diff --git a/tierb_study_20260826_144644/moonep/tierbprobe/manifest.txt b/tierb_study_20260826_144644/moonep/tierbprobe/manifest.txt index 2c7277251517ac14b50256b100c8e93ec5f4c29f..a07fd39d931d88cb40aa36a3fbb7c4bd2f0b67f1 100644 --- a/tierb_study_20260826_144644/moonep/tierbprobe/manifest.txt +++ b/tierb_study_20260826_144644/moonep/tierbprobe/manifest.txt @@ -1,9 +1,9 @@ -timestamp: 2026-08-26T15:05:39+00:00 +timestamp: 2026-08-26T15:53:39+00:00 host: sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 gpu: NVIDIA B200 x8 driver: 580.95.05 study_scripts_branch: granularity-moonep -study_scripts_commit: 0a7994a +study_scripts_commit: f28bc3b campaign: moonep backend: moonep phase: tierbprobe diff --git a/tierb_study_20260826_144644/moonep/tierbprobe/provenance.txt b/tierb_study_20260826_144644/moonep/tierbprobe/provenance.txt index fb21e40b7f8265ffe1296ff19e1e53fa989206fa..7b862cf986396b7f112e43f553994eabdf21f9e0 100644 --- a/tierb_study_20260826_144644/moonep/tierbprobe/provenance.txt +++ b/tierb_study_20260826_144644/moonep/tierbprobe/provenance.txt @@ -1,7 +1,7 @@ ============================================================== PROVENANCE: moonep__tierbprobe__gbs8 ============================================================== -captured_at : 2026-08-26T15:05:24+00:00 +captured_at : 2026-08-26T15:53:25+00:00 hostname : sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 kernel : 6.8.0-1043-nvidia cwd : /workspace/blackwell_run_scripts @@ -17,13 +17,13 @@ command : run_ladder.sh --phase tierbprobe --gbs 8 --out /workspace/tierb_ 6, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 7, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 persistence/clocks: - 0, 120 MHz, 1965 MHz, 1000.00 W, 28 - 1, 120 MHz, 1965 MHz, 1000.00 W, 33 - 2, 120 MHz, 1965 MHz, 1000.00 W, 33 - 3, 120 MHz, 1965 MHz, 1000.00 W, 29 - 4, 120 MHz, 1965 MHz, 1000.00 W, 29 - 5, 120 MHz, 1965 MHz, 1000.00 W, 33 - 6, 120 MHz, 1965 MHz, 1000.00 W, 33 + 0, 135 MHz, 1965 MHz, 1000.00 W, 29 + 1, 120 MHz, 1965 MHz, 1000.00 W, 34 + 2, 120 MHz, 1965 MHz, 1000.00 W, 34 + 3, 120 MHz, 1965 MHz, 1000.00 W, 30 + 4, 120 MHz, 1965 MHz, 1000.00 W, 30 + 5, 120 MHz, 1965 MHz, 1000.00 W, 34 + 6, 120 MHz, 1965 MHz, 1000.00 W, 34 7, 120 MHz, 1965 MHz, 1000.00 W, 29 NVLink topology: @@ -64,7 +64,7 @@ command : run_ladder.sh --phase tierbprobe --gbs 8 --out /workspace/tierb_ UltraEP main 94cab099b44f DeepEP HEAD 34152ae28f80 (+3 untracked: build output) MoonEP megatron-integration c27d35da15f2 - blackwell_run_scripts granularity-moonep 0a7994a5c549 + blackwell_run_scripts granularity-moonep f28bc3bed7df ---- environment --------------------------------------------- CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0 diff --git a/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1/mem_samples.csv b/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1/mem_samples.csv index 54a66e4e522bacda864f6fbd0476e5347526de75..25649c14ba7bd5ec5a982d1998934c44c8eaaad2 100644 --- a/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1/mem_samples.csv +++ b/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1/mem_samples.csv @@ -14,7 +14,7 @@ 5, 0 6, 0 7, 0 -0, 1 +0, 2 1, 1 2, 1 3, 1 @@ -37,23 +37,23 @@ 4, 4 5, 4 6, 4 -7, 4 +7, 2 0, 4 -1, 629 -2, 629 -3, 629 -4, 629 -5, 629 -6, 629 -7, 629 -0, 891 -1, 901 -2, 887 -3, 889 -4, 899 -5, 885 -6, 897 -7, 901 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 821 +1, 821 +2, 821 +3, 821 +4, 821 +5, 821 +6, 821 +7, 821 0, 1955 1, 1955 2, 1955 @@ -62,38 +62,38 @@ 5, 1955 6, 1955 7, 1955 -0, 3925 -1, 3937 -2, 3903 -3, 3925 -4, 3941 -5, 3909 -6, 3921 -7, 3895 -0, 22631 -1, 22631 -2, 22631 -3, 22631 -4, 22631 -5, 22631 -6, 22631 -7, 22631 -0, 35805 -1, 35959 -2, 35811 -3, 35851 -4, 35751 -5, 35885 -6, 35829 -7, 35893 -0, 45895 -1, 45969 -2, 45845 -3, 45907 -4, 45879 -5, 45919 -6, 45893 -7, 45985 +0, 3869 +1, 3845 +2, 3871 +3, 3863 +4, 3855 +5, 3877 +6, 3849 +7, 3865 +0, 25667 +1, 31739 +2, 31739 +3, 25667 +4, 31739 +5, 25667 +6, 31739 +7, 31739 +0, 35825 +1, 35935 +2, 35807 +3, 35841 +4, 35837 +5, 35921 +6, 35837 +7, 35957 +0, 45779 +1, 45891 +2, 45789 +3, 45843 +4, 45777 +5, 45817 +6, 45839 +7, 45889 0, 46289 1, 46409 2, 46283 @@ -102,38 +102,38 @@ 5, 46339 6, 46331 7, 46409 -0, 46291 -1, 46411 -2, 46285 -3, 46345 -4, 46297 -5, 46341 -6, 46333 -7, 46411 -0, 69799 -1, 69857 -2, 69793 -3, 69789 -4, 69869 -5, 69851 -6, 69905 -7, 71969 -0, 138955 -1, 138949 -2, 138923 -3, 138927 -4, 138937 -5, 138947 -6, 139737 -7, 138937 -0, 139647 -1, 139651 -2, 139669 -3, 139659 -4, 139641 -5, 139623 -6, 140457 -7, 139645 +0, 46293 +1, 46415 +2, 46287 +3, 46347 +4, 46299 +5, 46345 +6, 46335 +7, 46415 +0, 116061 +1, 116045 +2, 116043 +3, 116037 +4, 116053 +5, 116035 +6, 116027 +7, 116041 +0, 139371 +1, 139375 +2, 139373 +3, 139365 +4, 139387 +5, 139377 +6, 140155 +7, 139375 +0, 139757 +1, 139733 +2, 139737 +3, 139739 +4, 139739 +5, 139719 +6, 140503 +7, 139697 0, 139801 1, 139793 2, 139795 @@ -142,22 +142,22 @@ 5, 139787 6, 140565 7, 139793 -0, 142039 -1, 142059 -2, 142053 -3, 142049 -4, 142031 -5, 142003 -6, 142783 -7, 142047 -0, 143185 -1, 143209 -2, 143177 -3, 143173 -4, 143199 -5, 143113 -6, 143935 -7, 143135 +0, 142109 +1, 142087 +2, 142103 +3, 142099 +4, 142085 +5, 142093 +6, 142861 +7, 142085 +0, 143195 +1, 143171 +2, 143117 +3, 143091 +4, 143145 +5, 143149 +6, 143881 +7, 143149 0, 157149 1, 157133 2, 157143 @@ -190,35 +190,43 @@ 5, 157127 6, 156339 7, 157133 -0, 157149 -1, 157133 -2, 157143 -3, 157135 +0, 156823 +1, 156563 +2, 156497 +3, 156553 4, 157151 -5, 157127 -6, 156339 +5, 156875 +6, 155757 7, 157133 -0, 154621 -1, 155057 -2, 155261 -3, 155315 -4, 154623 -5, 154839 -6, 154845 -7, 155313 -0, 152791 -1, 154217 -2, 153639 -3, 153573 -4, 152875 -5, 152983 -6, 152777 -7, 153459 -0, 9111 -1, 9112 -2, 9112 -3, 9112 -4, 9112 -5, 9112 -6, 152103 -7, 9112 +0, 154941 +1, 154993 +2, 154615 +3, 154607 +4, 155011 +5, 154599 +6, 153875 +7, 155251 +0, 152617 +1, 153571 +2, 153255 +3, 152539 +4, 152943 +5, 152531 +6, 152131 +7, 152925 +0, 4 +1, 143033 +2, 4 +3, 4 +4, 143051 +5, 4 +6, 4 +7, 4 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 diff --git a/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1/train.log b/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1/train.log index 2bae09e08904e2d777e53a1121f593f45b3a8b5c..d798f03dc88c5b12d86f6f7e34d6ed3eddf6bcff 100644 --- a/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1/train.log +++ b/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1/train.log @@ -1,5 +1,5 @@ -[15:08:13] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) -[15:08:13] WARN: moonep_backend_token_routing_analysis lacks --moe-debug-drop-logging; probe will have no pre/post rank loads +[15:56:03] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +[15:56:03] WARN: moonep_backend_token_routing_analysis lacks --moe-debug-drop-logging; probe will have no pre/post rank loads ============================================================== DSV2_LITE_L24 / moonep (slots/rank=8 policy=matched) E=64 K=6 H'=1408 m=8 @@ -8,10 +8,10 @@ memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1/memory_snapshots/snapshot.pickle repo=/workspace/Megatron-MoE-Echo @ 35ae6e7f ============================================================== -W0826 15:08:14.956000 68051 torch/distributed/run.py:766] -W0826 15:08:14.956000 68051 torch/distributed/run.py:766] ***************************************** -W0826 15:08:14.956000 68051 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -W0826 15:08:14.956000 68051 torch/distributed/run.py:766] ***************************************** +W0826 15:56:04.896000 157880 torch/distributed/run.py:766] +W0826 15:56:04.896000 157880 torch/distributed/run.py:766] ***************************************** +W0826 15:56:04.896000 157880 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 15:56:04.896000 157880 torch/distributed/run.py:766] ***************************************** /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled @@ -20,6 +20,7 @@ W0826 15:08:14.956000 68051 torch/distributed/run.py:766] ********************** warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 @@ -333,9 +334,7 @@ using torch.bfloat16 for parameters ... moe_moonep_num_sms .............................. None moe_moonep_traffic_logging ...................... True moe_moonep_traffic_logging_interval ............. 1 - moe_moonep_traffic_warmup_iters ................. 2/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") - + moe_moonep_traffic_warmup_iters ................. 2 moe_num_echo_experts ............................ None moe_pad_expert_input_to_capacity ................ False moe_per_layer_logging ........................... False @@ -574,57 +573,58 @@ using torch.bfloat16 for parameters ... -------------------- end of arguments --------------------- INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 > building GPT2BPETokenizer tokenizer ... -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS > initializing torch distributed ... -WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it -[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank 1342[Gloo] Rank 6 is connected to is connected to is connected to is connected to 7 is connected to 7777 is connected to 7 peer ranks. peer ranks. peer ranks. peer ranks. 7 peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : peer ranks. Expected number of connected peer ranks is : 7777Expected number of connected peer ranks is : -7 - - - -7 -[Gloo] Rank [Gloo] Rank 72 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 - -[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank 5 is connected to 67 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : -7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7[Gloo] Rank +[Gloo] Rank 25 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 > initialized tensor model parallel with size 1 > initialized pipeline model parallel with size 1 > setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0[Gloo] Rank peer ranks. 0[Gloo] Rank Expected number of connected peer ranks is : is connected to 000 + is connected to peer ranks. 0Expected number of connected peer ranks is : peer ranks. 0Expected number of connected peer ranks is : +0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to [Gloo] Rank 0 peer ranks. Expected number of connected peer ranks is : 00 is connected to -0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[rank4]:[W826 15:08:33.461820708 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank6]:[W826 15:08:33.461862030 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank5]:[W826 15:08:33.461880241 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank7]:[W826 15:08:33.461924814 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank3]:[W826 15:08:33.461950469 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank2]:[W826 15:08:33.462154190 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank1]:[W826 15:08:33.462900270 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 15:56:22.707030890 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 15:56:22.707378444 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 15:56:22.707721504 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 15:56:22.707732391 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 15:56:22.707801771 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 15:56:22.707802162 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 15:56:22.707863714 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. > compiling dataset index builder ... make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' make: Nothing to be done for 'default'. make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' ->>> done with dataset index builder. Compilation time: 0.034 seconds +>>> done with dataset index builder. Compilation time: 0.033 seconds WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. > compiling and loading fused kernels ... -[rank0]:[W826 15:08:33.615133695 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. ->>> done with compiling and loading fused kernels. Compilation time: 3.570 seconds -time to initialize megatron (seconds): 7.028 -[after megatron is initialized] datetime: 2026-08-26 15:08:39 +[rank0]:[W826 15:56:22.869156314 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.787 seconds +time to initialize megatron (seconds): 6.978 +[after megatron is initialized] datetime: 2026-08-26 15:56:28 building GPT model ... /workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1542: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. warnings.warn( @@ -664,58 +664,58 @@ INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradi Params for bucket 1 (103022592 elements, 103022592 padded size): module.output_layer.weight Params for bucket 2 (48503296 elements, 48503296 padded size): - module.decoder.layers.23.pre_mlp_layernorm.weight - module.decoder.layers.23.mlp.router.weight - module.decoder.layers.23.self_attention.linear_proj.weight - module.decoder.layers.23.input_layernorm.weight - module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight module.decoder.final_layernorm.weight - module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight module.decoder.layers.23.self_attention.linear_kv_up_proj.weight module.decoder.layers.23.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.23.self_attention.linear_kv_down_proj.weight module.decoder.layers.23.self_attention.linear_q_proj.weight + module.decoder.layers.23.self_attention.linear_proj.weight + module.decoder.layers.23.input_layernorm.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.23.pre_mlp_layernorm.weight + module.decoder.layers.23.mlp.router.weight Params for bucket 3 (40901632 elements, 40901632 padded size): - module.decoder.layers.22.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.22.self_attention.linear_kv_down_proj.weight module.decoder.layers.22.self_attention.linear_kv_up_proj.weight - module.decoder.layers.22.self_attention.linear_q_proj.weight - module.decoder.layers.22.self_attention.linear_proj.weight - module.decoder.layers.22.input_layernorm.weight + module.decoder.layers.22.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.21.mlp.router.weight module.decoder.layers.21.pre_mlp_layernorm.weight + module.decoder.layers.21.self_attention.linear_q_proj.weight module.decoder.layers.22.mlp.router.weight module.decoder.layers.22.pre_mlp_layernorm.weight + module.decoder.layers.22.self_attention.linear_q_proj.weight + module.decoder.layers.22.self_attention.linear_proj.weight + module.decoder.layers.22.input_layernorm.weight module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight module.decoder.layers.21.self_attention.linear_kv_up_proj.weight module.decoder.layers.21.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.21.self_attention.linear_kv_down_proj.weight - module.decoder.layers.21.self_attention.linear_q_proj.weight Params for bucket 4 (41163264 elements, 41163264 padded size): - module.decoder.layers.20.mlp.router.weight - module.decoder.layers.20.pre_mlp_layernorm.weight - module.decoder.layers.20.self_attention.linear_proj.weight - module.decoder.layers.20.input_layernorm.weight - module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight module.decoder.layers.21.self_attention.linear_proj.weight - module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight module.decoder.layers.21.input_layernorm.weight module.decoder.layers.20.self_attention.linear_kv_up_proj.weight module.decoder.layers.20.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.20.self_attention.linear_kv_down_proj.weight module.decoder.layers.20.self_attention.linear_q_proj.weight + module.decoder.layers.20.mlp.router.weight + module.decoder.layers.20.pre_mlp_layernorm.weight + module.decoder.layers.20.self_attention.linear_proj.weight + module.decoder.layers.20.input_layernorm.weight + module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight Params for bucket 5 (42734080 elements, 42734080 padded size): + module.decoder.layers.19.pre_mlp_layernorm.weight + module.decoder.layers.19.mlp.router.weight module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.19.self_attention.linear_kv_up_proj.weight module.decoder.layers.19.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.19.self_attention.linear_kv_up_proj.weight module.decoder.layers.19.self_attention.linear_kv_down_proj.weight module.decoder.layers.19.self_attention.linear_q_proj.weight - module.decoder.layers.19.mlp.router.weight - module.decoder.layers.19.pre_mlp_layernorm.weight module.decoder.layers.19.self_attention.linear_proj.weight module.decoder.layers.19.input_layernorm.weight module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight @@ -723,12 +723,6 @@ Params for bucket 5 (42734080 elements, 42734080 padded size): Params for bucket 6 (40901632 elements, 40901632 padded size): module.decoder.layers.18.mlp.router.weight module.decoder.layers.18.pre_mlp_layernorm.weight - module.decoder.layers.17.mlp.router.weight - module.decoder.layers.17.pre_mlp_layernorm.weight - module.decoder.layers.18.self_attention.linear_kv_up_proj.weight - module.decoder.layers.18.self_attention.linear_kv_down_proj.weight - module.decoder.layers.18.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.18.self_attention.linear_q_proj.weight module.decoder.layers.18.self_attention.linear_proj.weight module.decoder.layers.18.input_layernorm.weight module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight @@ -737,38 +731,41 @@ Params for bucket 6 (40901632 elements, 40901632 padded size): module.decoder.layers.17.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.17.self_attention.linear_kv_down_proj.weight module.decoder.layers.17.self_attention.linear_q_proj.weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.weight + module.decoder.layers.18.self_attention.linear_kv_down_proj.weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.18.self_attention.linear_q_proj.weight + module.decoder.layers.17.mlp.router.weight + module.decoder.layers.17.pre_mlp_layernorm.weight Params for bucket 7 (41163264 elements, 41163264 padded size): module.decoder.layers.17.self_attention.linear_proj.weight - module.decoder.layers.17.input_layernorm.weight - module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.16.mlp.router.weight + module.decoder.layers.16.pre_mlp_layernorm.weight + module.decoder.layers.16.input_layernorm.weight + module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.17.input_layernorm.weight module.decoder.layers.16.self_attention.linear_kv_up_proj.weight module.decoder.layers.16.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.16.self_attention.linear_kv_down_proj.weight module.decoder.layers.16.self_attention.linear_q_proj.weight module.decoder.layers.16.self_attention.linear_proj.weight - module.decoder.layers.16.input_layernorm.weight - module.decoder.layers.16.mlp.router.weight - module.decoder.layers.16.pre_mlp_layernorm.weight - module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight Params for bucket 8 (42734080 elements, 42734080 padded size): module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight module.decoder.layers.15.self_attention.linear_kv_up_proj.weight module.decoder.layers.15.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.15.self_attention.linear_kv_down_proj.weight module.decoder.layers.15.self_attention.linear_q_proj.weight + module.decoder.layers.15.self_attention.linear_proj.weight module.decoder.layers.15.input_layernorm.weight module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.15.mlp.router.weight module.decoder.layers.15.pre_mlp_layernorm.weight - module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.15.mlp.router.weight Params for bucket 9 (40901632 elements, 40901632 padded size): - module.decoder.layers.14.self_attention.linear_kv_up_proj.weight - module.decoder.layers.14.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.14.self_attention.linear_kv_down_proj.weight + module.decoder.layers.14.pre_mlp_layernorm.weight module.decoder.layers.14.self_attention.linear_q_proj.weight - module.decoder.layers.14.self_attention.linear_proj.weight module.decoder.layers.14.input_layernorm.weight module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight @@ -776,97 +773,101 @@ Params for bucket 9 (40901632 elements, 40901632 padded size): module.decoder.layers.13.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.13.self_attention.linear_kv_down_proj.weight module.decoder.layers.14.mlp.router.weight - module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.self_attention.linear_kv_down_proj.weight + module.decoder.layers.14.self_attention.linear_kv_up_proj.weight + module.decoder.layers.14.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.14.self_attention.linear_proj.weight module.decoder.layers.13.mlp.router.weight module.decoder.layers.13.pre_mlp_layernorm.weight module.decoder.layers.13.self_attention.linear_q_proj.weight Params for bucket 10 (41163264 elements, 41163264 padded size): module.decoder.layers.13.self_attention.linear_proj.weight - module.decoder.layers.13.input_layernorm.weight - module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.12.self_attention.linear_kv_up_proj.weight - module.decoder.layers.12.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.12.self_attention.linear_kv_down_proj.weight - module.decoder.layers.12.self_attention.linear_proj.weight module.decoder.layers.12.mlp.router.weight module.decoder.layers.12.pre_mlp_layernorm.weight module.decoder.layers.12.self_attention.linear_q_proj.weight + module.decoder.layers.12.self_attention.linear_proj.weight module.decoder.layers.12.input_layernorm.weight module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.13.input_layernorm.weight + module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.12.self_attention.linear_kv_up_proj.weight + module.decoder.layers.12.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.12.self_attention.linear_kv_down_proj.weight Params for bucket 11 (42734080 elements, 42734080 padded size): - module.decoder.layers.11.pre_mlp_layernorm.weight - module.decoder.layers.11.mlp.router.weight - module.decoder.layers.11.self_attention.linear_proj.weight - module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.11.self_attention.linear_kv_up_proj.weight module.decoder.layers.11.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.weight module.decoder.layers.11.self_attention.linear_kv_down_proj.weight module.decoder.layers.11.self_attention.linear_q_proj.weight + module.decoder.layers.11.self_attention.linear_proj.weight module.decoder.layers.11.input_layernorm.weight module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.pre_mlp_layernorm.weight Params for bucket 12 (40901632 elements, 40901632 padded size): - module.decoder.layers.10.mlp.router.weight - module.decoder.layers.10.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.10.self_attention.linear_kv_up_proj.weight module.decoder.layers.10.self_attention.linear_kv_down_proj.weight + module.decoder.layers.10.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.10.self_attention.linear_q_proj.weight - module.decoder.layers.10.self_attention.linear_proj.weight module.decoder.layers.10.input_layernorm.weight module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.9.self_attention.linear_kv_up_proj.weight - module.decoder.layers.10.pre_mlp_layernorm.weight module.decoder.layers.9.mlp.router.weight module.decoder.layers.9.pre_mlp_layernorm.weight - module.decoder.layers.9.self_attention.linear_kv_down_proj.weight module.decoder.layers.9.self_attention.linear_q_proj.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.9.self_attention.linear_kv_up_proj.weight module.decoder.layers.9.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.9.self_attention.linear_kv_down_proj.weight Params for bucket 13 (41163264 elements, 41163264 padded size): - module.decoder.layers.8.mlp.router.weight module.decoder.layers.9.self_attention.linear_proj.weight module.decoder.layers.9.input_layernorm.weight - module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight module.decoder.layers.8.self_attention.linear_kv_up_proj.weight module.decoder.layers.8.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.8.self_attention.linear_kv_down_proj.weight module.decoder.layers.8.self_attention.linear_q_proj.weight module.decoder.layers.8.self_attention.linear_proj.weight - module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight module.decoder.layers.8.input_layernorm.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight Params for bucket 14 (42734080 elements, 42734080 padded size): module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.7.self_attention.linear_kv_up_proj.weight module.decoder.layers.7.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.7.self_attention.linear_kv_up_proj.weight module.decoder.layers.7.self_attention.linear_kv_down_proj.weight - module.decoder.layers.7.self_attention.linear_q_proj.weight - module.decoder.layers.7.self_attention.linear_proj.weight module.decoder.layers.7.input_layernorm.weight module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight module.decoder.layers.7.pre_mlp_layernorm.weight module.decoder.layers.7.mlp.router.weight - module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.7.self_attention.linear_q_proj.weight + module.decoder.layers.7.self_attention.linear_proj.weight Params for bucket 15 (40901632 elements, 40901632 padded size): - module.decoder.layers.6.mlp.router.weight - module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_kv_down_proj.weight + module.decoder.layers.6.self_attention.linear_kv_up_proj.weight + module.decoder.layers.6.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.6.self_attention.linear_q_proj.weight - module.decoder.layers.6.self_attention.linear_proj.weight module.decoder.layers.6.input_layernorm.weight module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_kv_down_proj.weight + module.decoder.layers.5.self_attention.linear_q_proj.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_proj.weight module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.5.pre_mlp_layernorm.weight module.decoder.layers.5.self_attention.linear_kv_up_proj.weight module.decoder.layers.5.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.5.self_attention.linear_kv_down_proj.weight - module.decoder.layers.6.self_attention.linear_kv_up_proj.weight - module.decoder.layers.6.self_attention.linear_kv_down_proj.weight - module.decoder.layers.6.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.5.mlp.router.weight - module.decoder.layers.5.pre_mlp_layernorm.weight - module.decoder.layers.5.self_attention.linear_q_proj.weight Params for bucket 16 (41163264 elements, 41163264 padded size): + module.decoder.layers.5.self_attention.linear_proj.weight module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight module.decoder.layers.4.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.4.self_attention.linear_kv_down_proj.weight @@ -874,100 +875,99 @@ Params for bucket 16 (41163264 elements, 41163264 padded size): module.decoder.layers.4.self_attention.linear_proj.weight module.decoder.layers.4.input_layernorm.weight module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.5.self_attention.linear_proj.weight module.decoder.layers.4.mlp.router.weight - module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight module.decoder.layers.5.input_layernorm.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight module.decoder.layers.4.pre_mlp_layernorm.weight module.decoder.layers.4.self_attention.linear_kv_up_proj.weight Params for bucket 17 (42734080 elements, 42734080 padded size): module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.3.self_attention.linear_kv_up_proj.weight - module.decoder.layers.3.pre_mlp_layernorm.weight module.decoder.layers.3.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.3.self_attention.linear_kv_up_proj.weight module.decoder.layers.3.self_attention.linear_kv_down_proj.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.pre_mlp_layernorm.weight module.decoder.layers.3.self_attention.linear_q_proj.weight module.decoder.layers.3.self_attention.linear_proj.weight module.decoder.layers.3.input_layernorm.weight module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight Params for bucket 18 (40901632 elements, 40901632 padded size): + module.decoder.layers.2.self_attention.linear_kv_down_proj.weight + module.decoder.layers.2.self_attention.linear_kv_up_proj.weight + module.decoder.layers.2.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.2.self_attention.linear_q_proj.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_kv_down_proj.weight + module.decoder.layers.1.self_attention.linear_q_proj.weight module.decoder.layers.2.mlp.router.weight module.decoder.layers.2.pre_mlp_layernorm.weight module.decoder.layers.2.self_attention.linear_proj.weight module.decoder.layers.2.input_layernorm.weight module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight module.decoder.layers.1.mlp.router.weight - module.decoder.layers.1.pre_mlp_layernorm.weight module.decoder.layers.1.self_attention.linear_kv_up_proj.weight module.decoder.layers.1.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.1.self_attention.linear_q_proj.weight - module.decoder.layers.2.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.2.self_attention.linear_kv_down_proj.weight - module.decoder.layers.2.self_attention.linear_kv_up_proj.weight - module.decoder.layers.2.self_attention.linear_q_proj.weight - module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.1.self_attention.linear_kv_down_proj.weight Params for bucket 19 (71436288 elements, 71436288 padded size): module.decoder.layers.1.self_attention.linear_proj.weight - module.decoder.layers.0.mlp.linear_fc1.weight - module.decoder.layers.0.mlp.linear_fc2.weight module.decoder.layers.1.input_layernorm.weight + module.decoder.layers.0.mlp.linear_fc2.weight + module.decoder.layers.0.mlp.linear_fc1.weight Params for bucket 20 (116789760 elements, 116789760 padded size): module.decoder.layers.0.mlp.linear_fc1.layer_norm_weight - module.decoder.layers.0.self_attention.linear_kv_up_proj.weight module.decoder.layers.0.self_attention.linear_q_proj.weight - module.decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight - module.embedding.word_embeddings.weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.weight module.decoder.layers.0.self_attention.linear_kv_down_proj.weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.0.self_attention.linear_proj.weight module.decoder.layers.0.input_layernorm.weight + module.embedding.word_embeddings.weight INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 40 Params for bucket 1 (40370176 elements, 40370176 padded size): module.decoder.layers.23.mlp.experts.linear_fc2.weight7 - module.decoder.layers.23.mlp.experts.linear_fc2.weight5 module.decoder.layers.23.mlp.experts.linear_fc2.weight3 - module.decoder.layers.23.mlp.experts.linear_fc2.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight5 module.decoder.layers.23.mlp.experts.linear_fc2.weight0 - module.decoder.layers.23.mlp.experts.linear_fc1.weight6 + module.decoder.layers.23.mlp.experts.linear_fc1.weight7 + module.decoder.layers.23.mlp.experts.linear_fc1.weight5 module.decoder.layers.23.mlp.experts.linear_fc2.weight6 module.decoder.layers.23.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc2.weight2 module.decoder.layers.23.mlp.experts.linear_fc2.weight1 - module.decoder.layers.23.mlp.experts.linear_fc1.weight7 - module.decoder.layers.23.mlp.experts.linear_fc1.weight5 + module.decoder.layers.23.mlp.experts.linear_fc1.weight6 Params for bucket 2 (40370176 elements, 40370176 padded size): + module.decoder.layers.23.mlp.experts.linear_fc1.weight1 + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 + module.decoder.layers.22.mlp.experts.linear_fc2.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight5 module.decoder.layers.23.mlp.experts.linear_fc1.weight4 module.decoder.layers.23.mlp.experts.linear_fc1.weight3 module.decoder.layers.23.mlp.experts.linear_fc1.weight0 module.decoder.layers.22.mlp.experts.linear_fc2.weight6 module.decoder.layers.22.mlp.experts.linear_fc2.weight4 - module.decoder.layers.23.mlp.experts.linear_fc1.weight2 - module.decoder.layers.23.mlp.experts.linear_fc1.weight1 - module.decoder.layers.22.mlp.experts.linear_fc2.weight7 - module.decoder.layers.22.mlp.experts.linear_fc2.weight5 Params for bucket 3 (40370176 elements, 40370176 padded size): + module.decoder.layers.22.mlp.experts.linear_fc2.weight3 module.decoder.layers.22.mlp.experts.linear_fc1.weight7 - module.decoder.layers.22.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc2.weight0 module.decoder.layers.22.mlp.experts.linear_fc1.weight5 - module.decoder.layers.22.mlp.experts.linear_fc2.weight3 + module.decoder.layers.22.mlp.experts.linear_fc2.weight1 module.decoder.layers.22.mlp.experts.linear_fc2.weight2 - module.decoder.layers.22.mlp.experts.linear_fc2.weight0 module.decoder.layers.22.mlp.experts.linear_fc1.weight6 module.decoder.layers.22.mlp.experts.linear_fc1.weight4 module.decoder.layers.22.mlp.experts.linear_fc1.weight3 Params for bucket 4 (40370176 elements, 40370176 padded size): module.decoder.layers.22.mlp.experts.linear_fc1.weight2 - module.decoder.layers.22.mlp.experts.linear_fc1.weight1 - module.decoder.layers.21.mlp.experts.linear_fc2.weight6 - module.decoder.layers.21.mlp.experts.linear_fc2.weight4 - module.decoder.layers.21.mlp.experts.linear_fc2.weight2 - module.decoder.layers.21.mlp.experts.linear_fc2.weight1 - module.decoder.layers.22.mlp.experts.linear_fc1.weight0 module.decoder.layers.21.mlp.experts.linear_fc2.weight7 + module.decoder.layers.22.mlp.experts.linear_fc1.weight1 module.decoder.layers.21.mlp.experts.linear_fc2.weight5 module.decoder.layers.21.mlp.experts.linear_fc2.weight3 + module.decoder.layers.21.mlp.experts.linear_fc2.weight2 module.decoder.layers.21.mlp.experts.linear_fc2.weight0 + module.decoder.layers.21.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight0 + module.decoder.layers.21.mlp.experts.linear_fc2.weight4 + module.decoder.layers.21.mlp.experts.linear_fc2.weight1 Params for bucket 5 (40370176 elements, 40370176 padded size): module.decoder.layers.21.mlp.experts.linear_fc1.weight4 module.decoder.layers.21.mlp.experts.linear_fc1.weight6 @@ -978,50 +978,50 @@ Params for bucket 5 (40370176 elements, 40370176 padded size): module.decoder.layers.21.mlp.experts.linear_fc1.weight1 Params for bucket 6 (40370176 elements, 40370176 padded size): module.decoder.layers.21.mlp.experts.linear_fc1.weight0 + module.decoder.layers.20.mlp.experts.linear_fc2.weight4 + module.decoder.layers.20.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc2.weight1 + module.decoder.layers.20.mlp.experts.linear_fc1.weight7 module.decoder.layers.20.mlp.experts.linear_fc2.weight7 module.decoder.layers.20.mlp.experts.linear_fc2.weight5 module.decoder.layers.20.mlp.experts.linear_fc2.weight3 module.decoder.layers.20.mlp.experts.linear_fc2.weight2 module.decoder.layers.20.mlp.experts.linear_fc2.weight0 module.decoder.layers.20.mlp.experts.linear_fc1.weight6 - module.decoder.layers.20.mlp.experts.linear_fc2.weight6 - module.decoder.layers.20.mlp.experts.linear_fc2.weight4 - module.decoder.layers.20.mlp.experts.linear_fc2.weight1 - module.decoder.layers.20.mlp.experts.linear_fc1.weight7 Params for bucket 7 (40370176 elements, 40370176 padded size): - module.decoder.layers.20.mlp.experts.linear_fc1.weight4 - module.decoder.layers.20.mlp.experts.linear_fc1.weight3 - module.decoder.layers.20.mlp.experts.linear_fc1.weight0 - module.decoder.layers.19.mlp.experts.linear_fc2.weight6 module.decoder.layers.20.mlp.experts.linear_fc1.weight5 module.decoder.layers.20.mlp.experts.linear_fc1.weight2 module.decoder.layers.20.mlp.experts.linear_fc1.weight1 module.decoder.layers.19.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc1.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight3 + module.decoder.layers.20.mlp.experts.linear_fc1.weight0 + module.decoder.layers.19.mlp.experts.linear_fc2.weight6 Params for bucket 8 (40370176 elements, 40370176 padded size): - module.decoder.layers.19.mlp.experts.linear_fc2.weight1 - module.decoder.layers.19.mlp.experts.linear_fc2.weight4 - module.decoder.layers.19.mlp.experts.linear_fc1.weight7 - module.decoder.layers.19.mlp.experts.linear_fc1.weight5 module.decoder.layers.19.mlp.experts.linear_fc2.weight5 module.decoder.layers.19.mlp.experts.linear_fc2.weight2 module.decoder.layers.19.mlp.experts.linear_fc2.weight3 module.decoder.layers.19.mlp.experts.linear_fc2.weight0 module.decoder.layers.19.mlp.experts.linear_fc1.weight6 module.decoder.layers.19.mlp.experts.linear_fc1.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight1 + module.decoder.layers.19.mlp.experts.linear_fc1.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 Params for bucket 9 (40370176 elements, 40370176 padded size): - module.decoder.layers.19.mlp.experts.linear_fc1.weight1 - module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 + module.decoder.layers.19.mlp.experts.linear_fc1.weight0 module.decoder.layers.18.mlp.experts.linear_fc2.weight5 module.decoder.layers.18.mlp.experts.linear_fc2.weight3 module.decoder.layers.18.mlp.experts.linear_fc2.weight2 - module.decoder.layers.19.mlp.experts.linear_fc1.weight3 module.decoder.layers.18.mlp.experts.linear_fc2.weight7 - module.decoder.layers.19.mlp.experts.linear_fc1.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 + module.decoder.layers.19.mlp.experts.linear_fc1.weight2 module.decoder.layers.18.mlp.experts.linear_fc2.weight6 module.decoder.layers.18.mlp.experts.linear_fc2.weight4 Params for bucket 10 (40370176 elements, 40370176 padded size): - module.decoder.layers.18.mlp.experts.linear_fc1.weight6 module.decoder.layers.18.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight6 module.decoder.layers.18.mlp.experts.linear_fc1.weight4 module.decoder.layers.18.mlp.experts.linear_fc1.weight3 module.decoder.layers.18.mlp.experts.linear_fc2.weight1 @@ -1030,32 +1030,32 @@ Params for bucket 10 (40370176 elements, 40370176 padded size): module.decoder.layers.18.mlp.experts.linear_fc1.weight2 Params for bucket 11 (40370176 elements, 40370176 padded size): module.decoder.layers.17.mlp.experts.linear_fc2.weight7 + module.decoder.layers.17.mlp.experts.linear_fc2.weight5 module.decoder.layers.18.mlp.experts.linear_fc1.weight0 - module.decoder.layers.17.mlp.experts.linear_fc2.weight4 - module.decoder.layers.17.mlp.experts.linear_fc2.weight3 - module.decoder.layers.17.mlp.experts.linear_fc2.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight2 + module.decoder.layers.17.mlp.experts.linear_fc1.weight7 module.decoder.layers.18.mlp.experts.linear_fc1.weight1 module.decoder.layers.17.mlp.experts.linear_fc2.weight6 - module.decoder.layers.17.mlp.experts.linear_fc2.weight5 - module.decoder.layers.17.mlp.experts.linear_fc2.weight2 + module.decoder.layers.17.mlp.experts.linear_fc2.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight3 module.decoder.layers.17.mlp.experts.linear_fc2.weight1 - module.decoder.layers.17.mlp.experts.linear_fc1.weight7 + module.decoder.layers.17.mlp.experts.linear_fc2.weight0 Params for bucket 12 (40370176 elements, 40370176 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight2 + module.decoder.layers.17.mlp.experts.linear_fc1.weight5 + module.decoder.layers.17.mlp.experts.linear_fc1.weight1 module.decoder.layers.17.mlp.experts.linear_fc1.weight6 module.decoder.layers.17.mlp.experts.linear_fc1.weight4 module.decoder.layers.17.mlp.experts.linear_fc1.weight3 module.decoder.layers.17.mlp.experts.linear_fc1.weight0 - module.decoder.layers.17.mlp.experts.linear_fc1.weight5 - module.decoder.layers.17.mlp.experts.linear_fc1.weight2 - module.decoder.layers.17.mlp.experts.linear_fc1.weight1 Params for bucket 13 (40370176 elements, 40370176 padded size): module.decoder.layers.16.mlp.experts.linear_fc2.weight7 module.decoder.layers.16.mlp.experts.linear_fc2.weight3 module.decoder.layers.16.mlp.experts.linear_fc2.weight5 module.decoder.layers.16.mlp.experts.linear_fc1.weight7 module.decoder.layers.16.mlp.experts.linear_fc1.weight5 - module.decoder.layers.16.mlp.experts.linear_fc2.weight4 module.decoder.layers.16.mlp.experts.linear_fc2.weight6 + module.decoder.layers.16.mlp.experts.linear_fc2.weight4 module.decoder.layers.16.mlp.experts.linear_fc2.weight2 module.decoder.layers.16.mlp.experts.linear_fc2.weight1 module.decoder.layers.16.mlp.experts.linear_fc2.weight0 @@ -1063,20 +1063,20 @@ Params for bucket 13 (40370176 elements, 40370176 padded size): Params for bucket 14 (40370176 elements, 40370176 padded size): module.decoder.layers.16.mlp.experts.linear_fc1.weight1 module.decoder.layers.16.mlp.experts.linear_fc1.weight2 - module.decoder.layers.15.mlp.experts.linear_fc2.weight6 - module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 module.decoder.layers.16.mlp.experts.linear_fc1.weight4 module.decoder.layers.16.mlp.experts.linear_fc1.weight0 module.decoder.layers.16.mlp.experts.linear_fc1.weight3 - module.decoder.layers.15.mlp.experts.linear_fc2.weight7 - module.decoder.layers.15.mlp.experts.linear_fc2.weight5 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 Params for bucket 15 (40370176 elements, 40370176 padded size): - module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 module.decoder.layers.15.mlp.experts.linear_fc1.weight7 - module.decoder.layers.15.mlp.experts.linear_fc2.weight2 module.decoder.layers.15.mlp.experts.linear_fc1.weight5 - module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 module.decoder.layers.15.mlp.experts.linear_fc1.weight6 module.decoder.layers.15.mlp.experts.linear_fc1.weight4 module.decoder.layers.15.mlp.experts.linear_fc1.weight3 @@ -1101,8 +1101,8 @@ Params for bucket 17 (40370176 elements, 40370176 padded size): module.decoder.layers.14.mlp.experts.linear_fc1.weight3 module.decoder.layers.14.mlp.experts.linear_fc1.weight4 Params for bucket 18 (40370176 elements, 40370176 padded size): - module.decoder.layers.13.mlp.experts.linear_fc2.weight6 module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 module.decoder.layers.13.mlp.experts.linear_fc2.weight2 module.decoder.layers.13.mlp.experts.linear_fc2.weight1 module.decoder.layers.13.mlp.experts.linear_fc1.weight7 @@ -1114,162 +1114,162 @@ Params for bucket 18 (40370176 elements, 40370176 padded size): module.decoder.layers.13.mlp.experts.linear_fc1.weight6 Params for bucket 19 (40370176 elements, 40370176 padded size): module.decoder.layers.13.mlp.experts.linear_fc1.weight5 - module.decoder.layers.13.mlp.experts.linear_fc1.weight1 module.decoder.layers.13.mlp.experts.linear_fc1.weight2 - module.decoder.layers.12.mlp.experts.linear_fc2.weight6 - module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 module.decoder.layers.13.mlp.experts.linear_fc1.weight0 - module.decoder.layers.12.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 Params for bucket 20 (40370176 elements, 40370176 padded size): - module.decoder.layers.12.mlp.experts.linear_fc2.weight2 - module.decoder.layers.12.mlp.experts.linear_fc2.weight1 - module.decoder.layers.12.mlp.experts.linear_fc2.weight4 - module.decoder.layers.12.mlp.experts.linear_fc1.weight7 - module.decoder.layers.12.mlp.experts.linear_fc1.weight5 module.decoder.layers.12.mlp.experts.linear_fc2.weight5 module.decoder.layers.12.mlp.experts.linear_fc2.weight3 module.decoder.layers.12.mlp.experts.linear_fc2.weight0 module.decoder.layers.12.mlp.experts.linear_fc1.weight6 module.decoder.layers.12.mlp.experts.linear_fc1.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 Params for bucket 21 (40370176 elements, 40370176 padded size): - module.decoder.layers.12.mlp.experts.linear_fc1.weight1 - module.decoder.layers.12.mlp.experts.linear_fc1.weight2 - module.decoder.layers.11.mlp.experts.linear_fc2.weight7 - module.decoder.layers.11.mlp.experts.linear_fc2.weight5 - module.decoder.layers.11.mlp.experts.linear_fc2.weight3 module.decoder.layers.12.mlp.experts.linear_fc1.weight3 module.decoder.layers.12.mlp.experts.linear_fc1.weight0 module.decoder.layers.11.mlp.experts.linear_fc2.weight6 module.decoder.layers.11.mlp.experts.linear_fc2.weight4 module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 Params for bucket 22 (40370176 elements, 40370176 padded size): - module.decoder.layers.11.mlp.experts.linear_fc2.weight0 - module.decoder.layers.11.mlp.experts.linear_fc1.weight6 - module.decoder.layers.11.mlp.experts.linear_fc1.weight4 - module.decoder.layers.11.mlp.experts.linear_fc1.weight3 module.decoder.layers.11.mlp.experts.linear_fc2.weight1 module.decoder.layers.11.mlp.experts.linear_fc1.weight7 module.decoder.layers.11.mlp.experts.linear_fc1.weight5 module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 Params for bucket 23 (40370176 elements, 40370176 padded size): - module.decoder.layers.10.mlp.experts.linear_fc2.weight5 - module.decoder.layers.10.mlp.experts.linear_fc2.weight7 - module.decoder.layers.11.mlp.experts.linear_fc1.weight0 - module.decoder.layers.10.mlp.experts.linear_fc2.weight3 - module.decoder.layers.10.mlp.experts.linear_fc2.weight0 module.decoder.layers.11.mlp.experts.linear_fc1.weight1 module.decoder.layers.10.mlp.experts.linear_fc2.weight6 module.decoder.layers.10.mlp.experts.linear_fc2.weight4 module.decoder.layers.10.mlp.experts.linear_fc2.weight2 module.decoder.layers.10.mlp.experts.linear_fc2.weight1 module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 Params for bucket 24 (40370176 elements, 40370176 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 module.decoder.layers.10.mlp.experts.linear_fc1.weight6 module.decoder.layers.10.mlp.experts.linear_fc1.weight4 module.decoder.layers.10.mlp.experts.linear_fc1.weight3 module.decoder.layers.10.mlp.experts.linear_fc1.weight0 - module.decoder.layers.10.mlp.experts.linear_fc1.weight2 - module.decoder.layers.10.mlp.experts.linear_fc1.weight5 - module.decoder.layers.10.mlp.experts.linear_fc1.weight1 Params for bucket 25 (40370176 elements, 40370176 padded size): module.decoder.layers.9.mlp.experts.linear_fc2.weight6 module.decoder.layers.9.mlp.experts.linear_fc2.weight4 module.decoder.layers.9.mlp.experts.linear_fc2.weight2 module.decoder.layers.9.mlp.experts.linear_fc2.weight1 - module.decoder.layers.9.mlp.experts.linear_fc1.weight7 - module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 module.decoder.layers.9.mlp.experts.linear_fc2.weight7 module.decoder.layers.9.mlp.experts.linear_fc2.weight3 module.decoder.layers.9.mlp.experts.linear_fc2.weight5 - module.decoder.layers.9.mlp.experts.linear_fc2.weight0 - module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 Params for bucket 26 (40370176 elements, 40370176 padded size): - module.decoder.layers.9.mlp.experts.linear_fc1.weight2 - module.decoder.layers.9.mlp.experts.linear_fc1.weight1 - module.decoder.layers.8.mlp.experts.linear_fc2.weight7 - module.decoder.layers.8.mlp.experts.linear_fc2.weight5 module.decoder.layers.9.mlp.experts.linear_fc1.weight4 module.decoder.layers.9.mlp.experts.linear_fc1.weight3 module.decoder.layers.9.mlp.experts.linear_fc1.weight0 module.decoder.layers.8.mlp.experts.linear_fc2.weight6 module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 Params for bucket 27 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 module.decoder.layers.8.mlp.experts.linear_fc2.weight3 module.decoder.layers.8.mlp.experts.linear_fc2.weight0 module.decoder.layers.8.mlp.experts.linear_fc1.weight6 module.decoder.layers.8.mlp.experts.linear_fc1.weight4 module.decoder.layers.8.mlp.experts.linear_fc1.weight3 - module.decoder.layers.8.mlp.experts.linear_fc1.weight7 - module.decoder.layers.8.mlp.experts.linear_fc2.weight2 - module.decoder.layers.8.mlp.experts.linear_fc2.weight1 - module.decoder.layers.8.mlp.experts.linear_fc1.weight5 Params for bucket 28 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 module.decoder.layers.8.mlp.experts.linear_fc1.weight0 module.decoder.layers.7.mlp.experts.linear_fc2.weight7 module.decoder.layers.7.mlp.experts.linear_fc2.weight5 module.decoder.layers.7.mlp.experts.linear_fc2.weight3 - module.decoder.layers.7.mlp.experts.linear_fc2.weight0 - module.decoder.layers.8.mlp.experts.linear_fc1.weight2 - module.decoder.layers.7.mlp.experts.linear_fc2.weight6 - module.decoder.layers.8.mlp.experts.linear_fc1.weight1 - module.decoder.layers.7.mlp.experts.linear_fc2.weight4 module.decoder.layers.7.mlp.experts.linear_fc2.weight2 - module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 Params for bucket 29 (40370176 elements, 40370176 padded size): - module.decoder.layers.7.mlp.experts.linear_fc1.weight6 - module.decoder.layers.7.mlp.experts.linear_fc1.weight3 - module.decoder.layers.7.mlp.experts.linear_fc1.weight4 module.decoder.layers.7.mlp.experts.linear_fc1.weight7 module.decoder.layers.7.mlp.experts.linear_fc1.weight5 module.decoder.layers.7.mlp.experts.linear_fc1.weight2 module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 Params for bucket 30 (40370176 elements, 40370176 padded size): - module.decoder.layers.7.mlp.experts.linear_fc1.weight0 - module.decoder.layers.6.mlp.experts.linear_fc2.weight4 module.decoder.layers.6.mlp.experts.linear_fc2.weight6 - module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 module.decoder.layers.6.mlp.experts.linear_fc2.weight1 - module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 module.decoder.layers.6.mlp.experts.linear_fc2.weight7 module.decoder.layers.6.mlp.experts.linear_fc2.weight5 module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 module.decoder.layers.6.mlp.experts.linear_fc2.weight0 - module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 Params for bucket 31 (40370176 elements, 40370176 padded size): - module.decoder.layers.6.mlp.experts.linear_fc1.weight4 - module.decoder.layers.6.mlp.experts.linear_fc1.weight3 - module.decoder.layers.6.mlp.experts.linear_fc1.weight0 - module.decoder.layers.5.mlp.experts.linear_fc2.weight6 module.decoder.layers.6.mlp.experts.linear_fc1.weight5 module.decoder.layers.6.mlp.experts.linear_fc1.weight2 module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 module.decoder.layers.5.mlp.experts.linear_fc2.weight7 Params for bucket 32 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 module.decoder.layers.5.mlp.experts.linear_fc2.weight4 module.decoder.layers.5.mlp.experts.linear_fc2.weight1 - module.decoder.layers.5.mlp.experts.linear_fc1.weight7 - module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 module.decoder.layers.5.mlp.experts.linear_fc2.weight5 - module.decoder.layers.5.mlp.experts.linear_fc2.weight2 module.decoder.layers.5.mlp.experts.linear_fc2.weight3 module.decoder.layers.5.mlp.experts.linear_fc2.weight0 - module.decoder.layers.5.mlp.experts.linear_fc1.weight6 - module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 Params for bucket 33 (40370176 elements, 40370176 padded size): - module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 module.decoder.layers.4.mlp.experts.linear_fc2.weight7 - module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 module.decoder.layers.4.mlp.experts.linear_fc2.weight5 module.decoder.layers.4.mlp.experts.linear_fc2.weight3 - module.decoder.layers.5.mlp.experts.linear_fc1.weight3 - module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 module.decoder.layers.4.mlp.experts.linear_fc2.weight6 module.decoder.layers.4.mlp.experts.linear_fc2.weight4 module.decoder.layers.4.mlp.experts.linear_fc2.weight2 Params for bucket 34 (40370176 elements, 40370176 padded size): - module.decoder.layers.4.mlp.experts.linear_fc1.weight7 module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 module.decoder.layers.4.mlp.experts.linear_fc1.weight5 module.decoder.layers.4.mlp.experts.linear_fc1.weight2 module.decoder.layers.4.mlp.experts.linear_fc2.weight1 @@ -1282,15 +1282,15 @@ Params for bucket 35 (40370176 elements, 40370176 padded size): module.decoder.layers.3.mlp.experts.linear_fc2.weight4 module.decoder.layers.3.mlp.experts.linear_fc2.weight1 module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 module.decoder.layers.3.mlp.experts.linear_fc2.weight5 module.decoder.layers.4.mlp.experts.linear_fc1.weight0 - module.decoder.layers.3.mlp.experts.linear_fc2.weight7 module.decoder.layers.3.mlp.experts.linear_fc2.weight3 module.decoder.layers.3.mlp.experts.linear_fc2.weight2 module.decoder.layers.3.mlp.experts.linear_fc2.weight0 Params for bucket 36 (40370176 elements, 40370176 padded size): - module.decoder.layers.3.mlp.experts.linear_fc1.weight2 module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 module.decoder.layers.3.mlp.experts.linear_fc1.weight1 module.decoder.layers.3.mlp.experts.linear_fc1.weight6 module.decoder.layers.3.mlp.experts.linear_fc1.weight4 @@ -1300,22 +1300,22 @@ Params for bucket 37 (40370176 elements, 40370176 padded size): module.decoder.layers.2.mlp.experts.linear_fc2.weight7 module.decoder.layers.2.mlp.experts.linear_fc2.weight3 module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 module.decoder.layers.2.mlp.experts.linear_fc2.weight0 - module.decoder.layers.2.mlp.experts.linear_fc1.weight7 - module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 module.decoder.layers.2.mlp.experts.linear_fc2.weight4 module.decoder.layers.2.mlp.experts.linear_fc2.weight6 - module.decoder.layers.2.mlp.experts.linear_fc2.weight2 module.decoder.layers.2.mlp.experts.linear_fc2.weight1 - module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 Params for bucket 38 (40370176 elements, 40370176 padded size): - module.decoder.layers.2.mlp.experts.linear_fc1.weight1 - module.decoder.layers.2.mlp.experts.linear_fc1.weight2 - module.decoder.layers.1.mlp.experts.linear_fc2.weight7 - module.decoder.layers.1.mlp.experts.linear_fc2.weight5 module.decoder.layers.2.mlp.experts.linear_fc1.weight4 module.decoder.layers.2.mlp.experts.linear_fc1.weight3 module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 module.decoder.layers.1.mlp.experts.linear_fc2.weight6 module.decoder.layers.1.mlp.experts.linear_fc2.weight4 Params for bucket 39 (40370176 elements, 40370176 padded size): @@ -1332,9 +1332,9 @@ Params for bucket 40 (17301504 elements, 17301504 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight2 module.decoder.layers.1.mlp.experts.linear_fc1.weight1 module.decoder.layers.1.mlp.experts.linear_fc1.weight0 -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine -[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 15:08:43 +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 15:56:32 > building train, validation, and test datasets ... > datasets target sizes (minimum size): train: 96 @@ -1342,7 +1342,7 @@ INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine test: 0 INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] > building train, validation, and test datasets for GPT ... -INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers @@ -1355,14 +1355,14 @@ INFO:megatron.core.datasets.gpt_dataset: Load the sample index from eadc3756447e INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-shuffle_index.npy INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 > finished creating GPT datasets ... -[after dataloaders are built] datetime: 2026-08-26 15:08:43 +[after dataloaders are built] datetime: 2026-08-26 15:56:32 done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3545.60, 3559.13) + train/valid/test-data-iterators-setup ..........: (62.05, 68.15) training ... Overwriting rerun_state_machine.current_iteration from -1 to 0... -(min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (3699.16, 3711.30) - train/valid/test-data-iterators-setup ..........: (62.28, 69.71) -[before the start of training step] datetime: 2026-08-26 15:08:43 +[before the start of training step] datetime: 2026-08-26 15:56:32 INFO:moonep.api:MoonEP comm buffer sizes: config : R=8, S=8192, H=2048, E=64, K=6, NvS=51184, NvS_padded=51200 sm_count : num_sms=32, num_sms_dedup=148 @@ -1371,5640 +1371,5636 @@ INFO:moonep.api:MoonEP comm buffer sizes: local_temp_buffer_size : 2.01 MiB total_size : 204.01 MiB WARNING:megatron.core.rerun_state_machine:Result validation enabled -Number of parameters in transformer block in billions: 13.53 + [2026-08-26 15:56:55] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 22661.3 | throughput per GPU (TFLOP/s/GPU): 5.6 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.092206E+01 | loss scale: 1.0 | grad norm: 30.650 | number of skipped iterations: 0 | number of nan iterations: 0 |Number of parameters in transformer block in billions: 13.53 + Number of parameters in embedding layers in billions: 0.21 Total number of parameters in billions: 13.74 Number of parameters in most loaded shard in billions: 13.7356 - [2026-08-26 15:09:07] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 23827.1 | throughput per GPU (TFLOP/s/GPU): 5.4 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.092206E+01 | loss scale: 1.0 | grad norm: 30.650 | number of skipped iterations: 0 | number of nan iterations: 0 |Theoretical memory footprints: weight and optimizer=98244.96 MB - -[Rank 0] (after 1 iterations) memory (MB) | allocated: 66080.0146484375 | max allocated: 124949.189453125 | reserved: 127502.0 | max reserved: 127502.0 - [2026-08-26 15:09:07] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 818.9 | throughput per GPU (TFLOP/s/GPU): 155.8 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.091986E+01 | loss scale: 1.0 | grad norm: 30.565 | number of skipped iterations: 0 | number of nan iterations: 0 | +Theoretical memory footprints: weight and optimizer=98244.96 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 66080.0703125 | max allocated: 124949.189453125 | reserved: 127502.0 | max reserved: 127502.0 + [2026-08-26 15:56:55] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 797.3 | throughput per GPU (TFLOP/s/GPU): 160.0 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.091986E+01 | loss scale: 1.0 | grad norm: 30.565 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74164 payload_rows_remote=63572 dup_rows=24140 dup_rows_remote=19704 dedup_ratio=0.245565 router_tokens_per_expert=[1368, 800, 687, 608, 583, 912, 1307, 180, 827, 878, 1254, 626, 159, 1029, 512, 1030, 1115, 891, 708, 849, 582, 479, 1066, 534, 704, 830, 315, 415, 878, 959, 312, 962, 999, 506, 801, 868, 466, 385, 988, 943, 486, 499, 374, 520, 1580, 1567, 806, 370, 463, 645, 745, 830, 1104, 1602, 677, 481, 459, 1032, 1468, 789, 531, 421, 1071, 317] payload_rows_per_dst=[10592, 9606, 9306, 10196, 8760, 9706, 6736, 9262] dup_rows_per_dst=[4436, 3376, 2790, 3758, 3152, 2698, 1016, 2914] combine_rows_per_src=[10592, 9606, 9306, 10196, 8760, 9706, 6736, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74152 payload_rows_remote=65746 dup_rows=24152 dup_rows_remote=21160 dedup_ratio=0.245687 router_tokens_per_expert=[1471, 694, 544, 633, 638, 995, 1377, 184, 768, 1244, 1175, 448, 142, 1022, 472, 993, 1069, 871, 752, 873, 577, 481, 957, 617, 796, 696, 336, 403, 834, 1006, 385, 937, 1113, 425, 761, 834, 441, 374, 965, 786, 451, 519, 369, 479, 1386, 1486, 765, 370, 611, 773, 735, 786, 1012, 1751, 798, 509, 547, 971, 1510, 839, 504, 391, 1177, 324] payload_rows_per_dst=[9744, 9302, 9600, 8196, 8406, 9170, 10158, 9576] dup_rows_per_dst=[3328, 3226, 2794, 2590, 2992, 2480, 3792, 2950] combine_rows_per_src=[9744, 9302, 9600, 8196, 8406, 9170, 10158, 9576] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74634 payload_rows_remote=64782 dup_rows=23670 dup_rows_remote=19798 dedup_ratio=0.240784 router_tokens_per_expert=[1384, 896, 557, 585, 534, 959, 1303, 231, 941, 1164, 1301, 605, 153, 1233, 420, 1045, 1012, 867, 772, 829, 573, 411, 1007, 644, 849, 830, 339, 480, 878, 751, 431, 901, 890, 511, 820, 837, 504, 378, 857, 788, 459, 544, 318, 513, 1416, 1590, 771, 319, 522, 737, 881, 822, 1130, 1686, 727, 555, 523, 807, 1378, 767, 476, 434, 1049, 258] payload_rows_per_dst=[9594, 9852, 9414, 10074, 8426, 9420, 8962, 8892] dup_rows_per_dst=[3304, 3872, 2816, 4216, 2744, 2440, 1786, 2492] combine_rows_per_src=[9594, 9852, 9414, 10074, 8426, 9420, 8962, 8892] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74366 payload_rows_remote=64898 dup_rows=23938 dup_rows_remote=20786 dedup_ratio=0.243510 router_tokens_per_expert=[1467, 723, 585, 512, 602, 980, 1268, 241, 683, 1252, 1204, 362, 177, 936, 442, 1032, 1304, 830, 742, 849, 594, 473, 1008, 535, 895, 766, 325, 452, 831, 1020, 430, 905, 771, 434, 803, 797, 431, 400, 922, 913, 403, 517, 406, 838, 1400, 1547, 841, 358, 620, 628, 787, 759, 1071, 2002, 695, 508, 489, 835, 1443, 716, 557, 420, 1165, 251] payload_rows_per_dst=[9414, 9156, 9718, 8528, 8712, 9468, 10176, 9194] dup_rows_per_dst=[2844, 3020, 2952, 2720, 2728, 3152, 3964, 2558] combine_rows_per_src=[9414, 9156, 9718, 8528, 8712, 9468, 10176, 9194] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75044 payload_rows_remote=64192 dup_rows=23260 dup_rows_remote=19654 dedup_ratio=0.236613 router_tokens_per_expert=[1386, 677, 582, 589, 473, 947, 1297, 233, 925, 1183, 1308, 460, 118, 1110, 420, 1020, 1073, 799, 826, 837, 581, 495, 859, 573, 811, 878, 343, 480, 791, 1055, 454, 886, 954, 443, 763, 882, 420, 364, 850, 975, 501, 479, 376, 512, 1515, 1543, 776, 326, 610, 801, 755, 822, 1153, 1633, 808, 501, 606, 857, 1360, 746, 499, 434, 1153, 266] payload_rows_per_dst=[7318, 8438, 9352, 8664, 10308, 9376, 10736, 10852] dup_rows_per_dst=[2278, 1260, 2734, 2732, 3766, 2680, 4204, 3606] combine_rows_per_src=[7318, 8438, 9352, 8664, 10308, 9376, 10736, 10852] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73764 payload_rows_remote=65216 dup_rows=24540 dup_rows_remote=21874 dedup_ratio=0.249634 router_tokens_per_expert=[1280, 745, 587, 641, 619, 930, 1346, 162, 823, 781, 1301, 442, 124, 1034, 442, 950, 954, 841, 696, 803, 617, 492, 1080, 552, 810, 811, 323, 365, 866, 1025, 475, 932, 1232, 438, 1015, 785, 488, 393, 877, 941, 500, 553, 361, 780, 1386, 1520, 810, 336, 474, 770, 696, 850, 1182, 1727, 824, 512, 559, 929, 1297, 761, 569, 383, 1049, 306] payload_rows_per_dst=[9388, 8920, 9266, 8548, 8946, 10022, 9684, 8990] dup_rows_per_dst=[3232, 2874, 2804, 2666, 3392, 3614, 3242, 2716] combine_rows_per_src=[9388, 8920, 9266, 8548, 8946, 10022, 9684, 8990] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74366 payload_rows_remote=64898 dup_rows=23938 dup_rows_remote=20786 dedup_ratio=0.243510 router_tokens_per_expert=[1467, 723, 585, 512, 602, 980, 1268, 241, 683, 1252, 1204, 362, 177, 936, 442, 1032, 1304, 830, 742, 849, 594, 473, 1008, 535, 895, 766, 325, 452, 831, 1020, 430, 905, 771, 434, 803, 797, 431, 400, 922, 913, 403, 517, 406, 838, 1400, 1547, 841, 358, 620, 628, 787, 759, 1071, 2002, 695, 508, 489, 835, 1443, 716, 557, 420, 1165, 251] payload_rows_per_dst=[9414, 9156, 9718, 8528, 8712, 9468, 10176, 9194] dup_rows_per_dst=[2844, 3020, 2952, 2720, 2728, 3152, 3964, 2558] combine_rows_per_src=[9414, 9156, 9718, 8528, 8712, 9468, 10176, 9194] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=260390912 dispatch_meta_bytes=333104 combine_payload_bytes=260390912 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=260724016 ingress_bytes=329596928 link_bytes=329596928 token_bytes=521114928 weight_movement_bytes=69206016 total_nvlink_bytes=590320944 weight_to_token_ratio=0.132804 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.240246705e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=64254 dup_rows=23924 dup_rows_remote=20304 dedup_ratio=0.243368 router_tokens_per_expert=[1466, 712, 583, 604, 507, 937, 1337, 190, 898, 1179, 1285, 467, 169, 927, 462, 993, 994, 874, 762, 778, 645, 461, 1037, 581, 799, 838, 286, 492, 863, 942, 360, 926, 1051, 438, 894, 841, 541, 392, 939, 924, 443, 464, 329, 498, 1523, 1546, 833, 382, 564, 748, 770, 777, 1181, 1525, 813, 495, 477, 953, 1446, 751, 540, 407, 1050, 263] payload_rows_per_dst=[7460, 9554, 9494, 8436, 10484, 9470, 10126, 9356] dup_rows_per_dst=[2280, 2992, 2770, 2576, 4488, 2566, 3620, 2632] combine_rows_per_src=[7460, 9554, 9494, 8436, 10484, 9470, 10126, 9356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74872 payload_rows_remote=65326 dup_rows=23432 dup_rows_remote=20484 dedup_ratio=0.238363 router_tokens_per_expert=[1425, 764, 612, 536, 673, 847, 1422, 267, 911, 1081, 1310, 401, 145, 1132, 496, 952, 916, 809, 807, 789, 638, 488, 1125, 675, 731, 870, 428, 487, 881, 909, 406, 907, 798, 487, 777, 720, 540, 377, 877, 924, 513, 510, 371, 683, 1437, 1456, 624, 352, 641, 695, 741, 878, 967, 1658, 693, 436, 561, 913, 1442, 765, 648, 468, 1065, 295] payload_rows_per_dst=[9856, 9666, 9546, 10224, 8488, 9194, 8580, 9318] dup_rows_per_dst=[3236, 3190, 2948, 4180, 2512, 2848, 1522, 2996] combine_rows_per_src=[9856, 9666, 9546, 10224, 8488, 9194, 8580, 9318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75044 payload_rows_remote=64192 dup_rows=23260 dup_rows_remote=19654 dedup_ratio=0.236613 router_tokens_per_expert=[1386, 677, 582, 589, 473, 947, 1297, 233, 925, 1183, 1308, 460, 118, 1110, 420, 1020, 1073, 799, 826, 837, 581, 495, 859, 573, 811, 878, 343, 480, 791, 1055, 454, 886, 954, 443, 763, 882, 420, 364, 850, 975, 501, 479, 376, 512, 1515, 1543, 776, 326, 610, 801, 755, 822, 1153, 1633, 808, 501, 606, 857, 1360, 746, 499, 434, 1153, 266] payload_rows_per_dst=[7318, 8438, 9352, 8664, 10308, 9376, 10736, 10852] dup_rows_per_dst=[2278, 1260, 2734, 2732, 3766, 2680, 4204, 3606] combine_rows_per_src=[7318, 8438, 9352, 8664, 10308, 9376, 10736, 10852] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=269295616 dispatch_meta_bytes=347624 combine_payload_bytes=269295616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269643240 ingress_bytes=303898624 link_bytes=303898624 token_bytes=538938856 weight_movement_bytes=34603008 total_nvlink_bytes=573541864 weight_to_token_ratio=0.064206 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=7808 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.143546056e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265347072 dispatch_meta_bytes=338320 combine_payload_bytes=265347072 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265685392 ingress_bytes=369156096 link_bytes=369156096 token_bytes=531032464 weight_movement_bytes=103809024 total_nvlink_bytes=634841488 weight_to_token_ratio=0.195485 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=8832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389104670e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267124736 dispatch_meta_bytes=348360 combine_payload_bytes=267124736 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267473096 ingress_bytes=301727744 link_bytes=301727744 token_bytes=534597832 weight_movement_bytes=34603008 total_nvlink_bytes=569200840 weight_to_token_ratio=0.064727 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=7680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135377210e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265822208 dispatch_meta_bytes=342736 combine_payload_bytes=265822208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266164944 ingress_bytes=300425216 link_bytes=300425216 token_bytes=531987152 weight_movement_bytes=34603008 total_nvlink_bytes=566590160 weight_to_token_ratio=0.065045 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=12288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130475902e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74152 payload_rows_remote=65746 dup_rows=24152 dup_rows_remote=21160 dedup_ratio=0.245687 router_tokens_per_expert=[1471, 694, 544, 633, 638, 995, 1377, 184, 768, 1244, 1175, 448, 142, 1022, 472, 993, 1069, 871, 752, 873, 577, 481, 957, 617, 796, 696, 336, 403, 834, 1006, 385, 937, 1113, 425, 761, 834, 441, 374, 965, 786, 451, 519, 369, 479, 1386, 1486, 765, 370, 611, 773, 735, 786, 1012, 1751, 798, 509, 547, 971, 1510, 839, 504, 391, 1177, 324] payload_rows_per_dst=[9744, 9302, 9600, 8196, 8406, 9170, 10158, 9576] dup_rows_per_dst=[3328, 3226, 2794, 2590, 2992, 2480, 3792, 2950] combine_rows_per_src=[9744, 9302, 9600, 8196, 8406, 9170, 10158, 9576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=263184384 dispatch_meta_bytes=338232 combine_payload_bytes=263184384 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=263522616 ingress_bytes=401596416 link_bytes=401596416 token_bytes=526707000 weight_movement_bytes=138412032 total_nvlink_bytes=665119032 weight_to_token_ratio=0.262788 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=8192 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.511174982e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76358 payload_rows_remote=65448 dup_rows=21946 dup_rows_remote=18306 dedup_ratio=0.223246 router_tokens_per_expert=[264, 540, 419, 543, 1246, 601, 922, 480, 259, 332, 1306, 233, 1166, 587, 869, 2260, 818, 1374, 911, 587, 1038, 1015, 1433, 670, 650, 835, 924, 1742, 244, 904, 437, 671, 350, 331, 478, 659, 982, 334, 327, 981, 1492, 747, 39, 747, 324, 452, 45, 1121, 3000, 367, 1144, 158, 539, 1033, 811, 946, 499, 936, 334, 500, 1812, 385, 548, 451] payload_rows_per_dst=[10910, 9504, 9930, 8942, 11726, 8276, 8106, 8964] dup_rows_per_dst=[3640, 2748, 3278, 2878, 3158, 2388, 1890, 1966] combine_rows_per_src=[10910, 9504, 9930, 8942, 11726, 8276, 8106, 8964] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=262930432 dispatch_meta_bytes=335384 combine_payload_bytes=262930432 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263265816 ingress_bytes=297533440 link_bytes=297533440 token_bytes=526196248 weight_movement_bytes=34603008 total_nvlink_bytes=560799256 weight_to_token_ratio=0.065761 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.119594381e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267124736 dispatch_meta_bytes=348360 combine_payload_bytes=267124736 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267473096 ingress_bytes=301727744 link_bytes=301727744 token_bytes=534597832 weight_movement_bytes=34603008 total_nvlink_bytes=569200840 weight_to_token_ratio=0.064727 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=7680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135377210e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267575296 dispatch_meta_bytes=343240 combine_payload_bytes=267575296 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267918536 ingress_bytes=302178304 link_bytes=302178304 token_bytes=535493832 weight_movement_bytes=34603008 total_nvlink_bytes=570096840 weight_to_token_ratio=0.064619 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=8320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137072631e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74164 payload_rows_remote=63572 dup_rows=24140 dup_rows_remote=19704 dedup_ratio=0.245565 router_tokens_per_expert=[1368, 800, 687, 608, 583, 912, 1307, 180, 827, 878, 1254, 626, 159, 1029, 512, 1030, 1115, 891, 708, 849, 582, 479, 1066, 534, 704, 830, 315, 415, 878, 959, 312, 962, 999, 506, 801, 868, 466, 385, 988, 943, 486, 499, 374, 520, 1580, 1567, 806, 370, 463, 645, 745, 830, 1104, 1602, 677, 481, 459, 1032, 1468, 789, 531, 421, 1071, 317] payload_rows_per_dst=[10592, 9606, 9306, 10196, 8760, 9706, 6736, 9262] dup_rows_per_dst=[4436, 3376, 2790, 3758, 3152, 2698, 1016, 2914] combine_rows_per_src=[10592, 9606, 9306, 10196, 8760, 9706, 6736, 9262] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75536 payload_rows_remote=65560 dup_rows=22768 dup_rows_remote=20384 dedup_ratio=0.231608 router_tokens_per_expert=[156, 613, 212, 596, 1342, 713, 881, 628, 296, 293, 1175, 218, 1057, 524, 887, 2243, 930, 1239, 852, 485, 994, 1066, 1289, 675, 596, 764, 777, 1873, 186, 797, 425, 729, 336, 331, 418, 573, 1030, 281, 396, 880, 1601, 795, 32, 852, 453, 516, 49, 877, 3321, 370, 1252, 91, 686, 1104, 821, 833, 540, 968, 561, 501, 1814, 387, 494, 478] payload_rows_per_dst=[8248, 10172, 9372, 9170, 9976, 9732, 9628, 9238] dup_rows_per_dst=[2034, 3214, 3110, 3124, 2384, 3196, 3458, 2248] combine_rows_per_src=[8248, 10172, 9372, 9170, 9976, 9732, 9628, 9238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=268075008 dispatch_meta_bytes=335016 combine_payload_bytes=268075008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268410024 ingress_bytes=302678016 link_bytes=302678016 token_bytes=536485032 weight_movement_bytes=34603008 total_nvlink_bytes=571088040 weight_to_token_ratio=0.064499 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138953007e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76064 payload_rows_remote=66752 dup_rows=22240 dup_rows_remote=19652 dedup_ratio=0.226237 router_tokens_per_expert=[183, 456, 527, 500, 1258, 793, 922, 592, 236, 335, 1226, 187, 1239, 623, 904, 2214, 828, 1200, 885, 563, 944, 957, 1212, 611, 560, 807, 918, 1891, 203, 873, 447, 563, 382, 407, 479, 624, 1051, 357, 393, 981, 1545, 748, 45, 794, 346, 484, 46, 1014, 3069, 283, 1315, 99, 662, 1067, 789, 822, 479, 963, 369, 667, 1802, 462, 560, 391] payload_rows_per_dst=[10928, 9312, 7614, 9344, 12096, 9408, 8140, 9222] dup_rows_per_dst=[3962, 2588, 1962, 3180, 3390, 3060, 1934, 2164] combine_rows_per_src=[10928, 9312, 7614, 9344, 12096, 9408, 8140, 9222] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74872 payload_rows_remote=65326 dup_rows=23432 dup_rows_remote=20484 dedup_ratio=0.238363 router_tokens_per_expert=[1425, 764, 612, 536, 673, 847, 1422, 267, 911, 1081, 1310, 401, 145, 1132, 496, 952, 916, 809, 807, 789, 638, 488, 1125, 675, 731, 870, 428, 487, 881, 909, 406, 907, 798, 487, 777, 720, 540, 377, 877, 924, 513, 510, 371, 683, 1437, 1456, 624, 352, 641, 695, 741, 878, 967, 1658, 693, 436, 561, 913, 1442, 765, 648, 468, 1065, 295] payload_rows_per_dst=[9856, 9666, 9546, 10224, 8488, 9194, 8580, 9318] dup_rows_per_dst=[3236, 3190, 2948, 4180, 2512, 2848, 1522, 2996] combine_rows_per_src=[9856, 9666, 9546, 10224, 8488, 9194, 8580, 9318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76362 payload_rows_remote=66832 dup_rows=21942 dup_rows_remote=18460 dedup_ratio=0.223206 router_tokens_per_expert=[232, 547, 838, 562, 1273, 651, 970, 647, 300, 395, 1076, 286, 983, 550, 753, 1869, 873, 1219, 847, 309, 963, 1229, 1398, 517, 655, 797, 1001, 1804, 242, 871, 453, 683, 266, 350, 397, 712, 1105, 281, 317, 1041, 1510, 828, 42, 673, 264, 511, 62, 864, 3209, 505, 1422, 131, 789, 1087, 778, 880, 369, 860, 614, 592, 1516, 398, 532, 454] payload_rows_per_dst=[10472, 8236, 9030, 9530, 12020, 9370, 8868, 8836] dup_rows_per_dst=[3494, 1662, 2884, 3482, 3336, 2934, 2316, 1834] combine_rows_per_src=[10472, 8236, 9030, 9530, 12020, 9370, 8868, 8836] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75518 payload_rows_remote=65498 dup_rows=22786 dup_rows_remote=20038 dedup_ratio=0.231791 router_tokens_per_expert=[216, 392, 782, 558, 1226, 590, 936, 534, 281, 284, 900, 272, 1167, 470, 912, 2235, 833, 1289, 938, 865, 905, 944, 1311, 589, 631, 764, 902, 1910, 257, 914, 470, 589, 294, 351, 454, 594, 997, 223, 340, 1049, 1664, 494, 22, 823, 391, 498, 38, 1143, 3276, 316, 1294, 70, 541, 1084, 768, 751, 511, 1035, 415, 603, 1747, 421, 637, 442] payload_rows_per_dst=[8100, 10200, 9680, 9670, 7364, 10020, 11216, 9268] dup_rows_per_dst=[2368, 2842, 3046, 3204, 1240, 2748, 4984, 2354] combine_rows_per_src=[8100, 10200, 9680, 9670, 7364, 10020, 11216, 9268] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75560 payload_rows_remote=63084 dup_rows=22744 dup_rows_remote=17350 dedup_ratio=0.231364 router_tokens_per_expert=[224, 455, 380, 625, 1305, 715, 835, 614, 265, 328, 1198, 232, 1111, 544, 932, 2049, 924, 1210, 967, 882, 925, 973, 1479, 599, 605, 750, 954, 1823, 295, 844, 426, 645, 342, 316, 413, 522, 875, 313, 393, 982, 1702, 455, 50, 750, 409, 469, 32, 929, 3133, 358, 1396, 118, 697, 1012, 782, 789, 544, 966, 588, 527, 1613, 413, 688, 463] payload_rows_per_dst=[8118, 10212, 9862, 8198, 7078, 9868, 9748, 12476] dup_rows_per_dst=[2188, 3106, 3098, 2320, 1234, 2682, 2722, 5394] combine_rows_per_src=[8118, 10212, 9862, 8198, 7078, 9868, 9748, 12476] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=64254 dup_rows=23924 dup_rows_remote=20304 dedup_ratio=0.243368 router_tokens_per_expert=[1466, 712, 583, 604, 507, 937, 1337, 190, 898, 1179, 1285, 467, 169, 927, 462, 993, 994, 874, 762, 778, 645, 461, 1037, 581, 799, 838, 286, 492, 863, 942, 360, 926, 1051, 438, 894, 841, 541, 392, 939, 924, 443, 464, 329, 498, 1523, 1546, 833, 382, 564, 748, 770, 777, 1181, 1525, 813, 495, 477, 953, 1446, 751, 540, 407, 1050, 263] payload_rows_per_dst=[7460, 9554, 9494, 8436, 10484, 9470, 10126, 9356] dup_rows_per_dst=[2280, 2992, 2770, 2576, 4488, 2566, 3620, 2632] combine_rows_per_src=[7460, 9554, 9494, 8436, 10484, 9470, 10126, 9356] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=269295616 dispatch_meta_bytes=347624 combine_payload_bytes=269295616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269643240 ingress_bytes=303898624 link_bytes=303898624 token_bytes=538938856 weight_movement_bytes=34603008 total_nvlink_bytes=573541864 weight_to_token_ratio=0.064206 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=7808 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.143546056e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=273416192 dispatch_meta_bytes=345616 combine_payload_bytes=273416192 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=273761808 ingress_bytes=342622208 link_bytes=342622208 token_bytes=547178000 weight_movement_bytes=69206016 total_nvlink_bytes=616384016 weight_to_token_ratio=0.126478 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.289259785e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=268279808 dispatch_meta_bytes=342144 combine_payload_bytes=268279808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268621952 ingress_bytes=302882816 link_bytes=302882816 token_bytes=536901760 weight_movement_bytes=34603008 total_nvlink_bytes=571504768 weight_to_token_ratio=0.064449 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.139723653e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=260390912 dispatch_meta_bytes=333104 combine_payload_bytes=260390912 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=260724016 ingress_bytes=329596928 link_bytes=329596928 token_bytes=521114928 weight_movement_bytes=69206016 total_nvlink_bytes=590320944 weight_to_token_ratio=0.132804 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.240246705e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=258392064 dispatch_meta_bytes=321736 combine_payload_bytes=258392064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258713800 ingress_bytes=292995072 link_bytes=292995072 token_bytes=517105864 weight_movement_bytes=34603008 total_nvlink_bytes=551708872 weight_to_token_ratio=0.066917 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.102516868e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76968 payload_rows_remote=67166 dup_rows=21336 dup_rows_remote=17952 dedup_ratio=0.217041 router_tokens_per_expert=[211, 433, 519, 609, 1305, 543, 780, 515, 302, 386, 1248, 331, 1173, 600, 869, 2040, 992, 1263, 989, 1043, 1057, 1091, 1289, 578, 578, 783, 858, 2071, 238, 875, 455, 485, 368, 358, 431, 541, 974, 256, 397, 1040, 1628, 554, 58, 773, 321, 435, 23, 729, 3031, 338, 1366, 136, 619, 961, 718, 878, 567, 965, 578, 510, 1533, 407, 639, 511] payload_rows_per_dst=[10856, 8368, 9802, 9488, 11766, 9330, 8118, 9240] dup_rows_per_dst=[3054, 2290, 3384, 3198, 3026, 2290, 1914, 2180] combine_rows_per_src=[10856, 8368, 9802, 9488, 11766, 9330, 8118, 9240] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76362 payload_rows_remote=66832 dup_rows=21942 dup_rows_remote=18460 dedup_ratio=0.223206 router_tokens_per_expert=[232, 547, 838, 562, 1273, 651, 970, 647, 300, 395, 1076, 286, 983, 550, 753, 1869, 873, 1219, 847, 309, 963, 1229, 1398, 517, 655, 797, 1001, 1804, 242, 871, 453, 683, 266, 350, 397, 712, 1105, 281, 317, 1041, 1510, 828, 42, 673, 264, 511, 62, 864, 3209, 505, 1422, 131, 789, 1087, 778, 880, 369, 860, 614, 592, 1516, 398, 532, 454] payload_rows_per_dst=[10472, 8236, 9030, 9530, 12020, 9370, 8868, 8836] dup_rows_per_dst=[3494, 1662, 2884, 3482, 3336, 2934, 2316, 1834] combine_rows_per_src=[10472, 8236, 9030, 9530, 12020, 9370, 8868, 8836] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=263184384 dispatch_meta_bytes=338232 combine_payload_bytes=263184384 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=263522616 ingress_bytes=401596416 link_bytes=401596416 token_bytes=526707000 weight_movement_bytes=138412032 total_nvlink_bytes=665119032 weight_to_token_ratio=0.262788 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=8192 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.511174982e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76464 payload_rows_remote=66790 dup_rows=21840 dup_rows_remote=19244 dedup_ratio=0.222168 router_tokens_per_expert=[189, 833, 81, 454, 272, 483, 223, 2000, 40, 1283, 254, 1721, 702, 1114, 413, 608, 2626, 436, 552, 1040, 480, 266, 1699, 161, 572, 1617, 2877, 1247, 816, 508, 463, 126, 552, 742, 1721, 1388, 1462, 834, 1977, 442, 330, 827, 113, 180, 1421, 1608, 109, 410, 782, 1082, 701, 321, 578, 442, 198, 183, 1184, 580, 256, 297, 378, 763, 549, 586] payload_rows_per_dst=[10908, 9674, 10746, 8670, 10978, 8256, 10224, 7008] dup_rows_per_dst=[3916, 2596, 3774, 2028, 3304, 1740, 2304, 2178] combine_rows_per_src=[10908, 9674, 10746, 8670, 10978, 8256, 10224, 7008] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75090 payload_rows_remote=64300 dup_rows=23214 dup_rows_remote=20144 dedup_ratio=0.236145 router_tokens_per_expert=[211, 803, 108, 380, 269, 756, 220, 1773, 26, 1139, 220, 1896, 694, 1379, 343, 625, 1936, 461, 613, 1128, 442, 313, 1934, 165, 471, 1552, 2629, 1072, 751, 672, 218, 108, 878, 647, 1932, 1223, 1875, 895, 1814, 433, 332, 685, 107, 152, 1301, 1616, 108, 693, 929, 804, 633, 305, 505, 404, 190, 210, 1477, 671, 293, 408, 293, 752, 545, 735] payload_rows_per_dst=[8606, 9940, 8040, 9432, 9412, 10790, 9550, 9320] dup_rows_per_dst=[2556, 2704, 2072, 3392, 2490, 3070, 2038, 4892] combine_rows_per_src=[8606, 9940, 8040, 9432, 9412, 10790, 9550, 9320] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75536 payload_rows_remote=65560 dup_rows=22768 dup_rows_remote=20384 dedup_ratio=0.231608 router_tokens_per_expert=[156, 613, 212, 596, 1342, 713, 881, 628, 296, 293, 1175, 218, 1057, 524, 887, 2243, 930, 1239, 852, 485, 994, 1066, 1289, 675, 596, 764, 777, 1873, 186, 797, 425, 729, 336, 331, 418, 573, 1030, 281, 396, 880, 1601, 795, 32, 852, 453, 516, 49, 877, 3321, 370, 1252, 91, 686, 1104, 821, 833, 540, 968, 561, 501, 1814, 387, 494, 478] payload_rows_per_dst=[8248, 10172, 9372, 9170, 9976, 9732, 9628, 9238] dup_rows_per_dst=[2034, 3214, 3110, 3124, 2384, 3196, 3458, 2248] combine_rows_per_src=[8248, 10172, 9372, 9170, 9976, 9732, 9628, 9238] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=65754 dup_rows=23860 dup_rows_remote=20054 dedup_ratio=0.242716 router_tokens_per_expert=[179, 818, 117, 454, 275, 457, 252, 1837, 37, 1216, 245, 1755, 964, 1246, 344, 729, 2021, 568, 654, 830, 562, 400, 1492, 234, 534, 1701, 2698, 1151, 753, 646, 476, 92, 609, 733, 1485, 1337, 1610, 904, 1755, 415, 332, 897, 117, 149, 1583, 1651, 96, 526, 957, 1023, 728, 324, 590, 510, 178, 143, 1331, 579, 288, 317, 266, 737, 589, 656] payload_rows_per_dst=[7260, 7874, 9168, 11156, 8364, 11482, 10450, 8690] dup_rows_per_dst=[1518, 1688, 3362, 4946, 2320, 3262, 2958, 3806] combine_rows_per_src=[7260, 7874, 9168, 11156, 8364, 11482, 10450, 8690] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=275111936 dispatch_meta_bytes=340472 combine_payload_bytes=275111936 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=275452408 ingress_bytes=378920960 link_bytes=378920960 token_bytes=550564344 weight_movement_bytes=103809024 total_nvlink_bytes=654373368 weight_to_token_ratio=0.188550 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=8448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.425849067e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=273743872 dispatch_meta_bytes=341168 combine_payload_bytes=273743872 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=274085040 ingress_bytes=342949888 link_bytes=342949888 token_bytes=547828912 weight_movement_bytes=69206016 total_nvlink_bytes=617034928 weight_to_token_ratio=0.126328 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=13440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.290492819e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76358 payload_rows_remote=65448 dup_rows=21946 dup_rows_remote=18306 dedup_ratio=0.223246 router_tokens_per_expert=[264, 540, 419, 543, 1246, 601, 922, 480, 259, 332, 1306, 233, 1166, 587, 869, 2260, 818, 1374, 911, 587, 1038, 1015, 1433, 670, 650, 835, 924, 1742, 244, 904, 437, 671, 350, 331, 478, 659, 982, 334, 327, 981, 1492, 747, 39, 747, 324, 452, 45, 1121, 3000, 367, 1144, 158, 539, 1033, 811, 946, 499, 936, 334, 500, 1812, 385, 548, 451] payload_rows_per_dst=[10910, 9504, 9930, 8942, 11726, 8276, 8106, 8964] dup_rows_per_dst=[3640, 2748, 3278, 2878, 3158, 2388, 1890, 1966] combine_rows_per_src=[10910, 9504, 9930, 8942, 11726, 8276, 8106, 8964] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=273571840 dispatch_meta_bytes=344136 combine_payload_bytes=273571840 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=273915976 ingress_bytes=342777856 link_bytes=342777856 token_bytes=547487816 weight_movement_bytes=69206016 total_nvlink_bytes=616693832 weight_to_token_ratio=0.126406 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=12416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.289845476e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=268533760 dispatch_meta_bytes=343776 combine_payload_bytes=268533760 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268877536 ingress_bytes=303136768 link_bytes=303136768 token_bytes=537411296 weight_movement_bytes=34603008 total_nvlink_bytes=572014304 weight_to_token_ratio=0.064388 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.140679253e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=263372800 dispatch_meta_bytes=337776 combine_payload_bytes=263372800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263710576 ingress_bytes=297975808 link_bytes=297975808 token_bytes=527083376 weight_movement_bytes=34603008 total_nvlink_bytes=561686384 weight_to_token_ratio=0.065650 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=13568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.121258976e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=269328384 dispatch_meta_bytes=343232 combine_payload_bytes=269328384 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269671616 ingress_bytes=303931392 link_bytes=303931392 token_bytes=539000000 weight_movement_bytes=34603008 total_nvlink_bytes=573603008 weight_to_token_ratio=0.064199 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.143669360e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=268075008 dispatch_meta_bytes=335016 combine_payload_bytes=268075008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268410024 ingress_bytes=302678016 link_bytes=302678016 token_bytes=536485032 weight_movement_bytes=34603008 total_nvlink_bytes=571088040 weight_to_token_ratio=0.064499 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138953007e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74792 payload_rows_remote=64026 dup_rows=23512 dup_rows_remote=19016 dedup_ratio=0.239176 router_tokens_per_expert=[175, 439, 370, 540, 1293, 632, 926, 591, 254, 261, 1255, 322, 1115, 573, 852, 2293, 765, 1140, 863, 678, 944, 1055, 1453, 681, 594, 701, 826, 1903, 182, 917, 465, 700, 280, 328, 483, 567, 939, 250, 400, 1008, 1586, 724, 40, 740, 344, 457, 37, 1120, 3238, 379, 1297, 110, 764, 1081, 844, 725, 596, 963, 453, 517, 1674, 437, 544, 469] payload_rows_per_dst=[7790, 10442, 9266, 9492, 7248, 9842, 10766, 9946] dup_rows_per_dst=[2142, 3408, 2986, 3084, 1262, 3160, 4496, 2974] combine_rows_per_src=[7790, 10442, 9266, 9492, 7248, 9842, 10766, 9946] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76288 payload_rows_remote=66450 dup_rows=22016 dup_rows_remote=18364 dedup_ratio=0.223958 router_tokens_per_expert=[165, 868, 136, 394, 262, 523, 231, 1665, 36, 1445, 284, 1841, 636, 1247, 382, 916, 2171, 511, 652, 981, 528, 318, 1373, 211, 553, 1525, 2811, 1256, 815, 465, 456, 82, 664, 706, 1668, 1154, 1737, 947, 1698, 513, 318, 879, 124, 154, 1580, 1738, 125, 448, 746, 1005, 762, 362, 561, 312, 177, 216, 1286, 620, 256, 375, 360, 757, 537, 628] payload_rows_per_dst=[10732, 10348, 9838, 8588, 10560, 8840, 9604, 7778] dup_rows_per_dst=[3378, 3226, 3652, 1716, 3380, 1892, 2074, 2698] combine_rows_per_src=[10732, 10348, 9838, 8588, 10560, 8840, 9604, 7778] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76398 payload_rows_remote=68220 dup_rows=21906 dup_rows_remote=20388 dedup_ratio=0.222839 router_tokens_per_expert=[113, 762, 145, 465, 323, 405, 240, 1340, 43, 994, 249, 1801, 573, 1672, 392, 914, 1996, 372, 716, 1061, 366, 293, 1684, 210, 496, 1425, 2594, 1034, 673, 681, 376, 163, 577, 689, 1952, 1210, 1778, 855, 1875, 572, 250, 876, 169, 126, 1693, 1599, 157, 409, 955, 909, 641, 475, 666, 448, 212, 227, 1632, 486, 294, 335, 402, 677, 718, 717] payload_rows_per_dst=[10084, 10362, 10078, 8178, 9170, 8608, 10160, 9758] dup_rows_per_dst=[2690, 2914, 3318, 1518, 2192, 1950, 2656, 4668] combine_rows_per_src=[10084, 10362, 10078, 8178, 9170, 8608, 10160, 9758] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=273416192 dispatch_meta_bytes=345616 combine_payload_bytes=273416192 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=273761808 ingress_bytes=342622208 link_bytes=342622208 token_bytes=547178000 weight_movement_bytes=69206016 total_nvlink_bytes=616384016 weight_to_token_ratio=0.126478 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.289259785e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75560 payload_rows_remote=63084 dup_rows=22744 dup_rows_remote=17350 dedup_ratio=0.231364 router_tokens_per_expert=[224, 455, 380, 625, 1305, 715, 835, 614, 265, 328, 1198, 232, 1111, 544, 932, 2049, 924, 1210, 967, 882, 925, 973, 1479, 599, 605, 750, 954, 1823, 295, 844, 426, 645, 342, 316, 413, 522, 875, 313, 393, 982, 1702, 455, 50, 750, 409, 469, 32, 929, 3133, 358, 1396, 118, 697, 1012, 782, 789, 544, 966, 588, 527, 1613, 413, 688, 463] payload_rows_per_dst=[8118, 10212, 9862, 8198, 7078, 9868, 9748, 12476] dup_rows_per_dst=[2188, 3106, 3098, 2320, 1234, 2682, 2722, 5394] combine_rows_per_src=[8118, 10212, 9862, 8198, 7078, 9868, 9748, 12476] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75002 payload_rows_remote=63774 dup_rows=23302 dup_rows_remote=18916 dedup_ratio=0.237040 router_tokens_per_expert=[1874, 787, 963, 46, 88, 182, 1773, 1905, 89, 238, 713, 304, 924, 537, 384, 1583, 2999, 251, 461, 1452, 433, 1224, 108, 1582, 131, 290, 563, 217, 850, 1145, 3035, 582, 1904, 880, 198, 93, 502, 823, 212, 354, 1165, 2352, 331, 654, 294, 332, 448, 539, 1205, 846, 622, 354, 99, 1231, 169, 2164, 901, 8, 178, 245, 1060, 544, 497, 235] payload_rows_per_dst=[11508, 11228, 10756, 6686, 7846, 9520, 7676, 9782] dup_rows_per_dst=[3728, 4386, 4594, 870, 2086, 2710, 1376, 3552] combine_rows_per_src=[11508, 11228, 10756, 6686, 7846, 9520, 7676, 9782] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=273743872 dispatch_meta_bytes=341168 combine_payload_bytes=273743872 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=274085040 ingress_bytes=342949888 link_bytes=342949888 token_bytes=547828912 weight_movement_bytes=69206016 total_nvlink_bytes=617034928 weight_to_token_ratio=0.126328 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=13440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.290492819e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=268279808 dispatch_meta_bytes=342144 combine_payload_bytes=268279808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268621952 ingress_bytes=302882816 link_bytes=302882816 token_bytes=536901760 weight_movement_bytes=34603008 total_nvlink_bytes=571504768 weight_to_token_ratio=0.064449 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.139723653e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76172 payload_rows_remote=65818 dup_rows=22132 dup_rows_remote=18792 dedup_ratio=0.225138 router_tokens_per_expert=[124, 934, 134, 454, 318, 434, 262, 1446, 60, 1171, 192, 1853, 588, 1418, 400, 755, 1904, 493, 624, 953, 566, 322, 1438, 170, 506, 1671, 2741, 1071, 923, 565, 388, 110, 922, 785, 1862, 1145, 1767, 749, 1881, 559, 316, 882, 120, 209, 1313, 1583, 113, 399, 834, 922, 648, 472, 528, 487, 197, 172, 1379, 564, 296, 490, 469, 711, 584, 806] payload_rows_per_dst=[10354, 10150, 9642, 9220, 10546, 8044, 10170, 8046] dup_rows_per_dst=[3340, 2724, 3298, 2610, 3670, 1826, 2112, 2552] combine_rows_per_src=[10354, 10150, 9642, 9220, 10546, 8044, 10170, 8046] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267575296 dispatch_meta_bytes=343240 combine_payload_bytes=267575296 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267918536 ingress_bytes=302178304 link_bytes=302178304 token_bytes=535493832 weight_movement_bytes=34603008 total_nvlink_bytes=570096840 weight_to_token_ratio=0.064619 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=8320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137072631e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=262250496 dispatch_meta_bytes=332168 combine_payload_bytes=262250496 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=262582664 ingress_bytes=366059520 link_bytes=366059520 token_bytes=524833160 weight_movement_bytes=103809024 total_nvlink_bytes=628642184 weight_to_token_ratio=0.197794 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.377452504e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=272179200 dispatch_meta_bytes=339256 combine_payload_bytes=272179200 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=272518456 ingress_bytes=341385216 link_bytes=341385216 token_bytes=544697656 weight_movement_bytes=69206016 total_nvlink_bytes=613903672 weight_to_token_ratio=0.127054 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=13312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.284605084e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76046 payload_rows_remote=66090 dup_rows=22258 dup_rows_remote=19046 dedup_ratio=0.226420 router_tokens_per_expert=[1714, 697, 655, 60, 86, 243, 1768, 1772, 146, 253, 784, 295, 1005, 677, 265, 1825, 2163, 371, 630, 1435, 550, 1035, 76, 1333, 175, 274, 975, 194, 700, 1224, 2827, 712, 1989, 1227, 201, 99, 649, 647, 172, 355, 1135, 2812, 505, 686, 166, 400, 478, 402, 745, 1071, 580, 271, 101, 1306, 283, 2454, 832, 22, 234, 213, 737, 768, 514, 179] payload_rows_per_dst=[9032, 8448, 8504, 10622, 9994, 9956, 10526, 8964] dup_rows_per_dst=[1530, 2052, 2356, 3540, 4112, 3212, 3096, 2360] combine_rows_per_src=[9032, 8448, 8504, 10622, 9994, 9956, 10526, 8964] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74438 payload_rows_remote=65208 dup_rows=23866 dup_rows_remote=20878 dedup_ratio=0.242778 router_tokens_per_expert=[2051, 689, 726, 68, 121, 286, 1857, 1489, 128, 238, 763, 308, 771, 631, 353, 1938, 2296, 348, 542, 1373, 433, 1053, 85, 1064, 278, 333, 837, 214, 766, 1229, 2536, 793, 1844, 781, 250, 140, 671, 665, 294, 362, 1269, 2851, 374, 520, 191, 452, 483, 505, 1071, 890, 1003, 352, 79, 964, 379, 2352, 861, 38, 186, 256, 883, 633, 692, 264] payload_rows_per_dst=[7854, 8506, 8106, 10566, 10222, 7680, 12274, 9230] dup_rows_per_dst=[1282, 1754, 1690, 3406, 3894, 1758, 7094, 2988] combine_rows_per_src=[7854, 8506, 8106, 10566, 10222, 7680, 12274, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=269590528 dispatch_meta_bytes=338440 combine_payload_bytes=269590528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269928968 ingress_bytes=304193536 link_bytes=304193536 token_bytes=539519496 weight_movement_bytes=34603008 total_nvlink_bytes=574122504 weight_to_token_ratio=0.064137 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.144655786e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=258392064 dispatch_meta_bytes=321736 combine_payload_bytes=258392064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258713800 ingress_bytes=292995072 link_bytes=292995072 token_bytes=517105864 weight_movement_bytes=34603008 total_nvlink_bytes=551708872 weight_to_token_ratio=0.066917 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.102516868e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76802 payload_rows_remote=67520 dup_rows=21502 dup_rows_remote=19120 dedup_ratio=0.218730 router_tokens_per_expert=[110, 714, 86, 456, 271, 465, 256, 1833, 29, 985, 244, 2027, 664, 1330, 370, 633, 1974, 529, 674, 944, 369, 318, 1804, 132, 541, 1813, 2881, 1149, 711, 502, 399, 99, 770, 785, 1563, 1245, 1927, 643, 1796, 462, 259, 990, 152, 155, 1417, 1829, 98, 463, 906, 996, 790, 309, 487, 409, 160, 116, 1525, 578, 260, 429, 319, 766, 564, 672] payload_rows_per_dst=[10818, 9892, 7950, 8702, 9282, 11072, 9792, 9294] dup_rows_per_dst=[3326, 2672, 1910, 1726, 2382, 3282, 2146, 4058] combine_rows_per_src=[10818, 9892, 7950, 8702, 9282, 11072, 9792, 9294] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279429120 dispatch_meta_bytes=354432 combine_payload_bytes=279429120 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=279783552 ingress_bytes=348635136 link_bytes=348635136 token_bytes=559212672 weight_movement_bytes=69206016 total_nvlink_bytes=628418688 weight_to_token_ratio=0.123756 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.311885949e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=261218304 dispatch_meta_bytes=330760 combine_payload_bytes=261218304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=261549064 ingress_bytes=295821312 link_bytes=295821312 token_bytes=522767368 weight_movement_bytes=34603008 total_nvlink_bytes=557370376 weight_to_token_ratio=0.066192 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.113151781e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76172 payload_rows_remote=65818 dup_rows=22132 dup_rows_remote=18792 dedup_ratio=0.225138 router_tokens_per_expert=[124, 934, 134, 454, 318, 434, 262, 1446, 60, 1171, 192, 1853, 588, 1418, 400, 755, 1904, 493, 624, 953, 566, 322, 1438, 170, 506, 1671, 2741, 1071, 923, 565, 388, 110, 922, 785, 1862, 1145, 1767, 749, 1881, 559, 316, 882, 120, 209, 1313, 1583, 113, 399, 834, 922, 648, 472, 528, 487, 197, 172, 1379, 564, 296, 490, 469, 711, 584, 806] payload_rows_per_dst=[10354, 10150, 9642, 9220, 10546, 8044, 10170, 8046] dup_rows_per_dst=[3340, 2724, 3298, 2610, 3670, 1826, 2112, 2552] combine_rows_per_src=[10354, 10150, 9642, 9220, 10546, 8044, 10170, 8046] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76464 payload_rows_remote=66790 dup_rows=21840 dup_rows_remote=19244 dedup_ratio=0.222168 router_tokens_per_expert=[189, 833, 81, 454, 272, 483, 223, 2000, 40, 1283, 254, 1721, 702, 1114, 413, 608, 2626, 436, 552, 1040, 480, 266, 1699, 161, 572, 1617, 2877, 1247, 816, 508, 463, 126, 552, 742, 1721, 1388, 1462, 834, 1977, 442, 330, 827, 113, 180, 1421, 1608, 109, 410, 782, 1082, 701, 321, 578, 442, 198, 183, 1184, 580, 256, 297, 378, 763, 549, 586] payload_rows_per_dst=[10908, 9674, 10746, 8670, 10978, 8256, 10224, 7008] dup_rows_per_dst=[3916, 2596, 3774, 2028, 3304, 1740, 2304, 2178] combine_rows_per_src=[10908, 9674, 10746, 8670, 10978, 8256, 10224, 7008] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=270704640 dispatch_meta_bytes=340544 combine_payload_bytes=270704640 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=271045184 ingress_bytes=374513664 link_bytes=374513664 token_bytes=541749824 weight_movement_bytes=103809024 total_nvlink_bytes=645558848 weight_to_token_ratio=0.191618 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.409264767e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267091968 dispatch_meta_bytes=344344 combine_payload_bytes=267091968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267436312 ingress_bytes=301694976 link_bytes=301694976 token_bytes=534528280 weight_movement_bytes=34603008 total_nvlink_bytes=569131288 weight_to_token_ratio=0.064736 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135253906e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=276561920 dispatch_meta_bytes=346560 combine_payload_bytes=276561920 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=276908480 ingress_bytes=449576960 link_bytes=449576960 token_bytes=553470400 weight_movement_bytes=173015040 total_nvlink_bytes=726485440 weight_to_token_ratio=0.312600 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=12672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.691721907e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=269590528 dispatch_meta_bytes=338440 combine_payload_bytes=269590528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269928968 ingress_bytes=304193536 link_bytes=304193536 token_bytes=539519496 weight_movement_bytes=34603008 total_nvlink_bytes=574122504 weight_to_token_ratio=0.064137 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.144655786e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75090 payload_rows_remote=64300 dup_rows=23214 dup_rows_remote=20144 dedup_ratio=0.236145 router_tokens_per_expert=[211, 803, 108, 380, 269, 756, 220, 1773, 26, 1139, 220, 1896, 694, 1379, 343, 625, 1936, 461, 613, 1128, 442, 313, 1934, 165, 471, 1552, 2629, 1072, 751, 672, 218, 108, 878, 647, 1932, 1223, 1875, 895, 1814, 433, 332, 685, 107, 152, 1301, 1616, 108, 693, 929, 804, 633, 305, 505, 404, 190, 210, 1477, 671, 293, 408, 293, 752, 545, 735] payload_rows_per_dst=[8606, 9940, 8040, 9432, 9412, 10790, 9550, 9320] dup_rows_per_dst=[2556, 2704, 2072, 3392, 2490, 3070, 2038, 4892] combine_rows_per_src=[8606, 9940, 8040, 9432, 9412, 10790, 9550, 9320] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76398 payload_rows_remote=68220 dup_rows=21906 dup_rows_remote=20388 dedup_ratio=0.222839 router_tokens_per_expert=[113, 762, 145, 465, 323, 405, 240, 1340, 43, 994, 249, 1801, 573, 1672, 392, 914, 1996, 372, 716, 1061, 366, 293, 1684, 210, 496, 1425, 2594, 1034, 673, 681, 376, 163, 577, 689, 1952, 1210, 1778, 855, 1875, 572, 250, 876, 169, 126, 1693, 1599, 157, 409, 955, 909, 641, 475, 666, 448, 212, 227, 1632, 486, 294, 335, 402, 677, 718, 717] payload_rows_per_dst=[10084, 10362, 10078, 8178, 9170, 8608, 10160, 9758] dup_rows_per_dst=[2690, 2914, 3318, 1518, 2192, 1950, 2656, 4668] combine_rows_per_src=[10084, 10362, 10078, 8178, 9170, 8608, 10160, 9758] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=273571840 dispatch_meta_bytes=344136 combine_payload_bytes=273571840 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=273915976 ingress_bytes=342777856 link_bytes=342777856 token_bytes=547487816 weight_movement_bytes=69206016 total_nvlink_bytes=616693832 weight_to_token_ratio=0.126406 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=12416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.289845476e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74894 payload_rows_remote=65120 dup_rows=23410 dup_rows_remote=21118 dedup_ratio=0.238139 router_tokens_per_expert=[126, 739, 92, 457, 310, 513, 226, 1946, 26, 1203, 208, 2030, 776, 1246, 416, 712, 2183, 482, 519, 1031, 557, 318, 1818, 156, 541, 1675, 2852, 1251, 733, 628, 439, 92, 581, 596, 1793, 1269, 1574, 670, 1879, 437, 327, 738, 125, 204, 1125, 1822, 146, 425, 714, 1025, 588, 374, 544, 565, 195, 149, 1386, 641, 261, 422, 300, 743, 544, 689] payload_rows_per_dst=[7222, 9172, 8722, 11210, 8358, 11230, 9774, 9206] dup_rows_per_dst=[1596, 2368, 2734, 5212, 1896, 2960, 2292, 4352] combine_rows_per_src=[7222, 9172, 8722, 11210, 8358, 11230, 9774, 9206] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75554 payload_rows_remote=64782 dup_rows=22750 dup_rows_remote=18824 dedup_ratio=0.231425 router_tokens_per_expert=[1888, 805, 797, 48, 106, 293, 1803, 1881, 118, 292, 483, 323, 1108, 654, 525, 1518, 2594, 303, 554, 1387, 448, 1274, 101, 1178, 194, 423, 778, 239, 796, 1289, 2640, 911, 1412, 883, 222, 219, 632, 575, 249, 649, 1302, 2217, 252, 655, 335, 371, 459, 506, 1145, 823, 593, 230, 147, 1104, 311, 2104, 812, 81, 190, 327, 1078, 664, 550, 304] payload_rows_per_dst=[10836, 10992, 10772, 7712, 8608, 9620, 7462, 9552] dup_rows_per_dst=[3310, 4330, 3926, 1548, 2170, 2574, 1244, 3648] combine_rows_per_src=[10836, 10992, 10772, 7712, 8608, 9620, 7462, 9552] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76480 payload_rows_remote=66906 dup_rows=21824 dup_rows_remote=19416 dedup_ratio=0.222005 router_tokens_per_expert=[1663, 868, 948, 37, 53, 301, 1864, 1992, 140, 319, 619, 354, 912, 642, 286, 1765, 2300, 206, 833, 1072, 509, 1000, 144, 1445, 181, 319, 605, 336, 583, 1087, 2673, 582, 2039, 921, 182, 138, 669, 757, 137, 489, 1304, 2701, 486, 721, 246, 430, 550, 586, 1213, 735, 564, 336, 137, 1133, 286, 1922, 766, 76, 229, 398, 977, 649, 500, 237] payload_rows_per_dst=[9902, 8856, 9202, 9574, 10094, 10448, 9028, 9376] dup_rows_per_dst=[2224, 1968, 2206, 2408, 3896, 3600, 2634, 2888] combine_rows_per_src=[9902, 8856, 9202, 9574, 10094, 10448, 9028, 9376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76450 payload_rows_remote=67494 dup_rows=21854 dup_rows_remote=18872 dedup_ratio=0.222310 router_tokens_per_expert=[352, 654, 1390, 567, 381, 399, 359, 1181, 126, 849, 673, 485, 326, 1203, 2190, 117, 244, 4238, 27, 993, 55, 508, 2819, 301, 385, 376, 2348, 1941, 1292, 2555, 760, 0, 46, 93, 11, 2028, 282, 326, 534, 469, 169, 122, 210, 1297, 2144, 20, 1102, 670, 274, 367, 651, 1762, 1057, 494, 494, 639, 67, 1520, 71, 391, 1340, 30, 45, 333] payload_rows_per_dst=[11514, 8956, 12988, 7760, 6378, 9500, 8800, 10554] dup_rows_per_dst=[3748, 2982, 5382, 1748, 1200, 1968, 2676, 2150] combine_rows_per_src=[11514, 8956, 12988, 7760, 6378, 9500, 8800, 10554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=63322 dup_rows=24012 dup_rows_remote=20532 dedup_ratio=0.244263 router_tokens_per_expert=[2144, 753, 587, 85, 101, 282, 1638, 1635, 108, 231, 671, 396, 1111, 600, 245, 1938, 2182, 321, 533, 1443, 474, 1157, 98, 1365, 186, 382, 811, 216, 724, 1200, 2701, 685, 1794, 839, 153, 178, 380, 615, 264, 466, 1250, 2609, 587, 605, 231, 598, 502, 446, 1020, 956, 494, 300, 140, 1126, 418, 2049, 1127, 37, 213, 258, 992, 709, 584, 209] payload_rows_per_dst=[10970, 11250, 11308, 9034, 7388, 8074, 7532, 8736] dup_rows_per_dst=[3480, 4752, 4808, 2664, 1990, 2292, 1376, 2650] combine_rows_per_src=[10970, 11250, 11308, 9034, 7388, 8074, 7532, 8736] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76968 payload_rows_remote=67166 dup_rows=21336 dup_rows_remote=17952 dedup_ratio=0.217041 router_tokens_per_expert=[211, 433, 519, 609, 1305, 543, 780, 515, 302, 386, 1248, 331, 1173, 600, 869, 2040, 992, 1263, 989, 1043, 1057, 1091, 1289, 578, 578, 783, 858, 2071, 238, 875, 455, 485, 368, 358, 431, 541, 974, 256, 397, 1040, 1628, 554, 58, 773, 321, 435, 23, 729, 3031, 338, 1366, 136, 619, 961, 718, 878, 567, 965, 578, 510, 1533, 407, 639, 511] payload_rows_per_dst=[10856, 8368, 9802, 9488, 11766, 9330, 8118, 9240] dup_rows_per_dst=[3054, 2290, 3384, 3198, 3026, 2290, 1914, 2180] combine_rows_per_src=[10856, 8368, 9802, 9488, 11766, 9330, 8118, 9240] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77140 payload_rows_remote=67954 dup_rows=21164 dup_rows_remote=19254 dedup_ratio=0.215291 router_tokens_per_expert=[432, 346, 1473, 516, 309, 351, 396, 1193, 112, 1129, 871, 463, 210, 1198, 2297, 166, 219, 4086, 33, 871, 57, 757, 3178, 226, 558, 435, 2354, 1750, 1359, 2478, 648, 0, 55, 106, 8, 1652, 287, 392, 571, 604, 190, 155, 151, 1264, 2029, 16, 1136, 607, 240, 347, 629, 1325, 1063, 424, 548, 590, 87, 1394, 72, 97, 1956, 29, 41, 616] payload_rows_per_dst=[8408, 9506, 9456, 10292, 11422, 9186, 8110, 10760] dup_rows_per_dst=[1624, 3386, 1226, 3916, 4100, 1910, 2222, 2780] combine_rows_per_src=[8408, 9506, 9456, 10292, 11422, 9186, 8110, 10760] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75030 payload_rows_remote=65036 dup_rows=23274 dup_rows_remote=20562 dedup_ratio=0.236755 router_tokens_per_expert=[489, 570, 1209, 302, 543, 388, 418, 983, 117, 956, 730, 488, 313, 1320, 2421, 150, 208, 3738, 26, 1022, 60, 449, 3816, 458, 452, 84, 2225, 1817, 1093, 2158, 519, 0, 45, 147, 10, 1948, 314, 429, 444, 545, 200, 155, 134, 1568, 1925, 34, 1706, 880, 319, 297, 514, 1737, 1019, 583, 567, 755, 226, 1077, 78, 181, 1205, 30, 101, 457] payload_rows_per_dst=[8368, 9536, 4142, 8642, 11542, 9962, 12844, 9994] dup_rows_per_dst=[1436, 3454, 304, 2058, 3698, 3242, 6370, 2712] combine_rows_per_src=[8368, 9536, 4142, 8642, 11542, 9962, 12844, 9994] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=263372800 dispatch_meta_bytes=337776 combine_payload_bytes=263372800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263710576 ingress_bytes=297975808 link_bytes=297975808 token_bytes=527083376 weight_movement_bytes=34603008 total_nvlink_bytes=561686384 weight_to_token_ratio=0.065650 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=13568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.121258976e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=65754 dup_rows=23860 dup_rows_remote=20054 dedup_ratio=0.242716 router_tokens_per_expert=[179, 818, 117, 454, 275, 457, 252, 1837, 37, 1216, 245, 1755, 964, 1246, 344, 729, 2021, 568, 654, 830, 562, 400, 1492, 234, 534, 1701, 2698, 1151, 753, 646, 476, 92, 609, 733, 1485, 1337, 1610, 904, 1755, 415, 332, 897, 117, 149, 1583, 1651, 96, 526, 957, 1023, 728, 324, 590, 510, 178, 143, 1331, 579, 288, 317, 266, 737, 589, 656] payload_rows_per_dst=[7260, 7874, 9168, 11156, 8364, 11482, 10450, 8690] dup_rows_per_dst=[1518, 1688, 3362, 4946, 2320, 3262, 2958, 3806] combine_rows_per_src=[7260, 7874, 9168, 11156, 8364, 11482, 10450, 8690] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279429120 dispatch_meta_bytes=354432 combine_payload_bytes=279429120 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=279783552 ingress_bytes=348635136 link_bytes=348635136 token_bytes=559212672 weight_movement_bytes=69206016 total_nvlink_bytes=628418688 weight_to_token_ratio=0.123756 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.311885949e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=344952 combine_payload_bytes=266731520 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=267076472 ingress_bytes=335937536 link_bytes=335937536 token_bytes=533807992 weight_movement_bytes=69206016 total_nvlink_bytes=603014008 weight_to_token_ratio=0.129646 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.032500 gemm_active_groups=11 gemm_peak_group_rows=14720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264105903e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265347072 dispatch_meta_bytes=334424 combine_payload_bytes=265347072 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265681496 ingress_bytes=334553088 link_bytes=334553088 token_bytes=531028568 weight_movement_bytes=69206016 total_nvlink_bytes=600234584 weight_to_token_ratio=0.130324 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258896336e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=274046976 dispatch_meta_bytes=345288 combine_payload_bytes=274046976 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=274392264 ingress_bytes=343252992 link_bytes=343252992 token_bytes=548439240 weight_movement_bytes=69206016 total_nvlink_bytes=617645256 weight_to_token_ratio=0.126187 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.291633375e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=276455424 dispatch_meta_bytes=345464 combine_payload_bytes=276455424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276800888 ingress_bytes=311058432 link_bytes=311058432 token_bytes=553256312 weight_movement_bytes=34603008 total_nvlink_bytes=587859320 weight_to_token_ratio=0.062544 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.170487837e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=63322 dup_rows=24012 dup_rows_remote=20532 dedup_ratio=0.244263 router_tokens_per_expert=[2144, 753, 587, 85, 101, 282, 1638, 1635, 108, 231, 671, 396, 1111, 600, 245, 1938, 2182, 321, 533, 1443, 474, 1157, 98, 1365, 186, 382, 811, 216, 724, 1200, 2701, 685, 1794, 839, 153, 178, 380, 615, 264, 466, 1250, 2609, 587, 605, 231, 598, 502, 446, 1020, 956, 494, 300, 140, 1126, 418, 2049, 1127, 37, 213, 258, 992, 709, 584, 209] payload_rows_per_dst=[10970, 11250, 11308, 9034, 7388, 8074, 7532, 8736] dup_rows_per_dst=[3480, 4752, 4808, 2664, 1990, 2292, 1376, 2650] combine_rows_per_src=[10970, 11250, 11308, 9034, 7388, 8074, 7532, 8736] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=259366912 dispatch_meta_bytes=335416 combine_payload_bytes=259366912 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=259702328 ingress_bytes=328572928 link_bytes=328572928 token_bytes=519069240 weight_movement_bytes=69206016 total_nvlink_bytes=588275256 weight_to_token_ratio=0.133327 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=14208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.236393476e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=275111936 dispatch_meta_bytes=340472 combine_payload_bytes=275111936 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=275452408 ingress_bytes=378920960 link_bytes=378920960 token_bytes=550564344 weight_movement_bytes=103809024 total_nvlink_bytes=654373368 weight_to_token_ratio=0.188550 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=8448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.425849067e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76320 payload_rows_remote=66898 dup_rows=21984 dup_rows_remote=18812 dedup_ratio=0.223633 router_tokens_per_expert=[1704, 956, 695, 39, 175, 284, 1762, 1867, 137, 334, 725, 259, 1142, 521, 286, 1538, 2099, 336, 387, 1403, 466, 1072, 41, 1164, 166, 386, 941, 194, 820, 1261, 2959, 963, 1708, 842, 210, 174, 397, 589, 233, 440, 1709, 2600, 409, 618, 228, 442, 376, 375, 813, 947, 774, 231, 114, 984, 298, 2404, 832, 22, 359, 278, 966, 915, 572, 211] payload_rows_per_dst=[9628, 8288, 8104, 11220, 9422, 10250, 9952, 9456] dup_rows_per_dst=[1928, 1596, 1634, 4160, 3172, 3264, 3178, 3052] combine_rows_per_src=[9628, 8288, 8104, 11220, 9422, 10250, 9952, 9456] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266387456 dispatch_meta_bytes=342392 combine_payload_bytes=266387456 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=266729848 ingress_bytes=335593472 link_bytes=335593472 token_bytes=533117304 weight_movement_bytes=69206016 total_nvlink_bytes=602323320 weight_to_token_ratio=0.129814 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=18432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262811218e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278339584 dispatch_meta_bytes=348832 combine_payload_bytes=278339584 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=278688416 ingress_bytes=347545600 link_bytes=347545600 token_bytes=557028000 weight_movement_bytes=69206016 total_nvlink_bytes=626234016 weight_to_token_ratio=0.124242 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=15104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.307786113e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=269328384 dispatch_meta_bytes=343232 combine_payload_bytes=269328384 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269671616 ingress_bytes=303931392 link_bytes=303931392 token_bytes=539000000 weight_movement_bytes=34603008 total_nvlink_bytes=573603008 weight_to_token_ratio=0.064199 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.143669360e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75002 payload_rows_remote=63774 dup_rows=23302 dup_rows_remote=18916 dedup_ratio=0.237040 router_tokens_per_expert=[1874, 787, 963, 46, 88, 182, 1773, 1905, 89, 238, 713, 304, 924, 537, 384, 1583, 2999, 251, 461, 1452, 433, 1224, 108, 1582, 131, 290, 563, 217, 850, 1145, 3035, 582, 1904, 880, 198, 93, 502, 823, 212, 354, 1165, 2352, 331, 654, 294, 332, 448, 539, 1205, 846, 622, 354, 99, 1231, 169, 2164, 901, 8, 178, 245, 1060, 544, 497, 235] payload_rows_per_dst=[11508, 11228, 10756, 6686, 7846, 9520, 7676, 9782] dup_rows_per_dst=[3728, 4386, 4594, 870, 2086, 2710, 1376, 3552] combine_rows_per_src=[11508, 11228, 10756, 6686, 7846, 9520, 7676, 9782] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=259366912 dispatch_meta_bytes=335416 combine_payload_bytes=259366912 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=259702328 ingress_bytes=328572928 link_bytes=328572928 token_bytes=519069240 weight_movement_bytes=69206016 total_nvlink_bytes=588275256 weight_to_token_ratio=0.133327 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=14208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.236393476e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=274014208 dispatch_meta_bytes=342840 combine_payload_bytes=274014208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274357048 ingress_bytes=308617216 link_bytes=308617216 token_bytes=548371256 weight_movement_bytes=34603008 total_nvlink_bytes=582974264 weight_to_token_ratio=0.063101 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.161301738e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76046 payload_rows_remote=66090 dup_rows=22258 dup_rows_remote=19046 dedup_ratio=0.226420 router_tokens_per_expert=[1714, 697, 655, 60, 86, 243, 1768, 1772, 146, 253, 784, 295, 1005, 677, 265, 1825, 2163, 371, 630, 1435, 550, 1035, 76, 1333, 175, 274, 975, 194, 700, 1224, 2827, 712, 1989, 1227, 201, 99, 649, 647, 172, 355, 1135, 2812, 505, 686, 166, 400, 478, 402, 745, 1071, 580, 271, 101, 1306, 283, 2454, 832, 22, 234, 213, 737, 768, 514, 179] payload_rows_per_dst=[9032, 8448, 8504, 10622, 9994, 9956, 10526, 8964] dup_rows_per_dst=[1530, 2052, 2356, 3540, 4112, 3212, 3096, 2360] combine_rows_per_src=[9032, 8448, 8504, 10622, 9994, 9956, 10526, 8964] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=261218304 dispatch_meta_bytes=330760 combine_payload_bytes=261218304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=261549064 ingress_bytes=295821312 link_bytes=295821312 token_bytes=522767368 weight_movement_bytes=34603008 total_nvlink_bytes=557370376 weight_to_token_ratio=0.066192 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.113151781e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76324 payload_rows_remote=66044 dup_rows=21980 dup_rows_remote=18404 dedup_ratio=0.223592 router_tokens_per_expert=[1999, 880, 695, 71, 87, 257, 2001, 1555, 123, 304, 792, 313, 874, 508, 341, 1694, 2351, 259, 731, 1377, 519, 974, 65, 1244, 223, 383, 729, 218, 796, 1218, 2972, 608, 1623, 745, 234, 145, 649, 721, 146, 443, 1368, 2453, 475, 581, 274, 393, 442, 687, 1164, 893, 647, 269, 123, 1327, 291, 2214, 838, 25, 233, 270, 949, 615, 478, 276] payload_rows_per_dst=[9238, 8230, 8382, 10962, 10072, 9974, 10280, 9186] dup_rows_per_dst=[1854, 1668, 1956, 3332, 3338, 3372, 3576, 2884] combine_rows_per_src=[9238, 8230, 8382, 10962, 10072, 9974, 10280, 9186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75956 payload_rows_remote=63834 dup_rows=22348 dup_rows_remote=17856 dedup_ratio=0.227336 router_tokens_per_expert=[390, 565, 1247, 429, 372, 477, 550, 928, 127, 903, 728, 457, 293, 1423, 1797, 146, 230, 3444, 30, 1091, 49, 422, 2733, 308, 386, 70, 2354, 2010, 1892, 2249, 632, 1, 71, 110, 9, 2349, 382, 317, 450, 599, 236, 198, 139, 1923, 1798, 42, 1501, 1099, 364, 359, 660, 1289, 899, 1088, 571, 715, 165, 752, 154, 208, 1464, 53, 55, 400] payload_rows_per_dst=[11278, 8798, 12122, 8594, 7070, 10104, 8758, 9232] dup_rows_per_dst=[3192, 2950, 4492, 1542, 1504, 3768, 3132, 1768] combine_rows_per_src=[11278, 8798, 12122, 8594, 7070, 10104, 8758, 9232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75684 payload_rows_remote=66024 dup_rows=22620 dup_rows_remote=19538 dedup_ratio=0.230103 router_tokens_per_expert=[1023, 876, 1152, 364, 576, 558, 786, 613, 95, 714, 814, 650, 534, 1257, 1913, 206, 276, 3434, 68, 964, 37, 817, 2544, 259, 400, 290, 2158, 1519, 1268, 2031, 735, 1, 64, 159, 21, 1682, 327, 376, 555, 659, 273, 184, 232, 1158, 2240, 22, 1005, 1049, 323, 414, 783, 1298, 1051, 840, 512, 763, 153, 982, 77, 261, 2130, 84, 110, 433] payload_rows_per_dst=[9628, 9002, 10272, 9660, 8500, 9718, 8762, 10142] dup_rows_per_dst=[2268, 3364, 2980, 3082, 2732, 2608, 3206, 2380] combine_rows_per_src=[9628, 9002, 10272, 9660, 8500, 9718, 8762, 10142] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75224 payload_rows_remote=66466 dup_rows=23080 dup_rows_remote=20362 dedup_ratio=0.234782 router_tokens_per_expert=[213, 983, 935, 29, 639, 1007, 88, 4140, 488, 147, 152, 1895, 321, 933, 1309, 493, 3356, 2557, 59, 225, 1430, 43, 1811, 46, 146, 56, 700, 950, 1, 501, 1314, 69, 771, 26, 409, 686, 31, 93, 3567, 2341, 1582, 127, 17, 676, 3, 603, 842, 159, 170, 543, 1820, 2672, 451, 699, 108, 591, 96, 688, 888, 44, 34, 599, 17, 763] payload_rows_per_dst=[13664, 8758, 10056, 10996, 7392, 7054, 7686, 9618] dup_rows_per_dst=[7748, 2718, 3108, 3612, 1322, 964, 1078, 2530] combine_rows_per_src=[13664, 8758, 10056, 10996, 7392, 7054, 7686, 9618] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75016 payload_rows_remote=63248 dup_rows=23288 dup_rows_remote=19176 dedup_ratio=0.236898 router_tokens_per_expert=[462, 522, 1135, 413, 773, 532, 547, 1091, 143, 523, 916, 553, 257, 1323, 2148, 193, 255, 3925, 41, 956, 32, 427, 3432, 270, 521, 353, 2465, 1645, 1623, 2153, 594, 0, 43, 112, 6, 2216, 419, 424, 532, 424, 228, 130, 149, 1267, 2024, 28, 1156, 794, 333, 484, 641, 1678, 952, 518, 331, 706, 116, 1214, 53, 170, 1208, 85, 79, 409] payload_rows_per_dst=[11768, 9064, 13454, 7678, 6842, 8054, 8608, 9548] dup_rows_per_dst=[4112, 3374, 6560, 1794, 1510, 1834, 2678, 1426] combine_rows_per_src=[11768, 9064, 13454, 7678, 6842, 8054, 8608, 9548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76480 payload_rows_remote=66906 dup_rows=21824 dup_rows_remote=19416 dedup_ratio=0.222005 router_tokens_per_expert=[1663, 868, 948, 37, 53, 301, 1864, 1992, 140, 319, 619, 354, 912, 642, 286, 1765, 2300, 206, 833, 1072, 509, 1000, 144, 1445, 181, 319, 605, 336, 583, 1087, 2673, 582, 2039, 921, 182, 138, 669, 757, 137, 489, 1304, 2701, 486, 721, 246, 430, 550, 586, 1213, 735, 564, 336, 137, 1133, 286, 1922, 766, 76, 229, 398, 977, 649, 500, 237] payload_rows_per_dst=[9902, 8856, 9202, 9574, 10094, 10448, 9028, 9376] dup_rows_per_dst=[2224, 1968, 2206, 2408, 3896, 3600, 2634, 2888] combine_rows_per_src=[9902, 8856, 9202, 9574, 10094, 10448, 9028, 9376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75572 payload_rows_remote=65304 dup_rows=22732 dup_rows_remote=20090 dedup_ratio=0.231242 router_tokens_per_expert=[111, 821, 938, 75, 582, 972, 291, 3833, 563, 164, 194, 2259, 412, 1099, 1655, 510, 3384, 2074, 40, 271, 1654, 63, 1430, 62, 83, 132, 771, 760, 3, 461, 1708, 77, 713, 35, 212, 568, 22, 98, 3774, 2125, 1677, 176, 5, 627, 5, 468, 1137, 293, 296, 335, 1479, 2520, 639, 899, 120, 406, 147, 475, 747, 59, 69, 728, 9, 837] payload_rows_per_dst=[6930, 8254, 7360, 6838, 13106, 11878, 10938, 10268] dup_rows_per_dst=[650, 2182, 1568, 1152, 6136, 4564, 3838, 2642] combine_rows_per_src=[6930, 8254, 7360, 6838, 13106, 11878, 10938, 10268] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76782 payload_rows_remote=65172 dup_rows=21522 dup_rows_remote=17546 dedup_ratio=0.218933 router_tokens_per_expert=[249, 981, 692, 57, 496, 720, 166, 3890, 604, 182, 237, 2199, 288, 1099, 1198, 259, 3629, 3486, 41, 354, 1212, 38, 1627, 15, 113, 90, 563, 428, 0, 623, 1879, 55, 670, 24, 141, 881, 5, 89, 3399, 2586, 1702, 147, 2, 367, 4, 584, 871, 226, 117, 397, 1645, 2661, 648, 799, 41, 213, 140, 665, 1049, 34, 29, 636, 6, 904] payload_rows_per_dst=[6130, 9188, 10242, 6776, 11928, 11610, 10068, 10840] dup_rows_per_dst=[592, 2944, 3304, 726, 3662, 3976, 2974, 3344] combine_rows_per_src=[6130, 9188, 10242, 6776, 11928, 11610, 10068, 10840] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76288 payload_rows_remote=66450 dup_rows=22016 dup_rows_remote=18364 dedup_ratio=0.223958 router_tokens_per_expert=[165, 868, 136, 394, 262, 523, 231, 1665, 36, 1445, 284, 1841, 636, 1247, 382, 916, 2171, 511, 652, 981, 528, 318, 1373, 211, 553, 1525, 2811, 1256, 815, 465, 456, 82, 664, 706, 1668, 1154, 1737, 947, 1698, 513, 318, 879, 124, 154, 1580, 1738, 125, 448, 746, 1005, 762, 362, 561, 312, 177, 216, 1286, 620, 256, 375, 360, 757, 537, 628] payload_rows_per_dst=[10732, 10348, 9838, 8588, 10560, 8840, 9604, 7778] dup_rows_per_dst=[3378, 3226, 3652, 1716, 3380, 1892, 2074, 2698] combine_rows_per_src=[10732, 10348, 9838, 8588, 10560, 8840, 9604, 7778] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=270704640 dispatch_meta_bytes=340544 combine_payload_bytes=270704640 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=271045184 ingress_bytes=374513664 link_bytes=374513664 token_bytes=541749824 weight_movement_bytes=103809024 total_nvlink_bytes=645558848 weight_to_token_ratio=0.191618 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.409264767e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74438 payload_rows_remote=65208 dup_rows=23866 dup_rows_remote=20878 dedup_ratio=0.242778 router_tokens_per_expert=[2051, 689, 726, 68, 121, 286, 1857, 1489, 128, 238, 763, 308, 771, 631, 353, 1938, 2296, 348, 542, 1373, 433, 1053, 85, 1064, 278, 333, 837, 214, 766, 1229, 2536, 793, 1844, 781, 250, 140, 671, 665, 294, 362, 1269, 2851, 374, 520, 191, 452, 483, 505, 1071, 890, 1003, 352, 79, 964, 379, 2352, 861, 38, 186, 256, 883, 633, 692, 264] payload_rows_per_dst=[7854, 8506, 8106, 10566, 10222, 7680, 12274, 9230] dup_rows_per_dst=[1282, 1754, 1690, 3406, 3894, 1758, 7094, 2988] combine_rows_per_src=[7854, 8506, 8106, 10566, 10222, 7680, 12274, 9230] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=270516224 dispatch_meta_bytes=337792 combine_payload_bytes=270516224 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=270854016 ingress_bytes=339722240 link_bytes=339722240 token_bytes=541370240 weight_movement_bytes=69206016 total_nvlink_bytes=610576256 weight_to_token_ratio=0.127835 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.278347439e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=261464064 dispatch_meta_bytes=326760 combine_payload_bytes=261464064 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=261790824 ingress_bytes=399876096 link_bytes=399876096 token_bytes=523254888 weight_movement_bytes=138412032 total_nvlink_bytes=661666920 weight_to_token_ratio=0.264521 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.504701557e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=270434304 dispatch_meta_bytes=342248 combine_payload_bytes=270434304 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=270776552 ingress_bytes=408846336 link_bytes=408846336 token_bytes=541210856 weight_movement_bytes=138412032 total_nvlink_bytes=679622888 weight_to_token_ratio=0.255745 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=13952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.538455848e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=272244736 dispatch_meta_bytes=347312 combine_payload_bytes=272244736 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=272592048 ingress_bytes=341450752 link_bytes=341450752 token_bytes=544836784 weight_movement_bytes=69206016 total_nvlink_bytes=614042800 weight_to_token_ratio=0.127022 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=15616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.284851691e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75016 payload_rows_remote=63248 dup_rows=23288 dup_rows_remote=19176 dedup_ratio=0.236898 router_tokens_per_expert=[462, 522, 1135, 413, 773, 532, 547, 1091, 143, 523, 916, 553, 257, 1323, 2148, 193, 255, 3925, 41, 956, 32, 427, 3432, 270, 521, 353, 2465, 1645, 1623, 2153, 594, 0, 43, 112, 6, 2216, 419, 424, 532, 424, 228, 130, 149, 1267, 2024, 28, 1156, 794, 333, 484, 641, 1678, 952, 518, 331, 706, 116, 1214, 53, 170, 1208, 85, 79, 409] payload_rows_per_dst=[11768, 9064, 13454, 7678, 6842, 8054, 8608, 9548] dup_rows_per_dst=[4112, 3374, 6560, 1794, 1510, 1834, 2678, 1426] combine_rows_per_src=[11768, 9064, 13454, 7678, 6842, 8054, 8608, 9548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=259063808 dispatch_meta_bytes=329696 combine_payload_bytes=259063808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=259393504 ingress_bytes=293666816 link_bytes=293666816 token_bytes=518457312 weight_movement_bytes=34603008 total_nvlink_bytes=553060320 weight_to_token_ratio=0.066742 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.105044586e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=274046976 dispatch_meta_bytes=345288 combine_payload_bytes=274046976 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=274392264 ingress_bytes=343252992 link_bytes=343252992 token_bytes=548439240 weight_movement_bytes=69206016 total_nvlink_bytes=617645256 weight_to_token_ratio=0.126187 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.291633375e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75958 payload_rows_remote=64204 dup_rows=22346 dup_rows_remote=18220 dedup_ratio=0.227315 router_tokens_per_expert=[391, 628, 1289, 445, 355, 645, 437, 1027, 174, 929, 779, 493, 332, 1157, 1893, 128, 233, 4099, 37, 842, 60, 584, 2593, 463, 475, 113, 2341, 2077, 1335, 2307, 524, 2, 36, 107, 23, 1980, 335, 404, 402, 554, 128, 126, 265, 1533, 2027, 29, 1327, 1156, 353, 363, 835, 1146, 1115, 749, 369, 584, 95, 1486, 119, 240, 1281, 94, 76, 628] payload_rows_per_dst=[8700, 8614, 8078, 10662, 11754, 9816, 8204, 10130] dup_rows_per_dst=[1734, 3156, 1546, 3072, 4126, 3366, 2824, 2522] combine_rows_per_src=[8700, 8614, 8078, 10662, 11754, 9816, 8204, 10130] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267485184 dispatch_meta_bytes=341576 combine_payload_bytes=267485184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267826760 ingress_bytes=302088192 link_bytes=302088192 token_bytes=535311944 weight_movement_bytes=34603008 total_nvlink_bytes=569914952 weight_to_token_ratio=0.064641 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=23168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136733546e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266944512 dispatch_meta_bytes=330872 combine_payload_bytes=266944512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267275384 ingress_bytes=301547520 link_bytes=301547520 token_bytes=534219896 weight_movement_bytes=34603008 total_nvlink_bytes=568822904 weight_to_token_ratio=0.064773 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134699041e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=272179200 dispatch_meta_bytes=339256 combine_payload_bytes=272179200 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=272518456 ingress_bytes=341385216 link_bytes=341385216 token_bytes=544697656 weight_movement_bytes=69206016 total_nvlink_bytes=613903672 weight_to_token_ratio=0.127054 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=13312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.284605084e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267091968 dispatch_meta_bytes=344344 combine_payload_bytes=267091968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267436312 ingress_bytes=301694976 link_bytes=301694976 token_bytes=534528280 weight_movement_bytes=34603008 total_nvlink_bytes=569131288 weight_to_token_ratio=0.064736 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135253906e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=259063808 dispatch_meta_bytes=329696 combine_payload_bytes=259063808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=259393504 ingress_bytes=293666816 link_bytes=293666816 token_bytes=518457312 weight_movement_bytes=34603008 total_nvlink_bytes=553060320 weight_to_token_ratio=0.066742 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.105044586e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76450 payload_rows_remote=67494 dup_rows=21854 dup_rows_remote=18872 dedup_ratio=0.222310 router_tokens_per_expert=[352, 654, 1390, 567, 381, 399, 359, 1181, 126, 849, 673, 485, 326, 1203, 2190, 117, 244, 4238, 27, 993, 55, 508, 2819, 301, 385, 376, 2348, 1941, 1292, 2555, 760, 0, 46, 93, 11, 2028, 282, 326, 534, 469, 169, 122, 210, 1297, 2144, 20, 1102, 670, 274, 367, 651, 1762, 1057, 494, 494, 639, 67, 1520, 71, 391, 1340, 30, 45, 333] payload_rows_per_dst=[11514, 8956, 12988, 7760, 6378, 9500, 8800, 10554] dup_rows_per_dst=[3748, 2982, 5382, 1748, 1200, 1968, 2676, 2150] combine_rows_per_src=[11514, 8956, 12988, 7760, 6378, 9500, 8800, 10554] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=262979584 dispatch_meta_bytes=329696 combine_payload_bytes=262979584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263309280 ingress_bytes=297582592 link_bytes=297582592 token_bytes=526288864 weight_movement_bytes=34603008 total_nvlink_bytes=560891872 weight_to_token_ratio=0.065749 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.119779336e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77448 payload_rows_remote=67358 dup_rows=20856 dup_rows_remote=18296 dedup_ratio=0.212158 router_tokens_per_expert=[264, 898, 850, 67, 374, 854, 283, 3892, 511, 163, 102, 1827, 368, 1011, 1382, 788, 3558, 2079, 54, 406, 2109, 52, 1489, 136, 83, 81, 820, 743, 11, 626, 1286, 134, 815, 121, 349, 519, 19, 155, 3579, 1920, 1674, 124, 28, 604, 31, 703, 935, 544, 397, 430, 1274, 2910, 290, 785, 135, 568, 78, 699, 634, 58, 34, 822, 26, 591] payload_rows_per_dst=[11418, 9304, 10090, 11056, 6784, 7678, 10552, 10566] dup_rows_per_dst=[3644, 3000, 2560, 3670, 1012, 1608, 2928, 2434] combine_rows_per_src=[11418, 9304, 10090, 11056, 6784, 7678, 10552, 10566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77140 payload_rows_remote=67954 dup_rows=21164 dup_rows_remote=19254 dedup_ratio=0.215291 router_tokens_per_expert=[432, 346, 1473, 516, 309, 351, 396, 1193, 112, 1129, 871, 463, 210, 1198, 2297, 166, 219, 4086, 33, 871, 57, 757, 3178, 226, 558, 435, 2354, 1750, 1359, 2478, 648, 0, 55, 106, 8, 1652, 287, 392, 571, 604, 190, 155, 151, 1264, 2029, 16, 1136, 607, 240, 347, 629, 1325, 1063, 424, 548, 590, 87, 1394, 72, 97, 1956, 29, 41, 616] payload_rows_per_dst=[8408, 9506, 9456, 10292, 11422, 9186, 8110, 10760] dup_rows_per_dst=[1624, 3386, 1226, 3916, 4100, 1910, 2222, 2780] combine_rows_per_src=[8408, 9506, 9456, 10292, 11422, 9186, 8110, 10760] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74728 payload_rows_remote=66432 dup_rows=23576 dup_rows_remote=20762 dedup_ratio=0.239827 router_tokens_per_expert=[446, 750, 1004, 518, 624, 402, 418, 1101, 159, 715, 504, 551, 298, 1220, 2470, 164, 286, 3887, 41, 811, 56, 512, 2609, 281, 623, 312, 1890, 1954, 1393, 2460, 810, 0, 33, 155, 5, 2151, 357, 412, 490, 473, 236, 154, 150, 1320, 1681, 36, 1814, 1243, 247, 377, 897, 1229, 714, 674, 467, 925, 129, 1396, 44, 139, 1258, 85, 80, 512] payload_rows_per_dst=[8892, 9200, 4818, 12076, 11714, 9730, 8296, 10002] dup_rows_per_dst=[1634, 2962, 484, 5728, 4212, 3538, 2814, 2204] combine_rows_per_src=[8892, 9200, 4818, 12076, 11714, 9730, 8296, 10002] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77144 payload_rows_remote=66090 dup_rows=21160 dup_rows_remote=16268 dedup_ratio=0.215251 router_tokens_per_expert=[1193, 66, 313, 4387, 759, 3, 161, 885, 350, 1158, 797, 430, 4068, 1917, 21, 1693, 672, 56, 33, 930, 25, 590, 1247, 1173, 31, 515, 418, 1003, 448, 23, 45, 50, 30, 348, 1311, 323, 913, 6, 86, 467, 691, 5405, 29, 1919, 2875, 26, 1799, 228, 164, 729, 115, 709, 2548, 629, 842, 18, 238, 30, 29, 36, 54, 2, 1071, 22] payload_rows_per_dst=[11842, 11054, 7906, 8284, 11816, 6004, 9412, 10826] dup_rows_per_dst=[3692, 4892, 1546, 1704, 4740, 540, 2096, 1950] combine_rows_per_src=[11842, 11054, 7906, 8284, 11816, 6004, 9412, 10826] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77162 payload_rows_remote=65274 dup_rows=21142 dup_rows_remote=17468 dedup_ratio=0.215068 router_tokens_per_expert=[285, 1214, 630, 30, 491, 795, 217, 3428, 685, 145, 202, 2253, 433, 730, 1702, 447, 2953, 2665, 57, 180, 1523, 42, 1378, 46, 87, 54, 575, 833, 0, 653, 1662, 58, 652, 38, 388, 863, 9, 65, 3859, 2271, 1618, 157, 34, 300, 16, 510, 970, 261, 120, 214, 1916, 2586, 644, 936, 67, 505, 115, 754, 1179, 62, 61, 670, 7, 852] payload_rows_per_dst=[10044, 9854, 9816, 11888, 7340, 7778, 10376, 10066] dup_rows_per_dst=[2654, 3340, 1966, 3674, 1232, 1436, 3600, 3240] combine_rows_per_src=[10044, 9854, 9816, 11888, 7340, 7778, 10376, 10066] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=276455424 dispatch_meta_bytes=345464 combine_payload_bytes=276455424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276800888 ingress_bytes=311058432 link_bytes=311058432 token_bytes=553256312 weight_movement_bytes=34603008 total_nvlink_bytes=587859320 weight_to_token_ratio=0.062544 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.170487837e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75684 payload_rows_remote=66024 dup_rows=22620 dup_rows_remote=19538 dedup_ratio=0.230103 router_tokens_per_expert=[1023, 876, 1152, 364, 576, 558, 786, 613, 95, 714, 814, 650, 534, 1257, 1913, 206, 276, 3434, 68, 964, 37, 817, 2544, 259, 400, 290, 2158, 1519, 1268, 2031, 735, 1, 64, 159, 21, 1682, 327, 376, 555, 659, 273, 184, 232, 1158, 2240, 22, 1005, 1049, 323, 414, 783, 1298, 1051, 840, 512, 763, 153, 982, 77, 261, 2130, 84, 110, 433] payload_rows_per_dst=[9628, 9002, 10272, 9660, 8500, 9718, 8762, 10142] dup_rows_per_dst=[2268, 3364, 2980, 3082, 2732, 2608, 3206, 2380] combine_rows_per_src=[9628, 9002, 10272, 9660, 8500, 9718, 8762, 10142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73916 payload_rows_remote=60546 dup_rows=24388 dup_rows_remote=17550 dedup_ratio=0.248088 router_tokens_per_expert=[132, 897, 699, 65, 759, 749, 243, 4203, 453, 209, 157, 2241, 368, 584, 1786, 513, 3430, 2697, 42, 182, 1468, 61, 1829, 52, 133, 67, 652, 483, 0, 912, 1942, 41, 729, 41, 271, 685, 16, 99, 3597, 2496, 1630, 231, 19, 423, 24, 410, 869, 156, 191, 217, 1744, 2357, 440, 853, 109, 332, 144, 512, 1148, 106, 31, 612, 11, 600] payload_rows_per_dst=[13370, 9568, 12736, 11726, 7466, 6732, 6938, 5380] dup_rows_per_dst=[6838, 3054, 6786, 3928, 1208, 792, 834, 948] combine_rows_per_src=[13370, 9568, 12736, 11726, 7466, 6732, 6938, 5380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78932 payload_rows_remote=67704 dup_rows=19372 dup_rows_remote=17382 dedup_ratio=0.197062 router_tokens_per_expert=[1373, 72, 233, 4340, 1093, 4, 213, 1024, 516, 645, 883, 516, 3938, 1670, 59, 1583, 759, 28, 90, 1037, 26, 766, 1213, 931, 34, 776, 477, 752, 298, 26, 49, 31, 79, 365, 864, 249, 929, 2, 166, 313, 559, 5099, 19, 2649, 2380, 73, 2041, 265, 250, 832, 140, 845, 2130, 1189, 695, 54, 257, 76, 11, 22, 75, 6, 1042, 21] payload_rows_per_dst=[7084, 7524, 13216, 10344, 8786, 11070, 9680, 11228] dup_rows_per_dst=[940, 880, 5164, 2418, 1908, 3482, 2590, 1990] combine_rows_per_src=[7084, 7524, 13216, 10344, 8786, 11070, 9680, 11228] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80474 payload_rows_remote=68954 dup_rows=17830 dup_rows_remote=14704 dedup_ratio=0.181376 router_tokens_per_expert=[1660, 41, 90, 4853, 863, 5, 206, 1008, 499, 694, 775, 424, 4684, 2493, 7, 1476, 654, 14, 42, 936, 34, 530, 1292, 794, 21, 433, 280, 961, 600, 17, 42, 9, 11, 257, 908, 159, 644, 0, 29, 485, 849, 5603, 5, 2031, 2962, 7, 1691, 247, 85, 627, 173, 816, 2714, 798, 328, 7, 234, 36, 30, 17, 40, 1, 897, 24] payload_rows_per_dst=[6970, 7216, 12700, 11582, 9166, 11520, 9312, 12008] dup_rows_per_dst=[776, 946, 5186, 2512, 1744, 3126, 1784, 1756] combine_rows_per_src=[6970, 7216, 12700, 11582, 9166, 11520, 9312, 12008] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=275898368 dispatch_meta_bytes=342616 combine_payload_bytes=275898368 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=276240984 ingress_bytes=379707392 link_bytes=379707392 token_bytes=552139352 weight_movement_bytes=103809024 total_nvlink_bytes=655948376 weight_to_token_ratio=0.188012 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.428808347e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278339584 dispatch_meta_bytes=348832 combine_payload_bytes=278339584 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=278688416 ingress_bytes=347545600 link_bytes=347545600 token_bytes=557028000 weight_movement_bytes=69206016 total_nvlink_bytes=626234016 weight_to_token_ratio=0.124242 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=15104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.307786113e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=272105472 dispatch_meta_bytes=348776 combine_payload_bytes=272105472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272454248 ingress_bytes=306708480 link_bytes=306708480 token_bytes=544559720 weight_movement_bytes=34603008 total_nvlink_bytes=579162728 weight_to_token_ratio=0.063543 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.154119318e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=270704640 dispatch_meta_bytes=329432 combine_payload_bytes=270704640 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=271034072 ingress_bytes=374513664 link_bytes=374513664 token_bytes=541738712 weight_movement_bytes=103809024 total_nvlink_bytes=645547736 weight_to_token_ratio=0.191622 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=12160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.409264767e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73916 payload_rows_remote=60546 dup_rows=24388 dup_rows_remote=17550 dedup_ratio=0.248088 router_tokens_per_expert=[132, 897, 699, 65, 759, 749, 243, 4203, 453, 209, 157, 2241, 368, 584, 1786, 513, 3430, 2697, 42, 182, 1468, 61, 1829, 52, 133, 67, 652, 483, 0, 912, 1942, 41, 729, 41, 271, 685, 16, 99, 3597, 2496, 1630, 231, 19, 423, 24, 410, 869, 156, 191, 217, 1744, 2357, 440, 853, 109, 332, 144, 512, 1148, 106, 31, 612, 11, 600] payload_rows_per_dst=[13370, 9568, 12736, 11726, 7466, 6732, 6938, 5380] dup_rows_per_dst=[6838, 3054, 6786, 3928, 1208, 792, 834, 948] combine_rows_per_src=[13370, 9568, 12736, 11726, 7466, 6732, 6938, 5380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75554 payload_rows_remote=64782 dup_rows=22750 dup_rows_remote=18824 dedup_ratio=0.231425 router_tokens_per_expert=[1888, 805, 797, 48, 106, 293, 1803, 1881, 118, 292, 483, 323, 1108, 654, 525, 1518, 2594, 303, 554, 1387, 448, 1274, 101, 1178, 194, 423, 778, 239, 796, 1289, 2640, 911, 1412, 883, 222, 219, 632, 575, 249, 649, 1302, 2217, 252, 655, 335, 371, 459, 506, 1145, 823, 593, 230, 147, 1104, 311, 2104, 812, 81, 190, 327, 1078, 664, 550, 304] payload_rows_per_dst=[10836, 10992, 10772, 7712, 8608, 9620, 7462, 9552] dup_rows_per_dst=[3310, 4330, 3926, 1548, 2170, 2574, 1244, 3648] combine_rows_per_src=[10836, 10992, 10772, 7712, 8608, 9620, 7462, 9552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=270434304 dispatch_meta_bytes=342248 combine_payload_bytes=270434304 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=270776552 ingress_bytes=408846336 link_bytes=408846336 token_bytes=541210856 weight_movement_bytes=138412032 total_nvlink_bytes=679622888 weight_to_token_ratio=0.255745 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=13952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.538455848e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=247996416 dispatch_meta_bytes=312384 combine_payload_bytes=247996416 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=248308800 ingress_bytes=317202432 link_bytes=317202432 token_bytes=496305216 weight_movement_bytes=69206016 total_nvlink_bytes=565511232 weight_to_token_ratio=0.139442 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.193607215e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75030 payload_rows_remote=65036 dup_rows=23274 dup_rows_remote=20562 dedup_ratio=0.236755 router_tokens_per_expert=[489, 570, 1209, 302, 543, 388, 418, 983, 117, 956, 730, 488, 313, 1320, 2421, 150, 208, 3738, 26, 1022, 60, 449, 3816, 458, 452, 84, 2225, 1817, 1093, 2158, 519, 0, 45, 147, 10, 1948, 314, 429, 444, 545, 200, 155, 134, 1568, 1925, 34, 1706, 880, 319, 297, 514, 1737, 1019, 583, 567, 755, 226, 1077, 78, 181, 1205, 30, 101, 457] payload_rows_per_dst=[8368, 9536, 4142, 8642, 11542, 9962, 12844, 9994] dup_rows_per_dst=[1436, 3454, 304, 2058, 3698, 3242, 6370, 2712] combine_rows_per_src=[8368, 9536, 4142, 8642, 11542, 9962, 12844, 9994] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77222 payload_rows_remote=69356 dup_rows=21082 dup_rows_remote=19642 dedup_ratio=0.214457 router_tokens_per_expert=[209, 998, 1020, 37, 606, 926, 231, 4048, 296, 112, 250, 2317, 277, 1493, 1476, 473, 3455, 2155, 34, 141, 1047, 61, 1791, 72, 88, 73, 614, 480, 8, 797, 1522, 49, 840, 38, 254, 510, 27, 110, 3679, 2508, 1610, 120, 30, 360, 16, 375, 763, 301, 163, 659, 1622, 2453, 448, 772, 73, 415, 137, 639, 1223, 54, 43, 715, 10, 1029] payload_rows_per_dst=[7242, 9694, 8708, 10908, 7866, 11460, 10306, 11038] dup_rows_per_dst=[812, 3694, 1894, 2980, 1440, 3786, 2904, 3572] combine_rows_per_src=[7242, 9694, 8708, 10908, 7866, 11460, 10306, 11038] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267362304 dispatch_meta_bytes=330968 combine_payload_bytes=267362304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267693272 ingress_bytes=301965312 link_bytes=301965312 token_bytes=535055576 weight_movement_bytes=34603008 total_nvlink_bytes=569658584 weight_to_token_ratio=0.064672 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136271159e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277315584 dispatch_meta_bytes=340344 combine_payload_bytes=277315584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277655928 ingress_bytes=311918592 link_bytes=311918592 token_bytes=554971512 weight_movement_bytes=34603008 total_nvlink_bytes=589574520 weight_to_token_ratio=0.062351 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=38784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.173724550e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=282435584 dispatch_meta_bytes=334632 combine_payload_bytes=282435584 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=282770216 ingress_bytes=420847616 link_bytes=420847616 token_bytes=565205800 weight_movement_bytes=138412032 total_nvlink_bytes=703617832 weight_to_token_ratio=0.244888 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.583615698e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265347072 dispatch_meta_bytes=334424 combine_payload_bytes=265347072 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265681496 ingress_bytes=334553088 link_bytes=334553088 token_bytes=531028568 weight_movement_bytes=69206016 total_nvlink_bytes=600234584 weight_to_token_ratio=0.130324 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258896336e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75224 payload_rows_remote=66466 dup_rows=23080 dup_rows_remote=20362 dedup_ratio=0.234782 router_tokens_per_expert=[213, 983, 935, 29, 639, 1007, 88, 4140, 488, 147, 152, 1895, 321, 933, 1309, 493, 3356, 2557, 59, 225, 1430, 43, 1811, 46, 146, 56, 700, 950, 1, 501, 1314, 69, 771, 26, 409, 686, 31, 93, 3567, 2341, 1582, 127, 17, 676, 3, 603, 842, 159, 170, 543, 1820, 2672, 451, 699, 108, 591, 96, 688, 888, 44, 34, 599, 17, 763] payload_rows_per_dst=[13664, 8758, 10056, 10996, 7392, 7054, 7686, 9618] dup_rows_per_dst=[7748, 2718, 3108, 3612, 1322, 964, 1078, 2530] combine_rows_per_src=[13664, 8758, 10056, 10996, 7392, 7054, 7686, 9618] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266387456 dispatch_meta_bytes=342392 combine_payload_bytes=266387456 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=266729848 ingress_bytes=335593472 link_bytes=335593472 token_bytes=533117304 weight_movement_bytes=69206016 total_nvlink_bytes=602323320 weight_to_token_ratio=0.129814 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=18432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262811218e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=284082176 dispatch_meta_bytes=355992 combine_payload_bytes=284082176 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=284438168 ingress_bytes=353288192 link_bytes=353288192 token_bytes=568520344 weight_movement_bytes=69206016 total_nvlink_bytes=637726360 weight_to_token_ratio=0.121730 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=19072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.329395024e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=247996416 dispatch_meta_bytes=312384 combine_payload_bytes=247996416 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=248308800 ingress_bytes=317202432 link_bytes=317202432 token_bytes=496305216 weight_movement_bytes=69206016 total_nvlink_bytes=565511232 weight_to_token_ratio=0.139442 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.193607215e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74904 payload_rows_remote=63212 dup_rows=23400 dup_rows_remote=19532 dedup_ratio=0.238037 router_tokens_per_expert=[1727, 711, 60, 58, 1743, 55, 202, 248, 14, 272, 5733, 32, 38, 1527, 783, 1574, 1770, 78, 150, 86, 230, 647, 12, 675, 249, 1192, 627, 578, 270, 7173, 1847, 12, 492, 46, 1267, 392, 2018, 257, 44, 4, 1093, 918, 0, 140, 99, 812, 105, 18, 2484, 271, 11, 43, 151, 380, 2442, 1311, 93, 720, 16, 968, 10, 486, 1167, 521] payload_rows_per_dst=[10084, 11692, 6534, 15380, 10838, 5406, 8098, 6872] dup_rows_per_dst=[3910, 3868, 762, 8516, 3086, 964, 1204, 1090] combine_rows_per_src=[10084, 11692, 6534, 15380, 10838, 5406, 8098, 6872] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75042 payload_rows_remote=64602 dup_rows=23262 dup_rows_remote=20418 dedup_ratio=0.236633 router_tokens_per_expert=[277, 894, 748, 55, 443, 704, 163, 4166, 517, 126, 241, 2190, 390, 1157, 1172, 583, 3225, 2412, 53, 245, 1205, 25, 1744, 50, 93, 42, 730, 840, 3, 898, 1543, 38, 699, 62, 367, 783, 14, 47, 3765, 2789, 1921, 158, 22, 510, 14, 443, 1088, 172, 149, 630, 1826, 2156, 655, 861, 50, 315, 143, 461, 728, 69, 59, 574, 19, 631] payload_rows_per_dst=[5942, 9234, 7052, 6998, 13506, 12122, 10440, 9748] dup_rows_per_dst=[626, 3518, 1058, 1376, 6904, 4866, 2844, 2070] combine_rows_per_src=[5942, 9234, 7052, 6998, 13506, 12122, 10440, 9748] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79142 payload_rows_remote=71626 dup_rows=19162 dup_rows_remote=17574 dedup_ratio=0.194926 router_tokens_per_expert=[1361, 99, 334, 3704, 1109, 27, 151, 980, 303, 1103, 852, 372, 3924, 1772, 33, 1843, 618, 62, 126, 866, 32, 651, 1080, 1117, 15, 544, 493, 1004, 382, 60, 40, 63, 60, 212, 830, 477, 829, 4, 114, 977, 754, 6016, 13, 2535, 2678, 29, 1407, 281, 145, 659, 224, 801, 2147, 800, 334, 33, 305, 73, 3, 16, 70, 3, 1165, 38] payload_rows_per_dst=[11930, 9496, 7516, 10528, 10578, 7686, 8528, 12880] dup_rows_per_dst=[3600, 3060, 1588, 2522, 2994, 1142, 1758, 2498] combine_rows_per_src=[11930, 9496, 7516, 10528, 10578, 7686, 8528, 12880] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79708 payload_rows_remote=69356 dup_rows=18596 dup_rows_remote=16838 dedup_ratio=0.189168 router_tokens_per_expert=[1173, 56, 214, 4705, 281, 3, 125, 1160, 555, 799, 1014, 346, 4109, 1783, 33, 1348, 640, 10, 20, 903, 34, 353, 1352, 726, 15, 444, 230, 707, 314, 109, 21, 106, 20, 321, 1073, 402, 812, 1, 144, 465, 411, 5377, 29, 2235, 2192, 25, 2350, 299, 243, 1702, 148, 960, 2553, 1810, 888, 65, 253, 49, 19, 20, 28, 2, 530, 38] payload_rows_per_dst=[11858, 9018, 6854, 10352, 8986, 9322, 11806, 11512] dup_rows_per_dst=[3576, 2162, 1222, 1758, 1874, 1952, 4932, 1120] combine_rows_per_src=[11858, 9018, 6854, 10352, 8986, 9322, 11806, 11512] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=59798 dup_rows=24098 dup_rows_remote=17732 dedup_ratio=0.245138 router_tokens_per_expert=[1845, 624, 18, 54, 2480, 41, 466, 149, 41, 370, 4656, 50, 141, 1509, 848, 974, 1938, 51, 105, 104, 155, 753, 6, 655, 266, 1449, 621, 682, 271, 6551, 2393, 4, 512, 56, 1509, 302, 1873, 251, 189, 0, 1360, 571, 3, 70, 120, 565, 120, 15, 2350, 290, 19, 135, 324, 308, 3263, 841, 127, 713, 18, 555, 12, 760, 1122, 529] payload_rows_per_dst=[8568, 6614, 12302, 5736, 8216, 5156, 13206, 14408] dup_rows_per_dst=[2786, 1252, 4544, 850, 1168, 492, 6640, 6366] combine_rows_per_src=[8568, 6614, 12302, 5736, 8216, 5156, 13206, 14408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76782 payload_rows_remote=65172 dup_rows=21522 dup_rows_remote=17546 dedup_ratio=0.218933 router_tokens_per_expert=[249, 981, 692, 57, 496, 720, 166, 3890, 604, 182, 237, 2199, 288, 1099, 1198, 259, 3629, 3486, 41, 354, 1212, 38, 1627, 15, 113, 90, 563, 428, 0, 623, 1879, 55, 670, 24, 141, 881, 5, 89, 3399, 2586, 1702, 147, 2, 367, 4, 584, 871, 226, 117, 397, 1645, 2661, 648, 799, 41, 213, 140, 665, 1049, 34, 29, 636, 6, 904] payload_rows_per_dst=[6130, 9188, 10242, 6776, 11928, 11610, 10068, 10840] dup_rows_per_dst=[592, 2944, 3304, 726, 3662, 3976, 2974, 3344] combine_rows_per_src=[6130, 9188, 10242, 6776, 11928, 11610, 10068, 10840] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=272244736 dispatch_meta_bytes=347312 combine_payload_bytes=272244736 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=272592048 ingress_bytes=341450752 link_bytes=341450752 token_bytes=544836784 weight_movement_bytes=69206016 total_nvlink_bytes=614042800 weight_to_token_ratio=0.127022 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=15616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.284851691e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72932 payload_rows_remote=60382 dup_rows=25372 dup_rows_remote=19796 dedup_ratio=0.258097 router_tokens_per_expert=[1440, 91, 171, 4561, 809, 27, 119, 888, 217, 619, 891, 602, 4550, 2557, 48, 1793, 699, 25, 33, 1438, 50, 381, 986, 800, 43, 757, 606, 1252, 480, 62, 65, 51, 41, 386, 885, 86, 765, 2, 72, 355, 285, 4913, 14, 2351, 2080, 50, 1788, 267, 123, 867, 57, 627, 1902, 2357, 774, 26, 192, 147, 10, 66, 52, 2, 485, 14] payload_rows_per_dst=[12550, 13680, 7322, 5764, 10760, 11788, 9176, 1892] dup_rows_per_dst=[5576, 8874, 1502, 868, 3286, 2846, 2376, 44] combine_rows_per_src=[12550, 13680, 7322, 5764, 10760, 11788, 9176, 1892] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75732 payload_rows_remote=65364 dup_rows=22572 dup_rows_remote=19480 dedup_ratio=0.229614 router_tokens_per_expert=[1265, 727, 74, 27, 2706, 18, 258, 273, 18, 191, 5136, 44, 147, 1654, 693, 1309, 2687, 79, 58, 97, 285, 305, 7, 571, 311, 922, 387, 718, 332, 7252, 2087, 16, 361, 55, 1658, 427, 1780, 228, 72, 1, 1356, 711, 0, 40, 97, 757, 121, 8, 2068, 343, 6, 129, 261, 319, 3047, 1200, 161, 513, 11, 557, 2, 625, 1246, 338] payload_rows_per_dst=[8168, 6810, 13140, 6400, 10746, 10368, 9448, 10652] dup_rows_per_dst=[2528, 1302, 5310, 1150, 3260, 3092, 2452, 3478] combine_rows_per_src=[8168, 6810, 13140, 6400, 10746, 10368, 9448, 10652] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=258916352 dispatch_meta_bytes=330976 combine_payload_bytes=258916352 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=259247328 ingress_bytes=362725376 link_bytes=362725376 token_bytes=518163680 weight_movement_bytes=103809024 total_nvlink_bytes=621972704 weight_to_token_ratio=0.200340 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.364906388e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77162 payload_rows_remote=65274 dup_rows=21142 dup_rows_remote=17468 dedup_ratio=0.215068 router_tokens_per_expert=[285, 1214, 630, 30, 491, 795, 217, 3428, 685, 145, 202, 2253, 433, 730, 1702, 447, 2953, 2665, 57, 180, 1523, 42, 1378, 46, 87, 54, 575, 833, 0, 653, 1662, 58, 652, 38, 388, 863, 9, 65, 3859, 2271, 1618, 157, 34, 300, 16, 510, 970, 261, 120, 214, 1916, 2586, 644, 936, 67, 505, 115, 754, 1179, 62, 61, 670, 7, 852] payload_rows_per_dst=[10044, 9854, 9816, 11888, 7340, 7778, 10376, 10066] dup_rows_per_dst=[2654, 3340, 1966, 3674, 1232, 1436, 3600, 3240] combine_rows_per_src=[10044, 9854, 9816, 11888, 7340, 7778, 10376, 10066] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=264609792 dispatch_meta_bytes=340080 combine_payload_bytes=264609792 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=264949872 ingress_bytes=333815808 link_bytes=333815808 token_bytes=529559664 weight_movement_bytes=69206016 total_nvlink_bytes=598765680 weight_to_token_ratio=0.130686 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256122011e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=293380096 dispatch_meta_bytes=356800 combine_payload_bytes=293380096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293736896 ingress_bytes=327983104 link_bytes=327983104 token_bytes=587116992 weight_movement_bytes=34603008 total_nvlink_bytes=621720000 weight_to_token_ratio=0.058937 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=12800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.234174016e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266944512 dispatch_meta_bytes=330872 combine_payload_bytes=266944512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267275384 ingress_bytes=301547520 link_bytes=301547520 token_bytes=534219896 weight_movement_bytes=34603008 total_nvlink_bytes=568822904 weight_to_token_ratio=0.064773 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134699041e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=247324672 dispatch_meta_bytes=320712 combine_payload_bytes=247324672 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=247645384 ingress_bytes=351133696 link_bytes=351133696 token_bytes=494970056 weight_movement_bytes=103809024 total_nvlink_bytes=598779080 weight_to_token_ratio=0.209728 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.321287829e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75956 payload_rows_remote=63834 dup_rows=22348 dup_rows_remote=17856 dedup_ratio=0.227336 router_tokens_per_expert=[390, 565, 1247, 429, 372, 477, 550, 928, 127, 903, 728, 457, 293, 1423, 1797, 146, 230, 3444, 30, 1091, 49, 422, 2733, 308, 386, 70, 2354, 2010, 1892, 2249, 632, 1, 71, 110, 9, 2349, 382, 317, 450, 599, 236, 198, 139, 1923, 1798, 42, 1501, 1099, 364, 359, 660, 1289, 899, 1088, 571, 715, 165, 752, 154, 208, 1464, 53, 55, 400] payload_rows_per_dst=[11278, 8798, 12122, 8594, 7070, 10104, 8758, 9232] dup_rows_per_dst=[3192, 2950, 4492, 1542, 1504, 3768, 3132, 1768] combine_rows_per_src=[11278, 8798, 12122, 8594, 7070, 10104, 8758, 9232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75572 payload_rows_remote=65304 dup_rows=22732 dup_rows_remote=20090 dedup_ratio=0.231242 router_tokens_per_expert=[111, 821, 938, 75, 582, 972, 291, 3833, 563, 164, 194, 2259, 412, 1099, 1655, 510, 3384, 2074, 40, 271, 1654, 63, 1430, 62, 83, 132, 771, 760, 3, 461, 1708, 77, 713, 35, 212, 568, 22, 98, 3774, 2125, 1677, 176, 5, 627, 5, 468, 1137, 293, 296, 335, 1479, 2520, 639, 899, 120, 406, 147, 475, 747, 59, 69, 728, 9, 837] payload_rows_per_dst=[6930, 8254, 7360, 6838, 13106, 11878, 10938, 10268] dup_rows_per_dst=[650, 2182, 1568, 1152, 6136, 4564, 3838, 2642] combine_rows_per_src=[6930, 8254, 7360, 6838, 13106, 11878, 10938, 10268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267362304 dispatch_meta_bytes=330968 combine_payload_bytes=267362304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267693272 ingress_bytes=301965312 link_bytes=301965312 token_bytes=535055576 weight_movement_bytes=34603008 total_nvlink_bytes=569658584 weight_to_token_ratio=0.064672 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136271159e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78410 payload_rows_remote=70082 dup_rows=19894 dup_rows_remote=18304 dedup_ratio=0.202372 router_tokens_per_expert=[1338, 73, 159, 4215, 693, 11, 154, 1228, 350, 635, 839, 380, 5028, 2209, 16, 1427, 710, 27, 15, 1435, 53, 954, 766, 942, 17, 752, 453, 859, 278, 30, 34, 36, 32, 390, 815, 160, 701, 0, 98, 441, 587, 6091, 18, 2026, 2322, 24, 2111, 242, 133, 349, 88, 962, 1803, 1831, 252, 30, 202, 117, 13, 46, 65, 2, 1071, 14] payload_rows_per_dst=[8078, 9872, 7752, 11960, 8328, 10826, 8630, 12964] dup_rows_per_dst=[1648, 3438, 2052, 3014, 1590, 3608, 2266, 2278] combine_rows_per_src=[8078, 9872, 7752, 11960, 8328, 10826, 8630, 12964] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=284082176 dispatch_meta_bytes=344776 combine_payload_bytes=284082176 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=284426952 ingress_bytes=318685184 link_bytes=318685184 token_bytes=568509128 weight_movement_bytes=34603008 total_nvlink_bytes=603112136 weight_to_token_ratio=0.060866 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=29056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.199186691e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72932 payload_rows_remote=60382 dup_rows=25372 dup_rows_remote=19796 dedup_ratio=0.258097 router_tokens_per_expert=[1440, 91, 171, 4561, 809, 27, 119, 888, 217, 619, 891, 602, 4550, 2557, 48, 1793, 699, 25, 33, 1438, 50, 381, 986, 800, 43, 757, 606, 1252, 480, 62, 65, 51, 41, 386, 885, 86, 765, 2, 72, 355, 285, 4913, 14, 2351, 2080, 50, 1788, 267, 123, 867, 57, 627, 1902, 2357, 774, 26, 192, 147, 10, 66, 52, 2, 485, 14] payload_rows_per_dst=[12550, 13680, 7322, 5764, 10760, 11788, 9176, 1892] dup_rows_per_dst=[5576, 8874, 1502, 868, 3286, 2846, 2376, 44] combine_rows_per_src=[12550, 13680, 7322, 5764, 10760, 11788, 9176, 1892] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=244932608 dispatch_meta_bytes=310120 combine_payload_bytes=244932608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=245242728 ingress_bytes=279535616 link_bytes=279535616 token_bytes=490175336 weight_movement_bytes=34603008 total_nvlink_bytes=524778344 weight_to_token_ratio=0.070593 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.051870019e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=267730944 dispatch_meta_bytes=339376 combine_payload_bytes=267730944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268070320 ingress_bytes=302333952 link_bytes=302333952 token_bytes=535801264 weight_movement_bytes=34603008 total_nvlink_bytes=570404272 weight_to_token_ratio=0.064582 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137658321e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77144 payload_rows_remote=66090 dup_rows=21160 dup_rows_remote=16268 dedup_ratio=0.215251 router_tokens_per_expert=[1193, 66, 313, 4387, 759, 3, 161, 885, 350, 1158, 797, 430, 4068, 1917, 21, 1693, 672, 56, 33, 930, 25, 590, 1247, 1173, 31, 515, 418, 1003, 448, 23, 45, 50, 30, 348, 1311, 323, 913, 6, 86, 467, 691, 5405, 29, 1919, 2875, 26, 1799, 228, 164, 729, 115, 709, 2548, 629, 842, 18, 238, 30, 29, 36, 54, 2, 1071, 22] payload_rows_per_dst=[11842, 11054, 7906, 8284, 11816, 6004, 9412, 10826] dup_rows_per_dst=[3692, 4892, 1546, 1704, 4740, 540, 2096, 1950] combine_rows_per_src=[11842, 11054, 7906, 8284, 11816, 6004, 9412, 10826] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=261464064 dispatch_meta_bytes=326760 combine_payload_bytes=261464064 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=261790824 ingress_bytes=399876096 link_bytes=399876096 token_bytes=523254888 weight_movement_bytes=138412032 total_nvlink_bytes=661666920 weight_to_token_ratio=0.264521 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.504701557e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267485184 dispatch_meta_bytes=341576 combine_payload_bytes=267485184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267826760 ingress_bytes=302088192 link_bytes=302088192 token_bytes=535311944 weight_movement_bytes=34603008 total_nvlink_bytes=569914952 weight_to_token_ratio=0.064641 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=23168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136733546e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=287055872 dispatch_meta_bytes=353544 combine_payload_bytes=287055872 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=287409416 ingress_bytes=356261888 link_bytes=356261888 token_bytes=574465288 weight_movement_bytes=69206016 total_nvlink_bytes=643671304 weight_to_token_ratio=0.120470 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=19840 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.340584803e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=247324672 dispatch_meta_bytes=320712 combine_payload_bytes=247324672 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=247645384 ingress_bytes=351133696 link_bytes=351133696 token_bytes=494970056 weight_movement_bytes=103809024 total_nvlink_bytes=598779080 weight_to_token_ratio=0.209728 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.321287829e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80544 payload_rows_remote=69214 dup_rows=17760 dup_rows_remote=14950 dedup_ratio=0.180664 router_tokens_per_expert=[4101, 409, 463, 514, 26, 330, 750, 1910, 16, 0, 0, 1136, 2, 581, 2, 712, 16, 5, 975, 36, 450, 53, 56, 149, 925, 79, 2930, 0, 7, 4621, 861, 24, 84, 1073, 24, 1609, 212, 1262, 753, 0, 57, 3661, 75, 21, 3919, 191, 26, 1625, 180, 10, 17, 373, 708, 1033, 35, 2, 231, 1268, 370, 579, 0, 833, 5850, 932] payload_rows_per_dst=[12130, 11330, 12828, 8440, 8584, 9456, 10616, 7160] dup_rows_per_dst=[4876, 2810, 2352, 1212, 1450, 1856, 1938, 1266] combine_rows_per_src=[12130, 11330, 12828, 8440, 8584, 9456, 10616, 7160] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76344 payload_rows_remote=70294 dup_rows=21960 dup_rows_remote=21006 dedup_ratio=0.223389 router_tokens_per_expert=[1433, 709, 56, 50, 1499, 42, 362, 122, 39, 431, 4571, 17, 28, 1933, 755, 969, 1915, 137, 123, 111, 354, 406, 7, 449, 366, 1215, 254, 737, 402, 7236, 2595, 83, 508, 275, 1184, 319, 1625, 421, 278, 11, 2026, 780, 6, 30, 174, 353, 60, 31, 3088, 546, 35, 105, 227, 410, 2410, 905, 191, 766, 10, 446, 6, 513, 1415, 592] payload_rows_per_dst=[6648, 12950, 6050, 10932, 10872, 12976, 9278, 6638] dup_rows_per_dst=[1898, 4536, 954, 4534, 3190, 4254, 1354, 1240] combine_rows_per_src=[6648, 12950, 6050, 10932, 10872, 12976, 9278, 6638] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78924 payload_rows_remote=68438 dup_rows=19380 dup_rows_remote=15908 dedup_ratio=0.197144 router_tokens_per_expert=[1733, 39, 277, 3757, 681, 2, 246, 1114, 398, 634, 865, 298, 4796, 1635, 18, 883, 607, 18, 23, 1021, 30, 369, 1261, 1303, 10, 769, 428, 862, 339, 42, 55, 46, 23, 502, 1522, 166, 753, 0, 25, 579, 664, 5584, 10, 2028, 2304, 28, 2001, 330, 182, 1611, 172, 684, 2384, 1258, 652, 36, 225, 56, 9, 29, 16, 5, 736, 19] payload_rows_per_dst=[7014, 5656, 12388, 11708, 9454, 10352, 10486, 11866] dup_rows_per_dst=[1170, 536, 4390, 2986, 2294, 3040, 3472, 1492] combine_rows_per_src=[7014, 5656, 12388, 11708, 9454, 10352, 10486, 11866] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76652 payload_rows_remote=69734 dup_rows=21652 dup_rows_remote=20384 dedup_ratio=0.220256 router_tokens_per_expert=[1352, 545, 19, 60, 2648, 41, 372, 548, 60, 309, 4768, 43, 167, 1681, 820, 1127, 2070, 85, 162, 50, 271, 920, 13, 309, 151, 1069, 388, 513, 445, 7118, 1509, 18, 809, 35, 1135, 382, 1315, 261, 90, 17, 1424, 922, 0, 84, 134, 628, 255, 23, 2465, 276, 39, 78, 583, 259, 2306, 1628, 236, 808, 38, 479, 12, 617, 1484, 679] payload_rows_per_dst=[8250, 13334, 6572, 6918, 9796, 15334, 9232, 7216] dup_rows_per_dst=[2920, 4616, 1188, 1268, 2904, 5842, 1424, 1490] combine_rows_per_src=[8250, 13334, 6572, 6918, 9796, 15334, 9232, 7216] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78010 payload_rows_remote=68660 dup_rows=20294 dup_rows_remote=18624 dedup_ratio=0.206441 router_tokens_per_expert=[4315, 279, 116, 616, 2, 671, 624, 1769, 41, 0, 0, 1847, 2, 1897, 0, 801, 33, 3, 1607, 8, 268, 50, 53, 113, 999, 40, 2354, 6, 4, 4221, 469, 25, 87, 922, 22, 2145, 22, 445, 754, 0, 50, 3617, 36, 7, 4159, 262, 6, 1532, 138, 15, 19, 218, 213, 832, 6, 2, 105, 2026, 416, 574, 1, 986, 5734, 568] payload_rows_per_dst=[7090, 8374, 12346, 11488, 11734, 9350, 9684, 7944] dup_rows_per_dst=[1064, 1844, 2706, 3706, 5690, 1670, 1520, 2094] combine_rows_per_src=[7090, 8374, 12346, 11488, 11734, 9350, 9684, 7944] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=270704640 dispatch_meta_bytes=329432 combine_payload_bytes=270704640 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=271034072 ingress_bytes=374513664 link_bytes=374513664 token_bytes=541738712 weight_movement_bytes=103809024 total_nvlink_bytes=645547736 weight_to_token_ratio=0.191622 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=12160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.409264767e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74832 payload_rows_remote=61372 dup_rows=23472 dup_rows_remote=17326 dedup_ratio=0.238770 router_tokens_per_expert=[1767, 474, 28, 55, 1389, 31, 602, 147, 16, 294, 5310, 20, 126, 1374, 1082, 1299, 3180, 78, 74, 119, 198, 649, 9, 262, 186, 1251, 304, 780, 234, 7069, 2143, 14, 481, 36, 1402, 359, 1348, 341, 149, 7, 1114, 1641, 2, 78, 47, 630, 74, 8, 2068, 438, 34, 33, 299, 269, 2708, 1121, 133, 860, 21, 525, 5, 627, 1332, 398] payload_rows_per_dst=[13460, 8398, 8038, 14260, 10480, 5880, 7638, 6678] dup_rows_per_dst=[6146, 2262, 1100, 7464, 3182, 1308, 886, 1124] combine_rows_per_src=[13460, 8398, 8038, 14260, 10480, 5880, 7638, 6678] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=283500544 dispatch_meta_bytes=336656 combine_payload_bytes=283500544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=283837200 ingress_bytes=318103552 link_bytes=318103552 token_bytes=567337744 weight_movement_bytes=34603008 total_nvlink_bytes=601940752 weight_to_token_ratio=0.060992 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.196998057e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72270 payload_rows_remote=57412 dup_rows=26034 dup_rows_remote=14946 dedup_ratio=0.264832 router_tokens_per_expert=[4273, 279, 531, 653, 4, 405, 679, 816, 12, 0, 0, 1620, 3, 1378, 3, 795, 14, 11, 1191, 10, 390, 91, 101, 92, 759, 194, 3026, 3, 5, 4076, 728, 35, 190, 1016, 70, 1494, 102, 1256, 893, 0, 45, 3230, 39, 25, 3616, 209, 16, 2052, 216, 18, 8, 451, 556, 770, 8, 4, 259, 1782, 516, 533, 2, 785, 5862, 952] payload_rows_per_dst=[6018, 6530, 3536, 12796, 8324, 8572, 11636, 14858] dup_rows_per_dst=[716, 1092, 264, 4856, 1718, 2670, 3630, 11088] combine_rows_per_src=[6018, 6530, 3536, 12796, 8324, 8572, 11636, 14858] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=287924224 dispatch_meta_bytes=365200 combine_payload_bytes=287924224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288289424 ingress_bytes=322527232 link_bytes=322527232 token_bytes=576213648 weight_movement_bytes=34603008 total_nvlink_bytes=610816656 weight_to_token_ratio=0.060052 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.213644008e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80474 payload_rows_remote=68954 dup_rows=17830 dup_rows_remote=14704 dedup_ratio=0.181376 router_tokens_per_expert=[1660, 41, 90, 4853, 863, 5, 206, 1008, 499, 694, 775, 424, 4684, 2493, 7, 1476, 654, 14, 42, 936, 34, 530, 1292, 794, 21, 433, 280, 961, 600, 17, 42, 9, 11, 257, 908, 159, 644, 0, 29, 485, 849, 5603, 5, 2031, 2962, 7, 1691, 247, 85, 627, 173, 816, 2714, 798, 328, 7, 234, 36, 30, 17, 40, 1, 897, 24] payload_rows_per_dst=[6970, 7216, 12700, 11582, 9166, 11520, 9312, 12008] dup_rows_per_dst=[776, 946, 5186, 2512, 1744, 3126, 1784, 1756] combine_rows_per_src=[6970, 7216, 12700, 11582, 9166, 11520, 9312, 12008] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79708 payload_rows_remote=69356 dup_rows=18596 dup_rows_remote=16838 dedup_ratio=0.189168 router_tokens_per_expert=[1173, 56, 214, 4705, 281, 3, 125, 1160, 555, 799, 1014, 346, 4109, 1783, 33, 1348, 640, 10, 20, 903, 34, 353, 1352, 726, 15, 444, 230, 707, 314, 109, 21, 106, 20, 321, 1073, 402, 812, 1, 144, 465, 411, 5377, 29, 2235, 2192, 25, 2350, 299, 243, 1702, 148, 960, 2553, 1810, 888, 65, 253, 49, 19, 20, 28, 2, 530, 38] payload_rows_per_dst=[11858, 9018, 6854, 10352, 8986, 9322, 11806, 11512] dup_rows_per_dst=[3576, 2162, 1222, 1758, 1874, 1952, 4932, 1120] combine_rows_per_src=[11858, 9018, 6854, 10352, 8986, 9322, 11806, 11512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280322048 dispatch_meta_bytes=337384 combine_payload_bytes=280322048 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=280659432 ingress_bytes=314925056 link_bytes=314925056 token_bytes=560981480 weight_movement_bytes=34603008 total_nvlink_bytes=595584488 weight_to_token_ratio=0.061683 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=17280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.185037632e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=251379712 dispatch_meta_bytes=314792 combine_payload_bytes=251379712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=251694504 ingress_bytes=285982720 link_bytes=285982720 token_bytes=503074216 weight_movement_bytes=34603008 total_nvlink_bytes=537677224 weight_to_token_ratio=0.068783 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.076129952e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77448 payload_rows_remote=67358 dup_rows=20856 dup_rows_remote=18296 dedup_ratio=0.212158 router_tokens_per_expert=[264, 898, 850, 67, 374, 854, 283, 3892, 511, 163, 102, 1827, 368, 1011, 1382, 788, 3558, 2079, 54, 406, 2109, 52, 1489, 136, 83, 81, 820, 743, 11, 626, 1286, 134, 815, 121, 349, 519, 19, 155, 3579, 1920, 1674, 124, 28, 604, 31, 703, 935, 544, 397, 430, 1274, 2910, 290, 785, 135, 568, 78, 699, 634, 58, 34, 822, 26, 591] payload_rows_per_dst=[11418, 9304, 10090, 11056, 6784, 7678, 10552, 10566] dup_rows_per_dst=[3644, 3000, 2560, 3670, 1012, 1608, 2928, 2434] combine_rows_per_src=[11418, 9304, 10090, 11056, 6784, 7678, 10552, 10566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78932 payload_rows_remote=67704 dup_rows=19372 dup_rows_remote=17382 dedup_ratio=0.197062 router_tokens_per_expert=[1373, 72, 233, 4340, 1093, 4, 213, 1024, 516, 645, 883, 516, 3938, 1670, 59, 1583, 759, 28, 90, 1037, 26, 766, 1213, 931, 34, 776, 477, 752, 298, 26, 49, 31, 79, 365, 864, 249, 929, 2, 166, 313, 559, 5099, 19, 2649, 2380, 73, 2041, 265, 250, 832, 140, 845, 2130, 1189, 695, 54, 257, 76, 11, 22, 75, 6, 1042, 21] payload_rows_per_dst=[7084, 7524, 13216, 10344, 8786, 11070, 9680, 11228] dup_rows_per_dst=[940, 880, 5164, 2418, 1908, 3482, 2590, 1990] combine_rows_per_src=[7084, 7524, 13216, 10344, 8786, 11070, 9680, 11228] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=282435584 dispatch_meta_bytes=334632 combine_payload_bytes=282435584 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=282770216 ingress_bytes=420847616 link_bytes=420847616 token_bytes=565205800 weight_movement_bytes=138412032 total_nvlink_bytes=703617832 weight_to_token_ratio=0.244888 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.583615698e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=284082176 dispatch_meta_bytes=344776 combine_payload_bytes=284082176 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=284426952 ingress_bytes=318685184 link_bytes=318685184 token_bytes=568509128 weight_movement_bytes=34603008 total_nvlink_bytes=603112136 weight_to_token_ratio=0.060866 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=29056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.199186691e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78096 payload_rows_remote=68494 dup_rows=20208 dup_rows_remote=18384 dedup_ratio=0.205566 router_tokens_per_expert=[2341, 775, 22, 83, 1978, 25, 554, 203, 14, 202, 5231, 18, 38, 1476, 863, 1222, 2424, 121, 64, 60, 383, 366, 6, 167, 340, 1229, 320, 472, 144, 7305, 2121, 5, 419, 53, 854, 375, 1343, 242, 251, 16, 1199, 935, 4, 77, 100, 499, 588, 16, 2979, 286, 4, 28, 135, 280, 2160, 1167, 112, 834, 14, 1026, 19, 543, 1632, 390] payload_rows_per_dst=[9036, 9718, 10114, 7896, 9602, 15586, 8498, 7646] dup_rows_per_dst=[2926, 2584, 2894, 1366, 1824, 5860, 1260, 1494] combine_rows_per_src=[9036, 9718, 10114, 7896, 9602, 15586, 8498, 7646] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=285630464 dispatch_meta_bytes=360472 combine_payload_bytes=285630464 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=285990936 ingress_bytes=424042496 link_bytes=424042496 token_bytes=571621400 weight_movement_bytes=138412032 total_nvlink_bytes=710033432 weight_to_token_ratio=0.242139 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.595637774e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74832 payload_rows_remote=61372 dup_rows=23472 dup_rows_remote=17326 dedup_ratio=0.238770 router_tokens_per_expert=[1767, 474, 28, 55, 1389, 31, 602, 147, 16, 294, 5310, 20, 126, 1374, 1082, 1299, 3180, 78, 74, 119, 198, 649, 9, 262, 186, 1251, 304, 780, 234, 7069, 2143, 14, 481, 36, 1402, 359, 1348, 341, 149, 7, 1114, 1641, 2, 78, 47, 630, 74, 8, 2068, 438, 34, 33, 299, 269, 2708, 1121, 133, 860, 21, 525, 5, 627, 1332, 398] payload_rows_per_dst=[13460, 8398, 8038, 14260, 10480, 5880, 7638, 6678] dup_rows_per_dst=[6146, 2262, 1100, 7464, 3182, 1308, 886, 1124] combine_rows_per_src=[13460, 8398, 8038, 14260, 10480, 5880, 7638, 6678] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=281231360 dispatch_meta_bytes=349136 combine_payload_bytes=281231360 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=281580496 ingress_bytes=385040384 link_bytes=385040384 token_bytes=562811856 weight_movement_bytes=103809024 total_nvlink_bytes=666620880 weight_to_token_ratio=0.184447 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=27648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.448875967e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74904 payload_rows_remote=63212 dup_rows=23400 dup_rows_remote=19532 dedup_ratio=0.238037 router_tokens_per_expert=[1727, 711, 60, 58, 1743, 55, 202, 248, 14, 272, 5733, 32, 38, 1527, 783, 1574, 1770, 78, 150, 86, 230, 647, 12, 675, 249, 1192, 627, 578, 270, 7173, 1847, 12, 492, 46, 1267, 392, 2018, 257, 44, 4, 1093, 918, 0, 140, 99, 812, 105, 18, 2484, 271, 11, 43, 151, 380, 2442, 1311, 93, 720, 16, 968, 10, 486, 1167, 521] payload_rows_per_dst=[10084, 11692, 6534, 15380, 10838, 5406, 8098, 6872] dup_rows_per_dst=[3910, 3868, 762, 8516, 3086, 964, 1204, 1090] combine_rows_per_src=[10084, 11692, 6534, 15380, 10838, 5406, 8098, 6872] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=235159552 dispatch_meta_bytes=289432 combine_payload_bytes=235159552 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=235448984 ingress_bytes=304365568 link_bytes=304365568 token_bytes=470608536 weight_movement_bytes=69206016 total_nvlink_bytes=539814552 weight_to_token_ratio=0.147056 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.145303129e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=275898368 dispatch_meta_bytes=342616 combine_payload_bytes=275898368 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=276240984 ingress_bytes=379707392 link_bytes=379707392 token_bytes=552139352 weight_movement_bytes=103809024 total_nvlink_bytes=655948376 weight_to_token_ratio=0.188012 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.428808347e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277315584 dispatch_meta_bytes=340344 combine_payload_bytes=277315584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277655928 ingress_bytes=311918592 link_bytes=311918592 token_bytes=554971512 weight_movement_bytes=34603008 total_nvlink_bytes=589574520 weight_to_token_ratio=0.062351 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=38784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.173724550e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=280551424 dispatch_meta_bytes=347512 combine_payload_bytes=280551424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280898936 ingress_bytes=315154432 link_bytes=315154432 token_bytes=561450360 weight_movement_bytes=34603008 total_nvlink_bytes=596053368 weight_to_token_ratio=0.061631 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.185900756e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=251379712 dispatch_meta_bytes=314792 combine_payload_bytes=251379712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=251694504 ingress_bytes=285982720 link_bytes=285982720 token_bytes=503074216 weight_movement_bytes=34603008 total_nvlink_bytes=537677224 weight_to_token_ratio=0.068783 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.076129952e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79600 payload_rows_remote=69356 dup_rows=18704 dup_rows_remote=17184 dedup_ratio=0.190267 router_tokens_per_expert=[680, 128, 204, 540, 130, 1052, 763, 22, 13, 357, 640, 18, 68, 927, 60, 165, 16, 14, 1, 1019, 24, 8, 517, 1827, 1982, 3617, 1138, 283, 2204, 3620, 2824, 306, 48, 1807, 1104, 264, 2503, 284, 27, 73, 101, 2730, 20, 4, 622, 0, 0, 2, 4850, 481, 21, 2412, 28, 2, 38, 213, 16, 2660, 310, 131, 3079, 1, 0, 154] payload_rows_per_dst=[10362, 10244, 12222, 9190, 9954, 11396, 5836, 10396] dup_rows_per_dst=[2324, 1520, 4330, 2602, 2266, 2802, 554, 2306] combine_rows_per_src=[10362, 10244, 12222, 9190, 9954, 11396, 5836, 10396] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78094 payload_rows_remote=65516 dup_rows=20210 dup_rows_remote=16348 dedup_ratio=0.205587 router_tokens_per_expert=[4030, 323, 325, 631, 12, 280, 1006, 2474, 54, 5, 2, 885, 42, 762, 8, 542, 13, 17, 1944, 29, 551, 62, 68, 149, 952, 315, 2239, 3, 2, 4065, 919, 25, 223, 1037, 214, 2005, 178, 992, 797, 7, 66, 2869, 155, 9, 3527, 249, 10, 1625, 171, 28, 82, 419, 526, 827, 114, 9, 353, 2135, 546, 498, 7, 386, 5387, 967] payload_rows_per_dst=[12566, 10374, 12578, 7860, 8848, 8502, 9720, 7646] dup_rows_per_dst=[5596, 2356, 3862, 1050, 2058, 1464, 1686, 2138] combine_rows_per_src=[12566, 10374, 12578, 7860, 8848, 8502, 9720, 7646] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74150 payload_rows_remote=61900 dup_rows=24154 dup_rows_remote=18718 dedup_ratio=0.245707 router_tokens_per_expert=[1620, 589, 18, 76, 2469, 34, 200, 482, 10, 205, 5471, 13, 59, 1495, 840, 1562, 2390, 60, 76, 50, 284, 645, 16, 409, 401, 960, 522, 757, 165, 6829, 1707, 12, 586, 52, 1392, 535, 1689, 235, 72, 9, 1280, 700, 0, 46, 152, 616, 95, 4, 2139, 308, 18, 50, 264, 325, 2811, 1221, 111, 806, 44, 770, 5, 544, 1275, 572] payload_rows_per_dst=[8192, 7156, 13506, 4872, 7962, 5082, 12250, 15130] dup_rows_per_dst=[2784, 1212, 5296, 762, 1178, 704, 5436, 6782] combine_rows_per_src=[8192, 7156, 13506, 4872, 7962, 5082, 12250, 15130] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79218 payload_rows_remote=70800 dup_rows=19086 dup_rows_remote=17806 dedup_ratio=0.194153 router_tokens_per_expert=[3818, 299, 352, 456, 14, 427, 1438, 1170, 85, 0, 0, 1819, 2, 791, 0, 442, 58, 7, 1103, 78, 228, 29, 71, 173, 781, 198, 3145, 5, 11, 4056, 681, 28, 197, 1335, 99, 2111, 331, 698, 784, 0, 37, 3250, 58, 5, 3500, 307, 23, 2473, 381, 48, 16, 438, 537, 1002, 18, 12, 159, 2160, 590, 806, 14, 530, 5241, 227] payload_rows_per_dst=[11848, 10852, 11904, 8418, 9236, 9424, 9936, 7600] dup_rows_per_dst=[4100, 3538, 2072, 1280, 1874, 2882, 1968, 1372] combine_rows_per_src=[11848, 10852, 11904, 8418, 9236, 9424, 9936, 7600] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77384 payload_rows_remote=65928 dup_rows=20920 dup_rows_remote=18366 dedup_ratio=0.212809 router_tokens_per_expert=[226, 139, 151, 1755, 150, 908, 1102, 9, 4, 348, 736, 71, 63, 701, 37, 269, 48, 8, 1, 619, 9, 6, 286, 1251, 1019, 4076, 468, 817, 2097, 4437, 3306, 173, 17, 1850, 785, 284, 2010, 147, 25, 94, 66, 1748, 41, 0, 712, 0, 0, 1, 5918, 455, 34, 2854, 55, 15, 6, 205, 4, 2739, 207, 84, 3426, 0, 0, 80] payload_rows_per_dst=[7412, 10788, 4298, 10918, 8920, 11456, 13346, 10246] dup_rows_per_dst=[1468, 1822, 158, 4842, 1504, 2554, 5738, 2834] combine_rows_per_src=[7412, 10788, 4298, 10918, 8920, 11456, 13346, 10246] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=258916352 dispatch_meta_bytes=330976 combine_payload_bytes=258916352 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=259247328 ingress_bytes=362725376 link_bytes=362725376 token_bytes=518163680 weight_movement_bytes=103809024 total_nvlink_bytes=621972704 weight_to_token_ratio=0.200340 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.364906388e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=284082176 dispatch_meta_bytes=346160 combine_payload_bytes=284082176 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=284428336 ingress_bytes=353288192 link_bytes=353288192 token_bytes=568510512 weight_movement_bytes=69206016 total_nvlink_bytes=637716528 weight_to_token_ratio=0.121732 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=30080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.329395024e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76220 payload_rows_remote=66320 dup_rows=22084 dup_rows_remote=20112 dedup_ratio=0.224650 router_tokens_per_expert=[806, 194, 236, 1346, 177, 903, 1118, 37, 29, 597, 586, 37, 42, 1101, 57, 197, 143, 22, 3, 869, 8, 13, 813, 1589, 1176, 4126, 847, 374, 1705, 3696, 2373, 163, 36, 2174, 938, 246, 1505, 1117, 100, 100, 152, 1810, 37, 15, 862, 0, 0, 0, 5757, 411, 49, 2387, 51, 67, 34, 244, 10, 2753, 165, 87, 2536, 3, 2, 121] payload_rows_per_dst=[8232, 11342, 6560, 4530, 10962, 10530, 14164, 9900] dup_rows_per_dst=[1402, 2202, 360, 590, 4880, 2614, 8064, 1972] combine_rows_per_src=[8232, 11342, 6560, 4530, 10962, 10530, 14164, 9900] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=268353536 dispatch_meta_bytes=327456 combine_payload_bytes=268353536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268680992 ingress_bytes=302956544 link_bytes=302956544 token_bytes=537034528 weight_movement_bytes=34603008 total_nvlink_bytes=571637536 weight_to_token_ratio=0.064433 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=33920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.140001085e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78486 payload_rows_remote=67752 dup_rows=19818 dup_rows_remote=14920 dedup_ratio=0.201599 router_tokens_per_expert=[4285, 387, 106, 660, 40, 342, 1443, 1944, 42, 0, 0, 2189, 2, 1159, 3, 656, 50, 15, 559, 21, 575, 30, 81, 168, 570, 73, 2665, 2, 1, 3460, 797, 37, 174, 1854, 64, 1511, 92, 580, 829, 1, 34, 4223, 29, 20, 3846, 151, 10, 1724, 135, 8, 27, 288, 386, 779, 110, 1, 183, 1674, 319, 375, 3, 642, 5863, 855] payload_rows_per_dst=[8176, 11304, 12574, 9348, 10734, 10162, 9742, 6446] dup_rows_per_dst=[1710, 3718, 2150, 2048, 4898, 2220, 1418, 1656] combine_rows_per_src=[8176, 11304, 12574, 9348, 10734, 10162, 9742, 6446] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=253542400 dispatch_meta_bytes=322472 combine_payload_bytes=253542400 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=253864872 ingress_bytes=322748416 link_bytes=322748416 token_bytes=507407272 weight_movement_bytes=69206016 total_nvlink_bytes=576613288 weight_to_token_ratio=0.136391 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.214476306e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=289996800 dispatch_meta_bytes=354424 combine_payload_bytes=289996800 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=290351224 ingress_bytes=359202816 link_bytes=359202816 token_bytes=580348024 weight_movement_bytes=69206016 total_nvlink_bytes=649554040 weight_to_token_ratio=0.119249 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.351651278e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80116 payload_rows_remote=68624 dup_rows=18188 dup_rows_remote=14304 dedup_ratio=0.185018 router_tokens_per_expert=[4124, 244, 166, 409, 22, 378, 786, 1559, 52, 0, 0, 1886, 3, 964, 1, 390, 62, 22, 556, 10, 400, 61, 83, 201, 569, 279, 2916, 4, 7, 4519, 620, 33, 327, 1393, 118, 1619, 270, 413, 803, 0, 18, 4114, 42, 22, 3997, 243, 5, 2130, 250, 19, 74, 257, 715, 716, 139, 8, 236, 1496, 628, 625, 1, 283, 6106, 759] payload_rows_per_dst=[11492, 11874, 13080, 7916, 8358, 10304, 10542, 6550] dup_rows_per_dst=[3884, 3756, 1922, 940, 1528, 2844, 1808, 1506] combine_rows_per_src=[11492, 11874, 13080, 7916, 8358, 10304, 10542, 6550] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=270041088 dispatch_meta_bytes=337176 combine_payload_bytes=270041088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270378264 ingress_bytes=304644096 link_bytes=304644096 token_bytes=540419352 weight_movement_bytes=34603008 total_nvlink_bytes=575022360 weight_to_token_ratio=0.064030 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=25216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.146351207e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=271646720 dispatch_meta_bytes=345728 combine_payload_bytes=271646720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271992448 ingress_bytes=306249728 link_bytes=306249728 token_bytes=543639168 weight_movement_bytes=34603008 total_nvlink_bytes=578242176 weight_to_token_ratio=0.063651 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=23040 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.152393071e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75732 payload_rows_remote=65364 dup_rows=22572 dup_rows_remote=19480 dedup_ratio=0.229614 router_tokens_per_expert=[1265, 727, 74, 27, 2706, 18, 258, 273, 18, 191, 5136, 44, 147, 1654, 693, 1309, 2687, 79, 58, 97, 285, 305, 7, 571, 311, 922, 387, 718, 332, 7252, 2087, 16, 361, 55, 1658, 427, 1780, 228, 72, 1, 1356, 711, 0, 40, 97, 757, 121, 8, 2068, 343, 6, 129, 261, 319, 3047, 1200, 161, 513, 11, 557, 2, 625, 1246, 338] payload_rows_per_dst=[8168, 6810, 13140, 6400, 10746, 10368, 9448, 10652] dup_rows_per_dst=[2528, 1302, 5310, 1150, 3260, 3092, 2452, 3478] combine_rows_per_src=[8168, 6810, 13140, 6400, 10746, 10368, 9448, 10652] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76652 payload_rows_remote=69734 dup_rows=21652 dup_rows_remote=20384 dedup_ratio=0.220256 router_tokens_per_expert=[1352, 545, 19, 60, 2648, 41, 372, 548, 60, 309, 4768, 43, 167, 1681, 820, 1127, 2070, 85, 162, 50, 271, 920, 13, 309, 151, 1069, 388, 513, 445, 7118, 1509, 18, 809, 35, 1135, 382, 1315, 261, 90, 17, 1424, 922, 0, 84, 134, 628, 255, 23, 2465, 276, 39, 78, 583, 259, 2306, 1628, 236, 808, 38, 479, 12, 617, 1484, 679] payload_rows_per_dst=[8250, 13334, 6572, 6918, 9796, 15334, 9232, 7216] dup_rows_per_dst=[2920, 4616, 1188, 1268, 2904, 5842, 1424, 1490] combine_rows_per_src=[8250, 13334, 6572, 6918, 9796, 15334, 9232, 7216] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=253542400 dispatch_meta_bytes=322472 combine_payload_bytes=253542400 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=253864872 ingress_bytes=322748416 link_bytes=322748416 token_bytes=507407272 weight_movement_bytes=69206016 total_nvlink_bytes=576613288 weight_to_token_ratio=0.136391 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.214476306e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=277512192 dispatch_meta_bytes=330688 combine_payload_bytes=277512192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277842880 ingress_bytes=312115200 link_bytes=312115200 token_bytes=555355072 weight_movement_bytes=34603008 total_nvlink_bytes=589958080 weight_to_token_ratio=0.062308 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.174464370e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=281083904 dispatch_meta_bytes=331712 combine_payload_bytes=281083904 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=281415616 ingress_bytes=419495936 link_bytes=419495936 token_bytes=562499520 weight_movement_bytes=138412032 total_nvlink_bytes=700911552 weight_to_token_ratio=0.246066 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=16128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.578529435e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77282 payload_rows_remote=63606 dup_rows=21022 dup_rows_remote=14930 dedup_ratio=0.213847 router_tokens_per_expert=[81, 1110, 198, 297, 1992, 4457, 9, 3, 284, 1674, 656, 19, 253, 545, 2006, 168, 92, 27, 138, 2401, 0, 217, 712, 1, 261, 122, 105, 243, 5262, 330, 7, 3358, 9, 111, 50, 2, 823, 884, 0, 0, 1498, 1240, 2, 5012, 815, 51, 85, 998, 0, 4279, 26, 325, 227, 1, 118, 1979, 37, 84, 727, 2517, 184, 9, 25, 6] payload_rows_per_dst=[13890, 13676, 6674, 14026, 11390, 5766, 5184, 6676] dup_rows_per_dst=[5400, 6092, 502, 5350, 2392, 616, 168, 502] combine_rows_per_src=[13890, 13676, 6674, 14026, 11390, 5766, 5184, 6676] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77608 payload_rows_remote=64340 dup_rows=20696 dup_rows_remote=14702 dedup_ratio=0.210531 router_tokens_per_expert=[893, 82, 420, 1508, 245, 1659, 925, 23, 20, 714, 1040, 47, 45, 829, 55, 120, 125, 105, 4, 1520, 90, 6, 514, 1983, 1221, 3480, 1274, 672, 2389, 3027, 2638, 385, 74, 1800, 1125, 243, 1216, 218, 34, 98, 157, 1924, 32, 44, 730, 2, 27, 3, 5284, 564, 112, 1423, 70, 18, 44, 271, 12, 2938, 175, 394, 1941, 3, 2, 116] payload_rows_per_dst=[12286, 10712, 13268, 9388, 8064, 9870, 4520, 9500] dup_rows_per_dst=[4500, 1988, 5994, 2494, 1552, 2022, 484, 1662] combine_rows_per_src=[12286, 10712, 13268, 9388, 8064, 9870, 4520, 9500] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78190 payload_rows_remote=67756 dup_rows=20114 dup_rows_remote=15690 dedup_ratio=0.204610 router_tokens_per_expert=[528, 209, 149, 1920, 356, 1485, 927, 44, 17, 258, 753, 14, 28, 284, 45, 297, 32, 11, 3, 737, 4, 3, 363, 1305, 706, 3734, 814, 671, 2096, 3280, 3237, 152, 33, 1202, 1211, 240, 1656, 1060, 46, 179, 94, 1966, 91, 3, 986, 3, 1, 4, 5343, 401, 20, 2620, 55, 21, 25, 281, 11, 3179, 35, 575, 3229, 0, 0, 120] payload_rows_per_dst=[9014, 9594, 12418, 10434, 9066, 10332, 6186, 11146] dup_rows_per_dst=[2716, 1266, 3184, 4424, 2188, 2524, 660, 3152] combine_rows_per_src=[9014, 9594, 12418, 10434, 9066, 10332, 6186, 11146] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78486 payload_rows_remote=67752 dup_rows=19818 dup_rows_remote=14920 dedup_ratio=0.201599 router_tokens_per_expert=[4285, 387, 106, 660, 40, 342, 1443, 1944, 42, 0, 0, 2189, 2, 1159, 3, 656, 50, 15, 559, 21, 575, 30, 81, 168, 570, 73, 2665, 2, 1, 3460, 797, 37, 174, 1854, 64, 1511, 92, 580, 829, 1, 34, 4223, 29, 20, 3846, 151, 10, 1724, 135, 8, 27, 288, 386, 779, 110, 1, 183, 1674, 319, 375, 3, 642, 5863, 855] payload_rows_per_dst=[8176, 11304, 12574, 9348, 10734, 10162, 9742, 6446] dup_rows_per_dst=[1710, 3718, 2150, 2048, 4898, 2220, 1418, 1656] combine_rows_per_src=[8176, 11304, 12574, 9348, 10734, 10162, 9742, 6446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=267730944 dispatch_meta_bytes=339376 combine_payload_bytes=267730944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268070320 ingress_bytes=302333952 link_bytes=302333952 token_bytes=535801264 weight_movement_bytes=34603008 total_nvlink_bytes=570404272 weight_to_token_ratio=0.064582 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137658321e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79142 payload_rows_remote=71626 dup_rows=19162 dup_rows_remote=17574 dedup_ratio=0.194926 router_tokens_per_expert=[1361, 99, 334, 3704, 1109, 27, 151, 980, 303, 1103, 852, 372, 3924, 1772, 33, 1843, 618, 62, 126, 866, 32, 651, 1080, 1117, 15, 544, 493, 1004, 382, 60, 40, 63, 60, 212, 830, 477, 829, 4, 114, 977, 754, 6016, 13, 2535, 2678, 29, 1407, 281, 145, 659, 224, 801, 2147, 800, 334, 33, 305, 73, 3, 16, 70, 3, 1165, 38] payload_rows_per_dst=[11930, 9496, 7516, 10528, 10578, 7686, 8528, 12880] dup_rows_per_dst=[3600, 3060, 1588, 2522, 2994, 1142, 1758, 2498] combine_rows_per_src=[11930, 9496, 7516, 10528, 10578, 7686, 8528, 12880] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=59798 dup_rows=24098 dup_rows_remote=17732 dedup_ratio=0.245138 router_tokens_per_expert=[1845, 624, 18, 54, 2480, 41, 466, 149, 41, 370, 4656, 50, 141, 1509, 848, 974, 1938, 51, 105, 104, 155, 753, 6, 655, 266, 1449, 621, 682, 271, 6551, 2393, 4, 512, 56, 1509, 302, 1873, 251, 189, 0, 1360, 571, 3, 70, 120, 565, 120, 15, 2350, 290, 19, 135, 324, 308, 3263, 841, 127, 713, 18, 555, 12, 760, 1122, 529] payload_rows_per_dst=[8568, 6614, 12302, 5736, 8216, 5156, 13206, 14408] dup_rows_per_dst=[2786, 1252, 4544, 850, 1168, 492, 6640, 6366] combine_rows_per_src=[8568, 6614, 12302, 5736, 8216, 5156, 13206, 14408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=285630464 dispatch_meta_bytes=360472 combine_payload_bytes=285630464 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=285990936 ingress_bytes=424042496 link_bytes=424042496 token_bytes=571621400 weight_movement_bytes=138412032 total_nvlink_bytes=710033432 weight_to_token_ratio=0.242139 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.595637774e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80544 payload_rows_remote=69214 dup_rows=17760 dup_rows_remote=14950 dedup_ratio=0.180664 router_tokens_per_expert=[4101, 409, 463, 514, 26, 330, 750, 1910, 16, 0, 0, 1136, 2, 581, 2, 712, 16, 5, 975, 36, 450, 53, 56, 149, 925, 79, 2930, 0, 7, 4621, 861, 24, 84, 1073, 24, 1609, 212, 1262, 753, 0, 57, 3661, 75, 21, 3919, 191, 26, 1625, 180, 10, 17, 373, 708, 1033, 35, 2, 231, 1268, 370, 579, 0, 833, 5850, 932] payload_rows_per_dst=[12130, 11330, 12828, 8440, 8584, 9456, 10616, 7160] dup_rows_per_dst=[4876, 2810, 2352, 1212, 1450, 1856, 1938, 1266] combine_rows_per_src=[12130, 11330, 12828, 8440, 8584, 9456, 10616, 7160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=277512192 dispatch_meta_bytes=330688 combine_payload_bytes=277512192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277842880 ingress_bytes=312115200 link_bytes=312115200 token_bytes=555355072 weight_movement_bytes=34603008 total_nvlink_bytes=589958080 weight_to_token_ratio=0.062308 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.174464370e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=293380096 dispatch_meta_bytes=356800 combine_payload_bytes=293380096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293736896 ingress_bytes=327983104 link_bytes=327983104 token_bytes=587116992 weight_movement_bytes=34603008 total_nvlink_bytes=621720000 weight_to_token_ratio=0.058937 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=12800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.234174016e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=244932608 dispatch_meta_bytes=310120 combine_payload_bytes=244932608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=245242728 ingress_bytes=279535616 link_bytes=279535616 token_bytes=490175336 weight_movement_bytes=34603008 total_nvlink_bytes=524778344 weight_to_token_ratio=0.070593 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.051870019e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75296 payload_rows_remote=64744 dup_rows=23008 dup_rows_remote=21100 dedup_ratio=0.234049 router_tokens_per_expert=[4029, 344, 267, 564, 16, 257, 906, 1165, 29, 0, 0, 1711, 0, 1595, 2, 939, 70, 0, 1176, 30, 428, 52, 41, 268, 680, 121, 2930, 2, 6, 3605, 1057, 72, 103, 996, 42, 1623, 163, 830, 575, 0, 14, 4614, 26, 7, 4120, 140, 8, 1929, 146, 12, 7, 256, 485, 1176, 25, 3, 183, 1939, 234, 723, 0, 692, 4828, 891] payload_rows_per_dst=[6282, 7224, 3926, 12224, 9970, 12374, 10552, 12744] dup_rows_per_dst=[756, 1328, 204, 4722, 3300, 4554, 1908, 6236] combine_rows_per_src=[6282, 7224, 3926, 12224, 9970, 12374, 10552, 12744] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=7 export_slots=7 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 2, 0, 0, 1, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79216 payload_rows_remote=70646 dup_rows=19088 dup_rows_remote=16910 dedup_ratio=0.194173 router_tokens_per_expert=[72, 776, 303, 281, 2194, 3843, 57, 9, 109, 1247, 205, 17, 218, 549, 2214, 57, 42, 49, 44, 4158, 0, 104, 279, 0, 306, 20, 147, 316, 5703, 508, 27, 2726, 61, 96, 73, 15, 410, 823, 0, 0, 1486, 1352, 2, 4806, 1289, 8, 24, 1213, 3, 4306, 28, 637, 34, 0, 22, 2183, 90, 23, 1254, 1932, 277, 6, 93, 26] payload_rows_per_dst=[6836, 7364, 12366, 7398, 11018, 8570, 11874, 13790] dup_rows_per_dst=[548, 1868, 4672, 702, 1550, 2178, 2552, 5018] combine_rows_per_src=[6836, 7364, 12366, 7398, 11018, 8570, 11874, 13790] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71548 payload_rows_remote=57722 dup_rows=26756 dup_rows_remote=22126 dedup_ratio=0.272176 router_tokens_per_expert=[129, 590, 259, 330, 1841, 3899, 77, 5, 162, 723, 573, 12, 409, 386, 2062, 38, 108, 116, 136, 4285, 0, 305, 325, 0, 171, 144, 206, 333, 5908, 232, 32, 2176, 32, 155, 62, 13, 281, 711, 0, 0, 1163, 1460, 0, 5558, 1022, 11, 275, 1889, 0, 3673, 34, 509, 22, 0, 184, 2806, 25, 31, 650, 2272, 254, 9, 51, 28] payload_rows_per_dst=[5916, 7070, 12810, 2172, 2450, 15084, 12220, 13826] dup_rows_per_dst=[546, 1660, 5538, 64, 58, 10258, 4002, 4630] combine_rows_per_src=[5916, 7070, 12810, 2172, 2450, 15084, 12220, 13826] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=283500544 dispatch_meta_bytes=336656 combine_payload_bytes=283500544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=283837200 ingress_bytes=318103552 link_bytes=318103552 token_bytes=567337744 weight_movement_bytes=34603008 total_nvlink_bytes=601940752 weight_to_token_ratio=0.060992 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.196998057e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=260530176 dispatch_meta_bytes=314144 combine_payload_bytes=260530176 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260844320 ingress_bytes=295133184 link_bytes=295133184 token_bytes=521374496 weight_movement_bytes=34603008 total_nvlink_bytes=555977504 weight_to_token_ratio=0.066369 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.110562411e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=263536640 dispatch_meta_bytes=316168 combine_payload_bytes=263536640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263852808 ingress_bytes=298139648 link_bytes=298139648 token_bytes=527389448 weight_movement_bytes=34603008 total_nvlink_bytes=561992456 weight_to_token_ratio=0.065612 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.121875493e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77976 payload_rows_remote=68926 dup_rows=20328 dup_rows_remote=18598 dedup_ratio=0.206787 router_tokens_per_expert=[619, 76, 505, 1221, 190, 1991, 672, 42, 90, 393, 579, 13, 139, 929, 118, 89, 63, 19, 8, 1124, 23, 2, 357, 1294, 1399, 3558, 1600, 197, 2574, 2570, 3158, 115, 40, 1892, 922, 258, 1875, 245, 42, 116, 126, 1943, 88, 6, 663, 0, 0, 0, 5594, 571, 33, 2418, 112, 39, 41, 203, 12, 3044, 211, 304, 2494, 2, 1, 130] payload_rows_per_dst=[8482, 9932, 9318, 12490, 9050, 9014, 9298, 10392] dup_rows_per_dst=[2150, 1884, 2218, 5596, 1730, 1778, 2968, 2004] combine_rows_per_src=[8482, 9932, 9318, 12490, 9050, 9014, 9298, 10392] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=277528576 dispatch_meta_bytes=333784 combine_payload_bytes=277528576 prefetch_bytes=0 reduce_bytes=242221056 egress_bytes=277862360 ingress_bytes=519749632 link_bytes=519749632 token_bytes=555390936 weight_movement_bytes=242221056 total_nvlink_bytes=797611992 weight_to_token_ratio=0.436127 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=11776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.955776022e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78038 payload_rows_remote=65178 dup_rows=20266 dup_rows_remote=14706 dedup_ratio=0.206156 router_tokens_per_expert=[434, 233, 228, 2496, 569, 1581, 932, 52, 30, 53, 584, 19, 53, 225, 56, 146, 60, 13, 3, 1248, 7, 6, 325, 1791, 1271, 3241, 912, 724, 1288, 3859, 2685, 275, 147, 1442, 573, 240, 3067, 257, 54, 132, 215, 1897, 42, 25, 1168, 0, 1, 0, 5161, 474, 86, 3168, 53, 18, 20, 163, 12, 2672, 149, 45, 2374, 7, 0, 91] payload_rows_per_dst=[12860, 9408, 12712, 8980, 9580, 8206, 7152, 9140] dup_rows_per_dst=[5560, 1982, 4516, 2104, 2244, 1488, 812, 1560] combine_rows_per_src=[12860, 9408, 12712, 8980, 9580, 8206, 7152, 9140] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78010 payload_rows_remote=68660 dup_rows=20294 dup_rows_remote=18624 dedup_ratio=0.206441 router_tokens_per_expert=[4315, 279, 116, 616, 2, 671, 624, 1769, 41, 0, 0, 1847, 2, 1897, 0, 801, 33, 3, 1607, 8, 268, 50, 53, 113, 999, 40, 2354, 6, 4, 4221, 469, 25, 87, 922, 22, 2145, 22, 445, 754, 0, 50, 3617, 36, 7, 4159, 262, 6, 1532, 138, 15, 19, 218, 213, 832, 6, 2, 105, 2026, 416, 574, 1, 986, 5734, 568] payload_rows_per_dst=[7090, 8374, 12346, 11488, 11734, 9350, 9684, 7944] dup_rows_per_dst=[1064, 1844, 2706, 3706, 5690, 1670, 1520, 2094] combine_rows_per_src=[7090, 8374, 12346, 11488, 11734, 9350, 9684, 7944] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265191424 dispatch_meta_bytes=343376 combine_payload_bytes=265191424 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=265534800 ingress_bytes=334397440 link_bytes=334397440 token_bytes=530726224 weight_movement_bytes=69206016 total_nvlink_bytes=599932240 weight_to_token_ratio=0.130399 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=30720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258310646e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=289366016 dispatch_meta_bytes=350224 combine_payload_bytes=289366016 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=289716240 ingress_bytes=393175040 link_bytes=393175040 token_bytes=579082256 weight_movement_bytes=103809024 total_nvlink_bytes=682891280 weight_to_token_ratio=0.179265 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.479486022e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=236429312 dispatch_meta_bytes=319392 combine_payload_bytes=236429312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=236748704 ingress_bytes=271032320 link_bytes=271032320 token_bytes=473178016 weight_movement_bytes=34603008 total_nvlink_bytes=507781024 weight_to_token_ratio=0.073129 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=21376 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.019872800e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=282320896 dispatch_meta_bytes=350096 combine_payload_bytes=282320896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282670992 ingress_bytes=316923904 link_bytes=316923904 token_bytes=564991888 weight_movement_bytes=34603008 total_nvlink_bytes=599594896 weight_to_token_ratio=0.061245 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.192559136e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79218 payload_rows_remote=70800 dup_rows=19086 dup_rows_remote=17806 dedup_ratio=0.194153 router_tokens_per_expert=[3818, 299, 352, 456, 14, 427, 1438, 1170, 85, 0, 0, 1819, 2, 791, 0, 442, 58, 7, 1103, 78, 228, 29, 71, 173, 781, 198, 3145, 5, 11, 4056, 681, 28, 197, 1335, 99, 2111, 331, 698, 784, 0, 37, 3250, 58, 5, 3500, 307, 23, 2473, 381, 48, 16, 438, 537, 1002, 18, 12, 159, 2160, 590, 806, 14, 530, 5241, 227] payload_rows_per_dst=[11848, 10852, 11904, 8418, 9236, 9424, 9936, 7600] dup_rows_per_dst=[4100, 3538, 2072, 1280, 1874, 2882, 1968, 1372] combine_rows_per_src=[11848, 10852, 11904, 8418, 9236, 9424, 9936, 7600] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266969088 dispatch_meta_bytes=319536 combine_payload_bytes=266969088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267288624 ingress_bytes=301572096 link_bytes=301572096 token_bytes=534257712 weight_movement_bytes=34603008 total_nvlink_bytes=568860720 weight_to_token_ratio=0.064768 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=11648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134791519e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76346 payload_rows_remote=70090 dup_rows=21958 dup_rows_remote=21390 dedup_ratio=0.223368 router_tokens_per_expert=[109, 1083, 712, 498, 2366, 4362, 40, 13, 380, 1706, 629, 65, 322, 551, 1404, 281, 334, 70, 228, 2308, 18, 178, 276, 0, 337, 177, 105, 330, 3093, 455, 7, 3170, 24, 173, 54, 16, 1057, 1641, 2, 37, 975, 1083, 0, 4746, 1070, 6, 124, 1390, 0, 4287, 20, 1187, 372, 2, 52, 2447, 114, 30, 604, 1648, 307, 3, 61, 13] payload_rows_per_dst=[13736, 8880, 6256, 11670, 11802, 6434, 12344, 5224] dup_rows_per_dst=[6580, 2214, 568, 3678, 3698, 912, 3972, 336] combine_rows_per_src=[13736, 8880, 6256, 11670, 11802, 6434, 12344, 5224] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79946 payload_rows_remote=68174 dup_rows=18358 dup_rows_remote=16060 dedup_ratio=0.186747 router_tokens_per_expert=[6, 1832, 128, 4949, 151, 44, 198, 20, 214, 31, 236, 159, 934, 0, 1206, 110, 1059, 3, 391, 39, 377, 458, 48, 2837, 235, 1053, 6, 75, 149, 1366, 259, 350, 3230, 145, 242, 262, 295, 7, 2173, 2451, 104, 9, 1240, 336, 4, 7, 0, 142, 959, 187, 4145, 1527, 6795, 0, 626, 4, 910, 33, 0, 2517, 435, 137, 246, 1061] payload_rows_per_dst=[12404, 11772, 12098, 6008, 8870, 14394, 5742, 8658] dup_rows_per_dst=[2252, 2298, 4786, 978, 2280, 2880, 864, 2020] combine_rows_per_src=[12404, 11772, 12098, 6008, 8870, 14394, 5742, 8658] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79182 payload_rows_remote=65848 dup_rows=19122 dup_rows_remote=13950 dedup_ratio=0.194519 router_tokens_per_expert=[184, 1095, 338, 281, 1101, 4578, 12, 26, 483, 723, 274, 9, 253, 724, 2338, 101, 22, 84, 146, 2050, 0, 323, 345, 6, 351, 187, 280, 280, 4945, 465, 109, 2636, 70, 183, 102, 59, 331, 835, 2, 3, 1849, 2251, 4, 4954, 1132, 14, 42, 1980, 0, 3921, 26, 654, 34, 0, 53, 1591, 115, 105, 1607, 2264, 122, 34, 27, 39] payload_rows_per_dst=[10090, 7766, 8028, 13334, 11184, 10290, 10700, 7790] dup_rows_per_dst=[2476, 2044, 1622, 5172, 1894, 3220, 1858, 836] combine_rows_per_src=[10090, 7766, 8028, 13334, 11184, 10290, 10700, 7790] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77976 payload_rows_remote=68926 dup_rows=20328 dup_rows_remote=18598 dedup_ratio=0.206787 router_tokens_per_expert=[619, 76, 505, 1221, 190, 1991, 672, 42, 90, 393, 579, 13, 139, 929, 118, 89, 63, 19, 8, 1124, 23, 2, 357, 1294, 1399, 3558, 1600, 197, 2574, 2570, 3158, 115, 40, 1892, 922, 258, 1875, 245, 42, 116, 126, 1943, 88, 6, 663, 0, 0, 0, 5594, 571, 33, 2418, 112, 39, 41, 203, 12, 3044, 211, 304, 2494, 2, 1, 130] payload_rows_per_dst=[8482, 9932, 9318, 12490, 9050, 9014, 9298, 10392] dup_rows_per_dst=[2150, 1884, 2218, 5596, 1730, 1778, 2968, 2004] combine_rows_per_src=[8482, 9932, 9318, 12490, 9050, 9014, 9298, 10392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76344 payload_rows_remote=70294 dup_rows=21960 dup_rows_remote=21006 dedup_ratio=0.223389 router_tokens_per_expert=[1433, 709, 56, 50, 1499, 42, 362, 122, 39, 431, 4571, 17, 28, 1933, 755, 969, 1915, 137, 123, 111, 354, 406, 7, 449, 366, 1215, 254, 737, 402, 7236, 2595, 83, 508, 275, 1184, 319, 1625, 421, 278, 11, 2026, 780, 6, 30, 174, 353, 60, 31, 3088, 546, 35, 105, 227, 410, 2410, 905, 191, 766, 10, 446, 6, 513, 1415, 592] payload_rows_per_dst=[6648, 12950, 6050, 10932, 10872, 12976, 9278, 6638] dup_rows_per_dst=[1898, 4536, 954, 4534, 3190, 4254, 1354, 1240] combine_rows_per_src=[6648, 12950, 6050, 10932, 10872, 12976, 9278, 6638] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79600 payload_rows_remote=69356 dup_rows=18704 dup_rows_remote=17184 dedup_ratio=0.190267 router_tokens_per_expert=[680, 128, 204, 540, 130, 1052, 763, 22, 13, 357, 640, 18, 68, 927, 60, 165, 16, 14, 1, 1019, 24, 8, 517, 1827, 1982, 3617, 1138, 283, 2204, 3620, 2824, 306, 48, 1807, 1104, 264, 2503, 284, 27, 73, 101, 2730, 20, 4, 622, 0, 0, 2, 4850, 481, 21, 2412, 28, 2, 38, 213, 16, 2660, 310, 131, 3079, 1, 0, 154] payload_rows_per_dst=[10362, 10244, 12222, 9190, 9954, 11396, 5836, 10396] dup_rows_per_dst=[2324, 1520, 4330, 2602, 2266, 2802, 554, 2306] combine_rows_per_src=[10362, 10244, 12222, 9190, 9954, 11396, 5836, 10396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=281231360 dispatch_meta_bytes=349136 combine_payload_bytes=281231360 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=281580496 ingress_bytes=385040384 link_bytes=385040384 token_bytes=562811856 weight_movement_bytes=103809024 total_nvlink_bytes=666620880 weight_to_token_ratio=0.184447 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=27648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.448875967e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72270 payload_rows_remote=57412 dup_rows=26034 dup_rows_remote=14946 dedup_ratio=0.264832 router_tokens_per_expert=[4273, 279, 531, 653, 4, 405, 679, 816, 12, 0, 0, 1620, 3, 1378, 3, 795, 14, 11, 1191, 10, 390, 91, 101, 92, 759, 194, 3026, 3, 5, 4076, 728, 35, 190, 1016, 70, 1494, 102, 1256, 893, 0, 45, 3230, 39, 25, 3616, 209, 16, 2052, 216, 18, 8, 451, 556, 770, 8, 4, 259, 1782, 516, 533, 2, 785, 5862, 952] payload_rows_per_dst=[6018, 6530, 3536, 12796, 8324, 8572, 11636, 14858] dup_rows_per_dst=[716, 1092, 264, 4856, 1718, 2670, 3630, 11088] combine_rows_per_src=[6018, 6530, 3536, 12796, 8324, 8572, 11636, 14858] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=289996800 dispatch_meta_bytes=354424 combine_payload_bytes=289996800 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=290351224 ingress_bytes=359202816 link_bytes=359202816 token_bytes=580348024 weight_movement_bytes=69206016 total_nvlink_bytes=649554040 weight_to_token_ratio=0.119249 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.351651278e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=282320896 dispatch_meta_bytes=350096 combine_payload_bytes=282320896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282670992 ingress_bytes=316923904 link_bytes=316923904 token_bytes=564991888 weight_movement_bytes=34603008 total_nvlink_bytes=599594896 weight_to_token_ratio=0.061245 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.192559136e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=287924224 dispatch_meta_bytes=365200 combine_payload_bytes=287924224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288289424 ingress_bytes=322527232 link_bytes=322527232 token_bytes=576213648 weight_movement_bytes=34603008 total_nvlink_bytes=610816656 weight_to_token_ratio=0.060052 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.213644008e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77184 payload_rows_remote=64526 dup_rows=21120 dup_rows_remote=15256 dedup_ratio=0.214844 router_tokens_per_expert=[542, 191, 296, 669, 174, 2034, 848, 14, 48, 295, 497, 4, 45, 1450, 64, 283, 46, 17, 4, 971, 5, 0, 443, 1888, 820, 4124, 651, 497, 2698, 2984, 3089, 327, 72, 1544, 952, 254, 2199, 617, 24, 120, 111, 2174, 80, 2, 501, 7, 0, 0, 4597, 351, 77, 3067, 70, 30, 22, 216, 14, 3017, 347, 257, 2303, 0, 0, 109] payload_rows_per_dst=[7988, 11092, 6438, 7780, 11594, 9692, 12658, 9942] dup_rows_per_dst=[1548, 2528, 310, 1942, 4750, 2026, 5864, 2152] combine_rows_per_src=[7988, 11092, 6438, 7780, 11594, 9692, 12658, 9942] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=287088640 dispatch_meta_bytes=365920 combine_payload_bytes=287088640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=287454560 ingress_bytes=321691648 link_bytes=321691648 token_bytes=574543200 weight_movement_bytes=34603008 total_nvlink_bytes=609146208 weight_to_token_ratio=0.060227 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.210499773e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73380 payload_rows_remote=62750 dup_rows=24924 dup_rows_remote=21070 dedup_ratio=0.253540 router_tokens_per_expert=[10, 3425, 595, 4543, 183, 101, 267, 13, 146, 12, 85, 113, 1098, 0, 1466, 16, 934, 0, 701, 58, 658, 338, 116, 2089, 681, 597, 49, 60, 61, 1911, 364, 423, 4296, 187, 213, 455, 457, 4, 1035, 737, 130, 3, 1312, 125, 6, 13, 7, 267, 333, 669, 3689, 1804, 5786, 1, 1208, 7, 387, 23, 4, 2394, 425, 84, 496, 1482] payload_rows_per_dst=[8386, 5506, 8496, 12830, 8774, 3574, 15184, 10630] dup_rows_per_dst=[802, 366, 1292, 4548, 2100, 152, 11810, 3854] combine_rows_per_src=[8386, 5506, 8496, 12830, 8774, 3574, 15184, 10630] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79006 payload_rows_remote=68672 dup_rows=19298 dup_rows_remote=17760 dedup_ratio=0.196309 router_tokens_per_expert=[2, 2236, 841, 4213, 110, 36, 340, 1, 92, 13, 79, 78, 370, 0, 1408, 41, 1280, 3, 568, 23, 653, 297, 60, 1024, 229, 2231, 14, 79, 53, 1366, 336, 457, 4405, 107, 224, 446, 380, 1, 884, 1827, 157, 7, 1062, 102, 0, 2, 3, 173, 451, 328, 5255, 1291, 6709, 0, 1020, 0, 509, 64, 0, 3659, 435, 41, 303, 774] payload_rows_per_dst=[6672, 12494, 6836, 12710, 11976, 10334, 8544, 9440] dup_rows_per_dst=[460, 2178, 980, 5246, 4572, 1538, 2194, 2130] combine_rows_per_src=[6672, 12494, 6836, 12710, 11976, 10334, 8544, 9440] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=269713408 dispatch_meta_bytes=319192 combine_payload_bytes=269713408 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=270032600 ingress_bytes=373522432 link_bytes=373522432 token_bytes=539746008 weight_movement_bytes=103809024 total_nvlink_bytes=643555032 weight_to_token_ratio=0.192329 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=19456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.405534841e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78648 payload_rows_remote=66872 dup_rows=19656 dup_rows_remote=16918 dedup_ratio=0.199951 router_tokens_per_expert=[258, 784, 455, 287, 1870, 4250, 33, 23, 121, 984, 587, 18, 476, 553, 1687, 142, 126, 25, 237, 2649, 0, 373, 167, 14, 414, 219, 130, 397, 5216, 597, 5, 2272, 23, 144, 94, 0, 982, 665, 1, 0, 2006, 1384, 1, 5348, 849, 47, 69, 1665, 1, 3831, 19, 786, 39, 4, 119, 2500, 64, 8, 816, 2009, 239, 10, 46, 14] payload_rows_per_dst=[6830, 7150, 11812, 8098, 11776, 9584, 11496, 11902] dup_rows_per_dst=[590, 1986, 3870, 1334, 2738, 2458, 3102, 3578] combine_rows_per_src=[6830, 7150, 11812, 8098, 11776, 9584, 11496, 11902] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=279240704 dispatch_meta_bytes=336936 combine_payload_bytes=279240704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279577640 ingress_bytes=313843712 link_bytes=313843712 token_bytes=558818344 weight_movement_bytes=34603008 total_nvlink_bytes=593421352 weight_to_token_ratio=0.061922 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=25856 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.180968622e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=284082176 dispatch_meta_bytes=346160 combine_payload_bytes=284082176 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=284428336 ingress_bytes=353288192 link_bytes=353288192 token_bytes=568510512 weight_movement_bytes=69206016 total_nvlink_bytes=637716528 weight_to_token_ratio=0.121732 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=30080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.329395024e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=235159552 dispatch_meta_bytes=289432 combine_payload_bytes=235159552 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=235448984 ingress_bytes=304365568 link_bytes=304365568 token_bytes=470608536 weight_movement_bytes=69206016 total_nvlink_bytes=539814552 weight_to_token_ratio=0.147056 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.145303129e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77058 payload_rows_remote=64056 dup_rows=21246 dup_rows_remote=15960 dedup_ratio=0.216125 router_tokens_per_expert=[131, 803, 181, 279, 2535, 3767, 22, 2, 218, 1667, 1301, 41, 213, 301, 1913, 58, 36, 75, 191, 3370, 0, 274, 193, 3, 498, 202, 225, 119, 4935, 186, 5, 2696, 12, 112, 13, 52, 558, 2128, 0, 0, 1424, 964, 1, 5534, 760, 30, 22, 941, 0, 4034, 23, 664, 18, 0, 65, 1833, 103, 14, 792, 2361, 184, 4, 29, 37] payload_rows_per_dst=[13002, 13610, 7746, 13438, 12830, 4932, 4976, 6524] dup_rows_per_dst=[5286, 5882, 538, 4294, 3988, 504, 230, 524] combine_rows_per_src=[13002, 13610, 7746, 13438, 12830, 4932, 4976, 6524] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=264298496 dispatch_meta_bytes=319128 combine_payload_bytes=264298496 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=264617624 ingress_bytes=333504512 link_bytes=333504512 token_bytes=528916120 weight_movement_bytes=69206016 total_nvlink_bytes=598122136 weight_to_token_ratio=0.130845 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.254950629e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=257024000 dispatch_meta_bytes=335280 combine_payload_bytes=257024000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=257359280 ingress_bytes=291627008 link_bytes=291627008 token_bytes=514383280 weight_movement_bytes=34603008 total_nvlink_bytes=548986288 weight_to_token_ratio=0.067271 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=25088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.097368953e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=281280512 dispatch_meta_bytes=345728 combine_payload_bytes=281280512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281626240 ingress_bytes=315883520 link_bytes=315883520 token_bytes=562906752 weight_movement_bytes=34603008 total_nvlink_bytes=597509760 weight_to_token_ratio=0.061472 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=35968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.188644255e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=273907712 dispatch_meta_bytes=335160 combine_payload_bytes=273907712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274242872 ingress_bytes=308510720 link_bytes=308510720 token_bytes=548150584 weight_movement_bytes=34603008 total_nvlink_bytes=582753592 weight_to_token_ratio=0.063127 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=33280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.160901002e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77384 payload_rows_remote=65928 dup_rows=20920 dup_rows_remote=18366 dedup_ratio=0.212809 router_tokens_per_expert=[226, 139, 151, 1755, 150, 908, 1102, 9, 4, 348, 736, 71, 63, 701, 37, 269, 48, 8, 1, 619, 9, 6, 286, 1251, 1019, 4076, 468, 817, 2097, 4437, 3306, 173, 17, 1850, 785, 284, 2010, 147, 25, 94, 66, 1748, 41, 0, 712, 0, 0, 1, 5918, 455, 34, 2854, 55, 15, 6, 205, 4, 2739, 207, 84, 3426, 0, 0, 80] payload_rows_per_dst=[7412, 10788, 4298, 10918, 8920, 11456, 13346, 10246] dup_rows_per_dst=[1468, 1822, 158, 4842, 1504, 2554, 5738, 2834] combine_rows_per_src=[7412, 10788, 4298, 10918, 8920, 11456, 13346, 10246] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78190 payload_rows_remote=67756 dup_rows=20114 dup_rows_remote=15690 dedup_ratio=0.204610 router_tokens_per_expert=[528, 209, 149, 1920, 356, 1485, 927, 44, 17, 258, 753, 14, 28, 284, 45, 297, 32, 11, 3, 737, 4, 3, 363, 1305, 706, 3734, 814, 671, 2096, 3280, 3237, 152, 33, 1202, 1211, 240, 1656, 1060, 46, 179, 94, 1966, 91, 3, 986, 3, 1, 4, 5343, 401, 20, 2620, 55, 21, 25, 281, 11, 3179, 35, 575, 3229, 0, 0, 120] payload_rows_per_dst=[9014, 9594, 12418, 10434, 9066, 10332, 6186, 11146] dup_rows_per_dst=[2716, 1266, 3184, 4424, 2188, 2524, 660, 3152] combine_rows_per_src=[9014, 9594, 12418, 10434, 9066, 10332, 6186, 11146] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=262373376 dispatch_meta_bytes=320064 combine_payload_bytes=262373376 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=262693440 ingress_bytes=331579392 link_bytes=331579392 token_bytes=525066816 weight_movement_bytes=69206016 total_nvlink_bytes=594272832 weight_to_token_ratio=0.131804 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=15616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.247706558e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78690 payload_rows_remote=66246 dup_rows=19614 dup_rows_remote=13576 dedup_ratio=0.199524 router_tokens_per_expert=[40, 2359, 259, 3350, 151, 114, 176, 34, 165, 13, 1074, 94, 395, 4, 2073, 90, 585, 33, 688, 60, 843, 773, 59, 2500, 401, 639, 45, 50, 182, 1418, 413, 459, 3700, 280, 221, 390, 434, 2, 1705, 2418, 114, 10, 501, 323, 6, 14, 17, 413, 823, 271, 3640, 1764, 6447, 5, 1017, 1, 531, 52, 0, 2379, 382, 158, 149, 1446] payload_rows_per_dst=[11018, 11876, 12444, 6160, 8554, 13664, 6662, 8312] dup_rows_per_dst=[1948, 3220, 6038, 1054, 2346, 2026, 1100, 1882] combine_rows_per_src=[11018, 11876, 12444, 6160, 8554, 13664, 6662, 8312] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80598 payload_rows_remote=73222 dup_rows=17706 dup_rows_remote=16060 dedup_ratio=0.180115 router_tokens_per_expert=[3, 2646, 494, 4233, 119, 48, 237, 16, 88, 8, 120, 17, 337, 4, 1271, 54, 658, 0, 444, 15, 540, 441, 34, 1928, 223, 1752, 17, 142, 45, 1623, 388, 321, 3595, 180, 276, 448, 589, 1, 471, 1195, 371, 14, 632, 348, 6, 49, 2, 160, 288, 250, 4015, 1578, 7045, 30, 1812, 0, 996, 51, 0, 4278, 603, 135, 386, 1082] payload_rows_per_dst=[12478, 10380, 8838, 7376, 8888, 14626, 6692, 11320] dup_rows_per_dst=[3114, 1448, 1740, 1646, 2164, 2628, 1224, 3742] combine_rows_per_src=[12478, 10380, 8838, 7376, 8888, 14626, 6692, 11320] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80642 payload_rows_remote=68814 dup_rows=17662 dup_rows_remote=12150 dedup_ratio=0.179667 router_tokens_per_expert=[1054, 385, 7, 1, 48, 614, 0, 23, 2368, 1129, 23, 0, 650, 3204, 732, 564, 2116, 3702, 0, 269, 997, 1400, 70, 952, 94, 2, 5854, 55, 307, 474, 18, 1340, 530, 202, 377, 25, 59, 0, 1172, 7, 1344, 379, 108, 59, 26, 6805, 206, 45, 119, 1588, 0, 68, 82, 1053, 1766, 0, 58, 93, 3977, 148, 129, 256, 4, 15] payload_rows_per_dst=[10126, 11828, 9214, 13664, 14740, 4122, 8042, 8906] dup_rows_per_dst=[1542, 5512, 2394, 2624, 3614, 212, 1310, 454] combine_rows_per_src=[10126, 11828, 9214, 13664, 14740, 4122, 8042, 8906] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78648 payload_rows_remote=66872 dup_rows=19656 dup_rows_remote=16918 dedup_ratio=0.199951 router_tokens_per_expert=[258, 784, 455, 287, 1870, 4250, 33, 23, 121, 984, 587, 18, 476, 553, 1687, 142, 126, 25, 237, 2649, 0, 373, 167, 14, 414, 219, 130, 397, 5216, 597, 5, 2272, 23, 144, 94, 0, 982, 665, 1, 0, 2006, 1384, 1, 5348, 849, 47, 69, 1665, 1, 3831, 19, 786, 39, 4, 119, 2500, 64, 8, 816, 2009, 239, 10, 46, 14] payload_rows_per_dst=[6830, 7150, 11812, 8098, 11776, 9584, 11496, 11902] dup_rows_per_dst=[590, 1986, 3870, 1334, 2738, 2458, 3102, 3578] combine_rows_per_src=[6830, 7150, 11812, 8098, 11776, 9584, 11496, 11902] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=7 export_slots=7 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 2, 0, 0, 1, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=270041088 dispatch_meta_bytes=337176 combine_payload_bytes=270041088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270378264 ingress_bytes=304644096 link_bytes=304644096 token_bytes=540419352 weight_movement_bytes=34603008 total_nvlink_bytes=575022360 weight_to_token_ratio=0.064030 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=25216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.146351207e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78094 payload_rows_remote=65516 dup_rows=20210 dup_rows_remote=16348 dedup_ratio=0.205587 router_tokens_per_expert=[4030, 323, 325, 631, 12, 280, 1006, 2474, 54, 5, 2, 885, 42, 762, 8, 542, 13, 17, 1944, 29, 551, 62, 68, 149, 952, 315, 2239, 3, 2, 4065, 919, 25, 223, 1037, 214, 2005, 178, 992, 797, 7, 66, 2869, 155, 9, 3527, 249, 10, 1625, 171, 28, 82, 419, 526, 827, 114, 9, 353, 2135, 546, 498, 7, 386, 5387, 967] payload_rows_per_dst=[12566, 10374, 12578, 7860, 8848, 8502, 9720, 7646] dup_rows_per_dst=[5596, 2356, 3862, 1050, 2058, 1464, 1686, 2138] combine_rows_per_src=[12566, 10374, 12578, 7860, 8848, 8502, 9720, 7646] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77282 payload_rows_remote=63606 dup_rows=21022 dup_rows_remote=14930 dedup_ratio=0.213847 router_tokens_per_expert=[81, 1110, 198, 297, 1992, 4457, 9, 3, 284, 1674, 656, 19, 253, 545, 2006, 168, 92, 27, 138, 2401, 0, 217, 712, 1, 261, 122, 105, 243, 5262, 330, 7, 3358, 9, 111, 50, 2, 823, 884, 0, 0, 1498, 1240, 2, 5012, 815, 51, 85, 998, 0, 4279, 26, 325, 227, 1, 118, 1979, 37, 84, 727, 2517, 184, 9, 25, 6] payload_rows_per_dst=[13890, 13676, 6674, 14026, 11390, 5766, 5184, 6676] dup_rows_per_dst=[5400, 6092, 502, 5350, 2392, 616, 168, 502] combine_rows_per_src=[13890, 13676, 6674, 14026, 11390, 5766, 5184, 6676] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=277528576 dispatch_meta_bytes=333784 combine_payload_bytes=277528576 prefetch_bytes=0 reduce_bytes=242221056 egress_bytes=277862360 ingress_bytes=519749632 link_bytes=519749632 token_bytes=555390936 weight_movement_bytes=242221056 total_nvlink_bytes=797611992 weight_to_token_ratio=0.436127 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=11776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.955776022e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76220 payload_rows_remote=66320 dup_rows=22084 dup_rows_remote=20112 dedup_ratio=0.224650 router_tokens_per_expert=[806, 194, 236, 1346, 177, 903, 1118, 37, 29, 597, 586, 37, 42, 1101, 57, 197, 143, 22, 3, 869, 8, 13, 813, 1589, 1176, 4126, 847, 374, 1705, 3696, 2373, 163, 36, 2174, 938, 246, 1505, 1117, 100, 100, 152, 1810, 37, 15, 862, 0, 0, 0, 5757, 411, 49, 2387, 51, 67, 34, 244, 10, 2753, 165, 87, 2536, 3, 2, 121] payload_rows_per_dst=[8232, 11342, 6560, 4530, 10962, 10530, 14164, 9900] dup_rows_per_dst=[1402, 2202, 360, 590, 4880, 2614, 8064, 1972] combine_rows_per_src=[8232, 11342, 6560, 4530, 10962, 10530, 14164, 9900] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=273907712 dispatch_meta_bytes=335160 combine_payload_bytes=273907712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274242872 ingress_bytes=308510720 link_bytes=308510720 token_bytes=548150584 weight_movement_bytes=34603008 total_nvlink_bytes=582753592 weight_to_token_ratio=0.063127 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=33280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.160901002e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76542 payload_rows_remote=65390 dup_rows=21762 dup_rows_remote=19288 dedup_ratio=0.221375 router_tokens_per_expert=[288, 658, 328, 288, 1619, 4132, 102, 6, 289, 1680, 556, 20, 597, 789, 1481, 109, 34, 13, 206, 2653, 0, 246, 366, 1, 333, 117, 114, 231, 5227, 471, 9, 2913, 37, 160, 123, 53, 598, 1004, 0, 0, 1380, 1623, 2, 5402, 1021, 21, 96, 1596, 1, 4062, 22, 601, 34, 1, 160, 1932, 62, 51, 891, 2078, 180, 35, 32, 18] payload_rows_per_dst=[6002, 8184, 11210, 6876, 8164, 11872, 11152, 13082] dup_rows_per_dst=[576, 2858, 4092, 698, 1372, 5626, 2474, 4066] combine_rows_per_src=[6002, 8184, 11210, 6876, 8164, 11872, 11152, 13082] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75970 payload_rows_remote=64056 dup_rows=22334 dup_rows_remote=18668 dedup_ratio=0.227193 router_tokens_per_expert=[877, 36, 9, 4, 61, 480, 0, 29, 2791, 1056, 2, 0, 676, 3229, 763, 474, 1211, 3948, 0, 419, 1226, 1321, 135, 501, 197, 0, 4881, 186, 261, 237, 95, 968, 389, 495, 463, 177, 87, 0, 1714, 43, 1349, 1023, 179, 44, 1, 5318, 833, 57, 11, 2004, 1, 118, 360, 1290, 2562, 0, 58, 162, 3850, 179, 62, 224, 2, 24] payload_rows_per_dst=[9894, 5424, 8834, 6908, 5912, 12882, 14202, 11914] dup_rows_per_dst=[994, 518, 2340, 1014, 824, 4726, 8252, 3666] combine_rows_per_src=[9894, 5424, 8834, 6908, 5912, 12882, 14202, 11914] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75210 payload_rows_remote=60460 dup_rows=23094 dup_rows_remote=13490 dedup_ratio=0.234924 router_tokens_per_expert=[1259, 43, 1, 0, 46, 806, 0, 57, 2355, 919, 2, 0, 748, 3360, 352, 612, 2259, 3663, 0, 250, 2086, 1895, 47, 379, 45, 0, 5965, 77, 121, 361, 29, 1169, 335, 273, 457, 59, 16, 0, 1181, 19, 997, 424, 97, 53, 15, 5622, 68, 6, 4, 2246, 0, 29, 565, 1259, 1714, 0, 28, 108, 4300, 122, 54, 170, 0, 25] payload_rows_per_dst=[10314, 8360, 10396, 5312, 4198, 14750, 9520, 12360] dup_rows_per_dst=[1436, 1616, 3436, 432, 482, 9604, 2114, 3974] combine_rows_per_src=[10314, 8360, 10396, 5312, 4198, 14750, 9520, 12360] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=271343616 dispatch_meta_bytes=319288 combine_payload_bytes=271343616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271662904 ingress_bytes=305946624 link_bytes=305946624 token_bytes=543006520 weight_movement_bytes=34603008 total_nvlink_bytes=577609528 weight_to_token_ratio=0.063725 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.151252515e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78408 payload_rows_remote=66828 dup_rows=19896 dup_rows_remote=15450 dedup_ratio=0.202393 router_tokens_per_expert=[9, 2152, 548, 3778, 98, 136, 72, 4, 136, 6, 327, 63, 526, 15, 2180, 12, 1950, 2, 775, 114, 432, 479, 21, 2515, 323, 913, 12, 69, 431, 2162, 133, 640, 3986, 99, 198, 228, 248, 14, 1485, 1755, 137, 13, 1407, 118, 6, 26, 29, 152, 170, 114, 3965, 1116, 5875, 1, 1282, 0, 463, 74, 0, 2742, 421, 138, 148, 1709] payload_rows_per_dst=[8718, 11588, 9998, 9674, 11580, 12910, 4662, 9278] dup_rows_per_dst=[1358, 2872, 2578, 3210, 4446, 2616, 704, 2112] combine_rows_per_src=[8718, 11588, 9998, 9674, 11580, 12910, 4662, 9278] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=299917312 dispatch_meta_bytes=357128 combine_payload_bytes=299917312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=300274440 ingress_bytes=334520320 link_bytes=334520320 token_bytes=600191752 weight_movement_bytes=34603008 total_nvlink_bytes=634794760 weight_to_token_ratio=0.057653 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=15104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258773033e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=281862144 dispatch_meta_bytes=323856 combine_payload_bytes=281862144 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=282186000 ingress_bytes=385671168 link_bytes=385671168 token_bytes=564048144 weight_movement_bytes=103809024 total_nvlink_bytes=667857168 weight_to_token_ratio=0.184043 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=13568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.451249556e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=268353536 dispatch_meta_bytes=327456 combine_payload_bytes=268353536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268680992 ingress_bytes=302956544 link_bytes=302956544 token_bytes=537034528 weight_movement_bytes=34603008 total_nvlink_bytes=571637536 weight_to_token_ratio=0.064433 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=33920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.140001085e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=260530176 dispatch_meta_bytes=314144 combine_payload_bytes=260530176 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260844320 ingress_bytes=295133184 link_bytes=295133184 token_bytes=521374496 weight_movement_bytes=34603008 total_nvlink_bytes=555977504 weight_to_token_ratio=0.066369 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.110562411e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76672 payload_rows_remote=62544 dup_rows=21632 dup_rows_remote=13186 dedup_ratio=0.220052 router_tokens_per_expert=[5, 3637, 788, 4335, 116, 53, 79, 21, 161, 12, 105, 54, 571, 0, 2142, 91, 666, 0, 512, 78, 335, 353, 24, 2572, 502, 921, 21, 54, 141, 1561, 314, 806, 2775, 122, 84, 344, 345, 1, 904, 1451, 110, 5, 1189, 239, 2, 34, 16, 97, 256, 221, 4691, 1841, 6054, 3, 1199, 3, 559, 164, 1, 3478, 477, 217, 156, 1084] payload_rows_per_dst=[14128, 12322, 8622, 7154, 6698, 11922, 6132, 9694] dup_rows_per_dst=[8446, 3332, 1502, 1486, 1340, 2034, 914, 2578] combine_rows_per_src=[14128, 12322, 8622, 7154, 6698, 11922, 6132, 9694] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=267837440 dispatch_meta_bytes=338712 combine_payload_bytes=267837440 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=268176152 ingress_bytes=337043456 link_bytes=337043456 token_bytes=536013592 weight_movement_bytes=69206016 total_nvlink_bytes=605219608 weight_to_token_ratio=0.129112 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.268267391e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=271646720 dispatch_meta_bytes=345728 combine_payload_bytes=271646720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271992448 ingress_bytes=306249728 link_bytes=306249728 token_bytes=543639168 weight_movement_bytes=34603008 total_nvlink_bytes=578242176 weight_to_token_ratio=0.063651 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=23040 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.152393071e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79216 payload_rows_remote=70646 dup_rows=19088 dup_rows_remote=16910 dedup_ratio=0.194173 router_tokens_per_expert=[72, 776, 303, 281, 2194, 3843, 57, 9, 109, 1247, 205, 17, 218, 549, 2214, 57, 42, 49, 44, 4158, 0, 104, 279, 0, 306, 20, 147, 316, 5703, 508, 27, 2726, 61, 96, 73, 15, 410, 823, 0, 0, 1486, 1352, 2, 4806, 1289, 8, 24, 1213, 3, 4306, 28, 637, 34, 0, 22, 2183, 90, 23, 1254, 1932, 277, 6, 93, 26] payload_rows_per_dst=[6836, 7364, 12366, 7398, 11018, 8570, 11874, 13790] dup_rows_per_dst=[548, 1868, 4672, 702, 1550, 2178, 2552, 5018] combine_rows_per_src=[6836, 7364, 12366, 7398, 11018, 8570, 11874, 13790] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=262373376 dispatch_meta_bytes=330896 combine_payload_bytes=262373376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262704272 ingress_bytes=296976384 link_bytes=296976384 token_bytes=525077648 weight_movement_bytes=34603008 total_nvlink_bytes=559680656 weight_to_token_ratio=0.065901 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.117498224e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=247644160 dispatch_meta_bytes=295800 combine_payload_bytes=247644160 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=247939960 ingress_bytes=316850176 link_bytes=316850176 token_bytes=495584120 weight_movement_bytes=69206016 total_nvlink_bytes=564790136 weight_to_token_ratio=0.139645 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=17792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.192281704e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=273727488 dispatch_meta_bytes=329112 combine_payload_bytes=273727488 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=274056600 ingress_bytes=412139520 link_bytes=412139520 token_bytes=547784088 weight_movement_bytes=138412032 total_nvlink_bytes=686196120 weight_to_token_ratio=0.252676 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=16128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.550847834e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=267837440 dispatch_meta_bytes=338712 combine_payload_bytes=267837440 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=268176152 ingress_bytes=337043456 link_bytes=337043456 token_bytes=536013592 weight_movement_bytes=69206016 total_nvlink_bytes=605219608 weight_to_token_ratio=0.129112 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.268267391e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79182 payload_rows_remote=65848 dup_rows=19122 dup_rows_remote=13950 dedup_ratio=0.194519 router_tokens_per_expert=[184, 1095, 338, 281, 1101, 4578, 12, 26, 483, 723, 274, 9, 253, 724, 2338, 101, 22, 84, 146, 2050, 0, 323, 345, 6, 351, 187, 280, 280, 4945, 465, 109, 2636, 70, 183, 102, 59, 331, 835, 2, 3, 1849, 2251, 4, 4954, 1132, 14, 42, 1980, 0, 3921, 26, 654, 34, 0, 53, 1591, 115, 105, 1607, 2264, 122, 34, 27, 39] payload_rows_per_dst=[10090, 7766, 8028, 13334, 11184, 10290, 10700, 7790] dup_rows_per_dst=[2476, 2044, 1622, 5172, 1894, 3220, 1858, 836] combine_rows_per_src=[10090, 7766, 8028, 13334, 11184, 10290, 10700, 7790] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=256180224 dispatch_meta_bytes=302920 combine_payload_bytes=256180224 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=256483144 ingress_bytes=325386240 link_bytes=325386240 token_bytes=512663368 weight_movement_bytes=69206016 total_nvlink_bytes=581869384 weight_to_token_ratio=0.134993 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=20224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.224402225e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82368 payload_rows_remote=73224 dup_rows=15936 dup_rows_remote=13624 dedup_ratio=0.162109 router_tokens_per_expert=[955, 226, 44, 4, 83, 864, 0, 10, 1723, 616, 160, 3, 564, 2759, 938, 837, 2265, 4872, 25, 310, 1348, 723, 53, 1004, 105, 14, 5828, 155, 200, 230, 34, 761, 314, 210, 77, 252, 47, 12, 1198, 33, 1993, 452, 86, 47, 50, 6509, 266, 75, 217, 1493, 4, 21, 66, 747, 2400, 3, 103, 140, 3896, 200, 171, 311, 20, 26] payload_rows_per_dst=[12072, 10894, 9144, 13122, 14236, 5552, 8294, 9054] dup_rows_per_dst=[2044, 4306, 2312, 1532, 3068, 386, 1608, 680] combine_rows_per_src=[12072, 10894, 9144, 13122, 14236, 5552, 8294, 9054] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81030 payload_rows_remote=67560 dup_rows=17274 dup_rows_remote=14252 dedup_ratio=0.175720 router_tokens_per_expert=[1764, 145, 9, 1, 22, 783, 0, 26, 1980, 880, 51, 0, 617, 2407, 542, 677, 1407, 2920, 0, 844, 2204, 2023, 47, 519, 78, 0, 5573, 55, 116, 685, 81, 1658, 230, 133, 595, 144, 139, 0, 814, 29, 1894, 805, 93, 164, 37, 5885, 33, 26, 18, 1980, 2, 106, 348, 786, 1881, 1, 61, 61, 4209, 112, 128, 306, 10, 8] payload_rows_per_dst=[9816, 10598, 10206, 13470, 13296, 5548, 8744, 9352] dup_rows_per_dst=[1524, 3710, 3882, 3022, 2642, 556, 1500, 438] combine_rows_per_src=[9816, 10598, 10206, 13470, 13296, 5548, 8744, 9352] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80530 payload_rows_remote=67778 dup_rows=17774 dup_rows_remote=17402 dedup_ratio=0.180806 router_tokens_per_expert=[26, 28, 3, 0, 0, 2841, 1178, 1398, 33, 7, 136, 94, 1, 47, 20, 1, 1024, 24, 1238, 3015, 6223, 678, 127, 63, 1752, 0, 75, 1077, 1, 4225, 33, 922, 12, 685, 1549, 1, 1, 567, 57, 0, 82, 12, 2627, 159, 0, 67, 1466, 0, 641, 337, 4552, 363, 4520, 184, 27, 389, 2786, 7, 0, 145, 1552, 11, 62, 1] payload_rows_per_dst=[11236, 12752, 10172, 9352, 11850, 13422, 3466, 8280] dup_rows_per_dst=[4824, 372, 2166, 1706, 2998, 4444, 416, 848] combine_rows_per_src=[11236, 12752, 10172, 9352, 11850, 13422, 3466, 8280] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78408 payload_rows_remote=66828 dup_rows=19896 dup_rows_remote=15450 dedup_ratio=0.202393 router_tokens_per_expert=[9, 2152, 548, 3778, 98, 136, 72, 4, 136, 6, 327, 63, 526, 15, 2180, 12, 1950, 2, 775, 114, 432, 479, 21, 2515, 323, 913, 12, 69, 431, 2162, 133, 640, 3986, 99, 198, 228, 248, 14, 1485, 1755, 137, 13, 1407, 118, 6, 26, 29, 152, 170, 114, 3965, 1116, 5875, 1, 1282, 0, 463, 74, 0, 2742, 421, 138, 148, 1709] payload_rows_per_dst=[8718, 11588, 9998, 9674, 11580, 12910, 4662, 9278] dup_rows_per_dst=[1358, 2872, 2578, 3210, 4446, 2616, 704, 2112] combine_rows_per_src=[8718, 11588, 9998, 9674, 11580, 12910, 4662, 9278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=289366016 dispatch_meta_bytes=350224 combine_payload_bytes=289366016 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=289716240 ingress_bytes=393175040 link_bytes=393175040 token_bytes=579082256 weight_movement_bytes=103809024 total_nvlink_bytes=682891280 weight_to_token_ratio=0.179265 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.479486022e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77608 payload_rows_remote=64340 dup_rows=20696 dup_rows_remote=14702 dedup_ratio=0.210531 router_tokens_per_expert=[893, 82, 420, 1508, 245, 1659, 925, 23, 20, 714, 1040, 47, 45, 829, 55, 120, 125, 105, 4, 1520, 90, 6, 514, 1983, 1221, 3480, 1274, 672, 2389, 3027, 2638, 385, 74, 1800, 1125, 243, 1216, 218, 34, 98, 157, 1924, 32, 44, 730, 2, 27, 3, 5284, 564, 112, 1423, 70, 18, 44, 271, 12, 2938, 175, 394, 1941, 3, 2, 116] payload_rows_per_dst=[12286, 10712, 13268, 9388, 8064, 9870, 4520, 9500] dup_rows_per_dst=[4500, 1988, 5994, 2494, 1552, 2022, 484, 1662] combine_rows_per_src=[12286, 10712, 13268, 9388, 8064, 9870, 4520, 9500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=269713408 dispatch_meta_bytes=319192 combine_payload_bytes=269713408 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=270032600 ingress_bytes=373522432 link_bytes=373522432 token_bytes=539746008 weight_movement_bytes=103809024 total_nvlink_bytes=643555032 weight_to_token_ratio=0.192329 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=19456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.405534841e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79946 payload_rows_remote=68174 dup_rows=18358 dup_rows_remote=16060 dedup_ratio=0.186747 router_tokens_per_expert=[6, 1832, 128, 4949, 151, 44, 198, 20, 214, 31, 236, 159, 934, 0, 1206, 110, 1059, 3, 391, 39, 377, 458, 48, 2837, 235, 1053, 6, 75, 149, 1366, 259, 350, 3230, 145, 242, 262, 295, 7, 2173, 2451, 104, 9, 1240, 336, 4, 7, 0, 142, 959, 187, 4145, 1527, 6795, 0, 626, 4, 910, 33, 0, 2517, 435, 137, 246, 1061] payload_rows_per_dst=[12404, 11772, 12098, 6008, 8870, 14394, 5742, 8658] dup_rows_per_dst=[2252, 2298, 4786, 978, 2280, 2880, 864, 2020] combine_rows_per_src=[12404, 11772, 12098, 6008, 8870, 14394, 5742, 8658] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71548 payload_rows_remote=57722 dup_rows=26756 dup_rows_remote=22126 dedup_ratio=0.272176 router_tokens_per_expert=[129, 590, 259, 330, 1841, 3899, 77, 5, 162, 723, 573, 12, 409, 386, 2062, 38, 108, 116, 136, 4285, 0, 305, 325, 0, 171, 144, 206, 333, 5908, 232, 32, 2176, 32, 155, 62, 13, 281, 711, 0, 0, 1163, 1460, 0, 5558, 1022, 11, 275, 1889, 0, 3673, 34, 509, 22, 0, 184, 2806, 25, 31, 650, 2272, 254, 9, 51, 28] payload_rows_per_dst=[5916, 7070, 12810, 2172, 2450, 15084, 12220, 13826] dup_rows_per_dst=[546, 1660, 5538, 64, 58, 10258, 4002, 4630] combine_rows_per_src=[5916, 7070, 12810, 2172, 2450, 15084, 12220, 13826] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=273727488 dispatch_meta_bytes=329112 combine_payload_bytes=273727488 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=274056600 ingress_bytes=412139520 link_bytes=412139520 token_bytes=547784088 weight_movement_bytes=138412032 total_nvlink_bytes=686196120 weight_to_token_ratio=0.252676 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=16128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.550847834e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=263536640 dispatch_meta_bytes=316168 combine_payload_bytes=263536640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263852808 ingress_bytes=298139648 link_bytes=298139648 token_bytes=527389448 weight_movement_bytes=34603008 total_nvlink_bytes=561992456 weight_to_token_ratio=0.065612 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.121875493e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72768 payload_rows_remote=57666 dup_rows=25536 dup_rows_remote=14762 dedup_ratio=0.259766 router_tokens_per_expert=[0, 1879, 776, 4577, 73, 36, 278, 18, 164, 6, 77, 114, 865, 0, 1986, 22, 1315, 10, 775, 26, 419, 1003, 106, 2374, 239, 1351, 7, 145, 402, 1800, 220, 371, 2994, 128, 78, 366, 300, 8, 833, 2151, 290, 14, 862, 151, 9, 4, 1, 172, 470, 131, 2979, 2011, 6503, 4, 932, 0, 418, 66, 0, 3534, 475, 89, 378, 1367] payload_rows_per_dst=[5710, 6248, 10012, 12636, 10220, 2884, 15102, 9956] dup_rows_per_dst=[410, 404, 2044, 5588, 3496, 122, 10774, 2698] combine_rows_per_src=[5710, 6248, 10012, 12636, 10220, 2884, 15102, 9956] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=299925504 dispatch_meta_bytes=347392 combine_payload_bytes=299925504 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=300272896 ingress_bytes=403734528 link_bytes=403734528 token_bytes=600198400 weight_movement_bytes=103809024 total_nvlink_bytes=704007424 weight_to_token_ratio=0.172958 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=17280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519220526e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76162 payload_rows_remote=63596 dup_rows=22142 dup_rows_remote=17754 dedup_ratio=0.225240 router_tokens_per_expert=[123, 5, 42, 4, 1, 2698, 629, 1981, 52, 4, 252, 259, 1, 29, 44, 2, 1729, 29, 1003, 3130, 5800, 781, 323, 72, 957, 1, 188, 1253, 17, 3192, 71, 828, 12, 741, 1542, 13, 0, 419, 248, 1, 279, 10, 2705, 236, 3, 123, 1048, 0, 621, 71, 4564, 292, 4434, 234, 314, 457, 3585, 17, 1, 358, 1176, 3, 144, 1] payload_rows_per_dst=[9122, 4574, 12010, 10466, 5126, 7538, 14760, 12566] dup_rows_per_dst=[1844, 170, 4006, 2424, 826, 1270, 7214, 4388] combine_rows_per_src=[9122, 4574, 12010, 10466, 5126, 7538, 14760, 12566] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81392 payload_rows_remote=70370 dup_rows=16912 dup_rows_remote=14210 dedup_ratio=0.172038 router_tokens_per_expert=[724, 219, 32, 2, 53, 616, 0, 21, 2378, 880, 14, 0, 597, 2789, 772, 212, 2664, 5573, 0, 378, 1701, 936, 68, 430, 166, 0, 6110, 117, 64, 535, 57, 971, 550, 512, 221, 74, 20, 1, 737, 93, 1574, 579, 347, 54, 42, 5976, 109, 26, 164, 1907, 0, 47, 476, 1034, 1382, 0, 42, 78, 3577, 124, 41, 245, 37, 4] payload_rows_per_dst=[12746, 9656, 9868, 13896, 11022, 6858, 8110, 9236] dup_rows_per_dst=[1734, 3034, 2486, 2144, 2702, 1248, 1910, 1654] combine_rows_per_src=[12746, 9656, 9868, 13896, 11022, 6858, 8110, 9236] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78878 payload_rows_remote=71336 dup_rows=19426 dup_rows_remote=18944 dedup_ratio=0.197611 router_tokens_per_expert=[21, 9, 15, 0, 1, 3371, 617, 966, 129, 0, 244, 75, 1, 33, 1, 0, 1153, 77, 969, 2047, 6520, 377, 80, 193, 1896, 0, 113, 1214, 1, 4207, 59, 449, 6, 416, 1074, 0, 0, 1232, 48, 0, 28, 14, 3031, 83, 0, 151, 705, 0, 890, 442, 4681, 445, 5419, 164, 114, 624, 3450, 33, 0, 169, 1012, 1, 81, 1] payload_rows_per_dst=[8496, 13362, 8578, 10534, 4790, 7542, 15464, 10112] dup_rows_per_dst=[1504, 644, 1214, 2484, 762, 482, 10094, 2242] combine_rows_per_src=[8496, 13362, 8578, 10534, 4790, 7542, 15464, 10112] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=277618688 dispatch_meta_bytes=340720 combine_payload_bytes=277618688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277959408 ingress_bytes=312221696 link_bytes=312221696 token_bytes=555578096 weight_movement_bytes=34603008 total_nvlink_bytes=590181104 weight_to_token_ratio=0.062283 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=44800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.174865106e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=276725760 dispatch_meta_bytes=327248 combine_payload_bytes=276725760 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=277053008 ingress_bytes=380534784 link_bytes=380534784 token_bytes=553778768 weight_movement_bytes=103809024 total_nvlink_bytes=657587792 weight_to_token_ratio=0.187456 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=30848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.431921757e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=279240704 dispatch_meta_bytes=336936 combine_payload_bytes=279240704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279577640 ingress_bytes=313843712 link_bytes=313843712 token_bytes=558818344 weight_movement_bytes=34603008 total_nvlink_bytes=593421352 weight_to_token_ratio=0.061922 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=25856 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.180968622e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82602 payload_rows_remote=72026 dup_rows=15702 dup_rows_remote=13306 dedup_ratio=0.159729 router_tokens_per_expert=[951, 98, 11, 0, 4, 965, 0, 29, 890, 623, 15, 0, 462, 2507, 741, 275, 3362, 3604, 0, 483, 1091, 990, 55, 399, 131, 0, 6397, 131, 159, 550, 40, 1066, 117, 360, 584, 29, 39, 0, 1434, 37, 2315, 387, 228, 109, 29, 6162, 352, 26, 406, 2087, 1, 40, 63, 1045, 1486, 0, 28, 143, 4917, 146, 234, 300, 10, 9] payload_rows_per_dst=[10576, 8762, 8940, 14520, 14172, 6356, 8420, 10856] dup_rows_per_dst=[2396, 2264, 2172, 2428, 3352, 536, 1836, 718] combine_rows_per_src=[10576, 8762, 8940, 14520, 14172, 6356, 8420, 10856] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=236199936 dispatch_meta_bytes=289712 combine_payload_bytes=236199936 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=236489648 ingress_bytes=340008960 link_bytes=340008960 token_bytes=472689584 weight_movement_bytes=103809024 total_nvlink_bytes=576498608 weight_to_token_ratio=0.219614 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.279426344e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=236429312 dispatch_meta_bytes=319392 combine_payload_bytes=236429312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=236748704 ingress_bytes=271032320 link_bytes=271032320 token_bytes=473178016 weight_movement_bytes=34603008 total_nvlink_bytes=507781024 weight_to_token_ratio=0.073129 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=21376 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.019872800e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79006 payload_rows_remote=68672 dup_rows=19298 dup_rows_remote=17760 dedup_ratio=0.196309 router_tokens_per_expert=[2, 2236, 841, 4213, 110, 36, 340, 1, 92, 13, 79, 78, 370, 0, 1408, 41, 1280, 3, 568, 23, 653, 297, 60, 1024, 229, 2231, 14, 79, 53, 1366, 336, 457, 4405, 107, 224, 446, 380, 1, 884, 1827, 157, 7, 1062, 102, 0, 2, 3, 173, 451, 328, 5255, 1291, 6709, 0, 1020, 0, 509, 64, 0, 3659, 435, 41, 303, 774] payload_rows_per_dst=[6672, 12494, 6836, 12710, 11976, 10334, 8544, 9440] dup_rows_per_dst=[460, 2178, 980, 5246, 4572, 1538, 2194, 2130] combine_rows_per_src=[6672, 12494, 6836, 12710, 11976, 10334, 8544, 9440] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=260489216 dispatch_meta_bytes=325400 combine_payload_bytes=260489216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260814616 ingress_bytes=295092224 link_bytes=295092224 token_bytes=521303832 weight_movement_bytes=34603008 total_nvlink_bytes=555906840 weight_to_token_ratio=0.066378 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=27264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.110408282e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=288235520 dispatch_meta_bytes=338320 combine_payload_bytes=288235520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288573840 ingress_bytes=322838528 link_bytes=322838528 token_bytes=576809360 weight_movement_bytes=34603008 total_nvlink_bytes=611412368 weight_to_token_ratio=0.059990 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=30080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.214815390e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=292192256 dispatch_meta_bytes=361120 combine_payload_bytes=292192256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=292553376 ingress_bytes=326795264 link_bytes=326795264 token_bytes=584745632 weight_movement_bytes=34603008 total_nvlink_bytes=619348640 weight_to_token_ratio=0.059176 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=19328 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.229704269e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=236199936 dispatch_meta_bytes=289712 combine_payload_bytes=236199936 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=236489648 ingress_bytes=340008960 link_bytes=340008960 token_bytes=472689584 weight_movement_bytes=103809024 total_nvlink_bytes=576498608 weight_to_token_ratio=0.219614 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.279426344e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80598 payload_rows_remote=73222 dup_rows=17706 dup_rows_remote=16060 dedup_ratio=0.180115 router_tokens_per_expert=[3, 2646, 494, 4233, 119, 48, 237, 16, 88, 8, 120, 17, 337, 4, 1271, 54, 658, 0, 444, 15, 540, 441, 34, 1928, 223, 1752, 17, 142, 45, 1623, 388, 321, 3595, 180, 276, 448, 589, 1, 471, 1195, 371, 14, 632, 348, 6, 49, 2, 160, 288, 250, 4015, 1578, 7045, 30, 1812, 0, 996, 51, 0, 4278, 603, 135, 386, 1082] payload_rows_per_dst=[12478, 10380, 8838, 7376, 8888, 14626, 6692, 11320] dup_rows_per_dst=[3114, 1448, 1740, 1646, 2164, 2628, 1224, 3742] combine_rows_per_src=[12478, 10380, 8838, 7376, 8888, 14626, 6692, 11320] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=295018496 dispatch_meta_bytes=341328 combine_payload_bytes=295018496 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=295359824 ingress_bytes=364224512 link_bytes=364224512 token_bytes=590378320 weight_movement_bytes=69206016 total_nvlink_bytes=659584336 weight_to_token_ratio=0.117223 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=31744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.370547516e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82020 payload_rows_remote=72634 dup_rows=16284 dup_rows_remote=14828 dedup_ratio=0.165649 router_tokens_per_expert=[31, 270, 12, 6, 4, 4185, 870, 1415, 47, 149, 281, 133, 2, 96, 2, 7, 1031, 46, 2261, 1327, 6076, 595, 118, 43, 1215, 2, 53, 817, 26, 3099, 116, 559, 51, 420, 563, 5, 13, 1360, 208, 1, 66, 226, 2190, 373, 109, 58, 680, 0, 911, 425, 4770, 739, 5037, 230, 398, 492, 3063, 10, 1, 320, 1443, 22, 71, 3] payload_rows_per_dst=[11022, 12738, 9386, 10054, 11494, 13180, 5370, 8776] dup_rows_per_dst=[2564, 848, 1456, 1720, 3288, 4298, 1020, 1090] combine_rows_per_src=[11022, 12738, 9386, 10054, 11494, 13180, 5370, 8776] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81312 payload_rows_remote=67132 dup_rows=16992 dup_rows_remote=13332 dedup_ratio=0.172852 router_tokens_per_expert=[69, 17, 30, 3, 1, 2355, 1388, 487, 16, 0, 306, 138, 1, 30, 21, 0, 1160, 12, 1574, 2124, 5071, 732, 19, 12, 1207, 0, 295, 1124, 3, 5700, 80, 511, 12, 1437, 1891, 8, 0, 838, 35, 0, 25, 20, 2075, 36, 2, 62, 1042, 0, 436, 153, 4793, 660, 4205, 175, 48, 796, 4185, 24, 2, 148, 1342, 2, 213, 1] payload_rows_per_dst=[7264, 10656, 9256, 14180, 13056, 12066, 4276, 10558] dup_rows_per_dst=[1436, 510, 2010, 3660, 4972, 2868, 260, 1276] combine_rows_per_src=[7264, 10656, 9256, 14180, 13056, 12066, 4276, 10558] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80304 payload_rows_remote=65954 dup_rows=18000 dup_rows_remote=11466 dedup_ratio=0.183105 router_tokens_per_expert=[14, 451, 0, 0, 483, 87, 88, 2, 20, 31, 3952, 0, 1322, 3294, 1823, 0, 91, 2, 355, 55, 1, 0, 0, 2011, 580, 706, 5179, 0, 6, 34, 0, 19, 281, 0, 259, 545, 0, 1, 1042, 38, 1357, 1486, 178, 7112, 35, 0, 1341, 5726, 641, 11, 143, 202, 138, 1, 262, 2659, 1436, 821, 53, 2148, 8, 0, 390, 232] payload_rows_per_dst=[14474, 14350, 4634, 12612, 12802, 5388, 7510, 8534] dup_rows_per_dst=[2000, 6534, 396, 3408, 2982, 434, 604, 1642] combine_rows_per_src=[14474, 14350, 4634, 12612, 12802, 5388, 7510, 8534] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=297508864 dispatch_meta_bytes=349848 combine_payload_bytes=297508864 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=297858712 ingress_bytes=401317888 link_bytes=401317888 token_bytes=595367576 weight_movement_bytes=103809024 total_nvlink_bytes=699176600 weight_to_token_ratio=0.174361 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.510126904e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81392 payload_rows_remote=70370 dup_rows=16912 dup_rows_remote=14210 dedup_ratio=0.172038 router_tokens_per_expert=[724, 219, 32, 2, 53, 616, 0, 21, 2378, 880, 14, 0, 597, 2789, 772, 212, 2664, 5573, 0, 378, 1701, 936, 68, 430, 166, 0, 6110, 117, 64, 535, 57, 971, 550, 512, 221, 74, 20, 1, 737, 93, 1574, 579, 347, 54, 42, 5976, 109, 26, 164, 1907, 0, 47, 476, 1034, 1382, 0, 42, 78, 3577, 124, 41, 245, 37, 4] payload_rows_per_dst=[12746, 9656, 9868, 13896, 11022, 6858, 8110, 9236] dup_rows_per_dst=[1734, 3034, 2486, 2144, 2702, 1248, 1910, 1654] combine_rows_per_src=[12746, 9656, 9868, 13896, 11022, 6858, 8110, 9236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76346 payload_rows_remote=70090 dup_rows=21958 dup_rows_remote=21390 dedup_ratio=0.223368 router_tokens_per_expert=[109, 1083, 712, 498, 2366, 4362, 40, 13, 380, 1706, 629, 65, 322, 551, 1404, 281, 334, 70, 228, 2308, 18, 178, 276, 0, 337, 177, 105, 330, 3093, 455, 7, 3170, 24, 173, 54, 16, 1057, 1641, 2, 37, 975, 1083, 0, 4746, 1070, 6, 124, 1390, 0, 4287, 20, 1187, 372, 2, 52, 2447, 114, 30, 604, 1648, 307, 3, 61, 13] payload_rows_per_dst=[13736, 8880, 6256, 11670, 11802, 6434, 12344, 5224] dup_rows_per_dst=[6580, 2214, 568, 3678, 3698, 912, 3972, 336] combine_rows_per_src=[13736, 8880, 6256, 11670, 11802, 6434, 12344, 5224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=281280512 dispatch_meta_bytes=345728 combine_payload_bytes=281280512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281626240 ingress_bytes=315883520 link_bytes=315883520 token_bytes=562906752 weight_movement_bytes=34603008 total_nvlink_bytes=597509760 weight_to_token_ratio=0.061472 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=35968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.188644255e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=299917312 dispatch_meta_bytes=357128 combine_payload_bytes=299917312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=300274440 ingress_bytes=334520320 link_bytes=334520320 token_bytes=600191752 weight_movement_bytes=34603008 total_nvlink_bytes=634794760 weight_to_token_ratio=0.057653 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=15104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258773033e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80642 payload_rows_remote=68814 dup_rows=17662 dup_rows_remote=12150 dedup_ratio=0.179667 router_tokens_per_expert=[1054, 385, 7, 1, 48, 614, 0, 23, 2368, 1129, 23, 0, 650, 3204, 732, 564, 2116, 3702, 0, 269, 997, 1400, 70, 952, 94, 2, 5854, 55, 307, 474, 18, 1340, 530, 202, 377, 25, 59, 0, 1172, 7, 1344, 379, 108, 59, 26, 6805, 206, 45, 119, 1588, 0, 68, 82, 1053, 1766, 0, 58, 93, 3977, 148, 129, 256, 4, 15] payload_rows_per_dst=[10126, 11828, 9214, 13664, 14740, 4122, 8042, 8906] dup_rows_per_dst=[1542, 5512, 2394, 2624, 3614, 212, 1310, 454] combine_rows_per_src=[10126, 11828, 9214, 13664, 14740, 4122, 8042, 8906] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=288235520 dispatch_meta_bytes=338320 combine_payload_bytes=288235520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288573840 ingress_bytes=322838528 link_bytes=322838528 token_bytes=576809360 weight_movement_bytes=34603008 total_nvlink_bytes=611412368 weight_to_token_ratio=0.059990 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=30080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.214815390e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73380 payload_rows_remote=62750 dup_rows=24924 dup_rows_remote=21070 dedup_ratio=0.253540 router_tokens_per_expert=[10, 3425, 595, 4543, 183, 101, 267, 13, 146, 12, 85, 113, 1098, 0, 1466, 16, 934, 0, 701, 58, 658, 338, 116, 2089, 681, 597, 49, 60, 61, 1911, 364, 423, 4296, 187, 213, 455, 457, 4, 1035, 737, 130, 3, 1312, 125, 6, 13, 7, 267, 333, 669, 3689, 1804, 5786, 1, 1208, 7, 387, 23, 4, 2394, 425, 84, 496, 1482] payload_rows_per_dst=[8386, 5506, 8496, 12830, 8774, 3574, 15184, 10630] dup_rows_per_dst=[802, 366, 1292, 4548, 2100, 152, 11810, 3854] combine_rows_per_src=[8386, 5506, 8496, 12830, 8774, 3574, 15184, 10630] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=287088640 dispatch_meta_bytes=365920 combine_payload_bytes=287088640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=287454560 ingress_bytes=321691648 link_bytes=321691648 token_bytes=574543200 weight_movement_bytes=34603008 total_nvlink_bytes=609146208 weight_to_token_ratio=0.060227 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.210499773e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73568 payload_rows_remote=62770 dup_rows=24736 dup_rows_remote=21092 dedup_ratio=0.251628 router_tokens_per_expert=[842, 280, 0, 4, 64, 949, 0, 14, 2698, 1481, 30, 0, 1161, 2932, 776, 652, 2733, 3392, 0, 282, 943, 1164, 49, 282, 212, 0, 6032, 112, 112, 545, 59, 1068, 517, 161, 273, 174, 72, 0, 789, 36, 1107, 561, 531, 38, 19, 5528, 70, 22, 56, 1809, 1, 74, 514, 1034, 2304, 5, 21, 90, 3939, 132, 88, 307, 12, 0] payload_rows_per_dst=[9558, 9028, 9818, 4044, 3596, 14924, 10798, 11802] dup_rows_per_dst=[1532, 2794, 2862, 172, 448, 10044, 3644, 3240] combine_rows_per_src=[9558, 9028, 9818, 4044, 3596, 14924, 10798, 11802] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81282 payload_rows_remote=69890 dup_rows=17022 dup_rows_remote=12640 dedup_ratio=0.173157 router_tokens_per_expert=[16, 26, 13, 0, 0, 3314, 910, 908, 2, 8, 343, 155, 4, 68, 70, 1, 1127, 7, 1195, 2160, 6247, 1148, 117, 29, 881, 1, 27, 1291, 2, 4806, 36, 799, 66, 1205, 1230, 1, 0, 620, 299, 5, 67, 30, 2706, 95, 1, 315, 898, 0, 556, 273, 6042, 360, 3305, 173, 116, 669, 3389, 42, 1, 135, 778, 1, 62, 1] payload_rows_per_dst=[8570, 13074, 9574, 12978, 11392, 10526, 6850, 8318] dup_rows_per_dst=[1804, 722, 1992, 2708, 4382, 2698, 2216, 500] combine_rows_per_src=[8570, 13074, 9574, 12978, 11392, 10526, 6850, 8318] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=274972672 dispatch_meta_bytes=321856 combine_payload_bytes=274972672 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=275294528 ingress_bytes=378781696 link_bytes=378781696 token_bytes=550267200 weight_movement_bytes=103809024 total_nvlink_bytes=654076224 weight_to_token_ratio=0.188652 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=18176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.425325028e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=270147584 dispatch_meta_bytes=309680 combine_payload_bytes=270147584 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=270457264 ingress_bytes=339353600 link_bytes=339353600 token_bytes=540604848 weight_movement_bytes=69206016 total_nvlink_bytes=609810864 weight_to_token_ratio=0.128016 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.276960277e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70530 payload_rows_remote=55686 dup_rows=27774 dup_rows_remote=20098 dedup_ratio=0.282532 router_tokens_per_expert=[77, 54, 0, 8, 451, 102, 67, 6, 19, 184, 2611, 1, 1520, 2216, 1265, 0, 102, 10, 238, 39, 17, 2, 0, 3152, 425, 558, 5967, 2, 2, 22, 9, 4, 243, 0, 338, 730, 0, 21, 1122, 177, 1787, 2716, 560, 6874, 7, 0, 973, 5275, 1001, 13, 266, 168, 385, 2, 166, 1905, 1615, 1221, 238, 1877, 3, 0, 113, 226] payload_rows_per_dst=[1518, 7100, 15050, 2004, 5038, 14506, 10470, 14844] dup_rows_per_dst=[12, 990, 5818, 40, 224, 10450, 2564, 7676] combine_rows_per_src=[1518, 7100, 15050, 2004, 5038, 14506, 10470, 14844] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=281862144 dispatch_meta_bytes=323856 combine_payload_bytes=281862144 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=282186000 ingress_bytes=385671168 link_bytes=385671168 token_bytes=564048144 weight_movement_bytes=103809024 total_nvlink_bytes=667857168 weight_to_token_ratio=0.184043 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=13568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.451249556e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78754 payload_rows_remote=65022 dup_rows=19550 dup_rows_remote=13114 dedup_ratio=0.198873 router_tokens_per_expert=[22, 14, 17, 0, 3, 2253, 980, 1197, 11, 3, 300, 40, 10, 118, 12, 0, 2082, 10, 2193, 1851, 6304, 1123, 184, 28, 968, 0, 200, 1152, 3, 5598, 58, 402, 27, 494, 952, 2, 2, 514, 404, 0, 31, 30, 2283, 161, 5, 122, 836, 0, 729, 236, 4899, 501, 4235, 181, 207, 278, 3262, 11, 6, 134, 1256, 155, 63, 0] payload_rows_per_dst=[13732, 12946, 12494, 5268, 11936, 9896, 3718, 8764] dup_rows_per_dst=[6436, 650, 5622, 298, 2652, 2336, 546, 1010] combine_rows_per_src=[13732, 12946, 12494, 5268, 11936, 9896, 3718, 8764] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78924 payload_rows_remote=69122 dup_rows=19380 dup_rows_remote=16936 dedup_ratio=0.197144 router_tokens_per_expert=[23, 193, 0, 0, 578, 313, 62, 0, 57, 72, 3251, 0, 1405, 2078, 1221, 0, 153, 0, 354, 31, 1, 0, 0, 2665, 795, 718, 4326, 0, 0, 53, 1, 13, 245, 0, 228, 665, 0, 0, 1849, 9, 2079, 2134, 300, 7292, 35, 0, 1575, 4023, 1594, 12, 263, 104, 343, 4, 574, 1729, 1392, 2135, 59, 1830, 5, 0, 238, 73] payload_rows_per_dst=[5598, 8444, 12674, 10318, 11056, 9802, 11908, 9124] dup_rows_per_dst=[724, 1222, 4334, 1494, 2982, 2444, 3840, 2340] combine_rows_per_src=[5598, 8444, 12674, 10318, 11056, 9802, 11908, 9124] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=257105920 dispatch_meta_bytes=335448 combine_payload_bytes=257105920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=257441368 ingress_bytes=291708928 link_bytes=291708928 token_bytes=514547288 weight_movement_bytes=34603008 total_nvlink_bytes=549150296 weight_to_token_ratio=0.067249 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=15616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.097677212e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=257024000 dispatch_meta_bytes=335280 combine_payload_bytes=257024000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=257359280 ingress_bytes=291627008 link_bytes=291627008 token_bytes=514383280 weight_movement_bytes=34603008 total_nvlink_bytes=548986288 weight_to_token_ratio=0.067271 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=25088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.097368953e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 2, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=286269440 dispatch_meta_bytes=330120 combine_payload_bytes=286269440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286599560 ingress_bytes=320872448 link_bytes=320872448 token_bytes=572869000 weight_movement_bytes=34603008 total_nvlink_bytes=607472008 weight_to_token_ratio=0.060403 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.207417189e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=228089856 dispatch_meta_bytes=303136 combine_payload_bytes=228089856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=228392992 ingress_bytes=262692864 link_bytes=262692864 token_bytes=456482848 weight_movement_bytes=34603008 total_nvlink_bytes=491085856 weight_to_token_ratio=0.075804 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.884920987e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=257105920 dispatch_meta_bytes=335448 combine_payload_bytes=257105920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=257441368 ingress_bytes=291708928 link_bytes=291708928 token_bytes=514547288 weight_movement_bytes=34603008 total_nvlink_bytes=549150296 weight_to_token_ratio=0.067249 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=15616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.097677212e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75210 payload_rows_remote=60460 dup_rows=23094 dup_rows_remote=13490 dedup_ratio=0.234924 router_tokens_per_expert=[1259, 43, 1, 0, 46, 806, 0, 57, 2355, 919, 2, 0, 748, 3360, 352, 612, 2259, 3663, 0, 250, 2086, 1895, 47, 379, 45, 0, 5965, 77, 121, 361, 29, 1169, 335, 273, 457, 59, 16, 0, 1181, 19, 997, 424, 97, 53, 15, 5622, 68, 6, 4, 2246, 0, 29, 565, 1259, 1714, 0, 28, 108, 4300, 122, 54, 170, 0, 25] payload_rows_per_dst=[10314, 8360, 10396, 5312, 4198, 14750, 9520, 12360] dup_rows_per_dst=[1436, 1616, 3436, 432, 482, 9604, 2114, 3974] combine_rows_per_src=[10314, 8360, 10396, 5312, 4198, 14750, 9520, 12360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81030 payload_rows_remote=67560 dup_rows=17274 dup_rows_remote=14252 dedup_ratio=0.175720 router_tokens_per_expert=[1764, 145, 9, 1, 22, 783, 0, 26, 1980, 880, 51, 0, 617, 2407, 542, 677, 1407, 2920, 0, 844, 2204, 2023, 47, 519, 78, 0, 5573, 55, 116, 685, 81, 1658, 230, 133, 595, 144, 139, 0, 814, 29, 1894, 805, 93, 164, 37, 5885, 33, 26, 18, 1980, 2, 106, 348, 786, 1881, 1, 61, 61, 4209, 112, 128, 306, 10, 8] payload_rows_per_dst=[9816, 10598, 10206, 13470, 13296, 5548, 8744, 9352] dup_rows_per_dst=[1524, 3710, 3882, 3022, 2642, 556, 1500, 438] combine_rows_per_src=[9816, 10598, 10206, 13470, 13296, 5548, 8744, 9352] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266330112 dispatch_meta_bytes=312544 combine_payload_bytes=266330112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266642656 ingress_bytes=300933120 link_bytes=300933120 token_bytes=532972768 weight_movement_bytes=34603008 total_nvlink_bytes=567575776 weight_to_token_ratio=0.064925 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=23552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132387103e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=283123712 dispatch_meta_bytes=344232 combine_payload_bytes=283123712 prefetch_bytes=34603008 reduce_bytes=173015040 egress_bytes=283467944 ingress_bytes=490741760 link_bytes=490741760 token_bytes=566591656 weight_movement_bytes=207618048 total_nvlink_bytes=774209704 weight_to_token_ratio=0.366433 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=12800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.846621735e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80052 payload_rows_remote=73736 dup_rows=18252 dup_rows_remote=17392 dedup_ratio=0.185669 router_tokens_per_expert=[61, 515, 0, 8, 1514, 707, 77, 75, 150, 101, 3484, 2, 1188, 2082, 873, 3, 231, 2, 977, 73, 7, 0, 0, 2298, 738, 905, 5791, 5, 2, 50, 0, 12, 294, 5, 450, 650, 3, 19, 981, 112, 2021, 1677, 341, 6606, 63, 0, 1891, 4761, 1052, 12, 258, 146, 218, 14, 327, 1279, 1172, 1035, 37, 1372, 13, 8, 238, 166] payload_rows_per_dst=[14624, 12396, 6316, 12824, 11846, 9020, 6086, 6940] dup_rows_per_dst=[4502, 3370, 860, 3316, 2704, 1832, 526, 1142] combine_rows_per_src=[14624, 12396, 6316, 12824, 11846, 9020, 6086, 6940] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83090 payload_rows_remote=71202 dup_rows=15214 dup_rows_remote=14146 dedup_ratio=0.154765 router_tokens_per_expert=[119, 127, 0, 0, 578, 227, 190, 0, 63, 158, 3023, 8, 1110, 2579, 808, 0, 219, 7, 775, 119, 0, 3, 0, 3452, 420, 396, 5639, 0, 0, 21, 0, 2, 216, 4, 380, 560, 0, 0, 1132, 19, 2366, 1699, 151, 7409, 21, 0, 810, 4103, 1710, 7, 162, 145, 164, 4, 756, 3088, 880, 1154, 39, 1899, 4, 1, 157, 69] payload_rows_per_dst=[15038, 9828, 8030, 11888, 10554, 8632, 12100, 7020] dup_rows_per_dst=[2262, 1804, 1120, 1068, 2274, 1462, 3838, 1386] combine_rows_per_src=[15038, 9828, 8030, 11888, 10554, 8632, 12100, 7020] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70334 payload_rows_remote=54108 dup_rows=27970 dup_rows_remote=11192 dedup_ratio=0.284526 router_tokens_per_expert=[3461, 13, 0, 846, 25, 3, 3483, 534, 5217, 7637, 82, 6, 0, 89, 4, 113, 241, 12, 1835, 355, 266, 220, 28, 6, 485, 1523, 29, 14, 198, 0, 20, 371, 0, 18, 328, 816, 1, 604, 16, 1, 182, 641, 348, 773, 2188, 839, 336, 525, 4456, 1, 4, 9, 3, 218, 0, 6, 578, 50, 75, 7, 5, 1180, 3969, 3859] payload_rows_per_dst=[12542, 16226, 10742, 4674, 3044, 9278, 9224, 4604] dup_rows_per_dst=[4188, 16778, 2902, 606, 524, 2386, 170, 416] combine_rows_per_src=[12542, 16226, 10742, 4674, 3044, 9278, 9224, 4604] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=302022656 dispatch_meta_bytes=364512 combine_payload_bytes=302022656 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=302387168 ingress_bytes=371228672 link_bytes=371228672 token_bytes=604409824 weight_movement_bytes=69206016 total_nvlink_bytes=673615840 weight_to_token_ratio=0.114502 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.022500 gemm_active_groups=11 gemm_peak_group_rows=22016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396903606e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81282 payload_rows_remote=69890 dup_rows=17022 dup_rows_remote=12640 dedup_ratio=0.173157 router_tokens_per_expert=[16, 26, 13, 0, 0, 3314, 910, 908, 2, 8, 343, 155, 4, 68, 70, 1, 1127, 7, 1195, 2160, 6247, 1148, 117, 29, 881, 1, 27, 1291, 2, 4806, 36, 799, 66, 1205, 1230, 1, 0, 620, 299, 5, 67, 30, 2706, 95, 1, 315, 898, 0, 556, 273, 6042, 360, 3305, 173, 116, 669, 3389, 42, 1, 135, 778, 1, 62, 1] payload_rows_per_dst=[8570, 13074, 9574, 12978, 11392, 10526, 6850, 8318] dup_rows_per_dst=[1804, 722, 1992, 2708, 4382, 2698, 2216, 500] combine_rows_per_src=[8570, 13074, 9574, 12978, 11392, 10526, 6850, 8318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78690 payload_rows_remote=66246 dup_rows=19614 dup_rows_remote=13576 dedup_ratio=0.199524 router_tokens_per_expert=[40, 2359, 259, 3350, 151, 114, 176, 34, 165, 13, 1074, 94, 395, 4, 2073, 90, 585, 33, 688, 60, 843, 773, 59, 2500, 401, 639, 45, 50, 182, 1418, 413, 459, 3700, 280, 221, 390, 434, 2, 1705, 2418, 114, 10, 501, 323, 6, 14, 17, 413, 823, 271, 3640, 1764, 6447, 5, 1017, 1, 531, 52, 0, 2379, 382, 158, 149, 1446] payload_rows_per_dst=[11018, 11876, 12444, 6160, 8554, 13664, 6662, 8312] dup_rows_per_dst=[1948, 3220, 6038, 1054, 2346, 2026, 1100, 1882] combine_rows_per_src=[11018, 11876, 12444, 6160, 8554, 13664, 6662, 8312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=247644160 dispatch_meta_bytes=295800 combine_payload_bytes=247644160 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=247939960 ingress_bytes=316850176 link_bytes=316850176 token_bytes=495584120 weight_movement_bytes=69206016 total_nvlink_bytes=564790136 weight_to_token_ratio=0.139645 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=17792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.192281704e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=276725760 dispatch_meta_bytes=327248 combine_payload_bytes=276725760 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=277053008 ingress_bytes=380534784 link_bytes=380534784 token_bytes=553778768 weight_movement_bytes=103809024 total_nvlink_bytes=657587792 weight_to_token_ratio=0.187456 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=30848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.431921757e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80530 payload_rows_remote=67778 dup_rows=17774 dup_rows_remote=17402 dedup_ratio=0.180806 router_tokens_per_expert=[26, 28, 3, 0, 0, 2841, 1178, 1398, 33, 7, 136, 94, 1, 47, 20, 1, 1024, 24, 1238, 3015, 6223, 678, 127, 63, 1752, 0, 75, 1077, 1, 4225, 33, 922, 12, 685, 1549, 1, 1, 567, 57, 0, 82, 12, 2627, 159, 0, 67, 1466, 0, 641, 337, 4552, 363, 4520, 184, 27, 389, 2786, 7, 0, 145, 1552, 11, 62, 1] payload_rows_per_dst=[11236, 12752, 10172, 9352, 11850, 13422, 3466, 8280] dup_rows_per_dst=[4824, 372, 2166, 1706, 2998, 4444, 416, 848] combine_rows_per_src=[11236, 12752, 10172, 9352, 11850, 13422, 3466, 8280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=286269440 dispatch_meta_bytes=330120 combine_payload_bytes=286269440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286599560 ingress_bytes=320872448 link_bytes=320872448 token_bytes=572869000 weight_movement_bytes=34603008 total_nvlink_bytes=607472008 weight_to_token_ratio=0.060403 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.207417189e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75970 payload_rows_remote=64056 dup_rows=22334 dup_rows_remote=18668 dedup_ratio=0.227193 router_tokens_per_expert=[877, 36, 9, 4, 61, 480, 0, 29, 2791, 1056, 2, 0, 676, 3229, 763, 474, 1211, 3948, 0, 419, 1226, 1321, 135, 501, 197, 0, 4881, 186, 261, 237, 95, 968, 389, 495, 463, 177, 87, 0, 1714, 43, 1349, 1023, 179, 44, 1, 5318, 833, 57, 11, 2004, 1, 118, 360, 1290, 2562, 0, 58, 162, 3850, 179, 62, 224, 2, 24] payload_rows_per_dst=[9894, 5424, 8834, 6908, 5912, 12882, 14202, 11914] dup_rows_per_dst=[994, 518, 2340, 1014, 824, 4726, 8252, 3666] combine_rows_per_src=[9894, 5424, 8834, 6908, 5912, 12882, 14202, 11914] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=271343616 dispatch_meta_bytes=319288 combine_payload_bytes=271343616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271662904 ingress_bytes=305946624 link_bytes=305946624 token_bytes=543006520 weight_movement_bytes=34603008 total_nvlink_bytes=577609528 weight_to_token_ratio=0.063725 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.151252515e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78520 payload_rows_remote=63554 dup_rows=19784 dup_rows_remote=12116 dedup_ratio=0.201253 router_tokens_per_expert=[78, 17, 12, 2, 6, 2481, 989, 640, 84, 3, 169, 77, 1, 265, 8, 0, 977, 17, 968, 2759, 6536, 257, 61, 32, 2579, 1, 86, 1060, 1, 4924, 48, 574, 6, 477, 1945, 2, 1, 1840, 119, 0, 88, 15, 1635, 35, 1, 38, 1055, 2, 851, 229, 4844, 557, 4140, 210, 62, 424, 3475, 8, 0, 194, 1129, 0, 58, 0] payload_rows_per_dst=[7198, 13516, 7468, 9006, 7278, 5502, 14966, 13586] dup_rows_per_dst=[1252, 770, 900, 1466, 1502, 236, 7668, 5990] combine_rows_per_src=[7198, 13516, 7468, 9006, 7278, 5502, 14966, 13586] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81460 payload_rows_remote=69546 dup_rows=16844 dup_rows_remote=13108 dedup_ratio=0.171346 router_tokens_per_expert=[8, 170, 0, 0, 789, 639, 207, 5, 25, 68, 4377, 0, 903, 1502, 1118, 0, 237, 7, 301, 55, 13, 0, 0, 2419, 349, 582, 6457, 0, 0, 7, 1, 6, 286, 0, 869, 779, 0, 40, 1262, 152, 1238, 1604, 797, 7372, 28, 0, 1222, 4437, 816, 5, 68, 46, 313, 2, 228, 2566, 782, 1432, 183, 1830, 83, 0, 189, 278] payload_rows_per_dst=[15202, 6608, 5730, 13320, 11914, 8218, 12586, 7882] dup_rows_per_dst=[3178, 624, 334, 1484, 3736, 1560, 4256, 1672] combine_rows_per_src=[15202, 6608, 5730, 13320, 11914, 8218, 12586, 7882] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=291643392 dispatch_meta_bytes=341392 combine_payload_bytes=291643392 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=291984784 ingress_bytes=360849408 link_bytes=360849408 token_bytes=583628176 weight_movement_bytes=69206016 total_nvlink_bytes=652834192 weight_to_token_ratio=0.118579 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=33408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.357847272e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=221626368 dispatch_meta_bytes=261200 combine_payload_bytes=221626368 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=221887568 ingress_bytes=325435392 link_bytes=325435392 token_bytes=443513936 weight_movement_bytes=103809024 total_nvlink_bytes=547322960 weight_to_token_ratio=0.234060 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=19968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.224587180e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76456 payload_rows_remote=62980 dup_rows=21848 dup_rows_remote=14710 dedup_ratio=0.222249 router_tokens_per_expert=[2963, 8, 9, 570, 81, 7, 4255, 1123, 4596, 7154, 108, 7, 0, 68, 104, 95, 204, 4, 1465, 488, 386, 227, 51, 0, 494, 1126, 42, 1, 146, 13, 77, 926, 0, 13, 193, 1302, 8, 103, 19, 2, 297, 378, 198, 1550, 2506, 743, 160, 509, 4156, 4, 4, 55, 123, 137, 0, 2, 654, 81, 54, 2, 20, 1832, 3482, 3767] payload_rows_per_dst=[8528, 764, 5264, 11864, 14778, 9144, 12638, 13476] dup_rows_per_dst=[994, 0, 386, 2978, 2810, 2708, 4834, 7138] combine_rows_per_src=[8528, 764, 5264, 11864, 14778, 9144, 12638, 13476] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79964 payload_rows_remote=71994 dup_rows=18340 dup_rows_remote=16608 dedup_ratio=0.186564 router_tokens_per_expert=[2829, 0, 0, 518, 52, 1, 3659, 618, 5468, 8098, 105, 2, 0, 276, 18, 183, 169, 10, 1907, 234, 590, 909, 41, 1, 1325, 1374, 50, 6, 80, 0, 41, 590, 0, 2, 109, 811, 3, 210, 4, 1, 305, 485, 279, 605, 1788, 807, 322, 260, 2706, 0, 2, 9, 9, 320, 0, 0, 790, 50, 46, 1, 4, 1760, 3965, 4345] payload_rows_per_dst=[7672, 1166, 12002, 13282, 16226, 7970, 10914, 10732] dup_rows_per_dst=[364, 2, 4410, 4586, 2250, 1732, 2496, 2500] combine_rows_per_src=[7672, 1166, 12002, 13282, 16226, 7970, 10914, 10732] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=277618688 dispatch_meta_bytes=340720 combine_payload_bytes=277618688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277959408 ingress_bytes=312221696 link_bytes=312221696 token_bytes=555578096 weight_movement_bytes=34603008 total_nvlink_bytes=590181104 weight_to_token_ratio=0.062283 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=44800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.174865106e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78912 payload_rows_remote=64992 dup_rows=19392 dup_rows_remote=17086 dedup_ratio=0.197266 router_tokens_per_expert=[131, 128, 5, 7, 612, 498, 120, 3, 39, 98, 3742, 26, 768, 1830, 539, 2, 186, 0, 324, 46, 3, 1, 3, 3650, 408, 466, 5685, 0, 0, 5, 0, 16, 253, 3, 592, 629, 0, 1, 1136, 27, 3214, 3878, 527, 6609, 126, 0, 1180, 4145, 930, 1, 208, 151, 503, 5, 123, 1541, 1489, 1530, 159, 558, 2, 0, 175, 116] payload_rows_per_dst=[13920, 11862, 10416, 14056, 10986, 4608, 6132, 6932] dup_rows_per_dst=[2306, 2226, 2994, 6860, 2586, 502, 792, 1126] combine_rows_per_src=[13920, 11862, 10416, 14056, 10986, 4608, 6132, 6932] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=260317184 dispatch_meta_bytes=302680 combine_payload_bytes=260317184 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=260619864 ingress_bytes=364126208 link_bytes=364126208 token_bytes=520937048 weight_movement_bytes=103809024 total_nvlink_bytes=624746072 weight_to_token_ratio=0.199274 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=15744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.370177606e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=262373376 dispatch_meta_bytes=330896 combine_payload_bytes=262373376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262704272 ingress_bytes=296976384 link_bytes=296976384 token_bytes=525077648 weight_movement_bytes=34603008 total_nvlink_bytes=559680656 weight_to_token_ratio=0.065901 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.117498224e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=284860416 dispatch_meta_bytes=330616 combine_payload_bytes=284860416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=285191032 ingress_bytes=319463424 link_bytes=319463424 token_bytes=570051448 weight_movement_bytes=34603008 total_nvlink_bytes=604654456 weight_to_token_ratio=0.060702 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=27648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.202115146e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=257966080 dispatch_meta_bytes=310760 combine_payload_bytes=257966080 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=258276840 ingress_bytes=361775104 link_bytes=361775104 token_bytes=516242920 weight_movement_bytes=103809024 total_nvlink_bytes=620051944 weight_to_token_ratio=0.201086 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=22400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.361330591e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76336 payload_rows_remote=66940 dup_rows=21968 dup_rows_remote=20184 dedup_ratio=0.223470 router_tokens_per_expert=[3241, 17, 23, 949, 452, 19, 3820, 349, 5230, 6938, 175, 1, 1, 51, 30, 321, 241, 17, 1314, 353, 342, 159, 14, 3, 1217, 915, 39, 21, 103, 20, 57, 447, 0, 31, 396, 662, 17, 592, 8, 1, 591, 925, 184, 597, 1887, 1859, 387, 676, 2866, 203, 1, 1, 70, 552, 0, 41, 970, 723, 87, 1, 81, 1126, 3591, 3147] payload_rows_per_dst=[10576, 14246, 9396, 4910, 6660, 10476, 10088, 9984] dup_rows_per_dst=[2112, 6970, 1784, 728, 1032, 3736, 2432, 3174] combine_rows_per_src=[10576, 14246, 9396, 4910, 6660, 10476, 10088, 9984] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=294887424 dispatch_meta_bytes=354408 combine_payload_bytes=294887424 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=295241832 ingress_bytes=364093440 link_bytes=364093440 token_bytes=590129256 weight_movement_bytes=69206016 total_nvlink_bytes=659335272 weight_to_token_ratio=0.117273 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.370054303e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=260317184 dispatch_meta_bytes=302680 combine_payload_bytes=260317184 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=260619864 ingress_bytes=364126208 link_bytes=364126208 token_bytes=520937048 weight_movement_bytes=103809024 total_nvlink_bytes=624746072 weight_to_token_ratio=0.199274 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=15744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.370177606e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78878 payload_rows_remote=71336 dup_rows=19426 dup_rows_remote=18944 dedup_ratio=0.197611 router_tokens_per_expert=[21, 9, 15, 0, 1, 3371, 617, 966, 129, 0, 244, 75, 1, 33, 1, 0, 1153, 77, 969, 2047, 6520, 377, 80, 193, 1896, 0, 113, 1214, 1, 4207, 59, 449, 6, 416, 1074, 0, 0, 1232, 48, 0, 28, 14, 3031, 83, 0, 151, 705, 0, 890, 442, 4681, 445, 5419, 164, 114, 624, 3450, 33, 0, 169, 1012, 1, 81, 1] payload_rows_per_dst=[8496, 13362, 8578, 10534, 4790, 7542, 15464, 10112] dup_rows_per_dst=[1504, 644, 1214, 2484, 762, 482, 10094, 2242] combine_rows_per_src=[8496, 13362, 8578, 10534, 4790, 7542, 15464, 10112] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266207232 dispatch_meta_bytes=328312 combine_payload_bytes=266207232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266535544 ingress_bytes=300810240 link_bytes=300810240 token_bytes=532742776 weight_movement_bytes=34603008 total_nvlink_bytes=567345784 weight_to_token_ratio=0.064953 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=37120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131924716e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81938 payload_rows_remote=68874 dup_rows=16366 dup_rows_remote=11076 dedup_ratio=0.166484 router_tokens_per_expert=[3187, 1, 5, 1528, 24, 3, 3342, 1410, 6569, 7421, 115, 26, 0, 146, 14, 234, 335, 23, 1234, 133, 160, 727, 20, 0, 809, 800, 12, 2, 376, 2, 67, 2779, 0, 5, 107, 338, 11, 147, 7, 1, 50, 347, 570, 582, 2001, 507, 60, 401, 2020, 13, 2, 284, 7, 545, 0, 15, 467, 83, 22, 1, 8, 1900, 3612, 3535] payload_rows_per_dst=[10696, 5232, 10106, 13064, 14902, 7858, 10198, 9882] dup_rows_per_dst=[1620, 316, 2228, 5290, 1172, 1178, 2258, 2304] combine_rows_per_src=[10696, 5232, 10106, 13064, 14902, 7858, 10198, 9882] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=274186240 dispatch_meta_bytes=348496 combine_payload_bytes=274186240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274534736 ingress_bytes=308789248 link_bytes=308789248 token_bytes=548720976 weight_movement_bytes=34603008 total_nvlink_bytes=583323984 weight_to_token_ratio=0.063061 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=25472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.161949081e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80476 payload_rows_remote=71348 dup_rows=17828 dup_rows_remote=16114 dedup_ratio=0.181356 router_tokens_per_expert=[0, 283, 1080, 3368, 182, 2628, 1, 40, 0, 17, 0, 5, 1190, 264, 2618, 1327, 1, 3488, 0, 0, 2, 4, 30, 4, 80, 192, 457, 53, 45, 50, 677, 0, 180, 112, 998, 627, 28, 4996, 0, 3, 250, 0, 139, 3300, 0, 0, 2081, 579, 269, 25, 223, 2876, 1272, 4113, 518, 2, 0, 340, 0, 1, 4451, 3669, 14, 0] payload_rows_per_dst=[11838, 9128, 11448, 8558, 11374, 10170, 8542, 9418] dup_rows_per_dst=[3326, 1714, 3836, 1888, 2514, 2528, 1828, 194] combine_rows_per_src=[11838, 9128, 11448, 8558, 11374, 10170, 8542, 9418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81312 payload_rows_remote=67132 dup_rows=16992 dup_rows_remote=13332 dedup_ratio=0.172852 router_tokens_per_expert=[69, 17, 30, 3, 1, 2355, 1388, 487, 16, 0, 306, 138, 1, 30, 21, 0, 1160, 12, 1574, 2124, 5071, 732, 19, 12, 1207, 0, 295, 1124, 3, 5700, 80, 511, 12, 1437, 1891, 8, 0, 838, 35, 0, 25, 20, 2075, 36, 2, 62, 1042, 0, 436, 153, 4793, 660, 4205, 175, 48, 796, 4185, 24, 2, 148, 1342, 2, 213, 1] payload_rows_per_dst=[7264, 10656, 9256, 14180, 13056, 12066, 4276, 10558] dup_rows_per_dst=[1436, 510, 2010, 3660, 4972, 2868, 260, 1276] combine_rows_per_src=[7264, 10656, 9256, 14180, 13056, 12066, 4276, 10558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81460 payload_rows_remote=69546 dup_rows=16844 dup_rows_remote=13108 dedup_ratio=0.171346 router_tokens_per_expert=[8, 170, 0, 0, 789, 639, 207, 5, 25, 68, 4377, 0, 903, 1502, 1118, 0, 237, 7, 301, 55, 13, 0, 0, 2419, 349, 582, 6457, 0, 0, 7, 1, 6, 286, 0, 869, 779, 0, 40, 1262, 152, 1238, 1604, 797, 7372, 28, 0, 1222, 4437, 816, 5, 68, 46, 313, 2, 228, 2566, 782, 1432, 183, 1830, 83, 0, 189, 278] payload_rows_per_dst=[15202, 6608, 5730, 13320, 11914, 8218, 12586, 7882] dup_rows_per_dst=[3178, 624, 334, 1484, 3736, 1560, 4256, 1672] combine_rows_per_src=[15202, 6608, 5730, 13320, 11914, 8218, 12586, 7882] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82368 payload_rows_remote=73224 dup_rows=15936 dup_rows_remote=13624 dedup_ratio=0.162109 router_tokens_per_expert=[955, 226, 44, 4, 83, 864, 0, 10, 1723, 616, 160, 3, 564, 2759, 938, 837, 2265, 4872, 25, 310, 1348, 723, 53, 1004, 105, 14, 5828, 155, 200, 230, 34, 761, 314, 210, 77, 252, 47, 12, 1198, 33, 1993, 452, 86, 47, 50, 6509, 266, 75, 217, 1493, 4, 21, 66, 747, 2400, 3, 103, 140, 3896, 200, 171, 311, 20, 26] payload_rows_per_dst=[12072, 10894, 9144, 13122, 14236, 5552, 8294, 9054] dup_rows_per_dst=[2044, 4306, 2312, 1532, 3068, 386, 1608, 680] combine_rows_per_src=[12072, 10894, 9144, 13122, 14236, 5552, 8294, 9054] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=292192256 dispatch_meta_bytes=361120 combine_payload_bytes=292192256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=292553376 ingress_bytes=326795264 link_bytes=326795264 token_bytes=584745632 weight_movement_bytes=34603008 total_nvlink_bytes=619348640 weight_to_token_ratio=0.059176 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=19328 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.229704269e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=274972672 dispatch_meta_bytes=321856 combine_payload_bytes=274972672 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=275294528 ingress_bytes=378781696 link_bytes=378781696 token_bytes=550267200 weight_movement_bytes=103809024 total_nvlink_bytes=654076224 weight_to_token_ratio=0.188652 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=18176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.425325028e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80304 payload_rows_remote=65954 dup_rows=18000 dup_rows_remote=11466 dedup_ratio=0.183105 router_tokens_per_expert=[14, 451, 0, 0, 483, 87, 88, 2, 20, 31, 3952, 0, 1322, 3294, 1823, 0, 91, 2, 355, 55, 1, 0, 0, 2011, 580, 706, 5179, 0, 6, 34, 0, 19, 281, 0, 259, 545, 0, 1, 1042, 38, 1357, 1486, 178, 7112, 35, 0, 1341, 5726, 641, 11, 143, 202, 138, 1, 262, 2659, 1436, 821, 53, 2148, 8, 0, 390, 232] payload_rows_per_dst=[14474, 14350, 4634, 12612, 12802, 5388, 7510, 8534] dup_rows_per_dst=[2000, 6534, 396, 3408, 2982, 434, 604, 1642] combine_rows_per_src=[14474, 14350, 4634, 12612, 12802, 5388, 7510, 8534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=284860416 dispatch_meta_bytes=330616 combine_payload_bytes=284860416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=285191032 ingress_bytes=319463424 link_bytes=319463424 token_bytes=570051448 weight_movement_bytes=34603008 total_nvlink_bytes=604654456 weight_to_token_ratio=0.060702 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=27648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.202115146e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76162 payload_rows_remote=63596 dup_rows=22142 dup_rows_remote=17754 dedup_ratio=0.225240 router_tokens_per_expert=[123, 5, 42, 4, 1, 2698, 629, 1981, 52, 4, 252, 259, 1, 29, 44, 2, 1729, 29, 1003, 3130, 5800, 781, 323, 72, 957, 1, 188, 1253, 17, 3192, 71, 828, 12, 741, 1542, 13, 0, 419, 248, 1, 279, 10, 2705, 236, 3, 123, 1048, 0, 621, 71, 4564, 292, 4434, 234, 314, 457, 3585, 17, 1, 358, 1176, 3, 144, 1] payload_rows_per_dst=[9122, 4574, 12010, 10466, 5126, 7538, 14760, 12566] dup_rows_per_dst=[1844, 170, 4006, 2424, 826, 1270, 7214, 4388] combine_rows_per_src=[9122, 4574, 12010, 10466, 5126, 7538, 14760, 12566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=299925504 dispatch_meta_bytes=347392 combine_payload_bytes=299925504 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=300272896 ingress_bytes=403734528 link_bytes=403734528 token_bytes=600198400 weight_movement_bytes=103809024 total_nvlink_bytes=704007424 weight_to_token_ratio=0.172958 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=17280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519220526e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75556 payload_rows_remote=63936 dup_rows=22748 dup_rows_remote=19276 dedup_ratio=0.231405 router_tokens_per_expert=[50, 242, 0, 2, 628, 143, 404, 4, 15, 106, 3110, 0, 1459, 1934, 1959, 0, 154, 1, 68, 13, 3, 1, 0, 2276, 703, 747, 5712, 0, 1, 8, 7, 23, 234, 0, 693, 516, 0, 1, 1526, 5, 1126, 2212, 547, 7283, 25, 0, 1569, 4579, 1028, 25, 51, 181, 332, 0, 684, 2135, 1030, 835, 104, 2320, 4, 1, 170, 163] payload_rows_per_dst=[2850, 9104, 15298, 3388, 6118, 12894, 11620, 14284] dup_rows_per_dst=[96, 1842, 4300, 224, 502, 6552, 3472, 5760] combine_rows_per_src=[2850, 9104, 15298, 3388, 6118, 12894, 11620, 14284] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81798 payload_rows_remote=66998 dup_rows=16506 dup_rows_remote=14884 dedup_ratio=0.167908 router_tokens_per_expert=[3969, 1, 5, 693, 15, 2, 2939, 335, 5766, 7322, 131, 13, 15, 244, 10, 148, 313, 23, 1605, 120, 448, 222, 215, 16, 628, 675, 27, 21, 31, 3, 72, 280, 0, 24, 197, 324, 4, 335, 5, 0, 180, 324, 117, 686, 2850, 1723, 174, 678, 2922, 10, 5, 20, 17, 334, 0, 4, 1082, 324, 33, 2, 12, 1934, 4123, 4402] payload_rows_per_dst=[9474, 1108, 11674, 12570, 14800, 10634, 10180, 11358] dup_rows_per_dst=[566, 14, 3054, 2436, 1622, 2830, 2322, 3662] combine_rows_per_src=[9474, 1108, 11674, 12570, 14800, 10634, 10180, 11358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=282107904 dispatch_meta_bytes=319800 combine_payload_bytes=282107904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282427704 ingress_bytes=316710912 link_bytes=316710912 token_bytes=564535608 weight_movement_bytes=34603008 total_nvlink_bytes=599138616 weight_to_token_ratio=0.061295 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=25344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.191757665e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74584 payload_rows_remote=61072 dup_rows=23720 dup_rows_remote=15176 dedup_ratio=0.241292 router_tokens_per_expert=[0, 160, 840, 2358, 705, 1895, 0, 4, 0, 9, 0, 13, 1408, 236, 2723, 1451, 33, 3386, 0, 0, 7, 13, 88, 1, 158, 310, 426, 310, 40, 99, 505, 0, 240, 117, 799, 1573, 157, 4169, 0, 0, 387, 0, 206, 3365, 2, 0, 1396, 789, 146, 2, 230, 2462, 1382, 4151, 456, 1, 1, 351, 0, 23, 3855, 5707, 3, 4] payload_rows_per_dst=[6488, 9678, 6906, 8180, 6304, 9934, 13582, 13512] dup_rows_per_dst=[720, 2002, 150, 1686, 1046, 2356, 7216, 8544] combine_rows_per_src=[6488, 9678, 6906, 8180, 6304, 9934, 13582, 13512] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78680 payload_rows_remote=67544 dup_rows=19624 dup_rows_remote=15636 dedup_ratio=0.199626 router_tokens_per_expert=[1, 182, 1021, 1615, 506, 1531, 0, 41, 0, 3, 1, 58, 1084, 166, 2618, 1307, 11, 3158, 0, 0, 1, 0, 41, 1, 213, 244, 646, 92, 27, 302, 428, 0, 184, 66, 1330, 2963, 152, 4330, 1, 0, 286, 3, 63, 1659, 13, 0, 1480, 696, 174, 0, 1339, 2507, 1369, 4360, 684, 1, 0, 331, 0, 9, 5211, 4629, 13, 1] payload_rows_per_dst=[8184, 8876, 10102, 10634, 8030, 11136, 10306, 11412] dup_rows_per_dst=[1610, 1598, 2924, 2528, 1362, 3988, 3960, 1654] combine_rows_per_src=[8184, 8876, 10102, 10634, 8030, 11136, 10306, 11412] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=292241408 dispatch_meta_bytes=349848 combine_payload_bytes=292241408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=292591256 ingress_bytes=326844416 link_bytes=326844416 token_bytes=584832664 weight_movement_bytes=34603008 total_nvlink_bytes=619435672 weight_to_token_ratio=0.059167 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=23552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.229889224e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=261881856 dispatch_meta_bytes=332848 combine_payload_bytes=261881856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262214704 ingress_bytes=296484864 link_bytes=296484864 token_bytes=524096560 weight_movement_bytes=34603008 total_nvlink_bytes=558699568 weight_to_token_ratio=0.066024 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.115648674e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=270147584 dispatch_meta_bytes=309680 combine_payload_bytes=270147584 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=270457264 ingress_bytes=339353600 link_bytes=339353600 token_bytes=540604848 weight_movement_bytes=69206016 total_nvlink_bytes=609810864 weight_to_token_ratio=0.128016 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.276960277e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=67510 payload_rows_remote=54226 dup_rows=30794 dup_rows_remote=24642 dedup_ratio=0.313253 router_tokens_per_expert=[2993, 4, 0, 702, 17, 42, 3262, 421, 4604, 7519, 82, 5, 0, 131, 10, 292, 165, 18, 1726, 450, 892, 258, 209, 3, 377, 1675, 4, 8, 79, 1, 15, 443, 0, 4, 95, 770, 3, 235, 37, 2, 274, 219, 250, 1202, 4000, 1521, 221, 233, 2398, 7, 0, 182, 27, 286, 1, 7, 1091, 80, 58, 1, 37, 1110, 4646, 3748] payload_rows_per_dst=[13284, 16274, 11624, 4604, 2250, 9296, 5720, 4458] dup_rows_per_dst=[6152, 18304, 3314, 600, 42, 1990, 96, 296] combine_rows_per_src=[13284, 16274, 11624, 4604, 2250, 9296, 5720, 4458] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=274423808 dispatch_meta_bytes=327528 combine_payload_bytes=274423808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274751336 ingress_bytes=309026816 link_bytes=309026816 token_bytes=549175144 weight_movement_bytes=34603008 total_nvlink_bytes=583778152 weight_to_token_ratio=0.063009 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=39424 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.162843030e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=260489216 dispatch_meta_bytes=325400 combine_payload_bytes=260489216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260814616 ingress_bytes=295092224 link_bytes=295092224 token_bytes=521303832 weight_movement_bytes=34603008 total_nvlink_bytes=555906840 weight_to_token_ratio=0.066378 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=27264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.110408282e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=250150912 dispatch_meta_bytes=304992 combine_payload_bytes=250150912 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=250455904 ingress_bytes=319356928 link_bytes=319356928 token_bytes=500606816 weight_movement_bytes=69206016 total_nvlink_bytes=569812832 weight_to_token_ratio=0.138244 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=34816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.201714410e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=276660224 dispatch_meta_bytes=332720 combine_payload_bytes=276660224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276992944 ingress_bytes=311263232 link_bytes=311263232 token_bytes=553653168 weight_movement_bytes=34603008 total_nvlink_bytes=588256176 weight_to_token_ratio=0.062499 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=19968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.171258483e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80172 payload_rows_remote=69008 dup_rows=18132 dup_rows_remote=15280 dedup_ratio=0.184448 router_tokens_per_expert=[13, 166, 1387, 3372, 1015, 1613, 0, 119, 19, 82, 0, 109, 1039, 27, 3051, 998, 1, 2035, 0, 8, 5, 6, 49, 35, 99, 247, 510, 749, 16, 54, 505, 5, 556, 53, 1118, 771, 219, 3885, 0, 0, 441, 0, 57, 3366, 99, 9, 1838, 748, 289, 14, 472, 2341, 863, 4869, 483, 5, 2, 286, 0, 30, 3999, 4852, 148, 5] payload_rows_per_dst=[11882, 9068, 11164, 11130, 10522, 10378, 7370, 8658] dup_rows_per_dst=[3488, 1582, 2852, 2944, 2682, 2738, 1564, 282] combine_rows_per_src=[11882, 9068, 11164, 11130, 10522, 10378, 7370, 8658] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=261881856 dispatch_meta_bytes=332848 combine_payload_bytes=261881856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262214704 ingress_bytes=296484864 link_bytes=296484864 token_bytes=524096560 weight_movement_bytes=34603008 total_nvlink_bytes=558699568 weight_to_token_ratio=0.066024 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.115648674e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78924 payload_rows_remote=69122 dup_rows=19380 dup_rows_remote=16936 dedup_ratio=0.197144 router_tokens_per_expert=[23, 193, 0, 0, 578, 313, 62, 0, 57, 72, 3251, 0, 1405, 2078, 1221, 0, 153, 0, 354, 31, 1, 0, 0, 2665, 795, 718, 4326, 0, 0, 53, 1, 13, 245, 0, 228, 665, 0, 0, 1849, 9, 2079, 2134, 300, 7292, 35, 0, 1575, 4023, 1594, 12, 263, 104, 343, 4, 574, 1729, 1392, 2135, 59, 1830, 5, 0, 238, 73] payload_rows_per_dst=[5598, 8444, 12674, 10318, 11056, 9802, 11908, 9124] dup_rows_per_dst=[724, 1222, 4334, 1494, 2982, 2444, 3840, 2340] combine_rows_per_src=[5598, 8444, 12674, 10318, 11056, 9802, 11908, 9124] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83090 payload_rows_remote=71202 dup_rows=15214 dup_rows_remote=14146 dedup_ratio=0.154765 router_tokens_per_expert=[119, 127, 0, 0, 578, 227, 190, 0, 63, 158, 3023, 8, 1110, 2579, 808, 0, 219, 7, 775, 119, 0, 3, 0, 3452, 420, 396, 5639, 0, 0, 21, 0, 2, 216, 4, 380, 560, 0, 0, 1132, 19, 2366, 1699, 151, 7409, 21, 0, 810, 4103, 1710, 7, 162, 145, 164, 4, 756, 3088, 880, 1154, 39, 1899, 4, 1, 157, 69] payload_rows_per_dst=[15038, 9828, 8030, 11888, 10554, 8632, 12100, 7020] dup_rows_per_dst=[2262, 1804, 1120, 1068, 2274, 1462, 3838, 1386] combine_rows_per_src=[15038, 9828, 8030, 11888, 10554, 8632, 12100, 7020] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=222109696 dispatch_meta_bytes=315472 combine_payload_bytes=222109696 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=222425168 ingress_bytes=291315712 link_bytes=291315712 token_bytes=444534864 weight_movement_bytes=69206016 total_nvlink_bytes=513740880 weight_to_token_ratio=0.155682 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=26240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.096197571e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78366 payload_rows_remote=68372 dup_rows=19938 dup_rows_remote=17666 dedup_ratio=0.202820 router_tokens_per_expert=[0, 55, 1160, 2210, 375, 939, 0, 46, 0, 8, 0, 51, 1935, 212, 4507, 2218, 5, 3249, 0, 0, 17, 0, 77, 0, 105, 297, 588, 31, 72, 158, 700, 0, 323, 229, 1675, 2270, 201, 3552, 0, 0, 404, 0, 58, 1178, 55, 0, 1738, 355, 277, 5, 107, 2172, 2294, 4785, 528, 1, 0, 265, 0, 12, 3414, 4182, 32, 25] payload_rows_per_dst=[8166, 12972, 12032, 9994, 12014, 6830, 9056, 7302] dup_rows_per_dst=[1404, 4890, 4234, 2272, 4486, 746, 1712, 194] combine_rows_per_src=[8166, 12972, 12032, 9994, 12014, 6830, 9056, 7302] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=282656768 dispatch_meta_bytes=337152 combine_payload_bytes=282656768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282993920 ingress_bytes=317259776 link_bytes=317259776 token_bytes=565650688 weight_movement_bytes=34603008 total_nvlink_bytes=600253696 weight_to_token_ratio=0.061174 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=24960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.193822996e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81798 payload_rows_remote=66998 dup_rows=16506 dup_rows_remote=14884 dedup_ratio=0.167908 router_tokens_per_expert=[3969, 1, 5, 693, 15, 2, 2939, 335, 5766, 7322, 131, 13, 15, 244, 10, 148, 313, 23, 1605, 120, 448, 222, 215, 16, 628, 675, 27, 21, 31, 3, 72, 280, 0, 24, 197, 324, 4, 335, 5, 0, 180, 324, 117, 686, 2850, 1723, 174, 678, 2922, 10, 5, 20, 17, 334, 0, 4, 1082, 324, 33, 2, 12, 1934, 4123, 4402] payload_rows_per_dst=[9474, 1108, 11674, 12570, 14800, 10634, 10180, 11358] dup_rows_per_dst=[566, 14, 3054, 2436, 1622, 2830, 2322, 3662] combine_rows_per_src=[9474, 1108, 11674, 12570, 14800, 10634, 10180, 11358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 2, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82020 payload_rows_remote=72634 dup_rows=16284 dup_rows_remote=14828 dedup_ratio=0.165649 router_tokens_per_expert=[31, 270, 12, 6, 4, 4185, 870, 1415, 47, 149, 281, 133, 2, 96, 2, 7, 1031, 46, 2261, 1327, 6076, 595, 118, 43, 1215, 2, 53, 817, 26, 3099, 116, 559, 51, 420, 563, 5, 13, 1360, 208, 1, 66, 226, 2190, 373, 109, 58, 680, 0, 911, 425, 4770, 739, 5037, 230, 398, 492, 3063, 10, 1, 320, 1443, 22, 71, 3] payload_rows_per_dst=[11022, 12738, 9386, 10054, 11494, 13180, 5370, 8776] dup_rows_per_dst=[2564, 848, 1456, 1720, 3288, 4298, 1020, 1090] combine_rows_per_src=[11022, 12738, 9386, 10054, 11494, 13180, 5370, 8776] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=291643392 dispatch_meta_bytes=341392 combine_payload_bytes=291643392 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=291984784 ingress_bytes=360849408 link_bytes=360849408 token_bytes=583628176 weight_movement_bytes=69206016 total_nvlink_bytes=652834192 weight_to_token_ratio=0.118579 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=33408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.357847272e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=283123712 dispatch_meta_bytes=344232 combine_payload_bytes=283123712 prefetch_bytes=34603008 reduce_bytes=173015040 egress_bytes=283467944 ingress_bytes=490741760 link_bytes=490741760 token_bytes=566591656 weight_movement_bytes=207618048 total_nvlink_bytes=774209704 weight_to_token_ratio=0.366433 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=12800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.846621735e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70334 payload_rows_remote=54108 dup_rows=27970 dup_rows_remote=11192 dedup_ratio=0.284526 router_tokens_per_expert=[3461, 13, 0, 846, 25, 3, 3483, 534, 5217, 7637, 82, 6, 0, 89, 4, 113, 241, 12, 1835, 355, 266, 220, 28, 6, 485, 1523, 29, 14, 198, 0, 20, 371, 0, 18, 328, 816, 1, 604, 16, 1, 182, 641, 348, 773, 2188, 839, 336, 525, 4456, 1, 4, 9, 3, 218, 0, 6, 578, 50, 75, 7, 5, 1180, 3969, 3859] payload_rows_per_dst=[12542, 16226, 10742, 4674, 3044, 9278, 9224, 4604] dup_rows_per_dst=[4188, 16778, 2902, 606, 524, 2386, 170, 416] combine_rows_per_src=[12542, 16226, 10742, 4674, 3044, 9278, 9224, 4604] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=274423808 dispatch_meta_bytes=327528 combine_payload_bytes=274423808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274751336 ingress_bytes=309026816 link_bytes=309026816 token_bytes=549175144 weight_movement_bytes=34603008 total_nvlink_bytes=583778152 weight_to_token_ratio=0.063009 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=39424 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.162843030e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70530 payload_rows_remote=55686 dup_rows=27774 dup_rows_remote=20098 dedup_ratio=0.282532 router_tokens_per_expert=[77, 54, 0, 8, 451, 102, 67, 6, 19, 184, 2611, 1, 1520, 2216, 1265, 0, 102, 10, 238, 39, 17, 2, 0, 3152, 425, 558, 5967, 2, 2, 22, 9, 4, 243, 0, 338, 730, 0, 21, 1122, 177, 1787, 2716, 560, 6874, 7, 0, 973, 5275, 1001, 13, 266, 168, 385, 2, 166, 1905, 1615, 1221, 238, 1877, 3, 0, 113, 226] payload_rows_per_dst=[1518, 7100, 15050, 2004, 5038, 14506, 10470, 14844] dup_rows_per_dst=[12, 990, 5818, 40, 224, 10450, 2564, 7676] combine_rows_per_src=[1518, 7100, 15050, 2004, 5038, 14506, 10470, 14844] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79868 payload_rows_remote=67836 dup_rows=18436 dup_rows_remote=15734 dedup_ratio=0.187541 router_tokens_per_expert=[3501, 5, 2, 653, 75, 7, 2192, 525, 5172, 7237, 135, 12, 0, 146, 26, 133, 186, 10, 899, 184, 550, 425, 22, 3, 462, 1140, 30, 31, 484, 0, 60, 677, 0, 6, 82, 286, 0, 485, 0, 0, 533, 486, 331, 1201, 3372, 1181, 143, 1297, 4811, 3, 16, 31, 11, 301, 0, 2, 764, 150, 42, 0, 10, 1748, 2939, 3937] payload_rows_per_dst=[8868, 884, 8026, 12554, 14628, 12256, 12032, 10620] dup_rows_per_dst=[668, 20, 1392, 3558, 1564, 4832, 2702, 3700] combine_rows_per_src=[8868, 884, 8026, 12554, 14628, 12256, 12032, 10620] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80866 payload_rows_remote=70388 dup_rows=17438 dup_rows_remote=16472 dedup_ratio=0.177389 router_tokens_per_expert=[432, 195, 195, 10, 0, 34, 963, 143, 0, 1, 29, 4542, 4, 1146, 6136, 0, 67, 3, 192, 332, 2994, 6053, 135, 26, 9, 4473, 346, 569, 10, 0, 483, 1, 0, 0, 4, 674, 0, 2610, 696, 0, 0, 338, 15, 1201, 1940, 1875, 65, 1691, 3573, 15, 1, 312, 0, 6, 1894, 90, 361, 247, 516, 686, 211, 0, 0, 608] payload_rows_per_dst=[13402, 10478, 13802, 10408, 6946, 11450, 10082, 4298] dup_rows_per_dst=[2814, 966, 5802, 1374, 1022, 2800, 1700, 960] combine_rows_per_src=[13402, 10478, 13802, 10408, 6946, 11450, 10082, 4298] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78150 payload_rows_remote=67544 dup_rows=20154 dup_rows_remote=17198 dedup_ratio=0.205017 router_tokens_per_expert=[2, 411, 1730, 2453, 475, 1734, 0, 2, 1, 32, 0, 105, 1727, 186, 2499, 933, 10, 2497, 0, 0, 7, 0, 58, 0, 65, 685, 411, 203, 10, 93, 610, 0, 597, 690, 800, 1528, 68, 4070, 0, 3, 661, 3, 194, 2708, 11, 0, 3733, 521, 227, 3, 442, 2409, 569, 3629, 459, 3, 0, 135, 0, 26, 3245, 5462, 17, 0] payload_rows_per_dst=[10528, 9046, 9838, 12142, 10606, 12314, 6874, 6802] dup_rows_per_dst=[3086, 1920, 2564, 2936, 2956, 5298, 1350, 44] combine_rows_per_src=[10528, 9046, 9838, 12142, 10606, 12314, 6874, 6802] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78960 payload_rows_remote=65740 dup_rows=19344 dup_rows_remote=16396 dedup_ratio=0.196777 router_tokens_per_expert=[505, 240, 296, 88, 0, 28, 388, 348, 0, 11, 91, 3488, 0, 1073, 6942, 16, 278, 3, 291, 518, 1421, 6217, 422, 8, 19, 6314, 317, 785, 17, 36, 519, 14, 0, 0, 4, 971, 2, 3814, 886, 2, 0, 198, 83, 861, 1448, 1848, 120, 1016, 3071, 3, 0, 549, 0, 14, 1415, 287, 83, 31, 297, 362, 134, 0, 7, 953] payload_rows_per_dst=[3358, 14160, 5462, 9282, 11882, 9262, 12334, 13220] dup_rows_per_dst=[428, 2106, 420, 1484, 4752, 1886, 5320, 2948] combine_rows_per_src=[3358, 14160, 5462, 9282, 11882, 9262, 12334, 13220] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79224 payload_rows_remote=66292 dup_rows=19080 dup_rows_remote=13864 dedup_ratio=0.194092 router_tokens_per_expert=[387, 221, 190, 13, 0, 241, 302, 710, 0, 7, 5, 4190, 1, 1749, 6538, 4, 123, 9, 243, 625, 3365, 5459, 205, 3, 6, 3559, 485, 256, 10, 0, 596, 22, 0, 0, 2, 525, 0, 3557, 2456, 0, 0, 412, 43, 479, 1540, 3299, 7, 446, 2801, 0, 0, 163, 0, 2, 1313, 144, 162, 299, 441, 354, 30, 0, 0, 1153] payload_rows_per_dst=[9684, 7870, 8252, 8820, 10466, 12932, 8772, 12428] dup_rows_per_dst=[1818, 1364, 894, 1048, 2614, 5216, 2758, 3368] combine_rows_per_src=[9684, 7870, 8252, 8820, 10466, 12932, 8772, 12428] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=280051712 dispatch_meta_bytes=344152 combine_payload_bytes=280051712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280395864 ingress_bytes=314654720 link_bytes=314654720 token_bytes=560447576 weight_movement_bytes=34603008 total_nvlink_bytes=595050584 weight_to_token_ratio=0.061742 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=34176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.184020380e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=277856256 dispatch_meta_bytes=334280 combine_payload_bytes=277856256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278190536 ingress_bytes=312459264 link_bytes=312459264 token_bytes=556046792 weight_movement_bytes=34603008 total_nvlink_bytes=590649800 weight_to_token_ratio=0.062230 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=26368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.175759055e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=276660224 dispatch_meta_bytes=338968 combine_payload_bytes=276660224 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=276999192 ingress_bytes=380469248 link_bytes=380469248 token_bytes=553659416 weight_movement_bytes=103809024 total_nvlink_bytes=657468440 weight_to_token_ratio=0.187496 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=21120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.431675150e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=288309248 dispatch_meta_bytes=347440 combine_payload_bytes=288309248 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=288656688 ingress_bytes=426721280 link_bytes=426721280 token_bytes=576965936 weight_movement_bytes=138412032 total_nvlink_bytes=715377968 weight_to_token_ratio=0.239896 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=19456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.605717823e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=297508864 dispatch_meta_bytes=349848 combine_payload_bytes=297508864 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=297858712 ingress_bytes=401317888 link_bytes=401317888 token_bytes=595367576 weight_movement_bytes=103809024 total_nvlink_bytes=699176600 weight_to_token_ratio=0.174361 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.510126904e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=221626368 dispatch_meta_bytes=261200 combine_payload_bytes=221626368 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=221887568 ingress_bytes=325435392 link_bytes=325435392 token_bytes=443513936 weight_movement_bytes=103809024 total_nvlink_bytes=547322960 weight_to_token_ratio=0.234060 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=19968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.224587180e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81806 payload_rows_remote=71610 dup_rows=16498 dup_rows_remote=13950 dedup_ratio=0.167826 router_tokens_per_expert=[0, 139, 1599, 2761, 548, 1317, 0, 8, 0, 56, 0, 16, 2067, 85, 2321, 2106, 9, 1699, 0, 0, 5, 1, 147, 1, 234, 141, 427, 316, 27, 129, 597, 0, 125, 150, 1732, 659, 139, 5580, 0, 0, 351, 6, 237, 2077, 1, 8, 2037, 259, 220, 2, 409, 1665, 938, 4207, 428, 48, 0, 242, 0, 5, 5682, 5094, 85, 10] payload_rows_per_dst=[10196, 10372, 9844, 11360, 13118, 8648, 6528, 11740] dup_rows_per_dst=[2548, 2930, 2294, 2570, 3652, 1304, 892, 308] combine_rows_per_src=[10196, 10372, 9844, 11360, 13118, 8648, 6528, 11740] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=269271040 dispatch_meta_bytes=328544 combine_payload_bytes=269271040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269599584 ingress_bytes=303874048 link_bytes=303874048 token_bytes=538870624 weight_movement_bytes=34603008 total_nvlink_bytes=573473632 weight_to_token_ratio=0.064214 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=28288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.143453579e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=271532032 dispatch_meta_bytes=320624 combine_payload_bytes=271532032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271852656 ingress_bytes=306135040 link_bytes=306135040 token_bytes=543384688 weight_movement_bytes=34603008 total_nvlink_bytes=577987696 weight_to_token_ratio=0.063680 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.151961510e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=228089856 dispatch_meta_bytes=303136 combine_payload_bytes=228089856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=228392992 ingress_bytes=262692864 link_bytes=262692864 token_bytes=456482848 weight_movement_bytes=34603008 total_nvlink_bytes=491085856 weight_to_token_ratio=0.075804 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.884920987e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81698 payload_rows_remote=68632 dup_rows=16606 dup_rows_remote=11792 dedup_ratio=0.168925 router_tokens_per_expert=[629, 150, 426, 15, 7, 79, 598, 584, 57, 3, 24, 3774, 25, 949, 6331, 0, 267, 3, 278, 556, 1969, 5583, 104, 180, 27, 5173, 288, 355, 6, 33, 372, 7, 1, 2, 0, 188, 47, 3692, 1173, 0, 3, 413, 0, 866, 1489, 1412, 66, 1907, 4149, 0, 6, 156, 0, 53, 1585, 122, 323, 484, 268, 1060, 290, 0, 0, 545] payload_rows_per_dst=[14066, 8990, 13066, 11306, 8850, 9958, 10292, 5170] dup_rows_per_dst=[3572, 674, 4814, 1216, 1356, 2354, 1850, 770] combine_rows_per_src=[14066, 8990, 13066, 11306, 8850, 9958, 10292, 5170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=277856256 dispatch_meta_bytes=334280 combine_payload_bytes=277856256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278190536 ingress_bytes=312459264 link_bytes=312459264 token_bytes=556046792 weight_movement_bytes=34603008 total_nvlink_bytes=590649800 weight_to_token_ratio=0.062230 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=26368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.175759055e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81938 payload_rows_remote=68874 dup_rows=16366 dup_rows_remote=11076 dedup_ratio=0.166484 router_tokens_per_expert=[3187, 1, 5, 1528, 24, 3, 3342, 1410, 6569, 7421, 115, 26, 0, 146, 14, 234, 335, 23, 1234, 133, 160, 727, 20, 0, 809, 800, 12, 2, 376, 2, 67, 2779, 0, 5, 107, 338, 11, 147, 7, 1, 50, 347, 570, 582, 2001, 507, 60, 401, 2020, 13, 2, 284, 7, 545, 0, 15, 467, 83, 22, 1, 8, 1900, 3612, 3535] payload_rows_per_dst=[10696, 5232, 10106, 13064, 14902, 7858, 10198, 9882] dup_rows_per_dst=[1620, 316, 2228, 5290, 1172, 1178, 2258, 2304] combine_rows_per_src=[10696, 5232, 10106, 13064, 14902, 7858, 10198, 9882] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79964 payload_rows_remote=71994 dup_rows=18340 dup_rows_remote=16608 dedup_ratio=0.186564 router_tokens_per_expert=[2829, 0, 0, 518, 52, 1, 3659, 618, 5468, 8098, 105, 2, 0, 276, 18, 183, 169, 10, 1907, 234, 590, 909, 41, 1, 1325, 1374, 50, 6, 80, 0, 41, 590, 0, 2, 109, 811, 3, 210, 4, 1, 305, 485, 279, 605, 1788, 807, 322, 260, 2706, 0, 2, 9, 9, 320, 0, 0, 790, 50, 46, 1, 4, 1760, 3965, 4345] payload_rows_per_dst=[7672, 1166, 12002, 13282, 16226, 7970, 10914, 10732] dup_rows_per_dst=[364, 2, 4410, 4586, 2250, 1732, 2496, 2500] combine_rows_per_src=[7672, 1166, 12002, 13282, 16226, 7970, 10914, 10732] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=293314560 dispatch_meta_bytes=342240 combine_payload_bytes=293314560 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=293656800 ingress_bytes=397123584 link_bytes=397123584 token_bytes=586971360 weight_movement_bytes=103809024 total_nvlink_bytes=690780384 weight_to_token_ratio=0.176855 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.494344076e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82294 payload_rows_remote=71340 dup_rows=16010 dup_rows_remote=14362 dedup_ratio=0.162862 router_tokens_per_expert=[444, 245, 127, 270, 0, 59, 122, 252, 0, 3, 4, 4522, 0, 1621, 6787, 15, 123, 69, 393, 563, 3092, 6347, 65, 30, 2, 4640, 553, 637, 23, 2, 404, 40, 0, 0, 0, 1209, 1, 2968, 1218, 0, 0, 374, 17, 478, 1457, 1891, 14, 261, 3059, 10, 0, 78, 0, 1, 1035, 339, 139, 290, 486, 1218, 491, 2, 2, 660] payload_rows_per_dst=[14228, 10664, 9588, 10954, 8788, 8134, 7994, 11944] dup_rows_per_dst=[2384, 1666, 2802, 1648, 2004, 850, 1050, 3606] combine_rows_per_src=[14228, 10664, 9588, 10954, 8788, 8134, 7994, 11944] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82104 payload_rows_remote=71034 dup_rows=16200 dup_rows_remote=13946 dedup_ratio=0.164795 router_tokens_per_expert=[332, 199, 162, 20, 0, 35, 216, 98, 0, 2, 69, 3685, 0, 2112, 6303, 1, 69, 2, 194, 319, 2573, 5354, 462, 19, 105, 4990, 820, 520, 3, 2, 687, 0, 0, 0, 4, 1019, 7, 4728, 904, 0, 0, 490, 17, 343, 830, 2306, 99, 828, 3661, 8, 3, 95, 0, 1, 1184, 130, 116, 148, 583, 222, 880, 1, 0, 1192] payload_rows_per_dst=[13190, 8064, 6714, 11832, 11070, 9778, 8788, 12668] dup_rows_per_dst=[1540, 1086, 562, 2882, 2254, 2176, 1376, 4324] combine_rows_per_src=[13190, 8064, 6714, 11832, 11070, 9778, 8788, 12668] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78150 payload_rows_remote=67544 dup_rows=20154 dup_rows_remote=17198 dedup_ratio=0.205017 router_tokens_per_expert=[2, 411, 1730, 2453, 475, 1734, 0, 2, 1, 32, 0, 105, 1727, 186, 2499, 933, 10, 2497, 0, 0, 7, 0, 58, 0, 65, 685, 411, 203, 10, 93, 610, 0, 597, 690, 800, 1528, 68, 4070, 0, 3, 661, 3, 194, 2708, 11, 0, 3733, 521, 227, 3, 442, 2409, 569, 3629, 459, 3, 0, 135, 0, 26, 3245, 5462, 17, 0] payload_rows_per_dst=[10528, 9046, 9838, 12142, 10606, 12314, 6874, 6802] dup_rows_per_dst=[3086, 1920, 2564, 2936, 2956, 5298, 1350, 44] combine_rows_per_src=[10528, 9046, 9838, 12142, 10606, 12314, 6874, 6802] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=282107904 dispatch_meta_bytes=319800 combine_payload_bytes=282107904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282427704 ingress_bytes=316710912 link_bytes=316710912 token_bytes=564535608 weight_movement_bytes=34603008 total_nvlink_bytes=599138616 weight_to_token_ratio=0.061295 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=25344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.191757665e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80052 payload_rows_remote=73736 dup_rows=18252 dup_rows_remote=17392 dedup_ratio=0.185669 router_tokens_per_expert=[61, 515, 0, 8, 1514, 707, 77, 75, 150, 101, 3484, 2, 1188, 2082, 873, 3, 231, 2, 977, 73, 7, 0, 0, 2298, 738, 905, 5791, 5, 2, 50, 0, 12, 294, 5, 450, 650, 3, 19, 981, 112, 2021, 1677, 341, 6606, 63, 0, 1891, 4761, 1052, 12, 258, 146, 218, 14, 327, 1279, 1172, 1035, 37, 1372, 13, 8, 238, 166] payload_rows_per_dst=[14624, 12396, 6316, 12824, 11846, 9020, 6086, 6940] dup_rows_per_dst=[4502, 3370, 860, 3316, 2704, 1832, 526, 1142] combine_rows_per_src=[14624, 12396, 6316, 12824, 11846, 9020, 6086, 6940] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80476 payload_rows_remote=71348 dup_rows=17828 dup_rows_remote=16114 dedup_ratio=0.181356 router_tokens_per_expert=[0, 283, 1080, 3368, 182, 2628, 1, 40, 0, 17, 0, 5, 1190, 264, 2618, 1327, 1, 3488, 0, 0, 2, 4, 30, 4, 80, 192, 457, 53, 45, 50, 677, 0, 180, 112, 998, 627, 28, 4996, 0, 3, 250, 0, 139, 3300, 0, 0, 2081, 579, 269, 25, 223, 2876, 1272, 4113, 518, 2, 0, 340, 0, 1, 4451, 3669, 14, 0] payload_rows_per_dst=[11838, 9128, 11448, 8558, 11374, 10170, 8542, 9418] dup_rows_per_dst=[3326, 1714, 3836, 1888, 2514, 2528, 1828, 194] combine_rows_per_src=[11838, 9128, 11448, 8558, 11374, 10170, 8542, 9418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=294887424 dispatch_meta_bytes=354408 combine_payload_bytes=294887424 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=295241832 ingress_bytes=364093440 link_bytes=364093440 token_bytes=590129256 weight_movement_bytes=69206016 total_nvlink_bytes=659335272 weight_to_token_ratio=0.117273 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.370054303e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76456 payload_rows_remote=62980 dup_rows=21848 dup_rows_remote=14710 dedup_ratio=0.222249 router_tokens_per_expert=[2963, 8, 9, 570, 81, 7, 4255, 1123, 4596, 7154, 108, 7, 0, 68, 104, 95, 204, 4, 1465, 488, 386, 227, 51, 0, 494, 1126, 42, 1, 146, 13, 77, 926, 0, 13, 193, 1302, 8, 103, 19, 2, 297, 378, 198, 1550, 2506, 743, 160, 509, 4156, 4, 4, 55, 123, 137, 0, 2, 654, 81, 54, 2, 20, 1832, 3482, 3767] payload_rows_per_dst=[8528, 764, 5264, 11864, 14778, 9144, 12638, 13476] dup_rows_per_dst=[994, 0, 386, 2978, 2810, 2708, 4834, 7138] combine_rows_per_src=[8528, 764, 5264, 11864, 14778, 9144, 12638, 13476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=276660224 dispatch_meta_bytes=338968 combine_payload_bytes=276660224 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=276999192 ingress_bytes=380469248 link_bytes=380469248 token_bytes=553659416 weight_movement_bytes=103809024 total_nvlink_bytes=657468440 weight_to_token_ratio=0.187496 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=21120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.431675150e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78030 payload_rows_remote=65210 dup_rows=20274 dup_rows_remote=15570 dedup_ratio=0.206238 router_tokens_per_expert=[0, 218, 2361, 2713, 285, 2391, 0, 26, 4, 23, 0, 88, 1542, 241, 3144, 1192, 9, 3967, 0, 0, 51, 7, 23, 1, 57, 264, 485, 130, 45, 86, 588, 0, 305, 313, 688, 1694, 275, 3977, 0, 0, 252, 3, 53, 2308, 31, 3, 1832, 486, 11, 3, 309, 2116, 1229, 4537, 551, 6, 0, 233, 0, 5, 4841, 3086, 18, 46] payload_rows_per_dst=[10982, 10222, 8058, 7838, 6340, 8728, 12820, 13042] dup_rows_per_dst=[3858, 2306, 58, 1644, 1298, 1208, 4704, 5198] combine_rows_per_src=[10982, 10222, 8058, 7838, 6340, 8728, 12820, 13042] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=281116672 dispatch_meta_bytes=321696 combine_payload_bytes=281116672 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=281438368 ingress_bytes=350322688 link_bytes=350322688 token_bytes=562555040 weight_movement_bytes=69206016 total_nvlink_bytes=631761056 weight_to_token_ratio=0.123021 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.318236072e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78962 payload_rows_remote=69182 dup_rows=19342 dup_rows_remote=16276 dedup_ratio=0.196757 router_tokens_per_expert=[0, 5, 0, 2840, 369, 56, 0, 8, 1121, 1291, 10, 0, 4, 208, 1411, 2041, 102, 4, 126, 660, 3786, 659, 3, 2, 294, 759, 127, 987, 2955, 3413, 5, 1481, 155, 0, 69, 604, 28, 65, 854, 4, 1, 0, 98, 13, 0, 1, 15, 3, 850, 1, 4936, 4157, 912, 0, 519, 1636, 1382, 720, 4792, 0, 2, 0, 1462, 1146] payload_rows_per_dst=[12544, 9780, 9424, 10808, 8268, 12900, 7614, 7624] dup_rows_per_dst=[3596, 3066, 1260, 2650, 1200, 4824, 946, 1800] combine_rows_per_src=[12544, 9780, 9424, 10808, 8268, 12900, 7614, 7624] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=302022656 dispatch_meta_bytes=364512 combine_payload_bytes=302022656 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=302387168 ingress_bytes=371228672 link_bytes=371228672 token_bytes=604409824 weight_movement_bytes=69206016 total_nvlink_bytes=673615840 weight_to_token_ratio=0.114502 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.022500 gemm_active_groups=11 gemm_peak_group_rows=22016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396903606e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=292208640 dispatch_meta_bytes=342808 combine_payload_bytes=292208640 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=292551448 ingress_bytes=396017664 link_bytes=396017664 token_bytes=584760088 weight_movement_bytes=103809024 total_nvlink_bytes=688569112 weight_to_token_ratio=0.177524 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=34304 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.490182588e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=290955264 dispatch_meta_bytes=339920 combine_payload_bytes=290955264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291295184 ingress_bytes=325558272 link_bytes=325558272 token_bytes=582250448 weight_movement_bytes=34603008 total_nvlink_bytes=616853456 weight_to_token_ratio=0.059430 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=27904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.225049568e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=267100160 dispatch_meta_bytes=323120 combine_payload_bytes=267100160 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267423280 ingress_bytes=336306176 link_bytes=336306176 token_bytes=534523440 weight_movement_bytes=69206016 total_nvlink_bytes=603729456 weight_to_token_ratio=0.129472 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265493065e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78076 payload_rows_remote=63934 dup_rows=20228 dup_rows_remote=10790 dedup_ratio=0.205770 router_tokens_per_expert=[0, 48, 0, 1825, 225, 28, 7, 14, 70, 852, 1, 5, 1, 61, 3249, 2012, 57, 1, 464, 460, 2850, 2862, 2, 2, 174, 655, 1031, 655, 2813, 3592, 18, 2161, 77, 1, 75, 611, 224, 260, 1234, 23, 0, 3, 119, 22, 1, 0, 52, 0, 1080, 1, 4944, 3166, 386, 0, 67, 2413, 521, 1776, 4889, 9, 5, 4, 700, 294] payload_rows_per_dst=[4130, 10490, 8884, 8150, 9118, 10098, 13064, 14142] dup_rows_per_dst=[164, 2012, 1078, 1238, 1518, 184, 4596, 9438] combine_rows_per_src=[4130, 10490, 8884, 8150, 9118, 10098, 13064, 14142] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84336 payload_rows_remote=72780 dup_rows=13968 dup_rows_remote=13778 dedup_ratio=0.142090 router_tokens_per_expert=[0, 0, 0, 2098, 263, 13, 0, 3, 606, 584, 2, 0, 3, 36, 3396, 2230, 87, 0, 101, 200, 4951, 2446, 0, 3, 14, 2128, 160, 274, 4366, 2583, 0, 1501, 153, 0, 80, 1042, 151, 105, 446, 5, 0, 0, 125, 4, 0, 0, 22, 0, 375, 0, 5722, 3645, 546, 0, 112, 2397, 325, 1160, 3108, 0, 0, 0, 695, 886] payload_rows_per_dst=[9978, 10878, 13222, 9386, 11082, 11556, 11778, 6456] dup_rows_per_dst=[992, 2836, 2354, 2208, 1614, 190, 2372, 1402] combine_rows_per_src=[9978, 10878, 13222, 9386, 11082, 11556, 11778, 6456] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=283369472 dispatch_meta_bytes=341832 combine_payload_bytes=283369472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=283711304 ingress_bytes=317972480 link_bytes=317972480 token_bytes=567080776 weight_movement_bytes=34603008 total_nvlink_bytes=601683784 weight_to_token_ratio=0.061020 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.196504843e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79324 payload_rows_remote=66984 dup_rows=18980 dup_rows_remote=17288 dedup_ratio=0.193075 router_tokens_per_expert=[266, 207, 168, 3, 0, 274, 221, 128, 0, 3, 52, 4192, 0, 1681, 5749, 34, 100, 19, 390, 537, 2860, 4312, 24, 5, 33, 3051, 795, 1501, 8, 0, 1228, 1, 0, 0, 12, 1334, 216, 2990, 2454, 2, 0, 372, 3, 1283, 1189, 2486, 57, 794, 3563, 2, 5, 327, 0, 20, 677, 434, 272, 438, 606, 640, 62, 0, 0, 1072] payload_rows_per_dst=[12340, 10496, 12726, 8904, 10386, 10614, 8720, 5138] dup_rows_per_dst=[1692, 1428, 6314, 1784, 3630, 1754, 1336, 1042] combine_rows_per_src=[12340, 10496, 12726, 8904, 10386, 10614, 8720, 5138] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=292241408 dispatch_meta_bytes=349848 combine_payload_bytes=292241408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=292591256 ingress_bytes=326844416 link_bytes=326844416 token_bytes=584832664 weight_movement_bytes=34603008 total_nvlink_bytes=619435672 weight_to_token_ratio=0.059167 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=23552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.229889224e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=257966080 dispatch_meta_bytes=310760 combine_payload_bytes=257966080 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=258276840 ingress_bytes=361775104 link_bytes=361775104 token_bytes=516242920 weight_movement_bytes=103809024 total_nvlink_bytes=620051944 weight_to_token_ratio=0.201086 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=22400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.361330591e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=267100160 dispatch_meta_bytes=323120 combine_payload_bytes=267100160 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267423280 ingress_bytes=336306176 link_bytes=336306176 token_bytes=534523440 weight_movement_bytes=69206016 total_nvlink_bytes=603729456 weight_to_token_ratio=0.129472 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265493065e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=261873664 dispatch_meta_bytes=298896 combine_payload_bytes=261873664 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=262172560 ingress_bytes=331079680 link_bytes=331079680 token_bytes=524046224 weight_movement_bytes=69206016 total_nvlink_bytes=593252240 weight_to_token_ratio=0.132061 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=12544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.245826182e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80918 payload_rows_remote=74244 dup_rows=17386 dup_rows_remote=16866 dedup_ratio=0.176860 router_tokens_per_expert=[0, 110, 7, 2608, 287, 60, 2, 424, 905, 1474, 11, 0, 0, 157, 2650, 2602, 166, 13, 158, 237, 2435, 569, 17, 2, 99, 1954, 222, 844, 4224, 1928, 11, 2651, 246, 5, 189, 387, 160, 164, 795, 36, 22, 2, 139, 75, 0, 0, 61, 1, 567, 0, 5401, 1956, 172, 0, 537, 2040, 1003, 1465, 4491, 2, 65, 5, 1258, 1081] payload_rows_per_dst=[12446, 11360, 6674, 11866, 10764, 11122, 9190, 7496] dup_rows_per_dst=[3532, 4238, 520, 3552, 1648, 280, 1354, 2262] combine_rows_per_src=[12446, 11360, 6674, 11866, 10764, 11122, 9190, 7496] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=298106880 dispatch_meta_bytes=346232 combine_payload_bytes=298106880 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=298453112 ingress_bytes=367312896 link_bytes=367312896 token_bytes=596559992 weight_movement_bytes=69206016 total_nvlink_bytes=665766008 weight_to_token_ratio=0.116008 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=39936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.382168857e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78366 payload_rows_remote=68372 dup_rows=19938 dup_rows_remote=17666 dedup_ratio=0.202820 router_tokens_per_expert=[0, 55, 1160, 2210, 375, 939, 0, 46, 0, 8, 0, 51, 1935, 212, 4507, 2218, 5, 3249, 0, 0, 17, 0, 77, 0, 105, 297, 588, 31, 72, 158, 700, 0, 323, 229, 1675, 2270, 201, 3552, 0, 0, 404, 0, 58, 1178, 55, 0, 1738, 355, 277, 5, 107, 2172, 2294, 4785, 528, 1, 0, 265, 0, 12, 3414, 4182, 32, 25] payload_rows_per_dst=[8166, 12972, 12032, 9994, 12014, 6830, 9056, 7302] dup_rows_per_dst=[1404, 4890, 4234, 2272, 4486, 746, 1712, 194] combine_rows_per_src=[8166, 12972, 12032, 9994, 12014, 6830, 9056, 7302] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=274366464 dispatch_meta_bytes=337088 combine_payload_bytes=274366464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274703552 ingress_bytes=308969472 link_bytes=308969472 token_bytes=549070016 weight_movement_bytes=34603008 total_nvlink_bytes=583673024 weight_to_token_ratio=0.063021 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=35072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.162627249e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79388 payload_rows_remote=66998 dup_rows=18916 dup_rows_remote=14242 dedup_ratio=0.192424 router_tokens_per_expert=[1, 178, 0, 1995, 507, 19, 8, 27, 915, 1206, 1, 0, 14, 66, 999, 1485, 60, 16, 76, 154, 4219, 1990, 5, 4, 81, 1748, 177, 570, 3172, 3969, 0, 1987, 248, 0, 94, 2247, 327, 125, 1310, 1, 2, 0, 324, 19, 1, 0, 24, 0, 489, 0, 3826, 4296, 238, 1, 433, 2311, 660, 2132, 2412, 1, 1, 0, 1163, 818] payload_rows_per_dst=[8718, 7568, 11220, 12390, 11446, 8108, 12270, 7668] dup_rows_per_dst=[1576, 1804, 1828, 4674, 3602, 284, 3266, 1882] combine_rows_per_src=[8718, 7568, 11220, 12390, 11446, 8108, 12270, 7668] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81436 payload_rows_remote=68536 dup_rows=16868 dup_rows_remote=14142 dedup_ratio=0.171590 router_tokens_per_expert=[0, 15, 0, 2582, 367, 3, 2, 129, 586, 1186, 11, 0, 3, 132, 850, 1091, 177, 15, 388, 154, 4360, 1626, 22, 0, 276, 1503, 787, 1452, 5660, 2142, 2, 488, 289, 0, 96, 469, 311, 129, 845, 14, 1, 0, 178, 2, 0, 2, 8, 0, 465, 1, 4156, 2545, 372, 1, 714, 1316, 1801, 1962, 5722, 2, 15, 17, 804, 906] payload_rows_per_dst=[13590, 6336, 11532, 10404, 12900, 8532, 9364, 8778] dup_rows_per_dst=[4050, 1382, 1952, 2896, 2726, 162, 1464, 2236] combine_rows_per_src=[13590, 6336, 11532, 10404, 12900, 8532, 9364, 8778] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78680 payload_rows_remote=67544 dup_rows=19624 dup_rows_remote=15636 dedup_ratio=0.199626 router_tokens_per_expert=[1, 182, 1021, 1615, 506, 1531, 0, 41, 0, 3, 1, 58, 1084, 166, 2618, 1307, 11, 3158, 0, 0, 1, 0, 41, 1, 213, 244, 646, 92, 27, 302, 428, 0, 184, 66, 1330, 2963, 152, 4330, 1, 0, 286, 3, 63, 1659, 13, 0, 1480, 696, 174, 0, 1339, 2507, 1369, 4360, 684, 1, 0, 331, 0, 9, 5211, 4629, 13, 1] payload_rows_per_dst=[8184, 8876, 10102, 10634, 8030, 11136, 10306, 11412] dup_rows_per_dst=[1610, 1598, 2924, 2528, 1362, 3988, 3960, 1654] combine_rows_per_src=[8184, 8876, 10102, 10634, 8030, 11136, 10306, 11412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82104 payload_rows_remote=71034 dup_rows=16200 dup_rows_remote=13946 dedup_ratio=0.164795 router_tokens_per_expert=[332, 199, 162, 20, 0, 35, 216, 98, 0, 2, 69, 3685, 0, 2112, 6303, 1, 69, 2, 194, 319, 2573, 5354, 462, 19, 105, 4990, 820, 520, 3, 2, 687, 0, 0, 0, 4, 1019, 7, 4728, 904, 0, 0, 490, 17, 343, 830, 2306, 99, 828, 3661, 8, 3, 95, 0, 1, 1184, 130, 116, 148, 583, 222, 880, 1, 0, 1192] payload_rows_per_dst=[13190, 8064, 6714, 11832, 11070, 9778, 8788, 12668] dup_rows_per_dst=[1540, 1086, 562, 2882, 2254, 2176, 1376, 4324] combine_rows_per_src=[13190, 8064, 6714, 11832, 11070, 9778, 8788, 12668] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76336 payload_rows_remote=66940 dup_rows=21968 dup_rows_remote=20184 dedup_ratio=0.223470 router_tokens_per_expert=[3241, 17, 23, 949, 452, 19, 3820, 349, 5230, 6938, 175, 1, 1, 51, 30, 321, 241, 17, 1314, 353, 342, 159, 14, 3, 1217, 915, 39, 21, 103, 20, 57, 447, 0, 31, 396, 662, 17, 592, 8, 1, 591, 925, 184, 597, 1887, 1859, 387, 676, 2866, 203, 1, 1, 70, 552, 0, 41, 970, 723, 87, 1, 81, 1126, 3591, 3147] payload_rows_per_dst=[10576, 14246, 9396, 4910, 6660, 10476, 10088, 9984] dup_rows_per_dst=[2112, 6970, 1784, 728, 1032, 3736, 2432, 3174] combine_rows_per_src=[10576, 14246, 9396, 4910, 6660, 10476, 10088, 9984] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=280051712 dispatch_meta_bytes=344152 combine_payload_bytes=280051712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280395864 ingress_bytes=314654720 link_bytes=314654720 token_bytes=560447576 weight_movement_bytes=34603008 total_nvlink_bytes=595050584 weight_to_token_ratio=0.061742 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=34176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.184020380e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80866 payload_rows_remote=70388 dup_rows=17438 dup_rows_remote=16472 dedup_ratio=0.177389 router_tokens_per_expert=[432, 195, 195, 10, 0, 34, 963, 143, 0, 1, 29, 4542, 4, 1146, 6136, 0, 67, 3, 192, 332, 2994, 6053, 135, 26, 9, 4473, 346, 569, 10, 0, 483, 1, 0, 0, 4, 674, 0, 2610, 696, 0, 0, 338, 15, 1201, 1940, 1875, 65, 1691, 3573, 15, 1, 312, 0, 6, 1894, 90, 361, 247, 516, 686, 211, 0, 0, 608] payload_rows_per_dst=[13402, 10478, 13802, 10408, 6946, 11450, 10082, 4298] dup_rows_per_dst=[2814, 966, 5802, 1374, 1022, 2800, 1700, 960] combine_rows_per_src=[13402, 10478, 13802, 10408, 6946, 11450, 10082, 4298] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74584 payload_rows_remote=61072 dup_rows=23720 dup_rows_remote=15176 dedup_ratio=0.241292 router_tokens_per_expert=[0, 160, 840, 2358, 705, 1895, 0, 4, 0, 9, 0, 13, 1408, 236, 2723, 1451, 33, 3386, 0, 0, 7, 13, 88, 1, 158, 310, 426, 310, 40, 99, 505, 0, 240, 117, 799, 1573, 157, 4169, 0, 0, 387, 0, 206, 3365, 2, 0, 1396, 789, 146, 2, 230, 2462, 1382, 4151, 456, 1, 1, 351, 0, 23, 3855, 5707, 3, 4] payload_rows_per_dst=[6488, 9678, 6906, 8180, 6304, 9934, 13582, 13512] dup_rows_per_dst=[720, 2002, 150, 1686, 1046, 2356, 7216, 8544] combine_rows_per_src=[6488, 9678, 6906, 8180, 6304, 9934, 13582, 13512] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81340 payload_rows_remote=69714 dup_rows=16964 dup_rows_remote=12658 dedup_ratio=0.172567 router_tokens_per_expert=[364, 428, 241, 107, 0, 143, 362, 249, 4, 18, 148, 3584, 0, 1621, 7353, 2, 39, 2, 219, 216, 2941, 6671, 121, 5, 4, 5968, 260, 371, 28, 2, 1047, 1, 0, 0, 0, 619, 1, 3543, 1979, 0, 0, 356, 34, 234, 1218, 2027, 38, 663, 3174, 2, 2, 255, 0, 7, 807, 135, 108, 176, 350, 298, 67, 0, 5, 535] payload_rows_per_dst=[3216, 15174, 6632, 12840, 9858, 8048, 11626, 13946] dup_rows_per_dst=[572, 3118, 454, 2522, 2426, 1092, 4306, 2474] combine_rows_per_src=[3216, 15174, 6632, 12840, 9858, 8048, 11626, 13946] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=304103424 dispatch_meta_bytes=364440 combine_payload_bytes=304103424 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=304467864 ingress_bytes=373309440 link_bytes=373309440 token_bytes=608571288 weight_movement_bytes=69206016 total_nvlink_bytes=677777304 weight_to_token_ratio=0.113719 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=28288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.404733369e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77628 payload_rows_remote=67044 dup_rows=20676 dup_rows_remote=18234 dedup_ratio=0.210327 router_tokens_per_expert=[0, 245, 104, 308, 204, 1739, 4, 0, 0, 4487, 0, 455, 23, 783, 10, 755, 0, 914, 2743, 1, 1574, 732, 0, 36, 0, 6735, 2, 80, 6842, 152, 0, 0, 498, 1, 39, 0, 109, 53, 17, 25, 66, 253, 782, 3678, 0, 1, 1184, 19, 2, 1, 362, 372, 0, 0, 714, 4395, 4376, 2396, 87, 0, 0, 5, 782, 7] payload_rows_per_dst=[11654, 10584, 9730, 15224, 5474, 8726, 10304, 5932] dup_rows_per_dst=[2306, 2442, 2270, 9838, 358, 1452, 1388, 622] combine_rows_per_src=[11654, 10584, 9730, 15224, 5474, 8726, 10304, 5932] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=276660224 dispatch_meta_bytes=332720 combine_payload_bytes=276660224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276992944 ingress_bytes=311263232 link_bytes=311263232 token_bytes=553653168 weight_movement_bytes=34603008 total_nvlink_bytes=588256176 weight_to_token_ratio=0.062499 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=19968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.171258483e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=290955264 dispatch_meta_bytes=339920 combine_payload_bytes=290955264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291295184 ingress_bytes=325558272 link_bytes=325558272 token_bytes=582250448 weight_movement_bytes=34603008 total_nvlink_bytes=616853456 weight_to_token_ratio=0.059430 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=27904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.225049568e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=274423808 dispatch_meta_bytes=324960 combine_payload_bytes=274423808 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=274748768 ingress_bytes=412835840 link_bytes=412835840 token_bytes=549172576 weight_movement_bytes=138412032 total_nvlink_bytes=687584608 weight_to_token_ratio=0.252037 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=12928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.553468030e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=280723456 dispatch_meta_bytes=330712 combine_payload_bytes=280723456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281054168 ingress_bytes=315326464 link_bytes=315326464 token_bytes=561777624 weight_movement_bytes=34603008 total_nvlink_bytes=596380632 weight_to_token_ratio=0.061596 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=29056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.186548098e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=285548544 dispatch_meta_bytes=329488 combine_payload_bytes=285548544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=285878032 ingress_bytes=320151552 link_bytes=320151552 token_bytes=571426576 weight_movement_bytes=34603008 total_nvlink_bytes=606029584 weight_to_token_ratio=0.060555 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=27136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.204704516e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77012 payload_rows_remote=65812 dup_rows=21292 dup_rows_remote=18294 dedup_ratio=0.216593 router_tokens_per_expert=[0, 269, 120, 114, 130, 1963, 5, 0, 20, 5250, 6, 605, 27, 525, 67, 455, 0, 1419, 2446, 10, 1450, 505, 6, 132, 8, 4384, 10, 382, 6223, 263, 0, 0, 650, 0, 250, 0, 687, 276, 104, 135, 380, 1171, 761, 3430, 0, 12, 338, 40, 0, 7, 398, 289, 1, 0, 344, 4466, 3966, 3297, 310, 6, 2, 25, 1013, 0] payload_rows_per_dst=[11136, 3274, 11232, 1288, 13864, 11084, 13934, 11200] dup_rows_per_dst=[1998, 136, 3302, 38, 2786, 4190, 5844, 2998] combine_rows_per_src=[11136, 3274, 11232, 1288, 13864, 11084, 13934, 11200] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84054 payload_rows_remote=73736 dup_rows=14250 dup_rows_remote=12174 dedup_ratio=0.144958 router_tokens_per_expert=[0, 102, 8, 131, 136, 1620, 6, 0, 0, 5383, 0, 610, 2, 574, 21, 553, 0, 1041, 3111, 3, 1955, 669, 2, 10, 3, 6457, 3, 131, 6683, 178, 0, 0, 825, 0, 36, 0, 288, 57, 1, 6, 44, 370, 717, 3699, 0, 0, 1357, 10, 0, 0, 337, 500, 0, 0, 450, 2273, 3889, 3421, 50, 0, 0, 1, 1429, 0] payload_rows_per_dst=[10262, 11778, 6610, 10730, 14150, 10318, 8072, 12134] dup_rows_per_dst=[1522, 2508, 750, 246, 1642, 2076, 1616, 3890] combine_rows_per_src=[10262, 11778, 6610, 10730, 14150, 10318, 8072, 12134] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=274612224 dispatch_meta_bytes=341112 combine_payload_bytes=274612224 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=274953336 ingress_bytes=343818240 link_bytes=343818240 token_bytes=549565560 weight_movement_bytes=69206016 total_nvlink_bytes=618771576 weight_to_token_ratio=0.125929 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=33536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.293760357e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=274186240 dispatch_meta_bytes=348496 combine_payload_bytes=274186240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274534736 ingress_bytes=308789248 link_bytes=308789248 token_bytes=548720976 weight_movement_bytes=34603008 total_nvlink_bytes=583323984 weight_to_token_ratio=0.063061 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=25472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.161949081e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78540 payload_rows_remote=67172 dup_rows=19764 dup_rows_remote=17508 dedup_ratio=0.201050 router_tokens_per_expert=[0, 45, 0, 2238, 328, 1, 6, 79, 254, 792, 1, 0, 5, 75, 2206, 907, 224, 14, 52, 186, 3985, 2161, 1, 0, 9, 1612, 146, 900, 4029, 4242, 0, 1113, 53, 1, 73, 556, 387, 350, 975, 129, 5, 0, 41, 7, 0, 19, 149, 0, 483, 2, 5560, 3737, 616, 0, 370, 1482, 1304, 1666, 4115, 0, 0, 5, 385, 1071] payload_rows_per_dst=[11368, 12014, 11828, 8652, 7930, 13552, 5596, 7600] dup_rows_per_dst=[2256, 4950, 2538, 1610, 1354, 5484, 310, 1262] combine_rows_per_src=[11368, 12014, 11828, 8652, 7930, 13552, 5596, 7600] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=288309248 dispatch_meta_bytes=347440 combine_payload_bytes=288309248 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=288656688 ingress_bytes=426721280 link_bytes=426721280 token_bytes=576965936 weight_movement_bytes=138412032 total_nvlink_bytes=715377968 weight_to_token_ratio=0.239896 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=19456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.605717823e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=250150912 dispatch_meta_bytes=304992 combine_payload_bytes=250150912 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=250455904 ingress_bytes=319356928 link_bytes=319356928 token_bytes=500606816 weight_movement_bytes=69206016 total_nvlink_bytes=569812832 weight_to_token_ratio=0.138244 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=34816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.201714410e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=285548544 dispatch_meta_bytes=329488 combine_payload_bytes=285548544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=285878032 ingress_bytes=320151552 link_bytes=320151552 token_bytes=571426576 weight_movement_bytes=34603008 total_nvlink_bytes=606029584 weight_to_token_ratio=0.060555 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=27136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.204704516e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=269565952 dispatch_meta_bytes=336424 combine_payload_bytes=269565952 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=269902376 ingress_bytes=373374976 link_bytes=373374976 token_bytes=539468328 weight_movement_bytes=103809024 total_nvlink_bytes=643277352 weight_to_token_ratio=0.192428 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=19200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.404979976e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80262 payload_rows_remote=68932 dup_rows=18042 dup_rows_remote=14258 dedup_ratio=0.183533 router_tokens_per_expert=[1, 204, 138, 232, 46, 945, 21, 2, 46, 5214, 0, 476, 62, 1203, 14, 620, 3, 920, 2888, 23, 3993, 2020, 10, 14, 1, 4790, 9, 146, 5071, 455, 0, 2, 288, 13, 8, 36, 476, 76, 41, 74, 286, 358, 1875, 5746, 2, 7, 2086, 9, 1, 3, 460, 302, 3, 0, 307, 1682, 1631, 810, 47, 2, 0, 4, 2903, 47] payload_rows_per_dst=[5706, 12126, 11330, 10390, 11190, 13802, 5214, 10504] dup_rows_per_dst=[734, 3144, 3784, 416, 976, 6936, 302, 1750] combine_rows_per_src=[5706, 12126, 11330, 10390, 11190, 13802, 5214, 10504] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=302022656 dispatch_meta_bytes=343640 combine_payload_bytes=302022656 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=302366296 ingress_bytes=371228672 link_bytes=371228672 token_bytes=604388952 weight_movement_bytes=69206016 total_nvlink_bytes=673594968 weight_to_token_ratio=0.114506 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=26240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396903606e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82294 payload_rows_remote=71340 dup_rows=16010 dup_rows_remote=14362 dedup_ratio=0.162862 router_tokens_per_expert=[444, 245, 127, 270, 0, 59, 122, 252, 0, 3, 4, 4522, 0, 1621, 6787, 15, 123, 69, 393, 563, 3092, 6347, 65, 30, 2, 4640, 553, 637, 23, 2, 404, 40, 0, 0, 0, 1209, 1, 2968, 1218, 0, 0, 374, 17, 478, 1457, 1891, 14, 261, 3059, 10, 0, 78, 0, 1, 1035, 339, 139, 290, 486, 1218, 491, 2, 2, 660] payload_rows_per_dst=[14228, 10664, 9588, 10954, 8788, 8134, 7994, 11944] dup_rows_per_dst=[2384, 1666, 2802, 1648, 2004, 850, 1050, 3606] combine_rows_per_src=[14228, 10664, 9588, 10954, 8788, 8134, 7994, 11944] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=275136512 dispatch_meta_bytes=338720 combine_payload_bytes=275136512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275475232 ingress_bytes=309739520 link_bytes=309739520 token_bytes=550611744 weight_movement_bytes=34603008 total_nvlink_bytes=585214752 weight_to_token_ratio=0.062845 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=26240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.165524878e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84600 payload_rows_remote=73326 dup_rows=13704 dup_rows_remote=13272 dedup_ratio=0.139404 router_tokens_per_expert=[0, 123, 630, 580, 255, 1330, 11, 0, 1, 5791, 0, 848, 25, 454, 5, 1057, 0, 1167, 3121, 6, 2492, 1425, 0, 2, 2, 5316, 1, 15, 6337, 519, 0, 0, 655, 1, 28, 0, 117, 18, 44, 82, 163, 285, 719, 2668, 0, 4, 857, 16, 0, 1, 213, 142, 0, 0, 1197, 2464, 5011, 2049, 50, 1, 0, 0, 854, 0] payload_rows_per_dst=[12592, 12984, 8526, 11274, 13492, 8326, 7254, 10152] dup_rows_per_dst=[3288, 3378, 1658, 432, 1072, 1098, 780, 1998] combine_rows_per_src=[12592, 12984, 8526, 11274, 13492, 8326, 7254, 10152] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80850 payload_rows_remote=68262 dup_rows=17454 dup_rows_remote=16480 dedup_ratio=0.177551 router_tokens_per_expert=[0, 97, 292, 374, 79, 563, 2, 0, 1, 3328, 6, 259, 57, 411, 27, 163, 0, 1812, 3526, 5, 2127, 1253, 3, 16, 0, 5293, 9, 903, 5788, 291, 0, 0, 392, 27, 94, 0, 152, 30, 194, 104, 241, 1905, 889, 4463, 1, 0, 711, 28, 0, 0, 381, 988, 0, 0, 389, 2795, 2693, 3456, 25, 0, 1, 1, 2503, 4] payload_rows_per_dst=[7398, 7444, 8564, 11884, 12588, 12314, 7882, 12776] dup_rows_per_dst=[802, 1060, 1868, 1108, 974, 4162, 1224, 6256] combine_rows_per_src=[7398, 7444, 8564, 11884, 12588, 12314, 7882, 12776] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79224 payload_rows_remote=66292 dup_rows=19080 dup_rows_remote=13864 dedup_ratio=0.194092 router_tokens_per_expert=[387, 221, 190, 13, 0, 241, 302, 710, 0, 7, 5, 4190, 1, 1749, 6538, 4, 123, 9, 243, 625, 3365, 5459, 205, 3, 6, 3559, 485, 256, 10, 0, 596, 22, 0, 0, 2, 525, 0, 3557, 2456, 0, 0, 412, 43, 479, 1540, 3299, 7, 446, 2801, 0, 0, 163, 0, 2, 1313, 144, 162, 299, 441, 354, 30, 0, 0, 1153] payload_rows_per_dst=[9684, 7870, 8252, 8820, 10466, 12932, 8772, 12428] dup_rows_per_dst=[1818, 1364, 894, 1048, 2614, 5216, 2758, 3368] combine_rows_per_src=[9684, 7870, 8252, 8820, 10466, 12932, 8772, 12428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81436 payload_rows_remote=68536 dup_rows=16868 dup_rows_remote=14142 dedup_ratio=0.171590 router_tokens_per_expert=[0, 15, 0, 2582, 367, 3, 2, 129, 586, 1186, 11, 0, 3, 132, 850, 1091, 177, 15, 388, 154, 4360, 1626, 22, 0, 276, 1503, 787, 1452, 5660, 2142, 2, 488, 289, 0, 96, 469, 311, 129, 845, 14, 1, 0, 178, 2, 0, 2, 8, 0, 465, 1, 4156, 2545, 372, 1, 714, 1316, 1801, 1962, 5722, 2, 15, 17, 804, 906] payload_rows_per_dst=[13590, 6336, 11532, 10404, 12900, 8532, 9364, 8778] dup_rows_per_dst=[4050, 1382, 1952, 2896, 2726, 162, 1464, 2236] combine_rows_per_src=[13590, 6336, 11532, 10404, 12900, 8532, 9364, 8778] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80172 payload_rows_remote=69008 dup_rows=18132 dup_rows_remote=15280 dedup_ratio=0.184448 router_tokens_per_expert=[13, 166, 1387, 3372, 1015, 1613, 0, 119, 19, 82, 0, 109, 1039, 27, 3051, 998, 1, 2035, 0, 8, 5, 6, 49, 35, 99, 247, 510, 749, 16, 54, 505, 5, 556, 53, 1118, 771, 219, 3885, 0, 0, 441, 0, 57, 3366, 99, 9, 1838, 748, 289, 14, 472, 2341, 863, 4869, 483, 5, 2, 286, 0, 30, 3999, 4852, 148, 5] payload_rows_per_dst=[11882, 9068, 11164, 11130, 10522, 10378, 7370, 8658] dup_rows_per_dst=[3488, 1582, 2852, 2944, 2682, 2738, 1564, 282] combine_rows_per_src=[11882, 9068, 11164, 11130, 10522, 10378, 7370, 8658] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=292208640 dispatch_meta_bytes=342808 combine_payload_bytes=292208640 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=292551448 ingress_bytes=396017664 link_bytes=396017664 token_bytes=584760088 weight_movement_bytes=103809024 total_nvlink_bytes=688569112 weight_to_token_ratio=0.177524 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=34304 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.490182588e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78962 payload_rows_remote=69182 dup_rows=19342 dup_rows_remote=16276 dedup_ratio=0.196757 router_tokens_per_expert=[0, 5, 0, 2840, 369, 56, 0, 8, 1121, 1291, 10, 0, 4, 208, 1411, 2041, 102, 4, 126, 660, 3786, 659, 3, 2, 294, 759, 127, 987, 2955, 3413, 5, 1481, 155, 0, 69, 604, 28, 65, 854, 4, 1, 0, 98, 13, 0, 1, 15, 3, 850, 1, 4936, 4157, 912, 0, 519, 1636, 1382, 720, 4792, 0, 2, 0, 1462, 1146] payload_rows_per_dst=[12544, 9780, 9424, 10808, 8268, 12900, 7614, 7624] dup_rows_per_dst=[3596, 3066, 1260, 2650, 1200, 4824, 946, 1800] combine_rows_per_src=[12544, 9780, 9424, 10808, 8268, 12900, 7614, 7624] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78960 payload_rows_remote=65740 dup_rows=19344 dup_rows_remote=16396 dedup_ratio=0.196777 router_tokens_per_expert=[505, 240, 296, 88, 0, 28, 388, 348, 0, 11, 91, 3488, 0, 1073, 6942, 16, 278, 3, 291, 518, 1421, 6217, 422, 8, 19, 6314, 317, 785, 17, 36, 519, 14, 0, 0, 4, 971, 2, 3814, 886, 2, 0, 198, 83, 861, 1448, 1848, 120, 1016, 3071, 3, 0, 549, 0, 14, 1415, 287, 83, 31, 297, 362, 134, 0, 7, 953] payload_rows_per_dst=[3358, 14160, 5462, 9282, 11882, 9262, 12334, 13220] dup_rows_per_dst=[428, 2106, 420, 1484, 4752, 1886, 5320, 2948] combine_rows_per_src=[3358, 14160, 5462, 9282, 11882, 9262, 12334, 13220] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80864 payload_rows_remote=68410 dup_rows=17440 dup_rows_remote=14774 dedup_ratio=0.177409 router_tokens_per_expert=[0, 1, 3, 2806, 286, 7, 1, 11, 276, 1171, 6, 0, 1, 60, 1888, 1393, 37, 2, 119, 439, 3320, 3074, 4, 0, 336, 705, 260, 1024, 3669, 2285, 0, 1585, 97, 1, 103, 1204, 165, 133, 1122, 73, 0, 16, 111, 9, 0, 6, 16, 1, 613, 0, 5355, 4649, 686, 0, 272, 1340, 982, 1608, 4513, 5, 2, 2, 576, 723] payload_rows_per_dst=[8302, 8240, 11646, 6514, 10884, 10828, 12454, 11996] dup_rows_per_dst=[1062, 1350, 2344, 1306, 2250, 200, 2666, 6262] combine_rows_per_src=[8302, 8240, 11646, 6514, 10884, 10828, 12454, 11996] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=282345472 dispatch_meta_bytes=332760 combine_payload_bytes=282345472 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=282678232 ingress_bytes=351551488 link_bytes=351551488 token_bytes=565023704 weight_movement_bytes=69206016 total_nvlink_bytes=634229720 weight_to_token_ratio=0.122483 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=19968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.322859947e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77882 payload_rows_remote=65848 dup_rows=20422 dup_rows_remote=19160 dedup_ratio=0.207743 router_tokens_per_expert=[1084, 70, 1, 7055, 2, 8, 275, 692, 11, 215, 42, 3, 0, 0, 51, 1293, 0, 0, 27, 0, 0, 21, 17, 4937, 3, 0, 2, 7, 2696, 0, 0, 0, 35, 883, 722, 93, 7098, 1, 1748, 5033, 0, 0, 7, 274, 2, 0, 8004, 9, 55, 275, 0, 0, 1, 26, 34, 1, 453, 0, 0, 2759, 4, 1791, 32, 1300] payload_rows_per_dst=[15422, 12034, 12976, 5402, 6216, 584, 14440, 10808] dup_rows_per_dst=[10956, 1262, 5032, 14, 748, 0, 540, 1870] combine_rows_per_src=[15422, 12034, 12976, 5402, 6216, 584, 14440, 10808] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=271532032 dispatch_meta_bytes=320624 combine_payload_bytes=271532032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271852656 ingress_bytes=306135040 link_bytes=306135040 token_bytes=543384688 weight_movement_bytes=34603008 total_nvlink_bytes=577987696 weight_to_token_ratio=0.063680 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.151961510e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=280723456 dispatch_meta_bytes=330712 combine_payload_bytes=280723456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281054168 ingress_bytes=315326464 link_bytes=315326464 token_bytes=561777624 weight_movement_bytes=34603008 total_nvlink_bytes=596380632 weight_to_token_ratio=0.061596 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=29056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.186548098e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=282656768 dispatch_meta_bytes=337152 combine_payload_bytes=282656768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282993920 ingress_bytes=317259776 link_bytes=317259776 token_bytes=565650688 weight_movement_bytes=34603008 total_nvlink_bytes=600253696 weight_to_token_ratio=0.061174 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=24960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.193822996e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=300343296 dispatch_meta_bytes=346392 combine_payload_bytes=300343296 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=300689688 ingress_bytes=404152320 link_bytes=404152320 token_bytes=601032984 weight_movement_bytes=103809024 total_nvlink_bytes=704842008 weight_to_token_ratio=0.172718 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=30848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520792643e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=279601152 dispatch_meta_bytes=338968 combine_payload_bytes=279601152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279940120 ingress_bytes=314204160 link_bytes=314204160 token_bytes=559541272 weight_movement_bytes=34603008 total_nvlink_bytes=594144280 weight_to_token_ratio=0.061842 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=39936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.182324959e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=280207360 dispatch_meta_bytes=332736 combine_payload_bytes=280207360 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=280540096 ingress_bytes=384016384 link_bytes=384016384 token_bytes=560747456 weight_movement_bytes=103809024 total_nvlink_bytes=664556480 weight_to_token_ratio=0.185126 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.445022737e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85246 payload_rows_remote=73470 dup_rows=13058 dup_rows_remote=9776 dedup_ratio=0.132833 router_tokens_per_expert=[715, 68, 5, 5168, 0, 12, 442, 441, 5, 326, 52, 56, 2, 0, 61, 3326, 1, 1, 57, 2, 0, 10, 10, 5092, 5, 0, 24, 0, 1801, 2, 1, 0, 29, 1852, 105, 884, 6234, 5, 1551, 4878, 0, 0, 0, 487, 3, 0, 7529, 6, 59, 222, 25, 1, 0, 60, 8, 0, 411, 5, 0, 3060, 11, 3072, 100, 870] payload_rows_per_dst=[3206, 7252, 10290, 12374, 12104, 15382, 12862, 11776] dup_rows_per_dst=[160, 404, 56, 1628, 3402, 3770, 356, 3282] combine_rows_per_src=[3206, 7252, 10290, 12374, 12104, 15382, 12862, 11776] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85686 payload_rows_remote=69524 dup_rows=12618 dup_rows_remote=8324 dedup_ratio=0.128357 router_tokens_per_expert=[2294, 150, 0, 6113, 0, 4, 177, 284, 0, 45, 110, 7, 0, 0, 162, 2318, 0, 1, 6, 0, 0, 24, 210, 4699, 3, 0, 1, 0, 2325, 0, 0, 0, 20, 1748, 449, 58, 5972, 0, 1481, 4824, 0, 0, 7, 745, 18, 0, 7977, 0, 50, 86, 0, 0, 0, 34, 0, 0, 268, 0, 0, 3228, 14, 1617, 37, 1586] payload_rows_per_dst=[5636, 8130, 9752, 14204, 8348, 16162, 12162, 11292] dup_rows_per_dst=[182, 1168, 128, 2680, 1836, 4294, 122, 2208] combine_rows_per_src=[5636, 8130, 9752, 14204, 8348, 16162, 12162, 11292] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=269713408 dispatch_meta_bytes=340032 combine_payload_bytes=269713408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270053440 ingress_bytes=304316416 link_bytes=304316416 token_bytes=539766848 weight_movement_bytes=34603008 total_nvlink_bytes=574369856 weight_to_token_ratio=0.064107 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=30976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.145118174e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72866 payload_rows_remote=60736 dup_rows=25438 dup_rows_remote=21226 dedup_ratio=0.258769 router_tokens_per_expert=[3, 80, 320, 289, 36, 2536, 10, 0, 1, 5038, 1, 923, 10, 1794, 1, 221, 0, 1942, 2197, 0, 4206, 449, 0, 2, 2, 3540, 8, 59, 4201, 372, 1, 0, 366, 1, 202, 0, 236, 368, 266, 31, 400, 758, 618, 3995, 2, 0, 353, 26, 3, 0, 736, 321, 6, 0, 495, 5009, 1904, 3490, 17, 0, 0, 8, 1299, 0] payload_rows_per_dst=[12130, 12282, 12642, 14508, 2570, 4130, 11146, 3458] dup_rows_per_dst=[4212, 3696, 4950, 9848, 370, 184, 1994, 184] combine_rows_per_src=[12130, 12282, 12642, 14508, 2570, 4130, 11146, 3458] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=300933120 dispatch_meta_bytes=332984 combine_payload_bytes=300933120 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=301266104 ingress_bytes=335536128 link_bytes=335536128 token_bytes=602199224 weight_movement_bytes=34603008 total_nvlink_bytes=636802232 weight_to_token_ratio=0.057461 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=18688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262595437e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=284770304 dispatch_meta_bytes=311392 combine_payload_bytes=284770304 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=285081696 ingress_bytes=388579328 link_bytes=388579328 token_bytes=569852000 weight_movement_bytes=103809024 total_nvlink_bytes=673661024 weight_to_token_ratio=0.182168 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=38144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.462192728e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81912 payload_rows_remote=65968 dup_rows=16392 dup_rows_remote=10702 dedup_ratio=0.166748 router_tokens_per_expert=[925, 293, 3, 5757, 51, 21, 861, 466, 3, 499, 18, 92, 38, 0, 16, 1328, 2, 13, 48, 1, 18, 27, 40, 2782, 1, 3, 301, 0, 3235, 1, 3, 6, 30, 2086, 276, 25, 6598, 5, 3148, 6123, 1, 9, 0, 150, 72, 53, 7886, 0, 48, 722, 0, 1, 0, 236, 0, 20, 491, 10, 5, 1418, 5, 1847, 92, 943] payload_rows_per_dst=[12106, 14020, 15944, 8042, 8914, 550, 14024, 8312] dup_rows_per_dst=[3042, 2214, 5690, 664, 2226, 20, 1226, 1310] combine_rows_per_src=[12106, 14020, 15944, 8042, 8914, 550, 14024, 8312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=280207360 dispatch_meta_bytes=332736 combine_payload_bytes=280207360 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=280540096 ingress_bytes=384016384 link_bytes=384016384 token_bytes=560747456 weight_movement_bytes=103809024 total_nvlink_bytes=664556480 weight_to_token_ratio=0.185126 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.445022737e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=283369472 dispatch_meta_bytes=341832 combine_payload_bytes=283369472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=283711304 ingress_bytes=317972480 link_bytes=317972480 token_bytes=567080776 weight_movement_bytes=34603008 total_nvlink_bytes=601683784 weight_to_token_ratio=0.061020 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.196504843e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=269271040 dispatch_meta_bytes=328544 combine_payload_bytes=269271040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269599584 ingress_bytes=303874048 link_bytes=303874048 token_bytes=538870624 weight_movement_bytes=34603008 total_nvlink_bytes=573473632 weight_to_token_ratio=0.064214 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=28288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.143453579e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79388 payload_rows_remote=66998 dup_rows=18916 dup_rows_remote=14242 dedup_ratio=0.192424 router_tokens_per_expert=[1, 178, 0, 1995, 507, 19, 8, 27, 915, 1206, 1, 0, 14, 66, 999, 1485, 60, 16, 76, 154, 4219, 1990, 5, 4, 81, 1748, 177, 570, 3172, 3969, 0, 1987, 248, 0, 94, 2247, 327, 125, 1310, 1, 2, 0, 324, 19, 1, 0, 24, 0, 489, 0, 3826, 4296, 238, 1, 433, 2311, 660, 2132, 2412, 1, 1, 0, 1163, 818] payload_rows_per_dst=[8718, 7568, 11220, 12390, 11446, 8108, 12270, 7668] dup_rows_per_dst=[1576, 1804, 1828, 4674, 3602, 284, 3266, 1882] combine_rows_per_src=[8718, 7568, 11220, 12390, 11446, 8108, 12270, 7668] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=248774656 dispatch_meta_bytes=327848 combine_payload_bytes=248774656 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=249102504 ingress_bytes=317980672 link_bytes=317980672 token_bytes=497877160 weight_movement_bytes=69206016 total_nvlink_bytes=567083176 weight_to_token_ratio=0.139002 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=27776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.196535669e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85824 payload_rows_remote=72376 dup_rows=12480 dup_rows_remote=9856 dedup_ratio=0.126953 router_tokens_per_expert=[2118, 357, 17, 5723, 0, 33, 199, 291, 8, 141, 52, 18, 0, 0, 34, 1535, 0, 3, 45, 0, 0, 240, 44, 4261, 7, 0, 2, 0, 2304, 0, 0, 0, 57, 1847, 1440, 71, 5198, 0, 2338, 5914, 0, 0, 12, 902, 9, 0, 8028, 0, 67, 151, 0, 0, 0, 89, 10, 0, 488, 5, 0, 2515, 2, 1136, 42, 1399] payload_rows_per_dst=[5656, 13290, 9968, 13448, 8662, 14442, 10828, 9530] dup_rows_per_dst=[374, 2114, 1164, 2624, 2094, 2264, 202, 1644] combine_rows_per_src=[5656, 13290, 9968, 13448, 8662, 14442, 10828, 9530] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84936 payload_rows_remote=79378 dup_rows=13368 dup_rows_remote=13108 dedup_ratio=0.135986 router_tokens_per_expert=[899, 113, 29, 6028, 17, 51, 539, 226, 36, 220, 14, 9, 0, 0, 14, 1544, 1, 1, 8, 0, 13, 293, 4, 3959, 2, 4, 7, 3, 2308, 0, 0, 6, 1, 2252, 646, 10, 5865, 0, 1870, 6019, 0, 0, 0, 268, 33, 0, 7801, 3, 26, 312, 1, 0, 0, 149, 0, 0, 336, 1, 2, 2411, 12, 1887, 76, 2823] payload_rows_per_dst=[3616, 13794, 8324, 13750, 5558, 15840, 12272, 11782] dup_rows_per_dst=[132, 1918, 234, 2966, 260, 4110, 434, 3314] combine_rows_per_src=[3616, 13794, 8324, 13750, 5558, 15840, 12272, 11782] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84336 payload_rows_remote=72780 dup_rows=13968 dup_rows_remote=13778 dedup_ratio=0.142090 router_tokens_per_expert=[0, 0, 0, 2098, 263, 13, 0, 3, 606, 584, 2, 0, 3, 36, 3396, 2230, 87, 0, 101, 200, 4951, 2446, 0, 3, 14, 2128, 160, 274, 4366, 2583, 0, 1501, 153, 0, 80, 1042, 151, 105, 446, 5, 0, 0, 125, 4, 0, 0, 22, 0, 375, 0, 5722, 3645, 546, 0, 112, 2397, 325, 1160, 3108, 0, 0, 0, 695, 886] payload_rows_per_dst=[9978, 10878, 13222, 9386, 11082, 11556, 11778, 6456] dup_rows_per_dst=[992, 2836, 2354, 2208, 1614, 190, 2372, 1402] combine_rows_per_src=[9978, 10878, 13222, 9386, 11082, 11556, 11778, 6456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81698 payload_rows_remote=68632 dup_rows=16606 dup_rows_remote=11792 dedup_ratio=0.168925 router_tokens_per_expert=[629, 150, 426, 15, 7, 79, 598, 584, 57, 3, 24, 3774, 25, 949, 6331, 0, 267, 3, 278, 556, 1969, 5583, 104, 180, 27, 5173, 288, 355, 6, 33, 372, 7, 1, 2, 0, 188, 47, 3692, 1173, 0, 3, 413, 0, 866, 1489, 1412, 66, 1907, 4149, 0, 6, 156, 0, 53, 1585, 122, 323, 484, 268, 1060, 290, 0, 0, 545] payload_rows_per_dst=[14066, 8990, 13066, 11306, 8850, 9958, 10292, 5170] dup_rows_per_dst=[3572, 674, 4814, 1216, 1356, 2354, 1850, 770] combine_rows_per_src=[14066, 8990, 13066, 11306, 8850, 9958, 10292, 5170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80850 payload_rows_remote=68262 dup_rows=17454 dup_rows_remote=16480 dedup_ratio=0.177551 router_tokens_per_expert=[0, 97, 292, 374, 79, 563, 2, 0, 1, 3328, 6, 259, 57, 411, 27, 163, 0, 1812, 3526, 5, 2127, 1253, 3, 16, 0, 5293, 9, 903, 5788, 291, 0, 0, 392, 27, 94, 0, 152, 30, 194, 104, 241, 1905, 889, 4463, 1, 0, 711, 28, 0, 0, 381, 988, 0, 0, 389, 2795, 2693, 3456, 25, 0, 1, 1, 2503, 4] payload_rows_per_dst=[7398, 7444, 8564, 11884, 12588, 12314, 7882, 12776] dup_rows_per_dst=[802, 1060, 1868, 1108, 974, 4162, 1224, 6256] combine_rows_per_src=[7398, 7444, 8564, 11884, 12588, 12314, 7882, 12776] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=274423808 dispatch_meta_bytes=324960 combine_payload_bytes=274423808 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=274748768 ingress_bytes=412835840 link_bytes=412835840 token_bytes=549172576 weight_movement_bytes=138412032 total_nvlink_bytes=687584608 weight_to_token_ratio=0.252037 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=12928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.553468030e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81588 payload_rows_remote=66438 dup_rows=16716 dup_rows_remote=10516 dedup_ratio=0.170044 router_tokens_per_expert=[3, 88, 219, 151, 240, 1335, 30, 1, 4, 5880, 0, 432, 8, 421, 3, 573, 0, 916, 2780, 11, 2049, 972, 0, 9, 1, 6447, 13, 128, 7605, 398, 0, 0, 423, 3, 27, 0, 160, 17, 60, 104, 341, 148, 639, 3335, 1, 0, 282, 7, 0, 0, 360, 457, 0, 5, 246, 3160, 4215, 3251, 105, 1, 0, 0, 1088, 0] payload_rows_per_dst=[10922, 9840, 9052, 1072, 15694, 8406, 15150, 11452] dup_rows_per_dst=[1642, 1628, 2036, 8, 1104, 1100, 6200, 2998] combine_rows_per_src=[10922, 9840, 9052, 1072, 15694, 8406, 15150, 11452] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84550 payload_rows_remote=69778 dup_rows=13754 dup_rows_remote=5730 dedup_ratio=0.139913 router_tokens_per_expert=[0, 54, 69, 1017, 6, 42, 7509, 3, 592, 452, 2, 0, 3907, 0, 1507, 2, 10, 7496, 860, 0, 0, 1, 1, 169, 34, 0, 0, 0, 0, 10, 14, 0, 1310, 637, 3, 18, 0, 55, 1077, 0, 12, 6, 2585, 0, 1, 1013, 4339, 0, 14, 6538, 1, 1250, 24, 1, 5482, 3, 0, 15, 1008, 0, 0, 1, 0, 2] payload_rows_per_dst=[15362, 14772, 15286, 13088, 5576, 6916, 3454, 10096] dup_rows_per_dst=[2038, 8024, 1788, 104, 624, 318, 224, 634] combine_rows_per_src=[15362, 14772, 15286, 13088, 5576, 6916, 3454, 10096] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=270204928 dispatch_meta_bytes=306680 combine_payload_bytes=270204928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270511608 ingress_bytes=304807936 link_bytes=304807936 token_bytes=540716536 weight_movement_bytes=34603008 total_nvlink_bytes=575319544 weight_to_token_ratio=0.063995 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=34816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.146967724e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77628 payload_rows_remote=67044 dup_rows=20676 dup_rows_remote=18234 dedup_ratio=0.210327 router_tokens_per_expert=[0, 245, 104, 308, 204, 1739, 4, 0, 0, 4487, 0, 455, 23, 783, 10, 755, 0, 914, 2743, 1, 1574, 732, 0, 36, 0, 6735, 2, 80, 6842, 152, 0, 0, 498, 1, 39, 0, 109, 53, 17, 25, 66, 253, 782, 3678, 0, 1, 1184, 19, 2, 1, 362, 372, 0, 0, 714, 4395, 4376, 2396, 87, 0, 0, 5, 782, 7] payload_rows_per_dst=[11654, 10584, 9730, 15224, 5474, 8726, 10304, 5932] dup_rows_per_dst=[2306, 2442, 2270, 9838, 358, 1452, 1388, 622] combine_rows_per_src=[11654, 10584, 9730, 15224, 5474, 8726, 10304, 5932] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=298106880 dispatch_meta_bytes=346232 combine_payload_bytes=298106880 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=298453112 ingress_bytes=367312896 link_bytes=367312896 token_bytes=596559992 weight_movement_bytes=69206016 total_nvlink_bytes=665766008 weight_to_token_ratio=0.116008 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=39936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.382168857e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=281116672 dispatch_meta_bytes=321696 combine_payload_bytes=281116672 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=281438368 ingress_bytes=350322688 link_bytes=350322688 token_bytes=562555040 weight_movement_bytes=69206016 total_nvlink_bytes=631761056 weight_to_token_ratio=0.123021 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.318236072e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78076 payload_rows_remote=63934 dup_rows=20228 dup_rows_remote=10790 dedup_ratio=0.205770 router_tokens_per_expert=[0, 48, 0, 1825, 225, 28, 7, 14, 70, 852, 1, 5, 1, 61, 3249, 2012, 57, 1, 464, 460, 2850, 2862, 2, 2, 174, 655, 1031, 655, 2813, 3592, 18, 2161, 77, 1, 75, 611, 224, 260, 1234, 23, 0, 3, 119, 22, 1, 0, 52, 0, 1080, 1, 4944, 3166, 386, 0, 67, 2413, 521, 1776, 4889, 9, 5, 4, 700, 294] payload_rows_per_dst=[4130, 10490, 8884, 8150, 9118, 10098, 13064, 14142] dup_rows_per_dst=[164, 2012, 1078, 1238, 1518, 184, 4596, 9438] combine_rows_per_src=[4130, 10490, 8884, 8150, 9118, 10098, 13064, 14142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=279601152 dispatch_meta_bytes=338968 combine_payload_bytes=279601152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279940120 ingress_bytes=314204160 link_bytes=314204160 token_bytes=559541272 weight_movement_bytes=34603008 total_nvlink_bytes=594144280 weight_to_token_ratio=0.061842 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=39936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.182324959e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=296452096 dispatch_meta_bytes=328928 combine_payload_bytes=296452096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296781024 ingress_bytes=331055104 link_bytes=331055104 token_bytes=593233120 weight_movement_bytes=34603008 total_nvlink_bytes=627836128 weight_to_token_ratio=0.058330 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=29952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.245733704e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=325132288 dispatch_meta_bytes=369944 combine_payload_bytes=325132288 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=325502232 ingress_bytes=463544320 link_bytes=463544320 token_bytes=650634520 weight_movement_bytes=138412032 total_nvlink_bytes=789046552 weight_to_token_ratio=0.212734 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.744279958e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=272130048 dispatch_meta_bytes=307816 combine_payload_bytes=272130048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272437864 ingress_bytes=306733056 link_bytes=306733056 token_bytes=544567912 weight_movement_bytes=34603008 total_nvlink_bytes=579170920 weight_to_token_ratio=0.063542 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=26368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.154211796e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73674 payload_rows_remote=62536 dup_rows=24630 dup_rows_remote=20736 dedup_ratio=0.250549 router_tokens_per_expert=[0, 78, 114, 1930, 12, 115, 7473, 42, 247, 175, 1, 11, 3440, 0, 750, 49, 120, 7549, 757, 4, 0, 1, 11, 152, 356, 2, 0, 0, 3, 79, 16, 23, 939, 1781, 5, 3, 1, 36, 1181, 72, 41, 17, 2855, 0, 0, 1231, 3395, 0, 56, 6690, 0, 1857, 49, 6, 4145, 16, 1, 3, 1229, 1, 6, 1, 0, 25] payload_rows_per_dst=[4426, 8692, 2078, 3208, 15520, 13690, 14922, 11138] dup_rows_per_dst=[156, 654, 12, 212, 7462, 2170, 10070, 3894] combine_rows_per_src=[4426, 8692, 2078, 3208, 15520, 13690, 14922, 11138] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84374 payload_rows_remote=74606 dup_rows=13930 dup_rows_remote=11932 dedup_ratio=0.141703 router_tokens_per_expert=[0, 8, 218, 1232, 4, 6, 6649, 1, 258, 146, 0, 0, 3671, 0, 1595, 1, 7, 7389, 1386, 0, 0, 1, 1, 41, 94, 0, 0, 1, 0, 20, 176, 10, 983, 643, 0, 3, 0, 34, 1018, 0, 25, 21, 3783, 0, 0, 2345, 2983, 0, 8, 7068, 0, 1812, 628, 0, 4014, 6, 0, 5, 846, 0, 10, 0, 0, 2] payload_rows_per_dst=[8292, 10282, 13048, 14410, 9142, 9768, 10584, 8848] dup_rows_per_dst=[1038, 1060, 2594, 328, 2492, 1998, 2352, 2068] combine_rows_per_src=[8292, 10282, 13048, 14410, 9142, 9768, 10584, 8848] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=285810688 dispatch_meta_bytes=302032 combine_payload_bytes=285810688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286112720 ingress_bytes=320413696 link_bytes=320413696 token_bytes=571923408 weight_movement_bytes=34603008 total_nvlink_bytes=606526416 weight_to_token_ratio=0.060503 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=23808 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.205690943e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70176 payload_rows_remote=53848 dup_rows=28128 dup_rows_remote=11276 dedup_ratio=0.286133 router_tokens_per_expert=[2124, 47, 0, 6172, 1, 3, 315, 276, 0, 301, 108, 29, 9, 0, 79, 1189, 0, 1, 17, 14, 0, 107, 31, 3602, 0, 0, 4, 0, 2867, 0, 0, 0, 11, 1691, 248, 57, 5942, 0, 4032, 5835, 0, 0, 0, 710, 5, 2, 7652, 0, 109, 153, 0, 19, 0, 9, 3, 3, 534, 3, 15, 2381, 2, 1559, 70, 811] payload_rows_per_dst=[16328, 13178, 7476, 5742, 12996, 1432, 6938, 6086] dup_rows_per_dst=[16852, 1922, 68, 0, 8566, 2, 146, 572] combine_rows_per_src=[16328, 13178, 7476, 5742, 12996, 1432, 6938, 6086] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=256147456 dispatch_meta_bytes=333088 combine_payload_bytes=256147456 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=256480544 ingress_bytes=325353472 link_bytes=325353472 token_bytes=512628000 weight_movement_bytes=69206016 total_nvlink_bytes=581834016 weight_to_token_ratio=0.135002 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.022500 gemm_active_groups=11 gemm_peak_group_rows=32512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.224278922e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=305586176 dispatch_meta_bytes=346152 combine_payload_bytes=305586176 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=305932328 ingress_bytes=409395200 link_bytes=409395200 token_bytes=611518504 weight_movement_bytes=103809024 total_nvlink_bytes=715327528 weight_to_token_ratio=0.169756 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=21760 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.540521179e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=274612224 dispatch_meta_bytes=341112 combine_payload_bytes=274612224 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=274953336 ingress_bytes=343818240 link_bytes=343818240 token_bytes=549565560 weight_movement_bytes=69206016 total_nvlink_bytes=618771576 weight_to_token_ratio=0.125929 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=33536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.293760357e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=272130048 dispatch_meta_bytes=307816 combine_payload_bytes=272130048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272437864 ingress_bytes=306733056 link_bytes=306733056 token_bytes=544567912 weight_movement_bytes=34603008 total_nvlink_bytes=579170920 weight_to_token_ratio=0.063542 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=26368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.154211796e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87112 payload_rows_remote=73522 dup_rows=11192 dup_rows_remote=9222 dedup_ratio=0.113851 router_tokens_per_expert=[56, 91, 66, 2212, 8, 38, 6597, 12, 799, 286, 26, 0, 1359, 0, 951, 176, 0, 6381, 1066, 8, 0, 7, 45, 273, 112, 35, 0, 32, 0, 25, 34, 34, 1663, 713, 1, 4, 36, 30, 2053, 0, 61, 52, 5013, 0, 1, 701, 4513, 9, 73, 6847, 0, 1467, 293, 18, 2939, 142, 7, 7, 1661, 3, 2, 75, 3, 36] payload_rows_per_dst=[14580, 6702, 13590, 13806, 7648, 11034, 8576, 11176] dup_rows_per_dst=[3580, 492, 1970, 432, 1352, 640, 1288, 1438] combine_rows_per_src=[14580, 6702, 13590, 13806, 7648, 11034, 8576, 11176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=261873664 dispatch_meta_bytes=298896 combine_payload_bytes=261873664 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=262172560 ingress_bytes=331079680 link_bytes=331079680 token_bytes=524046224 weight_movement_bytes=69206016 total_nvlink_bytes=593252240 weight_to_token_ratio=0.132061 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=12544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.245826182e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84600 payload_rows_remote=73326 dup_rows=13704 dup_rows_remote=13272 dedup_ratio=0.139404 router_tokens_per_expert=[0, 123, 630, 580, 255, 1330, 11, 0, 1, 5791, 0, 848, 25, 454, 5, 1057, 0, 1167, 3121, 6, 2492, 1425, 0, 2, 2, 5316, 1, 15, 6337, 519, 0, 0, 655, 1, 28, 0, 117, 18, 44, 82, 163, 285, 719, 2668, 0, 4, 857, 16, 0, 1, 213, 142, 0, 0, 1197, 2464, 5011, 2049, 50, 1, 0, 0, 854, 0] payload_rows_per_dst=[12592, 12984, 8526, 11274, 13492, 8326, 7254, 10152] dup_rows_per_dst=[3288, 3378, 1658, 432, 1072, 1098, 780, 1998] combine_rows_per_src=[12592, 12984, 8526, 11274, 13492, 8326, 7254, 10152] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=220561408 dispatch_meta_bytes=260496 combine_payload_bytes=220561408 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=220821904 ingress_bytes=289767424 link_bytes=289767424 token_bytes=441383312 weight_movement_bytes=69206016 total_nvlink_bytes=510589328 weight_to_token_ratio=0.156793 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=18304 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.090371488e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89228 payload_rows_remote=76054 dup_rows=9076 dup_rows_remote=8794 dedup_ratio=0.092326 router_tokens_per_expert=[0, 1, 170, 941, 2, 11, 6649, 6, 205, 152, 0, 0, 2923, 0, 1508, 3, 4, 7242, 1011, 0, 0, 4, 1, 248, 181, 1, 0, 1, 0, 22, 5, 14, 1638, 1303, 3, 0, 0, 13, 2358, 1, 71, 4, 3539, 0, 0, 1117, 4049, 2, 100, 6504, 1, 740, 45, 1, 5272, 8, 0, 3, 1061, 2, 4, 0, 0, 8] payload_rows_per_dst=[13936, 8726, 14958, 13174, 8912, 8864, 11266, 9392] dup_rows_per_dst=[1624, 856, 2062, 282, 1720, 602, 1068, 862] combine_rows_per_src=[13936, 8726, 14958, 13174, 8912, 8864, 11266, 9392] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88878 payload_rows_remote=81306 dup_rows=9426 dup_rows_remote=8132 dedup_ratio=0.095886 router_tokens_per_expert=[0, 38, 30, 1799, 0, 19, 5178, 8, 61, 73, 3, 0, 2164, 0, 1099, 3, 13, 6872, 505, 0, 0, 0, 18, 60, 40, 8, 0, 0, 0, 39, 11, 17, 2096, 705, 14, 13, 1, 12, 1589, 3, 34, 46, 5243, 0, 0, 1988, 4913, 3, 29, 6288, 1, 1101, 11, 0, 6002, 26, 0, 4, 962, 0, 9, 1, 0, 0] payload_rows_per_dst=[12250, 6396, 14164, 12646, 7572, 12330, 12690, 10830] dup_rows_per_dst=[1894, 410, 772, 160, 1294, 2298, 1650, 948] combine_rows_per_src=[12250, 6396, 14164, 12646, 7572, 12330, 12690, 10830] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84054 payload_rows_remote=73736 dup_rows=14250 dup_rows_remote=12174 dedup_ratio=0.144958 router_tokens_per_expert=[0, 102, 8, 131, 136, 1620, 6, 0, 0, 5383, 0, 610, 2, 574, 21, 553, 0, 1041, 3111, 3, 1955, 669, 2, 10, 3, 6457, 3, 131, 6683, 178, 0, 0, 825, 0, 36, 0, 288, 57, 1, 6, 44, 370, 717, 3699, 0, 0, 1357, 10, 0, 0, 337, 500, 0, 0, 450, 2273, 3889, 3421, 50, 0, 0, 1, 1429, 0] payload_rows_per_dst=[10262, 11778, 6610, 10730, 14150, 10318, 8072, 12134] dup_rows_per_dst=[1522, 2508, 750, 246, 1642, 2076, 1616, 3890] combine_rows_per_src=[10262, 11778, 6610, 10730, 14150, 10318, 8072, 12134] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80918 payload_rows_remote=74244 dup_rows=17386 dup_rows_remote=16866 dedup_ratio=0.176860 router_tokens_per_expert=[0, 110, 7, 2608, 287, 60, 2, 424, 905, 1474, 11, 0, 0, 157, 2650, 2602, 166, 13, 158, 237, 2435, 569, 17, 2, 99, 1954, 222, 844, 4224, 1928, 11, 2651, 246, 5, 189, 387, 160, 164, 795, 36, 22, 2, 139, 75, 0, 0, 61, 1, 567, 0, 5401, 1956, 172, 0, 537, 2040, 1003, 1465, 4491, 2, 65, 5, 1258, 1081] payload_rows_per_dst=[12446, 11360, 6674, 11866, 10764, 11122, 9190, 7496] dup_rows_per_dst=[3532, 4238, 520, 3552, 1648, 280, 1354, 2262] combine_rows_per_src=[12446, 11360, 6674, 11866, 10764, 11122, 9190, 7496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84936 payload_rows_remote=79378 dup_rows=13368 dup_rows_remote=13108 dedup_ratio=0.135986 router_tokens_per_expert=[899, 113, 29, 6028, 17, 51, 539, 226, 36, 220, 14, 9, 0, 0, 14, 1544, 1, 1, 8, 0, 13, 293, 4, 3959, 2, 4, 7, 3, 2308, 0, 0, 6, 1, 2252, 646, 10, 5865, 0, 1870, 6019, 0, 0, 0, 268, 33, 0, 7801, 3, 26, 312, 1, 0, 0, 149, 0, 0, 336, 1, 2, 2411, 12, 1887, 76, 2823] payload_rows_per_dst=[3616, 13794, 8324, 13750, 5558, 15840, 12272, 11782] dup_rows_per_dst=[132, 1918, 234, 2966, 260, 4110, 434, 3314] combine_rows_per_src=[3616, 13794, 8324, 13750, 5558, 15840, 12272, 11782] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85668 payload_rows_remote=74242 dup_rows=12636 dup_rows_remote=12310 dedup_ratio=0.128540 router_tokens_per_expert=[1374, 31, 8, 6061, 0, 14, 305, 586, 3, 179, 50, 0, 3, 0, 114, 2453, 0, 6, 56, 0, 0, 99, 35, 5405, 0, 0, 0, 2, 1317, 2, 1, 0, 35, 2151, 1068, 33, 5478, 0, 1268, 4900, 4, 0, 12, 665, 0, 0, 7734, 43, 41, 331, 0, 9, 0, 15, 2, 0, 366, 2, 0, 2883, 11, 1736, 68, 2193] payload_rows_per_dst=[4458, 5422, 10990, 13252, 12888, 15686, 11426, 11546] dup_rows_per_dst=[178, 182, 212, 1514, 3486, 3766, 326, 2972] combine_rows_per_src=[4458, 5422, 10990, 13252, 12888, 15686, 11426, 11546] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82608 payload_rows_remote=70584 dup_rows=15696 dup_rows_remote=12120 dedup_ratio=0.159668 router_tokens_per_expert=[6, 0, 6135, 0, 13, 0, 0, 0, 0, 25, 0, 0, 4, 2285, 1098, 137, 4251, 459, 300, 10, 278, 3733, 0, 776, 45, 0, 2, 23, 964, 4, 2348, 36, 0, 1, 23, 0, 806, 28, 4, 2663, 72, 4, 0, 1469, 0, 0, 5952, 1, 1108, 1, 1292, 3439, 244, 1255, 889, 4, 0, 139, 52, 6752, 10, 8, 3, 1] payload_rows_per_dst=[12278, 12024, 9342, 13880, 11202, 7446, 8076, 8360] dup_rows_per_dst=[30, 3576, 1770, 4868, 2726, 1026, 1510, 190] combine_rows_per_src=[12278, 12024, 9342, 13880, 11202, 7446, 8076, 8360] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=301146112 dispatch_meta_bytes=330976 combine_payload_bytes=301146112 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=301477088 ingress_bytes=404955136 link_bytes=404955136 token_bytes=602623200 weight_movement_bytes=103809024 total_nvlink_bytes=706432224 weight_to_token_ratio=0.172262 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=40064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523813575e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77882 payload_rows_remote=65848 dup_rows=20422 dup_rows_remote=19160 dedup_ratio=0.207743 router_tokens_per_expert=[1084, 70, 1, 7055, 2, 8, 275, 692, 11, 215, 42, 3, 0, 0, 51, 1293, 0, 0, 27, 0, 0, 21, 17, 4937, 3, 0, 2, 7, 2696, 0, 0, 0, 35, 883, 722, 93, 7098, 1, 1748, 5033, 0, 0, 7, 274, 2, 0, 8004, 9, 55, 275, 0, 0, 1, 26, 34, 1, 453, 0, 0, 2759, 4, 1791, 32, 1300] payload_rows_per_dst=[15422, 12034, 12976, 5402, 6216, 584, 14440, 10808] dup_rows_per_dst=[10956, 1262, 5032, 14, 748, 0, 540, 1870] combine_rows_per_src=[15422, 12034, 12976, 5402, 6216, 584, 14440, 10808] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=300343296 dispatch_meta_bytes=346392 combine_payload_bytes=300343296 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=300689688 ingress_bytes=404152320 link_bytes=404152320 token_bytes=601032984 weight_movement_bytes=103809024 total_nvlink_bytes=704842008 weight_to_token_ratio=0.172718 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=30848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520792643e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77012 payload_rows_remote=65812 dup_rows=21292 dup_rows_remote=18294 dedup_ratio=0.216593 router_tokens_per_expert=[0, 269, 120, 114, 130, 1963, 5, 0, 20, 5250, 6, 605, 27, 525, 67, 455, 0, 1419, 2446, 10, 1450, 505, 6, 132, 8, 4384, 10, 382, 6223, 263, 0, 0, 650, 0, 250, 0, 687, 276, 104, 135, 380, 1171, 761, 3430, 0, 12, 338, 40, 0, 7, 398, 289, 1, 0, 344, 4466, 3966, 3297, 310, 6, 2, 25, 1013, 0] payload_rows_per_dst=[11136, 3274, 11232, 1288, 13864, 11084, 13934, 11200] dup_rows_per_dst=[1998, 136, 3302, 38, 2786, 4190, 5844, 2998] combine_rows_per_src=[11136, 3274, 11232, 1288, 13864, 11084, 13934, 11200] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=311517184 dispatch_meta_bytes=339392 combine_payload_bytes=311517184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=311856576 ingress_bytes=346120192 link_bytes=346120192 token_bytes=623373760 weight_movement_bytes=34603008 total_nvlink_bytes=657976768 weight_to_token_ratio=0.055509 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=47104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.302422418e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=302022656 dispatch_meta_bytes=343640 combine_payload_bytes=302022656 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=302366296 ingress_bytes=371228672 link_bytes=371228672 token_bytes=604388952 weight_movement_bytes=69206016 total_nvlink_bytes=673594968 weight_to_token_ratio=0.114506 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=26240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396903606e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=325132288 dispatch_meta_bytes=369944 combine_payload_bytes=325132288 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=325502232 ingress_bytes=463544320 link_bytes=463544320 token_bytes=650634520 weight_movement_bytes=138412032 total_nvlink_bytes=789046552 weight_to_token_ratio=0.212734 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.744279958e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=304103424 dispatch_meta_bytes=364440 combine_payload_bytes=304103424 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=304467864 ingress_bytes=373309440 link_bytes=373309440 token_bytes=608571288 weight_movement_bytes=69206016 total_nvlink_bytes=677777304 weight_to_token_ratio=0.113719 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=28288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.404733369e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=304095232 dispatch_meta_bytes=346208 combine_payload_bytes=304095232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=304441440 ingress_bytes=338698240 link_bytes=338698240 token_bytes=608536672 weight_movement_bytes=34603008 total_nvlink_bytes=643139680 weight_to_token_ratio=0.056863 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=45696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274494210e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75890 payload_rows_remote=61440 dup_rows=22414 dup_rows_remote=17408 dedup_ratio=0.228007 router_tokens_per_expert=[5, 0, 5765, 16, 49, 0, 4, 0, 0, 0, 9, 0, 0, 2444, 2712, 162, 4790, 786, 358, 11, 749, 2385, 5, 516, 2, 0, 0, 52, 1025, 75, 1350, 69, 20, 0, 1, 6, 1083, 9, 9, 855, 339, 11, 0, 2978, 4, 0, 5652, 3, 1041, 0, 917, 4264, 72, 936, 287, 9, 1, 325, 61, 6881, 1, 31, 2, 15] payload_rows_per_dst=[5676, 9312, 13122, 4022, 5234, 13396, 10678, 14450] dup_rows_per_dst=[72, 1342, 6078, 1124, 426, 4578, 3788, 5006] combine_rows_per_src=[5676, 9312, 13122, 4022, 5234, 13396, 10678, 14450] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=333029376 dispatch_meta_bytes=357752 combine_payload_bytes=333029376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=333387128 ingress_bytes=367632384 link_bytes=367632384 token_bytes=666416504 weight_movement_bytes=34603008 total_nvlink_bytes=701019512 weight_to_token_ratio=0.051924 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18304 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.383371064e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=289112064 dispatch_meta_bytes=330816 combine_payload_bytes=289112064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289442880 ingress_bytes=323715072 link_bytes=323715072 token_bytes=578554944 weight_movement_bytes=34603008 total_nvlink_bytes=613157952 weight_to_token_ratio=0.059809 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=18432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.218113755e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84122 payload_rows_remote=69390 dup_rows=14182 dup_rows_remote=9476 dedup_ratio=0.144267 router_tokens_per_expert=[36, 0, 6767, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1936, 1293, 105, 2550, 840, 193, 6, 287, 2392, 0, 145, 16, 0, 0, 0, 630, 1, 2438, 34, 0, 0, 0, 0, 886, 17, 5, 2535, 5, 0, 0, 2824, 0, 0, 6890, 0, 2219, 0, 966, 3860, 246, 1357, 260, 0, 0, 343, 40, 7011, 7, 3, 0, 0] payload_rows_per_dst=[13540, 6062, 9864, 5448, 11678, 14732, 8632, 14166] dup_rows_per_dst=[84, 606, 2962, 790, 2928, 4706, 1464, 642] combine_rows_per_src=[13540, 6062, 9864, 5448, 11678, 14732, 8632, 14166] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=269713408 dispatch_meta_bytes=340032 combine_payload_bytes=269713408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270053440 ingress_bytes=304316416 link_bytes=304316416 token_bytes=539766848 weight_movement_bytes=34603008 total_nvlink_bytes=574369856 weight_to_token_ratio=0.064107 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=30976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.145118174e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=269565952 dispatch_meta_bytes=336424 combine_payload_bytes=269565952 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=269902376 ingress_bytes=373374976 link_bytes=373374976 token_bytes=539468328 weight_movement_bytes=103809024 total_nvlink_bytes=643277352 weight_to_token_ratio=0.192428 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=19200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.404979976e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83444 payload_rows_remote=68766 dup_rows=14860 dup_rows_remote=12482 dedup_ratio=0.151164 router_tokens_per_expert=[3, 1, 174, 1429, 2, 57, 6861, 1, 95, 72, 6, 0, 4580, 0, 816, 14, 2, 6004, 730, 0, 2, 1, 17, 135, 173, 7, 0, 1, 3, 28, 2, 17, 1497, 492, 0, 13, 0, 1, 2064, 0, 43, 102, 3092, 0, 0, 1235, 4815, 1, 14, 5515, 0, 1174, 7, 8, 6618, 5, 0, 1, 1170, 0, 24, 9, 0, 19] payload_rows_per_dst=[14678, 15640, 12850, 11258, 7364, 8296, 2414, 10944] dup_rows_per_dst=[2378, 8762, 932, 234, 770, 650, 2, 1132] combine_rows_per_src=[14678, 15640, 12850, 11258, 7364, 8296, 2414, 10944] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=251658240 dispatch_meta_bytes=315392 combine_payload_bytes=251658240 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=251973632 ingress_bytes=320864256 link_bytes=320864256 token_bytes=503631872 weight_movement_bytes=69206016 total_nvlink_bytes=572837888 weight_to_token_ratio=0.137414 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=44032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.207386364e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12594937392 prefetch_bytes=622854144 reduce_bytes=830472192 weight_movement_bytes=1453326336 egress_bytes=6301285936 ingress_bytes=7746977792 link_bytes=7746977792 total_nvlink_bytes=14048263728 weight_to_token_ratio=0.115390 prefetch_share_of_ingress=0.080400 dedup_ratio=0.204756 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=24 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=28194 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.267445815e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=284221440 dispatch_meta_bytes=315464 combine_payload_bytes=284221440 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=284536904 ingress_bytes=353427456 link_bytes=353427456 token_bytes=568758344 weight_movement_bytes=69206016 total_nvlink_bytes=637964360 weight_to_token_ratio=0.121679 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=25472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.329919064e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81012 payload_rows_remote=73798 dup_rows=17292 dup_rows_remote=16148 dedup_ratio=0.175903 router_tokens_per_expert=[23, 4, 7155, 0, 87, 0, 0, 30, 0, 1, 0, 0, 12, 2386, 1153, 230, 2911, 1250, 159, 5, 322, 2246, 0, 197, 254, 0, 0, 9, 532, 14, 2842, 19, 5, 0, 17, 50, 848, 46, 12, 377, 12, 2, 15, 1232, 6, 0, 6888, 0, 1124, 7, 2191, 1936, 2363, 3456, 491, 59, 16, 292, 195, 5643, 3, 7, 3, 15] payload_rows_per_dst=[14368, 10540, 7214, 14960, 6092, 2518, 13538, 11782] dup_rows_per_dst=[230, 2846, 1144, 6156, 490, 16, 5844, 566] combine_rows_per_src=[14368, 10540, 7214, 14960, 6092, 2518, 13538, 11782] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85824 payload_rows_remote=72376 dup_rows=12480 dup_rows_remote=9856 dedup_ratio=0.126953 router_tokens_per_expert=[2118, 357, 17, 5723, 0, 33, 199, 291, 8, 141, 52, 18, 0, 0, 34, 1535, 0, 3, 45, 0, 0, 240, 44, 4261, 7, 0, 2, 0, 2304, 0, 0, 0, 57, 1847, 1440, 71, 5198, 0, 2338, 5914, 0, 0, 12, 902, 9, 0, 8028, 0, 67, 151, 0, 0, 0, 89, 10, 0, 488, 5, 0, 2515, 2, 1136, 42, 1399] payload_rows_per_dst=[5656, 13290, 9968, 13448, 8662, 14442, 10828, 9530] dup_rows_per_dst=[374, 2114, 1164, 2624, 2094, 2264, 202, 1644] combine_rows_per_src=[5656, 13290, 9968, 13448, 8662, 14442, 10828, 9530] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=281665536 dispatch_meta_bytes=324992 combine_payload_bytes=281665536 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=281990528 ingress_bytes=350871552 link_bytes=350871552 token_bytes=563656064 weight_movement_bytes=69206016 total_nvlink_bytes=632862080 weight_to_token_ratio=0.122781 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=36096 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.320301403e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=11972773824 prefetch_bytes=830472192 reduce_bytes=311427072 weight_movement_bytes=1141899264 egress_bytes=5990131648 ingress_bytes=7124541440 link_bytes=7124541440 total_nvlink_bytes=13114673088 weight_to_token_ratio=0.095375 prefetch_share_of_ingress=0.116565 dedup_ratio=0.230778 prefetch_slots=48 unused_prefetch_slots=320 slot_utilization=0.130435 reduce_slots=9 remote_demand=24 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027500 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.165611994e-04 prefetch_bytes_per_gemm_flop=1.358695652e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85156 payload_rows_remote=70822 dup_rows=13148 dup_rows_remote=11070 dedup_ratio=0.133748 router_tokens_per_expert=[120, 0, 6912, 0, 9, 0, 4, 2, 0, 3, 0, 0, 2, 2773, 1340, 8, 2849, 978, 84, 20, 235, 2225, 0, 355, 36, 0, 0, 6, 26, 26, 1047, 137, 3, 0, 0, 0, 2298, 14, 3, 846, 5, 26, 0, 641, 0, 0, 6928, 3, 2096, 8, 2013, 4838, 920, 1669, 184, 0, 5, 330, 49, 7061, 4, 7, 0, 4] payload_rows_per_dst=[13848, 11258, 6796, 14334, 12290, 1350, 11006, 14274] dup_rows_per_dst=[246, 2692, 998, 2078, 3714, 0, 2774, 646] combine_rows_per_src=[13848, 11258, 6796, 14334, 12290, 1350, 11006, 14274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88878 payload_rows_remote=81306 dup_rows=9426 dup_rows_remote=8132 dedup_ratio=0.095886 router_tokens_per_expert=[0, 38, 30, 1799, 0, 19, 5178, 8, 61, 73, 3, 0, 2164, 0, 1099, 3, 13, 6872, 505, 0, 0, 0, 18, 60, 40, 8, 0, 0, 0, 39, 11, 17, 2096, 705, 14, 13, 1, 12, 1589, 3, 34, 46, 5243, 0, 0, 1988, 4913, 3, 29, 6288, 1, 1101, 11, 0, 6002, 26, 0, 4, 962, 0, 9, 1, 0, 0] payload_rows_per_dst=[12250, 6396, 14164, 12646, 7572, 12330, 12690, 10830] dup_rows_per_dst=[1894, 410, 772, 160, 1294, 2298, 1650, 948] combine_rows_per_src=[12250, 6396, 14164, 12646, 7572, 12330, 12690, 10830] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85686 payload_rows_remote=69524 dup_rows=12618 dup_rows_remote=8324 dedup_ratio=0.128357 router_tokens_per_expert=[2294, 150, 0, 6113, 0, 4, 177, 284, 0, 45, 110, 7, 0, 0, 162, 2318, 0, 1, 6, 0, 0, 24, 210, 4699, 3, 0, 1, 0, 2325, 0, 0, 0, 20, 1748, 449, 58, 5972, 0, 1481, 4824, 0, 0, 7, 745, 18, 0, 7977, 0, 50, 86, 0, 0, 0, 34, 0, 0, 268, 0, 0, 3228, 14, 1617, 37, 1586] payload_rows_per_dst=[5636, 8130, 9752, 14204, 8348, 16162, 12162, 11292] dup_rows_per_dst=[182, 1168, 128, 2680, 1836, 4294, 122, 2208] combine_rows_per_src=[5636, 8130, 9752, 14204, 8348, 16162, 12162, 11292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80262 payload_rows_remote=68932 dup_rows=18042 dup_rows_remote=14258 dedup_ratio=0.183533 router_tokens_per_expert=[1, 204, 138, 232, 46, 945, 21, 2, 46, 5214, 0, 476, 62, 1203, 14, 620, 3, 920, 2888, 23, 3993, 2020, 10, 14, 1, 4790, 9, 146, 5071, 455, 0, 2, 288, 13, 8, 36, 476, 76, 41, 74, 286, 358, 1875, 5746, 2, 7, 2086, 9, 1, 3, 460, 302, 3, 0, 307, 1682, 1631, 810, 47, 2, 0, 4, 2903, 47] payload_rows_per_dst=[5706, 12126, 11330, 10390, 11190, 13802, 5214, 10504] dup_rows_per_dst=[734, 3144, 3784, 416, 976, 6936, 302, 1750] combine_rows_per_src=[5706, 12126, 11330, 10390, 11190, 13802, 5214, 10504] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80288 payload_rows_remote=66362 dup_rows=18016 dup_rows_remote=14198 dedup_ratio=0.183268 router_tokens_per_expert=[0, 0, 183, 828, 7, 13, 7645, 5, 114, 67, 0, 0, 1658, 0, 1576, 3, 58, 7135, 938, 0, 0, 0, 0, 104, 35, 0, 0, 0, 0, 58, 180, 159, 1386, 515, 0, 0, 1, 75, 806, 0, 48, 8, 2605, 0, 0, 2172, 3411, 2, 18, 7049, 1, 2519, 41, 0, 6280, 13, 0, 3, 1419, 3, 4, 4, 3, 0] payload_rows_per_dst=[2024, 6474, 2174, 14326, 15802, 14598, 13926, 10964] dup_rows_per_dst=[48, 362, 26, 636, 5054, 4132, 3818, 3940] combine_rows_per_src=[2024, 6474, 2174, 14326, 15802, 14598, 13926, 10964] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12860363600 prefetch_bytes=761266176 reduce_bytes=865075200 weight_movement_bytes=1626341376 egress_bytes=6434067280 ingress_bytes=8052637696 link_bytes=8052637696 total_nvlink_bytes=14486704976 weight_to_token_ratio=0.126462 prefetch_share_of_ingress=0.094536 dedup_ratio=0.195833 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=25 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=28194 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.317453363e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84826 payload_rows_remote=74368 dup_rows=13478 dup_rows_remote=11670 dedup_ratio=0.137105 router_tokens_per_expert=[156, 0, 6711, 0, 7, 0, 0, 11, 0, 9, 93, 0, 5, 2854, 2709, 238, 2045, 567, 497, 86, 305, 3284, 0, 922, 432, 0, 0, 23, 305, 1, 2077, 134, 0, 0, 0, 27, 1262, 35, 4, 606, 164, 17, 2, 940, 76, 0, 7043, 0, 1639, 0, 908, 4199, 337, 1022, 546, 6, 3, 33, 90, 6618, 59, 42, 3, 0] payload_rows_per_dst=[13502, 11478, 9704, 5706, 10458, 12998, 7618, 13362] dup_rows_per_dst=[268, 3806, 2240, 1814, 1808, 1910, 1298, 334] combine_rows_per_src=[13502, 11478, 9704, 5706, 10458, 12998, 7618, 13362] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=302276608 dispatch_meta_bytes=359784 combine_payload_bytes=302276608 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=302636392 ingress_bytes=371482624 link_bytes=371482624 token_bytes=604913000 weight_movement_bytes=69206016 total_nvlink_bytes=674119016 weight_to_token_ratio=0.114407 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20480 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397859207e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84550 payload_rows_remote=69778 dup_rows=13754 dup_rows_remote=5730 dedup_ratio=0.139913 router_tokens_per_expert=[0, 54, 69, 1017, 6, 42, 7509, 3, 592, 452, 2, 0, 3907, 0, 1507, 2, 10, 7496, 860, 0, 0, 1, 1, 169, 34, 0, 0, 0, 0, 10, 14, 0, 1310, 637, 3, 18, 0, 55, 1077, 0, 12, 6, 2585, 0, 1, 1013, 4339, 0, 14, 6538, 1, 1250, 24, 1, 5482, 3, 0, 15, 1008, 0, 0, 1, 0, 2] payload_rows_per_dst=[15362, 14772, 15286, 13088, 5576, 6916, 3454, 10096] dup_rows_per_dst=[2038, 8024, 1788, 104, 624, 318, 224, 634] combine_rows_per_src=[15362, 14772, 15286, 13088, 5576, 6916, 3454, 10096] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=296452096 dispatch_meta_bytes=328928 combine_payload_bytes=296452096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296781024 ingress_bytes=331055104 link_bytes=331055104 token_bytes=593233120 weight_movement_bytes=34603008 total_nvlink_bytes=627836128 weight_to_token_ratio=0.058330 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=29952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.245733704e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85246 payload_rows_remote=73470 dup_rows=13058 dup_rows_remote=9776 dedup_ratio=0.132833 router_tokens_per_expert=[715, 68, 5, 5168, 0, 12, 442, 441, 5, 326, 52, 56, 2, 0, 61, 3326, 1, 1, 57, 2, 0, 10, 10, 5092, 5, 0, 24, 0, 1801, 2, 1, 0, 29, 1852, 105, 884, 6234, 5, 1551, 4878, 0, 0, 0, 487, 3, 0, 7529, 6, 59, 222, 25, 1, 0, 60, 8, 0, 411, 5, 0, 3060, 11, 3072, 100, 870] payload_rows_per_dst=[3206, 7252, 10290, 12374, 12104, 15382, 12862, 11776] dup_rows_per_dst=[160, 404, 56, 1628, 3402, 3770, 356, 3282] combine_rows_per_src=[3206, 7252, 10290, 12374, 12104, 15382, 12862, 11776] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=290086912 dispatch_meta_bytes=327568 combine_payload_bytes=290086912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=290414480 ingress_bytes=324689920 link_bytes=324689920 token_bytes=580501392 weight_movement_bytes=34603008 total_nvlink_bytes=615104400 weight_to_token_ratio=0.059609 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=27264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.221782029e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=333029376 dispatch_meta_bytes=357752 combine_payload_bytes=333029376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=333387128 ingress_bytes=367632384 link_bytes=367632384 token_bytes=666416504 weight_movement_bytes=34603008 total_nvlink_bytes=701019512 weight_to_token_ratio=0.051924 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18304 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.383371064e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=282345472 dispatch_meta_bytes=332760 combine_payload_bytes=282345472 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=282678232 ingress_bytes=351551488 link_bytes=351551488 token_bytes=565023704 weight_movement_bytes=69206016 total_nvlink_bytes=634229720 weight_to_token_ratio=0.122483 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=19968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.322859947e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=284770304 dispatch_meta_bytes=311392 combine_payload_bytes=284770304 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=285081696 ingress_bytes=388579328 link_bytes=388579328 token_bytes=569852000 weight_movement_bytes=103809024 total_nvlink_bytes=673661024 weight_to_token_ratio=0.182168 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=38144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.462192728e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=271818752 dispatch_meta_bytes=322240 combine_payload_bytes=271818752 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=272140992 ingress_bytes=375627776 link_bytes=375627776 token_bytes=543959744 weight_movement_bytes=103809024 total_nvlink_bytes=647768768 weight_to_token_ratio=0.190840 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=29312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.413457081e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=304611328 dispatch_meta_bytes=344152 combine_payload_bytes=304611328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=304955480 ingress_bytes=339214336 link_bytes=339214336 token_bytes=609566808 weight_movement_bytes=34603008 total_nvlink_bytes=644169816 weight_to_token_ratio=0.056767 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=28928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.276436237e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12981233208 prefetch_bytes=795869184 reduce_bytes=692060160 weight_movement_bytes=1487929344 egress_bytes=6494537272 ingress_bytes=7974625280 link_bytes=7974625280 total_nvlink_bytes=14469162552 weight_to_token_ratio=0.114622 prefetch_share_of_ingress=0.099800 dedup_ratio=0.196294 prefetch_slots=46 unused_prefetch_slots=322 slot_utilization=0.125000 reduce_slots=20 remote_demand=23 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027935 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.304690127e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13026883584 prefetch_bytes=622854144 reduce_bytes=1211105280 weight_movement_bytes=1833959424 egress_bytes=6517340160 ingress_bytes=8343502848 link_bytes=8343502848 total_nvlink_bytes=14860843008 weight_to_token_ratio=0.140783 prefetch_share_of_ingress=0.074651 dedup_ratio=0.185480 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=35 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28067 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.365040413e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=285810688 dispatch_meta_bytes=302032 combine_payload_bytes=285810688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286112720 ingress_bytes=320413696 link_bytes=320413696 token_bytes=571923408 weight_movement_bytes=34603008 total_nvlink_bytes=606526416 weight_to_token_ratio=0.060503 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=23808 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.205690943e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=300933120 dispatch_meta_bytes=332984 combine_payload_bytes=300933120 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=301266104 ingress_bytes=335536128 link_bytes=335536128 token_bytes=602199224 weight_movement_bytes=34603008 total_nvlink_bytes=636802232 weight_to_token_ratio=0.057461 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=18688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262595437e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82022 payload_rows_remote=70466 dup_rows=16282 dup_rows_remote=15994 dedup_ratio=0.165629 router_tokens_per_expert=[189, 0, 5723, 0, 5, 0, 0, 5, 1, 2, 3, 0, 101, 1137, 2809, 20, 3707, 345, 396, 0, 184, 1391, 3, 1283, 7, 0, 0, 0, 1023, 5, 1850, 37, 3, 0, 1, 21, 1246, 11, 0, 997, 38, 2, 0, 774, 7, 0, 6628, 0, 4020, 1, 797, 5402, 152, 943, 648, 4, 3, 51, 179, 6979, 3, 13, 3, 0] payload_rows_per_dst=[11556, 12340, 8736, 14170, 12440, 14198, 8086, 496] dup_rows_per_dst=[288, 3220, 2056, 4930, 2922, 1402, 1456, 8] combine_rows_per_src=[11556, 12340, 8736, 14170, 12440, 14198, 8086, 496] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13077949840 prefetch_bytes=830472192 reduce_bytes=449839104 weight_movement_bytes=1280311296 egress_bytes=6542904720 ingress_bytes=7815356416 link_bytes=7815356416 total_nvlink_bytes=14358261136 weight_to_token_ratio=0.097898 prefetch_share_of_ingress=0.106262 dedup_ratio=0.189238 prefetch_slots=48 unused_prefetch_slots=320 slot_utilization=0.130435 reduce_slots=13 remote_demand=24 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.278632913e-04 prefetch_bytes_per_gemm_flop=1.358695652e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89228 payload_rows_remote=76054 dup_rows=9076 dup_rows_remote=8794 dedup_ratio=0.092326 router_tokens_per_expert=[0, 1, 170, 941, 2, 11, 6649, 6, 205, 152, 0, 0, 2923, 0, 1508, 3, 4, 7242, 1011, 0, 0, 4, 1, 248, 181, 1, 0, 1, 0, 22, 5, 14, 1638, 1303, 3, 0, 0, 13, 2358, 1, 71, 4, 3539, 0, 0, 1117, 4049, 2, 100, 6504, 1, 740, 45, 1, 5272, 8, 0, 3, 1061, 2, 4, 0, 0, 8] payload_rows_per_dst=[13936, 8726, 14958, 13174, 8912, 8864, 11266, 9392] dup_rows_per_dst=[1624, 856, 2062, 282, 1720, 602, 1068, 862] combine_rows_per_src=[13936, 8726, 14958, 13174, 8912, 8864, 11266, 9392] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=288628736 dispatch_meta_bytes=345840 combine_payload_bytes=288628736 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=288974576 ingress_bytes=357834752 link_bytes=357834752 token_bytes=577603312 weight_movement_bytes=69206016 total_nvlink_bytes=646809328 weight_to_token_ratio=0.119816 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=48384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.346503364e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84826 payload_rows_remote=74368 dup_rows=13478 dup_rows_remote=11670 dedup_ratio=0.137105 router_tokens_per_expert=[156, 0, 6711, 0, 7, 0, 0, 11, 0, 9, 93, 0, 5, 2854, 2709, 238, 2045, 567, 497, 86, 305, 3284, 0, 922, 432, 0, 0, 23, 305, 1, 2077, 134, 0, 0, 0, 27, 1262, 35, 4, 606, 164, 17, 2, 940, 76, 0, 7043, 0, 1639, 0, 908, 4199, 337, 1022, 546, 6, 3, 33, 90, 6618, 59, 42, 3, 0] payload_rows_per_dst=[13502, 11478, 9704, 5706, 10458, 12998, 7618, 13362] dup_rows_per_dst=[268, 3806, 2240, 1814, 1808, 1910, 1298, 334] combine_rows_per_src=[13502, 11478, 9704, 5706, 10458, 12998, 7618, 13362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81912 payload_rows_remote=65968 dup_rows=16392 dup_rows_remote=10702 dedup_ratio=0.166748 router_tokens_per_expert=[925, 293, 3, 5757, 51, 21, 861, 466, 3, 499, 18, 92, 38, 0, 16, 1328, 2, 13, 48, 1, 18, 27, 40, 2782, 1, 3, 301, 0, 3235, 1, 3, 6, 30, 2086, 276, 25, 6598, 5, 3148, 6123, 1, 9, 0, 150, 72, 53, 7886, 0, 48, 722, 0, 1, 0, 236, 0, 20, 491, 10, 5, 1418, 5, 1847, 92, 943] payload_rows_per_dst=[12106, 14020, 15944, 8042, 8914, 550, 14024, 8312] dup_rows_per_dst=[3042, 2214, 5690, 664, 2226, 20, 1226, 1310] combine_rows_per_src=[12106, 14020, 15944, 8042, 8914, 550, 14024, 8312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84374 payload_rows_remote=74606 dup_rows=13930 dup_rows_remote=11932 dedup_ratio=0.141703 router_tokens_per_expert=[0, 8, 218, 1232, 4, 6, 6649, 1, 258, 146, 0, 0, 3671, 0, 1595, 1, 7, 7389, 1386, 0, 0, 1, 1, 41, 94, 0, 0, 1, 0, 20, 176, 10, 983, 643, 0, 3, 0, 34, 1018, 0, 25, 21, 3783, 0, 0, 2345, 2983, 0, 8, 7068, 0, 1812, 628, 0, 4014, 6, 0, 5, 846, 0, 10, 0, 0, 2] payload_rows_per_dst=[8292, 10282, 13048, 14410, 9142, 9768, 10584, 8848] dup_rows_per_dst=[1038, 1060, 2594, 328, 2492, 1998, 2352, 2068] combine_rows_per_src=[8292, 10282, 13048, 14410, 9142, 9768, 10584, 8848] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=311517184 dispatch_meta_bytes=339392 combine_payload_bytes=311517184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=311856576 ingress_bytes=346120192 link_bytes=346120192 token_bytes=623373760 weight_movement_bytes=34603008 total_nvlink_bytes=657976768 weight_to_token_ratio=0.055509 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=47104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.302422418e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83410 payload_rows_remote=73066 dup_rows=14894 dup_rows_remote=12702 dedup_ratio=0.151510 router_tokens_per_expert=[49, 0, 6103, 0, 3, 0, 4, 0, 1, 4, 2, 0, 0, 2544, 971, 79, 3243, 535, 57, 3, 220, 2795, 0, 1286, 0, 0, 0, 0, 177, 4, 1767, 64, 2, 0, 3, 9, 856, 40, 7, 1689, 384, 2, 0, 2059, 1, 0, 6567, 1, 1724, 0, 1965, 4342, 122, 1793, 664, 0, 0, 235, 62, 6664, 2, 47, 1, 0] payload_rows_per_dst=[12220, 6752, 11572, 3848, 11252, 13894, 10344, 13528] dup_rows_per_dst=[98, 450, 4706, 176, 2644, 4134, 2192, 494] combine_rows_per_src=[12220, 6752, 11572, 3848, 11252, 13894, 10344, 13528] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82608 payload_rows_remote=70584 dup_rows=15696 dup_rows_remote=12120 dedup_ratio=0.159668 router_tokens_per_expert=[6, 0, 6135, 0, 13, 0, 0, 0, 0, 25, 0, 0, 4, 2285, 1098, 137, 4251, 459, 300, 10, 278, 3733, 0, 776, 45, 0, 2, 23, 964, 4, 2348, 36, 0, 1, 23, 0, 806, 28, 4, 2663, 72, 4, 0, 1469, 0, 0, 5952, 1, 1108, 1, 1292, 3439, 244, 1255, 889, 4, 0, 139, 52, 6752, 10, 8, 3, 1] payload_rows_per_dst=[12278, 12024, 9342, 13880, 11202, 7446, 8076, 8360] dup_rows_per_dst=[30, 3576, 1770, 4868, 2726, 1026, 1510, 190] combine_rows_per_src=[12278, 12024, 9342, 13880, 11202, 7446, 8076, 8360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12131284608 prefetch_bytes=692060160 reduce_bytes=726663168 weight_movement_bytes=1418723328 egress_bytes=6069376640 ingress_bytes=7480631296 link_bytes=7480631296 total_nvlink_bytes=13550007936 weight_to_token_ratio=0.116947 prefetch_share_of_ingress=0.092514 dedup_ratio=0.219138 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=21 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.223870145e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73674 payload_rows_remote=62536 dup_rows=24630 dup_rows_remote=20736 dedup_ratio=0.250549 router_tokens_per_expert=[0, 78, 114, 1930, 12, 115, 7473, 42, 247, 175, 1, 11, 3440, 0, 750, 49, 120, 7549, 757, 4, 0, 1, 11, 152, 356, 2, 0, 0, 3, 79, 16, 23, 939, 1781, 5, 3, 1, 36, 1181, 72, 41, 17, 2855, 0, 0, 1231, 3395, 0, 56, 6690, 0, 1857, 49, 6, 4145, 16, 1, 3, 1229, 1, 6, 1, 0, 25] payload_rows_per_dst=[4426, 8692, 2078, 3208, 15520, 13690, 14922, 11138] dup_rows_per_dst=[156, 654, 12, 212, 7462, 2170, 10070, 3894] combine_rows_per_src=[4426, 8692, 2078, 3208, 15520, 13690, 14922, 11138] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=270204928 dispatch_meta_bytes=306680 combine_payload_bytes=270204928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270511608 ingress_bytes=304807936 link_bytes=304807936 token_bytes=540716536 weight_movement_bytes=34603008 total_nvlink_bytes=575319544 weight_to_token_ratio=0.063995 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=34816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.146967724e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=304611328 dispatch_meta_bytes=344152 combine_payload_bytes=304611328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=304955480 ingress_bytes=339214336 link_bytes=339214336 token_bytes=609566808 weight_movement_bytes=34603008 total_nvlink_bytes=644169816 weight_to_token_ratio=0.056767 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=28928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.276436237e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=305586176 dispatch_meta_bytes=346152 combine_payload_bytes=305586176 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=305932328 ingress_bytes=409395200 link_bytes=409395200 token_bytes=611518504 weight_movement_bytes=103809024 total_nvlink_bytes=715327528 weight_to_token_ratio=0.169756 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=21760 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.540521179e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=299278336 dispatch_meta_bytes=343072 combine_payload_bytes=299278336 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=299621408 ingress_bytes=403087360 link_bytes=403087360 token_bytes=598899744 weight_movement_bytes=103809024 total_nvlink_bytes=702708768 weight_to_token_ratio=0.173333 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.516785284e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=289112064 dispatch_meta_bytes=330816 combine_payload_bytes=289112064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289442880 ingress_bytes=323715072 link_bytes=323715072 token_bytes=578554944 weight_movement_bytes=34603008 total_nvlink_bytes=613157952 weight_to_token_ratio=0.059809 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=18432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.218113755e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=256147456 dispatch_meta_bytes=333088 combine_payload_bytes=256147456 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=256480544 ingress_bytes=325353472 link_bytes=325353472 token_bytes=512628000 weight_movement_bytes=69206016 total_nvlink_bytes=581834016 weight_to_token_ratio=0.135002 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.022500 gemm_active_groups=11 gemm_peak_group_rows=32512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.224278922e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13077949840 prefetch_bytes=830472192 reduce_bytes=449839104 weight_movement_bytes=1280311296 egress_bytes=6542904720 ingress_bytes=7815356416 link_bytes=7815356416 total_nvlink_bytes=14358261136 weight_to_token_ratio=0.097898 prefetch_share_of_ingress=0.106262 dedup_ratio=0.189238 prefetch_slots=48 unused_prefetch_slots=320 slot_utilization=0.130435 reduce_slots=13 remote_demand=24 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.278632913e-04 prefetch_bytes_per_gemm_flop=1.358695652e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85156 payload_rows_remote=70822 dup_rows=13148 dup_rows_remote=11070 dedup_ratio=0.133748 router_tokens_per_expert=[120, 0, 6912, 0, 9, 0, 4, 2, 0, 3, 0, 0, 2, 2773, 1340, 8, 2849, 978, 84, 20, 235, 2225, 0, 355, 36, 0, 0, 6, 26, 26, 1047, 137, 3, 0, 0, 0, 2298, 14, 3, 846, 5, 26, 0, 641, 0, 0, 6928, 3, 2096, 8, 2013, 4838, 920, 1669, 184, 0, 5, 330, 49, 7061, 4, 7, 0, 4] payload_rows_per_dst=[13848, 11258, 6796, 14334, 12290, 1350, 11006, 14274] dup_rows_per_dst=[246, 2692, 998, 2078, 3714, 0, 2774, 646] combine_rows_per_src=[13848, 11258, 6796, 14334, 12290, 1350, 11006, 14274] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12415861280 prefetch_bytes=761266176 reduce_bytes=830472192 weight_movement_bytes=1591738368 egress_bytes=6211732000 ingress_bytes=7795867648 link_bytes=7795867648 total_nvlink_bytes=14007599648 weight_to_token_ratio=0.128202 prefetch_share_of_ingress=0.097650 dedup_ratio=0.211088 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=24 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027500 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.275444450e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none - [2026-08-26 15:09:08] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 793.9 | throughput per GPU (TFLOP/s/GPU): 160.7 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.956084E+00 | loss scale: 1.0 | grad norm: 42.533 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12594937392 prefetch_bytes=622854144 reduce_bytes=830472192 weight_movement_bytes=1453326336 egress_bytes=6301285936 ingress_bytes=7746977792 link_bytes=7746977792 total_nvlink_bytes=14048263728 weight_to_token_ratio=0.115390 prefetch_share_of_ingress=0.080400 dedup_ratio=0.204756 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=24 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=28194 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.267445815e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87112 payload_rows_remote=73522 dup_rows=11192 dup_rows_remote=9222 dedup_ratio=0.113851 router_tokens_per_expert=[56, 91, 66, 2212, 8, 38, 6597, 12, 799, 286, 26, 0, 1359, 0, 951, 176, 0, 6381, 1066, 8, 0, 7, 45, 273, 112, 35, 0, 32, 0, 25, 34, 34, 1663, 713, 1, 4, 36, 30, 2053, 0, 61, 52, 5013, 0, 1, 701, 4513, 9, 73, 6847, 0, 1467, 293, 18, 2939, 142, 7, 7, 1661, 3, 2, 75, 3, 36] payload_rows_per_dst=[14580, 6702, 13590, 13806, 7648, 11034, 8576, 11176] dup_rows_per_dst=[3580, 492, 1970, 432, 1352, 640, 1288, 1438] combine_rows_per_src=[14580, 6702, 13590, 13806, 7648, 11034, 8576, 11176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=290086912 dispatch_meta_bytes=327568 combine_payload_bytes=290086912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=290414480 ingress_bytes=324689920 link_bytes=324689920 token_bytes=580501392 weight_movement_bytes=34603008 total_nvlink_bytes=615104400 weight_to_token_ratio=0.059609 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=27264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.221782029e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84122 payload_rows_remote=69390 dup_rows=14182 dup_rows_remote=9476 dedup_ratio=0.144267 router_tokens_per_expert=[36, 0, 6767, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1936, 1293, 105, 2550, 840, 193, 6, 287, 2392, 0, 145, 16, 0, 0, 0, 630, 1, 2438, 34, 0, 0, 0, 0, 886, 17, 5, 2535, 5, 0, 0, 2824, 0, 0, 6890, 0, 2219, 0, 966, 3860, 246, 1357, 260, 0, 0, 343, 40, 7011, 7, 3, 0, 0] payload_rows_per_dst=[13540, 6062, 9864, 5448, 11678, 14732, 8632, 14166] dup_rows_per_dst=[84, 606, 2962, 790, 2928, 4706, 1464, 642] combine_rows_per_src=[13540, 6062, 9864, 5448, 11678, 14732, 8632, 14166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75890 payload_rows_remote=61440 dup_rows=22414 dup_rows_remote=17408 dedup_ratio=0.228007 router_tokens_per_expert=[5, 0, 5765, 16, 49, 0, 4, 0, 0, 0, 9, 0, 0, 2444, 2712, 162, 4790, 786, 358, 11, 749, 2385, 5, 516, 2, 0, 0, 52, 1025, 75, 1350, 69, 20, 0, 1, 6, 1083, 9, 9, 855, 339, 11, 0, 2978, 4, 0, 5652, 3, 1041, 0, 917, 4264, 72, 936, 287, 9, 1, 325, 61, 6881, 1, 31, 2, 15] payload_rows_per_dst=[5676, 9312, 13122, 4022, 5234, 13396, 10678, 14450] dup_rows_per_dst=[72, 1342, 6078, 1124, 426, 4578, 3788, 5006] combine_rows_per_src=[5676, 9312, 13122, 4022, 5234, 13396, 10678, 14450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=301146112 dispatch_meta_bytes=330976 combine_payload_bytes=301146112 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=301477088 ingress_bytes=404955136 link_bytes=404955136 token_bytes=602623200 weight_movement_bytes=103809024 total_nvlink_bytes=706432224 weight_to_token_ratio=0.172262 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=40064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523813575e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13026883584 prefetch_bytes=622854144 reduce_bytes=1211105280 weight_movement_bytes=1833959424 egress_bytes=6517340160 ingress_bytes=8343502848 link_bytes=8343502848 total_nvlink_bytes=14860843008 weight_to_token_ratio=0.140783 prefetch_share_of_ingress=0.074651 dedup_ratio=0.185480 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=35 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28067 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.365040413e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=284221440 dispatch_meta_bytes=315464 combine_payload_bytes=284221440 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=284536904 ingress_bytes=353427456 link_bytes=353427456 token_bytes=568758344 weight_movement_bytes=69206016 total_nvlink_bytes=637964360 weight_to_token_ratio=0.121679 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=25472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.329919064e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=251658240 dispatch_meta_bytes=315392 combine_payload_bytes=251658240 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=251973632 ingress_bytes=320864256 link_bytes=320864256 token_bytes=503631872 weight_movement_bytes=69206016 total_nvlink_bytes=572837888 weight_to_token_ratio=0.137414 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=44032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.207386364e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12860363600 prefetch_bytes=761266176 reduce_bytes=865075200 weight_movement_bytes=1626341376 egress_bytes=6434067280 ingress_bytes=8052637696 link_bytes=8052637696 total_nvlink_bytes=14486704976 weight_to_token_ratio=0.126462 prefetch_share_of_ingress=0.094536 dedup_ratio=0.195833 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=25 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=28194 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.317453363e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=11972773824 prefetch_bytes=830472192 reduce_bytes=311427072 weight_movement_bytes=1141899264 egress_bytes=5990131648 ingress_bytes=7124541440 link_bytes=7124541440 total_nvlink_bytes=13114673088 weight_to_token_ratio=0.095375 prefetch_share_of_ingress=0.116565 dedup_ratio=0.230778 prefetch_slots=48 unused_prefetch_slots=320 slot_utilization=0.130435 reduce_slots=9 remote_demand=24 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027500 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.165611994e-04 prefetch_bytes_per_gemm_flop=1.358695652e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81012 payload_rows_remote=73798 dup_rows=17292 dup_rows_remote=16148 dedup_ratio=0.175903 router_tokens_per_expert=[23, 4, 7155, 0, 87, 0, 0, 30, 0, 1, 0, 0, 12, 2386, 1153, 230, 2911, 1250, 159, 5, 322, 2246, 0, 197, 254, 0, 0, 9, 532, 14, 2842, 19, 5, 0, 17, 50, 848, 46, 12, 377, 12, 2, 15, 1232, 6, 0, 6888, 0, 1124, 7, 2191, 1936, 2363, 3456, 491, 59, 16, 292, 195, 5643, 3, 7, 3, 15] payload_rows_per_dst=[14368, 10540, 7214, 14960, 6092, 2518, 13538, 11782] dup_rows_per_dst=[230, 2846, 1144, 6156, 490, 16, 5844, 566] combine_rows_per_src=[14368, 10540, 7214, 14960, 6092, 2518, 13538, 11782] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=302276608 dispatch_meta_bytes=359784 combine_payload_bytes=302276608 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=302636392 ingress_bytes=371482624 link_bytes=371482624 token_bytes=604913000 weight_movement_bytes=69206016 total_nvlink_bytes=674119016 weight_to_token_ratio=0.114407 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20480 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397859207e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12981233208 prefetch_bytes=795869184 reduce_bytes=692060160 weight_movement_bytes=1487929344 egress_bytes=6494537272 ingress_bytes=7974625280 link_bytes=7974625280 total_nvlink_bytes=14469162552 weight_to_token_ratio=0.114622 prefetch_share_of_ingress=0.099800 dedup_ratio=0.196294 prefetch_slots=46 unused_prefetch_slots=322 slot_utilization=0.125000 reduce_slots=20 remote_demand=23 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027935 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.304690127e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 anomalies=none + [2026-08-26 15:56:56] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 795.9 | throughput per GPU (TFLOP/s/GPU): 160.3 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.956084E+00 | loss scale: 1.0 | grad norm: 42.533 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74634 payload_rows_remote=62728 dup_rows=23670 dup_rows_remote=17654 dedup_ratio=0.240784 router_tokens_per_expert=[2641, 653, 429, 788, 389, 1311, 978, 84, 2484, 683, 1298, 474, 20, 1176, 303, 582, 1074, 1205, 514, 834, 258, 358, 490, 579, 756, 837, 141, 397, 947, 765, 427, 1047, 1260, 324, 1194, 1429, 546, 370, 889, 414, 181, 381, 406, 230, 1718, 1842, 729, 234, 311, 610, 643, 958, 1031, 1411, 623, 292, 414, 702, 1497, 544, 412, 487, 1489, 659] payload_rows_per_dst=[11906, 9162, 8242, 8100, 9248, 9208, 9354, 9414] dup_rows_per_dst=[6016, 2162, 2382, 2534, 3604, 2234, 2404, 2334] combine_rows_per_src=[11906, 9162, 8242, 8100, 9248, 9208, 9354, 9414] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74908 payload_rows_remote=64270 dup_rows=23396 dup_rows_remote=19560 dedup_ratio=0.237996 router_tokens_per_expert=[2756, 594, 463, 601, 423, 1455, 970, 83, 2803, 692, 1254, 462, 38, 1144, 240, 604, 888, 1175, 432, 901, 318, 352, 512, 634, 801, 824, 147, 419, 914, 802, 391, 977, 1157, 347, 976, 1810, 569, 408, 995, 321, 148, 322, 398, 197, 1716, 1983, 784, 171, 284, 645, 604, 792, 907, 1544, 705, 212, 440, 781, 1544, 520, 420, 464, 1455, 464] payload_rows_per_dst=[10968, 10638, 8148, 7986, 9342, 9258, 8922, 9646] dup_rows_per_dst=[3722, 3836, 2276, 2564, 3824, 2180, 2464, 2530] combine_rows_per_src=[10968, 10638, 8148, 7986, 9342, 9258, 8922, 9646] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75138 payload_rows_remote=67086 dup_rows=23166 dup_rows_remote=20772 dedup_ratio=0.235657 router_tokens_per_expert=[2779, 671, 380, 712, 527, 1204, 1112, 100, 2432, 601, 1220, 530, 28, 1143, 260, 672, 1051, 1088, 502, 898, 302, 335, 496, 551, 907, 811, 130, 440, 871, 859, 420, 1017, 1143, 351, 1053, 1571, 630, 385, 978, 442, 202, 360, 373, 225, 1648, 1880, 723, 179, 349, 562, 666, 756, 1013, 1267, 606, 216, 365, 913, 1522, 503, 495, 547, 1484, 696] payload_rows_per_dst=[11180, 10366, 8052, 8118, 9402, 8946, 8910, 10164] dup_rows_per_dst=[3790, 3406, 2394, 2792, 3704, 2234, 1960, 2886] combine_rows_per_src=[11180, 10366, 8052, 8118, 9402, 8946, 8910, 10164] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75274 payload_rows_remote=65854 dup_rows=23030 dup_rows_remote=20502 dedup_ratio=0.234273 router_tokens_per_expert=[2760, 669, 420, 727, 379, 1296, 974, 98, 2583, 707, 1317, 494, 18, 1144, 275, 581, 988, 1206, 480, 774, 291, 356, 522, 586, 798, 779, 166, 353, 944, 821, 445, 1095, 1306, 340, 1049, 1826, 610, 350, 894, 368, 168, 344, 410, 274, 1652, 1954, 785, 165, 301, 583, 613, 812, 1032, 1323, 592, 257, 400, 827, 1484, 516, 397, 369, 1435, 670] payload_rows_per_dst=[6954, 9640, 11606, 9420, 9374, 9420, 9060, 9800] dup_rows_per_dst=[2172, 2798, 4320, 3182, 3668, 2528, 1966, 2396] combine_rows_per_src=[6954, 9640, 11606, 9420, 9374, 9420, 9060, 9800] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74816 payload_rows_remote=64918 dup_rows=23488 dup_rows_remote=20596 dedup_ratio=0.238932 router_tokens_per_expert=[2653, 588, 408, 758, 454, 1290, 938, 87, 2844, 641, 1278, 527, 23, 1112, 252, 584, 1015, 1176, 488, 762, 288, 355, 495, 541, 825, 914, 118, 385, 926, 771, 402, 1115, 1211, 300, 971, 1718, 626, 407, 888, 359, 211, 358, 379, 186, 1679, 1900, 741, 158, 312, 627, 561, 847, 1085, 1350, 722, 249, 376, 774, 1605, 593, 357, 465, 1527, 597] payload_rows_per_dst=[6914, 7798, 7908, 12186, 7414, 10822, 11876, 9898] dup_rows_per_dst=[2132, 1036, 2332, 4414, 1908, 3838, 4936, 2892] combine_rows_per_src=[6914, 7798, 7908, 12186, 7414, 10822, 11876, 9898] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75262 payload_rows_remote=66936 dup_rows=23042 dup_rows_remote=20396 dedup_ratio=0.234395 router_tokens_per_expert=[2567, 569, 402, 778, 376, 1363, 967, 106, 2601, 735, 1254, 522, 21, 1070, 277, 487, 932, 1224, 460, 923, 369, 359, 486, 608, 801, 793, 147, 400, 972, 769, 430, 1174, 1283, 358, 990, 1657, 693, 380, 954, 371, 183, 365, 369, 249, 1599, 2024, 695, 179, 263, 619, 642, 819, 996, 1583, 642, 358, 414, 779, 1505, 496, 434, 447, 1386, 478] payload_rows_per_dst=[10776, 10382, 8352, 8326, 9518, 9050, 9392, 9466] dup_rows_per_dst=[3244, 3552, 2606, 2646, 3854, 2276, 2452, 2412] combine_rows_per_src=[10776, 10382, 8352, 8326, 9518, 9050, 9392, 9466] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=263249920 dispatch_meta_bytes=335320 combine_payload_bytes=263249920 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=263585240 ingress_bytes=367058944 link_bytes=367058944 token_bytes=526835160 weight_movement_bytes=103809024 total_nvlink_bytes=630644184 weight_to_token_ratio=0.197043 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.381213256e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74634 payload_rows_remote=62728 dup_rows=23670 dup_rows_remote=17654 dedup_ratio=0.240784 router_tokens_per_expert=[2641, 653, 429, 788, 389, 1311, 978, 84, 2484, 683, 1298, 474, 20, 1176, 303, 582, 1074, 1205, 514, 834, 258, 358, 490, 579, 756, 837, 141, 397, 947, 765, 427, 1047, 1260, 324, 1194, 1429, 546, 370, 889, 414, 181, 381, 406, 230, 1718, 1842, 729, 234, 311, 610, 643, 958, 1031, 1411, 623, 292, 414, 702, 1497, 544, 412, 487, 1489, 659] payload_rows_per_dst=[11906, 9162, 8242, 8100, 9248, 9208, 9354, 9414] dup_rows_per_dst=[6016, 2162, 2382, 2534, 3604, 2234, 2404, 2334] combine_rows_per_src=[11906, 9162, 8242, 8100, 9248, 9208, 9354, 9414] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=274784256 dispatch_meta_bytes=351432 combine_payload_bytes=274784256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275135688 ingress_bytes=309387264 link_bytes=309387264 token_bytes=549919944 weight_movement_bytes=34603008 total_nvlink_bytes=584522952 weight_to_token_ratio=0.062924 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.164199367e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=269737984 dispatch_meta_bytes=345424 combine_payload_bytes=269737984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270083408 ingress_bytes=304340992 link_bytes=304340992 token_bytes=539821392 weight_movement_bytes=34603008 total_nvlink_bytes=574424400 weight_to_token_ratio=0.064101 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=15488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.145210651e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=342056 combine_payload_bytes=265904128 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266246184 ingress_bytes=335110144 link_bytes=335110144 token_bytes=532150312 weight_movement_bytes=69206016 total_nvlink_bytes=601356328 weight_to_token_ratio=0.130050 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=12032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260992493e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=274169856 dispatch_meta_bytes=349328 combine_payload_bytes=274169856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274519184 ingress_bytes=308772864 link_bytes=308772864 token_bytes=548689040 weight_movement_bytes=34603008 total_nvlink_bytes=583292048 weight_to_token_ratio=0.063065 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=8320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.161887429e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74808 payload_rows_remote=65360 dup_rows=23496 dup_rows_remote=19504 dedup_ratio=0.239014 router_tokens_per_expert=[2715, 640, 488, 746, 384, 1314, 949, 108, 2759, 669, 1343, 493, 27, 1166, 265, 577, 946, 1143, 433, 824, 279, 341, 470, 535, 766, 780, 142, 380, 967, 862, 364, 873, 1245, 360, 985, 1741, 637, 369, 993, 390, 185, 390, 380, 244, 1623, 1988, 704, 206, 321, 634, 601, 816, 1013, 1482, 702, 284, 380, 699, 1635, 482, 396, 403, 1444, 672] payload_rows_per_dst=[7068, 10794, 11366, 7668, 9448, 9258, 9374, 9832] dup_rows_per_dst=[2190, 3804, 4006, 2600, 3992, 2182, 2332, 2390] combine_rows_per_src=[7068, 10794, 11366, 7668, 9448, 9258, 9374, 9832] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74816 payload_rows_remote=64918 dup_rows=23488 dup_rows_remote=20596 dedup_ratio=0.238932 router_tokens_per_expert=[2653, 588, 408, 758, 454, 1290, 938, 87, 2844, 641, 1278, 527, 23, 1112, 252, 584, 1015, 1176, 488, 762, 288, 355, 495, 541, 825, 914, 118, 385, 926, 771, 402, 1115, 1211, 300, 971, 1718, 626, 407, 888, 359, 211, 358, 379, 186, 1679, 1900, 741, 158, 312, 627, 561, 847, 1085, 1350, 722, 249, 376, 774, 1605, 593, 357, 465, 1527, 597] payload_rows_per_dst=[6914, 7798, 7908, 12186, 7414, 10822, 11876, 9898] dup_rows_per_dst=[2132, 1036, 2332, 4414, 1908, 3838, 4936, 2892] combine_rows_per_src=[6914, 7798, 7908, 12186, 7414, 10822, 11876, 9898] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75872 payload_rows_remote=65950 dup_rows=22432 dup_rows_remote=19452 dedup_ratio=0.228190 router_tokens_per_expert=[2682, 722, 503, 716, 404, 1196, 968, 127, 2552, 693, 1209, 490, 30, 1118, 286, 639, 949, 1177, 431, 793, 329, 346, 536, 550, 833, 898, 140, 397, 1022, 738, 375, 929, 1289, 392, 923, 1668, 578, 356, 1060, 404, 174, 430, 407, 257, 1729, 1821, 750, 199, 294, 549, 573, 874, 1038, 1528, 688, 271, 385, 733, 1513, 568, 403, 417, 1510, 593] payload_rows_per_dst=[7076, 7862, 10672, 11752, 7872, 10928, 9922, 9788] dup_rows_per_dst=[2196, 1068, 3642, 4016, 2132, 3942, 2980, 2456] combine_rows_per_src=[7076, 7862, 10672, 11752, 7872, 10928, 9922, 9788] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75274 payload_rows_remote=65854 dup_rows=23030 dup_rows_remote=20502 dedup_ratio=0.234273 router_tokens_per_expert=[2760, 669, 420, 727, 379, 1296, 974, 98, 2583, 707, 1317, 494, 18, 1144, 275, 581, 988, 1206, 480, 774, 291, 356, 522, 586, 798, 779, 166, 353, 944, 821, 445, 1095, 1306, 340, 1049, 1826, 610, 350, 894, 368, 168, 344, 410, 274, 1652, 1954, 785, 165, 301, 583, 613, 812, 1032, 1323, 592, 257, 400, 827, 1484, 516, 397, 369, 1435, 670] payload_rows_per_dst=[6954, 9640, 11606, 9420, 9374, 9420, 9060, 9800] dup_rows_per_dst=[2172, 2798, 4320, 3182, 3668, 2528, 1966, 2396] combine_rows_per_src=[6954, 9640, 11606, 9420, 9374, 9420, 9060, 9800] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=256933888 dispatch_meta_bytes=321528 combine_payload_bytes=256933888 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=257255416 ingress_bytes=360742912 link_bytes=360742912 token_bytes=514189304 weight_movement_bytes=103809024 total_nvlink_bytes=617998328 weight_to_token_ratio=0.201889 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=10752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.357446536e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79338 payload_rows_remote=69456 dup_rows=18966 dup_rows_remote=16558 dedup_ratio=0.192932 router_tokens_per_expert=[109, 343, 154, 139, 1808, 304, 2918, 212, 23, 330, 930, 332, 703, 699, 89, 3039, 539, 1033, 1223, 1313, 437, 1688, 752, 437, 190, 226, 1114, 4830, 38, 1146, 74, 741, 122, 9, 9, 227, 448, 10, 219, 463, 1559, 605, 0, 457, 148, 334, 7, 1155, 5949, 143, 2012, 14, 319, 278, 934, 665, 976, 1092, 582, 564, 969, 181, 393, 396] payload_rows_per_dst=[9744, 9882, 10838, 12484, 13224, 7060, 7420, 8686] dup_rows_per_dst=[2230, 2408, 4006, 4234, 1688, 1470, 1310, 1620] combine_rows_per_src=[9744, 9882, 10838, 12484, 13224, 7060, 7420, 8686] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=263249920 dispatch_meta_bytes=335320 combine_payload_bytes=263249920 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=263585240 ingress_bytes=367058944 link_bytes=367058944 token_bytes=526835160 weight_movement_bytes=103809024 total_nvlink_bytes=630644184 weight_to_token_ratio=0.197043 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.381213256e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=274169856 dispatch_meta_bytes=349328 combine_payload_bytes=274169856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274519184 ingress_bytes=308772864 link_bytes=308772864 token_bytes=548689040 weight_movement_bytes=34603008 total_nvlink_bytes=583292048 weight_to_token_ratio=0.063065 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=8320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.161887429e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267714560 dispatch_meta_bytes=339456 combine_payload_bytes=267714560 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=268054016 ingress_bytes=336920576 link_bytes=336920576 token_bytes=535768576 weight_movement_bytes=69206016 total_nvlink_bytes=604974592 weight_to_token_ratio=0.129171 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=9856 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.267805003e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79098 payload_rows_remote=68500 dup_rows=19206 dup_rows_remote=15456 dedup_ratio=0.195374 router_tokens_per_expert=[213, 347, 214, 157, 1915, 288, 2820, 209, 34, 400, 1084, 418, 725, 742, 161, 2777, 663, 934, 1193, 1299, 532, 1448, 689, 416, 284, 273, 1065, 4309, 42, 1030, 101, 709, 176, 17, 13, 246, 546, 15, 256, 616, 1460, 729, 1, 475, 148, 398, 7, 925, 5580, 202, 1834, 14, 243, 342, 1034, 750, 1025, 1234, 649, 633, 897, 298, 490, 408] payload_rows_per_dst=[9796, 9742, 10598, 11976, 13156, 7168, 7576, 9086] dup_rows_per_dst=[2530, 2940, 3750, 3650, 1774, 1118, 1262, 2182] combine_rows_per_src=[9796, 9742, 10598, 11976, 13156, 7168, 7576, 9086] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74568 payload_rows_remote=64238 dup_rows=23736 dup_rows_remote=20520 dedup_ratio=0.241455 router_tokens_per_expert=[166, 372, 155, 141, 1924, 317, 3084, 210, 25, 380, 1239, 443, 731, 568, 135, 2773, 587, 995, 1222, 1146, 437, 1551, 863, 438, 219, 209, 931, 4672, 44, 973, 57, 877, 161, 9, 12, 244, 507, 27, 251, 481, 1637, 654, 0, 415, 156, 355, 3, 879, 5690, 195, 1922, 27, 233, 341, 1075, 772, 889, 1021, 616, 510, 1008, 235, 518, 425] payload_rows_per_dst=[9950, 8514, 8022, 7310, 3320, 12678, 14444, 10330] dup_rows_per_dst=[2788, 2308, 1166, 972, 64, 4864, 8358, 3216] combine_rows_per_src=[9950, 8514, 8022, 7310, 3320, 12678, 14444, 10330] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78558 payload_rows_remote=66206 dup_rows=19746 dup_rows_remote=14730 dedup_ratio=0.200867 router_tokens_per_expert=[128, 397, 272, 187, 1833, 304, 2681, 168, 40, 336, 1007, 388, 726, 749, 121, 2881, 584, 1084, 1226, 1358, 510, 1588, 834, 417, 221, 203, 993, 4555, 51, 975, 77, 834, 179, 14, 21, 243, 524, 19, 250, 603, 1565, 648, 0, 485, 167, 366, 7, 891, 5635, 203, 1862, 21, 313, 319, 922, 785, 930, 1154, 613, 543, 1028, 260, 446, 408] payload_rows_per_dst=[9502, 9736, 8024, 6148, 13162, 12352, 9130, 10504] dup_rows_per_dst=[2438, 2760, 1286, 560, 1814, 5016, 2436, 3436] combine_rows_per_src=[9502, 9736, 8024, 6148, 13162, 12352, 9130, 10504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=342056 combine_payload_bytes=265904128 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266246184 ingress_bytes=335110144 link_bytes=335110144 token_bytes=532150312 weight_movement_bytes=69206016 total_nvlink_bytes=601356328 weight_to_token_ratio=0.130050 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=12032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260992493e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=270131200 dispatch_meta_bytes=341608 combine_payload_bytes=270131200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270472808 ingress_bytes=304734208 link_bytes=304734208 token_bytes=540604008 weight_movement_bytes=34603008 total_nvlink_bytes=575207016 weight_to_token_ratio=0.064008 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.146690292e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79138 payload_rows_remote=67140 dup_rows=19166 dup_rows_remote=15520 dedup_ratio=0.194967 router_tokens_per_expert=[170, 420, 228, 147, 1863, 417, 2693, 176, 33, 349, 897, 423, 754, 634, 166, 2891, 512, 1011, 1233, 1408, 508, 1610, 788, 387, 202, 210, 973, 4589, 36, 937, 76, 799, 134, 14, 18, 291, 509, 14, 211, 617, 1438, 611, 0, 495, 143, 385, 3, 1250, 5586, 173, 1921, 17, 241, 316, 982, 807, 935, 1076, 665, 571, 941, 228, 560, 460] payload_rows_per_dst=[9836, 9700, 10980, 11998, 12934, 7234, 7506, 8950] dup_rows_per_dst=[2392, 2594, 3934, 3646, 1854, 1416, 1408, 1922] combine_rows_per_src=[9836, 9700, 10980, 11998, 12934, 7234, 7506, 8950] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=284491776 dispatch_meta_bytes=344056 combine_payload_bytes=284491776 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=284835832 ingress_bytes=353697792 link_bytes=353697792 token_bytes=569327608 weight_movement_bytes=69206016 total_nvlink_bytes=638533624 weight_to_token_ratio=0.121557 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=22528 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.330936316e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280576000 dispatch_meta_bytes=335824 combine_payload_bytes=280576000 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=280911824 ingress_bytes=418988032 link_bytes=418988032 token_bytes=561487824 weight_movement_bytes=138412032 total_nvlink_bytes=699899856 weight_to_token_ratio=0.246509 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=9472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.576618233e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=271179776 dispatch_meta_bytes=323744 combine_payload_bytes=271179776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271503520 ingress_bytes=305782784 link_bytes=305782784 token_bytes=542683296 weight_movement_bytes=34603008 total_nvlink_bytes=577286304 weight_to_token_ratio=0.063763 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=15488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.150635999e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=263118848 dispatch_meta_bytes=339032 combine_payload_bytes=263118848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263457880 ingress_bytes=297721856 link_bytes=297721856 token_bytes=526576728 weight_movement_bytes=34603008 total_nvlink_bytes=561179736 weight_to_token_ratio=0.065713 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=8960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.120303376e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=269737984 dispatch_meta_bytes=345424 combine_payload_bytes=269737984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270083408 ingress_bytes=304340992 link_bytes=304340992 token_bytes=539821392 weight_movement_bytes=34603008 total_nvlink_bytes=574424400 weight_to_token_ratio=0.064101 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=15488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.145210651e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75138 payload_rows_remote=67086 dup_rows=23166 dup_rows_remote=20772 dedup_ratio=0.235657 router_tokens_per_expert=[2779, 671, 380, 712, 527, 1204, 1112, 100, 2432, 601, 1220, 530, 28, 1143, 260, 672, 1051, 1088, 502, 898, 302, 335, 496, 551, 907, 811, 130, 440, 871, 859, 420, 1017, 1143, 351, 1053, 1571, 630, 385, 978, 442, 202, 360, 373, 225, 1648, 1880, 723, 179, 349, 562, 666, 756, 1013, 1267, 606, 216, 365, 913, 1522, 503, 495, 547, 1484, 696] payload_rows_per_dst=[11180, 10366, 8052, 8118, 9402, 8946, 8910, 10164] dup_rows_per_dst=[3790, 3406, 2394, 2792, 3704, 2234, 1960, 2886] combine_rows_per_src=[11180, 10366, 8052, 8118, 9402, 8946, 8910, 10164] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78968 payload_rows_remote=68448 dup_rows=19336 dup_rows_remote=16552 dedup_ratio=0.196696 router_tokens_per_expert=[126, 440, 223, 151, 1991, 308, 2764, 158, 35, 343, 963, 345, 810, 507, 149, 2956, 522, 1035, 1188, 1451, 573, 1499, 812, 454, 248, 234, 874, 4532, 39, 1089, 76, 737, 182, 12, 21, 266, 539, 18, 223, 630, 1520, 628, 0, 484, 129, 361, 8, 1075, 5528, 182, 1877, 23, 328, 265, 1025, 655, 958, 1081, 570, 549, 1050, 243, 583, 507] payload_rows_per_dst=[10520, 9590, 10354, 12096, 13036, 7038, 7302, 9032] dup_rows_per_dst=[2784, 2626, 3732, 3562, 1802, 1372, 1408, 2050] combine_rows_per_src=[10520, 9590, 10354, 12096, 13036, 7038, 7302, 9032] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=275005440 dispatch_meta_bytes=330640 combine_payload_bytes=275005440 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=275336080 ingress_bytes=344211456 link_bytes=344211456 token_bytes=550341520 weight_movement_bytes=69206016 total_nvlink_bytes=619547536 weight_to_token_ratio=0.125751 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.295239998e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78822 payload_rows_remote=65832 dup_rows=19482 dup_rows_remote=17888 dedup_ratio=0.198181 router_tokens_per_expert=[122, 346, 155, 163, 1870, 385, 2870, 153, 19, 282, 900, 367, 753, 630, 141, 3020, 610, 1023, 1419, 1277, 434, 1626, 790, 306, 213, 240, 1036, 4703, 25, 969, 49, 812, 122, 18, 17, 248, 411, 13, 221, 506, 1540, 704, 0, 494, 178, 336, 10, 1186, 5736, 164, 1891, 9, 270, 273, 954, 774, 924, 1118, 566, 536, 961, 217, 562, 485] payload_rows_per_dst=[9710, 9732, 9570, 9974, 12990, 9274, 7458, 10114] dup_rows_per_dst=[2418, 2492, 2752, 2666, 1594, 3076, 1346, 3138] combine_rows_per_src=[9710, 9732, 9570, 9974, 12990, 9274, 7458, 10114] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79138 payload_rows_remote=67140 dup_rows=19166 dup_rows_remote=15520 dedup_ratio=0.194967 router_tokens_per_expert=[170, 420, 228, 147, 1863, 417, 2693, 176, 33, 349, 897, 423, 754, 634, 166, 2891, 512, 1011, 1233, 1408, 508, 1610, 788, 387, 202, 210, 973, 4589, 36, 937, 76, 799, 134, 14, 18, 291, 509, 14, 211, 617, 1438, 611, 0, 495, 143, 385, 3, 1250, 5586, 173, 1921, 17, 241, 316, 982, 807, 935, 1076, 665, 571, 941, 228, 560, 460] payload_rows_per_dst=[9836, 9700, 10980, 11998, 12934, 7234, 7506, 8950] dup_rows_per_dst=[2392, 2594, 3934, 3646, 1854, 1416, 1408, 1922] combine_rows_per_src=[9836, 9700, 10980, 11998, 12934, 7234, 7506, 8950] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280363008 dispatch_meta_bytes=340000 combine_payload_bytes=280363008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280703008 ingress_bytes=314966016 link_bytes=314966016 token_bytes=561066016 weight_movement_bytes=34603008 total_nvlink_bytes=595669024 weight_to_token_ratio=0.061674 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22656 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.185191761e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78822 payload_rows_remote=65832 dup_rows=19482 dup_rows_remote=17888 dedup_ratio=0.198181 router_tokens_per_expert=[122, 346, 155, 163, 1870, 385, 2870, 153, 19, 282, 900, 367, 753, 630, 141, 3020, 610, 1023, 1419, 1277, 434, 1626, 790, 306, 213, 240, 1036, 4703, 25, 969, 49, 812, 122, 18, 17, 248, 411, 13, 221, 506, 1540, 704, 0, 494, 178, 336, 10, 1186, 5736, 164, 1891, 9, 270, 273, 954, 774, 924, 1118, 566, 536, 961, 217, 562, 485] payload_rows_per_dst=[9710, 9732, 9570, 9974, 12990, 9274, 7458, 10114] dup_rows_per_dst=[2418, 2492, 2752, 2666, 1594, 3076, 1346, 3138] combine_rows_per_src=[9710, 9732, 9570, 9974, 12990, 9274, 7458, 10114] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79338 payload_rows_remote=69456 dup_rows=18966 dup_rows_remote=16558 dedup_ratio=0.192932 router_tokens_per_expert=[109, 343, 154, 139, 1808, 304, 2918, 212, 23, 330, 930, 332, 703, 699, 89, 3039, 539, 1033, 1223, 1313, 437, 1688, 752, 437, 190, 226, 1114, 4830, 38, 1146, 74, 741, 122, 9, 9, 227, 448, 10, 219, 463, 1559, 605, 0, 457, 148, 334, 7, 1155, 5949, 143, 2012, 14, 319, 278, 934, 665, 976, 1092, 582, 564, 969, 181, 393, 396] payload_rows_per_dst=[9744, 9882, 10838, 12484, 13224, 7060, 7420, 8686] dup_rows_per_dst=[2230, 2408, 4006, 4234, 1688, 1470, 1310, 1620] combine_rows_per_src=[9744, 9882, 10838, 12484, 13224, 7060, 7420, 8686] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=274784256 dispatch_meta_bytes=351432 combine_payload_bytes=274784256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275135688 ingress_bytes=309387264 link_bytes=309387264 token_bytes=549919944 weight_movement_bytes=34603008 total_nvlink_bytes=584522952 weight_to_token_ratio=0.062924 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.164199367e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75560 payload_rows_remote=62636 dup_rows=22744 dup_rows_remote=15728 dedup_ratio=0.231364 router_tokens_per_expert=[176, 416, 194, 146, 1681, 288, 2723, 209, 44, 439, 1092, 406, 792, 680, 152, 3035, 571, 1175, 1143, 1192, 520, 1647, 864, 373, 229, 251, 1139, 4523, 48, 975, 60, 812, 165, 12, 21, 250, 549, 24, 228, 532, 1481, 638, 0, 437, 172, 390, 8, 935, 5475, 195, 1743, 15, 257, 285, 1148, 825, 998, 1060, 619, 565, 972, 220, 485, 453] payload_rows_per_dst=[9448, 10222, 8036, 6406, 5404, 12394, 12924, 10726] dup_rows_per_dst=[2218, 3058, 1256, 622, 488, 4774, 7016, 3312] combine_rows_per_src=[9448, 10222, 8036, 6406, 5404, 12394, 12924, 10726] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=269647872 dispatch_meta_bytes=334880 combine_payload_bytes=269647872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269982752 ingress_bytes=304250880 link_bytes=304250880 token_bytes=539630624 weight_movement_bytes=34603008 total_nvlink_bytes=574233632 weight_to_token_ratio=0.064124 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=35200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.144871567e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76968 payload_rows_remote=66030 dup_rows=21336 dup_rows_remote=15516 dedup_ratio=0.217041 router_tokens_per_expert=[2, 222, 12, 189, 25, 24, 2048, 5208, 1, 846, 415, 1512, 2112, 114, 161, 94, 3124, 1051, 923, 893, 83, 7, 4524, 316, 347, 1247, 4766, 1045, 644, 301, 130, 17, 267, 434, 1476, 108, 4243, 402, 487, 153, 181, 237, 0, 69, 1353, 1869, 82, 27, 223, 1693, 261, 19, 44, 90, 397, 5, 1093, 50, 24, 37, 2, 652, 24, 747] payload_rows_per_dst=[12692, 10938, 10926, 12454, 11706, 6698, 6512, 5042] dup_rows_per_dst=[2768, 5820, 2694, 4540, 3434, 938, 926, 216] combine_rows_per_src=[12692, 10938, 10926, 12454, 11706, 6698, 6512, 5042] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74568 payload_rows_remote=64238 dup_rows=23736 dup_rows_remote=20520 dedup_ratio=0.241455 router_tokens_per_expert=[166, 372, 155, 141, 1924, 317, 3084, 210, 25, 380, 1239, 443, 731, 568, 135, 2773, 587, 995, 1222, 1146, 437, 1551, 863, 438, 219, 209, 931, 4672, 44, 973, 57, 877, 161, 9, 12, 244, 507, 27, 251, 481, 1637, 654, 0, 415, 156, 355, 3, 879, 5690, 195, 1922, 27, 233, 341, 1075, 772, 889, 1021, 616, 510, 1008, 235, 518, 425] payload_rows_per_dst=[9950, 8514, 8022, 7310, 3320, 12678, 14444, 10330] dup_rows_per_dst=[2788, 2308, 1166, 972, 64, 4864, 8358, 3216] combine_rows_per_src=[9950, 8514, 8022, 7310, 3320, 12678, 14444, 10330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78558 payload_rows_remote=66206 dup_rows=19746 dup_rows_remote=14730 dedup_ratio=0.200867 router_tokens_per_expert=[128, 397, 272, 187, 1833, 304, 2681, 168, 40, 336, 1007, 388, 726, 749, 121, 2881, 584, 1084, 1226, 1358, 510, 1588, 834, 417, 221, 203, 993, 4555, 51, 975, 77, 834, 179, 14, 21, 243, 524, 19, 250, 603, 1565, 648, 0, 485, 167, 366, 7, 891, 5635, 203, 1862, 21, 313, 319, 922, 785, 930, 1154, 613, 543, 1028, 260, 446, 408] payload_rows_per_dst=[9502, 9736, 8024, 6148, 13162, 12352, 9130, 10504] dup_rows_per_dst=[2438, 2760, 1286, 560, 1814, 5016, 2436, 3436] combine_rows_per_src=[9502, 9736, 8024, 6148, 13162, 12352, 9130, 10504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280363008 dispatch_meta_bytes=340000 combine_payload_bytes=280363008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280703008 ingress_bytes=314966016 link_bytes=314966016 token_bytes=561066016 weight_movement_bytes=34603008 total_nvlink_bytes=595669024 weight_to_token_ratio=0.061674 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22656 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.185191761e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76232 payload_rows_remote=69776 dup_rows=22072 dup_rows_remote=20956 dedup_ratio=0.224528 router_tokens_per_expert=[1, 376, 8, 157, 47, 32, 1575, 4403, 0, 741, 351, 1449, 2110, 168, 287, 159, 3103, 1271, 961, 1141, 118, 4, 4123, 291, 454, 1425, 4283, 1043, 701, 351, 190, 32, 279, 427, 1309, 151, 4143, 488, 673, 190, 304, 331, 3, 142, 1250, 2112, 106, 44, 429, 1279, 208, 41, 54, 103, 552, 8, 1269, 94, 34, 64, 9, 626, 56, 1019] payload_rows_per_dst=[10824, 10880, 6456, 7610, 11454, 7370, 10534, 11104] dup_rows_per_dst=[2374, 5856, 1116, 1812, 3866, 1214, 3060, 2774] combine_rows_per_src=[10824, 10880, 6456, 7610, 11454, 7370, 10534, 11104] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76924 payload_rows_remote=65024 dup_rows=21380 dup_rows_remote=18408 dedup_ratio=0.217489 router_tokens_per_expert=[2, 302, 11, 211, 24, 34, 1768, 4817, 0, 718, 325, 1279, 1922, 115, 188, 100, 3052, 1385, 949, 1114, 77, 1, 4394, 322, 468, 1479, 4684, 1006, 676, 338, 149, 12, 262, 390, 1272, 105, 4210, 428, 625, 169, 235, 236, 2, 127, 1270, 2247, 77, 63, 377, 1558, 238, 19, 46, 61, 459, 2, 1090, 42, 49, 48, 3, 585, 29, 906] payload_rows_per_dst=[4522, 7906, 6932, 12738, 9384, 12382, 11160, 11900] dup_rows_per_dst=[182, 1388, 1206, 5152, 2780, 4552, 3148, 2972] combine_rows_per_src=[4522, 7906, 6932, 12738, 9384, 12382, 11160, 11900] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78612 payload_rows_remote=66490 dup_rows=19692 dup_rows_remote=15304 dedup_ratio=0.200317 router_tokens_per_expert=[1, 259, 14, 199, 33, 35, 1694, 4695, 1, 802, 341, 1388, 1992, 119, 155, 92, 2948, 1315, 1002, 1098, 82, 3, 4697, 307, 391, 1367, 4581, 1054, 602, 345, 182, 24, 272, 456, 1416, 112, 4223, 410, 571, 178, 199, 299, 0, 104, 1147, 2095, 128, 60, 309, 1634, 284, 43, 47, 71, 447, 4, 1112, 70, 72, 57, 8, 592, 34, 880] payload_rows_per_dst=[11518, 8168, 10216, 6854, 6098, 12122, 11756, 11880] dup_rows_per_dst=[2342, 1612, 3294, 1076, 732, 4388, 3316, 2932] combine_rows_per_src=[11518, 8168, 10216, 6854, 6098, 12122, 11756, 11880] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=275005440 dispatch_meta_bytes=330640 combine_payload_bytes=275005440 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=275336080 ingress_bytes=344211456 link_bytes=344211456 token_bytes=550341520 weight_movement_bytes=69206016 total_nvlink_bytes=619547536 weight_to_token_ratio=0.125751 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.295239998e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=269647872 dispatch_meta_bytes=334880 combine_payload_bytes=269647872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269982752 ingress_bytes=304250880 link_bytes=304250880 token_bytes=539630624 weight_movement_bytes=34603008 total_nvlink_bytes=574233632 weight_to_token_ratio=0.064124 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=35200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.144871567e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=284491776 dispatch_meta_bytes=344056 combine_payload_bytes=284491776 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=284835832 ingress_bytes=353697792 link_bytes=353697792 token_bytes=569327608 weight_movement_bytes=69206016 total_nvlink_bytes=638533624 weight_to_token_ratio=0.121557 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=22528 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.330936316e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=256557056 dispatch_meta_bytes=313456 combine_payload_bytes=256557056 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=256870512 ingress_bytes=325763072 link_bytes=325763072 token_bytes=513427568 weight_movement_bytes=69206016 total_nvlink_bytes=582633584 weight_to_token_ratio=0.134792 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=15104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.225820214e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78706 payload_rows_remote=72106 dup_rows=19598 dup_rows_remote=18632 dedup_ratio=0.199361 router_tokens_per_expert=[3, 340, 3, 176, 30, 57, 1771, 4772, 0, 777, 373, 1356, 1910, 123, 185, 136, 3348, 1286, 858, 1139, 78, 4, 4391, 350, 437, 1396, 4589, 941, 550, 269, 174, 16, 269, 396, 1357, 136, 4440, 422, 588, 173, 190, 309, 0, 97, 1199, 2050, 110, 78, 330, 1559, 259, 27, 65, 78, 463, 3, 1076, 54, 42, 64, 7, 545, 47, 881] payload_rows_per_dst=[11714, 8570, 10026, 6600, 11838, 7050, 11280, 11628] dup_rows_per_dst=[2590, 2120, 3130, 966, 3724, 1016, 3070, 2982] combine_rows_per_src=[11714, 8570, 10026, 6600, 11838, 7050, 11280, 11628] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=270458880 dispatch_meta_bytes=326184 combine_payload_bytes=270458880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270785064 ingress_bytes=305061888 link_bytes=305061888 token_bytes=541243944 weight_movement_bytes=34603008 total_nvlink_bytes=575846952 weight_to_token_ratio=0.063932 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=15488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.147923325e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=285802496 dispatch_meta_bytes=362928 combine_payload_bytes=285802496 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=286165424 ingress_bytes=424214528 link_bytes=424214528 token_bytes=571967920 weight_movement_bytes=138412032 total_nvlink_bytes=710379952 weight_to_token_ratio=0.241993 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=12288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.596285117e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=272343040 dispatch_meta_bytes=327176 combine_payload_bytes=272343040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272670216 ingress_bytes=306946048 link_bytes=306946048 token_bytes=545013256 weight_movement_bytes=34603008 total_nvlink_bytes=579616264 weight_to_token_ratio=0.063490 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=16896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.155013267e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=333728 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266672032 ingress_bytes=300941312 link_bytes=300941312 token_bytes=533010336 weight_movement_bytes=34603008 total_nvlink_bytes=567613344 weight_to_token_ratio=0.064920 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=26880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132417929e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=263118848 dispatch_meta_bytes=339032 combine_payload_bytes=263118848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263457880 ingress_bytes=297721856 link_bytes=297721856 token_bytes=526576728 weight_movement_bytes=34603008 total_nvlink_bytes=561179736 weight_to_token_ratio=0.065713 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=8960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.120303376e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=271179776 dispatch_meta_bytes=323744 combine_payload_bytes=271179776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271503520 ingress_bytes=305782784 link_bytes=305782784 token_bytes=542683296 weight_movement_bytes=34603008 total_nvlink_bytes=577286304 weight_to_token_ratio=0.063763 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=15488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.150635999e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79098 payload_rows_remote=68500 dup_rows=19206 dup_rows_remote=15456 dedup_ratio=0.195374 router_tokens_per_expert=[213, 347, 214, 157, 1915, 288, 2820, 209, 34, 400, 1084, 418, 725, 742, 161, 2777, 663, 934, 1193, 1299, 532, 1448, 689, 416, 284, 273, 1065, 4309, 42, 1030, 101, 709, 176, 17, 13, 246, 546, 15, 256, 616, 1460, 729, 1, 475, 148, 398, 7, 925, 5580, 202, 1834, 14, 243, 342, 1034, 750, 1025, 1234, 649, 633, 897, 298, 490, 408] payload_rows_per_dst=[9796, 9742, 10598, 11976, 13156, 7168, 7576, 9086] dup_rows_per_dst=[2530, 2940, 3750, 3650, 1774, 1118, 1262, 2182] combine_rows_per_src=[9796, 9742, 10598, 11976, 13156, 7168, 7576, 9086] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76236 payload_rows_remote=64506 dup_rows=22068 dup_rows_remote=19540 dedup_ratio=0.224487 router_tokens_per_expert=[5, 381, 14, 68, 36, 44, 1914, 4667, 2, 853, 383, 1455, 1825, 122, 204, 123, 2959, 1132, 988, 1023, 86, 5, 4006, 275, 464, 1436, 4496, 984, 622, 319, 167, 28, 248, 405, 1459, 119, 4637, 367, 568, 160, 254, 324, 1, 165, 1819, 2035, 118, 100, 313, 1373, 219, 39, 70, 80, 409, 5, 1149, 67, 75, 59, 4, 567, 41, 817] payload_rows_per_dst=[11730, 10662, 11620, 12106, 11978, 8088, 4744, 5308] dup_rows_per_dst=[2528, 5190, 3410, 4926, 3948, 1544, 272, 250] combine_rows_per_src=[11730, 10662, 11620, 12106, 11978, 8088, 4744, 5308] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=295346176 dispatch_meta_bytes=362952 combine_payload_bytes=295346176 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=295709128 ingress_bytes=364552192 link_bytes=364552192 token_bytes=591055304 weight_movement_bytes=69206016 total_nvlink_bytes=660261320 weight_to_token_ratio=0.117089 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.371780550e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78706 payload_rows_remote=72106 dup_rows=19598 dup_rows_remote=18632 dedup_ratio=0.199361 router_tokens_per_expert=[3, 340, 3, 176, 30, 57, 1771, 4772, 0, 777, 373, 1356, 1910, 123, 185, 136, 3348, 1286, 858, 1139, 78, 4, 4391, 350, 437, 1396, 4589, 941, 550, 269, 174, 16, 269, 396, 1357, 136, 4440, 422, 588, 173, 190, 309, 0, 97, 1199, 2050, 110, 78, 330, 1559, 259, 27, 65, 78, 463, 3, 1076, 54, 42, 64, 7, 545, 47, 881] payload_rows_per_dst=[11714, 8570, 10026, 6600, 11838, 7050, 11280, 11628] dup_rows_per_dst=[2590, 2120, 3130, 966, 3724, 1016, 3070, 2982] combine_rows_per_src=[11714, 8570, 10026, 6600, 11838, 7050, 11280, 11628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79222 payload_rows_remote=70906 dup_rows=19082 dup_rows_remote=17178 dedup_ratio=0.194112 router_tokens_per_expert=[2, 227, 4, 215, 34, 28, 1678, 4865, 1, 760, 459, 1288, 2149, 126, 173, 113, 3014, 1326, 1052, 865, 85, 3, 4692, 398, 334, 1365, 4823, 1036, 598, 303, 115, 18, 238, 519, 1429, 98, 4295, 482, 612, 148, 192, 264, 0, 88, 1397, 1975, 58, 65, 223, 1536, 231, 20, 41, 57, 339, 4, 1055, 34, 35, 47, 3, 537, 28, 953] payload_rows_per_dst=[11656, 8510, 10302, 6596, 8316, 10370, 11400, 12072] dup_rows_per_dst=[2450, 1628, 3184, 942, 1904, 3130, 2886, 2958] combine_rows_per_src=[11656, 8510, 10302, 6596, 8316, 10370, 11400, 12072] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77452 payload_rows_remote=66230 dup_rows=20852 dup_rows_remote=17834 dedup_ratio=0.212118 router_tokens_per_expert=[3, 342, 13, 210, 34, 18, 1955, 4568, 1, 862, 305, 1440, 1443, 152, 191, 134, 3130, 1342, 929, 947, 130, 3, 4421, 299, 528, 1361, 4345, 1060, 645, 350, 169, 33, 283, 465, 1509, 144, 4110, 403, 577, 190, 217, 395, 0, 122, 1289, 2394, 101, 93, 316, 1603, 265, 64, 57, 78, 307, 9, 1011, 86, 57, 66, 3, 617, 49, 909] payload_rows_per_dst=[7268, 7508, 10508, 10440, 6362, 12718, 11222, 11426] dup_rows_per_dst=[1146, 1548, 3284, 3492, 780, 4724, 3018, 2860] combine_rows_per_src=[7268, 7508, 10508, 10440, 6362, 12718, 11222, 11426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76968 payload_rows_remote=66030 dup_rows=21336 dup_rows_remote=15516 dedup_ratio=0.217041 router_tokens_per_expert=[2, 222, 12, 189, 25, 24, 2048, 5208, 1, 846, 415, 1512, 2112, 114, 161, 94, 3124, 1051, 923, 893, 83, 7, 4524, 316, 347, 1247, 4766, 1045, 644, 301, 130, 17, 267, 434, 1476, 108, 4243, 402, 487, 153, 181, 237, 0, 69, 1353, 1869, 82, 27, 223, 1693, 261, 19, 44, 90, 397, 5, 1093, 50, 24, 37, 2, 652, 24, 747] payload_rows_per_dst=[12692, 10938, 10926, 12454, 11706, 6698, 6512, 5042] dup_rows_per_dst=[2768, 5820, 2694, 4540, 3434, 938, 926, 216] combine_rows_per_src=[12692, 10938, 10926, 12454, 11706, 6698, 6512, 5042] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78612 payload_rows_remote=66490 dup_rows=19692 dup_rows_remote=15304 dedup_ratio=0.200317 router_tokens_per_expert=[1, 259, 14, 199, 33, 35, 1694, 4695, 1, 802, 341, 1388, 1992, 119, 155, 92, 2948, 1315, 1002, 1098, 82, 3, 4697, 307, 391, 1367, 4581, 1054, 602, 345, 182, 24, 272, 456, 1416, 112, 4223, 410, 571, 178, 199, 299, 0, 104, 1147, 2095, 128, 60, 309, 1634, 284, 43, 47, 71, 447, 4, 1112, 70, 72, 57, 8, 592, 34, 880] payload_rows_per_dst=[11518, 8168, 10216, 6854, 6098, 12122, 11756, 11880] dup_rows_per_dst=[2342, 1612, 3294, 1076, 732, 4388, 3316, 2932] combine_rows_per_src=[11518, 8168, 10216, 6854, 6098, 12122, 11756, 11880] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76924 payload_rows_remote=65024 dup_rows=21380 dup_rows_remote=18408 dedup_ratio=0.217489 router_tokens_per_expert=[2, 302, 11, 211, 24, 34, 1768, 4817, 0, 718, 325, 1279, 1922, 115, 188, 100, 3052, 1385, 949, 1114, 77, 1, 4394, 322, 468, 1479, 4684, 1006, 676, 338, 149, 12, 262, 390, 1272, 105, 4210, 428, 625, 169, 235, 236, 2, 127, 1270, 2247, 77, 63, 377, 1558, 238, 19, 46, 61, 459, 2, 1090, 42, 49, 48, 3, 585, 29, 906] payload_rows_per_dst=[4522, 7906, 6932, 12738, 9384, 12382, 11160, 11900] dup_rows_per_dst=[182, 1388, 1206, 5152, 2780, 4552, 3148, 2972] combine_rows_per_src=[4522, 7906, 6932, 12738, 9384, 12382, 11160, 11900] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280576000 dispatch_meta_bytes=335824 combine_payload_bytes=280576000 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=280911824 ingress_bytes=418988032 link_bytes=418988032 token_bytes=561487824 weight_movement_bytes=138412032 total_nvlink_bytes=699899856 weight_to_token_ratio=0.246509 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=9472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.576618233e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=264216576 dispatch_meta_bytes=336184 combine_payload_bytes=264216576 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=264552760 ingress_bytes=333422592 link_bytes=333422592 token_bytes=528769336 weight_movement_bytes=69206016 total_nvlink_bytes=597975352 weight_to_token_ratio=0.130881 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=30336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.254642371e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=295346176 dispatch_meta_bytes=362952 combine_payload_bytes=295346176 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=295709128 ingress_bytes=364552192 link_bytes=364552192 token_bytes=591055304 weight_movement_bytes=69206016 total_nvlink_bytes=660261320 weight_to_token_ratio=0.117089 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.371780550e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=290430976 dispatch_meta_bytes=352336 combine_payload_bytes=290430976 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=290783312 ingress_bytes=359636992 link_bytes=359636992 token_bytes=581214288 weight_movement_bytes=69206016 total_nvlink_bytes=650420304 weight_to_token_ratio=0.119071 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=19072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.353285048e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77452 payload_rows_remote=66230 dup_rows=20852 dup_rows_remote=17834 dedup_ratio=0.212118 router_tokens_per_expert=[3, 342, 13, 210, 34, 18, 1955, 4568, 1, 862, 305, 1440, 1443, 152, 191, 134, 3130, 1342, 929, 947, 130, 3, 4421, 299, 528, 1361, 4345, 1060, 645, 350, 169, 33, 283, 465, 1509, 144, 4110, 403, 577, 190, 217, 395, 0, 122, 1289, 2394, 101, 93, 316, 1603, 265, 64, 57, 78, 307, 9, 1011, 86, 57, 66, 3, 617, 49, 909] payload_rows_per_dst=[7268, 7508, 10508, 10440, 6362, 12718, 11222, 11426] dup_rows_per_dst=[1146, 1548, 3284, 3492, 780, 4724, 3018, 2860] combine_rows_per_src=[7268, 7508, 10508, 10440, 6362, 12718, 11222, 11426] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77242 payload_rows_remote=64642 dup_rows=21062 dup_rows_remote=18226 dedup_ratio=0.214254 router_tokens_per_expert=[784, 58, 1680, 15, 593, 210, 574, 493, 217, 108, 86, 47, 307, 8, 732, 184, 5628, 327, 184, 3, 380, 91, 0, 4062, 237, 0, 2041, 2742, 1009, 1061, 6029, 64, 605, 996, 29, 39, 1374, 299, 998, 199, 331, 1631, 37, 2038, 102, 409, 0, 1106, 1, 135, 2156, 769, 40, 697, 29, 1311, 219, 0, 888, 87, 141, 2096, 187, 249] payload_rows_per_dst=[13688, 12600, 9134, 9940, 7844, 8368, 8720, 6948] dup_rows_per_dst=[6382, 2836, 960, 4368, 1234, 2940, 1556, 786] combine_rows_per_src=[13688, 12600, 9134, 9940, 7844, 8368, 8720, 6948] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77962 payload_rows_remote=68888 dup_rows=20342 dup_rows_remote=18514 dedup_ratio=0.206930 router_tokens_per_expert=[815, 71, 1322, 35, 680, 283, 453, 502, 273, 127, 127, 130, 317, 24, 669, 214, 5138, 320, 220, 8, 257, 157, 0, 3659, 290, 2, 2327, 2812, 852, 1132, 5940, 87, 709, 1015, 17, 47, 1440, 184, 979, 230, 370, 1922, 83, 1827, 43, 326, 6, 1067, 4, 128, 2278, 1289, 72, 598, 33, 1042, 295, 0, 807, 82, 248, 2303, 197, 268] payload_rows_per_dst=[12044, 12638, 9074, 8844, 7902, 8220, 9126, 10114] dup_rows_per_dst=[4894, 3004, 1828, 2052, 1340, 3068, 1762, 2394] combine_rows_per_src=[12044, 12638, 9074, 8844, 7902, 8220, 9126, 10114] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80684 payload_rows_remote=72484 dup_rows=17620 dup_rows_remote=14676 dedup_ratio=0.179240 router_tokens_per_expert=[845, 68, 1463, 33, 709, 269, 604, 502, 141, 121, 102, 76, 317, 19, 586, 290, 5486, 305, 189, 4, 275, 116, 0, 3907, 250, 0, 1918, 2661, 852, 1179, 6022, 56, 768, 1135, 23, 42, 1180, 301, 1028, 200, 427, 1729, 52, 1735, 61, 365, 5, 1198, 1, 154, 2289, 1129, 81, 746, 23, 981, 321, 0, 817, 88, 228, 2301, 194, 185] payload_rows_per_dst=[7898, 12796, 11714, 7680, 12540, 8200, 9066, 10790] dup_rows_per_dst=[1088, 2552, 1036, 830, 4628, 2944, 1742, 2800] combine_rows_per_src=[7898, 12796, 11714, 7680, 12540, 8200, 9066, 10790] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73590 payload_rows_remote=62404 dup_rows=24714 dup_rows_remote=21750 dedup_ratio=0.251404 router_tokens_per_expert=[860, 67, 1355, 22, 661, 257, 426, 516, 236, 115, 70, 55, 333, 21, 489, 240, 5671, 348, 199, 7, 290, 122, 1, 3877, 204, 0, 2113, 2517, 915, 1170, 5967, 70, 549, 881, 8, 43, 1182, 223, 1123, 190, 448, 1717, 52, 1849, 71, 472, 5, 1033, 0, 156, 2490, 1055, 54, 535, 35, 1229, 265, 0, 1100, 113, 222, 2393, 215, 250] payload_rows_per_dst=[7476, 3026, 1836, 13248, 11976, 10680, 14162, 11186] dup_rows_per_dst=[852, 92, 98, 3404, 4176, 4840, 8288, 2964] combine_rows_per_src=[7476, 3026, 1836, 13248, 11976, 10680, 14162, 11186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78264 payload_rows_remote=70690 dup_rows=20040 dup_rows_remote=19262 dedup_ratio=0.203857 router_tokens_per_expert=[868, 49, 1292, 14, 613, 215, 822, 373, 116, 116, 94, 62, 353, 38, 568, 196, 5157, 348, 272, 2, 372, 105, 1, 4120, 216, 0, 2049, 2757, 868, 1006, 6132, 37, 735, 982, 16, 60, 1428, 378, 1024, 210, 331, 1582, 60, 1997, 156, 389, 1, 1126, 2, 132, 2208, 908, 61, 718, 30, 968, 239, 0, 1125, 66, 260, 2290, 190, 249] payload_rows_per_dst=[7582, 12954, 13832, 7574, 8194, 8398, 8410, 11320] dup_rows_per_dst=[910, 2396, 6922, 778, 1472, 2886, 1644, 3032] combine_rows_per_src=[7582, 12954, 13832, 7574, 8194, 8398, 8410, 11320] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=271278080 dispatch_meta_bytes=336256 combine_payload_bytes=271278080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271614336 ingress_bytes=305881088 link_bytes=305881088 token_bytes=542892416 weight_movement_bytes=34603008 total_nvlink_bytes=577495424 weight_to_token_ratio=0.063738 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=27776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.151005909e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=264773632 dispatch_meta_bytes=331472 combine_payload_bytes=264773632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265105104 ingress_bytes=299376640 link_bytes=299376640 token_bytes=529878736 weight_movement_bytes=34603008 total_nvlink_bytes=564481744 weight_to_token_ratio=0.065304 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=36608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126530195e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=282165248 dispatch_meta_bytes=349608 combine_payload_bytes=282165248 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=282514856 ingress_bytes=420577280 link_bytes=420577280 token_bytes=564680104 weight_movement_bytes=138412032 total_nvlink_bytes=703092136 weight_to_token_ratio=0.245116 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=21120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.582598445e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=296894464 dispatch_meta_bytes=348640 combine_payload_bytes=296894464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=297243104 ingress_bytes=331497472 link_bytes=331497472 token_bytes=594137568 weight_movement_bytes=34603008 total_nvlink_bytes=628740576 weight_to_token_ratio=0.058241 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.247398299e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=270458880 dispatch_meta_bytes=326184 combine_payload_bytes=270458880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270785064 ingress_bytes=305061888 link_bytes=305061888 token_bytes=541243944 weight_movement_bytes=34603008 total_nvlink_bytes=575846952 weight_to_token_ratio=0.063932 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=15488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.147923325e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=272343040 dispatch_meta_bytes=327176 combine_payload_bytes=272343040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272670216 ingress_bytes=306946048 link_bytes=306946048 token_bytes=545013256 weight_movement_bytes=34603008 total_nvlink_bytes=579616264 weight_to_token_ratio=0.063490 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=16896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.155013267e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=333728 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266672032 ingress_bytes=300941312 link_bytes=300941312 token_bytes=533010336 weight_movement_bytes=34603008 total_nvlink_bytes=567613344 weight_to_token_ratio=0.064920 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=26880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132417929e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76232 payload_rows_remote=69776 dup_rows=22072 dup_rows_remote=20956 dedup_ratio=0.224528 router_tokens_per_expert=[1, 376, 8, 157, 47, 32, 1575, 4403, 0, 741, 351, 1449, 2110, 168, 287, 159, 3103, 1271, 961, 1141, 118, 4, 4123, 291, 454, 1425, 4283, 1043, 701, 351, 190, 32, 279, 427, 1309, 151, 4143, 488, 673, 190, 304, 331, 3, 142, 1250, 2112, 106, 44, 429, 1279, 208, 41, 54, 103, 552, 8, 1269, 94, 34, 64, 9, 626, 56, 1019] payload_rows_per_dst=[10824, 10880, 6456, 7610, 11454, 7370, 10534, 11104] dup_rows_per_dst=[2374, 5856, 1116, 1812, 3866, 1214, 3060, 2774] combine_rows_per_src=[10824, 10880, 6456, 7610, 11454, 7370, 10534, 11104] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77948 payload_rows_remote=64704 dup_rows=20356 dup_rows_remote=14678 dedup_ratio=0.207072 router_tokens_per_expert=[827, 95, 1430, 20, 533, 226, 495, 430, 193, 128, 105, 61, 351, 13, 598, 199, 5405, 247, 230, 14, 320, 139, 0, 3553, 267, 1, 2200, 2800, 896, 1128, 6053, 90, 659, 1007, 23, 69, 1168, 351, 1044, 208, 343, 1828, 62, 2127, 139, 375, 5, 1186, 4, 141, 2284, 804, 74, 700, 30, 1043, 268, 0, 911, 129, 317, 2379, 244, 183] payload_rows_per_dst=[13244, 12802, 9586, 9410, 7724, 8754, 8642, 7786] dup_rows_per_dst=[5678, 2600, 1602, 3172, 1334, 3376, 1518, 1076] combine_rows_per_src=[13244, 12802, 9586, 9410, 7724, 8754, 8642, 7786] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=255606784 dispatch_meta_bytes=336616 combine_payload_bytes=255606784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=255943400 ingress_bytes=290209792 link_bytes=290209792 token_bytes=511550184 weight_movement_bytes=34603008 total_nvlink_bytes=546153192 weight_to_token_ratio=0.067643 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.092036083e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78264 payload_rows_remote=70690 dup_rows=20040 dup_rows_remote=19262 dedup_ratio=0.203857 router_tokens_per_expert=[868, 49, 1292, 14, 613, 215, 822, 373, 116, 116, 94, 62, 353, 38, 568, 196, 5157, 348, 272, 2, 372, 105, 1, 4120, 216, 0, 2049, 2757, 868, 1006, 6132, 37, 735, 982, 16, 60, 1428, 378, 1024, 210, 331, 1582, 60, 1997, 156, 389, 1, 1126, 2, 132, 2208, 908, 61, 718, 30, 968, 239, 0, 1125, 66, 260, 2290, 190, 249] payload_rows_per_dst=[7582, 12954, 13832, 7574, 8194, 8398, 8410, 11320] dup_rows_per_dst=[910, 2396, 6922, 778, 1472, 2886, 1644, 3032] combine_rows_per_src=[7582, 12954, 13832, 7574, 8194, 8398, 8410, 11320] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78324 payload_rows_remote=69360 dup_rows=19980 dup_rows_remote=17884 dedup_ratio=0.203247 router_tokens_per_expert=[797, 54, 1449, 26, 572, 205, 662, 472, 90, 84, 76, 50, 337, 17, 497, 204, 5699, 342, 228, 3, 556, 103, 0, 3904, 267, 0, 1945, 2669, 796, 946, 6152, 82, 790, 1151, 23, 54, 1456, 250, 992, 218, 321, 1635, 44, 1931, 125, 448, 0, 1244, 2, 120, 2232, 871, 50, 631, 29, 1238, 235, 0, 1028, 89, 247, 1974, 162, 278] payload_rows_per_dst=[7490, 12798, 13818, 7328, 8964, 8464, 8786, 10676] dup_rows_per_dst=[984, 2216, 6660, 744, 2096, 3032, 1560, 2688] combine_rows_per_src=[7490, 12798, 13818, 7328, 8964, 8464, 8786, 10676] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=289546240 dispatch_meta_bytes=359808 combine_payload_bytes=289546240 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=289906048 ingress_bytes=427958272 link_bytes=427958272 token_bytes=579452288 weight_movement_bytes=138412032 total_nvlink_bytes=717864320 weight_to_token_ratio=0.238867 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=12288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.610372524e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76220 payload_rows_remote=66156 dup_rows=22084 dup_rows_remote=18826 dedup_ratio=0.224650 router_tokens_per_expert=[830, 51, 1273, 19, 490, 215, 710, 459, 225, 103, 77, 57, 330, 20, 445, 87, 5314, 384, 265, 1, 417, 139, 0, 3869, 208, 2, 2064, 2640, 1061, 1373, 6055, 61, 735, 1035, 28, 38, 1220, 230, 1043, 242, 380, 1710, 62, 2162, 168, 471, 3, 906, 1, 160, 2228, 810, 81, 692, 32, 1239, 269, 0, 1042, 76, 349, 2073, 240, 183] payload_rows_per_dst=[7266, 2864, 9050, 13774, 12214, 10050, 10064, 10938] dup_rows_per_dst=[828, 130, 1154, 5158, 4666, 4084, 3258, 2806] combine_rows_per_src=[7266, 2864, 9050, 13774, 12214, 10050, 10064, 10938] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77242 payload_rows_remote=64642 dup_rows=21062 dup_rows_remote=18226 dedup_ratio=0.214254 router_tokens_per_expert=[784, 58, 1680, 15, 593, 210, 574, 493, 217, 108, 86, 47, 307, 8, 732, 184, 5628, 327, 184, 3, 380, 91, 0, 4062, 237, 0, 2041, 2742, 1009, 1061, 6029, 64, 605, 996, 29, 39, 1374, 299, 998, 199, 331, 1631, 37, 2038, 102, 409, 0, 1106, 1, 135, 2156, 769, 40, 697, 29, 1311, 219, 0, 888, 87, 141, 2096, 187, 249] payload_rows_per_dst=[13688, 12600, 9134, 9940, 7844, 8368, 8720, 6948] dup_rows_per_dst=[6382, 2836, 960, 4368, 1234, 2940, 1556, 786] combine_rows_per_src=[13688, 12600, 9134, 9940, 7844, 8368, 8720, 6948] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80684 payload_rows_remote=72484 dup_rows=17620 dup_rows_remote=14676 dedup_ratio=0.179240 router_tokens_per_expert=[845, 68, 1463, 33, 709, 269, 604, 502, 141, 121, 102, 76, 317, 19, 586, 290, 5486, 305, 189, 4, 275, 116, 0, 3907, 250, 0, 1918, 2661, 852, 1179, 6022, 56, 768, 1135, 23, 42, 1180, 301, 1028, 200, 427, 1729, 52, 1735, 61, 365, 5, 1198, 1, 154, 2289, 1129, 81, 746, 23, 981, 321, 0, 817, 88, 228, 2301, 194, 185] payload_rows_per_dst=[7898, 12796, 11714, 7680, 12540, 8200, 9066, 10790] dup_rows_per_dst=[1088, 2552, 1036, 830, 4628, 2944, 1742, 2800] combine_rows_per_src=[7898, 12796, 11714, 7680, 12540, 8200, 9066, 10790] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73590 payload_rows_remote=62404 dup_rows=24714 dup_rows_remote=21750 dedup_ratio=0.251404 router_tokens_per_expert=[860, 67, 1355, 22, 661, 257, 426, 516, 236, 115, 70, 55, 333, 21, 489, 240, 5671, 348, 199, 7, 290, 122, 1, 3877, 204, 0, 2113, 2517, 915, 1170, 5967, 70, 549, 881, 8, 43, 1182, 223, 1123, 190, 448, 1717, 52, 1849, 71, 472, 5, 1033, 0, 156, 2490, 1055, 54, 535, 35, 1229, 265, 0, 1100, 113, 222, 2393, 215, 250] payload_rows_per_dst=[7476, 3026, 1836, 13248, 11976, 10680, 14162, 11186] dup_rows_per_dst=[852, 92, 98, 3404, 4176, 4840, 8288, 2964] combine_rows_per_src=[7476, 3026, 1836, 13248, 11976, 10680, 14162, 11186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=285802496 dispatch_meta_bytes=362928 combine_payload_bytes=285802496 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=286165424 ingress_bytes=424214528 link_bytes=424214528 token_bytes=571967920 weight_movement_bytes=138412032 total_nvlink_bytes=710379952 weight_to_token_ratio=0.241993 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=12288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.596285117e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=289546240 dispatch_meta_bytes=359808 combine_payload_bytes=289546240 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=289906048 ingress_bytes=427958272 link_bytes=427958272 token_bytes=579452288 weight_movement_bytes=138412032 total_nvlink_bytes=717864320 weight_to_token_ratio=0.238867 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=12288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.610372524e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265027584 dispatch_meta_bytes=317528 combine_payload_bytes=265027584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265345112 ingress_bytes=299630592 link_bytes=299630592 token_bytes=530372696 weight_movement_bytes=34603008 total_nvlink_bytes=564975704 weight_to_token_ratio=0.065243 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=15360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127485795e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=284098560 dispatch_meta_bytes=348976 combine_payload_bytes=284098560 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=284447536 ingress_bytes=318701568 link_bytes=318701568 token_bytes=568546096 weight_movement_bytes=34603008 total_nvlink_bytes=603149104 weight_to_token_ratio=0.060862 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=12288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.199248343e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76220 payload_rows_remote=66156 dup_rows=22084 dup_rows_remote=18826 dedup_ratio=0.224650 router_tokens_per_expert=[830, 51, 1273, 19, 490, 215, 710, 459, 225, 103, 77, 57, 330, 20, 445, 87, 5314, 384, 265, 1, 417, 139, 0, 3869, 208, 2, 2064, 2640, 1061, 1373, 6055, 61, 735, 1035, 28, 38, 1220, 230, 1043, 242, 380, 1710, 62, 2162, 168, 471, 3, 906, 1, 160, 2228, 810, 81, 692, 32, 1239, 269, 0, 1042, 76, 349, 2073, 240, 183] payload_rows_per_dst=[7266, 2864, 9050, 13774, 12214, 10050, 10064, 10938] dup_rows_per_dst=[828, 130, 1154, 5158, 4666, 4084, 3258, 2806] combine_rows_per_src=[7266, 2864, 9050, 13774, 12214, 10050, 10064, 10938] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83878 payload_rows_remote=70830 dup_rows=14426 dup_rows_remote=12962 dedup_ratio=0.146749 router_tokens_per_expert=[1, 192, 1514, 0, 12, 347, 11, 346, 1, 421, 474, 186, 0, 0, 10, 98, 0, 1048, 0, 7298, 611, 170, 416, 854, 77, 0, 321, 4736, 77, 6066, 54, 0, 0, 2, 0, 7490, 2257, 3, 50, 581, 9, 1, 17, 1145, 264, 0, 2819, 47, 0, 56, 1253, 32, 4, 358, 1185, 1265, 11, 660, 0, 7, 4203, 81, 3, 8] payload_rows_per_dst=[15412, 13048, 5682, 9910, 15794, 7876, 6850, 9306] dup_rows_per_dst=[4030, 1464, 516, 620, 4972, 728, 1456, 640] combine_rows_per_src=[15412, 13048, 5682, 9910, 15794, 7876, 6850, 9306] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82286 payload_rows_remote=76372 dup_rows=16018 dup_rows_remote=15500 dedup_ratio=0.162944 router_tokens_per_expert=[5, 230, 1727, 2, 21, 353, 20, 468, 5, 576, 526, 362, 0, 0, 18, 63, 0, 1046, 0, 7228, 474, 127, 437, 1132, 75, 0, 316, 4870, 95, 4901, 50, 1, 0, 2, 0, 7281, 2310, 13, 87, 787, 15, 10, 27, 1392, 267, 0, 2929, 70, 1, 94, 834, 51, 8, 370, 1356, 1213, 18, 718, 2, 16, 4028, 101, 8, 16] payload_rows_per_dst=[15332, 11444, 5914, 10084, 15616, 8488, 6322, 9086] dup_rows_per_dst=[4776, 1458, 518, 730, 5344, 932, 1532, 728] combine_rows_per_src=[15332, 11444, 5914, 10084, 15616, 8488, 6322, 9086] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80486 payload_rows_remote=67142 dup_rows=17818 dup_rows_remote=12466 dedup_ratio=0.181254 router_tokens_per_expert=[3, 269, 1554, 0, 20, 415, 26, 389, 1, 465, 476, 201, 0, 0, 4, 90, 0, 1115, 0, 7288, 562, 172, 363, 906, 85, 1, 345, 4593, 129, 5546, 45, 0, 0, 4, 0, 7561, 2355, 3, 66, 590, 11, 7, 10, 1122, 322, 0, 2489, 68, 0, 55, 1415, 23, 5, 330, 1245, 1261, 8, 720, 1, 5, 4338, 62, 9, 4] payload_rows_per_dst=[5042, 12162, 15198, 9648, 5604, 13344, 9880, 9608] dup_rows_per_dst=[310, 1404, 5614, 748, 432, 5352, 3272, 686] combine_rows_per_src=[5042, 12162, 15198, 9648, 5604, 13344, 9880, 9608] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73338 payload_rows_remote=57498 dup_rows=24966 dup_rows_remote=15814 dedup_ratio=0.253967 router_tokens_per_expert=[0, 420, 1432, 1, 11, 446, 15, 403, 0, 444, 537, 226, 0, 0, 4, 88, 1, 1085, 0, 7543, 574, 285, 470, 1010, 94, 1, 425, 4599, 80, 5572, 44, 1, 0, 3, 0, 7422, 2393, 6, 67, 583, 14, 7, 34, 1138, 287, 0, 2610, 45, 1, 44, 767, 44, 8, 341, 1400, 1174, 18, 560, 2, 8, 4295, 57, 4, 9] payload_rows_per_dst=[5246, 2490, 6192, 14484, 5866, 7596, 15624, 15840] dup_rows_per_dst=[210, 108, 658, 6886, 500, 674, 6778, 9152] combine_rows_per_src=[5246, 2490, 6192, 14484, 5866, 7596, 15624, 15840] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=270974976 dispatch_meta_bytes=339928 combine_payload_bytes=270974976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271314904 ingress_bytes=305577984 link_bytes=305577984 token_bytes=542289880 weight_movement_bytes=34603008 total_nvlink_bytes=576892888 weight_to_token_ratio=0.063809 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.149865353e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=264773632 dispatch_meta_bytes=331472 combine_payload_bytes=264773632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265105104 ingress_bytes=299376640 link_bytes=299376640 token_bytes=529878736 weight_movement_bytes=34603008 total_nvlink_bytes=564481744 weight_to_token_ratio=0.065304 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=36608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126530195e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=296894464 dispatch_meta_bytes=348640 combine_payload_bytes=296894464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=297243104 ingress_bytes=331497472 link_bytes=331497472 token_bytes=594137568 weight_movement_bytes=34603008 total_nvlink_bytes=628740576 weight_to_token_ratio=0.058241 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.247398299e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=255606784 dispatch_meta_bytes=336616 combine_payload_bytes=255606784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=255943400 ingress_bytes=290209792 link_bytes=290209792 token_bytes=511550184 weight_movement_bytes=34603008 total_nvlink_bytes=546153192 weight_to_token_ratio=0.067643 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.092036083e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77962 payload_rows_remote=68888 dup_rows=20342 dup_rows_remote=18514 dedup_ratio=0.206930 router_tokens_per_expert=[815, 71, 1322, 35, 680, 283, 453, 502, 273, 127, 127, 130, 317, 24, 669, 214, 5138, 320, 220, 8, 257, 157, 0, 3659, 290, 2, 2327, 2812, 852, 1132, 5940, 87, 709, 1015, 17, 47, 1440, 184, 979, 230, 370, 1922, 83, 1827, 43, 326, 6, 1067, 4, 128, 2278, 1289, 72, 598, 33, 1042, 295, 0, 807, 82, 248, 2303, 197, 268] payload_rows_per_dst=[12044, 12638, 9074, 8844, 7902, 8220, 9126, 10114] dup_rows_per_dst=[4894, 3004, 1828, 2052, 1340, 3068, 1762, 2394] combine_rows_per_src=[12044, 12638, 9074, 8844, 7902, 8220, 9126, 10114] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82222 payload_rows_remote=72056 dup_rows=16082 dup_rows_remote=15248 dedup_ratio=0.163595 router_tokens_per_expert=[3, 232, 1787, 0, 20, 447, 18, 451, 0, 509, 441, 223, 1, 0, 13, 57, 0, 1179, 0, 7335, 571, 183, 306, 797, 63, 0, 399, 4841, 90, 5560, 105, 2, 0, 7, 0, 7453, 2221, 6, 57, 573, 7, 4, 18, 1098, 390, 0, 2757, 75, 0, 46, 989, 57, 10, 327, 1333, 1207, 7, 704, 0, 13, 4064, 76, 7, 13] payload_rows_per_dst=[13414, 12082, 7610, 10166, 13766, 9482, 6562, 9140] dup_rows_per_dst=[4142, 1526, 1492, 834, 4024, 2060, 1376, 628] combine_rows_per_src=[13414, 12082, 7610, 10166, 13766, 9482, 6562, 9140] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=290119680 dispatch_meta_bytes=335168 combine_payload_bytes=290119680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=290454848 ingress_bytes=324722688 link_bytes=324722688 token_bytes=580574528 weight_movement_bytes=34603008 total_nvlink_bytes=615177536 weight_to_token_ratio=0.059601 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=38656 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.221905333e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82560 payload_rows_remote=67272 dup_rows=15744 dup_rows_remote=10956 dedup_ratio=0.160156 router_tokens_per_expert=[12, 308, 1649, 7, 22, 487, 19, 394, 3, 602, 554, 316, 3, 0, 23, 55, 0, 1040, 0, 7140, 481, 124, 521, 925, 60, 0, 493, 4726, 106, 5673, 54, 0, 0, 5, 0, 7373, 1892, 11, 56, 572, 33, 3, 45, 1209, 347, 0, 2855, 65, 0, 57, 841, 37, 10, 391, 1463, 1259, 16, 652, 0, 17, 4014, 117, 7, 8] payload_rows_per_dst=[15288, 12566, 8922, 10090, 11854, 8272, 6590, 8978] dup_rows_per_dst=[4788, 1892, 2146, 788, 3078, 842, 1526, 684] combine_rows_per_src=[15288, 12566, 8922, 10090, 11854, 8272, 6590, 8978] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=312819712 dispatch_meta_bytes=367488 combine_payload_bytes=312819712 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=313187200 ingress_bytes=416628736 link_bytes=416628736 token_bytes=626006912 weight_movement_bytes=103809024 total_nvlink_bytes=729815936 weight_to_token_ratio=0.165827 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.567740392e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=275013632 dispatch_meta_bytes=318432 combine_payload_bytes=275013632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275332064 ingress_bytes=309616640 link_bytes=309616640 token_bytes=550345696 weight_movement_bytes=34603008 total_nvlink_bytes=584948704 weight_to_token_ratio=0.062875 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=22016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.165062490e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79360 payload_rows_remote=63620 dup_rows=18944 dup_rows_remote=11772 dedup_ratio=0.192708 router_tokens_per_expert=[8, 311, 1190, 3, 24, 469, 23, 490, 4, 392, 577, 215, 2, 0, 25, 75, 0, 1272, 0, 7184, 523, 251, 416, 938, 59, 2, 442, 4680, 99, 5802, 70, 2, 2, 1, 0, 7548, 2219, 8, 73, 463, 18, 5, 29, 1202, 390, 0, 2703, 84, 0, 48, 1174, 52, 7, 333, 1124, 1170, 22, 619, 0, 34, 4159, 83, 21, 13] payload_rows_per_dst=[4762, 11226, 12602, 10744, 5262, 8120, 15740, 10904] dup_rows_per_dst=[274, 1400, 4416, 1522, 270, 742, 7172, 3148] combine_rows_per_src=[4762, 11226, 12602, 10744, 5262, 8120, 15740, 10904] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80282 payload_rows_remote=74716 dup_rows=18022 dup_rows_remote=17666 dedup_ratio=0.183329 router_tokens_per_expert=[6, 199, 1664, 1, 9, 369, 10, 381, 0, 392, 499, 176, 0, 0, 12, 75, 0, 1189, 0, 7261, 614, 194, 346, 808, 94, 0, 330, 4977, 70, 5863, 54, 0, 0, 2, 0, 7592, 2328, 7, 66, 558, 4, 3, 21, 1161, 267, 0, 2808, 42, 0, 42, 1161, 29, 9, 305, 1143, 1123, 9, 628, 0, 9, 4155, 66, 3, 18] payload_rows_per_dst=[4970, 12668, 15242, 10364, 5566, 15826, 6478, 9168] dup_rows_per_dst=[308, 1366, 5582, 686, 356, 7970, 1146, 608] combine_rows_per_src=[4970, 12668, 15242, 10364, 5566, 15826, 6478, 9168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83878 payload_rows_remote=70830 dup_rows=14426 dup_rows_remote=12962 dedup_ratio=0.146749 router_tokens_per_expert=[1, 192, 1514, 0, 12, 347, 11, 346, 1, 421, 474, 186, 0, 0, 10, 98, 0, 1048, 0, 7298, 611, 170, 416, 854, 77, 0, 321, 4736, 77, 6066, 54, 0, 0, 2, 0, 7490, 2257, 3, 50, 581, 9, 1, 17, 1145, 264, 0, 2819, 47, 0, 56, 1253, 32, 4, 358, 1185, 1265, 11, 660, 0, 7, 4203, 81, 3, 8] payload_rows_per_dst=[15412, 13048, 5682, 9910, 15794, 7876, 6850, 9306] dup_rows_per_dst=[4030, 1464, 516, 620, 4972, 728, 1456, 640] combine_rows_per_src=[15412, 13048, 5682, 9910, 15794, 7876, 6850, 9306] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80486 payload_rows_remote=67142 dup_rows=17818 dup_rows_remote=12466 dedup_ratio=0.181254 router_tokens_per_expert=[3, 269, 1554, 0, 20, 415, 26, 389, 1, 465, 476, 201, 0, 0, 4, 90, 0, 1115, 0, 7288, 562, 172, 363, 906, 85, 1, 345, 4593, 129, 5546, 45, 0, 0, 4, 0, 7561, 2355, 3, 66, 590, 11, 7, 10, 1122, 322, 0, 2489, 68, 0, 55, 1415, 23, 5, 330, 1245, 1261, 8, 720, 1, 5, 4338, 62, 9, 4] payload_rows_per_dst=[5042, 12162, 15198, 9648, 5604, 13344, 9880, 9608] dup_rows_per_dst=[310, 1404, 5614, 748, 432, 5352, 3272, 686] combine_rows_per_src=[5042, 12162, 15198, 9648, 5604, 13344, 9880, 9608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=282165248 dispatch_meta_bytes=349608 combine_payload_bytes=282165248 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=282514856 ingress_bytes=420577280 link_bytes=420577280 token_bytes=564680104 weight_movement_bytes=138412032 total_nvlink_bytes=703092136 weight_to_token_ratio=0.245116 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=21120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.582598445e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=235511808 dispatch_meta_bytes=293248 combine_payload_bytes=235511808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=235805056 ingress_bytes=270114816 link_bytes=270114816 token_bytes=471316864 weight_movement_bytes=34603008 total_nvlink_bytes=505919872 weight_to_token_ratio=0.073418 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=33280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.016420307e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=295141376 dispatch_meta_bytes=349216 combine_payload_bytes=295141376 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=295490592 ingress_bytes=364347392 link_bytes=364347392 token_bytes=590631968 weight_movement_bytes=69206016 total_nvlink_bytes=659837984 weight_to_token_ratio=0.117173 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=38016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.371009904e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=275546112 dispatch_meta_bytes=312912 combine_payload_bytes=275546112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275859024 ingress_bytes=310149120 link_bytes=310149120 token_bytes=551405136 weight_movement_bytes=34603008 total_nvlink_bytes=586008144 weight_to_token_ratio=0.062754 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=27520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.167066170e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=306036736 dispatch_meta_bytes=369528 combine_payload_bytes=306036736 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=306406264 ingress_bytes=444448768 link_bytes=444448768 token_bytes=612443000 weight_movement_bytes=138412032 total_nvlink_bytes=750855032 weight_to_token_ratio=0.226000 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=25856 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.672424933e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73420 payload_rows_remote=62318 dup_rows=24884 dup_rows_remote=21408 dedup_ratio=0.253133 router_tokens_per_expert=[0, 582, 1948, 0, 439, 23, 11, 314, 71, 0, 4, 1193, 0, 3, 2444, 1291, 5884, 7852, 22, 4, 5172, 2283, 2355, 0, 0, 29, 264, 1, 0, 0, 34, 0, 2, 0, 1, 2, 0, 0, 2991, 179, 7066, 4, 164, 1, 0, 3, 0, 1498, 4, 11, 267, 3449, 4, 15, 0, 646, 0, 16, 0, 6, 0, 490, 0, 110] payload_rows_per_dst=[15130, 11102, 16114, 656, 6304, 3266, 8710, 12138] dup_rows_per_dst=[5636, 3476, 13828, 0, 46, 74, 950, 874] combine_rows_per_src=[15130, 11102, 16114, 656, 6304, 3266, 8710, 12138] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79360 payload_rows_remote=63620 dup_rows=18944 dup_rows_remote=11772 dedup_ratio=0.192708 router_tokens_per_expert=[8, 311, 1190, 3, 24, 469, 23, 490, 4, 392, 577, 215, 2, 0, 25, 75, 0, 1272, 0, 7184, 523, 251, 416, 938, 59, 2, 442, 4680, 99, 5802, 70, 2, 2, 1, 0, 7548, 2219, 8, 73, 463, 18, 5, 29, 1202, 390, 0, 2703, 84, 0, 48, 1174, 52, 7, 333, 1124, 1170, 22, 619, 0, 34, 4159, 83, 21, 13] payload_rows_per_dst=[4762, 11226, 12602, 10744, 5262, 8120, 15740, 10904] dup_rows_per_dst=[274, 1400, 4416, 1522, 270, 742, 7172, 3148] combine_rows_per_src=[4762, 11226, 12602, 10744, 5262, 8120, 15740, 10904] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82078 payload_rows_remote=72584 dup_rows=16226 dup_rows_remote=14904 dedup_ratio=0.165059 router_tokens_per_expert=[11, 680, 1943, 0, 560, 26, 45, 432, 82, 0, 8, 1454, 1, 2, 2187, 1669, 5636, 7768, 5, 6, 4431, 2103, 2090, 0, 0, 46, 336, 0, 0, 1, 39, 0, 6, 0, 0, 1, 0, 3, 3084, 322, 6955, 6, 229, 2, 1, 5, 0, 1583, 16, 15, 253, 3256, 2, 65, 0, 926, 2, 27, 0, 23, 0, 652, 0, 157] payload_rows_per_dst=[15034, 11062, 9494, 13294, 6922, 3560, 10908, 11804] dup_rows_per_dst=[6270, 3942, 1322, 672, 400, 92, 2338, 1190] combine_rows_per_src=[15034, 11062, 9494, 13294, 6922, 3560, 10908, 11804] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84984 payload_rows_remote=70776 dup_rows=13320 dup_rows_remote=10394 dedup_ratio=0.135498 router_tokens_per_expert=[7, 707, 2011, 0, 444, 24, 33, 417, 124, 0, 10, 1361, 0, 3, 2478, 1577, 5939, 7814, 12, 4, 4734, 2064, 2075, 0, 0, 42, 363, 0, 0, 0, 35, 0, 11, 0, 0, 1, 0, 1, 2773, 178, 6698, 6, 268, 0, 0, 10, 0, 1585, 14, 14, 248, 3203, 8, 26, 0, 1076, 1, 18, 0, 8, 0, 615, 0, 112] payload_rows_per_dst=[6572, 8982, 4150, 15652, 12258, 14208, 10878, 12284] dup_rows_per_dst=[714, 2124, 10, 856, 3138, 2926, 2450, 1102] combine_rows_per_src=[6572, 8982, 4150, 15652, 12258, 14208, 10878, 12284] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85082 payload_rows_remote=72288 dup_rows=13222 dup_rows_remote=12056 dedup_ratio=0.134501 router_tokens_per_expert=[1, 683, 1748, 0, 489, 26, 20, 455, 44, 0, 4, 1201, 0, 4, 2305, 1244, 6193, 7890, 12, 5, 4837, 2171, 2096, 0, 0, 23, 328, 0, 0, 1, 21, 0, 8, 0, 2, 1, 0, 2, 3096, 182, 6831, 11, 258, 0, 0, 13, 0, 1734, 12, 5, 245, 3236, 6, 34, 0, 888, 0, 31, 0, 13, 0, 619, 0, 124] payload_rows_per_dst=[6266, 8184, 4358, 15784, 12508, 14458, 10730, 12794] dup_rows_per_dst=[578, 1420, 18, 742, 3748, 3236, 2314, 1166] combine_rows_per_src=[6266, 8184, 4358, 15784, 12508, 14458, 10730, 12794] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85218 payload_rows_remote=69334 dup_rows=13086 dup_rows_remote=12318 dedup_ratio=0.133118 router_tokens_per_expert=[0, 654, 2125, 0, 438, 31, 45, 277, 122, 0, 12, 1312, 0, 1, 2373, 1133, 6031, 7939, 19, 3, 4360, 2280, 2566, 0, 1, 33, 321, 0, 0, 0, 32, 0, 10, 0, 0, 2, 0, 1, 2613, 315, 6810, 4, 212, 1, 0, 14, 1, 1710, 15, 9, 237, 3616, 6, 37, 0, 696, 6, 26, 1, 29, 0, 549, 0, 124] payload_rows_per_dst=[7226, 8526, 4654, 15884, 11462, 13372, 11624, 12470] dup_rows_per_dst=[1166, 1380, 20, 768, 3140, 2810, 2740, 1062] combine_rows_per_src=[7226, 8526, 4654, 15884, 11462, 13372, 11624, 12470] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=255254528 dispatch_meta_bytes=334904 combine_payload_bytes=255254528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=255589432 ingress_bytes=289857536 link_bytes=289857536 token_bytes=510843960 weight_movement_bytes=34603008 total_nvlink_bytes=545446968 weight_to_token_ratio=0.067737 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=19968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.090710573e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73338 payload_rows_remote=57498 dup_rows=24966 dup_rows_remote=15814 dedup_ratio=0.253967 router_tokens_per_expert=[0, 420, 1432, 1, 11, 446, 15, 403, 0, 444, 537, 226, 0, 0, 4, 88, 1, 1085, 0, 7543, 574, 285, 470, 1010, 94, 1, 425, 4599, 80, 5572, 44, 1, 0, 3, 0, 7422, 2393, 6, 67, 583, 14, 7, 34, 1138, 287, 0, 2610, 45, 1, 44, 767, 44, 8, 341, 1400, 1174, 18, 560, 2, 8, 4295, 57, 4, 9] payload_rows_per_dst=[5246, 2490, 6192, 14484, 5866, 7596, 15624, 15840] dup_rows_per_dst=[210, 108, 658, 6886, 500, 674, 6778, 9152] combine_rows_per_src=[5246, 2490, 6192, 14484, 5866, 7596, 15624, 15840] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=295141376 dispatch_meta_bytes=349216 combine_payload_bytes=295141376 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=295490592 ingress_bytes=364347392 link_bytes=364347392 token_bytes=590631968 weight_movement_bytes=69206016 total_nvlink_bytes=659837984 weight_to_token_ratio=0.117173 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=38016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.371009904e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=260587520 dispatch_meta_bytes=301568 combine_payload_bytes=260587520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260889088 ingress_bytes=295190528 link_bytes=295190528 token_bytes=521476608 weight_movement_bytes=34603008 total_nvlink_bytes=556079616 weight_to_token_ratio=0.066356 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.110778192e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=275546112 dispatch_meta_bytes=312912 combine_payload_bytes=275546112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275859024 ingress_bytes=310149120 link_bytes=310149120 token_bytes=551405136 weight_movement_bytes=34603008 total_nvlink_bytes=586008144 weight_to_token_ratio=0.062754 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=27520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.167066170e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=306036736 dispatch_meta_bytes=369528 combine_payload_bytes=306036736 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=306406264 ingress_bytes=444448768 link_bytes=444448768 token_bytes=612443000 weight_movement_bytes=138412032 total_nvlink_bytes=750855032 weight_to_token_ratio=0.226000 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=25856 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.672424933e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=290119680 dispatch_meta_bytes=335168 combine_payload_bytes=290119680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=290454848 ingress_bytes=324722688 link_bytes=324722688 token_bytes=580574528 weight_movement_bytes=34603008 total_nvlink_bytes=615177536 weight_to_token_ratio=0.059601 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=38656 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.221905333e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=275013632 dispatch_meta_bytes=318432 combine_payload_bytes=275013632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275332064 ingress_bytes=309616640 link_bytes=309616640 token_bytes=550345696 weight_movement_bytes=34603008 total_nvlink_bytes=584948704 weight_to_token_ratio=0.062875 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=22016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.165062490e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=235511808 dispatch_meta_bytes=293248 combine_payload_bytes=235511808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=235805056 ingress_bytes=270114816 link_bytes=270114816 token_bytes=471316864 weight_movement_bytes=34603008 total_nvlink_bytes=505919872 weight_to_token_ratio=0.073418 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=33280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.016420307e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82286 payload_rows_remote=76372 dup_rows=16018 dup_rows_remote=15500 dedup_ratio=0.162944 router_tokens_per_expert=[5, 230, 1727, 2, 21, 353, 20, 468, 5, 576, 526, 362, 0, 0, 18, 63, 0, 1046, 0, 7228, 474, 127, 437, 1132, 75, 0, 316, 4870, 95, 4901, 50, 1, 0, 2, 0, 7281, 2310, 13, 87, 787, 15, 10, 27, 1392, 267, 0, 2929, 70, 1, 94, 834, 51, 8, 370, 1356, 1213, 18, 718, 2, 16, 4028, 101, 8, 16] payload_rows_per_dst=[15332, 11444, 5914, 10084, 15616, 8488, 6322, 9086] dup_rows_per_dst=[4776, 1458, 518, 730, 5344, 932, 1532, 728] combine_rows_per_src=[15332, 11444, 5914, 10084, 15616, 8488, 6322, 9086] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83994 payload_rows_remote=73466 dup_rows=14310 dup_rows_remote=12036 dedup_ratio=0.145569 router_tokens_per_expert=[5, 593, 1623, 0, 402, 26, 13, 362, 117, 0, 13, 1470, 1, 10, 3040, 1476, 5589, 7832, 17, 4, 4951, 1992, 2180, 0, 0, 27, 366, 0, 0, 1, 55, 0, 14, 0, 0, 1, 0, 1, 3021, 257, 6623, 7, 274, 1, 0, 17, 0, 1839, 7, 9, 342, 3069, 3, 28, 0, 763, 2, 40, 0, 11, 1, 517, 1, 139] payload_rows_per_dst=[5582, 9674, 4014, 15686, 12744, 14124, 10528, 11642] dup_rows_per_dst=[466, 2580, 12, 876, 3746, 3398, 2274, 958] combine_rows_per_src=[5582, 9674, 4014, 15686, 12744, 14124, 10528, 11642] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85218 payload_rows_remote=69334 dup_rows=13086 dup_rows_remote=12318 dedup_ratio=0.133118 router_tokens_per_expert=[0, 654, 2125, 0, 438, 31, 45, 277, 122, 0, 12, 1312, 0, 1, 2373, 1133, 6031, 7939, 19, 3, 4360, 2280, 2566, 0, 1, 33, 321, 0, 0, 0, 32, 0, 10, 0, 0, 2, 0, 1, 2613, 315, 6810, 4, 212, 1, 0, 14, 1, 1710, 15, 9, 237, 3616, 6, 37, 0, 696, 6, 26, 1, 29, 0, 549, 0, 124] payload_rows_per_dst=[7226, 8526, 4654, 15884, 11462, 13372, 11624, 12470] dup_rows_per_dst=[1166, 1380, 20, 768, 3140, 2810, 2740, 1062] combine_rows_per_src=[7226, 8526, 4654, 15884, 11462, 13372, 11624, 12470] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=67268 payload_rows_remote=52278 dup_rows=31036 dup_rows_remote=25078 dedup_ratio=0.315715 router_tokens_per_expert=[6, 636, 2072, 0, 485, 37, 19, 341, 131, 0, 16, 1334, 0, 2, 2404, 1459, 5598, 7857, 31, 6, 4444, 2185, 2222, 0, 0, 24, 361, 0, 0, 0, 39, 4, 8, 0, 4, 3, 0, 6, 3010, 352, 6878, 9, 265, 2, 0, 5, 2, 1919, 14, 9, 274, 3228, 4, 33, 0, 703, 1, 54, 5, 17, 0, 494, 1, 139] payload_rows_per_dst=[14990, 11242, 16274, 856, 6432, 4264, 7880, 5330] dup_rows_per_dst=[5958, 3820, 19636, 0, 334, 140, 650, 498] combine_rows_per_src=[14990, 11242, 16274, 856, 6432, 4264, 7880, 5330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=312819712 dispatch_meta_bytes=367488 combine_payload_bytes=312819712 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=313187200 ingress_bytes=416628736 link_bytes=416628736 token_bytes=626006912 weight_movement_bytes=103809024 total_nvlink_bytes=729815936 weight_to_token_ratio=0.165827 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.567740392e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73420 payload_rows_remote=62318 dup_rows=24884 dup_rows_remote=21408 dedup_ratio=0.253133 router_tokens_per_expert=[0, 582, 1948, 0, 439, 23, 11, 314, 71, 0, 4, 1193, 0, 3, 2444, 1291, 5884, 7852, 22, 4, 5172, 2283, 2355, 0, 0, 29, 264, 1, 0, 0, 34, 0, 2, 0, 1, 2, 0, 0, 2991, 179, 7066, 4, 164, 1, 0, 3, 0, 1498, 4, 11, 267, 3449, 4, 15, 0, 646, 0, 16, 0, 6, 0, 490, 0, 110] payload_rows_per_dst=[15130, 11102, 16114, 656, 6304, 3266, 8710, 12138] dup_rows_per_dst=[5636, 3476, 13828, 0, 46, 74, 950, 874] combine_rows_per_src=[15130, 11102, 16114, 656, 6304, 3266, 8710, 12138] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85184 payload_rows_remote=73142 dup_rows=13120 dup_rows_remote=10040 dedup_ratio=0.133464 router_tokens_per_expert=[0, 535, 2088, 0, 409, 27, 27, 272, 63, 0, 2, 1344, 0, 2, 2704, 1280, 5806, 7970, 16, 6, 4922, 2178, 2378, 0, 0, 24, 268, 0, 0, 1, 26, 0, 3, 0, 0, 2, 0, 0, 2419, 215, 7159, 4, 207, 1, 0, 16, 0, 1801, 6, 7, 250, 3275, 3, 13, 0, 733, 1, 29, 0, 11, 0, 543, 0, 106] payload_rows_per_dst=[6118, 8918, 4386, 15942, 12042, 14902, 10862, 12014] dup_rows_per_dst=[598, 1872, 14, 636, 3080, 3474, 2468, 978] combine_rows_per_src=[6118, 8918, 4386, 15942, 12042, 14902, 10862, 12014] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=297304064 dispatch_meta_bytes=349952 combine_payload_bytes=297304064 prefetch_bytes=34603008 reduce_bytes=173015040 egress_bytes=297654016 ingress_bytes=504922112 link_bytes=504922112 token_bytes=594958080 weight_movement_bytes=207618048 total_nvlink_bytes=802576128 weight_to_token_ratio=0.348962 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.899981258e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84984 payload_rows_remote=70776 dup_rows=13320 dup_rows_remote=10394 dedup_ratio=0.135498 router_tokens_per_expert=[7, 707, 2011, 0, 444, 24, 33, 417, 124, 0, 10, 1361, 0, 3, 2478, 1577, 5939, 7814, 12, 4, 4734, 2064, 2075, 0, 0, 42, 363, 0, 0, 0, 35, 0, 11, 0, 0, 1, 0, 1, 2773, 178, 6698, 6, 268, 0, 0, 10, 0, 1585, 14, 14, 248, 3203, 8, 26, 0, 1076, 1, 18, 0, 8, 0, 615, 0, 112] payload_rows_per_dst=[6572, 8982, 4150, 15652, 12258, 14208, 10878, 12284] dup_rows_per_dst=[714, 2124, 10, 856, 3138, 2926, 2450, 1102] combine_rows_per_src=[6572, 8982, 4150, 15652, 12258, 14208, 10878, 12284] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=289898496 dispatch_meta_bytes=324680 combine_payload_bytes=289898496 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=290223176 ingress_bytes=359104512 link_bytes=359104512 token_bytes=580121672 weight_movement_bytes=69206016 total_nvlink_bytes=649327688 weight_to_token_ratio=0.119296 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=33536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.351281368e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85082 payload_rows_remote=72288 dup_rows=13222 dup_rows_remote=12056 dedup_ratio=0.134501 router_tokens_per_expert=[1, 683, 1748, 0, 489, 26, 20, 455, 44, 0, 4, 1201, 0, 4, 2305, 1244, 6193, 7890, 12, 5, 4837, 2171, 2096, 0, 0, 23, 328, 0, 0, 1, 21, 0, 8, 0, 2, 1, 0, 2, 3096, 182, 6831, 11, 258, 0, 0, 13, 0, 1734, 12, 5, 245, 3236, 6, 34, 0, 888, 0, 31, 0, 13, 0, 619, 0, 124] payload_rows_per_dst=[6266, 8184, 4358, 15784, 12508, 14458, 10730, 12794] dup_rows_per_dst=[578, 1420, 18, 742, 3748, 3236, 2314, 1166] combine_rows_per_src=[6266, 8184, 4358, 15784, 12508, 14458, 10730, 12794] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=300916736 dispatch_meta_bytes=342008 combine_payload_bytes=300916736 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=301258744 ingress_bytes=335519744 link_bytes=335519744 token_bytes=602175480 weight_movement_bytes=34603008 total_nvlink_bytes=636778488 weight_to_token_ratio=0.057463 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=26368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262533785e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=283992064 dispatch_meta_bytes=326608 combine_payload_bytes=283992064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=284318672 ingress_bytes=318595072 link_bytes=318595072 token_bytes=568310736 weight_movement_bytes=34603008 total_nvlink_bytes=602913744 weight_to_token_ratio=0.060887 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=46080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.198847607e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=296091648 dispatch_meta_bytes=337376 combine_payload_bytes=296091648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296429024 ingress_bytes=330694656 link_bytes=330694656 token_bytes=592520672 weight_movement_bytes=34603008 total_nvlink_bytes=627123680 weight_to_token_ratio=0.058400 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=43392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.244377367e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=214130688 dispatch_meta_bytes=309424 combine_payload_bytes=214130688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=214440112 ingress_bytes=248733696 link_bytes=248733696 token_bytes=428570800 weight_movement_bytes=34603008 total_nvlink_bytes=463173808 weight_to_token_ratio=0.080740 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=21632 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.359648733e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=255254528 dispatch_meta_bytes=334904 combine_payload_bytes=255254528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=255589432 ingress_bytes=289857536 link_bytes=289857536 token_bytes=510843960 weight_movement_bytes=34603008 total_nvlink_bytes=545446968 weight_to_token_ratio=0.067737 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=19968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.090710573e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=299589632 dispatch_meta_bytes=332728 combine_payload_bytes=299589632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=299922360 ingress_bytes=334192640 link_bytes=334192640 token_bytes=599511992 weight_movement_bytes=34603008 total_nvlink_bytes=634115000 weight_to_token_ratio=0.057719 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=24064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257540000e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83452 payload_rows_remote=70010 dup_rows=14852 dup_rows_remote=11212 dedup_ratio=0.151082 router_tokens_per_expert=[1932, 0, 7136, 6, 0, 0, 0, 4997, 0, 0, 699, 1834, 6400, 5669, 334, 5, 0, 0, 0, 124, 0, 2, 2887, 567, 0, 58, 148, 1614, 0, 0, 1, 0, 2, 0, 5140, 1, 0, 0, 0, 0, 1, 1986, 0, 40, 5, 0, 1091, 119, 29, 0, 5658, 595, 30, 12, 0, 0, 27, 1, 0, 0, 2, 0, 0, 0] payload_rows_per_dst=[14648, 13442, 6604, 7388, 10284, 6184, 12096, 12806] dup_rows_per_dst=[8746, 3640, 556, 1002, 2, 300, 552, 54] combine_rows_per_src=[14648, 13442, 6604, 7388, 10284, 6184, 12096, 12806] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83994 payload_rows_remote=73466 dup_rows=14310 dup_rows_remote=12036 dedup_ratio=0.145569 router_tokens_per_expert=[5, 593, 1623, 0, 402, 26, 13, 362, 117, 0, 13, 1470, 1, 10, 3040, 1476, 5589, 7832, 17, 4, 4951, 1992, 2180, 0, 0, 27, 366, 0, 0, 1, 55, 0, 14, 0, 0, 1, 0, 1, 3021, 257, 6623, 7, 274, 1, 0, 17, 0, 1839, 7, 9, 342, 3069, 3, 28, 0, 763, 2, 40, 0, 11, 1, 517, 1, 139] payload_rows_per_dst=[5582, 9674, 4014, 15686, 12744, 14124, 10528, 11642] dup_rows_per_dst=[466, 2580, 12, 876, 3746, 3398, 2274, 958] combine_rows_per_src=[5582, 9674, 4014, 15686, 12744, 14124, 10528, 11642] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86206 payload_rows_remote=79846 dup_rows=12098 dup_rows_remote=11394 dedup_ratio=0.123067 router_tokens_per_expert=[2122, 0, 6830, 10, 0, 0, 0, 4842, 0, 0, 758, 2151, 6548, 5941, 440, 13, 0, 0, 1, 264, 0, 14, 2274, 979, 0, 96, 161, 1803, 0, 0, 1, 0, 6, 33, 4797, 6, 0, 0, 0, 0, 1, 1961, 0, 67, 13, 0, 1140, 118, 21, 0, 5065, 464, 111, 36, 0, 0, 59, 4, 0, 0, 2, 0, 0, 0] payload_rows_per_dst=[11412, 13890, 6360, 14428, 9662, 6298, 11038, 13118] dup_rows_per_dst=[2536, 4716, 704, 3354, 22, 302, 356, 108] combine_rows_per_src=[11412, 13890, 6360, 14428, 9662, 6298, 11038, 13118] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85824 payload_rows_remote=71524 dup_rows=12480 dup_rows_remote=8226 dedup_ratio=0.126953 router_tokens_per_expert=[1951, 0, 7110, 11, 0, 0, 2, 4948, 0, 0, 728, 1996, 6570, 6228, 401, 14, 0, 0, 0, 227, 0, 3, 2335, 673, 0, 76, 145, 1935, 0, 0, 0, 0, 3, 1, 4940, 1, 0, 0, 0, 0, 3, 1838, 0, 30, 7, 0, 1066, 105, 16, 0, 5160, 533, 29, 31, 0, 0, 27, 3, 0, 0, 1, 0, 5, 0] payload_rows_per_dst=[3908, 5824, 12706, 14942, 9886, 14300, 11110, 13148] dup_rows_per_dst=[20, 454, 3666, 3590, 4, 4254, 428, 64] combine_rows_per_src=[3908, 5824, 12706, 14942, 9886, 14300, 11110, 13148] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=289898496 dispatch_meta_bytes=324680 combine_payload_bytes=289898496 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=290223176 ingress_bytes=359104512 link_bytes=359104512 token_bytes=580121672 weight_movement_bytes=69206016 total_nvlink_bytes=649327688 weight_to_token_ratio=0.119296 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=33536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.351281368e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=296091648 dispatch_meta_bytes=337376 combine_payload_bytes=296091648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296429024 ingress_bytes=330694656 link_bytes=330694656 token_bytes=592520672 weight_movement_bytes=34603008 total_nvlink_bytes=627123680 weight_to_token_ratio=0.058400 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=43392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.244377367e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82078 payload_rows_remote=72584 dup_rows=16226 dup_rows_remote=14904 dedup_ratio=0.165059 router_tokens_per_expert=[11, 680, 1943, 0, 560, 26, 45, 432, 82, 0, 8, 1454, 1, 2, 2187, 1669, 5636, 7768, 5, 6, 4431, 2103, 2090, 0, 0, 46, 336, 0, 0, 1, 39, 0, 6, 0, 0, 1, 0, 3, 3084, 322, 6955, 6, 229, 2, 1, 5, 0, 1583, 16, 15, 253, 3256, 2, 65, 0, 926, 2, 27, 0, 23, 0, 652, 0, 157] payload_rows_per_dst=[15034, 11062, 9494, 13294, 6922, 3560, 10908, 11804] dup_rows_per_dst=[6270, 3942, 1322, 672, 400, 92, 2338, 1190] combine_rows_per_src=[15034, 11062, 9494, 13294, 6922, 3560, 10908, 11804] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83520 payload_rows_remote=72482 dup_rows=14784 dup_rows_remote=14342 dedup_ratio=0.150391 router_tokens_per_expert=[1953, 0, 7047, 11, 0, 0, 1, 4711, 0, 2, 768, 1959, 6759, 5943, 419, 12, 0, 0, 0, 362, 0, 0, 2636, 765, 0, 75, 142, 1645, 0, 1, 0, 0, 7, 4, 4880, 2, 0, 1, 0, 0, 1, 1794, 0, 65, 29, 0, 1289, 96, 17, 0, 5102, 579, 13, 29, 0, 0, 26, 3, 1, 0, 3, 0, 0, 0] payload_rows_per_dst=[3916, 5910, 12896, 9736, 12352, 14146, 11038, 13526] dup_rows_per_dst=[14, 410, 4052, 1582, 3938, 4288, 442, 58] combine_rows_per_src=[3916, 5910, 12896, 9736, 12352, 14146, 11038, 13526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86766 payload_rows_remote=71698 dup_rows=11538 dup_rows_remote=8710 dedup_ratio=0.117371 router_tokens_per_expert=[1928, 1, 7244, 7, 1, 0, 0, 5156, 0, 0, 868, 2161, 6232, 6095, 216, 8, 0, 0, 2, 167, 0, 2, 2444, 675, 0, 70, 128, 1500, 0, 0, 6, 0, 3, 3, 5285, 1, 0, 0, 0, 0, 7, 1570, 0, 42, 3, 0, 1214, 112, 28, 0, 5185, 665, 20, 24, 1, 0, 43, 1, 0, 0, 0, 0, 31, 3] payload_rows_per_dst=[9850, 12650, 7794, 15068, 10580, 7022, 11238, 12564] dup_rows_per_dst=[1822, 3988, 1300, 2828, 4, 932, 608, 56] combine_rows_per_src=[9850, 12650, 7794, 15068, 10580, 7022, 11238, 12564] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80074 payload_rows_remote=66284 dup_rows=18230 dup_rows_remote=18192 dedup_ratio=0.185445 router_tokens_per_expert=[2057, 0, 7164, 6, 0, 0, 0, 4699, 0, 0, 812, 2359, 6880, 6035, 383, 10, 0, 0, 0, 197, 0, 11, 2496, 787, 0, 98, 154, 1743, 0, 0, 2, 0, 0, 7, 4056, 3, 1, 0, 0, 0, 3, 1983, 0, 54, 5, 1, 1179, 124, 21, 0, 5263, 492, 8, 25, 0, 0, 18, 8, 0, 0, 5, 0, 2, 1] payload_rows_per_dst=[4114, 6620, 12564, 3974, 14918, 11604, 12490, 13790] dup_rows_per_dst=[12, 508, 3816, 20, 7544, 3318, 2974, 38] combine_rows_per_src=[4114, 6620, 12564, 3974, 14918, 11604, 12490, 13790] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=286760960 dispatch_meta_bytes=324888 combine_payload_bytes=286760960 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=287085848 ingress_bytes=425172992 link_bytes=425172992 token_bytes=573846808 weight_movement_bytes=138412032 total_nvlink_bytes=712258840 weight_to_token_ratio=0.241200 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=19456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.599891740e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=300916736 dispatch_meta_bytes=342008 combine_payload_bytes=300916736 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=301258744 ingress_bytes=335519744 link_bytes=335519744 token_bytes=602175480 weight_movement_bytes=34603008 total_nvlink_bytes=636778488 weight_to_token_ratio=0.057463 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=26368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262533785e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=297304064 dispatch_meta_bytes=349952 combine_payload_bytes=297304064 prefetch_bytes=34603008 reduce_bytes=173015040 egress_bytes=297654016 ingress_bytes=504922112 link_bytes=504922112 token_bytes=594958080 weight_movement_bytes=207618048 total_nvlink_bytes=802576128 weight_to_token_ratio=0.348962 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.899981258e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73558 payload_rows_remote=57338 dup_rows=24746 dup_rows_remote=7178 dedup_ratio=0.251729 router_tokens_per_expert=[1696, 0, 7128, 13, 0, 0, 0, 4914, 0, 1, 774, 2338, 6903, 5873, 324, 18, 0, 0, 0, 101, 2, 11, 2453, 773, 0, 67, 120, 1447, 0, 0, 0, 0, 5, 22, 4547, 4, 1, 0, 0, 2, 11, 1671, 0, 78, 6, 0, 1403, 117, 31, 0, 5650, 565, 9, 21, 0, 0, 40, 0, 1, 0, 7, 0, 5, 0] payload_rows_per_dst=[16220, 14598, 6110, 3250, 9144, 6104, 12038, 6094] dup_rows_per_dst=[17568, 5558, 570, 18, 18, 468, 514, 32] combine_rows_per_src=[16220, 14598, 6110, 3250, 9144, 6104, 12038, 6094] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85740 payload_rows_remote=75800 dup_rows=12564 dup_rows_remote=12558 dedup_ratio=0.127808 router_tokens_per_expert=[1751, 0, 7142, 8, 0, 0, 0, 5415, 0, 0, 649, 2029, 6393, 6136, 364, 6, 0, 0, 0, 127, 0, 0, 3139, 622, 0, 62, 118, 1234, 0, 1, 5, 0, 5, 0, 4965, 3, 0, 0, 0, 0, 11, 2122, 0, 48, 3, 0, 959, 113, 12, 1, 5169, 501, 1, 7, 0, 0, 24, 1, 0, 0, 0, 0, 6, 0] payload_rows_per_dst=[3502, 5680, 13776, 14860, 9940, 14192, 10984, 12806] dup_rows_per_dst=[16, 416, 4830, 2264, 6, 4592, 398, 42] combine_rows_per_src=[3502, 5680, 13776, 14860, 9940, 14192, 10984, 12806] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=327049216 dispatch_meta_bytes=364960 combine_payload_bytes=327049216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=327414176 ingress_bytes=361652224 link_bytes=361652224 token_bytes=654463392 weight_movement_bytes=34603008 total_nvlink_bytes=689066400 weight_to_token_ratio=0.052872 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=21120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.360868204e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=292962304 dispatch_meta_bytes=319000 combine_payload_bytes=292962304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293281304 ingress_bytes=327565312 link_bytes=327565312 token_bytes=586243608 weight_movement_bytes=34603008 total_nvlink_bytes=620846616 weight_to_token_ratio=0.059025 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=23552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.232601898e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83452 payload_rows_remote=70010 dup_rows=14852 dup_rows_remote=11212 dedup_ratio=0.151082 router_tokens_per_expert=[1932, 0, 7136, 6, 0, 0, 0, 4997, 0, 0, 699, 1834, 6400, 5669, 334, 5, 0, 0, 0, 124, 0, 2, 2887, 567, 0, 58, 148, 1614, 0, 0, 1, 0, 2, 0, 5140, 1, 0, 0, 0, 0, 1, 1986, 0, 40, 5, 0, 1091, 119, 29, 0, 5658, 595, 30, 12, 0, 0, 27, 1, 0, 0, 2, 0, 0, 0] payload_rows_per_dst=[14648, 13442, 6604, 7388, 10284, 6184, 12096, 12806] dup_rows_per_dst=[8746, 3640, 556, 1002, 2, 300, 552, 54] combine_rows_per_src=[14648, 13442, 6604, 7388, 10284, 6184, 12096, 12806] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85824 payload_rows_remote=71524 dup_rows=12480 dup_rows_remote=8226 dedup_ratio=0.126953 router_tokens_per_expert=[1951, 0, 7110, 11, 0, 0, 2, 4948, 0, 0, 728, 1996, 6570, 6228, 401, 14, 0, 0, 0, 227, 0, 3, 2335, 673, 0, 76, 145, 1935, 0, 0, 0, 0, 3, 1, 4940, 1, 0, 0, 0, 0, 3, 1838, 0, 30, 7, 0, 1066, 105, 16, 0, 5160, 533, 29, 31, 0, 0, 27, 3, 0, 0, 1, 0, 5, 0] payload_rows_per_dst=[3908, 5824, 12706, 14942, 9886, 14300, 11110, 13148] dup_rows_per_dst=[20, 454, 3666, 3590, 4, 4254, 428, 64] combine_rows_per_src=[3908, 5824, 12706, 14942, 9886, 14300, 11110, 13148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=296886272 dispatch_meta_bytes=347296 combine_payload_bytes=296886272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=297233568 ingress_bytes=331489280 link_bytes=331489280 token_bytes=594119840 weight_movement_bytes=34603008 total_nvlink_bytes=628722848 weight_to_token_ratio=0.058242 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=42368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.247367474e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80074 payload_rows_remote=66284 dup_rows=18230 dup_rows_remote=18192 dedup_ratio=0.185445 router_tokens_per_expert=[2057, 0, 7164, 6, 0, 0, 0, 4699, 0, 0, 812, 2359, 6880, 6035, 383, 10, 0, 0, 0, 197, 0, 11, 2496, 787, 0, 98, 154, 1743, 0, 0, 2, 0, 0, 7, 4056, 3, 1, 0, 0, 0, 3, 1983, 0, 54, 5, 1, 1179, 124, 21, 0, 5263, 492, 8, 25, 0, 0, 18, 8, 0, 0, 5, 0, 2, 1] payload_rows_per_dst=[4114, 6620, 12564, 3974, 14918, 11604, 12490, 13790] dup_rows_per_dst=[12, 508, 3816, 20, 7544, 3318, 2974, 38] combine_rows_per_src=[4114, 6620, 12564, 3974, 14918, 11604, 12490, 13790] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=293675008 dispatch_meta_bytes=321632 combine_payload_bytes=293675008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293996640 ingress_bytes=328278016 link_bytes=328278016 token_bytes=587671648 weight_movement_bytes=34603008 total_nvlink_bytes=622274656 weight_to_token_ratio=0.058882 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=34560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.235283746e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=271499264 dispatch_meta_bytes=337904 combine_payload_bytes=271499264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271837168 ingress_bytes=306102272 link_bytes=306102272 token_bytes=543336432 weight_movement_bytes=34603008 total_nvlink_bytes=577939440 weight_to_token_ratio=0.063686 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=48896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.151838206e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=310476800 dispatch_meta_bytes=353432 combine_payload_bytes=310476800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=310830232 ingress_bytes=345079808 link_bytes=345079808 token_bytes=621307032 weight_movement_bytes=34603008 total_nvlink_bytes=655910040 weight_to_token_ratio=0.055694 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=38656 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.298507536e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=234856448 dispatch_meta_bytes=258064 combine_payload_bytes=234856448 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=235114512 ingress_bytes=373268480 link_bytes=373268480 token_bytes=469970960 weight_movement_bytes=138412032 total_nvlink_bytes=608382992 weight_to_token_ratio=0.294512 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=18688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.404579240e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=286760960 dispatch_meta_bytes=324888 combine_payload_bytes=286760960 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=287085848 ingress_bytes=425172992 link_bytes=425172992 token_bytes=573846808 weight_movement_bytes=138412032 total_nvlink_bytes=712258840 weight_to_token_ratio=0.241200 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=19456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.599891740e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=310476800 dispatch_meta_bytes=353432 combine_payload_bytes=310476800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=310830232 ingress_bytes=345079808 link_bytes=345079808 token_bytes=621307032 weight_movement_bytes=34603008 total_nvlink_bytes=655910040 weight_to_token_ratio=0.055694 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=38656 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.298507536e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=292962304 dispatch_meta_bytes=319000 combine_payload_bytes=292962304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293281304 ingress_bytes=327565312 link_bytes=327565312 token_bytes=586243608 weight_movement_bytes=34603008 total_nvlink_bytes=620846616 weight_to_token_ratio=0.059025 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=23552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.232601898e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=271499264 dispatch_meta_bytes=337904 combine_payload_bytes=271499264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271837168 ingress_bytes=306102272 link_bytes=306102272 token_bytes=543336432 weight_movement_bytes=34603008 total_nvlink_bytes=577939440 weight_to_token_ratio=0.063686 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=48896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.151838206e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86206 payload_rows_remote=79846 dup_rows=12098 dup_rows_remote=11394 dedup_ratio=0.123067 router_tokens_per_expert=[2122, 0, 6830, 10, 0, 0, 0, 4842, 0, 0, 758, 2151, 6548, 5941, 440, 13, 0, 0, 1, 264, 0, 14, 2274, 979, 0, 96, 161, 1803, 0, 0, 1, 0, 6, 33, 4797, 6, 0, 0, 0, 0, 1, 1961, 0, 67, 13, 0, 1140, 118, 21, 0, 5065, 464, 111, 36, 0, 0, 59, 4, 0, 0, 2, 0, 0, 0] payload_rows_per_dst=[11412, 13890, 6360, 14428, 9662, 6298, 11038, 13118] dup_rows_per_dst=[2536, 4716, 704, 3354, 22, 302, 356, 108] combine_rows_per_src=[11412, 13890, 6360, 14428, 9662, 6298, 11038, 13118] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86180 payload_rows_remote=74262 dup_rows=12124 dup_rows_remote=11708 dedup_ratio=0.123332 router_tokens_per_expert=[974, 478, 0, 124, 0, 0, 0, 0, 2, 73, 7705, 0, 0, 2643, 7492, 39, 0, 0, 0, 0, 1192, 38, 0, 12, 21, 344, 9, 555, 0, 8185, 1, 58, 2738, 0, 1215, 0, 4961, 0, 36, 0, 424, 2335, 0, 5, 3, 197, 0, 0, 273, 0, 2, 6, 71, 5, 5810, 0, 5, 606, 0, 0, 0, 0, 492, 23] payload_rows_per_dst=[11890, 5474, 15300, 1942, 7550, 16384, 11918, 15722] dup_rows_per_dst=[1184, 40, 2168, 34, 428, 5914, 416, 1940] combine_rows_per_src=[11890, 5474, 15300, 1942, 7550, 16384, 11918, 15722] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89780 payload_rows_remote=73404 dup_rows=8524 dup_rows_remote=6088 dedup_ratio=0.086711 router_tokens_per_expert=[792, 407, 0, 86, 0, 0, 1, 3, 0, 60, 7736, 0, 2, 2612, 7473, 17, 0, 0, 2, 0, 1174, 53, 0, 4, 27, 237, 10, 888, 3, 8180, 0, 61, 3037, 0, 1297, 0, 5503, 1, 20, 0, 358, 1574, 2, 1, 7, 161, 0, 0, 247, 1, 0, 16, 73, 14, 5971, 0, 4, 669, 0, 1, 0, 1, 349, 17] payload_rows_per_dst=[12440, 5366, 15274, 16376, 8218, 4162, 12208, 15736] dup_rows_per_dst=[1144, 16, 2138, 2436, 492, 44, 436, 1818] combine_rows_per_src=[12440, 5366, 15274, 16376, 8218, 4162, 12208, 15736] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=327049216 dispatch_meta_bytes=364960 combine_payload_bytes=327049216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=327414176 ingress_bytes=361652224 link_bytes=361652224 token_bytes=654463392 weight_movement_bytes=34603008 total_nvlink_bytes=689066400 weight_to_token_ratio=0.052872 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=21120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.360868204e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77760 payload_rows_remote=61700 dup_rows=20544 dup_rows_remote=8270 dedup_ratio=0.208984 router_tokens_per_expert=[955, 411, 0, 58, 0, 0, 0, 0, 0, 45, 7871, 0, 0, 2258, 7501, 15, 0, 0, 0, 0, 1083, 26, 0, 0, 25, 210, 3, 843, 0, 8192, 0, 11, 3807, 0, 1036, 0, 5713, 0, 15, 0, 372, 1578, 0, 0, 8, 196, 0, 0, 145, 0, 0, 15, 62, 0, 5726, 0, 0, 663, 0, 0, 0, 0, 277, 32] payload_rows_per_dst=[12934, 16060, 2214, 16384, 12378, 4266, 11610, 1914] dup_rows_per_dst=[1340, 12274, 4, 2184, 4384, 42, 286, 30] combine_rows_per_src=[12934, 16060, 2214, 16384, 12378, 4266, 11610, 1914] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83520 payload_rows_remote=72482 dup_rows=14784 dup_rows_remote=14342 dedup_ratio=0.150391 router_tokens_per_expert=[1953, 0, 7047, 11, 0, 0, 1, 4711, 0, 2, 768, 1959, 6759, 5943, 419, 12, 0, 0, 0, 362, 0, 0, 2636, 765, 0, 75, 142, 1645, 0, 1, 0, 0, 7, 4, 4880, 2, 0, 1, 0, 0, 1, 1794, 0, 65, 29, 0, 1289, 96, 17, 0, 5102, 579, 13, 29, 0, 0, 26, 3, 1, 0, 3, 0, 0, 0] payload_rows_per_dst=[3916, 5910, 12896, 9736, 12352, 14146, 11038, 13526] dup_rows_per_dst=[14, 410, 4052, 1582, 3938, 4288, 442, 58] combine_rows_per_src=[3916, 5910, 12896, 9736, 12352, 14146, 11038, 13526] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86336 payload_rows_remote=73536 dup_rows=11968 dup_rows_remote=10442 dedup_ratio=0.121745 router_tokens_per_expert=[867, 460, 0, 48, 0, 0, 0, 0, 4, 97, 7726, 2, 0, 2746, 7361, 14, 0, 0, 0, 0, 1046, 37, 0, 4, 58, 225, 5, 607, 0, 8187, 1, 42, 3083, 1, 1136, 0, 5164, 0, 30, 0, 397, 2607, 0, 1, 3, 198, 0, 0, 193, 0, 3, 7, 73, 9, 5667, 0, 5, 613, 0, 0, 1, 0, 392, 32] payload_rows_per_dst=[12062, 7414, 12800, 16378, 13668, 6352, 11606, 6056] dup_rows_per_dst=[1016, 882, 1526, 1872, 5862, 60, 298, 452] combine_rows_per_src=[12062, 7414, 12800, 16378, 13668, 6352, 11606, 6056] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86806 payload_rows_remote=70422 dup_rows=11498 dup_rows_remote=5708 dedup_ratio=0.116964 router_tokens_per_expert=[892, 383, 0, 60, 0, 0, 0, 0, 0, 57, 7831, 0, 0, 2510, 7376, 8, 0, 0, 0, 0, 1084, 42, 0, 5, 18, 222, 11, 548, 0, 8191, 2, 49, 3467, 0, 1137, 0, 5350, 0, 38, 0, 411, 2336, 0, 0, 2, 147, 0, 0, 214, 0, 1, 13, 41, 0, 5680, 0, 2, 659, 0, 0, 0, 0, 336, 29] payload_rows_per_dst=[12296, 5128, 15102, 1686, 8770, 16384, 11584, 15856] dup_rows_per_dst=[1074, 22, 1912, 14, 514, 5790, 314, 1858] combine_rows_per_src=[12296, 5128, 15102, 1686, 8770, 16384, 11584, 15856] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89780 payload_rows_remote=73404 dup_rows=8524 dup_rows_remote=6088 dedup_ratio=0.086711 router_tokens_per_expert=[792, 407, 0, 86, 0, 0, 1, 3, 0, 60, 7736, 0, 2, 2612, 7473, 17, 0, 0, 2, 0, 1174, 53, 0, 4, 27, 237, 10, 888, 3, 8180, 0, 61, 3037, 0, 1297, 0, 5503, 1, 20, 0, 358, 1574, 2, 1, 7, 161, 0, 0, 247, 1, 0, 16, 73, 14, 5971, 0, 4, 669, 0, 1, 0, 1, 349, 17] payload_rows_per_dst=[12440, 5366, 15274, 16376, 8218, 4162, 12208, 15736] dup_rows_per_dst=[1144, 16, 2138, 2436, 492, 44, 436, 1818] combine_rows_per_src=[12440, 5366, 15274, 16376, 8218, 4162, 12208, 15736] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=296886272 dispatch_meta_bytes=347296 combine_payload_bytes=296886272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=297233568 ingress_bytes=331489280 link_bytes=331489280 token_bytes=594119840 weight_movement_bytes=34603008 total_nvlink_bytes=628722848 weight_to_token_ratio=0.058242 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=42368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.247367474e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82980 payload_rows_remote=67276 dup_rows=15324 dup_rows_remote=13678 dedup_ratio=0.155884 router_tokens_per_expert=[851, 399, 0, 79, 1, 0, 0, 0, 4, 60, 7755, 0, 1, 2478, 7573, 27, 0, 0, 0, 0, 1222, 30, 0, 19, 13, 230, 1, 425, 0, 8188, 0, 16, 3190, 0, 981, 0, 5613, 0, 29, 0, 384, 2319, 0, 3, 2, 183, 0, 0, 218, 0, 1, 14, 51, 0, 5872, 0, 4, 526, 0, 0, 0, 0, 356, 34] payload_rows_per_dst=[2778, 4012, 8380, 8898, 14358, 16384, 12466, 15704] dup_rows_per_dst=[40, 24, 1142, 638, 5110, 5774, 950, 1646] combine_rows_per_src=[2778, 4012, 8380, 8898, 14358, 16384, 12466, 15704] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=252723200 dispatch_meta_bytes=279880 combine_payload_bytes=252723200 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=253003080 ingress_bytes=425738240 link_bytes=425738240 token_bytes=505726280 weight_movement_bytes=173015040 total_nvlink_bytes=678741320 weight_to_token_ratio=0.342112 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=19840 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.602018722e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89336 payload_rows_remote=80844 dup_rows=8968 dup_rows_remote=8512 dedup_ratio=0.091227 router_tokens_per_expert=[840, 363, 0, 59, 0, 0, 0, 1, 1, 59, 7804, 0, 0, 2714, 7458, 38, 0, 1, 0, 0, 1320, 20, 0, 2, 17, 241, 4, 740, 2, 8191, 5, 15, 3552, 1, 915, 0, 5788, 0, 6, 0, 382, 1572, 0, 0, 4, 117, 0, 0, 167, 0, 0, 10, 76, 3, 5732, 0, 0, 641, 0, 0, 0, 0, 265, 26] payload_rows_per_dst=[12906, 5604, 15310, 9786, 8492, 9798, 11684, 15756] dup_rows_per_dst=[1196, 20, 2292, 1160, 456, 1836, 292, 1716] combine_rows_per_src=[12906, 5604, 15310, 9786, 8492, 9798, 11684, 15756] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=61702 dup_rows=24020 dup_rows_remote=22598 dedup_ratio=0.244344 router_tokens_per_expert=[986, 479, 0, 76, 0, 0, 0, 1, 1, 78, 7708, 0, 0, 2420, 7666, 111, 1, 0, 1, 0, 1307, 47, 0, 15, 50, 333, 14, 641, 0, 8166, 5, 49, 2770, 0, 1026, 0, 5460, 0, 16, 0, 400, 1902, 0, 0, 33, 222, 0, 0, 232, 0, 2, 15, 94, 3, 5772, 0, 0, 598, 0, 0, 0, 0, 395, 57] payload_rows_per_dst=[12582, 16376, 2732, 16356, 7306, 4996, 11852, 2084] dup_rows_per_dst=[1422, 19592, 10, 2160, 318, 118, 384, 16] combine_rows_per_src=[12582, 16376, 2732, 16356, 7306, 4996, 11852, 2084] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89336 payload_rows_remote=80844 dup_rows=8968 dup_rows_remote=8512 dedup_ratio=0.091227 router_tokens_per_expert=[840, 363, 0, 59, 0, 0, 0, 1, 1, 59, 7804, 0, 0, 2714, 7458, 38, 0, 1, 0, 0, 1320, 20, 0, 2, 17, 241, 4, 740, 2, 8191, 5, 15, 3552, 1, 915, 0, 5788, 0, 6, 0, 382, 1572, 0, 0, 4, 117, 0, 0, 167, 0, 0, 10, 76, 3, 5732, 0, 0, 641, 0, 0, 0, 0, 265, 26] payload_rows_per_dst=[12906, 5604, 15310, 9786, 8492, 9798, 11684, 15756] dup_rows_per_dst=[1196, 20, 2292, 1160, 456, 1836, 292, 1716] combine_rows_per_src=[12906, 5604, 15310, 9786, 8492, 9798, 11684, 15756] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=301203456 dispatch_meta_bytes=335912 combine_payload_bytes=301203456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=301539368 ingress_bytes=335806464 link_bytes=335806464 token_bytes=602742824 weight_movement_bytes=34603008 total_nvlink_bytes=637345832 weight_to_token_ratio=0.057409 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=39424 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263612689e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=288448512 dispatch_meta_bytes=304520 combine_payload_bytes=288448512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288753032 ingress_bytes=323051520 link_bytes=323051520 token_bytes=577201544 weight_movement_bytes=34603008 total_nvlink_bytes=611804552 weight_to_token_ratio=0.059950 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=28416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.215616862e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=304177152 dispatch_meta_bytes=343880 combine_payload_bytes=304177152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=304521032 ingress_bytes=338780160 link_bytes=338780160 token_bytes=608698184 weight_movement_bytes=34603008 total_nvlink_bytes=643301192 weight_to_token_ratio=0.056848 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=46336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274802468e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82980 payload_rows_remote=67276 dup_rows=15324 dup_rows_remote=13678 dedup_ratio=0.155884 router_tokens_per_expert=[851, 399, 0, 79, 1, 0, 0, 0, 4, 60, 7755, 0, 1, 2478, 7573, 27, 0, 0, 0, 0, 1222, 30, 0, 19, 13, 230, 1, 425, 0, 8188, 0, 16, 3190, 0, 981, 0, 5613, 0, 29, 0, 384, 2319, 0, 3, 2, 183, 0, 0, 218, 0, 1, 14, 51, 0, 5872, 0, 4, 526, 0, 0, 0, 0, 356, 34] payload_rows_per_dst=[2778, 4012, 8380, 8898, 14358, 16384, 12466, 15704] dup_rows_per_dst=[40, 24, 1142, 638, 5110, 5774, 950, 1646] combine_rows_per_src=[2778, 4012, 8380, 8898, 14358, 16384, 12466, 15704] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=300662784 dispatch_meta_bytes=317968 combine_payload_bytes=300662784 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=300980752 ingress_bytes=369868800 link_bytes=369868800 token_bytes=601643536 weight_movement_bytes=69206016 total_nvlink_bytes=670849552 weight_to_token_ratio=0.115028 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=37248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391786518e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=275562496 dispatch_meta_bytes=323816 combine_payload_bytes=275562496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275886312 ingress_bytes=310165504 link_bytes=310165504 token_bytes=551448808 weight_movement_bytes=34603008 total_nvlink_bytes=586051816 weight_to_token_ratio=0.062749 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=41088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.167127821e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=331137024 dispatch_meta_bytes=357424 combine_payload_bytes=331137024 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=331494448 ingress_bytes=400343040 link_bytes=400343040 token_bytes=662631472 weight_movement_bytes=69206016 total_nvlink_bytes=731837488 weight_to_token_ratio=0.104441 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=25728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.506458629e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=252723200 dispatch_meta_bytes=279880 combine_payload_bytes=252723200 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=253003080 ingress_bytes=425738240 link_bytes=425738240 token_bytes=505726280 weight_movement_bytes=173015040 total_nvlink_bytes=678741320 weight_to_token_ratio=0.342112 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=19840 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.602018722e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=288448512 dispatch_meta_bytes=304520 combine_payload_bytes=288448512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288753032 ingress_bytes=323051520 link_bytes=323051520 token_bytes=577201544 weight_movement_bytes=34603008 total_nvlink_bytes=611804552 weight_to_token_ratio=0.059950 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=28416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.215616862e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=252731392 dispatch_meta_bytes=337200 combine_payload_bytes=252731392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=253068592 ingress_bytes=287334400 link_bytes=287334400 token_bytes=505799984 weight_movement_bytes=34603008 total_nvlink_bytes=540402992 weight_to_token_ratio=0.068412 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=38144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.081216215e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86180 payload_rows_remote=74262 dup_rows=12124 dup_rows_remote=11708 dedup_ratio=0.123332 router_tokens_per_expert=[974, 478, 0, 124, 0, 0, 0, 0, 2, 73, 7705, 0, 0, 2643, 7492, 39, 0, 0, 0, 0, 1192, 38, 0, 12, 21, 344, 9, 555, 0, 8185, 1, 58, 2738, 0, 1215, 0, 4961, 0, 36, 0, 424, 2335, 0, 5, 3, 197, 0, 0, 273, 0, 2, 6, 71, 5, 5810, 0, 5, 606, 0, 0, 0, 0, 492, 23] payload_rows_per_dst=[11890, 5474, 15300, 1942, 7550, 16384, 11918, 15722] dup_rows_per_dst=[1184, 40, 2168, 34, 428, 5914, 416, 1940] combine_rows_per_src=[11890, 5474, 15300, 1942, 7550, 16384, 11918, 15722] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=331137024 dispatch_meta_bytes=357424 combine_payload_bytes=331137024 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=331494448 ingress_bytes=400343040 link_bytes=400343040 token_bytes=662631472 weight_movement_bytes=69206016 total_nvlink_bytes=731837488 weight_to_token_ratio=0.104441 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=25728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.506458629e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=275562496 dispatch_meta_bytes=323816 combine_payload_bytes=275562496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275886312 ingress_bytes=310165504 link_bytes=310165504 token_bytes=551448808 weight_movement_bytes=34603008 total_nvlink_bytes=586051816 weight_to_token_ratio=0.062749 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=41088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.167127821e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86336 payload_rows_remote=73536 dup_rows=11968 dup_rows_remote=10442 dedup_ratio=0.121745 router_tokens_per_expert=[867, 460, 0, 48, 0, 0, 0, 0, 4, 97, 7726, 2, 0, 2746, 7361, 14, 0, 0, 0, 0, 1046, 37, 0, 4, 58, 225, 5, 607, 0, 8187, 1, 42, 3083, 1, 1136, 0, 5164, 0, 30, 0, 397, 2607, 0, 1, 3, 198, 0, 0, 193, 0, 3, 7, 73, 9, 5667, 0, 5, 613, 0, 0, 1, 0, 392, 32] payload_rows_per_dst=[12062, 7414, 12800, 16378, 13668, 6352, 11606, 6056] dup_rows_per_dst=[1016, 882, 1526, 1872, 5862, 60, 298, 452] combine_rows_per_src=[12062, 7414, 12800, 16378, 13668, 6352, 11606, 6056] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71522 payload_rows_remote=58956 dup_rows=26782 dup_rows_remote=22950 dedup_ratio=0.272441 router_tokens_per_expert=[2762, 789, 0, 192, 12, 1, 535, 0, 0, 0, 0, 2854, 0, 0, 4, 111, 0, 17, 0, 0, 403, 86, 0, 0, 0, 0, 776, 0, 0, 1756, 3656, 59, 0, 845, 0, 0, 0, 1, 0, 0, 0, 1408, 10, 1, 7646, 0, 0, 4705, 4, 0, 1, 1, 2832, 26, 629, 1, 118, 1300, 0, 152, 0, 0, 7930, 7529] payload_rows_per_dst=[7734, 5904, 1012, 10592, 1692, 15648, 12566, 16374] dup_rows_per_dst=[848, 34, 0, 1902, 0, 2482, 3832, 17684] combine_rows_per_src=[7734, 5904, 1012, 10592, 1692, 15648, 12566, 16374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=301203456 dispatch_meta_bytes=335912 combine_payload_bytes=301203456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=301539368 ingress_bytes=335806464 link_bytes=335806464 token_bytes=602742824 weight_movement_bytes=34603008 total_nvlink_bytes=637345832 weight_to_token_ratio=0.057409 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=39424 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263612689e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85206 payload_rows_remote=74896 dup_rows=13098 dup_rows_remote=11204 dedup_ratio=0.133240 router_tokens_per_expert=[2835, 662, 0, 205, 20, 0, 463, 0, 0, 0, 0, 2897, 0, 1, 4, 125, 1, 33, 0, 0, 275, 119, 1, 0, 0, 0, 823, 0, 2, 1695, 3475, 107, 0, 757, 1, 0, 0, 0, 0, 0, 0, 1235, 20, 5, 7712, 0, 0, 4855, 0, 1, 0, 0, 3183, 13, 531, 0, 37, 1355, 0, 125, 0, 15, 7907, 7657] payload_rows_per_dst=[7532, 15756, 15826, 10310, 15666, 7126, 10054, 2936] dup_rows_per_dst=[838, 5612, 846, 1894, 1274, 714, 1792, 128] combine_rows_per_src=[7532, 15756, 15826, 10310, 15666, 7126, 10054, 2936] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85518 payload_rows_remote=78878 dup_rows=12786 dup_rows_remote=12328 dedup_ratio=0.130066 router_tokens_per_expert=[2806, 673, 0, 208, 2, 0, 437, 0, 0, 0, 0, 2921, 0, 0, 0, 114, 0, 42, 0, 0, 44, 106, 0, 0, 0, 0, 732, 0, 0, 1580, 3906, 46, 0, 946, 0, 0, 0, 0, 0, 0, 0, 1204, 6, 0, 7717, 0, 0, 5287, 0, 0, 0, 0, 2483, 4, 615, 0, 62, 1358, 0, 112, 0, 6, 8001, 7734] payload_rows_per_dst=[15588, 6640, 16004, 10620, 15778, 11712, 6138, 3038] dup_rows_per_dst=[7104, 458, 382, 1908, 1548, 1282, 66, 38] combine_rows_per_src=[15588, 6640, 16004, 10620, 15778, 11712, 6138, 3038] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85746 payload_rows_remote=69904 dup_rows=12558 dup_rows_remote=11448 dedup_ratio=0.127747 router_tokens_per_expert=[3290, 561, 0, 197, 9, 0, 422, 0, 0, 0, 0, 3095, 0, 1, 1, 123, 0, 23, 2, 0, 428, 106, 0, 1, 0, 0, 801, 0, 0, 1758, 3280, 54, 0, 869, 0, 0, 0, 1, 0, 0, 0, 1254, 9, 0, 7487, 0, 0, 4778, 2, 0, 0, 1, 2860, 19, 687, 0, 118, 1488, 0, 101, 0, 0, 7916, 7410] payload_rows_per_dst=[8184, 15274, 15842, 9924, 15354, 10816, 7002, 3350] dup_rows_per_dst=[774, 5986, 1110, 1862, 1360, 1266, 136, 64] combine_rows_per_src=[8184, 15274, 15842, 9924, 15354, 10816, 7002, 3350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=241483776 dispatch_meta_bytes=327624 combine_payload_bytes=241483776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=241811400 ingress_bytes=276086784 link_bytes=276086784 token_bytes=483295176 weight_movement_bytes=34603008 total_nvlink_bytes=517898184 weight_to_token_ratio=0.071598 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=22272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.038892341e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85302 payload_rows_remote=69238 dup_rows=13002 dup_rows_remote=5558 dedup_ratio=0.132263 router_tokens_per_expert=[2815, 697, 0, 217, 24, 0, 591, 0, 0, 1, 0, 2878, 0, 0, 6, 134, 0, 28, 2, 0, 448, 107, 0, 0, 0, 0, 868, 0, 0, 1640, 3569, 96, 1, 810, 1, 0, 0, 2, 0, 0, 0, 894, 14, 2, 7686, 0, 0, 5081, 1, 0, 0, 2, 3136, 6, 542, 0, 114, 1343, 0, 104, 0, 5, 7877, 7410] payload_rows_per_dst=[16064, 6000, 15774, 10350, 15636, 11070, 7314, 3094] dup_rows_per_dst=[7444, 38, 1150, 1996, 1364, 912, 60, 38] combine_rows_per_src=[16064, 6000, 15774, 10350, 15636, 11070, 7314, 3094] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81382 payload_rows_remote=68742 dup_rows=16922 dup_rows_remote=14402 dedup_ratio=0.172139 router_tokens_per_expert=[3241, 709, 0, 182, 4, 0, 382, 0, 0, 0, 0, 2768, 0, 0, 3, 135, 0, 47, 0, 0, 351, 98, 0, 0, 0, 0, 794, 0, 0, 1572, 3603, 62, 0, 822, 0, 0, 0, 0, 0, 0, 0, 1150, 9, 0, 7721, 0, 0, 4880, 1, 0, 0, 0, 2742, 6, 736, 0, 62, 1304, 0, 102, 0, 4, 7949, 7713] payload_rows_per_dst=[8138, 7564, 12198, 10608, 8464, 12640, 12888, 8882] dup_rows_per_dst=[898, 1062, 818, 2138, 646, 2520, 3842, 4998] combine_rows_per_src=[8138, 7564, 12198, 10608, 8464, 12640, 12888, 8882] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84244 payload_rows_remote=68378 dup_rows=14060 dup_rows_remote=12372 dedup_ratio=0.143026 router_tokens_per_expert=[2709, 740, 0, 196, 5, 1, 436, 0, 0, 0, 0, 2792, 0, 0, 7, 92, 0, 50, 0, 0, 133, 132, 0, 0, 0, 0, 777, 0, 0, 1561, 3626, 51, 0, 956, 0, 1, 0, 0, 0, 0, 0, 1283, 7, 1, 7820, 0, 0, 5005, 1, 0, 0, 0, 2733, 9, 638, 0, 60, 1533, 0, 93, 0, 5, 7929, 7770] payload_rows_per_dst=[7384, 15834, 15862, 10288, 15866, 2582, 13110, 3318] dup_rows_per_dst=[790, 5488, 626, 1742, 1688, 0, 3662, 64] combine_rows_per_src=[7384, 15834, 15862, 10288, 15866, 2582, 13110, 3318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72332 payload_rows_remote=55950 dup_rows=25972 dup_rows_remote=8056 dedup_ratio=0.264201 router_tokens_per_expert=[3112, 757, 0, 170, 6, 0, 447, 0, 0, 0, 0, 3228, 0, 0, 3, 96, 0, 23, 0, 0, 303, 99, 0, 2, 0, 0, 759, 0, 0, 1642, 3613, 71, 1, 979, 0, 1, 3, 0, 0, 0, 0, 1164, 10, 5, 7588, 0, 0, 4925, 1, 0, 0, 0, 2299, 3, 693, 0, 50, 1265, 0, 101, 0, 2, 8052, 7679] payload_rows_per_dst=[8118, 6614, 850, 10332, 1964, 15506, 12566, 16382] dup_rows_per_dst=[866, 40, 4, 1838, 4, 2028, 3276, 17916] combine_rows_per_src=[8118, 6614, 850, 10332, 1964, 15506, 12566, 16382] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85206 payload_rows_remote=74896 dup_rows=13098 dup_rows_remote=11204 dedup_ratio=0.133240 router_tokens_per_expert=[2835, 662, 0, 205, 20, 0, 463, 0, 0, 0, 0, 2897, 0, 1, 4, 125, 1, 33, 0, 0, 275, 119, 1, 0, 0, 0, 823, 0, 2, 1695, 3475, 107, 0, 757, 1, 0, 0, 0, 0, 0, 0, 1235, 20, 5, 7712, 0, 0, 4855, 0, 1, 0, 0, 3183, 13, 531, 0, 37, 1355, 0, 125, 0, 15, 7907, 7657] payload_rows_per_dst=[7532, 15756, 15826, 10310, 15666, 7126, 10054, 2936] dup_rows_per_dst=[838, 5612, 846, 1894, 1274, 714, 1792, 128] combine_rows_per_src=[7532, 15756, 15826, 10310, 15666, 7126, 10054, 2936] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=304177152 dispatch_meta_bytes=343880 combine_payload_bytes=304177152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=304521032 ingress_bytes=338780160 link_bytes=338780160 token_bytes=608698184 weight_movement_bytes=34603008 total_nvlink_bytes=643301192 weight_to_token_ratio=0.056848 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=46336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274802468e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72332 payload_rows_remote=55950 dup_rows=25972 dup_rows_remote=8056 dedup_ratio=0.264201 router_tokens_per_expert=[3112, 757, 0, 170, 6, 0, 447, 0, 0, 0, 0, 3228, 0, 0, 3, 96, 0, 23, 0, 0, 303, 99, 0, 2, 0, 0, 759, 0, 0, 1642, 3613, 71, 1, 979, 0, 1, 3, 0, 0, 0, 0, 1164, 10, 5, 7588, 0, 0, 4925, 1, 0, 0, 0, 2299, 3, 693, 0, 50, 1265, 0, 101, 0, 2, 8052, 7679] payload_rows_per_dst=[8118, 6614, 850, 10332, 1964, 15506, 12566, 16382] dup_rows_per_dst=[866, 40, 4, 1838, 4, 2028, 3276, 17916] combine_rows_per_src=[8118, 6614, 850, 10332, 1964, 15506, 12566, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=306774016 dispatch_meta_bytes=344400 combine_payload_bytes=306774016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307118416 ingress_bytes=341377024 link_bytes=341377024 token_bytes=613892432 weight_movement_bytes=34603008 total_nvlink_bytes=648495440 weight_to_token_ratio=0.056367 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=28800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.284574258e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 1, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=323084288 dispatch_meta_bytes=364824 combine_payload_bytes=323084288 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=323449112 ingress_bytes=357687296 link_bytes=357687296 token_bytes=646533400 weight_movement_bytes=34603008 total_nvlink_bytes=681136408 weight_to_token_ratio=0.053521 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=24832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.345948499e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84244 payload_rows_remote=68378 dup_rows=14060 dup_rows_remote=12372 dedup_ratio=0.143026 router_tokens_per_expert=[2709, 740, 0, 196, 5, 1, 436, 0, 0, 0, 0, 2792, 0, 0, 7, 92, 0, 50, 0, 0, 133, 132, 0, 0, 0, 0, 777, 0, 0, 1561, 3626, 51, 0, 956, 0, 1, 0, 0, 0, 0, 0, 1283, 7, 1, 7820, 0, 0, 5005, 1, 0, 0, 0, 2733, 9, 638, 0, 60, 1533, 0, 93, 0, 5, 7929, 7770] payload_rows_per_dst=[7384, 15834, 15862, 10288, 15866, 2582, 13110, 3318] dup_rows_per_dst=[790, 5488, 626, 1742, 1688, 0, 3662, 64] combine_rows_per_src=[7384, 15834, 15862, 10288, 15866, 2582, 13110, 3318] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85302 payload_rows_remote=69238 dup_rows=13002 dup_rows_remote=5558 dedup_ratio=0.132263 router_tokens_per_expert=[2815, 697, 0, 217, 24, 0, 591, 0, 0, 1, 0, 2878, 0, 0, 6, 134, 0, 28, 2, 0, 448, 107, 0, 0, 0, 0, 868, 0, 0, 1640, 3569, 96, 1, 810, 1, 0, 0, 2, 0, 0, 0, 894, 14, 2, 7686, 0, 0, 5081, 1, 0, 0, 2, 3136, 6, 542, 0, 114, 1343, 0, 104, 0, 5, 7877, 7410] payload_rows_per_dst=[16064, 6000, 15774, 10350, 15636, 11070, 7314, 3094] dup_rows_per_dst=[7444, 38, 1150, 1996, 1364, 912, 60, 38] combine_rows_per_src=[16064, 6000, 15774, 10350, 15636, 11070, 7314, 3094] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=286326784 dispatch_meta_bytes=325408 combine_payload_bytes=286326784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286652192 ingress_bytes=320929792 link_bytes=320929792 token_bytes=572978976 weight_movement_bytes=34603008 total_nvlink_bytes=607581984 weight_to_token_ratio=0.060391 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=45696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.207632970e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=283598848 dispatch_meta_bytes=299184 combine_payload_bytes=283598848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=283898032 ingress_bytes=318201856 link_bytes=318201856 token_bytes=567496880 weight_movement_bytes=34603008 total_nvlink_bytes=602099888 weight_to_token_ratio=0.060975 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.197367967e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=281567232 dispatch_meta_bytes=332576 combine_payload_bytes=281567232 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=281899808 ingress_bytes=385376256 link_bytes=385376256 token_bytes=563467040 weight_movement_bytes=103809024 total_nvlink_bytes=667276064 weight_to_token_ratio=0.184233 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=19328 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.450139826e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=306774016 dispatch_meta_bytes=344400 combine_payload_bytes=306774016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307118416 ingress_bytes=341377024 link_bytes=341377024 token_bytes=613892432 weight_movement_bytes=34603008 total_nvlink_bytes=648495440 weight_to_token_ratio=0.056367 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=28800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.284574258e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=229171200 dispatch_meta_bytes=256024 combine_payload_bytes=229171200 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=229427224 ingress_bytes=402186240 link_bytes=402186240 token_bytes=458598424 weight_movement_bytes=173015040 total_nvlink_bytes=631613464 weight_to_token_ratio=0.377269 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.035000 gemm_active_groups=7 gemm_peak_group_rows=18816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.513394442e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85746 payload_rows_remote=69904 dup_rows=12558 dup_rows_remote=11448 dedup_ratio=0.127747 router_tokens_per_expert=[3290, 561, 0, 197, 9, 0, 422, 0, 0, 0, 0, 3095, 0, 1, 1, 123, 0, 23, 2, 0, 428, 106, 0, 1, 0, 0, 801, 0, 0, 1758, 3280, 54, 0, 869, 0, 0, 0, 1, 0, 0, 0, 1254, 9, 0, 7487, 0, 0, 4778, 2, 0, 0, 1, 2860, 19, 687, 0, 118, 1488, 0, 101, 0, 0, 7916, 7410] payload_rows_per_dst=[8184, 15274, 15842, 9924, 15354, 10816, 7002, 3350] dup_rows_per_dst=[774, 5986, 1110, 1862, 1360, 1266, 136, 64] combine_rows_per_src=[8184, 15274, 15842, 9924, 15354, 10816, 7002, 3350] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=280076288 dispatch_meta_bytes=323000 combine_payload_bytes=280076288 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280399288 ingress_bytes=314679296 link_bytes=314679296 token_bytes=560475576 weight_movement_bytes=34603008 total_nvlink_bytes=595078584 weight_to_token_ratio=0.061739 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=42240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.184112857e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=283598848 dispatch_meta_bytes=299184 combine_payload_bytes=283598848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=283898032 ingress_bytes=318201856 link_bytes=318201856 token_bytes=567496880 weight_movement_bytes=34603008 total_nvlink_bytes=602099888 weight_to_token_ratio=0.060975 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.197367967e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71522 payload_rows_remote=58956 dup_rows=26782 dup_rows_remote=22950 dedup_ratio=0.272441 router_tokens_per_expert=[2762, 789, 0, 192, 12, 1, 535, 0, 0, 0, 0, 2854, 0, 0, 4, 111, 0, 17, 0, 0, 403, 86, 0, 0, 0, 0, 776, 0, 0, 1756, 3656, 59, 0, 845, 0, 0, 0, 1, 0, 0, 0, 1408, 10, 1, 7646, 0, 0, 4705, 4, 0, 1, 1, 2832, 26, 629, 1, 118, 1300, 0, 152, 0, 0, 7930, 7529] payload_rows_per_dst=[7734, 5904, 1012, 10592, 1692, 15648, 12566, 16374] dup_rows_per_dst=[848, 34, 0, 1902, 0, 2482, 3832, 17684] combine_rows_per_src=[7734, 5904, 1012, 10592, 1692, 15648, 12566, 16374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=229171200 dispatch_meta_bytes=256024 combine_payload_bytes=229171200 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=229427224 ingress_bytes=402186240 link_bytes=402186240 token_bytes=458598424 weight_movement_bytes=173015040 total_nvlink_bytes=631613464 weight_to_token_ratio=0.377269 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.035000 gemm_active_groups=7 gemm_peak_group_rows=18816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.513394442e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81532 payload_rows_remote=69852 dup_rows=16772 dup_rows_remote=15756 dedup_ratio=0.170614 router_tokens_per_expert=[965, 2373, 5824, 843, 0, 5462, 39, 0, 0, 0, 0, 0, 1, 21, 0, 0, 0, 0, 0, 4948, 0, 0, 0, 372, 1, 230, 1221, 2815, 8192, 0, 138, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1464, 4716, 0, 0, 301, 0, 0, 0, 4, 5291, 0, 1053, 0, 0, 0, 0, 0, 10, 1160, 0, 1701, 0, 0, 5] payload_rows_per_dst=[7274, 44, 10630, 16384, 11650, 10916, 11680, 12954] dup_rows_per_dst=[1166, 0, 10, 8810, 2, 2046, 1016, 3722] combine_rows_per_src=[7274, 44, 10630, 16384, 11650, 10916, 11680, 12954] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=286326784 dispatch_meta_bytes=325408 combine_payload_bytes=286326784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286652192 ingress_bytes=320929792 link_bytes=320929792 token_bytes=572978976 weight_movement_bytes=34603008 total_nvlink_bytes=607581984 weight_to_token_ratio=0.060391 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=45696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.207632970e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89092 payload_rows_remote=80464 dup_rows=9212 dup_rows_remote=8050 dedup_ratio=0.093709 router_tokens_per_expert=[1425, 2403, 5965, 839, 0, 4862, 62, 0, 0, 0, 0, 1, 0, 99, 0, 2, 0, 0, 0, 4646, 0, 0, 0, 328, 2, 268, 1188, 3278, 8188, 0, 159, 0, 1, 0, 0, 0, 7, 0, 3, 1, 1813, 3678, 0, 0, 351, 2, 1, 2, 1, 5808, 0, 966, 0, 0, 1, 0, 1, 19, 1056, 0, 1718, 0, 0, 8] payload_rows_per_dst=[8088, 16380, 9936, 8628, 11940, 9836, 12166, 12118] dup_rows_per_dst=[1370, 200, 12, 1162, 14, 1858, 1386, 3210] combine_rows_per_src=[8088, 16380, 9936, 8628, 11940, 9836, 12166, 12118] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=286113792 dispatch_meta_bytes=342432 combine_payload_bytes=286113792 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=286456224 ingress_bytes=320716800 link_bytes=320716800 token_bytes=572570016 weight_movement_bytes=34603008 total_nvlink_bytes=607173024 weight_to_token_ratio=0.060435 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=41984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.206831499e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85212 payload_rows_remote=68828 dup_rows=13092 dup_rows_remote=12730 dedup_ratio=0.133179 router_tokens_per_expert=[1380, 2351, 6066, 590, 0, 4608, 21, 0, 0, 0, 0, 0, 1, 180, 0, 0, 0, 0, 0, 5259, 0, 0, 0, 345, 2, 200, 1085, 3198, 8192, 0, 188, 0, 0, 0, 0, 0, 6, 0, 1, 0, 1855, 4720, 0, 0, 306, 0, 0, 0, 0, 4951, 0, 707, 0, 0, 0, 0, 0, 27, 1236, 0, 1677, 0, 0, 0] payload_rows_per_dst=[12950, 16384, 13004, 2932, 12134, 11408, 10844, 5556] dup_rows_per_dst=[4950, 362, 4600, 18, 12, 2354, 472, 324] combine_rows_per_src=[12950, 16384, 13004, 2932, 12134, 11408, 10844, 5556] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87100 payload_rows_remote=76206 dup_rows=11204 dup_rows_remote=9522 dedup_ratio=0.113973 router_tokens_per_expert=[1042, 2552, 6210, 863, 0, 5628, 24, 0, 0, 0, 0, 0, 1, 117, 0, 0, 0, 0, 0, 4564, 0, 0, 0, 340, 7, 254, 1183, 2842, 8192, 0, 221, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1617, 4352, 0, 0, 262, 0, 0, 0, 0, 4647, 0, 953, 0, 0, 0, 0, 0, 19, 1302, 0, 1951, 0, 0, 4] payload_rows_per_dst=[12336, 16384, 10894, 5652, 12420, 10442, 10612, 8360] dup_rows_per_dst=[4602, 236, 1682, 594, 10, 2020, 588, 1472] combine_rows_per_src=[12336, 16384, 10894, 5652, 12420, 10442, 10612, 8360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83870 payload_rows_remote=69954 dup_rows=14434 dup_rows_remote=6662 dedup_ratio=0.146830 router_tokens_per_expert=[1453, 2469, 5451, 741, 0, 5228, 57, 0, 3, 0, 0, 0, 2, 61, 3, 0, 0, 0, 0, 4779, 0, 0, 0, 361, 10, 255, 1211, 2962, 8192, 0, 133, 0, 0, 0, 0, 0, 6, 2, 11, 0, 1761, 4635, 0, 0, 349, 0, 0, 0, 0, 5260, 0, 921, 0, 0, 0, 0, 0, 5, 1129, 0, 1701, 0, 0, 1] payload_rows_per_dst=[13916, 16384, 12396, 7496, 10924, 6174, 11238, 5342] dup_rows_per_dst=[7772, 138, 3808, 748, 16, 828, 794, 330] combine_rows_per_src=[13916, 16384, 12396, 7496, 10924, 6174, 11238, 5342] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88486 payload_rows_remote=77514 dup_rows=9818 dup_rows_remote=7814 dedup_ratio=0.099874 router_tokens_per_expert=[1407, 2520, 5990, 756, 0, 5044, 58, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 0, 4797, 0, 0, 0, 380, 3, 257, 1091, 2803, 8192, 0, 169, 0, 0, 0, 0, 0, 3, 0, 6, 0, 1621, 4541, 0, 0, 326, 0, 0, 0, 0, 5137, 0, 884, 0, 0, 0, 0, 0, 29, 1331, 0, 1715, 0, 0, 0] payload_rows_per_dst=[8288, 14130, 10348, 8928, 11980, 10972, 11288, 12552] dup_rows_per_dst=[1194, 182, 6, 1974, 18, 2004, 754, 3686] combine_rows_per_src=[8288, 14130, 10348, 8928, 11980, 10972, 11288, 12552] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89812 payload_rows_remote=77568 dup_rows=8492 dup_rows_remote=8470 dedup_ratio=0.086385 router_tokens_per_expert=[1234, 2437, 6120, 767, 0, 4440, 52, 0, 1, 0, 0, 0, 0, 346, 0, 0, 0, 0, 0, 4944, 0, 0, 0, 362, 2, 247, 1250, 3069, 8191, 0, 151, 1, 0, 0, 0, 0, 4, 0, 9, 0, 1778, 4968, 0, 0, 288, 0, 0, 0, 0, 5536, 0, 716, 0, 0, 0, 0, 0, 20, 795, 1, 1422, 0, 0, 1] payload_rows_per_dst=[7760, 16382, 10596, 8416, 12244, 11506, 11960, 10948] dup_rows_per_dst=[1220, 694, 16, 1024, 22, 2562, 544, 2410] combine_rows_per_src=[7760, 16382, 10596, 8416, 12244, 11506, 11960, 10948] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89092 payload_rows_remote=80464 dup_rows=9212 dup_rows_remote=8050 dedup_ratio=0.093709 router_tokens_per_expert=[1425, 2403, 5965, 839, 0, 4862, 62, 0, 0, 0, 0, 1, 0, 99, 0, 2, 0, 0, 0, 4646, 0, 0, 0, 328, 2, 268, 1188, 3278, 8188, 0, 159, 0, 1, 0, 0, 0, 7, 0, 3, 1, 1813, 3678, 0, 0, 351, 2, 1, 2, 1, 5808, 0, 966, 0, 0, 1, 0, 1, 19, 1056, 0, 1718, 0, 0, 8] payload_rows_per_dst=[8088, 16380, 9936, 8628, 11940, 9836, 12166, 12118] dup_rows_per_dst=[1370, 200, 12, 1162, 14, 1858, 1386, 3210] combine_rows_per_src=[8088, 16380, 9936, 8628, 11940, 9836, 12166, 12118] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79798 payload_rows_remote=68018 dup_rows=18506 dup_rows_remote=14966 dedup_ratio=0.188253 router_tokens_per_expert=[1186, 2342, 5793, 807, 0, 4540, 28, 0, 0, 0, 0, 0, 1, 201, 0, 0, 0, 1, 0, 4840, 0, 0, 0, 342, 3, 218, 1206, 2976, 8191, 0, 174, 1, 0, 0, 0, 0, 8, 0, 4, 0, 1681, 4631, 1, 0, 376, 0, 0, 0, 0, 5512, 0, 969, 0, 0, 0, 0, 0, 17, 1020, 1, 2081, 0, 0, 1] payload_rows_per_dst=[4948, 404, 10366, 16384, 13100, 11082, 11734, 11780] dup_rows_per_dst=[470, 0, 0, 9154, 1818, 2296, 1228, 3540] combine_rows_per_src=[4948, 404, 10366, 16384, 13100, 11082, 11734, 11780] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=329580544 dispatch_meta_bytes=354056 combine_payload_bytes=329580544 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=329934600 ingress_bytes=433389568 link_bytes=433389568 token_bytes=659515144 weight_movement_bytes=103809024 total_nvlink_bytes=763324168 weight_to_token_ratio=0.157402 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=17536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.630810054e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=241483776 dispatch_meta_bytes=327624 combine_payload_bytes=241483776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=241811400 ingress_bytes=276086784 link_bytes=276086784 token_bytes=483295176 weight_movement_bytes=34603008 total_nvlink_bytes=517898184 weight_to_token_ratio=0.071598 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=22272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.038892341e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=281919488 dispatch_meta_bytes=326232 combine_payload_bytes=281919488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282245720 ingress_bytes=316522496 link_bytes=316522496 token_bytes=564165208 weight_movement_bytes=34603008 total_nvlink_bytes=598768216 weight_to_token_ratio=0.061335 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=48128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.191048670e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79798 payload_rows_remote=68018 dup_rows=18506 dup_rows_remote=14966 dedup_ratio=0.188253 router_tokens_per_expert=[1186, 2342, 5793, 807, 0, 4540, 28, 0, 0, 0, 0, 0, 1, 201, 0, 0, 0, 1, 0, 4840, 0, 0, 0, 342, 3, 218, 1206, 2976, 8191, 0, 174, 1, 0, 0, 0, 0, 8, 0, 4, 0, 1681, 4631, 1, 0, 376, 0, 0, 0, 0, 5512, 0, 969, 0, 0, 0, 0, 0, 17, 1020, 1, 2081, 0, 0, 1] payload_rows_per_dst=[4948, 404, 10366, 16384, 13100, 11082, 11734, 11780] dup_rows_per_dst=[470, 0, 0, 9154, 1818, 2296, 1228, 3540] combine_rows_per_src=[4948, 404, 10366, 16384, 13100, 11082, 11734, 11780] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89812 payload_rows_remote=77568 dup_rows=8492 dup_rows_remote=8470 dedup_ratio=0.086385 router_tokens_per_expert=[1234, 2437, 6120, 767, 0, 4440, 52, 0, 1, 0, 0, 0, 0, 346, 0, 0, 0, 0, 0, 4944, 0, 0, 0, 362, 2, 247, 1250, 3069, 8191, 0, 151, 1, 0, 0, 0, 0, 4, 0, 9, 0, 1778, 4968, 0, 0, 288, 0, 0, 0, 0, 5536, 0, 716, 0, 0, 0, 0, 0, 20, 795, 1, 1422, 0, 0, 1] payload_rows_per_dst=[7760, 16382, 10596, 8416, 12244, 11506, 11960, 10948] dup_rows_per_dst=[1220, 694, 16, 1024, 22, 2562, 544, 2410] combine_rows_per_src=[7760, 16382, 10596, 8416, 12244, 11506, 11960, 10948] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83870 payload_rows_remote=69954 dup_rows=14434 dup_rows_remote=6662 dedup_ratio=0.146830 router_tokens_per_expert=[1453, 2469, 5451, 741, 0, 5228, 57, 0, 3, 0, 0, 0, 2, 61, 3, 0, 0, 0, 0, 4779, 0, 0, 0, 361, 10, 255, 1211, 2962, 8192, 0, 133, 0, 0, 0, 0, 0, 6, 2, 11, 0, 1761, 4635, 0, 0, 349, 0, 0, 0, 0, 5260, 0, 921, 0, 0, 0, 0, 0, 5, 1129, 0, 1701, 0, 0, 1] payload_rows_per_dst=[13916, 16384, 12396, 7496, 10924, 6174, 11238, 5342] dup_rows_per_dst=[7772, 138, 3808, 748, 16, 828, 794, 330] combine_rows_per_src=[13916, 16384, 12396, 7496, 10924, 6174, 11238, 5342] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=312139776 dispatch_meta_bytes=342912 combine_payload_bytes=312139776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=312482688 ingress_bytes=346742784 link_bytes=346742784 token_bytes=624622464 weight_movement_bytes=34603008 total_nvlink_bytes=659225472 weight_to_token_ratio=0.055398 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=38784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.304765181e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=317497344 dispatch_meta_bytes=341312 combine_payload_bytes=317497344 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=317838656 ingress_bytes=421306368 link_bytes=421306368 token_bytes=635336000 weight_movement_bytes=103809024 total_nvlink_bytes=739145024 weight_to_token_ratio=0.163392 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=32896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.585341945e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=329580544 dispatch_meta_bytes=354056 combine_payload_bytes=329580544 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=329934600 ingress_bytes=433389568 link_bytes=433389568 token_bytes=659515144 weight_movement_bytes=103809024 total_nvlink_bytes=763324168 weight_to_token_ratio=0.157402 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=17536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.630810054e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=278601728 dispatch_meta_bytes=331936 combine_payload_bytes=278601728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278933664 ingress_bytes=313204736 link_bytes=313204736 token_bytes=557535392 weight_movement_bytes=34603008 total_nvlink_bytes=592138400 weight_to_token_ratio=0.062064 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=26112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.178564207e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=317718528 dispatch_meta_bytes=344152 combine_payload_bytes=317718528 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=318062680 ingress_bytes=386924544 link_bytes=386924544 token_bytes=635781208 weight_movement_bytes=69206016 total_nvlink_bytes=704987224 weight_to_token_ratio=0.108852 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=47488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.455965909e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=286531584 dispatch_meta_bytes=306464 combine_payload_bytes=286531584 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=286838048 ingress_bytes=424943616 link_bytes=424943616 token_bytes=573369632 weight_movement_bytes=138412032 total_nvlink_bytes=711781664 weight_to_token_ratio=0.241401 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.035000 gemm_active_groups=7 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.599028616e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81532 payload_rows_remote=69852 dup_rows=16772 dup_rows_remote=15756 dedup_ratio=0.170614 router_tokens_per_expert=[965, 2373, 5824, 843, 0, 5462, 39, 0, 0, 0, 0, 0, 1, 21, 0, 0, 0, 0, 0, 4948, 0, 0, 0, 372, 1, 230, 1221, 2815, 8192, 0, 138, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1464, 4716, 0, 0, 301, 0, 0, 0, 4, 5291, 0, 1053, 0, 0, 0, 0, 0, 10, 1160, 0, 1701, 0, 0, 5] payload_rows_per_dst=[7274, 44, 10630, 16384, 11650, 10916, 11680, 12954] dup_rows_per_dst=[1166, 0, 10, 8810, 2, 2046, 1016, 3722] combine_rows_per_src=[7274, 44, 10630, 16384, 11650, 10916, 11680, 12954] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87100 payload_rows_remote=76206 dup_rows=11204 dup_rows_remote=9522 dedup_ratio=0.113973 router_tokens_per_expert=[1042, 2552, 6210, 863, 0, 5628, 24, 0, 0, 0, 0, 0, 1, 117, 0, 0, 0, 0, 0, 4564, 0, 0, 0, 340, 7, 254, 1183, 2842, 8192, 0, 221, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1617, 4352, 0, 0, 262, 0, 0, 0, 0, 4647, 0, 953, 0, 0, 0, 0, 0, 19, 1302, 0, 1951, 0, 0, 4] payload_rows_per_dst=[12336, 16384, 10894, 5652, 12420, 10442, 10612, 8360] dup_rows_per_dst=[4602, 236, 1682, 594, 10, 2020, 588, 1472] combine_rows_per_src=[12336, 16384, 10894, 5652, 12420, 10442, 10612, 8360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=317497344 dispatch_meta_bytes=341312 combine_payload_bytes=317497344 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=317838656 ingress_bytes=421306368 link_bytes=421306368 token_bytes=635336000 weight_movement_bytes=103809024 total_nvlink_bytes=739145024 weight_to_token_ratio=0.163392 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=32896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.585341945e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=317718528 dispatch_meta_bytes=344152 combine_payload_bytes=317718528 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=318062680 ingress_bytes=386924544 link_bytes=386924544 token_bytes=635781208 weight_movement_bytes=69206016 total_nvlink_bytes=704987224 weight_to_token_ratio=0.108852 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=47488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.455965909e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=278601728 dispatch_meta_bytes=331936 combine_payload_bytes=278601728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278933664 ingress_bytes=313204736 link_bytes=313204736 token_bytes=557535392 weight_movement_bytes=34603008 total_nvlink_bytes=592138400 weight_to_token_ratio=0.062064 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=26112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.178564207e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81764 payload_rows_remote=66466 dup_rows=16540 dup_rows_remote=9410 dedup_ratio=0.168254 router_tokens_per_expert=[0, 0, 0, 7979, 59, 2948, 0, 0, 449, 0, 6860, 0, 0, 0, 0, 206, 0, 1452, 37, 0, 0, 0, 0, 0, 1, 3, 0, 0, 6, 154, 0, 5353, 0, 0, 1, 1, 1161, 0, 0, 0, 0, 835, 458, 8192, 1183, 0, 0, 6421, 0, 4002, 0, 20, 0, 0, 0, 332, 0, 0, 451, 324, 194, 70, 0, 0] payload_rows_per_dst=[6000, 1310, 16106, 10844, 2324, 16384, 15298, 13498] dup_rows_per_dst=[14, 0, 2830, 190, 2, 4952, 7130, 1422] combine_rows_per_src=[6000, 1310, 16106, 10844, 2324, 16384, 15298, 13498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=312139776 dispatch_meta_bytes=342912 combine_payload_bytes=312139776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=312482688 ingress_bytes=346742784 link_bytes=346742784 token_bytes=624622464 weight_movement_bytes=34603008 total_nvlink_bytes=659225472 weight_to_token_ratio=0.055398 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=38784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.304765181e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=272244736 dispatch_meta_bytes=303504 combine_payload_bytes=272244736 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272548240 ingress_bytes=306847744 link_bytes=306847744 token_bytes=544792976 weight_movement_bytes=34603008 total_nvlink_bytes=579395984 weight_to_token_ratio=0.063516 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=32128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.154643357e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91112 payload_rows_remote=80676 dup_rows=7192 dup_rows_remote=7038 dedup_ratio=0.073161 router_tokens_per_expert=[0, 0, 4, 7928, 1, 3137, 0, 2, 391, 0, 6798, 0, 0, 1, 0, 177, 0, 1275, 46, 1, 2, 0, 0, 2, 0, 1, 0, 0, 5, 137, 1, 5151, 1, 0, 1, 0, 957, 2, 1, 0, 0, 666, 475, 8187, 1484, 0, 0, 6623, 0, 4342, 0, 0, 0, 0, 0, 387, 0, 0, 453, 282, 161, 70, 0, 0] payload_rows_per_dst=[6284, 14204, 16070, 10436, 16380, 5074, 8852, 13812] dup_rows_per_dst=[4, 530, 2438, 154, 1918, 176, 606, 1366] combine_rows_per_src=[6284, 14204, 16070, 10436, 16380, 5074, 8852, 13812] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86298 payload_rows_remote=72496 dup_rows=12006 dup_rows_remote=11518 dedup_ratio=0.122131 router_tokens_per_expert=[0, 0, 0, 7924, 7, 3164, 0, 0, 363, 0, 6599, 0, 0, 0, 0, 183, 0, 1485, 11, 1, 0, 0, 0, 0, 0, 0, 0, 2, 2, 99, 0, 5914, 0, 0, 0, 0, 767, 0, 0, 0, 0, 675, 393, 8192, 1272, 0, 0, 6889, 0, 4039, 0, 3, 0, 0, 0, 356, 0, 0, 489, 198, 82, 43, 0, 0] payload_rows_per_dst=[16026, 13802, 2994, 11898, 16384, 13002, 8250, 3942] dup_rows_per_dst=[6164, 488, 0, 136, 1534, 2934, 546, 204] combine_rows_per_src=[16026, 13802, 2994, 11898, 16384, 13002, 8250, 3942] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88202 payload_rows_remote=81094 dup_rows=10102 dup_rows_remote=9240 dedup_ratio=0.102763 router_tokens_per_expert=[0, 0, 0, 7937, 80, 3007, 0, 0, 345, 0, 6777, 0, 0, 0, 0, 278, 0, 1667, 47, 5, 0, 0, 0, 0, 0, 1, 0, 0, 8, 152, 0, 4820, 0, 0, 0, 0, 1043, 0, 0, 0, 1, 857, 424, 8192, 1314, 0, 0, 6789, 0, 3943, 0, 0, 0, 0, 0, 244, 0, 0, 646, 324, 208, 43, 0, 0] payload_rows_per_dst=[13306, 14228, 7108, 9814, 16384, 5034, 8036, 14292] dup_rows_per_dst=[4210, 572, 862, 148, 2086, 158, 338, 1728] combine_rows_per_src=[13306, 14228, 7108, 9814, 16384, 5034, 8036, 14292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73112 payload_rows_remote=56996 dup_rows=25192 dup_rows_remote=14718 dedup_ratio=0.256266 router_tokens_per_expert=[0, 0, 0, 7943, 28, 3169, 0, 0, 364, 0, 6588, 0, 0, 0, 0, 227, 0, 1140, 32, 0, 0, 0, 0, 0, 0, 5, 0, 1, 3, 191, 0, 5137, 1, 0, 9, 0, 859, 2, 0, 1, 3, 819, 645, 8185, 1553, 0, 0, 6592, 0, 4106, 0, 55, 0, 0, 0, 323, 0, 0, 625, 300, 203, 43, 0, 0] payload_rows_per_dst=[16116, 15376, 2340, 10482, 3380, 14782, 8362, 2274] dup_rows_per_dst=[10474, 9186, 4, 192, 220, 4442, 606, 68] combine_rows_per_src=[16116, 15376, 2340, 10482, 3380, 14782, 8362, 2274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90598 payload_rows_remote=74216 dup_rows=7706 dup_rows_remote=5558 dedup_ratio=0.078389 router_tokens_per_expert=[0, 0, 0, 7988, 20, 3317, 0, 0, 358, 0, 6667, 0, 0, 0, 0, 144, 0, 1706, 16, 5, 0, 0, 0, 0, 0, 0, 0, 0, 2, 164, 0, 5284, 0, 0, 0, 0, 1074, 0, 0, 0, 0, 599, 385, 8191, 1403, 0, 0, 6868, 0, 3865, 0, 0, 0, 0, 0, 329, 0, 0, 414, 185, 142, 26, 0, 0] payload_rows_per_dst=[6668, 13946, 16172, 10712, 16382, 4674, 7862, 14182] dup_rows_per_dst=[6, 392, 3258, 188, 2148, 100, 526, 1088] combine_rows_per_src=[6668, 13946, 16172, 10712, 16382, 4674, 7862, 14182] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91278 payload_rows_remote=86522 dup_rows=7026 dup_rows_remote=6798 dedup_ratio=0.071472 router_tokens_per_expert=[0, 0, 0, 7939, 35, 3194, 0, 0, 337, 0, 6810, 0, 0, 0, 0, 166, 0, 1340, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 162, 0, 5832, 0, 0, 0, 0, 751, 0, 0, 0, 0, 722, 334, 8192, 1243, 0, 0, 6811, 0, 4157, 0, 1, 0, 0, 0, 325, 0, 0, 311, 284, 136, 32, 0, 0] payload_rows_per_dst=[6448, 12872, 16078, 11792, 16070, 4756, 9174, 14088] dup_rows_per_dst=[10, 394, 2544, 208, 1430, 228, 1152, 1060] combine_rows_per_src=[6448, 12872, 16078, 11792, 16070, 4756, 9174, 14088] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91112 payload_rows_remote=80676 dup_rows=7192 dup_rows_remote=7038 dedup_ratio=0.073161 router_tokens_per_expert=[0, 0, 4, 7928, 1, 3137, 0, 2, 391, 0, 6798, 0, 0, 1, 0, 177, 0, 1275, 46, 1, 2, 0, 0, 2, 0, 1, 0, 0, 5, 137, 1, 5151, 1, 0, 1, 0, 957, 2, 1, 0, 0, 666, 475, 8187, 1484, 0, 0, 6623, 0, 4342, 0, 0, 0, 0, 0, 387, 0, 0, 453, 282, 161, 70, 0, 0] payload_rows_per_dst=[6284, 14204, 16070, 10436, 16380, 5074, 8852, 13812] dup_rows_per_dst=[4, 530, 2438, 154, 1918, 176, 606, 1366] combine_rows_per_src=[6284, 14204, 16070, 10436, 16380, 5074, 8852, 13812] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78128 payload_rows_remote=64030 dup_rows=20176 dup_rows_remote=18994 dedup_ratio=0.205241 router_tokens_per_expert=[0, 0, 0, 8030, 47, 3240, 0, 0, 362, 0, 6837, 1, 0, 0, 0, 139, 0, 1385, 15, 2, 0, 1, 0, 0, 0, 0, 0, 0, 1, 176, 0, 5695, 0, 0, 1, 0, 829, 0, 0, 1, 0, 851, 437, 8181, 1335, 0, 0, 6777, 0, 3647, 0, 10, 0, 0, 0, 289, 0, 0, 369, 272, 197, 25, 0, 0] payload_rows_per_dst=[6568, 1004, 8538, 14590, 1662, 16366, 15302, 14098] dup_rows_per_dst=[6, 0, 962, 6520, 0, 5242, 6264, 1182] combine_rows_per_src=[6568, 1004, 8538, 14590, 1662, 16366, 15302, 14098] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=330448896 dispatch_meta_bytes=350856 combine_payload_bytes=330448896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=330799752 ingress_bytes=365051904 link_bytes=365051904 token_bytes=661248648 weight_movement_bytes=34603008 total_nvlink_bytes=695851656 weight_to_token_ratio=0.052330 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=43264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.373660926e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88202 payload_rows_remote=81094 dup_rows=10102 dup_rows_remote=9240 dedup_ratio=0.102763 router_tokens_per_expert=[0, 0, 0, 7937, 80, 3007, 0, 0, 345, 0, 6777, 0, 0, 0, 0, 278, 0, 1667, 47, 5, 0, 0, 0, 0, 0, 1, 0, 0, 8, 152, 0, 4820, 0, 0, 0, 0, 1043, 0, 0, 0, 1, 857, 424, 8192, 1314, 0, 0, 6789, 0, 3943, 0, 0, 0, 0, 0, 244, 0, 0, 646, 324, 208, 43, 0, 0] payload_rows_per_dst=[13306, 14228, 7108, 9814, 16384, 5034, 8036, 14292] dup_rows_per_dst=[4210, 572, 862, 148, 2086, 158, 338, 1728] combine_rows_per_src=[13306, 14228, 7108, 9814, 16384, 5034, 8036, 14292] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=286113792 dispatch_meta_bytes=342432 combine_payload_bytes=286113792 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=286456224 ingress_bytes=320716800 link_bytes=320716800 token_bytes=572570016 weight_movement_bytes=34603008 total_nvlink_bytes=607173024 weight_to_token_ratio=0.060435 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=41984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.206831499e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=296943616 dispatch_meta_bytes=336056 combine_payload_bytes=296943616 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=297279672 ingress_bytes=366149632 link_bytes=366149632 token_bytes=594223288 weight_movement_bytes=69206016 total_nvlink_bytes=663429304 weight_to_token_ratio=0.116465 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=39680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.377791588e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90598 payload_rows_remote=74216 dup_rows=7706 dup_rows_remote=5558 dedup_ratio=0.078389 router_tokens_per_expert=[0, 0, 0, 7988, 20, 3317, 0, 0, 358, 0, 6667, 0, 0, 0, 0, 144, 0, 1706, 16, 5, 0, 0, 0, 0, 0, 0, 0, 0, 2, 164, 0, 5284, 0, 0, 0, 0, 1074, 0, 0, 0, 0, 599, 385, 8191, 1403, 0, 0, 6868, 0, 3865, 0, 0, 0, 0, 0, 329, 0, 0, 414, 185, 142, 26, 0, 0] payload_rows_per_dst=[6668, 13946, 16172, 10712, 16382, 4674, 7862, 14182] dup_rows_per_dst=[6, 392, 3258, 188, 2148, 100, 526, 1088] combine_rows_per_src=[6668, 13946, 16172, 10712, 16382, 4674, 7862, 14182] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73112 payload_rows_remote=56996 dup_rows=25192 dup_rows_remote=14718 dedup_ratio=0.256266 router_tokens_per_expert=[0, 0, 0, 7943, 28, 3169, 0, 0, 364, 0, 6588, 0, 0, 0, 0, 227, 0, 1140, 32, 0, 0, 0, 0, 0, 0, 5, 0, 1, 3, 191, 0, 5137, 1, 0, 9, 0, 859, 2, 0, 1, 3, 819, 645, 8185, 1553, 0, 0, 6592, 0, 4106, 0, 55, 0, 0, 0, 323, 0, 0, 625, 300, 203, 43, 0, 0] payload_rows_per_dst=[16116, 15376, 2340, 10482, 3380, 14782, 8362, 2274] dup_rows_per_dst=[10474, 9186, 4, 192, 220, 4442, 606, 68] combine_rows_per_src=[16116, 15376, 2340, 10482, 3380, 14782, 8362, 2274] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=332161024 dispatch_meta_bytes=361336 combine_payload_bytes=332161024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=332522360 ingress_bytes=366764032 link_bytes=366764032 token_bytes=664683384 weight_movement_bytes=34603008 total_nvlink_bytes=699286392 weight_to_token_ratio=0.052059 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=37504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.380103525e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=233455616 dispatch_meta_bytes=286856 combine_payload_bytes=233455616 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=233742472 ingress_bytes=337264640 link_bytes=337264640 token_bytes=467198088 weight_movement_bytes=103809024 total_nvlink_bytes=571007112 weight_to_token_ratio=0.222195 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=25216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.269099688e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=303988736 dispatch_meta_bytes=319096 combine_payload_bytes=303988736 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=304307832 ingress_bytes=338591744 link_bytes=338591744 token_bytes=608296568 weight_movement_bytes=34603008 total_nvlink_bytes=642899576 weight_to_token_ratio=0.056885 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=41728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274093474e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68212 payload_rows_remote=53470 dup_rows=30092 dup_rows_remote=25080 dedup_ratio=0.306112 router_tokens_per_expert=[0, 8081, 21, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1455, 0, 6598, 0, 1, 0, 8, 176, 0, 2158, 2, 2357, 0, 5, 0, 1236, 473, 3, 8, 7067, 0, 0, 172, 0, 0, 0, 250, 1, 6764, 0, 0, 117, 0, 0, 0, 0, 0, 0, 26, 0, 2479, 0, 305, 0, 0, 0, 0, 8177, 1211, 0, 0, 0] payload_rows_per_dst=[44, 2910, 16204, 3424, 846, 13660, 14742, 16382] dup_rows_per_dst=[0, 0, 9362, 26, 0, 102, 5012, 15590] combine_rows_per_src=[44, 2910, 16204, 3424, 846, 13660, 14742, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=354394112 dispatch_meta_bytes=373280 combine_payload_bytes=354394112 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=354767392 ingress_bytes=492806144 link_bytes=492806144 token_bytes=709161504 weight_movement_bytes=138412032 total_nvlink_bytes=847573536 weight_to_token_ratio=0.195177 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=16640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.854389846e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=330448896 dispatch_meta_bytes=350856 combine_payload_bytes=330448896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=330799752 ingress_bytes=365051904 link_bytes=365051904 token_bytes=661248648 weight_movement_bytes=34603008 total_nvlink_bytes=695851656 weight_to_token_ratio=0.052330 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=43264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.373660926e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=262266880 dispatch_meta_bytes=332096 combine_payload_bytes=262266880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262598976 ingress_bytes=296869888 link_bytes=296869888 token_bytes=524865856 weight_movement_bytes=34603008 total_nvlink_bytes=559468864 weight_to_token_ratio=0.065927 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=41600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.117097489e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=303988736 dispatch_meta_bytes=319096 combine_payload_bytes=303988736 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=304307832 ingress_bytes=338591744 link_bytes=338591744 token_bytes=608296568 weight_movement_bytes=34603008 total_nvlink_bytes=642899576 weight_to_token_ratio=0.056885 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=41728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274093474e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=233455616 dispatch_meta_bytes=286856 combine_payload_bytes=233455616 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=233742472 ingress_bytes=337264640 link_bytes=337264640 token_bytes=467198088 weight_movement_bytes=103809024 total_nvlink_bytes=571007112 weight_to_token_ratio=0.222195 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=25216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.269099688e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81764 payload_rows_remote=66466 dup_rows=16540 dup_rows_remote=9410 dedup_ratio=0.168254 router_tokens_per_expert=[0, 0, 0, 7979, 59, 2948, 0, 0, 449, 0, 6860, 0, 0, 0, 0, 206, 0, 1452, 37, 0, 0, 0, 0, 0, 1, 3, 0, 0, 6, 154, 0, 5353, 0, 0, 1, 1, 1161, 0, 0, 0, 0, 835, 458, 8192, 1183, 0, 0, 6421, 0, 4002, 0, 20, 0, 0, 0, 332, 0, 0, 451, 324, 194, 70, 0, 0] payload_rows_per_dst=[6000, 1310, 16106, 10844, 2324, 16384, 15298, 13498] dup_rows_per_dst=[14, 0, 2830, 190, 2, 4952, 7130, 1422] combine_rows_per_src=[6000, 1310, 16106, 10844, 2324, 16384, 15298, 13498] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89274 payload_rows_remote=74832 dup_rows=9030 dup_rows_remote=6268 dedup_ratio=0.091858 router_tokens_per_expert=[0, 8066, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1693, 0, 6909, 0, 0, 0, 4, 247, 0, 1898, 0, 1840, 0, 11, 0, 1135, 434, 0, 0, 7002, 0, 0, 333, 0, 0, 0, 153, 0, 7335, 0, 0, 159, 0, 0, 0, 0, 0, 0, 21, 0, 2531, 0, 183, 0, 0, 0, 0, 8188, 1002, 0, 0, 0] payload_rows_per_dst=[16132, 14442, 7486, 14554, 16376, 12880, 5400, 2004] dup_rows_per_dst=[16, 2762, 492, 4466, 972, 252, 70, 0] combine_rows_per_src=[16132, 14442, 7486, 14554, 16376, 12880, 5400, 2004] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91174 payload_rows_remote=83784 dup_rows=7130 dup_rows_remote=6760 dedup_ratio=0.072530 router_tokens_per_expert=[0, 8041, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1329, 0, 6799, 0, 0, 0, 3, 207, 0, 1920, 0, 1750, 0, 5, 0, 1175, 449, 3, 4, 7208, 0, 0, 288, 1, 0, 0, 237, 0, 6658, 0, 0, 225, 0, 0, 0, 0, 0, 0, 53, 0, 2886, 0, 432, 0, 0, 0, 0, 8176, 1252, 0, 0, 0] payload_rows_per_dst=[16086, 14060, 7390, 14592, 16352, 13628, 6562, 2504] dup_rows_per_dst=[98, 2196, 370, 3096, 1052, 138, 180, 0] combine_rows_per_src=[16086, 14060, 7390, 14592, 16352, 13628, 6562, 2504] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82030 payload_rows_remote=67810 dup_rows=16274 dup_rows_remote=16142 dedup_ratio=0.165548 router_tokens_per_expert=[0, 8092, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1554, 0, 7104, 0, 0, 0, 12, 216, 0, 1478, 0, 2174, 0, 7, 0, 1012, 362, 2, 2, 7109, 0, 0, 441, 0, 0, 0, 223, 0, 7031, 0, 0, 145, 0, 0, 0, 0, 0, 0, 19, 0, 2559, 0, 260, 0, 0, 0, 0, 8179, 1148, 0, 0, 0] payload_rows_per_dst=[15818, 5622, 7540, 2758, 8112, 14220, 14842, 13118] dup_rows_per_dst=[44, 384, 588, 12, 596, 132, 5052, 9466] combine_rows_per_src=[15818, 5622, 7540, 2758, 8112, 14220, 14842, 13118] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=332161024 dispatch_meta_bytes=361336 combine_payload_bytes=332161024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=332522360 ingress_bytes=366764032 link_bytes=366764032 token_bytes=664683384 weight_movement_bytes=34603008 total_nvlink_bytes=699286392 weight_to_token_ratio=0.052059 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=37504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.380103525e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=219013120 dispatch_meta_bytes=314200 combine_payload_bytes=219013120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=219327320 ingress_bytes=253616128 link_bytes=253616128 token_bytes=438340440 weight_movement_bytes=34603008 total_nvlink_bytes=472943448 weight_to_token_ratio=0.078941 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=26112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.543370719e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90482 payload_rows_remote=76212 dup_rows=7822 dup_rows_remote=5230 dedup_ratio=0.079569 router_tokens_per_expert=[0, 8064, 32, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1644, 1, 7142, 1, 3, 0, 8, 228, 0, 1866, 0, 2220, 0, 8, 0, 1212, 324, 2, 10, 6875, 0, 0, 486, 0, 0, 0, 190, 0, 6747, 0, 0, 167, 0, 0, 0, 0, 3, 6, 16, 0, 2265, 2, 287, 0, 0, 0, 0, 8183, 1159, 0, 0, 0] payload_rows_per_dst=[16138, 14796, 7836, 14270, 16368, 13670, 5086, 2318] dup_rows_per_dst=[56, 2780, 814, 2592, 1350, 158, 72, 0] combine_rows_per_src=[16138, 14796, 7836, 14270, 16368, 13670, 5086, 2318] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69064 payload_rows_remote=52680 dup_rows=29240 dup_rows_remote=13150 dedup_ratio=0.297445 router_tokens_per_expert=[0, 8059, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1439, 0, 7016, 0, 0, 0, 2, 284, 0, 1334, 0, 2077, 0, 4, 0, 1197, 355, 4, 3, 7202, 0, 0, 299, 0, 0, 0, 245, 1, 7188, 0, 0, 114, 0, 0, 0, 0, 0, 0, 53, 0, 2621, 0, 410, 0, 0, 0, 0, 8184, 1035, 0, 0, 2] payload_rows_per_dst=[48, 2878, 16146, 3110, 1090, 14454, 14954, 16384] dup_rows_per_dst=[0, 0, 7366, 16, 0, 150, 5618, 16090] combine_rows_per_src=[48, 2878, 16146, 3110, 1090, 14454, 14954, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=272244736 dispatch_meta_bytes=303504 combine_payload_bytes=272244736 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272548240 ingress_bytes=306847744 link_bytes=306847744 token_bytes=544792976 weight_movement_bytes=34603008 total_nvlink_bytes=579395984 weight_to_token_ratio=0.063516 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=32128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.154643357e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=306511872 dispatch_meta_bytes=324400 combine_payload_bytes=306511872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=306836272 ingress_bytes=375717888 link_bytes=375717888 token_bytes=613348144 weight_movement_bytes=69206016 total_nvlink_bytes=682554160 weight_to_token_ratio=0.112833 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=35968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.413796165e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89530 payload_rows_remote=73180 dup_rows=8774 dup_rows_remote=7508 dedup_ratio=0.089254 router_tokens_per_expert=[0, 8086, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1585, 0, 7013, 0, 0, 0, 0, 311, 0, 1824, 0, 2003, 0, 13, 0, 1278, 554, 2, 5, 6844, 0, 0, 457, 0, 0, 0, 177, 0, 7136, 0, 0, 144, 0, 0, 0, 0, 0, 0, 7, 0, 2301, 0, 189, 0, 0, 0, 0, 8174, 1015, 0, 0, 0] payload_rows_per_dst=[16172, 14528, 7642, 7060, 16350, 14302, 11446, 2030] dup_rows_per_dst=[68, 2668, 634, 868, 1266, 258, 3012, 0] combine_rows_per_src=[16172, 14528, 7642, 7060, 16350, 14302, 11446, 2030] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89274 payload_rows_remote=74832 dup_rows=9030 dup_rows_remote=6268 dedup_ratio=0.091858 router_tokens_per_expert=[0, 8066, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1693, 0, 6909, 0, 0, 0, 4, 247, 0, 1898, 0, 1840, 0, 11, 0, 1135, 434, 0, 0, 7002, 0, 0, 333, 0, 0, 0, 153, 0, 7335, 0, 0, 159, 0, 0, 0, 0, 0, 0, 21, 0, 2531, 0, 183, 0, 0, 0, 0, 8188, 1002, 0, 0, 0] payload_rows_per_dst=[16132, 14442, 7486, 14554, 16376, 12880, 5400, 2004] dup_rows_per_dst=[16, 2762, 492, 4466, 972, 252, 70, 0] combine_rows_per_src=[16132, 14442, 7486, 14554, 16376, 12880, 5400, 2004] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79232 payload_rows_remote=63046 dup_rows=19072 dup_rows_remote=17674 dedup_ratio=0.194010 router_tokens_per_expert=[0, 8084, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1202, 0, 6648, 0, 0, 0, 3, 174, 0, 1833, 0, 2207, 0, 7, 0, 1194, 631, 0, 3, 7308, 0, 2, 355, 0, 0, 0, 245, 0, 6560, 0, 0, 150, 0, 0, 0, 0, 0, 1, 42, 0, 2800, 0, 285, 0, 0, 0, 1, 8185, 1208, 0, 0, 0] payload_rows_per_dst=[16186, 14148, 7900, 15778, 16370, 300, 6134, 2416] dup_rows_per_dst=[1398, 4130, 534, 11682, 1204, 0, 122, 2] combine_rows_per_src=[16186, 14148, 7900, 15778, 16370, 300, 6134, 2416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89530 payload_rows_remote=73180 dup_rows=8774 dup_rows_remote=7508 dedup_ratio=0.089254 router_tokens_per_expert=[0, 8086, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1585, 0, 7013, 0, 0, 0, 0, 311, 0, 1824, 0, 2003, 0, 13, 0, 1278, 554, 2, 5, 6844, 0, 0, 457, 0, 0, 0, 177, 0, 7136, 0, 0, 144, 0, 0, 0, 0, 0, 0, 7, 0, 2301, 0, 189, 0, 0, 0, 0, 8174, 1015, 0, 0, 0] payload_rows_per_dst=[16172, 14528, 7642, 7060, 16350, 14302, 11446, 2030] dup_rows_per_dst=[68, 2668, 634, 868, 1266, 258, 3012, 0] combine_rows_per_src=[16172, 14528, 7642, 7060, 16350, 14302, 11446, 2030] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=343179264 dispatch_meta_bytes=362176 combine_payload_bytes=343179264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=343541440 ingress_bytes=377782272 link_bytes=377782272 token_bytes=686720704 weight_movement_bytes=34603008 total_nvlink_bytes=721323712 weight_to_token_ratio=0.050389 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=16640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.421564276e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=277749760 dispatch_meta_bytes=335808 combine_payload_bytes=277749760 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=278085568 ingress_bytes=381558784 link_bytes=381558784 token_bytes=555835328 weight_movement_bytes=103809024 total_nvlink_bytes=659644352 weight_to_token_ratio=0.186762 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.037500 gemm_active_groups=3 gemm_peak_group_rows=48000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.435774986e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69064 payload_rows_remote=52680 dup_rows=29240 dup_rows_remote=13150 dedup_ratio=0.297445 router_tokens_per_expert=[0, 8059, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1439, 0, 7016, 0, 0, 0, 2, 284, 0, 1334, 0, 2077, 0, 4, 0, 1197, 355, 4, 3, 7202, 0, 0, 299, 0, 0, 0, 245, 1, 7188, 0, 0, 114, 0, 0, 0, 0, 0, 0, 53, 0, 2621, 0, 410, 0, 0, 0, 0, 8184, 1035, 0, 0, 2] payload_rows_per_dst=[48, 2878, 16146, 3110, 1090, 14454, 14954, 16384] dup_rows_per_dst=[0, 0, 7366, 16, 0, 150, 5618, 16090] combine_rows_per_src=[48, 2878, 16146, 3110, 1090, 14454, 14954, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82030 payload_rows_remote=67810 dup_rows=16274 dup_rows_remote=16142 dedup_ratio=0.165548 router_tokens_per_expert=[0, 8092, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1554, 0, 7104, 0, 0, 0, 12, 216, 0, 1478, 0, 2174, 0, 7, 0, 1012, 362, 2, 2, 7109, 0, 0, 441, 0, 0, 0, 223, 0, 7031, 0, 0, 145, 0, 0, 0, 0, 0, 0, 19, 0, 2559, 0, 260, 0, 0, 0, 0, 8179, 1148, 0, 0, 0] payload_rows_per_dst=[15818, 5622, 7540, 2758, 8112, 14220, 14842, 13118] dup_rows_per_dst=[44, 384, 588, 12, 596, 132, 5052, 9466] combine_rows_per_src=[15818, 5622, 7540, 2758, 8112, 14220, 14842, 13118] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91174 payload_rows_remote=83784 dup_rows=7130 dup_rows_remote=6760 dedup_ratio=0.072530 router_tokens_per_expert=[0, 8041, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1329, 0, 6799, 0, 0, 0, 3, 207, 0, 1920, 0, 1750, 0, 5, 0, 1175, 449, 3, 4, 7208, 0, 0, 288, 1, 0, 0, 237, 0, 6658, 0, 0, 225, 0, 0, 0, 0, 0, 0, 53, 0, 2886, 0, 432, 0, 0, 0, 0, 8176, 1252, 0, 0, 0] payload_rows_per_dst=[16086, 14060, 7390, 14592, 16352, 13628, 6562, 2504] dup_rows_per_dst=[98, 2196, 370, 3096, 1052, 138, 180, 0] combine_rows_per_src=[16086, 14060, 7390, 14592, 16352, 13628, 6562, 2504] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=312164352 dispatch_meta_bytes=325768 combine_payload_bytes=312164352 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=312490120 ingress_bytes=381370368 link_bytes=381370368 token_bytes=624654472 weight_movement_bytes=69206016 total_nvlink_bytes=693860488 weight_to_token_ratio=0.110791 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=30592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.435065992e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=215777280 dispatch_meta_bytes=263320 combine_payload_bytes=215777280 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=216040600 ingress_bytes=284983296 link_bytes=284983296 token_bytes=431817880 weight_movement_bytes=69206016 total_nvlink_bytes=501023896 weight_to_token_ratio=0.160267 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=20864 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.072369200e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=306511872 dispatch_meta_bytes=324400 combine_payload_bytes=306511872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=306836272 ingress_bytes=375717888 link_bytes=375717888 token_bytes=613348144 weight_movement_bytes=69206016 total_nvlink_bytes=682554160 weight_to_token_ratio=0.112833 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=35968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.413796165e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=299745280 dispatch_meta_bytes=322752 combine_payload_bytes=299745280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=300068032 ingress_bytes=334348288 link_bytes=334348288 token_bytes=599813312 weight_movement_bytes=34603008 total_nvlink_bytes=634416320 weight_to_token_ratio=0.057690 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=44672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258125690e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=258236416 dispatch_meta_bytes=322880 combine_payload_bytes=258236416 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=258559296 ingress_bytes=327442432 link_bytes=327442432 token_bytes=516795712 weight_movement_bytes=69206016 total_nvlink_bytes=586001728 weight_to_token_ratio=0.133914 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=48256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.232139510e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68212 payload_rows_remote=53470 dup_rows=30092 dup_rows_remote=25080 dedup_ratio=0.306112 router_tokens_per_expert=[0, 8081, 21, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1455, 0, 6598, 0, 1, 0, 8, 176, 0, 2158, 2, 2357, 0, 5, 0, 1236, 473, 3, 8, 7067, 0, 0, 172, 0, 0, 0, 250, 1, 6764, 0, 0, 117, 0, 0, 0, 0, 0, 0, 26, 0, 2479, 0, 305, 0, 0, 0, 0, 8177, 1211, 0, 0, 0] payload_rows_per_dst=[44, 2910, 16204, 3424, 846, 13660, 14742, 16382] dup_rows_per_dst=[0, 0, 9362, 26, 0, 102, 5012, 15590] combine_rows_per_src=[44, 2910, 16204, 3424, 846, 13660, 14742, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78140 payload_rows_remote=62506 dup_rows=20164 dup_rows_remote=20158 dedup_ratio=0.205119 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1881, 0, 0, 0, 0, 2716, 0, 3025, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 1682, 403, 0, 0, 0, 4, 0, 0, 5029, 0, 6684, 8189, 0, 0, 1107, 1263, 0, 0, 0, 3, 0, 0, 0, 7817, 0, 0, 0, 246, 0, 175, 0, 0, 734, 0] payload_rows_per_dst=[14, 8594, 3086, 7988, 10066, 16374, 15634, 16384] dup_rows_per_dst=[0, 600, 0, 1328, 0, 15922, 6, 2308] combine_rows_per_src=[14, 8594, 3086, 7988, 10066, 16374, 15634, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=343179264 dispatch_meta_bytes=362176 combine_payload_bytes=343179264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=343541440 ingress_bytes=377782272 link_bytes=377782272 token_bytes=686720704 weight_movement_bytes=34603008 total_nvlink_bytes=721323712 weight_to_token_ratio=0.050389 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=16640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.421564276e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=215777280 dispatch_meta_bytes=263320 combine_payload_bytes=215777280 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=216040600 ingress_bytes=284983296 link_bytes=284983296 token_bytes=431817880 weight_movement_bytes=69206016 total_nvlink_bytes=501023896 weight_to_token_ratio=0.160267 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=20864 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.072369200e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=277749760 dispatch_meta_bytes=335808 combine_payload_bytes=277749760 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=278085568 ingress_bytes=381558784 link_bytes=381558784 token_bytes=555835328 weight_movement_bytes=103809024 total_nvlink_bytes=659644352 weight_to_token_ratio=0.186762 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.037500 gemm_active_groups=3 gemm_peak_group_rows=48000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.435774986e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=256024576 dispatch_meta_bytes=330656 combine_payload_bytes=256024576 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=256355232 ingress_bytes=325230592 link_bytes=325230592 token_bytes=512379808 weight_movement_bytes=69206016 total_nvlink_bytes=581585824 weight_to_token_ratio=0.135068 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.223816534e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77820 payload_rows_remote=64438 dup_rows=20484 dup_rows_remote=15208 dedup_ratio=0.208374 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2146, 0, 0, 0, 0, 2485, 0, 3443, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2165, 301, 0, 0, 0, 10, 0, 0, 4630, 0, 6779, 8192, 0, 0, 1064, 923, 0, 0, 0, 0, 0, 0, 0, 7845, 0, 0, 0, 159, 0, 142, 0, 0, 675, 0] payload_rows_per_dst=[16384, 13382, 16384, 7966, 14002, 3230, 4538, 1934] dup_rows_per_dst=[0, 5276, 6886, 1130, 6430, 744, 0, 18] combine_rows_per_src=[16384, 13382, 16384, 7966, 14002, 3230, 4538, 1934] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90260 payload_rows_remote=87108 dup_rows=8044 dup_rows_remote=7288 dedup_ratio=0.081828 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2535, 0, 0, 0, 0, 2602, 0, 3507, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1824, 384, 0, 0, 0, 17, 0, 0, 3865, 1, 6718, 8192, 0, 0, 997, 957, 0, 0, 0, 0, 0, 0, 0, 7790, 0, 0, 0, 265, 0, 511, 0, 0, 791, 0] payload_rows_per_dst=[16384, 9514, 7014, 14472, 7760, 3152, 15580, 16384] dup_rows_per_dst=[0, 760, 0, 3388, 6, 756, 0, 3134] combine_rows_per_src=[16384, 9514, 7014, 14472, 7760, 3152, 15580, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89512 payload_rows_remote=75312 dup_rows=8792 dup_rows_remote=5354 dedup_ratio=0.089437 router_tokens_per_expert=[2, 0, 0, 0, 0, 0, 0, 0, 0, 2167, 0, 0, 0, 0, 2347, 0, 3934, 8189, 1, 2, 0, 0, 0, 0, 0, 1, 3, 0, 0, 1, 2038, 359, 0, 0, 0, 4, 0, 0, 4079, 0, 6417, 8186, 0, 0, 1211, 988, 3, 0, 0, 0, 0, 0, 2, 7850, 0, 4, 0, 265, 2, 461, 2, 0, 633, 1] payload_rows_per_dst=[16376, 8536, 9392, 14200, 8164, 3672, 15708, 13464] dup_rows_per_dst=[0, 492, 1860, 3438, 2, 732, 4, 2264] combine_rows_per_src=[16376, 8536, 9392, 14200, 8164, 3672, 15708, 13464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91528 payload_rows_remote=80816 dup_rows=6776 dup_rows_remote=6766 dedup_ratio=0.068929 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1903, 0, 0, 1, 0, 2530, 0, 2933, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1818, 238, 0, 0, 0, 16, 0, 0, 5344, 1, 6936, 8191, 0, 0, 1153, 992, 0, 0, 0, 0, 0, 0, 0, 7818, 0, 0, 0, 276, 0, 227, 0, 0, 582, 0] payload_rows_per_dst=[16382, 8364, 5866, 14704, 10712, 3480, 15636, 16384] dup_rows_per_dst=[0, 504, 0, 3282, 10, 810, 0, 2170] combine_rows_per_src=[16382, 8364, 5866, 14704, 10712, 3480, 15636, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73360 payload_rows_remote=56980 dup_rows=24944 dup_rows_remote=24944 dedup_ratio=0.253743 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2036, 0, 0, 0, 0, 2433, 0, 3225, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1877, 259, 0, 1, 0, 13, 0, 0, 4930, 0, 6780, 8190, 0, 0, 1167, 983, 0, 0, 0, 0, 0, 0, 0, 7821, 0, 0, 0, 286, 0, 224, 0, 0, 731, 0] payload_rows_per_dst=[16380, 15444, 16384, 4218, 15034, 3484, 0, 2416] dup_rows_per_dst=[0, 7054, 8302, 62, 8644, 816, 0, 66] combine_rows_per_src=[16380, 15444, 16384, 4218, 15034, 3484, 0, 2416] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=263938048 dispatch_meta_bytes=318584 combine_payload_bytes=263938048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264256632 ingress_bytes=298541056 link_bytes=298541056 token_bytes=528194680 weight_movement_bytes=34603008 total_nvlink_bytes=562797688 weight_to_token_ratio=0.065512 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=20224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123385959e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86010 payload_rows_remote=69626 dup_rows=12294 dup_rows_remote=5550 dedup_ratio=0.125061 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2786, 0, 0, 1, 0, 2739, 0, 3372, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1921, 546, 0, 0, 0, 4, 0, 0, 3946, 2, 6343, 8192, 0, 0, 916, 899, 0, 0, 0, 0, 0, 0, 0, 7798, 0, 0, 0, 274, 0, 509, 0, 0, 708, 0] payload_rows_per_dst=[16384, 10042, 16384, 14120, 7902, 3052, 15596, 2530] dup_rows_per_dst=[0, 1010, 6744, 3508, 2, 578, 0, 452] combine_rows_per_src=[16384, 10042, 16384, 14120, 7902, 3052, 15596, 2530] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89512 payload_rows_remote=75312 dup_rows=8792 dup_rows_remote=5354 dedup_ratio=0.089437 router_tokens_per_expert=[2, 0, 0, 0, 0, 0, 0, 0, 0, 2167, 0, 0, 0, 0, 2347, 0, 3934, 8189, 1, 2, 0, 0, 0, 0, 0, 1, 3, 0, 0, 1, 2038, 359, 0, 0, 0, 4, 0, 0, 4079, 0, 6417, 8186, 0, 0, 1211, 988, 3, 0, 0, 0, 0, 0, 2, 7850, 0, 4, 0, 265, 2, 461, 2, 0, 633, 1] payload_rows_per_dst=[16376, 8536, 9392, 14200, 8164, 3672, 15708, 13464] dup_rows_per_dst=[0, 492, 1860, 3438, 2, 732, 4, 2264] combine_rows_per_src=[16376, 8536, 9392, 14200, 8164, 3672, 15708, 13464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70686 payload_rows_remote=54302 dup_rows=27618 dup_rows_remote=25606 dedup_ratio=0.280945 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2084, 0, 0, 1, 0, 2281, 0, 3568, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1899, 350, 0, 0, 0, 15, 0, 0, 4768, 2, 7091, 8191, 0, 0, 894, 1007, 0, 0, 0, 0, 0, 0, 0, 7801, 0, 0, 0, 200, 0, 265, 0, 0, 541, 0] payload_rows_per_dst=[0, 8312, 2, 4440, 9562, 16384, 15602, 16384] dup_rows_per_dst=[0, 420, 0, 60, 8, 25118, 0, 2012] combine_rows_per_src=[0, 8312, 2, 4440, 9562, 16384, 15602, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=263938048 dispatch_meta_bytes=318584 combine_payload_bytes=263938048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264256632 ingress_bytes=298541056 link_bytes=298541056 token_bytes=528194680 weight_movement_bytes=34603008 total_nvlink_bytes=562797688 weight_to_token_ratio=0.065512 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=20224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123385959e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=219013120 dispatch_meta_bytes=314200 combine_payload_bytes=219013120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=219327320 ingress_bytes=253616128 link_bytes=253616128 token_bytes=438340440 weight_movement_bytes=34603008 total_nvlink_bytes=472943448 weight_to_token_ratio=0.078941 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=26112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.543370719e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=308477952 dispatch_meta_bytes=322664 combine_payload_bytes=308477952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=308800616 ingress_bytes=343080960 link_bytes=343080960 token_bytes=617278568 weight_movement_bytes=34603008 total_nvlink_bytes=651881576 weight_to_token_ratio=0.056057 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=31104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.290986032e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90260 payload_rows_remote=87108 dup_rows=8044 dup_rows_remote=7288 dedup_ratio=0.081828 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2535, 0, 0, 0, 0, 2602, 0, 3507, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1824, 384, 0, 0, 0, 17, 0, 0, 3865, 1, 6718, 8192, 0, 0, 997, 957, 0, 0, 0, 0, 0, 0, 0, 7790, 0, 0, 0, 265, 0, 511, 0, 0, 791, 0] payload_rows_per_dst=[16384, 9514, 7014, 14472, 7760, 3152, 15580, 16384] dup_rows_per_dst=[0, 760, 0, 3388, 6, 756, 0, 3134] combine_rows_per_src=[16384, 9514, 7014, 14472, 7760, 3152, 15580, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=331022336 dispatch_meta_bytes=350328 combine_payload_bytes=331022336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=331372664 ingress_bytes=365625344 link_bytes=365625344 token_bytes=662395000 weight_movement_bytes=34603008 total_nvlink_bytes=696998008 weight_to_token_ratio=0.052239 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=36608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.375818734e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73360 payload_rows_remote=56980 dup_rows=24944 dup_rows_remote=24944 dedup_ratio=0.253743 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2036, 0, 0, 0, 0, 2433, 0, 3225, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1877, 259, 0, 1, 0, 13, 0, 0, 4930, 0, 6780, 8190, 0, 0, 1167, 983, 0, 0, 0, 0, 0, 0, 0, 7821, 0, 0, 0, 286, 0, 224, 0, 0, 731, 0] payload_rows_per_dst=[16380, 15444, 16384, 4218, 15034, 3484, 0, 2416] dup_rows_per_dst=[0, 7054, 8302, 62, 8644, 816, 0, 66] combine_rows_per_src=[16380, 15444, 16384, 4218, 15034, 3484, 0, 2416] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91528 payload_rows_remote=80816 dup_rows=6776 dup_rows_remote=6766 dedup_ratio=0.068929 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1903, 0, 0, 1, 0, 2530, 0, 2933, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1818, 238, 0, 0, 0, 16, 0, 0, 5344, 1, 6936, 8191, 0, 0, 1153, 992, 0, 0, 0, 0, 0, 0, 0, 7818, 0, 0, 0, 276, 0, 227, 0, 0, 582, 0] payload_rows_per_dst=[16382, 8364, 5866, 14704, 10712, 3480, 15636, 16384] dup_rows_per_dst=[0, 504, 0, 3282, 10, 810, 0, 2170] combine_rows_per_src=[16382, 8364, 5866, 14704, 10712, 3480, 15636, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=233390080 dispatch_meta_bytes=327696 combine_payload_bytes=233390080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=233717776 ingress_bytes=267993088 link_bytes=267993088 token_bytes=467107856 weight_movement_bytes=34603008 total_nvlink_bytes=501710864 weight_to_token_ratio=0.074079 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.037500 gemm_active_groups=3 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.008436415e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80486 payload_rows_remote=66192 dup_rows=17818 dup_rows_remote=15658 dedup_ratio=0.181254 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4368, 0, 0, 0, 34, 1, 2294, 3283, 71, 0, 0, 367, 0, 0, 5106, 0, 1, 0, 0, 0, 0, 8120, 0, 0, 0, 0, 0, 0, 0, 7446, 0, 0, 2, 0, 0, 1, 0, 1904, 0, 6256, 66, 0, 0, 7838, 0, 0, 1, 0, 1993, 0, 0] payload_rows_per_dst=[5262, 8736, 10374, 10592, 0, 14896, 14294, 16332] dup_rows_per_dst=[0, 0, 992, 356, 0, 0, 2160, 14310] combine_rows_per_src=[5262, 8736, 10374, 10592, 0, 14896, 14294, 16332] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=356794368 dispatch_meta_bytes=377584 combine_payload_bytes=356794368 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=357171952 ingress_bytes=495206400 link_bytes=495206400 token_bytes=713966320 weight_movement_bytes=138412032 total_nvlink_bytes=852378352 weight_to_token_ratio=0.193864 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.863421816e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=285188096 dispatch_meta_bytes=300704 combine_payload_bytes=285188096 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=285488800 ingress_bytes=388997120 link_bytes=388997120 token_bytes=570676896 weight_movement_bytes=103809024 total_nvlink_bytes=674485920 weight_to_token_ratio=0.181905 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=26368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.463764846e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=308477952 dispatch_meta_bytes=322664 combine_payload_bytes=308477952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=308800616 ingress_bytes=343080960 link_bytes=343080960 token_bytes=617278568 weight_movement_bytes=34603008 total_nvlink_bytes=651881576 weight_to_token_ratio=0.056057 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=31104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.290986032e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=222420992 dispatch_meta_bytes=319632 combine_payload_bytes=222420992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=222740624 ingress_bytes=257024000 link_bytes=257024000 token_bytes=445161616 weight_movement_bytes=34603008 total_nvlink_bytes=479764624 weight_to_token_ratio=0.077731 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=39296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.671606199e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=233390080 dispatch_meta_bytes=327696 combine_payload_bytes=233390080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=233717776 ingress_bytes=267993088 link_bytes=267993088 token_bytes=467107856 weight_movement_bytes=34603008 total_nvlink_bytes=501710864 weight_to_token_ratio=0.074079 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.037500 gemm_active_groups=3 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.008436415e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=331022336 dispatch_meta_bytes=350328 combine_payload_bytes=331022336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=331372664 ingress_bytes=365625344 link_bytes=365625344 token_bytes=662395000 weight_movement_bytes=34603008 total_nvlink_bytes=696998008 weight_to_token_ratio=0.052239 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=36608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.375818734e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=356794368 dispatch_meta_bytes=377584 combine_payload_bytes=356794368 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=357171952 ingress_bytes=495206400 link_bytes=495206400 token_bytes=713966320 weight_movement_bytes=138412032 total_nvlink_bytes=852378352 weight_to_token_ratio=0.193864 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.863421816e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=271122432 dispatch_meta_bytes=327400 combine_payload_bytes=271122432 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=271449832 ingress_bytes=340328448 link_bytes=340328448 token_bytes=542572264 weight_movement_bytes=69206016 total_nvlink_bytes=611778280 weight_to_token_ratio=0.127552 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=31232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.280628551e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85124 payload_rows_remote=70334 dup_rows=13180 dup_rows_remote=7656 dedup_ratio=0.134074 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4489, 0, 0, 0, 39, 0, 2970, 3328, 2, 0, 0, 260, 0, 0, 5466, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 7357, 0, 0, 0, 0, 0, 0, 0, 1915, 0, 5668, 0, 0, 0, 7741, 0, 0, 0, 0, 1727, 0, 0] payload_rows_per_dst=[15482, 14790, 11098, 11042, 16382, 9046, 3830, 3454] dup_rows_per_dst=[0, 5524, 1580, 410, 5666, 0, 0, 0] combine_rows_per_src=[15482, 14790, 11098, 11042, 16382, 9046, 3830, 3454] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78140 payload_rows_remote=62506 dup_rows=20164 dup_rows_remote=20158 dedup_ratio=0.205119 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1881, 0, 0, 0, 0, 2716, 0, 3025, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 1682, 403, 0, 0, 0, 4, 0, 0, 5029, 0, 6684, 8189, 0, 0, 1107, 1263, 0, 0, 0, 3, 0, 0, 0, 7817, 0, 0, 0, 246, 0, 175, 0, 0, 734, 0] payload_rows_per_dst=[14, 8594, 3086, 7988, 10066, 16374, 15634, 16384] dup_rows_per_dst=[0, 600, 0, 1328, 0, 15922, 6, 2308] combine_rows_per_src=[14, 8594, 3086, 7988, 10066, 16374, 15634, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92086 payload_rows_remote=77206 dup_rows=6218 dup_rows_remote=6218 dedup_ratio=0.063253 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 0, 35, 0, 2813, 3670, 37, 0, 0, 295, 0, 0, 5491, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 7440, 0, 0, 0, 0, 0, 0, 0, 1842, 0, 5967, 20, 0, 0, 7771, 0, 0, 0, 0, 1557, 0, 0] payload_rows_per_dst=[15542, 8106, 11476, 11170, 5220, 14880, 13744, 11948] dup_rows_per_dst=[0, 0, 1634, 402, 0, 0, 1914, 2268] combine_rows_per_src=[15542, 8106, 11476, 11170, 5220, 14880, 13744, 11948] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91966 payload_rows_remote=80542 dup_rows=6338 dup_rows_remote=4500 dedup_ratio=0.064473 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3971, 0, 0, 2, 35, 0, 3004, 3539, 51, 0, 0, 447, 0, 0, 5746, 0, 0, 0, 0, 0, 0, 8108, 0, 0, 0, 0, 0, 0, 0, 7457, 0, 0, 0, 0, 0, 0, 0, 1904, 0, 5818, 70, 0, 0, 7765, 0, 0, 0, 0, 1235, 0, 0] payload_rows_per_dst=[15530, 12840, 11424, 11780, 16216, 14914, 6792, 2470] dup_rows_per_dst=[0, 3230, 1838, 606, 0, 0, 664, 0] combine_rows_per_src=[15530, 12840, 11424, 11780, 16216, 14914, 6792, 2470] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94370 payload_rows_remote=83908 dup_rows=3934 dup_rows_remote=3604 dedup_ratio=0.040019 router_tokens_per_expert=[0, 2, 0, 1, 0, 0, 0, 0, 1, 0, 4, 0, 0, 4271, 3, 0, 0, 38, 3, 2833, 3706, 36, 0, 1, 294, 0, 1, 5092, 1, 7, 1, 0, 0, 0, 8147, 1, 0, 3, 0, 0, 0, 0, 7421, 0, 0, 0, 0, 0, 0, 0, 1920, 1, 5583, 38, 0, 0, 7765, 0, 0, 0, 1, 1976, 0, 1] payload_rows_per_dst=[15536, 8554, 11562, 10462, 16296, 14842, 13164, 3954] dup_rows_per_dst=[0, 4, 1672, 330, 6, 0, 1920, 2] combine_rows_per_src=[15536, 8554, 11562, 10462, 16296, 14842, 13164, 3954] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71520 payload_rows_remote=55136 dup_rows=26784 dup_rows_remote=6654 dedup_ratio=0.272461 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3755, 0, 0, 1, 23, 0, 2862, 3422, 82, 0, 1, 284, 0, 0, 5147, 0, 2, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 7426, 0, 0, 1, 0, 0, 0, 0, 2094, 0, 5756, 39, 0, 0, 7861, 0, 0, 0, 0, 2243, 0, 0] payload_rows_per_dst=[0, 7510, 7492, 10478, 0, 14854, 14802, 16384] dup_rows_per_dst=[0, 0, 628, 388, 0, 0, 5638, 20130] combine_rows_per_src=[0, 7510, 7492, 10478, 0, 14854, 14802, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=288088064 dispatch_meta_bytes=311960 combine_payload_bytes=288088064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288400024 ingress_bytes=322691072 link_bytes=322691072 token_bytes=576488088 weight_movement_bytes=34603008 total_nvlink_bytes=611091096 weight_to_token_ratio=0.060024 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=33664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.214260525e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=256024576 dispatch_meta_bytes=330656 combine_payload_bytes=256024576 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=256355232 ingress_bytes=325230592 link_bytes=325230592 token_bytes=512379808 weight_movement_bytes=69206016 total_nvlink_bytes=581585824 weight_to_token_ratio=0.135068 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.223816534e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94612 payload_rows_remote=78252 dup_rows=3692 dup_rows_remote=3692 dedup_ratio=0.037557 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4647, 0, 0, 1, 28, 0, 2972, 3208, 49, 0, 0, 234, 0, 0, 5069, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 7435, 0, 0, 1, 0, 0, 0, 0, 1810, 0, 5844, 11, 0, 0, 7713, 0, 0, 0, 0, 1949, 0, 0] payload_rows_per_dst=[15426, 9294, 10914, 10314, 16360, 14870, 13536, 3898] dup_rows_per_dst=[2, 0, 1602, 292, 0, 2, 1794, 0] combine_rows_per_src=[15426, 9294, 10914, 10314, 16360, 14870, 13536, 3898] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=316235776 dispatch_meta_bytes=333696 combine_payload_bytes=316235776 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=316569472 ingress_bytes=420044800 link_bytes=420044800 token_bytes=632805248 weight_movement_bytes=103809024 total_nvlink_bytes=736614272 weight_to_token_ratio=0.164046 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.037500 gemm_active_groups=3 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.580594766e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79744 payload_rows_remote=64220 dup_rows=18560 dup_rows_remote=18560 dedup_ratio=0.188802 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 4019, 0, 0, 5, 61, 0, 3116, 3631, 49, 0, 0, 311, 0, 0, 5266, 0, 0, 0, 0, 0, 0, 8145, 0, 0, 3, 0, 0, 0, 0, 7342, 0, 0, 0, 0, 0, 0, 0, 2016, 0, 5835, 10, 0, 0, 7762, 0, 0, 0, 0, 1577, 0, 0] payload_rows_per_dst=[15524, 14734, 11906, 14086, 16308, 0, 4032, 3154] dup_rows_per_dst=[0, 4982, 2112, 5596, 5850, 0, 20, 0] combine_rows_per_src=[15524, 14734, 11906, 14086, 16308, 0, 4032, 3154] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=329900032 dispatch_meta_bytes=340168 combine_payload_bytes=329900032 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=330240200 ingress_bytes=399106048 link_bytes=399106048 token_bytes=660140232 weight_movement_bytes=69206016 total_nvlink_bytes=729346248 weight_to_token_ratio=0.104835 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=25472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.501803928e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=288088064 dispatch_meta_bytes=311960 combine_payload_bytes=288088064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288400024 ingress_bytes=322691072 link_bytes=322691072 token_bytes=576488088 weight_movement_bytes=34603008 total_nvlink_bytes=611091096 weight_to_token_ratio=0.060024 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=33664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.214260525e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91966 payload_rows_remote=80542 dup_rows=6338 dup_rows_remote=4500 dedup_ratio=0.064473 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3971, 0, 0, 2, 35, 0, 3004, 3539, 51, 0, 0, 447, 0, 0, 5746, 0, 0, 0, 0, 0, 0, 8108, 0, 0, 0, 0, 0, 0, 0, 7457, 0, 0, 0, 0, 0, 0, 0, 1904, 0, 5818, 70, 0, 0, 7765, 0, 0, 0, 0, 1235, 0, 0] payload_rows_per_dst=[15530, 12840, 11424, 11780, 16216, 14914, 6792, 2470] dup_rows_per_dst=[0, 3230, 1838, 606, 0, 0, 664, 0] combine_rows_per_src=[15530, 12840, 11424, 11780, 16216, 14914, 6792, 2470] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=343687168 dispatch_meta_bytes=350048 combine_payload_bytes=343687168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=344037216 ingress_bytes=378290176 link_bytes=378290176 token_bytes=687724384 weight_movement_bytes=34603008 total_nvlink_bytes=722327392 weight_to_token_ratio=0.050315 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=42496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.423475477e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=225837056 dispatch_meta_bytes=247160 combine_payload_bytes=225837056 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=226084216 ingress_bytes=295043072 link_bytes=295043072 token_bytes=451921272 weight_movement_bytes=69206016 total_nvlink_bytes=521127288 weight_to_token_ratio=0.153137 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=21888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.110223327e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71520 payload_rows_remote=55136 dup_rows=26784 dup_rows_remote=6654 dedup_ratio=0.272461 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3755, 0, 0, 1, 23, 0, 2862, 3422, 82, 0, 1, 284, 0, 0, 5147, 0, 2, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 7426, 0, 0, 1, 0, 0, 0, 0, 2094, 0, 5756, 39, 0, 0, 7861, 0, 0, 0, 0, 2243, 0, 0] payload_rows_per_dst=[0, 7510, 7492, 10478, 0, 14854, 14802, 16384] dup_rows_per_dst=[0, 0, 628, 388, 0, 0, 5638, 20130] combine_rows_per_src=[0, 7510, 7492, 10478, 0, 14854, 14802, 16384] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=320520192 dispatch_meta_bytes=327776 combine_payload_bytes=320520192 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=320847968 ingress_bytes=389726208 link_bytes=389726208 token_bytes=641368160 weight_movement_bytes=69206016 total_nvlink_bytes=710574176 weight_to_token_ratio=0.107904 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=43392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.466508345e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=263045120 dispatch_meta_bytes=331120 combine_payload_bytes=263045120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263376240 ingress_bytes=297648128 link_bytes=297648128 token_bytes=526421360 weight_movement_bytes=34603008 total_nvlink_bytes=561024368 weight_to_token_ratio=0.065733 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.120025943e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80486 payload_rows_remote=66192 dup_rows=17818 dup_rows_remote=15658 dedup_ratio=0.181254 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4368, 0, 0, 0, 34, 1, 2294, 3283, 71, 0, 0, 367, 0, 0, 5106, 0, 1, 0, 0, 0, 0, 8120, 0, 0, 0, 0, 0, 0, 0, 7446, 0, 0, 2, 0, 0, 1, 0, 1904, 0, 6256, 66, 0, 0, 7838, 0, 0, 1, 0, 1993, 0, 0] payload_rows_per_dst=[5262, 8736, 10374, 10592, 0, 14896, 14294, 16332] dup_rows_per_dst=[0, 0, 992, 356, 0, 0, 2160, 14310] combine_rows_per_src=[5262, 8736, 10374, 10592, 0, 14896, 14294, 16332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87782 payload_rows_remote=76692 dup_rows=10522 dup_rows_remote=9438 dedup_ratio=0.107035 router_tokens_per_expert=[0, 0, 0, 1, 0, 0, 297, 552, 13, 0, 0, 0, 3427, 0, 0, 0, 7959, 0, 2, 0, 0, 0, 0, 7585, 0, 1, 7326, 0, 0, 0, 0, 0, 0, 0, 743, 123, 0, 0, 4122, 0, 0, 2873, 6848, 130, 0, 0, 97, 1, 0, 0, 0, 0, 5029, 0, 1058, 0, 0, 0, 0, 0, 945, 0, 0, 20] payload_rows_per_dst=[1694, 6872, 15174, 9738, 11564, 15540, 11090, 16110] dup_rows_per_dst=[6, 8, 0, 2, 3326, 4358, 1084, 1738] combine_rows_per_src=[1694, 6872, 15174, 9738, 11564, 15540, 11090, 16110] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92086 payload_rows_remote=77206 dup_rows=6218 dup_rows_remote=6218 dedup_ratio=0.063253 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 0, 35, 0, 2813, 3670, 37, 0, 0, 295, 0, 0, 5491, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 7440, 0, 0, 0, 0, 0, 0, 0, 1842, 0, 5967, 20, 0, 0, 7771, 0, 0, 0, 0, 1557, 0, 0] payload_rows_per_dst=[15542, 8106, 11476, 11170, 5220, 14880, 13744, 11948] dup_rows_per_dst=[0, 0, 1634, 402, 0, 0, 1914, 2268] combine_rows_per_src=[15542, 8106, 11476, 11170, 5220, 14880, 13744, 11948] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=329900032 dispatch_meta_bytes=340168 combine_payload_bytes=329900032 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=330240200 ingress_bytes=399106048 link_bytes=399106048 token_bytes=660140232 weight_movement_bytes=69206016 total_nvlink_bytes=729346248 weight_to_token_ratio=0.104835 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=25472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.501803928e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=225837056 dispatch_meta_bytes=247160 combine_payload_bytes=225837056 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=226084216 ingress_bytes=295043072 link_bytes=295043072 token_bytes=451921272 weight_movement_bytes=69206016 total_nvlink_bytes=521127288 weight_to_token_ratio=0.153137 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=21888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.110223327e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=314130432 dispatch_meta_bytes=344520 combine_payload_bytes=314130432 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=314474952 ingress_bytes=348733440 link_bytes=348733440 token_bytes=628605384 weight_movement_bytes=34603008 total_nvlink_bytes=663208392 weight_to_token_ratio=0.055047 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.037500 gemm_active_groups=4 gemm_peak_group_rows=38016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.312255859e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=316235776 dispatch_meta_bytes=333696 combine_payload_bytes=316235776 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=316569472 ingress_bytes=420044800 link_bytes=420044800 token_bytes=632805248 weight_movement_bytes=103809024 total_nvlink_bytes=736614272 weight_to_token_ratio=0.164046 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.037500 gemm_active_groups=3 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.580594766e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88942 payload_rows_remote=73540 dup_rows=9362 dup_rows_remote=3032 dedup_ratio=0.095235 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 302, 578, 8, 0, 0, 0, 4075, 0, 0, 0, 7931, 0, 7, 0, 0, 0, 0, 7813, 0, 0, 7350, 0, 0, 0, 0, 0, 0, 0, 582, 85, 0, 0, 4638, 0, 0, 2875, 6783, 140, 0, 0, 151, 0, 0, 0, 0, 0, 4499, 0, 649, 0, 0, 0, 0, 0, 644, 0, 0, 42] payload_rows_per_dst=[15920, 15402, 15862, 14700, 10026, 6050, 9610, 1372] dup_rows_per_dst=[1466, 6330, 14, 0, 584, 282, 686, 0] combine_rows_per_src=[15920, 15402, 15862, 14700, 10026, 6050, 9610, 1372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90438 payload_rows_remote=74862 dup_rows=7866 dup_rows_remote=3168 dedup_ratio=0.080017 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 289, 525, 9, 0, 0, 0, 3934, 0, 0, 0, 7990, 0, 13, 0, 0, 0, 0, 7502, 0, 0, 7488, 0, 0, 0, 0, 0, 0, 0, 743, 134, 0, 0, 4164, 0, 0, 3005, 6822, 178, 0, 0, 132, 0, 0, 0, 0, 0, 4839, 0, 693, 0, 0, 0, 0, 0, 680, 0, 0, 12] payload_rows_per_dst=[7614, 7884, 8478, 14976, 9514, 15576, 10284, 16112] dup_rows_per_dst=[552, 2, 14, 0, 568, 4698, 780, 1252] combine_rows_per_src=[7614, 7884, 8478, 14976, 9514, 15576, 10284, 16112] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89928 payload_rows_remote=74818 dup_rows=8376 dup_rows_remote=6388 dedup_ratio=0.085205 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 325, 527, 5, 0, 0, 0, 3693, 0, 0, 0, 7902, 0, 37, 0, 0, 0, 0, 7736, 0, 0, 7325, 0, 0, 0, 0, 0, 0, 0, 837, 91, 0, 0, 4650, 0, 0, 2602, 6936, 178, 0, 0, 160, 0, 0, 0, 0, 0, 5012, 0, 517, 0, 0, 0, 0, 0, 611, 0, 0, 8] payload_rows_per_dst=[15752, 11682, 15110, 14650, 10260, 5596, 10438, 6440] dup_rows_per_dst=[1424, 2796, 1988, 0, 896, 284, 620, 368] combine_rows_per_src=[15752, 11682, 15110, 14650, 10260, 5596, 10438, 6440] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=271122432 dispatch_meta_bytes=327400 combine_payload_bytes=271122432 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=271449832 ingress_bytes=340328448 link_bytes=340328448 token_bytes=542572264 weight_movement_bytes=69206016 total_nvlink_bytes=611778280 weight_to_token_ratio=0.127552 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=31232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.280628551e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92576 payload_rows_remote=77858 dup_rows=5728 dup_rows_remote=5728 dedup_ratio=0.058268 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 290, 760, 7, 0, 0, 3, 3901, 0, 0, 2, 7925, 0, 32, 2, 0, 0, 0, 7626, 0, 0, 7359, 0, 0, 0, 0, 0, 0, 0, 733, 109, 1, 0, 4503, 0, 0, 2339, 6874, 152, 1, 0, 89, 0, 0, 0, 0, 0, 4840, 0, 780, 0, 0, 4, 0, 0, 798, 0, 2, 19] payload_rows_per_dst=[15668, 7820, 10216, 14718, 10046, 7648, 10412, 16048] dup_rows_per_dst=[1686, 6, 46, 0, 646, 1068, 828, 1448] combine_rows_per_src=[15668, 7820, 10216, 14718, 10046, 7648, 10412, 16048] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80022 payload_rows_remote=63892 dup_rows=18282 dup_rows_remote=16732 dedup_ratio=0.185974 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 236, 515, 10, 0, 0, 0, 3972, 0, 0, 0, 7989, 0, 2, 0, 0, 0, 0, 7706, 0, 0, 7233, 0, 0, 0, 0, 0, 0, 0, 522, 135, 0, 0, 4451, 0, 0, 3339, 6575, 142, 0, 0, 133, 0, 0, 0, 0, 0, 4713, 0, 628, 0, 0, 0, 0, 0, 826, 0, 0, 25] payload_rows_per_dst=[1502, 7964, 15412, 0, 14680, 9816, 14518, 16130] dup_rows_per_dst=[0, 0, 4, 0, 7026, 2054, 7648, 1550] combine_rows_per_src=[1502, 7964, 15412, 0, 14680, 9816, 14518, 16130] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=301219840 dispatch_meta_bytes=306288 combine_payload_bytes=301219840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=301526128 ingress_bytes=335822848 link_bytes=335822848 token_bytes=602745968 weight_movement_bytes=34603008 total_nvlink_bytes=637348976 weight_to_token_ratio=0.057409 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=32000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263674341e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89794 payload_rows_remote=79928 dup_rows=8510 dup_rows_remote=8116 dedup_ratio=0.086568 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 235, 663, 6, 0, 0, 0, 4532, 0, 0, 0, 7974, 0, 6, 0, 0, 0, 0, 7684, 0, 0, 7248, 0, 0, 0, 1, 1, 0, 0, 431, 109, 0, 0, 4590, 0, 0, 2976, 6631, 130, 0, 0, 102, 0, 0, 0, 0, 0, 4312, 0, 616, 0, 0, 0, 0, 0, 865, 0, 0, 40] payload_rows_per_dst=[15700, 9074, 12, 14498, 9866, 15310, 9242, 16092] dup_rows_per_dst=[1464, 2, 0, 2, 394, 4368, 614, 1666] combine_rows_per_src=[15700, 9074, 12, 14498, 9866, 15310, 9242, 16092] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=306634752 dispatch_meta_bytes=312120 combine_payload_bytes=306634752 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=306946872 ingress_bytes=445046784 link_bytes=445046784 token_bytes=613581624 weight_movement_bytes=138412032 total_nvlink_bytes=751993656 weight_to_token_ratio=0.225580 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=22912 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.674675219e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88142 payload_rows_remote=72242 dup_rows=10162 dup_rows_remote=8206 dedup_ratio=0.103373 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 434, 688, 12, 0, 0, 0, 4370, 0, 0, 0, 7967, 0, 20, 0, 0, 0, 0, 7806, 0, 0, 7510, 0, 0, 0, 0, 0, 0, 0, 471, 110, 0, 0, 3873, 0, 0, 2436, 6849, 275, 0, 0, 117, 0, 0, 0, 0, 0, 4693, 0, 532, 0, 0, 0, 0, 0, 971, 0, 0, 18] payload_rows_per_dst=[15900, 15606, 15934, 15020, 8564, 5318, 9822, 1978] dup_rows_per_dst=[1956, 6856, 40, 0, 344, 338, 628, 0] combine_rows_per_src=[15900, 15606, 15934, 15020, 8564, 5318, 9822, 1978] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=306454528 dispatch_meta_bytes=324824 combine_payload_bytes=306454528 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=306779352 ingress_bytes=410263552 link_bytes=410263552 token_bytes=613233880 weight_movement_bytes=103809024 total_nvlink_bytes=717042904 weight_to_token_ratio=0.169281 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=21120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.543788717e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=301219840 dispatch_meta_bytes=306288 combine_payload_bytes=301219840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=301526128 ingress_bytes=335822848 link_bytes=335822848 token_bytes=602745968 weight_movement_bytes=34603008 total_nvlink_bytes=637348976 weight_to_token_ratio=0.057409 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=32000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263674341e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89928 payload_rows_remote=74818 dup_rows=8376 dup_rows_remote=6388 dedup_ratio=0.085205 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 325, 527, 5, 0, 0, 0, 3693, 0, 0, 0, 7902, 0, 37, 0, 0, 0, 0, 7736, 0, 0, 7325, 0, 0, 0, 0, 0, 0, 0, 837, 91, 0, 0, 4650, 0, 0, 2602, 6936, 178, 0, 0, 160, 0, 0, 0, 0, 0, 5012, 0, 517, 0, 0, 0, 0, 0, 611, 0, 0, 8] payload_rows_per_dst=[15752, 11682, 15110, 14650, 10260, 5596, 10438, 6440] dup_rows_per_dst=[1424, 2796, 1988, 0, 896, 284, 620, 368] combine_rows_per_src=[15752, 11682, 15110, 14650, 10260, 5596, 10438, 6440] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=318906368 dispatch_meta_bytes=334344 combine_payload_bytes=318906368 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=319240712 ingress_bytes=388112384 link_bytes=388112384 token_bytes=638147080 weight_movement_bytes=69206016 total_nvlink_bytes=707353096 weight_to_token_ratio=0.108448 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.460435655e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=261701632 dispatch_meta_bytes=322496 combine_payload_bytes=261701632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262024128 ingress_bytes=296304640 link_bytes=296304640 token_bytes=523725760 weight_movement_bytes=34603008 total_nvlink_bytes=558328768 weight_to_token_ratio=0.066071 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=42624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.114970506e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=327385088 dispatch_meta_bytes=352176 combine_payload_bytes=327385088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=327737264 ingress_bytes=361988096 link_bytes=361988096 token_bytes=655122352 weight_movement_bytes=34603008 total_nvlink_bytes=689725360 weight_to_token_ratio=0.052819 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=35072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.362132063e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94576 payload_rows_remote=79180 dup_rows=3728 dup_rows_remote=3726 dedup_ratio=0.037923 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6803, 0, 0, 0, 8192, 0, 5535, 0, 351, 0, 0, 1, 50, 0, 0, 0, 0, 175, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 0, 5044, 0, 0, 5659, 0, 0, 6571, 0, 0, 0, 0, 0, 0, 7695, 0, 0, 0, 4, 3065, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 16384, 13354, 13802, 10088, 11318, 15396, 14234] dup_rows_per_dst=[0, 0, 592, 168, 0, 0, 2, 2966] combine_rows_per_src=[0, 16384, 13354, 13802, 10088, 11318, 15396, 14234] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=295903232 dispatch_meta_bytes=321792 combine_payload_bytes=295903232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296225024 ingress_bytes=330506240 link_bytes=330506240 token_bytes=592128256 weight_movement_bytes=34603008 total_nvlink_bytes=626731264 weight_to_token_ratio=0.058438 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=41984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.243668373e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87782 payload_rows_remote=76692 dup_rows=10522 dup_rows_remote=9438 dedup_ratio=0.107035 router_tokens_per_expert=[0, 0, 0, 1, 0, 0, 297, 552, 13, 0, 0, 0, 3427, 0, 0, 0, 7959, 0, 2, 0, 0, 0, 0, 7585, 0, 1, 7326, 0, 0, 0, 0, 0, 0, 0, 743, 123, 0, 0, 4122, 0, 0, 2873, 6848, 130, 0, 0, 97, 1, 0, 0, 0, 0, 5029, 0, 1058, 0, 0, 0, 0, 0, 945, 0, 0, 20] payload_rows_per_dst=[1694, 6872, 15174, 9738, 11564, 15540, 11090, 16110] dup_rows_per_dst=[6, 8, 0, 2, 3326, 4358, 1084, 1738] combine_rows_per_src=[1694, 6872, 15174, 9738, 11564, 15540, 11090, 16110] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80022 payload_rows_remote=63892 dup_rows=18282 dup_rows_remote=16732 dedup_ratio=0.185974 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 236, 515, 10, 0, 0, 0, 3972, 0, 0, 0, 7989, 0, 2, 0, 0, 0, 0, 7706, 0, 0, 7233, 0, 0, 0, 0, 0, 0, 0, 522, 135, 0, 0, 4451, 0, 0, 3339, 6575, 142, 0, 0, 133, 0, 0, 0, 0, 0, 4713, 0, 628, 0, 0, 0, 0, 0, 826, 0, 0, 25] payload_rows_per_dst=[1502, 7964, 15412, 0, 14680, 9816, 14518, 16130] dup_rows_per_dst=[0, 0, 4, 0, 7026, 2054, 7648, 1550] combine_rows_per_src=[1502, 7964, 15412, 0, 14680, 9816, 14518, 16130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90438 payload_rows_remote=74862 dup_rows=7866 dup_rows_remote=3168 dedup_ratio=0.080017 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 289, 525, 9, 0, 0, 0, 3934, 0, 0, 0, 7990, 0, 13, 0, 0, 0, 0, 7502, 0, 0, 7488, 0, 0, 0, 0, 0, 0, 0, 743, 134, 0, 0, 4164, 0, 0, 3005, 6822, 178, 0, 0, 132, 0, 0, 0, 0, 0, 4839, 0, 693, 0, 0, 0, 0, 0, 680, 0, 0, 12] payload_rows_per_dst=[7614, 7884, 8478, 14976, 9514, 15576, 10284, 16112] dup_rows_per_dst=[552, 2, 14, 0, 568, 4698, 780, 1252] combine_rows_per_src=[7614, 7884, 8478, 14976, 9514, 15576, 10284, 16112] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=306454528 dispatch_meta_bytes=324824 combine_payload_bytes=306454528 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=306779352 ingress_bytes=410263552 link_bytes=410263552 token_bytes=613233880 weight_movement_bytes=103809024 total_nvlink_bytes=717042904 weight_to_token_ratio=0.169281 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=21120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.543788717e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=324321280 dispatch_meta_bytes=331624 combine_payload_bytes=324321280 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=324652904 ingress_bytes=428130304 link_bytes=428130304 token_bytes=648974184 weight_movement_bytes=103809024 total_nvlink_bytes=752783208 weight_to_token_ratio=0.159959 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=47616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.611019867e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=261701632 dispatch_meta_bytes=322496 combine_payload_bytes=261701632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262024128 ingress_bytes=296304640 link_bytes=296304640 token_bytes=523725760 weight_movement_bytes=34603008 total_nvlink_bytes=558328768 weight_to_token_ratio=0.066071 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=42624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.114970506e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=306634752 dispatch_meta_bytes=312120 combine_payload_bytes=306634752 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=306946872 ingress_bytes=445046784 link_bytes=445046784 token_bytes=613581624 weight_movement_bytes=138412032 total_nvlink_bytes=751993656 weight_to_token_ratio=0.225580 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=22912 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.674675219e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90310 payload_rows_remote=78066 dup_rows=7994 dup_rows_remote=7994 dedup_ratio=0.081319 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6979, 0, 0, 0, 8192, 0, 6122, 0, 360, 0, 0, 0, 7, 0, 0, 0, 0, 91, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 5122, 0, 0, 5678, 0, 0, 6322, 0, 0, 0, 0, 0, 0, 7671, 0, 0, 0, 2, 2605, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 12244, 12866, 14082, 15272, 11356, 2896, 5210] dup_rows_per_dst=[0, 0, 512, 60, 7420, 0, 2, 0] combine_rows_per_src=[16384, 12244, 12866, 14082, 15272, 11356, 2896, 5210] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94712 payload_rows_remote=83754 dup_rows=3592 dup_rows_remote=3592 dedup_ratio=0.036540 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6835, 0, 0, 0, 8192, 0, 5660, 0, 399, 0, 0, 0, 12, 0, 0, 0, 0, 134, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 5354, 0, 0, 5479, 0, 0, 6607, 0, 0, 0, 0, 0, 0, 7627, 0, 0, 0, 2, 2850, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 0, 13352, 13842, 10708, 10958, 15254, 14214] dup_rows_per_dst=[0, 0, 684, 98, 0, 0, 4, 2806] combine_rows_per_src=[16384, 0, 13352, 13842, 10708, 10958, 15254, 14214] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97484 payload_rows_remote=83840 dup_rows=820 dup_rows_remote=126 dedup_ratio=0.008341 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6846, 0, 0, 0, 8192, 0, 5821, 0, 458, 0, 0, 0, 13, 0, 0, 0, 0, 177, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 5361, 0, 0, 4676, 0, 0, 6698, 0, 0, 0, 0, 0, 0, 7648, 0, 0, 0, 2, 3257, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 11642, 13644, 13930, 10722, 9352, 15296, 6514] dup_rows_per_dst=[0, 0, 694, 122, 0, 0, 4, 0] combine_rows_per_src=[16384, 11642, 13644, 13930, 10722, 9352, 15296, 6514] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=314130432 dispatch_meta_bytes=344520 combine_payload_bytes=314130432 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=314474952 ingress_bytes=348733440 link_bytes=348733440 token_bytes=628605384 weight_movement_bytes=34603008 total_nvlink_bytes=663208392 weight_to_token_ratio=0.055047 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.037500 gemm_active_groups=4 gemm_peak_group_rows=38016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.312255859e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95962 payload_rows_remote=82046 dup_rows=2342 dup_rows_remote=2250 dedup_ratio=0.023824 router_tokens_per_expert=[1, 1, 0, 0, 0, 0, 0, 0, 5, 6870, 0, 0, 0, 8188, 0, 5996, 0, 395, 0, 0, 0, 27, 0, 0, 0, 0, 127, 0, 0, 0, 5, 2, 2, 0, 0, 0, 2, 0, 5290, 0, 0, 5704, 0, 0, 6234, 0, 0, 2, 0, 0, 0, 7713, 3, 0, 0, 12, 2571, 0, 1, 1, 0, 0, 0, 0] payload_rows_per_dst=[16378, 5350, 12688, 13916, 10584, 11412, 15440, 10194] dup_rows_per_dst=[2, 6, 624, 92, 4, 0, 16, 1598] combine_rows_per_src=[16378, 5350, 12688, 13916, 10584, 11412, 15440, 10194] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86606 payload_rows_remote=72122 dup_rows=11698 dup_rows_remote=8700 dedup_ratio=0.118998 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 5771, 0, 0, 0, 8190, 0, 5968, 0, 477, 0, 0, 0, 17, 0, 0, 0, 0, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5488, 0, 0, 5755, 0, 0, 6783, 0, 0, 0, 0, 0, 0, 7749, 0, 0, 0, 1, 2769, 0, 0, 0, 4, 0, 0, 0] payload_rows_per_dst=[0, 16380, 2338, 11860, 10976, 14880, 15688, 14484] dup_rows_per_dst=[0, 0, 80, 42, 0, 5556, 3022, 2998] combine_rows_per_src=[0, 16380, 2338, 11860, 10976, 14880, 15688, 14484] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94672 payload_rows_remote=84880 dup_rows=3632 dup_rows_remote=3632 dedup_ratio=0.036947 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 7027, 0, 0, 0, 8192, 0, 5972, 0, 412, 0, 0, 0, 6, 0, 0, 0, 0, 130, 0, 0, 0, 5, 3, 0, 0, 0, 0, 0, 0, 4896, 0, 0, 5851, 0, 0, 6335, 0, 0, 0, 0, 0, 0, 7642, 0, 0, 0, 0, 2681, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 0, 12862, 14200, 9792, 11702, 15284, 14448] dup_rows_per_dst=[0, 0, 644, 130, 0, 0, 0, 2858] combine_rows_per_src=[16384, 0, 12862, 14200, 9792, 11702, 15284, 14448] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=319758336 dispatch_meta_bytes=344240 combine_payload_bytes=319758336 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=320102576 ingress_bytes=458170368 link_bytes=458170368 token_bytes=639860912 weight_movement_bytes=138412032 total_nvlink_bytes=778272944 weight_to_token_ratio=0.216316 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=20608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.724058209e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75594 payload_rows_remote=59210 dup_rows=22710 dup_rows_remote=22706 dedup_ratio=0.231018 router_tokens_per_expert=[0, 0, 0, 0, 3, 0, 0, 0, 0, 6707, 0, 0, 0, 8191, 0, 5905, 0, 346, 0, 0, 0, 38, 1, 0, 0, 0, 163, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 5061, 0, 1, 5884, 0, 0, 6300, 0, 0, 0, 0, 0, 0, 7660, 0, 0, 0, 0, 2890, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 15788, 12762, 1950, 11160, 11770, 0, 5780] dup_rows_per_dst=[4, 20510, 608, 10, 1578, 0, 0, 0] combine_rows_per_src=[16384, 15788, 12762, 1950, 11160, 11770, 0, 5780] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=343056384 dispatch_meta_bytes=349384 combine_payload_bytes=343056384 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=343405768 ingress_bytes=412262400 link_bytes=412262400 token_bytes=686462152 weight_movement_bytes=69206016 total_nvlink_bytes=755668168 weight_to_token_ratio=0.100815 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=43136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.551310221e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=343408640 dispatch_meta_bytes=335864 combine_payload_bytes=343408640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=343744504 ingress_bytes=378011648 link_bytes=378011648 token_bytes=687153144 weight_movement_bytes=34603008 total_nvlink_bytes=721756152 weight_to_token_ratio=0.050357 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=45824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.422427399e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=319758336 dispatch_meta_bytes=344240 combine_payload_bytes=319758336 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=320102576 ingress_bytes=458170368 link_bytes=458170368 token_bytes=639860912 weight_movement_bytes=138412032 total_nvlink_bytes=778272944 weight_to_token_ratio=0.216316 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=20608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.724058209e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=336060416 dispatch_meta_bytes=337184 combine_payload_bytes=336060416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=336397600 ingress_bytes=370663424 link_bytes=370663424 token_bytes=672458016 weight_movement_bytes=34603008 total_nvlink_bytes=707061024 weight_to_token_ratio=0.051457 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=48000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394776624e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=295411712 dispatch_meta_bytes=323288 combine_payload_bytes=295411712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=295735000 ingress_bytes=330014720 link_bytes=330014720 token_bytes=591146712 weight_movement_bytes=34603008 total_nvlink_bytes=625749720 weight_to_token_ratio=0.058535 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=26496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.241818823e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=347668480 dispatch_meta_bytes=354048 combine_payload_bytes=347668480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=348022528 ingress_bytes=382271488 link_bytes=382271488 token_bytes=695691008 weight_movement_bytes=34603008 total_nvlink_bytes=730294016 weight_to_token_ratio=0.049739 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=41728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.438456834e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=336060416 dispatch_meta_bytes=337184 combine_payload_bytes=336060416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=336397600 ingress_bytes=370663424 link_bytes=370663424 token_bytes=672458016 weight_movement_bytes=34603008 total_nvlink_bytes=707061024 weight_to_token_ratio=0.051457 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=48000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394776624e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97484 payload_rows_remote=83840 dup_rows=820 dup_rows_remote=126 dedup_ratio=0.008341 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6846, 0, 0, 0, 8192, 0, 5821, 0, 458, 0, 0, 0, 13, 0, 0, 0, 0, 177, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 5361, 0, 0, 4676, 0, 0, 6698, 0, 0, 0, 0, 0, 0, 7648, 0, 0, 0, 2, 3257, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 11642, 13644, 13930, 10722, 9352, 15296, 6514] dup_rows_per_dst=[0, 0, 694, 122, 0, 0, 4, 0] combine_rows_per_src=[16384, 11642, 13644, 13930, 10722, 9352, 15296, 6514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86938 payload_rows_remote=80134 dup_rows=11366 dup_rows_remote=11366 dedup_ratio=0.115621 router_tokens_per_expert=[0, 0, 4153, 0, 0, 0, 1541, 3, 0, 7644, 0, 0, 1, 0, 0, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 629, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 2, 1, 63, 0, 0, 0, 624, 31, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 6270, 5906, 7582, 6364] payload_rows_per_dst=[10284, 15304, 12728, 13406, 0, 12040, 6804, 16372] dup_rows_per_dst=[1110, 286, 0, 394, 0, 676, 0, 8900] combine_rows_per_src=[10284, 15304, 12728, 13406, 0, 12040, 6804, 16372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86606 payload_rows_remote=72122 dup_rows=11698 dup_rows_remote=8700 dedup_ratio=0.118998 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 5771, 0, 0, 0, 8190, 0, 5968, 0, 477, 0, 0, 0, 17, 0, 0, 0, 0, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5488, 0, 0, 5755, 0, 0, 6783, 0, 0, 0, 0, 0, 0, 7749, 0, 0, 0, 1, 2769, 0, 0, 0, 4, 0, 0, 0] payload_rows_per_dst=[0, 16380, 2338, 11860, 10976, 14880, 15688, 14484] dup_rows_per_dst=[0, 0, 80, 42, 0, 5556, 3022, 2998] combine_rows_per_src=[0, 16380, 2338, 11860, 10976, 14880, 15688, 14484] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=242524160 dispatch_meta_bytes=327664 combine_payload_bytes=242524160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=242851824 ingress_bytes=277127168 link_bytes=277127168 token_bytes=485375984 weight_movement_bytes=34603008 total_nvlink_bytes=519978992 weight_to_token_ratio=0.071291 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.042807223e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94576 payload_rows_remote=79180 dup_rows=3728 dup_rows_remote=3726 dedup_ratio=0.037923 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6803, 0, 0, 0, 8192, 0, 5535, 0, 351, 0, 0, 1, 50, 0, 0, 0, 0, 175, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 0, 5044, 0, 0, 5659, 0, 0, 6571, 0, 0, 0, 0, 0, 0, 7695, 0, 0, 0, 4, 3065, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 16384, 13354, 13802, 10088, 11318, 15396, 14234] dup_rows_per_dst=[0, 0, 592, 168, 0, 0, 2, 2966] combine_rows_per_src=[0, 16384, 13354, 13802, 10088, 11318, 15396, 14234] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94712 payload_rows_remote=83754 dup_rows=3592 dup_rows_remote=3592 dedup_ratio=0.036540 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6835, 0, 0, 0, 8192, 0, 5660, 0, 399, 0, 0, 0, 12, 0, 0, 0, 0, 134, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 5354, 0, 0, 5479, 0, 0, 6607, 0, 0, 0, 0, 0, 0, 7627, 0, 0, 0, 2, 2850, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 0, 13352, 13842, 10708, 10958, 15254, 14214] dup_rows_per_dst=[0, 0, 684, 98, 0, 0, 4, 2806] combine_rows_per_src=[16384, 0, 13352, 13842, 10708, 10958, 15254, 14214] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=343408640 dispatch_meta_bytes=335864 combine_payload_bytes=343408640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=343744504 ingress_bytes=378011648 link_bytes=378011648 token_bytes=687153144 weight_movement_bytes=34603008 total_nvlink_bytes=721756152 weight_to_token_ratio=0.050357 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=45824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.422427399e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=328228864 dispatch_meta_bytes=366000 combine_payload_bytes=328228864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=328594864 ingress_bytes=362831872 link_bytes=362831872 token_bytes=656823728 weight_movement_bytes=34603008 total_nvlink_bytes=691426736 weight_to_token_ratio=0.052682 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.365307124e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=295411712 dispatch_meta_bytes=323288 combine_payload_bytes=295411712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=295735000 ingress_bytes=330014720 link_bytes=330014720 token_bytes=591146712 weight_movement_bytes=34603008 total_nvlink_bytes=625749720 weight_to_token_ratio=0.058535 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=26496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.241818823e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=343056384 dispatch_meta_bytes=349384 combine_payload_bytes=343056384 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=343405768 ingress_bytes=412262400 link_bytes=412262400 token_bytes=686462152 weight_movement_bytes=69206016 total_nvlink_bytes=755668168 weight_to_token_ratio=0.100815 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=43136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.551310221e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95900 payload_rows_remote=80838 dup_rows=2404 dup_rows_remote=2182 dedup_ratio=0.024455 router_tokens_per_expert=[0, 0, 3587, 0, 0, 0, 1391, 0, 0, 7692, 0, 0, 0, 0, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 725, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 59, 0, 0, 0, 540, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6103, 6356, 7586, 6799] payload_rows_per_dst=[9186, 15062, 13598, 13198, 16376, 13308, 15172, 0] dup_rows_per_dst=[1096, 222, 0, 458, 0, 628, 0, 0] combine_rows_per_src=[9186, 15062, 13598, 13198, 16376, 13308, 15172, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96156 payload_rows_remote=82446 dup_rows=2148 dup_rows_remote=1394 dedup_ratio=0.021851 router_tokens_per_expert=[0, 0, 3366, 0, 0, 0, 1393, 3, 0, 7722, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 753, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 38, 0, 0, 0, 690, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5823, 6467, 7736, 6810] payload_rows_per_dst=[8814, 15446, 13620, 12722, 5556, 13710, 15472, 10816] dup_rows_per_dst=[710, 254, 0, 430, 0, 754, 0, 0] combine_rows_per_src=[8814, 15446, 13620, 12722, 5556, 13710, 15472, 10816] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=324321280 dispatch_meta_bytes=331624 combine_payload_bytes=324321280 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=324652904 ingress_bytes=428130304 link_bytes=428130304 token_bytes=648974184 weight_movement_bytes=103809024 total_nvlink_bytes=752783208 weight_to_token_ratio=0.159959 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=47616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.611019867e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95638 payload_rows_remote=82918 dup_rows=2666 dup_rows_remote=2666 dedup_ratio=0.027120 router_tokens_per_expert=[0, 0, 3765, 0, 0, 0, 1891, 11, 0, 7741, 0, 0, 0, 0, 0, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 745, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 59, 0, 0, 0, 609, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5704, 6380, 7556, 6360] payload_rows_per_dst=[10010, 15484, 12720, 12544, 16356, 13412, 15112, 0] dup_rows_per_dst=[1324, 268, 0, 354, 0, 720, 0, 0] combine_rows_per_src=[10010, 15484, 12720, 12544, 16356, 13412, 15112, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96136 payload_rows_remote=83174 dup_rows=2168 dup_rows_remote=1722 dedup_ratio=0.022054 router_tokens_per_expert=[0, 0, 3368, 0, 0, 0, 1519, 11, 0, 7636, 1, 0, 0, 1, 0, 127, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 1, 723, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 2, 64, 0, 0, 4, 493, 19, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 5972, 6166, 7777, 7074] payload_rows_per_dst=[8856, 15284, 14150, 12962, 16362, 12960, 15558, 4] dup_rows_per_dst=[940, 246, 0, 446, 0, 536, 0, 0] combine_rows_per_src=[8856, 15284, 14150, 12962, 16362, 12960, 15558, 4] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=51400 payload_rows_remote=35016 dup_rows=46904 dup_rows_remote=1078 dedup_ratio=0.477132 router_tokens_per_expert=[0, 0, 3407, 0, 0, 0, 1726, 10, 0, 7748, 0, 0, 0, 0, 0, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 706, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 101, 1, 0, 0, 755, 33, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 5415, 6695, 7674, 6550] payload_rows_per_dst=[9416, 4772, 4028, 4122, 10900, 1774, 4, 16384] dup_rows_per_dst=[870, 104, 0, 98, 0, 6, 0, 45826] combine_rows_per_src=[9416, 4772, 4028, 4122, 10900, 1774, 4, 16384] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96270 payload_rows_remote=79900 dup_rows=2034 dup_rows_remote=2034 dedup_ratio=0.020691 router_tokens_per_expert=[0, 0, 3774, 0, 0, 0, 1507, 1, 0, 7558, 0, 0, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 635, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 52, 0, 0, 0, 554, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6222, 6161, 7615, 6770] payload_rows_per_dst=[9632, 15122, 13540, 13342, 16370, 13034, 15230, 0] dup_rows_per_dst=[932, 202, 0, 372, 0, 528, 0, 0] combine_rows_per_src=[9632, 15122, 13540, 13342, 16370, 13034, 15230, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=331112448 dispatch_meta_bytes=332080 combine_payload_bytes=331112448 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=331444528 ingress_bytes=400318464 link_bytes=400318464 token_bytes=662556976 weight_movement_bytes=69206016 total_nvlink_bytes=731762992 weight_to_token_ratio=0.104453 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=48128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.506366152e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96136 payload_rows_remote=83174 dup_rows=2168 dup_rows_remote=1722 dedup_ratio=0.022054 router_tokens_per_expert=[0, 0, 3368, 0, 0, 0, 1519, 11, 0, 7636, 1, 0, 0, 1, 0, 127, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 1, 723, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 2, 64, 0, 0, 4, 493, 19, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 5972, 6166, 7777, 7074] payload_rows_per_dst=[8856, 15284, 14150, 12962, 16362, 12960, 15558, 4] dup_rows_per_dst=[940, 246, 0, 446, 0, 536, 0, 0] combine_rows_per_src=[8856, 15284, 14150, 12962, 16362, 12960, 15558, 4] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88350 payload_rows_remote=72002 dup_rows=9954 dup_rows_remote=1178 dedup_ratio=0.101257 router_tokens_per_expert=[0, 0, 3185, 0, 0, 0, 1636, 2, 0, 7739, 0, 0, 0, 0, 0, 124, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 759, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 1, 71, 0, 0, 0, 558, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5969, 6439, 7476, 6951] payload_rows_per_dst=[16348, 248, 13904, 12968, 16384, 13544, 14952, 2] dup_rows_per_dst=[8776, 0, 16, 488, 0, 674, 0, 0] combine_rows_per_src=[16348, 248, 13904, 12968, 16384, 13544, 14952, 2] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=337698816 dispatch_meta_bytes=335360 combine_payload_bytes=337698816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=338034176 ingress_bytes=372301824 link_bytes=372301824 token_bytes=675732992 weight_movement_bytes=34603008 total_nvlink_bytes=710336000 weight_to_token_ratio=0.051208 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=43648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.400941791e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=339632128 dispatch_meta_bytes=342336 combine_payload_bytes=339632128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=339974464 ingress_bytes=374235136 link_bytes=374235136 token_bytes=679606592 weight_movement_bytes=34603008 total_nvlink_bytes=714209600 weight_to_token_ratio=0.050916 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.408216688e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=331112448 dispatch_meta_bytes=332080 combine_payload_bytes=331112448 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=331444528 ingress_bytes=400318464 link_bytes=400318464 token_bytes=662556976 weight_movement_bytes=69206016 total_nvlink_bytes=731762992 weight_to_token_ratio=0.104453 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=48128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.506366152e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=340680704 dispatch_meta_bytes=339584 combine_payload_bytes=340680704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=341020288 ingress_bytes=375283712 link_bytes=375283712 token_bytes=681700992 weight_movement_bytes=34603008 total_nvlink_bytes=716304000 weight_to_token_ratio=0.050760 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=43520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.412162395e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=143425536 dispatch_meta_bytes=144376 combine_payload_bytes=143425536 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=143569912 ingress_bytes=316440576 link_bytes=316440576 token_bytes=286995448 weight_movement_bytes=173015040 total_nvlink_bytes=460010488 weight_to_token_ratio=0.602849 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=21888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.190740412e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=327270400 dispatch_meta_bytes=327736 combine_payload_bytes=327270400 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=327598136 ingress_bytes=396476416 link_bytes=396476416 token_bytes=654868536 weight_movement_bytes=69206016 total_nvlink_bytes=724074552 weight_to_token_ratio=0.105679 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.037500 gemm_active_groups=3 gemm_peak_group_rows=43776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.491908834e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86938 payload_rows_remote=80134 dup_rows=11366 dup_rows_remote=11366 dedup_ratio=0.115621 router_tokens_per_expert=[0, 0, 4153, 0, 0, 0, 1541, 3, 0, 7644, 0, 0, 1, 0, 0, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 629, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 2, 1, 63, 0, 0, 0, 624, 31, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 6270, 5906, 7582, 6364] payload_rows_per_dst=[10284, 15304, 12728, 13406, 0, 12040, 6804, 16372] dup_rows_per_dst=[1110, 286, 0, 394, 0, 676, 0, 8900] combine_rows_per_src=[10284, 15304, 12728, 13406, 0, 12040, 6804, 16372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=340680704 dispatch_meta_bytes=339584 combine_payload_bytes=340680704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=341020288 ingress_bytes=375283712 link_bytes=375283712 token_bytes=681700992 weight_movement_bytes=34603008 total_nvlink_bytes=716304000 weight_to_token_ratio=0.050760 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=43520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.412162395e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95638 payload_rows_remote=82918 dup_rows=2666 dup_rows_remote=2666 dedup_ratio=0.027120 router_tokens_per_expert=[0, 0, 3765, 0, 0, 0, 1891, 11, 0, 7741, 0, 0, 0, 0, 0, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 745, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 59, 0, 0, 0, 609, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5704, 6380, 7556, 6360] payload_rows_per_dst=[10010, 15484, 12720, 12544, 16356, 13412, 15112, 0] dup_rows_per_dst=[1324, 268, 0, 354, 0, 720, 0, 0] combine_rows_per_src=[10010, 15484, 12720, 12544, 16356, 13412, 15112, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85218 payload_rows_remote=74076 dup_rows=13086 dup_rows_remote=11212 dedup_ratio=0.133118 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8112, 0, 81, 1, 5, 0, 195, 3481, 0, 784, 0, 0, 0, 0, 307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6476, 8110, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 8186, 4844, 0, 0, 209, 0, 0, 0, 0, 0, 0, 45, 0, 112, 0, 8, 0] payload_rows_per_dst=[6, 174, 16380, 11448, 16384, 13462, 11142, 16222] dup_rows_per_dst=[0, 0, 8764, 418, 1700, 2, 1874, 328] combine_rows_per_src=[6, 174, 16380, 11448, 16384, 13462, 11142, 16222] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=51400 payload_rows_remote=35016 dup_rows=46904 dup_rows_remote=1078 dedup_ratio=0.477132 router_tokens_per_expert=[0, 0, 3407, 0, 0, 0, 1726, 10, 0, 7748, 0, 0, 0, 0, 0, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 706, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 101, 1, 0, 0, 755, 33, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 5415, 6695, 7674, 6550] payload_rows_per_dst=[9416, 4772, 4028, 4122, 10900, 1774, 4, 16384] dup_rows_per_dst=[870, 104, 0, 98, 0, 6, 0, 45826] combine_rows_per_src=[9416, 4772, 4028, 4122, 10900, 1774, 4, 16384] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=294920192 dispatch_meta_bytes=292720 combine_payload_bytes=294920192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=295212912 ingress_bytes=329523200 link_bytes=329523200 token_bytes=590133104 weight_movement_bytes=34603008 total_nvlink_bytes=624736112 weight_to_token_ratio=0.058636 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=28672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.239969273e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=328228864 dispatch_meta_bytes=366000 combine_payload_bytes=328228864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=328594864 ingress_bytes=362831872 link_bytes=362831872 token_bytes=656823728 weight_movement_bytes=34603008 total_nvlink_bytes=691426736 weight_to_token_ratio=0.052682 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.365307124e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96156 payload_rows_remote=82446 dup_rows=2148 dup_rows_remote=1394 dedup_ratio=0.021851 router_tokens_per_expert=[0, 0, 3366, 0, 0, 0, 1393, 3, 0, 7722, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 753, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 38, 0, 0, 0, 690, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5823, 6467, 7736, 6810] payload_rows_per_dst=[8814, 15446, 13620, 12722, 5556, 13710, 15472, 10816] dup_rows_per_dst=[710, 254, 0, 430, 0, 754, 0, 0] combine_rows_per_src=[8814, 15446, 13620, 12722, 5556, 13710, 15472, 10816] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=339632128 dispatch_meta_bytes=342336 combine_payload_bytes=339632128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=339974464 ingress_bytes=374235136 link_bytes=374235136 token_bytes=679606592 weight_movement_bytes=34603008 total_nvlink_bytes=714209600 weight_to_token_ratio=0.050916 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.408216688e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=303415296 dispatch_meta_bytes=341152 combine_payload_bytes=303415296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=303756448 ingress_bytes=338018304 link_bytes=338018304 token_bytes=607171744 weight_movement_bytes=34603008 total_nvlink_bytes=641774752 weight_to_token_ratio=0.056990 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=38400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.271935665e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=143425536 dispatch_meta_bytes=144376 combine_payload_bytes=143425536 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=143569912 ingress_bytes=316440576 link_bytes=316440576 token_bytes=286995448 weight_movement_bytes=173015040 total_nvlink_bytes=460010488 weight_to_token_ratio=0.602849 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=21888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.190740412e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=337698816 dispatch_meta_bytes=335360 combine_payload_bytes=337698816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=338034176 ingress_bytes=372301824 link_bytes=372301824 token_bytes=675732992 weight_movement_bytes=34603008 total_nvlink_bytes=710336000 weight_to_token_ratio=0.051208 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=43648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.400941791e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97650 payload_rows_remote=81420 dup_rows=654 dup_rows_remote=534 dedup_ratio=0.006653 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8115, 0, 60, 0, 0, 0, 247, 3035, 0, 442, 0, 0, 0, 0, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6869, 8158, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 8185, 5116, 0, 0, 165, 0, 0, 0, 0, 0, 0, 165, 0, 141, 0, 5, 0] payload_rows_per_dst=[16384, 16230, 7424, 13802, 16316, 16370, 10502, 622] dup_rows_per_dst=[6, 120, 24, 440, 0, 4, 60, 0] combine_rows_per_src=[16384, 16230, 7424, 13802, 16316, 16370, 10502, 622] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96504 payload_rows_remote=87734 dup_rows=1800 dup_rows_remote=1718 dedup_ratio=0.018311 router_tokens_per_expert=[0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 8108, 0, 53, 0, 8, 0, 278, 3561, 0, 587, 0, 0, 0, 0, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6602, 8183, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 8114, 4316, 0, 0, 143, 0, 0, 0, 0, 0, 0, 182, 0, 350, 0, 6, 0] payload_rows_per_dst=[16384, 16216, 8770, 13372, 4668, 16228, 8890, 11976] dup_rows_per_dst=[8, 122, 82, 760, 0, 2, 28, 798] combine_rows_per_src=[16384, 16216, 8770, 13372, 4668, 16228, 8890, 11976] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96930 payload_rows_remote=96930 dup_rows=1374 dup_rows_remote=1374 dedup_ratio=0.013977 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8133, 0, 102, 0, 1, 0, 336, 2916, 0, 482, 0, 0, 0, 0, 366, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7002, 8170, 8192, 0, 0, 0, 0, 0, 0, 6, 0, 8179, 4934, 0, 0, 97, 0, 0, 0, 0, 0, 0, 112, 0, 120, 0, 1, 0] payload_rows_per_dst=[16384, 16266, 7430, 14104, 0, 16358, 10048, 16340] dup_rows_per_dst=[4, 206, 38, 634, 0, 12, 14, 466] combine_rows_per_src=[16384, 16266, 7430, 14104, 0, 16358, 10048, 16340] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96608 payload_rows_remote=83354 dup_rows=1696 dup_rows_remote=932 dedup_ratio=0.017253 router_tokens_per_expert=[3, 0, 4, 1, 1, 0, 2, 0, 1, 0, 0, 8124, 1, 105, 0, 2, 2, 336, 3113, 2, 483, 0, 1, 0, 0, 476, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6533, 8174, 8187, 0, 0, 7, 0, 0, 0, 2, 0, 8169, 4991, 0, 0, 126, 0, 0, 0, 0, 0, 0, 98, 0, 200, 0, 6, 1] payload_rows_per_dst=[16380, 16254, 7794, 13254, 2, 16352, 10214, 16358] dup_rows_per_dst=[16, 212, 80, 764, 0, 4, 20, 600] combine_rows_per_src=[16380, 16254, 7794, 13254, 2, 16352, 10214, 16358] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83834 payload_rows_remote=67450 dup_rows=14470 dup_rows_remote=14462 dedup_ratio=0.147196 router_tokens_per_expert=[1, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 8156, 0, 235, 1, 2, 1, 348, 3159, 0, 629, 0, 0, 0, 0, 423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6852, 8181, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 8188, 4402, 0, 0, 118, 0, 0, 0, 0, 0, 0, 117, 0, 139, 0, 3, 0] payload_rows_per_dst=[16384, 16312, 8182, 13814, 16380, 3212, 9032, 518] dup_rows_per_dst=[8, 1008, 92, 736, 12618, 0, 8, 0] combine_rows_per_src=[16384, 16312, 8182, 13814, 16380, 3212, 9032, 518] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=66436 payload_rows_remote=50120 dup_rows=31868 dup_rows_remote=31074 dedup_ratio=0.324178 router_tokens_per_expert=[1, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 8137, 0, 131, 0, 11, 0, 291, 2810, 0, 612, 0, 0, 0, 0, 337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6898, 8158, 8192, 0, 0, 0, 0, 0, 0, 3, 0, 8162, 4864, 0, 0, 143, 0, 0, 0, 0, 0, 0, 86, 0, 307, 0, 4, 0] payload_rows_per_dst=[12, 276, 16384, 674, 16384, 6, 16384, 16316] dup_rows_per_dst=[0, 8, 7316, 0, 13796, 0, 9954, 794] combine_rows_per_src=[12, 276, 16384, 674, 16384, 6, 16384, 16316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=333496320 dispatch_meta_bytes=327816 combine_payload_bytes=333496320 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=333824136 ingress_bytes=402702336 link_bytes=402702336 token_bytes=667320456 weight_movement_bytes=69206016 total_nvlink_bytes=736526472 weight_to_token_ratio=0.103707 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=48128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.515336470e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96930 payload_rows_remote=96930 dup_rows=1374 dup_rows_remote=1374 dedup_ratio=0.013977 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8133, 0, 102, 0, 1, 0, 336, 2916, 0, 482, 0, 0, 0, 0, 366, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7002, 8170, 8192, 0, 0, 0, 0, 0, 0, 6, 0, 8179, 4934, 0, 0, 97, 0, 0, 0, 0, 0, 0, 112, 0, 120, 0, 1, 0] payload_rows_per_dst=[16384, 16266, 7430, 14104, 0, 16358, 10048, 16340] dup_rows_per_dst=[4, 206, 38, 634, 0, 12, 14, 466] combine_rows_per_src=[16384, 16266, 7430, 14104, 0, 16358, 10048, 16340] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=359358464 dispatch_meta_bytes=357808 combine_payload_bytes=359358464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=359716272 ingress_bytes=393961472 link_bytes=393961472 token_bytes=719074736 weight_movement_bytes=34603008 total_nvlink_bytes=753677744 weight_to_token_ratio=0.048122 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=25472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.482445303e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96010 payload_rows_remote=79720 dup_rows=2294 dup_rows_remote=2292 dedup_ratio=0.023336 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8122, 0, 49, 0, 4, 0, 256, 3296, 0, 495, 0, 0, 0, 0, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6605, 8168, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 8145, 4901, 0, 0, 166, 0, 0, 0, 0, 0, 0, 112, 0, 202, 0, 1, 0] payload_rows_per_dst=[16328, 14806, 8736, 13378, 56, 16290, 10080, 16336] dup_rows_per_dst=[0, 94, 808, 706, 0, 2, 54, 630] combine_rows_per_src=[16328, 14806, 8736, 13378, 56, 16290, 10080, 16336] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=397025280 dispatch_meta_bytes=393216 combine_payload_bytes=397025280 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=397418496 ingress_bytes=535437312 link_bytes=535437312 token_bytes=794443776 weight_movement_bytes=138412032 total_nvlink_bytes=932855808 weight_to_token_ratio=0.174225 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=18688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=2.014807499e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=341417984 dispatch_meta_bytes=337144 combine_payload_bytes=341417984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=341755128 ingress_bytes=376020992 link_bytes=376020992 token_bytes=683173112 weight_movement_bytes=34603008 total_nvlink_bytes=717776120 weight_to_token_ratio=0.050650 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=46208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.414936721e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85218 payload_rows_remote=74076 dup_rows=13086 dup_rows_remote=11212 dedup_ratio=0.133118 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8112, 0, 81, 1, 5, 0, 195, 3481, 0, 784, 0, 0, 0, 0, 307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6476, 8110, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 8186, 4844, 0, 0, 209, 0, 0, 0, 0, 0, 0, 45, 0, 112, 0, 8, 0] payload_rows_per_dst=[6, 174, 16380, 11448, 16384, 13462, 11142, 16222] dup_rows_per_dst=[0, 0, 8764, 418, 1700, 2, 1874, 328] combine_rows_per_src=[6, 174, 16380, 11448, 16384, 13462, 11142, 16222] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96504 payload_rows_remote=87734 dup_rows=1800 dup_rows_remote=1718 dedup_ratio=0.018311 router_tokens_per_expert=[0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 8108, 0, 53, 0, 8, 0, 278, 3561, 0, 587, 0, 0, 0, 0, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6602, 8183, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 8114, 4316, 0, 0, 143, 0, 0, 0, 0, 0, 0, 182, 0, 350, 0, 6, 0] payload_rows_per_dst=[16384, 16216, 8770, 13372, 4668, 16228, 8890, 11976] dup_rows_per_dst=[8, 122, 82, 760, 0, 2, 28, 798] combine_rows_per_src=[16384, 16216, 8770, 13372, 4668, 16228, 8890, 11976] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=62042 payload_rows_remote=46292 dup_rows=36262 dup_rows_remote=4742 dedup_ratio=0.368876 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7605, 608, 2690, 625, 0, 0, 0, 0, 0, 269, 0, 0, 0, 0, 7978, 0, 63, 0, 0, 0, 308, 0, 0, 733, 0, 0, 0, 0, 0, 0, 0, 0, 7836, 6880, 424, 0, 8110, 0, 0, 2644, 131, 0, 0, 0, 0, 2238] payload_rows_per_dst=[664, 2374, 7308, 15230, 3372, 6732, 15750, 10612] dup_rows_per_dst=[0, 10, 538, 518, 148, 384, 31520, 3144] combine_rows_per_src=[664, 2374, 7308, 15230, 3372, 6732, 15750, 10612] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=276275200 dispatch_meta_bytes=327648 combine_payload_bytes=276275200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276602848 ingress_bytes=310878208 link_bytes=310878208 token_bytes=552878048 weight_movement_bytes=34603008 total_nvlink_bytes=587481056 weight_to_token_ratio=0.062587 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.169809669e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=205291520 dispatch_meta_bytes=324776 combine_payload_bytes=205291520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=205616296 ingress_bytes=239894528 link_bytes=239894528 token_bytes=410907816 weight_movement_bytes=34603008 total_nvlink_bytes=445510824 weight_to_token_ratio=0.084211 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=46720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.027037958e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=66436 payload_rows_remote=50120 dup_rows=31868 dup_rows_remote=31074 dedup_ratio=0.324178 router_tokens_per_expert=[1, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 8137, 0, 131, 0, 11, 0, 291, 2810, 0, 612, 0, 0, 0, 0, 337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6898, 8158, 8192, 0, 0, 0, 0, 0, 0, 3, 0, 8162, 4864, 0, 0, 143, 0, 0, 0, 0, 0, 0, 86, 0, 307, 0, 4, 0] payload_rows_per_dst=[12, 276, 16384, 674, 16384, 6, 16384, 16316] dup_rows_per_dst=[0, 8, 7316, 0, 13796, 0, 9954, 794] combine_rows_per_src=[12, 276, 16384, 674, 16384, 6, 16384, 16316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96010 payload_rows_remote=79720 dup_rows=2294 dup_rows_remote=2292 dedup_ratio=0.023336 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8122, 0, 49, 0, 4, 0, 256, 3296, 0, 495, 0, 0, 0, 0, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6605, 8168, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 8145, 4901, 0, 0, 166, 0, 0, 0, 0, 0, 0, 112, 0, 202, 0, 1, 0] payload_rows_per_dst=[16328, 14806, 8736, 13378, 56, 16290, 10080, 16336] dup_rows_per_dst=[0, 94, 808, 706, 0, 2, 54, 630] combine_rows_per_src=[16328, 14806, 8736, 13378, 56, 16290, 10080, 16336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=397025280 dispatch_meta_bytes=393216 combine_payload_bytes=397025280 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=397418496 ingress_bytes=535437312 link_bytes=535437312 token_bytes=794443776 weight_movement_bytes=138412032 total_nvlink_bytes=932855808 weight_to_token_ratio=0.174225 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=18688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=2.014807499e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=189612032 dispatch_meta_bytes=204136 combine_payload_bytes=189612032 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=189816168 ingress_bytes=328024064 link_bytes=328024064 token_bytes=379428200 weight_movement_bytes=138412032 total_nvlink_bytes=517840232 weight_to_token_ratio=0.364791 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=16000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.234328145e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=359358464 dispatch_meta_bytes=357808 combine_payload_bytes=359358464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=359716272 ingress_bytes=393961472 link_bytes=393961472 token_bytes=719074736 weight_movement_bytes=34603008 total_nvlink_bytes=753677744 weight_to_token_ratio=0.048122 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=25472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.482445303e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=326533120 dispatch_meta_bytes=328048 combine_payload_bytes=326533120 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=326861168 ingress_bytes=430342144 link_bytes=430342144 token_bytes=653394288 weight_movement_bytes=103809024 total_nvlink_bytes=757203312 weight_to_token_ratio=0.158877 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.619342842e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=303415296 dispatch_meta_bytes=341152 combine_payload_bytes=303415296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=303756448 ingress_bytes=338018304 link_bytes=338018304 token_bytes=607171744 weight_movement_bytes=34603008 total_nvlink_bytes=641774752 weight_to_token_ratio=0.056990 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=38400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.271935665e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94370 payload_rows_remote=86668 dup_rows=3934 dup_rows_remote=3060 dedup_ratio=0.040019 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7921, 805, 2773, 710, 0, 0, 0, 0, 0, 267, 0, 0, 0, 0, 7870, 0, 77, 0, 0, 0, 245, 0, 0, 825, 0, 0, 0, 0, 0, 0, 0, 0, 7857, 7034, 348, 0, 8119, 0, 0, 1664, 55, 0, 0, 0, 0, 2570] payload_rows_per_dst=[16238, 15714, 7702, 15870, 15742, 14478, 696, 7930] dup_rows_per_dst=[0, 24, 874, 506, 642, 1240, 0, 648] combine_rows_per_src=[16238, 15714, 7702, 15870, 15742, 14478, 696, 7930] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=205291520 dispatch_meta_bytes=324776 combine_payload_bytes=205291520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=205616296 ingress_bytes=239894528 link_bytes=239894528 token_bytes=410907816 weight_movement_bytes=34603008 total_nvlink_bytes=445510824 weight_to_token_ratio=0.084211 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=46720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.027037958e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90352 payload_rows_remote=74096 dup_rows=7952 dup_rows_remote=7934 dedup_ratio=0.080892 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7725, 782, 1840, 629, 0, 0, 0, 0, 0, 290, 0, 0, 0, 0, 7933, 0, 46, 0, 0, 0, 394, 0, 0, 600, 0, 0, 0, 0, 0, 0, 0, 0, 8128, 7048, 365, 0, 8153, 26, 0, 3006, 141, 0, 0, 0, 0, 2037] payload_rows_per_dst=[16306, 16256, 15544, 1478, 15886, 14358, 782, 9742] dup_rows_per_dst=[0, 18, 5482, 28, 860, 938, 0, 626] combine_rows_per_src=[16306, 16256, 15544, 1478, 15886, 14358, 782, 9742] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94808 payload_rows_remote=79084 dup_rows=3496 dup_rows_remote=2658 dedup_ratio=0.035563 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7632, 717, 2222, 530, 0, 0, 0, 0, 0, 280, 0, 0, 0, 0, 7854, 0, 57, 0, 0, 0, 370, 0, 0, 443, 0, 0, 0, 0, 0, 0, 0, 0, 8104, 6801, 317, 0, 8118, 6, 0, 2942, 185, 0, 0, 0, 0, 2566] payload_rows_per_dst=[16236, 16208, 6766, 15294, 15724, 13796, 646, 10138] dup_rows_per_dst=[2, 12, 174, 530, 838, 692, 0, 1248] combine_rows_per_src=[16236, 16208, 6766, 15294, 15724, 13796, 646, 10138] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95024 payload_rows_remote=79276 dup_rows=3280 dup_rows_remote=2758 dedup_ratio=0.033366 router_tokens_per_expert=[0, 0, 1, 0, 1, 0, 0, 1, 10, 0, 0, 0, 1, 0, 1, 0, 3, 0, 7829, 892, 2635, 737, 2, 0, 3, 0, 0, 303, 0, 0, 0, 0, 7902, 0, 42, 0, 0, 0, 270, 0, 1, 494, 0, 3, 0, 1, 0, 2, 2, 0, 8000, 6888, 402, 0, 8137, 6, 1, 2033, 154, 0, 0, 0, 0, 2395] payload_rows_per_dst=[16278, 16002, 7816, 15748, 15830, 14022, 820, 8508] dup_rows_per_dst=[2, 22, 722, 522, 598, 756, 0, 658] combine_rows_per_src=[16278, 16002, 7816, 15748, 15830, 14022, 820, 8508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83468 payload_rows_remote=67190 dup_rows=14836 dup_rows_remote=14836 dedup_ratio=0.150920 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7695, 561, 2858, 767, 0, 0, 3, 0, 0, 258, 0, 0, 0, 0, 7918, 0, 73, 0, 0, 0, 294, 0, 0, 433, 0, 0, 0, 0, 0, 0, 0, 0, 7915, 7004, 275, 0, 8139, 8, 0, 2407, 92, 0, 0, 0, 0, 2434] payload_rows_per_dst=[16278, 15830, 16282, 522, 10710, 14268, 566, 9012] dup_rows_per_dst=[0, 36, 12886, 0, 454, 606, 0, 854] combine_rows_per_src=[16278, 15830, 16282, 522, 10710, 14268, 566, 9012] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=303497216 dispatch_meta_bytes=328120 combine_payload_bytes=303497216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=303825336 ingress_bytes=338100224 link_bytes=338100224 token_bytes=607322552 weight_movement_bytes=34603008 total_nvlink_bytes=641925560 weight_to_token_ratio=0.056976 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.272243924e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=57504 payload_rows_remote=41200 dup_rows=40800 dup_rows_remote=31648 dedup_ratio=0.415039 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7775, 765, 2587, 677, 0, 0, 3, 0, 0, 256, 0, 0, 0, 1, 7947, 0, 82, 0, 0, 0, 354, 0, 0, 477, 0, 0, 0, 0, 0, 0, 0, 0, 7768, 7101, 402, 0, 8169, 0, 1, 2143, 81, 0, 0, 0, 0, 2556] payload_rows_per_dst=[0, 14, 7416, 15560, 872, 954, 16384, 16304] dup_rows_per_dst=[0, 0, 642, 510, 0, 0, 30496, 9152] combine_rows_per_src=[0, 14, 7416, 15560, 872, 954, 16384, 16304] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=354992128 dispatch_meta_bytes=358912 combine_payload_bytes=354992128 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=355351040 ingress_bytes=424198144 link_bytes=424198144 token_bytes=710343168 weight_movement_bytes=69206016 total_nvlink_bytes=779549184 weight_to_token_ratio=0.097426 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=20224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.596223465e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94808 payload_rows_remote=79084 dup_rows=3496 dup_rows_remote=2658 dedup_ratio=0.035563 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7632, 717, 2222, 530, 0, 0, 0, 0, 0, 280, 0, 0, 0, 0, 7854, 0, 57, 0, 0, 0, 370, 0, 0, 443, 0, 0, 0, 0, 0, 0, 0, 0, 8104, 6801, 317, 0, 8118, 6, 0, 2942, 185, 0, 0, 0, 0, 2566] payload_rows_per_dst=[16236, 16208, 6766, 15294, 15724, 13796, 646, 10138] dup_rows_per_dst=[2, 12, 174, 530, 838, 692, 0, 1248] combine_rows_per_src=[16236, 16208, 6766, 15294, 15724, 13796, 646, 10138] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=303497216 dispatch_meta_bytes=328120 combine_payload_bytes=303497216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=303825336 ingress_bytes=338100224 link_bytes=338100224 token_bytes=607322552 weight_movement_bytes=34603008 total_nvlink_bytes=641925560 weight_to_token_ratio=0.056976 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.272243924e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94428 payload_rows_remote=80200 dup_rows=3876 dup_rows_remote=3024 dedup_ratio=0.039429 router_tokens_per_expert=[0, 0, 0, 0, 1, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7736, 782, 2589, 705, 0, 0, 0, 0, 0, 258, 0, 0, 0, 0, 7926, 0, 73, 0, 0, 0, 315, 0, 0, 615, 0, 0, 0, 0, 0, 0, 0, 0, 7878, 6925, 321, 0, 8149, 1, 0, 2254, 57, 0, 0, 0, 0, 2553] payload_rows_per_dst=[16298, 15758, 7404, 15484, 15874, 14228, 644, 8738] dup_rows_per_dst=[2, 26, 748, 504, 754, 852, 0, 990] combine_rows_per_src=[16298, 15758, 7404, 15484, 15874, 14228, 644, 8738] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=323928064 dispatch_meta_bytes=326968 combine_payload_bytes=323928064 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=324255032 ingress_bytes=427737088 link_bytes=427737088 token_bytes=648183096 weight_movement_bytes=103809024 total_nvlink_bytes=751992120 weight_to_token_ratio=0.160154 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=46208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.609540226e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=324714496 dispatch_meta_bytes=328136 combine_payload_bytes=324714496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=325042632 ingress_bytes=359317504 link_bytes=359317504 token_bytes=649757128 weight_movement_bytes=34603008 total_nvlink_bytes=684360136 weight_to_token_ratio=0.053255 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=46976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.352082840e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=62042 payload_rows_remote=46292 dup_rows=36262 dup_rows_remote=4742 dedup_ratio=0.368876 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7605, 608, 2690, 625, 0, 0, 0, 0, 0, 269, 0, 0, 0, 0, 7978, 0, 63, 0, 0, 0, 308, 0, 0, 733, 0, 0, 0, 0, 0, 0, 0, 0, 7836, 6880, 424, 0, 8110, 0, 0, 2644, 131, 0, 0, 0, 0, 2238] payload_rows_per_dst=[664, 2374, 7308, 15230, 3372, 6732, 15750, 10612] dup_rows_per_dst=[0, 10, 538, 518, 148, 384, 31520, 3144] combine_rows_per_src=[664, 2374, 7308, 15230, 3372, 6732, 15750, 10612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93468 payload_rows_remote=77084 dup_rows=4836 dup_rows_remote=4836 dedup_ratio=0.049194 router_tokens_per_expert=[0, 4981, 0, 1, 0, 0, 0, 0, 0, 938, 451, 0, 0, 0, 0, 0, 0, 3269, 0, 0, 0, 0, 0, 289, 74, 8026, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1543, 0, 0, 292, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2816, 0, 5272, 0, 0, 0, 4816, 0] payload_rows_per_dst=[9964, 2564, 6942, 16052, 12352, 16384, 16384, 12826] dup_rows_per_dst=[0, 214, 174, 148, 1278, 584, 0, 2438] combine_rows_per_src=[9964, 2564, 6942, 16052, 12352, 16384, 16384, 12826] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=275210240 dispatch_meta_bytes=328104 combine_payload_bytes=275210240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275538344 ingress_bytes=309813248 link_bytes=309813248 token_bytes=550748584 weight_movement_bytes=34603008 total_nvlink_bytes=585351592 weight_to_token_ratio=0.062829 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.165802310e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=168755200 dispatch_meta_bytes=291392 combine_payload_bytes=168755200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=169046592 ingress_bytes=203358208 link_bytes=203358208 token_bytes=337801792 weight_movement_bytes=34603008 total_nvlink_bytes=372404800 weight_to_token_ratio=0.102436 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=19456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=7.652205650e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94370 payload_rows_remote=86668 dup_rows=3934 dup_rows_remote=3060 dedup_ratio=0.040019 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7921, 805, 2773, 710, 0, 0, 0, 0, 0, 267, 0, 0, 0, 0, 7870, 0, 77, 0, 0, 0, 245, 0, 0, 825, 0, 0, 0, 0, 0, 0, 0, 0, 7857, 7034, 348, 0, 8119, 0, 0, 1664, 55, 0, 0, 0, 0, 2570] payload_rows_per_dst=[16238, 15714, 7702, 15870, 15742, 14478, 696, 7930] dup_rows_per_dst=[0, 24, 874, 506, 642, 1240, 0, 648] combine_rows_per_src=[16238, 15714, 7702, 15870, 15742, 14478, 696, 7930] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94428 payload_rows_remote=80200 dup_rows=3876 dup_rows_remote=3024 dedup_ratio=0.039429 router_tokens_per_expert=[0, 0, 0, 0, 1, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7736, 782, 2589, 705, 0, 0, 0, 0, 0, 258, 0, 0, 0, 0, 7926, 0, 73, 0, 0, 0, 315, 0, 0, 615, 0, 0, 0, 0, 0, 0, 0, 0, 7878, 6925, 321, 0, 8149, 1, 0, 2254, 57, 0, 0, 0, 0, 2553] payload_rows_per_dst=[16298, 15758, 7404, 15484, 15874, 14228, 644, 8738] dup_rows_per_dst=[2, 26, 748, 504, 754, 852, 0, 990] combine_rows_per_src=[16298, 15758, 7404, 15484, 15874, 14228, 644, 8738] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=57504 payload_rows_remote=41200 dup_rows=40800 dup_rows_remote=31648 dedup_ratio=0.415039 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7775, 765, 2587, 677, 0, 0, 3, 0, 0, 256, 0, 0, 0, 1, 7947, 0, 82, 0, 0, 0, 354, 0, 0, 477, 0, 0, 0, 0, 0, 0, 0, 0, 7768, 7101, 402, 0, 8169, 0, 1, 2143, 81, 0, 0, 0, 0, 2556] payload_rows_per_dst=[0, 14, 7416, 15560, 872, 954, 16384, 16304] dup_rows_per_dst=[0, 0, 642, 510, 0, 0, 30496, 9152] combine_rows_per_src=[0, 14, 7416, 15560, 872, 954, 16384, 16304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=323928064 dispatch_meta_bytes=326968 combine_payload_bytes=323928064 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=324255032 ingress_bytes=427737088 link_bytes=427737088 token_bytes=648183096 weight_movement_bytes=103809024 total_nvlink_bytes=751992120 weight_to_token_ratio=0.160154 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=46208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.609540226e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=315736064 dispatch_meta_bytes=327680 combine_payload_bytes=315736064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=316063744 ingress_bytes=350339072 link_bytes=350339072 token_bytes=631799808 weight_movement_bytes=34603008 total_nvlink_bytes=666402816 weight_to_token_ratio=0.054769 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.318297723e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=354992128 dispatch_meta_bytes=358912 combine_payload_bytes=354992128 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=355351040 ingress_bytes=424198144 link_bytes=424198144 token_bytes=710343168 weight_movement_bytes=69206016 total_nvlink_bytes=779549184 weight_to_token_ratio=0.097426 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=20224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.596223465e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=328499200 dispatch_meta_bytes=332896 combine_payload_bytes=328499200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=328832096 ingress_bytes=363102208 link_bytes=363102208 token_bytes=657331296 weight_movement_bytes=34603008 total_nvlink_bytes=691934304 weight_to_token_ratio=0.052642 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=44544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.366324377e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=189612032 dispatch_meta_bytes=204136 combine_payload_bytes=189612032 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=189816168 ingress_bytes=328024064 link_bytes=328024064 token_bytes=379428200 weight_movement_bytes=138412032 total_nvlink_bytes=517840232 weight_to_token_ratio=0.364791 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=16000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.234328145e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79090 payload_rows_remote=62738 dup_rows=19214 dup_rows_remote=15970 dedup_ratio=0.195455 router_tokens_per_expert=[0, 4455, 0, 0, 0, 0, 0, 0, 0, 1276, 413, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 0, 0, 0, 343, 104, 8109, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1660, 0, 0, 187, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2919, 0, 4813, 0, 0, 0, 4428, 0] payload_rows_per_dst=[13254, 16352, 16384, 16384, 11796, 374, 0, 4546] dup_rows_per_dst=[4512, 3244, 8808, 1500, 1150, 0, 0, 0] combine_rows_per_src=[13254, 16352, 16384, 16384, 11796, 374, 0, 4546] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88778 payload_rows_remote=78072 dup_rows=9526 dup_rows_remote=6450 dedup_ratio=0.096903 router_tokens_per_expert=[0, 4826, 0, 0, 0, 0, 0, 0, 0, 1048, 560, 0, 0, 0, 0, 0, 0, 3423, 0, 0, 0, 0, 0, 381, 77, 7884, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1159, 0, 0, 337, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3217, 0, 5253, 0, 0, 0, 4603, 0] payload_rows_per_dst=[10400, 16162, 10706, 154, 12022, 12396, 16384, 10554] dup_rows_per_dst=[826, 2822, 3076, 0, 802, 440, 0, 1560] combine_rows_per_src=[10400, 16162, 10706, 154, 12022, 12396, 16384, 10554] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=168755200 dispatch_meta_bytes=291392 combine_payload_bytes=168755200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=169046592 ingress_bytes=203358208 link_bytes=203358208 token_bytes=337801792 weight_movement_bytes=34603008 total_nvlink_bytes=372404800 weight_to_token_ratio=0.102436 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=19456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=7.652205650e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92184 payload_rows_remote=80252 dup_rows=6120 dup_rows_remote=4886 dedup_ratio=0.062256 router_tokens_per_expert=[0, 4529, 0, 0, 0, 0, 0, 0, 0, 1341, 330, 0, 0, 0, 0, 0, 0, 3487, 0, 0, 0, 0, 0, 403, 63, 8104, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1495, 0, 0, 239, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3363, 0, 5088, 0, 0, 0, 4326, 0] payload_rows_per_dst=[9058, 10096, 7534, 7676, 11932, 16384, 16384, 13120] dup_rows_per_dst=[0, 1830, 246, 74, 1234, 478, 0, 2258] combine_rows_per_src=[9058, 10096, 7534, 7676, 11932, 16384, 16384, 13120] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90936 payload_rows_remote=90828 dup_rows=7368 dup_rows_remote=7368 dedup_ratio=0.074951 router_tokens_per_expert=[0, 5037, 1, 0, 0, 0, 0, 0, 0, 1332, 361, 1, 1, 2, 1, 0, 1, 3375, 0, 1, 0, 0, 0, 319, 53, 8020, 0, 0, 8187, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1520, 0, 0, 269, 0, 0, 1, 0, 8190, 4, 3, 0, 0, 0, 0, 2, 0, 0, 3746, 0, 4099, 0, 0, 0, 4625, 0] payload_rows_per_dst=[10076, 16318, 7292, 108, 10588, 16382, 16380, 13792] dup_rows_per_dst=[0, 3118, 100, 0, 650, 546, 4, 2950] combine_rows_per_src=[10076, 16318, 7292, 108, 10588, 16382, 16380, 13792] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=256974848 dispatch_meta_bytes=314832 combine_payload_bytes=256974848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=257289680 ingress_bytes=291577856 link_bytes=291577856 token_bytes=514264528 weight_movement_bytes=34603008 total_nvlink_bytes=548867536 weight_to_token_ratio=0.067286 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=36480 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.097183998e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74728 payload_rows_remote=60534 dup_rows=23576 dup_rows_remote=17900 dedup_ratio=0.239827 router_tokens_per_expert=[0, 5186, 0, 1, 0, 0, 0, 0, 0, 1081, 434, 0, 0, 0, 0, 0, 0, 2664, 0, 0, 0, 0, 0, 681, 55, 8082, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1498, 0, 0, 196, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3683, 0, 4458, 0, 0, 0, 4748, 0] payload_rows_per_dst=[14194, 16328, 16384, 16384, 11046, 392, 0, 0] dup_rows_per_dst=[5676, 2866, 6690, 7476, 868, 0, 0, 0] combine_rows_per_src=[14194, 16328, 16384, 16384, 11046, 392, 0, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=319782912 dispatch_meta_bytes=338088 combine_payload_bytes=319782912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=320121000 ingress_bytes=354385920 link_bytes=354385920 token_bytes=639903912 weight_movement_bytes=34603008 total_nvlink_bytes=674506920 weight_to_token_ratio=0.054075 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=26496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.333525687e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91922 payload_rows_remote=78172 dup_rows=6382 dup_rows_remote=2368 dedup_ratio=0.064921 router_tokens_per_expert=[0, 5167, 0, 8, 0, 0, 0, 0, 0, 1237, 425, 0, 0, 0, 0, 0, 0, 3242, 0, 0, 0, 0, 0, 378, 112, 7915, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1941, 0, 0, 340, 0, 0, 0, 0, 8186, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3308, 0, 4279, 0, 0, 0, 4421, 0] payload_rows_per_dst=[10334, 2982, 7066, 15830, 9200, 16376, 16384, 13750] dup_rows_per_dst=[16, 342, 174, 224, 934, 678, 0, 4014] combine_rows_per_src=[10334, 2982, 7066, 15830, 9200, 16376, 16384, 13750] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92184 payload_rows_remote=80252 dup_rows=6120 dup_rows_remote=4886 dedup_ratio=0.062256 router_tokens_per_expert=[0, 4529, 0, 0, 0, 0, 0, 0, 0, 1341, 330, 0, 0, 0, 0, 0, 0, 3487, 0, 0, 0, 0, 0, 403, 63, 8104, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1495, 0, 0, 239, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3363, 0, 5088, 0, 0, 0, 4326, 0] payload_rows_per_dst=[9058, 10096, 7534, 7676, 11932, 16384, 16384, 13120] dup_rows_per_dst=[0, 1830, 246, 74, 1234, 478, 0, 2258] combine_rows_per_src=[9058, 10096, 7534, 7676, 11932, 16384, 16384, 13120] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=256974848 dispatch_meta_bytes=314832 combine_payload_bytes=256974848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=257289680 ingress_bytes=291577856 link_bytes=291577856 token_bytes=514264528 weight_movement_bytes=34603008 total_nvlink_bytes=548867536 weight_to_token_ratio=0.067286 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=36480 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.097183998e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93286 payload_rows_remote=76902 dup_rows=5018 dup_rows_remote=4512 dedup_ratio=0.051046 router_tokens_per_expert=[0, 4997, 0, 0, 0, 0, 0, 0, 0, 1104, 429, 0, 0, 0, 0, 0, 0, 2902, 0, 0, 0, 0, 0, 462, 89, 8058, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1564, 0, 0, 253, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3605, 0, 4662, 0, 0, 0, 4643, 0] payload_rows_per_dst=[9994, 2878, 6552, 16116, 11400, 16384, 16384, 13578] dup_rows_per_dst=[0, 188, 176, 178, 1052, 506, 0, 2918] combine_rows_per_src=[9994, 2878, 6552, 16116, 11400, 16384, 16384, 13578] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=328712192 dispatch_meta_bytes=340552 combine_payload_bytes=328712192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=329052744 ingress_bytes=363315200 link_bytes=363315200 token_bytes=657764936 weight_movement_bytes=34603008 total_nvlink_bytes=692367944 weight_to_token_ratio=0.052607 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=36864 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.367125848e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=372031488 dispatch_meta_bytes=392784 combine_payload_bytes=372031488 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=372424272 ingress_bytes=475840512 link_bytes=475840512 token_bytes=744455760 weight_movement_bytes=103809024 total_nvlink_bytes=848264784 weight_to_token_ratio=0.139443 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=27904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.790549538e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93468 payload_rows_remote=77084 dup_rows=4836 dup_rows_remote=4836 dedup_ratio=0.049194 router_tokens_per_expert=[0, 4981, 0, 1, 0, 0, 0, 0, 0, 938, 451, 0, 0, 0, 0, 0, 0, 3269, 0, 0, 0, 0, 0, 289, 74, 8026, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1543, 0, 0, 292, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2816, 0, 5272, 0, 0, 0, 4816, 0] payload_rows_per_dst=[9964, 2564, 6942, 16052, 12352, 16384, 16384, 12826] dup_rows_per_dst=[0, 214, 174, 148, 1278, 584, 0, 2438] combine_rows_per_src=[9964, 2564, 6942, 16052, 12352, 16384, 16384, 12826] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=247947264 dispatch_meta_bytes=313736 combine_payload_bytes=247947264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=248261000 ingress_bytes=282550272 link_bytes=282550272 token_bytes=496208264 weight_movement_bytes=34603008 total_nvlink_bytes=530811272 weight_to_token_ratio=0.069735 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=39296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.063213926e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87590 payload_rows_remote=71524 dup_rows=10714 dup_rows_remote=10712 dedup_ratio=0.108988 router_tokens_per_expert=[4185, 0, 0, 8192, 0, 0, 0, 754, 3, 0, 0, 0, 0, 0, 531, 0, 0, 0, 0, 0, 0, 8192, 0, 22, 0, 0, 0, 0, 3302, 0, 0, 0, 0, 1796, 0, 8033, 0, 0, 7507, 0, 0, 0, 0, 97, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5204, 1333] payload_rows_per_dst=[8452, 4058, 1286, 16384, 13114, 16384, 16066, 11846] dup_rows_per_dst=[182, 362, 2, 6604, 2140, 194, 2, 1228] combine_rows_per_src=[8452, 4058, 1286, 16384, 13114, 16384, 16066, 11846] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88778 payload_rows_remote=78072 dup_rows=9526 dup_rows_remote=6450 dedup_ratio=0.096903 router_tokens_per_expert=[0, 4826, 0, 0, 0, 0, 0, 0, 0, 1048, 560, 0, 0, 0, 0, 0, 0, 3423, 0, 0, 0, 0, 0, 381, 77, 7884, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1159, 0, 0, 337, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3217, 0, 5253, 0, 0, 0, 4603, 0] payload_rows_per_dst=[10400, 16162, 10706, 154, 12022, 12396, 16384, 10554] dup_rows_per_dst=[826, 2822, 3076, 0, 802, 440, 0, 1560] combine_rows_per_src=[10400, 16162, 10706, 154, 12022, 12396, 16384, 10554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93286 payload_rows_remote=76902 dup_rows=5018 dup_rows_remote=4512 dedup_ratio=0.051046 router_tokens_per_expert=[0, 4997, 0, 0, 0, 0, 0, 0, 0, 1104, 429, 0, 0, 0, 0, 0, 0, 2902, 0, 0, 0, 0, 0, 462, 89, 8058, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1564, 0, 0, 253, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3605, 0, 4662, 0, 0, 0, 4643, 0] payload_rows_per_dst=[9994, 2878, 6552, 16116, 11400, 16384, 16384, 13578] dup_rows_per_dst=[0, 188, 176, 178, 1052, 506, 0, 2918] combine_rows_per_src=[9994, 2878, 6552, 16116, 11400, 16384, 16384, 13578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91922 payload_rows_remote=78172 dup_rows=6382 dup_rows_remote=2368 dedup_ratio=0.064921 router_tokens_per_expert=[0, 5167, 0, 8, 0, 0, 0, 0, 0, 1237, 425, 0, 0, 0, 0, 0, 0, 3242, 0, 0, 0, 0, 0, 378, 112, 7915, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1941, 0, 0, 340, 0, 0, 0, 0, 8186, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3308, 0, 4279, 0, 0, 0, 4421, 0] payload_rows_per_dst=[10334, 2982, 7066, 15830, 9200, 16376, 16384, 13750] dup_rows_per_dst=[16, 342, 174, 224, 934, 678, 0, 4014] combine_rows_per_src=[10334, 2982, 7066, 15830, 9200, 16376, 16384, 13750] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=320192512 dispatch_meta_bytes=322160 combine_payload_bytes=320192512 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=320514672 ingress_bytes=458604544 link_bytes=458604544 token_bytes=640707184 weight_movement_bytes=138412032 total_nvlink_bytes=779119216 weight_to_token_ratio=0.216030 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=25728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.725691978e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=328712192 dispatch_meta_bytes=340552 combine_payload_bytes=328712192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=329052744 ingress_bytes=363315200 link_bytes=363315200 token_bytes=657764936 weight_movement_bytes=34603008 total_nvlink_bytes=692367944 weight_to_token_ratio=0.052607 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=36864 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.367125848e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=292962304 dispatch_meta_bytes=328944 combine_payload_bytes=292962304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293291248 ingress_bytes=327565312 link_bytes=327565312 token_bytes=586253552 weight_movement_bytes=34603008 total_nvlink_bytes=620856560 weight_to_token_ratio=0.059024 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.232601898e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=319782912 dispatch_meta_bytes=338088 combine_payload_bytes=319782912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=320121000 ingress_bytes=354385920 link_bytes=354385920 token_bytes=639903912 weight_movement_bytes=34603008 total_nvlink_bytes=674506920 weight_to_token_ratio=0.054075 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=26496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.333525687e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=314990592 dispatch_meta_bytes=325656 combine_payload_bytes=314990592 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=315316248 ingress_bytes=384196608 link_bytes=384196608 token_bytes=630306840 weight_movement_bytes=69206016 total_nvlink_bytes=699512856 weight_to_token_ratio=0.109797 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=46080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.445700906e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=315736064 dispatch_meta_bytes=327680 combine_payload_bytes=315736064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=316063744 ingress_bytes=350339072 link_bytes=350339072 token_bytes=631799808 weight_movement_bytes=34603008 total_nvlink_bytes=666402816 weight_to_token_ratio=0.054769 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.318297723e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80776 payload_rows_remote=66184 dup_rows=17528 dup_rows_remote=16348 dedup_ratio=0.178304 router_tokens_per_expert=[3881, 0, 0, 8192, 0, 0, 0, 533, 0, 0, 0, 0, 0, 0, 645, 0, 0, 0, 0, 0, 0, 8192, 0, 8, 0, 0, 0, 0, 3829, 0, 0, 0, 0, 1620, 0, 8067, 0, 0, 7241, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5117, 1755] payload_rows_per_dst=[16384, 14592, 16384, 7658, 14590, 144, 1962, 9062] dup_rows_per_dst=[8828, 1180, 16, 0, 6322, 0, 0, 1182] combine_rows_per_src=[16384, 14592, 16384, 7658, 14590, 144, 1962, 9062] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94798 payload_rows_remote=78414 dup_rows=3506 dup_rows_remote=3502 dedup_ratio=0.035665 router_tokens_per_expert=[4096, 0, 0, 8192, 0, 0, 0, 622, 0, 0, 0, 0, 0, 0, 336, 0, 0, 0, 0, 0, 0, 8192, 0, 2, 0, 0, 0, 0, 3411, 0, 0, 0, 0, 1892, 0, 7921, 0, 0, 7362, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5404, 1617] payload_rows_per_dst=[9866, 14774, 16384, 6822, 3784, 15038, 15842, 12288] dup_rows_per_dst=[946, 622, 4, 0, 0, 180, 0, 1754] combine_rows_per_src=[9866, 14774, 16384, 6822, 3784, 15038, 15842, 12288] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=271089664 dispatch_meta_bytes=330128 combine_payload_bytes=271089664 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271419792 ingress_bytes=305692672 link_bytes=305692672 token_bytes=542509456 weight_movement_bytes=34603008 total_nvlink_bytes=577112464 weight_to_token_ratio=0.063783 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=45312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.150296914e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=320192512 dispatch_meta_bytes=322160 combine_payload_bytes=320192512 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=320514672 ingress_bytes=458604544 link_bytes=458604544 token_bytes=640707184 weight_movement_bytes=138412032 total_nvlink_bytes=779119216 weight_to_token_ratio=0.216030 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=25728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.725691978e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95728 payload_rows_remote=92804 dup_rows=2576 dup_rows_remote=2576 dedup_ratio=0.026204 router_tokens_per_expert=[3721, 0, 0, 8192, 0, 0, 0, 517, 0, 0, 0, 0, 0, 0, 446, 0, 0, 0, 0, 0, 0, 8192, 0, 7, 0, 0, 0, 0, 3954, 0, 0, 0, 0, 1462, 0, 8080, 0, 0, 7412, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5489, 1634] payload_rows_per_dst=[8400, 14898, 16384, 7908, 2924, 16384, 16160, 12670] dup_rows_per_dst=[76, 818, 14, 0, 0, 92, 0, 1576] combine_rows_per_src=[8400, 14898, 16384, 7908, 2924, 16384, 16160, 12670] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95484 payload_rows_remote=88404 dup_rows=2820 dup_rows_remote=2818 dedup_ratio=0.028687 router_tokens_per_expert=[4183, 0, 0, 8190, 0, 1, 0, 590, 0, 0, 0, 1, 1, 0, 563, 0, 0, 0, 0, 0, 0, 8187, 0, 6, 1, 3, 0, 0, 3537, 0, 0, 0, 0, 1775, 0, 8085, 0, 0, 7091, 0, 0, 2, 1, 110, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 1, 5216, 1604] payload_rows_per_dst=[9408, 14374, 16374, 7080, 3550, 16384, 16174, 12140] dup_rows_per_dst=[140, 938, 12, 2, 0, 224, 0, 1504] combine_rows_per_src=[9408, 14374, 16374, 7080, 3550, 16384, 16174, 12140] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=321183744 dispatch_meta_bytes=327664 combine_payload_bytes=321183744 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=321511408 ingress_bytes=390389760 link_bytes=390389760 token_bytes=642695152 weight_movement_bytes=69206016 total_nvlink_bytes=711901168 weight_to_token_ratio=0.107681 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.037500 gemm_active_groups=4 gemm_peak_group_rows=45440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.469005238e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74864 payload_rows_remote=58480 dup_rows=23440 dup_rows_remote=14168 dedup_ratio=0.238444 router_tokens_per_expert=[4012, 0, 0, 8192, 0, 0, 0, 624, 3, 0, 0, 0, 0, 0, 471, 0, 0, 0, 0, 0, 0, 8192, 0, 38, 0, 0, 0, 0, 3533, 0, 0, 0, 0, 1603, 0, 8083, 0, 0, 7497, 0, 0, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5113, 1681] payload_rows_per_dst=[16384, 15066, 16384, 7066, 16382, 220, 0, 3362] dup_rows_per_dst=[9272, 876, 76, 0, 13216, 0, 0, 0] combine_rows_per_src=[16384, 15066, 16384, 7066, 16382, 220, 0, 3362] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95728 payload_rows_remote=92804 dup_rows=2576 dup_rows_remote=2576 dedup_ratio=0.026204 router_tokens_per_expert=[3721, 0, 0, 8192, 0, 0, 0, 517, 0, 0, 0, 0, 0, 0, 446, 0, 0, 0, 0, 0, 0, 8192, 0, 7, 0, 0, 0, 0, 3954, 0, 0, 0, 0, 1462, 0, 8080, 0, 0, 7412, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5489, 1634] payload_rows_per_dst=[8400, 14898, 16384, 7908, 2924, 16384, 16160, 12670] dup_rows_per_dst=[76, 818, 14, 0, 0, 92, 0, 1576] combine_rows_per_src=[8400, 14898, 16384, 7908, 2924, 16384, 16160, 12670] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85252 payload_rows_remote=72266 dup_rows=13052 dup_rows_remote=9956 dedup_ratio=0.132772 router_tokens_per_expert=[4031, 0, 0, 8192, 0, 0, 0, 527, 0, 0, 0, 0, 0, 0, 418, 0, 0, 0, 0, 0, 0, 8191, 0, 15, 0, 0, 0, 0, 3667, 0, 0, 0, 0, 1542, 0, 7989, 0, 0, 7434, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5485, 1599] payload_rows_per_dst=[1054, 836, 6180, 16382, 15448, 16384, 15982, 12986] dup_rows_per_dst=[0, 0, 0, 7334, 2504, 118, 0, 3096] combine_rows_per_src=[1054, 836, 6180, 16382, 15448, 16384, 15982, 12986] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=271089664 dispatch_meta_bytes=330128 combine_payload_bytes=271089664 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271419792 ingress_bytes=305692672 link_bytes=305692672 token_bytes=542509456 weight_movement_bytes=34603008 total_nvlink_bytes=577112464 weight_to_token_ratio=0.063783 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=45312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.150296914e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=380125184 dispatch_meta_bytes=381520 combine_payload_bytes=380125184 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=380506704 ingress_bytes=483934208 link_bytes=483934208 token_bytes=760631888 weight_movement_bytes=103809024 total_nvlink_bytes=864440912 weight_to_token_ratio=0.136477 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.821005465e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92146 payload_rows_remote=75762 dup_rows=6158 dup_rows_remote=5984 dedup_ratio=0.062642 router_tokens_per_expert=[3917, 0, 0, 8192, 0, 0, 0, 539, 0, 0, 0, 0, 0, 0, 526, 0, 0, 0, 0, 0, 0, 8192, 0, 6, 0, 0, 0, 0, 3736, 0, 0, 0, 0, 2165, 0, 8058, 0, 0, 6929, 0, 0, 0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5229, 1576] payload_rows_per_dst=[8824, 13952, 8794, 11588, 4330, 16384, 16116, 12158] dup_rows_per_dst=[88, 958, 10, 3476, 0, 174, 0, 1452] combine_rows_per_src=[8824, 13952, 8794, 11588, 4330, 16384, 16116, 12158] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=362102784 dispatch_meta_bytes=364888 combine_payload_bytes=362102784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=362467672 ingress_bytes=396705792 link_bytes=396705792 token_bytes=724570456 weight_movement_bytes=34603008 total_nvlink_bytes=759173464 weight_to_token_ratio=0.047757 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=29056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.492771958e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87590 payload_rows_remote=71524 dup_rows=10714 dup_rows_remote=10712 dedup_ratio=0.108988 router_tokens_per_expert=[4185, 0, 0, 8192, 0, 0, 0, 754, 3, 0, 0, 0, 0, 0, 531, 0, 0, 0, 0, 0, 0, 8192, 0, 22, 0, 0, 0, 0, 3302, 0, 0, 0, 0, 1796, 0, 8033, 0, 0, 7507, 0, 0, 0, 0, 97, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5204, 1333] payload_rows_per_dst=[8452, 4058, 1286, 16384, 13114, 16384, 16066, 11846] dup_rows_per_dst=[182, 362, 2, 6604, 2140, 194, 2, 1228] combine_rows_per_src=[8452, 4058, 1286, 16384, 13114, 16384, 16066, 11846] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74858 payload_rows_remote=58474 dup_rows=23446 dup_rows_remote=19218 dedup_ratio=0.238505 router_tokens_per_expert=[0, 0, 55, 0, 7876, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 3753, 0, 0, 0, 1697, 0, 8192, 0, 17, 80, 0, 0, 0, 0, 1, 544, 0, 0, 118, 8192, 0, 0, 8192, 16, 0, 0, 1582, 0, 0, 0, 0, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[2782, 226, 16384, 5064, 1250, 16384, 16384, 16384] dup_rows_per_dst=[18, 0, 15076, 396, 0, 3728, 4228, 0] combine_rows_per_src=[2782, 226, 16384, 5064, 1250, 16384, 16384, 16384] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=239534080 dispatch_meta_bytes=290592 combine_payload_bytes=239534080 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=239824672 ingress_bytes=343343104 link_bytes=343343104 token_bytes=479358752 weight_movement_bytes=103809024 total_nvlink_bytes=583167776 weight_to_token_ratio=0.216558 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=27392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.291972459e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94798 payload_rows_remote=78414 dup_rows=3506 dup_rows_remote=3502 dedup_ratio=0.035665 router_tokens_per_expert=[4096, 0, 0, 8192, 0, 0, 0, 622, 0, 0, 0, 0, 0, 0, 336, 0, 0, 0, 0, 0, 0, 8192, 0, 2, 0, 0, 0, 0, 3411, 0, 0, 0, 0, 1892, 0, 7921, 0, 0, 7362, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5404, 1617] payload_rows_per_dst=[9866, 14774, 16384, 6822, 3784, 15038, 15842, 12288] dup_rows_per_dst=[946, 622, 4, 0, 0, 180, 0, 1754] combine_rows_per_src=[9866, 14774, 16384, 6822, 3784, 15038, 15842, 12288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92146 payload_rows_remote=75762 dup_rows=6158 dup_rows_remote=5984 dedup_ratio=0.062642 router_tokens_per_expert=[3917, 0, 0, 8192, 0, 0, 0, 539, 0, 0, 0, 0, 0, 0, 526, 0, 0, 0, 0, 0, 0, 8192, 0, 6, 0, 0, 0, 0, 3736, 0, 0, 0, 0, 2165, 0, 8058, 0, 0, 6929, 0, 0, 0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5229, 1576] payload_rows_per_dst=[8824, 13952, 8794, 11588, 4330, 16384, 16116, 12158] dup_rows_per_dst=[88, 958, 10, 3476, 0, 174, 0, 1452] combine_rows_per_src=[8824, 13952, 8794, 11588, 4330, 16384, 16116, 12158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85252 payload_rows_remote=72266 dup_rows=13052 dup_rows_remote=9956 dedup_ratio=0.132772 router_tokens_per_expert=[4031, 0, 0, 8192, 0, 0, 0, 527, 0, 0, 0, 0, 0, 0, 418, 0, 0, 0, 0, 0, 0, 8191, 0, 15, 0, 0, 0, 0, 3667, 0, 0, 0, 0, 1542, 0, 7989, 0, 0, 7434, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5485, 1599] payload_rows_per_dst=[1054, 836, 6180, 16382, 15448, 16384, 15982, 12986] dup_rows_per_dst=[0, 0, 0, 7334, 2504, 118, 0, 3096] combine_rows_per_src=[1054, 836, 6180, 16382, 15448, 16384, 15982, 12986] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=239509504 dispatch_meta_bytes=310768 combine_payload_bytes=239509504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=239820272 ingress_bytes=274112512 link_bytes=274112512 token_bytes=479329776 weight_movement_bytes=34603008 total_nvlink_bytes=513932784 weight_to_token_ratio=0.072190 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.037500 gemm_active_groups=4 gemm_peak_group_rows=31744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.031463315e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=380125184 dispatch_meta_bytes=381520 combine_payload_bytes=380125184 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=380506704 ingress_bytes=483934208 link_bytes=483934208 token_bytes=760631888 weight_movement_bytes=103809024 total_nvlink_bytes=864440912 weight_to_token_ratio=0.136477 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.821005465e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=296001536 dispatch_meta_bytes=328888 combine_payload_bytes=296001536 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=296330424 ingress_bytes=365207552 link_bytes=365207552 token_bytes=592331960 weight_movement_bytes=69206016 total_nvlink_bytes=661537976 weight_to_token_ratio=0.116837 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=35456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.374246617e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=310321152 dispatch_meta_bytes=326984 combine_payload_bytes=310321152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=310648136 ingress_bytes=344924160 link_bytes=344924160 token_bytes=620969288 weight_movement_bytes=34603008 total_nvlink_bytes=655572296 weight_to_token_ratio=0.055724 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=48512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.297921845e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=321183744 dispatch_meta_bytes=327664 combine_payload_bytes=321183744 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=321511408 ingress_bytes=390389760 link_bytes=390389760 token_bytes=642695152 weight_movement_bytes=69206016 total_nvlink_bytes=711901168 weight_to_token_ratio=0.107681 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.037500 gemm_active_groups=4 gemm_peak_group_rows=45440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.469005238e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92352 payload_rows_remote=75970 dup_rows=5952 dup_rows_remote=5556 dedup_ratio=0.060547 router_tokens_per_expert=[0, 0, 62, 0, 7922, 0, 0, 0, 0, 0, 0, 0, 198, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 3201, 0, 0, 0, 1898, 0, 8192, 0, 19, 93, 0, 0, 0, 0, 0, 584, 0, 0, 119, 8192, 0, 0, 8192, 3, 0, 0, 1620, 0, 0, 0, 0, 666, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15844, 16382, 6402, 16384, 16384, 16384, 4572, 0] dup_rows_per_dst=[124, 396, 0, 3834, 1354, 244, 0, 0] combine_rows_per_src=[15844, 16382, 6402, 16384, 16384, 16384, 4572, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=292962304 dispatch_meta_bytes=328944 combine_payload_bytes=292962304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293291248 ingress_bytes=327565312 link_bytes=327565312 token_bytes=586253552 weight_movement_bytes=34603008 total_nvlink_bytes=620856560 weight_to_token_ratio=0.059024 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.232601898e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91584 payload_rows_remote=85512 dup_rows=6720 dup_rows_remote=6720 dedup_ratio=0.068359 router_tokens_per_expert=[0, 0, 123, 0, 7818, 0, 0, 0, 0, 0, 0, 0, 825, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 3036, 0, 0, 0, 1633, 0, 8192, 0, 14, 94, 0, 0, 0, 0, 1, 676, 0, 0, 112, 8192, 0, 0, 8192, 52, 0, 0, 1175, 0, 0, 0, 0, 827, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15652, 16380, 6072, 16384, 16384, 332, 4002, 16378] dup_rows_per_dst=[230, 1650, 0, 3294, 1542, 2, 2, 0] combine_rows_per_src=[15652, 16380, 6072, 16384, 16384, 332, 4002, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=311173120 dispatch_meta_bytes=326104 combine_payload_bytes=311173120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=311499224 ingress_bytes=345776128 link_bytes=345776128 token_bytes=622672344 weight_movement_bytes=34603008 total_nvlink_bytes=657275352 weight_to_token_ratio=0.055572 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=46848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.301127732e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92140 payload_rows_remote=75756 dup_rows=6164 dup_rows_remote=5002 dedup_ratio=0.062703 router_tokens_per_expert=[0, 0, 74, 0, 7922, 0, 0, 0, 0, 0, 0, 0, 162, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 3475, 0, 0, 0, 1694, 0, 8192, 0, 17, 77, 0, 0, 0, 0, 1, 503, 0, 0, 109, 8192, 0, 0, 8192, 16, 0, 0, 1725, 0, 0, 0, 0, 608, 0, 0, 0, 0, 0, 1, 0, 0, 0] payload_rows_per_dst=[15846, 16384, 6950, 5084, 16384, 250, 14858, 16384] dup_rows_per_dst=[146, 324, 0, 390, 1162, 0, 4140, 2] combine_rows_per_src=[15846, 16384, 6950, 5084, 16384, 250, 14858, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=310321152 dispatch_meta_bytes=326984 combine_payload_bytes=310321152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=310648136 ingress_bytes=344924160 link_bytes=344924160 token_bytes=620969288 weight_movement_bytes=34603008 total_nvlink_bytes=655572296 weight_to_token_ratio=0.055724 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=48512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.297921845e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=296001536 dispatch_meta_bytes=328888 combine_payload_bytes=296001536 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=296330424 ingress_bytes=365207552 link_bytes=365207552 token_bytes=592331960 weight_movement_bytes=69206016 total_nvlink_bytes=661537976 weight_to_token_ratio=0.116837 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=35456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.374246617e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92476 payload_rows_remote=76096 dup_rows=5828 dup_rows_remote=1998 dedup_ratio=0.059285 router_tokens_per_expert=[6, 0, 79, 0, 7823, 0, 0, 1, 1, 1, 0, 2, 221, 1, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 3274, 0, 0, 2, 1892, 0, 8185, 0, 26, 83, 0, 0, 0, 1, 7, 613, 2, 0, 126, 8188, 0, 4, 8190, 13, 0, 0, 1552, 0, 0, 0, 0, 669, 0, 0, 0, 0, 0, 0, 2, 0, 0] payload_rows_per_dst=[15674, 16382, 6548, 16380, 16382, 286, 4440, 16384] dup_rows_per_dst=[144, 446, 0, 3830, 1406, 0, 2, 0] combine_rows_per_src=[15674, 16382, 6548, 16380, 16382, 286, 4440, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=350257152 dispatch_meta_bytes=368928 combine_payload_bytes=350257152 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=350626080 ingress_bytes=419463168 link_bytes=419463168 token_bytes=700883232 weight_movement_bytes=69206016 total_nvlink_bytes=770089248 weight_to_token_ratio=0.098741 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=26752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.578406132e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91946 payload_rows_remote=76202 dup_rows=6358 dup_rows_remote=6208 dedup_ratio=0.064677 router_tokens_per_expert=[0, 0, 85, 0, 7862, 0, 0, 0, 0, 0, 0, 0, 132, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 3030, 0, 0, 0, 2128, 0, 8192, 0, 19, 83, 0, 0, 0, 0, 1, 619, 0, 0, 78, 8192, 0, 0, 8192, 39, 0, 0, 1790, 0, 0, 0, 0, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15744, 16382, 6060, 16384, 16384, 16384, 4608, 0] dup_rows_per_dst=[150, 264, 0, 4294, 1406, 234, 10, 0] combine_rows_per_src=[15744, 16382, 6060, 16384, 16384, 16384, 4608, 0] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92140 payload_rows_remote=75756 dup_rows=6164 dup_rows_remote=5002 dedup_ratio=0.062703 router_tokens_per_expert=[0, 0, 74, 0, 7922, 0, 0, 0, 0, 0, 0, 0, 162, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 3475, 0, 0, 0, 1694, 0, 8192, 0, 17, 77, 0, 0, 0, 0, 1, 503, 0, 0, 109, 8192, 0, 0, 8192, 16, 0, 0, 1725, 0, 0, 0, 0, 608, 0, 0, 0, 0, 0, 1, 0, 0, 0] payload_rows_per_dst=[15846, 16384, 6950, 5084, 16384, 250, 14858, 16384] dup_rows_per_dst=[146, 324, 0, 390, 1162, 0, 4140, 2] combine_rows_per_src=[15846, 16384, 6950, 5084, 16384, 250, 14858, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70842 payload_rows_remote=54458 dup_rows=27462 dup_rows_remote=27458 dedup_ratio=0.279358 router_tokens_per_expert=[0, 0, 26, 0, 7866, 0, 0, 0, 0, 0, 0, 0, 173, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 3799, 0, 0, 0, 1593, 0, 8189, 0, 6, 96, 0, 0, 0, 0, 1, 762, 0, 0, 141, 8192, 0, 0, 8192, 7, 0, 0, 1312, 0, 0, 0, 0, 607, 0, 0, 0, 1, 0, 1, 0, 0, 0] payload_rows_per_dst=[52, 346, 16376, 3198, 1718, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 7598, 0, 0, 16028, 3832, 4] combine_rows_per_src=[52, 346, 16376, 3198, 1718, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=311173120 dispatch_meta_bytes=326104 combine_payload_bytes=311173120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=311499224 ingress_bytes=345776128 link_bytes=345776128 token_bytes=622672344 weight_movement_bytes=34603008 total_nvlink_bytes=657275352 weight_to_token_ratio=0.055572 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=46848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.301127732e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=310296576 dispatch_meta_bytes=323032 combine_payload_bytes=310296576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=310619608 ingress_bytes=344899584 link_bytes=344899584 token_bytes=620916184 weight_movement_bytes=34603008 total_nvlink_bytes=655519192 weight_to_token_ratio=0.055729 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=43520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.297829368e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84848 payload_rows_remote=68464 dup_rows=13456 dup_rows_remote=7536 dedup_ratio=0.136882 router_tokens_per_expert=[0, 0, 3443, 0, 0, 0, 0, 0, 0, 190, 8192, 0, 0, 135, 4919, 0, 0, 0, 0, 398, 0, 0, 0, 0, 0, 0, 0, 0, 2116, 233, 0, 0, 0, 0, 0, 1202, 2134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 2678, 0, 8191, 0, 0, 8092, 1, 0, 0, 0, 0, 140, 7052, 0, 0, 0] payload_rows_per_dst=[6886, 10264, 16384, 4648, 16016, 72, 16384, 14194] dup_rows_per_dst=[0, 224, 796, 50, 6276, 0, 5920, 190] combine_rows_per_src=[6886, 10264, 16384, 4648, 16016, 72, 16384, 14194] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=311689216 dispatch_meta_bytes=312376 combine_payload_bytes=311689216 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=312001592 ingress_bytes=380895232 link_bytes=380895232 token_bytes=623690808 weight_movement_bytes=69206016 total_nvlink_bytes=692896824 weight_to_token_ratio=0.110962 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=33792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.433278093e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90604 payload_rows_remote=79182 dup_rows=7700 dup_rows_remote=7458 dedup_ratio=0.078328 router_tokens_per_expert=[0, 0, 55, 0, 7894, 0, 0, 0, 0, 0, 0, 0, 543, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 3057, 0, 0, 0, 1687, 0, 8192, 0, 7, 80, 0, 0, 0, 0, 2, 660, 0, 0, 152, 8192, 0, 0, 8192, 20, 0, 0, 1396, 0, 0, 0, 0, 833, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15792, 11990, 9096, 3388, 6148, 11422, 16384, 16384] dup_rows_per_dst=[106, 752, 1742, 0, 400, 242, 4458, 0] combine_rows_per_src=[15792, 11990, 9096, 3388, 6148, 11422, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74858 payload_rows_remote=58474 dup_rows=23446 dup_rows_remote=19218 dedup_ratio=0.238505 router_tokens_per_expert=[0, 0, 55, 0, 7876, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 3753, 0, 0, 0, 1697, 0, 8192, 0, 17, 80, 0, 0, 0, 0, 1, 544, 0, 0, 118, 8192, 0, 0, 8192, 16, 0, 0, 1582, 0, 0, 0, 0, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[2782, 226, 16384, 5064, 1250, 16384, 16384, 16384] dup_rows_per_dst=[18, 0, 15076, 396, 0, 3728, 4228, 0] combine_rows_per_src=[2782, 226, 16384, 5064, 1250, 16384, 16384, 16384] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=312123392 dispatch_meta_bytes=329640 combine_payload_bytes=312123392 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=312453032 ingress_bytes=415932416 link_bytes=415932416 token_bytes=624576424 weight_movement_bytes=103809024 total_nvlink_bytes=728385448 weight_to_token_ratio=0.166207 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=48640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.565120196e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91584 payload_rows_remote=85512 dup_rows=6720 dup_rows_remote=6720 dedup_ratio=0.068359 router_tokens_per_expert=[0, 0, 123, 0, 7818, 0, 0, 0, 0, 0, 0, 0, 825, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 3036, 0, 0, 0, 1633, 0, 8192, 0, 14, 94, 0, 0, 0, 0, 1, 676, 0, 0, 112, 8192, 0, 0, 8192, 52, 0, 0, 1175, 0, 0, 0, 0, 827, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15652, 16380, 6072, 16384, 16384, 332, 4002, 16378] dup_rows_per_dst=[230, 1650, 0, 3294, 1542, 2, 2, 0] combine_rows_per_src=[15652, 16380, 6072, 16384, 16384, 332, 4002, 16378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=280428544 dispatch_meta_bytes=304000 combine_payload_bytes=280428544 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=280732544 ingress_bytes=418840576 link_bytes=418840576 token_bytes=561161088 weight_movement_bytes=138412032 total_nvlink_bytes=699573120 weight_to_token_ratio=0.246653 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.576063368e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90604 payload_rows_remote=79182 dup_rows=7700 dup_rows_remote=7458 dedup_ratio=0.078328 router_tokens_per_expert=[0, 0, 55, 0, 7894, 0, 0, 0, 0, 0, 0, 0, 543, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 3057, 0, 0, 0, 1687, 0, 8192, 0, 7, 80, 0, 0, 0, 0, 2, 660, 0, 0, 152, 8192, 0, 0, 8192, 20, 0, 0, 1396, 0, 0, 0, 0, 833, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15792, 11990, 9096, 3388, 6148, 11422, 16384, 16384] dup_rows_per_dst=[106, 752, 1742, 0, 400, 242, 4458, 0] combine_rows_per_src=[15792, 11990, 9096, 3388, 6148, 11422, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70842 payload_rows_remote=54458 dup_rows=27462 dup_rows_remote=27458 dedup_ratio=0.279358 router_tokens_per_expert=[0, 0, 26, 0, 7866, 0, 0, 0, 0, 0, 0, 0, 173, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 3799, 0, 0, 0, 1593, 0, 8189, 0, 6, 96, 0, 0, 0, 0, 1, 762, 0, 0, 141, 8192, 0, 0, 8192, 7, 0, 0, 1312, 0, 0, 0, 0, 607, 0, 0, 0, 1, 0, 1, 0, 0, 0] payload_rows_per_dst=[52, 346, 16376, 3198, 1718, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 7598, 0, 0, 16028, 3832, 4] combine_rows_per_src=[52, 346, 16376, 3198, 1718, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=223059968 dispatch_meta_bytes=327664 combine_payload_bytes=223059968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=223387632 ingress_bytes=257662976 link_bytes=257662976 token_bytes=446447600 weight_movement_bytes=34603008 total_nvlink_bytes=481050608 weight_to_token_ratio=0.077507 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.695650351e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=310296576 dispatch_meta_bytes=323032 combine_payload_bytes=310296576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=310619608 ingress_bytes=344899584 link_bytes=344899584 token_bytes=620916184 weight_movement_bytes=34603008 total_nvlink_bytes=655519192 weight_to_token_ratio=0.055729 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=43520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.297829368e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87414 payload_rows_remote=71030 dup_rows=10890 dup_rows_remote=10286 dedup_ratio=0.110779 router_tokens_per_expert=[0, 0, 4396, 0, 0, 0, 0, 1, 0, 164, 8192, 0, 0, 138, 5108, 0, 0, 2, 0, 379, 0, 0, 0, 0, 0, 0, 0, 0, 2121, 104, 0, 0, 0, 0, 0, 784, 1673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 2475, 0, 8192, 0, 0, 8179, 0, 0, 0, 0, 0, 66, 7169, 0, 0, 0] payload_rows_per_dst=[14264, 16384, 760, 16370, 4790, 16384, 6948, 11514] dup_rows_per_dst=[4746, 604, 2, 4438, 124, 18, 890, 68] combine_rows_per_src=[14264, 16384, 760, 16370, 4790, 16384, 6948, 11514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91812 payload_rows_remote=75450 dup_rows=6492 dup_rows_remote=1288 dedup_ratio=0.066040 router_tokens_per_expert=[0, 0, 4130, 0, 0, 0, 0, 0, 0, 191, 8192, 0, 0, 64, 5389, 0, 0, 0, 0, 436, 0, 0, 0, 0, 0, 0, 0, 0, 2347, 173, 0, 0, 0, 0, 0, 856, 1755, 0, 0, 0, 1, 0, 0, 0, 0, 0, 40, 0, 2027, 0, 8192, 0, 0, 8172, 0, 0, 0, 0, 0, 98, 7089, 0, 0, 0] payload_rows_per_dst=[8260, 11114, 16384, 4994, 16362, 16384, 4054, 14260] dup_rows_per_dst=[0, 174, 872, 46, 5204, 82, 0, 114] combine_rows_per_src=[8260, 11114, 16384, 4994, 16362, 16384, 4054, 14260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=350257152 dispatch_meta_bytes=368928 combine_payload_bytes=350257152 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=350626080 ingress_bytes=419463168 link_bytes=419463168 token_bytes=700883232 weight_movement_bytes=69206016 total_nvlink_bytes=770089248 weight_to_token_ratio=0.098741 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=26752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.578406132e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12739182128 prefetch_bytes=726663168 reduce_bytes=449839104 weight_movement_bytes=1176502272 egress_bytes=6373326384 ingress_bytes=7542358016 link_bytes=7542358016 total_nvlink_bytes=13915684400 weight_to_token_ratio=0.092353 prefetch_share_of_ingress=0.096344 dedup_ratio=0.178620 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=13 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030978 gemm_inner_pad_upper_bound=20447 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.233968957e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=324329472 dispatch_meta_bytes=346560 combine_payload_bytes=324329472 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=324676032 ingress_bytes=428138496 link_bytes=428138496 token_bytes=649005504 weight_movement_bytes=103809024 total_nvlink_bytes=752814528 weight_to_token_ratio=0.159951 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=22144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.611050692e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=239509504 dispatch_meta_bytes=310768 combine_payload_bytes=239509504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=239820272 ingress_bytes=274112512 link_bytes=274112512 token_bytes=479329776 weight_movement_bytes=34603008 total_nvlink_bytes=513932784 weight_to_token_ratio=0.072190 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.037500 gemm_active_groups=4 gemm_peak_group_rows=31744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.031463315e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88822 payload_rows_remote=78104 dup_rows=9482 dup_rows_remote=8872 dedup_ratio=0.096456 router_tokens_per_expert=[0, 0, 3673, 0, 0, 0, 0, 1, 0, 110, 8192, 0, 0, 184, 4664, 0, 0, 0, 0, 530, 0, 0, 0, 0, 0, 0, 0, 0, 2241, 176, 0, 0, 0, 0, 0, 932, 2276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 2503, 0, 8192, 0, 0, 8039, 0, 0, 0, 0, 0, 119, 7301, 0, 0, 0] payload_rows_per_dst=[13316, 6448, 10718, 16260, 6024, 16384, 5006, 14666] dup_rows_per_dst=[3360, 256, 610, 4652, 392, 38, 0, 174] combine_rows_per_src=[13316, 6448, 10718, 16260, 6024, 16384, 5006, 14666] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=290938880 dispatch_meta_bytes=325264 combine_payload_bytes=290938880 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=291264144 ingress_bytes=394747904 link_bytes=394747904 token_bytes=582203024 weight_movement_bytes=103809024 total_nvlink_bytes=686012048 weight_to_token_ratio=0.178304 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.485404583e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89736 payload_rows_remote=76810 dup_rows=8568 dup_rows_remote=4878 dedup_ratio=0.087158 router_tokens_per_expert=[0, 0, 4139, 0, 0, 0, 0, 2, 0, 223, 8185, 0, 0, 72, 5219, 1, 0, 5, 0, 398, 1, 0, 0, 0, 1, 0, 0, 0, 2452, 168, 0, 0, 0, 2, 0, 797, 1719, 0, 0, 4, 0, 0, 0, 0, 0, 0, 11, 0, 2202, 0, 8189, 1, 0, 8125, 1, 0, 2, 0, 0, 98, 7132, 0, 3, 0] payload_rows_per_dst=[10776, 6220, 16382, 12926, 8328, 16378, 4408, 14318] dup_rows_per_dst=[2182, 134, 796, 3690, 1592, 22, 0, 152] combine_rows_per_src=[10776, 6220, 16382, 12926, 8328, 16378, 4408, 14318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=223059968 dispatch_meta_bytes=327664 combine_payload_bytes=223059968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=223387632 ingress_bytes=257662976 link_bytes=257662976 token_bytes=446447600 weight_movement_bytes=34603008 total_nvlink_bytes=481050608 weight_to_token_ratio=0.077507 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.695650351e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79378 payload_rows_remote=65838 dup_rows=18926 dup_rows_remote=14898 dedup_ratio=0.192525 router_tokens_per_expert=[0, 0, 4106, 0, 0, 0, 0, 2, 0, 139, 8192, 0, 0, 125, 4676, 0, 0, 0, 0, 322, 0, 0, 0, 0, 0, 0, 0, 0, 2009, 178, 0, 0, 0, 0, 0, 1106, 2290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 2492, 0, 8192, 0, 0, 8141, 0, 0, 0, 0, 0, 123, 7041, 0, 0, 0] payload_rows_per_dst=[13540, 16384, 644, 16330, 6460, 16384, 9390, 246] dup_rows_per_dst=[4028, 8412, 0, 4326, 332, 36, 1792, 0] combine_rows_per_src=[13540, 16384, 644, 16330, 6460, 16384, 9390, 246] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=319913984 dispatch_meta_bytes=347904 combine_payload_bytes=319913984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=320261888 ingress_bytes=354516992 link_bytes=354516992 token_bytes=640175872 weight_movement_bytes=34603008 total_nvlink_bytes=674778880 weight_to_token_ratio=0.054052 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=46208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.334018900e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91812 payload_rows_remote=75450 dup_rows=6492 dup_rows_remote=1288 dedup_ratio=0.066040 router_tokens_per_expert=[0, 0, 4130, 0, 0, 0, 0, 0, 0, 191, 8192, 0, 0, 64, 5389, 0, 0, 0, 0, 436, 0, 0, 0, 0, 0, 0, 0, 0, 2347, 173, 0, 0, 0, 0, 0, 856, 1755, 0, 0, 0, 1, 0, 0, 0, 0, 0, 40, 0, 2027, 0, 8192, 0, 0, 8172, 0, 0, 0, 0, 0, 98, 7089, 0, 0, 0] payload_rows_per_dst=[8260, 11114, 16384, 4994, 16362, 16384, 4054, 14260] dup_rows_per_dst=[0, 174, 872, 46, 5204, 82, 0, 114] combine_rows_per_src=[8260, 11114, 16384, 4994, 16362, 16384, 4054, 14260] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75888 payload_rows_remote=61936 dup_rows=22416 dup_rows_remote=22326 dedup_ratio=0.228027 router_tokens_per_expert=[0, 0, 3977, 0, 0, 0, 0, 0, 0, 124, 8192, 0, 0, 63, 5591, 0, 0, 0, 0, 341, 0, 0, 0, 0, 0, 0, 0, 0, 2140, 182, 0, 0, 0, 0, 0, 1054, 1618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 2512, 0, 8192, 0, 0, 8125, 0, 0, 0, 0, 0, 65, 6956, 0, 0, 0] payload_rows_per_dst=[7954, 11402, 16384, 4620, 5152, 40, 16384, 13952] dup_rows_per_dst=[0, 154, 682, 24, 192, 0, 21274, 90] combine_rows_per_src=[7954, 11402, 16384, 4620, 5152, 40, 16384, 13952] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=290938880 dispatch_meta_bytes=325264 combine_payload_bytes=290938880 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=291264144 ingress_bytes=394747904 link_bytes=394747904 token_bytes=582203024 weight_movement_bytes=103809024 total_nvlink_bytes=686012048 weight_to_token_ratio=0.178304 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.485404583e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=309043200 dispatch_meta_bytes=306952 combine_payload_bytes=309043200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=309350152 ingress_bytes=343646208 link_bytes=343646208 token_bytes=618393352 weight_movement_bytes=34603008 total_nvlink_bytes=652996360 weight_to_token_ratio=0.055956 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=26624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.293113015e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13302686048 prefetch_bytes=657457152 reduce_bytes=795869184 weight_movement_bytes=1453326336 egress_bytes=6655107424 ingress_bytes=8100904960 link_bytes=8100904960 total_nvlink_bytes=14756012384 weight_to_token_ratio=0.109251 prefetch_share_of_ingress=0.081158 dedup_ratio=0.142413 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=23 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029457 gemm_inner_pad_upper_bound=21844 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.325350139e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=314613760 dispatch_meta_bytes=326752 combine_payload_bytes=314613760 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=314940512 ingress_bytes=349216768 link_bytes=349216768 token_bytes=629554272 weight_movement_bytes=34603008 total_nvlink_bytes=664157280 weight_to_token_ratio=0.054964 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=30080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.314074584e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92582 payload_rows_remote=76568 dup_rows=5722 dup_rows_remote=5654 dedup_ratio=0.058207 router_tokens_per_expert=[0, 0, 4495, 0, 0, 0, 0, 1, 0, 134, 8192, 0, 0, 146, 4785, 0, 0, 0, 0, 253, 0, 0, 0, 0, 0, 0, 0, 0, 2242, 223, 0, 0, 0, 0, 0, 765, 1624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 2742, 0, 8192, 0, 0, 8141, 0, 0, 0, 0, 0, 71, 7112, 0, 0, 0] payload_rows_per_dst=[8990, 9980, 16384, 4882, 16346, 16014, 5722, 14264] dup_rows_per_dst=[2, 150, 506, 48, 4714, 68, 132, 102] combine_rows_per_src=[8990, 9980, 16384, 4882, 16346, 16014, 5722, 14264] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=269672448 dispatch_meta_bytes=322944 combine_payload_bytes=269672448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269995392 ingress_bytes=304275456 link_bytes=304275456 token_bytes=539667840 weight_movement_bytes=34603008 total_nvlink_bytes=574270848 weight_to_token_ratio=0.064119 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=32384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.144964045e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84848 payload_rows_remote=68464 dup_rows=13456 dup_rows_remote=7536 dedup_ratio=0.136882 router_tokens_per_expert=[0, 0, 3443, 0, 0, 0, 0, 0, 0, 190, 8192, 0, 0, 135, 4919, 0, 0, 0, 0, 398, 0, 0, 0, 0, 0, 0, 0, 0, 2116, 233, 0, 0, 0, 0, 0, 1202, 2134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 2678, 0, 8191, 0, 0, 8092, 1, 0, 0, 0, 0, 140, 7052, 0, 0, 0] payload_rows_per_dst=[6886, 10264, 16384, 4648, 16016, 72, 16384, 14194] dup_rows_per_dst=[0, 224, 796, 50, 6276, 0, 5920, 190] combine_rows_per_src=[6886, 10264, 16384, 4648, 16016, 72, 16384, 14194] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14539119344 prefetch_bytes=795869184 reduce_bytes=830472192 weight_movement_bytes=1626341376 egress_bytes=7273536240 ingress_bytes=8891924480 link_bytes=8891924480 total_nvlink_bytes=16165460720 weight_to_token_ratio=0.111860 prefetch_share_of_ingress=0.089505 dedup_ratio=0.109488 prefetch_slots=46 unused_prefetch_slots=322 slot_utilization=0.125000 reduce_slots=24 remote_demand=23 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029891 gemm_inner_pad_upper_bound=21844 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.454765042e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=309043200 dispatch_meta_bytes=306952 combine_payload_bytes=309043200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=309350152 ingress_bytes=343646208 link_bytes=343646208 token_bytes=618393352 weight_movement_bytes=34603008 total_nvlink_bytes=652996360 weight_to_token_ratio=0.055956 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=26624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.293113015e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=253689856 dispatch_meta_bytes=337048 combine_payload_bytes=253689856 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=254026904 ingress_bytes=322895872 link_bytes=322895872 token_bytes=507716760 weight_movement_bytes=69206016 total_nvlink_bytes=576922776 weight_to_token_ratio=0.136308 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=48384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.215031171e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13302686048 prefetch_bytes=657457152 reduce_bytes=795869184 weight_movement_bytes=1453326336 egress_bytes=6655107424 ingress_bytes=8100904960 link_bytes=8100904960 total_nvlink_bytes=14756012384 weight_to_token_ratio=0.109251 prefetch_share_of_ingress=0.081158 dedup_ratio=0.142413 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=23 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029457 gemm_inner_pad_upper_bound=21844 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.325350139e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14535198032 prefetch_bytes=761266176 reduce_bytes=588251136 weight_movement_bytes=1349517312 egress_bytes=7271540048 ingress_bytes=8613175296 link_bytes=8613175296 total_nvlink_bytes=15884715344 weight_to_token_ratio=0.092845 prefetch_share_of_ingress=0.088384 dedup_ratio=0.099342 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=17 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030217 gemm_inner_pad_upper_bound=20574 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.409160227e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88822 payload_rows_remote=78104 dup_rows=9482 dup_rows_remote=8872 dedup_ratio=0.096456 router_tokens_per_expert=[0, 0, 3673, 0, 0, 0, 0, 1, 0, 110, 8192, 0, 0, 184, 4664, 0, 0, 0, 0, 530, 0, 0, 0, 0, 0, 0, 0, 0, 2241, 176, 0, 0, 0, 0, 0, 932, 2276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 2503, 0, 8192, 0, 0, 8039, 0, 0, 0, 0, 0, 119, 7301, 0, 0, 0] payload_rows_per_dst=[13316, 6448, 10718, 16260, 6024, 16384, 5006, 14666] dup_rows_per_dst=[3360, 256, 610, 4652, 392, 38, 0, 174] combine_rows_per_src=[13316, 6448, 10718, 16260, 6024, 16384, 5006, 14666] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14529617312 prefetch_bytes=692060160 reduce_bytes=588251136 weight_movement_bytes=1280311296 egress_bytes=7268728224 ingress_bytes=8541200384 link_bytes=8541200384 total_nvlink_bytes=15809928608 weight_to_token_ratio=0.088117 prefetch_share_of_ingress=0.081026 dedup_ratio=0.097711 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=17 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030326 gemm_inner_pad_upper_bound=21463 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.397384757e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=313622528 dispatch_meta_bytes=328888 combine_payload_bytes=313622528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=313951416 ingress_bytes=348225536 link_bytes=348225536 token_bytes=627573944 weight_movement_bytes=34603008 total_nvlink_bytes=662176952 weight_to_token_ratio=0.055138 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.310344658e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92582 payload_rows_remote=76568 dup_rows=5722 dup_rows_remote=5654 dedup_ratio=0.058207 router_tokens_per_expert=[0, 0, 4495, 0, 0, 0, 0, 1, 0, 134, 8192, 0, 0, 146, 4785, 0, 0, 0, 0, 253, 0, 0, 0, 0, 0, 0, 0, 0, 2242, 223, 0, 0, 0, 0, 0, 765, 1624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 2742, 0, 8192, 0, 0, 8141, 0, 0, 0, 0, 0, 71, 7112, 0, 0, 0] payload_rows_per_dst=[8990, 9980, 16384, 4882, 16346, 16014, 5722, 14264] dup_rows_per_dst=[2, 150, 506, 48, 4714, 68, 132, 102] combine_rows_per_src=[8990, 9980, 16384, 4882, 16346, 16014, 5722, 14264] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12119609016 prefetch_bytes=692060160 reduce_bytes=657457152 weight_movement_bytes=1349517312 egress_bytes=6063435448 ingress_bytes=7405690880 link_bytes=7405690880 total_nvlink_bytes=13469126328 weight_to_token_ratio=0.111350 prefetch_share_of_ingress=0.093450 dedup_ratio=0.194876 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=19 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030761 gemm_inner_pad_upper_bound=20193 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.211609503e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=280428544 dispatch_meta_bytes=304000 combine_payload_bytes=280428544 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=280732544 ingress_bytes=418840576 link_bytes=418840576 token_bytes=561161088 weight_movement_bytes=138412032 total_nvlink_bytes=699573120 weight_to_token_ratio=0.246653 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.576063368e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14535198032 prefetch_bytes=761266176 reduce_bytes=588251136 weight_movement_bytes=1349517312 egress_bytes=7271540048 ingress_bytes=8613175296 link_bytes=8613175296 total_nvlink_bytes=15884715344 weight_to_token_ratio=0.092845 prefetch_share_of_ingress=0.088384 dedup_ratio=0.099342 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=17 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030217 gemm_inner_pad_upper_bound=20574 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.409160227e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=11469591808 prefetch_bytes=692060160 reduce_bytes=657457152 weight_movement_bytes=1349517312 egress_bytes=5738353920 ingress_bytes=7080755200 link_bytes=7080755200 total_nvlink_bytes=12819109120 weight_to_token_ratio=0.117660 prefetch_share_of_ingress=0.097738 dedup_ratio=0.235392 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=19 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029891 gemm_inner_pad_upper_bound=21844 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.158448338e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75888 payload_rows_remote=61936 dup_rows=22416 dup_rows_remote=22326 dedup_ratio=0.228027 router_tokens_per_expert=[0, 0, 3977, 0, 0, 0, 0, 0, 0, 124, 8192, 0, 0, 63, 5591, 0, 0, 0, 0, 341, 0, 0, 0, 0, 0, 0, 0, 0, 2140, 182, 0, 0, 0, 0, 0, 1054, 1618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 2512, 0, 8192, 0, 0, 8125, 0, 0, 0, 0, 0, 65, 6956, 0, 0, 0] payload_rows_per_dst=[7954, 11402, 16384, 4620, 5152, 40, 16384, 13952] dup_rows_per_dst=[0, 154, 682, 24, 192, 0, 21274, 90] combine_rows_per_src=[7954, 11402, 16384, 4620, 5152, 40, 16384, 13952] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=319913984 dispatch_meta_bytes=347904 combine_payload_bytes=319913984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=320261888 ingress_bytes=354516992 link_bytes=354516992 token_bytes=640175872 weight_movement_bytes=34603008 total_nvlink_bytes=674778880 weight_to_token_ratio=0.054052 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=46208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.334018900e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=313622528 dispatch_meta_bytes=328888 combine_payload_bytes=313622528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=313951416 ingress_bytes=348225536 link_bytes=348225536 token_bytes=627573944 weight_movement_bytes=34603008 total_nvlink_bytes=662176952 weight_to_token_ratio=0.055138 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.310344658e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=253689856 dispatch_meta_bytes=337048 combine_payload_bytes=253689856 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=254026904 ingress_bytes=322895872 link_bytes=322895872 token_bytes=507716760 weight_movement_bytes=69206016 total_nvlink_bytes=576922776 weight_to_token_ratio=0.136308 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=48384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.215031171e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14539119344 prefetch_bytes=795869184 reduce_bytes=830472192 weight_movement_bytes=1626341376 egress_bytes=7273536240 ingress_bytes=8891924480 link_bytes=8891924480 total_nvlink_bytes=16165460720 weight_to_token_ratio=0.111860 prefetch_share_of_ingress=0.089505 dedup_ratio=0.109488 prefetch_slots=46 unused_prefetch_slots=322 slot_utilization=0.125000 reduce_slots=24 remote_demand=23 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029891 gemm_inner_pad_upper_bound=21844 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.454765042e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14140497400 prefetch_bytes=588251136 reduce_bytes=1038090240 weight_movement_bytes=1626341376 egress_bytes=7074094584 ingress_bytes=8692744192 link_bytes=8692744192 total_nvlink_bytes=15766838776 weight_to_token_ratio=0.115013 prefetch_share_of_ingress=0.067672 dedup_ratio=0.107864 prefetch_slots=34 unused_prefetch_slots=334 slot_utilization=0.092391 reduce_slots=30 remote_demand=17 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030326 gemm_inner_pad_upper_bound=21209 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.422178112e-04 prefetch_bytes_per_gemm_flop=9.624094203e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12739182128 prefetch_bytes=726663168 reduce_bytes=449839104 weight_movement_bytes=1176502272 egress_bytes=6373326384 ingress_bytes=7542358016 link_bytes=7542358016 total_nvlink_bytes=13915684400 weight_to_token_ratio=0.092353 prefetch_share_of_ingress=0.096344 dedup_ratio=0.178620 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=13 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030978 gemm_inner_pad_upper_bound=20447 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.233968957e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none - [2026-08-26 15:09:09] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 791.2 | throughput per GPU (TFLOP/s/GPU): 161.3 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.478545E+00 | loss scale: 1.0 | grad norm: 7.152 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=11469591808 prefetch_bytes=692060160 reduce_bytes=657457152 weight_movement_bytes=1349517312 egress_bytes=5738353920 ingress_bytes=7080755200 link_bytes=7080755200 total_nvlink_bytes=12819109120 weight_to_token_ratio=0.117660 prefetch_share_of_ingress=0.097738 dedup_ratio=0.235392 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=19 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029891 gemm_inner_pad_upper_bound=21844 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.158448338e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none + [2026-08-26 15:56:57] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 799.7 | throughput per GPU (TFLOP/s/GPU): 159.5 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.478545E+00 | loss scale: 1.0 | grad norm: 7.152 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76012 payload_rows_remote=65022 dup_rows=22292 dup_rows_remote=18482 dedup_ratio=0.226766 router_tokens_per_expert=[2892, 585, 178, 1462, 395, 670, 630, 138, 2993, 284, 500, 904, 30, 1025, 496, 374, 726, 1170, 347, 652, 355, 280, 554, 1050, 1149, 809, 297, 478, 983, 376, 1153, 955, 1292, 578, 857, 982, 1456, 808, 784, 607, 253, 627, 424, 311, 740, 1436, 834, 177, 149, 482, 558, 1149, 763, 1930, 512, 215, 420, 661, 1346, 447, 209, 669, 1583, 1003] payload_rows_per_dst=[10990, 11098, 8094, 8466, 11558, 8242, 9510, 8054] dup_rows_per_dst=[3810, 4412, 2174, 1636, 4474, 2324, 2006, 1456] combine_rows_per_src=[10990, 11098, 8094, 8466, 11558, 8242, 9510, 8054] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76226 payload_rows_remote=65184 dup_rows=22078 dup_rows_remote=18182 dedup_ratio=0.224589 router_tokens_per_expert=[2600, 558, 281, 1467, 477, 800, 713, 153, 2905, 322, 446, 971, 55, 769, 548, 502, 964, 1189, 399, 694, 385, 276, 509, 1016, 1354, 920, 331, 415, 1034, 385, 1161, 987, 1092, 649, 916, 984, 1229, 823, 826, 663, 266, 567, 433, 302, 847, 1494, 806, 232, 161, 418, 548, 1195, 667, 1434, 502, 259, 371, 717, 1246, 488, 255, 598, 1545, 1033] payload_rows_per_dst=[10566, 11042, 8424, 9128, 9602, 9090, 8558, 9816] dup_rows_per_dst=[3532, 3896, 2440, 2144, 2674, 3262, 1810, 2320] combine_rows_per_src=[10566, 11042, 8424, 9128, 9602, 9090, 8558, 9816] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76306 payload_rows_remote=67936 dup_rows=21998 dup_rows_remote=19754 dedup_ratio=0.223775 router_tokens_per_expert=[2699, 559, 283, 1664, 456, 863, 543, 103, 2884, 386, 503, 914, 33, 931, 539, 378, 845, 1279, 400, 650, 375, 282, 475, 1001, 1236, 936, 279, 490, 941, 422, 1250, 1014, 1185, 578, 810, 1089, 1384, 716, 734, 624, 244, 538, 414, 272, 798, 1566, 789, 178, 166, 488, 510, 1250, 730, 1575, 501, 245, 407, 598, 1268, 465, 207, 658, 1547, 1005] payload_rows_per_dst=[10676, 10064, 8370, 9968, 9234, 9090, 9010, 9894] dup_rows_per_dst=[3664, 3072, 2244, 3168, 2238, 3276, 1920, 2416] combine_rows_per_src=[10676, 10064, 8370, 9968, 9234, 9090, 9010, 9894] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76408 payload_rows_remote=67186 dup_rows=21896 dup_rows_remote=18430 dedup_ratio=0.222738 router_tokens_per_expert=[2709, 582, 232, 1598, 448, 750, 686, 115, 3033, 308, 481, 860, 32, 971, 504, 400, 795, 1202, 388, 748, 317, 263, 458, 1049, 1254, 907, 301, 470, 870, 359, 1157, 788, 1266, 562, 849, 1066, 1379, 777, 714, 599, 267, 520, 396, 312, 822, 1578, 851, 219, 143, 431, 552, 1021, 726, 1706, 547, 304, 416, 654, 1325, 533, 218, 639, 1686, 1039] payload_rows_per_dst=[7954, 9982, 11176, 9596, 9308, 9222, 8846, 10324] dup_rows_per_dst=[1904, 3196, 3646, 2616, 2358, 3466, 2014, 2696] combine_rows_per_src=[7954, 9982, 11176, 9596, 9308, 9222, 8846, 10324] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75230 payload_rows_remote=65002 dup_rows=23074 dup_rows_remote=20138 dedup_ratio=0.234721 router_tokens_per_expert=[2835, 535, 241, 1562, 474, 858, 600, 87, 2851, 285, 448, 952, 36, 888, 556, 489, 657, 1186, 369, 658, 360, 237, 493, 915, 1321, 831, 286, 426, 993, 350, 1151, 988, 1146, 576, 795, 1107, 1378, 877, 758, 681, 213, 535, 425, 364, 806, 1505, 827, 185, 166, 480, 500, 1249, 812, 1497, 584, 248, 448, 631, 1317, 584, 224, 654, 1587, 1075] payload_rows_per_dst=[7310, 6178, 11326, 9748, 9428, 9134, 11878, 10228] dup_rows_per_dst=[1404, 1130, 4094, 2820, 2452, 3342, 4896, 2936] combine_rows_per_src=[7310, 6178, 11326, 9748, 9428, 9134, 11878, 10228] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76654 payload_rows_remote=67172 dup_rows=21650 dup_rows_remote=19282 dedup_ratio=0.220235 router_tokens_per_expert=[2649, 571, 261, 1325, 403, 769, 728, 134, 2807, 327, 428, 842, 41, 967, 575, 435, 787, 1148, 396, 680, 368, 239, 479, 1017, 1243, 974, 318, 434, 1090, 390, 1176, 1030, 1111, 599, 821, 1063, 1336, 798, 786, 747, 312, 574, 437, 320, 868, 1553, 739, 185, 150, 453, 522, 1082, 779, 1649, 552, 292, 379, 618, 1329, 495, 243, 677, 1579, 1073] payload_rows_per_dst=[10478, 9840, 8134, 10234, 9482, 9284, 8976, 10226] dup_rows_per_dst=[3202, 3004, 2094, 3076, 2368, 3364, 1982, 2560] combine_rows_per_src=[10478, 9840, 8134, 10234, 9482, 9284, 8976, 10226] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76444 payload_rows_remote=66788 dup_rows=21860 dup_rows_remote=19050 dedup_ratio=0.222371 router_tokens_per_expert=[2859, 602, 267, 1604, 456, 707, 569, 113, 3174, 258, 416, 993, 43, 1073, 499, 477, 753, 1167, 320, 690, 310, 260, 516, 1095, 1272, 918, 287, 434, 934, 329, 1154, 905, 1137, 578, 752, 1109, 1292, 774, 682, 681, 263, 602, 428, 292, 722, 1485, 835, 198, 154, 440, 505, 1177, 730, 1825, 520, 231, 380, 684, 1324, 569, 191, 639, 1545, 954] payload_rows_per_dst=[10950, 10346, 8048, 9656, 9266, 8968, 9194, 10016] dup_rows_per_dst=[3404, 3520, 2174, 2810, 2160, 3266, 1970, 2556] combine_rows_per_src=[10950, 10346, 8048, 9656, 9266, 8968, 9194, 10016] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75804 payload_rows_remote=64452 dup_rows=22500 dup_rows_remote=18118 dedup_ratio=0.228882 router_tokens_per_expert=[2814, 534, 216, 1717, 337, 805, 502, 119, 2886, 294, 400, 939, 26, 808, 497, 454, 836, 1279, 279, 713, 367, 205, 485, 955, 1374, 939, 292, 453, 905, 326, 1257, 1073, 1161, 643, 854, 1024, 1331, 734, 745, 690, 311, 601, 437, 367, 787, 1542, 885, 226, 141, 449, 554, 1183, 731, 1513, 517, 237, 413, 643, 1275, 585, 187, 601, 1580, 1119] payload_rows_per_dst=[7214, 6390, 11840, 9982, 9326, 9484, 11352, 10216] dup_rows_per_dst=[1246, 1134, 4026, 3256, 2376, 3490, 4382, 2590] combine_rows_per_src=[7214, 6390, 11840, 9982, 9326, 9484, 11352, 10216] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75230 payload_rows_remote=65002 dup_rows=23074 dup_rows_remote=20138 dedup_ratio=0.234721 router_tokens_per_expert=[2835, 535, 241, 1562, 474, 858, 600, 87, 2851, 285, 448, 952, 36, 888, 556, 489, 657, 1186, 369, 658, 360, 237, 493, 915, 1321, 831, 286, 426, 993, 350, 1151, 988, 1146, 576, 795, 1107, 1378, 877, 758, 681, 213, 535, 425, 364, 806, 1505, 827, 185, 166, 480, 500, 1249, 812, 1497, 584, 248, 448, 631, 1317, 584, 224, 654, 1587, 1075] payload_rows_per_dst=[7310, 6178, 11326, 9748, 9428, 9134, 11878, 10228] dup_rows_per_dst=[1404, 1130, 4094, 2820, 2452, 3342, 4896, 2936] combine_rows_per_src=[7310, 6178, 11326, 9748, 9428, 9134, 11878, 10228] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266330112 dispatch_meta_bytes=334016 combine_payload_bytes=266330112 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266664128 ingress_bytes=335536128 link_bytes=335536128 token_bytes=532994240 weight_movement_bytes=69206016 total_nvlink_bytes=602200256 weight_to_token_ratio=0.129844 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262595437e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266993664 dispatch_meta_bytes=333464 combine_payload_bytes=266993664 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267327128 ingress_bytes=336199680 link_bytes=336199680 token_bytes=534320792 weight_movement_bytes=69206016 total_nvlink_bytes=603526808 weight_to_token_ratio=0.129521 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265092330e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=275136512 dispatch_meta_bytes=345816 combine_payload_bytes=275136512 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=275482328 ingress_bytes=344342528 link_bytes=344342528 token_bytes=550618840 weight_movement_bytes=69206016 total_nvlink_bytes=619824856 weight_to_token_ratio=0.125688 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.295733211e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278265856 dispatch_meta_bytes=350760 combine_payload_bytes=278265856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278616616 ingress_bytes=312868864 link_bytes=312868864 token_bytes=556882472 weight_movement_bytes=34603008 total_nvlink_bytes=591485480 weight_to_token_ratio=0.062137 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.177300347e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76654 payload_rows_remote=67172 dup_rows=21650 dup_rows_remote=19282 dedup_ratio=0.220235 router_tokens_per_expert=[2649, 571, 261, 1325, 403, 769, 728, 134, 2807, 327, 428, 842, 41, 967, 575, 435, 787, 1148, 396, 680, 368, 239, 479, 1017, 1243, 974, 318, 434, 1090, 390, 1176, 1030, 1111, 599, 821, 1063, 1336, 798, 786, 747, 312, 574, 437, 320, 868, 1553, 739, 185, 150, 453, 522, 1082, 779, 1649, 552, 292, 379, 618, 1329, 495, 243, 677, 1579, 1073] payload_rows_per_dst=[10478, 9840, 8134, 10234, 9482, 9284, 8976, 10226] dup_rows_per_dst=[3202, 3004, 2094, 3076, 2368, 3364, 1982, 2560] combine_rows_per_src=[10478, 9840, 8134, 10234, 9482, 9284, 8976, 10226] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=275193856 dispatch_meta_bytes=342464 combine_payload_bytes=275193856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275536320 ingress_bytes=309796864 link_bytes=309796864 token_bytes=550730176 weight_movement_bytes=34603008 total_nvlink_bytes=585333184 weight_to_token_ratio=0.062831 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=12160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.165740659e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76012 payload_rows_remote=65022 dup_rows=22292 dup_rows_remote=18482 dedup_ratio=0.226766 router_tokens_per_expert=[2892, 585, 178, 1462, 395, 670, 630, 138, 2993, 284, 500, 904, 30, 1025, 496, 374, 726, 1170, 347, 652, 355, 280, 554, 1050, 1149, 809, 297, 478, 983, 376, 1153, 955, 1292, 578, 857, 982, 1456, 808, 784, 607, 253, 627, 424, 311, 740, 1436, 834, 177, 149, 482, 558, 1149, 763, 1930, 512, 215, 420, 661, 1346, 447, 209, 669, 1583, 1003] payload_rows_per_dst=[10990, 11098, 8094, 8466, 11558, 8242, 9510, 8054] dup_rows_per_dst=[3810, 4412, 2174, 1636, 4474, 2324, 2006, 1456] combine_rows_per_src=[10990, 11098, 8094, 8466, 11558, 8242, 9510, 8054] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=340560 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266588752 ingress_bytes=370057216 link_bytes=370057216 token_bytes=532836944 weight_movement_bytes=103809024 total_nvlink_bytes=636645968 weight_to_token_ratio=0.194823 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392495512e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=273563648 dispatch_meta_bytes=343352 combine_payload_bytes=273563648 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=273907000 ingress_bytes=342769664 link_bytes=342769664 token_bytes=547470648 weight_movement_bytes=69206016 total_nvlink_bytes=616676664 weight_to_token_ratio=0.126410 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=9472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.289814650e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75804 payload_rows_remote=64452 dup_rows=22500 dup_rows_remote=18118 dedup_ratio=0.228882 router_tokens_per_expert=[2814, 534, 216, 1717, 337, 805, 502, 119, 2886, 294, 400, 939, 26, 808, 497, 454, 836, 1279, 279, 713, 367, 205, 485, 955, 1374, 939, 292, 453, 905, 326, 1257, 1073, 1161, 643, 854, 1024, 1331, 734, 745, 690, 311, 601, 437, 367, 787, 1542, 885, 226, 141, 449, 554, 1183, 731, 1513, 517, 237, 413, 643, 1275, 585, 187, 601, 1580, 1119] payload_rows_per_dst=[7214, 6390, 11840, 9982, 9326, 9484, 11352, 10216] dup_rows_per_dst=[1246, 1134, 4026, 3256, 2376, 3490, 4382, 2590] combine_rows_per_src=[7214, 6390, 11840, 9982, 9326, 9484, 11352, 10216] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=275136512 dispatch_meta_bytes=345816 combine_payload_bytes=275136512 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=275482328 ingress_bytes=344342528 link_bytes=344342528 token_bytes=550618840 weight_movement_bytes=69206016 total_nvlink_bytes=619824856 weight_to_token_ratio=0.125688 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.295733211e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266330112 dispatch_meta_bytes=334016 combine_payload_bytes=266330112 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266664128 ingress_bytes=335536128 link_bytes=335536128 token_bytes=532994240 weight_movement_bytes=69206016 total_nvlink_bytes=602200256 weight_to_token_ratio=0.129844 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262595437e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=263995392 dispatch_meta_bytes=330280 combine_payload_bytes=263995392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264325672 ingress_bytes=298598400 link_bytes=298598400 token_bytes=528321064 weight_movement_bytes=34603008 total_nvlink_bytes=562924072 weight_to_token_ratio=0.065496 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123601740e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76408 payload_rows_remote=67186 dup_rows=21896 dup_rows_remote=18430 dedup_ratio=0.222738 router_tokens_per_expert=[2709, 582, 232, 1598, 448, 750, 686, 115, 3033, 308, 481, 860, 32, 971, 504, 400, 795, 1202, 388, 748, 317, 263, 458, 1049, 1254, 907, 301, 470, 870, 359, 1157, 788, 1266, 562, 849, 1066, 1379, 777, 714, 599, 267, 520, 396, 312, 822, 1578, 851, 219, 143, 431, 552, 1021, 726, 1706, 547, 304, 416, 654, 1325, 533, 218, 639, 1686, 1039] payload_rows_per_dst=[7954, 9982, 11176, 9596, 9308, 9222, 8846, 10324] dup_rows_per_dst=[1904, 3196, 3646, 2616, 2358, 3466, 2014, 2696] combine_rows_per_src=[7954, 9982, 11176, 9596, 9308, 9222, 8846, 10324] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=340560 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266588752 ingress_bytes=370057216 link_bytes=370057216 token_bytes=532836944 weight_movement_bytes=103809024 total_nvlink_bytes=636645968 weight_to_token_ratio=0.194823 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392495512e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76306 payload_rows_remote=67936 dup_rows=21998 dup_rows_remote=19754 dedup_ratio=0.223775 router_tokens_per_expert=[2699, 559, 283, 1664, 456, 863, 543, 103, 2884, 386, 503, 914, 33, 931, 539, 378, 845, 1279, 400, 650, 375, 282, 475, 1001, 1236, 936, 279, 490, 941, 422, 1250, 1014, 1185, 578, 810, 1089, 1384, 716, 734, 624, 244, 538, 414, 272, 798, 1566, 789, 178, 166, 488, 510, 1250, 730, 1575, 501, 245, 407, 598, 1268, 465, 207, 658, 1547, 1005] payload_rows_per_dst=[10676, 10064, 8370, 9968, 9234, 9090, 9010, 9894] dup_rows_per_dst=[3664, 3072, 2244, 3168, 2238, 3276, 1920, 2416] combine_rows_per_src=[10676, 10064, 8370, 9968, 9234, 9090, 9010, 9894] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71460 payload_rows_remote=57728 dup_rows=26844 dup_rows_remote=16496 dedup_ratio=0.273071 router_tokens_per_expert=[315, 935, 370, 632, 2325, 95, 4544, 297, 602, 704, 429, 979, 451, 1080, 45, 1479, 571, 974, 1061, 2036, 31, 643, 11, 374, 134, 845, 927, 2527, 50, 396, 253, 2226, 23, 33, 520, 129, 534, 12, 276, 531, 783, 579, 287, 791, 217, 182, 270, 1265, 2718, 156, 1670, 397, 451, 172, 862, 194, 1052, 1125, 1229, 989, 591, 602, 1337, 834] payload_rows_per_dst=[13732, 11750, 11134, 5616, 3992, 10454, 7156, 7626] dup_rows_per_dst=[10348, 5186, 4720, 920, 124, 3426, 1290, 830] combine_rows_per_src=[13732, 11750, 11134, 5616, 3992, 10454, 7156, 7626] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77028 payload_rows_remote=67660 dup_rows=21276 dup_rows_remote=18856 dedup_ratio=0.216431 router_tokens_per_expert=[273, 951, 447, 586, 2195, 88, 4481, 317, 649, 638, 434, 1025, 552, 1162, 49, 1385, 553, 936, 730, 1926, 38, 604, 20, 429, 132, 861, 1021, 2323, 51, 625, 253, 2164, 26, 28, 604, 231, 537, 23, 405, 549, 729, 554, 307, 884, 271, 163, 290, 1247, 2327, 187, 1754, 391, 378, 183, 968, 173, 876, 1092, 1214, 1320, 723, 614, 1290, 916] payload_rows_per_dst=[11170, 9368, 10854, 5190, 11000, 10470, 10136, 8840] dup_rows_per_dst=[3984, 2420, 3946, 696, 1974, 3428, 2586, 2242] combine_rows_per_src=[11170, 9368, 10854, 5190, 11000, 10470, 10136, 8840] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78060 payload_rows_remote=68362 dup_rows=20244 dup_rows_remote=17060 dedup_ratio=0.205933 router_tokens_per_expert=[236, 867, 339, 597, 2282, 100, 4510, 320, 539, 562, 453, 1173, 487, 1011, 60, 1488, 506, 1001, 1097, 2169, 21, 708, 27, 428, 144, 759, 950, 2402, 43, 467, 251, 2257, 19, 19, 512, 168, 535, 16, 382, 501, 834, 518, 282, 792, 226, 166, 244, 1080, 2719, 155, 1833, 406, 424, 155, 856, 125, 1186, 1059, 1172, 1062, 602, 533, 1383, 934] payload_rows_per_dst=[8994, 9130, 9698, 9418, 11238, 10066, 10426, 9090] dup_rows_per_dst=[2420, 2416, 3184, 2228, 2086, 3248, 2920, 1742] combine_rows_per_src=[8994, 9130, 9698, 9418, 11238, 10066, 10426, 9090] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=275193856 dispatch_meta_bytes=342464 combine_payload_bytes=275193856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275536320 ingress_bytes=309796864 link_bytes=309796864 token_bytes=550730176 weight_movement_bytes=34603008 total_nvlink_bytes=585333184 weight_to_token_ratio=0.062831 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=12160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.165740659e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77340 payload_rows_remote=67264 dup_rows=20964 dup_rows_remote=18202 dedup_ratio=0.213257 router_tokens_per_expert=[217, 855, 488, 630, 2102, 104, 4708, 292, 625, 713, 364, 1015, 502, 1240, 82, 1379, 497, 952, 690, 1905, 22, 611, 21, 388, 161, 692, 839, 2243, 62, 608, 190, 2484, 14, 20, 589, 295, 446, 27, 392, 546, 785, 575, 280, 909, 242, 156, 231, 1204, 2430, 156, 1771, 467, 436, 154, 885, 120, 888, 1018, 1369, 1511, 591, 606, 1433, 925] payload_rows_per_dst=[7496, 9520, 8398, 11234, 11708, 9074, 10076, 9834] dup_rows_per_dst=[1880, 2320, 1774, 3324, 2366, 2556, 2762, 3982] combine_rows_per_src=[7496, 9520, 8398, 11234, 11708, 9074, 10076, 9834] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77640 payload_rows_remote=68524 dup_rows=20664 dup_rows_remote=17992 dedup_ratio=0.210205 router_tokens_per_expert=[223, 847, 376, 510, 2182, 126, 4607, 270, 516, 678, 414, 1239, 527, 1033, 37, 1343, 518, 976, 1020, 2052, 11, 657, 16, 343, 123, 721, 904, 2706, 38, 569, 200, 2283, 21, 37, 522, 148, 485, 10, 380, 524, 799, 591, 291, 818, 218, 241, 253, 1269, 2729, 144, 1545, 409, 493, 151, 883, 139, 954, 1072, 1436, 1085, 593, 589, 1414, 844] payload_rows_per_dst=[7420, 9222, 8830, 11690, 11472, 9116, 10086, 9804] dup_rows_per_dst=[1648, 2352, 2356, 3398, 1996, 2672, 2900, 3342] combine_rows_per_src=[7420, 9222, 8830, 11690, 11472, 9116, 10086, 9804] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77868 payload_rows_remote=67378 dup_rows=20436 dup_rows_remote=16424 dedup_ratio=0.207886 router_tokens_per_expert=[311, 902, 416, 654, 2331, 110, 4806, 282, 730, 654, 369, 913, 527, 1067, 49, 1304, 566, 1069, 958, 1990, 17, 569, 10, 422, 121, 794, 908, 2037, 45, 385, 245, 2526, 23, 15, 686, 194, 515, 25, 252, 501, 765, 511, 313, 818, 232, 157, 228, 1279, 2274, 148, 1461, 494, 430, 161, 841, 115, 1245, 1121, 1260, 1385, 745, 586, 1376, 909] payload_rows_per_dst=[7834, 9002, 8878, 11274, 11722, 8988, 9680, 10490] dup_rows_per_dst=[2178, 2224, 2324, 2848, 2312, 2370, 2168, 4012] combine_rows_per_src=[7834, 9002, 8878, 11274, 11722, 8988, 9680, 10490] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277135360 dispatch_meta_bytes=346064 combine_payload_bytes=277135360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277481424 ingress_bytes=311738368 link_bytes=311738368 token_bytes=554616784 weight_movement_bytes=34603008 total_nvlink_bytes=589219792 weight_to_token_ratio=0.062391 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.173046382e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77276 payload_rows_remote=66344 dup_rows=21028 dup_rows_remote=18996 dedup_ratio=0.213908 router_tokens_per_expert=[294, 885, 540, 553, 2273, 111, 4210, 309, 555, 691, 409, 982, 496, 1199, 69, 1391, 585, 998, 847, 2034, 31, 660, 33, 408, 120, 802, 987, 2475, 74, 472, 288, 2437, 36, 42, 560, 186, 605, 23, 310, 510, 841, 569, 310, 818, 202, 170, 259, 1203, 2565, 170, 1685, 430, 328, 166, 920, 228, 1009, 1081, 1184, 1199, 654, 598, 1239, 834] payload_rows_per_dst=[7734, 9114, 8880, 11724, 10932, 8720, 10360, 9812] dup_rows_per_dst=[2196, 2470, 2312, 3586, 2032, 2502, 2624, 3306] combine_rows_per_src=[7734, 9114, 8880, 11724, 10932, 8720, 10360, 9812] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77484 payload_rows_remote=65840 dup_rows=20820 dup_rows_remote=17306 dedup_ratio=0.211792 router_tokens_per_expert=[252, 821, 373, 626, 2189, 121, 4379, 253, 686, 721, 351, 1056, 547, 1103, 50, 1391, 532, 931, 797, 2123, 31, 604, 25, 465, 126, 752, 975, 2559, 31, 375, 201, 2560, 25, 29, 585, 212, 549, 27, 332, 537, 799, 517, 317, 837, 187, 214, 240, 1259, 2528, 156, 1547, 443, 491, 183, 982, 237, 1029, 962, 1289, 1270, 593, 591, 1237, 942] payload_rows_per_dst=[7476, 9292, 8886, 11644, 11086, 8936, 10324, 9840] dup_rows_per_dst=[1794, 2518, 2130, 3514, 2264, 2278, 2810, 3512] combine_rows_per_src=[7476, 9292, 8886, 11644, 11086, 8936, 10324, 9840] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280010752 dispatch_meta_bytes=341688 combine_payload_bytes=280010752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280352440 ingress_bytes=314613760 link_bytes=314613760 token_bytes=560363192 weight_movement_bytes=34603008 total_nvlink_bytes=594966200 weight_to_token_ratio=0.061751 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.183866250e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 2, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=263995392 dispatch_meta_bytes=330280 combine_payload_bytes=263995392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264325672 ingress_bytes=298598400 link_bytes=298598400 token_bytes=528321064 weight_movement_bytes=34603008 total_nvlink_bytes=562924072 weight_to_token_ratio=0.065496 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123601740e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=236453888 dispatch_meta_bytes=296896 combine_payload_bytes=236453888 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=236750784 ingress_bytes=305659904 link_bytes=305659904 token_bytes=473204672 weight_movement_bytes=69206016 total_nvlink_bytes=542410688 weight_to_token_ratio=0.146250 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.150173611e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278265856 dispatch_meta_bytes=350760 combine_payload_bytes=278265856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278616616 ingress_bytes=312868864 link_bytes=312868864 token_bytes=556882472 weight_movement_bytes=34603008 total_nvlink_bytes=591485480 weight_to_token_ratio=0.062137 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.177300347e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277135360 dispatch_meta_bytes=346064 combine_payload_bytes=277135360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277481424 ingress_bytes=311738368 link_bytes=311738368 token_bytes=554616784 weight_movement_bytes=34603008 total_nvlink_bytes=589219792 weight_to_token_ratio=0.062391 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.173046382e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280674304 dispatch_meta_bytes=346064 combine_payload_bytes=280674304 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=281020368 ingress_bytes=349880320 link_bytes=349880320 token_bytes=561694672 weight_movement_bytes=69206016 total_nvlink_bytes=630900688 weight_to_token_ratio=0.123209 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=10752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.316571476e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=275980288 dispatch_meta_bytes=335208 combine_payload_bytes=275980288 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=276315496 ingress_bytes=414392320 link_bytes=414392320 token_bytes=552295784 weight_movement_bytes=138412032 total_nvlink_bytes=690707816 weight_to_token_ratio=0.250612 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=8960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.559324939e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77276 payload_rows_remote=66344 dup_rows=21028 dup_rows_remote=18996 dedup_ratio=0.213908 router_tokens_per_expert=[294, 885, 540, 553, 2273, 111, 4210, 309, 555, 691, 409, 982, 496, 1199, 69, 1391, 585, 998, 847, 2034, 31, 660, 33, 408, 120, 802, 987, 2475, 74, 472, 288, 2437, 36, 42, 560, 186, 605, 23, 310, 510, 841, 569, 310, 818, 202, 170, 259, 1203, 2565, 170, 1685, 430, 328, 166, 920, 228, 1009, 1081, 1184, 1199, 654, 598, 1239, 834] payload_rows_per_dst=[7734, 9114, 8880, 11724, 10932, 8720, 10360, 9812] dup_rows_per_dst=[2196, 2470, 2312, 3586, 2032, 2502, 2624, 3306] combine_rows_per_src=[7734, 9114, 8880, 11724, 10932, 8720, 10360, 9812] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71460 payload_rows_remote=57728 dup_rows=26844 dup_rows_remote=16496 dedup_ratio=0.273071 router_tokens_per_expert=[315, 935, 370, 632, 2325, 95, 4544, 297, 602, 704, 429, 979, 451, 1080, 45, 1479, 571, 974, 1061, 2036, 31, 643, 11, 374, 134, 845, 927, 2527, 50, 396, 253, 2226, 23, 33, 520, 129, 534, 12, 276, 531, 783, 579, 287, 791, 217, 182, 270, 1265, 2718, 156, 1670, 397, 451, 172, 862, 194, 1052, 1125, 1229, 989, 591, 602, 1337, 834] payload_rows_per_dst=[13732, 11750, 11134, 5616, 3992, 10454, 7156, 7626] dup_rows_per_dst=[10348, 5186, 4720, 920, 124, 3426, 1290, 830] combine_rows_per_src=[13732, 11750, 11134, 5616, 3992, 10454, 7156, 7626] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77868 payload_rows_remote=67378 dup_rows=20436 dup_rows_remote=16424 dedup_ratio=0.207886 router_tokens_per_expert=[311, 902, 416, 654, 2331, 110, 4806, 282, 730, 654, 369, 913, 527, 1067, 49, 1304, 566, 1069, 958, 1990, 17, 569, 10, 422, 121, 794, 908, 2037, 45, 385, 245, 2526, 23, 15, 686, 194, 515, 25, 252, 501, 765, 511, 313, 818, 232, 157, 228, 1279, 2274, 148, 1461, 494, 430, 161, 841, 115, 1245, 1121, 1260, 1385, 745, 586, 1376, 909] payload_rows_per_dst=[7834, 9002, 8878, 11274, 11722, 8988, 9680, 10490] dup_rows_per_dst=[2178, 2224, 2324, 2848, 2312, 2370, 2168, 4012] combine_rows_per_src=[7834, 9002, 8878, 11274, 11722, 8988, 9680, 10490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=275513344 dispatch_meta_bytes=341864 combine_payload_bytes=275513344 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=275855208 ingress_bytes=344719360 link_bytes=344719360 token_bytes=551368552 weight_movement_bytes=69206016 total_nvlink_bytes=620574568 weight_to_token_ratio=0.125517 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.297151199e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=271745024 dispatch_meta_bytes=341360 combine_payload_bytes=271745024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272086384 ingress_bytes=306348032 link_bytes=306348032 token_bytes=543831408 weight_movement_bytes=34603008 total_nvlink_bytes=578434416 weight_to_token_ratio=0.063628 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=31360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.152762981e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=269680640 dispatch_meta_bytes=332584 combine_payload_bytes=269680640 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=270013224 ingress_bytes=373489664 link_bytes=373489664 token_bytes=539693864 weight_movement_bytes=103809024 total_nvlink_bytes=643502888 weight_to_token_ratio=0.192348 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.405411537e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 2, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=275980288 dispatch_meta_bytes=335208 combine_payload_bytes=275980288 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=276315496 ingress_bytes=414392320 link_bytes=414392320 token_bytes=552295784 weight_movement_bytes=138412032 total_nvlink_bytes=690707816 weight_to_token_ratio=0.250612 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=8960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.559324939e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77640 payload_rows_remote=68524 dup_rows=20664 dup_rows_remote=17992 dedup_ratio=0.210205 router_tokens_per_expert=[223, 847, 376, 510, 2182, 126, 4607, 270, 516, 678, 414, 1239, 527, 1033, 37, 1343, 518, 976, 1020, 2052, 11, 657, 16, 343, 123, 721, 904, 2706, 38, 569, 200, 2283, 21, 37, 522, 148, 485, 10, 380, 524, 799, 591, 291, 818, 218, 241, 253, 1269, 2729, 144, 1545, 409, 493, 151, 883, 139, 954, 1072, 1436, 1085, 593, 589, 1414, 844] payload_rows_per_dst=[7420, 9222, 8830, 11690, 11472, 9116, 10086, 9804] dup_rows_per_dst=[1648, 2352, 2356, 3398, 1996, 2672, 2900, 3342] combine_rows_per_src=[7420, 9222, 8830, 11690, 11472, 9116, 10086, 9804] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73114 payload_rows_remote=62328 dup_rows=25190 dup_rows_remote=21326 dedup_ratio=0.256246 router_tokens_per_expert=[458, 367, 445, 1198, 686, 57, 1423, 793, 1035, 970, 1898, 866, 641, 1261, 114, 214, 3616, 949, 4920, 469, 489, 36, 1826, 1593, 321, 836, 550, 153, 189, 1233, 836, 788, 238, 293, 935, 999, 415, 462, 591, 168, 924, 429, 6, 264, 262, 799, 316, 107, 2352, 1048, 546, 937, 73, 882, 1179, 369, 644, 1099, 301, 41, 225, 45, 12, 961] payload_rows_per_dst=[10786, 8310, 13344, 8226, 12266, 5686, 8282, 6214] dup_rows_per_dst=[3864, 1892, 9316, 1586, 5776, 528, 1786, 442] combine_rows_per_src=[10786, 8310, 13344, 8226, 12266, 5686, 8282, 6214] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=65826 dup_rows=24010 dup_rows_remote=22016 dedup_ratio=0.244242 router_tokens_per_expert=[422, 300, 450, 1204, 903, 57, 1104, 710, 1156, 814, 1797, 835, 643, 1437, 165, 181, 3219, 1183, 4621, 565, 654, 39, 1393, 1434, 395, 952, 616, 157, 136, 1208, 972, 743, 235, 334, 944, 1277, 277, 523, 771, 199, 1039, 561, 19, 312, 303, 876, 297, 104, 2255, 1087, 441, 855, 107, 1013, 1348, 268, 596, 876, 456, 44, 209, 44, 22, 995] payload_rows_per_dst=[10152, 8468, 10458, 10096, 12252, 6224, 8258, 8386] dup_rows_per_dst=[3742, 1994, 4542, 3328, 6110, 798, 1980, 1516] combine_rows_per_src=[10152, 8468, 10458, 10096, 12252, 6224, 8258, 8386] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=271745024 dispatch_meta_bytes=341360 combine_payload_bytes=271745024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272086384 ingress_bytes=306348032 link_bytes=306348032 token_bytes=543831408 weight_movement_bytes=34603008 total_nvlink_bytes=578434416 weight_to_token_ratio=0.063628 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=31360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.152762981e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=236453888 dispatch_meta_bytes=296896 combine_payload_bytes=236453888 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=236750784 ingress_bytes=305659904 link_bytes=305659904 token_bytes=473204672 weight_movement_bytes=69206016 total_nvlink_bytes=542410688 weight_to_token_ratio=0.146250 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.150173611e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78060 payload_rows_remote=68362 dup_rows=20244 dup_rows_remote=17060 dedup_ratio=0.205933 router_tokens_per_expert=[236, 867, 339, 597, 2282, 100, 4510, 320, 539, 562, 453, 1173, 487, 1011, 60, 1488, 506, 1001, 1097, 2169, 21, 708, 27, 428, 144, 759, 950, 2402, 43, 467, 251, 2257, 19, 19, 512, 168, 535, 16, 382, 501, 834, 518, 282, 792, 226, 166, 244, 1080, 2719, 155, 1833, 406, 424, 155, 856, 125, 1186, 1059, 1172, 1062, 602, 533, 1383, 934] payload_rows_per_dst=[8994, 9130, 9698, 9418, 11238, 10066, 10426, 9090] dup_rows_per_dst=[2420, 2416, 3184, 2228, 2086, 3248, 2920, 1742] combine_rows_per_src=[8994, 9130, 9698, 9418, 11238, 10066, 10426, 9090] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76752 payload_rows_remote=65756 dup_rows=21552 dup_rows_remote=17362 dedup_ratio=0.219238 router_tokens_per_expert=[535, 322, 476, 1195, 834, 68, 1308, 720, 1143, 1060, 1849, 710, 701, 1313, 131, 163, 3511, 904, 4522, 498, 691, 25, 1495, 1535, 439, 848, 523, 138, 166, 1199, 885, 777, 186, 299, 886, 1160, 309, 566, 693, 194, 1076, 491, 7, 322, 262, 706, 290, 94, 2696, 1259, 396, 811, 109, 937, 1099, 286, 629, 808, 423, 23, 254, 55, 16, 1126] payload_rows_per_dst=[8760, 10424, 8542, 8176, 7072, 11596, 10996, 11186] dup_rows_per_dst=[2156, 3716, 1754, 1774, 1514, 3944, 4190, 2504] combine_rows_per_src=[8760, 10424, 8542, 8176, 7072, 11596, 10996, 11186] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76032 payload_rows_remote=67284 dup_rows=22272 dup_rows_remote=20040 dedup_ratio=0.226562 router_tokens_per_expert=[308, 403, 312, 1013, 698, 53, 1480, 723, 1164, 913, 1845, 919, 625, 1440, 99, 275, 3685, 1052, 4542, 487, 665, 46, 1656, 1584, 487, 1075, 585, 159, 150, 1324, 1033, 558, 181, 233, 876, 1033, 426, 477, 380, 164, 1033, 439, 11, 255, 245, 879, 269, 114, 2596, 1152, 302, 974, 81, 813, 1275, 274, 552, 921, 456, 42, 195, 52, 18, 1076] payload_rows_per_dst=[10226, 8836, 8748, 11194, 11912, 5846, 7938, 11332] dup_rows_per_dst=[3444, 2034, 2232, 4740, 4712, 644, 1804, 2662] combine_rows_per_src=[10226, 8836, 8748, 11194, 11912, 5846, 7938, 11332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76262 payload_rows_remote=68926 dup_rows=22042 dup_rows_remote=20446 dedup_ratio=0.224223 router_tokens_per_expert=[426, 335, 457, 1078, 703, 71, 1214, 741, 1031, 1147, 1468, 857, 719, 1407, 150, 249, 3204, 953, 4520, 549, 725, 48, 1432, 1688, 367, 839, 631, 211, 173, 1182, 959, 784, 227, 373, 913, 1135, 337, 441, 852, 188, 1016, 498, 33, 295, 314, 847, 346, 141, 2210, 1207, 464, 907, 87, 982, 1269, 368, 593, 754, 486, 50, 283, 59, 20, 1139] payload_rows_per_dst=[8064, 10188, 8732, 10868, 7336, 11990, 8472, 10612] dup_rows_per_dst=[1986, 3868, 2058, 3844, 1596, 4030, 2096, 2564] combine_rows_per_src=[8064, 10188, 8732, 10868, 7336, 11990, 8472, 10612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77322 payload_rows_remote=66680 dup_rows=20982 dup_rows_remote=17300 dedup_ratio=0.213440 router_tokens_per_expert=[472, 316, 463, 1307, 875, 46, 1340, 708, 1043, 1045, 2009, 836, 687, 1442, 134, 236, 3486, 866, 4506, 490, 499, 25, 1647, 1537, 458, 751, 495, 195, 192, 1171, 820, 897, 255, 296, 1032, 1206, 244, 521, 727, 204, 920, 466, 14, 289, 277, 657, 323, 103, 2183, 1182, 398, 884, 94, 935, 1189, 332, 646, 1098, 400, 41, 228, 44, 7, 963] payload_rows_per_dst=[10086, 9564, 8532, 10642, 7450, 11622, 8212, 11214] dup_rows_per_dst=[3486, 2782, 1596, 3682, 1534, 3474, 1816, 2612] combine_rows_per_src=[10086, 9564, 8532, 10642, 7450, 11622, 8212, 11214] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280674304 dispatch_meta_bytes=346064 combine_payload_bytes=280674304 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=281020368 ingress_bytes=349880320 link_bytes=349880320 token_bytes=561694672 weight_movement_bytes=69206016 total_nvlink_bytes=630900688 weight_to_token_ratio=0.123209 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=10752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.316571476e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=255295488 dispatch_meta_bytes=334616 combine_payload_bytes=255295488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=255630104 ingress_bytes=289898496 link_bytes=289898496 token_bytes=510925592 weight_movement_bytes=34603008 total_nvlink_bytes=545528600 weight_to_token_ratio=0.067726 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.090864702e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=269623296 dispatch_meta_bytes=351368 combine_payload_bytes=269623296 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=269974664 ingress_bytes=338829312 link_bytes=338829312 token_bytes=539597960 weight_movement_bytes=69206016 total_nvlink_bytes=608803976 weight_to_token_ratio=0.128255 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=9728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274987423e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77412 payload_rows_remote=65182 dup_rows=20892 dup_rows_remote=16748 dedup_ratio=0.212524 router_tokens_per_expert=[458, 265, 379, 1015, 747, 44, 1530, 818, 959, 1102, 1917, 915, 671, 1377, 103, 207, 3464, 932, 4893, 413, 466, 20, 1671, 1607, 462, 774, 558, 262, 170, 1242, 830, 689, 249, 287, 1081, 1107, 402, 419, 587, 176, 994, 515, 14, 317, 264, 788, 300, 102, 2241, 1105, 427, 874, 62, 906, 1295, 268, 652, 999, 376, 29, 258, 72, 8, 1018] payload_rows_per_dst=[8522, 10494, 8496, 8612, 7488, 12230, 10306, 11264] dup_rows_per_dst=[1990, 4008, 1722, 1784, 1128, 4144, 3628, 2488] combine_rows_per_src=[8522, 10494, 8496, 8612, 7488, 12230, 10306, 11264] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77340 payload_rows_remote=67264 dup_rows=20964 dup_rows_remote=18202 dedup_ratio=0.213257 router_tokens_per_expert=[217, 855, 488, 630, 2102, 104, 4708, 292, 625, 713, 364, 1015, 502, 1240, 82, 1379, 497, 952, 690, 1905, 22, 611, 21, 388, 161, 692, 839, 2243, 62, 608, 190, 2484, 14, 20, 589, 295, 446, 27, 392, 546, 785, 575, 280, 909, 242, 156, 231, 1204, 2430, 156, 1771, 467, 436, 154, 885, 120, 888, 1018, 1369, 1511, 591, 606, 1433, 925] payload_rows_per_dst=[7496, 9520, 8398, 11234, 11708, 9074, 10076, 9834] dup_rows_per_dst=[1880, 2320, 1774, 3324, 2366, 2556, 2762, 3982] combine_rows_per_src=[7496, 9520, 8398, 11234, 11708, 9074, 10076, 9834] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75874 payload_rows_remote=65076 dup_rows=22430 dup_rows_remote=19920 dedup_ratio=0.228170 router_tokens_per_expert=[567, 340, 419, 1330, 1014, 71, 1156, 623, 1133, 888, 1990, 792, 735, 1516, 114, 206, 3258, 943, 4610, 398, 640, 45, 1411, 1512, 405, 795, 498, 146, 171, 1100, 818, 943, 233, 293, 923, 1088, 283, 513, 699, 214, 923, 488, 15, 247, 284, 760, 329, 118, 2503, 1182, 441, 850, 142, 1057, 1185, 399, 537, 1113, 385, 31, 238, 50, 15, 1027] payload_rows_per_dst=[8848, 9164, 6890, 8328, 8242, 11892, 11712, 10798] dup_rows_per_dst=[2192, 2562, 1342, 1424, 1916, 3656, 6828, 2510] combine_rows_per_src=[8848, 9164, 6890, 8328, 8242, 11892, 11712, 10798] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77322 payload_rows_remote=66680 dup_rows=20982 dup_rows_remote=17300 dedup_ratio=0.213440 router_tokens_per_expert=[472, 316, 463, 1307, 875, 46, 1340, 708, 1043, 1045, 2009, 836, 687, 1442, 134, 236, 3486, 866, 4506, 490, 499, 25, 1647, 1537, 458, 751, 495, 195, 192, 1171, 820, 897, 255, 296, 1032, 1206, 244, 521, 727, 204, 920, 466, 14, 289, 277, 657, 323, 103, 2183, 1182, 398, 884, 94, 935, 1189, 332, 646, 1098, 400, 41, 228, 44, 7, 963] payload_rows_per_dst=[10086, 9564, 8532, 10642, 7450, 11622, 8212, 11214] dup_rows_per_dst=[3486, 2782, 1596, 3682, 1534, 3474, 1816, 2612] combine_rows_per_src=[10086, 9564, 8532, 10642, 7450, 11622, 8212, 11214] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=275595264 dispatch_meta_bytes=349296 combine_payload_bytes=275595264 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=275944560 ingress_bytes=448610304 link_bytes=448610304 token_bytes=551539824 weight_movement_bytes=173015040 total_nvlink_bytes=724554864 weight_to_token_ratio=0.313695 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=12544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.688084458e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280010752 dispatch_meta_bytes=341688 combine_payload_bytes=280010752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280352440 ingress_bytes=314613760 link_bytes=314613760 token_bytes=560363192 weight_movement_bytes=34603008 total_nvlink_bytes=594966200 weight_to_token_ratio=0.061751 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.183866250e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=269336576 dispatch_meta_bytes=332472 combine_payload_bytes=269336576 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=269669048 ingress_bytes=373145600 link_bytes=373145600 token_bytes=539005624 weight_movement_bytes=103809024 total_nvlink_bytes=642814648 weight_to_token_ratio=0.192594 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=9856 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.404116852e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=282320896 dispatch_meta_bytes=357488 combine_payload_bytes=282320896 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=282678384 ingress_bytes=351526912 link_bytes=351526912 token_bytes=564999280 weight_movement_bytes=69206016 total_nvlink_bytes=634205296 weight_to_token_ratio=0.122489 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=14208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.322767470e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=273121280 dispatch_meta_bytes=335920 combine_payload_bytes=273121280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273457200 ingress_bytes=307724288 link_bytes=307724288 token_bytes=546578480 weight_movement_bytes=34603008 total_nvlink_bytes=581181488 weight_to_token_ratio=0.063308 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.157941722e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266985472 dispatch_meta_bytes=327720 combine_payload_bytes=266985472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267313192 ingress_bytes=301588480 link_bytes=301588480 token_bytes=534298664 weight_movement_bytes=34603008 total_nvlink_bytes=568901672 weight_to_token_ratio=0.064763 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=23168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134853170e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76262 payload_rows_remote=68926 dup_rows=22042 dup_rows_remote=20446 dedup_ratio=0.224223 router_tokens_per_expert=[426, 335, 457, 1078, 703, 71, 1214, 741, 1031, 1147, 1468, 857, 719, 1407, 150, 249, 3204, 953, 4520, 549, 725, 48, 1432, 1688, 367, 839, 631, 211, 173, 1182, 959, 784, 227, 373, 913, 1135, 337, 441, 852, 188, 1016, 498, 33, 295, 314, 847, 346, 141, 2210, 1207, 464, 907, 87, 982, 1269, 368, 593, 754, 486, 50, 283, 59, 20, 1139] payload_rows_per_dst=[8064, 10188, 8732, 10868, 7336, 11990, 8472, 10612] dup_rows_per_dst=[1986, 3868, 2058, 3844, 1596, 4030, 2096, 2564] combine_rows_per_src=[8064, 10188, 8732, 10868, 7336, 11990, 8472, 10612] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=275513344 dispatch_meta_bytes=341864 combine_payload_bytes=275513344 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=275855208 ingress_bytes=344719360 link_bytes=344719360 token_bytes=551368552 weight_movement_bytes=69206016 total_nvlink_bytes=620574568 weight_to_token_ratio=0.125517 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.297151199e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266551296 dispatch_meta_bytes=339984 combine_payload_bytes=266551296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266891280 ingress_bytes=301154304 link_bytes=301154304 token_bytes=533442576 weight_movement_bytes=34603008 total_nvlink_bytes=568045584 weight_to_token_ratio=0.064867 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=22400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133219401e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73114 payload_rows_remote=62328 dup_rows=25190 dup_rows_remote=21326 dedup_ratio=0.256246 router_tokens_per_expert=[458, 367, 445, 1198, 686, 57, 1423, 793, 1035, 970, 1898, 866, 641, 1261, 114, 214, 3616, 949, 4920, 469, 489, 36, 1826, 1593, 321, 836, 550, 153, 189, 1233, 836, 788, 238, 293, 935, 999, 415, 462, 591, 168, 924, 429, 6, 264, 262, 799, 316, 107, 2352, 1048, 546, 937, 73, 882, 1179, 369, 644, 1099, 301, 41, 225, 45, 12, 961] payload_rows_per_dst=[10786, 8310, 13344, 8226, 12266, 5686, 8282, 6214] dup_rows_per_dst=[3864, 1892, 9316, 1586, 5776, 528, 1786, 442] combine_rows_per_src=[10786, 8310, 13344, 8226, 12266, 5686, 8282, 6214] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=273121280 dispatch_meta_bytes=335920 combine_payload_bytes=273121280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273457200 ingress_bytes=307724288 link_bytes=307724288 token_bytes=546578480 weight_movement_bytes=34603008 total_nvlink_bytes=581181488 weight_to_token_ratio=0.063308 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.157941722e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77412 payload_rows_remote=65182 dup_rows=20892 dup_rows_remote=16748 dedup_ratio=0.212524 router_tokens_per_expert=[458, 265, 379, 1015, 747, 44, 1530, 818, 959, 1102, 1917, 915, 671, 1377, 103, 207, 3464, 932, 4893, 413, 466, 20, 1671, 1607, 462, 774, 558, 262, 170, 1242, 830, 689, 249, 287, 1081, 1107, 402, 419, 587, 176, 994, 515, 14, 317, 264, 788, 300, 102, 2241, 1105, 427, 874, 62, 906, 1295, 268, 652, 999, 376, 29, 258, 72, 8, 1018] payload_rows_per_dst=[8522, 10494, 8496, 8612, 7488, 12230, 10306, 11264] dup_rows_per_dst=[1990, 4008, 1722, 1784, 1128, 4144, 3628, 2488] combine_rows_per_src=[8522, 10494, 8496, 8612, 7488, 12230, 10306, 11264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76620 payload_rows_remote=66336 dup_rows=21684 dup_rows_remote=17514 dedup_ratio=0.220581 router_tokens_per_expert=[11, 13, 3, 1452, 1515, 2824, 563, 112, 1069, 736, 363, 58, 425, 121, 1106, 173, 469, 343, 1, 5, 1025, 147, 162, 129, 3033, 1415, 7, 858, 909, 415, 1395, 1372, 4, 142, 2758, 424, 5046, 189, 1011, 251, 600, 453, 691, 664, 268, 242, 2, 201, 725, 352, 1299, 408, 3960, 165, 733, 206, 542, 31, 99, 1108, 32, 1830, 116, 2371] payload_rows_per_dst=[10284, 12396, 11962, 11264, 8118, 5990, 6952, 9654] dup_rows_per_dst=[4170, 3626, 2692, 4644, 1440, 1684, 824, 2604] combine_rows_per_src=[10284, 12396, 11962, 11264, 8118, 5990, 6952, 9654] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77078 payload_rows_remote=65024 dup_rows=21226 dup_rows_remote=17222 dedup_ratio=0.215922 router_tokens_per_expert=[10, 45, 3, 1737, 1261, 2839, 393, 228, 1179, 778, 383, 189, 472, 185, 937, 245, 378, 299, 1, 10, 775, 174, 410, 118, 3259, 1419, 30, 694, 859, 338, 1298, 1403, 11, 141, 2782, 591, 4632, 173, 1207, 263, 711, 428, 1032, 607, 203, 293, 5, 168, 746, 487, 1111, 395, 3661, 131, 978, 165, 722, 51, 65, 1254, 22, 1392, 156, 2220] payload_rows_per_dst=[9968, 12054, 11290, 8934, 8636, 9830, 7058, 9308] dup_rows_per_dst=[3064, 4004, 2304, 3148, 1700, 3582, 968, 2456] combine_rows_per_src=[9968, 12054, 11290, 8934, 8636, 9830, 7058, 9308] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=282320896 dispatch_meta_bytes=357488 combine_payload_bytes=282320896 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=282678384 ingress_bytes=351526912 link_bytes=351526912 token_bytes=564999280 weight_movement_bytes=69206016 total_nvlink_bytes=634205296 weight_to_token_ratio=0.122489 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=14208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.322767470e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=255295488 dispatch_meta_bytes=334616 combine_payload_bytes=255295488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=255630104 ingress_bytes=289898496 link_bytes=289898496 token_bytes=510925592 weight_movement_bytes=34603008 total_nvlink_bytes=545528600 weight_to_token_ratio=0.067726 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.090864702e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76460 payload_rows_remote=64754 dup_rows=21844 dup_rows_remote=19552 dedup_ratio=0.222209 router_tokens_per_expert=[22, 26, 4, 1558, 1593, 3171, 653, 119, 919, 705, 365, 108, 433, 107, 1111, 259, 495, 311, 9, 2, 792, 130, 176, 160, 2977, 1407, 6, 902, 1261, 400, 1487, 1186, 5, 159, 2381, 331, 4924, 211, 1238, 224, 606, 418, 1004, 653, 178, 281, 4, 204, 656, 363, 1183, 448, 3962, 160, 650, 110, 545, 47, 85, 895, 114, 1580, 189, 2490] payload_rows_per_dst=[10156, 12252, 11706, 9628, 7762, 9406, 6400, 9150] dup_rows_per_dst=[4136, 3686, 2292, 3670, 1336, 3244, 740, 2740] combine_rows_per_src=[10156, 12252, 11706, 9628, 7762, 9406, 6400, 9150] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=328984 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266667288 ingress_bytes=300941312 link_bytes=300941312 token_bytes=533005592 weight_movement_bytes=34603008 total_nvlink_bytes=567608600 weight_to_token_ratio=0.064921 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=17792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132417929e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74992 payload_rows_remote=65730 dup_rows=23312 dup_rows_remote=20070 dedup_ratio=0.237142 router_tokens_per_expert=[18, 20, 3, 1580, 1388, 2682, 741, 126, 896, 699, 339, 125, 407, 126, 1095, 113, 350, 323, 1, 2, 959, 131, 148, 137, 2997, 1537, 20, 773, 1430, 378, 1468, 1297, 4, 106, 2535, 466, 5077, 154, 1070, 226, 576, 453, 744, 763, 286, 251, 2, 180, 742, 362, 1417, 404, 4218, 148, 805, 121, 534, 34, 109, 909, 36, 1537, 61, 2513] payload_rows_per_dst=[9482, 6346, 11820, 8644, 7896, 9262, 12468, 9074] dup_rows_per_dst=[3634, 1254, 2436, 3316, 1226, 3242, 5812, 2392] combine_rows_per_src=[9482, 6346, 11820, 8644, 7896, 9262, 12468, 9074] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76752 payload_rows_remote=65756 dup_rows=21552 dup_rows_remote=17362 dedup_ratio=0.219238 router_tokens_per_expert=[535, 322, 476, 1195, 834, 68, 1308, 720, 1143, 1060, 1849, 710, 701, 1313, 131, 163, 3511, 904, 4522, 498, 691, 25, 1495, 1535, 439, 848, 523, 138, 166, 1199, 885, 777, 186, 299, 886, 1160, 309, 566, 693, 194, 1076, 491, 7, 322, 262, 706, 290, 94, 2696, 1259, 396, 811, 109, 937, 1099, 286, 629, 808, 423, 23, 254, 55, 16, 1126] payload_rows_per_dst=[8760, 10424, 8542, 8176, 7072, 11596, 10996, 11186] dup_rows_per_dst=[2156, 3716, 1754, 1774, 1514, 3944, 4190, 2504] combine_rows_per_src=[8760, 10424, 8542, 8176, 7072, 11596, 10996, 11186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72354 payload_rows_remote=59916 dup_rows=25950 dup_rows_remote=19646 dedup_ratio=0.263977 router_tokens_per_expert=[3, 43, 4, 1716, 1413, 3241, 489, 52, 839, 727, 292, 148, 394, 118, 981, 136, 372, 286, 2, 11, 817, 154, 255, 124, 3363, 1472, 13, 881, 1045, 405, 1493, 1314, 3, 174, 2557, 483, 5179, 250, 895, 195, 581, 445, 926, 641, 204, 191, 4, 203, 898, 391, 1120, 430, 3901, 128, 857, 174, 658, 57, 66, 1011, 50, 1317, 121, 2439] payload_rows_per_dst=[7910, 6086, 7530, 7462, 12244, 9604, 12438, 9080] dup_rows_per_dst=[2016, 1184, 1310, 2840, 6426, 3512, 6304, 2358] combine_rows_per_src=[7910, 6086, 7530, 7462, 12244, 9604, 12438, 9080] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76032 payload_rows_remote=67284 dup_rows=22272 dup_rows_remote=20040 dedup_ratio=0.226562 router_tokens_per_expert=[308, 403, 312, 1013, 698, 53, 1480, 723, 1164, 913, 1845, 919, 625, 1440, 99, 275, 3685, 1052, 4542, 487, 665, 46, 1656, 1584, 487, 1075, 585, 159, 150, 1324, 1033, 558, 181, 233, 876, 1033, 426, 477, 380, 164, 1033, 439, 11, 255, 245, 879, 269, 114, 2596, 1152, 302, 974, 81, 813, 1275, 274, 552, 921, 456, 42, 195, 52, 18, 1076] payload_rows_per_dst=[10226, 8836, 8748, 11194, 11912, 5846, 7938, 11332] dup_rows_per_dst=[3444, 2034, 2232, 4740, 4712, 644, 1804, 2662] combine_rows_per_src=[10226, 8836, 8748, 11194, 11912, 5846, 7938, 11332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76272 payload_rows_remote=68104 dup_rows=22032 dup_rows_remote=20546 dedup_ratio=0.224121 router_tokens_per_expert=[43, 61, 7, 1494, 1332, 2637, 603, 154, 1087, 824, 304, 281, 450, 123, 953, 284, 497, 330, 5, 6, 837, 182, 327, 134, 3256, 1334, 17, 801, 940, 368, 1569, 1391, 13, 195, 2354, 505, 4592, 225, 1281, 254, 737, 397, 919, 679, 207, 229, 5, 147, 798, 455, 1042, 380, 3614, 152, 841, 177, 659, 40, 100, 1170, 58, 1797, 149, 2350] payload_rows_per_dst=[9224, 10142, 11400, 9374, 8168, 9820, 8462, 9682] dup_rows_per_dst=[3438, 2996, 2420, 3466, 1486, 3332, 1930, 2964] combine_rows_per_src=[9224, 10142, 11400, 9374, 8168, 9820, 8462, 9682] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77408 payload_rows_remote=68250 dup_rows=20896 dup_rows_remote=17514 dedup_ratio=0.212565 router_tokens_per_expert=[13, 50, 0, 1789, 1281, 3180, 387, 124, 647, 701, 269, 165, 530, 148, 1002, 176, 362, 314, 5, 1, 1014, 133, 299, 127, 3318, 1504, 5, 807, 1049, 365, 1288, 1252, 14, 122, 2565, 557, 5199, 158, 1094, 268, 631, 406, 936, 616, 285, 186, 1, 299, 824, 443, 1070, 475, 3822, 169, 862, 174, 571, 52, 91, 1058, 37, 1587, 68, 2207] payload_rows_per_dst=[10254, 11734, 12104, 9158, 8210, 9884, 7012, 9052] dup_rows_per_dst=[3394, 3186, 2804, 3382, 1346, 3472, 1022, 2290] combine_rows_per_src=[10254, 11734, 12104, 9158, 8210, 9884, 7012, 9052] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266985472 dispatch_meta_bytes=327720 combine_payload_bytes=266985472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267313192 ingress_bytes=301588480 link_bytes=301588480 token_bytes=534298664 weight_movement_bytes=34603008 total_nvlink_bytes=568901672 weight_to_token_ratio=0.064763 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=23168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134853170e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70756 payload_rows_remote=59840 dup_rows=27548 dup_rows_remote=22964 dedup_ratio=0.280233 router_tokens_per_expert=[0, 20, 1, 2003, 1133, 2990, 280, 150, 743, 743, 312, 214, 457, 126, 901, 183, 302, 311, 2, 4, 871, 155, 351, 121, 3198, 1826, 6, 801, 766, 332, 1154, 1628, 9, 119, 2939, 694, 4901, 92, 918, 239, 632, 374, 899, 504, 205, 228, 10, 170, 991, 415, 1161, 417, 3834, 144, 928, 241, 607, 67, 96, 1292, 26, 1371, 119, 2426] payload_rows_per_dst=[6474, 6060, 3860, 6896, 14376, 9348, 12826, 10916] dup_rows_per_dst=[700, 1298, 374, 2478, 7934, 3092, 7088, 4584] combine_rows_per_src=[6474, 6060, 3860, 6896, 14376, 9348, 12826, 10916] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265232384 dispatch_meta_bytes=337224 combine_payload_bytes=265232384 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265569608 ingress_bytes=299835392 link_bytes=299835392 token_bytes=530801992 weight_movement_bytes=34603008 total_nvlink_bytes=565405000 weight_to_token_ratio=0.065190 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=31872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128256441e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=269230080 dispatch_meta_bytes=343200 combine_payload_bytes=269230080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269573280 ingress_bytes=303833088 link_bytes=303833088 token_bytes=538803360 weight_movement_bytes=34603008 total_nvlink_bytes=573406368 weight_to_token_ratio=0.064222 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.143299450e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=271712256 dispatch_meta_bytes=335400 combine_payload_bytes=271712256 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=272047656 ingress_bytes=375521280 link_bytes=375521280 token_bytes=543759912 weight_movement_bytes=103809024 total_nvlink_bytes=647568936 weight_to_token_ratio=0.190910 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=18688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.413056345e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=328984 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266667288 ingress_bytes=300941312 link_bytes=300941312 token_bytes=533005592 weight_movement_bytes=34603008 total_nvlink_bytes=567608600 weight_to_token_ratio=0.064921 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=17792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132417929e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76620 payload_rows_remote=66336 dup_rows=21684 dup_rows_remote=17514 dedup_ratio=0.220581 router_tokens_per_expert=[11, 13, 3, 1452, 1515, 2824, 563, 112, 1069, 736, 363, 58, 425, 121, 1106, 173, 469, 343, 1, 5, 1025, 147, 162, 129, 3033, 1415, 7, 858, 909, 415, 1395, 1372, 4, 142, 2758, 424, 5046, 189, 1011, 251, 600, 453, 691, 664, 268, 242, 2, 201, 725, 352, 1299, 408, 3960, 165, 733, 206, 542, 31, 99, 1108, 32, 1830, 116, 2371] payload_rows_per_dst=[10284, 12396, 11962, 11264, 8118, 5990, 6952, 9654] dup_rows_per_dst=[4170, 3626, 2692, 4644, 1440, 1684, 824, 2604] combine_rows_per_src=[10284, 12396, 11962, 11264, 8118, 5990, 6952, 9654] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=269336576 dispatch_meta_bytes=332472 combine_payload_bytes=269336576 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=269669048 ingress_bytes=373145600 link_bytes=373145600 token_bytes=539005624 weight_movement_bytes=103809024 total_nvlink_bytes=642814648 weight_to_token_ratio=0.192594 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=9856 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.404116852e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=245415936 dispatch_meta_bytes=318248 combine_payload_bytes=245415936 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=245734184 ingress_bytes=314621952 link_bytes=314621952 token_bytes=491150120 weight_movement_bytes=69206016 total_nvlink_bytes=560356136 weight_to_token_ratio=0.140906 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=13312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.183897076e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278953984 dispatch_meta_bytes=354600 combine_payload_bytes=278953984 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=279308584 ingress_bytes=348160000 link_bytes=348160000 token_bytes=558262568 weight_movement_bytes=69206016 total_nvlink_bytes=627468584 weight_to_token_ratio=0.123967 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=20992 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.310098051e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=275595264 dispatch_meta_bytes=349296 combine_payload_bytes=275595264 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=275944560 ingress_bytes=448610304 link_bytes=448610304 token_bytes=551539824 weight_movement_bytes=173015040 total_nvlink_bytes=724554864 weight_to_token_ratio=0.313695 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=12544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.688084458e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279552000 dispatch_meta_bytes=343056 combine_payload_bytes=279552000 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=279895056 ingress_bytes=383361024 link_bytes=383361024 token_bytes=559447056 weight_movement_bytes=103809024 total_nvlink_bytes=663256080 weight_to_token_ratio=0.185556 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=10880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.442556670e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76272 payload_rows_remote=68104 dup_rows=22032 dup_rows_remote=20546 dedup_ratio=0.224121 router_tokens_per_expert=[43, 61, 7, 1494, 1332, 2637, 603, 154, 1087, 824, 304, 281, 450, 123, 953, 284, 497, 330, 5, 6, 837, 182, 327, 134, 3256, 1334, 17, 801, 940, 368, 1569, 1391, 13, 195, 2354, 505, 4592, 225, 1281, 254, 737, 397, 919, 679, 207, 229, 5, 147, 798, 455, 1042, 380, 3614, 152, 841, 177, 659, 40, 100, 1170, 58, 1797, 149, 2350] payload_rows_per_dst=[9224, 10142, 11400, 9374, 8168, 9820, 8462, 9682] dup_rows_per_dst=[3438, 2996, 2420, 3466, 1486, 3332, 1930, 2964] combine_rows_per_src=[9224, 10142, 11400, 9374, 8168, 9820, 8462, 9682] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=245104640 dispatch_meta_bytes=331216 combine_payload_bytes=245104640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=245435856 ingress_bytes=279707648 link_bytes=279707648 token_bytes=490540496 weight_movement_bytes=34603008 total_nvlink_bytes=525143504 weight_to_token_ratio=0.070541 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=19072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.052517361e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74992 payload_rows_remote=65730 dup_rows=23312 dup_rows_remote=20070 dedup_ratio=0.237142 router_tokens_per_expert=[18, 20, 3, 1580, 1388, 2682, 741, 126, 896, 699, 339, 125, 407, 126, 1095, 113, 350, 323, 1, 2, 959, 131, 148, 137, 2997, 1537, 20, 773, 1430, 378, 1468, 1297, 4, 106, 2535, 466, 5077, 154, 1070, 226, 576, 453, 744, 763, 286, 251, 2, 180, 742, 362, 1417, 404, 4218, 148, 805, 121, 534, 34, 109, 909, 36, 1537, 61, 2513] payload_rows_per_dst=[9482, 6346, 11820, 8644, 7896, 9262, 12468, 9074] dup_rows_per_dst=[3634, 1254, 2436, 3316, 1226, 3242, 5812, 2392] combine_rows_per_src=[9482, 6346, 11820, 8644, 7896, 9262, 12468, 9074] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73576 payload_rows_remote=63474 dup_rows=24728 dup_rows_remote=20638 dedup_ratio=0.251546 router_tokens_per_expert=[1971, 2922, 1707, 85, 44, 0, 0, 367, 296, 1822, 0, 682, 148, 1597, 382, 40, 896, 474, 24, 664, 810, 260, 631, 1472, 1124, 624, 4, 1289, 5, 1557, 4, 3493, 451, 1313, 614, 232, 2985, 0, 1090, 457, 13, 1298, 424, 1392, 0, 397, 210, 1697, 478, 2917, 313, 308, 1058, 321, 865, 559, 875, 6, 330, 793, 737, 1119, 379, 127] payload_rows_per_dst=[10102, 11584, 11912, 7558, 8008, 10686, 6410, 7316] dup_rows_per_dst=[4090, 4320, 5536, 1656, 2674, 3642, 1394, 1416] combine_rows_per_src=[10102, 11584, 11912, 7558, 8008, 10686, 6410, 7316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74726 payload_rows_remote=64882 dup_rows=23578 dup_rows_remote=20764 dedup_ratio=0.239848 router_tokens_per_expert=[2103, 2586, 1145, 58, 124, 3, 0, 315, 202, 1510, 0, 686, 93, 1518, 375, 73, 958, 274, 20, 563, 695, 238, 788, 1340, 1321, 629, 2, 979, 1, 1661, 6, 3655, 444, 1750, 728, 157, 3296, 3, 859, 815, 19, 1671, 341, 1460, 1, 518, 234, 1837, 540, 2748, 199, 318, 835, 271, 869, 549, 990, 4, 487, 842, 469, 1415, 449, 113] payload_rows_per_dst=[7954, 7410, 8066, 9382, 11066, 9600, 9844, 11404] dup_rows_per_dst=[1520, 1504, 1686, 3010, 5038, 2562, 2814, 5444] combine_rows_per_src=[7954, 7410, 8066, 9382, 11066, 9600, 9844, 11404] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75788 payload_rows_remote=64464 dup_rows=22516 dup_rows_remote=19428 dedup_ratio=0.229045 router_tokens_per_expert=[2145, 2569, 1180, 162, 213, 3, 0, 394, 347, 1710, 0, 619, 100, 1811, 397, 56, 928, 240, 163, 361, 877, 288, 761, 1561, 1313, 660, 0, 1167, 1, 1498, 6, 3963, 545, 1802, 569, 112, 2166, 4, 738, 532, 22, 1180, 406, 1298, 0, 700, 207, 2010, 872, 2872, 130, 387, 1027, 364, 697, 533, 1124, 11, 330, 695, 497, 1403, 295, 131] payload_rows_per_dst=[9782, 11324, 12814, 7758, 7200, 10518, 8856, 7536] dup_rows_per_dst=[3550, 3088, 5470, 1532, 1404, 3472, 2564, 1436] combine_rows_per_src=[9782, 11324, 12814, 7758, 7200, 10518, 8856, 7536] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=271712256 dispatch_meta_bytes=335400 combine_payload_bytes=271712256 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=272047656 ingress_bytes=375521280 link_bytes=375521280 token_bytes=543759912 weight_movement_bytes=103809024 total_nvlink_bytes=647568936 weight_to_token_ratio=0.190910 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=18688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.413056345e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75202 payload_rows_remote=64780 dup_rows=23102 dup_rows_remote=19218 dedup_ratio=0.235006 router_tokens_per_expert=[1505, 2909, 1448, 141, 296, 7, 0, 259, 246, 1565, 6, 673, 108, 1277, 406, 77, 857, 374, 19, 595, 684, 367, 749, 1497, 1305, 623, 22, 1197, 9, 1629, 11, 3437, 551, 1670, 439, 201, 2913, 12, 846, 521, 11, 1600, 450, 1276, 3, 550, 338, 2161, 630, 2470, 367, 356, 1072, 349, 819, 620, 843, 39, 419, 866, 563, 1347, 419, 133] payload_rows_per_dst=[9662, 7334, 8450, 11320, 10422, 10040, 9966, 8008] dup_rows_per_dst=[3468, 1382, 1834, 4282, 3884, 2738, 3400, 2114] combine_rows_per_src=[9662, 7334, 8450, 11320, 10422, 10040, 9966, 8008] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=269230080 dispatch_meta_bytes=343200 combine_payload_bytes=269230080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269573280 ingress_bytes=303833088 link_bytes=303833088 token_bytes=538803360 weight_movement_bytes=34603008 total_nvlink_bytes=573406368 weight_to_token_ratio=0.064222 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.143299450e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=259989504 dispatch_meta_bytes=336448 combine_payload_bytes=259989504 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=260325952 ingress_bytes=329195520 link_bytes=329195520 token_bytes=520315456 weight_movement_bytes=69206016 total_nvlink_bytes=589521472 weight_to_token_ratio=0.133008 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.238736239e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278953984 dispatch_meta_bytes=354600 combine_payload_bytes=278953984 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=279308584 ingress_bytes=348160000 link_bytes=348160000 token_bytes=558262568 weight_movement_bytes=69206016 total_nvlink_bytes=627468584 weight_to_token_ratio=0.123967 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=20992 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.310098051e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75032 payload_rows_remote=64640 dup_rows=23272 dup_rows_remote=19296 dedup_ratio=0.236735 router_tokens_per_expert=[1467, 2644, 1500, 86, 202, 0, 1, 345, 246, 1811, 0, 751, 154, 1304, 424, 52, 944, 471, 12, 879, 627, 355, 690, 1508, 1263, 614, 18, 978, 3, 1743, 10, 3556, 334, 1416, 402, 297, 3246, 0, 908, 522, 4, 1869, 389, 1790, 0, 401, 248, 1823, 377, 2590, 236, 287, 962, 284, 876, 614, 1030, 9, 329, 886, 453, 1336, 521, 55] payload_rows_per_dst=[9388, 11694, 10392, 9736, 6466, 10246, 9636, 7474] dup_rows_per_dst=[3102, 4282, 3976, 3238, 1292, 2802, 2816, 1764] combine_rows_per_src=[9388, 11694, 10392, 9736, 6466, 10246, 9636, 7474] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=264044544 dispatch_meta_bytes=335568 combine_payload_bytes=264044544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264380112 ingress_bytes=298647552 link_bytes=298647552 token_bytes=528424656 weight_movement_bytes=34603008 total_nvlink_bytes=563027664 weight_to_token_ratio=0.065483 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123786695e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74610 payload_rows_remote=65820 dup_rows=23694 dup_rows_remote=21452 dedup_ratio=0.241028 router_tokens_per_expert=[1995, 2736, 1267, 81, 51, 0, 0, 318, 292, 1640, 0, 643, 123, 1788, 394, 40, 845, 446, 39, 625, 745, 250, 848, 1480, 919, 653, 8, 1090, 2, 1721, 1, 3709, 513, 1616, 636, 145, 2712, 0, 1031, 720, 11, 1172, 387, 1727, 0, 470, 229, 1520, 442, 3225, 271, 350, 1083, 249, 711, 496, 891, 10, 346, 918, 552, 1309, 535, 126] payload_rows_per_dst=[9662, 8070, 8540, 7284, 10294, 8790, 10518, 11452] dup_rows_per_dst=[3234, 1770, 2016, 1504, 4452, 2242, 3136, 5340] combine_rows_per_src=[9662, 8070, 8540, 7284, 10294, 8790, 10518, 11452] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72354 payload_rows_remote=59916 dup_rows=25950 dup_rows_remote=19646 dedup_ratio=0.263977 router_tokens_per_expert=[3, 43, 4, 1716, 1413, 3241, 489, 52, 839, 727, 292, 148, 394, 118, 981, 136, 372, 286, 2, 11, 817, 154, 255, 124, 3363, 1472, 13, 881, 1045, 405, 1493, 1314, 3, 174, 2557, 483, 5179, 250, 895, 195, 581, 445, 926, 641, 204, 191, 4, 203, 898, 391, 1120, 430, 3901, 128, 857, 174, 658, 57, 66, 1011, 50, 1317, 121, 2439] payload_rows_per_dst=[7910, 6086, 7530, 7462, 12244, 9604, 12438, 9080] dup_rows_per_dst=[2016, 1184, 1310, 2840, 6426, 3512, 6304, 2358] combine_rows_per_src=[7910, 6086, 7530, 7462, 12244, 9604, 12438, 9080] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76312 payload_rows_remote=63800 dup_rows=21992 dup_rows_remote=17154 dedup_ratio=0.223714 router_tokens_per_expert=[2247, 2852, 1480, 101, 44, 0, 0, 363, 324, 1605, 2, 679, 132, 1898, 407, 64, 1046, 242, 42, 505, 696, 270, 711, 1105, 1345, 588, 3, 1360, 4, 1611, 10, 3754, 516, 1665, 924, 107, 2466, 0, 963, 402, 25, 941, 419, 1171, 0, 484, 194, 1975, 449, 3110, 259, 353, 1008, 275, 703, 543, 1036, 17, 283, 799, 595, 1542, 316, 122] payload_rows_per_dst=[10188, 10798, 7778, 12512, 8106, 8652, 10376, 7902] dup_rows_per_dst=[3986, 3058, 1456, 4838, 2346, 1766, 3024, 1518] combine_rows_per_src=[10188, 10798, 7778, 12512, 8106, 8652, 10376, 7902] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=264765440 dispatch_meta_bytes=335744 combine_payload_bytes=264765440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265101184 ingress_bytes=299368448 link_bytes=299368448 token_bytes=529866624 weight_movement_bytes=34603008 total_nvlink_bytes=564469632 weight_to_token_ratio=0.065305 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126499369e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=269598720 dispatch_meta_bytes=349088 combine_payload_bytes=269598720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269947808 ingress_bytes=304201728 link_bytes=304201728 token_bytes=539546528 weight_movement_bytes=34603008 total_nvlink_bytes=574149536 weight_to_token_ratio=0.064134 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.144686612e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76460 payload_rows_remote=64754 dup_rows=21844 dup_rows_remote=19552 dedup_ratio=0.222209 router_tokens_per_expert=[22, 26, 4, 1558, 1593, 3171, 653, 119, 919, 705, 365, 108, 433, 107, 1111, 259, 495, 311, 9, 2, 792, 130, 176, 160, 2977, 1407, 6, 902, 1261, 400, 1487, 1186, 5, 159, 2381, 331, 4924, 211, 1238, 224, 606, 418, 1004, 653, 178, 281, 4, 204, 656, 363, 1183, 448, 3962, 160, 650, 110, 545, 47, 85, 895, 114, 1580, 189, 2490] payload_rows_per_dst=[10156, 12252, 11706, 9628, 7762, 9406, 6400, 9150] dup_rows_per_dst=[4136, 3686, 2292, 3670, 1336, 3244, 740, 2740] combine_rows_per_src=[10156, 12252, 11706, 9628, 7762, 9406, 6400, 9150] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75422 payload_rows_remote=63104 dup_rows=22882 dup_rows_remote=17020 dedup_ratio=0.232768 router_tokens_per_expert=[2778, 2379, 1105, 122, 24, 0, 0, 279, 337, 1466, 1, 645, 88, 2307, 437, 62, 1010, 185, 117, 406, 730, 233, 652, 856, 1128, 589, 2, 946, 1, 1480, 3, 4125, 736, 2045, 1049, 84, 2306, 0, 848, 758, 12, 906, 365, 1032, 1, 596, 216, 2146, 715, 3216, 157, 354, 1016, 280, 389, 467, 1168, 8, 364, 771, 611, 1698, 211, 134] payload_rows_per_dst=[7812, 8738, 7030, 9380, 10662, 8816, 10666, 12318] dup_rows_per_dst=[804, 1948, 1348, 3314, 4990, 1732, 2884, 5862] combine_rows_per_src=[7812, 8738, 7030, 9380, 10662, 8816, 10666, 12318] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76312 payload_rows_remote=63800 dup_rows=21992 dup_rows_remote=17154 dedup_ratio=0.223714 router_tokens_per_expert=[2247, 2852, 1480, 101, 44, 0, 0, 363, 324, 1605, 2, 679, 132, 1898, 407, 64, 1046, 242, 42, 505, 696, 270, 711, 1105, 1345, 588, 3, 1360, 4, 1611, 10, 3754, 516, 1665, 924, 107, 2466, 0, 963, 402, 25, 941, 419, 1171, 0, 484, 194, 1975, 449, 3110, 259, 353, 1008, 275, 703, 543, 1036, 17, 283, 799, 595, 1542, 316, 122] payload_rows_per_dst=[10188, 10798, 7778, 12512, 8106, 8652, 10376, 7902] dup_rows_per_dst=[3986, 3058, 1456, 4838, 2346, 1766, 3024, 1518] combine_rows_per_src=[10188, 10798, 7778, 12512, 8106, 8652, 10376, 7902] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=342584 combine_payload_bytes=265756672 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266099256 ingress_bytes=369565696 link_bytes=369565696 token_bytes=531855928 weight_movement_bytes=103809024 total_nvlink_bytes=635664952 weight_to_token_ratio=0.195183 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=19072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390645962e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=264044544 dispatch_meta_bytes=335568 combine_payload_bytes=264044544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264380112 ingress_bytes=298647552 link_bytes=298647552 token_bytes=528424656 weight_movement_bytes=34603008 total_nvlink_bytes=563027664 weight_to_token_ratio=0.065483 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123786695e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265338880 dispatch_meta_bytes=335992 combine_payload_bytes=265338880 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265674872 ingress_bytes=334544896 link_bytes=334544896 token_bytes=531013752 weight_movement_bytes=69206016 total_nvlink_bytes=600219768 weight_to_token_ratio=0.130328 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=13312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258865511e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73576 payload_rows_remote=63474 dup_rows=24728 dup_rows_remote=20638 dedup_ratio=0.251546 router_tokens_per_expert=[1971, 2922, 1707, 85, 44, 0, 0, 367, 296, 1822, 0, 682, 148, 1597, 382, 40, 896, 474, 24, 664, 810, 260, 631, 1472, 1124, 624, 4, 1289, 5, 1557, 4, 3493, 451, 1313, 614, 232, 2985, 0, 1090, 457, 13, 1298, 424, 1392, 0, 397, 210, 1697, 478, 2917, 313, 308, 1058, 321, 865, 559, 875, 6, 330, 793, 737, 1119, 379, 127] payload_rows_per_dst=[10102, 11584, 11912, 7558, 8008, 10686, 6410, 7316] dup_rows_per_dst=[4090, 4320, 5536, 1656, 2674, 3642, 1394, 1416] combine_rows_per_src=[10102, 11584, 11912, 7558, 8008, 10686, 6410, 7316] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=245415936 dispatch_meta_bytes=318248 combine_payload_bytes=245415936 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=245734184 ingress_bytes=314621952 link_bytes=314621952 token_bytes=491150120 weight_movement_bytes=69206016 total_nvlink_bytes=560356136 weight_to_token_ratio=0.140906 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=13312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.183897076e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=261324800 dispatch_meta_bytes=323816 combine_payload_bytes=261324800 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=261648616 ingress_bytes=365133824 link_bytes=365133824 token_bytes=522973416 weight_movement_bytes=103809024 total_nvlink_bytes=626782440 weight_to_token_ratio=0.198498 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.373969184e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265232384 dispatch_meta_bytes=337224 combine_payload_bytes=265232384 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265569608 ingress_bytes=299835392 link_bytes=299835392 token_bytes=530801992 weight_movement_bytes=34603008 total_nvlink_bytes=565405000 weight_to_token_ratio=0.065190 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=31872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128256441e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=258473984 dispatch_meta_bytes=320496 combine_payload_bytes=258473984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258794480 ingress_bytes=293076992 link_bytes=293076992 token_bytes=517268464 weight_movement_bytes=34603008 total_nvlink_bytes=551871472 weight_to_token_ratio=0.066896 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.102825126e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75202 payload_rows_remote=64780 dup_rows=23102 dup_rows_remote=19218 dedup_ratio=0.235006 router_tokens_per_expert=[1505, 2909, 1448, 141, 296, 7, 0, 259, 246, 1565, 6, 673, 108, 1277, 406, 77, 857, 374, 19, 595, 684, 367, 749, 1497, 1305, 623, 22, 1197, 9, 1629, 11, 3437, 551, 1670, 439, 201, 2913, 12, 846, 521, 11, 1600, 450, 1276, 3, 550, 338, 2161, 630, 2470, 367, 356, 1072, 349, 819, 620, 843, 39, 419, 866, 563, 1347, 419, 133] payload_rows_per_dst=[9662, 7334, 8450, 11320, 10422, 10040, 9966, 8008] dup_rows_per_dst=[3468, 1382, 1834, 4282, 3884, 2738, 3400, 2114] combine_rows_per_src=[9662, 7334, 8450, 11320, 10422, 10040, 9966, 8008] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=261324800 dispatch_meta_bytes=323816 combine_payload_bytes=261324800 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=261648616 ingress_bytes=365133824 link_bytes=365133824 token_bytes=522973416 weight_movement_bytes=103809024 total_nvlink_bytes=626782440 weight_to_token_ratio=0.198498 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.373969184e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74610 payload_rows_remote=65820 dup_rows=23694 dup_rows_remote=21452 dedup_ratio=0.241028 router_tokens_per_expert=[1995, 2736, 1267, 81, 51, 0, 0, 318, 292, 1640, 0, 643, 123, 1788, 394, 40, 845, 446, 39, 625, 745, 250, 848, 1480, 919, 653, 8, 1090, 2, 1721, 1, 3709, 513, 1616, 636, 145, 2712, 0, 1031, 720, 11, 1172, 387, 1727, 0, 470, 229, 1520, 442, 3225, 271, 350, 1083, 249, 711, 496, 891, 10, 346, 918, 552, 1309, 535, 126] payload_rows_per_dst=[9662, 8070, 8540, 7284, 10294, 8790, 10518, 11452] dup_rows_per_dst=[3234, 1770, 2016, 1504, 4452, 2242, 3136, 5340] combine_rows_per_src=[9662, 8070, 8540, 7284, 10294, 8790, 10518, 11452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75332 payload_rows_remote=62930 dup_rows=22972 dup_rows_remote=17038 dedup_ratio=0.233683 router_tokens_per_expert=[127, 2103, 1515, 1634, 1263, 1831, 28, 112, 485, 647, 56, 1262, 3506, 158, 28, 2658, 762, 1, 286, 987, 37, 34, 334, 833, 29, 1488, 581, 82, 55, 0, 265, 1371, 448, 58, 24, 1309, 135, 1, 1021, 13, 4, 223, 2, 169, 2630, 1853, 93, 1403, 873, 509, 590, 603, 93, 3, 1614, 433, 288, 3100, 578, 1897, 2766, 1076, 650, 135] payload_rows_per_dst=[12402, 10404, 9996, 6934, 8012, 11220, 11396, 4968] dup_rows_per_dst=[5934, 2456, 2752, 808, 2212, 4752, 3572, 486] combine_rows_per_src=[12402, 10404, 9996, 6934, 8012, 11220, 11396, 4968] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76302 payload_rows_remote=68528 dup_rows=22002 dup_rows_remote=20972 dedup_ratio=0.223816 router_tokens_per_expert=[102, 1874, 1526, 1669, 1002, 2142, 83, 47, 392, 529, 97, 1296, 3578, 127, 139, 2214, 703, 39, 244, 685, 104, 21, 703, 711, 27, 1177, 546, 107, 69, 0, 421, 2246, 520, 169, 51, 1458, 114, 0, 1073, 17, 9, 282, 7, 110, 2012, 1498, 84, 1194, 1026, 489, 510, 313, 212, 8, 1355, 489, 427, 3531, 501, 1918, 3042, 1141, 842, 130] payload_rows_per_dst=[7872, 7816, 9776, 11780, 10930, 8344, 7774, 12010] dup_rows_per_dst=[986, 1772, 2884, 4562, 3906, 2048, 1030, 4814] combine_rows_per_src=[7872, 7816, 9776, 11780, 10930, 8344, 7774, 12010] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76354 payload_rows_remote=66022 dup_rows=21950 dup_rows_remote=19570 dedup_ratio=0.223287 router_tokens_per_expert=[133, 1255, 1606, 1942, 738, 1878, 23, 156, 383, 780, 97, 1066, 3550, 352, 73, 1903, 769, 6, 403, 677, 24, 36, 343, 1301, 147, 1256, 625, 158, 118, 158, 301, 2101, 463, 433, 145, 1725, 195, 83, 609, 4, 3, 363, 7, 137, 2036, 1916, 115, 1127, 1388, 513, 468, 255, 354, 17, 1964, 406, 472, 3110, 570, 1633, 2499, 786, 785, 213] payload_rows_per_dst=[11854, 10332, 10128, 8490, 8000, 10362, 11682, 5506] dup_rows_per_dst=[4794, 2380, 3210, 1238, 1824, 3890, 4046, 568] combine_rows_per_src=[11854, 10332, 10128, 8490, 8000, 10362, 11682, 5506] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=259989504 dispatch_meta_bytes=336448 combine_payload_bytes=259989504 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=260325952 ingress_bytes=329195520 link_bytes=329195520 token_bytes=520315456 weight_movement_bytes=69206016 total_nvlink_bytes=589521472 weight_to_token_ratio=0.133008 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.238736239e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78078 payload_rows_remote=67396 dup_rows=20226 dup_rows_remote=16842 dedup_ratio=0.205750 router_tokens_per_expert=[106, 2553, 1567, 1495, 1443, 2121, 82, 97, 361, 474, 68, 1396, 3364, 87, 103, 2611, 720, 24, 467, 963, 98, 42, 752, 773, 24, 1591, 565, 96, 53, 4, 324, 1783, 632, 63, 41, 1287, 143, 1, 1017, 12, 21, 201, 7, 146, 2337, 1805, 109, 987, 874, 424, 472, 306, 282, 15, 1057, 342, 447, 3194, 518, 1593, 2795, 1020, 655, 142] payload_rows_per_dst=[11264, 12192, 10682, 7900, 8772, 8946, 10768, 7554] dup_rows_per_dst=[2558, 4736, 3384, 980, 2726, 2280, 2366, 1196] combine_rows_per_src=[11264, 12192, 10682, 7900, 8772, 8946, 10768, 7554] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76672 payload_rows_remote=67354 dup_rows=21632 dup_rows_remote=19340 dedup_ratio=0.220052 router_tokens_per_expert=[113, 2743, 1419, 1777, 1000, 1901, 60, 76, 489, 520, 81, 1508, 3383, 122, 36, 2415, 576, 3, 321, 702, 26, 38, 503, 806, 35, 1600, 596, 149, 69, 0, 503, 1848, 391, 336, 48, 1191, 132, 1, 864, 36, 1, 241, 4, 176, 2356, 1677, 67, 1283, 1090, 490, 562, 727, 85, 4, 1536, 356, 287, 2938, 522, 1953, 2398, 1068, 738, 176] payload_rows_per_dst=[7600, 8240, 9436, 11668, 11368, 9318, 8318, 10724] dup_rows_per_dst=[1290, 2102, 2390, 4698, 3918, 2292, 1382, 3560] combine_rows_per_src=[7600, 8240, 9436, 11668, 11368, 9318, 8318, 10724] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265338880 dispatch_meta_bytes=335992 combine_payload_bytes=265338880 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265674872 ingress_bytes=334544896 link_bytes=334544896 token_bytes=531013752 weight_movement_bytes=69206016 total_nvlink_bytes=600219768 weight_to_token_ratio=0.130328 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=13312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258865511e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=270426112 dispatch_meta_bytes=342368 combine_payload_bytes=270426112 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=270768480 ingress_bytes=374235136 link_bytes=374235136 token_bytes=541194592 weight_movement_bytes=103809024 total_nvlink_bytes=645003616 weight_to_token_ratio=0.191815 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.408216688e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=276054016 dispatch_meta_bytes=336952 combine_payload_bytes=276054016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276390968 ingress_bytes=310657024 link_bytes=310657024 token_bytes=552444984 weight_movement_bytes=34603008 total_nvlink_bytes=587047992 weight_to_token_ratio=0.062636 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22656 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.168977371e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74726 payload_rows_remote=64882 dup_rows=23578 dup_rows_remote=20764 dedup_ratio=0.239848 router_tokens_per_expert=[2103, 2586, 1145, 58, 124, 3, 0, 315, 202, 1510, 0, 686, 93, 1518, 375, 73, 958, 274, 20, 563, 695, 238, 788, 1340, 1321, 629, 2, 979, 1, 1661, 6, 3655, 444, 1750, 728, 157, 3296, 3, 859, 815, 19, 1671, 341, 1460, 1, 518, 234, 1837, 540, 2748, 199, 318, 835, 271, 869, 549, 990, 4, 487, 842, 469, 1415, 449, 113] payload_rows_per_dst=[7954, 7410, 8066, 9382, 11066, 9600, 9844, 11404] dup_rows_per_dst=[1520, 1504, 1686, 3010, 5038, 2562, 2814, 5444] combine_rows_per_src=[7954, 7410, 8066, 9382, 11066, 9600, 9844, 11404] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78636 payload_rows_remote=70324 dup_rows=19668 dup_rows_remote=18638 dedup_ratio=0.200073 router_tokens_per_expert=[116, 1955, 1501, 1369, 919, 2001, 52, 79, 405, 696, 103, 1168, 3172, 228, 35, 2106, 800, 3, 238, 613, 136, 19, 400, 976, 44, 1398, 561, 96, 67, 12, 304, 2189, 409, 326, 48, 1568, 147, 2, 909, 6, 1, 293, 6, 177, 2293, 1830, 100, 1022, 915, 469, 446, 399, 109, 6, 1787, 469, 462, 3512, 498, 2351, 3008, 961, 755, 107] payload_rows_per_dst=[10544, 11470, 10126, 8312, 8570, 9044, 11382, 9188] dup_rows_per_dst=[1530, 4356, 3268, 1030, 2170, 2400, 3422, 1492] combine_rows_per_src=[10544, 11470, 10126, 8312, 8570, 9044, 11382, 9188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=275881984 dispatch_meta_bytes=346776 combine_payload_bytes=275881984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276228760 ingress_bytes=310484992 link_bytes=310484992 token_bytes=552110744 weight_movement_bytes=34603008 total_nvlink_bytes=586713752 weight_to_token_ratio=0.062674 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.168330029e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77320 payload_rows_remote=68138 dup_rows=20984 dup_rows_remote=18374 dedup_ratio=0.213460 router_tokens_per_expert=[119, 1884, 1478, 1693, 1124, 1884, 50, 122, 392, 625, 86, 1335, 3064, 135, 166, 2175, 671, 93, 458, 490, 108, 49, 413, 764, 62, 1528, 552, 149, 32, 102, 301, 1744, 496, 320, 36, 1643, 137, 3, 968, 26, 5, 299, 16, 248, 2455, 1917, 105, 982, 1004, 530, 506, 496, 139, 10, 1551, 474, 239, 3645, 509, 1786, 2938, 955, 672, 194] payload_rows_per_dst=[10194, 8316, 10142, 10642, 9182, 9516, 8210, 11118] dup_rows_per_dst=[1980, 2108, 3240, 3830, 2610, 2538, 1210, 3468] combine_rows_per_src=[10194, 8316, 10142, 10642, 9182, 9516, 8210, 11118] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=269598720 dispatch_meta_bytes=349088 combine_payload_bytes=269598720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269947808 ingress_bytes=304201728 link_bytes=304201728 token_bytes=539546528 weight_movement_bytes=34603008 total_nvlink_bytes=574149536 weight_to_token_ratio=0.064134 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.144686612e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=257761280 dispatch_meta_bytes=319872 combine_payload_bytes=257761280 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=258081152 ingress_bytes=361570304 link_bytes=361570304 token_bytes=515842432 weight_movement_bytes=103809024 total_nvlink_bytes=619651456 weight_to_token_ratio=0.201242 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=13440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.360559945e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75032 payload_rows_remote=64640 dup_rows=23272 dup_rows_remote=19296 dedup_ratio=0.236735 router_tokens_per_expert=[1467, 2644, 1500, 86, 202, 0, 1, 345, 246, 1811, 0, 751, 154, 1304, 424, 52, 944, 471, 12, 879, 627, 355, 690, 1508, 1263, 614, 18, 978, 3, 1743, 10, 3556, 334, 1416, 402, 297, 3246, 0, 908, 522, 4, 1869, 389, 1790, 0, 401, 248, 1823, 377, 2590, 236, 287, 962, 284, 876, 614, 1030, 9, 329, 886, 453, 1336, 521, 55] payload_rows_per_dst=[9388, 11694, 10392, 9736, 6466, 10246, 9636, 7474] dup_rows_per_dst=[3102, 4282, 3976, 3238, 1292, 2802, 2816, 1764] combine_rows_per_src=[9388, 11694, 10392, 9736, 6466, 10246, 9636, 7474] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76102 payload_rows_remote=62202 dup_rows=22202 dup_rows_remote=14450 dedup_ratio=0.225850 router_tokens_per_expert=[110, 1612, 1462, 1792, 264, 1849, 9, 114, 455, 803, 154, 1105, 2958, 332, 54, 1909, 698, 13, 263, 804, 67, 21, 278, 1301, 95, 1010, 638, 100, 154, 94, 398, 2269, 326, 382, 152, 1896, 123, 26, 681, 5, 3, 452, 5, 128, 2417, 1542, 106, 1024, 1361, 436, 481, 257, 234, 4, 2659, 441, 478, 3097, 534, 1875, 3053, 756, 894, 139] payload_rows_per_dst=[6914, 7814, 6366, 11480, 10692, 9100, 9836, 13900] dup_rows_per_dst=[588, 1810, 524, 3952, 3412, 2254, 1910, 7752] combine_rows_per_src=[6914, 7814, 6366, 11480, 10692, 9100, 9836, 13900] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75332 payload_rows_remote=62930 dup_rows=22972 dup_rows_remote=17038 dedup_ratio=0.233683 router_tokens_per_expert=[127, 2103, 1515, 1634, 1263, 1831, 28, 112, 485, 647, 56, 1262, 3506, 158, 28, 2658, 762, 1, 286, 987, 37, 34, 334, 833, 29, 1488, 581, 82, 55, 0, 265, 1371, 448, 58, 24, 1309, 135, 1, 1021, 13, 4, 223, 2, 169, 2630, 1853, 93, 1403, 873, 509, 590, 603, 93, 3, 1614, 433, 288, 3100, 578, 1897, 2766, 1076, 650, 135] payload_rows_per_dst=[12402, 10404, 9996, 6934, 8012, 11220, 11396, 4968] dup_rows_per_dst=[5934, 2456, 2752, 808, 2212, 4752, 3572, 486] combine_rows_per_src=[12402, 10404, 9996, 6934, 8012, 11220, 11396, 4968] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78636 payload_rows_remote=70324 dup_rows=19668 dup_rows_remote=18638 dedup_ratio=0.200073 router_tokens_per_expert=[116, 1955, 1501, 1369, 919, 2001, 52, 79, 405, 696, 103, 1168, 3172, 228, 35, 2106, 800, 3, 238, 613, 136, 19, 400, 976, 44, 1398, 561, 96, 67, 12, 304, 2189, 409, 326, 48, 1568, 147, 2, 909, 6, 1, 293, 6, 177, 2293, 1830, 100, 1022, 915, 469, 446, 399, 109, 6, 1787, 469, 462, 3512, 498, 2351, 3008, 961, 755, 107] payload_rows_per_dst=[10544, 11470, 10126, 8312, 8570, 9044, 11382, 9188] dup_rows_per_dst=[1530, 4356, 3268, 1030, 2170, 2400, 3422, 1492] combine_rows_per_src=[10544, 11470, 10126, 8312, 8570, 9044, 11382, 9188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279093248 dispatch_meta_bytes=346048 combine_payload_bytes=279093248 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=279439296 ingress_bytes=348299264 link_bytes=348299264 token_bytes=558532544 weight_movement_bytes=69206016 total_nvlink_bytes=627738560 weight_to_token_ratio=0.123907 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.030000 gemm_active_groups=11 gemm_peak_group_rows=16000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.310622090e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280690688 dispatch_meta_bytes=358000 combine_payload_bytes=280690688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281048688 ingress_bytes=315293696 link_bytes=315293696 token_bytes=561739376 weight_movement_bytes=34603008 total_nvlink_bytes=596342384 weight_to_token_ratio=0.061600 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.186424795e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=270426112 dispatch_meta_bytes=342368 combine_payload_bytes=270426112 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=270768480 ingress_bytes=374235136 link_bytes=374235136 token_bytes=541194592 weight_movement_bytes=103809024 total_nvlink_bytes=645003616 weight_to_token_ratio=0.191815 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.408216688e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=342584 combine_payload_bytes=265756672 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266099256 ingress_bytes=369565696 link_bytes=369565696 token_bytes=531855928 weight_movement_bytes=103809024 total_nvlink_bytes=635664952 weight_to_token_ratio=0.195183 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=19072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390645962e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=288047104 dispatch_meta_bytes=355848 combine_payload_bytes=288047104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288402952 ingress_bytes=322650112 link_bytes=322650112 token_bytes=576450056 weight_movement_bytes=34603008 total_nvlink_bytes=611053064 weight_to_token_ratio=0.060028 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.214106396e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77320 payload_rows_remote=68138 dup_rows=20984 dup_rows_remote=18374 dedup_ratio=0.213460 router_tokens_per_expert=[119, 1884, 1478, 1693, 1124, 1884, 50, 122, 392, 625, 86, 1335, 3064, 135, 166, 2175, 671, 93, 458, 490, 108, 49, 413, 764, 62, 1528, 552, 149, 32, 102, 301, 1744, 496, 320, 36, 1643, 137, 3, 968, 26, 5, 299, 16, 248, 2455, 1917, 105, 982, 1004, 530, 506, 496, 139, 10, 1551, 474, 239, 3645, 509, 1786, 2938, 955, 672, 194] payload_rows_per_dst=[10194, 8316, 10142, 10642, 9182, 9516, 8210, 11118] dup_rows_per_dst=[1980, 2108, 3240, 3830, 2610, 2538, 1210, 3468] combine_rows_per_src=[10194, 8316, 10142, 10642, 9182, 9516, 8210, 11118] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=264765440 dispatch_meta_bytes=335744 combine_payload_bytes=264765440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265101184 ingress_bytes=299368448 link_bytes=299368448 token_bytes=529866624 weight_movement_bytes=34603008 total_nvlink_bytes=564469632 weight_to_token_ratio=0.065305 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126499369e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=254779392 dispatch_meta_bytes=306608 combine_payload_bytes=254779392 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=255086000 ingress_bytes=393191424 link_bytes=393191424 token_bytes=509865392 weight_movement_bytes=138412032 total_nvlink_bytes=648277424 weight_to_token_ratio=0.271468 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=11776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.479547674e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=288047104 dispatch_meta_bytes=355848 combine_payload_bytes=288047104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288402952 ingress_bytes=322650112 link_bytes=322650112 token_bytes=576450056 weight_movement_bytes=34603008 total_nvlink_bytes=611053064 weight_to_token_ratio=0.060028 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.214106396e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76672 payload_rows_remote=67354 dup_rows=21632 dup_rows_remote=19340 dedup_ratio=0.220052 router_tokens_per_expert=[113, 2743, 1419, 1777, 1000, 1901, 60, 76, 489, 520, 81, 1508, 3383, 122, 36, 2415, 576, 3, 321, 702, 26, 38, 503, 806, 35, 1600, 596, 149, 69, 0, 503, 1848, 391, 336, 48, 1191, 132, 1, 864, 36, 1, 241, 4, 176, 2356, 1677, 67, 1283, 1090, 490, 562, 727, 85, 4, 1536, 356, 287, 2938, 522, 1953, 2398, 1068, 738, 176] payload_rows_per_dst=[7600, 8240, 9436, 11668, 11368, 9318, 8318, 10724] dup_rows_per_dst=[1290, 2102, 2390, 4698, 3918, 2292, 1382, 3560] combine_rows_per_src=[7600, 8240, 9436, 11668, 11368, 9318, 8318, 10724] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77334 payload_rows_remote=66114 dup_rows=20970 dup_rows_remote=18162 dedup_ratio=0.213318 router_tokens_per_expert=[520, 983, 472, 4, 108, 38, 455, 4110, 846, 5, 694, 138, 381, 679, 5241, 137, 6, 3, 26, 1951, 168, 64, 504, 45, 16, 676, 1480, 1, 1227, 446, 42, 2471, 207, 177, 997, 18, 1810, 203, 1, 32, 9, 1735, 910, 244, 27, 305, 126, 477, 1558, 243, 71, 23, 2483, 53, 9, 1680, 233, 470, 10, 3859, 449, 1520, 3073, 2203] payload_rows_per_dst=[11220, 12876, 10206, 12070, 10048, 5990, 6286, 8638] dup_rows_per_dst=[2808, 3366, 3046, 4814, 2988, 1676, 1140, 1132] combine_rows_per_src=[11220, 12876, 10206, 12070, 10048, 5990, 6286, 8638] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76778 payload_rows_remote=66642 dup_rows=21526 dup_rows_remote=18422 dedup_ratio=0.218974 router_tokens_per_expert=[525, 813, 408, 8, 147, 56, 578, 3884, 1209, 3, 918, 147, 438, 681, 3958, 328, 13, 6, 29, 2659, 191, 5, 427, 61, 43, 704, 1699, 0, 1206, 471, 54, 2407, 183, 190, 1290, 32, 1771, 219, 6, 76, 99, 1865, 1273, 233, 107, 361, 113, 542, 1357, 325, 264, 47, 2181, 91, 14, 1446, 311, 465, 31, 3838, 466, 1190, 2853, 1837] payload_rows_per_dst=[10626, 6122, 10672, 10344, 10136, 12382, 8650, 7846] dup_rows_per_dst=[2212, 1326, 3786, 2824, 3104, 4720, 2800, 754] combine_rows_per_src=[10626, 6122, 10672, 10344, 10136, 12382, 8650, 7846] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76482 payload_rows_remote=66944 dup_rows=21822 dup_rows_remote=18626 dedup_ratio=0.221985 router_tokens_per_expert=[349, 867, 339, 11, 83, 52, 469, 4178, 985, 7, 1070, 115, 491, 878, 5027, 202, 14, 6, 47, 2189, 104, 49, 452, 62, 33, 615, 2177, 0, 1017, 425, 58, 2398, 190, 116, 1370, 24, 1704, 238, 0, 39, 13, 1335, 981, 319, 16, 236, 163, 472, 1330, 299, 84, 25, 2550, 125, 20, 1934, 259, 759, 21, 3273, 435, 1365, 2903, 1785] payload_rows_per_dst=[10738, 6194, 9606, 7730, 7912, 12862, 9538, 11902] dup_rows_per_dst=[1958, 1302, 2786, 920, 1658, 4262, 3196, 5740] combine_rows_per_src=[10738, 6194, 9606, 7730, 7912, 12862, 9538, 11902] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=275881984 dispatch_meta_bytes=346776 combine_payload_bytes=275881984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276228760 ingress_bytes=310484992 link_bytes=310484992 token_bytes=552110744 weight_movement_bytes=34603008 total_nvlink_bytes=586713752 weight_to_token_ratio=0.062674 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.168330029e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78336 payload_rows_remote=65980 dup_rows=19968 dup_rows_remote=17804 dedup_ratio=0.203125 router_tokens_per_expert=[457, 1302, 188, 2, 172, 118, 742, 3455, 1289, 17, 233, 150, 328, 20, 5071, 152, 7, 22, 25, 1530, 276, 8, 247, 20, 22, 874, 1634, 0, 1407, 432, 62, 2677, 372, 285, 613, 19, 2506, 593, 3, 176, 4, 1086, 1748, 145, 40, 328, 53, 371, 1347, 403, 112, 9, 2100, 43, 25, 3110, 170, 779, 22, 3479, 399, 1590, 2247, 2036] payload_rows_per_dst=[10434, 12356, 9138, 10894, 10278, 6164, 9998, 9074] dup_rows_per_dst=[2438, 2164, 2090, 3322, 3350, 1386, 4300, 918] combine_rows_per_src=[10434, 12356, 9138, 10894, 10278, 6164, 9998, 9074] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=257761280 dispatch_meta_bytes=319872 combine_payload_bytes=257761280 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=258081152 ingress_bytes=361570304 link_bytes=361570304 token_bytes=515842432 weight_movement_bytes=103809024 total_nvlink_bytes=619651456 weight_to_token_ratio=0.201242 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=13440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.360559945e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=270802944 dispatch_meta_bytes=337104 combine_payload_bytes=270802944 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=271140048 ingress_bytes=340008960 link_bytes=340008960 token_bytes=541942992 weight_movement_bytes=69206016 total_nvlink_bytes=611149008 weight_to_token_ratio=0.127700 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=27648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.279426344e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77536 payload_rows_remote=67872 dup_rows=20768 dup_rows_remote=17642 dedup_ratio=0.211263 router_tokens_per_expert=[618, 635, 887, 4, 113, 44, 439, 3677, 564, 9, 1201, 120, 573, 1184, 5217, 281, 3, 6, 51, 2391, 107, 12, 485, 80, 28, 735, 2223, 0, 1037, 223, 46, 2032, 214, 128, 1401, 39, 1578, 114, 1, 17, 49, 1782, 484, 252, 51, 426, 179, 598, 1298, 249, 153, 25, 2838, 91, 19, 817, 234, 1115, 18, 3260, 390, 1106, 3506, 1695] payload_rows_per_dst=[10594, 13516, 9664, 9754, 10762, 6114, 8836, 8296] dup_rows_per_dst=[2240, 4782, 3126, 2894, 3234, 1528, 2144, 820] combine_rows_per_src=[10594, 13516, 9664, 9754, 10762, 6114, 8836, 8296] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77656 payload_rows_remote=64860 dup_rows=20648 dup_rows_remote=16340 dedup_ratio=0.210042 router_tokens_per_expert=[523, 1038, 361, 9, 76, 46, 314, 4077, 910, 12, 961, 136, 407, 947, 5018, 194, 10, 2, 44, 2084, 104, 5, 493, 68, 15, 846, 1680, 0, 1279, 356, 54, 2150, 179, 119, 1277, 55, 1809, 159, 0, 34, 30, 1576, 775, 185, 26, 303, 105, 534, 1566, 252, 138, 16, 2579, 248, 12, 1269, 266, 482, 10, 3693, 434, 1286, 3393, 2123] payload_rows_per_dst=[10750, 6142, 9936, 7936, 10752, 12796, 9062, 10282] dup_rows_per_dst=[2138, 992, 3070, 1112, 3298, 4308, 3098, 2632] combine_rows_per_src=[10750, 6142, 9936, 7936, 10752, 12796, 9062, 10282] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279093248 dispatch_meta_bytes=346048 combine_payload_bytes=279093248 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=279439296 ingress_bytes=348299264 link_bytes=348299264 token_bytes=558532544 weight_movement_bytes=69206016 total_nvlink_bytes=627738560 weight_to_token_ratio=0.123907 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.030000 gemm_active_groups=11 gemm_peak_group_rows=16000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.310622090e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=270254080 dispatch_meta_bytes=335136 combine_payload_bytes=270254080 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=270589216 ingress_bytes=339460096 link_bytes=339460096 token_bytes=540843296 weight_movement_bytes=69206016 total_nvlink_bytes=610049312 weight_to_token_ratio=0.127959 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20096 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.277361012e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76302 payload_rows_remote=68528 dup_rows=22002 dup_rows_remote=20972 dedup_ratio=0.223816 router_tokens_per_expert=[102, 1874, 1526, 1669, 1002, 2142, 83, 47, 392, 529, 97, 1296, 3578, 127, 139, 2214, 703, 39, 244, 685, 104, 21, 703, 711, 27, 1177, 546, 107, 69, 0, 421, 2246, 520, 169, 51, 1458, 114, 0, 1073, 17, 9, 282, 7, 110, 2012, 1498, 84, 1194, 1026, 489, 510, 313, 212, 8, 1355, 489, 427, 3531, 501, 1918, 3042, 1141, 842, 130] payload_rows_per_dst=[7872, 7816, 9776, 11780, 10930, 8344, 7774, 12010] dup_rows_per_dst=[986, 1772, 2884, 4562, 3906, 2048, 1030, 4814] combine_rows_per_src=[7872, 7816, 9776, 11780, 10930, 8344, 7774, 12010] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278003712 dispatch_meta_bytes=342056 combine_payload_bytes=278003712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278345768 ingress_bytes=312606720 link_bytes=312606720 token_bytes=556349480 weight_movement_bytes=34603008 total_nvlink_bytes=590952488 weight_to_token_ratio=0.062197 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=28032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.176313920e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265666560 dispatch_meta_bytes=324800 combine_payload_bytes=265666560 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265991360 ingress_bytes=300269568 link_bytes=300269568 token_bytes=531657920 weight_movement_bytes=34603008 total_nvlink_bytes=566260928 weight_to_token_ratio=0.065085 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=19328 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129890211e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77662 payload_rows_remote=66024 dup_rows=20642 dup_rows_remote=17014 dedup_ratio=0.209981 router_tokens_per_expert=[796, 1349, 346, 19, 122, 59, 411, 4172, 1312, 16, 383, 152, 460, 336, 4659, 99, 8, 9, 38, 1563, 309, 2, 470, 81, 40, 773, 1717, 0, 1739, 424, 37, 2903, 211, 166, 955, 25, 1871, 330, 1, 116, 5, 1224, 946, 228, 24, 256, 130, 458, 1270, 272, 84, 23, 2299, 104, 12, 2473, 199, 353, 11, 3687, 380, 1553, 2686, 1996] payload_rows_per_dst=[11672, 11816, 9632, 11638, 9832, 5510, 9450, 8112] dup_rows_per_dst=[2876, 2622, 2702, 3628, 2890, 1428, 3624, 872] combine_rows_per_src=[11672, 11816, 9632, 11638, 9832, 5510, 9450, 8112] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78078 payload_rows_remote=67396 dup_rows=20226 dup_rows_remote=16842 dedup_ratio=0.205750 router_tokens_per_expert=[106, 2553, 1567, 1495, 1443, 2121, 82, 97, 361, 474, 68, 1396, 3364, 87, 103, 2611, 720, 24, 467, 963, 98, 42, 752, 773, 24, 1591, 565, 96, 53, 4, 324, 1783, 632, 63, 41, 1287, 143, 1, 1017, 12, 21, 201, 7, 146, 2337, 1805, 109, 987, 874, 424, 472, 306, 282, 15, 1057, 342, 447, 3194, 518, 1593, 2795, 1020, 655, 142] payload_rows_per_dst=[11264, 12192, 10682, 7900, 8772, 8946, 10768, 7554] dup_rows_per_dst=[2558, 4736, 3384, 980, 2726, 2280, 2366, 1196] combine_rows_per_src=[11264, 12192, 10682, 7900, 8772, 8946, 10768, 7554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=272965632 dispatch_meta_bytes=340256 combine_payload_bytes=272965632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273305888 ingress_bytes=307568640 link_bytes=307568640 token_bytes=546271520 weight_movement_bytes=34603008 total_nvlink_bytes=580874528 weight_to_token_ratio=0.063344 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.157356031e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74736 payload_rows_remote=61448 dup_rows=23568 dup_rows_remote=15570 dedup_ratio=0.239746 router_tokens_per_expert=[589, 1114, 114, 6, 154, 154, 606, 4220, 1358, 41, 77, 154, 404, 77, 5080, 96, 4, 17, 25, 1128, 239, 55, 119, 14, 30, 941, 1387, 0, 1693, 123, 38, 2942, 364, 547, 255, 13, 2268, 421, 0, 163, 3, 961, 1733, 153, 49, 243, 100, 267, 1428, 264, 40, 10, 1814, 35, 3, 4497, 112, 368, 24, 3652, 373, 1711, 2200, 2082] payload_rows_per_dst=[5184, 10114, 7668, 7162, 7072, 12956, 11292, 13288] dup_rows_per_dst=[416, 2614, 1176, 1262, 990, 4222, 4890, 7998] combine_rows_per_src=[5184, 10114, 7668, 7162, 7072, 12956, 11292, 13288] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77334 payload_rows_remote=66114 dup_rows=20970 dup_rows_remote=18162 dedup_ratio=0.213318 router_tokens_per_expert=[520, 983, 472, 4, 108, 38, 455, 4110, 846, 5, 694, 138, 381, 679, 5241, 137, 6, 3, 26, 1951, 168, 64, 504, 45, 16, 676, 1480, 1, 1227, 446, 42, 2471, 207, 177, 997, 18, 1810, 203, 1, 32, 9, 1735, 910, 244, 27, 305, 126, 477, 1558, 243, 71, 23, 2483, 53, 9, 1680, 233, 470, 10, 3859, 449, 1520, 3073, 2203] payload_rows_per_dst=[11220, 12876, 10206, 12070, 10048, 5990, 6286, 8638] dup_rows_per_dst=[2808, 3366, 3046, 4814, 2988, 1676, 1140, 1132] combine_rows_per_src=[11220, 12876, 10206, 12070, 10048, 5990, 6286, 8638] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280690688 dispatch_meta_bytes=358000 combine_payload_bytes=280690688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281048688 ingress_bytes=315293696 link_bytes=315293696 token_bytes=561739376 weight_movement_bytes=34603008 total_nvlink_bytes=596342384 weight_to_token_ratio=0.061600 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.186424795e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77662 payload_rows_remote=66024 dup_rows=20642 dup_rows_remote=17014 dedup_ratio=0.209981 router_tokens_per_expert=[796, 1349, 346, 19, 122, 59, 411, 4172, 1312, 16, 383, 152, 460, 336, 4659, 99, 8, 9, 38, 1563, 309, 2, 470, 81, 40, 773, 1717, 0, 1739, 424, 37, 2903, 211, 166, 955, 25, 1871, 330, 1, 116, 5, 1224, 946, 228, 24, 256, 130, 458, 1270, 272, 84, 23, 2299, 104, 12, 2473, 199, 353, 11, 3687, 380, 1553, 2686, 1996] payload_rows_per_dst=[11672, 11816, 9632, 11638, 9832, 5510, 9450, 8112] dup_rows_per_dst=[2876, 2622, 2702, 3628, 2890, 1428, 3624, 872] combine_rows_per_src=[11672, 11816, 9632, 11638, 9832, 5510, 9450, 8112] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=274202624 dispatch_meta_bytes=342280 combine_payload_bytes=274202624 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=274544904 ingress_bytes=343408640 link_bytes=343408640 token_bytes=548747528 weight_movement_bytes=69206016 total_nvlink_bytes=617953544 weight_to_token_ratio=0.126116 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.292219066e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=270254080 dispatch_meta_bytes=335136 combine_payload_bytes=270254080 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=270589216 ingress_bytes=339460096 link_bytes=339460096 token_bytes=540843296 weight_movement_bytes=69206016 total_nvlink_bytes=610049312 weight_to_token_ratio=0.127959 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20096 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.277361012e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=270434304 dispatch_meta_bytes=332152 combine_payload_bytes=270434304 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=270766456 ingress_bytes=339640320 link_bytes=339640320 token_bytes=541200760 weight_movement_bytes=69206016 total_nvlink_bytes=610406776 weight_to_token_ratio=0.127875 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.278039181e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76778 payload_rows_remote=66642 dup_rows=21526 dup_rows_remote=18422 dedup_ratio=0.218974 router_tokens_per_expert=[525, 813, 408, 8, 147, 56, 578, 3884, 1209, 3, 918, 147, 438, 681, 3958, 328, 13, 6, 29, 2659, 191, 5, 427, 61, 43, 704, 1699, 0, 1206, 471, 54, 2407, 183, 190, 1290, 32, 1771, 219, 6, 76, 99, 1865, 1273, 233, 107, 361, 113, 542, 1357, 325, 264, 47, 2181, 91, 14, 1446, 311, 465, 31, 3838, 466, 1190, 2853, 1837] payload_rows_per_dst=[10626, 6122, 10672, 10344, 10136, 12382, 8650, 7846] dup_rows_per_dst=[2212, 1326, 3786, 2824, 3104, 4720, 2800, 754] combine_rows_per_src=[10626, 6122, 10672, 10344, 10136, 12382, 8650, 7846] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77088 payload_rows_remote=68598 dup_rows=21216 dup_rows_remote=19672 dedup_ratio=0.215820 router_tokens_per_expert=[257, 377, 2300, 28, 366, 3545, 16, 644, 179, 257, 229, 3206, 323, 2, 309, 512, 650, 656, 497, 227, 225, 214, 1, 1300, 278, 44, 181, 114, 1076, 327, 228, 0, 836, 476, 0, 97, 563, 36, 467, 5, 309, 0, 3883, 86, 52, 147, 526, 366, 46, 540, 51, 166, 2242, 2, 756, 1384, 4958, 1, 3912, 0, 6126, 1723, 604, 224] payload_rows_per_dst=[11124, 8490, 6298, 14312, 11818, 13020, 7172, 4854] dup_rows_per_dst=[3942, 1544, 1242, 2436, 3058, 5542, 3202, 250] combine_rows_per_src=[11124, 8490, 6298, 14312, 11818, 13020, 7172, 4854] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=276054016 dispatch_meta_bytes=336952 combine_payload_bytes=276054016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276390968 ingress_bytes=310657024 link_bytes=310657024 token_bytes=552444984 weight_movement_bytes=34603008 total_nvlink_bytes=587047992 weight_to_token_ratio=0.062636 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22656 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.168977371e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=251691008 dispatch_meta_bytes=308072 combine_payload_bytes=251691008 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=251999080 ingress_bytes=355500032 link_bytes=355500032 token_bytes=503690088 weight_movement_bytes=103809024 total_nvlink_bytes=607499112 weight_to_token_ratio=0.206097 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.337718000e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=270802944 dispatch_meta_bytes=337104 combine_payload_bytes=270802944 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=271140048 ingress_bytes=340008960 link_bytes=340008960 token_bytes=541942992 weight_movement_bytes=69206016 total_nvlink_bytes=611149008 weight_to_token_ratio=0.127700 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=27648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.279426344e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77628 payload_rows_remote=68786 dup_rows=20676 dup_rows_remote=18220 dedup_ratio=0.210327 router_tokens_per_expert=[248, 329, 365, 57, 593, 3453, 1, 1119, 180, 827, 501, 4790, 155, 26, 367, 887, 1375, 1207, 8, 112, 986, 353, 6, 1602, 248, 169, 12, 452, 1103, 511, 297, 2, 431, 74, 3, 125, 1397, 118, 1130, 11, 360, 3, 1901, 64, 197, 141, 1086, 20, 420, 302, 69, 195, 1939, 41, 1335, 1836, 4498, 0, 1198, 0, 5229, 1358, 1045, 285] payload_rows_per_dst=[9746, 12290, 8842, 13456, 11380, 8344, 8404, 5166] dup_rows_per_dst=[2584, 3176, 2456, 2590, 4194, 1596, 3870, 210] combine_rows_per_src=[9746, 12290, 8842, 13456, 11380, 8344, 8404, 5166] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76286 payload_rows_remote=67478 dup_rows=22018 dup_rows_remote=20074 dedup_ratio=0.223979 router_tokens_per_expert=[255, 337, 468, 127, 495, 3451, 1, 1207, 264, 688, 414, 4222, 136, 30, 367, 1011, 1141, 1473, 21, 166, 1158, 209, 3, 1690, 259, 135, 37, 419, 852, 686, 294, 18, 589, 187, 0, 208, 1324, 190, 996, 15, 343, 2, 2170, 68, 215, 112, 1117, 10, 548, 387, 26, 159, 1724, 14, 1192, 1816, 4231, 1, 1339, 0, 5219, 1255, 1439, 222] payload_rows_per_dst=[9940, 10692, 9624, 11854, 8176, 8808, 7974, 9218] dup_rows_per_dst=[2742, 2554, 3116, 2102, 2632, 1944, 3758, 3170] combine_rows_per_src=[9940, 10692, 9624, 11854, 8176, 8808, 7974, 9218] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=272965632 dispatch_meta_bytes=340256 combine_payload_bytes=272965632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273305888 ingress_bytes=307568640 link_bytes=307568640 token_bytes=546271520 weight_movement_bytes=34603008 total_nvlink_bytes=580874528 weight_to_token_ratio=0.063344 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.157356031e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=280977408 dispatch_meta_bytes=353080 combine_payload_bytes=280977408 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=281330488 ingress_bytes=350183424 link_bytes=350183424 token_bytes=562307896 weight_movement_bytes=69206016 total_nvlink_bytes=631513912 weight_to_token_ratio=0.123075 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=25216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.317712032e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=281747456 dispatch_meta_bytes=348024 combine_payload_bytes=281747456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282095480 ingress_bytes=316350464 link_bytes=316350464 token_bytes=563842936 weight_movement_bytes=34603008 total_nvlink_bytes=598445944 weight_to_token_ratio=0.061370 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.190401328e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=276389888 dispatch_meta_bytes=350208 combine_payload_bytes=276389888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276740096 ingress_bytes=310992896 link_bytes=310992896 token_bytes=553129984 weight_movement_bytes=34603008 total_nvlink_bytes=587732992 weight_to_token_ratio=0.062559 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20864 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.170241231e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76482 payload_rows_remote=66944 dup_rows=21822 dup_rows_remote=18626 dedup_ratio=0.221985 router_tokens_per_expert=[349, 867, 339, 11, 83, 52, 469, 4178, 985, 7, 1070, 115, 491, 878, 5027, 202, 14, 6, 47, 2189, 104, 49, 452, 62, 33, 615, 2177, 0, 1017, 425, 58, 2398, 190, 116, 1370, 24, 1704, 238, 0, 39, 13, 1335, 981, 319, 16, 236, 163, 472, 1330, 299, 84, 25, 2550, 125, 20, 1934, 259, 759, 21, 3273, 435, 1365, 2903, 1785] payload_rows_per_dst=[10738, 6194, 9606, 7730, 7912, 12862, 9538, 11902] dup_rows_per_dst=[1958, 1302, 2786, 920, 1658, 4262, 3196, 5740] combine_rows_per_src=[10738, 6194, 9606, 7730, 7912, 12862, 9538, 11902] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77490 payload_rows_remote=63664 dup_rows=20814 dup_rows_remote=18038 dedup_ratio=0.211731 router_tokens_per_expert=[269, 374, 863, 55, 499, 4048, 32, 1292, 190, 378, 318, 4562, 184, 7, 386, 897, 910, 876, 32, 175, 856, 309, 5, 1583, 258, 127, 92, 355, 1399, 421, 257, 1, 497, 328, 4, 168, 1165, 69, 802, 12, 360, 0, 2356, 86, 93, 110, 714, 47, 268, 435, 45, 198, 1777, 18, 1026, 1481, 4312, 0, 2699, 0, 5391, 1387, 1007, 287] payload_rows_per_dst=[11058, 11160, 7662, 13826, 11282, 10306, 7142, 5054] dup_rows_per_dst=[3806, 2684, 1830, 2776, 3432, 2624, 3354, 308] combine_rows_per_src=[11058, 11160, 7662, 13826, 11282, 10306, 7142, 5054] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=66752 payload_rows_remote=51302 dup_rows=31552 dup_rows_remote=11932 dedup_ratio=0.320964 router_tokens_per_expert=[253, 386, 2270, 53, 323, 4154, 14, 563, 205, 227, 154, 3806, 352, 0, 368, 681, 718, 745, 209, 172, 495, 269, 2, 1247, 213, 28, 175, 141, 1016, 380, 227, 0, 634, 659, 0, 127, 401, 66, 363, 3, 351, 4, 3587, 73, 22, 160, 356, 598, 105, 486, 11, 40, 1838, 8, 700, 1044, 4947, 0, 4197, 2, 6102, 1576, 655, 191] payload_rows_per_dst=[6450, 3198, 12042, 4200, 3956, 10254, 11202, 15450] dup_rows_per_dst=[1274, 776, 3284, 160, 550, 1458, 4430, 19620] combine_rows_per_src=[6450, 3198, 12042, 4200, 3956, 10254, 11202, 15450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=270434304 dispatch_meta_bytes=332152 combine_payload_bytes=270434304 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=270766456 ingress_bytes=339640320 link_bytes=339640320 token_bytes=541200760 weight_movement_bytes=69206016 total_nvlink_bytes=610406776 weight_to_token_ratio=0.127875 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.278039181e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77656 payload_rows_remote=64860 dup_rows=20648 dup_rows_remote=16340 dedup_ratio=0.210042 router_tokens_per_expert=[523, 1038, 361, 9, 76, 46, 314, 4077, 910, 12, 961, 136, 407, 947, 5018, 194, 10, 2, 44, 2084, 104, 5, 493, 68, 15, 846, 1680, 0, 1279, 356, 54, 2150, 179, 119, 1277, 55, 1809, 159, 0, 34, 30, 1576, 775, 185, 26, 303, 105, 534, 1566, 252, 138, 16, 2579, 248, 12, 1269, 266, 482, 10, 3693, 434, 1286, 3393, 2123] payload_rows_per_dst=[10750, 6142, 9936, 7936, 10752, 12796, 9062, 10282] dup_rows_per_dst=[2138, 992, 3070, 1112, 3298, 4308, 3098, 2632] combine_rows_per_src=[10750, 6142, 9936, 7936, 10752, 12796, 9062, 10282] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73854 payload_rows_remote=63840 dup_rows=24450 dup_rows_remote=21828 dedup_ratio=0.248718 router_tokens_per_expert=[242, 396, 667, 103, 410, 3600, 3, 897, 210, 487, 324, 4217, 170, 15, 344, 799, 1051, 1425, 18, 73, 924, 241, 3, 1805, 252, 99, 45, 449, 1387, 510, 324, 3, 382, 61, 0, 181, 1469, 122, 814, 7, 338, 0, 2517, 82, 119, 137, 705, 21, 458, 461, 50, 158, 1865, 9, 1142, 1799, 4381, 1, 1795, 0, 5754, 1239, 1353, 239] payload_rows_per_dst=[10014, 13390, 8572, 8386, 10994, 9246, 7936, 5316] dup_rows_per_dst=[2622, 7668, 2508, 1334, 3840, 2182, 3948, 348] combine_rows_per_src=[10014, 13390, 8572, 8386, 10994, 9246, 7936, 5316] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=274202624 dispatch_meta_bytes=342280 combine_payload_bytes=274202624 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=274544904 ingress_bytes=343408640 link_bytes=343408640 token_bytes=548747528 weight_movement_bytes=69206016 total_nvlink_bytes=617953544 weight_to_token_ratio=0.126116 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.292219066e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77372 payload_rows_remote=65600 dup_rows=20932 dup_rows_remote=16170 dedup_ratio=0.212931 router_tokens_per_expert=[261, 412, 468, 73, 413, 3225, 1, 879, 159, 660, 311, 4660, 204, 7, 326, 1226, 1189, 918, 16, 202, 606, 269, 12, 2082, 291, 153, 199, 249, 1253, 566, 252, 33, 711, 184, 1, 146, 1513, 69, 1583, 34, 317, 1, 2197, 73, 89, 148, 938, 5, 509, 381, 133, 251, 1866, 23, 1202, 1510, 4026, 0, 1964, 1, 4965, 1208, 1187, 342] payload_rows_per_dst=[9340, 12208, 8374, 13208, 11772, 9440, 7918, 5112] dup_rows_per_dst=[2124, 2898, 2214, 2714, 4762, 2024, 3832, 364] combine_rows_per_src=[9340, 12208, 8374, 13208, 11772, 9440, 7918, 5112] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=260767744 dispatch_meta_bytes=326808 combine_payload_bytes=260767744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=261094552 ingress_bytes=295370752 link_bytes=295370752 token_bytes=521862296 weight_movement_bytes=34603008 total_nvlink_bytes=556465304 weight_to_token_ratio=0.066307 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=27904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.111456360e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75870 payload_rows_remote=63402 dup_rows=22434 dup_rows_remote=17464 dedup_ratio=0.228210 router_tokens_per_expert=[46, 71, 474, 287, 38, 877, 193, 0, 265, 659, 47, 54, 2981, 2, 473, 46, 1657, 57, 299, 1057, 0, 670, 2495, 1133, 1186, 648, 147, 3969, 12, 0, 244, 911, 319, 565, 0, 190, 25, 759, 242, 4757, 2642, 6, 1252, 146, 0, 463, 1231, 25, 3, 139, 84, 0, 2043, 4192, 141, 251, 2226, 1050, 135, 248, 4222, 111, 276, 411] payload_rows_per_dst=[7222, 12468, 7856, 10488, 12238, 11818, 5210, 8570] dup_rows_per_dst=[1202, 4970, 1890, 3746, 3260, 4702, 112, 2552] combine_rows_per_src=[7222, 12468, 7856, 10488, 12238, 11818, 5210, 8570] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69738 payload_rows_remote=60522 dup_rows=28566 dup_rows_remote=23620 dedup_ratio=0.290588 router_tokens_per_expert=[247, 339, 528, 128, 355, 3959, 2, 1117, 267, 554, 317, 4732, 106, 90, 389, 1065, 1152, 1167, 12, 169, 720, 277, 16, 1875, 216, 192, 26, 394, 977, 458, 212, 0, 445, 163, 1, 100, 1398, 121, 988, 13, 354, 3, 2248, 32, 143, 67, 890, 46, 711, 409, 186, 177, 1779, 27, 1268, 1618, 4279, 7, 1975, 0, 5099, 1256, 1062, 229] payload_rows_per_dst=[9152, 4544, 13612, 4680, 5026, 9414, 9216, 14094] dup_rows_per_dst=[2386, 1032, 6628, 270, 1432, 2102, 4946, 9770] combine_rows_per_src=[9152, 4544, 13612, 4680, 5026, 9414, 9216, 14094] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265666560 dispatch_meta_bytes=324800 combine_payload_bytes=265666560 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265991360 ingress_bytes=300269568 link_bytes=300269568 token_bytes=531657920 weight_movement_bytes=34603008 total_nvlink_bytes=566260928 weight_to_token_ratio=0.065085 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=19328 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129890211e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=210132992 dispatch_meta_bytes=252936 combine_payload_bytes=210132992 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=210385928 ingress_bytes=383148032 link_bytes=383148032 token_bytes=420518920 weight_movement_bytes=173015040 total_nvlink_bytes=593533960 weight_to_token_ratio=0.411432 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.441755198e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77682 payload_rows_remote=67518 dup_rows=20622 dup_rows_remote=15762 dedup_ratio=0.209778 router_tokens_per_expert=[59, 81, 238, 1783, 45, 351, 198, 0, 190, 423, 10, 310, 140, 5, 457, 420, 1539, 976, 1065, 1708, 0, 954, 1739, 1270, 724, 1211, 287, 2725, 1, 1, 82, 725, 846, 424, 0, 95, 4, 979, 38, 2308, 838, 0, 969, 83, 28, 14, 603, 1517, 26, 129, 139, 0, 2651, 6773, 165, 20, 4174, 677, 207, 205, 2425, 328, 718, 2052] payload_rows_per_dst=[11070, 13862, 10164, 8742, 8354, 9226, 5962, 10302] dup_rows_per_dst=[2788, 3594, 4860, 2770, 1034, 2356, 298, 2922] combine_rows_per_src=[11070, 13862, 10164, 8742, 8354, 9226, 5962, 10302] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77088 payload_rows_remote=68598 dup_rows=21216 dup_rows_remote=19672 dedup_ratio=0.215820 router_tokens_per_expert=[257, 377, 2300, 28, 366, 3545, 16, 644, 179, 257, 229, 3206, 323, 2, 309, 512, 650, 656, 497, 227, 225, 214, 1, 1300, 278, 44, 181, 114, 1076, 327, 228, 0, 836, 476, 0, 97, 563, 36, 467, 5, 309, 0, 3883, 86, 52, 147, 526, 366, 46, 540, 51, 166, 2242, 2, 756, 1384, 4958, 1, 3912, 0, 6126, 1723, 604, 224] payload_rows_per_dst=[11124, 8490, 6298, 14312, 11818, 13020, 7172, 4854] dup_rows_per_dst=[3942, 1544, 1242, 2436, 3058, 5542, 3202, 250] combine_rows_per_src=[11124, 8490, 6298, 14312, 11818, 13020, 7172, 4854] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=261488640 dispatch_meta_bytes=342672 combine_payload_bytes=261488640 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=261831312 ingress_bytes=330694656 link_bytes=330694656 token_bytes=523319952 weight_movement_bytes=69206016 total_nvlink_bytes=592525968 weight_to_token_ratio=0.132244 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=24448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.244377367e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75976 payload_rows_remote=66952 dup_rows=22328 dup_rows_remote=19998 dedup_ratio=0.227132 router_tokens_per_expert=[66, 26, 266, 1881, 52, 334, 170, 0, 230, 313, 7, 294, 156, 1, 305, 355, 1594, 942, 1172, 1575, 0, 720, 1734, 925, 1154, 1001, 316, 2317, 0, 1, 80, 861, 1631, 313, 0, 84, 2, 742, 93, 2525, 950, 0, 936, 58, 24, 8, 613, 1354, 36, 50, 72, 0, 2448, 6830, 153, 23, 4415, 706, 158, 180, 2348, 477, 739, 2336] payload_rows_per_dst=[11424, 2782, 8326, 8792, 9302, 9024, 15574, 10752] dup_rows_per_dst=[2996, 540, 2516, 2668, 1478, 2330, 6664, 3136] combine_rows_per_src=[11424, 2782, 8326, 8792, 9302, 9024, 15574, 10752] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=268697600 dispatch_meta_bytes=327080 combine_payload_bytes=268697600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269024680 ingress_bytes=303300608 link_bytes=303300608 token_bytes=537722280 weight_movement_bytes=34603008 total_nvlink_bytes=572325288 weight_to_token_ratio=0.064351 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=24192 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.141295770e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=259694592 dispatch_meta_bytes=323464 combine_payload_bytes=259694592 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260018056 ingress_bytes=294297600 link_bytes=294297600 token_bytes=519712648 weight_movement_bytes=34603008 total_nvlink_bytes=554315656 weight_to_token_ratio=0.066581 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=28160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.107418176e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=276553728 dispatch_meta_bytes=333120 combine_payload_bytes=276553728 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=276886848 ingress_bytes=380362752 link_bytes=380362752 token_bytes=553440576 weight_movement_bytes=103809024 total_nvlink_bytes=657249600 weight_to_token_ratio=0.187570 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.431274414e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69738 payload_rows_remote=60522 dup_rows=28566 dup_rows_remote=23620 dedup_ratio=0.290588 router_tokens_per_expert=[247, 339, 528, 128, 355, 3959, 2, 1117, 267, 554, 317, 4732, 106, 90, 389, 1065, 1152, 1167, 12, 169, 720, 277, 16, 1875, 216, 192, 26, 394, 977, 458, 212, 0, 445, 163, 1, 100, 1398, 121, 988, 13, 354, 3, 2248, 32, 143, 67, 890, 46, 711, 409, 186, 177, 1779, 27, 1268, 1618, 4279, 7, 1975, 0, 5099, 1256, 1062, 229] payload_rows_per_dst=[9152, 4544, 13612, 4680, 5026, 9414, 9216, 14094] dup_rows_per_dst=[2386, 1032, 6628, 270, 1432, 2102, 4946, 9770] combine_rows_per_src=[9152, 4544, 13612, 4680, 5026, 9414, 9216, 14094] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=274235392 dispatch_meta_bytes=347800 combine_payload_bytes=274235392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274583192 ingress_bytes=308838400 link_bytes=308838400 token_bytes=548818584 weight_movement_bytes=34603008 total_nvlink_bytes=583421592 weight_to_token_ratio=0.063050 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.162134036e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77902 payload_rows_remote=68332 dup_rows=20402 dup_rows_remote=17214 dedup_ratio=0.207540 router_tokens_per_expert=[51, 54, 372, 1662, 27, 349, 198, 0, 267, 401, 11, 281, 771, 2, 238, 289, 1464, 610, 1068, 1881, 1, 692, 1942, 957, 1228, 784, 314, 3138, 0, 1, 79, 835, 1039, 478, 1, 114, 9, 577, 107, 3273, 1570, 0, 873, 66, 6, 72, 810, 828, 16, 110, 124, 0, 2080, 6722, 103, 126, 4252, 1004, 170, 149, 2403, 267, 500, 1336] payload_rows_per_dst=[11028, 14026, 9688, 9570, 9730, 9472, 4928, 9460] dup_rows_per_dst=[2902, 3938, 3658, 3188, 1466, 2862, 190, 2198] combine_rows_per_src=[11028, 14026, 9688, 9570, 9730, 9472, 4928, 9460] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77536 payload_rows_remote=67872 dup_rows=20768 dup_rows_remote=17642 dedup_ratio=0.211263 router_tokens_per_expert=[618, 635, 887, 4, 113, 44, 439, 3677, 564, 9, 1201, 120, 573, 1184, 5217, 281, 3, 6, 51, 2391, 107, 12, 485, 80, 28, 735, 2223, 0, 1037, 223, 46, 2032, 214, 128, 1401, 39, 1578, 114, 1, 17, 49, 1782, 484, 252, 51, 426, 179, 598, 1298, 249, 153, 25, 2838, 91, 19, 817, 234, 1115, 18, 3260, 390, 1106, 3506, 1695] payload_rows_per_dst=[10594, 13516, 9664, 9754, 10762, 6114, 8836, 8296] dup_rows_per_dst=[2240, 4782, 3126, 2894, 3234, 1528, 2144, 820] combine_rows_per_src=[10594, 13516, 9664, 9754, 10762, 6114, 8836, 8296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=268697600 dispatch_meta_bytes=327080 combine_payload_bytes=268697600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269024680 ingress_bytes=303300608 link_bytes=303300608 token_bytes=537722280 weight_movement_bytes=34603008 total_nvlink_bytes=572325288 weight_to_token_ratio=0.064351 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=24192 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.141295770e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=66752 payload_rows_remote=51302 dup_rows=31552 dup_rows_remote=11932 dedup_ratio=0.320964 router_tokens_per_expert=[253, 386, 2270, 53, 323, 4154, 14, 563, 205, 227, 154, 3806, 352, 0, 368, 681, 718, 745, 209, 172, 495, 269, 2, 1247, 213, 28, 175, 141, 1016, 380, 227, 0, 634, 659, 0, 127, 401, 66, 363, 3, 351, 4, 3587, 73, 22, 160, 356, 598, 105, 486, 11, 40, 1838, 8, 700, 1044, 4947, 0, 4197, 2, 6102, 1576, 655, 191] payload_rows_per_dst=[6450, 3198, 12042, 4200, 3956, 10254, 11202, 15450] dup_rows_per_dst=[1274, 776, 3284, 160, 550, 1458, 4430, 19620] combine_rows_per_src=[6450, 3198, 12042, 4200, 3956, 10254, 11202, 15450] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=247898112 dispatch_meta_bytes=336568 combine_payload_bytes=247898112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=248234680 ingress_bytes=282501120 link_bytes=282501120 token_bytes=496132792 weight_movement_bytes=34603008 total_nvlink_bytes=530735800 weight_to_token_ratio=0.069745 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.063028971e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77228 payload_rows_remote=66384 dup_rows=21076 dup_rows_remote=16980 dedup_ratio=0.214396 router_tokens_per_expert=[42, 7, 402, 216, 35, 798, 189, 0, 279, 594, 33, 32, 2984, 1, 462, 44, 1536, 76, 504, 1828, 0, 615, 2030, 1168, 1377, 820, 180, 3726, 12, 0, 152, 952, 527, 433, 8, 97, 12, 519, 236, 4546, 2356, 2, 1215, 90, 4, 297, 1009, 63, 1, 130, 73, 0, 2048, 5063, 157, 149, 3072, 943, 224, 182, 3741, 76, 290, 495] payload_rows_per_dst=[5802, 7936, 7006, 10474, 11154, 10760, 13252, 10844] dup_rows_per_dst=[682, 922, 792, 3964, 1602, 3372, 5646, 4096] combine_rows_per_src=[5802, 7936, 7006, 10474, 11154, 10760, 13252, 10844] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77490 payload_rows_remote=63664 dup_rows=20814 dup_rows_remote=18038 dedup_ratio=0.211731 router_tokens_per_expert=[269, 374, 863, 55, 499, 4048, 32, 1292, 190, 378, 318, 4562, 184, 7, 386, 897, 910, 876, 32, 175, 856, 309, 5, 1583, 258, 127, 92, 355, 1399, 421, 257, 1, 497, 328, 4, 168, 1165, 69, 802, 12, 360, 0, 2356, 86, 93, 110, 714, 47, 268, 435, 45, 198, 1777, 18, 1026, 1481, 4312, 0, 2699, 0, 5391, 1387, 1007, 287] payload_rows_per_dst=[11058, 11160, 7662, 13826, 11282, 10306, 7142, 5054] dup_rows_per_dst=[3806, 2684, 1830, 2776, 3432, 2624, 3354, 308] combine_rows_per_src=[11058, 11160, 7662, 13826, 11282, 10306, 7142, 5054] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=280977408 dispatch_meta_bytes=353080 combine_payload_bytes=280977408 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=281330488 ingress_bytes=350183424 link_bytes=350183424 token_bytes=562307896 weight_movement_bytes=69206016 total_nvlink_bytes=631513912 weight_to_token_ratio=0.123075 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=25216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.317712032e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278003712 dispatch_meta_bytes=342056 combine_payload_bytes=278003712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278345768 ingress_bytes=312606720 link_bytes=312606720 token_bytes=556349480 weight_movement_bytes=34603008 total_nvlink_bytes=590952488 weight_to_token_ratio=0.062197 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=28032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.176313920e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=210132992 dispatch_meta_bytes=252936 combine_payload_bytes=210132992 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=210385928 ingress_bytes=383148032 link_bytes=383148032 token_bytes=420518920 weight_movement_bytes=173015040 total_nvlink_bytes=593533960 weight_to_token_ratio=0.411432 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.441755198e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76286 payload_rows_remote=67478 dup_rows=22018 dup_rows_remote=20074 dedup_ratio=0.223979 router_tokens_per_expert=[255, 337, 468, 127, 495, 3451, 1, 1207, 264, 688, 414, 4222, 136, 30, 367, 1011, 1141, 1473, 21, 166, 1158, 209, 3, 1690, 259, 135, 37, 419, 852, 686, 294, 18, 589, 187, 0, 208, 1324, 190, 996, 15, 343, 2, 2170, 68, 215, 112, 1117, 10, 548, 387, 26, 159, 1724, 14, 1192, 1816, 4231, 1, 1339, 0, 5219, 1255, 1439, 222] payload_rows_per_dst=[9940, 10692, 9624, 11854, 8176, 8808, 7974, 9218] dup_rows_per_dst=[2742, 2554, 3116, 2102, 2632, 1944, 3758, 3170] combine_rows_per_src=[9940, 10692, 9624, 11854, 8176, 8808, 7974, 9218] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=260767744 dispatch_meta_bytes=326808 combine_payload_bytes=260767744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=261094552 ingress_bytes=295370752 link_bytes=295370752 token_bytes=521862296 weight_movement_bytes=34603008 total_nvlink_bytes=556465304 weight_to_token_ratio=0.066307 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=27904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.111456360e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76786 payload_rows_remote=65698 dup_rows=21518 dup_rows_remote=18874 dedup_ratio=0.218892 router_tokens_per_expert=[71, 83, 287, 1534, 31, 375, 236, 0, 238, 442, 2, 251, 456, 1, 508, 272, 1473, 658, 919, 1826, 1, 709, 1539, 1419, 1483, 1137, 297, 2754, 1, 1, 81, 1049, 803, 449, 0, 103, 1, 659, 123, 3128, 1308, 1, 765, 111, 9, 25, 625, 641, 24, 63, 95, 0, 2376, 6449, 262, 52, 4249, 726, 299, 179, 2829, 337, 556, 1771] payload_rows_per_dst=[11088, 13478, 11966, 8420, 12108, 7176, 5552, 6998] dup_rows_per_dst=[2644, 3760, 6404, 2394, 3754, 1632, 192, 738] combine_rows_per_src=[11088, 13478, 11966, 8420, 12108, 7176, 5552, 6998] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279887872 dispatch_meta_bytes=342184 combine_payload_bytes=279887872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=280230056 ingress_bytes=349093888 link_bytes=349093888 token_bytes=560117928 weight_movement_bytes=69206016 total_nvlink_bytes=629323944 weight_to_token_ratio=0.123556 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=22528 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.313612196e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77148 payload_rows_remote=60948 dup_rows=21156 dup_rows_remote=12400 dedup_ratio=0.215210 router_tokens_per_expert=[0, 0, 7, 0, 2311, 100, 6, 13, 43, 3256, 288, 243, 24, 283, 0, 293, 1046, 18, 8048, 26, 321, 1712, 162, 309, 612, 3, 0, 81, 0, 4788, 80, 1009, 3073, 290, 118, 2443, 2, 1, 214, 5, 83, 0, 3525, 318, 0, 505, 2369, 79, 2851, 0, 71, 0, 355, 0, 8, 40, 2333, 0, 131, 4, 114, 3, 5122, 13] payload_rows_per_dst=[11630, 16200, 5766, 3210, 10510, 11250, 6524, 12058] dup_rows_per_dst=[2820, 8756, 1422, 360, 1782, 2508, 126, 3382] combine_rows_per_src=[11630, 16200, 5766, 3210, 10510, 11250, 6524, 12058] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77326 payload_rows_remote=67246 dup_rows=20978 dup_rows_remote=19004 dedup_ratio=0.213399 router_tokens_per_expert=[70, 24, 373, 1639, 160, 576, 213, 0, 268, 407, 3, 302, 531, 10, 389, 270, 1575, 778, 922, 1476, 8, 696, 2120, 1008, 869, 1168, 325, 2828, 7, 6, 86, 818, 1040, 480, 1, 86, 28, 694, 130, 3568, 1297, 0, 1047, 126, 13, 25, 711, 841, 39, 274, 79, 2, 2195, 6521, 160, 84, 3965, 791, 194, 233, 2471, 213, 682, 1237] payload_rows_per_dst=[10954, 9662, 9250, 9184, 10080, 9552, 9240, 9404] dup_rows_per_dst=[3086, 2564, 3676, 3030, 1974, 2808, 1602, 2238] combine_rows_per_src=[10954, 9662, 9250, 9184, 10080, 9552, 9240, 9404] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73404 payload_rows_remote=57692 dup_rows=24900 dup_rows_remote=14278 dedup_ratio=0.253296 router_tokens_per_expert=[1, 256, 258, 2, 788, 2496, 29, 25, 18, 1331, 13, 20, 33, 14, 12, 1708, 333, 1, 7471, 3903, 136, 467, 1374, 966, 1300, 911, 129, 269, 24, 5755, 160, 3230, 987, 417, 177, 281, 2, 9, 3876, 2, 212, 3, 519, 173, 8, 306, 312, 0, 4075, 0, 110, 487, 338, 0, 5, 0, 622, 0, 91, 9, 410, 0, 2066, 222] payload_rows_per_dst=[13090, 7838, 15712, 9112, 9914, 3022, 8728, 5988] dup_rows_per_dst=[6130, 1428, 10622, 2934, 1588, 44, 1302, 852] combine_rows_per_src=[13090, 7838, 15712, 9112, 9914, 3022, 8728, 5988] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=271908864 dispatch_meta_bytes=333456 combine_payload_bytes=271908864 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=272242320 ingress_bytes=375717888 link_bytes=375717888 token_bytes=544151184 weight_movement_bytes=103809024 total_nvlink_bytes=647960208 weight_to_token_ratio=0.190772 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.413796165e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78492 payload_rows_remote=63206 dup_rows=19812 dup_rows_remote=16784 dedup_ratio=0.201538 router_tokens_per_expert=[0, 340, 255, 0, 885, 2404, 8, 9, 5, 1660, 12, 45, 23, 1, 12, 2030, 252, 10, 7446, 3235, 160, 486, 901, 1198, 1228, 423, 35, 225, 0, 6066, 97, 3279, 1014, 380, 91, 358, 0, 1, 3449, 5, 152, 0, 729, 241, 11, 244, 334, 0, 4320, 0, 58, 601, 415, 0, 12, 1, 1036, 0, 125, 5, 283, 3, 2407, 147] payload_rows_per_dst=[7090, 6788, 7194, 14126, 11062, 15286, 10018, 6928] dup_rows_per_dst=[712, 788, 1578, 7352, 3246, 3028, 2024, 1084] combine_rows_per_src=[7090, 6788, 7194, 14126, 11062, 15286, 10018, 6928] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75442 payload_rows_remote=59796 dup_rows=22862 dup_rows_remote=14626 dedup_ratio=0.232564 router_tokens_per_expert=[72, 41, 258, 1348, 120, 328, 205, 0, 225, 410, 5, 269, 251, 8, 255, 478, 1548, 910, 1072, 2172, 1, 638, 1586, 957, 716, 937, 384, 2540, 2, 1, 81, 776, 1025, 342, 0, 75, 5, 693, 105, 3140, 1288, 1, 841, 49, 14, 35, 680, 1113, 20, 96, 90, 0, 2418, 6935, 146, 64, 4336, 711, 238, 171, 2869, 560, 654, 1844] payload_rows_per_dst=[10982, 3248, 8088, 8554, 9392, 8850, 15646, 10682] dup_rows_per_dst=[2434, 554, 2164, 2320, 1378, 2364, 8236, 3412] combine_rows_per_src=[10982, 3248, 8088, 8554, 9392, 8850, 15646, 10682] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=276389888 dispatch_meta_bytes=350208 combine_payload_bytes=276389888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276740096 ingress_bytes=310992896 link_bytes=310992896 token_bytes=553129984 weight_movement_bytes=34603008 total_nvlink_bytes=587732992 weight_to_token_ratio=0.062559 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20864 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.170241231e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=269099008 dispatch_meta_bytes=338288 combine_payload_bytes=269099008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269437296 ingress_bytes=303702016 link_bytes=303702016 token_bytes=538536304 weight_movement_bytes=34603008 total_nvlink_bytes=573139312 weight_to_token_ratio=0.064254 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=29184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.142806236e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=249643008 dispatch_meta_bytes=293392 combine_payload_bytes=249643008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=249936400 ingress_bytes=284246016 link_bytes=284246016 token_bytes=499579408 weight_movement_bytes=34603008 total_nvlink_bytes=534182416 weight_to_token_ratio=0.069264 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.069594875e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=275439616 dispatch_meta_bytes=345000 combine_payload_bytes=275439616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275784616 ingress_bytes=310042624 link_bytes=310042624 token_bytes=551224232 weight_movement_bytes=34603008 total_nvlink_bytes=585827240 weight_to_token_ratio=0.062775 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=27264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.166665434e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75442 payload_rows_remote=59796 dup_rows=22862 dup_rows_remote=14626 dedup_ratio=0.232564 router_tokens_per_expert=[72, 41, 258, 1348, 120, 328, 205, 0, 225, 410, 5, 269, 251, 8, 255, 478, 1548, 910, 1072, 2172, 1, 638, 1586, 957, 716, 937, 384, 2540, 2, 1, 81, 776, 1025, 342, 0, 75, 5, 693, 105, 3140, 1288, 1, 841, 49, 14, 35, 680, 1113, 20, 96, 90, 0, 2418, 6935, 146, 64, 4336, 711, 238, 171, 2869, 560, 654, 1844] payload_rows_per_dst=[10982, 3248, 8088, 8554, 9392, 8850, 15646, 10682] dup_rows_per_dst=[2434, 554, 2164, 2320, 1378, 2364, 8236, 3412] combine_rows_per_src=[10982, 3248, 8088, 8554, 9392, 8850, 15646, 10682] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=236306432 dispatch_meta_bytes=287880 combine_payload_bytes=236306432 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=236594312 ingress_bytes=409321472 link_bytes=409321472 token_bytes=472900744 weight_movement_bytes=173015040 total_nvlink_bytes=645915784 weight_to_token_ratio=0.365859 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=14336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.540243746e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=258891776 dispatch_meta_bytes=319960 combine_payload_bytes=258891776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=259211736 ingress_bytes=293494784 link_bytes=293494784 token_bytes=518103512 weight_movement_bytes=34603008 total_nvlink_bytes=552706520 weight_to_token_ratio=0.066788 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22656 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.104397244e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75870 payload_rows_remote=63402 dup_rows=22434 dup_rows_remote=17464 dedup_ratio=0.228210 router_tokens_per_expert=[46, 71, 474, 287, 38, 877, 193, 0, 265, 659, 47, 54, 2981, 2, 473, 46, 1657, 57, 299, 1057, 0, 670, 2495, 1133, 1186, 648, 147, 3969, 12, 0, 244, 911, 319, 565, 0, 190, 25, 759, 242, 4757, 2642, 6, 1252, 146, 0, 463, 1231, 25, 3, 139, 84, 0, 2043, 4192, 141, 251, 2226, 1050, 135, 248, 4222, 111, 276, 411] payload_rows_per_dst=[7222, 12468, 7856, 10488, 12238, 11818, 5210, 8570] dup_rows_per_dst=[1202, 4970, 1890, 3746, 3260, 4702, 112, 2552] combine_rows_per_src=[7222, 12468, 7856, 10488, 12238, 11818, 5210, 8570] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76832 payload_rows_remote=65752 dup_rows=21472 dup_rows_remote=17642 dedup_ratio=0.218424 router_tokens_per_expert=[0, 281, 234, 0, 1970, 1409, 6, 14, 22, 2181, 26, 123, 10, 1, 10, 1671, 253, 3, 7915, 1507, 274, 650, 705, 718, 907, 566, 23, 238, 31, 6453, 84, 2663, 1521, 441, 86, 830, 1, 5, 2096, 7, 143, 0, 1338, 230, 4, 302, 822, 4, 3552, 1, 95, 434, 691, 0, 10, 3, 1533, 3, 238, 9, 186, 3, 3582, 34] payload_rows_per_dst=[10808, 7086, 16366, 11080, 8700, 5128, 8358, 9306] dup_rows_per_dst=[4040, 1002, 7684, 3830, 1274, 558, 1214, 1870] combine_rows_per_src=[10808, 7086, 16366, 11080, 8700, 5128, 8358, 9306] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=275439616 dispatch_meta_bytes=345000 combine_payload_bytes=275439616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275784616 ingress_bytes=310042624 link_bytes=310042624 token_bytes=551224232 weight_movement_bytes=34603008 total_nvlink_bytes=585827240 weight_to_token_ratio=0.062775 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=27264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.166665434e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77628 payload_rows_remote=68786 dup_rows=20676 dup_rows_remote=18220 dedup_ratio=0.210327 router_tokens_per_expert=[248, 329, 365, 57, 593, 3453, 1, 1119, 180, 827, 501, 4790, 155, 26, 367, 887, 1375, 1207, 8, 112, 986, 353, 6, 1602, 248, 169, 12, 452, 1103, 511, 297, 2, 431, 74, 3, 125, 1397, 118, 1130, 11, 360, 3, 1901, 64, 197, 141, 1086, 20, 420, 302, 69, 195, 1939, 41, 1335, 1836, 4498, 0, 1198, 0, 5229, 1358, 1045, 285] payload_rows_per_dst=[9746, 12290, 8842, 13456, 11380, 8344, 8404, 5166] dup_rows_per_dst=[2584, 3176, 2456, 2590, 4194, 1596, 3870, 210] combine_rows_per_src=[9746, 12290, 8842, 13456, 11380, 8344, 8404, 5166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=244924416 dispatch_meta_bytes=297688 combine_payload_bytes=244924416 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=245222104 ingress_bytes=314130432 link_bytes=314130432 token_bytes=490146520 weight_movement_bytes=69206016 total_nvlink_bytes=559352536 weight_to_token_ratio=0.141195 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=21504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.182047526e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71698 payload_rows_remote=55966 dup_rows=26606 dup_rows_remote=11824 dedup_ratio=0.270650 router_tokens_per_expert=[1, 35, 30, 0, 2297, 221, 13, 38, 55, 2845, 128, 209, 14, 1, 4, 767, 487, 47, 8039, 189, 308, 1553, 246, 362, 692, 99, 12, 79, 0, 5899, 71, 1650, 2565, 292, 90, 2510, 5, 1, 262, 12, 131, 2, 3485, 302, 5, 437, 1628, 43, 2950, 0, 65, 64, 237, 0, 1, 6, 2823, 0, 134, 6, 120, 4, 4557, 24] payload_rows_per_dst=[5148, 7134, 4706, 4628, 10062, 10916, 13372, 15732] dup_rows_per_dst=[122, 912, 1300, 578, 1790, 2050, 5072, 14782] combine_rows_per_src=[5148, 7134, 4706, 4628, 10062, 10916, 13372, 15732] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=269320192 dispatch_meta_bytes=333576 combine_payload_bytes=269320192 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=269653768 ingress_bytes=373129216 link_bytes=373129216 token_bytes=538973960 weight_movement_bytes=103809024 total_nvlink_bytes=642782984 weight_to_token_ratio=0.192605 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=19968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.404055200e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77228 payload_rows_remote=66384 dup_rows=21076 dup_rows_remote=16980 dedup_ratio=0.214396 router_tokens_per_expert=[42, 7, 402, 216, 35, 798, 189, 0, 279, 594, 33, 32, 2984, 1, 462, 44, 1536, 76, 504, 1828, 0, 615, 2030, 1168, 1377, 820, 180, 3726, 12, 0, 152, 952, 527, 433, 8, 97, 12, 519, 236, 4546, 2356, 2, 1215, 90, 4, 297, 1009, 63, 1, 130, 73, 0, 2048, 5063, 157, 149, 3072, 943, 224, 182, 3741, 76, 290, 495] payload_rows_per_dst=[5802, 7936, 7006, 10474, 11154, 10760, 13252, 10844] dup_rows_per_dst=[682, 922, 792, 3964, 1602, 3372, 5646, 4096] combine_rows_per_src=[5802, 7936, 7006, 10474, 11154, 10760, 13252, 10844] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77902 payload_rows_remote=68332 dup_rows=20402 dup_rows_remote=17214 dedup_ratio=0.207540 router_tokens_per_expert=[51, 54, 372, 1662, 27, 349, 198, 0, 267, 401, 11, 281, 771, 2, 238, 289, 1464, 610, 1068, 1881, 1, 692, 1942, 957, 1228, 784, 314, 3138, 0, 1, 79, 835, 1039, 478, 1, 114, 9, 577, 107, 3273, 1570, 0, 873, 66, 6, 72, 810, 828, 16, 110, 124, 0, 2080, 6722, 103, 126, 4252, 1004, 170, 149, 2403, 267, 500, 1336] payload_rows_per_dst=[11028, 14026, 9688, 9570, 9730, 9472, 4928, 9460] dup_rows_per_dst=[2902, 3938, 3658, 3188, 1466, 2862, 190, 2198] combine_rows_per_src=[11028, 14026, 9688, 9570, 9730, 9472, 4928, 9460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=259694592 dispatch_meta_bytes=323464 combine_payload_bytes=259694592 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260018056 ingress_bytes=294297600 link_bytes=294297600 token_bytes=519712648 weight_movement_bytes=34603008 total_nvlink_bytes=554315656 weight_to_token_ratio=0.066581 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=28160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.107418176e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=281747456 dispatch_meta_bytes=348024 combine_payload_bytes=281747456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282095480 ingress_bytes=316350464 link_bytes=316350464 token_bytes=563842936 weight_movement_bytes=34603008 total_nvlink_bytes=598445944 weight_to_token_ratio=0.061370 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.190401328e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=271908864 dispatch_meta_bytes=333456 combine_payload_bytes=271908864 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=272242320 ingress_bytes=375717888 link_bytes=375717888 token_bytes=544151184 weight_movement_bytes=103809024 total_nvlink_bytes=647960208 weight_to_token_ratio=0.190772 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.413796165e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279887872 dispatch_meta_bytes=342184 combine_payload_bytes=279887872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=280230056 ingress_bytes=349093888 link_bytes=349093888 token_bytes=560117928 weight_movement_bytes=69206016 total_nvlink_bytes=629323944 weight_to_token_ratio=0.123556 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=22528 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.313612196e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75976 payload_rows_remote=66952 dup_rows=22328 dup_rows_remote=19998 dedup_ratio=0.227132 router_tokens_per_expert=[66, 26, 266, 1881, 52, 334, 170, 0, 230, 313, 7, 294, 156, 1, 305, 355, 1594, 942, 1172, 1575, 0, 720, 1734, 925, 1154, 1001, 316, 2317, 0, 1, 80, 861, 1631, 313, 0, 84, 2, 742, 93, 2525, 950, 0, 936, 58, 24, 8, 613, 1354, 36, 50, 72, 0, 2448, 6830, 153, 23, 4415, 706, 158, 180, 2348, 477, 739, 2336] payload_rows_per_dst=[11424, 2782, 8326, 8792, 9302, 9024, 15574, 10752] dup_rows_per_dst=[2996, 540, 2516, 2668, 1478, 2330, 6664, 3136] combine_rows_per_src=[11424, 2782, 8326, 8792, 9302, 9024, 15574, 10752] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75584 payload_rows_remote=61520 dup_rows=22720 dup_rows_remote=16140 dedup_ratio=0.231120 router_tokens_per_expert=[0, 169, 176, 2, 1943, 1631, 20, 13, 48, 2409, 25, 114, 19, 4, 13, 1767, 349, 5, 7840, 1486, 231, 724, 806, 857, 995, 274, 28, 259, 0, 6368, 91, 2503, 1877, 418, 101, 556, 1, 0, 2114, 11, 246, 1, 1497, 275, 5, 395, 592, 0, 4107, 0, 158, 430, 265, 0, 4, 2, 1457, 0, 259, 16, 171, 2, 2950, 73] payload_rows_per_dst=[14064, 15922, 7228, 7000, 8628, 5618, 8790, 8334] dup_rows_per_dst=[6580, 8556, 2016, 972, 1528, 404, 1142, 1522] combine_rows_per_src=[14064, 15922, 7228, 7000, 8628, 5618, 8790, 8334] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73466 payload_rows_remote=65866 dup_rows=24838 dup_rows_remote=23890 dedup_ratio=0.252665 router_tokens_per_expert=[4745, 529, 2372, 1, 468, 4, 1688, 1214, 43, 0, 0, 410, 119, 680, 284, 0, 135, 0, 1, 212, 1672, 140, 6, 644, 1080, 2, 1601, 457, 0, 1, 2183, 0, 592, 1141, 7, 1597, 0, 1231, 330, 3, 12, 0, 1791, 72, 10, 2886, 779, 0, 16, 124, 5799, 0, 485, 416, 209, 143, 635, 7735, 0, 0, 0, 17, 1717, 714] payload_rows_per_dst=[11124, 7600, 12338, 11264, 1836, 7236, 16198, 5870] dup_rows_per_dst=[3890, 948, 2772, 2966, 28, 282, 13656, 296] combine_rows_per_src=[11124, 7600, 12338, 11264, 1836, 7236, 16198, 5870] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77692 payload_rows_remote=67252 dup_rows=20612 dup_rows_remote=18772 dedup_ratio=0.209676 router_tokens_per_expert=[0, 408, 416, 4, 1498, 1861, 45, 72, 52, 1794, 14, 93, 47, 1, 6, 1397, 360, 16, 7592, 2346, 207, 538, 1039, 601, 954, 1092, 51, 329, 21, 5795, 209, 2137, 1346, 306, 101, 821, 3, 30, 3494, 39, 507, 13, 984, 277, 14, 401, 523, 3, 3495, 0, 215, 517, 654, 0, 8, 6, 1004, 14, 207, 2, 341, 20, 2621, 191] payload_rows_per_dst=[7716, 6064, 8104, 14032, 10440, 15348, 8456, 7532] dup_rows_per_dst=[892, 744, 2792, 7144, 1840, 4598, 1334, 1268] combine_rows_per_src=[7716, 6064, 8104, 14032, 10440, 15348, 8456, 7532] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79058 payload_rows_remote=70178 dup_rows=19246 dup_rows_remote=17754 dedup_ratio=0.195780 router_tokens_per_expert=[2754, 858, 2037, 214, 52, 193, 1084, 0, 841, 0, 15, 21, 77, 282, 0, 0, 1285, 0, 232, 15, 737, 133, 6, 24, 2818, 3, 419, 309, 2, 342, 409, 123, 1227, 3551, 2289, 4285, 133, 2924, 31, 1, 83, 10, 6079, 402, 6, 1645, 536, 1, 0, 750, 498, 0, 10, 2594, 211, 3, 1184, 3915, 0, 56, 3, 903, 317, 220] payload_rows_per_dst=[10984, 13316, 8880, 10964, 11896, 5206, 11110, 6702] dup_rows_per_dst=[3740, 4828, 1492, 3144, 2366, 160, 2924, 592] combine_rows_per_src=[10984, 13316, 8880, 10964, 11896, 5206, 11110, 6702] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=229236736 dispatch_meta_bytes=271160 combine_payload_bytes=229236736 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=229507896 ingress_bytes=263839744 link_bytes=263839744 token_bytes=458744632 weight_movement_bytes=34603008 total_nvlink_bytes=493347640 weight_to_token_ratio=0.075430 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=25600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.928077158e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80762 payload_rows_remote=66688 dup_rows=17542 dup_rows_remote=14454 dedup_ratio=0.178446 router_tokens_per_expert=[3551, 385, 2176, 178, 1, 34, 952, 0, 729, 0, 28, 23, 342, 248, 0, 0, 1647, 0, 442, 7, 757, 89, 3, 9, 2735, 7, 463, 248, 0, 317, 554, 38, 1107, 3370, 1360, 4292, 282, 3471, 29, 0, 62, 17, 6315, 353, 0, 1605, 229, 0, 0, 624, 666, 0, 39, 2279, 183, 0, 1313, 4012, 0, 107, 0, 834, 488, 152] payload_rows_per_dst=[7078, 9770, 10744, 7526, 12884, 14074, 7082, 11604] dup_rows_per_dst=[374, 1554, 2266, 1198, 6354, 3088, 500, 2208] combine_rows_per_src=[7078, 9770, 10744, 7526, 12884, 14074, 7082, 11604] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=251985920 dispatch_meta_bytes=310640 combine_payload_bytes=251985920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=252296560 ingress_bytes=286588928 link_bytes=286588928 token_bytes=504282480 weight_movement_bytes=34603008 total_nvlink_bytes=538885488 weight_to_token_ratio=0.068618 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=20480 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.078411064e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77022 payload_rows_remote=62488 dup_rows=21282 dup_rows_remote=13972 dedup_ratio=0.216492 router_tokens_per_expert=[0, 386, 328, 1, 914, 2040, 33, 29, 54, 1975, 13, 49, 21, 0, 11, 2420, 185, 6, 7441, 2769, 181, 539, 923, 1073, 1026, 774, 87, 287, 61, 5713, 80, 3658, 898, 328, 88, 445, 0, 27, 3408, 7, 201, 0, 583, 241, 6, 342, 457, 2, 3948, 0, 138, 680, 433, 0, 9, 1, 817, 1, 274, 1, 304, 1, 2209, 226] payload_rows_per_dst=[6856, 7828, 5164, 9430, 11174, 15350, 14534, 6686] dup_rows_per_dst=[606, 1258, 650, 2516, 4766, 3196, 7310, 980] combine_rows_per_src=[6856, 7828, 5164, 9430, 11174, 15350, 14534, 6686] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=269787136 dispatch_meta_bytes=359024 combine_payload_bytes=269787136 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=270146160 ingress_bytes=338993152 link_bytes=338993152 token_bytes=539933296 weight_movement_bytes=69206016 total_nvlink_bytes=609139312 weight_to_token_ratio=0.128175 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=26624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.275603940e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=275464192 dispatch_meta_bytes=344096 combine_payload_bytes=275464192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275808288 ingress_bytes=310067200 link_bytes=310067200 token_bytes=551272480 weight_movement_bytes=34603008 total_nvlink_bytes=585875488 weight_to_token_ratio=0.062769 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.166757911e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=274235392 dispatch_meta_bytes=347800 combine_payload_bytes=274235392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274583192 ingress_bytes=308838400 link_bytes=308838400 token_bytes=548818584 weight_movement_bytes=34603008 total_nvlink_bytes=583421592 weight_to_token_ratio=0.063050 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.162134036e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=251985920 dispatch_meta_bytes=310640 combine_payload_bytes=251985920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=252296560 ingress_bytes=286588928 link_bytes=286588928 token_bytes=504282480 weight_movement_bytes=34603008 total_nvlink_bytes=538885488 weight_to_token_ratio=0.068618 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=20480 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.078411064e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77148 payload_rows_remote=60948 dup_rows=21156 dup_rows_remote=12400 dedup_ratio=0.215210 router_tokens_per_expert=[0, 0, 7, 0, 2311, 100, 6, 13, 43, 3256, 288, 243, 24, 283, 0, 293, 1046, 18, 8048, 26, 321, 1712, 162, 309, 612, 3, 0, 81, 0, 4788, 80, 1009, 3073, 290, 118, 2443, 2, 1, 214, 5, 83, 0, 3525, 318, 0, 505, 2369, 79, 2851, 0, 71, 0, 355, 0, 8, 40, 2333, 0, 131, 4, 114, 3, 5122, 13] payload_rows_per_dst=[11630, 16200, 5766, 3210, 10510, 11250, 6524, 12058] dup_rows_per_dst=[2820, 8756, 1422, 360, 1782, 2508, 126, 3382] combine_rows_per_src=[11630, 16200, 5766, 3210, 10510, 11250, 6524, 12058] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=287449088 dispatch_meta_bytes=351728 combine_payload_bytes=287449088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=287800816 ingress_bytes=322052096 link_bytes=322052096 token_bytes=575249904 weight_movement_bytes=34603008 total_nvlink_bytes=609852912 weight_to_token_ratio=0.060153 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=29824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.211856110e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81022 payload_rows_remote=72284 dup_rows=17282 dup_rows_remote=15666 dedup_ratio=0.175802 router_tokens_per_expert=[4302, 925, 1993, 184, 261, 61, 1490, 45, 604, 0, 18, 88, 143, 361, 8, 0, 646, 0, 143, 33, 898, 117, 3, 24, 2877, 0, 729, 419, 0, 312, 728, 112, 1087, 3528, 503, 3299, 99, 3070, 151, 0, 78, 4, 5076, 376, 0, 2109, 218, 0, 0, 315, 1994, 1, 86, 924, 207, 2, 1478, 4905, 0, 89, 1, 798, 760, 470] payload_rows_per_dst=[8860, 11844, 10876, 8738, 9100, 11900, 6732, 12972] dup_rows_per_dst=[1358, 4254, 1456, 1616, 1906, 2336, 326, 4030] combine_rows_per_src=[8860, 11844, 10876, 8738, 9100, 11900, 6732, 12972] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=273154048 dispatch_meta_bytes=324568 combine_payload_bytes=273154048 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=273478616 ingress_bytes=376963072 link_bytes=376963072 token_bytes=546632664 weight_movement_bytes=103809024 total_nvlink_bytes=650441688 weight_to_token_ratio=0.189906 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23424 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.418481692e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=275464192 dispatch_meta_bytes=344096 combine_payload_bytes=275464192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275808288 ingress_bytes=310067200 link_bytes=310067200 token_bytes=551272480 weight_movement_bytes=34603008 total_nvlink_bytes=585875488 weight_to_token_ratio=0.062769 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.166757911e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77682 payload_rows_remote=67518 dup_rows=20622 dup_rows_remote=15762 dedup_ratio=0.209778 router_tokens_per_expert=[59, 81, 238, 1783, 45, 351, 198, 0, 190, 423, 10, 310, 140, 5, 457, 420, 1539, 976, 1065, 1708, 0, 954, 1739, 1270, 724, 1211, 287, 2725, 1, 1, 82, 725, 846, 424, 0, 95, 4, 979, 38, 2308, 838, 0, 969, 83, 28, 14, 603, 1517, 26, 129, 139, 0, 2651, 6773, 165, 20, 4174, 677, 207, 205, 2425, 328, 718, 2052] payload_rows_per_dst=[11070, 13862, 10164, 8742, 8354, 9226, 5962, 10302] dup_rows_per_dst=[2788, 3594, 4860, 2770, 1034, 2356, 298, 2922] combine_rows_per_src=[11070, 13862, 10164, 8742, 8354, 9226, 5962, 10302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=255950848 dispatch_meta_bytes=305840 combine_payload_bytes=255950848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=256256688 ingress_bytes=290553856 link_bytes=290553856 token_bytes=512207536 weight_movement_bytes=34603008 total_nvlink_bytes=546810544 weight_to_token_ratio=0.067557 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=29312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.093330769e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=296075264 dispatch_meta_bytes=351800 combine_payload_bytes=296075264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296427064 ingress_bytes=330678272 link_bytes=330678272 token_bytes=592502328 weight_movement_bytes=34603008 total_nvlink_bytes=627105336 weight_to_token_ratio=0.058401 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.244315716e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81470 payload_rows_remote=66182 dup_rows=16834 dup_rows_remote=11400 dedup_ratio=0.171244 router_tokens_per_expert=[5304, 598, 1958, 14, 288, 20, 1164, 376, 190, 1, 0, 299, 205, 552, 72, 0, 251, 0, 24, 440, 1539, 129, 5, 371, 1502, 4, 1372, 230, 0, 9, 1759, 2, 1450, 2471, 74, 1870, 5, 1563, 310, 2, 34, 0, 2717, 73, 26, 2891, 472, 0, 14, 115, 5017, 0, 311, 309, 277, 112, 603, 7401, 0, 3, 0, 96, 1708, 550] payload_rows_per_dst=[7758, 5764, 13034, 8782, 9238, 10430, 11176, 15288] dup_rows_per_dst=[1078, 614, 3092, 974, 2512, 1996, 1134, 5434] combine_rows_per_src=[7758, 5764, 13034, 8782, 9238, 10430, 11176, 15288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71698 payload_rows_remote=55966 dup_rows=26606 dup_rows_remote=11824 dedup_ratio=0.270650 router_tokens_per_expert=[1, 35, 30, 0, 2297, 221, 13, 38, 55, 2845, 128, 209, 14, 1, 4, 767, 487, 47, 8039, 189, 308, 1553, 246, 362, 692, 99, 12, 79, 0, 5899, 71, 1650, 2565, 292, 90, 2510, 5, 1, 262, 12, 131, 2, 3485, 302, 5, 437, 1628, 43, 2950, 0, 65, 64, 237, 0, 1, 6, 2823, 0, 134, 6, 120, 4, 4557, 24] payload_rows_per_dst=[5148, 7134, 4706, 4628, 10062, 10916, 13372, 15732] dup_rows_per_dst=[122, 912, 1300, 578, 1790, 2050, 5072, 14782] combine_rows_per_src=[5148, 7134, 4706, 4628, 10062, 10916, 13372, 15732] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76832 payload_rows_remote=65752 dup_rows=21472 dup_rows_remote=17642 dedup_ratio=0.218424 router_tokens_per_expert=[0, 281, 234, 0, 1970, 1409, 6, 14, 22, 2181, 26, 123, 10, 1, 10, 1671, 253, 3, 7915, 1507, 274, 650, 705, 718, 907, 566, 23, 238, 31, 6453, 84, 2663, 1521, 441, 86, 830, 1, 5, 2096, 7, 143, 0, 1338, 230, 4, 302, 822, 4, 3552, 1, 95, 434, 691, 0, 10, 3, 1533, 3, 238, 9, 186, 3, 3582, 34] payload_rows_per_dst=[10808, 7086, 16366, 11080, 8700, 5128, 8358, 9306] dup_rows_per_dst=[4040, 1002, 7684, 3830, 1274, 558, 1214, 1870] combine_rows_per_src=[10808, 7086, 16366, 11080, 8700, 5128, 8358, 9306] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=249643008 dispatch_meta_bytes=293392 combine_payload_bytes=249643008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=249936400 ingress_bytes=284246016 link_bytes=284246016 token_bytes=499579408 weight_movement_bytes=34603008 total_nvlink_bytes=534182416 weight_to_token_ratio=0.069264 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.069594875e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=276553728 dispatch_meta_bytes=333120 combine_payload_bytes=276553728 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=276886848 ingress_bytes=380362752 link_bytes=380362752 token_bytes=553440576 weight_movement_bytes=103809024 total_nvlink_bytes=657249600 weight_to_token_ratio=0.187570 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.431274414e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=229236736 dispatch_meta_bytes=271160 combine_payload_bytes=229236736 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=229507896 ingress_bytes=263839744 link_bytes=263839744 token_bytes=458744632 weight_movement_bytes=34603008 total_nvlink_bytes=493347640 weight_to_token_ratio=0.075430 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=25600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.928077158e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=269320192 dispatch_meta_bytes=333576 combine_payload_bytes=269320192 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=269653768 ingress_bytes=373129216 link_bytes=373129216 token_bytes=538973960 weight_movement_bytes=103809024 total_nvlink_bytes=642782984 weight_to_token_ratio=0.192605 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=19968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.404055200e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78492 payload_rows_remote=63206 dup_rows=19812 dup_rows_remote=16784 dedup_ratio=0.201538 router_tokens_per_expert=[0, 340, 255, 0, 885, 2404, 8, 9, 5, 1660, 12, 45, 23, 1, 12, 2030, 252, 10, 7446, 3235, 160, 486, 901, 1198, 1228, 423, 35, 225, 0, 6066, 97, 3279, 1014, 380, 91, 358, 0, 1, 3449, 5, 152, 0, 729, 241, 11, 244, 334, 0, 4320, 0, 58, 601, 415, 0, 12, 1, 1036, 0, 125, 5, 283, 3, 2407, 147] payload_rows_per_dst=[7090, 6788, 7194, 14126, 11062, 15286, 10018, 6928] dup_rows_per_dst=[712, 788, 1578, 7352, 3246, 3028, 2024, 1084] combine_rows_per_src=[7090, 6788, 7194, 14126, 11062, 15286, 10018, 6928] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73976 payload_rows_remote=60994 dup_rows=24328 dup_rows_remote=16950 dedup_ratio=0.247477 router_tokens_per_expert=[4530, 781, 2439, 234, 108, 40, 1068, 8, 486, 1, 13, 97, 185, 306, 2, 0, 723, 0, 112, 67, 795, 125, 52, 26, 3046, 3, 763, 317, 0, 236, 852, 3, 873, 2977, 426, 3420, 39, 3502, 41, 2, 77, 9, 4764, 344, 0, 2049, 1125, 0, 0, 343, 1702, 0, 93, 1704, 217, 2, 1179, 5230, 0, 68, 0, 747, 570, 231] payload_rows_per_dst=[12982, 11520, 7920, 13760, 2554, 8770, 11086, 5384] dup_rows_per_dst=[7378, 3454, 940, 6208, 208, 2324, 3610, 206] combine_rows_per_src=[12982, 11520, 7920, 13760, 2554, 8770, 11086, 5384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78208 payload_rows_remote=62822 dup_rows=20096 dup_rows_remote=7018 dedup_ratio=0.204427 router_tokens_per_expert=[663, 0, 0, 169, 62, 56, 128, 1258, 47, 6349, 344, 7, 1, 6388, 0, 1096, 70, 106, 0, 0, 127, 1, 867, 1, 382, 39, 4734, 1, 1, 111, 0, 0, 264, 0, 0, 0, 0, 7247, 2, 407, 416, 139, 2506, 0, 68, 4669, 6532, 0, 404, 0, 9, 99, 0, 104, 0, 3151, 18, 1, 0, 0, 0, 0, 28, 80] payload_rows_per_dst=[4820, 15386, 10980, 9974, 14508, 15436, 6852, 252] dup_rows_per_dst=[684, 13078, 702, 562, 1332, 3054, 682, 2] combine_rows_per_src=[4820, 15386, 10980, 9974, 14508, 15436, 6852, 252] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81146 payload_rows_remote=69028 dup_rows=17158 dup_rows_remote=12636 dedup_ratio=0.174540 router_tokens_per_expert=[3738, 1192, 2017, 254, 195, 90, 1834, 7, 917, 0, 43, 94, 370, 461, 0, 8, 1020, 0, 268, 20, 841, 176, 0, 19, 3042, 2, 715, 334, 4, 511, 512, 248, 1181, 3813, 898, 3850, 149, 2328, 101, 3, 94, 26, 4628, 310, 14, 1838, 361, 9, 0, 400, 1145, 1, 59, 1561, 141, 0, 1349, 3789, 13, 87, 4, 1419, 307, 342] payload_rows_per_dst=[9544, 9598, 10348, 8972, 12118, 12406, 6194, 11966] dup_rows_per_dst=[1634, 2194, 1816, 1764, 4522, 2154, 420, 2654] combine_rows_per_src=[9544, 9598, 10348, 8972, 12118, 12406, 6194, 11966] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77782 payload_rows_remote=66536 dup_rows=20522 dup_rows_remote=17426 dedup_ratio=0.208761 router_tokens_per_expert=[239, 0, 22, 96, 788, 346, 85, 2346, 33, 5742, 98, 524, 221, 3025, 0, 863, 2682, 102, 0, 0, 69, 373, 512, 492, 503, 0, 5566, 550, 33, 340, 30, 11, 811, 0, 0, 1270, 127, 366, 210, 584, 7717, 1, 164, 10, 225, 2941, 93, 0, 1514, 49, 1285, 3299, 313, 88, 198, 106, 1646, 54, 10, 0, 154, 96, 6, 124] payload_rows_per_dst=[15522, 9058, 11246, 12520, 5774, 984, 11026, 11652] dup_rows_per_dst=[7756, 1796, 3096, 1546, 962, 2, 2678, 2686] combine_rows_per_src=[15522, 9058, 11246, 12520, 5774, 984, 11026, 11652] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79540 payload_rows_remote=73686 dup_rows=18764 dup_rows_remote=18462 dedup_ratio=0.190877 router_tokens_per_expert=[2858, 1082, 2257, 205, 101, 186, 1430, 0, 716, 0, 79, 91, 310, 313, 1, 3, 729, 2, 805, 59, 609, 92, 7, 4, 3539, 4, 317, 309, 0, 352, 482, 211, 1049, 4239, 2028, 4010, 286, 2753, 36, 0, 33, 24, 5372, 372, 1, 2005, 164, 0, 0, 597, 802, 0, 19, 1434, 226, 0, 1104, 3227, 0, 184, 0, 1296, 473, 265] payload_rows_per_dst=[9074, 9272, 9046, 9086, 13424, 13048, 5854, 10736] dup_rows_per_dst=[1448, 1774, 1284, 1342, 7358, 2894, 302, 2362] combine_rows_per_src=[9074, 9272, 9046, 9086, 13424, 13048, 5854, 10736] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81502 payload_rows_remote=66054 dup_rows=16802 dup_rows_remote=11296 dedup_ratio=0.170919 router_tokens_per_expert=[298, 0, 0, 44, 744, 288, 47, 2763, 44, 6375, 196, 178, 142, 2399, 0, 1091, 3437, 173, 0, 0, 20, 387, 373, 86, 336, 0, 5557, 618, 60, 281, 5, 14, 521, 0, 0, 1621, 108, 722, 207, 578, 7573, 33, 120, 1, 221, 3506, 311, 0, 723, 65, 1357, 2861, 601, 140, 263, 214, 885, 44, 6, 2, 284, 91, 16, 122] payload_rows_per_dst=[7520, 3260, 9554, 12480, 9694, 15448, 9968, 13578] dup_rows_per_dst=[848, 42, 1974, 1262, 2618, 5506, 2480, 2072] combine_rows_per_src=[7520, 3260, 9554, 12480, 9694, 15448, 9968, 13578] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=271081472 dispatch_meta_bytes=310328 combine_payload_bytes=271081472 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=271391800 ingress_bytes=340287488 link_bytes=340287488 token_bytes=542473272 weight_movement_bytes=69206016 total_nvlink_bytes=611679288 weight_to_token_ratio=0.127575 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=24320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.280474422e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 2, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=249831424 dispatch_meta_bytes=311776 combine_payload_bytes=249831424 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=250143200 ingress_bytes=319037440 link_bytes=319037440 token_bytes=499974624 weight_movement_bytes=69206016 total_nvlink_bytes=569180640 weight_to_token_ratio=0.138419 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=17280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.200512202e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=257318912 dispatch_meta_bytes=279360 combine_payload_bytes=257318912 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=257598272 ingress_bytes=361127936 link_bytes=361127936 token_bytes=514917184 weight_movement_bytes=103809024 total_nvlink_bytes=618726208 weight_to_token_ratio=0.201603 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20480 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.358895350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=258891776 dispatch_meta_bytes=319960 combine_payload_bytes=258891776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=259211736 ingress_bytes=293494784 link_bytes=293494784 token_bytes=518103512 weight_movement_bytes=34603008 total_nvlink_bytes=552706520 weight_to_token_ratio=0.066788 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22656 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.104397244e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73466 payload_rows_remote=65866 dup_rows=24838 dup_rows_remote=23890 dedup_ratio=0.252665 router_tokens_per_expert=[4745, 529, 2372, 1, 468, 4, 1688, 1214, 43, 0, 0, 410, 119, 680, 284, 0, 135, 0, 1, 212, 1672, 140, 6, 644, 1080, 2, 1601, 457, 0, 1, 2183, 0, 592, 1141, 7, 1597, 0, 1231, 330, 3, 12, 0, 1791, 72, 10, 2886, 779, 0, 16, 124, 5799, 0, 485, 416, 209, 143, 635, 7735, 0, 0, 0, 17, 1717, 714] payload_rows_per_dst=[11124, 7600, 12338, 11264, 1836, 7236, 16198, 5870] dup_rows_per_dst=[3890, 948, 2772, 2966, 28, 282, 13656, 296] combine_rows_per_src=[11124, 7600, 12338, 11264, 1836, 7236, 16198, 5870] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=249831424 dispatch_meta_bytes=311776 combine_payload_bytes=249831424 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=250143200 ingress_bytes=319037440 link_bytes=319037440 token_bytes=499974624 weight_movement_bytes=69206016 total_nvlink_bytes=569180640 weight_to_token_ratio=0.138419 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=17280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.200512202e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=282738688 dispatch_meta_bytes=326656 combine_payload_bytes=282738688 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=283065344 ingress_bytes=421150720 link_bytes=421150720 token_bytes=565804032 weight_movement_bytes=138412032 total_nvlink_bytes=704216064 weight_to_token_ratio=0.244629 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=13824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.584756254e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=272531456 dispatch_meta_bytes=335848 combine_payload_bytes=272531456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272867304 ingress_bytes=307134464 link_bytes=307134464 token_bytes=545398760 weight_movement_bytes=34603008 total_nvlink_bytes=580001768 weight_to_token_ratio=0.063445 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.155722262e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=301817856 dispatch_meta_bytes=368592 combine_payload_bytes=301817856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=302186448 ingress_bytes=336420864 link_bytes=336420864 token_bytes=604004304 weight_movement_bytes=34603008 total_nvlink_bytes=638607312 weight_to_token_ratio=0.057289 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265924627e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=270557184 dispatch_meta_bytes=309400 combine_payload_bytes=270557184 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=270866584 ingress_bytes=408969216 link_bytes=408969216 token_bytes=541423768 weight_movement_bytes=138412032 total_nvlink_bytes=679835800 weight_to_token_ratio=0.255645 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.538918235e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82390 payload_rows_remote=70082 dup_rows=15914 dup_rows_remote=15054 dedup_ratio=0.161886 router_tokens_per_expert=[442, 0, 46, 61, 351, 163, 78, 1645, 4, 6834, 139, 104, 180, 3757, 0, 1279, 2919, 165, 0, 0, 115, 169, 463, 152, 289, 0, 5718, 374, 6, 190, 0, 7, 533, 0, 0, 864, 39, 2549, 141, 477, 6224, 85, 364, 0, 144, 4328, 1483, 0, 906, 23, 954, 1566, 331, 69, 80, 1026, 860, 24, 0, 3, 142, 99, 74, 114] payload_rows_per_dst=[13834, 9232, 12654, 12308, 8036, 4104, 7836, 14386] dup_rows_per_dst=[4186, 1694, 3968, 860, 1170, 48, 2074, 1914] combine_rows_per_src=[13834, 9232, 12654, 12308, 8036, 4104, 7836, 14386] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=287055872 dispatch_meta_bytes=340544 combine_payload_bytes=287055872 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=287396416 ingress_bytes=356261888 link_bytes=356261888 token_bytes=574452288 weight_movement_bytes=69206016 total_nvlink_bytes=643658304 weight_to_token_ratio=0.120473 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=40320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.340584803e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69354 payload_rows_remote=55598 dup_rows=28950 dup_rows_remote=28626 dedup_ratio=0.294495 router_tokens_per_expert=[768, 0, 2, 83, 53, 63, 119, 1506, 54, 6838, 192, 4, 10, 5905, 1, 1253, 777, 72, 2, 0, 84, 3, 759, 0, 275, 0, 5628, 39, 2, 142, 3, 0, 288, 0, 0, 99, 3, 6379, 8, 374, 1560, 70, 1394, 0, 91, 5255, 4921, 1, 239, 9, 85, 293, 34, 81, 34, 3095, 50, 3, 0, 0, 3, 5, 56, 85] payload_rows_per_dst=[4832, 3008, 2976, 5514, 15502, 15150, 8616, 13756] dup_rows_per_dst=[356, 20, 418, 332, 10610, 14646, 2244, 324] combine_rows_per_src=[4832, 3008, 2976, 5514, 15502, 15150, 8616, 13756] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82274 payload_rows_remote=81440 dup_rows=16030 dup_rows_remote=16018 dedup_ratio=0.163066 router_tokens_per_expert=[358, 0, 70, 0, 3, 366, 3811, 62, 0, 4, 0, 0, 2, 0, 0, 0, 417, 0, 4895, 36, 158, 440, 177, 8, 0, 7765, 0, 99, 3418, 2635, 8, 0, 10, 0, 4671, 0, 829, 2, 0, 151, 0, 1, 0, 692, 0, 0, 262, 7562, 7, 0, 0, 3488, 285, 0, 7, 2567, 125, 0, 28, 85, 25, 24, 0, 3599] payload_rows_per_dst=[16218, 834, 10972, 10562, 10002, 15642, 10418, 7626] dup_rows_per_dst=[8652, 12, 456, 1758, 1324, 1392, 2290, 146] combine_rows_per_src=[16218, 834, 10972, 10562, 10002, 15642, 10418, 7626] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73702 payload_rows_remote=59744 dup_rows=24602 dup_rows_remote=19300 dedup_ratio=0.250264 router_tokens_per_expert=[425, 0, 2, 73, 336, 188, 113, 2069, 94, 7243, 349, 157, 95, 3687, 0, 1503, 2960, 348, 1, 0, 71, 161, 559, 25, 501, 1, 5869, 466, 42, 153, 3, 2, 500, 0, 0, 759, 194, 2027, 126, 713, 6424, 79, 271, 0, 121, 4517, 858, 0, 899, 40, 603, 1530, 280, 80, 48, 749, 472, 10, 0, 0, 128, 24, 9, 195] payload_rows_per_dst=[13958, 15464, 12970, 12228, 7572, 2652, 7206, 1652] dup_rows_per_dst=[5302, 11532, 4314, 1106, 1066, 6, 1252, 24] combine_rows_per_src=[13958, 15464, 12970, 12228, 7572, 2652, 7206, 1652] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73404 payload_rows_remote=57692 dup_rows=24900 dup_rows_remote=14278 dedup_ratio=0.253296 router_tokens_per_expert=[1, 256, 258, 2, 788, 2496, 29, 25, 18, 1331, 13, 20, 33, 14, 12, 1708, 333, 1, 7471, 3903, 136, 467, 1374, 966, 1300, 911, 129, 269, 24, 5755, 160, 3230, 987, 417, 177, 281, 2, 9, 3876, 2, 212, 3, 519, 173, 8, 306, 312, 0, 4075, 0, 110, 487, 338, 0, 5, 0, 622, 0, 91, 9, 410, 0, 2066, 222] payload_rows_per_dst=[13090, 7838, 15712, 9112, 9914, 3022, 8728, 5988] dup_rows_per_dst=[6130, 1428, 10622, 2934, 1588, 44, 1302, 852] combine_rows_per_src=[13090, 7838, 15712, 9112, 9914, 3022, 8728, 5988] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81470 payload_rows_remote=66182 dup_rows=16834 dup_rows_remote=11400 dedup_ratio=0.171244 router_tokens_per_expert=[5304, 598, 1958, 14, 288, 20, 1164, 376, 190, 1, 0, 299, 205, 552, 72, 0, 251, 0, 24, 440, 1539, 129, 5, 371, 1502, 4, 1372, 230, 0, 9, 1759, 2, 1450, 2471, 74, 1870, 5, 1563, 310, 2, 34, 0, 2717, 73, 26, 2891, 472, 0, 14, 115, 5017, 0, 311, 309, 277, 112, 603, 7401, 0, 3, 0, 96, 1708, 550] payload_rows_per_dst=[7758, 5764, 13034, 8782, 9238, 10430, 11176, 15288] dup_rows_per_dst=[1078, 614, 3092, 974, 2512, 1996, 1134, 5434] combine_rows_per_src=[7758, 5764, 13034, 8782, 9238, 10430, 11176, 15288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81022 payload_rows_remote=72284 dup_rows=17282 dup_rows_remote=15666 dedup_ratio=0.175802 router_tokens_per_expert=[4302, 925, 1993, 184, 261, 61, 1490, 45, 604, 0, 18, 88, 143, 361, 8, 0, 646, 0, 143, 33, 898, 117, 3, 24, 2877, 0, 729, 419, 0, 312, 728, 112, 1087, 3528, 503, 3299, 99, 3070, 151, 0, 78, 4, 5076, 376, 0, 2109, 218, 0, 0, 315, 1994, 1, 86, 924, 207, 2, 1478, 4905, 0, 89, 1, 798, 760, 470] payload_rows_per_dst=[8860, 11844, 10876, 8738, 9100, 11900, 6732, 12972] dup_rows_per_dst=[1358, 4254, 1456, 1616, 1906, 2336, 326, 4030] combine_rows_per_src=[8860, 11844, 10876, 8738, 9100, 11900, 6732, 12972] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=269787136 dispatch_meta_bytes=359024 combine_payload_bytes=269787136 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=270146160 ingress_bytes=338993152 link_bytes=338993152 token_bytes=539933296 weight_movement_bytes=69206016 total_nvlink_bytes=609139312 weight_to_token_ratio=0.128175 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=26624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.275603940e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=236306432 dispatch_meta_bytes=287880 combine_payload_bytes=236306432 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=236594312 ingress_bytes=409321472 link_bytes=409321472 token_bytes=472900744 weight_movement_bytes=173015040 total_nvlink_bytes=645915784 weight_to_token_ratio=0.365859 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=14336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.540243746e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=271081472 dispatch_meta_bytes=310328 combine_payload_bytes=271081472 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=271391800 ingress_bytes=340287488 link_bytes=340287488 token_bytes=542473272 weight_movement_bytes=69206016 total_nvlink_bytes=611679288 weight_to_token_ratio=0.127575 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=24320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.280474422e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=296075264 dispatch_meta_bytes=351800 combine_payload_bytes=296075264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296427064 ingress_bytes=330678272 link_bytes=330678272 token_bytes=592502328 weight_movement_bytes=34603008 total_nvlink_bytes=627105336 weight_to_token_ratio=0.058401 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.244315716e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80762 payload_rows_remote=66688 dup_rows=17542 dup_rows_remote=14454 dedup_ratio=0.178446 router_tokens_per_expert=[3551, 385, 2176, 178, 1, 34, 952, 0, 729, 0, 28, 23, 342, 248, 0, 0, 1647, 0, 442, 7, 757, 89, 3, 9, 2735, 7, 463, 248, 0, 317, 554, 38, 1107, 3370, 1360, 4292, 282, 3471, 29, 0, 62, 17, 6315, 353, 0, 1605, 229, 0, 0, 624, 666, 0, 39, 2279, 183, 0, 1313, 4012, 0, 107, 0, 834, 488, 152] payload_rows_per_dst=[7078, 9770, 10744, 7526, 12884, 14074, 7082, 11604] dup_rows_per_dst=[374, 1554, 2266, 1198, 6354, 3088, 500, 2208] combine_rows_per_src=[7078, 9770, 10744, 7526, 12884, 14074, 7082, 11604] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81244 payload_rows_remote=74588 dup_rows=17060 dup_rows_remote=16120 dedup_ratio=0.173543 router_tokens_per_expert=[389, 9, 66, 62, 478, 502, 105, 1995, 27, 6133, 122, 193, 290, 3565, 1, 1364, 3029, 129, 0, 2, 106, 186, 455, 698, 349, 6, 5332, 717, 41, 388, 35, 40, 603, 0, 1, 881, 71, 1391, 201, 649, 6599, 57, 196, 1, 165, 2980, 823, 0, 936, 62, 1038, 2050, 277, 159, 112, 456, 1791, 114, 19, 25, 151, 300, 19, 211] payload_rows_per_dst=[7832, 9170, 11640, 12262, 6656, 11364, 8616, 13704] dup_rows_per_dst=[2274, 1952, 3530, 1554, 940, 1424, 1564, 3822] combine_rows_per_src=[7832, 9170, 11640, 12262, 6656, 11364, 8616, 13704] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77216 payload_rows_remote=65852 dup_rows=21088 dup_rows_remote=17028 dedup_ratio=0.214518 router_tokens_per_expert=[238, 0, 44, 0, 2265, 458, 482, 19, 2, 3153, 150, 20, 317, 67, 900, 10, 7205, 3, 3563, 197, 1190, 2722, 36, 1, 0, 3216, 1, 384, 407, 1398, 2137, 59, 2432, 233, 445, 36, 226, 0, 6, 155, 91, 802, 50, 93, 1, 0, 332, 1956, 2255, 4, 0, 1314, 198, 0, 124, 1032, 343, 0, 547, 5002, 0, 1, 29, 801] payload_rows_per_dst=[10762, 15628, 11364, 7852, 6328, 5930, 8120, 11232] dup_rows_per_dst=[2682, 8020, 4060, 920, 738, 720, 1734, 2214] combine_rows_per_src=[10762, 15628, 11364, 7852, 6328, 5930, 8120, 11232] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77398 payload_rows_remote=62380 dup_rows=20906 dup_rows_remote=10714 dedup_ratio=0.212667 router_tokens_per_expert=[254, 0, 105, 44, 1204, 384, 25, 2274, 72, 6141, 87, 243, 286, 2687, 4, 1114, 2898, 144, 0, 0, 40, 518, 356, 185, 510, 0, 5386, 694, 26, 263, 3, 16, 454, 0, 0, 1017, 150, 759, 194, 671, 7386, 18, 157, 6, 154, 2600, 383, 2, 966, 63, 1621, 2504, 346, 130, 223, 286, 2252, 52, 3, 14, 200, 442, 16, 120] payload_rows_per_dst=[7332, 3560, 7470, 12390, 9324, 8112, 15018, 14192] dup_rows_per_dst=[1248, 52, 812, 1406, 2540, 368, 10192, 4288] combine_rows_per_src=[7332, 3560, 7470, 12390, 9324, 8112, 15018, 14192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77092 payload_rows_remote=70196 dup_rows=21212 dup_rows_remote=20348 dedup_ratio=0.215780 router_tokens_per_expert=[277, 0, 63, 1, 2285, 446, 605, 27, 0, 2574, 129, 26, 662, 77, 912, 15, 7148, 12, 3507, 313, 511, 2332, 24, 13, 0, 3536, 0, 422, 392, 1123, 2136, 36, 3227, 610, 292, 7, 311, 0, 0, 125, 74, 573, 7, 169, 1, 0, 388, 2668, 2673, 3, 0, 1304, 222, 0, 96, 1451, 287, 0, 612, 3579, 0, 0, 2, 867] payload_rows_per_dst=[6876, 8714, 12118, 11998, 11800, 6896, 9586, 9104] dup_rows_per_dst=[532, 4112, 4552, 3292, 4358, 864, 1912, 1590] combine_rows_per_src=[6876, 8714, 12118, 11998, 11800, 6896, 9586, 9104] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=227729408 dispatch_meta_bytes=336896 combine_payload_bytes=227729408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=228066304 ingress_bytes=262332416 link_bytes=262332416 token_bytes=455795712 weight_movement_bytes=34603008 total_nvlink_bytes=490398720 weight_to_token_ratio=0.075918 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=37504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.871357619e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73702 payload_rows_remote=59744 dup_rows=24602 dup_rows_remote=19300 dedup_ratio=0.250264 router_tokens_per_expert=[425, 0, 2, 73, 336, 188, 113, 2069, 94, 7243, 349, 157, 95, 3687, 0, 1503, 2960, 348, 1, 0, 71, 161, 559, 25, 501, 1, 5869, 466, 42, 153, 3, 2, 500, 0, 0, 759, 194, 2027, 126, 713, 6424, 79, 271, 0, 121, 4517, 858, 0, 899, 40, 603, 1530, 280, 80, 48, 749, 472, 10, 0, 0, 128, 24, 9, 195] payload_rows_per_dst=[13958, 15464, 12970, 12228, 7572, 2652, 7206, 1652] dup_rows_per_dst=[5302, 11532, 4314, 1106, 1066, 6, 1252, 24] combine_rows_per_src=[13958, 15464, 12970, 12228, 7572, 2652, 7206, 1652] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=333578240 dispatch_meta_bytes=389832 combine_payload_bytes=333578240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=333968072 ingress_bytes=368181248 link_bytes=368181248 token_bytes=667546312 weight_movement_bytes=34603008 total_nvlink_bytes=702149320 weight_to_token_ratio=0.051836 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=27904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.385436395e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=244711424 dispatch_meta_bytes=316176 combine_payload_bytes=244711424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=245027600 ingress_bytes=279314432 link_bytes=279314432 token_bytes=489739024 weight_movement_bytes=34603008 total_nvlink_bytes=524342032 weight_to_token_ratio=0.070656 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=22272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.051037721e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=273154048 dispatch_meta_bytes=324568 combine_payload_bytes=273154048 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=273478616 ingress_bytes=376963072 link_bytes=376963072 token_bytes=546632664 weight_movement_bytes=103809024 total_nvlink_bytes=650441688 weight_to_token_ratio=0.189906 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23424 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.418481692e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78208 payload_rows_remote=62822 dup_rows=20096 dup_rows_remote=7018 dedup_ratio=0.204427 router_tokens_per_expert=[663, 0, 0, 169, 62, 56, 128, 1258, 47, 6349, 344, 7, 1, 6388, 0, 1096, 70, 106, 0, 0, 127, 1, 867, 1, 382, 39, 4734, 1, 1, 111, 0, 0, 264, 0, 0, 0, 0, 7247, 2, 407, 416, 139, 2506, 0, 68, 4669, 6532, 0, 404, 0, 9, 99, 0, 104, 0, 3151, 18, 1, 0, 0, 0, 0, 28, 80] payload_rows_per_dst=[4820, 15386, 10980, 9974, 14508, 15436, 6852, 252] dup_rows_per_dst=[684, 13078, 702, 562, 1332, 3054, 682, 2] combine_rows_per_src=[4820, 15386, 10980, 9974, 14508, 15436, 6852, 252] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=305512448 dispatch_meta_bytes=362832 combine_payload_bytes=305512448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=305875280 ingress_bytes=340115456 link_bytes=340115456 token_bytes=611387728 weight_movement_bytes=34603008 total_nvlink_bytes=645990736 weight_to_token_ratio=0.056597 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=21504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.279827079e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=269729792 dispatch_meta_bytes=331520 combine_payload_bytes=269729792 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=270061312 ingress_bytes=442744832 link_bytes=442744832 token_bytes=539791104 weight_movement_bytes=173015040 total_nvlink_bytes=712806144 weight_to_token_ratio=0.320522 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.666013159e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=255508480 dispatch_meta_bytes=292376 combine_payload_bytes=255508480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=255800856 ingress_bytes=290111488 link_bytes=290111488 token_bytes=511309336 weight_movement_bytes=34603008 total_nvlink_bytes=545912344 weight_to_token_ratio=0.067675 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.091666173e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=287522816 dispatch_meta_bytes=362176 combine_payload_bytes=287522816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=287884992 ingress_bytes=322125824 link_bytes=322125824 token_bytes=575407808 weight_movement_bytes=34603008 total_nvlink_bytes=610010816 weight_to_token_ratio=0.060136 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=34688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.212133542e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79124 payload_rows_remote=68142 dup_rows=19180 dup_rows_remote=15396 dedup_ratio=0.195109 router_tokens_per_expert=[288, 0, 55, 0, 971, 692, 1227, 66, 0, 1289, 64, 2, 208, 31, 741, 5, 5967, 17, 4768, 404, 282, 1708, 124, 16, 0, 5124, 1, 643, 792, 2410, 971, 51, 1944, 399, 1556, 2, 400, 0, 1, 129, 29, 302, 17, 451, 0, 2, 307, 4694, 1459, 2, 0, 1662, 200, 0, 46, 2429, 520, 0, 308, 2053, 0, 0, 12, 1311] payload_rows_per_dst=[9696, 12814, 5426, 10982, 12500, 10490, 9854, 7362] dup_rows_per_dst=[2120, 3800, 890, 3784, 4684, 1114, 1742, 1046] combine_rows_per_src=[9696, 12814, 5426, 10982, 12500, 10490, 9854, 7362] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=279109632 dispatch_meta_bytes=334152 combine_payload_bytes=279109632 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=279443784 ingress_bytes=382918656 link_bytes=382918656 token_bytes=558553416 weight_movement_bytes=103809024 total_nvlink_bytes=662362440 weight_to_token_ratio=0.185853 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=14720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.440892075e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77252 payload_rows_remote=61258 dup_rows=21052 dup_rows_remote=15024 dedup_ratio=0.214152 router_tokens_per_expert=[141, 0, 86, 1, 93, 218, 3514, 66, 0, 125, 5, 0, 42, 0, 50, 1, 1612, 0, 5469, 92, 81, 555, 91, 4, 0, 7534, 0, 105, 3175, 1998, 93, 0, 59, 201, 4527, 0, 736, 1, 0, 156, 2, 31, 1, 678, 0, 2, 257, 7268, 27, 0, 0, 3253, 211, 0, 82, 3032, 228, 0, 66, 122, 1, 3, 0, 3057] payload_rows_per_dst=[7878, 390, 4480, 9462, 10324, 15698, 13026, 15994] dup_rows_per_dst=[360, 56, 390, 1280, 1036, 6510, 5392, 6028] combine_rows_per_src=[7878, 390, 4480, 9462, 10324, 15698, 13026, 15994] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79968 payload_rows_remote=73648 dup_rows=18336 dup_rows_remote=18148 dedup_ratio=0.186523 router_tokens_per_expert=[4323, 5506, 552, 1786, 2521, 3, 0, 27, 8, 193, 1, 2750, 53, 1, 236, 12, 0, 6, 344, 0, 0, 0, 0, 9, 0, 1, 8, 0, 0, 0, 4275, 871, 4, 0, 0, 1961, 391, 1, 1344, 0, 1576, 601, 0, 5, 1633, 1, 0, 7366, 552, 7611, 0, 201, 0, 952, 1, 0, 0, 2, 152, 790, 22, 497, 0, 3] payload_rows_per_dst=[14144, 6320, 924, 10610, 6872, 15250, 16030, 9818] dup_rows_per_dst=[6372, 188, 14, 2852, 584, 3962, 2604, 1760] combine_rows_per_src=[14144, 6320, 924, 10610, 6872, 15250, 16030, 9818] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80624 payload_rows_remote=67394 dup_rows=17680 dup_rows_remote=13272 dedup_ratio=0.179850 router_tokens_per_expert=[468, 0, 78, 0, 654, 720, 1358, 25, 1, 1144, 129, 16, 168, 78, 734, 4, 5949, 20, 4705, 328, 560, 1898, 137, 6, 0, 5516, 2, 425, 657, 2128, 663, 1, 1558, 395, 1256, 14, 734, 1, 0, 154, 23, 168, 4, 388, 0, 3, 294, 5024, 813, 6, 0, 1601, 253, 0, 158, 2806, 615, 1, 280, 2454, 0, 3, 2, 1572] payload_rows_per_dst=[13230, 13116, 12184, 6434, 7098, 10832, 9406, 8324] dup_rows_per_dst=[4408, 3330, 3772, 670, 1126, 976, 1868, 1530] combine_rows_per_src=[13230, 13116, 12184, 6434, 7098, 10832, 9406, 8324] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=244711424 dispatch_meta_bytes=316176 combine_payload_bytes=244711424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=245027600 ingress_bytes=279314432 link_bytes=279314432 token_bytes=489739024 weight_movement_bytes=34603008 total_nvlink_bytes=524342032 weight_to_token_ratio=0.070656 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=22272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.051037721e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79058 payload_rows_remote=70178 dup_rows=19246 dup_rows_remote=17754 dedup_ratio=0.195780 router_tokens_per_expert=[2754, 858, 2037, 214, 52, 193, 1084, 0, 841, 0, 15, 21, 77, 282, 0, 0, 1285, 0, 232, 15, 737, 133, 6, 24, 2818, 3, 419, 309, 2, 342, 409, 123, 1227, 3551, 2289, 4285, 133, 2924, 31, 1, 83, 10, 6079, 402, 6, 1645, 536, 1, 0, 750, 498, 0, 10, 2594, 211, 3, 1184, 3915, 0, 56, 3, 903, 317, 220] payload_rows_per_dst=[10984, 13316, 8880, 10964, 11896, 5206, 11110, 6702] dup_rows_per_dst=[3740, 4828, 1492, 3144, 2366, 160, 2924, 592] combine_rows_per_src=[10984, 13316, 8880, 10964, 11896, 5206, 11110, 6702] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69354 payload_rows_remote=55598 dup_rows=28950 dup_rows_remote=28626 dedup_ratio=0.294495 router_tokens_per_expert=[768, 0, 2, 83, 53, 63, 119, 1506, 54, 6838, 192, 4, 10, 5905, 1, 1253, 777, 72, 2, 0, 84, 3, 759, 0, 275, 0, 5628, 39, 2, 142, 3, 0, 288, 0, 0, 99, 3, 6379, 8, 374, 1560, 70, 1394, 0, 91, 5255, 4921, 1, 239, 9, 85, 293, 34, 81, 34, 3095, 50, 3, 0, 0, 3, 5, 56, 85] payload_rows_per_dst=[4832, 3008, 2976, 5514, 15502, 15150, 8616, 13756] dup_rows_per_dst=[356, 20, 418, 332, 10610, 14646, 2244, 324] combine_rows_per_src=[4832, 3008, 2976, 5514, 15502, 15150, 8616, 13756] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82390 payload_rows_remote=70082 dup_rows=15914 dup_rows_remote=15054 dedup_ratio=0.161886 router_tokens_per_expert=[442, 0, 46, 61, 351, 163, 78, 1645, 4, 6834, 139, 104, 180, 3757, 0, 1279, 2919, 165, 0, 0, 115, 169, 463, 152, 289, 0, 5718, 374, 6, 190, 0, 7, 533, 0, 0, 864, 39, 2549, 141, 477, 6224, 85, 364, 0, 144, 4328, 1483, 0, 906, 23, 954, 1566, 331, 69, 80, 1026, 860, 24, 0, 3, 142, 99, 74, 114] payload_rows_per_dst=[13834, 9232, 12654, 12308, 8036, 4104, 7836, 14386] dup_rows_per_dst=[4186, 1694, 3968, 860, 1170, 48, 2074, 1914] combine_rows_per_src=[13834, 9232, 12654, 12308, 8036, 4104, 7836, 14386] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=257318912 dispatch_meta_bytes=279360 combine_payload_bytes=257318912 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=257598272 ingress_bytes=361127936 link_bytes=361127936 token_bytes=514917184 weight_movement_bytes=103809024 total_nvlink_bytes=618726208 weight_to_token_ratio=0.201603 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20480 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.358895350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=287449088 dispatch_meta_bytes=351728 combine_payload_bytes=287449088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=287800816 ingress_bytes=322052096 link_bytes=322052096 token_bytes=575249904 weight_movement_bytes=34603008 total_nvlink_bytes=609852912 weight_to_token_ratio=0.060153 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=29824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.211856110e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=227729408 dispatch_meta_bytes=336896 combine_payload_bytes=227729408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=228066304 ingress_bytes=262332416 link_bytes=262332416 token_bytes=455795712 weight_movement_bytes=34603008 total_nvlink_bytes=490398720 weight_to_token_ratio=0.075918 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=37504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.871357619e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81502 payload_rows_remote=66054 dup_rows=16802 dup_rows_remote=11296 dedup_ratio=0.170919 router_tokens_per_expert=[298, 0, 0, 44, 744, 288, 47, 2763, 44, 6375, 196, 178, 142, 2399, 0, 1091, 3437, 173, 0, 0, 20, 387, 373, 86, 336, 0, 5557, 618, 60, 281, 5, 14, 521, 0, 0, 1621, 108, 722, 207, 578, 7573, 33, 120, 1, 221, 3506, 311, 0, 723, 65, 1357, 2861, 601, 140, 263, 214, 885, 44, 6, 2, 284, 91, 16, 122] payload_rows_per_dst=[7520, 3260, 9554, 12480, 9694, 15448, 9968, 13578] dup_rows_per_dst=[848, 42, 1974, 1262, 2618, 5506, 2480, 2072] combine_rows_per_src=[7520, 3260, 9554, 12480, 9694, 15448, 9968, 13578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=287055872 dispatch_meta_bytes=340544 combine_payload_bytes=287055872 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=287396416 ingress_bytes=356261888 link_bytes=356261888 token_bytes=574452288 weight_movement_bytes=69206016 total_nvlink_bytes=643658304 weight_to_token_ratio=0.120473 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=40320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.340584803e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77892 payload_rows_remote=65862 dup_rows=20412 dup_rows_remote=15992 dedup_ratio=0.207642 router_tokens_per_expert=[343, 0, 62, 4, 2074, 681, 793, 109, 0, 1715, 163, 42, 344, 105, 747, 16, 6237, 33, 3576, 486, 601, 2723, 83, 17, 0, 4274, 1, 552, 518, 1835, 1521, 100, 2761, 435, 898, 44, 346, 0, 8, 157, 35, 515, 16, 400, 3, 1, 346, 3356, 1711, 0, 0, 1491, 358, 1, 61, 1558, 580, 4, 326, 2852, 9, 1, 78, 1046] payload_rows_per_dst=[7320, 13864, 6912, 12868, 12030, 8200, 8388, 8310] dup_rows_per_dst=[812, 4874, 974, 4734, 4420, 1144, 1972, 1482] combine_rows_per_src=[7320, 13864, 6912, 12868, 12030, 8200, 8388, 8310] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78892 payload_rows_remote=68418 dup_rows=19412 dup_rows_remote=17290 dedup_ratio=0.197469 router_tokens_per_expert=[196, 0, 36, 0, 1988, 518, 598, 43, 0, 2557, 123, 32, 610, 63, 965, 20, 6789, 15, 3069, 352, 506, 2202, 12, 1, 0, 3416, 5, 466, 411, 1476, 2461, 142, 3052, 388, 508, 11, 299, 0, 0, 185, 37, 1066, 43, 231, 0, 0, 287, 2040, 3058, 1, 0, 1412, 212, 0, 50, 1565, 376, 0, 477, 3983, 0, 0, 13, 786] payload_rows_per_dst=[6398, 7578, 14136, 9946, 8780, 9296, 10474, 12284] dup_rows_per_dst=[360, 1162, 5618, 1762, 1756, 2600, 2122, 4032] combine_rows_per_src=[6398, 7578, 14136, 9946, 8780, 9296, 10474, 12284] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81090 payload_rows_remote=67362 dup_rows=17214 dup_rows_remote=12686 dedup_ratio=0.175110 router_tokens_per_expert=[2385, 751, 4503, 5997, 597, 30, 15, 5652, 2153, 1504, 447, 27, 0, 0, 709, 14, 46, 998, 0, 455, 8, 51, 0, 1573, 552, 0, 64, 100, 0, 0, 1317, 941, 215, 30, 0, 14, 1037, 225, 555, 0, 4844, 179, 14, 587, 1497, 2, 0, 299, 588, 3701, 0, 2, 40, 518, 0, 0, 23, 33, 1, 807, 2188, 608, 0, 256] payload_rows_per_dst=[7842, 10332, 13728, 12352, 13780, 4662, 8454, 9940] dup_rows_per_dst=[372, 2954, 4528, 3284, 1676, 494, 1244, 2662] combine_rows_per_src=[7842, 10332, 13728, 12352, 13780, 4662, 8454, 9940] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79314 payload_rows_remote=67314 dup_rows=18990 dup_rows_remote=16722 dedup_ratio=0.193176 router_tokens_per_expert=[1923, 951, 3963, 6397, 633, 15, 39, 5829, 2374, 1487, 455, 107, 0, 0, 615, 34, 40, 1158, 0, 479, 22, 55, 0, 1550, 538, 1, 70, 188, 0, 0, 1353, 786, 134, 4, 0, 9, 1159, 474, 402, 0, 4872, 60, 1, 1061, 1359, 2, 0, 377, 555, 3651, 0, 2, 0, 388, 0, 0, 1, 3, 0, 458, 2490, 569, 0, 59] payload_rows_per_dst=[3192, 12596, 14350, 6032, 13804, 12000, 8068, 9272] dup_rows_per_dst=[84, 5474, 5052, 1036, 2218, 2268, 1124, 1734] combine_rows_per_src=[3192, 12596, 14350, 6032, 13804, 12000, 8068, 9272] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=250912768 dispatch_meta_bytes=305128 combine_payload_bytes=250912768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=251217896 ingress_bytes=285515776 link_bytes=285515776 token_bytes=502130664 weight_movement_bytes=34603008 total_nvlink_bytes=536733672 weight_to_token_ratio=0.068912 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=20224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.074372879e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=301662208 dispatch_meta_bytes=367184 combine_payload_bytes=301662208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=302029392 ingress_bytes=336265216 link_bytes=336265216 token_bytes=603691600 weight_movement_bytes=34603008 total_nvlink_bytes=638294608 weight_to_token_ratio=0.057319 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265338936e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=276045824 dispatch_meta_bytes=322664 combine_payload_bytes=276045824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276368488 ingress_bytes=310648832 link_bytes=310648832 token_bytes=552414312 weight_movement_bytes=34603008 total_nvlink_bytes=587017320 weight_to_token_ratio=0.062640 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=19200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.168946546e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80624 payload_rows_remote=67394 dup_rows=17680 dup_rows_remote=13272 dedup_ratio=0.179850 router_tokens_per_expert=[468, 0, 78, 0, 654, 720, 1358, 25, 1, 1144, 129, 16, 168, 78, 734, 4, 5949, 20, 4705, 328, 560, 1898, 137, 6, 0, 5516, 2, 425, 657, 2128, 663, 1, 1558, 395, 1256, 14, 734, 1, 0, 154, 23, 168, 4, 388, 0, 3, 294, 5024, 813, 6, 0, 1601, 253, 0, 158, 2806, 615, 1, 280, 2454, 0, 3, 2, 1572] payload_rows_per_dst=[13230, 13116, 12184, 6434, 7098, 10832, 9406, 8324] dup_rows_per_dst=[4408, 3330, 3772, 670, 1126, 976, 1868, 1530] combine_rows_per_src=[13230, 13116, 12184, 6434, 7098, 10832, 9406, 8324] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=275914752 dispatch_meta_bytes=320192 combine_payload_bytes=275914752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276234944 ingress_bytes=310517760 link_bytes=310517760 token_bytes=552149696 weight_movement_bytes=34603008 total_nvlink_bytes=586752704 weight_to_token_ratio=0.062670 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=34432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.168453332e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81060 payload_rows_remote=68736 dup_rows=17244 dup_rows_remote=12632 dedup_ratio=0.175415 router_tokens_per_expert=[3414, 1636, 2978, 6274, 1583, 20, 8, 3337, 1330, 1548, 326, 481, 0, 0, 415, 35, 27, 474, 0, 220, 17, 13, 0, 510, 428, 0, 97, 106, 0, 0, 2537, 1205, 41, 0, 0, 99, 887, 376, 735, 0, 4095, 186, 15, 603, 1685, 4, 0, 1827, 656, 5276, 0, 6, 9, 386, 0, 0, 21, 8, 3, 767, 1748, 628, 0, 72] payload_rows_per_dst=[5970, 10848, 12970, 12324, 9582, 7494, 11196, 10676] dup_rows_per_dst=[524, 3378, 2100, 4612, 1368, 1146, 1470, 2646] combine_rows_per_src=[5970, 10848, 12970, 12324, 9582, 7494, 11196, 10676] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=270557184 dispatch_meta_bytes=309400 combine_payload_bytes=270557184 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=270866584 ingress_bytes=408969216 link_bytes=408969216 token_bytes=541423768 weight_movement_bytes=138412032 total_nvlink_bytes=679835800 weight_to_token_ratio=0.255645 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.538918235e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=269770752 dispatch_meta_bytes=327416 combine_payload_bytes=269770752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270098168 ingress_bytes=304373760 link_bytes=304373760 token_bytes=539868920 weight_movement_bytes=34603008 total_nvlink_bytes=574471928 weight_to_token_ratio=0.064095 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.145333955e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82274 payload_rows_remote=81440 dup_rows=16030 dup_rows_remote=16018 dedup_ratio=0.163066 router_tokens_per_expert=[358, 0, 70, 0, 3, 366, 3811, 62, 0, 4, 0, 0, 2, 0, 0, 0, 417, 0, 4895, 36, 158, 440, 177, 8, 0, 7765, 0, 99, 3418, 2635, 8, 0, 10, 0, 4671, 0, 829, 2, 0, 151, 0, 1, 0, 692, 0, 0, 262, 7562, 7, 0, 0, 3488, 285, 0, 7, 2567, 125, 0, 28, 85, 25, 24, 0, 3599] payload_rows_per_dst=[16218, 834, 10972, 10562, 10002, 15642, 10418, 7626] dup_rows_per_dst=[8652, 12, 456, 1758, 1324, 1392, 2290, 146] combine_rows_per_src=[16218, 834, 10972, 10562, 10002, 15642, 10418, 7626] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=280240128 dispatch_meta_bytes=342832 combine_payload_bytes=280240128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280582960 ingress_bytes=314843136 link_bytes=314843136 token_bytes=560823088 weight_movement_bytes=34603008 total_nvlink_bytes=595426096 weight_to_token_ratio=0.061700 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=16512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.184729374e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=275718144 dispatch_meta_bytes=336144 combine_payload_bytes=275718144 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=276054288 ingress_bytes=344924160 link_bytes=344924160 token_bytes=551772432 weight_movement_bytes=69206016 total_nvlink_bytes=620978448 weight_to_token_ratio=0.125425 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=18176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.297921845e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=276045824 dispatch_meta_bytes=322664 combine_payload_bytes=276045824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276368488 ingress_bytes=310648832 link_bytes=310648832 token_bytes=552414312 weight_movement_bytes=34603008 total_nvlink_bytes=587017320 weight_to_token_ratio=0.062640 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=19200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.168946546e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77252 payload_rows_remote=61258 dup_rows=21052 dup_rows_remote=15024 dedup_ratio=0.214152 router_tokens_per_expert=[141, 0, 86, 1, 93, 218, 3514, 66, 0, 125, 5, 0, 42, 0, 50, 1, 1612, 0, 5469, 92, 81, 555, 91, 4, 0, 7534, 0, 105, 3175, 1998, 93, 0, 59, 201, 4527, 0, 736, 1, 0, 156, 2, 31, 1, 678, 0, 2, 257, 7268, 27, 0, 0, 3253, 211, 0, 82, 3032, 228, 0, 66, 122, 1, 3, 0, 3057] payload_rows_per_dst=[7878, 390, 4480, 9462, 10324, 15698, 13026, 15994] dup_rows_per_dst=[360, 56, 390, 1280, 1036, 6510, 5392, 6028] combine_rows_per_src=[7878, 390, 4480, 9462, 10324, 15698, 13026, 15994] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77782 payload_rows_remote=66536 dup_rows=20522 dup_rows_remote=17426 dedup_ratio=0.208761 router_tokens_per_expert=[239, 0, 22, 96, 788, 346, 85, 2346, 33, 5742, 98, 524, 221, 3025, 0, 863, 2682, 102, 0, 0, 69, 373, 512, 492, 503, 0, 5566, 550, 33, 340, 30, 11, 811, 0, 0, 1270, 127, 366, 210, 584, 7717, 1, 164, 10, 225, 2941, 93, 0, 1514, 49, 1285, 3299, 313, 88, 198, 106, 1646, 54, 10, 0, 154, 96, 6, 124] payload_rows_per_dst=[15522, 9058, 11246, 12520, 5774, 984, 11026, 11652] dup_rows_per_dst=[7756, 1796, 3096, 1546, 962, 2, 2678, 2686] combine_rows_per_src=[15522, 9058, 11246, 12520, 5774, 984, 11026, 11652] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79124 payload_rows_remote=68142 dup_rows=19180 dup_rows_remote=15396 dedup_ratio=0.195109 router_tokens_per_expert=[288, 0, 55, 0, 971, 692, 1227, 66, 0, 1289, 64, 2, 208, 31, 741, 5, 5967, 17, 4768, 404, 282, 1708, 124, 16, 0, 5124, 1, 643, 792, 2410, 971, 51, 1944, 399, 1556, 2, 400, 0, 1, 129, 29, 302, 17, 451, 0, 2, 307, 4694, 1459, 2, 0, 1662, 200, 0, 46, 2429, 520, 0, 308, 2053, 0, 0, 12, 1311] payload_rows_per_dst=[9696, 12814, 5426, 10982, 12500, 10490, 9854, 7362] dup_rows_per_dst=[2120, 3800, 890, 3784, 4684, 1114, 1742, 1046] combine_rows_per_src=[9696, 12814, 5426, 10982, 12500, 10490, 9854, 7362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=333578240 dispatch_meta_bytes=389832 combine_payload_bytes=333578240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=333968072 ingress_bytes=368181248 link_bytes=368181248 token_bytes=667546312 weight_movement_bytes=34603008 total_nvlink_bytes=702149320 weight_to_token_ratio=0.051836 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=27904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.385436395e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=250912768 dispatch_meta_bytes=305128 combine_payload_bytes=250912768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=251217896 ingress_bytes=285515776 link_bytes=285515776 token_bytes=502130664 weight_movement_bytes=34603008 total_nvlink_bytes=536733672 weight_to_token_ratio=0.068912 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=20224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.074372879e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=272531456 dispatch_meta_bytes=335848 combine_payload_bytes=272531456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272867304 ingress_bytes=307134464 link_bytes=307134464 token_bytes=545398760 weight_movement_bytes=34603008 total_nvlink_bytes=580001768 weight_to_token_ratio=0.063445 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.155722262e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77092 payload_rows_remote=70196 dup_rows=21212 dup_rows_remote=20348 dedup_ratio=0.215780 router_tokens_per_expert=[277, 0, 63, 1, 2285, 446, 605, 27, 0, 2574, 129, 26, 662, 77, 912, 15, 7148, 12, 3507, 313, 511, 2332, 24, 13, 0, 3536, 0, 422, 392, 1123, 2136, 36, 3227, 610, 292, 7, 311, 0, 0, 125, 74, 573, 7, 169, 1, 0, 388, 2668, 2673, 3, 0, 1304, 222, 0, 96, 1451, 287, 0, 612, 3579, 0, 0, 2, 867] payload_rows_per_dst=[6876, 8714, 12118, 11998, 11800, 6896, 9586, 9104] dup_rows_per_dst=[532, 4112, 4552, 3292, 4358, 864, 1912, 1590] combine_rows_per_src=[6876, 8714, 12118, 11998, 11800, 6896, 9586, 9104] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80144 payload_rows_remote=68064 dup_rows=18160 dup_rows_remote=16676 dedup_ratio=0.184733 router_tokens_per_expert=[2804, 1025, 3602, 5514, 1528, 60, 54, 4714, 1671, 1513, 319, 168, 0, 0, 562, 27, 70, 1150, 0, 653, 54, 57, 0, 752, 725, 8, 115, 111, 0, 0, 1052, 667, 231, 11, 12, 13, 835, 196, 768, 2, 3532, 251, 58, 879, 1663, 2, 1, 670, 899, 5117, 3, 3, 47, 411, 0, 0, 143, 26, 3, 825, 2845, 596, 0, 135] payload_rows_per_dst=[4718, 11556, 12846, 10174, 12080, 6086, 11186, 11498] dup_rows_per_dst=[616, 4168, 3654, 2246, 1484, 962, 1774, 3256] combine_rows_per_src=[4718, 11556, 12846, 10174, 12080, 6086, 11186, 11498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=279109632 dispatch_meta_bytes=334152 combine_payload_bytes=279109632 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=279443784 ingress_bytes=382918656 link_bytes=382918656 token_bytes=558553416 weight_movement_bytes=103809024 total_nvlink_bytes=662362440 weight_to_token_ratio=0.185853 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=14720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.440892075e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79258 payload_rows_remote=72248 dup_rows=19046 dup_rows_remote=17966 dedup_ratio=0.193746 router_tokens_per_expert=[2867, 833, 3383, 5996, 1143, 36, 64, 5866, 2503, 1584, 391, 32, 0, 0, 559, 30, 89, 1134, 0, 416, 30, 37, 0, 1058, 981, 1, 69, 163, 0, 0, 1313, 804, 271, 14, 0, 16, 825, 191, 472, 2, 4987, 124, 30, 1318, 1348, 1, 0, 284, 607, 3000, 2, 1, 82, 353, 0, 0, 32, 22, 0, 867, 2107, 613, 0, 201] payload_rows_per_dst=[3754, 12094, 13710, 5874, 13600, 12652, 7010, 10564] dup_rows_per_dst=[398, 4870, 3810, 788, 1714, 3532, 1080, 2854] combine_rows_per_src=[3754, 12094, 13710, 5874, 13600, 12652, 7010, 10564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68034 payload_rows_remote=52442 dup_rows=30270 dup_rows_remote=9874 dedup_ratio=0.307922 router_tokens_per_expert=[3113, 2868, 3891, 5917, 1376, 11, 0, 3732, 948, 863, 115, 713, 0, 0, 323, 14, 9, 323, 0, 71, 12, 9, 0, 543, 211, 3, 139, 29, 0, 0, 2357, 1056, 46, 0, 0, 139, 522, 213, 771, 0, 4205, 321, 5, 155, 1824, 24, 0, 1817, 632, 5978, 18, 2, 14, 424, 0, 0, 1, 17, 28, 796, 1908, 612, 0, 34] payload_rows_per_dst=[15592, 5164, 1750, 12014, 10104, 7130, 7484, 8796] dup_rows_per_dst=[20396, 788, 184, 3986, 742, 1162, 1134, 1878] combine_rows_per_src=[15592, 5164, 1750, 12014, 10104, 7130, 7484, 8796] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=287522816 dispatch_meta_bytes=362176 combine_payload_bytes=287522816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=287884992 ingress_bytes=322125824 link_bytes=322125824 token_bytes=575407808 weight_movement_bytes=34603008 total_nvlink_bytes=610010816 weight_to_token_ratio=0.060136 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=34688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.212133542e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=278790144 dispatch_meta_bytes=338960 combine_payload_bytes=278790144 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=279129104 ingress_bytes=347996160 link_bytes=347996160 token_bytes=557919248 weight_movement_bytes=69206016 total_nvlink_bytes=627125264 weight_to_token_ratio=0.124043 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=29824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.309481534e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 2, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=295927808 dispatch_meta_bytes=360856 combine_payload_bytes=295927808 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=296288664 ingress_bytes=365133824 link_bytes=365133824 token_bytes=592216472 weight_movement_bytes=69206016 total_nvlink_bytes=661422488 weight_to_token_ratio=0.116859 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=39040 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.373969184e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79968 payload_rows_remote=73648 dup_rows=18336 dup_rows_remote=18148 dedup_ratio=0.186523 router_tokens_per_expert=[4323, 5506, 552, 1786, 2521, 3, 0, 27, 8, 193, 1, 2750, 53, 1, 236, 12, 0, 6, 344, 0, 0, 0, 0, 9, 0, 1, 8, 0, 0, 0, 4275, 871, 4, 0, 0, 1961, 391, 1, 1344, 0, 1576, 601, 0, 5, 1633, 1, 0, 7366, 552, 7611, 0, 201, 0, 952, 1, 0, 0, 2, 152, 790, 22, 497, 0, 3] payload_rows_per_dst=[14144, 6320, 924, 10610, 6872, 15250, 16030, 9818] dup_rows_per_dst=[6372, 188, 14, 2852, 584, 3962, 2604, 1760] combine_rows_per_src=[14144, 6320, 924, 10610, 6872, 15250, 16030, 9818] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=214802432 dispatch_meta_bytes=249264 combine_payload_bytes=214802432 prefetch_bytes=34603008 reduce_bytes=173015040 egress_bytes=215051696 ingress_bytes=422420480 link_bytes=422420480 token_bytes=429854128 weight_movement_bytes=207618048 total_nvlink_bytes=637472176 weight_to_token_ratio=0.482997 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.589534259e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81302 payload_rows_remote=72046 dup_rows=17002 dup_rows_remote=15308 dedup_ratio=0.172953 router_tokens_per_expert=[3632, 5505, 778, 4042, 3152, 1, 2, 543, 164, 680, 20, 1824, 13, 5, 284, 19, 1, 58, 180, 5, 0, 5, 0, 51, 12, 0, 48, 25, 2, 0, 3744, 1229, 25, 3, 4, 678, 419, 15, 1165, 1, 2281, 238, 1, 46, 1696, 6, 0, 5936, 609, 7346, 5, 119, 0, 687, 5, 0, 0, 8, 87, 774, 365, 608, 0, 1] payload_rows_per_dst=[11526, 6398, 8460, 8808, 7332, 14182, 15340, 9256] dup_rows_per_dst=[2804, 1176, 224, 1312, 1364, 5322, 3106, 1694] combine_rows_per_src=[11526, 6398, 8460, 8808, 7332, 14182, 15340, 9256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77216 payload_rows_remote=65852 dup_rows=21088 dup_rows_remote=17028 dedup_ratio=0.214518 router_tokens_per_expert=[238, 0, 44, 0, 2265, 458, 482, 19, 2, 3153, 150, 20, 317, 67, 900, 10, 7205, 3, 3563, 197, 1190, 2722, 36, 1, 0, 3216, 1, 384, 407, 1398, 2137, 59, 2432, 233, 445, 36, 226, 0, 6, 155, 91, 802, 50, 93, 1, 0, 332, 1956, 2255, 4, 0, 1314, 198, 0, 124, 1032, 343, 0, 547, 5002, 0, 1, 29, 801] payload_rows_per_dst=[10762, 15628, 11364, 7852, 6328, 5930, 8120, 11232] dup_rows_per_dst=[2682, 8020, 4060, 920, 738, 720, 1734, 2214] combine_rows_per_src=[10762, 15628, 11364, 7852, 6328, 5930, 8120, 11232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81060 payload_rows_remote=68736 dup_rows=17244 dup_rows_remote=12632 dedup_ratio=0.175415 router_tokens_per_expert=[3414, 1636, 2978, 6274, 1583, 20, 8, 3337, 1330, 1548, 326, 481, 0, 0, 415, 35, 27, 474, 0, 220, 17, 13, 0, 510, 428, 0, 97, 106, 0, 0, 2537, 1205, 41, 0, 0, 99, 887, 376, 735, 0, 4095, 186, 15, 603, 1685, 4, 0, 1827, 656, 5276, 0, 6, 9, 386, 0, 0, 21, 8, 3, 767, 1748, 628, 0, 72] payload_rows_per_dst=[5970, 10848, 12970, 12324, 9582, 7494, 11196, 10676] dup_rows_per_dst=[524, 3378, 2100, 4612, 1368, 1146, 1470, 2646] combine_rows_per_src=[5970, 10848, 12970, 12324, 9582, 7494, 11196, 10676] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=301662208 dispatch_meta_bytes=367184 combine_payload_bytes=301662208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=302029392 ingress_bytes=336265216 link_bytes=336265216 token_bytes=603691600 weight_movement_bytes=34603008 total_nvlink_bytes=638294608 weight_to_token_ratio=0.057319 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265338936e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=295100416 dispatch_meta_bytes=349416 combine_payload_bytes=295100416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=295449832 ingress_bytes=329703424 link_bytes=329703424 token_bytes=590550248 weight_movement_bytes=34603008 total_nvlink_bytes=625153256 weight_to_token_ratio=0.058595 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=23296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.240647441e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79314 payload_rows_remote=67314 dup_rows=18990 dup_rows_remote=16722 dedup_ratio=0.193176 router_tokens_per_expert=[1923, 951, 3963, 6397, 633, 15, 39, 5829, 2374, 1487, 455, 107, 0, 0, 615, 34, 40, 1158, 0, 479, 22, 55, 0, 1550, 538, 1, 70, 188, 0, 0, 1353, 786, 134, 4, 0, 9, 1159, 474, 402, 0, 4872, 60, 1, 1061, 1359, 2, 0, 377, 555, 3651, 0, 2, 0, 388, 0, 0, 1, 3, 0, 458, 2490, 569, 0, 59] payload_rows_per_dst=[3192, 12596, 14350, 6032, 13804, 12000, 8068, 9272] dup_rows_per_dst=[84, 5474, 5052, 1036, 2218, 2268, 1124, 1734] combine_rows_per_src=[3192, 12596, 14350, 6032, 13804, 12000, 8068, 9272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=269729792 dispatch_meta_bytes=331520 combine_payload_bytes=269729792 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=270061312 ingress_bytes=442744832 link_bytes=442744832 token_bytes=539791104 weight_movement_bytes=173015040 total_nvlink_bytes=712806144 weight_to_token_ratio=0.320522 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.666013159e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=281542656 dispatch_meta_bytes=325472 combine_payload_bytes=281542656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281868128 ingress_bytes=316145664 link_bytes=316145664 token_bytes=563410784 weight_movement_bytes=34603008 total_nvlink_bytes=598013792 weight_to_token_ratio=0.061417 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=18944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.189630682e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81302 payload_rows_remote=72046 dup_rows=17002 dup_rows_remote=15308 dedup_ratio=0.172953 router_tokens_per_expert=[3632, 5505, 778, 4042, 3152, 1, 2, 543, 164, 680, 20, 1824, 13, 5, 284, 19, 1, 58, 180, 5, 0, 5, 0, 51, 12, 0, 48, 25, 2, 0, 3744, 1229, 25, 3, 4, 678, 419, 15, 1165, 1, 2281, 238, 1, 46, 1696, 6, 0, 5936, 609, 7346, 5, 119, 0, 687, 5, 0, 0, 8, 87, 774, 365, 608, 0, 1] payload_rows_per_dst=[11526, 6398, 8460, 8808, 7332, 14182, 15340, 9256] dup_rows_per_dst=[2804, 1176, 224, 1312, 1364, 5322, 3106, 1694] combine_rows_per_src=[11526, 6398, 8460, 8808, 7332, 14182, 15340, 9256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78438 payload_rows_remote=63614 dup_rows=19866 dup_rows_remote=16324 dedup_ratio=0.202087 router_tokens_per_expert=[3, 1, 28, 662, 1123, 1139, 0, 74, 86, 1709, 0, 7095, 0, 2, 13, 7446, 81, 3007, 10, 819, 102, 5, 2047, 0, 199, 165, 58, 699, 731, 8, 220, 926, 837, 619, 58, 167, 10, 205, 192, 0, 463, 11, 494, 768, 95, 1101, 3343, 87, 815, 472, 42, 785, 415, 976, 1875, 304, 220, 2011, 858, 39, 238, 336, 48, 2810] payload_rows_per_dst=[5600, 4676, 9296, 15242, 14824, 9890, 9514, 9396] dup_rows_per_dst=[460, 138, 2750, 4468, 3542, 2930, 1854, 3724] combine_rows_per_src=[5600, 4676, 9296, 15242, 14824, 9890, 9514, 9396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76842 payload_rows_remote=66444 dup_rows=21462 dup_rows_remote=18736 dedup_ratio=0.218323 router_tokens_per_expert=[4, 1, 4, 1235, 1213, 726, 0, 75, 35, 1354, 1, 6976, 0, 0, 7, 7378, 84, 2729, 19, 343, 20, 7, 1402, 0, 390, 225, 152, 941, 503, 23, 173, 1265, 416, 752, 16, 311, 10, 85, 320, 2, 564, 3, 762, 1027, 22, 561, 3856, 76, 595, 390, 24, 1173, 905, 621, 2600, 254, 220, 2003, 649, 15, 264, 320, 3, 3048] payload_rows_per_dst=[5950, 2778, 6172, 15824, 14422, 11996, 10398, 9302] dup_rows_per_dst=[566, 16, 232, 6276, 3354, 4550, 2726, 3742] combine_rows_per_src=[5950, 2778, 6172, 15824, 14422, 11996, 10398, 9302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76320 payload_rows_remote=60148 dup_rows=21984 dup_rows_remote=16510 dedup_ratio=0.223633 router_tokens_per_expert=[0, 4, 585, 281, 733, 1107, 0, 75, 7, 1559, 0, 5547, 0, 0, 2, 8038, 15, 2877, 16, 1652, 39, 2, 2953, 0, 21, 164, 1, 799, 508, 7, 236, 103, 2036, 617, 0, 68, 3, 27, 62, 0, 19, 0, 177, 75, 192, 1021, 2610, 109, 1519, 987, 50, 606, 123, 1906, 835, 92, 50, 3694, 613, 3, 273, 129, 0, 3925] payload_rows_per_dst=[16172, 12770, 8610, 3306, 5476, 7626, 13190, 9170] dup_rows_per_dst=[5474, 4788, 3170, 372, 150, 780, 6434, 816] combine_rows_per_src=[16172, 12770, 8610, 3306, 5476, 7626, 13190, 9170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=275718144 dispatch_meta_bytes=336144 combine_payload_bytes=275718144 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=276054288 ingress_bytes=344924160 link_bytes=344924160 token_bytes=551772432 weight_movement_bytes=69206016 total_nvlink_bytes=620978448 weight_to_token_ratio=0.125425 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=18176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.297921845e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=260562944 dispatch_meta_bytes=319752 combine_payload_bytes=260562944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260882696 ingress_bytes=295165952 link_bytes=295165952 token_bytes=521445640 weight_movement_bytes=34603008 total_nvlink_bytes=556048648 weight_to_token_ratio=0.066360 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=23808 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.110685715e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=272154624 dispatch_meta_bytes=340720 combine_payload_bytes=272154624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272495344 ingress_bytes=306757632 link_bytes=306757632 token_bytes=544649968 weight_movement_bytes=34603008 total_nvlink_bytes=579252976 weight_to_token_ratio=0.063533 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.154304273e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71562 payload_rows_remote=63282 dup_rows=26742 dup_rows_remote=26288 dedup_ratio=0.272034 router_tokens_per_expert=[0, 0, 5327, 4, 300, 149, 6, 526, 0, 3907, 0, 459, 0, 0, 1, 7941, 0, 34, 1, 6536, 1, 0, 6017, 0, 2, 1, 0, 258, 12, 0, 1, 2, 5741, 60, 0, 0, 0, 949, 0, 0, 2, 0, 0, 9, 179, 2662, 67, 181, 191, 106, 14, 125, 1, 2772, 3, 0, 2, 3116, 0, 0, 121, 0, 0, 1366] payload_rows_per_dst=[16288, 8280, 15112, 552, 11892, 5950, 10306, 3182] dup_rows_per_dst=[12218, 454, 10066, 0, 1608, 250, 1742, 404] combine_rows_per_src=[16288, 8280, 15112, 552, 11892, 5950, 10306, 3182] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68034 payload_rows_remote=52442 dup_rows=30270 dup_rows_remote=9874 dedup_ratio=0.307922 router_tokens_per_expert=[3113, 2868, 3891, 5917, 1376, 11, 0, 3732, 948, 863, 115, 713, 0, 0, 323, 14, 9, 323, 0, 71, 12, 9, 0, 543, 211, 3, 139, 29, 0, 0, 2357, 1056, 46, 0, 0, 139, 522, 213, 771, 0, 4205, 321, 5, 155, 1824, 24, 0, 1817, 632, 5978, 18, 2, 14, 424, 0, 0, 1, 17, 28, 796, 1908, 612, 0, 34] payload_rows_per_dst=[15592, 5164, 1750, 12014, 10104, 7130, 7484, 8796] dup_rows_per_dst=[20396, 788, 184, 3986, 742, 1162, 1134, 1878] combine_rows_per_src=[15592, 5164, 1750, 12014, 10104, 7130, 7484, 8796] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73448 payload_rows_remote=64908 dup_rows=24856 dup_rows_remote=23266 dedup_ratio=0.252848 router_tokens_per_expert=[0, 5, 5, 1183, 837, 1057, 0, 72, 68, 1220, 0, 6669, 0, 0, 18, 7868, 16, 3448, 36, 466, 73, 18, 1007, 1, 181, 285, 15, 761, 846, 4, 297, 683, 587, 464, 37, 203, 3, 0, 315, 0, 201, 3, 557, 572, 54, 564, 4660, 89, 722, 670, 78, 701, 118, 1045, 2506, 159, 280, 3168, 652, 239, 300, 258, 6, 2802] payload_rows_per_dst=[9576, 15314, 8540, 5418, 2976, 11066, 9862, 10696] dup_rows_per_dst=[3178, 9936, 1590, 726, 242, 2334, 2136, 4714] combine_rows_per_src=[9576, 15314, 8540, 5418, 2976, 11066, 9862, 10696] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83570 payload_rows_remote=72964 dup_rows=14734 dup_rows_remote=10550 dedup_ratio=0.149882 router_tokens_per_expert=[0, 0, 3416, 39, 362, 419, 0, 511, 16, 3541, 1, 1623, 8, 0, 0, 8169, 0, 570, 24, 4695, 0, 1, 6030, 0, 15, 18, 0, 922, 15, 0, 14, 26, 4885, 428, 0, 13, 9, 371, 10, 1, 70, 0, 41, 9, 253, 2673, 302, 180, 479, 290, 13, 181, 66, 1936, 151, 17, 10, 3517, 96, 4, 198, 17, 0, 2497] payload_rows_per_dst=[8840, 7132, 10086, 16344, 12450, 12278, 5834, 10606] dup_rows_per_dst=[654, 0, 494, 2014, 2230, 4726, 432, 4184] combine_rows_per_src=[8840, 7132, 10086, 16344, 12450, 12278, 5834, 10606] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=246366208 dispatch_meta_bytes=306632 combine_payload_bytes=246366208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=246672840 ingress_bytes=280969216 link_bytes=280969216 token_bytes=493039048 weight_movement_bytes=34603008 total_nvlink_bytes=527642056 weight_to_token_ratio=0.070183 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=19200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.057264540e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 2, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79258 payload_rows_remote=72248 dup_rows=19046 dup_rows_remote=17966 dedup_ratio=0.193746 router_tokens_per_expert=[2867, 833, 3383, 5996, 1143, 36, 64, 5866, 2503, 1584, 391, 32, 0, 0, 559, 30, 89, 1134, 0, 416, 30, 37, 0, 1058, 981, 1, 69, 163, 0, 0, 1313, 804, 271, 14, 0, 16, 825, 191, 472, 2, 4987, 124, 30, 1318, 1348, 1, 0, 284, 607, 3000, 2, 1, 82, 353, 0, 0, 32, 22, 0, 867, 2107, 613, 0, 201] payload_rows_per_dst=[3754, 12094, 13710, 5874, 13600, 12652, 7010, 10564] dup_rows_per_dst=[398, 4870, 3810, 788, 1714, 3532, 1080, 2854] combine_rows_per_src=[3754, 12094, 13710, 5874, 13600, 12652, 7010, 10564] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80144 payload_rows_remote=68064 dup_rows=18160 dup_rows_remote=16676 dedup_ratio=0.184733 router_tokens_per_expert=[2804, 1025, 3602, 5514, 1528, 60, 54, 4714, 1671, 1513, 319, 168, 0, 0, 562, 27, 70, 1150, 0, 653, 54, 57, 0, 752, 725, 8, 115, 111, 0, 0, 1052, 667, 231, 11, 12, 13, 835, 196, 768, 2, 3532, 251, 58, 879, 1663, 2, 1, 670, 899, 5117, 3, 3, 47, 411, 0, 0, 143, 26, 3, 825, 2845, 596, 0, 135] payload_rows_per_dst=[4718, 11556, 12846, 10174, 12080, 6086, 11186, 11498] dup_rows_per_dst=[616, 4168, 3654, 2246, 1484, 962, 1774, 3256] combine_rows_per_src=[4718, 11556, 12846, 10174, 12080, 6086, 11186, 11498] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76946 payload_rows_remote=65086 dup_rows=21358 dup_rows_remote=17330 dedup_ratio=0.217265 router_tokens_per_expert=[1, 1, 11, 978, 2065, 825, 0, 77, 27, 817, 0, 6959, 0, 0, 4, 7514, 130, 2649, 18, 384, 41, 4, 1212, 0, 475, 409, 35, 1464, 419, 1, 233, 863, 396, 561, 8, 269, 9, 23, 520, 2, 477, 6, 651, 650, 41, 914, 3915, 78, 793, 696, 38, 737, 608, 1044, 2751, 208, 232, 2021, 934, 62, 254, 281, 0, 2357] payload_rows_per_dst=[7090, 1694, 6172, 15742, 14386, 11860, 11000, 9002] dup_rows_per_dst=[826, 2, 280, 7084, 3108, 4028, 2750, 3280] combine_rows_per_src=[7090, 1694, 6172, 15742, 14386, 11860, 11000, 9002] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=295100416 dispatch_meta_bytes=349416 combine_payload_bytes=295100416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=295449832 ingress_bytes=329703424 link_bytes=329703424 token_bytes=590550248 weight_movement_bytes=34603008 total_nvlink_bytes=625153256 weight_to_token_ratio=0.058595 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=23296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.240647441e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=259203072 dispatch_meta_bytes=358280 combine_payload_bytes=259203072 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=259561352 ingress_bytes=397615104 link_bytes=397615104 token_bytes=518764424 weight_movement_bytes=138412032 total_nvlink_bytes=657176456 weight_to_token_ratio=0.266811 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=18048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.496193626e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=214802432 dispatch_meta_bytes=249264 combine_payload_bytes=214802432 prefetch_bytes=34603008 reduce_bytes=173015040 egress_bytes=215051696 ingress_bytes=422420480 link_bytes=422420480 token_bytes=429854128 weight_movement_bytes=207618048 total_nvlink_bytes=637472176 weight_to_token_ratio=0.482997 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.589534259e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81090 payload_rows_remote=67362 dup_rows=17214 dup_rows_remote=12686 dedup_ratio=0.175110 router_tokens_per_expert=[2385, 751, 4503, 5997, 597, 30, 15, 5652, 2153, 1504, 447, 27, 0, 0, 709, 14, 46, 998, 0, 455, 8, 51, 0, 1573, 552, 0, 64, 100, 0, 0, 1317, 941, 215, 30, 0, 14, 1037, 225, 555, 0, 4844, 179, 14, 587, 1497, 2, 0, 299, 588, 3701, 0, 2, 40, 518, 0, 0, 23, 33, 1, 807, 2188, 608, 0, 256] payload_rows_per_dst=[7842, 10332, 13728, 12352, 13780, 4662, 8454, 9940] dup_rows_per_dst=[372, 2954, 4528, 3284, 1676, 494, 1244, 2662] combine_rows_per_src=[7842, 10332, 13728, 12352, 13780, 4662, 8454, 9940] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70320 payload_rows_remote=65756 dup_rows=27984 dup_rows_remote=27612 dedup_ratio=0.284668 router_tokens_per_expert=[0, 1, 330, 618, 700, 981, 0, 76, 35, 2041, 1, 6437, 0, 0, 4, 7745, 163, 2379, 5, 1528, 34, 6, 3752, 0, 107, 95, 53, 1068, 350, 4, 194, 597, 1539, 677, 6, 147, 8, 233, 112, 0, 178, 3, 332, 431, 165, 1037, 1785, 89, 784, 560, 45, 617, 372, 1246, 1289, 150, 98, 3084, 762, 16, 226, 163, 0, 3694] payload_rows_per_dst=[5120, 16294, 11116, 4564, 6278, 6978, 8668, 11302] dup_rows_per_dst=[292, 14032, 4618, 372, 1366, 1062, 1458, 4784] combine_rows_per_src=[5120, 16294, 11116, 4564, 6278, 6978, 8668, 11302] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265863168 dispatch_meta_bytes=352696 combine_payload_bytes=265863168 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266215864 ingress_bytes=369672192 link_bytes=369672192 token_bytes=532079032 weight_movement_bytes=103809024 total_nvlink_bytes=635888056 weight_to_token_ratio=0.195101 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=16512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391046697e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=295927808 dispatch_meta_bytes=360856 combine_payload_bytes=295927808 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=296288664 ingress_bytes=365133824 link_bytes=365133824 token_bytes=592216472 weight_movement_bytes=69206016 total_nvlink_bytes=661422488 weight_to_token_ratio=0.116859 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=39040 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.373969184e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=278790144 dispatch_meta_bytes=338960 combine_payload_bytes=278790144 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=279129104 ingress_bytes=347996160 link_bytes=347996160 token_bytes=557919248 weight_movement_bytes=69206016 total_nvlink_bytes=627125264 weight_to_token_ratio=0.124043 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=29824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.309481534e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=259203072 dispatch_meta_bytes=358280 combine_payload_bytes=259203072 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=259561352 ingress_bytes=397615104 link_bytes=397615104 token_bytes=518764424 weight_movement_bytes=138412032 total_nvlink_bytes=657176456 weight_to_token_ratio=0.266811 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=18048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.496193626e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=298860544 dispatch_meta_bytes=334056 combine_payload_bytes=298860544 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=299194600 ingress_bytes=368066560 link_bytes=368066560 token_bytes=598055144 weight_movement_bytes=69206016 total_nvlink_bytes=667261160 weight_to_token_ratio=0.115718 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=22528 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.385004833e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266592256 dispatch_meta_bytes=329664 combine_payload_bytes=266592256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266921920 ingress_bytes=301195264 link_bytes=301195264 token_bytes=533514176 weight_movement_bytes=34603008 total_nvlink_bytes=568117184 weight_to_token_ratio=0.064859 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133373530e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=275914752 dispatch_meta_bytes=320192 combine_payload_bytes=275914752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276234944 ingress_bytes=310517760 link_bytes=310517760 token_bytes=552149696 weight_movement_bytes=34603008 total_nvlink_bytes=586752704 weight_to_token_ratio=0.062670 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=34432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.168453332e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82222 payload_rows_remote=77388 dup_rows=16082 dup_rows_remote=15716 dedup_ratio=0.163595 router_tokens_per_expert=[1286, 206, 61, 363, 2158, 406, 0, 0, 0, 206, 391, 89, 4481, 7, 873, 355, 0, 727, 85, 4615, 0, 737, 248, 7, 1646, 0, 602, 2621, 1, 2443, 468, 687, 4382, 463, 701, 384, 451, 473, 40, 789, 0, 367, 557, 257, 651, 260, 1266, 66, 3041, 0, 306, 541, 550, 732, 1581, 602, 731, 3763, 185, 0, 12, 0, 122, 110] payload_rows_per_dst=[9534, 11552, 10988, 11460, 4834, 12540, 11610, 9704] dup_rows_per_dst=[2718, 1252, 1850, 2184, 366, 3072, 3096, 1544] combine_rows_per_src=[9534, 11552, 10988, 11460, 4834, 12540, 11610, 9704] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76946 payload_rows_remote=65086 dup_rows=21358 dup_rows_remote=17330 dedup_ratio=0.217265 router_tokens_per_expert=[1, 1, 11, 978, 2065, 825, 0, 77, 27, 817, 0, 6959, 0, 0, 4, 7514, 130, 2649, 18, 384, 41, 4, 1212, 0, 475, 409, 35, 1464, 419, 1, 233, 863, 396, 561, 8, 269, 9, 23, 520, 2, 477, 6, 651, 650, 41, 914, 3915, 78, 793, 696, 38, 737, 608, 1044, 2751, 208, 232, 2021, 934, 62, 254, 281, 0, 2357] payload_rows_per_dst=[7090, 1694, 6172, 15742, 14386, 11860, 11000, 9002] dup_rows_per_dst=[826, 2, 280, 7084, 3108, 4028, 2750, 3280] combine_rows_per_src=[7090, 1694, 6172, 15742, 14386, 11860, 11000, 9002] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=269336576 dispatch_meta_bytes=373472 combine_payload_bytes=269336576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269710048 ingress_bytes=303939584 link_bytes=303939584 token_bytes=539046624 weight_movement_bytes=34603008 total_nvlink_bytes=573649632 weight_to_token_ratio=0.064193 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=29952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.143700185e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83570 payload_rows_remote=72964 dup_rows=14734 dup_rows_remote=10550 dedup_ratio=0.149882 router_tokens_per_expert=[0, 0, 3416, 39, 362, 419, 0, 511, 16, 3541, 1, 1623, 8, 0, 0, 8169, 0, 570, 24, 4695, 0, 1, 6030, 0, 15, 18, 0, 922, 15, 0, 14, 26, 4885, 428, 0, 13, 9, 371, 10, 1, 70, 0, 41, 9, 253, 2673, 302, 180, 479, 290, 13, 181, 66, 1936, 151, 17, 10, 3517, 96, 4, 198, 17, 0, 2497] payload_rows_per_dst=[8840, 7132, 10086, 16344, 12450, 12278, 5834, 10606] dup_rows_per_dst=[654, 0, 494, 2014, 2230, 4726, 432, 4184] combine_rows_per_src=[8840, 7132, 10086, 16344, 12450, 12278, 5834, 10606] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81556 payload_rows_remote=68170 dup_rows=16748 dup_rows_remote=10822 dedup_ratio=0.170369 router_tokens_per_expert=[2018, 14, 18, 183, 1808, 294, 1, 1, 0, 254, 278, 100, 3591, 9, 1552, 441, 0, 751, 16, 4291, 0, 686, 121, 0, 1068, 0, 958, 2165, 2, 1564, 752, 809, 2958, 128, 395, 332, 104, 491, 24, 1170, 0, 312, 340, 206, 1281, 248, 944, 0, 4185, 0, 221, 799, 571, 1535, 1666, 679, 816, 5846, 66, 0, 4, 0, 2, 84] payload_rows_per_dst=[8814, 10904, 10354, 10606, 4272, 10532, 13386, 12688] dup_rows_per_dst=[1996, 1546, 1376, 1894, 226, 2046, 5926, 1738] combine_rows_per_src=[8814, 10904, 10354, 10606, 4272, 10532, 13386, 12688] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71796 payload_rows_remote=61144 dup_rows=26508 dup_rows_remote=22526 dedup_ratio=0.269653 router_tokens_per_expert=[973, 488, 109, 131, 2490, 725, 0, 0, 0, 89, 459, 28, 5931, 0, 309, 131, 0, 186, 2, 3306, 0, 713, 89, 37, 2401, 0, 128, 2556, 4, 3060, 238, 763, 7207, 1008, 2340, 83, 1533, 554, 44, 347, 0, 685, 350, 24, 258, 93, 369, 91, 3048, 0, 601, 140, 403, 369, 490, 728, 237, 1867, 712, 3, 2, 0, 105, 115] payload_rows_per_dst=[10652, 13318, 11244, 6756, 15566, 3442, 5014, 5804] dup_rows_per_dst=[3982, 576, 3542, 622, 16762, 298, 448, 278] combine_rows_per_src=[10652, 13318, 11244, 6756, 15566, 3442, 5014, 5804] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=316981248 dispatch_meta_bytes=372416 combine_payload_bytes=316981248 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=317353664 ingress_bytes=420790272 link_bytes=420790272 token_bytes=634334912 weight_movement_bytes=103809024 total_nvlink_bytes=738143936 weight_to_token_ratio=0.163650 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.583399917e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266592256 dispatch_meta_bytes=329664 combine_payload_bytes=266592256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266921920 ingress_bytes=301195264 link_bytes=301195264 token_bytes=533514176 weight_movement_bytes=34603008 total_nvlink_bytes=568117184 weight_to_token_ratio=0.064859 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133373530e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73528 payload_rows_remote=59778 dup_rows=24776 dup_rows_remote=24362 dedup_ratio=0.252035 router_tokens_per_expert=[5, 385, 98, 1442, 34, 787, 0, 0, 23, 1, 428, 0, 6628, 0, 1, 1, 0, 0, 0, 77, 1, 4, 48, 1301, 6434, 0, 3, 687, 10, 3697, 4, 1, 8180, 2133, 4787, 0, 4593, 0, 2, 2, 0, 417, 41, 1, 1, 2, 3, 173, 1114, 0, 802, 7, 450, 8, 9, 336, 2, 24, 1114, 2728, 0, 56, 1, 66] payload_rows_per_dst=[14534, 13750, 9132, 1406, 14760, 6704, 3694, 9548] dup_rows_per_dst=[3836, 414, 1124, 4, 16790, 500, 650, 1458] combine_rows_per_src=[14534, 13750, 9132, 1406, 14760, 6704, 3694, 9548] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76320 payload_rows_remote=60148 dup_rows=21984 dup_rows_remote=16510 dedup_ratio=0.223633 router_tokens_per_expert=[0, 4, 585, 281, 733, 1107, 0, 75, 7, 1559, 0, 5547, 0, 0, 2, 8038, 15, 2877, 16, 1652, 39, 2, 2953, 0, 21, 164, 1, 799, 508, 7, 236, 103, 2036, 617, 0, 68, 3, 27, 62, 0, 19, 0, 177, 75, 192, 1021, 2610, 109, 1519, 987, 50, 606, 123, 1906, 835, 92, 50, 3694, 613, 3, 273, 129, 0, 3925] payload_rows_per_dst=[16172, 12770, 8610, 3306, 5476, 7626, 13190, 9170] dup_rows_per_dst=[5474, 4788, 3170, 372, 150, 780, 6434, 816] combine_rows_per_src=[16172, 12770, 8610, 3306, 5476, 7626, 13190, 9170] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79822 payload_rows_remote=66230 dup_rows=18482 dup_rows_remote=12906 dedup_ratio=0.188009 router_tokens_per_expert=[1705, 167, 28, 103, 2507, 297, 0, 0, 0, 256, 361, 71, 4313, 0, 845, 378, 0, 1039, 0, 5499, 0, 838, 168, 0, 351, 0, 269, 1883, 9, 2040, 438, 1040, 3332, 439, 665, 266, 152, 1113, 6, 617, 0, 404, 492, 174, 569, 215, 994, 9, 3621, 1, 328, 733, 219, 1017, 1437, 760, 1014, 5723, 119, 0, 18, 2, 2, 106] payload_rows_per_dst=[8426, 10952, 13592, 6762, 4832, 10192, 12056, 13010] dup_rows_per_dst=[1890, 1496, 5576, 516, 354, 2186, 4176, 2288] combine_rows_per_src=[8426, 10952, 13592, 6762, 4832, 10192, 12056, 13010] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=279224320 dispatch_meta_bytes=315968 combine_payload_bytes=279224320 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=279540288 ingress_bytes=348430336 link_bytes=348430336 token_bytes=558764608 weight_movement_bytes=69206016 total_nvlink_bytes=627970624 weight_to_token_ratio=0.123855 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.311115303e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73312 payload_rows_remote=62186 dup_rows=24992 dup_rows_remote=21660 dedup_ratio=0.254232 router_tokens_per_expert=[207, 462, 110, 1188, 349, 880, 2, 1, 1, 24, 696, 1, 6974, 0, 43, 23, 0, 16, 0, 671, 0, 86, 166, 752, 5857, 0, 1, 1777, 3, 4333, 16, 109, 8048, 1122, 4322, 14, 3719, 63, 139, 28, 0, 167, 231, 1, 49, 2, 68, 143, 1979, 1, 438, 53, 567, 39, 48, 256, 70, 224, 1279, 1253, 4, 1, 3, 73] payload_rows_per_dst=[5638, 5674, 3180, 15610, 9062, 16266, 6756, 11126] dup_rows_per_dst=[852, 416, 202, 17230, 1108, 1152, 700, 3332] combine_rows_per_src=[5638, 5674, 3180, 15610, 9062, 16266, 6756, 11126] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=250445824 dispatch_meta_bytes=334680 combine_payload_bytes=250445824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=250780504 ingress_bytes=285048832 link_bytes=285048832 token_bytes=501226328 weight_movement_bytes=34603008 total_nvlink_bytes=535829336 weight_to_token_ratio=0.069037 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.072615807e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76842 payload_rows_remote=66444 dup_rows=21462 dup_rows_remote=18736 dedup_ratio=0.218323 router_tokens_per_expert=[4, 1, 4, 1235, 1213, 726, 0, 75, 35, 1354, 1, 6976, 0, 0, 7, 7378, 84, 2729, 19, 343, 20, 7, 1402, 0, 390, 225, 152, 941, 503, 23, 173, 1265, 416, 752, 16, 311, 10, 85, 320, 2, 564, 3, 762, 1027, 22, 561, 3856, 76, 595, 390, 24, 1173, 905, 621, 2600, 254, 220, 2003, 649, 15, 264, 320, 3, 3048] payload_rows_per_dst=[5950, 2778, 6172, 15824, 14422, 11996, 10398, 9302] dup_rows_per_dst=[566, 16, 232, 6276, 3354, 4550, 2726, 3742] combine_rows_per_src=[5950, 2778, 6172, 15824, 14422, 11996, 10398, 9302] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78438 payload_rows_remote=63614 dup_rows=19866 dup_rows_remote=16324 dedup_ratio=0.202087 router_tokens_per_expert=[3, 1, 28, 662, 1123, 1139, 0, 74, 86, 1709, 0, 7095, 0, 2, 13, 7446, 81, 3007, 10, 819, 102, 5, 2047, 0, 199, 165, 58, 699, 731, 8, 220, 926, 837, 619, 58, 167, 10, 205, 192, 0, 463, 11, 494, 768, 95, 1101, 3343, 87, 815, 472, 42, 785, 415, 976, 1875, 304, 220, 2011, 858, 39, 238, 336, 48, 2810] payload_rows_per_dst=[5600, 4676, 9296, 15242, 14824, 9890, 9514, 9396] dup_rows_per_dst=[460, 138, 2750, 4468, 3542, 2930, 1854, 3724] combine_rows_per_src=[5600, 4676, 9296, 15242, 14824, 9890, 9514, 9396] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=298860544 dispatch_meta_bytes=334056 combine_payload_bytes=298860544 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=299194600 ingress_bytes=368066560 link_bytes=368066560 token_bytes=598055144 weight_movement_bytes=69206016 total_nvlink_bytes=667261160 weight_to_token_ratio=0.115718 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=22528 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.385004833e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82084 payload_rows_remote=70998 dup_rows=16220 dup_rows_remote=13924 dedup_ratio=0.164998 router_tokens_per_expert=[1858, 101, 46, 123, 2345, 204, 0, 0, 0, 232, 264, 148, 3933, 0, 963, 369, 0, 1112, 0, 4723, 0, 600, 183, 0, 1204, 0, 509, 2528, 0, 1414, 1135, 844, 3441, 231, 445, 117, 158, 419, 37, 885, 0, 341, 335, 69, 966, 284, 1252, 3, 3747, 0, 451, 705, 211, 1143, 1093, 640, 1083, 5990, 125, 0, 27, 0, 7, 109] payload_rows_per_dst=[9260, 10516, 11454, 10824, 3586, 11086, 12214, 13144] dup_rows_per_dst=[2502, 1302, 1782, 2036, 108, 2296, 3766, 2428] combine_rows_per_src=[9260, 10516, 11454, 10824, 3586, 11086, 12214, 13144] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=244850688 dispatch_meta_bytes=336560 combine_payload_bytes=244850688 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=245187248 ingress_bytes=314056704 link_bytes=314056704 token_bytes=490037936 weight_movement_bytes=69206016 total_nvlink_bytes=559243952 weight_to_token_ratio=0.141226 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=36864 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.181770094e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=246366208 dispatch_meta_bytes=306632 combine_payload_bytes=246366208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=246672840 ingress_bytes=280969216 link_bytes=280969216 token_bytes=493039048 weight_movement_bytes=34603008 total_nvlink_bytes=527642056 weight_to_token_ratio=0.070183 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=19200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.057264540e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73448 payload_rows_remote=64908 dup_rows=24856 dup_rows_remote=23266 dedup_ratio=0.252848 router_tokens_per_expert=[0, 5, 5, 1183, 837, 1057, 0, 72, 68, 1220, 0, 6669, 0, 0, 18, 7868, 16, 3448, 36, 466, 73, 18, 1007, 1, 181, 285, 15, 761, 846, 4, 297, 683, 587, 464, 37, 203, 3, 0, 315, 0, 201, 3, 557, 572, 54, 564, 4660, 89, 722, 670, 78, 701, 118, 1045, 2506, 159, 280, 3168, 652, 239, 300, 258, 6, 2802] payload_rows_per_dst=[9576, 15314, 8540, 5418, 2976, 11066, 9862, 10696] dup_rows_per_dst=[3178, 9936, 1590, 726, 242, 2334, 2136, 4714] combine_rows_per_src=[9576, 15314, 8540, 5418, 2976, 11066, 9862, 10696] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83712 payload_rows_remote=74414 dup_rows=14592 dup_rows_remote=13496 dedup_ratio=0.148438 router_tokens_per_expert=[1277, 181, 125, 385, 1236, 495, 0, 0, 0, 242, 394, 200, 5637, 5, 695, 145, 0, 236, 29, 4073, 0, 465, 115, 31, 3136, 0, 354, 2793, 4, 2397, 438, 716, 5904, 559, 1001, 110, 941, 312, 83, 834, 0, 360, 364, 72, 495, 160, 602, 39, 3487, 0, 395, 427, 586, 718, 1167, 626, 427, 2904, 623, 2, 4, 0, 0, 146] payload_rows_per_dst=[10788, 13346, 10270, 9298, 5346, 13854, 11682, 9128] dup_rows_per_dst=[2882, 1290, 2638, 1096, 332, 2138, 3130, 1086] combine_rows_per_src=[10788, 13346, 10270, 9298, 5346, 13854, 11682, 9128] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=271278080 dispatch_meta_bytes=316544 combine_payload_bytes=271278080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271594624 ingress_bytes=305881088 link_bytes=305881088 token_bytes=542872704 weight_movement_bytes=34603008 total_nvlink_bytes=577475712 weight_to_token_ratio=0.063741 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=27264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.151005909e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=272154624 dispatch_meta_bytes=340720 combine_payload_bytes=272154624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272495344 ingress_bytes=306757632 link_bytes=306757632 token_bytes=544649968 weight_movement_bytes=34603008 total_nvlink_bytes=579252976 weight_to_token_ratio=0.063533 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.154304273e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=260562944 dispatch_meta_bytes=319752 combine_payload_bytes=260562944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260882696 ingress_bytes=295165952 link_bytes=295165952 token_bytes=521445640 weight_movement_bytes=34603008 total_nvlink_bytes=556048648 weight_to_token_ratio=0.066360 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=23808 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.110685715e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=244850688 dispatch_meta_bytes=336560 combine_payload_bytes=244850688 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=245187248 ingress_bytes=314056704 link_bytes=314056704 token_bytes=490037936 weight_movement_bytes=69206016 total_nvlink_bytes=559243952 weight_to_token_ratio=0.141226 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=36864 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.181770094e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=254713856 dispatch_meta_bytes=335384 combine_payload_bytes=254713856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=255049240 ingress_bytes=289316864 link_bytes=289316864 token_bytes=509763096 weight_movement_bytes=34603008 total_nvlink_bytes=544366104 weight_to_token_ratio=0.067881 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=26368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.088676067e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=290807808 dispatch_meta_bytes=339688 combine_payload_bytes=290807808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291147496 ingress_bytes=325410816 link_bytes=325410816 token_bytes=581955304 weight_movement_bytes=34603008 total_nvlink_bytes=616558312 weight_to_token_ratio=0.059460 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=31104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.224494703e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79980 payload_rows_remote=67928 dup_rows=18324 dup_rows_remote=15258 dedup_ratio=0.186401 router_tokens_per_expert=[0, 6, 50, 70, 8, 13, 16, 6, 743, 43, 1443, 0, 1622, 55, 948, 9, 1740, 952, 811, 119, 56, 865, 4, 3549, 0, 507, 140, 909, 6177, 19, 0, 1498, 2945, 6, 3, 0, 2672, 351, 1806, 3, 1, 19, 7129, 1867, 0, 969, 0, 10, 2, 469, 3683, 0, 377, 14, 4, 20, 20, 219, 2969, 2, 1045, 52, 0, 117] payload_rows_per_dst=[12448, 8396, 8226, 5818, 12052, 5650, 11932, 15458] dup_rows_per_dst=[244, 1330, 868, 782, 3066, 82, 4304, 7648] combine_rows_per_src=[12448, 8396, 8226, 5818, 12052, 5650, 11932, 15458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265863168 dispatch_meta_bytes=352696 combine_payload_bytes=265863168 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266215864 ingress_bytes=369672192 link_bytes=369672192 token_bytes=532079032 weight_movement_bytes=103809024 total_nvlink_bytes=635888056 weight_to_token_ratio=0.195101 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=16512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391046697e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82084 payload_rows_remote=70998 dup_rows=16220 dup_rows_remote=13924 dedup_ratio=0.164998 router_tokens_per_expert=[1858, 101, 46, 123, 2345, 204, 0, 0, 0, 232, 264, 148, 3933, 0, 963, 369, 0, 1112, 0, 4723, 0, 600, 183, 0, 1204, 0, 509, 2528, 0, 1414, 1135, 844, 3441, 231, 445, 117, 158, 419, 37, 885, 0, 341, 335, 69, 966, 284, 1252, 3, 3747, 0, 451, 705, 211, 1143, 1093, 640, 1083, 5990, 125, 0, 27, 0, 7, 109] payload_rows_per_dst=[9260, 10516, 11454, 10824, 3586, 11086, 12214, 13144] dup_rows_per_dst=[2502, 1302, 1782, 2036, 108, 2296, 3766, 2428] combine_rows_per_src=[9260, 10516, 11454, 10824, 3586, 11086, 12214, 13144] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=304799744 dispatch_meta_bytes=351640 combine_payload_bytes=304799744 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=305151384 ingress_bytes=408608768 link_bytes=408608768 token_bytes=609951128 weight_movement_bytes=103809024 total_nvlink_bytes=713760152 weight_to_token_ratio=0.170192 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.537561898e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73312 payload_rows_remote=62186 dup_rows=24992 dup_rows_remote=21660 dedup_ratio=0.254232 router_tokens_per_expert=[207, 462, 110, 1188, 349, 880, 2, 1, 1, 24, 696, 1, 6974, 0, 43, 23, 0, 16, 0, 671, 0, 86, 166, 752, 5857, 0, 1, 1777, 3, 4333, 16, 109, 8048, 1122, 4322, 14, 3719, 63, 139, 28, 0, 167, 231, 1, 49, 2, 68, 143, 1979, 1, 438, 53, 567, 39, 48, 256, 70, 224, 1279, 1253, 4, 1, 3, 73] payload_rows_per_dst=[5638, 5674, 3180, 15610, 9062, 16266, 6756, 11126] dup_rows_per_dst=[852, 416, 202, 17230, 1108, 1152, 700, 3332] combine_rows_per_src=[5638, 5674, 3180, 15610, 9062, 16266, 6756, 11126] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79022 payload_rows_remote=70310 dup_rows=19282 dup_rows_remote=17192 dedup_ratio=0.196147 router_tokens_per_expert=[0, 1, 341, 165, 4, 4, 7, 3, 694, 76, 2336, 0, 2807, 8, 378, 6, 1550, 669, 783, 61, 19, 1148, 4, 1848, 0, 471, 175, 1581, 6172, 3, 0, 1160, 4781, 0, 0, 0, 2313, 367, 887, 0, 1, 15, 7777, 2560, 0, 472, 0, 1, 11, 489, 2881, 0, 142, 12, 0, 18, 13, 54, 2050, 0, 1583, 170, 0, 81] payload_rows_per_dst=[6780, 10288, 7650, 10476, 13052, 6056, 8712, 16008] dup_rows_per_dst=[708, 2322, 818, 2210, 3644, 42, 2090, 7448] combine_rows_per_src=[6780, 10288, 7650, 10476, 13052, 6056, 8712, 16008] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=278233088 dispatch_meta_bytes=332744 combine_payload_bytes=278233088 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=278565832 ingress_bytes=382042112 link_bytes=382042112 token_bytes=556798920 weight_movement_bytes=103809024 total_nvlink_bytes=660607944 weight_to_token_ratio=0.186439 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=18944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.437593711e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84382 payload_rows_remote=68810 dup_rows=13922 dup_rows_remote=13108 dedup_ratio=0.141622 router_tokens_per_expert=[1, 97, 42, 105, 10, 170, 1, 0, 538, 22, 1323, 0, 199, 86, 2018, 2, 1133, 1119, 2537, 101, 203, 37, 8, 5486, 0, 270, 21, 317, 7767, 465, 4, 1723, 1199, 1, 0, 0, 2792, 90, 3085, 0, 1, 0, 3925, 517, 0, 3834, 13, 2, 0, 826, 3191, 0, 677, 59, 102, 101, 34, 5, 2608, 0, 215, 5, 0, 65] payload_rows_per_dst=[15572, 13790, 9988, 9672, 7584, 13038, 8938, 5800] dup_rows_per_dst=[814, 5558, 1726, 1512, 328, 2946, 974, 64] combine_rows_per_src=[15572, 13790, 9988, 9672, 7584, 13038, 8938, 5800] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=290807808 dispatch_meta_bytes=339688 combine_payload_bytes=290807808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291147496 ingress_bytes=325410816 link_bytes=325410816 token_bytes=581955304 weight_movement_bytes=34603008 total_nvlink_bytes=616558312 weight_to_token_ratio=0.059460 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=31104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.224494703e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=287989760 dispatch_meta_bytes=350008 combine_payload_bytes=287989760 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288339768 ingress_bytes=322592768 link_bytes=322592768 token_bytes=576329528 weight_movement_bytes=34603008 total_nvlink_bytes=610932536 weight_to_token_ratio=0.060040 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=21504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.213890615e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83542 payload_rows_remote=71342 dup_rows=14762 dup_rows_remote=12108 dedup_ratio=0.150167 router_tokens_per_expert=[289, 78, 0, 0, 28, 299, 0, 0, 82, 0, 432, 0, 6, 272, 4891, 0, 602, 328, 3371, 47, 188, 4, 0, 7366, 0, 2, 1, 0, 7220, 5256, 0, 104, 10, 0, 0, 0, 3034, 0, 3387, 0, 0, 0, 28, 5, 0, 7888, 348, 0, 0, 25, 396, 0, 589, 0, 1020, 190, 1319, 0, 41, 0, 3, 1, 0, 2] payload_rows_per_dst=[14602, 12200, 13880, 13424, 6794, 15832, 4080, 2730] dup_rows_per_dst=[1226, 2654, 6444, 3370, 0, 706, 360, 2] combine_rows_per_src=[14602, 12200, 13880, 13424, 6794, 15832, 4080, 2730] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71796 payload_rows_remote=61144 dup_rows=26508 dup_rows_remote=22526 dedup_ratio=0.269653 router_tokens_per_expert=[973, 488, 109, 131, 2490, 725, 0, 0, 0, 89, 459, 28, 5931, 0, 309, 131, 0, 186, 2, 3306, 0, 713, 89, 37, 2401, 0, 128, 2556, 4, 3060, 238, 763, 7207, 1008, 2340, 83, 1533, 554, 44, 347, 0, 685, 350, 24, 258, 93, 369, 91, 3048, 0, 601, 140, 403, 369, 490, 728, 237, 1867, 712, 3, 2, 0, 105, 115] payload_rows_per_dst=[10652, 13318, 11244, 6756, 15566, 3442, 5014, 5804] dup_rows_per_dst=[3982, 576, 3542, 622, 16762, 298, 448, 278] combine_rows_per_src=[10652, 13318, 11244, 6756, 15566, 3442, 5014, 5804] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=254713856 dispatch_meta_bytes=335384 combine_payload_bytes=254713856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=255049240 ingress_bytes=289316864 link_bytes=289316864 token_bytes=509763096 weight_movement_bytes=34603008 total_nvlink_bytes=544366104 weight_to_token_ratio=0.067881 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=26368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.088676067e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82002 payload_rows_remote=70080 dup_rows=16302 dup_rows_remote=12334 dedup_ratio=0.165833 router_tokens_per_expert=[0, 12, 388, 286, 1, 32, 8, 0, 588, 57, 1991, 0, 1582, 0, 534, 6, 1849, 918, 1064, 161, 50, 519, 2, 3382, 0, 671, 316, 486, 7032, 1, 0, 1388, 3570, 1, 0, 1, 1852, 324, 1480, 13, 0, 1, 7789, 2613, 0, 565, 0, 5, 30, 564, 3468, 0, 259, 89, 2, 6, 11, 92, 2560, 0, 405, 36, 1, 91] payload_rows_per_dst=[14234, 8032, 11922, 8304, 9436, 15794, 8038, 6242] dup_rows_per_dst=[1284, 1484, 3968, 1124, 1342, 6152, 798, 150] combine_rows_per_src=[14234, 8032, 11922, 8304, 9436, 15794, 8038, 6242] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81556 payload_rows_remote=68170 dup_rows=16748 dup_rows_remote=10822 dedup_ratio=0.170369 router_tokens_per_expert=[2018, 14, 18, 183, 1808, 294, 1, 1, 0, 254, 278, 100, 3591, 9, 1552, 441, 0, 751, 16, 4291, 0, 686, 121, 0, 1068, 0, 958, 2165, 2, 1564, 752, 809, 2958, 128, 395, 332, 104, 491, 24, 1170, 0, 312, 340, 206, 1281, 248, 944, 0, 4185, 0, 221, 799, 571, 1535, 1666, 679, 816, 5846, 66, 0, 4, 0, 2, 84] payload_rows_per_dst=[8814, 10904, 10354, 10606, 4272, 10532, 13386, 12688] dup_rows_per_dst=[1996, 1546, 1376, 1894, 226, 2046, 5926, 1738] combine_rows_per_src=[8814, 10904, 10354, 10606, 4272, 10532, 13386, 12688] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=281845760 dispatch_meta_bytes=327672 combine_payload_bytes=281845760 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282173432 ingress_bytes=316448768 link_bytes=316448768 token_bytes=564019192 weight_movement_bytes=34603008 total_nvlink_bytes=598622200 weight_to_token_ratio=0.061351 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=44928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.190771238e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84246 payload_rows_remote=81150 dup_rows=14058 dup_rows_remote=13808 dedup_ratio=0.143005 router_tokens_per_expert=[299, 98, 22, 45, 18, 148, 0, 0, 178, 2, 483, 0, 37, 158, 4648, 0, 702, 844, 3905, 435, 144, 1, 1, 7065, 0, 125, 20, 14, 7541, 2528, 0, 748, 195, 16, 1, 10, 3206, 8, 4202, 0, 5, 5, 382, 146, 0, 7060, 174, 0, 1, 77, 1175, 0, 679, 0, 151, 159, 655, 0, 603, 0, 9, 17, 0, 7] payload_rows_per_dst=[1252, 10298, 11060, 16086, 12182, 14412, 15860, 3096] dup_rows_per_dst=[8, 714, 1004, 5866, 3094, 368, 2754, 250] combine_rows_per_src=[1252, 10298, 11060, 16086, 12182, 14412, 15860, 3096] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79822 payload_rows_remote=66230 dup_rows=18482 dup_rows_remote=12906 dedup_ratio=0.188009 router_tokens_per_expert=[1705, 167, 28, 103, 2507, 297, 0, 0, 0, 256, 361, 71, 4313, 0, 845, 378, 0, 1039, 0, 5499, 0, 838, 168, 0, 351, 0, 269, 1883, 9, 2040, 438, 1040, 3332, 439, 665, 266, 152, 1113, 6, 617, 0, 404, 492, 174, 569, 215, 994, 9, 3621, 1, 328, 733, 219, 1017, 1437, 760, 1014, 5723, 119, 0, 18, 2, 2, 106] payload_rows_per_dst=[8426, 10952, 13592, 6762, 4832, 10192, 12056, 13010] dup_rows_per_dst=[1890, 1496, 5576, 516, 354, 2186, 4176, 2288] combine_rows_per_src=[8426, 10952, 13592, 6762, 4832, 10192, 12056, 13010] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84232 payload_rows_remote=72916 dup_rows=14072 dup_rows_remote=11964 dedup_ratio=0.143148 router_tokens_per_expert=[4, 27, 82, 59, 8, 25, 1, 0, 652, 4, 1626, 0, 1271, 83, 1221, 3, 1743, 1115, 2034, 65, 80, 477, 12, 4403, 0, 316, 11, 1026, 6737, 201, 0, 1741, 2785, 0, 0, 0, 3417, 239, 2307, 0, 2, 0, 5299, 880, 0, 2235, 15, 0, 0, 493, 3085, 0, 480, 3, 10, 66, 27, 22, 1832, 0, 733, 138, 0, 57] payload_rows_per_dst=[13514, 8412, 10294, 11316, 9644, 9694, 11862, 9496] dup_rows_per_dst=[372, 1308, 2224, 2108, 1018, 1690, 3752, 1600] combine_rows_per_src=[13514, 8412, 10294, 11316, 9644, 9694, 11862, 9496] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=292216832 dispatch_meta_bytes=333800 combine_payload_bytes=292216832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=292550632 ingress_bytes=326819840 link_bytes=326819840 token_bytes=584767464 weight_movement_bytes=34603008 total_nvlink_bytes=619370472 weight_to_token_ratio=0.059174 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.229796747e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=250445824 dispatch_meta_bytes=334680 combine_payload_bytes=250445824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=250780504 ingress_bytes=285048832 link_bytes=285048832 token_bytes=501226328 weight_movement_bytes=34603008 total_nvlink_bytes=535829336 weight_to_token_ratio=0.069037 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.072615807e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82222 payload_rows_remote=77388 dup_rows=16082 dup_rows_remote=15716 dedup_ratio=0.163595 router_tokens_per_expert=[1286, 206, 61, 363, 2158, 406, 0, 0, 0, 206, 391, 89, 4481, 7, 873, 355, 0, 727, 85, 4615, 0, 737, 248, 7, 1646, 0, 602, 2621, 1, 2443, 468, 687, 4382, 463, 701, 384, 451, 473, 40, 789, 0, 367, 557, 257, 651, 260, 1266, 66, 3041, 0, 306, 541, 550, 732, 1581, 602, 731, 3763, 185, 0, 12, 0, 122, 110] payload_rows_per_dst=[9534, 11552, 10988, 11460, 4834, 12540, 11610, 9704] dup_rows_per_dst=[2718, 1252, 1850, 2184, 366, 3072, 3096, 1544] combine_rows_per_src=[9534, 11552, 10988, 11460, 4834, 12540, 11610, 9704] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76556 payload_rows_remote=64716 dup_rows=21748 dup_rows_remote=17820 dedup_ratio=0.221232 router_tokens_per_expert=[938, 27, 157, 155, 569, 78, 0, 1690, 1293, 144, 99, 0, 344, 201, 17, 31, 897, 606, 881, 1236, 1490, 2, 158, 290, 778, 463, 207, 0, 1382, 55, 26, 5908, 1329, 1068, 194, 0, 1933, 24, 587, 4216, 776, 705, 81, 2181, 1126, 204, 1683, 898, 0, 431, 6, 316, 1805, 152, 375, 34, 1282, 24, 649, 178, 83, 5193, 190, 1307] payload_rows_per_dst=[8212, 11170, 8160, 13372, 11840, 9414, 8428, 5960] dup_rows_per_dst=[1950, 3474, 2960, 4266, 3928, 1532, 2172, 1466] combine_rows_per_src=[8212, 11170, 8160, 13372, 11840, 9414, 8428, 5960] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=332390400 dispatch_meta_bytes=379832 combine_payload_bytes=332390400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=332770232 ingress_bytes=366993408 link_bytes=366993408 token_bytes=665160632 weight_movement_bytes=34603008 total_nvlink_bytes=699763640 weight_to_token_ratio=0.052022 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=25856 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.380966649e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=271278080 dispatch_meta_bytes=316544 combine_payload_bytes=271278080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271594624 ingress_bytes=305881088 link_bytes=305881088 token_bytes=542872704 weight_movement_bytes=34603008 total_nvlink_bytes=577475712 weight_to_token_ratio=0.063741 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=27264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.151005909e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80544 payload_rows_remote=74714 dup_rows=17760 dup_rows_remote=17748 dedup_ratio=0.180664 router_tokens_per_expert=[0, 9, 651, 235, 1, 6, 0, 0, 1100, 21, 2425, 1, 1764, 1, 442, 5, 1518, 750, 1236, 41, 16, 550, 5, 2627, 0, 591, 273, 1286, 6721, 0, 0, 968, 4114, 0, 0, 0, 2198, 350, 872, 0, 2, 0, 7709, 2574, 0, 344, 0, 1, 11, 485, 3530, 0, 254, 9, 0, 9, 20, 51, 2407, 0, 742, 138, 1, 88] payload_rows_per_dst=[13668, 9330, 7368, 5578, 12126, 5830, 10714, 15930] dup_rows_per_dst=[1578, 2188, 864, 658, 2942, 12, 3136, 6382] combine_rows_per_src=[13668, 9330, 7368, 5578, 12126, 5830, 10714, 15930] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=287047680 dispatch_meta_bytes=329656 combine_payload_bytes=287047680 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=287377336 ingress_bytes=390856704 link_bytes=390856704 token_bytes=574425016 weight_movement_bytes=103809024 total_nvlink_bytes=678234040 weight_to_token_ratio=0.180718 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.470762311e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84232 payload_rows_remote=72916 dup_rows=14072 dup_rows_remote=11964 dedup_ratio=0.143148 router_tokens_per_expert=[4, 27, 82, 59, 8, 25, 1, 0, 652, 4, 1626, 0, 1271, 83, 1221, 3, 1743, 1115, 2034, 65, 80, 477, 12, 4403, 0, 316, 11, 1026, 6737, 201, 0, 1741, 2785, 0, 0, 0, 3417, 239, 2307, 0, 2, 0, 5299, 880, 0, 2235, 15, 0, 0, 493, 3085, 0, 480, 3, 10, 66, 27, 22, 1832, 0, 733, 138, 0, 57] payload_rows_per_dst=[13514, 8412, 10294, 11316, 9644, 9694, 11862, 9496] dup_rows_per_dst=[372, 1308, 2224, 2108, 1018, 1690, 3752, 1600] combine_rows_per_src=[13514, 8412, 10294, 11316, 9644, 9694, 11862, 9496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72488 payload_rows_remote=64706 dup_rows=25816 dup_rows_remote=23740 dedup_ratio=0.262614 router_tokens_per_expert=[1862, 41, 18, 102, 219, 58, 0, 1479, 441, 195, 1, 0, 299, 233, 8, 44, 2134, 534, 1047, 766, 1742, 2, 46, 114, 1188, 618, 249, 0, 2318, 9, 8, 5978, 2049, 782, 74, 0, 2875, 8, 733, 4556, 274, 502, 16, 1751, 711, 455, 1606, 1333, 1, 866, 0, 291, 1886, 24, 97, 13, 530, 5, 1067, 146, 102, 3696, 186, 764] payload_rows_per_dst=[6642, 2462, 9240, 13826, 14328, 8516, 7782, 9692] dup_rows_per_dst=[916, 52, 3530, 6910, 7826, 1278, 2076, 3228] combine_rows_per_src=[6642, 2462, 9240, 13826, 14328, 8516, 7782, 9692] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=298663936 dispatch_meta_bytes=339520 combine_payload_bytes=298663936 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=299003456 ingress_bytes=367869952 link_bytes=367869952 token_bytes=597667392 weight_movement_bytes=69206016 total_nvlink_bytes=666873408 weight_to_token_ratio=0.115794 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=11392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.384265013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=279224320 dispatch_meta_bytes=315968 combine_payload_bytes=279224320 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=279540288 ingress_bytes=348430336 link_bytes=348430336 token_bytes=558764608 weight_movement_bytes=69206016 total_nvlink_bytes=627970624 weight_to_token_ratio=0.123855 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.311115303e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=316981248 dispatch_meta_bytes=372416 combine_payload_bytes=316981248 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=317353664 ingress_bytes=420790272 link_bytes=420790272 token_bytes=634334912 weight_movement_bytes=103809024 total_nvlink_bytes=738143936 weight_to_token_ratio=0.163650 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.583399917e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265076736 dispatch_meta_bytes=330144 combine_payload_bytes=265076736 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265406880 ingress_bytes=403488768 link_bytes=403488768 token_bytes=530483616 weight_movement_bytes=138412032 total_nvlink_bytes=668895648 weight_to_token_ratio=0.260917 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=13696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.518295750e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77370 payload_rows_remote=67318 dup_rows=20934 dup_rows_remote=18768 dedup_ratio=0.212952 router_tokens_per_expert=[312, 16, 889, 229, 2914, 17, 0, 585, 1895, 54, 0, 0, 119, 109, 1, 0, 290, 930, 301, 119, 199, 0, 364, 582, 83, 115, 25, 0, 361, 179, 76, 6293, 613, 2403, 93, 0, 681, 2, 42, 1147, 919, 845, 25, 5670, 908, 88, 2083, 90, 0, 137, 173, 52, 2165, 1229, 78, 99, 2724, 0, 462, 509, 101, 7243, 248, 1266] payload_rows_per_dst=[10052, 14700, 14234, 2460, 6270, 14338, 7104, 8212] dup_rows_per_dst=[2166, 4142, 3922, 138, 658, 6738, 762, 2408] combine_rows_per_src=[10052, 14700, 14234, 2460, 6270, 14338, 7104, 8212] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=306028544 dispatch_meta_bytes=369848 combine_payload_bytes=306028544 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=306398392 ingress_bytes=375234560 link_bytes=375234560 token_bytes=612426936 weight_movement_bytes=69206016 total_nvlink_bytes=681632952 weight_to_token_ratio=0.113003 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=23168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.411977440e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=298663936 dispatch_meta_bytes=339520 combine_payload_bytes=298663936 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=299003456 ingress_bytes=367869952 link_bytes=367869952 token_bytes=597667392 weight_movement_bytes=69206016 total_nvlink_bytes=666873408 weight_to_token_ratio=0.115794 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=11392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.384265013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84246 payload_rows_remote=81150 dup_rows=14058 dup_rows_remote=13808 dedup_ratio=0.143005 router_tokens_per_expert=[299, 98, 22, 45, 18, 148, 0, 0, 178, 2, 483, 0, 37, 158, 4648, 0, 702, 844, 3905, 435, 144, 1, 1, 7065, 0, 125, 20, 14, 7541, 2528, 0, 748, 195, 16, 1, 10, 3206, 8, 4202, 0, 5, 5, 382, 146, 0, 7060, 174, 0, 1, 77, 1175, 0, 679, 0, 151, 159, 655, 0, 603, 0, 9, 17, 0, 7] payload_rows_per_dst=[1252, 10298, 11060, 16086, 12182, 14412, 15860, 3096] dup_rows_per_dst=[8, 714, 1004, 5866, 3094, 368, 2754, 250] combine_rows_per_src=[1252, 10298, 11060, 16086, 12182, 14412, 15860, 3096] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265035776 dispatch_meta_bytes=353784 combine_payload_bytes=265035776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265389560 ingress_bytes=299638784 link_bytes=299638784 token_bytes=530425336 weight_movement_bytes=34603008 total_nvlink_bytes=565028344 weight_to_token_ratio=0.065236 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127516621e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91358 payload_rows_remote=74996 dup_rows=6946 dup_rows_remote=5648 dedup_ratio=0.070658 router_tokens_per_expert=[7, 0, 1231, 1, 8088, 0, 3, 3, 648, 0, 0, 0, 2, 0, 0, 0, 3, 260, 3, 1, 0, 0, 1052, 1, 6, 5, 0, 0, 2, 195, 25, 4852, 18, 4499, 1, 0, 10, 0, 1, 33, 66, 598, 115, 7772, 1, 2, 50, 0, 0, 1, 540, 0, 5, 5225, 0, 25, 5284, 0, 5, 0, 2, 8180, 0, 331] payload_rows_per_dst=[16268, 16362, 11670, 460, 9028, 15602, 11362, 10606] dup_rows_per_dst=[2464, 1298, 674, 6, 30, 1606, 230, 638] combine_rows_per_src=[16268, 16362, 11670, 460, 9028, 15602, 11362, 10606] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84382 payload_rows_remote=68810 dup_rows=13922 dup_rows_remote=13108 dedup_ratio=0.141622 router_tokens_per_expert=[1, 97, 42, 105, 10, 170, 1, 0, 538, 22, 1323, 0, 199, 86, 2018, 2, 1133, 1119, 2537, 101, 203, 37, 8, 5486, 0, 270, 21, 317, 7767, 465, 4, 1723, 1199, 1, 0, 0, 2792, 90, 3085, 0, 1, 0, 3925, 517, 0, 3834, 13, 2, 0, 826, 3191, 0, 677, 59, 102, 101, 34, 5, 2608, 0, 215, 5, 0, 65] payload_rows_per_dst=[15572, 13790, 9988, 9672, 7584, 13038, 8938, 5800] dup_rows_per_dst=[814, 5558, 1726, 1512, 328, 2946, 974, 64] combine_rows_per_src=[15572, 13790, 9988, 9672, 7584, 13038, 8938, 5800] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76796 payload_rows_remote=61666 dup_rows=21508 dup_rows_remote=15068 dedup_ratio=0.218791 router_tokens_per_expert=[879, 57, 289, 308, 355, 38, 0, 1106, 1068, 270, 0, 2, 471, 158, 5, 2, 869, 1038, 754, 498, 312, 0, 88, 724, 1012, 1521, 120, 0, 1308, 0, 61, 6502, 843, 1006, 92, 0, 1791, 0, 126, 3206, 507, 872, 64, 2487, 1526, 308, 1504, 866, 43, 961, 10, 445, 3647, 68, 335, 40, 890, 14, 865, 225, 167, 5240, 260, 929] payload_rows_per_dst=[10188, 11506, 15130, 6646, 6906, 11618, 9306, 5496] dup_rows_per_dst=[2288, 2926, 6440, 1398, 810, 4650, 1792, 1204] combine_rows_per_src=[10188, 11506, 15130, 6646, 6906, 11618, 9306, 5496] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=332390400 dispatch_meta_bytes=379832 combine_payload_bytes=332390400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=332770232 ingress_bytes=366993408 link_bytes=366993408 token_bytes=665160632 weight_movement_bytes=34603008 total_nvlink_bytes=699763640 weight_to_token_ratio=0.052022 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=25856 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.380966649e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=275734528 dispatch_meta_bytes=344344 combine_payload_bytes=275734528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276078872 ingress_bytes=310337536 link_bytes=310337536 token_bytes=551813400 weight_movement_bytes=34603008 total_nvlink_bytes=586416408 weight_to_token_ratio=0.062708 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.167775164e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70186 payload_rows_remote=53990 dup_rows=28118 dup_rows_remote=10152 dedup_ratio=0.286031 router_tokens_per_expert=[42, 1, 817, 48, 7044, 3, 1, 50, 1317, 7, 1, 5, 11, 28, 6, 3, 76, 308, 12, 65, 1, 0, 673, 65, 18, 70, 10, 0, 35, 176, 14, 6380, 144, 4851, 16, 0, 90, 1, 2, 401, 192, 617, 58, 7359, 119, 48, 560, 6, 4, 4, 435, 7, 171, 3379, 28, 24, 4687, 0, 54, 87, 41, 7998, 26, 456] payload_rows_per_dst=[14538, 2736, 2300, 12990, 2924, 3010, 15492, 16196] dup_rows_per_dst=[1474, 20, 100, 416, 622, 190, 7330, 17966] combine_rows_per_src=[14538, 2736, 2300, 12990, 2924, 3010, 15492, 16196] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82002 payload_rows_remote=70080 dup_rows=16302 dup_rows_remote=12334 dedup_ratio=0.165833 router_tokens_per_expert=[0, 12, 388, 286, 1, 32, 8, 0, 588, 57, 1991, 0, 1582, 0, 534, 6, 1849, 918, 1064, 161, 50, 519, 2, 3382, 0, 671, 316, 486, 7032, 1, 0, 1388, 3570, 1, 0, 1, 1852, 324, 1480, 13, 0, 1, 7789, 2613, 0, 565, 0, 5, 30, 564, 3468, 0, 259, 89, 2, 6, 11, 92, 2560, 0, 405, 36, 1, 91] payload_rows_per_dst=[14234, 8032, 11922, 8304, 9436, 15794, 8038, 6242] dup_rows_per_dst=[1284, 1484, 3968, 1124, 1342, 6152, 798, 150] combine_rows_per_src=[14234, 8032, 11922, 8304, 9436, 15794, 8038, 6242] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=307183616 dispatch_meta_bytes=322576 combine_payload_bytes=307183616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307506192 ingress_bytes=341786624 link_bytes=341786624 token_bytes=614689808 weight_movement_bytes=34603008 total_nvlink_bytes=649292816 weight_to_token_ratio=0.056293 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=36096 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.286115550e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79022 payload_rows_remote=70310 dup_rows=19282 dup_rows_remote=17192 dedup_ratio=0.196147 router_tokens_per_expert=[0, 1, 341, 165, 4, 4, 7, 3, 694, 76, 2336, 0, 2807, 8, 378, 6, 1550, 669, 783, 61, 19, 1148, 4, 1848, 0, 471, 175, 1581, 6172, 3, 0, 1160, 4781, 0, 0, 0, 2313, 367, 887, 0, 1, 15, 7777, 2560, 0, 472, 0, 1, 11, 489, 2881, 0, 142, 12, 0, 18, 13, 54, 2050, 0, 1583, 170, 0, 81] payload_rows_per_dst=[6780, 10288, 7650, 10476, 13052, 6056, 8712, 16008] dup_rows_per_dst=[708, 2322, 818, 2210, 3644, 42, 2090, 7448] combine_rows_per_src=[6780, 10288, 7650, 10476, 13052, 6056, 8712, 16008] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79980 payload_rows_remote=67928 dup_rows=18324 dup_rows_remote=15258 dedup_ratio=0.186401 router_tokens_per_expert=[0, 6, 50, 70, 8, 13, 16, 6, 743, 43, 1443, 0, 1622, 55, 948, 9, 1740, 952, 811, 119, 56, 865, 4, 3549, 0, 507, 140, 909, 6177, 19, 0, 1498, 2945, 6, 3, 0, 2672, 351, 1806, 3, 1, 19, 7129, 1867, 0, 969, 0, 10, 2, 469, 3683, 0, 377, 14, 4, 20, 20, 219, 2969, 2, 1045, 52, 0, 117] payload_rows_per_dst=[12448, 8396, 8226, 5818, 12052, 5650, 11932, 15458] dup_rows_per_dst=[244, 1330, 868, 782, 3066, 82, 4304, 7648] combine_rows_per_src=[12448, 8396, 8226, 5818, 12052, 5650, 11932, 15458] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=281845760 dispatch_meta_bytes=327672 combine_payload_bytes=281845760 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282173432 ingress_bytes=316448768 link_bytes=316448768 token_bytes=564019192 weight_movement_bytes=34603008 total_nvlink_bytes=598622200 weight_to_token_ratio=0.061351 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=44928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.190771238e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75544 payload_rows_remote=67180 dup_rows=22760 dup_rows_remote=21502 dedup_ratio=0.231527 router_tokens_per_expert=[1274, 13, 75, 101, 224, 66, 0, 1730, 685, 229, 0, 0, 339, 353, 5, 1, 1180, 600, 1499, 815, 1434, 0, 49, 410, 759, 779, 116, 0, 1434, 0, 10, 6375, 1668, 948, 81, 0, 2916, 0, 392, 4478, 453, 524, 6, 1667, 903, 377, 1450, 1098, 0, 662, 0, 260, 2984, 39, 189, 16, 564, 0, 1354, 219, 161, 4040, 226, 922] payload_rows_per_dst=[6296, 9176, 9040, 13970, 14054, 8364, 9150, 5494] dup_rows_per_dst=[670, 2128, 2934, 4976, 6912, 1258, 2484, 1398] combine_rows_per_src=[6296, 9176, 9040, 13970, 14054, 8364, 9150, 5494] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=252583936 dispatch_meta_bytes=306936 combine_payload_bytes=252583936 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=252890872 ingress_bytes=287186944 link_bytes=287186944 token_bytes=505474808 weight_movement_bytes=34603008 total_nvlink_bytes=540077816 weight_to_token_ratio=0.068456 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=18176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.080661350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79110 payload_rows_remote=65760 dup_rows=19194 dup_rows_remote=16074 dedup_ratio=0.195251 router_tokens_per_expert=[670, 11, 131, 129, 1234, 31, 0, 824, 1555, 109, 1, 0, 167, 131, 4, 10, 960, 514, 386, 478, 823, 3, 168, 194, 668, 287, 68, 0, 1111, 107, 34, 6419, 1769, 2313, 48, 0, 1500, 0, 454, 4060, 772, 684, 25, 4167, 841, 56, 1913, 502, 0, 248, 73, 162, 1545, 433, 66, 25, 1886, 0, 673, 393, 85, 6099, 184, 949] payload_rows_per_dst=[12076, 12634, 5420, 13350, 10000, 8974, 10202, 6454] dup_rows_per_dst=[2104, 3518, 2550, 3120, 2168, 1216, 2632, 1886] combine_rows_per_src=[12076, 12634, 5420, 13350, 10000, 8974, 10202, 6454] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=287989760 dispatch_meta_bytes=350008 combine_payload_bytes=287989760 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288339768 ingress_bytes=322592768 link_bytes=322592768 token_bytes=576329528 weight_movement_bytes=34603008 total_nvlink_bytes=610932536 weight_to_token_ratio=0.060040 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=21504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.213890615e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=278233088 dispatch_meta_bytes=332744 combine_payload_bytes=278233088 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=278565832 ingress_bytes=382042112 link_bytes=382042112 token_bytes=556798920 weight_movement_bytes=103809024 total_nvlink_bytes=660607944 weight_to_token_ratio=0.186439 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=18944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.437593711e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82034 payload_rows_remote=71738 dup_rows=16270 dup_rows_remote=14444 dedup_ratio=0.165507 router_tokens_per_expert=[119, 0, 50, 658, 1285, 0, 6, 0, 54, 2703, 87, 570, 1554, 26, 374, 2, 2724, 0, 3525, 0, 21, 14, 1160, 1095, 2356, 0, 0, 765, 0, 298, 116, 0, 414, 63, 3229, 1740, 124, 0, 4, 487, 5196, 442, 216, 158, 1351, 1768, 4799, 2359, 7, 843, 485, 15, 0, 333, 1, 8, 7, 24, 283, 584, 225, 3576, 442, 407] payload_rows_per_dst=[5648, 9104, 13026, 11462, 10296, 11254, 11872, 9372] dup_rows_per_dst=[908, 1636, 1732, 2138, 1826, 5196, 1110, 1724] combine_rows_per_src=[5648, 9104, 13026, 11462, 10296, 11254, 11872, 9372] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=275169280 dispatch_meta_bytes=354728 combine_payload_bytes=275169280 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=275524008 ingress_bytes=344375296 link_bytes=344375296 token_bytes=550693288 weight_movement_bytes=69206016 total_nvlink_bytes=619899304 weight_to_token_ratio=0.125671 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=16256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.295856514e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=221143040 dispatch_meta_bytes=256568 combine_payload_bytes=221143040 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=221399608 ingress_bytes=290349056 link_bytes=290349056 token_bytes=442542648 weight_movement_bytes=69206016 total_nvlink_bytes=511748664 weight_to_token_ratio=0.156383 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.092560123e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=287047680 dispatch_meta_bytes=329656 combine_payload_bytes=287047680 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=287377336 ingress_bytes=390856704 link_bytes=390856704 token_bytes=574425016 weight_movement_bytes=103809024 total_nvlink_bytes=678234040 weight_to_token_ratio=0.180718 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.470762311e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75544 payload_rows_remote=67180 dup_rows=22760 dup_rows_remote=21502 dedup_ratio=0.231527 router_tokens_per_expert=[1274, 13, 75, 101, 224, 66, 0, 1730, 685, 229, 0, 0, 339, 353, 5, 1, 1180, 600, 1499, 815, 1434, 0, 49, 410, 759, 779, 116, 0, 1434, 0, 10, 6375, 1668, 948, 81, 0, 2916, 0, 392, 4478, 453, 524, 6, 1667, 903, 377, 1450, 1098, 0, 662, 0, 260, 2984, 39, 189, 16, 564, 0, 1354, 219, 161, 4040, 226, 922] payload_rows_per_dst=[6296, 9176, 9040, 13970, 14054, 8364, 9150, 5494] dup_rows_per_dst=[670, 2128, 2934, 4976, 6912, 1258, 2484, 1398] combine_rows_per_src=[6296, 9176, 9040, 13970, 14054, 8364, 9150, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80584 payload_rows_remote=67826 dup_rows=17720 dup_rows_remote=17018 dedup_ratio=0.180257 router_tokens_per_expert=[121, 0, 0, 908, 931, 0, 0, 0, 62, 2131, 66, 1384, 523, 2, 349, 3, 1657, 0, 5341, 0, 13, 30, 602, 776, 3378, 0, 0, 771, 0, 714, 41, 0, 444, 4, 4870, 449, 15, 0, 1, 1119, 3877, 713, 133, 21, 1122, 1673, 5704, 1516, 7, 468, 439, 5, 0, 106, 0, 1, 0, 15, 399, 554, 29, 4707, 334, 624] payload_rows_per_dst=[4398, 8338, 10662, 8146, 13172, 9982, 12758, 13128] dup_rows_per_dst=[726, 702, 618, 1662, 4986, 4400, 702, 3924] combine_rows_per_src=[4398, 8338, 10662, 8146, 13172, 9982, 12758, 13128] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=269352960 dispatch_meta_bytes=327336 combine_payload_bytes=269352960 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=269680296 ingress_bytes=338558976 link_bytes=338558976 token_bytes=539033256 weight_movement_bytes=69206016 total_nvlink_bytes=608239272 weight_to_token_ratio=0.128389 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=30720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.273970170e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70186 payload_rows_remote=53990 dup_rows=28118 dup_rows_remote=10152 dedup_ratio=0.286031 router_tokens_per_expert=[42, 1, 817, 48, 7044, 3, 1, 50, 1317, 7, 1, 5, 11, 28, 6, 3, 76, 308, 12, 65, 1, 0, 673, 65, 18, 70, 10, 0, 35, 176, 14, 6380, 144, 4851, 16, 0, 90, 1, 2, 401, 192, 617, 58, 7359, 119, 48, 560, 6, 4, 4, 435, 7, 171, 3379, 28, 24, 4687, 0, 54, 87, 41, 7998, 26, 456] payload_rows_per_dst=[14538, 2736, 2300, 12990, 2924, 3010, 15492, 16196] dup_rows_per_dst=[1474, 20, 100, 416, 622, 190, 7330, 17966] combine_rows_per_src=[14538, 2736, 2300, 12990, 2924, 3010, 15492, 16196] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78536 payload_rows_remote=69516 dup_rows=19768 dup_rows_remote=18560 dedup_ratio=0.201090 router_tokens_per_expert=[150, 0, 625, 402, 550, 0, 0, 0, 31, 4221, 22, 151, 3002, 5, 31, 9, 4699, 0, 1241, 0, 0, 5, 3387, 986, 384, 0, 0, 98, 0, 206, 254, 0, 130, 160, 988, 5311, 179, 1, 0, 256, 7208, 224, 364, 1143, 1339, 613, 1311, 969, 3, 3131, 184, 2, 0, 1208, 0, 5, 0, 36, 21, 734, 2076, 707, 340, 50] payload_rows_per_dst=[9020, 14984, 12236, 15362, 2784, 8446, 8818, 6886] dup_rows_per_dst=[1208, 10582, 2852, 938, 644, 1318, 1184, 1042] combine_rows_per_src=[9020, 14984, 12236, 15362, 2784, 8446, 8818, 6886] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=293838848 dispatch_meta_bytes=344728 combine_payload_bytes=293838848 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=294183576 ingress_bytes=363044864 link_bytes=363044864 token_bytes=588022424 weight_movement_bytes=69206016 total_nvlink_bytes=657228440 weight_to_token_ratio=0.117693 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=19328 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.366108596e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=275169280 dispatch_meta_bytes=354728 combine_payload_bytes=275169280 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=275524008 ingress_bytes=344375296 link_bytes=344375296 token_bytes=550693288 weight_movement_bytes=69206016 total_nvlink_bytes=619899304 weight_to_token_ratio=0.125671 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=16256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.295856514e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=277815296 dispatch_meta_bytes=339376 combine_payload_bytes=277815296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278154672 ingress_bytes=312418304 link_bytes=312418304 token_bytes=555969968 weight_movement_bytes=34603008 total_nvlink_bytes=590572976 weight_to_token_ratio=0.062239 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=25728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.175604926e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82580 payload_rows_remote=74718 dup_rows=15724 dup_rows_remote=14524 dedup_ratio=0.159953 router_tokens_per_expert=[5, 0, 7035, 5, 21, 0, 0, 0, 0, 90, 0, 1, 2470, 0, 0, 2, 1252, 0, 8, 0, 0, 0, 7488, 104, 0, 0, 0, 1, 0, 8, 40, 0, 1, 755, 10, 7109, 134, 0, 0, 3, 7769, 1, 15, 2755, 22, 5, 11, 51, 0, 5264, 5, 0, 0, 584, 0, 0, 0, 0, 1, 1, 6113, 7, 1, 5] payload_rows_per_dst=[16350, 7862, 2712, 15552, 11040, 5678, 11138, 12248] dup_rows_per_dst=[12758, 1200, 26, 84, 1048, 10, 590, 8] combine_rows_per_src=[16350, 7862, 2712, 15552, 11040, 5678, 11138, 12248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77370 payload_rows_remote=67318 dup_rows=20934 dup_rows_remote=18768 dedup_ratio=0.212952 router_tokens_per_expert=[312, 16, 889, 229, 2914, 17, 0, 585, 1895, 54, 0, 0, 119, 109, 1, 0, 290, 930, 301, 119, 199, 0, 364, 582, 83, 115, 25, 0, 361, 179, 76, 6293, 613, 2403, 93, 0, 681, 2, 42, 1147, 919, 845, 25, 5670, 908, 88, 2083, 90, 0, 137, 173, 52, 2165, 1229, 78, 99, 2724, 0, 462, 509, 101, 7243, 248, 1266] payload_rows_per_dst=[10052, 14700, 14234, 2460, 6270, 14338, 7104, 8212] dup_rows_per_dst=[2166, 4142, 3922, 138, 658, 6738, 762, 2408] combine_rows_per_src=[10052, 14700, 14234, 2460, 6270, 14338, 7104, 8212] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=221143040 dispatch_meta_bytes=256568 combine_payload_bytes=221143040 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=221399608 ingress_bytes=290349056 link_bytes=290349056 token_bytes=442542648 weight_movement_bytes=69206016 total_nvlink_bytes=511748664 weight_to_token_ratio=0.156383 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.092560123e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84146 payload_rows_remote=70406 dup_rows=14158 dup_rows_remote=10776 dedup_ratio=0.144023 router_tokens_per_expert=[29, 0, 0, 2209, 1881, 0, 0, 0, 79, 3020, 66, 351, 1056, 28, 43, 84, 3418, 0, 3987, 0, 0, 1, 335, 894, 1388, 0, 0, 261, 0, 890, 36, 0, 347, 1, 3319, 1557, 40, 0, 29, 978, 5851, 462, 157, 48, 1309, 2919, 5657, 627, 6, 609, 676, 6, 1, 351, 0, 27, 17, 45, 53, 291, 132, 2689, 590, 302] payload_rows_per_dst=[7564, 8340, 13740, 14674, 10586, 8992, 12992, 7258] dup_rows_per_dst=[1344, 1114, 3382, 2178, 1956, 1530, 1674, 980] combine_rows_per_src=[7564, 8340, 13740, 14674, 10586, 8992, 12992, 7258] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72488 payload_rows_remote=64706 dup_rows=25816 dup_rows_remote=23740 dedup_ratio=0.262614 router_tokens_per_expert=[1862, 41, 18, 102, 219, 58, 0, 1479, 441, 195, 1, 0, 299, 233, 8, 44, 2134, 534, 1047, 766, 1742, 2, 46, 114, 1188, 618, 249, 0, 2318, 9, 8, 5978, 2049, 782, 74, 0, 2875, 8, 733, 4556, 274, 502, 16, 1751, 711, 455, 1606, 1333, 1, 866, 0, 291, 1886, 24, 97, 13, 530, 5, 1067, 146, 102, 3696, 186, 764] payload_rows_per_dst=[6642, 2462, 9240, 13826, 14328, 8516, 7782, 9692] dup_rows_per_dst=[916, 52, 3530, 6910, 7826, 1278, 2076, 3228] combine_rows_per_src=[6642, 2462, 9240, 13826, 14328, 8516, 7782, 9692] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=284737536 dispatch_meta_bytes=352304 combine_payload_bytes=284737536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=285089840 ingress_bytes=319340544 link_bytes=319340544 token_bytes=569827376 weight_movement_bytes=34603008 total_nvlink_bytes=604430384 weight_to_token_ratio=0.060725 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=22272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.201652758e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76796 payload_rows_remote=61666 dup_rows=21508 dup_rows_remote=15068 dedup_ratio=0.218791 router_tokens_per_expert=[879, 57, 289, 308, 355, 38, 0, 1106, 1068, 270, 0, 2, 471, 158, 5, 2, 869, 1038, 754, 498, 312, 0, 88, 724, 1012, 1521, 120, 0, 1308, 0, 61, 6502, 843, 1006, 92, 0, 1791, 0, 126, 3206, 507, 872, 64, 2487, 1526, 308, 1504, 866, 43, 961, 10, 445, 3647, 68, 335, 40, 890, 14, 865, 225, 167, 5240, 260, 929] payload_rows_per_dst=[10188, 11506, 15130, 6646, 6906, 11618, 9306, 5496] dup_rows_per_dst=[2288, 2926, 6440, 1398, 810, 4650, 1792, 1204] combine_rows_per_src=[10188, 11506, 15130, 6646, 6906, 11618, 9306, 5496] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=306044928 dispatch_meta_bytes=356968 combine_payload_bytes=306044928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=306401896 ingress_bytes=340647936 link_bytes=340647936 token_bytes=612446824 weight_movement_bytes=34603008 total_nvlink_bytes=647049832 weight_to_token_ratio=0.056500 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.281830759e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76556 payload_rows_remote=64716 dup_rows=21748 dup_rows_remote=17820 dedup_ratio=0.221232 router_tokens_per_expert=[938, 27, 157, 155, 569, 78, 0, 1690, 1293, 144, 99, 0, 344, 201, 17, 31, 897, 606, 881, 1236, 1490, 2, 158, 290, 778, 463, 207, 0, 1382, 55, 26, 5908, 1329, 1068, 194, 0, 1933, 24, 587, 4216, 776, 705, 81, 2181, 1126, 204, 1683, 898, 0, 431, 6, 316, 1805, 152, 375, 34, 1282, 24, 649, 178, 83, 5193, 190, 1307] payload_rows_per_dst=[8212, 11170, 8160, 13372, 11840, 9414, 8428, 5960] dup_rows_per_dst=[1950, 3474, 2960, 4266, 3928, 1532, 2172, 1466] combine_rows_per_src=[8212, 11170, 8160, 13372, 11840, 9414, 8428, 5960] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=275734528 dispatch_meta_bytes=344344 combine_payload_bytes=275734528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276078872 ingress_bytes=310337536 link_bytes=310337536 token_bytes=551813400 weight_movement_bytes=34603008 total_nvlink_bytes=586416408 weight_to_token_ratio=0.062708 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.167775164e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79660 payload_rows_remote=66740 dup_rows=18644 dup_rows_remote=10772 dedup_ratio=0.189657 router_tokens_per_expert=[105, 0, 0, 1481, 784, 0, 2, 0, 71, 3359, 99, 644, 687, 2, 147, 49, 2593, 0, 4806, 0, 0, 41, 753, 693, 2217, 0, 0, 390, 0, 1307, 61, 0, 526, 14, 4586, 710, 30, 0, 2, 1254, 4924, 945, 147, 52, 1005, 2163, 5829, 1160, 0, 580, 396, 3, 0, 192, 0, 4, 0, 27, 103, 477, 47, 2851, 395, 439] payload_rows_per_dst=[5492, 9016, 13984, 6402, 11264, 12920, 12976, 7606] dup_rows_per_dst=[758, 1100, 2282, 1548, 2980, 7872, 1032, 1072] combine_rows_per_src=[5492, 9016, 13984, 6402, 11264, 12920, 12976, 7606] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=288382976 dispatch_meta_bytes=324728 combine_payload_bytes=288382976 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=288707704 ingress_bytes=392192000 link_bytes=392192000 token_bytes=577090680 weight_movement_bytes=103809024 total_nvlink_bytes=680899704 weight_to_token_ratio=0.179883 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.475786922e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78054 payload_rows_remote=62238 dup_rows=20250 dup_rows_remote=10618 dedup_ratio=0.205994 router_tokens_per_expert=[51, 0, 5331, 47, 81, 7, 0, 0, 0, 1085, 5, 4, 4153, 0, 24, 11, 2002, 2, 174, 0, 0, 0, 6915, 333, 0, 1, 0, 7, 0, 52, 234, 0, 3, 361, 76, 5533, 213, 0, 0, 96, 7805, 16, 127, 3314, 179, 12, 159, 783, 2, 4319, 86, 1, 7, 610, 0, 12, 1, 11, 3, 114, 4697, 61, 30, 2] payload_rows_per_dst=[14702, 9552, 7860, 586, 11728, 8144, 9666, 15816] dup_rows_per_dst=[6154, 1012, 822, 2, 1184, 718, 726, 9632] combine_rows_per_src=[14702, 9552, 7860, 586, 11728, 8144, 9666, 15816] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84950 payload_rows_remote=69716 dup_rows=13354 dup_rows_remote=11018 dedup_ratio=0.135844 router_tokens_per_expert=[246, 0, 323, 625, 898, 0, 0, 0, 22, 3895, 51, 432, 2373, 0, 108, 0, 2784, 0, 2344, 0, 1, 0, 3545, 1056, 1518, 0, 0, 342, 0, 745, 210, 0, 282, 129, 2170, 3178, 112, 0, 3, 531, 5970, 374, 273, 607, 1167, 941, 3068, 2560, 2, 1627, 227, 1, 0, 608, 0, 0, 2, 10, 130, 595, 494, 2152, 238, 183] payload_rows_per_dst=[9462, 11116, 11202, 15234, 11056, 9968, 10106, 6806] dup_rows_per_dst=[1812, 2646, 1168, 2336, 1754, 1876, 960, 802] combine_rows_per_src=[9462, 11116, 11202, 15234, 11056, 9968, 10106, 6806] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265035776 dispatch_meta_bytes=353784 combine_payload_bytes=265035776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265389560 ingress_bytes=299638784 link_bytes=299638784 token_bytes=530425336 weight_movement_bytes=34603008 total_nvlink_bytes=565028344 weight_to_token_ratio=0.065236 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127516621e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265076736 dispatch_meta_bytes=330144 combine_payload_bytes=265076736 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265406880 ingress_bytes=403488768 link_bytes=403488768 token_bytes=530483616 weight_movement_bytes=138412032 total_nvlink_bytes=668895648 weight_to_token_ratio=0.260917 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=13696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.518295750e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=273367040 dispatch_meta_bytes=310048 combine_payload_bytes=273367040 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=273677088 ingress_bytes=411779072 link_bytes=411779072 token_bytes=547044128 weight_movement_bytes=138412032 total_nvlink_bytes=685456160 weight_to_token_ratio=0.253018 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=10112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.549491497e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78054 payload_rows_remote=62238 dup_rows=20250 dup_rows_remote=10618 dedup_ratio=0.205994 router_tokens_per_expert=[51, 0, 5331, 47, 81, 7, 0, 0, 0, 1085, 5, 4, 4153, 0, 24, 11, 2002, 2, 174, 0, 0, 0, 6915, 333, 0, 1, 0, 7, 0, 52, 234, 0, 3, 361, 76, 5533, 213, 0, 0, 96, 7805, 16, 127, 3314, 179, 12, 159, 783, 2, 4319, 86, 1, 7, 610, 0, 12, 1, 11, 3, 114, 4697, 61, 30, 2] payload_rows_per_dst=[14702, 9552, 7860, 586, 11728, 8144, 9666, 15816] dup_rows_per_dst=[6154, 1012, 822, 2, 1184, 718, 726, 9632] combine_rows_per_src=[14702, 9552, 7860, 586, 11728, 8144, 9666, 15816] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80806 payload_rows_remote=70146 dup_rows=17498 dup_rows_remote=14318 dedup_ratio=0.177999 router_tokens_per_expert=[947, 57, 25, 1, 351, 195, 3951, 0, 52, 1225, 1595, 2502, 5, 4577, 407, 2528, 2531, 51, 702, 2865, 109, 0, 629, 572, 1, 121, 4058, 507, 1852, 162, 456, 0, 0, 94, 72, 1605, 477, 1094, 26, 3552, 264, 94, 35, 64, 389, 0, 12, 2, 105, 4972, 0, 499, 0, 254, 62, 117, 282, 0, 0, 13, 12, 122, 1227, 673] payload_rows_per_dst=[10050, 8668, 8720, 11502, 10660, 11946, 10690, 8570] dup_rows_per_dst=[1004, 2904, 1136, 2812, 3180, 3984, 1328, 1150] combine_rows_per_src=[10050, 8668, 8720, 11502, 10660, 11946, 10690, 8570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=252583936 dispatch_meta_bytes=306936 combine_payload_bytes=252583936 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=252890872 ingress_bytes=287186944 link_bytes=287186944 token_bytes=505474808 weight_movement_bytes=34603008 total_nvlink_bytes=540077816 weight_to_token_ratio=0.068456 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=18176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.080661350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79660 payload_rows_remote=66740 dup_rows=18644 dup_rows_remote=10772 dedup_ratio=0.189657 router_tokens_per_expert=[105, 0, 0, 1481, 784, 0, 2, 0, 71, 3359, 99, 644, 687, 2, 147, 49, 2593, 0, 4806, 0, 0, 41, 753, 693, 2217, 0, 0, 390, 0, 1307, 61, 0, 526, 14, 4586, 710, 30, 0, 2, 1254, 4924, 945, 147, 52, 1005, 2163, 5829, 1160, 0, 580, 396, 3, 0, 192, 0, 4, 0, 27, 103, 477, 47, 2851, 395, 439] payload_rows_per_dst=[5492, 9016, 13984, 6402, 11264, 12920, 12976, 7606] dup_rows_per_dst=[758, 1100, 2282, 1548, 2980, 7872, 1032, 1072] combine_rows_per_src=[5492, 9016, 13984, 6402, 11264, 12920, 12976, 7606] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79310 payload_rows_remote=66752 dup_rows=18994 dup_rows_remote=17420 dedup_ratio=0.193217 router_tokens_per_expert=[979, 128, 30, 0, 389, 225, 2862, 0, 17, 2581, 1425, 2729, 0, 3652, 46, 3005, 842, 18, 837, 2347, 4, 4, 298, 733, 0, 239, 3973, 267, 2601, 294, 189, 4, 0, 35, 20, 1688, 207, 1826, 2, 4435, 11, 115, 24, 140, 155, 0, 11, 4, 0, 6023, 0, 653, 5, 30, 34, 321, 337, 0, 0, 3, 0, 130, 2129, 96] payload_rows_per_dst=[8100, 9502, 7708, 11804, 12116, 10830, 12558, 6692] dup_rows_per_dst=[1126, 4094, 774, 3330, 4310, 3404, 1574, 382] combine_rows_per_src=[8100, 9502, 7708, 11804, 12116, 10830, 12558, 6692] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=254926848 dispatch_meta_bytes=291424 combine_payload_bytes=254926848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=255218272 ingress_bytes=289529856 link_bytes=289529856 token_bytes=510145120 weight_movement_bytes=34603008 total_nvlink_bytes=544748128 weight_to_token_ratio=0.067830 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.089477539e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=285556736 dispatch_meta_bytes=322936 combine_payload_bytes=285556736 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=285879672 ingress_bytes=320159744 link_bytes=320159744 token_bytes=571436408 weight_movement_bytes=34603008 total_nvlink_bytes=606039416 weight_to_token_ratio=0.060554 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.035000 gemm_active_groups=7 gemm_peak_group_rows=30080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.204735342e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=287318016 dispatch_meta_bytes=337856 combine_payload_bytes=287318016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=287655872 ingress_bytes=321921024 link_bytes=321921024 token_bytes=574973888 weight_movement_bytes=34603008 total_nvlink_bytes=609576896 weight_to_token_ratio=0.060182 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=17408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.211362897e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80862 payload_rows_remote=66818 dup_rows=17442 dup_rows_remote=15346 dedup_ratio=0.177429 router_tokens_per_expert=[1100, 22, 8, 0, 80, 156, 6704, 0, 0, 468, 512, 414, 13, 4933, 1402, 1991, 6605, 96, 96, 1312, 203, 0, 1457, 86, 0, 9, 2096, 154, 422, 47, 2284, 2, 0, 438, 0, 699, 569, 154, 2, 1049, 394, 7, 0, 59, 1307, 0, 11, 5, 1008, 4302, 0, 422, 0, 2972, 2, 90, 55, 0, 0, 171, 0, 210, 289, 2265] payload_rows_per_dst=[14044, 8090, 15512, 8020, 9646, 11694, 8068, 5788] dup_rows_per_dst=[2096, 1510, 8390, 882, 1714, 1738, 920, 192] combine_rows_per_src=[14044, 8090, 15512, 8020, 9646, 11694, 8068, 5788] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=273367040 dispatch_meta_bytes=310048 combine_payload_bytes=273367040 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=273677088 ingress_bytes=411779072 link_bytes=411779072 token_bytes=547044128 weight_movement_bytes=138412032 total_nvlink_bytes=685456160 weight_to_token_ratio=0.253018 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=10112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.549491497e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=273416192 dispatch_meta_bytes=336688 combine_payload_bytes=273416192 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=273752880 ingress_bytes=377225216 link_bytes=377225216 token_bytes=547169072 weight_movement_bytes=103809024 total_nvlink_bytes=650978096 weight_to_token_ratio=0.189720 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=22016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.419468119e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92004 payload_rows_remote=81312 dup_rows=6300 dup_rows_remote=6188 dedup_ratio=0.064087 router_tokens_per_expert=[53, 0, 0, 0, 0, 4, 7633, 2, 0, 6, 1, 9, 142, 1363, 5190, 54, 8173, 0, 1, 12, 254, 0, 1264, 4, 0, 0, 135, 0, 4, 0, 7093, 0, 0, 431, 1, 13, 3, 4, 1, 14, 186, 0, 0, 0, 365, 0, 1, 0, 293, 486, 0, 3, 0, 8093, 0, 3, 2, 0, 0, 851, 0, 512, 3, 6490] payload_rows_per_dst=[15374, 10692, 15260, 14194, 1854, 3820, 16200, 14610] dup_rows_per_dst=[10, 112, 3010, 0, 52, 10, 584, 2522] combine_rows_per_src=[15374, 10692, 15260, 14194, 1854, 3820, 16200, 14610] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=254926848 dispatch_meta_bytes=291424 combine_payload_bytes=254926848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=255218272 ingress_bytes=289529856 link_bytes=289529856 token_bytes=510145120 weight_movement_bytes=34603008 total_nvlink_bytes=544748128 weight_to_token_ratio=0.067830 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.089477539e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77394 payload_rows_remote=67152 dup_rows=20910 dup_rows_remote=19008 dedup_ratio=0.212708 router_tokens_per_expert=[1059, 91, 98, 0, 140, 219, 4358, 1, 74, 1120, 1905, 2217, 3, 4425, 142, 4134, 2276, 246, 968, 1480, 123, 5, 334, 141, 0, 102, 2775, 734, 2009, 247, 308, 2, 17, 65, 2, 1293, 1127, 532, 3, 2273, 238, 46, 26, 148, 574, 0, 23, 297, 204, 6348, 0, 1091, 0, 143, 93, 574, 272, 0, 0, 14, 0, 66, 1690, 257] payload_rows_per_dst=[10562, 11844, 10242, 5862, 15304, 11006, 4148, 8426] dup_rows_per_dst=[1370, 5426, 1902, 942, 8016, 2468, 62, 724] combine_rows_per_src=[10562, 11844, 10242, 5862, 15304, 11006, 4148, 8426] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78536 payload_rows_remote=69516 dup_rows=19768 dup_rows_remote=18560 dedup_ratio=0.201090 router_tokens_per_expert=[150, 0, 625, 402, 550, 0, 0, 0, 31, 4221, 22, 151, 3002, 5, 31, 9, 4699, 0, 1241, 0, 0, 5, 3387, 986, 384, 0, 0, 98, 0, 206, 254, 0, 130, 160, 988, 5311, 179, 1, 0, 256, 7208, 224, 364, 1143, 1339, 613, 1311, 969, 3, 3131, 184, 2, 0, 1208, 0, 5, 0, 36, 21, 734, 2076, 707, 340, 50] payload_rows_per_dst=[9020, 14984, 12236, 15362, 2784, 8446, 8818, 6886] dup_rows_per_dst=[1208, 10582, 2852, 938, 644, 1318, 1184, 1042] combine_rows_per_src=[9020, 14984, 12236, 15362, 2784, 8446, 8818, 6886] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80584 payload_rows_remote=67826 dup_rows=17720 dup_rows_remote=17018 dedup_ratio=0.180257 router_tokens_per_expert=[121, 0, 0, 908, 931, 0, 0, 0, 62, 2131, 66, 1384, 523, 2, 349, 3, 1657, 0, 5341, 0, 13, 30, 602, 776, 3378, 0, 0, 771, 0, 714, 41, 0, 444, 4, 4870, 449, 15, 0, 1, 1119, 3877, 713, 133, 21, 1122, 1673, 5704, 1516, 7, 468, 439, 5, 0, 106, 0, 1, 0, 15, 399, 554, 29, 4707, 334, 624] payload_rows_per_dst=[4398, 8338, 10662, 8146, 13172, 9982, 12758, 13128] dup_rows_per_dst=[726, 702, 618, 1662, 4986, 4400, 702, 3924] combine_rows_per_src=[4398, 8338, 10662, 8146, 13172, 9982, 12758, 13128] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=273686528 dispatch_meta_bytes=328656 combine_payload_bytes=273686528 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=274015184 ingress_bytes=342892544 link_bytes=342892544 token_bytes=547701712 weight_movement_bytes=69206016 total_nvlink_bytes=616907728 weight_to_token_ratio=0.126357 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=38784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.290277038e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84146 payload_rows_remote=70406 dup_rows=14158 dup_rows_remote=10776 dedup_ratio=0.144023 router_tokens_per_expert=[29, 0, 0, 2209, 1881, 0, 0, 0, 79, 3020, 66, 351, 1056, 28, 43, 84, 3418, 0, 3987, 0, 0, 1, 335, 894, 1388, 0, 0, 261, 0, 890, 36, 0, 347, 1, 3319, 1557, 40, 0, 29, 978, 5851, 462, 157, 48, 1309, 2919, 5657, 627, 6, 609, 676, 6, 1, 351, 0, 27, 17, 45, 53, 291, 132, 2689, 590, 302] payload_rows_per_dst=[7564, 8340, 13740, 14674, 10586, 8992, 12992, 7258] dup_rows_per_dst=[1344, 1114, 3382, 2178, 1956, 1530, 1674, 980] combine_rows_per_src=[7564, 8340, 13740, 14674, 10586, 8992, 12992, 7258] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=333053952 dispatch_meta_bytes=350000 combine_payload_bytes=333053952 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=333403952 ingress_bytes=436862976 link_bytes=436862976 token_bytes=666457904 weight_movement_bytes=103809024 total_nvlink_bytes=770266928 weight_to_token_ratio=0.155762 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=13696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.643880208e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82034 payload_rows_remote=71738 dup_rows=16270 dup_rows_remote=14444 dedup_ratio=0.165507 router_tokens_per_expert=[119, 0, 50, 658, 1285, 0, 6, 0, 54, 2703, 87, 570, 1554, 26, 374, 2, 2724, 0, 3525, 0, 21, 14, 1160, 1095, 2356, 0, 0, 765, 0, 298, 116, 0, 414, 63, 3229, 1740, 124, 0, 4, 487, 5196, 442, 216, 158, 1351, 1768, 4799, 2359, 7, 843, 485, 15, 0, 333, 1, 8, 7, 24, 283, 584, 225, 3576, 442, 407] payload_rows_per_dst=[5648, 9104, 13026, 11462, 10296, 11254, 11872, 9372] dup_rows_per_dst=[908, 1636, 1732, 2138, 1826, 5196, 1110, 1724] combine_rows_per_src=[5648, 9104, 13026, 11462, 10296, 11254, 11872, 9372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81586 payload_rows_remote=70222 dup_rows=16718 dup_rows_remote=13088 dedup_ratio=0.170064 router_tokens_per_expert=[973, 112, 58, 0, 285, 295, 3612, 0, 0, 1638, 1734, 1903, 0, 3456, 128, 3164, 1550, 42, 1355, 2666, 57, 14, 478, 489, 0, 82, 3319, 384, 1821, 210, 222, 0, 0, 24, 10, 1982, 672, 1049, 7, 3765, 38, 292, 12, 184, 271, 0, 68, 12, 46, 6750, 0, 695, 0, 83, 56, 299, 213, 0, 0, 7, 1, 194, 2266, 109] payload_rows_per_dst=[9382, 8352, 8848, 9966, 11562, 11364, 13944, 8168] dup_rows_per_dst=[1288, 2454, 1354, 2110, 3456, 3630, 1914, 512] combine_rows_per_src=[9382, 8352, 8848, 9966, 11562, 11364, 13944, 8168] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=275054592 dispatch_meta_bytes=344640 combine_payload_bytes=275054592 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=275399232 ingress_bytes=344260608 link_bytes=344260608 token_bytes=550453824 weight_movement_bytes=69206016 total_nvlink_bytes=619659840 weight_to_token_ratio=0.125725 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.295424953e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77510 payload_rows_remote=66758 dup_rows=20794 dup_rows_remote=18910 dedup_ratio=0.211528 router_tokens_per_expert=[1393, 44, 6, 0, 203, 208, 5157, 0, 7, 947, 1139, 1310, 4, 4392, 1038, 2583, 3854, 8, 650, 2133, 114, 0, 1284, 397, 0, 168, 4005, 136, 915, 119, 1531, 1, 0, 239, 36, 1242, 212, 1001, 8, 2058, 195, 148, 12, 60, 603, 0, 0, 2, 116, 5502, 0, 296, 0, 825, 17, 212, 196, 0, 0, 81, 2, 227, 906, 1210] payload_rows_per_dst=[11978, 6874, 8558, 10752, 9594, 11932, 7416, 10406] dup_rows_per_dst=[2044, 2016, 1728, 1884, 5034, 4058, 1484, 2546] combine_rows_per_src=[11978, 6874, 8558, 10752, 9594, 11932, 7416, 10406] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=284737536 dispatch_meta_bytes=352304 combine_payload_bytes=284737536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=285089840 ingress_bytes=319340544 link_bytes=319340544 token_bytes=569827376 weight_movement_bytes=34603008 total_nvlink_bytes=604430384 weight_to_token_ratio=0.060725 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=22272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.201652758e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=277815296 dispatch_meta_bytes=339376 combine_payload_bytes=277815296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278154672 ingress_bytes=312418304 link_bytes=312418304 token_bytes=555969968 weight_movement_bytes=34603008 total_nvlink_bytes=590572976 weight_to_token_ratio=0.062239 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=25728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.175604926e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=293838848 dispatch_meta_bytes=344728 combine_payload_bytes=293838848 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=294183576 ingress_bytes=363044864 link_bytes=363044864 token_bytes=588022424 weight_movement_bytes=69206016 total_nvlink_bytes=657228440 weight_to_token_ratio=0.117693 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=19328 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.366108596e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78966 payload_rows_remote=63098 dup_rows=19338 dup_rows_remote=7590 dedup_ratio=0.196716 router_tokens_per_expert=[443, 14, 7, 0, 1, 13, 7224, 2, 9, 98, 67, 47, 4, 2298, 5225, 161, 7803, 30, 12, 324, 252, 0, 1953, 10, 5, 3, 623, 5, 13, 12, 6461, 0, 11, 539, 0, 95, 72, 28, 0, 158, 319, 8, 0, 29, 347, 0, 21, 27, 231, 1170, 5, 85, 1, 6866, 5, 11, 10, 0, 1, 367, 0, 429, 35, 5163] payload_rows_per_dst=[9680, 12394, 4886, 13628, 1674, 5958, 14878, 15868] dup_rows_per_dst=[196, 3372, 276, 616, 132, 462, 2536, 11748] combine_rows_per_src=[9680, 12394, 4886, 13628, 1674, 5958, 14878, 15868] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=287629312 dispatch_meta_bytes=333240 combine_payload_bytes=287629312 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=287962552 ingress_bytes=356835328 link_bytes=356835328 token_bytes=575591864 weight_movement_bytes=69206016 total_nvlink_bytes=644797880 weight_to_token_ratio=0.120235 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=29184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.342742612e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=273440768 dispatch_meta_bytes=342672 combine_payload_bytes=273440768 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=273783440 ingress_bytes=342646784 link_bytes=342646784 token_bytes=547224208 weight_movement_bytes=69206016 total_nvlink_bytes=616430224 weight_to_token_ratio=0.126467 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.289352263e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75290 payload_rows_remote=70880 dup_rows=23014 dup_rows_remote=23000 dedup_ratio=0.234111 router_tokens_per_expert=[0, 726, 1211, 0, 0, 8, 402, 0, 1, 1468, 0, 726, 441, 2, 1, 0, 0, 7607, 17, 0, 0, 0, 0, 0, 1, 0, 10, 2, 0, 175, 3151, 0, 0, 0, 0, 0, 0, 12, 0, 14, 78, 7899, 0, 17, 3561, 7600, 9, 5, 5, 1, 0, 0, 5707, 0, 9, 8180, 0, 88, 1, 0, 1, 4, 10, 2] payload_rows_per_dst=[4854, 4410, 15800, 6838, 24, 15572, 11966, 15826] dup_rows_per_dst=[722, 14, 6664, 380, 0, 13690, 1360, 184] combine_rows_per_src=[4854, 4410, 15800, 6838, 24, 15572, 11966, 15826] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79694 payload_rows_remote=66240 dup_rows=18610 dup_rows_remote=10994 dedup_ratio=0.189311 router_tokens_per_expert=[30, 215, 651, 7, 0, 2698, 0, 24, 415, 94, 0, 2103, 6373, 112, 69, 28, 27, 81, 3506, 0, 0, 353, 0, 0, 466, 18, 1807, 111, 42, 98, 499, 10, 634, 0, 0, 0, 0, 3632, 0, 6269, 2254, 1051, 42, 3480, 0, 271, 956, 187, 0, 72, 253, 8, 4, 1034, 217, 4699, 2, 101, 511, 0, 60, 2136, 1369, 73] payload_rows_per_dst=[6592, 13764, 7830, 10948, 13454, 6892, 11474, 8740] dup_rows_per_dst=[658, 4624, 104, 2114, 7616, 528, 1100, 1866] combine_rows_per_src=[6592, 13764, 7830, 10948, 13454, 6892, 11474, 8740] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79452 payload_rows_remote=71012 dup_rows=18852 dup_rows_remote=18230 dedup_ratio=0.191772 router_tokens_per_expert=[2, 28, 317, 0, 0, 3673, 0, 63, 673, 13, 0, 1313, 5676, 43, 10, 59, 41, 9, 2928, 0, 0, 258, 0, 0, 147, 5, 3388, 159, 0, 37, 307, 0, 398, 0, 0, 0, 0, 4364, 0, 7155, 2467, 87, 0, 5209, 0, 25, 1614, 134, 0, 281, 146, 7, 0, 831, 244, 3022, 1, 8, 517, 0, 44, 2188, 1049, 212] payload_rows_per_dst=[7734, 12746, 6418, 13528, 14992, 8058, 8440, 7536] dup_rows_per_dst=[432, 2828, 54, 4976, 8842, 422, 622, 676] combine_rows_per_src=[7734, 12746, 6418, 13528, 14992, 8058, 8440, 7536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81586 payload_rows_remote=70222 dup_rows=16718 dup_rows_remote=13088 dedup_ratio=0.170064 router_tokens_per_expert=[973, 112, 58, 0, 285, 295, 3612, 0, 0, 1638, 1734, 1903, 0, 3456, 128, 3164, 1550, 42, 1355, 2666, 57, 14, 478, 489, 0, 82, 3319, 384, 1821, 210, 222, 0, 0, 24, 10, 1982, 672, 1049, 7, 3765, 38, 292, 12, 184, 271, 0, 68, 12, 46, 6750, 0, 695, 0, 83, 56, 299, 213, 0, 0, 7, 1, 194, 2266, 109] payload_rows_per_dst=[9382, 8352, 8848, 9966, 11562, 11364, 13944, 8168] dup_rows_per_dst=[1288, 2454, 1354, 2110, 3456, 3630, 1914, 512] combine_rows_per_src=[9382, 8352, 8848, 9966, 11562, 11364, 13944, 8168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=288382976 dispatch_meta_bytes=324728 combine_payload_bytes=288382976 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=288707704 ingress_bytes=392192000 link_bytes=392192000 token_bytes=577090680 weight_movement_bytes=103809024 total_nvlink_bytes=680899704 weight_to_token_ratio=0.179883 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.475786922e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=258449408 dispatch_meta_bytes=282752 combine_payload_bytes=258449408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258732160 ingress_bytes=293052416 link_bytes=293052416 token_bytes=517181568 weight_movement_bytes=34603008 total_nvlink_bytes=551784576 weight_to_token_ratio=0.066907 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=19584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.102732649e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=273440768 dispatch_meta_bytes=342672 combine_payload_bytes=273440768 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=273783440 ingress_bytes=342646784 link_bytes=342646784 token_bytes=547224208 weight_movement_bytes=69206016 total_nvlink_bytes=616430224 weight_to_token_ratio=0.126467 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.289352263e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=271319040 dispatch_meta_bytes=308936 combine_payload_bytes=271319040 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=271627976 ingress_bytes=340525056 link_bytes=340525056 token_bytes=542947016 weight_movement_bytes=69206016 total_nvlink_bytes=612153032 weight_to_token_ratio=0.127464 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=28416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.281368371e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=290865152 dispatch_meta_bytes=356968 combine_payload_bytes=290865152 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=291222120 ingress_bytes=394674176 link_bytes=394674176 token_bytes=582087272 weight_movement_bytes=103809024 total_nvlink_bytes=685896296 weight_to_token_ratio=0.178339 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=31232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.485127150e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=287629312 dispatch_meta_bytes=333240 combine_payload_bytes=287629312 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=287962552 ingress_bytes=356835328 link_bytes=356835328 token_bytes=575591864 weight_movement_bytes=69206016 total_nvlink_bytes=644797880 weight_to_token_ratio=0.120235 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=29184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.342742612e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71884 payload_rows_remote=56334 dup_rows=26420 dup_rows_remote=18252 dedup_ratio=0.268758 router_tokens_per_expert=[13, 2206, 1658, 0, 1, 1159, 44, 12, 48, 212, 0, 3049, 6766, 78, 22, 6, 20, 3840, 1967, 0, 0, 24, 1, 0, 300, 63, 1325, 16, 9, 172, 569, 353, 122, 0, 0, 0, 0, 959, 0, 1979, 2661, 5688, 40, 688, 2, 2125, 152, 330, 0, 40, 197, 0, 239, 35, 90, 7207, 24, 738, 472, 0, 44, 844, 517, 26] payload_rows_per_dst=[15550, 9480, 15154, 9492, 1950, 9648, 1188, 9422] dup_rows_per_dst=[8168, 1308, 10964, 1748, 212, 2496, 14, 1510] combine_rows_per_src=[15550, 9480, 15154, 9492, 1950, 9648, 1188, 9422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75290 payload_rows_remote=70880 dup_rows=23014 dup_rows_remote=23000 dedup_ratio=0.234111 router_tokens_per_expert=[0, 726, 1211, 0, 0, 8, 402, 0, 1, 1468, 0, 726, 441, 2, 1, 0, 0, 7607, 17, 0, 0, 0, 0, 0, 1, 0, 10, 2, 0, 175, 3151, 0, 0, 0, 0, 0, 0, 12, 0, 14, 78, 7899, 0, 17, 3561, 7600, 9, 5, 5, 1, 0, 0, 5707, 0, 9, 8180, 0, 88, 1, 0, 1, 4, 10, 2] payload_rows_per_dst=[4854, 4410, 15800, 6838, 24, 15572, 11966, 15826] dup_rows_per_dst=[722, 14, 6664, 380, 0, 13690, 1360, 184] combine_rows_per_src=[4854, 4410, 15800, 6838, 24, 15572, 11966, 15826] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82754 payload_rows_remote=74910 dup_rows=15550 dup_rows_remote=15432 dedup_ratio=0.158183 router_tokens_per_expert=[1, 509, 499, 7, 0, 2410, 0, 11, 106, 47, 0, 3582, 6826, 7, 6, 58, 3, 83, 3830, 1, 0, 64, 0, 0, 441, 25, 2982, 196, 69, 87, 244, 2, 96, 0, 0, 0, 0, 3265, 0, 5670, 1797, 1057, 28, 2978, 0, 94, 716, 105, 0, 113, 190, 0, 0, 359, 760, 4977, 2, 84, 511, 0, 56, 3119, 977, 102] payload_rows_per_dst=[15002, 11430, 7844, 10866, 10854, 5312, 11712, 9734] dup_rows_per_dst=[5524, 1734, 118, 3182, 1656, 168, 1086, 2082] combine_rows_per_src=[15002, 11430, 7844, 10866, 10854, 5312, 11712, 9734] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80862 payload_rows_remote=66818 dup_rows=17442 dup_rows_remote=15346 dedup_ratio=0.177429 router_tokens_per_expert=[1100, 22, 8, 0, 80, 156, 6704, 0, 0, 468, 512, 414, 13, 4933, 1402, 1991, 6605, 96, 96, 1312, 203, 0, 1457, 86, 0, 9, 2096, 154, 422, 47, 2284, 2, 0, 438, 0, 699, 569, 154, 2, 1049, 394, 7, 0, 59, 1307, 0, 11, 5, 1008, 4302, 0, 422, 0, 2972, 2, 90, 55, 0, 0, 171, 0, 210, 289, 2265] payload_rows_per_dst=[14044, 8090, 15512, 8020, 9646, 11694, 8068, 5788] dup_rows_per_dst=[2096, 1510, 8390, 882, 1714, 1738, 920, 192] combine_rows_per_src=[14044, 8090, 15512, 8020, 9646, 11694, 8068, 5788] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=230744064 dispatch_meta_bytes=298344 combine_payload_bytes=230744064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=231042408 ingress_bytes=265347072 link_bytes=265347072 token_bytes=461786472 weight_movement_bytes=34603008 total_nvlink_bytes=496389480 weight_to_token_ratio=0.074933 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=19584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.984796697e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=290324480 dispatch_meta_bytes=375520 combine_payload_bytes=290324480 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=290700000 ingress_bytes=359530496 link_bytes=359530496 token_bytes=581024480 weight_movement_bytes=69206016 total_nvlink_bytes=650230496 weight_to_token_ratio=0.119110 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=22144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.352884312e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79310 payload_rows_remote=66752 dup_rows=18994 dup_rows_remote=17420 dedup_ratio=0.193217 router_tokens_per_expert=[979, 128, 30, 0, 389, 225, 2862, 0, 17, 2581, 1425, 2729, 0, 3652, 46, 3005, 842, 18, 837, 2347, 4, 4, 298, 733, 0, 239, 3973, 267, 2601, 294, 189, 4, 0, 35, 20, 1688, 207, 1826, 2, 4435, 11, 115, 24, 140, 155, 0, 11, 4, 0, 6023, 0, 653, 5, 30, 34, 321, 337, 0, 0, 3, 0, 130, 2129, 96] payload_rows_per_dst=[8100, 9502, 7708, 11804, 12116, 10830, 12558, 6692] dup_rows_per_dst=[1126, 4094, 774, 3330, 4310, 3404, 1574, 382] combine_rows_per_src=[8100, 9502, 7708, 11804, 12116, 10830, 12558, 6692] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78288 payload_rows_remote=71346 dup_rows=20016 dup_rows_remote=19768 dedup_ratio=0.203613 router_tokens_per_expert=[21, 89, 416, 0, 0, 2759, 0, 30, 284, 0, 0, 2444, 6367, 79, 4, 20, 92, 21, 2571, 0, 0, 198, 0, 0, 268, 0, 3099, 80, 47, 53, 240, 5, 334, 0, 0, 0, 0, 4020, 0, 7093, 2665, 304, 7, 3871, 0, 59, 759, 105, 0, 244, 322, 2, 0, 466, 423, 3862, 1, 39, 545, 0, 45, 3489, 1231, 79] payload_rows_per_dst=[6112, 13496, 5714, 11696, 14674, 6942, 9794, 9860] dup_rows_per_dst=[518, 4900, 50, 3630, 8220, 248, 844, 1606] combine_rows_per_src=[6112, 13496, 5714, 11696, 14674, 6942, 9794, 9860] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80806 payload_rows_remote=70146 dup_rows=17498 dup_rows_remote=14318 dedup_ratio=0.177999 router_tokens_per_expert=[947, 57, 25, 1, 351, 195, 3951, 0, 52, 1225, 1595, 2502, 5, 4577, 407, 2528, 2531, 51, 702, 2865, 109, 0, 629, 572, 1, 121, 4058, 507, 1852, 162, 456, 0, 0, 94, 72, 1605, 477, 1094, 26, 3552, 264, 94, 35, 64, 389, 0, 12, 2, 105, 4972, 0, 499, 0, 254, 62, 117, 282, 0, 0, 13, 12, 122, 1227, 673] payload_rows_per_dst=[10050, 8668, 8720, 11502, 10660, 11946, 10690, 8570] dup_rows_per_dst=[1004, 2904, 1136, 2812, 3180, 3984, 1328, 1150] combine_rows_per_src=[10050, 8668, 8720, 11502, 10660, 11946, 10690, 8570] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=306831360 dispatch_meta_bytes=361368 combine_payload_bytes=306831360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307192728 ingress_bytes=341434368 link_bytes=341434368 token_bytes=614024088 weight_movement_bytes=34603008 total_nvlink_bytes=648627096 weight_to_token_ratio=0.056354 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.284790039e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=273686528 dispatch_meta_bytes=328656 combine_payload_bytes=273686528 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=274015184 ingress_bytes=342892544 link_bytes=342892544 token_bytes=547701712 weight_movement_bytes=69206016 total_nvlink_bytes=616907728 weight_to_token_ratio=0.126357 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=38784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.290277038e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80372 payload_rows_remote=70978 dup_rows=17932 dup_rows_remote=16610 dedup_ratio=0.182414 router_tokens_per_expert=[7, 379, 1375, 0, 0, 2184, 1, 22, 164, 135, 0, 2560, 6624, 70, 2, 11, 8, 915, 2484, 0, 0, 234, 0, 0, 210, 25, 1305, 110, 0, 119, 1283, 41, 336, 0, 0, 0, 0, 2527, 0, 4702, 3632, 2967, 46, 2265, 0, 1218, 448, 274, 0, 139, 176, 8, 80, 622, 250, 5662, 2, 128, 409, 0, 62, 1468, 1391, 72] payload_rows_per_dst=[13452, 6914, 7032, 9394, 10004, 9782, 13100, 10694] dup_rows_per_dst=[5490, 1212, 250, 1322, 5126, 1454, 774, 2304] combine_rows_per_src=[13452, 6914, 7032, 9394, 10004, 9782, 13100, 10694] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=273416192 dispatch_meta_bytes=336688 combine_payload_bytes=273416192 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=273752880 ingress_bytes=377225216 link_bytes=377225216 token_bytes=547169072 weight_movement_bytes=103809024 total_nvlink_bytes=650978096 weight_to_token_ratio=0.189720 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=22016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.419468119e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77394 payload_rows_remote=67152 dup_rows=20910 dup_rows_remote=19008 dedup_ratio=0.212708 router_tokens_per_expert=[1059, 91, 98, 0, 140, 219, 4358, 1, 74, 1120, 1905, 2217, 3, 4425, 142, 4134, 2276, 246, 968, 1480, 123, 5, 334, 141, 0, 102, 2775, 734, 2009, 247, 308, 2, 17, 65, 2, 1293, 1127, 532, 3, 2273, 238, 46, 26, 148, 574, 0, 23, 297, 204, 6348, 0, 1091, 0, 143, 93, 574, 272, 0, 0, 14, 0, 66, 1690, 257] payload_rows_per_dst=[10562, 11844, 10242, 5862, 15304, 11006, 4148, 8426] dup_rows_per_dst=[1370, 5426, 1902, 942, 8016, 2468, 62, 724] combine_rows_per_src=[10562, 11844, 10242, 5862, 15304, 11006, 4148, 8426] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82798 payload_rows_remote=67232 dup_rows=15506 dup_rows_remote=14808 dedup_ratio=0.157735 router_tokens_per_expert=[21, 981, 2325, 3, 0, 199, 219, 0, 0, 1005, 3, 1153, 2554, 7, 0, 0, 0, 7048, 104, 0, 0, 0, 2, 0, 85, 33, 217, 23, 4, 204, 3216, 178, 0, 2, 2, 0, 1, 80, 3, 407, 930, 7610, 11, 135, 932, 6476, 48, 149, 6, 0, 10, 0, 4170, 2, 47, 8025, 4, 133, 99, 0, 7, 137, 138, 4] payload_rows_per_dst=[6850, 7496, 14256, 7800, 550, 14118, 16162, 15566] dup_rows_per_dst=[646, 1948, 52, 390, 130, 2974, 8668, 698] combine_rows_per_src=[6850, 7496, 14256, 7800, 550, 14118, 16162, 15566] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=292233216 dispatch_meta_bytes=364456 combine_payload_bytes=292233216 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=292597672 ingress_bytes=430645248 link_bytes=430645248 token_bytes=584830888 weight_movement_bytes=138412032 total_nvlink_bytes=723242920 weight_to_token_ratio=0.236670 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=16512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.620483398e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=287318016 dispatch_meta_bytes=337856 combine_payload_bytes=287318016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=287655872 ingress_bytes=321921024 link_bytes=321921024 token_bytes=574973888 weight_movement_bytes=34603008 total_nvlink_bytes=609576896 weight_to_token_ratio=0.060182 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=17408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.211362897e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80372 payload_rows_remote=70978 dup_rows=17932 dup_rows_remote=16610 dedup_ratio=0.182414 router_tokens_per_expert=[7, 379, 1375, 0, 0, 2184, 1, 22, 164, 135, 0, 2560, 6624, 70, 2, 11, 8, 915, 2484, 0, 0, 234, 0, 0, 210, 25, 1305, 110, 0, 119, 1283, 41, 336, 0, 0, 0, 0, 2527, 0, 4702, 3632, 2967, 46, 2265, 0, 1218, 448, 274, 0, 139, 176, 8, 80, 622, 250, 5662, 2, 128, 409, 0, 62, 1468, 1391, 72] payload_rows_per_dst=[13452, 6914, 7032, 9394, 10004, 9782, 13100, 10694] dup_rows_per_dst=[5490, 1212, 250, 1322, 5126, 1454, 774, 2304] combine_rows_per_src=[13452, 6914, 7032, 9394, 10004, 9782, 13100, 10694] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79106 payload_rows_remote=72500 dup_rows=19198 dup_rows_remote=18900 dedup_ratio=0.195292 router_tokens_per_expert=[0, 0, 3, 412, 192, 3203, 18, 119, 5, 4, 136, 100, 2367, 150, 728, 1535, 6402, 163, 319, 888, 357, 1770, 185, 2, 233, 1192, 4251, 512, 66, 795, 0, 530, 1166, 125, 0, 2, 1746, 74, 339, 0, 219, 144, 0, 92, 81, 2108, 4, 210, 1923, 1300, 487, 184, 253, 2571, 6, 93, 3040, 3, 1546, 0, 1000, 178, 3620, 1] payload_rows_per_dst=[11068, 8546, 6702, 11622, 6606, 13734, 10700, 10128] dup_rows_per_dst=[4066, 1504, 666, 3536, 298, 4786, 2934, 1408] combine_rows_per_src=[11068, 8546, 6702, 11622, 6606, 13734, 10700, 10128] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79626 payload_rows_remote=68464 dup_rows=18678 dup_rows_remote=16844 dedup_ratio=0.190002 router_tokens_per_expert=[0, 0, 5, 494, 133, 3401, 5, 11, 0, 0, 32, 51, 3271, 22, 567, 1828, 7224, 404, 432, 572, 105, 1183, 526, 0, 154, 761, 2282, 66, 13, 1395, 0, 1174, 1561, 25, 0, 0, 653, 96, 380, 2, 90, 109, 0, 20, 13, 3069, 6, 162, 257, 1451, 405, 27, 243, 4086, 5, 24, 3069, 0, 1601, 0, 1456, 42, 4189, 0] payload_rows_per_dst=[11280, 10296, 5788, 9794, 5060, 15294, 11162, 10952] dup_rows_per_dst=[5196, 1246, 656, 1896, 374, 6092, 1834, 1384] combine_rows_per_src=[11280, 10296, 5788, 9794, 5060, 15294, 11162, 10952] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78288 payload_rows_remote=71346 dup_rows=20016 dup_rows_remote=19768 dedup_ratio=0.203613 router_tokens_per_expert=[21, 89, 416, 0, 0, 2759, 0, 30, 284, 0, 0, 2444, 6367, 79, 4, 20, 92, 21, 2571, 0, 0, 198, 0, 0, 268, 0, 3099, 80, 47, 53, 240, 5, 334, 0, 0, 0, 0, 4020, 0, 7093, 2665, 304, 7, 3871, 0, 59, 759, 105, 0, 244, 322, 2, 0, 466, 423, 3862, 1, 39, 545, 0, 45, 3489, 1231, 79] payload_rows_per_dst=[6112, 13496, 5714, 11696, 14674, 6942, 9794, 9860] dup_rows_per_dst=[518, 4900, 50, 3630, 8220, 248, 844, 1606] combine_rows_per_src=[6112, 13496, 5714, 11696, 14674, 6942, 9794, 9860] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=275054592 dispatch_meta_bytes=344640 combine_payload_bytes=275054592 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=275399232 ingress_bytes=344260608 link_bytes=344260608 token_bytes=550453824 weight_movement_bytes=69206016 total_nvlink_bytes=619659840 weight_to_token_ratio=0.125725 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.295424953e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=275382272 dispatch_meta_bytes=328160 combine_payload_bytes=275382272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275710432 ingress_bytes=309985280 link_bytes=309985280 token_bytes=551092704 weight_movement_bytes=34603008 total_nvlink_bytes=585695712 weight_to_token_ratio=0.062790 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=23808 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.166449653e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=290725888 dispatch_meta_bytes=350352 combine_payload_bytes=290725888 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=291076240 ingress_bytes=359931904 link_bytes=359931904 token_bytes=581802128 weight_movement_bytes=69206016 total_nvlink_bytes=651008144 weight_to_token_ratio=0.118951 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=18688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.354394778e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=296960000 dispatch_meta_bytes=365600 combine_payload_bytes=296960000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=297325600 ingress_bytes=331563008 link_bytes=331563008 token_bytes=594285600 weight_movement_bytes=34603008 total_nvlink_bytes=628888608 weight_to_token_ratio=0.058226 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=32128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.247644906e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81534 payload_rows_remote=66178 dup_rows=16770 dup_rows_remote=7300 dedup_ratio=0.170593 router_tokens_per_expert=[0, 1, 0, 0, 0, 6, 0, 1173, 4982, 0, 227, 15, 5, 571, 0, 0, 35, 0, 5, 173, 5, 914, 1, 0, 1, 4, 8155, 0, 4, 3, 0, 4, 2, 377, 0, 0, 8157, 2, 2, 0, 0, 5, 0, 1, 6685, 4, 0, 0, 7928, 3, 10, 463, 2, 12, 0, 0, 30, 5111, 2, 11, 1, 869, 41, 3150] payload_rows_per_dst=[2424, 15356, 2236, 16334, 16322, 13382, 1124, 14356] dup_rows_per_dst=[78, 9470, 30, 2474, 758, 8, 20, 3932] combine_rows_per_src=[2424, 15356, 2236, 16334, 16322, 13382, 1124, 14356] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=275382272 dispatch_meta_bytes=328160 combine_payload_bytes=275382272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275710432 ingress_bytes=309985280 link_bytes=309985280 token_bytes=551092704 weight_movement_bytes=34603008 total_nvlink_bytes=585695712 weight_to_token_ratio=0.062790 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=23808 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.166449653e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76610 payload_rows_remote=67030 dup_rows=21694 dup_rows_remote=18878 dedup_ratio=0.220683 router_tokens_per_expert=[0, 1, 3, 97, 66, 447, 101, 821, 489, 0, 191, 230, 495, 744, 35, 838, 2512, 120, 89, 2823, 220, 3494, 136, 0, 162, 550, 7115, 99, 65, 231, 0, 95, 210, 496, 0, 0, 6506, 31, 56, 0, 70, 72, 0, 356, 1538, 317, 0, 63, 6399, 1067, 677, 862, 227, 630, 2, 218, 1962, 638, 445, 0, 80, 386, 2700, 875] payload_rows_per_dst=[9580, 14692, 15038, 8278, 13596, 4712, 5992, 4722] dup_rows_per_dst=[2816, 4150, 11608, 498, 1002, 120, 1374, 126] combine_rows_per_src=[9580, 14692, 15038, 8278, 13596, 4712, 5992, 4722] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=280428544 dispatch_meta_bytes=341232 combine_payload_bytes=280428544 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=280769776 ingress_bytes=384237568 link_bytes=384237568 token_bytes=561198320 weight_movement_bytes=103809024 total_nvlink_bytes=665007344 weight_to_token_ratio=0.184977 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.445855035e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=292233216 dispatch_meta_bytes=364456 combine_payload_bytes=292233216 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=292597672 ingress_bytes=430645248 link_bytes=430645248 token_bytes=584830888 weight_movement_bytes=138412032 total_nvlink_bytes=723242920 weight_to_token_ratio=0.236670 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=16512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.620483398e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77838 payload_rows_remote=61756 dup_rows=20466 dup_rows_remote=13174 dedup_ratio=0.208191 router_tokens_per_expert=[0, 9, 1, 786, 316, 2386, 88, 141, 17, 0, 135, 50, 1272, 47, 290, 1738, 7318, 270, 913, 1025, 197, 1722, 242, 0, 127, 1087, 4028, 311, 119, 724, 0, 521, 506, 309, 0, 3, 1946, 358, 61, 1, 9, 394, 1, 206, 89, 1067, 0, 193, 2255, 2711, 415, 77, 248, 2750, 11, 56, 2891, 2, 1437, 0, 630, 49, 4594, 3] payload_rows_per_dst=[12296, 6698, 16082, 11450, 6052, 3660, 12682, 8918] dup_rows_per_dst=[4482, 400, 7292, 2384, 316, 258, 4364, 970] combine_rows_per_src=[12296, 6698, 16082, 11450, 6052, 3660, 12682, 8918] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=271065088 dispatch_meta_bytes=293912 combine_payload_bytes=271065088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271359000 ingress_bytes=305668096 link_bytes=305668096 token_bytes=542424088 weight_movement_bytes=34603008 total_nvlink_bytes=577027096 weight_to_token_ratio=0.063793 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.150204437e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71884 payload_rows_remote=56334 dup_rows=26420 dup_rows_remote=18252 dedup_ratio=0.268758 router_tokens_per_expert=[13, 2206, 1658, 0, 1, 1159, 44, 12, 48, 212, 0, 3049, 6766, 78, 22, 6, 20, 3840, 1967, 0, 0, 24, 1, 0, 300, 63, 1325, 16, 9, 172, 569, 353, 122, 0, 0, 0, 0, 959, 0, 1979, 2661, 5688, 40, 688, 2, 2125, 152, 330, 0, 40, 197, 0, 239, 35, 90, 7207, 24, 738, 472, 0, 44, 844, 517, 26] payload_rows_per_dst=[15550, 9480, 15154, 9492, 1950, 9648, 1188, 9422] dup_rows_per_dst=[8168, 1308, 10964, 1748, 212, 2496, 14, 1510] combine_rows_per_src=[15550, 9480, 15154, 9492, 1950, 9648, 1188, 9422] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79452 payload_rows_remote=71012 dup_rows=18852 dup_rows_remote=18230 dedup_ratio=0.191772 router_tokens_per_expert=[2, 28, 317, 0, 0, 3673, 0, 63, 673, 13, 0, 1313, 5676, 43, 10, 59, 41, 9, 2928, 0, 0, 258, 0, 0, 147, 5, 3388, 159, 0, 37, 307, 0, 398, 0, 0, 0, 0, 4364, 0, 7155, 2467, 87, 0, 5209, 0, 25, 1614, 134, 0, 281, 146, 7, 0, 831, 244, 3022, 1, 8, 517, 0, 44, 2188, 1049, 212] payload_rows_per_dst=[7734, 12746, 6418, 13528, 14992, 8058, 8440, 7536] dup_rows_per_dst=[432, 2828, 54, 4976, 8842, 422, 622, 676] combine_rows_per_src=[7734, 12746, 6418, 13528, 14992, 8058, 8440, 7536] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79946 payload_rows_remote=64872 dup_rows=18358 dup_rows_remote=13520 dedup_ratio=0.186747 router_tokens_per_expert=[0, 0, 0, 588, 94, 3074, 19, 28, 0, 0, 87, 22, 2672, 22, 475, 1816, 7276, 304, 302, 775, 183, 1811, 285, 0, 159, 1228, 3167, 139, 81, 437, 0, 980, 1058, 7, 0, 0, 1327, 65, 156, 0, 69, 401, 0, 12, 19, 1993, 0, 186, 910, 3057, 317, 60, 224, 3180, 13, 48, 3143, 1, 1319, 0, 877, 9, 4677, 0] payload_rows_per_dst=[12282, 8990, 6662, 9998, 5138, 15074, 12044, 9758] dup_rows_per_dst=[4678, 1198, 658, 2384, 88, 4838, 3574, 940] combine_rows_per_src=[12282, 8990, 6662, 9998, 5138, 15074, 12044, 9758] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=252952576 dispatch_meta_bytes=299720 combine_payload_bytes=252952576 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=253252296 ingress_bytes=322158592 link_bytes=322158592 token_bytes=506204872 weight_movement_bytes=69206016 total_nvlink_bytes=575410888 weight_to_token_ratio=0.136715 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.212256846e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79694 payload_rows_remote=66240 dup_rows=18610 dup_rows_remote=10994 dedup_ratio=0.189311 router_tokens_per_expert=[30, 215, 651, 7, 0, 2698, 0, 24, 415, 94, 0, 2103, 6373, 112, 69, 28, 27, 81, 3506, 0, 0, 353, 0, 0, 466, 18, 1807, 111, 42, 98, 499, 10, 634, 0, 0, 0, 0, 3632, 0, 6269, 2254, 1051, 42, 3480, 0, 271, 956, 187, 0, 72, 253, 8, 4, 1034, 217, 4699, 2, 101, 511, 0, 60, 2136, 1369, 73] payload_rows_per_dst=[6592, 13764, 7830, 10948, 13454, 6892, 11474, 8740] dup_rows_per_dst=[658, 4624, 104, 2114, 7616, 528, 1100, 1866] combine_rows_per_src=[6592, 13764, 7830, 10948, 13454, 6892, 11474, 8740] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=230744064 dispatch_meta_bytes=298344 combine_payload_bytes=230744064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=231042408 ingress_bytes=265347072 link_bytes=265347072 token_bytes=461786472 weight_movement_bytes=34603008 total_nvlink_bytes=496389480 weight_to_token_ratio=0.074933 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=19584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.984796697e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83172 payload_rows_remote=69380 dup_rows=15132 dup_rows_remote=12572 dedup_ratio=0.153931 router_tokens_per_expert=[0, 0, 0, 230, 51, 2217, 15, 431, 31, 0, 162, 244, 1508, 475, 308, 639, 4721, 149, 248, 1356, 402, 2833, 234, 0, 206, 857, 5734, 135, 16, 496, 0, 732, 777, 259, 0, 0, 3857, 70, 197, 0, 224, 48, 0, 78, 214, 1318, 13, 203, 3128, 605, 779, 670, 258, 1698, 9, 121, 3535, 110, 980, 0, 750, 875, 3709, 237] payload_rows_per_dst=[10246, 6150, 13872, 13792, 9864, 6634, 11516, 11098] dup_rows_per_dst=[3060, 584, 3152, 2560, 456, 424, 3020, 1876] combine_rows_per_src=[10246, 6150, 13872, 13792, 9864, 6634, 11516, 11098] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=274554880 dispatch_meta_bytes=343632 combine_payload_bytes=274554880 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=274898512 ingress_bytes=343760896 link_bytes=343760896 token_bytes=549453392 weight_movement_bytes=69206016 total_nvlink_bytes=618659408 weight_to_token_ratio=0.125954 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=24064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.293544577e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82754 payload_rows_remote=74910 dup_rows=15550 dup_rows_remote=15432 dedup_ratio=0.158183 router_tokens_per_expert=[1, 509, 499, 7, 0, 2410, 0, 11, 106, 47, 0, 3582, 6826, 7, 6, 58, 3, 83, 3830, 1, 0, 64, 0, 0, 441, 25, 2982, 196, 69, 87, 244, 2, 96, 0, 0, 0, 0, 3265, 0, 5670, 1797, 1057, 28, 2978, 0, 94, 716, 105, 0, 113, 190, 0, 0, 359, 760, 4977, 2, 84, 511, 0, 56, 3119, 977, 102] payload_rows_per_dst=[15002, 11430, 7844, 10866, 10854, 5312, 11712, 9734] dup_rows_per_dst=[5524, 1734, 118, 3182, 1656, 168, 1086, 2082] combine_rows_per_src=[15002, 11430, 7844, 10866, 10854, 5312, 11712, 9734] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73188 payload_rows_remote=59994 dup_rows=25116 dup_rows_remote=20574 dedup_ratio=0.255493 router_tokens_per_expert=[2, 29, 0, 11, 7, 7, 8, 1867, 2536, 0, 256, 25, 13, 1588, 5, 89, 286, 80, 40, 1301, 4, 2479, 27, 0, 106, 243, 8018, 2, 0, 8, 0, 26, 0, 498, 0, 0, 7936, 0, 2, 0, 10, 112, 1, 70, 4123, 17, 5, 20, 7673, 70, 468, 663, 24, 86, 14, 83, 346, 4012, 12, 2, 5, 1320, 445, 2072] payload_rows_per_dst=[4576, 7832, 7602, 1070, 15992, 8986, 13936, 13194] dup_rows_per_dst=[176, 1192, 260, 234, 16382, 302, 2028, 4542] combine_rows_per_src=[4576, 7832, 7602, 1070, 15992, 8986, 13936, 13194] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=290865152 dispatch_meta_bytes=356968 combine_payload_bytes=290865152 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=291222120 ingress_bytes=394674176 link_bytes=394674176 token_bytes=582087272 weight_movement_bytes=103809024 total_nvlink_bytes=685896296 weight_to_token_ratio=0.178339 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=31232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.485127150e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265715712 dispatch_meta_bytes=313568 combine_payload_bytes=265715712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266029280 ingress_bytes=300318720 link_bytes=300318720 token_bytes=531744992 weight_movement_bytes=34603008 total_nvlink_bytes=566348000 weight_to_token_ratio=0.065074 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=22656 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130075166e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=271319040 dispatch_meta_bytes=308936 combine_payload_bytes=271319040 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=271627976 ingress_bytes=340525056 link_bytes=340525056 token_bytes=542947016 weight_movement_bytes=69206016 total_nvlink_bytes=612153032 weight_to_token_ratio=0.127464 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=28416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.281368371e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83172 payload_rows_remote=69380 dup_rows=15132 dup_rows_remote=12572 dedup_ratio=0.153931 router_tokens_per_expert=[0, 0, 0, 230, 51, 2217, 15, 431, 31, 0, 162, 244, 1508, 475, 308, 639, 4721, 149, 248, 1356, 402, 2833, 234, 0, 206, 857, 5734, 135, 16, 496, 0, 732, 777, 259, 0, 0, 3857, 70, 197, 0, 224, 48, 0, 78, 214, 1318, 13, 203, 3128, 605, 779, 670, 258, 1698, 9, 121, 3535, 110, 980, 0, 750, 875, 3709, 237] payload_rows_per_dst=[10246, 6150, 13872, 13792, 9864, 6634, 11516, 11098] dup_rows_per_dst=[3060, 584, 3152, 2560, 456, 424, 3020, 1876] combine_rows_per_src=[10246, 6150, 13872, 13792, 9864, 6634, 11516, 11098] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77762 payload_rows_remote=69146 dup_rows=20542 dup_rows_remote=19678 dedup_ratio=0.208964 router_tokens_per_expert=[30, 2351, 1, 418, 301, 1154, 1143, 1, 5084, 237, 676, 682, 0, 53, 449, 1147, 0, 2, 35, 29, 0, 2729, 1081, 5, 0, 59, 2686, 1124, 9, 136, 0, 0, 277, 1412, 235, 152, 366, 2740, 142, 713, 289, 595, 4225, 250, 17, 123, 0, 2058, 200, 588, 0, 1596, 3, 1893, 11, 1132, 0, 1685, 1638, 1814, 0, 3376, 0, 0] payload_rows_per_dst=[7542, 13200, 9760, 8546, 8616, 5672, 14182, 10244] dup_rows_per_dst=[948, 3226, 826, 1790, 864, 992, 5114, 6782] combine_rows_per_src=[7542, 13200, 9760, 8546, 8616, 5672, 14182, 10244] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78298 payload_rows_remote=70782 dup_rows=20006 dup_rows_remote=19978 dedup_ratio=0.203512 router_tokens_per_expert=[0, 7, 0, 7852, 8150, 6954, 0, 0, 54, 0, 0, 10, 0, 1, 4, 3703, 0, 3, 0, 0, 0, 14, 1649, 0, 0, 80, 34, 427, 0, 0, 0, 0, 4821, 8170, 44, 0, 0, 7, 0, 0, 1, 18, 4820, 255, 0, 2, 0, 5, 1, 872, 0, 112, 0, 7, 0, 13, 1046, 4, 2, 4, 0, 6, 0, 0] payload_rows_per_dst=[16350, 7516, 16364, 14078, 9698, 10172, 2004, 2116] dup_rows_per_dst=[15668, 28, 3308, 912, 46, 30, 6, 8] combine_rows_per_src=[16350, 7516, 16364, 14078, 9698, 10172, 2004, 2116] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=284180480 dispatch_meta_bytes=327808 combine_payload_bytes=284180480 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=284508288 ingress_bytes=387989504 link_bytes=387989504 token_bytes=568688768 weight_movement_bytes=103809024 total_nvlink_bytes=672497792 weight_to_token_ratio=0.182541 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=31104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.459973268e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76808 payload_rows_remote=63098 dup_rows=21496 dup_rows_remote=17342 dedup_ratio=0.218669 router_tokens_per_expert=[70, 1795, 0, 83, 9, 392, 1714, 0, 3554, 251, 550, 899, 0, 32, 769, 410, 0, 0, 36, 42, 0, 4218, 498, 0, 0, 49, 2539, 554, 21, 380, 0, 0, 158, 213, 117, 207, 90, 5198, 69, 758, 669, 506, 3116, 298, 60, 75, 0, 2373, 440, 312, 0, 790, 3, 3106, 1, 1164, 0, 1304, 2064, 2934, 0, 4262, 0, 0] payload_rows_per_dst=[6522, 5404, 9734, 6862, 12062, 6606, 13710, 15908] dup_rows_per_dst=[820, 418, 280, 1008, 1160, 1356, 4154, 12300] combine_rows_per_src=[6522, 5404, 9734, 6862, 12062, 6606, 13710, 15908] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79946 payload_rows_remote=64872 dup_rows=18358 dup_rows_remote=13520 dedup_ratio=0.186747 router_tokens_per_expert=[0, 0, 0, 588, 94, 3074, 19, 28, 0, 0, 87, 22, 2672, 22, 475, 1816, 7276, 304, 302, 775, 183, 1811, 285, 0, 159, 1228, 3167, 139, 81, 437, 0, 980, 1058, 7, 0, 0, 1327, 65, 156, 0, 69, 401, 0, 12, 19, 1993, 0, 186, 910, 3057, 317, 60, 224, 3180, 13, 48, 3143, 1, 1319, 0, 877, 9, 4677, 0] payload_rows_per_dst=[12282, 8990, 6662, 9998, 5138, 15074, 12044, 9758] dup_rows_per_dst=[4678, 1198, 658, 2384, 88, 4838, 3574, 940] combine_rows_per_src=[12282, 8990, 6662, 9998, 5138, 15074, 12044, 9758] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=306831360 dispatch_meta_bytes=361368 combine_payload_bytes=306831360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307192728 ingress_bytes=341434368 link_bytes=341434368 token_bytes=614024088 weight_movement_bytes=34603008 total_nvlink_bytes=648627096 weight_to_token_ratio=0.056354 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.284790039e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=245735424 dispatch_meta_bytes=322272 combine_payload_bytes=245735424 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=246057696 ingress_bytes=349544448 link_bytes=349544448 token_bytes=491793120 weight_movement_bytes=103809024 total_nvlink_bytes=595602144 weight_to_token_ratio=0.211083 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=16000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.315307617e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=284180480 dispatch_meta_bytes=327808 combine_payload_bytes=284180480 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=284508288 ingress_bytes=387989504 link_bytes=387989504 token_bytes=568688768 weight_movement_bytes=103809024 total_nvlink_bytes=672497792 weight_to_token_ratio=0.182541 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=31104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.459973268e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=283222016 dispatch_meta_bytes=355296 combine_payload_bytes=283222016 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=283577312 ingress_bytes=352428032 link_bytes=352428032 token_bytes=566799328 weight_movement_bytes=69206016 total_nvlink_bytes=636005344 weight_to_token_ratio=0.122100 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.326158312e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78298 payload_rows_remote=70782 dup_rows=20006 dup_rows_remote=19978 dedup_ratio=0.203512 router_tokens_per_expert=[0, 7, 0, 7852, 8150, 6954, 0, 0, 54, 0, 0, 10, 0, 1, 4, 3703, 0, 3, 0, 0, 0, 14, 1649, 0, 0, 80, 34, 427, 0, 0, 0, 0, 4821, 8170, 44, 0, 0, 7, 0, 0, 1, 18, 4820, 255, 0, 2, 0, 5, 1, 872, 0, 112, 0, 7, 0, 13, 1046, 4, 2, 4, 0, 6, 0, 0] payload_rows_per_dst=[16350, 7516, 16364, 14078, 9698, 10172, 2004, 2116] dup_rows_per_dst=[15668, 28, 3308, 912, 46, 30, 6, 8] combine_rows_per_src=[16350, 7516, 16364, 14078, 9698, 10172, 2004, 2116] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=258449408 dispatch_meta_bytes=321760 combine_payload_bytes=258449408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258771168 ingress_bytes=293052416 link_bytes=293052416 token_bytes=517220576 weight_movement_bytes=34603008 total_nvlink_bytes=551823584 weight_to_token_ratio=0.066902 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.102732649e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78724 payload_rows_remote=65572 dup_rows=19580 dup_rows_remote=13332 dedup_ratio=0.199178 router_tokens_per_expert=[8, 1075, 0, 4188, 4144, 3754, 285, 0, 5817, 20, 102, 67, 0, 73, 111, 3299, 0, 72, 2, 0, 0, 659, 1328, 1, 0, 82, 779, 1389, 6, 128, 24, 0, 1042, 5843, 705, 79, 61, 802, 84, 118, 92, 130, 5856, 723, 4, 44, 0, 1047, 12, 406, 0, 1623, 0, 314, 4, 426, 0, 684, 700, 249, 1, 690, 0, 0] payload_rows_per_dst=[13152, 14052, 13512, 10528, 5502, 13436, 5350, 3192] dup_rows_per_dst=[6248, 4926, 2298, 1796, 280, 2356, 220, 1456] combine_rows_per_src=[13152, 14052, 13512, 10528, 5502, 13436, 5350, 3192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265715712 dispatch_meta_bytes=313568 combine_payload_bytes=265715712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266029280 ingress_bytes=300318720 link_bytes=300318720 token_bytes=531744992 weight_movement_bytes=34603008 total_nvlink_bytes=566348000 weight_to_token_ratio=0.065074 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=22656 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130075166e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75420 payload_rows_remote=65818 dup_rows=22884 dup_rows_remote=21700 dedup_ratio=0.232788 router_tokens_per_expert=[6, 2004, 0, 539, 283, 987, 422, 0, 6267, 44, 37, 319, 0, 107, 555, 1525, 0, 18, 54, 98, 0, 2205, 1605, 2, 0, 83, 1616, 513, 81, 335, 85, 0, 169, 1411, 600, 173, 392, 2975, 409, 853, 699, 1230, 4613, 509, 30, 174, 0, 3894, 8, 339, 0, 1369, 2, 1297, 52, 1391, 0, 1674, 1688, 1206, 5, 2200, 0, 0] payload_rows_per_dst=[5908, 13898, 9602, 6604, 9462, 12780, 8180, 8986] dup_rows_per_dst=[600, 3810, 1184, 796, 1680, 9518, 736, 4560] combine_rows_per_src=[5908, 13898, 9602, 6604, 9462, 12780, 8180, 8986] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76610 payload_rows_remote=67030 dup_rows=21694 dup_rows_remote=18878 dedup_ratio=0.220683 router_tokens_per_expert=[0, 1, 3, 97, 66, 447, 101, 821, 489, 0, 191, 230, 495, 744, 35, 838, 2512, 120, 89, 2823, 220, 3494, 136, 0, 162, 550, 7115, 99, 65, 231, 0, 95, 210, 496, 0, 0, 6506, 31, 56, 0, 70, 72, 0, 356, 1538, 317, 0, 63, 6399, 1067, 677, 862, 227, 630, 2, 218, 1962, 638, 445, 0, 80, 386, 2700, 875] payload_rows_per_dst=[9580, 14692, 15038, 8278, 13596, 4712, 5992, 4722] dup_rows_per_dst=[2816, 4150, 11608, 498, 1002, 120, 1374, 126] combine_rows_per_src=[9580, 14692, 15038, 8278, 13596, 4712, 5992, 4722] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 2, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12878820176 prefetch_bytes=865075200 reduce_bytes=588251136 weight_movement_bytes=1453326336 egress_bytes=6443340624 ingress_bytes=7888805888 link_bytes=7888805888 total_nvlink_bytes=14332146512 weight_to_token_ratio=0.112846 prefetch_share_of_ingress=0.109659 dedup_ratio=0.201599 prefetch_slots=50 unused_prefetch_slots=318 slot_utilization=0.135870 reduce_slots=17 remote_demand=25 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028152 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.290649629e-04 prefetch_bytes_per_gemm_flop=1.415307971e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12492842888 prefetch_bytes=726663168 reduce_bytes=622854144 weight_movement_bytes=1349517312 egress_bytes=6250284936 ingress_bytes=7592075264 link_bytes=7592075264 total_nvlink_bytes=13842360200 weight_to_token_ratio=0.108023 prefetch_share_of_ingress=0.095713 dedup_ratio=0.215947 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=18 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027826 gemm_inner_pad_upper_bound=28321 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.242102957e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=289923072 dispatch_meta_bytes=363040 combine_payload_bytes=289923072 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=290286112 ingress_bytes=393732096 link_bytes=393732096 token_bytes=580209184 weight_movement_bytes=103809024 total_nvlink_bytes=684018208 weight_to_token_ratio=0.178917 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=21888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.481582179e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79626 payload_rows_remote=68464 dup_rows=18678 dup_rows_remote=16844 dedup_ratio=0.190002 router_tokens_per_expert=[0, 0, 5, 494, 133, 3401, 5, 11, 0, 0, 32, 51, 3271, 22, 567, 1828, 7224, 404, 432, 572, 105, 1183, 526, 0, 154, 761, 2282, 66, 13, 1395, 0, 1174, 1561, 25, 0, 0, 653, 96, 380, 2, 90, 109, 0, 20, 13, 3069, 6, 162, 257, 1451, 405, 27, 243, 4086, 5, 24, 3069, 0, 1601, 0, 1456, 42, 4189, 0] payload_rows_per_dst=[11280, 10296, 5788, 9794, 5060, 15294, 11162, 10952] dup_rows_per_dst=[5196, 1246, 656, 1896, 374, 6092, 1834, 1384] combine_rows_per_src=[11280, 10296, 5788, 9794, 5060, 15294, 11162, 10952] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76586 payload_rows_remote=69018 dup_rows=21718 dup_rows_remote=20298 dedup_ratio=0.220927 router_tokens_per_expert=[43, 2199, 1, 258, 49, 879, 1456, 0, 4378, 371, 228, 678, 0, 67, 645, 341, 0, 0, 134, 99, 0, 3713, 962, 2, 0, 66, 2249, 793, 4, 153, 0, 0, 128, 592, 101, 333, 316, 3900, 125, 894, 652, 482, 3956, 337, 68, 54, 0, 2901, 66, 133, 0, 962, 3, 1917, 0, 1815, 0, 1462, 1885, 2678, 1, 3623, 0, 0] payload_rows_per_dst=[7116, 4482, 10342, 6948, 15516, 7568, 13412, 11202] dup_rows_per_dst=[896, 178, 662, 1340, 4834, 1420, 4292, 8096] combine_rows_per_src=[7116, 4482, 10342, 6948, 15516, 7568, 13412, 11202] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=269590528 dispatch_meta_bytes=350072 combine_payload_bytes=269590528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269940600 ingress_bytes=304193536 link_bytes=304193536 token_bytes=539531128 weight_movement_bytes=34603008 total_nvlink_bytes=574134136 weight_to_token_ratio=0.064135 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.144655786e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79106 payload_rows_remote=72500 dup_rows=19198 dup_rows_remote=18900 dedup_ratio=0.195292 router_tokens_per_expert=[0, 0, 3, 412, 192, 3203, 18, 119, 5, 4, 136, 100, 2367, 150, 728, 1535, 6402, 163, 319, 888, 357, 1770, 185, 2, 233, 1192, 4251, 512, 66, 795, 0, 530, 1166, 125, 0, 2, 1746, 74, 339, 0, 219, 144, 0, 92, 81, 2108, 4, 210, 1923, 1300, 487, 184, 253, 2571, 6, 93, 3040, 3, 1546, 0, 1000, 178, 3620, 1] payload_rows_per_dst=[11068, 8546, 6702, 11622, 6606, 13734, 10700, 10128] dup_rows_per_dst=[4066, 1504, 666, 3536, 298, 4786, 2934, 1408] combine_rows_per_src=[11068, 8546, 6702, 11622, 6606, 13734, 10700, 10128] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12870514128 prefetch_bytes=761266176 reduce_bytes=657457152 weight_movement_bytes=1418723328 egress_bytes=6439171536 ingress_bytes=7850065920 link_bytes=7850065920 total_nvlink_bytes=14289237456 weight_to_token_ratio=0.110231 prefetch_share_of_ingress=0.096976 dedup_ratio=0.199987 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=19 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027826 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.284311568e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=274554880 dispatch_meta_bytes=343632 combine_payload_bytes=274554880 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=274898512 ingress_bytes=343760896 link_bytes=343760896 token_bytes=549453392 weight_movement_bytes=69206016 total_nvlink_bytes=618659408 weight_to_token_ratio=0.125954 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=24064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.293544577e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78180 payload_rows_remote=67750 dup_rows=20124 dup_rows_remote=17310 dedup_ratio=0.204712 router_tokens_per_expert=[24, 2264, 0, 1668, 998, 2199, 884, 0, 4718, 37, 320, 416, 0, 27, 239, 1770, 0, 26, 57, 20, 0, 1800, 2067, 0, 0, 66, 2814, 1381, 8, 154, 0, 0, 882, 3025, 352, 104, 142, 1880, 63, 350, 237, 524, 5347, 344, 82, 80, 0, 1162, 215, 652, 0, 2082, 5, 1348, 0, 796, 1, 1330, 959, 1215, 2, 2016, 0, 0] payload_rows_per_dst=[9680, 12048, 11616, 10430, 7160, 8754, 11338, 7154] dup_rows_per_dst=[1996, 3006, 2374, 2814, 386, 2990, 2666, 3892] combine_rows_per_src=[9680, 12048, 11616, 10430, 7160, 8754, 11338, 7154] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=280428544 dispatch_meta_bytes=341232 combine_payload_bytes=280428544 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=280769776 ingress_bytes=384237568 link_bytes=384237568 token_bytes=561198320 weight_movement_bytes=103809024 total_nvlink_bytes=665007344 weight_to_token_ratio=0.184977 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.445855035e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=268582912 dispatch_meta_bytes=315616 combine_payload_bytes=268582912 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=268898528 ingress_bytes=406994944 link_bytes=406994944 token_bytes=537481440 weight_movement_bytes=138412032 total_nvlink_bytes=675893472 weight_to_token_ratio=0.257520 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=16000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.531489208e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77838 payload_rows_remote=61756 dup_rows=20466 dup_rows_remote=13174 dedup_ratio=0.208191 router_tokens_per_expert=[0, 9, 1, 786, 316, 2386, 88, 141, 17, 0, 135, 50, 1272, 47, 290, 1738, 7318, 270, 913, 1025, 197, 1722, 242, 0, 127, 1087, 4028, 311, 119, 724, 0, 521, 506, 309, 0, 3, 1946, 358, 61, 1, 9, 394, 1, 206, 89, 1067, 0, 193, 2255, 2711, 415, 77, 248, 2750, 11, 56, 2891, 2, 1437, 0, 630, 49, 4594, 3] payload_rows_per_dst=[12296, 6698, 16082, 11450, 6052, 3660, 12682, 8918] dup_rows_per_dst=[4482, 400, 7292, 2384, 316, 258, 4364, 970] combine_rows_per_src=[12296, 6698, 16082, 11450, 6052, 3660, 12682, 8918] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77432 payload_rows_remote=75362 dup_rows=20872 dup_rows_remote=20534 dedup_ratio=0.212321 router_tokens_per_expert=[30, 346, 0, 7366, 7543, 6427, 12, 0, 935, 0, 0, 19, 0, 18, 11, 4557, 0, 21, 41, 1, 13, 18, 2661, 0, 3, 81, 125, 329, 0, 0, 19, 0, 2853, 7832, 185, 4, 3, 41, 50, 17, 24, 147, 5261, 417, 28, 11, 0, 225, 13, 419, 0, 660, 0, 20, 4, 93, 60, 88, 94, 5, 1, 20, 1, 0] payload_rows_per_dst=[3272, 10418, 6062, 14860, 15148, 13896, 11706, 2070] dup_rows_per_dst=[1586, 1694, 380, 8992, 5890, 758, 1234, 338] combine_rows_per_src=[3272, 10418, 6062, 14860, 15148, 13896, 11706, 2070] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=282697728 dispatch_meta_bytes=357264 combine_payload_bytes=282697728 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=283054992 ingress_bytes=351903744 link_bytes=351903744 token_bytes=565752720 weight_movement_bytes=69206016 total_nvlink_bytes=634958736 weight_to_token_ratio=0.122326 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=18816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.324185458e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12583178440 prefetch_bytes=761266176 reduce_bytes=795869184 weight_movement_bytes=1557135360 egress_bytes=6295433416 ingress_bytes=7844880384 link_bytes=7844880384 total_nvlink_bytes=14140313800 weight_to_token_ratio=0.123747 prefetch_share_of_ingress=0.097040 dedup_ratio=0.207877 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=23 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=28194 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.283463188e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=296960000 dispatch_meta_bytes=365600 combine_payload_bytes=296960000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=297325600 ingress_bytes=331563008 link_bytes=331563008 token_bytes=594285600 weight_movement_bytes=34603008 total_nvlink_bytes=628888608 weight_to_token_ratio=0.058226 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=32128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.247644906e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78180 payload_rows_remote=67750 dup_rows=20124 dup_rows_remote=17310 dedup_ratio=0.204712 router_tokens_per_expert=[24, 2264, 0, 1668, 998, 2199, 884, 0, 4718, 37, 320, 416, 0, 27, 239, 1770, 0, 26, 57, 20, 0, 1800, 2067, 0, 0, 66, 2814, 1381, 8, 154, 0, 0, 882, 3025, 352, 104, 142, 1880, 63, 350, 237, 524, 5347, 344, 82, 80, 0, 1162, 215, 652, 0, 2082, 5, 1348, 0, 796, 1, 1330, 959, 1215, 2, 2016, 0, 0] payload_rows_per_dst=[9680, 12048, 11616, 10430, 7160, 8754, 11338, 7154] dup_rows_per_dst=[1996, 3006, 2374, 2814, 386, 2990, 2666, 3892] combine_rows_per_src=[9680, 12048, 11616, 10430, 7160, 8754, 11338, 7154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12870514128 prefetch_bytes=761266176 reduce_bytes=657457152 weight_movement_bytes=1418723328 egress_bytes=6439171536 ingress_bytes=7850065920 link_bytes=7850065920 total_nvlink_bytes=14289237456 weight_to_token_ratio=0.110231 prefetch_share_of_ingress=0.096976 dedup_ratio=0.199987 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=19 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027826 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.284311568e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=11953454480 prefetch_bytes=692060160 reduce_bytes=795869184 weight_movement_bytes=1487929344 egress_bytes=5980446096 ingress_bytes=7460937728 link_bytes=7460937728 total_nvlink_bytes=13441383824 weight_to_token_ratio=0.124477 prefetch_share_of_ingress=0.092758 dedup_ratio=0.229324 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=23 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028152 gemm_inner_pad_upper_bound=27940 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.220648175e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=277504000 dispatch_meta_bytes=340240 combine_payload_bytes=277504000 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=277844240 ingress_bytes=346710016 link_bytes=346710016 token_bytes=555348240 weight_movement_bytes=69206016 total_nvlink_bytes=624554256 weight_to_token_ratio=0.124617 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.304641878e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76586 payload_rows_remote=69018 dup_rows=21718 dup_rows_remote=20298 dedup_ratio=0.220927 router_tokens_per_expert=[43, 2199, 1, 258, 49, 879, 1456, 0, 4378, 371, 228, 678, 0, 67, 645, 341, 0, 0, 134, 99, 0, 3713, 962, 2, 0, 66, 2249, 793, 4, 153, 0, 0, 128, 592, 101, 333, 316, 3900, 125, 894, 652, 482, 3956, 337, 68, 54, 0, 2901, 66, 133, 0, 962, 3, 1917, 0, 1815, 0, 1462, 1885, 2678, 1, 3623, 0, 0] payload_rows_per_dst=[7116, 4482, 10342, 6948, 15516, 7568, 13412, 11202] dup_rows_per_dst=[896, 178, 662, 1340, 4834, 1420, 4292, 8096] combine_rows_per_src=[7116, 4482, 10342, 6948, 15516, 7568, 13412, 11202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=252952576 dispatch_meta_bytes=299720 combine_payload_bytes=252952576 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=253252296 ingress_bytes=322158592 link_bytes=322158592 token_bytes=506204872 weight_movement_bytes=69206016 total_nvlink_bytes=575410888 weight_to_token_ratio=0.136715 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.212256846e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=308682752 dispatch_meta_bytes=383584 combine_payload_bytes=308682752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=309066336 ingress_bytes=343285760 link_bytes=343285760 token_bytes=617749088 weight_movement_bytes=34603008 total_nvlink_bytes=652352096 weight_to_token_ratio=0.056015 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=16256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.291756678e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12727685112 prefetch_bytes=726663168 reduce_bytes=657457152 weight_movement_bytes=1384120320 egress_bytes=6367743992 ingress_bytes=7744061440 link_bytes=7744061440 total_nvlink_bytes=14111805432 weight_to_token_ratio=0.108749 prefetch_share_of_ingress=0.093835 dedup_ratio=0.205659 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=19 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028370 gemm_inner_pad_upper_bound=28321 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.266968685e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12853885432 prefetch_bytes=692060160 reduce_bytes=899678208 weight_movement_bytes=1591738368 egress_bytes=6430841336 ingress_bytes=8014782464 link_bytes=8014782464 total_nvlink_bytes=14445623800 weight_to_token_ratio=0.123833 prefetch_share_of_ingress=0.086348 dedup_ratio=0.195642 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=26 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029022 gemm_inner_pad_upper_bound=27559 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.311260051e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78724 payload_rows_remote=65572 dup_rows=19580 dup_rows_remote=13332 dedup_ratio=0.199178 router_tokens_per_expert=[8, 1075, 0, 4188, 4144, 3754, 285, 0, 5817, 20, 102, 67, 0, 73, 111, 3299, 0, 72, 2, 0, 0, 659, 1328, 1, 0, 82, 779, 1389, 6, 128, 24, 0, 1042, 5843, 705, 79, 61, 802, 84, 118, 92, 130, 5856, 723, 4, 44, 0, 1047, 12, 406, 0, 1623, 0, 314, 4, 426, 0, 684, 700, 249, 1, 690, 0, 0] payload_rows_per_dst=[13152, 14052, 13512, 10528, 5502, 13436, 5350, 3192] dup_rows_per_dst=[6248, 4926, 2298, 1796, 280, 2356, 220, 1456] combine_rows_per_src=[13152, 14052, 13512, 10528, 5502, 13436, 5350, 3192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76808 payload_rows_remote=63098 dup_rows=21496 dup_rows_remote=17342 dedup_ratio=0.218669 router_tokens_per_expert=[70, 1795, 0, 83, 9, 392, 1714, 0, 3554, 251, 550, 899, 0, 32, 769, 410, 0, 0, 36, 42, 0, 4218, 498, 0, 0, 49, 2539, 554, 21, 380, 0, 0, 158, 213, 117, 207, 90, 5198, 69, 758, 669, 506, 3116, 298, 60, 75, 0, 2373, 440, 312, 0, 790, 3, 3106, 1, 1164, 0, 1304, 2064, 2934, 0, 4262, 0, 0] payload_rows_per_dst=[6522, 5404, 9734, 6862, 12062, 6606, 13710, 15908] dup_rows_per_dst=[820, 418, 280, 1008, 1160, 1356, 4154, 12300] combine_rows_per_src=[6522, 5404, 9734, 6862, 12062, 6606, 13710, 15908] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=277504000 dispatch_meta_bytes=340240 combine_payload_bytes=277504000 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=277844240 ingress_bytes=346710016 link_bytes=346710016 token_bytes=555348240 weight_movement_bytes=69206016 total_nvlink_bytes=624554256 weight_to_token_ratio=0.124617 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.304641878e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=282697728 dispatch_meta_bytes=357264 combine_payload_bytes=282697728 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=283054992 ingress_bytes=351903744 link_bytes=351903744 token_bytes=565752720 weight_movement_bytes=69206016 total_nvlink_bytes=634958736 weight_to_token_ratio=0.122326 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=18816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.324185458e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12057787496 prefetch_bytes=657457152 reduce_bytes=865075200 weight_movement_bytes=1522532352 egress_bytes=6032571496 ingress_bytes=7547748352 link_bytes=7547748352 total_nvlink_bytes=13580319848 weight_to_token_ratio=0.126270 prefetch_share_of_ingress=0.087106 dedup_ratio=0.224767 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=25 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029239 gemm_inner_pad_upper_bound=27940 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.234850844e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 2, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77762 payload_rows_remote=69146 dup_rows=20542 dup_rows_remote=19678 dedup_ratio=0.208964 router_tokens_per_expert=[30, 2351, 1, 418, 301, 1154, 1143, 1, 5084, 237, 676, 682, 0, 53, 449, 1147, 0, 2, 35, 29, 0, 2729, 1081, 5, 0, 59, 2686, 1124, 9, 136, 0, 0, 277, 1412, 235, 152, 366, 2740, 142, 713, 289, 595, 4225, 250, 17, 123, 0, 2058, 200, 588, 0, 1596, 3, 1893, 11, 1132, 0, 1685, 1638, 1814, 0, 3376, 0, 0] payload_rows_per_dst=[7542, 13200, 9760, 8546, 8616, 5672, 14182, 10244] dup_rows_per_dst=[948, 3226, 826, 1790, 864, 992, 5114, 6782] combine_rows_per_src=[7542, 13200, 9760, 8546, 8616, 5672, 14182, 10244] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=268582912 dispatch_meta_bytes=315616 combine_payload_bytes=268582912 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=268898528 ingress_bytes=406994944 link_bytes=406994944 token_bytes=537481440 weight_movement_bytes=138412032 total_nvlink_bytes=675893472 weight_to_token_ratio=0.257520 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=16000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.531489208e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=258449408 dispatch_meta_bytes=321760 combine_payload_bytes=258449408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258771168 ingress_bytes=293052416 link_bytes=293052416 token_bytes=517220576 weight_movement_bytes=34603008 total_nvlink_bytes=551823584 weight_to_token_ratio=0.066902 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.102732649e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=283222016 dispatch_meta_bytes=355296 combine_payload_bytes=283222016 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=283577312 ingress_bytes=352428032 link_bytes=352428032 token_bytes=566799328 weight_movement_bytes=69206016 total_nvlink_bytes=636005344 weight_to_token_ratio=0.122100 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.326158312e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=11953454480 prefetch_bytes=692060160 reduce_bytes=795869184 weight_movement_bytes=1487929344 egress_bytes=5980446096 ingress_bytes=7460937728 link_bytes=7460937728 total_nvlink_bytes=13441383824 weight_to_token_ratio=0.124477 prefetch_share_of_ingress=0.092758 dedup_ratio=0.229324 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=23 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028152 gemm_inner_pad_upper_bound=27940 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.220648175e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12492842888 prefetch_bytes=726663168 reduce_bytes=622854144 weight_movement_bytes=1349517312 egress_bytes=6250284936 ingress_bytes=7592075264 link_bytes=7592075264 total_nvlink_bytes=13842360200 weight_to_token_ratio=0.108023 prefetch_share_of_ingress=0.095713 dedup_ratio=0.215947 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=18 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027826 gemm_inner_pad_upper_bound=28321 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.242102957e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12878820176 prefetch_bytes=865075200 reduce_bytes=588251136 weight_movement_bytes=1453326336 egress_bytes=6443340624 ingress_bytes=7888805888 link_bytes=7888805888 total_nvlink_bytes=14332146512 weight_to_token_ratio=0.112846 prefetch_share_of_ingress=0.109659 dedup_ratio=0.201599 prefetch_slots=50 unused_prefetch_slots=318 slot_utilization=0.135870 reduce_slots=17 remote_demand=25 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028152 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.290649629e-04 prefetch_bytes_per_gemm_flop=1.415307971e-05 anomalies=none - [2026-08-26 15:09:10] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 799.6 | throughput per GPU (TFLOP/s/GPU): 159.6 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.091896E+01 | loss scale: 1.0 | grad norm: 452.732 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12853885432 prefetch_bytes=692060160 reduce_bytes=899678208 weight_movement_bytes=1591738368 egress_bytes=6430841336 ingress_bytes=8014782464 link_bytes=8014782464 total_nvlink_bytes=14445623800 weight_to_token_ratio=0.123833 prefetch_share_of_ingress=0.086348 dedup_ratio=0.195642 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=26 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029022 gemm_inner_pad_upper_bound=27559 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.311260051e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12727685112 prefetch_bytes=726663168 reduce_bytes=657457152 weight_movement_bytes=1384120320 egress_bytes=6367743992 ingress_bytes=7744061440 link_bytes=7744061440 total_nvlink_bytes=14111805432 weight_to_token_ratio=0.108749 prefetch_share_of_ingress=0.093835 dedup_ratio=0.205659 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=19 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028370 gemm_inner_pad_upper_bound=28321 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.266968685e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75420 payload_rows_remote=65818 dup_rows=22884 dup_rows_remote=21700 dedup_ratio=0.232788 router_tokens_per_expert=[6, 2004, 0, 539, 283, 987, 422, 0, 6267, 44, 37, 319, 0, 107, 555, 1525, 0, 18, 54, 98, 0, 2205, 1605, 2, 0, 83, 1616, 513, 81, 335, 85, 0, 169, 1411, 600, 173, 392, 2975, 409, 853, 699, 1230, 4613, 509, 30, 174, 0, 3894, 8, 339, 0, 1369, 2, 1297, 52, 1391, 0, 1674, 1688, 1206, 5, 2200, 0, 0] payload_rows_per_dst=[5908, 13898, 9602, 6604, 9462, 12780, 8180, 8986] dup_rows_per_dst=[600, 3810, 1184, 796, 1680, 9518, 736, 4560] combine_rows_per_src=[5908, 13898, 9602, 6604, 9462, 12780, 8180, 8986] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=269590528 dispatch_meta_bytes=350072 combine_payload_bytes=269590528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269940600 ingress_bytes=304193536 link_bytes=304193536 token_bytes=539531128 weight_movement_bytes=34603008 total_nvlink_bytes=574134136 weight_to_token_ratio=0.064135 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.144655786e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12583178440 prefetch_bytes=761266176 reduce_bytes=795869184 weight_movement_bytes=1557135360 egress_bytes=6295433416 ingress_bytes=7844880384 link_bytes=7844880384 total_nvlink_bytes=14140313800 weight_to_token_ratio=0.123747 prefetch_share_of_ingress=0.097040 dedup_ratio=0.207877 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=23 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=28194 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.283463188e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none + [2026-08-26 15:56:58] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 792.9 | throughput per GPU (TFLOP/s/GPU): 160.9 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.091896E+01 | loss scale: 1.0 | grad norm: 452.732 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73948 payload_rows_remote=63418 dup_rows=24356 dup_rows_remote=21060 dedup_ratio=0.247762 router_tokens_per_expert=[1827, 620, 368, 1343, 305, 1340, 778, 93, 3164, 781, 1325, 299, 12, 2462, 607, 620, 639, 839, 685, 708, 177, 208, 269, 918, 695, 812, 486, 549, 981, 253, 742, 837, 1089, 523, 1665, 1079, 615, 523, 912, 543, 126, 362, 370, 140, 719, 1404, 607, 114, 166, 396, 533, 836, 832, 2148, 273, 283, 1036, 607, 1550, 625, 212, 694, 1635, 793] payload_rows_per_dst=[10530, 12686, 7332, 10332, 8182, 6704, 9028, 9154] dup_rows_per_dst=[3296, 9184, 1554, 3648, 1908, 980, 1906, 1880] combine_rows_per_src=[10530, 12686, 7332, 10332, 8182, 6704, 9028, 9154] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75618 payload_rows_remote=65350 dup_rows=22686 dup_rows_remote=19240 dedup_ratio=0.230774 router_tokens_per_expert=[1906, 699, 343, 1391, 284, 1171, 767, 112, 3340, 729, 1207, 378, 2, 2309, 584, 734, 600, 874, 706, 685, 148, 220, 266, 903, 549, 745, 423, 429, 989, 260, 846, 936, 913, 542, 1748, 1122, 659, 441, 917, 595, 169, 423, 396, 156, 763, 1340, 528, 81, 201, 418, 489, 1039, 912, 2078, 267, 297, 995, 582, 1595, 538, 210, 695, 1680, 828] payload_rows_per_dst=[10138, 6508, 9748, 10268, 10126, 10430, 9286, 9114] dup_rows_per_dst=[3208, 1130, 3674, 3446, 3378, 3962, 2116, 1772] combine_rows_per_src=[10138, 6508, 9748, 10268, 10126, 10430, 9286, 9114] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74532 payload_rows_remote=63244 dup_rows=23772 dup_rows_remote=20134 dedup_ratio=0.241821 router_tokens_per_expert=[1894, 649, 380, 1353, 214, 1336, 775, 112, 3344, 883, 1175, 306, 5, 2528, 512, 735, 663, 848, 654, 599, 160, 200, 246, 993, 487, 827, 528, 573, 959, 269, 731, 680, 960, 486, 1538, 1150, 680, 497, 918, 574, 170, 427, 431, 143, 787, 1415, 548, 85, 178, 389, 521, 901, 810, 2376, 288, 269, 1127, 599, 1640, 541, 177, 581, 1611, 717] payload_rows_per_dst=[7922, 6124, 10030, 8446, 9640, 10536, 10546, 11288] dup_rows_per_dst=[1716, 1108, 3752, 1662, 3026, 4164, 4706, 3638] combine_rows_per_src=[7922, 6124, 10030, 8446, 9640, 10536, 10546, 11288] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74902 payload_rows_remote=65888 dup_rows=23402 dup_rows_remote=20632 dedup_ratio=0.238057 router_tokens_per_expert=[1808, 614, 338, 1512, 199, 1180, 714, 89, 3208, 792, 1214, 394, 2, 2439, 575, 724, 670, 969, 666, 697, 136, 163, 265, 992, 618, 644, 491, 447, 965, 293, 826, 932, 1035, 568, 1711, 1129, 613, 525, 1058, 494, 198, 406, 405, 166, 774, 1429, 577, 107, 142, 382, 496, 933, 804, 2089, 251, 331, 1004, 622, 1572, 597, 171, 610, 1567, 810] payload_rows_per_dst=[10090, 11378, 9014, 10082, 8662, 7650, 9060, 8966] dup_rows_per_dst=[2818, 6668, 2770, 3484, 2182, 1878, 1796, 1806] combine_rows_per_src=[10090, 11378, 9014, 10082, 8662, 7650, 9060, 8966] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73908 payload_rows_remote=61032 dup_rows=24396 dup_rows_remote=15132 dedup_ratio=0.248169 router_tokens_per_expert=[1884, 602, 375, 1549, 254, 1165, 790, 79, 3481, 693, 1212, 357, 3, 2505, 553, 665, 604, 894, 636, 707, 123, 230, 231, 898, 529, 643, 430, 577, 946, 231, 689, 700, 957, 519, 1601, 1138, 700, 408, 1013, 537, 150, 384, 416, 124, 807, 1455, 553, 101, 161, 343, 558, 951, 842, 2354, 262, 223, 1046, 605, 1558, 640, 206, 746, 1712, 847] payload_rows_per_dst=[10332, 12876, 7064, 9682, 8332, 6974, 9320, 9328] dup_rows_per_dst=[3064, 9264, 1582, 3232, 2212, 1006, 2068, 1968] combine_rows_per_src=[10332, 12876, 7064, 9682, 8332, 6974, 9320, 9328] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75270 payload_rows_remote=65340 dup_rows=23034 dup_rows_remote=19742 dedup_ratio=0.234314 router_tokens_per_expert=[1805, 666, 398, 1528, 247, 1307, 744, 101, 3236, 748, 1222, 353, 4, 2458, 685, 573, 679, 869, 707, 698, 137, 199, 249, 868, 578, 756, 504, 520, 1021, 258, 749, 754, 929, 483, 1574, 1172, 614, 437, 889, 513, 170, 448, 501, 161, 802, 1389, 554, 69, 175, 408, 493, 998, 892, 2139, 245, 288, 1018, 521, 1706, 639, 176, 702, 1675, 751] payload_rows_per_dst=[10380, 6088, 9972, 10002, 9930, 10490, 9294, 9114] dup_rows_per_dst=[3212, 1082, 3756, 3376, 3292, 4170, 1982, 2164] combine_rows_per_src=[10380, 6088, 9972, 10002, 9930, 10490, 9294, 9114] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73908 payload_rows_remote=61032 dup_rows=24396 dup_rows_remote=15132 dedup_ratio=0.248169 router_tokens_per_expert=[1884, 602, 375, 1549, 254, 1165, 790, 79, 3481, 693, 1212, 357, 3, 2505, 553, 665, 604, 894, 636, 707, 123, 230, 231, 898, 529, 643, 430, 577, 946, 231, 689, 700, 957, 519, 1601, 1138, 700, 408, 1013, 537, 150, 384, 416, 124, 807, 1455, 553, 101, 161, 343, 558, 951, 842, 2354, 262, 223, 1046, 605, 1558, 640, 206, 746, 1712, 847] payload_rows_per_dst=[10332, 12876, 7064, 9682, 8332, 6974, 9320, 9328] dup_rows_per_dst=[3064, 9264, 1582, 3232, 2212, 1006, 2068, 1968] combine_rows_per_src=[10332, 12876, 7064, 9682, 8332, 6974, 9320, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=64050 dup_rows=24008 dup_rows_remote=19378 dedup_ratio=0.244222 router_tokens_per_expert=[1933, 642, 343, 1449, 222, 1262, 676, 116, 3383, 660, 1144, 385, 8, 2566, 564, 629, 663, 933, 629, 628, 157, 205, 252, 906, 574, 729, 505, 509, 1015, 308, 798, 781, 977, 515, 1668, 1161, 718, 495, 887, 581, 160, 422, 435, 140, 760, 1458, 589, 91, 184, 402, 555, 938, 879, 1988, 269, 290, 942, 525, 1597, 582, 187, 697, 1630, 856] payload_rows_per_dst=[7686, 5848, 10030, 8658, 10316, 10550, 10246, 10962] dup_rows_per_dst=[1734, 932, 3848, 1780, 3688, 4326, 4630, 3070] combine_rows_per_src=[7686, 5848, 10030, 8658, 10316, 10550, 10246, 10962] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74532 payload_rows_remote=63244 dup_rows=23772 dup_rows_remote=20134 dedup_ratio=0.241821 router_tokens_per_expert=[1894, 649, 380, 1353, 214, 1336, 775, 112, 3344, 883, 1175, 306, 5, 2528, 512, 735, 663, 848, 654, 599, 160, 200, 246, 993, 487, 827, 528, 573, 959, 269, 731, 680, 960, 486, 1538, 1150, 680, 497, 918, 574, 170, 427, 431, 143, 787, 1415, 548, 85, 178, 389, 521, 901, 810, 2376, 288, 269, 1127, 599, 1640, 541, 177, 581, 1611, 717] payload_rows_per_dst=[7922, 6124, 10030, 8446, 9640, 10536, 10546, 11288] dup_rows_per_dst=[1716, 1108, 3752, 1662, 3026, 4164, 4706, 3638] combine_rows_per_src=[7922, 6124, 10030, 8446, 9640, 10536, 10546, 11288] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75604 payload_rows_remote=65262 dup_rows=22700 dup_rows_remote=18558 dedup_ratio=0.230916 router_tokens_per_expert=[1865, 601, 334, 1571, 260, 1116, 742, 91, 3355, 820, 1384, 319, 0, 2529, 587, 604, 652, 833, 714, 766, 139, 185, 252, 1019, 549, 670, 505, 598, 946, 249, 862, 724, 912, 562, 1681, 1058, 689, 429, 842, 542, 126, 363, 409, 147, 755, 1406, 608, 73, 138, 341, 550, 912, 859, 2308, 281, 280, 963, 526, 1641, 550, 213, 683, 1669, 795] payload_rows_per_dst=[9570, 6266, 10394, 8504, 10106, 10342, 9518, 10904] dup_rows_per_dst=[2608, 1162, 3784, 1702, 3324, 4142, 2802, 3176] combine_rows_per_src=[9570, 6266, 10394, 8504, 10106, 10342, 9518, 10904] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=64050 dup_rows=24008 dup_rows_remote=19378 dedup_ratio=0.244222 router_tokens_per_expert=[1933, 642, 343, 1449, 222, 1262, 676, 116, 3383, 660, 1144, 385, 8, 2566, 564, 629, 663, 933, 629, 628, 157, 205, 252, 906, 574, 729, 505, 509, 1015, 308, 798, 781, 977, 515, 1668, 1161, 718, 495, 887, 581, 160, 422, 435, 140, 760, 1458, 589, 91, 184, 402, 555, 938, 879, 1988, 269, 290, 942, 525, 1597, 582, 187, 697, 1630, 856] payload_rows_per_dst=[7686, 5848, 10030, 8658, 10316, 10550, 10246, 10962] dup_rows_per_dst=[1734, 932, 3848, 1780, 3688, 4326, 4630, 3070] combine_rows_per_src=[7686, 5848, 10030, 8658, 10316, 10550, 10246, 10962] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267673600 dispatch_meta_bytes=338360 combine_payload_bytes=267673600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268011960 ingress_bytes=302276608 link_bytes=302276608 token_bytes=535685560 weight_movement_bytes=34603008 total_nvlink_bytes=570288568 weight_to_token_ratio=0.064596 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=8192 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137442541e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=259047424 dispatch_meta_bytes=333512 combine_payload_bytes=259047424 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=259380936 ingress_bytes=328253440 link_bytes=328253440 token_bytes=518428360 weight_movement_bytes=69206016 total_nvlink_bytes=587634376 weight_to_token_ratio=0.133492 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.235191268e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=269877248 dispatch_meta_bytes=346080 combine_payload_bytes=269877248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270223328 ingress_bytes=304480256 link_bytes=304480256 token_bytes=540100576 weight_movement_bytes=34603008 total_nvlink_bytes=574703584 weight_to_token_ratio=0.064068 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.145734691e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=249987072 dispatch_meta_bytes=304656 combine_payload_bytes=249987072 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=250291728 ingress_bytes=353796096 link_bytes=353796096 token_bytes=500278800 weight_movement_bytes=103809024 total_nvlink_bytes=604087824 weight_to_token_ratio=0.207502 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.331306226e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=259760128 dispatch_meta_bytes=337912 combine_payload_bytes=259760128 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=260098040 ingress_bytes=328966144 link_bytes=328966144 token_bytes=519858168 weight_movement_bytes=69206016 total_nvlink_bytes=589064184 weight_to_token_ratio=0.133125 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.237873116e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267673600 dispatch_meta_bytes=338360 combine_payload_bytes=267673600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268011960 ingress_bytes=302276608 link_bytes=302276608 token_bytes=535685560 weight_movement_bytes=34603008 total_nvlink_bytes=570288568 weight_to_token_ratio=0.064596 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=8192 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137442541e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267632640 dispatch_meta_bytes=340328 combine_payload_bytes=267632640 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267972968 ingress_bytes=371441664 link_bytes=371441664 token_bytes=535605608 weight_movement_bytes=103809024 total_nvlink_bytes=639414632 weight_to_token_ratio=0.193816 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=8320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397705078e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73948 payload_rows_remote=63418 dup_rows=24356 dup_rows_remote=21060 dedup_ratio=0.247762 router_tokens_per_expert=[1827, 620, 368, 1343, 305, 1340, 778, 93, 3164, 781, 1325, 299, 12, 2462, 607, 620, 639, 839, 685, 708, 177, 208, 269, 918, 695, 812, 486, 549, 981, 253, 742, 837, 1089, 523, 1665, 1079, 615, 523, 912, 543, 126, 362, 370, 140, 719, 1404, 607, 114, 166, 396, 533, 836, 832, 2148, 273, 283, 1036, 607, 1550, 625, 212, 694, 1635, 793] payload_rows_per_dst=[10530, 12686, 7332, 10332, 8182, 6704, 9028, 9154] dup_rows_per_dst=[3296, 9184, 1554, 3648, 1908, 980, 1906, 1880] combine_rows_per_src=[10530, 12686, 7332, 10332, 8182, 6704, 9028, 9154] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267313152 dispatch_meta_bytes=335280 combine_payload_bytes=267313152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267648432 ingress_bytes=301916160 link_bytes=301916160 token_bytes=534961584 weight_movement_bytes=34603008 total_nvlink_bytes=569564592 weight_to_token_ratio=0.064683 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136086204e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=249987072 dispatch_meta_bytes=304656 combine_payload_bytes=249987072 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=250291728 ingress_bytes=353796096 link_bytes=353796096 token_bytes=500278800 weight_movement_bytes=103809024 total_nvlink_bytes=604087824 weight_to_token_ratio=0.207502 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.331306226e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=262348800 dispatch_meta_bytes=333712 combine_payload_bytes=262348800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262682512 ingress_bytes=296951808 link_bytes=296951808 token_bytes=525031312 weight_movement_bytes=34603008 total_nvlink_bytes=559634320 weight_to_token_ratio=0.065907 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.117405747e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=259760128 dispatch_meta_bytes=337912 combine_payload_bytes=259760128 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=260098040 ingress_bytes=328966144 link_bytes=328966144 token_bytes=519858168 weight_movement_bytes=69206016 total_nvlink_bytes=589064184 weight_to_token_ratio=0.133125 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.237873116e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80272 payload_rows_remote=69946 dup_rows=18032 dup_rows_remote=15650 dedup_ratio=0.183431 router_tokens_per_expert=[182, 424, 54, 163, 4885, 59, 1623, 422, 39, 707, 1438, 424, 503, 788, 27, 3446, 422, 973, 546, 663, 492, 912, 425, 389, 462, 365, 810, 3901, 0, 59, 94, 663, 7, 0, 2, 16, 335, 4, 208, 509, 789, 445, 85, 671, 19, 897, 0, 223, 6763, 228, 3052, 0, 29, 226, 412, 1386, 2697, 411, 793, 298, 384, 152, 1477, 274] payload_rows_per_dst=[7932, 10900, 7894, 10326, 14080, 12906, 6004, 10230] dup_rows_per_dst=[2152, 3844, 1750, 2382, 1608, 3122, 432, 2742] combine_rows_per_src=[7932, 10900, 7894, 10326, 14080, 12906, 6004, 10230] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75622 payload_rows_remote=67768 dup_rows=22682 dup_rows_remote=21268 dedup_ratio=0.230733 router_tokens_per_expert=[142, 559, 20, 148, 4882, 53, 1702, 334, 45, 504, 1394, 456, 528, 817, 46, 3284, 321, 978, 551, 460, 521, 885, 454, 464, 422, 425, 832, 3763, 2, 72, 97, 733, 2, 0, 5, 15, 365, 4, 234, 496, 781, 496, 58, 689, 22, 938, 3, 205, 6905, 226, 3654, 0, 12, 203, 404, 1361, 2590, 302, 687, 452, 404, 207, 1290, 248] payload_rows_per_dst=[13322, 10860, 7854, 10288, 14300, 4994, 4308, 9696] dup_rows_per_dst=[9582, 3288, 1414, 2404, 1752, 1474, 104, 2664] combine_rows_per_src=[13322, 10860, 7854, 10288, 14300, 4994, 4308, 9696] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71654 payload_rows_remote=61418 dup_rows=26650 dup_rows_remote=24038 dedup_ratio=0.271098 router_tokens_per_expert=[163, 420, 36, 127, 4397, 82, 1508, 302, 32, 485, 1527, 401, 501, 665, 51, 3496, 303, 960, 507, 632, 465, 1139, 541, 482, 455, 444, 812, 4033, 3, 98, 102, 615, 4, 3, 1, 4, 318, 1, 275, 499, 801, 479, 81, 592, 12, 967, 0, 247, 6956, 251, 3364, 0, 26, 205, 456, 1402, 2755, 392, 764, 189, 356, 152, 1620, 196] payload_rows_per_dst=[4708, 6374, 12206, 7828, 2202, 12112, 15988, 10236] dup_rows_per_dst=[568, 950, 4844, 1686, 8, 3040, 12942, 2612] combine_rows_per_src=[4708, 6374, 12206, 7828, 2202, 12112, 15988, 10236] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76554 payload_rows_remote=65626 dup_rows=21750 dup_rows_remote=18708 dedup_ratio=0.221252 router_tokens_per_expert=[190, 452, 40, 138, 4732, 53, 1605, 374, 20, 480, 1493, 423, 502, 636, 31, 3400, 354, 1014, 488, 637, 483, 961, 470, 419, 399, 373, 725, 3861, 0, 84, 82, 661, 3, 3, 1, 4, 326, 5, 290, 542, 777, 503, 41, 674, 16, 855, 0, 212, 6927, 181, 3788, 0, 17, 205, 420, 1383, 2957, 307, 818, 178, 343, 133, 1415, 248] payload_rows_per_dst=[13548, 10928, 8100, 10276, 14400, 4738, 4318, 10246] dup_rows_per_dst=[9196, 3042, 1552, 2094, 1802, 1418, 94, 2552] combine_rows_per_src=[13548, 10928, 8100, 10276, 14400, 4738, 4318, 10246] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=259047424 dispatch_meta_bytes=333512 combine_payload_bytes=259047424 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=259380936 ingress_bytes=328253440 link_bytes=328253440 token_bytes=518428360 weight_movement_bytes=69206016 total_nvlink_bytes=587634376 weight_to_token_ratio=0.133492 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.235191268e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74902 payload_rows_remote=65888 dup_rows=23402 dup_rows_remote=20632 dedup_ratio=0.238057 router_tokens_per_expert=[1808, 614, 338, 1512, 199, 1180, 714, 89, 3208, 792, 1214, 394, 2, 2439, 575, 724, 670, 969, 666, 697, 136, 163, 265, 992, 618, 644, 491, 447, 965, 293, 826, 932, 1035, 568, 1711, 1129, 613, 525, 1058, 494, 198, 406, 405, 166, 774, 1429, 577, 107, 142, 382, 496, 933, 804, 2089, 251, 331, 1004, 622, 1572, 597, 171, 610, 1567, 810] payload_rows_per_dst=[10090, 11378, 9014, 10082, 8662, 7650, 9060, 8966] dup_rows_per_dst=[2818, 6668, 2770, 3484, 2182, 1878, 1796, 1806] combine_rows_per_src=[10090, 11378, 9014, 10082, 8662, 7650, 9060, 8966] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267313152 dispatch_meta_bytes=335280 combine_payload_bytes=267313152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267648432 ingress_bytes=301916160 link_bytes=301916160 token_bytes=534961584 weight_movement_bytes=34603008 total_nvlink_bytes=569564592 weight_to_token_ratio=0.064683 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136086204e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=269877248 dispatch_meta_bytes=346080 combine_payload_bytes=269877248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270223328 ingress_bytes=304480256 link_bytes=304480256 token_bytes=540100576 weight_movement_bytes=34603008 total_nvlink_bytes=574703584 weight_to_token_ratio=0.064068 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.145734691e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81390 payload_rows_remote=67430 dup_rows=16914 dup_rows_remote=15286 dedup_ratio=0.172058 router_tokens_per_expert=[217, 490, 33, 130, 4586, 91, 1559, 319, 40, 524, 1666, 493, 548, 595, 55, 3404, 328, 964, 571, 880, 581, 854, 522, 448, 406, 354, 715, 3770, 0, 100, 131, 703, 1, 2, 14, 14, 387, 0, 258, 463, 755, 428, 173, 479, 16, 996, 1, 214, 6655, 295, 3367, 1, 18, 221, 423, 1331, 2733, 350, 883, 214, 439, 193, 1526, 225] payload_rows_per_dst=[5000, 11120, 8480, 10152, 13960, 12296, 9958, 10424] dup_rows_per_dst=[678, 3530, 1816, 2206, 1628, 3000, 1354, 2702] combine_rows_per_src=[5000, 11120, 8480, 10152, 13960, 12296, 9958, 10424] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75366 payload_rows_remote=62750 dup_rows=22938 dup_rows_remote=15106 dedup_ratio=0.233337 router_tokens_per_expert=[216, 519, 32, 124, 4549, 63, 1359, 392, 58, 656, 1366, 373, 466, 683, 70, 3121, 406, 906, 512, 1044, 583, 1114, 468, 511, 455, 368, 743, 3841, 1, 83, 95, 590, 1, 0, 6, 5, 378, 0, 290, 500, 774, 461, 199, 503, 16, 1195, 0, 262, 6817, 251, 2970, 0, 25, 210, 424, 1287, 2931, 348, 778, 198, 383, 218, 1711, 244] payload_rows_per_dst=[12616, 12118, 8770, 10728, 14246, 5394, 4272, 7222] dup_rows_per_dst=[7832, 5708, 2318, 2898, 1748, 1426, 122, 886] combine_rows_per_src=[12616, 12118, 8770, 10728, 14246, 5394, 4272, 7222] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76554 payload_rows_remote=65626 dup_rows=21750 dup_rows_remote=18708 dedup_ratio=0.221252 router_tokens_per_expert=[190, 452, 40, 138, 4732, 53, 1605, 374, 20, 480, 1493, 423, 502, 636, 31, 3400, 354, 1014, 488, 637, 483, 961, 470, 419, 399, 373, 725, 3861, 0, 84, 82, 661, 3, 3, 1, 4, 326, 5, 290, 542, 777, 503, 41, 674, 16, 855, 0, 212, 6927, 181, 3788, 0, 17, 205, 420, 1383, 2957, 307, 818, 178, 343, 133, 1415, 248] payload_rows_per_dst=[13548, 10928, 8100, 10276, 14400, 4738, 4318, 10246] dup_rows_per_dst=[9196, 3042, 1552, 2094, 1802, 1418, 94, 2552] combine_rows_per_src=[13548, 10928, 8100, 10276, 14400, 4738, 4318, 10246] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80272 payload_rows_remote=69946 dup_rows=18032 dup_rows_remote=15650 dedup_ratio=0.183431 router_tokens_per_expert=[182, 424, 54, 163, 4885, 59, 1623, 422, 39, 707, 1438, 424, 503, 788, 27, 3446, 422, 973, 546, 663, 492, 912, 425, 389, 462, 365, 810, 3901, 0, 59, 94, 663, 7, 0, 2, 16, 335, 4, 208, 509, 789, 445, 85, 671, 19, 897, 0, 223, 6763, 228, 3052, 0, 29, 226, 412, 1386, 2697, 411, 793, 298, 384, 152, 1477, 274] payload_rows_per_dst=[7932, 10900, 7894, 10326, 14080, 12906, 6004, 10230] dup_rows_per_dst=[2152, 3844, 1750, 2382, 1608, 3122, 432, 2742] combine_rows_per_src=[7932, 10900, 7894, 10326, 14080, 12906, 6004, 10230] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74434 payload_rows_remote=58666 dup_rows=23870 dup_rows_remote=15010 dedup_ratio=0.242818 router_tokens_per_expert=[219, 463, 42, 153, 4754, 80, 1557, 341, 49, 523, 1352, 411, 502, 786, 74, 3210, 336, 914, 512, 880, 470, 927, 530, 474, 464, 351, 790, 3891, 0, 71, 106, 616, 1, 2, 4, 17, 319, 3, 265, 518, 818, 496, 167, 695, 17, 858, 1, 278, 6885, 220, 3091, 0, 21, 191, 504, 1402, 2625, 347, 802, 368, 406, 160, 1552, 271] payload_rows_per_dst=[4972, 6400, 11898, 10376, 2254, 12566, 15768, 10200] dup_rows_per_dst=[738, 994, 4608, 2202, 4, 3602, 8860, 2862] combine_rows_per_src=[4972, 6400, 11898, 10376, 2254, 12566, 15768, 10200] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79556 payload_rows_remote=66940 dup_rows=18748 dup_rows_remote=15148 dedup_ratio=0.190715 router_tokens_per_expert=[153, 497, 59, 140, 4661, 76, 1362, 443, 50, 510, 1207, 457, 436, 721, 71, 3314, 370, 922, 531, 530, 500, 1006, 420, 479, 396, 422, 799, 3897, 5, 82, 80, 548, 3, 0, 2, 8, 298, 0, 292, 539, 809, 505, 49, 712, 26, 1099, 2, 245, 6891, 250, 3796, 0, 11, 200, 437, 1319, 2813, 125, 709, 466, 356, 155, 1595, 296] payload_rows_per_dst=[4820, 6714, 11086, 10212, 12770, 12616, 11090, 10248] dup_rows_per_dst=[640, 1534, 3714, 2246, 1542, 3600, 2690, 2782] combine_rows_per_src=[4820, 6714, 11086, 10212, 12770, 12616, 11090, 10248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71654 payload_rows_remote=61418 dup_rows=26650 dup_rows_remote=24038 dedup_ratio=0.271098 router_tokens_per_expert=[163, 420, 36, 127, 4397, 82, 1508, 302, 32, 485, 1527, 401, 501, 665, 51, 3496, 303, 960, 507, 632, 465, 1139, 541, 482, 455, 444, 812, 4033, 3, 98, 102, 615, 4, 3, 1, 4, 318, 1, 275, 499, 801, 479, 81, 592, 12, 967, 0, 247, 6956, 251, 3364, 0, 26, 205, 456, 1402, 2755, 392, 764, 189, 356, 152, 1620, 196] payload_rows_per_dst=[4708, 6374, 12206, 7828, 2202, 12112, 15988, 10236] dup_rows_per_dst=[568, 950, 4844, 1686, 8, 3040, 12942, 2612] combine_rows_per_src=[4708, 6374, 12206, 7828, 2202, 12112, 15988, 10236] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=286498816 dispatch_meta_bytes=342384 combine_payload_bytes=286498816 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=286841200 ingress_bytes=355704832 link_bytes=355704832 token_bytes=573340016 weight_movement_bytes=69206016 total_nvlink_bytes=642546032 weight_to_token_ratio=0.120707 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=29184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.338488646e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277577728 dispatch_meta_bytes=356144 combine_payload_bytes=277577728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277933872 ingress_bytes=312180736 link_bytes=312180736 token_bytes=555511600 weight_movement_bytes=34603008 total_nvlink_bytes=590114608 weight_to_token_ratio=0.062290 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=9472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.174710977e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=251568128 dispatch_meta_bytes=341824 combine_payload_bytes=251568128 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=251909952 ingress_bytes=320774144 link_bytes=320774144 token_bytes=503478080 weight_movement_bytes=69206016 total_nvlink_bytes=572684096 weight_to_token_ratio=0.137456 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=19456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.207047279e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=268804096 dispatch_meta_bytes=337336 combine_payload_bytes=268804096 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=269141432 ingress_bytes=338010112 link_bytes=338010112 token_bytes=537945528 weight_movement_bytes=69206016 total_nvlink_bytes=607151544 weight_to_token_ratio=0.128649 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.271904839e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79556 payload_rows_remote=66940 dup_rows=18748 dup_rows_remote=15148 dedup_ratio=0.190715 router_tokens_per_expert=[153, 497, 59, 140, 4661, 76, 1362, 443, 50, 510, 1207, 457, 436, 721, 71, 3314, 370, 922, 531, 530, 500, 1006, 420, 479, 396, 422, 799, 3897, 5, 82, 80, 548, 3, 0, 2, 8, 298, 0, 292, 539, 809, 505, 49, 712, 26, 1099, 2, 245, 6891, 250, 3796, 0, 11, 200, 437, 1319, 2813, 125, 709, 466, 356, 155, 1595, 296] payload_rows_per_dst=[4820, 6714, 11086, 10212, 12770, 12616, 11090, 10248] dup_rows_per_dst=[640, 1534, 3714, 2246, 1542, 3600, 2690, 2782] combine_rows_per_src=[4820, 6714, 11086, 10212, 12770, 12616, 11090, 10248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=276193280 dispatch_meta_bytes=330864 combine_payload_bytes=276193280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276524144 ingress_bytes=310796288 link_bytes=310796288 token_bytes=552717424 weight_movement_bytes=34603008 total_nvlink_bytes=587320432 weight_to_token_ratio=0.062605 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=40192 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.169501411e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=257024000 dispatch_meta_bytes=311424 combine_payload_bytes=257024000 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=257335424 ingress_bytes=326230016 link_bytes=326230016 token_bytes=514359424 weight_movement_bytes=69206016 total_nvlink_bytes=583565440 weight_to_token_ratio=0.134548 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=14208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.227577287e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=268804096 dispatch_meta_bytes=337336 combine_payload_bytes=268804096 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=269141432 ingress_bytes=338010112 link_bytes=338010112 token_bytes=537945528 weight_movement_bytes=69206016 total_nvlink_bytes=607151544 weight_to_token_ratio=0.128649 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.271904839e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=286498816 dispatch_meta_bytes=342384 combine_payload_bytes=286498816 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=286841200 ingress_bytes=355704832 link_bytes=355704832 token_bytes=573340016 weight_movement_bytes=69206016 total_nvlink_bytes=642546032 weight_to_token_ratio=0.120707 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=29184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.338488646e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=240295936 dispatch_meta_bytes=294704 combine_payload_bytes=240295936 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=240590640 ingress_bytes=344104960 link_bytes=344104960 token_bytes=480886576 weight_movement_bytes=103809024 total_nvlink_bytes=584695600 weight_to_token_ratio=0.215870 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.294839262e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=251568128 dispatch_meta_bytes=341824 combine_payload_bytes=251568128 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=251909952 ingress_bytes=320774144 link_bytes=320774144 token_bytes=503478080 weight_movement_bytes=69206016 total_nvlink_bytes=572684096 weight_to_token_ratio=0.137456 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=19456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.207047279e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=274186240 dispatch_meta_bytes=328352 combine_payload_bytes=274186240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274514592 ingress_bytes=308789248 link_bytes=308789248 token_bytes=548700832 weight_movement_bytes=34603008 total_nvlink_bytes=583303840 weight_to_token_ratio=0.063064 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23424 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.161949081e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75366 payload_rows_remote=62750 dup_rows=22938 dup_rows_remote=15106 dedup_ratio=0.233337 router_tokens_per_expert=[216, 519, 32, 124, 4549, 63, 1359, 392, 58, 656, 1366, 373, 466, 683, 70, 3121, 406, 906, 512, 1044, 583, 1114, 468, 511, 455, 368, 743, 3841, 1, 83, 95, 590, 1, 0, 6, 5, 378, 0, 290, 500, 774, 461, 199, 503, 16, 1195, 0, 262, 6817, 251, 2970, 0, 25, 210, 424, 1287, 2931, 348, 778, 198, 383, 218, 1711, 244] payload_rows_per_dst=[12616, 12118, 8770, 10728, 14246, 5394, 4272, 7222] dup_rows_per_dst=[7832, 5708, 2318, 2898, 1748, 1426, 122, 886] combine_rows_per_src=[12616, 12118, 8770, 10728, 14246, 5394, 4272, 7222] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=257024000 dispatch_meta_bytes=311424 combine_payload_bytes=257024000 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=257335424 ingress_bytes=326230016 link_bytes=326230016 token_bytes=514359424 weight_movement_bytes=69206016 total_nvlink_bytes=583565440 weight_to_token_ratio=0.134548 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=14208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.227577287e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79516 payload_rows_remote=69112 dup_rows=18788 dup_rows_remote=15510 dedup_ratio=0.191121 router_tokens_per_expert=[0, 225, 72, 154, 0, 4, 351, 2086, 20, 2422, 168, 1803, 347, 376, 1, 1, 4545, 845, 5254, 143, 1, 0, 5121, 764, 216, 967, 2061, 2915, 1, 455, 226, 0, 48, 550, 195, 167, 4356, 415, 166, 79, 1011, 0, 0, 21, 424, 2302, 92, 0, 1095, 6, 451, 373, 1, 0, 238, 0, 3269, 154, 11, 12, 0, 1153, 0, 1019] payload_rows_per_dst=[12900, 8938, 3314, 10404, 9982, 12942, 11644, 9392] dup_rows_per_dst=[3126, 1338, 192, 3278, 1970, 3848, 3192, 1844] combine_rows_per_src=[12900, 8938, 3314, 10404, 9982, 12942, 11644, 9392] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80066 payload_rows_remote=70802 dup_rows=18238 dup_rows_remote=16928 dedup_ratio=0.185527 router_tokens_per_expert=[0, 164, 74, 155, 1, 4, 326, 2198, 40, 2706, 174, 1575, 312, 340, 0, 0, 4426, 1000, 5269, 123, 0, 0, 5189, 646, 272, 1043, 2311, 2495, 2, 274, 209, 0, 47, 620, 242, 125, 4438, 378, 143, 88, 986, 0, 0, 12, 441, 2295, 76, 1, 1017, 6, 412, 417, 2, 0, 186, 0, 3399, 185, 19, 6, 0, 1205, 1, 1077] payload_rows_per_dst=[12924, 11594, 9264, 7396, 10798, 6892, 11508, 9690] dup_rows_per_dst=[3298, 3108, 1310, 1408, 2916, 994, 3110, 2094] combine_rows_per_src=[12924, 11594, 9264, 7396, 10798, 6892, 11508, 9690] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77222 payload_rows_remote=65316 dup_rows=21082 dup_rows_remote=17028 dedup_ratio=0.214457 router_tokens_per_expert=[1, 132, 43, 156, 0, 1, 343, 1997, 5, 2544, 112, 1782, 212, 323, 0, 2, 4750, 824, 5270, 114, 0, 0, 5322, 692, 201, 882, 2158, 3000, 2, 408, 213, 0, 21, 595, 172, 95, 4597, 404, 205, 99, 834, 0, 0, 11, 448, 2083, 86, 0, 940, 5, 516, 426, 2, 1, 215, 1, 3873, 163, 4, 4, 0, 1013, 0, 850] payload_rows_per_dst=[12904, 11906, 13636, 6804, 10392, 6164, 5478, 9938] dup_rows_per_dst=[3086, 4054, 7388, 924, 1984, 760, 1010, 1876] combine_rows_per_src=[12904, 11906, 13636, 6804, 10392, 6164, 5478, 9938] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78938 payload_rows_remote=66170 dup_rows=19366 dup_rows_remote=14050 dedup_ratio=0.197001 router_tokens_per_expert=[0, 201, 46, 188, 0, 16, 371, 1914, 2, 2589, 112, 1701, 369, 352, 0, 2, 4477, 773, 5163, 130, 1, 1, 5339, 798, 213, 928, 2326, 3392, 3, 372, 254, 0, 40, 578, 156, 95, 4729, 363, 180, 104, 1037, 0, 1, 23, 414, 2202, 77, 0, 959, 2, 398, 307, 2, 0, 168, 0, 3067, 144, 10, 4, 1, 1020, 0, 1038] payload_rows_per_dst=[5264, 9174, 9776, 12314, 10560, 7734, 11348, 12768] dup_rows_per_dst=[208, 1080, 1760, 5210, 1930, 1212, 2650, 5316] combine_rows_per_src=[5264, 9174, 9776, 12314, 10560, 7734, 11348, 12768] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75622 payload_rows_remote=67768 dup_rows=22682 dup_rows_remote=21268 dedup_ratio=0.230733 router_tokens_per_expert=[142, 559, 20, 148, 4882, 53, 1702, 334, 45, 504, 1394, 456, 528, 817, 46, 3284, 321, 978, 551, 460, 521, 885, 454, 464, 422, 425, 832, 3763, 2, 72, 97, 733, 2, 0, 5, 15, 365, 4, 234, 496, 781, 496, 58, 689, 22, 938, 3, 205, 6905, 226, 3654, 0, 12, 203, 404, 1361, 2590, 302, 687, 452, 404, 207, 1290, 248] payload_rows_per_dst=[13322, 10860, 7854, 10288, 14300, 4994, 4308, 9696] dup_rows_per_dst=[9582, 3288, 1414, 2404, 1752, 1474, 104, 2664] combine_rows_per_src=[13322, 10860, 7854, 10288, 14300, 4994, 4308, 9696] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277577728 dispatch_meta_bytes=356144 combine_payload_bytes=277577728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277933872 ingress_bytes=312180736 link_bytes=312180736 token_bytes=555511600 weight_movement_bytes=34603008 total_nvlink_bytes=590114608 weight_to_token_ratio=0.062290 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=9472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.174710977e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75944 payload_rows_remote=63416 dup_rows=22360 dup_rows_remote=19126 dedup_ratio=0.227458 router_tokens_per_expert=[0, 244, 41, 148, 0, 7, 366, 2075, 38, 2472, 121, 1735, 370, 313, 0, 0, 4662, 855, 5128, 143, 2, 0, 5000, 696, 240, 940, 2362, 2995, 1, 470, 223, 0, 44, 598, 203, 122, 4351, 386, 207, 107, 1125, 0, 0, 18, 425, 2110, 103, 1, 987, 5, 369, 458, 0, 0, 259, 0, 3221, 130, 11, 14, 0, 1221, 0, 1030] payload_rows_per_dst=[12528, 12050, 14056, 7158, 9956, 6716, 3916, 9564] dup_rows_per_dst=[3234, 4038, 8916, 1314, 2080, 848, 240, 1690] combine_rows_per_src=[12528, 12050, 14056, 7158, 9956, 6716, 3916, 9564] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79854 payload_rows_remote=69834 dup_rows=18450 dup_rows_remote=16328 dedup_ratio=0.187683 router_tokens_per_expert=[0, 155, 50, 134, 0, 5, 409, 2016, 33, 2720, 132, 1798, 362, 343, 0, 0, 4475, 799, 5465, 86, 0, 0, 4828, 725, 223, 968, 2251, 3243, 0, 225, 222, 0, 43, 677, 208, 109, 4355, 386, 204, 89, 1048, 0, 0, 37, 446, 2175, 76, 0, 953, 5, 334, 415, 7, 0, 221, 0, 3258, 173, 10, 19, 0, 1177, 0, 1060] payload_rows_per_dst=[12334, 9364, 3088, 10860, 10020, 12854, 11762, 9572] dup_rows_per_dst=[2860, 1412, 132, 3404, 2122, 3660, 3038, 1822] combine_rows_per_src=[12334, 9364, 3088, 10860, 10020, 12854, 11762, 9572] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77222 payload_rows_remote=65316 dup_rows=21082 dup_rows_remote=17028 dedup_ratio=0.214457 router_tokens_per_expert=[1, 132, 43, 156, 0, 1, 343, 1997, 5, 2544, 112, 1782, 212, 323, 0, 2, 4750, 824, 5270, 114, 0, 0, 5322, 692, 201, 882, 2158, 3000, 2, 408, 213, 0, 21, 595, 172, 95, 4597, 404, 205, 99, 834, 0, 0, 11, 448, 2083, 86, 0, 940, 5, 516, 426, 2, 1, 215, 1, 3873, 163, 4, 4, 0, 1013, 0, 850] payload_rows_per_dst=[12904, 11906, 13636, 6804, 10392, 6164, 5478, 9938] dup_rows_per_dst=[3086, 4054, 7388, 924, 1984, 760, 1010, 1876] combine_rows_per_src=[12904, 11906, 13636, 6804, 10392, 6164, 5478, 9938] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79516 payload_rows_remote=69112 dup_rows=18788 dup_rows_remote=15510 dedup_ratio=0.191121 router_tokens_per_expert=[0, 225, 72, 154, 0, 4, 351, 2086, 20, 2422, 168, 1803, 347, 376, 1, 1, 4545, 845, 5254, 143, 1, 0, 5121, 764, 216, 967, 2061, 2915, 1, 455, 226, 0, 48, 550, 195, 167, 4356, 415, 166, 79, 1011, 0, 0, 21, 424, 2302, 92, 0, 1095, 6, 451, 373, 1, 0, 238, 0, 3269, 154, 11, 12, 0, 1153, 0, 1019] payload_rows_per_dst=[12900, 8938, 3314, 10404, 9982, 12942, 11644, 9392] dup_rows_per_dst=[3126, 1338, 192, 3278, 1970, 3848, 3192, 1844] combine_rows_per_src=[12900, 8938, 3314, 10404, 9982, 12942, 11644, 9392] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80062 payload_rows_remote=68558 dup_rows=18242 dup_rows_remote=15128 dedup_ratio=0.185567 router_tokens_per_expert=[1, 190, 55, 192, 0, 7, 359, 2062, 22, 2632, 150, 1563, 297, 337, 0, 1, 4519, 856, 5292, 169, 1, 0, 5187, 872, 259, 1022, 2284, 2682, 1, 315, 224, 0, 56, 578, 183, 140, 4268, 381, 180, 68, 1022, 0, 3, 28, 527, 2233, 91, 1, 916, 3, 422, 497, 8, 0, 169, 2, 3473, 156, 16, 5, 1, 1094, 1, 1079] payload_rows_per_dst=[5482, 8874, 11696, 10330, 9722, 12952, 11504, 9502] dup_rows_per_dst=[250, 1130, 2474, 3244, 1986, 3896, 3114, 2148] combine_rows_per_src=[5482, 8874, 11696, 10330, 9722, 12952, 11504, 9502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78938 payload_rows_remote=66170 dup_rows=19366 dup_rows_remote=14050 dedup_ratio=0.197001 router_tokens_per_expert=[0, 201, 46, 188, 0, 16, 371, 1914, 2, 2589, 112, 1701, 369, 352, 0, 2, 4477, 773, 5163, 130, 1, 1, 5339, 798, 213, 928, 2326, 3392, 3, 372, 254, 0, 40, 578, 156, 95, 4729, 363, 180, 104, 1037, 0, 1, 23, 414, 2202, 77, 0, 959, 2, 398, 307, 2, 0, 168, 0, 3067, 144, 10, 4, 1, 1020, 0, 1038] payload_rows_per_dst=[5264, 9174, 9776, 12314, 10560, 7734, 11348, 12768] dup_rows_per_dst=[208, 1080, 1760, 5210, 1930, 1212, 2650, 5316] combine_rows_per_src=[5264, 9174, 9776, 12314, 10560, 7734, 11348, 12768] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80364 payload_rows_remote=67260 dup_rows=17940 dup_rows_remote=14212 dedup_ratio=0.182495 router_tokens_per_expert=[0, 183, 61, 179, 0, 4, 408, 1945, 2, 2734, 156, 1638, 253, 339, 0, 0, 4555, 675, 5251, 102, 1, 0, 5236, 810, 204, 996, 2278, 2964, 2, 446, 238, 0, 32, 553, 205, 119, 4328, 431, 192, 83, 1145, 0, 0, 22, 415, 2200, 79, 0, 1050, 4, 476, 353, 2, 0, 259, 0, 3258, 152, 11, 5, 0, 1063, 0, 1055] payload_rows_per_dst=[6966, 8978, 9684, 10772, 9952, 13104, 11534, 9374] dup_rows_per_dst=[940, 1266, 1618, 3484, 1934, 3728, 3256, 1714] combine_rows_per_src=[6966, 8978, 9684, 10772, 9952, 13104, 11534, 9374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=283082752 dispatch_meta_bytes=338488 combine_payload_bytes=283082752 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=283421240 ingress_bytes=352288768 link_bytes=352288768 token_bytes=566503992 weight_movement_bytes=69206016 total_nvlink_bytes=635710008 weight_to_token_ratio=0.122163 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=18048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.325634272e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=290004992 dispatch_meta_bytes=350920 combine_payload_bytes=290004992 prefetch_bytes=0 reduce_bytes=207618048 egress_bytes=290355912 ingress_bytes=497623040 link_bytes=497623040 token_bytes=580360904 weight_movement_bytes=207618048 total_nvlink_bytes=787978952 weight_to_token_ratio=0.357740 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=13440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.872515438e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=259751936 dispatch_meta_bytes=330168 combine_payload_bytes=259751936 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260082104 ingress_bytes=294354944 link_bytes=294354944 token_bytes=519834040 weight_movement_bytes=34603008 total_nvlink_bytes=554437048 weight_to_token_ratio=0.066565 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=26752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.107633957e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=286040064 dispatch_meta_bytes=344648 combine_payload_bytes=286040064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286384712 ingress_bytes=320643072 link_bytes=320643072 token_bytes=572424776 weight_movement_bytes=34603008 total_nvlink_bytes=607027784 weight_to_token_ratio=0.060450 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=35456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.206554066e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267534336 dispatch_meta_bytes=329376 combine_payload_bytes=267534336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267863712 ingress_bytes=302137344 link_bytes=302137344 token_bytes=535398048 weight_movement_bytes=34603008 total_nvlink_bytes=570001056 weight_to_token_ratio=0.064630 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=20864 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136918501e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=271032320 dispatch_meta_bytes=320880 combine_payload_bytes=271032320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271353200 ingress_bytes=305635328 link_bytes=305635328 token_bytes=542385520 weight_movement_bytes=34603008 total_nvlink_bytes=576988528 weight_to_token_ratio=0.063798 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=26880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.150081134e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=283082752 dispatch_meta_bytes=338488 combine_payload_bytes=283082752 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=283421240 ingress_bytes=352288768 link_bytes=352288768 token_bytes=566503992 weight_movement_bytes=69206016 total_nvlink_bytes=635710008 weight_to_token_ratio=0.122163 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=18048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.325634272e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=286040064 dispatch_meta_bytes=344648 combine_payload_bytes=286040064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286384712 ingress_bytes=320643072 link_bytes=320643072 token_bytes=572424776 weight_movement_bytes=34603008 total_nvlink_bytes=607027784 weight_to_token_ratio=0.060450 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=35456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.206554066e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=280813568 dispatch_meta_bytes=334744 combine_payload_bytes=280813568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281148312 ingress_bytes=315416576 link_bytes=315416576 token_bytes=561961880 weight_movement_bytes=34603008 total_nvlink_bytes=596564888 weight_to_token_ratio=0.061575 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=32896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.186887182e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75944 payload_rows_remote=63416 dup_rows=22360 dup_rows_remote=19126 dedup_ratio=0.227458 router_tokens_per_expert=[0, 244, 41, 148, 0, 7, 366, 2075, 38, 2472, 121, 1735, 370, 313, 0, 0, 4662, 855, 5128, 143, 2, 0, 5000, 696, 240, 940, 2362, 2995, 1, 470, 223, 0, 44, 598, 203, 122, 4351, 386, 207, 107, 1125, 0, 0, 18, 425, 2110, 103, 1, 987, 5, 369, 458, 0, 0, 259, 0, 3221, 130, 11, 14, 0, 1221, 0, 1030] payload_rows_per_dst=[12528, 12050, 14056, 7158, 9956, 6716, 3916, 9564] dup_rows_per_dst=[3234, 4038, 8916, 1314, 2080, 848, 240, 1690] combine_rows_per_src=[12528, 12050, 14056, 7158, 9956, 6716, 3916, 9564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=271032320 dispatch_meta_bytes=320880 combine_payload_bytes=271032320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271353200 ingress_bytes=305635328 link_bytes=305635328 token_bytes=542385520 weight_movement_bytes=34603008 total_nvlink_bytes=576988528 weight_to_token_ratio=0.063798 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=26880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.150081134e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=275496960 dispatch_meta_bytes=325888 combine_payload_bytes=275496960 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275822848 ingress_bytes=310099968 link_bytes=310099968 token_bytes=551319808 weight_movement_bytes=34603008 total_nvlink_bytes=585922816 weight_to_token_ratio=0.062764 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=19072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.166881214e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=259751936 dispatch_meta_bytes=330168 combine_payload_bytes=259751936 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260082104 ingress_bytes=294354944 link_bytes=294354944 token_bytes=519834040 weight_movement_bytes=34603008 total_nvlink_bytes=554437048 weight_to_token_ratio=0.066565 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=26752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.107633957e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82708 payload_rows_remote=73736 dup_rows=15596 dup_rows_remote=12566 dedup_ratio=0.158651 router_tokens_per_expert=[0, 0, 3, 0, 2905, 779, 1033, 117, 416, 82, 326, 0, 245, 0, 6969, 0, 2772, 1538, 15, 1, 1208, 266, 0, 184, 1211, 1037, 1234, 816, 517, 697, 489, 0, 0, 11, 5, 0, 1302, 20, 116, 1, 534, 5687, 103, 534, 259, 0, 0, 2202, 0, 2, 81, 516, 137, 2592, 0, 0, 0, 0, 675, 0, 2874, 5952, 0, 689] payload_rows_per_dst=[8478, 14700, 9662, 8972, 13130, 6686, 13188, 7892] dup_rows_per_dst=[1196, 1376, 2306, 3030, 1684, 1212, 4208, 584] combine_rows_per_src=[8478, 14700, 9662, 8972, 13130, 6686, 13188, 7892] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82404 payload_rows_remote=73130 dup_rows=15900 dup_rows_remote=13724 dedup_ratio=0.161743 router_tokens_per_expert=[0, 0, 0, 0, 3151, 780, 1136, 60, 414, 74, 396, 0, 248, 0, 7002, 1, 2436, 1450, 14, 1, 1403, 272, 0, 149, 1170, 975, 1145, 715, 427, 645, 491, 0, 0, 7, 59, 3, 1348, 30, 107, 0, 727, 5962, 104, 540, 227, 0, 0, 2460, 0, 0, 69, 487, 96, 2446, 0, 0, 0, 0, 667, 1, 2873, 5661, 0, 723] payload_rows_per_dst=[8976, 14716, 9274, 8312, 12758, 14270, 6142, 7956] dup_rows_per_dst=[1278, 1554, 2176, 2824, 1672, 5770, 54, 572] combine_rows_per_src=[8976, 14716, 9274, 8312, 12758, 14270, 6142, 7956] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80066 payload_rows_remote=70802 dup_rows=18238 dup_rows_remote=16928 dedup_ratio=0.185527 router_tokens_per_expert=[0, 164, 74, 155, 1, 4, 326, 2198, 40, 2706, 174, 1575, 312, 340, 0, 0, 4426, 1000, 5269, 123, 0, 0, 5189, 646, 272, 1043, 2311, 2495, 2, 274, 209, 0, 47, 620, 242, 125, 4438, 378, 143, 88, 986, 0, 0, 12, 441, 2295, 76, 1, 1017, 6, 412, 417, 2, 0, 186, 0, 3399, 185, 19, 6, 0, 1205, 1, 1077] payload_rows_per_dst=[12924, 11594, 9264, 7396, 10798, 6892, 11508, 9690] dup_rows_per_dst=[3298, 3108, 1310, 1408, 2916, 994, 3110, 2094] combine_rows_per_src=[12924, 11594, 9264, 7396, 10798, 6892, 11508, 9690] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77210 payload_rows_remote=61816 dup_rows=21094 dup_rows_remote=12876 dedup_ratio=0.214579 router_tokens_per_expert=[0, 0, 2, 0, 2996, 796, 989, 122, 388, 81, 383, 0, 255, 0, 6901, 0, 2622, 1316, 34, 1, 1294, 311, 0, 138, 1200, 957, 1134, 686, 611, 685, 451, 0, 0, 9, 41, 1, 1332, 31, 164, 0, 631, 5803, 75, 608, 238, 0, 0, 2385, 0, 2, 86, 477, 92, 2546, 0, 0, 1, 0, 850, 0, 2957, 5753, 1, 716] payload_rows_per_dst=[15394, 2242, 9234, 8718, 12936, 14012, 6318, 8356] dup_rows_per_dst=[8218, 36, 2198, 2730, 1726, 5404, 88, 694] combine_rows_per_src=[15394, 2242, 9234, 8718, 12936, 14012, 6318, 8356] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80140 payload_rows_remote=71112 dup_rows=18164 dup_rows_remote=17544 dedup_ratio=0.184774 router_tokens_per_expert=[0, 0, 0, 2, 2789, 733, 1129, 119, 382, 86, 249, 1, 248, 0, 6883, 4, 2793, 1354, 9, 3, 1382, 279, 0, 134, 1183, 929, 1241, 724, 512, 731, 461, 0, 0, 6, 47, 0, 1427, 16, 156, 0, 929, 6017, 71, 648, 226, 0, 1, 2122, 1, 0, 86, 561, 51, 2626, 0, 2, 1, 1, 689, 2, 2860, 5544, 2, 700] payload_rows_per_dst=[11450, 9028, 9508, 8760, 12682, 14332, 6528, 7852] dup_rows_per_dst=[4152, 620, 2400, 2802, 1710, 5696, 126, 658] combine_rows_per_src=[11450, 9028, 9508, 8760, 12682, 14332, 6528, 7852] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70252 payload_rows_remote=54308 dup_rows=28052 dup_rows_remote=14186 dedup_ratio=0.285360 router_tokens_per_expert=[0, 0, 6, 3, 3055, 704, 1104, 68, 424, 78, 298, 0, 243, 0, 6988, 1, 2834, 1594, 23, 2, 1290, 284, 0, 108, 1160, 787, 1173, 808, 666, 759, 455, 1, 0, 10, 41, 6, 1325, 13, 144, 2, 586, 5993, 81, 552, 219, 3, 1, 2276, 2, 3, 72, 504, 113, 2346, 0, 0, 5, 0, 683, 0, 2698, 5850, 2, 706] payload_rows_per_dst=[8676, 5926, 11098, 11742, 3060, 6396, 7410, 15944] dup_rows_per_dst=[1204, 352, 5008, 5826, 22, 1040, 734, 13866] combine_rows_per_src=[8676, 5926, 11098, 11742, 3060, 6396, 7410, 15944] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=290004992 dispatch_meta_bytes=350920 combine_payload_bytes=290004992 prefetch_bytes=0 reduce_bytes=207618048 egress_bytes=290355912 ingress_bytes=497623040 link_bytes=497623040 token_bytes=580360904 weight_movement_bytes=207618048 total_nvlink_bytes=787978952 weight_to_token_ratio=0.357740 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=13440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.872515438e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82708 payload_rows_remote=73736 dup_rows=15596 dup_rows_remote=12566 dedup_ratio=0.158651 router_tokens_per_expert=[0, 0, 3, 0, 2905, 779, 1033, 117, 416, 82, 326, 0, 245, 0, 6969, 0, 2772, 1538, 15, 1, 1208, 266, 0, 184, 1211, 1037, 1234, 816, 517, 697, 489, 0, 0, 11, 5, 0, 1302, 20, 116, 1, 534, 5687, 103, 534, 259, 0, 0, 2202, 0, 2, 81, 516, 137, 2592, 0, 0, 0, 0, 675, 0, 2874, 5952, 0, 689] payload_rows_per_dst=[8478, 14700, 9662, 8972, 13130, 6686, 13188, 7892] dup_rows_per_dst=[1196, 1376, 2306, 3030, 1684, 1212, 4208, 584] combine_rows_per_src=[8478, 14700, 9662, 8972, 13130, 6686, 13188, 7892] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82264 payload_rows_remote=68968 dup_rows=16040 dup_rows_remote=14178 dedup_ratio=0.163167 router_tokens_per_expert=[0, 0, 3, 0, 2818, 723, 1106, 96, 437, 74, 354, 0, 252, 0, 6838, 0, 2487, 1367, 25, 0, 1552, 324, 0, 149, 1190, 928, 1212, 694, 484, 758, 483, 0, 0, 8, 10, 3, 1479, 14, 163, 0, 584, 5759, 118, 505, 239, 0, 0, 2293, 0, 0, 73, 537, 142, 2563, 0, 0, 2, 0, 715, 0, 2971, 5902, 0, 718] payload_rows_per_dst=[8178, 14418, 9512, 8700, 13296, 6542, 13584, 8034] dup_rows_per_dst=[1314, 1492, 2296, 2798, 1862, 936, 4564, 778] combine_rows_per_src=[8178, 14418, 9512, 8700, 13296, 6542, 13584, 8034] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80566 payload_rows_remote=66898 dup_rows=17738 dup_rows_remote=13714 dedup_ratio=0.180440 router_tokens_per_expert=[0, 1, 1, 1, 2954, 866, 1096, 63, 366, 90, 362, 0, 237, 1, 6869, 2, 2726, 1404, 13, 1, 1400, 278, 0, 124, 1217, 784, 1241, 790, 472, 712, 445, 0, 0, 11, 49, 11, 1340, 17, 147, 0, 718, 5791, 92, 584, 299, 0, 0, 2361, 1, 1, 90, 503, 91, 2367, 2, 0, 10, 3, 710, 0, 2815, 5880, 0, 743] payload_rows_per_dst=[8682, 14532, 9554, 8366, 6436, 6918, 13668, 12410] dup_rows_per_dst=[1282, 1322, 2338, 2956, 584, 1190, 4024, 4042] combine_rows_per_src=[8682, 14532, 9554, 8366, 6436, 6918, 13668, 12410] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70252 payload_rows_remote=54308 dup_rows=28052 dup_rows_remote=14186 dedup_ratio=0.285360 router_tokens_per_expert=[0, 0, 6, 3, 3055, 704, 1104, 68, 424, 78, 298, 0, 243, 0, 6988, 1, 2834, 1594, 23, 2, 1290, 284, 0, 108, 1160, 787, 1173, 808, 666, 759, 455, 1, 0, 10, 41, 6, 1325, 13, 144, 2, 586, 5993, 81, 552, 219, 3, 1, 2276, 2, 3, 72, 504, 113, 2346, 0, 0, 5, 0, 683, 0, 2698, 5850, 2, 706] payload_rows_per_dst=[8676, 5926, 11098, 11742, 3060, 6396, 7410, 15944] dup_rows_per_dst=[1204, 352, 5008, 5826, 22, 1040, 734, 13866] combine_rows_per_src=[8676, 5926, 11098, 11742, 3060, 6396, 7410, 15944] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83002 payload_rows_remote=76238 dup_rows=15302 dup_rows_remote=14166 dedup_ratio=0.155660 router_tokens_per_expert=[0, 0, 4, 0, 2933, 839, 1005, 132, 357, 78, 313, 0, 255, 0, 6988, 4, 2611, 1323, 15, 1, 1514, 320, 0, 86, 1304, 769, 1203, 783, 560, 813, 452, 0, 0, 5, 46, 2, 1199, 17, 185, 1, 701, 5991, 94, 670, 299, 0, 0, 2186, 0, 0, 76, 522, 75, 2170, 0, 0, 1, 0, 749, 1, 2764, 5918, 2, 816] payload_rows_per_dst=[8554, 14778, 9388, 8750, 13102, 6764, 13774, 7892] dup_rows_per_dst=[1272, 1212, 2352, 3018, 1644, 1136, 3894, 774] combine_rows_per_src=[8554, 14778, 9388, 8750, 13102, 6764, 13774, 7892] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=302022656 dispatch_meta_bytes=345208 combine_payload_bytes=302022656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=302367864 ingress_bytes=336625664 link_bytes=336625664 token_bytes=604390520 weight_movement_bytes=34603008 total_nvlink_bytes=638993528 weight_to_token_ratio=0.057253 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266695273e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83002 payload_rows_remote=76238 dup_rows=15302 dup_rows_remote=14166 dedup_ratio=0.155660 router_tokens_per_expert=[0, 0, 4, 0, 2933, 839, 1005, 132, 357, 78, 313, 0, 255, 0, 6988, 4, 2611, 1323, 15, 1, 1514, 320, 0, 86, 1304, 769, 1203, 783, 560, 813, 452, 0, 0, 5, 46, 2, 1199, 17, 185, 1, 701, 5991, 94, 670, 299, 0, 0, 2186, 0, 0, 76, 522, 75, 2170, 0, 0, 1, 0, 749, 1, 2764, 5918, 2, 816] payload_rows_per_dst=[8554, 14778, 9388, 8750, 13102, 6764, 13774, 7892] dup_rows_per_dst=[1272, 1212, 2352, 3018, 1644, 1136, 3894, 774] combine_rows_per_src=[8554, 14778, 9388, 8750, 13102, 6764, 13774, 7892] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=299540480 dispatch_meta_bytes=347416 combine_payload_bytes=299540480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=299887896 ingress_bytes=334143488 link_bytes=334143488 token_bytes=599428376 weight_movement_bytes=34603008 total_nvlink_bytes=634031384 weight_to_token_ratio=0.057727 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=21376 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257355045e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=291274752 dispatch_meta_bytes=354624 combine_payload_bytes=291274752 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=291629376 ingress_bytes=429686784 link_bytes=429686784 token_bytes=582904128 weight_movement_bytes=138412032 total_nvlink_bytes=721316160 weight_to_token_ratio=0.237452 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=40704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.616876776e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=222445568 dispatch_meta_bytes=273976 combine_payload_bytes=222445568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=222719544 ingress_bytes=291651584 link_bytes=291651584 token_bytes=445165112 weight_movement_bytes=69206016 total_nvlink_bytes=514371128 weight_to_token_ratio=0.155461 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=22912 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.097461431e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=253198336 dispatch_meta_bytes=298768 combine_payload_bytes=253198336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=253497104 ingress_bytes=287801344 link_bytes=287801344 token_bytes=506695440 weight_movement_bytes=34603008 total_nvlink_bytes=541298448 weight_to_token_ratio=0.068292 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.082973288e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=302022656 dispatch_meta_bytes=345208 combine_payload_bytes=302022656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=302367864 ingress_bytes=336625664 link_bytes=336625664 token_bytes=604390520 weight_movement_bytes=34603008 total_nvlink_bytes=638993528 weight_to_token_ratio=0.057253 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266695273e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=282492928 dispatch_meta_bytes=332584 combine_payload_bytes=282492928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282825512 ingress_bytes=317095936 link_bytes=317095936 token_bytes=565318440 weight_movement_bytes=34603008 total_nvlink_bytes=599921448 weight_to_token_ratio=0.061210 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=36736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.193206479e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=274014208 dispatch_meta_bytes=322448 combine_payload_bytes=274014208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274336656 ingress_bytes=308617216 link_bytes=308617216 token_bytes=548350864 weight_movement_bytes=34603008 total_nvlink_bytes=582953872 weight_to_token_ratio=0.063104 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=24064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.161301738e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77210 payload_rows_remote=61816 dup_rows=21094 dup_rows_remote=12876 dedup_ratio=0.214579 router_tokens_per_expert=[0, 0, 2, 0, 2996, 796, 989, 122, 388, 81, 383, 0, 255, 0, 6901, 0, 2622, 1316, 34, 1, 1294, 311, 0, 138, 1200, 957, 1134, 686, 611, 685, 451, 0, 0, 9, 41, 1, 1332, 31, 164, 0, 631, 5803, 75, 608, 238, 0, 0, 2385, 0, 2, 86, 477, 92, 2546, 0, 0, 1, 0, 850, 0, 2957, 5753, 1, 716] payload_rows_per_dst=[15394, 2242, 9234, 8718, 12936, 14012, 6318, 8356] dup_rows_per_dst=[8218, 36, 2198, 2730, 1726, 5404, 88, 694] combine_rows_per_src=[15394, 2242, 9234, 8718, 12936, 14012, 6318, 8356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=222445568 dispatch_meta_bytes=273976 combine_payload_bytes=222445568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=222719544 ingress_bytes=291651584 link_bytes=291651584 token_bytes=445165112 weight_movement_bytes=69206016 total_nvlink_bytes=514371128 weight_to_token_ratio=0.155461 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=22912 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.097461431e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=312270848 dispatch_meta_bytes=361616 combine_payload_bytes=312270848 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=312632464 ingress_bytes=416079872 link_bytes=416079872 token_bytes=624903312 weight_movement_bytes=103809024 total_nvlink_bytes=728712336 weight_to_token_ratio=0.166120 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=18304 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.565675061e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=253198336 dispatch_meta_bytes=298768 combine_payload_bytes=253198336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=253497104 ingress_bytes=287801344 link_bytes=287801344 token_bytes=506695440 weight_movement_bytes=34603008 total_nvlink_bytes=541298448 weight_to_token_ratio=0.068292 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.082973288e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86086 payload_rows_remote=78068 dup_rows=12218 dup_rows_remote=11428 dedup_ratio=0.124288 router_tokens_per_expert=[0, 529, 3386, 0, 0, 16, 0, 112, 0, 0, 0, 60, 72, 0, 0, 0, 0, 3116, 0, 3079, 2, 246, 6503, 7, 8, 0, 309, 76, 0, 1995, 0, 0, 0, 0, 0, 598, 1701, 1, 3908, 159, 0, 2196, 0, 7845, 60, 0, 151, 0, 0, 507, 7220, 0, 0, 382, 4228, 430, 0, 0, 0, 1, 248, 0, 1, 0] payload_rows_per_dst=[11858, 13012, 8018, 15794, 10536, 4694, 7732, 14442] dup_rows_per_dst=[2386, 258, 790, 4672, 2198, 120, 1296, 498] combine_rows_per_src=[11858, 13012, 8018, 15794, 10536, 4694, 7732, 14442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82404 payload_rows_remote=73130 dup_rows=15900 dup_rows_remote=13724 dedup_ratio=0.161743 router_tokens_per_expert=[0, 0, 0, 0, 3151, 780, 1136, 60, 414, 74, 396, 0, 248, 0, 7002, 1, 2436, 1450, 14, 1, 1403, 272, 0, 149, 1170, 975, 1145, 715, 427, 645, 491, 0, 0, 7, 59, 3, 1348, 30, 107, 0, 727, 5962, 104, 540, 227, 0, 0, 2460, 0, 0, 69, 487, 96, 2446, 0, 0, 0, 0, 667, 1, 2873, 5661, 0, 723] payload_rows_per_dst=[8976, 14716, 9274, 8312, 12758, 14270, 6142, 7956] dup_rows_per_dst=[1278, 1554, 2176, 2824, 1672, 5770, 54, 572] combine_rows_per_src=[8976, 14716, 9274, 8312, 12758, 14270, 6142, 7956] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84922 payload_rows_remote=71706 dup_rows=13382 dup_rows_remote=12954 dedup_ratio=0.136129 router_tokens_per_expert=[0, 612, 3777, 0, 0, 4, 0, 106, 0, 1, 0, 88, 129, 0, 0, 0, 0, 2930, 0, 3330, 2, 199, 6604, 6, 1, 0, 274, 61, 0, 1905, 0, 0, 1, 0, 2, 517, 1603, 0, 3776, 263, 0, 2103, 1, 7817, 86, 0, 136, 0, 0, 227, 7292, 0, 0, 392, 4213, 446, 0, 0, 1, 0, 246, 0, 1, 0] payload_rows_per_dst=[12746, 13216, 12408, 15736, 10260, 4480, 7434, 8642] dup_rows_per_dst=[2912, 428, 2292, 4380, 2064, 172, 866, 268] combine_rows_per_src=[12746, 13216, 12408, 15736, 10260, 4480, 7434, 8642] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84636 payload_rows_remote=72406 dup_rows=13668 dup_rows_remote=11206 dedup_ratio=0.139038 router_tokens_per_expert=[0, 646, 3401, 0, 0, 3, 0, 96, 0, 0, 0, 61, 84, 0, 0, 0, 0, 2869, 0, 3200, 0, 216, 6539, 10, 5, 0, 562, 52, 0, 1851, 0, 0, 0, 0, 0, 648, 1487, 0, 3782, 291, 0, 2316, 0, 7860, 60, 0, 144, 0, 0, 256, 7301, 0, 0, 382, 4317, 457, 0, 0, 0, 0, 255, 0, 1, 0] payload_rows_per_dst=[12230, 13078, 12422, 16196, 10054, 4920, 15224, 512] dup_rows_per_dst=[2462, 290, 2402, 4464, 2362, 120, 1568, 0] combine_rows_per_src=[12230, 13078, 12422, 16196, 10054, 4920, 15224, 512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=299540480 dispatch_meta_bytes=347416 combine_payload_bytes=299540480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=299887896 ingress_bytes=334143488 link_bytes=334143488 token_bytes=599428376 weight_movement_bytes=34603008 total_nvlink_bytes=634031384 weight_to_token_ratio=0.057727 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=21376 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257355045e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86370 payload_rows_remote=73502 dup_rows=11934 dup_rows_remote=8182 dedup_ratio=0.121399 router_tokens_per_expert=[0, 546, 3109, 0, 0, 10, 0, 78, 0, 0, 0, 52, 73, 0, 0, 0, 0, 3330, 0, 3015, 0, 235, 6503, 6, 2, 0, 275, 96, 0, 2099, 0, 0, 0, 0, 0, 527, 1947, 0, 3889, 209, 0, 2059, 0, 7852, 52, 0, 140, 0, 0, 378, 7113, 0, 0, 414, 4435, 455, 0, 1, 0, 0, 247, 0, 5, 0] payload_rows_per_dst=[11474, 13024, 6984, 12868, 10688, 7344, 9752, 14236] dup_rows_per_dst=[2042, 232, 158, 3752, 2456, 1186, 1612, 496] combine_rows_per_src=[11474, 13024, 6984, 12868, 10688, 7344, 9752, 14236] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77982 payload_rows_remote=63636 dup_rows=20322 dup_rows_remote=19854 dedup_ratio=0.206726 router_tokens_per_expert=[0, 533, 3167, 1, 2, 40, 2, 49, 0, 1, 0, 61, 75, 0, 0, 0, 0, 3027, 0, 3499, 0, 214, 6680, 12, 0, 0, 354, 65, 0, 2051, 1, 0, 2, 0, 0, 523, 1490, 3, 3841, 199, 0, 2348, 1, 7844, 52, 1, 132, 0, 1, 244, 7161, 1, 1, 424, 4332, 472, 0, 0, 0, 5, 238, 0, 2, 1] payload_rows_per_dst=[7004, 4124, 14546, 4822, 8766, 11544, 12830, 14346] dup_rows_per_dst=[584, 114, 8354, 120, 1710, 2862, 6110, 468] combine_rows_per_src=[7004, 4124, 14546, 4822, 8766, 11544, 12830, 14346] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85798 payload_rows_remote=76118 dup_rows=12506 dup_rows_remote=10742 dedup_ratio=0.127218 router_tokens_per_expert=[0, 599, 3242, 5, 0, 13, 2, 111, 1, 2, 1, 51, 70, 0, 2, 0, 0, 3176, 0, 2957, 0, 225, 6411, 8, 6, 0, 523, 106, 0, 2055, 1, 2, 1, 0, 3, 514, 1621, 0, 4070, 169, 3, 2110, 0, 7851, 49, 0, 147, 1, 1, 487, 7081, 7, 0, 406, 4374, 447, 0, 3, 3, 0, 234, 0, 0, 1] payload_rows_per_dst=[7320, 12842, 10408, 5180, 10340, 15846, 9680, 14182] dup_rows_per_dst=[624, 234, 2324, 206, 2416, 4476, 1764, 462] combine_rows_per_src=[7320, 12842, 10408, 5180, 10340, 15846, 9680, 14182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77982 payload_rows_remote=63636 dup_rows=20322 dup_rows_remote=19854 dedup_ratio=0.206726 router_tokens_per_expert=[0, 533, 3167, 1, 2, 40, 2, 49, 0, 1, 0, 61, 75, 0, 0, 0, 0, 3027, 0, 3499, 0, 214, 6680, 12, 0, 0, 354, 65, 0, 2051, 1, 0, 2, 0, 0, 523, 1490, 3, 3841, 199, 0, 2348, 1, 7844, 52, 1, 132, 0, 1, 244, 7161, 1, 1, 424, 4332, 472, 0, 0, 0, 5, 238, 0, 2, 1] payload_rows_per_dst=[7004, 4124, 14546, 4822, 8766, 11544, 12830, 14346] dup_rows_per_dst=[584, 114, 8354, 120, 1710, 2862, 6110, 468] combine_rows_per_src=[7004, 4124, 14546, 4822, 8766, 11544, 12830, 14346] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86386 payload_rows_remote=76558 dup_rows=11918 dup_rows_remote=9692 dedup_ratio=0.121236 router_tokens_per_expert=[0, 574, 3491, 1, 0, 2, 0, 69, 0, 0, 0, 73, 110, 0, 0, 0, 0, 3097, 0, 3162, 1, 260, 6569, 9, 4, 0, 610, 23, 0, 1945, 0, 0, 0, 0, 0, 604, 1357, 0, 3784, 282, 0, 2166, 0, 7834, 68, 0, 114, 0, 0, 523, 7196, 0, 0, 415, 4124, 462, 0, 0, 0, 0, 222, 0, 1, 0] payload_rows_per_dst=[12240, 13138, 6546, 5118, 9828, 15842, 9282, 14392] dup_rows_per_dst=[2358, 366, 188, 46, 2226, 4522, 1766, 446] combine_rows_per_src=[12240, 13138, 6546, 5118, 9828, 15842, 9282, 14392] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86582 payload_rows_remote=70726 dup_rows=11722 dup_rows_remote=7402 dedup_ratio=0.119242 router_tokens_per_expert=[0, 601, 3477, 0, 0, 34, 0, 63, 0, 0, 0, 91, 103, 0, 0, 0, 0, 3183, 0, 2686, 0, 216, 6780, 4, 2, 0, 308, 70, 0, 2095, 0, 0, 0, 0, 0, 521, 1880, 0, 3539, 260, 0, 2069, 0, 7817, 88, 0, 114, 0, 0, 406, 7199, 0, 0, 414, 4398, 476, 0, 0, 0, 0, 256, 0, 2, 0] payload_rows_per_dst=[9010, 13560, 9106, 4820, 10148, 15856, 9684, 14398] dup_rows_per_dst=[1054, 388, 1358, 130, 2252, 4320, 1704, 516] combine_rows_per_src=[9010, 13560, 9106, 4820, 10148, 15856, 9684, 14398] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=293707776 dispatch_meta_bytes=338640 combine_payload_bytes=293707776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=294046416 ingress_bytes=328310784 link_bytes=328310784 token_bytes=587754192 weight_movement_bytes=34603008 total_nvlink_bytes=622357200 weight_to_token_ratio=0.058873 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=48000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.235407049e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=296574976 dispatch_meta_bytes=334448 combine_payload_bytes=296574976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296909424 ingress_bytes=331177984 link_bytes=331177984 token_bytes=593484400 weight_movement_bytes=34603008 total_nvlink_bytes=628087408 weight_to_token_ratio=0.058305 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=27136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.246196092e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=319766528 dispatch_meta_bytes=357984 combine_payload_bytes=319766528 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=320124512 ingress_bytes=423575552 link_bytes=423575552 token_bytes=639891040 weight_movement_bytes=103809024 total_nvlink_bytes=743700064 weight_to_token_ratio=0.162229 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=24832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.593880702e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=260653056 dispatch_meta_bytes=333960 combine_payload_bytes=260653056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260987016 ingress_bytes=295256064 link_bytes=295256064 token_bytes=521640072 weight_movement_bytes=34603008 total_nvlink_bytes=556243080 weight_to_token_ratio=0.066335 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=47232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.111024799e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=301064192 dispatch_meta_bytes=326736 combine_payload_bytes=301064192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=301390928 ingress_bytes=335667200 link_bytes=335667200 token_bytes=602455120 weight_movement_bytes=34603008 total_nvlink_bytes=637058128 weight_to_token_ratio=0.057437 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=29440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263088650e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=301064192 dispatch_meta_bytes=326736 combine_payload_bytes=301064192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=301390928 ingress_bytes=335667200 link_bytes=335667200 token_bytes=602455120 weight_movement_bytes=34603008 total_nvlink_bytes=637058128 weight_to_token_ratio=0.057437 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=29440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263088650e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=311779328 dispatch_meta_bytes=347440 combine_payload_bytes=311779328 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=312126768 ingress_bytes=415588352 link_bytes=415588352 token_bytes=623906096 weight_movement_bytes=103809024 total_nvlink_bytes=727715120 weight_to_token_ratio=0.166386 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=24960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.563825511e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84636 payload_rows_remote=72406 dup_rows=13668 dup_rows_remote=11206 dedup_ratio=0.139038 router_tokens_per_expert=[0, 646, 3401, 0, 0, 3, 0, 96, 0, 0, 0, 61, 84, 0, 0, 0, 0, 2869, 0, 3200, 0, 216, 6539, 10, 5, 0, 562, 52, 0, 1851, 0, 0, 0, 0, 0, 648, 1487, 0, 3782, 291, 0, 2316, 0, 7860, 60, 0, 144, 0, 0, 256, 7301, 0, 0, 382, 4317, 457, 0, 0, 0, 0, 255, 0, 1, 0] payload_rows_per_dst=[12230, 13078, 12422, 16196, 10054, 4920, 15224, 512] dup_rows_per_dst=[2462, 290, 2402, 4464, 2362, 120, 1568, 0] combine_rows_per_src=[12230, 13078, 12422, 16196, 10054, 4920, 15224, 512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=260653056 dispatch_meta_bytes=333960 combine_payload_bytes=260653056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260987016 ingress_bytes=295256064 link_bytes=295256064 token_bytes=521640072 weight_movement_bytes=34603008 total_nvlink_bytes=556243080 weight_to_token_ratio=0.066335 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=47232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.111024799e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=313581568 dispatch_meta_bytes=345000 combine_payload_bytes=313581568 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=313926568 ingress_bytes=348184576 link_bytes=348184576 token_bytes=627508136 weight_movement_bytes=34603008 total_nvlink_bytes=662111144 weight_to_token_ratio=0.055144 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=29824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.310190528e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=289693696 dispatch_meta_bytes=312512 combine_payload_bytes=289693696 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=290006208 ingress_bytes=393502720 link_bytes=393502720 token_bytes=579699904 weight_movement_bytes=103809024 total_nvlink_bytes=683508928 weight_to_token_ratio=0.179074 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=29440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.480719056e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=296574976 dispatch_meta_bytes=334448 combine_payload_bytes=296574976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296909424 ingress_bytes=331177984 link_bytes=331177984 token_bytes=593484400 weight_movement_bytes=34603008 total_nvlink_bytes=628087408 weight_to_token_ratio=0.058305 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=27136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.246196092e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86086 payload_rows_remote=78068 dup_rows=12218 dup_rows_remote=11428 dedup_ratio=0.124288 router_tokens_per_expert=[0, 529, 3386, 0, 0, 16, 0, 112, 0, 0, 0, 60, 72, 0, 0, 0, 0, 3116, 0, 3079, 2, 246, 6503, 7, 8, 0, 309, 76, 0, 1995, 0, 0, 0, 0, 0, 598, 1701, 1, 3908, 159, 0, 2196, 0, 7845, 60, 0, 151, 0, 0, 507, 7220, 0, 0, 382, 4228, 430, 0, 0, 0, 1, 248, 0, 1, 0] payload_rows_per_dst=[11858, 13012, 8018, 15794, 10536, 4694, 7732, 14442] dup_rows_per_dst=[2386, 258, 790, 4672, 2198, 120, 1296, 498] combine_rows_per_src=[11858, 13012, 8018, 15794, 10536, 4694, 7732, 14442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76504 payload_rows_remote=62732 dup_rows=21800 dup_rows_remote=15354 dedup_ratio=0.221761 router_tokens_per_expert=[88, 7445, 87, 0, 6693, 2546, 3556, 0, 0, 2, 0, 1069, 248, 0, 0, 3221, 321, 715, 540, 44, 74, 713, 5569, 0, 2, 1247, 319, 0, 0, 0, 185, 0, 603, 1, 2, 0, 1, 2, 244, 1472, 6155, 0, 0, 2, 865, 1, 82, 35, 0, 1, 2128, 5, 0, 0, 1, 706, 0, 0, 86, 0, 134, 1720, 3, 219] payload_rows_per_dst=[14930, 13772, 3966, 15298, 10148, 8958, 5606, 3826] dup_rows_per_dst=[8350, 6446, 848, 3098, 1614, 870, 76, 498] combine_rows_per_src=[14930, 13772, 3966, 15298, 10148, 8958, 5606, 3826] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82044 payload_rows_remote=74634 dup_rows=16260 dup_rows_remote=15042 dedup_ratio=0.165405 router_tokens_per_expert=[70, 7477, 70, 0, 6612, 2153, 3407, 0, 0, 0, 0, 896, 330, 0, 2, 3057, 337, 678, 521, 57, 12, 819, 5712, 0, 0, 1071, 335, 0, 0, 0, 174, 0, 837, 0, 0, 0, 0, 0, 232, 1677, 6277, 0, 0, 0, 836, 0, 71, 37, 0, 0, 2728, 15, 0, 0, 0, 628, 0, 0, 71, 0, 90, 1672, 0, 191] payload_rows_per_dst=[14222, 11504, 7410, 15250, 10220, 13218, 6610, 3610] dup_rows_per_dst=[3588, 4710, 1218, 2864, 2086, 1224, 132, 438] combine_rows_per_src=[14222, 11504, 7410, 15250, 10220, 13218, 6610, 3610] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83142 payload_rows_remote=68338 dup_rows=15162 dup_rows_remote=12426 dedup_ratio=0.154236 router_tokens_per_expert=[70, 7503, 76, 0, 6894, 1927, 3657, 0, 0, 0, 1, 1021, 220, 0, 0, 3149, 352, 694, 525, 53, 65, 749, 5799, 0, 2, 1300, 350, 2, 1, 0, 183, 0, 689, 0, 0, 3, 1, 1, 213, 1491, 6327, 0, 1, 0, 920, 10, 52, 26, 0, 0, 2297, 14, 0, 0, 1, 637, 0, 0, 75, 0, 80, 1515, 1, 205] payload_rows_per_dst=[4144, 7302, 13398, 3634, 10516, 13346, 15998, 14804] dup_rows_per_dst=[2, 1480, 2210, 42, 1594, 2192, 4906, 2736] combine_rows_per_src=[4144, 7302, 13398, 3634, 10516, 13346, 15998, 14804] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85570 payload_rows_remote=70308 dup_rows=12734 dup_rows_remote=9520 dedup_ratio=0.129537 router_tokens_per_expert=[85, 7462, 81, 1, 6477, 2291, 3860, 0, 0, 0, 0, 1041, 324, 0, 0, 2863, 372, 677, 577, 82, 170, 586, 5515, 0, 0, 1212, 372, 0, 0, 0, 192, 0, 601, 0, 0, 0, 0, 0, 222, 1667, 6121, 0, 0, 0, 928, 0, 79, 37, 0, 0, 2564, 2, 0, 0, 0, 666, 0, 0, 81, 0, 84, 1625, 0, 235] payload_rows_per_dst=[4926, 7014, 13940, 15262, 10954, 12958, 6320, 14196] dup_rows_per_dst=[0, 1442, 2018, 3214, 1746, 1372, 144, 2798] combine_rows_per_src=[4926, 7014, 13940, 15262, 10954, 12958, 6320, 14196] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=319766528 dispatch_meta_bytes=357984 combine_payload_bytes=319766528 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=320124512 ingress_bytes=423575552 link_bytes=423575552 token_bytes=639891040 weight_movement_bytes=103809024 total_nvlink_bytes=743700064 weight_to_token_ratio=0.162229 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=24832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.593880702e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=67844 payload_rows_remote=52062 dup_rows=30460 dup_rows_remote=11862 dedup_ratio=0.309855 router_tokens_per_expert=[80, 7324, 78, 0, 6922, 1970, 3962, 0, 0, 0, 0, 991, 248, 0, 0, 3046, 309, 669, 469, 83, 133, 682, 5386, 0, 0, 1417, 306, 0, 0, 0, 164, 0, 740, 0, 0, 0, 0, 0, 266, 1671, 6110, 0, 0, 0, 834, 0, 78, 30, 0, 0, 2405, 74, 0, 0, 0, 696, 0, 0, 78, 0, 91, 1626, 0, 214] payload_rows_per_dst=[15782, 13532, 3874, 14248, 8778, 1882, 6174, 3574] dup_rows_per_dst=[18598, 5810, 816, 3078, 1536, 2, 176, 444] combine_rows_per_src=[15782, 13532, 3874, 14248, 8778, 1882, 6174, 3574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85570 payload_rows_remote=70308 dup_rows=12734 dup_rows_remote=9520 dedup_ratio=0.129537 router_tokens_per_expert=[85, 7462, 81, 1, 6477, 2291, 3860, 0, 0, 0, 0, 1041, 324, 0, 0, 2863, 372, 677, 577, 82, 170, 586, 5515, 0, 0, 1212, 372, 0, 0, 0, 192, 0, 601, 0, 0, 0, 0, 0, 222, 1667, 6121, 0, 0, 0, 928, 0, 79, 37, 0, 0, 2564, 2, 0, 0, 0, 666, 0, 0, 81, 0, 84, 1625, 0, 235] payload_rows_per_dst=[4926, 7014, 13940, 15262, 10954, 12958, 6320, 14196] dup_rows_per_dst=[0, 1442, 2018, 3214, 1746, 1372, 144, 2798] combine_rows_per_src=[4926, 7014, 13940, 15262, 10954, 12958, 6320, 14196] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=256950272 dispatch_meta_bytes=312344 combine_payload_bytes=256950272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=257262616 ingress_bytes=291553280 link_bytes=291553280 token_bytes=514212888 weight_movement_bytes=34603008 total_nvlink_bytes=548815896 weight_to_token_ratio=0.067293 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=24320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.097091521e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82898 payload_rows_remote=66882 dup_rows=15406 dup_rows_remote=9852 dedup_ratio=0.156718 router_tokens_per_expert=[83, 7466, 85, 0, 6375, 2048, 3675, 0, 0, 0, 3, 979, 282, 0, 0, 2976, 351, 722, 436, 86, 73, 745, 5614, 0, 0, 1345, 350, 0, 0, 1, 187, 2, 677, 22, 0, 0, 1, 0, 237, 1817, 6039, 0, 0, 1, 929, 6, 88, 36, 0, 1, 2697, 22, 0, 0, 2, 597, 1, 0, 92, 0, 85, 1711, 4, 203] payload_rows_per_dst=[4432, 7022, 13928, 3708, 10880, 12864, 16016, 14048] dup_rows_per_dst=[0, 1458, 2126, 62, 1978, 1334, 5554, 2894] combine_rows_per_src=[4432, 7022, 13928, 3708, 10880, 12864, 16016, 14048] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83142 payload_rows_remote=68338 dup_rows=15162 dup_rows_remote=12426 dedup_ratio=0.154236 router_tokens_per_expert=[70, 7503, 76, 0, 6894, 1927, 3657, 0, 0, 0, 1, 1021, 220, 0, 0, 3149, 352, 694, 525, 53, 65, 749, 5799, 0, 2, 1300, 350, 2, 1, 0, 183, 0, 689, 0, 0, 3, 1, 1, 213, 1491, 6327, 0, 1, 0, 920, 10, 52, 26, 0, 0, 2297, 14, 0, 0, 1, 637, 0, 0, 75, 0, 80, 1515, 1, 205] payload_rows_per_dst=[4144, 7302, 13398, 3634, 10516, 13346, 15998, 14804] dup_rows_per_dst=[2, 1480, 2210, 42, 1594, 2192, 4906, 2736] combine_rows_per_src=[4144, 7302, 13398, 3634, 10516, 13346, 15998, 14804] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84328 payload_rows_remote=73682 dup_rows=13976 dup_rows_remote=11852 dedup_ratio=0.142171 router_tokens_per_expert=[72, 7386, 72, 0, 6738, 2163, 3771, 0, 0, 0, 0, 886, 246, 0, 0, 2896, 347, 624, 553, 67, 70, 775, 5690, 0, 0, 1335, 346, 0, 0, 0, 194, 0, 782, 0, 0, 0, 0, 0, 209, 1623, 6001, 0, 0, 0, 1005, 0, 72, 27, 0, 0, 2388, 28, 0, 0, 0, 718, 0, 0, 72, 0, 118, 1634, 0, 244] payload_rows_per_dst=[4614, 6720, 14028, 12670, 10646, 12828, 8224, 14598] dup_rows_per_dst=[0, 1336, 2224, 2614, 2124, 1382, 1282, 3014] combine_rows_per_src=[4614, 6720, 14028, 12670, 10646, 12828, 8224, 14598] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83192 payload_rows_remote=70574 dup_rows=15112 dup_rows_remote=13856 dedup_ratio=0.153727 router_tokens_per_expert=[81, 7379, 76, 0, 6567, 2201, 3754, 0, 0, 0, 0, 1034, 341, 0, 0, 3003, 337, 689, 542, 71, 48, 701, 5762, 0, 0, 1156, 336, 0, 0, 0, 139, 0, 928, 0, 0, 0, 0, 0, 137, 1923, 5876, 0, 0, 0, 973, 1, 63, 24, 0, 0, 2342, 120, 0, 0, 0, 702, 0, 0, 76, 0, 111, 1444, 0, 215] payload_rows_per_dst=[4716, 7110, 14140, 3246, 11112, 12618, 15980, 14270] dup_rows_per_dst=[0, 1646, 2160, 16, 2372, 1256, 5106, 2556] combine_rows_per_src=[4716, 7110, 14140, 3246, 11112, 12618, 15980, 14270] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=256950272 dispatch_meta_bytes=312344 combine_payload_bytes=256950272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=257262616 ingress_bytes=291553280 link_bytes=291553280 token_bytes=514212888 weight_movement_bytes=34603008 total_nvlink_bytes=548815896 weight_to_token_ratio=0.067293 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=24320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.097091521e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=287981568 dispatch_meta_bytes=319312 combine_payload_bytes=287981568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288300880 ingress_bytes=322584576 link_bytes=322584576 token_bytes=576282448 weight_movement_bytes=34603008 total_nvlink_bytes=610885456 weight_to_token_ratio=0.060045 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=34944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.213859789e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=305700864 dispatch_meta_bytes=358704 combine_payload_bytes=305700864 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=306059568 ingress_bytes=374906880 link_bytes=374906880 token_bytes=611760432 weight_movement_bytes=69206016 total_nvlink_bytes=680966448 weight_to_token_ratio=0.113126 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=29952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.410744407e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279912448 dispatch_meta_bytes=323056 combine_payload_bytes=279912448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280235504 ingress_bytes=314515456 link_bytes=314515456 token_bytes=560147952 weight_movement_bytes=34603008 total_nvlink_bytes=594750960 weight_to_token_ratio=0.061775 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=33152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.183496340e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=67844 payload_rows_remote=52062 dup_rows=30460 dup_rows_remote=11862 dedup_ratio=0.309855 router_tokens_per_expert=[80, 7324, 78, 0, 6922, 1970, 3962, 0, 0, 0, 0, 991, 248, 0, 0, 3046, 309, 669, 469, 83, 133, 682, 5386, 0, 0, 1417, 306, 0, 0, 0, 164, 0, 740, 0, 0, 0, 0, 0, 266, 1671, 6110, 0, 0, 0, 834, 0, 78, 30, 0, 0, 2405, 74, 0, 0, 0, 696, 0, 0, 78, 0, 91, 1626, 0, 214] payload_rows_per_dst=[15782, 13532, 3874, 14248, 8778, 1882, 6174, 3574] dup_rows_per_dst=[18598, 5810, 816, 3078, 1536, 2, 176, 444] combine_rows_per_src=[15782, 13532, 3874, 14248, 8778, 1882, 6174, 3574] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=287981568 dispatch_meta_bytes=319312 combine_payload_bytes=287981568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288300880 ingress_bytes=322584576 link_bytes=322584576 token_bytes=576282448 weight_movement_bytes=34603008 total_nvlink_bytes=610885456 weight_to_token_ratio=0.060045 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=34944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.213859789e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=213245952 dispatch_meta_bytes=255696 combine_payload_bytes=213245952 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=213501648 ingress_bytes=386260992 link_bytes=386260992 token_bytes=426747600 weight_movement_bytes=173015040 total_nvlink_bytes=599762640 weight_to_token_ratio=0.405427 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=20352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.453469016e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=273948672 dispatch_meta_bytes=306936 combine_payload_bytes=273948672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274255608 ingress_bytes=308551680 link_bytes=308551680 token_bytes=548204280 weight_movement_bytes=34603008 total_nvlink_bytes=582807288 weight_to_token_ratio=0.063121 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=24064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.161055131e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279912448 dispatch_meta_bytes=323056 combine_payload_bytes=279912448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280235504 ingress_bytes=314515456 link_bytes=314515456 token_bytes=560147952 weight_movement_bytes=34603008 total_nvlink_bytes=594750960 weight_to_token_ratio=0.061775 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=33152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.183496340e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=301801472 dispatch_meta_bytes=342136 combine_payload_bytes=301801472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=302143608 ingress_bytes=336404480 link_bytes=336404480 token_bytes=603945080 weight_movement_bytes=34603008 total_nvlink_bytes=638548088 weight_to_token_ratio=0.057295 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=28160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265862975e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82044 payload_rows_remote=74634 dup_rows=16260 dup_rows_remote=15042 dedup_ratio=0.165405 router_tokens_per_expert=[70, 7477, 70, 0, 6612, 2153, 3407, 0, 0, 0, 0, 896, 330, 0, 2, 3057, 337, 678, 521, 57, 12, 819, 5712, 0, 0, 1071, 335, 0, 0, 0, 174, 0, 837, 0, 0, 0, 0, 0, 232, 1677, 6277, 0, 0, 0, 836, 0, 71, 37, 0, 0, 2728, 15, 0, 0, 0, 628, 0, 0, 71, 0, 90, 1672, 0, 191] payload_rows_per_dst=[14222, 11504, 7410, 15250, 10220, 13218, 6610, 3610] dup_rows_per_dst=[3588, 4710, 1218, 2864, 2086, 1224, 132, 438] combine_rows_per_src=[14222, 11504, 7410, 15250, 10220, 13218, 6610, 3610] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=289071104 dispatch_meta_bytes=337720 combine_payload_bytes=289071104 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=289408824 ingress_bytes=358277120 link_bytes=358277120 token_bytes=578479928 weight_movement_bytes=69206016 total_nvlink_bytes=647685944 weight_to_token_ratio=0.119634 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=40576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.348167959e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=213245952 dispatch_meta_bytes=255696 combine_payload_bytes=213245952 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=213501648 ingress_bytes=386260992 link_bytes=386260992 token_bytes=426747600 weight_movement_bytes=173015040 total_nvlink_bytes=599762640 weight_to_token_ratio=0.405427 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=20352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.453469016e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83510 payload_rows_remote=67856 dup_rows=14794 dup_rows_remote=10148 dedup_ratio=0.150492 router_tokens_per_expert=[401, 0, 33, 0, 0, 0, 2, 466, 0, 1, 3374, 0, 2845, 7869, 0, 3680, 0, 2, 1, 361, 0, 1, 468, 117, 0, 39, 88, 0, 1, 86, 0, 0, 0, 0, 5617, 0, 2, 2, 0, 0, 0, 7508, 0, 0, 0, 133, 860, 728, 0, 1, 0, 94, 3915, 2283, 65, 1, 1, 1, 1, 0, 1, 1, 8103, 0] payload_rows_per_dst=[12928, 15654, 15290, 1378, 11238, 16378, 10638, 6] dup_rows_per_dst=[2984, 4646, 1626, 180, 4, 3270, 2080, 4] combine_rows_per_src=[12928, 15654, 15290, 1378, 11238, 16378, 10638, 6] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85492 payload_rows_remote=69272 dup_rows=12812 dup_rows_remote=7054 dedup_ratio=0.130330 router_tokens_per_expert=[363, 0, 46, 0, 0, 2, 0, 421, 0, 0, 4161, 1, 2521, 7824, 1, 3720, 0, 0, 0, 230, 0, 2, 483, 118, 1, 39, 73, 0, 1, 71, 1, 0, 3, 0, 5142, 0, 2, 1, 0, 0, 27, 7584, 0, 0, 0, 122, 751, 627, 0, 0, 4, 57, 4133, 2508, 0, 0, 3, 2, 0, 0, 0, 0, 8107, 0] payload_rows_per_dst=[9024, 8098, 1514, 15654, 12224, 15742, 7016, 16220] dup_rows_per_dst=[80, 0, 152, 366, 3114, 2480, 862, 5758] combine_rows_per_src=[9024, 8098, 1514, 15654, 12224, 15742, 7016, 16220] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89922 payload_rows_remote=74454 dup_rows=8382 dup_rows_remote=6814 dedup_ratio=0.085266 router_tokens_per_expert=[340, 0, 19, 0, 0, 0, 0, 446, 0, 0, 3584, 0, 2751, 7852, 0, 3535, 0, 0, 0, 304, 0, 0, 461, 141, 0, 17, 71, 0, 0, 70, 0, 0, 0, 0, 5525, 0, 0, 0, 0, 0, 1, 7612, 0, 0, 0, 128, 727, 585, 0, 0, 0, 61, 3840, 2887, 74, 0, 0, 0, 0, 0, 0, 0, 8121, 0] payload_rows_per_dst=[12878, 5502, 15468, 15706, 11050, 6698, 11206, 11414] dup_rows_per_dst=[2970, 0, 1568, 314, 0, 1012, 2518, 0] combine_rows_per_src=[12878, 5502, 15468, 15706, 11050, 6698, 11206, 11414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=305700864 dispatch_meta_bytes=358704 combine_payload_bytes=305700864 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=306059568 ingress_bytes=374906880 link_bytes=374906880 token_bytes=611760432 weight_movement_bytes=69206016 total_nvlink_bytes=680966448 weight_to_token_ratio=0.113126 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=29952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.410744407e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 1, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92370 payload_rows_remote=81770 dup_rows=5934 dup_rows_remote=5934 dedup_ratio=0.060364 router_tokens_per_expert=[380, 0, 16, 0, 0, 0, 0, 475, 0, 0, 3705, 0, 2751, 7844, 0, 4055, 0, 0, 0, 296, 0, 0, 459, 99, 0, 62, 72, 0, 0, 72, 0, 0, 0, 0, 5300, 0, 0, 0, 0, 0, 0, 7505, 0, 0, 0, 120, 765, 592, 0, 0, 0, 74, 3599, 2768, 26, 0, 1, 0, 0, 0, 0, 0, 8116, 0] payload_rows_per_dst=[9780, 11532, 15186, 15688, 10600, 2664, 10688, 16232] dup_rows_per_dst=[72, 1380, 1532, 412, 0, 290, 2246, 2] combine_rows_per_src=[9780, 11532, 15186, 15688, 10600, 2664, 10688, 16232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92292 payload_rows_remote=76660 dup_rows=6012 dup_rows_remote=5564 dedup_ratio=0.061157 router_tokens_per_expert=[376, 0, 54, 0, 0, 0, 0, 463, 0, 0, 3453, 0, 2534, 7816, 0, 4118, 0, 0, 0, 329, 0, 0, 518, 103, 0, 61, 82, 0, 0, 81, 0, 0, 0, 0, 5631, 0, 0, 0, 0, 0, 1, 7545, 0, 0, 0, 127, 817, 600, 0, 0, 1, 53, 3845, 2379, 52, 0, 2, 0, 0, 0, 0, 0, 8111, 0] payload_rows_per_dst=[10216, 10230, 15296, 15632, 11262, 2770, 10664, 16222] dup_rows_per_dst=[514, 1036, 1694, 448, 0, 320, 1996, 4] combine_rows_per_src=[10216, 10230, 15296, 15632, 11262, 2770, 10664, 16222] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277938176 dispatch_meta_bytes=312016 combine_payload_bytes=277938176 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=278250192 ingress_bytes=450953216 link_bytes=450953216 token_bytes=556188368 weight_movement_bytes=173015040 total_nvlink_bytes=729203408 weight_to_token_ratio=0.311073 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=18432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.696900647e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83152 payload_rows_remote=69166 dup_rows=15152 dup_rows_remote=11170 dedup_ratio=0.154134 router_tokens_per_expert=[337, 0, 39, 0, 0, 0, 0, 400, 0, 0, 4047, 0, 2741, 7882, 0, 4161, 0, 0, 0, 250, 0, 0, 435, 118, 0, 46, 78, 0, 0, 78, 0, 0, 0, 0, 5529, 0, 0, 0, 0, 0, 69, 7409, 0, 0, 0, 117, 726, 685, 0, 0, 0, 62, 3646, 2176, 12, 0, 0, 0, 0, 0, 0, 0, 8109, 0] payload_rows_per_dst=[13986, 16376, 15004, 248, 11058, 16384, 10096, 0] dup_rows_per_dst=[3982, 4870, 1420, 156, 0, 3028, 1696, 0] combine_rows_per_src=[13986, 16376, 15004, 248, 11058, 16384, 10096, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92370 payload_rows_remote=81770 dup_rows=5934 dup_rows_remote=5934 dedup_ratio=0.060364 router_tokens_per_expert=[380, 0, 16, 0, 0, 0, 0, 475, 0, 0, 3705, 0, 2751, 7844, 0, 4055, 0, 0, 0, 296, 0, 0, 459, 99, 0, 62, 72, 0, 0, 72, 0, 0, 0, 0, 5300, 0, 0, 0, 0, 0, 0, 7505, 0, 0, 0, 120, 765, 592, 0, 0, 0, 74, 3599, 2768, 26, 0, 1, 0, 0, 0, 0, 0, 8116, 0] payload_rows_per_dst=[9780, 11532, 15186, 15688, 10600, 2664, 10688, 16232] dup_rows_per_dst=[72, 1380, 1532, 412, 0, 290, 2246, 2] combine_rows_per_src=[9780, 11532, 15186, 15688, 10600, 2664, 10688, 16232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85492 payload_rows_remote=69272 dup_rows=12812 dup_rows_remote=7054 dedup_ratio=0.130330 router_tokens_per_expert=[363, 0, 46, 0, 0, 2, 0, 421, 0, 0, 4161, 1, 2521, 7824, 1, 3720, 0, 0, 0, 230, 0, 2, 483, 118, 1, 39, 73, 0, 1, 71, 1, 0, 3, 0, 5142, 0, 2, 1, 0, 0, 27, 7584, 0, 0, 0, 122, 751, 627, 0, 0, 4, 57, 4133, 2508, 0, 0, 3, 2, 0, 0, 0, 0, 8107, 0] payload_rows_per_dst=[9024, 8098, 1514, 15654, 12224, 15742, 7016, 16220] dup_rows_per_dst=[80, 0, 152, 366, 3114, 2480, 862, 5758] combine_rows_per_src=[9024, 8098, 1514, 15654, 12224, 15742, 7016, 16220] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89790 payload_rows_remote=79058 dup_rows=8514 dup_rows_remote=6520 dedup_ratio=0.086609 router_tokens_per_expert=[342, 0, 28, 0, 4, 0, 0, 460, 0, 1, 3648, 0, 2532, 7833, 1, 4015, 1, 4, 0, 290, 0, 3, 491, 140, 3, 47, 81, 2, 0, 82, 0, 1, 1, 1, 5570, 0, 1, 0, 3, 0, 12, 7541, 1, 0, 0, 134, 753, 660, 0, 0, 1, 48, 3606, 2688, 16, 4, 1, 0, 6, 0, 1, 4, 8089, 2] payload_rows_per_dst=[9648, 7300, 1680, 15676, 12834, 15726, 10732, 16194] dup_rows_per_dst=[50, 0, 178, 422, 3382, 2476, 1994, 12] combine_rows_per_src=[9648, 7300, 1680, 15676, 12834, 15726, 10732, 16194] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91418 payload_rows_remote=83504 dup_rows=6886 dup_rows_remote=5734 dedup_ratio=0.070048 router_tokens_per_expert=[364, 0, 22, 0, 0, 0, 0, 450, 0, 0, 3903, 0, 2546, 7854, 0, 3760, 0, 0, 0, 303, 0, 0, 476, 97, 0, 6, 76, 0, 0, 76, 0, 0, 0, 0, 5160, 0, 0, 0, 0, 0, 0, 7455, 0, 0, 0, 120, 811, 619, 0, 0, 0, 92, 3979, 2816, 49, 0, 2, 0, 0, 0, 0, 0, 8116, 0] payload_rows_per_dst=[9128, 10686, 9704, 15708, 10722, 7914, 11324, 16232] dup_rows_per_dst=[64, 994, 992, 316, 816, 1152, 2548, 4] combine_rows_per_src=[9128, 10686, 9704, 15708, 10722, 7914, 11324, 16232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277938176 dispatch_meta_bytes=312016 combine_payload_bytes=277938176 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=278250192 ingress_bytes=450953216 link_bytes=450953216 token_bytes=556188368 weight_movement_bytes=173015040 total_nvlink_bytes=729203408 weight_to_token_ratio=0.311073 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=18432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.696900647e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=313999360 dispatch_meta_bytes=328896 combine_payload_bytes=313999360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=314328256 ingress_bytes=348602368 link_bytes=348602368 token_bytes=628327616 weight_movement_bytes=34603008 total_nvlink_bytes=662930624 weight_to_token_ratio=0.055072 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=47616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.311762646e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=283738112 dispatch_meta_bytes=305304 combine_payload_bytes=283738112 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=284043416 ingress_bytes=352944128 link_bytes=352944128 token_bytes=567781528 weight_movement_bytes=69206016 total_nvlink_bytes=636987544 weight_to_token_ratio=0.121888 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=46336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.328100339e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=304963584 dispatch_meta_bytes=325072 combine_payload_bytes=304963584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=305288656 ingress_bytes=339566592 link_bytes=339566592 token_bytes=610252240 weight_movement_bytes=34603008 total_nvlink_bytes=644855248 weight_to_token_ratio=0.056703 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=42112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.277761748e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83152 payload_rows_remote=69166 dup_rows=15152 dup_rows_remote=11170 dedup_ratio=0.154134 router_tokens_per_expert=[337, 0, 39, 0, 0, 0, 0, 400, 0, 0, 4047, 0, 2741, 7882, 0, 4161, 0, 0, 0, 250, 0, 0, 435, 118, 0, 46, 78, 0, 0, 78, 0, 0, 0, 0, 5529, 0, 0, 0, 0, 0, 69, 7409, 0, 0, 0, 117, 726, 685, 0, 0, 0, 62, 3646, 2176, 12, 0, 0, 0, 0, 0, 0, 0, 8109, 0] payload_rows_per_dst=[13986, 16376, 15004, 248, 11058, 16384, 10096, 0] dup_rows_per_dst=[3982, 4870, 1420, 156, 0, 3028, 1696, 0] combine_rows_per_src=[13986, 16376, 15004, 248, 11058, 16384, 10096, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=283303936 dispatch_meta_bytes=321344 combine_payload_bytes=283303936 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=283625280 ingress_bytes=352509952 link_bytes=352509952 token_bytes=566929216 weight_movement_bytes=69206016 total_nvlink_bytes=636135232 weight_to_token_ratio=0.122072 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=31104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.326466570e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=334929920 dispatch_meta_bytes=350816 combine_payload_bytes=334929920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=335280736 ingress_bytes=369532928 link_bytes=369532928 token_bytes=670210656 weight_movement_bytes=34603008 total_nvlink_bytes=704813664 weight_to_token_ratio=0.051630 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=43520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390522658e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=283738112 dispatch_meta_bytes=305304 combine_payload_bytes=283738112 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=284043416 ingress_bytes=352944128 link_bytes=352944128 token_bytes=567781528 weight_movement_bytes=69206016 total_nvlink_bytes=636987544 weight_to_token_ratio=0.121888 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=46336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.328100339e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=323821568 dispatch_meta_bytes=342312 combine_payload_bytes=323821568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=324163880 ingress_bytes=393027584 link_bytes=393027584 token_bytes=647985448 weight_movement_bytes=69206016 total_nvlink_bytes=717191464 weight_to_token_ratio=0.106802 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=27776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.478931157e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=313999360 dispatch_meta_bytes=328896 combine_payload_bytes=313999360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=314328256 ingress_bytes=348602368 link_bytes=348602368 token_bytes=628327616 weight_movement_bytes=34603008 total_nvlink_bytes=662930624 weight_to_token_ratio=0.055072 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=47616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.311762646e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=342032384 dispatch_meta_bytes=356952 combine_payload_bytes=342032384 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=342389336 ingress_bytes=411238400 link_bytes=411238400 token_bytes=684421720 weight_movement_bytes=69206016 total_nvlink_bytes=753627736 weight_to_token_ratio=0.101116 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=18688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.547456992e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=283303936 dispatch_meta_bytes=321344 combine_payload_bytes=283303936 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=283625280 ingress_bytes=352509952 link_bytes=352509952 token_bytes=566929216 weight_movement_bytes=69206016 total_nvlink_bytes=636135232 weight_to_token_ratio=0.122072 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=31104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.326466570e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89922 payload_rows_remote=74454 dup_rows=8382 dup_rows_remote=6814 dedup_ratio=0.085266 router_tokens_per_expert=[340, 0, 19, 0, 0, 0, 0, 446, 0, 0, 3584, 0, 2751, 7852, 0, 3535, 0, 0, 0, 304, 0, 0, 461, 141, 0, 17, 71, 0, 0, 70, 0, 0, 0, 0, 5525, 0, 0, 0, 0, 0, 1, 7612, 0, 0, 0, 128, 727, 585, 0, 0, 0, 61, 3840, 2887, 74, 0, 0, 0, 0, 0, 0, 0, 8121, 0] payload_rows_per_dst=[12878, 5502, 15468, 15706, 11050, 6698, 11206, 11414] dup_rows_per_dst=[2970, 0, 1568, 314, 0, 1012, 2518, 0] combine_rows_per_src=[12878, 5502, 15468, 15706, 11050, 6698, 11206, 11414] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86088 payload_rows_remote=75154 dup_rows=12216 dup_rows_remote=8604 dedup_ratio=0.124268 router_tokens_per_expert=[248, 1, 3, 0, 249, 0, 0, 2, 0, 577, 2228, 1948, 2, 1, 320, 4, 712, 2204, 0, 0, 7745, 0, 0, 1669, 0, 0, 0, 7157, 1, 316, 4, 0, 0, 0, 0, 0, 1663, 506, 352, 0, 0, 1, 1, 0, 0, 0, 0, 2, 1, 2, 0, 1, 94, 7224, 6156, 1, 0, 0, 0, 1, 0, 0, 7755, 1] payload_rows_per_dst=[15496, 10934, 9316, 14322, 16180, 14454, 5382, 4] dup_rows_per_dst=[1000, 3612, 2520, 634, 4372, 2, 76, 0] combine_rows_per_src=[15496, 10934, 9316, 14322, 16180, 14454, 5382, 4] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=67452 payload_rows_remote=51574 dup_rows=30852 dup_rows_remote=16616 dedup_ratio=0.313843 router_tokens_per_expert=[243, 5, 0, 0, 243, 0, 0, 4, 1, 954, 1762, 2196, 2, 4, 370, 0, 580, 1981, 0, 0, 7854, 2, 0, 1624, 0, 0, 0, 7251, 1, 350, 12, 2, 1, 1, 0, 0, 1672, 418, 290, 0, 2, 4, 0, 0, 0, 8, 0, 2, 1, 3, 0, 0, 75, 7261, 6165, 2, 0, 0, 0, 0, 0, 1, 7805, 0] payload_rows_per_dst=[504, 9084, 9894, 11274, 4672, 28, 16118, 15878] dup_rows_per_dst=[486, 1494, 3166, 478, 92, 4, 10896, 14236] combine_rows_per_src=[504, 9084, 9894, 11274, 4672, 28, 16118, 15878] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89610 payload_rows_remote=81646 dup_rows=8694 dup_rows_remote=7236 dedup_ratio=0.088440 router_tokens_per_expert=[248, 6, 0, 0, 248, 0, 0, 7, 0, 940, 1930, 1795, 0, 0, 335, 0, 653, 2285, 0, 0, 7872, 0, 0, 1773, 0, 0, 0, 7225, 1, 335, 0, 0, 0, 0, 1, 0, 1617, 429, 295, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 75, 7291, 5974, 3, 0, 0, 0, 0, 0, 0, 7811, 0] payload_rows_per_dst=[15744, 8630, 7964, 14452, 16264, 14586, 11970, 0] dup_rows_per_dst=[1018, 1370, 1458, 670, 4042, 2, 134, 0] combine_rows_per_src=[15744, 8630, 7964, 14452, 16264, 14586, 11970, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=304963584 dispatch_meta_bytes=325072 combine_payload_bytes=304963584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=305288656 ingress_bytes=339566592 link_bytes=339566592 token_bytes=610252240 weight_movement_bytes=34603008 total_nvlink_bytes=644855248 weight_to_token_ratio=0.056703 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=42112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.277761748e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90256 payload_rows_remote=75804 dup_rows=8048 dup_rows_remote=7302 dedup_ratio=0.081868 router_tokens_per_expert=[245, 1, 0, 0, 245, 0, 0, 3, 0, 806, 1796, 1956, 0, 0, 394, 0, 670, 2310, 0, 0, 7807, 0, 0, 1560, 0, 1, 0, 7198, 0, 370, 28, 2, 0, 0, 2, 0, 1673, 430, 302, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 2, 84, 7260, 6183, 9, 0, 0, 0, 0, 0, 0, 7812, 0] payload_rows_per_dst=[15614, 8530, 7788, 14452, 14278, 14524, 12388, 2682] dup_rows_per_dst=[988, 1374, 1292, 746, 3478, 2, 168, 0] combine_rows_per_src=[15614, 8530, 7788, 14452, 14278, 14524, 12388, 2682] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=307830784 dispatch_meta_bytes=335032 combine_payload_bytes=307830784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=308165816 ingress_bytes=342433792 link_bytes=342433792 token_bytes=615996600 weight_movement_bytes=34603008 total_nvlink_bytes=650599608 weight_to_token_ratio=0.056174 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=16000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.288550791e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83610 payload_rows_remote=67984 dup_rows=14694 dup_rows_remote=13658 dedup_ratio=0.149475 router_tokens_per_expert=[255, 1, 0, 0, 255, 0, 0, 7, 0, 742, 2061, 1983, 0, 0, 315, 0, 654, 1929, 0, 0, 7813, 0, 0, 1763, 0, 1, 0, 7146, 1, 307, 28, 0, 0, 0, 1, 0, 1573, 472, 321, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 80, 7338, 6291, 12, 0, 0, 0, 0, 0, 0, 7799, 0] payload_rows_per_dst=[15626, 15078, 7460, 14352, 16228, 14676, 190, 0] dup_rows_per_dst=[1036, 7706, 1232, 614, 4104, 2, 0, 0] combine_rows_per_src=[15626, 15078, 7460, 14352, 16228, 14676, 190, 0] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93446 payload_rows_remote=88896 dup_rows=4858 dup_rows_remote=4680 dedup_ratio=0.049418 router_tokens_per_expert=[252, 0, 0, 0, 250, 0, 0, 3, 0, 713, 2204, 2121, 0, 0, 406, 5, 711, 1985, 0, 0, 7791, 0, 0, 1557, 0, 2, 0, 7120, 0, 348, 46, 0, 0, 0, 0, 0, 1494, 491, 379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 7303, 6098, 5, 0, 0, 0, 0, 0, 0, 7795, 0] payload_rows_per_dst=[15582, 9170, 7398, 14330, 4550, 14606, 12220, 15590] dup_rows_per_dst=[1010, 1728, 1108, 702, 178, 0, 132, 0] combine_rows_per_src=[15582, 9170, 7398, 14330, 4550, 14606, 12220, 15590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=67452 payload_rows_remote=51574 dup_rows=30852 dup_rows_remote=16616 dedup_ratio=0.313843 router_tokens_per_expert=[243, 5, 0, 0, 243, 0, 0, 4, 1, 954, 1762, 2196, 2, 4, 370, 0, 580, 1981, 0, 0, 7854, 2, 0, 1624, 0, 0, 0, 7251, 1, 350, 12, 2, 1, 1, 0, 0, 1672, 418, 290, 0, 2, 4, 0, 0, 0, 8, 0, 2, 1, 3, 0, 0, 75, 7261, 6165, 2, 0, 0, 0, 0, 0, 1, 7805, 0] payload_rows_per_dst=[504, 9084, 9894, 11274, 4672, 28, 16118, 15878] dup_rows_per_dst=[486, 1494, 3166, 478, 92, 4, 10896, 14236] combine_rows_per_src=[504, 9084, 9894, 11274, 4672, 28, 16118, 15878] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70782 payload_rows_remote=57470 dup_rows=27522 dup_rows_remote=24740 dedup_ratio=0.279968 router_tokens_per_expert=[236, 1, 0, 2, 236, 0, 1, 5, 0, 765, 1705, 2168, 0, 0, 354, 2, 679, 1972, 4, 0, 7864, 0, 0, 1856, 1, 0, 1, 7198, 5, 332, 0, 0, 1, 0, 2, 4, 1603, 418, 391, 0, 6, 0, 0, 1, 2, 0, 0, 0, 3, 6, 0, 4, 104, 7231, 6181, 3, 0, 4, 5, 1, 0, 1, 7793, 1] payload_rows_per_dst=[488, 8568, 16186, 676, 4670, 10986, 13312, 15896] dup_rows_per_dst=[474, 1420, 8564, 2, 168, 2, 2782, 14110] combine_rows_per_src=[488, 8568, 16186, 676, 4670, 10986, 13312, 15896] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90256 payload_rows_remote=75804 dup_rows=8048 dup_rows_remote=7302 dedup_ratio=0.081868 router_tokens_per_expert=[245, 1, 0, 0, 245, 0, 0, 3, 0, 806, 1796, 1956, 0, 0, 394, 0, 670, 2310, 0, 0, 7807, 0, 0, 1560, 0, 1, 0, 7198, 0, 370, 28, 2, 0, 0, 2, 0, 1673, 430, 302, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 2, 84, 7260, 6183, 9, 0, 0, 0, 0, 0, 0, 7812, 0] payload_rows_per_dst=[15614, 8530, 7788, 14452, 14278, 14524, 12388, 2682] dup_rows_per_dst=[988, 1374, 1292, 746, 3478, 2, 168, 0] combine_rows_per_src=[15614, 8530, 7788, 14452, 14278, 14524, 12388, 2682] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87948 payload_rows_remote=73518 dup_rows=10356 dup_rows_remote=10356 dedup_ratio=0.105347 router_tokens_per_expert=[255, 0, 0, 0, 255, 0, 0, 6, 0, 710, 1914, 1832, 0, 0, 345, 2, 753, 2161, 0, 0, 7774, 0, 0, 1894, 0, 0, 0, 7145, 0, 338, 0, 0, 0, 0, 0, 0, 1646, 497, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 78, 7215, 6184, 1, 0, 0, 0, 0, 0, 0, 7771, 0] payload_rows_per_dst=[12322, 8250, 9900, 10148, 4852, 14430, 12426, 15620] dup_rows_per_dst=[916, 1356, 3058, 452, 176, 0, 110, 4288] combine_rows_per_src=[12322, 8250, 9900, 10148, 4852, 14430, 12426, 15620] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=307830784 dispatch_meta_bytes=335032 combine_payload_bytes=307830784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=308165816 ingress_bytes=342433792 link_bytes=342433792 token_bytes=615996600 weight_movement_bytes=34603008 total_nvlink_bytes=650599608 weight_to_token_ratio=0.056174 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=16000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.288550791e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=310493184 dispatch_meta_bytes=332424 combine_payload_bytes=310493184 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=310825608 ingress_bytes=379699200 link_bytes=379699200 token_bytes=621318792 weight_movement_bytes=69206016 total_nvlink_bytes=690524808 weight_to_token_ratio=0.111386 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=40832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.428777521e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=211247104 dispatch_meta_bytes=272760 combine_payload_bytes=211247104 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=211519864 ingress_bytes=280453120 link_bytes=280453120 token_bytes=422766968 weight_movement_bytes=69206016 total_nvlink_bytes=491972984 weight_to_token_ratio=0.163698 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.055322512e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=334422016 dispatch_meta_bytes=355528 combine_payload_bytes=334422016 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=334777544 ingress_bytes=438231040 link_bytes=438231040 token_bytes=669199560 weight_movement_bytes=103809024 total_nvlink_bytes=773008584 weight_to_token_ratio=0.155124 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=16896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.649028123e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83610 payload_rows_remote=67984 dup_rows=14694 dup_rows_remote=13658 dedup_ratio=0.149475 router_tokens_per_expert=[255, 1, 0, 0, 255, 0, 0, 7, 0, 742, 2061, 1983, 0, 0, 315, 0, 654, 1929, 0, 0, 7813, 0, 0, 1763, 0, 1, 0, 7146, 1, 307, 28, 0, 0, 0, 1, 0, 1573, 472, 321, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 80, 7338, 6291, 12, 0, 0, 0, 0, 0, 0, 7799, 0] payload_rows_per_dst=[15626, 15078, 7460, 14352, 16228, 14676, 190, 0] dup_rows_per_dst=[1036, 7706, 1232, 614, 4104, 2, 0, 0] combine_rows_per_src=[15626, 15078, 7460, 14352, 16228, 14676, 190, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278462464 dispatch_meta_bytes=326568 combine_payload_bytes=278462464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278789032 ingress_bytes=313065472 link_bytes=313065472 token_bytes=557251496 weight_movement_bytes=34603008 total_nvlink_bytes=591854504 weight_to_token_ratio=0.062096 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=45184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.178040167e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=364118016 dispatch_meta_bytes=374304 combine_payload_bytes=364118016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=364492320 ingress_bytes=398721024 link_bytes=398721024 token_bytes=728610336 weight_movement_bytes=34603008 total_nvlink_bytes=763213344 weight_to_token_ratio=0.047492 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=29952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.500355114e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=211247104 dispatch_meta_bytes=272760 combine_payload_bytes=211247104 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=211519864 ingress_bytes=280453120 link_bytes=280453120 token_bytes=422766968 weight_movement_bytes=69206016 total_nvlink_bytes=491972984 weight_to_token_ratio=0.163698 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.055322512e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=235397120 dispatch_meta_bytes=328840 combine_payload_bytes=235397120 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=235725960 ingress_bytes=339206144 link_bytes=339206144 token_bytes=471123080 weight_movement_bytes=103809024 total_nvlink_bytes=574932104 weight_to_token_ratio=0.220344 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=39424 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.276405412e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=310493184 dispatch_meta_bytes=332424 combine_payload_bytes=310493184 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=310825608 ingress_bytes=379699200 link_bytes=379699200 token_bytes=621318792 weight_movement_bytes=69206016 total_nvlink_bytes=690524808 weight_to_token_ratio=0.111386 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=40832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.428777521e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=301129728 dispatch_meta_bytes=335496 combine_payload_bytes=301129728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=301465224 ingress_bytes=335732736 link_bytes=335732736 token_bytes=602594952 weight_movement_bytes=34603008 total_nvlink_bytes=637197960 weight_to_token_ratio=0.057423 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263335257e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278462464 dispatch_meta_bytes=326568 combine_payload_bytes=278462464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278789032 ingress_bytes=313065472 link_bytes=313065472 token_bytes=557251496 weight_movement_bytes=34603008 total_nvlink_bytes=591854504 weight_to_token_ratio=0.062096 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=45184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.178040167e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89610 payload_rows_remote=81646 dup_rows=8694 dup_rows_remote=7236 dedup_ratio=0.088440 router_tokens_per_expert=[248, 6, 0, 0, 248, 0, 0, 7, 0, 940, 1930, 1795, 0, 0, 335, 0, 653, 2285, 0, 0, 7872, 0, 0, 1773, 0, 0, 0, 7225, 1, 335, 0, 0, 0, 0, 1, 0, 1617, 429, 295, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 75, 7291, 5974, 3, 0, 0, 0, 0, 0, 0, 7811, 0] payload_rows_per_dst=[15744, 8630, 7964, 14452, 16264, 14586, 11970, 0] dup_rows_per_dst=[1018, 1370, 1458, 670, 4042, 2, 134, 0] combine_rows_per_src=[15744, 8630, 7964, 14452, 16264, 14586, 11970, 0] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89076 payload_rows_remote=72788 dup_rows=9228 dup_rows_remote=9226 dedup_ratio=0.093872 router_tokens_per_expert=[7203, 0, 3, 3150, 51, 3, 194, 0, 0, 1, 0, 0, 2, 0, 4, 8138, 0, 6272, 0, 0, 1, 0, 2, 0, 0, 4340, 0, 0, 0, 85, 1, 1, 1658, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 7202, 243, 0, 0, 0, 0, 940, 0, 0, 57, 2, 0, 704, 334, 199, 468, 7890] payload_rows_per_dst=[15646, 16288, 12548, 8854, 16040, 14408, 2364, 2928] dup_rows_per_dst=[5562, 2, 2, 0, 3060, 0, 2, 600] combine_rows_per_src=[15646, 16288, 12548, 8854, 16040, 14408, 2364, 2928] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72530 payload_rows_remote=56166 dup_rows=25774 dup_rows_remote=9488 dedup_ratio=0.262187 router_tokens_per_expert=[7328, 0, 1, 3513, 20, 0, 227, 0, 0, 0, 1, 2, 0, 0, 9, 7955, 0, 6377, 0, 0, 0, 0, 3, 5, 0, 3896, 0, 0, 0, 69, 0, 0, 2087, 0, 0, 1, 0, 0, 4, 0, 0, 2, 0, 0, 8, 3, 4, 7236, 280, 3, 0, 0, 0, 518, 0, 0, 63, 2, 0, 799, 324, 153, 340, 7919] payload_rows_per_dst=[7384, 22, 12762, 15910, 4184, 1042, 14862, 16364] dup_rows_per_dst=[138, 2, 8, 7930, 0, 14, 1396, 16286] combine_rows_per_src=[7384, 22, 12762, 15910, 4184, 1042, 14862, 16364] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91554 payload_rows_remote=79034 dup_rows=6750 dup_rows_remote=6750 dedup_ratio=0.068665 router_tokens_per_expert=[7347, 0, 0, 3685, 73, 0, 175, 0, 0, 0, 0, 1, 0, 0, 0, 8171, 0, 6260, 0, 0, 0, 0, 0, 0, 0, 3974, 0, 0, 0, 67, 0, 0, 1502, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 7377, 38, 0, 0, 0, 0, 992, 0, 0, 71, 0, 0, 656, 261, 145, 442, 7912] payload_rows_per_dst=[10690, 16342, 12520, 8082, 15964, 14754, 10476, 2726] dup_rows_per_dst=[2362, 2, 0, 0, 2870, 0, 1092, 424] combine_rows_per_src=[10690, 16342, 12520, 8082, 15964, 14754, 10476, 2726] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84440 payload_rows_remote=69782 dup_rows=13864 dup_rows_remote=11710 dedup_ratio=0.141032 router_tokens_per_expert=[7203, 0, 0, 3567, 54, 2, 200, 0, 2, 0, 4, 2, 9, 0, 4, 7892, 0, 6026, 0, 0, 1, 3, 9, 1, 1, 4141, 0, 5, 0, 89, 1, 0, 1800, 2, 2, 0, 0, 1, 1, 7, 2, 1, 0, 6, 0, 2, 0, 7255, 346, 0, 0, 0, 0, 856, 1, 0, 52, 1, 0, 727, 291, 203, 442, 7938] payload_rows_per_dst=[7562, 144, 12076, 15732, 3618, 14528, 14658, 16122] dup_rows_per_dst=[84, 18, 4, 8406, 8, 4, 2154, 3186] combine_rows_per_src=[7562, 144, 12076, 15732, 3618, 14528, 14658, 16122] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93336 payload_rows_remote=84764 dup_rows=4968 dup_rows_remote=3534 dedup_ratio=0.050537 router_tokens_per_expert=[7336, 0, 0, 3604, 47, 0, 208, 0, 0, 0, 0, 0, 0, 0, 4, 7948, 0, 6173, 0, 0, 0, 0, 0, 0, 0, 4118, 0, 0, 0, 72, 0, 0, 2074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7276, 277, 0, 0, 0, 0, 595, 0, 0, 78, 0, 0, 655, 338, 139, 337, 7873] payload_rows_per_dst=[7600, 15896, 12346, 8380, 8572, 14552, 14870, 11120] dup_rows_per_dst=[118, 8, 0, 0, 1434, 0, 1546, 1862] combine_rows_per_src=[7600, 15896, 12346, 8380, 8572, 14552, 14870, 11120] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92884 payload_rows_remote=84972 dup_rows=5420 dup_rows_remote=5420 dedup_ratio=0.055135 router_tokens_per_expert=[7365, 0, 0, 3805, 103, 0, 205, 0, 0, 0, 0, 0, 0, 0, 3, 8131, 0, 6116, 0, 0, 0, 0, 0, 0, 0, 3877, 0, 0, 0, 79, 0, 0, 1623, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7264, 197, 0, 1, 0, 0, 753, 0, 0, 67, 0, 0, 721, 289, 210, 419, 7923] payload_rows_per_dst=[8108, 16262, 12232, 7912, 15990, 14528, 14956, 2896] dup_rows_per_dst=[118, 6, 0, 0, 3104, 0, 1676, 516] combine_rows_per_src=[8108, 16262, 12232, 7912, 15990, 14528, 14956, 2896] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93216 payload_rows_remote=78854 dup_rows=5088 dup_rows_remote=5088 dedup_ratio=0.051758 router_tokens_per_expert=[7193, 0, 0, 3296, 125, 0, 219, 0, 0, 0, 0, 1, 0, 0, 0, 8069, 0, 6250, 0, 0, 0, 0, 0, 0, 0, 4100, 0, 0, 0, 74, 0, 0, 1902, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7181, 214, 0, 0, 0, 0, 657, 0, 0, 55, 0, 0, 924, 340, 224, 404, 7923] payload_rows_per_dst=[7252, 16138, 12500, 8348, 3806, 14362, 14666, 16144] dup_rows_per_dst=[28, 2, 0, 0, 0, 0, 1462, 3596] combine_rows_per_src=[7252, 16138, 12500, 8348, 3806, 14362, 14666, 16144] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=334422016 dispatch_meta_bytes=355528 combine_payload_bytes=334422016 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=334777544 ingress_bytes=438231040 link_bytes=438231040 token_bytes=669199560 weight_movement_bytes=103809024 total_nvlink_bytes=773008584 weight_to_token_ratio=0.155124 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=16896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.649028123e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=298139648 dispatch_meta_bytes=328056 combine_payload_bytes=298139648 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=298467704 ingress_bytes=367345664 link_bytes=367345664 token_bytes=596607352 weight_movement_bytes=69206016 total_nvlink_bytes=665813368 weight_to_token_ratio=0.115999 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=48640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.382292160e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=230055936 dispatch_meta_bytes=262616 combine_payload_bytes=230055936 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=230318552 ingress_bytes=299261952 link_bytes=299261952 token_bytes=460374488 weight_movement_bytes=69206016 total_nvlink_bytes=529580504 weight_to_token_ratio=0.150325 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=28800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126098633e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82740 payload_rows_remote=67036 dup_rows=15564 dup_rows_remote=5616 dedup_ratio=0.158325 router_tokens_per_expert=[7178, 0, 0, 3438, 41, 0, 193, 0, 0, 0, 0, 0, 0, 0, 0, 8016, 0, 5941, 0, 0, 0, 0, 0, 0, 0, 4298, 0, 0, 0, 77, 0, 0, 1886, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7314, 260, 0, 0, 0, 0, 802, 0, 0, 57, 0, 0, 792, 287, 175, 456, 7940] payload_rows_per_dst=[15704, 16086, 10908, 8750, 16048, 10112, 2124, 3008] dup_rows_per_dst=[9948, 1086, 398, 0, 3606, 0, 0, 526] combine_rows_per_src=[15704, 16086, 10908, 8750, 16048, 10112, 2124, 3008] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=323723264 dispatch_meta_bytes=343136 combine_payload_bytes=323723264 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=324066400 ingress_bytes=392929280 link_bytes=392929280 token_bytes=647789664 weight_movement_bytes=69206016 total_nvlink_bytes=716995680 weight_to_token_ratio=0.106834 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=48896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.478561247e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72530 payload_rows_remote=56166 dup_rows=25774 dup_rows_remote=9488 dedup_ratio=0.262187 router_tokens_per_expert=[7328, 0, 1, 3513, 20, 0, 227, 0, 0, 0, 1, 2, 0, 0, 9, 7955, 0, 6377, 0, 0, 0, 0, 3, 5, 0, 3896, 0, 0, 0, 69, 0, 0, 2087, 0, 0, 1, 0, 0, 4, 0, 0, 2, 0, 0, 8, 3, 4, 7236, 280, 3, 0, 0, 0, 518, 0, 0, 63, 2, 0, 799, 324, 153, 340, 7919] payload_rows_per_dst=[7384, 22, 12762, 15910, 4184, 1042, 14862, 16364] dup_rows_per_dst=[138, 2, 8, 7930, 0, 14, 1396, 16286] combine_rows_per_src=[7384, 22, 12762, 15910, 4184, 1042, 14862, 16364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93336 payload_rows_remote=84764 dup_rows=4968 dup_rows_remote=3534 dedup_ratio=0.050537 router_tokens_per_expert=[7336, 0, 0, 3604, 47, 0, 208, 0, 0, 0, 0, 0, 0, 0, 4, 7948, 0, 6173, 0, 0, 0, 0, 0, 0, 0, 4118, 0, 0, 0, 72, 0, 0, 2074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7276, 277, 0, 0, 0, 0, 595, 0, 0, 78, 0, 0, 655, 338, 139, 337, 7873] payload_rows_per_dst=[7600, 15896, 12346, 8380, 8572, 14552, 14870, 11120] dup_rows_per_dst=[118, 8, 0, 0, 1434, 0, 1546, 1862] combine_rows_per_src=[7600, 15896, 12346, 8380, 8572, 14552, 14870, 11120] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93216 payload_rows_remote=78854 dup_rows=5088 dup_rows_remote=5088 dedup_ratio=0.051758 router_tokens_per_expert=[7193, 0, 0, 3296, 125, 0, 219, 0, 0, 0, 0, 1, 0, 0, 0, 8069, 0, 6250, 0, 0, 0, 0, 0, 0, 0, 4100, 0, 0, 0, 74, 0, 0, 1902, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7181, 214, 0, 0, 0, 0, 657, 0, 0, 55, 0, 0, 924, 340, 224, 404, 7923] payload_rows_per_dst=[7252, 16138, 12500, 8348, 3806, 14362, 14666, 16144] dup_rows_per_dst=[28, 2, 0, 0, 0, 0, 1462, 3596] combine_rows_per_src=[7252, 16138, 12500, 8348, 3806, 14362, 14666, 16144] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84440 payload_rows_remote=69782 dup_rows=13864 dup_rows_remote=11710 dedup_ratio=0.141032 router_tokens_per_expert=[7203, 0, 0, 3567, 54, 2, 200, 0, 2, 0, 4, 2, 9, 0, 4, 7892, 0, 6026, 0, 0, 1, 3, 9, 1, 1, 4141, 0, 5, 0, 89, 1, 0, 1800, 2, 2, 0, 0, 1, 1, 7, 2, 1, 0, 6, 0, 2, 0, 7255, 346, 0, 0, 0, 0, 856, 1, 0, 52, 1, 0, 727, 291, 203, 442, 7938] payload_rows_per_dst=[7562, 144, 12076, 15732, 3618, 14528, 14658, 16122] dup_rows_per_dst=[84, 18, 4, 8406, 8, 4, 2154, 3186] combine_rows_per_src=[7562, 144, 12076, 15732, 3618, 14528, 14658, 16122] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=348045312 dispatch_meta_bytes=361568 combine_payload_bytes=348045312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=348406880 ingress_bytes=382648320 link_bytes=382648320 token_bytes=696452192 weight_movement_bytes=34603008 total_nvlink_bytes=731055200 weight_to_token_ratio=0.049685 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.439874822e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=285827072 dispatch_meta_bytes=325968 combine_payload_bytes=285827072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286153040 ingress_bytes=320430080 link_bytes=320430080 token_bytes=571980112 weight_movement_bytes=34603008 total_nvlink_bytes=606583120 weight_to_token_ratio=0.060497 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=41216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.205752594e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=274579456 dispatch_meta_bytes=290608 combine_payload_bytes=274579456 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=274870064 ingress_bytes=343785472 link_bytes=343785472 token_bytes=549449520 weight_movement_bytes=69206016 total_nvlink_bytes=618655536 weight_to_token_ratio=0.125955 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=28160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.293637054e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=230055936 dispatch_meta_bytes=262616 combine_payload_bytes=230055936 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=230318552 ingress_bytes=299261952 link_bytes=299261952 token_bytes=460374488 weight_movement_bytes=69206016 total_nvlink_bytes=529580504 weight_to_token_ratio=0.150325 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=28800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126098633e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=347193344 dispatch_meta_bytes=353192 combine_payload_bytes=347193344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=347546536 ingress_bytes=381796352 link_bytes=381796352 token_bytes=694739880 weight_movement_bytes=34603008 total_nvlink_bytes=729342888 weight_to_token_ratio=0.049807 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=34688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.436668935e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=348045312 dispatch_meta_bytes=361568 combine_payload_bytes=348045312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=348406880 ingress_bytes=382648320 link_bytes=382648320 token_bytes=696452192 weight_movement_bytes=34603008 total_nvlink_bytes=731055200 weight_to_token_ratio=0.049685 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.439874822e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=322985984 dispatch_meta_bytes=335768 combine_payload_bytes=322985984 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=323321752 ingress_bytes=426795008 link_bytes=426795008 token_bytes=646307736 weight_movement_bytes=103809024 total_nvlink_bytes=750116760 weight_to_token_ratio=0.160619 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.605995255e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=274579456 dispatch_meta_bytes=290608 combine_payload_bytes=274579456 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=274870064 ingress_bytes=343785472 link_bytes=343785472 token_bytes=549449520 weight_movement_bytes=69206016 total_nvlink_bytes=618655536 weight_to_token_ratio=0.125955 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=28160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.293637054e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72840 payload_rows_remote=56460 dup_rows=25464 dup_rows_remote=25194 dedup_ratio=0.259033 router_tokens_per_expert=[0, 7451, 0, 0, 0, 6, 0, 0, 0, 2, 1, 1, 1, 555, 0, 0, 0, 0, 0, 8186, 0, 0, 160, 8182, 0, 253, 0, 0, 0, 0, 0, 2501, 1, 0, 81, 0, 1, 5195, 7560, 0, 71, 0, 3, 3, 60, 0, 0, 1, 8051, 2, 0, 682, 0, 2, 0, 1, 0, 1, 0, 75, 56, 1, 1, 5] payload_rows_per_dst=[12, 1114, 16376, 5022, 2580, 15126, 16230, 16380] dup_rows_per_dst=[0, 6, 12758, 486, 38, 270, 11636, 270] combine_rows_per_src=[12, 1114, 16376, 5022, 2580, 15126, 16230, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91102 payload_rows_remote=88650 dup_rows=7202 dup_rows_remote=7166 dedup_ratio=0.073263 router_tokens_per_expert=[0, 7481, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 578, 0, 0, 0, 0, 0, 8192, 0, 0, 165, 8190, 0, 268, 0, 0, 0, 0, 0, 2474, 0, 0, 71, 0, 0, 5071, 7556, 0, 70, 0, 0, 0, 101, 0, 0, 0, 8044, 0, 0, 769, 0, 0, 0, 0, 0, 0, 0, 72, 46, 0, 0, 0] payload_rows_per_dst=[14962, 16384, 2452, 16100, 10284, 15112, 1538, 14270] dup_rows_per_dst=[8, 1152, 36, 5472, 0, 342, 0, 192] combine_rows_per_src=[14962, 16384, 2452, 16100, 10284, 15112, 1538, 14270] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=285827072 dispatch_meta_bytes=325968 combine_payload_bytes=285827072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286153040 ingress_bytes=320430080 link_bytes=320430080 token_bytes=571980112 weight_movement_bytes=34603008 total_nvlink_bytes=606583120 weight_to_token_ratio=0.060497 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=41216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.205752594e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84384 payload_rows_remote=68006 dup_rows=13920 dup_rows_remote=12894 dedup_ratio=0.141602 router_tokens_per_expert=[1, 7375, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 513, 0, 0, 0, 0, 0, 8188, 0, 2, 224, 8184, 0, 259, 1, 1, 0, 0, 0, 2470, 0, 0, 86, 2, 0, 5434, 7516, 0, 90, 0, 1, 0, 70, 0, 0, 2, 8007, 0, 0, 519, 0, 3, 0, 1, 0, 2, 0, 128, 68, 0, 0, 0] payload_rows_per_dst=[14752, 16378, 16380, 16018, 14292, 5124, 1044, 396] dup_rows_per_dst=[0, 1026, 448, 5458, 6842, 144, 2, 0] combine_rows_per_src=[14752, 16378, 16380, 16018, 14292, 5124, 1044, 396] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85430 payload_rows_remote=69270 dup_rows=12874 dup_rows_remote=7690 dedup_ratio=0.130961 router_tokens_per_expert=[0, 7415, 0, 0, 0, 0, 0, 0, 0, 10, 7, 0, 0, 559, 0, 1, 0, 0, 0, 8181, 0, 1, 181, 8170, 0, 258, 2, 2, 0, 0, 0, 2463, 0, 0, 88, 3, 1, 5144, 7542, 4, 81, 2, 8, 7, 109, 3, 0, 1, 8058, 0, 0, 705, 0, 0, 0, 6, 0, 1, 0, 70, 61, 0, 7, 1] payload_rows_per_dst=[8658, 1140, 16344, 4974, 6670, 15106, 16160, 16378] dup_rows_per_dst=[0, 14, 6532, 476, 4, 400, 5184, 264] combine_rows_per_src=[8658, 1140, 16344, 4974, 6670, 15106, 16160, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94614 payload_rows_remote=84182 dup_rows=3690 dup_rows_remote=3690 dedup_ratio=0.037537 router_tokens_per_expert=[0, 7501, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 577, 0, 0, 0, 0, 0, 8192, 0, 0, 175, 8186, 0, 260, 0, 6, 0, 0, 0, 2404, 0, 0, 72, 0, 0, 5144, 7613, 0, 72, 0, 0, 0, 90, 0, 0, 0, 8001, 0, 0, 723, 0, 0, 0, 0, 0, 1, 0, 110, 23, 0, 0, 0] payload_rows_per_dst=[15002, 16378, 350, 4836, 10432, 15226, 16006, 16384] dup_rows_per_dst=[4, 1148, 0, 504, 0, 324, 1442, 268] combine_rows_per_src=[15002, 16378, 350, 4836, 10432, 15226, 16006, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91554 payload_rows_remote=79034 dup_rows=6750 dup_rows_remote=6750 dedup_ratio=0.068665 router_tokens_per_expert=[7347, 0, 0, 3685, 73, 0, 175, 0, 0, 0, 0, 1, 0, 0, 0, 8171, 0, 6260, 0, 0, 0, 0, 0, 0, 0, 3974, 0, 0, 0, 67, 0, 0, 1502, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 7377, 38, 0, 0, 0, 0, 992, 0, 0, 71, 0, 0, 656, 261, 145, 442, 7912] payload_rows_per_dst=[10690, 16342, 12520, 8082, 15964, 14754, 10476, 2726] dup_rows_per_dst=[2362, 2, 0, 0, 2870, 0, 1092, 424] combine_rows_per_src=[10690, 16342, 12520, 8082, 15964, 14754, 10476, 2726] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93140 payload_rows_remote=83340 dup_rows=5164 dup_rows_remote=2524 dedup_ratio=0.052531 router_tokens_per_expert=[0, 7452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 585, 0, 0, 0, 0, 0, 8192, 0, 0, 216, 8192, 0, 254, 0, 0, 0, 0, 0, 2327, 0, 0, 81, 0, 0, 5199, 7584, 0, 81, 0, 0, 0, 97, 0, 0, 0, 8047, 0, 0, 740, 0, 0, 0, 0, 0, 2, 0, 72, 31, 0, 0, 0] payload_rows_per_dst=[14904, 16384, 432, 9800, 10560, 15168, 9508, 16384] dup_rows_per_dst=[0, 1170, 0, 2640, 0, 356, 788, 210] combine_rows_per_src=[14904, 16384, 432, 9800, 10560, 15168, 9508, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=231260160 dispatch_meta_bytes=326616 combine_payload_bytes=231260160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=231586776 ingress_bytes=265863168 link_bytes=265863168 token_bytes=462846936 weight_movement_bytes=34603008 total_nvlink_bytes=497449944 weight_to_token_ratio=0.074761 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=48128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.000421697e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95266 payload_rows_remote=80230 dup_rows=3038 dup_rows_remote=2782 dedup_ratio=0.030904 router_tokens_per_expert=[0, 7512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 0, 2, 0, 0, 0, 8192, 0, 0, 224, 8192, 0, 274, 0, 0, 0, 0, 0, 2312, 0, 0, 76, 0, 0, 5391, 7518, 0, 76, 0, 0, 0, 52, 0, 0, 0, 8033, 0, 0, 650, 0, 0, 0, 0, 0, 1, 0, 103, 19, 0, 0, 0] payload_rows_per_dst=[15024, 8154, 9006, 4662, 10934, 15036, 16066, 16384] dup_rows_per_dst=[0, 516, 210, 510, 0, 256, 1300, 246] combine_rows_per_src=[15024, 8154, 9006, 4662, 10934, 15036, 16066, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=278552576 dispatch_meta_bytes=323600 combine_payload_bytes=278552576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278876176 ingress_bytes=313155584 link_bytes=313155584 token_bytes=557428752 weight_movement_bytes=34603008 total_nvlink_bytes=592031760 weight_to_token_ratio=0.062076 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=44800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.178379251e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93140 payload_rows_remote=83340 dup_rows=5164 dup_rows_remote=2524 dedup_ratio=0.052531 router_tokens_per_expert=[0, 7452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 585, 0, 0, 0, 0, 0, 8192, 0, 0, 216, 8192, 0, 254, 0, 0, 0, 0, 0, 2327, 0, 0, 81, 0, 0, 5199, 7584, 0, 81, 0, 0, 0, 97, 0, 0, 0, 8047, 0, 0, 740, 0, 0, 0, 0, 0, 2, 0, 72, 31, 0, 0, 0] payload_rows_per_dst=[14904, 16384, 432, 9800, 10560, 15168, 9508, 16384] dup_rows_per_dst=[0, 1170, 0, 2640, 0, 356, 788, 210] combine_rows_per_src=[14904, 16384, 432, 9800, 10560, 15168, 9508, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=323723264 dispatch_meta_bytes=343136 combine_payload_bytes=323723264 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=324066400 ingress_bytes=392929280 link_bytes=392929280 token_bytes=647789664 weight_movement_bytes=69206016 total_nvlink_bytes=716995680 weight_to_token_ratio=0.106834 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=48896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.478561247e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81718 payload_rows_remote=66744 dup_rows=16586 dup_rows_remote=16582 dedup_ratio=0.168722 router_tokens_per_expert=[0, 7487, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 484, 0, 0, 0, 0, 0, 8192, 0, 0, 298, 8186, 0, 286, 0, 0, 0, 0, 0, 2337, 0, 0, 78, 0, 0, 5188, 7568, 0, 78, 0, 0, 0, 79, 0, 0, 0, 8050, 0, 0, 700, 0, 0, 0, 0, 0, 0, 0, 84, 55, 0, 0, 0] payload_rows_per_dst=[14974, 16384, 16384, 16110, 15874, 314, 1400, 278] dup_rows_per_dst=[4, 956, 596, 5236, 9794, 0, 0, 0] combine_rows_per_src=[14974, 16384, 16384, 16110, 15874, 314, 1400, 278] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=363110400 dispatch_meta_bytes=383264 combine_payload_bytes=363110400 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=363493664 ingress_bytes=466919424 link_bytes=466919424 token_bytes=726604064 weight_movement_bytes=103809024 total_nvlink_bytes=830413088 weight_to_token_ratio=0.142869 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.756980202e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=283729920 dispatch_meta_bytes=307840 combine_payload_bytes=283729920 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=284037760 ingress_bytes=352935936 link_bytes=352935936 token_bytes=567767680 weight_movement_bytes=69206016 total_nvlink_bytes=636973696 weight_to_token_ratio=0.121891 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=36608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.328069513e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=344809472 dispatch_meta_bytes=351488 combine_payload_bytes=344809472 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=345160960 ingress_bytes=448618496 link_bytes=448618496 token_bytes=689970432 weight_movement_bytes=103809024 total_nvlink_bytes=793779456 weight_to_token_ratio=0.150454 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=34688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.688115284e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72840 payload_rows_remote=56460 dup_rows=25464 dup_rows_remote=25194 dedup_ratio=0.259033 router_tokens_per_expert=[0, 7451, 0, 0, 0, 6, 0, 0, 0, 2, 1, 1, 1, 555, 0, 0, 0, 0, 0, 8186, 0, 0, 160, 8182, 0, 253, 0, 0, 0, 0, 0, 2501, 1, 0, 81, 0, 1, 5195, 7560, 0, 71, 0, 3, 3, 60, 0, 0, 1, 8051, 2, 0, 682, 0, 2, 0, 1, 0, 1, 0, 75, 56, 1, 1, 5] payload_rows_per_dst=[12, 1114, 16376, 5022, 2580, 15126, 16230, 16380] dup_rows_per_dst=[0, 6, 12758, 486, 38, 270, 11636, 270] combine_rows_per_src=[12, 1114, 16376, 5022, 2580, 15126, 16230, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85430 payload_rows_remote=69270 dup_rows=12874 dup_rows_remote=7690 dedup_ratio=0.130961 router_tokens_per_expert=[0, 7415, 0, 0, 0, 0, 0, 0, 0, 10, 7, 0, 0, 559, 0, 1, 0, 0, 0, 8181, 0, 1, 181, 8170, 0, 258, 2, 2, 0, 0, 0, 2463, 0, 0, 88, 3, 1, 5144, 7542, 4, 81, 2, 8, 7, 109, 3, 0, 1, 8058, 0, 0, 705, 0, 0, 0, 6, 0, 1, 0, 70, 61, 0, 7, 1] payload_rows_per_dst=[8658, 1140, 16344, 4974, 6670, 15106, 16160, 16378] dup_rows_per_dst=[0, 14, 6532, 476, 4, 400, 5184, 264] combine_rows_per_src=[8658, 1140, 16344, 4974, 6670, 15106, 16160, 16378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94614 payload_rows_remote=84182 dup_rows=3690 dup_rows_remote=3690 dedup_ratio=0.037537 router_tokens_per_expert=[0, 7501, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 577, 0, 0, 0, 0, 0, 8192, 0, 0, 175, 8186, 0, 260, 0, 6, 0, 0, 0, 2404, 0, 0, 72, 0, 0, 5144, 7613, 0, 72, 0, 0, 0, 90, 0, 0, 0, 8001, 0, 0, 723, 0, 0, 0, 0, 0, 1, 0, 110, 23, 0, 0, 0] payload_rows_per_dst=[15002, 16378, 350, 4836, 10432, 15226, 16006, 16384] dup_rows_per_dst=[4, 1148, 0, 504, 0, 324, 1442, 268] combine_rows_per_src=[15002, 16378, 350, 4836, 10432, 15226, 16006, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95266 payload_rows_remote=80230 dup_rows=3038 dup_rows_remote=2782 dedup_ratio=0.030904 router_tokens_per_expert=[0, 7512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 0, 2, 0, 0, 0, 8192, 0, 0, 224, 8192, 0, 274, 0, 0, 0, 0, 0, 2312, 0, 0, 76, 0, 0, 5391, 7518, 0, 76, 0, 0, 0, 52, 0, 0, 0, 8033, 0, 0, 650, 0, 0, 0, 0, 0, 1, 0, 103, 19, 0, 0, 0] payload_rows_per_dst=[15024, 8154, 9006, 4662, 10934, 15036, 16066, 16384] dup_rows_per_dst=[0, 516, 210, 510, 0, 256, 1300, 246] combine_rows_per_src=[15024, 8154, 9006, 4662, 10934, 15036, 16066, 16384] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=341360640 dispatch_meta_bytes=343456 combine_payload_bytes=341360640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=341704096 ingress_bytes=375963648 link_bytes=375963648 token_bytes=683064736 weight_movement_bytes=34603008 total_nvlink_bytes=717667744 weight_to_token_ratio=0.050658 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=27776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.414720940e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=328622080 dispatch_meta_bytes=332048 combine_payload_bytes=328622080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=328954128 ingress_bytes=363225088 link_bytes=363225088 token_bytes=657576208 weight_movement_bytes=34603008 total_nvlink_bytes=692179216 weight_to_token_ratio=0.052622 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=47872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.366786764e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=273383424 dispatch_meta_bytes=333304 combine_payload_bytes=273383424 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=273716728 ingress_bytes=342589440 link_bytes=342589440 token_bytes=547100152 weight_movement_bytes=69206016 total_nvlink_bytes=616306168 weight_to_token_ratio=0.126496 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.289136482e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69656 payload_rows_remote=53276 dup_rows=28648 dup_rows_remote=28458 dedup_ratio=0.291423 router_tokens_per_expert=[0, 0, 0, 709, 0, 527, 0, 3, 7053, 0, 0, 0, 0, 0, 2, 73, 0, 0, 4220, 0, 5, 3, 0, 0, 133, 0, 0, 0, 0, 137, 0, 0, 224, 0, 7885, 8188, 8108, 0, 2, 1, 0, 0, 3250, 343, 1, 0, 6, 0, 3, 0, 3, 0, 0, 8174, 0, 2, 94, 2, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[2446, 150, 10214, 540, 16374, 7198, 16354, 16380] dup_rows_per_dst=[32, 0, 1566, 0, 23102, 2, 3756, 190] combine_rows_per_src=[2446, 150, 10214, 540, 16374, 7198, 16354, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=344809472 dispatch_meta_bytes=351488 combine_payload_bytes=344809472 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=345160960 ingress_bytes=448618496 link_bytes=448618496 token_bytes=689970432 weight_movement_bytes=103809024 total_nvlink_bytes=793779456 weight_to_token_ratio=0.150454 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=34688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.688115284e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=231260160 dispatch_meta_bytes=326616 combine_payload_bytes=231260160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=231586776 ingress_bytes=265863168 link_bytes=265863168 token_bytes=462846936 weight_movement_bytes=34603008 total_nvlink_bytes=497449944 weight_to_token_ratio=0.074761 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=48128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.000421697e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87982 payload_rows_remote=73744 dup_rows=10322 dup_rows_remote=10318 dedup_ratio=0.105001 router_tokens_per_expert=[0, 0, 0, 518, 2, 463, 0, 3, 7029, 1, 0, 2, 0, 2, 1, 86, 0, 0, 4302, 0, 3, 0, 0, 0, 74, 1, 0, 0, 1, 185, 2, 0, 300, 0, 7843, 8189, 8095, 1, 0, 1, 0, 0, 3412, 320, 1, 0, 0, 2, 0, 0, 3, 0, 0, 8189, 1, 0, 118, 2, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15934, 14238, 8610, 16196, 16380, 16378, 6, 240] dup_rows_per_dst=[1724, 4, 0, 520, 602, 7470, 2, 0] combine_rows_per_src=[15934, 14238, 8610, 16196, 16380, 16378, 6, 240] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94432 payload_rows_remote=85768 dup_rows=3872 dup_rows_remote=3872 dedup_ratio=0.039388 router_tokens_per_expert=[0, 0, 0, 423, 0, 478, 0, 0, 7047, 0, 1, 0, 0, 0, 0, 70, 0, 0, 4332, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 156, 0, 0, 214, 0, 7845, 8192, 8114, 0, 0, 0, 0, 0, 3581, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 152, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15964, 14234, 8664, 16232, 2460, 9918, 12656, 14304] dup_rows_per_dst=[1528, 2, 0, 354, 68, 1636, 0, 284] combine_rows_per_src=[15964, 14234, 8664, 16232, 2460, 9918, 12656, 14304] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87866 payload_rows_remote=71494 dup_rows=10438 dup_rows_remote=10428 dedup_ratio=0.106181 router_tokens_per_expert=[0, 0, 0, 452, 8, 499, 0, 8, 6998, 1, 0, 7, 0, 1, 4, 83, 0, 0, 4391, 0, 5, 5, 0, 0, 136, 1, 0, 0, 0, 152, 2, 0, 240, 0, 7872, 8180, 8088, 0, 0, 1, 0, 0, 3385, 352, 1, 5, 1, 1, 2, 0, 6, 0, 2, 8175, 0, 6, 72, 8, 0, 2, 0, 0, 0, 0] payload_rows_per_dst=[1922, 12570, 16212, 582, 16340, 7486, 16372, 16382] dup_rows_per_dst=[12, 14, 8334, 0, 1922, 4, 10, 142] combine_rows_per_src=[1922, 12570, 16212, 582, 16340, 7486, 16372, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95822 payload_rows_remote=95314 dup_rows=2482 dup_rows_remote=2482 dedup_ratio=0.025248 router_tokens_per_expert=[0, 0, 0, 514, 0, 484, 0, 0, 7135, 0, 0, 0, 0, 0, 0, 72, 0, 0, 4063, 0, 2, 0, 0, 0, 50, 0, 0, 0, 0, 170, 0, 0, 254, 0, 7870, 8192, 8116, 0, 0, 0, 0, 0, 3536, 344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 158, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16010, 14414, 8130, 16232, 508, 7760, 16384, 16384] dup_rows_per_dst=[1726, 0, 0, 440, 0, 0, 0, 316] combine_rows_per_src=[16010, 14414, 8130, 16232, 508, 7760, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=283729920 dispatch_meta_bytes=307840 combine_payload_bytes=283729920 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=284037760 ingress_bytes=352935936 link_bytes=352935936 token_bytes=567767680 weight_movement_bytes=69206016 total_nvlink_bytes=636973696 weight_to_token_ratio=0.121891 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=36608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.328069513e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=328622080 dispatch_meta_bytes=332048 combine_payload_bytes=328622080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=328954128 ingress_bytes=363225088 link_bytes=363225088 token_bytes=657576208 weight_movement_bytes=34603008 total_nvlink_bytes=692179216 weight_to_token_ratio=0.052622 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=47872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.366786764e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91102 payload_rows_remote=88650 dup_rows=7202 dup_rows_remote=7166 dedup_ratio=0.073263 router_tokens_per_expert=[0, 7481, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 578, 0, 0, 0, 0, 0, 8192, 0, 0, 165, 8190, 0, 268, 0, 0, 0, 0, 0, 2474, 0, 0, 71, 0, 0, 5071, 7556, 0, 70, 0, 0, 0, 101, 0, 0, 0, 8044, 0, 0, 769, 0, 0, 0, 0, 0, 0, 0, 72, 46, 0, 0, 0] payload_rows_per_dst=[14962, 16384, 2452, 16100, 10284, 15112, 1538, 14270] dup_rows_per_dst=[8, 1152, 36, 5472, 0, 342, 0, 192] combine_rows_per_src=[14962, 16384, 2452, 16100, 10284, 15112, 1538, 14270] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=218218496 dispatch_meta_bytes=326936 combine_payload_bytes=218218496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=218545432 ingress_bytes=252821504 link_bytes=252821504 token_bytes=436763928 weight_movement_bytes=34603008 total_nvlink_bytes=471366936 weight_to_token_ratio=0.079226 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=48128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.513469658e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95990 payload_rows_remote=79786 dup_rows=2314 dup_rows_remote=1776 dedup_ratio=0.023539 router_tokens_per_expert=[0, 0, 0, 437, 0, 514, 0, 0, 7015, 0, 0, 0, 0, 0, 0, 81, 0, 0, 3976, 0, 4, 0, 0, 0, 140, 0, 0, 0, 0, 129, 0, 0, 233, 0, 7856, 8192, 8102, 0, 0, 0, 0, 0, 3846, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15970, 14192, 7958, 16204, 466, 8434, 16382, 16384] dup_rows_per_dst=[1644, 0, 2, 538, 0, 2, 0, 128] combine_rows_per_src=[15970, 14192, 7958, 16204, 466, 8434, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91284 payload_rows_remote=83256 dup_rows=7020 dup_rows_remote=7020 dedup_ratio=0.071411 router_tokens_per_expert=[0, 0, 0, 459, 0, 486, 0, 0, 7018, 0, 0, 0, 0, 0, 0, 76, 0, 0, 4125, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 178, 0, 0, 257, 0, 7824, 8192, 8112, 0, 0, 0, 0, 0, 3678, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 217, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5468, 14188, 14112, 13552, 3170, 8028, 16382, 16384] dup_rows_per_dst=[434, 0, 5778, 324, 50, 0, 0, 434] combine_rows_per_src=[5468, 14188, 14112, 13552, 3170, 8028, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=302055424 dispatch_meta_bytes=336248 combine_payload_bytes=302055424 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=302391672 ingress_bytes=371261440 link_bytes=371261440 token_bytes=604447096 weight_movement_bytes=69206016 total_nvlink_bytes=673653112 weight_to_token_ratio=0.114495 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=48512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397026910e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=363110400 dispatch_meta_bytes=383264 combine_payload_bytes=363110400 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=363493664 ingress_bytes=466919424 link_bytes=466919424 token_bytes=726604064 weight_movement_bytes=103809024 total_nvlink_bytes=830413088 weight_to_token_ratio=0.142869 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.756980202e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95990 payload_rows_remote=79786 dup_rows=2314 dup_rows_remote=1776 dedup_ratio=0.023539 router_tokens_per_expert=[0, 0, 0, 437, 0, 514, 0, 0, 7015, 0, 0, 0, 0, 0, 0, 81, 0, 0, 3976, 0, 4, 0, 0, 0, 140, 0, 0, 0, 0, 129, 0, 0, 233, 0, 7856, 8192, 8102, 0, 0, 0, 0, 0, 3846, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15970, 14192, 7958, 16204, 466, 8434, 16382, 16384] dup_rows_per_dst=[1644, 0, 2, 538, 0, 2, 0, 128] combine_rows_per_src=[15970, 14192, 7958, 16204, 466, 8434, 16382, 16384] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88280 payload_rows_remote=72362 dup_rows=10024 dup_rows_remote=8452 dedup_ratio=0.101969 router_tokens_per_expert=[0, 0, 0, 446, 0, 452, 0, 0, 7071, 0, 0, 0, 0, 0, 0, 78, 0, 0, 4362, 0, 0, 0, 0, 0, 17, 1, 0, 0, 0, 164, 0, 0, 239, 0, 7847, 8192, 8113, 0, 0, 0, 0, 0, 3499, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 176, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15918, 14298, 8724, 16226, 16384, 16378, 0, 352] dup_rows_per_dst=[1572, 0, 0, 364, 478, 7610, 0, 0] combine_rows_per_src=[15918, 14298, 8724, 16226, 16384, 16378, 0, 352] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=351305728 dispatch_meta_bytes=358560 combine_payload_bytes=351305728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=351664288 ingress_bytes=385908736 link_bytes=385908736 token_bytes=702970016 weight_movement_bytes=34603008 total_nvlink_bytes=737573024 weight_to_token_ratio=0.049224 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=33792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.452143505e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=292839424 dispatch_meta_bytes=327688 combine_payload_bytes=292839424 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=293167112 ingress_bytes=362045440 link_bytes=362045440 token_bytes=586006536 weight_movement_bytes=69206016 total_nvlink_bytes=655212552 weight_to_token_ratio=0.118098 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=47360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.362347844e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=390406144 dispatch_meta_bytes=391184 combine_payload_bytes=390406144 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=390797328 ingress_bytes=563421184 link_bytes=563421184 token_bytes=781203472 weight_movement_bytes=173015040 total_nvlink_bytes=954218512 weight_to_token_ratio=0.221472 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=2.120108556e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=326803456 dispatch_meta_bytes=326248 combine_payload_bytes=326803456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=327129704 ingress_bytes=361406464 link_bytes=361406464 token_bytes=653933160 weight_movement_bytes=34603008 total_nvlink_bytes=688536168 weight_to_token_ratio=0.052915 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=47360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.359943428e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95822 payload_rows_remote=95314 dup_rows=2482 dup_rows_remote=2482 dedup_ratio=0.025248 router_tokens_per_expert=[0, 0, 0, 514, 0, 484, 0, 0, 7135, 0, 0, 0, 0, 0, 0, 72, 0, 0, 4063, 0, 2, 0, 0, 0, 50, 0, 0, 0, 0, 170, 0, 0, 254, 0, 7870, 8192, 8116, 0, 0, 0, 0, 0, 3536, 344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 158, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16010, 14414, 8130, 16232, 508, 7760, 16384, 16384] dup_rows_per_dst=[1726, 0, 0, 440, 0, 0, 0, 316] combine_rows_per_src=[16010, 14414, 8130, 16232, 508, 7760, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69656 payload_rows_remote=53276 dup_rows=28648 dup_rows_remote=28458 dedup_ratio=0.291423 router_tokens_per_expert=[0, 0, 0, 709, 0, 527, 0, 3, 7053, 0, 0, 0, 0, 0, 2, 73, 0, 0, 4220, 0, 5, 3, 0, 0, 133, 0, 0, 0, 0, 137, 0, 0, 224, 0, 7885, 8188, 8108, 0, 2, 1, 0, 0, 3250, 343, 1, 0, 6, 0, 3, 0, 3, 0, 0, 8174, 0, 2, 94, 2, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[2446, 150, 10214, 540, 16374, 7198, 16354, 16380] dup_rows_per_dst=[32, 0, 1566, 0, 23102, 2, 3756, 190] combine_rows_per_src=[2446, 150, 10214, 540, 16374, 7198, 16354, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87866 payload_rows_remote=71494 dup_rows=10438 dup_rows_remote=10428 dedup_ratio=0.106181 router_tokens_per_expert=[0, 0, 0, 452, 8, 499, 0, 8, 6998, 1, 0, 7, 0, 1, 4, 83, 0, 0, 4391, 0, 5, 5, 0, 0, 136, 1, 0, 0, 0, 152, 2, 0, 240, 0, 7872, 8180, 8088, 0, 0, 1, 0, 0, 3385, 352, 1, 5, 1, 1, 2, 0, 6, 0, 2, 8175, 0, 6, 72, 8, 0, 2, 0, 0, 0, 0] payload_rows_per_dst=[1922, 12570, 16212, 582, 16340, 7486, 16372, 16382] dup_rows_per_dst=[12, 14, 8334, 0, 1922, 4, 10, 142] combine_rows_per_src=[1922, 12570, 16212, 582, 16340, 7486, 16372, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91284 payload_rows_remote=83256 dup_rows=7020 dup_rows_remote=7020 dedup_ratio=0.071411 router_tokens_per_expert=[0, 0, 0, 459, 0, 486, 0, 0, 7018, 0, 0, 0, 0, 0, 0, 76, 0, 0, 4125, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 178, 0, 0, 257, 0, 7824, 8192, 8112, 0, 0, 0, 0, 0, 3678, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 217, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5468, 14188, 14112, 13552, 3170, 8028, 16382, 16384] dup_rows_per_dst=[434, 0, 5778, 324, 50, 0, 0, 434] combine_rows_per_src=[5468, 14188, 14112, 13552, 3170, 8028, 16382, 16384] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=341016576 dispatch_meta_bytes=361104 combine_payload_bytes=341016576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=341377680 ingress_bytes=375619584 link_bytes=375619584 token_bytes=682394256 weight_movement_bytes=34603008 total_nvlink_bytes=716997264 weight_to_token_ratio=0.050708 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=28288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.413426255e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=326803456 dispatch_meta_bytes=326248 combine_payload_bytes=326803456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=327129704 ingress_bytes=361406464 link_bytes=361406464 token_bytes=653933160 weight_movement_bytes=34603008 total_nvlink_bytes=688536168 weight_to_token_ratio=0.052915 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=47360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.359943428e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=296394752 dispatch_meta_bytes=323256 combine_payload_bytes=296394752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296718008 ingress_bytes=330997760 link_bytes=330997760 token_bytes=593112760 weight_movement_bytes=34603008 total_nvlink_bytes=627715768 weight_to_token_ratio=0.058341 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=41344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.245517923e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68266 payload_rows_remote=53128 dup_rows=30038 dup_rows_remote=23714 dedup_ratio=0.305562 router_tokens_per_expert=[0, 0, 186, 0, 6, 0, 0, 3, 1, 0, 0, 5, 0, 3, 3, 0, 1, 2, 0, 0, 1, 8185, 3, 0, 1, 1, 0, 8186, 8116, 0, 1, 3, 0, 0, 0, 3089, 0, 2, 76, 27, 8185, 3, 0, 0, 0, 0, 0, 0, 1, 0, 3419, 71, 214, 0, 26, 0, 5799, 0, 5, 0, 1071, 2455, 2, 0] payload_rows_per_dst=[390, 12, 10, 16378, 6384, 13578, 16376, 15138] dup_rows_per_dst=[0, 12, 4, 16238, 4, 0, 7456, 6324] combine_rows_per_src=[390, 12, 10, 16378, 6384, 13578, 16376, 15138] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91674 payload_rows_remote=75292 dup_rows=6630 dup_rows_remote=6626 dedup_ratio=0.067444 router_tokens_per_expert=[0, 0, 228, 1, 0, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 2, 8187, 1, 0, 0, 0, 0, 8189, 8061, 0, 0, 0, 1, 1, 0, 3977, 0, 2, 88, 45, 8186, 3, 1, 0, 0, 0, 2, 0, 2, 0, 2739, 83, 193, 0, 18, 2, 6152, 0, 0, 0, 1065, 1918, 0, 0] payload_rows_per_dst=[16128, 16382, 16380, 13556, 14484, 2822, 6074, 5848] dup_rows_per_dst=[454, 4, 0, 0, 6048, 6, 0, 118] combine_rows_per_src=[16128, 16382, 16380, 13556, 14484, 2822, 6074, 5848] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92418 payload_rows_remote=76034 dup_rows=5886 dup_rows_remote=5886 dedup_ratio=0.059875 router_tokens_per_expert=[0, 0, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8191, 8129, 0, 0, 1, 0, 0, 0, 3146, 0, 0, 70, 57, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2968, 70, 177, 0, 22, 0, 6186, 0, 0, 0, 1121, 2319, 0, 0] payload_rows_per_dst=[16262, 16382, 16384, 2, 13786, 16384, 6470, 6748] dup_rows_per_dst=[618, 0, 0, 0, 5132, 0, 4, 132] combine_rows_per_src=[16262, 16382, 16384, 2, 13786, 16384, 6470, 6748] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71144 payload_rows_remote=54772 dup_rows=27160 dup_rows_remote=19926 dedup_ratio=0.276286 router_tokens_per_expert=[1, 0, 277, 8, 3, 0, 0, 4, 4, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 3, 8178, 4, 1, 0, 0, 2, 8181, 8109, 1, 0, 4, 1, 0, 3, 3123, 1, 8, 93, 49, 8157, 2, 13, 0, 0, 3, 6, 0, 1, 0, 3315, 81, 174, 0, 48, 6, 6196, 0, 3, 1, 948, 2132, 0, 0] payload_rows_per_dst=[584, 22, 16, 16362, 6544, 16340, 16372, 14904] dup_rows_per_dst=[2, 2, 0, 16232, 12, 22, 7234, 3656] combine_rows_per_src=[584, 22, 16, 16362, 6544, 16340, 16372, 14904] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93546 payload_rows_remote=84200 dup_rows=4758 dup_rows_remote=2938 dedup_ratio=0.048401 router_tokens_per_expert=[0, 0, 308, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 8096, 0, 0, 0, 0, 0, 0, 3213, 0, 0, 72, 11, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3184, 72, 231, 0, 47, 0, 6084, 0, 0, 0, 1064, 2193, 0, 0] payload_rows_per_dst=[16192, 16384, 16384, 0, 9346, 16384, 7068, 11788] dup_rows_per_dst=[618, 0, 0, 0, 1820, 0, 0, 2320] combine_rows_per_src=[16192, 16384, 16384, 0, 9346, 16384, 7068, 11788] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=390406144 dispatch_meta_bytes=391184 combine_payload_bytes=390406144 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=390797328 ingress_bytes=563421184 link_bytes=563421184 token_bytes=781203472 weight_movement_bytes=173015040 total_nvlink_bytes=954218512 weight_to_token_ratio=0.221472 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=2.120108556e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=218218496 dispatch_meta_bytes=326936 combine_payload_bytes=218218496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=218545432 ingress_bytes=252821504 link_bytes=252821504 token_bytes=436763928 weight_movement_bytes=34603008 total_nvlink_bytes=471366936 weight_to_token_ratio=0.079226 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=48128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.513469658e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94432 payload_rows_remote=85768 dup_rows=3872 dup_rows_remote=3872 dedup_ratio=0.039388 router_tokens_per_expert=[0, 0, 0, 423, 0, 478, 0, 0, 7047, 0, 1, 0, 0, 0, 0, 70, 0, 0, 4332, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 156, 0, 0, 214, 0, 7845, 8192, 8114, 0, 0, 0, 0, 0, 3581, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 152, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15964, 14234, 8664, 16232, 2460, 9918, 12656, 14304] dup_rows_per_dst=[1528, 2, 0, 354, 68, 1636, 0, 284] combine_rows_per_src=[15964, 14234, 8664, 16232, 2460, 9918, 12656, 14304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=292839424 dispatch_meta_bytes=327688 combine_payload_bytes=292839424 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=293167112 ingress_bytes=362045440 link_bytes=362045440 token_bytes=586006536 weight_movement_bytes=69206016 total_nvlink_bytes=655212552 weight_to_token_ratio=0.118098 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=47360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.362347844e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=341016576 dispatch_meta_bytes=361104 combine_payload_bytes=341016576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=341377680 ingress_bytes=375619584 link_bytes=375619584 token_bytes=682394256 weight_movement_bytes=34603008 total_nvlink_bytes=716997264 weight_to_token_ratio=0.050708 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=28288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.413426255e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=217612288 dispatch_meta_bytes=307368 combine_payload_bytes=217612288 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=217919656 ingress_bytes=321421312 link_bytes=321421312 token_bytes=435531944 weight_movement_bytes=103809024 total_nvlink_bytes=539340968 weight_to_token_ratio=0.238350 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=21888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.209482521e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92836 payload_rows_remote=92832 dup_rows=5468 dup_rows_remote=5468 dedup_ratio=0.055623 router_tokens_per_expert=[0, 0, 253, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 8078, 0, 0, 2, 0, 0, 0, 3201, 0, 0, 81, 21, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3340, 81, 225, 0, 48, 0, 6026, 0, 5, 0, 959, 2246, 0, 0] payload_rows_per_dst=[16156, 16384, 16384, 4, 13822, 16384, 7384, 6318] dup_rows_per_dst=[526, 0, 0, 0, 4836, 0, 4, 102] combine_rows_per_src=[16156, 16384, 16384, 4, 13822, 16384, 7384, 6318] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89968 payload_rows_remote=73584 dup_rows=8336 dup_rows_remote=8336 dedup_ratio=0.084798 router_tokens_per_expert=[0, 0, 308, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 8057, 0, 0, 1, 0, 0, 0, 3217, 0, 0, 76, 39, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3305, 72, 201, 0, 25, 0, 6014, 0, 3, 0, 980, 2277, 0, 0] payload_rows_per_dst=[13142, 16336, 7044, 3098, 6656, 16384, 12494, 14814] dup_rows_per_dst=[492, 0, 0, 50, 8, 0, 4052, 3734] combine_rows_per_src=[13142, 16336, 7044, 3098, 6656, 16384, 12494, 14814] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=308396032 dispatch_meta_bytes=327672 combine_payload_bytes=308396032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=308723704 ingress_bytes=342999040 link_bytes=342999040 token_bytes=617119736 weight_movement_bytes=34603008 total_nvlink_bytes=651722744 weight_to_token_ratio=0.056072 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.290677774e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=351305728 dispatch_meta_bytes=358560 combine_payload_bytes=351305728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=351664288 ingress_bytes=385908736 link_bytes=385908736 token_bytes=702970016 weight_movement_bytes=34603008 total_nvlink_bytes=737573024 weight_to_token_ratio=0.049224 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=33792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.452143505e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92836 payload_rows_remote=92832 dup_rows=5468 dup_rows_remote=5468 dedup_ratio=0.055623 router_tokens_per_expert=[0, 0, 253, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 8078, 0, 0, 2, 0, 0, 0, 3201, 0, 0, 81, 21, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3340, 81, 225, 0, 48, 0, 6026, 0, 5, 0, 959, 2246, 0, 0] payload_rows_per_dst=[16156, 16384, 16384, 4, 13822, 16384, 7384, 6318] dup_rows_per_dst=[526, 0, 0, 0, 4836, 0, 4, 102] combine_rows_per_src=[16156, 16384, 16384, 4, 13822, 16384, 7384, 6318] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87198 payload_rows_remote=71056 dup_rows=11106 dup_rows_remote=10508 dedup_ratio=0.112976 router_tokens_per_expert=[0, 0, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 8071, 0, 0, 0, 0, 0, 0, 3083, 0, 0, 78, 18, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3333, 78, 216, 0, 31, 0, 6189, 0, 1, 0, 925, 2254, 0, 0] payload_rows_per_dst=[16142, 16384, 16384, 16384, 8310, 0, 7316, 6278] dup_rows_per_dst=[598, 0, 9314, 0, 1112, 0, 0, 82] combine_rows_per_src=[16142, 16384, 16384, 16384, 8310, 0, 7316, 6278] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=311435264 dispatch_meta_bytes=327680 combine_payload_bytes=311435264 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=311762944 ingress_bytes=380641280 link_bytes=380641280 token_bytes=623198208 weight_movement_bytes=69206016 total_nvlink_bytes=692404224 weight_to_token_ratio=0.111050 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=44544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.432322493e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=224346112 dispatch_meta_bytes=298792 combine_payload_bytes=224346112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=224644904 ingress_bytes=258949120 link_bytes=258949120 token_bytes=448991016 weight_movement_bytes=34603008 total_nvlink_bytes=483594024 weight_to_token_ratio=0.077068 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=25728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.744046914e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=344883200 dispatch_meta_bytes=348552 combine_payload_bytes=344883200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=345231752 ingress_bytes=379486208 link_bytes=379486208 token_bytes=690114952 weight_movement_bytes=34603008 total_nvlink_bytes=724717960 weight_to_token_ratio=0.050141 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=26112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.427976050e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93546 payload_rows_remote=84200 dup_rows=4758 dup_rows_remote=2938 dedup_ratio=0.048401 router_tokens_per_expert=[0, 0, 308, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 8096, 0, 0, 0, 0, 0, 0, 3213, 0, 0, 72, 11, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3184, 72, 231, 0, 47, 0, 6084, 0, 0, 0, 1064, 2193, 0, 0] payload_rows_per_dst=[16192, 16384, 16384, 0, 9346, 16384, 7068, 11788] dup_rows_per_dst=[618, 0, 0, 0, 1820, 0, 0, 2320] combine_rows_per_src=[16192, 16384, 16384, 0, 9346, 16384, 7068, 11788] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68266 payload_rows_remote=53128 dup_rows=30038 dup_rows_remote=23714 dedup_ratio=0.305562 router_tokens_per_expert=[0, 0, 186, 0, 6, 0, 0, 3, 1, 0, 0, 5, 0, 3, 3, 0, 1, 2, 0, 0, 1, 8185, 3, 0, 1, 1, 0, 8186, 8116, 0, 1, 3, 0, 0, 0, 3089, 0, 2, 76, 27, 8185, 3, 0, 0, 0, 0, 0, 0, 1, 0, 3419, 71, 214, 0, 26, 0, 5799, 0, 5, 0, 1071, 2455, 2, 0] payload_rows_per_dst=[390, 12, 10, 16378, 6384, 13578, 16376, 15138] dup_rows_per_dst=[0, 12, 4, 16238, 4, 0, 7456, 6324] combine_rows_per_src=[390, 12, 10, 16378, 6384, 13578, 16376, 15138] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71144 payload_rows_remote=54772 dup_rows=27160 dup_rows_remote=19926 dedup_ratio=0.276286 router_tokens_per_expert=[1, 0, 277, 8, 3, 0, 0, 4, 4, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 3, 8178, 4, 1, 0, 0, 2, 8181, 8109, 1, 0, 4, 1, 0, 3, 3123, 1, 8, 93, 49, 8157, 2, 13, 0, 0, 3, 6, 0, 1, 0, 3315, 81, 174, 0, 48, 6, 6196, 0, 3, 1, 948, 2132, 0, 0] payload_rows_per_dst=[584, 22, 16, 16362, 6544, 16340, 16372, 14904] dup_rows_per_dst=[2, 2, 0, 16232, 12, 22, 7234, 3656] combine_rows_per_src=[584, 22, 16, 16362, 6544, 16340, 16372, 14904] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89968 payload_rows_remote=73584 dup_rows=8336 dup_rows_remote=8336 dedup_ratio=0.084798 router_tokens_per_expert=[0, 0, 308, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 8057, 0, 0, 1, 0, 0, 0, 3217, 0, 0, 76, 39, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3305, 72, 201, 0, 25, 0, 6014, 0, 3, 0, 980, 2277, 0, 0] payload_rows_per_dst=[13142, 16336, 7044, 3098, 6656, 16384, 12494, 14814] dup_rows_per_dst=[492, 0, 0, 50, 8, 0, 4052, 3734] combine_rows_per_src=[13142, 16336, 7044, 3098, 6656, 16384, 12494, 14814] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=380239872 dispatch_meta_bytes=393200 combine_payload_bytes=380239872 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=380633072 ingress_bytes=484048896 link_bytes=484048896 token_bytes=760872944 weight_movement_bytes=103809024 total_nvlink_bytes=864681968 weight_to_token_ratio=0.136434 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=17792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.821437027e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=301400064 dispatch_meta_bytes=327680 combine_payload_bytes=301400064 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=301727744 ingress_bytes=370606080 link_bytes=370606080 token_bytes=603127808 weight_movement_bytes=69206016 total_nvlink_bytes=672333824 weight_to_token_ratio=0.114745 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394560843e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=291045376 dispatch_meta_bytes=326256 combine_payload_bytes=291045376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291371632 ingress_bytes=325648384 link_bytes=325648384 token_bytes=582417008 weight_movement_bytes=34603008 total_nvlink_bytes=617020016 weight_to_token_ratio=0.059413 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=46976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.225388652e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88734 payload_rows_remote=73228 dup_rows=9570 dup_rows_remote=8934 dedup_ratio=0.097351 router_tokens_per_expert=[3, 0, 0, 0, 4027, 0, 0, 1, 2, 7908, 0, 0, 8106, 8, 228, 113, 181, 0, 4, 2, 1, 113, 2, 1, 0, 3, 3, 243, 0, 1, 8185, 0, 2, 81, 1, 5, 2, 0, 0, 1, 0, 1, 0, 1, 2, 0, 7205, 2, 4640, 0, 0, 2, 1, 0, 0, 0, 3, 0, 388, 28, 0, 0, 7652, 0] payload_rows_per_dst=[8060, 696, 15840, 16380, 16368, 10, 15874, 15506] dup_rows_per_dst=[2, 6, 584, 490, 28, 2, 7822, 636] combine_rows_per_src=[8060, 696, 15840, 16380, 16368, 10, 15874, 15506] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=63862 payload_rows_remote=47484 dup_rows=34442 dup_rows_remote=8030 dedup_ratio=0.350362 router_tokens_per_expert=[0, 0, 1, 0, 3392, 0, 0, 0, 1, 7900, 0, 0, 8097, 13, 197, 158, 179, 0, 1, 0, 0, 385, 0, 0, 0, 3, 1, 248, 1, 0, 8188, 0, 1, 95, 2, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 7312, 4, 4634, 3, 0, 0, 1, 0, 0, 0, 2, 0, 468, 126, 1, 0, 7733, 2] payload_rows_per_dst=[16378, 16378, 1130, 4860, 198, 14700, 9276, 942] dup_rows_per_dst=[6784, 26412, 0, 136, 2, 852, 0, 256] combine_rows_per_src=[16378, 16378, 1130, 4860, 198, 14700, 9276, 942] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91110 payload_rows_remote=77376 dup_rows=7194 dup_rows_remote=6858 dedup_ratio=0.073181 router_tokens_per_expert=[0, 0, 0, 0, 3387, 0, 0, 0, 0, 7905, 0, 0, 8120, 11, 175, 152, 162, 0, 0, 0, 0, 153, 0, 0, 0, 0, 0, 248, 0, 0, 8192, 0, 2, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7254, 0, 5120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 368, 68, 0, 0, 7758, 0] payload_rows_per_dst=[11766, 2964, 13734, 8182, 15552, 14686, 10240, 13986] dup_rows_per_dst=[3470, 916, 336, 236, 12, 1590, 0, 634] combine_rows_per_src=[11766, 2964, 13734, 8182, 15552, 14686, 10240, 13986] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92964 payload_rows_remote=80778 dup_rows=5340 dup_rows_remote=1816 dedup_ratio=0.054321 router_tokens_per_expert=[7, 1, 5, 3, 3697, 0, 1, 0, 5, 7907, 0, 0, 8089, 17, 200, 142, 174, 0, 3, 0, 2, 117, 1, 0, 0, 4, 0, 236, 5, 0, 8182, 0, 3, 124, 6, 0, 8, 4, 0, 0, 1, 0, 0, 1, 0, 1, 7195, 6, 4858, 1, 0, 1, 0, 0, 0, 7, 2, 0, 359, 64, 0, 4, 7707, 2] payload_rows_per_dst=[7416, 720, 15856, 16380, 16364, 8426, 12186, 15616] dup_rows_per_dst=[12, 8, 552, 474, 104, 6, 3524, 660] combine_rows_per_src=[7416, 720, 15856, 16380, 16364, 8426, 12186, 15616] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96368 payload_rows_remote=79984 dup_rows=1936 dup_rows_remote=1744 dedup_ratio=0.019694 router_tokens_per_expert=[0, 0, 0, 0, 3418, 0, 0, 0, 0, 7911, 0, 0, 8120, 5, 208, 169, 187, 0, 0, 0, 0, 156, 0, 0, 0, 0, 0, 252, 0, 0, 8192, 0, 0, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7309, 0, 4888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 374, 103, 0, 0, 7692, 0] payload_rows_per_dst=[6836, 764, 15966, 16384, 16384, 14618, 9776, 15640] dup_rows_per_dst=[0, 0, 542, 504, 192, 0, 0, 698] combine_rows_per_src=[6836, 764, 15966, 16384, 16384, 14618, 9776, 15640] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92418 payload_rows_remote=76034 dup_rows=5886 dup_rows_remote=5886 dedup_ratio=0.059875 router_tokens_per_expert=[0, 0, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8191, 8129, 0, 0, 1, 0, 0, 0, 3146, 0, 0, 70, 57, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2968, 70, 177, 0, 22, 0, 6186, 0, 0, 0, 1121, 2319, 0, 0] payload_rows_per_dst=[16262, 16382, 16384, 2, 13786, 16384, 6470, 6748] dup_rows_per_dst=[618, 0, 0, 0, 5132, 0, 4, 132] combine_rows_per_src=[16262, 16382, 16384, 2, 13786, 16384, 6470, 6748] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=344883200 dispatch_meta_bytes=348552 combine_payload_bytes=344883200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=345231752 ingress_bytes=379486208 link_bytes=379486208 token_bytes=690114952 weight_movement_bytes=34603008 total_nvlink_bytes=724717960 weight_to_token_ratio=0.050141 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=26112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.427976050e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=224346112 dispatch_meta_bytes=298792 combine_payload_bytes=224346112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=224644904 ingress_bytes=258949120 link_bytes=258949120 token_bytes=448991016 weight_movement_bytes=34603008 total_nvlink_bytes=483594024 weight_to_token_ratio=0.077068 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=25728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.744046914e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=217612288 dispatch_meta_bytes=307368 combine_payload_bytes=217612288 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=217919656 ingress_bytes=321421312 link_bytes=321421312 token_bytes=435531944 weight_movement_bytes=103809024 total_nvlink_bytes=539340968 weight_to_token_ratio=0.238350 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=21888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.209482521e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=301400064 dispatch_meta_bytes=327680 combine_payload_bytes=301400064 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=301727744 ingress_bytes=370606080 link_bytes=370606080 token_bytes=603127808 weight_movement_bytes=69206016 total_nvlink_bytes=672333824 weight_to_token_ratio=0.114745 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394560843e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=299941888 dispatch_meta_bytes=328648 combine_payload_bytes=299941888 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=300270536 ingress_bytes=403750912 link_bytes=403750912 token_bytes=600212424 weight_movement_bytes=103809024 total_nvlink_bytes=704021448 weight_to_token_ratio=0.172954 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=45440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519282177e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96512 payload_rows_remote=80128 dup_rows=1792 dup_rows_remote=1302 dedup_ratio=0.018229 router_tokens_per_expert=[0, 0, 0, 0, 3457, 0, 0, 0, 0, 7910, 0, 0, 8108, 5, 205, 162, 168, 0, 0, 0, 0, 264, 0, 0, 0, 0, 0, 245, 0, 0, 8192, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7187, 0, 5001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 357, 67, 0, 0, 7726, 0] payload_rows_per_dst=[6914, 744, 16092, 16384, 16378, 14374, 10002, 15624] dup_rows_per_dst=[0, 0, 592, 490, 34, 0, 0, 676] combine_rows_per_src=[6914, 744, 16092, 16384, 16378, 14374, 10002, 15624] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96164 payload_rows_remote=81582 dup_rows=2140 dup_rows_remote=2140 dedup_ratio=0.021769 router_tokens_per_expert=[0, 0, 0, 0, 3491, 0, 0, 0, 0, 7909, 0, 0, 8106, 4, 218, 142, 162, 0, 0, 0, 0, 245, 0, 0, 0, 0, 0, 255, 0, 0, 8192, 0, 1, 155, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7290, 0, 4672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 78, 0, 0, 7746, 0] payload_rows_per_dst=[6982, 728, 16062, 16384, 16366, 14582, 9344, 15716] dup_rows_per_dst=[0, 0, 570, 510, 158, 0, 0, 902] combine_rows_per_src=[6982, 728, 16062, 16384, 16366, 14582, 9344, 15716] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=194494464 dispatch_meta_bytes=222056 combine_payload_bytes=194494464 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=194716520 ingress_bytes=298303488 link_bytes=298303488 token_bytes=389210984 weight_movement_bytes=103809024 total_nvlink_bytes=493020008 weight_to_token_ratio=0.266717 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.122492010e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=311435264 dispatch_meta_bytes=327680 combine_payload_bytes=311435264 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=311762944 ingress_bytes=380641280 link_bytes=380641280 token_bytes=623198208 weight_movement_bytes=69206016 total_nvlink_bytes=692404224 weight_to_token_ratio=0.111050 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=44544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.432322493e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96512 payload_rows_remote=80128 dup_rows=1792 dup_rows_remote=1302 dedup_ratio=0.018229 router_tokens_per_expert=[0, 0, 0, 0, 3457, 0, 0, 0, 0, 7910, 0, 0, 8108, 5, 205, 162, 168, 0, 0, 0, 0, 264, 0, 0, 0, 0, 0, 245, 0, 0, 8192, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7187, 0, 5001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 357, 67, 0, 0, 7726, 0] payload_rows_per_dst=[6914, 744, 16092, 16384, 16378, 14374, 10002, 15624] dup_rows_per_dst=[0, 0, 592, 490, 34, 0, 0, 676] combine_rows_per_src=[6914, 744, 16092, 16384, 16378, 14374, 10002, 15624] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=59264 payload_rows_remote=42880 dup_rows=39040 dup_rows_remote=31740 dedup_ratio=0.397135 router_tokens_per_expert=[0, 0, 0, 0, 3650, 0, 0, 0, 0, 7881, 0, 0, 8112, 5, 193, 140, 168, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 255, 0, 0, 8192, 0, 2, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7229, 0, 5130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 328, 78, 0, 0, 7653, 0] payload_rows_per_dst=[16384, 16384, 448, 510, 164, 14458, 10260, 656] dup_rows_per_dst=[7300, 31584, 0, 0, 0, 0, 0, 156] combine_rows_per_src=[16384, 16384, 448, 510, 164, 14458, 10260, 656] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=316932096 dispatch_meta_bytes=336936 combine_payload_bytes=316932096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=317269032 ingress_bytes=351535104 link_bytes=351535104 token_bytes=634201128 weight_movement_bytes=34603008 total_nvlink_bytes=668804136 weight_to_token_ratio=0.054562 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=46336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.322798295e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=330866688 dispatch_meta_bytes=330376 combine_payload_bytes=330866688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=331197064 ingress_bytes=365469696 link_bytes=365469696 token_bytes=662063752 weight_movement_bytes=34603008 total_nvlink_bytes=696666760 weight_to_token_ratio=0.052265 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=39040 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.375233043e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=327614464 dispatch_meta_bytes=326912 combine_payload_bytes=327614464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=327941376 ingress_bytes=362217472 link_bytes=362217472 token_bytes=655555840 weight_movement_bytes=34603008 total_nvlink_bytes=690158848 weight_to_token_ratio=0.052784 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=48256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.362995186e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96368 payload_rows_remote=79984 dup_rows=1936 dup_rows_remote=1744 dedup_ratio=0.019694 router_tokens_per_expert=[0, 0, 0, 0, 3418, 0, 0, 0, 0, 7911, 0, 0, 8120, 5, 208, 169, 187, 0, 0, 0, 0, 156, 0, 0, 0, 0, 0, 252, 0, 0, 8192, 0, 0, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7309, 0, 4888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 374, 103, 0, 0, 7692, 0] payload_rows_per_dst=[6836, 764, 15966, 16384, 16384, 14618, 9776, 15640] dup_rows_per_dst=[0, 0, 542, 504, 192, 0, 0, 698] combine_rows_per_src=[6836, 764, 15966, 16384, 16384, 14618, 9776, 15640] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92964 payload_rows_remote=80778 dup_rows=5340 dup_rows_remote=1816 dedup_ratio=0.054321 router_tokens_per_expert=[7, 1, 5, 3, 3697, 0, 1, 0, 5, 7907, 0, 0, 8089, 17, 200, 142, 174, 0, 3, 0, 2, 117, 1, 0, 0, 4, 0, 236, 5, 0, 8182, 0, 3, 124, 6, 0, 8, 4, 0, 0, 1, 0, 0, 1, 0, 1, 7195, 6, 4858, 1, 0, 1, 0, 0, 0, 7, 2, 0, 359, 64, 0, 4, 7707, 2] payload_rows_per_dst=[7416, 720, 15856, 16380, 16364, 8426, 12186, 15616] dup_rows_per_dst=[12, 8, 552, 474, 104, 6, 3524, 660] combine_rows_per_src=[7416, 720, 15856, 16380, 16364, 8426, 12186, 15616] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88734 payload_rows_remote=73228 dup_rows=9570 dup_rows_remote=8934 dedup_ratio=0.097351 router_tokens_per_expert=[3, 0, 0, 0, 4027, 0, 0, 1, 2, 7908, 0, 0, 8106, 8, 228, 113, 181, 0, 4, 2, 1, 113, 2, 1, 0, 3, 3, 243, 0, 1, 8185, 0, 2, 81, 1, 5, 2, 0, 0, 1, 0, 1, 0, 1, 2, 0, 7205, 2, 4640, 0, 0, 2, 1, 0, 0, 0, 3, 0, 388, 28, 0, 0, 7652, 0] payload_rows_per_dst=[8060, 696, 15840, 16380, 16368, 10, 15874, 15506] dup_rows_per_dst=[2, 6, 584, 490, 28, 2, 7822, 636] combine_rows_per_src=[8060, 696, 15840, 16380, 16368, 10, 15874, 15506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96164 payload_rows_remote=81582 dup_rows=2140 dup_rows_remote=2140 dedup_ratio=0.021769 router_tokens_per_expert=[0, 0, 0, 0, 3491, 0, 0, 0, 0, 7909, 0, 0, 8106, 4, 218, 142, 162, 0, 0, 0, 0, 245, 0, 0, 0, 0, 0, 255, 0, 0, 8192, 0, 1, 155, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7290, 0, 4672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 78, 0, 0, 7746, 0] payload_rows_per_dst=[6982, 728, 16062, 16384, 16366, 14582, 9344, 15716] dup_rows_per_dst=[0, 0, 570, 510, 158, 0, 0, 902] combine_rows_per_src=[6982, 728, 16062, 16384, 16366, 14582, 9344, 15716] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=328204288 dispatch_meta_bytes=325720 combine_payload_bytes=328204288 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=328530008 ingress_bytes=397410304 link_bytes=397410304 token_bytes=656734296 weight_movement_bytes=69206016 total_nvlink_bytes=725940312 weight_to_token_ratio=0.105379 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=44928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.495422980e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=334159872 dispatch_meta_bytes=334888 combine_payload_bytes=334159872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=334494760 ingress_bytes=403365888 link_bytes=403365888 token_bytes=668654632 weight_movement_bytes=69206016 total_nvlink_bytes=737860648 weight_to_token_ratio=0.103500 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=47872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.517833363e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=175636480 dispatch_meta_bytes=298480 combine_payload_bytes=175636480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=175934960 ingress_bytes=210239488 link_bytes=210239488 token_bytes=351571440 weight_movement_bytes=34603008 total_nvlink_bytes=386174448 weight_to_token_ratio=0.098424 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=28544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=7.911142677e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70296 payload_rows_remote=57178 dup_rows=28008 dup_rows_remote=27904 dedup_ratio=0.284912 router_tokens_per_expert=[1, 1, 0, 97, 0, 2, 0, 8185, 3019, 0, 0, 3, 3, 3, 12, 0, 499, 6209, 4, 0, 4845, 0, 0, 3996, 4386, 0, 0, 747, 0, 0, 0, 0, 0, 653, 1, 2, 10, 0, 0, 2, 6, 0, 6830, 8139, 0, 0, 0, 6, 0, 0, 0, 0, 1087, 0, 2, 0, 51, 0, 0, 0, 0, 351, 0, 0] payload_rows_per_dst=[200, 6070, 4186, 14740, 1334, 16380, 14268, 13118] dup_rows_per_dst=[2, 10, 292, 8424, 2, 13582, 5592, 104] combine_rows_per_src=[200, 6070, 4186, 14740, 1334, 16380, 14268, 13118] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89672 payload_rows_remote=75824 dup_rows=8632 dup_rows_remote=3374 dedup_ratio=0.087809 router_tokens_per_expert=[2, 0, 0, 66, 0, 2, 0, 8187, 2657, 0, 0, 2, 0, 3, 1, 0, 483, 6320, 4, 0, 4257, 0, 0, 4408, 4640, 0, 0, 854, 0, 0, 0, 0, 0, 652, 1, 0, 4, 0, 0, 0, 5, 1, 6890, 8101, 0, 0, 0, 3, 3, 0, 0, 0, 1202, 1, 2, 0, 81, 0, 0, 0, 0, 320, 0, 0] payload_rows_per_dst=[16380, 13848, 9274, 10150, 16206, 16, 10520, 13278] dup_rows_per_dst=[134, 5258, 214, 838, 1310, 2, 712, 164] combine_rows_per_src=[16380, 13848, 9274, 10150, 16206, 16, 10520, 13278] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75112 payload_rows_remote=63670 dup_rows=23192 dup_rows_remote=21274 dedup_ratio=0.235921 router_tokens_per_expert=[1, 1, 0, 63, 1, 1, 0, 8178, 2988, 0, 7, 2, 7, 10, 6, 1, 496, 6312, 11, 0, 4234, 0, 5, 4242, 4663, 0, 0, 766, 2, 0, 0, 0, 0, 660, 1, 0, 15, 0, 1, 0, 4, 4, 6858, 8127, 1, 0, 0, 7, 1, 0, 0, 1, 1010, 4, 4, 1, 86, 0, 11, 0, 0, 359, 0, 0] payload_rows_per_dst=[134, 6026, 16380, 10064, 1352, 16370, 11442, 13344] dup_rows_per_dst=[0, 16, 6634, 798, 2, 13632, 1918, 192] combine_rows_per_src=[134, 6026, 16380, 10064, 1352, 16370, 11442, 13344] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88628 payload_rows_remote=79568 dup_rows=9676 dup_rows_remote=9416 dedup_ratio=0.098429 router_tokens_per_expert=[0, 0, 0, 80, 0, 0, 0, 8192, 3336, 0, 0, 0, 0, 0, 11, 0, 455, 6234, 0, 0, 4205, 0, 0, 4255, 4475, 0, 0, 750, 0, 0, 0, 0, 0, 655, 0, 0, 6, 0, 0, 0, 0, 0, 6992, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 969, 0, 0, 0, 66, 0, 0, 0, 0, 339, 0, 0] payload_rows_per_dst=[16384, 14028, 9060, 9718, 16264, 0, 10036, 13138] dup_rows_per_dst=[160, 6650, 260, 732, 1322, 0, 412, 140] combine_rows_per_src=[16384, 14028, 9060, 9718, 16264, 0, 10036, 13138] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91110 payload_rows_remote=77376 dup_rows=7194 dup_rows_remote=6858 dedup_ratio=0.073181 router_tokens_per_expert=[0, 0, 0, 0, 3387, 0, 0, 0, 0, 7905, 0, 0, 8120, 11, 175, 152, 162, 0, 0, 0, 0, 153, 0, 0, 0, 0, 0, 248, 0, 0, 8192, 0, 2, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7254, 0, 5120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 368, 68, 0, 0, 7758, 0] payload_rows_per_dst=[11766, 2964, 13734, 8182, 15552, 14686, 10240, 13986] dup_rows_per_dst=[3470, 916, 336, 236, 12, 1590, 0, 634] combine_rows_per_src=[11766, 2964, 13734, 8182, 15552, 14686, 10240, 13986] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=327614464 dispatch_meta_bytes=326912 combine_payload_bytes=327614464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=327941376 ingress_bytes=362217472 link_bytes=362217472 token_bytes=655555840 weight_movement_bytes=34603008 total_nvlink_bytes=690158848 weight_to_token_ratio=0.052784 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=48256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.362995186e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=330866688 dispatch_meta_bytes=330376 combine_payload_bytes=330866688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=331197064 ingress_bytes=365469696 link_bytes=365469696 token_bytes=662063752 weight_movement_bytes=34603008 total_nvlink_bytes=696666760 weight_to_token_ratio=0.052265 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=39040 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.375233043e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=334159872 dispatch_meta_bytes=334888 combine_payload_bytes=334159872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=334494760 ingress_bytes=403365888 link_bytes=403365888 token_bytes=668654632 weight_movement_bytes=69206016 total_nvlink_bytes=737860648 weight_to_token_ratio=0.103500 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=47872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.517833363e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=299941888 dispatch_meta_bytes=328648 combine_payload_bytes=299941888 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=300270536 ingress_bytes=403750912 link_bytes=403750912 token_bytes=600212424 weight_movement_bytes=103809024 total_nvlink_bytes=704021448 weight_to_token_ratio=0.172954 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=45440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519282177e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95132 payload_rows_remote=78894 dup_rows=3172 dup_rows_remote=1808 dedup_ratio=0.032267 router_tokens_per_expert=[0, 0, 0, 91, 0, 0, 0, 8192, 2935, 0, 0, 0, 0, 0, 15, 0, 563, 6305, 0, 0, 4103, 0, 0, 4331, 4901, 0, 0, 627, 0, 0, 0, 0, 0, 666, 0, 0, 16, 0, 0, 0, 0, 0, 6968, 8119, 0, 0, 0, 0, 0, 0, 0, 0, 901, 0, 0, 0, 73, 0, 0, 0, 0, 346, 0, 0] payload_rows_per_dst=[16384, 5900, 8960, 10430, 16238, 13936, 9980, 13304] dup_rows_per_dst=[182, 0, 372, 626, 1364, 0, 484, 144] combine_rows_per_src=[16384, 5900, 8960, 10430, 16238, 13936, 9980, 13304] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=234201088 dispatch_meta_bytes=340328 combine_payload_bytes=234201088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=234541416 ingress_bytes=268804096 link_bytes=268804096 token_bytes=468742504 weight_movement_bytes=34603008 total_nvlink_bytes=503345512 weight_to_token_ratio=0.073821 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=45952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.011488173e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88950 payload_rows_remote=79200 dup_rows=9354 dup_rows_remote=8818 dedup_ratio=0.095154 router_tokens_per_expert=[0, 0, 0, 70, 0, 0, 0, 8192, 3210, 0, 0, 0, 0, 0, 3, 0, 511, 5961, 0, 0, 4273, 0, 0, 4540, 4540, 0, 0, 603, 0, 0, 0, 0, 0, 681, 0, 0, 11, 0, 0, 0, 0, 0, 6912, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 0, 0, 0, 78, 0, 0, 0, 0, 372, 0, 0] payload_rows_per_dst=[16384, 13444, 9250, 9750, 16280, 856, 10320, 12666] dup_rows_per_dst=[140, 5950, 318, 536, 1384, 0, 870, 156] combine_rows_per_src=[16384, 13444, 9250, 9750, 16280, 856, 10320, 12666] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84016 payload_rows_remote=68704 dup_rows=14288 dup_rows_remote=6070 dedup_ratio=0.145345 router_tokens_per_expert=[0, 0, 0, 63, 0, 0, 0, 8192, 3019, 0, 0, 0, 0, 0, 11, 0, 472, 6268, 0, 0, 4415, 0, 0, 4305, 4636, 0, 0, 695, 0, 0, 0, 0, 0, 654, 0, 0, 8, 0, 0, 0, 0, 0, 6851, 8107, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 0, 0, 0, 62, 0, 0, 0, 0, 336, 0, 0] payload_rows_per_dst=[9930, 6060, 12406, 9920, 7170, 15312, 10008, 13210] dup_rows_per_dst=[80, 0, 3868, 742, 540, 8218, 718, 122] combine_rows_per_src=[9930, 6060, 12406, 9920, 7170, 15312, 10008, 13210] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=310575104 dispatch_meta_bytes=316792 combine_payload_bytes=310575104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=310891896 ingress_bytes=345178112 link_bytes=345178112 token_bytes=621467000 weight_movement_bytes=34603008 total_nvlink_bytes=656070008 weight_to_token_ratio=0.055680 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=24704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.298877446e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=316932096 dispatch_meta_bytes=336936 combine_payload_bytes=316932096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=317269032 ingress_bytes=351535104 link_bytes=351535104 token_bytes=634201128 weight_movement_bytes=34603008 total_nvlink_bytes=668804136 weight_to_token_ratio=0.054562 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=46336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.322798295e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82466 payload_rows_remote=66082 dup_rows=15838 dup_rows_remote=10412 dedup_ratio=0.161112 router_tokens_per_expert=[0, 0, 0, 82, 0, 0, 0, 8192, 3215, 0, 0, 0, 0, 0, 3, 0, 461, 6169, 0, 0, 4014, 0, 0, 4586, 4758, 0, 0, 660, 0, 0, 0, 0, 0, 627, 0, 0, 9, 0, 0, 0, 0, 0, 6935, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 926, 0, 0, 0, 77, 0, 0, 0, 0, 306, 0, 0] payload_rows_per_dst=[16384, 11132, 12872, 10210, 16264, 0, 10428, 5176] dup_rows_per_dst=[5426, 3912, 3958, 626, 1272, 0, 596, 48] combine_rows_per_src=[16384, 11132, 12872, 10210, 16264, 0, 10428, 5176] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=325910528 dispatch_meta_bytes=355936 combine_payload_bytes=325910528 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=326266464 ingress_bytes=464322560 link_bytes=464322560 token_bytes=652176992 weight_movement_bytes=138412032 total_nvlink_bytes=790589024 weight_to_token_ratio=0.212231 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=28160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.747208412e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=260792320 dispatch_meta_bytes=339776 combine_payload_bytes=260792320 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=261132096 ingress_bytes=329998336 link_bytes=329998336 token_bytes=521924416 weight_movement_bytes=69206016 total_nvlink_bytes=591130432 weight_to_token_ratio=0.132598 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=34688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.241757171e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88950 payload_rows_remote=79200 dup_rows=9354 dup_rows_remote=8818 dedup_ratio=0.095154 router_tokens_per_expert=[0, 0, 0, 70, 0, 0, 0, 8192, 3210, 0, 0, 0, 0, 0, 3, 0, 511, 5961, 0, 0, 4273, 0, 0, 4540, 4540, 0, 0, 603, 0, 0, 0, 0, 0, 681, 0, 0, 11, 0, 0, 0, 0, 0, 6912, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 0, 0, 0, 78, 0, 0, 0, 0, 372, 0, 0] payload_rows_per_dst=[16384, 13444, 9250, 9750, 16280, 856, 10320, 12666] dup_rows_per_dst=[140, 5950, 318, 536, 1384, 0, 870, 156] combine_rows_per_src=[16384, 13444, 9250, 9750, 16280, 856, 10320, 12666] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=323149824 dispatch_meta_bytes=322808 combine_payload_bytes=323149824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=323472632 ingress_bytes=357752832 link_bytes=357752832 token_bytes=646622456 weight_movement_bytes=34603008 total_nvlink_bytes=681225464 weight_to_token_ratio=0.053513 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=43904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.346195105e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=324403200 dispatch_meta_bytes=352072 combine_payload_bytes=324403200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=324755272 ingress_bytes=359006208 link_bytes=359006208 token_bytes=649158472 weight_movement_bytes=34603008 total_nvlink_bytes=683761480 weight_to_token_ratio=0.053304 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=37120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.350911458e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=281411584 dispatch_meta_bytes=299096 combine_payload_bytes=281411584 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=281710680 ingress_bytes=385220608 link_bytes=385220608 token_bytes=563122264 weight_movement_bytes=103809024 total_nvlink_bytes=666931288 weight_to_token_ratio=0.184345 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=28032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.449554135e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95132 payload_rows_remote=78894 dup_rows=3172 dup_rows_remote=1808 dedup_ratio=0.032267 router_tokens_per_expert=[0, 0, 0, 91, 0, 0, 0, 8192, 2935, 0, 0, 0, 0, 0, 15, 0, 563, 6305, 0, 0, 4103, 0, 0, 4331, 4901, 0, 0, 627, 0, 0, 0, 0, 0, 666, 0, 0, 16, 0, 0, 0, 0, 0, 6968, 8119, 0, 0, 0, 0, 0, 0, 0, 0, 901, 0, 0, 0, 73, 0, 0, 0, 0, 346, 0, 0] payload_rows_per_dst=[16384, 5900, 8960, 10430, 16238, 13936, 9980, 13304] dup_rows_per_dst=[182, 0, 372, 626, 1364, 0, 484, 144] combine_rows_per_src=[16384, 5900, 8960, 10430, 16238, 13936, 9980, 13304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75112 payload_rows_remote=63670 dup_rows=23192 dup_rows_remote=21274 dedup_ratio=0.235921 router_tokens_per_expert=[1, 1, 0, 63, 1, 1, 0, 8178, 2988, 0, 7, 2, 7, 10, 6, 1, 496, 6312, 11, 0, 4234, 0, 5, 4242, 4663, 0, 0, 766, 2, 0, 0, 0, 0, 660, 1, 0, 15, 0, 1, 0, 4, 4, 6858, 8127, 1, 0, 0, 7, 1, 0, 0, 1, 1010, 4, 4, 1, 86, 0, 11, 0, 0, 359, 0, 0] payload_rows_per_dst=[134, 6026, 16380, 10064, 1352, 16370, 11442, 13344] dup_rows_per_dst=[0, 16, 6634, 798, 2, 13632, 1918, 192] combine_rows_per_src=[134, 6026, 16380, 10064, 1352, 16370, 11442, 13344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84016 payload_rows_remote=68704 dup_rows=14288 dup_rows_remote=6070 dedup_ratio=0.145345 router_tokens_per_expert=[0, 0, 0, 63, 0, 0, 0, 8192, 3019, 0, 0, 0, 0, 0, 11, 0, 472, 6268, 0, 0, 4415, 0, 0, 4305, 4636, 0, 0, 695, 0, 0, 0, 0, 0, 654, 0, 0, 8, 0, 0, 0, 0, 0, 6851, 8107, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 0, 0, 0, 62, 0, 0, 0, 0, 336, 0, 0] payload_rows_per_dst=[9930, 6060, 12406, 9920, 7170, 15312, 10008, 13210] dup_rows_per_dst=[80, 0, 3868, 742, 540, 8218, 718, 122] combine_rows_per_src=[9930, 6060, 12406, 9920, 7170, 15312, 10008, 13210] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70296 payload_rows_remote=57178 dup_rows=28008 dup_rows_remote=27904 dedup_ratio=0.284912 router_tokens_per_expert=[1, 1, 0, 97, 0, 2, 0, 8185, 3019, 0, 0, 3, 3, 3, 12, 0, 499, 6209, 4, 0, 4845, 0, 0, 3996, 4386, 0, 0, 747, 0, 0, 0, 0, 0, 653, 1, 2, 10, 0, 0, 2, 6, 0, 6830, 8139, 0, 0, 0, 6, 0, 0, 0, 0, 1087, 0, 2, 0, 51, 0, 0, 0, 0, 351, 0, 0] payload_rows_per_dst=[200, 6070, 4186, 14740, 1334, 16380, 14268, 13118] dup_rows_per_dst=[2, 10, 292, 8424, 2, 13582, 5592, 104] combine_rows_per_src=[200, 6070, 4186, 14740, 1334, 16380, 14268, 13118] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=270671872 dispatch_meta_bytes=305976 combine_payload_bytes=270671872 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=270977848 ingress_bytes=374480896 link_bytes=374480896 token_bytes=541649720 weight_movement_bytes=103809024 total_nvlink_bytes=645458744 weight_to_token_ratio=0.191653 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=45952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.409141463e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74566 payload_rows_remote=61388 dup_rows=23738 dup_rows_remote=22932 dedup_ratio=0.241475 router_tokens_per_expert=[0, 2, 5, 352, 9, 0, 0, 0, 4, 0, 0, 6321, 0, 0, 3, 0, 7968, 123, 42, 0, 1, 0, 3, 0, 3, 0, 6587, 0, 0, 18, 0, 8006, 5, 0, 0, 0, 1, 5, 0, 0, 5851, 0, 0, 8184, 0, 1, 0, 0, 1, 4, 0, 0, 0, 0, 5248, 0, 0, 2, 0, 1, 402, 0, 0, 0] payload_rows_per_dst=[732, 12650, 338, 42, 16024, 16376, 15226, 13178] dup_rows_per_dst=[4, 6, 0, 0, 10, 14774, 8138, 806] combine_rows_per_src=[732, 12650, 338, 42, 16024, 16376, 15226, 13178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=324403200 dispatch_meta_bytes=352072 combine_payload_bytes=324403200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=324755272 ingress_bytes=359006208 link_bytes=359006208 token_bytes=649158472 weight_movement_bytes=34603008 total_nvlink_bytes=683761480 weight_to_token_ratio=0.053304 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=37120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.350911458e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83972 payload_rows_remote=71450 dup_rows=14332 dup_rows_remote=14330 dedup_ratio=0.145793 router_tokens_per_expert=[0, 0, 4, 395, 5, 0, 3, 0, 1, 1, 0, 6259, 1, 0, 0, 0, 7831, 205, 97, 0, 2, 0, 2, 0, 1, 0, 6782, 0, 0, 3, 0, 7994, 3, 0, 0, 0, 0, 0, 0, 0, 5842, 0, 0, 8186, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 5082, 0, 0, 4, 0, 0, 442, 1, 0, 3] payload_rows_per_dst=[16380, 12522, 16100, 15860, 1832, 4, 10166, 11108] dup_rows_per_dst=[806, 2, 174, 12614, 0, 0, 0, 736] combine_rows_per_src=[16380, 12522, 16100, 15860, 1832, 4, 10166, 11108] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70322 payload_rows_remote=59512 dup_rows=27982 dup_rows_remote=27980 dedup_ratio=0.284648 router_tokens_per_expert=[0, 0, 15, 466, 3, 0, 2, 4, 0, 0, 0, 5996, 1, 11, 0, 1, 7954, 151, 23, 11, 0, 0, 7, 1, 3, 0, 6713, 0, 0, 2, 1, 8014, 8, 0, 0, 0, 0, 2, 0, 0, 5702, 0, 0, 8167, 0, 1, 0, 0, 3, 0, 1, 0, 0, 3, 5399, 0, 0, 10, 0, 0, 472, 0, 1, 4] payload_rows_per_dst=[968, 12016, 668, 12, 16046, 16346, 10810, 13456] dup_rows_per_dst=[12, 2, 48, 0, 2, 26972, 2, 944] combine_rows_per_src=[968, 12016, 668, 12, 16046, 16346, 10810, 13456] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96802 payload_rows_remote=80720 dup_rows=1502 dup_rows_remote=1372 dedup_ratio=0.015279 router_tokens_per_expert=[0, 0, 0, 328, 0, 0, 0, 0, 0, 0, 0, 6313, 0, 0, 0, 0, 7974, 125, 7, 0, 0, 0, 0, 0, 0, 0, 6717, 0, 0, 2, 0, 8079, 0, 0, 0, 0, 0, 0, 0, 0, 5700, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5359, 0, 0, 0, 0, 0, 356, 0, 0, 0] payload_rows_per_dst=[16384, 12626, 16082, 11400, 16158, 0, 10718, 13434] dup_rows_per_dst=[656, 0, 130, 4, 0, 0, 0, 712] combine_rows_per_src=[16384, 12626, 16082, 11400, 16158, 0, 10718, 13434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88628 payload_rows_remote=79568 dup_rows=9676 dup_rows_remote=9416 dedup_ratio=0.098429 router_tokens_per_expert=[0, 0, 0, 80, 0, 0, 0, 8192, 3336, 0, 0, 0, 0, 0, 11, 0, 455, 6234, 0, 0, 4205, 0, 0, 4255, 4475, 0, 0, 750, 0, 0, 0, 0, 0, 655, 0, 0, 6, 0, 0, 0, 0, 0, 6992, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 969, 0, 0, 0, 66, 0, 0, 0, 0, 339, 0, 0] payload_rows_per_dst=[16384, 14028, 9060, 9718, 16264, 0, 10036, 13138] dup_rows_per_dst=[160, 6650, 260, 732, 1322, 0, 412, 140] combine_rows_per_src=[16384, 14028, 9060, 9718, 16264, 0, 10036, 13138] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=323149824 dispatch_meta_bytes=322808 combine_payload_bytes=323149824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=323472632 ingress_bytes=357752832 link_bytes=357752832 token_bytes=646622456 weight_movement_bytes=34603008 total_nvlink_bytes=681225464 weight_to_token_ratio=0.053513 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=43904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.346195105e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=260792320 dispatch_meta_bytes=339776 combine_payload_bytes=260792320 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=261132096 ingress_bytes=329998336 link_bytes=329998336 token_bytes=521924416 weight_movement_bytes=69206016 total_nvlink_bytes=591130432 weight_to_token_ratio=0.132598 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=34688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.241757171e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96550 payload_rows_remote=80454 dup_rows=1754 dup_rows_remote=1754 dedup_ratio=0.017843 router_tokens_per_expert=[0, 0, 0, 310, 0, 0, 0, 0, 0, 0, 0, 6096, 0, 0, 0, 0, 7903, 157, 61, 0, 0, 0, 0, 0, 0, 0, 6679, 0, 0, 8, 0, 8048, 0, 0, 0, 0, 0, 0, 0, 0, 5854, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5346, 0, 0, 0, 0, 0, 498, 0, 0, 0] payload_rows_per_dst=[16384, 12192, 16120, 5680, 16096, 6028, 10692, 13358] dup_rows_per_dst=[620, 0, 122, 16, 0, 0, 0, 996] combine_rows_per_src=[16384, 12192, 16120, 5680, 16096, 6028, 10692, 13358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=251445248 dispatch_meta_bytes=337280 combine_payload_bytes=251445248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=251782528 ingress_bytes=286048256 link_bytes=286048256 token_bytes=503227776 weight_movement_bytes=34603008 total_nvlink_bytes=537830784 weight_to_token_ratio=0.068762 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=45696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.076376559e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96546 payload_rows_remote=84908 dup_rows=1758 dup_rows_remote=1752 dedup_ratio=0.017883 router_tokens_per_expert=[0, 0, 0, 370, 0, 0, 0, 0, 2, 0, 0, 6083, 0, 0, 0, 0, 7916, 192, 17, 0, 0, 0, 0, 0, 0, 0, 6650, 0, 0, 3, 0, 8054, 0, 0, 0, 0, 0, 0, 0, 0, 5819, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5426, 0, 0, 0, 0, 0, 428, 0, 0, 0] payload_rows_per_dst=[16384, 12168, 16094, 11638, 16108, 0, 10852, 13302] dup_rows_per_dst=[740, 2, 156, 6, 0, 0, 0, 854] combine_rows_per_src=[16384, 12168, 16094, 11638, 16108, 0, 10852, 13302] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=281411584 dispatch_meta_bytes=299096 combine_payload_bytes=281411584 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=281710680 ingress_bytes=385220608 link_bytes=385220608 token_bytes=563122264 weight_movement_bytes=103809024 total_nvlink_bytes=666931288 weight_to_token_ratio=0.184345 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=28032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.449554135e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=234201088 dispatch_meta_bytes=340328 combine_payload_bytes=234201088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=234541416 ingress_bytes=268804096 link_bytes=268804096 token_bytes=468742504 weight_movement_bytes=34603008 total_nvlink_bytes=503345512 weight_to_token_ratio=0.073821 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=45952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.011488173e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=292659200 dispatch_meta_bytes=343120 combine_payload_bytes=292659200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293002320 ingress_bytes=327262208 link_bytes=327262208 token_bytes=585661520 weight_movement_bytes=34603008 total_nvlink_bytes=620264528 weight_to_token_ratio=0.059084 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.231461342e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=325910528 dispatch_meta_bytes=355936 combine_payload_bytes=325910528 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=326266464 ingress_bytes=464322560 link_bytes=464322560 token_bytes=652176992 weight_movement_bytes=138412032 total_nvlink_bytes=790589024 weight_to_token_ratio=0.212231 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=28160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.747208412e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72686 payload_rows_remote=56302 dup_rows=25618 dup_rows_remote=24918 dedup_ratio=0.260600 router_tokens_per_expert=[0, 0, 0, 350, 0, 0, 0, 0, 0, 0, 0, 5917, 0, 0, 0, 0, 7886, 200, 16, 0, 0, 0, 0, 0, 2, 0, 6586, 0, 0, 1, 0, 8089, 0, 0, 0, 0, 0, 0, 0, 0, 5830, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5593, 0, 0, 0, 0, 0, 490, 0, 0, 0] payload_rows_per_dst=[16384, 11864, 15888, 16384, 0, 0, 11186, 980] dup_rows_per_dst=[700, 132, 154, 24632, 0, 0, 0, 0] combine_rows_per_src=[16384, 11864, 15888, 16384, 0, 0, 11186, 980] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96546 payload_rows_remote=84908 dup_rows=1758 dup_rows_remote=1752 dedup_ratio=0.017883 router_tokens_per_expert=[0, 0, 0, 370, 0, 0, 0, 0, 2, 0, 0, 6083, 0, 0, 0, 0, 7916, 192, 17, 0, 0, 0, 0, 0, 0, 0, 6650, 0, 0, 3, 0, 8054, 0, 0, 0, 0, 0, 0, 0, 0, 5819, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5426, 0, 0, 0, 0, 0, 428, 0, 0, 0] payload_rows_per_dst=[16384, 12168, 16094, 11638, 16108, 0, 10852, 13302] dup_rows_per_dst=[740, 2, 156, 6, 0, 0, 0, 854] combine_rows_per_src=[16384, 12168, 16094, 11638, 16108, 0, 10852, 13302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96550 payload_rows_remote=80454 dup_rows=1754 dup_rows_remote=1754 dedup_ratio=0.017843 router_tokens_per_expert=[0, 0, 0, 310, 0, 0, 0, 0, 0, 0, 0, 6096, 0, 0, 0, 0, 7903, 157, 61, 0, 0, 0, 0, 0, 0, 0, 6679, 0, 0, 8, 0, 8048, 0, 0, 0, 0, 0, 0, 0, 0, 5854, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5346, 0, 0, 0, 0, 0, 498, 0, 0, 0] payload_rows_per_dst=[16384, 12192, 16120, 5680, 16096, 6028, 10692, 13358] dup_rows_per_dst=[620, 0, 122, 16, 0, 0, 0, 996] combine_rows_per_src=[16384, 12192, 16120, 5680, 16096, 6028, 10692, 13358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70322 payload_rows_remote=59512 dup_rows=27982 dup_rows_remote=27980 dedup_ratio=0.284648 router_tokens_per_expert=[0, 0, 15, 466, 3, 0, 2, 4, 0, 0, 0, 5996, 1, 11, 0, 1, 7954, 151, 23, 11, 0, 0, 7, 1, 3, 0, 6713, 0, 0, 2, 1, 8014, 8, 0, 0, 0, 0, 2, 0, 0, 5702, 0, 0, 8167, 0, 1, 0, 0, 3, 0, 1, 0, 0, 3, 5399, 0, 0, 10, 0, 0, 472, 0, 1, 4] payload_rows_per_dst=[968, 12016, 668, 12, 16046, 16346, 10810, 13456] dup_rows_per_dst=[12, 2, 48, 0, 2, 26972, 2, 944] combine_rows_per_src=[968, 12016, 668, 12, 16046, 16346, 10810, 13456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=230612992 dispatch_meta_bytes=324880 combine_payload_bytes=230612992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=230937872 ingress_bytes=265216000 link_bytes=265216000 token_bytes=461550864 weight_movement_bytes=34603008 total_nvlink_bytes=496153872 weight_to_token_ratio=0.074971 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=46336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.979864564e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74566 payload_rows_remote=61388 dup_rows=23738 dup_rows_remote=22932 dedup_ratio=0.241475 router_tokens_per_expert=[0, 2, 5, 352, 9, 0, 0, 0, 4, 0, 0, 6321, 0, 0, 3, 0, 7968, 123, 42, 0, 1, 0, 3, 0, 3, 0, 6587, 0, 0, 18, 0, 8006, 5, 0, 0, 0, 1, 5, 0, 0, 5851, 0, 0, 8184, 0, 1, 0, 0, 1, 4, 0, 0, 0, 0, 5248, 0, 0, 2, 0, 1, 402, 0, 0, 0] payload_rows_per_dst=[732, 12650, 338, 42, 16024, 16376, 15226, 13178] dup_rows_per_dst=[4, 6, 0, 0, 10, 14774, 8138, 806] combine_rows_per_src=[732, 12650, 338, 42, 16024, 16376, 15226, 13178] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92610 payload_rows_remote=79082 dup_rows=5694 dup_rows_remote=1418 dedup_ratio=0.057922 router_tokens_per_expert=[0, 0, 0, 309, 0, 0, 0, 0, 0, 0, 0, 6061, 0, 0, 0, 0, 7875, 193, 47, 0, 0, 0, 0, 0, 0, 0, 6763, 0, 0, 8, 0, 8014, 0, 0, 0, 0, 0, 0, 0, 0, 5976, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5257, 0, 0, 0, 0, 0, 457, 0, 0, 0] payload_rows_per_dst=[10772, 12122, 16104, 16, 16028, 13528, 10514, 13526] dup_rows_per_dst=[378, 0, 126, 0, 0, 4276, 0, 914] combine_rows_per_src=[10772, 12122, 16104, 16, 16028, 13528, 10514, 13526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=347783168 dispatch_meta_bytes=346640 combine_payload_bytes=347783168 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=348129808 ingress_bytes=451592192 link_bytes=451592192 token_bytes=695912976 weight_movement_bytes=103809024 total_nvlink_bytes=799722000 weight_to_token_ratio=0.149170 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=26112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.699305062e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=243761152 dispatch_meta_bytes=349968 combine_payload_bytes=243761152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=244111120 ingress_bytes=278364160 link_bytes=278364160 token_bytes=487872272 weight_movement_bytes=34603008 total_nvlink_bytes=522475280 weight_to_token_ratio=0.070926 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=42752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.047461924e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=330629120 dispatch_meta_bytes=328368 combine_payload_bytes=330629120 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=330957488 ingress_bytes=434438144 link_bytes=434438144 token_bytes=661586608 weight_movement_bytes=103809024 total_nvlink_bytes=765395632 weight_to_token_ratio=0.156909 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=47488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.634755761e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93094 payload_rows_remote=81460 dup_rows=5210 dup_rows_remote=5036 dedup_ratio=0.052999 router_tokens_per_expert=[0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 603, 0, 0, 0, 0, 2, 1924, 0, 0, 0, 1, 0, 3, 3, 0, 8013, 0, 0, 0, 166, 0, 4, 6, 2, 0, 2, 0, 0, 380, 0, 0, 0, 7996, 8184, 5299, 0, 0, 2, 0, 0, 272, 0, 8185, 0, 0, 0, 8096, 1, 4, 0, 0, 0, 0] payload_rows_per_dst=[16372, 11634, 16384, 16094, 16384, 0, 15338, 888] dup_rows_per_dst=[0, 174, 3846, 270, 396, 0, 522, 2] combine_rows_per_src=[16372, 11634, 16384, 16094, 16384, 0, 15338, 888] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=329539584 dispatch_meta_bytes=328832 combine_payload_bytes=329539584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=329868416 ingress_bytes=364142592 link_bytes=364142592 token_bytes=659408000 weight_movement_bytes=34603008 total_nvlink_bytes=694011008 weight_to_token_ratio=0.052476 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.370239258e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=347783168 dispatch_meta_bytes=346640 combine_payload_bytes=347783168 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=348129808 ingress_bytes=451592192 link_bytes=451592192 token_bytes=695912976 weight_movement_bytes=103809024 total_nvlink_bytes=799722000 weight_to_token_ratio=0.149170 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=26112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.699305062e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=230612992 dispatch_meta_bytes=324880 combine_payload_bytes=230612992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=230937872 ingress_bytes=265216000 link_bytes=265216000 token_bytes=461550864 weight_movement_bytes=34603008 total_nvlink_bytes=496153872 weight_to_token_ratio=0.074971 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=46336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.979864564e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96802 payload_rows_remote=80720 dup_rows=1502 dup_rows_remote=1372 dedup_ratio=0.015279 router_tokens_per_expert=[0, 0, 0, 328, 0, 0, 0, 0, 0, 0, 0, 6313, 0, 0, 0, 0, 7974, 125, 7, 0, 0, 0, 0, 0, 0, 0, 6717, 0, 0, 2, 0, 8079, 0, 0, 0, 0, 0, 0, 0, 0, 5700, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5359, 0, 0, 0, 0, 0, 356, 0, 0, 0] payload_rows_per_dst=[16384, 12626, 16082, 11400, 16158, 0, 10718, 13434] dup_rows_per_dst=[656, 0, 130, 4, 0, 0, 0, 712] combine_rows_per_src=[16384, 12626, 16082, 11400, 16158, 0, 10718, 13434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=243761152 dispatch_meta_bytes=349968 combine_payload_bytes=243761152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=244111120 ingress_bytes=278364160 link_bytes=278364160 token_bytes=487872272 weight_movement_bytes=34603008 total_nvlink_bytes=522475280 weight_to_token_ratio=0.070926 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=42752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.047461924e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=323919872 dispatch_meta_bytes=322000 combine_payload_bytes=323919872 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=324241872 ingress_bytes=427728896 link_bytes=427728896 token_bytes=648161744 weight_movement_bytes=103809024 total_nvlink_bytes=751970768 weight_to_token_ratio=0.160159 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=20608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.609509401e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=65644 payload_rows_remote=49414 dup_rows=32660 dup_rows_remote=32594 dedup_ratio=0.332235 router_tokens_per_expert=[1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 606, 2, 0, 0, 0, 0, 1667, 0, 0, 0, 0, 0, 0, 0, 0, 8031, 0, 0, 0, 127, 2, 5, 3, 3, 0, 4, 0, 0, 450, 0, 6, 0, 7958, 8186, 5537, 0, 0, 2, 3, 0, 254, 0, 8185, 0, 0, 0, 8110, 3, 4, 0, 1, 0, 0] payload_rows_per_dst=[6, 12138, 3334, 258, 916, 16384, 16378, 16230] dup_rows_per_dst=[0, 152, 0, 0, 14, 31918, 510, 66] combine_rows_per_src=[6, 12138, 3334, 258, 916, 16384, 16378, 16230] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93094 payload_rows_remote=81460 dup_rows=5210 dup_rows_remote=5036 dedup_ratio=0.052999 router_tokens_per_expert=[0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 603, 0, 0, 0, 0, 2, 1924, 0, 0, 0, 1, 0, 3, 3, 0, 8013, 0, 0, 0, 166, 0, 4, 6, 2, 0, 2, 0, 0, 380, 0, 0, 0, 7996, 8184, 5299, 0, 0, 2, 0, 0, 272, 0, 8185, 0, 0, 0, 8096, 1, 4, 0, 0, 0, 0] payload_rows_per_dst=[16372, 11634, 16384, 16094, 16384, 0, 15338, 888] dup_rows_per_dst=[0, 174, 3846, 270, 396, 0, 522, 2] combine_rows_per_src=[16372, 11634, 16384, 16094, 16384, 0, 15338, 888] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=65518 payload_rows_remote=49166 dup_rows=32786 dup_rows_remote=32172 dedup_ratio=0.333516 router_tokens_per_expert=[3, 2, 0, 0, 2, 2, 1, 1, 0, 0, 1, 738, 3, 1, 0, 0, 7, 2018, 0, 0, 0, 0, 0, 9, 7, 0, 8004, 1, 0, 2, 183, 0, 13, 10, 2, 0, 8, 1, 0, 491, 0, 7, 1, 7967, 8177, 4911, 0, 1, 2, 0, 2, 309, 0, 8170, 0, 0, 0, 8077, 4, 12, 0, 1, 0, 1] payload_rows_per_dst=[14, 11132, 4062, 384, 1026, 16370, 16352, 16178] dup_rows_per_dst=[8, 176, 6, 2, 24, 31944, 614, 12] combine_rows_per_src=[14, 11132, 4062, 384, 1026, 16370, 16352, 16178] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93598 payload_rows_remote=77214 dup_rows=4706 dup_rows_remote=1038 dedup_ratio=0.047872 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 650, 0, 0, 0, 0, 0, 1834, 0, 0, 0, 0, 0, 0, 0, 0, 8044, 0, 0, 0, 139, 0, 0, 0, 0, 0, 0, 0, 0, 490, 0, 0, 0, 8017, 8191, 5200, 0, 0, 0, 0, 0, 277, 0, 8192, 0, 0, 0, 8118, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16382, 11556, 16384, 16102, 16384, 0, 554, 16236] dup_rows_per_dst=[0, 144, 3668, 264, 630, 0, 0, 0] combine_rows_per_src=[16382, 11556, 16384, 16102, 16384, 0, 554, 16236] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=202399744 dispatch_meta_bytes=328032 combine_payload_bytes=202399744 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=202727776 ingress_bytes=271605760 link_bytes=271605760 token_bytes=405127520 weight_movement_bytes=69206016 total_nvlink_bytes=474333536 weight_to_token_ratio=0.170825 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.022030609e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96156 payload_rows_remote=79772 dup_rows=2148 dup_rows_remote=1506 dedup_ratio=0.021851 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 568, 0, 0, 0, 0, 0, 1715, 0, 0, 0, 0, 0, 0, 0, 0, 8048, 0, 0, 0, 124, 0, 0, 0, 0, 0, 0, 0, 0, 529, 0, 0, 0, 7984, 8192, 5382, 0, 0, 0, 0, 0, 298, 0, 8192, 0, 0, 0, 8120, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 11756, 6040, 16106, 16384, 0, 13246, 16240] dup_rows_per_dst=[0, 144, 658, 238, 642, 0, 466, 0] combine_rows_per_src=[16384, 11756, 6040, 16106, 16384, 0, 13246, 16240] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=251445248 dispatch_meta_bytes=337280 combine_payload_bytes=251445248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=251782528 ingress_bytes=286048256 link_bytes=286048256 token_bytes=503227776 weight_movement_bytes=34603008 total_nvlink_bytes=537830784 weight_to_token_ratio=0.068762 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=45696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.076376559e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=333660160 dispatch_meta_bytes=345984 combine_payload_bytes=333660160 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=334006144 ingress_bytes=402866176 link_bytes=402866176 token_bytes=667666304 weight_movement_bytes=69206016 total_nvlink_bytes=736872320 weight_to_token_ratio=0.103654 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=41856 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.515952987e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=330629120 dispatch_meta_bytes=328368 combine_payload_bytes=330629120 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=330957488 ingress_bytes=434438144 link_bytes=434438144 token_bytes=661586608 weight_movement_bytes=103809024 total_nvlink_bytes=765395632 weight_to_token_ratio=0.156909 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=47488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.634755761e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90330 payload_rows_remote=73946 dup_rows=7974 dup_rows_remote=7974 dedup_ratio=0.081116 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 698, 0, 0, 0, 0, 0, 1773, 0, 0, 0, 0, 0, 0, 0, 0, 8036, 0, 0, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 492, 0, 0, 0, 7998, 8192, 5256, 0, 0, 0, 0, 0, 278, 0, 8192, 0, 0, 0, 8114, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 13052, 16384, 16092, 12190, 0, 16228, 0] dup_rows_per_dst=[0, 3188, 3546, 226, 458, 0, 556, 0] combine_rows_per_src=[16384, 13052, 16384, 16092, 12190, 0, 16228, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96260 payload_rows_remote=95114 dup_rows=2044 dup_rows_remote=1502 dedup_ratio=0.020793 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 0, 0, 0, 0, 0, 1952, 0, 0, 0, 0, 0, 0, 0, 0, 8041, 0, 0, 0, 145, 0, 0, 0, 0, 0, 0, 0, 0, 475, 0, 0, 0, 7972, 8192, 5239, 0, 0, 0, 0, 0, 303, 0, 8192, 0, 0, 0, 8116, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16370, 11378, 3904, 15574, 15272, 1146, 16384, 16232] dup_rows_per_dst=[0, 150, 0, 264, 482, 542, 606, 0] combine_rows_per_src=[16370, 11378, 3904, 15574, 15272, 1146, 16384, 16232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93420 payload_rows_remote=77288 dup_rows=4884 dup_rows_remote=4586 dedup_ratio=0.049683 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 795, 0, 0, 0, 0, 0, 1852, 0, 0, 0, 0, 0, 0, 0, 0, 8029, 0, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 565, 0, 0, 0, 7986, 8192, 4970, 0, 0, 0, 0, 0, 275, 0, 8192, 0, 0, 0, 8110, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 11368, 16384, 16132, 16382, 0, 550, 16220] dup_rows_per_dst=[0, 162, 3704, 298, 720, 0, 0, 0] combine_rows_per_src=[16384, 11368, 16384, 16132, 16382, 0, 550, 16220] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=201383936 dispatch_meta_bytes=325352 combine_payload_bytes=201383936 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=201709288 ingress_bytes=270589952 link_bytes=270589952 token_bytes=403093224 weight_movement_bytes=69206016 total_nvlink_bytes=472299240 weight_to_token_ratio=0.171687 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=31232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.018208205e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=316268544 dispatch_meta_bytes=313008 combine_payload_bytes=316268544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=316581552 ingress_bytes=350871552 link_bytes=350871552 token_bytes=632850096 weight_movement_bytes=34603008 total_nvlink_bytes=667453104 weight_to_token_ratio=0.054678 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=34432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.320301403e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 2, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=326746112 dispatch_meta_bytes=325112 combine_payload_bytes=326746112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=327071224 ingress_bytes=361349120 link_bytes=361349120 token_bytes=653817336 weight_movement_bytes=34603008 total_nvlink_bytes=688420344 weight_to_token_ratio=0.052925 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=45312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.359727648e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96156 payload_rows_remote=79772 dup_rows=2148 dup_rows_remote=1506 dedup_ratio=0.021851 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 568, 0, 0, 0, 0, 0, 1715, 0, 0, 0, 0, 0, 0, 0, 0, 8048, 0, 0, 0, 124, 0, 0, 0, 0, 0, 0, 0, 0, 529, 0, 0, 0, 7984, 8192, 5382, 0, 0, 0, 0, 0, 298, 0, 8192, 0, 0, 0, 8120, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 11756, 6040, 16106, 16384, 0, 13246, 16240] dup_rows_per_dst=[0, 144, 658, 238, 642, 0, 466, 0] combine_rows_per_src=[16384, 11756, 6040, 16106, 16384, 0, 13246, 16240] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=302882816 dispatch_meta_bytes=327680 combine_payload_bytes=302882816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=303210496 ingress_bytes=337485824 link_bytes=337485824 token_bytes=606093312 weight_movement_bytes=34603008 total_nvlink_bytes=640696320 weight_to_token_ratio=0.057092 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.269931986e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=389586944 dispatch_meta_bytes=386464 combine_payload_bytes=389586944 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=389973408 ingress_bytes=562601984 link_bytes=562601984 token_bytes=779560352 weight_movement_bytes=173015040 total_nvlink_bytes=952575392 weight_to_token_ratio=0.221939 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=16512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=2.117025973e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=65518 payload_rows_remote=49166 dup_rows=32786 dup_rows_remote=32172 dedup_ratio=0.333516 router_tokens_per_expert=[3, 2, 0, 0, 2, 2, 1, 1, 0, 0, 1, 738, 3, 1, 0, 0, 7, 2018, 0, 0, 0, 0, 0, 9, 7, 0, 8004, 1, 0, 2, 183, 0, 13, 10, 2, 0, 8, 1, 0, 491, 0, 7, 1, 7967, 8177, 4911, 0, 1, 2, 0, 2, 309, 0, 8170, 0, 0, 0, 8077, 4, 12, 0, 1, 0, 1] payload_rows_per_dst=[14, 11132, 4062, 384, 1026, 16370, 16352, 16178] dup_rows_per_dst=[8, 176, 6, 2, 24, 31944, 614, 12] combine_rows_per_src=[14, 11132, 4062, 384, 1026, 16370, 16352, 16178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96260 payload_rows_remote=95114 dup_rows=2044 dup_rows_remote=1502 dedup_ratio=0.020793 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 0, 0, 0, 0, 0, 1952, 0, 0, 0, 0, 0, 0, 0, 0, 8041, 0, 0, 0, 145, 0, 0, 0, 0, 0, 0, 0, 0, 475, 0, 0, 0, 7972, 8192, 5239, 0, 0, 0, 0, 0, 303, 0, 8192, 0, 0, 0, 8116, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16370, 11378, 3904, 15574, 15272, 1146, 16384, 16232] dup_rows_per_dst=[0, 150, 0, 264, 482, 542, 606, 0] combine_rows_per_src=[16370, 11378, 3904, 15574, 15272, 1146, 16384, 16232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=65644 payload_rows_remote=49414 dup_rows=32660 dup_rows_remote=32594 dedup_ratio=0.332235 router_tokens_per_expert=[1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 606, 2, 0, 0, 0, 0, 1667, 0, 0, 0, 0, 0, 0, 0, 0, 8031, 0, 0, 0, 127, 2, 5, 3, 3, 0, 4, 0, 0, 450, 0, 6, 0, 7958, 8186, 5537, 0, 0, 2, 3, 0, 254, 0, 8185, 0, 0, 0, 8110, 3, 4, 0, 1, 0, 0] payload_rows_per_dst=[6, 12138, 3334, 258, 916, 16384, 16378, 16230] dup_rows_per_dst=[0, 152, 0, 0, 14, 31918, 510, 66] combine_rows_per_src=[6, 12138, 3334, 258, 916, 16384, 16378, 16230] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=316571648 dispatch_meta_bytes=327496 combine_payload_bytes=316571648 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=316899144 ingress_bytes=385777664 link_bytes=385777664 token_bytes=633470792 weight_movement_bytes=69206016 total_nvlink_bytes=702676808 weight_to_token_ratio=0.109249 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=48000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.451650292e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=50760 payload_rows_remote=34384 dup_rows=47544 dup_rows_remote=33058 dedup_ratio=0.483643 router_tokens_per_expert=[1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 7939, 0, 6, 0, 12, 0, 0, 0, 6, 0, 0, 3748, 1, 1, 1, 0, 2572, 0, 2, 0, 0, 8185, 8170, 0, 5, 5, 0, 4247, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 6055, 8186, 0, 0] payload_rows_per_dst=[10, 4, 14, 34, 9444, 16382, 8496, 16376] dup_rows_per_dst=[0, 0, 0, 2, 830, 32226, 0, 14486] combine_rows_per_src=[10, 4, 14, 34, 9444, 16382, 8496, 16376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 2, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92376 payload_rows_remote=75992 dup_rows=5928 dup_rows_remote=5926 dedup_ratio=0.060303 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 0, 2, 7927, 0, 2, 0, 5, 0, 0, 0, 4, 0, 0, 3838, 0, 0, 0, 0, 2670, 0, 0, 0, 0, 8187, 8183, 0, 4, 1, 0, 3828, 2, 0, 0, 0, 0, 4, 1, 0, 0, 0, 0, 6300, 8188, 0, 0] payload_rows_per_dst=[16376, 16384, 15860, 16376, 11750, 8, 15622, 0] dup_rows_per_dst=[0, 2, 2, 8, 1266, 2, 4648, 0] combine_rows_per_src=[16376, 16384, 15860, 16376, 11750, 8, 15622, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=52738 payload_rows_remote=45042 dup_rows=45566 dup_rows_remote=45520 dedup_ratio=0.463521 router_tokens_per_expert=[0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 5, 15, 0, 0, 0, 2, 0, 9, 7874, 1, 0, 0, 8, 0, 0, 0, 16, 0, 0, 3996, 1, 0, 0, 0, 2762, 1, 0, 0, 0, 8170, 8168, 3, 16, 8, 0, 3835, 25, 0, 0, 0, 0, 10, 1, 2, 0, 0, 0, 6038, 8178, 0, 1] payload_rows_per_dst=[14, 30, 24, 48, 12184, 16382, 7696, 16360] dup_rows_per_dst=[0, 10, 0, 0, 1336, 32096, 46, 12078] combine_rows_per_src=[14, 30, 24, 48, 12184, 16382, 7696, 16360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=326746112 dispatch_meta_bytes=325112 combine_payload_bytes=326746112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=327071224 ingress_bytes=361349120 link_bytes=361349120 token_bytes=653817336 weight_movement_bytes=34603008 total_nvlink_bytes=688420344 weight_to_token_ratio=0.052925 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=45312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.359727648e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93598 payload_rows_remote=77214 dup_rows=4706 dup_rows_remote=1038 dedup_ratio=0.047872 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 650, 0, 0, 0, 0, 0, 1834, 0, 0, 0, 0, 0, 0, 0, 0, 8044, 0, 0, 0, 139, 0, 0, 0, 0, 0, 0, 0, 0, 490, 0, 0, 0, 8017, 8191, 5200, 0, 0, 0, 0, 0, 277, 0, 8192, 0, 0, 0, 8118, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16382, 11556, 16384, 16102, 16384, 0, 554, 16236] dup_rows_per_dst=[0, 144, 3668, 264, 630, 0, 0, 0] combine_rows_per_src=[16382, 11556, 16384, 16102, 16384, 0, 554, 16236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=201383936 dispatch_meta_bytes=325352 combine_payload_bytes=201383936 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=201709288 ingress_bytes=270589952 link_bytes=270589952 token_bytes=403093224 weight_movement_bytes=69206016 total_nvlink_bytes=472299240 weight_to_token_ratio=0.171687 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=31232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.018208205e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=389586944 dispatch_meta_bytes=386464 combine_payload_bytes=389586944 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=389973408 ingress_bytes=562601984 link_bytes=562601984 token_bytes=779560352 weight_movement_bytes=173015040 total_nvlink_bytes=952575392 weight_to_token_ratio=0.221939 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=16512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=2.117025973e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93410 payload_rows_remote=77560 dup_rows=4894 dup_rows_remote=4894 dedup_ratio=0.049784 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3797, 0, 0, 0, 0, 2795, 0, 0, 0, 0, 8192, 8185, 0, 0, 0, 0, 3745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6321, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 15850, 16370, 11980, 0, 14378, 2064] dup_rows_per_dst=[0, 0, 0, 0, 1204, 0, 3690, 0] combine_rows_per_src=[16384, 16384, 15850, 16370, 11980, 0, 14378, 2064] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=140836864 dispatch_meta_bytes=269768 combine_payload_bytes=140836864 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=141106632 ingress_bytes=244645888 link_bytes=244645888 token_bytes=281943496 weight_movement_bytes=103809024 total_nvlink_bytes=385752520 weight_to_token_ratio=0.368191 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=31616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.205827809e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=202399744 dispatch_meta_bytes=328032 combine_payload_bytes=202399744 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=202727776 ingress_bytes=271605760 link_bytes=271605760 token_bytes=405127520 weight_movement_bytes=69206016 total_nvlink_bytes=474333536 weight_to_token_ratio=0.170825 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.022030609e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=311263232 dispatch_meta_bytes=327672 combine_payload_bytes=311263232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=311590904 ingress_bytes=345866240 link_bytes=345866240 token_bytes=622854136 weight_movement_bytes=34603008 total_nvlink_bytes=657457144 weight_to_token_ratio=0.055556 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.301466817e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97038 payload_rows_remote=85122 dup_rows=1266 dup_rows_remote=0 dedup_ratio=0.012878 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 2521, 0, 0, 0, 0, 8192, 8188, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5990, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 15880, 16376, 11916, 0, 8118, 11980] dup_rows_per_dst=[0, 0, 0, 0, 1266, 0, 0, 0] combine_rows_per_src=[16384, 16384, 15880, 16376, 11916, 0, 8118, 11980] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=316268544 dispatch_meta_bytes=313008 combine_payload_bytes=316268544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=316581552 ingress_bytes=350871552 link_bytes=350871552 token_bytes=632850096 weight_movement_bytes=34603008 total_nvlink_bytes=667453104 weight_to_token_ratio=0.054678 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=34432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.320301403e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89186 payload_rows_remote=72802 dup_rows=9118 dup_rows_remote=9118 dedup_ratio=0.092753 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4183, 0, 0, 0, 0, 2491, 0, 0, 0, 0, 8192, 8183, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5907, 8191, 0, 0] payload_rows_per_dst=[16384, 16382, 16094, 16366, 8360, 0, 15600, 0] dup_rows_per_dst=[0, 0, 4046, 0, 694, 0, 4378, 0] combine_rows_per_src=[16384, 16382, 16094, 16366, 8360, 0, 15600, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97042 payload_rows_remote=95864 dup_rows=1262 dup_rows_remote=1232 dedup_ratio=0.012838 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3628, 0, 0, 0, 0, 2773, 0, 0, 0, 0, 8192, 8178, 0, 0, 0, 0, 3948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6296, 8192, 0, 0] payload_rows_per_dst=[16368, 16340, 14712, 16342, 11604, 1178, 7896, 12602] dup_rows_per_dst=[0, 0, 0, 0, 1198, 30, 0, 34] combine_rows_per_src=[16368, 16340, 14712, 16342, 11604, 1178, 7896, 12602] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97138 payload_rows_remote=80756 dup_rows=1166 dup_rows_remote=1166 dedup_ratio=0.011861 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 2565, 0, 0, 0, 0, 8192, 8191, 0, 0, 0, 0, 3868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6162, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 15822, 16382, 12106, 0, 7736, 12324] dup_rows_per_dst=[0, 0, 0, 0, 1166, 0, 0, 0] combine_rows_per_src=[16384, 16384, 15822, 16382, 12106, 0, 7736, 12324] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=184492032 dispatch_meta_bytes=362248 combine_payload_bytes=184492032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=184854280 ingress_bytes=219095040 link_bytes=219095040 token_bytes=369346312 weight_movement_bytes=34603008 total_nvlink_bytes=403949320 weight_to_token_ratio=0.093687 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=31616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.244369969e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=317685760 dispatch_meta_bytes=329816 combine_payload_bytes=317685760 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=318015576 ingress_bytes=386891776 link_bytes=386891776 token_bytes=635701336 weight_movement_bytes=69206016 total_nvlink_bytes=704907352 weight_to_token_ratio=0.108866 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=47104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.455842606e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=348659712 dispatch_meta_bytes=340488 combine_payload_bytes=348659712 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=349000200 ingress_bytes=417865728 link_bytes=417865728 token_bytes=697659912 weight_movement_bytes=69206016 total_nvlink_bytes=766865928 weight_to_token_ratio=0.099197 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=28032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.572395094e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=298196992 dispatch_meta_bytes=327680 combine_payload_bytes=298196992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=298524672 ingress_bytes=332800000 link_bytes=332800000 token_bytes=596721664 weight_movement_bytes=34603008 total_nvlink_bytes=631324672 weight_to_token_ratio=0.057989 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.252299607e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=392658944 dispatch_meta_bytes=388384 combine_payload_bytes=392658944 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=393047328 ingress_bytes=496467968 link_bytes=496467968 token_bytes=785706272 weight_movement_bytes=103809024 total_nvlink_bytes=889515296 weight_to_token_ratio=0.132122 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=16512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.868168995e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97038 payload_rows_remote=85122 dup_rows=1266 dup_rows_remote=0 dedup_ratio=0.012878 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 2521, 0, 0, 0, 0, 8192, 8188, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5990, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 15880, 16376, 11916, 0, 8118, 11980] dup_rows_per_dst=[0, 0, 0, 0, 1266, 0, 0, 0] combine_rows_per_src=[16384, 16384, 15880, 16376, 11916, 0, 8118, 11980] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=52738 payload_rows_remote=45042 dup_rows=45566 dup_rows_remote=45520 dedup_ratio=0.463521 router_tokens_per_expert=[0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 5, 15, 0, 0, 0, 2, 0, 9, 7874, 1, 0, 0, 8, 0, 0, 0, 16, 0, 0, 3996, 1, 0, 0, 0, 2762, 1, 0, 0, 0, 8170, 8168, 3, 16, 8, 0, 3835, 25, 0, 0, 0, 0, 10, 1, 2, 0, 0, 0, 6038, 8178, 0, 1] payload_rows_per_dst=[14, 30, 24, 48, 12184, 16382, 7696, 16360] dup_rows_per_dst=[0, 10, 0, 0, 1336, 32096, 46, 12078] combine_rows_per_src=[14, 30, 24, 48, 12184, 16382, 7696, 16360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97042 payload_rows_remote=95864 dup_rows=1262 dup_rows_remote=1232 dedup_ratio=0.012838 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3628, 0, 0, 0, 0, 2773, 0, 0, 0, 0, 8192, 8178, 0, 0, 0, 0, 3948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6296, 8192, 0, 0] payload_rows_per_dst=[16368, 16340, 14712, 16342, 11604, 1178, 7896, 12602] dup_rows_per_dst=[0, 0, 0, 0, 1198, 30, 0, 34] combine_rows_per_src=[16368, 16340, 14712, 16342, 11604, 1178, 7896, 12602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=50760 payload_rows_remote=34384 dup_rows=47544 dup_rows_remote=33058 dedup_ratio=0.483643 router_tokens_per_expert=[1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 7939, 0, 6, 0, 12, 0, 0, 0, 6, 0, 0, 3748, 1, 1, 1, 0, 2572, 0, 2, 0, 0, 8185, 8170, 0, 5, 5, 0, 4247, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 6055, 8186, 0, 0] payload_rows_per_dst=[10, 4, 14, 34, 9444, 16382, 8496, 16376] dup_rows_per_dst=[0, 0, 0, 2, 830, 32226, 0, 14486] combine_rows_per_src=[10, 4, 14, 34, 9444, 16382, 8496, 16376] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=330776576 dispatch_meta_bytes=327688 combine_payload_bytes=330776576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=331104264 ingress_bytes=365379584 link_bytes=365379584 token_bytes=661880840 weight_movement_bytes=34603008 total_nvlink_bytes=696483848 weight_to_token_ratio=0.052280 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.374893959e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87514 payload_rows_remote=71568 dup_rows=10790 dup_rows_remote=10186 dedup_ratio=0.109762 router_tokens_per_expert=[8186, 0, 5, 0, 6, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 7525, 7909, 1, 3049, 0, 8169, 1, 4, 0, 0, 0, 0, 0, 2, 4, 0, 0, 0, 1, 4, 0, 932, 0, 3748, 2, 0, 0, 0, 6, 4, 1010, 0, 30, 0, 0, 1, 0, 0, 1, 0, 0, 0, 74, 282, 0, 2, 6, 0, 2] payload_rows_per_dst=[12, 0, 6110, 16380, 16340, 16384, 16342, 15946] dup_rows_per_dst=[10, 0, 0, 4, 8084, 2088, 0, 604] combine_rows_per_src=[12, 0, 6110, 16380, 16340, 16384, 16342, 15946] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=59706 payload_rows_remote=45144 dup_rows=38598 dup_rows_remote=38598 dedup_ratio=0.392639 router_tokens_per_expert=[8187, 0, 3, 0, 4, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 1, 7280, 7792, 1, 3036, 0, 8170, 0, 3, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 2, 0, 0, 841, 0, 4100, 0, 1, 2, 0, 3, 4, 887, 0, 17, 0, 0, 1, 0, 2, 3, 0, 0, 2, 105, 510, 0, 1, 4, 0, 0] payload_rows_per_dst=[16384, 14562, 16378, 6, 9208, 1820, 106, 1242] dup_rows_per_dst=[22450, 0, 15460, 0, 678, 8, 0, 2] combine_rows_per_src=[16384, 14562, 16378, 6, 9208, 1820, 106, 1242] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92670 payload_rows_remote=76348 dup_rows=5634 dup_rows_remote=5634 dedup_ratio=0.057312 router_tokens_per_expert=[8174, 0, 14, 0, 15, 0, 0, 8177, 0, 0, 0, 0, 0, 7, 0, 7348, 7850, 4, 2964, 0, 8145, 0, 0, 0, 0, 0, 2, 0, 0, 12, 0, 0, 0, 4, 0, 0, 919, 0, 3929, 0, 2, 1, 0, 9, 34, 1003, 0, 41, 0, 0, 0, 2, 0, 14, 0, 0, 7, 75, 381, 0, 1, 14, 0, 4] payload_rows_per_dst=[36, 10058, 7828, 16372, 9792, 16376, 16322, 15886] dup_rows_per_dst=[22, 8, 1124, 4, 1540, 2158, 0, 778] combine_rows_per_src=[36, 10058, 7828, 16372, 9792, 16376, 16322, 15886] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=293142528 dispatch_meta_bytes=327016 combine_payload_bytes=293142528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293469544 ingress_bytes=327745536 link_bytes=327745536 token_bytes=586612072 weight_movement_bytes=34603008 total_nvlink_bytes=621215080 weight_to_token_ratio=0.058988 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=45440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.233280066e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92046 payload_rows_remote=88078 dup_rows=6258 dup_rows_remote=6258 dedup_ratio=0.063660 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 7320, 7957, 0, 2516, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 880, 0, 4628, 0, 0, 0, 0, 0, 0, 886, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 313, 0, 0, 0, 0, 0] payload_rows_per_dst=[16344, 14640, 3968, 16382, 10338, 1856, 16382, 12136] dup_rows_per_dst=[5030, 0, 0, 2, 678, 4, 0, 544] combine_rows_per_src=[16344, 14640, 3968, 16382, 10338, 1856, 16382, 12136] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93410 payload_rows_remote=77560 dup_rows=4894 dup_rows_remote=4894 dedup_ratio=0.049784 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3797, 0, 0, 0, 0, 2795, 0, 0, 0, 0, 8192, 8185, 0, 0, 0, 0, 3745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6321, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 15850, 16370, 11980, 0, 14378, 2064] dup_rows_per_dst=[0, 0, 0, 0, 1204, 0, 3690, 0] combine_rows_per_src=[16384, 16384, 15850, 16370, 11980, 0, 14378, 2064] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=348659712 dispatch_meta_bytes=340488 combine_payload_bytes=348659712 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=349000200 ingress_bytes=417865728 link_bytes=417865728 token_bytes=697659912 weight_movement_bytes=69206016 total_nvlink_bytes=766865928 weight_to_token_ratio=0.099197 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=28032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.572395094e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=184492032 dispatch_meta_bytes=362248 combine_payload_bytes=184492032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=184854280 ingress_bytes=219095040 link_bytes=219095040 token_bytes=369346312 weight_movement_bytes=34603008 total_nvlink_bytes=403949320 weight_to_token_ratio=0.093687 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=31616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.244369969e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=392658944 dispatch_meta_bytes=388384 combine_payload_bytes=392658944 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=393047328 ingress_bytes=496467968 link_bytes=496467968 token_bytes=785706272 weight_movement_bytes=103809024 total_nvlink_bytes=889515296 weight_to_token_ratio=0.132122 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=16512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.868168995e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=140836864 dispatch_meta_bytes=269768 combine_payload_bytes=140836864 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=141106632 ingress_bytes=244645888 link_bytes=244645888 token_bytes=281943496 weight_movement_bytes=103809024 total_nvlink_bytes=385752520 weight_to_token_ratio=0.368191 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=31616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.205827809e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95004 payload_rows_remote=85542 dup_rows=3300 dup_rows_remote=2644 dedup_ratio=0.033569 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7332, 7910, 0, 2925, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 838, 0, 4221, 0, 0, 0, 0, 0, 0, 919, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 350, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 14664, 5850, 16384, 9462, 16384, 16384, 15876] dup_rows_per_dst=[0, 0, 0, 0, 656, 1866, 0, 778] combine_rows_per_src=[0, 14664, 5850, 16384, 9462, 16384, 16384, 15876] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=312721408 dispatch_meta_bytes=327928 combine_payload_bytes=312721408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=313049336 ingress_bytes=347324416 link_bytes=347324416 token_bytes=625770744 weight_movement_bytes=34603008 total_nvlink_bytes=660373752 weight_to_token_ratio=0.055297 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.306953815e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94706 payload_rows_remote=78322 dup_rows=3598 dup_rows_remote=1436 dedup_ratio=0.036601 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7393, 7848, 0, 2705, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 895, 0, 4184, 0, 0, 0, 0, 0, 0, 1071, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 340, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 14786, 5410, 16384, 9496, 16384, 16384, 15862] dup_rows_per_dst=[0, 0, 0, 2, 662, 2162, 0, 772] combine_rows_per_src=[0, 14786, 5410, 16384, 9496, 16384, 16384, 15862] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=184909824 dispatch_meta_bytes=334968 combine_payload_bytes=184909824 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=185244792 ingress_bytes=254115840 link_bytes=254115840 token_bytes=370154616 weight_movement_bytes=69206016 total_nvlink_bytes=439360632 weight_to_token_ratio=0.186965 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.562174479e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=61094 payload_rows_remote=44710 dup_rows=37210 dup_rows_remote=16486 dedup_ratio=0.378520 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7389, 7906, 0, 2170, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 951, 0, 4580, 0, 0, 0, 0, 0, 0, 1065, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 378, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 14778, 16384, 0, 10388, 2166, 0, 994] dup_rows_per_dst=[20724, 0, 15812, 0, 674, 0, 0, 0] combine_rows_per_src=[16384, 14778, 16384, 0, 10388, 2166, 0, 994] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=317685760 dispatch_meta_bytes=329816 combine_payload_bytes=317685760 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=318015576 ingress_bytes=386891776 link_bytes=386891776 token_bytes=635701336 weight_movement_bytes=69206016 total_nvlink_bytes=704907352 weight_to_token_ratio=0.108866 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=47104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.455842606e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94404 payload_rows_remote=78020 dup_rows=3900 dup_rows_remote=3900 dedup_ratio=0.039673 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 7404, 7900, 0, 2444, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 857, 0, 4297, 0, 0, 0, 0, 0, 0, 1094, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 429, 0, 0, 0, 0, 0] payload_rows_per_dst=[908, 14808, 3980, 16384, 9652, 16384, 16384, 15904] dup_rows_per_dst=[0, 0, 0, 0, 656, 2264, 0, 980] combine_rows_per_src=[908, 14808, 3980, 16384, 9652, 16384, 16384, 15904] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=360767488 dispatch_meta_bytes=377344 combine_payload_bytes=360767488 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=361144832 ingress_bytes=499179520 link_bytes=499179520 token_bytes=721912320 weight_movement_bytes=138412032 total_nvlink_bytes=860324352 weight_to_token_ratio=0.191730 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=17792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.878372347e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=350380032 dispatch_meta_bytes=352744 combine_payload_bytes=350380032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=350732776 ingress_bytes=384983040 link_bytes=384983040 token_bytes=701112808 weight_movement_bytes=34603008 total_nvlink_bytes=735715816 weight_to_token_ratio=0.049354 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=33792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.448660186e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=320806912 dispatch_meta_bytes=319032 combine_payload_bytes=320806912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=321125944 ingress_bytes=355409920 link_bytes=355409920 token_bytes=641932856 weight_movement_bytes=34603008 total_nvlink_bytes=676535864 weight_to_token_ratio=0.053904 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=41088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.337378916e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95004 payload_rows_remote=85542 dup_rows=3300 dup_rows_remote=2644 dedup_ratio=0.033569 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7332, 7910, 0, 2925, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 838, 0, 4221, 0, 0, 0, 0, 0, 0, 919, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 350, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 14664, 5850, 16384, 9462, 16384, 16384, 15876] dup_rows_per_dst=[0, 0, 0, 0, 656, 1866, 0, 778] combine_rows_per_src=[0, 14664, 5850, 16384, 9462, 16384, 16384, 15876] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92670 payload_rows_remote=76348 dup_rows=5634 dup_rows_remote=5634 dedup_ratio=0.057312 router_tokens_per_expert=[8174, 0, 14, 0, 15, 0, 0, 8177, 0, 0, 0, 0, 0, 7, 0, 7348, 7850, 4, 2964, 0, 8145, 0, 0, 0, 0, 0, 2, 0, 0, 12, 0, 0, 0, 4, 0, 0, 919, 0, 3929, 0, 2, 1, 0, 9, 34, 1003, 0, 41, 0, 0, 0, 2, 0, 14, 0, 0, 7, 75, 381, 0, 1, 14, 0, 4] payload_rows_per_dst=[36, 10058, 7828, 16372, 9792, 16376, 16322, 15886] dup_rows_per_dst=[22, 8, 1124, 4, 1540, 2158, 0, 778] combine_rows_per_src=[36, 10058, 7828, 16372, 9792, 16376, 16322, 15886] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=183132160 dispatch_meta_bytes=244784 combine_payload_bytes=183132160 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=183376944 ingress_bytes=286941184 link_bytes=286941184 token_bytes=366509104 weight_movement_bytes=103809024 total_nvlink_bytes=470318128 weight_to_token_ratio=0.283237 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=24576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.079736575e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87514 payload_rows_remote=71568 dup_rows=10790 dup_rows_remote=10186 dedup_ratio=0.109762 router_tokens_per_expert=[8186, 0, 5, 0, 6, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 7525, 7909, 1, 3049, 0, 8169, 1, 4, 0, 0, 0, 0, 0, 2, 4, 0, 0, 0, 1, 4, 0, 932, 0, 3748, 2, 0, 0, 0, 6, 4, 1010, 0, 30, 0, 0, 1, 0, 0, 1, 0, 0, 0, 74, 282, 0, 2, 6, 0, 2] payload_rows_per_dst=[12, 0, 6110, 16380, 16340, 16384, 16342, 15946] dup_rows_per_dst=[10, 0, 0, 4, 8084, 2088, 0, 604] combine_rows_per_src=[12, 0, 6110, 16380, 16340, 16384, 16342, 15946] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=319569920 dispatch_meta_bytes=327680 combine_payload_bytes=319569920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=319897600 ingress_bytes=354172928 link_bytes=354172928 token_bytes=639467520 weight_movement_bytes=34603008 total_nvlink_bytes=674070528 weight_to_token_ratio=0.054112 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.332724215e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=65284 payload_rows_remote=48900 dup_rows=33020 dup_rows_remote=16502 dedup_ratio=0.335897 router_tokens_per_expert=[0, 0, 4, 0, 1, 0, 0, 1, 0, 0, 0, 0, 5, 0, 6917, 2, 0, 0, 8183, 0, 0, 3, 3, 0, 8011, 4, 0, 0, 0, 8184, 0, 0, 0, 0, 8186, 5, 0, 0, 0, 2, 0, 0, 0, 3, 0, 2, 1290, 0, 0, 6, 0, 0, 0, 0, 0, 75, 2, 0, 8186, 0, 6, 71, 0, 0] payload_rows_per_dst=[12, 1398, 1796, 16378, 10680, 2590, 16046, 16384] dup_rows_per_dst=[0, 0, 0, 16364, 0, 0, 138, 16518] combine_rows_per_src=[12, 1398, 1796, 16378, 10680, 2590, 16046, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=65306 payload_rows_remote=49396 dup_rows=32998 dup_rows_remote=32878 dedup_ratio=0.335673 router_tokens_per_expert=[0, 0, 14, 0, 3, 0, 0, 0, 1, 0, 0, 0, 1, 0, 7045, 2, 0, 0, 8171, 0, 0, 0, 9, 0, 7940, 14, 0, 0, 0, 8176, 0, 0, 0, 2, 8174, 17, 0, 0, 2, 0, 3, 0, 0, 15, 1, 1, 1211, 0, 0, 6, 1, 0, 0, 0, 0, 68, 2, 0, 8175, 0, 17, 81, 0, 0] payload_rows_per_dst=[34, 14092, 18, 16378, 38, 2454, 15910, 16382] dup_rows_per_dst=[0, 6, 0, 16344, 4, 8, 120, 16516] combine_rows_per_src=[34, 14092, 18, 16378, 38, 2454, 15910, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94706 payload_rows_remote=78322 dup_rows=3598 dup_rows_remote=1436 dedup_ratio=0.036601 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7393, 7848, 0, 2705, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 895, 0, 4184, 0, 0, 0, 0, 0, 0, 1071, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 340, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 14786, 5410, 16384, 9496, 16384, 16384, 15862] dup_rows_per_dst=[0, 0, 0, 2, 662, 2162, 0, 772] combine_rows_per_src=[0, 14786, 5410, 16384, 9496, 16384, 16384, 15862] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95168 payload_rows_remote=81836 dup_rows=3136 dup_rows_remote=3136 dedup_ratio=0.031901 router_tokens_per_expert=[0, 0, 3, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 6662, 0, 2, 0, 8186, 0, 0, 0, 2, 0, 7943, 1, 0, 0, 0, 8187, 0, 0, 0, 0, 8187, 5, 0, 0, 1, 0, 0, 0, 1, 2, 0, 0, 1564, 0, 1, 3, 0, 0, 0, 1, 0, 118, 1, 0, 8187, 0, 5, 86, 0, 0] payload_rows_per_dst=[16380, 13332, 16380, 14978, 16384, 16378, 1154, 182] dup_rows_per_dst=[0, 0, 0, 0, 2, 3130, 2, 2] combine_rows_per_src=[16380, 13332, 16380, 14978, 16384, 16378, 1154, 182] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=200294400 dispatch_meta_bytes=261608 combine_payload_bytes=200294400 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=200556008 ingress_bytes=269500416 link_bytes=269500416 token_bytes=400850408 weight_movement_bytes=69206016 total_nvlink_bytes=470056424 weight_to_token_ratio=0.172648 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=26880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.014108369e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95904 payload_rows_remote=79520 dup_rows=2400 dup_rows_remote=2400 dedup_ratio=0.024414 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7106, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8008, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1140, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 8192, 0, 0, 70, 0, 0] payload_rows_per_dst=[16384, 14212, 16384, 0, 16384, 16384, 16016, 140] dup_rows_per_dst=[0, 0, 0, 0, 0, 2280, 120, 0] combine_rows_per_src=[16384, 14212, 16384, 0, 16384, 16384, 16016, 140] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=350380032 dispatch_meta_bytes=352744 combine_payload_bytes=350380032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=350732776 ingress_bytes=384983040 link_bytes=384983040 token_bytes=701112808 weight_movement_bytes=34603008 total_nvlink_bytes=735715816 weight_to_token_ratio=0.049354 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=33792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.448660186e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=312721408 dispatch_meta_bytes=327928 combine_payload_bytes=312721408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=313049336 ingress_bytes=347324416 link_bytes=347324416 token_bytes=625770744 weight_movement_bytes=34603008 total_nvlink_bytes=660373752 weight_to_token_ratio=0.055297 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.306953815e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92046 payload_rows_remote=88078 dup_rows=6258 dup_rows_remote=6258 dedup_ratio=0.063660 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 7320, 7957, 0, 2516, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 880, 0, 4628, 0, 0, 0, 0, 0, 0, 886, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 313, 0, 0, 0, 0, 0] payload_rows_per_dst=[16344, 14640, 3968, 16382, 10338, 1856, 16382, 12136] dup_rows_per_dst=[5030, 0, 0, 2, 678, 4, 0, 544] combine_rows_per_src=[16344, 14640, 3968, 16382, 10338, 1856, 16382, 12136] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=293142528 dispatch_meta_bytes=327016 combine_payload_bytes=293142528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293469544 ingress_bytes=327745536 link_bytes=327745536 token_bytes=586612072 weight_movement_bytes=34603008 total_nvlink_bytes=621215080 weight_to_token_ratio=0.058988 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=45440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.233280066e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=320806912 dispatch_meta_bytes=319032 combine_payload_bytes=320806912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=321125944 ingress_bytes=355409920 link_bytes=355409920 token_bytes=641932856 weight_movement_bytes=34603008 total_nvlink_bytes=676535864 weight_to_token_ratio=0.053904 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=41088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.337378916e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96030 payload_rows_remote=79646 dup_rows=2274 dup_rows_remote=2274 dedup_ratio=0.023132 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6768, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8010, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1438, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 8192, 0, 0, 72, 0, 0] payload_rows_per_dst=[16384, 13536, 16384, 0, 16384, 12760, 16020, 4562] dup_rows_per_dst=[0, 0, 0, 0, 0, 2042, 192, 40] combine_rows_per_src=[16384, 13536, 16384, 0, 16384, 12760, 16020, 4562] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=202326016 dispatch_meta_bytes=329096 combine_payload_bytes=202326016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=202655112 ingress_bytes=236929024 link_bytes=236929024 token_bytes=404981128 weight_movement_bytes=34603008 total_nvlink_bytes=439584136 weight_to_token_ratio=0.085444 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=48384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.915448430e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87126 payload_rows_remote=84764 dup_rows=11178 dup_rows_remote=11178 dedup_ratio=0.113708 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7032, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 7965, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1181, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 8192, 0, 0, 76, 0, 0] payload_rows_per_dst=[16342, 14064, 14572, 1812, 5660, 2362, 15930, 16384] dup_rows_per_dst=[0, 0, 0, 42, 0, 0, 260, 10876] combine_rows_per_src=[16342, 14064, 14572, 1812, 5660, 2362, 15930, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=335200256 dispatch_meta_bytes=339888 combine_payload_bytes=335200256 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=335540144 ingress_bytes=404406272 link_bytes=404406272 token_bytes=670740400 weight_movement_bytes=69206016 total_nvlink_bytes=739946416 weight_to_token_ratio=0.103179 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521748244e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95772 payload_rows_remote=79388 dup_rows=2532 dup_rows_remote=2532 dedup_ratio=0.025757 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6914, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8004, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1266, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 8192, 0, 0, 78, 0, 0] payload_rows_per_dst=[16384, 13828, 16384, 16008, 16384, 16384, 244, 156] dup_rows_per_dst=[0, 0, 0, 0, 0, 2532, 0, 0] combine_rows_per_src=[16384, 13828, 16384, 16008, 16384, 16384, 244, 156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=360767488 dispatch_meta_bytes=377344 combine_payload_bytes=360767488 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=361144832 ingress_bytes=499179520 link_bytes=499179520 token_bytes=721912320 weight_movement_bytes=138412032 total_nvlink_bytes=860324352 weight_to_token_ratio=0.191730 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=17792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.878372347e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95458 payload_rows_remote=95458 dup_rows=2846 dup_rows_remote=2846 dedup_ratio=0.028951 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6918, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 7962, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1308, 0, 0, 0, 0, 0, 0, 0, 0, 114, 0, 0, 8192, 0, 0, 81, 0, 0] payload_rows_per_dst=[16384, 13836, 16384, 0, 16384, 16384, 15924, 162] dup_rows_per_dst=[0, 0, 2, 0, 0, 2616, 228, 0] combine_rows_per_src=[16384, 13836, 16384, 0, 16384, 16384, 15924, 162] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=325713920 dispatch_meta_bytes=327680 combine_payload_bytes=325713920 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=326041600 ingress_bytes=394919936 link_bytes=394919936 token_bytes=651755520 weight_movement_bytes=69206016 total_nvlink_bytes=720961536 weight_to_token_ratio=0.106184 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=48256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.486051926e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=326230016 dispatch_meta_bytes=327680 combine_payload_bytes=326230016 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=326557696 ingress_bytes=395436032 link_bytes=395436032 token_bytes=652787712 weight_movement_bytes=69206016 total_nvlink_bytes=721993728 weight_to_token_ratio=0.106016 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=43904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.487993953e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=347193344 dispatch_meta_bytes=383768 combine_payload_bytes=347193344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=347577112 ingress_bytes=381796352 link_bytes=381796352 token_bytes=694770456 weight_movement_bytes=34603008 total_nvlink_bytes=729373464 weight_to_token_ratio=0.049805 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=38784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.436668935e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=65306 payload_rows_remote=49396 dup_rows=32998 dup_rows_remote=32878 dedup_ratio=0.335673 router_tokens_per_expert=[0, 0, 14, 0, 3, 0, 0, 0, 1, 0, 0, 0, 1, 0, 7045, 2, 0, 0, 8171, 0, 0, 0, 9, 0, 7940, 14, 0, 0, 0, 8176, 0, 0, 0, 2, 8174, 17, 0, 0, 2, 0, 3, 0, 0, 15, 1, 1, 1211, 0, 0, 6, 1, 0, 0, 0, 0, 68, 2, 0, 8175, 0, 17, 81, 0, 0] payload_rows_per_dst=[34, 14092, 18, 16378, 38, 2454, 15910, 16382] dup_rows_per_dst=[0, 6, 0, 16344, 4, 8, 120, 16516] combine_rows_per_src=[34, 14092, 18, 16378, 38, 2454, 15910, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96030 payload_rows_remote=79646 dup_rows=2274 dup_rows_remote=2274 dedup_ratio=0.023132 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6768, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8010, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1438, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 8192, 0, 0, 72, 0, 0] payload_rows_per_dst=[16384, 13536, 16384, 0, 16384, 12760, 16020, 4562] dup_rows_per_dst=[0, 0, 0, 0, 0, 2042, 192, 40] combine_rows_per_src=[16384, 13536, 16384, 0, 16384, 12760, 16020, 4562] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=325173248 dispatch_meta_bytes=327680 combine_payload_bytes=325173248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=325500928 ingress_bytes=359776256 link_bytes=359776256 token_bytes=650674176 weight_movement_bytes=34603008 total_nvlink_bytes=685277184 weight_to_token_ratio=0.053180 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.353809087e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92504 payload_rows_remote=76292 dup_rows=5800 dup_rows_remote=5792 dedup_ratio=0.059001 router_tokens_per_expert=[0, 0, 0, 1, 0, 0, 0, 0, 0, 8183, 8101, 1052, 2, 0, 1, 1, 6, 0, 1235, 0, 6, 1008, 0, 7449, 0, 6262, 0, 141, 0, 0, 5, 0, 2, 0, 0, 0, 6538, 8177, 0, 0, 2, 0, 0, 0, 1, 3, 523, 436, 3, 0, 0, 0, 0, 5, 0, 0, 0, 1, 0, 0, 0, 0, 4, 4] payload_rows_per_dst=[2, 16372, 4494, 10538, 13490, 15030, 16366, 16212] dup_rows_per_dst=[0, 2106, 16, 116, 1752, 1798, 4, 8] combine_rows_per_src=[2, 16372, 4494, 10538, 13490, 15030, 16366, 16212] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=65284 payload_rows_remote=48900 dup_rows=33020 dup_rows_remote=16502 dedup_ratio=0.335897 router_tokens_per_expert=[0, 0, 4, 0, 1, 0, 0, 1, 0, 0, 0, 0, 5, 0, 6917, 2, 0, 0, 8183, 0, 0, 3, 3, 0, 8011, 4, 0, 0, 0, 8184, 0, 0, 0, 0, 8186, 5, 0, 0, 0, 2, 0, 0, 0, 3, 0, 2, 1290, 0, 0, 6, 0, 0, 0, 0, 0, 75, 2, 0, 8186, 0, 6, 71, 0, 0] payload_rows_per_dst=[12, 1398, 1796, 16378, 10680, 2590, 16046, 16384] dup_rows_per_dst=[0, 0, 0, 16364, 0, 0, 138, 16518] combine_rows_per_src=[12, 1398, 1796, 16378, 10680, 2590, 16046, 16384] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=390995968 dispatch_meta_bytes=393216 combine_payload_bytes=390995968 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=391389184 ingress_bytes=494804992 link_bytes=494804992 token_bytes=782385152 weight_movement_bytes=103809024 total_nvlink_bytes=886194176 weight_to_token_ratio=0.132683 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=17280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.861911350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93702 payload_rows_remote=77360 dup_rows=4602 dup_rows_remote=4596 dedup_ratio=0.046814 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 10, 0, 0, 8169, 8080, 1174, 2, 0, 9, 0, 16, 7, 1292, 0, 12, 986, 0, 7408, 2, 5963, 0, 141, 0, 0, 15, 0, 0, 0, 0, 0, 6607, 8154, 0, 0, 4, 0, 0, 0, 1, 2, 550, 497, 4, 0, 0, 1, 0, 15, 0, 0, 0, 0, 0, 1, 0, 1, 20, 8] payload_rows_per_dst=[32, 16352, 4592, 12076, 13214, 14916, 16342, 16178] dup_rows_per_dst=[0, 2346, 34, 166, 0, 2008, 6, 42] combine_rows_per_src=[32, 16352, 4592, 12076, 13214, 14916, 16342, 16178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87126 payload_rows_remote=84764 dup_rows=11178 dup_rows_remote=11178 dedup_ratio=0.113708 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7032, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 7965, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1181, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 8192, 0, 0, 76, 0, 0] payload_rows_per_dst=[16342, 14064, 14572, 1812, 5660, 2362, 15930, 16384] dup_rows_per_dst=[0, 0, 0, 42, 0, 0, 260, 10876] combine_rows_per_src=[16342, 14064, 14572, 1812, 5660, 2362, 15930, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79418 payload_rows_remote=64226 dup_rows=18886 dup_rows_remote=12918 dedup_ratio=0.192118 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 8090, 1222, 3, 0, 3, 0, 4, 1, 1525, 0, 1, 1085, 0, 7392, 0, 5864, 0, 203, 0, 0, 3, 0, 0, 0, 0, 0, 6386, 8184, 0, 0, 0, 0, 0, 0, 2, 0, 634, 351, 2, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 0, 1, 1, 3] payload_rows_per_dst=[16372, 15192, 16356, 11970, 16330, 1970, 48, 1180] dup_rows_per_dst=[0, 5968, 12736, 170, 0, 4, 4, 4] combine_rows_per_src=[16372, 15192, 16356, 11970, 16330, 1970, 48, 1180] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=312492032 dispatch_meta_bytes=328336 combine_payload_bytes=312492032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=312820368 ingress_bytes=347095040 link_bytes=347095040 token_bytes=625312400 weight_movement_bytes=34603008 total_nvlink_bytes=659915408 weight_to_token_ratio=0.055337 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.306090692e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88214 payload_rows_remote=72242 dup_rows=10090 dup_rows_remote=1118 dedup_ratio=0.102641 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 8108, 999, 0, 0, 0, 0, 0, 0, 1330, 0, 0, 1059, 0, 7517, 0, 6297, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 6405, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 510, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0] payload_rows_per_dst=[16378, 1998, 15972, 12704, 0, 8560, 16384, 16218] dup_rows_per_dst=[0, 0, 8972, 150, 0, 948, 0, 20] combine_rows_per_src=[16378, 1998, 15972, 12704, 0, 8560, 16384, 16218] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=202326016 dispatch_meta_bytes=329096 combine_payload_bytes=202326016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=202655112 ingress_bytes=236929024 link_bytes=236929024 token_bytes=404981128 weight_movement_bytes=34603008 total_nvlink_bytes=439584136 weight_to_token_ratio=0.085444 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=48384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.915448430e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=326230016 dispatch_meta_bytes=327680 combine_payload_bytes=326230016 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=326557696 ingress_bytes=395436032 link_bytes=395436032 token_bytes=652787712 weight_movement_bytes=69206016 total_nvlink_bytes=721993728 weight_to_token_ratio=0.106016 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=43904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.487993953e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=200294400 dispatch_meta_bytes=261608 combine_payload_bytes=200294400 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=200556008 ingress_bytes=269500416 link_bytes=269500416 token_bytes=400850408 weight_movement_bytes=69206016 total_nvlink_bytes=470056424 weight_to_token_ratio=0.172648 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=26880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.014108369e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96394 payload_rows_remote=83596 dup_rows=1910 dup_rows_remote=1910 dedup_ratio=0.019430 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8033, 1123, 0, 0, 0, 0, 0, 0, 1286, 0, 0, 1065, 0, 7739, 0, 6076, 0, 167, 0, 0, 0, 0, 0, 0, 0, 0, 6399, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 493, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0] payload_rows_per_dst=[16382, 2246, 4682, 12344, 12798, 15492, 16382, 16068] dup_rows_per_dst=[0, 0, 20, 142, 0, 1746, 0, 2] combine_rows_per_src=[16382, 2246, 4682, 12344, 12798, 15492, 16382, 16068] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=316866560 dispatch_meta_bytes=327824 combine_payload_bytes=316866560 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=317194384 ingress_bytes=351469568 link_bytes=351469568 token_bytes=634060944 weight_movement_bytes=34603008 total_nvlink_bytes=668663952 weight_to_token_ratio=0.054574 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.322551689e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95904 payload_rows_remote=79520 dup_rows=2400 dup_rows_remote=2400 dedup_ratio=0.024414 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7106, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8008, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1140, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 8192, 0, 0, 70, 0, 0] payload_rows_per_dst=[16384, 14212, 16384, 0, 16384, 16384, 16016, 140] dup_rows_per_dst=[0, 0, 0, 0, 0, 2280, 120, 0] combine_rows_per_src=[16384, 14212, 16384, 0, 16384, 16384, 16016, 140] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=347193344 dispatch_meta_bytes=383768 combine_payload_bytes=347193344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=347577112 ingress_bytes=381796352 link_bytes=381796352 token_bytes=694770456 weight_movement_bytes=34603008 total_nvlink_bytes=729373464 weight_to_token_ratio=0.049805 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=38784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.436668935e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=263069696 dispatch_meta_bytes=308576 combine_payload_bytes=263069696 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=263378272 ingress_bytes=366878720 link_bytes=366878720 token_bytes=526447968 weight_movement_bytes=103809024 total_nvlink_bytes=630256992 weight_to_token_ratio=0.197188 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.380535087e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96394 payload_rows_remote=81300 dup_rows=1910 dup_rows_remote=196 dedup_ratio=0.019430 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8105, 1206, 0, 0, 0, 0, 0, 0, 1247, 0, 0, 1037, 0, 7492, 0, 6210, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0, 6351, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 607, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 2412, 4554, 12654, 12702, 15094, 16384, 16210] dup_rows_per_dst=[0, 0, 14, 182, 0, 1714, 0, 0] combine_rows_per_src=[16384, 2412, 4554, 12654, 12702, 15094, 16384, 16210] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79720 payload_rows_remote=63346 dup_rows=18584 dup_rows_remote=18584 dedup_ratio=0.189046 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 8097, 990, 0, 0, 0, 0, 0, 0, 1439, 0, 0, 1035, 0, 7571, 0, 6132, 0, 200, 0, 0, 0, 0, 0, 0, 0, 0, 6336, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 561, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0] payload_rows_per_dst=[16374, 16228, 16288, 12500, 16384, 1932, 0, 14] dup_rows_per_dst=[0, 14618, 3802, 164, 0, 0, 0, 0] combine_rows_per_src=[16374, 16228, 16288, 12500, 16384, 1932, 0, 14] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95988 payload_rows_remote=83722 dup_rows=2316 dup_rows_remote=2154 dedup_ratio=0.023560 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8072, 1233, 0, 0, 0, 0, 0, 0, 1278, 0, 0, 984, 0, 7513, 0, 6019, 0, 195, 0, 0, 0, 0, 0, 0, 0, 0, 6441, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 549, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0] payload_rows_per_dst=[16380, 2466, 5636, 12266, 11634, 15078, 16384, 16144] dup_rows_per_dst=[0, 0, 136, 162, 0, 2014, 0, 4] combine_rows_per_src=[16380, 2466, 5636, 12266, 11634, 15078, 16384, 16144] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=295903232 dispatch_meta_bytes=293440 combine_payload_bytes=295903232 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=296196672 ingress_bytes=365109248 link_bytes=365109248 token_bytes=592099904 weight_movement_bytes=69206016 total_nvlink_bytes=661305920 weight_to_token_ratio=0.116882 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=18688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.373876707e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=325713920 dispatch_meta_bytes=327680 combine_payload_bytes=325713920 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=326041600 ingress_bytes=394919936 link_bytes=394919936 token_bytes=651755520 weight_movement_bytes=69206016 total_nvlink_bytes=720961536 weight_to_token_ratio=0.106184 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=48256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.486051926e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=342409216 dispatch_meta_bytes=342024 combine_payload_bytes=342409216 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=342751240 ingress_bytes=480821248 link_bytes=480821248 token_bytes=685160456 weight_movement_bytes=138412032 total_nvlink_bytes=823572488 weight_to_token_ratio=0.202014 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=31744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.809291647e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=333004800 dispatch_meta_bytes=325984 combine_payload_bytes=333004800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=333330784 ingress_bytes=367607808 link_bytes=367607808 token_bytes=666335584 weight_movement_bytes=34603008 total_nvlink_bytes=700938592 weight_to_token_ratio=0.051930 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=41472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.383278587e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93702 payload_rows_remote=77360 dup_rows=4602 dup_rows_remote=4596 dedup_ratio=0.046814 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 10, 0, 0, 8169, 8080, 1174, 2, 0, 9, 0, 16, 7, 1292, 0, 12, 986, 0, 7408, 2, 5963, 0, 141, 0, 0, 15, 0, 0, 0, 0, 0, 6607, 8154, 0, 0, 4, 0, 0, 0, 1, 2, 550, 497, 4, 0, 0, 1, 0, 15, 0, 0, 0, 0, 0, 1, 0, 1, 20, 8] payload_rows_per_dst=[32, 16352, 4592, 12076, 13214, 14916, 16342, 16178] dup_rows_per_dst=[0, 2346, 34, 166, 0, 2008, 6, 42] combine_rows_per_src=[32, 16352, 4592, 12076, 13214, 14916, 16342, 16178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96394 payload_rows_remote=83596 dup_rows=1910 dup_rows_remote=1910 dedup_ratio=0.019430 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8033, 1123, 0, 0, 0, 0, 0, 0, 1286, 0, 0, 1065, 0, 7739, 0, 6076, 0, 167, 0, 0, 0, 0, 0, 0, 0, 0, 6399, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 493, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0] payload_rows_per_dst=[16382, 2246, 4682, 12344, 12798, 15492, 16382, 16068] dup_rows_per_dst=[0, 0, 20, 142, 0, 1746, 0, 2] combine_rows_per_src=[16382, 2246, 4682, 12344, 12798, 15492, 16382, 16068] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=259465216 dispatch_meta_bytes=327720 combine_payload_bytes=259465216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=259792936 ingress_bytes=294068224 link_bytes=294068224 token_bytes=519258152 weight_movement_bytes=34603008 total_nvlink_bytes=553861160 weight_to_token_ratio=0.066639 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.106555052e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=342925312 dispatch_meta_bytes=343504 combine_payload_bytes=342925312 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=343268816 ingress_bytes=377528320 link_bytes=377528320 token_bytes=686194128 weight_movement_bytes=34603008 total_nvlink_bytes=720797136 weight_to_token_ratio=0.050427 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=47744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.420608675e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77000 payload_rows_remote=60632 dup_rows=21304 dup_rows_remote=17550 dedup_ratio=0.216715 router_tokens_per_expert=[0, 3, 0, 0, 0, 0, 0, 895, 491, 5, 1, 0, 0, 0, 1, 1242, 0, 5, 3897, 0, 0, 0, 0, 0, 0, 3, 4158, 0, 0, 0, 0, 4, 7793, 3, 1, 0, 0, 8184, 1, 8184, 0, 2, 0, 8183, 8, 6, 87, 0, 2, 3719, 0, 0, 4, 0, 0, 2, 0, 0, 608, 0, 0, 1553, 0, 107] payload_rows_per_dst=[1796, 2500, 7804, 8330, 16374, 16382, 7446, 16368] dup_rows_per_dst=[0, 980, 0, 0, 16372, 190, 8, 3754] combine_rows_per_src=[1796, 2500, 7804, 8330, 16374, 16382, 7446, 16368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79954 payload_rows_remote=72908 dup_rows=18350 dup_rows_remote=18328 dedup_ratio=0.186666 router_tokens_per_expert=[0, 2, 0, 0, 0, 0, 4, 1127, 531, 16, 0, 0, 0, 0, 0, 1050, 0, 5, 4239, 0, 0, 0, 5, 0, 2, 18, 3908, 0, 0, 0, 0, 7, 7752, 14, 0, 0, 0, 8173, 0, 8172, 0, 13, 0, 8168, 15, 12, 49, 0, 1, 3509, 0, 1, 12, 0, 0, 11, 0, 2, 698, 0, 0, 1536, 0, 100] payload_rows_per_dst=[2266, 2142, 11558, 7864, 16360, 16370, 7046, 16348] dup_rows_per_dst=[0, 1052, 2994, 6, 10304, 144, 22, 3828] combine_rows_per_src=[2266, 2142, 11558, 7864, 16360, 16370, 7046, 16348] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79152 payload_rows_remote=62770 dup_rows=19152 dup_rows_remote=15884 dedup_ratio=0.194824 router_tokens_per_expert=[0, 2, 0, 0, 0, 0, 1, 1004, 546, 5, 0, 0, 0, 0, 0, 1088, 0, 1, 4062, 0, 0, 0, 2, 0, 0, 6, 4169, 0, 0, 0, 0, 1, 7757, 3, 0, 0, 0, 8187, 0, 8186, 0, 3, 1, 8186, 4, 3, 55, 0, 0, 3778, 0, 0, 3, 1, 0, 2, 0, 2, 619, 0, 0, 1367, 0, 108] payload_rows_per_dst=[16380, 16382, 8130, 16376, 4642, 118, 13016, 4108] dup_rows_per_dst=[2008, 3268, 0, 8348, 0, 14, 5430, 84] combine_rows_per_src=[16380, 16382, 8130, 16376, 4642, 118, 13016, 4108] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=248348672 dispatch_meta_bytes=312728 combine_payload_bytes=248348672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=248661400 ingress_bytes=282951680 link_bytes=282951680 token_bytes=497010072 weight_movement_bytes=34603008 total_nvlink_bytes=531613080 weight_to_token_ratio=0.069622 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=32000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.064724392e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85488 payload_rows_remote=77142 dup_rows=12816 dup_rows_remote=12816 dedup_ratio=0.130371 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1093, 501, 0, 0, 0, 0, 0, 0, 1228, 0, 0, 4173, 0, 0, 0, 0, 0, 0, 0, 4119, 0, 0, 0, 0, 0, 7805, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 94, 0, 0, 3452, 0, 0, 0, 1, 0, 0, 0, 0, 650, 0, 0, 1377, 0, 83] payload_rows_per_dst=[16384, 16384, 8346, 8240, 1306, 15032, 15610, 4186] dup_rows_per_dst=[2186, 3458, 0, 48, 0, 184, 6906, 34] combine_rows_per_src=[16384, 16384, 8346, 8240, 1306, 15032, 15610, 4186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92504 payload_rows_remote=76292 dup_rows=5800 dup_rows_remote=5792 dedup_ratio=0.059001 router_tokens_per_expert=[0, 0, 0, 1, 0, 0, 0, 0, 0, 8183, 8101, 1052, 2, 0, 1, 1, 6, 0, 1235, 0, 6, 1008, 0, 7449, 0, 6262, 0, 141, 0, 0, 5, 0, 2, 0, 0, 0, 6538, 8177, 0, 0, 2, 0, 0, 0, 1, 3, 523, 436, 3, 0, 0, 0, 0, 5, 0, 0, 0, 1, 0, 0, 0, 0, 4, 4] payload_rows_per_dst=[2, 16372, 4494, 10538, 13490, 15030, 16366, 16212] dup_rows_per_dst=[0, 2106, 16, 116, 1752, 1798, 4, 8] combine_rows_per_src=[2, 16372, 4494, 10538, 13490, 15030, 16366, 16212] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96394 payload_rows_remote=81300 dup_rows=1910 dup_rows_remote=196 dedup_ratio=0.019430 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8105, 1206, 0, 0, 0, 0, 0, 0, 1247, 0, 0, 1037, 0, 7492, 0, 6210, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0, 6351, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 607, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 2412, 4554, 12654, 12702, 15094, 16384, 16210] dup_rows_per_dst=[0, 0, 14, 182, 0, 1714, 0, 0] combine_rows_per_src=[16384, 2412, 4554, 12654, 12702, 15094, 16384, 16210] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=316866560 dispatch_meta_bytes=327824 combine_payload_bytes=316866560 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=317194384 ingress_bytes=351469568 link_bytes=351469568 token_bytes=634060944 weight_movement_bytes=34603008 total_nvlink_bytes=668663952 weight_to_token_ratio=0.054574 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.322551689e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79152 payload_rows_remote=62770 dup_rows=19152 dup_rows_remote=15884 dedup_ratio=0.194824 router_tokens_per_expert=[0, 2, 0, 0, 0, 0, 1, 1004, 546, 5, 0, 0, 0, 0, 0, 1088, 0, 1, 4062, 0, 0, 0, 2, 0, 0, 6, 4169, 0, 0, 0, 0, 1, 7757, 3, 0, 0, 0, 8187, 0, 8186, 0, 3, 1, 8186, 4, 3, 55, 0, 0, 3778, 0, 0, 3, 1, 0, 2, 0, 2, 619, 0, 0, 1367, 0, 108] payload_rows_per_dst=[16380, 16382, 8130, 16376, 4642, 118, 13016, 4108] dup_rows_per_dst=[2008, 3268, 0, 8348, 0, 14, 5430, 84] combine_rows_per_src=[16380, 16382, 8130, 16376, 4642, 118, 13016, 4108] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=342409216 dispatch_meta_bytes=342024 combine_payload_bytes=342409216 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=342751240 ingress_bytes=480821248 link_bytes=480821248 token_bytes=685160456 weight_movement_bytes=138412032 total_nvlink_bytes=823572488 weight_to_token_ratio=0.202014 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=31744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.809291647e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88214 payload_rows_remote=72242 dup_rows=10090 dup_rows_remote=1118 dedup_ratio=0.102641 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 8108, 999, 0, 0, 0, 0, 0, 0, 1330, 0, 0, 1059, 0, 7517, 0, 6297, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 6405, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 510, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0] payload_rows_per_dst=[16378, 1998, 15972, 12704, 0, 8560, 16384, 16218] dup_rows_per_dst=[0, 0, 8972, 150, 0, 948, 0, 20] combine_rows_per_src=[16378, 1998, 15972, 12704, 0, 8560, 16384, 16218] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=312492032 dispatch_meta_bytes=328336 combine_payload_bytes=312492032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=312820368 ingress_bytes=347095040 link_bytes=347095040 token_bytes=625312400 weight_movement_bytes=34603008 total_nvlink_bytes=659915408 weight_to_token_ratio=0.055337 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.306090692e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85640 payload_rows_remote=84724 dup_rows=12664 dup_rows_remote=12664 dedup_ratio=0.128825 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1257, 449, 0, 0, 0, 0, 0, 0, 1054, 0, 0, 4183, 0, 0, 0, 0, 0, 0, 0, 3924, 0, 0, 0, 0, 0, 7865, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 92, 0, 0, 3658, 0, 0, 0, 5, 0, 0, 0, 0, 528, 0, 0, 1476, 0, 85] payload_rows_per_dst=[15626, 8022, 13134, 7848, 916, 16384, 7326, 16384] dup_rows_per_dst=[2356, 1756, 4844, 0, 0, 184, 0, 3524] combine_rows_per_src=[15626, 8022, 13134, 7848, 916, 16384, 7326, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=298631168 dispatch_meta_bytes=364944 combine_payload_bytes=298631168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=298996112 ingress_bytes=333234176 link_bytes=333234176 token_bytes=597627280 weight_movement_bytes=34603008 total_nvlink_bytes=632230288 weight_to_token_ratio=0.057901 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=28800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.253933377e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=333004800 dispatch_meta_bytes=325984 combine_payload_bytes=333004800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=333330784 ingress_bytes=367607808 link_bytes=367607808 token_bytes=666335584 weight_movement_bytes=34603008 total_nvlink_bytes=700938592 weight_to_token_ratio=0.051930 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=41472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.383278587e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=257105920 dispatch_meta_bytes=314616 combine_payload_bytes=257105920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=257420536 ingress_bytes=291708928 link_bytes=291708928 token_bytes=514526456 weight_movement_bytes=34603008 total_nvlink_bytes=549129464 weight_to_token_ratio=0.067252 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=36224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.097677212e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85726 payload_rows_remote=69342 dup_rows=12578 dup_rows_remote=12448 dedup_ratio=0.127950 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1170, 531, 0, 0, 0, 0, 0, 0, 1229, 0, 0, 4035, 0, 0, 0, 0, 0, 0, 0, 4126, 0, 0, 0, 0, 0, 7773, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 65, 0, 0, 3570, 0, 0, 0, 0, 0, 0, 0, 0, 651, 0, 0, 1336, 0, 90] payload_rows_per_dst=[2340, 2458, 16384, 8252, 16384, 16384, 7140, 16384] dup_rows_per_dst=[0, 1062, 8070, 0, 0, 130, 0, 3316] combine_rows_per_src=[2340, 2458, 16384, 8252, 16384, 16384, 7140, 16384] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85072 payload_rows_remote=68688 dup_rows=13232 dup_rows_remote=11184 dedup_ratio=0.134603 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1024, 457, 0, 0, 0, 0, 0, 0, 1014, 0, 0, 4407, 0, 0, 0, 0, 0, 0, 0, 4104, 0, 0, 0, 0, 0, 7823, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 80, 0, 0, 3559, 0, 0, 0, 0, 0, 0, 0, 0, 572, 0, 0, 1451, 0, 85] payload_rows_per_dst=[16384, 16384, 8814, 16384, 15646, 160, 7118, 4182] dup_rows_per_dst=[2048, 2942, 0, 8208, 0, 0, 0, 34] combine_rows_per_src=[16384, 16384, 8814, 16384, 15646, 160, 7118, 4182] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=315973632 dispatch_meta_bytes=359832 combine_payload_bytes=315973632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=316333464 ingress_bytes=350576640 link_bytes=350576640 token_bytes=632307096 weight_movement_bytes=34603008 total_nvlink_bytes=666910104 weight_to_token_ratio=0.054725 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=33152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.319191673e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=295903232 dispatch_meta_bytes=293440 combine_payload_bytes=295903232 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=296196672 ingress_bytes=365109248 link_bytes=365109248 token_bytes=592099904 weight_movement_bytes=69206016 total_nvlink_bytes=661305920 weight_to_token_ratio=0.116882 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=18688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.373876707e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86024 payload_rows_remote=77624 dup_rows=12280 dup_rows_remote=12280 dedup_ratio=0.124919 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1074, 447, 0, 0, 0, 0, 0, 1, 1117, 0, 0, 4113, 0, 0, 0, 0, 0, 0, 0, 4200, 0, 0, 0, 0, 0, 7829, 0, 0, 0, 0, 8191, 0, 8192, 0, 0, 0, 8192, 0, 0, 122, 0, 0, 3528, 0, 0, 0, 0, 0, 0, 0, 0, 545, 0, 0, 1511, 0, 90] payload_rows_per_dst=[16384, 16384, 8226, 8400, 0, 16384, 14812, 5434] dup_rows_per_dst=[2146, 3130, 0, 0, 0, 244, 6424, 336] combine_rows_per_src=[16384, 16384, 8226, 8400, 0, 16384, 14812, 5434] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=347029504 dispatch_meta_bytes=389552 combine_payload_bytes=347029504 prefetch_bytes=34603008 reduce_bytes=173015040 egress_bytes=347419056 ingress_bytes=554647552 link_bytes=554647552 token_bytes=694448560 weight_movement_bytes=207618048 total_nvlink_bytes=902066608 weight_to_token_ratio=0.298968 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=25088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=2.087094085e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79954 payload_rows_remote=72908 dup_rows=18350 dup_rows_remote=18328 dedup_ratio=0.186666 router_tokens_per_expert=[0, 2, 0, 0, 0, 0, 4, 1127, 531, 16, 0, 0, 0, 0, 0, 1050, 0, 5, 4239, 0, 0, 0, 5, 0, 2, 18, 3908, 0, 0, 0, 0, 7, 7752, 14, 0, 0, 0, 8173, 0, 8172, 0, 13, 0, 8168, 15, 12, 49, 0, 1, 3509, 0, 1, 12, 0, 0, 11, 0, 2, 698, 0, 0, 1536, 0, 100] payload_rows_per_dst=[2266, 2142, 11558, 7864, 16360, 16370, 7046, 16348] dup_rows_per_dst=[0, 1052, 2994, 6, 10304, 144, 22, 3828] combine_rows_per_src=[2266, 2142, 11558, 7864, 16360, 16370, 7046, 16348] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=65172 payload_rows_remote=48790 dup_rows=33132 dup_rows_remote=16750 dedup_ratio=0.337036 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 8184, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 8184, 0, 1, 0, 4, 0, 0, 0, 0, 184, 0, 0, 8, 8185, 0, 0, 7, 79, 0, 0, 22, 7899, 0, 0, 0, 6, 0, 4, 8184, 0, 0, 4] payload_rows_per_dst=[2, 12, 16364, 6, 10, 16384, 16012, 16382] dup_rows_per_dst=[0, 0, 2, 0, 0, 16738, 10, 16382] combine_rows_per_src=[2, 12, 16364, 6, 10, 16384, 16012, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=284024832 dispatch_meta_bytes=327160 combine_payload_bytes=284024832 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=284351992 ingress_bytes=353230848 link_bytes=353230848 token_bytes=568376824 weight_movement_bytes=69206016 total_nvlink_bytes=637582840 weight_to_token_ratio=0.121761 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=48512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.329179244e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=281346048 dispatch_meta_bytes=319488 combine_payload_bytes=281346048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281665536 ingress_bytes=315949056 link_bytes=315949056 token_bytes=563011584 weight_movement_bytes=34603008 total_nvlink_bytes=597614592 weight_to_token_ratio=0.061461 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=40576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.188890862e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85640 payload_rows_remote=84724 dup_rows=12664 dup_rows_remote=12664 dedup_ratio=0.128825 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1257, 449, 0, 0, 0, 0, 0, 0, 1054, 0, 0, 4183, 0, 0, 0, 0, 0, 0, 0, 3924, 0, 0, 0, 0, 0, 7865, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 92, 0, 0, 3658, 0, 0, 0, 5, 0, 0, 0, 0, 528, 0, 0, 1476, 0, 85] payload_rows_per_dst=[15626, 8022, 13134, 7848, 916, 16384, 7326, 16384] dup_rows_per_dst=[2356, 1756, 4844, 0, 0, 184, 0, 3524] combine_rows_per_src=[15626, 8022, 13134, 7848, 916, 16384, 7326, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=317947904 dispatch_meta_bytes=359616 combine_payload_bytes=317947904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=318307520 ingress_bytes=352550912 link_bytes=352550912 token_bytes=636255424 weight_movement_bytes=34603008 total_nvlink_bytes=670858432 weight_to_token_ratio=0.054385 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.326620699e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=65184 payload_rows_remote=49228 dup_rows=33120 dup_rows_remote=33118 dedup_ratio=0.336914 router_tokens_per_expert=[5, 0, 0, 4, 0, 0, 2, 0, 17, 0, 0, 0, 8169, 0, 0, 1, 8173, 3, 0, 0, 0, 0, 0, 0, 0, 1, 7, 0, 4, 0, 0, 0, 8166, 5, 0, 1, 13, 0, 0, 0, 0, 208, 0, 1, 18, 8173, 0, 0, 21, 91, 0, 0, 74, 7793, 0, 0, 0, 14, 0, 13, 8173, 0, 0, 2] payload_rows_per_dst=[20, 36, 16348, 24, 36, 16382, 15956, 16382] dup_rows_per_dst=[2, 0, 4, 0, 2, 16750, 2, 16360] combine_rows_per_src=[20, 36, 16348, 24, 36, 16382, 15956, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77000 payload_rows_remote=60632 dup_rows=21304 dup_rows_remote=17550 dedup_ratio=0.216715 router_tokens_per_expert=[0, 3, 0, 0, 0, 0, 0, 895, 491, 5, 1, 0, 0, 0, 1, 1242, 0, 5, 3897, 0, 0, 0, 0, 0, 0, 3, 4158, 0, 0, 0, 0, 4, 7793, 3, 1, 0, 0, 8184, 1, 8184, 0, 2, 0, 8183, 8, 6, 87, 0, 2, 3719, 0, 0, 4, 0, 0, 2, 0, 0, 608, 0, 0, 1553, 0, 107] payload_rows_per_dst=[1796, 2500, 7804, 8330, 16374, 16382, 7446, 16368] dup_rows_per_dst=[0, 980, 0, 0, 16372, 190, 8, 3754] combine_rows_per_src=[1796, 2500, 7804, 8330, 16374, 16382, 7446, 16368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85726 payload_rows_remote=69342 dup_rows=12578 dup_rows_remote=12448 dedup_ratio=0.127950 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1170, 531, 0, 0, 0, 0, 0, 0, 1229, 0, 0, 4035, 0, 0, 0, 0, 0, 0, 0, 4126, 0, 0, 0, 0, 0, 7773, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 65, 0, 0, 3570, 0, 0, 0, 0, 0, 0, 0, 0, 651, 0, 0, 1336, 0, 90] payload_rows_per_dst=[2340, 2458, 16384, 8252, 16384, 16384, 7140, 16384] dup_rows_per_dst=[0, 1062, 8070, 0, 0, 130, 0, 3316] combine_rows_per_src=[2340, 2458, 16384, 8252, 16384, 16384, 7140, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=298631168 dispatch_meta_bytes=364944 combine_payload_bytes=298631168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=298996112 ingress_bytes=333234176 link_bytes=333234176 token_bytes=597627280 weight_movement_bytes=34603008 total_nvlink_bytes=632230288 weight_to_token_ratio=0.057901 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=28800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.253933377e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68220 payload_rows_remote=51836 dup_rows=30084 dup_rows_remote=13762 dedup_ratio=0.306030 router_tokens_per_expert=[2, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 8186, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 3, 0, 2, 0, 0, 211, 0, 0, 6, 8186, 0, 0, 4, 89, 0, 0, 11, 7876, 0, 0, 0, 3, 0, 4, 8187, 0, 0, 4] payload_rows_per_dst=[16376, 16384, 50, 16376, 16376, 434, 2214, 10] dup_rows_per_dst=[0, 16322, 0, 0, 13746, 0, 4, 12] combine_rows_per_src=[16376, 16384, 50, 16376, 16376, 434, 2214, 10] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=199843840 dispatch_meta_bytes=262160 combine_payload_bytes=199843840 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=200106000 ingress_bytes=269049856 link_bytes=269049856 token_bytes=399949840 weight_movement_bytes=69206016 total_nvlink_bytes=469155856 weight_to_token_ratio=0.173037 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.012412948e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=347029504 dispatch_meta_bytes=389552 combine_payload_bytes=347029504 prefetch_bytes=34603008 reduce_bytes=173015040 egress_bytes=347419056 ingress_bytes=554647552 link_bytes=554647552 token_bytes=694448560 weight_movement_bytes=207618048 total_nvlink_bytes=902066608 weight_to_token_ratio=0.298968 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=25088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=2.087094085e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85488 payload_rows_remote=77142 dup_rows=12816 dup_rows_remote=12816 dedup_ratio=0.130371 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1093, 501, 0, 0, 0, 0, 0, 0, 1228, 0, 0, 4173, 0, 0, 0, 0, 0, 0, 0, 4119, 0, 0, 0, 0, 0, 7805, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 94, 0, 0, 3452, 0, 0, 0, 1, 0, 0, 0, 0, 650, 0, 0, 1377, 0, 83] payload_rows_per_dst=[16384, 16384, 8346, 8240, 1306, 15032, 15610, 4186] dup_rows_per_dst=[2186, 3458, 0, 48, 0, 184, 6906, 34] combine_rows_per_src=[16384, 16384, 8346, 8240, 1306, 15032, 15610, 4186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=248348672 dispatch_meta_bytes=312728 combine_payload_bytes=248348672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=248661400 ingress_bytes=282951680 link_bytes=282951680 token_bytes=497010072 weight_movement_bytes=34603008 total_nvlink_bytes=531613080 weight_to_token_ratio=0.069622 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=32000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.064724392e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=201637888 dispatch_meta_bytes=329384 combine_payload_bytes=201637888 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=201967272 ingress_bytes=270843904 link_bytes=270843904 token_bytes=403605160 weight_movement_bytes=69206016 total_nvlink_bytes=472811176 weight_to_token_ratio=0.171470 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=48384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.019163806e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=98304 payload_rows_remote=81920 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 172, 0, 0, 0, 8192, 0, 0, 0, 71, 0, 0, 0, 7949, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 344, 16040, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 344, 16040, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97978 payload_rows_remote=94854 dup_rows=326 dup_rows_remote=326 dedup_ratio=0.003316 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 210, 0, 0, 0, 8192, 0, 0, 0, 72, 0, 0, 62, 7848, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 14, 16384, 16384, 3124, 13354, 15964, 16370] dup_rows_per_dst=[0, 0, 0, 0, 0, 326, 0, 0] combine_rows_per_src=[16384, 14, 16384, 16384, 3124, 13354, 15964, 16370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=284024832 dispatch_meta_bytes=327160 combine_payload_bytes=284024832 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=284351992 ingress_bytes=353230848 link_bytes=353230848 token_bytes=568376824 weight_movement_bytes=69206016 total_nvlink_bytes=637582840 weight_to_token_ratio=0.121761 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=48512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.329179244e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=212320256 dispatch_meta_bytes=262392 combine_payload_bytes=212320256 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=212582648 ingress_bytes=281526272 link_bytes=281526272 token_bytes=424902904 weight_movement_bytes=69206016 total_nvlink_bytes=494108920 weight_to_token_ratio=0.162875 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=32896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.059360697e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=66062 payload_rows_remote=49678 dup_rows=32242 dup_rows_remote=32242 dedup_ratio=0.327983 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 184, 0, 0, 0, 8192, 0, 0, 0, 78, 0, 0, 1, 7929, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 0, 16384, 16384, 368, 158, 0] dup_rows_per_dst=[0, 16384, 0, 0, 15858, 0, 0, 0] combine_rows_per_src=[16384, 16384, 0, 16384, 16384, 368, 158, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97820 payload_rows_remote=81436 dup_rows=484 dup_rows_remote=22 dedup_ratio=0.004924 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 223, 0, 0, 0, 8192, 0, 0, 0, 75, 0, 0, 0, 7894, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16368, 0, 16384, 16362, 0, 16384, 15938, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 462, 0, 22] combine_rows_per_src=[16368, 0, 16384, 16362, 0, 16384, 15938, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=315973632 dispatch_meta_bytes=359832 combine_payload_bytes=315973632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=316333464 ingress_bytes=350576640 link_bytes=350576640 token_bytes=632307096 weight_movement_bytes=34603008 total_nvlink_bytes=666910104 weight_to_token_ratio=0.054725 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=33152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.319191673e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=98304 payload_rows_remote=81920 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 189, 0, 0, 0, 8192, 0, 0, 0, 81, 0, 0, 17, 7905, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 378, 16006, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 378, 16006, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=335544320 dispatch_meta_bytes=327680 combine_payload_bytes=335544320 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=335872000 ingress_bytes=404750336 link_bytes=404750336 token_bytes=671416320 weight_movement_bytes=69206016 total_nvlink_bytes=740622336 weight_to_token_ratio=0.103075 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.037500 gemm_active_groups=3 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523042929e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=388521984 dispatch_meta_bytes=380720 combine_payload_bytes=388521984 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=388902704 ingress_bytes=457728000 link_bytes=457728000 token_bytes=777424688 weight_movement_bytes=69206016 total_nvlink_bytes=846630704 weight_to_token_ratio=0.089020 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=34432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.722393614e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=65184 payload_rows_remote=49228 dup_rows=33120 dup_rows_remote=33118 dedup_ratio=0.336914 router_tokens_per_expert=[5, 0, 0, 4, 0, 0, 2, 0, 17, 0, 0, 0, 8169, 0, 0, 1, 8173, 3, 0, 0, 0, 0, 0, 0, 0, 1, 7, 0, 4, 0, 0, 0, 8166, 5, 0, 1, 13, 0, 0, 0, 0, 208, 0, 1, 18, 8173, 0, 0, 21, 91, 0, 0, 74, 7793, 0, 0, 0, 14, 0, 13, 8173, 0, 0, 2] payload_rows_per_dst=[20, 36, 16348, 24, 36, 16382, 15956, 16382] dup_rows_per_dst=[2, 0, 4, 0, 2, 16750, 2, 16360] combine_rows_per_src=[20, 36, 16348, 24, 36, 16382, 15956, 16382] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73926 payload_rows_remote=57576 dup_rows=24378 dup_rows_remote=21788 dedup_ratio=0.247986 router_tokens_per_expert=[0, 0, 3, 7, 6, 0, 0, 372, 0, 824, 1331, 0, 0, 0, 3201, 0, 0, 4, 0, 6, 5, 8184, 3, 0, 2, 2, 0, 0, 8038, 0, 4, 0, 4, 6, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 2193, 0, 4, 0, 434, 1, 0, 0, 0, 8181, 0, 0, 0, 0, 7, 0, 0, 8171, 0, 0] payload_rows_per_dst=[760, 10540, 24, 10, 13504, 16372, 16366, 16350] dup_rows_per_dst=[16, 172, 12, 6, 8, 4390, 17184, 2590] combine_rows_per_src=[760, 10540, 24, 10, 13504, 16372, 16366, 16350] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=203481088 dispatch_meta_bytes=327680 combine_payload_bytes=203481088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=203808768 ingress_bytes=238084096 link_bytes=238084096 token_bytes=407289856 weight_movement_bytes=34603008 total_nvlink_bytes=441892864 weight_to_token_ratio=0.084959 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.958912859e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=333561856 dispatch_meta_bytes=325832 combine_payload_bytes=333561856 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=333887688 ingress_bytes=402767872 link_bytes=402767872 token_bytes=667449544 weight_movement_bytes=69206016 total_nvlink_bytes=736655560 weight_to_token_ratio=0.103687 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=31232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.515583077e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76278 payload_rows_remote=59936 dup_rows=22026 dup_rows_remote=4424 dedup_ratio=0.224060 router_tokens_per_expert=[2, 0, 9, 10, 19, 0, 0, 394, 0, 1100, 1215, 0, 0, 0, 2972, 0, 0, 12, 0, 22, 18, 8167, 2, 0, 2, 6, 0, 1, 7999, 0, 10, 0, 13, 15, 0, 0, 0, 8133, 5, 0, 0, 0, 0, 0, 2055, 0, 6, 0, 675, 0, 0, 4, 0, 8160, 0, 0, 1, 0, 3, 0, 0, 8117, 5, 0] payload_rows_per_dst=[834, 10366, 82, 11028, 5028, 16346, 16342, 16252] dup_rows_per_dst=[34, 208, 26, 20, 26, 4110, 17602, 0] combine_rows_per_src=[834, 10366, 82, 11028, 5028, 16346, 16342, 16252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97978 payload_rows_remote=94854 dup_rows=326 dup_rows_remote=326 dedup_ratio=0.003316 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 210, 0, 0, 0, 8192, 0, 0, 0, 72, 0, 0, 62, 7848, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 14, 16384, 16384, 3124, 13354, 15964, 16370] dup_rows_per_dst=[0, 0, 0, 0, 0, 326, 0, 0] combine_rows_per_src=[16384, 14, 16384, 16384, 3124, 13354, 15964, 16370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=65172 payload_rows_remote=48790 dup_rows=33132 dup_rows_remote=16750 dedup_ratio=0.337036 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 8184, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 8184, 0, 1, 0, 4, 0, 0, 0, 0, 184, 0, 0, 8, 8185, 0, 0, 7, 79, 0, 0, 22, 7899, 0, 0, 0, 6, 0, 4, 8184, 0, 0, 4] payload_rows_per_dst=[2, 12, 16364, 6, 10, 16384, 16012, 16382] dup_rows_per_dst=[0, 0, 2, 0, 0, 16738, 10, 16382] combine_rows_per_src=[2, 12, 16364, 6, 10, 16384, 16012, 16382] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=335544320 dispatch_meta_bytes=327680 combine_payload_bytes=335544320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=335872000 ingress_bytes=370147328 link_bytes=370147328 token_bytes=671416320 weight_movement_bytes=34603008 total_nvlink_bytes=706019328 weight_to_token_ratio=0.051537 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392834596e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97820 payload_rows_remote=81436 dup_rows=484 dup_rows_remote=22 dedup_ratio=0.004924 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 223, 0, 0, 0, 8192, 0, 0, 0, 75, 0, 0, 0, 7894, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16368, 0, 16384, 16362, 0, 16384, 15938, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 462, 0, 22] combine_rows_per_src=[16368, 0, 16384, 16362, 0, 16384, 15938, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=201637888 dispatch_meta_bytes=329384 combine_payload_bytes=201637888 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=201967272 ingress_bytes=270843904 link_bytes=270843904 token_bytes=403605160 weight_movement_bytes=69206016 total_nvlink_bytes=472811176 weight_to_token_ratio=0.171470 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=48384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.019163806e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81278 payload_rows_remote=69996 dup_rows=17026 dup_rows_remote=16848 dedup_ratio=0.173197 router_tokens_per_expert=[2, 0, 2, 2, 5, 0, 0, 238, 0, 954, 1120, 0, 0, 0, 3656, 0, 0, 1, 0, 7, 6, 8185, 2, 0, 1, 3, 0, 0, 8014, 0, 3, 0, 4, 4, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 1731, 0, 2, 0, 676, 1, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 8163, 3, 0] payload_rows_per_dst=[16384, 11282, 16382, 16034, 16372, 3464, 1354, 6] dup_rows_per_dst=[484, 178, 16346, 8, 8, 2, 0, 0] combine_rows_per_src=[16384, 11282, 16382, 16034, 16372, 3464, 1354, 6] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=235831296 dispatch_meta_bytes=317456 combine_payload_bytes=235831296 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=236148752 ingress_bytes=305037312 link_bytes=305037312 token_bytes=471980048 weight_movement_bytes=69206016 total_nvlink_bytes=541186064 weight_to_token_ratio=0.146629 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=47872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.147830848e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=388521984 dispatch_meta_bytes=380720 combine_payload_bytes=388521984 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=388902704 ingress_bytes=457728000 link_bytes=457728000 token_bytes=777424688 weight_movement_bytes=69206016 total_nvlink_bytes=846630704 weight_to_token_ratio=0.089020 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=34432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.722393614e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=98304 payload_rows_remote=81920 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 172, 0, 0, 0, 8192, 0, 0, 0, 71, 0, 0, 0, 7949, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 344, 16040, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 344, 16040, 0] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=245497856 dispatch_meta_bytes=257440 combine_payload_bytes=245497856 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=245755296 ingress_bytes=314703872 link_bytes=314703872 token_bytes=491253152 weight_movement_bytes=69206016 total_nvlink_bytes=560459168 weight_to_token_ratio=0.140876 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=25600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.184205335e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95166 payload_rows_remote=78782 dup_rows=3138 dup_rows_remote=780 dedup_ratio=0.031921 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 315, 0, 1160, 1269, 0, 0, 0, 3078, 0, 0, 0, 0, 3, 0, 8192, 0, 0, 0, 0, 0, 0, 8051, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1899, 0, 0, 0, 616, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0] payload_rows_per_dst=[16384, 10864, 16384, 16102, 16382, 3798, 1232, 14020] dup_rows_per_dst=[630, 150, 2358, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 10864, 16384, 16102, 16382, 3798, 1232, 14020] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93778 payload_rows_remote=79570 dup_rows=4526 dup_rows_remote=4526 dedup_ratio=0.046041 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 351, 0, 1061, 1793, 0, 0, 0, 2801, 0, 0, 0, 0, 9, 0, 8192, 0, 0, 0, 0, 0, 0, 8047, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1781, 0, 0, 0, 553, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0] payload_rows_per_dst=[16384, 11160, 18, 16094, 14208, 16384, 3170, 16360] dup_rows_per_dst=[702, 150, 0, 0, 0, 3562, 112, 0] combine_rows_per_src=[16384, 11160, 18, 16094, 14208, 16384, 3170, 16360] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=286703616 dispatch_meta_bytes=347376 combine_payload_bytes=286703616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=287050992 ingress_bytes=321306624 link_bytes=321306624 token_bytes=573754608 weight_movement_bytes=34603008 total_nvlink_bytes=608357616 weight_to_token_ratio=0.060310 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.037500 gemm_active_groups=5 gemm_peak_group_rows=24960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.209050959e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=11118279448 prefetch_bytes=726663168 reduce_bytes=622854144 weight_movement_bytes=1349517312 egress_bytes=5562710808 ingress_bytes=6905085952 link_bytes=6905085952 total_nvlink_bytes=12467796760 weight_to_token_ratio=0.121378 prefetch_share_of_ingress=0.105236 dedup_ratio=0.246288 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=18 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=22352 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.129707936e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=333561856 dispatch_meta_bytes=325832 combine_payload_bytes=333561856 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=333887688 ingress_bytes=402767872 link_bytes=402767872 token_bytes=667449544 weight_movement_bytes=69206016 total_nvlink_bytes=736655560 weight_to_token_ratio=0.103687 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=31232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.515583077e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=199843840 dispatch_meta_bytes=262160 combine_payload_bytes=199843840 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=200106000 ingress_bytes=269049856 link_bytes=269049856 token_bytes=399949840 weight_movement_bytes=69206016 total_nvlink_bytes=469155856 weight_to_token_ratio=0.173037 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.012412948e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75052 payload_rows_remote=58668 dup_rows=23252 dup_rows_remote=22620 dedup_ratio=0.236532 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 316, 0, 1135, 1630, 0, 0, 0, 2788, 0, 0, 0, 0, 9, 0, 8192, 0, 0, 0, 0, 0, 0, 8036, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1943, 0, 0, 0, 538, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0] payload_rows_per_dst=[16384, 14070, 16384, 6868, 16384, 3886, 1076, 0] dup_rows_per_dst=[632, 6240, 16380, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 14070, 16384, 6868, 16384, 3886, 1076, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12183793264 prefetch_bytes=795869184 reduce_bytes=449839104 weight_movement_bytes=1245708288 egress_bytes=6095654512 ingress_bytes=7333847040 link_bytes=7333847040 total_nvlink_bytes=13429501552 weight_to_token_ratio=0.102243 prefetch_share_of_ingress=0.108520 dedup_ratio=0.205104 prefetch_slots=46 unused_prefetch_slots=322 slot_utilization=0.125000 reduce_slots=13 remote_demand=23 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028152 gemm_inner_pad_upper_bound=22606 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.199855478e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88172 payload_rows_remote=71788 dup_rows=10132 dup_rows_remote=7150 dedup_ratio=0.103068 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 347, 0, 1032, 1488, 0, 0, 0, 3472, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8039, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 1491, 0, 0, 0, 520, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[11130, 11822, 0, 16078, 0, 16384, 16374, 16384] dup_rows_per_dst=[494, 162, 0, 0, 0, 2982, 6494, 0] combine_rows_per_src=[11130, 11822, 0, 16078, 0, 16384, 16374, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=286703616 dispatch_meta_bytes=347376 combine_payload_bytes=286703616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=287050992 ingress_bytes=321306624 link_bytes=321306624 token_bytes=573754608 weight_movement_bytes=34603008 total_nvlink_bytes=608357616 weight_to_token_ratio=0.060310 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.037500 gemm_active_groups=5 gemm_peak_group_rows=24960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.209050959e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=335544320 dispatch_meta_bytes=327680 combine_payload_bytes=335544320 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=335872000 ingress_bytes=404750336 link_bytes=404750336 token_bytes=671416320 weight_movement_bytes=69206016 total_nvlink_bytes=740622336 weight_to_token_ratio=0.103075 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.037500 gemm_active_groups=3 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523042929e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96888 payload_rows_remote=80832 dup_rows=1416 dup_rows_remote=1416 dedup_ratio=0.014404 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 334, 0, 911, 1637, 0, 0, 0, 2922, 0, 0, 0, 0, 19, 0, 8192, 0, 0, 0, 0, 0, 0, 8028, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 2055, 0, 0, 0, 499, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0] payload_rows_per_dst=[16384, 10774, 13872, 16056, 16384, 6078, 998, 16342] dup_rows_per_dst=[668, 166, 36, 0, 0, 546, 0, 0] combine_rows_per_src=[16384, 10774, 13872, 16056, 16384, 6078, 998, 16342] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12773668384 prefetch_bytes=692060160 reduce_bytes=795869184 weight_movement_bytes=1487929344 egress_bytes=6390535712 ingress_bytes=7871062016 link_bytes=7871062016 total_nvlink_bytes=14261597728 weight_to_token_ratio=0.116484 prefetch_share_of_ingress=0.087925 dedup_ratio=0.169443 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=23 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=22606 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.287746639e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=322691072 dispatch_meta_bytes=318248 combine_payload_bytes=322691072 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=323009320 ingress_bytes=391897088 link_bytes=391897088 token_bytes=645700392 weight_movement_bytes=69206016 total_nvlink_bytes=714906408 weight_to_token_ratio=0.107180 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=31616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.474677192e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=325918720 dispatch_meta_bytes=336384 combine_payload_bytes=325918720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=326255104 ingress_bytes=395124736 link_bytes=395124736 token_bytes=652173824 weight_movement_bytes=69206016 total_nvlink_bytes=721379840 weight_to_token_ratio=0.106116 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=39936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.486822571e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=240304128 dispatch_meta_bytes=325152 combine_payload_bytes=240304128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=240629280 ingress_bytes=274907136 link_bytes=274907136 token_bytes=480933408 weight_movement_bytes=34603008 total_nvlink_bytes=515536416 weight_to_token_ratio=0.071950 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=46464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.034453421e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=294043648 dispatch_meta_bytes=315752 combine_payload_bytes=294043648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=294359400 ingress_bytes=328646656 link_bytes=328646656 token_bytes=588403048 weight_movement_bytes=34603008 total_nvlink_bytes=623006056 weight_to_token_ratio=0.058808 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=34048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.236670908e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93778 payload_rows_remote=79570 dup_rows=4526 dup_rows_remote=4526 dedup_ratio=0.046041 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 351, 0, 1061, 1793, 0, 0, 0, 2801, 0, 0, 0, 0, 9, 0, 8192, 0, 0, 0, 0, 0, 0, 8047, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1781, 0, 0, 0, 553, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0] payload_rows_per_dst=[16384, 11160, 18, 16094, 14208, 16384, 3170, 16360] dup_rows_per_dst=[702, 150, 0, 0, 0, 3562, 112, 0] combine_rows_per_src=[16384, 11160, 18, 16094, 14208, 16384, 3170, 16360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76278 payload_rows_remote=59936 dup_rows=22026 dup_rows_remote=4424 dedup_ratio=0.224060 router_tokens_per_expert=[2, 0, 9, 10, 19, 0, 0, 394, 0, 1100, 1215, 0, 0, 0, 2972, 0, 0, 12, 0, 22, 18, 8167, 2, 0, 2, 6, 0, 1, 7999, 0, 10, 0, 13, 15, 0, 0, 0, 8133, 5, 0, 0, 0, 0, 0, 2055, 0, 6, 0, 675, 0, 0, 4, 0, 8160, 0, 0, 1, 0, 3, 0, 0, 8117, 5, 0] payload_rows_per_dst=[834, 10366, 82, 11028, 5028, 16346, 16342, 16252] dup_rows_per_dst=[34, 208, 26, 20, 26, 4110, 17602, 0] combine_rows_per_src=[834, 10366, 82, 11028, 5028, 16346, 16342, 16252] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=331087872 dispatch_meta_bytes=328992 combine_payload_bytes=331087872 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=331416864 ingress_bytes=434896896 link_bytes=434896896 token_bytes=662504736 weight_movement_bytes=103809024 total_nvlink_bytes=766313760 weight_to_token_ratio=0.156692 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.636482008e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14638774392 prefetch_bytes=657457152 reduce_bytes=1072693248 weight_movement_bytes=1730150400 egress_bytes=7323326584 ingress_bytes=9045598208 link_bytes=9045598208 total_nvlink_bytes=16368924792 weight_to_token_ratio=0.118190 prefetch_share_of_ingress=0.072683 dedup_ratio=0.093835 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=31 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029457 gemm_inner_pad_upper_bound=21844 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.479906862e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=15188540784 prefetch_bytes=692060160 reduce_bytes=795869184 weight_movement_bytes=1487929344 egress_bytes=7598259568 ingress_bytes=9078210560 link_bytes=9078210560 total_nvlink_bytes=16676470128 weight_to_token_ratio=0.097964 prefetch_share_of_ingress=0.076233 dedup_ratio=0.071782 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=23 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.026630 gemm_inner_pad_upper_bound=24003 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.485242412e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73926 payload_rows_remote=57576 dup_rows=24378 dup_rows_remote=21788 dedup_ratio=0.247986 router_tokens_per_expert=[0, 0, 3, 7, 6, 0, 0, 372, 0, 824, 1331, 0, 0, 0, 3201, 0, 0, 4, 0, 6, 5, 8184, 3, 0, 2, 2, 0, 0, 8038, 0, 4, 0, 4, 6, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 2193, 0, 4, 0, 434, 1, 0, 0, 0, 8181, 0, 0, 0, 0, 7, 0, 0, 8171, 0, 0] payload_rows_per_dst=[760, 10540, 24, 10, 13504, 16372, 16366, 16350] dup_rows_per_dst=[16, 172, 12, 6, 8, 4390, 17184, 2590] combine_rows_per_src=[760, 10540, 24, 10, 13504, 16372, 16366, 16350] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12022502504 prefetch_bytes=795869184 reduce_bytes=449839104 weight_movement_bytes=1245708288 egress_bytes=6014874728 ingress_bytes=7253336064 link_bytes=7253336064 total_nvlink_bytes=13268210792 weight_to_token_ratio=0.103615 prefetch_share_of_ingress=0.109725 dedup_ratio=0.195773 prefetch_slots=46 unused_prefetch_slots=322 slot_utilization=0.125000 reduce_slots=13 remote_demand=23 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=22225 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.186683464e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14566961624 prefetch_bytes=622854144 reduce_bytes=934281216 weight_movement_bytes=1557135360 egress_bytes=7287370200 ingress_bytes=8836726784 link_bytes=8836726784 total_nvlink_bytes=16124096984 weight_to_token_ratio=0.106895 prefetch_share_of_ingress=0.070485 dedup_ratio=0.091799 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=27 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027826 gemm_inner_pad_upper_bound=22987 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.445734412e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88172 payload_rows_remote=71788 dup_rows=10132 dup_rows_remote=7150 dedup_ratio=0.103068 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 347, 0, 1032, 1488, 0, 0, 0, 3472, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8039, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 1491, 0, 0, 0, 520, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[11130, 11822, 0, 16078, 0, 16384, 16374, 16384] dup_rows_per_dst=[494, 162, 0, 0, 0, 2982, 6494, 0] combine_rows_per_src=[11130, 11822, 0, 16078, 0, 16384, 16374, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=325918720 dispatch_meta_bytes=336384 combine_payload_bytes=325918720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=326255104 ingress_bytes=395124736 link_bytes=395124736 token_bytes=652173824 weight_movement_bytes=69206016 total_nvlink_bytes=721379840 weight_to_token_ratio=0.106116 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=39936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.486822571e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95166 payload_rows_remote=78782 dup_rows=3138 dup_rows_remote=780 dedup_ratio=0.031921 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 315, 0, 1160, 1269, 0, 0, 0, 3078, 0, 0, 0, 0, 3, 0, 8192, 0, 0, 0, 0, 0, 0, 8051, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1899, 0, 0, 0, 616, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0] payload_rows_per_dst=[16384, 10864, 16384, 16102, 16382, 3798, 1232, 14020] dup_rows_per_dst=[630, 150, 2358, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 10864, 16384, 16102, 16382, 3798, 1232, 14020] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=245497856 dispatch_meta_bytes=257440 combine_payload_bytes=245497856 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=245755296 ingress_bytes=314703872 link_bytes=314703872 token_bytes=491253152 weight_movement_bytes=69206016 total_nvlink_bytes=560459168 weight_to_token_ratio=0.140876 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=25600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.184205335e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14878007752 prefetch_bytes=692060160 reduce_bytes=553648128 weight_movement_bytes=1245708288 egress_bytes=7442932168 ingress_bytes=8680783872 link_bytes=8680783872 total_nvlink_bytes=16123716040 weight_to_token_ratio=0.083728 prefetch_share_of_ingress=0.079723 dedup_ratio=0.076917 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=16 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029130 gemm_inner_pad_upper_bound=21717 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.420221341e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none - [2026-08-26 15:09:11] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 800.5 | throughput per GPU (TFLOP/s/GPU): 159.4 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.375167E+00 | loss scale: 1.0 | grad norm: 4.992 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=235831296 dispatch_meta_bytes=317456 combine_payload_bytes=235831296 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=236148752 ingress_bytes=305037312 link_bytes=305037312 token_bytes=471980048 weight_movement_bytes=69206016 total_nvlink_bytes=541186064 weight_to_token_ratio=0.146629 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=47872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.147830848e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=15188540784 prefetch_bytes=692060160 reduce_bytes=795869184 weight_movement_bytes=1487929344 egress_bytes=7598259568 ingress_bytes=9078210560 link_bytes=9078210560 total_nvlink_bytes=16676470128 weight_to_token_ratio=0.097964 prefetch_share_of_ingress=0.076233 dedup_ratio=0.071782 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=23 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.026630 gemm_inner_pad_upper_bound=24003 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.485242412e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=294043648 dispatch_meta_bytes=315752 combine_payload_bytes=294043648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=294359400 ingress_bytes=328646656 link_bytes=328646656 token_bytes=588403048 weight_movement_bytes=34603008 total_nvlink_bytes=623006056 weight_to_token_ratio=0.058808 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=34048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.236670908e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=322691072 dispatch_meta_bytes=318248 combine_payload_bytes=322691072 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=323009320 ingress_bytes=391897088 link_bytes=391897088 token_bytes=645700392 weight_movement_bytes=69206016 total_nvlink_bytes=714906408 weight_to_token_ratio=0.107180 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=31616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.474677192e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12183793264 prefetch_bytes=795869184 reduce_bytes=449839104 weight_movement_bytes=1245708288 egress_bytes=6095654512 ingress_bytes=7333847040 link_bytes=7333847040 total_nvlink_bytes=13429501552 weight_to_token_ratio=0.102243 prefetch_share_of_ingress=0.108520 dedup_ratio=0.205104 prefetch_slots=46 unused_prefetch_slots=322 slot_utilization=0.125000 reduce_slots=13 remote_demand=23 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028152 gemm_inner_pad_upper_bound=22606 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.199855478e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=11118279448 prefetch_bytes=726663168 reduce_bytes=622854144 weight_movement_bytes=1349517312 egress_bytes=5562710808 ingress_bytes=6905085952 link_bytes=6905085952 total_nvlink_bytes=12467796760 weight_to_token_ratio=0.121378 prefetch_share_of_ingress=0.105236 dedup_ratio=0.246288 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=18 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=22352 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.129707936e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14566961624 prefetch_bytes=622854144 reduce_bytes=934281216 weight_movement_bytes=1557135360 egress_bytes=7287370200 ingress_bytes=8836726784 link_bytes=8836726784 total_nvlink_bytes=16124096984 weight_to_token_ratio=0.106895 prefetch_share_of_ingress=0.070485 dedup_ratio=0.091799 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=27 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027826 gemm_inner_pad_upper_bound=22987 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.445734412e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14638774392 prefetch_bytes=657457152 reduce_bytes=1072693248 weight_movement_bytes=1730150400 egress_bytes=7323326584 ingress_bytes=9045598208 link_bytes=9045598208 total_nvlink_bytes=16368924792 weight_to_token_ratio=0.118190 prefetch_share_of_ingress=0.072683 dedup_ratio=0.093835 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=31 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029457 gemm_inner_pad_upper_bound=21844 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.479906862e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none + [2026-08-26 15:56:59] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 794.3 | throughput per GPU (TFLOP/s/GPU): 160.6 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.375167E+00 | loss scale: 1.0 | grad norm: 4.992 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74156 payload_rows_remote=62908 dup_rows=24148 dup_rows_remote=18452 dedup_ratio=0.245646 router_tokens_per_expert=[1423, 604, 588, 1177, 485, 1407, 1265, 174, 2062, 922, 1268, 290, 10, 2585, 730, 734, 673, 922, 746, 653, 129, 290, 277, 970, 334, 737, 758, 482, 948, 180, 689, 526, 1104, 676, 1603, 1166, 646, 260, 866, 597, 261, 482, 634, 159, 796, 845, 505, 116, 398, 413, 539, 582, 803, 2053, 314, 494, 1871, 468, 1558, 674, 195, 751, 1612, 673] payload_rows_per_dst=[11248, 9952, 7974, 9718, 8662, 7682, 9090, 9830] dup_rows_per_dst=[5696, 4940, 1854, 3332, 1968, 2224, 2102, 2032] combine_rows_per_src=[11248, 9952, 7974, 9718, 8662, 7682, 9090, 9830] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74944 payload_rows_remote=65340 dup_rows=23360 dup_rows_remote=20086 dedup_ratio=0.237630 router_tokens_per_expert=[1603, 702, 553, 1309, 430, 1536, 1172, 132, 2277, 862, 1176, 268, 3, 2558, 692, 829, 691, 897, 737, 748, 103, 205, 275, 1102, 341, 715, 717, 414, 954, 178, 712, 497, 1033, 644, 1701, 1170, 690, 224, 854, 611, 210, 438, 590, 179, 707, 804, 507, 122, 307, 416, 572, 726, 770, 1997, 331, 474, 1911, 437, 1749, 618, 185, 738, 1375, 674] payload_rows_per_dst=[8684, 8574, 10002, 9604, 10216, 8964, 9060, 9840] dup_rows_per_dst=[2984, 3640, 2720, 3274, 3638, 3266, 2126, 1712] combine_rows_per_src=[8684, 8574, 10002, 9604, 10216, 8964, 9060, 9840] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=65582 dup_rows=24062 dup_rows_remote=20102 dedup_ratio=0.244771 router_tokens_per_expert=[1440, 578, 487, 1359, 463, 1487, 1319, 139, 2172, 1046, 1298, 330, 10, 2502, 712, 742, 709, 865, 824, 689, 110, 211, 261, 1006, 343, 724, 743, 529, 999, 159, 767, 574, 1027, 656, 1778, 1044, 573, 243, 775, 648, 245, 457, 610, 200, 764, 820, 496, 165, 321, 419, 502, 662, 797, 1963, 307, 435, 1864, 514, 1537, 664, 161, 673, 1504, 731] payload_rows_per_dst=[9570, 8660, 9526, 9834, 8906, 9072, 8834, 9840] dup_rows_per_dst=[4074, 3960, 2704, 3570, 2602, 3446, 1978, 1728] combine_rows_per_src=[9570, 8660, 9526, 9834, 8906, 9072, 8834, 9840] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74992 payload_rows_remote=63490 dup_rows=23312 dup_rows_remote=20022 dedup_ratio=0.237142 router_tokens_per_expert=[1581, 661, 583, 1255, 475, 1500, 1206, 167, 2125, 865, 1201, 239, 10, 2692, 730, 754, 680, 852, 757, 711, 95, 237, 286, 992, 307, 730, 668, 531, 982, 155, 629, 470, 1052, 663, 1699, 1113, 623, 200, 870, 635, 195, 528, 636, 130, 768, 868, 554, 173, 326, 363, 541, 778, 824, 2113, 326, 426, 1993, 487, 1579, 602, 203, 717, 1373, 668] payload_rows_per_dst=[8634, 8384, 9678, 8132, 8470, 9486, 11502, 10706] dup_rows_per_dst=[3060, 3464, 2704, 2202, 1842, 3602, 3290, 3148] combine_rows_per_src=[8634, 8384, 9678, 8132, 8470, 9486, 11502, 10706] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75614 payload_rows_remote=66450 dup_rows=22690 dup_rows_remote=19244 dedup_ratio=0.230815 router_tokens_per_expert=[1607, 614, 539, 1281, 463, 1400, 1036, 216, 2278, 715, 1215, 245, 8, 2532, 742, 832, 674, 910, 733, 709, 135, 240, 296, 1062, 354, 735, 716, 496, 996, 123, 664, 460, 951, 758, 1754, 1169, 824, 242, 774, 637, 237, 517, 611, 185, 787, 780, 532, 124, 316, 424, 589, 652, 802, 2048, 271, 472, 1923, 486, 1641, 691, 164, 673, 1380, 712] payload_rows_per_dst=[8238, 8564, 10184, 9548, 9830, 9164, 10396, 9690] dup_rows_per_dst=[2860, 3506, 2548, 3386, 2514, 3446, 2626, 1804] combine_rows_per_src=[8238, 8564, 10184, 9548, 9830, 9164, 10396, 9690] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74782 payload_rows_remote=64672 dup_rows=23522 dup_rows_remote=19876 dedup_ratio=0.239278 router_tokens_per_expert=[1556, 617, 563, 1238, 444, 1366, 1186, 153, 2146, 968, 1363, 228, 12, 2648, 698, 715, 758, 951, 785, 805, 134, 257, 263, 1029, 367, 690, 766, 475, 946, 217, 745, 551, 1103, 668, 1751, 1131, 587, 210, 783, 645, 255, 463, 610, 182, 695, 939, 565, 138, 349, 421, 514, 666, 810, 1906, 274, 487, 1958, 431, 1545, 576, 206, 642, 1322, 680] payload_rows_per_dst=[8468, 8606, 10092, 9848, 10110, 9448, 8884, 9326] dup_rows_per_dst=[2666, 3654, 2984, 3582, 3646, 3542, 1970, 1478] combine_rows_per_src=[8468, 8606, 10092, 9848, 10110, 9448, 8884, 9326] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75064 payload_rows_remote=63456 dup_rows=23240 dup_rows_remote=18818 dedup_ratio=0.236410 router_tokens_per_expert=[1526, 637, 579, 1256, 413, 1421, 1022, 134, 2302, 967, 1358, 265, 4, 2658, 662, 844, 694, 953, 816, 681, 153, 209, 247, 1110, 310, 693, 795, 455, 1022, 197, 636, 399, 998, 622, 1779, 1079, 613, 197, 854, 683, 199, 440, 575, 173, 675, 834, 537, 155, 327, 445, 591, 649, 811, 2105, 326, 472, 2051, 474, 1530, 614, 151, 632, 1425, 718] payload_rows_per_dst=[8184, 7536, 9894, 7460, 9332, 9274, 11776, 11608] dup_rows_per_dst=[2740, 2618, 2884, 1554, 2570, 3218, 3234, 4422] combine_rows_per_src=[8184, 7536, 9894, 7460, 9332, 9274, 11776, 11608] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75614 payload_rows_remote=66450 dup_rows=22690 dup_rows_remote=19244 dedup_ratio=0.230815 router_tokens_per_expert=[1607, 614, 539, 1281, 463, 1400, 1036, 216, 2278, 715, 1215, 245, 8, 2532, 742, 832, 674, 910, 733, 709, 135, 240, 296, 1062, 354, 735, 716, 496, 996, 123, 664, 460, 951, 758, 1754, 1169, 824, 242, 774, 637, 237, 517, 611, 185, 787, 780, 532, 124, 316, 424, 589, 652, 802, 2048, 271, 472, 1923, 486, 1641, 691, 164, 673, 1380, 712] payload_rows_per_dst=[8238, 8564, 10184, 9548, 9830, 9164, 10396, 9690] dup_rows_per_dst=[2860, 3506, 2548, 3386, 2514, 3446, 2626, 1804] combine_rows_per_src=[8238, 8564, 10184, 9548, 9830, 9164, 10396, 9690] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75124 payload_rows_remote=65414 dup_rows=23180 dup_rows_remote=20334 dedup_ratio=0.235799 router_tokens_per_expert=[1514, 604, 499, 1248, 447, 1531, 1160, 110, 2285, 732, 1276, 262, 4, 2685, 752, 812, 672, 925, 840, 712, 86, 191, 222, 1116, 301, 758, 663, 447, 940, 153, 724, 466, 1024, 636, 1838, 1069, 708, 204, 824, 696, 188, 413, 558, 162, 744, 816, 550, 119, 306, 389, 542, 647, 885, 2155, 356, 438, 2178, 411, 1609, 618, 143, 678, 1387, 724] payload_rows_per_dst=[8524, 8648, 9710, 9828, 10454, 9096, 9306, 9558] dup_rows_per_dst=[2674, 3598, 2846, 3432, 3544, 3374, 2130, 1582] combine_rows_per_src=[8524, 8648, 9710, 9828, 10454, 9096, 9306, 9558] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=65582 dup_rows=24062 dup_rows_remote=20102 dedup_ratio=0.244771 router_tokens_per_expert=[1440, 578, 487, 1359, 463, 1487, 1319, 139, 2172, 1046, 1298, 330, 10, 2502, 712, 742, 709, 865, 824, 689, 110, 211, 261, 1006, 343, 724, 743, 529, 999, 159, 767, 574, 1027, 656, 1778, 1044, 573, 243, 775, 648, 245, 457, 610, 200, 764, 820, 496, 165, 321, 419, 502, 662, 797, 1963, 307, 435, 1864, 514, 1537, 664, 161, 673, 1504, 731] payload_rows_per_dst=[9570, 8660, 9526, 9834, 8906, 9072, 8834, 9840] dup_rows_per_dst=[4074, 3960, 2704, 3570, 2602, 3446, 1978, 1728] combine_rows_per_src=[9570, 8660, 9526, 9834, 8906, 9072, 8834, 9840] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=257671168 dispatch_meta_bytes=325440 combine_payload_bytes=257671168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=257996608 ingress_bytes=326877184 link_bytes=326877184 token_bytes=515667776 weight_movement_bytes=69206016 total_nvlink_bytes=584873792 weight_to_token_ratio=0.134207 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=11648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.230012528e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=264896512 dispatch_meta_bytes=338192 combine_payload_bytes=264896512 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265234704 ingress_bytes=368705536 link_bytes=368705536 token_bytes=530131216 weight_movement_bytes=103809024 total_nvlink_bytes=633940240 weight_to_token_ratio=0.195818 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=8960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.387409249e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=272179200 dispatch_meta_bytes=342776 combine_payload_bytes=272179200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272521976 ingress_bytes=306782208 link_bytes=306782208 token_bytes=544701176 weight_movement_bytes=34603008 total_nvlink_bytes=579304184 weight_to_token_ratio=0.063527 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=19456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.154396751e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267632640 dispatch_meta_bytes=341704 combine_payload_bytes=267632640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267974344 ingress_bytes=302235648 link_bytes=302235648 token_bytes=535606984 weight_movement_bytes=34603008 total_nvlink_bytes=570209992 weight_to_token_ratio=0.064605 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=12416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137288411e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=259915776 dispatch_meta_bytes=329096 combine_payload_bytes=259915776 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=260244872 ingress_bytes=329121792 link_bytes=329121792 token_bytes=520160648 weight_movement_bytes=69206016 total_nvlink_bytes=589366664 weight_to_token_ratio=0.133047 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=12800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.238458807e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74992 payload_rows_remote=63490 dup_rows=23312 dup_rows_remote=20022 dedup_ratio=0.237142 router_tokens_per_expert=[1581, 661, 583, 1255, 475, 1500, 1206, 167, 2125, 865, 1201, 239, 10, 2692, 730, 754, 680, 852, 757, 711, 95, 237, 286, 992, 307, 730, 668, 531, 982, 155, 629, 470, 1052, 663, 1699, 1113, 623, 200, 870, 635, 195, 528, 636, 130, 768, 868, 554, 173, 326, 363, 541, 778, 824, 2113, 326, 426, 1993, 487, 1579, 602, 203, 717, 1373, 668] payload_rows_per_dst=[8634, 8384, 9678, 8132, 8470, 9486, 11502, 10706] dup_rows_per_dst=[3060, 3464, 2704, 2202, 1842, 3602, 3290, 3148] combine_rows_per_src=[8634, 8384, 9678, 8132, 8470, 9486, 11502, 10706] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267935744 dispatch_meta_bytes=342992 combine_payload_bytes=267935744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268278736 ingress_bytes=302538752 link_bytes=302538752 token_bytes=536214480 weight_movement_bytes=34603008 total_nvlink_bytes=570817488 weight_to_token_ratio=0.064532 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138428967e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74156 payload_rows_remote=62908 dup_rows=24148 dup_rows_remote=18452 dedup_ratio=0.245646 router_tokens_per_expert=[1423, 604, 588, 1177, 485, 1407, 1265, 174, 2062, 922, 1268, 290, 10, 2585, 730, 734, 673, 922, 746, 653, 129, 290, 277, 970, 334, 737, 758, 482, 948, 180, 689, 526, 1104, 676, 1603, 1166, 646, 260, 866, 597, 261, 482, 634, 159, 796, 845, 505, 116, 398, 413, 539, 582, 803, 2053, 314, 494, 1871, 468, 1558, 674, 195, 751, 1612, 673] payload_rows_per_dst=[11248, 9952, 7974, 9718, 8662, 7682, 9090, 9830] dup_rows_per_dst=[5696, 4940, 1854, 3332, 1968, 2224, 2102, 2032] combine_rows_per_src=[11248, 9952, 7974, 9718, 8662, 7682, 9090, 9830] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=268623872 dispatch_meta_bytes=342736 combine_payload_bytes=268623872 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=268966608 ingress_bytes=372432896 link_bytes=372432896 token_bytes=537590480 weight_movement_bytes=103809024 total_nvlink_bytes=641399504 weight_to_token_ratio=0.193101 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.401435004e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=260055040 dispatch_meta_bytes=334048 combine_payload_bytes=260055040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260389088 ingress_bytes=294658048 link_bytes=294658048 token_bytes=520444128 weight_movement_bytes=34603008 total_nvlink_bytes=555047136 weight_to_token_ratio=0.066487 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.108774513e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=257671168 dispatch_meta_bytes=325440 combine_payload_bytes=257671168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=257996608 ingress_bytes=326877184 link_bytes=326877184 token_bytes=515667776 weight_movement_bytes=69206016 total_nvlink_bytes=584873792 weight_to_token_ratio=0.134207 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=11648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.230012528e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=272179200 dispatch_meta_bytes=342776 combine_payload_bytes=272179200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272521976 ingress_bytes=306782208 link_bytes=306782208 token_bytes=544701176 weight_movement_bytes=34603008 total_nvlink_bytes=579304184 weight_to_token_ratio=0.063527 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=19456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.154396751e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=264896512 dispatch_meta_bytes=338192 combine_payload_bytes=264896512 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265234704 ingress_bytes=368705536 link_bytes=368705536 token_bytes=530131216 weight_movement_bytes=103809024 total_nvlink_bytes=633940240 weight_to_token_ratio=0.195818 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=8960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.387409249e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=259915776 dispatch_meta_bytes=329096 combine_payload_bytes=259915776 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=260244872 ingress_bytes=329121792 link_bytes=329121792 token_bytes=520160648 weight_movement_bytes=69206016 total_nvlink_bytes=589366664 weight_to_token_ratio=0.133047 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=12800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.238458807e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75124 payload_rows_remote=65414 dup_rows=23180 dup_rows_remote=20334 dedup_ratio=0.235799 router_tokens_per_expert=[1514, 604, 499, 1248, 447, 1531, 1160, 110, 2285, 732, 1276, 262, 4, 2685, 752, 812, 672, 925, 840, 712, 86, 191, 222, 1116, 301, 758, 663, 447, 940, 153, 724, 466, 1024, 636, 1838, 1069, 708, 204, 824, 696, 188, 413, 558, 162, 744, 816, 550, 119, 306, 389, 542, 647, 885, 2155, 356, 438, 2178, 411, 1609, 618, 143, 678, 1387, 724] payload_rows_per_dst=[8524, 8648, 9710, 9828, 10454, 9096, 9306, 9558] dup_rows_per_dst=[2674, 3598, 2846, 3432, 3544, 3374, 2130, 1582] combine_rows_per_src=[8524, 8648, 9710, 9828, 10454, 9096, 9306, 9558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78048 payload_rows_remote=68206 dup_rows=20256 dup_rows_remote=17404 dedup_ratio=0.206055 router_tokens_per_expert=[257, 623, 61, 279, 3620, 107, 378, 772, 6, 459, 3006, 323, 447, 668, 273, 3221, 418, 1052, 539, 271, 1449, 434, 1625, 1049, 1081, 354, 740, 2543, 2, 161, 83, 163, 8, 6, 250, 10, 532, 16, 458, 744, 1108, 522, 258, 750, 21, 1688, 1, 174, 5491, 376, 2782, 1, 13, 610, 157, 2048, 2049, 384, 818, 64, 559, 95, 519, 176] payload_rows_per_dst=[9842, 12960, 8680, 8710, 13406, 6972, 9612, 7866] dup_rows_per_dst=[2852, 7096, 1840, 1544, 1624, 2072, 1766, 1462] combine_rows_per_src=[9842, 12960, 8680, 8710, 13406, 6972, 9612, 7866] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80160 payload_rows_remote=71170 dup_rows=18144 dup_rows_remote=16682 dedup_ratio=0.184570 router_tokens_per_expert=[182, 547, 48, 255, 3590, 87, 475, 739, 8, 429, 3027, 334, 424, 697, 258, 3217, 332, 964, 513, 274, 1409, 388, 1575, 1156, 1149, 369, 726, 2661, 0, 140, 40, 141, 1, 7, 247, 6, 546, 11, 437, 664, 1033, 554, 245, 742, 13, 1782, 0, 167, 5855, 326, 3317, 0, 9, 512, 61, 2009, 1989, 383, 752, 24, 517, 102, 504, 183] payload_rows_per_dst=[9510, 11782, 10626, 8990, 14012, 7192, 10158, 7890] dup_rows_per_dst=[2336, 4756, 2596, 1462, 1536, 2130, 1912, 1416] combine_rows_per_src=[9510, 11782, 10626, 8990, 14012, 7192, 10158, 7890] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78578 payload_rows_remote=65766 dup_rows=19726 dup_rows_remote=12944 dedup_ratio=0.200663 router_tokens_per_expert=[240, 566, 47, 286, 3717, 77, 560, 798, 6, 468, 3026, 355, 445, 633, 286, 3047, 339, 1034, 533, 237, 1509, 437, 1531, 987, 1146, 374, 665, 2381, 0, 133, 39, 164, 9, 7, 260, 6, 476, 15, 569, 672, 1023, 508, 254, 731, 18, 1769, 0, 189, 5856, 384, 3164, 0, 12, 546, 96, 2005, 1991, 372, 841, 46, 494, 97, 492, 184] payload_rows_per_dst=[9970, 12812, 8456, 8400, 13998, 6926, 10340, 7676] dup_rows_per_dst=[2612, 6782, 1696, 1404, 1742, 2058, 2074, 1358] combine_rows_per_src=[9970, 12812, 8456, 8400, 13998, 6926, 10340, 7676] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75824 payload_rows_remote=62004 dup_rows=22480 dup_rows_remote=19020 dedup_ratio=0.228678 router_tokens_per_expert=[301, 587, 43, 217, 3886, 64, 368, 753, 10, 363, 3143, 380, 405, 540, 214, 3177, 314, 1137, 572, 249, 1330, 378, 1650, 1008, 1148, 345, 697, 2767, 1, 140, 31, 158, 7, 1, 262, 5, 522, 6, 533, 763, 990, 508, 263, 735, 7, 1608, 1, 211, 5893, 283, 3223, 0, 14, 467, 91, 1892, 1933, 402, 783, 47, 524, 88, 510, 204] payload_rows_per_dst=[9972, 3198, 10592, 12386, 4116, 10378, 13820, 11362] dup_rows_per_dst=[2466, 626, 2684, 4474, 82, 4622, 3460, 4066] combine_rows_per_src=[9972, 3198, 10592, 12386, 4116, 10378, 13820, 11362] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79228 payload_rows_remote=65014 dup_rows=19076 dup_rows_remote=17318 dedup_ratio=0.194051 router_tokens_per_expert=[157, 585, 36, 247, 3906, 36, 378, 675, 10, 407, 3125, 302, 390, 681, 285, 3326, 377, 1027, 489, 246, 1543, 414, 1401, 1188, 1059, 397, 692, 2498, 2, 127, 33, 142, 3, 1, 253, 4, 464, 31, 565, 707, 1005, 491, 252, 749, 15, 1755, 0, 151, 5958, 328, 3226, 0, 11, 623, 60, 1826, 2089, 365, 792, 42, 469, 115, 454, 167] payload_rows_per_dst=[9816, 8296, 10746, 8518, 14214, 10908, 5512, 11218] dup_rows_per_dst=[2224, 2104, 2624, 1382, 1758, 4580, 184, 4220] combine_rows_per_src=[9816, 8296, 10746, 8518, 14214, 10908, 5512, 11218] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76818 payload_rows_remote=66244 dup_rows=21486 dup_rows_remote=16540 dedup_ratio=0.218567 router_tokens_per_expert=[242, 545, 54, 252, 3660, 123, 452, 791, 4, 494, 2861, 384, 457, 652, 225, 3193, 303, 1033, 499, 255, 1443, 355, 1563, 1112, 1022, 473, 740, 2483, 4, 131, 72, 148, 6, 2, 277, 4, 532, 10, 467, 804, 1096, 565, 276, 837, 15, 1602, 1, 175, 5628, 411, 3040, 0, 11, 508, 84, 2147, 2002, 363, 849, 112, 487, 128, 489, 199] payload_rows_per_dst=[9694, 5148, 10666, 10678, 10600, 10574, 8552, 10906] dup_rows_per_dst=[2544, 1256, 2460, 3218, 1120, 4946, 1510, 4432] combine_rows_per_src=[9694, 5148, 10666, 10678, 10600, 10574, 8552, 10906] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80160 payload_rows_remote=71170 dup_rows=18144 dup_rows_remote=16682 dedup_ratio=0.184570 router_tokens_per_expert=[182, 547, 48, 255, 3590, 87, 475, 739, 8, 429, 3027, 334, 424, 697, 258, 3217, 332, 964, 513, 274, 1409, 388, 1575, 1156, 1149, 369, 726, 2661, 0, 140, 40, 141, 1, 7, 247, 6, 546, 11, 437, 664, 1033, 554, 245, 742, 13, 1782, 0, 167, 5855, 326, 3317, 0, 9, 512, 61, 2009, 1989, 383, 752, 24, 517, 102, 504, 183] payload_rows_per_dst=[9510, 11782, 10626, 8990, 14012, 7192, 10158, 7890] dup_rows_per_dst=[2336, 4756, 2596, 1462, 1536, 2130, 1912, 1416] combine_rows_per_src=[9510, 11782, 10626, 8990, 14012, 7192, 10158, 7890] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75320 payload_rows_remote=64136 dup_rows=22984 dup_rows_remote=18654 dedup_ratio=0.233805 router_tokens_per_expert=[157, 590, 44, 252, 3827, 55, 635, 658, 5, 408, 3044, 256, 407, 620, 241, 3533, 321, 990, 419, 242, 1346, 458, 1613, 846, 1148, 342, 686, 2769, 0, 79, 25, 169, 9, 3, 261, 2, 537, 13, 485, 754, 957, 547, 267, 826, 21, 1746, 0, 176, 5972, 296, 3135, 0, 12, 495, 62, 1769, 2098, 344, 836, 45, 475, 93, 559, 172] payload_rows_per_dst=[9896, 3180, 10178, 12094, 4068, 11058, 13662, 11184] dup_rows_per_dst=[2540, 694, 2292, 4430, 60, 5088, 3550, 4330] combine_rows_per_src=[9896, 3180, 10178, 12094, 4068, 11058, 13662, 11184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80478 payload_rows_remote=70624 dup_rows=17826 dup_rows_remote=15512 dedup_ratio=0.181335 router_tokens_per_expert=[170, 515, 33, 241, 3966, 70, 318, 702, 2, 410, 3173, 283, 426, 606, 222, 3426, 325, 973, 548, 262, 1441, 392, 1474, 928, 1197, 312, 684, 2840, 0, 119, 20, 135, 14, 3, 236, 1, 527, 1, 464, 644, 1005, 499, 245, 770, 6, 1712, 0, 163, 6094, 292, 3393, 0, 0, 477, 117, 1817, 2056, 392, 750, 24, 445, 89, 536, 167] payload_rows_per_dst=[9854, 12388, 9854, 9166, 14392, 6818, 10348, 7658] dup_rows_per_dst=[2176, 5226, 2314, 1448, 1576, 1982, 1844, 1260] combine_rows_per_src=[9854, 12388, 9854, 9166, 14392, 6818, 10348, 7658] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279371776 dispatch_meta_bytes=342440 combine_payload_bytes=279371776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279714216 ingress_bytes=313974784 link_bytes=313974784 token_bytes=559085992 weight_movement_bytes=34603008 total_nvlink_bytes=593689000 weight_to_token_ratio=0.061892 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=30208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.181461835e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78578 payload_rows_remote=65766 dup_rows=19726 dup_rows_remote=12944 dedup_ratio=0.200663 router_tokens_per_expert=[240, 566, 47, 286, 3717, 77, 560, 798, 6, 468, 3026, 355, 445, 633, 286, 3047, 339, 1034, 533, 237, 1509, 437, 1531, 987, 1146, 374, 665, 2381, 0, 133, 39, 164, 9, 7, 260, 6, 476, 15, 569, 672, 1023, 508, 254, 731, 18, 1769, 0, 189, 5856, 384, 3164, 0, 12, 546, 96, 2005, 1991, 372, 841, 46, 494, 97, 492, 184] payload_rows_per_dst=[9970, 12812, 8456, 8400, 13998, 6926, 10340, 7676] dup_rows_per_dst=[2612, 6782, 1696, 1404, 1742, 2058, 2074, 1358] combine_rows_per_src=[9970, 12812, 8456, 8400, 13998, 6926, 10340, 7676] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=291512320 dispatch_meta_bytes=351408 combine_payload_bytes=291512320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291863728 ingress_bytes=326115328 link_bytes=326115328 token_bytes=583376048 weight_movement_bytes=34603008 total_nvlink_bytes=617979056 weight_to_token_ratio=0.059315 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20992 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.227145725e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=269377536 dispatch_meta_bytes=314840 combine_payload_bytes=269377536 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=269692376 ingress_bytes=373186560 link_bytes=373186560 token_bytes=539069912 weight_movement_bytes=103809024 total_nvlink_bytes=642878936 weight_to_token_ratio=0.192571 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.404270981e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267935744 dispatch_meta_bytes=342992 combine_payload_bytes=267935744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268278736 ingress_bytes=302538752 link_bytes=302538752 token_bytes=536214480 weight_movement_bytes=34603008 total_nvlink_bytes=570817488 weight_to_token_ratio=0.064532 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138428967e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=253968384 dispatch_meta_bytes=324096 combine_payload_bytes=253968384 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=254292480 ingress_bytes=392380416 link_bytes=392380416 token_bytes=508260864 weight_movement_bytes=138412032 total_nvlink_bytes=646672896 weight_to_token_ratio=0.272325 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.476495916e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266297344 dispatch_meta_bytes=329328 combine_payload_bytes=266297344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266626672 ingress_bytes=300900352 link_bytes=300900352 token_bytes=532924016 weight_movement_bytes=34603008 total_nvlink_bytes=567527024 weight_to_token_ratio=0.064930 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=33024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132263800e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=271335424 dispatch_meta_bytes=331136 combine_payload_bytes=271335424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271666560 ingress_bytes=305938432 link_bytes=305938432 token_bytes=543001984 weight_movement_bytes=34603008 total_nvlink_bytes=577604992 weight_to_token_ratio=0.063725 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.151221690e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=291512320 dispatch_meta_bytes=351408 combine_payload_bytes=291512320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291863728 ingress_bytes=326115328 link_bytes=326115328 token_bytes=583376048 weight_movement_bytes=34603008 total_nvlink_bytes=617979056 weight_to_token_ratio=0.059315 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20992 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.227145725e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=289275904 dispatch_meta_bytes=344544 combine_payload_bytes=289275904 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=289620448 ingress_bytes=358481920 link_bytes=358481920 token_bytes=578896352 weight_movement_bytes=69206016 total_nvlink_bytes=648102368 weight_to_token_ratio=0.119548 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.348938605e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=262701056 dispatch_meta_bytes=331160 combine_payload_bytes=262701056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263032216 ingress_bytes=297304064 link_bytes=297304064 token_bytes=525733272 weight_movement_bytes=34603008 total_nvlink_bytes=560336280 weight_to_token_ratio=0.065819 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=16256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.118731258e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=269377536 dispatch_meta_bytes=314840 combine_payload_bytes=269377536 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=269692376 ingress_bytes=373186560 link_bytes=373186560 token_bytes=539069912 weight_movement_bytes=103809024 total_nvlink_bytes=642878936 weight_to_token_ratio=0.192571 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.404270981e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75824 payload_rows_remote=62004 dup_rows=22480 dup_rows_remote=19020 dedup_ratio=0.228678 router_tokens_per_expert=[301, 587, 43, 217, 3886, 64, 368, 753, 10, 363, 3143, 380, 405, 540, 214, 3177, 314, 1137, 572, 249, 1330, 378, 1650, 1008, 1148, 345, 697, 2767, 1, 140, 31, 158, 7, 1, 262, 5, 522, 6, 533, 763, 990, 508, 263, 735, 7, 1608, 1, 211, 5893, 283, 3223, 0, 14, 467, 91, 1892, 1933, 402, 783, 47, 524, 88, 510, 204] payload_rows_per_dst=[9972, 3198, 10592, 12386, 4116, 10378, 13820, 11362] dup_rows_per_dst=[2466, 626, 2684, 4474, 82, 4622, 3460, 4066] combine_rows_per_src=[9972, 3198, 10592, 12386, 4116, 10378, 13820, 11362] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78048 payload_rows_remote=68206 dup_rows=20256 dup_rows_remote=17404 dedup_ratio=0.206055 router_tokens_per_expert=[257, 623, 61, 279, 3620, 107, 378, 772, 6, 459, 3006, 323, 447, 668, 273, 3221, 418, 1052, 539, 271, 1449, 434, 1625, 1049, 1081, 354, 740, 2543, 2, 161, 83, 163, 8, 6, 250, 10, 532, 16, 458, 744, 1108, 522, 258, 750, 21, 1688, 1, 174, 5491, 376, 2782, 1, 13, 610, 157, 2048, 2049, 384, 818, 64, 559, 95, 519, 176] payload_rows_per_dst=[9842, 12960, 8680, 8710, 13406, 6972, 9612, 7866] dup_rows_per_dst=[2852, 7096, 1840, 1544, 1624, 2072, 1766, 1462] combine_rows_per_src=[9842, 12960, 8680, 8710, 13406, 6972, 9612, 7866] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=253968384 dispatch_meta_bytes=324096 combine_payload_bytes=253968384 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=254292480 ingress_bytes=392380416 link_bytes=392380416 token_bytes=508260864 weight_movement_bytes=138412032 total_nvlink_bytes=646672896 weight_to_token_ratio=0.272325 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.476495916e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78314 payload_rows_remote=70912 dup_rows=19990 dup_rows_remote=18190 dedup_ratio=0.203349 router_tokens_per_expert=[0, 1052, 87, 155, 0, 133, 127, 391, 54, 987, 629, 2413, 576, 332, 0, 0, 1666, 1668, 1696, 203, 0, 0, 4035, 959, 1037, 1635, 2851, 3892, 0, 63, 205, 0, 123, 798, 102, 68, 6219, 993, 188, 110, 789, 0, 5, 17, 818, 1644, 97, 96, 96, 0, 544, 620, 0, 0, 111, 2, 2993, 362, 0, 10, 0, 2671, 0, 2830] payload_rows_per_dst=[9876, 8812, 8832, 9368, 7402, 11234, 10372, 12418] dup_rows_per_dst=[1798, 1170, 2050, 2214, 1800, 3698, 1946, 5314] combine_rows_per_src=[9876, 8812, 8832, 9368, 7402, 11234, 10372, 12418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76370 payload_rows_remote=64278 dup_rows=21934 dup_rows_remote=17718 dedup_ratio=0.223124 router_tokens_per_expert=[0, 804, 91, 159, 0, 176, 159, 366, 39, 1060, 554, 2388, 544, 346, 2, 1, 1628, 1570, 1581, 188, 3, 2, 3887, 887, 816, 1622, 3118, 4020, 3, 71, 242, 0, 243, 916, 177, 70, 6207, 1002, 240, 108, 676, 3, 34, 16, 728, 2128, 94, 91, 161, 2, 580, 645, 1, 0, 155, 5, 2654, 345, 3, 10, 5, 2892, 0, 2634] payload_rows_per_dst=[12092, 11574, 12196, 10200, 13380, 6236, 9968, 724] dup_rows_per_dst=[4216, 3602, 4790, 2570, 4546, 1304, 904, 2] combine_rows_per_src=[12092, 11574, 12196, 10200, 13380, 6236, 9968, 724] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79454 payload_rows_remote=70030 dup_rows=18850 dup_rows_remote=16602 dedup_ratio=0.191752 router_tokens_per_expert=[0, 1018, 102, 117, 0, 132, 129, 352, 55, 1054, 551, 2415, 497, 355, 0, 0, 1619, 1780, 1566, 156, 0, 0, 4029, 956, 876, 1618, 3040, 4075, 0, 60, 242, 0, 122, 867, 171, 84, 6338, 909, 219, 95, 719, 0, 2, 15, 607, 1791, 80, 81, 124, 0, 472, 578, 0, 0, 116, 1, 2974, 305, 0, 7, 0, 2618, 0, 3063] payload_rows_per_dst=[10094, 9400, 9562, 9424, 13448, 5688, 9962, 11876] dup_rows_per_dst=[1756, 1930, 2592, 2248, 4162, 902, 678, 4582] combine_rows_per_src=[10094, 9400, 9562, 9424, 13448, 5688, 9962, 11876] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79864 payload_rows_remote=69606 dup_rows=18440 dup_rows_remote=15336 dedup_ratio=0.187581 router_tokens_per_expert=[0, 900, 103, 146, 0, 136, 112, 296, 44, 1154, 592, 2429, 468, 392, 0, 0, 1759, 1623, 1730, 153, 3, 0, 3925, 916, 844, 1722, 2972, 4225, 0, 66, 215, 0, 136, 853, 68, 38, 6507, 932, 183, 108, 655, 0, 3, 10, 561, 1761, 77, 75, 14, 0, 680, 597, 0, 0, 121, 0, 2859, 305, 1, 5, 0, 2701, 0, 2977] payload_rows_per_dst=[10148, 11720, 10258, 9382, 13588, 5430, 9912, 9426] dup_rows_per_dst=[1688, 4156, 3104, 2256, 4064, 854, 762, 1556] combine_rows_per_src=[10148, 11720, 10258, 9382, 13588, 5430, 9912, 9426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74810 payload_rows_remote=63114 dup_rows=23494 dup_rows_remote=19350 dedup_ratio=0.238993 router_tokens_per_expert=[1, 1015, 70, 166, 0, 149, 124, 350, 40, 959, 552, 2351, 585, 340, 1, 0, 1660, 1738, 1661, 172, 1, 0, 3894, 910, 1040, 1623, 2964, 3815, 1, 74, 221, 0, 202, 920, 130, 68, 6138, 1122, 243, 114, 703, 0, 43, 21, 832, 1612, 83, 82, 81, 0, 581, 721, 0, 0, 108, 6, 3092, 375, 5, 6, 0, 2674, 0, 2713] payload_rows_per_dst=[9600, 11696, 12524, 10944, 13612, 5716, 9946, 772] dup_rows_per_dst=[1780, 4144, 5186, 4770, 5742, 1036, 836, 0] combine_rows_per_src=[9600, 11696, 12524, 10944, 13612, 5716, 9946, 772] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78084 payload_rows_remote=67166 dup_rows=20220 dup_rows_remote=17112 dedup_ratio=0.205688 router_tokens_per_expert=[0, 915, 90, 155, 0, 132, 127, 311, 62, 990, 655, 2656, 502, 357, 0, 0, 1723, 1649, 1815, 156, 0, 0, 3874, 847, 884, 1647, 2980, 4191, 0, 91, 211, 1, 195, 900, 139, 46, 6414, 1081, 204, 85, 640, 0, 3, 17, 850, 1481, 85, 88, 20, 0, 574, 710, 0, 0, 112, 0, 2883, 271, 2, 5, 0, 2709, 0, 2617] payload_rows_per_dst=[10086, 9114, 7584, 9274, 4550, 14486, 10918, 12072] dup_rows_per_dst=[1756, 1330, 1350, 2354, 750, 4670, 3108, 4902] combine_rows_per_src=[10086, 9114, 7584, 9274, 4550, 14486, 10918, 12072] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78314 payload_rows_remote=70912 dup_rows=19990 dup_rows_remote=18190 dedup_ratio=0.203349 router_tokens_per_expert=[0, 1052, 87, 155, 0, 133, 127, 391, 54, 987, 629, 2413, 576, 332, 0, 0, 1666, 1668, 1696, 203, 0, 0, 4035, 959, 1037, 1635, 2851, 3892, 0, 63, 205, 0, 123, 798, 102, 68, 6219, 993, 188, 110, 789, 0, 5, 17, 818, 1644, 97, 96, 96, 0, 544, 620, 0, 0, 111, 2, 2993, 362, 0, 10, 0, 2671, 0, 2830] payload_rows_per_dst=[9876, 8812, 8832, 9368, 7402, 11234, 10372, 12418] dup_rows_per_dst=[1798, 1170, 2050, 2214, 1800, 3698, 1946, 5314] combine_rows_per_src=[9876, 8812, 8832, 9368, 7402, 11234, 10372, 12418] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77504 payload_rows_remote=63580 dup_rows=20800 dup_rows_remote=16492 dedup_ratio=0.211589 router_tokens_per_expert=[0, 832, 155, 145, 0, 135, 124, 423, 44, 1179, 605, 2291, 560, 326, 0, 1, 1869, 1518, 1662, 249, 5, 0, 3820, 999, 1026, 1491, 2958, 3886, 4, 71, 241, 0, 220, 842, 198, 88, 6100, 1220, 259, 117, 600, 0, 8, 9, 880, 1384, 70, 65, 39, 1, 873, 620, 0, 1, 100, 4, 2811, 417, 0, 3, 0, 2745, 1, 2858] payload_rows_per_dst=[9668, 8744, 7528, 9258, 4880, 13924, 11062, 12440] dup_rows_per_dst=[1732, 1268, 1338, 2324, 1008, 4308, 3592, 5230] combine_rows_per_src=[9668, 8744, 7528, 9258, 4880, 13924, 11062, 12440] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279371776 dispatch_meta_bytes=342440 combine_payload_bytes=279371776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279714216 ingress_bytes=313974784 link_bytes=313974784 token_bytes=559085992 weight_movement_bytes=34603008 total_nvlink_bytes=593689000 weight_to_token_ratio=0.061892 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=30208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.181461835e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77686 payload_rows_remote=65508 dup_rows=20618 dup_rows_remote=15806 dedup_ratio=0.209737 router_tokens_per_expert=[0, 987, 76, 177, 0, 97, 156, 298, 54, 1022, 655, 2247, 528, 316, 0, 0, 1917, 1714, 1694, 156, 3, 0, 3880, 1138, 907, 1748, 2929, 4216, 0, 38, 237, 0, 227, 940, 124, 63, 6441, 1041, 193, 98, 607, 0, 1, 16, 506, 1714, 63, 61, 45, 1, 721, 535, 0, 0, 70, 0, 2785, 319, 0, 14, 0, 2725, 0, 2652] payload_rows_per_dst=[10112, 8480, 7858, 9434, 4446, 14324, 10854, 12178] dup_rows_per_dst=[1902, 1164, 1552, 2284, 926, 4494, 3484, 4812] combine_rows_per_src=[10112, 8480, 7858, 9434, 4446, 14324, 10854, 12178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80478 payload_rows_remote=70624 dup_rows=17826 dup_rows_remote=15512 dedup_ratio=0.181335 router_tokens_per_expert=[170, 515, 33, 241, 3966, 70, 318, 702, 2, 410, 3173, 283, 426, 606, 222, 3426, 325, 973, 548, 262, 1441, 392, 1474, 928, 1197, 312, 684, 2840, 0, 119, 20, 135, 14, 3, 236, 1, 527, 1, 464, 644, 1005, 499, 245, 770, 6, 1712, 0, 163, 6094, 292, 3393, 0, 0, 477, 117, 1817, 2056, 392, 750, 24, 445, 89, 536, 167] payload_rows_per_dst=[9854, 12388, 9854, 9166, 14392, 6818, 10348, 7658] dup_rows_per_dst=[2176, 5226, 2314, 1448, 1576, 1982, 1844, 1260] combine_rows_per_src=[9854, 12388, 9854, 9166, 14392, 6818, 10348, 7658] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74810 payload_rows_remote=63114 dup_rows=23494 dup_rows_remote=19350 dedup_ratio=0.238993 router_tokens_per_expert=[1, 1015, 70, 166, 0, 149, 124, 350, 40, 959, 552, 2351, 585, 340, 1, 0, 1660, 1738, 1661, 172, 1, 0, 3894, 910, 1040, 1623, 2964, 3815, 1, 74, 221, 0, 202, 920, 130, 68, 6138, 1122, 243, 114, 703, 0, 43, 21, 832, 1612, 83, 82, 81, 0, 581, 721, 0, 0, 108, 6, 3092, 375, 5, 6, 0, 2674, 0, 2713] payload_rows_per_dst=[9600, 11696, 12524, 10944, 13612, 5716, 9946, 772] dup_rows_per_dst=[1780, 4144, 5186, 4770, 5742, 1036, 836, 0] combine_rows_per_src=[9600, 11696, 12524, 10944, 13612, 5716, 9946, 772] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=263282688 dispatch_meta_bytes=327984 combine_payload_bytes=263282688 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=263610672 ingress_bytes=332488704 link_bytes=332488704 token_bytes=526893360 weight_movement_bytes=69206016 total_nvlink_bytes=596099376 weight_to_token_ratio=0.131347 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=32384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.251128226e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=290455552 dispatch_meta_bytes=356408 combine_payload_bytes=290455552 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=290811960 ingress_bytes=359661568 link_bytes=359661568 token_bytes=581267512 weight_movement_bytes=69206016 total_nvlink_bytes=650473528 weight_to_token_ratio=0.119061 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=27520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.353377525e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=286842880 dispatch_meta_bytes=346528 combine_payload_bytes=286842880 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=287189408 ingress_bytes=390651904 link_bytes=390651904 token_bytes=574032288 weight_movement_bytes=103809024 total_nvlink_bytes=677841312 weight_to_token_ratio=0.180842 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=21504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.469991665e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=285106176 dispatch_meta_bytes=339768 combine_payload_bytes=285106176 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=285445944 ingress_bytes=388915200 link_bytes=388915200 token_bytes=570552120 weight_movement_bytes=103809024 total_nvlink_bytes=674361144 weight_to_token_ratio=0.181945 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=13440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.463456587e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=275111936 dispatch_meta_bytes=337112 combine_payload_bytes=275111936 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=275449048 ingress_bytes=344317952 link_bytes=344317952 token_bytes=550560984 weight_movement_bytes=69206016 total_nvlink_bytes=619767000 weight_to_token_ratio=0.125701 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=31360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.295640734e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=258514944 dispatch_meta_bytes=329856 combine_payload_bytes=258514944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258844800 ingress_bytes=293117952 link_bytes=293117952 token_bytes=517359744 weight_movement_bytes=34603008 total_nvlink_bytes=551962752 weight_to_token_ratio=0.066884 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=19200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.102979255e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=290455552 dispatch_meta_bytes=356408 combine_payload_bytes=290455552 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=290811960 ingress_bytes=359661568 link_bytes=359661568 token_bytes=581267512 weight_movement_bytes=69206016 total_nvlink_bytes=650473528 weight_to_token_ratio=0.119061 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=27520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.353377525e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=260423680 dispatch_meta_bytes=320288 combine_payload_bytes=260423680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260743968 ingress_bytes=295026688 link_bytes=295026688 token_bytes=521167648 weight_movement_bytes=34603008 total_nvlink_bytes=555770656 weight_to_token_ratio=0.066395 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23040 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.110161675e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=268320768 dispatch_meta_bytes=325256 combine_payload_bytes=268320768 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=268646024 ingress_bytes=406732800 link_bytes=406732800 token_bytes=536966792 weight_movement_bytes=138412032 total_nvlink_bytes=675378824 weight_to_token_ratio=0.257766 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=16512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.530502782e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=258514944 dispatch_meta_bytes=329856 combine_payload_bytes=258514944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258844800 ingress_bytes=293117952 link_bytes=293117952 token_bytes=517359744 weight_movement_bytes=34603008 total_nvlink_bytes=551962752 weight_to_token_ratio=0.066884 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=19200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.102979255e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78084 payload_rows_remote=67166 dup_rows=20220 dup_rows_remote=17112 dedup_ratio=0.205688 router_tokens_per_expert=[0, 915, 90, 155, 0, 132, 127, 311, 62, 990, 655, 2656, 502, 357, 0, 0, 1723, 1649, 1815, 156, 0, 0, 3874, 847, 884, 1647, 2980, 4191, 0, 91, 211, 1, 195, 900, 139, 46, 6414, 1081, 204, 85, 640, 0, 3, 17, 850, 1481, 85, 88, 20, 0, 574, 710, 0, 0, 112, 0, 2883, 271, 2, 5, 0, 2709, 0, 2617] payload_rows_per_dst=[10086, 9114, 7584, 9274, 4550, 14486, 10918, 12072] dup_rows_per_dst=[1756, 1330, 1350, 2354, 750, 4670, 3108, 4902] combine_rows_per_src=[10086, 9114, 7584, 9274, 4550, 14486, 10918, 12072] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76370 payload_rows_remote=64278 dup_rows=21934 dup_rows_remote=17718 dedup_ratio=0.223124 router_tokens_per_expert=[0, 804, 91, 159, 0, 176, 159, 366, 39, 1060, 554, 2388, 544, 346, 2, 1, 1628, 1570, 1581, 188, 3, 2, 3887, 887, 816, 1622, 3118, 4020, 3, 71, 242, 0, 243, 916, 177, 70, 6207, 1002, 240, 108, 676, 3, 34, 16, 728, 2128, 94, 91, 161, 2, 580, 645, 1, 0, 155, 5, 2654, 345, 3, 10, 5, 2892, 0, 2634] payload_rows_per_dst=[12092, 11574, 12196, 10200, 13380, 6236, 9968, 724] dup_rows_per_dst=[4216, 3602, 4790, 2570, 4546, 1304, 904, 2] combine_rows_per_src=[12092, 11574, 12196, 10200, 13380, 6236, 9968, 724] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=275111936 dispatch_meta_bytes=337112 combine_payload_bytes=275111936 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=275449048 ingress_bytes=344317952 link_bytes=344317952 token_bytes=550560984 weight_movement_bytes=69206016 total_nvlink_bytes=619767000 weight_to_token_ratio=0.125701 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=31360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.295640734e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83874 payload_rows_remote=68912 dup_rows=14430 dup_rows_remote=13904 dedup_ratio=0.146790 router_tokens_per_expert=[0, 0, 158, 0, 1367, 1047, 1173, 298, 4852, 193, 356, 1, 258, 0, 7068, 43, 2248, 0, 756, 318, 444, 219, 0, 81, 328, 1705, 428, 349, 188, 2633, 474, 62, 0, 2, 94, 0, 433, 21, 126, 0, 22, 4529, 92, 334, 99, 0, 0, 5193, 0, 108, 94, 621, 0, 3448, 0, 0, 0, 0, 913, 0, 2177, 3565, 0, 234] payload_rows_per_dst=[6608, 10738, 6848, 10604, 14962, 11904, 10924, 11286] dup_rows_per_dst=[1478, 564, 1388, 1730, 526, 2770, 3482, 2492] combine_rows_per_src=[6608, 10738, 6848, 10604, 14962, 11904, 10924, 11286] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=289275904 dispatch_meta_bytes=344544 combine_payload_bytes=289275904 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=289620448 ingress_bytes=358481920 link_bytes=358481920 token_bytes=578896352 weight_movement_bytes=69206016 total_nvlink_bytes=648102368 weight_to_token_ratio=0.119548 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.348938605e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71802 payload_rows_remote=59050 dup_rows=26502 dup_rows_remote=22008 dedup_ratio=0.269592 router_tokens_per_expert=[0, 0, 155, 2, 1224, 877, 1229, 275, 4271, 230, 420, 5, 259, 2, 7001, 39, 2455, 8, 757, 354, 517, 255, 0, 97, 635, 1715, 559, 481, 226, 2688, 453, 114, 0, 4, 90, 4, 536, 56, 169, 7, 31, 4328, 71, 365, 132, 3, 0, 4861, 5, 114, 88, 653, 4, 3364, 0, 5, 5, 6, 1065, 3, 1997, 3625, 0, 258] payload_rows_per_dst=[12752, 15722, 7316, 11004, 1716, 8772, 8154, 6366] dup_rows_per_dst=[4494, 15982, 1570, 2738, 16, 1088, 312, 302] combine_rows_per_src=[12752, 15722, 7316, 11004, 1716, 8772, 8154, 6366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79346 payload_rows_remote=67618 dup_rows=18958 dup_rows_remote=14264 dedup_ratio=0.192851 router_tokens_per_expert=[0, 0, 142, 0, 1187, 1026, 1171, 259, 4854, 232, 273, 0, 269, 0, 7042, 30, 2455, 11, 777, 373, 423, 267, 0, 67, 330, 1462, 500, 388, 197, 3007, 505, 82, 0, 5, 81, 0, 595, 42, 170, 0, 29, 3967, 56, 335, 99, 0, 0, 4759, 0, 110, 85, 646, 1, 3402, 0, 0, 0, 0, 948, 0, 2388, 3808, 0, 297] payload_rows_per_dst=[6236, 1590, 12574, 9266, 15022, 11058, 11728, 11872] dup_rows_per_dst=[1334, 18, 5880, 1358, 848, 1816, 4694, 3010] combine_rows_per_src=[6236, 1590, 12574, 9266, 15022, 11058, 11728, 11872] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83888 payload_rows_remote=72988 dup_rows=14416 dup_rows_remote=12202 dedup_ratio=0.146647 router_tokens_per_expert=[0, 0, 178, 0, 1411, 956, 1243, 253, 4783, 200, 349, 0, 252, 0, 7107, 31, 2544, 2, 770, 314, 367, 209, 0, 84, 418, 1604, 514, 367, 210, 2865, 486, 93, 0, 3, 80, 0, 491, 39, 157, 0, 16, 4180, 82, 378, 99, 0, 0, 5163, 0, 99, 80, 592, 1, 3217, 0, 0, 0, 0, 874, 0, 2220, 3545, 0, 226] payload_rows_per_dst=[11318, 10656, 7212, 10900, 15010, 9938, 7708, 11146] dup_rows_per_dst=[4234, 574, 1368, 2214, 744, 2428, 270, 2584] combine_rows_per_src=[11318, 10656, 7212, 10900, 15010, 9938, 7708, 11146] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80998 payload_rows_remote=70222 dup_rows=17306 dup_rows_remote=16456 dedup_ratio=0.176046 router_tokens_per_expert=[1, 0, 165, 0, 1368, 858, 1287, 194, 4591, 230, 399, 1, 285, 1, 7088, 39, 2167, 2, 773, 365, 486, 198, 0, 64, 604, 1476, 575, 401, 211, 2859, 438, 35, 0, 8, 70, 0, 440, 38, 172, 0, 24, 4582, 80, 400, 86, 1, 0, 5222, 0, 103, 70, 715, 0, 3367, 0, 0, 0, 0, 861, 3, 2140, 3364, 0, 245] payload_rows_per_dst=[6426, 1900, 12236, 10878, 15086, 10776, 12622, 11074] dup_rows_per_dst=[1320, 10, 5056, 2320, 546, 850, 5052, 2152] combine_rows_per_src=[6426, 1900, 12236, 10878, 15086, 10776, 12622, 11074] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=263282688 dispatch_meta_bytes=327984 combine_payload_bytes=263282688 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=263610672 ingress_bytes=332488704 link_bytes=332488704 token_bytes=526893360 weight_movement_bytes=69206016 total_nvlink_bytes=596099376 weight_to_token_ratio=0.131347 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=32384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.251128226e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83874 payload_rows_remote=68912 dup_rows=14430 dup_rows_remote=13904 dedup_ratio=0.146790 router_tokens_per_expert=[0, 0, 158, 0, 1367, 1047, 1173, 298, 4852, 193, 356, 1, 258, 0, 7068, 43, 2248, 0, 756, 318, 444, 219, 0, 81, 328, 1705, 428, 349, 188, 2633, 474, 62, 0, 2, 94, 0, 433, 21, 126, 0, 22, 4529, 92, 334, 99, 0, 0, 5193, 0, 108, 94, 621, 0, 3448, 0, 0, 0, 0, 913, 0, 2177, 3565, 0, 234] payload_rows_per_dst=[6608, 10738, 6848, 10604, 14962, 11904, 10924, 11286] dup_rows_per_dst=[1478, 564, 1388, 1730, 526, 2770, 3482, 2492] combine_rows_per_src=[6608, 10738, 6848, 10604, 14962, 11904, 10924, 11286] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73318 payload_rows_remote=58144 dup_rows=24986 dup_rows_remote=12610 dedup_ratio=0.254171 router_tokens_per_expert=[0, 0, 186, 0, 1205, 1094, 1224, 259, 4753, 221, 376, 1, 268, 0, 6985, 31, 2378, 6, 817, 350, 467, 247, 0, 95, 630, 1587, 495, 377, 207, 2961, 545, 91, 0, 11, 82, 0, 540, 46, 159, 1, 23, 4026, 80, 338, 100, 1, 1, 4901, 0, 121, 84, 635, 4, 2888, 0, 0, 2, 0, 990, 1, 2278, 3730, 0, 254] payload_rows_per_dst=[12850, 15174, 7224, 11886, 2468, 8132, 7196, 8388] dup_rows_per_dst=[4888, 12376, 1496, 3830, 48, 1006, 268, 1074] combine_rows_per_src=[12850, 15174, 7224, 11886, 2468, 8132, 7196, 8388] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78708 payload_rows_remote=68116 dup_rows=19596 dup_rows_remote=17644 dedup_ratio=0.199341 router_tokens_per_expert=[0, 0, 153, 0, 1469, 813, 1147, 193, 4859, 147, 320, 0, 273, 0, 7256, 19, 2892, 6, 800, 302, 538, 203, 0, 50, 233, 1534, 485, 323, 191, 2869, 543, 38, 1, 3, 62, 0, 412, 40, 115, 0, 16, 4486, 79, 412, 74, 0, 1, 5261, 0, 141, 63, 531, 4, 3523, 0, 0, 0, 0, 781, 0, 1888, 3390, 1, 212] payload_rows_per_dst=[6250, 1506, 12892, 6522, 15296, 13256, 12394, 10592] dup_rows_per_dst=[1300, 12, 6408, 172, 482, 4168, 5102, 1952] combine_rows_per_src=[6250, 1506, 12892, 6522, 15296, 13256, 12394, 10592] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=282263552 dispatch_meta_bytes=331264 combine_payload_bytes=282263552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282594816 ingress_bytes=316866560 link_bytes=316866560 token_bytes=564858368 weight_movement_bytes=34603008 total_nvlink_bytes=599461376 weight_to_token_ratio=0.061260 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=43136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.192343355e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=298958848 dispatch_meta_bytes=340760 combine_payload_bytes=298958848 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=299299608 ingress_bytes=368164864 link_bytes=368164864 token_bytes=598258456 weight_movement_bytes=69206016 total_nvlink_bytes=667464472 weight_to_token_ratio=0.115679 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.385374744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79864 payload_rows_remote=69606 dup_rows=18440 dup_rows_remote=15336 dedup_ratio=0.187581 router_tokens_per_expert=[0, 900, 103, 146, 0, 136, 112, 296, 44, 1154, 592, 2429, 468, 392, 0, 0, 1759, 1623, 1730, 153, 3, 0, 3925, 916, 844, 1722, 2972, 4225, 0, 66, 215, 0, 136, 853, 68, 38, 6507, 932, 183, 108, 655, 0, 3, 10, 561, 1761, 77, 75, 14, 0, 680, 597, 0, 0, 121, 0, 2859, 305, 1, 5, 0, 2701, 0, 2977] payload_rows_per_dst=[10148, 11720, 10258, 9382, 13588, 5430, 9912, 9426] dup_rows_per_dst=[1688, 4156, 3104, 2256, 4064, 854, 762, 1556] combine_rows_per_src=[10148, 11720, 10258, 9382, 13588, 5430, 9912, 9426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80998 payload_rows_remote=70222 dup_rows=17306 dup_rows_remote=16456 dedup_ratio=0.176046 router_tokens_per_expert=[1, 0, 165, 0, 1368, 858, 1287, 194, 4591, 230, 399, 1, 285, 1, 7088, 39, 2167, 2, 773, 365, 486, 198, 0, 64, 604, 1476, 575, 401, 211, 2859, 438, 35, 0, 8, 70, 0, 440, 38, 172, 0, 24, 4582, 80, 400, 86, 1, 0, 5222, 0, 103, 70, 715, 0, 3367, 0, 0, 0, 0, 861, 3, 2140, 3364, 0, 245] payload_rows_per_dst=[6426, 1900, 12236, 10878, 15086, 10776, 12622, 11074] dup_rows_per_dst=[1320, 10, 5056, 2320, 546, 850, 5052, 2152] combine_rows_per_src=[6426, 1900, 12236, 10878, 15086, 10776, 12622, 11074] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=241868800 dispatch_meta_bytes=324232 combine_payload_bytes=241868800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=242193032 ingress_bytes=276471808 link_bytes=276471808 token_bytes=484061832 weight_movement_bytes=34603008 total_nvlink_bytes=518664840 weight_to_token_ratio=0.071485 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=18304 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.040341156e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=276963328 dispatch_meta_bytes=327528 combine_payload_bytes=276963328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277290856 ingress_bytes=311566336 link_bytes=311566336 token_bytes=554254184 weight_movement_bytes=34603008 total_nvlink_bytes=588857192 weight_to_token_ratio=0.062432 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=26624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.172399039e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=287629312 dispatch_meta_bytes=346712 combine_payload_bytes=287629312 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=287976024 ingress_bytes=391438336 link_bytes=391438336 token_bytes=575605336 weight_movement_bytes=103809024 total_nvlink_bytes=679414360 weight_to_token_ratio=0.180348 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=21888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.472950945e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86156 payload_rows_remote=78752 dup_rows=12148 dup_rows_remote=10716 dedup_ratio=0.123576 router_tokens_per_expert=[0, 0, 123, 0, 1309, 806, 1117, 260, 5028, 179, 267, 0, 251, 0, 7110, 16, 2553, 1, 756, 313, 568, 187, 0, 40, 295, 1465, 480, 410, 185, 3086, 452, 68, 0, 2, 72, 0, 417, 28, 136, 2, 27, 4602, 77, 378, 87, 0, 0, 4793, 0, 78, 76, 645, 0, 3400, 0, 0, 0, 0, 899, 0, 2030, 3840, 0, 238] payload_rows_per_dst=[12634, 11050, 7404, 11006, 15024, 9268, 8180, 11590] dup_rows_per_dst=[4182, 432, 1432, 1876, 510, 1074, 218, 2424] combine_rows_per_src=[12634, 11050, 7404, 11006, 15024, 9268, 8180, 11590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=298958848 dispatch_meta_bytes=340760 combine_payload_bytes=298958848 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=299299608 ingress_bytes=368164864 link_bytes=368164864 token_bytes=598258456 weight_movement_bytes=69206016 total_nvlink_bytes=667464472 weight_to_token_ratio=0.115679 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.385374744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=238157824 dispatch_meta_bytes=283016 combine_payload_bytes=238157824 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=238440840 ingress_bytes=341966848 link_bytes=341966848 token_bytes=476598664 weight_movement_bytes=103809024 total_nvlink_bytes=580407688 weight_to_token_ratio=0.217812 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.286793718e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=282263552 dispatch_meta_bytes=331264 combine_payload_bytes=282263552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282594816 ingress_bytes=316866560 link_bytes=316866560 token_bytes=564858368 weight_movement_bytes=34603008 total_nvlink_bytes=599461376 weight_to_token_ratio=0.061260 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=43136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.192343355e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=285106176 dispatch_meta_bytes=339768 combine_payload_bytes=285106176 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=285445944 ingress_bytes=388915200 link_bytes=388915200 token_bytes=570552120 weight_movement_bytes=103809024 total_nvlink_bytes=674361144 weight_to_token_ratio=0.181945 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=13440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.463456587e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279003136 dispatch_meta_bytes=343040 combine_payload_bytes=279003136 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=279346176 ingress_bytes=348209152 link_bytes=348209152 token_bytes=558349312 weight_movement_bytes=69206016 total_nvlink_bytes=627555328 weight_to_token_ratio=0.123948 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=22784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.310283006e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79346 payload_rows_remote=67618 dup_rows=18958 dup_rows_remote=14264 dedup_ratio=0.192851 router_tokens_per_expert=[0, 0, 142, 0, 1187, 1026, 1171, 259, 4854, 232, 273, 0, 269, 0, 7042, 30, 2455, 11, 777, 373, 423, 267, 0, 67, 330, 1462, 500, 388, 197, 3007, 505, 82, 0, 5, 81, 0, 595, 42, 170, 0, 29, 3967, 56, 335, 99, 0, 0, 4759, 0, 110, 85, 646, 1, 3402, 0, 0, 0, 0, 948, 0, 2388, 3808, 0, 297] payload_rows_per_dst=[6236, 1590, 12574, 9266, 15022, 11058, 11728, 11872] dup_rows_per_dst=[1334, 18, 5880, 1358, 848, 1816, 4694, 3010] combine_rows_per_src=[6236, 1590, 12574, 9266, 15022, 11058, 11728, 11872] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=322568192 dispatch_meta_bytes=357872 combine_payload_bytes=322568192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=322926064 ingress_bytes=357171200 link_bytes=357171200 token_bytes=645494256 weight_movement_bytes=34603008 total_nvlink_bytes=680097264 weight_to_token_ratio=0.053607 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.344006471e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71802 payload_rows_remote=59050 dup_rows=26502 dup_rows_remote=22008 dedup_ratio=0.269592 router_tokens_per_expert=[0, 0, 155, 2, 1224, 877, 1229, 275, 4271, 230, 420, 5, 259, 2, 7001, 39, 2455, 8, 757, 354, 517, 255, 0, 97, 635, 1715, 559, 481, 226, 2688, 453, 114, 0, 4, 90, 4, 536, 56, 169, 7, 31, 4328, 71, 365, 132, 3, 0, 4861, 5, 114, 88, 653, 4, 3364, 0, 5, 5, 6, 1065, 3, 1997, 3625, 0, 258] payload_rows_per_dst=[12752, 15722, 7316, 11004, 1716, 8772, 8154, 6366] dup_rows_per_dst=[4494, 15982, 1570, 2738, 16, 1088, 312, 302] combine_rows_per_src=[12752, 15722, 7316, 11004, 1716, 8772, 8154, 6366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85894 payload_rows_remote=76382 dup_rows=12410 dup_rows_remote=10548 dedup_ratio=0.126241 router_tokens_per_expert=[0, 1915, 1185, 45, 138, 2504, 0, 2733, 0, 359, 392, 59, 69, 0, 0, 0, 0, 335, 0, 154, 0, 816, 5289, 8, 234, 0, 5718, 94, 171, 97, 150, 0, 57, 0, 0, 173, 1406, 0, 21, 1297, 0, 1370, 0, 6045, 59, 0, 439, 0, 0, 1200, 2492, 0, 0, 5756, 5253, 422, 1, 69, 0, 134, 493, 0, 0, 0] payload_rows_per_dst=[8422, 11322, 11782, 11934, 9512, 13896, 7266, 11760] dup_rows_per_dst=[3152, 942, 1422, 994, 1862, 1930, 962, 1146] combine_rows_per_src=[8422, 11322, 11782, 11934, 9512, 13896, 7266, 11760] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=276963328 dispatch_meta_bytes=327528 combine_payload_bytes=276963328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277290856 ingress_bytes=311566336 link_bytes=311566336 token_bytes=554254184 weight_movement_bytes=34603008 total_nvlink_bytes=588857192 weight_to_token_ratio=0.062432 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=26624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.172399039e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=287629312 dispatch_meta_bytes=346712 combine_payload_bytes=287629312 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=287976024 ingress_bytes=391438336 link_bytes=391438336 token_bytes=575605336 weight_movement_bytes=103809024 total_nvlink_bytes=679414360 weight_to_token_ratio=0.180348 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=21888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.472950945e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79958 payload_rows_remote=66084 dup_rows=18346 dup_rows_remote=9440 dedup_ratio=0.186625 router_tokens_per_expert=[5, 1956, 1288, 86, 124, 2625, 1, 2548, 0, 279, 611, 66, 91, 0, 3, 2, 0, 373, 7, 182, 1, 792, 5305, 4, 167, 7, 5805, 88, 202, 98, 143, 5, 106, 4, 0, 148, 1238, 0, 19, 1348, 2, 1425, 0, 5970, 66, 4, 396, 2, 1, 1261, 2085, 2, 5, 5751, 5168, 436, 0, 86, 0, 181, 583, 0, 1, 0] payload_rows_per_dst=[13874, 11418, 8644, 7950, 9250, 11806, 10264, 6752] dup_rows_per_dst=[8906, 1322, 928, 632, 1572, 1518, 2800, 668] combine_rows_per_src=[13874, 11418, 8644, 7950, 9250, 11806, 10264, 6752] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81104 payload_rows_remote=67098 dup_rows=17200 dup_rows_remote=10614 dedup_ratio=0.174967 router_tokens_per_expert=[0, 1859, 1268, 68, 124, 2816, 0, 2844, 0, 303, 453, 90, 126, 0, 0, 0, 0, 368, 0, 189, 0, 817, 5166, 12, 186, 0, 5685, 85, 188, 92, 164, 0, 5, 0, 0, 140, 1032, 0, 12, 1309, 0, 1408, 0, 6142, 90, 0, 401, 0, 0, 1313, 2414, 0, 0, 5727, 5054, 427, 0, 119, 0, 203, 453, 0, 0, 0] payload_rows_per_dst=[5234, 11062, 11536, 12012, 11882, 3612, 14006, 11760] dup_rows_per_dst=[1404, 990, 1568, 788, 4434, 186, 6586, 1244] combine_rows_per_src=[5234, 11062, 11536, 12012, 11882, 3612, 14006, 11760] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85952 payload_rows_remote=74062 dup_rows=12352 dup_rows_remote=11518 dedup_ratio=0.125651 router_tokens_per_expert=[0, 1943, 1238, 47, 154, 2537, 0, 2594, 0, 359, 397, 60, 74, 0, 0, 0, 0, 364, 0, 174, 0, 813, 5313, 14, 188, 0, 5737, 80, 148, 84, 125, 0, 55, 0, 0, 161, 1047, 0, 13, 1380, 0, 1542, 0, 6042, 60, 0, 420, 0, 0, 1277, 2603, 0, 0, 5697, 5233, 437, 0, 76, 0, 164, 502, 0, 0, 0] payload_rows_per_dst=[8540, 11310, 11806, 11890, 9092, 14052, 7602, 11660] dup_rows_per_dst=[3298, 936, 1550, 834, 1408, 2076, 1032, 1218] combine_rows_per_src=[8540, 11310, 11806, 11890, 9092, 14052, 7602, 11660] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84826 payload_rows_remote=72924 dup_rows=13478 dup_rows_remote=11712 dedup_ratio=0.137105 router_tokens_per_expert=[0, 1831, 1323, 95, 122, 2176, 0, 2558, 0, 436, 521, 70, 110, 0, 1, 0, 0, 349, 0, 183, 0, 818, 5494, 3, 206, 0, 5678, 70, 165, 72, 167, 0, 58, 0, 0, 101, 1177, 0, 16, 1264, 0, 1479, 0, 6040, 70, 0, 421, 0, 0, 1530, 2616, 0, 0, 5696, 5040, 484, 1, 96, 0, 172, 435, 0, 8, 0] payload_rows_per_dst=[7124, 11162, 12108, 11922, 9634, 11902, 9334, 11640] dup_rows_per_dst=[2890, 1194, 1586, 794, 1794, 1766, 2278, 1176] combine_rows_per_src=[7124, 11162, 12108, 11922, 9634, 11902, 9334, 11640] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85894 payload_rows_remote=76382 dup_rows=12410 dup_rows_remote=10548 dedup_ratio=0.126241 router_tokens_per_expert=[0, 1915, 1185, 45, 138, 2504, 0, 2733, 0, 359, 392, 59, 69, 0, 0, 0, 0, 335, 0, 154, 0, 816, 5289, 8, 234, 0, 5718, 94, 171, 97, 150, 0, 57, 0, 0, 173, 1406, 0, 21, 1297, 0, 1370, 0, 6045, 59, 0, 439, 0, 0, 1200, 2492, 0, 0, 5756, 5253, 422, 1, 69, 0, 134, 493, 0, 0, 0] payload_rows_per_dst=[8422, 11322, 11782, 11934, 9512, 13896, 7266, 11760] dup_rows_per_dst=[3152, 942, 1422, 994, 1862, 1930, 962, 1146] combine_rows_per_src=[8422, 11322, 11782, 11934, 9512, 13896, 7266, 11760] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83922 payload_rows_remote=72976 dup_rows=14382 dup_rows_remote=13308 dedup_ratio=0.146301 router_tokens_per_expert=[1, 1927, 1257, 87, 159, 2539, 1, 2904, 0, 365, 561, 69, 88, 0, 0, 0, 0, 353, 0, 153, 0, 918, 5209, 15, 207, 0, 5637, 83, 197, 94, 153, 0, 43, 0, 0, 170, 1231, 0, 18, 1206, 0, 1400, 0, 6104, 71, 0, 399, 1, 0, 1292, 2250, 0, 0, 5861, 4927, 427, 6, 88, 0, 185, 490, 0, 6, 0] payload_rows_per_dst=[11070, 10946, 7530, 11824, 9494, 14072, 6968, 12018] dup_rows_per_dst=[5732, 1074, 906, 918, 1650, 1878, 970, 1254] combine_rows_per_src=[11070, 10946, 7530, 11824, 9494, 14072, 6968, 12018] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=241868800 dispatch_meta_bytes=324232 combine_payload_bytes=241868800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=242193032 ingress_bytes=276471808 link_bytes=276471808 token_bytes=484061832 weight_movement_bytes=34603008 total_nvlink_bytes=518664840 weight_to_token_ratio=0.071485 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=18304 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.040341156e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=312860672 dispatch_meta_bytes=347720 combine_payload_bytes=312860672 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=313208392 ingress_bytes=382066688 link_bytes=382066688 token_bytes=626069064 weight_movement_bytes=69206016 total_nvlink_bytes=695275080 weight_to_token_ratio=0.110541 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=21632 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.437686188e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82110 payload_rows_remote=70318 dup_rows=16194 dup_rows_remote=15066 dedup_ratio=0.164734 router_tokens_per_expert=[0, 1902, 1214, 57, 135, 2746, 1, 2750, 0, 327, 473, 35, 66, 0, 0, 0, 0, 324, 0, 179, 1, 670, 5554, 14, 152, 0, 6031, 63, 174, 73, 116, 0, 19, 0, 0, 152, 1253, 0, 26, 1177, 0, 1269, 0, 6057, 35, 0, 455, 0, 0, 1139, 2355, 0, 1, 5796, 5228, 469, 0, 44, 0, 157, 462, 0, 1, 0] payload_rows_per_dst=[5054, 11218, 12112, 12430, 12160, 3446, 13898, 11792] dup_rows_per_dst=[1564, 1040, 1372, 788, 4086, 72, 6144, 1128] combine_rows_per_src=[5054, 11218, 12112, 12430, 12160, 3446, 13898, 11792] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86156 payload_rows_remote=78752 dup_rows=12148 dup_rows_remote=10716 dedup_ratio=0.123576 router_tokens_per_expert=[0, 0, 123, 0, 1309, 806, 1117, 260, 5028, 179, 267, 0, 251, 0, 7110, 16, 2553, 1, 756, 313, 568, 187, 0, 40, 295, 1465, 480, 410, 185, 3086, 452, 68, 0, 2, 72, 0, 417, 28, 136, 2, 27, 4602, 77, 378, 87, 0, 0, 4793, 0, 78, 76, 645, 0, 3400, 0, 0, 0, 0, 899, 0, 2030, 3840, 0, 238] payload_rows_per_dst=[12634, 11050, 7404, 11006, 15024, 9268, 8180, 11590] dup_rows_per_dst=[4182, 432, 1432, 1876, 510, 1074, 218, 2424] combine_rows_per_src=[12634, 11050, 7404, 11006, 15024, 9268, 8180, 11590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84826 payload_rows_remote=72924 dup_rows=13478 dup_rows_remote=11712 dedup_ratio=0.137105 router_tokens_per_expert=[0, 1831, 1323, 95, 122, 2176, 0, 2558, 0, 436, 521, 70, 110, 0, 1, 0, 0, 349, 0, 183, 0, 818, 5494, 3, 206, 0, 5678, 70, 165, 72, 167, 0, 58, 0, 0, 101, 1177, 0, 16, 1264, 0, 1479, 0, 6040, 70, 0, 421, 0, 0, 1530, 2616, 0, 0, 5696, 5040, 484, 1, 96, 0, 172, 435, 0, 8, 0] payload_rows_per_dst=[7124, 11162, 12108, 11922, 9634, 11902, 9334, 11640] dup_rows_per_dst=[2890, 1194, 1586, 794, 1794, 1766, 2278, 1176] combine_rows_per_src=[7124, 11162, 12108, 11922, 9634, 11902, 9334, 11640] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=274833408 dispatch_meta_bytes=310848 combine_payload_bytes=274833408 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=275144256 ingress_bytes=413245440 link_bytes=413245440 token_bytes=549977664 weight_movement_bytes=138412032 total_nvlink_bytes=688389696 weight_to_token_ratio=0.251668 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=18944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.555009322e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=270680064 dispatch_meta_bytes=302096 combine_payload_bytes=270680064 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=270982160 ingress_bytes=374489088 link_bytes=374489088 token_bytes=541662224 weight_movement_bytes=103809024 total_nvlink_bytes=645471248 weight_to_token_ratio=0.191649 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=15360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.409172289e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=303357952 dispatch_meta_bytes=342320 combine_payload_bytes=303357952 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=303700272 ingress_bytes=372563968 link_bytes=372563968 token_bytes=607058224 weight_movement_bytes=69206016 total_nvlink_bytes=676264240 weight_to_token_ratio=0.114002 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=42624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.401928218e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86434 payload_rows_remote=74546 dup_rows=11870 dup_rows_remote=10304 dedup_ratio=0.120748 router_tokens_per_expert=[0, 1909, 1274, 43, 118, 2676, 0, 2640, 0, 306, 417, 64, 86, 0, 0, 0, 0, 395, 0, 174, 0, 732, 5421, 5, 169, 0, 5666, 75, 181, 78, 128, 0, 34, 0, 0, 129, 1049, 0, 18, 1367, 0, 1220, 0, 6382, 64, 0, 380, 0, 0, 1475, 2186, 0, 0, 5916, 5311, 436, 0, 77, 0, 120, 431, 0, 0, 0] payload_rows_per_dst=[8666, 11404, 11888, 11758, 9114, 14298, 7268, 12038] dup_rows_per_dst=[3374, 964, 1566, 836, 1360, 1794, 926, 1050] combine_rows_per_src=[8666, 11404, 11888, 11758, 9114, 14298, 7268, 12038] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=298696704 dispatch_meta_bytes=338544 combine_payload_bytes=298696704 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=299035248 ingress_bytes=367902720 link_bytes=367902720 token_bytes=597731952 weight_movement_bytes=69206016 total_nvlink_bytes=666937968 weight_to_token_ratio=0.115781 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=34304 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.384388317e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=312860672 dispatch_meta_bytes=347720 combine_payload_bytes=312860672 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=313208392 ingress_bytes=382066688 link_bytes=382066688 token_bytes=626069064 weight_movement_bytes=69206016 total_nvlink_bytes=695275080 weight_to_token_ratio=0.110541 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=21632 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.437686188e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=298909696 dispatch_meta_bytes=345136 combine_payload_bytes=298909696 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=299254832 ingress_bytes=368115712 link_bytes=368115712 token_bytes=598164528 weight_movement_bytes=69206016 total_nvlink_bytes=667370544 weight_to_token_ratio=0.115697 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=41216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.385189789e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=288022528 dispatch_meta_bytes=341536 combine_payload_bytes=288022528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288364064 ingress_bytes=322625536 link_bytes=322625536 token_bytes=576386592 weight_movement_bytes=34603008 total_nvlink_bytes=610989600 weight_to_token_ratio=0.060034 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=43392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.214013918e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81104 payload_rows_remote=67098 dup_rows=17200 dup_rows_remote=10614 dedup_ratio=0.174967 router_tokens_per_expert=[0, 1859, 1268, 68, 124, 2816, 0, 2844, 0, 303, 453, 90, 126, 0, 0, 0, 0, 368, 0, 189, 0, 817, 5166, 12, 186, 0, 5685, 85, 188, 92, 164, 0, 5, 0, 0, 140, 1032, 0, 12, 1309, 0, 1408, 0, 6142, 90, 0, 401, 0, 0, 1313, 2414, 0, 0, 5727, 5054, 427, 0, 119, 0, 203, 453, 0, 0, 0] payload_rows_per_dst=[5234, 11062, 11536, 12012, 11882, 3612, 14006, 11760] dup_rows_per_dst=[1404, 990, 1568, 788, 4434, 186, 6586, 1244] combine_rows_per_src=[5234, 11062, 11536, 12012, 11882, 3612, 14006, 11760] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=305340416 dispatch_meta_bytes=339400 combine_payload_bytes=305340416 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=305679816 ingress_bytes=374546432 link_bytes=374546432 token_bytes=611020232 weight_movement_bytes=69206016 total_nvlink_bytes=680226248 weight_to_token_ratio=0.113263 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=35072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.409388070e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79958 payload_rows_remote=66084 dup_rows=18346 dup_rows_remote=9440 dedup_ratio=0.186625 router_tokens_per_expert=[5, 1956, 1288, 86, 124, 2625, 1, 2548, 0, 279, 611, 66, 91, 0, 3, 2, 0, 373, 7, 182, 1, 792, 5305, 4, 167, 7, 5805, 88, 202, 98, 143, 5, 106, 4, 0, 148, 1238, 0, 19, 1348, 2, 1425, 0, 5970, 66, 4, 396, 2, 1, 1261, 2085, 2, 5, 5751, 5168, 436, 0, 86, 0, 181, 583, 0, 1, 0] payload_rows_per_dst=[13874, 11418, 8644, 7950, 9250, 11806, 10264, 6752] dup_rows_per_dst=[8906, 1322, 928, 632, 1572, 1518, 2800, 668] combine_rows_per_src=[13874, 11418, 8644, 7950, 9250, 11806, 10264, 6752] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81712 payload_rows_remote=69802 dup_rows=16592 dup_rows_remote=12722 dedup_ratio=0.168783 router_tokens_per_expert=[186, 3488, 94, 69, 83, 2327, 1373, 92, 0, 0, 0, 501, 1837, 0, 0, 773, 326, 4077, 7070, 13, 0, 1419, 5381, 0, 0, 1840, 785, 0, 0, 0, 1182, 0, 3748, 9, 0, 0, 0, 0, 330, 315, 534, 0, 0, 96, 0, 0, 227, 1252, 81, 335, 936, 767, 0, 0, 0, 1083, 307, 0, 94, 0, 2, 414, 0, 5706] payload_rows_per_dst=[7526, 8792, 8380, 6984, 11910, 15216, 10988, 11916] dup_rows_per_dst=[922, 2176, 1152, 630, 3870, 3142, 3570, 1130] combine_rows_per_src=[7526, 8792, 8380, 6984, 11910, 15216, 10988, 11916] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=274833408 dispatch_meta_bytes=310848 combine_payload_bytes=274833408 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=275144256 ingress_bytes=413245440 link_bytes=413245440 token_bytes=549977664 weight_movement_bytes=138412032 total_nvlink_bytes=688389696 weight_to_token_ratio=0.251668 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=18944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.555009322e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=322568192 dispatch_meta_bytes=357872 combine_payload_bytes=322568192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=322926064 ingress_bytes=357171200 link_bytes=357171200 token_bytes=645494256 weight_movement_bytes=34603008 total_nvlink_bytes=680097264 weight_to_token_ratio=0.053607 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.344006471e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=298696704 dispatch_meta_bytes=338544 combine_payload_bytes=298696704 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=299035248 ingress_bytes=367902720 link_bytes=367902720 token_bytes=597731952 weight_movement_bytes=69206016 total_nvlink_bytes=666937968 weight_to_token_ratio=0.115781 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=34304 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.384388317e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82412 payload_rows_remote=71834 dup_rows=15892 dup_rows_remote=12288 dedup_ratio=0.161662 router_tokens_per_expert=[171, 3480, 85, 54, 42, 2580, 1406, 92, 0, 0, 0, 534, 1989, 0, 0, 763, 353, 3918, 6966, 43, 0, 1580, 5289, 0, 0, 1922, 924, 0, 0, 0, 1237, 0, 3654, 7, 0, 0, 0, 0, 265, 275, 567, 0, 0, 89, 0, 0, 179, 1208, 42, 334, 891, 763, 0, 0, 0, 1143, 302, 0, 85, 0, 2, 360, 0, 5558] payload_rows_per_dst=[7884, 5730, 12414, 7488, 11790, 14952, 10578, 11576] dup_rows_per_dst=[976, 842, 2116, 678, 3572, 3066, 3604, 1038] combine_rows_per_src=[7884, 5730, 12414, 7488, 11790, 14952, 10578, 11576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72940 payload_rows_remote=59874 dup_rows=25364 dup_rows_remote=17184 dedup_ratio=0.258016 router_tokens_per_expert=[194, 3312, 92, 69, 57, 2503, 1426, 105, 0, 5, 1, 542, 1614, 0, 1, 771, 341, 4111, 6778, 79, 2, 1537, 5224, 7, 5, 2072, 850, 6, 0, 1, 1309, 6, 3679, 284, 3, 0, 1, 7, 306, 202, 529, 2, 0, 127, 13, 2, 220, 1054, 13, 361, 935, 823, 0, 1, 8, 1112, 348, 1, 89, 0, 6, 404, 1, 5601] payload_rows_per_dst=[13066, 12838, 15498, 7650, 8512, 3380, 5510, 6486] dup_rows_per_dst=[8180, 3478, 10212, 848, 452, 514, 996, 684] combine_rows_per_src=[13066, 12838, 15498, 7650, 8512, 3380, 5510, 6486] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79712 payload_rows_remote=68780 dup_rows=18592 dup_rows_remote=15524 dedup_ratio=0.189128 router_tokens_per_expert=[172, 3532, 80, 46, 74, 2209, 1442, 92, 0, 0, 0, 537, 1932, 0, 0, 792, 346, 3804, 7089, 78, 0, 1589, 5341, 0, 0, 2065, 839, 0, 0, 0, 1231, 0, 3662, 4, 0, 0, 0, 0, 312, 379, 585, 0, 0, 87, 0, 0, 146, 1178, 4, 378, 930, 860, 0, 0, 0, 1087, 202, 0, 80, 0, 1, 334, 0, 5633] payload_rows_per_dst=[10660, 13084, 3870, 10932, 8672, 10370, 10422, 11702] dup_rows_per_dst=[3168, 4120, 156, 3068, 1508, 2070, 3704, 798] combine_rows_per_src=[10660, 13084, 3870, 10932, 8672, 10370, 10422, 11702] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82038 payload_rows_remote=66808 dup_rows=16266 dup_rows_remote=13078 dedup_ratio=0.165466 router_tokens_per_expert=[131, 3560, 70, 52, 61, 2482, 1163, 60, 0, 0, 0, 574, 1853, 0, 0, 803, 323, 4164, 7127, 32, 0, 1422, 5198, 0, 0, 1858, 781, 0, 0, 0, 1375, 0, 3789, 3, 0, 0, 0, 0, 253, 183, 579, 0, 0, 81, 2, 0, 200, 1220, 38, 400, 1070, 798, 0, 0, 0, 1105, 325, 1, 70, 0, 1, 336, 0, 5609] payload_rows_per_dst=[7204, 5520, 12070, 7338, 11838, 15230, 11186, 11652] dup_rows_per_dst=[834, 940, 1880, 690, 3738, 3188, 3964, 1032] combine_rows_per_src=[7204, 5520, 12070, 7338, 11838, 15230, 11186, 11652] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78922 payload_rows_remote=65926 dup_rows=19382 dup_rows_remote=15386 dedup_ratio=0.197164 router_tokens_per_expert=[170, 3515, 83, 72, 69, 2401, 1413, 99, 0, 0, 0, 527, 2013, 0, 1, 637, 335, 4000, 6979, 9, 0, 1452, 5318, 2, 1, 1748, 832, 1, 0, 0, 1287, 0, 3687, 153, 0, 0, 0, 1, 326, 260, 545, 0, 0, 110, 2, 0, 234, 1241, 75, 338, 874, 780, 0, 0, 0, 1166, 332, 0, 83, 0, 1, 424, 0, 5556] payload_rows_per_dst=[11664, 12996, 9636, 15560, 8274, 3696, 5466, 11630] dup_rows_per_dst=[3980, 3996, 2100, 5996, 580, 568, 1000, 1162] combine_rows_per_src=[11664, 12996, 9636, 15560, 8274, 3696, 5466, 11630] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81712 payload_rows_remote=69802 dup_rows=16592 dup_rows_remote=12722 dedup_ratio=0.168783 router_tokens_per_expert=[186, 3488, 94, 69, 83, 2327, 1373, 92, 0, 0, 0, 501, 1837, 0, 0, 773, 326, 4077, 7070, 13, 0, 1419, 5381, 0, 0, 1840, 785, 0, 0, 0, 1182, 0, 3748, 9, 0, 0, 0, 0, 330, 315, 534, 0, 0, 96, 0, 0, 227, 1252, 81, 335, 936, 767, 0, 0, 0, 1083, 307, 0, 94, 0, 2, 414, 0, 5706] payload_rows_per_dst=[7526, 8792, 8380, 6984, 11910, 15216, 10988, 11916] dup_rows_per_dst=[922, 2176, 1152, 630, 3870, 3142, 3570, 1130] combine_rows_per_src=[7526, 8792, 8380, 6984, 11910, 15216, 10988, 11916] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=270680064 dispatch_meta_bytes=302096 combine_payload_bytes=270680064 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=270982160 ingress_bytes=374489088 link_bytes=374489088 token_bytes=541662224 weight_movement_bytes=103809024 total_nvlink_bytes=645471248 weight_to_token_ratio=0.191649 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=15360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.409172289e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=285908992 dispatch_meta_bytes=330096 combine_payload_bytes=285908992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286239088 ingress_bytes=320512000 link_bytes=320512000 token_bytes=572148080 weight_movement_bytes=34603008 total_nvlink_bytes=606751088 weight_to_token_ratio=0.060479 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=29440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.206060853e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81140 payload_rows_remote=69136 dup_rows=17164 dup_rows_remote=14908 dedup_ratio=0.174601 router_tokens_per_expert=[109, 3753, 63, 57, 73, 1972, 1413, 46, 0, 0, 0, 587, 2008, 0, 0, 792, 312, 4138, 7190, 56, 0, 1395, 5307, 0, 0, 2074, 684, 0, 0, 0, 1441, 0, 3560, 0, 0, 0, 0, 0, 267, 185, 581, 0, 0, 66, 1, 0, 151, 1292, 10, 410, 1077, 816, 0, 0, 0, 1003, 300, 0, 63, 0, 0, 354, 1, 5545] payload_rows_per_dst=[6756, 5872, 10796, 7588, 11900, 15320, 10904, 12004] dup_rows_per_dst=[710, 902, 1610, 810, 3630, 3242, 4004, 2256] combine_rows_per_src=[6756, 5872, 10796, 7588, 11900, 15320, 10904, 12004] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78922 payload_rows_remote=65926 dup_rows=19382 dup_rows_remote=15386 dedup_ratio=0.197164 router_tokens_per_expert=[170, 3515, 83, 72, 69, 2401, 1413, 99, 0, 0, 0, 527, 2013, 0, 1, 637, 335, 4000, 6979, 9, 0, 1452, 5318, 2, 1, 1748, 832, 1, 0, 0, 1287, 0, 3687, 153, 0, 0, 0, 1, 326, 260, 545, 0, 0, 110, 2, 0, 234, 1241, 75, 338, 874, 780, 0, 0, 0, 1166, 332, 0, 83, 0, 1, 424, 0, 5556] payload_rows_per_dst=[11664, 12996, 9636, 15560, 8274, 3696, 5466, 11630] dup_rows_per_dst=[3980, 3996, 2100, 5996, 580, 568, 1000, 1162] combine_rows_per_src=[11664, 12996, 9636, 15560, 8274, 3696, 5466, 11630] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=294232064 dispatch_meta_bytes=336488 combine_payload_bytes=294232064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=294568552 ingress_bytes=328835072 link_bytes=328835072 token_bytes=588800616 weight_movement_bytes=34603008 total_nvlink_bytes=623403624 weight_to_token_ratio=0.058769 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=23040 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.237379903e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86434 payload_rows_remote=74546 dup_rows=11870 dup_rows_remote=10304 dedup_ratio=0.120748 router_tokens_per_expert=[0, 1909, 1274, 43, 118, 2676, 0, 2640, 0, 306, 417, 64, 86, 0, 0, 0, 0, 395, 0, 174, 0, 732, 5421, 5, 169, 0, 5666, 75, 181, 78, 128, 0, 34, 0, 0, 129, 1049, 0, 18, 1367, 0, 1220, 0, 6382, 64, 0, 380, 0, 0, 1475, 2186, 0, 0, 5916, 5311, 436, 0, 77, 0, 120, 431, 0, 0, 0] payload_rows_per_dst=[8666, 11404, 11888, 11758, 9114, 14298, 7268, 12038] dup_rows_per_dst=[3374, 964, 1566, 836, 1360, 1794, 926, 1050] combine_rows_per_src=[8666, 11404, 11888, 11758, 9114, 14298, 7268, 12038] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82038 payload_rows_remote=66808 dup_rows=16266 dup_rows_remote=13078 dedup_ratio=0.165466 router_tokens_per_expert=[131, 3560, 70, 52, 61, 2482, 1163, 60, 0, 0, 0, 574, 1853, 0, 0, 803, 323, 4164, 7127, 32, 0, 1422, 5198, 0, 0, 1858, 781, 0, 0, 0, 1375, 0, 3789, 3, 0, 0, 0, 0, 253, 183, 579, 0, 0, 81, 2, 0, 200, 1220, 38, 400, 1070, 798, 0, 0, 0, 1105, 325, 1, 70, 0, 1, 336, 0, 5609] payload_rows_per_dst=[7204, 5520, 12070, 7338, 11838, 15230, 11186, 11652] dup_rows_per_dst=[834, 940, 1880, 690, 3738, 3188, 3964, 1032] combine_rows_per_src=[7204, 5520, 12070, 7338, 11838, 15230, 11186, 11652] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=245243904 dispatch_meta_bytes=308232 combine_payload_bytes=245243904 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=245552136 ingress_bytes=314449920 link_bytes=314449920 token_bytes=490796040 weight_movement_bytes=69206016 total_nvlink_bytes=560002056 weight_to_token_ratio=0.141008 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=18816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.183249734e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=281722880 dispatch_meta_bytes=337216 combine_payload_bytes=281722880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282060096 ingress_bytes=316325888 link_bytes=316325888 token_bytes=563782976 weight_movement_bytes=34603008 total_nvlink_bytes=598385984 weight_to_token_ratio=0.061376 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=16896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.190308850e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78698 payload_rows_remote=73078 dup_rows=19606 dup_rows_remote=18812 dedup_ratio=0.199443 router_tokens_per_expert=[165, 3486, 76, 58, 62, 2255, 1195, 90, 0, 0, 0, 543, 1833, 0, 0, 789, 392, 4168, 7042, 49, 0, 1501, 5344, 0, 0, 1761, 860, 0, 0, 0, 1359, 0, 3558, 5, 0, 0, 0, 0, 275, 191, 533, 0, 0, 77, 0, 0, 164, 1401, 28, 376, 1012, 782, 0, 0, 0, 1103, 369, 0, 76, 0, 0, 333, 0, 5841] payload_rows_per_dst=[11232, 13028, 5620, 15790, 7630, 3842, 9484, 12072] dup_rows_per_dst=[3542, 3990, 794, 6254, 428, 508, 2924, 1166] combine_rows_per_src=[11232, 13028, 5620, 15790, 7630, 3842, 9484, 12072] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=273645568 dispatch_meta_bytes=319544 combine_payload_bytes=273645568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273965112 ingress_bytes=308248576 link_bytes=308248576 token_bytes=547610680 weight_movement_bytes=34603008 total_nvlink_bytes=582213688 weight_to_token_ratio=0.063189 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.159914575e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=285908992 dispatch_meta_bytes=330096 combine_payload_bytes=285908992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286239088 ingress_bytes=320512000 link_bytes=320512000 token_bytes=572148080 weight_movement_bytes=34603008 total_nvlink_bytes=606751088 weight_to_token_ratio=0.060479 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=29440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.206060853e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=270032896 dispatch_meta_bytes=325248 combine_payload_bytes=270032896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270358144 ingress_bytes=304635904 link_bytes=304635904 token_bytes=540391040 weight_movement_bytes=34603008 total_nvlink_bytes=574994048 weight_to_token_ratio=0.064033 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.146320382e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82412 payload_rows_remote=71834 dup_rows=15892 dup_rows_remote=12288 dedup_ratio=0.161662 router_tokens_per_expert=[171, 3480, 85, 54, 42, 2580, 1406, 92, 0, 0, 0, 534, 1989, 0, 0, 763, 353, 3918, 6966, 43, 0, 1580, 5289, 0, 0, 1922, 924, 0, 0, 0, 1237, 0, 3654, 7, 0, 0, 0, 0, 265, 275, 567, 0, 0, 89, 0, 0, 179, 1208, 42, 334, 891, 763, 0, 0, 0, 1143, 302, 0, 85, 0, 2, 360, 0, 5558] payload_rows_per_dst=[7884, 5730, 12414, 7488, 11790, 14952, 10578, 11576] dup_rows_per_dst=[976, 842, 2116, 678, 3572, 3066, 3604, 1038] combine_rows_per_src=[7884, 5730, 12414, 7488, 11790, 14952, 10578, 11576] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=283181056 dispatch_meta_bytes=336176 combine_payload_bytes=283181056 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=283517232 ingress_bytes=352387072 link_bytes=352387072 token_bytes=566698288 weight_movement_bytes=69206016 total_nvlink_bytes=635904304 weight_to_token_ratio=0.122121 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=42240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.326004182e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=299327488 dispatch_meta_bytes=367560 combine_payload_bytes=299327488 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=299695048 ingress_bytes=472342528 link_bytes=472342528 token_bytes=599022536 weight_movement_bytes=173015040 total_nvlink_bytes=772037576 weight_to_token_ratio=0.288829 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=18048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.777386906e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89380 payload_rows_remote=83578 dup_rows=8924 dup_rows_remote=8924 dedup_ratio=0.090780 router_tokens_per_expert=[2262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 1707, 6740, 0, 126, 0, 0, 0, 0, 0, 332, 419, 381, 0, 0, 7, 0, 0, 94, 0, 0, 0, 5674, 2901, 0, 0, 0, 0, 0, 748, 953, 0, 5, 0, 94, 2382, 2528, 0, 5, 0, 234, 92, 6346, 6615, 0, 376, 0, 0, 0, 0, 0, 8038, 0] payload_rows_per_dst=[12940, 5566, 12340, 13418, 5802, 10032, 13206, 16076] dup_rows_per_dst=[2882, 468, 1272, 14, 0, 3388, 148, 752] combine_rows_per_src=[12940, 5566, 12340, 13418, 5802, 10032, 13206, 16076] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=294232064 dispatch_meta_bytes=336488 combine_payload_bytes=294232064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=294568552 ingress_bytes=328835072 link_bytes=328835072 token_bytes=588800616 weight_movement_bytes=34603008 total_nvlink_bytes=623403624 weight_to_token_ratio=0.058769 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=23040 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.237379903e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72940 payload_rows_remote=59874 dup_rows=25364 dup_rows_remote=17184 dedup_ratio=0.258016 router_tokens_per_expert=[194, 3312, 92, 69, 57, 2503, 1426, 105, 0, 5, 1, 542, 1614, 0, 1, 771, 341, 4111, 6778, 79, 2, 1537, 5224, 7, 5, 2072, 850, 6, 0, 1, 1309, 6, 3679, 284, 3, 0, 1, 7, 306, 202, 529, 2, 0, 127, 13, 2, 220, 1054, 13, 361, 935, 823, 0, 1, 8, 1112, 348, 1, 89, 0, 6, 404, 1, 5601] payload_rows_per_dst=[13066, 12838, 15498, 7650, 8512, 3380, 5510, 6486] dup_rows_per_dst=[8180, 3478, 10212, 848, 452, 514, 996, 684] combine_rows_per_src=[13066, 12838, 15498, 7650, 8512, 3380, 5510, 6486] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=305340416 dispatch_meta_bytes=339400 combine_payload_bytes=305340416 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=305679816 ingress_bytes=374546432 link_bytes=374546432 token_bytes=611020232 weight_movement_bytes=69206016 total_nvlink_bytes=680226248 weight_to_token_ratio=0.113263 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=35072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.409388070e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=273645568 dispatch_meta_bytes=319544 combine_payload_bytes=273645568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273965112 ingress_bytes=308248576 link_bytes=308248576 token_bytes=547610680 weight_movement_bytes=34603008 total_nvlink_bytes=582213688 weight_to_token_ratio=0.063189 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.159914575e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88492 payload_rows_remote=74798 dup_rows=9812 dup_rows_remote=7494 dedup_ratio=0.099813 router_tokens_per_expert=[2064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 1928, 6787, 0, 64, 0, 0, 0, 0, 0, 337, 437, 321, 0, 0, 1, 0, 0, 85, 0, 0, 0, 5277, 3081, 0, 0, 0, 0, 0, 716, 975, 0, 8, 1, 85, 2370, 2746, 0, 21, 0, 299, 86, 6329, 6714, 0, 343, 0, 0, 0, 0, 0, 7988, 0] payload_rows_per_dst=[4128, 14812, 11456, 13598, 6162, 8666, 13694, 15976] dup_rows_per_dst=[0, 2924, 1288, 2, 0, 2594, 2318, 686] combine_rows_per_src=[4128, 14812, 11456, 13598, 6162, 8666, 13694, 15976] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82584 payload_rows_remote=68222 dup_rows=15720 dup_rows_remote=12984 dedup_ratio=0.159912 router_tokens_per_expert=[1965, 0, 2, 1, 5, 0, 3, 2, 3, 2, 139, 1, 1763, 6571, 0, 80, 0, 0, 5, 1, 4, 277, 428, 365, 1, 2, 2, 0, 0, 90, 0, 0, 1, 5717, 2981, 57, 0, 7, 3, 0, 760, 1051, 7, 41, 0, 88, 2472, 2878, 3, 61, 7, 268, 113, 6092, 6536, 8, 384, 0, 0, 2, 9, 4, 7890, 0] payload_rows_per_dst=[14362, 14516, 12280, 13258, 15976, 10584, 816, 792] dup_rows_per_dst=[2736, 1644, 1314, 4, 5902, 4010, 104, 6] combine_rows_per_src=[14362, 14516, 12280, 13258, 15976, 10584, 816, 792] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89276 payload_rows_remote=76142 dup_rows=9028 dup_rows_remote=9028 dedup_ratio=0.091838 router_tokens_per_expert=[2137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 2022, 6671, 0, 82, 0, 0, 0, 0, 0, 277, 426, 345, 0, 0, 0, 0, 0, 80, 0, 0, 0, 5785, 3051, 4, 0, 0, 0, 0, 757, 929, 0, 37, 0, 80, 2324, 2693, 0, 10, 0, 225, 92, 6066, 6487, 0, 394, 0, 0, 0, 0, 0, 8079, 0] payload_rows_per_dst=[14530, 4406, 12356, 13134, 6102, 9966, 12624, 16158] dup_rows_per_dst=[3086, 0, 1310, 0, 8, 3674, 162, 788] combine_rows_per_src=[14530, 4406, 12356, 13134, 6102, 9966, 12624, 16158] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=342335488 dispatch_meta_bytes=370008 combine_payload_bytes=342335488 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=342705496 ingress_bytes=411541504 link_bytes=411541504 token_bytes=685040984 weight_movement_bytes=69206016 total_nvlink_bytes=754247000 weight_to_token_ratio=0.101025 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=24832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.548597548e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=306372608 dispatch_meta_bytes=329168 combine_payload_bytes=306372608 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=306701776 ingress_bytes=410181632 link_bytes=410181632 token_bytes=613074384 weight_movement_bytes=103809024 total_nvlink_bytes=716883408 weight_to_token_ratio=0.169325 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=38016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.543480459e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89380 payload_rows_remote=83578 dup_rows=8924 dup_rows_remote=8924 dedup_ratio=0.090780 router_tokens_per_expert=[2262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 1707, 6740, 0, 126, 0, 0, 0, 0, 0, 332, 419, 381, 0, 0, 7, 0, 0, 94, 0, 0, 0, 5674, 2901, 0, 0, 0, 0, 0, 748, 953, 0, 5, 0, 94, 2382, 2528, 0, 5, 0, 234, 92, 6346, 6615, 0, 376, 0, 0, 0, 0, 0, 8038, 0] payload_rows_per_dst=[12940, 5566, 12340, 13418, 5802, 10032, 13206, 16076] dup_rows_per_dst=[2882, 468, 1272, 14, 0, 3388, 148, 752] combine_rows_per_src=[12940, 5566, 12340, 13418, 5802, 10032, 13206, 16076] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74866 payload_rows_remote=58618 dup_rows=23438 dup_rows_remote=18444 dedup_ratio=0.238424 router_tokens_per_expert=[2125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 1805, 6751, 0, 88, 0, 0, 0, 0, 0, 393, 374, 279, 0, 0, 0, 0, 0, 63, 0, 0, 0, 5781, 3079, 4, 0, 0, 0, 0, 779, 1096, 0, 0, 0, 63, 2393, 2748, 0, 18, 0, 177, 46, 6147, 6503, 0, 266, 0, 0, 0, 0, 0, 8122, 0] payload_rows_per_dst=[4250, 14558, 4530, 4542, 11896, 3962, 14880, 16248] dup_rows_per_dst=[0, 2834, 382, 0, 3012, 234, 11982, 4994] combine_rows_per_src=[4250, 14558, 4530, 4542, 11896, 3962, 14880, 16248] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90042 payload_rows_remote=79994 dup_rows=8262 dup_rows_remote=4494 dedup_ratio=0.084045 router_tokens_per_expert=[2277, 0, 0, 0, 0, 0, 0, 0, 0, 3, 67, 0, 1585, 6709, 0, 42, 0, 0, 0, 0, 0, 273, 392, 319, 0, 0, 0, 0, 0, 70, 0, 0, 0, 5563, 3436, 4, 0, 0, 0, 0, 805, 984, 0, 0, 0, 70, 2388, 2661, 0, 1, 0, 292, 60, 6235, 6601, 0, 358, 0, 0, 0, 0, 0, 7957, 0] payload_rows_per_dst=[4554, 14468, 11888, 13342, 6872, 10048, 12956, 15914] dup_rows_per_dst=[0, 2344, 1206, 0, 8, 3768, 220, 716] combine_rows_per_src=[4554, 14468, 11888, 13342, 6872, 10048, 12956, 15914] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=245243904 dispatch_meta_bytes=308232 combine_payload_bytes=245243904 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=245552136 ingress_bytes=314449920 link_bytes=314449920 token_bytes=490796040 weight_movement_bytes=69206016 total_nvlink_bytes=560002056 weight_to_token_ratio=0.141008 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=18816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.183249734e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78698 payload_rows_remote=73078 dup_rows=19606 dup_rows_remote=18812 dedup_ratio=0.199443 router_tokens_per_expert=[165, 3486, 76, 58, 62, 2255, 1195, 90, 0, 0, 0, 543, 1833, 0, 0, 789, 392, 4168, 7042, 49, 0, 1501, 5344, 0, 0, 1761, 860, 0, 0, 0, 1359, 0, 3558, 5, 0, 0, 0, 0, 275, 191, 533, 0, 0, 77, 0, 0, 164, 1401, 28, 376, 1012, 782, 0, 0, 0, 1103, 369, 0, 76, 0, 0, 333, 0, 5841] payload_rows_per_dst=[11232, 13028, 5620, 15790, 7630, 3842, 9484, 12072] dup_rows_per_dst=[3542, 3990, 794, 6254, 428, 508, 2924, 1166] combine_rows_per_src=[11232, 13028, 5620, 15790, 7630, 3842, 9484, 12072] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82470 payload_rows_remote=68722 dup_rows=15834 dup_rows_remote=13714 dedup_ratio=0.161072 router_tokens_per_expert=[1949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 0, 1903, 6703, 0, 110, 0, 0, 1, 0, 0, 291, 417, 357, 0, 0, 7, 1, 0, 84, 0, 0, 0, 5311, 3047, 0, 0, 0, 0, 1, 691, 918, 0, 41, 0, 83, 2559, 2643, 0, 14, 0, 265, 89, 6541, 6699, 1, 376, 0, 0, 0, 0, 1, 7946, 0] payload_rows_per_dst=[14538, 13748, 11464, 13566, 15996, 10246, 2158, 754] dup_rows_per_dst=[2766, 2120, 1290, 16, 5992, 3624, 26, 0] combine_rows_per_src=[14538, 13748, 11464, 13566, 15996, 10246, 2158, 754] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74866 payload_rows_remote=58618 dup_rows=23438 dup_rows_remote=18444 dedup_ratio=0.238424 router_tokens_per_expert=[2125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 1805, 6751, 0, 88, 0, 0, 0, 0, 0, 393, 374, 279, 0, 0, 0, 0, 0, 63, 0, 0, 0, 5781, 3079, 4, 0, 0, 0, 0, 779, 1096, 0, 0, 0, 63, 2393, 2748, 0, 18, 0, 177, 46, 6147, 6503, 0, 266, 0, 0, 0, 0, 0, 8122, 0] payload_rows_per_dst=[4250, 14558, 4530, 4542, 11896, 3962, 14880, 16248] dup_rows_per_dst=[0, 2834, 382, 0, 3012, 234, 11982, 4994] combine_rows_per_src=[4250, 14558, 4530, 4542, 11896, 3962, 14880, 16248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90042 payload_rows_remote=79994 dup_rows=8262 dup_rows_remote=4494 dedup_ratio=0.084045 router_tokens_per_expert=[2277, 0, 0, 0, 0, 0, 0, 0, 0, 3, 67, 0, 1585, 6709, 0, 42, 0, 0, 0, 0, 0, 273, 392, 319, 0, 0, 0, 0, 0, 70, 0, 0, 0, 5563, 3436, 4, 0, 0, 0, 0, 805, 984, 0, 0, 0, 70, 2388, 2661, 0, 1, 0, 292, 60, 6235, 6601, 0, 358, 0, 0, 0, 0, 0, 7957, 0] payload_rows_per_dst=[4554, 14468, 11888, 13342, 6872, 10048, 12956, 15914] dup_rows_per_dst=[0, 2344, 1206, 0, 8, 3768, 220, 716] combine_rows_per_src=[4554, 14468, 11888, 13342, 6872, 10048, 12956, 15914] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279437312 dispatch_meta_bytes=324824 combine_payload_bytes=279437312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279762136 ingress_bytes=314040320 link_bytes=314040320 token_bytes=559199448 weight_movement_bytes=34603008 total_nvlink_bytes=593802456 weight_to_token_ratio=0.061880 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=32384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.181708442e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=311877632 dispatch_meta_bytes=340680 combine_payload_bytes=311877632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=312218312 ingress_bytes=346480640 link_bytes=346480640 token_bytes=624095944 weight_movement_bytes=34603008 total_nvlink_bytes=658698952 weight_to_token_ratio=0.055445 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=48512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.303778754e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86900 payload_rows_remote=74342 dup_rows=11404 dup_rows_remote=10034 dedup_ratio=0.116007 router_tokens_per_expert=[2084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 1787, 6682, 0, 68, 0, 0, 0, 0, 0, 387, 469, 297, 0, 0, 2, 0, 0, 76, 0, 0, 0, 5811, 3170, 2, 0, 0, 0, 0, 759, 1108, 0, 6, 0, 76, 2337, 2468, 0, 14, 0, 232, 89, 6170, 6627, 0, 249, 0, 0, 0, 0, 0, 8089, 0] payload_rows_per_dst=[14518, 3894, 12558, 13406, 11504, 10034, 12824, 8162] dup_rows_per_dst=[3014, 2, 1370, 4, 3072, 3474, 186, 282] combine_rows_per_src=[14518, 3894, 12558, 13406, 11504, 10034, 12824, 8162] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88492 payload_rows_remote=74798 dup_rows=9812 dup_rows_remote=7494 dedup_ratio=0.099813 router_tokens_per_expert=[2064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 1928, 6787, 0, 64, 0, 0, 0, 0, 0, 337, 437, 321, 0, 0, 1, 0, 0, 85, 0, 0, 0, 5277, 3081, 0, 0, 0, 0, 0, 716, 975, 0, 8, 1, 85, 2370, 2746, 0, 21, 0, 299, 86, 6329, 6714, 0, 343, 0, 0, 0, 0, 0, 7988, 0] payload_rows_per_dst=[4128, 14812, 11456, 13598, 6162, 8666, 13694, 15976] dup_rows_per_dst=[0, 2924, 1288, 2, 0, 2594, 2318, 686] combine_rows_per_src=[4128, 14812, 11456, 13598, 6162, 8666, 13694, 15976] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=327655424 dispatch_meta_bytes=337952 combine_payload_bytes=327655424 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=327993376 ingress_bytes=396861440 link_bytes=396861440 token_bytes=655648800 weight_movement_bytes=69206016 total_nvlink_bytes=724854816 weight_to_token_ratio=0.105553 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=17408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.493357649e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=281485312 dispatch_meta_bytes=329744 combine_payload_bytes=281485312 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=281815056 ingress_bytes=350691328 link_bytes=350691328 token_bytes=563300368 weight_movement_bytes=69206016 total_nvlink_bytes=632506384 weight_to_token_ratio=0.122858 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=21376 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.319623234e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=342335488 dispatch_meta_bytes=370008 combine_payload_bytes=342335488 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=342705496 ingress_bytes=411541504 link_bytes=411541504 token_bytes=685040984 weight_movement_bytes=69206016 total_nvlink_bytes=754247000 weight_to_token_ratio=0.101025 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=24832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.548597548e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=240099328 dispatch_meta_bytes=308248 combine_payload_bytes=240099328 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=240407576 ingress_bytes=309305344 link_bytes=309305344 token_bytes=480506904 weight_movement_bytes=69206016 total_nvlink_bytes=549712920 weight_to_token_ratio=0.144027 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=44160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.163891108e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=304504832 dispatch_meta_bytes=337504 combine_payload_bytes=304504832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=304842336 ingress_bytes=339107840 link_bytes=339107840 token_bytes=609347168 weight_movement_bytes=34603008 total_nvlink_bytes=643950176 weight_to_token_ratio=0.056787 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=40576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.276035501e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89772 payload_rows_remote=78576 dup_rows=8532 dup_rows_remote=7556 dedup_ratio=0.086792 router_tokens_per_expert=[251, 407, 0, 24, 251, 0, 0, 0, 0, 3902, 98, 5087, 0, 0, 2012, 4479, 69, 357, 0, 0, 1020, 0, 0, 0, 257, 248, 0, 6554, 45, 0, 0, 0, 0, 0, 0, 0, 4509, 1333, 244, 0, 3, 0, 0, 0, 0, 92, 0, 0, 4210, 0, 0, 6938, 94, 2611, 925, 0, 0, 0, 0, 0, 0, 312, 2734, 86] payload_rows_per_dst=[13928, 10592, 10146, 13918, 11196, 10364, 6998, 12630] dup_rows_per_dst=[1814, 1432, 1166, 290, 976, 0, 262, 2592] combine_rows_per_src=[13928, 10592, 10146, 13918, 11196, 10364, 6998, 12630] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=306372608 dispatch_meta_bytes=329168 combine_payload_bytes=306372608 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=306701776 ingress_bytes=410181632 link_bytes=410181632 token_bytes=613074384 weight_movement_bytes=103809024 total_nvlink_bytes=716883408 weight_to_token_ratio=0.169325 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=38016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.543480459e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82584 payload_rows_remote=68222 dup_rows=15720 dup_rows_remote=12984 dedup_ratio=0.159912 router_tokens_per_expert=[1965, 0, 2, 1, 5, 0, 3, 2, 3, 2, 139, 1, 1763, 6571, 0, 80, 0, 0, 5, 1, 4, 277, 428, 365, 1, 2, 2, 0, 0, 90, 0, 0, 1, 5717, 2981, 57, 0, 7, 3, 0, 760, 1051, 7, 41, 0, 88, 2472, 2878, 3, 61, 7, 268, 113, 6092, 6536, 8, 384, 0, 0, 2, 9, 4, 7890, 0] payload_rows_per_dst=[14362, 14516, 12280, 13258, 15976, 10584, 816, 792] dup_rows_per_dst=[2736, 1644, 1314, 4, 5902, 4010, 104, 6] combine_rows_per_src=[14362, 14516, 12280, 13258, 15976, 10584, 816, 792] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=299327488 dispatch_meta_bytes=367560 combine_payload_bytes=299327488 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=299695048 ingress_bytes=472342528 link_bytes=472342528 token_bytes=599022536 weight_movement_bytes=173015040 total_nvlink_bytes=772037576 weight_to_token_ratio=0.288829 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=18048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.777386906e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=281485312 dispatch_meta_bytes=329744 combine_payload_bytes=281485312 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=281815056 ingress_bytes=350691328 link_bytes=350691328 token_bytes=563300368 weight_movement_bytes=69206016 total_nvlink_bytes=632506384 weight_to_token_ratio=0.122858 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=21376 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.319623234e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=327655424 dispatch_meta_bytes=337952 combine_payload_bytes=327655424 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=327993376 ingress_bytes=396861440 link_bytes=396861440 token_bytes=655648800 weight_movement_bytes=69206016 total_nvlink_bytes=724854816 weight_to_token_ratio=0.105553 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=17408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.493357649e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78446 payload_rows_remote=63522 dup_rows=19858 dup_rows_remote=10176 dedup_ratio=0.202006 router_tokens_per_expert=[260, 434, 0, 90, 260, 0, 0, 0, 0, 3771, 138, 5034, 0, 0, 2006, 4248, 126, 357, 0, 0, 1150, 0, 0, 0, 265, 270, 0, 6559, 36, 0, 0, 0, 0, 0, 0, 0, 4720, 1279, 136, 0, 0, 0, 0, 0, 0, 89, 0, 0, 3952, 0, 0, 6808, 91, 2718, 963, 0, 0, 0, 0, 0, 0, 389, 2917, 86] payload_rows_per_dst=[2236, 4108, 10032, 9840, 11580, 13088, 14924, 12638] dup_rows_per_dst=[604, 180, 1138, 222, 690, 4700, 9682, 2642] combine_rows_per_src=[2236, 4108, 10032, 9840, 11580, 13088, 14924, 12638] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 2, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=260186112 dispatch_meta_bytes=294792 combine_payload_bytes=260186112 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=260480904 ingress_bytes=398598144 link_bytes=398598144 token_bytes=520667016 weight_movement_bytes=138412032 total_nvlink_bytes=659079048 weight_to_token_ratio=0.265836 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=18816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.499892726e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83676 payload_rows_remote=69956 dup_rows=14628 dup_rows_remote=12724 dedup_ratio=0.148804 router_tokens_per_expert=[250, 460, 12, 85, 248, 0, 0, 0, 1, 3641, 113, 5133, 4, 1, 1895, 4428, 91, 404, 9, 3, 1064, 0, 0, 11, 269, 300, 0, 6454, 11, 3, 0, 0, 2, 2, 6, 7, 4504, 1364, 231, 0, 12, 0, 1, 4, 0, 104, 0, 9, 4454, 12, 0, 6757, 112, 2449, 1009, 0, 0, 3, 6, 0, 1, 347, 2762, 104] payload_rows_per_dst=[13720, 14168, 12976, 14318, 9440, 10514, 2262, 6278] dup_rows_per_dst=[1904, 5998, 3952, 1876, 672, 12, 46, 168] combine_rows_per_src=[13720, 14168, 12976, 14318, 9440, 10514, 2262, 6278] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90096 payload_rows_remote=76058 dup_rows=8208 dup_rows_remote=7952 dedup_ratio=0.083496 router_tokens_per_expert=[245, 448, 0, 60, 245, 0, 0, 0, 0, 3668, 122, 4995, 0, 0, 1824, 4284, 74, 397, 0, 0, 1063, 0, 0, 0, 253, 252, 0, 6628, 14, 0, 0, 0, 0, 0, 0, 0, 4797, 1400, 171, 0, 1, 0, 0, 0, 0, 92, 0, 0, 4420, 0, 0, 6815, 82, 2650, 1005, 0, 0, 0, 0, 0, 0, 314, 2741, 92] payload_rows_per_dst=[13764, 10060, 10646, 14038, 11804, 10176, 7212, 12396] dup_rows_per_dst=[1862, 1168, 1262, 256, 932, 0, 262, 2466] combine_rows_per_src=[13764, 10060, 10646, 14038, 11804, 10176, 7212, 12396] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=321847296 dispatch_meta_bytes=344528 combine_payload_bytes=321847296 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=322191824 ingress_bytes=391053312 link_bytes=391053312 token_bytes=644039120 weight_movement_bytes=69206016 total_nvlink_bytes=713245136 weight_to_token_ratio=0.107456 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=36352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.471502131e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279437312 dispatch_meta_bytes=324824 combine_payload_bytes=279437312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279762136 ingress_bytes=314040320 link_bytes=314040320 token_bytes=559199448 weight_movement_bytes=34603008 total_nvlink_bytes=593802456 weight_to_token_ratio=0.061880 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=32384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.181708442e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89772 payload_rows_remote=78576 dup_rows=8532 dup_rows_remote=7556 dedup_ratio=0.086792 router_tokens_per_expert=[251, 407, 0, 24, 251, 0, 0, 0, 0, 3902, 98, 5087, 0, 0, 2012, 4479, 69, 357, 0, 0, 1020, 0, 0, 0, 257, 248, 0, 6554, 45, 0, 0, 0, 0, 0, 0, 0, 4509, 1333, 244, 0, 3, 0, 0, 0, 0, 92, 0, 0, 4210, 0, 0, 6938, 94, 2611, 925, 0, 0, 0, 0, 0, 0, 312, 2734, 86] payload_rows_per_dst=[13928, 10592, 10146, 13918, 11196, 10364, 6998, 12630] dup_rows_per_dst=[1814, 1432, 1166, 290, 976, 0, 262, 2592] combine_rows_per_src=[13928, 10592, 10146, 13918, 11196, 10364, 6998, 12630] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69676 payload_rows_remote=57480 dup_rows=28628 dup_rows_remote=26186 dedup_ratio=0.291219 router_tokens_per_expert=[264, 432, 0, 47, 264, 0, 0, 0, 0, 3843, 67, 5183, 0, 0, 1819, 4157, 65, 451, 0, 0, 1020, 0, 0, 0, 269, 199, 0, 6647, 17, 0, 0, 0, 0, 0, 0, 0, 4690, 1403, 289, 1, 4, 0, 0, 0, 0, 47, 0, 0, 4426, 0, 0, 6809, 63, 2604, 910, 0, 0, 0, 0, 0, 0, 274, 2843, 45] payload_rows_per_dst=[1486, 3702, 10548, 936, 11650, 13364, 15794, 12196] dup_rows_per_dst=[528, 70, 1376, 34, 1116, 4790, 18272, 2442] combine_rows_per_src=[1486, 3702, 10548, 936, 11650, 13364, 15794, 12196] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89490 payload_rows_remote=79046 dup_rows=8814 dup_rows_remote=8558 dedup_ratio=0.089661 router_tokens_per_expert=[274, 433, 0, 89, 274, 0, 0, 0, 0, 3584, 115, 5096, 0, 0, 1811, 4657, 110, 434, 0, 0, 1018, 0, 0, 4, 276, 282, 0, 6551, 11, 0, 0, 0, 0, 0, 1, 0, 4684, 1266, 266, 0, 0, 0, 0, 0, 0, 64, 0, 0, 4428, 0, 0, 6641, 87, 2773, 961, 0, 0, 0, 0, 0, 0, 299, 2603, 60] payload_rows_per_dst=[13490, 9252, 10610, 14008, 11644, 10444, 7350, 12692] dup_rows_per_dst=[1932, 956, 1378, 232, 1222, 256, 292, 2546] combine_rows_per_src=[13490, 9252, 10610, 14008, 11644, 10444, 7350, 12692] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86900 payload_rows_remote=74342 dup_rows=11404 dup_rows_remote=10034 dedup_ratio=0.116007 router_tokens_per_expert=[2084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 1787, 6682, 0, 68, 0, 0, 0, 0, 0, 387, 469, 297, 0, 0, 2, 0, 0, 76, 0, 0, 0, 5811, 3170, 2, 0, 0, 0, 0, 759, 1108, 0, 6, 0, 76, 2337, 2468, 0, 14, 0, 232, 89, 6170, 6627, 0, 249, 0, 0, 0, 0, 0, 8089, 0] payload_rows_per_dst=[14518, 3894, 12558, 13406, 11504, 10034, 12824, 8162] dup_rows_per_dst=[3014, 2, 1370, 4, 3072, 3474, 186, 282] combine_rows_per_src=[14518, 3894, 12558, 13406, 11504, 10034, 12824, 8162] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86986 payload_rows_remote=72672 dup_rows=11318 dup_rows_remote=4776 dedup_ratio=0.115133 router_tokens_per_expert=[252, 433, 1, 38, 252, 0, 0, 0, 0, 3861, 121, 5031, 0, 0, 2015, 4431, 87, 375, 1, 0, 1013, 0, 0, 0, 258, 260, 0, 6371, 48, 0, 0, 0, 0, 0, 0, 1, 4627, 1368, 260, 0, 7, 0, 1, 0, 0, 89, 0, 1, 4323, 1, 0, 6758, 83, 2745, 934, 0, 0, 0, 0, 0, 0, 273, 2744, 89] payload_rows_per_dst=[13620, 14314, 10406, 13576, 11452, 10254, 7240, 6124] dup_rows_per_dst=[1848, 6542, 1192, 298, 1060, 4, 286, 88] combine_rows_per_src=[13620, 14314, 10406, 13576, 11452, 10254, 7240, 6124] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89490 payload_rows_remote=79046 dup_rows=8814 dup_rows_remote=8558 dedup_ratio=0.089661 router_tokens_per_expert=[274, 433, 0, 89, 274, 0, 0, 0, 0, 3584, 115, 5096, 0, 0, 1811, 4657, 110, 434, 0, 0, 1018, 0, 0, 4, 276, 282, 0, 6551, 11, 0, 0, 0, 0, 0, 1, 0, 4684, 1266, 266, 0, 0, 0, 0, 0, 0, 64, 0, 0, 4428, 0, 0, 6641, 87, 2773, 961, 0, 0, 0, 0, 0, 0, 299, 2603, 60] payload_rows_per_dst=[13490, 9252, 10610, 14008, 11644, 10444, 7350, 12692] dup_rows_per_dst=[1932, 956, 1378, 232, 1222, 256, 292, 2546] combine_rows_per_src=[13490, 9252, 10610, 14008, 11644, 10444, 7350, 12692] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=286539776 dispatch_meta_bytes=330720 combine_payload_bytes=286539776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286870496 ingress_bytes=321142784 link_bytes=321142784 token_bytes=573410272 weight_movement_bytes=34603008 total_nvlink_bytes=608013280 weight_to_token_ratio=0.060346 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.035000 gemm_active_groups=7 gemm_peak_group_rows=41088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.208434442e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=311533568 dispatch_meta_bytes=336040 combine_payload_bytes=311533568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=311869608 ingress_bytes=380739584 link_bytes=380739584 token_bytes=623403176 weight_movement_bytes=69206016 total_nvlink_bytes=692609192 weight_to_token_ratio=0.111013 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=43776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.432692403e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69676 payload_rows_remote=57480 dup_rows=28628 dup_rows_remote=26186 dedup_ratio=0.291219 router_tokens_per_expert=[264, 432, 0, 47, 264, 0, 0, 0, 0, 3843, 67, 5183, 0, 0, 1819, 4157, 65, 451, 0, 0, 1020, 0, 0, 0, 269, 199, 0, 6647, 17, 0, 0, 0, 0, 0, 0, 0, 4690, 1403, 289, 1, 4, 0, 0, 0, 0, 47, 0, 0, 4426, 0, 0, 6809, 63, 2604, 910, 0, 0, 0, 0, 0, 0, 274, 2843, 45] payload_rows_per_dst=[1486, 3702, 10548, 936, 11650, 13364, 15794, 12196] dup_rows_per_dst=[528, 70, 1376, 34, 1116, 4790, 18272, 2442] combine_rows_per_src=[1486, 3702, 10548, 936, 11650, 13364, 15794, 12196] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89052 payload_rows_remote=78840 dup_rows=9252 dup_rows_remote=7988 dedup_ratio=0.094116 router_tokens_per_expert=[244, 447, 0, 77, 245, 0, 0, 0, 0, 3692, 117, 5078, 0, 0, 1951, 4171, 86, 409, 0, 0, 1081, 0, 0, 0, 249, 238, 0, 6735, 9, 0, 0, 0, 0, 0, 0, 0, 4785, 1361, 289, 0, 1, 0, 0, 0, 0, 90, 0, 0, 4162, 0, 0, 6771, 83, 2624, 1004, 0, 0, 0, 0, 0, 0, 269, 2795, 89] payload_rows_per_dst=[13702, 12340, 10212, 14258, 11756, 10338, 7196, 9250] dup_rows_per_dst=[1866, 3422, 1264, 204, 1114, 0, 226, 1156] combine_rows_per_src=[13702, 12340, 10212, 14258, 11756, 10338, 7196, 9250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 2, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78446 payload_rows_remote=63522 dup_rows=19858 dup_rows_remote=10176 dedup_ratio=0.202006 router_tokens_per_expert=[260, 434, 0, 90, 260, 0, 0, 0, 0, 3771, 138, 5034, 0, 0, 2006, 4248, 126, 357, 0, 0, 1150, 0, 0, 0, 265, 270, 0, 6559, 36, 0, 0, 0, 0, 0, 0, 0, 4720, 1279, 136, 0, 0, 0, 0, 0, 0, 89, 0, 0, 3952, 0, 0, 6808, 91, 2718, 963, 0, 0, 0, 0, 0, 0, 389, 2917, 86] payload_rows_per_dst=[2236, 4108, 10032, 9840, 11580, 13088, 14924, 12638] dup_rows_per_dst=[604, 180, 1138, 222, 690, 4700, 9682, 2642] combine_rows_per_src=[2236, 4108, 10032, 9840, 11580, 13088, 14924, 12638] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=323772416 dispatch_meta_bytes=350416 combine_payload_bytes=323772416 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=324122832 ingress_bytes=392978432 link_bytes=392978432 token_bytes=647895248 weight_movement_bytes=69206016 total_nvlink_bytes=717101264 weight_to_token_ratio=0.106817 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=40704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.478746202e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=297664512 dispatch_meta_bytes=309792 combine_payload_bytes=297664512 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=297974304 ingress_bytes=436076544 link_bytes=436076544 token_bytes=595638816 weight_movement_bytes=138412032 total_nvlink_bytes=734050848 weight_to_token_ratio=0.232376 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=22016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.640920928e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 2, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=235438080 dispatch_meta_bytes=334664 combine_payload_bytes=235438080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=235772744 ingress_bytes=270041088 link_bytes=270041088 token_bytes=471210824 weight_movement_bytes=34603008 total_nvlink_bytes=505813832 weight_to_token_ratio=0.073434 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=23936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.016142874e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=322928640 dispatch_meta_bytes=347312 combine_payload_bytes=322928640 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=323275952 ingress_bytes=392134656 link_bytes=392134656 token_bytes=646204592 weight_movement_bytes=69206016 total_nvlink_bytes=715410608 weight_to_token_ratio=0.107096 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=34432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.475571141e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=260186112 dispatch_meta_bytes=294792 combine_payload_bytes=260186112 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=260480904 ingress_bytes=398598144 link_bytes=398598144 token_bytes=520667016 weight_movement_bytes=138412032 total_nvlink_bytes=659079048 weight_to_token_ratio=0.265836 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=18816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.499892726e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88330 payload_rows_remote=81290 dup_rows=9974 dup_rows_remote=9968 dedup_ratio=0.101461 router_tokens_per_expert=[6901, 94, 0, 185, 4678, 0, 389, 196, 0, 94, 0, 0, 0, 6, 0, 7770, 0, 563, 2667, 0, 0, 0, 286, 0, 0, 686, 0, 0, 0, 153, 0, 0, 3163, 0, 0, 0, 0, 360, 4941, 0, 0, 0, 0, 0, 0, 0, 0, 2092, 1251, 0, 0, 0, 0, 4940, 9, 0, 3887, 0, 0, 689, 238, 2352, 0, 562] payload_rows_per_dst=[10102, 15540, 6816, 13974, 7040, 11984, 11166, 11708] dup_rows_per_dst=[982, 200, 216, 1506, 6, 2082, 1234, 3748] combine_rows_per_src=[10102, 15540, 6816, 13974, 7040, 11984, 11166, 11708] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83676 payload_rows_remote=69956 dup_rows=14628 dup_rows_remote=12724 dedup_ratio=0.148804 router_tokens_per_expert=[250, 460, 12, 85, 248, 0, 0, 0, 1, 3641, 113, 5133, 4, 1, 1895, 4428, 91, 404, 9, 3, 1064, 0, 0, 11, 269, 300, 0, 6454, 11, 3, 0, 0, 2, 2, 6, 7, 4504, 1364, 231, 0, 12, 0, 1, 4, 0, 104, 0, 9, 4454, 12, 0, 6757, 112, 2449, 1009, 0, 0, 3, 6, 0, 1, 347, 2762, 104] payload_rows_per_dst=[13720, 14168, 12976, 14318, 9440, 10514, 2262, 6278] dup_rows_per_dst=[1904, 5998, 3952, 1876, 672, 12, 46, 168] combine_rows_per_src=[13720, 14168, 12976, 14318, 9440, 10514, 2262, 6278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81922 payload_rows_remote=70444 dup_rows=16382 dup_rows_remote=15050 dedup_ratio=0.166646 router_tokens_per_expert=[6789, 85, 0, 251, 4394, 0, 393, 194, 0, 85, 0, 0, 0, 66, 0, 7747, 0, 671, 2888, 0, 0, 0, 323, 0, 0, 668, 0, 0, 0, 175, 0, 0, 3403, 0, 0, 0, 0, 393, 4969, 0, 0, 0, 0, 0, 0, 0, 0, 1774, 1475, 0, 0, 0, 0, 4930, 0, 0, 3568, 0, 0, 711, 307, 2281, 0, 612] payload_rows_per_dst=[7352, 2578, 16212, 13778, 7566, 11604, 11478, 11354] dup_rows_per_dst=[1006, 0, 7046, 1486, 26, 1882, 1332, 3604] combine_rows_per_src=[7352, 2578, 16212, 13778, 7566, 11604, 11478, 11354] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=321847296 dispatch_meta_bytes=344528 combine_payload_bytes=321847296 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=322191824 ingress_bytes=391053312 link_bytes=391053312 token_bytes=644039120 weight_movement_bytes=69206016 total_nvlink_bytes=713245136 weight_to_token_ratio=0.107456 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=36352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.471502131e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=304504832 dispatch_meta_bytes=337504 combine_payload_bytes=304504832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=304842336 ingress_bytes=339107840 link_bytes=339107840 token_bytes=609347168 weight_movement_bytes=34603008 total_nvlink_bytes=643950176 weight_to_token_ratio=0.056787 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=40576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.276035501e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=297664512 dispatch_meta_bytes=309792 combine_payload_bytes=297664512 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=297974304 ingress_bytes=436076544 link_bytes=436076544 token_bytes=595638816 weight_movement_bytes=138412032 total_nvlink_bytes=734050848 weight_to_token_ratio=0.232376 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=22016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.640920928e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=323772416 dispatch_meta_bytes=350416 combine_payload_bytes=323772416 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=324122832 ingress_bytes=392978432 link_bytes=392978432 token_bytes=647895248 weight_movement_bytes=69206016 total_nvlink_bytes=717101264 weight_to_token_ratio=0.106817 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=40704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.478746202e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=288538624 dispatch_meta_bytes=341976 combine_payload_bytes=288538624 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=288880600 ingress_bytes=357744640 link_bytes=357744640 token_bytes=577419224 weight_movement_bytes=69206016 total_nvlink_bytes=646625240 weight_to_token_ratio=0.119854 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=37120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.346164280e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75874 payload_rows_remote=60464 dup_rows=22430 dup_rows_remote=13724 dedup_ratio=0.228170 router_tokens_per_expert=[6787, 88, 0, 211, 4088, 1, 395, 209, 6, 93, 9, 2, 14, 64, 4, 7735, 0, 658, 2699, 0, 10, 0, 371, 18, 1, 713, 0, 0, 0, 156, 8, 4, 3145, 3, 2, 2, 0, 374, 5198, 8, 1, 1, 1, 1, 0, 10, 11, 1746, 1608, 2, 0, 0, 0, 4943, 22, 8, 3652, 0, 0, 700, 314, 2512, 7, 537] payload_rows_per_dst=[15410, 15512, 7148, 1756, 13988, 3528, 11308, 7224] dup_rows_per_dst=[8706, 342, 364, 8, 10780, 14, 1300, 916] combine_rows_per_src=[15410, 15512, 7148, 1756, 13988, 3528, 11308, 7224] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88306 payload_rows_remote=74556 dup_rows=9998 dup_rows_remote=8524 dedup_ratio=0.101705 router_tokens_per_expert=[6809, 80, 0, 215, 4320, 0, 391, 185, 0, 80, 0, 0, 0, 72, 0, 7763, 0, 661, 2985, 0, 0, 0, 378, 0, 0, 663, 0, 0, 0, 140, 0, 0, 3060, 0, 0, 0, 0, 377, 5064, 0, 0, 0, 0, 0, 0, 0, 0, 1907, 1571, 0, 0, 0, 0, 4814, 1, 0, 3475, 0, 0, 720, 277, 2662, 0, 482] payload_rows_per_dst=[9342, 15526, 7740, 13750, 10824, 11838, 11506, 7780] dup_rows_per_dst=[1040, 304, 308, 1474, 2384, 2104, 1266, 1118] combine_rows_per_src=[9342, 15526, 7740, 13750, 10824, 11838, 11506, 7780] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88330 payload_rows_remote=81290 dup_rows=9974 dup_rows_remote=9968 dedup_ratio=0.101461 router_tokens_per_expert=[6901, 94, 0, 185, 4678, 0, 389, 196, 0, 94, 0, 0, 0, 6, 0, 7770, 0, 563, 2667, 0, 0, 0, 286, 0, 0, 686, 0, 0, 0, 153, 0, 0, 3163, 0, 0, 0, 0, 360, 4941, 0, 0, 0, 0, 0, 0, 0, 0, 2092, 1251, 0, 0, 0, 0, 4940, 9, 0, 3887, 0, 0, 689, 238, 2352, 0, 562] payload_rows_per_dst=[10102, 15540, 6816, 13974, 7040, 11984, 11166, 11708] dup_rows_per_dst=[982, 200, 216, 1506, 6, 2082, 1234, 3748] combine_rows_per_src=[10102, 15540, 6816, 13974, 7040, 11984, 11166, 11708] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=332963840 dispatch_meta_bytes=365032 combine_payload_bytes=332963840 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=333328872 ingress_bytes=402169856 link_bytes=402169856 token_bytes=666292712 weight_movement_bytes=69206016 total_nvlink_bytes=735498728 weight_to_token_ratio=0.103867 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=25472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.513332791e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=286539776 dispatch_meta_bytes=330720 combine_payload_bytes=286539776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286870496 ingress_bytes=321142784 link_bytes=321142784 token_bytes=573410272 weight_movement_bytes=34603008 total_nvlink_bytes=608013280 weight_to_token_ratio=0.060346 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.035000 gemm_active_groups=7 gemm_peak_group_rows=41088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.208434442e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80388 payload_rows_remote=67810 dup_rows=17916 dup_rows_remote=11706 dedup_ratio=0.182251 router_tokens_per_expert=[6978, 63, 0, 211, 4281, 0, 432, 114, 0, 64, 0, 1, 0, 8, 0, 7814, 0, 571, 2734, 0, 0, 0, 428, 0, 0, 697, 0, 0, 0, 98, 0, 0, 3210, 0, 0, 1, 0, 343, 5091, 0, 0, 0, 0, 0, 0, 0, 0, 2016, 1534, 0, 0, 0, 0, 4555, 8, 0, 3841, 0, 0, 574, 304, 2531, 0, 650] payload_rows_per_dst=[1316, 8708, 16256, 14040, 7072, 11964, 8454, 12578] dup_rows_per_dst=[324, 0, 6838, 1506, 36, 2250, 752, 6210] combine_rows_per_src=[1316, 8708, 16256, 14040, 7072, 11964, 8454, 12578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89052 payload_rows_remote=78840 dup_rows=9252 dup_rows_remote=7988 dedup_ratio=0.094116 router_tokens_per_expert=[244, 447, 0, 77, 245, 0, 0, 0, 0, 3692, 117, 5078, 0, 0, 1951, 4171, 86, 409, 0, 0, 1081, 0, 0, 0, 249, 238, 0, 6735, 9, 0, 0, 0, 0, 0, 0, 0, 4785, 1361, 289, 0, 1, 0, 0, 0, 0, 90, 0, 0, 4162, 0, 0, 6771, 83, 2624, 1004, 0, 0, 0, 0, 0, 0, 269, 2795, 89] payload_rows_per_dst=[13702, 12340, 10212, 14258, 11756, 10338, 7196, 9250] dup_rows_per_dst=[1866, 3422, 1264, 204, 1114, 0, 226, 1156] combine_rows_per_src=[13702, 12340, 10212, 14258, 11756, 10338, 7196, 9250] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85360 payload_rows_remote=74016 dup_rows=12944 dup_rows_remote=11010 dedup_ratio=0.131673 router_tokens_per_expert=[6728, 70, 0, 202, 4677, 0, 388, 150, 0, 70, 0, 3, 0, 7, 0, 7795, 0, 634, 2789, 0, 0, 0, 25, 1, 0, 725, 0, 0, 0, 140, 0, 0, 3154, 0, 0, 0, 0, 353, 4669, 0, 0, 0, 0, 0, 0, 0, 0, 1970, 1630, 0, 0, 0, 0, 4891, 26, 0, 4624, 0, 0, 597, 307, 1983, 0, 544] payload_rows_per_dst=[9890, 8082, 11286, 13620, 7002, 11344, 11750, 12386] dup_rows_per_dst=[1084, 76, 3204, 1566, 12, 1934, 1344, 3724] combine_rows_per_src=[9890, 8082, 11286, 13620, 7002, 11344, 11750, 12386] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=305381376 dispatch_meta_bytes=332320 combine_payload_bytes=305381376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=305713696 ingress_bytes=339984384 link_bytes=339984384 token_bytes=611095072 weight_movement_bytes=34603008 total_nvlink_bytes=645698080 weight_to_token_ratio=0.056625 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=42624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.279333866e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82062 payload_rows_remote=66520 dup_rows=16242 dup_rows_remote=15982 dedup_ratio=0.165222 router_tokens_per_expert=[6672, 83, 0, 210, 4500, 0, 391, 186, 1, 85, 1, 0, 1, 45, 0, 7768, 0, 585, 2818, 0, 0, 0, 304, 0, 1, 604, 2, 0, 0, 152, 0, 1, 3287, 0, 0, 2, 0, 378, 5016, 1, 0, 0, 0, 0, 0, 0, 0, 1779, 1451, 0, 0, 0, 0, 5036, 18, 1, 4098, 0, 0, 689, 259, 2207, 1, 519] payload_rows_per_dst=[14290, 15542, 7124, 3964, 12674, 10344, 11570, 6554] dup_rows_per_dst=[7030, 260, 290, 320, 4546, 1558, 1442, 796] combine_rows_per_src=[14290, 15542, 7124, 3964, 12674, 10344, 11570, 6554] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80388 payload_rows_remote=67810 dup_rows=17916 dup_rows_remote=11706 dedup_ratio=0.182251 router_tokens_per_expert=[6978, 63, 0, 211, 4281, 0, 432, 114, 0, 64, 0, 1, 0, 8, 0, 7814, 0, 571, 2734, 0, 0, 0, 428, 0, 0, 697, 0, 0, 0, 98, 0, 0, 3210, 0, 0, 1, 0, 343, 5091, 0, 0, 0, 0, 0, 0, 0, 0, 2016, 1534, 0, 0, 0, 0, 4555, 8, 0, 3841, 0, 0, 574, 304, 2531, 0, 650] payload_rows_per_dst=[1316, 8708, 16256, 14040, 7072, 11964, 8454, 12578] dup_rows_per_dst=[324, 0, 6838, 1506, 36, 2250, 752, 6210] combine_rows_per_src=[1316, 8708, 16256, 14040, 7072, 11964, 8454, 12578] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88084 payload_rows_remote=81022 dup_rows=10220 dup_rows_remote=9826 dedup_ratio=0.103963 router_tokens_per_expert=[6928, 76, 0, 228, 4304, 0, 433, 174, 0, 73, 0, 1, 0, 21, 0, 7722, 0, 705, 2595, 0, 0, 0, 428, 0, 0, 710, 0, 0, 0, 140, 0, 0, 2972, 0, 0, 0, 1, 422, 5087, 0, 0, 0, 0, 0, 0, 0, 0, 1850, 1439, 0, 0, 0, 0, 4740, 22, 0, 3779, 0, 0, 672, 334, 2558, 0, 738] payload_rows_per_dst=[9304, 15444, 7062, 14000, 11516, 11886, 11218, 7654] dup_rows_per_dst=[1126, 190, 394, 1556, 2832, 1988, 1184, 950] combine_rows_per_src=[9304, 15444, 7062, 14000, 11516, 11886, 11218, 7654] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=247660544 dispatch_meta_bytes=296752 combine_payload_bytes=247660544 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=247957296 ingress_bytes=351469568 link_bytes=351469568 token_bytes=495617840 weight_movement_bytes=103809024 total_nvlink_bytes=599426864 weight_to_token_ratio=0.209454 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=29824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.322551689e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=305381376 dispatch_meta_bytes=332320 combine_payload_bytes=305381376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=305713696 ingress_bytes=339984384 link_bytes=339984384 token_bytes=611095072 weight_movement_bytes=34603008 total_nvlink_bytes=645698080 weight_to_token_ratio=0.056625 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=42624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.279333866e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83984 payload_rows_remote=71902 dup_rows=14320 dup_rows_remote=14150 dedup_ratio=0.145671 router_tokens_per_expert=[0, 7755, 92, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1225, 0, 175, 0, 0, 0, 8107, 0, 0, 0, 8067, 0, 226, 0, 40, 0, 0, 0, 0, 0, 0, 85, 0, 55, 0, 7598, 0, 123, 0, 0, 0, 93, 0, 0, 0, 6041, 0, 85, 0, 0, 0, 0, 0, 0, 0, 7334, 2050, 0, 0, 0, 0] payload_rows_per_dst=[2754, 2800, 12912, 16214, 4750, 16214, 12082, 16258] dup_rows_per_dst=[30, 0, 0, 452, 80, 432, 170, 13156] combine_rows_per_src=[2754, 2800, 12912, 16214, 4750, 16214, 12082, 16258] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81922 payload_rows_remote=70444 dup_rows=16382 dup_rows_remote=15050 dedup_ratio=0.166646 router_tokens_per_expert=[6789, 85, 0, 251, 4394, 0, 393, 194, 0, 85, 0, 0, 0, 66, 0, 7747, 0, 671, 2888, 0, 0, 0, 323, 0, 0, 668, 0, 0, 0, 175, 0, 0, 3403, 0, 0, 0, 0, 393, 4969, 0, 0, 0, 0, 0, 0, 0, 0, 1774, 1475, 0, 0, 0, 0, 4930, 0, 0, 3568, 0, 0, 711, 307, 2281, 0, 612] payload_rows_per_dst=[7352, 2578, 16212, 13778, 7566, 11604, 11478, 11354] dup_rows_per_dst=[1006, 0, 7046, 1486, 26, 1882, 1332, 3604] combine_rows_per_src=[7352, 2578, 16212, 13778, 7566, 11604, 11478, 11354] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=332963840 dispatch_meta_bytes=365032 combine_payload_bytes=332963840 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=333328872 ingress_bytes=402169856 link_bytes=402169856 token_bytes=666292712 weight_movement_bytes=69206016 total_nvlink_bytes=735498728 weight_to_token_ratio=0.103867 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=25472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.513332791e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=277749760 dispatch_meta_bytes=318064 combine_payload_bytes=277749760 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=278067824 ingress_bytes=346955776 link_bytes=346955776 token_bytes=555817584 weight_movement_bytes=69206016 total_nvlink_bytes=625023600 weight_to_token_ratio=0.124512 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=19200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.305566653e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=322928640 dispatch_meta_bytes=347312 combine_payload_bytes=322928640 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=323275952 ingress_bytes=392134656 link_bytes=392134656 token_bytes=646204592 weight_movement_bytes=69206016 total_nvlink_bytes=715410608 weight_to_token_ratio=0.107096 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=34432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.475571141e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=303169536 dispatch_meta_bytes=340104 combine_payload_bytes=303169536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=303509640 ingress_bytes=337772544 link_bytes=337772544 token_bytes=606679176 weight_movement_bytes=34603008 total_nvlink_bytes=641282184 weight_to_token_ratio=0.057037 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=34048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.271010890e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=272465920 dispatch_meta_bytes=330008 combine_payload_bytes=272465920 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=272795928 ingress_bytes=341671936 link_bytes=341671936 token_bytes=545261848 weight_movement_bytes=69206016 total_nvlink_bytes=614467864 weight_to_token_ratio=0.126923 gemm_rows_defined=50176 gemm_rows_charged=51200 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020000 gemm_active_groups=10 gemm_peak_group_rows=42880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.285683988e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=277749760 dispatch_meta_bytes=318064 combine_payload_bytes=277749760 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=278067824 ingress_bytes=346955776 link_bytes=346955776 token_bytes=555817584 weight_movement_bytes=69206016 total_nvlink_bytes=625023600 weight_to_token_ratio=0.124512 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=19200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.305566653e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=331866112 dispatch_meta_bytes=363392 combine_payload_bytes=331866112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=332229504 ingress_bytes=366469120 link_bytes=366469120 token_bytes=664095616 weight_movement_bytes=34603008 total_nvlink_bytes=698698624 weight_to_token_ratio=0.052105 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=22272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.378993795e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=288538624 dispatch_meta_bytes=341976 combine_payload_bytes=288538624 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=288880600 ingress_bytes=357744640 link_bytes=357744640 token_bytes=577419224 weight_movement_bytes=69206016 total_nvlink_bytes=646625240 weight_to_token_ratio=0.119854 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=37120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.346164280e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94466 payload_rows_remote=79252 dup_rows=3838 dup_rows_remote=3588 dedup_ratio=0.039042 router_tokens_per_expert=[0, 7775, 109, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1246, 0, 153, 0, 0, 0, 8098, 0, 0, 0, 8017, 0, 237, 0, 81, 0, 0, 0, 0, 0, 1, 94, 0, 31, 0, 7606, 0, 112, 0, 0, 0, 97, 0, 0, 0, 6134, 0, 94, 0, 0, 0, 0, 0, 0, 0, 7350, 1915, 0, 0, 0, 0] payload_rows_per_dst=[15550, 15212, 0, 16196, 15214, 16196, 12268, 3830] dup_rows_per_dst=[222, 2286, 0, 474, 250, 418, 188, 0] combine_rows_per_src=[15550, 15212, 0, 16196, 15214, 16196, 12268, 3830] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75874 payload_rows_remote=60464 dup_rows=22430 dup_rows_remote=13724 dedup_ratio=0.228170 router_tokens_per_expert=[6787, 88, 0, 211, 4088, 1, 395, 209, 6, 93, 9, 2, 14, 64, 4, 7735, 0, 658, 2699, 0, 10, 0, 371, 18, 1, 713, 0, 0, 0, 156, 8, 4, 3145, 3, 2, 2, 0, 374, 5198, 8, 1, 1, 1, 1, 0, 10, 11, 1746, 1608, 2, 0, 0, 0, 4943, 22, 8, 3652, 0, 0, 700, 314, 2512, 7, 537] payload_rows_per_dst=[15410, 15512, 7148, 1756, 13988, 3528, 11308, 7224] dup_rows_per_dst=[8706, 342, 364, 8, 10780, 14, 1300, 916] combine_rows_per_src=[15410, 15512, 7148, 1756, 13988, 3528, 11308, 7224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=294510592 dispatch_meta_bytes=344208 combine_payload_bytes=294510592 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=294854800 ingress_bytes=363716608 link_bytes=363716608 token_bytes=589365392 weight_movement_bytes=69206016 total_nvlink_bytes=658571408 weight_to_token_ratio=0.117425 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=48512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.368636314e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77638 payload_rows_remote=62094 dup_rows=20666 dup_rows_remote=19622 dedup_ratio=0.210225 router_tokens_per_expert=[3, 7748, 146, 0, 0, 0, 0, 11, 0, 3, 7, 1, 0, 1263, 2, 154, 3, 1, 1, 8069, 2, 9, 0, 7991, 0, 239, 2, 53, 0, 0, 1, 0, 7, 39, 88, 6, 45, 0, 7520, 0, 135, 0, 16, 0, 98, 6, 5, 16, 6125, 5, 88, 2, 0, 2, 0, 12, 0, 6, 7277, 1942, 1, 0, 2, 0] payload_rows_per_dst=[15544, 15066, 16178, 590, 15188, 504, 10668, 3900] dup_rows_per_dst=[1044, 2348, 15974, 0, 1078, 48, 172, 2] combine_rows_per_src=[15544, 15066, 16178, 590, 15188, 504, 10668, 3900] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94418 payload_rows_remote=78194 dup_rows=3886 dup_rows_remote=3506 dedup_ratio=0.039530 router_tokens_per_expert=[0, 7766, 92, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1278, 0, 140, 0, 0, 0, 8112, 0, 0, 0, 8043, 0, 190, 0, 69, 0, 0, 0, 0, 0, 8, 80, 0, 65, 0, 7561, 0, 137, 0, 0, 0, 87, 0, 0, 0, 6232, 0, 80, 0, 0, 0, 0, 0, 0, 0, 7352, 1858, 0, 0, 0, 0] payload_rows_per_dst=[15532, 15120, 0, 16224, 15138, 16224, 12464, 3716] dup_rows_per_dst=[188, 2420, 0, 380, 290, 448, 160, 0] combine_rows_per_src=[15532, 15120, 0, 16224, 15138, 16224, 12464, 3716] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94466 payload_rows_remote=79252 dup_rows=3838 dup_rows_remote=3588 dedup_ratio=0.039042 router_tokens_per_expert=[0, 7775, 109, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1246, 0, 153, 0, 0, 0, 8098, 0, 0, 0, 8017, 0, 237, 0, 81, 0, 0, 0, 0, 0, 1, 94, 0, 31, 0, 7606, 0, 112, 0, 0, 0, 97, 0, 0, 0, 6134, 0, 94, 0, 0, 0, 0, 0, 0, 0, 7350, 1915, 0, 0, 0, 0] payload_rows_per_dst=[15550, 15212, 0, 16196, 15214, 16196, 12268, 3830] dup_rows_per_dst=[222, 2286, 0, 474, 250, 418, 188, 0] combine_rows_per_src=[15550, 15212, 0, 16196, 15214, 16196, 12268, 3830] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=324616192 dispatch_meta_bytes=331360 combine_payload_bytes=324616192 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=324947552 ingress_bytes=393822208 link_bytes=393822208 token_bytes=649563744 weight_movement_bytes=69206016 total_nvlink_bytes=718769760 weight_to_token_ratio=0.106542 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=47744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.481921263e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=247660544 dispatch_meta_bytes=296752 combine_payload_bytes=247660544 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=247957296 ingress_bytes=351469568 link_bytes=351469568 token_bytes=495617840 weight_movement_bytes=103809024 total_nvlink_bytes=599426864 weight_to_token_ratio=0.209454 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=29824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.322551689e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=320282624 dispatch_meta_bytes=326800 combine_payload_bytes=320282624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=320609424 ingress_bytes=354885632 link_bytes=354885632 token_bytes=640892048 weight_movement_bytes=34603008 total_nvlink_bytes=675495056 weight_to_token_ratio=0.053992 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=46848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.335406063e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80076 payload_rows_remote=63692 dup_rows=18228 dup_rows_remote=918 dedup_ratio=0.185425 router_tokens_per_expert=[0, 7823, 108, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1279, 0, 98, 0, 0, 0, 8122, 0, 0, 0, 8064, 0, 203, 0, 64, 0, 0, 0, 0, 0, 0, 63, 0, 53, 0, 7519, 0, 124, 0, 0, 0, 69, 0, 0, 0, 6170, 0, 63, 0, 0, 0, 0, 0, 0, 0, 7461, 1867, 0, 0, 0, 0] payload_rows_per_dst=[220, 2754, 15646, 16256, 232, 16244, 12340, 16384] dup_rows_per_dst=[0, 0, 0, 406, 0, 386, 126, 17310] combine_rows_per_src=[220, 2754, 15646, 16256, 232, 16244, 12340, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88084 payload_rows_remote=81022 dup_rows=10220 dup_rows_remote=9826 dedup_ratio=0.103963 router_tokens_per_expert=[6928, 76, 0, 228, 4304, 0, 433, 174, 0, 73, 0, 1, 0, 21, 0, 7722, 0, 705, 2595, 0, 0, 0, 428, 0, 0, 710, 0, 0, 0, 140, 0, 0, 2972, 0, 0, 0, 1, 422, 5087, 0, 0, 0, 0, 0, 0, 0, 0, 1850, 1439, 0, 0, 0, 0, 4740, 22, 0, 3779, 0, 0, 672, 334, 2558, 0, 738] payload_rows_per_dst=[9304, 15444, 7062, 14000, 11516, 11886, 11218, 7654] dup_rows_per_dst=[1126, 190, 394, 1556, 2832, 1988, 1184, 950] combine_rows_per_src=[9304, 15444, 7062, 14000, 11516, 11886, 11218, 7654] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94280 payload_rows_remote=78036 dup_rows=4024 dup_rows_remote=3708 dedup_ratio=0.040934 router_tokens_per_expert=[0, 7799, 106, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 1290, 2, 140, 0, 0, 0, 8122, 0, 0, 0, 8060, 0, 271, 0, 61, 0, 0, 0, 0, 0, 9, 70, 0, 45, 0, 7519, 0, 70, 0, 0, 0, 88, 0, 0, 0, 6110, 0, 70, 0, 0, 0, 0, 0, 0, 0, 7580, 1725, 0, 0, 0, 0] payload_rows_per_dst=[15598, 4792, 0, 16242, 15044, 16244, 12220, 14140] dup_rows_per_dst=[242, 374, 0, 542, 242, 316, 140, 2168] combine_rows_per_src=[15598, 4792, 0, 16242, 15044, 16244, 12220, 14140] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=254337024 dispatch_meta_bytes=326864 combine_payload_bytes=254337024 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=254663888 ingress_bytes=323543040 link_bytes=323543040 token_bytes=509000912 weight_movement_bytes=69206016 total_nvlink_bytes=578206928 weight_to_token_ratio=0.135964 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=48000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.217466412e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79288 payload_rows_remote=64152 dup_rows=19016 dup_rows_remote=16676 dedup_ratio=0.193441 router_tokens_per_expert=[0, 7776, 92, 0, 0, 0, 0, 3, 0, 0, 1, 1, 0, 1250, 0, 152, 0, 1, 0, 8103, 0, 1, 0, 8038, 0, 247, 1, 68, 0, 0, 0, 0, 0, 12, 83, 1, 42, 0, 7589, 0, 89, 0, 1, 0, 92, 0, 0, 1, 6264, 0, 83, 0, 0, 0, 0, 0, 0, 0, 7334, 1824, 3, 0, 0, 0] payload_rows_per_dst=[15552, 15136, 16218, 632, 15208, 360, 12528, 3654] dup_rows_per_dst=[190, 2340, 16068, 0, 246, 6, 166, 0] combine_rows_per_src=[15552, 15136, 16218, 632, 15208, 360, 12528, 3654] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80076 payload_rows_remote=63692 dup_rows=18228 dup_rows_remote=918 dedup_ratio=0.185425 router_tokens_per_expert=[0, 7823, 108, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1279, 0, 98, 0, 0, 0, 8122, 0, 0, 0, 8064, 0, 203, 0, 64, 0, 0, 0, 0, 0, 0, 63, 0, 53, 0, 7519, 0, 124, 0, 0, 0, 69, 0, 0, 0, 6170, 0, 63, 0, 0, 0, 0, 0, 0, 0, 7461, 1867, 0, 0, 0, 0] payload_rows_per_dst=[220, 2754, 15646, 16256, 232, 16244, 12340, 16384] dup_rows_per_dst=[0, 0, 0, 406, 0, 386, 126, 17310] combine_rows_per_src=[220, 2754, 15646, 16256, 232, 16244, 12340, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92278 payload_rows_remote=89310 dup_rows=6026 dup_rows_remote=3686 dedup_ratio=0.061300 router_tokens_per_expert=[0, 7725, 93, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1331, 0, 139, 0, 0, 0, 8116, 0, 0, 0, 8055, 0, 204, 0, 61, 0, 0, 0, 0, 0, 0, 76, 0, 26, 0, 7570, 0, 116, 0, 0, 0, 82, 0, 0, 0, 6228, 0, 78, 0, 0, 0, 0, 0, 0, 0, 7358, 1893, 0, 0, 0, 0] payload_rows_per_dst=[15450, 15172, 2968, 13324, 15140, 13978, 12460, 3786] dup_rows_per_dst=[188, 2484, 2340, 372, 204, 286, 152, 0] combine_rows_per_src=[15450, 15172, 2968, 13324, 15140, 13978, 12460, 3786] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=320282624 dispatch_meta_bytes=326800 combine_payload_bytes=320282624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=320609424 ingress_bytes=354885632 link_bytes=354885632 token_bytes=640892048 weight_movement_bytes=34603008 total_nvlink_bytes=675495056 weight_to_token_ratio=0.053992 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=46848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.335406063e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84282 payload_rows_remote=68330 dup_rows=14022 dup_rows_remote=14022 dedup_ratio=0.142639 router_tokens_per_expert=[0, 0, 0, 551, 0, 0, 0, 0, 453, 0, 0, 0, 1591, 0, 0, 67, 0, 0, 2179, 0, 334, 0, 0, 0, 3235, 7932, 0, 4344, 0, 7976, 0, 0, 638, 1303, 0, 6846, 0, 7855, 0, 0, 0, 0, 321, 0, 0, 540, 715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2121, 61, 90, 0, 0, 0] payload_rows_per_dst=[1102, 3614, 5026, 15864, 14640, 16230, 15952, 11854] dup_rows_per_dst=[0, 608, 0, 0, 9404, 2632, 0, 1378] combine_rows_per_src=[1102, 3614, 5026, 15864, 14640, 16230, 15952, 11854] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=324616192 dispatch_meta_bytes=331360 combine_payload_bytes=324616192 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=324947552 ingress_bytes=393822208 link_bytes=393822208 token_bytes=649563744 weight_movement_bytes=69206016 total_nvlink_bytes=718769760 weight_to_token_ratio=0.106542 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=47744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.481921263e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83984 payload_rows_remote=71902 dup_rows=14320 dup_rows_remote=14150 dedup_ratio=0.145671 router_tokens_per_expert=[0, 7755, 92, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1225, 0, 175, 0, 0, 0, 8107, 0, 0, 0, 8067, 0, 226, 0, 40, 0, 0, 0, 0, 0, 0, 85, 0, 55, 0, 7598, 0, 123, 0, 0, 0, 93, 0, 0, 0, 6041, 0, 85, 0, 0, 0, 0, 0, 0, 0, 7334, 2050, 0, 0, 0, 0] payload_rows_per_dst=[2754, 2800, 12912, 16214, 4750, 16214, 12082, 16258] dup_rows_per_dst=[30, 0, 0, 452, 80, 432, 170, 13156] combine_rows_per_src=[2754, 2800, 12912, 16214, 4750, 16214, 12082, 16258] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=260882432 dispatch_meta_bytes=258440 combine_payload_bytes=260882432 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=261140872 ingress_bytes=330088448 link_bytes=330088448 token_bytes=522023304 weight_movement_bytes=69206016 total_nvlink_bytes=591229320 weight_to_token_ratio=0.132573 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.242096256e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=331866112 dispatch_meta_bytes=363392 combine_payload_bytes=331866112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=332229504 ingress_bytes=366469120 link_bytes=366469120 token_bytes=664095616 weight_movement_bytes=34603008 total_nvlink_bytes=698698624 weight_to_token_ratio=0.052105 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=22272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.378993795e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=319635456 dispatch_meta_bytes=326976 combine_payload_bytes=319635456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=319962432 ingress_bytes=354238464 link_bytes=354238464 token_bytes=639597888 weight_movement_bytes=34603008 total_nvlink_bytes=674200896 weight_to_token_ratio=0.054101 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=47616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.332970821e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=262766592 dispatch_meta_bytes=323312 combine_payload_bytes=262766592 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263089904 ingress_bytes=297369600 link_bytes=297369600 token_bytes=525856496 weight_movement_bytes=34603008 total_nvlink_bytes=560459504 weight_to_token_ratio=0.065803 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=37888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.118977865e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=365813760 dispatch_meta_bytes=371984 combine_payload_bytes=365813760 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=366185744 ingress_bytes=469622784 link_bytes=469622784 token_bytes=731999504 weight_movement_bytes=103809024 total_nvlink_bytes=835808528 weight_to_token_ratio=0.141816 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=17536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.767152728e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=260882432 dispatch_meta_bytes=258440 combine_payload_bytes=260882432 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=261140872 ingress_bytes=330088448 link_bytes=330088448 token_bytes=522023304 weight_movement_bytes=69206016 total_nvlink_bytes=591229320 weight_to_token_ratio=0.132573 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.242096256e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88630 payload_rows_remote=79166 dup_rows=9674 dup_rows_remote=7620 dedup_ratio=0.098409 router_tokens_per_expert=[0, 0, 0, 430, 0, 0, 0, 0, 338, 0, 0, 0, 1617, 0, 0, 62, 0, 0, 2158, 0, 303, 0, 0, 0, 3395, 7945, 0, 4517, 0, 7958, 0, 0, 456, 1233, 0, 6928, 0, 7849, 0, 0, 0, 0, 319, 0, 0, 609, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2130, 88, 59, 0, 0, 0] payload_rows_per_dst=[16008, 12628, 6288, 0, 9464, 16200, 15916, 12126] dup_rows_per_dst=[742, 2006, 540, 0, 2054, 2870, 0, 1462] combine_rows_per_src=[16008, 12628, 6288, 0, 9464, 16200, 15916, 12126] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=294510592 dispatch_meta_bytes=344208 combine_payload_bytes=294510592 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=294854800 ingress_bytes=363716608 link_bytes=363716608 token_bytes=589365392 weight_movement_bytes=69206016 total_nvlink_bytes=658571408 weight_to_token_ratio=0.117425 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=48512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.368636314e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77638 payload_rows_remote=62094 dup_rows=20666 dup_rows_remote=19622 dedup_ratio=0.210225 router_tokens_per_expert=[3, 7748, 146, 0, 0, 0, 0, 11, 0, 3, 7, 1, 0, 1263, 2, 154, 3, 1, 1, 8069, 2, 9, 0, 7991, 0, 239, 2, 53, 0, 0, 1, 0, 7, 39, 88, 6, 45, 0, 7520, 0, 135, 0, 16, 0, 98, 6, 5, 16, 6125, 5, 88, 2, 0, 2, 0, 12, 0, 6, 7277, 1942, 1, 0, 2, 0] payload_rows_per_dst=[15544, 15066, 16178, 590, 15188, 504, 10668, 3900] dup_rows_per_dst=[1044, 2348, 15974, 0, 1078, 48, 172, 2] combine_rows_per_src=[15544, 15066, 16178, 590, 15188, 504, 10668, 3900] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=279879680 dispatch_meta_bytes=329408 combine_payload_bytes=279879680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280209088 ingress_bytes=314482688 link_bytes=314482688 token_bytes=560088768 weight_movement_bytes=34603008 total_nvlink_bytes=594691776 weight_to_token_ratio=0.061781 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.183373037e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75600 payload_rows_remote=59630 dup_rows=22704 dup_rows_remote=21894 dedup_ratio=0.230957 router_tokens_per_expert=[5, 1, 0, 443, 3, 0, 1, 18, 405, 21, 0, 12, 1677, 1, 7, 48, 0, 2, 2160, 1, 358, 10, 0, 3, 3369, 7919, 0, 3824, 14, 7941, 3, 5, 527, 1355, 0, 6726, 1, 7823, 9, 2, 0, 0, 350, 5, 10, 645, 858, 6, 0, 52, 14, 1, 2, 0, 0, 4, 1, 12, 2330, 82, 66, 0, 0, 20] payload_rows_per_dst=[15970, 15324, 15680, 16126, 3556, 3460, 140, 5344] dup_rows_per_dst=[810, 2470, 5034, 13628, 232, 288, 6, 236] combine_rows_per_src=[15970, 15324, 15680, 16126, 3556, 3460, 140, 5344] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89168 payload_rows_remote=84070 dup_rows=9136 dup_rows_remote=9136 dedup_ratio=0.092936 router_tokens_per_expert=[0, 0, 0, 527, 0, 0, 0, 0, 291, 0, 0, 0, 1646, 0, 0, 21, 0, 0, 2369, 0, 340, 0, 0, 0, 3101, 7972, 0, 4294, 0, 7983, 0, 0, 574, 1258, 0, 6934, 0, 7873, 0, 0, 0, 0, 278, 0, 0, 592, 817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2175, 47, 60, 0, 0, 0] payload_rows_per_dst=[16064, 15446, 9932, 5098, 4334, 10480, 15966, 11848] dup_rows_per_dst=[934, 2338, 2338, 0, 434, 1788, 0, 1304] combine_rows_per_src=[16064, 15446, 9932, 5098, 4334, 10480, 15966, 11848] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 2, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88630 payload_rows_remote=79166 dup_rows=9674 dup_rows_remote=7620 dedup_ratio=0.098409 router_tokens_per_expert=[0, 0, 0, 430, 0, 0, 0, 0, 338, 0, 0, 0, 1617, 0, 0, 62, 0, 0, 2158, 0, 303, 0, 0, 0, 3395, 7945, 0, 4517, 0, 7958, 0, 0, 456, 1233, 0, 6928, 0, 7849, 0, 0, 0, 0, 319, 0, 0, 609, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2130, 88, 59, 0, 0, 0] payload_rows_per_dst=[16008, 12628, 6288, 0, 9464, 16200, 15916, 12126] dup_rows_per_dst=[742, 2006, 540, 0, 2054, 2870, 0, 1462] combine_rows_per_src=[16008, 12628, 6288, 0, 9464, 16200, 15916, 12126] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84772 payload_rows_remote=72770 dup_rows=13532 dup_rows_remote=12188 dedup_ratio=0.137655 router_tokens_per_expert=[0, 0, 0, 517, 0, 0, 0, 0, 290, 1, 0, 0, 1547, 0, 0, 14, 0, 0, 2277, 0, 296, 4, 0, 1, 3398, 8031, 0, 4457, 0, 8029, 0, 0, 400, 1205, 0, 6980, 0, 7875, 0, 0, 0, 0, 251, 0, 0, 642, 721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2117, 63, 35, 0, 0, 0] payload_rows_per_dst=[1034, 3296, 5156, 16062, 14884, 16280, 16058, 12002] dup_rows_per_dst=[0, 408, 0, 0, 9082, 2698, 0, 1344] combine_rows_per_src=[1034, 3296, 5156, 16062, 14884, 16280, 16058, 12002] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92278 payload_rows_remote=89310 dup_rows=6026 dup_rows_remote=3686 dedup_ratio=0.061300 router_tokens_per_expert=[0, 7725, 93, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1331, 0, 139, 0, 0, 0, 8116, 0, 0, 0, 8055, 0, 204, 0, 61, 0, 0, 0, 0, 0, 0, 76, 0, 26, 0, 7570, 0, 116, 0, 0, 0, 82, 0, 0, 0, 6228, 0, 78, 0, 0, 0, 0, 0, 0, 0, 7358, 1893, 0, 0, 0, 0] payload_rows_per_dst=[15450, 15172, 2968, 13324, 15140, 13978, 12460, 3786] dup_rows_per_dst=[188, 2484, 2340, 372, 204, 286, 152, 0] combine_rows_per_src=[15450, 15172, 2968, 13324, 15140, 13978, 12460, 3786] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=324263936 dispatch_meta_bytes=347144 combine_payload_bytes=324263936 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=324611080 ingress_bytes=497278976 link_bytes=497278976 token_bytes=648875016 weight_movement_bytes=173015040 total_nvlink_bytes=821890056 weight_to_token_ratio=0.266638 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.871220752e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=254337024 dispatch_meta_bytes=326864 combine_payload_bytes=254337024 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=254663888 ingress_bytes=323543040 link_bytes=323543040 token_bytes=509000912 weight_movement_bytes=69206016 total_nvlink_bytes=578206928 weight_to_token_ratio=0.135964 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=48000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.217466412e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=344350720 dispatch_meta_bytes=372824 combine_payload_bytes=344350720 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=344723544 ingress_bytes=482762752 link_bytes=482762752 token_bytes=689074264 weight_movement_bytes=138412032 total_nvlink_bytes=827486296 weight_to_token_ratio=0.200867 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.816597370e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=244244480 dispatch_meta_bytes=326096 combine_payload_bytes=244244480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=244570576 ingress_bytes=278847488 link_bytes=278847488 token_bytes=488815056 weight_movement_bytes=34603008 total_nvlink_bytes=523418064 weight_to_token_ratio=0.070790 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=45312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.049280648e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 2, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84300 payload_rows_remote=68048 dup_rows=14004 dup_rows_remote=11522 dedup_ratio=0.142456 router_tokens_per_expert=[0, 0, 0, 475, 0, 0, 0, 0, 452, 3, 0, 0, 1641, 0, 0, 57, 0, 0, 2822, 0, 283, 0, 0, 0, 3310, 7982, 0, 3558, 0, 7991, 0, 0, 522, 1164, 0, 6683, 0, 7853, 0, 0, 0, 0, 348, 0, 0, 574, 592, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 2645, 104, 72, 0, 0, 0] payload_rows_per_dst=[11340, 3642, 6210, 5036, 14438, 16252, 15982, 11400] dup_rows_per_dst=[538, 664, 0, 0, 8920, 2482, 42, 1358] combine_rows_per_src=[11340, 3642, 6210, 5036, 14438, 16252, 15982, 11400] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82728 payload_rows_remote=67248 dup_rows=15576 dup_rows_remote=12936 dedup_ratio=0.158447 router_tokens_per_expert=[0, 0, 0, 469, 0, 0, 0, 1, 407, 1, 0, 0, 1761, 2, 1, 79, 0, 0, 2293, 0, 325, 0, 0, 0, 3038, 7956, 1, 4353, 0, 7967, 1, 0, 551, 1281, 0, 6809, 0, 7852, 0, 0, 0, 0, 371, 0, 0, 612, 794, 1, 0, 0, 1, 2, 0, 0, 0, 0, 0, 1, 2082, 69, 70, 0, 0, 1] payload_rows_per_dst=[16052, 15480, 15712, 14634, 3448, 3298, 2260, 11844] dup_rows_per_dst=[800, 2640, 5228, 5126, 216, 258, 0, 1308] combine_rows_per_src=[16052, 15480, 15712, 14634, 3448, 3298, 2260, 11844] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88126 payload_rows_remote=72354 dup_rows=10178 dup_rows_remote=4790 dedup_ratio=0.103536 router_tokens_per_expert=[0, 0, 0, 479, 0, 0, 0, 0, 370, 0, 0, 0, 1648, 0, 0, 34, 0, 0, 2336, 0, 372, 0, 0, 0, 3365, 7976, 0, 4540, 0, 7991, 0, 0, 500, 1099, 0, 6818, 0, 7872, 0, 0, 0, 0, 304, 0, 0, 504, 736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2113, 30, 65, 0, 0, 0] payload_rows_per_dst=[16080, 15478, 15772, 6730, 3008, 2876, 15982, 12200] dup_rows_per_dst=[830, 2262, 5388, 0, 190, 212, 0, 1296] combine_rows_per_src=[16080, 15478, 15772, 6730, 3008, 2876, 15982, 12200] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84772 payload_rows_remote=72770 dup_rows=13532 dup_rows_remote=12188 dedup_ratio=0.137655 router_tokens_per_expert=[0, 0, 0, 517, 0, 0, 0, 0, 290, 1, 0, 0, 1547, 0, 0, 14, 0, 0, 2277, 0, 296, 4, 0, 1, 3398, 8031, 0, 4457, 0, 8029, 0, 0, 400, 1205, 0, 6980, 0, 7875, 0, 0, 0, 0, 251, 0, 0, 642, 721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2117, 63, 35, 0, 0, 0] payload_rows_per_dst=[1034, 3296, 5156, 16062, 14884, 16280, 16058, 12002] dup_rows_per_dst=[0, 408, 0, 0, 9082, 2698, 0, 1344] combine_rows_per_src=[1034, 3296, 5156, 16062, 14884, 16280, 16058, 12002] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90248 payload_rows_remote=79694 dup_rows=8056 dup_rows_remote=7980 dedup_ratio=0.081950 router_tokens_per_expert=[0, 0, 4788, 126, 3519, 5088, 0, 353, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 0, 0, 0, 8192, 0, 412, 0, 0, 27, 8192, 1, 0, 0, 0, 288, 0, 0, 171, 0, 0, 124, 375, 389, 0, 0, 5813, 0, 0, 0, 0, 7936, 0, 0, 85, 440, 0, 2, 0, 8, 0, 908, 550, 1056, 88, 0, 85] payload_rows_per_dst=[7042, 10176, 1096, 16384, 16384, 12228, 10554, 16384] dup_rows_per_dst=[536, 0, 0, 474, 1916, 176, 76, 4878] combine_rows_per_src=[7042, 10176, 1096, 16384, 16384, 12228, 10554, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=344350720 dispatch_meta_bytes=372824 combine_payload_bytes=344350720 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=344723544 ingress_bytes=482762752 link_bytes=482762752 token_bytes=689074264 weight_movement_bytes=138412032 total_nvlink_bytes=827486296 weight_to_token_ratio=0.200867 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.816597370e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=324263936 dispatch_meta_bytes=347144 combine_payload_bytes=324263936 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=324611080 ingress_bytes=497278976 link_bytes=497278976 token_bytes=648875016 weight_movement_bytes=173015040 total_nvlink_bytes=821890056 weight_to_token_ratio=0.266638 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.871220752e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=298065920 dispatch_meta_bytes=339832 combine_payload_bytes=298065920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=298405752 ingress_bytes=332668928 link_bytes=332668928 token_bytes=596471672 weight_movement_bytes=34603008 total_nvlink_bytes=631074680 weight_to_token_ratio=0.058013 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=30464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.251806394e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84282 payload_rows_remote=68330 dup_rows=14022 dup_rows_remote=14022 dedup_ratio=0.142639 router_tokens_per_expert=[0, 0, 0, 551, 0, 0, 0, 0, 453, 0, 0, 0, 1591, 0, 0, 67, 0, 0, 2179, 0, 334, 0, 0, 0, 3235, 7932, 0, 4344, 0, 7976, 0, 0, 638, 1303, 0, 6846, 0, 7855, 0, 0, 0, 0, 321, 0, 0, 540, 715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2121, 61, 90, 0, 0, 0] payload_rows_per_dst=[1102, 3614, 5026, 15864, 14640, 16230, 15952, 11854] dup_rows_per_dst=[0, 608, 0, 0, 9404, 2632, 0, 1378] combine_rows_per_src=[1102, 3614, 5026, 15864, 14640, 16230, 15952, 11854] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=365813760 dispatch_meta_bytes=371984 combine_payload_bytes=365813760 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=366185744 ingress_bytes=469622784 link_bytes=469622784 token_bytes=731999504 weight_movement_bytes=103809024 total_nvlink_bytes=835808528 weight_to_token_ratio=0.141816 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=17536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.767152728e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=278724608 dispatch_meta_bytes=318280 combine_payload_bytes=278724608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279042888 ingress_bytes=313327616 link_bytes=313327616 token_bytes=557767496 weight_movement_bytes=34603008 total_nvlink_bytes=592370504 weight_to_token_ratio=0.062038 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=35968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.179026594e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=275447808 dispatch_meta_bytes=320736 combine_payload_bytes=275447808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275768544 ingress_bytes=310050816 link_bytes=310050816 token_bytes=551216352 weight_movement_bytes=34603008 total_nvlink_bytes=585819360 weight_to_token_ratio=0.062776 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.166696259e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=296361984 dispatch_meta_bytes=308576 combine_payload_bytes=296361984 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=296670560 ingress_bytes=365568000 link_bytes=365568000 token_bytes=593032544 weight_movement_bytes=69206016 total_nvlink_bytes=662238560 weight_to_token_ratio=0.116699 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=27008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.375602953e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=298065920 dispatch_meta_bytes=339832 combine_payload_bytes=298065920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=298405752 ingress_bytes=332668928 link_bytes=332668928 token_bytes=596471672 weight_movement_bytes=34603008 total_nvlink_bytes=631074680 weight_to_token_ratio=0.058013 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=30464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.251806394e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=279879680 dispatch_meta_bytes=329408 combine_payload_bytes=279879680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280209088 ingress_bytes=314482688 link_bytes=314482688 token_bytes=560088768 weight_movement_bytes=34603008 total_nvlink_bytes=594691776 weight_to_token_ratio=0.061781 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.183373037e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=326426624 dispatch_meta_bytes=350696 combine_payload_bytes=326426624 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=326777320 ingress_bytes=395632640 link_bytes=395632640 token_bytes=653203944 weight_movement_bytes=69206016 total_nvlink_bytes=722409960 weight_to_token_ratio=0.105949 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=34688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.488733773e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74090 payload_rows_remote=60568 dup_rows=24214 dup_rows_remote=19354 dedup_ratio=0.246318 router_tokens_per_expert=[1, 2, 5135, 97, 3115, 5083, 6, 340, 0, 0, 0, 3, 3, 10, 0, 5, 164, 0, 1, 0, 19, 8148, 7, 456, 0, 0, 44, 8172, 1, 5, 0, 4, 297, 5, 1, 192, 0, 10, 103, 412, 340, 13, 26, 5785, 0, 0, 15, 1, 7888, 0, 0, 77, 520, 0, 1, 4, 20, 0, 1027, 461, 927, 111, 7, 88] payload_rows_per_dst=[13522, 10200, 16384, 108, 1942, 11146, 15940, 4848] dup_rows_per_dst=[4860, 8, 17550, 0, 98, 224, 1040, 434] combine_rows_per_src=[13522, 10200, 16384, 108, 1942, 11146, 15940, 4848] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 2, 0, 1, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87090 payload_rows_remote=70706 dup_rows=11214 dup_rows_remote=9370 dedup_ratio=0.114075 router_tokens_per_expert=[0, 0, 4904, 69, 3509, 5171, 0, 325, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 0, 0, 0, 8192, 0, 403, 0, 0, 22, 8192, 0, 0, 0, 0, 251, 0, 0, 184, 0, 0, 71, 416, 317, 0, 0, 5914, 0, 0, 0, 0, 7912, 0, 0, 92, 467, 0, 17, 0, 8, 0, 971, 487, 970, 95, 0, 94] payload_rows_per_dst=[9456, 12766, 1004, 16384, 16384, 12272, 6550, 12274] dup_rows_per_dst=[2046, 3688, 0, 44, 1844, 190, 300, 3102] combine_rows_per_src=[9456, 12766, 1004, 16384, 16384, 12272, 6550, 12274] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75600 payload_rows_remote=59630 dup_rows=22704 dup_rows_remote=21894 dedup_ratio=0.230957 router_tokens_per_expert=[5, 1, 0, 443, 3, 0, 1, 18, 405, 21, 0, 12, 1677, 1, 7, 48, 0, 2, 2160, 1, 358, 10, 0, 3, 3369, 7919, 0, 3824, 14, 7941, 3, 5, 527, 1355, 0, 6726, 1, 7823, 9, 2, 0, 0, 350, 5, 10, 645, 858, 6, 0, 52, 14, 1, 2, 0, 0, 4, 1, 12, 2330, 82, 66, 0, 0, 20] payload_rows_per_dst=[15970, 15324, 15680, 16126, 3556, 3460, 140, 5344] dup_rows_per_dst=[810, 2470, 5034, 13628, 232, 288, 6, 236] combine_rows_per_src=[15970, 15324, 15680, 16126, 3556, 3460, 140, 5344] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90576 payload_rows_remote=74192 dup_rows=7728 dup_rows_remote=7686 dedup_ratio=0.078613 router_tokens_per_expert=[0, 0, 4800, 74, 3522, 5108, 0, 346, 0, 0, 0, 0, 0, 0, 0, 0, 179, 0, 0, 0, 0, 8192, 0, 447, 0, 0, 21, 8192, 0, 0, 0, 0, 264, 0, 0, 172, 0, 0, 75, 447, 295, 0, 0, 5957, 0, 0, 0, 0, 7940, 0, 0, 79, 468, 0, 1, 0, 8, 0, 872, 540, 977, 96, 0, 80] payload_rows_per_dst=[13256, 10216, 1252, 16384, 16384, 12342, 16038, 4704] dup_rows_per_dst=[4228, 0, 0, 42, 1916, 162, 938, 442] combine_rows_per_src=[13256, 10216, 1252, 16384, 16384, 12342, 16038, 4704] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=248086528 dispatch_meta_bytes=319688 combine_payload_bytes=248086528 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=248406216 ingress_bytes=421101568 link_bytes=421101568 token_bytes=496492744 weight_movement_bytes=173015040 total_nvlink_bytes=669507784 weight_to_token_ratio=0.348474 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=23936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.584571299e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84056 payload_rows_remote=67672 dup_rows=14248 dup_rows_remote=9808 dedup_ratio=0.144938 router_tokens_per_expert=[0, 0, 4732, 65, 3871, 5244, 0, 311, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 8192, 0, 400, 0, 0, 21, 8192, 0, 0, 0, 0, 264, 0, 0, 108, 0, 0, 69, 475, 226, 0, 0, 5949, 0, 0, 0, 0, 8020, 0, 0, 58, 471, 0, 6, 0, 5, 0, 887, 435, 948, 54, 0, 63] payload_rows_per_dst=[752, 10488, 972, 16384, 16384, 12226, 10466, 16384] dup_rows_per_dst=[0, 0, 0, 7784, 1832, 124, 68, 4440] combine_rows_per_src=[752, 10488, 972, 16384, 16384, 12226, 10466, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88126 payload_rows_remote=72354 dup_rows=10178 dup_rows_remote=4790 dedup_ratio=0.103536 router_tokens_per_expert=[0, 0, 0, 479, 0, 0, 0, 0, 370, 0, 0, 0, 1648, 0, 0, 34, 0, 0, 2336, 0, 372, 0, 0, 0, 3365, 7976, 0, 4540, 0, 7991, 0, 0, 500, 1099, 0, 6818, 0, 7872, 0, 0, 0, 0, 304, 0, 0, 504, 736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2113, 30, 65, 0, 0, 0] payload_rows_per_dst=[16080, 15478, 15772, 6730, 3008, 2876, 15982, 12200] dup_rows_per_dst=[830, 2262, 5388, 0, 190, 212, 0, 1296] combine_rows_per_src=[16080, 15478, 15772, 6730, 3008, 2876, 15982, 12200] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=289611776 dispatch_meta_bytes=320304 combine_payload_bytes=289611776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289932080 ingress_bytes=324214784 link_bytes=324214784 token_bytes=579543856 weight_movement_bytes=34603008 total_nvlink_bytes=614146864 weight_to_token_ratio=0.059707 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=41600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.219994131e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=244244480 dispatch_meta_bytes=326096 combine_payload_bytes=244244480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=244570576 ingress_bytes=278847488 link_bytes=278847488 token_bytes=488815056 weight_movement_bytes=34603008 total_nvlink_bytes=523418064 weight_to_token_ratio=0.070790 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=45312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.049280648e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=303890432 dispatch_meta_bytes=327512 combine_payload_bytes=303890432 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=304217944 ingress_bytes=373096448 link_bytes=373096448 token_bytes=608108376 weight_movement_bytes=69206016 total_nvlink_bytes=677314392 weight_to_token_ratio=0.113805 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=44928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.403931897e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86078 payload_rows_remote=73558 dup_rows=12226 dup_rows_remote=12078 dedup_ratio=0.124369 router_tokens_per_expert=[0, 0, 5115, 110, 3237, 5252, 0, 323, 0, 0, 0, 0, 0, 0, 0, 0, 196, 0, 0, 0, 0, 8192, 0, 429, 0, 0, 11, 8192, 0, 0, 0, 0, 274, 0, 0, 130, 0, 0, 114, 462, 325, 0, 0, 6009, 0, 0, 0, 0, 7992, 0, 0, 66, 465, 0, 10, 0, 14, 0, 689, 428, 983, 64, 0, 70] payload_rows_per_dst=[6800, 14090, 1250, 16384, 16384, 12520, 2266, 16384] dup_rows_per_dst=[540, 5454, 0, 22, 1960, 148, 6, 4096] combine_rows_per_src=[6800, 14090, 1250, 16384, 16384, 12520, 2266, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82880 payload_rows_remote=67552 dup_rows=15424 dup_rows_remote=5846 dedup_ratio=0.156901 router_tokens_per_expert=[0, 0, 4533, 86, 3801, 5423, 0, 390, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 0, 0, 0, 8192, 0, 444, 0, 0, 23, 8192, 0, 0, 0, 0, 244, 0, 0, 165, 0, 0, 88, 439, 301, 0, 0, 5844, 0, 0, 0, 0, 7951, 0, 0, 72, 441, 0, 12, 0, 4, 0, 794, 382, 1052, 96, 0, 76] payload_rows_per_dst=[13294, 10846, 15328, 7836, 2924, 12140, 16070, 4442] dup_rows_per_dst=[4326, 0, 9578, 26, 96, 150, 882, 366] combine_rows_per_src=[13294, 10846, 15328, 7836, 2924, 12140, 16070, 4442] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84056 payload_rows_remote=67672 dup_rows=14248 dup_rows_remote=9808 dedup_ratio=0.144938 router_tokens_per_expert=[0, 0, 4732, 65, 3871, 5244, 0, 311, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 8192, 0, 400, 0, 0, 21, 8192, 0, 0, 0, 0, 264, 0, 0, 108, 0, 0, 69, 475, 226, 0, 0, 5949, 0, 0, 0, 0, 8020, 0, 0, 58, 471, 0, 6, 0, 5, 0, 887, 435, 948, 54, 0, 63] payload_rows_per_dst=[752, 10488, 972, 16384, 16384, 12226, 10466, 16384] dup_rows_per_dst=[0, 0, 0, 7784, 1832, 124, 68, 4440] combine_rows_per_src=[752, 10488, 972, 16384, 16384, 12226, 10466, 16384] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74872 payload_rows_remote=64500 dup_rows=23432 dup_rows_remote=23432 dedup_ratio=0.238363 router_tokens_per_expert=[0, 0, 5082, 88, 3319, 5186, 0, 334, 0, 0, 0, 0, 0, 0, 0, 0, 131, 0, 0, 0, 1, 8187, 0, 435, 0, 0, 23, 8191, 0, 0, 0, 1, 262, 0, 0, 171, 0, 1, 89, 405, 323, 1, 1, 5802, 0, 0, 1, 0, 7936, 0, 5, 81, 528, 0, 5, 0, 15, 0, 932, 497, 944, 91, 1, 83] payload_rows_per_dst=[13380, 10372, 16384, 48, 1834, 12088, 16038, 4728] dup_rows_per_dst=[4266, 0, 17506, 0, 22, 168, 1072, 398] combine_rows_per_src=[13380, 10372, 16384, 48, 1834, 12088, 16038, 4728] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90248 payload_rows_remote=79694 dup_rows=8056 dup_rows_remote=7980 dedup_ratio=0.081950 router_tokens_per_expert=[0, 0, 4788, 126, 3519, 5088, 0, 353, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 0, 0, 0, 8192, 0, 412, 0, 0, 27, 8192, 1, 0, 0, 0, 288, 0, 0, 171, 0, 0, 124, 375, 389, 0, 0, 5813, 0, 0, 0, 0, 7936, 0, 0, 85, 440, 0, 2, 0, 8, 0, 908, 550, 1056, 88, 0, 85] payload_rows_per_dst=[7042, 10176, 1096, 16384, 16384, 12228, 10554, 16384] dup_rows_per_dst=[536, 0, 0, 474, 1916, 176, 76, 4878] combine_rows_per_src=[7042, 10176, 1096, 16384, 16384, 12228, 10554, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94614 payload_rows_remote=78400 dup_rows=3690 dup_rows_remote=3690 dedup_ratio=0.037537 router_tokens_per_expert=[904, 0, 0, 0, 97, 0, 0, 0, 0, 881, 0, 0, 87, 0, 7169, 8107, 7303, 5419, 0, 85, 0, 0, 0, 0, 169, 0, 0, 16, 0, 313, 7567, 1679, 260, 175, 0, 765, 0, 0, 0, 0, 0, 0, 85, 0, 0, 90, 260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7721, 0, 0, 0, 0, 0] payload_rows_per_dst=[2002, 15264, 10838, 4080, 15956, 14818, 16214, 15442] dup_rows_per_dst=[0, 1010, 170, 274, 1578, 658, 0, 0] combine_rows_per_src=[2002, 15264, 10838, 4080, 15956, 14818, 16214, 15442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=289611776 dispatch_meta_bytes=320304 combine_payload_bytes=289611776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289932080 ingress_bytes=324214784 link_bytes=324214784 token_bytes=579543856 weight_movement_bytes=34603008 total_nvlink_bytes=614146864 weight_to_token_ratio=0.059707 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=41600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.219994131e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=303890432 dispatch_meta_bytes=327512 combine_payload_bytes=303890432 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=304217944 ingress_bytes=373096448 link_bytes=373096448 token_bytes=608108376 weight_movement_bytes=69206016 total_nvlink_bytes=677314392 weight_to_token_ratio=0.113805 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=44928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.403931897e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=301293568 dispatch_meta_bytes=342544 combine_payload_bytes=301293568 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=301636112 ingress_bytes=335896576 link_bytes=335896576 token_bytes=602929680 weight_movement_bytes=34603008 total_nvlink_bytes=637532688 weight_to_token_ratio=0.057391 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=46592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263951774e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=276692992 dispatch_meta_bytes=293592 combine_payload_bytes=276692992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=276986584 ingress_bytes=345899008 link_bytes=345899008 token_bytes=553679576 weight_movement_bytes=69206016 total_nvlink_bytes=622885592 weight_to_token_ratio=0.124993 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=24064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.301590120e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=296361984 dispatch_meta_bytes=308576 combine_payload_bytes=296361984 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=296670560 ingress_bytes=365568000 link_bytes=365568000 token_bytes=593032544 weight_movement_bytes=69206016 total_nvlink_bytes=662238560 weight_to_token_ratio=0.116699 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=27008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.375602953e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=277184512 dispatch_meta_bytes=309920 combine_payload_bytes=277184512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277494432 ingress_bytes=311787520 link_bytes=311787520 token_bytes=554678944 weight_movement_bytes=34603008 total_nvlink_bytes=589281952 weight_to_token_ratio=0.062384 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=29056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.173231337e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=301293568 dispatch_meta_bytes=342544 combine_payload_bytes=301293568 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=301636112 ingress_bytes=335896576 link_bytes=335896576 token_bytes=602929680 weight_movement_bytes=34603008 total_nvlink_bytes=637532688 weight_to_token_ratio=0.057391 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=46592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263951774e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=321126400 dispatch_meta_bytes=328360 combine_payload_bytes=321126400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=321454760 ingress_bytes=355729408 link_bytes=355729408 token_bytes=642581160 weight_movement_bytes=34603008 total_nvlink_bytes=677184168 weight_to_token_ratio=0.053850 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.338581124e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=264192000 dispatch_meta_bytes=351728 combine_payload_bytes=264192000 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=264543728 ingress_bytes=333398016 link_bytes=333398016 token_bytes=528735728 weight_movement_bytes=69206016 total_nvlink_bytes=597941744 weight_to_token_ratio=0.130890 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=41600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.254549893e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=65310 payload_rows_remote=50410 dup_rows=32994 dup_rows_remote=31632 dedup_ratio=0.335632 router_tokens_per_expert=[935, 0, 0, 0, 74, 2, 0, 0, 3, 1052, 0, 0, 103, 0, 7120, 8072, 7222, 5040, 8, 89, 0, 0, 5, 5, 189, 16, 0, 146, 16, 362, 7788, 1700, 248, 155, 4, 534, 10, 5, 0, 1, 4, 0, 88, 9, 2, 145, 251, 22, 0, 0, 0, 0, 1, 0, 6, 0, 10, 0, 7691, 0, 4, 9, 0, 6] payload_rows_per_dst=[14900, 16328, 14968, 16140, 1904, 1016, 12, 42] dup_rows_per_dst=[1362, 12212, 4300, 15066, 10, 26, 2, 16] combine_rows_per_src=[14900, 16328, 14968, 16140, 1904, 1016, 12, 42] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=206479360 dispatch_meta_bytes=328168 combine_payload_bytes=206479360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=206807528 ingress_bytes=241082368 link_bytes=241082368 token_bytes=413286888 weight_movement_bytes=34603008 total_nvlink_bytes=447889896 weight_to_token_ratio=0.083726 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=42112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.071735421e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94182 payload_rows_remote=78228 dup_rows=4122 dup_rows_remote=2510 dedup_ratio=0.041931 router_tokens_per_expert=[875, 0, 0, 0, 64, 0, 0, 0, 0, 899, 0, 0, 95, 0, 7198, 8098, 7384, 5334, 0, 94, 0, 0, 0, 0, 180, 0, 0, 55, 0, 287, 7726, 1679, 251, 153, 0, 653, 0, 0, 0, 0, 0, 0, 94, 0, 0, 111, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7671, 0, 0, 0, 0, 0] payload_rows_per_dst=[14968, 1800, 10668, 4154, 15954, 15100, 16196, 15342] dup_rows_per_dst=[1306, 188, 188, 248, 1612, 580, 0, 0] combine_rows_per_src=[14968, 1800, 10668, 4154, 15954, 15100, 16196, 15342] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74090 payload_rows_remote=60568 dup_rows=24214 dup_rows_remote=19354 dedup_ratio=0.246318 router_tokens_per_expert=[1, 2, 5135, 97, 3115, 5083, 6, 340, 0, 0, 0, 3, 3, 10, 0, 5, 164, 0, 1, 0, 19, 8148, 7, 456, 0, 0, 44, 8172, 1, 5, 0, 4, 297, 5, 1, 192, 0, 10, 103, 412, 340, 13, 26, 5785, 0, 0, 15, 1, 7888, 0, 0, 77, 520, 0, 1, 4, 20, 0, 1027, 461, 927, 111, 7, 88] payload_rows_per_dst=[13522, 10200, 16384, 108, 1942, 11146, 15940, 4848] dup_rows_per_dst=[4860, 8, 17550, 0, 98, 224, 1040, 434] combine_rows_per_src=[13522, 10200, 16384, 108, 1942, 11146, 15940, 4848] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94162 payload_rows_remote=90192 dup_rows=4142 dup_rows_remote=3982 dedup_ratio=0.042135 router_tokens_per_expert=[861, 0, 0, 0, 67, 0, 0, 0, 0, 887, 0, 0, 82, 0, 7229, 8112, 7395, 5383, 0, 80, 0, 0, 0, 0, 201, 0, 0, 10, 0, 289, 7732, 1565, 244, 140, 0, 757, 0, 0, 0, 0, 0, 0, 80, 0, 0, 74, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7719, 0, 0, 0, 0, 0] payload_rows_per_dst=[14982, 1778, 10766, 3970, 16046, 14958, 16224, 15438] dup_rows_per_dst=[1332, 160, 160, 160, 1700, 630, 0, 0] combine_rows_per_src=[14982, 1778, 10766, 3970, 16046, 14958, 16224, 15438] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=326426624 dispatch_meta_bytes=350696 combine_payload_bytes=326426624 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=326777320 ingress_bytes=395632640 link_bytes=395632640 token_bytes=653203944 weight_movement_bytes=69206016 total_nvlink_bytes=722409960 weight_to_token_ratio=0.105949 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=34688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.488733773e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82880 payload_rows_remote=67552 dup_rows=15424 dup_rows_remote=5846 dedup_ratio=0.156901 router_tokens_per_expert=[0, 0, 4533, 86, 3801, 5423, 0, 390, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 0, 0, 0, 8192, 0, 444, 0, 0, 23, 8192, 0, 0, 0, 0, 244, 0, 0, 165, 0, 0, 88, 439, 301, 0, 0, 5844, 0, 0, 0, 0, 7951, 0, 0, 72, 441, 0, 12, 0, 4, 0, 794, 382, 1052, 96, 0, 76] payload_rows_per_dst=[13294, 10846, 15328, 7836, 2924, 12140, 16070, 4442] dup_rows_per_dst=[4326, 0, 9578, 26, 96, 150, 882, 366] combine_rows_per_src=[13294, 10846, 15328, 7836, 2924, 12140, 16070, 4442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94690 payload_rows_remote=79214 dup_rows=3614 dup_rows_remote=3614 dedup_ratio=0.036764 router_tokens_per_expert=[639, 0, 0, 0, 51, 0, 0, 0, 0, 832, 0, 0, 69, 0, 7322, 8129, 7380, 5401, 0, 63, 0, 0, 0, 0, 207, 0, 0, 37, 0, 228, 7963, 1786, 264, 99, 0, 568, 0, 0, 0, 0, 1, 0, 61, 0, 0, 50, 264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7738, 0, 0, 0, 0, 0] payload_rows_per_dst=[1380, 15418, 10802, 4318, 16104, 14934, 16258, 15476] dup_rows_per_dst=[0, 1028, 126, 198, 1684, 578, 0, 0] combine_rows_per_src=[1380, 15418, 10802, 4318, 16104, 14934, 16258, 15476] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 2, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=320421888 dispatch_meta_bytes=322952 combine_payload_bytes=320421888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=320744840 ingress_bytes=355024896 link_bytes=355024896 token_bytes=641166728 weight_movement_bytes=34603008 total_nvlink_bytes=675769736 weight_to_token_ratio=0.053969 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=40832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.335930102e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=369426432 dispatch_meta_bytes=376696 combine_payload_bytes=369426432 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=369803128 ingress_bytes=438632448 link_bytes=438632448 token_bytes=739229560 weight_movement_bytes=69206016 total_nvlink_bytes=808435576 weight_to_token_ratio=0.093619 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.650538589e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=248086528 dispatch_meta_bytes=319688 combine_payload_bytes=248086528 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=248406216 ingress_bytes=421101568 link_bytes=421101568 token_bytes=496492744 weight_movement_bytes=173015040 total_nvlink_bytes=669507784 weight_to_token_ratio=0.348474 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=23936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.584571299e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74774 payload_rows_remote=59682 dup_rows=23530 dup_rows_remote=23530 dedup_ratio=0.239360 router_tokens_per_expert=[0, 0, 0, 303, 0, 0, 0, 0, 0, 0, 6341, 0, 0, 0, 898, 0, 8017, 520, 0, 0, 7805, 0, 0, 0, 1183, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 7806, 0, 0, 0, 7829, 0, 41, 83, 0, 0, 0, 0, 0, 0, 0, 0, 7546, 0, 0, 0, 175, 0, 0, 0, 0, 516, 0, 0] payload_rows_per_dst=[606, 8446, 16102, 2544, 0, 16374, 15092, 15610] dup_rows_per_dst=[0, 412, 6592, 0, 0, 15144, 0, 1382] combine_rows_per_src=[606, 8446, 16102, 2544, 0, 16374, 15092, 15610] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94492 payload_rows_remote=79306 dup_rows=3812 dup_rows_remote=3388 dedup_ratio=0.038778 router_tokens_per_expert=[657, 0, 0, 0, 77, 0, 0, 0, 0, 971, 0, 0, 73, 0, 7270, 8122, 7239, 5472, 0, 70, 0, 0, 0, 0, 214, 0, 0, 133, 0, 292, 7787, 1601, 275, 140, 0, 477, 0, 0, 0, 0, 1, 0, 70, 0, 0, 221, 274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7716, 0, 0, 0, 0, 0] payload_rows_per_dst=[12984, 3996, 10944, 3990, 15716, 15186, 16244, 15432] dup_rows_per_dst=[836, 280, 140, 490, 1642, 424, 0, 0] combine_rows_per_src=[12984, 3996, 10944, 3990, 15716, 15186, 16244, 15432] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92916 payload_rows_remote=82110 dup_rows=5388 dup_rows_remote=5236 dedup_ratio=0.054810 router_tokens_per_expert=[833, 0, 0, 0, 66, 0, 0, 0, 0, 871, 0, 0, 76, 0, 7186, 8116, 7323, 5403, 0, 76, 0, 0, 0, 0, 204, 0, 0, 1, 0, 262, 7704, 1754, 244, 140, 0, 765, 0, 0, 0, 0, 0, 0, 74, 0, 0, 73, 244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7737, 0, 0, 0, 0, 0] payload_rows_per_dst=[14908, 1744, 10806, 13100, 5858, 14794, 16232, 15474] dup_rows_per_dst=[1262, 150, 152, 2836, 354, 634, 0, 0] combine_rows_per_src=[14908, 1744, 10806, 13100, 5858, 14794, 16232, 15474] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94690 payload_rows_remote=79214 dup_rows=3614 dup_rows_remote=3614 dedup_ratio=0.036764 router_tokens_per_expert=[639, 0, 0, 0, 51, 0, 0, 0, 0, 832, 0, 0, 69, 0, 7322, 8129, 7380, 5401, 0, 63, 0, 0, 0, 0, 207, 0, 0, 37, 0, 228, 7963, 1786, 264, 99, 0, 568, 0, 0, 0, 0, 1, 0, 61, 0, 0, 50, 264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7738, 0, 0, 0, 0, 0] payload_rows_per_dst=[1380, 15418, 10802, 4318, 16104, 14934, 16258, 15476] dup_rows_per_dst=[0, 1028, 126, 198, 1684, 578, 0, 0] combine_rows_per_src=[1380, 15418, 10802, 4318, 16104, 14934, 16258, 15476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72384 payload_rows_remote=56824 dup_rows=25920 dup_rows_remote=14532 dedup_ratio=0.263672 router_tokens_per_expert=[768, 0, 0, 0, 75, 0, 0, 0, 0, 975, 0, 0, 85, 0, 7198, 8108, 7330, 5523, 0, 83, 0, 0, 0, 1, 194, 1, 0, 48, 1, 257, 7695, 1559, 253, 153, 1, 686, 0, 0, 0, 0, 2, 0, 81, 0, 0, 116, 251, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7706, 0, 0, 0, 0, 0] payload_rows_per_dst=[14936, 15560, 13126, 16184, 2182, 3832, 922, 5642] dup_rows_per_dst=[1146, 11388, 150, 13096, 4, 136, 0, 0] combine_rows_per_src=[14936, 15560, 13126, 16184, 2182, 3832, 922, 5642] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94614 payload_rows_remote=78400 dup_rows=3690 dup_rows_remote=3690 dedup_ratio=0.037537 router_tokens_per_expert=[904, 0, 0, 0, 97, 0, 0, 0, 0, 881, 0, 0, 87, 0, 7169, 8107, 7303, 5419, 0, 85, 0, 0, 0, 0, 169, 0, 0, 16, 0, 313, 7567, 1679, 260, 175, 0, 765, 0, 0, 0, 0, 0, 0, 85, 0, 0, 90, 260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7721, 0, 0, 0, 0, 0] payload_rows_per_dst=[2002, 15264, 10838, 4080, 15956, 14818, 16214, 15442] dup_rows_per_dst=[0, 1010, 170, 274, 1578, 658, 0, 0] combine_rows_per_src=[2002, 15264, 10838, 4080, 15956, 14818, 16214, 15442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=320421888 dispatch_meta_bytes=322952 combine_payload_bytes=320421888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=320744840 ingress_bytes=355024896 link_bytes=355024896 token_bytes=641166728 weight_movement_bytes=34603008 total_nvlink_bytes=675769736 weight_to_token_ratio=0.053969 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=40832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.335930102e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=369426432 dispatch_meta_bytes=376696 combine_payload_bytes=369426432 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=369803128 ingress_bytes=438632448 link_bytes=438632448 token_bytes=739229560 weight_movement_bytes=69206016 total_nvlink_bytes=808435576 weight_to_token_ratio=0.093619 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.650538589e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=276692992 dispatch_meta_bytes=293592 combine_payload_bytes=276692992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=276986584 ingress_bytes=345899008 link_bytes=345899008 token_bytes=553679576 weight_movement_bytes=69206016 total_nvlink_bytes=622885592 weight_to_token_ratio=0.124993 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=24064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.301590120e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=324460544 dispatch_meta_bytes=331312 combine_payload_bytes=324460544 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=324791856 ingress_bytes=393666560 link_bytes=393666560 token_bytes=649252400 weight_movement_bytes=69206016 total_nvlink_bytes=718458416 weight_to_token_ratio=0.106593 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.481335573e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=244457472 dispatch_meta_bytes=332848 combine_payload_bytes=244457472 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=244790320 ingress_bytes=313663488 link_bytes=313663488 token_bytes=489247792 weight_movement_bytes=69206016 total_nvlink_bytes=558453808 weight_to_token_ratio=0.141454 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.180290453e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79820 payload_rows_remote=63666 dup_rows=18484 dup_rows_remote=18064 dedup_ratio=0.188029 router_tokens_per_expert=[4, 3, 0, 298, 0, 0, 0, 0, 0, 0, 6015, 9, 5, 23, 926, 5, 7982, 522, 17, 0, 7758, 0, 6, 0, 1418, 0, 0, 103, 2, 1, 1, 5, 0, 0, 4, 0, 7771, 2, 0, 1, 7827, 16, 55, 91, 1, 0, 0, 21, 0, 5, 0, 1, 7561, 16, 6, 0, 154, 0, 1, 0, 0, 516, 0, 0] payload_rows_per_dst=[16154, 13944, 16160, 15826, 16020, 334, 42, 1340] dup_rows_per_dst=[420, 3530, 446, 2888, 11150, 34, 14, 2] combine_rows_per_src=[16154, 13944, 16160, 15826, 16020, 334, 42, 1340] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=324837376 dispatch_meta_bytes=330776 combine_payload_bytes=324837376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=325168152 ingress_bytes=359440384 link_bytes=359440384 token_bytes=650005528 weight_movement_bytes=34603008 total_nvlink_bytes=684608536 weight_to_token_ratio=0.053235 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=45568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.352545228e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=336322560 dispatch_meta_bytes=349384 combine_payload_bytes=336322560 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=336671944 ingress_bytes=440131584 link_bytes=440131584 token_bytes=672994504 weight_movement_bytes=103809024 total_nvlink_bytes=776803528 weight_to_token_ratio=0.154249 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=33280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.656179717e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=324460544 dispatch_meta_bytes=331312 combine_payload_bytes=324460544 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=324791856 ingress_bytes=393666560 link_bytes=393666560 token_bytes=649252400 weight_movement_bytes=69206016 total_nvlink_bytes=718458416 weight_to_token_ratio=0.106593 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.481335573e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=232751104 dispatch_meta_bytes=285424 combine_payload_bytes=232751104 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=233036528 ingress_bytes=336560128 link_bytes=336560128 token_bytes=465787632 weight_movement_bytes=103809024 total_nvlink_bytes=569596656 weight_to_token_ratio=0.222868 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=15744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266448666e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=260775936 dispatch_meta_bytes=326920 combine_payload_bytes=260775936 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=261102856 ingress_bytes=295378944 link_bytes=295378944 token_bytes=521878792 weight_movement_bytes=34603008 total_nvlink_bytes=556481800 weight_to_token_ratio=0.066305 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=46976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.111487186e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93688 payload_rows_remote=78156 dup_rows=4616 dup_rows_remote=4616 dedup_ratio=0.046956 router_tokens_per_expert=[0, 0, 0, 285, 0, 0, 0, 0, 0, 0, 6222, 0, 0, 0, 848, 0, 8039, 492, 0, 0, 7765, 0, 0, 0, 1354, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 7766, 0, 0, 0, 7846, 0, 89, 92, 0, 0, 0, 0, 0, 0, 0, 0, 7613, 0, 0, 0, 153, 0, 0, 0, 0, 494, 0, 0] payload_rows_per_dst=[16266, 13448, 984, 13630, 15532, 3072, 15226, 15530] dup_rows_per_dst=[382, 692, 0, 2228, 0, 20, 0, 1294] combine_rows_per_src=[16266, 13448, 984, 13630, 15532, 3072, 15226, 15530] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93204 payload_rows_remote=77364 dup_rows=5100 dup_rows_remote=2424 dedup_ratio=0.051880 router_tokens_per_expert=[0, 0, 0, 296, 0, 0, 0, 0, 0, 0, 6080, 0, 0, 0, 905, 0, 8052, 545, 0, 0, 7786, 0, 0, 0, 1300, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 7799, 0, 0, 0, 7875, 0, 72, 65, 0, 0, 0, 0, 0, 0, 0, 0, 7627, 0, 0, 0, 140, 0, 0, 0, 0, 527, 0, 0] payload_rows_per_dst=[16264, 13312, 1090, 15840, 15598, 274, 15254, 15572] dup_rows_per_dst=[432, 658, 0, 2676, 0, 0, 0, 1334] combine_rows_per_src=[16264, 13312, 1090, 15840, 15598, 274, 15254, 15572] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=65310 payload_rows_remote=50410 dup_rows=32994 dup_rows_remote=31632 dedup_ratio=0.335632 router_tokens_per_expert=[935, 0, 0, 0, 74, 2, 0, 0, 3, 1052, 0, 0, 103, 0, 7120, 8072, 7222, 5040, 8, 89, 0, 0, 5, 5, 189, 16, 0, 146, 16, 362, 7788, 1700, 248, 155, 4, 534, 10, 5, 0, 1, 4, 0, 88, 9, 2, 145, 251, 22, 0, 0, 0, 0, 1, 0, 6, 0, 10, 0, 7691, 0, 4, 9, 0, 6] payload_rows_per_dst=[14900, 16328, 14968, 16140, 1904, 1016, 12, 42] dup_rows_per_dst=[1362, 12212, 4300, 15066, 10, 26, 2, 16] combine_rows_per_src=[14900, 16328, 14968, 16140, 1904, 1016, 12, 42] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=321126400 dispatch_meta_bytes=328360 combine_payload_bytes=321126400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=321454760 ingress_bytes=355729408 link_bytes=355729408 token_bytes=642581160 weight_movement_bytes=34603008 total_nvlink_bytes=677184168 weight_to_token_ratio=0.053850 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.338581124e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92916 payload_rows_remote=82110 dup_rows=5388 dup_rows_remote=5236 dedup_ratio=0.054810 router_tokens_per_expert=[833, 0, 0, 0, 66, 0, 0, 0, 0, 871, 0, 0, 76, 0, 7186, 8116, 7323, 5403, 0, 76, 0, 0, 0, 0, 204, 0, 0, 1, 0, 262, 7704, 1754, 244, 140, 0, 765, 0, 0, 0, 0, 0, 0, 74, 0, 0, 73, 244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7737, 0, 0, 0, 0, 0] payload_rows_per_dst=[14908, 1744, 10806, 13100, 5858, 14794, 16232, 15474] dup_rows_per_dst=[1262, 150, 152, 2836, 354, 634, 0, 0] combine_rows_per_src=[14908, 1744, 10806, 13100, 5858, 14794, 16232, 15474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68408 payload_rows_remote=52772 dup_rows=29896 dup_rows_remote=28684 dedup_ratio=0.304118 router_tokens_per_expert=[0, 0, 0, 253, 0, 0, 0, 0, 0, 0, 6329, 0, 0, 0, 845, 0, 8094, 486, 0, 0, 7818, 0, 0, 0, 1188, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 7801, 0, 0, 0, 7847, 0, 65, 38, 0, 0, 0, 0, 0, 0, 0, 0, 7717, 0, 0, 0, 98, 0, 0, 0, 0, 508, 0, 0] payload_rows_per_dst=[506, 1690, 16314, 2506, 0, 16322, 15434, 15636] dup_rows_per_dst=[0, 0, 13504, 0, 0, 15180, 0, 1212] combine_rows_per_src=[506, 1690, 16314, 2506, 0, 16322, 15434, 15636] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75476 payload_rows_remote=61564 dup_rows=22828 dup_rows_remote=21830 dedup_ratio=0.232218 router_tokens_per_expert=[0, 0, 0, 85, 0, 0, 0, 0, 7852, 0, 0, 0, 0, 0, 0, 0, 2398, 0, 8192, 0, 0, 0, 0, 0, 803, 0, 0, 0, 0, 85, 0, 81, 0, 0, 0, 0, 0, 0, 1977, 0, 390, 744, 2892, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6713, 1, 0, 0, 0, 0, 640, 8107, 0, 0] payload_rows_per_dst=[170, 3662, 4796, 16384, 3954, 16384, 13912, 16214] dup_rows_per_dst=[0, 0, 0, 1938, 0, 9864, 998, 10028] combine_rows_per_src=[170, 3662, 4796, 16384, 3954, 16384, 13912, 16214] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=320126976 dispatch_meta_bytes=331088 combine_payload_bytes=320126976 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=320458064 ingress_bytes=389332992 link_bytes=389332992 token_bytes=640585040 weight_movement_bytes=69206016 total_nvlink_bytes=709791056 weight_to_token_ratio=0.108036 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=39168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.465028705e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=316882944 dispatch_meta_bytes=319152 combine_payload_bytes=316882944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=317202096 ingress_bytes=351485952 link_bytes=351485952 token_bytes=634085040 weight_movement_bytes=34603008 total_nvlink_bytes=668688048 weight_to_token_ratio=0.054572 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=37888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.322613340e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=206479360 dispatch_meta_bytes=328168 combine_payload_bytes=206479360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=206807528 ingress_bytes=241082368 link_bytes=241082368 token_bytes=413286888 weight_movement_bytes=34603008 total_nvlink_bytes=447889896 weight_to_token_ratio=0.083726 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=42112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.071735421e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81048 payload_rows_remote=64720 dup_rows=17256 dup_rows_remote=2586 dedup_ratio=0.175537 router_tokens_per_expert=[0, 0, 0, 305, 0, 0, 0, 0, 0, 0, 5802, 0, 0, 0, 992, 0, 8052, 506, 0, 0, 7795, 0, 0, 0, 1631, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 7787, 0, 0, 0, 7789, 0, 67, 75, 0, 0, 0, 0, 0, 0, 0, 0, 7612, 0, 0, 0, 140, 0, 0, 0, 0, 521, 0, 0] payload_rows_per_dst=[13608, 12886, 3556, 3418, 438, 16328, 15224, 15590] dup_rows_per_dst=[400, 702, 162, 0, 0, 14670, 0, 1322] combine_rows_per_src=[13608, 12886, 3556, 3418, 438, 16328, 15224, 15590] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94038 payload_rows_remote=87290 dup_rows=4266 dup_rows_remote=4150 dedup_ratio=0.043396 router_tokens_per_expert=[0, 0, 0, 244, 0, 0, 0, 0, 0, 0, 6415, 0, 0, 0, 863, 0, 8052, 501, 0, 0, 7795, 0, 0, 0, 1097, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 7812, 0, 0, 0, 7871, 0, 61, 65, 0, 0, 0, 0, 0, 0, 0, 0, 7577, 0, 0, 0, 139, 0, 0, 0, 0, 584, 0, 0] payload_rows_per_dst=[16254, 13916, 6748, 15844, 15624, 252, 15154, 10246] dup_rows_per_dst=[338, 640, 116, 2244, 0, 0, 0, 928] combine_rows_per_src=[16254, 13916, 6748, 15844, 15624, 252, 15154, 10246] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68408 payload_rows_remote=52772 dup_rows=29896 dup_rows_remote=28684 dedup_ratio=0.304118 router_tokens_per_expert=[0, 0, 0, 253, 0, 0, 0, 0, 0, 0, 6329, 0, 0, 0, 845, 0, 8094, 486, 0, 0, 7818, 0, 0, 0, 1188, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 7801, 0, 0, 0, 7847, 0, 65, 38, 0, 0, 0, 0, 0, 0, 0, 0, 7717, 0, 0, 0, 98, 0, 0, 0, 0, 508, 0, 0] payload_rows_per_dst=[506, 1690, 16314, 2506, 0, 16322, 15434, 15636] dup_rows_per_dst=[0, 0, 13504, 0, 0, 15180, 0, 1212] combine_rows_per_src=[506, 1690, 16314, 2506, 0, 16322, 15434, 15636] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86078 payload_rows_remote=72724 dup_rows=12226 dup_rows_remote=11552 dedup_ratio=0.124369 router_tokens_per_expert=[0, 0, 0, 299, 0, 0, 0, 0, 0, 0, 6177, 0, 0, 1, 836, 0, 8038, 485, 1, 0, 7779, 0, 0, 0, 1410, 0, 0, 90, 0, 0, 0, 1, 0, 0, 0, 0, 7789, 0, 0, 0, 7829, 1, 72, 76, 0, 0, 0, 1, 1, 1, 0, 0, 7611, 1, 0, 0, 152, 0, 0, 0, 1, 500, 0, 0] payload_rows_per_dst=[16242, 13354, 16200, 15874, 15946, 298, 6862, 1302] dup_rows_per_dst=[432, 674, 330, 2786, 7996, 2, 2, 4] combine_rows_per_src=[16242, 13354, 16200, 15874, 15946, 298, 6862, 1302] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74774 payload_rows_remote=59682 dup_rows=23530 dup_rows_remote=23530 dedup_ratio=0.239360 router_tokens_per_expert=[0, 0, 0, 303, 0, 0, 0, 0, 0, 0, 6341, 0, 0, 0, 898, 0, 8017, 520, 0, 0, 7805, 0, 0, 0, 1183, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 7806, 0, 0, 0, 7829, 0, 41, 83, 0, 0, 0, 0, 0, 0, 0, 0, 7546, 0, 0, 0, 175, 0, 0, 0, 0, 516, 0, 0] payload_rows_per_dst=[606, 8446, 16102, 2544, 0, 16374, 15092, 15610] dup_rows_per_dst=[0, 412, 6592, 0, 0, 15144, 0, 1382] combine_rows_per_src=[606, 8446, 16102, 2544, 0, 16374, 15092, 15610] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=320126976 dispatch_meta_bytes=331088 combine_payload_bytes=320126976 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=320458064 ingress_bytes=389332992 link_bytes=389332992 token_bytes=640585040 weight_movement_bytes=69206016 total_nvlink_bytes=709791056 weight_to_token_ratio=0.108036 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=39168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.465028705e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=316882944 dispatch_meta_bytes=319152 combine_payload_bytes=316882944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=317202096 ingress_bytes=351485952 link_bytes=351485952 token_bytes=634085040 weight_movement_bytes=34603008 total_nvlink_bytes=668688048 weight_to_token_ratio=0.054572 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=37888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.322613340e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=336322560 dispatch_meta_bytes=349384 combine_payload_bytes=336322560 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=336671944 ingress_bytes=440131584 link_bytes=440131584 token_bytes=672994504 weight_movement_bytes=103809024 total_nvlink_bytes=776803528 weight_to_token_ratio=0.154249 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=33280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.656179717e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=216154112 dispatch_meta_bytes=325824 combine_payload_bytes=216154112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=216479936 ingress_bytes=250757120 link_bytes=250757120 token_bytes=432634048 weight_movement_bytes=34603008 total_nvlink_bytes=467237056 weight_to_token_ratio=0.079982 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=43904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.435788550e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76168 payload_rows_remote=59790 dup_rows=22136 dup_rows_remote=21890 dedup_ratio=0.225179 router_tokens_per_expert=[0, 0, 24, 87, 3, 14, 19, 3, 7765, 0, 0, 0, 3, 3, 0, 1, 2129, 0, 8169, 6, 3, 1, 12, 0, 844, 0, 0, 10, 1, 88, 0, 90, 15, 0, 0, 0, 0, 7, 2265, 0, 410, 652, 3317, 8162, 0, 13, 0, 0, 5, 3, 0, 0, 0, 6, 6308, 9, 6, 20, 0, 9, 607, 8056, 0, 7] payload_rows_per_dst=[16378, 15544, 16370, 2056, 16146, 8376, 42, 1256] dup_rows_per_dst=[246, 0, 16886, 10, 4540, 408, 4, 42] combine_rows_per_src=[16378, 15544, 16370, 2056, 16146, 8376, 42, 1256] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=252166144 dispatch_meta_bytes=333576 combine_payload_bytes=252166144 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=252499720 ingress_bytes=321372160 link_bytes=321372160 token_bytes=504665864 weight_movement_bytes=69206016 total_nvlink_bytes=573871880 weight_to_token_ratio=0.137132 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=45312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.209297565e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265093120 dispatch_meta_bytes=269224 combine_payload_bytes=265093120 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265362344 ingress_bytes=334299136 link_bytes=334299136 token_bytes=530455464 weight_movement_bytes=69206016 total_nvlink_bytes=599661480 weight_to_token_ratio=0.130465 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=24832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257940735e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=357539840 dispatch_meta_bytes=365760 combine_payload_bytes=357539840 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=357905600 ingress_bytes=461348864 link_bytes=461348864 token_bytes=715445440 weight_movement_bytes=103809024 total_nvlink_bytes=819254464 weight_to_token_ratio=0.145097 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.736018634e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92152 payload_rows_remote=75768 dup_rows=6152 dup_rows_remote=4428 dedup_ratio=0.062581 router_tokens_per_expert=[0, 0, 0, 80, 0, 0, 0, 0, 8006, 0, 0, 0, 0, 0, 0, 0, 2321, 0, 8192, 0, 0, 0, 0, 0, 713, 0, 0, 0, 0, 80, 0, 69, 0, 0, 0, 0, 0, 0, 2058, 0, 292, 788, 3181, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6465, 1, 0, 0, 0, 0, 602, 8112, 0, 0] payload_rows_per_dst=[16384, 16012, 4642, 16384, 16224, 8372, 12930, 1204] dup_rows_per_dst=[160, 0, 0, 1724, 4116, 150, 2, 0] combine_rows_per_src=[16384, 16012, 4642, 16384, 16224, 8372, 12930, 1204] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94776 payload_rows_remote=90028 dup_rows=3528 dup_rows_remote=3288 dedup_ratio=0.035889 router_tokens_per_expert=[0, 0, 0, 94, 0, 0, 0, 0, 7931, 0, 0, 0, 0, 0, 0, 0, 2310, 0, 8192, 0, 0, 0, 0, 0, 776, 0, 0, 0, 0, 94, 0, 63, 0, 0, 0, 0, 0, 0, 1995, 0, 342, 690, 3259, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6546, 7, 0, 0, 0, 0, 563, 8098, 0, 0] payload_rows_per_dst=[16384, 15862, 4620, 16384, 4748, 8406, 13104, 15268] dup_rows_per_dst=[188, 0, 0, 1866, 240, 176, 2, 1056] combine_rows_per_src=[16384, 15862, 4620, 16384, 4748, 8406, 13104, 15268] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=216154112 dispatch_meta_bytes=325824 combine_payload_bytes=216154112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=216479936 ingress_bytes=250757120 link_bytes=250757120 token_bytes=432634048 weight_movement_bytes=34603008 total_nvlink_bytes=467237056 weight_to_token_ratio=0.079982 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=43904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.435788550e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=297877504 dispatch_meta_bytes=337104 combine_payload_bytes=297877504 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=298214608 ingress_bytes=367083520 link_bytes=367083520 token_bytes=596092112 weight_movement_bytes=69206016 total_nvlink_bytes=665298128 weight_to_token_ratio=0.116100 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=40320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.381305733e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79820 payload_rows_remote=63666 dup_rows=18484 dup_rows_remote=18064 dedup_ratio=0.188029 router_tokens_per_expert=[4, 3, 0, 298, 0, 0, 0, 0, 0, 0, 6015, 9, 5, 23, 926, 5, 7982, 522, 17, 0, 7758, 0, 6, 0, 1418, 0, 0, 103, 2, 1, 1, 5, 0, 0, 4, 0, 7771, 2, 0, 1, 7827, 16, 55, 91, 1, 0, 0, 21, 0, 5, 0, 1, 7561, 16, 6, 0, 154, 0, 1, 0, 0, 516, 0, 0] payload_rows_per_dst=[16154, 13944, 16160, 15826, 16020, 334, 42, 1340] dup_rows_per_dst=[420, 3530, 446, 2888, 11150, 34, 14, 2] combine_rows_per_src=[16154, 13944, 16160, 15826, 16020, 334, 42, 1340] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=244457472 dispatch_meta_bytes=332848 combine_payload_bytes=244457472 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=244790320 ingress_bytes=313663488 link_bytes=313663488 token_bytes=489247792 weight_movement_bytes=69206016 total_nvlink_bytes=558453808 weight_to_token_ratio=0.141454 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.180290453e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=244899840 dispatch_meta_bytes=326720 combine_payload_bytes=244899840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=245226560 ingress_bytes=279502848 link_bytes=279502848 token_bytes=490126400 weight_movement_bytes=34603008 total_nvlink_bytes=524729408 weight_to_token_ratio=0.070600 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=48512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.051746715e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94776 payload_rows_remote=90028 dup_rows=3528 dup_rows_remote=3288 dedup_ratio=0.035889 router_tokens_per_expert=[0, 0, 0, 94, 0, 0, 0, 0, 7931, 0, 0, 0, 0, 0, 0, 0, 2310, 0, 8192, 0, 0, 0, 0, 0, 776, 0, 0, 0, 0, 94, 0, 63, 0, 0, 0, 0, 0, 0, 1995, 0, 342, 690, 3259, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6546, 7, 0, 0, 0, 0, 563, 8098, 0, 0] payload_rows_per_dst=[16384, 15862, 4620, 16384, 4748, 8406, 13104, 15268] dup_rows_per_dst=[188, 0, 0, 1866, 240, 176, 2, 1056] combine_rows_per_src=[16384, 15862, 4620, 16384, 4748, 8406, 13104, 15268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92152 payload_rows_remote=75768 dup_rows=6152 dup_rows_remote=4428 dedup_ratio=0.062581 router_tokens_per_expert=[0, 0, 0, 80, 0, 0, 0, 0, 8006, 0, 0, 0, 0, 0, 0, 0, 2321, 0, 8192, 0, 0, 0, 0, 0, 713, 0, 0, 0, 0, 80, 0, 69, 0, 0, 0, 0, 0, 0, 2058, 0, 292, 788, 3181, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6465, 1, 0, 0, 0, 0, 602, 8112, 0, 0] payload_rows_per_dst=[16384, 16012, 4642, 16384, 16224, 8372, 12930, 1204] dup_rows_per_dst=[160, 0, 0, 1724, 4116, 150, 2, 0] combine_rows_per_src=[16384, 16012, 4642, 16384, 16224, 8372, 12930, 1204] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94038 payload_rows_remote=87290 dup_rows=4266 dup_rows_remote=4150 dedup_ratio=0.043396 router_tokens_per_expert=[0, 0, 0, 244, 0, 0, 0, 0, 0, 0, 6415, 0, 0, 0, 863, 0, 8052, 501, 0, 0, 7795, 0, 0, 0, 1097, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 7812, 0, 0, 0, 7871, 0, 61, 65, 0, 0, 0, 0, 0, 0, 0, 0, 7577, 0, 0, 0, 139, 0, 0, 0, 0, 584, 0, 0] payload_rows_per_dst=[16254, 13916, 6748, 15844, 15624, 252, 15154, 10246] dup_rows_per_dst=[338, 640, 116, 2244, 0, 0, 0, 928] combine_rows_per_src=[16254, 13916, 6748, 15844, 15624, 252, 15154, 10246] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73654 payload_rows_remote=57396 dup_rows=24650 dup_rows_remote=7688 dedup_ratio=0.250753 router_tokens_per_expert=[0, 0, 0, 63, 0, 3, 0, 0, 7910, 0, 0, 0, 0, 0, 0, 0, 2430, 0, 8191, 0, 0, 0, 0, 0, 717, 0, 0, 0, 0, 63, 0, 52, 0, 0, 0, 0, 0, 0, 2083, 0, 326, 597, 3171, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6649, 5, 0, 0, 0, 0, 571, 8129, 0, 0] payload_rows_per_dst=[132, 0, 4860, 16382, 4166, 16384, 15472, 16258] dup_rows_per_dst=[0, 0, 0, 1664, 0, 1846, 4178, 16962] combine_rows_per_src=[132, 0, 4860, 16382, 4166, 16384, 15472, 16258] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=310345728 dispatch_meta_bytes=320784 combine_payload_bytes=310345728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=310666512 ingress_bytes=344948736 link_bytes=344948736 token_bytes=621012240 weight_movement_bytes=34603008 total_nvlink_bytes=655615248 weight_to_token_ratio=0.055720 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=41984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.298014323e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=368754688 dispatch_meta_bytes=373264 combine_payload_bytes=368754688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=369127952 ingress_bytes=403357696 link_bytes=403357696 token_bytes=737882640 weight_movement_bytes=34603008 total_nvlink_bytes=772485648 weight_to_token_ratio=0.046895 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=32896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.517802537e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=260775936 dispatch_meta_bytes=326920 combine_payload_bytes=260775936 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=261102856 ingress_bytes=295378944 link_bytes=295378944 token_bytes=521878792 weight_movement_bytes=34603008 total_nvlink_bytes=556481800 weight_to_token_ratio=0.066305 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=46976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.111487186e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73700 payload_rows_remote=59302 dup_rows=24604 dup_rows_remote=20518 dedup_ratio=0.250285 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 65, 0, 0, 205, 0, 1415, 0, 0, 0, 0, 0, 7922, 0, 0, 0, 5935, 0, 0, 0, 0, 8022, 0, 0, 0, 3, 4862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5615, 8192, 1650, 1918, 0, 0, 0, 0, 606, 0, 2701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41] payload_rows_per_dst=[130, 8588, 0, 16044, 5664, 13032, 14398, 15844] dup_rows_per_dst=[0, 736, 0, 9730, 0, 9970, 4086, 82] combine_rows_per_src=[130, 8588, 0, 16044, 5664, 13032, 14398, 15844] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94120 payload_rows_remote=84918 dup_rows=4184 dup_rows_remote=3202 dedup_ratio=0.042562 router_tokens_per_expert=[0, 0, 0, 69, 1, 0, 0, 0, 7886, 0, 0, 0, 0, 0, 0, 0, 2321, 0, 8191, 0, 0, 0, 0, 0, 727, 0, 0, 0, 0, 70, 0, 107, 0, 0, 0, 0, 0, 0, 2036, 0, 382, 599, 3435, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6378, 5, 0, 0, 0, 0, 631, 8122, 0, 0] payload_rows_per_dst=[15044, 15772, 4642, 16384, 4072, 9202, 12760, 16244] dup_rows_per_dst=[128, 0, 0, 1806, 0, 982, 6, 1262] combine_rows_per_src=[15044, 15772, 4642, 16384, 4072, 9202, 12760, 16244] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89636 payload_rows_remote=74658 dup_rows=8668 dup_rows_remote=4586 dedup_ratio=0.088175 router_tokens_per_expert=[0, 0, 0, 76, 0, 0, 0, 0, 7855, 0, 0, 0, 0, 0, 0, 0, 2310, 0, 8192, 0, 0, 0, 0, 0, 713, 0, 0, 0, 0, 76, 0, 67, 0, 0, 0, 0, 0, 0, 1913, 0, 452, 636, 3304, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6574, 6, 0, 0, 0, 0, 669, 8117, 0, 0] payload_rows_per_dst=[16384, 15710, 14978, 3420, 16234, 8420, 13152, 1338] dup_rows_per_dst=[152, 0, 4082, 236, 3826, 364, 8, 0] combine_rows_per_src=[16384, 15710, 14978, 3420, 16234, 8420, 13152, 1338] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73654 payload_rows_remote=57396 dup_rows=24650 dup_rows_remote=7688 dedup_ratio=0.250753 router_tokens_per_expert=[0, 0, 0, 63, 0, 3, 0, 0, 7910, 0, 0, 0, 0, 0, 0, 0, 2430, 0, 8191, 0, 0, 0, 0, 0, 717, 0, 0, 0, 0, 63, 0, 52, 0, 0, 0, 0, 0, 0, 2083, 0, 326, 597, 3171, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6649, 5, 0, 0, 0, 0, 571, 8129, 0, 0] payload_rows_per_dst=[132, 0, 4860, 16382, 4166, 16384, 15472, 16258] dup_rows_per_dst=[0, 0, 0, 1664, 0, 1846, 4178, 16962] combine_rows_per_src=[132, 0, 4860, 16382, 4166, 16384, 15472, 16258] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75476 payload_rows_remote=61564 dup_rows=22828 dup_rows_remote=21830 dedup_ratio=0.232218 router_tokens_per_expert=[0, 0, 0, 85, 0, 0, 0, 0, 7852, 0, 0, 0, 0, 0, 0, 0, 2398, 0, 8192, 0, 0, 0, 0, 0, 803, 0, 0, 0, 0, 85, 0, 81, 0, 0, 0, 0, 0, 0, 1977, 0, 390, 744, 2892, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6713, 1, 0, 0, 0, 0, 640, 8107, 0, 0] payload_rows_per_dst=[170, 3662, 4796, 16384, 3954, 16384, 13912, 16214] dup_rows_per_dst=[0, 0, 0, 1938, 0, 9864, 998, 10028] combine_rows_per_src=[170, 3662, 4796, 16384, 3954, 16384, 13912, 16214] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=368754688 dispatch_meta_bytes=373264 combine_payload_bytes=368754688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=369127952 ingress_bytes=403357696 link_bytes=403357696 token_bytes=737882640 weight_movement_bytes=34603008 total_nvlink_bytes=772485648 weight_to_token_ratio=0.046895 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=32896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.517802537e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87762 payload_rows_remote=72022 dup_rows=10542 dup_rows_remote=10540 dedup_ratio=0.107239 router_tokens_per_expert=[1, 0, 1, 83, 1, 0, 0, 0, 7870, 0, 0, 0, 0, 0, 0, 1, 2377, 0, 8191, 0, 0, 0, 1, 0, 666, 0, 0, 0, 0, 83, 0, 84, 1, 0, 0, 0, 0, 1, 1910, 0, 558, 678, 3248, 8191, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6505, 7, 0, 1, 0, 0, 585, 8107, 0, 0] payload_rows_per_dst=[16384, 15740, 16384, 1666, 16216, 8480, 11720, 1172] dup_rows_per_dst=[170, 2, 6044, 0, 3822, 490, 14, 0] combine_rows_per_src=[16384, 15740, 16384, 1666, 16216, 8480, 11720, 1172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=310345728 dispatch_meta_bytes=320784 combine_payload_bytes=310345728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=310666512 ingress_bytes=344948736 link_bytes=344948736 token_bytes=621012240 weight_movement_bytes=34603008 total_nvlink_bytes=655615248 weight_to_token_ratio=0.055720 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=41984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.298014323e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73750 payload_rows_remote=57788 dup_rows=24554 dup_rows_remote=24292 dedup_ratio=0.249776 router_tokens_per_expert=[2, 1, 1, 0, 1, 0, 133, 0, 14, 219, 6, 1429, 1, 0, 0, 0, 8, 7914, 0, 0, 1, 5822, 0, 5, 2, 5, 7974, 1, 0, 4, 45, 5287, 25, 30, 3, 0, 17, 0, 0, 0, 8, 18, 5, 5557, 8163, 1520, 2035, 6, 1, 0, 5, 546, 0, 2253, 0, 0, 0, 0, 0, 27, 0, 2, 0, 56] payload_rows_per_dst=[15962, 13164, 16356, 5818, 10668, 6270, 5346, 166] dup_rows_per_dst=[262, 1288, 21718, 56, 48, 914, 264, 4] combine_rows_per_src=[15962, 13164, 16356, 5818, 10668, 6270, 5346, 166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=357539840 dispatch_meta_bytes=365760 combine_payload_bytes=357539840 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=357905600 ingress_bytes=461348864 link_bytes=461348864 token_bytes=715445440 weight_movement_bytes=103809024 total_nvlink_bytes=819254464 weight_to_token_ratio=0.145097 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.736018634e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=235094016 dispatch_meta_bytes=260336 combine_payload_bytes=235094016 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=235354352 ingress_bytes=304300032 link_bytes=304300032 token_bytes=470448368 weight_movement_bytes=69206016 total_nvlink_bytes=539654384 weight_to_token_ratio=0.147107 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=32000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.145056522e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=242900992 dispatch_meta_bytes=319280 combine_payload_bytes=242900992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=243220272 ingress_bytes=277504000 link_bytes=277504000 token_bytes=486121264 weight_movement_bytes=34603008 total_nvlink_bytes=520724272 weight_to_token_ratio=0.071182 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=24576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.044225211e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=347824128 dispatch_meta_bytes=352480 combine_payload_bytes=347824128 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=348176608 ingress_bytes=451633152 link_bytes=451633152 token_bytes=696000736 weight_movement_bytes=103809024 total_nvlink_bytes=799809760 weight_to_token_ratio=0.149151 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=22016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.699459192e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=295002112 dispatch_meta_bytes=330248 combine_payload_bytes=295002112 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=295332360 ingress_bytes=364208128 link_bytes=364208128 token_bytes=590334472 weight_movement_bytes=69206016 total_nvlink_bytes=659540488 weight_to_token_ratio=0.117232 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.370485865e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=236699648 dispatch_meta_bytes=328320 combine_payload_bytes=236699648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=237027968 ingress_bytes=271302656 link_bytes=271302656 token_bytes=473727616 weight_movement_bytes=34603008 total_nvlink_bytes=508330624 weight_to_token_ratio=0.073044 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=48128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.020890053e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91712 payload_rows_remote=75328 dup_rows=6592 dup_rows_remote=6592 dedup_ratio=0.067057 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 106, 0, 0, 235, 0, 1381, 0, 0, 0, 0, 0, 7939, 0, 0, 0, 5964, 0, 0, 0, 0, 8004, 0, 0, 0, 3, 4910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5714, 8192, 1562, 1884, 0, 0, 0, 0, 586, 0, 2591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81] payload_rows_per_dst=[16008, 13254, 0, 9826, 16384, 6058, 14304, 15878] dup_rows_per_dst=[212, 1406, 0, 0, 0, 834, 3978, 162] combine_rows_per_src=[16008, 13254, 0, 9826, 16384, 6058, 14304, 15878] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=305799168 dispatch_meta_bytes=316976 combine_payload_bytes=305799168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=306116144 ingress_bytes=375005184 link_bytes=375005184 token_bytes=611915312 weight_movement_bytes=69206016 total_nvlink_bytes=681121328 weight_to_token_ratio=0.113097 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.411114317e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95250 payload_rows_remote=85042 dup_rows=3054 dup_rows_remote=3054 dedup_ratio=0.031067 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 0, 0, 256, 0, 1175, 0, 0, 0, 0, 0, 7947, 0, 0, 0, 6096, 0, 0, 0, 0, 8032, 0, 0, 0, 10, 5094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5602, 8192, 1491, 1854, 0, 0, 0, 0, 475, 0, 2731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69] payload_rows_per_dst=[16064, 13048, 11068, 10208, 16384, 5716, 6868, 15894] dup_rows_per_dst=[256, 1018, 0, 0, 0, 974, 668, 138] combine_rows_per_src=[16064, 13048, 11068, 10208, 16384, 5716, 6868, 15894] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=235094016 dispatch_meta_bytes=260336 combine_payload_bytes=235094016 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=235354352 ingress_bytes=304300032 link_bytes=304300032 token_bytes=470448368 weight_movement_bytes=69206016 total_nvlink_bytes=539654384 weight_to_token_ratio=0.147107 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=32000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.145056522e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=252166144 dispatch_meta_bytes=333576 combine_payload_bytes=252166144 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=252499720 ingress_bytes=321372160 link_bytes=321372160 token_bytes=504665864 weight_movement_bytes=69206016 total_nvlink_bytes=573871880 weight_to_token_ratio=0.137132 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=45312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.209297565e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76168 payload_rows_remote=59790 dup_rows=22136 dup_rows_remote=21890 dedup_ratio=0.225179 router_tokens_per_expert=[0, 0, 24, 87, 3, 14, 19, 3, 7765, 0, 0, 0, 3, 3, 0, 1, 2129, 0, 8169, 6, 3, 1, 12, 0, 844, 0, 0, 10, 1, 88, 0, 90, 15, 0, 0, 0, 0, 7, 2265, 0, 410, 652, 3317, 8162, 0, 13, 0, 0, 5, 3, 0, 0, 0, 6, 6308, 9, 6, 20, 0, 9, 607, 8056, 0, 7] payload_rows_per_dst=[16378, 15544, 16370, 2056, 16146, 8376, 42, 1256] dup_rows_per_dst=[246, 0, 16886, 10, 4540, 408, 4, 42] combine_rows_per_src=[16378, 15544, 16370, 2056, 16146, 8376, 42, 1256] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=295002112 dispatch_meta_bytes=330248 combine_payload_bytes=295002112 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=295332360 ingress_bytes=364208128 link_bytes=364208128 token_bytes=590334472 weight_movement_bytes=69206016 total_nvlink_bytes=659540488 weight_to_token_ratio=0.117232 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.370485865e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89636 payload_rows_remote=74658 dup_rows=8668 dup_rows_remote=4586 dedup_ratio=0.088175 router_tokens_per_expert=[0, 0, 0, 76, 0, 0, 0, 0, 7855, 0, 0, 0, 0, 0, 0, 0, 2310, 0, 8192, 0, 0, 0, 0, 0, 713, 0, 0, 0, 0, 76, 0, 67, 0, 0, 0, 0, 0, 0, 1913, 0, 452, 636, 3304, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6574, 6, 0, 0, 0, 0, 669, 8117, 0, 0] payload_rows_per_dst=[16384, 15710, 14978, 3420, 16234, 8420, 13152, 1338] dup_rows_per_dst=[152, 0, 4082, 236, 3826, 364, 8, 0] combine_rows_per_src=[16384, 15710, 14978, 3420, 16234, 8420, 13152, 1338] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=66538 payload_rows_remote=50690 dup_rows=31766 dup_rows_remote=31638 dedup_ratio=0.323140 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 202, 0, 0, 235, 0, 1500, 0, 0, 0, 0, 0, 7924, 0, 0, 0, 6185, 0, 0, 0, 0, 8066, 0, 0, 0, 7, 5048, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5408, 8192, 1543, 1904, 0, 0, 0, 0, 458, 0, 2415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64] payload_rows_per_dst=[404, 3470, 0, 16132, 2, 16384, 14298, 15848] dup_rows_per_dst=[0, 0, 0, 10110, 0, 17710, 3818, 128] combine_rows_per_src=[404, 3470, 0, 16132, 2, 16384, 14298, 15848] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87146 payload_rows_remote=87146 dup_rows=11158 dup_rows_remote=11158 dedup_ratio=0.113505 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5313, 0, 0, 6707, 0, 0, 0, 0, 0, 6756, 0, 0, 0, 0, 0, 221, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 318, 0, 0, 3489, 8192, 0, 0] payload_rows_per_dst=[13414, 10626, 13512, 442, 16384, 16384, 0, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 3544, 0, 7614] combine_rows_per_src=[13414, 10626, 13512, 442, 16384, 16384, 0, 16384] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=308543488 dispatch_meta_bytes=327680 combine_payload_bytes=308543488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=308871168 ingress_bytes=343146496 link_bytes=343146496 token_bytes=617414656 weight_movement_bytes=34603008 total_nvlink_bytes=652017664 weight_to_token_ratio=0.056045 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.291232639e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=348332032 dispatch_meta_bytes=352384 combine_payload_bytes=348332032 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=348684416 ingress_bytes=452141056 link_bytes=452141056 token_bytes=697016448 weight_movement_bytes=103809024 total_nvlink_bytes=800825472 weight_to_token_ratio=0.148933 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=30080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.701370393e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=244899840 dispatch_meta_bytes=326720 combine_payload_bytes=244899840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=245226560 ingress_bytes=279502848 link_bytes=279502848 token_bytes=490126400 weight_movement_bytes=34603008 total_nvlink_bytes=524729408 weight_to_token_ratio=0.070600 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=48512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.051746715e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91944 payload_rows_remote=85966 dup_rows=6360 dup_rows_remote=5410 dedup_ratio=0.064697 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 87, 0, 0, 242, 0, 1407, 0, 0, 0, 0, 0, 7906, 0, 0, 0, 5819, 0, 0, 0, 0, 8052, 0, 0, 0, 7, 5427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5332, 8192, 1482, 1982, 0, 0, 0, 0, 568, 0, 2588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61] payload_rows_per_dst=[15960, 12770, 0, 10916, 16384, 5978, 14124, 15812] dup_rows_per_dst=[174, 1192, 0, 96, 0, 950, 3826, 122] combine_rows_per_src=[15960, 12770, 0, 10916, 16384, 5978, 14124, 15812] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95302 payload_rows_remote=82488 dup_rows=3002 dup_rows_remote=2766 dedup_ratio=0.030538 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 209, 0, 0, 246, 0, 1451, 0, 0, 0, 0, 0, 7946, 0, 0, 0, 6376, 0, 0, 0, 0, 8040, 0, 0, 0, 9, 4820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5518, 8192, 1587, 1802, 0, 0, 0, 0, 510, 0, 2385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61] payload_rows_per_dst=[16080, 13362, 12814, 9360, 16384, 5880, 5530, 15892] dup_rows_per_dst=[418, 1068, 236, 0, 0, 898, 260, 122] combine_rows_per_src=[16080, 13362, 12814, 9360, 16384, 5880, 5530, 15892] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=66538 payload_rows_remote=50690 dup_rows=31766 dup_rows_remote=31638 dedup_ratio=0.323140 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 202, 0, 0, 235, 0, 1500, 0, 0, 0, 0, 0, 7924, 0, 0, 0, 6185, 0, 0, 0, 0, 8066, 0, 0, 0, 7, 5048, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5408, 8192, 1543, 1904, 0, 0, 0, 0, 458, 0, 2415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64] payload_rows_per_dst=[404, 3470, 0, 16132, 2, 16384, 14298, 15848] dup_rows_per_dst=[0, 0, 0, 10110, 0, 17710, 3818, 128] combine_rows_per_src=[404, 3470, 0, 16132, 2, 16384, 14298, 15848] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73700 payload_rows_remote=59302 dup_rows=24604 dup_rows_remote=20518 dedup_ratio=0.250285 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 65, 0, 0, 205, 0, 1415, 0, 0, 0, 0, 0, 7922, 0, 0, 0, 5935, 0, 0, 0, 0, 8022, 0, 0, 0, 3, 4862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5615, 8192, 1650, 1918, 0, 0, 0, 0, 606, 0, 2701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41] payload_rows_per_dst=[130, 8588, 0, 16044, 5664, 13032, 14398, 15844] dup_rows_per_dst=[0, 736, 0, 9730, 0, 9970, 4086, 82] combine_rows_per_src=[130, 8588, 0, 16044, 5664, 13032, 14398, 15844] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=348332032 dispatch_meta_bytes=352384 combine_payload_bytes=348332032 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=348684416 ingress_bytes=452141056 link_bytes=452141056 token_bytes=697016448 weight_movement_bytes=103809024 total_nvlink_bytes=800825472 weight_to_token_ratio=0.148933 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=30080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.701370393e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75790 payload_rows_remote=62926 dup_rows=22514 dup_rows_remote=21438 dedup_ratio=0.229024 router_tokens_per_expert=[2, 0, 0, 0, 0, 0, 161, 0, 0, 236, 0, 1417, 0, 0, 0, 0, 0, 7939, 0, 0, 0, 5908, 0, 1, 1, 0, 8024, 0, 0, 0, 3, 5243, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 5317, 8191, 1439, 1928, 0, 0, 0, 0, 563, 0, 2705, 0, 0, 0, 1, 0, 2, 0, 0, 0, 68] payload_rows_per_dst=[16052, 12864, 16176, 8, 16384, 5922, 6196, 2188] dup_rows_per_dst=[322, 1076, 19936, 0, 2, 814, 340, 24] combine_rows_per_src=[16052, 12864, 16176, 8, 16384, 5922, 6196, 2188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94854 payload_rows_remote=78470 dup_rows=3450 dup_rows_remote=3450 dedup_ratio=0.035095 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5425, 0, 0, 7074, 0, 0, 0, 0, 0, 6831, 0, 0, 0, 0, 0, 175, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 335, 0, 0, 3011, 8192, 0, 0] payload_rows_per_dst=[14148, 10850, 13662, 350, 16384, 16384, 16384, 6692] dup_rows_per_dst=[0, 0, 0, 0, 0, 3450, 0, 0] combine_rows_per_src=[14148, 10850, 13662, 350, 16384, 16384, 16384, 6692] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=352116736 dispatch_meta_bytes=365504 combine_payload_bytes=352116736 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=352482240 ingress_bytes=455925760 link_bytes=455925760 token_bytes=704598976 weight_movement_bytes=103809024 total_nvlink_bytes=808408000 weight_to_token_ratio=0.147331 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=15360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.715611930e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=337870848 dispatch_meta_bytes=341016 combine_payload_bytes=337870848 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=338211864 ingress_bytes=441679872 link_bytes=441679872 token_bytes=676082712 weight_movement_bytes=103809024 total_nvlink_bytes=779891736 weight_to_token_ratio=0.153545 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.662005800e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81344 payload_rows_remote=67834 dup_rows=16960 dup_rows_remote=16956 dedup_ratio=0.172526 router_tokens_per_expert=[1, 4, 0, 0, 1, 3, 3, 0, 0, 0, 3, 0, 5358, 10, 27, 6745, 3, 0, 2, 2, 10, 6702, 1, 6, 0, 0, 0, 199, 0, 27, 8162, 0, 0, 0, 1, 0, 0, 8162, 2, 3, 0, 0, 1802, 0, 0, 28, 27, 0, 11, 35, 0, 0, 0, 3, 14, 4, 8, 340, 0, 3, 3271, 8165, 4, 0] payload_rows_per_dst=[13510, 10868, 13274, 16380, 12, 3684, 16366, 7250] dup_rows_per_dst=[4, 102, 4, 16720, 0, 30, 98, 2] combine_rows_per_src=[13510, 10868, 13274, 16380, 12, 3684, 16366, 7250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=305799168 dispatch_meta_bytes=316976 combine_payload_bytes=305799168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=306116144 ingress_bytes=375005184 link_bytes=375005184 token_bytes=611915312 weight_movement_bytes=69206016 total_nvlink_bytes=681121328 weight_to_token_ratio=0.113097 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.411114317e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=207626240 dispatch_meta_bytes=329312 combine_payload_bytes=207626240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=207955552 ingress_bytes=242229248 link_bytes=242229248 token_bytes=415581792 weight_movement_bytes=34603008 total_nvlink_bytes=450184800 weight_to_token_ratio=0.083264 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=48640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.114891592e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=356950016 dispatch_meta_bytes=393216 combine_payload_bytes=356950016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=357343232 ingress_bytes=391553024 link_bytes=391553024 token_bytes=714293248 weight_movement_bytes=34603008 total_nvlink_bytes=748896256 weight_to_token_ratio=0.048444 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.473382507e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=352116736 dispatch_meta_bytes=365504 combine_payload_bytes=352116736 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=352482240 ingress_bytes=455925760 link_bytes=455925760 token_bytes=704598976 weight_movement_bytes=103809024 total_nvlink_bytes=808408000 weight_to_token_ratio=0.147331 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=15360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.715611930e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=257744896 dispatch_meta_bytes=337456 combine_payload_bytes=257744896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258082352 ingress_bytes=292347904 link_bytes=292347904 token_bytes=515827248 weight_movement_bytes=34603008 total_nvlink_bytes=550430256 weight_to_token_ratio=0.067083 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=36096 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.100081627e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=277848064 dispatch_meta_bytes=339160 combine_payload_bytes=277848064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278187224 ingress_bytes=312451072 link_bytes=312451072 token_bytes=556035288 weight_movement_bytes=34603008 total_nvlink_bytes=590638296 weight_to_token_ratio=0.062232 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.175728230e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94854 payload_rows_remote=78470 dup_rows=3450 dup_rows_remote=3450 dedup_ratio=0.035095 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5425, 0, 0, 7074, 0, 0, 0, 0, 0, 6831, 0, 0, 0, 0, 0, 175, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 335, 0, 0, 3011, 8192, 0, 0] payload_rows_per_dst=[14148, 10850, 13662, 350, 16384, 16384, 16384, 6692] dup_rows_per_dst=[0, 0, 0, 0, 0, 3450, 0, 0] combine_rows_per_src=[14148, 10850, 13662, 350, 16384, 16384, 16384, 6692] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97146 payload_rows_remote=85232 dup_rows=1158 dup_rows_remote=944 dedup_ratio=0.011780 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5315, 0, 0, 6931, 0, 0, 0, 0, 0, 6794, 0, 0, 0, 0, 0, 164, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 326, 0, 0, 3194, 8192, 0, 0] payload_rows_per_dst=[13862, 10630, 13588, 11914, 16384, 7344, 16384, 7040] dup_rows_per_dst=[0, 0, 0, 214, 0, 944, 0, 0] combine_rows_per_src=[13862, 10630, 13588, 11914, 16384, 7344, 16384, 7040] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73750 payload_rows_remote=57788 dup_rows=24554 dup_rows_remote=24292 dedup_ratio=0.249776 router_tokens_per_expert=[2, 1, 1, 0, 1, 0, 133, 0, 14, 219, 6, 1429, 1, 0, 0, 0, 8, 7914, 0, 0, 1, 5822, 0, 5, 2, 5, 7974, 1, 0, 4, 45, 5287, 25, 30, 3, 0, 17, 0, 0, 0, 8, 18, 5, 5557, 8163, 1520, 2035, 6, 1, 0, 5, 546, 0, 2253, 0, 0, 0, 0, 0, 27, 0, 2, 0, 56] payload_rows_per_dst=[15962, 13164, 16356, 5818, 10668, 6270, 5346, 166] dup_rows_per_dst=[262, 1288, 21718, 56, 48, 914, 264, 4] combine_rows_per_src=[15962, 13164, 16356, 5818, 10668, 6270, 5346, 166] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=257744896 dispatch_meta_bytes=337456 combine_payload_bytes=257744896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258082352 ingress_bytes=292347904 link_bytes=292347904 token_bytes=515827248 weight_movement_bytes=34603008 total_nvlink_bytes=550430256 weight_to_token_ratio=0.067083 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=36096 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.100081627e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=242900992 dispatch_meta_bytes=319280 combine_payload_bytes=242900992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=243220272 ingress_bytes=277504000 link_bytes=277504000 token_bytes=486121264 weight_movement_bytes=34603008 total_nvlink_bytes=520724272 weight_to_token_ratio=0.071182 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=24576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.044225211e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95302 payload_rows_remote=82488 dup_rows=3002 dup_rows_remote=2766 dedup_ratio=0.030538 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 209, 0, 0, 246, 0, 1451, 0, 0, 0, 0, 0, 7946, 0, 0, 0, 6376, 0, 0, 0, 0, 8040, 0, 0, 0, 9, 4820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5518, 8192, 1587, 1802, 0, 0, 0, 0, 510, 0, 2385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61] payload_rows_per_dst=[16080, 13362, 12814, 9360, 16384, 5880, 5530, 15892] dup_rows_per_dst=[418, 1068, 236, 0, 0, 898, 260, 122] combine_rows_per_src=[16080, 13362, 12814, 9360, 16384, 5880, 5530, 15892] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70310 payload_rows_remote=53926 dup_rows=27994 dup_rows_remote=12714 dedup_ratio=0.284770 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5325, 0, 0, 7144, 0, 0, 0, 0, 0, 6857, 0, 0, 0, 0, 0, 148, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 292, 0, 0, 3137, 8192, 0, 0] payload_rows_per_dst=[1852, 15426, 3584, 296, 16384, 16384, 0, 16384] dup_rows_per_dst=[0, 7660, 0, 0, 1708, 3346, 0, 15280] combine_rows_per_src=[1852, 15426, 3584, 296, 16384, 16384, 0, 16384] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=321413120 dispatch_meta_bytes=327680 combine_payload_bytes=321413120 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=321740800 ingress_bytes=390619136 link_bytes=390619136 token_bytes=643153920 weight_movement_bytes=69206016 total_nvlink_bytes=712359936 weight_to_token_ratio=0.107604 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=48384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.469868361e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82546 payload_rows_remote=66764 dup_rows=15758 dup_rows_remote=15758 dedup_ratio=0.160299 router_tokens_per_expert=[7570, 0, 0, 0, 0, 0, 0, 554, 0, 0, 5, 0, 0, 0, 0, 0, 98, 0, 753, 1697, 8192, 0, 0, 0, 0, 0, 207, 0, 567, 0, 0, 0, 476, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 7891, 7051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6411, 7179, 0, 86, 0, 0, 330] payload_rows_per_dst=[1108, 11032, 4906, 3968, 16384, 15758, 15782, 13608] dup_rows_per_dst=[0, 0, 190, 106, 1122, 10958, 0, 3382] combine_rows_per_src=[1108, 11032, 4906, 3968, 16384, 15758, 15782, 13608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=349110272 dispatch_meta_bytes=344704 combine_payload_bytes=349110272 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=349454976 ingress_bytes=418316288 link_bytes=418316288 token_bytes=698565248 weight_movement_bytes=69206016 total_nvlink_bytes=767771264 weight_to_token_ratio=0.099069 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=30464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.574090515e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=236699648 dispatch_meta_bytes=328320 combine_payload_bytes=236699648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=237027968 ingress_bytes=271302656 link_bytes=271302656 token_bytes=473727616 weight_movement_bytes=34603008 total_nvlink_bytes=508330624 weight_to_token_ratio=0.073044 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=48128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.020890053e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94588 payload_rows_remote=78204 dup_rows=3716 dup_rows_remote=36 dedup_ratio=0.037801 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5332, 0, 0, 6806, 0, 0, 0, 0, 0, 6792, 0, 0, 0, 0, 0, 198, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 329, 0, 0, 3279, 8192, 0, 0] payload_rows_per_dst=[13612, 10664, 13584, 396, 16384, 16384, 16302, 7262] dup_rows_per_dst=[0, 0, 0, 0, 0, 3680, 0, 36] combine_rows_per_src=[13612, 10664, 13584, 396, 16384, 16384, 16302, 7262] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96254 payload_rows_remote=82734 dup_rows=2050 dup_rows_remote=2050 dedup_ratio=0.020854 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5271, 0, 0, 7057, 0, 0, 0, 0, 0, 6760, 0, 0, 0, 0, 0, 165, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 325, 0, 0, 3343, 8192, 0, 0] payload_rows_per_dst=[14114, 10542, 13520, 16384, 14664, 3310, 16384, 7336] dup_rows_per_dst=[0, 0, 0, 2050, 0, 0, 0, 0] combine_rows_per_src=[14114, 10542, 13520, 16384, 14664, 3310, 16384, 7336] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70310 payload_rows_remote=53926 dup_rows=27994 dup_rows_remote=12714 dedup_ratio=0.284770 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5325, 0, 0, 7144, 0, 0, 0, 0, 0, 6857, 0, 0, 0, 0, 0, 148, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 292, 0, 0, 3137, 8192, 0, 0] payload_rows_per_dst=[1852, 15426, 3584, 296, 16384, 16384, 0, 16384] dup_rows_per_dst=[0, 7660, 0, 0, 1708, 3346, 0, 15280] combine_rows_per_src=[1852, 15426, 3584, 296, 16384, 16384, 0, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87146 payload_rows_remote=87146 dup_rows=11158 dup_rows_remote=11158 dedup_ratio=0.113505 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5313, 0, 0, 6707, 0, 0, 0, 0, 0, 6756, 0, 0, 0, 0, 0, 221, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 318, 0, 0, 3489, 8192, 0, 0] payload_rows_per_dst=[13414, 10626, 13512, 442, 16384, 16384, 0, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 3544, 0, 7614] combine_rows_per_src=[13414, 10626, 13512, 442, 16384, 16384, 0, 16384] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81526 payload_rows_remote=70592 dup_rows=16778 dup_rows_remote=16776 dedup_ratio=0.170675 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 5464, 1, 1, 6893, 0, 0, 0, 0, 0, 6788, 0, 0, 0, 1, 0, 196, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 1716, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 327, 0, 0, 3184, 8191, 0, 0] payload_rows_per_dst=[13788, 10934, 13576, 16384, 0, 3436, 16384, 7024] dup_rows_per_dst=[0, 2, 0, 16774, 0, 0, 2, 0] combine_rows_per_src=[13788, 10934, 13576, 16384, 0, 3436, 16384, 7024] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79790 payload_rows_remote=63438 dup_rows=18514 dup_rows_remote=7338 dedup_ratio=0.188334 router_tokens_per_expert=[7554, 0, 20, 0, 28, 0, 0, 505, 0, 0, 32, 0, 0, 0, 0, 0, 112, 2, 792, 1621, 8143, 0, 13, 0, 0, 0, 164, 0, 540, 12, 1, 0, 437, 11, 35, 0, 2, 0, 0, 88, 0, 11, 0, 21, 7898, 7012, 0, 0, 0, 0, 2, 0, 0, 24, 0, 0, 7, 6474, 7129, 0, 91, 26, 1, 344] payload_rows_per_dst=[16352, 14322, 16346, 14802, 1142, 15852, 50, 924] dup_rows_per_dst=[11176, 0, 6654, 656, 4, 8, 2, 14] combine_rows_per_src=[16352, 14322, 16346, 14802, 1142, 15852, 50, 924] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=337870848 dispatch_meta_bytes=341016 combine_payload_bytes=337870848 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=338211864 ingress_bytes=441679872 link_bytes=441679872 token_bytes=676082712 weight_movement_bytes=103809024 total_nvlink_bytes=779891736 weight_to_token_ratio=0.153545 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.662005800e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=220880896 dispatch_meta_bytes=266560 combine_payload_bytes=220880896 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=221147456 ingress_bytes=290086912 link_bytes=290086912 token_bytes=442028352 weight_movement_bytes=69206016 total_nvlink_bytes=511234368 weight_to_token_ratio=0.156565 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=25984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.091573696e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95884 payload_rows_remote=79500 dup_rows=2420 dup_rows_remote=1386 dedup_ratio=0.024618 router_tokens_per_expert=[7652, 0, 0, 0, 0, 0, 0, 497, 0, 0, 29, 0, 0, 0, 0, 0, 101, 0, 742, 1730, 8192, 0, 0, 0, 0, 0, 158, 0, 506, 0, 0, 0, 422, 0, 1, 0, 0, 0, 0, 94, 0, 0, 0, 0, 7972, 7144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6304, 7229, 0, 114, 0, 0, 265] payload_rows_per_dst=[16176, 14516, 5020, 15048, 16384, 0, 15944, 12796] dup_rows_per_dst=[122, 0, 126, 568, 1034, 0, 0, 570] combine_rows_per_src=[16176, 14516, 5020, 15048, 16384, 0, 15944, 12796] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=273465344 dispatch_meta_bytes=330088 combine_payload_bytes=273465344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273795432 ingress_bytes=308068352 link_bytes=308068352 token_bytes=547260776 weight_movement_bytes=34603008 total_nvlink_bytes=581863784 weight_to_token_ratio=0.063229 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.159236407e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=320323584 dispatch_meta_bytes=312960 combine_payload_bytes=320323584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=320636544 ingress_bytes=354926592 link_bytes=354926592 token_bytes=640960128 weight_movement_bytes=34603008 total_nvlink_bytes=675563136 weight_to_token_ratio=0.053986 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=35072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.335560192e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=338878464 dispatch_meta_bytes=339136 combine_payload_bytes=338878464 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=339217600 ingress_bytes=442687488 link_bytes=442687488 token_bytes=678096064 weight_movement_bytes=103809024 total_nvlink_bytes=781905088 weight_to_token_ratio=0.153089 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.665797378e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96042 payload_rows_remote=81036 dup_rows=2262 dup_rows_remote=1728 dedup_ratio=0.023010 router_tokens_per_expert=[7635, 0, 0, 0, 0, 0, 0, 507, 0, 0, 24, 0, 0, 0, 0, 0, 103, 0, 708, 1739, 8192, 0, 0, 0, 0, 0, 180, 0, 509, 0, 0, 0, 418, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 7966, 7081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6436, 7251, 0, 81, 0, 0, 242] payload_rows_per_dst=[16162, 14550, 4976, 15006, 16384, 0, 15932, 13032] dup_rows_per_dst=[122, 0, 124, 534, 996, 0, 0, 486] combine_rows_per_src=[16162, 14550, 4976, 15006, 16384, 0, 15932, 13032] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81344 payload_rows_remote=67834 dup_rows=16960 dup_rows_remote=16956 dedup_ratio=0.172526 router_tokens_per_expert=[1, 4, 0, 0, 1, 3, 3, 0, 0, 0, 3, 0, 5358, 10, 27, 6745, 3, 0, 2, 2, 10, 6702, 1, 6, 0, 0, 0, 199, 0, 27, 8162, 0, 0, 0, 1, 0, 0, 8162, 2, 3, 0, 0, 1802, 0, 0, 28, 27, 0, 11, 35, 0, 0, 0, 3, 14, 4, 8, 340, 0, 3, 3271, 8165, 4, 0] payload_rows_per_dst=[13510, 10868, 13274, 16380, 12, 3684, 16366, 7250] dup_rows_per_dst=[4, 102, 4, 16720, 0, 30, 98, 2] combine_rows_per_src=[13510, 10868, 13274, 16380, 12, 3684, 16366, 7250] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=220880896 dispatch_meta_bytes=266560 combine_payload_bytes=220880896 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=221147456 ingress_bytes=290086912 link_bytes=290086912 token_bytes=442028352 weight_movement_bytes=69206016 total_nvlink_bytes=511234368 weight_to_token_ratio=0.156565 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=25984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.091573696e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=356950016 dispatch_meta_bytes=393216 combine_payload_bytes=356950016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=357343232 ingress_bytes=391553024 link_bytes=391553024 token_bytes=714293248 weight_movement_bytes=34603008 total_nvlink_bytes=748896256 weight_to_token_ratio=0.048444 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.473382507e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=289144832 dispatch_meta_bytes=349472 combine_payload_bytes=289144832 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=289494304 ingress_bytes=358350848 link_bytes=358350848 token_bytes=578639136 weight_movement_bytes=69206016 total_nvlink_bytes=647845152 weight_to_token_ratio=0.119601 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=42880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.348445391e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=259842048 dispatch_meta_bytes=283104 combine_payload_bytes=259842048 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=260125152 ingress_bytes=329048064 link_bytes=329048064 token_bytes=519967200 weight_movement_bytes=69206016 total_nvlink_bytes=589173216 weight_to_token_ratio=0.133097 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=39168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.238181374e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95884 payload_rows_remote=79500 dup_rows=2420 dup_rows_remote=1386 dedup_ratio=0.024618 router_tokens_per_expert=[7652, 0, 0, 0, 0, 0, 0, 497, 0, 0, 29, 0, 0, 0, 0, 0, 101, 0, 742, 1730, 8192, 0, 0, 0, 0, 0, 158, 0, 506, 0, 0, 0, 422, 0, 1, 0, 0, 0, 0, 94, 0, 0, 0, 0, 7972, 7144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6304, 7229, 0, 114, 0, 0, 265] payload_rows_per_dst=[16176, 14516, 5020, 15048, 16384, 0, 15944, 12796] dup_rows_per_dst=[122, 0, 126, 568, 1034, 0, 0, 570] combine_rows_per_src=[16176, 14516, 5020, 15048, 16384, 0, 15944, 12796] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96042 payload_rows_remote=81036 dup_rows=2262 dup_rows_remote=1728 dedup_ratio=0.023010 router_tokens_per_expert=[7635, 0, 0, 0, 0, 0, 0, 507, 0, 0, 24, 0, 0, 0, 0, 0, 103, 0, 708, 1739, 8192, 0, 0, 0, 0, 0, 180, 0, 509, 0, 0, 0, 418, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 7966, 7081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6436, 7251, 0, 81, 0, 0, 242] payload_rows_per_dst=[16162, 14550, 4976, 15006, 16384, 0, 15932, 13032] dup_rows_per_dst=[122, 0, 124, 534, 996, 0, 0, 486] combine_rows_per_src=[16162, 14550, 4976, 15006, 16384, 0, 15932, 13032] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=325632000 dispatch_meta_bytes=323544 combine_payload_bytes=325632000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=325955544 ingress_bytes=360235008 link_bytes=360235008 token_bytes=651587544 weight_movement_bytes=34603008 total_nvlink_bytes=686190552 weight_to_token_ratio=0.053106 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=45056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.355535334e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96254 payload_rows_remote=82734 dup_rows=2050 dup_rows_remote=2050 dedup_ratio=0.020854 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5271, 0, 0, 7057, 0, 0, 0, 0, 0, 6760, 0, 0, 0, 0, 0, 165, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 325, 0, 0, 3343, 8192, 0, 0] payload_rows_per_dst=[14114, 10542, 13520, 16384, 14664, 3310, 16384, 7336] dup_rows_per_dst=[0, 0, 0, 2050, 0, 0, 0, 0] combine_rows_per_src=[14114, 10542, 13520, 16384, 14664, 3310, 16384, 7336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71922 payload_rows_remote=55820 dup_rows=26382 dup_rows_remote=14572 dedup_ratio=0.268372 router_tokens_per_expert=[7635, 0, 0, 0, 0, 0, 0, 603, 0, 0, 47, 0, 0, 0, 0, 0, 60, 0, 788, 1684, 8192, 0, 0, 0, 0, 0, 183, 0, 425, 0, 0, 0, 379, 0, 2, 0, 0, 0, 0, 63, 0, 0, 0, 0, 8044, 7091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6365, 7370, 0, 63, 0, 0, 158] payload_rows_per_dst=[1206, 94, 4976, 1216, 16384, 15856, 16088, 16102] dup_rows_per_dst=[0, 0, 88, 0, 888, 13596, 0, 11810] combine_rows_per_src=[1206, 94, 4976, 1216, 16384, 15856, 16088, 16102] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=331923456 dispatch_meta_bytes=331056 combine_payload_bytes=331923456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=332254512 ingress_bytes=366526464 link_bytes=366526464 token_bytes=664177968 weight_movement_bytes=34603008 total_nvlink_bytes=698780976 weight_to_token_ratio=0.052099 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=43776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.379209576e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=277848064 dispatch_meta_bytes=339160 combine_payload_bytes=277848064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278187224 ingress_bytes=312451072 link_bytes=312451072 token_bytes=556035288 weight_movement_bytes=34603008 total_nvlink_bytes=590638296 weight_to_token_ratio=0.062232 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.175728230e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=60822 payload_rows_remote=59986 dup_rows=37482 dup_rows_remote=37482 dedup_ratio=0.381287 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1269, 0, 0, 0, 99, 0, 2449, 6422, 0, 0, 7378, 0, 0, 0, 0, 738, 0, 0, 0, 0, 8107, 0, 0, 0, 0, 1016, 0, 7721, 0, 0, 0, 0, 4205, 0, 0, 0, 0, 85, 0, 0, 0, 333, 1250, 1, 7994, 0, 0, 85, 0, 0] payload_rows_per_dst=[5268, 2538, 4922, 1476, 16376, 13054, 836, 16352] dup_rows_per_dst=[0, 0, 174, 0, 16626, 5530, 0, 15152] combine_rows_per_src=[5268, 2538, 4922, 1476, 16376, 13054, 836, 16352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96032 payload_rows_remote=82956 dup_rows=2272 dup_rows_remote=2272 dedup_ratio=0.023112 router_tokens_per_expert=[7591, 0, 0, 0, 0, 0, 0, 528, 0, 0, 21, 0, 0, 0, 0, 0, 72, 0, 772, 1848, 8192, 0, 0, 0, 0, 0, 223, 0, 509, 0, 0, 0, 451, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 7952, 7071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6402, 7142, 0, 70, 0, 0, 238] payload_rows_per_dst=[3152, 14324, 5234, 15014, 16384, 13076, 15904, 12944] dup_rows_per_dst=[10, 2, 150, 592, 1042, 0, 0, 476] combine_rows_per_src=[3152, 14324, 5234, 15014, 16384, 13076, 15904, 12944] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91920 payload_rows_remote=81324 dup_rows=6384 dup_rows_remote=1638 dedup_ratio=0.064941 router_tokens_per_expert=[7616, 0, 0, 0, 0, 0, 0, 594, 0, 0, 65, 0, 0, 0, 0, 0, 104, 0, 764, 1624, 8192, 0, 0, 0, 0, 0, 146, 0, 329, 0, 0, 0, 443, 0, 4, 0, 0, 0, 0, 75, 0, 0, 0, 0, 7952, 7087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6378, 7406, 0, 76, 0, 0, 297] payload_rows_per_dst=[16178, 14942, 10596, 14708, 8632, 0, 15904, 10960] dup_rows_per_dst=[242, 0, 4746, 416, 482, 0, 0, 498] combine_rows_per_src=[16178, 14942, 10596, 14708, 8632, 0, 15904, 10960] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71922 payload_rows_remote=55820 dup_rows=26382 dup_rows_remote=14572 dedup_ratio=0.268372 router_tokens_per_expert=[7635, 0, 0, 0, 0, 0, 0, 603, 0, 0, 47, 0, 0, 0, 0, 0, 60, 0, 788, 1684, 8192, 0, 0, 0, 0, 0, 183, 0, 425, 0, 0, 0, 379, 0, 2, 0, 0, 0, 0, 63, 0, 0, 0, 0, 8044, 7091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6365, 7370, 0, 63, 0, 0, 158] payload_rows_per_dst=[1206, 94, 4976, 1216, 16384, 15856, 16088, 16102] dup_rows_per_dst=[0, 0, 88, 0, 888, 13596, 0, 11810] combine_rows_per_src=[1206, 94, 4976, 1216, 16384, 15856, 16088, 16102] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82546 payload_rows_remote=66764 dup_rows=15758 dup_rows_remote=15758 dedup_ratio=0.160299 router_tokens_per_expert=[7570, 0, 0, 0, 0, 0, 0, 554, 0, 0, 5, 0, 0, 0, 0, 0, 98, 0, 753, 1697, 8192, 0, 0, 0, 0, 0, 207, 0, 567, 0, 0, 0, 476, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 7891, 7051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6411, 7179, 0, 86, 0, 0, 330] payload_rows_per_dst=[1108, 11032, 4906, 3968, 16384, 15758, 15782, 13608] dup_rows_per_dst=[0, 0, 190, 106, 1122, 10958, 0, 3382] combine_rows_per_src=[1108, 11032, 4906, 3968, 16384, 15758, 15782, 13608] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79742 payload_rows_remote=65220 dup_rows=18562 dup_rows_remote=18562 dedup_ratio=0.188822 router_tokens_per_expert=[7602, 0, 0, 0, 1, 0, 0, 546, 0, 0, 5, 0, 0, 0, 0, 0, 106, 0, 762, 1738, 8190, 0, 1, 0, 0, 0, 139, 0, 592, 0, 0, 0, 436, 1, 0, 0, 1, 0, 0, 83, 0, 1, 0, 1, 7946, 7104, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6289, 7256, 0, 83, 1, 0, 267] payload_rows_per_dst=[16142, 14522, 16384, 15054, 1042, 13198, 2698, 702] dup_rows_per_dst=[156, 0, 17788, 616, 0, 2, 0, 0] combine_rows_per_src=[16142, 14522, 16384, 15054, 1042, 13198, 2698, 702] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89808 payload_rows_remote=74376 dup_rows=8496 dup_rows_remote=8492 dedup_ratio=0.086426 router_tokens_per_expert=[0, 0, 22, 0, 2, 3, 0, 0, 0, 0, 0, 0, 2, 0, 1216, 16, 0, 0, 109, 0, 2604, 6356, 14, 0, 7229, 17, 0, 0, 0, 810, 2, 0, 0, 10, 8071, 26, 0, 0, 0, 1178, 11, 7691, 3, 27, 1, 0, 3837, 0, 8, 6, 2, 88, 6, 7, 0, 410, 1337, 72, 7842, 0, 29, 88, 0, 0] payload_rows_per_dst=[15432, 16156, 14934, 14334, 2414, 7732, 15912, 2894] dup_rows_per_dst=[4, 2454, 3752, 1262, 14, 26, 826, 158] combine_rows_per_src=[15432, 16156, 14934, 14334, 2414, 7732, 15912, 2894] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=338878464 dispatch_meta_bytes=339136 combine_payload_bytes=338878464 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=339217600 ingress_bytes=442687488 link_bytes=442687488 token_bytes=678096064 weight_movement_bytes=103809024 total_nvlink_bytes=781905088 weight_to_token_ratio=0.153089 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.665797378e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=228638720 dispatch_meta_bytes=281568 combine_payload_bytes=228638720 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=228920288 ingress_bytes=332447744 link_bytes=332447744 token_bytes=457559008 weight_movement_bytes=103809024 total_nvlink_bytes=561368032 weight_to_token_ratio=0.226876 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=37376 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.250974096e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89012 payload_rows_remote=84728 dup_rows=9292 dup_rows_remote=9022 dedup_ratio=0.094523 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1147, 0, 0, 0, 102, 0, 2439, 6593, 0, 0, 7422, 0, 0, 0, 0, 661, 0, 0, 0, 0, 8098, 0, 0, 0, 0, 964, 0, 7753, 0, 0, 0, 0, 4242, 0, 0, 0, 0, 94, 0, 0, 0, 338, 1274, 4, 7927, 0, 0, 94, 0, 0] payload_rows_per_dst=[15506, 14008, 8862, 14962, 4284, 12092, 16042, 3256] dup_rows_per_dst=[0, 1856, 1440, 1204, 270, 3708, 676, 138] combine_rows_per_src=[15506, 14008, 8862, 14962, 4284, 12092, 16042, 3256] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=333103104 dispatch_meta_bytes=331848 combine_payload_bytes=333103104 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=333434952 ingress_bytes=402309120 link_bytes=402309120 token_bytes=666538056 weight_movement_bytes=69206016 total_nvlink_bytes=735744072 weight_to_token_ratio=0.103829 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=20608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.513856830e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90408 payload_rows_remote=75610 dup_rows=7896 dup_rows_remote=6522 dedup_ratio=0.080322 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1167, 0, 0, 0, 112, 0, 2685, 6399, 0, 0, 7338, 0, 0, 0, 0, 748, 0, 0, 0, 0, 8112, 0, 0, 0, 0, 959, 0, 7758, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 80, 0, 0, 0, 344, 1331, 1, 7971, 0, 0, 80, 0, 0] payload_rows_per_dst=[15516, 16224, 14924, 14798, 1918, 8134, 16102, 2792] dup_rows_per_dst=[0, 2334, 3468, 1374, 0, 0, 688, 32] combine_rows_per_src=[15516, 16224, 14924, 14798, 1918, 8134, 16102, 2792] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79790 payload_rows_remote=63438 dup_rows=18514 dup_rows_remote=7338 dedup_ratio=0.188334 router_tokens_per_expert=[7554, 0, 20, 0, 28, 0, 0, 505, 0, 0, 32, 0, 0, 0, 0, 0, 112, 2, 792, 1621, 8143, 0, 13, 0, 0, 0, 164, 0, 540, 12, 1, 0, 437, 11, 35, 0, 2, 0, 0, 88, 0, 11, 0, 21, 7898, 7012, 0, 0, 0, 0, 2, 0, 0, 24, 0, 0, 7, 6474, 7129, 0, 91, 26, 1, 344] payload_rows_per_dst=[16352, 14322, 16346, 14802, 1142, 15852, 50, 924] dup_rows_per_dst=[11176, 0, 6654, 656, 4, 8, 2, 14] combine_rows_per_src=[16352, 14322, 16346, 14802, 1142, 15852, 50, 924] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=245702656 dispatch_meta_bytes=389872 combine_payload_bytes=245702656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=246092528 ingress_bytes=280305664 link_bytes=280305664 token_bytes=491795184 weight_movement_bytes=34603008 total_nvlink_bytes=526398192 weight_to_token_ratio=0.070361 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=45824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.054767647e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=339787776 dispatch_meta_bytes=340912 combine_payload_bytes=339787776 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=340128688 ingress_bytes=443596800 link_bytes=443596800 token_bytes=679916464 weight_movement_bytes=103809024 total_nvlink_bytes=783725488 weight_to_token_ratio=0.152679 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=21760 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.669219046e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=228638720 dispatch_meta_bytes=281568 combine_payload_bytes=228638720 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=228920288 ingress_bytes=332447744 link_bytes=332447744 token_bytes=457559008 weight_movement_bytes=103809024 total_nvlink_bytes=561368032 weight_to_token_ratio=0.226876 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=37376 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.250974096e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=273465344 dispatch_meta_bytes=330088 combine_payload_bytes=273465344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273795432 ingress_bytes=308068352 link_bytes=308068352 token_bytes=547260776 weight_movement_bytes=34603008 total_nvlink_bytes=581863784 weight_to_token_ratio=0.063229 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.159236407e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89012 payload_rows_remote=84728 dup_rows=9292 dup_rows_remote=9022 dedup_ratio=0.094523 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1147, 0, 0, 0, 102, 0, 2439, 6593, 0, 0, 7422, 0, 0, 0, 0, 661, 0, 0, 0, 0, 8098, 0, 0, 0, 0, 964, 0, 7753, 0, 0, 0, 0, 4242, 0, 0, 0, 0, 94, 0, 0, 0, 338, 1274, 4, 7927, 0, 0, 94, 0, 0] payload_rows_per_dst=[15506, 14008, 8862, 14962, 4284, 12092, 16042, 3256] dup_rows_per_dst=[0, 1856, 1440, 1204, 270, 3708, 676, 138] combine_rows_per_src=[15506, 14008, 8862, 14962, 4284, 12092, 16042, 3256] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=267141120 dispatch_meta_bytes=335128 combine_payload_bytes=267141120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267476248 ingress_bytes=301744128 link_bytes=301744128 token_bytes=534617368 weight_movement_bytes=34603008 total_nvlink_bytes=569220376 weight_to_token_ratio=0.064725 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.037500 gemm_active_groups=3 gemm_peak_group_rows=49024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135438861e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=304644096 dispatch_meta_bytes=331472 combine_payload_bytes=304644096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=304975568 ingress_bytes=339247104 link_bytes=339247104 token_bytes=609619664 weight_movement_bytes=34603008 total_nvlink_bytes=644222672 weight_to_token_ratio=0.056762 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.276559541e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90408 payload_rows_remote=75610 dup_rows=7896 dup_rows_remote=6522 dedup_ratio=0.080322 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1167, 0, 0, 0, 112, 0, 2685, 6399, 0, 0, 7338, 0, 0, 0, 0, 748, 0, 0, 0, 0, 8112, 0, 0, 0, 0, 959, 0, 7758, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 80, 0, 0, 0, 344, 1331, 1, 7971, 0, 0, 80, 0, 0] payload_rows_per_dst=[15516, 16224, 14924, 14798, 1918, 8134, 16102, 2792] dup_rows_per_dst=[0, 2334, 3468, 1374, 0, 0, 688, 32] combine_rows_per_src=[15516, 16224, 14924, 14798, 1918, 8134, 16102, 2792] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=347045888 dispatch_meta_bytes=375000 combine_payload_bytes=347045888 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=347420888 ingress_bytes=416251904 link_bytes=416251904 token_bytes=694466776 weight_movement_bytes=69206016 total_nvlink_bytes=763672792 weight_to_token_ratio=0.099653 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=25728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.566322404e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91920 payload_rows_remote=81324 dup_rows=6384 dup_rows_remote=1638 dedup_ratio=0.064941 router_tokens_per_expert=[7616, 0, 0, 0, 0, 0, 0, 594, 0, 0, 65, 0, 0, 0, 0, 0, 104, 0, 764, 1624, 8192, 0, 0, 0, 0, 0, 146, 0, 329, 0, 0, 0, 443, 0, 4, 0, 0, 0, 0, 75, 0, 0, 0, 0, 7952, 7087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6378, 7406, 0, 76, 0, 0, 297] payload_rows_per_dst=[16178, 14942, 10596, 14708, 8632, 0, 15904, 10960] dup_rows_per_dst=[242, 0, 4746, 416, 482, 0, 0, 498] combine_rows_per_src=[16178, 14942, 10596, 14708, 8632, 0, 15904, 10960] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=57754 payload_rows_remote=41396 dup_rows=40550 dup_rows_remote=24474 dedup_ratio=0.412496 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1148, 0, 0, 0, 66, 0, 2428, 6803, 0, 0, 7391, 0, 0, 0, 0, 733, 0, 0, 0, 0, 8129, 0, 0, 0, 0, 879, 0, 7831, 0, 0, 0, 0, 4005, 0, 0, 0, 0, 63, 0, 0, 0, 262, 1339, 6, 8006, 0, 0, 63, 0, 0] payload_rows_per_dst=[0, 2296, 4892, 1466, 16368, 15724, 650, 16358] dup_rows_per_dst=[0, 0, 96, 0, 16430, 7948, 0, 16076] combine_rows_per_src=[0, 2296, 4892, 1466, 16368, 15724, 650, 16358] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=309698560 dispatch_meta_bytes=328528 combine_payload_bytes=309698560 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=310027088 ingress_bytes=378904576 link_bytes=378904576 token_bytes=619725648 weight_movement_bytes=69206016 total_nvlink_bytes=688931664 weight_to_token_ratio=0.111672 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=43136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.425787415e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=259842048 dispatch_meta_bytes=283104 combine_payload_bytes=259842048 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=260125152 ingress_bytes=329048064 link_bytes=329048064 token_bytes=519967200 weight_movement_bytes=69206016 total_nvlink_bytes=589173216 weight_to_token_ratio=0.133097 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=39168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.238181374e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90038 payload_rows_remote=74978 dup_rows=8266 dup_rows_remote=5454 dedup_ratio=0.084086 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1333, 0, 0, 0, 101, 0, 2006, 6829, 0, 0, 7270, 0, 0, 0, 0, 1107, 0, 0, 0, 0, 8117, 0, 0, 0, 0, 990, 0, 7783, 0, 0, 0, 0, 3746, 0, 0, 0, 0, 76, 0, 0, 0, 337, 1378, 13, 7990, 0, 0, 76, 0, 0] payload_rows_per_dst=[15566, 16234, 15060, 14672, 1980, 7492, 16132, 2902] dup_rows_per_dst=[0, 2666, 2812, 2082, 0, 0, 674, 32] combine_rows_per_src=[15566, 16234, 15060, 14672, 1980, 7492, 16132, 2902] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94772 payload_rows_remote=79078 dup_rows=3532 dup_rows_remote=3532 dedup_ratio=0.035929 router_tokens_per_expert=[0, 175, 0, 8134, 0, 85, 0, 0, 0, 0, 7847, 0, 0, 0, 0, 7474, 0, 0, 6686, 0, 0, 672, 0, 0, 0, 364, 0, 8192, 0, 0, 0, 0, 0, 0, 8110, 0, 0, 0, 0, 0, 0, 0, 197, 0, 0, 0, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 667, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[350, 0, 14690, 16276, 16220, 16384, 15694, 15158] dup_rows_per_dst=[170, 0, 26, 720, 0, 1492, 0, 1124] combine_rows_per_src=[350, 0, 14690, 16276, 16220, 16384, 15694, 15158] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88064 payload_rows_remote=79826 dup_rows=10240 dup_rows_remote=10240 dedup_ratio=0.104167 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1430, 0, 0, 0, 79, 0, 2335, 6584, 0, 0, 7359, 0, 0, 0, 0, 663, 0, 0, 0, 0, 8122, 0, 0, 0, 0, 1032, 0, 7777, 0, 0, 0, 0, 4119, 0, 0, 0, 0, 70, 0, 0, 0, 309, 1335, 0, 7868, 0, 0, 70, 0, 0] payload_rows_per_dst=[15554, 2860, 4702, 13556, 16256, 8238, 12522, 14376] dup_rows_per_dst=[0, 0, 126, 1102, 3438, 0, 512, 5062] combine_rows_per_src=[15554, 2860, 4702, 13556, 16256, 8238, 12522, 14376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=57754 payload_rows_remote=41396 dup_rows=40550 dup_rows_remote=24474 dedup_ratio=0.412496 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1148, 0, 0, 0, 66, 0, 2428, 6803, 0, 0, 7391, 0, 0, 0, 0, 733, 0, 0, 0, 0, 8129, 0, 0, 0, 0, 879, 0, 7831, 0, 0, 0, 0, 4005, 0, 0, 0, 0, 63, 0, 0, 0, 262, 1339, 6, 8006, 0, 0, 63, 0, 0] payload_rows_per_dst=[0, 2296, 4892, 1466, 16368, 15724, 650, 16358] dup_rows_per_dst=[0, 0, 96, 0, 16430, 7948, 0, 16076] combine_rows_per_src=[0, 2296, 4892, 1466, 16368, 15724, 650, 16358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=60822 payload_rows_remote=59986 dup_rows=37482 dup_rows_remote=37482 dedup_ratio=0.381287 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1269, 0, 0, 0, 99, 0, 2449, 6422, 0, 0, 7378, 0, 0, 0, 0, 738, 0, 0, 0, 0, 8107, 0, 0, 0, 0, 1016, 0, 7721, 0, 0, 0, 0, 4205, 0, 0, 0, 0, 85, 0, 0, 0, 333, 1250, 1, 7994, 0, 0, 85, 0, 0] payload_rows_per_dst=[5268, 2538, 4922, 1476, 16376, 13054, 836, 16352] dup_rows_per_dst=[0, 0, 174, 0, 16626, 5530, 0, 15152] combine_rows_per_src=[5268, 2538, 4922, 1476, 16376, 13054, 836, 16352] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90672 payload_rows_remote=74454 dup_rows=7632 dup_rows_remote=5172 dedup_ratio=0.077637 router_tokens_per_expert=[0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1229, 0, 0, 0, 106, 0, 2544, 6419, 1, 0, 7354, 0, 0, 0, 0, 654, 0, 0, 0, 1, 8108, 1, 0, 0, 0, 963, 0, 7728, 0, 0, 0, 1, 4312, 0, 0, 0, 0, 83, 1, 0, 0, 353, 1269, 9, 7929, 0, 1, 83, 0, 0] payload_rows_per_dst=[15458, 16218, 14886, 14846, 1928, 8626, 16024, 2686] dup_rows_per_dst=[0, 2460, 3254, 1170, 2, 0, 708, 38] combine_rows_per_src=[15458, 16218, 14886, 14846, 1928, 8626, 16024, 2686] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=333103104 dispatch_meta_bytes=331848 combine_payload_bytes=333103104 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=333434952 ingress_bytes=402309120 link_bytes=402309120 token_bytes=666538056 weight_movement_bytes=69206016 total_nvlink_bytes=735744072 weight_to_token_ratio=0.103829 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=20608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.513856830e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=54038 payload_rows_remote=37766 dup_rows=44266 dup_rows_remote=31620 dedup_ratio=0.450297 router_tokens_per_expert=[0, 154, 0, 8014, 0, 88, 0, 0, 1, 1, 7822, 0, 17, 1, 20, 7492, 28, 4, 6203, 0, 19, 638, 0, 0, 0, 339, 0, 8160, 0, 0, 11, 0, 0, 0, 8066, 0, 0, 0, 0, 0, 4, 0, 194, 0, 5, 1, 681, 17, 3, 18, 1, 0, 5, 31, 0, 0, 0, 1084, 0, 7, 0, 2, 0, 21] payload_rows_per_dst=[16272, 16216, 1336, 16342, 0, 1552, 100, 2220] dup_rows_per_dst=[12646, 30624, 42, 678, 0, 252, 16, 8] combine_rows_per_src=[16272, 16216, 1336, 16342, 0, 1552, 100, 2220] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=169558016 dispatch_meta_bytes=263480 combine_payload_bytes=169558016 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=169821496 ingress_bytes=238764032 link_bytes=238764032 token_bytes=339379512 weight_movement_bytes=69206016 total_nvlink_bytes=408585528 weight_to_token_ratio=0.203919 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=20224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.984498303e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95500 payload_rows_remote=79274 dup_rows=2804 dup_rows_remote=2804 dedup_ratio=0.028524 router_tokens_per_expert=[0, 153, 0, 8099, 0, 94, 0, 0, 0, 0, 7846, 0, 0, 0, 0, 7553, 0, 0, 6828, 0, 0, 663, 0, 0, 0, 356, 0, 8192, 0, 0, 0, 0, 0, 0, 8113, 0, 0, 0, 0, 0, 0, 0, 161, 0, 0, 0, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16326, 0, 14936, 712, 16226, 16384, 15692, 15224] dup_rows_per_dst=[366, 0, 46, 0, 0, 1432, 0, 960] combine_rows_per_src=[16326, 0, 14936, 712, 16226, 16384, 15692, 15224] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=307109888 dispatch_meta_bytes=321728 combine_payload_bytes=307109888 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=307431616 ingress_bytes=410918912 link_bytes=410918912 token_bytes=614541504 weight_movement_bytes=103809024 total_nvlink_bytes=718350528 weight_to_token_ratio=0.168921 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=28672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.546254784e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95238 payload_rows_remote=94550 dup_rows=3066 dup_rows_remote=3066 dedup_ratio=0.031189 router_tokens_per_expert=[0, 140, 0, 8103, 0, 80, 0, 0, 0, 0, 7865, 0, 0, 0, 0, 7497, 0, 0, 6700, 0, 0, 669, 0, 0, 0, 344, 0, 8192, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 0, 0, 0, 0, 152, 0, 0, 0, 587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 684, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16344, 0, 14672, 688, 16278, 16384, 15730, 15142] dup_rows_per_dst=[302, 0, 66, 0, 0, 1478, 0, 1220] combine_rows_per_src=[16344, 0, 14672, 688, 16278, 16384, 15730, 15142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95500 payload_rows_remote=79274 dup_rows=2804 dup_rows_remote=2804 dedup_ratio=0.028524 router_tokens_per_expert=[0, 153, 0, 8099, 0, 94, 0, 0, 0, 0, 7846, 0, 0, 0, 0, 7553, 0, 0, 6828, 0, 0, 663, 0, 0, 0, 356, 0, 8192, 0, 0, 0, 0, 0, 0, 8113, 0, 0, 0, 0, 0, 0, 0, 161, 0, 0, 0, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16326, 0, 14936, 712, 16226, 16384, 15692, 15224] dup_rows_per_dst=[366, 0, 46, 0, 0, 1432, 0, 960] combine_rows_per_src=[16326, 0, 14936, 712, 16226, 16384, 15692, 15224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=323903488 dispatch_meta_bytes=330440 combine_payload_bytes=323903488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=324233928 ingress_bytes=358506496 link_bytes=358506496 token_bytes=648137416 weight_movement_bytes=34603008 total_nvlink_bytes=682740424 weight_to_token_ratio=0.053388 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.349031082e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=326967296 dispatch_meta_bytes=360264 combine_payload_bytes=326967296 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=327327560 ingress_bytes=396173312 link_bytes=396173312 token_bytes=654294856 weight_movement_bytes=69206016 total_nvlink_bytes=723500872 weight_to_token_ratio=0.105772 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.490768278e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89808 payload_rows_remote=74376 dup_rows=8496 dup_rows_remote=8492 dedup_ratio=0.086426 router_tokens_per_expert=[0, 0, 22, 0, 2, 3, 0, 0, 0, 0, 0, 0, 2, 0, 1216, 16, 0, 0, 109, 0, 2604, 6356, 14, 0, 7229, 17, 0, 0, 0, 810, 2, 0, 0, 10, 8071, 26, 0, 0, 0, 1178, 11, 7691, 3, 27, 1, 0, 3837, 0, 8, 6, 2, 88, 6, 7, 0, 410, 1337, 72, 7842, 0, 29, 88, 0, 0] payload_rows_per_dst=[15432, 16156, 14934, 14334, 2414, 7732, 15912, 2894] dup_rows_per_dst=[4, 2454, 3752, 1262, 14, 26, 826, 158] combine_rows_per_src=[15432, 16156, 14934, 14334, 2414, 7732, 15912, 2894] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=169558016 dispatch_meta_bytes=263480 combine_payload_bytes=169558016 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=169821496 ingress_bytes=238764032 link_bytes=238764032 token_bytes=339379512 weight_movement_bytes=69206016 total_nvlink_bytes=408585528 weight_to_token_ratio=0.203919 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=20224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.984498303e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=245702656 dispatch_meta_bytes=389872 combine_payload_bytes=245702656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=246092528 ingress_bytes=280305664 link_bytes=280305664 token_bytes=491795184 weight_movement_bytes=34603008 total_nvlink_bytes=526398192 weight_to_token_ratio=0.070361 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=45824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.054767647e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=304963584 dispatch_meta_bytes=318504 combine_payload_bytes=304963584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=305282088 ingress_bytes=339566592 link_bytes=339566592 token_bytes=610245672 weight_movement_bytes=34603008 total_nvlink_bytes=644848680 weight_to_token_ratio=0.056703 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=39296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.277761748e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=154689536 dispatch_meta_bytes=277544 combine_payload_bytes=154689536 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=154967080 ingress_bytes=223895552 link_bytes=223895552 token_bytes=309656616 weight_movement_bytes=69206016 total_nvlink_bytes=378862632 weight_to_token_ratio=0.223493 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=40704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.425009371e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95238 payload_rows_remote=94550 dup_rows=3066 dup_rows_remote=3066 dedup_ratio=0.031189 router_tokens_per_expert=[0, 140, 0, 8103, 0, 80, 0, 0, 0, 0, 7865, 0, 0, 0, 0, 7497, 0, 0, 6700, 0, 0, 669, 0, 0, 0, 344, 0, 8192, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 0, 0, 0, 0, 152, 0, 0, 0, 587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 684, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16344, 0, 14672, 688, 16278, 16384, 15730, 15142] dup_rows_per_dst=[302, 0, 66, 0, 0, 1478, 0, 1220] combine_rows_per_src=[16344, 0, 14672, 688, 16278, 16384, 15730, 15142] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=324706304 dispatch_meta_bytes=328312 combine_payload_bytes=324706304 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=325034616 ingress_bytes=393912320 link_bytes=393912320 token_bytes=649740920 weight_movement_bytes=69206016 total_nvlink_bytes=718946936 weight_to_token_ratio=0.106513 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.037500 gemm_active_groups=3 gemm_peak_group_rows=45952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.482260348e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=387276800 dispatch_meta_bytes=390464 combine_payload_bytes=387276800 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=387667264 ingress_bytes=456482816 link_bytes=456482816 token_bytes=774944064 weight_movement_bytes=69206016 total_nvlink_bytes=844150080 weight_to_token_ratio=0.089305 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=24192 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.717708087e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=304644096 dispatch_meta_bytes=331472 combine_payload_bytes=304644096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=304975568 ingress_bytes=339247104 link_bytes=339247104 token_bytes=609619664 weight_movement_bytes=34603008 total_nvlink_bytes=644222672 weight_to_token_ratio=0.056762 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.276559541e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94018 payload_rows_remote=79024 dup_rows=4286 dup_rows_remote=4252 dedup_ratio=0.043599 router_tokens_per_expert=[0, 139, 0, 8112, 0, 75, 0, 0, 0, 0, 7871, 0, 0, 0, 0, 7506, 0, 0, 6845, 0, 0, 669, 0, 0, 0, 322, 0, 8192, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 171, 0, 0, 0, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16354, 5966, 14994, 12120, 10340, 5506, 15742, 12996] dup_rows_per_dst=[298, 2386, 34, 458, 0, 486, 0, 624] combine_rows_per_src=[16354, 5966, 14994, 12120, 10340, 5506, 15742, 12996] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95078 payload_rows_remote=78694 dup_rows=3226 dup_rows_remote=1824 dedup_ratio=0.032817 router_tokens_per_expert=[0, 140, 0, 8071, 0, 70, 0, 0, 0, 0, 7844, 0, 0, 0, 0, 7452, 0, 0, 6878, 0, 0, 713, 0, 0, 0, 353, 0, 8192, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 153, 0, 0, 0, 548, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 579, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[756, 0, 15086, 15766, 16316, 16384, 15690, 15080] dup_rows_per_dst=[142, 0, 96, 604, 0, 1402, 0, 982] combine_rows_per_src=[756, 0, 15086, 15766, 16316, 16384, 15690, 15080] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90038 payload_rows_remote=74978 dup_rows=8266 dup_rows_remote=5454 dedup_ratio=0.084086 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1333, 0, 0, 0, 101, 0, 2006, 6829, 0, 0, 7270, 0, 0, 0, 0, 1107, 0, 0, 0, 0, 8117, 0, 0, 0, 0, 990, 0, 7783, 0, 0, 0, 0, 3746, 0, 0, 0, 0, 76, 0, 0, 0, 337, 1378, 13, 7990, 0, 0, 76, 0, 0] payload_rows_per_dst=[15566, 16234, 15060, 14672, 1980, 7492, 16132, 2902] dup_rows_per_dst=[0, 2666, 2812, 2082, 0, 0, 674, 32] combine_rows_per_src=[15566, 16234, 15060, 14672, 1980, 7492, 16132, 2902] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88658 payload_rows_remote=73410 dup_rows=9646 dup_rows_remote=8632 dedup_ratio=0.098124 router_tokens_per_expert=[0, 98, 0, 8101, 1, 63, 0, 0, 0, 0, 7864, 0, 0, 0, 0, 7588, 0, 0, 6817, 0, 0, 676, 0, 0, 0, 338, 0, 8192, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[198, 0, 8416, 16306, 16378, 16384, 15728, 15248] dup_rows_per_dst=[126, 0, 8, 572, 6546, 1380, 0, 1014] combine_rows_per_src=[198, 0, 8416, 16306, 16378, 16384, 15728, 15248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94772 payload_rows_remote=79078 dup_rows=3532 dup_rows_remote=3532 dedup_ratio=0.035929 router_tokens_per_expert=[0, 175, 0, 8134, 0, 85, 0, 0, 0, 0, 7847, 0, 0, 0, 0, 7474, 0, 0, 6686, 0, 0, 672, 0, 0, 0, 364, 0, 8192, 0, 0, 0, 0, 0, 0, 8110, 0, 0, 0, 0, 0, 0, 0, 197, 0, 0, 0, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 667, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[350, 0, 14690, 16276, 16220, 16384, 15694, 15158] dup_rows_per_dst=[170, 0, 26, 720, 0, 1492, 0, 1124] combine_rows_per_src=[350, 0, 14690, 16276, 16220, 16384, 15694, 15158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=387276800 dispatch_meta_bytes=390464 combine_payload_bytes=387276800 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=387667264 ingress_bytes=456482816 link_bytes=456482816 token_bytes=774944064 weight_movement_bytes=69206016 total_nvlink_bytes=844150080 weight_to_token_ratio=0.089305 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=24192 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.717708087e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82452 payload_rows_remote=66068 dup_rows=15852 dup_rows_remote=7904 dedup_ratio=0.161255 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8192, 685, 0, 1244, 0, 0, 0, 0, 759, 0, 0, 0, 0, 0, 0, 0, 900, 0, 0, 0, 0, 0, 0, 0, 0, 4468, 0, 0, 0, 0, 424, 0, 8192, 0, 0, 0, 8192, 0, 0, 54, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 691, 0, 7196] payload_rows_per_dst=[0, 16384, 1800, 40, 16354, 16384, 16384, 15106] dup_rows_per_dst=[0, 5376, 0, 0, 1752, 108, 7948, 668] combine_rows_per_src=[0, 16384, 1800, 40, 16354, 16384, 16384, 15106] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69270 payload_rows_remote=53034 dup_rows=29034 dup_rows_remote=2158 dedup_ratio=0.295349 router_tokens_per_expert=[0, 152, 0, 8100, 0, 83, 0, 0, 0, 0, 7854, 0, 0, 0, 1, 7538, 1, 0, 6768, 0, 0, 651, 0, 0, 0, 340, 1, 8190, 0, 0, 0, 0, 0, 0, 8115, 0, 0, 0, 0, 0, 0, 0, 174, 0, 1, 0, 527, 0, 0, 1, 1, 0, 1, 1, 0, 0, 0, 650, 0, 0, 0, 0, 1, 1] payload_rows_per_dst=[16358, 16236, 13850, 16382, 0, 1230, 3910, 1304] dup_rows_per_dst=[1296, 26876, 6, 680, 0, 174, 2, 0] combine_rows_per_src=[16358, 16236, 13850, 16382, 0, 1230, 3910, 1304] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93292 payload_rows_remote=83308 dup_rows=5012 dup_rows_remote=4924 dedup_ratio=0.050985 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8192, 656, 0, 1095, 0, 0, 0, 0, 809, 0, 0, 0, 0, 0, 0, 0, 966, 0, 0, 0, 0, 0, 0, 0, 0, 4514, 0, 0, 0, 0, 522, 0, 8192, 0, 0, 0, 8192, 0, 0, 85, 0, 0, 0, 0, 0, 0, 8132, 0, 0, 0, 0, 0, 0, 0, 618, 0, 7179] payload_rows_per_dst=[9718, 9368, 16384, 16384, 9984, 170, 16264, 15020] dup_rows_per_dst=[0, 2418, 1932, 0, 88, 0, 0, 574] combine_rows_per_src=[9718, 9368, 16384, 16384, 9984, 170, 16264, 15020] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94924 payload_rows_remote=78540 dup_rows=3380 dup_rows_remote=3380 dedup_ratio=0.034383 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8192, 714, 0, 1075, 0, 0, 0, 0, 735, 0, 0, 0, 0, 0, 0, 0, 979, 0, 0, 0, 0, 0, 0, 0, 0, 4586, 0, 0, 0, 0, 488, 0, 8192, 0, 0, 0, 8192, 0, 0, 75, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 0, 0, 0, 643, 0, 7134] payload_rows_per_dst=[16384, 4290, 16384, 16384, 10052, 332, 16112, 14986] dup_rows_per_dst=[0, 758, 1958, 0, 96, 0, 0, 568] combine_rows_per_src=[16384, 4290, 16384, 16384, 10052, 332, 16112, 14986] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95078 payload_rows_remote=78694 dup_rows=3226 dup_rows_remote=1824 dedup_ratio=0.032817 router_tokens_per_expert=[0, 140, 0, 8071, 0, 70, 0, 0, 0, 0, 7844, 0, 0, 0, 0, 7452, 0, 0, 6878, 0, 0, 713, 0, 0, 0, 353, 0, 8192, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 153, 0, 0, 0, 548, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 579, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[756, 0, 15086, 15766, 16316, 16384, 15690, 15080] dup_rows_per_dst=[142, 0, 96, 604, 0, 1402, 0, 982] combine_rows_per_src=[756, 0, 15086, 15766, 16316, 16384, 15690, 15080] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=307109888 dispatch_meta_bytes=321728 combine_payload_bytes=307109888 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=307431616 ingress_bytes=410918912 link_bytes=410918912 token_bytes=614541504 weight_movement_bytes=103809024 total_nvlink_bytes=718350528 weight_to_token_ratio=0.168921 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=28672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.546254784e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81578 payload_rows_remote=65234 dup_rows=16726 dup_rows_remote=2978 dedup_ratio=0.170146 router_tokens_per_expert=[11, 20, 0, 0, 0, 0, 3, 8134, 584, 29, 1580, 0, 0, 0, 0, 727, 0, 8, 2, 0, 0, 0, 1, 941, 0, 0, 0, 0, 0, 0, 3, 15, 4637, 15, 0, 0, 0, 429, 0, 8155, 0, 29, 0, 8141, 20, 28, 52, 0, 0, 0, 0, 9, 17, 8010, 0, 18, 0, 9, 4, 2, 1, 640, 0, 6878] payload_rows_per_dst=[16344, 5108, 16312, 16332, 10040, 16084, 56, 1302] dup_rows_per_dst=[13748, 732, 1874, 14, 122, 194, 32, 10] combine_rows_per_src=[16344, 5108, 16312, 16332, 10040, 16084, 56, 1302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=54038 payload_rows_remote=37766 dup_rows=44266 dup_rows_remote=31620 dedup_ratio=0.450297 router_tokens_per_expert=[0, 154, 0, 8014, 0, 88, 0, 0, 1, 1, 7822, 0, 17, 1, 20, 7492, 28, 4, 6203, 0, 19, 638, 0, 0, 0, 339, 0, 8160, 0, 0, 11, 0, 0, 0, 8066, 0, 0, 0, 0, 0, 4, 0, 194, 0, 5, 1, 681, 17, 3, 18, 1, 0, 5, 31, 0, 0, 0, 1084, 0, 7, 0, 2, 0, 21] payload_rows_per_dst=[16272, 16216, 1336, 16342, 0, 1552, 100, 2220] dup_rows_per_dst=[12646, 30624, 42, 678, 0, 252, 16, 8] combine_rows_per_src=[16272, 16216, 1336, 16342, 0, 1552, 100, 2220] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=323682304 dispatch_meta_bytes=333104 combine_payload_bytes=323682304 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=324015408 ingress_bytes=392888320 link_bytes=392888320 token_bytes=647697712 weight_movement_bytes=69206016 total_nvlink_bytes=716903728 weight_to_token_ratio=0.106849 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=43776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.478407118e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=323903488 dispatch_meta_bytes=330440 combine_payload_bytes=323903488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=324233928 ingress_bytes=358506496 link_bytes=358506496 token_bytes=648137416 weight_movement_bytes=34603008 total_nvlink_bytes=682740424 weight_to_token_ratio=0.053388 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.349031082e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=322330624 dispatch_meta_bytes=322072 combine_payload_bytes=322330624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=322652696 ingress_bytes=356933632 link_bytes=356933632 token_bytes=644983320 weight_movement_bytes=34603008 total_nvlink_bytes=679586328 weight_to_token_ratio=0.053649 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=43264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.343112522e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=300687360 dispatch_meta_bytes=328168 combine_payload_bytes=300687360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=301015528 ingress_bytes=335290368 link_bytes=335290368 token_bytes=601702888 weight_movement_bytes=34603008 total_nvlink_bytes=636305896 weight_to_token_ratio=0.057508 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=43904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261670662e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93292 payload_rows_remote=83308 dup_rows=5012 dup_rows_remote=4924 dedup_ratio=0.050985 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8192, 656, 0, 1095, 0, 0, 0, 0, 809, 0, 0, 0, 0, 0, 0, 0, 966, 0, 0, 0, 0, 0, 0, 0, 0, 4514, 0, 0, 0, 0, 522, 0, 8192, 0, 0, 0, 8192, 0, 0, 85, 0, 0, 0, 0, 0, 0, 8132, 0, 0, 0, 0, 0, 0, 0, 618, 0, 7179] payload_rows_per_dst=[9718, 9368, 16384, 16384, 9984, 170, 16264, 15020] dup_rows_per_dst=[0, 2418, 1932, 0, 88, 0, 0, 574] combine_rows_per_src=[9718, 9368, 16384, 16384, 9984, 170, 16264, 15020] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=270614528 dispatch_meta_bytes=295888 combine_payload_bytes=270614528 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=270910416 ingress_bytes=339820544 link_bytes=339820544 token_bytes=541524944 weight_movement_bytes=69206016 total_nvlink_bytes=610730960 weight_to_token_ratio=0.127798 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=40832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.278717349e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=322330624 dispatch_meta_bytes=322072 combine_payload_bytes=322330624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=322652696 ingress_bytes=356933632 link_bytes=356933632 token_bytes=644983320 weight_movement_bytes=34603008 total_nvlink_bytes=679586328 weight_to_token_ratio=0.053649 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=43264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.343112522e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=217227264 dispatch_meta_bytes=220768 combine_payload_bytes=217227264 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=217448032 ingress_bytes=321036288 link_bytes=321036288 token_bytes=434675296 weight_movement_bytes=103809024 total_nvlink_bytes=538484320 weight_to_token_ratio=0.238820 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=19200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.208033706e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267198464 dispatch_meta_bytes=272848 combine_payload_bytes=267198464 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267471312 ingress_bytes=336404480 link_bytes=336404480 token_bytes=534669776 weight_movement_bytes=69206016 total_nvlink_bytes=603875792 weight_to_token_ratio=0.129437 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=37632 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265862975e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94924 payload_rows_remote=78540 dup_rows=3380 dup_rows_remote=3380 dedup_ratio=0.034383 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8192, 714, 0, 1075, 0, 0, 0, 0, 735, 0, 0, 0, 0, 0, 0, 0, 979, 0, 0, 0, 0, 0, 0, 0, 0, 4586, 0, 0, 0, 0, 488, 0, 8192, 0, 0, 0, 8192, 0, 0, 75, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 0, 0, 0, 643, 0, 7134] payload_rows_per_dst=[16384, 4290, 16384, 16384, 10052, 332, 16112, 14986] dup_rows_per_dst=[0, 758, 1958, 0, 96, 0, 0, 568] combine_rows_per_src=[16384, 4290, 16384, 16384, 10052, 332, 16112, 14986] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=341229568 dispatch_meta_bytes=352928 combine_payload_bytes=341229568 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=341582496 ingress_bytes=445038592 link_bytes=445038592 token_bytes=682812064 weight_movement_bytes=103809024 total_nvlink_bytes=786621088 weight_to_token_ratio=0.152032 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=27392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.674644393e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94018 payload_rows_remote=79024 dup_rows=4286 dup_rows_remote=4252 dedup_ratio=0.043599 router_tokens_per_expert=[0, 139, 0, 8112, 0, 75, 0, 0, 0, 0, 7871, 0, 0, 0, 0, 7506, 0, 0, 6845, 0, 0, 669, 0, 0, 0, 322, 0, 8192, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 171, 0, 0, 0, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16354, 5966, 14994, 12120, 10340, 5506, 15742, 12996] dup_rows_per_dst=[298, 2386, 34, 458, 0, 486, 0, 624] combine_rows_per_src=[16354, 5966, 14994, 12120, 10340, 5506, 15742, 12996] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=321699840 dispatch_meta_bytes=327680 combine_payload_bytes=321699840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=322027520 ingress_bytes=356302848 link_bytes=356302848 token_bytes=643727360 weight_movement_bytes=34603008 total_nvlink_bytes=678330368 weight_to_token_ratio=0.053754 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.340738932e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=154689536 dispatch_meta_bytes=277544 combine_payload_bytes=154689536 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=154967080 ingress_bytes=223895552 link_bytes=223895552 token_bytes=309656616 weight_movement_bytes=69206016 total_nvlink_bytes=378862632 weight_to_token_ratio=0.223493 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=40704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.425009371e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82452 payload_rows_remote=66068 dup_rows=15852 dup_rows_remote=7904 dedup_ratio=0.161255 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8192, 685, 0, 1244, 0, 0, 0, 0, 759, 0, 0, 0, 0, 0, 0, 0, 900, 0, 0, 0, 0, 0, 0, 0, 0, 4468, 0, 0, 0, 0, 424, 0, 8192, 0, 0, 0, 8192, 0, 0, 54, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 691, 0, 7196] payload_rows_per_dst=[0, 16384, 1800, 40, 16354, 16384, 16384, 15106] dup_rows_per_dst=[0, 5376, 0, 0, 1752, 108, 7948, 668] combine_rows_per_src=[0, 16384, 1800, 40, 16354, 16384, 16384, 15106] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94452 payload_rows_remote=78068 dup_rows=3852 dup_rows_remote=1602 dedup_ratio=0.039185 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8192, 670, 0, 1167, 0, 0, 0, 0, 838, 0, 0, 0, 0, 0, 0, 0, 1125, 0, 0, 0, 0, 0, 0, 0, 0, 4121, 0, 0, 0, 0, 654, 0, 8192, 0, 0, 0, 8192, 0, 0, 66, 0, 0, 0, 0, 0, 0, 8150, 0, 0, 0, 0, 0, 0, 0, 682, 0, 7103] payload_rows_per_dst=[16384, 4582, 16384, 16384, 9432, 16300, 0, 14986] dup_rows_per_dst=[0, 768, 2250, 0, 118, 132, 0, 584] combine_rows_per_src=[16384, 4582, 16384, 16384, 9432, 16300, 0, 14986] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92236 payload_rows_remote=76774 dup_rows=6068 dup_rows_remote=5936 dedup_ratio=0.061727 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8192, 650, 0, 1067, 0, 0, 0, 0, 891, 0, 0, 0, 0, 0, 0, 0, 870, 0, 0, 0, 0, 0, 0, 0, 0, 4589, 0, 0, 0, 0, 517, 0, 8192, 0, 0, 0, 8192, 0, 0, 71, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 611, 0, 7129] payload_rows_per_dst=[0, 16384, 2588, 16384, 10140, 15462, 16362, 14916] dup_rows_per_dst=[0, 5216, 84, 0, 72, 132, 0, 564] combine_rows_per_src=[0, 16384, 2588, 16384, 10140, 15462, 16362, 14916] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82482 payload_rows_remote=67540 dup_rows=15822 dup_rows_remote=15294 dedup_ratio=0.160950 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8192, 679, 0, 1210, 0, 0, 0, 0, 762, 0, 0, 0, 0, 0, 0, 0, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 4385, 0, 0, 0, 0, 544, 0, 8192, 0, 0, 0, 8192, 0, 0, 75, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 580, 0, 7155] payload_rows_per_dst=[0, 16384, 2004, 0, 16384, 16384, 16384, 14942] dup_rows_per_dst=[0, 5302, 0, 0, 1088, 150, 8754, 528] combine_rows_per_src=[0, 16384, 2004, 0, 16384, 16384, 16384, 14942] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=66366 payload_rows_remote=52074 dup_rows=31938 dup_rows_remote=30464 dedup_ratio=0.324890 router_tokens_per_expert=[0, 0, 3449, 6465, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 369, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 41, 0, 0, 129, 0, 0, 0, 8192, 8192, 0, 0, 1418, 0, 0, 0, 0, 0, 0, 0, 0, 163, 0, 4264, 0, 0, 0] payload_rows_per_dst=[6898, 8346, 738, 172, 5620, 16384, 14292, 13916] dup_rows_per_dst=[0, 0, 0, 0, 36, 24680, 1474, 5748] combine_rows_per_src=[6898, 8346, 738, 172, 5620, 16384, 14292, 13916] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92236 payload_rows_remote=76774 dup_rows=6068 dup_rows_remote=5936 dedup_ratio=0.061727 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8192, 650, 0, 1067, 0, 0, 0, 0, 891, 0, 0, 0, 0, 0, 0, 0, 870, 0, 0, 0, 0, 0, 0, 0, 0, 4589, 0, 0, 0, 0, 517, 0, 8192, 0, 0, 0, 8192, 0, 0, 71, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 611, 0, 7129] payload_rows_per_dst=[0, 16384, 2588, 16384, 10140, 15462, 16362, 14916] dup_rows_per_dst=[0, 5216, 84, 0, 72, 132, 0, 564] combine_rows_per_src=[0, 16384, 2588, 16384, 10140, 15462, 16362, 14916] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84210 payload_rows_remote=79666 dup_rows=14094 dup_rows_remote=13404 dedup_ratio=0.143372 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 0, 8190, 656, 1, 1227, 0, 0, 0, 0, 733, 0, 0, 0, 0, 0, 0, 0, 985, 0, 0, 0, 0, 0, 0, 0, 1, 4387, 0, 1, 0, 0, 497, 0, 8190, 0, 1, 0, 8190, 1, 1, 70, 0, 0, 0, 1, 1, 1, 8163, 0, 1, 0, 0, 0, 0, 0, 625, 0, 7228] payload_rows_per_dst=[16382, 4544, 16380, 16382, 10730, 16328, 6, 3458] dup_rows_per_dst=[9344, 690, 1970, 0, 1852, 144, 2, 92] combine_rows_per_src=[16382, 4544, 16380, 16382, 10730, 16328, 6, 3458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77880 payload_rows_remote=61672 dup_rows=20424 dup_rows_remote=1156 dedup_ratio=0.207764 router_tokens_per_expert=[7, 0, 3869, 6429, 0, 0, 0, 1, 32, 0, 0, 0, 8153, 0, 0, 0, 15, 399, 0, 0, 0, 0, 0, 0, 115, 1, 11, 0, 0, 0, 0, 0, 8142, 1, 0, 0, 32, 0, 54, 0, 0, 84, 1, 0, 31, 8146, 8149, 0, 31, 1245, 0, 0, 2, 0, 0, 0, 0, 31, 167, 23, 3974, 0, 0, 7] payload_rows_per_dst=[16208, 8038, 16314, 16330, 9854, 232, 2552, 8352] dup_rows_per_dst=[19268, 0, 806, 222, 72, 0, 4, 52] combine_rows_per_src=[16208, 8038, 16314, 16330, 9854, 232, 2552, 8352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=323682304 dispatch_meta_bytes=333104 combine_payload_bytes=323682304 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=324015408 ingress_bytes=392888320 link_bytes=392888320 token_bytes=647697712 weight_movement_bytes=69206016 total_nvlink_bytes=716903728 weight_to_token_ratio=0.106849 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=43776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.478407118e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=276643840 dispatch_meta_bytes=331336 combine_payload_bytes=276643840 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=276975176 ingress_bytes=345849856 link_bytes=345849856 token_bytes=553619016 weight_movement_bytes=69206016 total_nvlink_bytes=622825032 weight_to_token_ratio=0.125007 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=44160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.301405165e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95832 payload_rows_remote=79448 dup_rows=2472 dup_rows_remote=2310 dedup_ratio=0.025146 router_tokens_per_expert=[0, 0, 3317, 6511, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 81, 0, 0, 106, 0, 0, 0, 8192, 8192, 0, 0, 1414, 0, 0, 0, 0, 0, 0, 0, 0, 168, 0, 4357, 0, 0, 0] payload_rows_per_dst=[6634, 16384, 16384, 16384, 16384, 212, 14400, 9050] dup_rows_per_dst=[0, 0, 676, 184, 162, 0, 1450, 0] combine_rows_per_src=[6634, 16384, 16384, 16384, 16384, 212, 14400, 9050] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84210 payload_rows_remote=79666 dup_rows=14094 dup_rows_remote=13404 dedup_ratio=0.143372 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 0, 8190, 656, 1, 1227, 0, 0, 0, 0, 733, 0, 0, 0, 0, 0, 0, 0, 985, 0, 0, 0, 0, 0, 0, 0, 1, 4387, 0, 1, 0, 0, 497, 0, 8190, 0, 1, 0, 8190, 1, 1, 70, 0, 0, 0, 1, 1, 1, 8163, 0, 1, 0, 0, 0, 0, 0, 625, 0, 7228] payload_rows_per_dst=[16382, 4544, 16380, 16382, 10730, 16328, 6, 3458] dup_rows_per_dst=[9344, 690, 1970, 0, 1852, 144, 2, 92] combine_rows_per_src=[16382, 4544, 16380, 16382, 10730, 16328, 6, 3458] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=270614528 dispatch_meta_bytes=295888 combine_payload_bytes=270614528 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=270910416 ingress_bytes=339820544 link_bytes=339820544 token_bytes=541524944 weight_movement_bytes=69206016 total_nvlink_bytes=610730960 weight_to_token_ratio=0.127798 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=40832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.278717349e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96102 payload_rows_remote=79718 dup_rows=2202 dup_rows_remote=2018 dedup_ratio=0.022400 router_tokens_per_expert=[0, 0, 3516, 6499, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 367, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 69, 0, 0, 75, 0, 0, 0, 8192, 8192, 0, 0, 1194, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 4391, 0, 0, 0] payload_rows_per_dst=[7032, 16384, 16384, 16384, 16384, 150, 14240, 9144] dup_rows_per_dst=[0, 0, 734, 184, 138, 0, 1146, 0] combine_rows_per_src=[7032, 16384, 16384, 16384, 16384, 150, 14240, 9144] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=319766528 dispatch_meta_bytes=318680 combine_payload_bytes=319766528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=320085208 ingress_bytes=354369536 link_bytes=354369536 token_bytes=639851736 weight_movement_bytes=34603008 total_nvlink_bytes=674454744 weight_to_token_ratio=0.054080 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=41472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.333464035e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81578 payload_rows_remote=65234 dup_rows=16726 dup_rows_remote=2978 dedup_ratio=0.170146 router_tokens_per_expert=[11, 20, 0, 0, 0, 0, 3, 8134, 584, 29, 1580, 0, 0, 0, 0, 727, 0, 8, 2, 0, 0, 0, 1, 941, 0, 0, 0, 0, 0, 0, 3, 15, 4637, 15, 0, 0, 0, 429, 0, 8155, 0, 29, 0, 8141, 20, 28, 52, 0, 0, 0, 0, 9, 17, 8010, 0, 18, 0, 9, 4, 2, 1, 640, 0, 6878] payload_rows_per_dst=[16344, 5108, 16312, 16332, 10040, 16084, 56, 1302] dup_rows_per_dst=[13748, 732, 1874, 14, 122, 194, 32, 10] combine_rows_per_src=[16344, 5108, 16312, 16332, 10040, 16084, 56, 1302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=326311936 dispatch_meta_bytes=372280 combine_payload_bytes=326311936 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=326684216 ingress_bytes=360914944 link_bytes=360914944 token_bytes=652996152 weight_movement_bytes=34603008 total_nvlink_bytes=687599160 weight_to_token_ratio=0.052991 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=28032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.358093878e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=213295104 dispatch_meta_bytes=330152 combine_payload_bytes=213295104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=213625256 ingress_bytes=247898112 link_bytes=247898112 token_bytes=426920360 weight_movement_bytes=34603008 total_nvlink_bytes=461523368 weight_to_token_ratio=0.081053 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=38400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.328206380e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=314466304 dispatch_meta_bytes=330840 combine_payload_bytes=314466304 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=314797144 ingress_bytes=383672320 link_bytes=383672320 token_bytes=629263448 weight_movement_bytes=69206016 total_nvlink_bytes=698469464 weight_to_token_ratio=0.109979 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=24448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.443728052e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=276643840 dispatch_meta_bytes=331336 combine_payload_bytes=276643840 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=276975176 ingress_bytes=345849856 link_bytes=345849856 token_bytes=553619016 weight_movement_bytes=69206016 total_nvlink_bytes=622825032 weight_to_token_ratio=0.125007 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=44160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.301405165e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=252608512 dispatch_meta_bytes=251312 combine_payload_bytes=252608512 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=252859824 ingress_bytes=321814528 link_bytes=321814528 token_bytes=505468336 weight_movement_bytes=69206016 total_nvlink_bytes=574674352 weight_to_token_ratio=0.136915 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=27904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.210962161e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96102 payload_rows_remote=79718 dup_rows=2202 dup_rows_remote=2018 dedup_ratio=0.022400 router_tokens_per_expert=[0, 0, 3516, 6499, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 367, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 69, 0, 0, 75, 0, 0, 0, 8192, 8192, 0, 0, 1194, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 4391, 0, 0, 0] payload_rows_per_dst=[7032, 16384, 16384, 16384, 16384, 150, 14240, 9144] dup_rows_per_dst=[0, 0, 734, 184, 138, 0, 1146, 0] combine_rows_per_src=[7032, 16384, 16384, 16384, 16384, 150, 14240, 9144] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=325419008 dispatch_meta_bytes=327032 combine_payload_bytes=325419008 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=325746040 ingress_bytes=394625024 link_bytes=394625024 token_bytes=651165048 weight_movement_bytes=69206016 total_nvlink_bytes=720371064 weight_to_token_ratio=0.106280 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=48640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.484942195e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=326311936 dispatch_meta_bytes=372280 combine_payload_bytes=326311936 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=326684216 ingress_bytes=360914944 link_bytes=360914944 token_bytes=652996152 weight_movement_bytes=34603008 total_nvlink_bytes=687599160 weight_to_token_ratio=0.052991 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=28032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.358093878e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=326524928 dispatch_meta_bytes=326944 combine_payload_bytes=326524928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=326851872 ingress_bytes=361127936 link_bytes=361127936 token_bytes=653376800 weight_movement_bytes=34603008 total_nvlink_bytes=687979808 weight_to_token_ratio=0.052960 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=48512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.358895350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267198464 dispatch_meta_bytes=272848 combine_payload_bytes=267198464 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267471312 ingress_bytes=336404480 link_bytes=336404480 token_bytes=534669776 weight_movement_bytes=69206016 total_nvlink_bytes=603875792 weight_to_token_ratio=0.129437 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=37632 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265862975e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=66366 payload_rows_remote=52074 dup_rows=31938 dup_rows_remote=30464 dedup_ratio=0.324890 router_tokens_per_expert=[0, 0, 3449, 6465, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 369, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 41, 0, 0, 129, 0, 0, 0, 8192, 8192, 0, 0, 1418, 0, 0, 0, 0, 0, 0, 0, 0, 163, 0, 4264, 0, 0, 0] payload_rows_per_dst=[6898, 8346, 738, 172, 5620, 16384, 14292, 13916] dup_rows_per_dst=[0, 0, 0, 0, 36, 24680, 1474, 5748] combine_rows_per_src=[6898, 8346, 738, 172, 5620, 16384, 14292, 13916] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95000 payload_rows_remote=78616 dup_rows=3304 dup_rows_remote=2486 dedup_ratio=0.033610 router_tokens_per_expert=[0, 0, 3226, 6587, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 409, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 61, 0, 0, 81, 0, 0, 0, 8192, 8192, 0, 0, 1451, 0, 0, 0, 0, 0, 0, 0, 0, 191, 0, 4289, 0, 0, 0] payload_rows_per_dst=[7812, 16384, 16384, 16384, 16384, 162, 12530, 8960] dup_rows_per_dst=[690, 0, 818, 178, 122, 0, 1496, 0] combine_rows_per_src=[7812, 16384, 16384, 16384, 16384, 162, 12530, 8960] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95052 payload_rows_remote=89210 dup_rows=3252 dup_rows_remote=1928 dedup_ratio=0.033081 router_tokens_per_expert=[0, 0, 3451, 6674, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 58, 0, 0, 70, 0, 0, 0, 8192, 8192, 0, 0, 1203, 0, 0, 0, 0, 0, 0, 0, 0, 196, 0, 4332, 0, 0, 0] payload_rows_per_dst=[6902, 16384, 10856, 15124, 16384, 5842, 14504, 9056] dup_rows_per_dst=[0, 0, 454, 108, 116, 1324, 1250, 0] combine_rows_per_src=[6902, 16384, 10856, 15124, 16384, 5842, 14504, 9056] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94452 payload_rows_remote=78068 dup_rows=3852 dup_rows_remote=1602 dedup_ratio=0.039185 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8192, 670, 0, 1167, 0, 0, 0, 0, 838, 0, 0, 0, 0, 0, 0, 0, 1125, 0, 0, 0, 0, 0, 0, 0, 0, 4121, 0, 0, 0, 0, 654, 0, 8192, 0, 0, 0, 8192, 0, 0, 66, 0, 0, 0, 0, 0, 0, 8150, 0, 0, 0, 0, 0, 0, 0, 682, 0, 7103] payload_rows_per_dst=[16384, 4582, 16384, 16384, 9432, 16300, 0, 14986] dup_rows_per_dst=[0, 768, 2250, 0, 118, 132, 0, 584] combine_rows_per_src=[16384, 4582, 16384, 16384, 9432, 16300, 0, 14986] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=54516 payload_rows_remote=38132 dup_rows=43788 dup_rows_remote=34676 dedup_ratio=0.445435 router_tokens_per_expert=[0, 0, 3254, 6613, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 322, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 64, 0, 0, 29, 0, 0, 0, 8192, 8192, 0, 0, 1500, 0, 0, 0, 0, 0, 0, 0, 0, 194, 0, 4362, 0, 0, 0] payload_rows_per_dst=[6508, 0, 644, 92, 128, 16384, 14376, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 32826, 1850, 9112] combine_rows_per_src=[6508, 0, 644, 92, 128, 16384, 14376, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=213295104 dispatch_meta_bytes=330152 combine_payload_bytes=213295104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=213625256 ingress_bytes=247898112 link_bytes=247898112 token_bytes=426920360 weight_movement_bytes=34603008 total_nvlink_bytes=461523368 weight_to_token_ratio=0.081053 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=38400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.328206380e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97358 payload_rows_remote=80974 dup_rows=946 dup_rows_remote=612 dedup_ratio=0.009623 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7676, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 312, 0, 0, 0, 0, 0, 0, 0, 167, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 214, 0, 0, 0, 0, 92, 0, 0, 0, 0, 255, 63, 0, 7605, 0, 0] payload_rows_per_dst=[15352, 0, 16384, 624, 16384, 16384, 16384, 15846] dup_rows_per_dst=[0, 0, 0, 0, 334, 0, 612, 0] combine_rows_per_src=[15352, 0, 16384, 624, 16384, 16384, 16384, 15846] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91838 payload_rows_remote=75454 dup_rows=6466 dup_rows_remote=6466 dedup_ratio=0.065776 router_tokens_per_expert=[0, 0, 3700, 6476, 0, 0, 0, 0, 2, 0, 0, 0, 8190, 0, 0, 0, 1, 366, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 1, 0, 68, 1, 0, 52, 1, 0, 1, 8189, 8190, 0, 1, 1367, 0, 1, 1, 0, 0, 0, 0, 1, 164, 0, 4100, 0, 0, 0] payload_rows_per_dst=[14934, 16384, 16382, 16380, 16384, 106, 2738, 8530] dup_rows_per_dst=[5418, 0, 730, 178, 136, 2, 2, 0] combine_rows_per_src=[14934, 16384, 16382, 16380, 16384, 106, 2738, 8530] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97310 payload_rows_remote=89698 dup_rows=994 dup_rows_remote=588 dedup_ratio=0.010111 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 388, 0, 0, 0, 0, 0, 0, 0, 180, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0, 0, 92, 0, 0, 0, 0, 250, 63, 0, 7629, 0, 0] payload_rows_per_dst=[15370, 0, 16384, 7612, 9292, 16384, 16384, 15884] dup_rows_per_dst=[0, 0, 0, 406, 210, 0, 378, 0] combine_rows_per_src=[15370, 0, 16384, 7612, 9292, 16384, 16384, 15884] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97306 payload_rows_remote=80922 dup_rows=998 dup_rows_remote=324 dedup_ratio=0.010152 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 230, 0, 0, 0, 0, 0, 0, 0, 162, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 0, 0, 0, 86, 0, 0, 0, 0, 268, 91, 0, 7612, 0, 0] payload_rows_per_dst=[6142, 9226, 16384, 460, 16384, 16384, 16384, 15942] dup_rows_per_dst=[0, 0, 0, 0, 324, 0, 674, 0] combine_rows_per_src=[6142, 9226, 16384, 460, 16384, 16384, 16384, 15942] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77880 payload_rows_remote=61672 dup_rows=20424 dup_rows_remote=1156 dedup_ratio=0.207764 router_tokens_per_expert=[7, 0, 3869, 6429, 0, 0, 0, 1, 32, 0, 0, 0, 8153, 0, 0, 0, 15, 399, 0, 0, 0, 0, 0, 0, 115, 1, 11, 0, 0, 0, 0, 0, 8142, 1, 0, 0, 32, 0, 54, 0, 0, 84, 1, 0, 31, 8146, 8149, 0, 31, 1245, 0, 0, 2, 0, 0, 0, 0, 31, 167, 23, 3974, 0, 0, 7] payload_rows_per_dst=[16208, 8038, 16314, 16330, 9854, 232, 2552, 8352] dup_rows_per_dst=[19268, 0, 806, 222, 72, 0, 4, 52] combine_rows_per_src=[16208, 8038, 16314, 16330, 9854, 232, 2552, 8352] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=365404160 dispatch_meta_bytes=364552 combine_payload_bytes=365404160 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=365768712 ingress_bytes=469213184 link_bytes=469213184 token_bytes=731172872 weight_movement_bytes=103809024 total_nvlink_bytes=834981896 weight_to_token_ratio=0.141976 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=19328 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.765611437e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=322011136 dispatch_meta_bytes=324408 combine_payload_bytes=322011136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=322335544 ingress_bytes=356614144 link_bytes=356614144 token_bytes=644346680 weight_movement_bytes=34603008 total_nvlink_bytes=678949688 weight_to_token_ratio=0.053702 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=46336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.341910314e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95052 payload_rows_remote=89210 dup_rows=3252 dup_rows_remote=1928 dedup_ratio=0.033081 router_tokens_per_expert=[0, 0, 3451, 6674, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 58, 0, 0, 70, 0, 0, 0, 8192, 8192, 0, 0, 1203, 0, 0, 0, 0, 0, 0, 0, 0, 196, 0, 4332, 0, 0, 0] payload_rows_per_dst=[6902, 16384, 10856, 15124, 16384, 5842, 14504, 9056] dup_rows_per_dst=[0, 0, 454, 108, 116, 1324, 1250, 0] combine_rows_per_src=[6902, 16384, 10856, 15124, 16384, 5842, 14504, 9056] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=64688 payload_rows_remote=49374 dup_rows=33616 dup_rows_remote=33506 dedup_ratio=0.341960 router_tokens_per_expert=[0, 0, 17, 23, 35, 15, 0, 4, 1, 8152, 7618, 0, 0, 0, 2, 0, 0, 5, 0, 5, 31, 8145, 3, 0, 6, 22, 0, 0, 8150, 309, 14, 0, 18, 29, 0, 0, 0, 153, 0, 8156, 1, 0, 5, 0, 16, 1, 64, 0, 214, 1, 0, 0, 0, 104, 0, 0, 0, 0, 287, 73, 0, 7473, 0, 0] payload_rows_per_dst=[15314, 16308, 8176, 16338, 7052, 142, 638, 720] dup_rows_per_dst=[110, 16292, 6, 16976, 200, 32, 0, 0] combine_rows_per_src=[15314, 16308, 8176, 16338, 7052, 142, 638, 720] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=319766528 dispatch_meta_bytes=318680 combine_payload_bytes=319766528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=320085208 ingress_bytes=354369536 link_bytes=354369536 token_bytes=639851736 weight_movement_bytes=34603008 total_nvlink_bytes=674454744 weight_to_token_ratio=0.054080 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=41472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.333464035e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156188672 dispatch_meta_bytes=291232 combine_payload_bytes=156188672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=156479904 ingress_bytes=190791680 link_bytes=190791680 token_bytes=312668576 weight_movement_bytes=34603008 total_nvlink_bytes=347271584 weight_to_token_ratio=0.110670 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=34176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=7.179337318e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=331669504 dispatch_meta_bytes=326344 combine_payload_bytes=331669504 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=331995848 ingress_bytes=400875520 link_bytes=400875520 token_bytes=663665352 weight_movement_bytes=69206016 total_nvlink_bytes=732871368 weight_to_token_ratio=0.104278 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=37376 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.508462309e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97358 payload_rows_remote=80974 dup_rows=946 dup_rows_remote=612 dedup_ratio=0.009623 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7676, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 312, 0, 0, 0, 0, 0, 0, 0, 167, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 214, 0, 0, 0, 0, 92, 0, 0, 0, 0, 255, 63, 0, 7605, 0, 0] payload_rows_per_dst=[15352, 0, 16384, 624, 16384, 16384, 16384, 15846] dup_rows_per_dst=[0, 0, 0, 0, 334, 0, 612, 0] combine_rows_per_src=[15352, 0, 16384, 624, 16384, 16384, 16384, 15846] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=309059584 dispatch_meta_bytes=327680 combine_payload_bytes=309059584 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=309387264 ingress_bytes=412868608 link_bytes=412868608 token_bytes=618446848 weight_movement_bytes=103809024 total_nvlink_bytes=722255872 weight_to_token_ratio=0.167854 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=45952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.553591333e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97310 payload_rows_remote=89698 dup_rows=994 dup_rows_remote=588 dedup_ratio=0.010111 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 388, 0, 0, 0, 0, 0, 0, 0, 180, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0, 0, 92, 0, 0, 0, 0, 250, 63, 0, 7629, 0, 0] payload_rows_per_dst=[15370, 0, 16384, 7612, 9292, 16384, 16384, 15884] dup_rows_per_dst=[0, 0, 0, 406, 210, 0, 378, 0] combine_rows_per_src=[15370, 0, 16384, 7612, 9292, 16384, 16384, 15884] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=331456512 dispatch_meta_bytes=324984 combine_payload_bytes=331456512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=331781496 ingress_bytes=366059520 link_bytes=366059520 token_bytes=663238008 weight_movement_bytes=34603008 total_nvlink_bytes=697841016 weight_to_token_ratio=0.052173 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=46848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.377452504e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=365404160 dispatch_meta_bytes=364552 combine_payload_bytes=365404160 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=365768712 ingress_bytes=469213184 link_bytes=469213184 token_bytes=731172872 weight_movement_bytes=103809024 total_nvlink_bytes=834981896 weight_to_token_ratio=0.141976 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=19328 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.765611437e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=202235904 dispatch_meta_bytes=331520 combine_payload_bytes=202235904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=202567424 ingress_bytes=236838912 link_bytes=236838912 token_bytes=404803328 weight_movement_bytes=34603008 total_nvlink_bytes=439406336 weight_to_token_ratio=0.085481 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.912057588e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=331669504 dispatch_meta_bytes=326344 combine_payload_bytes=331669504 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=331995848 ingress_bytes=400875520 link_bytes=400875520 token_bytes=663665352 weight_movement_bytes=69206016 total_nvlink_bytes=732871368 weight_to_token_ratio=0.104278 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=37376 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.508462309e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=367403008 dispatch_meta_bytes=361144 combine_payload_bytes=367403008 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=367764152 ingress_bytes=436609024 link_bytes=436609024 token_bytes=735167160 weight_movement_bytes=69206016 total_nvlink_bytes=804373176 weight_to_token_ratio=0.094136 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=28160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.642924607e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=252608512 dispatch_meta_bytes=251312 combine_payload_bytes=252608512 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=252859824 ingress_bytes=321814528 link_bytes=321814528 token_bytes=505468336 weight_movement_bytes=69206016 total_nvlink_bytes=574674352 weight_to_token_ratio=0.136915 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=27904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.210962161e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97306 payload_rows_remote=80922 dup_rows=998 dup_rows_remote=324 dedup_ratio=0.010152 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 230, 0, 0, 0, 0, 0, 0, 0, 162, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 0, 0, 0, 86, 0, 0, 0, 0, 268, 91, 0, 7612, 0, 0] payload_rows_per_dst=[6142, 9226, 16384, 460, 16384, 16384, 16384, 15942] dup_rows_per_dst=[0, 0, 0, 0, 324, 0, 674, 0] combine_rows_per_src=[6142, 9226, 16384, 460, 16384, 16384, 16384, 15942] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12941903184 prefetch_bytes=692060160 reduce_bytes=726663168 weight_movement_bytes=1418723328 egress_bytes=6474786128 ingress_bytes=7885840384 link_bytes=7885840384 total_nvlink_bytes=14360626512 weight_to_token_ratio=0.109622 prefetch_share_of_ingress=0.087760 dedup_ratio=0.173323 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=21 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028804 gemm_inner_pad_upper_bound=22098 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.290164457e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=11579943400 prefetch_bytes=830472192 reduce_bytes=519045120 weight_movement_bytes=1349517312 egress_bytes=5793597928 ingress_bytes=7135862784 link_bytes=7135862784 total_nvlink_bytes=12929460712 weight_to_token_ratio=0.116539 prefetch_share_of_ingress=0.116380 dedup_ratio=0.227370 prefetch_slots=48 unused_prefetch_slots=320 slot_utilization=0.130435 reduce_slots=15 remote_demand=24 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029239 gemm_inner_pad_upper_bound=22098 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.167464225e-04 prefetch_bytes_per_gemm_flop=1.358695652e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95000 payload_rows_remote=78616 dup_rows=3304 dup_rows_remote=2486 dedup_ratio=0.033610 router_tokens_per_expert=[0, 0, 3226, 6587, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 409, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 61, 0, 0, 81, 0, 0, 0, 8192, 8192, 0, 0, 1451, 0, 0, 0, 0, 0, 0, 0, 0, 191, 0, 4289, 0, 0, 0] payload_rows_per_dst=[7812, 16384, 16384, 16384, 16384, 162, 12530, 8960] dup_rows_per_dst=[690, 0, 818, 178, 122, 0, 1496, 0] combine_rows_per_src=[7812, 16384, 16384, 16384, 16384, 162, 12530, 8960] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97394 payload_rows_remote=81382 dup_rows=910 dup_rows_remote=910 dedup_ratio=0.009257 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 251, 0, 0, 0, 0, 0, 0, 0, 193, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 216, 0, 0, 0, 0, 46, 0, 0, 0, 0, 264, 35, 0, 7707, 0, 0] payload_rows_per_dst=[0, 15344, 16384, 502, 16384, 16384, 16384, 16012] dup_rows_per_dst=[0, 0, 0, 0, 386, 0, 524, 0] combine_rows_per_src=[0, 15344, 16384, 502, 16384, 16384, 16384, 16012] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14554394376 prefetch_bytes=795869184 reduce_bytes=692060160 weight_movement_bytes=1487929344 egress_bytes=7281110792 ingress_bytes=8761212928 link_bytes=8761212928 total_nvlink_bytes=16042323720 weight_to_token_ratio=0.102232 prefetch_share_of_ingress=0.090840 dedup_ratio=0.089974 prefetch_slots=46 unused_prefetch_slots=322 slot_utilization=0.125000 reduce_slots=20 remote_demand=23 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029239 gemm_inner_pad_upper_bound=22860 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.433379953e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14739811392 prefetch_bytes=726663168 reduce_bytes=622854144 weight_movement_bytes=1349517312 egress_bytes=7373843520 ingress_bytes=8715485184 link_bytes=8715485184 total_nvlink_bytes=16089328704 weight_to_token_ratio=0.091556 prefetch_share_of_ingress=0.083376 dedup_ratio=0.085407 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=18 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029348 gemm_inner_pad_upper_bound=22098 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.425898656e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97250 payload_rows_remote=80866 dup_rows=1054 dup_rows_remote=1054 dedup_ratio=0.010722 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 240, 0, 0, 0, 0, 0, 0, 0, 200, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 266, 0, 0, 0, 0, 61, 0, 0, 0, 0, 293, 72, 0, 7604, 0, 0] payload_rows_per_dst=[15296, 0, 16384, 480, 16384, 16384, 16384, 15938] dup_rows_per_dst=[0, 0, 0, 0, 400, 0, 654, 0] combine_rows_per_src=[15296, 0, 16384, 480, 16384, 16384, 16384, 15938] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92446 payload_rows_remote=84246 dup_rows=5858 dup_rows_remote=5858 dedup_ratio=0.059591 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 260, 0, 0, 0, 0, 0, 0, 0, 189, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 206, 0, 0, 0, 0, 90, 0, 0, 0, 0, 293, 80, 0, 7599, 0, 0] payload_rows_per_dst=[15334, 8184, 8200, 16384, 378, 16208, 11814, 15944] dup_rows_per_dst=[0, 176, 0, 5240, 0, 0, 442, 0] combine_rows_per_src=[15334, 8184, 8200, 16384, 378, 16208, 11814, 15944] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=331456512 dispatch_meta_bytes=324984 combine_payload_bytes=331456512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=331781496 ingress_bytes=366059520 link_bytes=366059520 token_bytes=663238008 weight_movement_bytes=34603008 total_nvlink_bytes=697841016 weight_to_token_ratio=0.052173 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=46848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.377452504e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97394 payload_rows_remote=81382 dup_rows=910 dup_rows_remote=910 dedup_ratio=0.009257 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 251, 0, 0, 0, 0, 0, 0, 0, 193, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 216, 0, 0, 0, 0, 46, 0, 0, 0, 0, 264, 35, 0, 7707, 0, 0] payload_rows_per_dst=[0, 15344, 16384, 502, 16384, 16384, 16384, 16012] dup_rows_per_dst=[0, 0, 0, 0, 386, 0, 524, 0] combine_rows_per_src=[0, 15344, 16384, 502, 16384, 16384, 16384, 16012] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=64712 payload_rows_remote=48330 dup_rows=33592 dup_rows_remote=17200 dedup_ratio=0.341715 router_tokens_per_expert=[0, 0, 0, 1, 1, 1, 0, 0, 2, 8191, 7690, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 8190, 0, 0, 1, 1, 0, 0, 8188, 262, 1, 0, 1, 1, 0, 0, 0, 164, 0, 8188, 0, 0, 0, 0, 1, 0, 0, 0, 234, 0, 0, 1, 0, 89, 0, 0, 0, 0, 263, 71, 0, 7605, 0, 0] payload_rows_per_dst=[15382, 16382, 2, 16382, 13896, 2, 648, 2018] dup_rows_per_dst=[4, 16392, 0, 16900, 296, 0, 0, 0] combine_rows_per_src=[15382, 16382, 2, 16382, 13896, 2, 648, 2018] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=331227136 dispatch_meta_bytes=327680 combine_payload_bytes=331227136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=331554816 ingress_bytes=365830144 link_bytes=365830144 token_bytes=662781952 weight_movement_bytes=34603008 total_nvlink_bytes=697384960 weight_to_token_ratio=0.052209 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.376589380e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=345071616 dispatch_meta_bytes=360416 combine_payload_bytes=345071616 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=345432032 ingress_bytes=414277632 link_bytes=414277632 token_bytes=690503648 weight_movement_bytes=69206016 total_nvlink_bytes=759709664 weight_to_token_ratio=0.100225 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=45184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.558893377e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12941903184 prefetch_bytes=692060160 reduce_bytes=726663168 weight_movement_bytes=1418723328 egress_bytes=6474786128 ingress_bytes=7885840384 link_bytes=7885840384 total_nvlink_bytes=14360626512 weight_to_token_ratio=0.109622 prefetch_share_of_ingress=0.087760 dedup_ratio=0.173323 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=21 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028804 gemm_inner_pad_upper_bound=22098 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.290164457e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=64712 payload_rows_remote=48330 dup_rows=33592 dup_rows_remote=17200 dedup_ratio=0.341715 router_tokens_per_expert=[0, 0, 0, 1, 1, 1, 0, 0, 2, 8191, 7690, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 8190, 0, 0, 1, 1, 0, 0, 8188, 262, 1, 0, 1, 1, 0, 0, 0, 164, 0, 8188, 0, 0, 0, 0, 1, 0, 0, 0, 234, 0, 0, 1, 0, 89, 0, 0, 0, 0, 263, 71, 0, 7605, 0, 0] payload_rows_per_dst=[15382, 16382, 2, 16382, 13896, 2, 648, 2018] dup_rows_per_dst=[4, 16392, 0, 16900, 296, 0, 0, 0] combine_rows_per_src=[15382, 16382, 2, 16382, 13896, 2, 648, 2018] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97250 payload_rows_remote=80866 dup_rows=1054 dup_rows_remote=1054 dedup_ratio=0.010722 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 240, 0, 0, 0, 0, 0, 0, 0, 200, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 266, 0, 0, 0, 0, 61, 0, 0, 0, 0, 293, 72, 0, 7604, 0, 0] payload_rows_per_dst=[15296, 0, 16384, 480, 16384, 16384, 16384, 15938] dup_rows_per_dst=[0, 0, 0, 0, 400, 0, 654, 0] combine_rows_per_src=[15296, 0, 16384, 480, 16384, 16384, 16384, 15938] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=322011136 dispatch_meta_bytes=324408 combine_payload_bytes=322011136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=322335544 ingress_bytes=356614144 link_bytes=356614144 token_bytes=644346680 weight_movement_bytes=34603008 total_nvlink_bytes=678949688 weight_to_token_ratio=0.053702 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=46336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.341910314e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=333340672 dispatch_meta_bytes=329168 combine_payload_bytes=333340672 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=333669840 ingress_bytes=402546688 link_bytes=402546688 token_bytes=667010512 weight_movement_bytes=69206016 total_nvlink_bytes=736216528 weight_to_token_ratio=0.103756 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.037500 gemm_active_groups=4 gemm_peak_group_rows=46464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.514750779e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=64688 payload_rows_remote=49374 dup_rows=33616 dup_rows_remote=33506 dedup_ratio=0.341960 router_tokens_per_expert=[0, 0, 17, 23, 35, 15, 0, 4, 1, 8152, 7618, 0, 0, 0, 2, 0, 0, 5, 0, 5, 31, 8145, 3, 0, 6, 22, 0, 0, 8150, 309, 14, 0, 18, 29, 0, 0, 0, 153, 0, 8156, 1, 0, 5, 0, 16, 1, 64, 0, 214, 1, 0, 0, 0, 104, 0, 0, 0, 0, 287, 73, 0, 7473, 0, 0] payload_rows_per_dst=[15314, 16308, 8176, 16338, 7052, 142, 638, 720] dup_rows_per_dst=[110, 16292, 6, 16976, 200, 32, 0, 0] combine_rows_per_src=[15314, 16308, 8176, 16338, 7052, 142, 638, 720] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=197959680 dispatch_meta_bytes=262120 combine_payload_bytes=197959680 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=198221800 ingress_bytes=301768704 link_bytes=301768704 token_bytes=396181480 weight_movement_bytes=103809024 total_nvlink_bytes=499990504 weight_to_token_ratio=0.262024 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=20480 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135531339e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14264770808 prefetch_bytes=692060160 reduce_bytes=657457152 weight_movement_bytes=1349517312 egress_bytes=7136231672 ingress_bytes=8478056448 link_bytes=8478056448 total_nvlink_bytes=15614288120 weight_to_token_ratio=0.094605 prefetch_share_of_ingress=0.081630 dedup_ratio=0.103379 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=19 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=23876 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.387054081e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14597572344 prefetch_bytes=761266176 reduce_bytes=968884224 weight_movement_bytes=1730150400 egress_bytes=7302694648 ingress_bytes=9025028096 link_bytes=9025028096 total_nvlink_bytes=16327722744 weight_to_token_ratio=0.118523 prefetch_share_of_ingress=0.084351 dedup_ratio=0.097700 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=28 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=22987 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.476541485e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=331227136 dispatch_meta_bytes=327680 combine_payload_bytes=331227136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=331554816 ingress_bytes=365830144 link_bytes=365830144 token_bytes=662781952 weight_movement_bytes=34603008 total_nvlink_bytes=697384960 weight_to_token_ratio=0.052209 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.376589380e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=11806888608 prefetch_bytes=588251136 reduce_bytes=795869184 weight_movement_bytes=1384120320 egress_bytes=5907051168 ingress_bytes=7283957760 link_bytes=7283957760 total_nvlink_bytes=13191008928 weight_to_token_ratio=0.117230 prefetch_share_of_ingress=0.080760 dedup_ratio=0.217075 prefetch_slots=34 unused_prefetch_slots=334 slot_utilization=0.092391 reduce_slots=23 remote_demand=17 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029022 gemm_inner_pad_upper_bound=22225 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.191693332e-04 prefetch_bytes_per_gemm_flop=9.624094203e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=202235904 dispatch_meta_bytes=331520 combine_payload_bytes=202235904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=202567424 ingress_bytes=236838912 link_bytes=236838912 token_bytes=404803328 weight_movement_bytes=34603008 total_nvlink_bytes=439406336 weight_to_token_ratio=0.085481 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.912057588e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12513027392 prefetch_bytes=761266176 reduce_bytes=865075200 weight_movement_bytes=1626341376 egress_bytes=6260204864 ingress_bytes=7879163904 link_bytes=7879163904 total_nvlink_bytes=14139368768 weight_to_token_ratio=0.129972 prefetch_share_of_ingress=0.096618 dedup_ratio=0.187887 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=25 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029565 gemm_inner_pad_upper_bound=22225 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.289072150e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=11579943400 prefetch_bytes=830472192 reduce_bytes=519045120 weight_movement_bytes=1349517312 egress_bytes=5793597928 ingress_bytes=7135862784 link_bytes=7135862784 total_nvlink_bytes=12929460712 weight_to_token_ratio=0.116539 prefetch_share_of_ingress=0.116380 dedup_ratio=0.227370 prefetch_slots=48 unused_prefetch_slots=320 slot_utilization=0.130435 reduce_slots=15 remote_demand=24 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029239 gemm_inner_pad_upper_bound=22098 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.167464225e-04 prefetch_bytes_per_gemm_flop=1.358695652e-05 anomalies=none - [2026-08-26 15:09:11] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 796.1 | throughput per GPU (TFLOP/s/GPU): 160.3 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 8.982457E+00 | loss scale: 1.0 | grad norm: 3.717 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74738 payload_rows_remote=64918 dup_rows=23566 dup_rows_remote=20474 dedup_ratio=0.239726 router_tokens_per_expert=[1154, 449, 744, 1315, 581, 1125, 1094, 563, 1589, 564, 1245, 319, 87, 1889, 888, 952, 710, 967, 904, 730, 186, 525, 339, 1168, 375, 756, 750, 425, 862, 116, 729, 554, 1039, 738, 1504, 1002, 641, 254, 532, 827, 463, 603, 601, 495, 1050, 593, 601, 324, 611, 509, 771, 653, 927, 1630, 527, 638, 1436, 366, 1473, 724, 292, 698, 1204, 772] payload_rows_per_dst=[10260, 7884, 8778, 9820, 9858, 9172, 9560, 9406] dup_rows_per_dst=[3790, 3404, 2280, 3092, 3216, 3160, 2972, 1652] combine_rows_per_src=[10260, 7884, 8778, 9820, 9858, 9172, 9560, 9406] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74616 payload_rows_remote=65256 dup_rows=23688 dup_rows_remote=20308 dedup_ratio=0.240967 router_tokens_per_expert=[1204, 385, 647, 1088, 593, 1126, 1030, 441, 1698, 855, 1176, 235, 74, 1893, 876, 981, 803, 967, 1021, 747, 188, 502, 340, 1203, 442, 742, 797, 525, 879, 162, 656, 495, 1025, 711, 1588, 1069, 382, 257, 671, 808, 391, 483, 607, 371, 1189, 803, 656, 354, 738, 539, 697, 467, 846, 1574, 435, 592, 1516, 448, 1493, 568, 323, 797, 1205, 778] payload_rows_per_dst=[8118, 8058, 10474, 10050, 9600, 9360, 9354, 9602] dup_rows_per_dst=[2602, 3732, 3376, 3132, 3360, 3380, 2422, 1684] combine_rows_per_src=[8118, 8058, 10474, 10050, 9600, 9360, 9354, 9602] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74764 payload_rows_remote=65574 dup_rows=23540 dup_rows_remote=21214 dedup_ratio=0.239461 router_tokens_per_expert=[1174, 445, 801, 1296, 526, 1123, 919, 591, 1562, 637, 1060, 278, 73, 1948, 847, 901, 820, 1177, 831, 699, 218, 461, 372, 1180, 364, 728, 843, 405, 846, 117, 810, 580, 1062, 834, 1504, 1061, 496, 244, 648, 727, 391, 534, 620, 415, 1196, 735, 536, 316, 675, 528, 642, 598, 1037, 1674, 480, 576, 1560, 374, 1497, 630, 174, 724, 1286, 746] payload_rows_per_dst=[10032, 7418, 9190, 9988, 9682, 9352, 9692, 9410] dup_rows_per_dst=[3718, 3298, 2326, 3294, 3470, 3254, 2728, 1452] combine_rows_per_src=[10032, 7418, 9190, 9988, 9682, 9352, 9692, 9410] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74768 payload_rows_remote=65776 dup_rows=23536 dup_rows_remote=19214 dedup_ratio=0.239421 router_tokens_per_expert=[1222, 546, 702, 1162, 528, 1238, 862, 524, 1554, 842, 1099, 306, 106, 1885, 767, 916, 795, 1129, 864, 684, 200, 504, 346, 1268, 430, 667, 808, 340, 848, 144, 764, 541, 1050, 838, 1531, 1042, 387, 219, 673, 758, 369, 471, 607, 408, 1052, 723, 666, 283, 609, 525, 710, 582, 974, 1574, 559, 607, 1738, 433, 1536, 584, 207, 760, 1235, 851] payload_rows_per_dst=[9938, 8992, 9254, 8534, 9530, 9344, 9546, 9630] dup_rows_per_dst=[3630, 4322, 2326, 2186, 3466, 3290, 2734, 1582] combine_rows_per_src=[9938, 8992, 9254, 8534, 9530, 9344, 9546, 9630] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14264770808 prefetch_bytes=692060160 reduce_bytes=657457152 weight_movement_bytes=1349517312 egress_bytes=7136231672 ingress_bytes=8478056448 link_bytes=8478056448 total_nvlink_bytes=15614288120 weight_to_token_ratio=0.094605 prefetch_share_of_ingress=0.081630 dedup_ratio=0.103379 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=19 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=23876 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.387054081e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92446 payload_rows_remote=84246 dup_rows=5858 dup_rows_remote=5858 dedup_ratio=0.059591 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 260, 0, 0, 0, 0, 0, 0, 0, 189, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 206, 0, 0, 0, 0, 90, 0, 0, 0, 0, 293, 80, 0, 7599, 0, 0] payload_rows_per_dst=[15334, 8184, 8200, 16384, 378, 16208, 11814, 15944] dup_rows_per_dst=[0, 176, 0, 5240, 0, 0, 442, 0] combine_rows_per_src=[15334, 8184, 8200, 16384, 378, 16208, 11814, 15944] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=345071616 dispatch_meta_bytes=360416 combine_payload_bytes=345071616 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=345432032 ingress_bytes=414277632 link_bytes=414277632 token_bytes=690503648 weight_movement_bytes=69206016 total_nvlink_bytes=759709664 weight_to_token_ratio=0.100225 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=45184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.558893377e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14597572344 prefetch_bytes=761266176 reduce_bytes=968884224 weight_movement_bytes=1730150400 egress_bytes=7302694648 ingress_bytes=9025028096 link_bytes=9025028096 total_nvlink_bytes=16327722744 weight_to_token_ratio=0.118523 prefetch_share_of_ingress=0.084351 dedup_ratio=0.097700 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=28 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=22987 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.476541485e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none + [2026-08-26 15:56:59] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 794.0 | throughput per GPU (TFLOP/s/GPU): 160.7 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 8.982457E+00 | loss scale: 1.0 | grad norm: 3.717 | number of skipped iterations: 0 | number of nan iterations: 0 | WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74708 payload_rows_remote=65280 dup_rows=23596 dup_rows_remote=20280 dedup_ratio=0.240031 router_tokens_per_expert=[1247, 516, 622, 1184, 488, 1046, 955, 595, 1625, 877, 1139, 307, 99, 1887, 806, 883, 746, 1139, 841, 655, 201, 498, 339, 1263, 447, 767, 874, 421, 865, 123, 730, 483, 1074, 769, 1420, 1066, 398, 228, 636, 781, 457, 530, 672, 365, 1156, 667, 634, 326, 691, 565, 720, 592, 916, 1644, 507, 600, 1567, 413, 1392, 673, 192, 721, 1308, 804] payload_rows_per_dst=[9832, 7708, 9142, 10064, 9428, 9408, 9642, 9484] dup_rows_per_dst=[3474, 3764, 2222, 3130, 3316, 3340, 2828, 1522] combine_rows_per_src=[9832, 7708, 9142, 10064, 9428, 9408, 9642, 9484] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74738 payload_rows_remote=64918 dup_rows=23566 dup_rows_remote=20474 dedup_ratio=0.239726 router_tokens_per_expert=[1154, 449, 744, 1315, 581, 1125, 1094, 563, 1589, 564, 1245, 319, 87, 1889, 888, 952, 710, 967, 904, 730, 186, 525, 339, 1168, 375, 756, 750, 425, 862, 116, 729, 554, 1039, 738, 1504, 1002, 641, 254, 532, 827, 463, 603, 601, 495, 1050, 593, 601, 324, 611, 509, 771, 653, 927, 1630, 527, 638, 1436, 366, 1473, 724, 292, 698, 1204, 772] payload_rows_per_dst=[10260, 7884, 8778, 9820, 9858, 9172, 9560, 9406] dup_rows_per_dst=[3790, 3404, 2280, 3092, 3216, 3160, 2972, 1652] combine_rows_per_src=[10260, 7884, 8778, 9820, 9858, 9172, 9560, 9406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74548 payload_rows_remote=64852 dup_rows=23756 dup_rows_remote=20816 dedup_ratio=0.241659 router_tokens_per_expert=[1247, 525, 807, 1295, 500, 1161, 847, 506, 1562, 713, 1075, 286, 93, 1809, 736, 995, 757, 1088, 800, 624, 233, 593, 328, 1370, 428, 716, 762, 291, 850, 138, 778, 565, 1011, 871, 1380, 1004, 582, 209, 617, 735, 484, 571, 665, 458, 1109, 706, 648, 333, 578, 581, 701, 674, 952, 1520, 607, 705, 1511, 377, 1435, 671, 186, 683, 1249, 861] payload_rows_per_dst=[7996, 8150, 10844, 9666, 8230, 9360, 9696, 10606] dup_rows_per_dst=[2530, 3526, 3236, 3008, 1828, 3094, 2940, 3594] combine_rows_per_src=[7996, 8150, 10844, 9666, 8230, 9360, 9696, 10606] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74162 payload_rows_remote=61938 dup_rows=24142 dup_rows_remote=18898 dedup_ratio=0.245585 router_tokens_per_expert=[1212, 479, 648, 1089, 623, 1300, 1082, 550, 1499, 816, 1152, 388, 91, 1757, 877, 949, 733, 931, 877, 764, 181, 499, 330, 1057, 411, 712, 826, 440, 944, 168, 637, 502, 1158, 801, 1455, 1110, 365, 231, 665, 703, 461, 528, 645, 377, 1105, 673, 662, 330, 661, 565, 663, 566, 1005, 1481, 506, 633, 1381, 474, 1550, 714, 307, 873, 1257, 723] payload_rows_per_dst=[6600, 9528, 10010, 9924, 8524, 7846, 9506, 12224] dup_rows_per_dst=[2046, 4194, 3158, 2870, 2200, 1716, 2714, 5244] combine_rows_per_src=[6600, 9528, 10010, 9924, 8524, 7846, 9506, 12224] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74616 payload_rows_remote=65256 dup_rows=23688 dup_rows_remote=20308 dedup_ratio=0.240967 router_tokens_per_expert=[1204, 385, 647, 1088, 593, 1126, 1030, 441, 1698, 855, 1176, 235, 74, 1893, 876, 981, 803, 967, 1021, 747, 188, 502, 340, 1203, 442, 742, 797, 525, 879, 162, 656, 495, 1025, 711, 1588, 1069, 382, 257, 671, 808, 391, 483, 607, 371, 1189, 803, 656, 354, 738, 539, 697, 467, 846, 1574, 435, 592, 1516, 448, 1493, 568, 323, 797, 1205, 778] payload_rows_per_dst=[8118, 8058, 10474, 10050, 9600, 9360, 9354, 9602] dup_rows_per_dst=[2602, 3732, 3376, 3132, 3360, 3380, 2422, 1684] combine_rows_per_src=[8118, 8058, 10474, 10050, 9600, 9360, 9354, 9602] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74982 payload_rows_remote=64994 dup_rows=23322 dup_rows_remote=19572 dedup_ratio=0.237244 router_tokens_per_expert=[1237, 449, 766, 1169, 564, 1054, 1013, 617, 1517, 509, 1084, 280, 112, 1741, 904, 1016, 721, 1087, 949, 748, 189, 473, 335, 1342, 476, 750, 722, 374, 988, 126, 752, 585, 997, 779, 1335, 968, 662, 200, 579, 778, 421, 560, 587, 421, 1127, 775, 726, 358, 593, 494, 696, 607, 992, 1479, 498, 626, 1639, 420, 1417, 689, 276, 777, 1220, 807] payload_rows_per_dst=[9988, 9922, 9172, 7958, 9564, 9620, 9290, 9468] dup_rows_per_dst=[3750, 4404, 2516, 1588, 3032, 3608, 2680, 1744] combine_rows_per_src=[9988, 9922, 9172, 7958, 9564, 9620, 9290, 9468] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=341568 combine_payload_bytes=265904128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266245696 ingress_bytes=300507136 link_bytes=300507136 token_bytes=532149824 weight_movement_bytes=34603008 total_nvlink_bytes=566752832 weight_to_token_ratio=0.065025 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130784160e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267288576 dispatch_meta_bytes=342256 combine_payload_bytes=267288576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267630832 ingress_bytes=301891584 link_bytes=301891584 token_bytes=534919408 weight_movement_bytes=34603008 total_nvlink_bytes=569522416 weight_to_token_ratio=0.064688 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135993726e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=268591104 dispatch_meta_bytes=347152 combine_payload_bytes=268591104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268938256 ingress_bytes=303194112 link_bytes=303194112 token_bytes=537529360 weight_movement_bytes=34603008 total_nvlink_bytes=572132368 weight_to_token_ratio=0.064374 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9856 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.140895034e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74768 payload_rows_remote=65776 dup_rows=23536 dup_rows_remote=19214 dedup_ratio=0.239421 router_tokens_per_expert=[1222, 546, 702, 1162, 528, 1238, 862, 524, 1554, 842, 1099, 306, 106, 1885, 767, 916, 795, 1129, 864, 684, 200, 504, 346, 1268, 430, 667, 808, 340, 848, 144, 764, 541, 1050, 838, 1531, 1042, 387, 219, 673, 758, 369, 471, 607, 408, 1052, 723, 666, 283, 609, 525, 710, 582, 974, 1574, 559, 607, 1738, 433, 1536, 584, 207, 760, 1235, 851] payload_rows_per_dst=[9938, 8992, 9254, 8534, 9530, 9344, 9546, 9630] dup_rows_per_dst=[3630, 4322, 2326, 2186, 3466, 3290, 2734, 1582] combine_rows_per_src=[9938, 8992, 9254, 8534, 9530, 9344, 9546, 9630] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=269418496 dispatch_meta_bytes=339960 combine_payload_bytes=269418496 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=269758456 ingress_bytes=338624512 link_bytes=338624512 token_bytes=539176952 weight_movement_bytes=69206016 total_nvlink_bytes=608382968 weight_to_token_ratio=0.128355 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274216777e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267386880 dispatch_meta_bytes=342240 combine_payload_bytes=267386880 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267729120 ingress_bytes=336592896 link_bytes=336592896 token_bytes=535116000 weight_movement_bytes=69206016 total_nvlink_bytes=604322016 weight_to_token_ratio=0.129329 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=8960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266571970e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267386880 dispatch_meta_bytes=342240 combine_payload_bytes=267386880 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267729120 ingress_bytes=336592896 link_bytes=336592896 token_bytes=535116000 weight_movement_bytes=69206016 total_nvlink_bytes=604322016 weight_to_token_ratio=0.129329 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=8960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266571970e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=341568 combine_payload_bytes=265904128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266245696 ingress_bytes=300507136 link_bytes=300507136 token_bytes=532149824 weight_movement_bytes=34603008 total_nvlink_bytes=566752832 weight_to_token_ratio=0.065025 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130784160e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265633792 dispatch_meta_bytes=342672 combine_payload_bytes=265633792 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265976464 ingress_bytes=300236800 link_bytes=300236800 token_bytes=531610256 weight_movement_bytes=34603008 total_nvlink_bytes=566213264 weight_to_token_ratio=0.065091 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129766907e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=253698048 dispatch_meta_bytes=323344 combine_payload_bytes=253698048 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=254021392 ingress_bytes=322904064 link_bytes=322904064 token_bytes=507719440 weight_movement_bytes=69206016 total_nvlink_bytes=576925456 weight_to_token_ratio=0.136308 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.215061997e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74548 payload_rows_remote=64852 dup_rows=23756 dup_rows_remote=20816 dedup_ratio=0.241659 router_tokens_per_expert=[1247, 525, 807, 1295, 500, 1161, 847, 506, 1562, 713, 1075, 286, 93, 1809, 736, 995, 757, 1088, 800, 624, 233, 593, 328, 1370, 428, 716, 762, 291, 850, 138, 778, 565, 1011, 871, 1380, 1004, 582, 209, 617, 735, 484, 571, 665, 458, 1109, 706, 648, 333, 578, 581, 701, 674, 952, 1520, 607, 705, 1511, 377, 1435, 671, 186, 683, 1249, 861] payload_rows_per_dst=[7996, 8150, 10844, 9666, 8230, 9360, 9696, 10606] dup_rows_per_dst=[2530, 3526, 3236, 3008, 1828, 3094, 2940, 3594] combine_rows_per_src=[7996, 8150, 10844, 9666, 8230, 9360, 9696, 10606] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267288576 dispatch_meta_bytes=342256 combine_payload_bytes=267288576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267630832 ingress_bytes=301891584 link_bytes=301891584 token_bytes=534919408 weight_movement_bytes=34603008 total_nvlink_bytes=569522416 weight_to_token_ratio=0.064688 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135993726e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266215424 dispatch_meta_bytes=338264 combine_payload_bytes=266215424 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266553688 ingress_bytes=404627456 link_bytes=404627456 token_bytes=532769112 weight_movement_bytes=138412032 total_nvlink_bytes=671181144 weight_to_token_ratio=0.259797 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=8832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522580542e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74764 payload_rows_remote=65574 dup_rows=23540 dup_rows_remote=21214 dedup_ratio=0.239461 router_tokens_per_expert=[1174, 445, 801, 1296, 526, 1123, 919, 591, 1562, 637, 1060, 278, 73, 1948, 847, 901, 820, 1177, 831, 699, 218, 461, 372, 1180, 364, 728, 843, 405, 846, 117, 810, 580, 1062, 834, 1504, 1061, 496, 244, 648, 727, 391, 534, 620, 415, 1196, 735, 536, 316, 675, 528, 642, 598, 1037, 1674, 480, 576, 1560, 374, 1497, 630, 174, 724, 1286, 746] payload_rows_per_dst=[10032, 7418, 9190, 9988, 9682, 9352, 9692, 9410] dup_rows_per_dst=[3718, 3298, 2326, 3294, 3470, 3254, 2728, 1452] combine_rows_per_src=[10032, 7418, 9190, 9988, 9682, 9352, 9692, 9410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=269418496 dispatch_meta_bytes=339960 combine_payload_bytes=269418496 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=269758456 ingress_bytes=338624512 link_bytes=338624512 token_bytes=539176952 weight_movement_bytes=69206016 total_nvlink_bytes=608382968 weight_to_token_ratio=0.128355 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274216777e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76048 payload_rows_remote=66518 dup_rows=22256 dup_rows_remote=19742 dedup_ratio=0.226400 router_tokens_per_expert=[423, 654, 353, 422, 1818, 213, 612, 902, 30, 679, 2404, 326, 622, 849, 550, 1951, 571, 1289, 665, 218, 1673, 212, 1998, 1642, 752, 682, 898, 1295, 15, 172, 112, 137, 180, 31, 276, 34, 1009, 561, 997, 936, 1824, 1116, 450, 1152, 163, 1924, 8, 165, 2230, 741, 1689, 0, 40, 1330, 180, 1824, 1373, 527, 967, 235, 1065, 320, 356, 310] payload_rows_per_dst=[8488, 9526, 9712, 9918, 9530, 10042, 9330, 9502] dup_rows_per_dst=[2306, 3038, 2828, 2668, 2514, 3562, 2278, 3062] combine_rows_per_src=[8488, 9526, 9712, 9918, 9530, 10042, 9330, 9502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=268591104 dispatch_meta_bytes=347152 combine_payload_bytes=268591104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268938256 ingress_bytes=303194112 link_bytes=303194112 token_bytes=537529360 weight_movement_bytes=34603008 total_nvlink_bytes=572132368 weight_to_token_ratio=0.064374 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9856 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.140895034e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75256 payload_rows_remote=65986 dup_rows=23048 dup_rows_remote=20770 dedup_ratio=0.234456 router_tokens_per_expert=[434, 844, 455, 381, 1794, 215, 557, 1110, 19, 727, 2477, 345, 590, 892, 641, 2035, 565, 1306, 660, 243, 1783, 244, 2011, 1474, 808, 628, 815, 1119, 15, 188, 90, 139, 222, 51, 294, 97, 936, 717, 852, 1016, 1733, 1148, 477, 1021, 158, 1807, 10, 193, 2109, 766, 1724, 0, 53, 1300, 182, 1749, 1216, 613, 889, 144, 1169, 291, 321, 290] payload_rows_per_dst=[8912, 6606, 10666, 9406, 9964, 9670, 9270, 10762] dup_rows_per_dst=[2668, 1594, 3758, 2416, 2852, 3424, 2278, 4058] combine_rows_per_src=[8912, 6606, 10666, 9406, 9964, 9670, 9270, 10762] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75886 payload_rows_remote=66332 dup_rows=22418 dup_rows_remote=19910 dedup_ratio=0.228048 router_tokens_per_expert=[430, 771, 370, 423, 1471, 228, 550, 1073, 34, 743, 2272, 312, 686, 917, 655, 1918, 599, 1300, 693, 293, 1722, 211, 2072, 1382, 804, 572, 754, 1315, 15, 196, 141, 116, 238, 36, 323, 96, 904, 704, 993, 958, 2023, 1146, 544, 1173, 146, 1779, 13, 216, 2118, 728, 1494, 1, 57, 1373, 213, 1802, 1325, 306, 901, 410, 1140, 317, 320, 317] payload_rows_per_dst=[8430, 10946, 9668, 9554, 9852, 10268, 9110, 8058] dup_rows_per_dst=[2202, 4128, 2732, 2508, 2796, 3812, 2226, 2014] combine_rows_per_src=[8430, 10946, 9668, 9554, 9852, 10268, 9110, 8058] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265633792 dispatch_meta_bytes=342672 combine_payload_bytes=265633792 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265976464 ingress_bytes=300236800 link_bytes=300236800 token_bytes=531610256 weight_movement_bytes=34603008 total_nvlink_bytes=566213264 weight_to_token_ratio=0.065091 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129766907e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75370 payload_rows_remote=64342 dup_rows=22934 dup_rows_remote=18576 dedup_ratio=0.233297 router_tokens_per_expert=[484, 641, 286, 417, 1894, 170, 556, 1064, 26, 825, 2526, 391, 558, 900, 535, 1995, 647, 1191, 647, 424, 1435, 251, 1869, 1600, 797, 599, 821, 1394, 14, 310, 134, 166, 237, 37, 279, 58, 881, 555, 870, 888, 1777, 949, 395, 919, 155, 1914, 4, 226, 2400, 751, 1892, 0, 57, 1068, 197, 1909, 1436, 587, 946, 153, 1022, 345, 366, 312] payload_rows_per_dst=[8818, 5214, 8196, 8874, 10922, 9568, 12750, 11028] dup_rows_per_dst=[2206, 1256, 1426, 1952, 3296, 3110, 5330, 4358] combine_rows_per_src=[8818, 5214, 8196, 8874, 10922, 9568, 12750, 11028] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76146 payload_rows_remote=65950 dup_rows=22158 dup_rows_remote=18614 dedup_ratio=0.225403 router_tokens_per_expert=[458, 647, 303, 465, 1945, 136, 496, 1268, 24, 728, 2378, 380, 522, 788, 523, 1975, 637, 1203, 583, 327, 1495, 262, 1893, 1789, 824, 651, 853, 1405, 29, 289, 87, 98, 110, 26, 271, 34, 879, 514, 880, 903, 1839, 1133, 363, 1048, 137, 2127, 6, 217, 2719, 666, 1940, 0, 64, 1033, 141, 1695, 1404, 405, 1088, 282, 757, 324, 324, 362] payload_rows_per_dst=[9018, 7844, 9762, 10742, 8870, 10196, 8886, 10828] dup_rows_per_dst=[2418, 2036, 2830, 3168, 2150, 3544, 2192, 3820] combine_rows_per_src=[9018, 7844, 9762, 10742, 8870, 10196, 8886, 10828] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76032 payload_rows_remote=66304 dup_rows=22272 dup_rows_remote=19698 dedup_ratio=0.226562 router_tokens_per_expert=[592, 795, 369, 442, 1750, 179, 611, 1086, 18, 748, 2383, 346, 584, 928, 560, 1933, 536, 1308, 739, 234, 1606, 163, 2005, 1565, 779, 575, 801, 1228, 25, 159, 97, 148, 135, 28, 306, 57, 920, 650, 965, 985, 1868, 953, 455, 1013, 151, 2050, 6, 180, 2354, 681, 1659, 0, 20, 1292, 224, 1938, 1408, 559, 949, 167, 1027, 259, 292, 309] payload_rows_per_dst=[9246, 10770, 9728, 9592, 9642, 9760, 9340, 7954] dup_rows_per_dst=[2806, 4230, 2574, 2740, 2460, 3188, 2288, 1986] combine_rows_per_src=[9246, 10770, 9728, 9592, 9642, 9760, 9340, 7954] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75160 payload_rows_remote=64006 dup_rows=23144 dup_rows_remote=18838 dedup_ratio=0.235433 router_tokens_per_expert=[471, 697, 372, 428, 2113, 157, 561, 1022, 23, 752, 2405, 387, 619, 945, 552, 2047, 527, 1176, 669, 281, 1580, 280, 2016, 1440, 817, 637, 815, 1235, 15, 220, 122, 144, 169, 45, 256, 37, 838, 584, 854, 991, 1593, 1126, 414, 1040, 147, 2041, 7, 223, 2414, 677, 1876, 0, 33, 1248, 173, 1834, 1320, 522, 912, 292, 966, 281, 431, 283] payload_rows_per_dst=[10904, 11154, 9464, 9904, 9116, 7128, 9344, 8146] dup_rows_per_dst=[4820, 4306, 2442, 2934, 2464, 1972, 2338, 1868] combine_rows_per_src=[10904, 11154, 9464, 9904, 9116, 7128, 9344, 8146] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76048 payload_rows_remote=66518 dup_rows=22256 dup_rows_remote=19742 dedup_ratio=0.226400 router_tokens_per_expert=[423, 654, 353, 422, 1818, 213, 612, 902, 30, 679, 2404, 326, 622, 849, 550, 1951, 571, 1289, 665, 218, 1673, 212, 1998, 1642, 752, 682, 898, 1295, 15, 172, 112, 137, 180, 31, 276, 34, 1009, 561, 997, 936, 1824, 1116, 450, 1152, 163, 1924, 8, 165, 2230, 741, 1689, 0, 40, 1330, 180, 1824, 1373, 527, 967, 235, 1065, 320, 356, 310] payload_rows_per_dst=[8488, 9526, 9712, 9918, 9530, 10042, 9330, 9502] dup_rows_per_dst=[2306, 3038, 2828, 2668, 2514, 3562, 2278, 3062] combine_rows_per_src=[8488, 9526, 9712, 9918, 9530, 10042, 9330, 9502] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=271695872 dispatch_meta_bytes=344968 combine_payload_bytes=271695872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272040840 ingress_bytes=306298880 link_bytes=306298880 token_bytes=543736712 weight_movement_bytes=34603008 total_nvlink_bytes=578339720 weight_to_token_ratio=0.063639 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.152578026e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75370 payload_rows_remote=64342 dup_rows=22934 dup_rows_remote=18576 dedup_ratio=0.233297 router_tokens_per_expert=[484, 641, 286, 417, 1894, 170, 556, 1064, 26, 825, 2526, 391, 558, 900, 535, 1995, 647, 1191, 647, 424, 1435, 251, 1869, 1600, 797, 599, 821, 1394, 14, 310, 134, 166, 237, 37, 279, 58, 881, 555, 870, 888, 1777, 949, 395, 919, 155, 1914, 4, 226, 2400, 751, 1892, 0, 57, 1068, 197, 1909, 1436, 587, 946, 153, 1022, 345, 366, 312] payload_rows_per_dst=[8818, 5214, 8196, 8874, 10922, 9568, 12750, 11028] dup_rows_per_dst=[2206, 1256, 1426, 1952, 3296, 3110, 5330, 4358] combine_rows_per_src=[8818, 5214, 8196, 8874, 10922, 9568, 12750, 11028] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74988 payload_rows_remote=64422 dup_rows=23316 dup_rows_remote=18820 dedup_ratio=0.237183 router_tokens_per_expert=[487, 675, 437, 511, 1721, 176, 538, 1070, 49, 744, 2282, 335, 668, 967, 622, 1869, 709, 1282, 616, 365, 1663, 195, 1935, 1429, 778, 647, 838, 1211, 12, 301, 94, 121, 187, 16, 296, 105, 897, 672, 857, 973, 1830, 1061, 464, 1090, 150, 1916, 7, 220, 1962, 647, 1747, 2, 62, 1194, 242, 1778, 1405, 636, 965, 157, 1212, 289, 373, 393] payload_rows_per_dst=[10566, 11092, 9680, 9506, 9490, 7128, 9064, 8462] dup_rows_per_dst=[4496, 4564, 2838, 2422, 2386, 1932, 2280, 2398] combine_rows_per_src=[10566, 11092, 9680, 9506, 9490, 7128, 9064, 8462] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=270131200 dispatch_meta_bytes=338256 combine_payload_bytes=270131200 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=270469456 ingress_bytes=339337216 link_bytes=339337216 token_bytes=540600656 weight_movement_bytes=69206016 total_nvlink_bytes=609806672 weight_to_token_ratio=0.128017 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=14208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.276898625e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=271581184 dispatch_meta_bytes=344008 combine_payload_bytes=271581184 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=271925192 ingress_bytes=375390208 link_bytes=375390208 token_bytes=543506376 weight_movement_bytes=103809024 total_nvlink_bytes=647315400 weight_to_token_ratio=0.190999 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=12416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.412563131e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=262168576 dispatch_meta_bytes=331376 combine_payload_bytes=262168576 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=262499952 ingress_bytes=400580608 link_bytes=400580608 token_bytes=524668528 weight_movement_bytes=138412032 total_nvlink_bytes=663080560 weight_to_token_ratio=0.263809 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.507352579e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=272457728 dispatch_meta_bytes=345040 combine_payload_bytes=272457728 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=272802768 ingress_bytes=341663744 link_bytes=341663744 token_bytes=545260496 weight_movement_bytes=69206016 total_nvlink_bytes=614466512 weight_to_token_ratio=0.126923 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.030000 gemm_active_groups=11 gemm_peak_group_rows=15872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.285653162e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75160 payload_rows_remote=64006 dup_rows=23144 dup_rows_remote=18838 dedup_ratio=0.235433 router_tokens_per_expert=[471, 697, 372, 428, 2113, 157, 561, 1022, 23, 752, 2405, 387, 619, 945, 552, 2047, 527, 1176, 669, 281, 1580, 280, 2016, 1440, 817, 637, 815, 1235, 15, 220, 122, 144, 169, 45, 256, 37, 838, 584, 854, 991, 1593, 1126, 414, 1040, 147, 2041, 7, 223, 2414, 677, 1876, 0, 33, 1248, 173, 1834, 1320, 522, 912, 292, 966, 281, 431, 283] payload_rows_per_dst=[10904, 11154, 9464, 9904, 9116, 7128, 9344, 8146] dup_rows_per_dst=[4820, 4306, 2442, 2934, 2464, 1972, 2338, 1868] combine_rows_per_src=[10904, 11154, 9464, 9904, 9116, 7128, 9344, 8146] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=272457728 dispatch_meta_bytes=345040 combine_payload_bytes=272457728 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=272802768 ingress_bytes=341663744 link_bytes=341663744 token_bytes=545260496 weight_movement_bytes=69206016 total_nvlink_bytes=614466512 weight_to_token_ratio=0.126923 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.030000 gemm_active_groups=11 gemm_peak_group_rows=15872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.285653162e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=270278656 dispatch_meta_bytes=347024 combine_payload_bytes=270278656 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=270625680 ingress_bytes=339484672 link_bytes=339484672 token_bytes=540904336 weight_movement_bytes=69206016 total_nvlink_bytes=610110352 weight_to_token_ratio=0.127945 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.277453490e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=271695872 dispatch_meta_bytes=344968 combine_payload_bytes=271695872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272040840 ingress_bytes=306298880 link_bytes=306298880 token_bytes=543736712 weight_movement_bytes=34603008 total_nvlink_bytes=578339720 weight_to_token_ratio=0.063639 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.152578026e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=263544832 dispatch_meta_bytes=331672 combine_payload_bytes=263544832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263876504 ingress_bytes=298147840 link_bytes=298147840 token_bytes=527421336 weight_movement_bytes=34603008 total_nvlink_bytes=562024344 weight_to_token_ratio=0.065608 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.121906319e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=270131200 dispatch_meta_bytes=338256 combine_payload_bytes=270131200 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=270469456 ingress_bytes=339337216 link_bytes=339337216 token_bytes=540600656 weight_movement_bytes=69206016 total_nvlink_bytes=609806672 weight_to_token_ratio=0.128017 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=14208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.276898625e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=263872512 dispatch_meta_bytes=332968 combine_payload_bytes=263872512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264205480 ingress_bytes=298475520 link_bytes=298475520 token_bytes=528077992 weight_movement_bytes=34603008 total_nvlink_bytes=562681000 weight_to_token_ratio=0.065526 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123139353e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75256 payload_rows_remote=65986 dup_rows=23048 dup_rows_remote=20770 dedup_ratio=0.234456 router_tokens_per_expert=[434, 844, 455, 381, 1794, 215, 557, 1110, 19, 727, 2477, 345, 590, 892, 641, 2035, 565, 1306, 660, 243, 1783, 244, 2011, 1474, 808, 628, 815, 1119, 15, 188, 90, 139, 222, 51, 294, 97, 936, 717, 852, 1016, 1733, 1148, 477, 1021, 158, 1807, 10, 193, 2109, 766, 1724, 0, 53, 1300, 182, 1749, 1216, 613, 889, 144, 1169, 291, 321, 290] payload_rows_per_dst=[8912, 6606, 10666, 9406, 9964, 9670, 9270, 10762] dup_rows_per_dst=[2668, 1594, 3758, 2416, 2852, 3424, 2278, 4058] combine_rows_per_src=[8912, 6606, 10666, 9406, 9964, 9670, 9270, 10762] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76644 payload_rows_remote=67082 dup_rows=21660 dup_rows_remote=19166 dedup_ratio=0.220337 router_tokens_per_expert=[0, 1318, 393, 258, 8, 1010, 585, 200, 280, 572, 1179, 2005, 1080, 571, 0, 302, 970, 1557, 358, 595, 42, 45, 1980, 890, 853, 998, 1420, 1742, 24, 313, 642, 36, 374, 1500, 366, 506, 4082, 1907, 1410, 330, 770, 2, 200, 132, 1820, 1883, 121, 586, 457, 10, 640, 1128, 1, 3, 185, 207, 1876, 620, 42, 330, 0, 2333, 33, 3072] payload_rows_per_dst=[10784, 9558, 10164, 9562, 9590, 7848, 10288, 8850] dup_rows_per_dst=[2904, 2420, 2710, 2494, 3196, 3180, 3138, 1618] combine_rows_per_src=[10784, 9558, 10164, 9562, 9590, 7848, 10288, 8850] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77492 payload_rows_remote=67096 dup_rows=20812 dup_rows_remote=16562 dedup_ratio=0.211711 router_tokens_per_expert=[0, 1419, 394, 250, 0, 1154, 541, 178, 240, 513, 1226, 2179, 1181, 437, 0, 21, 960, 1637, 594, 505, 26, 0, 1619, 840, 1178, 1044, 1602, 2194, 18, 72, 658, 11, 325, 1426, 459, 364, 4744, 2197, 1304, 182, 984, 0, 139, 55, 1590, 1907, 114, 337, 344, 1, 404, 969, 0, 0, 195, 148, 2096, 645, 11, 139, 0, 2197, 4, 3181] payload_rows_per_dst=[9932, 9486, 9778, 10526, 6764, 10396, 11270, 9340] dup_rows_per_dst=[2728, 2108, 2584, 3028, 1356, 4250, 2340, 2418] combine_rows_per_src=[9932, 9486, 9778, 10526, 6764, 10396, 11270, 9340] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75688 payload_rows_remote=65794 dup_rows=22616 dup_rows_remote=19548 dedup_ratio=0.230062 router_tokens_per_expert=[0, 1513, 436, 232, 3, 942, 444, 210, 310, 529, 1290, 2004, 1189, 524, 0, 90, 1027, 1811, 454, 600, 36, 4, 1705, 917, 1075, 1043, 1480, 1735, 25, 73, 649, 23, 326, 1121, 516, 471, 4708, 2535, 1236, 223, 906, 0, 170, 196, 1710, 1875, 99, 488, 337, 0, 580, 1044, 2, 0, 172, 127, 1895, 633, 13, 181, 0, 1973, 17, 3225] payload_rows_per_dst=[10978, 9894, 9494, 9562, 11212, 7786, 8698, 8064] dup_rows_per_dst=[3032, 3068, 2524, 2644, 4842, 3102, 2044, 1360] combine_rows_per_src=[10978, 9894, 9494, 9562, 11212, 7786, 8698, 8064] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=270278656 dispatch_meta_bytes=347024 combine_payload_bytes=270278656 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=270625680 ingress_bytes=339484672 link_bytes=339484672 token_bytes=540904336 weight_movement_bytes=69206016 total_nvlink_bytes=610110352 weight_to_token_ratio=0.127945 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.277453490e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77296 payload_rows_remote=67270 dup_rows=21008 dup_rows_remote=18514 dedup_ratio=0.213704 router_tokens_per_expert=[0, 1635, 411, 268, 11, 979, 540, 176, 266, 461, 1181, 2142, 1116, 541, 0, 144, 1026, 1626, 327, 544, 36, 17, 1956, 728, 877, 1077, 1668, 1854, 21, 97, 698, 15, 403, 1237, 434, 458, 4692, 2285, 1125, 196, 738, 0, 196, 86, 1722, 1785, 116, 482, 382, 0, 652, 1135, 0, 3, 133, 151, 1743, 613, 23, 212, 0, 2594, 15, 3103] payload_rows_per_dst=[10942, 9456, 10026, 9928, 9458, 7462, 11174, 8850] dup_rows_per_dst=[3304, 2246, 2494, 2686, 2818, 2788, 3122, 1550] combine_rows_per_src=[10942, 9456, 10026, 9928, 9458, 7462, 11174, 8850] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=262168576 dispatch_meta_bytes=331376 combine_payload_bytes=262168576 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=262499952 ingress_bytes=400580608 link_bytes=400580608 token_bytes=524668528 weight_movement_bytes=138412032 total_nvlink_bytes=663080560 weight_to_token_ratio=0.263809 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.507352579e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76032 payload_rows_remote=66304 dup_rows=22272 dup_rows_remote=19698 dedup_ratio=0.226562 router_tokens_per_expert=[592, 795, 369, 442, 1750, 179, 611, 1086, 18, 748, 2383, 346, 584, 928, 560, 1933, 536, 1308, 739, 234, 1606, 163, 2005, 1565, 779, 575, 801, 1228, 25, 159, 97, 148, 135, 28, 306, 57, 920, 650, 965, 985, 1868, 953, 455, 1013, 151, 2050, 6, 180, 2354, 681, 1659, 0, 20, 1292, 224, 1938, 1408, 559, 949, 167, 1027, 259, 292, 309] payload_rows_per_dst=[9246, 10770, 9728, 9592, 9642, 9760, 9340, 7954] dup_rows_per_dst=[2806, 4230, 2574, 2740, 2460, 3188, 2288, 1986] combine_rows_per_src=[9246, 10770, 9728, 9592, 9642, 9760, 9340, 7954] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77406 payload_rows_remote=67906 dup_rows=20898 dup_rows_remote=17996 dedup_ratio=0.212585 router_tokens_per_expert=[1, 1569, 411, 232, 1, 1051, 505, 195, 260, 488, 1302, 1922, 1109, 552, 0, 133, 960, 1823, 346, 618, 27, 30, 1900, 857, 1100, 958, 1380, 1830, 33, 97, 638, 19, 353, 1172, 382, 499, 4446, 2573, 1383, 226, 734, 0, 169, 92, 1623, 1881, 113, 454, 375, 1, 586, 1097, 3, 0, 141, 169, 1836, 571, 27, 231, 0, 2645, 12, 3011] payload_rows_per_dst=[10874, 9270, 10340, 9654, 9500, 7914, 10818, 9036] dup_rows_per_dst=[3078, 2262, 2782, 2456, 2902, 2992, 2818, 1608] combine_rows_per_src=[10874, 9270, 10340, 9654, 9500, 7914, 10818, 9036] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=274767872 dispatch_meta_bytes=344992 combine_payload_bytes=274767872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=275112864 ingress_bytes=343973888 link_bytes=343973888 token_bytes=549880736 weight_movement_bytes=69206016 total_nvlink_bytes=619086752 weight_to_token_ratio=0.125856 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=13184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.294346048e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76644 payload_rows_remote=67082 dup_rows=21660 dup_rows_remote=19166 dedup_ratio=0.220337 router_tokens_per_expert=[0, 1318, 393, 258, 8, 1010, 585, 200, 280, 572, 1179, 2005, 1080, 571, 0, 302, 970, 1557, 358, 595, 42, 45, 1980, 890, 853, 998, 1420, 1742, 24, 313, 642, 36, 374, 1500, 366, 506, 4082, 1907, 1410, 330, 770, 2, 200, 132, 1820, 1883, 121, 586, 457, 10, 640, 1128, 1, 3, 185, 207, 1876, 620, 42, 330, 0, 2333, 33, 3072] payload_rows_per_dst=[10784, 9558, 10164, 9562, 9590, 7848, 10288, 8850] dup_rows_per_dst=[2904, 2420, 2710, 2494, 3196, 3180, 3138, 1618] combine_rows_per_src=[10784, 9558, 10164, 9562, 9590, 7848, 10288, 8850] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76100 payload_rows_remote=64580 dup_rows=22204 dup_rows_remote=19188 dedup_ratio=0.225871 router_tokens_per_expert=[0, 1487, 432, 245, 1, 966, 650, 188, 300, 432, 1302, 1714, 1092, 542, 4, 284, 995, 1647, 476, 637, 31, 63, 2106, 1043, 987, 961, 1330, 1601, 21, 93, 840, 33, 349, 1258, 398, 441, 4955, 2167, 1264, 292, 797, 0, 167, 185, 1638, 1762, 106, 477, 375, 1, 576, 1011, 4, 1, 157, 188, 1682, 498, 39, 301, 0, 2420, 16, 3124] payload_rows_per_dst=[6802, 9128, 10556, 9304, 6524, 10386, 11520, 11880] dup_rows_per_dst=[1136, 2212, 3440, 2428, 1480, 4212, 3016, 4280] combine_rows_per_src=[6802, 9128, 10556, 9304, 6524, 10386, 11520, 11880] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=271581184 dispatch_meta_bytes=344008 combine_payload_bytes=271581184 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=271925192 ingress_bytes=375390208 link_bytes=375390208 token_bytes=543506376 weight_movement_bytes=103809024 total_nvlink_bytes=647315400 weight_to_token_ratio=0.190999 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=12416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.412563131e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75396 payload_rows_remote=62800 dup_rows=22908 dup_rows_remote=16192 dedup_ratio=0.233032 router_tokens_per_expert=[0, 1255, 440, 273, 1, 1096, 455, 212, 278, 500, 1116, 2260, 1043, 483, 0, 63, 1001, 1767, 646, 569, 41, 0, 1722, 837, 1095, 1193, 1760, 1997, 29, 63, 624, 20, 323, 1256, 504, 455, 4358, 2224, 1318, 185, 968, 0, 172, 165, 1640, 1999, 152, 321, 363, 2, 370, 921, 1, 0, 251, 121, 2028, 724, 33, 210, 0, 2183, 12, 3054] payload_rows_per_dst=[6522, 9386, 9762, 8924, 6836, 10714, 10656, 12596] dup_rows_per_dst=[942, 2100, 2748, 1814, 1902, 4568, 2118, 6716] combine_rows_per_src=[6522, 9386, 9762, 8924, 6836, 10714, 10656, 12596] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278142976 dispatch_meta_bytes=343608 combine_payload_bytes=278142976 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=278486584 ingress_bytes=416555008 link_bytes=416555008 token_bytes=556629560 weight_movement_bytes=138412032 total_nvlink_bytes=695041592 weight_to_token_ratio=0.248661 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.567462960e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73104 payload_rows_remote=62050 dup_rows=25200 dup_rows_remote=21908 dedup_ratio=0.256348 router_tokens_per_expert=[0, 1588, 436, 307, 5, 1179, 514, 195, 282, 412, 1228, 1997, 1220, 604, 2, 297, 1009, 1609, 341, 574, 26, 17, 1813, 875, 1036, 928, 1479, 1536, 44, 103, 705, 28, 332, 1340, 505, 575, 4174, 2133, 1448, 243, 892, 0, 148, 92, 1753, 1868, 117, 483, 382, 4, 725, 1103, 2, 1, 229, 196, 1748, 527, 27, 321, 0, 2425, 21, 2949] payload_rows_per_dst=[11054, 11172, 7904, 10050, 12236, 7650, 4436, 8602] dup_rows_per_dst=[3292, 4538, 998, 3244, 7688, 3056, 848, 1536] combine_rows_per_src=[11054, 11172, 7904, 10050, 12236, 7650, 4436, 8602] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=274825216 dispatch_meta_bytes=334632 combine_payload_bytes=274825216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275159848 ingress_bytes=309428224 link_bytes=309428224 token_bytes=549985064 weight_movement_bytes=34603008 total_nvlink_bytes=584588072 weight_to_token_ratio=0.062916 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.164353496e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77492 payload_rows_remote=67096 dup_rows=20812 dup_rows_remote=16562 dedup_ratio=0.211711 router_tokens_per_expert=[0, 1419, 394, 250, 0, 1154, 541, 178, 240, 513, 1226, 2179, 1181, 437, 0, 21, 960, 1637, 594, 505, 26, 0, 1619, 840, 1178, 1044, 1602, 2194, 18, 72, 658, 11, 325, 1426, 459, 364, 4744, 2197, 1304, 182, 984, 0, 139, 55, 1590, 1907, 114, 337, 344, 1, 404, 969, 0, 0, 195, 148, 2096, 645, 11, 139, 0, 2197, 4, 3181] payload_rows_per_dst=[9932, 9486, 9778, 10526, 6764, 10396, 11270, 9340] dup_rows_per_dst=[2728, 2108, 2584, 3028, 1356, 4250, 2340, 2418] combine_rows_per_src=[9932, 9486, 9778, 10526, 6764, 10396, 11270, 9340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75688 payload_rows_remote=65794 dup_rows=22616 dup_rows_remote=19548 dedup_ratio=0.230062 router_tokens_per_expert=[0, 1513, 436, 232, 3, 942, 444, 210, 310, 529, 1290, 2004, 1189, 524, 0, 90, 1027, 1811, 454, 600, 36, 4, 1705, 917, 1075, 1043, 1480, 1735, 25, 73, 649, 23, 326, 1121, 516, 471, 4708, 2535, 1236, 223, 906, 0, 170, 196, 1710, 1875, 99, 488, 337, 0, 580, 1044, 2, 0, 172, 127, 1895, 633, 13, 181, 0, 1973, 17, 3225] payload_rows_per_dst=[10978, 9894, 9494, 9562, 11212, 7786, 8698, 8064] dup_rows_per_dst=[3032, 3068, 2524, 2644, 4842, 3102, 2044, 1360] combine_rows_per_src=[10978, 9894, 9494, 9562, 11212, 7786, 8698, 8064] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=269492224 dispatch_meta_bytes=341368 combine_payload_bytes=269492224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269833592 ingress_bytes=304095232 link_bytes=304095232 token_bytes=539325816 weight_movement_bytes=34603008 total_nvlink_bytes=573928824 weight_to_token_ratio=0.064160 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.144285876e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=275537920 dispatch_meta_bytes=343136 combine_payload_bytes=275537920 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=275881056 ingress_bytes=344743936 link_bytes=344743936 token_bytes=551418976 weight_movement_bytes=69206016 total_nvlink_bytes=620624992 weight_to_token_ratio=0.125505 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=13184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.297243677e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278142976 dispatch_meta_bytes=343608 combine_payload_bytes=278142976 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=278486584 ingress_bytes=416555008 link_bytes=416555008 token_bytes=556629560 weight_movement_bytes=138412032 total_nvlink_bytes=695041592 weight_to_token_ratio=0.248661 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.567462960e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=274767872 dispatch_meta_bytes=344992 combine_payload_bytes=274767872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=275112864 ingress_bytes=343973888 link_bytes=343973888 token_bytes=549880736 weight_movement_bytes=69206016 total_nvlink_bytes=619086752 weight_to_token_ratio=0.125856 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=13184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.294346048e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=264519680 dispatch_meta_bytes=335072 combine_payload_bytes=264519680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264854752 ingress_bytes=299122688 link_bytes=299122688 token_bytes=529374432 weight_movement_bytes=34603008 total_nvlink_bytes=563977440 weight_to_token_ratio=0.065366 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=30464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125574594e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=257228800 dispatch_meta_bytes=315968 combine_payload_bytes=257228800 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=257544768 ingress_bytes=326434816 link_bytes=326434816 token_bytes=514773568 weight_movement_bytes=69206016 total_nvlink_bytes=583979584 weight_to_token_ratio=0.134440 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=18816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.228347932e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=254156800 dispatch_meta_bytes=335832 combine_payload_bytes=254156800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=254492632 ingress_bytes=288759808 link_bytes=288759808 token_bytes=508649432 weight_movement_bytes=34603008 total_nvlink_bytes=543252440 weight_to_token_ratio=0.068029 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=17792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.086579910e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76100 payload_rows_remote=64580 dup_rows=22204 dup_rows_remote=19188 dedup_ratio=0.225871 router_tokens_per_expert=[0, 1487, 432, 245, 1, 966, 650, 188, 300, 432, 1302, 1714, 1092, 542, 4, 284, 995, 1647, 476, 637, 31, 63, 2106, 1043, 987, 961, 1330, 1601, 21, 93, 840, 33, 349, 1258, 398, 441, 4955, 2167, 1264, 292, 797, 0, 167, 185, 1638, 1762, 106, 477, 375, 1, 576, 1011, 4, 1, 157, 188, 1682, 498, 39, 301, 0, 2420, 16, 3124] payload_rows_per_dst=[6802, 9128, 10556, 9304, 6524, 10386, 11520, 11880] dup_rows_per_dst=[1136, 2212, 3440, 2428, 1480, 4212, 3016, 4280] combine_rows_per_src=[6802, 9128, 10556, 9304, 6524, 10386, 11520, 11880] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=274825216 dispatch_meta_bytes=334632 combine_payload_bytes=274825216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275159848 ingress_bytes=309428224 link_bytes=309428224 token_bytes=549985064 weight_movement_bytes=34603008 total_nvlink_bytes=584588072 weight_to_token_ratio=0.062916 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.164353496e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=269492224 dispatch_meta_bytes=341368 combine_payload_bytes=269492224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269833592 ingress_bytes=304095232 link_bytes=304095232 token_bytes=539325816 weight_movement_bytes=34603008 total_nvlink_bytes=573928824 weight_to_token_ratio=0.064160 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.144285876e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77296 payload_rows_remote=67270 dup_rows=21008 dup_rows_remote=18514 dedup_ratio=0.213704 router_tokens_per_expert=[0, 1635, 411, 268, 11, 979, 540, 176, 266, 461, 1181, 2142, 1116, 541, 0, 144, 1026, 1626, 327, 544, 36, 17, 1956, 728, 877, 1077, 1668, 1854, 21, 97, 698, 15, 403, 1237, 434, 458, 4692, 2285, 1125, 196, 738, 0, 196, 86, 1722, 1785, 116, 482, 382, 0, 652, 1135, 0, 3, 133, 151, 1743, 613, 23, 212, 0, 2594, 15, 3103] payload_rows_per_dst=[10942, 9456, 10026, 9928, 9458, 7462, 11174, 8850] dup_rows_per_dst=[3304, 2246, 2494, 2686, 2818, 2788, 3122, 1550] combine_rows_per_src=[10942, 9456, 10026, 9928, 9458, 7462, 11174, 8850] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78640 payload_rows_remote=66400 dup_rows=19664 dup_rows_remote=17794 dedup_ratio=0.200033 router_tokens_per_expert=[1, 71, 1184, 0, 358, 946, 991, 648, 3357, 2043, 195, 172, 707, 14, 5349, 285, 2592, 17, 2193, 342, 693, 276, 0, 268, 860, 1998, 456, 361, 269, 1680, 967, 144, 41, 183, 78, 67, 632, 193, 305, 207, 21, 3030, 85, 456, 85, 56, 0, 5430, 0, 128, 78, 1196, 143, 1908, 0, 64, 28, 0, 2275, 46, 1769, 932, 69, 210] payload_rows_per_dst=[7498, 9724, 9494, 10210, 12240, 6346, 13608, 9520] dup_rows_per_dst=[1910, 2812, 3268, 2332, 1870, 1120, 4286, 2066] combine_rows_per_src=[7498, 9724, 9494, 10210, 12240, 6346, 13608, 9520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78650 payload_rows_remote=65476 dup_rows=19654 dup_rows_remote=15894 dedup_ratio=0.199931 router_tokens_per_expert=[0, 66, 1300, 0, 382, 969, 956, 610, 3404, 2268, 208, 158, 619, 25, 5179, 282, 2661, 8, 2306, 359, 705, 287, 0, 268, 819, 1844, 488, 344, 282, 1683, 1164, 116, 27, 194, 65, 48, 658, 196, 542, 253, 21, 3190, 89, 453, 101, 55, 0, 5137, 0, 95, 63, 1057, 180, 1887, 0, 48, 40, 1, 2136, 9, 1724, 847, 92, 184] payload_rows_per_dst=[6876, 10590, 9888, 8332, 12220, 6684, 13174, 10886] dup_rows_per_dst=[1690, 3338, 3300, 1460, 2104, 1134, 3760, 2868] combine_rows_per_src=[6876, 10590, 9888, 8332, 12220, 6684, 13174, 10886] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79240 payload_rows_remote=68006 dup_rows=19064 dup_rows_remote=16206 dedup_ratio=0.193929 router_tokens_per_expert=[28, 67, 1237, 0, 369, 614, 1022, 600, 3041, 2205, 206, 145, 751, 17, 4902, 200, 2436, 14, 2540, 378, 658, 290, 0, 353, 848, 1999, 552, 361, 300, 1825, 993, 168, 41, 338, 69, 65, 757, 178, 583, 209, 40, 2943, 79, 469, 123, 60, 0, 5243, 0, 142, 75, 1146, 218, 1822, 0, 78, 28, 5, 2294, 173, 1691, 872, 79, 213] payload_rows_per_dst=[11100, 6546, 9864, 11234, 12038, 11356, 8138, 8964] dup_rows_per_dst=[2856, 502, 3474, 2858, 2246, 3834, 1548, 1746] combine_rows_per_src=[11100, 6546, 9864, 11234, 12038, 11356, 8138, 8964] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=264519680 dispatch_meta_bytes=335072 combine_payload_bytes=264519680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264854752 ingress_bytes=299122688 link_bytes=299122688 token_bytes=529374432 weight_movement_bytes=34603008 total_nvlink_bytes=563977440 weight_to_token_ratio=0.065366 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=30464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125574594e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=275537920 dispatch_meta_bytes=343136 combine_payload_bytes=275537920 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=275881056 ingress_bytes=344743936 link_bytes=344743936 token_bytes=551418976 weight_movement_bytes=69206016 total_nvlink_bytes=620624992 weight_to_token_ratio=0.125505 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=13184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.297243677e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78626 payload_rows_remote=66928 dup_rows=19678 dup_rows_remote=15890 dedup_ratio=0.200175 router_tokens_per_expert=[0, 43, 1146, 1, 527, 1194, 1022, 743, 3665, 2012, 250, 170, 604, 6, 5431, 224, 2891, 8, 2203, 328, 552, 197, 0, 270, 606, 2104, 425, 342, 249, 1628, 941, 272, 51, 167, 96, 75, 606, 197, 236, 250, 17, 2943, 83, 395, 114, 46, 0, 4654, 0, 105, 96, 1269, 129, 1757, 0, 173, 36, 0, 2327, 125, 1805, 1041, 57, 248] payload_rows_per_dst=[7346, 10534, 7490, 9978, 12374, 6536, 12670, 11698] dup_rows_per_dst=[2006, 3328, 1580, 2214, 1844, 1222, 3696, 3788] combine_rows_per_src=[7346, 10534, 7490, 9978, 12374, 6536, 12670, 11698] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=271974400 dispatch_meta_bytes=336776 combine_payload_bytes=271974400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272311176 ingress_bytes=306577408 link_bytes=306577408 token_bytes=544285576 weight_movement_bytes=34603008 total_nvlink_bytes=578888584 weight_to_token_ratio=0.063575 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=34176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.153626105e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76880 payload_rows_remote=64978 dup_rows=21424 dup_rows_remote=17258 dedup_ratio=0.217936 router_tokens_per_expert=[4, 73, 1260, 1, 465, 1043, 936, 693, 3559, 1831, 238, 118, 693, 13, 4982, 274, 2347, 22, 2344, 342, 573, 264, 0, 308, 841, 2000, 455, 360, 311, 1614, 897, 238, 49, 214, 91, 59, 716, 192, 598, 245, 31, 3065, 81, 430, 112, 28, 1, 5186, 0, 130, 94, 1211, 169, 1854, 0, 140, 38, 0, 2274, 57, 1686, 959, 69, 274] payload_rows_per_dst=[11902, 12860, 9312, 10896, 3936, 12600, 6454, 8920] dup_rows_per_dst=[4166, 4674, 3088, 2536, 392, 4032, 742, 1794] combine_rows_per_src=[11902, 12860, 9312, 10896, 3936, 12600, 6454, 8920] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78906 payload_rows_remote=72068 dup_rows=19398 dup_rows_remote=18240 dedup_ratio=0.197327 router_tokens_per_expert=[0, 73, 1095, 1, 430, 1117, 808, 657, 4101, 2106, 221, 145, 617, 14, 5659, 239, 2700, 6, 1938, 360, 676, 250, 0, 212, 760, 1877, 472, 307, 242, 1656, 922, 254, 27, 157, 91, 64, 555, 165, 302, 198, 23, 3320, 66, 452, 96, 41, 0, 4890, 0, 114, 90, 1089, 135, 1769, 0, 161, 36, 0, 2181, 10, 1701, 1138, 83, 283] payload_rows_per_dst=[6634, 11180, 9366, 7886, 12822, 6838, 12862, 11318] dup_rows_per_dst=[1728, 3706, 2918, 1340, 1614, 1158, 3634, 3300] combine_rows_per_src=[6634, 11180, 9366, 7886, 12822, 6838, 12862, 11318] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77346 payload_rows_remote=67594 dup_rows=20958 dup_rows_remote=18508 dedup_ratio=0.213196 router_tokens_per_expert=[0, 64, 1198, 1, 451, 1158, 901, 714, 3896, 2079, 194, 147, 629, 12, 5240, 264, 2517, 10, 2091, 337, 684, 285, 0, 236, 884, 1929, 580, 349, 265, 1868, 769, 154, 20, 88, 72, 82, 732, 135, 375, 235, 34, 3104, 106, 513, 86, 37, 0, 5116, 0, 96, 71, 1233, 103, 1797, 0, 81, 42, 0, 2054, 12, 1688, 1011, 76, 247] payload_rows_per_dst=[12124, 9752, 9366, 11002, 7446, 12930, 6128, 8598] dup_rows_per_dst=[4642, 2450, 2954, 2594, 960, 5062, 634, 1662] combine_rows_per_src=[12124, 9752, 9366, 11002, 7446, 12930, 6128, 8598] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78982 payload_rows_remote=69228 dup_rows=19322 dup_rows_remote=16134 dedup_ratio=0.196554 router_tokens_per_expert=[0, 93, 1157, 0, 439, 1103, 936, 690, 3495, 1899, 187, 142, 744, 12, 5123, 170, 2350, 11, 2432, 407, 698, 242, 0, 331, 749, 1907, 456, 372, 243, 1964, 865, 160, 18, 198, 82, 42, 697, 192, 541, 206, 21, 2956, 54, 384, 99, 22, 0, 5190, 0, 144, 83, 1266, 161, 1759, 0, 72, 8, 1, 2304, 6, 1933, 904, 112, 320] payload_rows_per_dst=[11754, 5800, 9754, 10850, 12216, 12952, 6338, 9318] dup_rows_per_dst=[4072, 508, 3188, 2582, 1982, 4500, 632, 1858] combine_rows_per_src=[11754, 5800, 9754, 10850, 12216, 12952, 6338, 9318] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78626 payload_rows_remote=66928 dup_rows=19678 dup_rows_remote=15890 dedup_ratio=0.200175 router_tokens_per_expert=[0, 43, 1146, 1, 527, 1194, 1022, 743, 3665, 2012, 250, 170, 604, 6, 5431, 224, 2891, 8, 2203, 328, 552, 197, 0, 270, 606, 2104, 425, 342, 249, 1628, 941, 272, 51, 167, 96, 75, 606, 197, 236, 250, 17, 2943, 83, 395, 114, 46, 0, 4654, 0, 105, 96, 1269, 129, 1757, 0, 173, 36, 0, 2327, 125, 1805, 1041, 57, 248] payload_rows_per_dst=[7346, 10534, 7490, 9978, 12374, 6536, 12670, 11698] dup_rows_per_dst=[2006, 3328, 1580, 2214, 1844, 1222, 3696, 3788] combine_rows_per_src=[7346, 10534, 7490, 9978, 12374, 6536, 12670, 11698] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278552576 dispatch_meta_bytes=336848 combine_payload_bytes=278552576 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=278889424 ingress_bytes=347758592 link_bytes=347758592 token_bytes=557442000 weight_movement_bytes=69206016 total_nvlink_bytes=626648016 weight_to_token_ratio=0.124149 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=13952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.308587585e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78640 payload_rows_remote=66400 dup_rows=19664 dup_rows_remote=17794 dedup_ratio=0.200033 router_tokens_per_expert=[1, 71, 1184, 0, 358, 946, 991, 648, 3357, 2043, 195, 172, 707, 14, 5349, 285, 2592, 17, 2193, 342, 693, 276, 0, 268, 860, 1998, 456, 361, 269, 1680, 967, 144, 41, 183, 78, 67, 632, 193, 305, 207, 21, 3030, 85, 456, 85, 56, 0, 5430, 0, 128, 78, 1196, 143, 1908, 0, 64, 28, 0, 2275, 46, 1769, 932, 69, 210] payload_rows_per_dst=[7498, 9724, 9494, 10210, 12240, 6346, 13608, 9520] dup_rows_per_dst=[1910, 2812, 3268, 2332, 1870, 1120, 4286, 2066] combine_rows_per_src=[7498, 9724, 9494, 10210, 12240, 6346, 13608, 9520] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77346 payload_rows_remote=67594 dup_rows=20958 dup_rows_remote=18508 dedup_ratio=0.213196 router_tokens_per_expert=[0, 64, 1198, 1, 451, 1158, 901, 714, 3896, 2079, 194, 147, 629, 12, 5240, 264, 2517, 10, 2091, 337, 684, 285, 0, 236, 884, 1929, 580, 349, 265, 1868, 769, 154, 20, 88, 72, 82, 732, 135, 375, 235, 34, 3104, 106, 513, 86, 37, 0, 5116, 0, 96, 71, 1233, 103, 1797, 0, 81, 42, 0, 2054, 12, 1688, 1011, 76, 247] payload_rows_per_dst=[12124, 9752, 9366, 11002, 7446, 12930, 6128, 8598] dup_rows_per_dst=[4642, 2450, 2954, 2594, 960, 5062, 634, 1662] combine_rows_per_src=[12124, 9752, 9366, 11002, 7446, 12930, 6128, 8598] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=295190528 dispatch_meta_bytes=361232 combine_payload_bytes=295190528 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=295551760 ingress_bytes=398999552 link_bytes=398999552 token_bytes=590742288 weight_movement_bytes=103809024 total_nvlink_bytes=694551312 weight_to_token_ratio=0.175726 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.501403192e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76880 payload_rows_remote=64978 dup_rows=21424 dup_rows_remote=17258 dedup_ratio=0.217936 router_tokens_per_expert=[4, 73, 1260, 1, 465, 1043, 936, 693, 3559, 1831, 238, 118, 693, 13, 4982, 274, 2347, 22, 2344, 342, 573, 264, 0, 308, 841, 2000, 455, 360, 311, 1614, 897, 238, 49, 214, 91, 59, 716, 192, 598, 245, 31, 3065, 81, 430, 112, 28, 1, 5186, 0, 130, 94, 1211, 169, 1854, 0, 140, 38, 0, 2274, 57, 1686, 959, 69, 274] payload_rows_per_dst=[11902, 12860, 9312, 10896, 3936, 12600, 6454, 8920] dup_rows_per_dst=[4166, 4674, 3088, 2536, 392, 4032, 742, 1794] combine_rows_per_src=[11902, 12860, 9312, 10896, 3936, 12600, 6454, 8920] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=276865024 dispatch_meta_bytes=344408 combine_payload_bytes=276865024 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=277209432 ingress_bytes=380674048 link_bytes=380674048 token_bytes=554074456 weight_movement_bytes=103809024 total_nvlink_bytes=657883480 weight_to_token_ratio=0.187356 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.432445796e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=283557888 dispatch_meta_bytes=341448 combine_payload_bytes=283557888 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=283899336 ingress_bytes=352763904 link_bytes=352763904 token_bytes=567457224 weight_movement_bytes=69206016 total_nvlink_bytes=636663240 weight_to_token_ratio=0.121958 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=18688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.327422171e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=268189696 dispatch_meta_bytes=325480 combine_payload_bytes=268189696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268515176 ingress_bytes=302792704 link_bytes=302792704 token_bytes=536704872 weight_movement_bytes=34603008 total_nvlink_bytes=571307880 weight_to_token_ratio=0.064473 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=21504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.139384568e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78650 payload_rows_remote=65476 dup_rows=19654 dup_rows_remote=15894 dedup_ratio=0.199931 router_tokens_per_expert=[0, 66, 1300, 0, 382, 969, 956, 610, 3404, 2268, 208, 158, 619, 25, 5179, 282, 2661, 8, 2306, 359, 705, 287, 0, 268, 819, 1844, 488, 344, 282, 1683, 1164, 116, 27, 194, 65, 48, 658, 196, 542, 253, 21, 3190, 89, 453, 101, 55, 0, 5137, 0, 95, 63, 1057, 180, 1887, 0, 48, 40, 1, 2136, 9, 1724, 847, 92, 184] payload_rows_per_dst=[6876, 10590, 9888, 8332, 12220, 6684, 13174, 10886] dup_rows_per_dst=[1690, 3338, 3300, 1460, 2104, 1134, 3760, 2868] combine_rows_per_src=[6876, 10590, 9888, 8332, 12220, 6684, 13174, 10886] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278552576 dispatch_meta_bytes=336848 combine_payload_bytes=278552576 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=278889424 ingress_bytes=347758592 link_bytes=347758592 token_bytes=557442000 weight_movement_bytes=69206016 total_nvlink_bytes=626648016 weight_to_token_ratio=0.124149 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=13952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.308587585e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=274137088 dispatch_meta_bytes=331272 combine_payload_bytes=274137088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274468360 ingress_bytes=308740096 link_bytes=308740096 token_bytes=548605448 weight_movement_bytes=34603008 total_nvlink_bytes=583208456 weight_to_token_ratio=0.063074 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.161764126e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=271974400 dispatch_meta_bytes=336776 combine_payload_bytes=271974400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272311176 ingress_bytes=306577408 link_bytes=306577408 token_bytes=544285576 weight_movement_bytes=34603008 total_nvlink_bytes=578888584 weight_to_token_ratio=0.063575 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=34176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.153626105e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=328944 combine_payload_bytes=266149888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266478832 ingress_bytes=300752896 link_bytes=300752896 token_bytes=532628720 weight_movement_bytes=34603008 total_nvlink_bytes=567231728 weight_to_token_ratio=0.064966 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131708935e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=268189696 dispatch_meta_bytes=325480 combine_payload_bytes=268189696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268515176 ingress_bytes=302792704 link_bytes=302792704 token_bytes=536704872 weight_movement_bytes=34603008 total_nvlink_bytes=571307880 weight_to_token_ratio=0.064473 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=21504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.139384568e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=295190528 dispatch_meta_bytes=361232 combine_payload_bytes=295190528 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=295551760 ingress_bytes=398999552 link_bytes=398999552 token_bytes=590742288 weight_movement_bytes=103809024 total_nvlink_bytes=694551312 weight_to_token_ratio=0.175726 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.501403192e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=276865024 dispatch_meta_bytes=344408 combine_payload_bytes=276865024 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=277209432 ingress_bytes=380674048 link_bytes=380674048 token_bytes=554074456 weight_movement_bytes=103809024 total_nvlink_bytes=657883480 weight_to_token_ratio=0.187356 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.432445796e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78982 payload_rows_remote=69228 dup_rows=19322 dup_rows_remote=16134 dedup_ratio=0.196554 router_tokens_per_expert=[0, 93, 1157, 0, 439, 1103, 936, 690, 3495, 1899, 187, 142, 744, 12, 5123, 170, 2350, 11, 2432, 407, 698, 242, 0, 331, 749, 1907, 456, 372, 243, 1964, 865, 160, 18, 198, 82, 42, 697, 192, 541, 206, 21, 2956, 54, 384, 99, 22, 0, 5190, 0, 144, 83, 1266, 161, 1759, 0, 72, 8, 1, 2304, 6, 1933, 904, 112, 320] payload_rows_per_dst=[11754, 5800, 9754, 10850, 12216, 12952, 6338, 9318] dup_rows_per_dst=[4072, 508, 3188, 2582, 1982, 4500, 632, 1858] combine_rows_per_src=[11754, 5800, 9754, 10850, 12216, 12952, 6338, 9318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81336 payload_rows_remote=68336 dup_rows=16968 dup_rows_remote=16188 dedup_ratio=0.172607 router_tokens_per_expert=[0, 895, 1219, 1014, 943, 2927, 4, 1834, 5, 1475, 1972, 63, 132, 164, 246, 10, 1697, 343, 0, 1, 44, 1026, 742, 3, 1090, 0, 3874, 96, 4718, 107, 205, 0, 144, 0, 0, 183, 188, 0, 0, 686, 28, 704, 0, 1205, 143, 0, 4477, 0, 0, 1369, 127, 169, 0, 5689, 4147, 662, 341, 189, 4, 924, 707, 64, 42, 111] payload_rows_per_dst=[8294, 8446, 10244, 9280, 13000, 11538, 8718, 11816] dup_rows_per_dst=[3524, 2084, 3322, 1464, 780, 1576, 1834, 2384] combine_rows_per_src=[8294, 8446, 10244, 9280, 13000, 11538, 8718, 11816] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76832 payload_rows_remote=63328 dup_rows=21472 dup_rows_remote=13206 dedup_ratio=0.218424 router_tokens_per_expert=[0, 978, 1209, 1344, 902, 2594, 0, 1853, 3, 1283, 1772, 60, 152, 189, 254, 16, 1704, 336, 0, 0, 72, 895, 679, 8, 1027, 0, 4389, 93, 4684, 93, 241, 0, 138, 0, 0, 243, 163, 0, 1, 707, 65, 816, 0, 1178, 147, 0, 4607, 0, 0, 1457, 111, 149, 46, 5152, 4008, 671, 320, 204, 5, 1143, 750, 62, 88, 91] payload_rows_per_dst=[8816, 6480, 9610, 9816, 11712, 4964, 13504, 11930] dup_rows_per_dst=[3756, 978, 2966, 1616, 690, 436, 8266, 2764] combine_rows_per_src=[8816, 6480, 9610, 9816, 11712, 4964, 13504, 11930] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79998 payload_rows_remote=69964 dup_rows=18306 dup_rows_remote=16540 dedup_ratio=0.186218 router_tokens_per_expert=[0, 1165, 1261, 1054, 946, 2626, 1, 1770, 9, 1224, 2234, 54, 132, 145, 261, 26, 1504, 345, 0, 8, 104, 1059, 744, 10, 1201, 0, 4292, 96, 4346, 91, 220, 0, 135, 0, 0, 227, 205, 0, 4, 729, 41, 680, 0, 1228, 135, 0, 4555, 0, 0, 1349, 133, 206, 55, 5309, 3776, 653, 368, 213, 5, 1118, 809, 76, 98, 117] payload_rows_per_dst=[8420, 11550, 9710, 10034, 12458, 11670, 4686, 11470] dup_rows_per_dst=[3974, 4172, 3090, 1766, 760, 1608, 106, 2830] combine_rows_per_src=[8420, 11550, 9710, 10034, 12458, 11670, 4686, 11470] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82264 payload_rows_remote=70004 dup_rows=16040 dup_rows_remote=14494 dedup_ratio=0.163167 router_tokens_per_expert=[0, 827, 1211, 1047, 801, 2833, 0, 1802, 0, 1431, 2029, 66, 153, 135, 246, 9, 1450, 304, 0, 2, 22, 1106, 750, 1, 928, 0, 3579, 99, 4862, 103, 193, 0, 150, 0, 0, 246, 280, 0, 4, 742, 2, 648, 0, 1324, 160, 0, 4769, 0, 0, 1317, 127, 236, 4, 6089, 4300, 569, 311, 228, 5, 855, 642, 40, 4, 111] payload_rows_per_dst=[8070, 7008, 9994, 8648, 13910, 12260, 10548, 11826] dup_rows_per_dst=[3306, 1130, 2942, 1156, 1112, 1546, 2558, 2290] combine_rows_per_src=[8070, 7008, 9994, 8648, 13910, 12260, 10548, 11826] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=283557888 dispatch_meta_bytes=341448 combine_payload_bytes=283557888 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=283899336 ingress_bytes=352763904 link_bytes=352763904 token_bytes=567457224 weight_movement_bytes=69206016 total_nvlink_bytes=636663240 weight_to_token_ratio=0.121958 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=18688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.327422171e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68686 payload_rows_remote=56620 dup_rows=29618 dup_rows_remote=27210 dedup_ratio=0.301290 router_tokens_per_expert=[0, 827, 1162, 1210, 892, 2541, 0, 1851, 1, 1230, 1879, 43, 96, 45, 254, 2, 1496, 337, 0, 1, 7, 1190, 654, 2, 1181, 0, 3923, 108, 5042, 114, 168, 0, 167, 0, 0, 233, 319, 0, 3, 764, 22, 662, 0, 1338, 146, 0, 4540, 0, 0, 1233, 155, 272, 0, 5950, 4301, 596, 331, 152, 5, 872, 677, 36, 7, 115] payload_rows_per_dst=[7992, 6368, 9600, 3854, 2588, 10986, 15232, 12066] dup_rows_per_dst=[3028, 732, 2856, 152, 384, 1196, 18862, 2408] combine_rows_per_src=[7992, 6368, 9600, 3854, 2588, 10986, 15232, 12066] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279904256 dispatch_meta_bytes=338096 combine_payload_bytes=279904256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280242352 ingress_bytes=314507264 link_bytes=314507264 token_bytes=560146608 weight_movement_bytes=34603008 total_nvlink_bytes=594749616 weight_to_token_ratio=0.061775 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=38400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.183465515e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76958 payload_rows_remote=66836 dup_rows=21346 dup_rows_remote=15432 dedup_ratio=0.217143 router_tokens_per_expert=[0, 922, 1186, 1192, 909, 2422, 1, 1702, 3, 1490, 2066, 54, 123, 131, 249, 11, 1738, 342, 1, 1, 32, 1097, 699, 3, 1297, 0, 3953, 126, 4365, 108, 195, 1, 162, 0, 1, 203, 319, 0, 10, 811, 43, 827, 1, 1276, 149, 0, 4594, 1, 0, 1309, 144, 249, 4, 5404, 4009, 631, 375, 205, 12, 946, 763, 119, 38, 128] payload_rows_per_dst=[10122, 12042, 7672, 12894, 12770, 12046, 4558, 4854] dup_rows_per_dst=[5914, 4230, 786, 7196, 1050, 1736, 116, 318] combine_rows_per_src=[10122, 12042, 7672, 12894, 12770, 12046, 4558, 4854] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82264 payload_rows_remote=70004 dup_rows=16040 dup_rows_remote=14494 dedup_ratio=0.163167 router_tokens_per_expert=[0, 827, 1211, 1047, 801, 2833, 0, 1802, 0, 1431, 2029, 66, 153, 135, 246, 9, 1450, 304, 0, 2, 22, 1106, 750, 1, 928, 0, 3579, 99, 4862, 103, 193, 0, 150, 0, 0, 246, 280, 0, 4, 742, 2, 648, 0, 1324, 160, 0, 4769, 0, 0, 1317, 127, 236, 4, 6089, 4300, 569, 311, 228, 5, 855, 642, 40, 4, 111] payload_rows_per_dst=[8070, 7008, 9994, 8648, 13910, 12260, 10548, 11826] dup_rows_per_dst=[3306, 1130, 2942, 1156, 1112, 1546, 2558, 2290] combine_rows_per_src=[8070, 7008, 9994, 8648, 13910, 12260, 10548, 11826] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77914 payload_rows_remote=65708 dup_rows=20390 dup_rows_remote=15806 dedup_ratio=0.207418 router_tokens_per_expert=[0, 908, 1273, 914, 852, 2569, 0, 1810, 0, 1484, 2000, 61, 136, 151, 239, 12, 1673, 348, 0, 3, 4, 1080, 728, 5, 1060, 0, 4036, 88, 4939, 84, 194, 0, 163, 0, 0, 257, 181, 0, 1, 733, 4, 693, 0, 1307, 145, 0, 4620, 0, 0, 1482, 124, 148, 0, 5453, 4312, 601, 276, 216, 6, 958, 647, 72, 6, 96] payload_rows_per_dst=[8112, 12206, 9810, 12682, 12874, 11940, 4608, 5682] dup_rows_per_dst=[3402, 4584, 3010, 6510, 702, 1598, 102, 482] combine_rows_per_src=[8112, 12206, 9810, 12682, 12874, 11940, 4608, 5682] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=259391488 dispatch_meta_bytes=306136 combine_payload_bytes=259391488 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=259697624 ingress_bytes=363200512 link_bytes=363200512 token_bytes=519089112 weight_movement_bytes=103809024 total_nvlink_bytes=622898136 weight_to_token_ratio=0.199983 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=15616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.366694287e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=286572544 dispatch_meta_bytes=346016 combine_payload_bytes=286572544 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=286918560 ingress_bytes=390381568 link_bytes=390381568 token_bytes=573491104 weight_movement_bytes=103809024 total_nvlink_bytes=677300128 weight_to_token_ratio=0.181012 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=28160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.468974412e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80698 payload_rows_remote=70928 dup_rows=17606 dup_rows_remote=14684 dedup_ratio=0.179097 router_tokens_per_expert=[0, 929, 1282, 934, 905, 2609, 0, 1737, 3, 1415, 2016, 90, 203, 57, 264, 12, 1641, 361, 0, 3, 2, 1046, 681, 3, 1073, 0, 4129, 101, 5099, 101, 214, 0, 126, 0, 0, 282, 201, 0, 5, 637, 4, 714, 0, 1235, 182, 0, 4467, 0, 0, 1314, 133, 221, 17, 5506, 4087, 618, 315, 272, 3, 1031, 689, 56, 13, 114] payload_rows_per_dst=[8086, 12016, 9770, 9708, 12748, 11572, 4488, 12310] dup_rows_per_dst=[3488, 4278, 2922, 1528, 766, 1632, 118, 2874] combine_rows_per_src=[8086, 12016, 9770, 9708, 12748, 11572, 4488, 12310] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68686 payload_rows_remote=56620 dup_rows=29618 dup_rows_remote=27210 dedup_ratio=0.301290 router_tokens_per_expert=[0, 827, 1162, 1210, 892, 2541, 0, 1851, 1, 1230, 1879, 43, 96, 45, 254, 2, 1496, 337, 0, 1, 7, 1190, 654, 2, 1181, 0, 3923, 108, 5042, 114, 168, 0, 167, 0, 0, 233, 319, 0, 3, 764, 22, 662, 0, 1338, 146, 0, 4540, 0, 0, 1233, 155, 272, 0, 5950, 4301, 596, 331, 152, 5, 872, 677, 36, 7, 115] payload_rows_per_dst=[7992, 6368, 9600, 3854, 2588, 10986, 15232, 12066] dup_rows_per_dst=[3028, 732, 2856, 152, 384, 1196, 18862, 2408] combine_rows_per_src=[7992, 6368, 9600, 3854, 2588, 10986, 15232, 12066] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81336 payload_rows_remote=68336 dup_rows=16968 dup_rows_remote=16188 dedup_ratio=0.172607 router_tokens_per_expert=[0, 895, 1219, 1014, 943, 2927, 4, 1834, 5, 1475, 1972, 63, 132, 164, 246, 10, 1697, 343, 0, 1, 44, 1026, 742, 3, 1090, 0, 3874, 96, 4718, 107, 205, 0, 144, 0, 0, 183, 188, 0, 0, 686, 28, 704, 0, 1205, 143, 0, 4477, 0, 0, 1369, 127, 169, 0, 5689, 4147, 662, 341, 189, 4, 924, 707, 64, 42, 111] payload_rows_per_dst=[8294, 8446, 10244, 9280, 13000, 11538, 8718, 11816] dup_rows_per_dst=[3524, 2084, 3322, 1464, 780, 1576, 1834, 2384] combine_rows_per_src=[8294, 8446, 10244, 9280, 13000, 11538, 8718, 11816] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76958 payload_rows_remote=66836 dup_rows=21346 dup_rows_remote=15432 dedup_ratio=0.217143 router_tokens_per_expert=[0, 922, 1186, 1192, 909, 2422, 1, 1702, 3, 1490, 2066, 54, 123, 131, 249, 11, 1738, 342, 1, 1, 32, 1097, 699, 3, 1297, 0, 3953, 126, 4365, 108, 195, 1, 162, 0, 1, 203, 319, 0, 10, 811, 43, 827, 1, 1276, 149, 0, 4594, 1, 0, 1309, 144, 249, 4, 5404, 4009, 631, 375, 205, 12, 946, 763, 119, 38, 128] payload_rows_per_dst=[10122, 12042, 7672, 12894, 12770, 12046, 4558, 4854] dup_rows_per_dst=[5914, 4230, 786, 7196, 1050, 1736, 116, 318] combine_rows_per_src=[10122, 12042, 7672, 12894, 12770, 12046, 4558, 4854] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=269139968 dispatch_meta_bytes=326056 combine_payload_bytes=269139968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269466024 ingress_bytes=303742976 link_bytes=303742976 token_bytes=538605992 weight_movement_bytes=34603008 total_nvlink_bytes=573209000 weight_to_token_ratio=0.064245 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=17408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.142960365e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=286736384 dispatch_meta_bytes=337992 combine_payload_bytes=286736384 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=287074376 ingress_bytes=355942400 link_bytes=355942400 token_bytes=573810760 weight_movement_bytes=69206016 total_nvlink_bytes=643016776 weight_to_token_ratio=0.120608 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=27904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.339382595e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76832 payload_rows_remote=63328 dup_rows=21472 dup_rows_remote=13206 dedup_ratio=0.218424 router_tokens_per_expert=[0, 978, 1209, 1344, 902, 2594, 0, 1853, 3, 1283, 1772, 60, 152, 189, 254, 16, 1704, 336, 0, 0, 72, 895, 679, 8, 1027, 0, 4389, 93, 4684, 93, 241, 0, 138, 0, 0, 243, 163, 0, 1, 707, 65, 816, 0, 1178, 147, 0, 4607, 0, 0, 1457, 111, 149, 46, 5152, 4008, 671, 320, 204, 5, 1143, 750, 62, 88, 91] payload_rows_per_dst=[8816, 6480, 9610, 9816, 11712, 4964, 13504, 11930] dup_rows_per_dst=[3756, 978, 2966, 1616, 690, 436, 8266, 2764] combine_rows_per_src=[8816, 6480, 9610, 9816, 11712, 4964, 13504, 11930] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=290521088 dispatch_meta_bytes=342448 combine_payload_bytes=290521088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=290863536 ingress_bytes=325124096 link_bytes=325124096 token_bytes=581384624 weight_movement_bytes=34603008 total_nvlink_bytes=615987632 weight_to_token_ratio=0.059518 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=19072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.223415799e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279904256 dispatch_meta_bytes=338096 combine_payload_bytes=279904256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280242352 ingress_bytes=314507264 link_bytes=314507264 token_bytes=560146608 weight_movement_bytes=34603008 total_nvlink_bytes=594749616 weight_to_token_ratio=0.061775 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=38400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.183465515e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=231915520 dispatch_meta_bytes=335320 combine_payload_bytes=231915520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=232250840 ingress_bytes=266518528 link_bytes=266518528 token_bytes=464166360 weight_movement_bytes=34603008 total_nvlink_bytes=498769368 weight_to_token_ratio=0.074549 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=29568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.002887764e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=273760256 dispatch_meta_bytes=329072 combine_payload_bytes=273760256 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=274089328 ingress_bytes=342966272 link_bytes=342966272 token_bytes=547849584 weight_movement_bytes=69206016 total_nvlink_bytes=617055600 weight_to_token_ratio=0.126323 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=13952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.290554470e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=286736384 dispatch_meta_bytes=337992 combine_payload_bytes=286736384 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=287074376 ingress_bytes=355942400 link_bytes=355942400 token_bytes=573810760 weight_movement_bytes=69206016 total_nvlink_bytes=643016776 weight_to_token_ratio=0.120608 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=27904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.339382595e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=269139968 dispatch_meta_bytes=326056 combine_payload_bytes=269139968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269466024 ingress_bytes=303742976 link_bytes=303742976 token_bytes=538605992 weight_movement_bytes=34603008 total_nvlink_bytes=573209000 weight_to_token_ratio=0.064245 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=17408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.142960365e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80698 payload_rows_remote=70928 dup_rows=17606 dup_rows_remote=14684 dedup_ratio=0.179097 router_tokens_per_expert=[0, 929, 1282, 934, 905, 2609, 0, 1737, 3, 1415, 2016, 90, 203, 57, 264, 12, 1641, 361, 0, 3, 2, 1046, 681, 3, 1073, 0, 4129, 101, 5099, 101, 214, 0, 126, 0, 0, 282, 201, 0, 5, 637, 4, 714, 0, 1235, 182, 0, 4467, 0, 0, 1314, 133, 221, 17, 5506, 4087, 618, 315, 272, 3, 1031, 689, 56, 13, 114] payload_rows_per_dst=[8086, 12016, 9770, 9708, 12748, 11572, 4488, 12310] dup_rows_per_dst=[3488, 4278, 2922, 1528, 766, 1632, 118, 2874] combine_rows_per_src=[8086, 12016, 9770, 9708, 12748, 11572, 4488, 12310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79226 payload_rows_remote=69436 dup_rows=19078 dup_rows_remote=18376 dedup_ratio=0.194071 router_tokens_per_expert=[161, 1517, 344, 772, 69, 114, 1335, 374, 0, 0, 25, 574, 2757, 0, 0, 769, 348, 672, 6575, 0, 11, 764, 1277, 0, 0, 4867, 1484, 0, 0, 0, 2512, 0, 4130, 403, 70, 0, 86, 0, 408, 149, 0, 0, 0, 355, 4, 143, 3707, 1491, 942, 936, 1040, 1004, 0, 192, 0, 993, 958, 244, 75, 11, 56, 286, 431, 3717] payload_rows_per_dst=[7748, 7338, 15150, 12548, 9790, 9450, 7860, 9342] dup_rows_per_dst=[1624, 912, 4144, 5178, 702, 1950, 2354, 2214] combine_rows_per_src=[7748, 7338, 15150, 12548, 9790, 9450, 7860, 9342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76726 payload_rows_remote=63150 dup_rows=21578 dup_rows_remote=15598 dedup_ratio=0.219503 router_tokens_per_expert=[136, 1451, 362, 816, 75, 213, 1462, 363, 0, 0, 16, 538, 2885, 0, 0, 735, 336, 651, 6640, 0, 3, 823, 1333, 0, 0, 5025, 1629, 0, 0, 0, 2194, 0, 4100, 392, 76, 3, 69, 0, 369, 133, 1, 0, 0, 395, 17, 206, 3860, 1550, 715, 778, 1247, 911, 0, 109, 0, 996, 811, 234, 65, 15, 34, 231, 424, 3725] payload_rows_per_dst=[8158, 7436, 6048, 7008, 15504, 9848, 13576, 9148] dup_rows_per_dst=[1598, 912, 244, 638, 8060, 2216, 5980, 1930] combine_rows_per_src=[8158, 7436, 6048, 7008, 15504, 9848, 13576, 9148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79386 payload_rows_remote=66946 dup_rows=18918 dup_rows_remote=13554 dedup_ratio=0.192444 router_tokens_per_expert=[188, 1654, 261, 850, 66, 233, 1490, 388, 0, 0, 50, 604, 2655, 0, 0, 756, 345, 722, 6527, 1, 7, 677, 1122, 0, 0, 4861, 1601, 0, 1, 0, 2439, 0, 3970, 412, 67, 0, 63, 0, 386, 149, 0, 0, 0, 340, 23, 173, 3782, 1402, 865, 977, 1256, 1031, 0, 270, 0, 1038, 769, 225, 123, 90, 59, 295, 451, 3438] payload_rows_per_dst=[8268, 7302, 15108, 12440, 9432, 9550, 8382, 8904] dup_rows_per_dst=[1992, 828, 3694, 5364, 662, 1890, 2492, 1996] combine_rows_per_src=[8268, 7302, 15108, 12440, 9432, 9550, 8382, 8904] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=259391488 dispatch_meta_bytes=306136 combine_payload_bytes=259391488 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=259697624 ingress_bytes=363200512 link_bytes=363200512 token_bytes=519089112 weight_movement_bytes=103809024 total_nvlink_bytes=622898136 weight_to_token_ratio=0.199983 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=15616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.366694287e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=290521088 dispatch_meta_bytes=342448 combine_payload_bytes=290521088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=290863536 ingress_bytes=325124096 link_bytes=325124096 token_bytes=581384624 weight_movement_bytes=34603008 total_nvlink_bytes=615987632 weight_to_token_ratio=0.059518 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=19072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.223415799e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77250 payload_rows_remote=64350 dup_rows=21054 dup_rows_remote=14474 dedup_ratio=0.214172 router_tokens_per_expert=[279, 1299, 124, 1129, 72, 143, 1208, 578, 0, 0, 24, 482, 2915, 0, 0, 809, 337, 617, 6519, 5, 13, 925, 1709, 0, 0, 4609, 1872, 0, 0, 0, 2084, 0, 3822, 330, 72, 0, 46, 0, 481, 179, 0, 0, 0, 282, 22, 155, 3653, 1492, 920, 753, 1166, 1014, 0, 338, 0, 968, 841, 181, 97, 2, 47, 296, 370, 3873] payload_rows_per_dst=[7864, 7750, 6922, 7158, 11558, 9318, 13780, 12900] dup_rows_per_dst=[1800, 710, 290, 754, 3274, 1890, 5756, 6580] combine_rows_per_src=[7864, 7750, 6922, 7158, 11558, 9318, 13780, 12900] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=284409856 dispatch_meta_bytes=351248 combine_payload_bytes=284409856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=284761104 ingress_bytes=319012864 link_bytes=319012864 token_bytes=569170960 weight_movement_bytes=34603008 total_nvlink_bytes=603773968 weight_to_token_ratio=0.060795 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=31104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.200419725e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77350 payload_rows_remote=63592 dup_rows=20954 dup_rows_remote=13786 dedup_ratio=0.213155 router_tokens_per_expert=[274, 1684, 341, 981, 69, 277, 1419, 475, 1, 1, 24, 545, 2551, 0, 1, 846, 346, 674, 6446, 2, 1, 750, 1441, 0, 1, 4943, 1558, 0, 1, 0, 2308, 0, 3631, 370, 70, 2, 59, 1, 486, 156, 0, 1, 0, 326, 6, 179, 3985, 1304, 933, 868, 930, 925, 0, 201, 0, 990, 939, 233, 92, 15, 45, 316, 383, 3746] payload_rows_per_dst=[13758, 14746, 6168, 7122, 8934, 9748, 7562, 9312] dup_rows_per_dst=[7168, 6084, 260, 614, 616, 1854, 2132, 2226] combine_rows_per_src=[13758, 14746, 6168, 7122, 8934, 9748, 7562, 9312] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77508 payload_rows_remote=62518 dup_rows=20796 dup_rows_remote=13892 dedup_ratio=0.211548 router_tokens_per_expert=[162, 1525, 411, 927, 92, 186, 1183, 422, 0, 0, 11, 588, 3044, 0, 0, 794, 349, 700, 6510, 3, 0, 857, 1676, 0, 0, 4717, 1641, 0, 0, 0, 2364, 0, 3867, 432, 93, 1, 65, 0, 427, 174, 0, 0, 0, 323, 1, 183, 3600, 1507, 843, 852, 1068, 980, 0, 39, 0, 922, 951, 228, 82, 5, 44, 239, 339, 3725] payload_rows_per_dst=[12402, 14990, 6824, 8308, 9366, 9346, 7268, 9004] dup_rows_per_dst=[4854, 6904, 346, 1696, 752, 1882, 2140, 2222] combine_rows_per_src=[12402, 14990, 6824, 8308, 9366, 9346, 7268, 9004] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77456 payload_rows_remote=65320 dup_rows=20848 dup_rows_remote=15148 dedup_ratio=0.212077 router_tokens_per_expert=[262, 1284, 353, 1155, 60, 230, 1419, 556, 0, 0, 29, 576, 2870, 0, 0, 858, 304, 663, 6586, 1, 6, 844, 1535, 0, 0, 4598, 1977, 0, 0, 0, 2479, 0, 3465, 404, 60, 0, 70, 0, 376, 153, 0, 0, 0, 342, 9, 122, 3642, 1588, 813, 706, 1046, 1169, 0, 47, 0, 980, 800, 220, 96, 11, 49, 267, 271, 3801] payload_rows_per_dst=[8712, 7748, 14360, 9398, 8742, 12136, 7366, 8994] dup_rows_per_dst=[1926, 918, 4358, 2906, 848, 5700, 2156, 2036] combine_rows_per_src=[8712, 7748, 14360, 9398, 8742, 12136, 7366, 8994] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=258662400 dispatch_meta_bytes=314992 combine_payload_bytes=258662400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258977392 ingress_bytes=293265408 link_bytes=293265408 token_bytes=517639792 weight_movement_bytes=34603008 total_nvlink_bytes=552242800 weight_to_token_ratio=0.066848 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=9728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.103534121e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=274210816 dispatch_meta_bytes=322000 combine_payload_bytes=274210816 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=274532816 ingress_bytes=412622848 link_bytes=412622848 token_bytes=548743632 weight_movement_bytes=138412032 total_nvlink_bytes=687155664 weight_to_token_ratio=0.252234 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.552666558e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78768 payload_rows_remote=67244 dup_rows=19536 dup_rows_remote=16672 dedup_ratio=0.198730 router_tokens_per_expert=[185, 1535, 430, 987, 44, 364, 1260, 411, 0, 0, 39, 582, 2695, 0, 0, 754, 362, 680, 6569, 3, 15, 927, 1431, 0, 0, 4569, 1887, 0, 0, 0, 2130, 0, 4034, 346, 46, 0, 97, 0, 317, 192, 1, 0, 0, 348, 1, 181, 3792, 1440, 872, 808, 1058, 935, 0, 28, 0, 1049, 932, 209, 82, 3, 39, 315, 379, 3789] payload_rows_per_dst=[8680, 10684, 11524, 12140, 9446, 9532, 7422, 9340] dup_rows_per_dst=[1752, 3042, 2864, 5032, 618, 1994, 2078, 2156] combine_rows_per_src=[8680, 10684, 11524, 12140, 9446, 9532, 7422, 9340] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79226 payload_rows_remote=69436 dup_rows=19078 dup_rows_remote=18376 dedup_ratio=0.194071 router_tokens_per_expert=[161, 1517, 344, 772, 69, 114, 1335, 374, 0, 0, 25, 574, 2757, 0, 0, 769, 348, 672, 6575, 0, 11, 764, 1277, 0, 0, 4867, 1484, 0, 0, 0, 2512, 0, 4130, 403, 70, 0, 86, 0, 408, 149, 0, 0, 0, 355, 4, 143, 3707, 1491, 942, 936, 1040, 1004, 0, 192, 0, 993, 958, 244, 75, 11, 56, 286, 431, 3717] payload_rows_per_dst=[7748, 7338, 15150, 12548, 9790, 9450, 7860, 9342] dup_rows_per_dst=[1624, 912, 4144, 5178, 702, 1950, 2354, 2214] combine_rows_per_src=[7748, 7338, 15150, 12548, 9790, 9450, 7860, 9342] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77250 payload_rows_remote=64350 dup_rows=21054 dup_rows_remote=14474 dedup_ratio=0.214172 router_tokens_per_expert=[279, 1299, 124, 1129, 72, 143, 1208, 578, 0, 0, 24, 482, 2915, 0, 0, 809, 337, 617, 6519, 5, 13, 925, 1709, 0, 0, 4609, 1872, 0, 0, 0, 2084, 0, 3822, 330, 72, 0, 46, 0, 481, 179, 0, 0, 0, 282, 22, 155, 3653, 1492, 920, 753, 1166, 1014, 0, 338, 0, 968, 841, 181, 97, 2, 47, 296, 370, 3873] payload_rows_per_dst=[7864, 7750, 6922, 7158, 11558, 9318, 13780, 12900] dup_rows_per_dst=[1800, 710, 290, 754, 3274, 1890, 5756, 6580] combine_rows_per_src=[7864, 7750, 6922, 7158, 11558, 9318, 13780, 12900] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=256073728 dispatch_meta_bytes=305640 combine_payload_bytes=256073728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=256379368 ingress_bytes=290676736 link_bytes=290676736 token_bytes=512453096 weight_movement_bytes=34603008 total_nvlink_bytes=547056104 weight_to_token_ratio=0.067524 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=22400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.093793156e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77350 payload_rows_remote=63592 dup_rows=20954 dup_rows_remote=13786 dedup_ratio=0.213155 router_tokens_per_expert=[274, 1684, 341, 981, 69, 277, 1419, 475, 1, 1, 24, 545, 2551, 0, 1, 846, 346, 674, 6446, 2, 1, 750, 1441, 0, 1, 4943, 1558, 0, 1, 0, 2308, 0, 3631, 370, 70, 2, 59, 1, 486, 156, 0, 1, 0, 326, 6, 179, 3985, 1304, 933, 868, 930, 925, 0, 201, 0, 990, 939, 233, 92, 15, 45, 316, 383, 3746] payload_rows_per_dst=[13758, 14746, 6168, 7122, 8934, 9748, 7562, 9312] dup_rows_per_dst=[7168, 6084, 260, 614, 616, 1854, 2132, 2226] combine_rows_per_src=[13758, 14746, 6168, 7122, 8934, 9748, 7562, 9312] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267550720 dispatch_meta_bytes=321872 combine_payload_bytes=267550720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267872592 ingress_bytes=302153728 link_bytes=302153728 token_bytes=535423312 weight_movement_bytes=34603008 total_nvlink_bytes=570026320 weight_to_token_ratio=0.064627 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=30080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136980153e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76726 payload_rows_remote=63150 dup_rows=21578 dup_rows_remote=15598 dedup_ratio=0.219503 router_tokens_per_expert=[136, 1451, 362, 816, 75, 213, 1462, 363, 0, 0, 16, 538, 2885, 0, 0, 735, 336, 651, 6640, 0, 3, 823, 1333, 0, 0, 5025, 1629, 0, 0, 0, 2194, 0, 4100, 392, 76, 3, 69, 0, 369, 133, 1, 0, 0, 395, 17, 206, 3860, 1550, 715, 778, 1247, 911, 0, 109, 0, 996, 811, 234, 65, 15, 34, 231, 424, 3725] payload_rows_per_dst=[8158, 7436, 6048, 7008, 15504, 9848, 13576, 9148] dup_rows_per_dst=[1598, 912, 244, 638, 8060, 2216, 5980, 1930] combine_rows_per_src=[8158, 7436, 6048, 7008, 15504, 9848, 13576, 9148] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=275431424 dispatch_meta_bytes=335664 combine_payload_bytes=275431424 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=275767088 ingress_bytes=413843456 link_bytes=413843456 token_bytes=551198512 weight_movement_bytes=138412032 total_nvlink_bytes=689610544 weight_to_token_ratio=0.251111 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=22912 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.557259608e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=284409856 dispatch_meta_bytes=351248 combine_payload_bytes=284409856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=284761104 ingress_bytes=319012864 link_bytes=319012864 token_bytes=569170960 weight_movement_bytes=34603008 total_nvlink_bytes=603773968 weight_to_token_ratio=0.060795 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=31104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.200419725e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=263577600 dispatch_meta_bytes=315296 combine_payload_bytes=263577600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263892896 ingress_bytes=298180608 link_bytes=298180608 token_bytes=527470496 weight_movement_bytes=34603008 total_nvlink_bytes=562073504 weight_to_token_ratio=0.065602 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=29824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.122029622e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=260472832 dispatch_meta_bytes=309512 combine_payload_bytes=260472832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260782344 ingress_bytes=295075840 link_bytes=295075840 token_bytes=521255176 weight_movement_bytes=34603008 total_nvlink_bytes=555858184 weight_to_token_ratio=0.066384 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.110346630e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=256073728 dispatch_meta_bytes=305640 combine_payload_bytes=256073728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=256379368 ingress_bytes=290676736 link_bytes=290676736 token_bytes=512453096 weight_movement_bytes=34603008 total_nvlink_bytes=547056104 weight_to_token_ratio=0.067524 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=22400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.093793156e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267550720 dispatch_meta_bytes=321872 combine_payload_bytes=267550720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267872592 ingress_bytes=302153728 link_bytes=302153728 token_bytes=535423312 weight_movement_bytes=34603008 total_nvlink_bytes=570026320 weight_to_token_ratio=0.064627 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=30080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136980153e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78768 payload_rows_remote=67244 dup_rows=19536 dup_rows_remote=16672 dedup_ratio=0.198730 router_tokens_per_expert=[185, 1535, 430, 987, 44, 364, 1260, 411, 0, 0, 39, 582, 2695, 0, 0, 754, 362, 680, 6569, 3, 15, 927, 1431, 0, 0, 4569, 1887, 0, 0, 0, 2130, 0, 4034, 346, 46, 0, 97, 0, 317, 192, 1, 0, 0, 348, 1, 181, 3792, 1440, 872, 808, 1058, 935, 0, 28, 0, 1049, 932, 209, 82, 3, 39, 315, 379, 3789] payload_rows_per_dst=[8680, 10684, 11524, 12140, 9446, 9532, 7422, 9340] dup_rows_per_dst=[1752, 3042, 2864, 5032, 618, 1994, 2078, 2156] combine_rows_per_src=[8680, 10684, 11524, 12140, 9446, 9532, 7422, 9340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86968 payload_rows_remote=83396 dup_rows=11336 dup_rows_remote=10962 dedup_ratio=0.115316 router_tokens_per_expert=[1944, 65, 0, 4549, 0, 0, 155, 0, 0, 129, 127, 131, 855, 453, 0, 365, 0, 0, 183, 0, 0, 690, 418, 600, 0, 69, 150, 0, 64, 75, 0, 0, 10, 6552, 206, 68, 0, 0, 0, 1689, 797, 464, 0, 791, 3141, 75, 720, 5147, 387, 75, 0, 947, 0, 5052, 6906, 0, 3799, 0, 0, 0, 0, 0, 1304, 0] payload_rows_per_dst=[10696, 14244, 12376, 14228, 3572, 10636, 11468, 9748] dup_rows_per_dst=[2730, 2980, 1700, 300, 374, 1340, 1454, 458] combine_rows_per_src=[10696, 14244, 12376, 14228, 3572, 10636, 11468, 9748] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80020 payload_rows_remote=66122 dup_rows=18284 dup_rows_remote=12948 dedup_ratio=0.185994 router_tokens_per_expert=[1518, 60, 0, 4556, 0, 0, 37, 0, 0, 159, 108, 109, 825, 403, 14, 222, 0, 0, 430, 0, 0, 811, 401, 610, 1, 73, 126, 0, 48, 65, 0, 0, 9, 6074, 280, 91, 0, 0, 0, 1715, 783, 530, 8, 720, 3386, 65, 703, 5135, 376, 63, 0, 864, 0, 6114, 6912, 0, 3532, 0, 0, 0, 0, 0, 1216, 0] payload_rows_per_dst=[10426, 3588, 6268, 9840, 13312, 10352, 13898, 12336] dup_rows_per_dst=[1916, 92, 1218, 210, 3026, 2554, 5336, 3932] combine_rows_per_src=[10426, 3588, 6268, 9840, 13312, 10352, 13898, 12336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87196 payload_rows_remote=72952 dup_rows=11108 dup_rows_remote=10832 dedup_ratio=0.112996 router_tokens_per_expert=[1524, 53, 0, 4480, 0, 0, 75, 0, 0, 129, 131, 117, 855, 408, 173, 465, 0, 0, 114, 0, 0, 806, 414, 709, 0, 66, 138, 0, 78, 69, 0, 0, 19, 5924, 247, 76, 0, 0, 0, 1618, 843, 573, 0, 735, 3465, 69, 679, 5247, 402, 69, 0, 1012, 0, 5329, 6909, 0, 3996, 0, 0, 0, 0, 0, 1136, 0] payload_rows_per_dst=[10266, 13862, 12716, 14244, 5870, 11206, 9326, 9706] dup_rows_per_dst=[1998, 2542, 1864, 276, 926, 1522, 1422, 558] combine_rows_per_src=[10266, 13862, 12716, 14244, 5870, 11206, 9326, 9706] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=258662400 dispatch_meta_bytes=314992 combine_payload_bytes=258662400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258977392 ingress_bytes=293265408 link_bytes=293265408 token_bytes=517639792 weight_movement_bytes=34603008 total_nvlink_bytes=552242800 weight_to_token_ratio=0.066848 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=9728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.103534121e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=341590016 dispatch_meta_bytes=377432 combine_payload_bytes=341590016 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=341967448 ingress_bytes=410796032 link_bytes=410796032 token_bytes=683557464 weight_movement_bytes=69206016 total_nvlink_bytes=752763480 weight_to_token_ratio=0.101244 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.545792397e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69112 payload_rows_remote=56464 dup_rows=29192 dup_rows_remote=25554 dedup_ratio=0.296956 router_tokens_per_expert=[1704, 45, 0, 4536, 0, 0, 122, 0, 0, 128, 216, 92, 824, 435, 37, 263, 0, 0, 179, 0, 0, 807, 433, 474, 0, 67, 189, 0, 174, 95, 0, 0, 1, 6174, 168, 56, 0, 0, 0, 1588, 759, 473, 0, 791, 2632, 96, 712, 5632, 326, 95, 0, 755, 0, 5663, 6900, 0, 3631, 0, 0, 0, 0, 0, 1880, 0] payload_rows_per_dst=[6320, 3652, 3492, 862, 13406, 13112, 15620, 12648] dup_rows_per_dst=[1034, 338, 294, 188, 2568, 3814, 17318, 3638] combine_rows_per_src=[6320, 3652, 3492, 862, 13406, 13112, 15620, 12648] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=275431424 dispatch_meta_bytes=335664 combine_payload_bytes=275431424 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=275767088 ingress_bytes=413843456 link_bytes=413843456 token_bytes=551198512 weight_movement_bytes=138412032 total_nvlink_bytes=689610544 weight_to_token_ratio=0.251111 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=22912 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.557259608e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85378 payload_rows_remote=73840 dup_rows=12926 dup_rows_remote=9076 dedup_ratio=0.131490 router_tokens_per_expert=[1881, 54, 0, 4777, 0, 1, 161, 0, 1, 131, 194, 132, 824, 470, 73, 296, 0, 0, 132, 1, 0, 782, 434, 736, 0, 70, 185, 0, 141, 91, 0, 0, 8, 6167, 259, 69, 1, 1, 0, 1621, 784, 562, 18, 965, 3067, 94, 615, 5148, 360, 91, 1, 860, 0, 4877, 6832, 1, 4092, 0, 0, 0, 2, 0, 1090, 0] payload_rows_per_dst=[11538, 13918, 12628, 14270, 11840, 9294, 2014, 9876] dup_rows_per_dst=[3850, 2658, 1838, 368, 1832, 1276, 612, 492] combine_rows_per_src=[11538, 13918, 12628, 14270, 11840, 9294, 2014, 9876] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86252 payload_rows_remote=72102 dup_rows=12052 dup_rows_remote=9400 dedup_ratio=0.122599 router_tokens_per_expert=[1630, 60, 0, 4526, 0, 0, 6, 0, 0, 168, 141, 124, 885, 435, 14, 242, 0, 0, 154, 0, 0, 849, 422, 665, 0, 90, 142, 0, 81, 71, 0, 0, 0, 6392, 264, 76, 0, 0, 0, 1748, 800, 546, 1, 711, 3097, 74, 748, 5095, 421, 71, 0, 890, 0, 5394, 6862, 0, 3824, 0, 0, 0, 0, 0, 1433, 0] payload_rows_per_dst=[10364, 14150, 12528, 14208, 12720, 10474, 1970, 9838] dup_rows_per_dst=[2080, 2652, 1842, 284, 2244, 1480, 794, 676] combine_rows_per_src=[10364, 14150, 12528, 14208, 12720, 10474, 1970, 9838] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=270835712 dispatch_meta_bytes=316280 combine_payload_bytes=270835712 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=271151992 ingress_bytes=374644736 link_bytes=374644736 token_bytes=541987704 weight_movement_bytes=103809024 total_nvlink_bytes=645796728 weight_to_token_ratio=0.191534 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=26624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.409757980e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86160 payload_rows_remote=78422 dup_rows=12144 dup_rows_remote=11132 dedup_ratio=0.123535 router_tokens_per_expert=[1829, 66, 0, 4255, 0, 0, 44, 0, 0, 142, 227, 9, 837, 369, 31, 91, 0, 0, 126, 0, 0, 936, 393, 408, 0, 60, 179, 0, 161, 90, 0, 0, 8, 6196, 164, 81, 0, 0, 0, 1574, 813, 373, 0, 814, 2962, 90, 603, 5426, 392, 90, 0, 942, 0, 5819, 6781, 0, 3806, 0, 0, 0, 0, 0, 1965, 0] payload_rows_per_dst=[10172, 6580, 12788, 14184, 10300, 7738, 12710, 11688] dup_rows_per_dst=[2216, 690, 1790, 358, 1888, 1012, 1776, 2414] combine_rows_per_src=[10172, 6580, 12788, 14184, 10300, 7738, 12710, 11688] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=298811392 dispatch_meta_bytes=335136 combine_payload_bytes=298811392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=299146528 ingress_bytes=333414400 link_bytes=333414400 token_bytes=597957920 weight_movement_bytes=34603008 total_nvlink_bytes=632560928 weight_to_token_ratio=0.057869 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=45952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.254611545e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86440 payload_rows_remote=73456 dup_rows=11864 dup_rows_remote=9886 dedup_ratio=0.120687 router_tokens_per_expert=[1701, 90, 0, 4547, 0, 0, 11, 0, 0, 143, 162, 134, 830, 487, 32, 529, 0, 0, 138, 0, 0, 909, 443, 713, 0, 40, 158, 0, 72, 82, 0, 0, 15, 5862, 231, 112, 0, 0, 0, 1590, 768, 549, 0, 725, 3316, 82, 807, 5278, 339, 82, 0, 872, 0, 5265, 6890, 0, 3789, 0, 0, 0, 0, 0, 1359, 0] payload_rows_per_dst=[10636, 13634, 12984, 14168, 12566, 10826, 1938, 9688] dup_rows_per_dst=[2062, 2724, 1978, 316, 1860, 1668, 648, 608] combine_rows_per_src=[10636, 13634, 12984, 14168, 12566, 10826, 1938, 9688] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86968 payload_rows_remote=83396 dup_rows=11336 dup_rows_remote=10962 dedup_ratio=0.115316 router_tokens_per_expert=[1944, 65, 0, 4549, 0, 0, 155, 0, 0, 129, 127, 131, 855, 453, 0, 365, 0, 0, 183, 0, 0, 690, 418, 600, 0, 69, 150, 0, 64, 75, 0, 0, 10, 6552, 206, 68, 0, 0, 0, 1689, 797, 464, 0, 791, 3141, 75, 720, 5147, 387, 75, 0, 947, 0, 5052, 6906, 0, 3799, 0, 0, 0, 0, 0, 1304, 0] payload_rows_per_dst=[10696, 14244, 12376, 14228, 3572, 10636, 11468, 9748] dup_rows_per_dst=[2730, 2980, 1700, 300, 374, 1340, 1454, 458] combine_rows_per_src=[10696, 14244, 12376, 14228, 3572, 10636, 11468, 9748] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=295329792 dispatch_meta_bytes=326008 combine_payload_bytes=295329792 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=295655800 ingress_bytes=329932800 link_bytes=329932800 token_bytes=590985592 weight_movement_bytes=34603008 total_nvlink_bytes=625588600 weight_to_token_ratio=0.058551 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=32896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.241510565e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69112 payload_rows_remote=56464 dup_rows=29192 dup_rows_remote=25554 dedup_ratio=0.296956 router_tokens_per_expert=[1704, 45, 0, 4536, 0, 0, 122, 0, 0, 128, 216, 92, 824, 435, 37, 263, 0, 0, 179, 0, 0, 807, 433, 474, 0, 67, 189, 0, 174, 95, 0, 0, 1, 6174, 168, 56, 0, 0, 0, 1588, 759, 473, 0, 791, 2632, 96, 712, 5632, 326, 95, 0, 755, 0, 5663, 6900, 0, 3631, 0, 0, 0, 0, 0, 1880, 0] payload_rows_per_dst=[6320, 3652, 3492, 862, 13406, 13112, 15620, 12648] dup_rows_per_dst=[1034, 338, 294, 188, 2568, 3814, 17318, 3638] combine_rows_per_src=[6320, 3652, 3492, 862, 13406, 13112, 15620, 12648] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85378 payload_rows_remote=73840 dup_rows=12926 dup_rows_remote=9076 dedup_ratio=0.131490 router_tokens_per_expert=[1881, 54, 0, 4777, 0, 1, 161, 0, 1, 131, 194, 132, 824, 470, 73, 296, 0, 0, 132, 1, 0, 782, 434, 736, 0, 70, 185, 0, 141, 91, 0, 0, 8, 6167, 259, 69, 1, 1, 0, 1621, 784, 562, 18, 965, 3067, 94, 615, 5148, 360, 91, 1, 860, 0, 4877, 6832, 1, 4092, 0, 0, 0, 2, 0, 1090, 0] payload_rows_per_dst=[11538, 13918, 12628, 14270, 11840, 9294, 2014, 9876] dup_rows_per_dst=[3850, 2658, 1838, 368, 1832, 1276, 612, 492] combine_rows_per_src=[11538, 13918, 12628, 14270, 11840, 9294, 2014, 9876] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=321216512 dispatch_meta_bytes=358216 combine_payload_bytes=321216512 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=321574728 ingress_bytes=425025536 link_bytes=425025536 token_bytes=642791240 weight_movement_bytes=103809024 total_nvlink_bytes=746600264 weight_to_token_ratio=0.161497 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.599336875e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80020 payload_rows_remote=66122 dup_rows=18284 dup_rows_remote=12948 dedup_ratio=0.185994 router_tokens_per_expert=[1518, 60, 0, 4556, 0, 0, 37, 0, 0, 159, 108, 109, 825, 403, 14, 222, 0, 0, 430, 0, 0, 811, 401, 610, 1, 73, 126, 0, 48, 65, 0, 0, 9, 6074, 280, 91, 0, 0, 0, 1715, 783, 530, 8, 720, 3386, 65, 703, 5135, 376, 63, 0, 864, 0, 6114, 6912, 0, 3532, 0, 0, 0, 0, 0, 1216, 0] payload_rows_per_dst=[10426, 3588, 6268, 9840, 13312, 10352, 13898, 12336] dup_rows_per_dst=[1916, 92, 1218, 210, 3026, 2554, 5336, 3932] combine_rows_per_src=[10426, 3588, 6268, 9840, 13312, 10352, 13898, 12336] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=300875776 dispatch_meta_bytes=333368 combine_payload_bytes=300875776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=301209144 ingress_bytes=335478784 link_bytes=335478784 token_bytes=602084920 weight_movement_bytes=34603008 total_nvlink_bytes=636687928 weight_to_token_ratio=0.057472 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=32896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262379656e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=341590016 dispatch_meta_bytes=377432 combine_payload_bytes=341590016 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=341967448 ingress_bytes=410796032 link_bytes=410796032 token_bytes=683557464 weight_movement_bytes=69206016 total_nvlink_bytes=752763480 weight_to_token_ratio=0.101244 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.545792397e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=231276544 dispatch_meta_bytes=328072 combine_payload_bytes=231276544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=231604616 ingress_bytes=265879552 link_bytes=265879552 token_bytes=462881160 weight_movement_bytes=34603008 total_nvlink_bytes=497484168 weight_to_token_ratio=0.074756 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=30592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.000483349e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=302448640 dispatch_meta_bytes=331664 combine_payload_bytes=302448640 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=302780304 ingress_bytes=371654656 link_bytes=371654656 token_bytes=605228944 weight_movement_bytes=69206016 total_nvlink_bytes=674434960 weight_to_token_ratio=0.114347 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=33536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.398506550e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=295329792 dispatch_meta_bytes=326008 combine_payload_bytes=295329792 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=295655800 ingress_bytes=329932800 link_bytes=329932800 token_bytes=590985592 weight_movement_bytes=34603008 total_nvlink_bytes=625588600 weight_to_token_ratio=0.058551 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=32896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.241510565e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=321216512 dispatch_meta_bytes=358216 combine_payload_bytes=321216512 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=321574728 ingress_bytes=425025536 link_bytes=425025536 token_bytes=642791240 weight_movement_bytes=103809024 total_nvlink_bytes=746600264 weight_to_token_ratio=0.161497 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.599336875e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86230 payload_rows_remote=73076 dup_rows=12074 dup_rows_remote=9176 dedup_ratio=0.122823 router_tokens_per_expert=[555, 7523, 0, 0, 327, 0, 63, 0, 0, 366, 64, 3624, 0, 69, 507, 1838, 69, 1409, 0, 0, 270, 63, 0, 394, 6586, 4143, 255, 479, 0, 0, 0, 42, 0, 239, 218, 0, 82, 1581, 462, 0, 407, 0, 0, 69, 0, 4486, 0, 0, 5444, 0, 0, 4080, 75, 28, 0, 0, 0, 0, 0, 0, 0, 2925, 202, 208] payload_rows_per_dst=[15322, 10942, 12920, 9972, 13154, 9110, 8216, 6594] dup_rows_per_dst=[1614, 1994, 3294, 1234, 2898, 814, 150, 76] combine_rows_per_src=[15322, 10942, 12920, 9972, 13154, 9110, 8216, 6594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86440 payload_rows_remote=73456 dup_rows=11864 dup_rows_remote=9886 dedup_ratio=0.120687 router_tokens_per_expert=[1701, 90, 0, 4547, 0, 0, 11, 0, 0, 143, 162, 134, 830, 487, 32, 529, 0, 0, 138, 0, 0, 909, 443, 713, 0, 40, 158, 0, 72, 82, 0, 0, 15, 5862, 231, 112, 0, 0, 0, 1590, 768, 549, 0, 725, 3316, 82, 807, 5278, 339, 82, 0, 872, 0, 5265, 6890, 0, 3789, 0, 0, 0, 0, 0, 1359, 0] payload_rows_per_dst=[10636, 13634, 12984, 14168, 12566, 10826, 1938, 9688] dup_rows_per_dst=[2062, 2724, 1978, 316, 1860, 1668, 648, 608] combine_rows_per_src=[10636, 13634, 12984, 14168, 12566, 10826, 1938, 9688] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76530 payload_rows_remote=62420 dup_rows=21774 dup_rows_remote=16574 dedup_ratio=0.221497 router_tokens_per_expert=[575, 7535, 0, 0, 330, 0, 60, 0, 0, 398, 60, 3284, 0, 92, 526, 1696, 92, 1282, 0, 0, 229, 60, 0, 378, 6414, 4338, 236, 551, 0, 0, 0, 51, 0, 216, 188, 0, 65, 1711, 358, 0, 386, 0, 0, 75, 0, 4626, 0, 0, 5676, 0, 0, 3871, 82, 26, 0, 0, 0, 0, 0, 0, 0, 3223, 274, 188] payload_rows_per_dst=[1422, 10292, 3204, 13490, 4722, 13784, 14110, 15506] dup_rows_per_dst=[508, 1820, 878, 1014, 354, 5066, 5200, 6934] combine_rows_per_src=[1422, 10292, 3204, 13490, 4722, 13784, 14110, 15506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86626 payload_rows_remote=76496 dup_rows=11678 dup_rows_remote=11126 dedup_ratio=0.118795 router_tokens_per_expert=[656, 7520, 0, 0, 405, 0, 53, 0, 0, 244, 53, 3356, 0, 79, 511, 1816, 79, 1264, 0, 0, 265, 53, 2, 404, 6561, 4549, 217, 546, 0, 0, 0, 29, 0, 214, 188, 0, 69, 1439, 476, 0, 425, 0, 0, 66, 0, 4748, 0, 0, 5720, 0, 0, 3861, 70, 47, 0, 0, 0, 0, 0, 0, 0, 2749, 205, 213] payload_rows_per_dst=[15284, 10184, 13954, 10130, 13354, 9628, 7818, 6274] dup_rows_per_dst=[1984, 1934, 3302, 552, 2858, 850, 138, 60] combine_rows_per_src=[15284, 10184, 13954, 10130, 13354, 9628, 7818, 6274] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=270835712 dispatch_meta_bytes=316280 combine_payload_bytes=270835712 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=271151992 ingress_bytes=374644736 link_bytes=374644736 token_bytes=541987704 weight_movement_bytes=103809024 total_nvlink_bytes=645796728 weight_to_token_ratio=0.191534 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=26624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.409757980e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=299319296 dispatch_meta_bytes=329008 combine_payload_bytes=299319296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=299648304 ingress_bytes=333922304 link_bytes=333922304 token_bytes=598967600 weight_movement_bytes=34603008 total_nvlink_bytes=633570608 weight_to_token_ratio=0.057771 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=28416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256522747e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74530 payload_rows_remote=58990 dup_rows=23774 dup_rows_remote=16282 dedup_ratio=0.241842 router_tokens_per_expert=[690, 7575, 0, 0, 370, 0, 42, 0, 0, 297, 42, 3710, 0, 57, 537, 1567, 57, 1400, 0, 0, 198, 42, 0, 336, 6600, 3973, 202, 515, 0, 0, 0, 44, 0, 242, 151, 0, 99, 1607, 486, 0, 511, 0, 0, 72, 0, 4367, 0, 0, 4935, 0, 0, 4382, 95, 10, 0, 0, 0, 0, 0, 0, 0, 3319, 281, 341] payload_rows_per_dst=[1696, 8354, 3226, 13824, 4756, 13100, 14034, 15540] dup_rows_per_dst=[508, 1618, 840, 964, 414, 4746, 7192, 7492] combine_rows_per_src=[1696, 8354, 3226, 13824, 4756, 13100, 14034, 15540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83832 payload_rows_remote=68462 dup_rows=14472 dup_rows_remote=9836 dedup_ratio=0.147217 router_tokens_per_expert=[706, 7523, 0, 1, 367, 0, 57, 0, 0, 332, 54, 3721, 1, 70, 526, 1834, 69, 1287, 1, 2, 299, 54, 4, 392, 6569, 3871, 210, 523, 0, 0, 0, 44, 1, 259, 181, 0, 96, 1483, 469, 0, 485, 0, 0, 69, 2, 4319, 0, 1, 5571, 1, 0, 4046, 96, 82, 0, 0, 0, 1, 2, 0, 0, 2954, 217, 300] payload_rows_per_dst=[15370, 11080, 14000, 6260, 13206, 8786, 8268, 6862] dup_rows_per_dst=[4636, 1996, 3354, 338, 2914, 966, 182, 86] combine_rows_per_src=[15370, 11080, 14000, 6260, 13206, 8786, 8268, 6862] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=300875776 dispatch_meta_bytes=333368 combine_payload_bytes=300875776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=301209144 ingress_bytes=335478784 link_bytes=335478784 token_bytes=602084920 weight_movement_bytes=34603008 total_nvlink_bytes=636687928 weight_to_token_ratio=0.057472 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=32896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262379656e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=255672320 dispatch_meta_bytes=315976 combine_payload_bytes=255672320 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=255988296 ingress_bytes=324878336 link_bytes=324878336 token_bytes=511660616 weight_movement_bytes=69206016 total_nvlink_bytes=580866632 weight_to_token_ratio=0.135258 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=32000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.222491024e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86386 payload_rows_remote=75400 dup_rows=11918 dup_rows_remote=9860 dedup_ratio=0.121236 router_tokens_per_expert=[626, 7490, 0, 0, 319, 0, 60, 0, 0, 372, 60, 3649, 0, 76, 538, 1827, 76, 1344, 0, 0, 237, 60, 0, 423, 6280, 4132, 211, 453, 0, 0, 0, 50, 0, 219, 197, 0, 80, 1903, 429, 0, 429, 0, 0, 92, 0, 4288, 0, 0, 5403, 0, 0, 4038, 111, 5, 0, 0, 0, 0, 0, 0, 0, 3183, 245, 247] payload_rows_per_dst=[15248, 10986, 13476, 9248, 13252, 8760, 8166, 7250] dup_rows_per_dst=[1742, 2058, 3364, 444, 3210, 858, 142, 100] combine_rows_per_src=[15248, 10986, 13476, 9248, 13252, 8760, 8166, 7250] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=313327616 dispatch_meta_bytes=350488 combine_payload_bytes=313327616 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=313678104 ingress_bytes=451739648 link_bytes=451739648 token_bytes=627005720 weight_movement_bytes=138412032 total_nvlink_bytes=765417752 weight_to_token_ratio=0.220751 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=22528 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.699859927e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77144 payload_rows_remote=66840 dup_rows=21160 dup_rows_remote=19502 dedup_ratio=0.215251 router_tokens_per_expert=[595, 7467, 0, 0, 349, 0, 66, 0, 0, 299, 66, 3470, 0, 87, 530, 1566, 87, 1669, 0, 0, 248, 66, 0, 399, 6725, 3744, 215, 602, 0, 0, 0, 23, 0, 201, 191, 0, 94, 1710, 497, 0, 464, 0, 0, 60, 0, 4732, 0, 0, 5447, 0, 0, 3757, 91, 0, 0, 0, 0, 0, 0, 0, 0, 2967, 357, 311] payload_rows_per_dst=[4984, 10258, 3974, 15208, 6456, 10304, 12692, 13268] dup_rows_per_dst=[806, 1778, 964, 5960, 826, 1658, 4002, 5166] combine_rows_per_src=[4984, 10258, 3974, 15208, 6456, 10304, 12692, 13268] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86298 payload_rows_remote=72188 dup_rows=12006 dup_rows_remote=8562 dedup_ratio=0.122131 router_tokens_per_expert=[621, 7485, 0, 0, 394, 0, 90, 0, 0, 297, 90, 3529, 0, 113, 589, 1887, 112, 1415, 0, 0, 298, 90, 0, 341, 6521, 4036, 201, 555, 1, 0, 0, 40, 0, 226, 187, 0, 87, 1711, 476, 0, 433, 0, 0, 44, 0, 4460, 0, 0, 5278, 0, 0, 3882, 82, 33, 0, 0, 0, 0, 0, 0, 0, 3121, 229, 198] payload_rows_per_dst=[15314, 10928, 14110, 9122, 12966, 9008, 7830, 7020] dup_rows_per_dst=[1866, 2082, 3444, 544, 2964, 866, 164, 76] combine_rows_per_src=[15314, 10928, 14110, 9122, 12966, 9008, 7830, 7020] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86230 payload_rows_remote=73076 dup_rows=12074 dup_rows_remote=9176 dedup_ratio=0.122823 router_tokens_per_expert=[555, 7523, 0, 0, 327, 0, 63, 0, 0, 366, 64, 3624, 0, 69, 507, 1838, 69, 1409, 0, 0, 270, 63, 0, 394, 6586, 4143, 255, 479, 0, 0, 0, 42, 0, 239, 218, 0, 82, 1581, 462, 0, 407, 0, 0, 69, 0, 4486, 0, 0, 5444, 0, 0, 4080, 75, 28, 0, 0, 0, 0, 0, 0, 0, 2925, 202, 208] payload_rows_per_dst=[15322, 10942, 12920, 9972, 13154, 9110, 8216, 6594] dup_rows_per_dst=[1614, 1994, 3294, 1234, 2898, 814, 150, 76] combine_rows_per_src=[15322, 10942, 12920, 9972, 13154, 9110, 8216, 6594] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=308838400 dispatch_meta_bytes=341040 combine_payload_bytes=308838400 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=309179440 ingress_bytes=378044416 link_bytes=378044416 token_bytes=618017840 weight_movement_bytes=69206016 total_nvlink_bytes=687223856 weight_to_token_ratio=0.111981 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=27136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.422550702e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74530 payload_rows_remote=58990 dup_rows=23774 dup_rows_remote=16282 dedup_ratio=0.241842 router_tokens_per_expert=[690, 7575, 0, 0, 370, 0, 42, 0, 0, 297, 42, 3710, 0, 57, 537, 1567, 57, 1400, 0, 0, 198, 42, 0, 336, 6600, 3973, 202, 515, 0, 0, 0, 44, 0, 242, 151, 0, 99, 1607, 486, 0, 511, 0, 0, 72, 0, 4367, 0, 0, 4935, 0, 0, 4382, 95, 10, 0, 0, 0, 0, 0, 0, 0, 3319, 281, 341] payload_rows_per_dst=[1696, 8354, 3226, 13824, 4756, 13100, 14034, 15540] dup_rows_per_dst=[508, 1618, 840, 964, 414, 4746, 7192, 7492] combine_rows_per_src=[1696, 8354, 3226, 13824, 4756, 13100, 14034, 15540] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83832 payload_rows_remote=68462 dup_rows=14472 dup_rows_remote=9836 dedup_ratio=0.147217 router_tokens_per_expert=[706, 7523, 0, 1, 367, 0, 57, 0, 0, 332, 54, 3721, 1, 70, 526, 1834, 69, 1287, 1, 2, 299, 54, 4, 392, 6569, 3871, 210, 523, 0, 0, 0, 44, 1, 259, 181, 0, 96, 1483, 469, 0, 485, 0, 0, 69, 2, 4319, 0, 1, 5571, 1, 0, 4046, 96, 82, 0, 0, 0, 1, 2, 0, 0, 2954, 217, 300] payload_rows_per_dst=[15370, 11080, 14000, 6260, 13206, 8786, 8268, 6862] dup_rows_per_dst=[4636, 1996, 3354, 338, 2914, 966, 182, 86] combine_rows_per_src=[15370, 11080, 14000, 6260, 13206, 8786, 8268, 6862] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76530 payload_rows_remote=62420 dup_rows=21774 dup_rows_remote=16574 dedup_ratio=0.221497 router_tokens_per_expert=[575, 7535, 0, 0, 330, 0, 60, 0, 0, 398, 60, 3284, 0, 92, 526, 1696, 92, 1282, 0, 0, 229, 60, 0, 378, 6414, 4338, 236, 551, 0, 0, 0, 51, 0, 216, 188, 0, 65, 1711, 358, 0, 386, 0, 0, 75, 0, 4626, 0, 0, 5676, 0, 0, 3871, 82, 26, 0, 0, 0, 0, 0, 0, 0, 3223, 274, 188] payload_rows_per_dst=[1422, 10292, 3204, 13490, 4722, 13784, 14110, 15506] dup_rows_per_dst=[508, 1820, 878, 1014, 354, 5066, 5200, 6934] combine_rows_per_src=[1422, 10292, 3204, 13490, 4722, 13784, 14110, 15506] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=273776640 dispatch_meta_bytes=345368 combine_payload_bytes=273776640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274122008 ingress_bytes=308379648 link_bytes=308379648 token_bytes=547898648 weight_movement_bytes=34603008 total_nvlink_bytes=582501656 weight_to_token_ratio=0.063156 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=36096 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.160407789e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=295682048 dispatch_meta_bytes=323000 combine_payload_bytes=295682048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296005048 ingress_bytes=330285056 link_bytes=330285056 token_bytes=591687096 weight_movement_bytes=34603008 total_nvlink_bytes=626290104 weight_to_token_ratio=0.058482 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=31872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.242836076e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=299319296 dispatch_meta_bytes=329008 combine_payload_bytes=299319296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=299648304 ingress_bytes=333922304 link_bytes=333922304 token_bytes=598967600 weight_movement_bytes=34603008 total_nvlink_bytes=633570608 weight_to_token_ratio=0.057771 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=28416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256522747e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=313327616 dispatch_meta_bytes=350488 combine_payload_bytes=313327616 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=313678104 ingress_bytes=451739648 link_bytes=451739648 token_bytes=627005720 weight_movement_bytes=138412032 total_nvlink_bytes=765417752 weight_to_token_ratio=0.220751 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=22528 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.699859927e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=241623040 dispatch_meta_bytes=301088 combine_payload_bytes=241623040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=241924128 ingress_bytes=276226048 link_bytes=276226048 token_bytes=483547168 weight_movement_bytes=34603008 total_nvlink_bytes=518150176 weight_to_token_ratio=0.071561 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=24448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.039416381e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=280420352 dispatch_meta_bytes=313192 combine_payload_bytes=280420352 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=280733544 ingress_bytes=349626368 link_bytes=349626368 token_bytes=561153896 weight_movement_bytes=69206016 total_nvlink_bytes=630359912 weight_to_token_ratio=0.123328 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=39552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.315615876e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=255672320 dispatch_meta_bytes=315976 combine_payload_bytes=255672320 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=255988296 ingress_bytes=324878336 link_bytes=324878336 token_bytes=511660616 weight_movement_bytes=69206016 total_nvlink_bytes=580866632 weight_to_token_ratio=0.135258 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=32000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.222491024e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86114 payload_rows_remote=71602 dup_rows=12190 dup_rows_remote=11870 dedup_ratio=0.124003 router_tokens_per_expert=[758, 69, 22, 154, 6937, 0, 7428, 5099, 0, 138, 0, 408, 0, 226, 0, 2755, 262, 345, 3079, 0, 712, 0, 402, 79, 0, 107, 69, 0, 0, 123, 0, 180, 5880, 0, 0, 69, 207, 1183, 2845, 0, 407, 0, 0, 0, 0, 0, 0, 223, 1178, 0, 0, 0, 0, 693, 70, 0, 4888, 0, 0, 812, 1267, 78, 0, 0] payload_rows_per_dst=[2006, 12972, 9272, 14512, 7126, 15720, 13764, 10742] dup_rows_per_dst=[0, 4280, 1574, 320, 1482, 396, 1878, 2260] combine_rows_per_src=[2006, 12972, 9272, 14512, 7126, 15720, 13764, 10742] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=64610 payload_rows_remote=56818 dup_rows=33694 dup_rows_remote=33162 dedup_ratio=0.342753 router_tokens_per_expert=[813, 71, 8, 157, 6912, 0, 7409, 4817, 0, 145, 0, 390, 0, 228, 0, 3399, 168, 348, 2618, 0, 397, 0, 421, 76, 0, 432, 74, 0, 0, 134, 0, 200, 6013, 0, 0, 74, 165, 1372, 2462, 0, 383, 0, 0, 0, 0, 0, 0, 229, 1034, 0, 0, 0, 0, 570, 97, 0, 4952, 0, 0, 1000, 1503, 81, 0, 0] payload_rows_per_dst=[16232, 7792, 13480, 4462, 13876, 774, 3394, 4600] dup_rows_per_dst=[21034, 532, 4480, 326, 6296, 450, 8, 568] combine_rows_per_src=[16232, 7792, 13480, 4462, 13876, 774, 3394, 4600] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=293281792 dispatch_meta_bytes=333888 combine_payload_bytes=293281792 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293615680 ingress_bytes=327884800 link_bytes=327884800 token_bytes=586897472 weight_movement_bytes=34603008 total_nvlink_bytes=621500480 weight_to_token_ratio=0.058959 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=43136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.233804106e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85482 payload_rows_remote=71938 dup_rows=12822 dup_rows_remote=8218 dedup_ratio=0.130432 router_tokens_per_expert=[813, 82, 8, 172, 6966, 0, 7369, 4826, 0, 164, 0, 417, 0, 267, 0, 3129, 252, 361, 2691, 0, 595, 0, 340, 113, 0, 181, 82, 0, 0, 172, 0, 173, 5845, 0, 0, 82, 224, 1346, 2787, 0, 459, 0, 0, 0, 0, 0, 0, 253, 1066, 0, 0, 0, 0, 622, 55, 0, 4722, 0, 0, 1015, 1431, 72, 0, 0] payload_rows_per_dst=[8528, 9066, 13544, 14608, 8060, 15000, 12150, 4526] dup_rows_per_dst=[1186, 1632, 4604, 540, 2342, 506, 1502, 510] combine_rows_per_src=[8528, 9066, 13544, 14608, 8060, 15000, 12150, 4526] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85054 payload_rows_remote=69560 dup_rows=13250 dup_rows_remote=12760 dedup_ratio=0.134786 router_tokens_per_expert=[790, 90, 6, 117, 6771, 0, 7322, 4663, 0, 166, 0, 494, 0, 338, 0, 3301, 105, 303, 2803, 0, 282, 0, 392, 87, 2, 464, 90, 0, 0, 157, 0, 176, 5901, 1, 0, 90, 204, 1286, 2772, 0, 423, 0, 0, 0, 0, 0, 0, 247, 1075, 0, 0, 0, 0, 627, 63, 0, 4725, 0, 0, 1185, 1473, 161, 0, 0] payload_rows_per_dst=[2006, 12814, 7600, 14258, 7226, 15494, 13646, 12010] dup_rows_per_dst=[0, 5110, 344, 1062, 1480, 490, 1686, 3078] combine_rows_per_src=[2006, 12814, 7600, 14258, 7226, 15494, 13646, 12010] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=308838400 dispatch_meta_bytes=341040 combine_payload_bytes=308838400 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=309179440 ingress_bytes=378044416 link_bytes=378044416 token_bytes=618017840 weight_movement_bytes=69206016 total_nvlink_bytes=687223856 weight_to_token_ratio=0.111981 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=27136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.422550702e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=273776640 dispatch_meta_bytes=345368 combine_payload_bytes=273776640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274122008 ingress_bytes=308379648 link_bytes=308379648 token_bytes=547898648 weight_movement_bytes=34603008 total_nvlink_bytes=582501656 weight_to_token_ratio=0.063156 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=36096 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.160407789e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85854 payload_rows_remote=79082 dup_rows=12450 dup_rows_remote=10978 dedup_ratio=0.126648 router_tokens_per_expert=[821, 75, 9, 179, 7125, 0, 7445, 4976, 0, 150, 0, 365, 0, 204, 0, 3070, 142, 344, 3037, 0, 357, 0, 387, 69, 0, 428, 75, 0, 0, 139, 0, 190, 5864, 0, 0, 75, 194, 1425, 2428, 0, 389, 0, 0, 0, 0, 0, 0, 242, 1109, 0, 0, 0, 0, 753, 72, 0, 4652, 0, 0, 916, 1380, 66, 0, 0] payload_rows_per_dst=[2168, 12876, 8326, 14930, 6772, 15698, 13710, 11374] dup_rows_per_dst=[0, 4654, 346, 984, 1472, 454, 1886, 2654] combine_rows_per_src=[2168, 12876, 8326, 14930, 6772, 15698, 13710, 11374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=232726528 dispatch_meta_bytes=359920 combine_payload_bytes=232726528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=233086448 ingress_bytes=267329536 link_bytes=267329536 token_bytes=465812976 weight_movement_bytes=34603008 total_nvlink_bytes=500415984 weight_to_token_ratio=0.074285 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=24960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.005939522e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86298 payload_rows_remote=72188 dup_rows=12006 dup_rows_remote=8562 dedup_ratio=0.122131 router_tokens_per_expert=[621, 7485, 0, 0, 394, 0, 90, 0, 0, 297, 90, 3529, 0, 113, 589, 1887, 112, 1415, 0, 0, 298, 90, 0, 341, 6521, 4036, 201, 555, 1, 0, 0, 40, 0, 226, 187, 0, 87, 1711, 476, 0, 433, 0, 0, 44, 0, 4460, 0, 0, 5278, 0, 0, 3882, 82, 33, 0, 0, 0, 0, 0, 0, 0, 3121, 229, 198] payload_rows_per_dst=[15314, 10928, 14110, 9122, 12966, 9008, 7830, 7020] dup_rows_per_dst=[1866, 2082, 3444, 544, 2964, 866, 164, 76] combine_rows_per_src=[15314, 10928, 14110, 9122, 12966, 9008, 7830, 7020] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85476 payload_rows_remote=71554 dup_rows=12828 dup_rows_remote=10912 dedup_ratio=0.130493 router_tokens_per_expert=[777, 63, 13, 152, 6946, 0, 7481, 5040, 0, 128, 0, 351, 0, 203, 0, 2834, 247, 336, 3079, 0, 445, 2, 376, 92, 0, 314, 65, 0, 0, 125, 0, 166, 5992, 0, 0, 65, 157, 1242, 2929, 0, 409, 0, 0, 0, 0, 0, 0, 185, 1197, 0, 0, 0, 0, 650, 80, 0, 4681, 0, 0, 906, 1356, 68, 0, 0] payload_rows_per_dst=[2010, 8422, 8488, 14498, 10106, 15780, 13922, 12250] dup_rows_per_dst=[0, 1376, 666, 734, 3212, 370, 1916, 4554] combine_rows_per_src=[2010, 8422, 8488, 14498, 10106, 15780, 13922, 12250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86114 payload_rows_remote=71602 dup_rows=12190 dup_rows_remote=11870 dedup_ratio=0.124003 router_tokens_per_expert=[758, 69, 22, 154, 6937, 0, 7428, 5099, 0, 138, 0, 408, 0, 226, 0, 2755, 262, 345, 3079, 0, 712, 0, 402, 79, 0, 107, 69, 0, 0, 123, 0, 180, 5880, 0, 0, 69, 207, 1183, 2845, 0, 407, 0, 0, 0, 0, 0, 0, 223, 1178, 0, 0, 0, 0, 693, 70, 0, 4888, 0, 0, 812, 1267, 78, 0, 0] payload_rows_per_dst=[2006, 12972, 9272, 14512, 7126, 15720, 13764, 10742] dup_rows_per_dst=[0, 4280, 1574, 320, 1482, 396, 1878, 2260] combine_rows_per_src=[2006, 12972, 9272, 14512, 7126, 15720, 13764, 10742] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=323919872 dispatch_meta_bytes=360240 combine_payload_bytes=323919872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=324280112 ingress_bytes=393125888 link_bytes=393125888 token_bytes=648199984 weight_movement_bytes=69206016 total_nvlink_bytes=717406000 weight_to_token_ratio=0.106766 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=21504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.479301067e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82256 payload_rows_remote=67946 dup_rows=16048 dup_rows_remote=5742 dedup_ratio=0.163249 router_tokens_per_expert=[965, 95, 7, 159, 6805, 0, 7301, 4833, 0, 190, 0, 523, 0, 326, 0, 3028, 257, 337, 2734, 0, 306, 0, 326, 57, 2, 408, 95, 0, 0, 138, 0, 207, 5811, 0, 0, 96, 200, 1284, 2797, 0, 405, 0, 0, 0, 0, 0, 0, 237, 1060, 0, 0, 0, 0, 610, 78, 0, 4679, 0, 0, 1106, 1516, 174, 0, 0] payload_rows_per_dst=[2452, 7242, 7788, 14408, 7130, 15422, 13504, 14310] dup_rows_per_dst=[0, 892, 246, 902, 1624, 464, 1614, 10306] combine_rows_per_src=[2452, 7242, 7788, 14408, 7130, 15422, 13504, 14310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=295682048 dispatch_meta_bytes=323000 combine_payload_bytes=295682048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296005048 ingress_bytes=330285056 link_bytes=330285056 token_bytes=591687096 weight_movement_bytes=34603008 total_nvlink_bytes=626290104 weight_to_token_ratio=0.058482 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=31872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.242836076e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=293085184 dispatch_meta_bytes=329864 combine_payload_bytes=293085184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293415048 ingress_bytes=327688192 link_bytes=327688192 token_bytes=586500232 weight_movement_bytes=34603008 total_nvlink_bytes=621103240 weight_to_token_ratio=0.058999 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=34560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.233064285e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=61574 payload_rows_remote=45332 dup_rows=36730 dup_rows_remote=12290 dedup_ratio=0.373637 router_tokens_per_expert=[816, 91, 27, 154, 6989, 1, 7299, 4964, 1, 182, 0, 466, 3, 298, 0, 3372, 96, 341, 2881, 0, 379, 3, 361, 69, 0, 446, 93, 0, 0, 147, 1, 193, 6059, 2, 0, 91, 216, 1260, 2420, 2, 409, 0, 1, 2, 0, 3, 0, 228, 1060, 3, 1, 0, 0, 722, 75, 1, 4643, 0, 0, 926, 1212, 143, 0, 0] payload_rows_per_dst=[16242, 7914, 13334, 1574, 14018, 854, 3696, 3942] dup_rows_per_dst=[24440, 730, 4212, 186, 6082, 432, 28, 620] combine_rows_per_src=[16242, 7914, 13334, 1574, 14018, 854, 3696, 3942] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=294658048 dispatch_meta_bytes=320624 combine_payload_bytes=294658048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=294978672 ingress_bytes=329261056 link_bytes=329261056 token_bytes=589636720 weight_movement_bytes=34603008 total_nvlink_bytes=624239728 weight_to_token_ratio=0.058685 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=23040 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.238982846e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=284917760 dispatch_meta_bytes=329280 combine_payload_bytes=284917760 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=285247040 ingress_bytes=319520768 link_bytes=319520768 token_bytes=570164800 weight_movement_bytes=34603008 total_nvlink_bytes=604767808 weight_to_token_ratio=0.060689 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=44032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.202330926e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=323919872 dispatch_meta_bytes=360240 combine_payload_bytes=323919872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=324280112 ingress_bytes=393125888 link_bytes=393125888 token_bytes=648199984 weight_movement_bytes=69206016 total_nvlink_bytes=717406000 weight_to_token_ratio=0.106766 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=21504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.479301067e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=64610 payload_rows_remote=56818 dup_rows=33694 dup_rows_remote=33162 dedup_ratio=0.342753 router_tokens_per_expert=[813, 71, 8, 157, 6912, 0, 7409, 4817, 0, 145, 0, 390, 0, 228, 0, 3399, 168, 348, 2618, 0, 397, 0, 421, 76, 0, 432, 74, 0, 0, 134, 0, 200, 6013, 0, 0, 74, 165, 1372, 2462, 0, 383, 0, 0, 0, 0, 0, 0, 229, 1034, 0, 0, 0, 0, 570, 97, 0, 4952, 0, 0, 1000, 1503, 81, 0, 0] payload_rows_per_dst=[16232, 7792, 13480, 4462, 13876, 774, 3394, 4600] dup_rows_per_dst=[21034, 532, 4480, 326, 6296, 450, 8, 568] combine_rows_per_src=[16232, 7792, 13480, 4462, 13876, 774, 3394, 4600] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85054 payload_rows_remote=69560 dup_rows=13250 dup_rows_remote=12760 dedup_ratio=0.134786 router_tokens_per_expert=[790, 90, 6, 117, 6771, 0, 7322, 4663, 0, 166, 0, 494, 0, 338, 0, 3301, 105, 303, 2803, 0, 282, 0, 392, 87, 2, 464, 90, 0, 0, 157, 0, 176, 5901, 1, 0, 90, 204, 1286, 2772, 0, 423, 0, 0, 0, 0, 0, 0, 247, 1075, 0, 0, 0, 0, 627, 63, 0, 4725, 0, 0, 1185, 1473, 161, 0, 0] payload_rows_per_dst=[2006, 12814, 7600, 14258, 7226, 15494, 13646, 12010] dup_rows_per_dst=[0, 5110, 344, 1062, 1480, 490, 1686, 3078] combine_rows_per_src=[2006, 12814, 7600, 14258, 7226, 15494, 13646, 12010] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=293085184 dispatch_meta_bytes=329864 combine_payload_bytes=293085184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293415048 ingress_bytes=327688192 link_bytes=327688192 token_bytes=586500232 weight_movement_bytes=34603008 total_nvlink_bytes=621103240 weight_to_token_ratio=0.058999 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=34560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.233064285e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90912 payload_rows_remote=74528 dup_rows=7392 dup_rows_remote=6250 dedup_ratio=0.075195 router_tokens_per_expert=[0, 8123, 4685, 0, 0, 53, 0, 312, 0, 0, 0, 0, 0, 1204, 1107, 122, 0, 69, 0, 718, 0, 0, 0, 1625, 78, 261, 0, 101, 0, 53, 147, 0, 217, 7911, 69, 0, 3923, 0, 6698, 0, 118, 0, 0, 0, 382, 0, 0, 0, 525, 0, 79, 0, 0, 0, 0, 0, 0, 0, 7288, 2753, 69, 77, 0, 385] payload_rows_per_dst=[9800, 11870, 15680, 16384, 572, 15822, 14358, 6426] dup_rows_per_dst=[300, 842, 3720, 1142, 0, 1000, 246, 142] combine_rows_per_src=[9800, 11870, 15680, 16384, 572, 15822, 14358, 6426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=293281792 dispatch_meta_bytes=333888 combine_payload_bytes=293281792 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293615680 ingress_bytes=327884800 link_bytes=327884800 token_bytes=586897472 weight_movement_bytes=34603008 total_nvlink_bytes=621500480 weight_to_token_ratio=0.058959 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=43136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.233804106e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278306816 dispatch_meta_bytes=294752 combine_payload_bytes=278306816 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=278601568 ingress_bytes=347512832 link_bytes=347512832 token_bytes=556908384 weight_movement_bytes=69206016 total_nvlink_bytes=626114400 weight_to_token_ratio=0.124268 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=23168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.307662810e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=185679872 dispatch_meta_bytes=230488 combine_payload_bytes=185679872 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=185910360 ingress_bytes=358694912 link_bytes=358694912 token_bytes=371590232 weight_movement_bytes=173015040 total_nvlink_bytes=544605272 weight_to_token_ratio=0.465607 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=15104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.349740077e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=232726528 dispatch_meta_bytes=359920 combine_payload_bytes=232726528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=233086448 ingress_bytes=267329536 link_bytes=267329536 token_bytes=465812976 weight_movement_bytes=34603008 total_nvlink_bytes=500415984 weight_to_token_ratio=0.074285 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=24960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.005939522e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=284917760 dispatch_meta_bytes=329280 combine_payload_bytes=284917760 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=285247040 ingress_bytes=319520768 link_bytes=319520768 token_bytes=570164800 weight_movement_bytes=34603008 total_nvlink_bytes=604767808 weight_to_token_ratio=0.060689 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=44032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.202330926e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90812 payload_rows_remote=86072 dup_rows=7492 dup_rows_remote=7458 dedup_ratio=0.076213 router_tokens_per_expert=[0, 8117, 4708, 0, 0, 63, 0, 175, 0, 0, 0, 0, 0, 1189, 1231, 138, 0, 75, 0, 696, 0, 0, 0, 1491, 64, 246, 0, 74, 0, 63, 139, 0, 255, 7909, 75, 0, 3898, 0, 6730, 0, 366, 0, 0, 0, 447, 0, 0, 0, 537, 0, 87, 0, 0, 0, 0, 0, 0, 0, 7179, 2629, 75, 81, 0, 415] payload_rows_per_dst=[5706, 12036, 14826, 16384, 4740, 15888, 14396, 6836] dup_rows_per_dst=[72, 876, 2984, 1022, 34, 1556, 312, 636] combine_rows_per_src=[5706, 12036, 14826, 16384, 4740, 15888, 14396, 6836] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88420 payload_rows_remote=74264 dup_rows=9884 dup_rows_remote=9686 dedup_ratio=0.100545 router_tokens_per_expert=[0, 8129, 4849, 0, 0, 60, 0, 235, 0, 0, 0, 0, 0, 1233, 961, 125, 0, 65, 0, 713, 0, 0, 0, 1515, 48, 254, 0, 102, 0, 60, 111, 0, 185, 7869, 65, 0, 4200, 0, 6633, 0, 86, 0, 0, 0, 401, 0, 0, 0, 477, 0, 67, 0, 0, 0, 0, 0, 0, 4, 7416, 2771, 63, 73, 0, 382] payload_rows_per_dst=[590, 12262, 3854, 16384, 10110, 15738, 14156, 15326] dup_rows_per_dst=[0, 776, 732, 1024, 88, 974, 198, 6092] combine_rows_per_src=[590, 12262, 3854, 16384, 10110, 15738, 14156, 15326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85482 payload_rows_remote=71938 dup_rows=12822 dup_rows_remote=8218 dedup_ratio=0.130432 router_tokens_per_expert=[813, 82, 8, 172, 6966, 0, 7369, 4826, 0, 164, 0, 417, 0, 267, 0, 3129, 252, 361, 2691, 0, 595, 0, 340, 113, 0, 181, 82, 0, 0, 172, 0, 173, 5845, 0, 0, 82, 224, 1346, 2787, 0, 459, 0, 0, 0, 0, 0, 0, 253, 1066, 0, 0, 0, 0, 622, 55, 0, 4722, 0, 0, 1015, 1431, 72, 0, 0] payload_rows_per_dst=[8528, 9066, 13544, 14608, 8060, 15000, 12150, 4526] dup_rows_per_dst=[1186, 1632, 4604, 540, 2342, 506, 1502, 510] combine_rows_per_src=[8528, 9066, 13544, 14608, 8060, 15000, 12150, 4526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=352550912 dispatch_meta_bytes=374120 combine_payload_bytes=352550912 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=352925032 ingress_bytes=490962944 link_bytes=490962944 token_bytes=705475944 weight_movement_bytes=138412032 total_nvlink_bytes=843887976 weight_to_token_ratio=0.196197 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=18816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.847454033e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=304185344 dispatch_meta_bytes=335800 combine_payload_bytes=304185344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=304521144 ingress_bytes=338788352 link_bytes=338788352 token_bytes=608706488 weight_movement_bytes=34603008 total_nvlink_bytes=643309496 weight_to_token_ratio=0.056847 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=44160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274833294e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=294658048 dispatch_meta_bytes=320624 combine_payload_bytes=294658048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=294978672 ingress_bytes=329261056 link_bytes=329261056 token_bytes=589636720 weight_movement_bytes=34603008 total_nvlink_bytes=624239728 weight_to_token_ratio=0.058685 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=23040 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.238982846e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90912 payload_rows_remote=74528 dup_rows=7392 dup_rows_remote=6250 dedup_ratio=0.075195 router_tokens_per_expert=[0, 8123, 4685, 0, 0, 53, 0, 312, 0, 0, 0, 0, 0, 1204, 1107, 122, 0, 69, 0, 718, 0, 0, 0, 1625, 78, 261, 0, 101, 0, 53, 147, 0, 217, 7911, 69, 0, 3923, 0, 6698, 0, 118, 0, 0, 0, 382, 0, 0, 0, 525, 0, 79, 0, 0, 0, 0, 0, 0, 0, 7288, 2753, 69, 77, 0, 385] payload_rows_per_dst=[9800, 11870, 15680, 16384, 572, 15822, 14358, 6426] dup_rows_per_dst=[300, 842, 3720, 1142, 0, 1000, 246, 142] combine_rows_per_src=[9800, 11870, 15680, 16384, 572, 15822, 14358, 6426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=62626 payload_rows_remote=46430 dup_rows=35678 dup_rows_remote=16636 dedup_ratio=0.362935 router_tokens_per_expert=[0, 8096, 4818, 0, 0, 54, 0, 233, 0, 0, 4, 0, 0, 1109, 1223, 145, 0, 91, 0, 663, 0, 3, 0, 1427, 140, 249, 0, 116, 0, 54, 231, 1, 168, 7891, 91, 1, 4055, 2, 6677, 1, 244, 1, 4, 0, 496, 2, 0, 2, 507, 1, 133, 0, 0, 0, 0, 3, 0, 1, 7160, 2473, 91, 99, 2, 390] payload_rows_per_dst=[16196, 12150, 8212, 1302, 16244, 1488, 1104, 5930] dup_rows_per_dst=[19042, 922, 1640, 280, 13418, 10, 184, 182] combine_rows_per_src=[16196, 12150, 8212, 1302, 16244, 1488, 1104, 5930] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83894 payload_rows_remote=68750 dup_rows=14410 dup_rows_remote=7324 dedup_ratio=0.146586 router_tokens_per_expert=[0, 8097, 4462, 0, 0, 42, 0, 177, 0, 0, 0, 0, 0, 1262, 879, 138, 0, 96, 0, 639, 0, 0, 0, 1393, 174, 253, 0, 72, 0, 42, 269, 0, 195, 7822, 96, 0, 4212, 0, 6453, 0, 228, 0, 0, 0, 417, 0, 0, 0, 504, 0, 115, 0, 0, 0, 0, 0, 0, 0, 7152, 3330, 94, 95, 0, 444] payload_rows_per_dst=[438, 7038, 3604, 16384, 11848, 15646, 13792, 15144] dup_rows_per_dst=[0, 268, 652, 1430, 3334, 1288, 352, 7086] combine_rows_per_src=[438, 7038, 3604, 16384, 11848, 15646, 13792, 15144] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78672 payload_rows_remote=66296 dup_rows=19632 dup_rows_remote=18776 dedup_ratio=0.199707 router_tokens_per_expert=[0, 8121, 4633, 0, 0, 60, 0, 194, 0, 0, 0, 0, 0, 1268, 993, 135, 0, 74, 0, 720, 0, 0, 0, 1529, 81, 239, 0, 97, 0, 60, 152, 0, 229, 7856, 74, 0, 4220, 0, 6561, 0, 164, 0, 0, 0, 405, 0, 0, 0, 553, 0, 117, 0, 0, 0, 0, 0, 0, 0, 7280, 2789, 71, 73, 0, 404] payload_rows_per_dst=[16242, 12376, 15768, 1096, 15992, 1138, 9528, 6532] dup_rows_per_dst=[9774, 856, 3438, 162, 5036, 0, 224, 142] combine_rows_per_src=[16242, 12376, 15768, 1096, 15992, 1138, 9528, 6532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87694 payload_rows_remote=72078 dup_rows=10610 dup_rows_remote=9598 dedup_ratio=0.107931 router_tokens_per_expert=[0, 8102, 4820, 0, 0, 66, 0, 154, 0, 0, 0, 0, 0, 1244, 1084, 156, 0, 90, 0, 723, 0, 0, 0, 1380, 161, 247, 0, 61, 0, 66, 251, 0, 246, 7808, 90, 0, 3831, 0, 6627, 0, 112, 0, 0, 0, 394, 0, 0, 0, 571, 0, 153, 0, 0, 0, 0, 0, 0, 0, 7179, 2981, 90, 90, 0, 375] payload_rows_per_dst=[440, 11888, 3610, 16384, 10218, 15616, 14350, 15188] dup_rows_per_dst=[0, 742, 776, 1392, 94, 1012, 352, 6242] combine_rows_per_src=[440, 11888, 3610, 16384, 10218, 15616, 14350, 15188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=305266688 dispatch_meta_bytes=323112 combine_payload_bytes=305266688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=305589800 ingress_bytes=339869696 link_bytes=339869696 token_bytes=610856488 weight_movement_bytes=34603008 total_nvlink_bytes=645459496 weight_to_token_ratio=0.056647 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=43776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.278902304e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=190177280 dispatch_meta_bytes=252264 combine_payload_bytes=190177280 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=190429544 ingress_bytes=293986304 link_bytes=293986304 token_bytes=380606824 weight_movement_bytes=103809024 total_nvlink_bytes=484415848 weight_to_token_ratio=0.272746 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=21504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.106246794e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=281600000 dispatch_meta_bytes=304296 combine_payload_bytes=281600000 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=281904296 ingress_bytes=350806016 link_bytes=350806016 token_bytes=563504296 weight_movement_bytes=69206016 total_nvlink_bytes=632710312 weight_to_token_ratio=0.122814 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=22400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.320054796e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=271548416 dispatch_meta_bytes=340288 combine_payload_bytes=271548416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271888704 ingress_bytes=306151424 link_bytes=306151424 token_bytes=543437120 weight_movement_bytes=34603008 total_nvlink_bytes=578040128 weight_to_token_ratio=0.063674 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=29824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.152023161e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=295231488 dispatch_meta_bytes=326704 combine_payload_bytes=295231488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=295558192 ingress_bytes=329834496 link_bytes=329834496 token_bytes=590789680 weight_movement_bytes=34603008 total_nvlink_bytes=625392688 weight_to_token_ratio=0.058571 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=44288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.241140655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87016 payload_rows_remote=71002 dup_rows=11288 dup_rows_remote=11192 dedup_ratio=0.114827 router_tokens_per_expert=[0, 0, 0, 250, 0, 0, 0, 0, 400, 60, 10, 0, 765, 63, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 1, 7942, 0, 1144, 60, 3429, 0, 0, 128, 4284, 0, 0, 0, 7313, 0, 439, 2918, 244, 60, 0, 373, 2126, 2874, 0, 0, 48, 0, 65, 0, 0, 0, 0, 0, 0, 6902, 6414, 685, 0, 0, 63] payload_rows_per_dst=[6896, 6592, 13988, 6246, 13146, 9626, 16014, 14508] dup_rows_per_dst=[196, 1752, 0, 274, 4590, 1816, 96, 2564] combine_rows_per_src=[6896, 6592, 13988, 6246, 13146, 9626, 16014, 14508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93818 payload_rows_remote=84332 dup_rows=4486 dup_rows_remote=4334 dedup_ratio=0.045634 router_tokens_per_expert=[0, 0, 0, 317, 0, 0, 0, 0, 366, 63, 3, 0, 1014, 73, 0, 29, 0, 0, 0, 0, 0, 69, 0, 0, 9, 7927, 0, 1109, 61, 3564, 0, 0, 150, 4214, 0, 0, 0, 7370, 0, 455, 2496, 262, 73, 0, 387, 2026, 3027, 0, 0, 64, 0, 75, 0, 0, 0, 0, 1, 0, 6957, 6323, 593, 0, 0, 75] payload_rows_per_dst=[14740, 11254, 14052, 9026, 9486, 5486, 16004, 13770] dup_rows_per_dst=[634, 2888, 0, 460, 152, 10, 128, 214] combine_rows_per_src=[14740, 11254, 14052, 9026, 9486, 5486, 16004, 13770] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86176 payload_rows_remote=70570 dup_rows=12128 dup_rows_remote=8926 dedup_ratio=0.123372 router_tokens_per_expert=[0, 8110, 4637, 0, 0, 90, 0, 156, 0, 0, 0, 0, 0, 1237, 1046, 172, 0, 82, 0, 713, 0, 0, 0, 1486, 72, 264, 0, 46, 0, 90, 154, 0, 222, 7827, 82, 0, 4246, 0, 6789, 0, 365, 0, 0, 0, 356, 0, 0, 0, 491, 0, 96, 0, 0, 0, 0, 0, 0, 0, 7123, 2652, 82, 83, 0, 383] payload_rows_per_dst=[13726, 12532, 15606, 7086, 6404, 10158, 14432, 6232] dup_rows_per_dst=[5978, 870, 3202, 448, 172, 970, 320, 168] combine_rows_per_src=[13726, 12532, 15606, 7086, 6404, 10158, 14432, 6232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87694 payload_rows_remote=72078 dup_rows=10610 dup_rows_remote=9598 dedup_ratio=0.107931 router_tokens_per_expert=[0, 8102, 4820, 0, 0, 66, 0, 154, 0, 0, 0, 0, 0, 1244, 1084, 156, 0, 90, 0, 723, 0, 0, 0, 1380, 161, 247, 0, 61, 0, 66, 251, 0, 246, 7808, 90, 0, 3831, 0, 6627, 0, 112, 0, 0, 0, 394, 0, 0, 0, 571, 0, 153, 0, 0, 0, 0, 0, 0, 0, 7179, 2981, 90, 90, 0, 375] payload_rows_per_dst=[440, 11888, 3610, 16384, 10218, 15616, 14350, 15188] dup_rows_per_dst=[0, 742, 776, 1392, 94, 1012, 352, 6242] combine_rows_per_src=[440, 11888, 3610, 16384, 10218, 15616, 14350, 15188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90812 payload_rows_remote=86072 dup_rows=7492 dup_rows_remote=7458 dedup_ratio=0.076213 router_tokens_per_expert=[0, 8117, 4708, 0, 0, 63, 0, 175, 0, 0, 0, 0, 0, 1189, 1231, 138, 0, 75, 0, 696, 0, 0, 0, 1491, 64, 246, 0, 74, 0, 63, 139, 0, 255, 7909, 75, 0, 3898, 0, 6730, 0, 366, 0, 0, 0, 447, 0, 0, 0, 537, 0, 87, 0, 0, 0, 0, 0, 0, 0, 7179, 2629, 75, 81, 0, 415] payload_rows_per_dst=[5706, 12036, 14826, 16384, 4740, 15888, 14396, 6836] dup_rows_per_dst=[72, 876, 2984, 1022, 34, 1556, 312, 636] combine_rows_per_src=[5706, 12036, 14826, 16384, 4740, 15888, 14396, 6836] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=271548416 dispatch_meta_bytes=340288 combine_payload_bytes=271548416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271888704 ingress_bytes=306151424 link_bytes=306151424 token_bytes=543437120 weight_movement_bytes=34603008 total_nvlink_bytes=578040128 weight_to_token_ratio=0.063674 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=29824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.152023161e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88420 payload_rows_remote=74264 dup_rows=9884 dup_rows_remote=9686 dedup_ratio=0.100545 router_tokens_per_expert=[0, 8129, 4849, 0, 0, 60, 0, 235, 0, 0, 0, 0, 0, 1233, 961, 125, 0, 65, 0, 713, 0, 0, 0, 1515, 48, 254, 0, 102, 0, 60, 111, 0, 185, 7869, 65, 0, 4200, 0, 6633, 0, 86, 0, 0, 0, 401, 0, 0, 0, 477, 0, 67, 0, 0, 0, 0, 0, 0, 4, 7416, 2771, 63, 73, 0, 382] payload_rows_per_dst=[590, 12262, 3854, 16384, 10110, 15738, 14156, 15326] dup_rows_per_dst=[0, 776, 732, 1024, 88, 974, 198, 6092] combine_rows_per_src=[590, 12262, 3854, 16384, 10110, 15738, 14156, 15326] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83894 payload_rows_remote=68750 dup_rows=14410 dup_rows_remote=7324 dedup_ratio=0.146586 router_tokens_per_expert=[0, 8097, 4462, 0, 0, 42, 0, 177, 0, 0, 0, 0, 0, 1262, 879, 138, 0, 96, 0, 639, 0, 0, 0, 1393, 174, 253, 0, 72, 0, 42, 269, 0, 195, 7822, 96, 0, 4212, 0, 6453, 0, 228, 0, 0, 0, 417, 0, 0, 0, 504, 0, 115, 0, 0, 0, 0, 0, 0, 0, 7152, 3330, 94, 95, 0, 444] payload_rows_per_dst=[438, 7038, 3604, 16384, 11848, 15646, 13792, 15144] dup_rows_per_dst=[0, 268, 652, 1430, 3334, 1288, 352, 7086] combine_rows_per_src=[438, 7038, 3604, 16384, 11848, 15646, 13792, 15144] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=62626 payload_rows_remote=46430 dup_rows=35678 dup_rows_remote=16636 dedup_ratio=0.362935 router_tokens_per_expert=[0, 8096, 4818, 0, 0, 54, 0, 233, 0, 0, 4, 0, 0, 1109, 1223, 145, 0, 91, 0, 663, 0, 3, 0, 1427, 140, 249, 0, 116, 0, 54, 231, 1, 168, 7891, 91, 1, 4055, 2, 6677, 1, 244, 1, 4, 0, 496, 2, 0, 2, 507, 1, 133, 0, 0, 0, 0, 3, 0, 1, 7160, 2473, 91, 99, 2, 390] payload_rows_per_dst=[16196, 12150, 8212, 1302, 16244, 1488, 1104, 5930] dup_rows_per_dst=[19042, 922, 1640, 280, 13418, 10, 184, 182] combine_rows_per_src=[16196, 12150, 8212, 1302, 16244, 1488, 1104, 5930] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=290824192 dispatch_meta_bytes=328776 combine_payload_bytes=290824192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291152968 ingress_bytes=325427200 link_bytes=325427200 token_bytes=581977160 weight_movement_bytes=34603008 total_nvlink_bytes=616580168 weight_to_token_ratio=0.059458 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=47744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.224556355e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=345423872 dispatch_meta_bytes=354664 combine_payload_bytes=345423872 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=345778536 ingress_bytes=414629888 link_bytes=414629888 token_bytes=691202408 weight_movement_bytes=69206016 total_nvlink_bytes=760408424 weight_to_token_ratio=0.100124 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=33280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.560218888e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=289054720 dispatch_meta_bytes=317984 combine_payload_bytes=289054720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289372704 ingress_bytes=323657728 link_bytes=323657728 token_bytes=578427424 weight_movement_bytes=34603008 total_nvlink_bytes=613030432 weight_to_token_ratio=0.059823 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=31104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.217897974e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=295231488 dispatch_meta_bytes=326704 combine_payload_bytes=295231488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=295558192 ingress_bytes=329834496 link_bytes=329834496 token_bytes=590789680 weight_movement_bytes=34603008 total_nvlink_bytes=625392688 weight_to_token_ratio=0.058571 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=44288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.241140655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=352550912 dispatch_meta_bytes=374120 combine_payload_bytes=352550912 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=352925032 ingress_bytes=490962944 link_bytes=490962944 token_bytes=705475944 weight_movement_bytes=138412032 total_nvlink_bytes=843887976 weight_to_token_ratio=0.196197 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=18816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.847454033e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93602 payload_rows_remote=84826 dup_rows=4702 dup_rows_remote=4402 dedup_ratio=0.047831 router_tokens_per_expert=[0, 0, 0, 294, 0, 0, 0, 0, 386, 54, 8, 0, 1119, 69, 0, 6, 0, 0, 0, 0, 0, 79, 0, 0, 3, 7932, 0, 965, 51, 3519, 0, 0, 138, 4077, 0, 0, 0, 7262, 0, 443, 2412, 274, 76, 0, 402, 2082, 3223, 0, 0, 78, 0, 70, 0, 0, 0, 0, 0, 0, 6989, 6412, 660, 0, 0, 69] payload_rows_per_dst=[14524, 11432, 14136, 8776, 9178, 5638, 16004, 13914] dup_rows_per_dst=[588, 3122, 0, 300, 138, 30, 156, 368] combine_rows_per_src=[14524, 11432, 14136, 8776, 9178, 5638, 16004, 13914] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=304185344 dispatch_meta_bytes=335800 combine_payload_bytes=304185344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=304521144 ingress_bytes=338788352 link_bytes=338788352 token_bytes=608706488 weight_movement_bytes=34603008 total_nvlink_bytes=643309496 weight_to_token_ratio=0.056847 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=44160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274833294e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=281600000 dispatch_meta_bytes=304296 combine_payload_bytes=281600000 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=281904296 ingress_bytes=350806016 link_bytes=350806016 token_bytes=563504296 weight_movement_bytes=69206016 total_nvlink_bytes=632710312 weight_to_token_ratio=0.122814 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=22400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.320054796e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=190177280 dispatch_meta_bytes=252264 combine_payload_bytes=190177280 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=190429544 ingress_bytes=293986304 link_bytes=293986304 token_bytes=380606824 weight_movement_bytes=103809024 total_nvlink_bytes=484415848 weight_to_token_ratio=0.272746 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=21504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.106246794e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=347447296 dispatch_meta_bytes=356912 combine_payload_bytes=347447296 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=347804208 ingress_bytes=451256320 link_bytes=451256320 token_bytes=695251504 weight_movement_bytes=103809024 total_nvlink_bytes=799060528 weight_to_token_ratio=0.149311 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=23168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.698041203e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81016 payload_rows_remote=66530 dup_rows=17288 dup_rows_remote=16694 dedup_ratio=0.175863 router_tokens_per_expert=[0, 0, 0, 297, 3, 0, 0, 4, 372, 55, 2, 3, 917, 90, 3, 70, 0, 0, 0, 0, 4, 69, 0, 0, 3, 7885, 0, 1016, 57, 3664, 2, 2, 182, 4156, 0, 0, 0, 7236, 1, 425, 2482, 339, 122, 0, 362, 2061, 3089, 2, 0, 140, 2, 91, 0, 3, 0, 0, 2, 3, 7012, 6256, 577, 0, 0, 91] payload_rows_per_dst=[14486, 11240, 14166, 16218, 8006, 15128, 470, 1302] dup_rows_per_dst=[594, 2926, 4, 10410, 152, 3156, 2, 44] combine_rows_per_src=[14486, 11240, 14166, 16218, 8006, 15128, 470, 1302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68810 payload_rows_remote=52838 dup_rows=29494 dup_rows_remote=12260 dedup_ratio=0.300028 router_tokens_per_expert=[0, 0, 0, 186, 0, 0, 0, 0, 252, 42, 3, 0, 994, 88, 0, 46, 0, 0, 0, 0, 0, 57, 0, 0, 34, 7911, 0, 1306, 40, 3109, 0, 0, 191, 4322, 0, 0, 0, 7258, 0, 368, 2336, 312, 42, 0, 326, 2763, 3236, 0, 0, 174, 0, 96, 0, 0, 0, 0, 5, 0, 6776, 6188, 596, 0, 0, 95] payload_rows_per_dst=[372, 7570, 446, 2760, 15776, 9900, 16014, 15972] dup_rows_per_dst=[0, 1752, 0, 0, 8502, 1658, 348, 17234] combine_rows_per_src=[372, 7570, 446, 2760, 15776, 9900, 16014, 15972] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82038 payload_rows_remote=71086 dup_rows=16266 dup_rows_remote=13576 dedup_ratio=0.165466 router_tokens_per_expert=[0, 0, 0, 254, 0, 0, 0, 0, 331, 60, 4, 0, 909, 67, 0, 14, 0, 0, 0, 0, 0, 76, 0, 0, 10, 7953, 0, 1056, 42, 3670, 0, 0, 145, 4097, 0, 0, 0, 7420, 0, 428, 2513, 280, 60, 0, 420, 2513, 2923, 0, 0, 81, 0, 74, 0, 0, 0, 0, 1, 0, 6892, 6172, 616, 0, 0, 71] payload_rows_per_dst=[14840, 10952, 13936, 16044, 9198, 15144, 670, 1254] dup_rows_per_dst=[508, 2690, 0, 9026, 142, 3746, 32, 122] combine_rows_per_src=[14840, 10952, 13936, 16044, 9198, 15144, 670, 1254] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91102 payload_rows_remote=77258 dup_rows=7202 dup_rows_remote=7202 dedup_ratio=0.073263 router_tokens_per_expert=[0, 0, 0, 257, 0, 0, 0, 0, 447, 90, 3, 0, 924, 73, 0, 33, 0, 0, 0, 0, 0, 113, 0, 0, 4, 7883, 0, 1154, 88, 3394, 0, 0, 164, 4180, 0, 0, 0, 7241, 0, 443, 2326, 319, 91, 0, 340, 2277, 3361, 0, 0, 72, 0, 82, 0, 0, 0, 0, 1, 0, 6809, 6243, 658, 0, 0, 82] payload_rows_per_dst=[14482, 11368, 13844, 9002, 9386, 13684, 15930, 3406] dup_rows_per_dst=[514, 3146, 0, 278, 188, 2802, 144, 130] combine_rows_per_src=[14482, 11368, 13844, 9002, 9386, 13684, 15930, 3406] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93818 payload_rows_remote=84332 dup_rows=4486 dup_rows_remote=4334 dedup_ratio=0.045634 router_tokens_per_expert=[0, 0, 0, 317, 0, 0, 0, 0, 366, 63, 3, 0, 1014, 73, 0, 29, 0, 0, 0, 0, 0, 69, 0, 0, 9, 7927, 0, 1109, 61, 3564, 0, 0, 150, 4214, 0, 0, 0, 7370, 0, 455, 2496, 262, 73, 0, 387, 2026, 3027, 0, 0, 64, 0, 75, 0, 0, 0, 0, 1, 0, 6957, 6323, 593, 0, 0, 75] payload_rows_per_dst=[14740, 11254, 14052, 9026, 9486, 5486, 16004, 13770] dup_rows_per_dst=[634, 2888, 0, 460, 152, 10, 128, 214] combine_rows_per_src=[14740, 11254, 14052, 9026, 9486, 5486, 16004, 13770] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93618 payload_rows_remote=84414 dup_rows=4686 dup_rows_remote=3410 dedup_ratio=0.047668 router_tokens_per_expert=[0, 0, 0, 192, 0, 0, 0, 0, 323, 66, 3, 0, 823, 71, 0, 23, 0, 0, 0, 0, 0, 87, 0, 0, 64, 7895, 0, 1394, 52, 3232, 0, 0, 180, 4593, 0, 0, 0, 7273, 0, 359, 2473, 355, 66, 0, 408, 2354, 3085, 0, 0, 161, 0, 90, 0, 0, 0, 0, 19, 0, 6779, 6041, 601, 0, 0, 90] payload_rows_per_dst=[14546, 7808, 13732, 9044, 10068, 9204, 15970, 13246] dup_rows_per_dst=[384, 1812, 0, 440, 196, 1276, 322, 256] combine_rows_per_src=[14546, 7808, 13732, 9044, 10068, 9204, 15970, 13246] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=347447296 dispatch_meta_bytes=356912 combine_payload_bytes=347447296 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=347804208 ingress_bytes=451256320 link_bytes=451256320 token_bytes=695251504 weight_movement_bytes=103809024 total_nvlink_bytes=799060528 weight_to_token_ratio=0.149311 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=23168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.698041203e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=272506880 dispatch_meta_bytes=332896 combine_payload_bytes=272506880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272839776 ingress_bytes=307109888 link_bytes=307109888 token_bytes=545346656 weight_movement_bytes=34603008 total_nvlink_bytes=579949664 weight_to_token_ratio=0.063451 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=47104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.155629784e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=216424448 dispatch_meta_bytes=260392 combine_payload_bytes=216424448 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=216684840 ingress_bytes=320233472 link_bytes=320233472 token_bytes=433109288 weight_movement_bytes=103809024 total_nvlink_bytes=536918312 weight_to_token_ratio=0.239683 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.205012774e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=291168256 dispatch_meta_bytes=338648 combine_payload_bytes=291168256 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=291506904 ingress_bytes=360374272 link_bytes=360374272 token_bytes=582675160 weight_movement_bytes=69206016 total_nvlink_bytes=651881176 weight_to_token_ratio=0.118773 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=24832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.356059373e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87644 payload_rows_remote=73868 dup_rows=10660 dup_rows_remote=8812 dedup_ratio=0.108439 router_tokens_per_expert=[0, 0, 254, 92, 0, 7503, 0, 399, 0, 0, 2, 0, 65, 0, 0, 0, 1653, 0, 0, 0, 0, 3256, 0, 5117, 0, 0, 34, 7938, 0, 0, 0, 26, 891, 0, 0, 204, 0, 0, 155, 314, 215, 0, 0, 2885, 0, 0, 0, 0, 6790, 0, 1568, 0, 6244, 0, 0, 0, 0, 49, 0, 495, 728, 2210, 0, 65] payload_rows_per_dst=[1490, 13710, 8818, 15876, 11792, 6200, 13776, 15982] dup_rows_per_dst=[0, 4, 1000, 120, 1570, 0, 1848, 6118] combine_rows_per_src=[1490, 13710, 8818, 15876, 11792, 6200, 13776, 15982] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=345759744 dispatch_meta_bytes=351296 combine_payload_bytes=345759744 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=346111040 ingress_bytes=449568768 link_bytes=449568768 token_bytes=691870784 weight_movement_bytes=103809024 total_nvlink_bytes=795679808 weight_to_token_ratio=0.150041 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=18304 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.691691081e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92454 payload_rows_remote=81182 dup_rows=5850 dup_rows_remote=4338 dedup_ratio=0.059509 router_tokens_per_expert=[0, 0, 246, 70, 0, 7471, 0, 419, 0, 0, 4, 0, 75, 0, 0, 0, 1826, 0, 0, 0, 0, 3175, 0, 4711, 0, 0, 35, 7946, 0, 0, 0, 1, 933, 0, 0, 208, 0, 0, 144, 396, 207, 0, 0, 3077, 0, 0, 0, 0, 6758, 0, 1513, 0, 6110, 0, 0, 0, 0, 64, 0, 533, 781, 2374, 0, 75] payload_rows_per_dst=[15726, 13666, 8896, 15892, 11272, 6568, 13550, 6884] dup_rows_per_dst=[686, 8, 1106, 72, 1512, 0, 1696, 770] combine_rows_per_src=[15726, 13666, 8896, 15892, 11272, 6568, 13550, 6884] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91102 payload_rows_remote=77258 dup_rows=7202 dup_rows_remote=7202 dedup_ratio=0.073263 router_tokens_per_expert=[0, 0, 0, 257, 0, 0, 0, 0, 447, 90, 3, 0, 924, 73, 0, 33, 0, 0, 0, 0, 0, 113, 0, 0, 4, 7883, 0, 1154, 88, 3394, 0, 0, 164, 4180, 0, 0, 0, 7241, 0, 443, 2326, 319, 91, 0, 340, 2277, 3361, 0, 0, 72, 0, 82, 0, 0, 0, 0, 1, 0, 6809, 6243, 658, 0, 0, 82] payload_rows_per_dst=[14482, 11368, 13844, 9002, 9386, 13684, 15930, 3406] dup_rows_per_dst=[514, 3146, 0, 278, 188, 2802, 144, 130] combine_rows_per_src=[14482, 11368, 13844, 9002, 9386, 13684, 15930, 3406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87016 payload_rows_remote=71002 dup_rows=11288 dup_rows_remote=11192 dedup_ratio=0.114827 router_tokens_per_expert=[0, 0, 0, 250, 0, 0, 0, 0, 400, 60, 10, 0, 765, 63, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 1, 7942, 0, 1144, 60, 3429, 0, 0, 128, 4284, 0, 0, 0, 7313, 0, 439, 2918, 244, 60, 0, 373, 2126, 2874, 0, 0, 48, 0, 65, 0, 0, 0, 0, 0, 0, 6902, 6414, 685, 0, 0, 63] payload_rows_per_dst=[6896, 6592, 13988, 6246, 13146, 9626, 16014, 14508] dup_rows_per_dst=[196, 1752, 0, 274, 4590, 1816, 96, 2564] combine_rows_per_src=[6896, 6592, 13988, 6246, 13146, 9626, 16014, 14508] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68810 payload_rows_remote=52838 dup_rows=29494 dup_rows_remote=12260 dedup_ratio=0.300028 router_tokens_per_expert=[0, 0, 0, 186, 0, 0, 0, 0, 252, 42, 3, 0, 994, 88, 0, 46, 0, 0, 0, 0, 0, 57, 0, 0, 34, 7911, 0, 1306, 40, 3109, 0, 0, 191, 4322, 0, 0, 0, 7258, 0, 368, 2336, 312, 42, 0, 326, 2763, 3236, 0, 0, 174, 0, 96, 0, 0, 0, 0, 5, 0, 6776, 6188, 596, 0, 0, 95] payload_rows_per_dst=[372, 7570, 446, 2760, 15776, 9900, 16014, 15972] dup_rows_per_dst=[0, 1752, 0, 0, 8502, 1658, 348, 17234] combine_rows_per_src=[372, 7570, 446, 2760, 15776, 9900, 16014, 15972] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81016 payload_rows_remote=66530 dup_rows=17288 dup_rows_remote=16694 dedup_ratio=0.175863 router_tokens_per_expert=[0, 0, 0, 297, 3, 0, 0, 4, 372, 55, 2, 3, 917, 90, 3, 70, 0, 0, 0, 0, 4, 69, 0, 0, 3, 7885, 0, 1016, 57, 3664, 2, 2, 182, 4156, 0, 0, 0, 7236, 1, 425, 2482, 339, 122, 0, 362, 2061, 3089, 2, 0, 140, 2, 91, 0, 3, 0, 0, 2, 3, 7012, 6256, 577, 0, 0, 91] payload_rows_per_dst=[14486, 11240, 14166, 16218, 8006, 15128, 470, 1302] dup_rows_per_dst=[594, 2926, 4, 10410, 152, 3156, 2, 44] combine_rows_per_src=[14486, 11240, 14166, 16218, 8006, 15128, 470, 1302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=302563328 dispatch_meta_bytes=330720 combine_payload_bytes=302563328 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=302894048 ingress_bytes=406372352 link_bytes=406372352 token_bytes=605457376 weight_movement_bytes=103809024 total_nvlink_bytes=709266400 weight_to_token_ratio=0.171456 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=25984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.529146445e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=332521472 dispatch_meta_bytes=342080 combine_payload_bytes=332521472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=332863552 ingress_bytes=367124480 link_bytes=367124480 token_bytes=665385024 weight_movement_bytes=34603008 total_nvlink_bytes=699988032 weight_to_token_ratio=0.052004 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=36096 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.381459862e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=316448768 dispatch_meta_bytes=337840 combine_payload_bytes=316448768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=316786608 ingress_bytes=351051776 link_bytes=351051776 token_bytes=633235376 weight_movement_bytes=34603008 total_nvlink_bytes=667838384 weight_to_token_ratio=0.054645 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=48512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.320979571e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=345423872 dispatch_meta_bytes=354664 combine_payload_bytes=345423872 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=345778536 ingress_bytes=414629888 link_bytes=414629888 token_bytes=691202408 weight_movement_bytes=69206016 total_nvlink_bytes=760408424 weight_to_token_ratio=0.100124 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=33280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.560218888e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=345759744 dispatch_meta_bytes=351296 combine_payload_bytes=345759744 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=346111040 ingress_bytes=449568768 link_bytes=449568768 token_bytes=691870784 weight_movement_bytes=103809024 total_nvlink_bytes=795679808 weight_to_token_ratio=0.150041 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=18304 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.691691081e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90508 payload_rows_remote=74646 dup_rows=7796 dup_rows_remote=7730 dedup_ratio=0.079305 router_tokens_per_expert=[0, 0, 261, 79, 0, 7497, 0, 414, 0, 0, 6, 0, 69, 0, 0, 0, 1730, 0, 0, 0, 0, 3481, 0, 4627, 0, 0, 30, 7931, 0, 0, 0, 3, 972, 0, 0, 226, 0, 0, 148, 532, 201, 0, 0, 2727, 0, 0, 0, 0, 6687, 0, 1656, 0, 6163, 1, 0, 0, 0, 78, 1, 512, 772, 2279, 0, 69] payload_rows_per_dst=[15812, 13512, 9152, 15862, 11404, 12476, 5684, 6606] dup_rows_per_dst=[690, 12, 1270, 66, 1606, 2864, 472, 816] combine_rows_per_src=[15812, 13512, 9152, 15862, 11404, 12476, 5684, 6606] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=290824192 dispatch_meta_bytes=328776 combine_payload_bytes=290824192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291152968 ingress_bytes=325427200 link_bytes=325427200 token_bytes=581977160 weight_movement_bytes=34603008 total_nvlink_bytes=616580168 weight_to_token_ratio=0.059458 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=47744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.224556355e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=216424448 dispatch_meta_bytes=260392 combine_payload_bytes=216424448 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=216684840 ingress_bytes=320233472 link_bytes=320233472 token_bytes=433109288 weight_movement_bytes=103809024 total_nvlink_bytes=536918312 weight_to_token_ratio=0.239683 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.205012774e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=272506880 dispatch_meta_bytes=332896 combine_payload_bytes=272506880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272839776 ingress_bytes=307109888 link_bytes=307109888 token_bytes=545346656 weight_movement_bytes=34603008 total_nvlink_bytes=579949664 weight_to_token_ratio=0.063451 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=47104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.155629784e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76046 payload_rows_remote=60116 dup_rows=22258 dup_rows_remote=14520 dedup_ratio=0.226420 router_tokens_per_expert=[1, 0, 249, 71, 0, 7364, 2, 434, 0, 0, 7, 0, 92, 4, 0, 0, 1715, 0, 0, 0, 0, 3038, 2, 4605, 0, 0, 22, 7936, 0, 2, 0, 10, 871, 9, 0, 300, 0, 2, 160, 498, 214, 3, 7, 3034, 0, 0, 4, 0, 6674, 0, 1623, 0, 6198, 0, 0, 1, 0, 141, 0, 538, 764, 2463, 1, 93] payload_rows_per_dst=[15930, 13538, 15196, 66, 6508, 14520, 3248, 7040] dup_rows_per_dst=[7738, 16, 7694, 2, 1448, 4400, 0, 960] combine_rows_per_src=[15930, 13538, 15196, 66, 6508, 14520, 3248, 7040] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68898 payload_rows_remote=52788 dup_rows=29406 dup_rows_remote=23938 dedup_ratio=0.299133 router_tokens_per_expert=[0, 0, 254, 58, 0, 7378, 0, 432, 0, 0, 0, 0, 96, 0, 0, 0, 1862, 0, 0, 0, 0, 3693, 0, 5067, 0, 0, 33, 7938, 0, 0, 0, 0, 783, 0, 3, 327, 0, 0, 152, 167, 194, 0, 0, 3185, 0, 0, 0, 0, 6673, 0, 1351, 0, 6095, 0, 0, 0, 0, 174, 0, 601, 761, 1779, 0, 96] payload_rows_per_dst=[1486, 3288, 9818, 4388, 11432, 6758, 15618, 16110] dup_rows_per_dst=[2, 0, 1292, 18, 1566, 0, 21060, 5468] combine_rows_per_src=[1486, 3288, 9818, 4388, 11432, 6758, 15618, 16110] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88550 payload_rows_remote=75004 dup_rows=9754 dup_rows_remote=9752 dedup_ratio=0.099223 router_tokens_per_expert=[0, 0, 239, 76, 0, 7497, 0, 419, 0, 0, 1, 0, 74, 0, 0, 0, 1848, 0, 0, 0, 0, 3509, 0, 4967, 0, 0, 33, 7953, 0, 0, 0, 6, 865, 0, 0, 230, 0, 0, 147, 271, 207, 0, 0, 3027, 0, 0, 0, 0, 6699, 0, 1596, 0, 6153, 0, 0, 0, 0, 82, 0, 512, 765, 1902, 0, 74] payload_rows_per_dst=[15766, 13546, 10364, 13962, 11310, 14528, 3192, 5882] dup_rows_per_dst=[696, 2, 2324, 48, 1650, 4246, 0, 788] combine_rows_per_src=[15766, 13546, 10364, 13962, 11310, 14528, 3192, 5882] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=305750016 dispatch_meta_bytes=329504 combine_payload_bytes=305750016 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=306079520 ingress_bytes=409559040 link_bytes=409559040 token_bytes=611829536 weight_movement_bytes=103809024 total_nvlink_bytes=715638560 weight_to_token_ratio=0.169670 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=48896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.541137695e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89116 payload_rows_remote=79388 dup_rows=9188 dup_rows_remote=7680 dedup_ratio=0.093465 router_tokens_per_expert=[0, 0, 264, 113, 0, 7409, 0, 443, 0, 0, 0, 0, 82, 0, 0, 0, 2037, 0, 0, 0, 0, 3581, 0, 4605, 0, 0, 46, 7928, 0, 0, 0, 9, 839, 0, 0, 267, 0, 0, 195, 331, 284, 0, 0, 3096, 0, 0, 0, 0, 6665, 0, 1607, 0, 6048, 0, 0, 0, 0, 72, 0, 550, 842, 1757, 0, 82] payload_rows_per_dst=[15736, 13494, 9728, 15856, 10944, 14374, 3214, 5770] dup_rows_per_dst=[722, 0, 1508, 110, 1530, 4482, 0, 836] combine_rows_per_src=[15736, 13494, 9728, 15856, 10944, 14374, 3214, 5770] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92454 payload_rows_remote=81182 dup_rows=5850 dup_rows_remote=4338 dedup_ratio=0.059509 router_tokens_per_expert=[0, 0, 246, 70, 0, 7471, 0, 419, 0, 0, 4, 0, 75, 0, 0, 0, 1826, 0, 0, 0, 0, 3175, 0, 4711, 0, 0, 35, 7946, 0, 0, 0, 1, 933, 0, 0, 208, 0, 0, 144, 396, 207, 0, 0, 3077, 0, 0, 0, 0, 6758, 0, 1513, 0, 6110, 0, 0, 0, 0, 64, 0, 533, 781, 2374, 0, 75] payload_rows_per_dst=[15726, 13666, 8896, 15892, 11272, 6568, 13550, 6884] dup_rows_per_dst=[686, 8, 1106, 72, 1512, 0, 1696, 770] combine_rows_per_src=[15726, 13666, 8896, 15892, 11272, 6568, 13550, 6884] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88630 payload_rows_remote=81742 dup_rows=9674 dup_rows_remote=9674 dedup_ratio=0.098409 router_tokens_per_expert=[0, 0, 247, 87, 0, 7396, 0, 393, 0, 0, 2, 0, 90, 0, 0, 0, 1701, 0, 0, 0, 0, 3368, 0, 4862, 0, 0, 37, 7945, 0, 0, 0, 1, 852, 0, 4, 337, 0, 0, 177, 148, 243, 0, 0, 3201, 0, 0, 0, 0, 6662, 0, 1465, 0, 6008, 0, 0, 0, 0, 161, 0, 551, 718, 2406, 0, 90] payload_rows_per_dst=[5620, 13504, 9014, 15890, 10994, 6888, 13264, 13456] dup_rows_per_dst=[204, 4, 1124, 76, 1766, 0, 1682, 4818] combine_rows_per_src=[5620, 13504, 9014, 15890, 10994, 6888, 13264, 13456] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=307216384 dispatch_meta_bytes=339024 combine_payload_bytes=307216384 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307555408 ingress_bytes=341819392 link_bytes=341819392 token_bytes=614771792 weight_movement_bytes=34603008 total_nvlink_bytes=649374800 weight_to_token_ratio=0.056286 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=48512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.286238853e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87644 payload_rows_remote=73868 dup_rows=10660 dup_rows_remote=8812 dedup_ratio=0.108439 router_tokens_per_expert=[0, 0, 254, 92, 0, 7503, 0, 399, 0, 0, 2, 0, 65, 0, 0, 0, 1653, 0, 0, 0, 0, 3256, 0, 5117, 0, 0, 34, 7938, 0, 0, 0, 26, 891, 0, 0, 204, 0, 0, 155, 314, 215, 0, 0, 2885, 0, 0, 0, 0, 6790, 0, 1568, 0, 6244, 0, 0, 0, 0, 49, 0, 495, 728, 2210, 0, 65] payload_rows_per_dst=[1490, 13710, 8818, 15876, 11792, 6200, 13776, 15982] dup_rows_per_dst=[0, 4, 1000, 120, 1570, 0, 1848, 6118] combine_rows_per_src=[1490, 13710, 8818, 15876, 11792, 6200, 13776, 15982] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68898 payload_rows_remote=52788 dup_rows=29406 dup_rows_remote=23938 dedup_ratio=0.299133 router_tokens_per_expert=[0, 0, 254, 58, 0, 7378, 0, 432, 0, 0, 0, 0, 96, 0, 0, 0, 1862, 0, 0, 0, 0, 3693, 0, 5067, 0, 0, 33, 7938, 0, 0, 0, 0, 783, 0, 3, 327, 0, 0, 152, 167, 194, 0, 0, 3185, 0, 0, 0, 0, 6673, 0, 1351, 0, 6095, 0, 0, 0, 0, 174, 0, 601, 761, 1779, 0, 96] payload_rows_per_dst=[1486, 3288, 9818, 4388, 11432, 6758, 15618, 16110] dup_rows_per_dst=[2, 0, 1292, 18, 1566, 0, 21060, 5468] combine_rows_per_src=[1486, 3288, 9818, 4388, 11432, 6758, 15618, 16110] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76046 payload_rows_remote=60116 dup_rows=22258 dup_rows_remote=14520 dedup_ratio=0.226420 router_tokens_per_expert=[1, 0, 249, 71, 0, 7364, 2, 434, 0, 0, 7, 0, 92, 4, 0, 0, 1715, 0, 0, 0, 0, 3038, 2, 4605, 0, 0, 22, 7936, 0, 2, 0, 10, 871, 9, 0, 300, 0, 2, 160, 498, 214, 3, 7, 3034, 0, 0, 4, 0, 6674, 0, 1623, 0, 6198, 0, 0, 1, 0, 141, 0, 538, 764, 2463, 1, 93] payload_rows_per_dst=[15930, 13538, 15196, 66, 6508, 14520, 3248, 7040] dup_rows_per_dst=[7738, 16, 7694, 2, 1448, 4400, 0, 960] combine_rows_per_src=[15930, 13538, 15196, 66, 6508, 14520, 3248, 7040] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=332521472 dispatch_meta_bytes=342080 combine_payload_bytes=332521472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=332863552 ingress_bytes=367124480 link_bytes=367124480 token_bytes=665385024 weight_movement_bytes=34603008 total_nvlink_bytes=699988032 weight_to_token_ratio=0.052004 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=36096 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.381459862e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=325173248 dispatch_meta_bytes=348272 combine_payload_bytes=325173248 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=325521520 ingress_bytes=394379264 link_bytes=394379264 token_bytes=650694768 weight_movement_bytes=69206016 total_nvlink_bytes=719900784 weight_to_token_ratio=0.106357 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=27136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.484017420e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=334815232 dispatch_meta_bytes=365664 combine_payload_bytes=334815232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=335180896 ingress_bytes=369418240 link_bytes=369418240 token_bytes=669996128 weight_movement_bytes=34603008 total_nvlink_bytes=704599136 weight_to_token_ratio=0.051647 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=24320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390091097e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=302563328 dispatch_meta_bytes=330720 combine_payload_bytes=302563328 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=302894048 ingress_bytes=406372352 link_bytes=406372352 token_bytes=605457376 weight_movement_bytes=103809024 total_nvlink_bytes=709266400 weight_to_token_ratio=0.171456 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=25984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.529146445e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94200 payload_rows_remote=80874 dup_rows=4104 dup_rows_remote=3900 dedup_ratio=0.041748 router_tokens_per_expert=[5323, 0, 0, 0, 53, 0, 0, 0, 0, 495, 0, 0, 38, 0, 591, 5964, 66, 7329, 0, 69, 0, 0, 66, 0, 7504, 0, 1, 261, 0, 6434, 69, 6654, 3909, 122, 0, 0, 0, 0, 0, 0, 69, 0, 426, 0, 0, 2513, 261, 0, 0, 0, 0, 334, 0, 0, 0, 0, 0, 112, 489, 0, 0, 0, 0, 0] payload_rows_per_dst=[10752, 12472, 14658, 13326, 7818, 5618, 15316, 14240] dup_rows_per_dst=[0, 1704, 402, 204, 244, 920, 360, 270] combine_rows_per_src=[10752, 12472, 14658, 13326, 7818, 5618, 15316, 14240] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=216219648 dispatch_meta_bytes=306904 combine_payload_bytes=216219648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=216526552 ingress_bytes=250822656 link_bytes=250822656 token_bytes=432746200 weight_movement_bytes=34603008 total_nvlink_bytes=467349208 weight_to_token_ratio=0.079961 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=20096 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.438254616e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=246235136 dispatch_meta_bytes=298544 combine_payload_bytes=246235136 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=246533680 ingress_bytes=315441152 link_bytes=315441152 token_bytes=492768816 weight_movement_bytes=69206016 total_nvlink_bytes=561974832 weight_to_token_ratio=0.140443 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=39424 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.186979660e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=331259904 dispatch_meta_bytes=339096 combine_payload_bytes=331259904 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=331599000 ingress_bytes=504274944 link_bytes=504274944 token_bytes=662858904 weight_movement_bytes=173015040 total_nvlink_bytes=835873944 weight_to_token_ratio=0.261013 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.897546017e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79424 payload_rows_remote=66114 dup_rows=18880 dup_rows_remote=17212 dedup_ratio=0.192057 router_tokens_per_expert=[6074, 0, 0, 0, 60, 0, 0, 0, 0, 428, 0, 0, 2, 0, 626, 6433, 92, 7320, 0, 74, 0, 0, 92, 0, 7466, 0, 2, 239, 0, 6303, 71, 6584, 2448, 131, 0, 0, 0, 0, 0, 0, 72, 0, 398, 0, 0, 2976, 239, 0, 0, 0, 0, 370, 0, 0, 0, 0, 0, 121, 531, 0, 0, 0, 0, 0] payload_rows_per_dst=[12268, 13310, 14640, 15264, 4896, 6454, 3062, 9530] dup_rows_per_dst=[0, 1668, 516, 15286, 262, 916, 74, 158] combine_rows_per_src=[12268, 13310, 14640, 15264, 4896, 6454, 3062, 9530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86774 payload_rows_remote=71458 dup_rows=11530 dup_rows_remote=11032 dedup_ratio=0.117289 router_tokens_per_expert=[5564, 0, 0, 0, 60, 0, 0, 0, 0, 408, 0, 0, 4, 0, 561, 6185, 75, 7391, 0, 65, 0, 0, 75, 0, 7539, 0, 2, 254, 0, 6472, 63, 6736, 3188, 123, 0, 0, 0, 0, 0, 0, 65, 0, 389, 0, 0, 2753, 254, 0, 0, 0, 0, 368, 0, 0, 0, 0, 0, 105, 453, 0, 0, 0, 0, 0] payload_rows_per_dst=[11248, 6838, 6968, 638, 15146, 16286, 15316, 14334] dup_rows_per_dst=[0, 708, 232, 0, 4420, 5418, 498, 254] combine_rows_per_src=[11248, 6838, 6968, 638, 15146, 16286, 15316, 14334] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=305750016 dispatch_meta_bytes=329504 combine_payload_bytes=305750016 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=306079520 ingress_bytes=409559040 link_bytes=409559040 token_bytes=611829536 weight_movement_bytes=103809024 total_nvlink_bytes=715638560 weight_to_token_ratio=0.169670 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=48896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.541137695e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89116 payload_rows_remote=79388 dup_rows=9188 dup_rows_remote=7680 dedup_ratio=0.093465 router_tokens_per_expert=[0, 0, 264, 113, 0, 7409, 0, 443, 0, 0, 0, 0, 82, 0, 0, 0, 2037, 0, 0, 0, 0, 3581, 0, 4605, 0, 0, 46, 7928, 0, 0, 0, 9, 839, 0, 0, 267, 0, 0, 195, 331, 284, 0, 0, 3096, 0, 0, 0, 0, 6665, 0, 1607, 0, 6048, 0, 0, 0, 0, 72, 0, 550, 842, 1757, 0, 82] payload_rows_per_dst=[15736, 13494, 9728, 15856, 10944, 14374, 3214, 5770] dup_rows_per_dst=[722, 0, 1508, 110, 1530, 4482, 0, 836] combine_rows_per_src=[15736, 13494, 9728, 15856, 10944, 14374, 3214, 5770] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91514 payload_rows_remote=77688 dup_rows=6790 dup_rows_remote=3624 dedup_ratio=0.069071 router_tokens_per_expert=[5625, 0, 0, 0, 63, 0, 0, 0, 0, 412, 0, 0, 17, 0, 599, 6421, 69, 7357, 0, 75, 0, 0, 69, 0, 7510, 0, 1, 246, 0, 6464, 75, 6746, 2809, 138, 0, 0, 0, 0, 0, 0, 75, 0, 407, 0, 0, 2811, 246, 0, 0, 0, 0, 321, 0, 0, 0, 0, 0, 114, 482, 0, 0, 0, 0, 0] payload_rows_per_dst=[11376, 13332, 14714, 2438, 13826, 6134, 15282, 14412] dup_rows_per_dst=[0, 1566, 426, 36, 3166, 944, 380, 272] combine_rows_per_src=[11376, 13332, 14714, 2438, 13826, 6134, 15282, 14412] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94156 payload_rows_remote=79468 dup_rows=4148 dup_rows_remote=3808 dedup_ratio=0.042196 router_tokens_per_expert=[5995, 0, 0, 0, 90, 0, 0, 0, 0, 435, 0, 0, 0, 0, 526, 6258, 44, 7344, 0, 82, 0, 0, 44, 0, 7460, 0, 9, 264, 0, 6353, 82, 6768, 2814, 172, 0, 0, 0, 0, 0, 0, 82, 0, 452, 0, 0, 2671, 264, 0, 0, 0, 0, 383, 0, 0, 0, 0, 0, 116, 444, 0, 0, 0, 0, 0] payload_rows_per_dst=[12170, 12926, 14688, 13218, 5628, 5930, 15230, 14366] dup_rows_per_dst=[0, 1512, 340, 198, 344, 1008, 456, 290] combine_rows_per_src=[12170, 12926, 14688, 13218, 5628, 5930, 15230, 14366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87870 payload_rows_remote=73066 dup_rows=10434 dup_rows_remote=5134 dedup_ratio=0.106140 router_tokens_per_expert=[6104, 0, 0, 0, 66, 0, 0, 0, 0, 379, 0, 0, 0, 0, 606, 6560, 60, 7236, 0, 90, 0, 0, 60, 0, 7427, 0, 8, 247, 0, 6246, 90, 6754, 2230, 156, 0, 0, 0, 0, 0, 0, 89, 0, 519, 0, 0, 2890, 247, 0, 0, 0, 0, 489, 0, 0, 0, 0, 0, 29, 570, 0, 0, 0, 0, 0] payload_rows_per_dst=[12340, 13590, 2114, 690, 14362, 14804, 15348, 14622] dup_rows_per_dst=[0, 1500, 164, 0, 2902, 5300, 484, 84] combine_rows_per_src=[12340, 13590, 2114, 690, 14362, 14804, 15348, 14622] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=216219648 dispatch_meta_bytes=306904 combine_payload_bytes=216219648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=216526552 ingress_bytes=250822656 link_bytes=250822656 token_bytes=432746200 weight_movement_bytes=34603008 total_nvlink_bytes=467349208 weight_to_token_ratio=0.079961 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=20096 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.438254616e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=307216384 dispatch_meta_bytes=339024 combine_payload_bytes=307216384 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307555408 ingress_bytes=341819392 link_bytes=341819392 token_bytes=614771792 weight_movement_bytes=34603008 total_nvlink_bytes=649374800 weight_to_token_ratio=0.056286 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=48512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.286238853e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=334815232 dispatch_meta_bytes=365664 combine_payload_bytes=334815232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=335180896 ingress_bytes=369418240 link_bytes=369418240 token_bytes=669996128 weight_movement_bytes=34603008 total_nvlink_bytes=704599136 weight_to_token_ratio=0.051647 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=24320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390091097e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86774 payload_rows_remote=71458 dup_rows=11530 dup_rows_remote=11032 dedup_ratio=0.117289 router_tokens_per_expert=[5564, 0, 0, 0, 60, 0, 0, 0, 0, 408, 0, 0, 4, 0, 561, 6185, 75, 7391, 0, 65, 0, 0, 75, 0, 7539, 0, 2, 254, 0, 6472, 63, 6736, 3188, 123, 0, 0, 0, 0, 0, 0, 65, 0, 389, 0, 0, 2753, 254, 0, 0, 0, 0, 368, 0, 0, 0, 0, 0, 105, 453, 0, 0, 0, 0, 0] payload_rows_per_dst=[11248, 6838, 6968, 638, 15146, 16286, 15316, 14334] dup_rows_per_dst=[0, 708, 232, 0, 4420, 5418, 498, 254] combine_rows_per_src=[11248, 6838, 6968, 638, 15146, 16286, 15316, 14334] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=270802944 dispatch_meta_bytes=333304 combine_payload_bytes=270802944 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=271136248 ingress_bytes=340008960 link_bytes=340008960 token_bytes=541939192 weight_movement_bytes=69206016 total_nvlink_bytes=611145208 weight_to_token_ratio=0.127701 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=40704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.279426344e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88020 payload_rows_remote=74106 dup_rows=10284 dup_rows_remote=10050 dedup_ratio=0.104614 router_tokens_per_expert=[6223, 0, 0, 0, 42, 0, 0, 0, 0, 416, 0, 0, 2, 0, 486, 6457, 72, 7301, 0, 96, 0, 0, 72, 0, 7419, 0, 8, 254, 0, 6399, 95, 6425, 2515, 137, 0, 0, 0, 0, 0, 0, 94, 0, 457, 0, 0, 2839, 254, 0, 0, 0, 0, 440, 0, 0, 0, 0, 0, 86, 563, 0, 0, 0, 0, 0] payload_rows_per_dst=[12530, 1808, 13106, 714, 14620, 16034, 15294, 13914] dup_rows_per_dst=[0, 0, 288, 0, 3482, 5856, 424, 234] combine_rows_per_src=[12530, 1808, 13106, 714, 14620, 16034, 15294, 13914] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71472 payload_rows_remote=58512 dup_rows=26832 dup_rows_remote=24172 dedup_ratio=0.272949 router_tokens_per_expert=[5617, 0, 0, 0, 54, 0, 0, 0, 1, 445, 0, 0, 8, 0, 542, 6226, 70, 7291, 1, 91, 0, 0, 71, 0, 7419, 5, 1, 249, 3, 6260, 92, 6719, 3447, 145, 1, 0, 1, 0, 0, 0, 91, 0, 371, 5, 0, 2620, 249, 5, 0, 0, 0, 389, 1, 0, 0, 0, 4, 129, 523, 0, 2, 4, 0, 0] payload_rows_per_dst=[12960, 13042, 14590, 16044, 6898, 5840, 780, 1318] dup_rows_per_dst=[2660, 2328, 458, 20248, 290, 842, 0, 6] combine_rows_per_src=[12960, 13042, 14590, 16044, 6898, 5840, 780, 1318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=292691968 dispatch_meta_bytes=329960 combine_payload_bytes=292691968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293021928 ingress_bytes=327294976 link_bytes=327294976 token_bytes=585713896 weight_movement_bytes=34603008 total_nvlink_bytes=620316904 weight_to_token_ratio=0.059078 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=46080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.231584645e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=325173248 dispatch_meta_bytes=348272 combine_payload_bytes=325173248 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=325521520 ingress_bytes=394379264 link_bytes=394379264 token_bytes=650694768 weight_movement_bytes=69206016 total_nvlink_bytes=719900784 weight_to_token_ratio=0.106357 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=27136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.484017420e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=318210048 dispatch_meta_bytes=325248 combine_payload_bytes=318210048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=318535296 ingress_bytes=352813056 link_bytes=352813056 token_bytes=636745344 weight_movement_bytes=34603008 total_nvlink_bytes=671348352 weight_to_token_ratio=0.054344 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=24704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.327607126e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=325500928 dispatch_meta_bytes=333104 combine_payload_bytes=325500928 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=325834032 ingress_bytes=394706944 link_bytes=394706944 token_bytes=651334960 weight_movement_bytes=69206016 total_nvlink_bytes=720540976 weight_to_token_ratio=0.106253 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=37632 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.485250454e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=299278336 dispatch_meta_bytes=312800 combine_payload_bytes=299278336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=299591136 ingress_bytes=333881344 link_bytes=333881344 token_bytes=598869472 weight_movement_bytes=34603008 total_nvlink_bytes=633472480 weight_to_token_ratio=0.057781 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=22144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256368618e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71472 payload_rows_remote=58512 dup_rows=26832 dup_rows_remote=24172 dedup_ratio=0.272949 router_tokens_per_expert=[5617, 0, 0, 0, 54, 0, 0, 0, 1, 445, 0, 0, 8, 0, 542, 6226, 70, 7291, 1, 91, 0, 0, 71, 0, 7419, 5, 1, 249, 3, 6260, 92, 6719, 3447, 145, 1, 0, 1, 0, 0, 0, 91, 0, 371, 5, 0, 2620, 249, 5, 0, 0, 0, 389, 1, 0, 0, 0, 4, 129, 523, 0, 2, 4, 0, 0] payload_rows_per_dst=[12960, 13042, 14590, 16044, 6898, 5840, 780, 1318] dup_rows_per_dst=[2660, 2328, 458, 20248, 290, 842, 0, 6] combine_rows_per_src=[12960, 13042, 14590, 16044, 6898, 5840, 780, 1318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94200 payload_rows_remote=80874 dup_rows=4104 dup_rows_remote=3900 dedup_ratio=0.041748 router_tokens_per_expert=[5323, 0, 0, 0, 53, 0, 0, 0, 0, 495, 0, 0, 38, 0, 591, 5964, 66, 7329, 0, 69, 0, 0, 66, 0, 7504, 0, 1, 261, 0, 6434, 69, 6654, 3909, 122, 0, 0, 0, 0, 0, 0, 69, 0, 426, 0, 0, 2513, 261, 0, 0, 0, 0, 334, 0, 0, 0, 0, 0, 112, 489, 0, 0, 0, 0, 0] payload_rows_per_dst=[10752, 12472, 14658, 13326, 7818, 5618, 15316, 14240] dup_rows_per_dst=[0, 1704, 402, 204, 244, 920, 360, 270] combine_rows_per_src=[10752, 12472, 14658, 13326, 7818, 5618, 15316, 14240] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=292691968 dispatch_meta_bytes=329960 combine_payload_bytes=292691968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293021928 ingress_bytes=327294976 link_bytes=327294976 token_bytes=585713896 weight_movement_bytes=34603008 total_nvlink_bytes=620316904 weight_to_token_ratio=0.059078 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=46080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.231584645e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84684 payload_rows_remote=69994 dup_rows=13620 dup_rows_remote=11652 dedup_ratio=0.138550 router_tokens_per_expert=[0, 0, 51, 152, 0, 3, 3357, 0, 0, 2268, 6850, 0, 0, 0, 761, 0, 670, 381, 0, 81, 3305, 2, 0, 0, 824, 0, 0, 328, 0, 0, 0, 0, 0, 32, 0, 0, 7525, 286, 0, 69, 601, 0, 1945, 0, 0, 0, 0, 0, 1247, 0, 69, 0, 7909, 0, 71, 0, 2334, 0, 0, 0, 7177, 528, 326, 0] payload_rows_per_dst=[6978, 13194, 8324, 14690, 15388, 6628, 13106, 6376] dup_rows_per_dst=[148, 4170, 554, 1968, 4138, 858, 1784, 0] combine_rows_per_src=[6978, 13194, 8324, 14690, 15388, 6628, 13106, 6376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=303538176 dispatch_meta_bytes=336624 combine_payload_bytes=303538176 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=303874800 ingress_bytes=338141184 link_bytes=338141184 token_bytes=607412976 weight_movement_bytes=34603008 total_nvlink_bytes=642015984 weight_to_token_ratio=0.056968 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=44288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.272398053e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88020 payload_rows_remote=74106 dup_rows=10284 dup_rows_remote=10050 dedup_ratio=0.104614 router_tokens_per_expert=[6223, 0, 0, 0, 42, 0, 0, 0, 0, 416, 0, 0, 2, 0, 486, 6457, 72, 7301, 0, 96, 0, 0, 72, 0, 7419, 0, 8, 254, 0, 6399, 95, 6425, 2515, 137, 0, 0, 0, 0, 0, 0, 94, 0, 457, 0, 0, 2839, 254, 0, 0, 0, 0, 440, 0, 0, 0, 0, 0, 86, 563, 0, 0, 0, 0, 0] payload_rows_per_dst=[12530, 1808, 13106, 714, 14620, 16034, 15294, 13914] dup_rows_per_dst=[0, 0, 288, 0, 3482, 5856, 424, 234] combine_rows_per_src=[12530, 1808, 13106, 714, 14620, 16034, 15294, 13914] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79424 payload_rows_remote=66114 dup_rows=18880 dup_rows_remote=17212 dedup_ratio=0.192057 router_tokens_per_expert=[6074, 0, 0, 0, 60, 0, 0, 0, 0, 428, 0, 0, 2, 0, 626, 6433, 92, 7320, 0, 74, 0, 0, 92, 0, 7466, 0, 2, 239, 0, 6303, 71, 6584, 2448, 131, 0, 0, 0, 0, 0, 0, 72, 0, 398, 0, 0, 2976, 239, 0, 0, 0, 0, 370, 0, 0, 0, 0, 0, 121, 531, 0, 0, 0, 0, 0] payload_rows_per_dst=[12268, 13310, 14640, 15264, 4896, 6454, 3062, 9530] dup_rows_per_dst=[0, 1668, 516, 15286, 262, 916, 74, 158] combine_rows_per_src=[12268, 13310, 14640, 15264, 4896, 6454, 3062, 9530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87870 payload_rows_remote=73066 dup_rows=10434 dup_rows_remote=5134 dedup_ratio=0.106140 router_tokens_per_expert=[6104, 0, 0, 0, 66, 0, 0, 0, 0, 379, 0, 0, 0, 0, 606, 6560, 60, 7236, 0, 90, 0, 0, 60, 0, 7427, 0, 8, 247, 0, 6246, 90, 6754, 2230, 156, 0, 0, 0, 0, 0, 0, 89, 0, 519, 0, 0, 2890, 247, 0, 0, 0, 0, 489, 0, 0, 0, 0, 0, 29, 570, 0, 0, 0, 0, 0] payload_rows_per_dst=[12340, 13590, 2114, 690, 14362, 14804, 15348, 14622] dup_rows_per_dst=[0, 1500, 164, 0, 2902, 5300, 484, 84] combine_rows_per_src=[12340, 13590, 2114, 690, 14362, 14804, 15348, 14622] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=239665152 dispatch_meta_bytes=330736 combine_payload_bytes=239665152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=239995888 ingress_bytes=274268160 link_bytes=274268160 token_bytes=479661040 weight_movement_bytes=34603008 total_nvlink_bytes=514264048 weight_to_token_ratio=0.072141 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.037500 gemm_active_groups=4 gemm_peak_group_rows=46592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.032049006e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=286695424 dispatch_meta_bytes=326584 combine_payload_bytes=286695424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=287022008 ingress_bytes=321298432 link_bytes=321298432 token_bytes=573717432 weight_movement_bytes=34603008 total_nvlink_bytes=608320440 weight_to_token_ratio=0.060314 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=40448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.209020133e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83228 payload_rows_remote=67902 dup_rows=15076 dup_rows_remote=10386 dedup_ratio=0.153361 router_tokens_per_expert=[0, 0, 41, 172, 0, 0, 2992, 0, 0, 1893, 7350, 0, 0, 0, 765, 0, 862, 358, 0, 76, 3905, 0, 0, 0, 733, 0, 0, 274, 0, 0, 0, 0, 0, 7, 0, 0, 7506, 339, 0, 62, 837, 0, 1884, 0, 0, 0, 0, 0, 998, 0, 71, 0, 7879, 0, 71, 0, 2352, 0, 0, 0, 6616, 546, 563, 0] payload_rows_per_dst=[16234, 15326, 10450, 13918, 12982, 5116, 2280, 6922] dup_rows_per_dst=[5934, 4690, 2214, 1328, 584, 326, 0, 0] combine_rows_per_src=[16234, 15326, 10450, 13918, 12982, 5116, 2280, 6922] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82852 payload_rows_remote=66614 dup_rows=15452 dup_rows_remote=13298 dedup_ratio=0.157186 router_tokens_per_expert=[0, 0, 18, 139, 0, 1, 2994, 0, 0, 2111, 6934, 0, 0, 0, 704, 0, 760, 364, 0, 92, 4114, 7, 0, 0, 810, 0, 0, 240, 0, 0, 0, 0, 0, 13, 0, 0, 7556, 335, 0, 63, 769, 0, 1780, 0, 0, 0, 0, 0, 1144, 0, 63, 0, 7897, 0, 92, 0, 2263, 0, 0, 0, 7154, 523, 212, 0] payload_rows_per_dst=[6150, 5630, 9816, 2100, 12246, 14780, 16238, 15892] dup_rows_per_dst=[154, 0, 858, 0, 542, 4186, 2154, 7558] combine_rows_per_src=[6150, 5630, 9816, 2100, 12246, 14780, 16238, 15892] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=331259904 dispatch_meta_bytes=339096 combine_payload_bytes=331259904 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=331599000 ingress_bytes=504274944 link_bytes=504274944 token_bytes=662858904 weight_movement_bytes=173015040 total_nvlink_bytes=835873944 weight_to_token_ratio=0.261013 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.897546017e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94156 payload_rows_remote=79468 dup_rows=4148 dup_rows_remote=3808 dedup_ratio=0.042196 router_tokens_per_expert=[5995, 0, 0, 0, 90, 0, 0, 0, 0, 435, 0, 0, 0, 0, 526, 6258, 44, 7344, 0, 82, 0, 0, 44, 0, 7460, 0, 9, 264, 0, 6353, 82, 6768, 2814, 172, 0, 0, 0, 0, 0, 0, 82, 0, 452, 0, 0, 2671, 264, 0, 0, 0, 0, 383, 0, 0, 0, 0, 0, 116, 444, 0, 0, 0, 0, 0] payload_rows_per_dst=[12170, 12926, 14688, 13218, 5628, 5930, 15230, 14366] dup_rows_per_dst=[0, 1512, 340, 198, 344, 1008, 456, 290] combine_rows_per_src=[12170, 12926, 14688, 13218, 5628, 5930, 15230, 14366] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88294 payload_rows_remote=73126 dup_rows=10010 dup_rows_remote=9238 dedup_ratio=0.101827 router_tokens_per_expert=[0, 0, 28, 178, 0, 1, 3249, 0, 0, 2064, 7247, 0, 0, 0, 743, 0, 754, 347, 0, 70, 3673, 0, 0, 0, 909, 0, 0, 309, 0, 0, 0, 0, 0, 2, 0, 0, 7509, 384, 0, 75, 733, 0, 1711, 0, 0, 0, 0, 0, 1161, 0, 75, 0, 7904, 0, 41, 0, 2175, 0, 0, 0, 6918, 560, 332, 0] payload_rows_per_dst=[6716, 5614, 8956, 14388, 15168, 15124, 16194, 6134] dup_rows_per_dst=[196, 0, 732, 1884, 772, 4258, 2168, 0] combine_rows_per_src=[6716, 5614, 8956, 14388, 15168, 15124, 16194, 6134] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82102 payload_rows_remote=72474 dup_rows=16202 dup_rows_remote=15230 dedup_ratio=0.164815 router_tokens_per_expert=[0, 0, 31, 129, 0, 0, 3347, 0, 0, 2087, 7034, 0, 0, 0, 700, 0, 764, 414, 0, 113, 4009, 0, 0, 0, 663, 0, 0, 314, 0, 0, 0, 0, 0, 13, 0, 0, 7537, 316, 0, 78, 652, 0, 1773, 0, 0, 0, 0, 0, 1191, 0, 82, 0, 7871, 0, 111, 0, 2144, 0, 0, 0, 6902, 546, 331, 0] payload_rows_per_dst=[14546, 14874, 9628, 14216, 15430, 4598, 2768, 6042] dup_rows_per_dst=[5330, 4768, 972, 1542, 3338, 252, 0, 0] combine_rows_per_src=[14546, 14874, 9628, 14216, 15430, 4598, 2768, 6042] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88284 payload_rows_remote=73086 dup_rows=10020 dup_rows_remote=5334 dedup_ratio=0.101929 router_tokens_per_expert=[0, 0, 30, 154, 0, 0, 3103, 0, 0, 1721, 7404, 0, 0, 0, 759, 0, 963, 361, 0, 87, 4158, 0, 0, 0, 785, 0, 0, 171, 0, 0, 0, 0, 0, 2, 0, 0, 7430, 377, 0, 44, 896, 0, 1642, 0, 0, 0, 0, 0, 1068, 0, 90, 0, 7865, 0, 76, 0, 2336, 0, 0, 0, 6661, 494, 475, 0] payload_rows_per_dst=[6432, 4960, 10050, 12708, 15044, 15198, 16242, 7650] dup_rows_per_dst=[142, 0, 1088, 948, 662, 4686, 1956, 538] combine_rows_per_src=[6432, 4960, 10050, 12708, 15044, 15198, 16242, 7650] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=303538176 dispatch_meta_bytes=336624 combine_payload_bytes=303538176 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=303874800 ingress_bytes=338141184 link_bytes=338141184 token_bytes=607412976 weight_movement_bytes=34603008 total_nvlink_bytes=642015984 weight_to_token_ratio=0.056968 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=44288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.272398053e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=299278336 dispatch_meta_bytes=312800 combine_payload_bytes=299278336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=299591136 ingress_bytes=333881344 link_bytes=333881344 token_bytes=598869472 weight_movement_bytes=34603008 total_nvlink_bytes=633472480 weight_to_token_ratio=0.057781 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=22144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256368618e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=270802944 dispatch_meta_bytes=333304 combine_payload_bytes=270802944 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=271136248 ingress_bytes=340008960 link_bytes=340008960 token_bytes=541939192 weight_movement_bytes=69206016 total_nvlink_bytes=611145208 weight_to_token_ratio=0.127701 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=40704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.279426344e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82852 payload_rows_remote=66614 dup_rows=15452 dup_rows_remote=13298 dedup_ratio=0.157186 router_tokens_per_expert=[0, 0, 18, 139, 0, 1, 2994, 0, 0, 2111, 6934, 0, 0, 0, 704, 0, 760, 364, 0, 92, 4114, 7, 0, 0, 810, 0, 0, 240, 0, 0, 0, 0, 0, 13, 0, 0, 7556, 335, 0, 63, 769, 0, 1780, 0, 0, 0, 0, 0, 1144, 0, 63, 0, 7897, 0, 92, 0, 2263, 0, 0, 0, 7154, 523, 212, 0] payload_rows_per_dst=[6150, 5630, 9816, 2100, 12246, 14780, 16238, 15892] dup_rows_per_dst=[154, 0, 858, 0, 542, 4186, 2154, 7558] combine_rows_per_src=[6150, 5630, 9816, 2100, 12246, 14780, 16238, 15892] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=278126592 dispatch_meta_bytes=313152 combine_payload_bytes=278126592 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=278439744 ingress_bytes=347332608 link_bytes=347332608 token_bytes=556566336 weight_movement_bytes=69206016 total_nvlink_bytes=625772352 weight_to_token_ratio=0.124345 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=27008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.306984641e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70348 payload_rows_remote=54054 dup_rows=27956 dup_rows_remote=9302 dedup_ratio=0.284383 router_tokens_per_expert=[0, 0, 45, 183, 0, 1, 3233, 0, 0, 1765, 7351, 0, 0, 0, 718, 0, 729, 337, 0, 57, 4254, 0, 0, 0, 702, 0, 0, 275, 0, 0, 0, 0, 0, 2, 0, 0, 7433, 430, 0, 48, 787, 0, 1775, 0, 0, 0, 0, 0, 981, 0, 95, 0, 7870, 0, 40, 0, 2426, 0, 0, 0, 6569, 488, 558, 0] payload_rows_per_dst=[6746, 4966, 9944, 1954, 956, 13286, 16202, 16294] dup_rows_per_dst=[178, 0, 810, 0, 4, 3778, 4532, 18654] combine_rows_per_src=[6746, 4966, 9944, 1954, 956, 13286, 16202, 16294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=272850944 dispatch_meta_bytes=319648 combine_payload_bytes=272850944 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=273170592 ingress_bytes=376659968 link_bytes=376659968 token_bytes=546021536 weight_movement_bytes=103809024 total_nvlink_bytes=649830560 weight_to_token_ratio=0.190119 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=37632 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.417341136e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=325500928 dispatch_meta_bytes=333104 combine_payload_bytes=325500928 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=325834032 ingress_bytes=394706944 link_bytes=394706944 token_bytes=651334960 weight_movement_bytes=69206016 total_nvlink_bytes=720540976 weight_to_token_ratio=0.106253 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=37632 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.485250454e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=299524096 dispatch_meta_bytes=329456 combine_payload_bytes=299524096 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=299853552 ingress_bytes=403333120 link_bytes=403333120 token_bytes=599377648 weight_movement_bytes=103809024 total_nvlink_bytes=703186672 weight_to_token_ratio=0.173195 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=42368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.517710060e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76194 payload_rows_remote=60002 dup_rows=22110 dup_rows_remote=15882 dedup_ratio=0.224915 router_tokens_per_expert=[1, 0, 88, 171, 1, 3, 3082, 0, 0, 2469, 6865, 0, 4, 6, 712, 0, 594, 362, 4, 69, 3494, 2, 0, 0, 1039, 0, 0, 339, 0, 1, 0, 1, 0, 34, 3, 2, 7429, 395, 0, 91, 695, 3, 1911, 0, 1, 0, 0, 5, 1182, 1, 91, 0, 7864, 1, 65, 0, 2314, 0, 1, 0, 6950, 520, 287, 0] payload_rows_per_dst=[16192, 14814, 15886, 14496, 982, 4902, 2680, 6242] dup_rows_per_dst=[6228, 5298, 8022, 2164, 68, 328, 0, 2] combine_rows_per_src=[16192, 14814, 15886, 14496, 982, 4902, 2680, 6242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84684 payload_rows_remote=69994 dup_rows=13620 dup_rows_remote=11652 dedup_ratio=0.138550 router_tokens_per_expert=[0, 0, 51, 152, 0, 3, 3357, 0, 0, 2268, 6850, 0, 0, 0, 761, 0, 670, 381, 0, 81, 3305, 2, 0, 0, 824, 0, 0, 328, 0, 0, 0, 0, 0, 32, 0, 0, 7525, 286, 0, 69, 601, 0, 1945, 0, 0, 0, 0, 0, 1247, 0, 69, 0, 7909, 0, 71, 0, 2334, 0, 0, 0, 7177, 528, 326, 0] payload_rows_per_dst=[6978, 13194, 8324, 14690, 15388, 6628, 13106, 6376] dup_rows_per_dst=[148, 4170, 554, 1968, 4138, 858, 1784, 0] combine_rows_per_src=[6978, 13194, 8324, 14690, 15388, 6628, 13106, 6376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=245768192 dispatch_meta_bytes=303536 combine_payload_bytes=245768192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=246071728 ingress_bytes=280371200 link_bytes=280371200 token_bytes=491839920 weight_movement_bytes=34603008 total_nvlink_bytes=526442928 weight_to_token_ratio=0.070354 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=25472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.055014254e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70348 payload_rows_remote=54054 dup_rows=27956 dup_rows_remote=9302 dedup_ratio=0.284383 router_tokens_per_expert=[0, 0, 45, 183, 0, 1, 3233, 0, 0, 1765, 7351, 0, 0, 0, 718, 0, 729, 337, 0, 57, 4254, 0, 0, 0, 702, 0, 0, 275, 0, 0, 0, 0, 0, 2, 0, 0, 7433, 430, 0, 48, 787, 0, 1775, 0, 0, 0, 0, 0, 981, 0, 95, 0, 7870, 0, 40, 0, 2426, 0, 0, 0, 6569, 488, 558, 0] payload_rows_per_dst=[6746, 4966, 9944, 1954, 956, 13286, 16202, 16294] dup_rows_per_dst=[178, 0, 810, 0, 4, 3778, 4532, 18654] combine_rows_per_src=[6746, 4966, 9944, 1954, 956, 13286, 16202, 16294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88284 payload_rows_remote=73086 dup_rows=10020 dup_rows_remote=5334 dedup_ratio=0.101929 router_tokens_per_expert=[0, 0, 30, 154, 0, 0, 3103, 0, 0, 1721, 7404, 0, 0, 0, 759, 0, 963, 361, 0, 87, 4158, 0, 0, 0, 785, 0, 0, 171, 0, 0, 0, 0, 0, 2, 0, 0, 7430, 377, 0, 44, 896, 0, 1642, 0, 0, 0, 0, 0, 1068, 0, 90, 0, 7865, 0, 76, 0, 2336, 0, 0, 0, 6661, 494, 475, 0] payload_rows_per_dst=[6432, 4960, 10050, 12708, 15044, 15198, 16242, 7650] dup_rows_per_dst=[142, 0, 1088, 948, 662, 4686, 1956, 538] combine_rows_per_src=[6432, 4960, 10050, 12708, 15044, 15198, 16242, 7650] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83228 payload_rows_remote=67902 dup_rows=15076 dup_rows_remote=10386 dedup_ratio=0.153361 router_tokens_per_expert=[0, 0, 41, 172, 0, 0, 2992, 0, 0, 1893, 7350, 0, 0, 0, 765, 0, 862, 358, 0, 76, 3905, 0, 0, 0, 733, 0, 0, 274, 0, 0, 0, 0, 0, 7, 0, 0, 7506, 339, 0, 62, 837, 0, 1884, 0, 0, 0, 0, 0, 998, 0, 71, 0, 7879, 0, 71, 0, 2352, 0, 0, 0, 6616, 546, 563, 0] payload_rows_per_dst=[16234, 15326, 10450, 13918, 12982, 5116, 2280, 6922] dup_rows_per_dst=[5934, 4690, 2214, 1328, 584, 326, 0, 0] combine_rows_per_src=[16234, 15326, 10450, 13918, 12982, 5116, 2280, 6922] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=299524096 dispatch_meta_bytes=329456 combine_payload_bytes=299524096 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=299853552 ingress_bytes=403333120 link_bytes=403333120 token_bytes=599377648 weight_movement_bytes=103809024 total_nvlink_bytes=703186672 weight_to_token_ratio=0.173195 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=42368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.517710060e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=296853504 dispatch_meta_bytes=350816 combine_payload_bytes=296853504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=297204320 ingress_bytes=331456512 link_bytes=331456512 token_bytes=594057824 weight_movement_bytes=34603008 total_nvlink_bytes=628660832 weight_to_token_ratio=0.058249 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=30976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.247244170e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75430 payload_rows_remote=60888 dup_rows=22874 dup_rows_remote=22126 dedup_ratio=0.232686 router_tokens_per_expert=[500, 0, 0, 65, 319, 4793, 0, 0, 5474, 59, 161, 0, 0, 0, 0, 0, 0, 0, 1955, 0, 0, 5, 0, 0, 141, 0, 0, 0, 1, 65, 0, 0, 0, 0, 48, 0, 0, 0, 720, 254, 0, 4546, 7601, 8132, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 374, 7206, 3750, 0, 0, 0, 614, 2304, 0, 0] payload_rows_per_dst=[11100, 11388, 9224, 414, 1932, 16304, 14542, 10526] dup_rows_per_dst=[254, 0, 958, 0, 112, 17992, 748, 2810] combine_rows_per_src=[11100, 11388, 9224, 414, 1932, 16304, 14542, 10526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82102 payload_rows_remote=72474 dup_rows=16202 dup_rows_remote=15230 dedup_ratio=0.164815 router_tokens_per_expert=[0, 0, 31, 129, 0, 0, 3347, 0, 0, 2087, 7034, 0, 0, 0, 700, 0, 764, 414, 0, 113, 4009, 0, 0, 0, 663, 0, 0, 314, 0, 0, 0, 0, 0, 13, 0, 0, 7537, 316, 0, 78, 652, 0, 1773, 0, 0, 0, 0, 0, 1191, 0, 82, 0, 7871, 0, 111, 0, 2144, 0, 0, 0, 6902, 546, 331, 0] payload_rows_per_dst=[14546, 14874, 9628, 14216, 15430, 4598, 2768, 6042] dup_rows_per_dst=[5330, 4768, 972, 1542, 3338, 252, 0, 0] combine_rows_per_src=[14546, 14874, 9628, 14216, 15430, 4598, 2768, 6042] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=286695424 dispatch_meta_bytes=326584 combine_payload_bytes=286695424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=287022008 ingress_bytes=321298432 link_bytes=321298432 token_bytes=573717432 weight_movement_bytes=34603008 total_nvlink_bytes=608320440 weight_to_token_ratio=0.060314 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=40448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.209020133e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90658 payload_rows_remote=75326 dup_rows=7646 dup_rows_remote=6046 dedup_ratio=0.077779 router_tokens_per_expert=[528, 0, 0, 75, 320, 4939, 0, 0, 5320, 31, 138, 0, 0, 0, 0, 0, 0, 0, 2154, 0, 0, 1, 0, 0, 149, 0, 0, 0, 0, 75, 0, 0, 0, 0, 65, 0, 0, 0, 630, 246, 0, 4713, 7525, 8129, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 387, 7177, 3554, 0, 0, 0, 628, 2292, 0, 1] payload_rows_per_dst=[11442, 10978, 12000, 16258, 15332, 0, 14504, 10144] dup_rows_per_dst=[282, 0, 1736, 448, 1600, 0, 774, 2806] combine_rows_per_src=[11442, 10978, 12000, 16258, 15332, 0, 14504, 10144] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=299360256 dispatch_meta_bytes=313680 combine_payload_bytes=299360256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=299673936 ingress_bytes=333963264 link_bytes=333963264 token_bytes=599034192 weight_movement_bytes=34603008 total_nvlink_bytes=633637200 weight_to_token_ratio=0.057765 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=28800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256676876e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=272850944 dispatch_meta_bytes=319648 combine_payload_bytes=272850944 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=273170592 ingress_bytes=376659968 link_bytes=376659968 token_bytes=546021536 weight_movement_bytes=103809024 total_nvlink_bytes=649830560 weight_to_token_ratio=0.190119 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=37632 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.417341136e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91062 payload_rows_remote=74784 dup_rows=7242 dup_rows_remote=6830 dedup_ratio=0.073669 router_tokens_per_expert=[503, 0, 0, 69, 330, 5159, 0, 0, 5509, 67, 153, 0, 0, 0, 0, 0, 0, 0, 1704, 0, 0, 2, 0, 0, 137, 0, 0, 0, 0, 69, 0, 0, 0, 0, 78, 0, 0, 0, 749, 261, 0, 4563, 7534, 8139, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 402, 7175, 3239, 0, 0, 0, 606, 2635, 0, 0] payload_rows_per_dst=[11866, 11458, 11336, 16278, 15384, 0, 14488, 10252] dup_rows_per_dst=[256, 0, 1202, 412, 1860, 0, 804, 2708] combine_rows_per_src=[11866, 11458, 11336, 16278, 15384, 0, 14488, 10252] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=221405184 dispatch_meta_bytes=253424 combine_payload_bytes=221405184 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=221658608 ingress_bytes=290611200 link_bytes=290611200 token_bytes=443063792 weight_movement_bytes=69206016 total_nvlink_bytes=512269808 weight_to_token_ratio=0.156199 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=18432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.093546549e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=245768192 dispatch_meta_bytes=303536 combine_payload_bytes=245768192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=246071728 ingress_bytes=280371200 link_bytes=280371200 token_bytes=491839920 weight_movement_bytes=34603008 total_nvlink_bytes=526442928 weight_to_token_ratio=0.070354 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=25472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.055014254e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90592 payload_rows_remote=79378 dup_rows=7712 dup_rows_remote=7712 dedup_ratio=0.078451 router_tokens_per_expert=[523, 0, 0, 74, 311, 4564, 0, 0, 5447, 29, 131, 0, 0, 0, 0, 0, 0, 0, 2519, 0, 0, 0, 0, 0, 163, 0, 0, 0, 0, 74, 0, 0, 0, 0, 81, 0, 0, 0, 660, 239, 0, 4618, 7546, 8132, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 420, 7176, 3705, 0, 0, 0, 621, 2045, 0, 0] payload_rows_per_dst=[10658, 11214, 12084, 16264, 15344, 7520, 7404, 10104] dup_rows_per_dst=[286, 0, 2190, 474, 1708, 0, 416, 2638] combine_rows_per_src=[10658, 11214, 12084, 16264, 15344, 7520, 7404, 10104] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=306315264 dispatch_meta_bytes=326456 combine_payload_bytes=306315264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=306641720 ingress_bytes=340918272 link_bytes=340918272 token_bytes=612956984 weight_movement_bytes=34603008 total_nvlink_bytes=647559992 weight_to_token_ratio=0.056453 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=47360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.282848011e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90658 payload_rows_remote=75326 dup_rows=7646 dup_rows_remote=6046 dedup_ratio=0.077779 router_tokens_per_expert=[528, 0, 0, 75, 320, 4939, 0, 0, 5320, 31, 138, 0, 0, 0, 0, 0, 0, 0, 2154, 0, 0, 1, 0, 0, 149, 0, 0, 0, 0, 75, 0, 0, 0, 0, 65, 0, 0, 0, 630, 246, 0, 4713, 7525, 8129, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 387, 7177, 3554, 0, 0, 0, 628, 2292, 0, 1] payload_rows_per_dst=[11442, 10978, 12000, 16258, 15332, 0, 14504, 10144] dup_rows_per_dst=[282, 0, 1736, 448, 1600, 0, 774, 2806] combine_rows_per_src=[11442, 10978, 12000, 16258, 15332, 0, 14504, 10144] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90708 payload_rows_remote=79046 dup_rows=7596 dup_rows_remote=5802 dedup_ratio=0.077271 router_tokens_per_expert=[591, 0, 0, 82, 346, 5039, 0, 0, 5561, 40, 172, 0, 0, 0, 0, 0, 0, 0, 2256, 0, 0, 0, 0, 0, 130, 0, 0, 0, 2, 82, 0, 0, 0, 0, 72, 0, 0, 0, 575, 264, 0, 4472, 7545, 8102, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 337, 7115, 3404, 0, 0, 0, 659, 2224, 0, 0] payload_rows_per_dst=[11760, 11546, 11662, 16204, 15378, 0, 14394, 9764] dup_rows_per_dst=[356, 0, 1794, 428, 1534, 0, 674, 2810] combine_rows_per_src=[11760, 11546, 11662, 16204, 15378, 0, 14394, 9764] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87780 payload_rows_remote=78936 dup_rows=10524 dup_rows_remote=7788 dedup_ratio=0.107056 router_tokens_per_expert=[626, 0, 0, 90, 336, 4735, 0, 0, 4788, 1, 156, 0, 0, 0, 0, 0, 0, 0, 2828, 0, 0, 0, 0, 0, 150, 0, 0, 0, 0, 90, 0, 0, 0, 0, 161, 0, 0, 0, 706, 247, 0, 4637, 7608, 8126, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 389, 7111, 3714, 0, 0, 0, 581, 1978, 0, 4] payload_rows_per_dst=[11254, 9890, 12274, 13404, 7660, 8844, 14402, 10052] dup_rows_per_dst=[320, 0, 2656, 426, 1106, 2736, 778, 2502] combine_rows_per_src=[11254, 9890, 12274, 13404, 7660, 8844, 14402, 10052] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=278126592 dispatch_meta_bytes=313152 combine_payload_bytes=278126592 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=278439744 ingress_bytes=347332608 link_bytes=347332608 token_bytes=556566336 weight_movement_bytes=69206016 total_nvlink_bytes=625772352 weight_to_token_ratio=0.124345 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=27008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.306984641e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=325132288 dispatch_meta_bytes=348360 combine_payload_bytes=325132288 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=325480648 ingress_bytes=359735296 link_bytes=359735296 token_bytes=650612936 weight_movement_bytes=34603008 total_nvlink_bytes=685215944 weight_to_token_ratio=0.053185 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=43008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.353654958e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75430 payload_rows_remote=60888 dup_rows=22874 dup_rows_remote=22126 dedup_ratio=0.232686 router_tokens_per_expert=[500, 0, 0, 65, 319, 4793, 0, 0, 5474, 59, 161, 0, 0, 0, 0, 0, 0, 0, 1955, 0, 0, 5, 0, 0, 141, 0, 0, 0, 1, 65, 0, 0, 0, 0, 48, 0, 0, 0, 720, 254, 0, 4546, 7601, 8132, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 374, 7206, 3750, 0, 0, 0, 614, 2304, 0, 0] payload_rows_per_dst=[11100, 11388, 9224, 414, 1932, 16304, 14542, 10526] dup_rows_per_dst=[254, 0, 958, 0, 112, 17992, 748, 2810] combine_rows_per_src=[11100, 11388, 9224, 414, 1932, 16304, 14542, 10526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=249397248 dispatch_meta_bytes=332056 combine_payload_bytes=249397248 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=249729304 ingress_bytes=353206272 link_bytes=353206272 token_bytes=499126552 weight_movement_bytes=103809024 total_nvlink_bytes=602935576 weight_to_token_ratio=0.207981 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=45568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.329086766e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83024 payload_rows_remote=69104 dup_rows=15280 dup_rows_remote=11968 dedup_ratio=0.155436 router_tokens_per_expert=[581, 0, 7, 91, 340, 5045, 1, 0, 5286, 30, 152, 0, 0, 0, 0, 0, 0, 0, 2123, 4, 2, 1, 4, 0, 162, 0, 0, 3, 0, 91, 0, 0, 4, 0, 140, 0, 0, 0, 759, 249, 0, 4921, 7432, 8128, 0, 4, 0, 0, 1, 1, 1, 91, 0, 0, 361, 7110, 3253, 8, 0, 2, 588, 2174, 0, 2] payload_rows_per_dst=[13920, 13650, 12162, 16262, 15162, 4818, 908, 6142] dup_rows_per_dst=[3312, 6696, 1948, 506, 2006, 0, 2, 810] combine_rows_per_src=[13920, 13650, 12162, 16262, 15162, 4818, 908, 6142] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69910 payload_rows_remote=59160 dup_rows=28394 dup_rows_remote=24908 dedup_ratio=0.288839 router_tokens_per_expert=[562, 0, 0, 96, 349, 4612, 0, 0, 5597, 9, 136, 0, 0, 0, 0, 0, 0, 0, 2539, 0, 0, 0, 0, 0, 167, 0, 0, 0, 0, 96, 0, 0, 0, 0, 174, 0, 0, 0, 665, 254, 0, 4349, 7545, 8151, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 326, 7123, 3561, 0, 0, 0, 545, 2200, 0, 0] payload_rows_per_dst=[10948, 11484, 5078, 526, 1836, 16384, 12904, 10750] dup_rows_per_dst=[290, 0, 0, 0, 350, 23706, 562, 3486] combine_rows_per_src=[10948, 11484, 5078, 526, 1836, 16384, 12904, 10750] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=296853504 dispatch_meta_bytes=350816 combine_payload_bytes=296853504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=297204320 ingress_bytes=331456512 link_bytes=331456512 token_bytes=594057824 weight_movement_bytes=34603008 total_nvlink_bytes=628660832 weight_to_token_ratio=0.058249 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=30976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.247244170e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=308535296 dispatch_meta_bytes=325488 combine_payload_bytes=308535296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=308860784 ingress_bytes=343138304 link_bytes=343138304 token_bytes=617396080 weight_movement_bytes=34603008 total_nvlink_bytes=651999088 weight_to_token_ratio=0.056047 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=40960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.291201813e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=323772416 dispatch_meta_bytes=339392 combine_payload_bytes=323772416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=324111808 ingress_bytes=358375424 link_bytes=358375424 token_bytes=647884224 weight_movement_bytes=34603008 total_nvlink_bytes=682487232 weight_to_token_ratio=0.053409 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=31104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.348537869e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=283049984 dispatch_meta_bytes=324288 combine_payload_bytes=283049984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=283374272 ingress_bytes=317652992 link_bytes=317652992 token_bytes=566424256 weight_movement_bytes=34603008 total_nvlink_bytes=601027264 weight_to_token_ratio=0.061090 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=38912 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.195302636e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91062 payload_rows_remote=74784 dup_rows=7242 dup_rows_remote=6830 dedup_ratio=0.073669 router_tokens_per_expert=[503, 0, 0, 69, 330, 5159, 0, 0, 5509, 67, 153, 0, 0, 0, 0, 0, 0, 0, 1704, 0, 0, 2, 0, 0, 137, 0, 0, 0, 0, 69, 0, 0, 0, 0, 78, 0, 0, 0, 749, 261, 0, 4563, 7534, 8139, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 402, 7175, 3239, 0, 0, 0, 606, 2635, 0, 0] payload_rows_per_dst=[11866, 11458, 11336, 16278, 15384, 0, 14488, 10252] dup_rows_per_dst=[256, 0, 1202, 412, 1860, 0, 804, 2708] combine_rows_per_src=[11866, 11458, 11336, 16278, 15384, 0, 14488, 10252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87780 payload_rows_remote=78936 dup_rows=10524 dup_rows_remote=7788 dedup_ratio=0.107056 router_tokens_per_expert=[626, 0, 0, 90, 336, 4735, 0, 0, 4788, 1, 156, 0, 0, 0, 0, 0, 0, 0, 2828, 0, 0, 0, 0, 0, 150, 0, 0, 0, 0, 90, 0, 0, 0, 0, 161, 0, 0, 0, 706, 247, 0, 4637, 7608, 8126, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 389, 7111, 3714, 0, 0, 0, 581, 1978, 0, 4] payload_rows_per_dst=[11254, 9890, 12274, 13404, 7660, 8844, 14402, 10052] dup_rows_per_dst=[320, 0, 2656, 426, 1106, 2736, 778, 2502] combine_rows_per_src=[11254, 9890, 12274, 13404, 7660, 8844, 14402, 10052] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69910 payload_rows_remote=59160 dup_rows=28394 dup_rows_remote=24908 dedup_ratio=0.288839 router_tokens_per_expert=[562, 0, 0, 96, 349, 4612, 0, 0, 5597, 9, 136, 0, 0, 0, 0, 0, 0, 0, 2539, 0, 0, 0, 0, 0, 167, 0, 0, 0, 0, 96, 0, 0, 0, 0, 174, 0, 0, 0, 665, 254, 0, 4349, 7545, 8151, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 326, 7123, 3561, 0, 0, 0, 545, 2200, 0, 0] payload_rows_per_dst=[10948, 11484, 5078, 526, 1836, 16384, 12904, 10750] dup_rows_per_dst=[290, 0, 0, 0, 350, 23706, 562, 3486] combine_rows_per_src=[10948, 11484, 5078, 526, 1836, 16384, 12904, 10750] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90592 payload_rows_remote=79378 dup_rows=7712 dup_rows_remote=7712 dedup_ratio=0.078451 router_tokens_per_expert=[523, 0, 0, 74, 311, 4564, 0, 0, 5447, 29, 131, 0, 0, 0, 0, 0, 0, 0, 2519, 0, 0, 0, 0, 0, 163, 0, 0, 0, 0, 74, 0, 0, 0, 0, 81, 0, 0, 0, 660, 239, 0, 4618, 7546, 8132, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 420, 7176, 3705, 0, 0, 0, 621, 2045, 0, 0] payload_rows_per_dst=[10658, 11214, 12084, 16264, 15344, 7520, 7404, 10104] dup_rows_per_dst=[286, 0, 2190, 474, 1708, 0, 416, 2638] combine_rows_per_src=[10658, 11214, 12084, 16264, 15344, 7520, 7404, 10104] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71084 payload_rows_remote=59570 dup_rows=27220 dup_rows_remote=26786 dedup_ratio=0.276896 router_tokens_per_expert=[778, 290, 0, 0, 0, 0, 1342, 0, 0, 286, 0, 286, 0, 0, 254, 0, 0, 464, 0, 0, 0, 626, 0, 0, 0, 0, 7810, 237, 119, 0, 60, 7627, 0, 0, 0, 0, 0, 0, 0, 299, 0, 0, 6950, 152, 7852, 69, 7510, 0, 5706, 0, 0, 205, 0, 63, 0, 0, 0, 0, 0, 0, 60, 0, 0, 107] payload_rows_per_dst=[3512, 1652, 2172, 16224, 4438, 15702, 11514, 15870] dup_rows_per_dst=[1308, 0, 8, 15482, 32, 9788, 434, 168] combine_rows_per_src=[3512, 1652, 2172, 16224, 4438, 15702, 11514, 15870] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=306315264 dispatch_meta_bytes=326456 combine_payload_bytes=306315264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=306641720 ingress_bytes=340918272 link_bytes=340918272 token_bytes=612956984 weight_movement_bytes=34603008 total_nvlink_bytes=647559992 weight_to_token_ratio=0.056453 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=47360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.282848011e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94148 payload_rows_remote=78640 dup_rows=4156 dup_rows_remote=4050 dedup_ratio=0.042277 router_tokens_per_expert=[822, 325, 0, 0, 0, 0, 1327, 0, 0, 304, 0, 260, 0, 0, 246, 0, 0, 495, 0, 0, 0, 635, 0, 0, 0, 0, 7796, 143, 101, 0, 63, 7600, 0, 0, 0, 0, 0, 0, 0, 303, 0, 0, 6863, 132, 7851, 75, 7504, 0, 5816, 0, 0, 256, 0, 75, 0, 0, 0, 0, 0, 0, 63, 0, 0, 97] payload_rows_per_dst=[3554, 16234, 14956, 15762, 15508, 414, 11844, 15876] dup_rows_per_dst=[1394, 978, 1030, 52, 106, 0, 450, 146] combine_rows_per_src=[3554, 16234, 14956, 15762, 15508, 414, 11844, 15876] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90708 payload_rows_remote=79046 dup_rows=7596 dup_rows_remote=5802 dedup_ratio=0.077271 router_tokens_per_expert=[591, 0, 0, 82, 346, 5039, 0, 0, 5561, 40, 172, 0, 0, 0, 0, 0, 0, 0, 2256, 0, 0, 0, 0, 0, 130, 0, 0, 0, 2, 82, 0, 0, 0, 0, 72, 0, 0, 0, 575, 264, 0, 4472, 7545, 8102, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 337, 7115, 3404, 0, 0, 0, 659, 2224, 0, 0] payload_rows_per_dst=[11760, 11546, 11662, 16204, 15378, 0, 14394, 9764] dup_rows_per_dst=[356, 0, 1794, 428, 1534, 0, 674, 2810] combine_rows_per_src=[11760, 11546, 11662, 16204, 15378, 0, 14394, 9764] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=323321856 dispatch_meta_bytes=346896 combine_payload_bytes=323321856 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=323668752 ingress_bytes=461733888 link_bytes=461733888 token_bytes=646990608 weight_movement_bytes=138412032 total_nvlink_bytes=785402640 weight_to_token_ratio=0.213932 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=19584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.737467448e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92212 payload_rows_remote=88428 dup_rows=6092 dup_rows_remote=6090 dedup_ratio=0.061971 router_tokens_per_expert=[821, 347, 0, 0, 0, 0, 1332, 0, 0, 266, 0, 260, 0, 0, 260, 0, 0, 483, 0, 0, 0, 585, 0, 0, 0, 0, 7793, 259, 80, 0, 53, 7597, 0, 0, 0, 0, 0, 0, 0, 302, 0, 0, 6847, 132, 7870, 69, 7515, 0, 5945, 0, 0, 142, 0, 69, 0, 0, 0, 0, 0, 0, 53, 0, 0, 72] payload_rows_per_dst=[13586, 16244, 14890, 3784, 15552, 402, 11896, 15858] dup_rows_per_dst=[3606, 914, 940, 2, 82, 0, 416, 132] combine_rows_per_src=[13586, 16244, 14890, 3784, 15552, 402, 11896, 15858] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=249397248 dispatch_meta_bytes=332056 combine_payload_bytes=249397248 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=249729304 ingress_bytes=353206272 link_bytes=353206272 token_bytes=499126552 weight_movement_bytes=103809024 total_nvlink_bytes=602935576 weight_to_token_ratio=0.207981 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=45568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.329086766e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=283049984 dispatch_meta_bytes=324288 combine_payload_bytes=283049984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=283374272 ingress_bytes=317652992 link_bytes=317652992 token_bytes=566424256 weight_movement_bytes=34603008 total_nvlink_bytes=601027264 weight_to_token_ratio=0.061090 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=38912 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.195302636e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=242319360 dispatch_meta_bytes=336272 combine_payload_bytes=242319360 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=242655632 ingress_bytes=311525376 link_bytes=311525376 token_bytes=484974992 weight_movement_bytes=69206016 total_nvlink_bytes=554181008 weight_to_token_ratio=0.142700 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=25728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.172244910e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91386 payload_rows_remote=75146 dup_rows=6918 dup_rows_remote=6084 dedup_ratio=0.070374 router_tokens_per_expert=[848, 332, 0, 0, 0, 0, 1344, 0, 0, 273, 0, 218, 0, 0, 237, 0, 0, 493, 0, 0, 0, 627, 0, 0, 0, 0, 7809, 109, 90, 0, 60, 7575, 0, 0, 0, 0, 0, 0, 0, 265, 0, 0, 6836, 136, 7893, 74, 7526, 0, 5881, 0, 0, 303, 0, 71, 0, 0, 0, 0, 0, 0, 60, 0, 0, 92] payload_rows_per_dst=[15932, 16240, 14910, 518, 15530, 14246, 12030, 1980] dup_rows_per_dst=[4266, 834, 1002, 0, 52, 264, 480, 20] combine_rows_per_src=[15932, 16240, 14910, 518, 15530, 14246, 12030, 1980] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=362201088 dispatch_meta_bytes=378072 combine_payload_bytes=362201088 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=362579160 ingress_bytes=500613120 link_bytes=500613120 token_bytes=724780248 weight_movement_bytes=138412032 total_nvlink_bytes=863192280 weight_to_token_ratio=0.190971 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=26880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.883766868e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94148 payload_rows_remote=78640 dup_rows=4156 dup_rows_remote=4050 dedup_ratio=0.042277 router_tokens_per_expert=[822, 325, 0, 0, 0, 0, 1327, 0, 0, 304, 0, 260, 0, 0, 246, 0, 0, 495, 0, 0, 0, 635, 0, 0, 0, 0, 7796, 143, 101, 0, 63, 7600, 0, 0, 0, 0, 0, 0, 0, 303, 0, 0, 6863, 132, 7851, 75, 7504, 0, 5816, 0, 0, 256, 0, 75, 0, 0, 0, 0, 0, 0, 63, 0, 0, 97] payload_rows_per_dst=[3554, 16234, 14956, 15762, 15508, 414, 11844, 15876] dup_rows_per_dst=[1394, 978, 1030, 52, 106, 0, 450, 146] combine_rows_per_src=[3554, 16234, 14956, 15762, 15508, 414, 11844, 15876] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=307798016 dispatch_meta_bytes=324920 combine_payload_bytes=307798016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=308122936 ingress_bytes=342401024 link_bytes=342401024 token_bytes=615920952 weight_movement_bytes=34603008 total_nvlink_bytes=650523960 weight_to_token_ratio=0.056181 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=43008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.288427488e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91580 payload_rows_remote=76524 dup_rows=6724 dup_rows_remote=5664 dedup_ratio=0.068400 router_tokens_per_expert=[787, 350, 0, 0, 0, 0, 1339, 0, 0, 272, 0, 253, 0, 0, 262, 0, 0, 525, 0, 0, 0, 626, 0, 0, 0, 0, 7764, 163, 88, 0, 90, 7559, 0, 0, 0, 0, 0, 0, 0, 298, 0, 0, 6907, 203, 7835, 83, 7414, 0, 5839, 0, 0, 279, 0, 82, 0, 0, 0, 0, 0, 0, 90, 0, 0, 44] payload_rows_per_dst=[15952, 16216, 15056, 682, 15356, 572, 11896, 15850] dup_rows_per_dst=[4118, 886, 1060, 0, 68, 0, 504, 88] combine_rows_per_src=[15952, 16216, 15056, 682, 15356, 572, 11896, 15850] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=325132288 dispatch_meta_bytes=348360 combine_payload_bytes=325132288 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=325480648 ingress_bytes=359735296 link_bytes=359735296 token_bytes=650612936 weight_movement_bytes=34603008 total_nvlink_bytes=685215944 weight_to_token_ratio=0.053185 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=43008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.353654958e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87144 payload_rows_remote=85286 dup_rows=11160 dup_rows_remote=11134 dedup_ratio=0.113525 router_tokens_per_expert=[790, 445, 0, 0, 0, 0, 1463, 0, 0, 283, 0, 233, 0, 0, 246, 0, 0, 483, 0, 0, 0, 550, 0, 0, 0, 0, 7764, 26, 96, 0, 66, 7478, 0, 0, 0, 0, 0, 0, 0, 277, 0, 0, 6865, 153, 7877, 90, 7445, 0, 5893, 0, 0, 407, 0, 90, 0, 0, 0, 0, 0, 0, 66, 0, 0, 66] payload_rows_per_dst=[3796, 9032, 13506, 15454, 15384, 1858, 12224, 15890] dup_rows_per_dst=[1600, 452, 892, 7446, 60, 26, 556, 128] combine_rows_per_src=[3796, 9032, 13506, 15454, 15384, 1858, 12224, 15890] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71084 payload_rows_remote=59570 dup_rows=27220 dup_rows_remote=26786 dedup_ratio=0.276896 router_tokens_per_expert=[778, 290, 0, 0, 0, 0, 1342, 0, 0, 286, 0, 286, 0, 0, 254, 0, 0, 464, 0, 0, 0, 626, 0, 0, 0, 0, 7810, 237, 119, 0, 60, 7627, 0, 0, 0, 0, 0, 0, 0, 299, 0, 0, 6950, 152, 7852, 69, 7510, 0, 5706, 0, 0, 205, 0, 63, 0, 0, 0, 0, 0, 0, 60, 0, 0, 107] payload_rows_per_dst=[3512, 1652, 2172, 16224, 4438, 15702, 11514, 15870] dup_rows_per_dst=[1308, 0, 8, 15482, 32, 9788, 434, 168] combine_rows_per_src=[3512, 1652, 2172, 16224, 4438, 15702, 11514, 15870] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=322109440 dispatch_meta_bytes=330760 combine_payload_bytes=322109440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=322440200 ingress_bytes=356712448 link_bytes=356712448 token_bytes=644549640 weight_movement_bytes=34603008 total_nvlink_bytes=679152648 weight_to_token_ratio=0.053686 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=46848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.342280224e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=243998720 dispatch_meta_bytes=345424 combine_payload_bytes=243998720 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=244344144 ingress_bytes=278601728 link_bytes=278601728 token_bytes=488342864 weight_movement_bytes=34603008 total_nvlink_bytes=522945872 weight_to_token_ratio=0.070858 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=46464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.048355873e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91302 payload_rows_remote=75360 dup_rows=7002 dup_rows_remote=2910 dedup_ratio=0.071228 router_tokens_per_expert=[801, 430, 0, 0, 0, 0, 1296, 0, 2, 272, 2, 351, 3, 0, 249, 0, 1, 527, 0, 0, 0, 568, 0, 1, 0, 1, 7750, 232, 113, 2, 54, 7490, 7, 3, 5, 0, 6, 0, 0, 283, 0, 4, 6765, 123, 7871, 93, 7486, 2, 5952, 0, 1, 177, 0, 91, 0, 0, 0, 0, 0, 8, 54, 0, 0, 76] payload_rows_per_dst=[15942, 16188, 14782, 1804, 15492, 15870, 10948, 276] dup_rows_per_dst=[4092, 1070, 942, 8, 88, 316, 486, 0] combine_rows_per_src=[15942, 16188, 14782, 1804, 15492, 15870, 10948, 276] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=322109440 dispatch_meta_bytes=330760 combine_payload_bytes=322109440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=322440200 ingress_bytes=356712448 link_bytes=356712448 token_bytes=644549640 weight_movement_bytes=34603008 total_nvlink_bytes=679152648 weight_to_token_ratio=0.053686 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=46848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.342280224e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=323772416 dispatch_meta_bytes=339392 combine_payload_bytes=323772416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=324111808 ingress_bytes=358375424 link_bytes=358375424 token_bytes=647884224 weight_movement_bytes=34603008 total_nvlink_bytes=682487232 weight_to_token_ratio=0.053409 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=31104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.348537869e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=308674560 dispatch_meta_bytes=313080 combine_payload_bytes=308674560 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=308987640 ingress_bytes=343277568 link_bytes=343277568 token_bytes=617662200 weight_movement_bytes=34603008 total_nvlink_bytes=652265208 weight_to_token_ratio=0.056023 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=28800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.291725852e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92212 payload_rows_remote=88428 dup_rows=6092 dup_rows_remote=6090 dedup_ratio=0.061971 router_tokens_per_expert=[821, 347, 0, 0, 0, 0, 1332, 0, 0, 266, 0, 260, 0, 0, 260, 0, 0, 483, 0, 0, 0, 585, 0, 0, 0, 0, 7793, 259, 80, 0, 53, 7597, 0, 0, 0, 0, 0, 0, 0, 302, 0, 0, 6847, 132, 7870, 69, 7515, 0, 5945, 0, 0, 142, 0, 69, 0, 0, 0, 0, 0, 0, 53, 0, 0, 72] payload_rows_per_dst=[13586, 16244, 14890, 3784, 15552, 402, 11896, 15858] dup_rows_per_dst=[3606, 914, 940, 2, 82, 0, 416, 132] combine_rows_per_src=[13586, 16244, 14890, 3784, 15552, 402, 11896, 15858] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87144 payload_rows_remote=85286 dup_rows=11160 dup_rows_remote=11134 dedup_ratio=0.113525 router_tokens_per_expert=[790, 445, 0, 0, 0, 0, 1463, 0, 0, 283, 0, 233, 0, 0, 246, 0, 0, 483, 0, 0, 0, 550, 0, 0, 0, 0, 7764, 26, 96, 0, 66, 7478, 0, 0, 0, 0, 0, 0, 0, 277, 0, 0, 6865, 153, 7877, 90, 7445, 0, 5893, 0, 0, 407, 0, 90, 0, 0, 0, 0, 0, 0, 66, 0, 0, 66] payload_rows_per_dst=[3796, 9032, 13506, 15454, 15384, 1858, 12224, 15890] dup_rows_per_dst=[1600, 452, 892, 7446, 60, 26, 556, 128] combine_rows_per_src=[3796, 9032, 13506, 15454, 15384, 1858, 12224, 15890] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68136 payload_rows_remote=52262 dup_rows=30168 dup_rows_remote=30020 dedup_ratio=0.306885 router_tokens_per_expert=[762, 452, 0, 0, 0, 0, 1609, 0, 0, 251, 0, 177, 0, 0, 254, 0, 0, 528, 0, 0, 0, 551, 0, 0, 0, 0, 7747, 45, 104, 0, 42, 7459, 0, 0, 0, 0, 0, 0, 0, 278, 0, 0, 6844, 99, 7893, 99, 7527, 0, 5822, 0, 0, 396, 0, 95, 0, 0, 0, 0, 0, 0, 42, 0, 0, 76] payload_rows_per_dst=[4138, 1360, 2156, 15776, 556, 16196, 12080, 15874] dup_rows_per_dst=[1508, 4, 2, 5310, 0, 22650, 546, 148] combine_rows_per_src=[4138, 1360, 2156, 15776, 556, 16196, 12080, 15874] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91386 payload_rows_remote=75146 dup_rows=6918 dup_rows_remote=6084 dedup_ratio=0.070374 router_tokens_per_expert=[848, 332, 0, 0, 0, 0, 1344, 0, 0, 273, 0, 218, 0, 0, 237, 0, 0, 493, 0, 0, 0, 627, 0, 0, 0, 0, 7809, 109, 90, 0, 60, 7575, 0, 0, 0, 0, 0, 0, 0, 265, 0, 0, 6836, 136, 7893, 74, 7526, 0, 5881, 0, 0, 303, 0, 71, 0, 0, 0, 0, 0, 0, 60, 0, 0, 92] payload_rows_per_dst=[15932, 16240, 14910, 518, 15530, 14246, 12030, 1980] dup_rows_per_dst=[4266, 834, 1002, 0, 52, 264, 480, 20] combine_rows_per_src=[15932, 16240, 14910, 518, 15530, 14246, 12030, 1980] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=313442304 dispatch_meta_bytes=328752 combine_payload_bytes=313442304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=313771056 ingress_bytes=348045312 link_bytes=348045312 token_bytes=627213360 weight_movement_bytes=34603008 total_nvlink_bytes=661816368 weight_to_token_ratio=0.055169 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=40448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.309666489e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92820 payload_rows_remote=76590 dup_rows=5484 dup_rows_remote=4976 dedup_ratio=0.055786 router_tokens_per_expert=[0, 0, 75, 0, 0, 0, 449, 0, 0, 0, 63, 0, 7180, 0, 0, 7196, 65, 0, 0, 0, 0, 2814, 0, 0, 0, 0, 0, 8129, 0, 0, 8115, 0, 0, 0, 63, 0, 0, 8131, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 0, 0, 0, 302, 0, 0, 765, 5213, 0, 0] payload_rows_per_dst=[898, 126, 15422, 16258, 16262, 15068, 16230, 12556] dup_rows_per_dst=[150, 0, 4696, 0, 126, 0, 508, 4] combine_rows_per_src=[898, 126, 15422, 16258, 16262, 15068, 16230, 12556] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93198 payload_rows_remote=76944 dup_rows=5106 dup_rows_remote=4956 dedup_ratio=0.051941 router_tokens_per_expert=[0, 0, 69, 0, 0, 0, 519, 0, 0, 0, 75, 0, 7026, 0, 0, 7309, 75, 0, 0, 0, 0, 2642, 0, 0, 0, 0, 0, 8117, 0, 0, 8163, 0, 0, 0, 75, 0, 0, 8127, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 0, 0, 0, 310, 0, 0, 904, 5155, 0, 0] payload_rows_per_dst=[16234, 3396, 13054, 0, 16254, 15298, 16326, 12636] dup_rows_per_dst=[1176, 0, 3186, 0, 150, 0, 492, 102] combine_rows_per_src=[16234, 3396, 13054, 0, 16254, 15298, 16326, 12636] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=362201088 dispatch_meta_bytes=378072 combine_payload_bytes=362201088 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=362579160 ingress_bytes=500613120 link_bytes=500613120 token_bytes=724780248 weight_movement_bytes=138412032 total_nvlink_bytes=863192280 weight_to_token_ratio=0.190971 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=26880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.883766868e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91580 payload_rows_remote=76524 dup_rows=6724 dup_rows_remote=5664 dedup_ratio=0.068400 router_tokens_per_expert=[787, 350, 0, 0, 0, 0, 1339, 0, 0, 272, 0, 253, 0, 0, 262, 0, 0, 525, 0, 0, 0, 626, 0, 0, 0, 0, 7764, 163, 88, 0, 90, 7559, 0, 0, 0, 0, 0, 0, 0, 298, 0, 0, 6907, 203, 7835, 83, 7414, 0, 5839, 0, 0, 279, 0, 82, 0, 0, 0, 0, 0, 0, 90, 0, 0, 44] payload_rows_per_dst=[15952, 16216, 15056, 682, 15356, 572, 11896, 15850] dup_rows_per_dst=[4118, 886, 1060, 0, 68, 0, 504, 88] combine_rows_per_src=[15952, 16216, 15056, 682, 15356, 572, 11896, 15850] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=349331456 dispatch_meta_bytes=385680 combine_payload_bytes=349331456 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=349717136 ingress_bytes=487743488 link_bytes=487743488 token_bytes=699048592 weight_movement_bytes=138412032 total_nvlink_bytes=837460624 weight_to_token_ratio=0.198001 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=14976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.835339479e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96450 payload_rows_remote=96450 dup_rows=1854 dup_rows_remote=1854 dedup_ratio=0.018860 router_tokens_per_expert=[0, 0, 66, 0, 0, 0, 530, 0, 0, 0, 69, 0, 7145, 0, 0, 7196, 69, 0, 0, 0, 0, 3355, 0, 0, 0, 0, 0, 8123, 0, 0, 8113, 0, 0, 0, 69, 0, 0, 8139, 0, 0, 0, 0, 345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 261, 0, 0, 0, 339, 0, 0, 804, 4529, 0, 0] payload_rows_per_dst=[16246, 14428, 6848, 0, 16278, 15082, 16226, 11342] dup_rows_per_dst=[1192, 0, 0, 0, 138, 0, 522, 2] combine_rows_per_src=[16246, 14428, 6848, 0, 16278, 15082, 16226, 11342] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=243998720 dispatch_meta_bytes=345424 combine_payload_bytes=243998720 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=244344144 ingress_bytes=278601728 link_bytes=278601728 token_bytes=488342864 weight_movement_bytes=34603008 total_nvlink_bytes=522945872 weight_to_token_ratio=0.070858 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=46464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.048355873e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=308674560 dispatch_meta_bytes=313080 combine_payload_bytes=308674560 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=308987640 ingress_bytes=343277568 link_bytes=343277568 token_bytes=617662200 weight_movement_bytes=34603008 total_nvlink_bytes=652265208 weight_to_token_ratio=0.056023 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=28800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.291725852e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=214065152 dispatch_meta_bytes=329128 combine_payload_bytes=214065152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=214394280 ingress_bytes=248668160 link_bytes=248668160 token_bytes=428459432 weight_movement_bytes=34603008 total_nvlink_bytes=463062440 weight_to_token_ratio=0.080761 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=48128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.357182667e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73260 payload_rows_remote=58030 dup_rows=25044 dup_rows_remote=17528 dedup_ratio=0.254761 router_tokens_per_expert=[0, 0, 92, 0, 0, 0, 529, 0, 0, 0, 71, 0, 6987, 0, 0, 7153, 74, 0, 0, 0, 0, 2500, 0, 0, 0, 0, 0, 8121, 0, 0, 8189, 0, 0, 0, 71, 0, 0, 8132, 0, 0, 0, 0, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 239, 0, 0, 0, 320, 0, 0, 831, 5461, 0, 0] payload_rows_per_dst=[16242, 15230, 5148, 16384, 142, 6440, 478, 13196] dup_rows_per_dst=[1242, 7516, 0, 16258, 0, 0, 0, 28] combine_rows_per_src=[16242, 15230, 5148, 16384, 142, 6440, 478, 13196] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=395059200 dispatch_meta_bytes=393216 combine_payload_bytes=395059200 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=395452416 ingress_bytes=498868224 link_bytes=498868224 token_bytes=790511616 weight_movement_bytes=103809024 total_nvlink_bytes=894320640 weight_to_token_ratio=0.131319 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=20608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.877200965e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93198 payload_rows_remote=76944 dup_rows=5106 dup_rows_remote=4956 dedup_ratio=0.051941 router_tokens_per_expert=[0, 0, 69, 0, 0, 0, 519, 0, 0, 0, 75, 0, 7026, 0, 0, 7309, 75, 0, 0, 0, 0, 2642, 0, 0, 0, 0, 0, 8117, 0, 0, 8163, 0, 0, 0, 75, 0, 0, 8127, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 0, 0, 0, 310, 0, 0, 904, 5155, 0, 0] payload_rows_per_dst=[16234, 3396, 13054, 0, 16254, 15298, 16326, 12636] dup_rows_per_dst=[1176, 0, 3186, 0, 150, 0, 492, 102] combine_rows_per_src=[16234, 3396, 13054, 0, 16254, 15298, 16326, 12636] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=237690880 dispatch_meta_bytes=302232 combine_payload_bytes=237690880 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=237993112 ingress_bytes=341499904 link_bytes=341499904 token_bytes=475683992 weight_movement_bytes=103809024 total_nvlink_bytes=579493016 weight_to_token_ratio=0.218231 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=29952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.285036646e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93804 payload_rows_remote=88060 dup_rows=4500 dup_rows_remote=4500 dedup_ratio=0.045776 router_tokens_per_expert=[0, 0, 44, 0, 0, 0, 495, 0, 0, 0, 82, 0, 7069, 0, 0, 7176, 82, 0, 0, 0, 0, 2790, 0, 0, 0, 0, 0, 8110, 0, 0, 8112, 0, 0, 0, 82, 0, 0, 8102, 0, 0, 0, 0, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 0, 0, 0, 336, 0, 0, 861, 5146, 0, 0] payload_rows_per_dst=[16220, 14302, 5744, 3514, 13364, 15154, 12836, 12670] dup_rows_per_dst=[1078, 0, 0, 2850, 136, 0, 420, 16] combine_rows_per_src=[16220, 14302, 5744, 3514, 13364, 15154, 12836, 12670] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=307798016 dispatch_meta_bytes=324920 combine_payload_bytes=307798016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=308122936 ingress_bytes=342401024 link_bytes=342401024 token_bytes=615920952 weight_movement_bytes=34603008 total_nvlink_bytes=650523960 weight_to_token_ratio=0.056181 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=43008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.288427488e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93276 payload_rows_remote=78028 dup_rows=5028 dup_rows_remote=5026 dedup_ratio=0.051147 router_tokens_per_expert=[0, 0, 61, 0, 0, 0, 559, 0, 0, 0, 89, 0, 6954, 0, 0, 7161, 90, 0, 0, 0, 0, 2325, 0, 0, 0, 0, 0, 8102, 0, 0, 8190, 0, 0, 0, 90, 0, 0, 8126, 0, 0, 0, 0, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 0, 0, 0, 408, 0, 0, 686, 5600, 0, 0] payload_rows_per_dst=[7984, 178, 15162, 8694, 16252, 15248, 16380, 13378] dup_rows_per_dst=[766, 0, 3576, 0, 180, 2, 494, 10] combine_rows_per_src=[7984, 178, 15162, 8694, 16252, 15248, 16380, 13378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=315162624 dispatch_meta_bytes=327600 combine_payload_bytes=315162624 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=315490224 ingress_bytes=418971648 link_bytes=418971648 token_bytes=630652848 weight_movement_bytes=103809024 total_nvlink_bytes=734461872 weight_to_token_ratio=0.164606 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=47360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.576556581e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92820 payload_rows_remote=76590 dup_rows=5484 dup_rows_remote=4976 dedup_ratio=0.055786 router_tokens_per_expert=[0, 0, 75, 0, 0, 0, 449, 0, 0, 0, 63, 0, 7180, 0, 0, 7196, 65, 0, 0, 0, 0, 2814, 0, 0, 0, 0, 0, 8129, 0, 0, 8115, 0, 0, 0, 63, 0, 0, 8131, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 0, 0, 0, 302, 0, 0, 765, 5213, 0, 0] payload_rows_per_dst=[898, 126, 15422, 16258, 16262, 15068, 16230, 12556] dup_rows_per_dst=[150, 0, 4696, 0, 126, 0, 508, 4] combine_rows_per_src=[898, 126, 15422, 16258, 16262, 15068, 16230, 12556] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=313712640 dispatch_meta_bytes=326264 combine_payload_bytes=313712640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=314038904 ingress_bytes=348315648 link_bytes=348315648 token_bytes=627751544 weight_movement_bytes=34603008 total_nvlink_bytes=662354552 weight_to_token_ratio=0.055122 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=47232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.310683742e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68452 payload_rows_remote=52268 dup_rows=29852 dup_rows_remote=28506 dedup_ratio=0.303670 router_tokens_per_expert=[0, 1, 74, 0, 0, 0, 600, 0, 0, 0, 91, 0, 7018, 0, 8, 7175, 93, 0, 0, 0, 1, 2946, 0, 4, 0, 1, 0, 8090, 0, 7, 8116, 0, 0, 0, 91, 0, 0, 8130, 5, 1, 0, 0, 423, 0, 1, 8, 6, 0, 0, 1, 0, 0, 0, 249, 4, 0, 1, 389, 0, 1, 853, 4763, 1, 0] payload_rows_per_dst=[16184, 16342, 6088, 16264, 2526, 866, 508, 9674] dup_rows_per_dst=[1346, 26598, 0, 1888, 6, 10, 0, 4] combine_rows_per_src=[16184, 16342, 6088, 16264, 2526, 866, 508, 9674] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=315162624 dispatch_meta_bytes=327600 combine_payload_bytes=315162624 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=315490224 ingress_bytes=418971648 link_bytes=418971648 token_bytes=630652848 weight_movement_bytes=103809024 total_nvlink_bytes=734461872 weight_to_token_ratio=0.164606 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=47360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.576556581e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=313442304 dispatch_meta_bytes=328752 combine_payload_bytes=313442304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=313771056 ingress_bytes=348045312 link_bytes=348045312 token_bytes=627213360 weight_movement_bytes=34603008 total_nvlink_bytes=661816368 weight_to_token_ratio=0.055169 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=40448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.309666489e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=214089728 dispatch_meta_bytes=323096 combine_payload_bytes=214089728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=214412824 ingress_bytes=248692736 link_bytes=248692736 token_bytes=428502552 weight_movement_bytes=34603008 total_nvlink_bytes=463105560 weight_to_token_ratio=0.080753 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=44416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.358107442e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96450 payload_rows_remote=96450 dup_rows=1854 dup_rows_remote=1854 dedup_ratio=0.018860 router_tokens_per_expert=[0, 0, 66, 0, 0, 0, 530, 0, 0, 0, 69, 0, 7145, 0, 0, 7196, 69, 0, 0, 0, 0, 3355, 0, 0, 0, 0, 0, 8123, 0, 0, 8113, 0, 0, 0, 69, 0, 0, 8139, 0, 0, 0, 0, 345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 261, 0, 0, 0, 339, 0, 0, 804, 4529, 0, 0] payload_rows_per_dst=[16246, 14428, 6848, 0, 16278, 15082, 16226, 11342] dup_rows_per_dst=[1192, 0, 0, 0, 138, 0, 522, 2] combine_rows_per_src=[16246, 14428, 6848, 0, 16278, 15082, 16226, 11342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93276 payload_rows_remote=78028 dup_rows=5028 dup_rows_remote=5026 dedup_ratio=0.051147 router_tokens_per_expert=[0, 0, 61, 0, 0, 0, 559, 0, 0, 0, 89, 0, 6954, 0, 0, 7161, 90, 0, 0, 0, 0, 2325, 0, 0, 0, 0, 0, 8102, 0, 0, 8190, 0, 0, 0, 90, 0, 0, 8126, 0, 0, 0, 0, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 0, 0, 0, 408, 0, 0, 686, 5600, 0, 0] payload_rows_per_dst=[7984, 178, 15162, 8694, 16252, 15248, 16380, 13378] dup_rows_per_dst=[766, 0, 3576, 0, 180, 2, 494, 10] combine_rows_per_src=[7984, 178, 15162, 8694, 16252, 15248, 16380, 13378] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93156 payload_rows_remote=80500 dup_rows=5148 dup_rows_remote=5118 dedup_ratio=0.052368 router_tokens_per_expert=[0, 0, 72, 0, 0, 0, 601, 0, 0, 0, 94, 0, 6902, 0, 0, 7125, 94, 0, 0, 0, 0, 2694, 0, 0, 0, 0, 0, 8097, 0, 0, 8186, 0, 0, 0, 95, 0, 0, 8150, 0, 0, 0, 0, 443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 428, 0, 0, 829, 5086, 0, 0] payload_rows_per_dst=[1202, 188, 15110, 16194, 16300, 15134, 16372, 12656] dup_rows_per_dst=[144, 0, 4270, 0, 190, 2, 512, 30] combine_rows_per_src=[1202, 188, 15110, 16194, 16300, 15134, 16372, 12656] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=360693760 dispatch_meta_bytes=370240 combine_payload_bytes=360693760 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=361064000 ingress_bytes=395296768 link_bytes=395296768 token_bytes=721757760 weight_movement_bytes=34603008 total_nvlink_bytes=756360768 weight_to_token_ratio=0.047943 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=26496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.487469914e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73260 payload_rows_remote=58030 dup_rows=25044 dup_rows_remote=17528 dedup_ratio=0.254761 router_tokens_per_expert=[0, 0, 92, 0, 0, 0, 529, 0, 0, 0, 71, 0, 6987, 0, 0, 7153, 74, 0, 0, 0, 0, 2500, 0, 0, 0, 0, 0, 8121, 0, 0, 8189, 0, 0, 0, 71, 0, 0, 8132, 0, 0, 0, 0, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 239, 0, 0, 0, 320, 0, 0, 831, 5461, 0, 0] payload_rows_per_dst=[16242, 15230, 5148, 16384, 142, 6440, 478, 13196] dup_rows_per_dst=[1242, 7516, 0, 16258, 0, 0, 0, 28] combine_rows_per_src=[16242, 15230, 5148, 16384, 142, 6440, 478, 13196] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85616 payload_rows_remote=71398 dup_rows=12688 dup_rows_remote=8700 dedup_ratio=0.129069 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 383, 0, 0, 0, 205, 0, 726, 0, 151, 8081, 8192, 0, 0, 0, 0, 0, 3133, 0, 7731, 0, 60, 0, 7881, 0, 0, 0, 0, 1261, 6374, 65, 0, 0, 0, 0, 126, 737, 0, 0, 0, 2637, 0, 92, 0, 0, 0, 0, 0, 0, 60, 11, 63, 0, 847, 336] payload_rows_per_dst=[0, 866, 16384, 6386, 15892, 15582, 14218, 16288] dup_rows_per_dst=[0, 310, 1754, 0, 2522, 1606, 3988, 2508] combine_rows_per_src=[0, 866, 16384, 6386, 15892, 15582, 14218, 16288] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88460 payload_rows_remote=86426 dup_rows=9844 dup_rows_remote=9694 dedup_ratio=0.100138 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 390, 0, 0, 0, 255, 0, 770, 0, 151, 8052, 8192, 0, 0, 0, 0, 0, 3320, 0, 7710, 0, 63, 0, 7838, 0, 0, 0, 0, 1017, 6192, 75, 0, 0, 0, 0, 246, 930, 0, 0, 0, 2488, 0, 70, 0, 0, 0, 0, 0, 0, 63, 3, 75, 0, 908, 344] payload_rows_per_dst=[16384, 15676, 1842, 6766, 2034, 15548, 13954, 16256] dup_rows_per_dst=[0, 1290, 0, 0, 150, 2224, 3546, 2634] combine_rows_per_src=[16384, 15676, 1842, 6766, 2034, 15548, 13954, 16256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93804 payload_rows_remote=88060 dup_rows=4500 dup_rows_remote=4500 dedup_ratio=0.045776 router_tokens_per_expert=[0, 0, 44, 0, 0, 0, 495, 0, 0, 0, 82, 0, 7069, 0, 0, 7176, 82, 0, 0, 0, 0, 2790, 0, 0, 0, 0, 0, 8110, 0, 0, 8112, 0, 0, 0, 82, 0, 0, 8102, 0, 0, 0, 0, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 0, 0, 0, 336, 0, 0, 861, 5146, 0, 0] payload_rows_per_dst=[16220, 14302, 5744, 3514, 13364, 15154, 12836, 12670] dup_rows_per_dst=[1078, 0, 0, 2850, 136, 0, 420, 16] combine_rows_per_src=[16220, 14302, 5744, 3514, 13364, 15154, 12836, 12670] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=395059200 dispatch_meta_bytes=393216 combine_payload_bytes=395059200 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=395452416 ingress_bytes=498868224 link_bytes=498868224 token_bytes=790511616 weight_movement_bytes=103809024 total_nvlink_bytes=894320640 weight_to_token_ratio=0.131319 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=20608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.877200965e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=319602688 dispatch_meta_bytes=332216 combine_payload_bytes=319602688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=319934904 ingress_bytes=354205696 link_bytes=354205696 token_bytes=639537592 weight_movement_bytes=34603008 total_nvlink_bytes=674140600 weight_to_token_ratio=0.054106 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=46080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.332847518e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88728 payload_rows_remote=80574 dup_rows=9576 dup_rows_remote=8478 dedup_ratio=0.097412 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 404, 0, 0, 0, 373, 0, 704, 0, 138, 8042, 8192, 0, 0, 0, 0, 0, 2976, 0, 7725, 0, 53, 0, 7888, 0, 0, 0, 0, 1191, 6295, 69, 0, 0, 0, 0, 165, 694, 0, 0, 0, 2834, 0, 79, 0, 0, 0, 0, 0, 0, 55, 24, 69, 0, 837, 345] payload_rows_per_dst=[16384, 15776, 2684, 8154, 7230, 15562, 10394, 12544] dup_rows_per_dst=[0, 1554, 78, 1098, 994, 1606, 2318, 1928] combine_rows_per_src=[16384, 15776, 2684, 8154, 7230, 15562, 10394, 12544] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=313712640 dispatch_meta_bytes=326264 combine_payload_bytes=313712640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=314038904 ingress_bytes=348315648 link_bytes=348315648 token_bytes=627751544 weight_movement_bytes=34603008 total_nvlink_bytes=662354552 weight_to_token_ratio=0.055122 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=47232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.310683742e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=214089728 dispatch_meta_bytes=323096 combine_payload_bytes=214089728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=214412824 ingress_bytes=248692736 link_bytes=248692736 token_bytes=428502552 weight_movement_bytes=34603008 total_nvlink_bytes=463105560 weight_to_token_ratio=0.080753 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=44416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.358107442e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87402 payload_rows_remote=71680 dup_rows=10902 dup_rows_remote=9052 dedup_ratio=0.110901 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 425, 0, 0, 0, 500, 0, 645, 0, 166, 8023, 8192, 0, 0, 0, 0, 0, 3538, 0, 7710, 0, 60, 0, 7861, 0, 0, 0, 0, 796, 6075, 74, 0, 0, 0, 0, 244, 1060, 0, 0, 0, 2355, 0, 76, 0, 0, 0, 0, 0, 0, 60, 0, 71, 0, 873, 348] payload_rows_per_dst=[16384, 15722, 16384, 16236, 12622, 2608, 4862, 2584] dup_rows_per_dst=[0, 1850, 1284, 6380, 1268, 0, 0, 120] combine_rows_per_src=[16384, 15722, 16384, 16236, 12622, 2608, 4862, 2584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=329728000 dispatch_meta_bytes=342472 combine_payload_bytes=329728000 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=330070472 ingress_bytes=364331008 link_bytes=364331008 token_bytes=659798472 weight_movement_bytes=34603008 total_nvlink_bytes=694401480 weight_to_token_ratio=0.052445 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=39808 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.370948252e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88460 payload_rows_remote=86426 dup_rows=9844 dup_rows_remote=9694 dedup_ratio=0.100138 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 390, 0, 0, 0, 255, 0, 770, 0, 151, 8052, 8192, 0, 0, 0, 0, 0, 3320, 0, 7710, 0, 63, 0, 7838, 0, 0, 0, 0, 1017, 6192, 75, 0, 0, 0, 0, 246, 930, 0, 0, 0, 2488, 0, 70, 0, 0, 0, 0, 0, 0, 63, 3, 75, 0, 908, 344] payload_rows_per_dst=[16384, 15676, 1842, 6766, 2034, 15548, 13954, 16256] dup_rows_per_dst=[0, 1290, 0, 0, 150, 2224, 3546, 2634] combine_rows_per_src=[16384, 15676, 1842, 6766, 2034, 15548, 13954, 16256] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=330031104 dispatch_meta_bytes=356208 combine_payload_bytes=330031104 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=330387312 ingress_bytes=399237120 link_bytes=399237120 token_bytes=660418416 weight_movement_bytes=69206016 total_nvlink_bytes=729624432 weight_to_token_ratio=0.104791 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=26112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.502297141e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=293601280 dispatch_meta_bytes=322928 combine_payload_bytes=293601280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293924208 ingress_bytes=328204288 link_bytes=328204288 token_bytes=587525488 weight_movement_bytes=34603008 total_nvlink_bytes=622128496 weight_to_token_ratio=0.058896 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=43392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.235006313e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=237690880 dispatch_meta_bytes=302232 combine_payload_bytes=237690880 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=237993112 ingress_bytes=341499904 link_bytes=341499904 token_bytes=475683992 weight_movement_bytes=103809024 total_nvlink_bytes=579493016 weight_to_token_ratio=0.218231 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=29952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.285036646e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89070 payload_rows_remote=72686 dup_rows=9234 dup_rows_remote=7586 dedup_ratio=0.093933 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 344, 0, 0, 0, 239, 0, 743, 0, 239, 8034, 8192, 0, 0, 0, 0, 0, 3221, 0, 7659, 0, 90, 0, 7835, 0, 0, 0, 0, 1073, 5977, 82, 0, 0, 0, 0, 297, 888, 0, 0, 0, 2632, 0, 114, 0, 0, 0, 0, 0, 0, 90, 1, 82, 0, 959, 361] payload_rows_per_dst=[16384, 15670, 16384, 13660, 12814, 5862, 5492, 2804] dup_rows_per_dst=[0, 1166, 1648, 4178, 1450, 610, 0, 182] combine_rows_per_src=[16384, 15670, 16384, 13660, 12814, 5862, 5492, 2804] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87000 payload_rows_remote=70622 dup_rows=11304 dup_rows_remote=11286 dedup_ratio=0.114990 router_tokens_per_expert=[0, 0, 8, 0, 7, 0, 1, 3, 0, 0, 371, 1, 0, 0, 311, 0, 912, 0, 245, 7938, 8179, 0, 3, 0, 0, 0, 2906, 0, 7687, 1, 55, 0, 7856, 2, 4, 0, 2, 1242, 6330, 91, 0, 3, 0, 6, 269, 836, 0, 0, 0, 2359, 0, 71, 0, 6, 0, 0, 2, 0, 70, 8, 91, 7, 923, 346] payload_rows_per_dst=[16378, 15716, 16372, 16176, 12534, 2222, 4872, 2730] dup_rows_per_dst=[18, 1362, 3036, 5122, 1596, 6, 0, 164] combine_rows_per_src=[16378, 15716, 16372, 16176, 12534, 2222, 4872, 2730] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=292446208 dispatch_meta_bytes=320392 combine_payload_bytes=292446208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=292766600 ingress_bytes=327049216 link_bytes=327049216 token_bytes=585212808 weight_movement_bytes=34603008 total_nvlink_bytes=619815816 weight_to_token_ratio=0.059129 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=27904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.230659870e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=354000896 dispatch_meta_bytes=384480 combine_payload_bytes=354000896 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=354385376 ingress_bytes=457809920 link_bytes=457809920 token_bytes=708386272 weight_movement_bytes=103809024 total_nvlink_bytes=812195296 weight_to_token_ratio=0.146543 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.722701872e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=360693760 dispatch_meta_bytes=370240 combine_payload_bytes=360693760 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=361064000 ingress_bytes=395296768 link_bytes=395296768 token_bytes=721757760 weight_movement_bytes=34603008 total_nvlink_bytes=756360768 weight_to_token_ratio=0.047943 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=26496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.487469914e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=289267712 dispatch_meta_bytes=327632 combine_payload_bytes=289267712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289595344 ingress_bytes=323870720 link_bytes=323870720 token_bytes=578863056 weight_movement_bytes=34603008 total_nvlink_bytes=613466064 weight_to_token_ratio=0.059778 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.218699446e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88728 payload_rows_remote=80574 dup_rows=9576 dup_rows_remote=8478 dedup_ratio=0.097412 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 404, 0, 0, 0, 373, 0, 704, 0, 138, 8042, 8192, 0, 0, 0, 0, 0, 2976, 0, 7725, 0, 53, 0, 7888, 0, 0, 0, 0, 1191, 6295, 69, 0, 0, 0, 0, 165, 694, 0, 0, 0, 2834, 0, 79, 0, 0, 0, 0, 0, 0, 55, 24, 69, 0, 837, 345] payload_rows_per_dst=[16384, 15776, 2684, 8154, 7230, 15562, 10394, 12544] dup_rows_per_dst=[0, 1554, 78, 1098, 994, 1606, 2318, 1928] combine_rows_per_src=[16384, 15776, 2684, 8154, 7230, 15562, 10394, 12544] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88008 payload_rows_remote=72398 dup_rows=10296 dup_rows_remote=8446 dedup_ratio=0.104736 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 399, 0, 0, 0, 436, 0, 736, 0, 291, 7934, 8192, 0, 0, 0, 0, 0, 3507, 0, 7733, 0, 66, 0, 7845, 0, 0, 0, 0, 893, 5855, 90, 0, 0, 0, 0, 226, 771, 0, 0, 0, 2518, 0, 108, 0, 0, 0, 0, 0, 0, 66, 3, 90, 0, 1090, 303] payload_rows_per_dst=[16372, 8478, 2066, 7146, 8700, 15610, 13574, 16062] dup_rows_per_dst=[0, 1230, 0, 0, 918, 1850, 3388, 2910] combine_rows_per_src=[16372, 8478, 2066, 7146, 8700, 15610, 13574, 16062] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85616 payload_rows_remote=71398 dup_rows=12688 dup_rows_remote=8700 dedup_ratio=0.129069 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 383, 0, 0, 0, 205, 0, 726, 0, 151, 8081, 8192, 0, 0, 0, 0, 0, 3133, 0, 7731, 0, 60, 0, 7881, 0, 0, 0, 0, 1261, 6374, 65, 0, 0, 0, 0, 126, 737, 0, 0, 0, 2637, 0, 92, 0, 0, 0, 0, 0, 0, 60, 11, 63, 0, 847, 336] payload_rows_per_dst=[0, 866, 16384, 6386, 15892, 15582, 14218, 16288] dup_rows_per_dst=[0, 310, 1754, 0, 2522, 1606, 3988, 2508] combine_rows_per_src=[0, 866, 16384, 6386, 15892, 15582, 14218, 16288] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87000 payload_rows_remote=70622 dup_rows=11304 dup_rows_remote=11286 dedup_ratio=0.114990 router_tokens_per_expert=[0, 0, 8, 0, 7, 0, 1, 3, 0, 0, 371, 1, 0, 0, 311, 0, 912, 0, 245, 7938, 8179, 0, 3, 0, 0, 0, 2906, 0, 7687, 1, 55, 0, 7856, 2, 4, 0, 2, 1242, 6330, 91, 0, 3, 0, 6, 269, 836, 0, 0, 0, 2359, 0, 71, 0, 6, 0, 0, 2, 0, 70, 8, 91, 7, 923, 346] payload_rows_per_dst=[16378, 15716, 16372, 16176, 12534, 2222, 4872, 2730] dup_rows_per_dst=[18, 1362, 3036, 5122, 1596, 6, 0, 164] combine_rows_per_src=[16378, 15716, 16372, 16176, 12534, 2222, 4872, 2730] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85930 payload_rows_remote=69894 dup_rows=12374 dup_rows_remote=9820 dedup_ratio=0.125875 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 328, 0, 0, 0, 491, 0, 754, 0, 248, 7921, 8192, 0, 0, 0, 0, 0, 3459, 0, 7717, 0, 42, 0, 7903, 0, 0, 0, 0, 791, 6011, 96, 0, 0, 0, 0, 239, 748, 0, 0, 0, 2782, 0, 56, 0, 0, 0, 0, 0, 0, 42, 13, 95, 0, 887, 337] payload_rows_per_dst=[0, 1032, 16384, 7002, 15998, 15522, 13956, 16036] dup_rows_per_dst=[0, 606, 2004, 0, 1582, 1886, 3742, 2554] combine_rows_per_src=[0, 1032, 16384, 7002, 15998, 15522, 13956, 16036] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=297721856 dispatch_meta_bytes=321088 combine_payload_bytes=297721856 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=298042944 ingress_bytes=436133888 link_bytes=436133888 token_bytes=595764800 weight_movement_bytes=138412032 total_nvlink_bytes=734176832 weight_to_token_ratio=0.232327 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=24576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.641136709e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87402 payload_rows_remote=71680 dup_rows=10902 dup_rows_remote=9052 dedup_ratio=0.110901 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 425, 0, 0, 0, 500, 0, 645, 0, 166, 8023, 8192, 0, 0, 0, 0, 0, 3538, 0, 7710, 0, 60, 0, 7861, 0, 0, 0, 0, 796, 6075, 74, 0, 0, 0, 0, 244, 1060, 0, 0, 0, 2355, 0, 76, 0, 0, 0, 0, 0, 0, 60, 0, 71, 0, 873, 348] payload_rows_per_dst=[16384, 15722, 16384, 16236, 12622, 2608, 4862, 2584] dup_rows_per_dst=[0, 1850, 1284, 6380, 1268, 0, 0, 120] combine_rows_per_src=[16384, 15722, 16384, 16236, 12622, 2608, 4862, 2584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96552 payload_rows_remote=86988 dup_rows=1752 dup_rows_remote=1752 dedup_ratio=0.017822 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 387, 0, 0, 0, 78, 210, 7868, 1332, 0, 0, 7656, 0, 0, 4651, 78, 53, 0, 0, 0, 0, 492, 0, 0, 0, 0, 4170, 0, 7707, 0, 0, 0, 0, 69, 0, 0, 0, 0, 272, 6329, 0, 74, 78, 7510, 69, 0, 0, 0, 69, 0, 0] payload_rows_per_dst=[15312, 15736, 3102, 9564, 9216, 15414, 13050, 15158] dup_rows_per_dst=[0, 774, 138, 0, 108, 138, 456, 138] combine_rows_per_src=[15312, 15736, 3102, 9564, 9216, 15414, 13050, 15158] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=292446208 dispatch_meta_bytes=320392 combine_payload_bytes=292446208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=292766600 ingress_bytes=327049216 link_bytes=327049216 token_bytes=585212808 weight_movement_bytes=34603008 total_nvlink_bytes=619815816 weight_to_token_ratio=0.059129 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=27904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.230659870e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86502 payload_rows_remote=70870 dup_rows=11802 dup_rows_remote=11062 dedup_ratio=0.120056 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 370, 0, 0, 0, 82, 241, 7816, 1314, 0, 0, 7639, 0, 0, 4339, 81, 60, 0, 0, 0, 0, 502, 0, 0, 0, 0, 4414, 0, 7764, 0, 0, 0, 0, 71, 0, 0, 0, 0, 302, 6113, 0, 179, 71, 7652, 71, 0, 0, 0, 71, 0, 0] payload_rows_per_dst=[15278, 15632, 15698, 8960, 12476, 522, 13242, 4694] dup_rows_per_dst=[0, 740, 3104, 0, 3704, 0, 4112, 142] combine_rows_per_src=[15278, 15632, 15698, 8960, 12476, 522, 13242, 4694] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89872 payload_rows_remote=76758 dup_rows=8432 dup_rows_remote=7986 dedup_ratio=0.085775 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 401, 0, 0, 0, 48, 208, 7872, 1288, 0, 0, 7668, 0, 0, 4680, 48, 60, 0, 0, 0, 0, 447, 0, 0, 0, 0, 4082, 0, 7799, 0, 0, 0, 0, 63, 0, 0, 0, 0, 289, 6383, 0, 60, 48, 7582, 63, 0, 0, 0, 63, 0, 0] payload_rows_per_dst=[6766, 802, 16368, 12914, 9020, 15598, 13114, 15290] dup_rows_per_dst=[0, 0, 2464, 5232, 38, 126, 446, 126] combine_rows_per_src=[6766, 802, 16368, 12914, 9020, 15598, 13114, 15290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96312 payload_rows_remote=86946 dup_rows=1992 dup_rows_remote=1872 dedup_ratio=0.020264 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 378, 0, 0, 0, 68, 219, 7814, 1263, 0, 0, 7640, 0, 0, 4708, 64, 63, 0, 0, 0, 0, 478, 0, 0, 0, 0, 4265, 0, 7739, 0, 0, 0, 0, 75, 0, 0, 0, 0, 286, 6156, 0, 165, 63, 7558, 75, 0, 0, 0, 75, 0, 0] payload_rows_per_dst=[15280, 15628, 2950, 9670, 9366, 15478, 12674, 15266] dup_rows_per_dst=[0, 756, 150, 0, 120, 150, 666, 150] combine_rows_per_src=[15280, 15628, 2950, 9670, 9366, 15478, 12674, 15266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89070 payload_rows_remote=72686 dup_rows=9234 dup_rows_remote=7586 dedup_ratio=0.093933 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 344, 0, 0, 0, 239, 0, 743, 0, 239, 8034, 8192, 0, 0, 0, 0, 0, 3221, 0, 7659, 0, 90, 0, 7835, 0, 0, 0, 0, 1073, 5977, 82, 0, 0, 0, 0, 297, 888, 0, 0, 0, 2632, 0, 114, 0, 0, 0, 0, 0, 0, 90, 1, 82, 0, 959, 361] payload_rows_per_dst=[16384, 15670, 16384, 13660, 12814, 5862, 5492, 2804] dup_rows_per_dst=[0, 1166, 1648, 4178, 1450, 610, 0, 182] combine_rows_per_src=[16384, 15670, 16384, 13660, 12814, 5862, 5492, 2804] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=330031104 dispatch_meta_bytes=356208 combine_payload_bytes=330031104 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=330387312 ingress_bytes=399237120 link_bytes=399237120 token_bytes=660418416 weight_movement_bytes=69206016 total_nvlink_bytes=729624432 weight_to_token_ratio=0.104791 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=26112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.502297141e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=296542208 dispatch_meta_bytes=323376 combine_payload_bytes=296542208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296865584 ingress_bytes=331145216 link_bytes=331145216 token_bytes=593407792 weight_movement_bytes=34603008 total_nvlink_bytes=628010800 weight_to_token_ratio=0.058312 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=40704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.246072788e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=289267712 dispatch_meta_bytes=327632 combine_payload_bytes=289267712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289595344 ingress_bytes=323870720 link_bytes=323870720 token_bytes=578863056 weight_movement_bytes=34603008 total_nvlink_bytes=613466064 weight_to_token_ratio=0.059778 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.218699446e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=286285824 dispatch_meta_bytes=318856 combine_payload_bytes=286285824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286604680 ingress_bytes=320888832 link_bytes=320888832 token_bytes=572890504 weight_movement_bytes=34603008 total_nvlink_bytes=607493512 weight_to_token_ratio=0.060401 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=38016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.207478841e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96312 payload_rows_remote=86946 dup_rows=1992 dup_rows_remote=1872 dedup_ratio=0.020264 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 378, 0, 0, 0, 68, 219, 7814, 1263, 0, 0, 7640, 0, 0, 4708, 64, 63, 0, 0, 0, 0, 478, 0, 0, 0, 0, 4265, 0, 7739, 0, 0, 0, 0, 75, 0, 0, 0, 0, 286, 6156, 0, 165, 63, 7558, 75, 0, 0, 0, 75, 0, 0] payload_rows_per_dst=[15280, 15628, 2950, 9670, 9366, 15478, 12674, 15266] dup_rows_per_dst=[0, 756, 150, 0, 120, 150, 666, 150] combine_rows_per_src=[15280, 15628, 2950, 9670, 9366, 15478, 12674, 15266] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=356302848 dispatch_meta_bytes=354960 combine_payload_bytes=356302848 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=356657808 ingress_bytes=425508864 link_bytes=425508864 token_bytes=712960656 weight_movement_bytes=69206016 total_nvlink_bytes=782166672 weight_to_token_ratio=0.097068 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=35584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.601155599e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=290283520 dispatch_meta_bytes=327728 combine_payload_bytes=290283520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=290611248 ingress_bytes=324886528 link_bytes=324886528 token_bytes=580894768 weight_movement_bytes=34603008 total_nvlink_bytes=615497776 weight_to_token_ratio=0.059568 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=46080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.222521849e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=293601280 dispatch_meta_bytes=322928 combine_payload_bytes=293601280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293924208 ingress_bytes=328204288 link_bytes=328204288 token_bytes=587525488 weight_movement_bytes=34603008 total_nvlink_bytes=622128496 weight_to_token_ratio=0.058896 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=43392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.235006313e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94742 payload_rows_remote=84704 dup_rows=3562 dup_rows_remote=1788 dedup_ratio=0.036235 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 0, 0, 0, 74, 207, 7788, 1303, 0, 0, 7577, 0, 0, 4156, 72, 90, 0, 0, 0, 0, 517, 0, 0, 0, 0, 4724, 0, 7762, 0, 0, 0, 0, 82, 0, 0, 0, 0, 253, 6141, 0, 107, 71, 7599, 82, 0, 0, 0, 82, 0, 0] payload_rows_per_dst=[15154, 15576, 10038, 8636, 10398, 6984, 12594, 15362] dup_rows_per_dst=[0, 930, 1774, 0, 84, 60, 550, 164] combine_rows_per_src=[15154, 15576, 10038, 8636, 10398, 6984, 12594, 15362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86550 payload_rows_remote=71328 dup_rows=11754 dup_rows_remote=11750 dedup_ratio=0.119568 router_tokens_per_expert=[0, 0, 7, 2, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 369, 9, 0, 0, 140, 275, 7760, 1197, 1, 0, 7598, 5, 0, 4745, 140, 54, 0, 0, 0, 6, 581, 7, 0, 0, 0, 3992, 5, 7641, 1, 7, 0, 0, 91, 0, 1, 1, 0, 280, 6267, 1, 195, 140, 7436, 93, 0, 0, 8, 91, 0, 0] payload_rows_per_dst=[15222, 15580, 15614, 10200, 15830, 12730, 1172, 202] dup_rows_per_dst=[4, 696, 2894, 390, 7512, 12, 64, 182] combine_rows_per_src=[15222, 15580, 15614, 10200, 15830, 12730, 1172, 202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=314400768 dispatch_meta_bytes=338976 combine_payload_bytes=314400768 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=314739744 ingress_bytes=383606784 link_bytes=383606784 token_bytes=629140512 weight_movement_bytes=69206016 total_nvlink_bytes=698346528 weight_to_token_ratio=0.110001 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=43008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.443481445e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=356130816 dispatch_meta_bytes=355272 combine_payload_bytes=356130816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=356486088 ingress_bytes=390733824 link_bytes=390733824 token_bytes=712616904 weight_movement_bytes=34603008 total_nvlink_bytes=747219912 weight_to_token_ratio=0.048558 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=34816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.470299923e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89872 payload_rows_remote=76758 dup_rows=8432 dup_rows_remote=7986 dedup_ratio=0.085775 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 401, 0, 0, 0, 48, 208, 7872, 1288, 0, 0, 7668, 0, 0, 4680, 48, 60, 0, 0, 0, 0, 447, 0, 0, 0, 0, 4082, 0, 7799, 0, 0, 0, 0, 63, 0, 0, 0, 0, 289, 6383, 0, 60, 48, 7582, 63, 0, 0, 0, 63, 0, 0] payload_rows_per_dst=[6766, 802, 16368, 12914, 9020, 15598, 13114, 15290] dup_rows_per_dst=[0, 0, 2464, 5232, 38, 126, 446, 126] combine_rows_per_src=[6766, 802, 16368, 12914, 9020, 15598, 13114, 15290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=297721856 dispatch_meta_bytes=321088 combine_payload_bytes=297721856 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=298042944 ingress_bytes=436133888 link_bytes=436133888 token_bytes=595764800 weight_movement_bytes=138412032 total_nvlink_bytes=734176832 weight_to_token_ratio=0.232327 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=24576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.641136709e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=292159488 dispatch_meta_bytes=332312 combine_payload_bytes=292159488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=292491800 ingress_bytes=326762496 link_bytes=326762496 token_bytes=584651288 weight_movement_bytes=34603008 total_nvlink_bytes=619254296 weight_to_token_ratio=0.059186 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.229580966e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96552 payload_rows_remote=86988 dup_rows=1752 dup_rows_remote=1752 dedup_ratio=0.017822 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 387, 0, 0, 0, 78, 210, 7868, 1332, 0, 0, 7656, 0, 0, 4651, 78, 53, 0, 0, 0, 0, 492, 0, 0, 0, 0, 4170, 0, 7707, 0, 0, 0, 0, 69, 0, 0, 0, 0, 272, 6329, 0, 74, 78, 7510, 69, 0, 0, 0, 69, 0, 0] payload_rows_per_dst=[15312, 15736, 3102, 9564, 9216, 15414, 13050, 15158] dup_rows_per_dst=[0, 774, 138, 0, 108, 138, 456, 138] combine_rows_per_src=[15312, 15736, 3102, 9564, 9216, 15414, 13050, 15158] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96102 payload_rows_remote=80734 dup_rows=2202 dup_rows_remote=2022 dedup_ratio=0.022400 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 399, 0, 0, 0, 164, 238, 7811, 1236, 0, 0, 7644, 0, 0, 4202, 161, 66, 0, 0, 0, 0, 554, 0, 0, 0, 0, 4537, 0, 7684, 0, 0, 0, 0, 90, 0, 0, 0, 0, 286, 5906, 0, 146, 126, 7722, 90, 0, 0, 0, 90, 0, 0] payload_rows_per_dst=[15288, 14040, 4532, 8858, 10102, 15368, 12290, 15624] dup_rows_per_dst=[0, 710, 414, 0, 80, 180, 638, 180] combine_rows_per_src=[15288, 14040, 4532, 8858, 10102, 15368, 12290, 15624] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86550 payload_rows_remote=71328 dup_rows=11754 dup_rows_remote=11750 dedup_ratio=0.119568 router_tokens_per_expert=[0, 0, 7, 2, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 369, 9, 0, 0, 140, 275, 7760, 1197, 1, 0, 7598, 5, 0, 4745, 140, 54, 0, 0, 0, 6, 581, 7, 0, 0, 0, 3992, 5, 7641, 1, 7, 0, 0, 91, 0, 1, 1, 0, 280, 6267, 1, 195, 140, 7436, 93, 0, 0, 8, 91, 0, 0] payload_rows_per_dst=[15222, 15580, 15614, 10200, 15830, 12730, 1172, 202] dup_rows_per_dst=[4, 696, 2894, 390, 7512, 12, 64, 182] combine_rows_per_src=[15222, 15580, 15614, 10200, 15830, 12730, 1172, 202] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=346947584 dispatch_meta_bytes=345968 combine_payload_bytes=346947584 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=347293552 ingress_bytes=416153600 link_bytes=416153600 token_bytes=694241136 weight_movement_bytes=69206016 total_nvlink_bytes=763447152 weight_to_token_ratio=0.099686 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=19840 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.565952494e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85982 payload_rows_remote=70546 dup_rows=12322 dup_rows_remote=12132 dedup_ratio=0.125346 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 352, 0, 0, 0, 179, 263, 7790, 1300, 0, 0, 7624, 0, 0, 4100, 174, 42, 0, 0, 0, 0, 606, 0, 0, 0, 0, 4519, 0, 7677, 0, 0, 0, 0, 95, 0, 0, 0, 0, 322, 6073, 0, 64, 159, 7623, 95, 0, 0, 0, 95, 0, 0] payload_rows_per_dst=[0, 704, 16354, 15332, 10148, 15354, 12654, 15436] dup_rows_per_dst=[0, 0, 2710, 8548, 102, 190, 582, 190] combine_rows_per_src=[0, 704, 16354, 15332, 10148, 15354, 12654, 15436] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86502 payload_rows_remote=70870 dup_rows=11802 dup_rows_remote=11062 dedup_ratio=0.120056 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 370, 0, 0, 0, 82, 241, 7816, 1314, 0, 0, 7639, 0, 0, 4339, 81, 60, 0, 0, 0, 0, 502, 0, 0, 0, 0, 4414, 0, 7764, 0, 0, 0, 0, 71, 0, 0, 0, 0, 302, 6113, 0, 179, 71, 7652, 71, 0, 0, 0, 71, 0, 0] payload_rows_per_dst=[15278, 15632, 15698, 8960, 12476, 522, 13242, 4694] dup_rows_per_dst=[0, 740, 3104, 0, 3704, 0, 4112, 142] combine_rows_per_src=[15278, 15632, 15698, 8960, 12476, 522, 13242, 4694] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95844 payload_rows_remote=79592 dup_rows=2460 dup_rows_remote=2216 dedup_ratio=0.025024 router_tokens_per_expert=[0, 133, 79, 1461, 8004, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4477, 0, 0, 307, 0, 0, 7057, 0, 0, 0, 0, 53, 8126, 69, 0, 0, 0, 0, 0, 7976, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 78, 0, 0, 601, 0, 0, 0, 0, 0, 7966, 69, 2401, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[3346, 8954, 14322, 16252, 15952, 16008, 16070, 4940] dup_rows_per_dst=[276, 0, 406, 244, 0, 470, 1064, 0] combine_rows_per_src=[3346, 8954, 14322, 16252, 15952, 16008, 16070, 4940] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=314400768 dispatch_meta_bytes=338976 combine_payload_bytes=314400768 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=314739744 ingress_bytes=383606784 link_bytes=383606784 token_bytes=629140512 weight_movement_bytes=69206016 total_nvlink_bytes=698346528 weight_to_token_ratio=0.110001 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=43008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.443481445e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69728 payload_rows_remote=57310 dup_rows=28576 dup_rows_remote=23860 dedup_ratio=0.290690 router_tokens_per_expert=[0, 142, 76, 1496, 7969, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4530, 0, 0, 295, 0, 0, 7032, 0, 0, 0, 0, 60, 8100, 71, 0, 0, 0, 0, 0, 7966, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 81, 0, 0, 579, 0, 0, 0, 0, 0, 7964, 71, 2418, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 12418, 14582, 1886, 3096, 314, 16070, 4978] dup_rows_per_dst=[19472, 4716, 3202, 8, 0, 162, 1016, 0] combine_rows_per_src=[16384, 12418, 14582, 1886, 3096, 314, 16070, 4978] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92226 payload_rows_remote=83368 dup_rows=6078 dup_rows_remote=5862 dedup_ratio=0.061829 router_tokens_per_expert=[0, 143, 92, 1423, 7994, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4604, 0, 0, 298, 0, 0, 7035, 0, 0, 0, 0, 60, 8117, 63, 0, 0, 0, 0, 0, 8016, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 0, 48, 0, 0, 451, 0, 0, 0, 0, 0, 7989, 63, 2488, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[3316, 9208, 6390, 16234, 16032, 15988, 8858, 16200] dup_rows_per_dst=[256, 0, 104, 246, 0, 376, 216, 4880] combine_rows_per_src=[3316, 9208, 6390, 16234, 16032, 15988, 8858, 16200] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93182 payload_rows_remote=77226 dup_rows=5122 dup_rows_remote=5122 dedup_ratio=0.052104 router_tokens_per_expert=[0, 157, 69, 1378, 7985, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4759, 0, 0, 278, 0, 0, 7069, 0, 0, 0, 0, 63, 8123, 75, 0, 0, 0, 0, 0, 7978, 0, 0, 0, 0, 0, 0, 0, 133, 0, 0, 0, 64, 0, 0, 509, 0, 0, 0, 0, 0, 7984, 75, 2303, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[3208, 9518, 14282, 16246, 15956, 15970, 4740, 13262] dup_rows_per_dst=[300, 0, 412, 276, 0, 394, 244, 3496] combine_rows_per_src=[3208, 9518, 14282, 16246, 15956, 15970, 4740, 13262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=356302848 dispatch_meta_bytes=354960 combine_payload_bytes=356302848 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=356657808 ingress_bytes=425508864 link_bytes=425508864 token_bytes=712960656 weight_movement_bytes=69206016 total_nvlink_bytes=782166672 weight_to_token_ratio=0.097068 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=35584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.601155599e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94742 payload_rows_remote=84704 dup_rows=3562 dup_rows_remote=1788 dedup_ratio=0.036235 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 0, 0, 0, 74, 207, 7788, 1303, 0, 0, 7577, 0, 0, 4156, 72, 90, 0, 0, 0, 0, 517, 0, 0, 0, 0, 4724, 0, 7762, 0, 0, 0, 0, 82, 0, 0, 0, 0, 253, 6141, 0, 107, 71, 7599, 82, 0, 0, 0, 82, 0, 0] payload_rows_per_dst=[15154, 15576, 10038, 8636, 10398, 6984, 12594, 15362] dup_rows_per_dst=[0, 930, 1774, 0, 84, 60, 550, 164] combine_rows_per_src=[15154, 15576, 10038, 8636, 10398, 6984, 12594, 15362] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=330686464 dispatch_meta_bytes=331024 combine_payload_bytes=330686464 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=331017488 ingress_bytes=399892480 link_bytes=399892480 token_bytes=661703952 weight_movement_bytes=69206016 total_nvlink_bytes=730909968 weight_to_token_ratio=0.104588 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=42112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.504763208e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=292159488 dispatch_meta_bytes=332312 combine_payload_bytes=292159488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=292491800 ingress_bytes=326762496 link_bytes=326762496 token_bytes=584651288 weight_movement_bytes=34603008 total_nvlink_bytes=619254296 weight_to_token_ratio=0.059186 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.229580966e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=288956416 dispatch_meta_bytes=330712 combine_payload_bytes=288956416 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=289287128 ingress_bytes=392765440 link_bytes=392765440 token_bytes=578243544 weight_movement_bytes=103809024 total_nvlink_bytes=682052568 weight_to_token_ratio=0.179525 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.037500 gemm_active_groups=5 gemm_peak_group_rows=47872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.477944731e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93182 payload_rows_remote=77226 dup_rows=5122 dup_rows_remote=5122 dedup_ratio=0.052104 router_tokens_per_expert=[0, 157, 69, 1378, 7985, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4759, 0, 0, 278, 0, 0, 7069, 0, 0, 0, 0, 63, 8123, 75, 0, 0, 0, 0, 0, 7978, 0, 0, 0, 0, 0, 0, 0, 133, 0, 0, 0, 64, 0, 0, 509, 0, 0, 0, 0, 0, 7984, 75, 2303, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[3208, 9518, 14282, 16246, 15956, 15970, 4740, 13262] dup_rows_per_dst=[300, 0, 412, 276, 0, 394, 244, 3496] combine_rows_per_src=[3208, 9518, 14282, 16246, 15956, 15970, 4740, 13262] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=326008832 dispatch_meta_bytes=327232 combine_payload_bytes=326008832 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=326336064 ingress_bytes=395214848 link_bytes=395214848 token_bytes=652344896 weight_movement_bytes=69206016 total_nvlink_bytes=721550912 weight_to_token_ratio=0.106088 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=47232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.487161656e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=234741760 dispatch_meta_bytes=324680 combine_payload_bytes=234741760 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=235066440 ingress_bytes=269344768 link_bytes=269344768 token_bytes=469808200 weight_movement_bytes=34603008 total_nvlink_bytes=504411208 weight_to_token_ratio=0.073653 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=37248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.013522678e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93298 payload_rows_remote=79076 dup_rows=5006 dup_rows_remote=4680 dedup_ratio=0.050924 router_tokens_per_expert=[0, 198, 113, 1405, 7976, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4706, 0, 0, 279, 0, 0, 6995, 0, 0, 0, 0, 90, 8148, 82, 0, 0, 0, 0, 0, 7956, 0, 0, 0, 0, 0, 0, 0, 185, 0, 0, 0, 73, 0, 0, 508, 0, 0, 0, 0, 0, 7925, 82, 2267, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5876, 9412, 14222, 13266, 15912, 13898, 16014, 4698] dup_rows_per_dst=[3116, 0, 326, 254, 0, 458, 852, 0] combine_rows_per_src=[5876, 9412, 14222, 13266, 15912, 13898, 16014, 4698] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=290283520 dispatch_meta_bytes=327728 combine_payload_bytes=290283520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=290611248 ingress_bytes=324886528 link_bytes=324886528 token_bytes=580894768 weight_movement_bytes=34603008 total_nvlink_bytes=615497776 weight_to_token_ratio=0.059568 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=46080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.222521849e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=67868 payload_rows_remote=51500 dup_rows=30436 dup_rows_remote=11172 dedup_ratio=0.309611 router_tokens_per_expert=[0, 159, 71, 1330, 7961, 182, 0, 0, 1, 0, 5, 0, 5, 0, 6, 4837, 8, 0, 270, 0, 5, 7015, 0, 0, 0, 0, 55, 8113, 91, 0, 5, 0, 0, 0, 7857, 0, 0, 0, 0, 0, 0, 0, 209, 0, 4, 0, 142, 5, 2, 493, 4, 0, 0, 8, 0, 7879, 91, 2334, 1, 3, 0, 1, 0, 0] payload_rows_per_dst=[16368, 15738, 14198, 302, 0, 440, 15962, 4860] dup_rows_per_dst=[19264, 9684, 398, 0, 0, 280, 810, 0] combine_rows_per_src=[16368, 15738, 14198, 302, 0, 440, 15962, 4860] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=341475328 dispatch_meta_bytes=356920 combine_payload_bytes=341475328 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=341832248 ingress_bytes=410681344 link_bytes=410681344 token_bytes=683307576 weight_movement_bytes=69206016 total_nvlink_bytes=752513592 weight_to_token_ratio=0.101281 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=33792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.545360835e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=316317696 dispatch_meta_bytes=329392 combine_payload_bytes=316317696 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=316647088 ingress_bytes=420126720 link_bytes=420126720 token_bytes=632964784 weight_movement_bytes=103809024 total_nvlink_bytes=736773808 weight_to_token_ratio=0.164004 gemm_rows_defined=49152 gemm_rows_charged=51200 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.040000 gemm_active_groups=2 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.580903024e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92226 payload_rows_remote=83368 dup_rows=6078 dup_rows_remote=5862 dedup_ratio=0.061829 router_tokens_per_expert=[0, 143, 92, 1423, 7994, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4604, 0, 0, 298, 0, 0, 7035, 0, 0, 0, 0, 60, 8117, 63, 0, 0, 0, 0, 0, 8016, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 0, 48, 0, 0, 451, 0, 0, 0, 0, 0, 7989, 63, 2488, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[3316, 9208, 6390, 16234, 16032, 15988, 8858, 16200] dup_rows_per_dst=[256, 0, 104, 246, 0, 376, 216, 4880] combine_rows_per_src=[3316, 9208, 6390, 16234, 16032, 15988, 8858, 16200] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=323895296 dispatch_meta_bytes=335024 combine_payload_bytes=323895296 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=324230320 ingress_bytes=393101312 link_bytes=393101312 token_bytes=648125616 weight_movement_bytes=69206016 total_nvlink_bytes=717331632 weight_to_token_ratio=0.106779 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=45440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.479208590e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=346947584 dispatch_meta_bytes=345968 combine_payload_bytes=346947584 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=347293552 ingress_bytes=416153600 link_bytes=416153600 token_bytes=694241136 weight_movement_bytes=69206016 total_nvlink_bytes=763447152 weight_to_token_ratio=0.099686 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=19840 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.565952494e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=210944000 dispatch_meta_bytes=250688 combine_payload_bytes=210944000 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=211194688 ingress_bytes=280150016 link_bytes=280150016 token_bytes=422138688 weight_movement_bytes=69206016 total_nvlink_bytes=491344704 weight_to_token_ratio=0.163941 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.035000 gemm_active_groups=7 gemm_peak_group_rows=17792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.054181956e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95844 payload_rows_remote=79592 dup_rows=2460 dup_rows_remote=2216 dedup_ratio=0.025024 router_tokens_per_expert=[0, 133, 79, 1461, 8004, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4477, 0, 0, 307, 0, 0, 7057, 0, 0, 0, 0, 53, 8126, 69, 0, 0, 0, 0, 0, 7976, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 78, 0, 0, 601, 0, 0, 0, 0, 0, 7966, 69, 2401, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[3346, 8954, 14322, 16252, 15952, 16008, 16070, 4940] dup_rows_per_dst=[276, 0, 406, 244, 0, 470, 1064, 0] combine_rows_per_src=[3346, 8954, 14322, 16252, 15952, 16008, 16070, 4940] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92696 payload_rows_remote=76744 dup_rows=5608 dup_rows_remote=4790 dedup_ratio=0.057048 router_tokens_per_expert=[0, 156, 87, 1359, 7976, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4796, 0, 0, 275, 0, 0, 7221, 0, 0, 0, 0, 66, 8132, 90, 0, 0, 0, 0, 0, 7851, 0, 0, 0, 0, 0, 0, 0, 248, 0, 0, 0, 161, 0, 0, 586, 0, 0, 0, 0, 0, 7854, 90, 2024, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[3204, 9592, 14616, 16264, 15702, 15952, 1172, 16194] dup_rows_per_dst=[360, 0, 376, 312, 0, 818, 0, 3742] combine_rows_per_src=[3204, 9592, 14616, 16264, 15702, 15952, 1172, 16194] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=67868 payload_rows_remote=51500 dup_rows=30436 dup_rows_remote=11172 dedup_ratio=0.309611 router_tokens_per_expert=[0, 159, 71, 1330, 7961, 182, 0, 0, 1, 0, 5, 0, 5, 0, 6, 4837, 8, 0, 270, 0, 5, 7015, 0, 0, 0, 0, 55, 8113, 91, 0, 5, 0, 0, 0, 7857, 0, 0, 0, 0, 0, 0, 0, 209, 0, 4, 0, 142, 5, 2, 493, 4, 0, 0, 8, 0, 7879, 91, 2334, 1, 3, 0, 1, 0, 0] payload_rows_per_dst=[16368, 15738, 14198, 302, 0, 440, 15962, 4860] dup_rows_per_dst=[19264, 9684, 398, 0, 0, 280, 810, 0] combine_rows_per_src=[16368, 15738, 14198, 302, 0, 440, 15962, 4860] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91754 payload_rows_remote=75486 dup_rows=6550 dup_rows_remote=1764 dedup_ratio=0.066630 router_tokens_per_expert=[0, 138, 57, 1393, 7983, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4532, 0, 0, 234, 0, 0, 7055, 0, 0, 0, 0, 42, 8120, 95, 0, 0, 0, 0, 0, 7827, 0, 0, 0, 0, 0, 0, 0, 231, 0, 0, 0, 174, 0, 0, 553, 0, 0, 0, 0, 0, 7866, 95, 2566, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[3176, 9064, 468, 16240, 15654, 15966, 14918, 16268] dup_rows_per_dst=[382, 0, 0, 274, 0, 810, 298, 4786] combine_rows_per_src=[3176, 9064, 468, 16240, 15654, 15966, 14918, 16268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69728 payload_rows_remote=57310 dup_rows=28576 dup_rows_remote=23860 dedup_ratio=0.290690 router_tokens_per_expert=[0, 142, 76, 1496, 7969, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4530, 0, 0, 295, 0, 0, 7032, 0, 0, 0, 0, 60, 8100, 71, 0, 0, 0, 0, 0, 7966, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 81, 0, 0, 579, 0, 0, 0, 0, 0, 7964, 71, 2418, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 12418, 14582, 1886, 3096, 314, 16070, 4978] dup_rows_per_dst=[19472, 4716, 3202, 8, 0, 162, 1016, 0] combine_rows_per_src=[16384, 12418, 14582, 1886, 3096, 314, 16070, 4978] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=341475328 dispatch_meta_bytes=356920 combine_payload_bytes=341475328 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=341832248 ingress_bytes=410681344 link_bytes=410681344 token_bytes=683307576 weight_movement_bytes=69206016 total_nvlink_bytes=752513592 weight_to_token_ratio=0.101281 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=33792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.545360835e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95612 payload_rows_remote=79930 dup_rows=2692 dup_rows_remote=2692 dedup_ratio=0.027384 router_tokens_per_expert=[7841, 0, 0, 0, 0, 0, 0, 2959, 407, 0, 4698, 0, 0, 0, 0, 78, 0, 0, 0, 170, 0, 78, 0, 7727, 0, 0, 0, 0, 0, 0, 0, 0, 474, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7418, 0, 0, 162, 0, 0, 0, 0, 261, 0, 7843, 0, 0, 0, 0, 0, 213, 148, 414, 0, 69] payload_rows_per_dst=[5918, 10356, 15610, 15682, 948, 15028, 15686, 16384] dup_rows_per_dst=[0, 10, 340, 0, 0, 132, 522, 1688] combine_rows_per_src=[5918, 10356, 15610, 15682, 948, 15028, 15686, 16384] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69950 payload_rows_remote=59330 dup_rows=28354 dup_rows_remote=28320 dedup_ratio=0.288432 router_tokens_per_expert=[7829, 0, 0, 0, 0, 0, 0, 2946, 438, 0, 4808, 0, 0, 0, 0, 81, 0, 0, 0, 167, 1, 81, 0, 7754, 0, 0, 0, 0, 0, 0, 0, 0, 483, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7329, 0, 0, 207, 0, 0, 0, 0, 239, 0, 7736, 0, 0, 0, 0, 0, 218, 147, 422, 0, 74] payload_rows_per_dst=[13018, 10620, 16118, 4850, 16384, 5328, 2058, 1574] dup_rows_per_dst=[14118, 34, 11412, 0, 966, 1618, 58, 148] combine_rows_per_src=[13018, 10620, 16118, 4850, 16384, 5328, 2058, 1574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95856 payload_rows_remote=80352 dup_rows=2448 dup_rows_remote=1940 dedup_ratio=0.024902 router_tokens_per_expert=[7863, 0, 0, 0, 0, 0, 0, 2787, 388, 0, 5021, 0, 0, 0, 0, 48, 0, 0, 0, 149, 0, 48, 0, 7752, 0, 0, 0, 0, 0, 0, 0, 0, 440, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7462, 0, 0, 178, 0, 0, 0, 0, 254, 0, 7752, 0, 0, 0, 0, 0, 197, 155, 401, 0, 65] payload_rows_per_dst=[5574, 10890, 394, 15726, 16254, 15130, 15504, 16384] dup_rows_per_dst=[0, 24, 0, 0, 130, 150, 508, 1636] combine_rows_per_src=[5574, 10890, 394, 15726, 16254, 15130, 15504, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=326008832 dispatch_meta_bytes=327232 combine_payload_bytes=326008832 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=326336064 ingress_bytes=395214848 link_bytes=395214848 token_bytes=652344896 weight_movement_bytes=69206016 total_nvlink_bytes=721550912 weight_to_token_ratio=0.106088 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=47232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.487161656e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95572 payload_rows_remote=94664 dup_rows=2732 dup_rows_remote=2732 dedup_ratio=0.027791 router_tokens_per_expert=[7845, 0, 0, 0, 0, 0, 0, 2650, 392, 0, 5112, 0, 0, 0, 0, 64, 0, 0, 0, 197, 3, 64, 0, 7745, 0, 0, 0, 0, 0, 0, 0, 0, 454, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7417, 0, 0, 194, 0, 0, 0, 0, 246, 0, 7728, 0, 0, 0, 0, 0, 211, 144, 419, 0, 75] payload_rows_per_dst=[5300, 11126, 15624, 15690, 908, 15084, 15456, 16384] dup_rows_per_dst=[0, 10, 394, 0, 0, 138, 492, 1698] combine_rows_per_src=[5300, 11126, 15624, 15690, 908, 15084, 15456, 16384] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=314343424 dispatch_meta_bytes=326136 combine_payload_bytes=314343424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=314669560 ingress_bytes=348946432 link_bytes=348946432 token_bytes=629012984 weight_movement_bytes=34603008 total_nvlink_bytes=663615992 weight_to_token_ratio=0.055012 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=46976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.313057331e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=210944000 dispatch_meta_bytes=250688 combine_payload_bytes=210944000 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=211194688 ingress_bytes=280150016 link_bytes=280150016 token_bytes=422138688 weight_movement_bytes=69206016 total_nvlink_bytes=491344704 weight_to_token_ratio=0.163941 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.035000 gemm_active_groups=7 gemm_peak_group_rows=17792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.054181956e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93298 payload_rows_remote=79076 dup_rows=5006 dup_rows_remote=4680 dedup_ratio=0.050924 router_tokens_per_expert=[0, 198, 113, 1405, 7976, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4706, 0, 0, 279, 0, 0, 6995, 0, 0, 0, 0, 90, 8148, 82, 0, 0, 0, 0, 0, 7956, 0, 0, 0, 0, 0, 0, 0, 185, 0, 0, 0, 73, 0, 0, 508, 0, 0, 0, 0, 0, 7925, 82, 2267, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5876, 9412, 14222, 13266, 15912, 13898, 16014, 4698] dup_rows_per_dst=[3116, 0, 326, 254, 0, 458, 852, 0] combine_rows_per_src=[5876, 9412, 14222, 13266, 15912, 13898, 16014, 4698] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=309190656 dispatch_meta_bytes=309000 combine_payload_bytes=309190656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=309499656 ingress_bytes=343793664 link_bytes=343793664 token_bytes=618690312 weight_movement_bytes=34603008 total_nvlink_bytes=653293320 weight_to_token_ratio=0.055929 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=29824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.293667880e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95572 payload_rows_remote=94664 dup_rows=2732 dup_rows_remote=2732 dedup_ratio=0.027791 router_tokens_per_expert=[7845, 0, 0, 0, 0, 0, 0, 2650, 392, 0, 5112, 0, 0, 0, 0, 64, 0, 0, 0, 197, 3, 64, 0, 7745, 0, 0, 0, 0, 0, 0, 0, 0, 454, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7417, 0, 0, 194, 0, 0, 0, 0, 246, 0, 7728, 0, 0, 0, 0, 0, 211, 144, 419, 0, 75] payload_rows_per_dst=[5300, 11126, 15624, 15690, 908, 15084, 15456, 16384] dup_rows_per_dst=[0, 10, 394, 0, 0, 138, 492, 1698] combine_rows_per_src=[5300, 11126, 15624, 15690, 908, 15084, 15456, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=327393280 dispatch_meta_bytes=330488 combine_payload_bytes=327393280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=327723768 ingress_bytes=361996288 link_bytes=361996288 token_bytes=655117048 weight_movement_bytes=34603008 total_nvlink_bytes=689720056 weight_to_token_ratio=0.052820 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.362162889e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96142 payload_rows_remote=80454 dup_rows=2162 dup_rows_remote=1772 dedup_ratio=0.021993 router_tokens_per_expert=[7751, 0, 0, 0, 0, 0, 0, 3130, 345, 0, 4614, 0, 0, 0, 0, 72, 0, 0, 0, 195, 0, 72, 0, 7772, 0, 0, 0, 0, 0, 0, 0, 0, 460, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7408, 0, 0, 165, 0, 0, 0, 0, 264, 0, 7793, 0, 0, 0, 0, 0, 199, 195, 443, 0, 82] payload_rows_per_dst=[6260, 10052, 15688, 15502, 14518, 15058, 15586, 3478] dup_rows_per_dst=[0, 10, 390, 0, 810, 88, 528, 336] combine_rows_per_src=[6260, 10052, 15688, 15502, 14518, 15058, 15586, 3478] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=243015680 dispatch_meta_bytes=350600 combine_payload_bytes=243015680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=243366280 ingress_bytes=277618688 link_bytes=277618688 token_bytes=486381960 weight_movement_bytes=34603008 total_nvlink_bytes=520984968 weight_to_token_ratio=0.071144 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=38784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.044656773e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=234741760 dispatch_meta_bytes=324680 combine_payload_bytes=234741760 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=235066440 ingress_bytes=269344768 link_bytes=269344768 token_bytes=469808200 weight_movement_bytes=34603008 total_nvlink_bytes=504411208 weight_to_token_ratio=0.073653 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=37248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.013522678e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=66164 payload_rows_remote=50066 dup_rows=32140 dup_rows_remote=11430 dedup_ratio=0.326945 router_tokens_per_expert=[7737, 5, 0, 0, 0, 0, 4, 2939, 378, 8, 4673, 0, 0, 0, 0, 139, 0, 2, 1, 169, 14, 140, 0, 7639, 0, 0, 0, 0, 0, 1, 0, 3, 563, 4, 0, 0, 0, 0, 1, 8183, 0, 9, 0, 7409, 4, 7, 195, 0, 0, 0, 0, 250, 4, 7719, 0, 4, 0, 0, 0, 262, 161, 434, 0, 91] payload_rows_per_dst=[16098, 15420, 15634, 8, 16372, 408, 510, 1714] dup_rows_per_dst=[20710, 7988, 2102, 0, 1130, 22, 6, 182] combine_rows_per_src=[16098, 15420, 15634, 8, 16372, 408, 510, 1714] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95856 payload_rows_remote=80352 dup_rows=2448 dup_rows_remote=1940 dedup_ratio=0.024902 router_tokens_per_expert=[7863, 0, 0, 0, 0, 0, 0, 2787, 388, 0, 5021, 0, 0, 0, 0, 48, 0, 0, 0, 149, 0, 48, 0, 7752, 0, 0, 0, 0, 0, 0, 0, 0, 440, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7462, 0, 0, 178, 0, 0, 0, 0, 254, 0, 7752, 0, 0, 0, 0, 0, 197, 155, 401, 0, 65] payload_rows_per_dst=[5574, 10890, 394, 15726, 16254, 15130, 15504, 16384] dup_rows_per_dst=[0, 24, 0, 0, 130, 150, 508, 1636] combine_rows_per_src=[5574, 10890, 394, 15726, 16254, 15130, 15504, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=329121792 dispatch_meta_bytes=329168 combine_payload_bytes=329121792 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=329450960 ingress_bytes=398327808 link_bytes=398327808 token_bytes=658572752 weight_movement_bytes=69206016 total_nvlink_bytes=727778768 weight_to_token_ratio=0.105085 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=47232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.498875473e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=387743744 dispatch_meta_bytes=389584 combine_payload_bytes=387743744 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=388133328 ingress_bytes=456949760 link_bytes=456949760 token_bytes=775877072 weight_movement_bytes=69206016 total_nvlink_bytes=845083088 weight_to_token_ratio=0.089197 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.719465159e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=329539584 dispatch_meta_bytes=328904 combine_payload_bytes=329539584 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=329868488 ingress_bytes=398745600 link_bytes=398745600 token_bytes=659408072 weight_movement_bytes=69206016 total_nvlink_bytes=728614088 weight_to_token_ratio=0.104952 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=40320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.500447591e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=323895296 dispatch_meta_bytes=335024 combine_payload_bytes=323895296 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=324230320 ingress_bytes=393101312 link_bytes=393101312 token_bytes=648125616 weight_movement_bytes=69206016 total_nvlink_bytes=717331632 weight_to_token_ratio=0.106779 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=45440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.479208590e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=205070336 dispatch_meta_bytes=245984 combine_payload_bytes=205070336 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=205316320 ingress_bytes=274276352 link_bytes=274276352 token_bytes=410386656 weight_movement_bytes=69206016 total_nvlink_bytes=479592672 weight_to_token_ratio=0.168636 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=23168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.032079832e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95612 payload_rows_remote=79930 dup_rows=2692 dup_rows_remote=2692 dedup_ratio=0.027384 router_tokens_per_expert=[7841, 0, 0, 0, 0, 0, 0, 2959, 407, 0, 4698, 0, 0, 0, 0, 78, 0, 0, 0, 170, 0, 78, 0, 7727, 0, 0, 0, 0, 0, 0, 0, 0, 474, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7418, 0, 0, 162, 0, 0, 0, 0, 261, 0, 7843, 0, 0, 0, 0, 0, 213, 148, 414, 0, 69] payload_rows_per_dst=[5918, 10356, 15610, 15682, 948, 15028, 15686, 16384] dup_rows_per_dst=[0, 10, 340, 0, 0, 132, 522, 1688] combine_rows_per_src=[5918, 10356, 15610, 15682, 948, 15028, 15686, 16384] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95458 payload_rows_remote=80440 dup_rows=2846 dup_rows_remote=2726 dedup_ratio=0.028951 router_tokens_per_expert=[7698, 0, 0, 0, 0, 0, 0, 2740, 419, 0, 4984, 0, 0, 0, 0, 161, 0, 0, 0, 201, 0, 161, 0, 7669, 0, 0, 0, 0, 0, 0, 0, 0, 558, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7406, 0, 0, 163, 0, 0, 0, 0, 247, 0, 7598, 0, 0, 0, 0, 0, 296, 177, 393, 0, 89] payload_rows_per_dst=[5480, 11074, 3816, 15396, 13094, 15018, 15196, 16384] dup_rows_per_dst=[0, 54, 94, 0, 174, 120, 494, 1910] combine_rows_per_src=[5480, 11074, 3816, 15396, 13094, 15018, 15196, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=66164 payload_rows_remote=50066 dup_rows=32140 dup_rows_remote=11430 dedup_ratio=0.326945 router_tokens_per_expert=[7737, 5, 0, 0, 0, 0, 4, 2939, 378, 8, 4673, 0, 0, 0, 0, 139, 0, 2, 1, 169, 14, 140, 0, 7639, 0, 0, 0, 0, 0, 1, 0, 3, 563, 4, 0, 0, 0, 0, 1, 8183, 0, 9, 0, 7409, 4, 7, 195, 0, 0, 0, 0, 250, 4, 7719, 0, 4, 0, 0, 0, 262, 161, 434, 0, 91] payload_rows_per_dst=[16098, 15420, 15634, 8, 16372, 408, 510, 1714] dup_rows_per_dst=[20710, 7988, 2102, 0, 1130, 22, 6, 182] combine_rows_per_src=[16098, 15420, 15634, 8, 16372, 408, 510, 1714] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69950 payload_rows_remote=59330 dup_rows=28354 dup_rows_remote=28320 dedup_ratio=0.288432 router_tokens_per_expert=[7829, 0, 0, 0, 0, 0, 0, 2946, 438, 0, 4808, 0, 0, 0, 0, 81, 0, 0, 0, 167, 1, 81, 0, 7754, 0, 0, 0, 0, 0, 0, 0, 0, 483, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7329, 0, 0, 207, 0, 0, 0, 0, 239, 0, 7736, 0, 0, 0, 0, 0, 218, 147, 422, 0, 74] payload_rows_per_dst=[13018, 10620, 16118, 4850, 16384, 5328, 2058, 1574] dup_rows_per_dst=[14118, 34, 11412, 0, 966, 1618, 58, 148] combine_rows_per_src=[13018, 10620, 16118, 4850, 16384, 5328, 2058, 1574] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95166 payload_rows_remote=78782 dup_rows=3138 dup_rows_remote=1118 dedup_ratio=0.031921 router_tokens_per_expert=[7725, 0, 0, 0, 0, 0, 0, 2914, 345, 0, 4800, 0, 0, 0, 0, 173, 0, 0, 0, 160, 10, 174, 0, 7715, 0, 0, 0, 0, 0, 0, 0, 0, 593, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7291, 0, 0, 200, 0, 0, 0, 0, 254, 0, 7596, 0, 0, 0, 0, 0, 330, 152, 433, 0, 95] payload_rows_per_dst=[5828, 10598, 688, 15450, 16188, 14838, 15192, 16384] dup_rows_per_dst=[0, 38, 0, 0, 428, 144, 508, 2020] combine_rows_per_src=[5828, 10598, 688, 15450, 16188, 14838, 15192, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=329121792 dispatch_meta_bytes=329168 combine_payload_bytes=329121792 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=329450960 ingress_bytes=398327808 link_bytes=398327808 token_bytes=658572752 weight_movement_bytes=69206016 total_nvlink_bytes=727778768 weight_to_token_ratio=0.105085 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=47232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.498875473e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94880 payload_rows_remote=81608 dup_rows=3424 dup_rows_remote=3110 dedup_ratio=0.034831 router_tokens_per_expert=[0, 0, 8060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 0, 0, 0, 0, 346, 6, 0, 0, 0, 53, 0, 346, 0, 0, 0, 0, 0, 0, 6447, 8139, 0, 0, 0, 0, 0, 65, 0, 69, 47, 0, 0, 0, 7971, 8192, 0, 0, 540, 0, 0, 0, 0, 69, 0, 0, 0, 679, 0, 0, 0, 0, 0] payload_rows_per_dst=[16120, 16246, 16384, 13272, 130, 232, 16218, 16278] dup_rows_per_dst=[0, 0, 810, 314, 0, 0, 942, 1358] combine_rows_per_src=[16120, 16246, 16384, 13272, 130, 232, 16218, 16278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=327393280 dispatch_meta_bytes=330488 combine_payload_bytes=327393280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=327723768 ingress_bytes=361996288 link_bytes=361996288 token_bytes=655117048 weight_movement_bytes=34603008 total_nvlink_bytes=689720056 weight_to_token_ratio=0.052820 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.362162889e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=329482240 dispatch_meta_bytes=332664 combine_payload_bytes=329482240 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=329814904 ingress_bytes=433291264 link_bytes=433291264 token_bytes=659297144 weight_movement_bytes=103809024 total_nvlink_bytes=763106168 weight_to_token_ratio=0.157454 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=46080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.630440144e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96718 payload_rows_remote=80482 dup_rows=1586 dup_rows_remote=1586 dedup_ratio=0.016134 router_tokens_per_expert=[0, 0, 8056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8118, 0, 0, 0, 0, 350, 0, 0, 0, 0, 60, 0, 310, 0, 0, 0, 0, 0, 0, 6147, 8132, 0, 0, 0, 0, 0, 92, 0, 74, 60, 0, 0, 0, 7964, 8192, 0, 0, 566, 0, 0, 0, 0, 74, 0, 0, 0, 957, 0, 0, 0, 0, 0] payload_rows_per_dst=[16112, 16236, 16384, 12600, 16264, 16076, 1132, 1914] dup_rows_per_dst=[0, 0, 820, 314, 184, 120, 148, 0] combine_rows_per_src=[16112, 16236, 16384, 12600, 16264, 16076, 1132, 1914] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=205070336 dispatch_meta_bytes=245984 combine_payload_bytes=205070336 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=205316320 ingress_bytes=274276352 link_bytes=274276352 token_bytes=410386656 weight_movement_bytes=69206016 total_nvlink_bytes=479592672 weight_to_token_ratio=0.168636 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=23168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.032079832e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=322691072 dispatch_meta_bytes=319600 combine_payload_bytes=322691072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=323010672 ingress_bytes=357294080 link_bytes=357294080 token_bytes=645701744 weight_movement_bytes=34603008 total_nvlink_bytes=680304752 weight_to_token_ratio=0.053590 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=41984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.344468859e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94496 payload_rows_remote=92100 dup_rows=3808 dup_rows_remote=3786 dedup_ratio=0.038737 router_tokens_per_expert=[0, 0, 8059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 0, 1, 0, 0, 345, 0, 0, 0, 0, 63, 0, 315, 0, 0, 0, 0, 0, 0, 6247, 8129, 0, 0, 0, 0, 0, 69, 0, 75, 63, 0, 0, 0, 7976, 8192, 0, 0, 536, 0, 0, 0, 0, 75, 0, 0, 0, 890, 0, 0, 0, 0, 0] payload_rows_per_dst=[16118, 13954, 16384, 12858, 2396, 276, 16252, 16258] dup_rows_per_dst=[0, 2, 816, 266, 22, 0, 922, 1780] combine_rows_per_src=[16118, 13954, 16384, 12858, 2396, 276, 16252, 16258] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95970 payload_rows_remote=79586 dup_rows=2334 dup_rows_remote=1428 dedup_ratio=0.023743 router_tokens_per_expert=[0, 0, 7986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8110, 0, 0, 0, 0, 363, 0, 0, 0, 0, 90, 0, 418, 0, 0, 0, 0, 0, 0, 6073, 8102, 0, 0, 0, 0, 0, 44, 0, 82, 90, 0, 0, 0, 7952, 8192, 0, 0, 507, 0, 0, 0, 0, 82, 0, 0, 0, 1061, 0, 0, 0, 0, 0] payload_rows_per_dst=[15972, 16220, 16384, 12612, 13770, 6908, 9872, 4232] dup_rows_per_dst=[0, 0, 906, 370, 74, 94, 552, 338] combine_rows_per_src=[15972, 16220, 16384, 12612, 13770, 6908, 9872, 4232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=334266368 dispatch_meta_bytes=338872 combine_payload_bytes=334266368 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=334605240 ingress_bytes=368869376 link_bytes=368869376 token_bytes=668871608 weight_movement_bytes=34603008 total_nvlink_bytes=703474616 weight_to_token_ratio=0.051733 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=46336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388025765e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=329654272 dispatch_meta_bytes=328272 combine_payload_bytes=329654272 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=329982544 ingress_bytes=398860288 link_bytes=398860288 token_bytes=659636816 weight_movement_bytes=69206016 total_nvlink_bytes=728842832 weight_to_token_ratio=0.104915 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=39424 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.500879153e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92972 payload_rows_remote=76686 dup_rows=5332 dup_rows_remote=4450 dedup_ratio=0.054240 router_tokens_per_expert=[0, 0, 8040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8127, 0, 0, 0, 0, 336, 0, 0, 0, 0, 60, 0, 294, 0, 0, 0, 0, 0, 0, 6475, 8132, 0, 0, 0, 0, 0, 75, 0, 65, 60, 0, 0, 0, 8015, 8192, 0, 0, 504, 0, 0, 0, 0, 65, 0, 0, 0, 712, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 13734, 792, 13258, 16254, 16384, 16286, 16264] dup_rows_per_dst=[0, 0, 0, 280, 150, 2596, 882, 1424] combine_rows_per_src=[0, 13734, 792, 13258, 16254, 16384, 16286, 16264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=243015680 dispatch_meta_bytes=350600 combine_payload_bytes=243015680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=243366280 ingress_bytes=277618688 link_bytes=277618688 token_bytes=486381960 weight_movement_bytes=34603008 total_nvlink_bytes=520984968 weight_to_token_ratio=0.071144 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=38784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.044656773e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96142 payload_rows_remote=80454 dup_rows=2162 dup_rows_remote=1772 dedup_ratio=0.021993 router_tokens_per_expert=[7751, 0, 0, 0, 0, 0, 0, 3130, 345, 0, 4614, 0, 0, 0, 0, 72, 0, 0, 0, 195, 0, 72, 0, 7772, 0, 0, 0, 0, 0, 0, 0, 0, 460, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7408, 0, 0, 165, 0, 0, 0, 0, 264, 0, 7793, 0, 0, 0, 0, 0, 199, 195, 443, 0, 82] payload_rows_per_dst=[6260, 10052, 15688, 15502, 14518, 15058, 15586, 3478] dup_rows_per_dst=[0, 10, 390, 0, 810, 88, 528, 336] combine_rows_per_src=[6260, 10052, 15688, 15502, 14518, 15058, 15586, 3478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=322691072 dispatch_meta_bytes=319600 combine_payload_bytes=322691072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=323010672 ingress_bytes=357294080 link_bytes=357294080 token_bytes=645701744 weight_movement_bytes=34603008 total_nvlink_bytes=680304752 weight_to_token_ratio=0.053590 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=41984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.344468859e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89368 payload_rows_remote=73206 dup_rows=8936 dup_rows_remote=1488 dedup_ratio=0.090902 router_tokens_per_expert=[0, 0, 8050, 0, 1, 0, 0, 1, 9, 0, 0, 0, 8088, 0, 9, 0, 0, 345, 1, 0, 0, 0, 54, 0, 411, 3, 0, 0, 0, 0, 0, 6354, 8121, 0, 0, 0, 8, 0, 69, 0, 96, 54, 0, 1, 8, 7859, 8183, 0, 9, 536, 0, 0, 1, 0, 91, 0, 0, 9, 773, 8, 0, 0, 0, 0] payload_rows_per_dst=[16162, 16196, 16366, 5814, 16258, 15918, 1090, 1564] dup_rows_per_dst=[7448, 16, 800, 216, 138, 118, 184, 16] combine_rows_per_src=[16162, 16196, 16366, 5814, 16258, 15918, 1090, 1564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=377241600 dispatch_meta_bytes=383544 combine_payload_bytes=377241600 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=377625144 ingress_bytes=446447616 link_bytes=446447616 token_bytes=754866744 weight_movement_bytes=69206016 total_nvlink_bytes=824072760 weight_to_token_ratio=0.091680 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=25472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.679946437e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=325984256 dispatch_meta_bytes=324056 combine_payload_bytes=325984256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=326308312 ingress_bytes=360587264 link_bytes=360587264 token_bytes=652292568 weight_movement_bytes=34603008 total_nvlink_bytes=686895576 weight_to_token_ratio=0.053048 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=45952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.356860845e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=314105856 dispatch_meta_bytes=324544 combine_payload_bytes=314105856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=314430400 ingress_bytes=348708864 link_bytes=348708864 token_bytes=628536256 weight_movement_bytes=34603008 total_nvlink_bytes=663139264 weight_to_token_ratio=0.055053 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=44288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.312163382e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=329539584 dispatch_meta_bytes=328904 combine_payload_bytes=329539584 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=329868488 ingress_bytes=398745600 link_bytes=398745600 token_bytes=659408072 weight_movement_bytes=69206016 total_nvlink_bytes=728614088 weight_to_token_ratio=0.104952 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=40320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.500447591e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=299851776 dispatch_meta_bytes=298776 combine_payload_bytes=299851776 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=300150552 ingress_bytes=403660800 link_bytes=403660800 token_bytes=600002328 weight_movement_bytes=103809024 total_nvlink_bytes=703811352 weight_to_token_ratio=0.173014 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=45440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.518943093e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94880 payload_rows_remote=81608 dup_rows=3424 dup_rows_remote=3110 dedup_ratio=0.034831 router_tokens_per_expert=[0, 0, 8060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 0, 0, 0, 0, 346, 6, 0, 0, 0, 53, 0, 346, 0, 0, 0, 0, 0, 0, 6447, 8139, 0, 0, 0, 0, 0, 65, 0, 69, 47, 0, 0, 0, 7971, 8192, 0, 0, 540, 0, 0, 0, 0, 69, 0, 0, 0, 679, 0, 0, 0, 0, 0] payload_rows_per_dst=[16120, 16246, 16384, 13272, 130, 232, 16218, 16278] dup_rows_per_dst=[0, 0, 810, 314, 0, 0, 942, 1358] combine_rows_per_src=[16120, 16246, 16384, 13272, 130, 232, 16218, 16278] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93888 payload_rows_remote=87266 dup_rows=4416 dup_rows_remote=4304 dedup_ratio=0.044922 router_tokens_per_expert=[0, 0, 8035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8103, 0, 1, 0, 0, 303, 0, 0, 0, 0, 66, 0, 436, 0, 0, 0, 0, 0, 0, 6029, 8126, 0, 0, 0, 0, 0, 60, 0, 89, 66, 0, 0, 0, 7845, 8192, 0, 0, 544, 0, 0, 0, 0, 90, 0, 0, 0, 1167, 0, 0, 0, 0, 0] payload_rows_per_dst=[10372, 5700, 10252, 12434, 16206, 6622, 16050, 16252] dup_rows_per_dst=[0, 0, 446, 496, 120, 112, 908, 2334] combine_rows_per_src=[10372, 5700, 10252, 12434, 16206, 6622, 16050, 16252] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89368 payload_rows_remote=73206 dup_rows=8936 dup_rows_remote=1488 dedup_ratio=0.090902 router_tokens_per_expert=[0, 0, 8050, 0, 1, 0, 0, 1, 9, 0, 0, 0, 8088, 0, 9, 0, 0, 345, 1, 0, 0, 0, 54, 0, 411, 3, 0, 0, 0, 0, 0, 6354, 8121, 0, 0, 0, 8, 0, 69, 0, 96, 54, 0, 1, 8, 7859, 8183, 0, 9, 536, 0, 0, 1, 0, 91, 0, 0, 9, 773, 8, 0, 0, 0, 0] payload_rows_per_dst=[16162, 16196, 16366, 5814, 16258, 15918, 1090, 1564] dup_rows_per_dst=[7448, 16, 800, 216, 138, 118, 184, 16] combine_rows_per_src=[16162, 16196, 16366, 5814, 16258, 15918, 1090, 1564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96718 payload_rows_remote=80482 dup_rows=1586 dup_rows_remote=1586 dedup_ratio=0.016134 router_tokens_per_expert=[0, 0, 8056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8118, 0, 0, 0, 0, 350, 0, 0, 0, 0, 60, 0, 310, 0, 0, 0, 0, 0, 0, 6147, 8132, 0, 0, 0, 0, 0, 92, 0, 74, 60, 0, 0, 0, 7964, 8192, 0, 0, 566, 0, 0, 0, 0, 74, 0, 0, 0, 957, 0, 0, 0, 0, 0] payload_rows_per_dst=[16112, 16236, 16384, 12600, 16264, 16076, 1132, 1914] dup_rows_per_dst=[0, 0, 820, 314, 184, 120, 148, 0] combine_rows_per_src=[16112, 16236, 16384, 12600, 16264, 16076, 1132, 1914] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78398 payload_rows_remote=62098 dup_rows=19906 dup_rows_remote=17914 dedup_ratio=0.202494 router_tokens_per_expert=[0, 0, 8093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8097, 0, 0, 0, 0, 338, 0, 0, 0, 0, 42, 0, 382, 0, 0, 0, 0, 0, 0, 6218, 8150, 0, 0, 0, 0, 0, 72, 0, 95, 44, 0, 0, 0, 7822, 8192, 0, 0, 515, 0, 0, 0, 0, 96, 0, 0, 0, 996, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 0, 760, 12738, 16194, 16384, 16022, 16300] dup_rows_per_dst=[0, 0, 0, 462, 144, 16464, 844, 1992] combine_rows_per_src=[0, 0, 760, 12738, 16194, 16384, 16022, 16300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=314105856 dispatch_meta_bytes=324544 combine_payload_bytes=314105856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=314430400 ingress_bytes=348708864 link_bytes=348708864 token_bytes=628536256 weight_movement_bytes=34603008 total_nvlink_bytes=663139264 weight_to_token_ratio=0.055053 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=44288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.312163382e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79206 payload_rows_remote=63300 dup_rows=19098 dup_rows_remote=6412 dedup_ratio=0.194275 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 329, 0, 71, 7877, 6348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7414, 0, 0, 0, 0, 0, 0, 1560, 1844, 0, 0, 0, 0, 0, 0, 0, 7144, 0, 8121, 74, 0, 0, 0, 0, 0, 269, 0, 239, 0, 0, 0, 0, 81, 0, 0, 0, 0, 313, 7467, 0, 0, 0, 1] payload_rows_per_dst=[16242, 15906, 14828, 15988, 7974, 686, 6954, 628] dup_rows_per_dst=[658, 12686, 0, 5754, 0, 0, 0, 0] combine_rows_per_src=[16242, 15906, 14828, 15988, 7974, 686, 6954, 628] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95774 payload_rows_remote=88730 dup_rows=2530 dup_rows_remote=2008 dedup_ratio=0.025736 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 325, 0, 69, 7853, 6105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7138, 0, 0, 0, 0, 0, 0, 1521, 2262, 0, 0, 0, 0, 0, 0, 0, 7216, 0, 8123, 69, 0, 0, 0, 0, 0, 252, 0, 260, 0, 0, 0, 0, 78, 0, 0, 0, 0, 330, 7523, 0, 0, 0, 28] payload_rows_per_dst=[16246, 12210, 14276, 7044, 0, 15706, 15108, 15184] dup_rows_per_dst=[650, 138, 0, 522, 0, 642, 0, 578] combine_rows_per_src=[16246, 12210, 14276, 7044, 0, 15706, 15108, 15184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=334266368 dispatch_meta_bytes=338872 combine_payload_bytes=334266368 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=334605240 ingress_bytes=368869376 link_bytes=368869376 token_bytes=668871608 weight_movement_bytes=34603008 total_nvlink_bytes=703474616 weight_to_token_ratio=0.051733 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=46336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388025765e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=357441536 dispatch_meta_bytes=366280 combine_payload_bytes=357441536 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=357807816 ingress_bytes=461250560 link_bytes=461250560 token_bytes=715249352 weight_movement_bytes=103809024 total_nvlink_bytes=819058376 weight_to_token_ratio=0.145137 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.735648724e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=299851776 dispatch_meta_bytes=298776 combine_payload_bytes=299851776 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=300150552 ingress_bytes=403660800 link_bytes=403660800 token_bytes=600002328 weight_movement_bytes=103809024 total_nvlink_bytes=703811352 weight_to_token_ratio=0.173014 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=45440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.518943093e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=254353408 dispatch_meta_bytes=320048 combine_payload_bytes=254353408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=254673456 ingress_bytes=288956416 link_bytes=288956416 token_bytes=509026864 weight_movement_bytes=34603008 total_nvlink_bytes=543629872 weight_to_token_ratio=0.067979 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=41984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.087319731e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95740 payload_rows_remote=93782 dup_rows=2564 dup_rows_remote=2564 dedup_ratio=0.026082 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 350, 0, 75, 7878, 6136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7233, 0, 0, 0, 0, 0, 0, 1452, 2200, 0, 0, 0, 0, 0, 0, 0, 7232, 0, 8117, 75, 0, 0, 0, 0, 0, 291, 0, 245, 0, 0, 0, 0, 64, 0, 0, 0, 0, 321, 7471, 0, 0, 0, 12] payload_rows_per_dst=[16234, 10340, 14466, 6812, 1958, 15756, 15082, 15092] dup_rows_per_dst=[700, 124, 0, 492, 0, 732, 0, 516] combine_rows_per_src=[16234, 10340, 14466, 6812, 1958, 15756, 15082, 15092] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91178 payload_rows_remote=76892 dup_rows=7126 dup_rows_remote=7126 dedup_ratio=0.072489 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 322, 0, 81, 7816, 6234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7143, 0, 0, 0, 0, 0, 0, 1616, 2116, 0, 0, 0, 0, 0, 0, 0, 7218, 0, 8111, 82, 0, 0, 0, 0, 0, 267, 0, 263, 0, 0, 0, 0, 72, 0, 0, 0, 0, 346, 7464, 0, 0, 0, 1] payload_rows_per_dst=[16222, 12812, 14286, 12668, 0, 14170, 15106, 5914] dup_rows_per_dst=[644, 1366, 0, 4294, 0, 612, 0, 210] combine_rows_per_src=[16222, 12812, 14286, 12668, 0, 14170, 15106, 5914] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=259276800 dispatch_meta_bytes=278848 combine_payload_bytes=259276800 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=259555648 ingress_bytes=363085824 link_bytes=363085824 token_bytes=518832448 weight_movement_bytes=103809024 total_nvlink_bytes=622641472 weight_to_token_ratio=0.200082 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=30080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.366262725e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=363438080 dispatch_meta_bytes=362952 combine_payload_bytes=363438080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=363801032 ingress_bytes=398041088 link_bytes=398041088 token_bytes=727239112 weight_movement_bytes=34603008 total_nvlink_bytes=761842120 weight_to_token_ratio=0.047581 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.497796569e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=329654272 dispatch_meta_bytes=328272 combine_payload_bytes=329654272 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=329982544 ingress_bytes=398860288 link_bytes=398860288 token_bytes=659636816 weight_movement_bytes=69206016 total_nvlink_bytes=728842832 weight_to_token_ratio=0.104915 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=39424 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.500879153e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84090 payload_rows_remote=68920 dup_rows=14214 dup_rows_remote=14214 dedup_ratio=0.144592 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 304, 0, 63, 7846, 6346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7081, 0, 0, 0, 0, 0, 0, 1473, 2156, 0, 0, 0, 0, 0, 0, 0, 7283, 0, 8129, 65, 0, 0, 0, 0, 0, 249, 0, 254, 0, 0, 0, 0, 48, 0, 0, 0, 0, 319, 7535, 0, 0, 0, 1] payload_rows_per_dst=[608, 126, 14162, 6748, 16384, 15692, 15170, 15200] dup_rows_per_dst=[0, 0, 0, 510, 12566, 628, 0, 510] combine_rows_per_src=[608, 126, 14162, 6748, 16384, 15692, 15170, 15200] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=384131072 dispatch_meta_bytes=385384 combine_payload_bytes=384131072 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=384516456 ingress_bytes=487940096 link_bytes=487940096 token_bytes=768647528 weight_movement_bytes=103809024 total_nvlink_bytes=872456552 weight_to_token_ratio=0.135054 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.836079299e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=314949632 dispatch_meta_bytes=336072 combine_payload_bytes=314949632 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=315285704 ingress_bytes=384155648 link_bytes=384155648 token_bytes=630235336 weight_movement_bytes=69206016 total_nvlink_bytes=699441352 weight_to_token_ratio=0.109810 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.445546776e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12767808408 prefetch_bytes=761266176 reduce_bytes=553648128 weight_movement_bytes=1314914304 egress_bytes=6387698584 ingress_bytes=7695024128 link_bytes=7695024128 total_nvlink_bytes=14082722712 weight_to_token_ratio=0.102987 prefetch_share_of_ingress=0.098930 dedup_ratio=0.180763 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=16 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030978 gemm_inner_pad_upper_bound=21844 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.258945926e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14477019568 prefetch_bytes=692060160 reduce_bytes=1003487232 weight_movement_bytes=1695547392 egress_bytes=7242459568 ingress_bytes=8930107392 link_bytes=8930107392 total_nvlink_bytes=16172566960 weight_to_token_ratio=0.117120 prefetch_share_of_ingress=0.077497 dedup_ratio=0.103558 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=29 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030870 gemm_inner_pad_upper_bound=21590 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.461011964e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92830 payload_rows_remote=77112 dup_rows=5474 dup_rows_remote=4732 dedup_ratio=0.055684 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 298, 0, 89, 7859, 6111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6995, 0, 0, 0, 0, 0, 0, 1631, 2314, 0, 0, 0, 0, 0, 0, 0, 7111, 0, 8107, 89, 0, 0, 0, 0, 0, 282, 0, 246, 0, 0, 0, 0, 161, 0, 0, 0, 0, 337, 7502, 0, 0, 0, 20] payload_rows_per_dst=[12214, 178, 13990, 7362, 13148, 15718, 15036, 15184] dup_rows_per_dst=[456, 0, 0, 528, 3214, 742, 0, 534] combine_rows_per_src=[12214, 178, 13990, 7362, 13148, 15718, 15036, 15184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95970 payload_rows_remote=79586 dup_rows=2334 dup_rows_remote=1428 dedup_ratio=0.023743 router_tokens_per_expert=[0, 0, 7986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8110, 0, 0, 0, 0, 363, 0, 0, 0, 0, 90, 0, 418, 0, 0, 0, 0, 0, 0, 6073, 8102, 0, 0, 0, 0, 0, 44, 0, 82, 90, 0, 0, 0, 7952, 8192, 0, 0, 507, 0, 0, 0, 0, 82, 0, 0, 0, 1061, 0, 0, 0, 0, 0] payload_rows_per_dst=[15972, 16220, 16384, 12612, 13770, 6908, 9872, 4232] dup_rows_per_dst=[0, 0, 906, 370, 74, 94, 552, 338] combine_rows_per_src=[15972, 16220, 16384, 12612, 13770, 6908, 9872, 4232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=254353408 dispatch_meta_bytes=320048 combine_payload_bytes=254353408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=254673456 ingress_bytes=288956416 link_bytes=288956416 token_bytes=509026864 weight_movement_bytes=34603008 total_nvlink_bytes=543629872 weight_to_token_ratio=0.067979 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=41984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.087319731e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74760 payload_rows_remote=58558 dup_rows=23544 dup_rows_remote=22880 dedup_ratio=0.239502 router_tokens_per_expert=[0, 0, 5, 6, 9, 5, 316, 0, 91, 7863, 6187, 0, 0, 0, 1, 0, 0, 1, 0, 0, 8, 7011, 5, 0, 0, 3, 0, 0, 1548, 2236, 6, 0, 1, 9, 0, 0, 8, 7141, 0, 8092, 91, 0, 0, 0, 1, 0, 293, 0, 250, 0, 0, 0, 0, 140, 0, 0, 0, 0, 341, 7464, 0, 2, 0, 18] payload_rows_per_dst=[16202, 15868, 10092, 16012, 14316, 768, 780, 722] dup_rows_per_dst=[664, 16364, 10, 6502, 2, 2, 0, 0] combine_rows_per_src=[16202, 15868, 10092, 16012, 14316, 768, 780, 722] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72894 payload_rows_remote=56708 dup_rows=25410 dup_rows_remote=13564 dedup_ratio=0.258484 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 282, 0, 93, 7882, 6272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7314, 0, 0, 0, 0, 0, 0, 1476, 2058, 0, 0, 0, 0, 0, 0, 0, 7107, 0, 8099, 96, 0, 0, 0, 0, 0, 247, 0, 253, 0, 0, 0, 0, 174, 0, 0, 0, 0, 349, 7450, 0, 0, 0, 0] payload_rows_per_dst=[564, 186, 2194, 6548, 16384, 15764, 15068, 16186] dup_rows_per_dst=[0, 0, 0, 520, 12358, 686, 0, 11846] combine_rows_per_src=[564, 186, 2194, 6548, 16384, 15764, 15068, 16186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14785478488 prefetch_bytes=726663168 reduce_bytes=865075200 weight_movement_bytes=1591738368 egress_bytes=7396769624 ingress_bytes=8980447232 link_bytes=8980447232 total_nvlink_bytes=16377216856 weight_to_token_ratio=0.107656 prefetch_share_of_ingress=0.080916 dedup_ratio=0.104343 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=25 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029565 gemm_inner_pad_upper_bound=22479 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.469247824e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=384131072 dispatch_meta_bytes=385384 combine_payload_bytes=384131072 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=384516456 ingress_bytes=487940096 link_bytes=487940096 token_bytes=768647528 weight_movement_bytes=103809024 total_nvlink_bytes=872456552 weight_to_token_ratio=0.135054 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.836079299e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=282296320 dispatch_meta_bytes=332536 combine_payload_bytes=282296320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282628856 ingress_bytes=316899328 link_bytes=316899328 token_bytes=564925176 weight_movement_bytes=34603008 total_nvlink_bytes=599528184 weight_to_token_ratio=0.061252 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.037500 gemm_active_groups=4 gemm_peak_group_rows=46336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.192466659e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14100577032 prefetch_bytes=657457152 reduce_bytes=692060160 weight_movement_bytes=1349517312 egress_bytes=7054162696 ingress_bytes=8395931648 link_bytes=8395931648 total_nvlink_bytes=15450094344 weight_to_token_ratio=0.095707 prefetch_share_of_ingress=0.078307 dedup_ratio=0.113706 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=20 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029891 gemm_inner_pad_upper_bound=21336 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.373618038e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=325984256 dispatch_meta_bytes=324056 combine_payload_bytes=325984256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=326308312 ingress_bytes=360587264 link_bytes=360587264 token_bytes=652292568 weight_movement_bytes=34603008 total_nvlink_bytes=686895576 weight_to_token_ratio=0.053048 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=45952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.356860845e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=239853568 dispatch_meta_bytes=325752 combine_payload_bytes=239853568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=240179320 ingress_bytes=274456576 link_bytes=274456576 token_bytes=480032888 weight_movement_bytes=34603008 total_nvlink_bytes=514635896 weight_to_token_ratio=0.072085 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=46720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.032758000e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95774 payload_rows_remote=88730 dup_rows=2530 dup_rows_remote=2008 dedup_ratio=0.025736 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 325, 0, 69, 7853, 6105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7138, 0, 0, 0, 0, 0, 0, 1521, 2262, 0, 0, 0, 0, 0, 0, 0, 7216, 0, 8123, 69, 0, 0, 0, 0, 0, 252, 0, 260, 0, 0, 0, 0, 78, 0, 0, 0, 0, 330, 7523, 0, 0, 0, 28] payload_rows_per_dst=[16246, 12210, 14276, 7044, 0, 15706, 15108, 15184] dup_rows_per_dst=[650, 138, 0, 522, 0, 642, 0, 578] combine_rows_per_src=[16246, 12210, 14276, 7044, 0, 15706, 15108, 15184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14785478488 prefetch_bytes=726663168 reduce_bytes=865075200 weight_movement_bytes=1591738368 egress_bytes=7396769624 ingress_bytes=8980447232 link_bytes=8980447232 total_nvlink_bytes=16377216856 weight_to_token_ratio=0.107656 prefetch_share_of_ingress=0.080916 dedup_ratio=0.104343 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=25 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029565 gemm_inner_pad_upper_bound=22479 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.469247824e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92830 payload_rows_remote=77112 dup_rows=5474 dup_rows_remote=4732 dedup_ratio=0.055684 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 298, 0, 89, 7859, 6111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6995, 0, 0, 0, 0, 0, 0, 1631, 2314, 0, 0, 0, 0, 0, 0, 0, 7111, 0, 8107, 89, 0, 0, 0, 0, 0, 282, 0, 246, 0, 0, 0, 0, 161, 0, 0, 0, 0, 337, 7502, 0, 0, 0, 20] payload_rows_per_dst=[12214, 178, 13990, 7362, 13148, 15718, 15036, 15184] dup_rows_per_dst=[456, 0, 0, 528, 3214, 742, 0, 534] combine_rows_per_src=[12214, 178, 13990, 7362, 13148, 15718, 15036, 15184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13108263848 prefetch_bytes=692060160 reduce_bytes=622854144 weight_movement_bytes=1314914304 egress_bytes=6557924264 ingress_bytes=7865253888 link_bytes=7865253888 total_nvlink_bytes=14423178152 weight_to_token_ratio=0.100312 prefetch_share_of_ingress=0.087990 dedup_ratio=0.153995 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=18 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030435 gemm_inner_pad_upper_bound=20701 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.286796399e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79206 payload_rows_remote=63300 dup_rows=19098 dup_rows_remote=6412 dedup_ratio=0.194275 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 329, 0, 71, 7877, 6348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7414, 0, 0, 0, 0, 0, 0, 1560, 1844, 0, 0, 0, 0, 0, 0, 0, 7144, 0, 8121, 74, 0, 0, 0, 0, 0, 269, 0, 239, 0, 0, 0, 0, 81, 0, 0, 0, 0, 313, 7467, 0, 0, 0, 1] payload_rows_per_dst=[16242, 15906, 14828, 15988, 7974, 686, 6954, 628] dup_rows_per_dst=[658, 12686, 0, 5754, 0, 0, 0, 0] combine_rows_per_src=[16242, 15906, 14828, 15988, 7974, 686, 6954, 628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=11788090256 prefetch_bytes=692060160 reduce_bytes=692060160 weight_movement_bytes=1384120320 egress_bytes=5897599888 ingress_bytes=7274610688 link_bytes=7274610688 total_nvlink_bytes=13172210576 weight_to_token_ratio=0.117417 prefetch_share_of_ingress=0.095134 dedup_ratio=0.218534 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=20 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030543 gemm_inner_pad_upper_bound=21082 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.190164103e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72894 payload_rows_remote=56708 dup_rows=25410 dup_rows_remote=13564 dedup_ratio=0.258484 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 282, 0, 93, 7882, 6272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7314, 0, 0, 0, 0, 0, 0, 1476, 2058, 0, 0, 0, 0, 0, 0, 0, 7107, 0, 8099, 96, 0, 0, 0, 0, 0, 247, 0, 253, 0, 0, 0, 0, 174, 0, 0, 0, 0, 349, 7450, 0, 0, 0, 0] payload_rows_per_dst=[564, 186, 2194, 6548, 16384, 15764, 15068, 16186] dup_rows_per_dst=[0, 0, 0, 520, 12358, 686, 0, 11846] combine_rows_per_src=[564, 186, 2194, 6548, 16384, 15764, 15068, 16186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=363438080 dispatch_meta_bytes=362952 combine_payload_bytes=363438080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=363801032 ingress_bytes=398041088 link_bytes=398041088 token_bytes=727239112 weight_movement_bytes=34603008 total_nvlink_bytes=761842120 weight_to_token_ratio=0.047581 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.497796569e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=315850752 dispatch_meta_bytes=327376 combine_payload_bytes=315850752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=316178128 ingress_bytes=350453760 link_bytes=350453760 token_bytes=632028880 weight_movement_bytes=34603008 total_nvlink_bytes=666631888 weight_to_token_ratio=0.054749 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=46464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.318729285e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=239853568 dispatch_meta_bytes=325752 combine_payload_bytes=239853568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=240179320 ingress_bytes=274456576 link_bytes=274456576 token_bytes=480032888 weight_movement_bytes=34603008 total_nvlink_bytes=514635896 weight_to_token_ratio=0.072085 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=46720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.032758000e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=259276800 dispatch_meta_bytes=278848 combine_payload_bytes=259276800 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=259555648 ingress_bytes=363085824 link_bytes=363085824 token_bytes=518832448 weight_movement_bytes=103809024 total_nvlink_bytes=622641472 weight_to_token_ratio=0.200082 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=30080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.366262725e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91178 payload_rows_remote=76892 dup_rows=7126 dup_rows_remote=7126 dedup_ratio=0.072489 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 322, 0, 81, 7816, 6234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7143, 0, 0, 0, 0, 0, 0, 1616, 2116, 0, 0, 0, 0, 0, 0, 0, 7218, 0, 8111, 82, 0, 0, 0, 0, 0, 267, 0, 263, 0, 0, 0, 0, 72, 0, 0, 0, 0, 346, 7464, 0, 0, 0, 1] payload_rows_per_dst=[16222, 12812, 14286, 12668, 0, 14170, 15106, 5914] dup_rows_per_dst=[644, 1366, 0, 4294, 0, 612, 0, 210] combine_rows_per_src=[16222, 12812, 14286, 12668, 0, 14170, 15106, 5914] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=232275968 dispatch_meta_bytes=281088 combine_payload_bytes=232275968 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=232557056 ingress_bytes=301481984 link_bytes=301481984 token_bytes=464833024 weight_movement_bytes=69206016 total_nvlink_bytes=534039040 weight_to_token_ratio=0.148884 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=40320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134452435e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13108263848 prefetch_bytes=692060160 reduce_bytes=622854144 weight_movement_bytes=1314914304 egress_bytes=6557924264 ingress_bytes=7865253888 link_bytes=7865253888 total_nvlink_bytes=14423178152 weight_to_token_ratio=0.100312 prefetch_share_of_ingress=0.087990 dedup_ratio=0.153995 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=18 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030435 gemm_inner_pad_upper_bound=20701 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.286796399e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14113173328 prefetch_bytes=726663168 reduce_bytes=726663168 weight_movement_bytes=1453326336 egress_bytes=7060492112 ingress_bytes=8506007552 link_bytes=8506007552 total_nvlink_bytes=15566499664 weight_to_token_ratio=0.102977 prefetch_share_of_ingress=0.085429 dedup_ratio=0.120512 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=21 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029130 gemm_inner_pad_upper_bound=22606 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.391627027e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=11788090256 prefetch_bytes=692060160 reduce_bytes=692060160 weight_movement_bytes=1384120320 egress_bytes=5897599888 ingress_bytes=7274610688 link_bytes=7274610688 total_nvlink_bytes=13172210576 weight_to_token_ratio=0.117417 prefetch_share_of_ingress=0.095134 dedup_ratio=0.218534 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=20 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030543 gemm_inner_pad_upper_bound=21082 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.190164103e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14477019568 prefetch_bytes=692060160 reduce_bytes=1003487232 weight_movement_bytes=1695547392 egress_bytes=7242459568 ingress_bytes=8930107392 link_bytes=8930107392 total_nvlink_bytes=16172566960 weight_to_token_ratio=0.117120 prefetch_share_of_ingress=0.077497 dedup_ratio=0.103558 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=29 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030870 gemm_inner_pad_upper_bound=21590 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.461011964e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12767808408 prefetch_bytes=761266176 reduce_bytes=553648128 weight_movement_bytes=1314914304 egress_bytes=6387698584 ingress_bytes=7695024128 link_bytes=7695024128 total_nvlink_bytes=14082722712 weight_to_token_ratio=0.102987 prefetch_share_of_ingress=0.098930 dedup_ratio=0.180763 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=16 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030978 gemm_inner_pad_upper_bound=21844 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.258945926e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12035948800 prefetch_bytes=692060160 reduce_bytes=484442112 weight_movement_bytes=1176502272 egress_bytes=6021587200 ingress_bytes=7190863872 link_bytes=7190863872 total_nvlink_bytes=13212451072 weight_to_token_ratio=0.097749 prefetch_share_of_ingress=0.096242 dedup_ratio=0.205356 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=14 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030435 gemm_inner_pad_upper_bound=22098 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.176462689e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none - [2026-08-26 15:09:12] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 791.8 | throughput per GPU (TFLOP/s/GPU): 161.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.787875E+00 | loss scale: 1.0 | grad norm: 4.876 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75832 payload_rows_remote=67308 dup_rows=22472 dup_rows_remote=20216 dedup_ratio=0.228597 router_tokens_per_expert=[1083, 359, 855, 1228, 621, 914, 826, 883, 1387, 295, 954, 312, 272, 1507, 909, 1038, 744, 1048, 851, 666, 570, 729, 352, 1421, 485, 706, 766, 347, 807, 132, 814, 533, 1066, 784, 1315, 871, 452, 199, 555, 864, 518, 642, 654, 686, 1080, 622, 670, 518, 707, 578, 799, 430, 984, 1425, 810, 679, 1274, 358, 1421, 616, 212, 815, 1175, 959] payload_rows_per_dst=[8272, 9360, 10226, 9628, 9158, 8524, 9800, 10864] dup_rows_per_dst=[2810, 3430, 2536, 2566, 3054, 2256, 3024, 2796] combine_rows_per_src=[8272, 9360, 10226, 9628, 9158, 8524, 9800, 10864] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=314949632 dispatch_meta_bytes=336072 combine_payload_bytes=314949632 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=315285704 ingress_bytes=384155648 link_bytes=384155648 token_bytes=630235336 weight_movement_bytes=69206016 total_nvlink_bytes=699441352 weight_to_token_ratio=0.109810 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.445546776e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14100577032 prefetch_bytes=657457152 reduce_bytes=692060160 weight_movement_bytes=1349517312 egress_bytes=7054162696 ingress_bytes=8395931648 link_bytes=8395931648 total_nvlink_bytes=15450094344 weight_to_token_ratio=0.095707 prefetch_share_of_ingress=0.078307 dedup_ratio=0.113706 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=20 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029891 gemm_inner_pad_upper_bound=21336 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.373618038e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none + [2026-08-26 15:57:00] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 791.5 | throughput per GPU (TFLOP/s/GPU): 161.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.787875E+00 | loss scale: 1.0 | grad norm: 4.876 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74730 payload_rows_remote=64926 dup_rows=23574 dup_rows_remote=20530 dedup_ratio=0.239807 router_tokens_per_expert=[1145, 432, 852, 1215, 575, 868, 802, 840, 1352, 435, 1055, 310, 289, 1646, 828, 1059, 733, 1108, 825, 573, 543, 645, 360, 1320, 439, 728, 840, 358, 807, 93, 877, 636, 1044, 828, 1351, 859, 457, 194, 522, 732, 599, 698, 620, 608, 1211, 634, 634, 490, 661, 586, 778, 487, 957, 1418, 733, 693, 1417, 295, 1309, 590, 159, 847, 1211, 942] payload_rows_per_dst=[9714, 7902, 9814, 9804, 9146, 9776, 9736, 8838] dup_rows_per_dst=[3472, 3026, 2400, 3044, 2828, 3830, 2890, 2084] combine_rows_per_src=[9714, 7902, 9814, 9804, 9146, 9776, 9736, 8838] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75250 payload_rows_remote=66014 dup_rows=23054 dup_rows_remote=20124 dedup_ratio=0.234517 router_tokens_per_expert=[1094, 371, 765, 1264, 657, 782, 823, 867, 1427, 441, 1047, 353, 302, 1577, 969, 1148, 697, 995, 912, 661, 581, 739, 413, 1194, 485, 618, 749, 377, 791, 118, 882, 643, 984, 890, 1198, 893, 505, 202, 558, 853, 521, 645, 546, 667, 1112, 600, 635, 529, 723, 491, 780, 546, 1011, 1371, 732, 751, 1282, 327, 1335, 599, 176, 905, 1096, 947] payload_rows_per_dst=[8072, 9924, 9760, 9730, 9236, 9022, 9910, 9596] dup_rows_per_dst=[2646, 3978, 2624, 2750, 2930, 2964, 2900, 2262] combine_rows_per_src=[8072, 9924, 9760, 9730, 9236, 9022, 9910, 9596] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74970 payload_rows_remote=66988 dup_rows=23334 dup_rows_remote=20268 dedup_ratio=0.237366 router_tokens_per_expert=[1180, 330, 832, 1125, 592, 824, 862, 798, 1430, 318, 1037, 299, 263, 1666, 1012, 1165, 720, 1094, 856, 602, 659, 769, 356, 1370, 435, 732, 887, 355, 818, 114, 778, 548, 1012, 861, 1215, 885, 358, 228, 554, 805, 547, 599, 543, 696, 1102, 629, 614, 455, 716, 587, 787, 473, 980, 1403, 862, 636, 1375, 290, 1380, 647, 215, 817, 1207, 848] payload_rows_per_dst=[9638, 7982, 10268, 9822, 8926, 9658, 9898, 8778] dup_rows_per_dst=[3448, 3066, 2584, 2844, 2910, 3472, 2990, 2020] combine_rows_per_src=[9638, 7982, 10268, 9822, 8926, 9658, 9898, 8778] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74534 payload_rows_remote=64456 dup_rows=23770 dup_rows_remote=21168 dedup_ratio=0.241801 router_tokens_per_expert=[1166, 381, 801, 1217, 581, 1000, 781, 734, 1434, 496, 949, 301, 225, 1652, 842, 1076, 793, 1178, 876, 608, 528, 551, 441, 1365, 494, 636, 774, 350, 864, 116, 854, 609, 1123, 909, 1190, 895, 456, 212, 504, 869, 650, 611, 613, 626, 1095, 661, 655, 559, 618, 573, 805, 489, 984, 1401, 675, 764, 1262, 329, 1342, 535, 161, 794, 1138, 981] payload_rows_per_dst=[9656, 7678, 10078, 9700, 9270, 9858, 9856, 8438] dup_rows_per_dst=[3666, 2968, 2602, 2998, 3046, 3766, 2762, 1962] combine_rows_per_src=[9656, 7678, 10078, 9700, 9270, 9858, 9856, 8438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75556 payload_rows_remote=66002 dup_rows=22748 dup_rows_remote=20184 dedup_ratio=0.231405 router_tokens_per_expert=[1153, 355, 816, 1253, 672, 928, 817, 788, 1356, 507, 984, 353, 257, 1466, 915, 1033, 764, 1089, 891, 650, 469, 656, 378, 1320, 549, 678, 822, 404, 804, 111, 871, 491, 1090, 801, 1332, 880, 354, 205, 569, 841, 494, 527, 558, 627, 1134, 691, 709, 567, 757, 559, 785, 474, 944, 1370, 715, 709, 1271, 434, 1430, 688, 217, 865, 1067, 888] payload_rows_per_dst=[8230, 9494, 9762, 9782, 9140, 8312, 9554, 11282] dup_rows_per_dst=[2828, 3822, 2672, 2610, 3004, 2302, 2564, 2946] combine_rows_per_src=[8230, 9494, 9762, 9782, 9140, 8312, 9554, 11282] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75472 payload_rows_remote=63572 dup_rows=22832 dup_rows_remote=18828 dedup_ratio=0.232259 router_tokens_per_expert=[1202, 381, 959, 1184, 619, 872, 778, 840, 1402, 419, 1042, 340, 298, 1488, 873, 1018, 705, 1065, 903, 595, 640, 646, 360, 1381, 439, 733, 812, 339, 804, 116, 833, 617, 1086, 914, 1184, 857, 407, 233, 479, 796, 561, 608, 556, 632, 1092, 642, 602, 628, 666, 580, 896, 551, 1003, 1320, 801, 723, 1227, 306, 1338, 624, 212, 833, 1178, 914] payload_rows_per_dst=[8414, 9484, 9070, 9522, 9658, 8366, 9058, 11900] dup_rows_per_dst=[2888, 3668, 1642, 2840, 3742, 2276, 1772, 4004] combine_rows_per_src=[8414, 9484, 9070, 9522, 9658, 8366, 9058, 11900] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75832 payload_rows_remote=67308 dup_rows=22472 dup_rows_remote=20216 dedup_ratio=0.228597 router_tokens_per_expert=[1083, 359, 855, 1228, 621, 914, 826, 883, 1387, 295, 954, 312, 272, 1507, 909, 1038, 744, 1048, 851, 666, 570, 729, 352, 1421, 485, 706, 766, 347, 807, 132, 814, 533, 1066, 784, 1315, 871, 452, 199, 555, 864, 518, 642, 654, 686, 1080, 622, 670, 518, 707, 578, 799, 430, 984, 1425, 810, 679, 1274, 358, 1421, 616, 212, 815, 1175, 959] payload_rows_per_dst=[8272, 9360, 10226, 9628, 9158, 8524, 9800, 10864] dup_rows_per_dst=[2810, 3430, 2536, 2566, 3054, 2256, 3024, 2796] combine_rows_per_src=[8272, 9360, 10226, 9628, 9158, 8524, 9800, 10864] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74734 payload_rows_remote=64302 dup_rows=23570 dup_rows_remote=19452 dedup_ratio=0.239766 router_tokens_per_expert=[1174, 409, 833, 1264, 651, 819, 804, 840, 1428, 377, 926, 355, 327, 1586, 877, 1067, 726, 1122, 807, 564, 680, 643, 391, 1200, 457, 708, 820, 312, 808, 105, 874, 566, 1071, 792, 1329, 856, 489, 201, 565, 825, 535, 601, 668, 716, 1071, 638, 666, 599, 674, 578, 790, 488, 992, 1328, 756, 670, 1327, 267, 1467, 549, 187, 781, 1113, 1043] payload_rows_per_dst=[10432, 8876, 9840, 8542, 9290, 9932, 9298, 8524] dup_rows_per_dst=[4118, 3646, 2426, 2122, 2966, 3990, 2292, 2010] combine_rows_per_src=[10432, 8876, 9840, 8542, 9290, 9932, 9298, 8524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=275693568 dispatch_meta_bytes=350096 combine_payload_bytes=275693568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276043664 ingress_bytes=310296576 link_bytes=310296576 token_bytes=551737232 weight_movement_bytes=34603008 total_nvlink_bytes=586340240 weight_to_token_ratio=0.062716 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=8960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.167621035e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265936896 dispatch_meta_bytes=341824 combine_payload_bytes=265936896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266278720 ingress_bytes=300539904 link_bytes=300539904 token_bytes=532215616 weight_movement_bytes=34603008 total_nvlink_bytes=566818624 weight_to_token_ratio=0.065017 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130907463e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=270393344 dispatch_meta_bytes=344552 combine_payload_bytes=270393344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270737896 ingress_bytes=304996352 link_bytes=304996352 token_bytes=541131240 weight_movement_bytes=34603008 total_nvlink_bytes=575734248 weight_to_token_ratio=0.063946 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.147676718e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=274382848 dispatch_meta_bytes=349024 combine_payload_bytes=274382848 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=274731872 ingress_bytes=343588864 link_bytes=343588864 token_bytes=549114720 weight_movement_bytes=69206016 total_nvlink_bytes=618320736 weight_to_token_ratio=0.126032 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.292897234e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=264011776 dispatch_meta_bytes=342496 combine_payload_bytes=264011776 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=264354272 ingress_bytes=333217792 link_bytes=333217792 token_bytes=528366048 weight_movement_bytes=69206016 total_nvlink_bytes=597572064 weight_to_token_ratio=0.130981 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=9728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.253871725e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=260390912 dispatch_meta_bytes=329600 combine_payload_bytes=260390912 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=260720512 ingress_bytes=329596928 link_bytes=329596928 token_bytes=521111424 weight_movement_bytes=69206016 total_nvlink_bytes=590317440 weight_to_token_ratio=0.132805 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.240246705e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=270344192 dispatch_meta_bytes=344744 combine_payload_bytes=270344192 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=270688936 ingress_bytes=374153216 link_bytes=374153216 token_bytes=541033128 weight_movement_bytes=103809024 total_nvlink_bytes=644842152 weight_to_token_ratio=0.191872 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.407908430e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75556 payload_rows_remote=66002 dup_rows=22748 dup_rows_remote=20184 dedup_ratio=0.231405 router_tokens_per_expert=[1153, 355, 816, 1253, 672, 928, 817, 788, 1356, 507, 984, 353, 257, 1466, 915, 1033, 764, 1089, 891, 650, 469, 656, 378, 1320, 549, 678, 822, 404, 804, 111, 871, 491, 1090, 801, 1332, 880, 354, 205, 569, 841, 494, 527, 558, 627, 1134, 691, 709, 567, 757, 559, 785, 474, 944, 1370, 715, 709, 1271, 434, 1430, 688, 217, 865, 1067, 888] payload_rows_per_dst=[8230, 9494, 9762, 9782, 9140, 8312, 9554, 11282] dup_rows_per_dst=[2828, 3822, 2672, 2610, 3004, 2302, 2564, 2946] combine_rows_per_src=[8230, 9494, 9762, 9782, 9140, 8312, 9554, 11282] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74730 payload_rows_remote=64926 dup_rows=23574 dup_rows_remote=20530 dedup_ratio=0.239807 router_tokens_per_expert=[1145, 432, 852, 1215, 575, 868, 802, 840, 1352, 435, 1055, 310, 289, 1646, 828, 1059, 733, 1108, 825, 573, 543, 645, 360, 1320, 439, 728, 840, 358, 807, 93, 877, 636, 1044, 828, 1351, 859, 457, 194, 522, 732, 599, 698, 620, 608, 1211, 634, 634, 490, 661, 586, 778, 487, 957, 1418, 733, 693, 1417, 295, 1309, 590, 159, 847, 1211, 942] payload_rows_per_dst=[9714, 7902, 9814, 9804, 9146, 9776, 9736, 8838] dup_rows_per_dst=[3472, 3026, 2400, 3044, 2828, 3830, 2890, 2084] combine_rows_per_src=[9714, 7902, 9814, 9804, 9146, 9776, 9736, 8838] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=260390912 dispatch_meta_bytes=329600 combine_payload_bytes=260390912 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=260720512 ingress_bytes=329596928 link_bytes=329596928 token_bytes=521111424 weight_movement_bytes=69206016 total_nvlink_bytes=590317440 weight_to_token_ratio=0.132805 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.240246705e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=275693568 dispatch_meta_bytes=350096 combine_payload_bytes=275693568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276043664 ingress_bytes=310296576 link_bytes=310296576 token_bytes=551737232 weight_movement_bytes=34603008 total_nvlink_bytes=586340240 weight_to_token_ratio=0.062716 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=8960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.167621035e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=263380992 dispatch_meta_bytes=335016 combine_payload_bytes=263380992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=263716008 ingress_bytes=332587008 link_bytes=332587008 token_bytes=527097000 weight_movement_bytes=69206016 total_nvlink_bytes=596303016 weight_to_token_ratio=0.131297 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=9216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.251498136e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75250 payload_rows_remote=66014 dup_rows=23054 dup_rows_remote=20124 dedup_ratio=0.234517 router_tokens_per_expert=[1094, 371, 765, 1264, 657, 782, 823, 867, 1427, 441, 1047, 353, 302, 1577, 969, 1148, 697, 995, 912, 661, 581, 739, 413, 1194, 485, 618, 749, 377, 791, 118, 882, 643, 984, 890, 1198, 893, 505, 202, 558, 853, 521, 645, 546, 667, 1112, 600, 635, 529, 723, 491, 780, 546, 1011, 1371, 732, 751, 1282, 327, 1335, 599, 176, 905, 1096, 947] payload_rows_per_dst=[8072, 9924, 9760, 9730, 9236, 9022, 9910, 9596] dup_rows_per_dst=[2646, 3978, 2624, 2750, 2930, 2964, 2900, 2262] combine_rows_per_src=[8072, 9924, 9760, 9730, 9236, 9022, 9910, 9596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74534 payload_rows_remote=64456 dup_rows=23770 dup_rows_remote=21168 dedup_ratio=0.241801 router_tokens_per_expert=[1166, 381, 801, 1217, 581, 1000, 781, 734, 1434, 496, 949, 301, 225, 1652, 842, 1076, 793, 1178, 876, 608, 528, 551, 441, 1365, 494, 636, 774, 350, 864, 116, 854, 609, 1123, 909, 1190, 895, 456, 212, 504, 869, 650, 611, 613, 626, 1095, 661, 655, 559, 618, 573, 805, 489, 984, 1401, 675, 764, 1262, 329, 1342, 535, 161, 794, 1138, 981] payload_rows_per_dst=[9656, 7678, 10078, 9700, 9270, 9858, 9856, 8438] dup_rows_per_dst=[3666, 2968, 2602, 2998, 3046, 3766, 2762, 1962] combine_rows_per_src=[9656, 7678, 10078, 9700, 9270, 9858, 9856, 8438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74624 payload_rows_remote=66190 dup_rows=23680 dup_rows_remote=21090 dedup_ratio=0.240885 router_tokens_per_expert=[645, 944, 712, 449, 1198, 286, 517, 1214, 90, 951, 1715, 349, 716, 1023, 748, 1590, 782, 1142, 688, 245, 1584, 177, 1773, 1226, 747, 615, 830, 933, 94, 211, 186, 123, 414, 178, 401, 163, 974, 1138, 970, 1054, 1863, 1399, 833, 1374, 319, 1586, 28, 283, 1224, 1099, 1183, 8, 87, 1584, 213, 1363, 935, 358, 917, 548, 1084, 354, 334, 381] payload_rows_per_dst=[9324, 10490, 9412, 8434, 8958, 8236, 10594, 9176] dup_rows_per_dst=[2606, 3874, 2276, 2590, 1626, 3408, 2928, 4372] combine_rows_per_src=[9324, 10490, 9412, 8434, 8958, 8236, 10594, 9176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74510 payload_rows_remote=64740 dup_rows=23794 dup_rows_remote=21376 dedup_ratio=0.242045 router_tokens_per_expert=[604, 1024, 815, 426, 1017, 242, 304, 1179, 80, 907, 1613, 310, 596, 940, 842, 1482, 955, 1245, 817, 377, 1513, 175, 1754, 1212, 639, 684, 726, 914, 95, 299, 162, 102, 411, 249, 434, 221, 1055, 1395, 1087, 1075, 2086, 1520, 943, 1350, 315, 1563, 13, 208, 1042, 1045, 1058, 4, 113, 1561, 214, 1369, 891, 340, 914, 523, 1062, 346, 313, 382] payload_rows_per_dst=[8782, 9840, 9884, 8338, 9770, 8296, 10188, 9412] dup_rows_per_dst=[2440, 3366, 2704, 2412, 2418, 3528, 2624, 4302] combine_rows_per_src=[8782, 9840, 9884, 8338, 9770, 8296, 10188, 9412] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75900 payload_rows_remote=65984 dup_rows=22404 dup_rows_remote=19026 dedup_ratio=0.227905 router_tokens_per_expert=[572, 961, 959, 365, 963, 289, 251, 1090, 44, 935, 1544, 332, 611, 944, 771, 1466, 871, 1166, 727, 456, 1575, 215, 1905, 1252, 763, 736, 761, 910, 97, 327, 156, 118, 308, 245, 410, 218, 979, 1416, 1043, 1047, 2242, 1533, 950, 1275, 355, 1624, 14, 267, 967, 1014, 1042, 7, 134, 1595, 204, 1435, 896, 407, 821, 454, 1064, 324, 381, 349] payload_rows_per_dst=[10474, 9916, 11410, 8820, 9336, 9890, 8334, 7720] dup_rows_per_dst=[3674, 3378, 4304, 2726, 1996, 3382, 1272, 1672] combine_rows_per_src=[10474, 9916, 11410, 8820, 9336, 9890, 8334, 7720] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=66306 dup_rows=23906 dup_rows_remote=20660 dedup_ratio=0.243184 router_tokens_per_expert=[570, 893, 956, 398, 1180, 241, 355, 1111, 51, 912, 1695, 362, 638, 881, 756, 1561, 844, 1188, 774, 413, 1483, 214, 1902, 1331, 732, 760, 889, 888, 113, 336, 178, 112, 321, 251, 374, 169, 961, 1173, 880, 1017, 2166, 1357, 860, 1293, 293, 1568, 22, 276, 1227, 1010, 1113, 5, 120, 1369, 187, 1469, 927, 353, 944, 555, 1071, 353, 351, 400] payload_rows_per_dst=[8906, 8240, 8834, 9964, 10416, 8092, 10324, 9622] dup_rows_per_dst=[2502, 2082, 2008, 3508, 3266, 3246, 2676, 4618] combine_rows_per_src=[8906, 8240, 8834, 9964, 10416, 8092, 10324, 9622] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=270344192 dispatch_meta_bytes=344744 combine_payload_bytes=270344192 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=270688936 ingress_bytes=374153216 link_bytes=374153216 token_bytes=541033128 weight_movement_bytes=103809024 total_nvlink_bytes=644842152 weight_to_token_ratio=0.191872 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.407908430e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75332 payload_rows_remote=65116 dup_rows=22972 dup_rows_remote=19804 dedup_ratio=0.233683 router_tokens_per_expert=[501, 1145, 677, 383, 1168, 295, 501, 1136, 52, 950, 1876, 340, 567, 958, 826, 1570, 890, 1190, 727, 325, 1540, 209, 1748, 1387, 776, 768, 820, 893, 96, 260, 157, 108, 308, 177, 411, 181, 1019, 1050, 999, 1134, 1784, 1422, 894, 1294, 269, 1502, 18, 268, 1226, 1106, 1315, 2, 94, 1513, 190, 1433, 830, 328, 867, 514, 1037, 351, 386, 391] payload_rows_per_dst=[9268, 10452, 10216, 8402, 8994, 9730, 10412, 7858] dup_rows_per_dst=[3000, 3826, 3168, 2850, 1564, 3956, 2498, 2110] combine_rows_per_src=[9268, 10452, 10216, 8402, 8994, 9730, 10412, 7858] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74498 payload_rows_remote=64216 dup_rows=23806 dup_rows_remote=21122 dedup_ratio=0.242167 router_tokens_per_expert=[661, 845, 879, 553, 1114, 177, 324, 1155, 67, 976, 1680, 314, 641, 984, 870, 1493, 956, 1133, 685, 366, 1523, 197, 1716, 1464, 733, 654, 785, 931, 186, 361, 197, 107, 352, 195, 373, 149, 894, 1209, 1147, 1010, 2099, 1406, 890, 1228, 277, 1570, 10, 250, 1179, 998, 1177, 4, 115, 1494, 164, 1352, 921, 421, 963, 475, 1077, 323, 308, 395] payload_rows_per_dst=[9008, 8560, 8124, 10368, 10746, 7998, 10282, 9412] dup_rows_per_dst=[2408, 2130, 1478, 4018, 3272, 3264, 2684, 4552] combine_rows_per_src=[9008, 8560, 8124, 10368, 10746, 7998, 10282, 9412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=264011776 dispatch_meta_bytes=342496 combine_payload_bytes=264011776 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=264354272 ingress_bytes=333217792 link_bytes=333217792 token_bytes=528366048 weight_movement_bytes=69206016 total_nvlink_bytes=597572064 weight_to_token_ratio=0.130981 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=9728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.253871725e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73692 payload_rows_remote=64436 dup_rows=24612 dup_rows_remote=20290 dedup_ratio=0.250366 router_tokens_per_expert=[629, 1050, 761, 412, 969, 333, 339, 1112, 41, 849, 1651, 293, 615, 1034, 784, 1507, 880, 1289, 731, 418, 1669, 132, 1929, 1156, 717, 674, 751, 902, 103, 319, 166, 110, 413, 193, 434, 241, 892, 1468, 1037, 1175, 2064, 1432, 988, 1446, 296, 1429, 14, 252, 939, 971, 947, 14, 137, 1643, 190, 1487, 812, 356, 869, 498, 1167, 334, 330, 359] payload_rows_per_dst=[8812, 6446, 7884, 10618, 11022, 8810, 10844, 9256] dup_rows_per_dst=[2398, 1624, 1328, 4062, 3986, 3922, 2970, 4322] combine_rows_per_src=[8812, 6446, 7884, 10618, 11022, 8810, 10844, 9256] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265936896 dispatch_meta_bytes=341824 combine_payload_bytes=265936896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266278720 ingress_bytes=300539904 link_bytes=300539904 token_bytes=532215616 weight_movement_bytes=34603008 total_nvlink_bytes=566818624 weight_to_token_ratio=0.065017 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130907463e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=270270464 dispatch_meta_bytes=340040 combine_payload_bytes=270270464 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=270610504 ingress_bytes=374079488 link_bytes=374079488 token_bytes=540880968 weight_movement_bytes=103809024 total_nvlink_bytes=644689992 weight_to_token_ratio=0.191926 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=11264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.407630997e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75876 payload_rows_remote=65224 dup_rows=22428 dup_rows_remote=18378 dedup_ratio=0.228149 router_tokens_per_expert=[701, 950, 850, 376, 1172, 273, 384, 1051, 67, 857, 1636, 354, 605, 1038, 882, 1601, 802, 1141, 726, 372, 1635, 182, 1876, 1304, 685, 674, 732, 924, 127, 254, 199, 126, 358, 162, 393, 173, 941, 1402, 893, 1145, 2104, 1338, 908, 1328, 285, 1594, 39, 271, 1161, 983, 917, 8, 119, 1475, 235, 1414, 936, 404, 898, 588, 1025, 375, 315, 409] payload_rows_per_dst=[10652, 10372, 11316, 8492, 9072, 9464, 8484, 8024] dup_rows_per_dst=[4050, 3708, 3958, 2702, 1862, 3082, 1190, 1876] combine_rows_per_src=[10652, 10372, 11316, 8492, 9072, 9464, 8484, 8024] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=271589376 dispatch_meta_bytes=347864 combine_payload_bytes=271589376 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=271937240 ingress_bytes=375398400 link_bytes=375398400 token_bytes=543526616 weight_movement_bytes=103809024 total_nvlink_bytes=647335640 weight_to_token_ratio=0.190992 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.412593957e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=66306 dup_rows=23906 dup_rows_remote=20660 dedup_ratio=0.243184 router_tokens_per_expert=[570, 893, 956, 398, 1180, 241, 355, 1111, 51, 912, 1695, 362, 638, 881, 756, 1561, 844, 1188, 774, 413, 1483, 214, 1902, 1331, 732, 760, 889, 888, 113, 336, 178, 112, 321, 251, 374, 169, 961, 1173, 880, 1017, 2166, 1357, 860, 1293, 293, 1568, 22, 276, 1227, 1010, 1113, 5, 120, 1369, 187, 1469, 927, 353, 944, 555, 1071, 353, 351, 400] payload_rows_per_dst=[8906, 8240, 8834, 9964, 10416, 8092, 10324, 9622] dup_rows_per_dst=[2502, 2082, 2008, 3508, 3266, 3246, 2676, 4618] combine_rows_per_src=[8906, 8240, 8834, 9964, 10416, 8092, 10324, 9622] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=271114240 dispatch_meta_bytes=349120 combine_payload_bytes=271114240 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=271463360 ingress_bytes=340320256 link_bytes=340320256 token_bytes=542577600 weight_movement_bytes=69206016 total_nvlink_bytes=611783616 weight_to_token_ratio=0.127550 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=14720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.280597725e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=270393344 dispatch_meta_bytes=344552 combine_payload_bytes=270393344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270737896 ingress_bytes=304996352 link_bytes=304996352 token_bytes=541131240 weight_movement_bytes=34603008 total_nvlink_bytes=575734248 weight_to_token_ratio=0.063946 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.147676718e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265175040 dispatch_meta_bytes=344464 combine_payload_bytes=265175040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265519504 ingress_bytes=299778048 link_bytes=299778048 token_bytes=530694544 weight_movement_bytes=34603008 total_nvlink_bytes=565297552 weight_to_token_ratio=0.065203 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128040661e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=270270464 dispatch_meta_bytes=340040 combine_payload_bytes=270270464 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=270610504 ingress_bytes=374079488 link_bytes=374079488 token_bytes=540880968 weight_movement_bytes=103809024 total_nvlink_bytes=644689992 weight_to_token_ratio=0.191926 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=11264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.407630997e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=263028736 dispatch_meta_bytes=341352 combine_payload_bytes=263028736 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=263370088 ingress_bytes=332234752 link_bytes=332234752 token_bytes=526398824 weight_movement_bytes=69206016 total_nvlink_bytes=595604840 weight_to_token_ratio=0.131471 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=11392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.250172625e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266715136 dispatch_meta_bytes=339680 combine_payload_bytes=266715136 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267054816 ingress_bytes=370524160 link_bytes=370524160 token_bytes=533769952 weight_movement_bytes=103809024 total_nvlink_bytes=637578976 weight_to_token_ratio=0.194483 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=10368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394252584e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=263929856 dispatch_meta_bytes=338904 combine_payload_bytes=263929856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264268760 ingress_bytes=298532864 link_bytes=298532864 token_bytes=528198616 weight_movement_bytes=34603008 total_nvlink_bytes=562801624 weight_to_token_ratio=0.065511 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123355133e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267157504 dispatch_meta_bytes=334408 combine_payload_bytes=267157504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267491912 ingress_bytes=301760512 link_bytes=301760512 token_bytes=534649416 weight_movement_bytes=34603008 total_nvlink_bytes=569252424 weight_to_token_ratio=0.064721 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135500513e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74498 payload_rows_remote=64216 dup_rows=23806 dup_rows_remote=21122 dedup_ratio=0.242167 router_tokens_per_expert=[661, 845, 879, 553, 1114, 177, 324, 1155, 67, 976, 1680, 314, 641, 984, 870, 1493, 956, 1133, 685, 366, 1523, 197, 1716, 1464, 733, 654, 785, 931, 186, 361, 197, 107, 352, 195, 373, 149, 894, 1209, 1147, 1010, 2099, 1406, 890, 1228, 277, 1570, 10, 250, 1179, 998, 1177, 4, 115, 1494, 164, 1352, 921, 421, 963, 475, 1077, 323, 308, 395] payload_rows_per_dst=[9008, 8560, 8124, 10368, 10746, 7998, 10282, 9412] dup_rows_per_dst=[2408, 2130, 1478, 4018, 3272, 3264, 2684, 4552] combine_rows_per_src=[9008, 8560, 8124, 10368, 10746, 7998, 10282, 9412] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=271589376 dispatch_meta_bytes=347864 combine_payload_bytes=271589376 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=271937240 ingress_bytes=375398400 link_bytes=375398400 token_bytes=543526616 weight_movement_bytes=103809024 total_nvlink_bytes=647335640 weight_to_token_ratio=0.190992 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.412593957e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75332 payload_rows_remote=65116 dup_rows=22972 dup_rows_remote=19804 dedup_ratio=0.233683 router_tokens_per_expert=[501, 1145, 677, 383, 1168, 295, 501, 1136, 52, 950, 1876, 340, 567, 958, 826, 1570, 890, 1190, 727, 325, 1540, 209, 1748, 1387, 776, 768, 820, 893, 96, 260, 157, 108, 308, 177, 411, 181, 1019, 1050, 999, 1134, 1784, 1422, 894, 1294, 269, 1502, 18, 268, 1226, 1106, 1315, 2, 94, 1513, 190, 1433, 830, 328, 867, 514, 1037, 351, 386, 391] payload_rows_per_dst=[9268, 10452, 10216, 8402, 8994, 9730, 10412, 7858] dup_rows_per_dst=[3000, 3826, 3168, 2850, 1564, 3956, 2498, 2110] combine_rows_per_src=[9268, 10452, 10216, 8402, 8994, 9730, 10412, 7858] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75104 payload_rows_remote=64612 dup_rows=23200 dup_rows_remote=20082 dedup_ratio=0.236003 router_tokens_per_expert=[16, 1247, 553, 350, 12, 1121, 721, 224, 530, 445, 985, 1505, 1557, 842, 54, 971, 957, 1704, 276, 712, 94, 78, 1202, 1062, 742, 977, 1005, 798, 181, 491, 821, 139, 429, 1145, 740, 657, 2815, 2449, 1725, 469, 714, 58, 314, 238, 1553, 1800, 130, 1180, 655, 87, 965, 1234, 20, 34, 189, 480, 1244, 557, 52, 441, 3, 1950, 125, 2328] payload_rows_per_dst=[9342, 8272, 9870, 10492, 8814, 8506, 9178, 10630] dup_rows_per_dst=[2260, 2392, 2300, 3118, 3112, 3468, 3780, 2770] combine_rows_per_src=[9342, 8272, 9870, 10492, 8814, 8506, 9178, 10630] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73756 payload_rows_remote=63934 dup_rows=24548 dup_rows_remote=20476 dedup_ratio=0.249715 router_tokens_per_expert=[24, 904, 486, 331, 33, 1251, 837, 233, 471, 427, 1084, 1562, 1537, 1025, 116, 1708, 923, 1594, 320, 653, 92, 90, 1294, 1222, 554, 819, 992, 843, 211, 566, 1094, 190, 417, 1111, 714, 567, 2394, 2119, 1656, 456, 746, 89, 312, 316, 1434, 1539, 131, 1181, 910, 117, 944, 1070, 18, 73, 262, 565, 1151, 394, 76, 520, 12, 1718, 310, 2344] payload_rows_per_dst=[10844, 9822, 9844, 11148, 7648, 8066, 8938, 7446] dup_rows_per_dst=[3552, 4072, 2532, 3628, 2194, 3430, 3768, 1372] combine_rows_per_src=[10844, 9822, 9844, 11148, 7648, 8066, 8938, 7446] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75518 payload_rows_remote=66876 dup_rows=22786 dup_rows_remote=19080 dedup_ratio=0.231791 router_tokens_per_expert=[18, 1122, 423, 349, 17, 1216, 688, 211, 532, 397, 1096, 1706, 1492, 899, 119, 1139, 981, 1538, 342, 634, 78, 84, 1438, 1125, 570, 882, 1028, 834, 157, 471, 967, 164, 447, 1330, 798, 569, 2882, 2017, 1608, 517, 803, 96, 247, 253, 1475, 1790, 135, 1375, 777, 92, 835, 992, 15, 78, 282, 532, 1046, 487, 83, 492, 10, 1835, 140, 2397] payload_rows_per_dst=[9170, 9174, 9810, 8590, 10394, 8642, 9344, 10394] dup_rows_per_dst=[1902, 2602, 2630, 1556, 4178, 3706, 3626, 2586] combine_rows_per_src=[9170, 9174, 9810, 8590, 10394, 8642, 9344, 10394] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74624 payload_rows_remote=66190 dup_rows=23680 dup_rows_remote=21090 dedup_ratio=0.240885 router_tokens_per_expert=[645, 944, 712, 449, 1198, 286, 517, 1214, 90, 951, 1715, 349, 716, 1023, 748, 1590, 782, 1142, 688, 245, 1584, 177, 1773, 1226, 747, 615, 830, 933, 94, 211, 186, 123, 414, 178, 401, 163, 974, 1138, 970, 1054, 1863, 1399, 833, 1374, 319, 1586, 28, 283, 1224, 1099, 1183, 8, 87, 1584, 213, 1363, 935, 358, 917, 548, 1084, 354, 334, 381] payload_rows_per_dst=[9324, 10490, 9412, 8434, 8958, 8236, 10594, 9176] dup_rows_per_dst=[2606, 3874, 2276, 2590, 1626, 3408, 2928, 4372] combine_rows_per_src=[9324, 10490, 9412, 8434, 8958, 8236, 10594, 9176] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=263028736 dispatch_meta_bytes=341352 combine_payload_bytes=263028736 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=263370088 ingress_bytes=332234752 link_bytes=332234752 token_bytes=526398824 weight_movement_bytes=69206016 total_nvlink_bytes=595604840 weight_to_token_ratio=0.131471 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=11392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.250172625e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74510 payload_rows_remote=64740 dup_rows=23794 dup_rows_remote=21376 dedup_ratio=0.242045 router_tokens_per_expert=[604, 1024, 815, 426, 1017, 242, 304, 1179, 80, 907, 1613, 310, 596, 940, 842, 1482, 955, 1245, 817, 377, 1513, 175, 1754, 1212, 639, 684, 726, 914, 95, 299, 162, 102, 411, 249, 434, 221, 1055, 1395, 1087, 1075, 2086, 1520, 943, 1350, 315, 1563, 13, 208, 1042, 1045, 1058, 4, 113, 1561, 214, 1369, 891, 340, 914, 523, 1062, 346, 313, 382] payload_rows_per_dst=[8782, 9840, 9884, 8338, 9770, 8296, 10188, 9412] dup_rows_per_dst=[2440, 3366, 2704, 2412, 2418, 3528, 2624, 4302] combine_rows_per_src=[8782, 9840, 9884, 8338, 9770, 8296, 10188, 9412] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=261873664 dispatch_meta_bytes=337640 combine_payload_bytes=261873664 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=262211304 ingress_bytes=365682688 link_bytes=365682688 token_bytes=524084968 weight_movement_bytes=103809024 total_nvlink_bytes=627893992 weight_to_token_ratio=0.198077 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=10752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.376034515e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75130 payload_rows_remote=66214 dup_rows=23174 dup_rows_remote=19648 dedup_ratio=0.235738 router_tokens_per_expert=[33, 1515, 503, 369, 26, 1133, 810, 229, 549, 360, 1068, 1517, 1792, 1156, 178, 941, 911, 1603, 308, 667, 69, 116, 1227, 1005, 662, 960, 975, 728, 191, 527, 1077, 190, 388, 1117, 548, 605, 2501, 1994, 1899, 484, 837, 138, 336, 219, 1559, 1750, 185, 1253, 914, 141, 712, 956, 27, 150, 341, 479, 1158, 400, 66, 466, 28, 1524, 193, 2389] payload_rows_per_dst=[10022, 8948, 9466, 9018, 10160, 8584, 8916, 10016] dup_rows_per_dst=[2798, 2590, 2346, 1602, 3910, 3970, 3526, 2432] combine_rows_per_src=[10022, 8948, 9466, 9018, 10160, 8584, 8916, 10016] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74994 payload_rows_remote=64488 dup_rows=23310 dup_rows_remote=19106 dedup_ratio=0.237122 router_tokens_per_expert=[39, 1061, 468, 399, 46, 1115, 793, 256, 469, 250, 936, 1572, 1588, 1132, 278, 1782, 903, 1325, 214, 647, 88, 198, 1315, 943, 539, 862, 954, 813, 156, 584, 896, 213, 417, 1152, 721, 665, 2111, 1913, 1992, 495, 587, 114, 272, 156, 1785, 1758, 159, 1234, 988, 114, 844, 957, 51, 140, 308, 694, 1153, 410, 69, 487, 93, 1700, 423, 2356] payload_rows_per_dst=[9082, 9556, 9456, 8696, 10506, 8556, 8682, 10460] dup_rows_per_dst=[2448, 3282, 1810, 1338, 4204, 3574, 3732, 2922] combine_rows_per_src=[9082, 9556, 9456, 8696, 10506, 8556, 8682, 10460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74104 payload_rows_remote=63200 dup_rows=24200 dup_rows_remote=20076 dedup_ratio=0.246175 router_tokens_per_expert=[81, 1019, 486, 291, 49, 1016, 837, 251, 542, 324, 844, 1433, 1572, 1174, 269, 1760, 903, 1599, 283, 711, 75, 145, 1238, 987, 483, 885, 942, 662, 187, 593, 908, 300, 433, 1091, 734, 645, 2220, 1929, 1861, 527, 553, 126, 345, 205, 1706, 1691, 134, 1264, 837, 125, 1089, 963, 49, 189, 297, 661, 1117, 393, 108, 533, 106, 1594, 442, 2336] payload_rows_per_dst=[8918, 9430, 9718, 8550, 8466, 9254, 8864, 10904] dup_rows_per_dst=[2286, 3262, 2164, 1370, 2692, 4306, 3996, 4124] combine_rows_per_src=[8918, 9430, 9718, 8550, 8466, 9254, 8864, 10904] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74968 payload_rows_remote=65012 dup_rows=23336 dup_rows_remote=20952 dedup_ratio=0.237386 router_tokens_per_expert=[12, 1111, 486, 358, 12, 1043, 660, 237, 570, 410, 1155, 1452, 1721, 984, 73, 991, 927, 1734, 270, 678, 177, 106, 1308, 970, 802, 927, 1085, 671, 104, 492, 847, 167, 433, 1025, 631, 595, 2826, 2478, 1725, 388, 716, 67, 383, 270, 1628, 1900, 123, 935, 850, 93, 777, 1098, 17, 83, 232, 443, 1325, 549, 81, 397, 2, 1839, 179, 2524] payload_rows_per_dst=[9074, 8450, 9956, 11598, 7562, 8514, 9008, 10806] dup_rows_per_dst=[2268, 2758, 2384, 3548, 2032, 3530, 3830, 2986] combine_rows_per_src=[9074, 8450, 9956, 11598, 7562, 8514, 9008, 10806] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=273924096 dispatch_meta_bytes=343824 combine_payload_bytes=273924096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274267920 ingress_bytes=308527104 link_bytes=308527104 token_bytes=548192016 weight_movement_bytes=34603008 total_nvlink_bytes=582795024 weight_to_token_ratio=0.063122 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=14080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.160962654e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266715136 dispatch_meta_bytes=339680 combine_payload_bytes=266715136 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267054816 ingress_bytes=370524160 link_bytes=370524160 token_bytes=533769952 weight_movement_bytes=103809024 total_nvlink_bytes=637578976 weight_to_token_ratio=0.194483 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=10368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394252584e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74158 payload_rows_remote=63286 dup_rows=24146 dup_rows_remote=20680 dedup_ratio=0.245626 router_tokens_per_expert=[36, 1095, 436, 351, 25, 1103, 776, 244, 554, 422, 890, 1560, 1543, 1039, 182, 1346, 898, 1578, 282, 685, 94, 88, 1291, 978, 600, 928, 898, 795, 156, 585, 997, 162, 408, 1297, 774, 543, 2678, 1905, 1833, 448, 673, 101, 310, 224, 1618, 1778, 153, 1242, 773, 86, 927, 1072, 24, 196, 234, 596, 1150, 524, 54, 541, 33, 1707, 219, 2414] payload_rows_per_dst=[10872, 9804, 10872, 10992, 9066, 8474, 6770, 7308] dup_rows_per_dst=[3466, 3890, 3146, 3060, 3366, 3724, 2346, 1148] combine_rows_per_src=[10872, 9804, 10872, 10992, 9066, 8474, 6770, 7308] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=271114240 dispatch_meta_bytes=349120 combine_payload_bytes=271114240 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=271463360 ingress_bytes=340320256 link_bytes=340320256 token_bytes=542577600 weight_movement_bytes=69206016 total_nvlink_bytes=611783616 weight_to_token_ratio=0.127550 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=14720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.280597725e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265175040 dispatch_meta_bytes=344464 combine_payload_bytes=265175040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265519504 ingress_bytes=299778048 link_bytes=299778048 token_bytes=530694544 weight_movement_bytes=34603008 total_nvlink_bytes=565297552 weight_to_token_ratio=0.065203 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128040661e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=261873664 dispatch_meta_bytes=337640 combine_payload_bytes=261873664 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=262211304 ingress_bytes=365682688 link_bytes=365682688 token_bytes=524084968 weight_movement_bytes=103809024 total_nvlink_bytes=627893992 weight_to_token_ratio=0.198077 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=10752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.376034515e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=271212544 dispatch_meta_bytes=343448 combine_payload_bytes=271212544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271555992 ingress_bytes=305815552 link_bytes=305815552 token_bytes=542768536 weight_movement_bytes=34603008 total_nvlink_bytes=577371544 weight_to_token_ratio=0.063753 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.150759302e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75518 payload_rows_remote=66876 dup_rows=22786 dup_rows_remote=19080 dedup_ratio=0.231791 router_tokens_per_expert=[18, 1122, 423, 349, 17, 1216, 688, 211, 532, 397, 1096, 1706, 1492, 899, 119, 1139, 981, 1538, 342, 634, 78, 84, 1438, 1125, 570, 882, 1028, 834, 157, 471, 967, 164, 447, 1330, 798, 569, 2882, 2017, 1608, 517, 803, 96, 247, 253, 1475, 1790, 135, 1375, 777, 92, 835, 992, 15, 78, 282, 532, 1046, 487, 83, 492, 10, 1835, 140, 2397] payload_rows_per_dst=[9170, 9174, 9810, 8590, 10394, 8642, 9344, 10394] dup_rows_per_dst=[1902, 2602, 2630, 1556, 4178, 3706, 3626, 2586] combine_rows_per_src=[9170, 9174, 9810, 8590, 10394, 8642, 9344, 10394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=264650752 dispatch_meta_bytes=338776 combine_payload_bytes=264650752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264989528 ingress_bytes=299253760 link_bytes=299253760 token_bytes=529640280 weight_movement_bytes=34603008 total_nvlink_bytes=564243288 weight_to_token_ratio=0.065333 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=8192 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126067807e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=264142848 dispatch_meta_bytes=334376 combine_payload_bytes=264142848 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=264477224 ingress_bytes=437157888 link_bytes=437157888 token_bytes=528620072 weight_movement_bytes=173015040 total_nvlink_bytes=701635112 weight_to_token_ratio=0.327296 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=11648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.644989938e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=258867200 dispatch_meta_bytes=333104 combine_payload_bytes=258867200 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=259200304 ingress_bytes=328073216 link_bytes=328073216 token_bytes=518067504 weight_movement_bytes=69206016 total_nvlink_bytes=587273520 weight_to_token_ratio=0.133585 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.234513100e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266289152 dispatch_meta_bytes=343856 combine_payload_bytes=266289152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266633008 ingress_bytes=300892160 link_bytes=300892160 token_bytes=532922160 weight_movement_bytes=34603008 total_nvlink_bytes=567525168 weight_to_token_ratio=0.064931 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132232974e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75130 payload_rows_remote=66214 dup_rows=23174 dup_rows_remote=19648 dedup_ratio=0.235738 router_tokens_per_expert=[33, 1515, 503, 369, 26, 1133, 810, 229, 549, 360, 1068, 1517, 1792, 1156, 178, 941, 911, 1603, 308, 667, 69, 116, 1227, 1005, 662, 960, 975, 728, 191, 527, 1077, 190, 388, 1117, 548, 605, 2501, 1994, 1899, 484, 837, 138, 336, 219, 1559, 1750, 185, 1253, 914, 141, 712, 956, 27, 150, 341, 479, 1158, 400, 66, 466, 28, 1524, 193, 2389] payload_rows_per_dst=[10022, 8948, 9466, 9018, 10160, 8584, 8916, 10016] dup_rows_per_dst=[2798, 2590, 2346, 1602, 3910, 3970, 3526, 2432] combine_rows_per_src=[10022, 8948, 9466, 9018, 10160, 8584, 8916, 10016] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=259219456 dispatch_meta_bytes=335864 combine_payload_bytes=259219456 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=259555320 ingress_bytes=328425472 link_bytes=328425472 token_bytes=518774776 weight_movement_bytes=69206016 total_nvlink_bytes=587980792 weight_to_token_ratio=0.133403 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=12928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.235838611e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=273924096 dispatch_meta_bytes=343824 combine_payload_bytes=273924096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274267920 ingress_bytes=308527104 link_bytes=308527104 token_bytes=548192016 weight_movement_bytes=34603008 total_nvlink_bytes=582795024 weight_to_token_ratio=0.063122 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=14080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.160962654e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74968 payload_rows_remote=65012 dup_rows=23336 dup_rows_remote=20952 dedup_ratio=0.237386 router_tokens_per_expert=[12, 1111, 486, 358, 12, 1043, 660, 237, 570, 410, 1155, 1452, 1721, 984, 73, 991, 927, 1734, 270, 678, 177, 106, 1308, 970, 802, 927, 1085, 671, 104, 492, 847, 167, 433, 1025, 631, 595, 2826, 2478, 1725, 388, 716, 67, 383, 270, 1628, 1900, 123, 935, 850, 93, 777, 1098, 17, 83, 232, 443, 1325, 549, 81, 397, 2, 1839, 179, 2524] payload_rows_per_dst=[9074, 8450, 9956, 11598, 7562, 8514, 9008, 10806] dup_rows_per_dst=[2268, 2758, 2384, 3548, 2032, 3530, 3830, 2986] combine_rows_per_src=[9074, 8450, 9956, 11598, 7562, 8514, 9008, 10806] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76148 payload_rows_remote=65142 dup_rows=22156 dup_rows_remote=18390 dedup_ratio=0.225382 router_tokens_per_expert=[172, 325, 1358, 109, 461, 525, 1218, 816, 2260, 2900, 234, 384, 1126, 330, 2153, 419, 2028, 45, 2492, 436, 757, 353, 0, 544, 933, 1416, 652, 355, 495, 1266, 1478, 616, 460, 586, 121, 127, 1351, 519, 1254, 531, 181, 2323, 111, 471, 239, 316, 301, 3753, 6, 233, 117, 1161, 503, 1098, 72, 119, 145, 100, 1786, 621, 853, 743, 105, 190] payload_rows_per_dst=[10060, 11006, 9606, 9620, 12656, 5774, 9670, 7756] dup_rows_per_dst=[2864, 3766, 3704, 1846, 4748, 1150, 2748, 1330] combine_rows_per_src=[10060, 11006, 9606, 9620, 12656, 5774, 9670, 7756] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=271212544 dispatch_meta_bytes=343448 combine_payload_bytes=271212544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271555992 ingress_bytes=305815552 link_bytes=305815552 token_bytes=542768536 weight_movement_bytes=34603008 total_nvlink_bytes=577371544 weight_to_token_ratio=0.063753 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.150759302e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77712 payload_rows_remote=66602 dup_rows=20592 dup_rows_remote=16418 dedup_ratio=0.209473 router_tokens_per_expert=[168, 263, 1403, 76, 494, 602, 1146, 858, 2651, 3007, 342, 355, 1024, 434, 2646, 343, 2076, 35, 2275, 413, 829, 419, 1, 522, 535, 1539, 563, 396, 467, 1164, 1395, 603, 522, 509, 104, 134, 1188, 385, 1164, 506, 164, 2665, 134, 433, 236, 281, 345, 3384, 1, 172, 145, 1081, 564, 1020, 41, 130, 133, 66, 1978, 573, 955, 812, 74, 209] payload_rows_per_dst=[9452, 9006, 9560, 9964, 7480, 11110, 9782, 11358] dup_rows_per_dst=[2322, 1282, 3580, 1606, 1544, 4174, 2540, 3544] combine_rows_per_src=[9452, 9006, 9560, 9964, 7480, 11110, 9782, 11358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266821632 dispatch_meta_bytes=334128 combine_payload_bytes=266821632 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267155760 ingress_bytes=370630656 link_bytes=370630656 token_bytes=533977392 weight_movement_bytes=103809024 total_nvlink_bytes=637786416 weight_to_token_ratio=0.194407 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=17536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394653320e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75104 payload_rows_remote=64612 dup_rows=23200 dup_rows_remote=20082 dedup_ratio=0.236003 router_tokens_per_expert=[16, 1247, 553, 350, 12, 1121, 721, 224, 530, 445, 985, 1505, 1557, 842, 54, 971, 957, 1704, 276, 712, 94, 78, 1202, 1062, 742, 977, 1005, 798, 181, 491, 821, 139, 429, 1145, 740, 657, 2815, 2449, 1725, 469, 714, 58, 314, 238, 1553, 1800, 130, 1180, 655, 87, 965, 1234, 20, 34, 189, 480, 1244, 557, 52, 441, 3, 1950, 125, 2328] payload_rows_per_dst=[9342, 8272, 9870, 10492, 8814, 8506, 9178, 10630] dup_rows_per_dst=[2260, 2392, 2300, 3118, 3112, 3468, 3780, 2770] combine_rows_per_src=[9342, 8272, 9870, 10492, 8814, 8506, 9178, 10630] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75986 payload_rows_remote=65166 dup_rows=22318 dup_rows_remote=19406 dedup_ratio=0.227030 router_tokens_per_expert=[279, 532, 1325, 97, 420, 403, 1093, 819, 1856, 2920, 246, 372, 1191, 353, 1589, 466, 1746, 48, 2367, 417, 716, 271, 5, 612, 1141, 1170, 660, 373, 417, 1211, 1461, 618, 512, 539, 146, 190, 1104, 592, 1405, 522, 325, 2513, 90, 485, 316, 434, 350, 3879, 9, 194, 184, 1103, 510, 1244, 151, 151, 115, 145, 2002, 623, 1022, 768, 117, 218] payload_rows_per_dst=[8342, 7378, 6680, 12600, 8334, 12070, 9762, 10820] dup_rows_per_dst=[1594, 1056, 956, 5580, 1686, 5364, 3170, 2912] combine_rows_per_src=[8342, 7378, 6680, 12600, 8334, 12070, 9762, 10820] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74994 payload_rows_remote=64488 dup_rows=23310 dup_rows_remote=19106 dedup_ratio=0.237122 router_tokens_per_expert=[39, 1061, 468, 399, 46, 1115, 793, 256, 469, 250, 936, 1572, 1588, 1132, 278, 1782, 903, 1325, 214, 647, 88, 198, 1315, 943, 539, 862, 954, 813, 156, 584, 896, 213, 417, 1152, 721, 665, 2111, 1913, 1992, 495, 587, 114, 272, 156, 1785, 1758, 159, 1234, 988, 114, 844, 957, 51, 140, 308, 694, 1153, 410, 69, 487, 93, 1700, 423, 2356] payload_rows_per_dst=[9082, 9556, 9456, 8696, 10506, 8556, 8682, 10460] dup_rows_per_dst=[2448, 3282, 1810, 1338, 4204, 3574, 3732, 2922] combine_rows_per_src=[9082, 9556, 9456, 8696, 10506, 8556, 8682, 10460] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76634 payload_rows_remote=66962 dup_rows=21670 dup_rows_remote=18064 dedup_ratio=0.220439 router_tokens_per_expert=[167, 273, 1592, 50, 527, 694, 1212, 916, 2282, 2930, 243, 391, 1245, 289, 2425, 509, 1960, 18, 2484, 417, 885, 327, 1, 547, 557, 1548, 635, 315, 276, 1332, 1531, 503, 429, 912, 129, 209, 1244, 381, 945, 588, 177, 2622, 123, 506, 145, 396, 79, 3620, 3, 190, 128, 1109, 318, 1171, 51, 91, 156, 46, 2026, 577, 844, 576, 92, 188] payload_rows_per_dst=[10306, 11402, 9672, 9148, 12196, 6534, 9528, 7848] dup_rows_per_dst=[3618, 3366, 3606, 1184, 4718, 1562, 2454, 1162] combine_rows_per_src=[10306, 11402, 9672, 9148, 12196, 6534, 9528, 7848] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=272801792 dispatch_meta_bytes=332080 combine_payload_bytes=272801792 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=273133872 ingress_bytes=376610816 link_bytes=376610816 token_bytes=545935664 weight_movement_bytes=103809024 total_nvlink_bytes=649744688 weight_to_token_ratio=0.190149 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.417156181e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=264650752 dispatch_meta_bytes=338776 combine_payload_bytes=264650752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264989528 ingress_bytes=299253760 link_bytes=299253760 token_bytes=529640280 weight_movement_bytes=34603008 total_nvlink_bytes=564243288 weight_to_token_ratio=0.065333 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=8192 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126067807e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74998 payload_rows_remote=64756 dup_rows=23306 dup_rows_remote=20542 dedup_ratio=0.237081 router_tokens_per_expert=[254, 411, 1313, 126, 477, 521, 1175, 813, 2220, 2903, 330, 352, 999, 430, 2337, 403, 2059, 61, 2175, 454, 803, 397, 1, 578, 707, 1409, 633, 420, 423, 1247, 1413, 611, 512, 494, 126, 171, 1158, 430, 1275, 454, 245, 2688, 131, 505, 275, 319, 361, 3495, 6, 225, 146, 1150, 391, 984, 99, 113, 130, 107, 2050, 643, 909, 797, 92, 216] payload_rows_per_dst=[10242, 13080, 9366, 9578, 12204, 3500, 8632, 8396] dup_rows_per_dst=[2764, 7752, 3690, 1322, 4026, 172, 2088, 1492] combine_rows_per_src=[10242, 13080, 9366, 9578, 12204, 3500, 8632, 8396] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266919936 dispatch_meta_bytes=338288 combine_payload_bytes=266919936 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267258224 ingress_bytes=301522944 link_bytes=301522944 token_bytes=534178160 weight_movement_bytes=34603008 total_nvlink_bytes=568781168 weight_to_token_ratio=0.064778 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134606564e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=264142848 dispatch_meta_bytes=334376 combine_payload_bytes=264142848 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=264477224 ingress_bytes=437157888 link_bytes=437157888 token_bytes=528620072 weight_movement_bytes=173015040 total_nvlink_bytes=701635112 weight_to_token_ratio=0.327296 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=11648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.644989938e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=274276352 dispatch_meta_bytes=340104 combine_payload_bytes=274276352 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=274616456 ingress_bytes=343482368 link_bytes=343482368 token_bytes=548892808 weight_movement_bytes=69206016 total_nvlink_bytes=618098824 weight_to_token_ratio=0.126083 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=16768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.292496498e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77316 payload_rows_remote=67592 dup_rows=20988 dup_rows_remote=18400 dedup_ratio=0.213501 router_tokens_per_expert=[182, 416, 1265, 144, 488, 710, 1116, 841, 2564, 2981, 262, 298, 982, 424, 2338, 361, 2016, 34, 2224, 397, 591, 350, 2, 434, 765, 1281, 614, 419, 425, 1196, 1508, 850, 455, 518, 179, 201, 1254, 443, 1166, 546, 238, 2479, 120, 478, 200, 357, 377, 3699, 4, 224, 178, 1027, 409, 1082, 80, 171, 162, 77, 2010, 621, 728, 859, 68, 264] payload_rows_per_dst=[8524, 8304, 9034, 11176, 7878, 11358, 9724, 11318] dup_rows_per_dst=[1800, 1026, 3062, 2940, 1646, 4538, 2588, 3388] combine_rows_per_src=[8524, 8304, 9034, 11176, 7878, 11358, 9724, 11318] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74884 payload_rows_remote=64142 dup_rows=23420 dup_rows_remote=19498 dedup_ratio=0.238241 router_tokens_per_expert=[0, 1487, 1314, 1100, 1308, 1848, 659, 1329, 179, 1353, 2784, 62, 156, 293, 262, 751, 1992, 307, 0, 56, 111, 756, 291, 116, 943, 0, 2825, 343, 3215, 542, 526, 0, 167, 0, 0, 636, 882, 16, 68, 1066, 51, 314, 9, 1018, 312, 0, 3401, 0, 3, 907, 242, 597, 297, 1949, 1169, 1024, 622, 615, 55, 1875, 1264, 1492, 1237, 956] payload_rows_per_dst=[9986, 10742, 11012, 8268, 8868, 11608, 7132, 7268] dup_rows_per_dst=[5648, 3922, 2676, 2090, 3156, 2352, 1346, 2230] combine_rows_per_src=[9986, 10742, 11012, 8268, 8868, 11608, 7132, 7268] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265240576 dispatch_meta_bytes=341192 combine_payload_bytes=265240576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265581768 ingress_bytes=299843584 link_bytes=299843584 token_bytes=530822344 weight_movement_bytes=34603008 total_nvlink_bytes=565425352 weight_to_token_ratio=0.065188 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=11008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128287267e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=276856832 dispatch_meta_bytes=343968 combine_payload_bytes=276856832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277200800 ingress_bytes=311459840 link_bytes=311459840 token_bytes=554057632 weight_movement_bytes=34603008 total_nvlink_bytes=588660640 weight_to_token_ratio=0.062454 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.171998303e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=262725632 dispatch_meta_bytes=334560 combine_payload_bytes=262725632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263060192 ingress_bytes=297328640 link_bytes=297328640 token_bytes=525785824 weight_movement_bytes=34603008 total_nvlink_bytes=560388832 weight_to_token_ratio=0.065812 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.118823735e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=64268 dup_rows=23906 dup_rows_remote=23176 dedup_ratio=0.243184 router_tokens_per_expert=[0, 1224, 1303, 1223, 1098, 1779, 430, 1454, 148, 1238, 2484, 51, 118, 271, 256, 623, 2003, 374, 0, 8, 103, 959, 413, 119, 826, 0, 3227, 273, 3742, 462, 393, 0, 172, 0, 0, 546, 706, 5, 45, 981, 49, 239, 0, 1007, 272, 0, 3863, 0, 0, 1109, 226, 597, 332, 2371, 1394, 1064, 605, 565, 33, 1692, 1182, 1323, 1049, 1123] payload_rows_per_dst=[7400, 8518, 8098, 11112, 8412, 10130, 10374, 10354] dup_rows_per_dst=[3156, 1860, 940, 5654, 2964, 730, 3812, 4790] combine_rows_per_src=[7400, 8518, 8098, 11112, 8412, 10130, 10374, 10354] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77224 payload_rows_remote=67748 dup_rows=21080 dup_rows_remote=19706 dedup_ratio=0.214437 router_tokens_per_expert=[194, 233, 1433, 67, 505, 836, 1070, 916, 2578, 3013, 357, 372, 1342, 393, 2326, 344, 1852, 32, 2484, 413, 924, 368, 1, 530, 634, 1650, 613, 358, 307, 1424, 1211, 439, 447, 527, 97, 179, 1217, 333, 1002, 420, 111, 2751, 118, 466, 137, 273, 146, 3918, 2, 181, 97, 1216, 405, 1047, 8, 58, 113, 49, 2009, 590, 991, 755, 83, 187] payload_rows_per_dst=[9834, 9674, 9580, 9476, 7652, 10794, 9644, 10570] dup_rows_per_dst=[3096, 1888, 3628, 1374, 1568, 4270, 2410, 2846] combine_rows_per_src=[9834, 9674, 9580, 9476, 7652, 10794, 9644, 10570] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72910 payload_rows_remote=62012 dup_rows=25394 dup_rows_remote=19250 dedup_ratio=0.258321 router_tokens_per_expert=[0, 1647, 1468, 1057, 1138, 1769, 670, 1060, 186, 1210, 2575, 59, 152, 280, 245, 920, 1935, 354, 0, 56, 125, 939, 280, 185, 962, 0, 2366, 460, 2721, 690, 670, 1, 184, 0, 3, 498, 1134, 50, 89, 1025, 70, 258, 22, 1009, 518, 0, 3132, 0, 30, 956, 301, 631, 359, 1921, 1152, 1109, 929, 594, 65, 1699, 1166, 1530, 1434, 1104] payload_rows_per_dst=[7238, 9076, 7072, 10606, 9356, 9204, 9460, 10898] dup_rows_per_dst=[3316, 2178, 676, 5134, 3674, 814, 3458, 6144] combine_rows_per_src=[7238, 9076, 7072, 10606, 9356, 9204, 9460, 10898] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76156 payload_rows_remote=68024 dup_rows=22148 dup_rows_remote=20312 dedup_ratio=0.225301 router_tokens_per_expert=[287, 618, 1236, 149, 405, 465, 1190, 819, 1834, 3078, 223, 387, 1044, 351, 1709, 445, 1778, 56, 2504, 468, 558, 348, 6, 572, 1081, 1040, 621, 441, 423, 1146, 1574, 693, 436, 748, 160, 174, 1151, 446, 1374, 495, 378, 2297, 118, 462, 288, 459, 339, 3626, 28, 223, 212, 1073, 552, 1222, 180, 214, 190, 130, 2042, 576, 876, 781, 136, 217] payload_rows_per_dst=[10460, 7298, 9406, 8998, 8132, 11200, 10090, 10572] dup_rows_per_dst=[3026, 1020, 3174, 1892, 1836, 4734, 3474, 2992] combine_rows_per_src=[10460, 7298, 9406, 8998, 8132, 11200, 10090, 10572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75986 payload_rows_remote=65166 dup_rows=22318 dup_rows_remote=19406 dedup_ratio=0.227030 router_tokens_per_expert=[279, 532, 1325, 97, 420, 403, 1093, 819, 1856, 2920, 246, 372, 1191, 353, 1589, 466, 1746, 48, 2367, 417, 716, 271, 5, 612, 1141, 1170, 660, 373, 417, 1211, 1461, 618, 512, 539, 146, 190, 1104, 592, 1405, 522, 325, 2513, 90, 485, 316, 434, 350, 3879, 9, 194, 184, 1103, 510, 1244, 151, 151, 115, 145, 2002, 623, 1022, 768, 117, 218] payload_rows_per_dst=[8342, 7378, 6680, 12600, 8334, 12070, 9762, 10820] dup_rows_per_dst=[1594, 1056, 956, 5580, 1686, 5364, 3170, 2912] combine_rows_per_src=[8342, 7378, 6680, 12600, 8334, 12070, 9762, 10820] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=62738 dup_rows=23920 dup_rows_remote=20032 dedup_ratio=0.243327 router_tokens_per_expert=[0, 1192, 1375, 1497, 1157, 1886, 426, 2079, 140, 1632, 2385, 51, 138, 265, 245, 417, 2445, 301, 0, 6, 103, 1138, 247, 111, 1117, 0, 2192, 280, 3416, 372, 462, 0, 189, 0, 0, 645, 770, 0, 52, 914, 48, 341, 1, 1001, 375, 0, 3735, 0, 2, 1179, 224, 551, 219, 2408, 1539, 981, 457, 586, 26, 1610, 1003, 1316, 876, 1029] payload_rows_per_dst=[8100, 9890, 11646, 8178, 9184, 11812, 9154, 6420] dup_rows_per_dst=[3650, 3288, 3888, 2242, 3886, 2410, 3022, 1534] combine_rows_per_src=[8100, 9890, 11646, 8178, 9184, 11812, 9154, 6420] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=263241728 dispatch_meta_bytes=349776 combine_payload_bytes=263241728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263591504 ingress_bytes=297844736 link_bytes=297844736 token_bytes=526833232 weight_movement_bytes=34603008 total_nvlink_bytes=561436240 weight_to_token_ratio=0.065681 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=28032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.120765763e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74998 payload_rows_remote=64756 dup_rows=23306 dup_rows_remote=20542 dedup_ratio=0.237081 router_tokens_per_expert=[254, 411, 1313, 126, 477, 521, 1175, 813, 2220, 2903, 330, 352, 999, 430, 2337, 403, 2059, 61, 2175, 454, 803, 397, 1, 578, 707, 1409, 633, 420, 423, 1247, 1413, 611, 512, 494, 126, 171, 1158, 430, 1275, 454, 245, 2688, 131, 505, 275, 319, 361, 3495, 6, 225, 146, 1150, 391, 984, 99, 113, 130, 107, 2050, 643, 909, 797, 92, 216] payload_rows_per_dst=[10242, 13080, 9366, 9578, 12204, 3500, 8632, 8396] dup_rows_per_dst=[2764, 7752, 3690, 1322, 4026, 172, 2088, 1492] combine_rows_per_src=[10242, 13080, 9366, 9578, 12204, 3500, 8632, 8396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266289152 dispatch_meta_bytes=343856 combine_payload_bytes=266289152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266633008 ingress_bytes=300892160 link_bytes=300892160 token_bytes=532922160 weight_movement_bytes=34603008 total_nvlink_bytes=567525168 weight_to_token_ratio=0.064931 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132232974e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77712 payload_rows_remote=66602 dup_rows=20592 dup_rows_remote=16418 dedup_ratio=0.209473 router_tokens_per_expert=[168, 263, 1403, 76, 494, 602, 1146, 858, 2651, 3007, 342, 355, 1024, 434, 2646, 343, 2076, 35, 2275, 413, 829, 419, 1, 522, 535, 1539, 563, 396, 467, 1164, 1395, 603, 522, 509, 104, 134, 1188, 385, 1164, 506, 164, 2665, 134, 433, 236, 281, 345, 3384, 1, 172, 145, 1081, 564, 1020, 41, 130, 133, 66, 1978, 573, 955, 812, 74, 209] payload_rows_per_dst=[9452, 9006, 9560, 9964, 7480, 11110, 9782, 11358] dup_rows_per_dst=[2322, 1282, 3580, 1606, 1544, 4174, 2540, 3544] combine_rows_per_src=[9452, 9006, 9560, 9964, 7480, 11110, 9782, 11358] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266821632 dispatch_meta_bytes=334128 combine_payload_bytes=266821632 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267155760 ingress_bytes=370630656 link_bytes=370630656 token_bytes=533977392 weight_movement_bytes=103809024 total_nvlink_bytes=637786416 weight_to_token_ratio=0.194407 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=17536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394653320e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=276856832 dispatch_meta_bytes=343968 combine_payload_bytes=276856832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277200800 ingress_bytes=311459840 link_bytes=311459840 token_bytes=554057632 weight_movement_bytes=34603008 total_nvlink_bytes=588660640 weight_to_token_ratio=0.062454 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.171998303e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277495808 dispatch_meta_bytes=349816 combine_payload_bytes=277495808 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=277845624 ingress_bytes=346701824 link_bytes=346701824 token_bytes=555341432 weight_movement_bytes=69206016 total_nvlink_bytes=624547448 weight_to_token_ratio=0.124619 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.304611052e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=254001152 dispatch_meta_bytes=325048 combine_payload_bytes=254001152 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=254326200 ingress_bytes=357810176 link_bytes=357810176 token_bytes=508327352 weight_movement_bytes=103809024 total_nvlink_bytes=612136376 weight_to_token_ratio=0.204217 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=9344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.346410886e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74982 payload_rows_remote=64786 dup_rows=23322 dup_rows_remote=17904 dedup_ratio=0.237244 router_tokens_per_expert=[0, 1385, 1447, 1109, 1202, 2096, 646, 1498, 221, 1000, 2428, 61, 147, 335, 251, 624, 1937, 366, 0, 32, 122, 780, 345, 108, 913, 2, 2706, 328, 3348, 536, 484, 0, 177, 0, 1, 603, 863, 16, 55, 1009, 56, 268, 8, 1054, 394, 0, 3558, 0, 11, 1056, 256, 557, 431, 2018, 1217, 1029, 642, 693, 49, 1466, 1357, 1344, 1277, 1230] payload_rows_per_dst=[10196, 9698, 11108, 7938, 9138, 11484, 7886, 7534] dup_rows_per_dst=[5418, 3124, 2968, 2000, 3498, 2124, 1626, 2564] combine_rows_per_src=[10196, 9698, 11108, 7938, 9138, 11484, 7886, 7534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278626304 dispatch_meta_bytes=353344 combine_payload_bytes=278626304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278979648 ingress_bytes=313229312 link_bytes=313229312 token_bytes=557605952 weight_movement_bytes=34603008 total_nvlink_bytes=592208960 weight_to_token_ratio=0.062056 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=11264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.178656684e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266919936 dispatch_meta_bytes=338288 combine_payload_bytes=266919936 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267258224 ingress_bytes=301522944 link_bytes=301522944 token_bytes=534178160 weight_movement_bytes=34603008 total_nvlink_bytes=568781168 weight_to_token_ratio=0.064778 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134606564e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265240576 dispatch_meta_bytes=341192 combine_payload_bytes=265240576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265581768 ingress_bytes=299843584 link_bytes=299843584 token_bytes=530822344 weight_movement_bytes=34603008 total_nvlink_bytes=565425352 weight_to_token_ratio=0.065188 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=11008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128287267e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=272801792 dispatch_meta_bytes=332080 combine_payload_bytes=272801792 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=273133872 ingress_bytes=376610816 link_bytes=376610816 token_bytes=545935664 weight_movement_bytes=103809024 total_nvlink_bytes=649744688 weight_to_token_ratio=0.190149 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.417156181e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76634 payload_rows_remote=66962 dup_rows=21670 dup_rows_remote=18064 dedup_ratio=0.220439 router_tokens_per_expert=[167, 273, 1592, 50, 527, 694, 1212, 916, 2282, 2930, 243, 391, 1245, 289, 2425, 509, 1960, 18, 2484, 417, 885, 327, 1, 547, 557, 1548, 635, 315, 276, 1332, 1531, 503, 429, 912, 129, 209, 1244, 381, 945, 588, 177, 2622, 123, 506, 145, 396, 79, 3620, 3, 190, 128, 1109, 318, 1171, 51, 91, 156, 46, 2026, 577, 844, 576, 92, 188] payload_rows_per_dst=[10306, 11402, 9672, 9148, 12196, 6534, 9528, 7848] dup_rows_per_dst=[3618, 3366, 3606, 1184, 4718, 1562, 2454, 1162] combine_rows_per_src=[10306, 11402, 9672, 9148, 12196, 6534, 9528, 7848] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73890 payload_rows_remote=63790 dup_rows=24414 dup_rows_remote=20946 dedup_ratio=0.248352 router_tokens_per_expert=[0, 1384, 1370, 1038, 1144, 2077, 652, 1551, 241, 1174, 2471, 42, 94, 268, 249, 548, 2201, 365, 8, 44, 147, 876, 265, 132, 1039, 4, 2383, 323, 3110, 647, 410, 1, 211, 2, 2, 620, 881, 7, 51, 1163, 56, 372, 33, 1139, 395, 1, 3489, 1, 13, 911, 305, 660, 423, 2193, 1339, 940, 783, 495, 43, 1462, 1184, 1330, 1194, 1196] payload_rows_per_dst=[7458, 8238, 7376, 10664, 9618, 10080, 10100, 10356] dup_rows_per_dst=[3718, 1936, 700, 5170, 3512, 892, 3468, 5018] combine_rows_per_src=[7458, 8238, 7376, 10664, 9618, 10080, 10100, 10356] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=256974848 dispatch_meta_bytes=331080 combine_payload_bytes=256974848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=257305928 ingress_bytes=291577856 link_bytes=291577856 token_bytes=514280776 weight_movement_bytes=34603008 total_nvlink_bytes=548883784 weight_to_token_ratio=0.067284 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.097183998e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78770 payload_rows_remote=66644 dup_rows=19534 dup_rows_remote=16066 dedup_ratio=0.198710 router_tokens_per_expert=[528, 2178, 512, 362, 62, 353, 2681, 454, 0, 0, 222, 657, 2312, 0, 0, 522, 395, 893, 5469, 26, 141, 961, 452, 0, 66, 4084, 1000, 0, 41, 0, 3574, 72, 2566, 420, 78, 388, 202, 166, 880, 296, 20, 0, 1, 1013, 85, 606, 2658, 740, 987, 1322, 1571, 1084, 19, 458, 0, 1147, 1015, 261, 445, 358, 181, 447, 613, 1108] payload_rows_per_dst=[10582, 12126, 13566, 8882, 8490, 8172, 9496, 7456] dup_rows_per_dst=[3678, 3468, 3808, 624, 1502, 2074, 2980, 1400] combine_rows_per_src=[10582, 12126, 13566, 8882, 8490, 8172, 9496, 7456] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 3, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=330760 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265694216 ingress_bytes=403775488 link_bytes=403775488 token_bytes=531057672 weight_movement_bytes=138412032 total_nvlink_bytes=669469704 weight_to_token_ratio=0.260635 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=11008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519374655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=261283840 dispatch_meta_bytes=338944 combine_payload_bytes=261283840 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=261622784 ingress_bytes=365092864 link_bytes=365092864 token_bytes=522906624 weight_movement_bytes=103809024 total_nvlink_bytes=626715648 weight_to_token_ratio=0.198523 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.373815055e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=272973824 dispatch_meta_bytes=330840 combine_payload_bytes=272973824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273304664 ingress_bytes=307576832 link_bytes=307576832 token_bytes=546278488 weight_movement_bytes=34603008 total_nvlink_bytes=580881496 weight_to_token_ratio=0.063343 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=19200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.157386857e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75644 payload_rows_remote=65272 dup_rows=22660 dup_rows_remote=18614 dedup_ratio=0.230509 router_tokens_per_expert=[408, 1968, 506, 381, 59, 335, 2493, 563, 0, 0, 192, 565, 2441, 0, 0, 770, 404, 823, 5652, 17, 56, 1117, 583, 0, 49, 3983, 1206, 0, 36, 0, 3308, 66, 2742, 385, 73, 307, 217, 27, 906, 313, 20, 0, 1, 1015, 61, 598, 2658, 1045, 1041, 1100, 1520, 1196, 9, 525, 0, 1086, 1053, 265, 78, 336, 166, 338, 559, 1531] payload_rows_per_dst=[8136, 7062, 5600, 12076, 11032, 10372, 10064, 11302] dup_rows_per_dst=[1668, 874, 400, 5220, 3488, 4046, 2890, 4074] combine_rows_per_src=[8136, 7062, 5600, 12076, 11032, 10372, 10064, 11302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74884 payload_rows_remote=64142 dup_rows=23420 dup_rows_remote=19498 dedup_ratio=0.238241 router_tokens_per_expert=[0, 1487, 1314, 1100, 1308, 1848, 659, 1329, 179, 1353, 2784, 62, 156, 293, 262, 751, 1992, 307, 0, 56, 111, 756, 291, 116, 943, 0, 2825, 343, 3215, 542, 526, 0, 167, 0, 0, 636, 882, 16, 68, 1066, 51, 314, 9, 1018, 312, 0, 3401, 0, 3, 907, 242, 597, 297, 1949, 1169, 1024, 622, 615, 55, 1875, 1264, 1492, 1237, 956] payload_rows_per_dst=[9986, 10742, 11012, 8268, 8868, 11608, 7132, 7268] dup_rows_per_dst=[5648, 3922, 2676, 2090, 3156, 2352, 1346, 2230] combine_rows_per_src=[9986, 10742, 11012, 8268, 8868, 11608, 7132, 7268] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75022 payload_rows_remote=66786 dup_rows=23282 dup_rows_remote=21614 dedup_ratio=0.236837 router_tokens_per_expert=[0, 1199, 1313, 1392, 1179, 1637, 258, 1604, 115, 1435, 2893, 50, 133, 266, 245, 322, 2319, 336, 0, 1, 91, 1178, 555, 170, 1105, 0, 2974, 198, 3998, 294, 381, 0, 154, 0, 0, 752, 672, 1, 36, 845, 38, 345, 3, 959, 296, 0, 3718, 0, 4, 1345, 168, 442, 285, 2554, 1491, 833, 523, 596, 19, 1496, 1120, 884, 881, 1051] payload_rows_per_dst=[7658, 9604, 12790, 8236, 8134, 11598, 10372, 6630] dup_rows_per_dst=[3024, 3082, 4506, 1668, 3268, 2112, 3872, 1750] combine_rows_per_src=[7658, 9604, 12790, 8236, 8134, 11598, 10372, 6630] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74934 payload_rows_remote=60912 dup_rows=23370 dup_rows_remote=17710 dedup_ratio=0.237732 router_tokens_per_expert=[614, 1783, 556, 322, 69, 381, 2885, 432, 1, 0, 297, 649, 2188, 0, 0, 842, 452, 1035, 5252, 28, 267, 759, 424, 0, 100, 3511, 957, 0, 78, 0, 3541, 125, 2729, 447, 70, 459, 310, 210, 958, 517, 36, 0, 7, 1332, 63, 498, 2638, 546, 1035, 1258, 1122, 1112, 28, 523, 0, 1117, 1032, 273, 189, 481, 225, 457, 605, 1327] payload_rows_per_dst=[6848, 7036, 5426, 11770, 9302, 10942, 9588, 14022] dup_rows_per_dst=[1466, 918, 504, 4854, 2098, 5068, 2802, 5660] combine_rows_per_src=[6848, 7036, 5426, 11770, 9302, 10942, 9588, 14022] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73826 payload_rows_remote=65074 dup_rows=24478 dup_rows_remote=21594 dedup_ratio=0.249003 router_tokens_per_expert=[1, 1690, 1413, 1136, 1266, 1763, 663, 1083, 260, 1381, 2697, 75, 176, 282, 254, 827, 1849, 353, 0, 101, 138, 846, 286, 131, 963, 0, 1951, 391, 2720, 792, 575, 0, 216, 0, 4, 503, 1077, 34, 64, 1074, 84, 283, 26, 1038, 406, 0, 3021, 0, 58, 903, 312, 765, 385, 2043, 1010, 1135, 989, 523, 76, 1612, 1253, 1655, 1288, 1252] payload_rows_per_dst=[7930, 10038, 10456, 7258, 8752, 9654, 9868, 9870] dup_rows_per_dst=[4408, 3110, 2392, 2086, 2884, 1300, 3354, 4944] combine_rows_per_src=[7930, 10038, 10456, 7258, 8752, 9654, 9868, 9870] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78828 payload_rows_remote=64754 dup_rows=19476 dup_rows_remote=15612 dedup_ratio=0.198120 router_tokens_per_expert=[316, 2097, 435, 228, 88, 542, 2276, 576, 0, 0, 206, 635, 2765, 0, 0, 769, 367, 894, 5783, 16, 45, 1412, 452, 0, 78, 3321, 1015, 1, 38, 0, 3562, 43, 3241, 421, 116, 313, 127, 29, 622, 371, 21, 0, 0, 864, 53, 617, 2933, 977, 1026, 1361, 1286, 1229, 20, 450, 0, 1106, 1101, 260, 123, 282, 168, 315, 529, 1231] payload_rows_per_dst=[10290, 11562, 14074, 8836, 8934, 8668, 9898, 6566] dup_rows_per_dst=[2826, 3830, 3864, 638, 1546, 2262, 3058, 1452] combine_rows_per_src=[10290, 11562, 14074, 8836, 8934, 8668, 9898, 6566] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267354112 dispatch_meta_bytes=335544 combine_payload_bytes=267354112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267689656 ingress_bytes=301957120 link_bytes=301957120 token_bytes=535043768 weight_movement_bytes=34603008 total_nvlink_bytes=569646776 weight_to_token_ratio=0.064673 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=21632 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136240333e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74982 payload_rows_remote=64786 dup_rows=23322 dup_rows_remote=17904 dedup_ratio=0.237244 router_tokens_per_expert=[0, 1385, 1447, 1109, 1202, 2096, 646, 1498, 221, 1000, 2428, 61, 147, 335, 251, 624, 1937, 366, 0, 32, 122, 780, 345, 108, 913, 2, 2706, 328, 3348, 536, 484, 0, 177, 0, 1, 603, 863, 16, 55, 1009, 56, 268, 8, 1054, 394, 0, 3558, 0, 11, 1056, 256, 557, 431, 2018, 1217, 1029, 642, 693, 49, 1466, 1357, 1344, 1277, 1230] payload_rows_per_dst=[10196, 9698, 11108, 7938, 9138, 11484, 7886, 7534] dup_rows_per_dst=[5418, 3124, 2968, 2000, 3498, 2124, 1626, 2564] combine_rows_per_src=[10196, 9698, 11108, 7938, 9138, 11484, 7886, 7534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=274276352 dispatch_meta_bytes=340104 combine_payload_bytes=274276352 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=274616456 ingress_bytes=343482368 link_bytes=343482368 token_bytes=548892808 weight_movement_bytes=69206016 total_nvlink_bytes=618098824 weight_to_token_ratio=0.126083 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=16768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.292496498e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72910 payload_rows_remote=62012 dup_rows=25394 dup_rows_remote=19250 dedup_ratio=0.258321 router_tokens_per_expert=[0, 1647, 1468, 1057, 1138, 1769, 670, 1060, 186, 1210, 2575, 59, 152, 280, 245, 920, 1935, 354, 0, 56, 125, 939, 280, 185, 962, 0, 2366, 460, 2721, 690, 670, 1, 184, 0, 3, 498, 1134, 50, 89, 1025, 70, 258, 22, 1009, 518, 0, 3132, 0, 30, 956, 301, 631, 359, 1921, 1152, 1109, 929, 594, 65, 1699, 1166, 1530, 1434, 1104] payload_rows_per_dst=[7238, 9076, 7072, 10606, 9356, 9204, 9460, 10898] dup_rows_per_dst=[3316, 2178, 676, 5134, 3674, 814, 3458, 6144] combine_rows_per_src=[7238, 9076, 7072, 10606, 9356, 9204, 9460, 10898] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=261283840 dispatch_meta_bytes=338944 combine_payload_bytes=261283840 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=261622784 ingress_bytes=365092864 link_bytes=365092864 token_bytes=522906624 weight_movement_bytes=103809024 total_nvlink_bytes=626715648 weight_to_token_ratio=0.198523 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.373815055e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=262725632 dispatch_meta_bytes=334560 combine_payload_bytes=262725632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263060192 ingress_bytes=297328640 link_bytes=297328640 token_bytes=525785824 weight_movement_bytes=34603008 total_nvlink_bytes=560388832 weight_to_token_ratio=0.065812 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.118823735e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=3 slot_utilization=0.375000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[6, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=64268 dup_rows=23906 dup_rows_remote=23176 dedup_ratio=0.243184 router_tokens_per_expert=[0, 1224, 1303, 1223, 1098, 1779, 430, 1454, 148, 1238, 2484, 51, 118, 271, 256, 623, 2003, 374, 0, 8, 103, 959, 413, 119, 826, 0, 3227, 273, 3742, 462, 393, 0, 172, 0, 0, 546, 706, 5, 45, 981, 49, 239, 0, 1007, 272, 0, 3863, 0, 0, 1109, 226, 597, 332, 2371, 1394, 1064, 605, 565, 33, 1692, 1182, 1323, 1049, 1123] payload_rows_per_dst=[7400, 8518, 8098, 11112, 8412, 10130, 10374, 10354] dup_rows_per_dst=[3156, 1860, 940, 5654, 2964, 730, 3812, 4790] combine_rows_per_src=[7400, 8518, 8098, 11112, 8412, 10130, 10374, 10354] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 3, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=273555456 dispatch_meta_bytes=353600 combine_payload_bytes=273555456 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=273909056 ingress_bytes=342761472 link_bytes=342761472 token_bytes=547464512 weight_movement_bytes=69206016 total_nvlink_bytes=616670528 weight_to_token_ratio=0.126412 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.289783825e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75010 payload_rows_remote=63474 dup_rows=23294 dup_rows_remote=17572 dedup_ratio=0.236959 router_tokens_per_expert=[468, 1744, 517, 323, 73, 325, 2769, 624, 0, 0, 277, 655, 2198, 0, 0, 787, 457, 930, 5249, 26, 75, 1023, 525, 0, 86, 3916, 1220, 0, 38, 0, 3406, 91, 2536, 486, 84, 333, 296, 156, 831, 360, 41, 0, 5, 1196, 53, 588, 2668, 696, 1021, 1112, 1598, 1219, 9, 499, 0, 1165, 1014, 250, 147, 358, 214, 399, 591, 1425] payload_rows_per_dst=[11536, 11732, 13998, 6222, 8636, 8334, 7396, 7156] dup_rows_per_dst=[5722, 3934, 5768, 474, 1528, 2160, 2068, 1640] combine_rows_per_src=[11536, 11732, 13998, 6222, 8636, 8334, 7396, 7156] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=249495552 dispatch_meta_bytes=314488 combine_payload_bytes=249495552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=249810040 ingress_bytes=284098560 link_bytes=284098560 token_bytes=499305592 weight_movement_bytes=34603008 total_nvlink_bytes=533908600 weight_to_token_ratio=0.069302 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.069040009e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=346672 combine_payload_bytes=266543104 prefetch_bytes=103809024 reduce_bytes=0 egress_bytes=266889776 ingress_bytes=370352128 link_bytes=370352128 token_bytes=533432880 weight_movement_bytes=103809024 total_nvlink_bytes=637241904 weight_to_token_ratio=0.194606 gemm_rows_defined=50176 gemm_rows_charged=51200 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020000 gemm_active_groups=12 gemm_peak_group_rows=14976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393605242e-04 prefetch_bytes_per_gemm_flop=3.906250000e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=330760 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265694216 ingress_bytes=403775488 link_bytes=403775488 token_bytes=531057672 weight_movement_bytes=138412032 total_nvlink_bytes=669469704 weight_to_token_ratio=0.260635 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=11008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519374655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=254001152 dispatch_meta_bytes=325048 combine_payload_bytes=254001152 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=254326200 ingress_bytes=357810176 link_bytes=357810176 token_bytes=508327352 weight_movement_bytes=103809024 total_nvlink_bytes=612136376 weight_to_token_ratio=0.204217 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=9344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.346410886e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=263241728 dispatch_meta_bytes=349776 combine_payload_bytes=263241728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263591504 ingress_bytes=297844736 link_bytes=297844736 token_bytes=526833232 weight_movement_bytes=34603008 total_nvlink_bytes=561436240 weight_to_token_ratio=0.065681 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=28032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.120765763e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=62738 dup_rows=23920 dup_rows_remote=20032 dedup_ratio=0.243327 router_tokens_per_expert=[0, 1192, 1375, 1497, 1157, 1886, 426, 2079, 140, 1632, 2385, 51, 138, 265, 245, 417, 2445, 301, 0, 6, 103, 1138, 247, 111, 1117, 0, 2192, 280, 3416, 372, 462, 0, 189, 0, 0, 645, 770, 0, 52, 914, 48, 341, 1, 1001, 375, 0, 3735, 0, 2, 1179, 224, 551, 219, 2408, 1539, 981, 457, 586, 26, 1610, 1003, 1316, 876, 1029] payload_rows_per_dst=[8100, 9890, 11646, 8178, 9184, 11812, 9154, 6420] dup_rows_per_dst=[3650, 3288, 3888, 2242, 3886, 2410, 3022, 1534] combine_rows_per_src=[8100, 9890, 11646, 8178, 9184, 11812, 9154, 6420] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75638 payload_rows_remote=65834 dup_rows=22666 dup_rows_remote=19762 dedup_ratio=0.230570 router_tokens_per_expert=[621, 1814, 640, 435, 79, 324, 2755, 531, 1, 0, 258, 580, 2217, 0, 1, 887, 579, 874, 5289, 29, 91, 869, 561, 0, 129, 3645, 1045, 5, 100, 0, 3043, 102, 2857, 456, 84, 652, 234, 197, 1092, 350, 37, 1, 3, 992, 72, 627, 2384, 758, 1048, 1072, 1344, 1179, 23, 513, 1, 1174, 1011, 295, 214, 369, 209, 389, 625, 1386] payload_rows_per_dst=[7332, 7018, 5488, 11386, 9914, 10658, 9804, 14038] dup_rows_per_dst=[1556, 870, 518, 4752, 1930, 4600, 2904, 5536] combine_rows_per_src=[7332, 7018, 5488, 11386, 9914, 10658, 9804, 14038] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82828 payload_rows_remote=70698 dup_rows=15476 dup_rows_remote=12442 dedup_ratio=0.157430 router_tokens_per_expert=[795, 97, 0, 5290, 72, 0, 0, 94, 0, 180, 247, 147, 849, 528, 427, 863, 0, 0, 453, 0, 0, 518, 375, 1714, 135, 53, 210, 0, 81, 379, 0, 0, 95, 2460, 299, 5, 0, 0, 0, 2657, 1082, 214, 208, 1996, 5559, 152, 1277, 4341, 410, 72, 0, 1526, 0, 2010, 6423, 0, 4492, 0, 0, 0, 0, 0, 308, 59] payload_rows_per_dst=[11728, 12130, 14304, 12500, 10894, 7866, 3842, 9564] dup_rows_per_dst=[968, 3034, 4662, 334, 4158, 1992, 174, 154] combine_rows_per_src=[11728, 12130, 14304, 12500, 10894, 7866, 3842, 9564] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265232384 dispatch_meta_bytes=321464 combine_payload_bytes=265232384 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265553848 ingress_bytes=369041408 link_bytes=369041408 token_bytes=530786232 weight_movement_bytes=103809024 total_nvlink_bytes=634595256 weight_to_token_ratio=0.195576 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=23296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388673108e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78770 payload_rows_remote=66644 dup_rows=19534 dup_rows_remote=16066 dedup_ratio=0.198710 router_tokens_per_expert=[528, 2178, 512, 362, 62, 353, 2681, 454, 0, 0, 222, 657, 2312, 0, 0, 522, 395, 893, 5469, 26, 141, 961, 452, 0, 66, 4084, 1000, 0, 41, 0, 3574, 72, 2566, 420, 78, 388, 202, 166, 880, 296, 20, 0, 1, 1013, 85, 606, 2658, 740, 987, 1322, 1571, 1084, 19, 458, 0, 1147, 1015, 261, 445, 358, 181, 447, 613, 1108] payload_rows_per_dst=[10582, 12126, 13566, 8882, 8490, 8172, 9496, 7456] dup_rows_per_dst=[3678, 3468, 3808, 624, 1502, 2074, 2980, 1400] combine_rows_per_src=[10582, 12126, 13566, 8882, 8490, 8172, 9496, 7456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=259989504 dispatch_meta_bytes=324184 combine_payload_bytes=259989504 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=260313688 ingress_bytes=329195520 link_bytes=329195520 token_bytes=520303192 weight_movement_bytes=69206016 total_nvlink_bytes=589509208 weight_to_token_ratio=0.133011 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.238736239e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=269656064 dispatch_meta_bytes=342384 combine_payload_bytes=269656064 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=269998448 ingress_bytes=338862080 link_bytes=338862080 token_bytes=539654512 weight_movement_bytes=69206016 total_nvlink_bytes=608860528 weight_to_token_ratio=0.128241 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=9472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.275110726e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=289579008 dispatch_meta_bytes=332560 combine_payload_bytes=289579008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289911568 ingress_bytes=324182016 link_bytes=324182016 token_bytes=579490576 weight_movement_bytes=34603008 total_nvlink_bytes=614093584 weight_to_token_ratio=0.059713 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=24064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.219870827e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81094 payload_rows_remote=72166 dup_rows=17210 dup_rows_remote=14086 dedup_ratio=0.175069 router_tokens_per_expert=[804, 80, 0, 5227, 58, 0, 0, 31, 0, 126, 207, 151, 849, 417, 373, 622, 0, 0, 514, 0, 0, 537, 445, 1336, 107, 59, 180, 0, 112, 77, 0, 0, 65, 2485, 235, 8, 0, 0, 0, 3148, 1041, 223, 198, 1706, 5550, 190, 1231, 4347, 365, 77, 0, 1492, 0, 2918, 6565, 0, 4572, 0, 0, 0, 0, 0, 399, 25] payload_rows_per_dst=[11386, 9486, 4770, 11892, 9770, 8928, 15080, 9782] dup_rows_per_dst=[1014, 1824, 894, 278, 2112, 3124, 7754, 210] combine_rows_per_src=[11386, 9486, 4770, 11892, 9770, 8928, 15080, 9782] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78602 payload_rows_remote=69780 dup_rows=19702 dup_rows_remote=18964 dedup_ratio=0.200419 router_tokens_per_expert=[277, 1839, 419, 471, 71, 258, 2341, 474, 0, 0, 224, 604, 2925, 0, 0, 601, 378, 786, 5642, 19, 53, 1239, 533, 0, 6, 4191, 1182, 0, 32, 0, 3536, 24, 2924, 398, 84, 172, 200, 39, 720, 329, 2, 0, 0, 938, 59, 663, 3091, 885, 1122, 1161, 1358, 1247, 8, 471, 0, 1080, 1067, 252, 135, 258, 187, 313, 549, 1315] payload_rows_per_dst=[9602, 12436, 13812, 8822, 8396, 9068, 9780, 6686] dup_rows_per_dst=[2698, 4654, 3488, 738, 1336, 2208, 3114, 1466] combine_rows_per_src=[9602, 12436, 13812, 8822, 8396, 9068, 9780, 6686] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76234 payload_rows_remote=62770 dup_rows=22070 dup_rows_remote=15362 dedup_ratio=0.224508 router_tokens_per_expert=[673, 1804, 632, 384, 77, 170, 2887, 555, 1, 0, 279, 632, 2027, 0, 0, 691, 531, 962, 5424, 26, 163, 763, 434, 0, 134, 3347, 942, 0, 63, 1, 3687, 93, 2756, 454, 86, 637, 296, 147, 1013, 389, 31, 0, 4, 1217, 79, 365, 2519, 621, 1018, 1246, 1115, 1103, 48, 489, 0, 1212, 953, 383, 394, 476, 254, 499, 661, 1305] payload_rows_per_dst=[10688, 9108, 6914, 10438, 13464, 7770, 9616, 8236] dup_rows_per_dst=[3676, 1676, 1076, 2572, 6708, 1902, 2846, 1614] combine_rows_per_src=[10688, 9108, 6914, 10438, 13464, 7770, 9616, 8236] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=295591936 dispatch_meta_bytes=345008 combine_payload_bytes=295591936 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=295936944 ingress_bytes=434003968 link_bytes=434003968 token_bytes=591528880 weight_movement_bytes=138412032 total_nvlink_bytes=729940912 weight_to_token_ratio=0.233990 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.633121991e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77276 payload_rows_remote=62556 dup_rows=21028 dup_rows_remote=12462 dedup_ratio=0.213908 router_tokens_per_expert=[766, 98, 0, 5005, 84, 0, 74, 88, 0, 161, 240, 139, 900, 633, 429, 969, 0, 0, 425, 0, 0, 458, 416, 2242, 104, 50, 334, 0, 102, 469, 0, 0, 88, 2539, 313, 3, 0, 0, 0, 2302, 1012, 134, 202, 2088, 5418, 185, 1421, 3786, 422, 86, 0, 1425, 2, 1897, 6421, 0, 4816, 0, 0, 0, 0, 0, 349, 57] payload_rows_per_dst=[10988, 6012, 6416, 9434, 8608, 12706, 8392, 14720] dup_rows_per_dst=[1068, 930, 666, 330, 1882, 6462, 1124, 8566] combine_rows_per_src=[10988, 6012, 6416, 9434, 8608, 12706, 8392, 14720] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=256974848 dispatch_meta_bytes=331080 combine_payload_bytes=256974848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=257305928 ingress_bytes=291577856 link_bytes=291577856 token_bytes=514280776 weight_movement_bytes=34603008 total_nvlink_bytes=548883784 weight_to_token_ratio=0.067284 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.097183998e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75010 payload_rows_remote=63474 dup_rows=23294 dup_rows_remote=17572 dedup_ratio=0.236959 router_tokens_per_expert=[468, 1744, 517, 323, 73, 325, 2769, 624, 0, 0, 277, 655, 2198, 0, 0, 787, 457, 930, 5249, 26, 75, 1023, 525, 0, 86, 3916, 1220, 0, 38, 0, 3406, 91, 2536, 486, 84, 333, 296, 156, 831, 360, 41, 0, 5, 1196, 53, 588, 2668, 696, 1021, 1112, 1598, 1219, 9, 499, 0, 1165, 1014, 250, 147, 358, 214, 399, 591, 1425] payload_rows_per_dst=[11536, 11732, 13998, 6222, 8636, 8334, 7396, 7156] dup_rows_per_dst=[5722, 3934, 5768, 474, 1528, 2160, 2068, 1640] combine_rows_per_src=[11536, 11732, 13998, 6222, 8636, 8334, 7396, 7156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=269656064 dispatch_meta_bytes=342384 combine_payload_bytes=269656064 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=269998448 ingress_bytes=338862080 link_bytes=338862080 token_bytes=539654512 weight_movement_bytes=69206016 total_nvlink_bytes=608860528 weight_to_token_ratio=0.128241 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=9472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.275110726e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74934 payload_rows_remote=60912 dup_rows=23370 dup_rows_remote=17710 dedup_ratio=0.237732 router_tokens_per_expert=[614, 1783, 556, 322, 69, 381, 2885, 432, 1, 0, 297, 649, 2188, 0, 0, 842, 452, 1035, 5252, 28, 267, 759, 424, 0, 100, 3511, 957, 0, 78, 0, 3541, 125, 2729, 447, 70, 459, 310, 210, 958, 517, 36, 0, 7, 1332, 63, 498, 2638, 546, 1035, 1258, 1122, 1112, 28, 523, 0, 1117, 1032, 273, 189, 481, 225, 457, 605, 1327] payload_rows_per_dst=[6848, 7036, 5426, 11770, 9302, 10942, 9588, 14022] dup_rows_per_dst=[1466, 918, 504, 4854, 2098, 5068, 2802, 5660] combine_rows_per_src=[6848, 7036, 5426, 11770, 9302, 10942, 9588, 14022] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=272973824 dispatch_meta_bytes=330840 combine_payload_bytes=272973824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273304664 ingress_bytes=307576832 link_bytes=307576832 token_bytes=546278488 weight_movement_bytes=34603008 total_nvlink_bytes=580881496 weight_to_token_ratio=0.063343 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=19200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.157386857e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75644 payload_rows_remote=65272 dup_rows=22660 dup_rows_remote=18614 dedup_ratio=0.230509 router_tokens_per_expert=[408, 1968, 506, 381, 59, 335, 2493, 563, 0, 0, 192, 565, 2441, 0, 0, 770, 404, 823, 5652, 17, 56, 1117, 583, 0, 49, 3983, 1206, 0, 36, 0, 3308, 66, 2742, 385, 73, 307, 217, 27, 906, 313, 20, 0, 1, 1015, 61, 598, 2658, 1045, 1041, 1100, 1520, 1196, 9, 525, 0, 1086, 1053, 265, 78, 336, 166, 338, 559, 1531] payload_rows_per_dst=[8136, 7062, 5600, 12076, 11032, 10372, 10064, 11302] dup_rows_per_dst=[1668, 874, 400, 5220, 3488, 4046, 2890, 4074] combine_rows_per_src=[8136, 7062, 5600, 12076, 11032, 10372, 10064, 11302] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84748 payload_rows_remote=70264 dup_rows=13556 dup_rows_remote=9516 dedup_ratio=0.137899 router_tokens_per_expert=[784, 85, 0, 5275, 58, 0, 0, 47, 0, 178, 169, 138, 828, 507, 382, 614, 0, 0, 206, 1, 0, 474, 372, 1730, 119, 88, 389, 0, 82, 73, 0, 0, 86, 2429, 295, 0, 0, 0, 0, 2758, 1040, 182, 176, 1804, 5622, 152, 1381, 4219, 415, 73, 0, 1617, 0, 2936, 6479, 0, 4483, 0, 0, 0, 0, 0, 389, 17] payload_rows_per_dst=[11514, 11772, 14484, 12376, 9214, 7540, 8280, 9568] dup_rows_per_dst=[984, 2298, 4040, 370, 2058, 1930, 1666, 210] combine_rows_per_src=[11514, 11772, 14484, 12376, 9214, 7540, 8280, 9568] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=259989504 dispatch_meta_bytes=324184 combine_payload_bytes=259989504 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=260313688 ingress_bytes=329195520 link_bytes=329195520 token_bytes=520303192 weight_movement_bytes=69206016 total_nvlink_bytes=589509208 weight_to_token_ratio=0.133011 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.238736239e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=285818880 dispatch_meta_bytes=354976 combine_payload_bytes=285818880 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=286173856 ingress_bytes=389627904 link_bytes=389627904 token_bytes=571992736 weight_movement_bytes=103809024 total_nvlink_bytes=675801760 weight_to_token_ratio=0.181487 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=25984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.466138435e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82262 payload_rows_remote=70926 dup_rows=16042 dup_rows_remote=15184 dedup_ratio=0.163188 router_tokens_per_expert=[732, 91, 0, 5112, 75, 0, 6, 81, 0, 170, 220, 115, 917, 511, 450, 887, 0, 0, 476, 0, 0, 468, 441, 1633, 121, 66, 230, 0, 87, 489, 0, 0, 82, 2489, 233, 4, 0, 0, 0, 3109, 1068, 205, 195, 1724, 5574, 163, 1330, 4037, 431, 78, 0, 1105, 0, 2616, 6477, 0, 4444, 0, 0, 0, 0, 0, 356, 54] payload_rows_per_dst=[11336, 11748, 14326, 12790, 11856, 7482, 3212, 9512] dup_rows_per_dst=[858, 2866, 4664, 344, 5210, 1888, 16, 196] combine_rows_per_src=[11336, 11748, 14326, 12790, 11856, 7482, 3212, 9512] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=257105920 dispatch_meta_bytes=312528 combine_payload_bytes=257105920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=257418448 ingress_bytes=291708928 link_bytes=291708928 token_bytes=514524368 weight_movement_bytes=34603008 total_nvlink_bytes=549127376 weight_to_token_ratio=0.067252 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=22400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.097677212e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80602 payload_rows_remote=76114 dup_rows=17702 dup_rows_remote=17032 dedup_ratio=0.180074 router_tokens_per_expert=[2214, 7341, 0, 118, 580, 0, 59, 0, 0, 131, 60, 785, 0, 338, 375, 890, 166, 1061, 0, 0, 279, 81, 343, 564, 4974, 4939, 762, 440, 0, 0, 44, 306, 0, 219, 503, 0, 73, 2447, 537, 2, 556, 2, 0, 53, 714, 6587, 6, 0, 6093, 0, 60, 1386, 406, 72, 0, 4, 0, 0, 0, 0, 0, 2234, 142, 206] payload_rows_per_dst=[15368, 4488, 12442, 11520, 14998, 2550, 14090, 5146] dup_rows_per_dst=[5256, 670, 2494, 1462, 5738, 112, 1952, 18] combine_rows_per_src=[15368, 4488, 12442, 11520, 14998, 2550, 14090, 5146] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=249495552 dispatch_meta_bytes=314488 combine_payload_bytes=249495552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=249810040 ingress_bytes=284098560 link_bytes=284098560 token_bytes=499305592 weight_movement_bytes=34603008 total_nvlink_bytes=533908600 weight_to_token_ratio=0.069302 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.069040009e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267354112 dispatch_meta_bytes=335544 combine_payload_bytes=267354112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267689656 ingress_bytes=301957120 link_bytes=301957120 token_bytes=535043768 weight_movement_bytes=34603008 total_nvlink_bytes=569646776 weight_to_token_ratio=0.064673 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=21632 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136240333e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78828 payload_rows_remote=64754 dup_rows=19476 dup_rows_remote=15612 dedup_ratio=0.198120 router_tokens_per_expert=[316, 2097, 435, 228, 88, 542, 2276, 576, 0, 0, 206, 635, 2765, 0, 0, 769, 367, 894, 5783, 16, 45, 1412, 452, 0, 78, 3321, 1015, 1, 38, 0, 3562, 43, 3241, 421, 116, 313, 127, 29, 622, 371, 21, 0, 0, 864, 53, 617, 2933, 977, 1026, 1361, 1286, 1229, 20, 450, 0, 1106, 1101, 260, 123, 282, 168, 315, 529, 1231] payload_rows_per_dst=[10290, 11562, 14074, 8836, 8934, 8668, 9898, 6566] dup_rows_per_dst=[2826, 3830, 3864, 638, 1546, 2262, 3058, 1452] combine_rows_per_src=[10290, 11562, 14074, 8836, 8934, 8668, 9898, 6566] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79754 payload_rows_remote=67686 dup_rows=18550 dup_rows_remote=14494 dedup_ratio=0.188700 router_tokens_per_expert=[819, 61, 0, 4770, 108, 0, 10, 49, 0, 202, 302, 325, 951, 524, 420, 698, 0, 0, 451, 1, 0, 543, 470, 1508, 96, 66, 339, 0, 154, 268, 0, 1, 49, 2385, 249, 1, 0, 1, 0, 2762, 1092, 217, 192, 2107, 5291, 247, 1410, 4205, 405, 110, 0, 1292, 78, 2252, 6467, 1, 4712, 0, 0, 1, 4, 0, 449, 37] payload_rows_per_dst=[10634, 5786, 5102, 12052, 9114, 12886, 12068, 12112] dup_rows_per_dst=[1000, 1058, 844, 378, 1780, 6054, 4056, 3380] combine_rows_per_src=[10634, 5786, 5102, 12052, 9114, 12886, 12068, 12112] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=256229376 dispatch_meta_bytes=300072 combine_payload_bytes=256229376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=256529448 ingress_bytes=290832384 link_bytes=290832384 token_bytes=512758824 weight_movement_bytes=34603008 total_nvlink_bytes=547361832 weight_to_token_ratio=0.067484 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=36992 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.094378847e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=287801344 dispatch_meta_bytes=319120 combine_payload_bytes=287801344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288120464 ingress_bytes=322404352 link_bytes=322404352 token_bytes=575921808 weight_movement_bytes=34603008 total_nvlink_bytes=610524816 weight_to_token_ratio=0.060083 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=24960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.213181621e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82828 payload_rows_remote=70698 dup_rows=15476 dup_rows_remote=12442 dedup_ratio=0.157430 router_tokens_per_expert=[795, 97, 0, 5290, 72, 0, 0, 94, 0, 180, 247, 147, 849, 528, 427, 863, 0, 0, 453, 0, 0, 518, 375, 1714, 135, 53, 210, 0, 81, 379, 0, 0, 95, 2460, 299, 5, 0, 0, 0, 2657, 1082, 214, 208, 1996, 5559, 152, 1277, 4341, 410, 72, 0, 1526, 0, 2010, 6423, 0, 4492, 0, 0, 0, 0, 0, 308, 59] payload_rows_per_dst=[11728, 12130, 14304, 12500, 10894, 7866, 3842, 9564] dup_rows_per_dst=[968, 3034, 4662, 334, 4158, 1992, 174, 154] combine_rows_per_src=[11728, 12130, 14304, 12500, 10894, 7866, 3842, 9564] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82730 payload_rows_remote=69244 dup_rows=15574 dup_rows_remote=13594 dedup_ratio=0.158427 router_tokens_per_expert=[2286, 7460, 0, 60, 566, 0, 50, 0, 0, 149, 50, 590, 0, 176, 480, 907, 143, 1257, 0, 0, 265, 50, 154, 499, 5555, 4413, 786, 405, 0, 0, 0, 313, 0, 284, 335, 0, 79, 2910, 666, 4, 662, 3, 0, 59, 472, 6537, 0, 0, 5705, 0, 35, 1375, 296, 77, 0, 0, 0, 0, 0, 0, 0, 2672, 118, 249] payload_rows_per_dst=[5412, 15504, 9428, 8372, 7994, 13486, 9342, 13192] dup_rows_per_dst=[512, 4120, 2030, 1070, 562, 1980, 1004, 4296] combine_rows_per_src=[5412, 15504, 9428, 8372, 7994, 13486, 9342, 13192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=290512896 dispatch_meta_bytes=344440 combine_payload_bytes=290512896 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=290857336 ingress_bytes=325115904 link_bytes=325115904 token_bytes=581370232 weight_movement_bytes=34603008 total_nvlink_bytes=615973240 weight_to_token_ratio=0.059520 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=41088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.223384973e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=311762944 dispatch_meta_bytes=372584 combine_payload_bytes=311762944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=312135528 ingress_bytes=346365952 link_bytes=346365952 token_bytes=623898472 weight_movement_bytes=34603008 total_nvlink_bytes=658501480 weight_to_token_ratio=0.055463 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=28544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.303347193e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265232384 dispatch_meta_bytes=321464 combine_payload_bytes=265232384 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265553848 ingress_bytes=369041408 link_bytes=369041408 token_bytes=530786232 weight_movement_bytes=103809024 total_nvlink_bytes=634595256 weight_to_token_ratio=0.195576 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=23296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388673108e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82262 payload_rows_remote=70926 dup_rows=16042 dup_rows_remote=15184 dedup_ratio=0.163188 router_tokens_per_expert=[732, 91, 0, 5112, 75, 0, 6, 81, 0, 170, 220, 115, 917, 511, 450, 887, 0, 0, 476, 0, 0, 468, 441, 1633, 121, 66, 230, 0, 87, 489, 0, 0, 82, 2489, 233, 4, 0, 0, 0, 3109, 1068, 205, 195, 1724, 5574, 163, 1330, 4037, 431, 78, 0, 1105, 0, 2616, 6477, 0, 4444, 0, 0, 0, 0, 0, 356, 54] payload_rows_per_dst=[11336, 11748, 14326, 12790, 11856, 7482, 3212, 9512] dup_rows_per_dst=[858, 2866, 4664, 344, 5210, 1888, 16, 196] combine_rows_per_src=[11336, 11748, 14326, 12790, 11856, 7482, 3212, 9512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277241856 dispatch_meta_bytes=328720 combine_payload_bytes=277241856 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=277570576 ingress_bytes=415653888 link_bytes=415653888 token_bytes=554812432 weight_movement_bytes=138412032 total_nvlink_bytes=693224464 weight_to_token_ratio=0.249475 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=18176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.564072118e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=283623424 dispatch_meta_bytes=331352 combine_payload_bytes=283623424 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=283954776 ingress_bytes=352829440 link_bytes=352829440 token_bytes=567578200 weight_movement_bytes=69206016 total_nvlink_bytes=636784216 weight_to_token_ratio=0.121932 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=34688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.327668778e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84682 payload_rows_remote=72374 dup_rows=13622 dup_rows_remote=13328 dedup_ratio=0.138570 router_tokens_per_expert=[807, 75, 0, 5465, 27, 0, 0, 22, 0, 150, 132, 129, 825, 528, 397, 632, 0, 0, 300, 0, 0, 548, 400, 1488, 123, 71, 334, 0, 52, 64, 0, 0, 79, 2323, 269, 0, 0, 0, 0, 2939, 1020, 190, 188, 1817, 5657, 109, 1105, 4420, 391, 64, 0, 1606, 8, 2895, 6630, 0, 4537, 0, 0, 0, 0, 0, 328, 8] payload_rows_per_dst=[11810, 11976, 12944, 12308, 9314, 7210, 9542, 9578] dup_rows_per_dst=[982, 2450, 3490, 294, 1906, 1648, 2684, 168] combine_rows_per_src=[11810, 11976, 12944, 12308, 9314, 7210, 9542, 9578] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83352 payload_rows_remote=74924 dup_rows=14952 dup_rows_remote=12952 dedup_ratio=0.152100 router_tokens_per_expert=[759, 92, 0, 4942, 91, 0, 43, 90, 0, 177, 329, 264, 878, 573, 426, 779, 0, 0, 458, 0, 0, 456, 408, 2027, 122, 54, 320, 0, 134, 418, 0, 0, 94, 2382, 337, 0, 0, 1, 0, 2400, 1013, 155, 146, 2122, 5264, 223, 1635, 4041, 397, 93, 0, 1555, 43, 1683, 6435, 1, 4861, 0, 0, 0, 0, 0, 367, 64] payload_rows_per_dst=[11076, 11750, 5966, 12252, 8428, 8564, 14890, 10426] dup_rows_per_dst=[958, 3184, 732, 372, 2000, 2024, 5524, 158] combine_rows_per_src=[11076, 11750, 5966, 12252, 8428, 8564, 14890, 10426] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76466 payload_rows_remote=62908 dup_rows=21838 dup_rows_remote=18404 dedup_ratio=0.222148 router_tokens_per_expert=[1870, 7238, 0, 120, 579, 0, 57, 1, 0, 136, 82, 619, 0, 328, 422, 1129, 244, 869, 0, 0, 291, 94, 759, 646, 4380, 4921, 927, 555, 0, 0, 83, 325, 0, 230, 701, 0, 86, 1893, 615, 18, 585, 10, 0, 50, 769, 6735, 4, 0, 6077, 0, 209, 1285, 486, 91, 167, 47, 0, 0, 0, 0, 0, 2016, 165, 238] payload_rows_per_dst=[4750, 8608, 4398, 10894, 6580, 15064, 12614, 13558] dup_rows_per_dst=[504, 2458, 1408, 2202, 506, 9528, 1798, 3434] combine_rows_per_src=[4750, 8608, 4398, 10894, 6580, 15064, 12614, 13558] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81930 payload_rows_remote=69276 dup_rows=16374 dup_rows_remote=14050 dedup_ratio=0.166565 router_tokens_per_expert=[2532, 7444, 0, 76, 518, 0, 51, 0, 0, 138, 52, 769, 0, 278, 373, 774, 159, 953, 0, 0, 215, 51, 278, 550, 5283, 4702, 835, 406, 0, 0, 0, 339, 0, 266, 315, 0, 74, 2675, 504, 0, 520, 0, 0, 88, 443, 6610, 2, 0, 5936, 0, 15, 1466, 172, 76, 0, 0, 0, 0, 0, 0, 0, 2829, 141, 244] payload_rows_per_dst=[15934, 4110, 12654, 11072, 13024, 4916, 13832, 6388] dup_rows_per_dst=[5308, 658, 2324, 1492, 4494, 560, 1498, 40] combine_rows_per_src=[15934, 4110, 12654, 11072, 13024, 4916, 13832, 6388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84682 payload_rows_remote=72374 dup_rows=13622 dup_rows_remote=13328 dedup_ratio=0.138570 router_tokens_per_expert=[807, 75, 0, 5465, 27, 0, 0, 22, 0, 150, 132, 129, 825, 528, 397, 632, 0, 0, 300, 0, 0, 548, 400, 1488, 123, 71, 334, 0, 52, 64, 0, 0, 79, 2323, 269, 0, 0, 0, 0, 2939, 1020, 190, 188, 1817, 5657, 109, 1105, 4420, 391, 64, 0, 1606, 8, 2895, 6630, 0, 4537, 0, 0, 0, 0, 0, 328, 8] payload_rows_per_dst=[11810, 11976, 12944, 12308, 9314, 7210, 9542, 9578] dup_rows_per_dst=[982, 2450, 3490, 294, 1906, 1648, 2684, 168] combine_rows_per_src=[11810, 11976, 12944, 12308, 9314, 7210, 9542, 9578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77276 payload_rows_remote=62556 dup_rows=21028 dup_rows_remote=12462 dedup_ratio=0.213908 router_tokens_per_expert=[766, 98, 0, 5005, 84, 0, 74, 88, 0, 161, 240, 139, 900, 633, 429, 969, 0, 0, 425, 0, 0, 458, 416, 2242, 104, 50, 334, 0, 102, 469, 0, 0, 88, 2539, 313, 3, 0, 0, 0, 2302, 1012, 134, 202, 2088, 5418, 185, 1421, 3786, 422, 86, 0, 1425, 2, 1897, 6421, 0, 4816, 0, 0, 0, 0, 0, 349, 57] payload_rows_per_dst=[10988, 6012, 6416, 9434, 8608, 12706, 8392, 14720] dup_rows_per_dst=[1068, 930, 666, 330, 1882, 6462, 1124, 8566] combine_rows_per_src=[10988, 6012, 6416, 9434, 8608, 12706, 8392, 14720] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81094 payload_rows_remote=72166 dup_rows=17210 dup_rows_remote=14086 dedup_ratio=0.175069 router_tokens_per_expert=[804, 80, 0, 5227, 58, 0, 0, 31, 0, 126, 207, 151, 849, 417, 373, 622, 0, 0, 514, 0, 0, 537, 445, 1336, 107, 59, 180, 0, 112, 77, 0, 0, 65, 2485, 235, 8, 0, 0, 0, 3148, 1041, 223, 198, 1706, 5550, 190, 1231, 4347, 365, 77, 0, 1492, 0, 2918, 6565, 0, 4572, 0, 0, 0, 0, 0, 399, 25] payload_rows_per_dst=[11386, 9486, 4770, 11892, 9770, 8928, 15080, 9782] dup_rows_per_dst=[1014, 1824, 894, 278, 2112, 3124, 7754, 210] combine_rows_per_src=[11386, 9486, 4770, 11892, 9770, 8928, 15080, 9782] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=289579008 dispatch_meta_bytes=332560 combine_payload_bytes=289579008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289911568 ingress_bytes=324182016 link_bytes=324182016 token_bytes=579490576 weight_movement_bytes=34603008 total_nvlink_bytes=614093584 weight_to_token_ratio=0.059713 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=24064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.219870827e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=290512896 dispatch_meta_bytes=344440 combine_payload_bytes=290512896 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=290857336 ingress_bytes=325115904 link_bytes=325115904 token_bytes=581370232 weight_movement_bytes=34603008 total_nvlink_bytes=615973240 weight_to_token_ratio=0.059520 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=41088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.223384973e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=306888704 dispatch_meta_bytes=351504 combine_payload_bytes=306888704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307240208 ingress_bytes=341491712 link_bytes=341491712 token_bytes=614128912 weight_movement_bytes=34603008 total_nvlink_bytes=648731920 weight_to_token_ratio=0.056345 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=22144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.285005820e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79986 payload_rows_remote=70558 dup_rows=18318 dup_rows_remote=16462 dedup_ratio=0.186340 router_tokens_per_expert=[553, 72, 3811, 1125, 3272, 0, 6909, 5659, 0, 144, 0, 410, 0, 772, 0, 505, 313, 300, 2946, 72, 855, 58, 410, 150, 0, 72, 485, 0, 0, 131, 0, 481, 6248, 271, 0, 72, 208, 1366, 727, 0, 425, 0, 0, 0, 0, 0, 0, 245, 2521, 0, 0, 57, 0, 1214, 92, 0, 2565, 0, 2, 1861, 1693, 80, 0, 0] payload_rows_per_dst=[12704, 9428, 11112, 13228, 11084, 5850, 7166, 9414] dup_rows_per_dst=[6158, 1856, 3188, 428, 2608, 490, 602, 2988] combine_rows_per_src=[12704, 9428, 11112, 13228, 11084, 5850, 7166, 9414] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=296443904 dispatch_meta_bytes=342808 combine_payload_bytes=296443904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296786712 ingress_bytes=331046912 link_bytes=331046912 token_bytes=593230616 weight_movement_bytes=34603008 total_nvlink_bytes=627833624 weight_to_token_ratio=0.058330 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=42368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.245702878e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80304 payload_rows_remote=64940 dup_rows=18000 dup_rows_remote=13352 dedup_ratio=0.183105 router_tokens_per_expert=[1936, 7369, 0, 84, 558, 0, 59, 0, 0, 115, 60, 561, 0, 318, 461, 1084, 159, 1253, 0, 0, 269, 60, 334, 587, 4965, 4523, 831, 454, 0, 0, 9, 300, 0, 240, 415, 0, 77, 2549, 709, 4, 631, 1, 0, 66, 674, 6545, 2, 0, 5945, 0, 109, 1434, 431, 80, 0, 0, 0, 0, 0, 0, 0, 2534, 127, 230] payload_rows_per_dst=[15364, 4388, 12552, 10724, 15050, 2616, 13854, 5756] dup_rows_per_dst=[4648, 810, 2702, 1510, 6028, 132, 2144, 26] combine_rows_per_src=[15364, 4388, 12552, 10724, 15050, 2616, 13854, 5756] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=256229376 dispatch_meta_bytes=300072 combine_payload_bytes=256229376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=256529448 ingress_bytes=290832384 link_bytes=290832384 token_bytes=512758824 weight_movement_bytes=34603008 total_nvlink_bytes=547361832 weight_to_token_ratio=0.067484 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=36992 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.094378847e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=295591936 dispatch_meta_bytes=345008 combine_payload_bytes=295591936 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=295936944 ingress_bytes=434003968 link_bytes=434003968 token_bytes=591528880 weight_movement_bytes=138412032 total_nvlink_bytes=729940912 weight_to_token_ratio=0.233990 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.633121991e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84748 payload_rows_remote=70264 dup_rows=13556 dup_rows_remote=9516 dedup_ratio=0.137899 router_tokens_per_expert=[784, 85, 0, 5275, 58, 0, 0, 47, 0, 178, 169, 138, 828, 507, 382, 614, 0, 0, 206, 1, 0, 474, 372, 1730, 119, 88, 389, 0, 82, 73, 0, 0, 86, 2429, 295, 0, 0, 0, 0, 2758, 1040, 182, 176, 1804, 5622, 152, 1381, 4219, 415, 73, 0, 1617, 0, 2936, 6479, 0, 4483, 0, 0, 0, 0, 0, 389, 17] payload_rows_per_dst=[11514, 11772, 14484, 12376, 9214, 7540, 8280, 9568] dup_rows_per_dst=[984, 2298, 4040, 370, 2058, 1930, 1666, 210] combine_rows_per_src=[11514, 11772, 14484, 12376, 9214, 7540, 8280, 9568] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81844 payload_rows_remote=70154 dup_rows=16460 dup_rows_remote=14968 dedup_ratio=0.167440 router_tokens_per_expert=[2130, 7290, 1, 58, 623, 0, 37, 1, 0, 122, 92, 742, 22, 293, 489, 943, 186, 1221, 1, 0, 297, 37, 462, 577, 5044, 4417, 793, 410, 1, 1, 11, 305, 0, 276, 479, 0, 107, 2514, 680, 13, 711, 3, 0, 66, 776, 6280, 7, 1, 5851, 1, 138, 1409, 511, 108, 5, 2, 0, 0, 1, 0, 1, 2088, 191, 327] payload_rows_per_dst=[5178, 15294, 4352, 11472, 7472, 13194, 11690, 13192] dup_rows_per_dst=[522, 4692, 1210, 1658, 666, 2494, 1492, 3726] combine_rows_per_src=[5178, 15294, 4352, 11472, 7472, 13194, 11690, 13192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=257671168 dispatch_meta_bytes=325248 combine_payload_bytes=257671168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=257996416 ingress_bytes=292274176 link_bytes=292274176 token_bytes=515667584 weight_movement_bytes=34603008 total_nvlink_bytes=550270592 weight_to_token_ratio=0.067103 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=27008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.099804194e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=283754496 dispatch_meta_bytes=333304 combine_payload_bytes=283754496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=284087800 ingress_bytes=318357504 link_bytes=318357504 token_bytes=567842296 weight_movement_bytes=34603008 total_nvlink_bytes=602445304 weight_to_token_ratio=0.060938 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=28544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.197953658e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80602 payload_rows_remote=76114 dup_rows=17702 dup_rows_remote=17032 dedup_ratio=0.180074 router_tokens_per_expert=[2214, 7341, 0, 118, 580, 0, 59, 0, 0, 131, 60, 785, 0, 338, 375, 890, 166, 1061, 0, 0, 279, 81, 343, 564, 4974, 4939, 762, 440, 0, 0, 44, 306, 0, 219, 503, 0, 73, 2447, 537, 2, 556, 2, 0, 53, 714, 6587, 6, 0, 6093, 0, 60, 1386, 406, 72, 0, 4, 0, 0, 0, 0, 0, 2234, 142, 206] payload_rows_per_dst=[15368, 4488, 12442, 11520, 14998, 2550, 14090, 5146] dup_rows_per_dst=[5256, 670, 2494, 1462, 5738, 112, 1952, 18] combine_rows_per_src=[15368, 4488, 12442, 11520, 14998, 2550, 14090, 5146] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82444 payload_rows_remote=67672 dup_rows=15860 dup_rows_remote=15358 dedup_ratio=0.161336 router_tokens_per_expert=[609, 77, 3340, 1082, 3256, 0, 6992, 5447, 0, 155, 0, 442, 0, 767, 0, 532, 312, 362, 2854, 77, 807, 57, 365, 88, 1, 77, 435, 0, 0, 128, 0, 560, 6251, 249, 0, 78, 205, 1825, 796, 0, 394, 0, 0, 0, 0, 0, 0, 251, 2410, 0, 0, 59, 1, 1157, 153, 0, 2569, 0, 1, 2087, 1711, 133, 0, 0] payload_rows_per_dst=[5456, 10880, 8808, 12862, 5080, 14772, 14836, 9750] dup_rows_per_dst=[500, 3684, 1036, 434, 1226, 502, 5226, 3252] combine_rows_per_src=[5456, 10880, 8808, 12862, 5080, 14772, 14836, 9750] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=289005568 dispatch_meta_bytes=348080 combine_payload_bytes=289005568 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=289353648 ingress_bytes=358211584 link_bytes=358211584 token_bytes=578359216 weight_movement_bytes=69206016 total_nvlink_bytes=647565232 weight_to_token_ratio=0.119659 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=28928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.347921352e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=313168 combine_payload_bytes=265994240 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266307408 ingress_bytes=369803264 link_bytes=369803264 token_bytes=532301648 weight_movement_bytes=103809024 total_nvlink_bytes=636110672 weight_to_token_ratio=0.195019 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=26112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391539911e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=287801344 dispatch_meta_bytes=319120 combine_payload_bytes=287801344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288120464 ingress_bytes=322404352 link_bytes=322404352 token_bytes=575921808 weight_movement_bytes=34603008 total_nvlink_bytes=610524816 weight_to_token_ratio=0.060083 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=24960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.213181621e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80304 payload_rows_remote=64940 dup_rows=18000 dup_rows_remote=13352 dedup_ratio=0.183105 router_tokens_per_expert=[1936, 7369, 0, 84, 558, 0, 59, 0, 0, 115, 60, 561, 0, 318, 461, 1084, 159, 1253, 0, 0, 269, 60, 334, 587, 4965, 4523, 831, 454, 0, 0, 9, 300, 0, 240, 415, 0, 77, 2549, 709, 4, 631, 1, 0, 66, 674, 6545, 2, 0, 5945, 0, 109, 1434, 431, 80, 0, 0, 0, 0, 0, 0, 0, 2534, 127, 230] payload_rows_per_dst=[15364, 4388, 12552, 10724, 15050, 2616, 13854, 5756] dup_rows_per_dst=[4648, 810, 2702, 1510, 6028, 132, 2144, 26] combine_rows_per_src=[15364, 4388, 12552, 10724, 15050, 2616, 13854, 5756] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=287350784 dispatch_meta_bytes=340488 combine_payload_bytes=287350784 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=287691272 ingress_bytes=356556800 link_bytes=356556800 token_bytes=575042056 weight_movement_bytes=69206016 total_nvlink_bytes=644248072 weight_to_token_ratio=0.120349 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20480 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.341694533e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=277184512 dispatch_meta_bytes=332120 combine_payload_bytes=277184512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277516632 ingress_bytes=311787520 link_bytes=311787520 token_bytes=554701144 weight_movement_bytes=34603008 total_nvlink_bytes=589304152 weight_to_token_ratio=0.062381 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=44032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.173231337e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81394 payload_rows_remote=74818 dup_rows=16910 dup_rows_remote=16364 dedup_ratio=0.172017 router_tokens_per_expert=[1919, 7201, 1, 117, 556, 3, 66, 0, 0, 120, 109, 814, 2, 327, 409, 1019, 392, 858, 0, 0, 272, 84, 807, 619, 4320, 4615, 950, 529, 0, 0, 91, 299, 0, 203, 614, 2, 93, 1911, 710, 28, 611, 2, 0, 54, 804, 6798, 7, 0, 6073, 0, 211, 1246, 483, 94, 305, 49, 0, 4, 0, 0, 0, 1923, 147, 281] payload_rows_per_dst=[4800, 15184, 11648, 11228, 6576, 14028, 4388, 13542] dup_rows_per_dst=[524, 4818, 3056, 1740, 546, 2524, 388, 3314] combine_rows_per_src=[4800, 15184, 11648, 11228, 6576, 14028, 4388, 13542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=311762944 dispatch_meta_bytes=372584 combine_payload_bytes=311762944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=312135528 ingress_bytes=346365952 link_bytes=346365952 token_bytes=623898472 weight_movement_bytes=34603008 total_nvlink_bytes=658501480 weight_to_token_ratio=0.055463 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=28544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.303347193e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83474 payload_rows_remote=72800 dup_rows=14830 dup_rows_remote=13616 dedup_ratio=0.150859 router_tokens_per_expert=[2643, 7507, 0, 43, 523, 0, 50, 0, 0, 129, 50, 824, 0, 153, 429, 1012, 143, 1180, 0, 0, 187, 50, 190, 513, 5540, 4418, 843, 379, 0, 0, 1, 303, 0, 271, 301, 0, 65, 2713, 679, 6, 572, 1, 0, 71, 326, 6590, 0, 0, 5666, 0, 18, 1338, 212, 64, 0, 0, 0, 0, 0, 0, 0, 2776, 185, 188] payload_rows_per_dst=[11030, 9778, 13080, 10674, 7520, 13496, 7562, 10334] dup_rows_per_dst=[3194, 2724, 2526, 1214, 550, 1624, 656, 2342] combine_rows_per_src=[11030, 9778, 13080, 10674, 7520, 13496, 7562, 10334] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84000 payload_rows_remote=74762 dup_rows=14304 dup_rows_remote=12016 dedup_ratio=0.145508 router_tokens_per_expert=[570, 84, 4048, 1109, 3547, 0, 6879, 5686, 0, 169, 0, 425, 0, 783, 0, 398, 291, 331, 3032, 84, 938, 59, 414, 140, 0, 84, 520, 0, 0, 142, 0, 493, 6125, 313, 0, 85, 197, 1189, 740, 0, 412, 0, 0, 0, 0, 0, 0, 224, 2481, 0, 0, 58, 0, 1259, 80, 0, 2694, 0, 1, 1774, 1193, 101, 0, 0] payload_rows_per_dst=[3526, 10260, 9280, 13342, 9126, 14584, 14644, 9238] dup_rows_per_dst=[0, 2324, 1298, 508, 2078, 446, 5362, 2288] combine_rows_per_src=[3526, 10260, 9280, 13342, 9126, 14584, 14644, 9238] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84578 payload_rows_remote=75300 dup_rows=13726 dup_rows_remote=12258 dedup_ratio=0.139628 router_tokens_per_expert=[617, 73, 3066, 787, 3535, 0, 6962, 5224, 0, 147, 0, 400, 0, 763, 0, 431, 329, 296, 3326, 73, 775, 50, 416, 108, 1, 73, 449, 0, 0, 125, 0, 443, 6115, 289, 0, 74, 195, 1977, 764, 0, 403, 0, 0, 0, 0, 0, 0, 248, 2426, 0, 0, 88, 0, 1558, 157, 0, 2575, 0, 0, 1916, 1816, 82, 0, 0] payload_rows_per_dst=[8898, 8034, 9278, 12220, 13952, 14738, 7648, 9810] dup_rows_per_dst=[1126, 1580, 1468, 410, 4876, 488, 810, 2968] combine_rows_per_src=[8898, 8034, 9278, 12220, 13952, 14738, 7648, 9810] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76466 payload_rows_remote=62908 dup_rows=21838 dup_rows_remote=18404 dedup_ratio=0.222148 router_tokens_per_expert=[1870, 7238, 0, 120, 579, 0, 57, 1, 0, 136, 82, 619, 0, 328, 422, 1129, 244, 869, 0, 0, 291, 94, 759, 646, 4380, 4921, 927, 555, 0, 0, 83, 325, 0, 230, 701, 0, 86, 1893, 615, 18, 585, 10, 0, 50, 769, 6735, 4, 0, 6077, 0, 209, 1285, 486, 91, 167, 47, 0, 0, 0, 0, 0, 2016, 165, 238] payload_rows_per_dst=[4750, 8608, 4398, 10894, 6580, 15064, 12614, 13558] dup_rows_per_dst=[504, 2458, 1408, 2202, 506, 9528, 1798, 3434] combine_rows_per_src=[4750, 8608, 4398, 10894, 6580, 15064, 12614, 13558] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82730 payload_rows_remote=69244 dup_rows=15574 dup_rows_remote=13594 dedup_ratio=0.158427 router_tokens_per_expert=[2286, 7460, 0, 60, 566, 0, 50, 0, 0, 149, 50, 590, 0, 176, 480, 907, 143, 1257, 0, 0, 265, 50, 154, 499, 5555, 4413, 786, 405, 0, 0, 0, 313, 0, 284, 335, 0, 79, 2910, 666, 4, 662, 3, 0, 59, 472, 6537, 0, 0, 5705, 0, 35, 1375, 296, 77, 0, 0, 0, 0, 0, 0, 0, 2672, 118, 249] payload_rows_per_dst=[5412, 15504, 9428, 8372, 7994, 13486, 9342, 13192] dup_rows_per_dst=[512, 4120, 2030, 1070, 562, 1980, 1004, 4296] combine_rows_per_src=[5412, 15504, 9428, 8372, 7994, 13486, 9342, 13192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=313168 combine_payload_bytes=265994240 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266307408 ingress_bytes=369803264 link_bytes=369803264 token_bytes=532301648 weight_movement_bytes=103809024 total_nvlink_bytes=636110672 weight_to_token_ratio=0.195019 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=26112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391539911e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=64670 payload_rows_remote=49490 dup_rows=33634 dup_rows_remote=13820 dedup_ratio=0.342143 router_tokens_per_expert=[596, 77, 3641, 967, 3117, 0, 6786, 5733, 0, 154, 0, 409, 0, 849, 0, 449, 322, 364, 2941, 77, 973, 44, 431, 137, 3, 77, 526, 0, 0, 136, 0, 553, 6205, 298, 0, 77, 224, 1629, 824, 0, 405, 0, 0, 0, 0, 0, 0, 243, 2501, 0, 0, 72, 0, 1075, 150, 0, 2591, 0, 2, 1908, 1499, 87, 0, 0] payload_rows_per_dst=[15180, 9134, 15526, 4262, 4748, 810, 7040, 7970] dup_rows_per_dst=[19814, 1870, 7462, 226, 1356, 486, 556, 1864] combine_rows_per_src=[15180, 9134, 15526, 4262, 4748, 810, 7040, 7970] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73264 payload_rows_remote=65574 dup_rows=25040 dup_rows_remote=24526 dedup_ratio=0.254720 router_tokens_per_expert=[0, 7817, 2809, 0, 0, 59, 247, 3471, 0, 0, 0, 0, 0, 949, 3022, 131, 0, 77, 66, 353, 2, 0, 0, 1076, 341, 258, 0, 127, 0, 59, 154, 0, 154, 7829, 72, 0, 3851, 0, 2510, 0, 570, 0, 0, 0, 2222, 241, 0, 0, 72, 0, 176, 304, 0, 59, 0, 0, 0, 401, 6762, 1161, 87, 925, 0, 738] payload_rows_per_dst=[16132, 7690, 15250, 1574, 16134, 9306, 1078, 6100] dup_rows_per_dst=[12674, 514, 1422, 304, 7850, 1608, 144, 524] combine_rows_per_src=[16132, 7690, 15250, 1574, 16134, 9306, 1078, 6100] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=306454528 dispatch_meta_bytes=364728 combine_payload_bytes=306454528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=306819256 ingress_bytes=341057536 link_bytes=341057536 token_bytes=613273784 weight_movement_bytes=34603008 total_nvlink_bytes=647876792 weight_to_token_ratio=0.056423 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.283372051e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81594 payload_rows_remote=66950 dup_rows=16710 dup_rows_remote=11976 dedup_ratio=0.169983 router_tokens_per_expert=[618, 108, 3860, 1106, 2950, 1, 6791, 5596, 1, 214, 0, 504, 1, 823, 1, 360, 315, 362, 2683, 107, 851, 47, 408, 97, 2, 107, 526, 0, 0, 144, 1, 546, 6153, 319, 0, 107, 207, 1633, 784, 0, 401, 2, 0, 0, 1, 1, 2, 245, 2300, 0, 3, 66, 0, 1057, 110, 0, 2761, 0, 12, 1978, 1728, 152, 0, 0] payload_rows_per_dst=[3662, 12514, 8620, 13306, 4712, 14388, 14644, 9748] dup_rows_per_dst=[4, 4914, 1120, 538, 1388, 498, 4734, 3514] combine_rows_per_src=[3662, 12514, 8620, 13306, 4712, 14388, 14644, 9748] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=298188800 dispatch_meta_bytes=345664 combine_payload_bytes=298188800 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=298534464 ingress_bytes=401997824 link_bytes=401997824 token_bytes=596723264 weight_movement_bytes=103809024 total_nvlink_bytes=700532288 weight_to_token_ratio=0.173965 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=24320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.512685448e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=306225152 dispatch_meta_bytes=347112 combine_payload_bytes=306225152 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=306572264 ingress_bytes=340828160 link_bytes=340828160 token_bytes=612797416 weight_movement_bytes=34603008 total_nvlink_bytes=647400424 weight_to_token_ratio=0.056467 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=20352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.282508927e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=308428800 dispatch_meta_bytes=350232 combine_payload_bytes=308428800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=308779032 ingress_bytes=343031808 link_bytes=343031808 token_bytes=617207832 weight_movement_bytes=34603008 total_nvlink_bytes=651810840 weight_to_token_ratio=0.056064 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.290801077e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 2, 0, 1, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=257671168 dispatch_meta_bytes=325248 combine_payload_bytes=257671168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=257996416 ingress_bytes=292274176 link_bytes=292274176 token_bytes=515667584 weight_movement_bytes=34603008 total_nvlink_bytes=550270592 weight_to_token_ratio=0.067103 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=27008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.099804194e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81930 payload_rows_remote=69276 dup_rows=16374 dup_rows_remote=14050 dedup_ratio=0.166565 router_tokens_per_expert=[2532, 7444, 0, 76, 518, 0, 51, 0, 0, 138, 52, 769, 0, 278, 373, 774, 159, 953, 0, 0, 215, 51, 278, 550, 5283, 4702, 835, 406, 0, 0, 0, 339, 0, 266, 315, 0, 74, 2675, 504, 0, 520, 0, 0, 88, 443, 6610, 2, 0, 5936, 0, 15, 1466, 172, 76, 0, 0, 0, 0, 0, 0, 0, 2829, 141, 244] payload_rows_per_dst=[15934, 4110, 12654, 11072, 13024, 4916, 13832, 6388] dup_rows_per_dst=[5308, 658, 2324, 1492, 4494, 560, 1498, 40] combine_rows_per_src=[15934, 4110, 12654, 11072, 13024, 4916, 13832, 6388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=283623424 dispatch_meta_bytes=331352 combine_payload_bytes=283623424 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=283954776 ingress_bytes=352829440 link_bytes=352829440 token_bytes=567578200 weight_movement_bytes=69206016 total_nvlink_bytes=636784216 weight_to_token_ratio=0.121932 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=34688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.327668778e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81594 payload_rows_remote=66950 dup_rows=16710 dup_rows_remote=11976 dedup_ratio=0.169983 router_tokens_per_expert=[618, 108, 3860, 1106, 2950, 1, 6791, 5596, 1, 214, 0, 504, 1, 823, 1, 360, 315, 362, 2683, 107, 851, 47, 408, 97, 2, 107, 526, 0, 0, 144, 1, 546, 6153, 319, 0, 107, 207, 1633, 784, 0, 401, 2, 0, 0, 1, 1, 2, 245, 2300, 0, 3, 66, 0, 1057, 110, 0, 2761, 0, 12, 1978, 1728, 152, 0, 0] payload_rows_per_dst=[3662, 12514, 8620, 13306, 4712, 14388, 14644, 9748] dup_rows_per_dst=[4, 4914, 1120, 538, 1388, 498, 4734, 3514] combine_rows_per_src=[3662, 12514, 8620, 13306, 4712, 14388, 14644, 9748] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79986 payload_rows_remote=70558 dup_rows=18318 dup_rows_remote=16462 dedup_ratio=0.186340 router_tokens_per_expert=[553, 72, 3811, 1125, 3272, 0, 6909, 5659, 0, 144, 0, 410, 0, 772, 0, 505, 313, 300, 2946, 72, 855, 58, 410, 150, 0, 72, 485, 0, 0, 131, 0, 481, 6248, 271, 0, 72, 208, 1366, 727, 0, 425, 0, 0, 0, 0, 0, 0, 245, 2521, 0, 0, 57, 0, 1214, 92, 0, 2565, 0, 2, 1861, 1693, 80, 0, 0] payload_rows_per_dst=[12704, 9428, 11112, 13228, 11084, 5850, 7166, 9414] dup_rows_per_dst=[6158, 1856, 3188, 428, 2608, 490, 602, 2988] combine_rows_per_src=[12704, 9428, 11112, 13228, 11084, 5850, 7166, 9414] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86018 payload_rows_remote=74758 dup_rows=12286 dup_rows_remote=9336 dedup_ratio=0.124980 router_tokens_per_expert=[0, 7754, 3038, 0, 0, 50, 275, 2886, 0, 0, 0, 0, 0, 1115, 3145, 128, 0, 78, 5, 317, 1, 0, 2, 1017, 493, 251, 0, 122, 0, 50, 231, 0, 264, 7822, 78, 0, 4483, 0, 1925, 0, 303, 0, 0, 0, 2076, 243, 0, 0, 78, 0, 226, 371, 0, 50, 0, 0, 0, 359, 6823, 1554, 127, 645, 0, 767] payload_rows_per_dst=[6726, 12056, 8826, 15662, 4478, 11260, 15902, 11108] dup_rows_per_dst=[0, 2492, 644, 2140, 56, 2950, 1192, 2812] combine_rows_per_src=[6726, 12056, 8826, 15662, 4478, 11260, 15902, 11108] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=202711040 dispatch_meta_bytes=253240 combine_payload_bytes=202711040 prefetch_bytes=34603008 reduce_bytes=173015040 egress_bytes=202964280 ingress_bytes=410329088 link_bytes=410329088 token_bytes=405675320 weight_movement_bytes=207618048 total_nvlink_bytes=613293368 weight_to_token_ratio=0.511784 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=18432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.544035324e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=268591104 dispatch_meta_bytes=360400 combine_payload_bytes=268591104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268951504 ingress_bytes=303194112 link_bytes=303194112 token_bytes=537542608 weight_movement_bytes=34603008 total_nvlink_bytes=572145616 weight_to_token_ratio=0.064373 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=25344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.140895034e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=283754496 dispatch_meta_bytes=333304 combine_payload_bytes=283754496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=284087800 ingress_bytes=318357504 link_bytes=318357504 token_bytes=567842296 weight_movement_bytes=34603008 total_nvlink_bytes=602445304 weight_to_token_ratio=0.060938 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=28544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.197953658e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=274227200 dispatch_meta_bytes=315704 combine_payload_bytes=274227200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274542904 ingress_bytes=308830208 link_bytes=308830208 token_bytes=548770104 weight_movement_bytes=34603008 total_nvlink_bytes=583373112 weight_to_token_ratio=0.063056 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=19072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.162103210e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=306208768 dispatch_meta_bytes=336376 combine_payload_bytes=306208768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=306545144 ingress_bytes=340811776 link_bytes=340811776 token_bytes=612753912 weight_movement_bytes=34603008 total_nvlink_bytes=647356920 weight_to_token_ratio=0.056471 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=26112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.282447275e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=64670 payload_rows_remote=49490 dup_rows=33634 dup_rows_remote=13820 dedup_ratio=0.342143 router_tokens_per_expert=[596, 77, 3641, 967, 3117, 0, 6786, 5733, 0, 154, 0, 409, 0, 849, 0, 449, 322, 364, 2941, 77, 973, 44, 431, 137, 3, 77, 526, 0, 0, 136, 0, 553, 6205, 298, 0, 77, 224, 1629, 824, 0, 405, 0, 0, 0, 0, 0, 0, 243, 2501, 0, 0, 72, 0, 1075, 150, 0, 2591, 0, 2, 1908, 1499, 87, 0, 0] payload_rows_per_dst=[15180, 9134, 15526, 4262, 4748, 810, 7040, 7970] dup_rows_per_dst=[19814, 1870, 7462, 226, 1356, 486, 556, 1864] combine_rows_per_src=[15180, 9134, 15526, 4262, 4748, 810, 7040, 7970] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86886 payload_rows_remote=73432 dup_rows=11418 dup_rows_remote=8768 dedup_ratio=0.116150 router_tokens_per_expert=[603, 91, 4269, 1022, 3570, 0, 6658, 5713, 1, 184, 0, 479, 0, 836, 1, 412, 311, 317, 3423, 90, 934, 50, 392, 148, 0, 91, 488, 0, 0, 157, 0, 475, 6023, 236, 0, 91, 216, 967, 715, 0, 437, 0, 0, 0, 0, 0, 0, 250, 2444, 0, 4, 67, 0, 1291, 75, 0, 2666, 0, 4, 1580, 1236, 135, 0, 0] payload_rows_per_dst=[9314, 10350, 9872, 13332, 13454, 14196, 7458, 8910] dup_rows_per_dst=[1258, 2014, 1458, 516, 2650, 494, 696, 2332] combine_rows_per_src=[9314, 10350, 9872, 13332, 13454, 14196, 7458, 8910] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83766 payload_rows_remote=71466 dup_rows=14538 dup_rows_remote=14146 dedup_ratio=0.147888 router_tokens_per_expert=[637, 64, 2881, 1044, 3607, 0, 7143, 5208, 0, 126, 0, 360, 0, 693, 0, 671, 317, 314, 2449, 64, 719, 51, 391, 84, 5, 64, 451, 0, 0, 115, 0, 503, 6359, 334, 0, 65, 204, 2241, 780, 0, 387, 0, 0, 0, 0, 0, 0, 251, 1965, 0, 0, 71, 0, 1149, 256, 0, 3049, 0, 0, 2349, 1676, 55, 0, 0] payload_rows_per_dst=[9360, 7990, 7668, 12300, 14438, 15074, 6356, 10580] dup_rows_per_dst=[1344, 1472, 1110, 392, 5528, 488, 526, 3678] combine_rows_per_src=[9360, 7990, 7668, 12300, 14438, 15074, 6356, 10580] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79460 payload_rows_remote=63944 dup_rows=18844 dup_rows_remote=8774 dedup_ratio=0.191691 router_tokens_per_expert=[0, 7793, 2478, 0, 0, 57, 43, 3858, 0, 0, 0, 0, 0, 636, 2685, 145, 0, 85, 94, 338, 1, 0, 0, 1105, 342, 241, 0, 160, 0, 57, 184, 0, 80, 7823, 85, 0, 3055, 0, 3271, 0, 908, 0, 0, 0, 2358, 236, 0, 0, 85, 0, 244, 333, 0, 57, 0, 0, 0, 419, 6673, 717, 85, 1548, 0, 873] payload_rows_per_dst=[200, 12724, 3244, 11078, 10022, 10746, 15930, 15516] dup_rows_per_dst=[0, 1924, 2, 1290, 2036, 2368, 1154, 10070] combine_rows_per_src=[200, 12724, 3244, 11078, 10022, 10746, 15930, 15516] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83764 payload_rows_remote=68630 dup_rows=14540 dup_rows_remote=13388 dedup_ratio=0.147909 router_tokens_per_expert=[0, 7886, 2665, 0, 0, 51, 202, 2558, 0, 0, 0, 0, 0, 1111, 3635, 125, 0, 74, 0, 331, 0, 0, 0, 938, 359, 244, 0, 168, 0, 51, 178, 0, 261, 7843, 74, 0, 4670, 0, 2452, 0, 392, 0, 0, 0, 1976, 242, 0, 0, 74, 0, 201, 300, 0, 51, 0, 0, 0, 365, 6800, 1414, 73, 670, 0, 718] payload_rows_per_dst=[12620, 8908, 15134, 9618, 10378, 11634, 9254, 6218] dup_rows_per_dst=[5284, 834, 1152, 1202, 2274, 2926, 606, 262] combine_rows_per_src=[12620, 8908, 15134, 9618, 10378, 11634, 9254, 6218] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=289005568 dispatch_meta_bytes=348080 combine_payload_bytes=289005568 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=289353648 ingress_bytes=358211584 link_bytes=358211584 token_bytes=578359216 weight_movement_bytes=69206016 total_nvlink_bytes=647565232 weight_to_token_ratio=0.119659 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=28928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.347921352e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84000 payload_rows_remote=74762 dup_rows=14304 dup_rows_remote=12016 dedup_ratio=0.145508 router_tokens_per_expert=[570, 84, 4048, 1109, 3547, 0, 6879, 5686, 0, 169, 0, 425, 0, 783, 0, 398, 291, 331, 3032, 84, 938, 59, 414, 140, 0, 84, 520, 0, 0, 142, 0, 493, 6125, 313, 0, 85, 197, 1189, 740, 0, 412, 0, 0, 0, 0, 0, 0, 224, 2481, 0, 0, 58, 0, 1259, 80, 0, 2694, 0, 1, 1774, 1193, 101, 0, 0] payload_rows_per_dst=[3526, 10260, 9280, 13342, 9126, 14584, 14644, 9238] dup_rows_per_dst=[0, 2324, 1298, 508, 2078, 446, 5362, 2288] combine_rows_per_src=[3526, 10260, 9280, 13342, 9126, 14584, 14644, 9238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 2, 0, 1, 1, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87270 payload_rows_remote=72770 dup_rows=11034 dup_rows_remote=6974 dedup_ratio=0.112244 router_tokens_per_expert=[0, 0, 0, 293, 0, 0, 0, 0, 412, 115, 1235, 0, 503, 323, 0, 72, 53, 0, 0, 0, 0, 95, 0, 0, 0, 6620, 0, 340, 1, 1974, 0, 60, 304, 4811, 0, 0, 0, 2240, 0, 681, 3762, 577, 1201, 0, 616, 1134, 2477, 0, 0, 80, 0, 486, 0, 0, 7, 0, 590, 0, 7249, 6481, 4187, 29, 0, 144] payload_rows_per_dst=[13130, 14500, 14702, 8938, 13140, 9716, 8624, 4520] dup_rows_per_dst=[418, 4060, 92, 1084, 2932, 2294, 46, 108] combine_rows_per_src=[13130, 14500, 14702, 8938, 13140, 9716, 8624, 4520] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69840 payload_rows_remote=53770 dup_rows=28464 dup_rows_remote=15634 dedup_ratio=0.289551 router_tokens_per_expert=[0, 7756, 2895, 0, 0, 59, 248, 3492, 0, 0, 0, 0, 0, 1146, 2905, 137, 0, 77, 37, 328, 0, 0, 0, 1082, 317, 246, 0, 127, 0, 59, 243, 0, 229, 7824, 77, 0, 4200, 0, 1968, 0, 400, 0, 0, 0, 2141, 245, 0, 0, 77, 0, 262, 366, 0, 59, 0, 0, 0, 425, 6744, 1219, 79, 899, 0, 784] payload_rows_per_dst=[16070, 7558, 15190, 1654, 16084, 5472, 1374, 6438] dup_rows_per_dst=[12830, 818, 1346, 330, 12512, 100, 154, 374] combine_rows_per_src=[16070, 7558, 15190, 1654, 16084, 5472, 1374, 6438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83766 payload_rows_remote=71466 dup_rows=14538 dup_rows_remote=14146 dedup_ratio=0.147888 router_tokens_per_expert=[637, 64, 2881, 1044, 3607, 0, 7143, 5208, 0, 126, 0, 360, 0, 693, 0, 671, 317, 314, 2449, 64, 719, 51, 391, 84, 5, 64, 451, 0, 0, 115, 0, 503, 6359, 334, 0, 65, 204, 2241, 780, 0, 387, 0, 0, 0, 0, 0, 0, 251, 1965, 0, 0, 71, 0, 1149, 256, 0, 3049, 0, 0, 2349, 1676, 55, 0, 0] payload_rows_per_dst=[9360, 7990, 7668, 12300, 14438, 15074, 6356, 10580] dup_rows_per_dst=[1344, 1472, 1110, 392, 5528, 488, 526, 3678] combine_rows_per_src=[9360, 7990, 7668, 12300, 14438, 15074, 6356, 10580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82444 payload_rows_remote=67672 dup_rows=15860 dup_rows_remote=15358 dedup_ratio=0.161336 router_tokens_per_expert=[609, 77, 3340, 1082, 3256, 0, 6992, 5447, 0, 155, 0, 442, 0, 767, 0, 532, 312, 362, 2854, 77, 807, 57, 365, 88, 1, 77, 435, 0, 0, 128, 0, 560, 6251, 249, 0, 78, 205, 1825, 796, 0, 394, 0, 0, 0, 0, 0, 0, 251, 2410, 0, 0, 59, 1, 1157, 153, 0, 2569, 0, 1, 2087, 1711, 133, 0, 0] payload_rows_per_dst=[5456, 10880, 8808, 12862, 5080, 14772, 14836, 9750] dup_rows_per_dst=[500, 3684, 1036, 434, 1226, 502, 5226, 3252] combine_rows_per_src=[5456, 10880, 8808, 12862, 5080, 14772, 14836, 9750] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=202711040 dispatch_meta_bytes=253240 combine_payload_bytes=202711040 prefetch_bytes=34603008 reduce_bytes=173015040 egress_bytes=202964280 ingress_bytes=410329088 link_bytes=410329088 token_bytes=405675320 weight_movement_bytes=207618048 total_nvlink_bytes=613293368 weight_to_token_ratio=0.511784 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=18432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.544035324e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=300777472 dispatch_meta_bytes=328800 combine_payload_bytes=300777472 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=301106272 ingress_bytes=404586496 link_bytes=404586496 token_bytes=601883744 weight_movement_bytes=103809024 total_nvlink_bytes=705692768 weight_to_token_ratio=0.172474 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=22528 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522426413e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84578 payload_rows_remote=75300 dup_rows=13726 dup_rows_remote=12258 dedup_ratio=0.139628 router_tokens_per_expert=[617, 73, 3066, 787, 3535, 0, 6962, 5224, 0, 147, 0, 400, 0, 763, 0, 431, 329, 296, 3326, 73, 775, 50, 416, 108, 1, 73, 449, 0, 0, 125, 0, 443, 6115, 289, 0, 74, 195, 1977, 764, 0, 403, 0, 0, 0, 0, 0, 0, 248, 2426, 0, 0, 88, 0, 1558, 157, 0, 2575, 0, 0, 1916, 1816, 82, 0, 0] payload_rows_per_dst=[8898, 8034, 9278, 12220, 13952, 14738, 7648, 9810] dup_rows_per_dst=[1126, 1580, 1468, 410, 4876, 488, 810, 2968] combine_rows_per_src=[8898, 8034, 9278, 12220, 13952, 14738, 7648, 9810] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=306225152 dispatch_meta_bytes=347112 combine_payload_bytes=306225152 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=306572264 ingress_bytes=340828160 link_bytes=340828160 token_bytes=612797416 weight_movement_bytes=34603008 total_nvlink_bytes=647400424 weight_to_token_ratio=0.056467 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=20352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.282508927e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79034 payload_rows_remote=63114 dup_rows=19270 dup_rows_remote=17976 dedup_ratio=0.196025 router_tokens_per_expert=[0, 7718, 3050, 0, 0, 37, 347, 3145, 0, 2, 0, 0, 1, 1136, 3349, 147, 0, 108, 152, 294, 3, 1, 5, 1097, 478, 245, 0, 162, 1, 37, 301, 0, 238, 7814, 107, 1, 3856, 0, 1909, 0, 306, 0, 1, 1, 2056, 245, 0, 1, 107, 1, 274, 366, 0, 44, 0, 1, 0, 405, 6676, 1029, 123, 891, 2, 882] payload_rows_per_dst=[1116, 12582, 3302, 15656, 4464, 10468, 15920, 15526] dup_rows_per_dst=[0, 2978, 18, 2228, 46, 2464, 1294, 10242] combine_rows_per_src=[1116, 12582, 3302, 15656, 4464, 10468, 15920, 15526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=277184512 dispatch_meta_bytes=332120 combine_payload_bytes=277184512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277516632 ingress_bytes=311787520 link_bytes=311787520 token_bytes=554701144 weight_movement_bytes=34603008 total_nvlink_bytes=589304152 weight_to_token_ratio=0.062381 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=44032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.173231337e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=292724736 dispatch_meta_bytes=342448 combine_payload_bytes=292724736 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293067184 ingress_bytes=327327744 link_bytes=327327744 token_bytes=585791920 weight_movement_bytes=34603008 total_nvlink_bytes=620394928 weight_to_token_ratio=0.059070 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=39552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.231707949e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=261914624 dispatch_meta_bytes=290872 combine_payload_bytes=261914624 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=262205496 ingress_bytes=331120640 link_bytes=331120640 token_bytes=524120120 weight_movement_bytes=69206016 total_nvlink_bytes=593326136 weight_to_token_ratio=0.132042 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=16896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.245980311e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=281108480 dispatch_meta_bytes=328072 combine_payload_bytes=281108480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281436552 ingress_bytes=315711488 link_bytes=315711488 token_bytes=562545032 weight_movement_bytes=34603008 total_nvlink_bytes=597148040 weight_to_token_ratio=0.061512 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=37120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.187996912e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87136 payload_rows_remote=73966 dup_rows=11168 dup_rows_remote=7738 dedup_ratio=0.113607 router_tokens_per_expert=[0, 0, 0, 248, 0, 0, 0, 0, 367, 110, 837, 0, 596, 288, 0, 77, 48, 0, 0, 0, 0, 66, 0, 0, 0, 6523, 0, 502, 0, 1760, 0, 191, 229, 4968, 0, 0, 0, 2474, 0, 637, 4464, 439, 910, 0, 568, 1415, 2601, 0, 0, 112, 0, 484, 0, 0, 10, 0, 646, 0, 7294, 6481, 3648, 4, 0, 155] payload_rows_per_dst=[13052, 3700, 14726, 14344, 6652, 13170, 12840, 8652] dup_rows_per_dst=[406, 850, 90, 3608, 28, 3430, 2502, 254] combine_rows_per_src=[13052, 3700, 14726, 14344, 6652, 13170, 12840, 8652] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73264 payload_rows_remote=65574 dup_rows=25040 dup_rows_remote=24526 dedup_ratio=0.254720 router_tokens_per_expert=[0, 7817, 2809, 0, 0, 59, 247, 3471, 0, 0, 0, 0, 0, 949, 3022, 131, 0, 77, 66, 353, 2, 0, 0, 1076, 341, 258, 0, 127, 0, 59, 154, 0, 154, 7829, 72, 0, 3851, 0, 2510, 0, 570, 0, 0, 0, 2222, 241, 0, 0, 72, 0, 176, 304, 0, 59, 0, 0, 0, 401, 6762, 1161, 87, 925, 0, 738] payload_rows_per_dst=[16132, 7690, 15250, 1574, 16134, 9306, 1078, 6100] dup_rows_per_dst=[12674, 514, 1422, 304, 7850, 1608, 144, 524] combine_rows_per_src=[16132, 7690, 15250, 1574, 16134, 9306, 1078, 6100] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=298065920 dispatch_meta_bytes=318976 combine_payload_bytes=298065920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=298384896 ingress_bytes=332668928 link_bytes=332668928 token_bytes=596450816 weight_movement_bytes=34603008 total_nvlink_bytes=631053824 weight_to_token_ratio=0.058015 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=29568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.251806394e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=220241920 dispatch_meta_bytes=277616 combine_payload_bytes=220241920 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=220519536 ingress_bytes=358653952 link_bytes=358653952 token_bytes=440761456 weight_movement_bytes=138412032 total_nvlink_bytes=579173488 weight_to_token_ratio=0.314029 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=19072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.349585947e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=308428800 dispatch_meta_bytes=350232 combine_payload_bytes=308428800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=308779032 ingress_bytes=343031808 link_bytes=343031808 token_bytes=617207832 weight_movement_bytes=34603008 total_nvlink_bytes=651810840 weight_to_token_ratio=0.056064 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.290801077e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=258514944 dispatch_meta_bytes=324360 combine_payload_bytes=258514944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258839304 ingress_bytes=293117952 link_bytes=293117952 token_bytes=517354248 weight_movement_bytes=34603008 total_nvlink_bytes=551957256 weight_to_token_ratio=0.066885 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=43520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.102979255e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=302964736 dispatch_meta_bytes=326816 combine_payload_bytes=302964736 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=303291552 ingress_bytes=406773760 link_bytes=406773760 token_bytes=606256288 weight_movement_bytes=103809024 total_nvlink_bytes=710065312 weight_to_token_ratio=0.171230 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=14208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.530656911e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88644 payload_rows_remote=81194 dup_rows=9660 dup_rows_remote=9620 dedup_ratio=0.098267 router_tokens_per_expert=[0, 7779, 2847, 0, 0, 66, 27, 3942, 0, 1, 0, 0, 0, 694, 2825, 157, 0, 91, 131, 363, 0, 0, 0, 1037, 420, 252, 0, 174, 0, 66, 234, 0, 54, 7803, 91, 0, 2843, 0, 3600, 0, 860, 0, 1, 0, 2058, 245, 0, 0, 91, 0, 286, 325, 0, 67, 1, 0, 0, 371, 6531, 685, 98, 1269, 0, 767] payload_rows_per_dst=[5880, 12904, 14924, 15744, 7450, 10062, 15920, 5760] dup_rows_per_dst=[0, 2334, 1382, 2106, 40, 1952, 1226, 620] combine_rows_per_src=[5880, 12904, 14924, 15744, 7450, 10062, 15920, 5760] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88130 payload_rows_remote=72062 dup_rows=10174 dup_rows_remote=8524 dedup_ratio=0.103495 router_tokens_per_expert=[0, 7970, 2559, 0, 0, 50, 143, 2641, 0, 0, 0, 0, 0, 1143, 3729, 115, 0, 91, 0, 323, 0, 0, 0, 912, 284, 244, 0, 143, 0, 50, 168, 0, 253, 7901, 65, 0, 4777, 0, 1922, 0, 395, 0, 0, 0, 2084, 238, 0, 0, 65, 0, 226, 336, 0, 50, 0, 0, 0, 350, 6908, 1555, 107, 611, 0, 744] payload_rows_per_dst=[7828, 10382, 15230, 16068, 4430, 11764, 16032, 6396] dup_rows_per_dst=[944, 1606, 1238, 1650, 50, 3224, 1124, 338] combine_rows_per_src=[7828, 10382, 15230, 16068, 4430, 11764, 16032, 6396] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=64942 payload_rows_remote=49204 dup_rows=33362 dup_rows_remote=10194 dedup_ratio=0.339376 router_tokens_per_expert=[0, 0, 0, 316, 0, 0, 0, 0, 386, 122, 1290, 0, 592, 324, 0, 84, 46, 0, 0, 0, 0, 88, 0, 0, 0, 6386, 0, 302, 0, 2175, 0, 21, 393, 4449, 0, 0, 0, 2477, 0, 733, 2947, 699, 1504, 0, 624, 998, 2439, 0, 0, 100, 0, 509, 0, 0, 50, 0, 596, 0, 7284, 6451, 4518, 81, 0, 168] payload_rows_per_dst=[4746, 4098, 268, 13582, 6840, 9070, 10600, 15738] dup_rows_per_dst=[116, 1498, 0, 4186, 366, 2538, 1490, 23168] combine_rows_per_src=[4746, 4098, 268, 13582, 6840, 9070, 10600, 15738] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88040 payload_rows_remote=73124 dup_rows=10264 dup_rows_remote=10258 dedup_ratio=0.104411 router_tokens_per_expert=[0, 0, 0, 185, 0, 0, 0, 0, 380, 101, 966, 0, 557, 301, 0, 73, 3, 0, 0, 0, 0, 86, 0, 0, 0, 6309, 0, 491, 0, 2035, 0, 0, 198, 5055, 0, 0, 0, 2592, 0, 571, 4231, 423, 1000, 0, 598, 1547, 2509, 0, 0, 82, 0, 447, 0, 0, 77, 0, 682, 0, 7372, 6452, 3681, 1, 0, 147] payload_rows_per_dst=[12910, 14250, 14916, 5052, 13460, 7850, 10884, 8718] dup_rows_per_dst=[364, 3124, 6, 0, 3372, 1870, 1224, 304] combine_rows_per_src=[12910, 14250, 14916, 5052, 13460, 7850, 10884, 8718] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69840 payload_rows_remote=53770 dup_rows=28464 dup_rows_remote=15634 dedup_ratio=0.289551 router_tokens_per_expert=[0, 7756, 2895, 0, 0, 59, 248, 3492, 0, 0, 0, 0, 0, 1146, 2905, 137, 0, 77, 37, 328, 0, 0, 0, 1082, 317, 246, 0, 127, 0, 59, 243, 0, 229, 7824, 77, 0, 4200, 0, 1968, 0, 400, 0, 0, 0, 2141, 245, 0, 0, 77, 0, 262, 366, 0, 59, 0, 0, 0, 425, 6744, 1219, 79, 899, 0, 784] payload_rows_per_dst=[16070, 7558, 15190, 1654, 16084, 5472, 1374, 6438] dup_rows_per_dst=[12830, 818, 1346, 330, 12512, 100, 154, 374] combine_rows_per_src=[16070, 7558, 15190, 1654, 16084, 5472, 1374, 6438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=268591104 dispatch_meta_bytes=360400 combine_payload_bytes=268591104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268951504 ingress_bytes=303194112 link_bytes=303194112 token_bytes=537542608 weight_movement_bytes=34603008 total_nvlink_bytes=572145616 weight_to_token_ratio=0.064373 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=25344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.140895034e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79460 payload_rows_remote=63944 dup_rows=18844 dup_rows_remote=8774 dedup_ratio=0.191691 router_tokens_per_expert=[0, 7793, 2478, 0, 0, 57, 43, 3858, 0, 0, 0, 0, 0, 636, 2685, 145, 0, 85, 94, 338, 1, 0, 0, 1105, 342, 241, 0, 160, 0, 57, 184, 0, 80, 7823, 85, 0, 3055, 0, 3271, 0, 908, 0, 0, 0, 2358, 236, 0, 0, 85, 0, 244, 333, 0, 57, 0, 0, 0, 419, 6673, 717, 85, 1548, 0, 873] payload_rows_per_dst=[200, 12724, 3244, 11078, 10022, 10746, 15930, 15516] dup_rows_per_dst=[0, 1924, 2, 1290, 2036, 2368, 1154, 10070] combine_rows_per_src=[200, 12724, 3244, 11078, 10022, 10746, 15930, 15516] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85222 payload_rows_remote=71898 dup_rows=13082 dup_rows_remote=13050 dedup_ratio=0.133077 router_tokens_per_expert=[0, 59, 258, 96, 0, 5142, 0, 375, 0, 0, 16, 0, 72, 0, 0, 0, 3631, 0, 0, 0, 0, 2068, 0, 1998, 0, 0, 25, 7258, 0, 0, 0, 59, 4796, 2, 71, 247, 0, 0, 167, 658, 226, 0, 0, 1949, 0, 0, 0, 0, 1078, 0, 4666, 72, 6590, 59, 629, 0, 0, 453, 1363, 613, 1006, 3371, 0, 79] payload_rows_per_dst=[11858, 13324, 15566, 6886, 11082, 10470, 10482, 5554] dup_rows_per_dst=[2, 32, 7082, 24, 1958, 1142, 1368, 1474] combine_rows_per_src=[11858, 13324, 15566, 6886, 11082, 10470, 10482, 5554] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86990 payload_rows_remote=74226 dup_rows=11314 dup_rows_remote=10908 dedup_ratio=0.115092 router_tokens_per_expert=[0, 0, 0, 209, 0, 0, 0, 0, 387, 129, 859, 0, 621, 316, 0, 77, 41, 0, 0, 0, 0, 82, 0, 0, 0, 6598, 0, 289, 0, 2001, 0, 243, 301, 4955, 0, 0, 0, 2349, 0, 632, 3785, 525, 1191, 0, 634, 1245, 2466, 0, 0, 86, 0, 463, 0, 0, 21, 0, 603, 0, 7346, 6376, 4155, 13, 0, 154] payload_rows_per_dst=[12764, 14518, 14860, 11392, 13386, 9894, 8636, 1540] dup_rows_per_dst=[406, 3456, 78, 1984, 3088, 2228, 74, 0] combine_rows_per_src=[12764, 14518, 14860, 11392, 13386, 9894, 8636, 1540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86018 payload_rows_remote=74758 dup_rows=12286 dup_rows_remote=9336 dedup_ratio=0.124980 router_tokens_per_expert=[0, 7754, 3038, 0, 0, 50, 275, 2886, 0, 0, 0, 0, 0, 1115, 3145, 128, 0, 78, 5, 317, 1, 0, 2, 1017, 493, 251, 0, 122, 0, 50, 231, 0, 264, 7822, 78, 0, 4483, 0, 1925, 0, 303, 0, 0, 0, 2076, 243, 0, 0, 78, 0, 226, 371, 0, 50, 0, 0, 0, 359, 6823, 1554, 127, 645, 0, 767] payload_rows_per_dst=[6726, 12056, 8826, 15662, 4478, 11260, 15902, 11108] dup_rows_per_dst=[0, 2492, 644, 2140, 56, 2950, 1192, 2812] combine_rows_per_src=[6726, 12056, 8826, 15662, 4478, 11260, 15902, 11108] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88130 payload_rows_remote=72062 dup_rows=10174 dup_rows_remote=8524 dedup_ratio=0.103495 router_tokens_per_expert=[0, 7970, 2559, 0, 0, 50, 143, 2641, 0, 0, 0, 0, 0, 1143, 3729, 115, 0, 91, 0, 323, 0, 0, 0, 912, 284, 244, 0, 143, 0, 50, 168, 0, 253, 7901, 65, 0, 4777, 0, 1922, 0, 395, 0, 0, 0, 2084, 238, 0, 0, 65, 0, 226, 336, 0, 50, 0, 0, 0, 350, 6908, 1555, 107, 611, 0, 744] payload_rows_per_dst=[7828, 10382, 15230, 16068, 4430, 11764, 16032, 6396] dup_rows_per_dst=[944, 1606, 1238, 1650, 50, 3224, 1124, 338] combine_rows_per_src=[7828, 10382, 15230, 16068, 4430, 11764, 16032, 6396] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=332570624 dispatch_meta_bytes=363256 combine_payload_bytes=332570624 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=332933880 ingress_bytes=436379648 link_bytes=436379648 token_bytes=665504504 weight_movement_bytes=103809024 total_nvlink_bytes=769313528 weight_to_token_ratio=0.155985 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=19584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.642061484e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=220241920 dispatch_meta_bytes=277616 combine_payload_bytes=220241920 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=220519536 ingress_bytes=358653952 link_bytes=358653952 token_bytes=440761456 weight_movement_bytes=138412032 total_nvlink_bytes=579173488 weight_to_token_ratio=0.314029 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=19072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.349585947e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83764 payload_rows_remote=68630 dup_rows=14540 dup_rows_remote=13388 dedup_ratio=0.147909 router_tokens_per_expert=[0, 7886, 2665, 0, 0, 51, 202, 2558, 0, 0, 0, 0, 0, 1111, 3635, 125, 0, 74, 0, 331, 0, 0, 0, 938, 359, 244, 0, 168, 0, 51, 178, 0, 261, 7843, 74, 0, 4670, 0, 2452, 0, 392, 0, 0, 0, 1976, 242, 0, 0, 74, 0, 201, 300, 0, 51, 0, 0, 0, 365, 6800, 1414, 73, 670, 0, 718] payload_rows_per_dst=[12620, 8908, 15134, 9618, 10378, 11634, 9254, 6218] dup_rows_per_dst=[5284, 834, 1152, 1202, 2274, 2926, 606, 262] combine_rows_per_src=[12620, 8908, 15134, 9618, 10378, 11634, 9254, 6218] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=261914624 dispatch_meta_bytes=290872 combine_payload_bytes=261914624 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=262205496 ingress_bytes=331120640 link_bytes=331120640 token_bytes=524120120 weight_movement_bytes=69206016 total_nvlink_bytes=593326136 weight_to_token_ratio=0.132042 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=16896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.245980311e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83292 payload_rows_remote=71270 dup_rows=15012 dup_rows_remote=12822 dedup_ratio=0.152710 router_tokens_per_expert=[1, 0, 0, 237, 0, 2, 0, 1, 331, 96, 941, 1, 638, 352, 1, 107, 66, 1, 0, 0, 1, 49, 0, 0, 0, 6510, 1, 406, 27, 1866, 0, 207, 320, 4928, 0, 0, 0, 2340, 0, 656, 3882, 659, 1186, 1, 596, 1010, 2525, 0, 0, 152, 1, 479, 0, 0, 66, 1, 678, 1, 7042, 6378, 4124, 71, 0, 214] payload_rows_per_dst=[12852, 3704, 8682, 14282, 6516, 13220, 12022, 12014] dup_rows_per_dst=[386, 1230, 2, 3752, 116, 3540, 2190, 3796] combine_rows_per_src=[12852, 3704, 8682, 14282, 6516, 13220, 12022, 12014] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=295165952 dispatch_meta_bytes=322344 combine_payload_bytes=295165952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=295488296 ingress_bytes=329768960 link_bytes=329768960 token_bytes=590654248 weight_movement_bytes=34603008 total_nvlink_bytes=625257256 weight_to_token_ratio=0.058584 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=40832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.240894048e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=201539584 dispatch_meta_bytes=237592 combine_payload_bytes=201539584 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=201777176 ingress_bytes=339951616 link_bytes=339951616 token_bytes=403316760 weight_movement_bytes=138412032 total_nvlink_bytes=541728792 weight_to_token_ratio=0.343184 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=25600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.279210563e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=299515904 dispatch_meta_bytes=333528 combine_payload_bytes=299515904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=299849432 ingress_bytes=334118912 link_bytes=334118912 token_bytes=599365336 weight_movement_bytes=34603008 total_nvlink_bytes=633968344 weight_to_token_ratio=0.057733 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=48256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257262567e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90832 payload_rows_remote=80036 dup_rows=7472 dup_rows_remote=5952 dedup_ratio=0.076009 router_tokens_per_expert=[0, 50, 251, 72, 0, 4935, 0, 444, 0, 0, 8, 0, 78, 0, 0, 0, 3619, 0, 0, 0, 0, 2225, 0, 2352, 0, 0, 25, 7320, 0, 0, 0, 45, 4548, 3, 78, 256, 0, 0, 143, 441, 192, 0, 0, 2347, 0, 0, 0, 0, 1241, 0, 4485, 77, 6684, 49, 296, 0, 0, 416, 1262, 640, 946, 3504, 0, 120] payload_rows_per_dst=[11480, 13526, 8642, 14640, 9668, 10796, 11046, 11034] dup_rows_per_dst=[24, 14, 512, 140, 1270, 1520, 1250, 2742] combine_rows_per_src=[11480, 13526, 8642, 14640, 9668, 10796, 11046, 11034] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=306208768 dispatch_meta_bytes=336376 combine_payload_bytes=306208768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=306545144 ingress_bytes=340811776 link_bytes=340811776 token_bytes=612753912 weight_movement_bytes=34603008 total_nvlink_bytes=647356920 weight_to_token_ratio=0.056471 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=26112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.282447275e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=294494208 dispatch_meta_bytes=339792 combine_payload_bytes=294494208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=294834000 ingress_bytes=329097216 link_bytes=329097216 token_bytes=589328208 weight_movement_bytes=34603008 total_nvlink_bytes=623931216 weight_to_token_ratio=0.058716 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=48512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.238366329e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=304029696 dispatch_meta_bytes=340536 combine_payload_bytes=304029696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=304370232 ingress_bytes=338632704 link_bytes=338632704 token_bytes=608399928 weight_movement_bytes=34603008 total_nvlink_bytes=643002936 weight_to_token_ratio=0.056875 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=47232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274247603e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=295165952 dispatch_meta_bytes=322344 combine_payload_bytes=295165952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=295488296 ingress_bytes=329768960 link_bytes=329768960 token_bytes=590654248 weight_movement_bytes=34603008 total_nvlink_bytes=625257256 weight_to_token_ratio=0.058584 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=40832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.240894048e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87270 payload_rows_remote=72770 dup_rows=11034 dup_rows_remote=6974 dedup_ratio=0.112244 router_tokens_per_expert=[0, 0, 0, 293, 0, 0, 0, 0, 412, 115, 1235, 0, 503, 323, 0, 72, 53, 0, 0, 0, 0, 95, 0, 0, 0, 6620, 0, 340, 1, 1974, 0, 60, 304, 4811, 0, 0, 0, 2240, 0, 681, 3762, 577, 1201, 0, 616, 1134, 2477, 0, 0, 80, 0, 486, 0, 0, 7, 0, 590, 0, 7249, 6481, 4187, 29, 0, 144] payload_rows_per_dst=[13130, 14500, 14702, 8938, 13140, 9716, 8624, 4520] dup_rows_per_dst=[418, 4060, 92, 1084, 2932, 2294, 46, 108] combine_rows_per_src=[13130, 14500, 14702, 8938, 13140, 9716, 8624, 4520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=281108480 dispatch_meta_bytes=328072 combine_payload_bytes=281108480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281436552 ingress_bytes=315711488 link_bytes=315711488 token_bytes=562545032 weight_movement_bytes=34603008 total_nvlink_bytes=597148040 weight_to_token_ratio=0.061512 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=37120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.187996912e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=291921920 dispatch_meta_bytes=336368 combine_payload_bytes=291921920 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=292258288 ingress_bytes=361127936 link_bytes=361127936 token_bytes=584180208 weight_movement_bytes=69206016 total_nvlink_bytes=653386224 weight_to_token_ratio=0.118467 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=30336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.358895350e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=327827456 dispatch_meta_bytes=343952 combine_payload_bytes=327827456 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=328171408 ingress_bytes=397033472 link_bytes=397033472 token_bytes=655998864 weight_movement_bytes=69206016 total_nvlink_bytes=725204880 weight_to_token_ratio=0.105497 gemm_rows_defined=50176 gemm_rows_charged=51200 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020000 gemm_active_groups=10 gemm_peak_group_rows=28800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.494004991e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86938 payload_rows_remote=77152 dup_rows=11366 dup_rows_remote=9662 dedup_ratio=0.115621 router_tokens_per_expert=[0, 0, 0, 265, 0, 0, 0, 0, 412, 128, 1151, 0, 541, 343, 0, 91, 52, 0, 0, 0, 0, 94, 0, 0, 0, 6295, 0, 315, 1, 2005, 0, 32, 353, 4761, 0, 0, 0, 2322, 0, 639, 2782, 766, 1536, 0, 550, 997, 2768, 0, 0, 134, 0, 465, 0, 0, 53, 0, 598, 0, 7275, 6414, 4742, 89, 0, 183] payload_rows_per_dst=[13040, 8884, 14776, 9312, 9786, 9714, 10700, 10726] dup_rows_per_dst=[318, 2598, 66, 1834, 1704, 2644, 1704, 498] combine_rows_per_src=[13040, 8884, 14776, 9312, 9786, 9714, 10700, 10726] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84264 payload_rows_remote=72470 dup_rows=14040 dup_rows_remote=10032 dedup_ratio=0.142822 router_tokens_per_expert=[0, 57, 241, 89, 0, 5503, 0, 415, 0, 0, 15, 0, 85, 0, 0, 0, 3655, 0, 0, 0, 0, 1805, 0, 1827, 0, 0, 12, 7007, 0, 0, 0, 55, 4898, 0, 9, 273, 0, 0, 172, 823, 228, 0, 0, 1731, 0, 0, 0, 0, 820, 0, 4994, 84, 6454, 57, 637, 0, 0, 576, 1526, 660, 999, 3360, 0, 85] payload_rows_per_dst=[12604, 5216, 5232, 14016, 10870, 11058, 13474, 11794] dup_rows_per_dst=[6, 20, 140, 132, 1480, 2062, 6192, 4008] combine_rows_per_src=[12604, 5216, 5232, 14016, 10870, 11058, 13474, 11794] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87554 payload_rows_remote=82600 dup_rows=10750 dup_rows_remote=10750 dedup_ratio=0.109355 router_tokens_per_expert=[0, 0, 0, 203, 0, 0, 0, 0, 373, 115, 617, 0, 594, 260, 0, 64, 33, 0, 0, 0, 0, 79, 0, 0, 0, 6894, 0, 368, 0, 2105, 0, 4, 205, 5047, 0, 0, 0, 2567, 0, 640, 4450, 327, 989, 0, 580, 1667, 2438, 0, 0, 52, 0, 437, 0, 0, 7, 0, 686, 0, 7408, 6495, 3318, 1, 0, 129] payload_rows_per_dst=[13018, 14666, 14974, 4954, 13712, 5674, 12516, 8040] dup_rows_per_dst=[378, 3168, 66, 0, 3206, 1452, 2252, 228] combine_rows_per_src=[13018, 14666, 14974, 4954, 13712, 5674, 12516, 8040] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88290 payload_rows_remote=77790 dup_rows=10014 dup_rows_remote=7120 dedup_ratio=0.101868 router_tokens_per_expert=[0, 51, 244, 87, 0, 4523, 0, 371, 0, 0, 9, 0, 74, 0, 0, 0, 3932, 0, 0, 0, 0, 2276, 0, 2046, 0, 0, 22, 7392, 0, 0, 0, 50, 4290, 0, 74, 241, 0, 0, 158, 531, 210, 0, 0, 2737, 0, 0, 0, 0, 1403, 0, 4303, 69, 6575, 51, 363, 0, 0, 374, 1046, 633, 1217, 3725, 0, 75] payload_rows_per_dst=[10548, 13298, 10500, 12182, 9092, 11788, 10990, 9892] dup_rows_per_dst=[4, 18, 2894, 98, 1496, 1970, 1388, 2146] combine_rows_per_src=[10548, 13298, 10500, 12182, 9092, 11788, 10990, 9892] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86990 payload_rows_remote=74226 dup_rows=11314 dup_rows_remote=10908 dedup_ratio=0.115092 router_tokens_per_expert=[0, 0, 0, 209, 0, 0, 0, 0, 387, 129, 859, 0, 621, 316, 0, 77, 41, 0, 0, 0, 0, 82, 0, 0, 0, 6598, 0, 289, 0, 2001, 0, 243, 301, 4955, 0, 0, 0, 2349, 0, 632, 3785, 525, 1191, 0, 634, 1245, 2466, 0, 0, 86, 0, 463, 0, 0, 21, 0, 603, 0, 7346, 6376, 4155, 13, 0, 154] payload_rows_per_dst=[12764, 14518, 14860, 11392, 13386, 9894, 8636, 1540] dup_rows_per_dst=[406, 3456, 78, 1984, 3088, 2228, 74, 0] combine_rows_per_src=[12764, 14518, 14860, 11392, 13386, 9894, 8636, 1540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=298065920 dispatch_meta_bytes=318976 combine_payload_bytes=298065920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=298384896 ingress_bytes=332668928 link_bytes=332668928 token_bytes=596450816 weight_movement_bytes=34603008 total_nvlink_bytes=631053824 weight_to_token_ratio=0.058015 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=29568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.251806394e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=64942 payload_rows_remote=49204 dup_rows=33362 dup_rows_remote=10194 dedup_ratio=0.339376 router_tokens_per_expert=[0, 0, 0, 316, 0, 0, 0, 0, 386, 122, 1290, 0, 592, 324, 0, 84, 46, 0, 0, 0, 0, 88, 0, 0, 0, 6386, 0, 302, 0, 2175, 0, 21, 393, 4449, 0, 0, 0, 2477, 0, 733, 2947, 699, 1504, 0, 624, 998, 2439, 0, 0, 100, 0, 509, 0, 0, 50, 0, 596, 0, 7284, 6451, 4518, 81, 0, 168] payload_rows_per_dst=[4746, 4098, 268, 13582, 6840, 9070, 10600, 15738] dup_rows_per_dst=[116, 1498, 0, 4186, 366, 2538, 1490, 23168] combine_rows_per_src=[4746, 4098, 268, 13582, 6840, 9070, 10600, 15738] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87006 payload_rows_remote=77876 dup_rows=11298 dup_rows_remote=9040 dedup_ratio=0.114929 router_tokens_per_expert=[251, 0, 59, 0, 59, 0, 0, 0, 0, 3230, 0, 80, 52, 4, 410, 1918, 53, 7313, 0, 72, 0, 0, 53, 0, 7530, 0, 167, 258, 0, 2163, 72, 6687, 7004, 2013, 0, 0, 0, 0, 0, 0, 72, 0, 2182, 0, 0, 1947, 258, 0, 6, 0, 0, 3198, 0, 0, 0, 0, 0, 1928, 113, 0, 0, 0, 0, 0] payload_rows_per_dst=[15278, 9130, 14626, 14856, 14742, 7954, 6404, 4016] dup_rows_per_dst=[520, 2258, 356, 3838, 3292, 964, 4, 66] combine_rows_per_src=[15278, 9130, 14626, 14856, 14742, 7954, 6404, 4016] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80236 payload_rows_remote=67132 dup_rows=18068 dup_rows_remote=15378 dedup_ratio=0.183797 router_tokens_per_expert=[0, 59, 246, 82, 0, 5242, 0, 441, 0, 0, 24, 0, 77, 0, 0, 0, 3276, 0, 0, 0, 0, 2057, 0, 2294, 0, 0, 21, 7322, 0, 0, 0, 57, 4807, 0, 71, 245, 0, 0, 159, 668, 218, 0, 0, 2122, 0, 0, 0, 0, 930, 0, 4763, 77, 6527, 59, 584, 0, 0, 521, 1348, 627, 906, 3245, 0, 77] payload_rows_per_dst=[13104, 13208, 15652, 6622, 13128, 10150, 3012, 5360] dup_rows_per_dst=[2690, 48, 7694, 24, 4644, 1082, 288, 1598] combine_rows_per_src=[13104, 13208, 15652, 6622, 13128, 10150, 3012, 5360] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87136 payload_rows_remote=73966 dup_rows=11168 dup_rows_remote=7738 dedup_ratio=0.113607 router_tokens_per_expert=[0, 0, 0, 248, 0, 0, 0, 0, 367, 110, 837, 0, 596, 288, 0, 77, 48, 0, 0, 0, 0, 66, 0, 0, 0, 6523, 0, 502, 0, 1760, 0, 191, 229, 4968, 0, 0, 0, 2474, 0, 637, 4464, 439, 910, 0, 568, 1415, 2601, 0, 0, 112, 0, 484, 0, 0, 10, 0, 646, 0, 7294, 6481, 3648, 4, 0, 155] payload_rows_per_dst=[13052, 3700, 14726, 14344, 6652, 13170, 12840, 8652] dup_rows_per_dst=[406, 850, 90, 3608, 28, 3430, 2502, 254] combine_rows_per_src=[13052, 3700, 14726, 14344, 6652, 13170, 12840, 8652] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=316014592 dispatch_meta_bytes=347256 combine_payload_bytes=316014592 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=316361848 ingress_bytes=385220608 link_bytes=385220608 token_bytes=632376440 weight_movement_bytes=69206016 total_nvlink_bytes=701582456 weight_to_token_ratio=0.109438 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=22400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.449554135e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=296837120 dispatch_meta_bytes=330008 combine_payload_bytes=296837120 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=297167128 ingress_bytes=366043136 link_bytes=366043136 token_bytes=594004248 weight_movement_bytes=69206016 total_nvlink_bytes=663210264 weight_to_token_ratio=0.116508 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=20352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.377390852e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=304029696 dispatch_meta_bytes=340536 combine_payload_bytes=304029696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=304370232 ingress_bytes=338632704 link_bytes=338632704 token_bytes=608399928 weight_movement_bytes=34603008 total_nvlink_bytes=643002936 weight_to_token_ratio=0.056875 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=47232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274247603e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87554 payload_rows_remote=82600 dup_rows=10750 dup_rows_remote=10750 dedup_ratio=0.109355 router_tokens_per_expert=[0, 0, 0, 203, 0, 0, 0, 0, 373, 115, 617, 0, 594, 260, 0, 64, 33, 0, 0, 0, 0, 79, 0, 0, 0, 6894, 0, 368, 0, 2105, 0, 4, 205, 5047, 0, 0, 0, 2567, 0, 640, 4450, 327, 989, 0, 580, 1667, 2438, 0, 0, 52, 0, 437, 0, 0, 7, 0, 686, 0, 7408, 6495, 3318, 1, 0, 129] payload_rows_per_dst=[13018, 14666, 14974, 4954, 13712, 5674, 12516, 8040] dup_rows_per_dst=[378, 3168, 66, 0, 3206, 1452, 2252, 228] combine_rows_per_src=[13018, 14666, 14974, 4954, 13712, 5674, 12516, 8040] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88040 payload_rows_remote=73124 dup_rows=10264 dup_rows_remote=10258 dedup_ratio=0.104411 router_tokens_per_expert=[0, 0, 0, 185, 0, 0, 0, 0, 380, 101, 966, 0, 557, 301, 0, 73, 3, 0, 0, 0, 0, 86, 0, 0, 0, 6309, 0, 491, 0, 2035, 0, 0, 198, 5055, 0, 0, 0, 2592, 0, 571, 4231, 423, 1000, 0, 598, 1547, 2509, 0, 0, 82, 0, 447, 0, 0, 77, 0, 682, 0, 7372, 6452, 3681, 1, 0, 147] payload_rows_per_dst=[12910, 14250, 14916, 5052, 13460, 7850, 10884, 8718] dup_rows_per_dst=[364, 3124, 6, 0, 3372, 1870, 1224, 304] combine_rows_per_src=[12910, 14250, 14916, 5052, 13460, 7850, 10884, 8718] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90446 payload_rows_remote=78148 dup_rows=7858 dup_rows_remote=6372 dedup_ratio=0.079936 router_tokens_per_expert=[0, 37, 245, 49, 0, 5070, 0, 470, 0, 0, 3, 0, 107, 0, 0, 1, 3064, 0, 0, 0, 1, 1828, 0, 2240, 0, 0, 37, 7144, 2, 0, 0, 37, 4442, 10, 107, 305, 0, 1, 156, 620, 193, 1, 1, 2356, 3, 0, 2, 1, 959, 1, 5183, 107, 6345, 37, 604, 1, 0, 447, 1584, 714, 971, 3550, 1, 115] payload_rows_per_dst=[11742, 12906, 7784, 14292, 9934, 10060, 12298, 11430] dup_rows_per_dst=[0, 6, 354, 148, 1348, 1182, 1486, 3334] combine_rows_per_src=[11742, 12906, 7784, 14292, 9934, 10060, 12298, 11430] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=318627840 dispatch_meta_bytes=339640 combine_payload_bytes=318627840 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=318967480 ingress_bytes=422436864 link_bytes=422436864 token_bytes=637595320 weight_movement_bytes=103809024 total_nvlink_bytes=741404344 weight_to_token_ratio=0.162813 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.589595910e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=338329600 dispatch_meta_bytes=373400 combine_payload_bytes=338329600 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=338703000 ingress_bytes=407535616 link_bytes=407535616 token_bytes=677032600 weight_movement_bytes=69206016 total_nvlink_bytes=746238616 weight_to_token_ratio=0.102220 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=22656 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.533523714e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85314 payload_rows_remote=76126 dup_rows=12990 dup_rows_remote=11806 dedup_ratio=0.132141 router_tokens_per_expert=[566, 0, 49, 0, 50, 0, 0, 0, 0, 3085, 0, 112, 6, 1, 360, 1872, 59, 7337, 0, 78, 0, 0, 59, 0, 7461, 0, 228, 251, 0, 2088, 77, 6566, 6855, 1593, 0, 0, 0, 0, 0, 0, 78, 0, 1987, 0, 0, 2870, 251, 0, 0, 0, 0, 3168, 0, 0, 0, 0, 0, 1924, 118, 0, 0, 0, 0, 3] payload_rows_per_dst=[15500, 8556, 14674, 5118, 3186, 9188, 14272, 14820] dup_rows_per_dst=[752, 2316, 392, 170, 0, 1184, 5774, 2402] combine_rows_per_src=[15500, 8556, 14674, 5118, 3186, 9188, 14272, 14820] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=201539584 dispatch_meta_bytes=237592 combine_payload_bytes=201539584 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=201777176 ingress_bytes=339951616 link_bytes=339951616 token_bytes=403316760 weight_movement_bytes=138412032 total_nvlink_bytes=541728792 weight_to_token_ratio=0.343184 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=25600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.279210563e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=274972672 dispatch_meta_bytes=330040 combine_payload_bytes=274972672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275302712 ingress_bytes=309575680 link_bytes=309575680 token_bytes=550275384 weight_movement_bytes=34603008 total_nvlink_bytes=584878392 weight_to_token_ratio=0.062883 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=40960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.164908361e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=318980096 dispatch_meta_bytes=347664 combine_payload_bytes=318980096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=319327760 ingress_bytes=353583104 link_bytes=353583104 token_bytes=638307856 weight_movement_bytes=34603008 total_nvlink_bytes=672910864 weight_to_token_ratio=0.054211 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=26496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.330504755e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=302964736 dispatch_meta_bytes=326816 combine_payload_bytes=302964736 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=303291552 ingress_bytes=406773760 link_bytes=406773760 token_bytes=606256288 weight_movement_bytes=103809024 total_nvlink_bytes=710065312 weight_to_token_ratio=0.171230 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=14208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.530656911e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85222 payload_rows_remote=71898 dup_rows=13082 dup_rows_remote=13050 dedup_ratio=0.133077 router_tokens_per_expert=[0, 59, 258, 96, 0, 5142, 0, 375, 0, 0, 16, 0, 72, 0, 0, 0, 3631, 0, 0, 0, 0, 2068, 0, 1998, 0, 0, 25, 7258, 0, 0, 0, 59, 4796, 2, 71, 247, 0, 0, 167, 658, 226, 0, 0, 1949, 0, 0, 0, 0, 1078, 0, 4666, 72, 6590, 59, 629, 0, 0, 453, 1363, 613, 1006, 3371, 0, 79] payload_rows_per_dst=[11858, 13324, 15566, 6886, 11082, 10470, 10482, 5554] dup_rows_per_dst=[2, 32, 7082, 24, 1958, 1142, 1368, 1474] combine_rows_per_src=[11858, 13324, 15566, 6886, 11082, 10470, 10482, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=338329600 dispatch_meta_bytes=373400 combine_payload_bytes=338329600 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=338703000 ingress_bytes=407535616 link_bytes=407535616 token_bytes=677032600 weight_movement_bytes=69206016 total_nvlink_bytes=746238616 weight_to_token_ratio=0.102220 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=22656 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.533523714e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=299515904 dispatch_meta_bytes=333528 combine_payload_bytes=299515904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=299849432 ingress_bytes=334118912 link_bytes=334118912 token_bytes=599365336 weight_movement_bytes=34603008 total_nvlink_bytes=633968344 weight_to_token_ratio=0.057733 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=48256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257262567e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=320094208 dispatch_meta_bytes=338080 combine_payload_bytes=320094208 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=320432288 ingress_bytes=458506240 link_bytes=458506240 token_bytes=640526496 weight_movement_bytes=138412032 total_nvlink_bytes=778938528 weight_to_token_ratio=0.216091 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=31744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.725322068e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=311812096 dispatch_meta_bytes=351728 combine_payload_bytes=311812096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=312163824 ingress_bytes=346415104 link_bytes=346415104 token_bytes=623975920 weight_movement_bytes=34603008 total_nvlink_bytes=658578928 weight_to_token_ratio=0.055456 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.303532148e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=294494208 dispatch_meta_bytes=339792 combine_payload_bytes=294494208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=294834000 ingress_bytes=329097216 link_bytes=329097216 token_bytes=589328208 weight_movement_bytes=34603008 total_nvlink_bytes=623931216 weight_to_token_ratio=0.058716 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=48512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.238366329e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91206 payload_rows_remote=81108 dup_rows=7098 dup_rows_remote=5624 dedup_ratio=0.072205 router_tokens_per_expert=[0, 66, 252, 95, 0, 5632, 0, 408, 0, 1, 5, 0, 91, 0, 0, 0, 3564, 0, 0, 0, 0, 1992, 0, 1881, 0, 0, 12, 6947, 0, 0, 0, 35, 4529, 0, 7, 319, 0, 0, 185, 746, 253, 0, 0, 1807, 0, 0, 0, 0, 738, 1, 4828, 91, 6476, 66, 604, 0, 0, 595, 1620, 690, 961, 3564, 0, 91] payload_rows_per_dst=[12842, 13134, 7374, 13894, 10098, 10378, 11586, 11900] dup_rows_per_dst=[64, 12, 372, 94, 1474, 870, 1070, 3142] combine_rows_per_src=[12842, 13134, 7374, 13894, 10098, 10378, 11586, 11900] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78106 payload_rows_remote=63072 dup_rows=20198 dup_rows_remote=17616 dedup_ratio=0.205465 router_tokens_per_expert=[158, 0, 57, 0, 57, 0, 0, 0, 0, 3604, 0, 100, 83, 33, 426, 1730, 42, 7290, 0, 85, 0, 0, 50, 0, 7482, 0, 115, 241, 0, 1848, 84, 6619, 7480, 1618, 0, 0, 0, 0, 0, 0, 85, 0, 2535, 0, 0, 1843, 241, 0, 133, 0, 0, 2924, 0, 0, 0, 0, 0, 2045, 142, 0, 0, 0, 0, 2] payload_rows_per_dst=[430, 9628, 270, 15132, 6490, 15362, 15760, 15034] dup_rows_per_dst=[114, 2324, 84, 712, 442, 8626, 5314, 2582] combine_rows_per_src=[430, 9628, 270, 15132, 6490, 15362, 15760, 15034] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80236 payload_rows_remote=67132 dup_rows=18068 dup_rows_remote=15378 dedup_ratio=0.183797 router_tokens_per_expert=[0, 59, 246, 82, 0, 5242, 0, 441, 0, 0, 24, 0, 77, 0, 0, 0, 3276, 0, 0, 0, 0, 2057, 0, 2294, 0, 0, 21, 7322, 0, 0, 0, 57, 4807, 0, 71, 245, 0, 0, 159, 668, 218, 0, 0, 2122, 0, 0, 0, 0, 930, 0, 4763, 77, 6527, 59, 584, 0, 0, 521, 1348, 627, 906, 3245, 0, 77] payload_rows_per_dst=[13104, 13208, 15652, 6622, 13128, 10150, 3012, 5360] dup_rows_per_dst=[2690, 48, 7694, 24, 4644, 1082, 288, 1598] combine_rows_per_src=[13104, 13208, 15652, 6622, 13128, 10150, 3012, 5360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84264 payload_rows_remote=72470 dup_rows=14040 dup_rows_remote=10032 dedup_ratio=0.142822 router_tokens_per_expert=[0, 57, 241, 89, 0, 5503, 0, 415, 0, 0, 15, 0, 85, 0, 0, 0, 3655, 0, 0, 0, 0, 1805, 0, 1827, 0, 0, 12, 7007, 0, 0, 0, 55, 4898, 0, 9, 273, 0, 0, 172, 823, 228, 0, 0, 1731, 0, 0, 0, 0, 820, 0, 4994, 84, 6454, 57, 637, 0, 0, 576, 1526, 660, 999, 3360, 0, 85] payload_rows_per_dst=[12604, 5216, 5232, 14016, 10870, 11058, 13474, 11794] dup_rows_per_dst=[6, 20, 140, 132, 1480, 2062, 6192, 4008] combine_rows_per_src=[12604, 5216, 5232, 14016, 10870, 11058, 13474, 11794] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86080 payload_rows_remote=71428 dup_rows=12224 dup_rows_remote=11726 dedup_ratio=0.124349 router_tokens_per_expert=[258, 0, 51, 0, 51, 0, 0, 0, 0, 3223, 0, 82, 0, 5, 416, 1881, 87, 7326, 0, 74, 0, 0, 88, 0, 7514, 0, 253, 244, 0, 2639, 73, 6461, 6920, 1519, 0, 0, 0, 0, 0, 0, 74, 0, 2218, 0, 0, 1949, 244, 0, 2, 0, 0, 3492, 0, 0, 0, 0, 0, 1646, 362, 0, 0, 0, 0, 0] payload_rows_per_dst=[15156, 8926, 14652, 14796, 14108, 7986, 6986, 3470] dup_rows_per_dst=[592, 2288, 498, 4544, 2770, 984, 2, 546] combine_rows_per_src=[15156, 8926, 14652, 14796, 14108, 7986, 6986, 3470] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90940 payload_rows_remote=75972 dup_rows=7364 dup_rows_remote=7220 dedup_ratio=0.074910 router_tokens_per_expert=[0, 50, 244, 84, 0, 4891, 0, 399, 0, 0, 16, 0, 65, 0, 0, 0, 3948, 0, 0, 0, 0, 2320, 0, 1933, 0, 0, 33, 7484, 0, 0, 0, 39, 4550, 0, 64, 154, 0, 0, 143, 470, 193, 0, 0, 2747, 0, 0, 0, 0, 1276, 0, 4508, 62, 6690, 50, 291, 0, 0, 298, 975, 602, 883, 3617, 0, 73] payload_rows_per_dst=[11304, 13510, 8128, 14968, 9472, 11710, 11084, 10764] dup_rows_per_dst=[32, 32, 378, 144, 1290, 2066, 1290, 2132] combine_rows_per_src=[11304, 13510, 8128, 14968, 9472, 11710, 11084, 10764] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90832 payload_rows_remote=80036 dup_rows=7472 dup_rows_remote=5952 dedup_ratio=0.076009 router_tokens_per_expert=[0, 50, 251, 72, 0, 4935, 0, 444, 0, 0, 8, 0, 78, 0, 0, 0, 3619, 0, 0, 0, 0, 2225, 0, 2352, 0, 0, 25, 7320, 0, 0, 0, 45, 4548, 3, 78, 256, 0, 0, 143, 441, 192, 0, 0, 2347, 0, 0, 0, 0, 1241, 0, 4485, 77, 6684, 49, 296, 0, 0, 416, 1262, 640, 946, 3504, 0, 120] payload_rows_per_dst=[11480, 13526, 8642, 14640, 9668, 10796, 11046, 11034] dup_rows_per_dst=[24, 14, 512, 140, 1270, 1520, 1250, 2742] combine_rows_per_src=[11480, 13526, 8642, 14640, 9668, 10796, 11046, 11034] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=332218368 dispatch_meta_bytes=346928 combine_payload_bytes=332218368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=332565296 ingress_bytes=366821376 link_bytes=366821376 token_bytes=664783664 weight_movement_bytes=34603008 total_nvlink_bytes=699386672 weight_to_token_ratio=0.052052 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=36864 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.380319306e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85862 payload_rows_remote=70812 dup_rows=12442 dup_rows_remote=11976 dedup_ratio=0.126567 router_tokens_per_expert=[182, 0, 59, 0, 59, 0, 0, 0, 0, 3183, 0, 86, 66, 5, 431, 1708, 66, 7293, 0, 77, 0, 0, 66, 0, 7458, 0, 145, 246, 0, 2451, 77, 6581, 6844, 1821, 0, 0, 0, 0, 0, 0, 77, 0, 2224, 0, 0, 2466, 246, 0, 0, 0, 0, 3205, 0, 0, 0, 0, 0, 1933, 97, 0, 0, 0, 0, 0] payload_rows_per_dst=[15050, 12626, 14586, 10010, 14208, 8918, 6410, 4054] dup_rows_per_dst=[466, 5410, 418, 1912, 3122, 1108, 0, 6] combine_rows_per_src=[15050, 12626, 14586, 10010, 14208, 8918, 6410, 4054] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80972 payload_rows_remote=69816 dup_rows=17332 dup_rows_remote=16090 dedup_ratio=0.176310 router_tokens_per_expert=[0, 0, 80, 502, 0, 0, 2665, 0, 0, 4203, 1238, 0, 0, 0, 758, 0, 270, 354, 0, 432, 2, 53, 0, 0, 4041, 0, 0, 309, 0, 0, 0, 0, 53, 330, 0, 0, 3386, 1471, 0, 72, 72, 0, 5130, 0, 0, 0, 0, 0, 627, 0, 72, 0, 7583, 0, 94, 0, 6783, 0, 0, 0, 7330, 512, 730, 0] payload_rows_per_dst=[15026, 11156, 15070, 15784, 9606, 10260, 1586, 2484] dup_rows_per_dst=[5034, 1242, 1812, 8082, 1018, 144, 0, 0] combine_rows_per_src=[15026, 11156, 15070, 15784, 9606, 10260, 1586, 2484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90940 payload_rows_remote=75972 dup_rows=7364 dup_rows_remote=7220 dedup_ratio=0.074910 router_tokens_per_expert=[0, 50, 244, 84, 0, 4891, 0, 399, 0, 0, 16, 0, 65, 0, 0, 0, 3948, 0, 0, 0, 0, 2320, 0, 1933, 0, 0, 33, 7484, 0, 0, 0, 39, 4550, 0, 64, 154, 0, 0, 143, 470, 193, 0, 0, 2747, 0, 0, 0, 0, 1276, 0, 4508, 62, 6690, 50, 291, 0, 0, 298, 975, 602, 883, 3617, 0, 73] payload_rows_per_dst=[11304, 13510, 8128, 14968, 9472, 11710, 11084, 10764] dup_rows_per_dst=[32, 32, 378, 144, 1290, 2066, 1290, 2132] combine_rows_per_src=[11304, 13510, 8128, 14968, 9472, 11710, 11084, 10764] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=274972672 dispatch_meta_bytes=330040 combine_payload_bytes=274972672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275302712 ingress_bytes=309575680 link_bytes=309575680 token_bytes=550275384 weight_movement_bytes=34603008 total_nvlink_bytes=584878392 weight_to_token_ratio=0.062883 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=40960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.164908361e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79906 payload_rows_remote=65432 dup_rows=18398 dup_rows_remote=12764 dedup_ratio=0.187154 router_tokens_per_expert=[158, 0, 37, 0, 37, 0, 0, 0, 2, 3322, 0, 156, 86, 59, 434, 1783, 29, 7243, 1, 107, 0, 0, 67, 1, 7416, 1, 162, 245, 1, 2075, 107, 6586, 6859, 1562, 2, 0, 2, 0, 0, 1, 107, 0, 2404, 2, 1, 2591, 245, 0, 86, 0, 0, 3109, 0, 0, 0, 0, 1, 1848, 209, 0, 0, 0, 0, 8] payload_rows_per_dst=[2262, 9152, 7444, 14062, 5346, 12234, 14474, 14932] dup_rows_per_dst=[100, 2532, 250, 2892, 286, 4332, 5634, 2372] combine_rows_per_src=[2262, 9152, 7444, 14062, 5346, 12234, 14474, 14932] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88290 payload_rows_remote=77790 dup_rows=10014 dup_rows_remote=7120 dedup_ratio=0.101868 router_tokens_per_expert=[0, 51, 244, 87, 0, 4523, 0, 371, 0, 0, 9, 0, 74, 0, 0, 0, 3932, 0, 0, 0, 0, 2276, 0, 2046, 0, 0, 22, 7392, 0, 0, 0, 50, 4290, 0, 74, 241, 0, 0, 158, 531, 210, 0, 0, 2737, 0, 0, 0, 0, 1403, 0, 4303, 69, 6575, 51, 363, 0, 0, 374, 1046, 633, 1217, 3725, 0, 75] payload_rows_per_dst=[10548, 13298, 10500, 12182, 9092, 11788, 10990, 9892] dup_rows_per_dst=[4, 18, 2894, 98, 1496, 1970, 1388, 2146] combine_rows_per_src=[10548, 13298, 10500, 12182, 9092, 11788, 10990, 9892] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=296837120 dispatch_meta_bytes=330008 combine_payload_bytes=296837120 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=297167128 ingress_bytes=366043136 link_bytes=366043136 token_bytes=594004248 weight_movement_bytes=69206016 total_nvlink_bytes=663210264 weight_to_token_ratio=0.116508 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=20352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.377390852e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=258342912 dispatch_meta_bytes=322752 combine_payload_bytes=258342912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258665664 ingress_bytes=292945920 link_bytes=292945920 token_bytes=517008576 weight_movement_bytes=34603008 total_nvlink_bytes=551611584 weight_to_token_ratio=0.066929 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.102331913e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79906 payload_rows_remote=65432 dup_rows=18398 dup_rows_remote=12764 dedup_ratio=0.187154 router_tokens_per_expert=[158, 0, 37, 0, 37, 0, 0, 0, 2, 3322, 0, 156, 86, 59, 434, 1783, 29, 7243, 1, 107, 0, 0, 67, 1, 7416, 1, 162, 245, 1, 2075, 107, 6586, 6859, 1562, 2, 0, 2, 0, 0, 1, 107, 0, 2404, 2, 1, 2591, 245, 0, 86, 0, 0, 3109, 0, 0, 0, 0, 1, 1848, 209, 0, 0, 0, 0, 8] payload_rows_per_dst=[2262, 9152, 7444, 14062, 5346, 12234, 14474, 14932] dup_rows_per_dst=[100, 2532, 250, 2892, 286, 4332, 5634, 2372] combine_rows_per_src=[2262, 9152, 7444, 14062, 5346, 12234, 14474, 14932] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=292569088 dispatch_meta_bytes=332616 combine_payload_bytes=292569088 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=292901704 ingress_bytes=361775104 link_bytes=361775104 token_bytes=585470792 weight_movement_bytes=69206016 total_nvlink_bytes=654676808 weight_to_token_ratio=0.118206 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=47616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.361330591e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=311181312 dispatch_meta_bytes=332768 combine_payload_bytes=311181312 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=311514080 ingress_bytes=380387328 link_bytes=380387328 token_bytes=622695392 weight_movement_bytes=69206016 total_nvlink_bytes=691901408 weight_to_token_ratio=0.111139 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=40960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.431366892e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86906 payload_rows_remote=76288 dup_rows=11398 dup_rows_remote=11244 dedup_ratio=0.115946 router_tokens_per_expert=[0, 0, 112, 458, 0, 6, 3041, 0, 0, 3409, 1387, 0, 0, 0, 728, 0, 192, 318, 0, 329, 13, 59, 0, 0, 3820, 0, 0, 326, 0, 0, 0, 0, 59, 329, 0, 0, 3472, 1791, 0, 77, 77, 0, 5309, 0, 0, 0, 0, 0, 701, 0, 77, 0, 7611, 0, 66, 0, 6888, 0, 0, 0, 7218, 570, 709, 0] payload_rows_per_dst=[7094, 9704, 9960, 8292, 10176, 10618, 15718, 15344] dup_rows_per_dst=[140, 1344, 1006, 0, 1280, 154, 1192, 6282] combine_rows_per_src=[7094, 9704, 9960, 8292, 10176, 10618, 15718, 15344] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=290045952 dispatch_meta_bytes=331152 combine_payload_bytes=290045952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=290377104 ingress_bytes=324648960 link_bytes=324648960 token_bytes=580423056 weight_movement_bytes=34603008 total_nvlink_bytes=615026064 weight_to_token_ratio=0.059617 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=45952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.221627900e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=285966336 dispatch_meta_bytes=343624 combine_payload_bytes=285966336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286309960 ingress_bytes=320569344 link_bytes=320569344 token_bytes=572276296 weight_movement_bytes=34603008 total_nvlink_bytes=606879304 weight_to_token_ratio=0.060466 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=30848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.206276634e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=327827456 dispatch_meta_bytes=343952 combine_payload_bytes=327827456 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=328171408 ingress_bytes=397033472 link_bytes=397033472 token_bytes=655998864 weight_movement_bytes=69206016 total_nvlink_bytes=725204880 weight_to_token_ratio=0.105497 gemm_rows_defined=50176 gemm_rows_charged=51200 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020000 gemm_active_groups=10 gemm_peak_group_rows=28800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.494004991e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87006 payload_rows_remote=77876 dup_rows=11298 dup_rows_remote=9040 dedup_ratio=0.114929 router_tokens_per_expert=[251, 0, 59, 0, 59, 0, 0, 0, 0, 3230, 0, 80, 52, 4, 410, 1918, 53, 7313, 0, 72, 0, 0, 53, 0, 7530, 0, 167, 258, 0, 2163, 72, 6687, 7004, 2013, 0, 0, 0, 0, 0, 0, 72, 0, 2182, 0, 0, 1947, 258, 0, 6, 0, 0, 3198, 0, 0, 0, 0, 0, 1928, 113, 0, 0, 0, 0, 0] payload_rows_per_dst=[15278, 9130, 14626, 14856, 14742, 7954, 6404, 4016] dup_rows_per_dst=[520, 2258, 356, 3838, 3292, 964, 4, 66] combine_rows_per_src=[15278, 9130, 14626, 14856, 14742, 7954, 6404, 4016] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=311181312 dispatch_meta_bytes=332768 combine_payload_bytes=311181312 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=311514080 ingress_bytes=380387328 link_bytes=380387328 token_bytes=622695392 weight_movement_bytes=69206016 total_nvlink_bytes=691901408 weight_to_token_ratio=0.111139 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=40960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.431366892e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=268009472 dispatch_meta_bytes=312784 combine_payload_bytes=268009472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268322256 ingress_bytes=302612480 link_bytes=302612480 token_bytes=536331728 weight_movement_bytes=34603008 total_nvlink_bytes=570934736 weight_to_token_ratio=0.064518 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=25344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138706400e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=312475648 dispatch_meta_bytes=350128 combine_payload_bytes=312475648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=312825776 ingress_bytes=347078656 link_bytes=347078656 token_bytes=625301424 weight_movement_bytes=34603008 total_nvlink_bytes=659904432 weight_to_token_ratio=0.055338 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=42368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.306029040e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=318627840 dispatch_meta_bytes=339640 combine_payload_bytes=318627840 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=318967480 ingress_bytes=422436864 link_bytes=422436864 token_bytes=637595320 weight_movement_bytes=103809024 total_nvlink_bytes=741404344 weight_to_token_ratio=0.162813 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.589595910e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=318980096 dispatch_meta_bytes=347664 combine_payload_bytes=318980096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=319327760 ingress_bytes=353583104 link_bytes=353583104 token_bytes=638307856 weight_movement_bytes=34603008 total_nvlink_bytes=672910864 weight_to_token_ratio=0.054211 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=26496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.330504755e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87418 payload_rows_remote=76518 dup_rows=10886 dup_rows_remote=8706 dedup_ratio=0.110738 router_tokens_per_expert=[136, 0, 66, 1, 66, 0, 0, 0, 0, 3699, 0, 138, 89, 41, 419, 1674, 30, 7254, 0, 91, 0, 0, 54, 0, 7436, 0, 133, 253, 0, 1970, 91, 6490, 7405, 1632, 0, 1, 0, 0, 0, 0, 91, 3, 2784, 0, 0, 1700, 252, 0, 14, 0, 0, 2949, 0, 0, 0, 0, 0, 1927, 263, 0, 0, 0, 0, 0] payload_rows_per_dst=[15042, 9816, 14510, 4714, 10900, 8764, 8948, 14724] dup_rows_per_dst=[368, 2304, 348, 180, 2180, 896, 1974, 2636] combine_rows_per_src=[15042, 9816, 14510, 4714, 10900, 8764, 8948, 14724] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77454 payload_rows_remote=61246 dup_rows=20850 dup_rows_remote=3668 dedup_ratio=0.212097 router_tokens_per_expert=[0, 0, 100, 608, 0, 1, 2214, 0, 0, 4384, 1211, 0, 0, 0, 756, 0, 221, 330, 0, 453, 2, 50, 0, 0, 4355, 0, 0, 294, 0, 0, 0, 0, 50, 326, 0, 0, 3372, 1245, 0, 84, 84, 0, 5262, 0, 0, 0, 0, 0, 673, 0, 84, 0, 7587, 0, 79, 0, 6770, 0, 0, 0, 7276, 536, 745, 0] payload_rows_per_dst=[5716, 11482, 1758, 9298, 9326, 10524, 13142, 16208] dup_rows_per_dst=[130, 1220, 354, 0, 828, 168, 968, 17182] combine_rows_per_src=[5716, 11482, 1758, 9298, 9326, 10524, 13142, 16208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85862 payload_rows_remote=70812 dup_rows=12442 dup_rows_remote=11976 dedup_ratio=0.126567 router_tokens_per_expert=[182, 0, 59, 0, 59, 0, 0, 0, 0, 3183, 0, 86, 66, 5, 431, 1708, 66, 7293, 0, 77, 0, 0, 66, 0, 7458, 0, 145, 246, 0, 2451, 77, 6581, 6844, 1821, 0, 0, 0, 0, 0, 0, 77, 0, 2224, 0, 0, 2466, 246, 0, 0, 0, 0, 3205, 0, 0, 0, 0, 0, 1933, 97, 0, 0, 0, 0, 0] payload_rows_per_dst=[15050, 12626, 14586, 10010, 14208, 8918, 6410, 4054] dup_rows_per_dst=[466, 5410, 418, 1912, 3122, 1108, 0, 6] combine_rows_per_src=[15050, 12626, 14586, 10010, 14208, 8918, 6410, 4054] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78106 payload_rows_remote=63072 dup_rows=20198 dup_rows_remote=17616 dedup_ratio=0.205465 router_tokens_per_expert=[158, 0, 57, 0, 57, 0, 0, 0, 0, 3604, 0, 100, 83, 33, 426, 1730, 42, 7290, 0, 85, 0, 0, 50, 0, 7482, 0, 115, 241, 0, 1848, 84, 6619, 7480, 1618, 0, 0, 0, 0, 0, 0, 85, 0, 2535, 0, 0, 1843, 241, 0, 133, 0, 0, 2924, 0, 0, 0, 0, 0, 2045, 142, 0, 0, 0, 0, 2] payload_rows_per_dst=[430, 9628, 270, 15132, 6490, 15362, 15760, 15034] dup_rows_per_dst=[114, 2324, 84, 712, 442, 8626, 5314, 2582] combine_rows_per_src=[430, 9628, 270, 15132, 6490, 15362, 15760, 15034] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86048 payload_rows_remote=70990 dup_rows=12256 dup_rows_remote=10808 dedup_ratio=0.124674 router_tokens_per_expert=[0, 0, 82, 529, 0, 0, 2967, 0, 0, 3247, 1258, 0, 0, 0, 763, 0, 211, 330, 0, 311, 5, 88, 0, 0, 3548, 0, 0, 316, 0, 0, 0, 0, 88, 318, 0, 0, 3228, 2520, 0, 73, 73, 0, 5564, 0, 0, 0, 0, 0, 620, 0, 73, 0, 7576, 0, 86, 0, 6787, 0, 0, 0, 7308, 480, 703, 0] payload_rows_per_dst=[15040, 9386, 15058, 9388, 10762, 11128, 12920, 2366] dup_rows_per_dst=[5690, 1150, 1448, 1196, 1692, 146, 934, 0] combine_rows_per_src=[15040, 9386, 15058, 9388, 10762, 11128, 12920, 2366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88000 payload_rows_remote=82886 dup_rows=10304 dup_rows_remote=10050 dedup_ratio=0.104818 router_tokens_per_expert=[699, 0, 47, 0, 50, 0, 0, 0, 0, 3085, 0, 52, 0, 3, 391, 1988, 71, 7391, 0, 65, 0, 0, 71, 0, 7541, 0, 255, 244, 0, 1901, 64, 6571, 6886, 1730, 0, 0, 0, 0, 0, 0, 65, 0, 2143, 0, 0, 2450, 244, 0, 2, 0, 0, 3172, 0, 0, 0, 0, 0, 1902, 69, 0, 0, 0, 0, 0] payload_rows_per_dst=[15760, 8850, 14782, 5114, 14024, 8732, 6348, 14390] dup_rows_per_dst=[914, 2188, 414, 254, 3208, 1072, 0, 2254] combine_rows_per_src=[15760, 8850, 14782, 5114, 14024, 8732, 6348, 14390] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=313417728 dispatch_meta_bytes=340896 combine_payload_bytes=313417728 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=313758624 ingress_bytes=382623744 link_bytes=382623744 token_bytes=627176352 weight_movement_bytes=69206016 total_nvlink_bytes=696382368 weight_to_token_ratio=0.110345 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.439782345e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80348 payload_rows_remote=65268 dup_rows=17956 dup_rows_remote=12340 dedup_ratio=0.182658 router_tokens_per_expert=[0, 0, 86, 527, 0, 3, 2888, 0, 0, 3596, 1288, 0, 0, 0, 779, 0, 250, 328, 0, 408, 7, 66, 0, 0, 3926, 0, 0, 308, 0, 0, 0, 0, 66, 323, 0, 0, 3675, 1380, 0, 77, 77, 0, 5249, 0, 0, 0, 0, 0, 641, 0, 77, 0, 7585, 0, 82, 0, 6844, 0, 0, 0, 7282, 567, 767, 0] payload_rows_per_dst=[15080, 12046, 14960, 13594, 9902, 10498, 1600, 2668] dup_rows_per_dst=[5616, 3552, 1722, 5772, 1140, 154, 0, 0] combine_rows_per_src=[15080, 12046, 14960, 13594, 9902, 10498, 1600, 2668] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86726 payload_rows_remote=81666 dup_rows=11578 dup_rows_remote=11574 dedup_ratio=0.117778 router_tokens_per_expert=[4305, 0, 0, 72, 717, 4963, 0, 0, 2087, 95, 350, 0, 0, 0, 0, 0, 0, 30, 164, 0, 0, 72, 0, 0, 180, 0, 0, 0, 0, 72, 0, 0, 0, 0, 2692, 0, 0, 0, 1137, 321, 0, 6386, 5374, 8133, 0, 0, 2856, 0, 0, 0, 0, 72, 0, 0, 0, 6959, 797, 0, 0, 302, 607, 133, 258, 18] payload_rows_per_dst=[14542, 5060, 16384, 12916, 11754, 11454, 12192, 2424] dup_rows_per_dst=[5572, 4, 414, 360, 2258, 1164, 0, 1806] combine_rows_per_src=[14542, 5060, 16384, 12916, 11754, 11454, 12192, 2424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=290045952 dispatch_meta_bytes=331152 combine_payload_bytes=290045952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=290377104 ingress_bytes=324648960 link_bytes=324648960 token_bytes=580423056 weight_movement_bytes=34603008 total_nvlink_bytes=615026064 weight_to_token_ratio=0.059617 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=45952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.221627900e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88000 payload_rows_remote=82886 dup_rows=10304 dup_rows_remote=10050 dedup_ratio=0.104818 router_tokens_per_expert=[699, 0, 47, 0, 50, 0, 0, 0, 0, 3085, 0, 52, 0, 3, 391, 1988, 71, 7391, 0, 65, 0, 0, 71, 0, 7541, 0, 255, 244, 0, 1901, 64, 6571, 6886, 1730, 0, 0, 0, 0, 0, 0, 65, 0, 2143, 0, 0, 2450, 244, 0, 2, 0, 0, 3172, 0, 0, 0, 0, 0, 1902, 69, 0, 0, 0, 0, 0] payload_rows_per_dst=[15760, 8850, 14782, 5114, 14024, 8732, 6348, 14390] dup_rows_per_dst=[914, 2188, 414, 254, 3208, 1072, 0, 2254] combine_rows_per_src=[15760, 8850, 14782, 5114, 14024, 8732, 6348, 14390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85314 payload_rows_remote=76126 dup_rows=12990 dup_rows_remote=11806 dedup_ratio=0.132141 router_tokens_per_expert=[566, 0, 49, 0, 50, 0, 0, 0, 0, 3085, 0, 112, 6, 1, 360, 1872, 59, 7337, 0, 78, 0, 0, 59, 0, 7461, 0, 228, 251, 0, 2088, 77, 6566, 6855, 1593, 0, 0, 0, 0, 0, 0, 78, 0, 1987, 0, 0, 2870, 251, 0, 0, 0, 0, 3168, 0, 0, 0, 0, 0, 1924, 118, 0, 0, 0, 0, 3] payload_rows_per_dst=[15500, 8556, 14674, 5118, 3186, 9188, 14272, 14820] dup_rows_per_dst=[752, 2316, 392, 170, 0, 1184, 5774, 2402] combine_rows_per_src=[15500, 8556, 14674, 5118, 3186, 9188, 14272, 14820] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79992 payload_rows_remote=64304 dup_rows=18312 dup_rows_remote=17182 dedup_ratio=0.186279 router_tokens_per_expert=[0, 0, 152, 562, 0, 1, 2607, 0, 0, 4151, 1253, 0, 1, 3, 768, 0, 233, 296, 2, 337, 1, 66, 0, 0, 4405, 0, 1, 319, 0, 0, 0, 2, 67, 352, 1, 2, 2994, 1348, 0, 107, 107, 0, 5239, 0, 0, 0, 1, 2, 676, 1, 107, 0, 7578, 1, 46, 0, 6809, 0, 0, 0, 7184, 554, 816, 0] payload_rows_per_dst=[6494, 11000, 1670, 9454, 8922, 10484, 15688, 16280] dup_rows_per_dst=[150, 1352, 200, 0, 820, 214, 1130, 14446] combine_rows_per_src=[6494, 11000, 1670, 9454, 8922, 10484, 15688, 16280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86080 payload_rows_remote=71428 dup_rows=12224 dup_rows_remote=11726 dedup_ratio=0.124349 router_tokens_per_expert=[258, 0, 51, 0, 51, 0, 0, 0, 0, 3223, 0, 82, 0, 5, 416, 1881, 87, 7326, 0, 74, 0, 0, 88, 0, 7514, 0, 253, 244, 0, 2639, 73, 6461, 6920, 1519, 0, 0, 0, 0, 0, 0, 74, 0, 2218, 0, 0, 1949, 244, 0, 2, 0, 0, 3492, 0, 0, 0, 0, 0, 1646, 362, 0, 0, 0, 0, 0] payload_rows_per_dst=[15156, 8926, 14652, 14796, 14108, 7986, 6986, 3470] dup_rows_per_dst=[592, 2288, 498, 4544, 2770, 984, 2, 546] combine_rows_per_src=[15156, 8926, 14652, 14796, 14108, 7986, 6986, 3470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=258342912 dispatch_meta_bytes=322752 combine_payload_bytes=258342912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258665664 ingress_bytes=292945920 link_bytes=292945920 token_bytes=517008576 weight_movement_bytes=34603008 total_nvlink_bytes=551611584 weight_to_token_ratio=0.066929 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.102331913e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=250863616 dispatch_meta_bytes=259656 combine_payload_bytes=250863616 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=251123272 ingress_bytes=423878656 link_bytes=423878656 token_bytes=501986888 weight_movement_bytes=173015040 total_nvlink_bytes=675001928 weight_to_token_ratio=0.344660 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=20608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.595021257e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=290775040 dispatch_meta_bytes=327192 combine_payload_bytes=290775040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291102232 ingress_bytes=325378048 link_bytes=325378048 token_bytes=581877272 weight_movement_bytes=34603008 total_nvlink_bytes=616480280 weight_to_token_ratio=0.059468 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=41216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.224371400e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87972 payload_rows_remote=84400 dup_rows=10332 dup_rows_remote=10332 dedup_ratio=0.105103 router_tokens_per_expert=[3821, 0, 0, 78, 700, 5164, 0, 0, 2029, 65, 347, 0, 0, 0, 0, 0, 0, 36, 124, 0, 0, 77, 0, 0, 188, 0, 0, 0, 1, 78, 0, 0, 0, 0, 2556, 0, 0, 0, 1016, 301, 0, 6428, 5816, 8142, 0, 0, 2774, 0, 0, 0, 0, 78, 0, 0, 0, 6900, 981, 0, 0, 363, 636, 173, 251, 29] payload_rows_per_dst=[8422, 12660, 12896, 13014, 11276, 3572, 13956, 12176] dup_rows_per_dst=[776, 2550, 290, 376, 2018, 0, 0, 4322] combine_rows_per_src=[8422, 12660, 12896, 13014, 11276, 3572, 13956, 12176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80972 payload_rows_remote=69816 dup_rows=17332 dup_rows_remote=16090 dedup_ratio=0.176310 router_tokens_per_expert=[0, 0, 80, 502, 0, 0, 2665, 0, 0, 4203, 1238, 0, 0, 0, 758, 0, 270, 354, 0, 432, 2, 53, 0, 0, 4041, 0, 0, 309, 0, 0, 0, 0, 53, 330, 0, 0, 3386, 1471, 0, 72, 72, 0, 5130, 0, 0, 0, 0, 0, 627, 0, 72, 0, 7583, 0, 94, 0, 6783, 0, 0, 0, 7330, 512, 730, 0] payload_rows_per_dst=[15026, 11156, 15070, 15784, 9606, 10260, 1586, 2484] dup_rows_per_dst=[5034, 1242, 1812, 8082, 1018, 144, 0, 0] combine_rows_per_src=[15026, 11156, 15070, 15784, 9606, 10260, 1586, 2484] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=339501056 dispatch_meta_bytes=371744 combine_payload_bytes=339501056 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=339872800 ingress_bytes=512516096 link_bytes=512516096 token_bytes=679373856 weight_movement_bytes=173015040 total_nvlink_bytes=852388896 weight_to_token_ratio=0.254668 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=16512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.928556808e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=334503936 dispatch_meta_bytes=372960 combine_payload_bytes=334503936 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=334876896 ingress_bytes=369106944 link_bytes=369106944 token_bytes=669380832 weight_movement_bytes=34603008 total_nvlink_bytes=703983840 weight_to_token_ratio=0.051694 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=29952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388919715e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267337728 dispatch_meta_bytes=310432 combine_payload_bytes=267337728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267648160 ingress_bytes=301940736 link_bytes=301940736 token_bytes=534985888 weight_movement_bytes=34603008 total_nvlink_bytes=569588896 weight_to_token_ratio=0.064680 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=22656 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136178681e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=311812096 dispatch_meta_bytes=351728 combine_payload_bytes=311812096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=312163824 ingress_bytes=346415104 link_bytes=346415104 token_bytes=623975920 weight_movement_bytes=34603008 total_nvlink_bytes=658578928 weight_to_token_ratio=0.055456 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.303532148e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=263389184 dispatch_meta_bytes=325944 combine_payload_bytes=263389184 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=263715128 ingress_bytes=367198208 link_bytes=367198208 token_bytes=527104312 weight_movement_bytes=103809024 total_nvlink_bytes=630913336 weight_to_token_ratio=0.196942 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=42752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.381737295e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=345702400 dispatch_meta_bytes=378928 combine_payload_bytes=345702400 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=346081328 ingress_bytes=518717440 link_bytes=518717440 token_bytes=691783728 weight_movement_bytes=173015040 total_nvlink_bytes=864798768 weight_to_token_ratio=0.250100 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=18176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.951891967e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=292569088 dispatch_meta_bytes=332616 combine_payload_bytes=292569088 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=292901704 ingress_bytes=361775104 link_bytes=361775104 token_bytes=585470792 weight_movement_bytes=69206016 total_nvlink_bytes=654676808 weight_to_token_ratio=0.118206 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=47616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.361330591e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=285966336 dispatch_meta_bytes=343624 combine_payload_bytes=285966336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286309960 ingress_bytes=320569344 link_bytes=320569344 token_bytes=572276296 weight_movement_bytes=34603008 total_nvlink_bytes=606879304 weight_to_token_ratio=0.060466 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=30848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.206276634e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83668 payload_rows_remote=74212 dup_rows=14636 dup_rows_remote=13192 dedup_ratio=0.148885 router_tokens_per_expert=[0, 0, 134, 511, 0, 0, 2329, 0, 0, 4645, 1156, 0, 0, 0, 768, 0, 282, 351, 0, 478, 4, 53, 0, 0, 4367, 0, 0, 292, 0, 0, 0, 0, 54, 342, 0, 0, 3118, 1310, 0, 91, 91, 0, 5192, 0, 0, 0, 0, 0, 581, 0, 91, 0, 7542, 1, 83, 0, 6825, 0, 0, 0, 7252, 475, 734, 0] payload_rows_per_dst=[5808, 11760, 14910, 9318, 9456, 14232, 15620, 2564] dup_rows_per_dst=[140, 1378, 1930, 0, 1444, 8758, 976, 10] combine_rows_per_src=[5808, 11760, 14910, 9318, 9456, 14232, 15620, 2564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80348 payload_rows_remote=65268 dup_rows=17956 dup_rows_remote=12340 dedup_ratio=0.182658 router_tokens_per_expert=[0, 0, 86, 527, 0, 3, 2888, 0, 0, 3596, 1288, 0, 0, 0, 779, 0, 250, 328, 0, 408, 7, 66, 0, 0, 3926, 0, 0, 308, 0, 0, 0, 0, 66, 323, 0, 0, 3675, 1380, 0, 77, 77, 0, 5249, 0, 0, 0, 0, 0, 641, 0, 77, 0, 7585, 0, 82, 0, 6844, 0, 0, 0, 7282, 567, 767, 0] payload_rows_per_dst=[15080, 12046, 14960, 13594, 9902, 10498, 1600, 2668] dup_rows_per_dst=[5616, 3552, 1722, 5772, 1140, 154, 0, 0] combine_rows_per_src=[15080, 12046, 14960, 13594, 9902, 10498, 1600, 2668] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77454 payload_rows_remote=61246 dup_rows=20850 dup_rows_remote=3668 dedup_ratio=0.212097 router_tokens_per_expert=[0, 0, 100, 608, 0, 1, 2214, 0, 0, 4384, 1211, 0, 0, 0, 756, 0, 221, 330, 0, 453, 2, 50, 0, 0, 4355, 0, 0, 294, 0, 0, 0, 0, 50, 326, 0, 0, 3372, 1245, 0, 84, 84, 0, 5262, 0, 0, 0, 0, 0, 673, 0, 84, 0, 7587, 0, 79, 0, 6770, 0, 0, 0, 7276, 536, 745, 0] payload_rows_per_dst=[5716, 11482, 1758, 9298, 9326, 10524, 13142, 16208] dup_rows_per_dst=[130, 1220, 354, 0, 828, 168, 968, 17182] combine_rows_per_src=[5716, 11482, 1758, 9298, 9326, 10524, 13142, 16208] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74064 payload_rows_remote=62694 dup_rows=24240 dup_rows_remote=20420 dedup_ratio=0.246582 router_tokens_per_expert=[4094, 0, 0, 85, 781, 4675, 0, 0, 1743, 88, 431, 0, 0, 0, 0, 0, 0, 59, 148, 0, 0, 83, 0, 0, 213, 0, 0, 0, 0, 85, 0, 0, 0, 0, 2799, 0, 0, 0, 1088, 299, 0, 6352, 5462, 8135, 0, 0, 3428, 0, 0, 0, 0, 85, 0, 0, 0, 6886, 816, 0, 0, 294, 624, 153, 241, 5] payload_rows_per_dst=[7578, 11896, 580, 5000, 7062, 16340, 14238, 11370] dup_rows_per_dst=[634, 1978, 0, 144, 1310, 14942, 1412, 3820] combine_rows_per_src=[7578, 11896, 580, 5000, 7062, 16340, 14238, 11370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89438 payload_rows_remote=73054 dup_rows=8866 dup_rows_remote=8490 dedup_ratio=0.090190 router_tokens_per_expert=[4543, 0, 0, 74, 732, 5365, 0, 0, 1934, 86, 358, 0, 0, 0, 0, 0, 0, 21, 145, 0, 0, 73, 0, 0, 208, 0, 0, 0, 0, 74, 0, 0, 0, 0, 2098, 0, 0, 0, 866, 295, 0, 6428, 5750, 8141, 0, 0, 2438, 0, 0, 0, 0, 74, 0, 0, 0, 7041, 1064, 0, 0, 296, 566, 170, 244, 68] payload_rows_per_dst=[9730, 12284, 16384, 13004, 10226, 6060, 14230, 7520] dup_rows_per_dst=[1714, 2456, 376, 416, 1168, 0, 0, 2736] combine_rows_per_src=[9730, 12284, 16384, 13004, 10226, 6060, 14230, 7520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267337728 dispatch_meta_bytes=310432 combine_payload_bytes=267337728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267648160 ingress_bytes=301940736 link_bytes=301940736 token_bytes=534985888 weight_movement_bytes=34603008 total_nvlink_bytes=569588896 weight_to_token_ratio=0.064680 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=22656 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136178681e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=303972352 dispatch_meta_bytes=349616 combine_payload_bytes=303972352 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=304321968 ingress_bytes=338575360 link_bytes=338575360 token_bytes=608294320 weight_movement_bytes=34603008 total_nvlink_bytes=642897328 weight_to_token_ratio=0.056885 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=26880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274031822e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85126 payload_rows_remote=77000 dup_rows=13178 dup_rows_remote=13178 dedup_ratio=0.134054 router_tokens_per_expert=[0, 0, 52, 494, 0, 0, 2898, 0, 0, 3198, 1338, 0, 0, 0, 730, 0, 209, 323, 0, 267, 0, 71, 0, 0, 3765, 0, 0, 298, 0, 0, 0, 0, 71, 309, 0, 0, 3545, 2228, 0, 64, 64, 0, 5347, 0, 0, 0, 0, 0, 653, 0, 64, 0, 7619, 0, 79, 0, 6878, 0, 0, 0, 7353, 524, 711, 0] payload_rows_per_dst=[9216, 9318, 15092, 8126, 14524, 10694, 15686, 2470] dup_rows_per_dst=[1930, 1214, 1354, 0, 7408, 128, 1144, 0] combine_rows_per_src=[9216, 9318, 15092, 8126, 14524, 10694, 15686, 2470] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78322 payload_rows_remote=64058 dup_rows=19982 dup_rows_remote=13606 dedup_ratio=0.203267 router_tokens_per_expert=[3651, 412, 0, 0, 0, 0, 1441, 1, 0, 1170, 0, 3225, 0, 0, 275, 0, 0, 144, 118, 0, 0, 479, 0, 0, 0, 0, 6918, 261, 546, 0, 130, 4099, 0, 0, 0, 0, 0, 0, 0, 2069, 0, 0, 1689, 154, 6978, 72, 7296, 0, 7048, 0, 0, 146, 0, 72, 0, 0, 95, 0, 0, 0, 83, 0, 0, 580] payload_rows_per_dst=[8948, 14264, 15488, 14544, 15162, 3640, 3038, 3238] dup_rows_per_dst=[2062, 6376, 586, 7490, 2932, 190, 194, 152] combine_rows_per_src=[8948, 14264, 15488, 14544, 15162, 3640, 3038, 3238] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85156 payload_rows_remote=71290 dup_rows=13148 dup_rows_remote=13140 dedup_ratio=0.133748 router_tokens_per_expert=[3798, 0, 3, 108, 791, 4677, 1, 0, 2058, 49, 409, 0, 0, 1, 0, 0, 0, 78, 92, 2, 0, 107, 1, 0, 212, 0, 0, 1, 1, 107, 0, 1, 2, 0, 2904, 0, 0, 0, 1139, 302, 0, 6319, 5501, 8148, 0, 1, 2879, 0, 0, 1, 0, 107, 0, 0, 4, 6825, 1086, 3, 0, 362, 604, 220, 245, 3] payload_rows_per_dst=[8648, 12096, 560, 12856, 9196, 16298, 13866, 11636] dup_rows_per_dst=[754, 2292, 0, 426, 2064, 3192, 8, 4412] combine_rows_per_src=[8648, 12096, 560, 12856, 9196, 16298, 13866, 11636] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78654 payload_rows_remote=64280 dup_rows=19650 dup_rows_remote=14406 dedup_ratio=0.199890 router_tokens_per_expert=[3894, 0, 0, 77, 784, 5054, 0, 0, 1788, 82, 386, 0, 0, 0, 0, 0, 0, 23, 141, 0, 0, 77, 0, 0, 198, 0, 0, 0, 0, 77, 0, 0, 0, 0, 2686, 0, 0, 0, 1028, 313, 0, 6358, 5708, 8133, 0, 0, 2905, 0, 0, 0, 0, 77, 0, 0, 0, 6931, 1038, 0, 0, 340, 649, 155, 246, 4] payload_rows_per_dst=[14374, 4506, 16384, 12870, 11582, 15898, 154, 2886] dup_rows_per_dst=[5244, 6, 364, 396, 2282, 9380, 0, 1978] combine_rows_per_src=[14374, 4506, 16384, 12870, 11582, 15898, 154, 2886] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=256794624 dispatch_meta_bytes=332456 combine_payload_bytes=256794624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=257127080 ingress_bytes=291397632 link_bytes=291397632 token_bytes=513921704 weight_movement_bytes=34603008 total_nvlink_bytes=548524712 weight_to_token_ratio=0.067331 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=30592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.096505830e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=299229184 dispatch_meta_bytes=326176 combine_payload_bytes=299229184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=299555360 ingress_bytes=333832192 link_bytes=333832192 token_bytes=598784544 weight_movement_bytes=34603008 total_nvlink_bytes=633387552 weight_to_token_ratio=0.057789 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=47104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256183663e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86906 payload_rows_remote=76288 dup_rows=11398 dup_rows_remote=11244 dedup_ratio=0.115946 router_tokens_per_expert=[0, 0, 112, 458, 0, 6, 3041, 0, 0, 3409, 1387, 0, 0, 0, 728, 0, 192, 318, 0, 329, 13, 59, 0, 0, 3820, 0, 0, 326, 0, 0, 0, 0, 59, 329, 0, 0, 3472, 1791, 0, 77, 77, 0, 5309, 0, 0, 0, 0, 0, 701, 0, 77, 0, 7611, 0, 66, 0, 6888, 0, 0, 0, 7218, 570, 709, 0] payload_rows_per_dst=[7094, 9704, 9960, 8292, 10176, 10618, 15718, 15344] dup_rows_per_dst=[140, 1344, 1006, 0, 1280, 154, 1192, 6282] combine_rows_per_src=[7094, 9704, 9960, 8292, 10176, 10618, 15718, 15344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86048 payload_rows_remote=70990 dup_rows=12256 dup_rows_remote=10808 dedup_ratio=0.124674 router_tokens_per_expert=[0, 0, 82, 529, 0, 0, 2967, 0, 0, 3247, 1258, 0, 0, 0, 763, 0, 211, 330, 0, 311, 5, 88, 0, 0, 3548, 0, 0, 316, 0, 0, 0, 0, 88, 318, 0, 0, 3228, 2520, 0, 73, 73, 0, 5564, 0, 0, 0, 0, 0, 620, 0, 73, 0, 7576, 0, 86, 0, 6787, 0, 0, 0, 7308, 480, 703, 0] payload_rows_per_dst=[15040, 9386, 15058, 9388, 10762, 11128, 12920, 2366] dup_rows_per_dst=[5690, 1150, 1448, 1196, 1692, 146, 934, 0] combine_rows_per_src=[15040, 9386, 15058, 9388, 10762, 11128, 12920, 2366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=250863616 dispatch_meta_bytes=259656 combine_payload_bytes=250863616 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=251123272 ingress_bytes=423878656 link_bytes=423878656 token_bytes=501986888 weight_movement_bytes=173015040 total_nvlink_bytes=675001928 weight_to_token_ratio=0.344660 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=20608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.595021257e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89478 payload_rows_remote=75492 dup_rows=8826 dup_rows_remote=5542 dedup_ratio=0.089783 router_tokens_per_expert=[3362, 466, 0, 0, 0, 0, 1525, 27, 0, 973, 0, 3098, 0, 0, 248, 0, 0, 155, 66, 0, 0, 452, 0, 0, 0, 0, 7014, 254, 425, 0, 100, 4574, 0, 0, 0, 0, 0, 0, 0, 1944, 0, 0, 1940, 116, 7000, 78, 7321, 0, 6930, 0, 0, 153, 0, 77, 0, 0, 66, 0, 0, 0, 95, 0, 0, 693] payload_rows_per_dst=[8810, 7910, 14930, 10462, 5178, 13986, 13864, 14338] dup_rows_per_dst=[1950, 728, 494, 244, 272, 3284, 456, 1398] combine_rows_per_src=[8810, 7910, 14930, 10462, 5178, 13986, 13864, 14338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86726 payload_rows_remote=81666 dup_rows=11578 dup_rows_remote=11574 dedup_ratio=0.117778 router_tokens_per_expert=[4305, 0, 0, 72, 717, 4963, 0, 0, 2087, 95, 350, 0, 0, 0, 0, 0, 0, 30, 164, 0, 0, 72, 0, 0, 180, 0, 0, 0, 0, 72, 0, 0, 0, 0, 2692, 0, 0, 0, 1137, 321, 0, 6386, 5374, 8133, 0, 0, 2856, 0, 0, 0, 0, 72, 0, 0, 0, 6959, 797, 0, 0, 302, 607, 133, 258, 18] payload_rows_per_dst=[14542, 5060, 16384, 12916, 11754, 11454, 12192, 2424] dup_rows_per_dst=[5572, 4, 414, 360, 2258, 1164, 0, 1806] combine_rows_per_src=[14542, 5060, 16384, 12916, 11754, 11454, 12192, 2424] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=315392000 dispatch_meta_bytes=360712 combine_payload_bytes=315392000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=315752712 ingress_bytes=349995008 link_bytes=349995008 token_bytes=631144712 weight_movement_bytes=34603008 total_nvlink_bytes=665747720 weight_to_token_ratio=0.054826 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=32256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.317003038e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=262381568 dispatch_meta_bytes=310656 combine_payload_bytes=262381568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262692224 ingress_bytes=296984576 link_bytes=296984576 token_bytes=525073792 weight_movement_bytes=34603008 total_nvlink_bytes=559676800 weight_to_token_ratio=0.065901 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=25344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.117529050e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=292003840 dispatch_meta_bytes=337720 combine_payload_bytes=292003840 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=292341560 ingress_bytes=361209856 link_bytes=361209856 token_bytes=584345400 weight_movement_bytes=69206016 total_nvlink_bytes=653551416 weight_to_token_ratio=0.118433 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=47744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.359203608e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=263290880 dispatch_meta_bytes=314744 combine_payload_bytes=263290880 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=263605624 ingress_bytes=332496896 link_bytes=332496896 token_bytes=526896504 weight_movement_bytes=69206016 total_nvlink_bytes=596102520 weight_to_token_ratio=0.131347 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=32256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.251159051e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=309215232 dispatch_meta_bytes=324136 combine_payload_bytes=309215232 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=309539368 ingress_bytes=482230272 link_bytes=482230272 token_bytes=618754600 weight_movement_bytes=173015040 total_nvlink_bytes=791769640 weight_to_token_ratio=0.279618 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=16640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.814593691e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=312475648 dispatch_meta_bytes=350128 combine_payload_bytes=312475648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=312825776 ingress_bytes=347078656 link_bytes=347078656 token_bytes=625301424 weight_movement_bytes=34603008 total_nvlink_bytes=659904432 weight_to_token_ratio=0.055338 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=42368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.306029040e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=290775040 dispatch_meta_bytes=327192 combine_payload_bytes=290775040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291102232 ingress_bytes=325378048 link_bytes=325378048 token_bytes=581877272 weight_movement_bytes=34603008 total_nvlink_bytes=616480280 weight_to_token_ratio=0.059468 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=41216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.224371400e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88272 payload_rows_remote=75946 dup_rows=10032 dup_rows_remote=7468 dedup_ratio=0.102051 router_tokens_per_expert=[4141, 0, 0, 92, 764, 4809, 0, 0, 1701, 94, 434, 0, 0, 0, 0, 0, 0, 70, 160, 0, 0, 91, 0, 0, 203, 0, 0, 0, 0, 91, 0, 0, 0, 0, 2716, 0, 0, 0, 1075, 327, 1, 6225, 5501, 8125, 0, 0, 3327, 1, 0, 0, 0, 91, 0, 0, 0, 6911, 863, 0, 0, 315, 569, 196, 251, 8] payload_rows_per_dst=[9252, 12096, 16384, 12632, 12326, 4, 14004, 11574] dup_rows_per_dst=[742, 1980, 508, 406, 2564, 0, 0, 3832] combine_rows_per_src=[9252, 12096, 16384, 12632, 12326, 4, 14004, 11574] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=60350 dup_rows=23908 dup_rows_remote=22612 dedup_ratio=0.243205 router_tokens_per_expert=[4191, 406, 0, 0, 0, 0, 1111, 10, 0, 1258, 0, 3537, 0, 0, 411, 0, 0, 169, 88, 0, 0, 446, 0, 0, 0, 0, 6854, 244, 540, 0, 130, 3055, 0, 0, 0, 0, 0, 0, 0, 2500, 0, 0, 1494, 145, 7009, 88, 7305, 0, 7116, 0, 0, 144, 0, 84, 0, 0, 88, 0, 0, 0, 93, 0, 0, 636] payload_rows_per_dst=[9606, 9300, 1406, 4518, 5000, 15738, 14782, 14046] dup_rows_per_dst=[1830, 1112, 0, 60, 0, 12300, 7310, 1296] combine_rows_per_src=[9606, 9300, 1406, 4518, 5000, 15738, 14782, 14046] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78654 payload_rows_remote=64280 dup_rows=19650 dup_rows_remote=14406 dedup_ratio=0.199890 router_tokens_per_expert=[3894, 0, 0, 77, 784, 5054, 0, 0, 1788, 82, 386, 0, 0, 0, 0, 0, 0, 23, 141, 0, 0, 77, 0, 0, 198, 0, 0, 0, 0, 77, 0, 0, 0, 0, 2686, 0, 0, 0, 1028, 313, 0, 6358, 5708, 8133, 0, 0, 2905, 0, 0, 0, 0, 77, 0, 0, 0, 6931, 1038, 0, 0, 340, 649, 155, 246, 4] payload_rows_per_dst=[14374, 4506, 16384, 12870, 11582, 15898, 154, 2886] dup_rows_per_dst=[5244, 6, 364, 396, 2282, 9380, 0, 1978] combine_rows_per_src=[14374, 4506, 16384, 12870, 11582, 15898, 154, 2886] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=334503936 dispatch_meta_bytes=372960 combine_payload_bytes=334503936 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=334876896 ingress_bytes=369106944 link_bytes=369106944 token_bytes=669380832 weight_movement_bytes=34603008 total_nvlink_bytes=703983840 weight_to_token_ratio=0.051694 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=29952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388919715e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74064 payload_rows_remote=62694 dup_rows=24240 dup_rows_remote=20420 dedup_ratio=0.246582 router_tokens_per_expert=[4094, 0, 0, 85, 781, 4675, 0, 0, 1743, 88, 431, 0, 0, 0, 0, 0, 0, 59, 148, 0, 0, 83, 0, 0, 213, 0, 0, 0, 0, 85, 0, 0, 0, 0, 2799, 0, 0, 0, 1088, 299, 0, 6352, 5462, 8135, 0, 0, 3428, 0, 0, 0, 0, 85, 0, 0, 0, 6886, 816, 0, 0, 294, 624, 153, 241, 5] payload_rows_per_dst=[7578, 11896, 580, 5000, 7062, 16340, 14238, 11370] dup_rows_per_dst=[634, 1978, 0, 144, 1310, 14942, 1412, 3820] combine_rows_per_src=[7578, 11896, 580, 5000, 7062, 16340, 14238, 11370] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90040 payload_rows_remote=74602 dup_rows=8264 dup_rows_remote=7812 dedup_ratio=0.084066 router_tokens_per_expert=[3362, 663, 0, 0, 0, 0, 1522, 21, 0, 915, 0, 2642, 0, 0, 249, 0, 0, 147, 46, 0, 0, 408, 0, 0, 0, 0, 7025, 244, 378, 0, 106, 4689, 0, 0, 0, 0, 0, 0, 0, 2014, 0, 0, 2155, 137, 7056, 74, 7344, 0, 6913, 0, 0, 184, 0, 73, 0, 0, 86, 0, 0, 0, 84, 0, 0, 615] payload_rows_per_dst=[9008, 6978, 15438, 10654, 15228, 4556, 13832, 14346] dup_rows_per_dst=[2128, 634, 452, 180, 2912, 176, 508, 1274] combine_rows_per_src=[9008, 6978, 15438, 10654, 15228, 4556, 13832, 14346] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=311074816 dispatch_meta_bytes=333656 combine_payload_bytes=311074816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=311408472 ingress_bytes=345677824 link_bytes=345677824 token_bytes=622483288 weight_movement_bytes=34603008 total_nvlink_bytes=657086296 weight_to_token_ratio=0.055589 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=20480 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.300757822e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84880 payload_rows_remote=69080 dup_rows=13424 dup_rows_remote=2200 dedup_ratio=0.136556 router_tokens_per_expert=[0, 0, 374, 0, 0, 0, 850, 0, 0, 0, 131, 95, 7388, 0, 0, 6760, 4, 0, 0, 0, 0, 4141, 0, 0, 0, 0, 0, 8120, 0, 0, 7676, 0, 0, 0, 72, 0, 0, 7814, 0, 0, 0, 0, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 460, 0, 71, 0, 2813, 0, 0, 484, 1420, 0, 0] payload_rows_per_dst=[16012, 15800, 8290, 16240, 15772, 958, 2586, 9222] dup_rows_per_dst=[1788, 11224, 0, 0, 0, 0, 200, 212] combine_rows_per_src=[16012, 15800, 8290, 16240, 15772, 958, 2586, 9222] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=263290880 dispatch_meta_bytes=314744 combine_payload_bytes=263290880 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=263605624 ingress_bytes=332496896 link_bytes=332496896 token_bytes=526896504 weight_movement_bytes=69206016 total_nvlink_bytes=596102520 weight_to_token_ratio=0.131347 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=32256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.251159051e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76180 payload_rows_remote=62260 dup_rows=22124 dup_rows_remote=21526 dedup_ratio=0.225057 router_tokens_per_expert=[3571, 647, 0, 0, 0, 0, 1306, 40, 0, 1292, 1, 3120, 1, 0, 251, 0, 1, 214, 62, 0, 2, 434, 0, 1, 0, 1, 6796, 246, 541, 0, 104, 3732, 3, 0, 0, 0, 3, 0, 0, 2326, 0, 2, 1693, 86, 7063, 107, 7334, 0, 6960, 1, 0, 191, 0, 107, 0, 0, 49, 0, 0, 3, 64, 0, 0, 797] payload_rows_per_dst=[9008, 8440, 1426, 9018, 4658, 15680, 13920, 14030] dup_rows_per_dst=[2120, 890, 2, 230, 6, 16890, 598, 1388] combine_rows_per_src=[9008, 8440, 1426, 9018, 4658, 15680, 13920, 14030] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87910 payload_rows_remote=74774 dup_rows=10394 dup_rows_remote=10042 dedup_ratio=0.105733 router_tokens_per_expert=[4397, 0, 0, 65, 699, 5581, 0, 0, 2198, 79, 362, 0, 0, 0, 0, 0, 0, 8, 176, 0, 0, 64, 0, 0, 176, 0, 0, 0, 0, 65, 0, 0, 0, 0, 2025, 0, 0, 0, 884, 294, 0, 6503, 5582, 8142, 0, 0, 2406, 0, 0, 0, 0, 65, 0, 0, 0, 7057, 998, 0, 0, 307, 603, 125, 244, 47] payload_rows_per_dst=[9004, 13120, 16384, 13136, 10170, 0, 14244, 11852] dup_rows_per_dst=[1318, 3320, 396, 352, 1048, 0, 0, 3960] combine_rows_per_src=[9004, 13120, 16384, 13136, 10170, 0, 14244, 11852] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74392 payload_rows_remote=61244 dup_rows=23912 dup_rows_remote=17080 dedup_ratio=0.243245 router_tokens_per_expert=[3817, 422, 0, 0, 0, 0, 1632, 8, 0, 1176, 0, 3105, 0, 0, 268, 0, 0, 154, 80, 0, 0, 465, 0, 0, 0, 0, 6856, 268, 547, 0, 128, 3396, 0, 0, 0, 0, 0, 0, 0, 2307, 0, 0, 1851, 141, 7004, 77, 7340, 0, 7005, 0, 0, 165, 0, 77, 0, 0, 82, 0, 0, 0, 111, 0, 0, 670] payload_rows_per_dst=[13148, 15434, 15554, 15122, 9104, 3974, 330, 1726] dup_rows_per_dst=[6832, 7674, 524, 7268, 1296, 164, 154, 0] combine_rows_per_src=[13148, 15434, 15554, 15122, 9104, 3974, 330, 1726] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=247193600 dispatch_meta_bytes=331848 combine_payload_bytes=247193600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=247525448 ingress_bytes=281796608 link_bytes=281796608 token_bytes=494719048 weight_movement_bytes=34603008 total_nvlink_bytes=529322056 weight_to_token_ratio=0.069945 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=42624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.060377949e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=305569792 dispatch_meta_bytes=329656 combine_payload_bytes=305569792 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=305899448 ingress_bytes=340172800 link_bytes=340172800 token_bytes=611469240 weight_movement_bytes=34603008 total_nvlink_bytes=646072248 weight_to_token_ratio=0.056590 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=43776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.280042860e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87972 payload_rows_remote=84400 dup_rows=10332 dup_rows_remote=10332 dedup_ratio=0.105103 router_tokens_per_expert=[3821, 0, 0, 78, 700, 5164, 0, 0, 2029, 65, 347, 0, 0, 0, 0, 0, 0, 36, 124, 0, 0, 77, 0, 0, 188, 0, 0, 0, 1, 78, 0, 0, 0, 0, 2556, 0, 0, 0, 1016, 301, 0, 6428, 5816, 8142, 0, 0, 2774, 0, 0, 0, 0, 78, 0, 0, 0, 6900, 981, 0, 0, 363, 636, 173, 251, 29] payload_rows_per_dst=[8422, 12660, 12896, 13014, 11276, 3572, 13956, 12176] dup_rows_per_dst=[776, 2550, 290, 376, 2018, 0, 0, 4322] combine_rows_per_src=[8422, 12660, 12896, 13014, 11276, 3572, 13956, 12176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89438 payload_rows_remote=73054 dup_rows=8866 dup_rows_remote=8490 dedup_ratio=0.090190 router_tokens_per_expert=[4543, 0, 0, 74, 732, 5365, 0, 0, 1934, 86, 358, 0, 0, 0, 0, 0, 0, 21, 145, 0, 0, 73, 0, 0, 208, 0, 0, 0, 0, 74, 0, 0, 0, 0, 2098, 0, 0, 0, 866, 295, 0, 6428, 5750, 8141, 0, 0, 2438, 0, 0, 0, 0, 74, 0, 0, 0, 7041, 1064, 0, 0, 296, 566, 170, 244, 68] payload_rows_per_dst=[9730, 12284, 16384, 13004, 10226, 6060, 14230, 7520] dup_rows_per_dst=[1714, 2456, 376, 416, 1168, 0, 0, 2736] combine_rows_per_src=[9730, 12284, 16384, 13004, 10226, 6060, 14230, 7520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=256794624 dispatch_meta_bytes=332456 combine_payload_bytes=256794624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=257127080 ingress_bytes=291397632 link_bytes=291397632 token_bytes=513921704 weight_movement_bytes=34603008 total_nvlink_bytes=548524712 weight_to_token_ratio=0.067331 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=30592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.096505830e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91066 payload_rows_remote=74682 dup_rows=7238 dup_rows_remote=6434 dedup_ratio=0.073629 router_tokens_per_expert=[0, 0, 467, 0, 0, 0, 966, 0, 0, 0, 127, 65, 7313, 0, 0, 6800, 0, 0, 0, 0, 0, 3356, 0, 0, 0, 0, 0, 8115, 0, 0, 7742, 0, 0, 0, 77, 0, 0, 7814, 0, 0, 0, 0, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 0, 78, 0, 3129, 0, 0, 507, 1610, 0, 0] payload_rows_per_dst=[2978, 384, 15014, 15192, 15782, 16384, 15058, 10274] dup_rows_per_dst=[180, 0, 5298, 0, 0, 804, 738, 218] combine_rows_per_src=[2978, 384, 15014, 15192, 15782, 16384, 15058, 10274] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=282951680 dispatch_meta_bytes=285120 combine_payload_bytes=282951680 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=283236800 ingress_bytes=386760704 link_bytes=386760704 token_bytes=566188480 weight_movement_bytes=103809024 total_nvlink_bytes=669997504 weight_to_token_ratio=0.183347 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=33664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.455349392e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=255016960 dispatch_meta_bytes=335144 combine_payload_bytes=255016960 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=255352104 ingress_bytes=324222976 link_bytes=324222976 token_bytes=510369064 weight_movement_bytes=69206016 total_nvlink_bytes=579575080 weight_to_token_ratio=0.135600 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=43520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.220024957e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=306274304 dispatch_meta_bytes=339264 combine_payload_bytes=306274304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=306613568 ingress_bytes=340877312 link_bytes=340877312 token_bytes=612887872 weight_movement_bytes=34603008 total_nvlink_bytes=647490880 weight_to_token_ratio=0.056459 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=46976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.282693882e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=250855424 dispatch_meta_bytes=313296 combine_payload_bytes=250855424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=251168720 ingress_bytes=285458432 link_bytes=285458432 token_bytes=502024144 weight_movement_bytes=34603008 total_nvlink_bytes=536627152 weight_to_token_ratio=0.068927 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=29568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.074157098e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=305897472 dispatch_meta_bytes=324464 combine_payload_bytes=305897472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=306221936 ingress_bytes=340500480 link_bytes=340500480 token_bytes=612119408 weight_movement_bytes=34603008 total_nvlink_bytes=646722416 weight_to_token_ratio=0.056530 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=45184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.281275894e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78322 payload_rows_remote=64058 dup_rows=19982 dup_rows_remote=13606 dedup_ratio=0.203267 router_tokens_per_expert=[3651, 412, 0, 0, 0, 0, 1441, 1, 0, 1170, 0, 3225, 0, 0, 275, 0, 0, 144, 118, 0, 0, 479, 0, 0, 0, 0, 6918, 261, 546, 0, 130, 4099, 0, 0, 0, 0, 0, 0, 0, 2069, 0, 0, 1689, 154, 6978, 72, 7296, 0, 7048, 0, 0, 146, 0, 72, 0, 0, 95, 0, 0, 0, 83, 0, 0, 580] payload_rows_per_dst=[8948, 14264, 15488, 14544, 15162, 3640, 3038, 3238] dup_rows_per_dst=[2062, 6376, 586, 7490, 2932, 190, 194, 152] combine_rows_per_src=[8948, 14264, 15488, 14544, 15162, 3640, 3038, 3238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=345702400 dispatch_meta_bytes=378928 combine_payload_bytes=345702400 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=346081328 ingress_bytes=518717440 link_bytes=518717440 token_bytes=691783728 weight_movement_bytes=173015040 total_nvlink_bytes=864798768 weight_to_token_ratio=0.250100 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=18176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.951891967e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=299229184 dispatch_meta_bytes=326176 combine_payload_bytes=299229184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=299555360 ingress_bytes=333832192 link_bytes=333832192 token_bytes=598784544 weight_movement_bytes=34603008 total_nvlink_bytes=633387552 weight_to_token_ratio=0.057789 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=47104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256183663e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89154 payload_rows_remote=74066 dup_rows=9150 dup_rows_remote=5882 dedup_ratio=0.093079 router_tokens_per_expert=[4318, 520, 0, 0, 0, 0, 1166, 33, 0, 1282, 0, 3304, 0, 0, 459, 0, 0, 182, 90, 0, 0, 412, 0, 0, 0, 0, 6902, 256, 496, 0, 123, 3042, 0, 0, 0, 0, 0, 0, 0, 2229, 0, 0, 1534, 160, 6949, 92, 7330, 0, 7145, 0, 0, 174, 0, 91, 0, 0, 94, 0, 0, 0, 92, 0, 0, 677] payload_rows_per_dst=[10078, 8930, 15518, 7696, 15088, 3384, 14292, 14168] dup_rows_per_dst=[1996, 1160, 510, 138, 3268, 188, 528, 1362] combine_rows_per_src=[10078, 8930, 15518, 7696, 15088, 3384, 14292, 14168] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94850 payload_rows_remote=88886 dup_rows=3454 dup_rows_remote=3454 dedup_ratio=0.035136 router_tokens_per_expert=[0, 0, 492, 0, 0, 0, 844, 0, 0, 0, 124, 86, 7379, 0, 0, 6945, 11, 0, 0, 0, 0, 2971, 0, 0, 0, 0, 0, 8118, 0, 0, 7725, 0, 0, 0, 73, 0, 0, 7827, 0, 0, 0, 0, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 0, 64, 0, 3130, 0, 0, 467, 1917, 0, 0] payload_rows_per_dst=[16080, 14310, 5964, 6950, 15800, 9740, 15154, 10852] dup_rows_per_dst=[2042, 0, 0, 0, 0, 542, 694, 176] combine_rows_per_src=[16080, 14310, 5964, 6950, 15800, 9740, 15154, 10852] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83652 payload_rows_remote=67268 dup_rows=14652 dup_rows_remote=6532 dedup_ratio=0.149048 router_tokens_per_expert=[0, 0, 426, 0, 0, 0, 817, 0, 0, 0, 141, 88, 7318, 0, 0, 6785, 6, 0, 0, 0, 0, 4412, 0, 0, 0, 0, 0, 8108, 0, 0, 7687, 0, 0, 0, 84, 0, 0, 7731, 0, 0, 0, 0, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 397, 0, 85, 0, 3029, 0, 0, 501, 991, 0, 0] payload_rows_per_dst=[2386, 458, 13104, 15374, 4692, 16384, 14870, 16384] dup_rows_per_dst=[100, 0, 4778, 0, 0, 924, 730, 8120] combine_rows_per_src=[2386, 458, 13104, 15374, 4692, 16384, 14870, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74392 payload_rows_remote=61244 dup_rows=23912 dup_rows_remote=17080 dedup_ratio=0.243245 router_tokens_per_expert=[3817, 422, 0, 0, 0, 0, 1632, 8, 0, 1176, 0, 3105, 0, 0, 268, 0, 0, 154, 80, 0, 0, 465, 0, 0, 0, 0, 6856, 268, 547, 0, 128, 3396, 0, 0, 0, 0, 0, 0, 0, 2307, 0, 0, 1851, 141, 7004, 77, 7340, 0, 7005, 0, 0, 165, 0, 77, 0, 0, 82, 0, 0, 0, 111, 0, 0, 670] payload_rows_per_dst=[13148, 15434, 15554, 15122, 9104, 3974, 330, 1726] dup_rows_per_dst=[6832, 7674, 524, 7268, 1296, 164, 154, 0] combine_rows_per_src=[13148, 15434, 15554, 15122, 9104, 3974, 330, 1726] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=262381568 dispatch_meta_bytes=310656 combine_payload_bytes=262381568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262692224 ingress_bytes=296984576 link_bytes=296984576 token_bytes=525073792 weight_movement_bytes=34603008 total_nvlink_bytes=559676800 weight_to_token_ratio=0.065901 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=25344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.117529050e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=60350 dup_rows=23908 dup_rows_remote=22612 dedup_ratio=0.243205 router_tokens_per_expert=[4191, 406, 0, 0, 0, 0, 1111, 10, 0, 1258, 0, 3537, 0, 0, 411, 0, 0, 169, 88, 0, 0, 446, 0, 0, 0, 0, 6854, 244, 540, 0, 130, 3055, 0, 0, 0, 0, 0, 0, 0, 2500, 0, 0, 1494, 145, 7009, 88, 7305, 0, 7116, 0, 0, 144, 0, 84, 0, 0, 88, 0, 0, 0, 93, 0, 0, 636] payload_rows_per_dst=[9606, 9300, 1406, 4518, 5000, 15738, 14782, 14046] dup_rows_per_dst=[1830, 1112, 0, 60, 0, 12300, 7310, 1296] combine_rows_per_src=[9606, 9300, 1406, 4518, 5000, 15738, 14782, 14046] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=303374336 dispatch_meta_bytes=319792 combine_payload_bytes=303374336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=303694128 ingress_bytes=337977344 link_bytes=337977344 token_bytes=607068464 weight_movement_bytes=34603008 total_nvlink_bytes=641671472 weight_to_token_ratio=0.057000 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=31872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.271781536e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=250855424 dispatch_meta_bytes=313296 combine_payload_bytes=250855424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=251168720 ingress_bytes=285458432 link_bytes=285458432 token_bytes=502024144 weight_movement_bytes=34603008 total_nvlink_bytes=536627152 weight_to_token_ratio=0.068927 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=29568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.074157098e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87664 payload_rows_remote=72742 dup_rows=10640 dup_rows_remote=9842 dedup_ratio=0.108236 router_tokens_per_expert=[0, 1, 393, 0, 0, 0, 978, 0, 0, 0, 144, 49, 7267, 0, 3, 6686, 6, 0, 0, 1, 1, 4301, 0, 1, 0, 0, 0, 8079, 0, 3, 7716, 0, 0, 0, 108, 0, 0, 7689, 3, 0, 1, 0, 516, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 486, 0, 107, 0, 3037, 0, 0, 527, 1043, 1, 0] payload_rows_per_dst=[2612, 392, 15152, 15438, 11946, 16384, 14922, 10818] dup_rows_per_dst=[132, 0, 6840, 0, 0, 818, 798, 2052] combine_rows_per_src=[2612, 392, 15152, 15438, 11946, 16384, 14922, 10818] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81850 payload_rows_remote=65734 dup_rows=16454 dup_rows_remote=15024 dedup_ratio=0.167379 router_tokens_per_expert=[7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 59, 799, 0, 1280, 0, 144, 7818, 7987, 0, 0, 0, 0, 0, 632, 0, 7758, 0, 59, 0, 7219, 0, 0, 0, 61, 5842, 7065, 72, 0, 0, 0, 0, 0, 0, 0, 53, 0, 933, 0, 443, 0, 0, 0, 0, 0, 0, 0, 100, 83, 0, 347, 294] payload_rows_per_dst=[14452, 16116, 14422, 16384, 14274, 2336, 2218, 1648] dup_rows_per_dst=[0, 1430, 2170, 12198, 122, 0, 534, 0] combine_rows_per_src=[14452, 16116, 14422, 16384, 14274, 2336, 2218, 1648] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90258 payload_rows_remote=79742 dup_rows=8046 dup_rows_remote=7882 dedup_ratio=0.081848 router_tokens_per_expert=[3280, 465, 0, 0, 0, 0, 1617, 28, 0, 778, 0, 3212, 0, 0, 247, 0, 0, 129, 33, 0, 0, 454, 0, 0, 0, 0, 7018, 244, 304, 0, 85, 4707, 0, 0, 0, 0, 0, 0, 0, 1899, 0, 0, 1883, 129, 7201, 65, 7372, 0, 7033, 0, 0, 152, 0, 64, 0, 0, 79, 0, 0, 0, 87, 0, 0, 587] payload_rows_per_dst=[8688, 7800, 15446, 10516, 15452, 3996, 14066, 14294] dup_rows_per_dst=[2092, 674, 530, 164, 2748, 158, 432, 1248] combine_rows_per_src=[8688, 7800, 15446, 10516, 15452, 3996, 14066, 14294] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=364077056 dispatch_meta_bytes=369360 combine_payload_bytes=364077056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=364446416 ingress_bytes=398680064 link_bytes=398680064 token_bytes=728523472 weight_movement_bytes=34603008 total_nvlink_bytes=763126480 weight_to_token_ratio=0.047497 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=30464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.500200984e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70446 payload_rows_remote=54368 dup_rows=27858 dup_rows_remote=25962 dedup_ratio=0.283386 router_tokens_per_expert=[0, 0, 439, 0, 0, 0, 830, 0, 0, 0, 136, 82, 7351, 0, 0, 6782, 2, 0, 0, 0, 0, 3759, 0, 0, 0, 0, 0, 8114, 0, 0, 7718, 0, 0, 0, 77, 0, 0, 7811, 0, 0, 0, 0, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 0, 77, 0, 3116, 0, 0, 468, 1452, 0, 0] payload_rows_per_dst=[16078, 15966, 7522, 16230, 2894, 932, 948, 9876] dup_rows_per_dst=[1896, 12736, 0, 12880, 0, 0, 150, 196] combine_rows_per_src=[16078, 15966, 7522, 16230, 2894, 932, 948, 9876] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=275529728 dispatch_meta_bytes=295200 combine_payload_bytes=275529728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275824928 ingress_bytes=310132736 link_bytes=310132736 token_bytes=551354656 weight_movement_bytes=34603008 total_nvlink_bytes=585957664 weight_to_token_ratio=0.062760 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=24320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.167004518e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89478 payload_rows_remote=75492 dup_rows=8826 dup_rows_remote=5542 dedup_ratio=0.089783 router_tokens_per_expert=[3362, 466, 0, 0, 0, 0, 1525, 27, 0, 973, 0, 3098, 0, 0, 248, 0, 0, 155, 66, 0, 0, 452, 0, 0, 0, 0, 7014, 254, 425, 0, 100, 4574, 0, 0, 0, 0, 0, 0, 0, 1944, 0, 0, 1940, 116, 7000, 78, 7321, 0, 6930, 0, 0, 153, 0, 77, 0, 0, 66, 0, 0, 0, 95, 0, 0, 693] payload_rows_per_dst=[8810, 7910, 14930, 10462, 5178, 13986, 13864, 14338] dup_rows_per_dst=[1950, 728, 494, 244, 272, 3284, 456, 1398] combine_rows_per_src=[8810, 7910, 14930, 10462, 5178, 13986, 13864, 14338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90040 payload_rows_remote=74602 dup_rows=8264 dup_rows_remote=7812 dedup_ratio=0.084066 router_tokens_per_expert=[3362, 663, 0, 0, 0, 0, 1522, 21, 0, 915, 0, 2642, 0, 0, 249, 0, 0, 147, 46, 0, 0, 408, 0, 0, 0, 0, 7025, 244, 378, 0, 106, 4689, 0, 0, 0, 0, 0, 0, 0, 2014, 0, 0, 2155, 137, 7056, 74, 7344, 0, 6913, 0, 0, 184, 0, 73, 0, 0, 86, 0, 0, 0, 84, 0, 0, 615] payload_rows_per_dst=[9008, 6978, 15438, 10654, 15228, 4556, 13832, 14346] dup_rows_per_dst=[2128, 634, 452, 180, 2912, 176, 508, 1274] combine_rows_per_src=[9008, 6978, 15438, 10654, 15228, 4556, 13832, 14346] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=247193600 dispatch_meta_bytes=331848 combine_payload_bytes=247193600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=247525448 ingress_bytes=281796608 link_bytes=281796608 token_bytes=494719048 weight_movement_bytes=34603008 total_nvlink_bytes=529322056 weight_to_token_ratio=0.069945 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=42624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.060377949e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84880 payload_rows_remote=69080 dup_rows=13424 dup_rows_remote=2200 dedup_ratio=0.136556 router_tokens_per_expert=[0, 0, 374, 0, 0, 0, 850, 0, 0, 0, 131, 95, 7388, 0, 0, 6760, 4, 0, 0, 0, 0, 4141, 0, 0, 0, 0, 0, 8120, 0, 0, 7676, 0, 0, 0, 72, 0, 0, 7814, 0, 0, 0, 0, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 460, 0, 71, 0, 2813, 0, 0, 484, 1420, 0, 0] payload_rows_per_dst=[16012, 15800, 8290, 16240, 15772, 958, 2586, 9222] dup_rows_per_dst=[1788, 11224, 0, 0, 0, 0, 200, 212] combine_rows_per_src=[16012, 15800, 8290, 16240, 15772, 958, 2586, 9222] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=297951232 dispatch_meta_bytes=330336 combine_payload_bytes=297951232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=298281568 ingress_bytes=332554240 link_bytes=332554240 token_bytes=596232800 weight_movement_bytes=34603008 total_nvlink_bytes=630835808 weight_to_token_ratio=0.058036 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.037500 gemm_active_groups=4 gemm_peak_group_rows=44928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.251374832e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91592 payload_rows_remote=75590 dup_rows=6712 dup_rows_remote=6710 dedup_ratio=0.068278 router_tokens_per_expert=[4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 259, 0, 0, 50, 767, 0, 1832, 0, 155, 7851, 7942, 0, 0, 0, 0, 0, 857, 0, 7697, 0, 50, 0, 7255, 0, 3, 0, 0, 4567, 7080, 78, 0, 0, 0, 0, 0, 1, 0, 59, 0, 1087, 0, 435, 0, 0, 0, 0, 0, 0, 0, 233, 189, 0, 334, 367] payload_rows_per_dst=[14518, 5490, 13012, 1814, 9134, 16002, 16128, 15494] dup_rows_per_dst=[0, 746, 2580, 0, 162, 2, 2310, 912] combine_rows_per_src=[14518, 5490, 13012, 1814, 9134, 16002, 16128, 15494] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=326623232 dispatch_meta_bytes=350496 combine_payload_bytes=326623232 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=326973728 ingress_bytes=395829248 link_bytes=395829248 token_bytes=653596960 weight_movement_bytes=69206016 total_nvlink_bytes=722802976 weight_to_token_ratio=0.105885 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=29696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.489473593e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=269246464 dispatch_meta_bytes=323032 combine_payload_bytes=269246464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269569496 ingress_bytes=303849472 link_bytes=303849472 token_bytes=538815960 weight_movement_bytes=34603008 total_nvlink_bytes=573418968 weight_to_token_ratio=0.064220 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=41472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.143361101e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=222691328 dispatch_meta_bytes=321320 combine_payload_bytes=222691328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=223012648 ingress_bytes=257294336 link_bytes=257294336 token_bytes=445703976 weight_movement_bytes=34603008 total_nvlink_bytes=480306984 weight_to_token_ratio=0.077637 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=38784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.681778725e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=305569792 dispatch_meta_bytes=329656 combine_payload_bytes=305569792 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=305899448 ingress_bytes=340172800 link_bytes=340172800 token_bytes=611469240 weight_movement_bytes=34603008 total_nvlink_bytes=646072248 weight_to_token_ratio=0.056590 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=43776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.280042860e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=309215232 dispatch_meta_bytes=324136 combine_payload_bytes=309215232 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=309539368 ingress_bytes=482230272 link_bytes=482230272 token_bytes=618754600 weight_movement_bytes=173015040 total_nvlink_bytes=791769640 weight_to_token_ratio=0.279618 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=16640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.814593691e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94010 payload_rows_remote=78500 dup_rows=4294 dup_rows_remote=4294 dedup_ratio=0.043681 router_tokens_per_expert=[0, 0, 415, 0, 0, 0, 827, 0, 0, 0, 157, 94, 7357, 0, 0, 6726, 47, 0, 0, 0, 0, 4477, 0, 0, 0, 0, 0, 8101, 0, 0, 7638, 0, 0, 0, 91, 0, 0, 7664, 0, 0, 0, 0, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 371, 0, 91, 0, 3101, 0, 0, 464, 963, 0, 0] payload_rows_per_dst=[15958, 12662, 9576, 0, 15510, 16384, 14984, 8936] dup_rows_per_dst=[1802, 0, 764, 0, 0, 954, 654, 120] combine_rows_per_src=[15958, 12662, 9576, 0, 15510, 16384, 14984, 8936] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=309616640 dispatch_meta_bytes=329200 combine_payload_bytes=309616640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=309945840 ingress_bytes=344219648 link_bytes=344219648 token_bytes=619562480 weight_movement_bytes=34603008 total_nvlink_bytes=654165488 weight_to_token_ratio=0.055851 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=48768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.295270823e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84930 payload_rows_remote=81982 dup_rows=13374 dup_rows_remote=13374 dedup_ratio=0.136047 router_tokens_per_expert=[4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 362, 0, 0, 51, 716, 0, 1327, 0, 147, 7985, 7947, 0, 0, 0, 0, 0, 974, 0, 7769, 0, 51, 0, 7566, 0, 2, 0, 0, 4635, 7032, 74, 0, 0, 0, 0, 0, 0, 0, 87, 0, 1033, 0, 401, 0, 0, 0, 0, 0, 0, 0, 217, 156, 0, 324, 292] payload_rows_per_dst=[15140, 16236, 2948, 16384, 1348, 16068, 2698, 14108] dup_rows_per_dst=[0, 1992, 0, 10474, 4, 0, 170, 734] combine_rows_per_src=[15140, 16236, 2948, 16384, 1348, 16068, 2698, 14108] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81148 payload_rows_remote=66082 dup_rows=17156 dup_rows_remote=16566 dedup_ratio=0.174520 router_tokens_per_expert=[2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 0, 57, 732, 0, 1420, 0, 169, 7765, 7951, 0, 0, 0, 0, 0, 642, 0, 7719, 0, 57, 0, 6921, 0, 1, 0, 138, 6005, 7062, 85, 0, 0, 0, 0, 0, 0, 0, 49, 0, 983, 0, 517, 0, 0, 0, 0, 0, 0, 0, 52, 84, 0, 346, 284] payload_rows_per_dst=[4, 1766, 15694, 1398, 15120, 16000, 16100, 15066] dup_rows_per_dst=[0, 34, 2734, 0, 11460, 0, 2338, 590] combine_rows_per_src=[4, 1766, 15694, 1398, 15120, 16000, 16100, 15066] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70446 payload_rows_remote=54368 dup_rows=27858 dup_rows_remote=25962 dedup_ratio=0.283386 router_tokens_per_expert=[0, 0, 439, 0, 0, 0, 830, 0, 0, 0, 136, 82, 7351, 0, 0, 6782, 2, 0, 0, 0, 0, 3759, 0, 0, 0, 0, 0, 8114, 0, 0, 7718, 0, 0, 0, 77, 0, 0, 7811, 0, 0, 0, 0, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 0, 77, 0, 3116, 0, 0, 468, 1452, 0, 0] payload_rows_per_dst=[16078, 15966, 7522, 16230, 2894, 932, 948, 9876] dup_rows_per_dst=[1896, 12736, 0, 12880, 0, 0, 150, 196] combine_rows_per_src=[16078, 15966, 7522, 16230, 2894, 932, 948, 9876] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=282951680 dispatch_meta_bytes=285120 combine_payload_bytes=282951680 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=283236800 ingress_bytes=386760704 link_bytes=386760704 token_bytes=566188480 weight_movement_bytes=103809024 total_nvlink_bytes=669997504 weight_to_token_ratio=0.183347 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=33664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.455349392e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89490 payload_rows_remote=73398 dup_rows=8814 dup_rows_remote=6310 dedup_ratio=0.089661 router_tokens_per_expert=[73, 0, 1, 0, 3, 0, 6, 0, 0, 0, 190, 0, 0, 37, 667, 0, 1666, 0, 214, 7607, 7787, 0, 3, 1, 0, 1, 683, 0, 7681, 0, 38, 0, 6876, 1, 0, 0, 235, 5560, 7060, 107, 0, 0, 0, 3, 2, 1, 0, 66, 0, 1051, 1, 563, 0, 2, 0, 0, 1, 0, 2, 104, 153, 3, 340, 363] payload_rows_per_dst=[11378, 1768, 15896, 1442, 11824, 15716, 16092, 15374] dup_rows_per_dst=[2, 20, 3086, 2, 2520, 2, 2504, 678] combine_rows_per_src=[11378, 1768, 15896, 1442, 11824, 15716, 16092, 15374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83652 payload_rows_remote=67268 dup_rows=14652 dup_rows_remote=6532 dedup_ratio=0.149048 router_tokens_per_expert=[0, 0, 426, 0, 0, 0, 817, 0, 0, 0, 141, 88, 7318, 0, 0, 6785, 6, 0, 0, 0, 0, 4412, 0, 0, 0, 0, 0, 8108, 0, 0, 7687, 0, 0, 0, 84, 0, 0, 7731, 0, 0, 0, 0, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 397, 0, 85, 0, 3029, 0, 0, 501, 991, 0, 0] payload_rows_per_dst=[2386, 458, 13104, 15374, 4692, 16384, 14870, 16384] dup_rows_per_dst=[100, 0, 4778, 0, 0, 924, 730, 8120] combine_rows_per_src=[2386, 458, 13104, 15374, 4692, 16384, 14870, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=321536000 dispatch_meta_bytes=331176 combine_payload_bytes=321536000 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=321867176 ingress_bytes=425345024 link_bytes=425345024 token_bytes=643403176 weight_movement_bytes=103809024 total_nvlink_bytes=747212200 weight_to_token_ratio=0.161344 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=46336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.600539082e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=222691328 dispatch_meta_bytes=321320 combine_payload_bytes=222691328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=223012648 ingress_bytes=257294336 link_bytes=257294336 token_bytes=445703976 weight_movement_bytes=34603008 total_nvlink_bytes=480306984 weight_to_token_ratio=0.077637 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=38784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.681778725e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89490 payload_rows_remote=73398 dup_rows=8814 dup_rows_remote=6310 dedup_ratio=0.089661 router_tokens_per_expert=[73, 0, 1, 0, 3, 0, 6, 0, 0, 0, 190, 0, 0, 37, 667, 0, 1666, 0, 214, 7607, 7787, 0, 3, 1, 0, 1, 683, 0, 7681, 0, 38, 0, 6876, 1, 0, 0, 235, 5560, 7060, 107, 0, 0, 0, 3, 2, 1, 0, 66, 0, 1051, 1, 563, 0, 2, 0, 0, 1, 0, 2, 104, 153, 3, 340, 363] payload_rows_per_dst=[11378, 1768, 15896, 1442, 11824, 15716, 16092, 15374] dup_rows_per_dst=[2, 20, 3086, 2, 2520, 2, 2504, 678] combine_rows_per_src=[11378, 1768, 15896, 1442, 11824, 15716, 16092, 15374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=275529728 dispatch_meta_bytes=295200 combine_payload_bytes=275529728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275824928 ingress_bytes=310132736 link_bytes=310132736 token_bytes=551354656 weight_movement_bytes=34603008 total_nvlink_bytes=585957664 weight_to_token_ratio=0.062760 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=24320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.167004518e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94850 payload_rows_remote=88886 dup_rows=3454 dup_rows_remote=3454 dedup_ratio=0.035136 router_tokens_per_expert=[0, 0, 492, 0, 0, 0, 844, 0, 0, 0, 124, 86, 7379, 0, 0, 6945, 11, 0, 0, 0, 0, 2971, 0, 0, 0, 0, 0, 8118, 0, 0, 7725, 0, 0, 0, 73, 0, 0, 7827, 0, 0, 0, 0, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 0, 64, 0, 3130, 0, 0, 467, 1917, 0, 0] payload_rows_per_dst=[16080, 14310, 5964, 6950, 15800, 9740, 15154, 10852] dup_rows_per_dst=[2042, 0, 0, 0, 0, 542, 694, 176] combine_rows_per_src=[16080, 14310, 5964, 6950, 15800, 9740, 15154, 10852] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94558 payload_rows_remote=94558 dup_rows=3746 dup_rows_remote=3746 dedup_ratio=0.038106 router_tokens_per_expert=[0, 0, 514, 0, 0, 0, 862, 0, 0, 0, 114, 79, 7362, 0, 0, 6993, 5, 0, 0, 0, 0, 2996, 0, 0, 0, 0, 0, 8128, 0, 0, 7754, 0, 0, 0, 64, 0, 0, 7890, 0, 0, 0, 0, 420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 0, 60, 0, 2947, 0, 0, 448, 2029, 0, 0] payload_rows_per_dst=[16118, 14372, 6002, 0, 15908, 16384, 15134, 10640] dup_rows_per_dst=[2142, 0, 0, 0, 0, 712, 684, 208] combine_rows_per_src=[16118, 14372, 6002, 0, 15908, 16384, 15134, 10640] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90948 payload_rows_remote=75470 dup_rows=7356 dup_rows_remote=6644 dedup_ratio=0.074829 router_tokens_per_expert=[0, 0, 0, 135, 0, 0, 0, 0, 0, 0, 0, 72, 0, 72, 535, 0, 0, 0, 242, 2586, 7416, 707, 0, 0, 7879, 0, 0, 7452, 83, 59, 0, 0, 0, 0, 155, 0, 0, 0, 0, 6992, 0, 2169, 0, 0, 0, 0, 72, 0, 0, 0, 0, 226, 6748, 0, 208, 136, 4283, 72, 0, 0, 0, 613, 0, 240] payload_rows_per_dst=[15758, 15478, 6678, 15258, 9422, 4482, 13842, 10030] dup_rows_per_dst=[270, 712, 392, 4712, 90, 0, 794, 386] combine_rows_per_src=[15758, 15478, 6678, 15258, 9422, 4482, 13842, 10030] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=335798272 dispatch_meta_bytes=381424 combine_payload_bytes=335798272 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=336179696 ingress_bytes=439607296 link_bytes=439607296 token_bytes=671977968 weight_movement_bytes=103809024 total_nvlink_bytes=775786992 weight_to_token_ratio=0.154483 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=21120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.654206864e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80982 payload_rows_remote=66982 dup_rows=17322 dup_rows_remote=17322 dedup_ratio=0.176208 router_tokens_per_expert=[4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 0, 0, 59, 735, 0, 1653, 0, 154, 7822, 7962, 0, 0, 0, 0, 0, 751, 0, 7692, 0, 59, 0, 6996, 0, 1, 0, 122, 5494, 6989, 77, 0, 0, 0, 0, 0, 0, 0, 64, 0, 973, 0, 526, 0, 0, 0, 0, 0, 0, 0, 82, 96, 0, 354, 364] payload_rows_per_dst=[14000, 15986, 16252, 16384, 14132, 128, 2312, 1788] dup_rows_per_dst=[0, 1492, 3286, 11608, 246, 0, 686, 4] combine_rows_per_src=[14000, 15986, 16252, 16384, 14132, 128, 2312, 1788] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=270671872 dispatch_meta_bytes=330592 combine_payload_bytes=270671872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271002464 ingress_bytes=305274880 link_bytes=305274880 token_bytes=541674336 weight_movement_bytes=34603008 total_nvlink_bytes=576277344 weight_to_token_ratio=0.063882 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=41856 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.148724797e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91066 payload_rows_remote=74682 dup_rows=7238 dup_rows_remote=6434 dedup_ratio=0.073629 router_tokens_per_expert=[0, 0, 467, 0, 0, 0, 966, 0, 0, 0, 127, 65, 7313, 0, 0, 6800, 0, 0, 0, 0, 0, 3356, 0, 0, 0, 0, 0, 8115, 0, 0, 7742, 0, 0, 0, 77, 0, 0, 7814, 0, 0, 0, 0, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 0, 78, 0, 3129, 0, 0, 507, 1610, 0, 0] payload_rows_per_dst=[2978, 384, 15014, 15192, 15782, 16384, 15058, 10274] dup_rows_per_dst=[180, 0, 5298, 0, 0, 804, 738, 218] combine_rows_per_src=[2978, 384, 15014, 15192, 15782, 16384, 15058, 10274] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81850 payload_rows_remote=65734 dup_rows=16454 dup_rows_remote=15024 dedup_ratio=0.167379 router_tokens_per_expert=[7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 59, 799, 0, 1280, 0, 144, 7818, 7987, 0, 0, 0, 0, 0, 632, 0, 7758, 0, 59, 0, 7219, 0, 0, 0, 61, 5842, 7065, 72, 0, 0, 0, 0, 0, 0, 0, 53, 0, 933, 0, 443, 0, 0, 0, 0, 0, 0, 0, 100, 83, 0, 347, 294] payload_rows_per_dst=[14452, 16116, 14422, 16384, 14274, 2336, 2218, 1648] dup_rows_per_dst=[0, 1430, 2170, 12198, 122, 0, 534, 0] combine_rows_per_src=[14452, 16116, 14422, 16384, 14274, 2336, 2218, 1648] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=300638208 dispatch_meta_bytes=318832 combine_payload_bytes=300638208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=300957040 ingress_bytes=335241216 link_bytes=335241216 token_bytes=601595248 weight_movement_bytes=34603008 total_nvlink_bytes=636198256 weight_to_token_ratio=0.057519 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=37248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261485707e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=364077056 dispatch_meta_bytes=369360 combine_payload_bytes=364077056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=364446416 ingress_bytes=398680064 link_bytes=398680064 token_bytes=728523472 weight_movement_bytes=34603008 total_nvlink_bytes=763126480 weight_to_token_ratio=0.047497 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=30464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.500200984e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=387309568 dispatch_meta_bytes=393216 combine_payload_bytes=387309568 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=387702784 ingress_bytes=491118592 link_bytes=491118592 token_bytes=775012352 weight_movement_bytes=103809024 total_nvlink_bytes=878821376 weight_to_token_ratio=0.133945 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=23040 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.848039723e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=309125120 dispatch_meta_bytes=328456 combine_payload_bytes=309125120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=309453576 ingress_bytes=343728128 link_bytes=343728128 token_bytes=618578696 weight_movement_bytes=34603008 total_nvlink_bytes=653181704 weight_to_token_ratio=0.055940 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=43776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.293421273e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90776 payload_rows_remote=75058 dup_rows=7528 dup_rows_remote=3650 dedup_ratio=0.076579 router_tokens_per_expert=[0, 0, 0, 191, 0, 0, 0, 0, 0, 0, 0, 78, 0, 78, 639, 0, 0, 0, 265, 2330, 7310, 701, 0, 0, 7718, 0, 0, 7327, 112, 50, 0, 0, 0, 0, 199, 0, 0, 0, 0, 6927, 0, 2394, 0, 0, 0, 0, 77, 0, 0, 0, 0, 236, 6698, 0, 253, 152, 4502, 77, 0, 0, 0, 553, 4, 281] payload_rows_per_dst=[15436, 14248, 6850, 324, 14098, 15718, 13776, 10326] dup_rows_per_dst=[382, 870, 834, 0, 154, 3878, 902, 508] combine_rows_per_src=[15436, 14248, 6850, 324, 14098, 15718, 13776, 10326] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=274358272 dispatch_meta_bytes=337216 combine_payload_bytes=274358272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274695488 ingress_bytes=308961280 link_bytes=308961280 token_bytes=549053760 weight_movement_bytes=34603008 total_nvlink_bytes=583656768 weight_to_token_ratio=0.063023 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=49024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.162596423e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93392 payload_rows_remote=81200 dup_rows=4912 dup_rows_remote=4282 dedup_ratio=0.049967 router_tokens_per_expert=[4, 0, 0, 0, 0, 0, 8, 0, 0, 0, 107, 0, 0, 66, 635, 0, 1495, 0, 182, 7712, 7906, 0, 0, 0, 0, 0, 593, 0, 7718, 0, 66, 0, 6783, 0, 0, 0, 224, 6096, 7092, 91, 0, 0, 0, 0, 0, 0, 0, 54, 0, 1018, 0, 545, 0, 0, 0, 0, 0, 0, 0, 36, 105, 0, 315, 301] payload_rows_per_dst=[13590, 15764, 3354, 1318, 12192, 15920, 16070, 15184] dup_rows_per_dst=[0, 1276, 0, 0, 630, 0, 2492, 514] combine_rows_per_src=[13590, 15764, 3354, 1318, 12192, 15920, 16070, 15184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95366 payload_rows_remote=89170 dup_rows=2938 dup_rows_remote=2524 dedup_ratio=0.029887 router_tokens_per_expert=[0, 0, 0, 161, 0, 0, 0, 0, 0, 0, 0, 74, 0, 74, 545, 0, 0, 0, 173, 2497, 7373, 635, 0, 0, 7857, 0, 0, 7375, 82, 51, 0, 0, 0, 0, 155, 0, 0, 0, 0, 7067, 0, 2332, 0, 0, 0, 0, 73, 0, 0, 0, 0, 209, 6623, 0, 257, 130, 4548, 73, 0, 0, 0, 508, 0, 280] payload_rows_per_dst=[15714, 15382, 6196, 15016, 14298, 4810, 13588, 10362] dup_rows_per_dst=[322, 750, 414, 0, 146, 0, 850, 456] combine_rows_per_src=[15714, 15382, 6196, 15016, 14298, 4810, 13588, 10362] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=307437568 dispatch_meta_bytes=314832 combine_payload_bytes=307437568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307752400 ingress_bytes=342040576 link_bytes=342040576 token_bytes=615189968 weight_movement_bytes=34603008 total_nvlink_bytes=649792976 weight_to_token_ratio=0.056248 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=30208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.287071151e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80220 payload_rows_remote=64254 dup_rows=18084 dup_rows_remote=10896 dedup_ratio=0.183960 router_tokens_per_expert=[0, 0, 0, 142, 0, 0, 0, 0, 0, 0, 0, 85, 0, 84, 503, 0, 0, 0, 360, 2863, 7376, 776, 0, 0, 7887, 0, 0, 7397, 109, 57, 0, 0, 0, 0, 184, 0, 0, 0, 0, 6904, 0, 2105, 0, 0, 0, 0, 84, 0, 0, 0, 0, 306, 6651, 0, 205, 148, 3836, 84, 0, 0, 0, 774, 1, 231] payload_rows_per_dst=[284, 1176, 15846, 15888, 14008, 15734, 1318, 15966] dup_rows_per_dst=[0, 168, 6904, 218, 168, 3438, 0, 7188] combine_rows_per_src=[284, 1176, 15846, 15888, 14008, 15734, 1318, 15966] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=305897472 dispatch_meta_bytes=324464 combine_payload_bytes=305897472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=306221936 ingress_bytes=340500480 link_bytes=340500480 token_bytes=612119408 weight_movement_bytes=34603008 total_nvlink_bytes=646722416 weight_to_token_ratio=0.056530 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=45184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.281275894e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80982 payload_rows_remote=66982 dup_rows=17322 dup_rows_remote=17322 dedup_ratio=0.176208 router_tokens_per_expert=[4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 0, 0, 59, 735, 0, 1653, 0, 154, 7822, 7962, 0, 0, 0, 0, 0, 751, 0, 7692, 0, 59, 0, 6996, 0, 1, 0, 122, 5494, 6989, 77, 0, 0, 0, 0, 0, 0, 0, 64, 0, 973, 0, 526, 0, 0, 0, 0, 0, 0, 0, 82, 96, 0, 354, 364] payload_rows_per_dst=[14000, 15986, 16252, 16384, 14132, 128, 2312, 1788] dup_rows_per_dst=[0, 1492, 3286, 11608, 246, 0, 686, 4] combine_rows_per_src=[14000, 15986, 16252, 16384, 14132, 128, 2312, 1788] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=269246464 dispatch_meta_bytes=323032 combine_payload_bytes=269246464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269569496 ingress_bytes=303849472 link_bytes=303849472 token_bytes=538815960 weight_movement_bytes=34603008 total_nvlink_bytes=573418968 weight_to_token_ratio=0.064220 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=41472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.143361101e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81148 payload_rows_remote=66082 dup_rows=17156 dup_rows_remote=16566 dedup_ratio=0.174520 router_tokens_per_expert=[2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 0, 57, 732, 0, 1420, 0, 169, 7765, 7951, 0, 0, 0, 0, 0, 642, 0, 7719, 0, 57, 0, 6921, 0, 1, 0, 138, 6005, 7062, 85, 0, 0, 0, 0, 0, 0, 0, 49, 0, 983, 0, 517, 0, 0, 0, 0, 0, 0, 0, 52, 84, 0, 346, 284] payload_rows_per_dst=[4, 1766, 15694, 1398, 15120, 16000, 16100, 15066] dup_rows_per_dst=[0, 34, 2734, 0, 11460, 0, 2338, 590] combine_rows_per_src=[4, 1766, 15694, 1398, 15120, 16000, 16100, 15066] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=332595200 dispatch_meta_bytes=341928 combine_payload_bytes=332595200 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=332937128 ingress_bytes=471007232 link_bytes=471007232 token_bytes=665532328 weight_movement_bytes=138412032 total_nvlink_bytes=803944360 weight_to_token_ratio=0.207972 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=24832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.772362295e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=274358272 dispatch_meta_bytes=337216 combine_payload_bytes=274358272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274695488 ingress_bytes=308961280 link_bytes=308961280 token_bytes=549053760 weight_movement_bytes=34603008 total_nvlink_bytes=583656768 weight_to_token_ratio=0.063023 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=49024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.162596423e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84846 payload_rows_remote=73218 dup_rows=13458 dup_rows_remote=12648 dedup_ratio=0.136902 router_tokens_per_expert=[0, 0, 1, 192, 1, 3, 0, 0, 0, 0, 0, 107, 1, 107, 356, 0, 0, 0, 403, 2590, 7284, 826, 1, 0, 7928, 2, 0, 7282, 187, 37, 0, 0, 0, 3, 260, 3, 1, 0, 0, 6726, 2, 2284, 0, 3, 0, 0, 108, 0, 0, 0, 0, 275, 6639, 1, 228, 156, 4203, 107, 0, 0, 2, 567, 3, 273] payload_rows_per_dst=[1984, 928, 15756, 14342, 13764, 15644, 11628, 10800] dup_rows_per_dst=[48, 214, 6452, 328, 222, 3714, 810, 1670] combine_rows_per_src=[1984, 928, 15756, 14342, 13764, 15644, 11628, 10800] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=270671872 dispatch_meta_bytes=330592 combine_payload_bytes=270671872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271002464 ingress_bytes=305274880 link_bytes=305274880 token_bytes=541674336 weight_movement_bytes=34603008 total_nvlink_bytes=576277344 weight_to_token_ratio=0.063882 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=41856 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.148724797e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84930 payload_rows_remote=81982 dup_rows=13374 dup_rows_remote=13374 dedup_ratio=0.136047 router_tokens_per_expert=[4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 362, 0, 0, 51, 716, 0, 1327, 0, 147, 7985, 7947, 0, 0, 0, 0, 0, 974, 0, 7769, 0, 51, 0, 7566, 0, 2, 0, 0, 4635, 7032, 74, 0, 0, 0, 0, 0, 0, 0, 87, 0, 1033, 0, 401, 0, 0, 0, 0, 0, 0, 0, 217, 156, 0, 324, 292] payload_rows_per_dst=[15140, 16236, 2948, 16384, 1348, 16068, 2698, 14108] dup_rows_per_dst=[0, 1992, 0, 10474, 4, 0, 170, 734] combine_rows_per_src=[15140, 16236, 2948, 16384, 1348, 16068, 2698, 14108] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92108 payload_rows_remote=84856 dup_rows=6196 dup_rows_remote=4550 dedup_ratio=0.063029 router_tokens_per_expert=[12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 390, 0, 0, 50, 629, 0, 1306, 0, 129, 8025, 8003, 0, 0, 0, 0, 0, 1028, 0, 7742, 0, 50, 0, 7598, 0, 3, 0, 0, 4527, 7047, 65, 0, 0, 0, 0, 0, 0, 0, 62, 0, 1120, 0, 308, 0, 0, 0, 0, 0, 0, 0, 242, 141, 0, 346, 329] payload_rows_per_dst=[15220, 16252, 2870, 7252, 5064, 16130, 14034, 15286] dup_rows_per_dst=[0, 1936, 0, 1646, 72, 0, 1618, 924] combine_rows_per_src=[15220, 16252, 2870, 7252, 5064, 16130, 14034, 15286] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81498 payload_rows_remote=79108 dup_rows=16806 dup_rows_remote=16806 dedup_ratio=0.170959 router_tokens_per_expert=[0, 657, 430, 987, 8008, 2081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 6494, 0, 0, 0, 0, 222, 6999, 72, 0, 0, 0, 0, 0, 2154, 0, 0, 4749, 0, 154, 0, 0, 175, 0, 0, 0, 48, 0, 0, 1090, 0, 0, 0, 0, 95, 7870, 112, 6647, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[14898, 2390, 13178, 14518, 4616, 16090, 2370, 13438] dup_rows_per_dst=[7038, 0, 26, 9566, 0, 96, 0, 80] combine_rows_per_src=[14898, 2390, 13178, 14518, 4616, 16090, 2370, 13438] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=263184384 dispatch_meta_bytes=300600 combine_payload_bytes=263184384 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263484984 ingress_bytes=297787392 link_bytes=297787392 token_bytes=526669368 weight_movement_bytes=34603008 total_nvlink_bytes=561272376 weight_to_token_ratio=0.065702 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=34048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.120549982e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88724 payload_rows_remote=73080 dup_rows=9580 dup_rows_remote=9206 dedup_ratio=0.097453 router_tokens_per_expert=[0, 0, 0, 187, 0, 0, 0, 0, 0, 0, 0, 77, 0, 77, 573, 0, 0, 0, 234, 2508, 7359, 834, 0, 0, 7822, 0, 0, 7358, 108, 59, 0, 0, 0, 0, 163, 0, 0, 0, 0, 6828, 0, 2334, 0, 0, 0, 0, 77, 0, 0, 0, 0, 298, 6633, 0, 185, 116, 4442, 77, 0, 0, 0, 578, 2, 223] payload_rows_per_dst=[15644, 15356, 13480, 15054, 3184, 4822, 10972, 10212] dup_rows_per_dst=[374, 816, 3906, 3418, 0, 0, 634, 432] combine_rows_per_src=[15644, 15356, 13480, 15054, 3184, 4822, 10972, 10212] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=365240320 dispatch_meta_bytes=366776 combine_payload_bytes=365240320 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=365607096 ingress_bytes=434446336 link_bytes=434446336 token_bytes=730847416 weight_movement_bytes=69206016 total_nvlink_bytes=800053432 weight_to_token_ratio=0.094693 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.634786587e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91592 payload_rows_remote=75590 dup_rows=6712 dup_rows_remote=6710 dedup_ratio=0.068278 router_tokens_per_expert=[4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 259, 0, 0, 50, 767, 0, 1832, 0, 155, 7851, 7942, 0, 0, 0, 0, 0, 857, 0, 7697, 0, 50, 0, 7255, 0, 3, 0, 0, 4567, 7080, 78, 0, 0, 0, 0, 0, 1, 0, 59, 0, 1087, 0, 435, 0, 0, 0, 0, 0, 0, 0, 233, 189, 0, 334, 367] payload_rows_per_dst=[14518, 5490, 13012, 1814, 9134, 16002, 16128, 15494] dup_rows_per_dst=[0, 746, 2580, 0, 162, 2, 2310, 912] combine_rows_per_src=[14518, 5490, 13012, 1814, 9134, 16002, 16128, 15494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90948 payload_rows_remote=75470 dup_rows=7356 dup_rows_remote=6644 dedup_ratio=0.074829 router_tokens_per_expert=[0, 0, 0, 135, 0, 0, 0, 0, 0, 0, 0, 72, 0, 72, 535, 0, 0, 0, 242, 2586, 7416, 707, 0, 0, 7879, 0, 0, 7452, 83, 59, 0, 0, 0, 0, 155, 0, 0, 0, 0, 6992, 0, 2169, 0, 0, 0, 0, 72, 0, 0, 0, 0, 226, 6748, 0, 208, 136, 4283, 72, 0, 0, 0, 613, 0, 240] payload_rows_per_dst=[15758, 15478, 6678, 15258, 9422, 4482, 13842, 10030] dup_rows_per_dst=[270, 712, 392, 4712, 90, 0, 794, 386] combine_rows_per_src=[15758, 15478, 6678, 15258, 9422, 4482, 13842, 10030] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=299900928 dispatch_meta_bytes=343464 combine_payload_bytes=299900928 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=300244392 ingress_bytes=369106944 link_bytes=369106944 token_bytes=600145320 weight_movement_bytes=69206016 total_nvlink_bytes=669351336 weight_to_token_ratio=0.115315 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=44288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388919715e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92980 payload_rows_remote=76974 dup_rows=5324 dup_rows_remote=5116 dedup_ratio=0.054159 router_tokens_per_expert=[0, 693, 394, 1072, 8006, 2336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 6683, 0, 0, 0, 0, 210, 7018, 77, 0, 0, 0, 0, 0, 2698, 0, 0, 3564, 2, 116, 0, 0, 178, 0, 0, 0, 104, 0, 0, 1251, 5, 0, 0, 0, 65, 7825, 106, 6668, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6844, 16012, 13474, 574, 11430, 16006, 15150, 13490] dup_rows_per_dst=[2146, 0, 54, 0, 1330, 208, 1528, 58] combine_rows_per_src=[6844, 16012, 13474, 574, 11430, 16006, 15150, 13490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=335798272 dispatch_meta_bytes=381424 combine_payload_bytes=335798272 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=336179696 ingress_bytes=439607296 link_bytes=439607296 token_bytes=671977968 weight_movement_bytes=103809024 total_nvlink_bytes=775786992 weight_to_token_ratio=0.154483 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=21120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.654206864e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=347570176 dispatch_meta_bytes=357624 combine_payload_bytes=347570176 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=347927800 ingress_bytes=416776192 link_bytes=416776192 token_bytes=695497976 weight_movement_bytes=69206016 total_nvlink_bytes=764703992 weight_to_token_ratio=0.099506 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=24576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.568295257e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=324026368 dispatch_meta_bytes=383656 combine_payload_bytes=324026368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=324410024 ingress_bytes=358629376 link_bytes=358629376 token_bytes=648436392 weight_movement_bytes=34603008 total_nvlink_bytes=683039400 weight_to_token_ratio=0.053364 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.349493470e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=299335680 dispatch_meta_bytes=329144 combine_payload_bytes=299335680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=299664824 ingress_bytes=333938688 link_bytes=333938688 token_bytes=599000504 weight_movement_bytes=34603008 total_nvlink_bytes=633603512 weight_to_token_ratio=0.057768 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=48000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256584399e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92260 payload_rows_remote=78118 dup_rows=6044 dup_rows_remote=5864 dedup_ratio=0.061483 router_tokens_per_expert=[0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 1, 91, 0, 90, 465, 0, 0, 0, 402, 3065, 7438, 765, 0, 1, 7925, 0, 0, 7369, 158, 66, 0, 0, 0, 0, 225, 0, 0, 0, 0, 6936, 0, 1978, 0, 0, 0, 0, 91, 0, 0, 0, 0, 316, 6614, 0, 189, 176, 3694, 91, 0, 0, 0, 711, 0, 219] payload_rows_per_dst=[15850, 15540, 7952, 400, 14142, 15684, 13692, 9000] dup_rows_per_dst=[152, 630, 514, 48, 180, 3192, 898, 430] combine_rows_per_src=[15850, 15540, 7952, 400, 14142, 15684, 13692, 9000] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=315285504 dispatch_meta_bytes=328360 combine_payload_bytes=315285504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=315613864 ingress_bytes=349888512 link_bytes=349888512 token_bytes=630899368 weight_movement_bytes=34603008 total_nvlink_bytes=665502376 weight_to_token_ratio=0.054847 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=47360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.316602302e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68446 payload_rows_remote=55080 dup_rows=29858 dup_rows_remote=28716 dedup_ratio=0.303731 router_tokens_per_expert=[0, 679, 412, 1115, 8001, 2185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 6264, 0, 0, 0, 0, 437, 6976, 84, 0, 0, 0, 0, 0, 1885, 0, 0, 5218, 0, 145, 0, 0, 189, 0, 0, 0, 6, 0, 0, 1159, 0, 0, 0, 0, 88, 7835, 133, 6267, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5732, 16002, 148, 1042, 15394, 378, 16384, 13366] dup_rows_per_dst=[1342, 0, 0, 0, 11630, 12, 15732, 1142] combine_rows_per_src=[5732, 16002, 148, 1042, 15394, 378, 16384, 13366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88724 payload_rows_remote=73080 dup_rows=9580 dup_rows_remote=9206 dedup_ratio=0.097453 router_tokens_per_expert=[0, 0, 0, 187, 0, 0, 0, 0, 0, 0, 0, 77, 0, 77, 573, 0, 0, 0, 234, 2508, 7359, 834, 0, 0, 7822, 0, 0, 7358, 108, 59, 0, 0, 0, 0, 163, 0, 0, 0, 0, 6828, 0, 2334, 0, 0, 0, 0, 77, 0, 0, 0, 0, 298, 6633, 0, 185, 116, 4442, 77, 0, 0, 0, 578, 2, 223] payload_rows_per_dst=[15644, 15356, 13480, 15054, 3184, 4822, 10972, 10212] dup_rows_per_dst=[374, 816, 3906, 3418, 0, 0, 634, 432] combine_rows_per_src=[15644, 15356, 13480, 15054, 3184, 4822, 10972, 10212] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=309616640 dispatch_meta_bytes=329200 combine_payload_bytes=309616640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=309945840 ingress_bytes=344219648 link_bytes=344219648 token_bytes=619562480 weight_movement_bytes=34603008 total_nvlink_bytes=654165488 weight_to_token_ratio=0.055851 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=48768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.295270823e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=309125120 dispatch_meta_bytes=328456 combine_payload_bytes=309125120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=309453576 ingress_bytes=343728128 link_bytes=343728128 token_bytes=618578696 weight_movement_bytes=34603008 total_nvlink_bytes=653181704 weight_to_token_ratio=0.055940 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=43776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.293421273e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80220 payload_rows_remote=64254 dup_rows=18084 dup_rows_remote=10896 dedup_ratio=0.183960 router_tokens_per_expert=[0, 0, 0, 142, 0, 0, 0, 0, 0, 0, 0, 85, 0, 84, 503, 0, 0, 0, 360, 2863, 7376, 776, 0, 0, 7887, 0, 0, 7397, 109, 57, 0, 0, 0, 0, 184, 0, 0, 0, 0, 6904, 0, 2105, 0, 0, 0, 0, 84, 0, 0, 0, 0, 306, 6651, 0, 205, 148, 3836, 84, 0, 0, 0, 774, 1, 231] payload_rows_per_dst=[284, 1176, 15846, 15888, 14008, 15734, 1318, 15966] dup_rows_per_dst=[0, 168, 6904, 218, 168, 3438, 0, 7188] combine_rows_per_src=[284, 1176, 15846, 15888, 14008, 15734, 1318, 15966] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88078 payload_rows_remote=73918 dup_rows=10226 dup_rows_remote=10116 dedup_ratio=0.104024 router_tokens_per_expert=[0, 657, 403, 1091, 7980, 2057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 0, 6996, 0, 0, 0, 0, 285, 6987, 73, 0, 0, 0, 0, 0, 2648, 0, 0, 3915, 4, 137, 0, 0, 168, 0, 0, 0, 82, 0, 0, 1020, 0, 0, 0, 0, 86, 7855, 106, 6463, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6398, 15960, 14160, 14652, 5578, 16046, 2212, 13072] dup_rows_per_dst=[2018, 0, 110, 7868, 0, 164, 0, 66] combine_rows_per_src=[6398, 15960, 14160, 14652, 5578, 16046, 2212, 13072] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=319971328 dispatch_meta_bytes=335928 combine_payload_bytes=319971328 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=320307256 ingress_bytes=423780352 link_bytes=423780352 token_bytes=640278584 weight_movement_bytes=103809024 total_nvlink_bytes=744087608 weight_to_token_ratio=0.162131 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=46336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.594651347e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=299335680 dispatch_meta_bytes=329144 combine_payload_bytes=299335680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=299664824 ingress_bytes=333938688 link_bytes=333938688 token_bytes=599000504 weight_movement_bytes=34603008 total_nvlink_bytes=633603512 weight_to_token_ratio=0.057768 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=48000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256584399e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=263184384 dispatch_meta_bytes=300600 combine_payload_bytes=263184384 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263484984 ingress_bytes=297787392 link_bytes=297787392 token_bytes=526669368 weight_movement_bytes=34603008 total_nvlink_bytes=561272376 weight_to_token_ratio=0.065702 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=34048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.120549982e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75000 payload_rows_remote=58638 dup_rows=23304 dup_rows_remote=7892 dedup_ratio=0.237061 router_tokens_per_expert=[0, 710, 401, 1195, 7977, 2290, 0, 0, 0, 0, 1, 0, 1, 0, 1, 2, 3, 1, 48, 0, 2, 6504, 0, 0, 0, 0, 422, 6952, 104, 0, 2, 0, 0, 0, 1898, 0, 0, 4693, 1, 86, 0, 0, 201, 0, 1, 1, 62, 0, 0, 1297, 0, 0, 1, 3, 49, 7768, 174, 6295, 0, 3, 3, 0, 0, 0] payload_rows_per_dst=[6882, 15958, 8154, 1056, 13008, 770, 16362, 12810] dup_rows_per_dst=[2310, 6, 6, 0, 5304, 126, 15412, 140] combine_rows_per_src=[6882, 15958, 8154, 1056, 13008, 770, 16362, 12810] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=225607680 dispatch_meta_bytes=335184 combine_payload_bytes=225607680 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=225942864 ingress_bytes=294813696 link_bytes=294813696 token_bytes=451550544 weight_movement_bytes=69206016 total_nvlink_bytes=520756560 weight_to_token_ratio=0.153263 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=47360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.109360204e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95024 payload_rows_remote=81728 dup_rows=3280 dup_rows_remote=3278 dedup_ratio=0.033366 router_tokens_per_expert=[0, 0, 0, 167, 0, 0, 0, 0, 0, 0, 0, 65, 0, 65, 521, 0, 0, 0, 155, 2243, 7367, 667, 0, 0, 7876, 0, 0, 7346, 53, 50, 0, 0, 0, 0, 126, 0, 0, 0, 0, 7088, 0, 2730, 0, 0, 0, 0, 64, 0, 0, 0, 0, 242, 6734, 0, 233, 91, 4508, 64, 0, 0, 0, 443, 0, 254] payload_rows_per_dst=[15752, 15350, 5690, 13296, 14300, 6774, 13742, 10120] dup_rows_per_dst=[334, 686, 440, 2, 128, 414, 858, 418] combine_rows_per_src=[15752, 15350, 5690, 13296, 14300, 6774, 13742, 10120] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86508 payload_rows_remote=70502 dup_rows=11796 dup_rows_remote=7920 dedup_ratio=0.119995 router_tokens_per_expert=[2503, 0, 0, 0, 0, 0, 0, 1601, 481, 0, 535, 0, 0, 0, 0, 1075, 0, 0, 0, 5017, 277, 345, 0, 6586, 0, 0, 0, 0, 0, 0, 0, 0, 1171, 0, 0, 0, 0, 0, 0, 7686, 0, 0, 0, 7320, 0, 0, 53, 0, 0, 0, 0, 2878, 0, 7850, 0, 0, 0, 0, 0, 3160, 167, 375, 0, 72] payload_rows_per_dst=[15926, 16006, 11262, 13172, 2342, 14640, 5756, 7404] dup_rows_per_dst=[7654, 3876, 16, 0, 0, 106, 0, 144] combine_rows_per_src=[15926, 16006, 11262, 13172, 2342, 14640, 5756, 7404] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73386 payload_rows_remote=57002 dup_rows=24918 dup_rows_remote=15858 dedup_ratio=0.253479 router_tokens_per_expert=[0, 621, 407, 1195, 7990, 2509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 6559, 0, 0, 0, 0, 200, 6933, 77, 0, 0, 0, 0, 0, 2286, 0, 0, 4092, 0, 141, 0, 0, 168, 0, 0, 0, 60, 0, 0, 1202, 1, 0, 0, 0, 82, 7860, 127, 6553, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 0, 14566, 14370, 4854, 16056, 2568, 4588] dup_rows_per_dst=[9060, 0, 7476, 8234, 0, 120, 2, 26] combine_rows_per_src=[16384, 0, 14566, 14370, 4854, 16056, 2568, 4588] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=302768128 dispatch_meta_bytes=336136 combine_payload_bytes=302768128 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=303104264 ingress_bytes=371974144 link_bytes=371974144 token_bytes=605872392 weight_movement_bytes=69206016 total_nvlink_bytes=675078408 weight_to_token_ratio=0.114225 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=44032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.399708757e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95366 payload_rows_remote=89170 dup_rows=2938 dup_rows_remote=2524 dedup_ratio=0.029887 router_tokens_per_expert=[0, 0, 0, 161, 0, 0, 0, 0, 0, 0, 0, 74, 0, 74, 545, 0, 0, 0, 173, 2497, 7373, 635, 0, 0, 7857, 0, 0, 7375, 82, 51, 0, 0, 0, 0, 155, 0, 0, 0, 0, 7067, 0, 2332, 0, 0, 0, 0, 73, 0, 0, 0, 0, 209, 6623, 0, 257, 130, 4548, 73, 0, 0, 0, 508, 0, 280] payload_rows_per_dst=[15714, 15382, 6196, 15016, 14298, 4810, 13588, 10362] dup_rows_per_dst=[322, 750, 414, 0, 146, 0, 850, 456] combine_rows_per_src=[15714, 15382, 6196, 15016, 14298, 4810, 13588, 10362] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88100 payload_rows_remote=76802 dup_rows=10204 dup_rows_remote=10124 dedup_ratio=0.103800 router_tokens_per_expert=[3014, 0, 0, 0, 0, 0, 0, 1463, 438, 0, 507, 0, 0, 0, 0, 981, 0, 0, 0, 4971, 193, 356, 0, 6391, 0, 0, 0, 0, 0, 0, 0, 0, 974, 0, 0, 0, 0, 0, 0, 7817, 0, 0, 0, 7333, 0, 0, 62, 0, 0, 0, 0, 2741, 0, 7728, 0, 0, 0, 0, 0, 3517, 143, 445, 0, 78] payload_rows_per_dst=[8394, 2974, 11040, 12782, 16042, 11298, 15456, 10114] dup_rows_per_dst=[560, 878, 0, 0, 1540, 80, 5482, 1664] combine_rows_per_src=[8394, 2974, 11040, 12782, 16042, 11298, 15456, 10114] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=288776192 dispatch_meta_bytes=313688 combine_payload_bytes=288776192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289089880 ingress_bytes=323379200 link_bytes=323379200 token_bytes=577866072 weight_movement_bytes=34603008 total_nvlink_bytes=612469080 weight_to_token_ratio=0.059881 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=33024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.216849895e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95024 payload_rows_remote=81728 dup_rows=3280 dup_rows_remote=3278 dedup_ratio=0.033366 router_tokens_per_expert=[0, 0, 0, 167, 0, 0, 0, 0, 0, 0, 0, 65, 0, 65, 521, 0, 0, 0, 155, 2243, 7367, 667, 0, 0, 7876, 0, 0, 7346, 53, 50, 0, 0, 0, 0, 126, 0, 0, 0, 0, 7088, 0, 2730, 0, 0, 0, 0, 64, 0, 0, 0, 0, 242, 6734, 0, 233, 91, 4508, 64, 0, 0, 0, 443, 0, 254] payload_rows_per_dst=[15752, 15350, 5690, 13296, 14300, 6774, 13742, 10120] dup_rows_per_dst=[334, 686, 440, 2, 128, 414, 858, 418] combine_rows_per_src=[15752, 15350, 5690, 13296, 14300, 6774, 13742, 10120] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90776 payload_rows_remote=75058 dup_rows=7528 dup_rows_remote=3650 dedup_ratio=0.076579 router_tokens_per_expert=[0, 0, 0, 191, 0, 0, 0, 0, 0, 0, 0, 78, 0, 78, 639, 0, 0, 0, 265, 2330, 7310, 701, 0, 0, 7718, 0, 0, 7327, 112, 50, 0, 0, 0, 0, 199, 0, 0, 0, 0, 6927, 0, 2394, 0, 0, 0, 0, 77, 0, 0, 0, 0, 236, 6698, 0, 253, 152, 4502, 77, 0, 0, 0, 553, 4, 281] payload_rows_per_dst=[15436, 14248, 6850, 324, 14098, 15718, 13776, 10326] dup_rows_per_dst=[382, 870, 834, 0, 154, 3878, 902, 508] combine_rows_per_src=[15436, 14248, 6850, 324, 14098, 15718, 13776, 10326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81498 payload_rows_remote=79108 dup_rows=16806 dup_rows_remote=16806 dedup_ratio=0.170959 router_tokens_per_expert=[0, 657, 430, 987, 8008, 2081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 6494, 0, 0, 0, 0, 222, 6999, 72, 0, 0, 0, 0, 0, 2154, 0, 0, 4749, 0, 154, 0, 0, 175, 0, 0, 0, 48, 0, 0, 1090, 0, 0, 0, 0, 95, 7870, 112, 6647, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[14898, 2390, 13178, 14518, 4616, 16090, 2370, 13438] dup_rows_per_dst=[7038, 0, 26, 9566, 0, 96, 0, 80] combine_rows_per_src=[14898, 2390, 13178, 14518, 4616, 16090, 2370, 13438] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=240181248 dispatch_meta_bytes=266120 combine_payload_bytes=240181248 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=240447368 ingress_bytes=309387264 link_bytes=309387264 token_bytes=480628616 weight_movement_bytes=69206016 total_nvlink_bytes=549834632 weight_to_token_ratio=0.143991 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=23936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.164199367e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=365240320 dispatch_meta_bytes=366776 combine_payload_bytes=365240320 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=365607096 ingress_bytes=434446336 link_bytes=434446336 token_bytes=730847416 weight_movement_bytes=69206016 total_nvlink_bytes=800053432 weight_to_token_ratio=0.094693 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.634786587e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93040 payload_rows_remote=80392 dup_rows=5264 dup_rows_remote=3240 dedup_ratio=0.053548 router_tokens_per_expert=[0, 676, 441, 1084, 7981, 2160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 6349, 0, 0, 0, 0, 568, 6982, 91, 0, 0, 0, 0, 0, 1896, 0, 0, 5281, 0, 159, 0, 0, 228, 0, 0, 0, 2, 0, 0, 1104, 0, 0, 0, 0, 94, 7783, 142, 6033, 0, 0, 1, 0, 0, 0] payload_rows_per_dst=[6632, 15962, 12886, 8982, 12648, 16022, 7660, 12248] dup_rows_per_dst=[2090, 0, 6, 488, 2024, 4, 548, 104] combine_rows_per_src=[6632, 15962, 12886, 8982, 12648, 16022, 7660, 12248] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=334757888 dispatch_meta_bytes=340024 combine_payload_bytes=334757888 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=335097912 ingress_bytes=438566912 link_bytes=438566912 token_bytes=669855800 weight_movement_bytes=103809024 total_nvlink_bytes=773664824 weight_to_token_ratio=0.154972 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=28800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.650291982e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73386 payload_rows_remote=57002 dup_rows=24918 dup_rows_remote=15858 dedup_ratio=0.253479 router_tokens_per_expert=[0, 621, 407, 1195, 7990, 2509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 6559, 0, 0, 0, 0, 200, 6933, 77, 0, 0, 0, 0, 0, 2286, 0, 0, 4092, 0, 141, 0, 0, 168, 0, 0, 0, 60, 0, 0, 1202, 1, 0, 0, 0, 82, 7860, 127, 6553, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 0, 14566, 14370, 4854, 16056, 2568, 4588] dup_rows_per_dst=[9060, 0, 7476, 8234, 0, 120, 2, 26] combine_rows_per_src=[16384, 0, 14566, 14370, 4854, 16056, 2568, 4588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=324026368 dispatch_meta_bytes=383656 combine_payload_bytes=324026368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=324410024 ingress_bytes=358629376 link_bytes=358629376 token_bytes=648436392 weight_movement_bytes=34603008 total_nvlink_bytes=683039400 weight_to_token_ratio=0.053364 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.349493470e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=307437568 dispatch_meta_bytes=314832 combine_payload_bytes=307437568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307752400 ingress_bytes=342040576 link_bytes=342040576 token_bytes=615189968 weight_movement_bytes=34603008 total_nvlink_bytes=649792976 weight_to_token_ratio=0.056248 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=30208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.287071151e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68446 payload_rows_remote=55080 dup_rows=29858 dup_rows_remote=28716 dedup_ratio=0.303731 router_tokens_per_expert=[0, 679, 412, 1115, 8001, 2185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 6264, 0, 0, 0, 0, 437, 6976, 84, 0, 0, 0, 0, 0, 1885, 0, 0, 5218, 0, 145, 0, 0, 189, 0, 0, 0, 6, 0, 0, 1159, 0, 0, 0, 0, 88, 7835, 133, 6267, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5732, 16002, 148, 1042, 15394, 378, 16384, 13366] dup_rows_per_dst=[1342, 0, 0, 0, 11630, 12, 15732, 1142] combine_rows_per_src=[5732, 16002, 148, 1042, 15394, 378, 16384, 13366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=329285632 dispatch_meta_bytes=334528 combine_payload_bytes=329285632 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=329620160 ingress_bytes=398491648 link_bytes=398491648 token_bytes=658905792 weight_movement_bytes=69206016 total_nvlink_bytes=728111808 weight_to_token_ratio=0.105032 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=20864 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.499491990e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=233480192 dispatch_meta_bytes=291440 combine_payload_bytes=233480192 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=233771632 ingress_bytes=302686208 link_bytes=302686208 token_bytes=467251824 weight_movement_bytes=69206016 total_nvlink_bytes=536457840 weight_to_token_ratio=0.148113 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138983832e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93040 payload_rows_remote=80392 dup_rows=5264 dup_rows_remote=3240 dedup_ratio=0.053548 router_tokens_per_expert=[0, 676, 441, 1084, 7981, 2160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 6349, 0, 0, 0, 0, 568, 6982, 91, 0, 0, 0, 0, 0, 1896, 0, 0, 5281, 0, 159, 0, 0, 228, 0, 0, 0, 2, 0, 0, 1104, 0, 0, 0, 0, 94, 7783, 142, 6033, 0, 0, 1, 0, 0, 0] payload_rows_per_dst=[6632, 15962, 12886, 8982, 12648, 16022, 7660, 12248] dup_rows_per_dst=[2090, 0, 6, 488, 2024, 4, 548, 104] combine_rows_per_src=[6632, 15962, 12886, 8982, 12648, 16022, 7660, 12248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=314580992 dispatch_meta_bytes=347704 combine_payload_bytes=314580992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=314928696 ingress_bytes=383787008 link_bytes=383787008 token_bytes=629509688 weight_movement_bytes=69206016 total_nvlink_bytes=698715704 weight_to_token_ratio=0.109936 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=42368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.444159614e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84020 payload_rows_remote=68646 dup_rows=14284 dup_rows_remote=9942 dedup_ratio=0.145304 router_tokens_per_expert=[2540, 2, 0, 0, 0, 0, 0, 1463, 512, 3, 504, 0, 1, 0, 1, 1131, 0, 2, 0, 5421, 294, 403, 0, 6424, 0, 0, 0, 0, 0, 0, 0, 1, 1049, 2, 0, 0, 0, 0, 0, 7768, 0, 3, 0, 7308, 2, 3, 73, 0, 0, 0, 0, 2170, 1, 7686, 0, 1, 0, 0, 0, 3652, 156, 469, 0, 107] payload_rows_per_dst=[7770, 3274, 12228, 12848, 16016, 152, 15374, 16358] dup_rows_per_dst=[240, 1030, 12, 2, 1622, 10, 4342, 7026] combine_rows_per_src=[7770, 3274, 12228, 12848, 16016, 152, 15374, 16358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=225607680 dispatch_meta_bytes=335184 combine_payload_bytes=225607680 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=225942864 ingress_bytes=294813696 link_bytes=294813696 token_bytes=451550544 weight_movement_bytes=69206016 total_nvlink_bytes=520756560 weight_to_token_ratio=0.153263 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=47360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.109360204e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88078 payload_rows_remote=73918 dup_rows=10226 dup_rows_remote=10116 dedup_ratio=0.104024 router_tokens_per_expert=[0, 657, 403, 1091, 7980, 2057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 0, 6996, 0, 0, 0, 0, 285, 6987, 73, 0, 0, 0, 0, 0, 2648, 0, 0, 3915, 4, 137, 0, 0, 168, 0, 0, 0, 82, 0, 0, 1020, 0, 0, 0, 0, 86, 7855, 106, 6463, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6398, 15960, 14160, 14652, 5578, 16046, 2212, 13072] dup_rows_per_dst=[2018, 0, 110, 7868, 0, 164, 0, 66] combine_rows_per_src=[6398, 15960, 14160, 14652, 5578, 16046, 2212, 13072] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92648 payload_rows_remote=78392 dup_rows=5656 dup_rows_remote=2958 dedup_ratio=0.057536 router_tokens_per_expert=[0, 626, 386, 1086, 8007, 2167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 6893, 0, 0, 0, 0, 180, 7017, 64, 0, 0, 0, 0, 0, 3031, 0, 0, 3300, 0, 129, 0, 0, 131, 0, 0, 0, 52, 0, 0, 1101, 8, 0, 0, 0, 79, 7920, 67, 6771, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6574, 16014, 13924, 14256, 9748, 16102, 2360, 13670] dup_rows_per_dst=[1956, 0, 136, 2698, 740, 104, 16, 6] combine_rows_per_src=[6574, 16014, 13924, 14256, 9748, 16102, 2360, 13670] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86508 payload_rows_remote=70502 dup_rows=11796 dup_rows_remote=7920 dedup_ratio=0.119995 router_tokens_per_expert=[2503, 0, 0, 0, 0, 0, 0, 1601, 481, 0, 535, 0, 0, 0, 0, 1075, 0, 0, 0, 5017, 277, 345, 0, 6586, 0, 0, 0, 0, 0, 0, 0, 0, 1171, 0, 0, 0, 0, 0, 0, 7686, 0, 0, 0, 7320, 0, 0, 53, 0, 0, 0, 0, 2878, 0, 7850, 0, 0, 0, 0, 0, 3160, 167, 375, 0, 72] payload_rows_per_dst=[15926, 16006, 11262, 13172, 2342, 14640, 5756, 7404] dup_rows_per_dst=[7654, 3876, 16, 0, 0, 106, 0, 144] combine_rows_per_src=[15926, 16006, 11262, 13172, 2342, 14640, 5756, 7404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=281174016 dispatch_meta_bytes=314352 combine_payload_bytes=281174016 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=281488368 ingress_bytes=384983040 link_bytes=384983040 token_bytes=562662384 weight_movement_bytes=103809024 total_nvlink_bytes=666471408 weight_to_token_ratio=0.184496 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=26496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.448660186e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92980 payload_rows_remote=76974 dup_rows=5324 dup_rows_remote=5116 dedup_ratio=0.054159 router_tokens_per_expert=[0, 693, 394, 1072, 8006, 2336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 6683, 0, 0, 0, 0, 210, 7018, 77, 0, 0, 0, 0, 0, 2698, 0, 0, 3564, 2, 116, 0, 0, 178, 0, 0, 0, 104, 0, 0, 1251, 5, 0, 0, 0, 65, 7825, 106, 6668, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6844, 16012, 13474, 574, 11430, 16006, 15150, 13490] dup_rows_per_dst=[2146, 0, 54, 0, 1330, 208, 1528, 58] combine_rows_per_src=[6844, 16012, 13474, 574, 11430, 16006, 15150, 13490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=302768128 dispatch_meta_bytes=336136 combine_payload_bytes=302768128 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=303104264 ingress_bytes=371974144 link_bytes=371974144 token_bytes=605872392 weight_movement_bytes=69206016 total_nvlink_bytes=675078408 weight_to_token_ratio=0.114225 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=44032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.399708757e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86976 payload_rows_remote=71070 dup_rows=11328 dup_rows_remote=4312 dedup_ratio=0.115234 router_tokens_per_expert=[1595, 0, 0, 0, 0, 0, 0, 1627, 483, 0, 404, 0, 0, 0, 0, 1144, 0, 0, 0, 5961, 248, 428, 0, 6520, 0, 0, 0, 0, 0, 0, 0, 0, 1247, 0, 0, 0, 0, 0, 0, 7562, 0, 0, 0, 7417, 0, 0, 63, 0, 0, 0, 0, 2923, 0, 7727, 0, 0, 0, 0, 0, 3119, 185, 408, 0, 91] payload_rows_per_dst=[6282, 6258, 13188, 13040, 15906, 14852, 10026, 7424] dup_rows_per_dst=[162, 1488, 86, 0, 7016, 108, 2286, 182] combine_rows_per_src=[6282, 6258, 13188, 13040, 15906, 14852, 10026, 7424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=321093632 dispatch_meta_bytes=325400 combine_payload_bytes=321093632 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=321419032 ingress_bytes=390299648 link_bytes=390299648 token_bytes=642512664 weight_movement_bytes=69206016 total_nvlink_bytes=711718680 weight_to_token_ratio=0.107712 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=32128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.468666154e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=288776192 dispatch_meta_bytes=313688 combine_payload_bytes=288776192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289089880 ingress_bytes=323379200 link_bytes=323379200 token_bytes=577866072 weight_movement_bytes=34603008 total_nvlink_bytes=612469080 weight_to_token_ratio=0.059881 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=33024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.216849895e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=315285504 dispatch_meta_bytes=328360 combine_payload_bytes=315285504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=315613864 ingress_bytes=349888512 link_bytes=349888512 token_bytes=630899368 weight_movement_bytes=34603008 total_nvlink_bytes=665502376 weight_to_token_ratio=0.054847 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=47360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.316602302e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82150 payload_rows_remote=66224 dup_rows=16154 dup_rows_remote=7910 dedup_ratio=0.164327 router_tokens_per_expert=[2632, 0, 0, 0, 0, 0, 0, 1723, 508, 0, 434, 0, 0, 0, 0, 1067, 0, 0, 0, 4950, 241, 355, 0, 6258, 0, 0, 0, 0, 0, 0, 0, 0, 1287, 0, 0, 0, 0, 0, 0, 7730, 0, 0, 0, 7320, 0, 0, 68, 0, 0, 0, 0, 2711, 0, 7811, 0, 0, 0, 0, 0, 3380, 159, 441, 0, 77] payload_rows_per_dst=[15926, 15946, 13250, 6428, 2574, 14644, 5422, 7960] dup_rows_per_dst=[8244, 3694, 3930, 0, 0, 132, 0, 154] combine_rows_per_src=[15926, 15946, 13250, 6428, 2574, 14644, 5422, 7960] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84530 payload_rows_remote=68150 dup_rows=13774 dup_rows_remote=7792 dedup_ratio=0.140116 router_tokens_per_expert=[1471, 0, 0, 0, 0, 0, 0, 1628, 505, 0, 405, 0, 0, 0, 0, 1132, 0, 0, 0, 5820, 287, 381, 0, 6383, 0, 0, 0, 0, 0, 0, 0, 0, 1377, 0, 0, 0, 0, 0, 0, 7785, 0, 0, 0, 7345, 0, 0, 52, 0, 0, 0, 0, 2954, 0, 7791, 0, 0, 0, 0, 0, 3163, 172, 416, 0, 85] payload_rows_per_dst=[5946, 3072, 12832, 12766, 5330, 12618, 15586, 16380] dup_rows_per_dst=[252, 1012, 80, 0, 386, 94, 5968, 5982] combine_rows_per_src=[5946, 3072, 12832, 12766, 5330, 12618, 15586, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=291102720 dispatch_meta_bytes=301528 combine_payload_bytes=291102720 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=291404248 ingress_bytes=394911744 link_bytes=394911744 token_bytes=582506968 weight_movement_bytes=103809024 total_nvlink_bytes=686315992 weight_to_token_ratio=0.178211 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=37888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.486021100e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90924 payload_rows_remote=80758 dup_rows=7380 dup_rows_remote=7380 dedup_ratio=0.075073 router_tokens_per_expert=[2385, 0, 0, 0, 0, 0, 0, 1296, 490, 0, 436, 0, 0, 0, 0, 1090, 0, 0, 0, 4515, 254, 314, 0, 6911, 0, 0, 0, 0, 0, 0, 0, 0, 853, 0, 0, 0, 0, 0, 0, 7737, 0, 0, 0, 7298, 0, 0, 88, 0, 0, 0, 0, 3354, 0, 7708, 0, 0, 0, 0, 0, 3820, 159, 370, 0, 74] payload_rows_per_dst=[9698, 15794, 10166, 13822, 11472, 14596, 6708, 8668] dup_rows_per_dst=[2410, 3654, 0, 0, 962, 176, 0, 178] combine_rows_per_src=[9698, 15794, 10166, 13822, 11472, 14596, 6708, 8668] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=329285632 dispatch_meta_bytes=334528 combine_payload_bytes=329285632 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=329620160 ingress_bytes=398491648 link_bytes=398491648 token_bytes=658905792 weight_movement_bytes=69206016 total_nvlink_bytes=728111808 weight_to_token_ratio=0.105032 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=20864 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.499491990e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86552 payload_rows_remote=71170 dup_rows=11752 dup_rows_remote=10842 dedup_ratio=0.119548 router_tokens_per_expert=[0, 0, 7639, 0, 0, 0, 0, 0, 3, 1, 0, 0, 8037, 0, 446, 0, 2, 362, 37, 0, 0, 0, 37, 0, 322, 1, 1, 0, 0, 0, 0, 6421, 7322, 0, 107, 0, 3, 0, 70, 0, 144, 251, 0, 0, 3, 7477, 8045, 0, 3, 314, 0, 0, 245, 0, 107, 0, 0, 3, 1747, 2, 0, 0, 0, 0] payload_rows_per_dst=[0, 15872, 802, 13086, 8860, 16384, 15382, 16166] dup_rows_per_dst=[0, 306, 74, 404, 226, 6420, 910, 3412] combine_rows_per_src=[0, 15872, 802, 13086, 8860, 16384, 15382, 16166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=271253504 dispatch_meta_bytes=296536 combine_payload_bytes=271253504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271550040 ingress_bytes=305856512 link_bytes=305856512 token_bytes=542803544 weight_movement_bytes=34603008 total_nvlink_bytes=577406552 weight_to_token_ratio=0.063749 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=19200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.150913431e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=279142400 dispatch_meta_bytes=303768 combine_payload_bytes=279142400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279446168 ingress_bytes=313745408 link_bytes=313745408 token_bytes=558588568 weight_movement_bytes=34603008 total_nvlink_bytes=593191576 weight_to_token_ratio=0.061947 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=27648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.180598712e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90924 payload_rows_remote=80758 dup_rows=7380 dup_rows_remote=7380 dedup_ratio=0.075073 router_tokens_per_expert=[2385, 0, 0, 0, 0, 0, 0, 1296, 490, 0, 436, 0, 0, 0, 0, 1090, 0, 0, 0, 4515, 254, 314, 0, 6911, 0, 0, 0, 0, 0, 0, 0, 0, 853, 0, 0, 0, 0, 0, 0, 7737, 0, 0, 0, 7298, 0, 0, 88, 0, 0, 0, 0, 3354, 0, 7708, 0, 0, 0, 0, 0, 3820, 159, 370, 0, 74] payload_rows_per_dst=[9698, 15794, 10166, 13822, 11472, 14596, 6708, 8668] dup_rows_per_dst=[2410, 3654, 0, 0, 962, 176, 0, 178] combine_rows_per_src=[9698, 15794, 10166, 13822, 11472, 14596, 6708, 8668] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84020 payload_rows_remote=68646 dup_rows=14284 dup_rows_remote=9942 dedup_ratio=0.145304 router_tokens_per_expert=[2540, 2, 0, 0, 0, 0, 0, 1463, 512, 3, 504, 0, 1, 0, 1, 1131, 0, 2, 0, 5421, 294, 403, 0, 6424, 0, 0, 0, 0, 0, 0, 0, 1, 1049, 2, 0, 0, 0, 0, 0, 7768, 0, 3, 0, 7308, 2, 3, 73, 0, 0, 0, 0, 2170, 1, 7686, 0, 1, 0, 0, 0, 3652, 156, 469, 0, 107] payload_rows_per_dst=[7770, 3274, 12228, 12848, 16016, 152, 15374, 16358] dup_rows_per_dst=[240, 1030, 12, 2, 1622, 10, 4342, 7026] combine_rows_per_src=[7770, 3274, 12228, 12848, 16016, 152, 15374, 16358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92748 payload_rows_remote=79304 dup_rows=5556 dup_rows_remote=5556 dedup_ratio=0.056519 router_tokens_per_expert=[3043, 0, 0, 0, 0, 0, 0, 1377, 456, 0, 556, 0, 0, 0, 0, 942, 0, 0, 0, 4579, 250, 278, 0, 6722, 0, 0, 0, 0, 0, 0, 0, 0, 759, 0, 0, 0, 0, 0, 0, 7789, 0, 0, 0, 7328, 0, 0, 71, 0, 0, 0, 0, 2949, 0, 7720, 0, 0, 0, 0, 0, 3725, 143, 400, 0, 65] payload_rows_per_dst=[8314, 15690, 10214, 13444, 15996, 14656, 5898, 8536] dup_rows_per_dst=[526, 3658, 0, 0, 1100, 142, 0, 130] combine_rows_per_src=[8314, 15690, 10214, 13444, 15996, 14656, 5898, 8536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=291512320 dispatch_meta_bytes=328048 combine_payload_bytes=291512320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291840368 ingress_bytes=326115328 link_bytes=326115328 token_bytes=583352688 weight_movement_bytes=34603008 total_nvlink_bytes=617955696 weight_to_token_ratio=0.059317 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=43392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.227145725e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95756 payload_rows_remote=79516 dup_rows=2548 dup_rows_remote=1684 dedup_ratio=0.025920 router_tokens_per_expert=[0, 0, 7577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8119, 0, 433, 0, 0, 303, 59, 0, 0, 0, 59, 0, 344, 0, 0, 0, 0, 0, 0, 6507, 7317, 0, 72, 0, 0, 0, 56, 0, 118, 203, 0, 0, 0, 7614, 8061, 0, 0, 464, 0, 0, 258, 0, 72, 0, 0, 0, 1516, 0, 0, 0, 0, 0] payload_rows_per_dst=[15154, 16240, 16240, 13352, 14634, 15660, 1444, 3032] dup_rows_per_dst=[0, 864, 724, 350, 256, 210, 144, 0] combine_rows_per_src=[15154, 16240, 16240, 13352, 14634, 15660, 1444, 3032] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92648 payload_rows_remote=78392 dup_rows=5656 dup_rows_remote=2958 dedup_ratio=0.057536 router_tokens_per_expert=[0, 626, 386, 1086, 8007, 2167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 6893, 0, 0, 0, 0, 180, 7017, 64, 0, 0, 0, 0, 0, 3031, 0, 0, 3300, 0, 129, 0, 0, 131, 0, 0, 0, 52, 0, 0, 1101, 8, 0, 0, 0, 79, 7920, 67, 6771, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6574, 16014, 13924, 14256, 9748, 16102, 2360, 13670] dup_rows_per_dst=[1956, 0, 136, 2698, 740, 104, 16, 6] combine_rows_per_src=[6574, 16014, 13924, 14256, 9748, 16102, 2360, 13670] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88100 payload_rows_remote=76802 dup_rows=10204 dup_rows_remote=10124 dedup_ratio=0.103800 router_tokens_per_expert=[3014, 0, 0, 0, 0, 0, 0, 1463, 438, 0, 507, 0, 0, 0, 0, 981, 0, 0, 0, 4971, 193, 356, 0, 6391, 0, 0, 0, 0, 0, 0, 0, 0, 974, 0, 0, 0, 0, 0, 0, 7817, 0, 0, 0, 7333, 0, 0, 62, 0, 0, 0, 0, 2741, 0, 7728, 0, 0, 0, 0, 0, 3517, 143, 445, 0, 78] payload_rows_per_dst=[8394, 2974, 11040, 12782, 16042, 11298, 15456, 10114] dup_rows_per_dst=[560, 878, 0, 0, 1540, 80, 5482, 1664] combine_rows_per_src=[8394, 2974, 11040, 12782, 16042, 11298, 15456, 10114] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=330784768 dispatch_meta_bytes=352552 combine_payload_bytes=330784768 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=331137320 ingress_bytes=399990784 link_bytes=399990784 token_bytes=661922088 weight_movement_bytes=69206016 total_nvlink_bytes=731128104 weight_to_token_ratio=0.104553 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=41216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.505133118e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82150 payload_rows_remote=66224 dup_rows=16154 dup_rows_remote=7910 dedup_ratio=0.164327 router_tokens_per_expert=[2632, 0, 0, 0, 0, 0, 0, 1723, 508, 0, 434, 0, 0, 0, 0, 1067, 0, 0, 0, 4950, 241, 355, 0, 6258, 0, 0, 0, 0, 0, 0, 0, 0, 1287, 0, 0, 0, 0, 0, 0, 7730, 0, 0, 0, 7320, 0, 0, 68, 0, 0, 0, 0, 2711, 0, 7811, 0, 0, 0, 0, 0, 3380, 159, 441, 0, 77] payload_rows_per_dst=[15926, 15946, 13250, 6428, 2574, 14644, 5422, 7960] dup_rows_per_dst=[8244, 3694, 3930, 0, 0, 132, 0, 154] combine_rows_per_src=[15926, 15946, 13250, 6428, 2574, 14644, 5422, 7960] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92684 payload_rows_remote=86656 dup_rows=5620 dup_rows_remote=5506 dedup_ratio=0.057170 router_tokens_per_expert=[0, 0, 7653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8102, 0, 429, 0, 0, 367, 50, 0, 0, 0, 50, 0, 340, 0, 0, 0, 0, 0, 0, 6513, 7252, 0, 77, 0, 0, 0, 59, 0, 112, 206, 0, 0, 0, 7584, 8065, 0, 0, 370, 0, 0, 251, 0, 78, 0, 0, 0, 1594, 0, 0, 0, 0, 0] payload_rows_per_dst=[11286, 4818, 10986, 13348, 14504, 6028, 15486, 16228] dup_rows_per_dst=[0, 60, 572, 358, 272, 114, 1080, 3164] combine_rows_per_src=[11286, 4818, 10986, 13348, 14504, 6028, 15486, 16228] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=281174016 dispatch_meta_bytes=314352 combine_payload_bytes=281174016 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=281488368 ingress_bytes=384983040 link_bytes=384983040 token_bytes=562662384 weight_movement_bytes=103809024 total_nvlink_bytes=666471408 weight_to_token_ratio=0.184496 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=26496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.448660186e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92754 payload_rows_remote=78112 dup_rows=5550 dup_rows_remote=5252 dedup_ratio=0.056458 router_tokens_per_expert=[0, 0, 7585, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8064, 0, 482, 0, 0, 317, 66, 0, 0, 0, 66, 0, 302, 0, 0, 0, 0, 0, 0, 6625, 7321, 0, 93, 0, 0, 0, 56, 0, 130, 250, 0, 0, 0, 7537, 8039, 0, 0, 512, 0, 0, 252, 0, 91, 0, 0, 0, 1363, 0, 0, 0, 0, 0] payload_rows_per_dst=[15170, 964, 16202, 13398, 14642, 752, 15426, 16200] dup_rows_per_dst=[0, 2, 774, 456, 298, 8, 1358, 2654] combine_rows_per_src=[15170, 964, 16202, 13398, 14642, 752, 15426, 16200] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85228 payload_rows_remote=69648 dup_rows=13076 dup_rows_remote=2358 dedup_ratio=0.133016 router_tokens_per_expert=[0, 0, 7648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 0, 414, 0, 0, 364, 59, 0, 0, 0, 59, 0, 347, 0, 0, 0, 0, 0, 0, 6377, 7295, 0, 77, 0, 0, 0, 71, 0, 116, 216, 0, 0, 0, 7617, 8056, 0, 0, 367, 0, 0, 245, 0, 77, 0, 0, 0, 1634, 0, 0, 0, 0, 0] payload_rows_per_dst=[15580, 16230, 16230, 2808, 14198, 15696, 1218, 3268] dup_rows_per_dst=[10718, 824, 846, 44, 282, 202, 160, 0] combine_rows_per_src=[15580, 16230, 16230, 2808, 14198, 15696, 1218, 3268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=324829184 dispatch_meta_bytes=339440 combine_payload_bytes=324829184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=325168624 ingress_bytes=359432192 link_bytes=359432192 token_bytes=649997808 weight_movement_bytes=34603008 total_nvlink_bytes=684600816 weight_to_token_ratio=0.053236 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.037500 gemm_active_groups=3 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.352514402e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=325697536 dispatch_meta_bytes=324800 combine_payload_bytes=325697536 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=326022336 ingress_bytes=394903552 link_bytes=394903552 token_bytes=651719872 weight_movement_bytes=69206016 total_nvlink_bytes=720925888 weight_to_token_ratio=0.106190 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=28416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.485990274e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=321093632 dispatch_meta_bytes=325400 combine_payload_bytes=321093632 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=321419032 ingress_bytes=390299648 link_bytes=390299648 token_bytes=642512664 weight_movement_bytes=69206016 total_nvlink_bytes=711718680 weight_to_token_ratio=0.107712 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=32128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.468666154e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=271253504 dispatch_meta_bytes=296536 combine_payload_bytes=271253504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271550040 ingress_bytes=305856512 link_bytes=305856512 token_bytes=542803544 weight_movement_bytes=34603008 total_nvlink_bytes=577406552 weight_to_token_ratio=0.063749 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=19200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.150913431e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=354942976 dispatch_meta_bytes=368648 combine_payload_bytes=354942976 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=355311624 ingress_bytes=458752000 link_bytes=458752000 token_bytes=710254600 weight_movement_bytes=103809024 total_nvlink_bytes=814063624 weight_to_token_ratio=0.146157 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=18944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.726246843e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=314580992 dispatch_meta_bytes=347704 combine_payload_bytes=314580992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=314928696 ingress_bytes=383787008 link_bytes=383787008 token_bytes=629509688 weight_movement_bytes=69206016 total_nvlink_bytes=698715704 weight_to_token_ratio=0.109936 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=42368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.444159614e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78336 payload_rows_remote=62122 dup_rows=19968 dup_rows_remote=17344 dedup_ratio=0.203125 router_tokens_per_expert=[0, 0, 7668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7995, 0, 429, 0, 0, 331, 57, 0, 0, 0, 57, 0, 247, 0, 0, 0, 0, 0, 0, 6553, 7436, 0, 84, 0, 0, 0, 50, 0, 134, 225, 0, 0, 0, 7568, 8051, 0, 0, 517, 0, 0, 241, 0, 85, 0, 0, 0, 1424, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 15994, 776, 13224, 268, 16384, 15476, 16214] dup_rows_per_dst=[0, 200, 114, 376, 0, 15308, 1346, 2624] combine_rows_per_src=[0, 15994, 776, 13224, 268, 16384, 15476, 16214] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86976 payload_rows_remote=71070 dup_rows=11328 dup_rows_remote=4312 dedup_ratio=0.115234 router_tokens_per_expert=[1595, 0, 0, 0, 0, 0, 0, 1627, 483, 0, 404, 0, 0, 0, 0, 1144, 0, 0, 0, 5961, 248, 428, 0, 6520, 0, 0, 0, 0, 0, 0, 0, 0, 1247, 0, 0, 0, 0, 0, 0, 7562, 0, 0, 0, 7417, 0, 0, 63, 0, 0, 0, 0, 2923, 0, 7727, 0, 0, 0, 0, 0, 3119, 185, 408, 0, 91] payload_rows_per_dst=[6282, 6258, 13188, 13040, 15906, 14852, 10026, 7424] dup_rows_per_dst=[162, 1488, 86, 0, 7016, 108, 2286, 182] combine_rows_per_src=[6282, 6258, 13188, 13040, 15906, 14852, 10026, 7424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=319946752 dispatch_meta_bytes=333456 combine_payload_bytes=319946752 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=320280208 ingress_bytes=389152768 link_bytes=389152768 token_bytes=640226960 weight_movement_bytes=69206016 total_nvlink_bytes=709432976 weight_to_token_ratio=0.108096 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=48000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.464350537e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91906 payload_rows_remote=75964 dup_rows=6398 dup_rows_remote=4954 dedup_ratio=0.065084 router_tokens_per_expert=[0, 1, 2, 3, 3, 1, 1228, 0, 107, 7853, 4816, 0, 0, 107, 0, 1, 0, 0, 0, 34, 3, 580, 3, 0, 0, 0, 0, 0, 851, 7740, 1, 0, 0, 3, 0, 0, 1, 7573, 0, 8082, 107, 0, 0, 116, 2, 0, 191, 0, 246, 0, 0, 0, 0, 874, 0, 0, 0, 0, 352, 7887, 0, 0, 0, 384] payload_rows_per_dst=[2464, 216, 15798, 15482, 9638, 16378, 15942, 15988] dup_rows_per_dst=[12, 214, 1148, 1702, 2, 618, 1444, 1258] combine_rows_per_src=[2464, 216, 15798, 15482, 9638, 16378, 15942, 15988] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=285278208 dispatch_meta_bytes=288024 combine_payload_bytes=285278208 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=285566232 ingress_bytes=354484224 link_bytes=354484224 token_bytes=570844440 weight_movement_bytes=69206016 total_nvlink_bytes=640050456 weight_to_token_ratio=0.121234 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.037500 gemm_active_groups=3 gemm_peak_group_rows=43776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.333895597e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95236 payload_rows_remote=78998 dup_rows=3068 dup_rows_remote=2370 dedup_ratio=0.031209 router_tokens_per_expert=[0, 0, 7601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 0, 412, 0, 0, 297, 51, 0, 0, 0, 51, 0, 339, 0, 0, 0, 0, 0, 0, 6445, 7337, 0, 71, 0, 0, 0, 92, 0, 114, 202, 0, 0, 0, 7583, 8069, 0, 0, 330, 0, 0, 244, 0, 74, 0, 0, 0, 1727, 0, 0, 0, 0, 0] payload_rows_per_dst=[15202, 16236, 16238, 13232, 14674, 4494, 11706, 3454] dup_rows_per_dst=[0, 814, 698, 336, 326, 66, 828, 0] combine_rows_per_src=[15202, 16236, 16238, 13232, 14674, 4494, 11706, 3454] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=254451712 dispatch_meta_bytes=317864 combine_payload_bytes=254451712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=254769576 ingress_bytes=289054720 link_bytes=289054720 token_bytes=509221288 weight_movement_bytes=34603008 total_nvlink_bytes=543824296 weight_to_token_ratio=0.067953 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=36352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.087689641e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=291102720 dispatch_meta_bytes=301528 combine_payload_bytes=291102720 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=291404248 ingress_bytes=394911744 link_bytes=394911744 token_bytes=582506968 weight_movement_bytes=103809024 total_nvlink_bytes=686315992 weight_to_token_ratio=0.178211 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=37888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.486021100e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86552 payload_rows_remote=71170 dup_rows=11752 dup_rows_remote=10842 dedup_ratio=0.119548 router_tokens_per_expert=[0, 0, 7639, 0, 0, 0, 0, 0, 3, 1, 0, 0, 8037, 0, 446, 0, 2, 362, 37, 0, 0, 0, 37, 0, 322, 1, 1, 0, 0, 0, 0, 6421, 7322, 0, 107, 0, 3, 0, 70, 0, 144, 251, 0, 0, 3, 7477, 8045, 0, 3, 314, 0, 0, 245, 0, 107, 0, 0, 3, 1747, 2, 0, 0, 0, 0] payload_rows_per_dst=[0, 15872, 802, 13086, 8860, 16384, 15382, 16166] dup_rows_per_dst=[0, 306, 74, 404, 226, 6420, 910, 3412] combine_rows_per_src=[0, 15872, 802, 13086, 8860, 16384, 15382, 16166] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92748 payload_rows_remote=79304 dup_rows=5556 dup_rows_remote=5556 dedup_ratio=0.056519 router_tokens_per_expert=[3043, 0, 0, 0, 0, 0, 0, 1377, 456, 0, 556, 0, 0, 0, 0, 942, 0, 0, 0, 4579, 250, 278, 0, 6722, 0, 0, 0, 0, 0, 0, 0, 0, 759, 0, 0, 0, 0, 0, 0, 7789, 0, 0, 0, 7328, 0, 0, 71, 0, 0, 0, 0, 2949, 0, 7720, 0, 0, 0, 0, 0, 3725, 143, 400, 0, 65] payload_rows_per_dst=[8314, 15690, 10214, 13444, 15996, 14656, 5898, 8536] dup_rows_per_dst=[526, 3658, 0, 0, 1100, 142, 0, 130] combine_rows_per_src=[8314, 15690, 10214, 13444, 15996, 14656, 5898, 8536] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69022 payload_rows_remote=53164 dup_rows=29282 dup_rows_remote=19280 dedup_ratio=0.297872 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1115, 0, 72, 7839, 4947, 0, 0, 72, 0, 0, 0, 0, 0, 57, 0, 751, 0, 0, 0, 0, 0, 0, 844, 7865, 0, 0, 0, 0, 0, 0, 0, 7710, 0, 8120, 72, 0, 0, 80, 0, 0, 172, 0, 259, 0, 0, 0, 0, 618, 0, 0, 0, 0, 330, 7911, 0, 0, 0, 318] payload_rows_per_dst=[15932, 15858, 1616, 15822, 16240, 504, 1754, 1296] dup_rows_per_dst=[2028, 10002, 0, 1688, 15420, 144, 0, 0] combine_rows_per_src=[15932, 15858, 1616, 15822, 16240, 504, 1754, 1296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=311148544 dispatch_meta_bytes=323672 combine_payload_bytes=311148544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=311472216 ingress_bytes=345751552 link_bytes=345751552 token_bytes=622620760 weight_movement_bytes=34603008 total_nvlink_bytes=657223768 weight_to_token_ratio=0.055576 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=41856 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.301035255e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=323575808 dispatch_meta_bytes=325472 combine_payload_bytes=323575808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=323901280 ingress_bytes=358178816 link_bytes=358178816 token_bytes=647477088 weight_movement_bytes=34603008 total_nvlink_bytes=682080096 weight_to_token_ratio=0.053443 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=45696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.347798049e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85228 payload_rows_remote=69648 dup_rows=13076 dup_rows_remote=2358 dedup_ratio=0.133016 router_tokens_per_expert=[0, 0, 7648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 0, 414, 0, 0, 364, 59, 0, 0, 0, 59, 0, 347, 0, 0, 0, 0, 0, 0, 6377, 7295, 0, 77, 0, 0, 0, 71, 0, 116, 216, 0, 0, 0, 7617, 8056, 0, 0, 367, 0, 0, 245, 0, 77, 0, 0, 0, 1634, 0, 0, 0, 0, 0] payload_rows_per_dst=[15580, 16230, 16230, 2808, 14198, 15696, 1218, 3268] dup_rows_per_dst=[10718, 824, 846, 44, 282, 202, 160, 0] combine_rows_per_src=[15580, 16230, 16230, 2808, 14198, 15696, 1218, 3268] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94078 payload_rows_remote=80810 dup_rows=4226 dup_rows_remote=3962 dedup_ratio=0.042989 router_tokens_per_expert=[0, 0, 7597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8099, 0, 375, 0, 0, 335, 50, 0, 0, 0, 50, 0, 310, 0, 0, 0, 0, 0, 0, 6456, 7297, 0, 64, 0, 0, 0, 71, 0, 103, 186, 0, 0, 0, 7603, 8079, 0, 0, 258, 0, 0, 244, 0, 65, 0, 0, 0, 1910, 0, 0, 0, 0, 0] payload_rows_per_dst=[15194, 8560, 16256, 13268, 14594, 578, 15486, 10142] dup_rows_per_dst=[0, 320, 772, 264, 270, 0, 854, 1746] combine_rows_per_src=[15194, 8560, 16256, 13268, 14594, 578, 15486, 10142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69022 payload_rows_remote=53164 dup_rows=29282 dup_rows_remote=19280 dedup_ratio=0.297872 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1115, 0, 72, 7839, 4947, 0, 0, 72, 0, 0, 0, 0, 0, 57, 0, 751, 0, 0, 0, 0, 0, 0, 844, 7865, 0, 0, 0, 0, 0, 0, 0, 7710, 0, 8120, 72, 0, 0, 80, 0, 0, 172, 0, 259, 0, 0, 0, 0, 618, 0, 0, 0, 0, 330, 7911, 0, 0, 0, 318] payload_rows_per_dst=[15932, 15858, 1616, 15822, 16240, 504, 1754, 1296] dup_rows_per_dst=[2028, 10002, 0, 1688, 15420, 144, 0, 0] combine_rows_per_src=[15932, 15858, 1616, 15822, 16240, 504, 1754, 1296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92684 payload_rows_remote=86656 dup_rows=5620 dup_rows_remote=5506 dedup_ratio=0.057170 router_tokens_per_expert=[0, 0, 7653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8102, 0, 429, 0, 0, 367, 50, 0, 0, 0, 50, 0, 340, 0, 0, 0, 0, 0, 0, 6513, 7252, 0, 77, 0, 0, 0, 59, 0, 112, 206, 0, 0, 0, 7584, 8065, 0, 0, 370, 0, 0, 251, 0, 78, 0, 0, 0, 1594, 0, 0, 0, 0, 0] payload_rows_per_dst=[11286, 4818, 10986, 13348, 14504, 6028, 15486, 16228] dup_rows_per_dst=[0, 60, 572, 358, 272, 114, 1080, 3164] combine_rows_per_src=[11286, 4818, 10986, 13348, 14504, 6028, 15486, 16228] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12852893120 prefetch_bytes=692060160 reduce_bytes=865075200 weight_movement_bytes=1557135360 egress_bytes=6430234048 ingress_bytes=7979794432 link_bytes=7979794432 total_nvlink_bytes=14410028480 weight_to_token_ratio=0.121151 prefetch_share_of_ingress=0.086727 dedup_ratio=0.173503 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=25 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029783 gemm_inner_pad_upper_bound=22225 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.305535827e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91930 payload_rows_remote=75546 dup_rows=6374 dup_rows_remote=5650 dedup_ratio=0.064840 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 987, 0, 77, 7875, 5006, 0, 0, 77, 0, 0, 0, 0, 0, 49, 0, 903, 0, 0, 0, 0, 0, 0, 803, 7745, 0, 0, 0, 0, 0, 0, 0, 7673, 0, 8115, 78, 0, 0, 85, 0, 0, 276, 0, 259, 0, 0, 0, 0, 592, 0, 0, 0, 0, 329, 7911, 0, 0, 0, 312] payload_rows_per_dst=[4016, 6070, 15760, 13554, 4096, 16384, 16072, 15978] dup_rows_per_dst=[182, 154, 1894, 1318, 0, 724, 976, 1126] combine_rows_per_src=[4016, 6070, 15760, 13554, 4096, 16384, 16072, 15978] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=291512320 dispatch_meta_bytes=328048 combine_payload_bytes=291512320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291840368 ingress_bytes=326115328 link_bytes=326115328 token_bytes=583352688 weight_movement_bytes=34603008 total_nvlink_bytes=617955696 weight_to_token_ratio=0.059317 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=43392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.227145725e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=324829184 dispatch_meta_bytes=339440 combine_payload_bytes=324829184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=325168624 ingress_bytes=359432192 link_bytes=359432192 token_bytes=649997808 weight_movement_bytes=34603008 total_nvlink_bytes=684600816 weight_to_token_ratio=0.053236 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.037500 gemm_active_groups=3 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.352514402e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90894 payload_rows_remote=90894 dup_rows=7410 dup_rows_remote=7410 dedup_ratio=0.075378 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1325, 0, 90, 7846, 4540, 0, 0, 91, 0, 0, 0, 0, 0, 65, 0, 604, 0, 0, 0, 0, 0, 0, 873, 7852, 0, 0, 0, 0, 0, 0, 0, 7624, 0, 8101, 91, 0, 0, 94, 0, 0, 99, 0, 252, 0, 0, 0, 0, 1014, 0, 0, 0, 0, 343, 7943, 0, 0, 0, 305] payload_rows_per_dst=[15720, 9262, 15692, 1746, 0, 16384, 16022, 16068] dup_rows_per_dst=[2634, 180, 1338, 0, 0, 386, 1758, 1114] combine_rows_per_src=[15720, 9262, 15692, 1746, 0, 16384, 16022, 16068] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68916 payload_rows_remote=53068 dup_rows=29388 dup_rows_remote=27664 dedup_ratio=0.298950 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 981, 0, 77, 7864, 5122, 0, 0, 77, 0, 0, 0, 0, 0, 59, 0, 697, 0, 0, 0, 0, 0, 0, 847, 7805, 0, 0, 0, 0, 0, 0, 0, 7706, 0, 8115, 77, 0, 0, 87, 0, 0, 195, 0, 246, 0, 0, 0, 0, 611, 0, 0, 0, 0, 323, 7912, 0, 0, 0, 351] payload_rows_per_dst=[15848, 15876, 1512, 15824, 16230, 564, 1714, 1348] dup_rows_per_dst=[1724, 10404, 0, 1694, 15412, 154, 0, 0] combine_rows_per_src=[15848, 15876, 1512, 15824, 16230, 564, 1714, 1348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=330997760 dispatch_meta_bytes=339088 combine_payload_bytes=330997760 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=331336848 ingress_bytes=400203776 link_bytes=400203776 token_bytes=662334608 weight_movement_bytes=69206016 total_nvlink_bytes=731540624 weight_to_token_ratio=0.104488 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=46464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.505934590e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=217759744 dispatch_meta_bytes=289776 combine_payload_bytes=217759744 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=218049520 ingress_bytes=321568768 link_bytes=321568768 token_bytes=435809264 weight_movement_bytes=103809024 total_nvlink_bytes=539618288 weight_to_token_ratio=0.238198 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=27520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.210037386e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=285278208 dispatch_meta_bytes=288024 combine_payload_bytes=285278208 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=285566232 ingress_bytes=354484224 link_bytes=354484224 token_bytes=570844440 weight_movement_bytes=69206016 total_nvlink_bytes=640050456 weight_to_token_ratio=0.121234 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.037500 gemm_active_groups=3 gemm_peak_group_rows=43776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.333895597e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=309436416 dispatch_meta_bytes=324784 combine_payload_bytes=309436416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=309761200 ingress_bytes=344039424 link_bytes=344039424 token_bytes=619197616 weight_movement_bytes=34603008 total_nvlink_bytes=653800624 weight_to_token_ratio=0.055884 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=46336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.294592655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=354942976 dispatch_meta_bytes=368648 combine_payload_bytes=354942976 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=355311624 ingress_bytes=458752000 link_bytes=458752000 token_bytes=710254600 weight_movement_bytes=103809024 total_nvlink_bytes=814063624 weight_to_token_ratio=0.146157 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=18944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.726246843e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90626 payload_rows_remote=74650 dup_rows=7678 dup_rows_remote=5358 dedup_ratio=0.078105 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1208, 0, 83, 7863, 4610, 0, 0, 84, 0, 0, 0, 0, 0, 56, 0, 837, 0, 0, 0, 0, 0, 0, 850, 7912, 0, 0, 0, 0, 0, 0, 0, 7682, 0, 8108, 85, 0, 0, 95, 0, 0, 94, 0, 246, 0, 0, 0, 0, 770, 0, 0, 0, 0, 326, 7903, 0, 0, 0, 340] payload_rows_per_dst=[2416, 168, 15750, 15824, 8062, 16384, 16046, 15976] dup_rows_per_dst=[0, 166, 1762, 1700, 0, 380, 1350, 2320] combine_rows_per_src=[2416, 168, 15750, 15824, 8062, 16384, 16046, 15976] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92754 payload_rows_remote=78112 dup_rows=5550 dup_rows_remote=5252 dedup_ratio=0.056458 router_tokens_per_expert=[0, 0, 7585, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8064, 0, 482, 0, 0, 317, 66, 0, 0, 0, 66, 0, 302, 0, 0, 0, 0, 0, 0, 6625, 7321, 0, 93, 0, 0, 0, 56, 0, 130, 250, 0, 0, 0, 7537, 8039, 0, 0, 512, 0, 0, 252, 0, 91, 0, 0, 0, 1363, 0, 0, 0, 0, 0] payload_rows_per_dst=[15170, 964, 16202, 13398, 14642, 752, 15426, 16200] dup_rows_per_dst=[0, 2, 774, 456, 298, 8, 1358, 2654] combine_rows_per_src=[15170, 964, 16202, 13398, 14642, 752, 15426, 16200] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=372301824 dispatch_meta_bytes=393216 combine_payload_bytes=372301824 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=372695040 ingress_bytes=476110848 link_bytes=476110848 token_bytes=744996864 weight_movement_bytes=103809024 total_nvlink_bytes=848805888 weight_to_token_ratio=0.139342 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.791566791e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82962 payload_rows_remote=75512 dup_rows=15342 dup_rows_remote=14586 dedup_ratio=0.156067 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 997, 0, 73, 7862, 4929, 0, 0, 73, 0, 0, 0, 0, 0, 49, 0, 849, 0, 0, 0, 0, 0, 0, 829, 7789, 0, 0, 0, 0, 0, 0, 0, 7719, 0, 8119, 74, 0, 0, 76, 0, 0, 255, 0, 277, 0, 0, 0, 0, 537, 0, 0, 0, 0, 318, 7936, 0, 0, 0, 391] payload_rows_per_dst=[15942, 13506, 7450, 9464, 8648, 11740, 8242, 7970] dup_rows_per_dst=[1630, 5958, 756, 936, 4556, 518, 410, 578] combine_rows_per_src=[15942, 13506, 7450, 9464, 8648, 11740, 8242, 7970] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=217366528 dispatch_meta_bytes=322928 combine_payload_bytes=217366528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=217689456 ingress_bytes=251969536 link_bytes=251969536 token_bytes=435055984 weight_movement_bytes=34603008 total_nvlink_bytes=469658992 weight_to_token_ratio=0.079537 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=40448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.481410788e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13167645768 prefetch_bytes=726663168 reduce_bytes=519045120 weight_movement_bytes=1245708288 egress_bytes=6587683912 ingress_bytes=7825670144 link_bytes=7825670144 total_nvlink_bytes=14413354056 weight_to_token_ratio=0.094604 prefetch_share_of_ingress=0.092856 dedup_ratio=0.174217 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=15 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030217 gemm_inner_pad_upper_bound=22860 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.280320293e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82962 payload_rows_remote=75512 dup_rows=15342 dup_rows_remote=14586 dedup_ratio=0.156067 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 997, 0, 73, 7862, 4929, 0, 0, 73, 0, 0, 0, 0, 0, 49, 0, 849, 0, 0, 0, 0, 0, 0, 829, 7789, 0, 0, 0, 0, 0, 0, 0, 7719, 0, 8119, 74, 0, 0, 76, 0, 0, 255, 0, 277, 0, 0, 0, 0, 537, 0, 0, 0, 0, 318, 7936, 0, 0, 0, 391] payload_rows_per_dst=[15942, 13506, 7450, 9464, 8648, 11740, 8242, 7970] dup_rows_per_dst=[1630, 5958, 756, 936, 4556, 518, 410, 578] combine_rows_per_src=[15942, 13506, 7450, 9464, 8648, 11740, 8242, 7970] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13836634616 prefetch_bytes=830472192 reduce_bytes=726663168 weight_movement_bytes=1557135360 egress_bytes=6922226168 ingress_bytes=8471543808 link_bytes=8471543808 total_nvlink_bytes=15393769976 weight_to_token_ratio=0.112537 prefetch_share_of_ingress=0.098031 dedup_ratio=0.136417 prefetch_slots=48 unused_prefetch_slots=320 slot_utilization=0.130435 reduce_slots=21 remote_demand=24 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028370 gemm_inner_pad_upper_bound=24003 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.385988580e-04 prefetch_bytes_per_gemm_flop=1.358695652e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=305766400 dispatch_meta_bytes=320032 combine_payload_bytes=305766400 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=306086432 ingress_bytes=374972416 link_bytes=374972416 token_bytes=611852832 weight_movement_bytes=69206016 total_nvlink_bytes=681058848 weight_to_token_ratio=0.113109 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=43264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.410991014e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=319946752 dispatch_meta_bytes=333456 combine_payload_bytes=319946752 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=320280208 ingress_bytes=389152768 link_bytes=389152768 token_bytes=640226960 weight_movement_bytes=69206016 total_nvlink_bytes=709432976 weight_to_token_ratio=0.108096 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=48000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.464350537e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91906 payload_rows_remote=75964 dup_rows=6398 dup_rows_remote=4954 dedup_ratio=0.065084 router_tokens_per_expert=[0, 1, 2, 3, 3, 1, 1228, 0, 107, 7853, 4816, 0, 0, 107, 0, 1, 0, 0, 0, 34, 3, 580, 3, 0, 0, 0, 0, 0, 851, 7740, 1, 0, 0, 3, 0, 0, 1, 7573, 0, 8082, 107, 0, 0, 116, 2, 0, 191, 0, 246, 0, 0, 0, 0, 874, 0, 0, 0, 0, 352, 7887, 0, 0, 0, 384] payload_rows_per_dst=[2464, 216, 15798, 15482, 9638, 16378, 15942, 15988] dup_rows_per_dst=[12, 214, 1148, 1702, 2, 618, 1444, 1258] combine_rows_per_src=[2464, 216, 15798, 15482, 9638, 16378, 15942, 15988] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94078 payload_rows_remote=80810 dup_rows=4226 dup_rows_remote=3962 dedup_ratio=0.042989 router_tokens_per_expert=[0, 0, 7597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8099, 0, 375, 0, 0, 335, 50, 0, 0, 0, 50, 0, 310, 0, 0, 0, 0, 0, 0, 6456, 7297, 0, 64, 0, 0, 0, 71, 0, 103, 186, 0, 0, 0, 7603, 8079, 0, 0, 258, 0, 0, 244, 0, 65, 0, 0, 0, 1910, 0, 0, 0, 0, 0] payload_rows_per_dst=[15194, 8560, 16256, 13268, 14594, 578, 15486, 10142] dup_rows_per_dst=[0, 320, 772, 264, 270, 0, 854, 1746] combine_rows_per_src=[15194, 8560, 16256, 13268, 14594, 578, 15486, 10142] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68916 payload_rows_remote=53068 dup_rows=29388 dup_rows_remote=27664 dedup_ratio=0.298950 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 981, 0, 77, 7864, 5122, 0, 0, 77, 0, 0, 0, 0, 0, 59, 0, 697, 0, 0, 0, 0, 0, 0, 847, 7805, 0, 0, 0, 0, 0, 0, 0, 7706, 0, 8115, 77, 0, 0, 87, 0, 0, 195, 0, 246, 0, 0, 0, 0, 611, 0, 0, 0, 0, 323, 7912, 0, 0, 0, 351] payload_rows_per_dst=[15848, 15876, 1512, 15824, 16230, 564, 1714, 1348] dup_rows_per_dst=[1724, 10404, 0, 1694, 15412, 154, 0, 0] combine_rows_per_src=[15848, 15876, 1512, 15824, 16230, 564, 1714, 1348] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=309297152 dispatch_meta_bytes=360392 combine_payload_bytes=309297152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=309657544 ingress_bytes=343900160 link_bytes=343900160 token_bytes=618954696 weight_movement_bytes=34603008 total_nvlink_bytes=653557704 weight_to_token_ratio=0.055906 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=42624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.294068616e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12050401408 prefetch_bytes=726663168 reduce_bytes=588251136 weight_movement_bytes=1314914304 egress_bytes=6028830848 ingress_bytes=7336484864 link_bytes=7336484864 total_nvlink_bytes=13365315712 weight_to_token_ratio=0.109118 prefetch_share_of_ingress=0.099048 dedup_ratio=0.210752 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=17 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030435 gemm_inner_pad_upper_bound=22733 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.200287039e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=311148544 dispatch_meta_bytes=323672 combine_payload_bytes=311148544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=311472216 ingress_bytes=345751552 link_bytes=345751552 token_bytes=622620760 weight_movement_bytes=34603008 total_nvlink_bytes=657223768 weight_to_token_ratio=0.055576 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=41856 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.301035255e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=330997760 dispatch_meta_bytes=339088 combine_payload_bytes=330997760 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=331336848 ingress_bytes=400203776 link_bytes=400203776 token_bytes=662334608 weight_movement_bytes=69206016 total_nvlink_bytes=731540624 weight_to_token_ratio=0.104488 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=46464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.505934590e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=217366528 dispatch_meta_bytes=322928 combine_payload_bytes=217366528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=217689456 ingress_bytes=251969536 link_bytes=251969536 token_bytes=435055984 weight_movement_bytes=34603008 total_nvlink_bytes=469658992 weight_to_token_ratio=0.079537 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=40448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.481410788e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13792672568 prefetch_bytes=622854144 reduce_bytes=657457152 weight_movement_bytes=1280311296 egress_bytes=6900251448 ingress_bytes=8172732416 link_bytes=8172732416 total_nvlink_bytes=15072983864 weight_to_token_ratio=0.092825 prefetch_share_of_ingress=0.076211 dedup_ratio=0.137555 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=19 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030109 gemm_inner_pad_upper_bound=22733 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.337101484e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12852893120 prefetch_bytes=692060160 reduce_bytes=865075200 weight_movement_bytes=1557135360 egress_bytes=6430234048 ingress_bytes=7979794432 link_bytes=7979794432 total_nvlink_bytes=14410028480 weight_to_token_ratio=0.121151 prefetch_share_of_ingress=0.086727 dedup_ratio=0.173503 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=25 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029783 gemm_inner_pad_upper_bound=22225 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.305535827e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90894 payload_rows_remote=90894 dup_rows=7410 dup_rows_remote=7410 dedup_ratio=0.075378 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1325, 0, 90, 7846, 4540, 0, 0, 91, 0, 0, 0, 0, 0, 65, 0, 604, 0, 0, 0, 0, 0, 0, 873, 7852, 0, 0, 0, 0, 0, 0, 0, 7624, 0, 8101, 91, 0, 0, 94, 0, 0, 99, 0, 252, 0, 0, 0, 0, 1014, 0, 0, 0, 0, 343, 7943, 0, 0, 0, 305] payload_rows_per_dst=[15720, 9262, 15692, 1746, 0, 16384, 16022, 16068] dup_rows_per_dst=[2634, 180, 1338, 0, 0, 386, 1758, 1114] combine_rows_per_src=[15720, 9262, 15692, 1746, 0, 16384, 16022, 16068] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14038833200 prefetch_bytes=761266176 reduce_bytes=899678208 weight_movement_bytes=1660944384 egress_bytes=7023343664 ingress_bytes=8676433920 link_bytes=8676433920 total_nvlink_bytes=15699777584 weight_to_token_ratio=0.118311 prefetch_share_of_ingress=0.087740 dedup_ratio=0.132581 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=26 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029783 gemm_inner_pad_upper_bound=22733 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.419509667e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=305766400 dispatch_meta_bytes=320032 combine_payload_bytes=305766400 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=306086432 ingress_bytes=374972416 link_bytes=374972416 token_bytes=611852832 weight_movement_bytes=69206016 total_nvlink_bytes=681058848 weight_to_token_ratio=0.113109 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=43264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.410991014e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12035531992 prefetch_bytes=657457152 reduce_bytes=795869184 weight_movement_bytes=1453326336 egress_bytes=6021424344 ingress_bytes=7467433984 link_bytes=7467433984 total_nvlink_bytes=13488858328 weight_to_token_ratio=0.120753 prefetch_share_of_ingress=0.088043 dedup_ratio=0.211343 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=23 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030870 gemm_inner_pad_upper_bound=21209 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.221710996e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=372301824 dispatch_meta_bytes=393216 combine_payload_bytes=372301824 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=372695040 ingress_bytes=476110848 link_bytes=476110848 token_bytes=744996864 weight_movement_bytes=103809024 total_nvlink_bytes=848805888 weight_to_token_ratio=0.139342 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.791566791e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=309297152 dispatch_meta_bytes=360392 combine_payload_bytes=309297152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=309657544 ingress_bytes=343900160 link_bytes=343900160 token_bytes=618954696 weight_movement_bytes=34603008 total_nvlink_bytes=653557704 weight_to_token_ratio=0.055906 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=42624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.294068616e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92184 payload_rows_remote=90634 dup_rows=6120 dup_rows_remote=6120 dedup_ratio=0.062256 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 963, 0, 64, 7869, 5007, 0, 0, 64, 0, 0, 0, 0, 0, 50, 0, 995, 0, 0, 0, 0, 0, 0, 775, 7915, 0, 0, 0, 0, 0, 0, 0, 7740, 0, 8128, 65, 0, 0, 75, 0, 0, 253, 0, 302, 0, 0, 0, 0, 245, 0, 0, 0, 0, 309, 7938, 0, 0, 0, 395] payload_rows_per_dst=[16206, 10142, 15824, 1550, 0, 16384, 16072, 16006] dup_rows_per_dst=[1550, 128, 2004, 0, 0, 658, 502, 1278] combine_rows_per_src=[16206, 10142, 15824, 1550, 0, 16384, 16072, 16006] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91930 payload_rows_remote=75546 dup_rows=6374 dup_rows_remote=5650 dedup_ratio=0.064840 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 987, 0, 77, 7875, 5006, 0, 0, 77, 0, 0, 0, 0, 0, 49, 0, 903, 0, 0, 0, 0, 0, 0, 803, 7745, 0, 0, 0, 0, 0, 0, 0, 7673, 0, 8115, 78, 0, 0, 85, 0, 0, 276, 0, 259, 0, 0, 0, 0, 592, 0, 0, 0, 0, 329, 7911, 0, 0, 0, 312] payload_rows_per_dst=[4016, 6070, 15760, 13554, 4096, 16384, 16072, 15978] dup_rows_per_dst=[182, 154, 1894, 1318, 0, 724, 976, 1126] combine_rows_per_src=[4016, 6070, 15760, 13554, 4096, 16384, 16072, 15978] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14038833200 prefetch_bytes=761266176 reduce_bytes=899678208 weight_movement_bytes=1660944384 egress_bytes=7023343664 ingress_bytes=8676433920 link_bytes=8676433920 total_nvlink_bytes=15699777584 weight_to_token_ratio=0.118311 prefetch_share_of_ingress=0.087740 dedup_ratio=0.132581 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=26 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029783 gemm_inner_pad_upper_bound=22733 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.419509667e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12050401408 prefetch_bytes=726663168 reduce_bytes=588251136 weight_movement_bytes=1314914304 egress_bytes=6028830848 ingress_bytes=7336484864 link_bytes=7336484864 total_nvlink_bytes=13365315712 weight_to_token_ratio=0.109118 prefetch_share_of_ingress=0.099048 dedup_ratio=0.210752 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=17 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030435 gemm_inner_pad_upper_bound=22733 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.200287039e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13792672568 prefetch_bytes=622854144 reduce_bytes=657457152 weight_movement_bytes=1280311296 egress_bytes=6900251448 ingress_bytes=8172732416 link_bytes=8172732416 total_nvlink_bytes=15072983864 weight_to_token_ratio=0.092825 prefetch_share_of_ingress=0.076211 dedup_ratio=0.137555 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=19 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030109 gemm_inner_pad_upper_bound=22733 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.337101484e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=371236864 dispatch_meta_bytes=387016 combine_payload_bytes=371236864 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=371623880 ingress_bytes=440442880 link_bytes=440442880 token_bytes=742860744 weight_movement_bytes=69206016 total_nvlink_bytes=812066760 weight_to_token_ratio=0.093161 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=22400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.657351099e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=309436416 dispatch_meta_bytes=324784 combine_payload_bytes=309436416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=309761200 ingress_bytes=344039424 link_bytes=344039424 token_bytes=619197616 weight_movement_bytes=34603008 total_nvlink_bytes=653800624 weight_to_token_ratio=0.055884 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=46336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.294592655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14360435568 prefetch_bytes=795869184 reduce_bytes=761266176 weight_movement_bytes=1557135360 egress_bytes=7184243568 ingress_bytes=8733327360 link_bytes=8733327360 total_nvlink_bytes=15917570928 weight_to_token_ratio=0.108432 prefetch_share_of_ingress=0.091130 dedup_ratio=0.124944 prefetch_slots=46 unused_prefetch_slots=322 slot_utilization=0.125000 reduce_slots=22 remote_demand=23 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029891 gemm_inner_pad_upper_bound=22479 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.428817729e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 anomalies=none - [2026-08-26 15:09:13] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 797.1 | throughput per GPU (TFLOP/s/GPU): 160.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.620937E+00 | loss scale: 1.0 | grad norm: 2.990 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75302 payload_rows_remote=66778 dup_rows=23002 dup_rows_remote=20538 dedup_ratio=0.233988 router_tokens_per_expert=[1080, 319, 849, 1190, 688, 835, 856, 1024, 1386, 361, 947, 343, 339, 1361, 1059, 945, 765, 1040, 800, 676, 718, 731, 395, 1415, 481, 608, 779, 351, 831, 109, 922, 626, 1060, 775, 1236, 862, 366, 225, 470, 803, 629, 635, 520, 775, 1076, 637, 577, 645, 870, 616, 817, 518, 934, 1279, 866, 774, 1105, 320, 1452, 566, 233, 881, 978, 823] payload_rows_per_dst=[8366, 9582, 10268, 9030, 8826, 8524, 10188, 10518] dup_rows_per_dst=[2936, 3900, 2812, 2764, 2768, 2464, 3160, 2198] combine_rows_per_src=[8366, 9582, 10268, 9030, 8826, 8524, 10188, 10518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13836634616 prefetch_bytes=830472192 reduce_bytes=726663168 weight_movement_bytes=1557135360 egress_bytes=6922226168 ingress_bytes=8471543808 link_bytes=8471543808 total_nvlink_bytes=15393769976 weight_to_token_ratio=0.112537 prefetch_share_of_ingress=0.098031 dedup_ratio=0.136417 prefetch_slots=48 unused_prefetch_slots=320 slot_utilization=0.130435 reduce_slots=21 remote_demand=24 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028370 gemm_inner_pad_upper_bound=24003 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.385988580e-04 prefetch_bytes_per_gemm_flop=1.358695652e-05 anomalies=none + [2026-08-26 15:57:01] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 792.3 | throughput per GPU (TFLOP/s/GPU): 161.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.620937E+00 | loss scale: 1.0 | grad norm: 2.990 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74996 payload_rows_remote=65646 dup_rows=23308 dup_rows_remote=18746 dedup_ratio=0.237101 router_tokens_per_expert=[1171, 375, 819, 1272, 638, 738, 795, 1055, 1365, 338, 919, 343, 434, 1330, 845, 914, 739, 1013, 770, 588, 632, 734, 388, 1391, 468, 674, 832, 351, 757, 103, 922, 840, 1083, 860, 1248, 846, 423, 216, 493, 805, 703, 703, 618, 821, 975, 631, 786, 629, 697, 547, 851, 517, 944, 1235, 835, 834, 1203, 252, 1128, 675, 185, 764, 1039, 1053] payload_rows_per_dst=[9350, 9796, 10032, 9334, 9034, 10474, 9436, 7540] dup_rows_per_dst=[4562, 2706, 2478, 3104, 2914, 3728, 2404, 1412] combine_rows_per_src=[9350, 9796, 10032, 9334, 9034, 10474, 9436, 7540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75464 payload_rows_remote=66130 dup_rows=22840 dup_rows_remote=19858 dedup_ratio=0.232340 router_tokens_per_expert=[1191, 317, 771, 1225, 620, 867, 748, 935, 1412, 300, 978, 378, 330, 1472, 847, 1099, 666, 996, 824, 613, 636, 727, 392, 1364, 466, 723, 810, 358, 771, 83, 844, 556, 1065, 831, 1305, 888, 502, 231, 528, 808, 587, 617, 596, 744, 1136, 600, 591, 624, 659, 583, 845, 526, 974, 1354, 865, 710, 1235, 306, 1488, 626, 208, 782, 1107, 912] payload_rows_per_dst=[8756, 9266, 9880, 8836, 9334, 8544, 10040, 10808] dup_rows_per_dst=[3334, 3174, 2556, 2836, 2982, 2446, 2992, 2520] combine_rows_per_src=[8756, 9266, 9880, 8836, 9334, 8544, 10040, 10808] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74992 payload_rows_remote=65998 dup_rows=23312 dup_rows_remote=20566 dedup_ratio=0.237142 router_tokens_per_expert=[1075, 355, 856, 1157, 705, 882, 861, 961, 1362, 365, 924, 336, 320, 1349, 947, 1113, 724, 1015, 823, 622, 757, 725, 357, 1352, 496, 644, 795, 420, 822, 114, 779, 643, 1055, 790, 1181, 901, 407, 214, 458, 901, 600, 683, 529, 726, 1059, 604, 624, 583, 862, 534, 843, 491, 1020, 1290, 926, 662, 1240, 292, 1261, 709, 259, 861, 1035, 896] payload_rows_per_dst=[9396, 8462, 10012, 8994, 8894, 9242, 9580, 10412] dup_rows_per_dst=[4718, 2246, 2738, 2746, 2920, 2692, 2558, 2694] combine_rows_per_src=[9396, 8462, 10012, 8994, 8894, 9242, 9580, 10412] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75798 payload_rows_remote=66022 dup_rows=22506 dup_rows_remote=19916 dedup_ratio=0.228943 router_tokens_per_expert=[1105, 324, 806, 1212, 672, 691, 794, 1081, 1279, 327, 892, 342, 380, 1280, 987, 1118, 703, 1008, 864, 648, 680, 738, 378, 1473, 556, 691, 768, 355, 822, 90, 782, 724, 1013, 845, 1148, 841, 343, 206, 477, 909, 584, 592, 560, 857, 1096, 582, 734, 694, 794, 554, 875, 435, 1053, 1223, 998, 824, 1230, 281, 1226, 590, 219, 866, 1013, 920] payload_rows_per_dst=[9038, 9776, 10398, 9106, 8860, 10368, 9104, 9148] dup_rows_per_dst=[4466, 2590, 2586, 2894, 2704, 3476, 1962, 1828] combine_rows_per_src=[9038, 9776, 10398, 9106, 8860, 10368, 9104, 9148] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74706 payload_rows_remote=64140 dup_rows=23598 dup_rows_remote=20964 dedup_ratio=0.240051 router_tokens_per_expert=[1154, 393, 843, 1263, 588, 906, 730, 1035, 1376, 355, 835, 395, 379, 1203, 783, 1069, 674, 1015, 876, 533, 769, 754, 422, 1393, 457, 669, 798, 324, 782, 100, 863, 752, 1084, 803, 1235, 876, 422, 209, 493, 771, 670, 645, 526, 821, 1106, 596, 708, 592, 767, 575, 815, 522, 999, 1243, 903, 773, 1221, 300, 1344, 673, 205, 777, 1071, 919] payload_rows_per_dst=[6974, 9068, 8552, 10596, 10244, 8790, 9916, 10566] dup_rows_per_dst=[2016, 3722, 1534, 3728, 4148, 2538, 3278, 2634] combine_rows_per_src=[6974, 9068, 8552, 10596, 10244, 8790, 9916, 10566] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75210 payload_rows_remote=65162 dup_rows=23094 dup_rows_remote=20354 dedup_ratio=0.234924 router_tokens_per_expert=[1093, 317, 749, 1191, 740, 831, 805, 1007, 1291, 316, 951, 389, 345, 1269, 920, 1038, 732, 1018, 867, 626, 705, 714, 418, 1314, 517, 675, 798, 400, 816, 135, 997, 596, 1111, 755, 1257, 934, 438, 197, 511, 874, 612, 618, 572, 769, 1145, 620, 673, 670, 704, 551, 794, 489, 1037, 1271, 851, 758, 1202, 312, 1340, 622, 232, 780, 1069, 804] payload_rows_per_dst=[9178, 8252, 10048, 9148, 9160, 10326, 8652, 10446] dup_rows_per_dst=[4488, 2204, 2740, 3102, 2994, 3574, 1716, 2276] combine_rows_per_src=[9178, 8252, 10048, 9148, 9160, 10326, 8652, 10446] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=273522688 dispatch_meta_bytes=349264 combine_payload_bytes=273522688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273871952 ingress_bytes=308125696 link_bytes=308125696 token_bytes=547394640 weight_movement_bytes=34603008 total_nvlink_bytes=581997648 weight_to_token_ratio=0.063214 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=8832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.159452188e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=270327808 dispatch_meta_bytes=346256 combine_payload_bytes=270327808 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=270674064 ingress_bytes=339533824 link_bytes=339533824 token_bytes=541001872 weight_movement_bytes=69206016 total_nvlink_bytes=610207888 weight_to_token_ratio=0.127922 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=9728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.277638445e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=270426112 dispatch_meta_bytes=343752 combine_payload_bytes=270426112 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=270769864 ingress_bytes=339632128 link_bytes=339632128 token_bytes=541195976 weight_movement_bytes=69206016 total_nvlink_bytes=610401992 weight_to_token_ratio=0.127876 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.278008355e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266903552 dispatch_meta_bytes=342064 combine_payload_bytes=266903552 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267245616 ingress_bytes=336109568 link_bytes=336109568 token_bytes=534149168 weight_movement_bytes=69206016 total_nvlink_bytes=603355184 weight_to_token_ratio=0.129563 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=9216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264753245e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=262717440 dispatch_meta_bytes=340416 combine_payload_bytes=262717440 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=263057856 ingress_bytes=366526464 link_bytes=366526464 token_bytes=525775296 weight_movement_bytes=103809024 total_nvlink_bytes=629584320 weight_to_token_ratio=0.197440 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=8960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.379209576e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75464 payload_rows_remote=66130 dup_rows=22840 dup_rows_remote=19858 dedup_ratio=0.232340 router_tokens_per_expert=[1191, 317, 771, 1225, 620, 867, 748, 935, 1412, 300, 978, 378, 330, 1472, 847, 1099, 666, 996, 824, 613, 636, 727, 392, 1364, 466, 723, 810, 358, 771, 83, 844, 556, 1065, 831, 1305, 888, 502, 231, 528, 808, 587, 617, 596, 744, 1136, 600, 591, 624, 659, 583, 845, 526, 974, 1354, 865, 710, 1235, 306, 1488, 626, 208, 782, 1107, 912] payload_rows_per_dst=[8756, 9266, 9880, 8836, 9334, 8544, 10040, 10808] dup_rows_per_dst=[3334, 3174, 2556, 2836, 2982, 2446, 2992, 2520] combine_rows_per_src=[8756, 9266, 9880, 8836, 9334, 8544, 10040, 10808] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74996 payload_rows_remote=65646 dup_rows=23308 dup_rows_remote=18746 dedup_ratio=0.237101 router_tokens_per_expert=[1171, 375, 819, 1272, 638, 738, 795, 1055, 1365, 338, 919, 343, 434, 1330, 845, 914, 739, 1013, 770, 588, 632, 734, 388, 1391, 468, 674, 832, 351, 757, 103, 922, 840, 1083, 860, 1248, 846, 423, 216, 493, 805, 703, 703, 618, 821, 975, 631, 786, 629, 697, 547, 851, 517, 944, 1235, 835, 834, 1203, 252, 1128, 675, 185, 764, 1039, 1053] payload_rows_per_dst=[9350, 9796, 10032, 9334, 9034, 10474, 9436, 7540] dup_rows_per_dst=[4562, 2706, 2478, 3104, 2914, 3728, 2404, 1412] combine_rows_per_src=[9350, 9796, 10032, 9334, 9034, 10474, 9436, 7540] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74678 payload_rows_remote=64546 dup_rows=23626 dup_rows_remote=20440 dedup_ratio=0.240336 router_tokens_per_expert=[1129, 366, 807, 1161, 644, 960, 870, 850, 1315, 409, 952, 369, 294, 1480, 855, 1106, 820, 1060, 800, 593, 668, 690, 397, 1338, 332, 653, 865, 397, 798, 94, 833, 647, 1068, 802, 1240, 886, 344, 254, 505, 631, 610, 642, 597, 721, 1228, 612, 583, 566, 883, 538, 826, 575, 940, 1323, 841, 733, 1185, 267, 1435, 635, 217, 925, 1161, 827] payload_rows_per_dst=[7990, 9530, 9390, 8994, 9218, 8774, 10132, 10650] dup_rows_per_dst=[2736, 4030, 2280, 3092, 3304, 2344, 3186, 2654] combine_rows_per_src=[7990, 9530, 9390, 8994, 9218, 8774, 10132, 10650] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75302 payload_rows_remote=66778 dup_rows=23002 dup_rows_remote=20538 dedup_ratio=0.233988 router_tokens_per_expert=[1080, 319, 849, 1190, 688, 835, 856, 1024, 1386, 361, 947, 343, 339, 1361, 1059, 945, 765, 1040, 800, 676, 718, 731, 395, 1415, 481, 608, 779, 351, 831, 109, 922, 626, 1060, 775, 1236, 862, 366, 225, 470, 803, 629, 635, 520, 775, 1076, 637, 577, 645, 870, 616, 817, 518, 934, 1279, 866, 774, 1105, 320, 1452, 566, 233, 881, 978, 823] payload_rows_per_dst=[8366, 9582, 10268, 9030, 8826, 8524, 10188, 10518] dup_rows_per_dst=[2936, 3900, 2812, 2764, 2768, 2464, 3160, 2198] combine_rows_per_src=[8366, 9582, 10268, 9030, 8826, 8524, 10188, 10518] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76454 payload_rows_remote=67742 dup_rows=21850 dup_rows_remote=18014 dedup_ratio=0.222270 router_tokens_per_expert=[649, 1029, 953, 431, 762, 249, 147, 1052, 71, 962, 1333, 300, 560, 898, 914, 1270, 1039, 1137, 831, 480, 1466, 189, 1791, 1205, 648, 695, 634, 838, 309, 470, 202, 78, 485, 420, 458, 199, 834, 1793, 1237, 1147, 2144, 1617, 1175, 1443, 457, 1294, 29, 259, 784, 1043, 881, 14, 208, 1602, 198, 1309, 743, 360, 860, 526, 1044, 314, 296, 387] payload_rows_per_dst=[8458, 9626, 9964, 9550, 10524, 8712, 9878, 9742] dup_rows_per_dst=[2086, 2990, 2730, 2486, 2622, 3836, 2200, 2900] combine_rows_per_src=[8458, 9626, 9964, 9550, 10524, 8712, 9878, 9742] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76438 payload_rows_remote=66992 dup_rows=21866 dup_rows_remote=19558 dedup_ratio=0.222432 router_tokens_per_expert=[689, 1071, 957, 409, 884, 303, 256, 1140, 79, 974, 1435, 317, 546, 1008, 899, 1330, 1011, 1185, 742, 317, 1555, 240, 1627, 1234, 648, 669, 696, 857, 271, 381, 251, 113, 373, 322, 436, 182, 903, 1812, 1213, 1076, 1991, 1611, 1101, 1353, 501, 1393, 36, 227, 728, 1156, 842, 19, 175, 1595, 229, 1227, 846, 344, 838, 559, 895, 395, 311, 369] payload_rows_per_dst=[9586, 9016, 9680, 9446, 10432, 8792, 9890, 9596] dup_rows_per_dst=[2796, 3196, 2888, 2308, 2202, 3652, 2052, 2772] combine_rows_per_src=[9586, 9016, 9680, 9446, 10432, 8792, 9890, 9596] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=268886016 dispatch_meta_bytes=337568 combine_payload_bytes=268886016 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=269223584 ingress_bytes=372695040 link_bytes=372695040 token_bytes=538109600 weight_movement_bytes=103809024 total_nvlink_bytes=641918624 weight_to_token_ratio=0.192914 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=8064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.402421431e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=270868480 dispatch_meta_bytes=343952 combine_payload_bytes=270868480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271212432 ingress_bytes=305471488 link_bytes=305471488 token_bytes=542080912 weight_movement_bytes=34603008 total_nvlink_bytes=576683920 weight_to_token_ratio=0.063834 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=9856 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.149464617e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75798 payload_rows_remote=66022 dup_rows=22506 dup_rows_remote=19916 dedup_ratio=0.228943 router_tokens_per_expert=[1105, 324, 806, 1212, 672, 691, 794, 1081, 1279, 327, 892, 342, 380, 1280, 987, 1118, 703, 1008, 864, 648, 680, 738, 378, 1473, 556, 691, 768, 355, 822, 90, 782, 724, 1013, 845, 1148, 841, 343, 206, 477, 909, 584, 592, 560, 857, 1096, 582, 734, 694, 794, 554, 875, 435, 1053, 1223, 998, 824, 1230, 281, 1226, 590, 219, 866, 1013, 920] payload_rows_per_dst=[9038, 9776, 10398, 9106, 8860, 10368, 9104, 9148] dup_rows_per_dst=[4466, 2590, 2586, 2894, 2704, 3476, 1962, 1828] combine_rows_per_src=[9038, 9776, 10398, 9106, 8860, 10368, 9104, 9148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=270327808 dispatch_meta_bytes=346256 combine_payload_bytes=270327808 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=270674064 ingress_bytes=339533824 link_bytes=339533824 token_bytes=541001872 weight_movement_bytes=69206016 total_nvlink_bytes=610207888 weight_to_token_ratio=0.127922 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=9728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.277638445e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=264380416 dispatch_meta_bytes=339944 combine_payload_bytes=264380416 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=264720360 ingress_bytes=333586432 link_bytes=333586432 token_bytes=529100776 weight_movement_bytes=69206016 total_nvlink_bytes=598306792 weight_to_token_ratio=0.130799 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=7936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.255258888e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76170 payload_rows_remote=68448 dup_rows=22134 dup_rows_remote=19804 dedup_ratio=0.225159 router_tokens_per_expert=[756, 1025, 968, 409, 798, 271, 206, 1079, 69, 946, 1120, 291, 569, 903, 1128, 1301, 1156, 1150, 713, 342, 1585, 129, 1821, 1151, 575, 653, 644, 761, 250, 391, 152, 58, 445, 281, 511, 252, 961, 1964, 1157, 1096, 2213, 1592, 1360, 1512, 469, 1213, 16, 218, 643, 1267, 685, 17, 176, 1789, 191, 1230, 673, 353, 891, 530, 1083, 315, 292, 387] payload_rows_per_dst=[10510, 7722, 9840, 10438, 10622, 7576, 9834, 9628] dup_rows_per_dst=[3116, 2330, 2758, 3964, 2712, 2176, 2162, 2916] combine_rows_per_src=[10510, 7722, 9840, 10438, 10622, 7576, 9834, 9628] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76452 payload_rows_remote=66994 dup_rows=21852 dup_rows_remote=19304 dedup_ratio=0.222290 router_tokens_per_expert=[729, 985, 925, 437, 923, 256, 310, 1179, 98, 928, 1412, 355, 577, 943, 1021, 1381, 966, 1161, 679, 358, 1396, 153, 1672, 1290, 631, 730, 642, 813, 295, 357, 192, 125, 440, 286, 500, 181, 874, 1731, 1225, 1011, 2075, 1414, 1177, 1326, 464, 1349, 37, 240, 891, 1120, 947, 16, 157, 1624, 196, 1163, 785, 380, 1000, 591, 890, 408, 299, 436] payload_rows_per_dst=[10936, 8018, 9458, 9498, 10172, 8410, 10078, 9882] dup_rows_per_dst=[3314, 2650, 2548, 2222, 2324, 3604, 2150, 3040] combine_rows_per_src=[10936, 8018, 9458, 9498, 10172, 8410, 10078, 9882] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277471232 dispatch_meta_bytes=343024 combine_payload_bytes=277471232 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=277814256 ingress_bytes=381280256 link_bytes=381280256 token_bytes=555285488 weight_movement_bytes=103809024 total_nvlink_bytes=659094512 weight_to_token_ratio=0.186947 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=11008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.434726908e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75940 payload_rows_remote=66446 dup_rows=22364 dup_rows_remote=19726 dedup_ratio=0.227498 router_tokens_per_expert=[778, 919, 975, 369, 786, 344, 406, 1025, 85, 1104, 1400, 349, 631, 1000, 913, 1389, 980, 1250, 742, 293, 1457, 158, 1773, 1076, 618, 612, 663, 822, 140, 274, 252, 108, 484, 352, 490, 280, 968, 1791, 1169, 1209, 2138, 1586, 1255, 1345, 435, 1215, 33, 269, 710, 1193, 680, 28, 216, 1733, 199, 1390, 718, 313, 757, 536, 1086, 296, 273, 314] payload_rows_per_dst=[8770, 10094, 8052, 9224, 9182, 10490, 10634, 9494] dup_rows_per_dst=[2434, 3648, 1368, 2030, 1814, 5368, 3064, 2638] combine_rows_per_src=[8770, 10094, 8052, 9224, 9182, 10490, 10634, 9494] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=274399232 dispatch_meta_bytes=346200 combine_payload_bytes=274399232 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=274745432 ingress_bytes=343605248 link_bytes=343605248 token_bytes=549144664 weight_movement_bytes=69206016 total_nvlink_bytes=618350680 weight_to_token_ratio=0.126025 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.030000 gemm_active_groups=11 gemm_peak_group_rows=16896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.292958886e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280363008 dispatch_meta_bytes=353008 combine_payload_bytes=280363008 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=280716016 ingress_bytes=349569024 link_bytes=349569024 token_bytes=561079024 weight_movement_bytes=69206016 total_nvlink_bytes=630285040 weight_to_token_ratio=0.123345 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=10880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.315400095e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=274407424 dispatch_meta_bytes=345192 combine_payload_bytes=274407424 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=274752616 ingress_bytes=412819456 link_bytes=412819456 token_bytes=549160040 weight_movement_bytes=138412032 total_nvlink_bytes=687572072 weight_to_token_ratio=0.252043 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=10240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.553406378e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=272162816 dispatch_meta_bytes=344688 combine_payload_bytes=272162816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272507504 ingress_bytes=306765824 link_bytes=306765824 token_bytes=544670320 weight_movement_bytes=34603008 total_nvlink_bytes=579273328 weight_to_token_ratio=0.063530 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.154335099e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=262717440 dispatch_meta_bytes=340416 combine_payload_bytes=262717440 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=263057856 ingress_bytes=366526464 link_bytes=366526464 token_bytes=525775296 weight_movement_bytes=103809024 total_nvlink_bytes=629584320 weight_to_token_ratio=0.197440 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=8960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.379209576e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=273522688 dispatch_meta_bytes=349264 combine_payload_bytes=273522688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273871952 ingress_bytes=308125696 link_bytes=308125696 token_bytes=547394640 weight_movement_bytes=34603008 total_nvlink_bytes=581997648 weight_to_token_ratio=0.063214 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=8832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.159452188e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=270426112 dispatch_meta_bytes=343752 combine_payload_bytes=270426112 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=270769864 ingress_bytes=339632128 link_bytes=339632128 token_bytes=541195976 weight_movement_bytes=69206016 total_nvlink_bytes=610401992 weight_to_token_ratio=0.127876 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.278008355e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75210 payload_rows_remote=65162 dup_rows=23094 dup_rows_remote=20354 dedup_ratio=0.234924 router_tokens_per_expert=[1093, 317, 749, 1191, 740, 831, 805, 1007, 1291, 316, 951, 389, 345, 1269, 920, 1038, 732, 1018, 867, 626, 705, 714, 418, 1314, 517, 675, 798, 400, 816, 135, 997, 596, 1111, 755, 1257, 934, 438, 197, 511, 874, 612, 618, 572, 769, 1145, 620, 673, 670, 704, 551, 794, 489, 1037, 1271, 851, 758, 1202, 312, 1340, 622, 232, 780, 1069, 804] payload_rows_per_dst=[9178, 8252, 10048, 9148, 9160, 10326, 8652, 10446] dup_rows_per_dst=[4488, 2204, 2740, 3102, 2994, 3574, 1716, 2276] combine_rows_per_src=[9178, 8252, 10048, 9148, 9160, 10326, 8652, 10446] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76324 payload_rows_remote=65330 dup_rows=21980 dup_rows_remote=18820 dedup_ratio=0.223592 router_tokens_per_expert=[650, 1003, 1031, 462, 922, 279, 308, 1053, 97, 1177, 1207, 329, 670, 1029, 995, 1369, 1027, 1252, 726, 191, 1425, 156, 1852, 1096, 639, 522, 592, 743, 194, 298, 219, 83, 463, 271, 539, 283, 852, 1901, 1044, 1151, 2223, 1590, 1200, 1410, 427, 1223, 26, 274, 767, 1183, 704, 38, 194, 1691, 175, 1262, 730, 375, 865, 551, 1144, 320, 332, 348] payload_rows_per_dst=[10994, 8592, 11116, 10476, 10380, 7122, 9968, 7676] dup_rows_per_dst=[3160, 3570, 3180, 3730, 2628, 1998, 2060, 1654] combine_rows_per_src=[10994, 8592, 11116, 10476, 10380, 7122, 9968, 7676] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76160 payload_rows_remote=66356 dup_rows=22144 dup_rows_remote=20008 dedup_ratio=0.225260 router_tokens_per_expert=[790, 947, 900, 462, 902, 315, 260, 1070, 57, 983, 1443, 374, 592, 976, 951, 1498, 971, 1148, 810, 268, 1479, 167, 1867, 1216, 665, 717, 749, 896, 184, 294, 218, 96, 342, 240, 419, 201, 948, 1601, 1174, 1045, 2136, 1503, 1018, 1294, 406, 1503, 28, 280, 943, 1018, 965, 14, 143, 1534, 172, 1255, 844, 362, 915, 574, 890, 402, 331, 387] payload_rows_per_dst=[8702, 10120, 9560, 9612, 9804, 8436, 9852, 10074] dup_rows_per_dst=[2590, 3628, 2558, 2298, 2136, 3628, 2236, 3070] combine_rows_per_src=[8702, 10120, 9560, 9612, 9804, 8436, 9852, 10074] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76434 payload_rows_remote=67262 dup_rows=21870 dup_rows_remote=18076 dedup_ratio=0.222473 router_tokens_per_expert=[172, 862, 479, 374, 78, 902, 1024, 224, 746, 270, 899, 1281, 1202, 1341, 773, 2047, 880, 1306, 301, 689, 97, 406, 1240, 1114, 300, 876, 932, 632, 370, 614, 803, 295, 405, 919, 686, 618, 1761, 1266, 1988, 466, 702, 196, 373, 296, 1676, 1410, 217, 1613, 1112, 683, 908, 912, 63, 253, 411, 763, 880, 266, 76, 697, 278, 1351, 487, 1891] payload_rows_per_dst=[9272, 8662, 9808, 9622, 9032, 9172, 10760, 10106] dup_rows_per_dst=[2934, 1800, 2258, 2584, 3210, 3794, 3544, 1746] combine_rows_per_src=[9272, 8662, 9808, 9622, 9032, 9172, 10760, 10106] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76836 payload_rows_remote=66676 dup_rows=21468 dup_rows_remote=19382 dedup_ratio=0.218384 router_tokens_per_expert=[706, 964, 927, 317, 880, 329, 326, 977, 60, 861, 1451, 371, 625, 922, 911, 1351, 829, 1242, 854, 379, 1593, 201, 2090, 1204, 693, 616, 685, 829, 189, 312, 168, 87, 492, 352, 429, 182, 893, 1699, 1248, 1127, 1932, 1599, 1139, 1284, 393, 1506, 38, 246, 839, 1097, 823, 18, 171, 1664, 246, 1265, 891, 424, 728, 512, 963, 367, 255, 381] payload_rows_per_dst=[8742, 9822, 9962, 9078, 9062, 9836, 10160, 10174] dup_rows_per_dst=[2110, 3282, 2642, 1944, 1688, 4668, 2086, 3048] combine_rows_per_src=[8742, 9822, 9962, 9078, 9062, 9836, 10160, 10174] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75832 payload_rows_remote=65842 dup_rows=22472 dup_rows_remote=19792 dedup_ratio=0.228597 router_tokens_per_expert=[150, 1002, 484, 384, 67, 1155, 974, 246, 697, 283, 909, 1404, 1395, 1198, 551, 1954, 939, 1376, 304, 667, 88, 289, 1184, 1085, 381, 864, 928, 710, 313, 629, 840, 266, 473, 1030, 752, 535, 1704, 1624, 1928, 480, 656, 152, 317, 273, 1457, 1574, 171, 1523, 1047, 437, 1028, 952, 61, 278, 315, 662, 942, 349, 80, 654, 174, 1453, 443, 1912] payload_rows_per_dst=[9554, 8290, 9702, 9990, 9540, 8592, 10188, 9976] dup_rows_per_dst=[3226, 1776, 2162, 2680, 3656, 3654, 3280, 2038] combine_rows_per_src=[9554, 8290, 9702, 9990, 9540, 8592, 10188, 9976] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75940 payload_rows_remote=66446 dup_rows=22364 dup_rows_remote=19726 dedup_ratio=0.227498 router_tokens_per_expert=[778, 919, 975, 369, 786, 344, 406, 1025, 85, 1104, 1400, 349, 631, 1000, 913, 1389, 980, 1250, 742, 293, 1457, 158, 1773, 1076, 618, 612, 663, 822, 140, 274, 252, 108, 484, 352, 490, 280, 968, 1791, 1169, 1209, 2138, 1586, 1255, 1345, 435, 1215, 33, 269, 710, 1193, 680, 28, 216, 1733, 199, 1390, 718, 313, 757, 536, 1086, 296, 273, 314] payload_rows_per_dst=[8770, 10094, 8052, 9224, 9182, 10490, 10634, 9494] dup_rows_per_dst=[2434, 3648, 1368, 2030, 1814, 5368, 3064, 2638] combine_rows_per_src=[8770, 10094, 8052, 9224, 9182, 10490, 10634, 9494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76438 payload_rows_remote=66992 dup_rows=21866 dup_rows_remote=19558 dedup_ratio=0.222432 router_tokens_per_expert=[689, 1071, 957, 409, 884, 303, 256, 1140, 79, 974, 1435, 317, 546, 1008, 899, 1330, 1011, 1185, 742, 317, 1555, 240, 1627, 1234, 648, 669, 696, 857, 271, 381, 251, 113, 373, 322, 436, 182, 903, 1812, 1213, 1076, 1991, 1611, 1101, 1353, 501, 1393, 36, 227, 728, 1156, 842, 19, 175, 1595, 229, 1227, 846, 344, 838, 559, 895, 395, 311, 369] payload_rows_per_dst=[9586, 9016, 9680, 9446, 10432, 8792, 9890, 9596] dup_rows_per_dst=[2796, 3196, 2888, 2308, 2202, 3652, 2052, 2772] combine_rows_per_src=[9586, 9016, 9680, 9446, 10432, 8792, 9890, 9596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76454 payload_rows_remote=67742 dup_rows=21850 dup_rows_remote=18014 dedup_ratio=0.222270 router_tokens_per_expert=[649, 1029, 953, 431, 762, 249, 147, 1052, 71, 962, 1333, 300, 560, 898, 914, 1270, 1039, 1137, 831, 480, 1466, 189, 1791, 1205, 648, 695, 634, 838, 309, 470, 202, 78, 485, 420, 458, 199, 834, 1793, 1237, 1147, 2144, 1617, 1175, 1443, 457, 1294, 29, 259, 784, 1043, 881, 14, 208, 1602, 198, 1309, 743, 360, 860, 526, 1044, 314, 296, 387] payload_rows_per_dst=[8458, 9626, 9964, 9550, 10524, 8712, 9878, 9742] dup_rows_per_dst=[2086, 2990, 2730, 2486, 2622, 3836, 2200, 2900] combine_rows_per_src=[8458, 9626, 9964, 9550, 10524, 8712, 9878, 9742] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75286 payload_rows_remote=63130 dup_rows=23018 dup_rows_remote=17344 dedup_ratio=0.234151 router_tokens_per_expert=[282, 786, 543, 327, 139, 1035, 1037, 261, 685, 281, 652, 1215, 1141, 1352, 977, 2266, 813, 1137, 243, 665, 89, 564, 1153, 844, 207, 868, 748, 497, 264, 628, 836, 354, 407, 1087, 787, 639, 1561, 1454, 2241, 548, 426, 209, 344, 174, 1731, 1514, 199, 1671, 1182, 634, 867, 801, 150, 446, 333, 899, 946, 345, 79, 726, 243, 1257, 575, 1788] payload_rows_per_dst=[9754, 12156, 10512, 9210, 7756, 7378, 8204, 10316] dup_rows_per_dst=[3548, 5674, 3036, 2024, 2088, 2626, 2420, 1602] combine_rows_per_src=[9754, 12156, 10512, 9210, 7756, 7378, 8204, 10316] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75092 payload_rows_remote=65776 dup_rows=23212 dup_rows_remote=21302 dedup_ratio=0.236125 router_tokens_per_expert=[117, 986, 599, 366, 85, 1088, 854, 192, 734, 327, 902, 1302, 1357, 1026, 392, 1585, 942, 1308, 294, 745, 81, 270, 1014, 959, 527, 995, 1024, 726, 335, 550, 844, 275, 435, 1077, 847, 580, 1861, 1796, 1957, 459, 648, 178, 369, 238, 1569, 1884, 172, 1567, 1011, 432, 892, 929, 108, 210, 276, 654, 1061, 514, 101, 570, 148, 1390, 397, 2021] payload_rows_per_dst=[9404, 9592, 9316, 10002, 9278, 9140, 8186, 10174] dup_rows_per_dst=[3084, 2978, 1910, 3154, 3484, 4110, 2262, 2230] combine_rows_per_src=[9404, 9592, 9316, 10002, 9278, 9140, 8186, 10174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267591680 dispatch_meta_bytes=336600 combine_payload_bytes=267591680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267928280 ingress_bytes=302194688 link_bytes=302194688 token_bytes=535519960 weight_movement_bytes=34603008 total_nvlink_bytes=570122968 weight_to_token_ratio=0.064616 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=8576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137134282e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=271794176 dispatch_meta_bytes=345456 combine_payload_bytes=271794176 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=272139632 ingress_bytes=341000192 link_bytes=341000192 token_bytes=543933808 weight_movement_bytes=69206016 total_nvlink_bytes=613139824 weight_to_token_ratio=0.127232 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.283156270e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=275505152 dispatch_meta_bytes=341352 combine_payload_bytes=275505152 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=275846504 ingress_bytes=344711168 link_bytes=344711168 token_bytes=551351656 weight_movement_bytes=69206016 total_nvlink_bytes=620557672 weight_to_token_ratio=0.125521 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.297120374e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=273104896 dispatch_meta_bytes=344232 combine_payload_bytes=273104896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273449128 ingress_bytes=307707904 link_bytes=307707904 token_bytes=546554024 weight_movement_bytes=34603008 total_nvlink_bytes=581157032 weight_to_token_ratio=0.063311 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.157880070e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=272162816 dispatch_meta_bytes=344688 combine_payload_bytes=272162816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272507504 ingress_bytes=306765824 link_bytes=306765824 token_bytes=544670320 weight_movement_bytes=34603008 total_nvlink_bytes=579273328 weight_to_token_ratio=0.063530 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.154335099e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=274399232 dispatch_meta_bytes=346200 combine_payload_bytes=274399232 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=274745432 ingress_bytes=343605248 link_bytes=343605248 token_bytes=549144664 weight_movement_bytes=69206016 total_nvlink_bytes=618350680 weight_to_token_ratio=0.126025 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.030000 gemm_active_groups=11 gemm_peak_group_rows=16896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.292958886e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277471232 dispatch_meta_bytes=343024 combine_payload_bytes=277471232 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=277814256 ingress_bytes=381280256 link_bytes=381280256 token_bytes=555285488 weight_movement_bytes=103809024 total_nvlink_bytes=659094512 weight_to_token_ratio=0.186947 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=11008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.434726908e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266903552 dispatch_meta_bytes=342064 combine_payload_bytes=266903552 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267245616 ingress_bytes=336109568 link_bytes=336109568 token_bytes=534149168 weight_movement_bytes=69206016 total_nvlink_bytes=603355184 weight_to_token_ratio=0.129563 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=9216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264753245e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76170 payload_rows_remote=68448 dup_rows=22134 dup_rows_remote=19804 dedup_ratio=0.225159 router_tokens_per_expert=[756, 1025, 968, 409, 798, 271, 206, 1079, 69, 946, 1120, 291, 569, 903, 1128, 1301, 1156, 1150, 713, 342, 1585, 129, 1821, 1151, 575, 653, 644, 761, 250, 391, 152, 58, 445, 281, 511, 252, 961, 1964, 1157, 1096, 2213, 1592, 1360, 1512, 469, 1213, 16, 218, 643, 1267, 685, 17, 176, 1789, 191, 1230, 673, 353, 891, 530, 1083, 315, 292, 387] payload_rows_per_dst=[10510, 7722, 9840, 10438, 10622, 7576, 9834, 9628] dup_rows_per_dst=[3116, 2330, 2758, 3964, 2712, 2176, 2162, 2916] combine_rows_per_src=[10510, 7722, 9840, 10438, 10622, 7576, 9834, 9628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280363008 dispatch_meta_bytes=353008 combine_payload_bytes=280363008 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=280716016 ingress_bytes=349569024 link_bytes=349569024 token_bytes=561079024 weight_movement_bytes=69206016 total_nvlink_bytes=630285040 weight_to_token_ratio=0.123345 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=10880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.315400095e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75052 payload_rows_remote=65168 dup_rows=23252 dup_rows_remote=19814 dedup_ratio=0.236532 router_tokens_per_expert=[188, 900, 575, 338, 100, 1139, 892, 248, 674, 272, 760, 1261, 1230, 1420, 781, 2057, 895, 1240, 254, 737, 70, 387, 1086, 950, 261, 871, 769, 515, 260, 633, 768, 428, 380, 1094, 759, 697, 1702, 1571, 2281, 645, 469, 185, 376, 211, 1666, 1533, 163, 1629, 959, 692, 878, 951, 92, 266, 238, 841, 914, 365, 91, 733, 131, 1254, 473, 1954] payload_rows_per_dst=[9884, 12282, 11070, 9156, 8066, 6574, 7902, 10118] dup_rows_per_dst=[3438, 5852, 3426, 2034, 2226, 2632, 1932, 1712] combine_rows_per_src=[9884, 12282, 11070, 9156, 8066, 6574, 7902, 10118] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74676 payload_rows_remote=64516 dup_rows=23628 dup_rows_remote=19026 dedup_ratio=0.240356 router_tokens_per_expert=[57, 917, 523, 344, 38, 1072, 908, 220, 570, 363, 780, 1471, 1528, 1027, 209, 1714, 916, 1587, 316, 721, 97, 181, 1184, 908, 519, 914, 892, 770, 249, 622, 892, 204, 473, 1057, 773, 557, 2132, 1873, 1895, 516, 643, 109, 278, 358, 1518, 1761, 159, 1641, 1050, 147, 795, 1163, 33, 191, 220, 710, 1072, 516, 75, 574, 69, 1476, 305, 2300] payload_rows_per_dst=[9228, 7162, 9642, 10216, 10160, 8986, 9200, 10082] dup_rows_per_dst=[2720, 1792, 2178, 2850, 4602, 3948, 2846, 2692] combine_rows_per_src=[9228, 7162, 9642, 10216, 10160, 8986, 9200, 10082] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76542 payload_rows_remote=66362 dup_rows=21762 dup_rows_remote=18688 dedup_ratio=0.221375 router_tokens_per_expert=[150, 938, 424, 426, 58, 897, 932, 273, 531, 311, 865, 1282, 1184, 1335, 649, 1895, 880, 1291, 376, 737, 85, 314, 1192, 1050, 300, 951, 976, 782, 397, 662, 926, 242, 403, 1028, 614, 506, 1648, 1503, 1853, 528, 702, 153, 397, 298, 1705, 1687, 161, 1617, 1263, 645, 599, 934, 62, 266, 314, 649, 982, 304, 116, 861, 135, 1523, 489, 1896] payload_rows_per_dst=[9216, 8348, 9650, 10090, 9170, 9352, 10180, 10536] dup_rows_per_dst=[2686, 1402, 2200, 2946, 3290, 4088, 3074, 2076] combine_rows_per_src=[9216, 8348, 9650, 10090, 9170, 9352, 10180, 10536] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76308 payload_rows_remote=65594 dup_rows=21996 dup_rows_remote=19810 dedup_ratio=0.223755 router_tokens_per_expert=[297, 871, 431, 390, 121, 1151, 1155, 274, 651, 275, 814, 1104, 1129, 1498, 902, 2197, 869, 1149, 313, 696, 72, 452, 1153, 1013, 231, 886, 769, 525, 294, 704, 885, 475, 465, 907, 694, 646, 1589, 1344, 1986, 524, 500, 188, 296, 257, 1512, 1526, 185, 1790, 1083, 777, 972, 735, 98, 248, 270, 783, 955, 289, 123, 879, 151, 1312, 600, 1722] payload_rows_per_dst=[8098, 8858, 9476, 9514, 9192, 9740, 10716, 10714] dup_rows_per_dst=[2258, 1680, 1958, 2232, 3554, 4518, 3610, 2186] combine_rows_per_src=[8098, 8858, 9476, 9514, 9192, 9740, 10716, 10714] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76434 payload_rows_remote=67262 dup_rows=21870 dup_rows_remote=18076 dedup_ratio=0.222473 router_tokens_per_expert=[172, 862, 479, 374, 78, 902, 1024, 224, 746, 270, 899, 1281, 1202, 1341, 773, 2047, 880, 1306, 301, 689, 97, 406, 1240, 1114, 300, 876, 932, 632, 370, 614, 803, 295, 405, 919, 686, 618, 1761, 1266, 1988, 466, 702, 196, 373, 296, 1676, 1410, 217, 1613, 1112, 683, 908, 912, 63, 253, 411, 763, 880, 266, 76, 697, 278, 1351, 487, 1891] payload_rows_per_dst=[9272, 8662, 9808, 9622, 9032, 9172, 10760, 10106] dup_rows_per_dst=[2934, 1800, 2258, 2584, 3210, 3794, 3544, 1746] combine_rows_per_src=[9272, 8662, 9808, 9622, 9032, 9172, 10760, 10106] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75832 payload_rows_remote=65842 dup_rows=22472 dup_rows_remote=19792 dedup_ratio=0.228597 router_tokens_per_expert=[150, 1002, 484, 384, 67, 1155, 974, 246, 697, 283, 909, 1404, 1395, 1198, 551, 1954, 939, 1376, 304, 667, 88, 289, 1184, 1085, 381, 864, 928, 710, 313, 629, 840, 266, 473, 1030, 752, 535, 1704, 1624, 1928, 480, 656, 152, 317, 273, 1457, 1574, 171, 1523, 1047, 437, 1028, 952, 61, 278, 315, 662, 942, 349, 80, 654, 174, 1453, 443, 1912] payload_rows_per_dst=[9554, 8290, 9702, 9990, 9540, 8592, 10188, 9976] dup_rows_per_dst=[3226, 1776, 2162, 2680, 3656, 3654, 3280, 2038] combine_rows_per_src=[9554, 8290, 9702, 9990, 9540, 8592, 10188, 9976] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266928128 dispatch_meta_bytes=339928 combine_payload_bytes=266928128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267268056 ingress_bytes=301531136 link_bytes=301531136 token_bytes=534196184 weight_movement_bytes=34603008 total_nvlink_bytes=568799192 weight_to_token_ratio=0.064776 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134637390e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=264257536 dispatch_meta_bytes=334168 combine_payload_bytes=264257536 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=264591704 ingress_bytes=402669568 link_bytes=402669568 token_bytes=528849240 weight_movement_bytes=138412032 total_nvlink_bytes=667261272 weight_to_token_ratio=0.261723 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=11264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.515213167e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=271818752 dispatch_meta_bytes=340200 combine_payload_bytes=271818752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272158952 ingress_bytes=306421760 link_bytes=306421760 token_bytes=543977704 weight_movement_bytes=34603008 total_nvlink_bytes=578580712 weight_to_token_ratio=0.063611 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.153040414e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=268673024 dispatch_meta_bytes=341616 combine_payload_bytes=268673024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269014640 ingress_bytes=303276032 link_bytes=303276032 token_bytes=537687664 weight_movement_bytes=34603008 total_nvlink_bytes=572290672 weight_to_token_ratio=0.064355 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.141203293e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76452 payload_rows_remote=66994 dup_rows=21852 dup_rows_remote=19304 dedup_ratio=0.222290 router_tokens_per_expert=[729, 985, 925, 437, 923, 256, 310, 1179, 98, 928, 1412, 355, 577, 943, 1021, 1381, 966, 1161, 679, 358, 1396, 153, 1672, 1290, 631, 730, 642, 813, 295, 357, 192, 125, 440, 286, 500, 181, 874, 1731, 1225, 1011, 2075, 1414, 1177, 1326, 464, 1349, 37, 240, 891, 1120, 947, 16, 157, 1624, 196, 1163, 785, 380, 1000, 591, 890, 408, 299, 436] payload_rows_per_dst=[10936, 8018, 9458, 9498, 10172, 8410, 10078, 9882] dup_rows_per_dst=[3314, 2650, 2548, 2222, 2324, 3604, 2150, 3040] combine_rows_per_src=[10936, 8018, 9458, 9498, 10172, 8410, 10078, 9882] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=275505152 dispatch_meta_bytes=341352 combine_payload_bytes=275505152 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=275846504 ingress_bytes=344711168 link_bytes=344711168 token_bytes=551351656 weight_movement_bytes=69206016 total_nvlink_bytes=620557672 weight_to_token_ratio=0.125521 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.297120374e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=269688832 dispatch_meta_bytes=342536 combine_payload_bytes=269688832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270031368 ingress_bytes=304291840 link_bytes=304291840 token_bytes=539720200 weight_movement_bytes=34603008 total_nvlink_bytes=574323208 weight_to_token_ratio=0.064113 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.145025696e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75286 payload_rows_remote=63130 dup_rows=23018 dup_rows_remote=17344 dedup_ratio=0.234151 router_tokens_per_expert=[282, 786, 543, 327, 139, 1035, 1037, 261, 685, 281, 652, 1215, 1141, 1352, 977, 2266, 813, 1137, 243, 665, 89, 564, 1153, 844, 207, 868, 748, 497, 264, 628, 836, 354, 407, 1087, 787, 639, 1561, 1454, 2241, 548, 426, 209, 344, 174, 1731, 1514, 199, 1671, 1182, 634, 867, 801, 150, 446, 333, 899, 946, 345, 79, 726, 243, 1257, 575, 1788] payload_rows_per_dst=[9754, 12156, 10512, 9210, 7756, 7378, 8204, 10316] dup_rows_per_dst=[3548, 5674, 3036, 2024, 2088, 2626, 2420, 1602] combine_rows_per_src=[9754, 12156, 10512, 9210, 7756, 7378, 8204, 10316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=274407424 dispatch_meta_bytes=345192 combine_payload_bytes=274407424 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=274752616 ingress_bytes=412819456 link_bytes=412819456 token_bytes=549160040 weight_movement_bytes=138412032 total_nvlink_bytes=687572072 weight_to_token_ratio=0.252043 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=10240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.553406378e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75780 payload_rows_remote=65296 dup_rows=22524 dup_rows_remote=18946 dedup_ratio=0.229126 router_tokens_per_expert=[557, 1045, 1173, 491, 362, 337, 965, 595, 1158, 2491, 218, 413, 830, 559, 970, 460, 1686, 51, 2249, 558, 671, 396, 30, 645, 1001, 756, 636, 595, 539, 820, 1506, 916, 554, 712, 181, 280, 893, 540, 1900, 721, 639, 1977, 168, 503, 344, 695, 682, 3520, 291, 367, 299, 924, 710, 1145, 279, 430, 177, 373, 1517, 613, 848, 693, 270, 228] payload_rows_per_dst=[10484, 12512, 8834, 8512, 9326, 10938, 7222, 7952] dup_rows_per_dst=[3578, 5768, 1932, 2014, 2236, 3842, 1668, 1486] combine_rows_per_src=[10484, 12512, 8834, 8512, 9326, 10938, 7222, 7952] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=258580480 dispatch_meta_bytes=321896 combine_payload_bytes=258580480 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=258902376 ingress_bytes=362389504 link_bytes=362389504 token_bytes=517482856 weight_movement_bytes=103809024 total_nvlink_bytes=621291880 weight_to_token_ratio=0.200604 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.363642529e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=269418496 dispatch_meta_bytes=348312 combine_payload_bytes=269418496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269766808 ingress_bytes=304021504 link_bytes=304021504 token_bytes=539185304 weight_movement_bytes=34603008 total_nvlink_bytes=573788312 weight_to_token_ratio=0.064176 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.144008444e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=268673024 dispatch_meta_bytes=341616 combine_payload_bytes=268673024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269014640 ingress_bytes=303276032 link_bytes=303276032 token_bytes=537687664 weight_movement_bytes=34603008 total_nvlink_bytes=572290672 weight_to_token_ratio=0.064355 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.141203293e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75052 payload_rows_remote=65168 dup_rows=23252 dup_rows_remote=19814 dedup_ratio=0.236532 router_tokens_per_expert=[188, 900, 575, 338, 100, 1139, 892, 248, 674, 272, 760, 1261, 1230, 1420, 781, 2057, 895, 1240, 254, 737, 70, 387, 1086, 950, 261, 871, 769, 515, 260, 633, 768, 428, 380, 1094, 759, 697, 1702, 1571, 2281, 645, 469, 185, 376, 211, 1666, 1533, 163, 1629, 959, 692, 878, 951, 92, 266, 238, 841, 914, 365, 91, 733, 131, 1254, 473, 1954] payload_rows_per_dst=[9884, 12282, 11070, 9156, 8066, 6574, 7902, 10118] dup_rows_per_dst=[3438, 5852, 3426, 2034, 2226, 2632, 1932, 1712] combine_rows_per_src=[9884, 12282, 11070, 9156, 8066, 6574, 7902, 10118] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77208 payload_rows_remote=68678 dup_rows=21096 dup_rows_remote=19336 dedup_ratio=0.214600 router_tokens_per_expert=[214, 481, 1547, 268, 528, 561, 1129, 759, 2158, 3027, 304, 300, 1038, 612, 1422, 452, 2044, 40, 2556, 428, 812, 446, 5, 554, 974, 968, 654, 435, 517, 1070, 1433, 725, 592, 549, 165, 173, 1050, 411, 1491, 714, 370, 2487, 119, 491, 243, 475, 405, 3059, 25, 234, 230, 1162, 478, 869, 111, 165, 192, 298, 1722, 636, 648, 737, 153, 237] payload_rows_per_dst=[8936, 10154, 9876, 10440, 8530, 7334, 10446, 11492] dup_rows_per_dst=[2038, 2418, 3894, 3112, 1760, 1846, 2220, 3808] combine_rows_per_src=[8936, 10154, 9876, 10440, 8530, 7334, 10446, 11492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75780 payload_rows_remote=64184 dup_rows=22524 dup_rows_remote=19348 dedup_ratio=0.229126 router_tokens_per_expert=[361, 878, 1338, 495, 478, 377, 959, 541, 1214, 2471, 188, 427, 779, 662, 1102, 452, 2008, 76, 2225, 454, 625, 435, 19, 628, 1107, 720, 613, 542, 557, 1073, 1732, 982, 571, 653, 190, 256, 961, 546, 2118, 781, 476, 1722, 176, 505, 336, 661, 590, 3190, 187, 404, 274, 1017, 829, 891, 203, 391, 255, 456, 1335, 574, 785, 730, 254, 317] payload_rows_per_dst=[8850, 6490, 9336, 11064, 10652, 6984, 11596, 10808] dup_rows_per_dst=[2004, 730, 3604, 3588, 3928, 1948, 3176, 3546] combine_rows_per_src=[8850, 6490, 9336, 11064, 10652, 6984, 11596, 10808] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75952 payload_rows_remote=68552 dup_rows=22352 dup_rows_remote=20298 dedup_ratio=0.227376 router_tokens_per_expert=[373, 851, 1220, 480, 404, 436, 996, 657, 1625, 2505, 236, 350, 693, 713, 1079, 432, 1618, 63, 2204, 519, 605, 373, 9, 527, 774, 729, 621, 521, 656, 945, 1462, 1137, 770, 691, 262, 281, 866, 593, 1872, 722, 420, 1963, 137, 511, 378, 638, 680, 3103, 257, 402, 344, 987, 608, 997, 451, 632, 241, 527, 1400, 598, 723, 702, 276, 307] payload_rows_per_dst=[8804, 7040, 8738, 10598, 10664, 7400, 11830, 10878] dup_rows_per_dst=[2030, 844, 3098, 3092, 3822, 2054, 3732, 3680] combine_rows_per_src=[8804, 7040, 8738, 10598, 10664, 7400, 11830, 10878] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74676 payload_rows_remote=64516 dup_rows=23628 dup_rows_remote=19026 dedup_ratio=0.240356 router_tokens_per_expert=[57, 917, 523, 344, 38, 1072, 908, 220, 570, 363, 780, 1471, 1528, 1027, 209, 1714, 916, 1587, 316, 721, 97, 181, 1184, 908, 519, 914, 892, 770, 249, 622, 892, 204, 473, 1057, 773, 557, 2132, 1873, 1895, 516, 643, 109, 278, 358, 1518, 1761, 159, 1641, 1050, 147, 795, 1163, 33, 191, 220, 710, 1072, 516, 75, 574, 69, 1476, 305, 2300] payload_rows_per_dst=[9228, 7162, 9642, 10216, 10160, 8986, 9200, 10082] dup_rows_per_dst=[2720, 1792, 2178, 2850, 4602, 3948, 2846, 2692] combine_rows_per_src=[9228, 7162, 9642, 10216, 10160, 8986, 9200, 10082] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76542 payload_rows_remote=66362 dup_rows=21762 dup_rows_remote=18688 dedup_ratio=0.221375 router_tokens_per_expert=[150, 938, 424, 426, 58, 897, 932, 273, 531, 311, 865, 1282, 1184, 1335, 649, 1895, 880, 1291, 376, 737, 85, 314, 1192, 1050, 300, 951, 976, 782, 397, 662, 926, 242, 403, 1028, 614, 506, 1648, 1503, 1853, 528, 702, 153, 397, 298, 1705, 1687, 161, 1617, 1263, 645, 599, 934, 62, 266, 314, 649, 982, 304, 116, 861, 135, 1523, 489, 1896] payload_rows_per_dst=[9216, 8348, 9650, 10090, 9170, 9352, 10180, 10536] dup_rows_per_dst=[2686, 1402, 2200, 2946, 3290, 4088, 3074, 2076] combine_rows_per_src=[9216, 8348, 9650, 10090, 9170, 9352, 10180, 10536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74852 payload_rows_remote=64206 dup_rows=23452 dup_rows_remote=19924 dedup_ratio=0.238566 router_tokens_per_expert=[443, 982, 1048, 571, 387, 334, 1007, 662, 1307, 2285, 225, 315, 662, 566, 1012, 391, 1721, 77, 2019, 542, 655, 344, 48, 623, 598, 598, 682, 862, 758, 928, 1552, 1009, 705, 624, 213, 283, 718, 719, 1836, 830, 522, 1834, 170, 564, 443, 799, 747, 3089, 343, 466, 416, 913, 870, 1067, 334, 632, 175, 481, 1452, 605, 870, 644, 362, 213] payload_rows_per_dst=[8688, 5742, 8882, 9934, 10414, 8386, 12160, 10646] dup_rows_per_dst=[2180, 600, 3176, 3286, 4056, 2526, 4100, 3528] combine_rows_per_src=[8688, 5742, 8882, 9934, 10414, 8386, 12160, 10646] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267452416 dispatch_meta_bytes=336968 combine_payload_bytes=267452416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267789384 ingress_bytes=302055424 link_bytes=302055424 token_bytes=535241800 weight_movement_bytes=34603008 total_nvlink_bytes=569844808 weight_to_token_ratio=0.064649 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=10624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136610243e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76296 payload_rows_remote=66208 dup_rows=22008 dup_rows_remote=19348 dedup_ratio=0.223877 router_tokens_per_expert=[353, 656, 1453, 267, 525, 445, 1049, 848, 1577, 2844, 283, 378, 845, 682, 1214, 369, 1756, 120, 2197, 553, 850, 343, 10, 654, 927, 817, 651, 444, 570, 1048, 1408, 876, 686, 669, 173, 265, 922, 541, 1506, 725, 510, 2234, 161, 460, 344, 626, 391, 3201, 112, 338, 245, 1130, 524, 1028, 249, 342, 234, 372, 1623, 596, 729, 769, 211, 224] payload_rows_per_dst=[9430, 10840, 9280, 10088, 8962, 8750, 9634, 9312] dup_rows_per_dst=[2496, 3420, 3686, 2660, 2012, 2974, 2432, 2328] combine_rows_per_src=[9430, 10840, 9280, 10088, 8962, 8750, 9634, 9312] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=281305088 dispatch_meta_bytes=352056 combine_payload_bytes=281305088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281657144 ingress_bytes=315908096 link_bytes=315908096 token_bytes=562962232 weight_movement_bytes=34603008 total_nvlink_bytes=597565240 weight_to_token_ratio=0.061466 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.188736733e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=262897664 dispatch_meta_bytes=334128 combine_payload_bytes=262897664 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263231792 ingress_bytes=297500672 link_bytes=297500672 token_bytes=526129456 weight_movement_bytes=34603008 total_nvlink_bytes=560732464 weight_to_token_ratio=0.065769 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.119471078e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74836 payload_rows_remote=63764 dup_rows=23468 dup_rows_remote=20736 dedup_ratio=0.238729 router_tokens_per_expert=[548, 1207, 1124, 483, 352, 325, 915, 545, 1258, 2420, 195, 351, 662, 649, 945, 422, 1604, 46, 1927, 471, 625, 287, 19, 561, 688, 476, 644, 704, 593, 695, 1411, 1092, 566, 776, 281, 274, 735, 664, 1959, 803, 724, 1910, 154, 561, 354, 965, 934, 3011, 585, 433, 517, 802, 642, 1129, 577, 829, 198, 568, 1386, 642, 767, 595, 333, 234] payload_rows_per_dst=[10288, 11072, 8272, 8168, 9438, 11326, 8376, 7896] dup_rows_per_dst=[3532, 2732, 2808, 1616, 2678, 5900, 2652, 1550] combine_rows_per_src=[10288, 11072, 8272, 8168, 9438, 11326, 8376, 7896] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75216 payload_rows_remote=65776 dup_rows=23088 dup_rows_remote=19640 dedup_ratio=0.234863 router_tokens_per_expert=[328, 581, 1699, 231, 501, 693, 1166, 882, 2005, 2902, 367, 334, 1035, 558, 1472, 298, 1685, 59, 2336, 503, 850, 376, 13, 622, 799, 1066, 607, 416, 407, 1036, 1402, 735, 615, 631, 175, 189, 915, 483, 1228, 677, 462, 2697, 158, 467, 251, 542, 384, 3087, 54, 271, 271, 1120, 445, 922, 194, 253, 167, 335, 1807, 580, 645, 775, 143, 245] payload_rows_per_dst=[11128, 12762, 9440, 8560, 8266, 11224, 5786, 8050] dup_rows_per_dst=[3838, 5180, 3448, 1572, 1560, 4872, 1274, 1344] combine_rows_per_src=[11128, 12762, 9440, 8560, 8266, 11224, 5786, 8050] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266928128 dispatch_meta_bytes=339928 combine_payload_bytes=266928128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267268056 ingress_bytes=301531136 link_bytes=301531136 token_bytes=534196184 weight_movement_bytes=34603008 total_nvlink_bytes=568799192 weight_to_token_ratio=0.064776 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134637390e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=280788992 dispatch_meta_bytes=355400 combine_payload_bytes=280788992 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=281144392 ingress_bytes=384598016 link_bytes=384598016 token_bytes=561933384 weight_movement_bytes=103809024 total_nvlink_bytes=665742408 weight_to_token_ratio=0.184735 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.447211372e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=271818752 dispatch_meta_bytes=340200 combine_payload_bytes=271818752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272158952 ingress_bytes=306421760 link_bytes=306421760 token_bytes=543977704 weight_movement_bytes=34603008 total_nvlink_bytes=578580712 weight_to_token_ratio=0.063611 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.153040414e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74852 payload_rows_remote=64206 dup_rows=23452 dup_rows_remote=19924 dedup_ratio=0.238566 router_tokens_per_expert=[443, 982, 1048, 571, 387, 334, 1007, 662, 1307, 2285, 225, 315, 662, 566, 1012, 391, 1721, 77, 2019, 542, 655, 344, 48, 623, 598, 598, 682, 862, 758, 928, 1552, 1009, 705, 624, 213, 283, 718, 719, 1836, 830, 522, 1834, 170, 564, 443, 799, 747, 3089, 343, 466, 416, 913, 870, 1067, 334, 632, 175, 481, 1452, 605, 870, 644, 362, 213] payload_rows_per_dst=[8688, 5742, 8882, 9934, 10414, 8386, 12160, 10646] dup_rows_per_dst=[2180, 600, 3176, 3286, 4056, 2526, 4100, 3528] combine_rows_per_src=[8688, 5742, 8882, 9934, 10414, 8386, 12160, 10646] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=262987776 dispatch_meta_bytes=336520 combine_payload_bytes=262987776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263324296 ingress_bytes=297590784 link_bytes=297590784 token_bytes=526312072 weight_movement_bytes=34603008 total_nvlink_bytes=560915080 weight_to_token_ratio=0.065746 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=12288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.119810162e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75092 payload_rows_remote=65776 dup_rows=23212 dup_rows_remote=21302 dedup_ratio=0.236125 router_tokens_per_expert=[117, 986, 599, 366, 85, 1088, 854, 192, 734, 327, 902, 1302, 1357, 1026, 392, 1585, 942, 1308, 294, 745, 81, 270, 1014, 959, 527, 995, 1024, 726, 335, 550, 844, 275, 435, 1077, 847, 580, 1861, 1796, 1957, 459, 648, 178, 369, 238, 1569, 1884, 172, 1567, 1011, 432, 892, 929, 108, 210, 276, 654, 1061, 514, 101, 570, 148, 1390, 397, 2021] payload_rows_per_dst=[9404, 9592, 9316, 10002, 9278, 9140, 8186, 10174] dup_rows_per_dst=[3084, 2978, 1910, 3154, 3484, 4110, 2262, 2230] combine_rows_per_src=[9404, 9592, 9316, 10002, 9278, 9140, 8186, 10174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74836 payload_rows_remote=63764 dup_rows=23468 dup_rows_remote=20736 dedup_ratio=0.238729 router_tokens_per_expert=[548, 1207, 1124, 483, 352, 325, 915, 545, 1258, 2420, 195, 351, 662, 649, 945, 422, 1604, 46, 1927, 471, 625, 287, 19, 561, 688, 476, 644, 704, 593, 695, 1411, 1092, 566, 776, 281, 274, 735, 664, 1959, 803, 724, 1910, 154, 561, 354, 965, 934, 3011, 585, 433, 517, 802, 642, 1129, 577, 829, 198, 568, 1386, 642, 767, 595, 333, 234] payload_rows_per_dst=[10288, 11072, 8272, 8168, 9438, 11326, 8376, 7896] dup_rows_per_dst=[3532, 2732, 2808, 1616, 2678, 5900, 2652, 1550] combine_rows_per_src=[10288, 11072, 8272, 8168, 9438, 11326, 8376, 7896] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=271187968 dispatch_meta_bytes=342224 combine_payload_bytes=271187968 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=271530192 ingress_bytes=340393984 link_bytes=340393984 token_bytes=542718160 weight_movement_bytes=69206016 total_nvlink_bytes=611924176 weight_to_token_ratio=0.127517 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=7552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.280875158e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=264257536 dispatch_meta_bytes=334168 combine_payload_bytes=264257536 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=264591704 ingress_bytes=402669568 link_bytes=402669568 token_bytes=528849240 weight_movement_bytes=138412032 total_nvlink_bytes=667261272 weight_to_token_ratio=0.261723 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=11264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.515213167e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73468 payload_rows_remote=63678 dup_rows=24836 dup_rows_remote=20010 dedup_ratio=0.252645 router_tokens_per_expert=[11, 1922, 1576, 693, 975, 1172, 920, 795, 555, 494, 1656, 92, 353, 277, 359, 1567, 1873, 462, 138, 165, 340, 415, 166, 170, 694, 390, 1804, 591, 1464, 972, 1144, 151, 382, 44, 192, 333, 1716, 768, 315, 1017, 153, 351, 779, 900, 848, 21, 2282, 16, 406, 510, 270, 1176, 539, 1014, 762, 1173, 974, 748, 194, 1166, 983, 1688, 1882, 1194] payload_rows_per_dst=[9790, 10300, 10906, 8460, 9814, 8940, 8842, 6416] dup_rows_per_dst=[4826, 4014, 3692, 2352, 3564, 1760, 2858, 1770] combine_rows_per_src=[9790, 10300, 10906, 8460, 9814, 8940, 8842, 6416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=269418496 dispatch_meta_bytes=348312 combine_payload_bytes=269418496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269766808 ingress_bytes=304021504 link_bytes=304021504 token_bytes=539185304 weight_movement_bytes=34603008 total_nvlink_bytes=573788312 weight_to_token_ratio=0.064176 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.144008444e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=261177344 dispatch_meta_bytes=338000 combine_payload_bytes=261177344 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=261515344 ingress_bytes=364986368 link_bytes=364986368 token_bytes=522692688 weight_movement_bytes=103809024 total_nvlink_bytes=626501712 weight_to_token_ratio=0.198604 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=9600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.373414319e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=269418496 dispatch_meta_bytes=341664 combine_payload_bytes=269418496 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=269760160 ingress_bytes=338624512 link_bytes=338624512 token_bytes=539178656 weight_movement_bytes=69206016 total_nvlink_bytes=608384672 weight_to_token_ratio=0.128355 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274216777e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=262987776 dispatch_meta_bytes=336520 combine_payload_bytes=262987776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263324296 ingress_bytes=297590784 link_bytes=297590784 token_bytes=526312072 weight_movement_bytes=34603008 total_nvlink_bytes=560915080 weight_to_token_ratio=0.065746 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=12288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.119810162e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75780 payload_rows_remote=65296 dup_rows=22524 dup_rows_remote=18946 dedup_ratio=0.229126 router_tokens_per_expert=[557, 1045, 1173, 491, 362, 337, 965, 595, 1158, 2491, 218, 413, 830, 559, 970, 460, 1686, 51, 2249, 558, 671, 396, 30, 645, 1001, 756, 636, 595, 539, 820, 1506, 916, 554, 712, 181, 280, 893, 540, 1900, 721, 639, 1977, 168, 503, 344, 695, 682, 3520, 291, 367, 299, 924, 710, 1145, 279, 430, 177, 373, 1517, 613, 848, 693, 270, 228] payload_rows_per_dst=[10484, 12512, 8834, 8512, 9326, 10938, 7222, 7952] dup_rows_per_dst=[3578, 5768, 1932, 2014, 2236, 3842, 1668, 1486] combine_rows_per_src=[10484, 12512, 8834, 8512, 9326, 10938, 7222, 7952] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74634 payload_rows_remote=67436 dup_rows=23670 dup_rows_remote=21428 dedup_ratio=0.240784 router_tokens_per_expert=[0, 1368, 1449, 1053, 887, 1440, 675, 1329, 303, 803, 2483, 70, 176, 319, 250, 1039, 2092, 394, 17, 61, 135, 738, 290, 161, 598, 184, 2565, 335, 2834, 713, 698, 0, 273, 0, 7, 659, 1134, 127, 104, 1048, 71, 285, 28, 980, 489, 0, 3212, 0, 16, 887, 290, 817, 454, 1630, 1041, 1122, 583, 960, 50, 1545, 1216, 1637, 1664, 1364] payload_rows_per_dst=[9194, 8952, 9390, 9842, 7198, 10520, 9284, 10254] dup_rows_per_dst=[4472, 1934, 1714, 3654, 2242, 1968, 3230, 4456] combine_rows_per_src=[9194, 8952, 9390, 9842, 7198, 10520, 9284, 10254] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75110 payload_rows_remote=66030 dup_rows=23194 dup_rows_remote=20306 dedup_ratio=0.235942 router_tokens_per_expert=[17, 1976, 1459, 395, 1006, 1625, 891, 851, 561, 360, 1597, 154, 336, 340, 410, 1419, 1940, 443, 163, 206, 368, 240, 256, 204, 407, 369, 2059, 606, 1925, 1172, 1109, 101, 323, 43, 97, 497, 1328, 646, 315, 1105, 146, 408, 876, 943, 619, 7, 2104, 7, 388, 535, 217, 1156, 540, 1080, 788, 1280, 894, 737, 169, 1361, 969, 1819, 1691, 1099] payload_rows_per_dst=[7914, 8272, 9372, 10710, 9476, 10552, 9080, 9734] dup_rows_per_dst=[1656, 2082, 1650, 3854, 3184, 3518, 2888, 4362] combine_rows_per_src=[7914, 8272, 9372, 10710, 9476, 10552, 9080, 9734] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73892 payload_rows_remote=63440 dup_rows=24412 dup_rows_remote=21234 dedup_ratio=0.248332 router_tokens_per_expert=[8, 1820, 1489, 520, 1142, 1198, 920, 855, 347, 443, 1676, 114, 321, 299, 377, 1597, 1581, 419, 256, 179, 445, 270, 210, 227, 549, 373, 2211, 591, 1543, 1003, 968, 96, 406, 32, 125, 395, 1639, 693, 301, 1343, 160, 285, 742, 1056, 837, 23, 2159, 10, 436, 394, 278, 1230, 561, 1028, 734, 1327, 831, 591, 236, 1333, 948, 1976, 1846, 1150] payload_rows_per_dst=[8050, 8386, 9260, 8894, 9798, 10452, 9196, 9856] dup_rows_per_dst=[4058, 1962, 1606, 2844, 3710, 3178, 2780, 4274] combine_rows_per_src=[8050, 8386, 9260, 8894, 9798, 10452, 9196, 9856] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75780 payload_rows_remote=64184 dup_rows=22524 dup_rows_remote=19348 dedup_ratio=0.229126 router_tokens_per_expert=[361, 878, 1338, 495, 478, 377, 959, 541, 1214, 2471, 188, 427, 779, 662, 1102, 452, 2008, 76, 2225, 454, 625, 435, 19, 628, 1107, 720, 613, 542, 557, 1073, 1732, 982, 571, 653, 190, 256, 961, 546, 2118, 781, 476, 1722, 176, 505, 336, 661, 590, 3190, 187, 404, 274, 1017, 829, 891, 203, 391, 255, 456, 1335, 574, 785, 730, 254, 317] payload_rows_per_dst=[8850, 6490, 9336, 11064, 10652, 6984, 11596, 10808] dup_rows_per_dst=[2004, 730, 3604, 3588, 3928, 1948, 3176, 3546] combine_rows_per_src=[8850, 6490, 9336, 11064, 10652, 6984, 11596, 10808] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73952 payload_rows_remote=64696 dup_rows=24352 dup_rows_remote=21968 dedup_ratio=0.247721 router_tokens_per_expert=[5, 1621, 1379, 673, 1299, 1289, 645, 1049, 324, 599, 2212, 84, 221, 309, 313, 1206, 1480, 371, 95, 88, 278, 522, 329, 255, 585, 325, 2296, 453, 2477, 1017, 870, 13, 295, 6, 28, 590, 1311, 296, 168, 1315, 100, 268, 245, 980, 697, 0, 2854, 4, 127, 636, 289, 955, 418, 1433, 857, 1306, 738, 766, 125, 1508, 1055, 1769, 2094, 1237] payload_rows_per_dst=[8510, 10642, 9806, 9256, 8448, 9118, 8920, 9252] dup_rows_per_dst=[4168, 4326, 2412, 2384, 2812, 1178, 3122, 3950] combine_rows_per_src=[8510, 10642, 9806, 9256, 8448, 9118, 8920, 9252] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77208 payload_rows_remote=68678 dup_rows=21096 dup_rows_remote=19336 dedup_ratio=0.214600 router_tokens_per_expert=[214, 481, 1547, 268, 528, 561, 1129, 759, 2158, 3027, 304, 300, 1038, 612, 1422, 452, 2044, 40, 2556, 428, 812, 446, 5, 554, 974, 968, 654, 435, 517, 1070, 1433, 725, 592, 549, 165, 173, 1050, 411, 1491, 714, 370, 2487, 119, 491, 243, 475, 405, 3059, 25, 234, 230, 1162, 478, 869, 111, 165, 192, 298, 1722, 636, 648, 737, 153, 237] payload_rows_per_dst=[8936, 10154, 9876, 10440, 8530, 7334, 10446, 11492] dup_rows_per_dst=[2038, 2418, 3894, 3112, 1760, 1846, 2220, 3808] combine_rows_per_src=[8936, 10154, 9876, 10440, 8530, 7334, 10446, 11492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74676 payload_rows_remote=65468 dup_rows=23628 dup_rows_remote=19526 dedup_ratio=0.240356 router_tokens_per_expert=[17, 1788, 1433, 525, 1151, 1558, 1147, 741, 400, 356, 1483, 178, 570, 301, 375, 1583, 1340, 474, 143, 195, 584, 321, 181, 208, 392, 399, 1710, 607, 1593, 1209, 1260, 158, 486, 143, 313, 298, 1131, 918, 393, 968, 216, 318, 1245, 910, 867, 61, 2000, 28, 460, 351, 269, 1667, 573, 887, 725, 1256, 937, 568, 259, 1151, 1122, 1470, 1696, 1086] payload_rows_per_dst=[8154, 8114, 8592, 10554, 9766, 11002, 9286, 9208] dup_rows_per_dst=[1928, 2378, 1692, 3782, 3306, 3474, 2966, 4102] combine_rows_per_src=[8154, 8114, 8592, 10554, 9766, 11002, 9286, 9208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=260825088 dispatch_meta_bytes=334752 combine_payload_bytes=260825088 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=261159840 ingress_bytes=399237120 link_bytes=399237120 token_bytes=521984928 weight_movement_bytes=138412032 total_nvlink_bytes=660396960 weight_to_token_ratio=0.265165 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=10496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.502297141e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73952 payload_rows_remote=64696 dup_rows=24352 dup_rows_remote=21968 dedup_ratio=0.247721 router_tokens_per_expert=[5, 1621, 1379, 673, 1299, 1289, 645, 1049, 324, 599, 2212, 84, 221, 309, 313, 1206, 1480, 371, 95, 88, 278, 522, 329, 255, 585, 325, 2296, 453, 2477, 1017, 870, 13, 295, 6, 28, 590, 1311, 296, 168, 1315, 100, 268, 245, 980, 697, 0, 2854, 4, 127, 636, 289, 955, 418, 1433, 857, 1306, 738, 766, 125, 1508, 1055, 1769, 2094, 1237] payload_rows_per_dst=[8510, 10642, 9806, 9256, 8448, 9118, 8920, 9252] dup_rows_per_dst=[4168, 4326, 2412, 2384, 2812, 1178, 3122, 3950] combine_rows_per_src=[8510, 10642, 9806, 9256, 8448, 9118, 8920, 9252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 2, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=276217856 dispatch_meta_bytes=355456 combine_payload_bytes=276217856 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=276573312 ingress_bytes=345423872 link_bytes=345423872 token_bytes=552791168 weight_movement_bytes=69206016 total_nvlink_bytes=621997184 weight_to_token_ratio=0.125194 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=13696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.299802221e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=270458880 dispatch_meta_bytes=345344 combine_payload_bytes=270458880 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=270804224 ingress_bytes=305061888 link_bytes=305061888 token_bytes=541263104 weight_movement_bytes=34603008 total_nvlink_bytes=575866112 weight_to_token_ratio=0.063930 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=9984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.147923325e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=259850240 dispatch_meta_bytes=338696 combine_payload_bytes=259850240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260188936 ingress_bytes=294453248 link_bytes=294453248 token_bytes=520039176 weight_movement_bytes=34603008 total_nvlink_bytes=554642184 weight_to_token_ratio=0.066539 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=19840 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.108003867e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=268156928 dispatch_meta_bytes=339976 combine_payload_bytes=268156928 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=268496904 ingress_bytes=406568960 link_bytes=406568960 token_bytes=536653832 weight_movement_bytes=138412032 total_nvlink_bytes=675065864 weight_to_token_ratio=0.257917 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=10112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.529886265e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75216 payload_rows_remote=65776 dup_rows=23088 dup_rows_remote=19640 dedup_ratio=0.234863 router_tokens_per_expert=[328, 581, 1699, 231, 501, 693, 1166, 882, 2005, 2902, 367, 334, 1035, 558, 1472, 298, 1685, 59, 2336, 503, 850, 376, 13, 622, 799, 1066, 607, 416, 407, 1036, 1402, 735, 615, 631, 175, 189, 915, 483, 1228, 677, 462, 2697, 158, 467, 251, 542, 384, 3087, 54, 271, 271, 1120, 445, 922, 194, 253, 167, 335, 1807, 580, 645, 775, 143, 245] payload_rows_per_dst=[11128, 12762, 9440, 8560, 8266, 11224, 5786, 8050] dup_rows_per_dst=[3838, 5180, 3448, 1572, 1560, 4872, 1274, 1344] combine_rows_per_src=[11128, 12762, 9440, 8560, 8266, 11224, 5786, 8050] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72962 payload_rows_remote=64404 dup_rows=25342 dup_rows_remote=22342 dedup_ratio=0.257792 router_tokens_per_expert=[58, 2128, 1426, 385, 994, 1082, 1264, 671, 453, 305, 1392, 147, 449, 288, 475, 1198, 1633, 469, 259, 220, 491, 265, 208, 173, 375, 424, 1072, 553, 1204, 1160, 1198, 516, 635, 422, 358, 255, 1358, 1034, 421, 1138, 210, 317, 1612, 989, 901, 84, 2033, 104, 684, 257, 195, 1953, 599, 919, 672, 1000, 837, 465, 306, 996, 891, 1796, 1353, 1423] payload_rows_per_dst=[8190, 8558, 10588, 8496, 10836, 10062, 9106, 7126] dup_rows_per_dst=[4148, 3000, 3146, 2364, 4662, 2438, 3452, 2132] combine_rows_per_src=[8190, 8558, 10588, 8496, 10836, 10062, 9106, 7126] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267452416 dispatch_meta_bytes=336968 combine_payload_bytes=267452416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267789384 ingress_bytes=302055424 link_bytes=302055424 token_bytes=535241800 weight_movement_bytes=34603008 total_nvlink_bytes=569844808 weight_to_token_ratio=0.064649 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=10624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136610243e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75224 payload_rows_remote=64430 dup_rows=23080 dup_rows_remote=19396 dedup_ratio=0.234782 router_tokens_per_expert=[0, 952, 1465, 1114, 990, 1125, 629, 1300, 255, 733, 2617, 68, 168, 291, 325, 938, 1709, 394, 48, 47, 166, 751, 366, 236, 658, 253, 2297, 246, 2818, 789, 662, 3, 312, 1, 23, 691, 1197, 208, 128, 1232, 83, 279, 42, 1015, 563, 0, 3090, 0, 56, 968, 224, 1045, 393, 1816, 959, 1186, 721, 780, 93, 1597, 1080, 1749, 1773, 1435] payload_rows_per_dst=[9094, 10890, 10794, 9054, 7506, 9258, 9982, 8646] dup_rows_per_dst=[4152, 4494, 3684, 1804, 1982, 886, 3312, 2766] combine_rows_per_src=[9094, 10890, 10794, 9054, 7506, 9258, 9982, 8646] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74676 payload_rows_remote=65468 dup_rows=23628 dup_rows_remote=19526 dedup_ratio=0.240356 router_tokens_per_expert=[17, 1788, 1433, 525, 1151, 1558, 1147, 741, 400, 356, 1483, 178, 570, 301, 375, 1583, 1340, 474, 143, 195, 584, 321, 181, 208, 392, 399, 1710, 607, 1593, 1209, 1260, 158, 486, 143, 313, 298, 1131, 918, 393, 968, 216, 318, 1245, 910, 867, 61, 2000, 28, 460, 351, 269, 1667, 573, 887, 725, 1256, 937, 568, 259, 1151, 1122, 1470, 1696, 1086] payload_rows_per_dst=[8154, 8114, 8592, 10554, 9766, 11002, 9286, 9208] dup_rows_per_dst=[1928, 2378, 1692, 3782, 3306, 3474, 2966, 4102] combine_rows_per_src=[8154, 8114, 8592, 10554, 9766, 11002, 9286, 9208] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=259850240 dispatch_meta_bytes=338696 combine_payload_bytes=259850240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260188936 ingress_bytes=294453248 link_bytes=294453248 token_bytes=520039176 weight_movement_bytes=34603008 total_nvlink_bytes=554642184 weight_to_token_ratio=0.066539 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=19840 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.108003867e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=262897664 dispatch_meta_bytes=334128 combine_payload_bytes=262897664 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263231792 ingress_bytes=297500672 link_bytes=297500672 token_bytes=526129456 weight_movement_bytes=34603008 total_nvlink_bytes=560732464 weight_to_token_ratio=0.065769 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.119471078e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=264994816 dispatch_meta_bytes=346656 combine_payload_bytes=264994816 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265341472 ingress_bytes=368803840 link_bytes=368803840 token_bytes=530336288 weight_movement_bytes=103809024 total_nvlink_bytes=634145312 weight_to_token_ratio=0.195742 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=9728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.387779159e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=281305088 dispatch_meta_bytes=352056 combine_payload_bytes=281305088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281657144 ingress_bytes=315908096 link_bytes=315908096 token_bytes=562962232 weight_movement_bytes=34603008 total_nvlink_bytes=597565240 weight_to_token_ratio=0.061466 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.188736733e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=263798784 dispatch_meta_bytes=346984 combine_payload_bytes=263798784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264145768 ingress_bytes=298401792 link_bytes=298401792 token_bytes=527944552 weight_movement_bytes=34603008 total_nvlink_bytes=562547560 weight_to_token_ratio=0.065543 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.122861920e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=263905280 dispatch_meta_bytes=335304 combine_payload_bytes=263905280 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=264240584 ingress_bytes=333111296 link_bytes=333111296 token_bytes=528145864 weight_movement_bytes=69206016 total_nvlink_bytes=597351880 weight_to_token_ratio=0.131036 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.030000 gemm_active_groups=11 gemm_peak_group_rows=14080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.253470989e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=268156928 dispatch_meta_bytes=339976 combine_payload_bytes=268156928 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=268496904 ingress_bytes=406568960 link_bytes=406568960 token_bytes=536653832 weight_movement_bytes=138412032 total_nvlink_bytes=675065864 weight_to_token_ratio=0.257917 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=10112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.529886265e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73468 payload_rows_remote=63678 dup_rows=24836 dup_rows_remote=20010 dedup_ratio=0.252645 router_tokens_per_expert=[11, 1922, 1576, 693, 975, 1172, 920, 795, 555, 494, 1656, 92, 353, 277, 359, 1567, 1873, 462, 138, 165, 340, 415, 166, 170, 694, 390, 1804, 591, 1464, 972, 1144, 151, 382, 44, 192, 333, 1716, 768, 315, 1017, 153, 351, 779, 900, 848, 21, 2282, 16, 406, 510, 270, 1176, 539, 1014, 762, 1173, 974, 748, 194, 1166, 983, 1688, 1882, 1194] payload_rows_per_dst=[9790, 10300, 10906, 8460, 9814, 8940, 8842, 6416] dup_rows_per_dst=[4826, 4014, 3692, 2352, 3564, 1760, 2858, 1770] combine_rows_per_src=[9790, 10300, 10906, 8460, 9814, 8940, 8842, 6416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76332 payload_rows_remote=64624 dup_rows=21972 dup_rows_remote=16620 dedup_ratio=0.223511 router_tokens_per_expert=[747, 1642, 1115, 307, 364, 45, 2777, 322, 28, 12, 1015, 842, 1487, 0, 23, 1189, 748, 880, 3063, 81, 647, 292, 348, 13, 111, 1902, 872, 124, 396, 238, 3219, 324, 1486, 569, 212, 1460, 621, 304, 1760, 1082, 185, 7, 187, 2098, 164, 406, 1505, 447, 980, 1209, 953, 782, 183, 700, 6, 1254, 750, 483, 450, 892, 290, 549, 1165, 840] payload_rows_per_dst=[11708, 11382, 8370, 7180, 10986, 8004, 9750, 8952] dup_rows_per_dst=[5352, 4248, 1352, 754, 4002, 1994, 2384, 1886] combine_rows_per_src=[11708, 11382, 8370, 7180, 10986, 8004, 9750, 8952] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=269418496 dispatch_meta_bytes=341664 combine_payload_bytes=269418496 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=269760160 ingress_bytes=338624512 link_bytes=338624512 token_bytes=539178656 weight_movement_bytes=69206016 total_nvlink_bytes=608384672 weight_to_token_ratio=0.128355 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274216777e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76848 payload_rows_remote=69816 dup_rows=21456 dup_rows_remote=20888 dedup_ratio=0.218262 router_tokens_per_expert=[586, 1697, 674, 282, 87, 65, 2837, 504, 0, 0, 832, 681, 2350, 0, 0, 822, 480, 1004, 4349, 20, 471, 1303, 405, 0, 75, 3164, 1018, 93, 151, 35, 3663, 236, 1705, 448, 94, 810, 457, 307, 1146, 538, 49, 0, 11, 1229, 62, 678, 1975, 699, 1020, 1118, 1438, 1142, 116, 555, 0, 1178, 1139, 398, 112, 498, 229, 415, 838, 864] payload_rows_per_dst=[8176, 8186, 12572, 11914, 7032, 9316, 9980, 9672] dup_rows_per_dst=[1934, 1184, 3492, 4956, 568, 3444, 3154, 2724] combine_rows_per_src=[8176, 8186, 12572, 11914, 7032, 9316, 9980, 9672] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77912 payload_rows_remote=68230 dup_rows=20392 dup_rows_remote=17986 dedup_ratio=0.207438 router_tokens_per_expert=[916, 1911, 1089, 348, 410, 299, 2695, 395, 28, 8, 946, 835, 1749, 0, 7, 1298, 759, 733, 3147, 83, 634, 426, 384, 12, 100, 1876, 945, 171, 344, 250, 2517, 413, 1718, 669, 198, 1417, 503, 331, 1760, 758, 283, 0, 61, 1796, 266, 401, 1431, 480, 1044, 1253, 1091, 446, 201, 626, 0, 1383, 869, 486, 440, 696, 336, 565, 1246, 670] payload_rows_per_dst=[9000, 8170, 10296, 10298, 9466, 10266, 9682, 10734] dup_rows_per_dst=[1736, 1572, 2060, 2934, 1806, 4560, 2406, 3318] combine_rows_per_src=[9000, 8170, 10296, 10298, 9466, 10266, 9682, 10734] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77354 payload_rows_remote=66308 dup_rows=20950 dup_rows_remote=15774 dedup_ratio=0.213114 router_tokens_per_expert=[882, 1566, 1136, 272, 367, 206, 3544, 511, 16, 1, 1038, 834, 1284, 0, 9, 1238, 715, 1007, 3383, 82, 564, 389, 312, 1, 186, 2353, 920, 128, 359, 313, 2805, 487, 1582, 587, 162, 1211, 574, 341, 1246, 958, 219, 0, 167, 1825, 159, 460, 1225, 512, 889, 970, 1063, 678, 167, 681, 0, 1295, 796, 446, 445, 739, 256, 578, 1212, 801] payload_rows_per_dst=[8216, 7542, 10794, 11186, 8750, 11046, 9348, 10472] dup_rows_per_dst=[1664, 1298, 2112, 3916, 1408, 5176, 2138, 3238] combine_rows_per_src=[8216, 7542, 10794, 11186, 8750, 11046, 9348, 10472] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75110 payload_rows_remote=66030 dup_rows=23194 dup_rows_remote=20306 dedup_ratio=0.235942 router_tokens_per_expert=[17, 1976, 1459, 395, 1006, 1625, 891, 851, 561, 360, 1597, 154, 336, 340, 410, 1419, 1940, 443, 163, 206, 368, 240, 256, 204, 407, 369, 2059, 606, 1925, 1172, 1109, 101, 323, 43, 97, 497, 1328, 646, 315, 1105, 146, 408, 876, 943, 619, 7, 2104, 7, 388, 535, 217, 1156, 540, 1080, 788, 1280, 894, 737, 169, 1361, 969, 1819, 1691, 1099] payload_rows_per_dst=[7914, 8272, 9372, 10710, 9476, 10552, 9080, 9734] dup_rows_per_dst=[1656, 2082, 1650, 3854, 3184, 3518, 2888, 4362] combine_rows_per_src=[7914, 8272, 9372, 10710, 9476, 10552, 9080, 9734] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77386 payload_rows_remote=68390 dup_rows=20918 dup_rows_remote=19444 dedup_ratio=0.212789 router_tokens_per_expert=[774, 1449, 749, 245, 171, 137, 3233, 527, 2, 4, 854, 708, 1716, 0, 12, 784, 536, 1051, 4099, 50, 529, 638, 352, 2, 119, 2728, 951, 149, 218, 146, 3234, 409, 1731, 523, 152, 962, 563, 318, 1119, 771, 118, 5, 97, 1608, 110, 673, 1599, 502, 1062, 1107, 1166, 1084, 168, 615, 6, 1292, 1046, 390, 340, 639, 297, 570, 1067, 876] payload_rows_per_dst=[10810, 10396, 11776, 8996, 9006, 7454, 9832, 9116] dup_rows_per_dst=[3760, 3202, 2738, 1474, 2256, 1970, 3168, 2350] combine_rows_per_src=[10810, 10396, 11776, 8996, 9006, 7454, 9832, 9116] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75774 payload_rows_remote=64498 dup_rows=22530 dup_rows_remote=17930 dedup_ratio=0.229187 router_tokens_per_expert=[1029, 931, 1476, 270, 554, 33, 2072, 364, 153, 159, 1276, 911, 989, 2, 87, 1722, 827, 783, 2101, 69, 613, 175, 304, 455, 460, 1344, 679, 327, 511, 438, 2639, 468, 1436, 788, 516, 2255, 744, 615, 1399, 811, 359, 75, 394, 1951, 347, 491, 925, 587, 885, 949, 820, 443, 256, 675, 39, 1495, 528, 512, 504, 1070, 206, 507, 1678, 671] payload_rows_per_dst=[10046, 11276, 9122, 7396, 11636, 7960, 9048, 9290] dup_rows_per_dst=[3412, 4600, 1532, 1058, 5492, 2298, 2076, 2062] combine_rows_per_src=[10046, 11276, 9122, 7396, 11636, 7960, 9048, 9290] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74634 payload_rows_remote=67436 dup_rows=23670 dup_rows_remote=21428 dedup_ratio=0.240784 router_tokens_per_expert=[0, 1368, 1449, 1053, 887, 1440, 675, 1329, 303, 803, 2483, 70, 176, 319, 250, 1039, 2092, 394, 17, 61, 135, 738, 290, 161, 598, 184, 2565, 335, 2834, 713, 698, 0, 273, 0, 7, 659, 1134, 127, 104, 1048, 71, 285, 28, 980, 489, 0, 3212, 0, 16, 887, 290, 817, 454, 1630, 1041, 1122, 583, 960, 50, 1545, 1216, 1637, 1664, 1364] payload_rows_per_dst=[9194, 8952, 9390, 9842, 7198, 10520, 9284, 10254] dup_rows_per_dst=[4472, 1934, 1714, 3654, 2242, 1968, 3230, 4456] combine_rows_per_src=[9194, 8952, 9390, 9842, 7198, 10520, 9284, 10254] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=260825088 dispatch_meta_bytes=334752 combine_payload_bytes=260825088 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=261159840 ingress_bytes=399237120 link_bytes=399237120 token_bytes=521984928 weight_movement_bytes=138412032 total_nvlink_bytes=660396960 weight_to_token_ratio=0.265165 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=10496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.502297141e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77526 payload_rows_remote=65678 dup_rows=20778 dup_rows_remote=17942 dedup_ratio=0.211365 router_tokens_per_expert=[549, 1616, 583, 222, 131, 8, 3012, 539, 1, 0, 716, 604, 1986, 0, 1, 739, 515, 945, 4047, 43, 432, 1047, 313, 0, 130, 3073, 1035, 76, 154, 103, 3778, 259, 2192, 434, 128, 780, 588, 252, 963, 828, 54, 0, 31, 1388, 68, 854, 1755, 613, 976, 1026, 1319, 1278, 141, 621, 0, 1201, 1131, 359, 313, 527, 268, 439, 885, 1083] payload_rows_per_dst=[10124, 11296, 11848, 8688, 9582, 7704, 10018, 8266] dup_rows_per_dst=[3196, 4354, 2836, 972, 2748, 1822, 3106, 1744] combine_rows_per_src=[10124, 11296, 11848, 8688, 9582, 7704, 10018, 8266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75542 payload_rows_remote=64554 dup_rows=22762 dup_rows_remote=19270 dedup_ratio=0.231547 router_tokens_per_expert=[867, 1221, 1178, 286, 527, 157, 2796, 349, 75, 61, 1228, 907, 1267, 0, 41, 1220, 695, 982, 3257, 120, 602, 278, 391, 185, 223, 1902, 837, 130, 387, 308, 2390, 481, 1417, 721, 247, 1407, 840, 410, 1412, 792, 310, 8, 178, 2001, 176, 506, 1191, 479, 957, 1087, 963, 592, 234, 751, 0, 1302, 737, 493, 452, 727, 297, 627, 1473, 1017] payload_rows_per_dst=[7572, 7996, 6828, 11484, 10432, 10616, 9626, 10988] dup_rows_per_dst=[1598, 1602, 576, 5098, 3134, 4674, 2588, 3492] combine_rows_per_src=[7572, 7996, 6828, 11484, 10432, 10616, 9626, 10988] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77386 payload_rows_remote=68390 dup_rows=20918 dup_rows_remote=19444 dedup_ratio=0.212789 router_tokens_per_expert=[774, 1449, 749, 245, 171, 137, 3233, 527, 2, 4, 854, 708, 1716, 0, 12, 784, 536, 1051, 4099, 50, 529, 638, 352, 2, 119, 2728, 951, 149, 218, 146, 3234, 409, 1731, 523, 152, 962, 563, 318, 1119, 771, 118, 5, 97, 1608, 110, 673, 1599, 502, 1062, 1107, 1166, 1084, 168, 615, 6, 1292, 1046, 390, 340, 639, 297, 570, 1067, 876] payload_rows_per_dst=[10810, 10396, 11776, 8996, 9006, 7454, 9832, 9116] dup_rows_per_dst=[3760, 3202, 2738, 1474, 2256, 1970, 3168, 2350] combine_rows_per_src=[10810, 10396, 11776, 8996, 9006, 7454, 9832, 9116] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=264699904 dispatch_meta_bytes=324976 combine_payload_bytes=264699904 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265024880 ingress_bytes=333905920 link_bytes=333905920 token_bytes=529724784 weight_movement_bytes=69206016 total_nvlink_bytes=598930800 weight_to_token_ratio=0.130645 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256461095e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=285966336 dispatch_meta_bytes=362816 combine_payload_bytes=285966336 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=286329152 ingress_bytes=355172352 link_bytes=355172352 token_bytes=572295488 weight_movement_bytes=69206016 total_nvlink_bytes=641501504 weight_to_token_ratio=0.120927 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=10880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.336484967e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279470080 dispatch_meta_bytes=344864 combine_payload_bytes=279470080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279814944 ingress_bytes=314073088 link_bytes=314073088 token_bytes=559285024 weight_movement_bytes=34603008 total_nvlink_bytes=593888032 weight_to_token_ratio=0.061870 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.181831745e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=271597568 dispatch_meta_bytes=328328 combine_payload_bytes=271597568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271925896 ingress_bytes=306200576 link_bytes=306200576 token_bytes=543523464 weight_movement_bytes=34603008 total_nvlink_bytes=578126472 weight_to_token_ratio=0.063664 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=13952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.152208116e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=264413184 dispatch_meta_bytes=335296 combine_payload_bytes=264413184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264748480 ingress_bytes=299016192 link_bytes=299016192 token_bytes=529161664 weight_movement_bytes=34603008 total_nvlink_bytes=563764672 weight_to_token_ratio=0.065392 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125173858e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75774 payload_rows_remote=64498 dup_rows=22530 dup_rows_remote=17930 dedup_ratio=0.229187 router_tokens_per_expert=[1029, 931, 1476, 270, 554, 33, 2072, 364, 153, 159, 1276, 911, 989, 2, 87, 1722, 827, 783, 2101, 69, 613, 175, 304, 455, 460, 1344, 679, 327, 511, 438, 2639, 468, 1436, 788, 516, 2255, 744, 615, 1399, 811, 359, 75, 394, 1951, 347, 491, 925, 587, 885, 949, 820, 443, 256, 675, 39, 1495, 528, 512, 504, 1070, 206, 507, 1678, 671] payload_rows_per_dst=[10046, 11276, 9122, 7396, 11636, 7960, 9048, 9290] dup_rows_per_dst=[3412, 4600, 1532, 1058, 5492, 2298, 2076, 2062] combine_rows_per_src=[10046, 11276, 9122, 7396, 11636, 7960, 9048, 9290] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280125440 dispatch_meta_bytes=351336 combine_payload_bytes=280125440 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=280476776 ingress_bytes=349331456 link_bytes=349331456 token_bytes=560602216 weight_movement_bytes=69206016 total_nvlink_bytes=629808232 weight_to_token_ratio=0.123449 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.314506145e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=270458880 dispatch_meta_bytes=345344 combine_payload_bytes=270458880 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=270804224 ingress_bytes=305061888 link_bytes=305061888 token_bytes=541263104 weight_movement_bytes=34603008 total_nvlink_bytes=575866112 weight_to_token_ratio=0.063930 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=9984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.147923325e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75224 payload_rows_remote=64430 dup_rows=23080 dup_rows_remote=19396 dedup_ratio=0.234782 router_tokens_per_expert=[0, 952, 1465, 1114, 990, 1125, 629, 1300, 255, 733, 2617, 68, 168, 291, 325, 938, 1709, 394, 48, 47, 166, 751, 366, 236, 658, 253, 2297, 246, 2818, 789, 662, 3, 312, 1, 23, 691, 1197, 208, 128, 1232, 83, 279, 42, 1015, 563, 0, 3090, 0, 56, 968, 224, 1045, 393, 1816, 959, 1186, 721, 780, 93, 1597, 1080, 1749, 1773, 1435] payload_rows_per_dst=[9094, 10890, 10794, 9054, 7506, 9258, 9982, 8646] dup_rows_per_dst=[4152, 4494, 3684, 1804, 1982, 886, 3312, 2766] combine_rows_per_src=[9094, 10890, 10794, 9054, 7506, 9258, 9982, 8646] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=264183808 dispatch_meta_bytes=329712 combine_payload_bytes=264183808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264513520 ingress_bytes=298786816 link_bytes=298786816 token_bytes=528697328 weight_movement_bytes=34603008 total_nvlink_bytes=563300336 weight_to_token_ratio=0.065450 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.124310734e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=276217856 dispatch_meta_bytes=355456 combine_payload_bytes=276217856 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=276573312 ingress_bytes=345423872 link_bytes=345423872 token_bytes=552791168 weight_movement_bytes=69206016 total_nvlink_bytes=621997184 weight_to_token_ratio=0.125194 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=13696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.299802221e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=269017088 dispatch_meta_bytes=334480 combine_payload_bytes=269017088 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=269351568 ingress_bytes=407429120 link_bytes=407429120 token_bytes=538368656 weight_movement_bytes=138412032 total_nvlink_bytes=676780688 weight_to_token_ratio=0.257095 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=23552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.533122978e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=264413184 dispatch_meta_bytes=335296 combine_payload_bytes=264413184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264748480 ingress_bytes=299016192 link_bytes=299016192 token_bytes=529161664 weight_movement_bytes=34603008 total_nvlink_bytes=563764672 weight_to_token_ratio=0.065392 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125173858e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76332 payload_rows_remote=64624 dup_rows=21972 dup_rows_remote=16620 dedup_ratio=0.223511 router_tokens_per_expert=[747, 1642, 1115, 307, 364, 45, 2777, 322, 28, 12, 1015, 842, 1487, 0, 23, 1189, 748, 880, 3063, 81, 647, 292, 348, 13, 111, 1902, 872, 124, 396, 238, 3219, 324, 1486, 569, 212, 1460, 621, 304, 1760, 1082, 185, 7, 187, 2098, 164, 406, 1505, 447, 980, 1209, 953, 782, 183, 700, 6, 1254, 750, 483, 450, 892, 290, 549, 1165, 840] payload_rows_per_dst=[11708, 11382, 8370, 7180, 10986, 8004, 9750, 8952] dup_rows_per_dst=[5352, 4248, 1352, 754, 4002, 1994, 2384, 1886] combine_rows_per_src=[11708, 11382, 8370, 7180, 10986, 8004, 9750, 8952] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78634 payload_rows_remote=64004 dup_rows=19670 dup_rows_remote=12488 dedup_ratio=0.200094 router_tokens_per_expert=[875, 175, 0, 3261, 76, 9, 261, 72, 3, 340, 1396, 712, 1040, 578, 432, 1723, 1, 0, 674, 21, 114, 531, 452, 2671, 113, 94, 703, 4, 370, 541, 0, 2, 68, 1921, 478, 13, 0, 5, 1, 2435, 1278, 243, 199, 1807, 5036, 272, 1236, 2844, 385, 285, 1, 813, 323, 1055, 6177, 4, 4287, 2, 0, 22, 386, 0, 285, 47] payload_rows_per_dst=[14630, 12588, 7882, 12334, 10202, 5616, 5554, 9828] dup_rows_per_dst=[7182, 5548, 1046, 1392, 3254, 840, 178, 230] combine_rows_per_src=[14630, 12588, 7882, 12334, 10202, 5616, 5554, 9828] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81430 payload_rows_remote=69784 dup_rows=16874 dup_rows_remote=13396 dedup_ratio=0.171651 router_tokens_per_expert=[677, 114, 0, 4593, 80, 0, 165, 97, 0, 258, 449, 426, 1004, 523, 444, 1152, 0, 0, 596, 2, 0, 519, 460, 1579, 107, 59, 260, 0, 82, 488, 0, 0, 97, 1947, 205, 4, 0, 0, 0, 3416, 1169, 82, 209, 1893, 5516, 158, 1553, 4086, 420, 89, 0, 1302, 110, 1706, 6371, 0, 4304, 0, 0, 0, 0, 0, 313, 68] payload_rows_per_dst=[10352, 6888, 6406, 12562, 11646, 10010, 12764, 10802] dup_rows_per_dst=[1100, 1624, 1256, 462, 3478, 3816, 3322, 1816] combine_rows_per_src=[10352, 6888, 6406, 12562, 11646, 10010, 12764, 10802] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80950 payload_rows_remote=65924 dup_rows=17354 dup_rows_remote=13802 dedup_ratio=0.176534 router_tokens_per_expert=[921, 181, 1, 3299, 76, 21, 249, 85, 0, 443, 1236, 803, 1096, 442, 505, 1609, 0, 7, 724, 16, 106, 641, 483, 2693, 147, 86, 543, 66, 394, 570, 0, 9, 91, 1600, 434, 30, 16, 0, 0, 2147, 1392, 267, 230, 2183, 4763, 246, 1287, 2677, 363, 536, 0, 672, 345, 1291, 6082, 0, 4224, 8, 0, 25, 357, 0, 350, 84] payload_rows_per_dst=[8450, 9670, 8142, 11918, 7078, 8718, 15026, 11948] dup_rows_per_dst=[1216, 2598, 1198, 1238, 1558, 2492, 3552, 3502] combine_rows_per_src=[8450, 9670, 8142, 11918, 7078, 8718, 15026, 11948] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81560 payload_rows_remote=73576 dup_rows=16744 dup_rows_remote=14890 dedup_ratio=0.170329 router_tokens_per_expert=[696, 135, 0, 3379, 100, 17, 233, 67, 0, 327, 1674, 857, 1050, 522, 459, 1586, 0, 1, 667, 4, 173, 571, 474, 2538, 112, 56, 653, 2, 370, 591, 0, 5, 67, 1729, 249, 17, 0, 0, 0, 2173, 1490, 183, 220, 1944, 4977, 317, 1230, 3254, 390, 295, 0, 942, 340, 1119, 5960, 0, 4104, 2, 0, 5, 339, 0, 421, 66] payload_rows_per_dst=[8260, 10264, 7694, 12274, 7730, 7984, 14760, 12594] dup_rows_per_dst=[994, 2686, 1162, 1258, 1670, 1854, 3332, 3788] combine_rows_per_src=[8260, 10264, 7694, 12274, 7730, 7984, 14760, 12594] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82102 payload_rows_remote=69620 dup_rows=16202 dup_rows_remote=15530 dedup_ratio=0.164815 router_tokens_per_expert=[703, 135, 1, 4301, 81, 1, 210, 89, 1, 315, 948, 561, 935, 682, 435, 1422, 0, 0, 602, 1, 6, 572, 395, 2246, 137, 65, 404, 0, 147, 485, 0, 4, 88, 1875, 171, 3, 7, 6, 8, 2905, 1330, 86, 189, 1780, 5335, 184, 1640, 3456, 374, 108, 1, 1354, 222, 1108, 6164, 4, 4304, 1, 3, 17, 139, 1, 343, 62] payload_rows_per_dst=[9904, 8414, 14020, 12482, 10804, 10764, 6132, 9582] dup_rows_per_dst=[1138, 2184, 5952, 672, 2882, 3006, 210, 158] combine_rows_per_src=[9904, 8414, 14020, 12482, 10804, 10764, 6132, 9582] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77912 payload_rows_remote=68230 dup_rows=20392 dup_rows_remote=17986 dedup_ratio=0.207438 router_tokens_per_expert=[916, 1911, 1089, 348, 410, 299, 2695, 395, 28, 8, 946, 835, 1749, 0, 7, 1298, 759, 733, 3147, 83, 634, 426, 384, 12, 100, 1876, 945, 171, 344, 250, 2517, 413, 1718, 669, 198, 1417, 503, 331, 1760, 758, 283, 0, 61, 1796, 266, 401, 1431, 480, 1044, 1253, 1091, 446, 201, 626, 0, 1383, 869, 486, 440, 696, 336, 565, 1246, 670] payload_rows_per_dst=[9000, 8170, 10296, 10298, 9466, 10266, 9682, 10734] dup_rows_per_dst=[1736, 1572, 2060, 2934, 1806, 4560, 2406, 3318] combine_rows_per_src=[9000, 8170, 10296, 10298, 9466, 10266, 9682, 10734] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77458 payload_rows_remote=67018 dup_rows=20846 dup_rows_remote=18136 dedup_ratio=0.212056 router_tokens_per_expert=[939, 357, 29, 1569, 120, 333, 358, 136, 31, 786, 1376, 847, 1179, 266, 279, 1737, 6, 121, 722, 61, 1504, 737, 692, 1822, 164, 494, 713, 442, 929, 765, 15, 114, 105, 1523, 539, 193, 72, 100, 27, 2855, 1320, 262, 563, 1587, 3112, 1036, 646, 1593, 438, 2063, 95, 379, 564, 869, 4700, 75, 2932, 453, 27, 165, 637, 36, 327, 216] payload_rows_per_dst=[10540, 10440, 10564, 10734, 10280, 8728, 7716, 8456] dup_rows_per_dst=[3358, 2710, 3950, 2762, 3722, 1964, 1250, 1130] combine_rows_per_src=[10540, 10440, 10564, 10734, 10280, 8728, 7716, 8456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=263905280 dispatch_meta_bytes=335304 combine_payload_bytes=263905280 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=264240584 ingress_bytes=333111296 link_bytes=333111296 token_bytes=528145864 weight_movement_bytes=69206016 total_nvlink_bytes=597351880 weight_to_token_ratio=0.131036 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.030000 gemm_active_groups=11 gemm_peak_group_rows=14080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.253470989e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78498 payload_rows_remote=67034 dup_rows=19806 dup_rows_remote=16772 dedup_ratio=0.201477 router_tokens_per_expert=[797, 207, 0, 3547, 78, 33, 243, 85, 0, 427, 1602, 914, 1104, 529, 460, 1850, 1, 2, 684, 5, 445, 689, 480, 2455, 113, 83, 539, 12, 406, 661, 0, 13, 89, 1695, 363, 58, 1, 1, 5, 2490, 1415, 240, 241, 1782, 4746, 278, 1411, 2495, 408, 389, 1, 1051, 285, 1003, 5486, 1, 3910, 3, 3, 30, 353, 0, 284, 171] payload_rows_per_dst=[8648, 10624, 8078, 4996, 7640, 12968, 14080, 11464] dup_rows_per_dst=[1332, 3148, 1444, 314, 1764, 5602, 3168, 3034] combine_rows_per_src=[8648, 10624, 8078, 4996, 7640, 12968, 14080, 11464] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=264699904 dispatch_meta_bytes=324976 combine_payload_bytes=264699904 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265024880 ingress_bytes=333905920 link_bytes=333905920 token_bytes=529724784 weight_movement_bytes=69206016 total_nvlink_bytes=598930800 weight_to_token_ratio=0.130645 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256461095e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76848 payload_rows_remote=69816 dup_rows=21456 dup_rows_remote=20888 dedup_ratio=0.218262 router_tokens_per_expert=[586, 1697, 674, 282, 87, 65, 2837, 504, 0, 0, 832, 681, 2350, 0, 0, 822, 480, 1004, 4349, 20, 471, 1303, 405, 0, 75, 3164, 1018, 93, 151, 35, 3663, 236, 1705, 448, 94, 810, 457, 307, 1146, 538, 49, 0, 11, 1229, 62, 678, 1975, 699, 1020, 1118, 1438, 1142, 116, 555, 0, 1178, 1139, 398, 112, 498, 229, 415, 838, 864] payload_rows_per_dst=[8176, 8186, 12572, 11914, 7032, 9316, 9980, 9672] dup_rows_per_dst=[1934, 1184, 3492, 4956, 568, 3444, 3154, 2724] combine_rows_per_src=[8176, 8186, 12572, 11914, 7032, 9316, 9980, 9672] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83548 payload_rows_remote=69246 dup_rows=14756 dup_rows_remote=9548 dedup_ratio=0.150106 router_tokens_per_expert=[547, 88, 0, 4639, 86, 0, 166, 74, 0, 278, 661, 420, 930, 735, 396, 1039, 0, 0, 531, 0, 9, 486, 434, 1921, 103, 72, 307, 0, 140, 378, 0, 0, 74, 1847, 208, 2, 0, 0, 0, 3230, 1240, 109, 177, 1937, 5453, 224, 1385, 3836, 377, 98, 0, 1410, 100, 1645, 6374, 0, 4582, 0, 0, 0, 0, 0, 352, 52] payload_rows_per_dst=[10382, 7290, 14302, 12448, 11538, 11032, 6712, 9844] dup_rows_per_dst=[818, 1628, 5208, 458, 3058, 2910, 548, 128] combine_rows_per_src=[10382, 7290, 14302, 12448, 11538, 11032, 6712, 9844] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82102 payload_rows_remote=69620 dup_rows=16202 dup_rows_remote=15530 dedup_ratio=0.164815 router_tokens_per_expert=[703, 135, 1, 4301, 81, 1, 210, 89, 1, 315, 948, 561, 935, 682, 435, 1422, 0, 0, 602, 1, 6, 572, 395, 2246, 137, 65, 404, 0, 147, 485, 0, 4, 88, 1875, 171, 3, 7, 6, 8, 2905, 1330, 86, 189, 1780, 5335, 184, 1640, 3456, 374, 108, 1, 1354, 222, 1108, 6164, 4, 4304, 1, 3, 17, 139, 1, 343, 62] payload_rows_per_dst=[9904, 8414, 14020, 12482, 10804, 10764, 6132, 9582] dup_rows_per_dst=[1138, 2184, 5952, 672, 2882, 3006, 210, 158] combine_rows_per_src=[9904, 8414, 14020, 12482, 10804, 10764, 6132, 9582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=262160384 dispatch_meta_bytes=305968 combine_payload_bytes=262160384 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262466352 ingress_bytes=296763392 link_bytes=296763392 token_bytes=524626736 weight_movement_bytes=34603008 total_nvlink_bytes=559229744 weight_to_token_ratio=0.065957 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=28672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.116696753e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=270024704 dispatch_meta_bytes=318904 combine_payload_bytes=270024704 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=270343608 ingress_bytes=373833728 link_bytes=373833728 token_bytes=540368312 weight_movement_bytes=103809024 total_nvlink_bytes=644177336 weight_to_token_ratio=0.192108 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=22016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.406706222e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=301367296 dispatch_meta_bytes=353864 combine_payload_bytes=301367296 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=301721160 ingress_bytes=474382336 link_bytes=474382336 token_bytes=603088456 weight_movement_bytes=173015040 total_nvlink_bytes=776103496 weight_to_token_ratio=0.286882 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.785062539e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78498 payload_rows_remote=67034 dup_rows=19806 dup_rows_remote=16772 dedup_ratio=0.201477 router_tokens_per_expert=[797, 207, 0, 3547, 78, 33, 243, 85, 0, 427, 1602, 914, 1104, 529, 460, 1850, 1, 2, 684, 5, 445, 689, 480, 2455, 113, 83, 539, 12, 406, 661, 0, 13, 89, 1695, 363, 58, 1, 1, 5, 2490, 1415, 240, 241, 1782, 4746, 278, 1411, 2495, 408, 389, 1, 1051, 285, 1003, 5486, 1, 3910, 3, 3, 30, 353, 0, 284, 171] payload_rows_per_dst=[8648, 10624, 8078, 4996, 7640, 12968, 14080, 11464] dup_rows_per_dst=[1332, 3148, 1444, 314, 1764, 5602, 3168, 3034] combine_rows_per_src=[8648, 10624, 8078, 4996, 7640, 12968, 14080, 11464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=274571264 dispatch_meta_bytes=335224 combine_payload_bytes=274571264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274906488 ingress_bytes=309174272 link_bytes=309174272 token_bytes=549477752 weight_movement_bytes=34603008 total_nvlink_bytes=584080760 weight_to_token_ratio=0.062974 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.163397895e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77458 payload_rows_remote=67018 dup_rows=20846 dup_rows_remote=18136 dedup_ratio=0.212056 router_tokens_per_expert=[939, 357, 29, 1569, 120, 333, 358, 136, 31, 786, 1376, 847, 1179, 266, 279, 1737, 6, 121, 722, 61, 1504, 737, 692, 1822, 164, 494, 713, 442, 929, 765, 15, 114, 105, 1523, 539, 193, 72, 100, 27, 2855, 1320, 262, 563, 1587, 3112, 1036, 646, 1593, 438, 2063, 95, 379, 564, 869, 4700, 75, 2932, 453, 27, 165, 637, 36, 327, 216] payload_rows_per_dst=[10540, 10440, 10564, 10734, 10280, 8728, 7716, 8456] dup_rows_per_dst=[3358, 2710, 3950, 2762, 3722, 1964, 1250, 1130] combine_rows_per_src=[10540, 10440, 10564, 10734, 10280, 8728, 7716, 8456] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=285163520 dispatch_meta_bytes=340600 combine_payload_bytes=285163520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=285504120 ingress_bytes=319766528 link_bytes=319766528 token_bytes=570667640 weight_movement_bytes=34603008 total_nvlink_bytes=605270648 weight_to_token_ratio=0.060636 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=35072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.203255702e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279470080 dispatch_meta_bytes=344864 combine_payload_bytes=279470080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279814944 ingress_bytes=314073088 link_bytes=314073088 token_bytes=559285024 weight_movement_bytes=34603008 total_nvlink_bytes=593888032 weight_to_token_ratio=0.061870 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.181831745e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=274505728 dispatch_meta_bytes=340616 combine_payload_bytes=274505728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274846344 ingress_bytes=309108736 link_bytes=309108736 token_bytes=549352072 weight_movement_bytes=34603008 total_nvlink_bytes=583955080 weight_to_token_ratio=0.062989 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.163151288e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=285966336 dispatch_meta_bytes=362816 combine_payload_bytes=285966336 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=286329152 ingress_bytes=355172352 link_bytes=355172352 token_bytes=572295488 weight_movement_bytes=69206016 total_nvlink_bytes=641501504 weight_to_token_ratio=0.120927 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=10880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.336484967e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=274571264 dispatch_meta_bytes=335224 combine_payload_bytes=274571264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274906488 ingress_bytes=309174272 link_bytes=309174272 token_bytes=549477752 weight_movement_bytes=34603008 total_nvlink_bytes=584080760 weight_to_token_ratio=0.062974 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.163397895e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=283631616 dispatch_meta_bytes=315176 combine_payload_bytes=283631616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=283946792 ingress_bytes=318234624 link_bytes=318234624 token_bytes=567578408 weight_movement_bytes=34603008 total_nvlink_bytes=602181416 weight_to_token_ratio=0.060966 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=18048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.197491270e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78634 payload_rows_remote=64004 dup_rows=19670 dup_rows_remote=12488 dedup_ratio=0.200094 router_tokens_per_expert=[875, 175, 0, 3261, 76, 9, 261, 72, 3, 340, 1396, 712, 1040, 578, 432, 1723, 1, 0, 674, 21, 114, 531, 452, 2671, 113, 94, 703, 4, 370, 541, 0, 2, 68, 1921, 478, 13, 0, 5, 1, 2435, 1278, 243, 199, 1807, 5036, 272, 1236, 2844, 385, 285, 1, 813, 323, 1055, 6177, 4, 4287, 2, 0, 22, 386, 0, 285, 47] payload_rows_per_dst=[14630, 12588, 7882, 12334, 10202, 5616, 5554, 9828] dup_rows_per_dst=[7182, 5548, 1046, 1392, 3254, 840, 178, 230] combine_rows_per_src=[14630, 12588, 7882, 12334, 10202, 5616, 5554, 9828] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77526 payload_rows_remote=65678 dup_rows=20778 dup_rows_remote=17942 dedup_ratio=0.211365 router_tokens_per_expert=[549, 1616, 583, 222, 131, 8, 3012, 539, 1, 0, 716, 604, 1986, 0, 1, 739, 515, 945, 4047, 43, 432, 1047, 313, 0, 130, 3073, 1035, 76, 154, 103, 3778, 259, 2192, 434, 128, 780, 588, 252, 963, 828, 54, 0, 31, 1388, 68, 854, 1755, 613, 976, 1026, 1319, 1278, 141, 621, 0, 1201, 1131, 359, 313, 527, 268, 439, 885, 1083] payload_rows_per_dst=[10124, 11296, 11848, 8688, 9582, 7704, 10018, 8266] dup_rows_per_dst=[3196, 4354, 2836, 972, 2748, 1822, 3106, 1744] combine_rows_per_src=[10124, 11296, 11848, 8688, 9582, 7704, 10018, 8266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79278 payload_rows_remote=65348 dup_rows=19026 dup_rows_remote=15450 dedup_ratio=0.193542 router_tokens_per_expert=[1703, 4512, 73, 223, 1036, 155, 70, 82, 11, 425, 177, 566, 82, 335, 540, 804, 409, 707, 0, 8, 666, 93, 2208, 762, 2498, 3942, 820, 298, 177, 38, 45, 497, 0, 216, 1942, 0, 75, 1646, 1006, 310, 702, 168, 0, 68, 2325, 5788, 560, 0, 5765, 0, 1237, 747, 437, 126, 368, 73, 33, 374, 0, 0, 0, 680, 184, 360] payload_rows_per_dst=[5660, 11306, 8076, 7246, 8604, 11868, 13930, 12588] dup_rows_per_dst=[1024, 2458, 1630, 1500, 1786, 4802, 3576, 2250] combine_rows_per_src=[5660, 11306, 8076, 7246, 8604, 11868, 13930, 12588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74656 payload_rows_remote=62582 dup_rows=23648 dup_rows_remote=19148 dedup_ratio=0.240560 router_tokens_per_expert=[1587, 4635, 78, 208, 983, 163, 48, 87, 3, 248, 148, 396, 65, 322, 554, 872, 394, 877, 6, 8, 745, 87, 2821, 814, 2463, 3848, 839, 288, 156, 27, 26, 482, 5, 232, 1672, 0, 72, 1422, 971, 204, 690, 223, 0, 75, 2165, 5835, 341, 5, 5957, 8, 1812, 743, 456, 93, 370, 94, 13, 393, 5, 2, 4, 554, 115, 343] payload_rows_per_dst=[12074, 10870, 13862, 7084, 7690, 13454, 6788, 2834] dup_rows_per_dst=[4500, 2042, 8560, 1478, 1466, 5214, 364, 24] combine_rows_per_src=[12074, 10870, 13862, 7084, 7690, 13454, 6788, 2834] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79182 payload_rows_remote=69036 dup_rows=19122 dup_rows_remote=15440 dedup_ratio=0.194519 router_tokens_per_expert=[1394, 4752, 72, 189, 948, 146, 56, 40, 0, 216, 140, 307, 160, 314, 564, 777, 407, 669, 0, 1, 634, 100, 2663, 610, 2570, 3876, 639, 393, 54, 0, 7, 573, 0, 224, 2051, 0, 100, 1252, 942, 193, 741, 225, 0, 54, 2711, 6139, 490, 0, 5716, 0, 1487, 1031, 476, 132, 348, 60, 5, 364, 0, 0, 0, 628, 107, 405] payload_rows_per_dst=[4866, 10868, 8494, 9442, 8078, 10146, 14262, 13026] dup_rows_per_dst=[824, 1840, 1674, 3148, 1446, 3682, 4238, 2270] combine_rows_per_src=[4866, 10868, 8494, 9442, 8078, 10146, 14262, 13026] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=285835264 dispatch_meta_bytes=332720 combine_payload_bytes=285835264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286167984 ingress_bytes=320438272 link_bytes=320438272 token_bytes=572003248 weight_movement_bytes=34603008 total_nvlink_bytes=606606256 weight_to_token_ratio=0.060494 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=21760 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.205783420e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79576 payload_rows_remote=67238 dup_rows=18728 dup_rows_remote=16938 dedup_ratio=0.190511 router_tokens_per_expert=[1452, 3458, 64, 391, 1113, 198, 71, 145, 3, 348, 164, 255, 52, 326, 549, 668, 396, 772, 4, 17, 755, 102, 2821, 678, 2493, 4062, 765, 359, 195, 271, 70, 644, 1, 264, 2663, 0, 76, 1313, 962, 305, 632, 292, 2, 71, 2570, 5710, 624, 0, 5586, 2, 1399, 641, 461, 123, 344, 96, 5, 386, 4, 0, 0, 530, 114, 315] payload_rows_per_dst=[5852, 7758, 9186, 10226, 9174, 11474, 13568, 12338] dup_rows_per_dst=[1016, 1264, 1904, 3200, 1994, 3824, 3736, 1790] combine_rows_per_src=[5852, 7758, 9186, 10226, 9174, 11474, 13568, 12338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79404 payload_rows_remote=70586 dup_rows=18900 dup_rows_remote=17364 dedup_ratio=0.192261 router_tokens_per_expert=[2004, 6354, 18, 142, 597, 23, 62, 13, 0, 170, 70, 320, 95, 330, 443, 763, 402, 1004, 12, 5, 476, 107, 1913, 599, 3301, 4309, 943, 421, 4, 1, 72, 435, 1, 272, 1183, 1, 76, 1739, 797, 132, 665, 49, 1, 63, 1963, 6449, 58, 12, 5760, 14, 754, 824, 461, 83, 295, 58, 2, 338, 11, 0, 3, 1255, 137, 288] payload_rows_per_dst=[13886, 11342, 7420, 8818, 12876, 5322, 5952, 13788] dup_rows_per_dst=[4540, 1658, 1616, 1536, 5262, 300, 810, 3178] combine_rows_per_src=[13886, 11342, 7420, 8818, 12876, 5322, 5952, 13788] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74890 payload_rows_remote=65074 dup_rows=23414 dup_rows_remote=21374 dedup_ratio=0.238180 router_tokens_per_expert=[1191, 1442, 199, 531, 2469, 520, 290, 362, 349, 503, 527, 719, 161, 427, 607, 781, 414, 1309, 179, 226, 713, 94, 2714, 1525, 2353, 1854, 765, 128, 333, 346, 247, 807, 141, 350, 2187, 24, 97, 1144, 1395, 165, 847, 451, 25, 336, 1866, 3513, 455, 47, 4180, 206, 2654, 596, 537, 207, 329, 308, 164, 495, 79, 13, 5, 162, 497, 592] payload_rows_per_dst=[10348, 9816, 12752, 8154, 10076, 11040, 8820, 3884] dup_rows_per_dst=[3660, 2040, 6836, 1804, 4050, 4040, 854, 130] combine_rows_per_src=[10348, 9816, 12752, 8154, 10076, 11040, 8820, 3884] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80950 payload_rows_remote=65924 dup_rows=17354 dup_rows_remote=13802 dedup_ratio=0.176534 router_tokens_per_expert=[921, 181, 1, 3299, 76, 21, 249, 85, 0, 443, 1236, 803, 1096, 442, 505, 1609, 0, 7, 724, 16, 106, 641, 483, 2693, 147, 86, 543, 66, 394, 570, 0, 9, 91, 1600, 434, 30, 16, 0, 0, 2147, 1392, 267, 230, 2183, 4763, 246, 1287, 2677, 363, 536, 0, 672, 345, 1291, 6082, 0, 4224, 8, 0, 25, 357, 0, 350, 84] payload_rows_per_dst=[8450, 9670, 8142, 11918, 7078, 8718, 15026, 11948] dup_rows_per_dst=[1216, 2598, 1198, 1238, 1558, 2492, 3552, 3502] combine_rows_per_src=[8450, 9670, 8142, 11918, 7078, 8718, 15026, 11948] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=269017088 dispatch_meta_bytes=334480 combine_payload_bytes=269017088 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=269351568 ingress_bytes=407429120 link_bytes=407429120 token_bytes=538368656 weight_movement_bytes=138412032 total_nvlink_bytes=676780688 weight_to_token_ratio=0.257095 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=23552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.533122978e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=267665408 dispatch_meta_bytes=323192 combine_payload_bytes=267665408 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267988600 ingress_bytes=371474432 link_bytes=371474432 token_bytes=535654008 weight_movement_bytes=103809024 total_nvlink_bytes=639463032 weight_to_token_ratio=0.193799 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=23808 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397828381e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=256335872 dispatch_meta_bytes=326920 combine_payload_bytes=256335872 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=256662792 ingress_bytes=360144896 link_bytes=360144896 token_bytes=512998664 weight_movement_bytes=103809024 total_nvlink_bytes=616807688 weight_to_token_ratio=0.202357 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=21376 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.355196250e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=262160384 dispatch_meta_bytes=305968 combine_payload_bytes=262160384 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262466352 ingress_bytes=296763392 link_bytes=296763392 token_bytes=524626736 weight_movement_bytes=34603008 total_nvlink_bytes=559229744 weight_to_token_ratio=0.065957 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=28672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.116696753e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81430 payload_rows_remote=69784 dup_rows=16874 dup_rows_remote=13396 dedup_ratio=0.171651 router_tokens_per_expert=[677, 114, 0, 4593, 80, 0, 165, 97, 0, 258, 449, 426, 1004, 523, 444, 1152, 0, 0, 596, 2, 0, 519, 460, 1579, 107, 59, 260, 0, 82, 488, 0, 0, 97, 1947, 205, 4, 0, 0, 0, 3416, 1169, 82, 209, 1893, 5516, 158, 1553, 4086, 420, 89, 0, 1302, 110, 1706, 6371, 0, 4304, 0, 0, 0, 0, 0, 313, 68] payload_rows_per_dst=[10352, 6888, 6406, 12562, 11646, 10010, 12764, 10802] dup_rows_per_dst=[1100, 1624, 1256, 462, 3478, 3816, 3322, 1816] combine_rows_per_src=[10352, 6888, 6406, 12562, 11646, 10010, 12764, 10802] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=282771456 dispatch_meta_bytes=337904 combine_payload_bytes=282771456 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=283109360 ingress_bytes=351977472 link_bytes=351977472 token_bytes=565880816 weight_movement_bytes=69206016 total_nvlink_bytes=635086832 weight_to_token_ratio=0.122298 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.324462891e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=275406848 dispatch_meta_bytes=336704 combine_payload_bytes=275406848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275743552 ingress_bytes=310009856 link_bytes=310009856 token_bytes=551150400 weight_movement_bytes=34603008 total_nvlink_bytes=585753408 weight_to_token_ratio=0.062783 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=34944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.166542130e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74890 payload_rows_remote=65074 dup_rows=23414 dup_rows_remote=21374 dedup_ratio=0.238180 router_tokens_per_expert=[1191, 1442, 199, 531, 2469, 520, 290, 362, 349, 503, 527, 719, 161, 427, 607, 781, 414, 1309, 179, 226, 713, 94, 2714, 1525, 2353, 1854, 765, 128, 333, 346, 247, 807, 141, 350, 2187, 24, 97, 1144, 1395, 165, 847, 451, 25, 336, 1866, 3513, 455, 47, 4180, 206, 2654, 596, 537, 207, 329, 308, 164, 495, 79, 13, 5, 162, 497, 592] payload_rows_per_dst=[10348, 9816, 12752, 8154, 10076, 11040, 8820, 3884] dup_rows_per_dst=[3660, 2040, 6836, 1804, 4050, 4040, 854, 130] combine_rows_per_src=[10348, 9816, 12752, 8154, 10076, 11040, 8820, 3884] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=289120256 dispatch_meta_bytes=351800 combine_payload_bytes=289120256 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=289472056 ingress_bytes=358326272 link_bytes=358326272 token_bytes=578592312 weight_movement_bytes=69206016 total_nvlink_bytes=647798328 weight_to_token_ratio=0.119611 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=24320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.348352914e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79576 payload_rows_remote=67238 dup_rows=18728 dup_rows_remote=16938 dedup_ratio=0.190511 router_tokens_per_expert=[1452, 3458, 64, 391, 1113, 198, 71, 145, 3, 348, 164, 255, 52, 326, 549, 668, 396, 772, 4, 17, 755, 102, 2821, 678, 2493, 4062, 765, 359, 195, 271, 70, 644, 1, 264, 2663, 0, 76, 1313, 962, 305, 632, 292, 2, 71, 2570, 5710, 624, 0, 5586, 2, 1399, 641, 461, 123, 344, 96, 5, 386, 4, 0, 0, 530, 114, 315] payload_rows_per_dst=[5852, 7758, 9186, 10226, 9174, 11474, 13568, 12338] dup_rows_per_dst=[1016, 1264, 1904, 3200, 1994, 3824, 3736, 1790] combine_rows_per_src=[5852, 7758, 9186, 10226, 9174, 11474, 13568, 12338] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80266 payload_rows_remote=73400 dup_rows=18038 dup_rows_remote=16600 dedup_ratio=0.183492 router_tokens_per_expert=[2139, 6977, 0, 112, 601, 8, 53, 0, 0, 119, 54, 292, 96, 334, 451, 790, 391, 1118, 0, 0, 338, 77, 1606, 622, 4045, 4200, 1125, 372, 0, 0, 78, 312, 0, 278, 728, 0, 90, 1681, 722, 56, 704, 39, 0, 72, 1369, 6530, 14, 0, 5565, 0, 794, 1044, 456, 94, 247, 62, 0, 253, 0, 0, 0, 1566, 161, 317] payload_rows_per_dst=[14896, 11100, 6866, 10182, 13488, 7676, 4992, 11066] dup_rows_per_dst=[4884, 1572, 1438, 1682, 4752, 1092, 402, 2216] combine_rows_per_src=[14896, 11100, 6866, 10182, 13488, 7676, 4992, 11066] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=270024704 dispatch_meta_bytes=318904 combine_payload_bytes=270024704 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=270343608 ingress_bytes=373833728 link_bytes=373833728 token_bytes=540368312 weight_movement_bytes=103809024 total_nvlink_bytes=644177336 weight_to_token_ratio=0.192108 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=22016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.406706222e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=345792 combine_payload_bytes=266543104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266888896 ingress_bytes=301146112 link_bytes=301146112 token_bytes=533432000 weight_movement_bytes=34603008 total_nvlink_bytes=568035008 weight_to_token_ratio=0.064869 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=28416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133188575e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=285835264 dispatch_meta_bytes=332720 combine_payload_bytes=285835264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286167984 ingress_bytes=320438272 link_bytes=320438272 token_bytes=572003248 weight_movement_bytes=34603008 total_nvlink_bytes=606606256 weight_to_token_ratio=0.060494 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=21760 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.205783420e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=275406848 dispatch_meta_bytes=336704 combine_payload_bytes=275406848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275743552 ingress_bytes=310009856 link_bytes=310009856 token_bytes=551150400 weight_movement_bytes=34603008 total_nvlink_bytes=585753408 weight_to_token_ratio=0.062783 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=34944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.166542130e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=300646400 dispatch_meta_bytes=360000 combine_payload_bytes=300646400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=301006400 ingress_bytes=335249408 link_bytes=335249408 token_bytes=601652800 weight_movement_bytes=34603008 total_nvlink_bytes=636255808 weight_to_token_ratio=0.057513 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261516533e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74656 payload_rows_remote=62582 dup_rows=23648 dup_rows_remote=19148 dedup_ratio=0.240560 router_tokens_per_expert=[1587, 4635, 78, 208, 983, 163, 48, 87, 3, 248, 148, 396, 65, 322, 554, 872, 394, 877, 6, 8, 745, 87, 2821, 814, 2463, 3848, 839, 288, 156, 27, 26, 482, 5, 232, 1672, 0, 72, 1422, 971, 204, 690, 223, 0, 75, 2165, 5835, 341, 5, 5957, 8, 1812, 743, 456, 93, 370, 94, 13, 393, 5, 2, 4, 554, 115, 343] payload_rows_per_dst=[12074, 10870, 13862, 7084, 7690, 13454, 6788, 2834] dup_rows_per_dst=[4500, 2042, 8560, 1478, 1466, 5214, 364, 24] combine_rows_per_src=[12074, 10870, 13862, 7084, 7690, 13454, 6788, 2834] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78014 payload_rows_remote=71184 dup_rows=20290 dup_rows_remote=19558 dedup_ratio=0.206401 router_tokens_per_expert=[2392, 6909, 0, 134, 596, 0, 68, 0, 0, 132, 68, 301, 12, 341, 475, 725, 365, 1427, 0, 0, 364, 107, 1061, 671, 4488, 4131, 1173, 382, 0, 0, 72, 325, 0, 262, 812, 0, 80, 1882, 719, 26, 701, 22, 0, 59, 1394, 6437, 9, 0, 5724, 0, 475, 1023, 487, 75, 187, 35, 1, 177, 0, 0, 0, 1470, 141, 235] payload_rows_per_dst=[8256, 10896, 6418, 14156, 6830, 4126, 13466, 13866] dup_rows_per_dst=[2018, 1474, 1572, 8648, 732, 244, 2546, 3056] combine_rows_per_src=[8256, 10896, 6418, 14156, 6830, 4126, 13466, 13866] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79358 payload_rows_remote=68062 dup_rows=18946 dup_rows_remote=14914 dedup_ratio=0.192729 router_tokens_per_expert=[584, 434, 3744, 2989, 1499, 0, 6672, 5992, 0, 149, 0, 412, 0, 1299, 0, 235, 330, 368, 2178, 93, 905, 50, 363, 160, 45, 74, 977, 0, 4, 145, 0, 208, 5819, 609, 0, 75, 224, 1066, 755, 0, 409, 0, 98, 0, 0, 0, 230, 394, 2715, 0, 27, 193, 15, 1495, 68, 0, 2137, 4, 17, 1279, 1531, 83, 0, 0] payload_rows_per_dst=[4530, 9504, 11804, 13322, 3672, 14608, 11296, 10622] dup_rows_per_dst=[504, 2174, 3068, 1568, 1786, 998, 4032, 4816] combine_rows_per_src=[4530, 9504, 11804, 13322, 3672, 14608, 11296, 10622] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83548 payload_rows_remote=69246 dup_rows=14756 dup_rows_remote=9548 dedup_ratio=0.150106 router_tokens_per_expert=[547, 88, 0, 4639, 86, 0, 166, 74, 0, 278, 661, 420, 930, 735, 396, 1039, 0, 0, 531, 0, 9, 486, 434, 1921, 103, 72, 307, 0, 140, 378, 0, 0, 74, 1847, 208, 2, 0, 0, 0, 3230, 1240, 109, 177, 1937, 5453, 224, 1385, 3836, 377, 98, 0, 1410, 100, 1645, 6374, 0, 4582, 0, 0, 0, 0, 0, 352, 52] payload_rows_per_dst=[10382, 7290, 14302, 12448, 11538, 11032, 6712, 9844] dup_rows_per_dst=[818, 1628, 5208, 458, 3058, 2910, 548, 128] combine_rows_per_src=[10382, 7290, 14302, 12448, 11538, 11032, 6712, 9844] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=66686 payload_rows_remote=52268 dup_rows=31618 dup_rows_remote=10668 dedup_ratio=0.321635 router_tokens_per_expert=[626, 517, 3853, 2492, 1794, 7, 6334, 5914, 5, 144, 9, 392, 7, 1253, 1, 305, 342, 367, 2072, 113, 867, 51, 390, 160, 33, 73, 1223, 0, 10, 119, 9, 540, 5858, 588, 2, 74, 274, 1130, 721, 3, 394, 2, 57, 3, 0, 5, 143, 287, 2799, 2, 12, 204, 5, 1681, 89, 2, 2722, 18, 70, 1026, 893, 63, 1, 2] payload_rows_per_dst=[14418, 9610, 7532, 3438, 13496, 1266, 8572, 8354] dup_rows_per_dst=[20950, 2328, 1192, 576, 3804, 516, 1016, 1236] combine_rows_per_src=[14418, 9610, 7532, 3438, 13496, 1266, 8572, 8354] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79100 payload_rows_remote=65396 dup_rows=19204 dup_rows_remote=11954 dedup_ratio=0.195353 router_tokens_per_expert=[605, 533, 3468, 2779, 1957, 2, 6353, 6300, 3, 149, 1, 368, 4, 1408, 0, 181, 319, 355, 2422, 106, 903, 79, 390, 171, 42, 73, 1205, 1, 4, 138, 8, 260, 5842, 684, 0, 76, 337, 948, 670, 1, 398, 0, 95, 0, 0, 0, 273, 329, 2948, 6, 3, 247, 1, 976, 92, 4, 2260, 0, 61, 1055, 1202, 55, 0, 2] payload_rows_per_dst=[1976, 10158, 11922, 14110, 5190, 14214, 7826, 13704] dup_rows_per_dst=[490, 2880, 3126, 1952, 2096, 682, 728, 7250] combine_rows_per_src=[1976, 10158, 11922, 14110, 5190, 14214, 7826, 13704] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80668 payload_rows_remote=66844 dup_rows=17636 dup_rows_remote=17094 dedup_ratio=0.179403 router_tokens_per_expert=[587, 473, 4273, 2230, 1873, 0, 6299, 6028, 0, 199, 0, 506, 0, 1285, 0, 181, 305, 361, 2480, 136, 955, 49, 377, 121, 12, 99, 999, 0, 0, 155, 0, 460, 5849, 663, 0, 100, 236, 956, 754, 0, 413, 0, 22, 0, 0, 0, 79, 370, 2984, 0, 6, 179, 2, 1031, 58, 0, 2288, 0, 59, 1214, 1286, 160, 0, 0] payload_rows_per_dst=[5312, 10264, 11364, 13960, 3432, 13824, 14124, 8388] dup_rows_per_dst=[554, 2624, 2664, 1546, 1986, 542, 6094, 1626] combine_rows_per_src=[5312, 10264, 11364, 13960, 3432, 13824, 14124, 8388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=291569664 dispatch_meta_bytes=362968 combine_payload_bytes=291569664 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291932632 ingress_bytes=326172672 link_bytes=326172672 token_bytes=583502296 weight_movement_bytes=34603008 total_nvlink_bytes=618105304 weight_to_token_ratio=0.059302 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.227361506e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 1, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84100 payload_rows_remote=70692 dup_rows=14204 dup_rows_remote=13128 dedup_ratio=0.144491 router_tokens_per_expert=[544, 356, 4364, 1983, 2239, 1, 6284, 5707, 10, 155, 4, 431, 16, 1107, 0, 239, 305, 318, 2701, 86, 925, 53, 380, 137, 13, 77, 753, 14, 0, 137, 16, 525, 5927, 652, 0, 78, 255, 1496, 723, 4, 404, 4, 12, 5, 0, 10, 14, 284, 2700, 4, 2, 117, 9, 1092, 122, 8, 1742, 0, 15, 1719, 1748, 116, 10, 0] payload_rows_per_dst=[7158, 10508, 9682, 13408, 13696, 13512, 7438, 8698] dup_rows_per_dst=[1286, 2144, 1930, 1076, 4574, 522, 670, 2002] combine_rows_per_src=[7158, 10508, 9682, 13408, 13696, 13512, 7438, 8698] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79278 payload_rows_remote=65348 dup_rows=19026 dup_rows_remote=15450 dedup_ratio=0.193542 router_tokens_per_expert=[1703, 4512, 73, 223, 1036, 155, 70, 82, 11, 425, 177, 566, 82, 335, 540, 804, 409, 707, 0, 8, 666, 93, 2208, 762, 2498, 3942, 820, 298, 177, 38, 45, 497, 0, 216, 1942, 0, 75, 1646, 1006, 310, 702, 168, 0, 68, 2325, 5788, 560, 0, 5765, 0, 1237, 747, 437, 126, 368, 73, 33, 374, 0, 0, 0, 680, 184, 360] payload_rows_per_dst=[5660, 11306, 8076, 7246, 8604, 11868, 13930, 12588] dup_rows_per_dst=[1024, 2458, 1630, 1500, 1786, 4802, 3576, 2250] combine_rows_per_src=[5660, 11306, 8076, 7246, 8604, 11868, 13930, 12588] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72724 payload_rows_remote=63128 dup_rows=25580 dup_rows_remote=22516 dedup_ratio=0.260213 router_tokens_per_expert=[679, 1416, 2855, 1911, 857, 142, 5235, 4341, 142, 188, 95, 488, 195, 1699, 27, 641, 440, 451, 1776, 146, 860, 49, 622, 612, 59, 251, 1637, 59, 253, 151, 246, 1030, 4780, 479, 41, 172, 359, 1565, 699, 147, 457, 260, 842, 340, 191, 150, 475, 483, 1861, 26, 140, 396, 180, 937, 171, 167, 1486, 474, 176, 370, 1201, 132, 429, 13] payload_rows_per_dst=[13452, 9596, 8546, 6220, 12378, 8192, 6792, 7548] dup_rows_per_dst=[12156, 3064, 1366, 1152, 4106, 1758, 964, 1014] combine_rows_per_src=[13452, 9596, 8546, 6220, 12378, 8192, 6792, 7548] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278781952 dispatch_meta_bytes=331904 combine_payload_bytes=278781952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279113856 ingress_bytes=313384960 link_bytes=313384960 token_bytes=557895808 weight_movement_bytes=34603008 total_nvlink_bytes=592498816 weight_to_token_ratio=0.062024 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.179242375e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=283631616 dispatch_meta_bytes=315176 combine_payload_bytes=283631616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=283946792 ingress_bytes=318234624 link_bytes=318234624 token_bytes=567578408 weight_movement_bytes=34603008 total_nvlink_bytes=602181416 weight_to_token_ratio=0.060966 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=18048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.197491270e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72724 payload_rows_remote=63128 dup_rows=25580 dup_rows_remote=22516 dedup_ratio=0.260213 router_tokens_per_expert=[679, 1416, 2855, 1911, 857, 142, 5235, 4341, 142, 188, 95, 488, 195, 1699, 27, 641, 440, 451, 1776, 146, 860, 49, 622, 612, 59, 251, 1637, 59, 253, 151, 246, 1030, 4780, 479, 41, 172, 359, 1565, 699, 147, 457, 260, 842, 340, 191, 150, 475, 483, 1861, 26, 140, 396, 180, 937, 171, 167, 1486, 474, 176, 370, 1201, 132, 429, 13] payload_rows_per_dst=[13452, 9596, 8546, 6220, 12378, 8192, 6792, 7548] dup_rows_per_dst=[12156, 3064, 1366, 1152, 4106, 1758, 964, 1014] combine_rows_per_src=[13452, 9596, 8546, 6220, 12378, 8192, 6792, 7548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=214089728 dispatch_meta_bytes=251744 combine_payload_bytes=214089728 prefetch_bytes=0 reduce_bytes=207618048 egress_bytes=214341472 ingress_bytes=421707776 link_bytes=421707776 token_bytes=428431200 weight_movement_bytes=207618048 total_nvlink_bytes=636049248 weight_to_token_ratio=0.484601 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=12800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.586852411e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267862016 dispatch_meta_bytes=309400 combine_payload_bytes=267862016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268171416 ingress_bytes=302465024 link_bytes=302465024 token_bytes=536033432 weight_movement_bytes=34603008 total_nvlink_bytes=570636440 weight_to_token_ratio=0.064554 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138151535e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=256335872 dispatch_meta_bytes=326920 combine_payload_bytes=256335872 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=256662792 ingress_bytes=360144896 link_bytes=360144896 token_bytes=512998664 weight_movement_bytes=103809024 total_nvlink_bytes=616807688 weight_to_token_ratio=0.202357 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=21376 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.355196250e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78014 payload_rows_remote=71184 dup_rows=20290 dup_rows_remote=19558 dedup_ratio=0.206401 router_tokens_per_expert=[2392, 6909, 0, 134, 596, 0, 68, 0, 0, 132, 68, 301, 12, 341, 475, 725, 365, 1427, 0, 0, 364, 107, 1061, 671, 4488, 4131, 1173, 382, 0, 0, 72, 325, 0, 262, 812, 0, 80, 1882, 719, 26, 701, 22, 0, 59, 1394, 6437, 9, 0, 5724, 0, 475, 1023, 487, 75, 187, 35, 1, 177, 0, 0, 0, 1470, 141, 235] payload_rows_per_dst=[8256, 10896, 6418, 14156, 6830, 4126, 13466, 13866] dup_rows_per_dst=[2018, 1474, 1572, 8648, 732, 244, 2546, 3056] combine_rows_per_src=[8256, 10896, 6418, 14156, 6830, 4126, 13466, 13866] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=273793024 dispatch_meta_bytes=335752 combine_payload_bytes=273793024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274128776 ingress_bytes=308396032 link_bytes=308396032 token_bytes=547921800 weight_movement_bytes=34603008 total_nvlink_bytes=582524808 weight_to_token_ratio=0.063153 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=39936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.160469440e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79100 payload_rows_remote=65396 dup_rows=19204 dup_rows_remote=11954 dedup_ratio=0.195353 router_tokens_per_expert=[605, 533, 3468, 2779, 1957, 2, 6353, 6300, 3, 149, 1, 368, 4, 1408, 0, 181, 319, 355, 2422, 106, 903, 79, 390, 171, 42, 73, 1205, 1, 4, 138, 8, 260, 5842, 684, 0, 76, 337, 948, 670, 1, 398, 0, 95, 0, 0, 0, 273, 329, 2948, 6, 3, 247, 1, 976, 92, 4, 2260, 0, 61, 1055, 1202, 55, 0, 2] payload_rows_per_dst=[1976, 10158, 11922, 14110, 5190, 14214, 7826, 13704] dup_rows_per_dst=[490, 2880, 3126, 1952, 2096, 682, 728, 7250] combine_rows_per_src=[1976, 10158, 11922, 14110, 5190, 14214, 7826, 13704] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=289554432 dispatch_meta_bytes=335280 combine_payload_bytes=289554432 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289889712 ingress_bytes=324157440 link_bytes=324157440 token_bytes=579444144 weight_movement_bytes=34603008 total_nvlink_bytes=614047152 weight_to_token_ratio=0.059718 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=34176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.219778350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83362 payload_rows_remote=74550 dup_rows=14942 dup_rows_remote=13774 dedup_ratio=0.151998 router_tokens_per_expert=[581, 255, 4052, 1697, 2398, 0, 6175, 5125, 0, 176, 0, 498, 0, 1115, 0, 191, 341, 335, 2752, 91, 912, 45, 377, 137, 11, 89, 642, 0, 0, 143, 0, 555, 6099, 646, 0, 90, 292, 1704, 711, 0, 428, 0, 0, 0, 0, 0, 0, 276, 2806, 0, 1, 123, 0, 1251, 245, 0, 2008, 1, 12, 1993, 1635, 138, 0, 0] payload_rows_per_dst=[8530, 10000, 8812, 11936, 13968, 13228, 7860, 9028] dup_rows_per_dst=[1686, 2064, 1168, 840, 5116, 530, 992, 2546] combine_rows_per_src=[8530, 10000, 8812, 11936, 13968, 13228, 7860, 9028] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=267665408 dispatch_meta_bytes=323192 combine_payload_bytes=267665408 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267988600 ingress_bytes=371474432 link_bytes=371474432 token_bytes=535654008 weight_movement_bytes=103809024 total_nvlink_bytes=639463032 weight_to_token_ratio=0.193799 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=23808 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397828381e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=258572288 dispatch_meta_bytes=342576 combine_payload_bytes=258572288 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=258914864 ingress_bytes=327778304 link_bytes=327778304 token_bytes=517487152 weight_movement_bytes=69206016 total_nvlink_bytes=586693168 weight_to_token_ratio=0.133735 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=30464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.233403370e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=291569664 dispatch_meta_bytes=362968 combine_payload_bytes=291569664 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291932632 ingress_bytes=326172672 link_bytes=326172672 token_bytes=583502296 weight_movement_bytes=34603008 total_nvlink_bytes=618105304 weight_to_token_ratio=0.059302 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.227361506e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267862016 dispatch_meta_bytes=309400 combine_payload_bytes=267862016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268171416 ingress_bytes=302465024 link_bytes=302465024 token_bytes=536033432 weight_movement_bytes=34603008 total_nvlink_bytes=570636440 weight_to_token_ratio=0.064554 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138151535e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=305356800 dispatch_meta_bytes=353296 combine_payload_bytes=305356800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=305710096 ingress_bytes=339959808 link_bytes=339959808 token_bytes=611066896 weight_movement_bytes=34603008 total_nvlink_bytes=645669904 weight_to_token_ratio=0.056627 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=19200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.279241388e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=66686 payload_rows_remote=52268 dup_rows=31618 dup_rows_remote=10668 dedup_ratio=0.321635 router_tokens_per_expert=[626, 517, 3853, 2492, 1794, 7, 6334, 5914, 5, 144, 9, 392, 7, 1253, 1, 305, 342, 367, 2072, 113, 867, 51, 390, 160, 33, 73, 1223, 0, 10, 119, 9, 540, 5858, 588, 2, 74, 274, 1130, 721, 3, 394, 2, 57, 3, 0, 5, 143, 287, 2799, 2, 12, 204, 5, 1681, 89, 2, 2722, 18, 70, 1026, 893, 63, 1, 2] payload_rows_per_dst=[14418, 9610, 7532, 3438, 13496, 1266, 8572, 8354] dup_rows_per_dst=[20950, 2328, 1192, 576, 3804, 516, 1016, 1236] combine_rows_per_src=[14418, 9610, 7532, 3438, 13496, 1266, 8572, 8354] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81038 payload_rows_remote=74550 dup_rows=17266 dup_rows_remote=14750 dedup_ratio=0.175639 router_tokens_per_expert=[548, 169, 3742, 1952, 1934, 0, 6268, 5113, 0, 160, 0, 408, 0, 1107, 0, 276, 317, 380, 2734, 83, 982, 43, 421, 152, 13, 80, 649, 0, 0, 148, 0, 597, 6029, 703, 0, 80, 266, 2256, 752, 0, 423, 0, 0, 0, 0, 0, 0, 265, 3097, 0, 0, 87, 0, 1059, 123, 0, 1609, 0, 14, 1987, 2045, 81, 0, 0] payload_rows_per_dst=[4888, 9456, 11338, 12426, 6488, 13420, 13998, 9024] dup_rows_per_dst=[414, 1930, 2790, 774, 2516, 492, 5902, 2448] combine_rows_per_src=[4888, 9456, 11338, 12426, 6488, 13420, 13998, 9024] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76416 payload_rows_remote=60936 dup_rows=21888 dup_rows_remote=20256 dedup_ratio=0.222656 router_tokens_per_expert=[0, 7301, 2397, 0, 19, 70, 446, 4465, 0, 0, 0, 0, 0, 626, 2509, 169, 160, 86, 82, 341, 84, 0, 241, 1093, 885, 252, 0, 404, 251, 70, 204, 3, 53, 7268, 84, 0, 1419, 0, 1817, 0, 561, 0, 0, 0, 2642, 251, 0, 0, 75, 0, 81, 514, 26, 485, 74, 0, 0, 363, 6056, 309, 222, 2501, 0, 2193] payload_rows_per_dst=[5762, 6604, 3538, 3748, 15078, 12132, 15480, 14074] dup_rows_per_dst=[102, 4, 636, 938, 6204, 3706, 1632, 8666] combine_rows_per_src=[5762, 6604, 3538, 3748, 15078, 12132, 15480, 14074] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80266 payload_rows_remote=73400 dup_rows=18038 dup_rows_remote=16600 dedup_ratio=0.183492 router_tokens_per_expert=[2139, 6977, 0, 112, 601, 8, 53, 0, 0, 119, 54, 292, 96, 334, 451, 790, 391, 1118, 0, 0, 338, 77, 1606, 622, 4045, 4200, 1125, 372, 0, 0, 78, 312, 0, 278, 728, 0, 90, 1681, 722, 56, 704, 39, 0, 72, 1369, 6530, 14, 0, 5565, 0, 794, 1044, 456, 94, 247, 62, 0, 253, 0, 0, 0, 1566, 161, 317] payload_rows_per_dst=[14896, 11100, 6866, 10182, 13488, 7676, 4992, 11066] dup_rows_per_dst=[4884, 1572, 1438, 1682, 4752, 1092, 402, 2216] combine_rows_per_src=[14896, 11100, 6866, 10182, 13488, 7676, 4992, 11066] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76966 payload_rows_remote=63176 dup_rows=21338 dup_rows_remote=11836 dedup_ratio=0.217061 router_tokens_per_expert=[1, 7301, 2351, 1, 66, 50, 410, 4856, 0, 7, 2, 2, 1, 698, 2925, 135, 56, 93, 63, 343, 64, 6, 198, 1060, 475, 262, 2, 412, 256, 47, 186, 0, 49, 7298, 84, 8, 1454, 6, 1601, 0, 683, 3, 11, 5, 2267, 258, 5, 9, 77, 7, 158, 496, 19, 522, 72, 8, 0, 385, 5802, 220, 232, 3091, 1, 1992] payload_rows_per_dst=[13790, 14930, 15538, 12890, 5744, 6344, 2418, 5312] dup_rows_per_dst=[9502, 5572, 2824, 1994, 660, 138, 300, 348] combine_rows_per_src=[13790, 14930, 15538, 12890, 5744, 6344, 2418, 5312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69700 payload_rows_remote=54084 dup_rows=28604 dup_rows_remote=10576 dedup_ratio=0.290975 router_tokens_per_expert=[2, 7162, 1779, 0, 55, 66, 388, 5147, 0, 8, 0, 1, 5, 552, 2574, 169, 250, 82, 53, 375, 86, 1, 253, 1067, 737, 240, 2, 396, 240, 64, 206, 1, 53, 7262, 98, 2, 1223, 0, 1626, 0, 649, 3, 7, 1, 2710, 240, 4, 1, 75, 5, 106, 523, 77, 532, 71, 5, 3, 388, 5947, 342, 200, 2467, 0, 2571] payload_rows_per_dst=[4580, 6578, 3548, 3008, 8302, 12888, 15180, 15616] dup_rows_per_dst=[0, 40, 786, 764, 2418, 4636, 1932, 18028] combine_rows_per_src=[4580, 6578, 3548, 3008, 8302, 12888, 15180, 15616] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80892 payload_rows_remote=68134 dup_rows=17412 dup_rows_remote=12484 dedup_ratio=0.177124 router_tokens_per_expert=[0, 7277, 2150, 0, 64, 55, 418, 5058, 0, 0, 0, 0, 0, 686, 2538, 169, 49, 101, 160, 329, 67, 0, 173, 1054, 907, 247, 0, 281, 234, 55, 319, 0, 58, 7338, 105, 0, 1337, 0, 1897, 0, 691, 0, 0, 0, 2847, 247, 0, 0, 100, 0, 183, 470, 43, 479, 95, 0, 0, 376, 5639, 369, 256, 2366, 0, 1865] payload_rows_per_dst=[5296, 6740, 3534, 12976, 15264, 12758, 15470, 8854] dup_rows_per_dst=[78, 46, 332, 2388, 6206, 4928, 1824, 1610] combine_rows_per_src=[5296, 6740, 3534, 12976, 15264, 12758, 15470, 8854] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 1, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=305356800 dispatch_meta_bytes=357200 combine_payload_bytes=305356800 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=305714000 ingress_bytes=409165824 link_bytes=409165824 token_bytes=611070800 weight_movement_bytes=103809024 total_nvlink_bytes=714879824 weight_to_token_ratio=0.169881 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.539658055e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=249593856 dispatch_meta_bytes=324768 combine_payload_bytes=249593856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=249918624 ingress_bytes=284196864 link_bytes=284196864 token_bytes=499512480 weight_movement_bytes=34603008 total_nvlink_bytes=534115488 weight_to_token_ratio=0.069274 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=37888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.069409920e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85126 payload_rows_remote=71826 dup_rows=13178 dup_rows_remote=10884 dedup_ratio=0.134054 router_tokens_per_expert=[2, 7485, 1983, 2, 1, 62, 387, 4630, 0, 16, 4, 0, 1, 803, 2607, 146, 1, 84, 116, 327, 88, 9, 130, 938, 820, 240, 2, 240, 238, 61, 277, 0, 116, 7366, 79, 9, 2570, 1, 1735, 1, 733, 0, 20, 0, 2803, 246, 0, 18, 77, 17, 118, 380, 5, 400, 61, 17, 3, 374, 5919, 898, 246, 2055, 0, 1185] payload_rows_per_dst=[10156, 6958, 15594, 13300, 7894, 10438, 15728, 5058] dup_rows_per_dst=[2470, 196, 2524, 2294, 1128, 2820, 1392, 354] combine_rows_per_src=[10156, 6958, 15594, 13300, 7894, 10438, 15728, 5058] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79358 payload_rows_remote=68062 dup_rows=18946 dup_rows_remote=14914 dedup_ratio=0.192729 router_tokens_per_expert=[584, 434, 3744, 2989, 1499, 0, 6672, 5992, 0, 149, 0, 412, 0, 1299, 0, 235, 330, 368, 2178, 93, 905, 50, 363, 160, 45, 74, 977, 0, 4, 145, 0, 208, 5819, 609, 0, 75, 224, 1066, 755, 0, 409, 0, 98, 0, 0, 0, 230, 394, 2715, 0, 27, 193, 15, 1495, 68, 0, 2137, 4, 17, 1279, 1531, 83, 0, 0] payload_rows_per_dst=[4530, 9504, 11804, 13322, 3672, 14608, 11296, 10622] dup_rows_per_dst=[504, 2174, 3068, 1568, 1786, 998, 4032, 4816] combine_rows_per_src=[4530, 9504, 11804, 13322, 3672, 14608, 11296, 10622] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75792 payload_rows_remote=61462 dup_rows=22512 dup_rows_remote=15950 dedup_ratio=0.229004 router_tokens_per_expert=[37, 5884, 3964, 192, 191, 167, 334, 2321, 31, 552, 131, 79, 58, 620, 2904, 187, 379, 143, 391, 284, 128, 197, 321, 1013, 425, 332, 416, 679, 256, 52, 475, 7, 327, 5018, 208, 83, 602, 379, 1988, 8, 238, 55, 497, 173, 1171, 328, 46, 216, 43, 221, 161, 686, 471, 675, 295, 361, 371, 429, 4057, 255, 370, 2819, 507, 2944] payload_rows_per_dst=[12502, 14330, 12730, 11084, 6374, 4762, 5150, 8860] dup_rows_per_dst=[7548, 6562, 3018, 2314, 816, 686, 676, 892] combine_rows_per_src=[12502, 14330, 12730, 11084, 6374, 4762, 5150, 8860] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=214089728 dispatch_meta_bytes=251744 combine_payload_bytes=214089728 prefetch_bytes=0 reduce_bytes=207618048 egress_bytes=214341472 ingress_bytes=421707776 link_bytes=421707776 token_bytes=428431200 weight_movement_bytes=207618048 total_nvlink_bytes=636049248 weight_to_token_ratio=0.484601 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=12800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.586852411e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=300646400 dispatch_meta_bytes=360000 combine_payload_bytes=300646400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=301006400 ingress_bytes=335249408 link_bytes=335249408 token_bytes=601652800 weight_movement_bytes=34603008 total_nvlink_bytes=636255808 weight_to_token_ratio=0.057513 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261516533e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=221528064 dispatch_meta_bytes=258640 combine_payload_bytes=221528064 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=221786704 ingress_bytes=325337088 link_bytes=325337088 token_bytes=443314768 weight_movement_bytes=103809024 total_nvlink_bytes=547123792 weight_to_token_ratio=0.234165 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.224217270e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=258768896 dispatch_meta_bytes=300048 combine_payload_bytes=258768896 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=259068944 ingress_bytes=397180928 link_bytes=397180928 token_bytes=517837840 weight_movement_bytes=138412032 total_nvlink_bytes=656249872 weight_to_token_ratio=0.267288 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.494559856e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81038 payload_rows_remote=74550 dup_rows=17266 dup_rows_remote=14750 dedup_ratio=0.175639 router_tokens_per_expert=[548, 169, 3742, 1952, 1934, 0, 6268, 5113, 0, 160, 0, 408, 0, 1107, 0, 276, 317, 380, 2734, 83, 982, 43, 421, 152, 13, 80, 649, 0, 0, 148, 0, 597, 6029, 703, 0, 80, 266, 2256, 752, 0, 423, 0, 0, 0, 0, 0, 0, 265, 3097, 0, 0, 87, 0, 1059, 123, 0, 1609, 0, 14, 1987, 2045, 81, 0, 0] payload_rows_per_dst=[4888, 9456, 11338, 12426, 6488, 13420, 13998, 9024] dup_rows_per_dst=[414, 1930, 2790, 774, 2516, 492, 5902, 2448] combine_rows_per_src=[4888, 9456, 11338, 12426, 6488, 13420, 13998, 9024] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=279076864 dispatch_meta_bytes=322472 combine_payload_bytes=279076864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279399336 ingress_bytes=313679872 link_bytes=313679872 token_bytes=558476200 weight_movement_bytes=34603008 total_nvlink_bytes=593079208 weight_to_token_ratio=0.061960 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=21632 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.180352105e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69700 payload_rows_remote=54084 dup_rows=28604 dup_rows_remote=10576 dedup_ratio=0.290975 router_tokens_per_expert=[2, 7162, 1779, 0, 55, 66, 388, 5147, 0, 8, 0, 1, 5, 552, 2574, 169, 250, 82, 53, 375, 86, 1, 253, 1067, 737, 240, 2, 396, 240, 64, 206, 1, 53, 7262, 98, 2, 1223, 0, 1626, 0, 649, 3, 7, 1, 2710, 240, 4, 1, 75, 5, 106, 523, 77, 532, 71, 5, 3, 388, 5947, 342, 200, 2467, 0, 2571] payload_rows_per_dst=[4580, 6578, 3548, 3008, 8302, 12888, 15180, 15616] dup_rows_per_dst=[0, 40, 786, 764, 2418, 4636, 1932, 18028] combine_rows_per_src=[4580, 6578, 3548, 3008, 8302, 12888, 15180, 15616] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80526 payload_rows_remote=64614 dup_rows=17778 dup_rows_remote=15348 dedup_ratio=0.180847 router_tokens_per_expert=[0, 7561, 1868, 0, 3, 53, 442, 4403, 0, 0, 0, 0, 0, 888, 3100, 153, 0, 94, 119, 339, 54, 0, 24, 941, 717, 265, 0, 262, 192, 53, 300, 0, 83, 7600, 90, 0, 3121, 0, 1559, 0, 731, 0, 0, 0, 2576, 265, 0, 0, 90, 0, 153, 378, 0, 312, 58, 0, 0, 374, 5941, 970, 214, 1890, 0, 916] payload_rows_per_dst=[12406, 15060, 15912, 13376, 8600, 6962, 3516, 4694] dup_rows_per_dst=[4912, 6432, 2430, 2084, 1106, 182, 378, 254] combine_rows_per_src=[12406, 15060, 15912, 13376, 8600, 6962, 3516, 4694] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278781952 dispatch_meta_bytes=331904 combine_payload_bytes=278781952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279113856 ingress_bytes=313384960 link_bytes=313384960 token_bytes=557895808 weight_movement_bytes=34603008 total_nvlink_bytes=592498816 weight_to_token_ratio=0.062024 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.179242375e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=294199296 dispatch_meta_bytes=330840 combine_payload_bytes=294199296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=294530136 ingress_bytes=328802304 link_bytes=328802304 token_bytes=588729432 weight_movement_bytes=34603008 total_nvlink_bytes=623332440 weight_to_token_ratio=0.058776 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=33664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.237256599e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=251748352 dispatch_meta_bytes=309648 combine_payload_bytes=251748352 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=252058000 ingress_bytes=286351360 link_bytes=286351360 token_bytes=503806352 weight_movement_bytes=34603008 total_nvlink_bytes=538409360 weight_to_token_ratio=0.068683 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=22272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.077517115e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=305356800 dispatch_meta_bytes=357200 combine_payload_bytes=305356800 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=305714000 ingress_bytes=409165824 link_bytes=409165824 token_bytes=611070800 weight_movement_bytes=103809024 total_nvlink_bytes=714879824 weight_to_token_ratio=0.169881 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.539658055e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=221528064 dispatch_meta_bytes=258640 combine_payload_bytes=221528064 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=221786704 ingress_bytes=325337088 link_bytes=325337088 token_bytes=443314768 weight_movement_bytes=103809024 total_nvlink_bytes=547123792 weight_to_token_ratio=0.234165 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.224217270e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=264658944 dispatch_meta_bytes=319848 combine_payload_bytes=264658944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264978792 ingress_bytes=299261952 link_bytes=299261952 token_bytes=529637736 weight_movement_bytes=34603008 total_nvlink_bytes=564240744 weight_to_token_ratio=0.065333 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=33408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126098633e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76966 payload_rows_remote=63176 dup_rows=21338 dup_rows_remote=11836 dedup_ratio=0.217061 router_tokens_per_expert=[1, 7301, 2351, 1, 66, 50, 410, 4856, 0, 7, 2, 2, 1, 698, 2925, 135, 56, 93, 63, 343, 64, 6, 198, 1060, 475, 262, 2, 412, 256, 47, 186, 0, 49, 7298, 84, 8, 1454, 6, 1601, 0, 683, 3, 11, 5, 2267, 258, 5, 9, 77, 7, 158, 496, 19, 522, 72, 8, 0, 385, 5802, 220, 232, 3091, 1, 1992] payload_rows_per_dst=[13790, 14930, 15538, 12890, 5744, 6344, 2418, 5312] dup_rows_per_dst=[9502, 5572, 2824, 1994, 660, 138, 300, 348] combine_rows_per_src=[13790, 14930, 15538, 12890, 5744, 6344, 2418, 5312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86610 payload_rows_remote=74668 dup_rows=11694 dup_rows_remote=9072 dedup_ratio=0.118958 router_tokens_per_expert=[0, 0, 0, 348, 0, 0, 0, 0, 405, 82, 1447, 0, 757, 325, 0, 108, 68, 0, 6, 0, 0, 84, 0, 0, 0, 4883, 0, 510, 259, 1036, 0, 341, 1012, 3474, 0, 0, 0, 1916, 0, 986, 2030, 2071, 3969, 0, 557, 539, 876, 0, 0, 84, 0, 3140, 21, 0, 68, 0, 840, 0, 5185, 5936, 4710, 930, 0, 149] payload_rows_per_dst=[11004, 5230, 12176, 12726, 11850, 10140, 11942, 11542] dup_rows_per_dst=[62, 1018, 12, 1332, 2926, 2006, 2622, 1716] combine_rows_per_src=[11004, 5230, 12176, 12726, 11850, 10140, 11942, 11542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85686 payload_rows_remote=75670 dup_rows=12618 dup_rows_remote=10124 dedup_ratio=0.128357 router_tokens_per_expert=[0, 7580, 2274, 0, 0, 68, 503, 4110, 0, 0, 0, 0, 0, 892, 3081, 153, 0, 82, 81, 344, 76, 0, 38, 966, 668, 247, 0, 199, 151, 68, 259, 0, 120, 7564, 80, 0, 3291, 0, 1293, 0, 798, 0, 0, 0, 2307, 247, 0, 0, 80, 0, 145, 395, 0, 363, 51, 0, 0, 418, 5981, 1146, 263, 1781, 0, 989] payload_rows_per_dst=[8274, 7956, 13248, 13384, 10016, 11588, 15920, 5300] dup_rows_per_dst=[926, 296, 2112, 1762, 2494, 3336, 1308, 384] combine_rows_per_src=[8274, 7956, 13248, 13384, 10016, 11588, 15920, 5300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83362 payload_rows_remote=74550 dup_rows=14942 dup_rows_remote=13774 dedup_ratio=0.151998 router_tokens_per_expert=[581, 255, 4052, 1697, 2398, 0, 6175, 5125, 0, 176, 0, 498, 0, 1115, 0, 191, 341, 335, 2752, 91, 912, 45, 377, 137, 11, 89, 642, 0, 0, 143, 0, 555, 6099, 646, 0, 90, 292, 1704, 711, 0, 428, 0, 0, 0, 0, 0, 0, 276, 2806, 0, 1, 123, 0, 1251, 245, 0, 2008, 1, 12, 1993, 1635, 138, 0, 0] payload_rows_per_dst=[8530, 10000, 8812, 11936, 13968, 13228, 7860, 9028] dup_rows_per_dst=[1686, 2064, 1168, 840, 5116, 530, 992, 2546] combine_rows_per_src=[8530, 10000, 8812, 11936, 13968, 13228, 7860, 9028] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84626 payload_rows_remote=71440 dup_rows=13678 dup_rows_remote=10300 dedup_ratio=0.139140 router_tokens_per_expert=[4, 1, 0, 334, 0, 1, 0, 5, 394, 80, 1122, 5, 787, 313, 3, 90, 71, 0, 9, 0, 7, 85, 0, 3, 1, 4792, 0, 303, 237, 966, 3, 262, 1819, 2524, 0, 2, 0, 1622, 0, 906, 1571, 1823, 4516, 0, 568, 517, 623, 0, 0, 81, 8, 3612, 199, 3, 71, 1, 789, 9, 5505, 5879, 5402, 1077, 0, 147] payload_rows_per_dst=[9912, 4536, 12088, 11880, 11054, 8686, 13284, 13186] dup_rows_per_dst=[72, 1052, 20, 1248, 2692, 1518, 3698, 3378] combine_rows_per_src=[9912, 4536, 12088, 11880, 11054, 8686, 13284, 13186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84774 payload_rows_remote=72286 dup_rows=13530 dup_rows_remote=13444 dedup_ratio=0.137634 router_tokens_per_expert=[0, 1, 2, 286, 1, 0, 0, 9, 367, 51, 1160, 12, 695, 330, 6, 85, 75, 2, 3, 0, 6, 64, 0, 3, 1, 4948, 3, 243, 224, 1182, 2, 291, 1002, 3048, 0, 4, 1, 2074, 3, 914, 1967, 1652, 4096, 0, 558, 438, 1042, 1, 0, 62, 11, 2726, 120, 7, 76, 4, 896, 10, 5988, 6166, 5063, 1025, 0, 146] payload_rows_per_dst=[12488, 12106, 12892, 9630, 9352, 11526, 10544, 6236] dup_rows_per_dst=[86, 3202, 1078, 358, 2578, 3138, 1904, 1186] combine_rows_per_src=[12488, 12106, 12892, 9630, 9352, 11526, 10544, 6236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=309944320 dispatch_meta_bytes=343176 combine_payload_bytes=309944320 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=310287496 ingress_bytes=379150336 link_bytes=379150336 token_bytes=620231816 weight_movement_bytes=69206016 total_nvlink_bytes=689437832 weight_to_token_ratio=0.111581 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.426712190e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=305840128 dispatch_meta_bytes=334960 combine_payload_bytes=305840128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=306175088 ingress_bytes=340443136 link_bytes=340443136 token_bytes=612015216 weight_movement_bytes=34603008 total_nvlink_bytes=646618224 weight_to_token_ratio=0.056539 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=27008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.281060113e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87458 payload_rows_remote=78174 dup_rows=10846 dup_rows_remote=9060 dedup_ratio=0.110331 router_tokens_per_expert=[0, 0, 0, 331, 0, 0, 0, 0, 369, 98, 1511, 0, 598, 346, 0, 100, 54, 0, 1, 0, 0, 67, 0, 0, 0, 5586, 0, 284, 228, 1447, 0, 323, 1448, 3209, 0, 0, 0, 1515, 0, 886, 2011, 1500, 3552, 0, 555, 567, 902, 0, 0, 160, 0, 2614, 52, 0, 54, 0, 538, 0, 5889, 6055, 5246, 857, 0, 199] payload_rows_per_dst=[12366, 5056, 12352, 13364, 11638, 9284, 10986, 12412] dup_rows_per_dst=[74, 988, 2, 2372, 2478, 1786, 1878, 1268] combine_rows_per_src=[12366, 5056, 12352, 13364, 11638, 9284, 10986, 12412] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76416 payload_rows_remote=60936 dup_rows=21888 dup_rows_remote=20256 dedup_ratio=0.222656 router_tokens_per_expert=[0, 7301, 2397, 0, 19, 70, 446, 4465, 0, 0, 0, 0, 0, 626, 2509, 169, 160, 86, 82, 341, 84, 0, 241, 1093, 885, 252, 0, 404, 251, 70, 204, 3, 53, 7268, 84, 0, 1419, 0, 1817, 0, 561, 0, 0, 0, 2642, 251, 0, 0, 75, 0, 81, 514, 26, 485, 74, 0, 0, 363, 6056, 309, 222, 2501, 0, 2193] payload_rows_per_dst=[5762, 6604, 3538, 3748, 15078, 12132, 15480, 14074] dup_rows_per_dst=[102, 4, 636, 938, 6204, 3706, 1632, 8666] combine_rows_per_src=[5762, 6604, 3538, 3748, 15078, 12132, 15480, 14074] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82286 payload_rows_remote=70528 dup_rows=16018 dup_rows_remote=12196 dedup_ratio=0.162944 router_tokens_per_expert=[223, 155, 0, 494, 311, 80, 27, 410, 412, 120, 686, 272, 1464, 490, 210, 150, 155, 135, 36, 2, 375, 96, 88, 471, 66, 3986, 26, 426, 449, 1551, 477, 555, 596, 1321, 1, 201, 10, 1800, 84, 1257, 2347, 3968, 3547, 97, 715, 392, 1007, 94, 7, 185, 355, 2480, 344, 446, 272, 85, 934, 355, 3994, 3231, 3101, 1343, 2, 183] payload_rows_per_dst=[10558, 11758, 8460, 8064, 9986, 11126, 12166, 10168] dup_rows_per_dst=[830, 3822, 718, 1678, 2606, 1420, 3276, 1668] combine_rows_per_src=[10558, 11758, 8460, 8064, 9986, 11126, 12166, 10168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88136 payload_rows_remote=77098 dup_rows=10168 dup_rows_remote=9448 dedup_ratio=0.103434 router_tokens_per_expert=[0, 0, 0, 343, 3, 0, 1, 21, 388, 166, 1437, 12, 642, 315, 7, 76, 62, 0, 0, 0, 11, 89, 0, 7, 1, 5650, 5, 334, 252, 1399, 16, 280, 715, 3592, 0, 2, 0, 1550, 5, 887, 2895, 1147, 3323, 3, 558, 1103, 1142, 2, 0, 116, 21, 1774, 6, 4, 63, 0, 603, 17, 6278, 6212, 4908, 558, 0, 151] payload_rows_per_dst=[13224, 13126, 12758, 11038, 5822, 10974, 9576, 11618] dup_rows_per_dst=[68, 4260, 4, 720, 496, 2726, 1038, 856] combine_rows_per_src=[13224, 13126, 12758, 11038, 5822, 10974, 9576, 11618] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 2, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82286 payload_rows_remote=70528 dup_rows=16018 dup_rows_remote=12196 dedup_ratio=0.162944 router_tokens_per_expert=[223, 155, 0, 494, 311, 80, 27, 410, 412, 120, 686, 272, 1464, 490, 210, 150, 155, 135, 36, 2, 375, 96, 88, 471, 66, 3986, 26, 426, 449, 1551, 477, 555, 596, 1321, 1, 201, 10, 1800, 84, 1257, 2347, 3968, 3547, 97, 715, 392, 1007, 94, 7, 185, 355, 2480, 344, 446, 272, 85, 934, 355, 3994, 3231, 3101, 1343, 2, 183] payload_rows_per_dst=[10558, 11758, 8460, 8064, 9986, 11126, 12166, 10168] dup_rows_per_dst=[830, 3822, 718, 1678, 2606, 1420, 3276, 1668] combine_rows_per_src=[10558, 11758, 8460, 8064, 9986, 11126, 12166, 10168] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=258768896 dispatch_meta_bytes=300048 combine_payload_bytes=258768896 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=259068944 ingress_bytes=397180928 link_bytes=397180928 token_bytes=517837840 weight_movement_bytes=138412032 total_nvlink_bytes=656249872 weight_to_token_ratio=0.267288 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.494559856e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=305356800 dispatch_meta_bytes=353296 combine_payload_bytes=305356800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=305710096 ingress_bytes=339959808 link_bytes=339959808 token_bytes=611066896 weight_movement_bytes=34603008 total_nvlink_bytes=645669904 weight_to_token_ratio=0.056627 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=19200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.279241388e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=292618240 dispatch_meta_bytes=326960 combine_payload_bytes=292618240 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=292945200 ingress_bytes=431030272 link_bytes=431030272 token_bytes=585563440 weight_movement_bytes=138412032 total_nvlink_bytes=723975472 weight_to_token_ratio=0.236374 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=34176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.621932213e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=296083456 dispatch_meta_bytes=342920 combine_payload_bytes=296083456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296426376 ingress_bytes=330686464 link_bytes=330686464 token_bytes=592509832 weight_movement_bytes=34603008 total_nvlink_bytes=627112840 weight_to_token_ratio=0.058401 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=45184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.244346542e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=320200704 dispatch_meta_bytes=348936 combine_payload_bytes=320200704 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=320549640 ingress_bytes=424009728 link_bytes=424009728 token_bytes=640750344 weight_movement_bytes=103809024 total_nvlink_bytes=744559368 weight_to_token_ratio=0.162012 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.595514471e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85686 payload_rows_remote=75670 dup_rows=12618 dup_rows_remote=10124 dedup_ratio=0.128357 router_tokens_per_expert=[0, 7580, 2274, 0, 0, 68, 503, 4110, 0, 0, 0, 0, 0, 892, 3081, 153, 0, 82, 81, 344, 76, 0, 38, 966, 668, 247, 0, 199, 151, 68, 259, 0, 120, 7564, 80, 0, 3291, 0, 1293, 0, 798, 0, 0, 0, 2307, 247, 0, 0, 80, 0, 145, 395, 0, 363, 51, 0, 0, 418, 5981, 1146, 263, 1781, 0, 989] payload_rows_per_dst=[8274, 7956, 13248, 13384, 10016, 11588, 15920, 5300] dup_rows_per_dst=[926, 296, 2112, 1762, 2494, 3336, 1308, 384] combine_rows_per_src=[8274, 7956, 13248, 13384, 10016, 11588, 15920, 5300] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84626 payload_rows_remote=71440 dup_rows=13678 dup_rows_remote=10300 dedup_ratio=0.139140 router_tokens_per_expert=[4, 1, 0, 334, 0, 1, 0, 5, 394, 80, 1122, 5, 787, 313, 3, 90, 71, 0, 9, 0, 7, 85, 0, 3, 1, 4792, 0, 303, 237, 966, 3, 262, 1819, 2524, 0, 2, 0, 1622, 0, 906, 1571, 1823, 4516, 0, 568, 517, 623, 0, 0, 81, 8, 3612, 199, 3, 71, 1, 789, 9, 5505, 5879, 5402, 1077, 0, 147] payload_rows_per_dst=[9912, 4536, 12088, 11880, 11054, 8686, 13284, 13186] dup_rows_per_dst=[72, 1052, 20, 1248, 2692, 1518, 3698, 3378] combine_rows_per_src=[9912, 4536, 12088, 11880, 11054, 8686, 13284, 13186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=249593856 dispatch_meta_bytes=324768 combine_payload_bytes=249593856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=249918624 ingress_bytes=284196864 link_bytes=284196864 token_bytes=499512480 weight_movement_bytes=34603008 total_nvlink_bytes=534115488 weight_to_token_ratio=0.069274 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=37888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.069409920e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88036 payload_rows_remote=75060 dup_rows=10268 dup_rows_remote=10268 dedup_ratio=0.104451 router_tokens_per_expert=[2, 0, 0, 293, 0, 0, 0, 0, 392, 156, 1845, 0, 581, 354, 0, 89, 70, 0, 0, 0, 0, 74, 0, 0, 0, 5561, 0, 298, 205, 1795, 0, 2, 532, 4102, 0, 0, 0, 1602, 0, 894, 3311, 771, 2416, 0, 555, 1136, 1526, 0, 0, 138, 0, 1268, 0, 0, 72, 0, 687, 0, 6682, 6344, 4795, 427, 0, 177] payload_rows_per_dst=[13912, 13116, 12976, 11498, 10514, 7548, 7256, 11216] dup_rows_per_dst=[42, 4840, 0, 1306, 1902, 690, 532, 956] combine_rows_per_src=[13912, 13116, 12976, 11498, 10514, 7548, 7256, 11216] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=288882688 dispatch_meta_bytes=330896 combine_payload_bytes=288882688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289213584 ingress_bytes=323485696 link_bytes=323485696 token_bytes=578096272 weight_movement_bytes=34603008 total_nvlink_bytes=612699280 weight_to_token_ratio=0.059857 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=24064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.217250631e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=315793408 dispatch_meta_bytes=346184 combine_payload_bytes=315793408 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=316139592 ingress_bytes=384999424 link_bytes=384999424 token_bytes=631933000 weight_movement_bytes=69206016 total_nvlink_bytes=701139016 weight_to_token_ratio=0.109515 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.448721838e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 2, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=309944320 dispatch_meta_bytes=343176 combine_payload_bytes=309944320 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=310287496 ingress_bytes=379150336 link_bytes=379150336 token_bytes=620231816 weight_movement_bytes=69206016 total_nvlink_bytes=689437832 weight_to_token_ratio=0.111581 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.426712190e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=292618240 dispatch_meta_bytes=326960 combine_payload_bytes=292618240 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=292945200 ingress_bytes=431030272 link_bytes=431030272 token_bytes=585563440 weight_movement_bytes=138412032 total_nvlink_bytes=723975472 weight_to_token_ratio=0.236374 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=34176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.621932213e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=307445760 dispatch_meta_bytes=341312 combine_payload_bytes=307445760 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=307787072 ingress_bytes=376651776 link_bytes=376651776 token_bytes=615232832 weight_movement_bytes=69206016 total_nvlink_bytes=684438848 weight_to_token_ratio=0.112488 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=46080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.417310310e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84774 payload_rows_remote=72286 dup_rows=13530 dup_rows_remote=13444 dedup_ratio=0.137634 router_tokens_per_expert=[0, 1, 2, 286, 1, 0, 0, 9, 367, 51, 1160, 12, 695, 330, 6, 85, 75, 2, 3, 0, 6, 64, 0, 3, 1, 4948, 3, 243, 224, 1182, 2, 291, 1002, 3048, 0, 4, 1, 2074, 3, 914, 1967, 1652, 4096, 0, 558, 438, 1042, 1, 0, 62, 11, 2726, 120, 7, 76, 4, 896, 10, 5988, 6166, 5063, 1025, 0, 146] payload_rows_per_dst=[12488, 12106, 12892, 9630, 9352, 11526, 10544, 6236] dup_rows_per_dst=[86, 3202, 1078, 358, 2578, 3138, 1904, 1186] combine_rows_per_src=[12488, 12106, 12892, 9630, 9352, 11526, 10544, 6236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=288882688 dispatch_meta_bytes=330896 combine_payload_bytes=288882688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289213584 ingress_bytes=323485696 link_bytes=323485696 token_bytes=578096272 weight_movement_bytes=34603008 total_nvlink_bytes=612699280 weight_to_token_ratio=0.059857 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=24064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.217250631e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84990 payload_rows_remote=71678 dup_rows=13314 dup_rows_remote=7910 dedup_ratio=0.135437 router_tokens_per_expert=[0, 70, 270, 85, 20, 4898, 0, 439, 0, 0, 18, 0, 75, 0, 0, 0, 1888, 0, 0, 0, 0, 1626, 0, 1854, 0, 16, 7, 1442, 0, 0, 0, 70, 5426, 0, 68, 339, 0, 0, 158, 1191, 265, 0, 0, 1326, 84, 0, 0, 8, 398, 0, 5271, 80, 5955, 70, 4205, 0, 15, 2520, 3229, 623, 2359, 2636, 0, 148] payload_rows_per_dst=[11494, 12064, 9394, 4292, 12232, 9116, 13312, 13086] dup_rows_per_dst=[70, 32, 1342, 110, 2132, 708, 5404, 3516] combine_rows_per_src=[11494, 12064, 9394, 4292, 12232, 9116, 13312, 13086] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87900 payload_rows_remote=77158 dup_rows=10404 dup_rows_remote=8736 dedup_ratio=0.105835 router_tokens_per_expert=[0, 0, 0, 334, 0, 0, 0, 0, 412, 153, 1673, 0, 646, 326, 0, 80, 59, 0, 1, 0, 0, 97, 0, 0, 0, 5821, 0, 362, 272, 1783, 0, 49, 477, 4395, 0, 0, 0, 1486, 0, 912, 3311, 815, 2448, 0, 629, 1128, 1985, 0, 0, 81, 0, 1213, 0, 0, 59, 0, 728, 0, 6469, 6227, 4301, 260, 0, 160] payload_rows_per_dst=[13588, 10900, 12766, 9756, 10742, 12602, 7184, 10362] dup_rows_per_dst=[18, 3294, 2, 1334, 1668, 3134, 418, 536] combine_rows_per_src=[13588, 10900, 12766, 9756, 10742, 12602, 7184, 10362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80526 payload_rows_remote=64614 dup_rows=17778 dup_rows_remote=15348 dedup_ratio=0.180847 router_tokens_per_expert=[0, 7561, 1868, 0, 3, 53, 442, 4403, 0, 0, 0, 0, 0, 888, 3100, 153, 0, 94, 119, 339, 54, 0, 24, 941, 717, 265, 0, 262, 192, 53, 300, 0, 83, 7600, 90, 0, 3121, 0, 1559, 0, 731, 0, 0, 0, 2576, 265, 0, 0, 90, 0, 153, 378, 0, 312, 58, 0, 0, 374, 5941, 970, 214, 1890, 0, 916] payload_rows_per_dst=[12406, 15060, 15912, 13376, 8600, 6962, 3516, 4694] dup_rows_per_dst=[4912, 6432, 2430, 2084, 1106, 182, 378, 254] combine_rows_per_src=[12406, 15060, 15912, 13376, 8600, 6962, 3516, 4694] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80144 payload_rows_remote=66920 dup_rows=18160 dup_rows_remote=13378 dedup_ratio=0.184733 router_tokens_per_expert=[1, 64, 302, 91, 58, 5499, 2, 425, 0, 0, 47, 2, 74, 7, 0, 3, 1942, 0, 0, 0, 0, 1655, 1, 1348, 0, 52, 25, 1398, 0, 0, 2, 64, 5235, 56, 73, 448, 0, 7, 158, 1379, 297, 6, 10, 1022, 85, 1, 9, 6, 402, 11, 4870, 87, 5858, 64, 4327, 4, 24, 3165, 3448, 628, 2198, 2083, 1, 128] payload_rows_per_dst=[12656, 11890, 8866, 3064, 6352, 9110, 14982, 13224] dup_rows_per_dst=[228, 92, 1026, 18, 970, 658, 10386, 4782] combine_rows_per_src=[12656, 11890, 8866, 3064, 6352, 9110, 14982, 13224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84558 payload_rows_remote=70242 dup_rows=13746 dup_rows_remote=8884 dedup_ratio=0.139832 router_tokens_per_expert=[0, 47, 333, 68, 17, 5388, 4, 449, 0, 0, 42, 0, 73, 12, 3, 3, 1883, 2, 0, 0, 4, 1217, 4, 1518, 1, 49, 10, 1363, 2, 1, 4, 51, 5591, 56, 73, 289, 0, 5, 136, 1133, 247, 12, 14, 1100, 64, 0, 12, 4, 406, 1, 5005, 76, 6069, 47, 4344, 2, 27, 3318, 3199, 673, 2165, 2393, 9, 134] payload_rows_per_dst=[14316, 12444, 12628, 12292, 6858, 11482, 9196, 5342] dup_rows_per_dst=[4862, 434, 3264, 680, 1142, 2608, 82, 674] combine_rows_per_src=[14316, 12444, 12628, 12292, 6858, 11482, 9196, 5342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=293593088 dispatch_meta_bytes=318352 combine_payload_bytes=293593088 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=293911440 ingress_bytes=432005120 link_bytes=432005120 token_bytes=587504528 weight_movement_bytes=138412032 total_nvlink_bytes=725916560 weight_to_token_ratio=0.235593 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=31232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.625600487e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89684 payload_rows_remote=79460 dup_rows=8620 dup_rows_remote=7982 dedup_ratio=0.087687 router_tokens_per_expert=[0, 55, 248, 67, 24, 4858, 0, 468, 0, 0, 38, 0, 100, 0, 0, 0, 1551, 0, 0, 0, 0, 1563, 0, 1480, 0, 50, 7, 1684, 0, 0, 0, 55, 5077, 38, 100, 451, 0, 0, 166, 1093, 242, 0, 0, 1156, 67, 0, 0, 0, 357, 0, 4654, 102, 6164, 55, 4679, 0, 17, 2861, 3966, 685, 2203, 2513, 0, 258] payload_rows_per_dst=[11402, 12530, 8238, 11916, 11800, 10224, 10370, 13204] dup_rows_per_dst=[38, 74, 950, 984, 2050, 638, 16, 3870] combine_rows_per_src=[11402, 12530, 8238, 11916, 11800, 10224, 10370, 13204] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86610 payload_rows_remote=74668 dup_rows=11694 dup_rows_remote=9072 dedup_ratio=0.118958 router_tokens_per_expert=[0, 0, 0, 348, 0, 0, 0, 0, 405, 82, 1447, 0, 757, 325, 0, 108, 68, 0, 6, 0, 0, 84, 0, 0, 0, 4883, 0, 510, 259, 1036, 0, 341, 1012, 3474, 0, 0, 0, 1916, 0, 986, 2030, 2071, 3969, 0, 557, 539, 876, 0, 0, 84, 0, 3140, 21, 0, 68, 0, 840, 0, 5185, 5936, 4710, 930, 0, 149] payload_rows_per_dst=[11004, 5230, 12176, 12726, 11850, 10140, 11942, 11542] dup_rows_per_dst=[62, 1018, 12, 1332, 2926, 2006, 2622, 1716] combine_rows_per_src=[11004, 5230, 12176, 12726, 11850, 10140, 11942, 11542] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81134 payload_rows_remote=72524 dup_rows=17170 dup_rows_remote=16232 dedup_ratio=0.174662 router_tokens_per_expert=[113, 170, 488, 361, 245, 3329, 430, 685, 5, 141, 57, 106, 307, 666, 102, 153, 433, 0, 41, 0, 54, 938, 124, 1596, 108, 489, 507, 817, 219, 140, 12, 142, 5005, 326, 77, 531, 10, 427, 154, 1568, 282, 479, 899, 1203, 99, 74, 595, 402, 391, 548, 3240, 242, 3237, 54, 3515, 32, 216, 3678, 3386, 605, 1402, 2897, 153, 447] payload_rows_per_dst=[10396, 8610, 11404, 9952, 12762, 13790, 8982, 5238] dup_rows_per_dst=[1246, 938, 2324, 1396, 3434, 6842, 582, 408] combine_rows_per_src=[10396, 8610, 11404, 9952, 12762, 13790, 8982, 5238] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=316039168 dispatch_meta_bytes=343576 combine_payload_bytes=316039168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=316382744 ingress_bytes=385245184 link_bytes=385245184 token_bytes=632421912 weight_movement_bytes=69206016 total_nvlink_bytes=701627928 weight_to_token_ratio=0.109430 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.449646613e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88284 payload_rows_remote=75204 dup_rows=10020 dup_rows_remote=7210 dedup_ratio=0.101929 router_tokens_per_expert=[2, 61, 243, 113, 16, 4284, 4, 395, 0, 2, 82, 9, 78, 12, 0, 0, 2343, 1, 0, 0, 10, 1863, 5, 1435, 0, 0, 47, 3404, 0, 1, 0, 68, 5232, 36, 82, 353, 1, 18, 164, 805, 225, 14, 24, 1267, 88, 2, 22, 0, 555, 0, 4425, 76, 6407, 61, 3975, 7, 10, 2206, 3575, 619, 1706, 2519, 7, 193] payload_rows_per_dst=[10198, 13014, 9878, 13080, 11800, 9668, 9020, 11626] dup_rows_per_dst=[38, 166, 1436, 2810, 1582, 766, 328, 2894] combine_rows_per_src=[10198, 13014, 9878, 13080, 11800, 9668, 9020, 11626] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=296083456 dispatch_meta_bytes=342920 combine_payload_bytes=296083456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296426376 ingress_bytes=330686464 link_bytes=330686464 token_bytes=592509832 weight_movement_bytes=34603008 total_nvlink_bytes=627112840 weight_to_token_ratio=0.058401 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=45184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.244346542e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81134 payload_rows_remote=72524 dup_rows=17170 dup_rows_remote=16232 dedup_ratio=0.174662 router_tokens_per_expert=[113, 170, 488, 361, 245, 3329, 430, 685, 5, 141, 57, 106, 307, 666, 102, 153, 433, 0, 41, 0, 54, 938, 124, 1596, 108, 489, 507, 817, 219, 140, 12, 142, 5005, 326, 77, 531, 10, 427, 154, 1568, 282, 479, 899, 1203, 99, 74, 595, 402, 391, 548, 3240, 242, 3237, 54, 3515, 32, 216, 3678, 3386, 605, 1402, 2897, 153, 447] payload_rows_per_dst=[10396, 8610, 11404, 9952, 12762, 13790, 8982, 5238] dup_rows_per_dst=[1246, 938, 2324, 1396, 3434, 6842, 582, 408] combine_rows_per_src=[10396, 8610, 11404, 9952, 12762, 13790, 8982, 5238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=264658944 dispatch_meta_bytes=319848 combine_payload_bytes=264658944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264978792 ingress_bytes=299261952 link_bytes=299261952 token_bytes=529637736 weight_movement_bytes=34603008 total_nvlink_bytes=564240744 weight_to_token_ratio=0.065333 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=33408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126098633e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=287711232 dispatch_meta_bytes=316504 combine_payload_bytes=287711232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288027736 ingress_bytes=322314240 link_bytes=322314240 token_bytes=575738968 weight_movement_bytes=34603008 total_nvlink_bytes=610341976 weight_to_token_ratio=0.060102 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=37248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.212842537e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=274104320 dispatch_meta_bytes=321192 combine_payload_bytes=274104320 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=274425512 ingress_bytes=412516352 link_bytes=412516352 token_bytes=548529832 weight_movement_bytes=138412032 total_nvlink_bytes=686941864 weight_to_token_ratio=0.252333 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=15104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.552265822e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=325468160 dispatch_meta_bytes=349768 combine_payload_bytes=325468160 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=325817928 ingress_bytes=394674176 link_bytes=394674176 token_bytes=651286088 weight_movement_bytes=69206016 total_nvlink_bytes=720492104 weight_to_token_ratio=0.106261 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.030000 gemm_active_groups=11 gemm_peak_group_rows=27392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.485127150e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80144 payload_rows_remote=66920 dup_rows=18160 dup_rows_remote=13378 dedup_ratio=0.184733 router_tokens_per_expert=[1, 64, 302, 91, 58, 5499, 2, 425, 0, 0, 47, 2, 74, 7, 0, 3, 1942, 0, 0, 0, 0, 1655, 1, 1348, 0, 52, 25, 1398, 0, 0, 2, 64, 5235, 56, 73, 448, 0, 7, 158, 1379, 297, 6, 10, 1022, 85, 1, 9, 6, 402, 11, 4870, 87, 5858, 64, 4327, 4, 24, 3165, 3448, 628, 2198, 2083, 1, 128] payload_rows_per_dst=[12656, 11890, 8866, 3064, 6352, 9110, 14982, 13224] dup_rows_per_dst=[228, 92, 1026, 18, 970, 658, 10386, 4782] combine_rows_per_src=[12656, 11890, 8866, 3064, 6352, 9110, 14982, 13224] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87900 payload_rows_remote=77158 dup_rows=10404 dup_rows_remote=8736 dedup_ratio=0.105835 router_tokens_per_expert=[0, 0, 0, 334, 0, 0, 0, 0, 412, 153, 1673, 0, 646, 326, 0, 80, 59, 0, 1, 0, 0, 97, 0, 0, 0, 5821, 0, 362, 272, 1783, 0, 49, 477, 4395, 0, 0, 0, 1486, 0, 912, 3311, 815, 2448, 0, 629, 1128, 1985, 0, 0, 81, 0, 1213, 0, 0, 59, 0, 728, 0, 6469, 6227, 4301, 260, 0, 160] payload_rows_per_dst=[13588, 10900, 12766, 9756, 10742, 12602, 7184, 10362] dup_rows_per_dst=[18, 3294, 2, 1334, 1668, 3134, 418, 536] combine_rows_per_src=[13588, 10900, 12766, 9756, 10742, 12602, 7184, 10362] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=305840128 dispatch_meta_bytes=334960 combine_payload_bytes=305840128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=306175088 ingress_bytes=340443136 link_bytes=340443136 token_bytes=612015216 weight_movement_bytes=34603008 total_nvlink_bytes=646618224 weight_to_token_ratio=0.056539 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=27008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.281060113e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=297058304 dispatch_meta_bytes=355024 combine_payload_bytes=297058304 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=297413328 ingress_bytes=366264320 link_bytes=366264320 token_bytes=594471632 weight_movement_bytes=69206016 total_nvlink_bytes=663677648 weight_to_token_ratio=0.116416 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=46464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.378223149e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=308035584 dispatch_meta_bytes=329656 combine_payload_bytes=308035584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=308365240 ingress_bytes=342638592 link_bytes=342638592 token_bytes=616400824 weight_movement_bytes=34603008 total_nvlink_bytes=651003832 weight_to_token_ratio=0.056137 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=27264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.289321437e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86356 payload_rows_remote=74348 dup_rows=11948 dup_rows_remote=9306 dedup_ratio=0.121541 router_tokens_per_expert=[0, 53, 271, 80, 7, 4465, 0, 432, 0, 0, 64, 0, 90, 0, 0, 0, 2152, 0, 0, 0, 0, 1917, 0, 1446, 0, 0, 15, 4506, 0, 0, 0, 53, 5032, 16, 90, 293, 0, 0, 163, 742, 215, 0, 0, 1536, 52, 0, 0, 0, 583, 0, 4408, 88, 6396, 53, 3375, 0, 5, 1821, 3500, 681, 1686, 2640, 0, 226] payload_rows_per_dst=[10586, 12972, 12008, 13818, 11268, 10952, 7834, 6918] dup_rows_per_dst=[30, 128, 2642, 4146, 1404, 2034, 364, 1200] combine_rows_per_src=[10586, 12972, 12008, 13818, 11268, 10952, 7834, 6918] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=274104320 dispatch_meta_bytes=321192 combine_payload_bytes=274104320 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=274425512 ingress_bytes=412516352 link_bytes=412516352 token_bytes=548529832 weight_movement_bytes=138412032 total_nvlink_bytes=686941864 weight_to_token_ratio=0.252333 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=15104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.552265822e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=316039168 dispatch_meta_bytes=343576 combine_payload_bytes=316039168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=316382744 ingress_bytes=385245184 link_bytes=385245184 token_bytes=632421912 weight_movement_bytes=69206016 total_nvlink_bytes=701627928 weight_to_token_ratio=0.109430 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.449646613e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=304529408 dispatch_meta_bytes=334616 combine_payload_bytes=304529408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=304864024 ingress_bytes=339132416 link_bytes=339132416 token_bytes=609393432 weight_movement_bytes=34603008 total_nvlink_bytes=643996440 weight_to_token_ratio=0.056783 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=14976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.276127979e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84558 payload_rows_remote=70242 dup_rows=13746 dup_rows_remote=8884 dedup_ratio=0.139832 router_tokens_per_expert=[0, 47, 333, 68, 17, 5388, 4, 449, 0, 0, 42, 0, 73, 12, 3, 3, 1883, 2, 0, 0, 4, 1217, 4, 1518, 1, 49, 10, 1363, 2, 1, 4, 51, 5591, 56, 73, 289, 0, 5, 136, 1133, 247, 12, 14, 1100, 64, 0, 12, 4, 406, 1, 5005, 76, 6069, 47, 4344, 2, 27, 3318, 3199, 673, 2165, 2393, 9, 134] payload_rows_per_dst=[14316, 12444, 12628, 12292, 6858, 11482, 9196, 5342] dup_rows_per_dst=[4862, 434, 3264, 680, 1142, 2608, 82, 674] combine_rows_per_src=[14316, 12444, 12628, 12292, 6858, 11482, 9196, 5342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=297058304 dispatch_meta_bytes=355024 combine_payload_bytes=297058304 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=297413328 ingress_bytes=366264320 link_bytes=366264320 token_bytes=594471632 weight_movement_bytes=69206016 total_nvlink_bytes=663677648 weight_to_token_ratio=0.116416 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=46464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.378223149e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88182 payload_rows_remote=72468 dup_rows=10122 dup_rows_remote=7348 dedup_ratio=0.102966 router_tokens_per_expert=[0, 0, 70, 0, 70, 0, 0, 0, 0, 2480, 0, 158, 75, 467, 52, 1352, 0, 7214, 0, 185, 0, 0, 68, 0, 7418, 0, 275, 251, 0, 1014, 74, 5548, 7556, 3638, 0, 358, 0, 0, 0, 0, 95, 67, 3836, 0, 0, 1918, 251, 0, 89, 0, 0, 1599, 0, 0, 0, 0, 0, 2825, 82, 0, 0, 0, 0, 67] payload_rows_per_dst=[10318, 7798, 14796, 6840, 7130, 11484, 15714, 14102] dup_rows_per_dst=[140, 1370, 138, 1046, 0, 1712, 2774, 2942] combine_rows_per_src=[10318, 7798, 14796, 6840, 7130, 11484, 15714, 14102] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86772 payload_rows_remote=75670 dup_rows=11532 dup_rows_remote=10116 dedup_ratio=0.117310 router_tokens_per_expert=[0, 68, 247, 98, 0, 4499, 0, 460, 0, 0, 48, 0, 80, 0, 0, 0, 2673, 0, 0, 0, 0, 2075, 0, 1704, 0, 0, 31, 5004, 0, 0, 0, 68, 4981, 1, 80, 259, 0, 0, 177, 761, 245, 0, 0, 1537, 72, 0, 0, 0, 595, 0, 4770, 80, 6330, 68, 2980, 0, 0, 1506, 3000, 653, 1324, 2517, 0, 161] payload_rows_per_dst=[10742, 12824, 11218, 14482, 11102, 8902, 7078, 10424] dup_rows_per_dst=[2, 92, 1686, 5264, 1416, 806, 368, 1898] combine_rows_per_src=[10742, 12824, 11218, 14482, 11102, 8902, 7078, 10424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78456 payload_rows_remote=63398 dup_rows=19848 dup_rows_remote=19746 dedup_ratio=0.201904 router_tokens_per_expert=[10, 1, 47, 1, 47, 0, 0, 0, 5, 2779, 0, 137, 74, 214, 52, 1291, 0, 7220, 8, 75, 0, 0, 81, 6, 7474, 15, 225, 256, 7, 1113, 72, 5553, 7541, 3508, 11, 167, 5, 0, 0, 1, 247, 79, 3314, 11, 0, 1779, 259, 10, 256, 0, 0, 1551, 1, 0, 0, 1, 6, 3488, 63, 0, 1, 13, 0, 77] payload_rows_per_dst=[15058, 15720, 1114, 14288, 11110, 10268, 3618, 7280] dup_rows_per_dst=[102, 7824, 10, 6484, 4282, 1130, 0, 16] combine_rows_per_src=[15058, 15720, 1114, 14288, 11110, 10268, 3618, 7280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=296828928 dispatch_meta_bytes=319264 combine_payload_bytes=296828928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=297148192 ingress_bytes=331431936 link_bytes=331431936 token_bytes=593977120 weight_movement_bytes=34603008 total_nvlink_bytes=628580128 weight_to_token_ratio=0.058256 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=33152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.247151693e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88036 payload_rows_remote=75060 dup_rows=10268 dup_rows_remote=10268 dedup_ratio=0.104451 router_tokens_per_expert=[2, 0, 0, 293, 0, 0, 0, 0, 392, 156, 1845, 0, 581, 354, 0, 89, 70, 0, 0, 0, 0, 74, 0, 0, 0, 5561, 0, 298, 205, 1795, 0, 2, 532, 4102, 0, 0, 0, 1602, 0, 894, 3311, 771, 2416, 0, 555, 1136, 1526, 0, 0, 138, 0, 1268, 0, 0, 72, 0, 687, 0, 6682, 6344, 4795, 427, 0, 177] payload_rows_per_dst=[13912, 13116, 12976, 11498, 10514, 7548, 7256, 11216] dup_rows_per_dst=[42, 4840, 0, 1306, 1902, 690, 532, 956] combine_rows_per_src=[13912, 13116, 12976, 11498, 10514, 7548, 7256, 11216] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82886 payload_rows_remote=68946 dup_rows=15418 dup_rows_remote=12014 dedup_ratio=0.156840 router_tokens_per_expert=[14, 1, 64, 0, 64, 0, 0, 0, 1, 2380, 0, 157, 74, 414, 2, 1310, 4, 7139, 5, 132, 0, 0, 71, 1, 7417, 8, 186, 249, 1, 771, 73, 5103, 7546, 4568, 6, 268, 5, 0, 0, 2, 368, 65, 3470, 4, 0, 1906, 243, 9, 143, 0, 0, 1339, 0, 0, 0, 0, 2, 3427, 62, 0, 4, 6, 0, 68] payload_rows_per_dst=[158, 7546, 14498, 15370, 554, 15012, 15808, 13940] dup_rows_per_dst=[128, 1130, 206, 2040, 8, 6254, 2248, 3404] combine_rows_per_src=[158, 7546, 14498, 15370, 554, 15012, 15808, 13940] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88410 payload_rows_remote=79450 dup_rows=9894 dup_rows_remote=8854 dedup_ratio=0.100647 router_tokens_per_expert=[0, 0, 55, 0, 55, 0, 0, 0, 0, 2972, 0, 259, 100, 290, 30, 1523, 1, 7111, 0, 142, 0, 0, 54, 0, 7386, 0, 205, 247, 0, 1140, 99, 5597, 7529, 3751, 0, 202, 0, 0, 0, 0, 139, 48, 2646, 0, 0, 1920, 247, 0, 205, 0, 0, 1875, 0, 0, 0, 0, 0, 3117, 160, 0, 0, 0, 0, 47] payload_rows_per_dst=[14882, 8780, 14460, 3184, 7906, 8960, 15952, 14286] dup_rows_per_dst=[110, 1568, 156, 198, 0, 1040, 3266, 3556] combine_rows_per_src=[14882, 8780, 14460, 3184, 7906, 8960, 15952, 14286] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84990 payload_rows_remote=71678 dup_rows=13314 dup_rows_remote=7910 dedup_ratio=0.135437 router_tokens_per_expert=[0, 70, 270, 85, 20, 4898, 0, 439, 0, 0, 18, 0, 75, 0, 0, 0, 1888, 0, 0, 0, 0, 1626, 0, 1854, 0, 16, 7, 1442, 0, 0, 0, 70, 5426, 0, 68, 339, 0, 0, 158, 1191, 265, 0, 0, 1326, 84, 0, 0, 8, 398, 0, 5271, 80, 5955, 70, 4205, 0, 15, 2520, 3229, 623, 2359, 2636, 0, 148] payload_rows_per_dst=[11494, 12064, 9394, 4292, 12232, 9116, 13312, 13086] dup_rows_per_dst=[70, 32, 1342, 110, 2132, 708, 5404, 3516] combine_rows_per_src=[11494, 12064, 9394, 4292, 12232, 9116, 13312, 13086] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80858 payload_rows_remote=70268 dup_rows=17446 dup_rows_remote=13140 dedup_ratio=0.177470 router_tokens_per_expert=[107, 78, 74, 90, 70, 78, 18, 1, 156, 1984, 13, 218, 168, 1405, 401, 579, 29, 5411, 268, 524, 31, 0, 558, 71, 5405, 467, 242, 418, 78, 1736, 148, 4357, 5889, 1798, 556, 472, 173, 156, 2, 208, 794, 186, 3665, 598, 26, 2283, 466, 873, 121, 62, 14, 1343, 80, 14, 54, 107, 289, 2544, 122, 4, 283, 624, 34, 129] payload_rows_per_dst=[11648, 10590, 8492, 12134, 14220, 13142, 3400, 7232] dup_rows_per_dst=[194, 4306, 244, 2758, 4288, 4640, 190, 826] combine_rows_per_src=[11648, 10590, 8492, 12134, 14220, 13142, 3400, 7232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=309944320 dispatch_meta_bytes=343144 combine_payload_bytes=309944320 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=310287464 ingress_bytes=413753344 link_bytes=413753344 token_bytes=620231784 weight_movement_bytes=103809024 total_nvlink_bytes=724040808 weight_to_token_ratio=0.167371 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=34688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.556920524e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87832 payload_rows_remote=75000 dup_rows=10472 dup_rows_remote=8642 dedup_ratio=0.106527 router_tokens_per_expert=[61, 1, 61, 1, 61, 0, 0, 0, 3, 3258, 0, 192, 77, 77, 89, 1419, 6, 7170, 12, 85, 0, 0, 65, 2, 7421, 18, 313, 239, 10, 1243, 76, 5738, 7421, 3967, 4, 238, 18, 12, 0, 1, 132, 37, 1888, 15, 1, 1955, 245, 18, 130, 0, 0, 2258, 1, 0, 1, 1, 10, 2878, 135, 0, 10, 17, 0, 61] payload_rows_per_dst=[15014, 8766, 14536, 12832, 12388, 7798, 9864, 6634] dup_rows_per_dst=[198, 1464, 144, 1830, 4296, 784, 1556, 200] combine_rows_per_src=[15014, 8766, 14536, 12832, 12388, 7798, 9864, 6634] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=287711232 dispatch_meta_bytes=316504 combine_payload_bytes=287711232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288027736 ingress_bytes=322314240 link_bytes=322314240 token_bytes=575738968 weight_movement_bytes=34603008 total_nvlink_bytes=610341976 weight_to_token_ratio=0.060102 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=37248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.212842537e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80858 payload_rows_remote=70268 dup_rows=17446 dup_rows_remote=13140 dedup_ratio=0.177470 router_tokens_per_expert=[107, 78, 74, 90, 70, 78, 18, 1, 156, 1984, 13, 218, 168, 1405, 401, 579, 29, 5411, 268, 524, 31, 0, 558, 71, 5405, 467, 242, 418, 78, 1736, 148, 4357, 5889, 1798, 556, 472, 173, 156, 2, 208, 794, 186, 3665, 598, 26, 2283, 466, 873, 121, 62, 14, 1343, 80, 14, 54, 107, 289, 2544, 122, 4, 283, 624, 34, 129] payload_rows_per_dst=[11648, 10590, 8492, 12134, 14220, 13142, 3400, 7232] dup_rows_per_dst=[194, 4306, 244, 2758, 4288, 4640, 190, 826] combine_rows_per_src=[11648, 10590, 8492, 12134, 14220, 13142, 3400, 7232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=259678208 dispatch_meta_bytes=332576 combine_payload_bytes=259678208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260010784 ingress_bytes=294281216 link_bytes=294281216 token_bytes=519688992 weight_movement_bytes=34603008 total_nvlink_bytes=554292000 weight_to_token_ratio=0.066584 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=47744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.107356524e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82886 payload_rows_remote=68946 dup_rows=15418 dup_rows_remote=12014 dedup_ratio=0.156840 router_tokens_per_expert=[14, 1, 64, 0, 64, 0, 0, 0, 1, 2380, 0, 157, 74, 414, 2, 1310, 4, 7139, 5, 132, 0, 0, 71, 1, 7417, 8, 186, 249, 1, 771, 73, 5103, 7546, 4568, 6, 268, 5, 0, 0, 2, 368, 65, 3470, 4, 0, 1906, 243, 9, 143, 0, 0, 1339, 0, 0, 0, 0, 2, 3427, 62, 0, 4, 6, 0, 68] payload_rows_per_dst=[158, 7546, 14498, 15370, 554, 15012, 15808, 13940] dup_rows_per_dst=[128, 1130, 206, 2040, 8, 6254, 2248, 3404] combine_rows_per_src=[158, 7546, 14498, 15370, 554, 15012, 15808, 13940] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=307445760 dispatch_meta_bytes=341312 combine_payload_bytes=307445760 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=307787072 ingress_bytes=376651776 link_bytes=376651776 token_bytes=615232832 weight_movement_bytes=69206016 total_nvlink_bytes=684438848 weight_to_token_ratio=0.112488 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=46080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.417310310e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=282402816 dispatch_meta_bytes=323840 combine_payload_bytes=282402816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282726656 ingress_bytes=317005824 link_bytes=317005824 token_bytes=565129472 weight_movement_bytes=34603008 total_nvlink_bytes=599732480 weight_to_token_ratio=0.061230 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=24320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.192867395e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=325427200 dispatch_meta_bytes=353216 combine_payload_bytes=325427200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=325780416 ingress_bytes=360030208 link_bytes=360030208 token_bytes=651207616 weight_movement_bytes=34603008 total_nvlink_bytes=685810624 weight_to_token_ratio=0.053137 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22528 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.354764688e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86772 payload_rows_remote=75670 dup_rows=11532 dup_rows_remote=10116 dedup_ratio=0.117310 router_tokens_per_expert=[0, 68, 247, 98, 0, 4499, 0, 460, 0, 0, 48, 0, 80, 0, 0, 0, 2673, 0, 0, 0, 0, 2075, 0, 1704, 0, 0, 31, 5004, 0, 0, 0, 68, 4981, 1, 80, 259, 0, 0, 177, 761, 245, 0, 0, 1537, 72, 0, 0, 0, 595, 0, 4770, 80, 6330, 68, 2980, 0, 0, 1506, 3000, 653, 1324, 2517, 0, 161] payload_rows_per_dst=[10742, 12824, 11218, 14482, 11102, 8902, 7078, 10424] dup_rows_per_dst=[2, 92, 1686, 5264, 1416, 806, 368, 1898] combine_rows_per_src=[10742, 12824, 11218, 14482, 11102, 8902, 7078, 10424] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=293593088 dispatch_meta_bytes=318352 combine_payload_bytes=293593088 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=293911440 ingress_bytes=432005120 link_bytes=432005120 token_bytes=587504528 weight_movement_bytes=138412032 total_nvlink_bytes=725916560 weight_to_token_ratio=0.235593 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=31232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.625600487e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=287817728 dispatch_meta_bytes=333632 combine_payload_bytes=287817728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288151360 ingress_bytes=322420736 link_bytes=322420736 token_bytes=575969088 weight_movement_bytes=34603008 total_nvlink_bytes=610572096 weight_to_token_ratio=0.060078 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.213243273e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=307200000 dispatch_meta_bytes=334568 combine_payload_bytes=307200000 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=307534568 ingress_bytes=411009024 link_bytes=411009024 token_bytes=614734568 weight_movement_bytes=103809024 total_nvlink_bytes=718543592 weight_to_token_ratio=0.168868 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=21120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.546593868e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86012 payload_rows_remote=71522 dup_rows=12292 dup_rows_remote=12038 dedup_ratio=0.125041 router_tokens_per_expert=[40, 0, 53, 0, 53, 0, 0, 0, 0, 3787, 0, 217, 67, 46, 27, 1450, 2, 7208, 0, 90, 0, 0, 72, 0, 7418, 0, 229, 265, 0, 1194, 89, 5738, 7399, 4125, 0, 28, 0, 0, 0, 0, 133, 1, 1646, 0, 0, 1829, 265, 0, 319, 0, 0, 2114, 0, 0, 0, 0, 0, 3037, 140, 0, 0, 0, 0, 71] payload_rows_per_dst=[14986, 9662, 14490, 13266, 15170, 7078, 4866, 6494] dup_rows_per_dst=[142, 1526, 254, 1764, 7934, 670, 0, 2] combine_rows_per_src=[14986, 9662, 14490, 13266, 15170, 7078, 4866, 6494] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=282402816 dispatch_meta_bytes=323840 combine_payload_bytes=282402816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282726656 ingress_bytes=317005824 link_bytes=317005824 token_bytes=565129472 weight_movement_bytes=34603008 total_nvlink_bytes=599732480 weight_to_token_ratio=0.061230 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=24320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.192867395e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=309944320 dispatch_meta_bytes=343144 combine_payload_bytes=309944320 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=310287464 ingress_bytes=413753344 link_bytes=413753344 token_bytes=620231784 weight_movement_bytes=103809024 total_nvlink_bytes=724040808 weight_to_token_ratio=0.167371 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=34688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.556920524e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=292954112 dispatch_meta_bytes=334240 combine_payload_bytes=292954112 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=293288352 ingress_bytes=362160128 link_bytes=362160128 token_bytes=586242464 weight_movement_bytes=69206016 total_nvlink_bytes=655448480 weight_to_token_ratio=0.118050 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=46080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.362779405e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78456 payload_rows_remote=63398 dup_rows=19848 dup_rows_remote=19746 dedup_ratio=0.201904 router_tokens_per_expert=[10, 1, 47, 1, 47, 0, 0, 0, 5, 2779, 0, 137, 74, 214, 52, 1291, 0, 7220, 8, 75, 0, 0, 81, 6, 7474, 15, 225, 256, 7, 1113, 72, 5553, 7541, 3508, 11, 167, 5, 0, 0, 1, 247, 79, 3314, 11, 0, 1779, 259, 10, 256, 0, 0, 1551, 1, 0, 0, 1, 6, 3488, 63, 0, 1, 13, 0, 77] payload_rows_per_dst=[15058, 15720, 1114, 14288, 11110, 10268, 3618, 7280] dup_rows_per_dst=[102, 7824, 10, 6484, 4282, 1130, 0, 16] combine_rows_per_src=[15058, 15720, 1114, 14288, 11110, 10268, 3618, 7280] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81070 payload_rows_remote=68882 dup_rows=17234 dup_rows_remote=12824 dedup_ratio=0.175313 router_tokens_per_expert=[0, 0, 160, 1238, 0, 0, 984, 0, 0, 5280, 837, 0, 0, 0, 752, 0, 208, 322, 0, 1427, 11, 54, 0, 0, 5485, 58, 0, 117, 0, 0, 0, 0, 58, 347, 0, 0, 2409, 776, 0, 99, 7, 0, 5086, 0, 0, 0, 0, 0, 610, 0, 99, 0, 7486, 0, 3, 27, 6748, 0, 0, 0, 7142, 543, 779, 0] payload_rows_per_dst=[4628, 12408, 3774, 11892, 6820, 12188, 15376, 13984] dup_rows_per_dst=[136, 1330, 270, 1238, 558, 4410, 1074, 8218] combine_rows_per_src=[4628, 12408, 3774, 11892, 6820, 12188, 15376, 13984] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88182 payload_rows_remote=72468 dup_rows=10122 dup_rows_remote=7348 dedup_ratio=0.102966 router_tokens_per_expert=[0, 0, 70, 0, 70, 0, 0, 0, 0, 2480, 0, 158, 75, 467, 52, 1352, 0, 7214, 0, 185, 0, 0, 68, 0, 7418, 0, 275, 251, 0, 1014, 74, 5548, 7556, 3638, 0, 358, 0, 0, 0, 0, 95, 67, 3836, 0, 0, 1918, 251, 0, 89, 0, 0, 1599, 0, 0, 0, 0, 0, 2825, 82, 0, 0, 0, 0, 67] payload_rows_per_dst=[10318, 7798, 14796, 6840, 7130, 11484, 15714, 14102] dup_rows_per_dst=[140, 1370, 138, 1046, 0, 1712, 2774, 2942] combine_rows_per_src=[10318, 7798, 14796, 6840, 7130, 11484, 15714, 14102] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79740 payload_rows_remote=68164 dup_rows=18564 dup_rows_remote=15888 dedup_ratio=0.188843 router_tokens_per_expert=[51, 94, 210, 866, 212, 342, 1096, 0, 0, 3595, 1278, 38, 700, 937, 575, 3, 204, 434, 512, 2212, 335, 70, 6, 55, 4127, 264, 12, 162, 137, 59, 55, 165, 228, 401, 767, 457, 1173, 1113, 299, 94, 426, 261, 4275, 33, 104, 139, 275, 897, 628, 247, 98, 16, 5931, 325, 224, 117, 4224, 14, 280, 134, 5378, 652, 1106, 30] payload_rows_per_dst=[11500, 11576, 13290, 9620, 15706, 11350, 2694, 4004] dup_rows_per_dst=[2690, 2676, 5122, 342, 5220, 1470, 616, 428] combine_rows_per_src=[11500, 11576, 13290, 9620, 15706, 11350, 2694, 4004] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88174 payload_rows_remote=76988 dup_rows=10130 dup_rows_remote=10128 dedup_ratio=0.103048 router_tokens_per_expert=[0, 1, 117, 964, 0, 9, 1601, 0, 0, 3860, 944, 1, 13, 28, 745, 1, 160, 427, 18, 1156, 25, 63, 3, 0, 5367, 4, 0, 203, 3, 2, 0, 15, 80, 315, 8, 6, 2616, 1043, 1, 76, 92, 11, 5563, 0, 1, 6, 0, 24, 620, 22, 76, 0, 7529, 17, 31, 4, 6834, 0, 2, 0, 7143, 509, 793, 0] payload_rows_per_dst=[15272, 9918, 14876, 11186, 7636, 11212, 15478, 2596] dup_rows_per_dst=[3780, 1266, 3114, 2, 654, 182, 1120, 12] combine_rows_per_src=[15272, 9918, 14876, 11186, 7636, 11212, 15478, 2596] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=259678208 dispatch_meta_bytes=332576 combine_payload_bytes=259678208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260010784 ingress_bytes=294281216 link_bytes=294281216 token_bytes=519688992 weight_movement_bytes=34603008 total_nvlink_bytes=554292000 weight_to_token_ratio=0.066584 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=47744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.107356524e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78500 payload_rows_remote=62326 dup_rows=19804 dup_rows_remote=5448 dedup_ratio=0.201457 router_tokens_per_expert=[0, 1, 52, 1367, 1, 11, 1123, 0, 0, 5237, 803, 1, 5, 9, 737, 0, 259, 341, 10, 1454, 227, 71, 0, 0, 5703, 82, 0, 89, 1, 0, 1, 0, 81, 314, 8, 4, 2332, 1012, 0, 73, 9, 0, 4628, 0, 0, 4, 0, 9, 619, 9, 73, 0, 7149, 6, 3, 28, 6679, 0, 0, 0, 7261, 528, 738, 0] payload_rows_per_dst=[4910, 10076, 4374, 11752, 7112, 9280, 14822, 16174] dup_rows_per_dst=[200, 1140, 350, 0, 536, 20, 3202, 14356] combine_rows_per_src=[4910, 10076, 4374, 11752, 7112, 9280, 14822, 16174] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85052 payload_rows_remote=70220 dup_rows=13252 dup_rows_remote=10188 dedup_ratio=0.134806 router_tokens_per_expert=[0, 0, 138, 784, 0, 0, 1618, 0, 0, 3385, 955, 0, 0, 0, 785, 0, 167, 467, 0, 1044, 0, 72, 0, 0, 5169, 0, 0, 306, 0, 0, 0, 0, 74, 355, 0, 0, 2285, 1656, 0, 89, 83, 0, 5897, 0, 0, 0, 0, 0, 622, 0, 89, 0, 7518, 7, 71, 0, 7012, 0, 0, 0, 7198, 521, 785, 0] payload_rows_per_dst=[15296, 8952, 14832, 10950, 11674, 11796, 8940, 2612] dup_rows_per_dst=[3808, 1298, 3064, 0, 4358, 164, 560, 0] combine_rows_per_src=[15296, 8952, 14832, 10950, 11674, 11796, 8940, 2612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80436 payload_rows_remote=65418 dup_rows=17868 dup_rows_remote=16718 dedup_ratio=0.181763 router_tokens_per_expert=[0, 0, 82, 1336, 0, 0, 1011, 0, 0, 5548, 850, 0, 0, 0, 727, 0, 256, 335, 0, 1359, 144, 68, 0, 0, 6024, 30, 0, 76, 0, 0, 0, 0, 78, 326, 0, 0, 1729, 1207, 0, 74, 45, 0, 4551, 0, 0, 0, 0, 0, 619, 0, 74, 0, 7359, 0, 1, 31, 6826, 0, 0, 0, 7098, 558, 730, 0] payload_rows_per_dst=[4580, 12872, 3974, 12260, 6414, 9102, 15018, 16216] dup_rows_per_dst=[278, 1378, 350, 0, 414, 90, 1150, 14208] combine_rows_per_src=[4580, 12872, 3974, 12260, 6414, 9102, 15018, 16216] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=287817728 dispatch_meta_bytes=333632 combine_payload_bytes=287817728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288151360 ingress_bytes=322420736 link_bytes=322420736 token_bytes=575969088 weight_movement_bytes=34603008 total_nvlink_bytes=610572096 weight_to_token_ratio=0.060078 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.213243273e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87806 payload_rows_remote=78788 dup_rows=10498 dup_rows_remote=10498 dedup_ratio=0.106791 router_tokens_per_expert=[50, 0, 68, 0, 68, 0, 0, 0, 0, 3085, 0, 151, 36, 99, 337, 1419, 3, 7241, 0, 80, 0, 0, 59, 0, 7427, 0, 356, 247, 0, 1452, 80, 5668, 7122, 4265, 0, 244, 0, 0, 0, 0, 88, 0, 2051, 0, 0, 1780, 247, 0, 101, 0, 0, 2301, 0, 0, 0, 0, 0, 2887, 81, 0, 0, 0, 0, 59] payload_rows_per_dst=[15042, 8316, 14550, 4094, 9018, 7602, 14952, 14232] dup_rows_per_dst=[184, 1938, 216, 176, 0, 730, 4096, 3158] combine_rows_per_src=[15042, 8316, 14550, 4094, 9018, 7602, 14952, 14232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=282140672 dispatch_meta_bytes=326824 combine_payload_bytes=282140672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282467496 ingress_bytes=316743680 link_bytes=316743680 token_bytes=564608168 weight_movement_bytes=34603008 total_nvlink_bytes=599211176 weight_to_token_ratio=0.061287 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=40832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.191880968e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=296828928 dispatch_meta_bytes=319264 combine_payload_bytes=296828928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=297148192 ingress_bytes=331431936 link_bytes=331431936 token_bytes=593977120 weight_movement_bytes=34603008 total_nvlink_bytes=628580128 weight_to_token_ratio=0.058256 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=33152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.247151693e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=279199744 dispatch_meta_bytes=336208 combine_payload_bytes=279199744 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=279535952 ingress_bytes=348405760 link_bytes=348405760 token_bytes=558735696 weight_movement_bytes=69206016 total_nvlink_bytes=627941712 weight_to_token_ratio=0.123862 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=33920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.311022826e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=315342848 dispatch_meta_bytes=348464 combine_payload_bytes=315342848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=315691312 ingress_bytes=349945856 link_bytes=349945856 token_bytes=631034160 weight_movement_bytes=34603008 total_nvlink_bytes=665637168 weight_to_token_ratio=0.054835 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=42752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.316818083e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=255287296 dispatch_meta_bytes=271096 combine_payload_bytes=255287296 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=255558392 ingress_bytes=428302336 link_bytes=428302336 token_bytes=510845688 weight_movement_bytes=173015040 total_nvlink_bytes=683860728 weight_to_token_ratio=0.338684 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=19456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.611667209e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=287621120 dispatch_meta_bytes=321632 combine_payload_bytes=287621120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=287942752 ingress_bytes=322224128 link_bytes=322224128 token_bytes=575563872 weight_movement_bytes=34603008 total_nvlink_bytes=610166880 weight_to_token_ratio=0.060120 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=31616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.212503452e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=322715648 dispatch_meta_bytes=357144 combine_payload_bytes=322715648 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=323072792 ingress_bytes=461127680 link_bytes=461127680 token_bytes=645788440 weight_movement_bytes=138412032 total_nvlink_bytes=784200472 weight_to_token_ratio=0.214330 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=24320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.735186336e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84540 payload_rows_remote=69236 dup_rows=13764 dup_rows_remote=11052 dedup_ratio=0.140015 router_tokens_per_expert=[0, 1, 63, 1226, 3, 7, 991, 0, 0, 5085, 868, 0, 13, 18, 688, 1, 249, 330, 5, 1281, 29, 75, 0, 1, 5698, 52, 0, 89, 0, 1, 2, 6, 87, 328, 8, 5, 1779, 1383, 3, 72, 17, 8, 5125, 0, 5, 3, 1, 16, 620, 12, 73, 0, 7482, 3, 15, 70, 6717, 0, 1, 0, 7278, 537, 722, 0] payload_rows_per_dst=[15304, 11980, 15022, 11692, 16294, 10288, 1440, 2520] dup_rows_per_dst=[2712, 1366, 3474, 4, 6000, 62, 146, 0] combine_rows_per_src=[15304, 11980, 15022, 11692, 16294, 10288, 1440, 2520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86356 payload_rows_remote=74348 dup_rows=11948 dup_rows_remote=9306 dedup_ratio=0.121541 router_tokens_per_expert=[0, 53, 271, 80, 7, 4465, 0, 432, 0, 0, 64, 0, 90, 0, 0, 0, 2152, 0, 0, 0, 0, 1917, 0, 1446, 0, 0, 15, 4506, 0, 0, 0, 53, 5032, 16, 90, 293, 0, 0, 163, 742, 215, 0, 0, 1536, 52, 0, 0, 0, 583, 0, 4408, 88, 6396, 53, 3375, 0, 5, 1821, 3500, 681, 1686, 2640, 0, 226] payload_rows_per_dst=[10586, 12972, 12008, 13818, 11268, 10952, 7834, 6918] dup_rows_per_dst=[30, 128, 2642, 4146, 1404, 2034, 364, 1200] combine_rows_per_src=[10586, 12972, 12008, 13818, 11268, 10952, 7834, 6918] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267952128 dispatch_meta_bytes=328544 combine_payload_bytes=267952128 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=268280672 ingress_bytes=337158144 link_bytes=337158144 token_bytes=536232800 weight_movement_bytes=69206016 total_nvlink_bytes=605438816 weight_to_token_ratio=0.129060 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=41088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.268698952e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78500 payload_rows_remote=62326 dup_rows=19804 dup_rows_remote=5448 dedup_ratio=0.201457 router_tokens_per_expert=[0, 1, 52, 1367, 1, 11, 1123, 0, 0, 5237, 803, 1, 5, 9, 737, 0, 259, 341, 10, 1454, 227, 71, 0, 0, 5703, 82, 0, 89, 1, 0, 1, 0, 81, 314, 8, 4, 2332, 1012, 0, 73, 9, 0, 4628, 0, 0, 4, 0, 9, 619, 9, 73, 0, 7149, 6, 3, 28, 6679, 0, 0, 0, 7261, 528, 738, 0] payload_rows_per_dst=[4910, 10076, 4374, 11752, 7112, 9280, 14822, 16174] dup_rows_per_dst=[200, 1140, 350, 0, 536, 20, 3202, 14356] combine_rows_per_src=[4910, 10076, 4374, 11752, 7112, 9280, 14822, 16174] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89500 payload_rows_remote=82544 dup_rows=8804 dup_rows_remote=8458 dedup_ratio=0.089559 router_tokens_per_expert=[3665, 0, 0, 174, 788, 2375, 0, 0, 1595, 67, 2067, 0, 0, 0, 0, 0, 0, 99, 122, 0, 0, 99, 0, 0, 78, 0, 0, 0, 0, 100, 0, 6, 0, 0, 6089, 0, 0, 0, 1922, 462, 0, 6746, 2010, 7753, 0, 0, 4597, 90, 0, 0, 0, 100, 0, 0, 2, 6353, 326, 0, 0, 360, 610, 249, 247, 1] payload_rows_per_dst=[8966, 6446, 13824, 15518, 13484, 6956, 12906, 11400] dup_rows_per_dst=[1936, 1012, 308, 356, 3462, 346, 4, 1380] combine_rows_per_src=[8966, 6446, 13824, 15518, 13484, 6956, 12906, 11400] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80436 payload_rows_remote=65418 dup_rows=17868 dup_rows_remote=16718 dedup_ratio=0.181763 router_tokens_per_expert=[0, 0, 82, 1336, 0, 0, 1011, 0, 0, 5548, 850, 0, 0, 0, 727, 0, 256, 335, 0, 1359, 144, 68, 0, 0, 6024, 30, 0, 76, 0, 0, 0, 0, 78, 326, 0, 0, 1729, 1207, 0, 74, 45, 0, 4551, 0, 0, 0, 0, 0, 619, 0, 74, 0, 7359, 0, 1, 31, 6826, 0, 0, 0, 7098, 558, 730, 0] payload_rows_per_dst=[4580, 12872, 3974, 12260, 6414, 9102, 15018, 16216] dup_rows_per_dst=[278, 1378, 350, 0, 414, 90, 1150, 14208] combine_rows_per_src=[4580, 12872, 3974, 12260, 6414, 9102, 15018, 16216] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83708 payload_rows_remote=72500 dup_rows=14596 dup_rows_remote=10520 dedup_ratio=0.148478 router_tokens_per_expert=[1367, 39, 1166, 196, 808, 3213, 326, 181, 2495, 163, 1012, 0, 112, 63, 50, 60, 3, 53, 451, 846, 113, 96, 395, 228, 213, 58, 160, 192, 59, 109, 73, 179, 666, 10, 3687, 4, 119, 175, 2834, 664, 51, 4985, 1855, 4797, 87, 561, 4509, 154, 79, 487, 131, 92, 97, 112, 433, 3534, 1227, 1044, 40, 575, 635, 245, 415, 369] payload_rows_per_dst=[12190, 11208, 12804, 11014, 7902, 5354, 9680, 13556] dup_rows_per_dst=[2402, 4076, 1536, 666, 1042, 62, 250, 4562] combine_rows_per_src=[12190, 11208, 12804, 11014, 7902, 5354, 9680, 13556] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88174 payload_rows_remote=76988 dup_rows=10130 dup_rows_remote=10128 dedup_ratio=0.103048 router_tokens_per_expert=[0, 1, 117, 964, 0, 9, 1601, 0, 0, 3860, 944, 1, 13, 28, 745, 1, 160, 427, 18, 1156, 25, 63, 3, 0, 5367, 4, 0, 203, 3, 2, 0, 15, 80, 315, 8, 6, 2616, 1043, 1, 76, 92, 11, 5563, 0, 1, 6, 0, 24, 620, 22, 76, 0, 7529, 17, 31, 4, 6834, 0, 2, 0, 7143, 509, 793, 0] payload_rows_per_dst=[15272, 9918, 14876, 11186, 7636, 11212, 15478, 2596] dup_rows_per_dst=[3780, 1266, 3114, 2, 654, 182, 1120, 12] combine_rows_per_src=[15272, 9918, 14876, 11186, 7636, 11212, 15478, 2596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81070 payload_rows_remote=68882 dup_rows=17234 dup_rows_remote=12824 dedup_ratio=0.175313 router_tokens_per_expert=[0, 0, 160, 1238, 0, 0, 984, 0, 0, 5280, 837, 0, 0, 0, 752, 0, 208, 322, 0, 1427, 11, 54, 0, 0, 5485, 58, 0, 117, 0, 0, 0, 0, 58, 347, 0, 0, 2409, 776, 0, 99, 7, 0, 5086, 0, 0, 0, 0, 0, 610, 0, 99, 0, 7486, 0, 3, 27, 6748, 0, 0, 0, 7142, 543, 779, 0] payload_rows_per_dst=[4628, 12408, 3774, 11892, 6820, 12188, 15376, 13984] dup_rows_per_dst=[136, 1330, 270, 1238, 558, 4410, 1074, 8218] combine_rows_per_src=[4628, 12408, 3774, 11892, 6820, 12188, 15376, 13984] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=322715648 dispatch_meta_bytes=357144 combine_payload_bytes=322715648 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=323072792 ingress_bytes=461127680 link_bytes=461127680 token_bytes=645788440 weight_movement_bytes=138412032 total_nvlink_bytes=784200472 weight_to_token_ratio=0.214330 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=24320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.735186336e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=283590656 dispatch_meta_bytes=321152 combine_payload_bytes=283590656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=283911808 ingress_bytes=318193664 link_bytes=318193664 token_bytes=567502464 weight_movement_bytes=34603008 total_nvlink_bytes=602105472 weight_to_token_ratio=0.060974 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=28544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.197337141e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88940 payload_rows_remote=73488 dup_rows=9364 dup_rows_remote=8902 dedup_ratio=0.095256 router_tokens_per_expert=[3189, 1, 32, 130, 728, 3753, 5, 3, 1817, 92, 1184, 0, 2, 3, 0, 2, 0, 76, 184, 23, 4, 76, 16, 0, 156, 0, 0, 14, 0, 76, 0, 35, 6, 0, 5620, 0, 0, 6, 1552, 416, 0, 6809, 2783, 7676, 0, 22, 4281, 18, 8, 3, 3, 76, 0, 1, 13, 6266, 528, 28, 0, 325, 588, 181, 240, 102] payload_rows_per_dst=[12608, 13850, 14038, 15452, 3724, 5642, 12724, 10902] dup_rows_per_dst=[3074, 3590, 338, 462, 236, 4, 16, 1644] combine_rows_per_src=[12608, 13850, 14038, 15452, 3724, 5642, 12724, 10902] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=66604 payload_rows_remote=54916 dup_rows=31700 dup_rows_remote=30130 dedup_ratio=0.322469 router_tokens_per_expert=[3595, 0, 12, 186, 788, 1585, 1, 1, 2039, 85, 2437, 0, 1, 3, 0, 0, 0, 74, 154, 8, 1, 73, 6, 0, 82, 0, 0, 5, 0, 74, 0, 0, 2, 0, 6416, 0, 0, 1, 1998, 357, 0, 6752, 1527, 7699, 0, 7, 4802, 71, 2, 0, 0, 74, 6, 1, 84, 6316, 385, 12, 0, 348, 613, 192, 240, 37] payload_rows_per_dst=[1824, 8064, 620, 322, 15086, 16194, 12806, 11688] dup_rows_per_dst=[152, 1066, 12, 0, 9652, 19088, 160, 1570] combine_rows_per_src=[1824, 8064, 620, 322, 15086, 16194, 12806, 11688] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89318 payload_rows_remote=75414 dup_rows=8986 dup_rows_remote=8676 dedup_ratio=0.091410 router_tokens_per_expert=[3407, 0, 0, 102, 762, 4100, 0, 0, 1317, 74, 735, 0, 0, 0, 0, 0, 0, 86, 128, 0, 0, 88, 0, 0, 229, 0, 0, 0, 0, 90, 0, 50, 0, 0, 5564, 0, 0, 0, 1234, 457, 0, 6805, 3009, 7916, 0, 0, 4180, 0, 0, 0, 0, 90, 0, 0, 0, 6575, 701, 0, 0, 338, 595, 209, 265, 46] payload_rows_per_dst=[13182, 13144, 13904, 15932, 3104, 6018, 13330, 10704] dup_rows_per_dst=[3560, 2236, 310, 638, 278, 0, 0, 1964] combine_rows_per_src=[13182, 13144, 13904, 15932, 3104, 6018, 13330, 10704] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=304529408 dispatch_meta_bytes=334616 combine_payload_bytes=304529408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=304864024 ingress_bytes=339132416 link_bytes=339132416 token_bytes=609393432 weight_movement_bytes=34603008 total_nvlink_bytes=643996440 weight_to_token_ratio=0.056783 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=14976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.276127979e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79740 payload_rows_remote=68164 dup_rows=18564 dup_rows_remote=15888 dedup_ratio=0.188843 router_tokens_per_expert=[51, 94, 210, 866, 212, 342, 1096, 0, 0, 3595, 1278, 38, 700, 937, 575, 3, 204, 434, 512, 2212, 335, 70, 6, 55, 4127, 264, 12, 162, 137, 59, 55, 165, 228, 401, 767, 457, 1173, 1113, 299, 94, 426, 261, 4275, 33, 104, 139, 275, 897, 628, 247, 98, 16, 5931, 325, 224, 117, 4224, 14, 280, 134, 5378, 652, 1106, 30] payload_rows_per_dst=[11500, 11576, 13290, 9620, 15706, 11350, 2694, 4004] dup_rows_per_dst=[2690, 2676, 5122, 342, 5220, 1470, 616, 428] combine_rows_per_src=[11500, 11576, 13290, 9620, 15706, 11350, 2694, 4004] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=255287296 dispatch_meta_bytes=271096 combine_payload_bytes=255287296 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=255558392 ingress_bytes=428302336 link_bytes=428302336 token_bytes=510845688 weight_movement_bytes=173015040 total_nvlink_bytes=683860728 weight_to_token_ratio=0.338684 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=19456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.611667209e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=315342848 dispatch_meta_bytes=348464 combine_payload_bytes=315342848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=315691312 ingress_bytes=349945856 link_bytes=349945856 token_bytes=631034160 weight_movement_bytes=34603008 total_nvlink_bytes=665637168 weight_to_token_ratio=0.054835 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=42752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.316818083e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=282140672 dispatch_meta_bytes=326824 combine_payload_bytes=282140672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282467496 ingress_bytes=316743680 link_bytes=316743680 token_bytes=564608168 weight_movement_bytes=34603008 total_nvlink_bytes=599211176 weight_to_token_ratio=0.061287 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=40832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.191880968e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79954 payload_rows_remote=66582 dup_rows=18350 dup_rows_remote=18346 dedup_ratio=0.186666 router_tokens_per_expert=[3808, 0, 0, 172, 738, 2178, 0, 0, 2260, 84, 2083, 0, 0, 0, 0, 0, 0, 75, 154, 0, 0, 74, 0, 0, 73, 0, 0, 0, 1, 75, 0, 0, 0, 0, 5953, 0, 0, 0, 1794, 403, 0, 6661, 1571, 7862, 0, 0, 4604, 74, 0, 0, 0, 75, 1, 0, 2, 6610, 328, 0, 0, 363, 642, 183, 251, 0] payload_rows_per_dst=[2658, 7730, 11892, 5664, 14678, 12612, 13372, 11348] dup_rows_per_dst=[288, 1124, 254, 106, 8112, 7068, 4, 1394] combine_rows_per_src=[2658, 7730, 11892, 5664, 14678, 12612, 13372, 11348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=279199744 dispatch_meta_bytes=336208 combine_payload_bytes=279199744 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=279535952 ingress_bytes=348405760 link_bytes=348405760 token_bytes=558735696 weight_movement_bytes=69206016 total_nvlink_bytes=627941712 weight_to_token_ratio=0.123862 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=33920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.311022826e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81902 payload_rows_remote=73758 dup_rows=16402 dup_rows_remote=15638 dedup_ratio=0.166850 router_tokens_per_expert=[0, 0, 81, 874, 0, 0, 1956, 0, 0, 3103, 1116, 0, 0, 0, 815, 0, 217, 423, 0, 957, 0, 59, 0, 0, 5163, 0, 0, 196, 0, 0, 0, 0, 59, 327, 0, 0, 2502, 1486, 0, 80, 76, 0, 5646, 0, 0, 0, 0, 0, 617, 0, 80, 0, 7544, 0, 93, 0, 7018, 0, 0, 0, 7176, 589, 899, 0] payload_rows_per_dst=[10610, 8676, 10652, 12734, 8144, 12516, 15594, 2976] dup_rows_per_dst=[2600, 1392, 1766, 4632, 764, 4174, 1074, 0] combine_rows_per_src=[10610, 8676, 10652, 12734, 8144, 12516, 15594, 2976] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=338100224 dispatch_meta_bytes=364008 combine_payload_bytes=338100224 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=338464232 ingress_bytes=476512256 link_bytes=476512256 token_bytes=676564456 weight_movement_bytes=138412032 total_nvlink_bytes=814976488 weight_to_token_ratio=0.204581 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.793077257e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267952128 dispatch_meta_bytes=328544 combine_payload_bytes=267952128 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=268280672 ingress_bytes=337158144 link_bytes=337158144 token_bytes=536232800 weight_movement_bytes=69206016 total_nvlink_bytes=605438816 weight_to_token_ratio=0.129060 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=41088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.268698952e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=296960000 dispatch_meta_bytes=332080 combine_payload_bytes=296960000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=297292080 ingress_bytes=331563008 link_bytes=331563008 token_bytes=594252080 weight_movement_bytes=34603008 total_nvlink_bytes=628855088 weight_to_token_ratio=0.058230 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=21376 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.247644906e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=301006848 dispatch_meta_bytes=329560 combine_payload_bytes=301006848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=301336408 ingress_bytes=335609856 link_bytes=335609856 token_bytes=602343256 weight_movement_bytes=34603008 total_nvlink_bytes=636946264 weight_to_token_ratio=0.057447 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=46592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262872869e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=224935936 dispatch_meta_bytes=340184 combine_payload_bytes=224935936 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=225276120 ingress_bytes=259538944 link_bytes=259538944 token_bytes=450212056 weight_movement_bytes=34603008 total_nvlink_bytes=484815064 weight_to_token_ratio=0.076859 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=30976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.766241517e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=308895744 dispatch_meta_bytes=336360 combine_payload_bytes=308895744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=309232104 ingress_bytes=343498752 link_bytes=343498752 token_bytes=618127848 weight_movement_bytes=34603008 total_nvlink_bytes=652730856 weight_to_token_ratio=0.055980 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=44800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.292558150e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=302112768 dispatch_meta_bytes=357584 combine_payload_bytes=302112768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=302470352 ingress_bytes=336715776 link_bytes=336715776 token_bytes=604583120 weight_movement_bytes=34603008 total_nvlink_bytes=639186128 weight_to_token_ratio=0.057234 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.267034357e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=272719872 dispatch_meta_bytes=339712 combine_payload_bytes=272719872 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=273059584 ingress_bytes=307322880 link_bytes=307322880 token_bytes=545779456 weight_movement_bytes=34603008 total_nvlink_bytes=580382464 weight_to_token_ratio=0.063401 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=47104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.156431256e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83744 payload_rows_remote=71296 dup_rows=14560 dup_rows_remote=9260 dedup_ratio=0.148112 router_tokens_per_expert=[3414, 0, 20, 150, 762, 2879, 5, 2, 1851, 71, 1839, 0, 1, 3, 0, 3, 0, 72, 119, 17, 3, 72, 6, 5, 89, 0, 0, 5, 0, 73, 0, 3, 7, 0, 5997, 0, 0, 1, 1556, 365, 0, 6790, 1833, 7611, 2, 13, 4875, 72, 2, 2, 1, 72, 0, 4, 32, 6518, 513, 16, 0, 367, 605, 151, 256, 27] payload_rows_per_dst=[12448, 14310, 13900, 15244, 3728, 11134, 9976, 3004] dup_rows_per_dst=[5300, 5220, 268, 318, 130, 1646, 2, 1676] combine_rows_per_src=[12448, 14310, 13900, 15244, 3728, 11134, 9976, 3004] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86012 payload_rows_remote=71522 dup_rows=12292 dup_rows_remote=12038 dedup_ratio=0.125041 router_tokens_per_expert=[40, 0, 53, 0, 53, 0, 0, 0, 0, 3787, 0, 217, 67, 46, 27, 1450, 2, 7208, 0, 90, 0, 0, 72, 0, 7418, 0, 229, 265, 0, 1194, 89, 5738, 7399, 4125, 0, 28, 0, 0, 0, 0, 133, 1, 1646, 0, 0, 1829, 265, 0, 319, 0, 0, 2114, 0, 0, 0, 0, 0, 3037, 140, 0, 0, 0, 0, 71] payload_rows_per_dst=[14986, 9662, 14490, 13266, 15170, 7078, 4866, 6494] dup_rows_per_dst=[142, 1526, 254, 1764, 7934, 670, 0, 2] combine_rows_per_src=[14986, 9662, 14490, 13266, 15170, 7078, 4866, 6494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=66604 payload_rows_remote=54916 dup_rows=31700 dup_rows_remote=30130 dedup_ratio=0.322469 router_tokens_per_expert=[3595, 0, 12, 186, 788, 1585, 1, 1, 2039, 85, 2437, 0, 1, 3, 0, 0, 0, 74, 154, 8, 1, 73, 6, 0, 82, 0, 0, 5, 0, 74, 0, 0, 2, 0, 6416, 0, 0, 1, 1998, 357, 0, 6752, 1527, 7699, 0, 7, 4802, 71, 2, 0, 0, 74, 6, 1, 84, 6316, 385, 12, 0, 348, 613, 192, 240, 37] payload_rows_per_dst=[1824, 8064, 620, 322, 15086, 16194, 12806, 11688] dup_rows_per_dst=[152, 1066, 12, 0, 9652, 19088, 160, 1570] combine_rows_per_src=[1824, 8064, 620, 322, 15086, 16194, 12806, 11688] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88940 payload_rows_remote=73488 dup_rows=9364 dup_rows_remote=8902 dedup_ratio=0.095256 router_tokens_per_expert=[3189, 1, 32, 130, 728, 3753, 5, 3, 1817, 92, 1184, 0, 2, 3, 0, 2, 0, 76, 184, 23, 4, 76, 16, 0, 156, 0, 0, 14, 0, 76, 0, 35, 6, 0, 5620, 0, 0, 6, 1552, 416, 0, 6809, 2783, 7676, 0, 22, 4281, 18, 8, 3, 3, 76, 0, 1, 13, 6266, 528, 28, 0, 325, 588, 181, 240, 102] payload_rows_per_dst=[12608, 13850, 14038, 15452, 3724, 5642, 12724, 10902] dup_rows_per_dst=[3074, 3590, 338, 462, 236, 4, 16, 1644] combine_rows_per_src=[12608, 13850, 14038, 15452, 3724, 5642, 12724, 10902] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=302112768 dispatch_meta_bytes=357584 combine_payload_bytes=302112768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=302470352 ingress_bytes=336715776 link_bytes=336715776 token_bytes=604583120 weight_movement_bytes=34603008 total_nvlink_bytes=639186128 weight_to_token_ratio=0.057234 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.267034357e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89500 payload_rows_remote=82544 dup_rows=8804 dup_rows_remote=8458 dedup_ratio=0.089559 router_tokens_per_expert=[3665, 0, 0, 174, 788, 2375, 0, 0, 1595, 67, 2067, 0, 0, 0, 0, 0, 0, 99, 122, 0, 0, 99, 0, 0, 78, 0, 0, 0, 0, 100, 0, 6, 0, 0, 6089, 0, 0, 0, 1922, 462, 0, 6746, 2010, 7753, 0, 0, 4597, 90, 0, 0, 0, 100, 0, 0, 2, 6353, 326, 0, 0, 360, 610, 249, 247, 1] payload_rows_per_dst=[8966, 6446, 13824, 15518, 13484, 6956, 12906, 11400] dup_rows_per_dst=[1936, 1012, 308, 356, 3462, 346, 4, 1380] combine_rows_per_src=[8966, 6446, 13824, 15518, 13484, 6956, 12906, 11400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=292028416 dispatch_meta_bytes=322224 combine_payload_bytes=292028416 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=292350640 ingress_bytes=395837440 link_bytes=395837440 token_bytes=584379056 weight_movement_bytes=103809024 total_nvlink_bytes=688188080 weight_to_token_ratio=0.177640 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=19584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.489504419e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82780 payload_rows_remote=69684 dup_rows=15524 dup_rows_remote=10178 dedup_ratio=0.157918 router_tokens_per_expert=[4607, 491, 0, 0, 0, 0, 1423, 2, 0, 2855, 0, 4286, 0, 0, 648, 0, 0, 199, 784, 0, 0, 357, 0, 0, 0, 0, 4284, 623, 1591, 0, 154, 1194, 0, 0, 0, 0, 0, 0, 0, 4581, 0, 0, 418, 122, 4487, 199, 7326, 0, 6888, 54, 0, 157, 0, 99, 0, 0, 67, 0, 0, 0, 288, 86, 0, 882] payload_rows_per_dst=[10884, 7296, 8382, 7082, 12412, 13096, 13780, 9848] dup_rows_per_dst=[2162, 158, 554, 42, 4874, 5346, 616, 1772] combine_rows_per_src=[10884, 7296, 8382, 7082, 12412, 13096, 13780, 9848] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79954 payload_rows_remote=66582 dup_rows=18350 dup_rows_remote=18346 dedup_ratio=0.186666 router_tokens_per_expert=[3808, 0, 0, 172, 738, 2178, 0, 0, 2260, 84, 2083, 0, 0, 0, 0, 0, 0, 75, 154, 0, 0, 74, 0, 0, 73, 0, 0, 0, 1, 75, 0, 0, 0, 0, 5953, 0, 0, 0, 1794, 403, 0, 6661, 1571, 7862, 0, 0, 4604, 74, 0, 0, 0, 75, 1, 0, 2, 6610, 328, 0, 0, 363, 642, 183, 251, 0] payload_rows_per_dst=[2658, 7730, 11892, 5664, 14678, 12612, 13372, 11348] dup_rows_per_dst=[288, 1124, 254, 106, 8112, 7068, 4, 1394] combine_rows_per_src=[2658, 7730, 11892, 5664, 14678, 12612, 13372, 11348] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81812 payload_rows_remote=71150 dup_rows=16492 dup_rows_remote=13402 dedup_ratio=0.167765 router_tokens_per_expert=[3678, 1213, 54, 6, 154, 250, 949, 113, 203, 2899, 498, 2160, 477, 77, 561, 1, 217, 248, 971, 230, 185, 520, 76, 163, 4, 221, 1759, 747, 1855, 453, 172, 952, 1143, 328, 163, 0, 1004, 94, 20, 3243, 69, 779, 1785, 71, 2526, 346, 5305, 502, 4983, 127, 69, 368, 0, 88, 109, 371, 61, 37, 11, 1215, 1238, 153, 186, 692] payload_rows_per_dst=[9982, 10662, 13364, 10290, 9560, 6082, 11624, 10248] dup_rows_per_dst=[2852, 3090, 2466, 2078, 2430, 1022, 564, 1990] combine_rows_per_src=[9982, 10662, 13364, 10290, 9560, 6082, 11624, 10248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88204 payload_rows_remote=78900 dup_rows=10100 dup_rows_remote=8764 dedup_ratio=0.102743 router_tokens_per_expert=[4782, 588, 0, 0, 1, 5, 1460, 31, 0, 1934, 7, 4173, 3, 0, 462, 0, 5, 153, 606, 0, 12, 439, 0, 9, 1, 1, 4965, 612, 1733, 15, 138, 1487, 29, 20, 2, 0, 20, 2, 0, 3457, 4, 26, 929, 149, 5366, 86, 7285, 13, 6703, 0, 3, 159, 0, 76, 0, 5, 88, 0, 0, 33, 289, 31, 0, 755] payload_rows_per_dst=[11420, 10986, 15848, 9304, 7012, 8618, 13496, 11520] dup_rows_per_dst=[2314, 2172, 1170, 1336, 48, 1060, 396, 1604] combine_rows_per_src=[11420, 10986, 15848, 9304, 7012, 8618, 13496, 11520] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78692 payload_rows_remote=69220 dup_rows=19612 dup_rows_remote=17506 dedup_ratio=0.199504 router_tokens_per_expert=[4680, 352, 0, 0, 2, 0, 1292, 8, 2, 4052, 3, 3888, 5, 0, 862, 0, 2, 147, 916, 0, 1, 349, 0, 4, 0, 1, 3399, 627, 1533, 7, 138, 1017, 11, 3, 0, 0, 8, 0, 0, 5410, 0, 10, 404, 149, 3921, 184, 7316, 3, 6897, 81, 1, 176, 0, 73, 0, 3, 85, 0, 0, 12, 335, 74, 1, 708] payload_rows_per_dst=[4918, 12826, 2836, 6610, 13136, 15064, 13830, 9472] dup_rows_per_dst=[972, 3800, 2, 36, 5504, 6560, 632, 2106] combine_rows_per_src=[4918, 12826, 2836, 6610, 13136, 15064, 13830, 9472] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86850 payload_rows_remote=71022 dup_rows=11454 dup_rows_remote=10556 dedup_ratio=0.116516 router_tokens_per_expert=[4845, 758, 0, 0, 0, 0, 1469, 55, 0, 1377, 0, 4302, 0, 0, 382, 0, 0, 179, 448, 0, 0, 416, 0, 0, 0, 0, 6000, 635, 1207, 0, 141, 1475, 0, 0, 0, 0, 0, 0, 0, 3207, 0, 0, 927, 126, 5627, 102, 7320, 0, 6767, 0, 0, 181, 0, 89, 0, 0, 74, 0, 0, 0, 230, 2, 0, 811] payload_rows_per_dst=[11714, 10446, 15828, 14772, 6414, 2164, 13646, 11866] dup_rows_per_dst=[2540, 1676, 898, 4144, 0, 146, 428, 1622] combine_rows_per_src=[11714, 10446, 15828, 14772, 6414, 2164, 13646, 11866] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83708 payload_rows_remote=72500 dup_rows=14596 dup_rows_remote=10520 dedup_ratio=0.148478 router_tokens_per_expert=[1367, 39, 1166, 196, 808, 3213, 326, 181, 2495, 163, 1012, 0, 112, 63, 50, 60, 3, 53, 451, 846, 113, 96, 395, 228, 213, 58, 160, 192, 59, 109, 73, 179, 666, 10, 3687, 4, 119, 175, 2834, 664, 51, 4985, 1855, 4797, 87, 561, 4509, 154, 79, 487, 131, 92, 97, 112, 433, 3534, 1227, 1044, 40, 575, 635, 245, 415, 369] payload_rows_per_dst=[12190, 11208, 12804, 11014, 7902, 5354, 9680, 13556] dup_rows_per_dst=[2402, 4076, 1536, 666, 1042, 62, 250, 4562] combine_rows_per_src=[12190, 11208, 12804, 11014, 7902, 5354, 9680, 13556] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=292954112 dispatch_meta_bytes=334240 combine_payload_bytes=292954112 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=293288352 ingress_bytes=362160128 link_bytes=362160128 token_bytes=586242464 weight_movement_bytes=69206016 total_nvlink_bytes=655448480 weight_to_token_ratio=0.118050 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=46080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.362779405e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=224935936 dispatch_meta_bytes=340184 combine_payload_bytes=224935936 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=225276120 ingress_bytes=259538944 link_bytes=259538944 token_bytes=450212056 weight_movement_bytes=34603008 total_nvlink_bytes=484815064 weight_to_token_ratio=0.076859 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=30976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.766241517e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=301006848 dispatch_meta_bytes=329560 combine_payload_bytes=301006848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=301336408 ingress_bytes=335609856 link_bytes=335609856 token_bytes=602343256 weight_movement_bytes=34603008 total_nvlink_bytes=636946264 weight_to_token_ratio=0.057447 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=46592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262872869e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=338100224 dispatch_meta_bytes=364008 combine_payload_bytes=338100224 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=338464232 ingress_bytes=476512256 link_bytes=476512256 token_bytes=676564456 weight_movement_bytes=138412032 total_nvlink_bytes=814976488 weight_to_token_ratio=0.204581 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.793077257e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79836 payload_rows_remote=65806 dup_rows=18468 dup_rows_remote=17878 dedup_ratio=0.187866 router_tokens_per_expert=[4063, 422, 0, 0, 0, 0, 1414, 0, 0, 3368, 0, 4237, 0, 0, 723, 0, 0, 149, 864, 0, 0, 365, 0, 0, 0, 0, 3762, 609, 1293, 0, 146, 1082, 0, 0, 0, 0, 0, 0, 0, 5454, 0, 0, 429, 154, 4690, 132, 7320, 0, 7015, 67, 0, 154, 0, 74, 0, 0, 84, 0, 0, 0, 227, 59, 0, 796] payload_rows_per_dst=[9864, 7996, 2756, 6246, 13116, 15734, 14030, 10094] dup_rows_per_dst=[1934, 186, 0, 14, 6266, 7860, 590, 1618] combine_rows_per_src=[9864, 7996, 2756, 6246, 13116, 15734, 14030, 10094] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=296960000 dispatch_meta_bytes=332080 combine_payload_bytes=296960000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=297292080 ingress_bytes=331563008 link_bytes=331563008 token_bytes=594252080 weight_movement_bytes=34603008 total_nvlink_bytes=628855088 weight_to_token_ratio=0.058230 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=21376 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.247644906e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89134 payload_rows_remote=77424 dup_rows=9170 dup_rows_remote=7194 dedup_ratio=0.093282 router_tokens_per_expert=[3201, 0, 0, 101, 776, 4780, 0, 0, 1340, 96, 741, 0, 0, 0, 0, 0, 0, 80, 165, 0, 0, 80, 0, 0, 172, 0, 0, 0, 6, 80, 0, 66, 0, 0, 5656, 0, 0, 0, 1228, 396, 0, 6823, 3090, 7982, 0, 0, 3523, 0, 0, 0, 0, 80, 0, 0, 0, 6457, 734, 0, 0, 380, 686, 140, 247, 46] payload_rows_per_dst=[13736, 4966, 13992, 16108, 11710, 6180, 13074, 9368] dup_rows_per_dst=[3980, 262, 304, 504, 1976, 0, 0, 2144] combine_rows_per_src=[13736, 4966, 13992, 16108, 11710, 6180, 13074, 9368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=291430400 dispatch_meta_bytes=338208 combine_payload_bytes=291430400 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=291768608 ingress_bytes=360636416 link_bytes=360636416 token_bytes=583199008 weight_movement_bytes=69206016 total_nvlink_bytes=652405024 weight_to_token_ratio=0.118666 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.357045800e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=285425664 dispatch_meta_bytes=319448 combine_payload_bytes=285425664 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=285745112 ingress_bytes=423837696 link_bytes=423837696 token_bytes=571170776 weight_movement_bytes=138412032 total_nvlink_bytes=709582808 weight_to_token_ratio=0.242330 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=20864 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.594867128e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=272719872 dispatch_meta_bytes=339712 combine_payload_bytes=272719872 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=273059584 ingress_bytes=307322880 link_bytes=307322880 token_bytes=545779456 weight_movement_bytes=34603008 total_nvlink_bytes=580382464 weight_to_token_ratio=0.063401 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=47104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.156431256e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=269541376 dispatch_meta_bytes=334736 combine_payload_bytes=269541376 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=269876112 ingress_bytes=338747392 link_bytes=338747392 token_bytes=539417488 weight_movement_bytes=69206016 total_nvlink_bytes=608623504 weight_to_token_ratio=0.128298 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=46080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274679165e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=323174400 dispatch_meta_bytes=350656 combine_payload_bytes=323174400 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=323525056 ingress_bytes=392380416 link_bytes=392380416 token_bytes=646699456 weight_movement_bytes=69206016 total_nvlink_bytes=715905472 weight_to_token_ratio=0.107014 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.476495916e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=283525120 dispatch_meta_bytes=346904 combine_payload_bytes=283525120 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=283872024 ingress_bytes=352731136 link_bytes=352731136 token_bytes=567397144 weight_movement_bytes=69206016 total_nvlink_bytes=636603160 weight_to_token_ratio=0.121971 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=36608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.327298868e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77118 payload_rows_remote=66486 dup_rows=21186 dup_rows_remote=19208 dedup_ratio=0.215515 router_tokens_per_expert=[4474, 468, 0, 0, 4, 2, 1327, 11, 2, 2436, 10, 4282, 3, 0, 748, 0, 5, 145, 774, 7, 1, 359, 1, 1, 4, 4, 4274, 639, 1527, 5, 113, 1120, 21, 10, 3, 0, 18, 3, 0, 5194, 1, 13, 542, 112, 4433, 142, 7379, 11, 7019, 48, 3, 159, 0, 72, 0, 27, 64, 0, 0, 19, 275, 72, 2, 764] payload_rows_per_dst=[10632, 12022, 15830, 16336, 10452, 1584, 474, 9788] dup_rows_per_dst=[1978, 2940, 1514, 13036, 46, 58, 144, 1470] combine_rows_per_src=[10632, 12022, 15830, 16336, 10452, 1584, 474, 9788] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=290906112 dispatch_meta_bytes=326312 combine_payload_bytes=290906112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291232424 ingress_bytes=325509120 link_bytes=325509120 token_bytes=582138536 weight_movement_bytes=34603008 total_nvlink_bytes=616741544 weight_to_token_ratio=0.059441 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=37760 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.224864613e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85052 payload_rows_remote=70220 dup_rows=13252 dup_rows_remote=10188 dedup_ratio=0.134806 router_tokens_per_expert=[0, 0, 138, 784, 0, 0, 1618, 0, 0, 3385, 955, 0, 0, 0, 785, 0, 167, 467, 0, 1044, 0, 72, 0, 0, 5169, 0, 0, 306, 0, 0, 0, 0, 74, 355, 0, 0, 2285, 1656, 0, 89, 83, 0, 5897, 0, 0, 0, 0, 0, 622, 0, 89, 0, 7518, 7, 71, 0, 7012, 0, 0, 0, 7198, 521, 785, 0] payload_rows_per_dst=[15296, 8952, 14832, 10950, 11674, 11796, 8940, 2612] dup_rows_per_dst=[3808, 1298, 3064, 0, 4358, 164, 560, 0] combine_rows_per_src=[15296, 8952, 14832, 10950, 11674, 11796, 8940, 2612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82780 payload_rows_remote=69684 dup_rows=15524 dup_rows_remote=10178 dedup_ratio=0.157918 router_tokens_per_expert=[4607, 491, 0, 0, 0, 0, 1423, 2, 0, 2855, 0, 4286, 0, 0, 648, 0, 0, 199, 784, 0, 0, 357, 0, 0, 0, 0, 4284, 623, 1591, 0, 154, 1194, 0, 0, 0, 0, 0, 0, 0, 4581, 0, 0, 418, 122, 4487, 199, 7326, 0, 6888, 54, 0, 157, 0, 99, 0, 0, 67, 0, 0, 0, 288, 86, 0, 882] payload_rows_per_dst=[10884, 7296, 8382, 7082, 12412, 13096, 13780, 9848] dup_rows_per_dst=[2162, 158, 554, 42, 4874, 5346, 616, 1772] combine_rows_per_src=[10884, 7296, 8382, 7082, 12412, 13096, 13780, 9848] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=317128704 dispatch_meta_bytes=338472 combine_payload_bytes=317128704 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=317467176 ingress_bytes=386334720 link_bytes=386334720 token_bytes=634595880 weight_movement_bytes=69206016 total_nvlink_bytes=703801896 weight_to_token_ratio=0.109055 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.453746449e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78692 payload_rows_remote=69220 dup_rows=19612 dup_rows_remote=17506 dedup_ratio=0.199504 router_tokens_per_expert=[4680, 352, 0, 0, 2, 0, 1292, 8, 2, 4052, 3, 3888, 5, 0, 862, 0, 2, 147, 916, 0, 1, 349, 0, 4, 0, 1, 3399, 627, 1533, 7, 138, 1017, 11, 3, 0, 0, 8, 0, 0, 5410, 0, 10, 404, 149, 3921, 184, 7316, 3, 6897, 81, 1, 176, 0, 73, 0, 3, 85, 0, 0, 12, 335, 74, 1, 708] payload_rows_per_dst=[4918, 12826, 2836, 6610, 13136, 15064, 13830, 9472] dup_rows_per_dst=[972, 3800, 2, 36, 5504, 6560, 632, 2106] combine_rows_per_src=[4918, 12826, 2836, 6610, 13136, 15064, 13830, 9472] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88204 payload_rows_remote=78900 dup_rows=10100 dup_rows_remote=8764 dedup_ratio=0.102743 router_tokens_per_expert=[4782, 588, 0, 0, 1, 5, 1460, 31, 0, 1934, 7, 4173, 3, 0, 462, 0, 5, 153, 606, 0, 12, 439, 0, 9, 1, 1, 4965, 612, 1733, 15, 138, 1487, 29, 20, 2, 0, 20, 2, 0, 3457, 4, 26, 929, 149, 5366, 86, 7285, 13, 6703, 0, 3, 159, 0, 76, 0, 5, 88, 0, 0, 33, 289, 31, 0, 755] payload_rows_per_dst=[11420, 10986, 15848, 9304, 7012, 8618, 13496, 11520] dup_rows_per_dst=[2314, 2172, 1170, 1336, 48, 1060, 396, 1604] combine_rows_per_src=[11420, 10986, 15848, 9304, 7012, 8618, 13496, 11520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81812 payload_rows_remote=71150 dup_rows=16492 dup_rows_remote=13402 dedup_ratio=0.167765 router_tokens_per_expert=[3678, 1213, 54, 6, 154, 250, 949, 113, 203, 2899, 498, 2160, 477, 77, 561, 1, 217, 248, 971, 230, 185, 520, 76, 163, 4, 221, 1759, 747, 1855, 453, 172, 952, 1143, 328, 163, 0, 1004, 94, 20, 3243, 69, 779, 1785, 71, 2526, 346, 5305, 502, 4983, 127, 69, 368, 0, 88, 109, 371, 61, 37, 11, 1215, 1238, 153, 186, 692] payload_rows_per_dst=[9982, 10662, 13364, 10290, 9560, 6082, 11624, 10248] dup_rows_per_dst=[2852, 3090, 2466, 2078, 2430, 1022, 564, 1990] combine_rows_per_src=[9982, 10662, 13364, 10290, 9560, 6082, 11624, 10248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=272326656 dispatch_meta_bytes=342776 combine_payload_bytes=272326656 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=272669432 ingress_bytes=341532672 link_bytes=341532672 token_bytes=544996088 weight_movement_bytes=69206016 total_nvlink_bytes=614202104 weight_to_token_ratio=0.126984 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=32512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.285159949e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74490 payload_rows_remote=60536 dup_rows=23814 dup_rows_remote=16310 dedup_ratio=0.242249 router_tokens_per_expert=[32, 196, 1750, 206, 2, 18, 1533, 0, 0, 55, 141, 66, 5588, 1, 908, 3970, 201, 16, 13, 132, 159, 3441, 165, 767, 52, 106, 57, 5933, 71, 1281, 3881, 120, 223, 27, 138, 5, 45, 3678, 443, 118, 143, 0, 561, 1, 481, 1405, 539, 55, 1, 286, 216, 202, 159, 301, 915, 121, 86, 3874, 0, 257, 857, 1849, 1130, 175] payload_rows_per_dst=[10990, 13954, 9642, 15508, 8750, 4594, 3898, 7154] dup_rows_per_dst=[4246, 7504, 146, 7480, 604, 1776, 504, 1554] combine_rows_per_src=[10990, 13954, 9642, 15508, 8750, 4594, 3898, 7154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88982 payload_rows_remote=72746 dup_rows=9322 dup_rows_remote=7452 dedup_ratio=0.094828 router_tokens_per_expert=[0, 0, 84, 0, 0, 0, 1599, 0, 0, 0, 144, 137, 7384, 0, 0, 5018, 2, 0, 0, 0, 0, 5497, 0, 0, 0, 0, 0, 8118, 0, 0, 5834, 0, 0, 0, 144, 0, 0, 6559, 0, 0, 0, 0, 860, 0, 0, 0, 0, 0, 0, 0, 259, 0, 0, 601, 0, 75, 0, 5915, 0, 0, 837, 85, 0, 0] payload_rows_per_dst=[3230, 562, 15070, 11668, 13406, 15150, 16236, 13660] dup_rows_per_dst=[136, 0, 5964, 0, 0, 1338, 1870, 14] combine_rows_per_src=[3230, 562, 15070, 11668, 13406, 15150, 16236, 13660] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=287621120 dispatch_meta_bytes=321632 combine_payload_bytes=287621120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=287942752 ingress_bytes=322224128 link_bytes=322224128 token_bytes=575563872 weight_movement_bytes=34603008 total_nvlink_bytes=610166880 weight_to_token_ratio=0.060120 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=31616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.212503452e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=285425664 dispatch_meta_bytes=319448 combine_payload_bytes=285425664 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=285745112 ingress_bytes=423837696 link_bytes=423837696 token_bytes=571170776 weight_movement_bytes=138412032 total_nvlink_bytes=709582808 weight_to_token_ratio=0.242330 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=20864 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.594867128e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=283525120 dispatch_meta_bytes=346904 combine_payload_bytes=283525120 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=283872024 ingress_bytes=352731136 link_bytes=352731136 token_bytes=567397144 weight_movement_bytes=69206016 total_nvlink_bytes=636603160 weight_to_token_ratio=0.121971 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=36608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.327298868e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=323174400 dispatch_meta_bytes=350656 combine_payload_bytes=323174400 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=323525056 ingress_bytes=392380416 link_bytes=392380416 token_bytes=646699456 weight_movement_bytes=69206016 total_nvlink_bytes=715905472 weight_to_token_ratio=0.107014 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.476495916e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=291430400 dispatch_meta_bytes=338208 combine_payload_bytes=291430400 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=291768608 ingress_bytes=360636416 link_bytes=360636416 token_bytes=583199008 weight_movement_bytes=69206016 total_nvlink_bytes=652405024 weight_to_token_ratio=0.118666 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.357045800e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=297967616 dispatch_meta_bytes=320792 combine_payload_bytes=297967616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=298288408 ingress_bytes=332570624 link_bytes=332570624 token_bytes=596256024 weight_movement_bytes=34603008 total_nvlink_bytes=630859032 weight_to_token_ratio=0.058034 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=40960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.251436484e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89370 payload_rows_remote=82336 dup_rows=8934 dup_rows_remote=8934 dedup_ratio=0.090881 router_tokens_per_expert=[4760, 716, 0, 0, 0, 0, 1538, 55, 0, 1313, 0, 4377, 0, 0, 334, 0, 0, 160, 511, 0, 0, 486, 0, 0, 0, 0, 5757, 666, 1087, 0, 151, 1585, 0, 0, 0, 0, 0, 0, 0, 3517, 0, 0, 951, 163, 5516, 84, 7271, 0, 6794, 0, 0, 153, 0, 80, 0, 0, 97, 0, 0, 0, 259, 0, 0, 771] payload_rows_per_dst=[11636, 10482, 15864, 6910, 7034, 12028, 13700, 11716] dup_rows_per_dst=[2502, 1566, 992, 68, 0, 1882, 354, 1570] combine_rows_per_src=[11636, 10482, 15864, 6910, 7034, 12028, 13700, 11716] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73972 payload_rows_remote=61106 dup_rows=24332 dup_rows_remote=21988 dedup_ratio=0.247518 router_tokens_per_expert=[4, 6, 201, 0, 0, 0, 1357, 0, 0, 0, 119, 83, 7415, 1, 22, 4898, 8, 0, 1, 0, 3, 5307, 2, 14, 1, 1, 0, 8067, 1, 23, 6037, 1, 2, 0, 119, 0, 0, 6816, 2, 3, 1, 0, 850, 0, 3, 24, 13, 0, 0, 8, 290, 2, 1, 633, 14, 72, 2, 5806, 0, 6, 790, 111, 10, 2] payload_rows_per_dst=[12866, 15772, 10668, 16214, 13462, 1758, 1404, 1828] dup_rows_per_dst=[2344, 9726, 2, 11586, 0, 24, 636, 14] combine_rows_per_src=[12866, 15772, 10668, 16214, 13462, 1758, 1404, 1828] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=337248256 dispatch_meta_bytes=365080 combine_payload_bytes=337248256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=337613336 ingress_bytes=371851264 link_bytes=371851264 token_bytes=674861592 weight_movement_bytes=34603008 total_nvlink_bytes=709464600 weight_to_token_ratio=0.051274 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=34176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.399246370e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92310 payload_rows_remote=77158 dup_rows=5994 dup_rows_remote=4266 dedup_ratio=0.060974 router_tokens_per_expert=[0, 0, 172, 0, 0, 0, 1551, 0, 0, 0, 154, 146, 7348, 0, 0, 4813, 0, 0, 0, 0, 0, 5241, 0, 0, 0, 0, 0, 8091, 0, 0, 5964, 0, 0, 0, 154, 0, 0, 6604, 0, 0, 0, 0, 1092, 0, 0, 0, 0, 0, 0, 0, 229, 0, 0, 633, 0, 100, 0, 5905, 0, 0, 842, 113, 0, 0] payload_rows_per_dst=[7464, 8160, 11338, 6800, 13516, 15152, 16186, 13694] dup_rows_per_dst=[1110, 0, 1210, 0, 0, 1728, 1920, 26] combine_rows_per_src=[7464, 8160, 11338, 6800, 13516, 15152, 16186, 13694] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79836 payload_rows_remote=65806 dup_rows=18468 dup_rows_remote=17878 dedup_ratio=0.187866 router_tokens_per_expert=[4063, 422, 0, 0, 0, 0, 1414, 0, 0, 3368, 0, 4237, 0, 0, 723, 0, 0, 149, 864, 0, 0, 365, 0, 0, 0, 0, 3762, 609, 1293, 0, 146, 1082, 0, 0, 0, 0, 0, 0, 0, 5454, 0, 0, 429, 154, 4690, 132, 7320, 0, 7015, 67, 0, 154, 0, 74, 0, 0, 84, 0, 0, 0, 227, 59, 0, 796] payload_rows_per_dst=[9864, 7996, 2756, 6246, 13116, 15734, 14030, 10094] dup_rows_per_dst=[1934, 186, 0, 14, 6266, 7860, 590, 1618] combine_rows_per_src=[9864, 7996, 2756, 6246, 13116, 15734, 14030, 10094] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78992 payload_rows_remote=63310 dup_rows=19312 dup_rows_remote=10490 dedup_ratio=0.196452 router_tokens_per_expert=[0, 0, 203, 0, 0, 1, 1520, 0, 0, 0, 139, 89, 7429, 0, 10, 5192, 0, 0, 1, 0, 1, 5592, 1, 6, 1, 0, 0, 8076, 0, 12, 5425, 0, 0, 0, 137, 0, 0, 6077, 7, 3, 1, 0, 1111, 0, 2, 12, 9, 1, 0, 2, 267, 1, 0, 600, 6, 74, 0, 6077, 0, 0, 859, 201, 7, 0] payload_rows_per_dst=[3266, 476, 15076, 10874, 2226, 15178, 16214, 15682] dup_rows_per_dst=[182, 0, 6510, 2, 6, 1952, 1838, 8822] combine_rows_per_src=[3266, 476, 15076, 10874, 2226, 15178, 16214, 15682] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89318 payload_rows_remote=75414 dup_rows=8986 dup_rows_remote=8676 dedup_ratio=0.091410 router_tokens_per_expert=[3407, 0, 0, 102, 762, 4100, 0, 0, 1317, 74, 735, 0, 0, 0, 0, 0, 0, 86, 128, 0, 0, 88, 0, 0, 229, 0, 0, 0, 0, 90, 0, 50, 0, 0, 5564, 0, 0, 0, 1234, 457, 0, 6805, 3009, 7916, 0, 0, 4180, 0, 0, 0, 0, 90, 0, 0, 0, 6575, 701, 0, 0, 338, 595, 209, 265, 46] payload_rows_per_dst=[13182, 13144, 13904, 15932, 3104, 6018, 13330, 10704] dup_rows_per_dst=[3560, 2236, 310, 638, 278, 0, 0, 1964] combine_rows_per_src=[13182, 13144, 13904, 15932, 3104, 6018, 13330, 10704] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92218 payload_rows_remote=92150 dup_rows=6086 dup_rows_remote=6078 dedup_ratio=0.061910 router_tokens_per_expert=[1, 6, 622, 2, 0, 1, 1319, 0, 0, 0, 137, 192, 7214, 2, 30, 5055, 3, 0, 3, 2, 2, 4289, 7, 20, 4, 0, 1, 7996, 1, 31, 6250, 1, 1, 1, 136, 0, 1, 7466, 7, 6, 2, 0, 974, 0, 1, 34, 27, 0, 0, 21, 82, 1, 0, 587, 21, 76, 4, 5191, 0, 3, 755, 546, 14, 4] payload_rows_per_dst=[13632, 10830, 8642, 68, 15222, 14998, 16190, 12636] dup_rows_per_dst=[2770, 2, 10, 8, 14, 1506, 1378, 398] combine_rows_per_src=[13632, 10830, 8642, 68, 15222, 14998, 16190, 12636] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78992 payload_rows_remote=63310 dup_rows=19312 dup_rows_remote=10490 dedup_ratio=0.196452 router_tokens_per_expert=[0, 0, 203, 0, 0, 1, 1520, 0, 0, 0, 139, 89, 7429, 0, 10, 5192, 0, 0, 1, 0, 1, 5592, 1, 6, 1, 0, 0, 8076, 0, 12, 5425, 0, 0, 0, 137, 0, 0, 6077, 7, 3, 1, 0, 1111, 0, 2, 12, 9, 1, 0, 2, 267, 1, 0, 600, 6, 74, 0, 6077, 0, 0, 859, 201, 7, 0] payload_rows_per_dst=[3266, 476, 15076, 10874, 2226, 15178, 16214, 15682] dup_rows_per_dst=[182, 0, 6510, 2, 6, 1952, 1838, 8822] combine_rows_per_src=[3266, 476, 15076, 10874, 2226, 15178, 16214, 15682] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85564 payload_rows_remote=78200 dup_rows=12740 dup_rows_remote=12740 dedup_ratio=0.129598 router_tokens_per_expert=[0, 0, 746, 0, 0, 0, 1228, 0, 0, 0, 142, 182, 7239, 0, 0, 5062, 0, 0, 0, 0, 0, 3682, 0, 0, 0, 0, 0, 8100, 0, 0, 7018, 0, 0, 0, 129, 0, 0, 7612, 0, 0, 0, 0, 800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 602, 0, 90, 0, 5365, 0, 0, 495, 660, 0, 0] payload_rows_per_dst=[14936, 13610, 7364, 15074, 15482, 6882, 2236, 9980] dup_rows_per_dst=[3048, 5766, 0, 3022, 0, 592, 280, 32] combine_rows_per_src=[14936, 13610, 7364, 15074, 15482, 6882, 2236, 9980] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89370 payload_rows_remote=82336 dup_rows=8934 dup_rows_remote=8934 dedup_ratio=0.090881 router_tokens_per_expert=[4760, 716, 0, 0, 0, 0, 1538, 55, 0, 1313, 0, 4377, 0, 0, 334, 0, 0, 160, 511, 0, 0, 486, 0, 0, 0, 0, 5757, 666, 1087, 0, 151, 1585, 0, 0, 0, 0, 0, 0, 0, 3517, 0, 0, 951, 163, 5516, 84, 7271, 0, 6794, 0, 0, 153, 0, 80, 0, 0, 97, 0, 0, 0, 259, 0, 0, 771] payload_rows_per_dst=[11636, 10482, 15864, 6910, 7034, 12028, 13700, 11716] dup_rows_per_dst=[2502, 1566, 992, 68, 0, 1882, 354, 1570] combine_rows_per_src=[11636, 10482, 15864, 6910, 7034, 12028, 13700, 11716] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=247955456 dispatch_meta_bytes=307384 combine_payload_bytes=247955456 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=248262840 ingress_bytes=351764480 link_bytes=351764480 token_bytes=496218296 weight_movement_bytes=103809024 total_nvlink_bytes=600027320 weight_to_token_ratio=0.209200 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=28416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.323661419e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=269541376 dispatch_meta_bytes=334736 combine_payload_bytes=269541376 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=269876112 ingress_bytes=338747392 link_bytes=338747392 token_bytes=539417488 weight_movement_bytes=69206016 total_nvlink_bytes=608623504 weight_to_token_ratio=0.128298 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=46080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274679165e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74490 payload_rows_remote=60536 dup_rows=23814 dup_rows_remote=16310 dedup_ratio=0.242249 router_tokens_per_expert=[32, 196, 1750, 206, 2, 18, 1533, 0, 0, 55, 141, 66, 5588, 1, 908, 3970, 201, 16, 13, 132, 159, 3441, 165, 767, 52, 106, 57, 5933, 71, 1281, 3881, 120, 223, 27, 138, 5, 45, 3678, 443, 118, 143, 0, 561, 1, 481, 1405, 539, 55, 1, 286, 216, 202, 159, 301, 915, 121, 86, 3874, 0, 257, 857, 1849, 1130, 175] payload_rows_per_dst=[10990, 13954, 9642, 15508, 8750, 4594, 3898, 7154] dup_rows_per_dst=[4246, 7504, 146, 7480, 604, 1776, 504, 1554] combine_rows_per_src=[10990, 13954, 9642, 15508, 8750, 4594, 3898, 7154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90606 payload_rows_remote=74512 dup_rows=7698 dup_rows_remote=3596 dedup_ratio=0.078308 router_tokens_per_expert=[0, 0, 0, 0, 0, 47, 143, 0, 0, 6, 88, 0, 0, 70, 918, 0, 3351, 0, 124, 7629, 7641, 0, 0, 0, 0, 128, 511, 0, 7677, 0, 70, 0, 3437, 0, 0, 0, 251, 6265, 7038, 75, 0, 0, 0, 0, 0, 0, 0, 78, 0, 1881, 0, 540, 0, 0, 0, 0, 0, 0, 0, 166, 404, 0, 339, 275] payload_rows_per_dst=[9546, 2160, 15530, 8088, 9778, 15276, 16094, 14134] dup_rows_per_dst=[52, 4, 0, 32, 2606, 138, 4102, 764] combine_rows_per_src=[9546, 2160, 15530, 8088, 9778, 15276, 16094, 14134] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=250290176 dispatch_meta_bytes=332376 combine_payload_bytes=250290176 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=250622552 ingress_bytes=284893184 link_bytes=284893184 token_bytes=500912728 weight_movement_bytes=34603008 total_nvlink_bytes=535515736 weight_to_token_ratio=0.069080 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=32896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.072030116e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=316039168 dispatch_meta_bytes=325696 combine_payload_bytes=316039168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=316364864 ingress_bytes=350642176 link_bytes=350642176 token_bytes=632404032 weight_movement_bytes=34603008 total_nvlink_bytes=667007040 weight_to_token_ratio=0.054717 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=39552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.319438279e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73972 payload_rows_remote=61106 dup_rows=24332 dup_rows_remote=21988 dedup_ratio=0.247518 router_tokens_per_expert=[4, 6, 201, 0, 0, 0, 1357, 0, 0, 0, 119, 83, 7415, 1, 22, 4898, 8, 0, 1, 0, 3, 5307, 2, 14, 1, 1, 0, 8067, 1, 23, 6037, 1, 2, 0, 119, 0, 0, 6816, 2, 3, 1, 0, 850, 0, 3, 24, 13, 0, 0, 8, 290, 2, 1, 633, 14, 72, 2, 5806, 0, 6, 790, 111, 10, 2] payload_rows_per_dst=[12866, 15772, 10668, 16214, 13462, 1758, 1404, 1828] dup_rows_per_dst=[2344, 9726, 2, 11586, 0, 24, 636, 14] combine_rows_per_src=[12866, 15772, 10668, 16214, 13462, 1758, 1404, 1828] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=377446400 dispatch_meta_bytes=392912 combine_payload_bytes=377446400 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=377839312 ingress_bytes=481255424 link_bytes=481255424 token_bytes=755285712 weight_movement_bytes=103809024 total_nvlink_bytes=859094736 weight_to_token_ratio=0.137443 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=21632 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.810925416e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=259317760 dispatch_meta_bytes=295200 combine_payload_bytes=259317760 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=259612960 ingress_bytes=328523776 link_bytes=328523776 token_bytes=518930720 weight_movement_bytes=69206016 total_nvlink_bytes=588136736 weight_to_token_ratio=0.133363 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=32256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.236208521e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=320307200 dispatch_meta_bytes=363760 combine_payload_bytes=320307200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=320670960 ingress_bytes=354910208 link_bytes=354910208 token_bytes=640978160 weight_movement_bytes=34603008 total_nvlink_bytes=675581168 weight_to_token_ratio=0.053985 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=36480 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.335498540e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=337248256 dispatch_meta_bytes=365080 combine_payload_bytes=337248256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=337613336 ingress_bytes=371851264 link_bytes=371851264 token_bytes=674861592 weight_movement_bytes=34603008 total_nvlink_bytes=709464600 weight_to_token_ratio=0.051274 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=34176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.399246370e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=250290176 dispatch_meta_bytes=332376 combine_payload_bytes=250290176 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=250622552 ingress_bytes=284893184 link_bytes=284893184 token_bytes=500912728 weight_movement_bytes=34603008 total_nvlink_bytes=535515736 weight_to_token_ratio=0.069080 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=32896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.072030116e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88982 payload_rows_remote=72746 dup_rows=9322 dup_rows_remote=7452 dedup_ratio=0.094828 router_tokens_per_expert=[0, 0, 84, 0, 0, 0, 1599, 0, 0, 0, 144, 137, 7384, 0, 0, 5018, 2, 0, 0, 0, 0, 5497, 0, 0, 0, 0, 0, 8118, 0, 0, 5834, 0, 0, 0, 144, 0, 0, 6559, 0, 0, 0, 0, 860, 0, 0, 0, 0, 0, 0, 0, 259, 0, 0, 601, 0, 75, 0, 5915, 0, 0, 837, 85, 0, 0] payload_rows_per_dst=[3230, 562, 15070, 11668, 13406, 15150, 16236, 13660] dup_rows_per_dst=[136, 0, 5964, 0, 0, 1338, 1870, 14] combine_rows_per_src=[3230, 562, 15070, 11668, 13406, 15150, 16236, 13660] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81606 payload_rows_remote=70442 dup_rows=16698 dup_rows_remote=15174 dedup_ratio=0.169861 router_tokens_per_expert=[19, 5, 1266, 1, 1413, 93, 502, 363, 39, 42, 544, 68, 16, 198, 356, 106, 3235, 523, 223, 5132, 4975, 11, 584, 188, 0, 1263, 749, 0, 5476, 536, 471, 0, 1824, 175, 422, 148, 219, 3120, 4687, 83, 3, 288, 2, 1052, 29, 66, 0, 138, 91, 547, 403, 1391, 0, 1090, 0, 4, 472, 72, 634, 259, 421, 1286, 1488, 341] payload_rows_per_dst=[13962, 11164, 12816, 13700, 8132, 5952, 6050, 9830] dup_rows_per_dst=[2736, 1524, 4296, 3290, 1324, 230, 1002, 2296] combine_rows_per_src=[13962, 11164, 12816, 13700, 8132, 5952, 6050, 9830] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=377446400 dispatch_meta_bytes=392912 combine_payload_bytes=377446400 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=377839312 ingress_bytes=481255424 link_bytes=481255424 token_bytes=755285712 weight_movement_bytes=103809024 total_nvlink_bytes=859094736 weight_to_token_ratio=0.137443 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=21632 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.810925416e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=308895744 dispatch_meta_bytes=336360 combine_payload_bytes=308895744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=309232104 ingress_bytes=343498752 link_bytes=343498752 token_bytes=618127848 weight_movement_bytes=34603008 total_nvlink_bytes=652730856 weight_to_token_ratio=0.055980 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=44800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.292558150e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=247955456 dispatch_meta_bytes=307384 combine_payload_bytes=247955456 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=248262840 ingress_bytes=351764480 link_bytes=351764480 token_bytes=496218296 weight_movement_bytes=103809024 total_nvlink_bytes=600027320 weight_to_token_ratio=0.209200 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=28416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.323661419e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=305201152 dispatch_meta_bytes=312432 combine_payload_bytes=305201152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=305513584 ingress_bytes=339804160 link_bytes=339804160 token_bytes=610714736 weight_movement_bytes=34603008 total_nvlink_bytes=645317744 weight_to_token_ratio=0.056660 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=31232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.278655698e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93070 payload_rows_remote=77454 dup_rows=5234 dup_rows_remote=5234 dedup_ratio=0.053243 router_tokens_per_expert=[0, 0, 674, 0, 0, 0, 1283, 0, 0, 0, 148, 174, 7236, 0, 0, 5649, 0, 0, 0, 0, 0, 3331, 0, 0, 0, 0, 0, 8111, 0, 0, 7085, 0, 0, 0, 128, 0, 0, 7680, 0, 0, 0, 0, 602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 566, 0, 80, 0, 5226, 0, 0, 505, 674, 0, 0] payload_rows_per_dst=[15070, 11942, 6662, 0, 15616, 14906, 16224, 12650] dup_rows_per_dst=[3014, 0, 0, 0, 0, 770, 1290, 160] combine_rows_per_src=[15070, 11942, 6662, 0, 15616, 14906, 16224, 12650] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=317251584 dispatch_meta_bytes=330752 combine_payload_bytes=317251584 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=317582336 ingress_bytes=386457600 link_bytes=386457600 token_bytes=634833920 weight_movement_bytes=69206016 total_nvlink_bytes=704039936 weight_to_token_ratio=0.109014 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=47232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.454208836e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82038 payload_rows_remote=67694 dup_rows=16266 dup_rows_remote=16228 dedup_ratio=0.165466 router_tokens_per_expert=[1, 0, 19, 0, 23, 68, 44, 2, 0, 1, 105, 1, 0, 51, 887, 0, 3208, 11, 149, 7595, 7606, 0, 12, 2, 0, 132, 545, 0, 7667, 6, 50, 0, 3630, 7, 7, 4, 256, 6599, 7034, 72, 0, 2, 0, 17, 4, 1, 0, 75, 0, 1529, 3, 518, 0, 20, 0, 0, 15, 0, 15, 198, 290, 23, 297, 351] payload_rows_per_dst=[14344, 16026, 16316, 16186, 13234, 198, 3410, 2324] dup_rows_per_dst=[38, 1276, 12898, 614, 656, 0, 730, 54] combine_rows_per_src=[14344, 16026, 16316, 16186, 13234, 198, 3410, 2324] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92266 payload_rows_remote=77296 dup_rows=6038 dup_rows_remote=5930 dedup_ratio=0.061422 router_tokens_per_expert=[3, 0, 0, 0, 0, 92, 7, 0, 0, 0, 113, 0, 0, 55, 1047, 0, 3716, 0, 199, 7485, 7544, 0, 0, 0, 0, 125, 493, 0, 7652, 0, 55, 0, 3283, 0, 0, 0, 247, 6568, 7064, 100, 0, 0, 0, 0, 0, 0, 0, 54, 0, 1619, 0, 596, 0, 0, 0, 0, 0, 0, 0, 164, 177, 0, 336, 358] payload_rows_per_dst=[14332, 10256, 6808, 8658, 6766, 14970, 16074, 14402] dup_rows_per_dst=[0, 852, 0, 120, 494, 108, 3660, 804] combine_rows_per_src=[14332, 10256, 6808, 8658, 6766, 14970, 16074, 14402] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88102 payload_rows_remote=73842 dup_rows=10202 dup_rows_remote=9168 dedup_ratio=0.103780 router_tokens_per_expert=[4, 0, 9, 0, 11, 71, 276, 0, 0, 32, 102, 3, 0, 64, 786, 0, 3669, 4, 93, 7664, 7661, 0, 9, 7, 0, 185, 522, 0, 7682, 3, 71, 0, 3012, 3, 6, 1, 240, 6725, 7032, 74, 0, 3, 0, 7, 1, 0, 0, 99, 0, 1623, 1, 464, 0, 11, 0, 0, 2, 0, 12, 173, 269, 11, 325, 130] payload_rows_per_dst=[620, 1930, 15544, 8730, 15530, 15402, 16086, 14260] dup_rows_per_dst=[122, 44, 4, 170, 5206, 146, 3476, 1034] combine_rows_per_src=[620, 1930, 15544, 8730, 15530, 15402, 16086, 14260] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93876 payload_rows_remote=78692 dup_rows=4428 dup_rows_remote=3618 dedup_ratio=0.045044 router_tokens_per_expert=[21, 0, 30, 0, 31, 1, 58, 1, 0, 0, 152, 6, 0, 63, 1026, 0, 2850, 11, 158, 7557, 7554, 0, 21, 6, 0, 123, 644, 0, 7626, 6, 74, 0, 4795, 14, 25, 2, 239, 5597, 6944, 76, 0, 15, 0, 27, 3, 0, 0, 62, 0, 1313, 1, 565, 0, 27, 0, 0, 6, 0, 31, 214, 449, 30, 363, 335] payload_rows_per_dst=[14134, 16120, 6084, 15184, 9792, 15190, 3992, 13380] dup_rows_per_dst=[38, 1482, 8, 810, 510, 138, 772, 670] combine_rows_per_src=[14134, 16120, 6084, 15184, 9792, 15190, 3992, 13380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86850 payload_rows_remote=71022 dup_rows=11454 dup_rows_remote=10556 dedup_ratio=0.116516 router_tokens_per_expert=[4845, 758, 0, 0, 0, 0, 1469, 55, 0, 1377, 0, 4302, 0, 0, 382, 0, 0, 179, 448, 0, 0, 416, 0, 0, 0, 0, 6000, 635, 1207, 0, 141, 1475, 0, 0, 0, 0, 0, 0, 0, 3207, 0, 0, 927, 126, 5627, 102, 7320, 0, 6767, 0, 0, 181, 0, 89, 0, 0, 74, 0, 0, 0, 230, 2, 0, 811] payload_rows_per_dst=[11714, 10446, 15828, 14772, 6414, 2164, 13646, 11866] dup_rows_per_dst=[2540, 1676, 898, 4144, 0, 146, 428, 1622] combine_rows_per_src=[11714, 10446, 15828, 14772, 6414, 2164, 13646, 11866] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88102 payload_rows_remote=73842 dup_rows=10202 dup_rows_remote=9168 dedup_ratio=0.103780 router_tokens_per_expert=[4, 0, 9, 0, 11, 71, 276, 0, 0, 32, 102, 3, 0, 64, 786, 0, 3669, 4, 93, 7664, 7661, 0, 9, 7, 0, 185, 522, 0, 7682, 3, 71, 0, 3012, 3, 6, 1, 240, 6725, 7032, 74, 0, 3, 0, 7, 1, 0, 0, 99, 0, 1623, 1, 464, 0, 11, 0, 0, 2, 0, 12, 173, 269, 11, 325, 130] payload_rows_per_dst=[620, 1930, 15544, 8730, 15530, 15402, 16086, 14260] dup_rows_per_dst=[122, 44, 4, 170, 5206, 146, 3476, 1034] combine_rows_per_src=[620, 1930, 15544, 8730, 15530, 15402, 16086, 14260] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94264 payload_rows_remote=88506 dup_rows=4040 dup_rows_remote=4040 dedup_ratio=0.041097 router_tokens_per_expert=[15, 0, 0, 0, 0, 0, 73, 0, 0, 0, 191, 0, 0, 53, 1052, 0, 2700, 0, 179, 7523, 7544, 0, 0, 0, 0, 48, 721, 0, 7674, 0, 53, 0, 5705, 0, 14, 0, 248, 4928, 7072, 90, 0, 0, 0, 0, 0, 0, 0, 72, 0, 1257, 0, 611, 0, 0, 0, 0, 0, 0, 0, 196, 468, 0, 322, 343] payload_rows_per_dst=[14320, 16238, 5758, 16154, 11590, 15046, 3050, 12108] dup_rows_per_dst=[0, 1442, 0, 838, 524, 144, 686, 406] combine_rows_per_src=[14320, 16238, 5758, 16154, 11590, 15046, 3050, 12108] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=297967616 dispatch_meta_bytes=320792 combine_payload_bytes=297967616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=298288408 ingress_bytes=332570624 link_bytes=332570624 token_bytes=596256024 weight_movement_bytes=34603008 total_nvlink_bytes=630859032 weight_to_token_ratio=0.058034 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=40960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.251436484e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93070 payload_rows_remote=77454 dup_rows=5234 dup_rows_remote=5234 dedup_ratio=0.053243 router_tokens_per_expert=[0, 0, 674, 0, 0, 0, 1283, 0, 0, 0, 148, 174, 7236, 0, 0, 5649, 0, 0, 0, 0, 0, 3331, 0, 0, 0, 0, 0, 8111, 0, 0, 7085, 0, 0, 0, 128, 0, 0, 7680, 0, 0, 0, 0, 602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 566, 0, 80, 0, 5226, 0, 0, 505, 674, 0, 0] payload_rows_per_dst=[15070, 11942, 6662, 0, 15616, 14906, 16224, 12650] dup_rows_per_dst=[3014, 0, 0, 0, 0, 770, 1290, 160] combine_rows_per_src=[15070, 11942, 6662, 0, 15616, 14906, 16224, 12650] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84532 payload_rows_remote=74044 dup_rows=13772 dup_rows_remote=12852 dedup_ratio=0.140096 router_tokens_per_expert=[0, 0, 0, 194, 0, 0, 0, 0, 0, 0, 3, 75, 0, 75, 742, 0, 0, 0, 1859, 2579, 7600, 953, 0, 72, 7918, 0, 0, 7342, 92, 70, 0, 0, 0, 0, 154, 0, 58, 0, 0, 6361, 0, 972, 0, 0, 0, 0, 77, 0, 0, 0, 0, 361, 4893, 0, 365, 85, 1446, 74, 0, 0, 0, 4350, 1, 381] payload_rows_per_dst=[2554, 1634, 15774, 14864, 12972, 14968, 10488, 11278] dup_rows_per_dst=[50, 156, 5194, 4238, 174, 1814, 920, 1226] combine_rows_per_src=[2554, 1634, 15774, 14864, 12972, 14968, 10488, 11278] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=288530432 dispatch_meta_bytes=342464 combine_payload_bytes=288530432 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288872896 ingress_bytes=323133440 link_bytes=323133440 token_bytes=577403328 weight_movement_bytes=34603008 total_nvlink_bytes=612006336 weight_to_token_ratio=0.059929 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=39680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.215925120e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81606 payload_rows_remote=70442 dup_rows=16698 dup_rows_remote=15174 dedup_ratio=0.169861 router_tokens_per_expert=[19, 5, 1266, 1, 1413, 93, 502, 363, 39, 42, 544, 68, 16, 198, 356, 106, 3235, 523, 223, 5132, 4975, 11, 584, 188, 0, 1263, 749, 0, 5476, 536, 471, 0, 1824, 175, 422, 148, 219, 3120, 4687, 83, 3, 288, 2, 1052, 29, 66, 0, 138, 91, 547, 403, 1391, 0, 1090, 0, 4, 472, 72, 634, 259, 421, 1286, 1488, 341] payload_rows_per_dst=[13962, 11164, 12816, 13700, 8132, 5952, 6050, 9830] dup_rows_per_dst=[2736, 1524, 4296, 3290, 1324, 230, 1002, 2296] combine_rows_per_src=[13962, 11164, 12816, 13700, 8132, 5952, 6050, 9830] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=277274624 dispatch_meta_bytes=335688 combine_payload_bytes=277274624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277610312 ingress_bytes=311877632 link_bytes=311877632 token_bytes=554884936 weight_movement_bytes=34603008 total_nvlink_bytes=589487944 weight_to_token_ratio=0.062361 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=44416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.173570421e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=316604416 dispatch_meta_bytes=332904 combine_payload_bytes=316604416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=316937320 ingress_bytes=351207424 link_bytes=351207424 token_bytes=633541736 weight_movement_bytes=34603008 total_nvlink_bytes=668144744 weight_to_token_ratio=0.054618 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=47104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.321565262e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82038 payload_rows_remote=67694 dup_rows=16266 dup_rows_remote=16228 dedup_ratio=0.165466 router_tokens_per_expert=[1, 0, 19, 0, 23, 68, 44, 2, 0, 1, 105, 1, 0, 51, 887, 0, 3208, 11, 149, 7595, 7606, 0, 12, 2, 0, 132, 545, 0, 7667, 6, 50, 0, 3630, 7, 7, 4, 256, 6599, 7034, 72, 0, 2, 0, 17, 4, 1, 0, 75, 0, 1529, 3, 518, 0, 20, 0, 0, 15, 0, 15, 198, 290, 23, 297, 351] payload_rows_per_dst=[14344, 16026, 16316, 16186, 13234, 198, 3410, 2324] dup_rows_per_dst=[38, 1276, 12898, 614, 656, 0, 730, 54] combine_rows_per_src=[14344, 16026, 16316, 16186, 13234, 198, 3410, 2324] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=322322432 dispatch_meta_bytes=329240 combine_payload_bytes=322322432 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=322651672 ingress_bytes=391528448 link_bytes=391528448 token_bytes=644974104 weight_movement_bytes=69206016 total_nvlink_bytes=714180120 weight_to_token_ratio=0.107300 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=28032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.473290029e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=302456832 dispatch_meta_bytes=332040 combine_payload_bytes=302456832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=302788872 ingress_bytes=337059840 link_bytes=337059840 token_bytes=605245704 weight_movement_bytes=34603008 total_nvlink_bytes=639848712 weight_to_token_ratio=0.057172 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=35712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.268329042e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=362520576 dispatch_meta_bytes=370184 combine_payload_bytes=362520576 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=362890760 ingress_bytes=500932608 link_bytes=500932608 token_bytes=725411336 weight_movement_bytes=138412032 total_nvlink_bytes=863823368 weight_to_token_ratio=0.190805 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.884969076e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=317251584 dispatch_meta_bytes=330752 combine_payload_bytes=317251584 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=317582336 ingress_bytes=386457600 link_bytes=386457600 token_bytes=634833920 weight_movement_bytes=69206016 total_nvlink_bytes=704039936 weight_to_token_ratio=0.109014 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=47232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.454208836e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=277274624 dispatch_meta_bytes=335688 combine_payload_bytes=277274624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277610312 ingress_bytes=311877632 link_bytes=311877632 token_bytes=554884936 weight_movement_bytes=34603008 total_nvlink_bytes=589487944 weight_to_token_ratio=0.062361 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=44416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.173570421e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90606 payload_rows_remote=74512 dup_rows=7698 dup_rows_remote=3596 dedup_ratio=0.078308 router_tokens_per_expert=[0, 0, 0, 0, 0, 47, 143, 0, 0, 6, 88, 0, 0, 70, 918, 0, 3351, 0, 124, 7629, 7641, 0, 0, 0, 0, 128, 511, 0, 7677, 0, 70, 0, 3437, 0, 0, 0, 251, 6265, 7038, 75, 0, 0, 0, 0, 0, 0, 0, 78, 0, 1881, 0, 540, 0, 0, 0, 0, 0, 0, 0, 166, 404, 0, 339, 275] payload_rows_per_dst=[9546, 2160, 15530, 8088, 9778, 15276, 16094, 14134] dup_rows_per_dst=[52, 4, 0, 32, 2606, 138, 4102, 764] combine_rows_per_src=[9546, 2160, 15530, 8088, 9778, 15276, 16094, 14134] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81442 payload_rows_remote=69796 dup_rows=16862 dup_rows_remote=15718 dedup_ratio=0.171529 router_tokens_per_expert=[12, 1, 887, 1464, 121, 350, 0, 0, 21, 27, 111, 69, 520, 1, 728, 607, 23, 303, 617, 1462, 4311, 666, 47, 91, 5440, 349, 11, 3659, 319, 72, 30, 56, 0, 1015, 636, 1469, 50, 0, 0, 4145, 828, 693, 152, 1289, 25, 0, 236, 0, 167, 329, 346, 867, 5542, 235, 2002, 293, 1892, 276, 247, 7, 1411, 1674, 187, 764] payload_rows_per_dst=[14026, 11646, 7648, 9234, 9296, 4596, 13550, 11446] dup_rows_per_dst=[2524, 1144, 1284, 550, 2028, 1850, 6012, 1470] combine_rows_per_src=[14026, 11646, 7648, 9234, 9296, 4596, 13550, 11446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=322322432 dispatch_meta_bytes=329240 combine_payload_bytes=322322432 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=322651672 ingress_bytes=391528448 link_bytes=391528448 token_bytes=644974104 weight_movement_bytes=69206016 total_nvlink_bytes=714180120 weight_to_token_ratio=0.107300 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=28032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.473290029e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=290906112 dispatch_meta_bytes=326312 combine_payload_bytes=290906112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291232424 ingress_bytes=325509120 link_bytes=325509120 token_bytes=582138536 weight_movement_bytes=34603008 total_nvlink_bytes=616741544 weight_to_token_ratio=0.059441 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=37760 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.224864613e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92640 payload_rows_remote=80974 dup_rows=5664 dup_rows_remote=5168 dedup_ratio=0.057617 router_tokens_per_expert=[34, 0, 0, 0, 0, 0, 46, 0, 0, 0, 167, 0, 0, 68, 1012, 0, 3039, 0, 160, 7624, 7646, 0, 0, 0, 0, 38, 681, 0, 7635, 0, 68, 0, 5753, 0, 8, 0, 240, 4443, 6950, 80, 0, 0, 0, 0, 0, 0, 0, 59, 0, 1325, 0, 600, 0, 0, 0, 0, 0, 0, 0, 236, 500, 0, 360, 380] payload_rows_per_dst=[14060, 16264, 2242, 5688, 11666, 15248, 16088, 11384] dup_rows_per_dst=[0, 1522, 0, 42, 496, 118, 3032, 454] combine_rows_per_src=[14060, 16264, 2242, 5688, 11666, 15248, 16088, 11384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=303284224 dispatch_meta_bytes=347584 combine_payload_bytes=303284224 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=303631808 ingress_bytes=372490240 link_bytes=372490240 token_bytes=606916032 weight_movement_bytes=69206016 total_nvlink_bytes=676122048 weight_to_token_ratio=0.114029 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=38656 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.401650785e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=288530432 dispatch_meta_bytes=342464 combine_payload_bytes=288530432 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288872896 ingress_bytes=323133440 link_bytes=323133440 token_bytes=577403328 weight_movement_bytes=34603008 total_nvlink_bytes=612006336 weight_to_token_ratio=0.059929 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=39680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.215925120e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=331669504 dispatch_meta_bytes=344568 combine_payload_bytes=331669504 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=332014072 ingress_bytes=435478528 link_bytes=435478528 token_bytes=663683576 weight_movement_bytes=103809024 total_nvlink_bytes=767492600 weight_to_token_ratio=0.156413 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=27648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.638670642e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86672 payload_rows_remote=71016 dup_rows=11632 dup_rows_remote=11136 dedup_ratio=0.118327 router_tokens_per_expert=[0, 0, 15, 240, 8, 14, 0, 0, 0, 0, 6, 72, 5, 70, 832, 7, 0, 0, 1775, 2498, 7475, 1008, 3, 70, 7799, 14, 0, 7298, 97, 49, 0, 0, 0, 19, 152, 26, 22, 0, 0, 6533, 14, 1010, 2, 22, 0, 0, 77, 0, 1, 5, 7, 426, 5236, 3, 303, 77, 1530, 76, 0, 0, 24, 3901, 4, 327] payload_rows_per_dst=[15656, 15604, 15488, 14866, 3702, 2218, 8052, 11086] dup_rows_per_dst=[496, 1330, 8286, 50, 72, 32, 728, 638] combine_rows_per_src=[15656, 15604, 15488, 14866, 3702, 2218, 8052, 11086] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90890 payload_rows_remote=76048 dup_rows=7414 dup_rows_remote=5436 dedup_ratio=0.075419 router_tokens_per_expert=[0, 0, 0, 181, 0, 0, 0, 0, 0, 0, 34, 100, 0, 100, 787, 0, 0, 0, 2281, 2423, 7558, 868, 0, 59, 7584, 0, 0, 7248, 180, 55, 0, 0, 0, 0, 260, 0, 28, 0, 0, 6710, 0, 1063, 0, 0, 0, 0, 99, 0, 0, 0, 0, 363, 4898, 0, 335, 167, 1658, 99, 0, 0, 0, 3649, 2, 363] payload_rows_per_dst=[15168, 13376, 7216, 5166, 13752, 14842, 10652, 10718] dup_rows_per_dst=[362, 1250, 1732, 150, 244, 1978, 874, 824] combine_rows_per_src=[15168, 13376, 7216, 5166, 13752, 14842, 10652, 10718] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91564 payload_rows_remote=86344 dup_rows=6740 dup_rows_remote=6608 dedup_ratio=0.068563 router_tokens_per_expert=[0, 0, 25, 202, 7, 21, 0, 0, 1, 0, 15, 75, 7, 75, 803, 31, 0, 0, 1724, 2469, 7421, 714, 7, 46, 7583, 18, 0, 7144, 116, 72, 1, 0, 0, 27, 233, 25, 0, 1, 0, 6872, 17, 1257, 1, 33, 1, 0, 75, 0, 9, 1, 5, 286, 5609, 10, 596, 182, 1909, 80, 13, 0, 30, 2893, 26, 384] payload_rows_per_dst=[15298, 15548, 4482, 5220, 14130, 14830, 12144, 9912] dup_rows_per_dst=[378, 1308, 500, 132, 186, 2226, 1252, 758] combine_rows_per_src=[15298, 15548, 4482, 5220, 14130, 14830, 12144, 9912] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81712 payload_rows_remote=68878 dup_rows=16592 dup_rows_remote=13214 dedup_ratio=0.168783 router_tokens_per_expert=[0, 0, 9, 214, 0, 9, 0, 0, 0, 0, 4, 74, 3, 74, 673, 9, 0, 1, 2357, 2192, 7740, 865, 1, 70, 7731, 7, 0, 7305, 89, 64, 0, 0, 0, 9, 126, 12, 55, 0, 0, 6445, 4, 997, 1, 11, 0, 0, 73, 0, 4, 1, 1, 412, 4674, 2, 354, 62, 1568, 75, 2, 0, 13, 4344, 27, 389] payload_rows_per_dst=[446, 1514, 16046, 15902, 13106, 14956, 6908, 12834] dup_rows_per_dst=[18, 160, 6022, 4264, 188, 1826, 736, 3378] combine_rows_per_src=[446, 1514, 16046, 15902, 13106, 14956, 6908, 12834] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91564 payload_rows_remote=86344 dup_rows=6740 dup_rows_remote=6608 dedup_ratio=0.068563 router_tokens_per_expert=[0, 0, 25, 202, 7, 21, 0, 0, 1, 0, 15, 75, 7, 75, 803, 31, 0, 0, 1724, 2469, 7421, 714, 7, 46, 7583, 18, 0, 7144, 116, 72, 1, 0, 0, 27, 233, 25, 0, 1, 0, 6872, 17, 1257, 1, 33, 1, 0, 75, 0, 9, 1, 5, 286, 5609, 10, 596, 182, 1909, 80, 13, 0, 30, 2893, 26, 384] payload_rows_per_dst=[15298, 15548, 4482, 5220, 14130, 14830, 12144, 9912] dup_rows_per_dst=[378, 1308, 500, 132, 186, 2226, 1252, 758] combine_rows_per_src=[15298, 15548, 4482, 5220, 14130, 14830, 12144, 9912] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91628 payload_rows_remote=76276 dup_rows=6676 dup_rows_remote=2968 dedup_ratio=0.067912 router_tokens_per_expert=[0, 2170, 567, 1068, 7467, 7215, 0, 0, 0, 0, 0, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1859, 0, 0, 0, 0, 1000, 5324, 74, 0, 0, 0, 0, 0, 486, 0, 0, 6474, 0, 154, 0, 0, 686, 0, 0, 0, 0, 0, 0, 1312, 0, 0, 0, 0, 84, 7453, 208, 5237, 0, 0, 74, 0, 0, 0] payload_rows_per_dst=[7310, 14934, 3718, 12364, 12492, 14836, 15352, 10622] dup_rows_per_dst=[300, 480, 0, 432, 374, 966, 3708, 416] combine_rows_per_src=[7310, 14934, 3718, 12364, 12492, 14836, 15352, 10622] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85564 payload_rows_remote=78200 dup_rows=12740 dup_rows_remote=12740 dedup_ratio=0.129598 router_tokens_per_expert=[0, 0, 746, 0, 0, 0, 1228, 0, 0, 0, 142, 182, 7239, 0, 0, 5062, 0, 0, 0, 0, 0, 3682, 0, 0, 0, 0, 0, 8100, 0, 0, 7018, 0, 0, 0, 129, 0, 0, 7612, 0, 0, 0, 0, 800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 602, 0, 90, 0, 5365, 0, 0, 495, 660, 0, 0] payload_rows_per_dst=[14936, 13610, 7364, 15074, 15482, 6882, 2236, 9980] dup_rows_per_dst=[3048, 5766, 0, 3022, 0, 592, 280, 32] combine_rows_per_src=[14936, 13610, 7364, 15074, 15482, 6882, 2236, 9980] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=305201152 dispatch_meta_bytes=312432 combine_payload_bytes=305201152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=305513584 ingress_bytes=339804160 link_bytes=339804160 token_bytes=610714736 weight_movement_bytes=34603008 total_nvlink_bytes=645317744 weight_to_token_ratio=0.056660 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=31232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.278655698e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92640 payload_rows_remote=80974 dup_rows=5664 dup_rows_remote=5168 dedup_ratio=0.057617 router_tokens_per_expert=[34, 0, 0, 0, 0, 0, 46, 0, 0, 0, 167, 0, 0, 68, 1012, 0, 3039, 0, 160, 7624, 7646, 0, 0, 0, 0, 38, 681, 0, 7635, 0, 68, 0, 5753, 0, 8, 0, 240, 4443, 6950, 80, 0, 0, 0, 0, 0, 0, 0, 59, 0, 1325, 0, 600, 0, 0, 0, 0, 0, 0, 0, 236, 500, 0, 360, 380] payload_rows_per_dst=[14060, 16264, 2242, 5688, 11666, 15248, 16088, 11384] dup_rows_per_dst=[0, 1522, 0, 42, 496, 118, 3032, 454] combine_rows_per_src=[14060, 16264, 2242, 5688, 11666, 15248, 16088, 11384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=285884416 dispatch_meta_bytes=342056 combine_payload_bytes=285884416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286226472 ingress_bytes=320487424 link_bytes=320487424 token_bytes=572110888 weight_movement_bytes=34603008 total_nvlink_bytes=606713896 weight_to_token_ratio=0.060483 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=40448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.205968375e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90164 payload_rows_remote=85866 dup_rows=8140 dup_rows_remote=7910 dedup_ratio=0.082804 router_tokens_per_expert=[0, 0, 0, 180, 0, 0, 0, 0, 0, 0, 5, 90, 0, 88, 764, 0, 0, 0, 1599, 2661, 7465, 657, 0, 8, 7688, 0, 0, 7223, 138, 53, 0, 0, 0, 0, 243, 0, 0, 0, 0, 6921, 0, 1352, 0, 0, 0, 0, 89, 0, 0, 0, 0, 239, 5476, 0, 524, 189, 2182, 89, 0, 0, 0, 2764, 2, 463] payload_rows_per_dst=[15380, 15636, 4298, 14634, 14150, 4136, 11788, 10142] dup_rows_per_dst=[356, 1188, 230, 4050, 178, 212, 1068, 858] combine_rows_per_src=[15380, 15636, 4298, 14634, 14150, 4136, 11788, 10142] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=311492608 dispatch_meta_bytes=325936 combine_payload_bytes=311492608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=311818544 ingress_bytes=346095616 link_bytes=346095616 token_bytes=623311152 weight_movement_bytes=34603008 total_nvlink_bytes=657914160 weight_to_token_ratio=0.055515 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=37120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.302329940e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86672 payload_rows_remote=71016 dup_rows=11632 dup_rows_remote=11136 dedup_ratio=0.118327 router_tokens_per_expert=[0, 0, 15, 240, 8, 14, 0, 0, 0, 0, 6, 72, 5, 70, 832, 7, 0, 0, 1775, 2498, 7475, 1008, 3, 70, 7799, 14, 0, 7298, 97, 49, 0, 0, 0, 19, 152, 26, 22, 0, 0, 6533, 14, 1010, 2, 22, 0, 0, 77, 0, 1, 5, 7, 426, 5236, 3, 303, 77, 1530, 76, 0, 0, 24, 3901, 4, 327] payload_rows_per_dst=[15656, 15604, 15488, 14866, 3702, 2218, 8052, 11086] dup_rows_per_dst=[496, 1330, 8286, 50, 72, 32, 728, 638] combine_rows_per_src=[15656, 15604, 15488, 14866, 3702, 2218, 8052, 11086] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=353665024 dispatch_meta_bytes=371808 combine_payload_bytes=353665024 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=354036832 ingress_bytes=457474048 link_bytes=457474048 token_bytes=707701856 weight_movement_bytes=103809024 total_nvlink_bytes=811510880 weight_to_token_ratio=0.146685 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.721438013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=282124288 dispatch_meta_bytes=328368 combine_payload_bytes=282124288 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282452656 ingress_bytes=316727296 link_bytes=316727296 token_bytes=564576944 weight_movement_bytes=34603008 total_nvlink_bytes=599179952 weight_to_token_ratio=0.061290 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=26240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.191819316e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=331669504 dispatch_meta_bytes=344568 combine_payload_bytes=331669504 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=332014072 ingress_bytes=435478528 link_bytes=435478528 token_bytes=663683576 weight_movement_bytes=103809024 total_nvlink_bytes=767492600 weight_to_token_ratio=0.156413 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=27648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.638670642e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=351707136 dispatch_meta_bytes=375104 combine_payload_bytes=351707136 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=352082240 ingress_bytes=455516160 link_bytes=455516160 token_bytes=703789376 weight_movement_bytes=103809024 total_nvlink_bytes=807598400 weight_to_token_ratio=0.147500 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.714070638e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=290881536 dispatch_meta_bytes=328608 combine_payload_bytes=290881536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291210144 ingress_bytes=325484544 link_bytes=325484544 token_bytes=582091680 weight_movement_bytes=34603008 total_nvlink_bytes=616694688 weight_to_token_ratio=0.059446 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=44800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.224772135e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84532 payload_rows_remote=74044 dup_rows=13772 dup_rows_remote=12852 dedup_ratio=0.140096 router_tokens_per_expert=[0, 0, 0, 194, 0, 0, 0, 0, 0, 0, 3, 75, 0, 75, 742, 0, 0, 0, 1859, 2579, 7600, 953, 0, 72, 7918, 0, 0, 7342, 92, 70, 0, 0, 0, 0, 154, 0, 58, 0, 0, 6361, 0, 972, 0, 0, 0, 0, 77, 0, 0, 0, 0, 361, 4893, 0, 365, 85, 1446, 74, 0, 0, 0, 4350, 1, 381] payload_rows_per_dst=[2554, 1634, 15774, 14864, 12972, 14968, 10488, 11278] dup_rows_per_dst=[50, 156, 5194, 4238, 174, 1814, 920, 1226] combine_rows_per_src=[2554, 1634, 15774, 14864, 12972, 14968, 10488, 11278] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74912 payload_rows_remote=71130 dup_rows=23392 dup_rows_remote=22718 dedup_ratio=0.237956 router_tokens_per_expert=[0, 1221, 465, 907, 4963, 5066, 117, 0, 78, 57, 114, 221, 187, 295, 505, 594, 1439, 319, 33, 41, 1266, 1733, 1, 0, 10, 0, 1002, 3891, 156, 2, 610, 56, 50, 52, 674, 27, 7, 4262, 187, 85, 340, 208, 716, 29, 844, 177, 147, 794, 500, 1233, 491, 0, 34, 1587, 76, 4783, 255, 4173, 114, 1208, 69, 80, 139, 462] payload_rows_per_dst=[12638, 3782, 14582, 10866, 9866, 5224, 6354, 11600] dup_rows_per_dst=[12486, 674, 4648, 588, 822, 1286, 1488, 1400] combine_rows_per_src=[12638, 3782, 14582, 10866, 9866, 5224, 6354, 11600] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81442 payload_rows_remote=69796 dup_rows=16862 dup_rows_remote=15718 dedup_ratio=0.171529 router_tokens_per_expert=[12, 1, 887, 1464, 121, 350, 0, 0, 21, 27, 111, 69, 520, 1, 728, 607, 23, 303, 617, 1462, 4311, 666, 47, 91, 5440, 349, 11, 3659, 319, 72, 30, 56, 0, 1015, 636, 1469, 50, 0, 0, 4145, 828, 693, 152, 1289, 25, 0, 236, 0, 167, 329, 346, 867, 5542, 235, 2002, 293, 1892, 276, 247, 7, 1411, 1674, 187, 764] payload_rows_per_dst=[14026, 11646, 7648, 9234, 9296, 4596, 13550, 11446] dup_rows_per_dst=[2524, 1144, 1284, 550, 2028, 1850, 6012, 1470] combine_rows_per_src=[14026, 11646, 7648, 9234, 9296, 4596, 13550, 11446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=290881536 dispatch_meta_bytes=328608 combine_payload_bytes=290881536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291210144 ingress_bytes=325484544 link_bytes=325484544 token_bytes=582091680 weight_movement_bytes=34603008 total_nvlink_bytes=616694688 weight_to_token_ratio=0.059446 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=44800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.224772135e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=311492608 dispatch_meta_bytes=325936 combine_payload_bytes=311492608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=311818544 ingress_bytes=346095616 link_bytes=346095616 token_bytes=623311152 weight_movement_bytes=34603008 total_nvlink_bytes=657914160 weight_to_token_ratio=0.055515 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=37120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.302329940e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=282124288 dispatch_meta_bytes=328368 combine_payload_bytes=282124288 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282452656 ingress_bytes=316727296 link_bytes=316727296 token_bytes=564576944 weight_movement_bytes=34603008 total_nvlink_bytes=599179952 weight_to_token_ratio=0.061290 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=26240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.191819316e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91702 payload_rows_remote=77736 dup_rows=6602 dup_rows_remote=6442 dedup_ratio=0.067159 router_tokens_per_expert=[0, 0, 0, 209, 0, 0, 0, 0, 0, 0, 0, 80, 0, 80, 840, 0, 0, 0, 1340, 2343, 7405, 712, 0, 0, 7587, 0, 0, 7228, 81, 68, 0, 0, 0, 0, 161, 0, 0, 0, 0, 6902, 0, 1602, 0, 0, 0, 0, 80, 0, 0, 0, 0, 215, 5661, 0, 542, 148, 2642, 80, 0, 0, 0, 2659, 4, 483] payload_rows_per_dst=[15174, 15462, 4058, 4848, 13966, 15446, 12004, 10744] dup_rows_per_dst=[418, 1348, 46, 136, 160, 2374, 1128, 992] combine_rows_per_src=[15174, 15462, 4058, 4848, 13966, 15446, 12004, 10744] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=353665024 dispatch_meta_bytes=371808 combine_payload_bytes=353665024 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=354036832 ingress_bytes=457474048 link_bytes=457474048 token_bytes=707701856 weight_movement_bytes=103809024 total_nvlink_bytes=811510880 weight_to_token_ratio=0.146685 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.721438013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=312426496 dispatch_meta_bytes=316976 combine_payload_bytes=312426496 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=312743472 ingress_bytes=381632512 link_bytes=381632512 token_bytes=625169968 weight_movement_bytes=69206016 total_nvlink_bytes=694375984 weight_to_token_ratio=0.110700 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=28800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.436052419e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=320307200 dispatch_meta_bytes=363760 combine_payload_bytes=320307200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=320670960 ingress_bytes=354910208 link_bytes=354910208 token_bytes=640978160 weight_movement_bytes=34603008 total_nvlink_bytes=675581168 weight_to_token_ratio=0.053985 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=36480 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.335498540e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=285884416 dispatch_meta_bytes=342056 combine_payload_bytes=285884416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286226472 ingress_bytes=320487424 link_bytes=320487424 token_bytes=572110888 weight_movement_bytes=34603008 total_nvlink_bytes=606713896 weight_to_token_ratio=0.060483 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=40448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.205968375e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=318406656 dispatch_meta_bytes=336712 combine_payload_bytes=318406656 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=318743368 ingress_bytes=387612672 link_bytes=387612672 token_bytes=637150024 weight_movement_bytes=69206016 total_nvlink_bytes=706356040 weight_to_token_ratio=0.108618 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=42752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.458555279e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74008 payload_rows_remote=57680 dup_rows=24296 dup_rows_remote=4520 dedup_ratio=0.247152 router_tokens_per_expert=[0, 1788, 528, 1142, 7466, 7125, 3, 0, 1, 0, 3, 227, 3, 4, 8, 8, 23, 4, 28, 0, 15, 1855, 0, 0, 0, 0, 963, 5219, 72, 0, 16, 0, 0, 1, 538, 0, 0, 6458, 8, 111, 4, 0, 653, 0, 14, 2, 3, 15, 8, 1303, 16, 0, 0, 27, 64, 7564, 190, 5567, 0, 23, 72, 1, 0, 9] payload_rows_per_dst=[16328, 500, 15578, 12302, 13804, 1370, 2798, 11328] dup_rows_per_dst=[19776, 8, 3400, 238, 428, 12, 38, 396] combine_rows_per_src=[16328, 500, 15578, 12302, 13804, 1370, 2798, 11328] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93130 payload_rows_remote=78582 dup_rows=5174 dup_rows_remote=4232 dedup_ratio=0.052633 router_tokens_per_expert=[0, 1813, 657, 1163, 7626, 6984, 0, 0, 0, 0, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2160, 0, 0, 0, 0, 997, 5081, 99, 0, 0, 0, 0, 0, 437, 0, 0, 6356, 0, 122, 0, 0, 761, 0, 0, 0, 0, 0, 0, 1205, 1, 0, 0, 0, 67, 7376, 203, 5738, 0, 0, 98, 0, 0, 0] payload_rows_per_dst=[6850, 15252, 4320, 11886, 13516, 14548, 15084, 11674] dup_rows_per_dst=[416, 416, 0, 468, 314, 942, 2214, 404] combine_rows_per_src=[6850, 15252, 4320, 11886, 13516, 14548, 15084, 11674] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91064 payload_rows_remote=77504 dup_rows=7240 dup_rows_remote=6618 dedup_ratio=0.073649 router_tokens_per_expert=[0, 1200, 509, 1097, 7796, 6384, 3, 0, 1, 1, 9, 118, 6, 2, 19, 10, 36, 1, 11, 0, 20, 2777, 2, 0, 0, 0, 987, 6002, 76, 0, 26, 0, 1, 1, 442, 0, 1, 5862, 8, 149, 1, 0, 673, 0, 21, 2, 22, 31, 3, 1133, 60, 0, 1, 36, 88, 7061, 152, 6188, 5, 28, 76, 3, 3, 8] payload_rows_per_dst=[5196, 15668, 15510, 13560, 12432, 13532, 2568, 12598] dup_rows_per_dst=[422, 256, 4306, 622, 496, 736, 74, 328] combine_rows_per_src=[5196, 15668, 15510, 13560, 12432, 13532, 2568, 12598] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80148 payload_rows_remote=68520 dup_rows=18156 dup_rows_remote=17052 dedup_ratio=0.184692 router_tokens_per_expert=[0, 2612, 530, 1176, 7570, 7125, 1, 0, 0, 0, 4, 211, 5, 1, 8, 10, 11, 0, 7, 0, 6, 1464, 0, 0, 0, 0, 994, 4815, 73, 0, 6, 0, 0, 0, 393, 0, 0, 6483, 17, 149, 0, 0, 682, 0, 8, 0, 2, 12, 1, 1209, 21, 0, 0, 12, 85, 7606, 207, 5550, 0, 11, 73, 0, 0, 2] payload_rows_per_dst=[5548, 14160, 2968, 6768, 8302, 14622, 16152, 11628] dup_rows_per_dst=[348, 412, 8, 318, 248, 1036, 14682, 1104] combine_rows_per_src=[5548, 14160, 2968, 6768, 8302, 14622, 16152, 11628] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=303284224 dispatch_meta_bytes=347584 combine_payload_bytes=303284224 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=303631808 ingress_bytes=372490240 link_bytes=372490240 token_bytes=606916032 weight_movement_bytes=69206016 total_nvlink_bytes=676122048 weight_to_token_ratio=0.114029 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=38656 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.401650785e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91064 payload_rows_remote=77504 dup_rows=7240 dup_rows_remote=6618 dedup_ratio=0.073649 router_tokens_per_expert=[0, 1200, 509, 1097, 7796, 6384, 3, 0, 1, 1, 9, 118, 6, 2, 19, 10, 36, 1, 11, 0, 20, 2777, 2, 0, 0, 0, 987, 6002, 76, 0, 26, 0, 1, 1, 442, 0, 1, 5862, 8, 149, 1, 0, 673, 0, 21, 2, 22, 31, 3, 1133, 60, 0, 1, 36, 88, 7061, 152, 6188, 5, 28, 76, 3, 3, 8] payload_rows_per_dst=[5196, 15668, 15510, 13560, 12432, 13532, 2568, 12598] dup_rows_per_dst=[422, 256, 4306, 622, 496, 736, 74, 328] combine_rows_per_src=[5196, 15668, 15510, 13560, 12432, 13532, 2568, 12598] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77062 payload_rows_remote=62348 dup_rows=21242 dup_rows_remote=11066 dedup_ratio=0.216085 router_tokens_per_expert=[29, 0, 0, 0, 0, 0, 0, 3092, 522, 0, 151, 0, 0, 0, 10, 3647, 0, 0, 0, 6415, 200, 1354, 0, 902, 0, 0, 0, 0, 0, 0, 0, 0, 3418, 0, 0, 0, 0, 0, 0, 7000, 0, 0, 0, 7939, 0, 0, 186, 0, 0, 0, 0, 1717, 0, 8037, 0, 0, 0, 0, 0, 3861, 158, 439, 0, 75] payload_rows_per_dst=[6242, 7742, 14468, 8558, 11382, 2276, 14714, 11680] dup_rows_per_dst=[0, 918, 3274, 0, 3616, 34, 10176, 3224] combine_rows_per_src=[6242, 7742, 14468, 8558, 11382, 2276, 14714, 11680] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91702 payload_rows_remote=77736 dup_rows=6602 dup_rows_remote=6442 dedup_ratio=0.067159 router_tokens_per_expert=[0, 0, 0, 209, 0, 0, 0, 0, 0, 0, 0, 80, 0, 80, 840, 0, 0, 0, 1340, 2343, 7405, 712, 0, 0, 7587, 0, 0, 7228, 81, 68, 0, 0, 0, 0, 161, 0, 0, 0, 0, 6902, 0, 1602, 0, 0, 0, 0, 80, 0, 0, 0, 0, 215, 5661, 0, 542, 148, 2642, 80, 0, 0, 0, 2659, 4, 483] payload_rows_per_dst=[15174, 15462, 4058, 4848, 13966, 15446, 12004, 10744] dup_rows_per_dst=[418, 1348, 46, 136, 160, 2374, 1128, 992] combine_rows_per_src=[15174, 15462, 4058, 4848, 13966, 15446, 12004, 10744] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94264 payload_rows_remote=88506 dup_rows=4040 dup_rows_remote=4040 dedup_ratio=0.041097 router_tokens_per_expert=[15, 0, 0, 0, 0, 0, 73, 0, 0, 0, 191, 0, 0, 53, 1052, 0, 2700, 0, 179, 7523, 7544, 0, 0, 0, 0, 48, 721, 0, 7674, 0, 53, 0, 5705, 0, 14, 0, 248, 4928, 7072, 90, 0, 0, 0, 0, 0, 0, 0, 72, 0, 1257, 0, 611, 0, 0, 0, 0, 0, 0, 0, 196, 468, 0, 322, 343] payload_rows_per_dst=[14320, 16238, 5758, 16154, 11590, 15046, 3050, 12108] dup_rows_per_dst=[0, 1442, 0, 838, 524, 144, 686, 406] combine_rows_per_src=[14320, 16238, 5758, 16154, 11590, 15046, 3050, 12108] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90532 payload_rows_remote=74888 dup_rows=7772 dup_rows_remote=2540 dedup_ratio=0.079061 router_tokens_per_expert=[0, 994, 531, 1165, 7902, 6745, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3441, 0, 0, 0, 0, 846, 6065, 89, 0, 0, 0, 0, 0, 509, 0, 0, 5687, 16, 127, 0, 0, 550, 0, 0, 0, 20, 0, 0, 1163, 31, 0, 0, 0, 74, 6997, 161, 5965, 0, 0, 69, 0, 0, 0] payload_rows_per_dst=[4876, 15804, 15644, 13348, 12272, 14006, 2474, 12108] dup_rows_per_dst=[540, 10, 5232, 652, 406, 588, 62, 282] combine_rows_per_src=[4876, 15804, 15644, 13348, 12272, 14006, 2474, 12108] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=291348480 dispatch_meta_bytes=375392 combine_payload_bytes=291348480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291723872 ingress_bytes=325951488 link_bytes=325951488 token_bytes=583072352 weight_movement_bytes=34603008 total_nvlink_bytes=617675360 weight_to_token_ratio=0.059346 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=46080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.226529208e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74912 payload_rows_remote=71130 dup_rows=23392 dup_rows_remote=22718 dedup_ratio=0.237956 router_tokens_per_expert=[0, 1221, 465, 907, 4963, 5066, 117, 0, 78, 57, 114, 221, 187, 295, 505, 594, 1439, 319, 33, 41, 1266, 1733, 1, 0, 10, 0, 1002, 3891, 156, 2, 610, 56, 50, 52, 674, 27, 7, 4262, 187, 85, 340, 208, 716, 29, 844, 177, 147, 794, 500, 1233, 491, 0, 34, 1587, 76, 4783, 255, 4173, 114, 1208, 69, 80, 139, 462] payload_rows_per_dst=[12638, 3782, 14582, 10866, 9866, 5224, 6354, 11600] dup_rows_per_dst=[12486, 674, 4648, 588, 822, 1286, 1488, 1400] combine_rows_per_src=[12638, 3782, 14582, 10866, 9866, 5224, 6354, 11600] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=321871872 dispatch_meta_bytes=331256 combine_payload_bytes=321871872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=322203128 ingress_bytes=356474880 link_bytes=356474880 token_bytes=644075000 weight_movement_bytes=34603008 total_nvlink_bytes=678678008 weight_to_token_ratio=0.053725 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=41216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.341386275e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74008 payload_rows_remote=57680 dup_rows=24296 dup_rows_remote=4520 dedup_ratio=0.247152 router_tokens_per_expert=[0, 1788, 528, 1142, 7466, 7125, 3, 0, 1, 0, 3, 227, 3, 4, 8, 8, 23, 4, 28, 0, 15, 1855, 0, 0, 0, 0, 963, 5219, 72, 0, 16, 0, 0, 1, 538, 0, 0, 6458, 8, 111, 4, 0, 653, 0, 14, 2, 3, 15, 8, 1303, 16, 0, 0, 27, 64, 7564, 190, 5567, 0, 23, 72, 1, 0, 9] payload_rows_per_dst=[16328, 500, 15578, 12302, 13804, 1370, 2798, 11328] dup_rows_per_dst=[19776, 8, 3400, 238, 428, 12, 38, 396] combine_rows_per_src=[16328, 500, 15578, 12302, 13804, 1370, 2798, 11328] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=317456384 dispatch_meta_bytes=336488 combine_payload_bytes=317456384 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=317792872 ingress_bytes=386662400 link_bytes=386662400 token_bytes=635249256 weight_movement_bytes=69206016 total_nvlink_bytes=704455272 weight_to_token_ratio=0.108943 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=38784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.454979482e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=280657920 dispatch_meta_bytes=342288 combine_payload_bytes=280657920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281000208 ingress_bytes=315260928 link_bytes=315260928 token_bytes=561658128 weight_movement_bytes=34603008 total_nvlink_bytes=596261136 weight_to_token_ratio=0.061609 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=44544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.186301491e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=318406656 dispatch_meta_bytes=336712 combine_payload_bytes=318406656 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=318743368 ingress_bytes=387612672 link_bytes=387612672 token_bytes=637150024 weight_movement_bytes=69206016 total_nvlink_bytes=706356040 weight_to_token_ratio=0.108618 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=42752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.458555279e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=306741248 dispatch_meta_bytes=309712 combine_payload_bytes=306741248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307050960 ingress_bytes=341344256 link_bytes=341344256 token_bytes=613792208 weight_movement_bytes=34603008 total_nvlink_bytes=648395216 weight_to_token_ratio=0.056376 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=27136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.284450955e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93196 payload_rows_remote=81158 dup_rows=5108 dup_rows_remote=4614 dedup_ratio=0.051961 router_tokens_per_expert=[0, 1031, 500, 1104, 7974, 6776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3517, 0, 0, 0, 0, 818, 6048, 80, 0, 0, 0, 0, 0, 623, 0, 0, 5479, 0, 164, 0, 0, 512, 0, 0, 0, 12, 0, 0, 1209, 12, 0, 0, 0, 97, 6985, 139, 6000, 0, 0, 69, 0, 0, 0] payload_rows_per_dst=[4954, 15948, 7604, 13308, 12038, 13992, 13192, 12160] dup_rows_per_dst=[316, 0, 786, 584, 494, 608, 2064, 256] combine_rows_per_src=[4954, 15948, 7604, 13308, 12038, 13992, 13192, 12160] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=236257280 dispatch_meta_bytes=248800 combine_payload_bytes=236257280 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=236506080 ingress_bytes=374669312 link_bytes=374669312 token_bytes=472763360 weight_movement_bytes=138412032 total_nvlink_bytes=611175392 weight_to_token_ratio=0.292772 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.035000 gemm_active_groups=7 gemm_peak_group_rows=12288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.409850458e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91628 payload_rows_remote=76276 dup_rows=6676 dup_rows_remote=2968 dedup_ratio=0.067912 router_tokens_per_expert=[0, 2170, 567, 1068, 7467, 7215, 0, 0, 0, 0, 0, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1859, 0, 0, 0, 0, 1000, 5324, 74, 0, 0, 0, 0, 0, 486, 0, 0, 6474, 0, 154, 0, 0, 686, 0, 0, 0, 0, 0, 0, 1312, 0, 0, 0, 0, 84, 7453, 208, 5237, 0, 0, 74, 0, 0, 0] payload_rows_per_dst=[7310, 14934, 3718, 12364, 12492, 14836, 15352, 10622] dup_rows_per_dst=[300, 480, 0, 432, 374, 966, 3708, 416] combine_rows_per_src=[7310, 14934, 3718, 12364, 12492, 14836, 15352, 10622] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81900 payload_rows_remote=67032 dup_rows=16404 dup_rows_remote=11382 dedup_ratio=0.166870 router_tokens_per_expert=[2114, 1259, 36, 87, 43, 54, 268, 956, 388, 1626, 2510, 89, 298, 0, 257, 1550, 34, 447, 90, 3227, 321, 818, 69, 804, 4, 44, 0, 55, 33, 109, 0, 107, 4615, 1580, 234, 0, 27, 0, 1, 3162, 0, 1580, 37, 3821, 903, 1384, 212, 36, 40, 76, 11, 1231, 315, 5203, 0, 815, 54, 280, 76, 4843, 302, 508, 40, 69] payload_rows_per_dst=[12840, 14868, 4848, 11034, 11698, 11396, 4288, 10928] dup_rows_per_dst=[3118, 5022, 318, 76, 1216, 4550, 688, 1416] combine_rows_per_src=[12840, 14868, 4848, 11034, 11698, 11396, 4288, 10928] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=255377408 dispatch_meta_bytes=293656 combine_payload_bytes=255377408 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=255671064 ingress_bytes=324583424 link_bytes=324583424 token_bytes=511048472 weight_movement_bytes=69206016 total_nvlink_bytes=580254488 weight_to_token_ratio=0.135420 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=21120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.221381293e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=280657920 dispatch_meta_bytes=342288 combine_payload_bytes=280657920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281000208 ingress_bytes=315260928 link_bytes=315260928 token_bytes=561658128 weight_movement_bytes=34603008 total_nvlink_bytes=596261136 weight_to_token_ratio=0.061609 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=44544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.186301491e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=317456384 dispatch_meta_bytes=336488 combine_payload_bytes=317456384 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=317792872 ingress_bytes=386662400 link_bytes=386662400 token_bytes=635249256 weight_movement_bytes=69206016 total_nvlink_bytes=704455272 weight_to_token_ratio=0.108943 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=38784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.454979482e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=291348480 dispatch_meta_bytes=375392 combine_payload_bytes=291348480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291723872 ingress_bytes=325951488 link_bytes=325951488 token_bytes=583072352 weight_movement_bytes=34603008 total_nvlink_bytes=617675360 weight_to_token_ratio=0.059346 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=46080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.226529208e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=362520576 dispatch_meta_bytes=370184 combine_payload_bytes=362520576 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=362890760 ingress_bytes=500932608 link_bytes=500932608 token_bytes=725411336 weight_movement_bytes=138412032 total_nvlink_bytes=863823368 weight_to_token_ratio=0.190805 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.884969076e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=332423168 dispatch_meta_bytes=343088 combine_payload_bytes=332423168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=332766256 ingress_bytes=401629184 link_bytes=401629184 token_bytes=665189424 weight_movement_bytes=69206016 total_nvlink_bytes=734395440 weight_to_token_ratio=0.104040 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=39936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.511298286e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87922 payload_rows_remote=71974 dup_rows=10382 dup_rows_remote=10106 dedup_ratio=0.105611 router_tokens_per_expert=[65, 0, 0, 0, 0, 0, 0, 2712, 543, 0, 202, 0, 0, 0, 2, 3807, 0, 0, 0, 6177, 188, 1294, 0, 863, 0, 0, 0, 0, 0, 0, 0, 0, 3251, 0, 0, 0, 0, 0, 0, 7017, 0, 0, 0, 7947, 0, 0, 165, 0, 0, 0, 0, 2777, 0, 7924, 0, 0, 0, 0, 0, 3490, 166, 462, 0, 100] payload_rows_per_dst=[5490, 8100, 14202, 15848, 14638, 15948, 5554, 8142] dup_rows_per_dst=[64, 1008, 2842, 0, 5898, 276, 0, 294] combine_rows_per_src=[5490, 8100, 14202, 15848, 14638, 15948, 5554, 8142] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=312426496 dispatch_meta_bytes=316976 combine_payload_bytes=312426496 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=312743472 ingress_bytes=381632512 link_bytes=381632512 token_bytes=625169968 weight_movement_bytes=69206016 total_nvlink_bytes=694375984 weight_to_token_ratio=0.110700 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=28800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.436052419e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86888 payload_rows_remote=71576 dup_rows=11416 dup_rows_remote=6456 dedup_ratio=0.116130 router_tokens_per_expert=[170, 17, 0, 0, 0, 0, 1, 3057, 516, 27, 220, 2, 2, 0, 4, 3569, 0, 7, 3, 6384, 182, 1183, 1, 925, 0, 1, 0, 0, 0, 2, 0, 2, 3445, 22, 2, 0, 0, 0, 0, 6891, 0, 27, 0, 7814, 13, 25, 158, 0, 1, 0, 0, 2237, 8, 7971, 0, 16, 0, 1, 1, 3597, 136, 440, 0, 72] payload_rows_per_dst=[15312, 15482, 4594, 15950, 6924, 15828, 4508, 8290] dup_rows_per_dst=[4960, 5966, 8, 2, 14, 246, 16, 204] combine_rows_per_src=[15312, 15482, 4594, 15950, 6924, 15828, 4508, 8290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86980 payload_rows_remote=71476 dup_rows=11324 dup_rows_remote=10402 dedup_ratio=0.115194 router_tokens_per_expert=[0, 0, 4144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7794, 0, 659, 0, 0, 365, 70, 0, 0, 0, 71, 0, 459, 0, 0, 0, 0, 17, 0, 6603, 7942, 0, 142, 0, 0, 0, 143, 0, 261, 719, 0, 0, 0, 7450, 7697, 0, 0, 437, 0, 0, 251, 0, 75, 0, 0, 0, 3853, 0, 0, 0, 0, 0] payload_rows_per_dst=[8288, 1318, 990, 13346, 15884, 15560, 15504, 16090] dup_rows_per_dst=[0, 0, 148, 812, 570, 1668, 922, 7204] combine_rows_per_src=[8288, 1318, 990, 13346, 15884, 15560, 15504, 16090] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69158 payload_rows_remote=53454 dup_rows=29146 dup_rows_remote=23406 dedup_ratio=0.296488 router_tokens_per_expert=[184, 9, 0, 0, 0, 0, 0, 2988, 545, 13, 244, 1, 0, 0, 49, 3037, 0, 6, 0, 6308, 184, 1774, 0, 869, 0, 0, 0, 0, 0, 0, 0, 4, 4321, 6, 0, 0, 0, 0, 0, 6951, 0, 12, 0, 7995, 7, 12, 207, 0, 0, 0, 0, 1595, 5, 8047, 0, 8, 0, 2, 0, 3118, 158, 419, 0, 74] payload_rows_per_dst=[6256, 6800, 12826, 8, 8724, 2530, 16310, 15704] dup_rows_per_dst=[106, 978, 3164, 0, 102, 66, 18990, 5740] combine_rows_per_src=[6256, 6800, 12826, 8, 8724, 2530, 16310, 15704] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89006 payload_rows_remote=73476 dup_rows=9298 dup_rows_remote=9298 dedup_ratio=0.094584 router_tokens_per_expert=[802, 30, 0, 2, 0, 0, 2, 2687, 535, 37, 437, 2, 0, 0, 28, 3155, 0, 11, 1, 5071, 256, 522, 7, 1331, 0, 0, 0, 0, 0, 2, 0, 10, 3028, 21, 3, 0, 0, 0, 0, 6981, 0, 34, 0, 7259, 18, 32, 194, 1, 0, 0, 0, 3599, 17, 7753, 0, 25, 0, 6, 2, 4614, 165, 395, 1, 76] payload_rows_per_dst=[11652, 7420, 13078, 15530, 9546, 14806, 7246, 9728] dup_rows_per_dst=[3656, 968, 1320, 0, 2258, 270, 36, 790] combine_rows_per_src=[11652, 7420, 13078, 15530, 9546, 14806, 7246, 9728] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93196 payload_rows_remote=81158 dup_rows=5108 dup_rows_remote=4614 dedup_ratio=0.051961 router_tokens_per_expert=[0, 1031, 500, 1104, 7974, 6776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3517, 0, 0, 0, 0, 818, 6048, 80, 0, 0, 0, 0, 0, 623, 0, 0, 5479, 0, 164, 0, 0, 512, 0, 0, 0, 12, 0, 0, 1209, 12, 0, 0, 0, 97, 6985, 139, 6000, 0, 0, 69, 0, 0, 0] payload_rows_per_dst=[4954, 15948, 7604, 13308, 12038, 13992, 13192, 12160] dup_rows_per_dst=[316, 0, 786, 584, 494, 608, 2064, 256] combine_rows_per_src=[4954, 15948, 7604, 13308, 12038, 13992, 13192, 12160] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89006 payload_rows_remote=73476 dup_rows=9298 dup_rows_remote=9298 dedup_ratio=0.094584 router_tokens_per_expert=[802, 30, 0, 2, 0, 0, 2, 2687, 535, 37, 437, 2, 0, 0, 28, 3155, 0, 11, 1, 5071, 256, 522, 7, 1331, 0, 0, 0, 0, 0, 2, 0, 10, 3028, 21, 3, 0, 0, 0, 0, 6981, 0, 34, 0, 7259, 18, 32, 194, 1, 0, 0, 0, 3599, 17, 7753, 0, 25, 0, 6, 2, 4614, 165, 395, 1, 76] payload_rows_per_dst=[11652, 7420, 13078, 15530, 9546, 14806, 7246, 9728] dup_rows_per_dst=[3656, 968, 1320, 0, 2258, 270, 36, 790] combine_rows_per_src=[11652, 7420, 13078, 15530, 9546, 14806, 7246, 9728] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81900 payload_rows_remote=67032 dup_rows=16404 dup_rows_remote=11382 dedup_ratio=0.166870 router_tokens_per_expert=[2114, 1259, 36, 87, 43, 54, 268, 956, 388, 1626, 2510, 89, 298, 0, 257, 1550, 34, 447, 90, 3227, 321, 818, 69, 804, 4, 44, 0, 55, 33, 109, 0, 107, 4615, 1580, 234, 0, 27, 0, 1, 3162, 0, 1580, 37, 3821, 903, 1384, 212, 36, 40, 76, 11, 1231, 315, 5203, 0, 815, 54, 280, 76, 4843, 302, 508, 40, 69] payload_rows_per_dst=[12840, 14868, 4848, 11034, 11698, 11396, 4288, 10928] dup_rows_per_dst=[3118, 5022, 318, 76, 1216, 4550, 688, 1416] combine_rows_per_src=[12840, 14868, 4848, 11034, 11698, 11396, 4288, 10928] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90164 payload_rows_remote=85866 dup_rows=8140 dup_rows_remote=7910 dedup_ratio=0.082804 router_tokens_per_expert=[0, 0, 0, 180, 0, 0, 0, 0, 0, 0, 5, 90, 0, 88, 764, 0, 0, 0, 1599, 2661, 7465, 657, 0, 8, 7688, 0, 0, 7223, 138, 53, 0, 0, 0, 0, 243, 0, 0, 0, 0, 6921, 0, 1352, 0, 0, 0, 0, 89, 0, 0, 0, 0, 239, 5476, 0, 524, 189, 2182, 89, 0, 0, 0, 2764, 2, 463] payload_rows_per_dst=[15380, 15636, 4298, 14634, 14150, 4136, 11788, 10142] dup_rows_per_dst=[356, 1188, 230, 4050, 178, 212, 1068, 858] combine_rows_per_src=[15380, 15636, 4298, 14634, 14150, 4136, 11788, 10142] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=294805504 dispatch_meta_bytes=328320 combine_payload_bytes=294805504 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=295133824 ingress_bytes=364011520 link_bytes=364011520 token_bytes=589939328 weight_movement_bytes=69206016 total_nvlink_bytes=659145344 weight_to_token_ratio=0.117310 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=42496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.369746044e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90170 payload_rows_remote=75492 dup_rows=8134 dup_rows_remote=4502 dedup_ratio=0.082743 router_tokens_per_expert=[698, 0, 0, 0, 0, 0, 0, 2885, 587, 0, 438, 0, 0, 0, 52, 3222, 0, 0, 0, 5153, 289, 426, 0, 1423, 0, 0, 0, 0, 0, 0, 0, 0, 2107, 0, 0, 0, 0, 0, 0, 7048, 0, 0, 0, 7312, 0, 0, 191, 0, 0, 0, 0, 3736, 0, 7842, 0, 0, 0, 0, 0, 5026, 177, 460, 0, 80] payload_rows_per_dst=[6276, 7468, 13138, 15684, 14678, 14888, 7472, 10566] dup_rows_per_dst=[890, 1130, 1444, 0, 3632, 118, 0, 920] combine_rows_per_src=[6276, 7468, 13138, 15684, 14678, 14888, 7472, 10566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=293175296 dispatch_meta_bytes=312128 combine_payload_bytes=293175296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293487424 ingress_bytes=327778304 link_bytes=327778304 token_bytes=586662720 weight_movement_bytes=34603008 total_nvlink_bytes=621265728 weight_to_token_ratio=0.058983 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=21504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.233403370e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=292765696 dispatch_meta_bytes=327512 combine_payload_bytes=292765696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293093208 ingress_bytes=327368704 link_bytes=327368704 token_bytes=585858904 weight_movement_bytes=34603008 total_nvlink_bytes=620461912 weight_to_token_ratio=0.059064 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=39680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.231862078e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=218947584 dispatch_meta_bytes=307440 combine_payload_bytes=218947584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=219255024 ingress_bytes=253550592 link_bytes=253550592 token_bytes=438202608 weight_movement_bytes=34603008 total_nvlink_bytes=472805616 weight_to_token_ratio=0.078966 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=33280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.540904652e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=274563072 dispatch_meta_bytes=313656 combine_payload_bytes=274563072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274876728 ingress_bytes=309166080 link_bytes=309166080 token_bytes=549439800 weight_movement_bytes=34603008 total_nvlink_bytes=584042808 weight_to_token_ratio=0.062979 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=25216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.163367069e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87526 payload_rows_remote=79314 dup_rows=10778 dup_rows_remote=10210 dedup_ratio=0.109639 router_tokens_per_expert=[589, 0, 0, 0, 0, 0, 0, 3056, 527, 0, 438, 0, 0, 0, 57, 3157, 0, 0, 0, 5336, 267, 489, 0, 1336, 0, 0, 0, 0, 0, 0, 0, 0, 1989, 0, 0, 0, 0, 0, 0, 7076, 0, 0, 0, 7353, 0, 0, 183, 0, 0, 0, 0, 4153, 0, 7807, 0, 0, 0, 0, 0, 4653, 163, 433, 0, 90] payload_rows_per_dst=[15406, 11270, 8212, 15614, 3978, 14926, 8306, 9814] dup_rows_per_dst=[6036, 3164, 568, 0, 0, 146, 0, 864] combine_rows_per_src=[15406, 11270, 8212, 15614, 3978, 14926, 8306, 9814] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86888 payload_rows_remote=71576 dup_rows=11416 dup_rows_remote=6456 dedup_ratio=0.116130 router_tokens_per_expert=[170, 17, 0, 0, 0, 0, 1, 3057, 516, 27, 220, 2, 2, 0, 4, 3569, 0, 7, 3, 6384, 182, 1183, 1, 925, 0, 1, 0, 0, 0, 2, 0, 2, 3445, 22, 2, 0, 0, 0, 0, 6891, 0, 27, 0, 7814, 13, 25, 158, 0, 1, 0, 0, 2237, 8, 7971, 0, 16, 0, 1, 1, 3597, 136, 440, 0, 72] payload_rows_per_dst=[15312, 15482, 4594, 15950, 6924, 15828, 4508, 8290] dup_rows_per_dst=[4960, 5966, 8, 2, 14, 246, 16, 204] combine_rows_per_src=[15312, 15482, 4594, 15950, 6924, 15828, 4508, 8290] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=300957696 dispatch_meta_bytes=331096 combine_payload_bytes=300957696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=301288792 ingress_bytes=335560704 link_bytes=335560704 token_bytes=602246488 weight_movement_bytes=34603008 total_nvlink_bytes=636849496 weight_to_token_ratio=0.057457 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=48896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262687914e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=294805504 dispatch_meta_bytes=328320 combine_payload_bytes=294805504 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=295133824 ingress_bytes=364011520 link_bytes=364011520 token_bytes=589939328 weight_movement_bytes=69206016 total_nvlink_bytes=659145344 weight_to_token_ratio=0.117310 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=42496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.369746044e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=218947584 dispatch_meta_bytes=307440 combine_payload_bytes=218947584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=219255024 ingress_bytes=253550592 link_bytes=253550592 token_bytes=438202608 weight_movement_bytes=34603008 total_nvlink_bytes=472805616 weight_to_token_ratio=0.078966 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=33280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.540904652e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=332423168 dispatch_meta_bytes=343088 combine_payload_bytes=332423168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=332766256 ingress_bytes=401629184 link_bytes=401629184 token_bytes=665189424 weight_movement_bytes=69206016 total_nvlink_bytes=734395440 weight_to_token_ratio=0.104040 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=39936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.511298286e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=324870144 dispatch_meta_bytes=358096 combine_payload_bytes=324870144 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=325228240 ingress_bytes=428679168 link_bytes=428679168 token_bytes=650098384 weight_movement_bytes=103809024 total_nvlink_bytes=753907408 weight_to_token_ratio=0.159682 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=30336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.613085198e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=293175296 dispatch_meta_bytes=312128 combine_payload_bytes=293175296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293487424 ingress_bytes=327778304 link_bytes=327778304 token_bytes=586662720 weight_movement_bytes=34603008 total_nvlink_bytes=621265728 weight_to_token_ratio=0.058983 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=21504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.233403370e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77062 payload_rows_remote=62348 dup_rows=21242 dup_rows_remote=11066 dedup_ratio=0.216085 router_tokens_per_expert=[29, 0, 0, 0, 0, 0, 0, 3092, 522, 0, 151, 0, 0, 0, 10, 3647, 0, 0, 0, 6415, 200, 1354, 0, 902, 0, 0, 0, 0, 0, 0, 0, 0, 3418, 0, 0, 0, 0, 0, 0, 7000, 0, 0, 0, 7939, 0, 0, 186, 0, 0, 0, 0, 1717, 0, 8037, 0, 0, 0, 0, 0, 3861, 158, 439, 0, 75] payload_rows_per_dst=[6242, 7742, 14468, 8558, 11382, 2276, 14714, 11680] dup_rows_per_dst=[0, 918, 3274, 0, 3616, 34, 10176, 3224] combine_rows_per_src=[6242, 7742, 14468, 8558, 11382, 2276, 14714, 11680] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78110 payload_rows_remote=65278 dup_rows=20194 dup_rows_remote=18344 dedup_ratio=0.205424 router_tokens_per_expert=[162, 281, 3318, 157, 60, 48, 31, 0, 1420, 220, 104, 8, 4392, 9, 1070, 118, 276, 651, 86, 21, 0, 0, 79, 136, 613, 229, 140, 2, 253, 41, 10, 3497, 4002, 39, 173, 0, 1397, 3, 434, 0, 2269, 2314, 92, 668, 1671, 4743, 4829, 60, 1723, 456, 14, 379, 953, 0, 72, 46, 0, 1723, 2017, 1411, 108, 72, 41, 11] payload_rows_per_dst=[11510, 12832, 11194, 8840, 3986, 15364, 6354, 8030] dup_rows_per_dst=[4608, 1850, 962, 730, 106, 8270, 932, 2736] combine_rows_per_src=[11510, 12832, 11194, 8840, 3986, 15364, 6354, 8030] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=255377408 dispatch_meta_bytes=293656 combine_payload_bytes=255377408 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=255671064 ingress_bytes=324583424 link_bytes=324583424 token_bytes=511048472 weight_movement_bytes=69206016 total_nvlink_bytes=580254488 weight_to_token_ratio=0.135420 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=21120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.221381293e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93492 payload_rows_remote=80056 dup_rows=4812 dup_rows_remote=3774 dedup_ratio=0.048950 router_tokens_per_expert=[3, 0, 4117, 12, 0, 0, 2, 0, 35, 2, 1, 0, 7689, 1, 602, 0, 4, 330, 61, 0, 0, 0, 74, 1, 704, 0, 13, 0, 0, 0, 0, 6520, 7168, 0, 136, 0, 38, 0, 138, 0, 502, 707, 4, 6, 35, 7449, 7991, 0, 38, 491, 0, 2, 243, 0, 76, 0, 0, 37, 3894, 25, 0, 0, 0, 1] payload_rows_per_dst=[8254, 15930, 16156, 13436, 14412, 2134, 15302, 7868] dup_rows_per_dst=[14, 730, 766, 1038, 548, 374, 1296, 46] combine_rows_per_src=[8254, 15930, 16156, 13436, 14412, 2134, 15302, 7868] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78628 payload_rows_remote=62576 dup_rows=19676 dup_rows_remote=13398 dedup_ratio=0.200155 router_tokens_per_expert=[1, 0, 4712, 4, 1, 0, 0, 0, 11, 0, 1, 0, 7761, 0, 542, 0, 0, 345, 64, 0, 0, 0, 81, 0, 344, 0, 4, 0, 0, 55, 0, 6582, 7953, 0, 144, 0, 13, 0, 147, 0, 293, 720, 0, 3, 12, 7415, 7725, 0, 14, 479, 0, 1, 247, 0, 74, 0, 0, 13, 3382, 9, 0, 0, 0, 0] payload_rows_per_dst=[9434, 1106, 852, 3936, 16030, 15756, 15462, 16052] dup_rows_per_dst=[2, 2, 128, 282, 7430, 4556, 998, 6278] combine_rows_per_src=[9434, 1106, 852, 3936, 16030, 15756, 15462, 16052] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=351707136 dispatch_meta_bytes=375104 combine_payload_bytes=351707136 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=352082240 ingress_bytes=455516160 link_bytes=455516160 token_bytes=703789376 weight_movement_bytes=103809024 total_nvlink_bytes=807598400 weight_to_token_ratio=0.147500 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.714070638e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=309215232 dispatch_meta_bytes=319976 combine_payload_bytes=309215232 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=309535208 ingress_bytes=413024256 link_bytes=413024256 token_bytes=618750440 weight_movement_bytes=103809024 total_nvlink_bytes=722559464 weight_to_token_ratio=0.167772 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=26240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.554177024e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93478 payload_rows_remote=91238 dup_rows=4826 dup_rows_remote=4502 dedup_ratio=0.049093 router_tokens_per_expert=[0, 0, 4159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7778, 0, 658, 0, 0, 362, 55, 0, 0, 0, 56, 0, 442, 0, 0, 0, 0, 0, 0, 6569, 7770, 0, 151, 0, 0, 0, 154, 0, 511, 771, 0, 0, 0, 7326, 7753, 0, 0, 487, 0, 0, 247, 0, 100, 0, 0, 0, 3803, 0, 0, 0, 0, 0] payload_rows_per_dst=[8318, 13496, 16186, 13254, 15540, 2240, 15256, 9188] dup_rows_per_dst=[0, 622, 266, 768, 610, 324, 1064, 1172] combine_rows_per_src=[8318, 13496, 16186, 13254, 15540, 2240, 15256, 9188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90170 payload_rows_remote=75492 dup_rows=8134 dup_rows_remote=4502 dedup_ratio=0.082743 router_tokens_per_expert=[698, 0, 0, 0, 0, 0, 0, 2885, 587, 0, 438, 0, 0, 0, 52, 3222, 0, 0, 0, 5153, 289, 426, 0, 1423, 0, 0, 0, 0, 0, 0, 0, 0, 2107, 0, 0, 0, 0, 0, 0, 7048, 0, 0, 0, 7312, 0, 0, 191, 0, 0, 0, 0, 3736, 0, 7842, 0, 0, 0, 0, 0, 5026, 177, 460, 0, 80] payload_rows_per_dst=[6276, 7468, 13138, 15684, 14678, 14888, 7472, 10566] dup_rows_per_dst=[890, 1130, 1444, 0, 3632, 118, 0, 920] combine_rows_per_src=[6276, 7468, 13138, 15684, 14678, 14888, 7472, 10566] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=267378688 dispatch_meta_bytes=334488 combine_payload_bytes=267378688 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267713176 ingress_bytes=336584704 link_bytes=336584704 token_bytes=535091864 weight_movement_bytes=69206016 total_nvlink_bytes=604297880 weight_to_token_ratio=0.129335 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=41728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266541144e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90066 payload_rows_remote=73860 dup_rows=8238 dup_rows_remote=7438 dedup_ratio=0.083801 router_tokens_per_expert=[0, 0, 4102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7774, 0, 627, 0, 0, 343, 53, 0, 0, 0, 57, 0, 544, 0, 0, 0, 0, 0, 0, 6562, 7191, 0, 145, 0, 0, 0, 161, 0, 444, 722, 0, 0, 0, 7548, 8050, 0, 0, 504, 0, 0, 265, 0, 90, 0, 0, 0, 3970, 0, 0, 0, 0, 0] payload_rows_per_dst=[11996, 16078, 16206, 13336, 7136, 11398, 5976, 7940] dup_rows_per_dst=[3610, 724, 800, 876, 456, 1088, 684, 0] combine_rows_per_src=[11996, 16078, 16206, 13336, 7136, 11398, 5976, 7940] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79466 payload_rows_remote=67246 dup_rows=18838 dup_rows_remote=16028 dedup_ratio=0.191630 router_tokens_per_expert=[0, 7, 0, 0, 0, 0, 1169, 0, 74, 7494, 815, 0, 0, 74, 0, 3, 34, 0, 0, 70, 0, 243, 0, 0, 0, 0, 0, 0, 539, 7701, 0, 0, 0, 0, 0, 0, 0, 7698, 0, 8118, 75, 0, 0, 684, 17, 0, 74, 0, 251, 0, 0, 0, 0, 5500, 0, 0, 0, 0, 326, 7906, 0, 0, 0, 280] payload_rows_per_dst=[2338, 15130, 694, 1078, 16350, 15694, 12220, 15962] dup_rows_per_dst=[14, 1790, 0, 0, 11760, 1402, 2810, 1062] combine_rows_per_src=[2338, 15130, 694, 1078, 16350, 15694, 12220, 15962] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86036 payload_rows_remote=70030 dup_rows=12268 dup_rows_remote=4158 dedup_ratio=0.124797 router_tokens_per_expert=[0, 0, 4321, 3, 4, 0, 0, 0, 17, 23, 0, 0, 7621, 0, 622, 2, 5, 372, 47, 0, 0, 0, 48, 0, 447, 3, 6, 0, 0, 2, 2, 6545, 7730, 0, 146, 0, 25, 0, 151, 0, 398, 760, 1, 13, 26, 7453, 7792, 1, 31, 457, 0, 2, 268, 0, 72, 0, 0, 30, 3679, 24, 1, 1, 1, 0] payload_rows_per_dst=[16006, 15806, 16174, 12820, 352, 15950, 1506, 7422] dup_rows_per_dst=[8110, 1188, 354, 766, 292, 1354, 154, 50] combine_rows_per_src=[16006, 15806, 16174, 12820, 352, 15950, 1506, 7422] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=327909376 dispatch_meta_bytes=335320 combine_payload_bytes=327909376 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=328244696 ingress_bytes=431718400 link_bytes=431718400 token_bytes=656154072 weight_movement_bytes=103809024 total_nvlink_bytes=759963096 weight_to_token_ratio=0.158208 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=44416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.624521583e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=256311296 dispatch_meta_bytes=303896 combine_payload_bytes=256311296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=256615192 ingress_bytes=290914304 link_bytes=290914304 token_bytes=512926488 weight_movement_bytes=34603008 total_nvlink_bytes=547529496 weight_to_token_ratio=0.067462 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=28800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.094687105e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=373710848 dispatch_meta_bytes=382960 combine_payload_bytes=373710848 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=374093808 ingress_bytes=477519872 link_bytes=477519872 token_bytes=747804656 weight_movement_bytes=103809024 total_nvlink_bytes=851613680 weight_to_token_ratio=0.138818 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=17280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.796868835e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78628 payload_rows_remote=62576 dup_rows=19676 dup_rows_remote=13398 dedup_ratio=0.200155 router_tokens_per_expert=[1, 0, 4712, 4, 1, 0, 0, 0, 11, 0, 1, 0, 7761, 0, 542, 0, 0, 345, 64, 0, 0, 0, 81, 0, 344, 0, 4, 0, 0, 55, 0, 6582, 7953, 0, 144, 0, 13, 0, 147, 0, 293, 720, 0, 3, 12, 7415, 7725, 0, 14, 479, 0, 1, 247, 0, 74, 0, 0, 13, 3382, 9, 0, 0, 0, 0] payload_rows_per_dst=[9434, 1106, 852, 3936, 16030, 15756, 15462, 16052] dup_rows_per_dst=[2, 2, 128, 282, 7430, 4556, 998, 6278] combine_rows_per_src=[9434, 1106, 852, 3936, 16030, 15756, 15462, 16052] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78110 payload_rows_remote=65278 dup_rows=20194 dup_rows_remote=18344 dedup_ratio=0.205424 router_tokens_per_expert=[162, 281, 3318, 157, 60, 48, 31, 0, 1420, 220, 104, 8, 4392, 9, 1070, 118, 276, 651, 86, 21, 0, 0, 79, 136, 613, 229, 140, 2, 253, 41, 10, 3497, 4002, 39, 173, 0, 1397, 3, 434, 0, 2269, 2314, 92, 668, 1671, 4743, 4829, 60, 1723, 456, 14, 379, 953, 0, 72, 46, 0, 1723, 2017, 1411, 108, 72, 41, 11] payload_rows_per_dst=[11510, 12832, 11194, 8840, 3986, 15364, 6354, 8030] dup_rows_per_dst=[4608, 1850, 962, 730, 106, 8270, 932, 2736] combine_rows_per_src=[11510, 12832, 11194, 8840, 3986, 15364, 6354, 8030] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=309215232 dispatch_meta_bytes=319976 combine_payload_bytes=309215232 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=309535208 ingress_bytes=413024256 link_bytes=413024256 token_bytes=618750440 weight_movement_bytes=103809024 total_nvlink_bytes=722559464 weight_to_token_ratio=0.167772 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=26240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.554177024e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=302530560 dispatch_meta_bytes=325192 combine_payload_bytes=302530560 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=302855752 ingress_bytes=337133568 link_bytes=337133568 token_bytes=605386312 weight_movement_bytes=34603008 total_nvlink_bytes=639989320 weight_to_token_ratio=0.057159 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=44544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.268606475e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93492 payload_rows_remote=80056 dup_rows=4812 dup_rows_remote=3774 dedup_ratio=0.048950 router_tokens_per_expert=[3, 0, 4117, 12, 0, 0, 2, 0, 35, 2, 1, 0, 7689, 1, 602, 0, 4, 330, 61, 0, 0, 0, 74, 1, 704, 0, 13, 0, 0, 0, 0, 6520, 7168, 0, 136, 0, 38, 0, 138, 0, 502, 707, 4, 6, 35, 7449, 7991, 0, 38, 491, 0, 2, 243, 0, 76, 0, 0, 37, 3894, 25, 0, 0, 0, 1] payload_rows_per_dst=[8254, 15930, 16156, 13436, 14412, 2134, 15302, 7868] dup_rows_per_dst=[14, 730, 766, 1038, 548, 374, 1296, 46] combine_rows_per_src=[8254, 15930, 16156, 13436, 14412, 2134, 15302, 7868] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=373710848 dispatch_meta_bytes=382960 combine_payload_bytes=373710848 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=374093808 ingress_bytes=477519872 link_bytes=477519872 token_bytes=747804656 weight_movement_bytes=103809024 total_nvlink_bytes=851613680 weight_to_token_ratio=0.138818 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=17280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.796868835e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90532 payload_rows_remote=74888 dup_rows=7772 dup_rows_remote=2540 dedup_ratio=0.079061 router_tokens_per_expert=[0, 994, 531, 1165, 7902, 6745, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3441, 0, 0, 0, 0, 846, 6065, 89, 0, 0, 0, 0, 0, 509, 0, 0, 5687, 16, 127, 0, 0, 550, 0, 0, 0, 20, 0, 0, 1163, 31, 0, 0, 0, 74, 6997, 161, 5965, 0, 0, 69, 0, 0, 0] payload_rows_per_dst=[4876, 15804, 15644, 13348, 12272, 14006, 2474, 12108] dup_rows_per_dst=[540, 10, 5232, 652, 406, 588, 62, 282] combine_rows_per_src=[4876, 15804, 15644, 13348, 12272, 14006, 2474, 12108] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93640 payload_rows_remote=79218 dup_rows=4664 dup_rows_remote=4134 dedup_ratio=0.047445 router_tokens_per_expert=[0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7781, 0, 530, 0, 0, 380, 68, 0, 0, 0, 69, 0, 522, 0, 0, 0, 0, 0, 0, 6510, 7211, 0, 126, 0, 0, 0, 139, 0, 391, 743, 0, 0, 0, 7568, 8044, 0, 0, 552, 0, 0, 247, 0, 80, 0, 0, 0, 4114, 0, 0, 0, 0, 0] payload_rows_per_dst=[8154, 16056, 16224, 13242, 14422, 1932, 15382, 8228] dup_rows_per_dst=[0, 566, 898, 822, 530, 336, 1512, 0] combine_rows_per_src=[8154, 16056, 16224, 13242, 14422, 1932, 15382, 8228] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=286842880 dispatch_meta_bytes=296752 combine_payload_bytes=286842880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=287139632 ingress_bytes=321445888 link_bytes=321445888 token_bytes=573982512 weight_movement_bytes=34603008 total_nvlink_bytes=608585520 weight_to_token_ratio=0.060286 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=33024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.209574998e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86980 payload_rows_remote=71476 dup_rows=11324 dup_rows_remote=10402 dedup_ratio=0.115194 router_tokens_per_expert=[0, 0, 4144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7794, 0, 659, 0, 0, 365, 70, 0, 0, 0, 71, 0, 459, 0, 0, 0, 0, 17, 0, 6603, 7942, 0, 142, 0, 0, 0, 143, 0, 261, 719, 0, 0, 0, 7450, 7697, 0, 0, 437, 0, 0, 251, 0, 75, 0, 0, 0, 3853, 0, 0, 0, 0, 0] payload_rows_per_dst=[8288, 1318, 990, 13346, 15884, 15560, 15504, 16090] dup_rows_per_dst=[0, 0, 148, 812, 570, 1668, 922, 7204] combine_rows_per_src=[8288, 1318, 990, 13346, 15884, 15560, 15504, 16090] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81662 payload_rows_remote=70066 dup_rows=16642 dup_rows_remote=14818 dedup_ratio=0.169291 router_tokens_per_expert=[6, 222, 1170, 1670, 1662, 853, 514, 115, 81, 5088, 783, 0, 0, 1158, 144, 108, 62, 336, 0, 212, 1580, 307, 1019, 0, 1, 408, 0, 54, 885, 4495, 975, 0, 56, 1732, 9, 160, 666, 3822, 204, 6221, 69, 31, 1, 2431, 818, 123, 1112, 92, 440, 39, 49, 3, 17, 823, 0, 0, 0, 0, 766, 4436, 43, 242, 85, 754] payload_rows_per_dst=[14506, 11596, 15940, 12724, 12868, 7814, 2616, 3598] dup_rows_per_dst=[8094, 1824, 3534, 912, 430, 1540, 126, 182] combine_rows_per_src=[14506, 11596, 15940, 12724, 12868, 7814, 2616, 3598] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=292765696 dispatch_meta_bytes=327512 combine_payload_bytes=292765696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293093208 ingress_bytes=327368704 link_bytes=327368704 token_bytes=585858904 weight_movement_bytes=34603008 total_nvlink_bytes=620461912 weight_to_token_ratio=0.059064 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=39680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.231862078e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82602 payload_rows_remote=66590 dup_rows=15702 dup_rows_remote=13646 dedup_ratio=0.159729 router_tokens_per_expert=[0, 2, 6, 11, 14, 9, 944, 1, 73, 7464, 782, 0, 0, 74, 0, 0, 23, 2, 0, 74, 12, 230, 10, 0, 0, 1, 0, 0, 569, 7676, 8, 0, 1, 13, 0, 1, 6, 7691, 1, 8105, 74, 0, 0, 855, 7, 1, 94, 0, 247, 0, 0, 0, 0, 5587, 0, 0, 0, 0, 323, 7905, 0, 0, 0, 256] payload_rows_per_dst=[1918, 15098, 680, 1156, 16238, 15770, 15730, 16012] dup_rows_per_dst=[56, 1688, 22, 0, 16, 1674, 10190, 2056] combine_rows_per_src=[1918, 15098, 680, 1156, 16238, 15770, 15730, 16012] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=275439616 dispatch_meta_bytes=333096 combine_payload_bytes=275439616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275772712 ingress_bytes=310042624 link_bytes=310042624 token_bytes=551212328 weight_movement_bytes=34603008 total_nvlink_bytes=585815336 weight_to_token_ratio=0.062776 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=38016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.166665434e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=256311296 dispatch_meta_bytes=303896 combine_payload_bytes=256311296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=256615192 ingress_bytes=290914304 link_bytes=290914304 token_bytes=512926488 weight_movement_bytes=34603008 total_nvlink_bytes=547529496 weight_to_token_ratio=0.067462 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=28800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.094687105e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=267378688 dispatch_meta_bytes=334488 combine_payload_bytes=267378688 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267713176 ingress_bytes=336584704 link_bytes=336584704 token_bytes=535091864 weight_movement_bytes=69206016 total_nvlink_bytes=604297880 weight_to_token_ratio=0.129335 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=41728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266541144e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=327909376 dispatch_meta_bytes=335320 combine_payload_bytes=327909376 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=328244696 ingress_bytes=431718400 link_bytes=431718400 token_bytes=656154072 weight_movement_bytes=103809024 total_nvlink_bytes=759963096 weight_to_token_ratio=0.158208 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=44416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.624521583e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=306741248 dispatch_meta_bytes=309712 combine_payload_bytes=306741248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307050960 ingress_bytes=341344256 link_bytes=341344256 token_bytes=613792208 weight_movement_bytes=34603008 total_nvlink_bytes=648395216 weight_to_token_ratio=0.056376 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=27136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.284450955e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=324476928 dispatch_meta_bytes=333408 combine_payload_bytes=324476928 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=324810336 ingress_bytes=393682944 link_bytes=393682944 token_bytes=649287264 weight_movement_bytes=69206016 total_nvlink_bytes=718493280 weight_to_token_ratio=0.106588 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=41600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.481397224e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12881843768 prefetch_bytes=588251136 reduce_bytes=657457152 weight_movement_bytes=1245708288 egress_bytes=6444709432 ingress_bytes=7682842624 link_bytes=7682842624 total_nvlink_bytes=14127552056 weight_to_token_ratio=0.096703 prefetch_share_of_ingress=0.076567 dedup_ratio=0.171990 prefetch_slots=34 unused_prefetch_slots=334 slot_utilization=0.092391 reduce_slots=19 remote_demand=17 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029457 gemm_inner_pad_upper_bound=27305 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.256952968e-04 prefetch_bytes_per_gemm_flop=9.624094203e-06 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90312 payload_rows_remote=74744 dup_rows=7992 dup_rows_remote=6400 dedup_ratio=0.081299 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 999, 0, 99, 7511, 829, 0, 0, 99, 0, 0, 3, 0, 0, 55, 0, 199, 0, 0, 0, 0, 0, 0, 642, 7772, 0, 0, 0, 0, 0, 0, 0, 7571, 0, 8093, 100, 0, 0, 804, 6, 0, 99, 0, 247, 0, 0, 0, 0, 5510, 0, 0, 0, 0, 347, 7905, 0, 0, 0, 261] payload_rows_per_dst=[5102, 12138, 9520, 13996, 6888, 15568, 11090, 16010] dup_rows_per_dst=[372, 1464, 322, 1124, 1678, 1592, 424, 1016] combine_rows_per_src=[5102, 12138, 9520, 13996, 6888, 15568, 11090, 16010] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90994 payload_rows_remote=75590 dup_rows=7310 dup_rows_remote=5772 dedup_ratio=0.074361 router_tokens_per_expert=[0, 0, 21, 33, 41, 21, 673, 11, 76, 7449, 1248, 0, 0, 77, 2, 0, 20, 3, 0, 65, 35, 249, 29, 0, 1, 8, 0, 0, 795, 7641, 26, 0, 1, 36, 0, 3, 6, 7496, 2, 8076, 76, 0, 3, 797, 47, 3, 163, 1, 290, 2, 0, 0, 0, 4792, 0, 0, 0, 0, 371, 7810, 0, 0, 1, 652] payload_rows_per_dst=[14990, 2654, 16228, 15404, 1574, 14254, 10078, 15812] dup_rows_per_dst=[1508, 152, 726, 1538, 4, 1436, 90, 1856] combine_rows_per_src=[14990, 2654, 16228, 15404, 1574, 14254, 10078, 15812] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93640 payload_rows_remote=79218 dup_rows=4664 dup_rows_remote=4134 dedup_ratio=0.047445 router_tokens_per_expert=[0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7781, 0, 530, 0, 0, 380, 68, 0, 0, 0, 69, 0, 522, 0, 0, 0, 0, 0, 0, 6510, 7211, 0, 126, 0, 0, 0, 139, 0, 391, 743, 0, 0, 0, 7568, 8044, 0, 0, 552, 0, 0, 247, 0, 80, 0, 0, 0, 4114, 0, 0, 0, 0, 0] payload_rows_per_dst=[8154, 16056, 16224, 13242, 14422, 1932, 15382, 8228] dup_rows_per_dst=[0, 566, 898, 822, 530, 336, 1512, 0] combine_rows_per_src=[8154, 16056, 16224, 13242, 14422, 1932, 15382, 8228] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=286990336 dispatch_meta_bytes=339536 combine_payload_bytes=286990336 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=287329872 ingress_bytes=356196352 link_bytes=356196352 token_bytes=574320208 weight_movement_bytes=69206016 total_nvlink_bytes=643526224 weight_to_token_ratio=0.120501 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.340338196e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92378 payload_rows_remote=76172 dup_rows=5926 dup_rows_remote=5310 dedup_ratio=0.060282 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 542, 0, 89, 7476, 1042, 0, 0, 89, 0, 0, 5, 0, 0, 53, 0, 250, 0, 0, 0, 0, 0, 0, 761, 7730, 0, 0, 0, 0, 0, 0, 0, 7580, 0, 8103, 90, 0, 0, 687, 15, 0, 113, 0, 273, 0, 0, 0, 0, 5467, 0, 0, 0, 0, 355, 7908, 0, 0, 0, 524] payload_rows_per_dst=[14952, 12650, 16206, 15460, 15160, 1600, 11246, 5104] dup_rows_per_dst=[1084, 1986, 616, 1522, 0, 210, 234, 274] combine_rows_per_src=[14952, 12650, 16206, 15460, 15160, 1600, 11246, 5104] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92090 payload_rows_remote=77118 dup_rows=6214 dup_rows_remote=3978 dedup_ratio=0.063212 router_tokens_per_expert=[0, 17, 15, 28, 27, 16, 1044, 3, 72, 7454, 715, 0, 0, 77, 3, 10, 14, 5, 0, 49, 27, 198, 18, 0, 0, 2, 0, 0, 550, 7650, 16, 0, 1, 28, 0, 3, 18, 7619, 0, 8091, 72, 0, 2, 1052, 40, 0, 106, 0, 261, 1, 0, 0, 0, 5247, 0, 0, 0, 0, 359, 7859, 0, 1, 1, 381] payload_rows_per_dst=[14972, 15814, 16240, 15350, 15332, 2320, 10578, 1484] dup_rows_per_dst=[2236, 1658, 564, 1086, 6, 224, 440, 0] combine_rows_per_src=[14972, 15814, 16240, 15350, 15332, 2320, 10578, 1484] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=272752640 dispatch_meta_bytes=320944 combine_payload_bytes=272752640 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=273073584 ingress_bytes=341958656 link_bytes=341958656 token_bytes=545826224 weight_movement_bytes=69206016 total_nvlink_bytes=615032240 weight_to_token_ratio=0.126791 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.035000 gemm_active_groups=8 gemm_peak_group_rows=41216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.286762893e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82602 payload_rows_remote=66590 dup_rows=15702 dup_rows_remote=13646 dedup_ratio=0.159729 router_tokens_per_expert=[0, 2, 6, 11, 14, 9, 944, 1, 73, 7464, 782, 0, 0, 74, 0, 0, 23, 2, 0, 74, 12, 230, 10, 0, 0, 1, 0, 0, 569, 7676, 8, 0, 1, 13, 0, 1, 6, 7691, 1, 8105, 74, 0, 0, 855, 7, 1, 94, 0, 247, 0, 0, 0, 0, 5587, 0, 0, 0, 0, 323, 7905, 0, 0, 0, 256] payload_rows_per_dst=[1918, 15098, 680, 1156, 16238, 15770, 15730, 16012] dup_rows_per_dst=[56, 1688, 22, 0, 16, 1674, 10190, 2056] combine_rows_per_src=[1918, 15098, 680, 1156, 16238, 15770, 15730, 16012] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=306151424 dispatch_meta_bytes=324576 combine_payload_bytes=306151424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=306476000 ingress_bytes=340754432 link_bytes=340754432 token_bytes=612627424 weight_movement_bytes=34603008 total_nvlink_bytes=647230432 weight_to_token_ratio=0.056483 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=37376 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.282231495e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=309616640 dispatch_meta_bytes=325448 combine_payload_bytes=309616640 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=309942088 ingress_bytes=413425664 link_bytes=413425664 token_bytes=619558728 weight_movement_bytes=103809024 total_nvlink_bytes=723367752 weight_to_token_ratio=0.167553 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=42112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.555687490e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=324476928 dispatch_meta_bytes=333408 combine_payload_bytes=324476928 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=324810336 ingress_bytes=393682944 link_bytes=393682944 token_bytes=649287264 weight_movement_bytes=69206016 total_nvlink_bytes=718493280 weight_to_token_ratio=0.106588 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=41600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.481397224e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81662 payload_rows_remote=70066 dup_rows=16642 dup_rows_remote=14818 dedup_ratio=0.169291 router_tokens_per_expert=[6, 222, 1170, 1670, 1662, 853, 514, 115, 81, 5088, 783, 0, 0, 1158, 144, 108, 62, 336, 0, 212, 1580, 307, 1019, 0, 1, 408, 0, 54, 885, 4495, 975, 0, 56, 1732, 9, 160, 666, 3822, 204, 6221, 69, 31, 1, 2431, 818, 123, 1112, 92, 440, 39, 49, 3, 17, 823, 0, 0, 0, 0, 766, 4436, 43, 242, 85, 754] payload_rows_per_dst=[14506, 11596, 15940, 12724, 12868, 7814, 2616, 3598] dup_rows_per_dst=[8094, 1824, 3534, 912, 430, 1540, 126, 182] combine_rows_per_src=[14506, 11596, 15940, 12724, 12868, 7814, 2616, 3598] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12675550512 prefetch_bytes=830472192 reduce_bytes=449839104 weight_movement_bytes=1280311296 egress_bytes=6341651760 ingress_bytes=7614210048 link_bytes=7614210048 total_nvlink_bytes=13955861808 weight_to_token_ratio=0.101006 prefetch_share_of_ingress=0.109069 dedup_ratio=0.205946 prefetch_slots=48 unused_prefetch_slots=320 slot_utilization=0.130435 reduce_slots=13 remote_demand=24 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029022 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.245724322e-04 prefetch_bytes_per_gemm_flop=1.358695652e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=312000512 dispatch_meta_bytes=325928 combine_payload_bytes=312000512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=312326440 ingress_bytes=346603520 link_bytes=346603520 token_bytes=624326952 weight_movement_bytes=34603008 total_nvlink_bytes=658929960 weight_to_token_ratio=0.055424 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=43392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.304241142e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79466 payload_rows_remote=67246 dup_rows=18838 dup_rows_remote=16028 dedup_ratio=0.191630 router_tokens_per_expert=[0, 7, 0, 0, 0, 0, 1169, 0, 74, 7494, 815, 0, 0, 74, 0, 3, 34, 0, 0, 70, 0, 243, 0, 0, 0, 0, 0, 0, 539, 7701, 0, 0, 0, 0, 0, 0, 0, 7698, 0, 8118, 75, 0, 0, 684, 17, 0, 74, 0, 251, 0, 0, 0, 0, 5500, 0, 0, 0, 0, 326, 7906, 0, 0, 0, 280] payload_rows_per_dst=[2338, 15130, 694, 1078, 16350, 15694, 12220, 15962] dup_rows_per_dst=[14, 1790, 0, 0, 11760, 1402, 2810, 1062] combine_rows_per_src=[2338, 15130, 694, 1078, 16350, 15694, 12220, 15962] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12255281648 prefetch_bytes=795869184 reduce_bytes=692060160 weight_movement_bytes=1487929344 egress_bytes=6131360240 ingress_bytes=7611850752 link_bytes=7611850752 total_nvlink_bytes=13743210992 weight_to_token_ratio=0.121411 prefetch_share_of_ingress=0.104557 dedup_ratio=0.208448 prefetch_slots=46 unused_prefetch_slots=322 slot_utilization=0.125000 reduce_slots=20 remote_demand=23 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028804 gemm_inner_pad_upper_bound=28321 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.245338329e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13794185856 prefetch_bytes=830472192 reduce_bytes=726663168 weight_movement_bytes=1557135360 egress_bytes=6901002880 ingress_bytes=8450318336 link_bytes=8450318336 total_nvlink_bytes=15351321216 weight_to_token_ratio=0.112883 prefetch_share_of_ingress=0.098277 dedup_ratio=0.143341 prefetch_slots=48 unused_prefetch_slots=320 slot_utilization=0.130435 reduce_slots=21 remote_demand=24 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029565 gemm_inner_pad_upper_bound=28448 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.382515982e-04 prefetch_bytes_per_gemm_flop=1.358695652e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13945272408 prefetch_bytes=761266176 reduce_bytes=761266176 weight_movement_bytes=1522532352 egress_bytes=6976583768 ingress_bytes=8491220992 link_bytes=8491220992 total_nvlink_bytes=15467804760 weight_to_token_ratio=0.109179 prefetch_share_of_ingress=0.089653 dedup_ratio=0.134268 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=22 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=28067 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.389207869e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=315875328 dispatch_meta_bytes=324384 combine_payload_bytes=315875328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=316199712 ingress_bytes=350478336 link_bytes=350478336 token_bytes=632075040 weight_movement_bytes=34603008 total_nvlink_bytes=666678048 weight_to_token_ratio=0.054745 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=36736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.318821763e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13688605240 prefetch_bytes=726663168 reduce_bytes=692060160 weight_movement_bytes=1418723328 egress_bytes=6848211512 ingress_bytes=8259117056 link_bytes=8259117056 total_nvlink_bytes=15107328568 weight_to_token_ratio=0.103643 prefetch_share_of_ingress=0.087983 dedup_ratio=0.144384 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=20 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=28067 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.351234459e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=275439616 dispatch_meta_bytes=333096 combine_payload_bytes=275439616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275772712 ingress_bytes=310042624 link_bytes=310042624 token_bytes=551212328 weight_movement_bytes=34603008 total_nvlink_bytes=585815336 weight_to_token_ratio=0.062776 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=38016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.166665434e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12466860072 prefetch_bytes=726663168 reduce_bytes=830472192 weight_movement_bytes=1557135360 egress_bytes=6237106216 ingress_bytes=7786889216 link_bytes=7786889216 total_nvlink_bytes=14023995432 weight_to_token_ratio=0.124902 prefetch_share_of_ingress=0.093319 dedup_ratio=0.193368 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=24 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=27686 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.273975531e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90994 payload_rows_remote=75590 dup_rows=7310 dup_rows_remote=5772 dedup_ratio=0.074361 router_tokens_per_expert=[0, 0, 21, 33, 41, 21, 673, 11, 76, 7449, 1248, 0, 0, 77, 2, 0, 20, 3, 0, 65, 35, 249, 29, 0, 1, 8, 0, 0, 795, 7641, 26, 0, 1, 36, 0, 3, 6, 7496, 2, 8076, 76, 0, 3, 797, 47, 3, 163, 1, 290, 2, 0, 0, 0, 4792, 0, 0, 0, 0, 371, 7810, 0, 0, 1, 652] payload_rows_per_dst=[14990, 2654, 16228, 15404, 1574, 14254, 10078, 15812] dup_rows_per_dst=[1508, 152, 726, 1538, 4, 1436, 90, 1856] combine_rows_per_src=[14990, 2654, 16228, 15404, 1574, 14254, 10078, 15812] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87526 payload_rows_remote=79314 dup_rows=10778 dup_rows_remote=10210 dedup_ratio=0.109639 router_tokens_per_expert=[589, 0, 0, 0, 0, 0, 0, 3056, 527, 0, 438, 0, 0, 0, 57, 3157, 0, 0, 0, 5336, 267, 489, 0, 1336, 0, 0, 0, 0, 0, 0, 0, 0, 1989, 0, 0, 0, 0, 0, 0, 7076, 0, 0, 0, 7353, 0, 0, 183, 0, 0, 0, 0, 4153, 0, 7807, 0, 0, 0, 0, 0, 4653, 163, 433, 0, 90] payload_rows_per_dst=[15406, 11270, 8212, 15614, 3978, 14926, 8306, 9814] dup_rows_per_dst=[6036, 3164, 568, 0, 0, 146, 0, 864] combine_rows_per_src=[15406, 11270, 8212, 15614, 3978, 14926, 8306, 9814] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90786 payload_rows_remote=90786 dup_rows=7518 dup_rows_remote=7518 dedup_ratio=0.076477 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 653, 0, 80, 7428, 1171, 0, 0, 80, 0, 0, 1, 0, 0, 68, 0, 325, 0, 0, 0, 0, 0, 0, 816, 7661, 0, 0, 0, 0, 0, 0, 0, 7634, 0, 8112, 80, 0, 0, 754, 24, 0, 111, 0, 248, 0, 0, 0, 0, 4968, 0, 0, 0, 0, 328, 7879, 0, 0, 0, 731] payload_rows_per_dst=[14856, 2502, 16224, 15324, 0, 15596, 10366, 15918] dup_rows_per_dst=[1306, 160, 788, 1630, 0, 1610, 66, 1958] combine_rows_per_src=[14856, 2502, 16224, 15324, 0, 15596, 10366, 15918] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12881843768 prefetch_bytes=588251136 reduce_bytes=657457152 weight_movement_bytes=1245708288 egress_bytes=6444709432 ingress_bytes=7682842624 link_bytes=7682842624 total_nvlink_bytes=14127552056 weight_to_token_ratio=0.096703 prefetch_share_of_ingress=0.076567 dedup_ratio=0.171990 prefetch_slots=34 unused_prefetch_slots=334 slot_utilization=0.092391 reduce_slots=19 remote_demand=17 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029457 gemm_inner_pad_upper_bound=27305 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.256952968e-04 prefetch_bytes_per_gemm_flop=9.624094203e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=315875328 dispatch_meta_bytes=324384 combine_payload_bytes=315875328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=316199712 ingress_bytes=350478336 link_bytes=350478336 token_bytes=632075040 weight_movement_bytes=34603008 total_nvlink_bytes=666678048 weight_to_token_ratio=0.054745 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=36736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.318821763e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=272752640 dispatch_meta_bytes=320944 combine_payload_bytes=272752640 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=273073584 ingress_bytes=341958656 link_bytes=341958656 token_bytes=545826224 weight_movement_bytes=69206016 total_nvlink_bytes=615032240 weight_to_token_ratio=0.126791 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.035000 gemm_active_groups=8 gemm_peak_group_rows=41216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.286762893e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13794185856 prefetch_bytes=830472192 reduce_bytes=726663168 weight_movement_bytes=1557135360 egress_bytes=6901002880 ingress_bytes=8450318336 link_bytes=8450318336 total_nvlink_bytes=15351321216 weight_to_token_ratio=0.112883 prefetch_share_of_ingress=0.098277 dedup_ratio=0.143341 prefetch_slots=48 unused_prefetch_slots=320 slot_utilization=0.130435 reduce_slots=21 remote_demand=24 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029565 gemm_inner_pad_upper_bound=28448 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.382515982e-04 prefetch_bytes_per_gemm_flop=1.358695652e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=286990336 dispatch_meta_bytes=339536 combine_payload_bytes=286990336 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=287329872 ingress_bytes=356196352 link_bytes=356196352 token_bytes=574320208 weight_movement_bytes=69206016 total_nvlink_bytes=643526224 weight_to_token_ratio=0.120501 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.340338196e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=309616640 dispatch_meta_bytes=325448 combine_payload_bytes=309616640 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=309942088 ingress_bytes=413425664 link_bytes=413425664 token_bytes=619558728 weight_movement_bytes=103809024 total_nvlink_bytes=723367752 weight_to_token_ratio=0.167553 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=42112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.555687490e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=324870144 dispatch_meta_bytes=358096 combine_payload_bytes=324870144 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=325228240 ingress_bytes=428679168 link_bytes=428679168 token_bytes=650098384 weight_movement_bytes=103809024 total_nvlink_bytes=753907408 weight_to_token_ratio=0.159682 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=30336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.613085198e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12466860072 prefetch_bytes=726663168 reduce_bytes=830472192 weight_movement_bytes=1557135360 egress_bytes=6237106216 ingress_bytes=7786889216 link_bytes=7786889216 total_nvlink_bytes=14023995432 weight_to_token_ratio=0.124902 prefetch_share_of_ingress=0.093319 dedup_ratio=0.193368 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=24 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=27686 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.273975531e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=371859456 dispatch_meta_bytes=393216 combine_payload_bytes=371859456 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=372252672 ingress_bytes=475668480 link_bytes=475668480 token_bytes=744112128 weight_movement_bytes=103809024 total_nvlink_bytes=847921152 weight_to_token_ratio=0.139507 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=19840 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.789902196e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12255281648 prefetch_bytes=795869184 reduce_bytes=692060160 weight_movement_bytes=1487929344 egress_bytes=6131360240 ingress_bytes=7611850752 link_bytes=7611850752 total_nvlink_bytes=13743210992 weight_to_token_ratio=0.121411 prefetch_share_of_ingress=0.104557 dedup_ratio=0.208448 prefetch_slots=46 unused_prefetch_slots=322 slot_utilization=0.125000 reduce_slots=20 remote_demand=23 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028804 gemm_inner_pad_upper_bound=28321 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.245338329e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12675550512 prefetch_bytes=830472192 reduce_bytes=449839104 weight_movement_bytes=1280311296 egress_bytes=6341651760 ingress_bytes=7614210048 link_bytes=7614210048 total_nvlink_bytes=13955861808 weight_to_token_ratio=0.101006 prefetch_share_of_ingress=0.109069 dedup_ratio=0.205946 prefetch_slots=48 unused_prefetch_slots=320 slot_utilization=0.130435 reduce_slots=13 remote_demand=24 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029022 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.245724322e-04 prefetch_bytes_per_gemm_flop=1.358695652e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13945272408 prefetch_bytes=761266176 reduce_bytes=761266176 weight_movement_bytes=1522532352 egress_bytes=6976583768 ingress_bytes=8491220992 link_bytes=8491220992 total_nvlink_bytes=15467804760 weight_to_token_ratio=0.109179 prefetch_share_of_ingress=0.089653 dedup_ratio=0.134268 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=22 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=28067 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.389207869e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14115210512 prefetch_bytes=622854144 reduce_bytes=1072693248 weight_movement_bytes=1695547392 egress_bytes=7061603600 ingress_bytes=8749154304 link_bytes=8749154304 total_nvlink_bytes=15810757904 weight_to_token_ratio=0.120122 prefetch_share_of_ingress=0.071190 dedup_ratio=0.137880 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=31 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028152 gemm_inner_pad_upper_bound=27686 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.431407099e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none - [2026-08-26 15:09:14] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 791.9 | throughput per GPU (TFLOP/s/GPU): 161.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.680684E+00 | loss scale: 1.0 | grad norm: 194.973 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74864 payload_rows_remote=65788 dup_rows=23440 dup_rows_remote=20506 dedup_ratio=0.238444 router_tokens_per_expert=[1124, 333, 768, 1145, 617, 755, 768, 1082, 1380, 288, 939, 347, 382, 1350, 916, 1107, 703, 986, 781, 599, 638, 703, 359, 1511, 529, 701, 740, 346, 786, 111, 889, 758, 1092, 809, 1243, 776, 552, 217, 439, 991, 625, 615, 602, 903, 1078, 634, 743, 652, 652, 525, 846, 521, 984, 1324, 866, 761, 1171, 316, 1138, 613, 142, 824, 1023, 1034] payload_rows_per_dst=[9598, 7794, 9998, 9076, 9370, 8938, 9920, 10170] dup_rows_per_dst=[3996, 2924, 2562, 2934, 2868, 2766, 3038, 2352] combine_rows_per_src=[9598, 7794, 9998, 9076, 9370, 8938, 9920, 10170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90066 payload_rows_remote=73860 dup_rows=8238 dup_rows_remote=7438 dedup_ratio=0.083801 router_tokens_per_expert=[0, 0, 4102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7774, 0, 627, 0, 0, 343, 53, 0, 0, 0, 57, 0, 544, 0, 0, 0, 0, 0, 0, 6562, 7191, 0, 145, 0, 0, 0, 161, 0, 444, 722, 0, 0, 0, 7548, 8050, 0, 0, 504, 0, 0, 265, 0, 90, 0, 0, 0, 3970, 0, 0, 0, 0, 0] payload_rows_per_dst=[11996, 16078, 16206, 13336, 7136, 11398, 5976, 7940] dup_rows_per_dst=[3610, 724, 800, 876, 456, 1088, 684, 0] combine_rows_per_src=[11996, 16078, 16206, 13336, 7136, 11398, 5976, 7940] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=302530560 dispatch_meta_bytes=325192 combine_payload_bytes=302530560 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=302855752 ingress_bytes=337133568 link_bytes=337133568 token_bytes=605386312 weight_movement_bytes=34603008 total_nvlink_bytes=639989320 weight_to_token_ratio=0.057159 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=44544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.268606475e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92378 payload_rows_remote=76172 dup_rows=5926 dup_rows_remote=5310 dedup_ratio=0.060282 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 542, 0, 89, 7476, 1042, 0, 0, 89, 0, 0, 5, 0, 0, 53, 0, 250, 0, 0, 0, 0, 0, 0, 761, 7730, 0, 0, 0, 0, 0, 0, 0, 7580, 0, 8103, 90, 0, 0, 687, 15, 0, 113, 0, 273, 0, 0, 0, 0, 5467, 0, 0, 0, 0, 355, 7908, 0, 0, 0, 524] payload_rows_per_dst=[14952, 12650, 16206, 15460, 15160, 1600, 11246, 5104] dup_rows_per_dst=[1084, 1986, 616, 1522, 0, 210, 234, 274] combine_rows_per_src=[14952, 12650, 16206, 15460, 15160, 1600, 11246, 5104] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=312000512 dispatch_meta_bytes=325928 combine_payload_bytes=312000512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=312326440 ingress_bytes=346603520 link_bytes=346603520 token_bytes=624326952 weight_movement_bytes=34603008 total_nvlink_bytes=658929960 weight_to_token_ratio=0.055424 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=43392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.304241142e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13688605240 prefetch_bytes=726663168 reduce_bytes=692060160 weight_movement_bytes=1418723328 egress_bytes=6848211512 ingress_bytes=8259117056 link_bytes=8259117056 total_nvlink_bytes=15107328568 weight_to_token_ratio=0.103643 prefetch_share_of_ingress=0.087983 dedup_ratio=0.144384 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=20 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=28067 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.351234459e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none + [2026-08-26 15:57:02] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 794.6 | throughput per GPU (TFLOP/s/GPU): 160.6 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.680684E+00 | loss scale: 1.0 | grad norm: 194.973 | number of skipped iterations: 0 | number of nan iterations: 0 | WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74994 payload_rows_remote=66186 dup_rows=23310 dup_rows_remote=19882 dedup_ratio=0.237122 router_tokens_per_expert=[1105, 349, 854, 1167, 606, 796, 816, 983, 1278, 378, 977, 345, 371, 1352, 923, 972, 760, 1071, 787, 597, 674, 749, 403, 1330, 464, 657, 890, 330, 824, 120, 859, 759, 1171, 836, 1182, 892, 349, 204, 579, 760, 590, 608, 607, 835, 1159, 632, 668, 632, 799, 604, 800, 460, 1001, 1198, 921, 804, 1204, 272, 1156, 638, 243, 843, 1076, 883] payload_rows_per_dst=[9652, 8808, 10092, 9198, 9008, 10414, 8928, 8894] dup_rows_per_dst=[4070, 3428, 2650, 2942, 2938, 3444, 1850, 1988] combine_rows_per_src=[9652, 8808, 10092, 9198, 9008, 10414, 8928, 8894] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74562 payload_rows_remote=64774 dup_rows=23742 dup_rows_remote=21030 dedup_ratio=0.241516 router_tokens_per_expert=[1158, 369, 863, 1175, 666, 999, 862, 817, 1312, 531, 1046, 324, 281, 1440, 826, 876, 763, 1014, 823, 639, 576, 768, 393, 1274, 415, 663, 873, 441, 828, 147, 746, 558, 1062, 814, 1279, 990, 428, 192, 545, 788, 525, 588, 658, 642, 1073, 746, 658, 471, 834, 624, 767, 499, 928, 1353, 896, 694, 1166, 342, 1395, 628, 222, 852, 1187, 840] payload_rows_per_dst=[9800, 7520, 9788, 8840, 9146, 9718, 9200, 10550] dup_rows_per_dst=[4548, 2872, 2712, 2852, 3050, 2732, 2262, 2714] combine_rows_per_src=[9800, 7520, 9788, 8840, 9146, 9718, 9200, 10550] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75346 payload_rows_remote=66536 dup_rows=22958 dup_rows_remote=20206 dedup_ratio=0.233541 router_tokens_per_expert=[1171, 361, 891, 1215, 700, 732, 809, 1069, 1294, 356, 902, 362, 331, 1297, 908, 1076, 744, 974, 849, 661, 657, 756, 398, 1399, 505, 711, 777, 379, 752, 94, 926, 687, 1003, 840, 1211, 855, 360, 220, 429, 863, 633, 660, 608, 779, 1115, 607, 688, 658, 762, 554, 907, 443, 940, 1290, 757, 779, 1218, 264, 1269, 628, 182, 813, 1084, 990] payload_rows_per_dst=[8648, 9220, 10330, 9194, 8810, 8788, 9914, 10442] dup_rows_per_dst=[2972, 3678, 2546, 2898, 2752, 2708, 2950, 2454] combine_rows_per_src=[8648, 9220, 10330, 9194, 8810, 8788, 9914, 10442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74970 payload_rows_remote=65332 dup_rows=23334 dup_rows_remote=19016 dedup_ratio=0.237366 router_tokens_per_expert=[1089, 421, 809, 1188, 657, 787, 915, 960, 1315, 408, 894, 367, 293, 1340, 890, 1038, 738, 1046, 830, 670, 603, 764, 401, 1344, 449, 617, 844, 453, 832, 99, 805, 497, 1082, 794, 1322, 893, 391, 216, 589, 815, 647, 546, 558, 805, 1112, 684, 647, 601, 793, 591, 771, 523, 962, 1328, 865, 775, 1157, 338, 1439, 561, 222, 876, 1103, 783] payload_rows_per_dst=[9638, 9634, 10186, 8800, 9116, 10320, 8892, 8384] dup_rows_per_dst=[4318, 3654, 2606, 2768, 3088, 3536, 1668, 1696] combine_rows_per_src=[9638, 9634, 10186, 8800, 9116, 10320, 8892, 8384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74864 payload_rows_remote=65788 dup_rows=23440 dup_rows_remote=20506 dedup_ratio=0.238444 router_tokens_per_expert=[1124, 333, 768, 1145, 617, 755, 768, 1082, 1380, 288, 939, 347, 382, 1350, 916, 1107, 703, 986, 781, 599, 638, 703, 359, 1511, 529, 701, 740, 346, 786, 111, 889, 758, 1092, 809, 1243, 776, 552, 217, 439, 991, 625, 615, 602, 903, 1078, 634, 743, 652, 652, 525, 846, 521, 984, 1324, 866, 761, 1171, 316, 1138, 613, 142, 824, 1023, 1034] payload_rows_per_dst=[9598, 7794, 9998, 9076, 9370, 8938, 9920, 10170] dup_rows_per_dst=[3996, 2924, 2562, 2934, 2868, 2766, 3038, 2352] combine_rows_per_src=[9598, 7794, 9998, 9076, 9370, 8938, 9920, 10170] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75246 payload_rows_remote=65336 dup_rows=23058 dup_rows_remote=20084 dedup_ratio=0.234558 router_tokens_per_expert=[1160, 324, 860, 1170, 671, 724, 767, 1049, 1229, 318, 901, 339, 383, 1325, 901, 1032, 803, 997, 794, 603, 750, 726, 437, 1366, 557, 661, 806, 330, 737, 93, 909, 817, 1073, 910, 1193, 872, 377, 224, 431, 811, 675, 638, 573, 818, 1105, 591, 775, 769, 750, 584, 863, 434, 954, 1203, 865, 789, 1182, 281, 1126, 616, 193, 856, 1073, 1009] payload_rows_per_dst=[8100, 9276, 9504, 9506, 9692, 9066, 9910, 10192] dup_rows_per_dst=[2616, 3580, 2114, 3048, 3424, 2822, 2974, 2480] combine_rows_per_src=[8100, 9276, 9504, 9506, 9692, 9066, 9910, 10192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75018 payload_rows_remote=64086 dup_rows=23286 dup_rows_remote=20664 dedup_ratio=0.236877 router_tokens_per_expert=[1143, 331, 908, 1182, 728, 704, 813, 999, 1317, 357, 937, 343, 374, 1343, 959, 1033, 746, 1098, 885, 687, 690, 718, 378, 1276, 537, 677, 719, 378, 777, 105, 931, 542, 1031, 838, 1179, 810, 386, 200, 497, 877, 649, 573, 585, 706, 1098, 673, 715, 687, 812, 519, 940, 380, 960, 1275, 931, 802, 1272, 301, 1408, 544, 211, 832, 1012, 834] payload_rows_per_dst=[6896, 9468, 8660, 10514, 9510, 8760, 10278, 10932] dup_rows_per_dst=[2070, 3858, 1744, 3468, 3570, 2612, 3342, 2622] combine_rows_per_src=[6896, 9468, 8660, 10514, 9510, 8760, 10278, 10932] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75242 payload_rows_remote=66138 dup_rows=23062 dup_rows_remote=20486 dedup_ratio=0.234599 router_tokens_per_expert=[1090, 342, 874, 1233, 686, 980, 835, 884, 1234, 370, 771, 374, 357, 1188, 841, 1160, 776, 1093, 792, 571, 670, 709, 370, 1425, 414, 640, 866, 303, 756, 95, 987, 839, 1076, 832, 1248, 886, 375, 214, 527, 823, 571, 662, 629, 878, 1095, 619, 740, 646, 633, 521, 790, 493, 1119, 1205, 999, 829, 1163, 281, 1150, 669, 218, 858, 1026, 852] payload_rows_per_dst=[8612, 8974, 10200, 9330, 9010, 9104, 9968, 10044] dup_rows_per_dst=[2770, 3616, 2612, 2936, 2952, 2576, 3210, 2390] combine_rows_per_src=[8612, 8974, 10200, 9330, 9010, 9104, 9968, 10044] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75018 payload_rows_remote=64086 dup_rows=23286 dup_rows_remote=20664 dedup_ratio=0.236877 router_tokens_per_expert=[1143, 331, 908, 1182, 728, 704, 813, 999, 1317, 357, 937, 343, 374, 1343, 959, 1033, 746, 1098, 885, 687, 690, 718, 378, 1276, 537, 677, 719, 378, 777, 105, 931, 542, 1031, 838, 1179, 810, 386, 200, 497, 877, 649, 573, 585, 706, 1098, 673, 715, 687, 812, 519, 940, 380, 960, 1275, 931, 802, 1272, 301, 1408, 544, 211, 832, 1012, 834] payload_rows_per_dst=[6896, 9468, 8660, 10514, 9510, 8760, 10278, 10932] dup_rows_per_dst=[2070, 3858, 1744, 3468, 3570, 2612, 3342, 2622] combine_rows_per_src=[6896, 9468, 8660, 10514, 9510, 8760, 10278, 10932] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=269467648 dispatch_meta_bytes=345176 combine_payload_bytes=269467648 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=269812824 ingress_bytes=338673664 link_bytes=338673664 token_bytes=539280472 weight_movement_bytes=69206016 total_nvlink_bytes=608486488 weight_to_token_ratio=0.128330 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=9600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274401732e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=271097856 dispatch_meta_bytes=344272 combine_payload_bytes=271097856 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=271442128 ingress_bytes=340303872 link_bytes=340303872 token_bytes=542539984 weight_movement_bytes=69206016 total_nvlink_bytes=611746000 weight_to_token_ratio=0.127559 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=10368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.280536074e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265314304 dispatch_meta_bytes=343216 combine_payload_bytes=265314304 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265657520 ingress_bytes=369123328 link_bytes=369123328 token_bytes=530971824 weight_movement_bytes=103809024 total_nvlink_bytes=634780848 weight_to_token_ratio=0.195508 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=9088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388981366e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=271097856 dispatch_meta_bytes=344272 combine_payload_bytes=271097856 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=271442128 ingress_bytes=340303872 link_bytes=340303872 token_bytes=542539984 weight_movement_bytes=69206016 total_nvlink_bytes=611746000 weight_to_token_ratio=0.127559 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=10368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.280536074e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=272531456 dispatch_meta_bytes=346968 combine_payload_bytes=272531456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272878424 ingress_bytes=307134464 link_bytes=307134464 token_bytes=545409880 weight_movement_bytes=34603008 total_nvlink_bytes=580012888 weight_to_token_ratio=0.063444 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=9984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.155722262e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267599872 dispatch_meta_bytes=337392 combine_payload_bytes=267599872 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267937264 ingress_bytes=371408896 link_bytes=371408896 token_bytes=535537136 weight_movement_bytes=103809024 total_nvlink_bytes=639346160 weight_to_token_ratio=0.193841 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=7936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397581775e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=269467648 dispatch_meta_bytes=345176 combine_payload_bytes=269467648 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=269812824 ingress_bytes=338673664 link_bytes=338673664 token_bytes=539280472 weight_movement_bytes=69206016 total_nvlink_bytes=608486488 weight_to_token_ratio=0.128330 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=9600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274401732e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267616256 dispatch_meta_bytes=341680 combine_payload_bytes=267616256 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267957936 ingress_bytes=336822272 link_bytes=336822272 token_bytes=535574192 weight_movement_bytes=69206016 total_nvlink_bytes=604780208 weight_to_token_ratio=0.129218 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=7936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.267435093e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=270901248 dispatch_meta_bytes=346496 combine_payload_bytes=270901248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271247744 ingress_bytes=305504256 link_bytes=305504256 token_bytes=542148992 weight_movement_bytes=34603008 total_nvlink_bytes=576752000 weight_to_token_ratio=0.063826 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=8960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.149587920e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74970 payload_rows_remote=65332 dup_rows=23334 dup_rows_remote=19016 dedup_ratio=0.237366 router_tokens_per_expert=[1089, 421, 809, 1188, 657, 787, 915, 960, 1315, 408, 894, 367, 293, 1340, 890, 1038, 738, 1046, 830, 670, 603, 764, 401, 1344, 449, 617, 844, 453, 832, 99, 805, 497, 1082, 794, 1322, 893, 391, 216, 589, 815, 647, 546, 558, 805, 1112, 684, 647, 601, 793, 591, 771, 523, 962, 1328, 865, 775, 1157, 338, 1439, 561, 222, 876, 1103, 783] payload_rows_per_dst=[9638, 9634, 10186, 8800, 9116, 10320, 8892, 8384] dup_rows_per_dst=[4318, 3654, 2606, 2768, 3088, 3536, 1668, 1696] combine_rows_per_src=[9638, 9634, 10186, 8800, 9116, 10320, 8892, 8384] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=262496256 dispatch_meta_bytes=339000 combine_payload_bytes=262496256 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=262835256 ingress_bytes=331702272 link_bytes=331702272 token_bytes=525331512 weight_movement_bytes=69206016 total_nvlink_bytes=594537528 weight_to_token_ratio=0.131738 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=9600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.248168945e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76718 payload_rows_remote=67398 dup_rows=21586 dup_rows_remote=19688 dedup_ratio=0.219584 router_tokens_per_expert=[678, 1074, 1130, 387, 939, 303, 374, 982, 110, 929, 1308, 340, 708, 986, 948, 1331, 985, 1225, 806, 353, 1565, 125, 1761, 1073, 645, 607, 687, 827, 239, 321, 225, 103, 394, 248, 509, 217, 1052, 1809, 1082, 1195, 1955, 1619, 1311, 1394, 490, 1146, 29, 213, 653, 1135, 854, 11, 145, 1546, 218, 1203, 677, 345, 833, 544, 1140, 339, 343, 429] payload_rows_per_dst=[9134, 10128, 9644, 9320, 10548, 8628, 9624, 9692] dup_rows_per_dst=[2600, 3192, 2620, 1898, 2464, 3776, 1906, 3130] combine_rows_per_src=[9134, 10128, 9644, 9320, 10548, 8628, 9624, 9692] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=270901248 dispatch_meta_bytes=346496 combine_payload_bytes=270901248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271247744 ingress_bytes=305504256 link_bytes=305504256 token_bytes=542148992 weight_movement_bytes=34603008 total_nvlink_bytes=576752000 weight_to_token_ratio=0.063826 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=8960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.149587920e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74562 payload_rows_remote=64774 dup_rows=23742 dup_rows_remote=21030 dedup_ratio=0.241516 router_tokens_per_expert=[1158, 369, 863, 1175, 666, 999, 862, 817, 1312, 531, 1046, 324, 281, 1440, 826, 876, 763, 1014, 823, 639, 576, 768, 393, 1274, 415, 663, 873, 441, 828, 147, 746, 558, 1062, 814, 1279, 990, 428, 192, 545, 788, 525, 588, 658, 642, 1073, 746, 658, 471, 834, 624, 767, 499, 928, 1353, 896, 694, 1166, 342, 1395, 628, 222, 852, 1187, 840] payload_rows_per_dst=[9800, 7520, 9788, 8840, 9146, 9718, 9200, 10550] dup_rows_per_dst=[4548, 2872, 2712, 2852, 3050, 2732, 2262, 2714] combine_rows_per_src=[9800, 7520, 9788, 8840, 9146, 9718, 9200, 10550] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76010 payload_rows_remote=65946 dup_rows=22294 dup_rows_remote=18906 dedup_ratio=0.226786 router_tokens_per_expert=[788, 972, 936, 379, 843, 310, 209, 1117, 72, 901, 1541, 363, 641, 946, 892, 1370, 930, 1141, 754, 408, 1500, 191, 1886, 1162, 700, 612, 671, 878, 219, 313, 238, 104, 433, 355, 443, 188, 986, 1577, 1057, 1173, 2134, 1618, 1020, 1269, 432, 1314, 47, 254, 914, 1110, 887, 10, 171, 1842, 190, 1212, 751, 344, 793, 492, 1179, 288, 306, 376] payload_rows_per_dst=[10186, 10064, 8914, 10022, 9924, 7666, 10248, 8986] dup_rows_per_dst=[3922, 3388, 1560, 3082, 2500, 2876, 2424, 2542] combine_rows_per_src=[10186, 10064, 8914, 10022, 9924, 7666, 10248, 8986] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267599872 dispatch_meta_bytes=337392 combine_payload_bytes=267599872 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267937264 ingress_bytes=371408896 link_bytes=371408896 token_bytes=535537136 weight_movement_bytes=103809024 total_nvlink_bytes=639346160 weight_to_token_ratio=0.193841 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=7936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397581775e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76008 payload_rows_remote=66514 dup_rows=22296 dup_rows_remote=20338 dedup_ratio=0.226807 router_tokens_per_expert=[733, 993, 1149, 398, 841, 256, 326, 1046, 77, 959, 1313, 376, 607, 1013, 1031, 1318, 956, 1062, 846, 373, 1534, 161, 1773, 1142, 710, 597, 645, 801, 244, 357, 159, 74, 442, 331, 471, 251, 921, 1902, 1103, 1120, 2038, 1631, 1200, 1300, 441, 1354, 27, 234, 701, 1137, 761, 15, 169, 1509, 213, 1221, 853, 413, 850, 528, 1132, 347, 302, 365] payload_rows_per_dst=[8968, 8112, 9622, 9190, 10516, 10202, 9494, 9904] dup_rows_per_dst=[2516, 2640, 2526, 2060, 2566, 4808, 1958, 3222] combine_rows_per_src=[8968, 8112, 9622, 9190, 10516, 10202, 9494, 9904] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76480 payload_rows_remote=65864 dup_rows=21824 dup_rows_remote=19082 dedup_ratio=0.222005 router_tokens_per_expert=[671, 955, 1153, 429, 792, 289, 379, 1041, 77, 911, 1138, 335, 685, 1030, 922, 1346, 1038, 1211, 762, 382, 1502, 199, 1787, 1244, 657, 619, 693, 822, 236, 335, 212, 51, 425, 308, 483, 205, 995, 1836, 1273, 1154, 2041, 1594, 1124, 1422, 456, 1106, 30, 200, 720, 1105, 844, 19, 207, 1697, 188, 1179, 787, 417, 949, 545, 911, 371, 282, 376] payload_rows_per_dst=[8872, 9852, 9912, 9140, 10616, 8364, 9826, 9898] dup_rows_per_dst=[2546, 3036, 2764, 2192, 2742, 3500, 2092, 2952] combine_rows_per_src=[8872, 9852, 9912, 9140, 10616, 8364, 9826, 9898] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76426 payload_rows_remote=66714 dup_rows=21878 dup_rows_remote=19300 dedup_ratio=0.222555 router_tokens_per_expert=[801, 1046, 904, 356, 896, 261, 207, 1128, 73, 867, 1496, 378, 507, 764, 848, 1407, 798, 1350, 778, 439, 1422, 297, 1929, 1332, 753, 770, 791, 883, 224, 351, 197, 96, 433, 386, 402, 187, 968, 1332, 1144, 1004, 2120, 1533, 891, 1212, 464, 1438, 50, 254, 1083, 1083, 950, 12, 148, 1603, 199, 1301, 849, 343, 827, 503, 1046, 331, 365, 342] payload_rows_per_dst=[9010, 9662, 9712, 9772, 9728, 8376, 10170, 9996] dup_rows_per_dst=[2730, 3018, 2578, 2598, 1984, 3308, 2588, 3074] combine_rows_per_src=[9010, 9662, 9712, 9772, 9728, 8376, 10170, 9996] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=276062208 dispatch_meta_bytes=348344 combine_payload_bytes=276062208 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=276410552 ingress_bytes=345268224 link_bytes=345268224 token_bytes=552472760 weight_movement_bytes=69206016 total_nvlink_bytes=621678776 weight_to_token_ratio=0.125266 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.299216531e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76444 payload_rows_remote=67076 dup_rows=21860 dup_rows_remote=17488 dedup_ratio=0.222371 router_tokens_per_expert=[745, 1028, 1068, 482, 788, 287, 275, 1096, 77, 985, 1247, 287, 542, 987, 1061, 1310, 994, 1296, 764, 316, 1592, 212, 1963, 1141, 601, 629, 629, 761, 243, 370, 197, 97, 477, 361, 539, 199, 887, 1892, 1027, 1094, 2161, 1523, 1282, 1194, 448, 1354, 48, 245, 749, 1156, 727, 21, 146, 1626, 219, 1319, 734, 353, 743, 499, 1116, 317, 264, 362] payload_rows_per_dst=[9084, 8744, 9984, 9242, 10416, 9368, 9874, 9732] dup_rows_per_dst=[2454, 2696, 2646, 2134, 2536, 4372, 2052, 2970] combine_rows_per_src=[9084, 8744, 9984, 9242, 10416, 9368, 9874, 9732] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=270114816 dispatch_meta_bytes=339408 combine_payload_bytes=270114816 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=270454224 ingress_bytes=339320832 link_bytes=339320832 token_bytes=540569040 weight_movement_bytes=69206016 total_nvlink_bytes=609775056 weight_to_token_ratio=0.128024 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=10624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.276836973e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75834 payload_rows_remote=65452 dup_rows=22470 dup_rows_remote=18182 dedup_ratio=0.228577 router_tokens_per_expert=[724, 933, 940, 539, 920, 235, 321, 1410, 73, 996, 1412, 411, 568, 1034, 904, 1381, 956, 1107, 715, 272, 1313, 235, 1680, 1532, 739, 618, 784, 898, 215, 416, 217, 96, 401, 310, 441, 155, 956, 1314, 1033, 995, 2244, 1476, 917, 1245, 367, 1436, 44, 305, 969, 1165, 1018, 8, 127, 1607, 196, 1075, 888, 375, 933, 528, 831, 410, 374, 415] payload_rows_per_dst=[10382, 9886, 10668, 11140, 9144, 6696, 9934, 7984] dup_rows_per_dst=[4288, 3672, 2674, 4266, 1718, 1932, 2396, 1524] combine_rows_per_src=[10382, 9886, 10668, 11140, 9144, 6696, 9934, 7984] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76718 payload_rows_remote=67398 dup_rows=21586 dup_rows_remote=19688 dedup_ratio=0.219584 router_tokens_per_expert=[678, 1074, 1130, 387, 939, 303, 374, 982, 110, 929, 1308, 340, 708, 986, 948, 1331, 985, 1225, 806, 353, 1565, 125, 1761, 1073, 645, 607, 687, 827, 239, 321, 225, 103, 394, 248, 509, 217, 1052, 1809, 1082, 1195, 1955, 1619, 1311, 1394, 490, 1146, 29, 213, 653, 1135, 854, 11, 145, 1546, 218, 1203, 677, 345, 833, 544, 1140, 339, 343, 429] payload_rows_per_dst=[9134, 10128, 9644, 9320, 10548, 8628, 9624, 9692] dup_rows_per_dst=[2600, 3192, 2620, 1898, 2464, 3776, 1906, 3130] combine_rows_per_src=[9134, 10128, 9644, 9320, 10548, 8628, 9624, 9692] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76008 payload_rows_remote=66514 dup_rows=22296 dup_rows_remote=20338 dedup_ratio=0.226807 router_tokens_per_expert=[733, 993, 1149, 398, 841, 256, 326, 1046, 77, 959, 1313, 376, 607, 1013, 1031, 1318, 956, 1062, 846, 373, 1534, 161, 1773, 1142, 710, 597, 645, 801, 244, 357, 159, 74, 442, 331, 471, 251, 921, 1902, 1103, 1120, 2038, 1631, 1200, 1300, 441, 1354, 27, 234, 701, 1137, 761, 15, 169, 1509, 213, 1221, 853, 413, 850, 528, 1132, 347, 302, 365] payload_rows_per_dst=[8968, 8112, 9622, 9190, 10516, 10202, 9494, 9904] dup_rows_per_dst=[2516, 2640, 2526, 2060, 2566, 4808, 1958, 3222] combine_rows_per_src=[8968, 8112, 9622, 9190, 10516, 10202, 9494, 9904] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75276 payload_rows_remote=65468 dup_rows=23028 dup_rows_remote=20138 dedup_ratio=0.234253 router_tokens_per_expert=[707, 880, 1110, 545, 767, 209, 378, 1210, 71, 1050, 1162, 335, 565, 930, 1060, 1318, 1023, 1182, 698, 321, 1394, 206, 1646, 1416, 603, 770, 641, 875, 321, 368, 221, 104, 405, 292, 547, 188, 1063, 1660, 1158, 1005, 2221, 1572, 1116, 1461, 432, 1068, 30, 219, 805, 1101, 997, 12, 131, 1610, 203, 1067, 745, 401, 889, 549, 960, 377, 360, 422] payload_rows_per_dst=[9102, 7766, 9728, 9738, 9116, 9662, 10356, 9808] dup_rows_per_dst=[2510, 2448, 2752, 2510, 1786, 4770, 3362, 2890] combine_rows_per_src=[9102, 7766, 9728, 9738, 9116, 9662, 10356, 9808] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76444 payload_rows_remote=67076 dup_rows=21860 dup_rows_remote=17488 dedup_ratio=0.222371 router_tokens_per_expert=[745, 1028, 1068, 482, 788, 287, 275, 1096, 77, 985, 1247, 287, 542, 987, 1061, 1310, 994, 1296, 764, 316, 1592, 212, 1963, 1141, 601, 629, 629, 761, 243, 370, 197, 97, 477, 361, 539, 199, 887, 1892, 1027, 1094, 2161, 1523, 1282, 1194, 448, 1354, 48, 245, 749, 1156, 727, 21, 146, 1626, 219, 1319, 734, 353, 743, 499, 1116, 317, 264, 362] payload_rows_per_dst=[9084, 8744, 9984, 9242, 10416, 9368, 9874, 9732] dup_rows_per_dst=[2454, 2696, 2646, 2134, 2536, 4372, 2052, 2970] combine_rows_per_src=[9084, 8744, 9984, 9242, 10416, 9368, 9874, 9732] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265314304 dispatch_meta_bytes=343216 combine_payload_bytes=265314304 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265657520 ingress_bytes=369123328 link_bytes=369123328 token_bytes=530971824 weight_movement_bytes=103809024 total_nvlink_bytes=634780848 weight_to_token_ratio=0.195508 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=9088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388981366e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=270114816 dispatch_meta_bytes=339408 combine_payload_bytes=270114816 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=270454224 ingress_bytes=339320832 link_bytes=339320832 token_bytes=540569040 weight_movement_bytes=69206016 total_nvlink_bytes=609775056 weight_to_token_ratio=0.128024 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=10624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.276836973e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=269778944 dispatch_meta_bytes=339784 combine_payload_bytes=269778944 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=270118728 ingress_bytes=373587968 link_bytes=373587968 token_bytes=539897672 weight_movement_bytes=103809024 total_nvlink_bytes=643706696 weight_to_token_ratio=0.192275 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.405781447e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=268091392 dispatch_meta_bytes=334536 combine_payload_bytes=268091392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268425928 ingress_bytes=302694400 link_bytes=302694400 token_bytes=536517320 weight_movement_bytes=34603008 total_nvlink_bytes=571120328 weight_to_token_ratio=0.064496 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.139014658e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=276062208 dispatch_meta_bytes=348344 combine_payload_bytes=276062208 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=276410552 ingress_bytes=345268224 link_bytes=345268224 token_bytes=552472760 weight_movement_bytes=69206016 total_nvlink_bytes=621678776 weight_to_token_ratio=0.125266 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.299216531e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=272441344 dispatch_meta_bytes=347408 combine_payload_bytes=272441344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272788752 ingress_bytes=307044352 link_bytes=307044352 token_bytes=545230096 weight_movement_bytes=34603008 total_nvlink_bytes=579833104 weight_to_token_ratio=0.063465 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=13056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.155383177e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=269778944 dispatch_meta_bytes=339784 combine_payload_bytes=269778944 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=270118728 ingress_bytes=373587968 link_bytes=373587968 token_bytes=539897672 weight_movement_bytes=103809024 total_nvlink_bytes=643706696 weight_to_token_ratio=0.192275 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.405781447e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=273260544 dispatch_meta_bytes=344056 combine_payload_bytes=273260544 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=273604600 ingress_bytes=377069568 link_bytes=377069568 token_bytes=546865144 weight_movement_bytes=103809024 total_nvlink_bytes=650674168 weight_to_token_ratio=0.189826 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.418882428e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=274743296 dispatch_meta_bytes=338256 combine_payload_bytes=274743296 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=275081552 ingress_bytes=378552320 link_bytes=378552320 token_bytes=549824848 weight_movement_bytes=103809024 total_nvlink_bytes=653633872 weight_to_token_ratio=0.188804 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=10624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.424461904e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=268156928 dispatch_meta_bytes=342424 combine_payload_bytes=268156928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268499352 ingress_bytes=302759936 link_bytes=302759936 token_bytes=536656280 weight_movement_bytes=34603008 total_nvlink_bytes=571259288 weight_to_token_ratio=0.064479 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.139261265e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75834 payload_rows_remote=65452 dup_rows=22470 dup_rows_remote=18182 dedup_ratio=0.228577 router_tokens_per_expert=[724, 933, 940, 539, 920, 235, 321, 1410, 73, 996, 1412, 411, 568, 1034, 904, 1381, 956, 1107, 715, 272, 1313, 235, 1680, 1532, 739, 618, 784, 898, 215, 416, 217, 96, 401, 310, 441, 155, 956, 1314, 1033, 995, 2244, 1476, 917, 1245, 367, 1436, 44, 305, 969, 1165, 1018, 8, 127, 1607, 196, 1075, 888, 375, 933, 528, 831, 410, 374, 415] payload_rows_per_dst=[10382, 9886, 10668, 11140, 9144, 6696, 9934, 7984] dup_rows_per_dst=[4288, 3672, 2674, 4266, 1718, 1932, 2396, 1524] combine_rows_per_src=[10382, 9886, 10668, 11140, 9144, 6696, 9934, 7984] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76838 payload_rows_remote=66368 dup_rows=21466 dup_rows_remote=18772 dedup_ratio=0.218363 router_tokens_per_expert=[242, 952, 404, 454, 121, 1055, 994, 228, 771, 326, 933, 1260, 1230, 1406, 839, 1968, 865, 1211, 250, 755, 70, 384, 1150, 898, 370, 852, 775, 508, 278, 727, 996, 329, 381, 1082, 740, 700, 1747, 1586, 2055, 498, 640, 140, 324, 305, 1556, 1607, 148, 1519, 1055, 646, 814, 873, 71, 309, 293, 666, 984, 382, 69, 701, 153, 1289, 475, 1743] payload_rows_per_dst=[9634, 10532, 9256, 10470, 7826, 8772, 10212, 10136] dup_rows_per_dst=[3376, 2998, 1910, 2694, 2148, 3706, 3178, 1456] combine_rows_per_src=[9634, 10532, 9256, 10470, 7826, 8772, 10212, 10136] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=274743296 dispatch_meta_bytes=338256 combine_payload_bytes=274743296 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=275081552 ingress_bytes=378552320 link_bytes=378552320 token_bytes=549824848 weight_movement_bytes=103809024 total_nvlink_bytes=653633872 weight_to_token_ratio=0.188804 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=10624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.424461904e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75484 payload_rows_remote=64316 dup_rows=22820 dup_rows_remote=18400 dedup_ratio=0.232137 router_tokens_per_expert=[106, 1008, 436, 359, 72, 929, 1010, 189, 654, 278, 1027, 1290, 1102, 1234, 412, 1797, 882, 1540, 333, 822, 97, 266, 1249, 1055, 342, 926, 997, 613, 396, 591, 984, 202, 390, 983, 663, 514, 1970, 1488, 1770, 504, 737, 137, 362, 272, 1587, 1804, 207, 1720, 1096, 562, 741, 908, 62, 149, 308, 688, 865, 346, 91, 731, 166, 1616, 374, 2143] payload_rows_per_dst=[9022, 11168, 10256, 10800, 7220, 9300, 7326, 10392] dup_rows_per_dst=[2736, 4420, 2776, 3242, 1864, 3808, 1702, 2272] combine_rows_per_src=[9022, 11168, 10256, 10800, 7220, 9300, 7326, 10392] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=271843328 dispatch_meta_bytes=340560 combine_payload_bytes=271843328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272183888 ingress_bytes=306446336 link_bytes=306446336 token_bytes=544027216 weight_movement_bytes=34603008 total_nvlink_bytes=578630224 weight_to_token_ratio=0.063605 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=9344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.153132891e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=268091392 dispatch_meta_bytes=334536 combine_payload_bytes=268091392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268425928 ingress_bytes=302694400 link_bytes=302694400 token_bytes=536517320 weight_movement_bytes=34603008 total_nvlink_bytes=571120328 weight_to_token_ratio=0.064496 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.139014658e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77254 payload_rows_remote=66720 dup_rows=21050 dup_rows_remote=17510 dedup_ratio=0.214132 router_tokens_per_expert=[220, 813, 455, 340, 92, 908, 929, 242, 641, 311, 823, 1257, 1344, 1342, 814, 2086, 890, 1256, 255, 715, 73, 408, 1197, 1033, 293, 836, 798, 533, 290, 640, 1023, 334, 337, 977, 759, 671, 1718, 1632, 2080, 553, 538, 197, 456, 284, 1635, 1431, 165, 1568, 1139, 643, 855, 887, 74, 260, 326, 767, 942, 419, 67, 697, 233, 1281, 502, 1868] payload_rows_per_dst=[9238, 8840, 9668, 8336, 11370, 9014, 10534, 10254] dup_rows_per_dst=[2920, 1540, 1986, 1158, 4608, 3534, 3540, 1764] combine_rows_per_src=[9238, 8840, 9668, 8336, 11370, 9014, 10534, 10254] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76984 payload_rows_remote=68966 dup_rows=21320 dup_rows_remote=19072 dedup_ratio=0.216878 router_tokens_per_expert=[286, 1012, 380, 442, 90, 966, 869, 241, 699, 324, 898, 1337, 1272, 1519, 880, 1914, 884, 1268, 266, 784, 73, 361, 1250, 968, 290, 825, 772, 429, 409, 643, 931, 322, 351, 1002, 699, 614, 1580, 1510, 2005, 512, 679, 199, 332, 269, 1593, 1540, 195, 1526, 1145, 689, 916, 757, 85, 308, 356, 850, 918, 313, 84, 690, 222, 1276, 536, 1797] payload_rows_per_dst=[9406, 10616, 9704, 9666, 8018, 8768, 10702, 10104] dup_rows_per_dst=[3176, 3084, 2004, 2004, 2248, 3898, 3338, 1568] combine_rows_per_src=[9406, 10616, 9704, 9666, 8018, 8768, 10702, 10104] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76074 payload_rows_remote=66260 dup_rows=22230 dup_rows_remote=19562 dedup_ratio=0.226135 router_tokens_per_expert=[92, 1114, 368, 446, 24, 971, 1036, 219, 574, 284, 1201, 1451, 1113, 1052, 298, 1762, 900, 1372, 555, 670, 112, 240, 1233, 1159, 495, 852, 1043, 643, 422, 624, 1042, 184, 438, 958, 741, 401, 2196, 1579, 1683, 496, 866, 162, 378, 397, 1482, 1626, 162, 1592, 1098, 424, 476, 965, 57, 159, 283, 661, 995, 310, 129, 641, 124, 1623, 370, 2129] payload_rows_per_dst=[9144, 9390, 9814, 11434, 7414, 9276, 9282, 10320] dup_rows_per_dst=[2762, 2600, 2668, 3568, 1812, 4054, 2444, 2322] combine_rows_per_src=[9144, 9390, 9814, 11434, 7414, 9276, 9282, 10320] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77098 payload_rows_remote=67400 dup_rows=21206 dup_rows_remote=17164 dedup_ratio=0.215719 router_tokens_per_expert=[238, 848, 378, 419, 91, 851, 1095, 224, 566, 211, 906, 1093, 1099, 1636, 839, 2081, 921, 1323, 254, 689, 54, 403, 1065, 970, 265, 976, 867, 543, 297, 619, 851, 310, 397, 1028, 712, 626, 1758, 1381, 2048, 510, 689, 202, 464, 272, 1587, 1571, 210, 1875, 1266, 702, 748, 854, 96, 286, 332, 769, 795, 291, 98, 870, 220, 1172, 591, 1750] payload_rows_per_dst=[9346, 8116, 9488, 8236, 11480, 9698, 10704, 10030] dup_rows_per_dst=[3038, 1312, 1870, 1220, 4616, 4042, 3564, 1544] combine_rows_per_src=[9346, 8116, 9488, 8236, 11480, 9698, 10704, 10030] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75694 payload_rows_remote=66424 dup_rows=22610 dup_rows_remote=19622 dedup_ratio=0.230001 router_tokens_per_expert=[90, 1140, 533, 403, 29, 1158, 866, 249, 603, 387, 991, 1293, 1576, 867, 236, 1290, 1004, 1449, 454, 729, 96, 151, 1053, 1112, 651, 1026, 1043, 773, 421, 562, 892, 208, 406, 1173, 905, 472, 2172, 1937, 1661, 433, 920, 151, 341, 351, 1396, 1774, 189, 1370, 998, 203, 657, 889, 59, 216, 292, 682, 1071, 441, 88, 453, 151, 1560, 216, 2190] payload_rows_per_dst=[9270, 10424, 11610, 11486, 8292, 8020, 6534, 10058] dup_rows_per_dst=[2988, 4062, 4034, 4010, 2360, 1416, 1458, 2282] combine_rows_per_src=[9270, 10424, 11610, 11486, 8292, 8020, 6534, 10058] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76426 payload_rows_remote=66714 dup_rows=21878 dup_rows_remote=19300 dedup_ratio=0.222555 router_tokens_per_expert=[801, 1046, 904, 356, 896, 261, 207, 1128, 73, 867, 1496, 378, 507, 764, 848, 1407, 798, 1350, 778, 439, 1422, 297, 1929, 1332, 753, 770, 791, 883, 224, 351, 197, 96, 433, 386, 402, 187, 968, 1332, 1144, 1004, 2120, 1533, 891, 1212, 464, 1438, 50, 254, 1083, 1083, 950, 12, 148, 1603, 199, 1301, 849, 343, 827, 503, 1046, 331, 365, 342] payload_rows_per_dst=[9010, 9662, 9712, 9772, 9728, 8376, 10170, 9996] dup_rows_per_dst=[2730, 3018, 2578, 2598, 1984, 3308, 2588, 3074] combine_rows_per_src=[9010, 9662, 9712, 9772, 9728, 8376, 10170, 9996] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76838 payload_rows_remote=66368 dup_rows=21466 dup_rows_remote=18772 dedup_ratio=0.218363 router_tokens_per_expert=[242, 952, 404, 454, 121, 1055, 994, 228, 771, 326, 933, 1260, 1230, 1406, 839, 1968, 865, 1211, 250, 755, 70, 384, 1150, 898, 370, 852, 775, 508, 278, 727, 996, 329, 381, 1082, 740, 700, 1747, 1586, 2055, 498, 640, 140, 324, 305, 1556, 1607, 148, 1519, 1055, 646, 814, 873, 71, 309, 293, 666, 984, 382, 69, 701, 153, 1289, 475, 1743] payload_rows_per_dst=[9634, 10532, 9256, 10470, 7826, 8772, 10212, 10136] dup_rows_per_dst=[3376, 2998, 1910, 2694, 2148, 3706, 3178, 1456] combine_rows_per_src=[9634, 10532, 9256, 10470, 7826, 8772, 10212, 10136] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77254 payload_rows_remote=66720 dup_rows=21050 dup_rows_remote=17510 dedup_ratio=0.214132 router_tokens_per_expert=[220, 813, 455, 340, 92, 908, 929, 242, 641, 311, 823, 1257, 1344, 1342, 814, 2086, 890, 1256, 255, 715, 73, 408, 1197, 1033, 293, 836, 798, 533, 290, 640, 1023, 334, 337, 977, 759, 671, 1718, 1632, 2080, 553, 538, 197, 456, 284, 1635, 1431, 165, 1568, 1139, 643, 855, 887, 74, 260, 326, 767, 942, 419, 67, 697, 233, 1281, 502, 1868] payload_rows_per_dst=[9238, 8840, 9668, 8336, 11370, 9014, 10534, 10254] dup_rows_per_dst=[2920, 1540, 1986, 1158, 4608, 3534, 3540, 1764] combine_rows_per_src=[9238, 8840, 9668, 8336, 11370, 9014, 10534, 10254] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76088 payload_rows_remote=64740 dup_rows=22216 dup_rows_remote=19762 dedup_ratio=0.225993 router_tokens_per_expert=[209, 866, 514, 436, 62, 1100, 958, 247, 776, 299, 953, 1182, 1293, 1243, 571, 1825, 945, 1254, 270, 715, 94, 294, 1017, 1041, 466, 780, 876, 600, 324, 610, 1039, 295, 396, 1175, 858, 659, 1596, 1712, 2135, 472, 699, 155, 319, 255, 1583, 1665, 173, 1487, 1196, 512, 911, 898, 143, 334, 300, 774, 989, 394, 82, 469, 195, 1120, 418, 1924] payload_rows_per_dst=[9230, 7952, 9374, 8592, 11568, 7668, 10356, 11348] dup_rows_per_dst=[3018, 1250, 1886, 1388, 5460, 3330, 3430, 2454] combine_rows_per_src=[9230, 7952, 9374, 8592, 11568, 7668, 10356, 11348] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=263438336 dispatch_meta_bytes=330864 combine_payload_bytes=263438336 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=263769200 ingress_bytes=367247360 link_bytes=367247360 token_bytes=527207536 weight_movement_bytes=103809024 total_nvlink_bytes=631016560 weight_to_token_ratio=0.196904 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=10112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.381922250e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77098 payload_rows_remote=67400 dup_rows=21206 dup_rows_remote=17164 dedup_ratio=0.215719 router_tokens_per_expert=[238, 848, 378, 419, 91, 851, 1095, 224, 566, 211, 906, 1093, 1099, 1636, 839, 2081, 921, 1323, 254, 689, 54, 403, 1065, 970, 265, 976, 867, 543, 297, 619, 851, 310, 397, 1028, 712, 626, 1758, 1381, 2048, 510, 689, 202, 464, 272, 1587, 1571, 210, 1875, 1266, 702, 748, 854, 96, 286, 332, 769, 795, 291, 98, 870, 220, 1172, 591, 1750] payload_rows_per_dst=[9346, 8116, 9488, 8236, 11480, 9698, 10704, 10030] dup_rows_per_dst=[3038, 1312, 1870, 1220, 4616, 4042, 3564, 1544] combine_rows_per_src=[9346, 8116, 9488, 8236, 11480, 9698, 10704, 10030] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=263438336 dispatch_meta_bytes=330864 combine_payload_bytes=263438336 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=263769200 ingress_bytes=367247360 link_bytes=367247360 token_bytes=527207536 weight_movement_bytes=103809024 total_nvlink_bytes=631016560 weight_to_token_ratio=0.196904 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=10112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.381922250e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=272072704 dispatch_meta_bytes=344184 combine_payload_bytes=272072704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272416888 ingress_bytes=306675712 link_bytes=306675712 token_bytes=544489592 weight_movement_bytes=34603008 total_nvlink_bytes=579092600 weight_to_token_ratio=0.063551 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.153996015e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=273285120 dispatch_meta_bytes=336920 combine_payload_bytes=273285120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273622040 ingress_bytes=307888128 link_bytes=307888128 token_bytes=546907160 weight_movement_bytes=34603008 total_nvlink_bytes=581510168 weight_to_token_ratio=0.063270 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.158558239e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=282484736 dispatch_meta_bytes=352152 combine_payload_bytes=282484736 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=282836888 ingress_bytes=386293760 link_bytes=386293760 token_bytes=565321624 weight_movement_bytes=103809024 total_nvlink_bytes=669130648 weight_to_token_ratio=0.183628 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=12928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.453592320e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=271400960 dispatch_meta_bytes=343288 combine_payload_bytes=271400960 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271744248 ingress_bytes=306003968 link_bytes=306003968 token_bytes=543145208 weight_movement_bytes=34603008 total_nvlink_bytes=577748216 weight_to_token_ratio=0.063709 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=10752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.151468296e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=276070400 dispatch_meta_bytes=338256 combine_payload_bytes=276070400 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=276408656 ingress_bytes=379879424 link_bytes=379879424 token_bytes=552479056 weight_movement_bytes=103809024 total_nvlink_bytes=656288080 weight_to_token_ratio=0.187897 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.429455690e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=273260544 dispatch_meta_bytes=344056 combine_payload_bytes=273260544 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=273604600 ingress_bytes=377069568 link_bytes=377069568 token_bytes=546865144 weight_movement_bytes=103809024 total_nvlink_bytes=650674168 weight_to_token_ratio=0.189826 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.418882428e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=271843328 dispatch_meta_bytes=340560 combine_payload_bytes=271843328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272183888 ingress_bytes=306446336 link_bytes=306446336 token_bytes=544027216 weight_movement_bytes=34603008 total_nvlink_bytes=578630224 weight_to_token_ratio=0.063605 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=9344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.153132891e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=273285120 dispatch_meta_bytes=336920 combine_payload_bytes=273285120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273622040 ingress_bytes=307888128 link_bytes=307888128 token_bytes=546907160 weight_movement_bytes=34603008 total_nvlink_bytes=581510168 weight_to_token_ratio=0.063270 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.158558239e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265175040 dispatch_meta_bytes=338008 combine_payload_bytes=265175040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265513048 ingress_bytes=299778048 link_bytes=299778048 token_bytes=530688088 weight_movement_bytes=34603008 total_nvlink_bytes=565291096 weight_to_token_ratio=0.065204 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=15616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128040661e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75392 payload_rows_remote=66104 dup_rows=22912 dup_rows_remote=20656 dedup_ratio=0.233073 router_tokens_per_expert=[402, 735, 1226, 398, 448, 461, 1005, 629, 1386, 2650, 215, 371, 925, 672, 977, 411, 1796, 61, 2431, 482, 721, 389, 14, 614, 1063, 856, 702, 461, 519, 1000, 1432, 1020, 563, 715, 197, 238, 1096, 575, 1853, 689, 467, 2141, 161, 589, 363, 613, 510, 3100, 168, 342, 281, 875, 746, 973, 310, 483, 265, 514, 1506, 678, 673, 575, 224, 197] payload_rows_per_dst=[8464, 9916, 9582, 9288, 9288, 10912, 10038, 7904] dup_rows_per_dst=[2144, 2560, 3434, 2256, 2564, 4976, 3618, 1360] combine_rows_per_src=[8464, 9916, 9582, 9288, 9288, 10912, 10038, 7904] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75694 payload_rows_remote=66424 dup_rows=22610 dup_rows_remote=19622 dedup_ratio=0.230001 router_tokens_per_expert=[90, 1140, 533, 403, 29, 1158, 866, 249, 603, 387, 991, 1293, 1576, 867, 236, 1290, 1004, 1449, 454, 729, 96, 151, 1053, 1112, 651, 1026, 1043, 773, 421, 562, 892, 208, 406, 1173, 905, 472, 2172, 1937, 1661, 433, 920, 151, 341, 351, 1396, 1774, 189, 1370, 998, 203, 657, 889, 59, 216, 292, 682, 1071, 441, 88, 453, 151, 1560, 216, 2190] payload_rows_per_dst=[9270, 10424, 11610, 11486, 8292, 8020, 6534, 10058] dup_rows_per_dst=[2988, 4062, 4034, 4010, 2360, 1416, 1458, 2282] combine_rows_per_src=[9270, 10424, 11610, 11486, 8292, 8020, 6534, 10058] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=276070400 dispatch_meta_bytes=338256 combine_payload_bytes=276070400 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=276408656 ingress_bytes=379879424 link_bytes=379879424 token_bytes=552479056 weight_movement_bytes=103809024 total_nvlink_bytes=656288080 weight_to_token_ratio=0.187897 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.429455690e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75530 payload_rows_remote=64428 dup_rows=22774 dup_rows_remote=19662 dedup_ratio=0.231669 router_tokens_per_expert=[313, 604, 1486, 351, 469, 463, 1116, 909, 1642, 2864, 250, 369, 812, 615, 1121, 362, 1628, 75, 2127, 452, 988, 443, 4, 615, 1045, 807, 625, 422, 553, 1124, 1514, 885, 566, 674, 176, 221, 1076, 633, 2052, 842, 327, 2228, 176, 473, 371, 586, 401, 2935, 41, 244, 252, 1096, 730, 896, 179, 266, 208, 338, 1612, 599, 748, 702, 192, 259] payload_rows_per_dst=[10942, 11102, 9122, 8942, 9742, 10530, 7268, 7882] dup_rows_per_dst=[3508, 3112, 3542, 1980, 2738, 4464, 1996, 1434] combine_rows_per_src=[10942, 11102, 9122, 8942, 9742, 10530, 7268, 7882] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=270761984 dispatch_meta_bytes=347040 combine_payload_bytes=270761984 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=271109024 ingress_bytes=409174016 link_bytes=409174016 token_bytes=541871008 weight_movement_bytes=138412032 total_nvlink_bytes=680283040 weight_to_token_ratio=0.255434 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=8064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.539688881e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75386 payload_rows_remote=64146 dup_rows=22918 dup_rows_remote=18786 dedup_ratio=0.233134 router_tokens_per_expert=[241, 436, 1536, 349, 614, 793, 1130, 921, 2317, 3059, 359, 335, 840, 716, 1352, 257, 1980, 54, 1954, 479, 931, 496, 3, 526, 923, 940, 632, 409, 519, 991, 1666, 788, 582, 615, 199, 242, 1093, 522, 1399, 731, 355, 2503, 211, 479, 215, 572, 377, 2723, 12, 253, 279, 1035, 484, 894, 167, 203, 224, 240, 1694, 578, 591, 762, 123, 249] payload_rows_per_dst=[11240, 13058, 9264, 8624, 10874, 9164, 5532, 7630] dup_rows_per_dst=[4132, 5412, 3582, 1780, 2936, 2662, 1122, 1292] combine_rows_per_src=[11240, 13058, 9264, 8624, 10874, 9164, 5532, 7630] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=272072704 dispatch_meta_bytes=344184 combine_payload_bytes=272072704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272416888 ingress_bytes=306675712 link_bytes=306675712 token_bytes=544489592 weight_movement_bytes=34603008 total_nvlink_bytes=579092600 weight_to_token_ratio=0.063551 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.153996015e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75260 payload_rows_remote=65808 dup_rows=23044 dup_rows_remote=20464 dedup_ratio=0.234416 router_tokens_per_expert=[413, 885, 1128, 415, 470, 367, 1026, 679, 1320, 2348, 145, 360, 830, 748, 1030, 445, 1491, 96, 2125, 485, 694, 352, 27, 634, 1034, 654, 682, 486, 497, 952, 1510, 1163, 528, 755, 241, 273, 889, 537, 2043, 750, 487, 2092, 188, 573, 322, 634, 660, 3129, 349, 449, 341, 935, 865, 999, 428, 601, 246, 436, 1461, 648, 699, 605, 272, 226] payload_rows_per_dst=[8652, 8506, 8652, 10582, 9452, 8082, 10434, 10900] dup_rows_per_dst=[2114, 1250, 3156, 3374, 2580, 2344, 4196, 4030] combine_rows_per_src=[8652, 8506, 8652, 10582, 9452, 8082, 10434, 10900] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75842 payload_rows_remote=66062 dup_rows=22462 dup_rows_remote=19102 dedup_ratio=0.228495 router_tokens_per_expert=[485, 831, 1330, 403, 468, 389, 1073, 694, 1531, 2540, 229, 342, 814, 738, 1047, 365, 1421, 52, 2263, 475, 728, 401, 15, 665, 1055, 884, 674, 475, 501, 1023, 1433, 1000, 565, 653, 227, 239, 1246, 530, 1837, 761, 423, 2177, 195, 601, 386, 612, 645, 3171, 175, 262, 289, 998, 591, 936, 331, 448, 220, 465, 1310, 666, 770, 623, 231, 225] payload_rows_per_dst=[9016, 8756, 8948, 10812, 9524, 7854, 9780, 11152] dup_rows_per_dst=[2330, 1376, 3092, 3278, 2592, 2224, 3360, 4210] combine_rows_per_src=[9016, 8756, 8948, 10812, 9524, 7854, 9780, 11152] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76074 payload_rows_remote=66260 dup_rows=22230 dup_rows_remote=19562 dedup_ratio=0.226135 router_tokens_per_expert=[92, 1114, 368, 446, 24, 971, 1036, 219, 574, 284, 1201, 1451, 1113, 1052, 298, 1762, 900, 1372, 555, 670, 112, 240, 1233, 1159, 495, 852, 1043, 643, 422, 624, 1042, 184, 438, 958, 741, 401, 2196, 1579, 1683, 496, 866, 162, 378, 397, 1482, 1626, 162, 1592, 1098, 424, 476, 965, 57, 159, 283, 661, 995, 310, 129, 641, 124, 1623, 370, 2129] payload_rows_per_dst=[9144, 9390, 9814, 11434, 7414, 9276, 9282, 10320] dup_rows_per_dst=[2762, 2600, 2668, 3568, 1812, 4054, 2444, 2322] combine_rows_per_src=[9144, 9390, 9814, 11434, 7414, 9276, 9282, 10320] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75764 payload_rows_remote=64798 dup_rows=22540 dup_rows_remote=18478 dedup_ratio=0.229289 router_tokens_per_expert=[388, 758, 1364, 404, 565, 522, 1070, 709, 1607, 2746, 284, 392, 881, 707, 1053, 389, 1598, 51, 1989, 521, 826, 292, 13, 557, 1056, 859, 619, 464, 536, 929, 1587, 1007, 570, 709, 273, 247, 994, 664, 1537, 752, 577, 2444, 196, 470, 229, 640, 519, 2871, 116, 368, 401, 911, 578, 915, 369, 416, 266, 478, 1546, 599, 563, 739, 196, 256] payload_rows_per_dst=[9254, 9158, 8546, 10202, 9110, 8394, 10134, 10966] dup_rows_per_dst=[2306, 1468, 3088, 2676, 2382, 3052, 3506, 4062] combine_rows_per_src=[9254, 9158, 8546, 10202, 9110, 8394, 10134, 10966] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75392 payload_rows_remote=66104 dup_rows=22912 dup_rows_remote=20656 dedup_ratio=0.233073 router_tokens_per_expert=[402, 735, 1226, 398, 448, 461, 1005, 629, 1386, 2650, 215, 371, 925, 672, 977, 411, 1796, 61, 2431, 482, 721, 389, 14, 614, 1063, 856, 702, 461, 519, 1000, 1432, 1020, 563, 715, 197, 238, 1096, 575, 1853, 689, 467, 2141, 161, 589, 363, 613, 510, 3100, 168, 342, 281, 875, 746, 973, 310, 483, 265, 514, 1506, 678, 673, 575, 224, 197] payload_rows_per_dst=[8464, 9916, 9582, 9288, 9288, 10912, 10038, 7904] dup_rows_per_dst=[2144, 2560, 3434, 2256, 2564, 4976, 3618, 1360] combine_rows_per_src=[8464, 9916, 9582, 9288, 9288, 10912, 10038, 7904] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=263897088 dispatch_meta_bytes=336360 combine_payload_bytes=263897088 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=264233448 ingress_bytes=333103104 link_bytes=333103104 token_bytes=528130536 weight_movement_bytes=69206016 total_nvlink_bytes=597336552 weight_to_token_ratio=0.131040 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.253440163e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75260 payload_rows_remote=65808 dup_rows=23044 dup_rows_remote=20464 dedup_ratio=0.234416 router_tokens_per_expert=[413, 885, 1128, 415, 470, 367, 1026, 679, 1320, 2348, 145, 360, 830, 748, 1030, 445, 1491, 96, 2125, 485, 694, 352, 27, 634, 1034, 654, 682, 486, 497, 952, 1510, 1163, 528, 755, 241, 273, 889, 537, 2043, 750, 487, 2092, 188, 573, 322, 634, 660, 3129, 349, 449, 341, 935, 865, 999, 428, 601, 246, 436, 1461, 648, 699, 605, 272, 226] payload_rows_per_dst=[8652, 8506, 8652, 10582, 9452, 8082, 10434, 10900] dup_rows_per_dst=[2114, 1250, 3156, 3374, 2580, 2344, 4196, 4030] combine_rows_per_src=[8652, 8506, 8652, 10582, 9452, 8082, 10434, 10900] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76214 payload_rows_remote=66936 dup_rows=22090 dup_rows_remote=18872 dedup_ratio=0.224711 router_tokens_per_expert=[228, 585, 1531, 423, 560, 509, 1135, 696, 1765, 2929, 218, 383, 767, 662, 1267, 468, 1918, 75, 2007, 460, 850, 356, 14, 568, 1009, 807, 548, 429, 578, 999, 1883, 1019, 561, 742, 239, 232, 1166, 424, 1806, 817, 366, 1938, 217, 398, 236, 619, 407, 2629, 61, 429, 261, 918, 819, 901, 263, 281, 362, 465, 1466, 528, 718, 782, 188, 267] payload_rows_per_dst=[9608, 10504, 9278, 8902, 9668, 9926, 10320, 8008] dup_rows_per_dst=[2884, 3164, 3218, 1876, 2306, 3694, 3404, 1544] combine_rows_per_src=[9608, 10504, 9278, 8902, 9668, 9926, 10320, 8008] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=262742016 dispatch_meta_bytes=331728 combine_payload_bytes=262742016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263073744 ingress_bytes=297345024 link_bytes=297345024 token_bytes=525815760 weight_movement_bytes=34603008 total_nvlink_bytes=560418768 weight_to_token_ratio=0.065808 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.118885387e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75764 payload_rows_remote=64798 dup_rows=22540 dup_rows_remote=18478 dedup_ratio=0.229289 router_tokens_per_expert=[388, 758, 1364, 404, 565, 522, 1070, 709, 1607, 2746, 284, 392, 881, 707, 1053, 389, 1598, 51, 1989, 521, 826, 292, 13, 557, 1056, 859, 619, 464, 536, 929, 1587, 1007, 570, 709, 273, 247, 994, 664, 1537, 752, 577, 2444, 196, 470, 229, 640, 519, 2871, 116, 368, 401, 911, 578, 915, 369, 416, 266, 478, 1546, 599, 563, 739, 196, 256] payload_rows_per_dst=[9254, 9158, 8546, 10202, 9110, 8394, 10134, 10966] dup_rows_per_dst=[2306, 1468, 3088, 2676, 2382, 3052, 3506, 4062] combine_rows_per_src=[9254, 9158, 8546, 10202, 9110, 8394, 10134, 10966] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75536 payload_rows_remote=67714 dup_rows=22768 dup_rows_remote=20564 dedup_ratio=0.231608 router_tokens_per_expert=[433, 1045, 1190, 645, 473, 444, 959, 666, 1151, 2200, 170, 385, 849, 712, 774, 440, 1528, 67, 2201, 463, 724, 298, 19, 600, 1015, 655, 536, 480, 638, 739, 1657, 1146, 593, 703, 272, 301, 915, 531, 2177, 782, 518, 1816, 157, 456, 392, 876, 798, 2900, 246, 376, 441, 933, 740, 992, 332, 538, 270, 721, 1470, 530, 692, 739, 359, 284] payload_rows_per_dst=[9200, 7950, 8826, 10472, 9792, 7822, 9932, 11542] dup_rows_per_dst=[2510, 1012, 2974, 3260, 2756, 2204, 3664, 4388] combine_rows_per_src=[9200, 7950, 8826, 10472, 9792, 7822, 9932, 11542] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=270589952 dispatch_meta_bytes=340656 combine_payload_bytes=270589952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270930608 ingress_bytes=305192960 link_bytes=305192960 token_bytes=541520560 weight_movement_bytes=34603008 total_nvlink_bytes=576123568 weight_to_token_ratio=0.063900 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.148416538e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=274169856 dispatch_meta_bytes=343232 combine_payload_bytes=274169856 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=274513088 ingress_bytes=308772864 link_bytes=308772864 token_bytes=548682944 weight_movement_bytes=34603008 total_nvlink_bytes=583285952 weight_to_token_ratio=0.063066 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=17152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.161887429e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73334 payload_rows_remote=64550 dup_rows=24970 dup_rows_remote=22172 dedup_ratio=0.254008 router_tokens_per_expert=[2, 1733, 1537, 655, 1034, 1209, 777, 823, 407, 568, 1914, 92, 244, 325, 342, 1577, 1628, 392, 142, 118, 362, 371, 206, 193, 478, 426, 1970, 572, 1644, 1152, 1112, 81, 372, 8, 50, 557, 1419, 562, 282, 1324, 124, 351, 290, 957, 717, 1, 2249, 0, 382, 487, 249, 1000, 520, 1219, 737, 1355, 698, 887, 116, 1244, 1207, 2377, 1874, 1451] payload_rows_per_dst=[8244, 9372, 9450, 8784, 9234, 10006, 8972, 9272] dup_rows_per_dst=[3830, 3094, 2128, 2798, 3380, 2660, 2926, 4154] combine_rows_per_src=[8244, 9372, 9450, 8784, 9234, 10006, 8972, 9272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=269549568 dispatch_meta_bytes=345088 combine_payload_bytes=269549568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269894656 ingress_bytes=304152576 link_bytes=304152576 token_bytes=539444224 weight_movement_bytes=34603008 total_nvlink_bytes=574047232 weight_to_token_ratio=0.064146 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.144501657e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=270589952 dispatch_meta_bytes=340656 combine_payload_bytes=270589952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270930608 ingress_bytes=305192960 link_bytes=305192960 token_bytes=541520560 weight_movement_bytes=34603008 total_nvlink_bytes=576123568 weight_to_token_ratio=0.063900 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.148416538e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=271400960 dispatch_meta_bytes=343288 combine_payload_bytes=271400960 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271744248 ingress_bytes=306003968 link_bytes=306003968 token_bytes=543145208 weight_movement_bytes=34603008 total_nvlink_bytes=577748216 weight_to_token_ratio=0.063709 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=10752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.151468296e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265412608 dispatch_meta_bytes=333104 combine_payload_bytes=265412608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265745712 ingress_bytes=300015616 link_bytes=300015616 token_bytes=531158320 weight_movement_bytes=34603008 total_nvlink_bytes=565761328 weight_to_token_ratio=0.065146 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128934610e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=270761984 dispatch_meta_bytes=347040 combine_payload_bytes=270761984 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=271109024 ingress_bytes=409174016 link_bytes=409174016 token_bytes=541871008 weight_movement_bytes=138412032 total_nvlink_bytes=680283040 weight_to_token_ratio=0.255434 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=8064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.539688881e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277356544 dispatch_meta_bytes=353112 combine_payload_bytes=277356544 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=277709656 ingress_bytes=381165568 link_bytes=381165568 token_bytes=555066200 weight_movement_bytes=103809024 total_nvlink_bytes=658875224 weight_to_token_ratio=0.187021 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=15872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.434295346e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75386 payload_rows_remote=64146 dup_rows=22918 dup_rows_remote=18786 dedup_ratio=0.233134 router_tokens_per_expert=[241, 436, 1536, 349, 614, 793, 1130, 921, 2317, 3059, 359, 335, 840, 716, 1352, 257, 1980, 54, 1954, 479, 931, 496, 3, 526, 923, 940, 632, 409, 519, 991, 1666, 788, 582, 615, 199, 242, 1093, 522, 1399, 731, 355, 2503, 211, 479, 215, 572, 377, 2723, 12, 253, 279, 1035, 484, 894, 167, 203, 224, 240, 1694, 578, 591, 762, 123, 249] payload_rows_per_dst=[11240, 13058, 9264, 8624, 10874, 9164, 5532, 7630] dup_rows_per_dst=[4132, 5412, 3582, 1780, 2936, 2662, 1122, 1292] combine_rows_per_src=[11240, 13058, 9264, 8624, 10874, 9164, 5532, 7630] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73988 payload_rows_remote=63472 dup_rows=24316 dup_rows_remote=20058 dedup_ratio=0.247355 router_tokens_per_expert=[0, 1602, 1436, 821, 1031, 1396, 666, 1095, 415, 504, 2193, 86, 185, 274, 280, 1318, 1610, 354, 112, 98, 254, 611, 264, 269, 572, 351, 2479, 419, 2399, 842, 859, 8, 297, 1, 20, 652, 1238, 307, 115, 1254, 97, 260, 108, 1017, 653, 0, 2438, 0, 105, 811, 303, 934, 423, 1267, 856, 1302, 679, 1062, 85, 1458, 1114, 1996, 2132, 1365] payload_rows_per_dst=[9082, 10516, 10380, 10772, 8320, 8580, 9022, 7316] dup_rows_per_dst=[3808, 4258, 3486, 4634, 2652, 1018, 2980, 1480] combine_rows_per_src=[9082, 10516, 10380, 10772, 8320, 8580, 9022, 7316] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=264396800 dispatch_meta_bytes=346888 combine_payload_bytes=264396800 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=264743688 ingress_bytes=333602816 link_bytes=333602816 token_bytes=529140488 weight_movement_bytes=69206016 total_nvlink_bytes=598346504 weight_to_token_ratio=0.130789 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.255320539e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=64960 dup_rows=24012 dup_rows_remote=18918 dedup_ratio=0.244263 router_tokens_per_expert=[0, 875, 1384, 1028, 865, 1416, 615, 1262, 476, 687, 2703, 69, 138, 349, 276, 1049, 1596, 350, 36, 33, 220, 709, 362, 211, 464, 311, 2305, 354, 2989, 707, 608, 0, 337, 1, 3, 992, 1087, 220, 69, 1369, 52, 271, 14, 973, 465, 0, 3243, 0, 36, 843, 236, 918, 474, 1715, 1030, 1186, 525, 986, 63, 1559, 1098, 1789, 1646, 1505] payload_rows_per_dst=[9332, 10662, 10322, 10922, 7396, 9300, 9644, 6714] dup_rows_per_dst=[5094, 4124, 3300, 4986, 1406, 736, 3232, 1134] combine_rows_per_src=[9332, 10662, 10322, 10922, 7396, 9300, 9644, 6714] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=262742016 dispatch_meta_bytes=331728 combine_payload_bytes=262742016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263073744 ingress_bytes=297345024 link_bytes=297345024 token_bytes=525815760 weight_movement_bytes=34603008 total_nvlink_bytes=560418768 weight_to_token_ratio=0.065808 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.118885387e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73562 payload_rows_remote=64548 dup_rows=24742 dup_rows_remote=21736 dedup_ratio=0.251689 router_tokens_per_expert=[3, 1883, 1557, 568, 1119, 1131, 821, 867, 235, 536, 1896, 93, 269, 323, 339, 1445, 1467, 397, 179, 148, 286, 281, 206, 356, 691, 418, 1905, 615, 1789, 918, 1110, 91, 349, 22, 84, 569, 1459, 462, 204, 1399, 172, 264, 352, 1020, 764, 6, 2378, 9, 361, 520, 270, 889, 535, 1243, 647, 1335, 878, 725, 164, 1236, 1138, 2378, 2001, 1377] payload_rows_per_dst=[8184, 8314, 9360, 8680, 9448, 10524, 9014, 10038] dup_rows_per_dst=[3528, 1958, 2036, 2816, 3414, 2984, 3006, 5000] combine_rows_per_src=[8184, 8314, 9360, 8680, 9448, 10524, 9014, 10038] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=259981312 dispatch_meta_bytes=334120 combine_payload_bytes=259981312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260315432 ingress_bytes=294584320 link_bytes=294584320 token_bytes=520296744 weight_movement_bytes=34603008 total_nvlink_bytes=554899752 weight_to_token_ratio=0.066506 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.108497080e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73664 payload_rows_remote=63988 dup_rows=24640 dup_rows_remote=21446 dedup_ratio=0.250651 router_tokens_per_expert=[27, 1561, 1393, 512, 1027, 2227, 691, 969, 484, 506, 2079, 76, 186, 433, 380, 1147, 1622, 325, 192, 160, 502, 393, 281, 241, 439, 338, 1963, 508, 2141, 1060, 975, 55, 378, 47, 28, 741, 1096, 548, 178, 1352, 127, 508, 507, 1032, 541, 27, 2163, 44, 194, 640, 239, 909, 444, 1372, 833, 1118, 622, 915, 135, 1225, 1111, 2016, 1747, 1422] payload_rows_per_dst=[9272, 10168, 9676, 8320, 9112, 10860, 8636, 7620] dup_rows_per_dst=[4420, 3908, 3194, 2356, 2746, 3320, 2862, 1834] combine_rows_per_src=[9272, 10168, 9676, 8320, 9112, 10860, 8636, 7620] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73014 payload_rows_remote=63030 dup_rows=25290 dup_rows_remote=21328 dedup_ratio=0.257263 router_tokens_per_expert=[10, 1862, 1492, 510, 1107, 1304, 944, 649, 362, 431, 1623, 131, 274, 323, 365, 1599, 1504, 429, 203, 169, 409, 318, 215, 326, 497, 413, 1503, 754, 1355, 1301, 1033, 115, 548, 81, 94, 393, 1526, 715, 267, 1487, 208, 392, 662, 1101, 821, 29, 1887, 18, 529, 381, 249, 1193, 530, 1052, 730, 1340, 861, 775, 203, 1087, 1040, 2329, 1738, 1356] payload_rows_per_dst=[8202, 8086, 9448, 8462, 9984, 10076, 9116, 9640] dup_rows_per_dst=[3830, 2130, 2356, 2770, 3962, 2870, 2892, 4480] combine_rows_per_src=[8202, 8086, 9448, 8462, 9984, 10076, 9116, 9640] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76214 payload_rows_remote=66936 dup_rows=22090 dup_rows_remote=18872 dedup_ratio=0.224711 router_tokens_per_expert=[228, 585, 1531, 423, 560, 509, 1135, 696, 1765, 2929, 218, 383, 767, 662, 1267, 468, 1918, 75, 2007, 460, 850, 356, 14, 568, 1009, 807, 548, 429, 578, 999, 1883, 1019, 561, 742, 239, 232, 1166, 424, 1806, 817, 366, 1938, 217, 398, 236, 619, 407, 2629, 61, 429, 261, 918, 819, 901, 263, 281, 362, 465, 1466, 528, 718, 782, 188, 267] payload_rows_per_dst=[9608, 10504, 9278, 8902, 9668, 9926, 10320, 8008] dup_rows_per_dst=[2884, 3164, 3218, 1876, 2306, 3694, 3404, 1544] combine_rows_per_src=[9608, 10504, 9278, 8902, 9668, 9926, 10320, 8008] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=259981312 dispatch_meta_bytes=334120 combine_payload_bytes=259981312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260315432 ingress_bytes=294584320 link_bytes=294584320 token_bytes=520296744 weight_movement_bytes=34603008 total_nvlink_bytes=554899752 weight_to_token_ratio=0.066506 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.108497080e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73998 payload_rows_remote=63732 dup_rows=24306 dup_rows_remote=18812 dedup_ratio=0.247253 router_tokens_per_expert=[3, 1598, 1591, 720, 986, 1073, 717, 1011, 309, 713, 2223, 62, 164, 414, 360, 1228, 1532, 371, 218, 95, 360, 478, 268, 273, 691, 402, 1766, 586, 1805, 984, 809, 36, 474, 14, 32, 622, 1384, 587, 177, 1432, 76, 307, 331, 1031, 604, 0, 2257, 5, 175, 608, 232, 919, 457, 1411, 731, 1230, 865, 985, 131, 1177, 1056, 2330, 1927, 1739] payload_rows_per_dst=[7716, 8602, 9504, 8128, 9302, 10316, 10164, 10266] dup_rows_per_dst=[1304, 2344, 2346, 2420, 3338, 2516, 4544, 5494] combine_rows_per_src=[7716, 8602, 9504, 8128, 9302, 10316, 10164, 10266] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73182 payload_rows_remote=63410 dup_rows=25122 dup_rows_remote=22356 dedup_ratio=0.255554 router_tokens_per_expert=[13, 1941, 1445, 499, 996, 1485, 958, 833, 405, 321, 1698, 169, 361, 291, 432, 1707, 1328, 413, 202, 233, 432, 308, 227, 242, 549, 333, 1466, 702, 1272, 1366, 1157, 155, 480, 81, 143, 552, 1591, 913, 360, 1244, 135, 341, 834, 925, 962, 26, 1741, 33, 529, 363, 279, 1151, 511, 997, 613, 1204, 865, 750, 203, 950, 1006, 2274, 1723, 1434] payload_rows_per_dst=[8564, 8552, 9158, 8458, 10370, 9772, 8748, 9560] dup_rows_per_dst=[3894, 2216, 2160, 2998, 4240, 2766, 2546, 4302] combine_rows_per_src=[8564, 8552, 9158, 8458, 10370, 9772, 8748, 9560] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=264388608 dispatch_meta_bytes=345136 combine_payload_bytes=264388608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264733744 ingress_bytes=298991616 link_bytes=298991616 token_bytes=529122352 weight_movement_bytes=34603008 total_nvlink_bytes=563725360 weight_to_token_ratio=0.065397 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=10368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125081380e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266076160 dispatch_meta_bytes=335512 combine_payload_bytes=266076160 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266411672 ingress_bytes=369885184 link_bytes=369885184 token_bytes=532487832 weight_movement_bytes=103809024 total_nvlink_bytes=636296856 weight_to_token_ratio=0.194951 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391848169e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73334 payload_rows_remote=64550 dup_rows=24970 dup_rows_remote=22172 dedup_ratio=0.254008 router_tokens_per_expert=[2, 1733, 1537, 655, 1034, 1209, 777, 823, 407, 568, 1914, 92, 244, 325, 342, 1577, 1628, 392, 142, 118, 362, 371, 206, 193, 478, 426, 1970, 572, 1644, 1152, 1112, 81, 372, 8, 50, 557, 1419, 562, 282, 1324, 124, 351, 290, 957, 717, 1, 2249, 0, 382, 487, 249, 1000, 520, 1219, 737, 1355, 698, 887, 116, 1244, 1207, 2377, 1874, 1451] payload_rows_per_dst=[8244, 9372, 9450, 8784, 9234, 10006, 8972, 9272] dup_rows_per_dst=[3830, 3094, 2128, 2798, 3380, 2660, 2926, 4154] combine_rows_per_src=[8244, 9372, 9450, 8784, 9234, 10006, 8972, 9272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73182 payload_rows_remote=63410 dup_rows=25122 dup_rows_remote=22356 dedup_ratio=0.255554 router_tokens_per_expert=[13, 1941, 1445, 499, 996, 1485, 958, 833, 405, 321, 1698, 169, 361, 291, 432, 1707, 1328, 413, 202, 233, 432, 308, 227, 242, 549, 333, 1466, 702, 1272, 1366, 1157, 155, 480, 81, 143, 552, 1591, 913, 360, 1244, 135, 341, 834, 925, 962, 26, 1741, 33, 529, 363, 279, 1151, 511, 997, 613, 1204, 865, 750, 203, 950, 1006, 2274, 1723, 1434] payload_rows_per_dst=[8564, 8552, 9158, 8458, 10370, 9772, 8748, 9560] dup_rows_per_dst=[3894, 2216, 2160, 2998, 4240, 2766, 2546, 4302] combine_rows_per_src=[8564, 8552, 9158, 8458, 10370, 9772, 8748, 9560] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 2, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=262094848 dispatch_meta_bytes=341736 combine_payload_bytes=262094848 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=262436584 ingress_bytes=331300864 link_bytes=331300864 token_bytes=524531432 weight_movement_bytes=69206016 total_nvlink_bytes=593737448 weight_to_token_ratio=0.131939 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=17536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.246658479e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78018 payload_rows_remote=66910 dup_rows=20286 dup_rows_remote=16746 dedup_ratio=0.206360 router_tokens_per_expert=[689, 1638, 987, 219, 238, 278, 3055, 388, 23, 0, 802, 911, 1459, 0, 1, 1064, 881, 877, 3527, 103, 455, 586, 244, 2, 102, 2142, 854, 124, 300, 198, 3162, 442, 1402, 544, 250, 1557, 559, 314, 1486, 842, 220, 0, 123, 1722, 105, 664, 1613, 581, 978, 1236, 1245, 831, 133, 695, 0, 1306, 956, 451, 489, 590, 308, 554, 1010, 637] payload_rows_per_dst=[8438, 10524, 10880, 11108, 8950, 9616, 10090, 8412] dup_rows_per_dst=[1716, 2826, 2470, 3540, 1844, 3554, 2758, 1578] combine_rows_per_src=[8438, 10524, 10880, 11108, 8950, 9616, 10090, 8412] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=264388608 dispatch_meta_bytes=345136 combine_payload_bytes=264388608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264733744 ingress_bytes=298991616 link_bytes=298991616 token_bytes=529122352 weight_movement_bytes=34603008 total_nvlink_bytes=563725360 weight_to_token_ratio=0.065397 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=10368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125081380e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=258170880 dispatch_meta_bytes=337432 combine_payload_bytes=258170880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258508312 ingress_bytes=292773888 link_bytes=292773888 token_bytes=516679192 weight_movement_bytes=34603008 total_nvlink_bytes=551282200 weight_to_token_ratio=0.066972 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=12544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.101684570e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=274169856 dispatch_meta_bytes=343232 combine_payload_bytes=274169856 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=274513088 ingress_bytes=308772864 link_bytes=308772864 token_bytes=548682944 weight_movement_bytes=34603008 total_nvlink_bytes=583285952 weight_to_token_ratio=0.063066 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=17152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.161887429e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=261046272 dispatch_meta_bytes=330176 combine_payload_bytes=261046272 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=261376448 ingress_bytes=434061312 link_bytes=434061312 token_bytes=522422720 weight_movement_bytes=173015040 total_nvlink_bytes=695437760 weight_to_token_ratio=0.331178 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=9728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.633337772e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=64960 dup_rows=24012 dup_rows_remote=18918 dedup_ratio=0.244263 router_tokens_per_expert=[0, 875, 1384, 1028, 865, 1416, 615, 1262, 476, 687, 2703, 69, 138, 349, 276, 1049, 1596, 350, 36, 33, 220, 709, 362, 211, 464, 311, 2305, 354, 2989, 707, 608, 0, 337, 1, 3, 992, 1087, 220, 69, 1369, 52, 271, 14, 973, 465, 0, 3243, 0, 36, 843, 236, 918, 474, 1715, 1030, 1186, 525, 986, 63, 1559, 1098, 1789, 1646, 1505] payload_rows_per_dst=[9332, 10662, 10322, 10922, 7396, 9300, 9644, 6714] dup_rows_per_dst=[5094, 4124, 3300, 4986, 1406, 736, 3232, 1134] combine_rows_per_src=[9332, 10662, 10322, 10922, 7396, 9300, 9644, 6714] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=264396800 dispatch_meta_bytes=346888 combine_payload_bytes=264396800 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=264743688 ingress_bytes=333602816 link_bytes=333602816 token_bytes=529140488 weight_movement_bytes=69206016 total_nvlink_bytes=598346504 weight_to_token_ratio=0.130789 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.255320539e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=259727360 dispatch_meta_bytes=343064 combine_payload_bytes=259727360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260070424 ingress_bytes=294330368 link_bytes=294330368 token_bytes=519797784 weight_movement_bytes=34603008 total_nvlink_bytes=554400792 weight_to_token_ratio=0.066570 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.107541479e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76498 payload_rows_remote=69274 dup_rows=21806 dup_rows_remote=20590 dedup_ratio=0.221822 router_tokens_per_expert=[790, 1849, 754, 211, 111, 192, 3082, 504, 7, 0, 767, 670, 1749, 0, 0, 1027, 595, 1004, 3815, 66, 522, 1049, 346, 0, 110, 2477, 993, 103, 166, 123, 3147, 378, 1699, 420, 125, 1324, 514, 317, 1276, 830, 139, 0, 35, 1684, 88, 798, 1438, 499, 1083, 1057, 1356, 1141, 147, 566, 0, 1352, 1152, 402, 153, 547, 324, 532, 841, 706] payload_rows_per_dst=[11880, 7224, 11630, 11126, 9930, 7542, 9390, 7776] dup_rows_per_dst=[4458, 1216, 3164, 3868, 3080, 1820, 2662, 1538] combine_rows_per_src=[11880, 7224, 11630, 11126, 9930, 7542, 9390, 7776] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=274063360 dispatch_meta_bytes=334624 combine_payload_bytes=274063360 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=274397984 ingress_bytes=343269376 link_bytes=343269376 token_bytes=548461344 weight_movement_bytes=69206016 total_nvlink_bytes=617667360 weight_to_token_ratio=0.126182 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.291695026e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76216 payload_rows_remote=63888 dup_rows=22088 dup_rows_remote=17584 dedup_ratio=0.224691 router_tokens_per_expert=[659, 1697, 652, 213, 111, 205, 2921, 577, 1, 0, 996, 620, 2245, 0, 0, 969, 524, 828, 3699, 31, 405, 1386, 278, 0, 104, 2626, 1305, 132, 160, 50, 3484, 355, 1695, 395, 158, 1323, 376, 296, 1143, 911, 51, 0, 22, 1164, 82, 1051, 1444, 673, 1011, 1079, 1548, 1417, 144, 549, 0, 1381, 1055, 431, 103, 435, 242, 385, 744, 611] payload_rows_per_dst=[12328, 8240, 11812, 11492, 9652, 7502, 8374, 6816] dup_rows_per_dst=[4504, 1422, 3594, 4940, 2942, 1472, 2018, 1196] combine_rows_per_src=[12328, 8240, 11812, 11492, 9652, 7502, 8374, 6816] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266076160 dispatch_meta_bytes=335512 combine_payload_bytes=266076160 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266411672 ingress_bytes=369885184 link_bytes=369885184 token_bytes=532487832 weight_movement_bytes=103809024 total_nvlink_bytes=636296856 weight_to_token_ratio=0.194951 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391848169e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77178 payload_rows_remote=67390 dup_rows=21126 dup_rows_remote=18514 dedup_ratio=0.214905 router_tokens_per_expert=[843, 1783, 1072, 204, 247, 312, 3302, 448, 62, 1, 975, 853, 1087, 0, 8, 1267, 796, 903, 2912, 64, 659, 543, 274, 0, 204, 2187, 940, 120, 291, 168, 3080, 413, 1272, 533, 211, 1455, 589, 328, 1421, 966, 369, 0, 178, 1742, 137, 648, 1306, 510, 784, 1155, 1339, 728, 165, 728, 0, 1301, 1077, 409, 413, 664, 317, 610, 1030, 749] payload_rows_per_dst=[8272, 11344, 8258, 9212, 8776, 9160, 9788, 12368] dup_rows_per_dst=[1546, 3766, 1320, 2158, 1864, 3530, 2612, 4330] combine_rows_per_src=[8272, 11344, 8258, 9212, 8776, 9160, 9788, 12368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77248 payload_rows_remote=68132 dup_rows=21056 dup_rows_remote=19006 dedup_ratio=0.214193 router_tokens_per_expert=[1038, 1565, 1183, 225, 304, 373, 2717, 399, 86, 4, 964, 910, 1230, 0, 30, 1419, 906, 940, 2886, 162, 647, 377, 209, 14, 217, 1515, 759, 221, 347, 244, 2421, 460, 1266, 586, 315, 1869, 458, 455, 1611, 892, 473, 1, 184, 2001, 214, 621, 1341, 524, 882, 1057, 1263, 614, 223, 697, 1, 1563, 881, 473, 492, 807, 260, 666, 1140, 550] payload_rows_per_dst=[8210, 10930, 10078, 8040, 9116, 10254, 9986, 10634] dup_rows_per_dst=[1964, 3790, 2204, 1238, 2050, 4202, 2614, 2994] combine_rows_per_src=[8210, 10930, 10078, 8040, 9116, 10254, 9986, 10634] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=283746304 dispatch_meta_bytes=359456 combine_payload_bytes=283746304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=284105760 ingress_bytes=318349312 link_bytes=318349312 token_bytes=567852064 weight_movement_bytes=34603008 total_nvlink_bytes=602455072 weight_to_token_ratio=0.060937 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=14464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.197922832e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76670 payload_rows_remote=65742 dup_rows=21634 dup_rows_remote=18846 dedup_ratio=0.220072 router_tokens_per_expert=[953, 2010, 824, 268, 208, 288, 2910, 522, 23, 20, 853, 651, 1812, 0, 3, 981, 837, 752, 3383, 46, 661, 792, 352, 35, 113, 2103, 1002, 247, 297, 244, 2484, 445, 1757, 445, 237, 2147, 465, 406, 1463, 723, 183, 55, 59, 1297, 172, 619, 1401, 581, 986, 990, 1463, 991, 294, 541, 38, 1584, 1004, 440, 363, 404, 249, 335, 868, 473] payload_rows_per_dst=[11518, 7694, 10928, 10618, 8766, 9546, 10380, 7220] dup_rows_per_dst=[4448, 992, 2788, 3252, 2230, 3478, 3394, 1052] combine_rows_per_src=[11518, 7694, 10928, 10618, 8766, 9546, 10380, 7220] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73664 payload_rows_remote=63988 dup_rows=24640 dup_rows_remote=21446 dedup_ratio=0.250651 router_tokens_per_expert=[27, 1561, 1393, 512, 1027, 2227, 691, 969, 484, 506, 2079, 76, 186, 433, 380, 1147, 1622, 325, 192, 160, 502, 393, 281, 241, 439, 338, 1963, 508, 2141, 1060, 975, 55, 378, 47, 28, 741, 1096, 548, 178, 1352, 127, 508, 507, 1032, 541, 27, 2163, 44, 194, 640, 239, 909, 444, 1372, 833, 1118, 622, 915, 135, 1225, 1111, 2016, 1747, 1422] payload_rows_per_dst=[9272, 10168, 9676, 8320, 9112, 10860, 8636, 7620] dup_rows_per_dst=[4420, 3908, 3194, 2356, 2746, 3320, 2862, 1834] combine_rows_per_src=[9272, 10168, 9676, 8320, 9112, 10860, 8636, 7620] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77248 payload_rows_remote=68132 dup_rows=21056 dup_rows_remote=19006 dedup_ratio=0.214193 router_tokens_per_expert=[1038, 1565, 1183, 225, 304, 373, 2717, 399, 86, 4, 964, 910, 1230, 0, 30, 1419, 906, 940, 2886, 162, 647, 377, 209, 14, 217, 1515, 759, 221, 347, 244, 2421, 460, 1266, 586, 315, 1869, 458, 455, 1611, 892, 473, 1, 184, 2001, 214, 621, 1341, 524, 882, 1057, 1263, 614, 223, 697, 1, 1563, 881, 473, 492, 807, 260, 666, 1140, 550] payload_rows_per_dst=[8210, 10930, 10078, 8040, 9116, 10254, 9986, 10634] dup_rows_per_dst=[1964, 3790, 2204, 1238, 2050, 4202, 2614, 2994] combine_rows_per_src=[8210, 10930, 10078, 8040, 9116, 10254, 9986, 10634] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76516 payload_rows_remote=63948 dup_rows=21788 dup_rows_remote=17678 dedup_ratio=0.221639 router_tokens_per_expert=[788, 1543, 840, 228, 193, 154, 3016, 456, 20, 0, 912, 667, 1428, 1, 5, 794, 793, 846, 3417, 105, 513, 638, 299, 10, 135, 2281, 767, 133, 292, 235, 3437, 503, 1665, 572, 170, 1812, 511, 315, 1300, 1056, 117, 3, 124, 1497, 144, 635, 1596, 592, 845, 958, 1376, 1132, 177, 655, 4, 1545, 1075, 473, 486, 599, 277, 479, 866, 647] payload_rows_per_dst=[7086, 10888, 5998, 7890, 12238, 9628, 10220, 12568] dup_rows_per_dst=[1318, 2798, 410, 1050, 5526, 3412, 3164, 4110] combine_rows_per_src=[7086, 10888, 5998, 7890, 12238, 9628, 10220, 12568] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=276029440 dispatch_meta_bytes=343616 combine_payload_bytes=276029440 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=276373056 ingress_bytes=345235456 link_bytes=345235456 token_bytes=552402496 weight_movement_bytes=69206016 total_nvlink_bytes=621608512 weight_to_token_ratio=0.125282 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=9984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.299093227e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=269279232 dispatch_meta_bytes=338352 combine_payload_bytes=269279232 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=269617584 ingress_bytes=373088256 link_bytes=373088256 token_bytes=538896816 weight_movement_bytes=103809024 total_nvlink_bytes=642705840 weight_to_token_ratio=0.192632 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=22016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.403901071e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=261685248 dispatch_meta_bytes=325888 combine_payload_bytes=261685248 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=262011136 ingress_bytes=330891264 link_bytes=330891264 token_bytes=523696384 weight_movement_bytes=69206016 total_nvlink_bytes=592902400 weight_to_token_ratio=0.132149 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.245117188e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78018 payload_rows_remote=66910 dup_rows=20286 dup_rows_remote=16746 dedup_ratio=0.206360 router_tokens_per_expert=[689, 1638, 987, 219, 238, 278, 3055, 388, 23, 0, 802, 911, 1459, 0, 1, 1064, 881, 877, 3527, 103, 455, 586, 244, 2, 102, 2142, 854, 124, 300, 198, 3162, 442, 1402, 544, 250, 1557, 559, 314, 1486, 842, 220, 0, 123, 1722, 105, 664, 1613, 581, 978, 1236, 1245, 831, 133, 695, 0, 1306, 956, 451, 489, 590, 308, 554, 1010, 637] payload_rows_per_dst=[8438, 10524, 10880, 11108, 8950, 9616, 10090, 8412] dup_rows_per_dst=[1716, 2826, 2470, 3540, 1844, 3554, 2758, 1578] combine_rows_per_src=[8438, 10524, 10880, 11108, 8950, 9616, 10090, 8412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76516 payload_rows_remote=63948 dup_rows=21788 dup_rows_remote=17678 dedup_ratio=0.221639 router_tokens_per_expert=[788, 1543, 840, 228, 193, 154, 3016, 456, 20, 0, 912, 667, 1428, 1, 5, 794, 793, 846, 3417, 105, 513, 638, 299, 10, 135, 2281, 767, 133, 292, 235, 3437, 503, 1665, 572, 170, 1812, 511, 315, 1300, 1056, 117, 3, 124, 1497, 144, 635, 1596, 592, 845, 958, 1376, 1132, 177, 655, 4, 1545, 1075, 473, 486, 599, 277, 479, 866, 647] payload_rows_per_dst=[7086, 10888, 5998, 7890, 12238, 9628, 10220, 12568] dup_rows_per_dst=[1318, 2798, 410, 1050, 5526, 3412, 3164, 4110] combine_rows_per_src=[7086, 10888, 5998, 7890, 12238, 9628, 10220, 12568] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77456 payload_rows_remote=67646 dup_rows=20848 dup_rows_remote=17070 dedup_ratio=0.212077 router_tokens_per_expert=[861, 1554, 1350, 253, 529, 358, 2956, 404, 39, 3, 900, 800, 1355, 0, 15, 1257, 1030, 876, 3085, 104, 611, 393, 279, 7, 283, 1716, 925, 143, 360, 280, 2701, 679, 1405, 434, 214, 1761, 697, 349, 1641, 930, 420, 0, 139, 1987, 122, 527, 1374, 464, 712, 1099, 869, 511, 194, 734, 0, 1308, 862, 464, 402, 603, 381, 635, 1166, 642] payload_rows_per_dst=[8620, 11068, 10322, 7734, 9220, 9810, 8868, 11814] dup_rows_per_dst=[1998, 3582, 2448, 1038, 2120, 3778, 1986, 3898] combine_rows_per_src=[8620, 11068, 10322, 7734, 9220, 9810, 8868, 11814] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80448 payload_rows_remote=67728 dup_rows=17856 dup_rows_remote=16496 dedup_ratio=0.181641 router_tokens_per_expert=[669, 116, 0, 3790, 63, 1, 236, 62, 0, 431, 1316, 814, 1011, 638, 511, 1269, 0, 0, 676, 0, 33, 559, 419, 2419, 121, 89, 727, 0, 290, 560, 0, 0, 62, 1533, 237, 10, 0, 0, 0, 2702, 1588, 189, 185, 2090, 5253, 124, 1531, 2791, 447, 173, 0, 1117, 294, 849, 6068, 0, 4316, 0, 0, 16, 414, 0, 289, 54] payload_rows_per_dst=[8908, 9310, 10982, 12720, 13824, 8804, 5604, 10296] dup_rows_per_dst=[966, 2670, 2812, 1360, 7400, 1960, 156, 532] combine_rows_per_src=[8908, 9310, 10982, 12720, 13824, 8804, 5604, 10296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=276029440 dispatch_meta_bytes=343616 combine_payload_bytes=276029440 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=276373056 ingress_bytes=345235456 link_bytes=345235456 token_bytes=552402496 weight_movement_bytes=69206016 total_nvlink_bytes=621608512 weight_to_token_ratio=0.125282 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=9984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.299093227e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279068672 dispatch_meta_bytes=348552 combine_payload_bytes=279068672 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=279417224 ingress_bytes=348274688 link_bytes=348274688 token_bytes=558485896 weight_movement_bytes=69206016 total_nvlink_bytes=627691912 weight_to_token_ratio=0.123917 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.310529613e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=261931008 dispatch_meta_bytes=326504 combine_payload_bytes=261931008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262257512 ingress_bytes=296534016 link_bytes=296534016 token_bytes=524188520 weight_movement_bytes=34603008 total_nvlink_bytes=558791528 weight_to_token_ratio=0.066013 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.115833629e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76216 payload_rows_remote=63888 dup_rows=22088 dup_rows_remote=17584 dedup_ratio=0.224691 router_tokens_per_expert=[659, 1697, 652, 213, 111, 205, 2921, 577, 1, 0, 996, 620, 2245, 0, 0, 969, 524, 828, 3699, 31, 405, 1386, 278, 0, 104, 2626, 1305, 132, 160, 50, 3484, 355, 1695, 395, 158, 1323, 376, 296, 1143, 911, 51, 0, 22, 1164, 82, 1051, 1444, 673, 1011, 1079, 1548, 1417, 144, 549, 0, 1381, 1055, 431, 103, 435, 242, 385, 744, 611] payload_rows_per_dst=[12328, 8240, 11812, 11492, 9652, 7502, 8374, 6816] dup_rows_per_dst=[4504, 1422, 3594, 4940, 2942, 1472, 2018, 1196] combine_rows_per_src=[12328, 8240, 11812, 11492, 9652, 7502, 8374, 6816] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80444 payload_rows_remote=70224 dup_rows=17860 dup_rows_remote=13920 dedup_ratio=0.181681 router_tokens_per_expert=[655, 98, 0, 3724, 98, 0, 179, 82, 0, 355, 1249, 654, 989, 518, 421, 1537, 0, 0, 594, 0, 3, 484, 432, 2379, 106, 61, 484, 0, 127, 481, 0, 0, 83, 1635, 176, 3, 0, 0, 0, 3344, 1348, 171, 176, 1953, 5371, 215, 1594, 3541, 382, 149, 0, 1019, 208, 1054, 6192, 0, 4360, 0, 0, 1, 81, 0, 322, 64] payload_rows_per_dst=[13408, 10220, 11512, 12472, 8888, 8970, 5454, 9520] dup_rows_per_dst=[5934, 3940, 3354, 788, 1594, 1944, 170, 136] combine_rows_per_src=[13408, 10220, 11512, 12472, 8888, 8970, 5454, 9520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=262094848 dispatch_meta_bytes=341736 combine_payload_bytes=262094848 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=262436584 ingress_bytes=331300864 link_bytes=331300864 token_bytes=524531432 weight_movement_bytes=69206016 total_nvlink_bytes=593737448 weight_to_token_ratio=0.131939 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=17536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.246658479e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=274063360 dispatch_meta_bytes=334624 combine_payload_bytes=274063360 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=274397984 ingress_bytes=343269376 link_bytes=343269376 token_bytes=548461344 weight_movement_bytes=69206016 total_nvlink_bytes=617667360 weight_to_token_ratio=0.126182 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.291695026e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=261931008 dispatch_meta_bytes=326504 combine_payload_bytes=261931008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262257512 ingress_bytes=296534016 link_bytes=296534016 token_bytes=524188520 weight_movement_bytes=34603008 total_nvlink_bytes=558791528 weight_to_token_ratio=0.066013 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.115833629e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277078016 dispatch_meta_bytes=338864 combine_payload_bytes=277078016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277416880 ingress_bytes=311681024 link_bytes=311681024 token_bytes=554494896 weight_movement_bytes=34603008 total_nvlink_bytes=589097904 weight_to_token_ratio=0.062405 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.172830601e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277413888 dispatch_meta_bytes=336896 combine_payload_bytes=277413888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277750784 ingress_bytes=312016896 link_bytes=312016896 token_bytes=555164672 weight_movement_bytes=34603008 total_nvlink_bytes=589767680 weight_to_token_ratio=0.062329 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=35840 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.174094460e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=261685248 dispatch_meta_bytes=325888 combine_payload_bytes=261685248 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=262011136 ingress_bytes=330891264 link_bytes=330891264 token_bytes=523696384 weight_movement_bytes=69206016 total_nvlink_bytes=592902400 weight_to_token_ratio=0.132149 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.245117188e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80444 payload_rows_remote=70224 dup_rows=17860 dup_rows_remote=13920 dedup_ratio=0.181681 router_tokens_per_expert=[655, 98, 0, 3724, 98, 0, 179, 82, 0, 355, 1249, 654, 989, 518, 421, 1537, 0, 0, 594, 0, 3, 484, 432, 2379, 106, 61, 484, 0, 127, 481, 0, 0, 83, 1635, 176, 3, 0, 0, 0, 3344, 1348, 171, 176, 1953, 5371, 215, 1594, 3541, 382, 149, 0, 1019, 208, 1054, 6192, 0, 4360, 0, 0, 1, 81, 0, 322, 64] payload_rows_per_dst=[13408, 10220, 11512, 12472, 8888, 8970, 5454, 9520] dup_rows_per_dst=[5934, 3940, 3354, 788, 1594, 1944, 170, 136] combine_rows_per_src=[13408, 10220, 11512, 12472, 8888, 8970, 5454, 9520] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80488 payload_rows_remote=65664 dup_rows=17816 dup_rows_remote=9178 dedup_ratio=0.181234 router_tokens_per_expert=[660, 81, 0, 4339, 89, 0, 179, 69, 0, 407, 711, 517, 923, 527, 409, 630, 0, 0, 652, 0, 0, 599, 433, 1563, 130, 95, 316, 1, 124, 452, 0, 0, 69, 1585, 137, 4, 0, 0, 0, 3982, 1515, 43, 180, 2032, 5451, 184, 1418, 3926, 376, 98, 0, 1349, 280, 1332, 6314, 0, 4489, 0, 0, 0, 0, 0, 421, 61] payload_rows_per_dst=[14824, 6684, 11362, 12684, 9946, 8736, 6554, 9698] dup_rows_per_dst=[8638, 1564, 2984, 454, 1608, 2008, 316, 244] combine_rows_per_src=[14824, 6684, 11362, 12684, 9946, 8736, 6554, 9698] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82584 payload_rows_remote=67590 dup_rows=15720 dup_rows_remote=12840 dedup_ratio=0.159912 router_tokens_per_expert=[667, 105, 0, 3684, 78, 2, 239, 70, 0, 438, 1223, 833, 1052, 581, 475, 1259, 0, 0, 648, 4, 74, 520, 450, 2677, 94, 87, 643, 9, 314, 601, 0, 5, 70, 1395, 262, 23, 0, 0, 0, 2837, 1524, 203, 203, 2069, 5186, 180, 1613, 2721, 421, 244, 0, 953, 324, 879, 6116, 0, 4410, 0, 0, 7, 335, 0, 294, 51] payload_rows_per_dst=[8598, 9194, 7760, 12394, 7534, 10202, 14994, 11908] dup_rows_per_dst=[1092, 2528, 986, 1178, 1640, 2992, 2880, 2424] combine_rows_per_src=[8598, 9194, 7760, 12394, 7534, 10202, 14994, 11908] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79324 payload_rows_remote=68426 dup_rows=18980 dup_rows_remote=15642 dedup_ratio=0.193075 router_tokens_per_expert=[814, 130, 16, 3268, 113, 32, 237, 98, 26, 593, 950, 770, 895, 345, 482, 939, 1, 15, 824, 32, 100, 799, 404, 1551, 135, 81, 464, 37, 162, 502, 7, 14, 105, 1464, 155, 36, 37, 73, 40, 3192, 1769, 199, 263, 2195, 5104, 203, 1401, 3392, 324, 572, 29, 1022, 463, 1124, 5857, 111, 4194, 77, 43, 52, 402, 4, 340, 74] payload_rows_per_dst=[8366, 10196, 10898, 12088, 11888, 9304, 6678, 9906] dup_rows_per_dst=[1050, 3962, 3338, 924, 5872, 2756, 612, 466] combine_rows_per_src=[8366, 10196, 10898, 12088, 11888, 9304, 6678, 9906] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=287637504 dispatch_meta_bytes=336576 combine_payload_bytes=287637504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=287974080 ingress_bytes=322240512 link_bytes=322240512 token_bytes=575611584 weight_movement_bytes=34603008 total_nvlink_bytes=610214592 weight_to_token_ratio=0.060115 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.212565104e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80802 payload_rows_remote=72974 dup_rows=17502 dup_rows_remote=15404 dedup_ratio=0.178040 router_tokens_per_expert=[834, 156, 0, 2949, 81, 37, 259, 65, 1, 527, 1596, 1050, 1092, 526, 529, 1082, 1, 6, 734, 30, 258, 624, 464, 2794, 121, 73, 906, 109, 465, 615, 5, 9, 66, 1404, 308, 74, 1, 0, 6, 2561, 1604, 268, 257, 2120, 4726, 318, 1208, 2366, 413, 434, 4, 930, 354, 920, 5694, 0, 4301, 17, 0, 23, 412, 5, 296, 64] payload_rows_per_dst=[7600, 10194, 11142, 12044, 7828, 6328, 13680, 11986] dup_rows_per_dst=[1162, 2612, 3412, 2014, 2098, 982, 2732, 2490] combine_rows_per_src=[7600, 10194, 11142, 12044, 7828, 6328, 13680, 11986] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76054 payload_rows_remote=64446 dup_rows=22250 dup_rows_remote=19742 dedup_ratio=0.226339 router_tokens_per_expert=[670, 88, 2, 3779, 91, 0, 220, 63, 1, 434, 1338, 668, 1005, 654, 476, 967, 0, 1, 717, 1, 39, 636, 433, 1861, 129, 53, 835, 5, 232, 538, 6, 5, 63, 1545, 171, 27, 0, 0, 7, 3077, 1361, 164, 193, 2129, 5168, 194, 1456, 3093, 414, 183, 3, 1548, 309, 1111, 6057, 3, 4118, 1, 0, 14, 403, 2, 329, 62] payload_rows_per_dst=[8892, 8746, 6404, 3414, 8178, 13646, 15166, 11608] dup_rows_per_dst=[934, 2340, 972, 192, 1602, 8380, 5322, 2508] combine_rows_per_src=[8892, 8746, 6404, 3414, 8178, 13646, 15166, 11608] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=276848640 dispatch_meta_bytes=321720 combine_payload_bytes=276848640 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=277170360 ingress_bytes=415260672 link_bytes=415260672 token_bytes=554019000 weight_movement_bytes=138412032 total_nvlink_bytes=692431032 weight_to_token_ratio=0.249833 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.562592478e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80448 payload_rows_remote=67728 dup_rows=17856 dup_rows_remote=16496 dedup_ratio=0.181641 router_tokens_per_expert=[669, 116, 0, 3790, 63, 1, 236, 62, 0, 431, 1316, 814, 1011, 638, 511, 1269, 0, 0, 676, 0, 33, 559, 419, 2419, 121, 89, 727, 0, 290, 560, 0, 0, 62, 1533, 237, 10, 0, 0, 0, 2702, 1588, 189, 185, 2090, 5253, 124, 1531, 2791, 447, 173, 0, 1117, 294, 849, 6068, 0, 4316, 0, 0, 16, 414, 0, 289, 54] payload_rows_per_dst=[8908, 9310, 10982, 12720, 13824, 8804, 5604, 10296] dup_rows_per_dst=[966, 2670, 2812, 1360, 7400, 1960, 156, 532] combine_rows_per_src=[8908, 9310, 10982, 12720, 13824, 8804, 5604, 10296] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280272896 dispatch_meta_bytes=336272 combine_payload_bytes=280272896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280609168 ingress_bytes=314875904 link_bytes=314875904 token_bytes=560882064 weight_movement_bytes=34603008 total_nvlink_bytes=595485072 weight_to_token_ratio=0.061694 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.184852677e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80128 payload_rows_remote=68796 dup_rows=18176 dup_rows_remote=13912 dedup_ratio=0.184896 router_tokens_per_expert=[2000, 5245, 0, 169, 676, 101, 54, 75, 0, 317, 124, 253, 83, 302, 529, 761, 357, 820, 0, 1, 506, 99, 2424, 693, 2465, 3927, 1102, 324, 142, 0, 64, 462, 0, 248, 1666, 0, 63, 1622, 757, 129, 703, 223, 0, 88, 2836, 6243, 294, 0, 5619, 0, 1221, 608, 491, 87, 342, 79, 21, 455, 0, 0, 0, 832, 118, 332] payload_rows_per_dst=[8088, 10612, 8158, 11332, 7678, 7788, 13138, 13334] dup_rows_per_dst=[2074, 1980, 1642, 4264, 1292, 904, 3352, 2668] combine_rows_per_src=[8088, 10612, 8158, 11332, 7678, 7788, 13138, 13334] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=298901504 dispatch_meta_bytes=353512 combine_payload_bytes=298901504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=299255016 ingress_bytes=333504512 link_bytes=333504512 token_bytes=598156520 weight_movement_bytes=34603008 total_nvlink_bytes=632759528 weight_to_token_ratio=0.057849 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=24192 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.254950629e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76670 payload_rows_remote=65742 dup_rows=21634 dup_rows_remote=18846 dedup_ratio=0.220072 router_tokens_per_expert=[953, 2010, 824, 268, 208, 288, 2910, 522, 23, 20, 853, 651, 1812, 0, 3, 981, 837, 752, 3383, 46, 661, 792, 352, 35, 113, 2103, 1002, 247, 297, 244, 2484, 445, 1757, 445, 237, 2147, 465, 406, 1463, 723, 183, 55, 59, 1297, 172, 619, 1401, 581, 986, 990, 1463, 991, 294, 541, 38, 1584, 1004, 440, 363, 404, 249, 335, 868, 473] payload_rows_per_dst=[11518, 7694, 10928, 10618, 8766, 9546, 10380, 7220] dup_rows_per_dst=[4448, 992, 2788, 3252, 2230, 3478, 3394, 1052] combine_rows_per_src=[11518, 7694, 10928, 10618, 8766, 9546, 10380, 7220] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=268959744 dispatch_meta_bytes=299368 combine_payload_bytes=268959744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269259112 ingress_bytes=303562752 link_bytes=303562752 token_bytes=538218856 weight_movement_bytes=34603008 total_nvlink_bytes=572821864 weight_to_token_ratio=0.064292 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=28288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.142282197e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80748 payload_rows_remote=71764 dup_rows=17556 dup_rows_remote=15248 dedup_ratio=0.178589 router_tokens_per_expert=[712, 129, 1, 2727, 88, 9, 228, 72, 0, 466, 1983, 963, 1036, 574, 467, 1583, 0, 7, 694, 3, 237, 562, 475, 2676, 122, 63, 820, 3, 379, 547, 0, 13, 72, 1365, 348, 36, 0, 0, 0, 2381, 1544, 200, 210, 2282, 4842, 319, 1524, 2388, 339, 464, 0, 1107, 356, 873, 5854, 0, 4258, 0, 0, 23, 312, 0, 311, 105] payload_rows_per_dst=[7002, 10842, 8684, 12002, 6822, 8984, 14584, 11828] dup_rows_per_dst=[930, 3302, 1702, 1576, 1582, 2308, 3402, 2754] combine_rows_per_src=[7002, 10842, 8684, 12002, 6822, 8984, 14584, 11828] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80488 payload_rows_remote=65664 dup_rows=17816 dup_rows_remote=9178 dedup_ratio=0.181234 router_tokens_per_expert=[660, 81, 0, 4339, 89, 0, 179, 69, 0, 407, 711, 517, 923, 527, 409, 630, 0, 0, 652, 0, 0, 599, 433, 1563, 130, 95, 316, 1, 124, 452, 0, 0, 69, 1585, 137, 4, 0, 0, 0, 3982, 1515, 43, 180, 2032, 5451, 184, 1418, 3926, 376, 98, 0, 1349, 280, 1332, 6314, 0, 4489, 0, 0, 0, 0, 0, 421, 61] payload_rows_per_dst=[14824, 6684, 11362, 12684, 9946, 8736, 6554, 9698] dup_rows_per_dst=[8638, 1564, 2984, 454, 1608, 2008, 316, 244] combine_rows_per_src=[14824, 6684, 11362, 12684, 9946, 8736, 6554, 9698] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=263970816 dispatch_meta_bytes=336752 combine_payload_bytes=263970816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264307568 ingress_bytes=298573824 link_bytes=298573824 token_bytes=528278384 weight_movement_bytes=34603008 total_nvlink_bytes=562881392 weight_to_token_ratio=0.065501 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=34560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123509263e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76054 payload_rows_remote=64446 dup_rows=22250 dup_rows_remote=19742 dedup_ratio=0.226339 router_tokens_per_expert=[670, 88, 2, 3779, 91, 0, 220, 63, 1, 434, 1338, 668, 1005, 654, 476, 967, 0, 1, 717, 1, 39, 636, 433, 1861, 129, 53, 835, 5, 232, 538, 6, 5, 63, 1545, 171, 27, 0, 0, 7, 3077, 1361, 164, 193, 2129, 5168, 194, 1456, 3093, 414, 183, 3, 1548, 309, 1111, 6057, 3, 4118, 1, 0, 14, 403, 2, 329, 62] payload_rows_per_dst=[8892, 8746, 6404, 3414, 8178, 13646, 15166, 11608] dup_rows_per_dst=[934, 2340, 972, 192, 1602, 8380, 5322, 2508] combine_rows_per_src=[8892, 8746, 6404, 3414, 8178, 13646, 15166, 11608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=298901504 dispatch_meta_bytes=353512 combine_payload_bytes=298901504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=299255016 ingress_bytes=333504512 link_bytes=333504512 token_bytes=598156520 weight_movement_bytes=34603008 total_nvlink_bytes=632759528 weight_to_token_ratio=0.057849 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=24192 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.254950629e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=276848640 dispatch_meta_bytes=321720 combine_payload_bytes=276848640 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=277170360 ingress_bytes=415260672 link_bytes=415260672 token_bytes=554019000 weight_movement_bytes=138412032 total_nvlink_bytes=692431032 weight_to_token_ratio=0.249833 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.562592478e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76784 payload_rows_remote=66776 dup_rows=21520 dup_rows_remote=20078 dedup_ratio=0.218913 router_tokens_per_expert=[2319, 6171, 0, 160, 565, 51, 64, 20, 0, 162, 65, 250, 109, 312, 488, 716, 408, 1027, 0, 0, 452, 88, 1630, 636, 3670, 3623, 1202, 384, 0, 0, 78, 493, 0, 248, 1026, 0, 98, 2229, 731, 140, 695, 129, 0, 60, 2730, 6279, 144, 0, 5085, 0, 786, 709, 446, 102, 224, 55, 9, 379, 0, 0, 0, 1259, 143, 303] payload_rows_per_dst=[13708, 10008, 13172, 9708, 7920, 13842, 4256, 4170] dup_rows_per_dst=[4992, 1442, 5480, 1946, 1024, 6232, 388, 16] combine_rows_per_src=[13708, 10008, 13172, 9708, 7920, 13842, 4256, 4170] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=281788416 dispatch_meta_bytes=330832 combine_payload_bytes=281788416 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=282119248 ingress_bytes=385597440 link_bytes=385597440 token_bytes=563907664 weight_movement_bytes=103809024 total_nvlink_bytes=667716688 weight_to_token_ratio=0.184089 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=22272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.450972124e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277413888 dispatch_meta_bytes=336896 combine_payload_bytes=277413888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277750784 ingress_bytes=312016896 link_bytes=312016896 token_bytes=555164672 weight_movement_bytes=34603008 total_nvlink_bytes=589767680 weight_to_token_ratio=0.062329 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=35840 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.174094460e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=269279232 dispatch_meta_bytes=338352 combine_payload_bytes=269279232 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=269617584 ingress_bytes=373088256 link_bytes=373088256 token_bytes=538896816 weight_movement_bytes=103809024 total_nvlink_bytes=642705840 weight_to_token_ratio=0.192632 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=22016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.403901071e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=293945344 dispatch_meta_bytes=348048 combine_payload_bytes=293945344 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=294293392 ingress_bytes=432357376 link_bytes=432357376 token_bytes=588238736 weight_movement_bytes=138412032 total_nvlink_bytes=726650768 weight_to_token_ratio=0.235299 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=11776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.626925998e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=268959744 dispatch_meta_bytes=299368 combine_payload_bytes=268959744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269259112 ingress_bytes=303562752 link_bytes=303562752 token_bytes=538218856 weight_movement_bytes=34603008 total_nvlink_bytes=572821864 weight_to_token_ratio=0.064292 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=28288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.142282197e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76784 payload_rows_remote=66776 dup_rows=21520 dup_rows_remote=20078 dedup_ratio=0.218913 router_tokens_per_expert=[2319, 6171, 0, 160, 565, 51, 64, 20, 0, 162, 65, 250, 109, 312, 488, 716, 408, 1027, 0, 0, 452, 88, 1630, 636, 3670, 3623, 1202, 384, 0, 0, 78, 493, 0, 248, 1026, 0, 98, 2229, 731, 140, 695, 129, 0, 60, 2730, 6279, 144, 0, 5085, 0, 786, 709, 446, 102, 224, 55, 9, 379, 0, 0, 0, 1259, 143, 303] payload_rows_per_dst=[13708, 10008, 13172, 9708, 7920, 13842, 4256, 4170] dup_rows_per_dst=[4992, 1442, 5480, 1946, 1024, 6232, 388, 16] combine_rows_per_src=[13708, 10008, 13172, 9708, 7920, 13842, 4256, 4170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=263970816 dispatch_meta_bytes=336752 combine_payload_bytes=263970816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264307568 ingress_bytes=298573824 link_bytes=298573824 token_bytes=528278384 weight_movement_bytes=34603008 total_nvlink_bytes=562881392 weight_to_token_ratio=0.065501 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=34560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123509263e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79208 payload_rows_remote=66036 dup_rows=19096 dup_rows_remote=15760 dedup_ratio=0.194255 router_tokens_per_expert=[2094, 4933, 26, 208, 694, 112, 51, 43, 6, 276, 124, 226, 98, 314, 529, 753, 389, 725, 0, 1, 484, 79, 2732, 640, 2581, 3854, 769, 340, 167, 0, 2, 503, 0, 228, 1819, 0, 78, 1829, 834, 169, 758, 254, 0, 84, 2959, 5874, 449, 0, 5276, 0, 1325, 615, 494, 100, 369, 75, 7, 431, 0, 0, 0, 896, 134, 342] payload_rows_per_dst=[5732, 10510, 8520, 7218, 13208, 8030, 13172, 12818] dup_rows_per_dst=[724, 1850, 1580, 1506, 6572, 978, 3336, 2550] combine_rows_per_src=[5732, 10510, 8520, 7218, 13208, 8030, 13172, 12818] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77890 payload_rows_remote=67594 dup_rows=20414 dup_rows_remote=16806 dedup_ratio=0.207662 router_tokens_per_expert=[2054, 5096, 187, 258, 959, 43, 96, 96, 1, 489, 227, 432, 109, 381, 535, 564, 428, 1585, 138, 110, 464, 88, 1454, 714, 2939, 3196, 1325, 201, 297, 59, 152, 520, 56, 377, 924, 16, 79, 2234, 1220, 132, 636, 119, 56, 69, 2776, 5538, 311, 92, 4544, 158, 1164, 535, 447, 109, 296, 47, 135, 360, 100, 33, 8, 734, 296, 354] payload_rows_per_dst=[12866, 10022, 10296, 8750, 8498, 11368, 5064, 11026] dup_rows_per_dst=[4712, 1846, 3608, 2236, 1578, 3748, 448, 2238] combine_rows_per_src=[12866, 10022, 10296, 8750, 8498, 11368, 5064, 11026] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76096 payload_rows_remote=61062 dup_rows=22208 dup_rows_remote=12780 dedup_ratio=0.225911 router_tokens_per_expert=[2961, 6779, 1, 103, 552, 1, 61, 3, 0, 146, 61, 203, 89, 321, 459, 550, 403, 1366, 0, 0, 268, 75, 1131, 611, 4214, 3644, 1297, 306, 0, 0, 61, 414, 0, 300, 706, 1, 91, 2475, 947, 68, 756, 50, 0, 95, 1757, 6459, 27, 0, 4804, 0, 670, 680, 459, 79, 202, 66, 8, 291, 0, 0, 0, 1483, 243, 355] payload_rows_per_dst=[15034, 9804, 10156, 10476, 8110, 13814, 4000, 4702] dup_rows_per_dst=[9428, 1216, 3620, 2034, 1066, 4474, 312, 58] combine_rows_per_src=[15034, 9804, 10156, 10476, 8110, 13814, 4000, 4702] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=273514496 dispatch_meta_bytes=347416 combine_payload_bytes=273514496 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=273861912 ingress_bytes=342720512 link_bytes=342720512 token_bytes=547376408 weight_movement_bytes=69206016 total_nvlink_bytes=616582424 weight_to_token_ratio=0.126432 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=29824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.289629695e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80466 payload_rows_remote=71630 dup_rows=17838 dup_rows_remote=15928 dedup_ratio=0.181458 router_tokens_per_expert=[1770, 3270, 80, 248, 1033, 130, 54, 191, 202, 440, 179, 365, 108, 304, 566, 715, 384, 793, 5, 16, 727, 84, 2828, 713, 2185, 3791, 792, 286, 243, 353, 39, 528, 0, 220, 2420, 0, 81, 1514, 923, 215, 745, 254, 0, 87, 3216, 5521, 784, 1, 5047, 2, 1725, 447, 489, 127, 357, 71, 27, 432, 0, 0, 0, 540, 107, 378] payload_rows_per_dst=[5948, 11212, 9250, 11222, 8836, 8758, 13148, 12092] dup_rows_per_dst=[1064, 2128, 1850, 4170, 1910, 1416, 3382, 1918] combine_rows_per_src=[5948, 11212, 9250, 11222, 8836, 8758, 13148, 12092] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=270483456 dispatch_meta_bytes=327184 combine_payload_bytes=270483456 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=270810640 ingress_bytes=374292480 link_bytes=374292480 token_bytes=541294096 weight_movement_bytes=103809024 total_nvlink_bytes=645103120 weight_to_token_ratio=0.191779 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=27008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.408432469e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80284 payload_rows_remote=66686 dup_rows=18020 dup_rows_remote=15228 dedup_ratio=0.183309 router_tokens_per_expert=[2236, 5720, 20, 209, 702, 62, 52, 47, 0, 246, 152, 239, 131, 315, 564, 820, 402, 902, 9, 4, 472, 79, 2325, 820, 2395, 3887, 1102, 286, 35, 11, 138, 371, 5, 273, 1118, 4, 88, 1474, 1033, 113, 700, 175, 1, 65, 2415, 6432, 166, 4, 5802, 4, 1266, 526, 489, 92, 339, 52, 31, 413, 2, 1, 0, 844, 137, 335] payload_rows_per_dst=[5946, 10820, 8390, 7466, 13872, 6614, 13578, 13598] dup_rows_per_dst=[710, 1888, 1636, 1210, 5784, 438, 3562, 2792] combine_rows_per_src=[5946, 10820, 8390, 7466, 13872, 6614, 13578, 13598] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79208 payload_rows_remote=66036 dup_rows=19096 dup_rows_remote=15760 dedup_ratio=0.194255 router_tokens_per_expert=[2094, 4933, 26, 208, 694, 112, 51, 43, 6, 276, 124, 226, 98, 314, 529, 753, 389, 725, 0, 1, 484, 79, 2732, 640, 2581, 3854, 769, 340, 167, 0, 2, 503, 0, 228, 1819, 0, 78, 1829, 834, 169, 758, 254, 0, 84, 2959, 5874, 449, 0, 5276, 0, 1325, 615, 494, 100, 369, 75, 7, 431, 0, 0, 0, 896, 134, 342] payload_rows_per_dst=[5732, 10510, 8520, 7218, 13208, 8030, 13172, 12818] dup_rows_per_dst=[724, 1850, 1580, 1506, 6572, 978, 3336, 2550] combine_rows_per_src=[5732, 10510, 8520, 7218, 13208, 8030, 13172, 12818] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=276865024 dispatch_meta_bytes=337600 combine_payload_bytes=276865024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277202624 ingress_bytes=311468032 link_bytes=311468032 token_bytes=554067648 weight_movement_bytes=34603008 total_nvlink_bytes=588670656 weight_to_token_ratio=0.062453 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.172029129e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82428 payload_rows_remote=69500 dup_rows=15876 dup_rows_remote=14198 dedup_ratio=0.161499 router_tokens_per_expert=[567, 465, 3995, 3032, 1380, 0, 6004, 5289, 0, 124, 0, 393, 0, 1607, 0, 121, 333, 356, 2173, 104, 950, 44, 446, 145, 50, 61, 1238, 0, 0, 117, 0, 548, 5862, 870, 0, 63, 217, 1675, 678, 0, 365, 0, 47, 0, 0, 0, 1, 334, 2622, 0, 6, 360, 1, 1131, 113, 0, 1857, 0, 50, 1834, 1436, 88, 0, 0] payload_rows_per_dst=[9218, 9954, 7806, 12928, 13468, 12926, 7606, 8522] dup_rows_per_dst=[1670, 2526, 1296, 1678, 5262, 576, 860, 2008] combine_rows_per_src=[9218, 9954, 7806, 12928, 13468, 12926, 7606, 8522] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=293396480 dispatch_meta_bytes=350232 combine_payload_bytes=293396480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293746712 ingress_bytes=327999488 link_bytes=327999488 token_bytes=587143192 weight_movement_bytes=34603008 total_nvlink_bytes=621746200 weight_to_token_ratio=0.058935 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.234235667e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=250109952 dispatch_meta_bytes=295368 combine_payload_bytes=250109952 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=250405320 ingress_bytes=353918976 link_bytes=353918976 token_bytes=500515272 weight_movement_bytes=103809024 total_nvlink_bytes=604324296 weight_to_token_ratio=0.207404 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20096 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.331768614e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80128 payload_rows_remote=68796 dup_rows=18176 dup_rows_remote=13912 dedup_ratio=0.184896 router_tokens_per_expert=[2000, 5245, 0, 169, 676, 101, 54, 75, 0, 317, 124, 253, 83, 302, 529, 761, 357, 820, 0, 1, 506, 99, 2424, 693, 2465, 3927, 1102, 324, 142, 0, 64, 462, 0, 248, 1666, 0, 63, 1622, 757, 129, 703, 223, 0, 88, 2836, 6243, 294, 0, 5619, 0, 1221, 608, 491, 87, 342, 79, 21, 455, 0, 0, 0, 832, 118, 332] payload_rows_per_dst=[8088, 10612, 8158, 11332, 7678, 7788, 13138, 13334] dup_rows_per_dst=[2074, 1980, 1642, 4264, 1292, 904, 3352, 2668] combine_rows_per_src=[8088, 10612, 8158, 11332, 7678, 7788, 13138, 13334] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81000 payload_rows_remote=72322 dup_rows=17304 dup_rows_remote=16082 dedup_ratio=0.176025 router_tokens_per_expert=[1780, 3666, 85, 348, 931, 242, 63, 190, 138, 326, 180, 327, 104, 329, 578, 636, 416, 502, 0, 6, 709, 84, 3019, 737, 1847, 3865, 697, 267, 245, 263, 101, 520, 0, 217, 2431, 0, 91, 1419, 894, 203, 724, 396, 0, 57, 3070, 5802, 703, 0, 5385, 0, 1710, 524, 412, 114, 370, 69, 5, 336, 0, 0, 0, 506, 110, 403] payload_rows_per_dst=[6140, 11138, 9376, 10866, 8872, 8678, 13492, 12438] dup_rows_per_dst=[1138, 1828, 1570, 3900, 2084, 1222, 3676, 1886] combine_rows_per_src=[6140, 11138, 9376, 10866, 8872, 8678, 13492, 12438] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76096 payload_rows_remote=61062 dup_rows=22208 dup_rows_remote=12780 dedup_ratio=0.225911 router_tokens_per_expert=[2961, 6779, 1, 103, 552, 1, 61, 3, 0, 146, 61, 203, 89, 321, 459, 550, 403, 1366, 0, 0, 268, 75, 1131, 611, 4214, 3644, 1297, 306, 0, 0, 61, 414, 0, 300, 706, 1, 91, 2475, 947, 68, 756, 50, 0, 95, 1757, 6459, 27, 0, 4804, 0, 670, 680, 459, 79, 202, 66, 8, 291, 0, 0, 0, 1483, 243, 355] payload_rows_per_dst=[15034, 9804, 10156, 10476, 8110, 13814, 4000, 4702] dup_rows_per_dst=[9428, 1216, 3620, 2034, 1066, 4474, 312, 58] combine_rows_per_src=[15034, 9804, 10156, 10476, 8110, 13814, 4000, 4702] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=273145856 dispatch_meta_bytes=327656 combine_payload_bytes=273145856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273473512 ingress_bytes=307748864 link_bytes=307748864 token_bytes=546619368 weight_movement_bytes=34603008 total_nvlink_bytes=581222376 weight_to_token_ratio=0.063304 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=34560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.158034199e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79324 payload_rows_remote=68426 dup_rows=18980 dup_rows_remote=15642 dedup_ratio=0.193075 router_tokens_per_expert=[814, 130, 16, 3268, 113, 32, 237, 98, 26, 593, 950, 770, 895, 345, 482, 939, 1, 15, 824, 32, 100, 799, 404, 1551, 135, 81, 464, 37, 162, 502, 7, 14, 105, 1464, 155, 36, 37, 73, 40, 3192, 1769, 199, 263, 2195, 5104, 203, 1401, 3392, 324, 572, 29, 1022, 463, 1124, 5857, 111, 4194, 77, 43, 52, 402, 4, 340, 74] payload_rows_per_dst=[8366, 10196, 10898, 12088, 11888, 9304, 6678, 9906] dup_rows_per_dst=[1050, 3962, 3338, 924, 5872, 2756, 612, 466] combine_rows_per_src=[8366, 10196, 10898, 12088, 11888, 9304, 6678, 9906] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=293396480 dispatch_meta_bytes=350232 combine_payload_bytes=293396480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293746712 ingress_bytes=327999488 link_bytes=327999488 token_bytes=587143192 weight_movement_bytes=34603008 total_nvlink_bytes=621746200 weight_to_token_ratio=0.058935 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.234235667e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80284 payload_rows_remote=66686 dup_rows=18020 dup_rows_remote=15228 dedup_ratio=0.183309 router_tokens_per_expert=[2236, 5720, 20, 209, 702, 62, 52, 47, 0, 246, 152, 239, 131, 315, 564, 820, 402, 902, 9, 4, 472, 79, 2325, 820, 2395, 3887, 1102, 286, 35, 11, 138, 371, 5, 273, 1118, 4, 88, 1474, 1033, 113, 700, 175, 1, 65, 2415, 6432, 166, 4, 5802, 4, 1266, 526, 489, 92, 339, 52, 31, 413, 2, 1, 0, 844, 137, 335] payload_rows_per_dst=[5946, 10820, 8390, 7466, 13872, 6614, 13578, 13598] dup_rows_per_dst=[710, 1888, 1636, 1210, 5784, 438, 3562, 2792] combine_rows_per_src=[5946, 10820, 8390, 7466, 13872, 6614, 13578, 13598] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=270483456 dispatch_meta_bytes=327184 combine_payload_bytes=270483456 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=270810640 ingress_bytes=374292480 link_bytes=374292480 token_bytes=541294096 weight_movement_bytes=103809024 total_nvlink_bytes=645103120 weight_to_token_ratio=0.191779 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=27008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.408432469e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=281788416 dispatch_meta_bytes=330832 combine_payload_bytes=281788416 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=282119248 ingress_bytes=385597440 link_bytes=385597440 token_bytes=563907664 weight_movement_bytes=103809024 total_nvlink_bytes=667716688 weight_to_token_ratio=0.184089 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=22272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.450972124e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75104 payload_rows_remote=65574 dup_rows=23200 dup_rows_remote=20806 dedup_ratio=0.236003 router_tokens_per_expert=[573, 389, 3674, 2819, 1452, 0, 6228, 4949, 0, 195, 0, 440, 0, 1511, 0, 142, 298, 333, 2460, 115, 936, 65, 382, 135, 62, 97, 1056, 0, 0, 162, 0, 565, 5854, 852, 0, 98, 270, 2164, 671, 0, 410, 0, 17, 0, 0, 0, 0, 312, 2363, 0, 4, 183, 0, 1111, 140, 0, 1250, 0, 16, 2265, 2024, 110, 0, 0] payload_rows_per_dst=[11720, 9530, 15274, 7096, 6032, 9436, 6974, 9042] dup_rows_per_dst=[8336, 2394, 5882, 1106, 2078, 488, 628, 2288] combine_rows_per_src=[11720, 9530, 15274, 7096, 6032, 9436, 6974, 9042] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=296230912 dispatch_meta_bytes=353616 combine_payload_bytes=296230912 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=296584528 ingress_bytes=365436928 link_bytes=365436928 token_bytes=592815440 weight_movement_bytes=69206016 total_nvlink_bytes=662021456 weight_to_token_ratio=0.116741 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=21760 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.375109740e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=250109952 dispatch_meta_bytes=295368 combine_payload_bytes=250109952 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=250405320 ingress_bytes=353918976 link_bytes=353918976 token_bytes=500515272 weight_movement_bytes=103809024 total_nvlink_bytes=604324296 weight_to_token_ratio=0.207404 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20096 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.331768614e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=284672000 dispatch_meta_bytes=334792 combine_payload_bytes=284672000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=285006792 ingress_bytes=319275008 link_bytes=319275008 token_bytes=569678792 weight_movement_bytes=34603008 total_nvlink_bytes=604281800 weight_to_token_ratio=0.060741 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=33024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.201406151e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280272896 dispatch_meta_bytes=336272 combine_payload_bytes=280272896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280609168 ingress_bytes=314875904 link_bytes=314875904 token_bytes=560882064 weight_movement_bytes=34603008 total_nvlink_bytes=595485072 weight_to_token_ratio=0.061694 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.184852677e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=273145856 dispatch_meta_bytes=327656 combine_payload_bytes=273145856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273473512 ingress_bytes=307748864 link_bytes=307748864 token_bytes=546619368 weight_movement_bytes=34603008 total_nvlink_bytes=581222376 weight_to_token_ratio=0.063304 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=34560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.158034199e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78774 payload_rows_remote=64538 dup_rows=19530 dup_rows_remote=13892 dedup_ratio=0.198669 router_tokens_per_expert=[574, 548, 4246, 3179, 1168, 0, 6044, 5277, 0, 156, 0, 420, 0, 1617, 0, 118, 336, 374, 2050, 108, 986, 27, 417, 144, 44, 78, 1220, 0, 1, 129, 0, 586, 5865, 945, 0, 78, 186, 1549, 664, 0, 392, 1, 69, 0, 0, 0, 7, 378, 2446, 0, 4, 463, 1, 1045, 113, 0, 1934, 0, 59, 1546, 1462, 98, 0, 0] payload_rows_per_dst=[4076, 10412, 7632, 12958, 5008, 13166, 14236, 11286] dup_rows_per_dst=[504, 2702, 1252, 1712, 2326, 616, 5638, 4780] combine_rows_per_src=[4076, 10412, 7632, 12958, 5008, 13166, 14236, 11286] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81396 payload_rows_remote=71124 dup_rows=16908 dup_rows_remote=14522 dedup_ratio=0.171997 router_tokens_per_expert=[539, 497, 4026, 2858, 822, 105, 5859, 3895, 62, 167, 90, 424, 192, 1750, 21, 270, 303, 292, 1936, 87, 684, 84, 440, 166, 62, 115, 1026, 25, 15, 133, 185, 628, 5606, 701, 51, 130, 479, 2113, 661, 40, 425, 39, 495, 87, 11, 157, 359, 438, 2208, 16, 38, 146, 140, 1088, 155, 155, 1168, 46, 87, 1764, 2422, 63, 90, 16] payload_rows_per_dst=[8386, 10840, 10272, 10602, 10898, 14072, 7200, 9126] dup_rows_per_dst=[1256, 3164, 2386, 1566, 3990, 1668, 692, 2186] combine_rows_per_src=[8386, 10840, 10272, 10602, 10898, 14072, 7200, 9126] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=66108 payload_rows_remote=51448 dup_rows=32196 dup_rows_remote=14164 dedup_ratio=0.327515 router_tokens_per_expert=[593, 365, 3934, 2538, 1391, 0, 5985, 4105, 0, 178, 0, 467, 0, 1723, 0, 126, 355, 342, 2033, 101, 896, 42, 376, 150, 59, 89, 792, 0, 0, 151, 0, 634, 5923, 909, 0, 90, 501, 2294, 718, 0, 410, 0, 13, 0, 0, 0, 0, 293, 2454, 0, 0, 199, 1, 1058, 299, 0, 1277, 0, 24, 2689, 2463, 112, 0, 0] payload_rows_per_dst=[14660, 8246, 15258, 2956, 6828, 928, 7172, 10060] dup_rows_per_dst=[18032, 1872, 5178, 494, 2196, 504, 850, 3070] combine_rows_per_src=[14660, 8246, 15258, 2956, 6828, 928, 7172, 10060] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=268591104 dispatch_meta_bytes=345520 combine_payload_bytes=268591104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268936624 ingress_bytes=303194112 link_bytes=303194112 token_bytes=537527728 weight_movement_bytes=34603008 total_nvlink_bytes=572130736 weight_to_token_ratio=0.064374 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=29824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.140895034e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 1, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82112 payload_rows_remote=68784 dup_rows=16192 dup_rows_remote=11416 dedup_ratio=0.164714 router_tokens_per_expert=[557, 629, 3607, 3174, 1204, 1, 6154, 5599, 7, 162, 1, 432, 0, 1609, 1, 179, 312, 363, 1985, 119, 969, 61, 438, 140, 52, 88, 1515, 1, 4, 132, 1, 301, 5723, 847, 0, 82, 222, 1236, 652, 0, 386, 6, 316, 2, 21, 8, 355, 438, 2191, 0, 10, 481, 21, 1314, 69, 0, 1929, 34, 72, 1477, 1342, 104, 17, 0] payload_rows_per_dst=[8924, 9588, 7574, 13124, 13328, 14258, 7296, 8020] dup_rows_per_dst=[1626, 2408, 1200, 2262, 4776, 1114, 876, 1930] combine_rows_per_src=[8924, 9588, 7574, 13124, 13328, 14258, 7296, 8020] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=264347648 dispatch_meta_bytes=313720 combine_payload_bytes=264347648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264661368 ingress_bytes=298950656 link_bytes=298950656 token_bytes=529009016 weight_movement_bytes=34603008 total_nvlink_bytes=563612024 weight_to_token_ratio=0.065411 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=19584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.124927251e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78268 payload_rows_remote=66798 dup_rows=20036 dup_rows_remote=14866 dedup_ratio=0.203817 router_tokens_per_expert=[565, 532, 4077, 2628, 1616, 7, 5899, 4769, 5, 178, 4, 470, 10, 1500, 3, 151, 307, 347, 2212, 113, 928, 56, 406, 124, 47, 92, 1190, 0, 0, 140, 8, 553, 5838, 907, 2, 89, 232, 1458, 679, 4, 402, 5, 68, 9, 0, 7, 17, 454, 2670, 5, 7, 344, 1, 1200, 119, 6, 1806, 15, 46, 2068, 1625, 122, 10, 0] payload_rows_per_dst=[4822, 10210, 7830, 12014, 4738, 13082, 14102, 11470] dup_rows_per_dst=[618, 2586, 1156, 1584, 2004, 640, 6278, 5170] combine_rows_per_src=[4822, 10210, 7830, 12014, 4738, 13082, 14102, 11470] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=291323904 dispatch_meta_bytes=342584 combine_payload_bytes=291323904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291666488 ingress_bytes=325926912 link_bytes=325926912 token_bytes=582990392 weight_movement_bytes=34603008 total_nvlink_bytes=617593400 weight_to_token_ratio=0.059354 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.226436731e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78774 payload_rows_remote=64538 dup_rows=19530 dup_rows_remote=13892 dedup_ratio=0.198669 router_tokens_per_expert=[574, 548, 4246, 3179, 1168, 0, 6044, 5277, 0, 156, 0, 420, 0, 1617, 0, 118, 336, 374, 2050, 108, 986, 27, 417, 144, 44, 78, 1220, 0, 1, 129, 0, 586, 5865, 945, 0, 78, 186, 1549, 664, 0, 392, 1, 69, 0, 0, 0, 7, 378, 2446, 0, 4, 463, 1, 1045, 113, 0, 1934, 0, 59, 1546, 1462, 98, 0, 0] payload_rows_per_dst=[4076, 10412, 7632, 12958, 5008, 13166, 14236, 11286] dup_rows_per_dst=[504, 2702, 1252, 1712, 2326, 616, 5638, 4780] combine_rows_per_src=[4076, 10412, 7632, 12958, 5008, 13166, 14236, 11286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=210731008 dispatch_meta_bytes=262448 combine_payload_bytes=210731008 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=210993456 ingress_bytes=383746048 link_bytes=383746048 token_bytes=421724464 weight_movement_bytes=173015040 total_nvlink_bytes=594739504 weight_to_token_ratio=0.410256 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=14208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.444005485e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=66108 payload_rows_remote=51448 dup_rows=32196 dup_rows_remote=14164 dedup_ratio=0.327515 router_tokens_per_expert=[593, 365, 3934, 2538, 1391, 0, 5985, 4105, 0, 178, 0, 467, 0, 1723, 0, 126, 355, 342, 2033, 101, 896, 42, 376, 150, 59, 89, 792, 0, 0, 151, 0, 634, 5923, 909, 0, 90, 501, 2294, 718, 0, 410, 0, 13, 0, 0, 0, 0, 293, 2454, 0, 0, 199, 1, 1058, 299, 0, 1277, 0, 24, 2689, 2463, 112, 0, 0] payload_rows_per_dst=[14660, 8246, 15258, 2956, 6828, 928, 7172, 10060] dup_rows_per_dst=[18032, 1872, 5178, 494, 2196, 504, 850, 3070] combine_rows_per_src=[14660, 8246, 15258, 2956, 6828, 928, 7172, 10060] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82428 payload_rows_remote=69500 dup_rows=15876 dup_rows_remote=14198 dedup_ratio=0.161499 router_tokens_per_expert=[567, 465, 3995, 3032, 1380, 0, 6004, 5289, 0, 124, 0, 393, 0, 1607, 0, 121, 333, 356, 2173, 104, 950, 44, 446, 145, 50, 61, 1238, 0, 0, 117, 0, 548, 5862, 870, 0, 63, 217, 1675, 678, 0, 365, 0, 47, 0, 0, 0, 1, 334, 2622, 0, 6, 360, 1, 1131, 113, 0, 1857, 0, 50, 1834, 1436, 88, 0, 0] payload_rows_per_dst=[9218, 9954, 7806, 12928, 13468, 12926, 7606, 8522] dup_rows_per_dst=[1670, 2526, 1296, 1678, 5262, 576, 860, 2008] combine_rows_per_src=[9218, 9954, 7806, 12928, 13468, 12926, 7606, 8522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79392 payload_rows_remote=65386 dup_rows=18912 dup_rows_remote=18088 dedup_ratio=0.192383 router_tokens_per_expert=[601, 608, 3621, 3421, 1019, 0, 6197, 5921, 0, 176, 0, 450, 0, 1548, 0, 141, 325, 375, 1974, 105, 838, 62, 330, 145, 49, 88, 1621, 0, 0, 150, 0, 283, 5816, 830, 0, 88, 281, 1023, 688, 0, 417, 0, 135, 0, 34, 0, 277, 355, 2604, 0, 16, 469, 1, 1330, 67, 0, 1604, 0, 97, 1567, 1296, 109, 0, 0] payload_rows_per_dst=[3906, 9668, 7234, 13572, 10624, 14006, 12772, 7610] dup_rows_per_dst=[550, 2204, 1074, 2652, 4748, 824, 5124, 1736] combine_rows_per_src=[3906, 9668, 7234, 13572, 10624, 14006, 12772, 7610] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=281739264 dispatch_meta_bytes=320800 combine_payload_bytes=281739264 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=282060064 ingress_bytes=385548288 link_bytes=385548288 token_bytes=563799328 weight_movement_bytes=103809024 total_nvlink_bytes=667608352 weight_to_token_ratio=0.184124 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.450787169e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83966 payload_rows_remote=68654 dup_rows=14338 dup_rows_remote=10518 dedup_ratio=0.145854 router_tokens_per_expert=[0, 7387, 1710, 0, 22, 54, 599, 4896, 0, 0, 0, 0, 0, 768, 2719, 128, 36, 84, 89, 300, 62, 0, 176, 898, 1046, 249, 0, 442, 240, 55, 210, 0, 97, 7324, 68, 0, 1971, 0, 989, 0, 950, 0, 0, 0, 2762, 242, 0, 0, 63, 0, 111, 500, 36, 500, 61, 0, 0, 446, 5510, 574, 235, 2698, 0, 1845] payload_rows_per_dst=[13114, 7188, 13108, 15312, 5780, 8030, 15612, 5822] dup_rows_per_dst=[6016, 42, 1202, 3820, 470, 706, 1704, 378] combine_rows_per_src=[13114, 7188, 13108, 15312, 5780, 8030, 15612, 5822] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=273604608 dispatch_meta_bytes=326656 combine_payload_bytes=273604608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273931264 ingress_bytes=308207616 link_bytes=308207616 token_bytes=547535872 weight_movement_bytes=34603008 total_nvlink_bytes=582138880 weight_to_token_ratio=0.063198 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.159760446e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77890 payload_rows_remote=67594 dup_rows=20414 dup_rows_remote=16806 dedup_ratio=0.207662 router_tokens_per_expert=[2054, 5096, 187, 258, 959, 43, 96, 96, 1, 489, 227, 432, 109, 381, 535, 564, 428, 1585, 138, 110, 464, 88, 1454, 714, 2939, 3196, 1325, 201, 297, 59, 152, 520, 56, 377, 924, 16, 79, 2234, 1220, 132, 636, 119, 56, 69, 2776, 5538, 311, 92, 4544, 158, 1164, 535, 447, 109, 296, 47, 135, 360, 100, 33, 8, 734, 296, 354] payload_rows_per_dst=[12866, 10022, 10296, 8750, 8498, 11368, 5064, 11026] dup_rows_per_dst=[4712, 1846, 3608, 2236, 1578, 3748, 448, 2238] combine_rows_per_src=[12866, 10022, 10296, 8750, 8498, 11368, 5064, 11026] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=264347648 dispatch_meta_bytes=313720 combine_payload_bytes=264347648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264661368 ingress_bytes=298950656 link_bytes=298950656 token_bytes=529009016 weight_movement_bytes=34603008 total_nvlink_bytes=563612024 weight_to_token_ratio=0.065411 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=19584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.124927251e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78268 payload_rows_remote=66798 dup_rows=20036 dup_rows_remote=14866 dedup_ratio=0.203817 router_tokens_per_expert=[565, 532, 4077, 2628, 1616, 7, 5899, 4769, 5, 178, 4, 470, 10, 1500, 3, 151, 307, 347, 2212, 113, 928, 56, 406, 124, 47, 92, 1190, 0, 0, 140, 8, 553, 5838, 907, 2, 89, 232, 1458, 679, 4, 402, 5, 68, 9, 0, 7, 17, 454, 2670, 5, 7, 344, 1, 1200, 119, 6, 1806, 15, 46, 2068, 1625, 122, 10, 0] payload_rows_per_dst=[4822, 10210, 7830, 12014, 4738, 13082, 14102, 11470] dup_rows_per_dst=[618, 2586, 1156, 1584, 2004, 640, 6278, 5170] combine_rows_per_src=[4822, 10210, 7830, 12014, 4738, 13082, 14102, 11470] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=284672000 dispatch_meta_bytes=334792 combine_payload_bytes=284672000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=285006792 ingress_bytes=319275008 link_bytes=319275008 token_bytes=569678792 weight_movement_bytes=34603008 total_nvlink_bytes=604281800 weight_to_token_ratio=0.060741 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=33024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.201406151e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79040 payload_rows_remote=63054 dup_rows=19264 dup_rows_remote=12894 dedup_ratio=0.195964 router_tokens_per_expert=[0, 7423, 1497, 0, 0, 64, 616, 4314, 0, 0, 0, 0, 0, 766, 2822, 167, 0, 164, 110, 313, 81, 0, 147, 857, 1526, 231, 0, 316, 229, 64, 285, 0, 117, 7270, 99, 0, 2819, 0, 930, 0, 999, 0, 0, 0, 2614, 231, 0, 0, 98, 0, 140, 428, 17, 393, 96, 0, 0, 382, 5096, 1063, 295, 2580, 0, 1493] payload_rows_per_dst=[12734, 15986, 12312, 15276, 7298, 7336, 2092, 6006] dup_rows_per_dst=[5408, 6370, 1224, 4566, 632, 352, 252, 460] combine_rows_per_src=[12734, 15986, 12312, 15276, 7298, 7336, 2092, 6006] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=210731008 dispatch_meta_bytes=262448 combine_payload_bytes=210731008 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=210993456 ingress_bytes=383746048 link_bytes=383746048 token_bytes=421724464 weight_movement_bytes=173015040 total_nvlink_bytes=594739504 weight_to_token_ratio=0.410256 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=14208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.444005485e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267821056 dispatch_meta_bytes=333896 combine_payload_bytes=267821056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268154952 ingress_bytes=302424064 link_bytes=302424064 token_bytes=535976008 weight_movement_bytes=34603008 total_nvlink_bytes=570579016 weight_to_token_ratio=0.064561 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=40448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137997406e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=281206784 dispatch_meta_bytes=316688 combine_payload_bytes=281206784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281523472 ingress_bytes=315809792 link_bytes=315809792 token_bytes=562730256 weight_movement_bytes=34603008 total_nvlink_bytes=597333264 weight_to_token_ratio=0.061491 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=29568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.188366823e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80264 payload_rows_remote=64786 dup_rows=18040 dup_rows_remote=16178 dedup_ratio=0.183512 router_tokens_per_expert=[0, 7283, 1820, 0, 56, 56, 646, 4819, 0, 5, 0, 0, 0, 786, 2756, 142, 39, 85, 98, 323, 70, 0, 230, 844, 1338, 259, 0, 347, 244, 51, 247, 1, 108, 7246, 95, 0, 1665, 0, 947, 0, 829, 0, 0, 0, 2766, 244, 0, 0, 78, 0, 110, 550, 67, 504, 78, 0, 1, 417, 5533, 462, 287, 2587, 0, 2033] payload_rows_per_dst=[5066, 7332, 9598, 4066, 15034, 12496, 15478, 11194] dup_rows_per_dst=[90, 46, 966, 908, 5088, 4820, 1862, 4260] combine_rows_per_src=[5066, 7332, 9598, 4066, 15034, 12496, 15478, 11194] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=276865024 dispatch_meta_bytes=337600 combine_payload_bytes=276865024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277202624 ingress_bytes=311468032 link_bytes=311468032 token_bytes=554067648 weight_movement_bytes=34603008 total_nvlink_bytes=588670656 weight_to_token_ratio=0.062453 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.172029129e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=273604608 dispatch_meta_bytes=326656 combine_payload_bytes=273604608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273931264 ingress_bytes=308207616 link_bytes=308207616 token_bytes=547535872 weight_movement_bytes=34603008 total_nvlink_bytes=582138880 weight_to_token_ratio=0.063198 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.159760446e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78406 payload_rows_remote=64692 dup_rows=19898 dup_rows_remote=11992 dedup_ratio=0.202413 router_tokens_per_expert=[0, 7451, 1549, 0, 13, 61, 571, 4005, 0, 1, 0, 0, 0, 974, 3245, 151, 0, 123, 123, 321, 67, 0, 163, 828, 1331, 253, 0, 333, 227, 65, 301, 0, 145, 7279, 90, 0, 3508, 0, 862, 0, 729, 0, 0, 0, 2490, 252, 0, 0, 90, 0, 112, 424, 3, 366, 64, 0, 0, 376, 5221, 1335, 286, 2165, 0, 1199] payload_rows_per_dst=[13714, 13774, 12498, 15282, 8556, 6712, 1932, 5938] dup_rows_per_dst=[7906, 4978, 1194, 4296, 654, 230, 186, 454] combine_rows_per_src=[13714, 13774, 12498, 15282, 8556, 6712, 1932, 5938] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=258269184 dispatch_meta_bytes=303792 combine_payload_bytes=258269184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258572976 ingress_bytes=292872192 link_bytes=292872192 token_bytes=516842160 weight_movement_bytes=34603008 total_nvlink_bytes=551445168 weight_to_token_ratio=0.066951 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=23296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.102054480e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77116 payload_rows_remote=64980 dup_rows=21188 dup_rows_remote=19852 dedup_ratio=0.215535 router_tokens_per_expert=[47, 6874, 3359, 92, 9, 60, 684, 2640, 6, 363, 202, 32, 13, 992, 3089, 142, 120, 142, 117, 307, 99, 141, 233, 704, 1402, 290, 87, 397, 262, 93, 264, 2, 332, 6461, 87, 53, 1974, 128, 541, 1, 312, 2, 290, 27, 2144, 300, 29, 197, 88, 187, 130, 458, 44, 558, 145, 275, 46, 326, 4873, 1105, 353, 2503, 152, 1767] payload_rows_per_dst=[12342, 14024, 12136, 14026, 5848, 6086, 5746, 6908] dup_rows_per_dst=[6446, 6446, 1336, 4490, 384, 516, 980, 590] combine_rows_per_src=[12342, 14024, 12136, 14026, 5848, 6086, 5746, 6908] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82112 payload_rows_remote=68850 dup_rows=16192 dup_rows_remote=12348 dedup_ratio=0.164714 router_tokens_per_expert=[0, 7017, 1875, 13, 75, 75, 550, 4510, 0, 22, 0, 3, 0, 716, 2756, 143, 166, 96, 109, 309, 62, 1, 262, 892, 1229, 249, 7, 426, 242, 75, 253, 0, 93, 7102, 123, 2, 1348, 8, 1041, 0, 688, 1, 48, 2, 2666, 243, 0, 0, 73, 1, 107, 717, 149, 536, 85, 0, 19, 409, 5394, 368, 272, 2603, 15, 2906] payload_rows_per_dst=[5520, 7222, 12990, 5644, 13262, 12164, 15046, 10264] dup_rows_per_dst=[264, 58, 1592, 1646, 3844, 4152, 2324, 2312] combine_rows_per_src=[5520, 7222, 12990, 5644, 13262, 12164, 15046, 10264] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=323856 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265687312 ingress_bytes=299966464 link_bytes=299966464 token_bytes=531050768 weight_movement_bytes=34603008 total_nvlink_bytes=565653776 weight_to_token_ratio=0.065160 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=37632 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128749655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71662 payload_rows_remote=56288 dup_rows=26642 dup_rows_remote=11052 dedup_ratio=0.271016 router_tokens_per_expert=[2, 7347, 1816, 7, 54, 57, 605, 4556, 1, 23, 4, 2, 2, 728, 3206, 154, 50, 114, 135, 307, 64, 2, 179, 809, 842, 276, 6, 385, 251, 59, 293, 0, 86, 7226, 90, 2, 1994, 13, 1235, 0, 886, 3, 24, 4, 2778, 252, 3, 13, 79, 13, 200, 607, 32, 506, 98, 11, 17, 420, 5258, 492, 280, 2670, 22, 1502] payload_rows_per_dst=[4922, 8206, 2964, 3212, 8994, 12378, 15612, 15374] dup_rows_per_dst=[160, 34, 356, 1012, 2656, 4660, 2174, 15590] combine_rows_per_src=[4922, 8206, 2964, 3212, 8994, 12378, 15612, 15374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266158080 dispatch_meta_bytes=339328 combine_payload_bytes=266158080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266497408 ingress_bytes=300761088 link_bytes=300761088 token_bytes=532655488 weight_movement_bytes=34603008 total_nvlink_bytes=567258496 weight_to_token_ratio=0.064963 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=35200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131739761e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80264 payload_rows_remote=64786 dup_rows=18040 dup_rows_remote=16178 dedup_ratio=0.183512 router_tokens_per_expert=[0, 7283, 1820, 0, 56, 56, 646, 4819, 0, 5, 0, 0, 0, 786, 2756, 142, 39, 85, 98, 323, 70, 0, 230, 844, 1338, 259, 0, 347, 244, 51, 247, 1, 108, 7246, 95, 0, 1665, 0, 947, 0, 829, 0, 0, 0, 2766, 244, 0, 0, 78, 0, 110, 550, 67, 504, 78, 0, 1, 417, 5533, 462, 287, 2587, 0, 2033] payload_rows_per_dst=[5066, 7332, 9598, 4066, 15034, 12496, 15478, 11194] dup_rows_per_dst=[90, 46, 966, 908, 5088, 4820, 1862, 4260] combine_rows_per_src=[5066, 7332, 9598, 4066, 15034, 12496, 15478, 11194] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=264978432 dispatch_meta_bytes=306736 combine_payload_bytes=264978432 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265285168 ingress_bytes=403390464 link_bytes=403390464 token_bytes=530263600 weight_movement_bytes=138412032 total_nvlink_bytes=668675632 weight_to_token_ratio=0.261025 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.517925840e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83966 payload_rows_remote=68654 dup_rows=14338 dup_rows_remote=10518 dedup_ratio=0.145854 router_tokens_per_expert=[0, 7387, 1710, 0, 22, 54, 599, 4896, 0, 0, 0, 0, 0, 768, 2719, 128, 36, 84, 89, 300, 62, 0, 176, 898, 1046, 249, 0, 442, 240, 55, 210, 0, 97, 7324, 68, 0, 1971, 0, 989, 0, 950, 0, 0, 0, 2762, 242, 0, 0, 63, 0, 111, 500, 36, 500, 61, 0, 0, 446, 5510, 574, 235, 2698, 0, 1845] payload_rows_per_dst=[13114, 7188, 13108, 15312, 5780, 8030, 15612, 5822] dup_rows_per_dst=[6016, 42, 1202, 3820, 470, 706, 1704, 378] combine_rows_per_src=[13114, 7188, 13108, 15312, 5780, 8030, 15612, 5822] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78406 payload_rows_remote=64692 dup_rows=19898 dup_rows_remote=11992 dedup_ratio=0.202413 router_tokens_per_expert=[0, 7451, 1549, 0, 13, 61, 571, 4005, 0, 1, 0, 0, 0, 974, 3245, 151, 0, 123, 123, 321, 67, 0, 163, 828, 1331, 253, 0, 333, 227, 65, 301, 0, 145, 7279, 90, 0, 3508, 0, 862, 0, 729, 0, 0, 0, 2490, 252, 0, 0, 90, 0, 112, 424, 3, 366, 64, 0, 0, 376, 5221, 1335, 286, 2165, 0, 1199] payload_rows_per_dst=[13714, 13774, 12498, 15282, 8556, 6712, 1932, 5938] dup_rows_per_dst=[7906, 4978, 1194, 4296, 654, 230, 186, 454] combine_rows_per_src=[13714, 13774, 12498, 15282, 8556, 6712, 1932, 5938] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=282009600 dispatch_meta_bytes=324792 combine_payload_bytes=282009600 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=282334392 ingress_bytes=351215616 link_bytes=351215616 token_bytes=564343992 weight_movement_bytes=69206016 total_nvlink_bytes=633550008 weight_to_token_ratio=0.122631 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=22912 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.321596088e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=230555648 dispatch_meta_bytes=269360 combine_payload_bytes=230555648 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=230825008 ingress_bytes=334364672 link_bytes=334364672 token_bytes=461380656 weight_movement_bytes=103809024 total_nvlink_bytes=565189680 weight_to_token_ratio=0.224996 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258187342e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81184 payload_rows_remote=68468 dup_rows=17120 dup_rows_remote=12170 dedup_ratio=0.174154 router_tokens_per_expert=[0, 7033, 1533, 0, 77, 66, 625, 5188, 0, 0, 0, 0, 0, 552, 2579, 168, 251, 111, 104, 377, 72, 0, 254, 829, 1233, 257, 0, 291, 257, 66, 270, 0, 26, 7296, 124, 0, 1262, 0, 1194, 0, 638, 0, 4, 0, 2747, 256, 0, 0, 85, 0, 117, 686, 134, 508, 89, 0, 3, 325, 5416, 306, 257, 2964, 0, 2522] payload_rows_per_dst=[4528, 6598, 12950, 3836, 15134, 12716, 15020, 10402] dup_rows_per_dst=[74, 0, 1878, 912, 4670, 4950, 2284, 2352] combine_rows_per_src=[4528, 6598, 12950, 3836, 15134, 12716, 15020, 10402] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85986 payload_rows_remote=74348 dup_rows=12318 dup_rows_remote=10982 dedup_ratio=0.125305 router_tokens_per_expert=[0, 0, 0, 226, 0, 0, 0, 0, 358, 138, 1861, 0, 642, 303, 0, 61, 88, 0, 0, 0, 0, 62, 0, 0, 0, 4702, 0, 252, 240, 1286, 1, 6, 1124, 3519, 0, 0, 0, 1190, 0, 1126, 2170, 1127, 4919, 0, 631, 633, 754, 0, 0, 96, 0, 2990, 93, 0, 88, 0, 986, 0, 5172, 5804, 5113, 1268, 0, 123] payload_rows_per_dst=[10714, 8540, 11908, 11638, 11466, 6036, 12902, 12782] dup_rows_per_dst=[82, 2526, 0, 1336, 2452, 370, 3354, 2198] combine_rows_per_src=[10714, 8540, 11908, 11638, 11466, 6036, 12902, 12782] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80398 payload_rows_remote=70278 dup_rows=17906 dup_rows_remote=14824 dedup_ratio=0.182149 router_tokens_per_expert=[0, 0, 0, 242, 0, 0, 0, 0, 376, 148, 1735, 0, 625, 328, 0, 97, 60, 0, 0, 0, 0, 82, 0, 0, 0, 5277, 0, 320, 240, 1543, 0, 110, 865, 3990, 0, 0, 0, 1101, 0, 1110, 3137, 1090, 4313, 0, 559, 1045, 1197, 0, 0, 111, 0, 2035, 0, 0, 60, 0, 911, 0, 5287, 5914, 4074, 975, 0, 195] payload_rows_per_dst=[12608, 10120, 12704, 12924, 11772, 12484, 4412, 3374] dup_rows_per_dst=[3812, 3082, 2194, 2056, 2360, 3614, 0, 788] combine_rows_per_src=[12608, 10120, 12704, 12924, 11772, 12484, 4412, 3374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=323856 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265687312 ingress_bytes=299966464 link_bytes=299966464 token_bytes=531050768 weight_movement_bytes=34603008 total_nvlink_bytes=565653776 weight_to_token_ratio=0.065160 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=37632 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128749655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=282009600 dispatch_meta_bytes=324792 combine_payload_bytes=282009600 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=282334392 ingress_bytes=351215616 link_bytes=351215616 token_bytes=564343992 weight_movement_bytes=69206016 total_nvlink_bytes=633550008 weight_to_token_ratio=0.122631 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=22912 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.321596088e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81396 payload_rows_remote=71124 dup_rows=16908 dup_rows_remote=14522 dedup_ratio=0.171997 router_tokens_per_expert=[539, 497, 4026, 2858, 822, 105, 5859, 3895, 62, 167, 90, 424, 192, 1750, 21, 270, 303, 292, 1936, 87, 684, 84, 440, 166, 62, 115, 1026, 25, 15, 133, 185, 628, 5606, 701, 51, 130, 479, 2113, 661, 40, 425, 39, 495, 87, 11, 157, 359, 438, 2208, 16, 38, 146, 140, 1088, 155, 155, 1168, 46, 87, 1764, 2422, 63, 90, 16] payload_rows_per_dst=[8386, 10840, 10272, 10602, 10898, 14072, 7200, 9126] dup_rows_per_dst=[1256, 3164, 2386, 1566, 3990, 1668, 692, 2186] combine_rows_per_src=[8386, 10840, 10272, 10602, 10898, 14072, 7200, 9126] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71662 payload_rows_remote=56288 dup_rows=26642 dup_rows_remote=11052 dedup_ratio=0.271016 router_tokens_per_expert=[2, 7347, 1816, 7, 54, 57, 605, 4556, 1, 23, 4, 2, 2, 728, 3206, 154, 50, 114, 135, 307, 64, 2, 179, 809, 842, 276, 6, 385, 251, 59, 293, 0, 86, 7226, 90, 2, 1994, 13, 1235, 0, 886, 3, 24, 4, 2778, 252, 3, 13, 79, 13, 200, 607, 32, 506, 98, 11, 17, 420, 5258, 492, 280, 2670, 22, 1502] payload_rows_per_dst=[4922, 8206, 2964, 3212, 8994, 12378, 15612, 15374] dup_rows_per_dst=[160, 34, 356, 1012, 2656, 4660, 2174, 15590] combine_rows_per_src=[4922, 8206, 2964, 3212, 8994, 12378, 15612, 15374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=281206784 dispatch_meta_bytes=316688 combine_payload_bytes=281206784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281523472 ingress_bytes=315809792 link_bytes=315809792 token_bytes=562730256 weight_movement_bytes=34603008 total_nvlink_bytes=597333264 weight_to_token_ratio=0.061491 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=29568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.188366823e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=264978432 dispatch_meta_bytes=306736 combine_payload_bytes=264978432 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265285168 ingress_bytes=403390464 link_bytes=403390464 token_bytes=530263600 weight_movement_bytes=138412032 total_nvlink_bytes=668675632 weight_to_token_ratio=0.261025 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.517925840e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85568 payload_rows_remote=72470 dup_rows=12736 dup_rows_remote=9340 dedup_ratio=0.129557 router_tokens_per_expert=[0, 0, 0, 258, 0, 0, 0, 0, 366, 117, 2043, 0, 601, 322, 0, 92, 84, 0, 1, 0, 0, 70, 0, 0, 0, 4686, 0, 254, 254, 1273, 1, 0, 1535, 2868, 0, 0, 0, 1112, 0, 1076, 2253, 1467, 4799, 0, 618, 659, 750, 0, 0, 123, 0, 3117, 124, 0, 84, 0, 939, 0, 5115, 5633, 4946, 1357, 0, 155] payload_rows_per_dst=[10662, 8878, 11574, 9668, 8352, 10830, 13098, 12506] dup_rows_per_dst=[84, 2710, 2, 980, 1382, 1894, 3396, 2288] combine_rows_per_src=[10662, 8878, 11574, 9668, 8352, 10830, 13098, 12506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80398 payload_rows_remote=70278 dup_rows=17906 dup_rows_remote=14824 dedup_ratio=0.182149 router_tokens_per_expert=[0, 0, 0, 242, 0, 0, 0, 0, 376, 148, 1735, 0, 625, 328, 0, 97, 60, 0, 0, 0, 0, 82, 0, 0, 0, 5277, 0, 320, 240, 1543, 0, 110, 865, 3990, 0, 0, 0, 1101, 0, 1110, 3137, 1090, 4313, 0, 559, 1045, 1197, 0, 0, 111, 0, 2035, 0, 0, 60, 0, 911, 0, 5287, 5914, 4074, 975, 0, 195] payload_rows_per_dst=[12608, 10120, 12704, 12924, 11772, 12484, 4412, 3374] dup_rows_per_dst=[3812, 3082, 2194, 2056, 2360, 3614, 0, 788] combine_rows_per_src=[12608, 10120, 12704, 12924, 11772, 12484, 4412, 3374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=280444928 dispatch_meta_bytes=322552 combine_payload_bytes=280444928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280767480 ingress_bytes=315047936 link_bytes=315047936 token_bytes=561212408 weight_movement_bytes=34603008 total_nvlink_bytes=595815416 weight_to_token_ratio=0.061658 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20992 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.185500020e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80198 payload_rows_remote=68548 dup_rows=18106 dup_rows_remote=15384 dedup_ratio=0.184184 router_tokens_per_expert=[40, 17, 2, 308, 170, 17, 27, 305, 422, 123, 1913, 161, 699, 395, 130, 86, 38, 45, 0, 11, 155, 118, 0, 197, 24, 4494, 26, 645, 338, 1569, 115, 369, 627, 3368, 0, 75, 0, 1277, 60, 1283, 3825, 1881, 3695, 180, 531, 879, 1624, 65, 3, 117, 267, 1563, 100, 179, 81, 43, 1321, 139, 4279, 4516, 3361, 712, 4, 138] payload_rows_per_dst=[10012, 11778, 11650, 12540, 10990, 12800, 4254, 6174] dup_rows_per_dst=[318, 3730, 2722, 2620, 2390, 4910, 452, 964] combine_rows_per_src=[10012, 11778, 11650, 12540, 10990, 12800, 4254, 6174] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=304529408 dispatch_meta_bytes=341320 combine_payload_bytes=304529408 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=304870728 ingress_bytes=373735424 link_bytes=373735424 token_bytes=609400136 weight_movement_bytes=69206016 total_nvlink_bytes=678606152 weight_to_token_ratio=0.113564 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=32128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.406336312e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=291323904 dispatch_meta_bytes=342584 combine_payload_bytes=291323904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291666488 ingress_bytes=325926912 link_bytes=325926912 token_bytes=582990392 weight_movement_bytes=34603008 total_nvlink_bytes=617593400 weight_to_token_ratio=0.059354 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.226436731e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=230555648 dispatch_meta_bytes=269360 combine_payload_bytes=230555648 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=230825008 ingress_bytes=334364672 link_bytes=334364672 token_bytes=461380656 weight_movement_bytes=103809024 total_nvlink_bytes=565189680 weight_to_token_ratio=0.224996 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258187342e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79860 payload_rows_remote=65802 dup_rows=18444 dup_rows_remote=12262 dedup_ratio=0.187622 router_tokens_per_expert=[1, 0, 0, 187, 0, 1, 0, 0, 382, 143, 2053, 0, 616, 341, 0, 89, 95, 0, 0, 0, 0, 69, 0, 0, 0, 5045, 0, 379, 241, 1603, 0, 43, 585, 4396, 0, 0, 0, 1028, 0, 1072, 3731, 928, 3242, 0, 556, 1335, 1320, 0, 0, 125, 0, 1605, 0, 0, 95, 0, 1067, 0, 5998, 5899, 3933, 772, 0, 177] payload_rows_per_dst=[14058, 11538, 12126, 12520, 12238, 10456, 3650, 3274] dup_rows_per_dst=[6182, 5028, 0, 2102, 1924, 2450, 0, 758] combine_rows_per_src=[14058, 11538, 12126, 12520, 12238, 10456, 3650, 3274] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=287858688 dispatch_meta_bytes=340408 combine_payload_bytes=287858688 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=288199096 ingress_bytes=357064704 link_bytes=357064704 token_bytes=576057784 weight_movement_bytes=69206016 total_nvlink_bytes=645263800 weight_to_token_ratio=0.120137 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=21120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.343605735e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=296837120 dispatch_meta_bytes=327240 combine_payload_bytes=296837120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=297164360 ingress_bytes=331440128 link_bytes=331440128 token_bytes=594001480 weight_movement_bytes=34603008 total_nvlink_bytes=628604488 weight_to_token_ratio=0.058254 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=24960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.247182519e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85568 payload_rows_remote=72470 dup_rows=12736 dup_rows_remote=9340 dedup_ratio=0.129557 router_tokens_per_expert=[0, 0, 0, 258, 0, 0, 0, 0, 366, 117, 2043, 0, 601, 322, 0, 92, 84, 0, 1, 0, 0, 70, 0, 0, 0, 4686, 0, 254, 254, 1273, 1, 0, 1535, 2868, 0, 0, 0, 1112, 0, 1076, 2253, 1467, 4799, 0, 618, 659, 750, 0, 0, 123, 0, 3117, 124, 0, 84, 0, 939, 0, 5115, 5633, 4946, 1357, 0, 155] payload_rows_per_dst=[10662, 8878, 11574, 9668, 8352, 10830, 13098, 12506] dup_rows_per_dst=[84, 2710, 2, 980, 1382, 1894, 3396, 2288] combine_rows_per_src=[10662, 8878, 11574, 9668, 8352, 10830, 13098, 12506] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84744 payload_rows_remote=74058 dup_rows=13560 dup_rows_remote=11010 dedup_ratio=0.137939 router_tokens_per_expert=[5, 0, 1, 279, 0, 4, 1, 4, 371, 65, 1302, 1, 700, 322, 2, 199, 69, 0, 17, 0, 4, 63, 0, 8, 4, 4612, 0, 274, 264, 1050, 14, 310, 1652, 2622, 0, 0, 0, 1208, 4, 1132, 1937, 2055, 5472, 3, 615, 461, 638, 4, 0, 202, 0, 3441, 256, 0, 78, 0, 1027, 1, 4636, 5094, 4828, 1685, 0, 156] payload_rows_per_dst=[9790, 7854, 10490, 11864, 10686, 7078, 14386, 12596] dup_rows_per_dst=[70, 1944, 20, 1192, 2550, 474, 4512, 2798] combine_rows_per_src=[9790, 7854, 10490, 11864, 10686, 7078, 14386, 12596] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82112 payload_rows_remote=67302 dup_rows=16192 dup_rows_remote=8686 dedup_ratio=0.164714 router_tokens_per_expert=[4, 2, 0, 223, 3, 2, 0, 12, 378, 92, 1659, 8, 594, 339, 5, 92, 51, 1, 2, 0, 13, 63, 0, 23, 3, 4645, 4, 338, 247, 1475, 17, 144, 1131, 3890, 0, 7, 0, 1010, 2, 1123, 2550, 1448, 4563, 8, 588, 654, 918, 4, 1, 130, 13, 2417, 33, 13, 62, 2, 1047, 12, 5338, 5523, 4905, 1152, 0, 169] payload_rows_per_dst=[11100, 9006, 11334, 4408, 7126, 12302, 12026, 14810] dup_rows_per_dst=[68, 2428, 18, 48, 964, 2718, 2442, 7506] combine_rows_per_src=[11100, 9006, 11334, 4408, 7126, 12302, 12026, 14810] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=280772608 dispatch_meta_bytes=335728 combine_payload_bytes=280772608 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=281108336 ingress_bytes=349978624 link_bytes=349978624 token_bytes=561880944 weight_movement_bytes=69206016 total_nvlink_bytes=631086960 weight_to_token_ratio=0.123168 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=44160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.316941387e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=269524992 dispatch_meta_bytes=312256 combine_payload_bytes=269524992 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=269837248 ingress_bytes=338731008 link_bytes=338731008 token_bytes=539362240 weight_movement_bytes=69206016 total_nvlink_bytes=608568256 weight_to_token_ratio=0.128311 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=40064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274617513e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85986 payload_rows_remote=74348 dup_rows=12318 dup_rows_remote=10982 dedup_ratio=0.125305 router_tokens_per_expert=[0, 0, 0, 226, 0, 0, 0, 0, 358, 138, 1861, 0, 642, 303, 0, 61, 88, 0, 0, 0, 0, 62, 0, 0, 0, 4702, 0, 252, 240, 1286, 1, 6, 1124, 3519, 0, 0, 0, 1190, 0, 1126, 2170, 1127, 4919, 0, 631, 633, 754, 0, 0, 96, 0, 2990, 93, 0, 88, 0, 986, 0, 5172, 5804, 5113, 1268, 0, 123] payload_rows_per_dst=[10714, 8540, 11908, 11638, 11466, 6036, 12902, 12782] dup_rows_per_dst=[82, 2526, 0, 1336, 2452, 370, 3354, 2198] combine_rows_per_src=[10714, 8540, 11908, 11638, 11466, 6036, 12902, 12782] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=303341568 dispatch_meta_bytes=340272 combine_payload_bytes=303341568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=303681840 ingress_bytes=372547584 link_bytes=372547584 token_bytes=607023408 weight_movement_bytes=69206016 total_nvlink_bytes=676229424 weight_to_token_ratio=0.114009 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=19584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.401866566e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79860 payload_rows_remote=65802 dup_rows=18444 dup_rows_remote=12262 dedup_ratio=0.187622 router_tokens_per_expert=[1, 0, 0, 187, 0, 1, 0, 0, 382, 143, 2053, 0, 616, 341, 0, 89, 95, 0, 0, 0, 0, 69, 0, 0, 0, 5045, 0, 379, 241, 1603, 0, 43, 585, 4396, 0, 0, 0, 1028, 0, 1072, 3731, 928, 3242, 0, 556, 1335, 1320, 0, 0, 125, 0, 1605, 0, 0, 95, 0, 1067, 0, 5998, 5899, 3933, 772, 0, 177] payload_rows_per_dst=[14058, 11538, 12126, 12520, 12238, 10456, 3650, 3274] dup_rows_per_dst=[6182, 5028, 0, 2102, 1924, 2450, 0, 758] combine_rows_per_src=[14058, 11538, 12126, 12520, 12238, 10456, 3650, 3274] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=275668992 dispatch_meta_bytes=303952 combine_payload_bytes=275668992 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=275972944 ingress_bytes=448684032 link_bytes=448684032 token_bytes=551641936 weight_movement_bytes=173015040 total_nvlink_bytes=724656976 weight_to_token_ratio=0.313636 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=26368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.688361890e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84314 payload_rows_remote=75696 dup_rows=13990 dup_rows_remote=12720 dedup_ratio=0.142314 router_tokens_per_expert=[0, 0, 0, 197, 0, 0, 0, 0, 395, 78, 1400, 0, 702, 345, 0, 179, 57, 0, 1, 0, 0, 72, 0, 0, 0, 4391, 0, 275, 222, 1099, 0, 217, 1843, 2740, 0, 0, 0, 1223, 0, 1124, 1472, 1949, 5246, 0, 484, 528, 879, 0, 0, 190, 0, 3981, 236, 0, 57, 0, 736, 0, 4175, 5651, 5200, 1632, 0, 176] payload_rows_per_dst=[8692, 7338, 11560, 11270, 9826, 8618, 14188, 12822] dup_rows_per_dst=[52, 1804, 2, 1138, 1826, 1270, 5232, 2666] combine_rows_per_src=[8692, 7338, 11560, 11270, 9826, 8618, 14188, 12822] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88090 payload_rows_remote=74712 dup_rows=10214 dup_rows_remote=8972 dedup_ratio=0.103902 router_tokens_per_expert=[0, 54, 243, 64, 30, 3225, 0, 442, 0, 0, 54, 0, 63, 0, 0, 0, 1636, 0, 0, 0, 0, 1581, 0, 1155, 0, 83, 12, 1189, 0, 0, 0, 54, 5740, 1, 63, 570, 0, 0, 123, 1211, 180, 0, 0, 979, 62, 0, 0, 1, 401, 0, 4311, 66, 5972, 54, 5927, 0, 58, 3648, 4012, 613, 2707, 2383, 0, 185] payload_rows_per_dst=[12452, 12028, 7710, 13378, 12812, 10116, 9288, 10306] dup_rows_per_dst=[2960, 60, 1034, 1242, 2604, 352, 376, 1586] combine_rows_per_src=[12452, 12028, 7710, 13378, 12812, 10116, 9288, 10306] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83208 payload_rows_remote=71148 dup_rows=15096 dup_rows_remote=15032 dedup_ratio=0.153564 router_tokens_per_expert=[0, 63, 231, 82, 17, 2912, 0, 437, 0, 0, 66, 0, 98, 0, 0, 0, 1792, 0, 0, 0, 0, 1586, 0, 1115, 0, 80, 23, 2018, 0, 0, 0, 63, 5562, 4, 98, 538, 0, 0, 179, 883, 242, 0, 0, 1281, 82, 0, 0, 0, 379, 0, 4416, 97, 6069, 63, 5898, 0, 12, 3604, 3864, 711, 2099, 2183, 0, 305] payload_rows_per_dst=[12132, 12060, 14514, 13690, 6034, 11172, 9556, 4050] dup_rows_per_dst=[2560, 64, 5596, 2816, 1568, 1726, 354, 412] combine_rows_per_src=[12132, 12060, 14514, 13690, 6034, 11172, 9556, 4050] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=296837120 dispatch_meta_bytes=327240 combine_payload_bytes=296837120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=297164360 ingress_bytes=331440128 link_bytes=331440128 token_bytes=594001480 weight_movement_bytes=34603008 total_nvlink_bytes=628604488 weight_to_token_ratio=0.058254 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=24960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.247182519e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77116 payload_rows_remote=64980 dup_rows=21188 dup_rows_remote=19852 dedup_ratio=0.215535 router_tokens_per_expert=[47, 6874, 3359, 92, 9, 60, 684, 2640, 6, 363, 202, 32, 13, 992, 3089, 142, 120, 142, 117, 307, 99, 141, 233, 704, 1402, 290, 87, 397, 262, 93, 264, 2, 332, 6461, 87, 53, 1974, 128, 541, 1, 312, 2, 290, 27, 2144, 300, 29, 197, 88, 187, 130, 458, 44, 558, 145, 275, 46, 326, 4873, 1105, 353, 2503, 152, 1767] payload_rows_per_dst=[12342, 14024, 12136, 14026, 5848, 6086, 5746, 6908] dup_rows_per_dst=[6446, 6446, 1336, 4490, 384, 516, 980, 590] combine_rows_per_src=[12342, 14024, 12136, 14026, 5848, 6086, 5746, 6908] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=303341568 dispatch_meta_bytes=340272 combine_payload_bytes=303341568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=303681840 ingress_bytes=372547584 link_bytes=372547584 token_bytes=607023408 weight_movement_bytes=69206016 total_nvlink_bytes=676229424 weight_to_token_ratio=0.114009 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=19584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.401866566e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82112 payload_rows_remote=67302 dup_rows=16192 dup_rows_remote=8686 dedup_ratio=0.164714 router_tokens_per_expert=[4, 2, 0, 223, 3, 2, 0, 12, 378, 92, 1659, 8, 594, 339, 5, 92, 51, 1, 2, 0, 13, 63, 0, 23, 3, 4645, 4, 338, 247, 1475, 17, 144, 1131, 3890, 0, 7, 0, 1010, 2, 1123, 2550, 1448, 4563, 8, 588, 654, 918, 4, 1, 130, 13, 2417, 33, 13, 62, 2, 1047, 12, 5338, 5523, 4905, 1152, 0, 169] payload_rows_per_dst=[11100, 9006, 11334, 4408, 7126, 12302, 12026, 14810] dup_rows_per_dst=[68, 2428, 18, 48, 964, 2718, 2442, 7506] combine_rows_per_src=[11100, 9006, 11334, 4408, 7126, 12302, 12026, 14810] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83122 payload_rows_remote=69432 dup_rows=15182 dup_rows_remote=9462 dedup_ratio=0.154439 router_tokens_per_expert=[0, 51, 244, 77, 63, 3223, 0, 473, 0, 0, 38, 0, 78, 0, 0, 0, 1530, 0, 0, 0, 0, 1426, 0, 1097, 0, 98, 16, 1053, 0, 0, 0, 51, 5421, 2, 78, 631, 0, 0, 148, 1262, 211, 0, 0, 1013, 70, 0, 0, 1, 363, 0, 4093, 79, 5947, 51, 6020, 0, 65, 3679, 4486, 663, 3082, 2029, 0, 240] payload_rows_per_dst=[8176, 12212, 7360, 3886, 12330, 11002, 13690, 14466] dup_rows_per_dst=[86, 60, 746, 206, 2754, 560, 5720, 5050] combine_rows_per_src=[8176, 12212, 7360, 3886, 12330, 11002, 13690, 14466] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=269524992 dispatch_meta_bytes=312256 combine_payload_bytes=269524992 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=269837248 ingress_bytes=338731008 link_bytes=338731008 token_bytes=539362240 weight_movement_bytes=69206016 total_nvlink_bytes=608568256 weight_to_token_ratio=0.128311 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=40064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274617513e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84118 payload_rows_remote=70550 dup_rows=14186 dup_rows_remote=9224 dedup_ratio=0.144307 router_tokens_per_expert=[44, 87, 305, 344, 57, 1939, 218, 542, 9, 33, 60, 87, 127, 345, 11, 51, 1406, 0, 14, 1, 49, 1215, 114, 1447, 47, 186, 453, 1580, 55, 41, 8, 118, 5220, 170, 84, 474, 2, 226, 157, 619, 212, 228, 446, 1044, 89, 23, 330, 47, 387, 14, 4202, 71, 5514, 49, 5744, 57, 40, 3191, 4080, 818, 1659, 2623, 76, 263] payload_rows_per_dst=[11260, 12482, 13568, 13050, 5804, 11110, 9208, 7636] dup_rows_per_dst=[2194, 452, 4962, 2954, 724, 1888, 352, 660] combine_rows_per_src=[11260, 12482, 13568, 13050, 5804, 11110, 9208, 7636] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83208 payload_rows_remote=71148 dup_rows=15096 dup_rows_remote=15032 dedup_ratio=0.153564 router_tokens_per_expert=[0, 63, 231, 82, 17, 2912, 0, 437, 0, 0, 66, 0, 98, 0, 0, 0, 1792, 0, 0, 0, 0, 1586, 0, 1115, 0, 80, 23, 2018, 0, 0, 0, 63, 5562, 4, 98, 538, 0, 0, 179, 883, 242, 0, 0, 1281, 82, 0, 0, 0, 379, 0, 4416, 97, 6069, 63, 5898, 0, 12, 3604, 3864, 711, 2099, 2183, 0, 305] payload_rows_per_dst=[12132, 12060, 14514, 13690, 6034, 11172, 9556, 4050] dup_rows_per_dst=[2560, 64, 5596, 2816, 1568, 1726, 354, 412] combine_rows_per_src=[12132, 12060, 14514, 13690, 6034, 11172, 9556, 4050] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=304529408 dispatch_meta_bytes=341320 combine_payload_bytes=304529408 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=304870728 ingress_bytes=373735424 link_bytes=373735424 token_bytes=609400136 weight_movement_bytes=69206016 total_nvlink_bytes=678606152 weight_to_token_ratio=0.113564 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=32128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.406336312e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 2, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=310050816 dispatch_meta_bytes=353664 combine_payload_bytes=310050816 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=310404480 ingress_bytes=448462848 link_bytes=448462848 token_bytes=620455296 weight_movement_bytes=138412032 total_nvlink_bytes=758867328 weight_to_token_ratio=0.223081 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=11392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.687529593e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=306020352 dispatch_meta_bytes=334736 combine_payload_bytes=306020352 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=306355088 ingress_bytes=340623360 link_bytes=340623360 token_bytes=612375440 weight_movement_bytes=34603008 total_nvlink_bytes=646978448 weight_to_token_ratio=0.056506 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=35456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.281738281e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80628 payload_rows_remote=69538 dup_rows=17676 dup_rows_remote=15674 dedup_ratio=0.179810 router_tokens_per_expert=[0, 61, 252, 119, 9, 2890, 0, 437, 0, 0, 74, 0, 90, 0, 0, 0, 1740, 0, 0, 0, 0, 1624, 0, 1260, 0, 0, 20, 2788, 0, 0, 0, 61, 5183, 13, 90, 420, 0, 0, 158, 761, 218, 0, 0, 1546, 69, 0, 0, 0, 475, 0, 4227, 89, 6067, 61, 5811, 0, 6, 2778, 4104, 683, 1973, 2693, 0, 302] payload_rows_per_dst=[11090, 11896, 14142, 13806, 5540, 13182, 9344, 1628] dup_rows_per_dst=[2002, 54, 5472, 4066, 1290, 4078, 360, 354] combine_rows_per_src=[11090, 11896, 14142, 13806, 5540, 13182, 9344, 1628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266158080 dispatch_meta_bytes=339328 combine_payload_bytes=266158080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266497408 ingress_bytes=300761088 link_bytes=300761088 token_bytes=532655488 weight_movement_bytes=34603008 total_nvlink_bytes=567258496 weight_to_token_ratio=0.064963 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=35200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131739761e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=275668992 dispatch_meta_bytes=303952 combine_payload_bytes=275668992 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=275972944 ingress_bytes=448684032 link_bytes=448684032 token_bytes=551641936 weight_movement_bytes=173015040 total_nvlink_bytes=724656976 weight_to_token_ratio=0.313636 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=26368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.688361890e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=291422208 dispatch_meta_bytes=344720 combine_payload_bytes=291422208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291766928 ingress_bytes=326025216 link_bytes=326025216 token_bytes=583189136 weight_movement_bytes=34603008 total_nvlink_bytes=617792144 weight_to_token_ratio=0.059334 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=47488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.226806641e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=284393472 dispatch_meta_bytes=315576 combine_payload_bytes=284393472 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=284709048 ingress_bytes=388202496 link_bytes=388202496 token_bytes=569102520 weight_movement_bytes=103809024 total_nvlink_bytes=672911544 weight_to_token_ratio=0.182408 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=34176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.460774740e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83122 payload_rows_remote=69432 dup_rows=15182 dup_rows_remote=9462 dedup_ratio=0.154439 router_tokens_per_expert=[0, 51, 244, 77, 63, 3223, 0, 473, 0, 0, 38, 0, 78, 0, 0, 0, 1530, 0, 0, 0, 0, 1426, 0, 1097, 0, 98, 16, 1053, 0, 0, 0, 51, 5421, 2, 78, 631, 0, 0, 148, 1262, 211, 0, 0, 1013, 70, 0, 0, 1, 363, 0, 4093, 79, 5947, 51, 6020, 0, 65, 3679, 4486, 663, 3082, 2029, 0, 240] payload_rows_per_dst=[8176, 12212, 7360, 3886, 12330, 11002, 13690, 14466] dup_rows_per_dst=[86, 60, 746, 206, 2754, 560, 5720, 5050] combine_rows_per_src=[8176, 12212, 7360, 3886, 12330, 11002, 13690, 14466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=284827648 dispatch_meta_bytes=340848 combine_payload_bytes=284827648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=285168496 ingress_bytes=319430656 link_bytes=319430656 token_bytes=569996144 weight_movement_bytes=34603008 total_nvlink_bytes=604599152 weight_to_token_ratio=0.060707 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=24576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.201991842e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87290 payload_rows_remote=74180 dup_rows=11014 dup_rows_remote=7924 dedup_ratio=0.112040 router_tokens_per_expert=[3, 52, 247, 71, 74, 3582, 0, 485, 0, 0, 37, 1, 82, 1, 0, 0, 1601, 0, 0, 9, 1, 1385, 0, 1120, 0, 135, 41, 847, 3, 3, 0, 64, 5711, 5, 84, 675, 5, 1, 142, 1477, 265, 1, 1, 905, 62, 3, 1, 9, 390, 29, 4468, 93, 5396, 52, 5968, 0, 47, 3923, 4009, 674, 2939, 1753, 0, 220] payload_rows_per_dst=[12694, 12124, 7438, 12370, 13110, 10070, 9620, 9864] dup_rows_per_dst=[3624, 54, 794, 608, 3090, 442, 444, 1958] combine_rows_per_src=[12694, 12124, 7438, 12370, 13110, 10070, 9620, 9864] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82130 payload_rows_remote=67714 dup_rows=16174 dup_rows_remote=9786 dedup_ratio=0.164530 router_tokens_per_expert=[3, 56, 261, 88, 21, 3013, 30, 462, 2, 6, 33, 2, 90, 16, 0, 9, 1320, 0, 1, 0, 1, 1322, 2, 1113, 6, 146, 64, 1342, 0, 9, 0, 56, 5684, 28, 90, 602, 0, 17, 151, 1055, 202, 15, 28, 1079, 63, 6, 16, 7, 360, 8, 4773, 87, 5634, 51, 6071, 3, 31, 3433, 4548, 717, 2300, 2337, 3, 279] payload_rows_per_dst=[7782, 12342, 6760, 3190, 12748, 11264, 13628, 14416] dup_rows_per_dst=[86, 90, 758, 56, 2506, 664, 5626, 6388] combine_rows_per_src=[7782, 12342, 6760, 3190, 12748, 11264, 13628, 14416] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=288972800 dispatch_meta_bytes=319096 combine_payload_bytes=288972800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289291896 ingress_bytes=323575808 link_bytes=323575808 token_bytes=578264696 weight_movement_bytes=34603008 total_nvlink_bytes=612867704 weight_to_token_ratio=0.059839 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=15872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.217589716e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=284393472 dispatch_meta_bytes=315576 combine_payload_bytes=284393472 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=284709048 ingress_bytes=388202496 link_bytes=388202496 token_bytes=569102520 weight_movement_bytes=103809024 total_nvlink_bytes=672911544 weight_to_token_ratio=0.182408 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=34176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.460774740e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88090 payload_rows_remote=74712 dup_rows=10214 dup_rows_remote=8972 dedup_ratio=0.103902 router_tokens_per_expert=[0, 54, 243, 64, 30, 3225, 0, 442, 0, 0, 54, 0, 63, 0, 0, 0, 1636, 0, 0, 0, 0, 1581, 0, 1155, 0, 83, 12, 1189, 0, 0, 0, 54, 5740, 1, 63, 570, 0, 0, 123, 1211, 180, 0, 0, 979, 62, 0, 0, 1, 401, 0, 4311, 66, 5972, 54, 5927, 0, 58, 3648, 4012, 613, 2707, 2383, 0, 185] payload_rows_per_dst=[12452, 12028, 7710, 13378, 12812, 10116, 9288, 10306] dup_rows_per_dst=[2960, 60, 1034, 1242, 2604, 352, 376, 1586] combine_rows_per_src=[12452, 12028, 7710, 13378, 12812, 10116, 9288, 10306] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=303841280 dispatch_meta_bytes=328416 combine_payload_bytes=303841280 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=304169696 ingress_bytes=373047296 link_bytes=373047296 token_bytes=608010976 weight_movement_bytes=69206016 total_nvlink_bytes=677216992 weight_to_token_ratio=0.113824 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=28288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.403746942e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=277356544 dispatch_meta_bytes=310000 combine_payload_bytes=277356544 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=277666544 ingress_bytes=450371584 link_bytes=450371584 token_bytes=555023088 weight_movement_bytes=173015040 total_nvlink_bytes=728038128 weight_to_token_ratio=0.311726 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=14464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.694712013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80628 payload_rows_remote=69538 dup_rows=17676 dup_rows_remote=15674 dedup_ratio=0.179810 router_tokens_per_expert=[0, 61, 252, 119, 9, 2890, 0, 437, 0, 0, 74, 0, 90, 0, 0, 0, 1740, 0, 0, 0, 0, 1624, 0, 1260, 0, 0, 20, 2788, 0, 0, 0, 61, 5183, 13, 90, 420, 0, 0, 158, 761, 218, 0, 0, 1546, 69, 0, 0, 0, 475, 0, 4227, 89, 6067, 61, 5811, 0, 6, 2778, 4104, 683, 1973, 2693, 0, 302] payload_rows_per_dst=[11090, 11896, 14142, 13806, 5540, 13182, 9344, 1628] dup_rows_per_dst=[2002, 54, 5472, 4066, 1290, 4078, 360, 354] combine_rows_per_src=[11090, 11896, 14142, 13806, 5540, 13182, 9344, 1628] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88184 payload_rows_remote=77970 dup_rows=10120 dup_rows_remote=9656 dedup_ratio=0.102946 router_tokens_per_expert=[0, 62, 263, 73, 30, 3686, 0, 489, 0, 0, 17, 0, 88, 0, 0, 0, 1403, 0, 0, 0, 0, 1248, 0, 1074, 0, 110, 2, 765, 0, 0, 0, 62, 5477, 17, 88, 698, 0, 0, 160, 1255, 306, 0, 0, 825, 72, 0, 0, 8, 307, 0, 4893, 107, 5554, 62, 6058, 0, 18, 3962, 4128, 715, 3047, 1827, 0, 196] payload_rows_per_dst=[9130, 12302, 6730, 12386, 12552, 10214, 10384, 14486] dup_rows_per_dst=[76, 24, 720, 600, 2838, 464, 354, 5044] combine_rows_per_src=[9130, 12302, 6730, 12386, 12552, 10214, 10384, 14486] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88134 payload_rows_remote=76256 dup_rows=10170 dup_rows_remote=8740 dedup_ratio=0.103455 router_tokens_per_expert=[0, 0, 54, 5, 54, 0, 0, 0, 0, 3060, 0, 154, 63, 254, 0, 1476, 2, 6974, 0, 100, 0, 0, 88, 0, 7226, 0, 328, 242, 0, 989, 61, 5034, 7548, 5710, 0, 366, 0, 0, 0, 0, 124, 85, 2145, 0, 0, 1376, 242, 0, 52, 0, 0, 1635, 0, 0, 0, 0, 0, 3528, 89, 0, 0, 0, 0, 88] payload_rows_per_dst=[14570, 8740, 14148, 11878, 13564, 7316, 10508, 7410] dup_rows_per_dst=[108, 1274, 180, 1430, 5018, 628, 1532, 0] combine_rows_per_src=[14570, 8740, 14148, 11878, 13564, 7316, 10508, 7410] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80044 payload_rows_remote=67968 dup_rows=18260 dup_rows_remote=13670 dedup_ratio=0.185750 router_tokens_per_expert=[2, 0, 63, 0, 63, 0, 0, 0, 0, 2781, 0, 215, 98, 116, 0, 1456, 0, 6918, 0, 120, 0, 0, 60, 0, 7182, 0, 439, 231, 0, 1221, 97, 4891, 7384, 5686, 0, 347, 0, 0, 0, 0, 214, 58, 1463, 0, 0, 1692, 231, 0, 64, 0, 0, 2227, 0, 0, 0, 0, 0, 3664, 109, 0, 0, 0, 0, 60] payload_rows_per_dst=[14494, 12076, 6804, 11914, 15834, 6698, 4558, 7666] dup_rows_per_dst=[126, 4590, 58, 1844, 11000, 618, 24, 0] combine_rows_per_src=[14494, 12076, 6804, 11914, 15834, 6698, 4558, 7666] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84160 payload_rows_remote=68598 dup_rows=14144 dup_rows_remote=11356 dedup_ratio=0.143880 router_tokens_per_expert=[0, 0, 51, 60, 51, 0, 0, 0, 0, 3080, 0, 192, 78, 271, 0, 1507, 0, 6925, 0, 309, 0, 0, 84, 0, 7158, 0, 280, 249, 0, 697, 78, 5112, 7488, 5299, 0, 386, 0, 0, 0, 0, 238, 84, 2101, 0, 0, 1425, 244, 0, 55, 0, 0, 1632, 0, 0, 0, 0, 0, 3843, 91, 0, 0, 0, 0, 84] payload_rows_per_dst=[12074, 8860, 14468, 4756, 772, 13662, 15562, 14006] dup_rows_per_dst=[102, 1396, 168, 316, 0, 4746, 2788, 4628] combine_rows_per_src=[12074, 8860, 14468, 4756, 772, 13662, 15562, 14006] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80198 payload_rows_remote=68548 dup_rows=18106 dup_rows_remote=15384 dedup_ratio=0.184184 router_tokens_per_expert=[40, 17, 2, 308, 170, 17, 27, 305, 422, 123, 1913, 161, 699, 395, 130, 86, 38, 45, 0, 11, 155, 118, 0, 197, 24, 4494, 26, 645, 338, 1569, 115, 369, 627, 3368, 0, 75, 0, 1277, 60, 1283, 3825, 1881, 3695, 180, 531, 879, 1624, 65, 3, 117, 267, 1563, 100, 179, 81, 43, 1321, 139, 4279, 4516, 3361, 712, 4, 138] payload_rows_per_dst=[10012, 11778, 11650, 12540, 10990, 12800, 4254, 6174] dup_rows_per_dst=[318, 3730, 2722, 2620, 2390, 4910, 452, 964] combine_rows_per_src=[10012, 11778, 11650, 12540, 10990, 12800, 4254, 6174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82130 payload_rows_remote=67714 dup_rows=16174 dup_rows_remote=9786 dedup_ratio=0.164530 router_tokens_per_expert=[3, 56, 261, 88, 21, 3013, 30, 462, 2, 6, 33, 2, 90, 16, 0, 9, 1320, 0, 1, 0, 1, 1322, 2, 1113, 6, 146, 64, 1342, 0, 9, 0, 56, 5684, 28, 90, 602, 0, 17, 151, 1055, 202, 15, 28, 1079, 63, 6, 16, 7, 360, 8, 4773, 87, 5634, 51, 6071, 3, 31, 3433, 4548, 717, 2300, 2337, 3, 279] payload_rows_per_dst=[7782, 12342, 6760, 3190, 12748, 11264, 13628, 14416] dup_rows_per_dst=[86, 90, 758, 56, 2506, 664, 5626, 6388] combine_rows_per_src=[7782, 12342, 6760, 3190, 12748, 11264, 13628, 14416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83958 payload_rows_remote=69726 dup_rows=14346 dup_rows_remote=14040 dedup_ratio=0.145935 router_tokens_per_expert=[127, 13, 65, 87, 48, 45, 13, 0, 30, 2734, 0, 141, 138, 705, 72, 780, 3, 6422, 169, 398, 8, 0, 159, 110, 6216, 256, 810, 267, 102, 1515, 81, 5230, 6400, 3772, 222, 526, 134, 109, 0, 96, 455, 68, 1505, 275, 15, 1962, 344, 402, 323, 8, 22, 1999, 20, 2, 50, 30, 172, 2794, 83, 21, 203, 305, 21, 70] payload_rows_per_dst=[13048, 8348, 14232, 13294, 14964, 8536, 4750, 6786] dup_rows_per_dst=[180, 852, 306, 3228, 7554, 1516, 158, 552] combine_rows_per_src=[13048, 8348, 14232, 13294, 14964, 8536, 4750, 6786] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=284827648 dispatch_meta_bytes=340848 combine_payload_bytes=284827648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=285168496 ingress_bytes=319430656 link_bytes=319430656 token_bytes=569996144 weight_movement_bytes=34603008 total_nvlink_bytes=604599152 weight_to_token_ratio=0.060707 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=24576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.201991842e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80044 payload_rows_remote=67968 dup_rows=18260 dup_rows_remote=13670 dedup_ratio=0.185750 router_tokens_per_expert=[2, 0, 63, 0, 63, 0, 0, 0, 0, 2781, 0, 215, 98, 116, 0, 1456, 0, 6918, 0, 120, 0, 0, 60, 0, 7182, 0, 439, 231, 0, 1221, 97, 4891, 7384, 5686, 0, 347, 0, 0, 0, 0, 214, 58, 1463, 0, 0, 1692, 231, 0, 64, 0, 0, 2227, 0, 0, 0, 0, 0, 3664, 109, 0, 0, 0, 0, 60] payload_rows_per_dst=[14494, 12076, 6804, 11914, 15834, 6698, 4558, 7666] dup_rows_per_dst=[126, 4590, 58, 1844, 11000, 618, 24, 0] combine_rows_per_src=[14494, 12076, 6804, 11914, 15834, 6698, 4558, 7666] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=306020352 dispatch_meta_bytes=334736 combine_payload_bytes=306020352 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=306355088 ingress_bytes=340623360 link_bytes=340623360 token_bytes=612375440 weight_movement_bytes=34603008 total_nvlink_bytes=646978448 weight_to_token_ratio=0.056506 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=35456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.281738281e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79168 payload_rows_remote=64674 dup_rows=19136 dup_rows_remote=19010 dedup_ratio=0.194661 router_tokens_per_expert=[3, 0, 61, 2, 61, 0, 0, 0, 0, 2544, 0, 212, 87, 87, 0, 1533, 10, 6904, 0, 102, 0, 0, 95, 0, 7183, 0, 631, 252, 0, 1049, 89, 5217, 7042, 6080, 0, 449, 0, 0, 0, 0, 394, 85, 1244, 0, 0, 1888, 252, 0, 40, 0, 0, 2072, 0, 0, 0, 0, 0, 3264, 125, 0, 0, 0, 0, 95] payload_rows_per_dst=[14494, 15464, 4254, 12756, 13906, 7106, 4220, 6968] dup_rows_per_dst=[126, 7270, 76, 2918, 8122, 620, 4, 0] combine_rows_per_src=[14494, 15464, 4254, 12756, 13906, 7106, 4220, 6968] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=319365120 dispatch_meta_bytes=350504 combine_payload_bytes=319365120 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=319715624 ingress_bytes=388571136 link_bytes=388571136 token_bytes=639080744 weight_movement_bytes=69206016 total_nvlink_bytes=708286760 weight_to_token_ratio=0.108290 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.030000 gemm_active_groups=11 gemm_peak_group_rows=33280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.462161902e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=312344576 dispatch_meta_bytes=339984 combine_payload_bytes=312344576 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=312684560 ingress_bytes=416153600 link_bytes=416153600 token_bytes=625029136 weight_movement_bytes=103809024 total_nvlink_bytes=728838160 weight_to_token_ratio=0.166087 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=26368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.565952494e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=280977408 dispatch_meta_bytes=319816 combine_payload_bytes=280977408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281297224 ingress_bytes=315580416 link_bytes=315580416 token_bytes=562274632 weight_movement_bytes=34603008 total_nvlink_bytes=596877640 weight_to_token_ratio=0.061541 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=34176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.187503699e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89590 payload_rows_remote=83756 dup_rows=8714 dup_rows_remote=8680 dedup_ratio=0.088643 router_tokens_per_expert=[0, 5, 56, 80, 63, 1, 3, 0, 1, 2149, 0, 213, 80, 681, 15, 1572, 3, 6834, 2, 495, 0, 3, 63, 0, 6748, 0, 327, 249, 1, 597, 80, 4794, 7442, 5468, 0, 468, 0, 1, 1, 0, 476, 63, 3025, 0, 0, 1653, 242, 0, 54, 0, 3, 1339, 0, 0, 0, 1, 0, 3706, 28, 5, 0, 0, 0, 62] payload_rows_per_dst=[13786, 7998, 14660, 10778, 5834, 10044, 15490, 11000] dup_rows_per_dst=[126, 1424, 140, 1318, 34, 874, 2188, 2610] combine_rows_per_src=[13786, 7998, 14660, 10778, 5834, 10044, 15490, 11000] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=280772608 dispatch_meta_bytes=335728 combine_payload_bytes=280772608 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=281108336 ingress_bytes=349978624 link_bytes=349978624 token_bytes=561880944 weight_movement_bytes=69206016 total_nvlink_bytes=631086960 weight_to_token_ratio=0.123168 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=44160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.316941387e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=277356544 dispatch_meta_bytes=310000 combine_payload_bytes=277356544 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=277666544 ingress_bytes=450371584 link_bytes=450371584 token_bytes=555023088 weight_movement_bytes=173015040 total_nvlink_bytes=728038128 weight_to_token_ratio=0.311726 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=14464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.694712013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=278396928 dispatch_meta_bytes=326552 combine_payload_bytes=278396928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278723480 ingress_bytes=312999936 link_bytes=312999936 token_bytes=557120408 weight_movement_bytes=34603008 total_nvlink_bytes=591723416 weight_to_token_ratio=0.062110 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=22272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.177793561e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82382 payload_rows_remote=68332 dup_rows=15922 dup_rows_remote=11350 dedup_ratio=0.161967 router_tokens_per_expert=[16, 2, 51, 56, 52, 1, 1, 0, 10, 3107, 0, 223, 91, 209, 25, 1390, 0, 6845, 15, 151, 0, 0, 58, 2, 7055, 11, 353, 251, 4, 758, 89, 5093, 7293, 5506, 12, 407, 9, 4, 0, 1, 493, 53, 1891, 21, 1, 1633, 256, 26, 59, 2, 0, 1745, 3, 1, 5, 7, 7, 3589, 130, 1, 9, 16, 1, 52] payload_rows_per_dst=[256, 8794, 14032, 15278, 854, 13946, 15172, 14050] dup_rows_per_dst=[102, 1316, 110, 1764, 12, 4988, 3058, 4572] combine_rows_per_src=[256, 8794, 14032, 15278, 854, 13946, 15172, 14050] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84160 payload_rows_remote=68598 dup_rows=14144 dup_rows_remote=11356 dedup_ratio=0.143880 router_tokens_per_expert=[0, 0, 51, 60, 51, 0, 0, 0, 0, 3080, 0, 192, 78, 271, 0, 1507, 0, 6925, 0, 309, 0, 0, 84, 0, 7158, 0, 280, 249, 0, 697, 78, 5112, 7488, 5299, 0, 386, 0, 0, 0, 0, 238, 84, 2101, 0, 0, 1425, 244, 0, 55, 0, 0, 1632, 0, 0, 0, 0, 0, 3843, 91, 0, 0, 0, 0, 84] payload_rows_per_dst=[12074, 8860, 14468, 4756, 772, 13662, 15562, 14006] dup_rows_per_dst=[102, 1396, 168, 316, 0, 4746, 2788, 4628] combine_rows_per_src=[12074, 8860, 14468, 4756, 772, 13662, 15562, 14006] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89590 payload_rows_remote=83756 dup_rows=8714 dup_rows_remote=8680 dedup_ratio=0.088643 router_tokens_per_expert=[0, 5, 56, 80, 63, 1, 3, 0, 1, 2149, 0, 213, 80, 681, 15, 1572, 3, 6834, 2, 495, 0, 3, 63, 0, 6748, 0, 327, 249, 1, 597, 80, 4794, 7442, 5468, 0, 468, 0, 1, 1, 0, 476, 63, 3025, 0, 0, 1653, 242, 0, 54, 0, 3, 1339, 0, 0, 0, 1, 0, 3706, 28, 5, 0, 0, 0, 62] payload_rows_per_dst=[13786, 7998, 14660, 10778, 5834, 10044, 15490, 11000] dup_rows_per_dst=[126, 1424, 140, 1318, 34, 874, 2188, 2610] combine_rows_per_src=[13786, 7998, 14660, 10778, 5834, 10044, 15490, 11000] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=264904704 dispatch_meta_bytes=334736 combine_payload_bytes=264904704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265239440 ingress_bytes=299507712 link_bytes=299507712 token_bytes=530144144 weight_movement_bytes=34603008 total_nvlink_bytes=564747152 weight_to_token_ratio=0.065271 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=47744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127023408e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=285597696 dispatch_meta_bytes=335064 combine_payload_bytes=285597696 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=285932760 ingress_bytes=354803712 link_bytes=354803712 token_bytes=571530456 weight_movement_bytes=69206016 total_nvlink_bytes=640736472 weight_to_token_ratio=0.121089 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=44288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.335097804e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=280977408 dispatch_meta_bytes=319816 combine_payload_bytes=280977408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281297224 ingress_bytes=315580416 link_bytes=315580416 token_bytes=562274632 weight_movement_bytes=34603008 total_nvlink_bytes=596877640 weight_to_token_ratio=0.061541 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=34176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.187503699e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=343064576 dispatch_meta_bytes=369744 combine_payload_bytes=343064576 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=343434320 ingress_bytes=481476608 link_bytes=481476608 token_bytes=686498896 weight_movement_bytes=138412032 total_nvlink_bytes=824910928 weight_to_token_ratio=0.201620 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=23168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.811757714e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=279887872 dispatch_meta_bytes=318728 combine_payload_bytes=279887872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280206600 ingress_bytes=314490880 link_bytes=314490880 token_bytes=560094472 weight_movement_bytes=34603008 total_nvlink_bytes=594697480 weight_to_token_ratio=0.061781 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=27904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.183403863e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79168 payload_rows_remote=64674 dup_rows=19136 dup_rows_remote=19010 dedup_ratio=0.194661 router_tokens_per_expert=[3, 0, 61, 2, 61, 0, 0, 0, 0, 2544, 0, 212, 87, 87, 0, 1533, 10, 6904, 0, 102, 0, 0, 95, 0, 7183, 0, 631, 252, 0, 1049, 89, 5217, 7042, 6080, 0, 449, 0, 0, 0, 0, 394, 85, 1244, 0, 0, 1888, 252, 0, 40, 0, 0, 2072, 0, 0, 0, 0, 0, 3264, 125, 0, 0, 0, 0, 95] payload_rows_per_dst=[14494, 15464, 4254, 12756, 13906, 7106, 4220, 6968] dup_rows_per_dst=[126, 7270, 76, 2918, 8122, 620, 4, 0] combine_rows_per_src=[14494, 15464, 4254, 12756, 13906, 7106, 4220, 6968] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88134 payload_rows_remote=76256 dup_rows=10170 dup_rows_remote=8740 dedup_ratio=0.103455 router_tokens_per_expert=[0, 0, 54, 5, 54, 0, 0, 0, 0, 3060, 0, 154, 63, 254, 0, 1476, 2, 6974, 0, 100, 0, 0, 88, 0, 7226, 0, 328, 242, 0, 989, 61, 5034, 7548, 5710, 0, 366, 0, 0, 0, 0, 124, 85, 2145, 0, 0, 1376, 242, 0, 52, 0, 0, 1635, 0, 0, 0, 0, 0, 3528, 89, 0, 0, 0, 0, 88] payload_rows_per_dst=[14570, 8740, 14148, 11878, 13564, 7316, 10508, 7410] dup_rows_per_dst=[108, 1274, 180, 1430, 5018, 628, 1532, 0] combine_rows_per_src=[14570, 8740, 14148, 11878, 13564, 7316, 10508, 7410] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85162 payload_rows_remote=72756 dup_rows=13142 dup_rows_remote=8542 dedup_ratio=0.133687 router_tokens_per_expert=[1, 0, 62, 76, 62, 0, 0, 0, 0, 2806, 0, 236, 88, 738, 0, 1083, 0, 6919, 0, 358, 0, 0, 57, 0, 7155, 0, 267, 264, 0, 647, 88, 4687, 7566, 5415, 0, 418, 0, 0, 0, 0, 194, 57, 2859, 0, 0, 1599, 257, 0, 151, 0, 0, 1429, 0, 0, 0, 0, 0, 3459, 97, 0, 0, 0, 0, 57] payload_rows_per_dst=[14588, 8630, 14554, 4480, 836, 12406, 15852, 13816] dup_rows_per_dst=[124, 1272, 114, 352, 0, 4600, 2440, 4240] combine_rows_per_src=[14588, 8630, 14554, 4480, 836, 12406, 15852, 13816] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84234 payload_rows_remote=70348 dup_rows=14070 dup_rows_remote=12974 dedup_ratio=0.143127 router_tokens_per_expert=[0, 0, 98, 1711, 0, 0, 1090, 0, 0, 3321, 814, 0, 0, 0, 726, 0, 179, 442, 0, 1758, 6, 83, 0, 0, 6042, 46, 0, 109, 0, 0, 0, 0, 569, 396, 0, 0, 1888, 1104, 0, 78, 2, 0, 5700, 0, 0, 1, 2, 0, 562, 0, 78, 0, 6773, 0, 14, 64, 6796, 0, 0, 0, 7080, 584, 1036, 0] payload_rows_per_dst=[5572, 8388, 4652, 12394, 12608, 11402, 13886, 15332] dup_rows_per_dst=[226, 1334, 284, 0, 4454, 8, 1096, 6668] combine_rows_per_src=[5572, 8388, 4652, 12394, 12608, 11402, 13886, 15332] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87020 payload_rows_remote=74716 dup_rows=11284 dup_rows_remote=11284 dedup_ratio=0.114787 router_tokens_per_expert=[0, 0, 89, 1655, 0, 0, 970, 0, 0, 3497, 826, 0, 0, 0, 753, 0, 174, 359, 0, 1778, 3, 88, 0, 0, 6019, 45, 0, 88, 0, 0, 0, 0, 564, 309, 0, 0, 1817, 1218, 0, 61, 1, 0, 5727, 0, 0, 0, 0, 0, 545, 0, 61, 0, 6928, 1, 12, 53, 6905, 0, 0, 0, 7176, 544, 886, 0] payload_rows_per_dst=[15702, 8846, 14876, 12304, 6854, 11454, 14128, 2856] dup_rows_per_dst=[3536, 1306, 4280, 0, 1084, 2, 1072, 4] combine_rows_per_src=[15702, 8846, 14876, 12304, 6854, 11454, 14128, 2856] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81296 payload_rows_remote=69866 dup_rows=17008 dup_rows_remote=13790 dedup_ratio=0.173014 router_tokens_per_expert=[0, 0, 110, 1197, 0, 1, 1230, 0, 0, 2643, 819, 0, 0, 0, 744, 0, 141, 474, 0, 1662, 0, 60, 0, 0, 5923, 5, 0, 249, 0, 0, 0, 0, 519, 335, 0, 0, 1964, 1429, 0, 97, 27, 0, 5801, 0, 0, 0, 0, 0, 588, 0, 97, 0, 7110, 3, 77, 29, 6951, 0, 0, 0, 7162, 512, 1193, 0] payload_rows_per_dst=[15454, 11430, 14802, 12354, 7420, 12712, 3718, 3406] dup_rows_per_dst=[3524, 3218, 4196, 0, 1268, 4570, 228, 4] combine_rows_per_src=[15454, 11430, 14802, 12354, 7420, 12712, 3718, 3406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82382 payload_rows_remote=68332 dup_rows=15922 dup_rows_remote=11350 dedup_ratio=0.161967 router_tokens_per_expert=[16, 2, 51, 56, 52, 1, 1, 0, 10, 3107, 0, 223, 91, 209, 25, 1390, 0, 6845, 15, 151, 0, 0, 58, 2, 7055, 11, 353, 251, 4, 758, 89, 5093, 7293, 5506, 12, 407, 9, 4, 0, 1, 493, 53, 1891, 21, 1, 1633, 256, 26, 59, 2, 0, 1745, 3, 1, 5, 7, 7, 3589, 130, 1, 9, 16, 1, 52] payload_rows_per_dst=[256, 8794, 14032, 15278, 854, 13946, 15172, 14050] dup_rows_per_dst=[102, 1316, 110, 1764, 12, 4988, 3058, 4572] combine_rows_per_src=[256, 8794, 14032, 15278, 854, 13946, 15172, 14050] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84118 payload_rows_remote=70550 dup_rows=14186 dup_rows_remote=9224 dedup_ratio=0.144307 router_tokens_per_expert=[44, 87, 305, 344, 57, 1939, 218, 542, 9, 33, 60, 87, 127, 345, 11, 51, 1406, 0, 14, 1, 49, 1215, 114, 1447, 47, 186, 453, 1580, 55, 41, 8, 118, 5220, 170, 84, 474, 2, 226, 157, 619, 212, 228, 446, 1044, 89, 23, 330, 47, 387, 14, 4202, 71, 5514, 49, 5744, 57, 40, 3191, 4080, 818, 1659, 2623, 76, 263] payload_rows_per_dst=[11260, 12482, 13568, 13050, 5804, 11110, 9208, 7636] dup_rows_per_dst=[2194, 452, 4962, 2954, 724, 1888, 352, 660] combine_rows_per_src=[11260, 12482, 13568, 13050, 5804, 11110, 9208, 7636] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=264904704 dispatch_meta_bytes=334736 combine_payload_bytes=264904704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265239440 ingress_bytes=299507712 link_bytes=299507712 token_bytes=530144144 weight_movement_bytes=34603008 total_nvlink_bytes=564747152 weight_to_token_ratio=0.065271 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=47744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127023408e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84716 payload_rows_remote=71262 dup_rows=13588 dup_rows_remote=9638 dedup_ratio=0.138224 router_tokens_per_expert=[44, 21, 122, 786, 43, 165, 1601, 0, 0, 1683, 1451, 15, 353, 477, 597, 21, 132, 420, 282, 1776, 82, 39, 8, 29, 5723, 37, 0, 324, 167, 15, 37, 74, 500, 354, 244, 165, 1408, 984, 84, 79, 160, 166, 5584, 0, 53, 108, 98, 496, 638, 238, 75, 1, 6784, 339, 107, 86, 5980, 6, 112, 57, 5934, 578, 1208, 2] payload_rows_per_dst=[13692, 7338, 13454, 12656, 6622, 12440, 14822, 3692] dup_rows_per_dst=[3832, 1856, 3950, 98, 1014, 890, 1714, 234] combine_rows_per_src=[13692, 7338, 13454, 12656, 6622, 12440, 14822, 3692] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81296 payload_rows_remote=69866 dup_rows=17008 dup_rows_remote=13790 dedup_ratio=0.173014 router_tokens_per_expert=[0, 0, 110, 1197, 0, 1, 1230, 0, 0, 2643, 819, 0, 0, 0, 744, 0, 141, 474, 0, 1662, 0, 60, 0, 0, 5923, 5, 0, 249, 0, 0, 0, 0, 519, 335, 0, 0, 1964, 1429, 0, 97, 27, 0, 5801, 0, 0, 0, 0, 0, 588, 0, 97, 0, 7110, 3, 77, 29, 6951, 0, 0, 0, 7162, 512, 1193, 0] payload_rows_per_dst=[15454, 11430, 14802, 12354, 7420, 12712, 3718, 3406] dup_rows_per_dst=[3524, 3218, 4196, 0, 1268, 4570, 228, 4] combine_rows_per_src=[15454, 11430, 14802, 12354, 7420, 12712, 3718, 3406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83378 payload_rows_remote=68156 dup_rows=14926 dup_rows_remote=11024 dedup_ratio=0.151835 router_tokens_per_expert=[0, 0, 124, 878, 0, 0, 1522, 0, 0, 2074, 857, 0, 0, 0, 721, 0, 185, 499, 0, 1327, 0, 95, 0, 0, 5923, 0, 0, 335, 0, 0, 0, 0, 597, 364, 0, 0, 1836, 1837, 0, 89, 68, 0, 5932, 0, 0, 0, 0, 0, 579, 0, 89, 0, 7082, 38, 69, 58, 7038, 0, 0, 0, 7162, 525, 1249, 0] payload_rows_per_dst=[15222, 15784, 14812, 12516, 8158, 11866, 1474, 3546] dup_rows_per_dst=[3902, 5684, 3724, 0, 1288, 134, 192, 2] combine_rows_per_src=[15222, 15784, 14812, 12516, 8158, 11866, 1474, 3546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=312344576 dispatch_meta_bytes=339984 combine_payload_bytes=312344576 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=312684560 ingress_bytes=416153600 link_bytes=416153600 token_bytes=625029136 weight_movement_bytes=103809024 total_nvlink_bytes=728838160 weight_to_token_ratio=0.166087 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=26368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.565952494e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=298008576 dispatch_meta_bytes=325192 combine_payload_bytes=298008576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=298333768 ingress_bytes=332611584 link_bytes=332611584 token_bytes=596342344 weight_movement_bytes=34603008 total_nvlink_bytes=630945352 weight_to_token_ratio=0.058025 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=16256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.251590613e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=288145408 dispatch_meta_bytes=333288 combine_payload_bytes=288145408 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=288478696 ingress_bytes=391954432 link_bytes=391954432 token_bytes=576624104 weight_movement_bytes=103809024 total_nvlink_bytes=680433128 weight_to_token_ratio=0.180029 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=41472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.474892973e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=306036736 dispatch_meta_bytes=344000 combine_payload_bytes=306036736 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=306380736 ingress_bytes=375242752 link_bytes=375242752 token_bytes=612417472 weight_movement_bytes=69206016 total_nvlink_bytes=681623488 weight_to_token_ratio=0.113005 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=47104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.412008266e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=279887872 dispatch_meta_bytes=318728 combine_payload_bytes=279887872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280206600 ingress_bytes=314490880 link_bytes=314490880 token_bytes=560094472 weight_movement_bytes=34603008 total_nvlink_bytes=594697480 weight_to_token_ratio=0.061781 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=27904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.183403863e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=288972800 dispatch_meta_bytes=319096 combine_payload_bytes=288972800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289291896 ingress_bytes=323575808 link_bytes=323575808 token_bytes=578264696 weight_movement_bytes=34603008 total_nvlink_bytes=612867704 weight_to_token_ratio=0.059839 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=15872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.217589716e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84234 payload_rows_remote=70348 dup_rows=14070 dup_rows_remote=12974 dedup_ratio=0.143127 router_tokens_per_expert=[0, 0, 98, 1711, 0, 0, 1090, 0, 0, 3321, 814, 0, 0, 0, 726, 0, 179, 442, 0, 1758, 6, 83, 0, 0, 6042, 46, 0, 109, 0, 0, 0, 0, 569, 396, 0, 0, 1888, 1104, 0, 78, 2, 0, 5700, 0, 0, 1, 2, 0, 562, 0, 78, 0, 6773, 0, 14, 64, 6796, 0, 0, 0, 7080, 584, 1036, 0] payload_rows_per_dst=[5572, 8388, 4652, 12394, 12608, 11402, 13886, 15332] dup_rows_per_dst=[226, 1334, 284, 0, 4454, 8, 1096, 6668] combine_rows_per_src=[5572, 8388, 4652, 12394, 12608, 11402, 13886, 15332] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=286171136 dispatch_meta_bytes=334624 combine_payload_bytes=286171136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286505760 ingress_bytes=320774144 link_bytes=320774144 token_bytes=572676896 weight_movement_bytes=34603008 total_nvlink_bytes=607279904 weight_to_token_ratio=0.060423 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=25088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.207047279e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87800 payload_rows_remote=73568 dup_rows=10504 dup_rows_remote=5800 dedup_ratio=0.106852 router_tokens_per_expert=[0, 0, 108, 2064, 0, 0, 911, 0, 0, 4251, 811, 0, 0, 0, 704, 2, 253, 306, 0, 2201, 48, 62, 0, 0, 6206, 75, 0, 82, 0, 0, 0, 0, 526, 353, 1, 0, 1630, 1152, 1, 80, 13, 0, 5178, 0, 0, 0, 1, 0, 580, 0, 80, 0, 6531, 2, 2, 83, 6542, 0, 0, 0, 6878, 585, 850, 0] payload_rows_per_dst=[6416, 10228, 6900, 12752, 14232, 10374, 13398, 13500] dup_rows_per_dst=[464, 1308, 1146, 122, 4704, 10, 1158, 1592] combine_rows_per_src=[6416, 10228, 6900, 12752, 14232, 10374, 13398, 13500] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77532 payload_rows_remote=61398 dup_rows=20772 dup_rows_remote=5796 dedup_ratio=0.211304 router_tokens_per_expert=[1, 1, 130, 1280, 1, 10, 1024, 0, 0, 2965, 926, 5, 20, 28, 724, 1, 140, 410, 17, 2021, 6, 54, 1, 2, 5892, 54, 0, 169, 7, 1, 0, 1, 595, 388, 16, 13, 1706, 1322, 6, 89, 52, 10, 5776, 0, 3, 3, 11, 27, 580, 7, 88, 0, 6891, 36, 18, 69, 6894, 0, 8, 1, 7158, 532, 962, 0] payload_rows_per_dst=[4748, 7996, 5042, 10416, 7202, 11634, 14360, 16134] dup_rows_per_dst=[146, 1342, 260, 4, 1068, 130, 2846, 14976] combine_rows_per_src=[4748, 7996, 5042, 10416, 7202, 11634, 14360, 16134] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=279166976 dispatch_meta_bytes=316720 combine_payload_bytes=279166976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279483696 ingress_bytes=313769984 link_bytes=313769984 token_bytes=558650672 weight_movement_bytes=34603008 total_nvlink_bytes=593253680 weight_to_token_ratio=0.061940 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=27264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.180691189e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=291889152 dispatch_meta_bytes=323600 combine_payload_bytes=291889152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=292212752 ingress_bytes=326492160 link_bytes=326492160 token_bytes=584101904 weight_movement_bytes=34603008 total_nvlink_bytes=618704912 weight_to_token_ratio=0.059241 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=28672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.228563713e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=288145408 dispatch_meta_bytes=333288 combine_payload_bytes=288145408 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=288478696 ingress_bytes=391954432 link_bytes=391954432 token_bytes=576624104 weight_movement_bytes=103809024 total_nvlink_bytes=680433128 weight_to_token_ratio=0.180029 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=41472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.474892973e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=301334528 dispatch_meta_bytes=317472 combine_payload_bytes=301334528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=301652000 ingress_bytes=335937536 link_bytes=335937536 token_bytes=602986528 weight_movement_bytes=34603008 total_nvlink_bytes=637589536 weight_to_token_ratio=0.057386 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264105903e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=251486208 dispatch_meta_bytes=268776 combine_payload_bytes=251486208 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=251754984 ingress_bytes=389898240 link_bytes=389898240 token_bytes=503241192 weight_movement_bytes=138412032 total_nvlink_bytes=641653224 weight_to_token_ratio=0.275041 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=26752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.467155688e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83378 payload_rows_remote=68156 dup_rows=14926 dup_rows_remote=11024 dedup_ratio=0.151835 router_tokens_per_expert=[0, 0, 124, 878, 0, 0, 1522, 0, 0, 2074, 857, 0, 0, 0, 721, 0, 185, 499, 0, 1327, 0, 95, 0, 0, 5923, 0, 0, 335, 0, 0, 0, 0, 597, 364, 0, 0, 1836, 1837, 0, 89, 68, 0, 5932, 0, 0, 0, 0, 0, 579, 0, 89, 0, 7082, 38, 69, 58, 7038, 0, 0, 0, 7162, 525, 1249, 0] payload_rows_per_dst=[15222, 15784, 14812, 12516, 8158, 11866, 1474, 3546] dup_rows_per_dst=[3902, 5684, 3724, 0, 1288, 134, 192, 2] combine_rows_per_src=[15222, 15784, 14812, 12516, 8158, 11866, 1474, 3546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87020 payload_rows_remote=74716 dup_rows=11284 dup_rows_remote=11284 dedup_ratio=0.114787 router_tokens_per_expert=[0, 0, 89, 1655, 0, 0, 970, 0, 0, 3497, 826, 0, 0, 0, 753, 0, 174, 359, 0, 1778, 3, 88, 0, 0, 6019, 45, 0, 88, 0, 0, 0, 0, 564, 309, 0, 0, 1817, 1218, 0, 61, 1, 0, 5727, 0, 0, 0, 0, 0, 545, 0, 61, 0, 6928, 1, 12, 53, 6905, 0, 0, 0, 7176, 544, 886, 0] payload_rows_per_dst=[15702, 8846, 14876, 12304, 6854, 11454, 14128, 2856] dup_rows_per_dst=[3536, 1306, 4280, 0, 1084, 2, 1072, 4] combine_rows_per_src=[15702, 8846, 14876, 12304, 6854, 11454, 14128, 2856] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89100 payload_rows_remote=78424 dup_rows=9204 dup_rows_remote=9202 dedup_ratio=0.093628 router_tokens_per_expert=[0, 0, 104, 2011, 0, 0, 819, 0, 0, 4642, 829, 0, 0, 0, 652, 0, 186, 334, 0, 2386, 0, 57, 0, 0, 6185, 71, 0, 88, 0, 0, 0, 0, 148, 389, 0, 0, 1674, 993, 0, 88, 0, 0, 5338, 0, 0, 1, 0, 0, 567, 0, 88, 0, 6371, 0, 0, 80, 6837, 0, 0, 0, 6872, 554, 788, 0] payload_rows_per_dst=[5718, 10870, 5630, 12688, 15736, 10676, 13094, 14688] dup_rows_per_dst=[150, 1376, 296, 0, 4592, 2, 1118, 1670] combine_rows_per_src=[5718, 10870, 5630, 12688, 15736, 10676, 13094, 14688] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78990 payload_rows_remote=67126 dup_rows=19314 dup_rows_remote=18974 dedup_ratio=0.196472 router_tokens_per_expert=[3432, 0, 0, 149, 802, 3434, 0, 0, 1428, 69, 2015, 0, 0, 0, 0, 0, 0, 78, 121, 0, 0, 78, 0, 0, 110, 0, 0, 0, 1, 78, 0, 18, 0, 0, 6962, 0, 0, 0, 1706, 513, 0, 6929, 847, 7596, 0, 0, 4800, 73, 0, 0, 0, 78, 90, 0, 0, 5683, 589, 0, 0, 372, 644, 186, 244, 27] payload_rows_per_dst=[7098, 6014, 10272, 4618, 10026, 14052, 11864, 15046] dup_rows_per_dst=[1166, 1010, 184, 108, 1280, 12224, 340, 3002] combine_rows_per_src=[7098, 6014, 10272, 4618, 10026, 14052, 11864, 15046] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86920 payload_rows_remote=71618 dup_rows=11384 dup_rows_remote=11022 dedup_ratio=0.115804 router_tokens_per_expert=[3009, 0, 0, 113, 840, 3487, 0, 0, 1499, 62, 2367, 0, 0, 0, 0, 0, 0, 63, 116, 0, 0, 61, 0, 0, 118, 0, 0, 0, 0, 63, 0, 22, 0, 0, 6907, 0, 0, 0, 1477, 744, 0, 6924, 724, 7629, 0, 0, 4789, 58, 0, 0, 0, 63, 75, 0, 0, 5873, 543, 0, 0, 356, 604, 177, 242, 147] payload_rows_per_dst=[12488, 13840, 14098, 15302, 9720, 1564, 12022, 7886] dup_rows_per_dst=[2410, 3594, 230, 362, 2462, 0, 0, 2326] combine_rows_per_src=[12488, 13840, 14098, 15302, 9720, 1564, 12022, 7886] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86682 payload_rows_remote=73886 dup_rows=11622 dup_rows_remote=9688 dedup_ratio=0.118225 router_tokens_per_expert=[3312, 0, 0, 104, 762, 4347, 0, 0, 1410, 82, 1605, 0, 0, 0, 0, 0, 0, 98, 146, 0, 0, 97, 0, 0, 132, 0, 0, 0, 0, 98, 0, 80, 0, 0, 6764, 0, 0, 0, 1559, 588, 0, 6953, 1080, 7601, 0, 0, 4268, 54, 0, 0, 0, 98, 25, 0, 0, 5808, 363, 0, 0, 334, 593, 170, 231, 390] payload_rows_per_dst=[13430, 12796, 14266, 15362, 14426, 2268, 11862, 2272] dup_rows_per_dst=[3620, 1934, 322, 460, 3396, 0, 0, 1890] combine_rows_per_src=[13430, 12796, 14266, 15362, 14426, 2268, 11862, 2272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77532 payload_rows_remote=61398 dup_rows=20772 dup_rows_remote=5796 dedup_ratio=0.211304 router_tokens_per_expert=[1, 1, 130, 1280, 1, 10, 1024, 0, 0, 2965, 926, 5, 20, 28, 724, 1, 140, 410, 17, 2021, 6, 54, 1, 2, 5892, 54, 0, 169, 7, 1, 0, 1, 595, 388, 16, 13, 1706, 1322, 6, 89, 52, 10, 5776, 0, 3, 3, 11, 27, 580, 7, 88, 0, 6891, 36, 18, 69, 6894, 0, 8, 1, 7158, 532, 962, 0] payload_rows_per_dst=[4748, 7996, 5042, 10416, 7202, 11634, 14360, 16134] dup_rows_per_dst=[146, 1342, 260, 4, 1068, 130, 2846, 14976] combine_rows_per_src=[4748, 7996, 5042, 10416, 7202, 11634, 14360, 16134] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83958 payload_rows_remote=69726 dup_rows=14346 dup_rows_remote=14040 dedup_ratio=0.145935 router_tokens_per_expert=[127, 13, 65, 87, 48, 45, 13, 0, 30, 2734, 0, 141, 138, 705, 72, 780, 3, 6422, 169, 398, 8, 0, 159, 110, 6216, 256, 810, 267, 102, 1515, 81, 5230, 6400, 3772, 222, 526, 134, 109, 0, 96, 455, 68, 1505, 275, 15, 1962, 344, 402, 323, 8, 22, 1999, 20, 2, 50, 30, 172, 2794, 83, 21, 203, 305, 21, 70] payload_rows_per_dst=[13048, 8348, 14232, 13294, 14964, 8536, 4750, 6786] dup_rows_per_dst=[180, 852, 306, 3228, 7554, 1516, 158, 552] combine_rows_per_src=[13048, 8348, 14232, 13294, 14964, 8536, 4750, 6786] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=279166976 dispatch_meta_bytes=316720 combine_payload_bytes=279166976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279483696 ingress_bytes=313769984 link_bytes=313769984 token_bytes=558650672 weight_movement_bytes=34603008 total_nvlink_bytes=593253680 weight_to_token_ratio=0.061940 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=27264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.180691189e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86682 payload_rows_remote=73886 dup_rows=11622 dup_rows_remote=9688 dedup_ratio=0.118225 router_tokens_per_expert=[3312, 0, 0, 104, 762, 4347, 0, 0, 1410, 82, 1605, 0, 0, 0, 0, 0, 0, 98, 146, 0, 0, 97, 0, 0, 132, 0, 0, 0, 0, 98, 0, 80, 0, 0, 6764, 0, 0, 0, 1559, 588, 0, 6953, 1080, 7601, 0, 0, 4268, 54, 0, 0, 0, 98, 25, 0, 0, 5808, 363, 0, 0, 334, 593, 170, 231, 390] payload_rows_per_dst=[13430, 12796, 14266, 15362, 14426, 2268, 11862, 2272] dup_rows_per_dst=[3620, 1934, 322, 460, 3396, 0, 0, 1890] combine_rows_per_src=[13430, 12796, 14266, 15362, 14426, 2268, 11862, 2272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87120 payload_rows_remote=73408 dup_rows=11184 dup_rows_remote=7826 dedup_ratio=0.113770 router_tokens_per_expert=[2805, 0, 0, 90, 755, 4885, 0, 0, 1339, 69, 1470, 0, 0, 0, 0, 0, 0, 89, 130, 0, 0, 89, 0, 0, 246, 0, 0, 0, 12, 90, 0, 57, 0, 0, 6725, 0, 0, 0, 1380, 726, 0, 6934, 1425, 7590, 0, 0, 4208, 0, 0, 0, 0, 90, 1, 0, 0, 5723, 380, 0, 0, 344, 593, 219, 252, 436] payload_rows_per_dst=[13712, 12518, 14184, 15318, 14384, 2850, 11626, 2528] dup_rows_per_dst=[3358, 1654, 300, 672, 3278, 0, 2, 1920] combine_rows_per_src=[13712, 12518, 14184, 15318, 14384, 2850, 11626, 2528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=306036736 dispatch_meta_bytes=344000 combine_payload_bytes=306036736 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=306380736 ingress_bytes=375242752 link_bytes=375242752 token_bytes=612417472 weight_movement_bytes=69206016 total_nvlink_bytes=681623488 weight_to_token_ratio=0.113005 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=47104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.412008266e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84920 payload_rows_remote=70986 dup_rows=13384 dup_rows_remote=12468 dedup_ratio=0.136149 router_tokens_per_expert=[1921, 12, 564, 75, 699, 5533, 175, 64, 1643, 176, 980, 0, 32, 48, 20, 22, 1, 66, 491, 378, 81, 78, 197, 54, 154, 11, 32, 175, 57, 84, 25, 52, 209, 1, 5055, 0, 5, 109, 2077, 544, 1, 6079, 1183, 6711, 61, 299, 4245, 86, 56, 218, 31, 69, 10, 94, 254, 4040, 1015, 519, 15, 581, 606, 104, 303, 672] payload_rows_per_dst=[14656, 12362, 13934, 14152, 12766, 3236, 9176, 4638] dup_rows_per_dst=[3430, 1970, 916, 450, 3234, 24, 368, 2992] combine_rows_per_src=[14656, 12362, 13934, 14152, 12766, 3236, 9176, 4638] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=321224704 dispatch_meta_bytes=350504 combine_payload_bytes=321224704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=321575208 ingress_bytes=355827712 link_bytes=355827712 token_bytes=642799912 weight_movement_bytes=34603008 total_nvlink_bytes=677402920 weight_to_token_ratio=0.053832 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=45056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.338951034e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=251486208 dispatch_meta_bytes=268776 combine_payload_bytes=251486208 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=251754984 ingress_bytes=389898240 link_bytes=389898240 token_bytes=503241192 weight_movement_bytes=138412032 total_nvlink_bytes=641653224 weight_to_token_ratio=0.275041 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=26752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.467155688e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78990 payload_rows_remote=67126 dup_rows=19314 dup_rows_remote=18974 dedup_ratio=0.196472 router_tokens_per_expert=[3432, 0, 0, 149, 802, 3434, 0, 0, 1428, 69, 2015, 0, 0, 0, 0, 0, 0, 78, 121, 0, 0, 78, 0, 0, 110, 0, 0, 0, 1, 78, 0, 18, 0, 0, 6962, 0, 0, 0, 1706, 513, 0, 6929, 847, 7596, 0, 0, 4800, 73, 0, 0, 0, 78, 90, 0, 0, 5683, 589, 0, 0, 372, 644, 186, 244, 27] payload_rows_per_dst=[7098, 6014, 10272, 4618, 10026, 14052, 11864, 15046] dup_rows_per_dst=[1166, 1010, 184, 108, 1280, 12224, 340, 3002] combine_rows_per_src=[7098, 6014, 10272, 4618, 10026, 14052, 11864, 15046] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=285597696 dispatch_meta_bytes=335064 combine_payload_bytes=285597696 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=285932760 ingress_bytes=354803712 link_bytes=354803712 token_bytes=571530456 weight_movement_bytes=69206016 total_nvlink_bytes=640736472 weight_to_token_ratio=0.121089 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=44288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.335097804e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88828 payload_rows_remote=83564 dup_rows=9476 dup_rows_remote=8418 dedup_ratio=0.096395 router_tokens_per_expert=[2611, 2, 0, 147, 885, 2223, 0, 1, 1796, 61, 2839, 0, 0, 0, 0, 0, 0, 80, 114, 0, 0, 80, 0, 0, 68, 0, 0, 0, 5, 80, 0, 7, 0, 0, 7196, 0, 0, 2, 2342, 817, 0, 6873, 637, 7597, 3, 0, 4837, 81, 0, 0, 0, 80, 109, 0, 29, 5692, 448, 0, 0, 355, 622, 189, 242, 2] payload_rows_per_dst=[10376, 10988, 14030, 15218, 5264, 5950, 11818, 15184] dup_rows_per_dst=[1362, 3270, 264, 296, 1058, 300, 2, 2924] combine_rows_per_src=[10376, 10988, 14030, 15218, 5264, 5950, 11818, 15184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=274948096 dispatch_meta_bytes=344400 combine_payload_bytes=274948096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275292496 ingress_bytes=309551104 link_bytes=309551104 token_bytes=550240592 weight_movement_bytes=34603008 total_nvlink_bytes=584843600 weight_to_token_ratio=0.062887 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=44288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.164815883e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=302637056 dispatch_meta_bytes=334296 combine_payload_bytes=302637056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=302971352 ingress_bytes=337240064 link_bytes=337240064 token_bytes=605608408 weight_movement_bytes=34603008 total_nvlink_bytes=640211416 weight_to_token_ratio=0.057138 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=19968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.269007211e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=321224704 dispatch_meta_bytes=350504 combine_payload_bytes=321224704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=321575208 ingress_bytes=355827712 link_bytes=355827712 token_bytes=642799912 weight_movement_bytes=34603008 total_nvlink_bytes=677402920 weight_to_token_ratio=0.053832 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=45056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.338951034e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=60722 payload_rows_remote=46096 dup_rows=37582 dup_rows_remote=34354 dedup_ratio=0.382304 router_tokens_per_expert=[2837, 1, 32, 93, 810, 3882, 5, 4, 1396, 64, 1894, 0, 3, 3, 1, 0, 0, 87, 145, 28, 0, 88, 8, 4, 95, 1, 4, 9, 7, 88, 1, 57, 14, 0, 6649, 0, 0, 6, 1587, 776, 4, 6821, 1196, 7496, 6, 20, 4656, 88, 3, 16, 0, 87, 45, 4, 26, 5727, 675, 26, 1, 365, 598, 182, 256, 175] payload_rows_per_dst=[1698, 5872, 690, 522, 7160, 16122, 14032, 14626] dup_rows_per_dst=[192, 850, 30, 2, 1462, 28360, 3458, 3228] combine_rows_per_src=[1698, 5872, 690, 522, 7160, 16122, 14032, 14626] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=293347328 dispatch_meta_bytes=330560 combine_payload_bytes=293347328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293677888 ingress_bytes=327950336 link_bytes=327950336 token_bytes=587025216 weight_movement_bytes=34603008 total_nvlink_bytes=621628224 weight_to_token_ratio=0.058946 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=46976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.234050712e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=300679168 dispatch_meta_bytes=324936 combine_payload_bytes=300679168 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=301004104 ingress_bytes=404488192 link_bytes=404488192 token_bytes=601683272 weight_movement_bytes=103809024 total_nvlink_bytes=705492296 weight_to_token_ratio=0.172531 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=21120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522056503e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=290758656 dispatch_meta_bytes=333816 combine_payload_bytes=290758656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291092472 ingress_bytes=325361664 link_bytes=325361664 token_bytes=581851128 weight_movement_bytes=34603008 total_nvlink_bytes=616454136 weight_to_token_ratio=0.059471 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=45696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.224309748e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=274948096 dispatch_meta_bytes=344400 combine_payload_bytes=274948096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275292496 ingress_bytes=309551104 link_bytes=309551104 token_bytes=550240592 weight_movement_bytes=34603008 total_nvlink_bytes=584843600 weight_to_token_ratio=0.062887 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=44288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.164815883e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=342278144 dispatch_meta_bytes=367928 combine_payload_bytes=342278144 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=342646072 ingress_bytes=411484160 link_bytes=411484160 token_bytes=684924216 weight_movement_bytes=69206016 total_nvlink_bytes=754130232 weight_to_token_ratio=0.101042 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=22528 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.548381767e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=188809216 dispatch_meta_bytes=321800 combine_payload_bytes=188809216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=189131016 ingress_bytes=223412224 link_bytes=223412224 token_bytes=377940232 weight_movement_bytes=34603008 total_nvlink_bytes=412543240 weight_to_token_ratio=0.091557 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=31104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.406822128e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87120 payload_rows_remote=73408 dup_rows=11184 dup_rows_remote=7826 dedup_ratio=0.113770 router_tokens_per_expert=[2805, 0, 0, 90, 755, 4885, 0, 0, 1339, 69, 1470, 0, 0, 0, 0, 0, 0, 89, 130, 0, 0, 89, 0, 0, 246, 0, 0, 0, 12, 90, 0, 57, 0, 0, 6725, 0, 0, 0, 1380, 726, 0, 6934, 1425, 7590, 0, 0, 4208, 0, 0, 0, 0, 90, 1, 0, 0, 5723, 380, 0, 0, 344, 593, 219, 252, 436] payload_rows_per_dst=[13712, 12518, 14184, 15318, 14384, 2850, 11626, 2528] dup_rows_per_dst=[3358, 1654, 300, 672, 3278, 0, 2, 1920] combine_rows_per_src=[13712, 12518, 14184, 15318, 14384, 2850, 11626, 2528] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84180 payload_rows_remote=70162 dup_rows=14124 dup_rows_remote=12430 dedup_ratio=0.143677 router_tokens_per_expert=[4930, 471, 0, 0, 0, 0, 1801, 68, 0, 2958, 0, 4373, 0, 0, 1146, 0, 0, 156, 2675, 0, 0, 401, 0, 0, 0, 0, 2568, 661, 1824, 0, 287, 406, 0, 0, 0, 0, 0, 0, 0, 5079, 0, 0, 417, 120, 2840, 174, 7339, 0, 6790, 62, 0, 173, 0, 78, 0, 0, 70, 0, 0, 0, 309, 78, 0, 898] payload_rows_per_dst=[3930, 13366, 12372, 9846, 10158, 5396, 14018, 15094] dup_rows_per_dst=[750, 4702, 2838, 1646, 0, 200, 1694, 2294] combine_rows_per_src=[3930, 13366, 12372, 9846, 10158, 5396, 14018, 15094] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78498 payload_rows_remote=67268 dup_rows=19806 dup_rows_remote=17650 dedup_ratio=0.201477 router_tokens_per_expert=[4912, 822, 0, 0, 0, 0, 1906, 97, 0, 2420, 0, 4136, 0, 0, 710, 0, 0, 195, 1959, 0, 0, 368, 0, 0, 0, 0, 3517, 613, 1715, 0, 317, 746, 0, 0, 0, 0, 0, 0, 0, 4597, 0, 0, 577, 143, 3142, 133, 7293, 0, 6671, 6, 0, 162, 0, 97, 0, 0, 82, 0, 0, 0, 732, 98, 0, 986] payload_rows_per_dst=[12242, 11230, 5908, 11174, 11974, 15724, 6872, 3374] dup_rows_per_dst=[3232, 2156, 282, 2642, 3948, 6852, 272, 422] combine_rows_per_src=[12242, 11230, 5908, 11174, 11974, 15724, 6872, 3374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86920 payload_rows_remote=71618 dup_rows=11384 dup_rows_remote=11022 dedup_ratio=0.115804 router_tokens_per_expert=[3009, 0, 0, 113, 840, 3487, 0, 0, 1499, 62, 2367, 0, 0, 0, 0, 0, 0, 63, 116, 0, 0, 61, 0, 0, 118, 0, 0, 0, 0, 63, 0, 22, 0, 0, 6907, 0, 0, 0, 1477, 744, 0, 6924, 724, 7629, 0, 0, 4789, 58, 0, 0, 0, 63, 75, 0, 0, 5873, 543, 0, 0, 356, 604, 177, 242, 147] payload_rows_per_dst=[12488, 13840, 14098, 15302, 9720, 1564, 12022, 7886] dup_rows_per_dst=[2410, 3594, 230, 362, 2462, 0, 0, 2326] combine_rows_per_src=[12488, 13840, 14098, 15302, 9720, 1564, 12022, 7886] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=60722 payload_rows_remote=46096 dup_rows=37582 dup_rows_remote=34354 dedup_ratio=0.382304 router_tokens_per_expert=[2837, 1, 32, 93, 810, 3882, 5, 4, 1396, 64, 1894, 0, 3, 3, 1, 0, 0, 87, 145, 28, 0, 88, 8, 4, 95, 1, 4, 9, 7, 88, 1, 57, 14, 0, 6649, 0, 0, 6, 1587, 776, 4, 6821, 1196, 7496, 6, 20, 4656, 88, 3, 16, 0, 87, 45, 4, 26, 5727, 675, 26, 1, 365, 598, 182, 256, 175] payload_rows_per_dst=[1698, 5872, 690, 522, 7160, 16122, 14032, 14626] dup_rows_per_dst=[192, 850, 30, 2, 1462, 28360, 3458, 3228] combine_rows_per_src=[1698, 5872, 690, 522, 7160, 16122, 14032, 14626] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90334 payload_rows_remote=79600 dup_rows=7970 dup_rows_remote=7350 dedup_ratio=0.081075 router_tokens_per_expert=[3112, 0, 0, 225, 736, 2429, 0, 0, 1520, 72, 3224, 0, 0, 0, 0, 0, 0, 88, 135, 0, 0, 88, 0, 0, 74, 0, 0, 0, 0, 88, 0, 2, 0, 0, 7158, 0, 0, 0, 1619, 727, 0, 6981, 656, 7567, 0, 0, 4937, 84, 0, 0, 0, 88, 101, 0, 0, 5678, 316, 0, 0, 368, 600, 164, 257, 58] payload_rows_per_dst=[8190, 8164, 14282, 15138, 7040, 10734, 11734, 15052] dup_rows_per_dst=[1118, 1468, 302, 324, 1348, 620, 0, 2790] combine_rows_per_src=[8190, 8164, 14282, 15138, 7040, 10734, 11734, 15052] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87656 payload_rows_remote=77400 dup_rows=10648 dup_rows_remote=8762 dedup_ratio=0.108317 router_tokens_per_expert=[5082, 491, 0, 0, 0, 0, 1781, 60, 0, 3165, 0, 4544, 0, 0, 1111, 0, 0, 124, 2175, 0, 0, 356, 0, 0, 0, 0, 2979, 688, 1699, 0, 272, 433, 0, 0, 0, 0, 0, 0, 0, 4923, 0, 0, 426, 113, 2704, 142, 7373, 0, 6791, 40, 0, 158, 0, 61, 0, 0, 62, 0, 0, 0, 471, 62, 0, 866] payload_rows_per_dst=[12126, 8384, 12076, 10256, 9846, 7930, 13714, 13324] dup_rows_per_dst=[2702, 168, 2322, 1886, 0, 1038, 386, 2146] combine_rows_per_src=[12126, 8384, 12076, 10256, 9846, 7930, 13714, 13324] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=300679168 dispatch_meta_bytes=324936 combine_payload_bytes=300679168 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=301004104 ingress_bytes=404488192 link_bytes=404488192 token_bytes=601683272 weight_movement_bytes=103809024 total_nvlink_bytes=705492296 weight_to_token_ratio=0.172531 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=21120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522056503e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84716 payload_rows_remote=71262 dup_rows=13588 dup_rows_remote=9638 dedup_ratio=0.138224 router_tokens_per_expert=[44, 21, 122, 786, 43, 165, 1601, 0, 0, 1683, 1451, 15, 353, 477, 597, 21, 132, 420, 282, 1776, 82, 39, 8, 29, 5723, 37, 0, 324, 167, 15, 37, 74, 500, 354, 244, 165, 1408, 984, 84, 79, 160, 166, 5584, 0, 53, 108, 98, 496, 638, 238, 75, 1, 6784, 339, 107, 86, 5980, 6, 112, 57, 5934, 578, 1208, 2] payload_rows_per_dst=[13692, 7338, 13454, 12656, 6622, 12440, 14822, 3692] dup_rows_per_dst=[3832, 1856, 3950, 98, 1014, 890, 1714, 234] combine_rows_per_src=[13692, 7338, 13454, 12656, 6622, 12440, 14822, 3692] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=61332 dup_rows=24072 dup_rows_remote=19804 dedup_ratio=0.244873 router_tokens_per_expert=[5020, 1166, 0, 0, 0, 0, 2120, 87, 0, 2422, 0, 3976, 0, 0, 465, 0, 0, 179, 1635, 0, 0, 373, 0, 0, 0, 0, 3883, 628, 1416, 0, 273, 857, 0, 0, 0, 0, 0, 0, 0, 4190, 0, 0, 633, 99, 3486, 103, 7306, 0, 6667, 0, 0, 195, 0, 89, 0, 0, 69, 0, 0, 0, 707, 72, 0, 1036] payload_rows_per_dst=[12900, 11556, 4374, 11390, 14616, 15542, 390, 3464] dup_rows_per_dst=[4268, 2170, 0, 2724, 7098, 7330, 178, 304] combine_rows_per_src=[12900, 11556, 4374, 11390, 14616, 15542, 390, 3464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79432 payload_rows_remote=69302 dup_rows=18872 dup_rows_remote=17230 dedup_ratio=0.191976 router_tokens_per_expert=[4409, 1620, 19, 4, 115, 101, 1625, 93, 89, 2229, 234, 3177, 68, 29, 458, 1, 105, 180, 1673, 42, 134, 393, 44, 138, 15, 69, 2172, 571, 2212, 187, 262, 1058, 567, 262, 69, 0, 464, 17, 7, 3461, 12, 443, 1417, 133, 3396, 180, 6267, 202, 5704, 89, 33, 168, 0, 76, 7, 306, 81, 23, 2, 541, 856, 96, 23, 724] payload_rows_per_dst=[12006, 6002, 10130, 10536, 8508, 15430, 12378, 4442] dup_rows_per_dst=[3966, 214, 1642, 2556, 1186, 8670, 388, 250] combine_rows_per_src=[12006, 6002, 10130, 10536, 8508, 15430, 12378, 4442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78498 payload_rows_remote=67268 dup_rows=19806 dup_rows_remote=17650 dedup_ratio=0.201477 router_tokens_per_expert=[4912, 822, 0, 0, 0, 0, 1906, 97, 0, 2420, 0, 4136, 0, 0, 710, 0, 0, 195, 1959, 0, 0, 368, 0, 0, 0, 0, 3517, 613, 1715, 0, 317, 746, 0, 0, 0, 0, 0, 0, 0, 4597, 0, 0, 577, 143, 3142, 133, 7293, 0, 6671, 6, 0, 162, 0, 97, 0, 0, 82, 0, 0, 0, 732, 98, 0, 986] payload_rows_per_dst=[12242, 11230, 5908, 11174, 11974, 15724, 6872, 3374] dup_rows_per_dst=[3232, 2156, 282, 2642, 3948, 6852, 272, 422] combine_rows_per_src=[12242, 11230, 5908, 11174, 11974, 15724, 6872, 3374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=293347328 dispatch_meta_bytes=330560 combine_payload_bytes=293347328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293677888 ingress_bytes=327950336 link_bytes=327950336 token_bytes=587025216 weight_movement_bytes=34603008 total_nvlink_bytes=621628224 weight_to_token_ratio=0.058946 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=46976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.234050712e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84180 payload_rows_remote=70162 dup_rows=14124 dup_rows_remote=12430 dedup_ratio=0.143677 router_tokens_per_expert=[4930, 471, 0, 0, 0, 0, 1801, 68, 0, 2958, 0, 4373, 0, 0, 1146, 0, 0, 156, 2675, 0, 0, 401, 0, 0, 0, 0, 2568, 661, 1824, 0, 287, 406, 0, 0, 0, 0, 0, 0, 0, 5079, 0, 0, 417, 120, 2840, 174, 7339, 0, 6790, 62, 0, 173, 0, 78, 0, 0, 70, 0, 0, 0, 309, 78, 0, 898] payload_rows_per_dst=[3930, 13366, 12372, 9846, 10158, 5396, 14018, 15094] dup_rows_per_dst=[750, 4702, 2838, 1646, 0, 200, 1694, 2294] combine_rows_per_src=[3930, 13366, 12372, 9846, 10158, 5396, 14018, 15094] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=326041600 dispatch_meta_bytes=347800 combine_payload_bytes=326041600 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=326389400 ingress_bytes=464453632 link_bytes=464453632 token_bytes=652431000 weight_movement_bytes=138412032 total_nvlink_bytes=790843032 weight_to_token_ratio=0.212148 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=16896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.747701626e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87670 payload_rows_remote=76906 dup_rows=10634 dup_rows_remote=10634 dedup_ratio=0.108175 router_tokens_per_expert=[4297, 435, 0, 0, 0, 0, 1564, 48, 0, 4170, 0, 4092, 3, 0, 1216, 0, 0, 160, 3281, 0, 0, 385, 7, 0, 0, 0, 2112, 684, 1773, 0, 281, 382, 0, 0, 0, 0, 0, 0, 0, 5382, 0, 0, 368, 111, 2215, 285, 7329, 0, 6533, 59, 0, 154, 0, 83, 23, 192, 62, 0, 0, 0, 537, 82, 0, 847] payload_rows_per_dst=[10478, 10218, 12518, 9144, 10764, 5896, 13594, 15058] dup_rows_per_dst=[2210, 560, 3332, 1320, 0, 62, 494, 2656] combine_rows_per_src=[10478, 10218, 12518, 9144, 10764, 5896, 13594, 15058] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80348 payload_rows_remote=65340 dup_rows=17956 dup_rows_remote=15076 dedup_ratio=0.182658 router_tokens_per_expert=[5290, 953, 2, 1, 4, 30, 1572, 80, 27, 2787, 16, 4397, 3, 0, 979, 0, 10, 175, 2267, 2, 10, 361, 3, 10, 0, 2, 3243, 650, 1553, 14, 303, 429, 32, 15, 8, 0, 24, 0, 0, 4462, 3, 24, 428, 114, 2730, 169, 7247, 9, 6697, 50, 0, 147, 0, 87, 19, 17, 63, 0, 0, 31, 604, 89, 0, 910] payload_rows_per_dst=[4226, 13242, 11998, 10778, 9006, 1470, 14620, 15008] dup_rows_per_dst=[1058, 4100, 2472, 2472, 76, 24, 4874, 2880] combine_rows_per_src=[4226, 13242, 11998, 10778, 9006, 1470, 14620, 15008] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=287383552 dispatch_meta_bytes=330368 combine_payload_bytes=287383552 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=287713920 ingress_bytes=356589568 link_bytes=356589568 token_bytes=575097472 weight_movement_bytes=69206016 total_nvlink_bytes=644303488 weight_to_token_ratio=0.120338 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=42752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.341817837e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=188809216 dispatch_meta_bytes=321800 combine_payload_bytes=188809216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=189131016 ingress_bytes=223412224 link_bytes=223412224 token_bytes=377940232 weight_movement_bytes=34603008 total_nvlink_bytes=412543240 weight_to_token_ratio=0.091557 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=31104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.406822128e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=291889152 dispatch_meta_bytes=323600 combine_payload_bytes=291889152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=292212752 ingress_bytes=326492160 link_bytes=326492160 token_bytes=584101904 weight_movement_bytes=34603008 total_nvlink_bytes=618704912 weight_to_token_ratio=0.059241 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=28672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.228563713e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=251215872 dispatch_meta_bytes=324544 combine_payload_bytes=251215872 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=251540416 ingress_bytes=355024896 link_bytes=355024896 token_bytes=502756288 weight_movement_bytes=103809024 total_nvlink_bytes=606565312 weight_to_token_ratio=0.206480 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=27904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.335930102e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=275529728 dispatch_meta_bytes=339672 combine_payload_bytes=275529728 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=275869400 ingress_bytes=344735744 link_bytes=344735744 token_bytes=551399128 weight_movement_bytes=69206016 total_nvlink_bytes=620605144 weight_to_token_ratio=0.125510 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=23296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.297212851e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=326041600 dispatch_meta_bytes=347800 combine_payload_bytes=326041600 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=326389400 ingress_bytes=464453632 link_bytes=464453632 token_bytes=652431000 weight_movement_bytes=138412032 total_nvlink_bytes=790843032 weight_to_token_ratio=0.212148 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=16896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.747701626e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=317030400 dispatch_meta_bytes=344648 combine_payload_bytes=317030400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=317375048 ingress_bytes=351633408 link_bytes=351633408 token_bytes=634405448 weight_movement_bytes=34603008 total_nvlink_bytes=669008456 weight_to_token_ratio=0.054544 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=22784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.323168205e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=283860992 dispatch_meta_bytes=346128 combine_payload_bytes=283860992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=284207120 ingress_bytes=318464000 link_bytes=318464000 token_bytes=568068112 weight_movement_bytes=34603008 total_nvlink_bytes=602671120 weight_to_token_ratio=0.060913 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=25728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.198354394e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=287383552 dispatch_meta_bytes=330368 combine_payload_bytes=287383552 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=287713920 ingress_bytes=356589568 link_bytes=356589568 token_bytes=575097472 weight_movement_bytes=69206016 total_nvlink_bytes=644303488 weight_to_token_ratio=0.120338 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=42752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.341817837e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=315006976 dispatch_meta_bytes=350160 combine_payload_bytes=315006976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=315357136 ingress_bytes=349609984 link_bytes=349609984 token_bytes=630364112 weight_movement_bytes=34603008 total_nvlink_bytes=664967120 weight_to_token_ratio=0.054894 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=37760 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.315554224e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=267632640 dispatch_meta_bytes=321664 combine_payload_bytes=267632640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267954304 ingress_bytes=302235648 link_bytes=302235648 token_bytes=535586944 weight_movement_bytes=34603008 total_nvlink_bytes=570189952 weight_to_token_ratio=0.064608 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=35584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137288411e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=61332 dup_rows=24072 dup_rows_remote=19804 dedup_ratio=0.244873 router_tokens_per_expert=[5020, 1166, 0, 0, 0, 0, 2120, 87, 0, 2422, 0, 3976, 0, 0, 465, 0, 0, 179, 1635, 0, 0, 373, 0, 0, 0, 0, 3883, 628, 1416, 0, 273, 857, 0, 0, 0, 0, 0, 0, 0, 4190, 0, 0, 633, 99, 3486, 103, 7306, 0, 6667, 0, 0, 195, 0, 89, 0, 0, 69, 0, 0, 0, 707, 72, 0, 1036] payload_rows_per_dst=[12900, 11556, 4374, 11390, 14616, 15542, 390, 3464] dup_rows_per_dst=[4268, 2170, 0, 2724, 7098, 7330, 178, 304] combine_rows_per_src=[12900, 11556, 4374, 11390, 14616, 15542, 390, 3464] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88634 payload_rows_remote=72406 dup_rows=9670 dup_rows_remote=7602 dedup_ratio=0.098368 router_tokens_per_expert=[0, 0, 860, 0, 0, 0, 2273, 0, 0, 0, 140, 276, 7251, 0, 0, 3933, 0, 0, 0, 0, 0, 4915, 0, 0, 0, 0, 0, 7871, 0, 0, 4365, 0, 0, 0, 129, 0, 0, 6274, 0, 0, 0, 0, 1998, 0, 0, 0, 0, 35, 0, 0, 307, 0, 0, 892, 0, 78, 0, 6499, 0, 0, 891, 165, 0, 0] payload_rows_per_dst=[5894, 832, 9830, 13102, 12806, 15026, 16228, 14916] dup_rows_per_dst=[372, 0, 0, 3494, 0, 3542, 2068, 194] combine_rows_per_src=[5894, 832, 9830, 13102, 12806, 15026, 16228, 14916] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80088 payload_rows_remote=64456 dup_rows=18216 dup_rows_remote=10680 dedup_ratio=0.185303 router_tokens_per_expert=[0, 0, 1281, 0, 0, 0, 2310, 0, 0, 0, 160, 249, 7167, 0, 0, 4008, 0, 0, 0, 0, 0, 3630, 0, 0, 0, 0, 0, 7859, 0, 0, 5015, 0, 0, 0, 160, 0, 0, 7094, 0, 0, 0, 0, 1256, 0, 0, 0, 0, 22, 0, 0, 270, 0, 0, 1055, 0, 98, 0, 6344, 0, 0, 827, 347, 0, 0] payload_rows_per_dst=[13208, 15632, 14322, 15718, 14508, 2556, 2116, 2028] dup_rows_per_dst=[4004, 7536, 5626, 0, 0, 0, 730, 320] combine_rows_per_src=[13208, 15632, 14322, 15718, 14508, 2556, 2116, 2028] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87656 payload_rows_remote=77400 dup_rows=10648 dup_rows_remote=8762 dedup_ratio=0.108317 router_tokens_per_expert=[5082, 491, 0, 0, 0, 0, 1781, 60, 0, 3165, 0, 4544, 0, 0, 1111, 0, 0, 124, 2175, 0, 0, 356, 0, 0, 0, 0, 2979, 688, 1699, 0, 272, 433, 0, 0, 0, 0, 0, 0, 0, 4923, 0, 0, 426, 113, 2704, 142, 7373, 0, 6791, 40, 0, 158, 0, 61, 0, 0, 62, 0, 0, 0, 471, 62, 0, 866] payload_rows_per_dst=[12126, 8384, 12076, 10256, 9846, 7930, 13714, 13324] dup_rows_per_dst=[2702, 168, 2322, 1886, 0, 1038, 386, 2146] combine_rows_per_src=[12126, 8384, 12076, 10256, 9846, 7930, 13714, 13324] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87778 payload_rows_remote=81178 dup_rows=10526 dup_rows_remote=10386 dedup_ratio=0.107076 router_tokens_per_expert=[4901, 507, 0, 0, 0, 0, 1500, 13, 0, 3071, 0, 4522, 0, 0, 1231, 0, 0, 176, 2989, 0, 0, 333, 0, 0, 0, 0, 2285, 582, 1489, 0, 307, 304, 0, 0, 0, 0, 0, 0, 0, 5562, 0, 0, 402, 134, 2617, 217, 7331, 0, 6906, 70, 0, 152, 0, 88, 0, 0, 72, 0, 0, 0, 439, 88, 0, 864] payload_rows_per_dst=[10360, 9204, 12874, 8664, 11124, 6600, 13838, 15114] dup_rows_per_dst=[1850, 1032, 3166, 1270, 0, 140, 594, 2474] combine_rows_per_src=[10360, 9204, 12874, 8664, 11124, 6600, 13838, 15114] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88634 payload_rows_remote=85802 dup_rows=9670 dup_rows_remote=9670 dedup_ratio=0.098368 router_tokens_per_expert=[0, 0, 954, 0, 0, 0, 2227, 0, 0, 0, 118, 272, 7329, 0, 0, 3990, 0, 0, 0, 0, 0, 4491, 0, 0, 0, 0, 0, 7889, 0, 0, 4608, 0, 0, 0, 115, 0, 0, 6535, 0, 0, 0, 0, 1741, 0, 0, 0, 0, 47, 0, 0, 307, 0, 0, 955, 0, 63, 0, 6413, 0, 0, 863, 235, 0, 0] payload_rows_per_dst=[12004, 10204, 8982, 2832, 13300, 12842, 13760, 14710] dup_rows_per_dst=[3574, 1432, 0, 0, 0, 2516, 1836, 312] combine_rows_per_src=[12004, 10204, 8982, 2832, 13300, 12842, 13760, 14710] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=251215872 dispatch_meta_bytes=324544 combine_payload_bytes=251215872 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=251540416 ingress_bytes=355024896 link_bytes=355024896 token_bytes=502756288 weight_movement_bytes=103809024 total_nvlink_bytes=606565312 weight_to_token_ratio=0.206480 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=27904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.335930102e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80348 payload_rows_remote=65340 dup_rows=17956 dup_rows_remote=15076 dedup_ratio=0.182658 router_tokens_per_expert=[5290, 953, 2, 1, 4, 30, 1572, 80, 27, 2787, 16, 4397, 3, 0, 979, 0, 10, 175, 2267, 2, 10, 361, 3, 10, 0, 2, 3243, 650, 1553, 14, 303, 429, 32, 15, 8, 0, 24, 0, 0, 4462, 3, 24, 428, 114, 2730, 169, 7247, 9, 6697, 50, 0, 147, 0, 87, 19, 17, 63, 0, 0, 31, 604, 89, 0, 910] payload_rows_per_dst=[4226, 13242, 11998, 10778, 9006, 1470, 14620, 15008] dup_rows_per_dst=[1058, 4100, 2472, 2472, 76, 24, 4874, 2880] combine_rows_per_src=[4226, 13242, 11998, 10778, 9006, 1470, 14620, 15008] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84920 payload_rows_remote=70986 dup_rows=13384 dup_rows_remote=12468 dedup_ratio=0.136149 router_tokens_per_expert=[1921, 12, 564, 75, 699, 5533, 175, 64, 1643, 176, 980, 0, 32, 48, 20, 22, 1, 66, 491, 378, 81, 78, 197, 54, 154, 11, 32, 175, 57, 84, 25, 52, 209, 1, 5055, 0, 5, 109, 2077, 544, 1, 6079, 1183, 6711, 61, 299, 4245, 86, 56, 218, 31, 69, 10, 94, 254, 4040, 1015, 519, 15, 581, 606, 104, 303, 672] payload_rows_per_dst=[14656, 12362, 13934, 14152, 12766, 3236, 9176, 4638] dup_rows_per_dst=[3430, 1970, 916, 450, 3234, 24, 368, 2992] combine_rows_per_src=[14656, 12362, 13934, 14152, 12766, 3236, 9176, 4638] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81386 payload_rows_remote=66796 dup_rows=16918 dup_rows_remote=11182 dedup_ratio=0.172099 router_tokens_per_expert=[0, 0, 1509, 0, 0, 0, 2500, 0, 0, 0, 150, 261, 7118, 0, 0, 3326, 1, 0, 0, 0, 0, 2960, 2, 0, 0, 0, 0, 7851, 0, 0, 6154, 0, 0, 0, 150, 0, 0, 7361, 0, 0, 0, 0, 1024, 0, 0, 0, 0, 39, 0, 0, 258, 0, 0, 1165, 0, 90, 0, 6153, 0, 0, 558, 522, 0, 0] payload_rows_per_dst=[14590, 15490, 14110, 15702, 15022, 2126, 2330, 2016] dup_rows_per_dst=[5736, 6220, 4122, 0, 0, 0, 696, 144] combine_rows_per_src=[14590, 15490, 14110, 15702, 15022, 2126, 2330, 2016] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91414 payload_rows_remote=81936 dup_rows=6890 dup_rows_remote=6848 dedup_ratio=0.070089 router_tokens_per_expert=[0, 9, 629, 7, 0, 0, 3109, 0, 4, 0, 207, 300, 7249, 0, 0, 4270, 0, 0, 0, 0, 0, 5466, 1, 0, 0, 0, 0, 7894, 2, 0, 4075, 0, 21, 0, 132, 0, 0, 4606, 1, 0, 0, 0, 1958, 0, 0, 0, 0, 112, 0, 0, 312, 3, 0, 928, 0, 80, 1, 6759, 0, 9, 944, 64, 0, 0] payload_rows_per_dst=[7298, 9434, 10934, 7784, 9478, 15006, 16224, 15256] dup_rows_per_dst=[580, 128, 0, 0, 42, 3632, 2210, 298] combine_rows_per_src=[7298, 9434, 10934, 7784, 9478, 15006, 16224, 15256] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83276 payload_rows_remote=74038 dup_rows=15028 dup_rows_remote=13636 dedup_ratio=0.152873 router_tokens_per_expert=[30, 142, 2457, 67, 13, 4, 2216, 0, 0, 6, 114, 169, 6248, 19, 535, 3113, 98, 0, 2, 44, 27, 2369, 143, 402, 24, 7, 32, 6842, 44, 636, 4195, 36, 55, 18, 117, 4, 20, 6669, 218, 90, 25, 0, 613, 1, 220, 697, 302, 144, 4, 128, 241, 103, 21, 621, 512, 77, 58, 4698, 0, 106, 676, 2205, 385, 90] payload_rows_per_dst=[13158, 14686, 9238, 15140, 14096, 3198, 2940, 10820] dup_rows_per_dst=[5090, 5722, 1392, 102, 286, 806, 474, 1156] combine_rows_per_src=[13158, 14686, 9238, 15140, 14096, 3198, 2940, 10820] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83944 payload_rows_remote=68702 dup_rows=14360 dup_rows_remote=8700 dedup_ratio=0.146077 router_tokens_per_expert=[0, 31, 1433, 7, 0, 1, 1780, 0, 0, 23, 138, 276, 7200, 0, 34, 4244, 5, 0, 1, 2, 4, 4262, 5, 19, 0, 2, 9, 7759, 7, 35, 4616, 2, 6, 1, 138, 2, 7, 6657, 12, 1, 3, 0, 1450, 0, 36, 38, 22, 36, 2, 18, 300, 5, 7, 901, 23, 87, 4, 6247, 0, 13, 880, 326, 24, 11] payload_rows_per_dst=[6148, 942, 8594, 12604, 9256, 15028, 16130, 15242] dup_rows_per_dst=[356, 0, 2, 3712, 14, 2542, 2074, 5660] combine_rows_per_src=[6148, 942, 8594, 12604, 9256, 15028, 16130, 15242] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=296574976 dispatch_meta_bytes=320032 combine_payload_bytes=296574976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296895008 ingress_bytes=331177984 link_bytes=331177984 token_bytes=593469984 weight_movement_bytes=34603008 total_nvlink_bytes=628072992 weight_to_token_ratio=0.058306 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=38016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.246196092e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=264011776 dispatch_meta_bytes=300544 combine_payload_bytes=264011776 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=264312320 ingress_bytes=367820800 link_bytes=367820800 token_bytes=528324096 weight_movement_bytes=103809024 total_nvlink_bytes=632133120 weight_to_token_ratio=0.196487 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=23296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.384080058e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=351444992 dispatch_meta_bytes=381888 combine_payload_bytes=351444992 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=351826880 ingress_bytes=455254016 link_bytes=455254016 token_bytes=703271872 weight_movement_bytes=103809024 total_nvlink_bytes=807080896 weight_to_token_ratio=0.147609 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=24064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.713084211e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=317030400 dispatch_meta_bytes=344648 combine_payload_bytes=317030400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=317375048 ingress_bytes=351633408 link_bytes=351633408 token_bytes=634405448 weight_movement_bytes=34603008 total_nvlink_bytes=669008456 weight_to_token_ratio=0.054544 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=22784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.323168205e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80088 payload_rows_remote=64456 dup_rows=18216 dup_rows_remote=10680 dedup_ratio=0.185303 router_tokens_per_expert=[0, 0, 1281, 0, 0, 0, 2310, 0, 0, 0, 160, 249, 7167, 0, 0, 4008, 0, 0, 0, 0, 0, 3630, 0, 0, 0, 0, 0, 7859, 0, 0, 5015, 0, 0, 0, 160, 0, 0, 7094, 0, 0, 0, 0, 1256, 0, 0, 0, 0, 22, 0, 0, 270, 0, 0, 1055, 0, 98, 0, 6344, 0, 0, 827, 347, 0, 0] payload_rows_per_dst=[13208, 15632, 14322, 15718, 14508, 2556, 2116, 2028] dup_rows_per_dst=[4004, 7536, 5626, 0, 0, 0, 730, 320] combine_rows_per_src=[13208, 15632, 14322, 15718, 14508, 2556, 2116, 2028] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88634 payload_rows_remote=72406 dup_rows=9670 dup_rows_remote=7602 dedup_ratio=0.098368 router_tokens_per_expert=[0, 0, 860, 0, 0, 0, 2273, 0, 0, 0, 140, 276, 7251, 0, 0, 3933, 0, 0, 0, 0, 0, 4915, 0, 0, 0, 0, 0, 7871, 0, 0, 4365, 0, 0, 0, 129, 0, 0, 6274, 0, 0, 0, 0, 1998, 0, 0, 0, 0, 35, 0, 0, 307, 0, 0, 892, 0, 78, 0, 6499, 0, 0, 891, 165, 0, 0] payload_rows_per_dst=[5894, 832, 9830, 13102, 12806, 15026, 16228, 14916] dup_rows_per_dst=[372, 0, 0, 3494, 0, 3542, 2068, 194] combine_rows_per_src=[5894, 832, 9830, 13102, 12806, 15026, 16228, 14916] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=332505088 dispatch_meta_bytes=366256 combine_payload_bytes=332505088 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=332871344 ingress_bytes=436314112 link_bytes=436314112 token_bytes=665376432 weight_movement_bytes=103809024 total_nvlink_bytes=769185456 weight_to_token_ratio=0.156015 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=21888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.641814877e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91414 payload_rows_remote=81936 dup_rows=6890 dup_rows_remote=6848 dedup_ratio=0.070089 router_tokens_per_expert=[0, 9, 629, 7, 0, 0, 3109, 0, 4, 0, 207, 300, 7249, 0, 0, 4270, 0, 0, 0, 0, 0, 5466, 1, 0, 0, 0, 0, 7894, 2, 0, 4075, 0, 21, 0, 132, 0, 0, 4606, 1, 0, 0, 0, 1958, 0, 0, 0, 0, 112, 0, 0, 312, 3, 0, 928, 0, 80, 1, 6759, 0, 9, 944, 64, 0, 0] payload_rows_per_dst=[7298, 9434, 10934, 7784, 9478, 15006, 16224, 15256] dup_rows_per_dst=[580, 128, 0, 0, 42, 3632, 2210, 298] combine_rows_per_src=[7298, 9434, 10934, 7784, 9478, 15006, 16224, 15256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=267632640 dispatch_meta_bytes=321664 combine_payload_bytes=267632640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267954304 ingress_bytes=302235648 link_bytes=302235648 token_bytes=535586944 weight_movement_bytes=34603008 total_nvlink_bytes=570189952 weight_to_token_ratio=0.064608 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=35584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137288411e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=290758656 dispatch_meta_bytes=333816 combine_payload_bytes=290758656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291092472 ingress_bytes=325361664 link_bytes=325361664 token_bytes=581851128 weight_movement_bytes=34603008 total_nvlink_bytes=616454136 weight_to_token_ratio=0.059471 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=45696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.224309748e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=273596416 dispatch_meta_bytes=311912 combine_payload_bytes=273596416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273908328 ingress_bytes=308199424 link_bytes=308199424 token_bytes=547504744 weight_movement_bytes=34603008 total_nvlink_bytes=582107752 weight_to_token_ratio=0.063201 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=20224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.159729620e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=303259648 dispatch_meta_bytes=350696 combine_payload_bytes=303259648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=303610344 ingress_bytes=337862656 link_bytes=337862656 token_bytes=606869992 weight_movement_bytes=34603008 total_nvlink_bytes=641473000 weight_to_token_ratio=0.057019 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=33792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.271349974e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=264011776 dispatch_meta_bytes=300544 combine_payload_bytes=264011776 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=264312320 ingress_bytes=367820800 link_bytes=367820800 token_bytes=528324096 weight_movement_bytes=103809024 total_nvlink_bytes=632133120 weight_to_token_ratio=0.196487 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=23296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.384080058e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=296574976 dispatch_meta_bytes=320032 combine_payload_bytes=296574976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296895008 ingress_bytes=331177984 link_bytes=331177984 token_bytes=593469984 weight_movement_bytes=34603008 total_nvlink_bytes=628072992 weight_to_token_ratio=0.058306 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=38016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.246196092e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=335609856 dispatch_meta_bytes=355136 combine_payload_bytes=335609856 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=335964992 ingress_bytes=404815872 link_bytes=404815872 token_bytes=671574848 weight_movement_bytes=69206016 total_nvlink_bytes=740780864 weight_to_token_ratio=0.103050 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=46848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523289536e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=281403392 dispatch_meta_bytes=309608 combine_payload_bytes=281403392 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=281713000 ingress_bytes=350609408 link_bytes=350609408 token_bytes=563116392 weight_movement_bytes=69206016 total_nvlink_bytes=632322408 weight_to_token_ratio=0.122898 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=35200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.319314976e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81386 payload_rows_remote=66796 dup_rows=16918 dup_rows_remote=11182 dedup_ratio=0.172099 router_tokens_per_expert=[0, 0, 1509, 0, 0, 0, 2500, 0, 0, 0, 150, 261, 7118, 0, 0, 3326, 1, 0, 0, 0, 0, 2960, 2, 0, 0, 0, 0, 7851, 0, 0, 6154, 0, 0, 0, 150, 0, 0, 7361, 0, 0, 0, 0, 1024, 0, 0, 0, 0, 39, 0, 0, 258, 0, 0, 1165, 0, 90, 0, 6153, 0, 0, 558, 522, 0, 0] payload_rows_per_dst=[14590, 15490, 14110, 15702, 15022, 2126, 2330, 2016] dup_rows_per_dst=[5736, 6220, 4122, 0, 0, 0, 696, 144] combine_rows_per_src=[14590, 15490, 14110, 15702, 15022, 2126, 2330, 2016] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90274 payload_rows_remote=78840 dup_rows=8030 dup_rows_remote=6194 dedup_ratio=0.081685 router_tokens_per_expert=[0, 0, 0, 0, 0, 78, 283, 0, 0, 46, 154, 0, 0, 51, 1364, 0, 4586, 0, 120, 7350, 7492, 0, 0, 0, 0, 199, 484, 0, 7504, 0, 51, 0, 3058, 0, 0, 0, 244, 5381, 7026, 78, 0, 0, 0, 0, 0, 0, 0, 84, 0, 1431, 0, 618, 0, 0, 0, 0, 0, 0, 0, 273, 495, 0, 330, 372] payload_rows_per_dst=[9638, 12548, 240, 15986, 9770, 15152, 11434, 15506] dup_rows_per_dst=[72, 1444, 0, 400, 2144, 0, 1836, 2134] combine_rows_per_src=[9638, 12548, 240, 15986, 9770, 15152, 11434, 15506] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82042 payload_rows_remote=71438 dup_rows=16262 dup_rows_remote=14854 dedup_ratio=0.165426 router_tokens_per_expert=[13, 0, 0, 0, 0, 86, 1, 0, 0, 0, 212, 0, 0, 63, 1681, 0, 4448, 0, 195, 7554, 7478, 0, 0, 0, 0, 168, 570, 0, 7461, 0, 64, 0, 4183, 0, 4, 0, 228, 4050, 7012, 98, 0, 0, 0, 0, 0, 1, 0, 60, 0, 1477, 0, 629, 0, 0, 0, 0, 0, 0, 0, 250, 479, 0, 353, 334] payload_rows_per_dst=[14224, 10604, 15840, 9714, 8562, 9190, 11076, 2832] dup_rows_per_dst=[0, 1408, 12138, 220, 464, 0, 2032, 0] combine_rows_per_src=[14224, 10604, 15840, 9714, 8562, 9190, 11076, 2832] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92186 payload_rows_remote=76078 dup_rows=6118 dup_rows_remote=5738 dedup_ratio=0.062236 router_tokens_per_expert=[2, 0, 0, 0, 0, 61, 231, 0, 0, 29, 136, 0, 0, 54, 1340, 0, 4234, 0, 98, 7461, 7549, 0, 0, 0, 0, 174, 497, 0, 7519, 0, 54, 0, 3536, 0, 3, 0, 242, 5211, 6995, 63, 0, 0, 0, 0, 0, 0, 0, 88, 0, 1579, 0, 570, 0, 0, 0, 0, 0, 0, 0, 258, 507, 0, 305, 356] payload_rows_per_dst=[14526, 12248, 1398, 16108, 7198, 15274, 10856, 14578] dup_rows_per_dst=[52, 1292, 0, 380, 490, 0, 1910, 1994] combine_rows_per_src=[14526, 12248, 1398, 16108, 7198, 15274, 10856, 14578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88634 payload_rows_remote=85802 dup_rows=9670 dup_rows_remote=9670 dedup_ratio=0.098368 router_tokens_per_expert=[0, 0, 954, 0, 0, 0, 2227, 0, 0, 0, 118, 272, 7329, 0, 0, 3990, 0, 0, 0, 0, 0, 4491, 0, 0, 0, 0, 0, 7889, 0, 0, 4608, 0, 0, 0, 115, 0, 0, 6535, 0, 0, 0, 0, 1741, 0, 0, 0, 0, 47, 0, 0, 307, 0, 0, 955, 0, 63, 0, 6413, 0, 0, 863, 235, 0, 0] payload_rows_per_dst=[12004, 10204, 8982, 2832, 13300, 12842, 13760, 14710] dup_rows_per_dst=[3574, 1432, 0, 0, 0, 2516, 1836, 312] combine_rows_per_src=[12004, 10204, 8982, 2832, 13300, 12842, 13760, 14710] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90778 payload_rows_remote=75788 dup_rows=7526 dup_rows_remote=3032 dedup_ratio=0.076558 router_tokens_per_expert=[0, 0, 639, 0, 0, 0, 2507, 0, 0, 0, 159, 319, 7246, 0, 0, 4838, 0, 0, 0, 0, 0, 5576, 0, 0, 0, 0, 0, 7962, 0, 0, 3764, 0, 0, 0, 150, 0, 0, 4598, 0, 0, 0, 0, 2450, 0, 0, 0, 0, 46, 0, 0, 336, 0, 0, 858, 0, 88, 0, 6746, 0, 0, 795, 75, 0, 0] payload_rows_per_dst=[5968, 9954, 11152, 7888, 9496, 14990, 16208, 15122] dup_rows_per_dst=[324, 92, 0, 226, 0, 4494, 2280, 110] combine_rows_per_src=[5968, 9954, 11152, 7888, 9496, 14990, 16208, 15122] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=351444992 dispatch_meta_bytes=381888 combine_payload_bytes=351444992 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=351826880 ingress_bytes=455254016 link_bytes=455254016 token_bytes=703271872 weight_movement_bytes=103809024 total_nvlink_bytes=807080896 weight_to_token_ratio=0.147609 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=24064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.713084211e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79432 payload_rows_remote=69302 dup_rows=18872 dup_rows_remote=17230 dedup_ratio=0.191976 router_tokens_per_expert=[4409, 1620, 19, 4, 115, 101, 1625, 93, 89, 2229, 234, 3177, 68, 29, 458, 1, 105, 180, 1673, 42, 134, 393, 44, 138, 15, 69, 2172, 571, 2212, 187, 262, 1058, 567, 262, 69, 0, 464, 17, 7, 3461, 12, 443, 1417, 133, 3396, 180, 6267, 202, 5704, 89, 33, 168, 0, 76, 7, 306, 81, 23, 2, 541, 856, 96, 23, 724] payload_rows_per_dst=[12006, 6002, 10130, 10536, 8508, 15430, 12378, 4442] dup_rows_per_dst=[3966, 214, 1642, 2556, 1186, 8670, 388, 250] combine_rows_per_src=[12006, 6002, 10130, 10536, 8508, 15430, 12378, 4442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83944 payload_rows_remote=68702 dup_rows=14360 dup_rows_remote=8700 dedup_ratio=0.146077 router_tokens_per_expert=[0, 31, 1433, 7, 0, 1, 1780, 0, 0, 23, 138, 276, 7200, 0, 34, 4244, 5, 0, 1, 2, 4, 4262, 5, 19, 0, 2, 9, 7759, 7, 35, 4616, 2, 6, 1, 138, 2, 7, 6657, 12, 1, 3, 0, 1450, 0, 36, 38, 22, 36, 2, 18, 300, 5, 7, 901, 23, 87, 4, 6247, 0, 13, 880, 326, 24, 11] payload_rows_per_dst=[6148, 942, 8594, 12604, 9256, 15028, 16130, 15242] dup_rows_per_dst=[356, 0, 2, 3712, 14, 2542, 2074, 5660] combine_rows_per_src=[6148, 942, 8594, 12604, 9256, 15028, 16130, 15242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=59008 payload_rows_remote=44972 dup_rows=39296 dup_rows_remote=39296 dedup_ratio=0.399740 router_tokens_per_expert=[28, 0, 0, 0, 0, 31, 29, 0, 0, 0, 239, 0, 0, 61, 1798, 0, 4333, 0, 179, 7546, 7452, 0, 0, 0, 0, 129, 719, 0, 7463, 0, 61, 0, 4782, 0, 18, 0, 242, 3354, 6930, 90, 0, 0, 0, 0, 0, 0, 0, 95, 0, 1474, 0, 652, 0, 0, 0, 0, 0, 0, 0, 288, 470, 0, 345, 344] payload_rows_per_dst=[14036, 9734, 16384, 1818, 9744, 190, 4220, 2882] dup_rows_per_dst=[0, 1170, 36748, 0, 520, 0, 846, 12] combine_rows_per_src=[14036, 9734, 16384, 1818, 9744, 190, 4220, 2882] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82042 payload_rows_remote=71438 dup_rows=16262 dup_rows_remote=14854 dedup_ratio=0.165426 router_tokens_per_expert=[13, 0, 0, 0, 0, 86, 1, 0, 0, 0, 212, 0, 0, 63, 1681, 0, 4448, 0, 195, 7554, 7478, 0, 0, 0, 0, 168, 570, 0, 7461, 0, 64, 0, 4183, 0, 4, 0, 228, 4050, 7012, 98, 0, 0, 0, 0, 0, 1, 0, 60, 0, 1477, 0, 629, 0, 0, 0, 0, 0, 0, 0, 250, 479, 0, 353, 334] payload_rows_per_dst=[14224, 10604, 15840, 9714, 8562, 9190, 11076, 2832] dup_rows_per_dst=[0, 1408, 12138, 220, 464, 0, 2032, 0] combine_rows_per_src=[14224, 10604, 15840, 9714, 8562, 9190, 11076, 2832] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90274 payload_rows_remote=78840 dup_rows=8030 dup_rows_remote=6194 dedup_ratio=0.081685 router_tokens_per_expert=[0, 0, 0, 0, 0, 78, 283, 0, 0, 46, 154, 0, 0, 51, 1364, 0, 4586, 0, 120, 7350, 7492, 0, 0, 0, 0, 199, 484, 0, 7504, 0, 51, 0, 3058, 0, 0, 0, 244, 5381, 7026, 78, 0, 0, 0, 0, 0, 0, 0, 84, 0, 1431, 0, 618, 0, 0, 0, 0, 0, 0, 0, 273, 495, 0, 330, 372] payload_rows_per_dst=[9638, 12548, 240, 15986, 9770, 15152, 11434, 15506] dup_rows_per_dst=[72, 1444, 0, 400, 2144, 0, 1836, 2134] combine_rows_per_src=[9638, 12548, 240, 15986, 9770, 15152, 11434, 15506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92004 payload_rows_remote=86546 dup_rows=6300 dup_rows_remote=5812 dedup_ratio=0.064087 router_tokens_per_expert=[1, 0, 0, 0, 0, 81, 441, 0, 0, 62, 115, 0, 0, 53, 1266, 0, 4528, 1, 83, 7219, 7513, 0, 0, 0, 0, 374, 417, 0, 7551, 0, 52, 0, 2636, 0, 1, 0, 242, 5679, 7048, 94, 0, 0, 0, 0, 0, 2, 0, 73, 0, 1605, 0, 605, 0, 0, 0, 0, 0, 0, 0, 231, 488, 0, 337, 354] payload_rows_per_dst=[14978, 12998, 168, 16160, 5458, 15156, 11620, 15466] dup_rows_per_dst=[164, 1352, 0, 628, 488, 20, 1856, 1792] combine_rows_per_src=[14978, 12998, 168, 16160, 5458, 15156, 11620, 15466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=310427648 dispatch_meta_bytes=315280 combine_payload_bytes=310427648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=310742928 ingress_bytes=345030656 link_bytes=345030656 token_bytes=621170576 weight_movement_bytes=34603008 total_nvlink_bytes=655773584 weight_to_token_ratio=0.055706 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=34944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.298322581e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=273596416 dispatch_meta_bytes=311912 combine_payload_bytes=273596416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273908328 ingress_bytes=308199424 link_bytes=308199424 token_bytes=547504744 weight_movement_bytes=34603008 total_nvlink_bytes=582107752 weight_to_token_ratio=0.063201 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=20224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.159729620e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88148 payload_rows_remote=73530 dup_rows=10156 dup_rows_remote=10050 dedup_ratio=0.103312 router_tokens_per_expert=[66, 0, 524, 0, 663, 70, 385, 49, 25, 61, 198, 96, 0, 107, 1194, 6, 4323, 409, 162, 6326, 5948, 1, 367, 97, 0, 628, 1513, 0, 6301, 206, 388, 8, 3054, 152, 328, 89, 248, 2303, 5454, 79, 1, 127, 6, 485, 13, 15, 3, 37, 32, 801, 66, 1865, 0, 520, 0, 1, 218, 26, 408, 281, 565, 608, 856, 390] payload_rows_per_dst=[13456, 7426, 14618, 14868, 7074, 13146, 12524, 5036] dup_rows_per_dst=[966, 554, 106, 3220, 826, 124, 2692, 1668] combine_rows_per_src=[13456, 7426, 14618, 14868, 7074, 13146, 12524, 5036] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92004 payload_rows_remote=86546 dup_rows=6300 dup_rows_remote=5812 dedup_ratio=0.064087 router_tokens_per_expert=[1, 0, 0, 0, 0, 81, 441, 0, 0, 62, 115, 0, 0, 53, 1266, 0, 4528, 1, 83, 7219, 7513, 0, 0, 0, 0, 374, 417, 0, 7551, 0, 52, 0, 2636, 0, 1, 0, 242, 5679, 7048, 94, 0, 0, 0, 0, 0, 2, 0, 73, 0, 1605, 0, 605, 0, 0, 0, 0, 0, 0, 0, 231, 488, 0, 337, 354] payload_rows_per_dst=[14978, 12998, 168, 16160, 5458, 15156, 11620, 15466] dup_rows_per_dst=[164, 1352, 0, 628, 488, 20, 1856, 1792] combine_rows_per_src=[14978, 12998, 168, 16160, 5458, 15156, 11620, 15466] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69090 payload_rows_remote=53672 dup_rows=29214 dup_rows_remote=27268 dedup_ratio=0.297180 router_tokens_per_expert=[0, 0, 29, 0, 38, 86, 316, 3, 1, 24, 166, 12, 0, 55, 1510, 1, 4011, 39, 176, 7169, 7386, 0, 16, 10, 0, 188, 549, 0, 7416, 11, 63, 0, 3737, 6, 23, 3, 250, 5123, 6909, 87, 3, 10, 1, 23, 0, 5, 0, 52, 24, 1343, 12, 726, 0, 27, 0, 0, 8, 1, 22, 225, 503, 36, 366, 352] payload_rows_per_dst=[874, 9860, 474, 1586, 15544, 14948, 10386, 15418] dup_rows_per_dst=[70, 1050, 8, 36, 24192, 12, 1900, 1946] combine_rows_per_src=[874, 9860, 474, 1586, 15544, 14948, 10386, 15418] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=283860992 dispatch_meta_bytes=346128 combine_payload_bytes=283860992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=284207120 ingress_bytes=318464000 link_bytes=318464000 token_bytes=568068112 weight_movement_bytes=34603008 total_nvlink_bytes=602671120 weight_to_token_ratio=0.060913 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=25728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.198354394e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=281403392 dispatch_meta_bytes=309608 combine_payload_bytes=281403392 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=281713000 ingress_bytes=350609408 link_bytes=350609408 token_bytes=563116392 weight_movement_bytes=69206016 total_nvlink_bytes=632322408 weight_to_token_ratio=0.122898 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=35200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.319314976e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=184205312 dispatch_meta_bytes=337072 combine_payload_bytes=184205312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=184542384 ingress_bytes=218808320 link_bytes=218808320 token_bytes=368747696 weight_movement_bytes=34603008 total_nvlink_bytes=403350704 weight_to_token_ratio=0.093839 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=45056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.233580926e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=292610048 dispatch_meta_bytes=345168 combine_payload_bytes=292610048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=292955216 ingress_bytes=327213056 link_bytes=327213056 token_bytes=585565264 weight_movement_bytes=34603008 total_nvlink_bytes=620168272 weight_to_token_ratio=0.059093 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=35584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.231276387e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=322928640 dispatch_meta_bytes=340136 combine_payload_bytes=322928640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=323268776 ingress_bytes=357531648 link_bytes=357531648 token_bytes=646197416 weight_movement_bytes=34603008 total_nvlink_bytes=680800424 weight_to_token_ratio=0.053549 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=30848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.345362808e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=311615488 dispatch_meta_bytes=327264 combine_payload_bytes=311615488 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=311942752 ingress_bytes=380821504 link_bytes=380821504 token_bytes=623558240 weight_movement_bytes=69206016 total_nvlink_bytes=692764256 weight_to_token_ratio=0.110986 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=40704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.433000661e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=310427648 dispatch_meta_bytes=315280 combine_payload_bytes=310427648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=310742928 ingress_bytes=345030656 link_bytes=345030656 token_bytes=621170576 weight_movement_bytes=34603008 total_nvlink_bytes=655773584 weight_to_token_ratio=0.055706 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=34944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.298322581e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=301178880 dispatch_meta_bytes=334320 combine_payload_bytes=301178880 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=301513200 ingress_bytes=439590912 link_bytes=439590912 token_bytes=602692080 weight_movement_bytes=138412032 total_nvlink_bytes=741104112 weight_to_token_ratio=0.229656 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.654145212e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=354492416 dispatch_meta_bytes=369432 combine_payload_bytes=354492416 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=354861848 ingress_bytes=458301440 link_bytes=458301440 token_bytes=709354264 weight_movement_bytes=103809024 total_nvlink_bytes=813163288 weight_to_token_ratio=0.146343 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=27648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.724551422e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=219840512 dispatch_meta_bytes=323760 combine_payload_bytes=219840512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=220164272 ingress_bytes=254443520 link_bytes=254443520 token_bytes=440004784 weight_movement_bytes=34603008 total_nvlink_bytes=474607792 weight_to_token_ratio=0.078642 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=35840 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.574504814e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=59008 payload_rows_remote=44972 dup_rows=39296 dup_rows_remote=39296 dedup_ratio=0.399740 router_tokens_per_expert=[28, 0, 0, 0, 0, 31, 29, 0, 0, 0, 239, 0, 0, 61, 1798, 0, 4333, 0, 179, 7546, 7452, 0, 0, 0, 0, 129, 719, 0, 7463, 0, 61, 0, 4782, 0, 18, 0, 242, 3354, 6930, 90, 0, 0, 0, 0, 0, 0, 0, 95, 0, 1474, 0, 652, 0, 0, 0, 0, 0, 0, 0, 288, 470, 0, 345, 344] payload_rows_per_dst=[14036, 9734, 16384, 1818, 9744, 190, 4220, 2882] dup_rows_per_dst=[0, 1170, 36748, 0, 520, 0, 846, 12] combine_rows_per_src=[14036, 9734, 16384, 1818, 9744, 190, 4220, 2882] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81990 payload_rows_remote=66744 dup_rows=16314 dup_rows_remote=15286 dedup_ratio=0.165955 router_tokens_per_expert=[0, 0, 0, 534, 0, 0, 0, 0, 0, 0, 0, 98, 0, 98, 812, 0, 0, 0, 2165, 2459, 7456, 757, 0, 98, 7574, 0, 0, 7129, 110, 64, 0, 0, 0, 0, 238, 0, 42, 0, 0, 6315, 0, 756, 0, 0, 0, 0, 97, 0, 0, 0, 0, 241, 3854, 0, 979, 250, 1488, 97, 0, 0, 0, 4725, 25, 691] payload_rows_per_dst=[15192, 15246, 15984, 924, 13084, 1706, 11790, 8064] dup_rows_per_dst=[1024, 1028, 9310, 0, 106, 0, 3696, 1150] combine_rows_per_src=[15192, 15246, 15984, 924, 13084, 1706, 11790, 8064] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91260 payload_rows_remote=85198 dup_rows=7044 dup_rows_remote=7010 dedup_ratio=0.071655 router_tokens_per_expert=[0, 0, 0, 636, 0, 0, 0, 0, 0, 0, 2, 63, 0, 63, 888, 0, 0, 0, 2548, 2905, 7575, 709, 0, 60, 7658, 0, 0, 7184, 89, 54, 0, 0, 0, 0, 152, 0, 39, 0, 0, 6307, 0, 583, 0, 0, 0, 0, 61, 0, 0, 0, 0, 212, 3489, 0, 725, 174, 1441, 61, 0, 0, 0, 4833, 6, 635] payload_rows_per_dst=[15318, 15370, 5942, 6062, 12912, 15526, 8054, 12076] dup_rows_per_dst=[1270, 1030, 692, 34, 84, 912, 1146, 1876] combine_rows_per_src=[15318, 15370, 5942, 6062, 12912, 15526, 8054, 12076] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73532 payload_rows_remote=65834 dup_rows=24772 dup_rows_remote=23672 dedup_ratio=0.251994 router_tokens_per_expert=[0, 0, 0, 585, 0, 0, 0, 0, 0, 0, 0, 78, 0, 78, 839, 0, 0, 0, 2588, 2701, 7618, 723, 0, 78, 7613, 0, 0, 7151, 98, 51, 0, 0, 0, 0, 186, 0, 50, 0, 0, 6288, 0, 615, 0, 0, 0, 0, 80, 0, 0, 0, 0, 235, 3282, 0, 718, 164, 1468, 78, 0, 0, 0, 5168, 2, 617] payload_rows_per_dst=[1170, 1834, 6070, 15640, 12932, 15490, 7698, 12698] dup_rows_per_dst=[0, 156, 708, 19588, 116, 1136, 1100, 1968] combine_rows_per_src=[1170, 1834, 6070, 15640, 12932, 15490, 7698, 12698] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91592 payload_rows_remote=76500 dup_rows=6712 dup_rows_remote=6712 dedup_ratio=0.068278 router_tokens_per_expert=[0, 0, 0, 0, 0, 88, 433, 0, 0, 71, 105, 0, 0, 62, 1228, 0, 4285, 0, 91, 7206, 7489, 0, 0, 0, 0, 492, 428, 0, 7608, 0, 62, 0, 2570, 0, 1, 0, 256, 5842, 7103, 88, 0, 0, 0, 0, 0, 0, 0, 57, 0, 1640, 0, 633, 0, 0, 0, 0, 0, 0, 0, 224, 415, 0, 310, 365] payload_rows_per_dst=[15078, 13100, 182, 16142, 5316, 15092, 11224, 15458] dup_rows_per_dst=[170, 1516, 0, 1038, 514, 0, 1892, 1582] combine_rows_per_src=[15078, 13100, 182, 16142, 5316, 15092, 11224, 15458] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=184205312 dispatch_meta_bytes=337072 combine_payload_bytes=184205312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=184542384 ingress_bytes=218808320 link_bytes=218808320 token_bytes=368747696 weight_movement_bytes=34603008 total_nvlink_bytes=403350704 weight_to_token_ratio=0.093839 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=45056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.233580926e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85282 payload_rows_remote=79226 dup_rows=13022 dup_rows_remote=12700 dedup_ratio=0.132467 router_tokens_per_expert=[0, 0, 468, 1194, 121, 278, 0, 1, 29, 19, 0, 73, 266, 52, 1052, 358, 9, 43, 801, 1674, 5788, 608, 81, 79, 6318, 260, 5, 4962, 76, 94, 23, 4, 0, 477, 514, 645, 46, 13, 0, 5286, 371, 920, 40, 610, 27, 0, 110, 2, 127, 102, 152, 290, 5888, 115, 2714, 228, 1790, 150, 237, 3, 619, 1718, 214, 1008] payload_rows_per_dst=[14854, 12304, 6056, 4220, 12988, 11044, 13604, 10212] dup_rows_per_dst=[1906, 1318, 322, 52, 974, 1556, 5628, 1266] combine_rows_per_src=[14854, 12304, 6056, 4220, 12988, 11044, 13604, 10212] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91206 payload_rows_remote=78564 dup_rows=7098 dup_rows_remote=6994 dedup_ratio=0.072205 router_tokens_per_expert=[0, 0, 0, 627, 0, 0, 0, 0, 0, 1, 1, 91, 0, 84, 786, 0, 0, 0, 2919, 3011, 7615, 685, 0, 80, 7615, 0, 0, 7190, 116, 52, 1, 0, 0, 0, 188, 0, 52, 0, 0, 6133, 0, 493, 0, 0, 0, 0, 86, 0, 0, 0, 2, 271, 2839, 0, 560, 130, 1469, 80, 1, 0, 0, 5390, 25, 559] payload_rows_per_dst=[15248, 15364, 6646, 6298, 12642, 15492, 6554, 12962] dup_rows_per_dst=[1236, 942, 722, 62, 104, 896, 1050, 2086] combine_rows_per_src=[15248, 15364, 6646, 6298, 12642, 15492, 6554, 12962] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72696 payload_rows_remote=60822 dup_rows=25608 dup_rows_remote=23100 dedup_ratio=0.260498 router_tokens_per_expert=[0, 0, 29, 653, 8, 19, 0, 0, 1, 1, 3, 87, 14, 78, 844, 22, 1, 1, 2399, 2830, 7363, 700, 3, 88, 7524, 18, 1, 6985, 134, 53, 1, 6, 0, 29, 253, 30, 51, 0, 0, 6249, 25, 771, 5, 35, 0, 0, 89, 0, 5, 4, 26, 255, 3962, 3, 735, 242, 1542, 94, 10, 0, 39, 4179, 40, 613] payload_rows_per_dst=[1400, 1926, 5744, 15612, 11744, 15344, 9052, 11874] dup_rows_per_dst=[18, 174, 640, 19492, 132, 1232, 1412, 2508] combine_rows_per_src=[1400, 1926, 5744, 15612, 11744, 15344, 9052, 11874] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92186 payload_rows_remote=76078 dup_rows=6118 dup_rows_remote=5738 dedup_ratio=0.062236 router_tokens_per_expert=[2, 0, 0, 0, 0, 61, 231, 0, 0, 29, 136, 0, 0, 54, 1340, 0, 4234, 0, 98, 7461, 7549, 0, 0, 0, 0, 174, 497, 0, 7519, 0, 54, 0, 3536, 0, 3, 0, 242, 5211, 6995, 63, 0, 0, 0, 0, 0, 0, 0, 88, 0, 1579, 0, 570, 0, 0, 0, 0, 0, 0, 0, 258, 507, 0, 305, 356] payload_rows_per_dst=[14526, 12248, 1398, 16108, 7198, 15274, 10856, 14578] dup_rows_per_dst=[52, 1292, 0, 380, 490, 0, 1910, 1994] combine_rows_per_src=[14526, 12248, 1398, 16108, 7198, 15274, 10856, 14578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91592 payload_rows_remote=76500 dup_rows=6712 dup_rows_remote=6712 dedup_ratio=0.068278 router_tokens_per_expert=[0, 0, 0, 0, 0, 88, 433, 0, 0, 71, 105, 0, 0, 62, 1228, 0, 4285, 0, 91, 7206, 7489, 0, 0, 0, 0, 492, 428, 0, 7608, 0, 62, 0, 2570, 0, 1, 0, 256, 5842, 7103, 88, 0, 0, 0, 0, 0, 0, 0, 57, 0, 1640, 0, 633, 0, 0, 0, 0, 0, 0, 0, 224, 415, 0, 310, 365] payload_rows_per_dst=[15078, 13100, 182, 16142, 5316, 15092, 11224, 15458] dup_rows_per_dst=[170, 1516, 0, 1038, 514, 0, 1892, 1582] combine_rows_per_src=[15078, 13100, 182, 16142, 5316, 15092, 11224, 15458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=311615488 dispatch_meta_bytes=327264 combine_payload_bytes=311615488 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=311942752 ingress_bytes=380821504 link_bytes=380821504 token_bytes=623558240 weight_movement_bytes=69206016 total_nvlink_bytes=692764256 weight_to_token_ratio=0.110986 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=40704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.433000661e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83276 payload_rows_remote=74038 dup_rows=15028 dup_rows_remote=13636 dedup_ratio=0.152873 router_tokens_per_expert=[30, 142, 2457, 67, 13, 4, 2216, 0, 0, 6, 114, 169, 6248, 19, 535, 3113, 98, 0, 2, 44, 27, 2369, 143, 402, 24, 7, 32, 6842, 44, 636, 4195, 36, 55, 18, 117, 4, 20, 6669, 218, 90, 25, 0, 613, 1, 220, 697, 302, 144, 4, 128, 241, 103, 21, 621, 512, 77, 58, 4698, 0, 106, 676, 2205, 385, 90] payload_rows_per_dst=[13158, 14686, 9238, 15140, 14096, 3198, 2940, 10820] dup_rows_per_dst=[5090, 5722, 1392, 102, 286, 806, 474, 1156] combine_rows_per_src=[13158, 14686, 9238, 15140, 14096, 3198, 2940, 10820] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69090 payload_rows_remote=53672 dup_rows=29214 dup_rows_remote=27268 dedup_ratio=0.297180 router_tokens_per_expert=[0, 0, 29, 0, 38, 86, 316, 3, 1, 24, 166, 12, 0, 55, 1510, 1, 4011, 39, 176, 7169, 7386, 0, 16, 10, 0, 188, 549, 0, 7416, 11, 63, 0, 3737, 6, 23, 3, 250, 5123, 6909, 87, 3, 10, 1, 23, 0, 5, 0, 52, 24, 1343, 12, 726, 0, 27, 0, 0, 8, 1, 22, 225, 503, 36, 366, 352] payload_rows_per_dst=[874, 9860, 474, 1586, 15544, 14948, 10386, 15418] dup_rows_per_dst=[70, 1050, 8, 36, 24192, 12, 1900, 1946] combine_rows_per_src=[874, 9860, 474, 1586, 15544, 14948, 10386, 15418] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75368 payload_rows_remote=60150 dup_rows=22936 dup_rows_remote=21876 dedup_ratio=0.233317 router_tokens_per_expert=[0, 0, 0, 532, 0, 0, 0, 0, 0, 0, 1, 90, 0, 89, 792, 0, 0, 0, 2352, 2501, 7344, 616, 0, 84, 7607, 0, 0, 7091, 124, 61, 0, 0, 0, 0, 245, 0, 34, 0, 0, 6331, 0, 824, 0, 0, 0, 0, 89, 0, 0, 0, 0, 212, 3836, 0, 1195, 236, 2006, 89, 0, 0, 0, 3970, 67, 734] payload_rows_per_dst=[15218, 15144, 15784, 370, 7494, 1826, 14222, 5310] dup_rows_per_dst=[1060, 982, 15686, 0, 50, 0, 4676, 482] combine_rows_per_src=[15218, 15144, 15784, 370, 7494, 1826, 14222, 5310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73532 payload_rows_remote=65834 dup_rows=24772 dup_rows_remote=23672 dedup_ratio=0.251994 router_tokens_per_expert=[0, 0, 0, 585, 0, 0, 0, 0, 0, 0, 0, 78, 0, 78, 839, 0, 0, 0, 2588, 2701, 7618, 723, 0, 78, 7613, 0, 0, 7151, 98, 51, 0, 0, 0, 0, 186, 0, 50, 0, 0, 6288, 0, 615, 0, 0, 0, 0, 80, 0, 0, 0, 0, 235, 3282, 0, 718, 164, 1468, 78, 0, 0, 0, 5168, 2, 617] payload_rows_per_dst=[1170, 1834, 6070, 15640, 12932, 15490, 7698, 12698] dup_rows_per_dst=[0, 156, 708, 19588, 116, 1136, 1100, 1968] combine_rows_per_src=[1170, 1834, 6070, 15640, 12932, 15490, 7698, 12698] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81990 payload_rows_remote=66744 dup_rows=16314 dup_rows_remote=15286 dedup_ratio=0.165955 router_tokens_per_expert=[0, 0, 0, 534, 0, 0, 0, 0, 0, 0, 0, 98, 0, 98, 812, 0, 0, 0, 2165, 2459, 7456, 757, 0, 98, 7574, 0, 0, 7129, 110, 64, 0, 0, 0, 0, 238, 0, 42, 0, 0, 6315, 0, 756, 0, 0, 0, 0, 97, 0, 0, 0, 0, 241, 3854, 0, 979, 250, 1488, 97, 0, 0, 0, 4725, 25, 691] payload_rows_per_dst=[15192, 15246, 15984, 924, 13084, 1706, 11790, 8064] dup_rows_per_dst=[1024, 1028, 9310, 0, 106, 0, 3696, 1150] combine_rows_per_src=[15192, 15246, 15984, 924, 13084, 1706, 11790, 8064] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=313344000 dispatch_meta_bytes=332848 combine_payload_bytes=313344000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=313676848 ingress_bytes=347947008 link_bytes=347947008 token_bytes=627020848 weight_movement_bytes=34603008 total_nvlink_bytes=661623856 weight_to_token_ratio=0.055186 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=48000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.309296579e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91206 payload_rows_remote=78564 dup_rows=7098 dup_rows_remote=6994 dedup_ratio=0.072205 router_tokens_per_expert=[0, 0, 0, 627, 0, 0, 0, 0, 0, 1, 1, 91, 0, 84, 786, 0, 0, 0, 2919, 3011, 7615, 685, 0, 80, 7615, 0, 0, 7190, 116, 52, 1, 0, 0, 0, 188, 0, 52, 0, 0, 6133, 0, 493, 0, 0, 0, 0, 86, 0, 0, 0, 2, 271, 2839, 0, 560, 130, 1469, 80, 1, 0, 0, 5390, 25, 559] payload_rows_per_dst=[15248, 15364, 6646, 6298, 12642, 15492, 6554, 12962] dup_rows_per_dst=[1236, 942, 722, 62, 104, 896, 1050, 2086] combine_rows_per_src=[15248, 15364, 6646, 6298, 12642, 15492, 6554, 12962] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=303259648 dispatch_meta_bytes=350696 combine_payload_bytes=303259648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=303610344 ingress_bytes=337862656 link_bytes=337862656 token_bytes=606869992 weight_movement_bytes=34603008 total_nvlink_bytes=641473000 weight_to_token_ratio=0.057019 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=33792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.271349974e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=219840512 dispatch_meta_bytes=323760 combine_payload_bytes=219840512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=220164272 ingress_bytes=254443520 link_bytes=254443520 token_bytes=440004784 weight_movement_bytes=34603008 total_nvlink_bytes=474607792 weight_to_token_ratio=0.078642 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=35840 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.574504814e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=269656064 dispatch_meta_bytes=358024 combine_payload_bytes=269656064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270014088 ingress_bytes=304259072 link_bytes=304259072 token_bytes=539670152 weight_movement_bytes=34603008 total_nvlink_bytes=574273160 weight_to_token_ratio=0.064119 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=30464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.144902393e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=348971008 dispatch_meta_bytes=368832 combine_payload_bytes=348971008 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=349339840 ingress_bytes=452780032 link_bytes=452780032 token_bytes=698310848 weight_movement_bytes=103809024 total_nvlink_bytes=802119872 weight_to_token_ratio=0.148657 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=16640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.703774809e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=246374400 dispatch_meta_bytes=328104 combine_payload_bytes=246374400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=246702504 ingress_bytes=280977408 link_bytes=280977408 token_bytes=493076904 weight_movement_bytes=34603008 total_nvlink_bytes=527679912 weight_to_token_ratio=0.070178 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=43008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.057295366e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=273383424 dispatch_meta_bytes=328120 combine_payload_bytes=273383424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273711544 ingress_bytes=307986432 link_bytes=307986432 token_bytes=547094968 weight_movement_bytes=34603008 total_nvlink_bytes=581697976 weight_to_token_ratio=0.063249 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=40320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.158928149e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=313344000 dispatch_meta_bytes=332848 combine_payload_bytes=313344000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=313676848 ingress_bytes=347947008 link_bytes=347947008 token_bytes=627020848 weight_movement_bytes=34603008 total_nvlink_bytes=661623856 weight_to_token_ratio=0.055186 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=48000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.309296579e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=324509696 dispatch_meta_bytes=367704 combine_payload_bytes=324509696 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=324877400 ingress_bytes=428318720 link_bytes=428318720 token_bytes=649387096 weight_movement_bytes=103809024 total_nvlink_bytes=753196120 weight_to_token_ratio=0.159857 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.611728861e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75368 payload_rows_remote=60150 dup_rows=22936 dup_rows_remote=21876 dedup_ratio=0.233317 router_tokens_per_expert=[0, 0, 0, 532, 0, 0, 0, 0, 0, 0, 1, 90, 0, 89, 792, 0, 0, 0, 2352, 2501, 7344, 616, 0, 84, 7607, 0, 0, 7091, 124, 61, 0, 0, 0, 0, 245, 0, 34, 0, 0, 6331, 0, 824, 0, 0, 0, 0, 89, 0, 0, 0, 0, 212, 3836, 0, 1195, 236, 2006, 89, 0, 0, 0, 3970, 67, 734] payload_rows_per_dst=[15218, 15144, 15784, 370, 7494, 1826, 14222, 5310] dup_rows_per_dst=[1060, 982, 15686, 0, 50, 0, 4676, 482] combine_rows_per_src=[15218, 15144, 15784, 370, 7494, 1826, 14222, 5310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=321798144 dispatch_meta_bytes=342232 combine_payload_bytes=321798144 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=322140376 ingress_bytes=391004160 link_bytes=391004160 token_bytes=643938520 weight_movement_bytes=69206016 total_nvlink_bytes=713144536 weight_to_token_ratio=0.107473 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=45696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.471317176e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90330 payload_rows_remote=74946 dup_rows=7974 dup_rows_remote=7214 dedup_ratio=0.081116 router_tokens_per_expert=[0, 0, 0, 487, 0, 0, 0, 0, 0, 0, 0, 88, 0, 88, 883, 0, 0, 0, 3169, 3031, 7588, 742, 0, 88, 7586, 0, 0, 7084, 104, 62, 0, 0, 0, 0, 192, 0, 61, 0, 0, 6218, 0, 382, 0, 0, 0, 0, 102, 0, 0, 0, 0, 320, 3213, 0, 634, 144, 1204, 88, 0, 0, 0, 5133, 1, 460] payload_rows_per_dst=[12616, 14462, 7258, 8072, 12814, 15384, 7434, 12290] dup_rows_per_dst=[774, 1024, 740, 1878, 128, 760, 1188, 1482] combine_rows_per_src=[12616, 14462, 7258, 8072, 12814, 15384, 7434, 12290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91260 payload_rows_remote=85198 dup_rows=7044 dup_rows_remote=7010 dedup_ratio=0.071655 router_tokens_per_expert=[0, 0, 0, 636, 0, 0, 0, 0, 0, 0, 2, 63, 0, 63, 888, 0, 0, 0, 2548, 2905, 7575, 709, 0, 60, 7658, 0, 0, 7184, 89, 54, 0, 0, 0, 0, 152, 0, 39, 0, 0, 6307, 0, 583, 0, 0, 0, 0, 61, 0, 0, 0, 0, 212, 3489, 0, 725, 174, 1441, 61, 0, 0, 0, 4833, 6, 635] payload_rows_per_dst=[15318, 15370, 5942, 6062, 12912, 15526, 8054, 12076] dup_rows_per_dst=[1270, 1030, 692, 34, 84, 912, 1146, 1876] combine_rows_per_src=[15318, 15370, 5942, 6062, 12912, 15526, 8054, 12076] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88148 payload_rows_remote=73530 dup_rows=10156 dup_rows_remote=10050 dedup_ratio=0.103312 router_tokens_per_expert=[66, 0, 524, 0, 663, 70, 385, 49, 25, 61, 198, 96, 0, 107, 1194, 6, 4323, 409, 162, 6326, 5948, 1, 367, 97, 0, 628, 1513, 0, 6301, 206, 388, 8, 3054, 152, 328, 89, 248, 2303, 5454, 79, 1, 127, 6, 485, 13, 15, 3, 37, 32, 801, 66, 1865, 0, 520, 0, 1, 218, 26, 408, 281, 565, 608, 856, 390] payload_rows_per_dst=[13456, 7426, 14618, 14868, 7074, 13146, 12524, 5036] dup_rows_per_dst=[966, 554, 106, 3220, 826, 124, 2692, 1668] combine_rows_per_src=[13456, 7426, 14618, 14868, 7074, 13146, 12524, 5036] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89796 payload_rows_remote=76582 dup_rows=8508 dup_rows_remote=5908 dedup_ratio=0.086548 router_tokens_per_expert=[0, 3213, 848, 1317, 7686, 7562, 0, 0, 0, 0, 0, 244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2240, 0, 0, 0, 0, 1334, 2237, 78, 0, 0, 0, 0, 0, 135, 0, 0, 6399, 13, 133, 0, 0, 1399, 0, 0, 0, 0, 0, 0, 1467, 17, 0, 0, 0, 70, 6353, 213, 6116, 0, 0, 78, 0, 0, 0] payload_rows_per_dst=[4010, 15372, 9424, 7000, 13002, 15386, 13214, 12388] dup_rows_per_dst=[320, 488, 1482, 298, 358, 2536, 2600, 426] combine_rows_per_src=[4010, 15372, 9424, 7000, 13002, 15386, 13214, 12388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72696 payload_rows_remote=60822 dup_rows=25608 dup_rows_remote=23100 dedup_ratio=0.260498 router_tokens_per_expert=[0, 0, 29, 653, 8, 19, 0, 0, 1, 1, 3, 87, 14, 78, 844, 22, 1, 1, 2399, 2830, 7363, 700, 3, 88, 7524, 18, 1, 6985, 134, 53, 1, 6, 0, 29, 253, 30, 51, 0, 0, 6249, 25, 771, 5, 35, 0, 0, 89, 0, 5, 4, 26, 255, 3962, 3, 735, 242, 1542, 94, 10, 0, 39, 4179, 40, 613] payload_rows_per_dst=[1400, 1926, 5744, 15612, 11744, 15344, 9052, 11874] dup_rows_per_dst=[18, 174, 640, 19492, 132, 1232, 1412, 2508] combine_rows_per_src=[1400, 1926, 5744, 15612, 11744, 15344, 9052, 11874] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75208 payload_rows_remote=70540 dup_rows=23096 dup_rows_remote=22964 dedup_ratio=0.234945 router_tokens_per_expert=[0, 2798, 795, 1186, 7761, 7537, 0, 0, 0, 0, 0, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2140, 0, 0, 0, 0, 1225, 3783, 97, 0, 0, 0, 0, 0, 131, 0, 0, 6133, 19, 149, 0, 0, 1222, 0, 0, 0, 0, 0, 0, 1449, 124, 0, 0, 0, 82, 5681, 219, 6294, 0, 0, 97, 0, 0, 0] payload_rows_per_dst=[16078, 4668, 8618, 15092, 12464, 2444, 3062, 12782] dup_rows_per_dst=[14140, 132, 1258, 6480, 400, 0, 248, 438] combine_rows_per_src=[16078, 4668, 8618, 15092, 12464, 2444, 3062, 12782] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89886 payload_rows_remote=82600 dup_rows=8418 dup_rows_remote=8128 dedup_ratio=0.085632 router_tokens_per_expert=[0, 3229, 780, 1240, 7754, 7583, 0, 0, 0, 0, 0, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2181, 0, 0, 0, 0, 1381, 2346, 61, 0, 0, 0, 0, 0, 144, 0, 0, 6487, 10, 120, 0, 0, 1377, 0, 0, 0, 0, 0, 0, 1415, 8, 0, 0, 0, 62, 6447, 203, 6022, 0, 0, 61, 0, 0, 0] payload_rows_per_dst=[3792, 15508, 9832, 7286, 13210, 14684, 13408, 12166] dup_rows_per_dst=[248, 482, 1740, 290, 312, 2484, 2456, 406] combine_rows_per_src=[3792, 15508, 9832, 7286, 13210, 14684, 13408, 12166] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=249126912 dispatch_meta_bytes=335688 combine_payload_bytes=249126912 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=249462600 ingress_bytes=318332928 link_bytes=318332928 token_bytes=498589512 weight_movement_bytes=69206016 total_nvlink_bytes=567795528 weight_to_token_ratio=0.138804 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=28800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.197861180e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=321798144 dispatch_meta_bytes=342232 combine_payload_bytes=321798144 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=322140376 ingress_bytes=391004160 link_bytes=391004160 token_bytes=643938520 weight_movement_bytes=69206016 total_nvlink_bytes=713144536 weight_to_token_ratio=0.107473 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=45696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.471317176e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89306 payload_rows_remote=75606 dup_rows=8998 dup_rows_remote=8736 dedup_ratio=0.091532 router_tokens_per_expert=[0, 3731, 927, 1168, 7448, 7547, 0, 0, 0, 0, 0, 242, 0, 0, 0, 0, 0, 0, 0, 41, 0, 1657, 1, 0, 0, 0, 1311, 1793, 80, 2, 0, 0, 0, 0, 246, 1, 0, 6614, 0, 120, 0, 62, 1719, 0, 0, 1, 0, 0, 0, 1681, 0, 0, 0, 0, 62, 6680, 197, 5740, 0, 0, 80, 0, 1, 0] payload_rows_per_dst=[3874, 14900, 9492, 6052, 13700, 15470, 14170, 11648] dup_rows_per_dst=[316, 480, 1368, 320, 262, 3188, 2676, 388] combine_rows_per_src=[3874, 14900, 9492, 6052, 13700, 15470, 14170, 11648] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=63000 payload_rows_remote=46738 dup_rows=35304 dup_rows_remote=9240 dedup_ratio=0.359131 router_tokens_per_expert=[0, 2225, 804, 1150, 7789, 7510, 0, 0, 0, 0, 0, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2870, 0, 0, 0, 0, 1202, 4315, 88, 0, 0, 0, 0, 0, 98, 0, 0, 6064, 53, 132, 0, 0, 900, 0, 0, 0, 11, 0, 0, 1620, 133, 0, 0, 0, 69, 5267, 245, 6267, 0, 0, 89, 0, 0, 0] payload_rows_per_dst=[16262, 502, 8972, 15168, 4750, 1800, 6676, 8870] dup_rows_per_dst=[26064, 0, 1218, 6576, 124, 22, 928, 372] combine_rows_per_src=[16262, 502, 8972, 15168, 4750, 1800, 6676, 8870] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=306978816 dispatch_meta_bytes=328640 combine_payload_bytes=306978816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307307456 ingress_bytes=341581824 link_bytes=341581824 token_bytes=614286272 weight_movement_bytes=34603008 total_nvlink_bytes=648889280 weight_to_token_ratio=0.056330 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=41984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.285344904e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=348971008 dispatch_meta_bytes=368832 combine_payload_bytes=348971008 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=349339840 ingress_bytes=452780032 link_bytes=452780032 token_bytes=698310848 weight_movement_bytes=103809024 total_nvlink_bytes=802119872 weight_to_token_ratio=0.148657 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=16640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.703774809e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=246374400 dispatch_meta_bytes=328104 combine_payload_bytes=246374400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=246702504 ingress_bytes=280977408 link_bytes=280977408 token_bytes=493076904 weight_movement_bytes=34603008 total_nvlink_bytes=527679912 weight_to_token_ratio=0.070178 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=43008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.057295366e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 2, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=301178880 dispatch_meta_bytes=334320 combine_payload_bytes=301178880 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=301513200 ingress_bytes=439590912 link_bytes=439590912 token_bytes=602692080 weight_movement_bytes=138412032 total_nvlink_bytes=741104112 weight_to_token_ratio=0.229656 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.654145212e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=249126912 dispatch_meta_bytes=335688 combine_payload_bytes=249126912 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=249462600 ingress_bytes=318332928 link_bytes=318332928 token_bytes=498589512 weight_movement_bytes=69206016 total_nvlink_bytes=567795528 weight_to_token_ratio=0.138804 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=28800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.197861180e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=313679872 dispatch_meta_bytes=329960 combine_payload_bytes=313679872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=314009832 ingress_bytes=382885888 link_bytes=382885888 token_bytes=627689704 weight_movement_bytes=69206016 total_nvlink_bytes=696895720 weight_to_token_ratio=0.110255 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=32896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.440768772e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=288931840 dispatch_meta_bytes=374016 combine_payload_bytes=288931840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289305856 ingress_bytes=323534848 link_bytes=323534848 token_bytes=578237696 weight_movement_bytes=34603008 total_nvlink_bytes=612840704 weight_to_token_ratio=0.059842 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=46464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.217435586e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=338329600 dispatch_meta_bytes=362912 combine_payload_bytes=338329600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=338692512 ingress_bytes=372932608 link_bytes=372932608 token_bytes=677022112 weight_movement_bytes=34603008 total_nvlink_bytes=711625120 weight_to_token_ratio=0.051111 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.403315380e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90330 payload_rows_remote=74946 dup_rows=7974 dup_rows_remote=7214 dedup_ratio=0.081116 router_tokens_per_expert=[0, 0, 0, 487, 0, 0, 0, 0, 0, 0, 0, 88, 0, 88, 883, 0, 0, 0, 3169, 3031, 7588, 742, 0, 88, 7586, 0, 0, 7084, 104, 62, 0, 0, 0, 0, 192, 0, 61, 0, 0, 6218, 0, 382, 0, 0, 0, 0, 102, 0, 0, 0, 0, 320, 3213, 0, 634, 144, 1204, 88, 0, 0, 0, 5133, 1, 460] payload_rows_per_dst=[12616, 14462, 7258, 8072, 12814, 15384, 7434, 12290] dup_rows_per_dst=[774, 1024, 740, 1878, 128, 760, 1188, 1482] combine_rows_per_src=[12616, 14462, 7258, 8072, 12814, 15384, 7434, 12290] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80974 payload_rows_remote=65654 dup_rows=17330 dup_rows_remote=9322 dedup_ratio=0.176290 router_tokens_per_expert=[1, 1860, 651, 1000, 6302, 6300, 102, 0, 32, 15, 94, 245, 134, 49, 243, 230, 669, 173, 40, 9, 465, 2846, 7, 0, 3, 0, 1125, 4493, 71, 1, 361, 23, 13, 40, 624, 25, 4, 4969, 461, 117, 129, 7, 654, 7, 397, 117, 135, 597, 217, 1241, 447, 0, 70, 712, 64, 3664, 146, 5917, 50, 441, 71, 33, 36, 203] payload_rows_per_dst=[4422, 13850, 15320, 13930, 11690, 3564, 4952, 13246] dup_rows_per_dst=[496, 838, 8008, 4960, 816, 522, 1142, 548] combine_rows_per_src=[4422, 13850, 15320, 13930, 11690, 3564, 4952, 13246] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=191438848 dispatch_meta_bytes=223912 combine_payload_bytes=191438848 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=191662760 ingress_bytes=364453888 link_bytes=364453888 token_bytes=383101608 weight_movement_bytes=173015040 total_nvlink_bytes=556116648 weight_to_token_ratio=0.451617 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=15872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.371410640e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=309682176 dispatch_meta_bytes=337368 combine_payload_bytes=309682176 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=310019544 ingress_bytes=378888192 link_bytes=378888192 token_bytes=619701720 weight_movement_bytes=69206016 total_nvlink_bytes=688907736 weight_to_token_ratio=0.111676 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=45568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.425725763e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89750 payload_rows_remote=74218 dup_rows=8554 dup_rows_remote=5768 dedup_ratio=0.087016 router_tokens_per_expert=[0, 3967, 961, 1158, 7576, 7578, 0, 0, 0, 0, 0, 257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1521, 0, 0, 0, 0, 1274, 1529, 88, 0, 0, 0, 0, 0, 204, 0, 0, 6659, 10, 140, 0, 0, 1581, 0, 0, 0, 0, 0, 0, 1459, 2, 0, 0, 0, 72, 6996, 212, 5820, 0, 0, 88, 0, 0, 0] payload_rows_per_dst=[3884, 15152, 9808, 5450, 13692, 15532, 14416, 11816] dup_rows_per_dst=[354, 514, 1168, 332, 334, 2786, 2642, 424] combine_rows_per_src=[3884, 15152, 9808, 5450, 13692, 15532, 14416, 11816] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89886 payload_rows_remote=82600 dup_rows=8418 dup_rows_remote=8128 dedup_ratio=0.085632 router_tokens_per_expert=[0, 3229, 780, 1240, 7754, 7583, 0, 0, 0, 0, 0, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2181, 0, 0, 0, 0, 1381, 2346, 61, 0, 0, 0, 0, 0, 144, 0, 0, 6487, 10, 120, 0, 0, 1377, 0, 0, 0, 0, 0, 0, 1415, 8, 0, 0, 0, 62, 6447, 203, 6022, 0, 0, 61, 0, 0, 0] payload_rows_per_dst=[3792, 15508, 9832, 7286, 13210, 14684, 13408, 12166] dup_rows_per_dst=[248, 482, 1740, 290, 312, 2484, 2456, 406] combine_rows_per_src=[3792, 15508, 9832, 7286, 13210, 14684, 13408, 12166] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89284 payload_rows_remote=77606 dup_rows=9020 dup_rows_remote=8312 dedup_ratio=0.091756 router_tokens_per_expert=[0, 2815, 900, 1113, 7418, 7484, 9, 0, 1, 1, 2, 250, 7, 3, 15, 12, 38, 5, 2, 4, 28, 2406, 0, 0, 0, 0, 1301, 3099, 110, 0, 23, 2, 1, 5, 350, 1, 0, 6257, 59, 123, 3, 0, 1363, 0, 11, 13, 37, 32, 11, 1604, 66, 0, 5, 39, 63, 6069, 188, 5663, 5, 27, 87, 3, 3, 16] payload_rows_per_dst=[3708, 14906, 9062, 8632, 12860, 15398, 13040, 11678] dup_rows_per_dst=[336, 512, 1534, 438, 330, 2488, 2674, 708] combine_rows_per_src=[3708, 14906, 9062, 8632, 12860, 15398, 13040, 11678] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89306 payload_rows_remote=75606 dup_rows=8998 dup_rows_remote=8736 dedup_ratio=0.091532 router_tokens_per_expert=[0, 3731, 927, 1168, 7448, 7547, 0, 0, 0, 0, 0, 242, 0, 0, 0, 0, 0, 0, 0, 41, 0, 1657, 1, 0, 0, 0, 1311, 1793, 80, 2, 0, 0, 0, 0, 246, 1, 0, 6614, 0, 120, 0, 62, 1719, 0, 0, 1, 0, 0, 0, 1681, 0, 0, 0, 0, 62, 6680, 197, 5740, 0, 0, 80, 0, 1, 0] payload_rows_per_dst=[3874, 14900, 9492, 6052, 13700, 15470, 14170, 11648] dup_rows_per_dst=[316, 480, 1368, 320, 262, 3188, 2676, 388] combine_rows_per_src=[3874, 14900, 9492, 6052, 13700, 15470, 14170, 11648] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77874 payload_rows_remote=65476 dup_rows=20430 dup_rows_remote=11174 dedup_ratio=0.207825 router_tokens_per_expert=[3, 0, 0, 0, 0, 0, 0, 2974, 561, 0, 216, 0, 0, 0, 269, 6029, 0, 0, 0, 4334, 256, 769, 0, 581, 0, 0, 0, 0, 0, 0, 0, 0, 3755, 0, 0, 0, 0, 0, 0, 5398, 0, 0, 0, 8060, 0, 0, 205, 0, 0, 0, 0, 3392, 0, 7855, 0, 0, 0, 0, 0, 3819, 148, 450, 0, 78] payload_rows_per_dst=[5948, 8146, 12602, 6914, 13556, 10218, 12398, 8092] dup_rows_per_dst=[6, 1002, 4280, 0, 4750, 238, 9256, 898] combine_rows_per_src=[5948, 8146, 12602, 6914, 13556, 10218, 12398, 8092] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=63000 payload_rows_remote=46738 dup_rows=35304 dup_rows_remote=9240 dedup_ratio=0.359131 router_tokens_per_expert=[0, 2225, 804, 1150, 7789, 7510, 0, 0, 0, 0, 0, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2870, 0, 0, 0, 0, 1202, 4315, 88, 0, 0, 0, 0, 0, 98, 0, 0, 6064, 53, 132, 0, 0, 900, 0, 0, 0, 11, 0, 0, 1620, 133, 0, 0, 0, 69, 5267, 245, 6267, 0, 0, 89, 0, 0, 0] payload_rows_per_dst=[16262, 502, 8972, 15168, 4750, 1800, 6676, 8870] dup_rows_per_dst=[26064, 0, 1218, 6576, 124, 22, 928, 372] combine_rows_per_src=[16262, 502, 8972, 15168, 4750, 1800, 6676, 8870] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=306978816 dispatch_meta_bytes=328640 combine_payload_bytes=306978816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307307456 ingress_bytes=341581824 link_bytes=341581824 token_bytes=614286272 weight_movement_bytes=34603008 total_nvlink_bytes=648889280 weight_to_token_ratio=0.056330 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=41984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.285344904e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=268918784 dispatch_meta_bytes=299904 combine_payload_bytes=268918784 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=269218688 ingress_bytes=338124800 link_bytes=338124800 token_bytes=538137472 weight_movement_bytes=69206016 total_nvlink_bytes=607343488 weight_to_token_ratio=0.128603 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.272336401e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91850 payload_rows_remote=76138 dup_rows=6454 dup_rows_remote=6454 dedup_ratio=0.065653 router_tokens_per_expert=[2, 0, 0, 0, 0, 0, 0, 2896, 536, 0, 250, 0, 0, 0, 223, 6097, 0, 0, 0, 4525, 248, 733, 0, 599, 0, 0, 0, 0, 0, 0, 0, 0, 3180, 0, 0, 0, 0, 0, 0, 5409, 0, 0, 0, 7992, 0, 0, 213, 0, 0, 0, 0, 3543, 0, 7856, 0, 0, 0, 0, 0, 4245, 123, 419, 0, 63] payload_rows_per_dst=[13698, 12526, 11620, 15712, 6360, 15986, 7086, 8862] dup_rows_per_dst=[2916, 1686, 590, 0, 0, 424, 0, 838] combine_rows_per_src=[13698, 12526, 11620, 15712, 6360, 15986, 7086, 8862] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85282 payload_rows_remote=79226 dup_rows=13022 dup_rows_remote=12700 dedup_ratio=0.132467 router_tokens_per_expert=[0, 0, 468, 1194, 121, 278, 0, 1, 29, 19, 0, 73, 266, 52, 1052, 358, 9, 43, 801, 1674, 5788, 608, 81, 79, 6318, 260, 5, 4962, 76, 94, 23, 4, 0, 477, 514, 645, 46, 13, 0, 5286, 371, 920, 40, 610, 27, 0, 110, 2, 127, 102, 152, 290, 5888, 115, 2714, 228, 1790, 150, 237, 3, 619, 1718, 214, 1008] payload_rows_per_dst=[14854, 12304, 6056, 4220, 12988, 11044, 13604, 10212] dup_rows_per_dst=[1906, 1318, 322, 52, 974, 1556, 5628, 1266] combine_rows_per_src=[14854, 12304, 6056, 4220, 12988, 11044, 13604, 10212] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91732 payload_rows_remote=79338 dup_rows=6572 dup_rows_remote=5016 dedup_ratio=0.066854 router_tokens_per_expert=[30, 0, 0, 0, 0, 0, 0, 3025, 495, 0, 317, 0, 0, 0, 202, 5961, 0, 0, 0, 3670, 275, 452, 0, 685, 0, 0, 0, 0, 0, 0, 0, 0, 2820, 0, 0, 0, 0, 0, 0, 5760, 0, 0, 0, 7821, 0, 0, 201, 0, 0, 0, 0, 4111, 0, 7750, 0, 0, 0, 0, 0, 4868, 179, 432, 0, 98] payload_rows_per_dst=[14198, 12394, 9830, 15500, 5640, 15658, 8222, 10290] dup_rows_per_dst=[3432, 1556, 334, 0, 0, 386, 0, 864] combine_rows_per_src=[14198, 12394, 9830, 15500, 5640, 15658, 8222, 10290] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=317874176 dispatch_meta_bytes=343672 combine_payload_bytes=317874176 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=318217848 ingress_bytes=387080192 link_bytes=387080192 token_bytes=636092024 weight_movement_bytes=69206016 total_nvlink_bytes=705298040 weight_to_token_ratio=0.108799 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=45952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.456551600e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90432 payload_rows_remote=75836 dup_rows=7872 dup_rows_remote=4130 dedup_ratio=0.080078 router_tokens_per_expert=[187, 0, 0, 0, 0, 0, 0, 2920, 467, 0, 312, 0, 0, 0, 93, 6228, 0, 0, 0, 3093, 268, 468, 0, 707, 0, 0, 0, 0, 0, 0, 0, 0, 1874, 0, 0, 0, 0, 0, 0, 6062, 0, 0, 0, 7679, 0, 0, 224, 0, 0, 0, 0, 4212, 0, 7771, 0, 0, 0, 0, 0, 5905, 158, 434, 0, 90] payload_rows_per_dst=[14596, 12866, 8768, 15542, 10898, 7032, 8424, 12306] dup_rows_per_dst=[3742, 1334, 304, 0, 1468, 156, 0, 868] combine_rows_per_src=[14596, 12866, 8768, 15542, 10898, 7032, 8424, 12306] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90124 payload_rows_remote=79414 dup_rows=8180 dup_rows_remote=6156 dedup_ratio=0.083211 router_tokens_per_expert=[0, 0, 0, 0, 0, 10, 0, 2945, 567, 0, 121, 0, 0, 0, 267, 5712, 1, 0, 0, 5513, 285, 1890, 0, 428, 0, 0, 0, 0, 0, 0, 0, 0, 3501, 0, 0, 0, 0, 0, 0, 5294, 0, 0, 0, 8076, 0, 0, 266, 1, 0, 0, 0, 1746, 0, 7901, 0, 0, 0, 0, 0, 3969, 142, 435, 0, 82] payload_rows_per_dst=[9498, 11898, 14166, 15802, 10710, 16154, 3492, 8404] dup_rows_per_dst=[1268, 1436, 2068, 0, 2024, 532, 0, 852] combine_rows_per_src=[9498, 11898, 14166, 15802, 10710, 16154, 3492, 8404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 2, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=309682176 dispatch_meta_bytes=337368 combine_payload_bytes=309682176 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=310019544 ingress_bytes=378888192 link_bytes=378888192 token_bytes=619701720 weight_movement_bytes=69206016 total_nvlink_bytes=688907736 weight_to_token_ratio=0.111676 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=45568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.425725763e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=303996928 dispatch_meta_bytes=319944 combine_payload_bytes=303996928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=304316872 ingress_bytes=338599936 link_bytes=338599936 token_bytes=608313800 weight_movement_bytes=34603008 total_nvlink_bytes=642916808 weight_to_token_ratio=0.056883 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=37504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274124300e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=338329600 dispatch_meta_bytes=362912 combine_payload_bytes=338329600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=338692512 ingress_bytes=372932608 link_bytes=372932608 token_bytes=677022112 weight_movement_bytes=34603008 total_nvlink_bytes=711625120 weight_to_token_ratio=0.051111 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.403315380e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=317874176 dispatch_meta_bytes=343672 combine_payload_bytes=317874176 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=318217848 ingress_bytes=387080192 link_bytes=387080192 token_bytes=636092024 weight_movement_bytes=69206016 total_nvlink_bytes=705298040 weight_to_token_ratio=0.108799 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=45952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.456551600e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=268189696 dispatch_meta_bytes=306600 combine_payload_bytes=268189696 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=268496296 ingress_bytes=337395712 link_bytes=337395712 token_bytes=536685992 weight_movement_bytes=69206016 total_nvlink_bytes=605892008 weight_to_token_ratio=0.128951 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=25728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.269592902e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=191438848 dispatch_meta_bytes=223912 combine_payload_bytes=191438848 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=191662760 ingress_bytes=364453888 link_bytes=364453888 token_bytes=383101608 weight_movement_bytes=173015040 total_nvlink_bytes=556116648 weight_to_token_ratio=0.451617 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=15872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.371410640e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=311861248 dispatch_meta_bytes=330368 combine_payload_bytes=311861248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=312191616 ingress_bytes=346464256 link_bytes=346464256 token_bytes=624052864 weight_movement_bytes=34603008 total_nvlink_bytes=658655872 weight_to_token_ratio=0.055449 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=48896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.303717103e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=324509696 dispatch_meta_bytes=367704 combine_payload_bytes=324509696 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=324877400 ingress_bytes=428318720 link_bytes=428318720 token_bytes=649387096 weight_movement_bytes=103809024 total_nvlink_bytes=753196120 weight_to_token_ratio=0.159857 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.611728861e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=324968448 dispatch_meta_bytes=337416 combine_payload_bytes=324968448 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=325305864 ingress_bytes=428777472 link_bytes=428777472 token_bytes=650274312 weight_movement_bytes=103809024 total_nvlink_bytes=754083336 weight_to_token_ratio=0.159639 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=38400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.613455108e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89750 payload_rows_remote=74218 dup_rows=8554 dup_rows_remote=5768 dedup_ratio=0.087016 router_tokens_per_expert=[0, 3967, 961, 1158, 7576, 7578, 0, 0, 0, 0, 0, 257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1521, 0, 0, 0, 0, 1274, 1529, 88, 0, 0, 0, 0, 0, 204, 0, 0, 6659, 10, 140, 0, 0, 1581, 0, 0, 0, 0, 0, 0, 1459, 2, 0, 0, 0, 72, 6996, 212, 5820, 0, 0, 88, 0, 0, 0] payload_rows_per_dst=[3884, 15152, 9808, 5450, 13692, 15532, 14416, 11816] dup_rows_per_dst=[354, 514, 1168, 332, 334, 2786, 2642, 424] combine_rows_per_src=[3884, 15152, 9808, 5450, 13692, 15532, 14416, 11816] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86956 payload_rows_remote=81032 dup_rows=11348 dup_rows_remote=10862 dedup_ratio=0.115438 router_tokens_per_expert=[1926, 675, 15, 23, 7, 10, 65, 1479, 426, 719, 2378, 51, 68, 4, 249, 5037, 4, 177, 40, 1543, 269, 350, 44, 778, 0, 13, 0, 14, 44, 77, 17, 99, 4111, 593, 108, 0, 9, 0, 0, 3952, 1, 710, 5, 5228, 340, 627, 181, 26, 12, 8, 2, 2749, 228, 6163, 0, 387, 7, 211, 5, 6266, 153, 385, 16, 68] payload_rows_per_dst=[13104, 14018, 5924, 12796, 9180, 12264, 6334, 13336] dup_rows_per_dst=[3200, 3846, 486, 58, 462, 1972, 438, 886] combine_rows_per_src=[13104, 14018, 5924, 12796, 9180, 12264, 6334, 13336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=310624256 dispatch_meta_bytes=319864 combine_payload_bytes=310624256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=310944120 ingress_bytes=345227264 link_bytes=345227264 token_bytes=621568376 weight_movement_bytes=34603008 total_nvlink_bytes=656171384 weight_to_token_ratio=0.055670 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.299062401e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=325279744 dispatch_meta_bytes=342280 combine_payload_bytes=325279744 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=325622024 ingress_bytes=394485760 link_bytes=394485760 token_bytes=650901768 weight_movement_bytes=69206016 total_nvlink_bytes=720107784 weight_to_token_ratio=0.106323 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.035000 gemm_active_groups=7 gemm_peak_group_rows=25216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.484418156e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90050 payload_rows_remote=73958 dup_rows=8254 dup_rows_remote=7818 dedup_ratio=0.083964 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 0, 3148, 479, 0, 164, 0, 0, 0, 267, 5720, 0, 0, 0, 5361, 180, 1485, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 3406, 0, 0, 0, 0, 0, 0, 5298, 0, 0, 0, 8046, 0, 0, 218, 0, 0, 0, 0, 2440, 0, 7979, 0, 0, 0, 0, 0, 3872, 160, 456, 0, 88] payload_rows_per_dst=[6298, 11940, 13288, 15958, 13346, 16092, 4880, 8248] dup_rows_per_dst=[0, 1320, 1532, 0, 4062, 436, 0, 904] combine_rows_per_src=[6298, 11940, 13288, 15958, 13346, 16092, 4880, 8248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91850 payload_rows_remote=76138 dup_rows=6454 dup_rows_remote=6454 dedup_ratio=0.065653 router_tokens_per_expert=[2, 0, 0, 0, 0, 0, 0, 2896, 536, 0, 250, 0, 0, 0, 223, 6097, 0, 0, 0, 4525, 248, 733, 0, 599, 0, 0, 0, 0, 0, 0, 0, 0, 3180, 0, 0, 0, 0, 0, 0, 5409, 0, 0, 0, 7992, 0, 0, 213, 0, 0, 0, 0, 3543, 0, 7856, 0, 0, 0, 0, 0, 4245, 123, 419, 0, 63] payload_rows_per_dst=[13698, 12526, 11620, 15712, 6360, 15986, 7086, 8862] dup_rows_per_dst=[2916, 1686, 590, 0, 0, 424, 0, 838] combine_rows_per_src=[13698, 12526, 11620, 15712, 6360, 15986, 7086, 8862] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=64804 payload_rows_remote=49592 dup_rows=33500 dup_rows_remote=26056 dedup_ratio=0.340780 router_tokens_per_expert=[177, 36, 1, 6, 1, 5, 2, 2972, 547, 42, 318, 1, 4, 0, 256, 6064, 0, 12, 1, 4250, 270, 665, 1, 672, 0, 1, 0, 0, 0, 1, 0, 9, 2510, 32, 5, 0, 0, 0, 0, 5377, 0, 39, 6, 7660, 29, 35, 183, 2, 1, 3, 0, 3509, 10, 7678, 0, 26, 0, 10, 1, 5052, 150, 432, 1, 87] payload_rows_per_dst=[6176, 2148, 11630, 22, 12844, 634, 16138, 15212] dup_rows_per_dst=[224, 188, 882, 0, 3004, 122, 21636, 7444] combine_rows_per_src=[6176, 2148, 11630, 22, 12844, 634, 16138, 15212] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90124 payload_rows_remote=79414 dup_rows=8180 dup_rows_remote=6156 dedup_ratio=0.083211 router_tokens_per_expert=[0, 0, 0, 0, 0, 10, 0, 2945, 567, 0, 121, 0, 0, 0, 267, 5712, 1, 0, 0, 5513, 285, 1890, 0, 428, 0, 0, 0, 0, 0, 0, 0, 0, 3501, 0, 0, 0, 0, 0, 0, 5294, 0, 0, 0, 8076, 0, 0, 266, 1, 0, 0, 0, 1746, 0, 7901, 0, 0, 0, 0, 0, 3969, 142, 435, 0, 82] payload_rows_per_dst=[9498, 11898, 14166, 15802, 10710, 16154, 3492, 8404] dup_rows_per_dst=[1268, 1436, 2068, 0, 2024, 532, 0, 852] combine_rows_per_src=[9498, 11898, 14166, 15802, 10710, 16154, 3492, 8404] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74636 payload_rows_remote=58812 dup_rows=23668 dup_rows_remote=22678 dedup_ratio=0.240763 router_tokens_per_expert=[0, 0, 909, 0, 0, 0, 0, 0, 0, 152, 0, 0, 7590, 0, 1641, 0, 0, 372, 51, 0, 0, 0, 262, 0, 724, 0, 0, 0, 0, 13, 0, 6510, 7585, 0, 183, 0, 0, 0, 183, 0, 447, 1880, 0, 0, 0, 7202, 7578, 76, 0, 495, 0, 0, 244, 0, 78, 0, 0, 0, 4977, 0, 0, 0, 0, 0] payload_rows_per_dst=[1818, 3282, 1268, 9084, 15364, 16384, 15824, 11612] dup_rows_per_dst=[0, 304, 102, 850, 538, 18974, 990, 1910] combine_rows_per_src=[1818, 3282, 1268, 9084, 15364, 16384, 15824, 11612] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90432 payload_rows_remote=75836 dup_rows=7872 dup_rows_remote=4130 dedup_ratio=0.080078 router_tokens_per_expert=[187, 0, 0, 0, 0, 0, 0, 2920, 467, 0, 312, 0, 0, 0, 93, 6228, 0, 0, 0, 3093, 268, 468, 0, 707, 0, 0, 0, 0, 0, 0, 0, 0, 1874, 0, 0, 0, 0, 0, 0, 6062, 0, 0, 0, 7679, 0, 0, 224, 0, 0, 0, 0, 4212, 0, 7771, 0, 0, 0, 0, 0, 5905, 158, 434, 0, 90] payload_rows_per_dst=[14596, 12866, 8768, 15542, 10898, 7032, 8424, 12306] dup_rows_per_dst=[3742, 1334, 304, 0, 1468, 156, 0, 868] combine_rows_per_src=[14596, 12866, 8768, 15542, 10898, 7032, 8424, 12306] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93490 payload_rows_remote=80298 dup_rows=4814 dup_rows_remote=3742 dedup_ratio=0.048971 router_tokens_per_expert=[0, 0, 900, 0, 0, 0, 0, 0, 0, 111, 0, 0, 7620, 0, 1598, 0, 0, 356, 54, 0, 0, 0, 249, 0, 608, 0, 0, 0, 0, 0, 0, 6524, 7612, 0, 143, 0, 0, 0, 164, 0, 376, 1964, 0, 0, 0, 7265, 7626, 87, 0, 596, 0, 0, 242, 0, 63, 0, 0, 0, 4994, 0, 0, 0, 0, 0] payload_rows_per_dst=[15280, 3196, 16072, 13192, 15386, 4526, 15850, 9988] dup_rows_per_dst=[1050, 222, 498, 1072, 452, 328, 1192, 0] combine_rows_per_src=[15280, 3196, 16072, 13192, 15386, 4526, 15850, 9988] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80974 payload_rows_remote=65654 dup_rows=17330 dup_rows_remote=9322 dedup_ratio=0.176290 router_tokens_per_expert=[1, 1860, 651, 1000, 6302, 6300, 102, 0, 32, 15, 94, 245, 134, 49, 243, 230, 669, 173, 40, 9, 465, 2846, 7, 0, 3, 0, 1125, 4493, 71, 1, 361, 23, 13, 40, 624, 25, 4, 4969, 461, 117, 129, 7, 654, 7, 397, 117, 135, 597, 217, 1241, 447, 0, 70, 712, 64, 3664, 146, 5917, 50, 441, 71, 33, 36, 203] payload_rows_per_dst=[4422, 13850, 15320, 13930, 11690, 3564, 4952, 13246] dup_rows_per_dst=[496, 838, 8008, 4960, 816, 522, 1142, 548] combine_rows_per_src=[4422, 13850, 15320, 13930, 11690, 3564, 4952, 13246] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77374 payload_rows_remote=61118 dup_rows=20930 dup_rows_remote=3714 dedup_ratio=0.212911 router_tokens_per_expert=[0, 0, 895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7541, 0, 1729, 0, 0, 334, 64, 0, 0, 0, 251, 0, 748, 0, 0, 0, 0, 0, 0, 6572, 7544, 0, 200, 0, 0, 0, 183, 0, 517, 1739, 0, 0, 0, 7187, 7597, 47, 0, 638, 0, 0, 231, 0, 98, 0, 0, 0, 5037, 0, 0, 0, 0, 0] payload_rows_per_dst=[15202, 16256, 15990, 13358, 3950, 1128, 1416, 10074] dup_rows_per_dst=[962, 17216, 502, 1282, 450, 0, 518, 0] combine_rows_per_src=[15202, 16256, 15990, 13358, 3950, 1128, 1416, 10074] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=303996928 dispatch_meta_bytes=319944 combine_payload_bytes=303996928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=304316872 ingress_bytes=338599936 link_bytes=338599936 token_bytes=608313800 weight_movement_bytes=34603008 total_nvlink_bytes=642916808 weight_to_token_ratio=0.056883 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=37504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274124300e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=331907072 dispatch_meta_bytes=367576 combine_payload_bytes=331907072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=332274648 ingress_bytes=366510080 link_bytes=366510080 token_bytes=664181720 weight_movement_bytes=34603008 total_nvlink_bytes=698784728 weight_to_token_ratio=0.052099 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=32384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.379147925e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=203128832 dispatch_meta_bytes=302592 combine_payload_bytes=203128832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=203431424 ingress_bytes=237731840 link_bytes=237731840 token_bytes=406560256 weight_movement_bytes=34603008 total_nvlink_bytes=441163264 weight_to_token_ratio=0.085112 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=38016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.945657749e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76996 payload_rows_remote=61952 dup_rows=21308 dup_rows_remote=20426 dedup_ratio=0.216756 router_tokens_per_expert=[0, 0, 791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7606, 0, 1909, 0, 0, 344, 61, 0, 0, 0, 242, 0, 979, 0, 0, 0, 0, 0, 0, 6516, 7212, 0, 149, 0, 0, 0, 198, 0, 566, 1390, 0, 0, 0, 7172, 7671, 59, 0, 697, 0, 0, 252, 0, 90, 0, 0, 0, 5248, 0, 0, 0, 0, 0] payload_rows_per_dst=[15044, 16262, 16150, 13332, 3052, 1250, 1410, 10496] dup_rows_per_dst=[882, 17192, 486, 1658, 422, 0, 668, 0] combine_rows_per_src=[15044, 16262, 16150, 13332, 3052, 1250, 1410, 10496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93608 payload_rows_remote=78182 dup_rows=4696 dup_rows_remote=4210 dedup_ratio=0.047770 router_tokens_per_expert=[0, 0, 887, 0, 0, 6, 0, 0, 0, 238, 0, 0, 7500, 0, 2213, 0, 0, 355, 52, 0, 0, 0, 269, 0, 534, 0, 0, 0, 0, 52, 0, 6570, 7605, 0, 192, 0, 0, 0, 158, 1, 355, 2103, 0, 0, 0, 7097, 7495, 61, 0, 488, 0, 0, 242, 0, 80, 0, 0, 0, 4598, 0, 0, 0, 1, 0] payload_rows_per_dst=[15038, 4428, 15810, 13244, 15426, 4820, 15644, 9198] dup_rows_per_dst=[942, 474, 532, 1068, 486, 218, 976, 0] combine_rows_per_src=[15038, 4428, 15810, 13244, 15426, 4820, 15644, 9198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=302931968 dispatch_meta_bytes=327104 combine_payload_bytes=302931968 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=303259072 ingress_bytes=406740992 link_bytes=406740992 token_bytes=606191040 weight_movement_bytes=103809024 total_nvlink_bytes=710000064 weight_to_token_ratio=0.171248 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=45568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.530533608e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=325279744 dispatch_meta_bytes=342280 combine_payload_bytes=325279744 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=325622024 ingress_bytes=394485760 link_bytes=394485760 token_bytes=650901768 weight_movement_bytes=69206016 total_nvlink_bytes=720107784 weight_to_token_ratio=0.106323 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.035000 gemm_active_groups=7 gemm_peak_group_rows=25216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.484418156e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=311861248 dispatch_meta_bytes=330368 combine_payload_bytes=311861248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=312191616 ingress_bytes=346464256 link_bytes=346464256 token_bytes=624052864 weight_movement_bytes=34603008 total_nvlink_bytes=658655872 weight_to_token_ratio=0.055449 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=48896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.303717103e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=203128832 dispatch_meta_bytes=302592 combine_payload_bytes=203128832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=203431424 ingress_bytes=237731840 link_bytes=237731840 token_bytes=406560256 weight_movement_bytes=34603008 total_nvlink_bytes=441163264 weight_to_token_ratio=0.085112 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=38016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.945657749e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=240893952 dispatch_meta_bytes=325960 combine_payload_bytes=240893952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=241219912 ingress_bytes=275496960 link_bytes=275496960 token_bytes=482113864 weight_movement_bytes=34603008 total_nvlink_bytes=516716872 weight_to_token_ratio=0.071774 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=40320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.036672881e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=310624256 dispatch_meta_bytes=319864 combine_payload_bytes=310624256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=310944120 ingress_bytes=345227264 link_bytes=345227264 token_bytes=621568376 weight_movement_bytes=34603008 total_nvlink_bytes=656171384 weight_to_token_ratio=0.055670 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.299062401e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=328900608 dispatch_meta_bytes=336160 combine_payload_bytes=328900608 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=329236768 ingress_bytes=398106624 link_bytes=398106624 token_bytes=658137376 weight_movement_bytes=69206016 total_nvlink_bytes=727343392 weight_to_token_ratio=0.105154 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=41472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.498043176e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=253755392 dispatch_meta_bytes=329512 combine_payload_bytes=253755392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=254084904 ingress_bytes=288358400 link_bytes=288358400 token_bytes=507840296 weight_movement_bytes=34603008 total_nvlink_bytes=542443304 weight_to_token_ratio=0.068138 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=41344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.085069444e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=250339328 dispatch_meta_bytes=259328 combine_payload_bytes=250339328 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=250598656 ingress_bytes=319545344 link_bytes=319545344 token_bytes=500937984 weight_movement_bytes=69206016 total_nvlink_bytes=570144000 weight_to_token_ratio=0.138153 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.202423404e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90050 payload_rows_remote=73958 dup_rows=8254 dup_rows_remote=7818 dedup_ratio=0.083964 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 0, 3148, 479, 0, 164, 0, 0, 0, 267, 5720, 0, 0, 0, 5361, 180, 1485, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 3406, 0, 0, 0, 0, 0, 0, 5298, 0, 0, 0, 8046, 0, 0, 218, 0, 0, 0, 0, 2440, 0, 7979, 0, 0, 0, 0, 0, 3872, 160, 456, 0, 88] payload_rows_per_dst=[6298, 11940, 13288, 15958, 13346, 16092, 4880, 8248] dup_rows_per_dst=[0, 1320, 1532, 0, 4062, 436, 0, 904] combine_rows_per_src=[6298, 11940, 13288, 15958, 13346, 16092, 4880, 8248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87586 payload_rows_remote=74490 dup_rows=10718 dup_rows_remote=10042 dedup_ratio=0.109029 router_tokens_per_expert=[39, 60, 1394, 114, 19, 7, 52, 24, 602, 134, 34, 0, 5123, 5, 2237, 25, 140, 363, 49, 2, 0, 8, 190, 46, 2442, 108, 104, 4, 19, 39, 8, 4419, 4329, 11, 118, 0, 675, 0, 327, 0, 1475, 2320, 58, 240, 747, 6267, 6088, 38, 741, 661, 4, 175, 890, 0, 68, 4, 3, 758, 4585, 659, 35, 26, 19, 21] payload_rows_per_dst=[13730, 9376, 13096, 12126, 10956, 8156, 9250, 10896] dup_rows_per_dst=[2222, 1714, 676, 2160, 476, 1088, 1066, 1316] combine_rows_per_src=[13730, 9376, 13096, 12126, 10956, 8156, 9250, 10896] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71642 payload_rows_remote=55818 dup_rows=26662 dup_rows_remote=19218 dedup_ratio=0.271220 router_tokens_per_expert=[4, 2, 872, 6, 5, 0, 2, 0, 40, 121, 1, 0, 7471, 1, 2101, 0, 19, 358, 52, 1, 0, 0, 238, 6, 693, 5, 4, 0, 0, 6, 0, 6528, 7306, 1, 218, 0, 42, 0, 164, 0, 632, 1429, 4, 11, 43, 7102, 7465, 48, 44, 604, 0, 6, 304, 0, 87, 0, 0, 44, 5023, 32, 3, 1, 1, 2] payload_rows_per_dst=[1768, 4282, 1234, 1408, 14974, 16376, 15776, 15824] dup_rows_per_dst=[14, 246, 114, 8, 488, 17092, 1256, 7444] combine_rows_per_src=[1768, 4282, 1234, 1408, 14974, 16376, 15776, 15824] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93608 payload_rows_remote=78182 dup_rows=4696 dup_rows_remote=4210 dedup_ratio=0.047770 router_tokens_per_expert=[0, 0, 887, 0, 0, 6, 0, 0, 0, 238, 0, 0, 7500, 0, 2213, 0, 0, 355, 52, 0, 0, 0, 269, 0, 534, 0, 0, 0, 0, 52, 0, 6570, 7605, 0, 192, 0, 0, 0, 158, 1, 355, 2103, 0, 0, 0, 7097, 7495, 61, 0, 488, 0, 0, 242, 0, 80, 0, 0, 0, 4598, 0, 0, 0, 1, 0] payload_rows_per_dst=[15038, 4428, 15810, 13244, 15426, 4820, 15644, 9198] dup_rows_per_dst=[942, 474, 532, 1068, 486, 218, 976, 0] combine_rows_per_src=[15038, 4428, 15810, 13244, 15426, 4820, 15644, 9198] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=54138 payload_rows_remote=42330 dup_rows=44166 dup_rows_remote=43688 dedup_ratio=0.449280 router_tokens_per_expert=[0, 244, 0, 0, 0, 0, 354, 31, 78, 7450, 0, 0, 0, 78, 0, 112, 84, 0, 0, 51, 0, 204, 0, 0, 0, 0, 0, 0, 585, 7739, 0, 0, 0, 0, 0, 0, 0, 7319, 0, 8114, 78, 0, 0, 1167, 35, 0, 79, 0, 244, 0, 0, 0, 0, 5899, 0, 0, 0, 0, 660, 7833, 0, 0, 0, 714] payload_rows_per_dst=[870, 4522, 678, 1170, 16384, 2492, 11808, 16214] dup_rows_per_dst=[388, 278, 0, 0, 29960, 226, 478, 12836] combine_rows_per_src=[870, 4522, 678, 1170, 16384, 2492, 11808, 16214] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76996 payload_rows_remote=61952 dup_rows=21308 dup_rows_remote=20426 dedup_ratio=0.216756 router_tokens_per_expert=[0, 0, 791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7606, 0, 1909, 0, 0, 344, 61, 0, 0, 0, 242, 0, 979, 0, 0, 0, 0, 0, 0, 6516, 7212, 0, 149, 0, 0, 0, 198, 0, 566, 1390, 0, 0, 0, 7172, 7671, 59, 0, 697, 0, 0, 252, 0, 90, 0, 0, 0, 5248, 0, 0, 0, 0, 0] payload_rows_per_dst=[15044, 16262, 16150, 13332, 3052, 1250, 1410, 10496] dup_rows_per_dst=[882, 17192, 486, 1658, 422, 0, 668, 0] combine_rows_per_src=[15044, 16262, 16150, 13332, 3052, 1250, 1410, 10496] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92314 payload_rows_remote=76588 dup_rows=5990 dup_rows_remote=4960 dedup_ratio=0.060933 router_tokens_per_expert=[0, 230, 0, 0, 0, 0, 326, 0, 61, 7430, 7, 0, 0, 61, 0, 108, 85, 0, 0, 54, 0, 198, 0, 0, 0, 0, 0, 0, 597, 7781, 0, 0, 0, 0, 0, 0, 0, 7416, 0, 8131, 63, 0, 0, 1206, 15, 0, 67, 0, 243, 0, 0, 0, 0, 5997, 0, 0, 0, 0, 568, 7882, 0, 0, 0, 626] payload_rows_per_dst=[14932, 14984, 16262, 15726, 0, 16024, 11998, 2388] dup_rows_per_dst=[1012, 350, 674, 1030, 0, 2442, 482, 0] combine_rows_per_src=[14932, 14984, 16262, 15726, 0, 16024, 11998, 2388] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92574 payload_rows_remote=77416 dup_rows=5730 dup_rows_remote=5262 dedup_ratio=0.058289 router_tokens_per_expert=[0, 189, 0, 0, 0, 0, 319, 4, 97, 7497, 27, 0, 0, 97, 0, 95, 60, 0, 0, 71, 0, 222, 0, 0, 0, 0, 0, 0, 574, 7786, 0, 0, 0, 0, 0, 0, 0, 7338, 0, 8095, 98, 0, 0, 1027, 83, 0, 116, 0, 233, 0, 0, 0, 0, 5729, 0, 0, 0, 0, 627, 7820, 0, 0, 0, 948] payload_rows_per_dst=[14754, 15158, 16190, 15792, 0, 16068, 11462, 3150] dup_rows_per_dst=[946, 468, 706, 928, 0, 2220, 462, 0] combine_rows_per_src=[14754, 15158, 16190, 15792, 0, 16068, 11462, 3150] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=302931968 dispatch_meta_bytes=327104 combine_payload_bytes=302931968 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=303259072 ingress_bytes=406740992 link_bytes=406740992 token_bytes=606191040 weight_movement_bytes=103809024 total_nvlink_bytes=710000064 weight_to_token_ratio=0.171248 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=45568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.530533608e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=305111040 dispatch_meta_bytes=338128 combine_payload_bytes=305111040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=305449168 ingress_bytes=339714048 link_bytes=339714048 token_bytes=610560208 weight_movement_bytes=34603008 total_nvlink_bytes=645163216 weight_to_token_ratio=0.056674 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=43776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.278316613e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=173383680 dispatch_meta_bytes=344072 combine_payload_bytes=173383680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=173727752 ingress_bytes=207986688 link_bytes=207986688 token_bytes=347111432 weight_movement_bytes=34603008 total_nvlink_bytes=381714440 weight_to_token_ratio=0.099688 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=44032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=7.826371626e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=228630528 dispatch_meta_bytes=300144 combine_payload_bytes=228630528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=228930672 ingress_bytes=263233536 link_bytes=263233536 token_bytes=457561200 weight_movement_bytes=34603008 total_nvlink_bytes=492164208 weight_to_token_ratio=0.075625 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=39296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.905266039e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=268918784 dispatch_meta_bytes=299904 combine_payload_bytes=268918784 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=269218688 ingress_bytes=338124800 link_bytes=338124800 token_bytes=538137472 weight_movement_bytes=69206016 total_nvlink_bytes=607343488 weight_to_token_ratio=0.128603 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.272336401e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=320233472 dispatch_meta_bytes=329568 combine_payload_bytes=320233472 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=320563040 ingress_bytes=389439488 link_bytes=389439488 token_bytes=640796512 weight_movement_bytes=69206016 total_nvlink_bytes=710002528 weight_to_token_ratio=0.108000 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=42112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.465429441e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=313704448 dispatch_meta_bytes=326192 combine_payload_bytes=313704448 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=314030640 ingress_bytes=417513472 link_bytes=417513472 token_bytes=627735088 weight_movement_bytes=103809024 total_nvlink_bytes=731544112 weight_to_token_ratio=0.165371 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=43776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.571069583e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=253755392 dispatch_meta_bytes=329512 combine_payload_bytes=253755392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=254084904 ingress_bytes=288358400 link_bytes=288358400 token_bytes=507840296 weight_movement_bytes=34603008 total_nvlink_bytes=542443304 weight_to_token_ratio=0.068138 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=41344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.085069444e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=317095936 dispatch_meta_bytes=330712 combine_payload_bytes=317095936 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=317426648 ingress_bytes=386301952 link_bytes=386301952 token_bytes=634522584 weight_movement_bytes=69206016 total_nvlink_bytes=703728600 weight_to_token_ratio=0.109068 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=45824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.453623146e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12606409080 prefetch_bytes=692060160 reduce_bytes=588251136 weight_movement_bytes=1280311296 egress_bytes=6307023224 ingress_bytes=7579697152 link_bytes=7579697152 total_nvlink_bytes=13886720376 weight_to_token_ratio=0.101560 prefetch_share_of_ingress=0.091304 dedup_ratio=0.192807 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=17 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027500 gemm_inner_pad_upper_bound=27813 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.240077833e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13799475920 prefetch_bytes=726663168 reduce_bytes=795869184 weight_movement_bytes=1522532352 egress_bytes=6903663312 ingress_bytes=8418344960 link_bytes=8418344960 total_nvlink_bytes=15322008272 weight_to_token_ratio=0.110333 prefetch_share_of_ingress=0.086319 dedup_ratio=0.139510 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=23 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027609 gemm_inner_pad_upper_bound=28067 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.377284972e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12874962448 prefetch_bytes=726663168 reduce_bytes=553648128 weight_movement_bytes=1280311296 egress_bytes=6441317904 ingress_bytes=7713955840 link_bytes=7713955840 total_nvlink_bytes=14155273744 weight_to_token_ratio=0.099442 prefetch_share_of_ingress=0.094201 dedup_ratio=0.188405 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=16 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28067 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.262043252e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92824 payload_rows_remote=86012 dup_rows=5480 dup_rows_remote=4328 dedup_ratio=0.055745 router_tokens_per_expert=[0, 0, 874, 0, 0, 2, 0, 0, 0, 243, 0, 0, 7475, 0, 1684, 0, 0, 369, 62, 0, 0, 0, 225, 0, 588, 0, 0, 0, 0, 57, 0, 6586, 7575, 0, 192, 0, 0, 0, 175, 0, 388, 2472, 0, 0, 0, 7134, 7462, 57, 0, 414, 0, 0, 257, 0, 88, 0, 0, 0, 4773, 0, 0, 0, 0, 0] payload_rows_per_dst=[13466, 3368, 15334, 13300, 15358, 6812, 15640, 9546] dup_rows_per_dst=[886, 486, 440, 1162, 526, 1152, 828, 0] combine_rows_per_src=[13466, 3368, 15334, 13300, 15358, 6812, 15640, 9546] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85690 payload_rows_remote=69604 dup_rows=12614 dup_rows_remote=10506 dedup_ratio=0.128316 router_tokens_per_expert=[1, 124, 504, 728, 787, 394, 303, 458, 69, 6390, 180, 0, 0, 360, 62, 72, 46, 136, 0, 221, 674, 246, 483, 0, 1, 139, 6, 20, 678, 5902, 548, 7, 64, 772, 0, 77, 138, 5540, 51, 7291, 68, 5, 25, 2039, 260, 87, 847, 27, 956, 13, 23, 7, 10, 1772, 0, 0, 5, 0, 744, 6021, 18, 31, 33, 2689] payload_rows_per_dst=[13640, 13396, 16086, 13840, 2060, 14224, 5438, 7006] dup_rows_per_dst=[4038, 870, 2108, 762, 144, 4534, 124, 34] combine_rows_per_src=[13640, 13396, 16086, 13840, 2060, 14224, 5438, 7006] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=51272 payload_rows_remote=35008 dup_rows=47032 dup_rows_remote=30092 dedup_ratio=0.478434 router_tokens_per_expert=[0, 229, 29, 42, 43, 24, 332, 86, 87, 7404, 10, 0, 0, 114, 5, 75, 51, 15, 0, 67, 38, 192, 28, 0, 0, 14, 0, 0, 598, 7598, 23, 0, 2, 43, 0, 4, 7, 7176, 2, 8052, 87, 0, 0, 964, 93, 3, 118, 6, 297, 0, 1, 0, 0, 5995, 0, 0, 0, 0, 715, 7741, 4, 15, 2, 721] payload_rows_per_dst=[1034, 380, 726, 1258, 16312, 2182, 13116, 16264] dup_rows_per_dst=[536, 202, 56, 12, 24700, 360, 4226, 16940] combine_rows_per_src=[1034, 380, 726, 1258, 16312, 2182, 13116, 16264] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92180 payload_rows_remote=92180 dup_rows=6124 dup_rows_remote=6124 dedup_ratio=0.062297 router_tokens_per_expert=[0, 257, 0, 0, 0, 0, 477, 18, 80, 7419, 1, 0, 0, 80, 2, 131, 62, 0, 0, 55, 0, 207, 0, 0, 0, 0, 0, 0, 579, 7772, 0, 0, 0, 0, 0, 0, 0, 7373, 0, 8104, 80, 0, 0, 1072, 11, 0, 80, 0, 242, 0, 0, 0, 0, 6342, 0, 0, 0, 0, 590, 7827, 0, 4, 0, 287] payload_rows_per_dst=[14852, 14964, 16210, 15706, 0, 15234, 12684, 2530] dup_rows_per_dst=[1398, 462, 646, 996, 0, 2032, 484, 106] combine_rows_per_src=[14852, 14964, 16210, 15706, 0, 15234, 12684, 2530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93490 payload_rows_remote=80298 dup_rows=4814 dup_rows_remote=3742 dedup_ratio=0.048971 router_tokens_per_expert=[0, 0, 900, 0, 0, 0, 0, 0, 0, 111, 0, 0, 7620, 0, 1598, 0, 0, 356, 54, 0, 0, 0, 249, 0, 608, 0, 0, 0, 0, 0, 0, 6524, 7612, 0, 143, 0, 0, 0, 164, 0, 376, 1964, 0, 0, 0, 7265, 7626, 87, 0, 596, 0, 0, 242, 0, 63, 0, 0, 0, 4994, 0, 0, 0, 0, 0] payload_rows_per_dst=[15280, 3196, 16072, 13192, 15386, 4526, 15850, 9988] dup_rows_per_dst=[1050, 222, 498, 1072, 452, 328, 1192, 0] combine_rows_per_src=[15280, 3196, 16072, 13192, 15386, 4526, 15850, 9988] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71642 payload_rows_remote=55818 dup_rows=26662 dup_rows_remote=19218 dedup_ratio=0.271220 router_tokens_per_expert=[4, 2, 872, 6, 5, 0, 2, 0, 40, 121, 1, 0, 7471, 1, 2101, 0, 19, 358, 52, 1, 0, 0, 238, 6, 693, 5, 4, 0, 0, 6, 0, 6528, 7306, 1, 218, 0, 42, 0, 164, 0, 632, 1429, 4, 11, 43, 7102, 7465, 48, 44, 604, 0, 6, 304, 0, 87, 0, 0, 44, 5023, 32, 3, 1, 1, 2] payload_rows_per_dst=[1768, 4282, 1234, 1408, 14974, 16376, 15776, 15824] dup_rows_per_dst=[14, 246, 114, 8, 488, 17092, 1256, 7444] combine_rows_per_src=[1768, 4282, 1234, 1408, 14974, 16376, 15776, 15824] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=54138 payload_rows_remote=42330 dup_rows=44166 dup_rows_remote=43688 dedup_ratio=0.449280 router_tokens_per_expert=[0, 244, 0, 0, 0, 0, 354, 31, 78, 7450, 0, 0, 0, 78, 0, 112, 84, 0, 0, 51, 0, 204, 0, 0, 0, 0, 0, 0, 585, 7739, 0, 0, 0, 0, 0, 0, 0, 7319, 0, 8114, 78, 0, 0, 1167, 35, 0, 79, 0, 244, 0, 0, 0, 0, 5899, 0, 0, 0, 0, 660, 7833, 0, 0, 0, 714] payload_rows_per_dst=[870, 4522, 678, 1170, 16384, 2492, 11808, 16214] dup_rows_per_dst=[388, 278, 0, 0, 29960, 226, 478, 12836] combine_rows_per_src=[870, 4522, 678, 1170, 16384, 2492, 11808, 16214] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92058 payload_rows_remote=77100 dup_rows=6246 dup_rows_remote=5334 dedup_ratio=0.063538 router_tokens_per_expert=[0, 175, 0, 0, 0, 0, 307, 2, 89, 7495, 20, 0, 0, 89, 0, 69, 86, 0, 0, 70, 0, 184, 0, 0, 0, 0, 0, 0, 571, 7748, 0, 0, 0, 0, 0, 0, 0, 7451, 0, 8103, 90, 0, 0, 810, 88, 0, 118, 0, 253, 0, 0, 0, 0, 6113, 0, 0, 0, 0, 494, 7834, 0, 0, 0, 893] payload_rows_per_dst=[14958, 15232, 16206, 14576, 0, 16082, 12230, 2774] dup_rows_per_dst=[912, 1544, 680, 810, 0, 1798, 502, 0] combine_rows_per_src=[14958, 15232, 16206, 14576, 0, 16082, 12230, 2774] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92574 payload_rows_remote=77416 dup_rows=5730 dup_rows_remote=5262 dedup_ratio=0.058289 router_tokens_per_expert=[0, 189, 0, 0, 0, 0, 319, 4, 97, 7497, 27, 0, 0, 97, 0, 95, 60, 0, 0, 71, 0, 222, 0, 0, 0, 0, 0, 0, 574, 7786, 0, 0, 0, 0, 0, 0, 0, 7338, 0, 8095, 98, 0, 0, 1027, 83, 0, 116, 0, 233, 0, 0, 0, 0, 5729, 0, 0, 0, 0, 627, 7820, 0, 0, 0, 948] payload_rows_per_dst=[14754, 15158, 16190, 15792, 0, 16068, 11462, 3150] dup_rows_per_dst=[946, 468, 706, 928, 0, 2220, 462, 0] combine_rows_per_src=[14754, 15158, 16190, 15792, 0, 16068, 11462, 3150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92180 payload_rows_remote=92180 dup_rows=6124 dup_rows_remote=6124 dedup_ratio=0.062297 router_tokens_per_expert=[0, 257, 0, 0, 0, 0, 477, 18, 80, 7419, 1, 0, 0, 80, 2, 131, 62, 0, 0, 55, 0, 207, 0, 0, 0, 0, 0, 0, 579, 7772, 0, 0, 0, 0, 0, 0, 0, 7373, 0, 8104, 80, 0, 0, 1072, 11, 0, 80, 0, 242, 0, 0, 0, 0, 6342, 0, 0, 0, 0, 590, 7827, 0, 4, 0, 287] payload_rows_per_dst=[14852, 14964, 16210, 15706, 0, 15234, 12684, 2530] dup_rows_per_dst=[1398, 462, 646, 996, 0, 2032, 484, 106] combine_rows_per_src=[14852, 14964, 16210, 15706, 0, 15234, 12684, 2530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86956 payload_rows_remote=81032 dup_rows=11348 dup_rows_remote=10862 dedup_ratio=0.115438 router_tokens_per_expert=[1926, 675, 15, 23, 7, 10, 65, 1479, 426, 719, 2378, 51, 68, 4, 249, 5037, 4, 177, 40, 1543, 269, 350, 44, 778, 0, 13, 0, 14, 44, 77, 17, 99, 4111, 593, 108, 0, 9, 0, 0, 3952, 1, 710, 5, 5228, 340, 627, 181, 26, 12, 8, 2, 2749, 228, 6163, 0, 387, 7, 211, 5, 6266, 153, 385, 16, 68] payload_rows_per_dst=[13104, 14018, 5924, 12796, 9180, 12264, 6334, 13336] dup_rows_per_dst=[3200, 3846, 486, 58, 462, 1972, 438, 886] combine_rows_per_src=[13104, 14018, 5924, 12796, 9180, 12264, 6334, 13336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=352305152 dispatch_meta_bytes=361360 combine_payload_bytes=352305152 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=352666512 ingress_bytes=490717184 link_bytes=490717184 token_bytes=704971664 weight_movement_bytes=138412032 total_nvlink_bytes=843383696 weight_to_token_ratio=0.196337 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.846529257e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=285097984 dispatch_meta_bytes=320440 combine_payload_bytes=285097984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=285418424 ingress_bytes=319700992 link_bytes=319700992 token_bytes=570516408 weight_movement_bytes=34603008 total_nvlink_bytes=605119416 weight_to_token_ratio=0.060652 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=45056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.203009095e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=143392768 dispatch_meta_bytes=260400 combine_payload_bytes=143392768 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=143653168 ingress_bytes=212598784 link_bytes=212598784 token_bytes=287045936 weight_movement_bytes=69206016 total_nvlink_bytes=356251952 weight_to_token_ratio=0.241097 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=23936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=7.999921086e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=377569280 dispatch_meta_bytes=393216 combine_payload_bytes=377569280 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=377962496 ingress_bytes=481378304 link_bytes=481378304 token_bytes=755531776 weight_movement_bytes=103809024 total_nvlink_bytes=859340800 weight_to_token_ratio=0.137399 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=19072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.811387804e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=328900608 dispatch_meta_bytes=336160 combine_payload_bytes=328900608 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=329236768 ingress_bytes=398106624 link_bytes=398106624 token_bytes=658137376 weight_movement_bytes=69206016 total_nvlink_bytes=727343392 weight_to_token_ratio=0.105154 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=41472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.498043176e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=228630528 dispatch_meta_bytes=300144 combine_payload_bytes=228630528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=228930672 ingress_bytes=263233536 link_bytes=263233536 token_bytes=457561200 weight_movement_bytes=34603008 total_nvlink_bytes=492164208 weight_to_token_ratio=0.075625 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=39296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.905266039e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=317095936 dispatch_meta_bytes=330712 combine_payload_bytes=317095936 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=317426648 ingress_bytes=386301952 link_bytes=386301952 token_bytes=634522584 weight_movement_bytes=69206016 total_nvlink_bytes=703728600 weight_to_token_ratio=0.109068 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=45824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.453623146e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=173383680 dispatch_meta_bytes=344072 combine_payload_bytes=173383680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=173727752 ingress_bytes=207986688 link_bytes=207986688 token_bytes=347111432 weight_movement_bytes=34603008 total_nvlink_bytes=381714440 weight_to_token_ratio=0.099688 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=44032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=7.826371626e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=315801600 dispatch_meta_bytes=329736 combine_payload_bytes=315801600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=316131336 ingress_bytes=350404608 link_bytes=350404608 token_bytes=631932936 weight_movement_bytes=34603008 total_nvlink_bytes=666535944 weight_to_token_ratio=0.054757 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=41600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.318544330e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13151731176 prefetch_bytes=830472192 reduce_bytes=484442112 weight_movement_bytes=1314914304 egress_bytes=6579756520 ingress_bytes=7886888960 link_bytes=7886888960 total_nvlink_bytes=14466645480 weight_to_token_ratio=0.099980 prefetch_share_of_ingress=0.105298 dedup_ratio=0.175322 prefetch_slots=48 unused_prefetch_slots=320 slot_utilization=0.130435 reduce_slots=14 remote_demand=24 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027391 gemm_inner_pad_upper_bound=28448 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.290336009e-04 prefetch_bytes_per_gemm_flop=1.358695652e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=11626757688 prefetch_bytes=726663168 reduce_bytes=830472192 weight_movement_bytes=1557135360 egress_bytes=5817024056 ingress_bytes=7366868992 link_bytes=7366868992 total_nvlink_bytes=13183893048 weight_to_token_ratio=0.133927 prefetch_share_of_ingress=0.098639 dedup_ratio=0.236226 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=24 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027935 gemm_inner_pad_upper_bound=28194 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.205258041e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=377569280 dispatch_meta_bytes=393216 combine_payload_bytes=377569280 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=377962496 ingress_bytes=481378304 link_bytes=481378304 token_bytes=755531776 weight_movement_bytes=103809024 total_nvlink_bytes=859340800 weight_to_token_ratio=0.137399 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=19072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.811387804e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=331907072 dispatch_meta_bytes=367576 combine_payload_bytes=331907072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=332274648 ingress_bytes=366510080 link_bytes=366510080 token_bytes=664181720 weight_movement_bytes=34603008 total_nvlink_bytes=698784728 weight_to_token_ratio=0.052099 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=32384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.379147925e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12874962448 prefetch_bytes=726663168 reduce_bytes=553648128 weight_movement_bytes=1280311296 egress_bytes=6441317904 ingress_bytes=7713955840 link_bytes=7713955840 total_nvlink_bytes=14155273744 weight_to_token_ratio=0.099442 prefetch_share_of_ingress=0.094201 dedup_ratio=0.188405 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=16 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28067 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.262043252e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12606409080 prefetch_bytes=692060160 reduce_bytes=588251136 weight_movement_bytes=1280311296 egress_bytes=6307023224 ingress_bytes=7579697152 link_bytes=7579697152 total_nvlink_bytes=13886720376 weight_to_token_ratio=0.101560 prefetch_share_of_ingress=0.091304 dedup_ratio=0.192807 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=17 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027500 gemm_inner_pad_upper_bound=27813 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.240077833e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14090289064 prefetch_bytes=657457152 reduce_bytes=934281216 weight_movement_bytes=1591738368 egress_bytes=7049125800 ingress_bytes=8632901632 link_bytes=8632901632 total_nvlink_bytes=15682027432 weight_to_token_ratio=0.112967 prefetch_share_of_ingress=0.076157 dedup_ratio=0.139498 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=27 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028804 gemm_inner_pad_upper_bound=27178 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.412387558e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12045631536 prefetch_bytes=761266176 reduce_bytes=830472192 weight_movement_bytes=1591738368 egress_bytes=6026469424 ingress_bytes=7610900480 link_bytes=7610900480 total_nvlink_bytes=13637369904 weight_to_token_ratio=0.132142 prefetch_share_of_ingress=0.100023 dedup_ratio=0.214449 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=24 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=27686 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.245182860e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89642 payload_rows_remote=87456 dup_rows=8662 dup_rows_remote=8462 dedup_ratio=0.088114 router_tokens_per_expert=[0, 216, 0, 0, 0, 0, 439, 0, 88, 7421, 2, 0, 0, 88, 0, 148, 57, 0, 0, 62, 0, 182, 0, 0, 0, 0, 0, 0, 501, 7842, 0, 0, 0, 0, 0, 0, 0, 7373, 0, 8104, 88, 0, 0, 1003, 12, 0, 90, 0, 257, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 581, 7856, 0, 0, 0, 594] payload_rows_per_dst=[12966, 14986, 10674, 14896, 5750, 2186, 12296, 15888] dup_rows_per_dst=[1068, 508, 402, 802, 2994, 200, 514, 2174] combine_rows_per_src=[12966, 14986, 10674, 14896, 5750, 2186, 12296, 15888] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92314 payload_rows_remote=76588 dup_rows=5990 dup_rows_remote=4960 dedup_ratio=0.060933 router_tokens_per_expert=[0, 230, 0, 0, 0, 0, 326, 0, 61, 7430, 7, 0, 0, 61, 0, 108, 85, 0, 0, 54, 0, 198, 0, 0, 0, 0, 0, 0, 597, 7781, 0, 0, 0, 0, 0, 0, 0, 7416, 0, 8131, 63, 0, 0, 1206, 15, 0, 67, 0, 243, 0, 0, 0, 0, 5997, 0, 0, 0, 0, 568, 7882, 0, 0, 0, 626] payload_rows_per_dst=[14932, 14984, 16262, 15726, 0, 16024, 11998, 2388] dup_rows_per_dst=[1012, 350, 674, 1030, 0, 2442, 482, 0] combine_rows_per_src=[14932, 14984, 16262, 15726, 0, 16024, 11998, 2388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=51272 payload_rows_remote=35008 dup_rows=47032 dup_rows_remote=30092 dedup_ratio=0.478434 router_tokens_per_expert=[0, 229, 29, 42, 43, 24, 332, 86, 87, 7404, 10, 0, 0, 114, 5, 75, 51, 15, 0, 67, 38, 192, 28, 0, 0, 14, 0, 0, 598, 7598, 23, 0, 2, 43, 0, 4, 7, 7176, 2, 8052, 87, 0, 0, 964, 93, 3, 118, 6, 297, 0, 1, 0, 0, 5995, 0, 0, 0, 0, 715, 7741, 4, 15, 2, 721] payload_rows_per_dst=[1034, 380, 726, 1258, 16312, 2182, 13116, 16264] dup_rows_per_dst=[536, 202, 56, 12, 24700, 360, 4226, 16940] combine_rows_per_src=[1034, 380, 726, 1258, 16312, 2182, 13116, 16264] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87586 payload_rows_remote=74490 dup_rows=10718 dup_rows_remote=10042 dedup_ratio=0.109029 router_tokens_per_expert=[39, 60, 1394, 114, 19, 7, 52, 24, 602, 134, 34, 0, 5123, 5, 2237, 25, 140, 363, 49, 2, 0, 8, 190, 46, 2442, 108, 104, 4, 19, 39, 8, 4419, 4329, 11, 118, 0, 675, 0, 327, 0, 1475, 2320, 58, 240, 747, 6267, 6088, 38, 741, 661, 4, 175, 890, 0, 68, 4, 3, 758, 4585, 659, 35, 26, 19, 21] payload_rows_per_dst=[13730, 9376, 13096, 12126, 10956, 8156, 9250, 10896] dup_rows_per_dst=[2222, 1714, 676, 2160, 476, 1088, 1066, 1316] combine_rows_per_src=[13730, 9376, 13096, 12126, 10956, 8156, 9250, 10896] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=358219776 dispatch_meta_bytes=383672 combine_payload_bytes=358219776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=358603448 ingress_bytes=392822784 link_bytes=392822784 token_bytes=716823224 weight_movement_bytes=34603008 total_nvlink_bytes=751426232 weight_to_token_ratio=0.048273 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=36992 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.478160511e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=313704448 dispatch_meta_bytes=326192 combine_payload_bytes=313704448 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=314030640 ingress_bytes=417513472 link_bytes=417513472 token_bytes=627735088 weight_movement_bytes=103809024 total_nvlink_bytes=731544112 weight_to_token_ratio=0.165371 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=43776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.571069583e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=143392768 dispatch_meta_bytes=260400 combine_payload_bytes=143392768 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=143653168 ingress_bytes=212598784 link_bytes=212598784 token_bytes=287045936 weight_movement_bytes=69206016 total_nvlink_bytes=356251952 weight_to_token_ratio=0.241097 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=23936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=7.999921086e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13867317520 prefetch_bytes=761266176 reduce_bytes=865075200 weight_movement_bytes=1626341376 egress_bytes=6937598224 ingress_bytes=8556060672 link_bytes=8556060672 total_nvlink_bytes=15493658896 weight_to_token_ratio=0.117279 prefetch_share_of_ingress=0.088974 dedup_ratio=0.143243 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=25 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=28448 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.399815978e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none - [2026-08-26 15:09:15] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 1018.7 | throughput per GPU (TFLOP/s/GPU): 125.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.583418E+00 | loss scale: 1.0 | grad norm: 24.333 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76404 payload_rows_remote=67448 dup_rows=21900 dup_rows_remote=19136 dedup_ratio=0.222778 router_tokens_per_expert=[1161, 380, 894, 1044, 633, 1043, 969, 621, 1275, 773, 995, 395, 235, 1418, 843, 908, 730, 1041, 757, 668, 398, 613, 433, 1256, 435, 664, 929, 395, 920, 159, 761, 579, 1114, 781, 1314, 1006, 370, 210, 622, 842, 572, 517, 653, 572, 1075, 703, 745, 462, 898, 707, 734, 655, 1035, 1258, 711, 726, 1156, 344, 1352, 616, 257, 884, 1114, 822] payload_rows_per_dst=[9906, 8956, 9284, 9570, 9272, 10162, 9070, 10184] dup_rows_per_dst=[3812, 2764, 2508, 2436, 3246, 2952, 1862, 2320] combine_rows_per_src=[9906, 8956, 9284, 9570, 9272, 10162, 9070, 10184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74650 payload_rows_remote=64048 dup_rows=23654 dup_rows_remote=21036 dedup_ratio=0.240621 router_tokens_per_expert=[1193, 615, 971, 1169, 656, 895, 814, 781, 1371, 372, 1031, 382, 322, 1271, 802, 1102, 746, 1040, 813, 563, 702, 744, 414, 1181, 453, 730, 758, 383, 778, 124, 753, 569, 1058, 888, 1211, 997, 371, 207, 512, 758, 574, 570, 574, 682, 1100, 649, 733, 594, 681, 564, 812, 598, 1032, 1330, 785, 764, 1194, 293, 1380, 624, 236, 843, 1091, 949] payload_rows_per_dst=[7162, 9396, 8892, 10246, 9744, 8662, 9946, 10602] dup_rows_per_dst=[2302, 3910, 1914, 3574, 3860, 2290, 3186, 2618] combine_rows_per_src=[7162, 9396, 8892, 10246, 9744, 8662, 9946, 10602] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76640 payload_rows_remote=67380 dup_rows=21664 dup_rows_remote=19370 dedup_ratio=0.220378 router_tokens_per_expert=[1208, 462, 892, 1137, 709, 825, 906, 791, 1371, 336, 1000, 377, 305, 1183, 879, 1044, 779, 1105, 794, 652, 610, 716, 392, 1222, 469, 719, 698, 414, 805, 140, 776, 548, 1016, 774, 1256, 921, 380, 195, 545, 844, 583, 594, 616, 643, 1165, 786, 768, 505, 760, 582, 869, 475, 1097, 1380, 811, 732, 1120, 337, 1371, 590, 223, 837, 1157, 956] payload_rows_per_dst=[10272, 8244, 9992, 9260, 8940, 9652, 9598, 10682] dup_rows_per_dst=[3914, 2004, 2548, 2294, 2922, 2820, 2662, 2500] combine_rows_per_src=[10272, 8244, 9992, 9260, 8940, 9652, 9598, 10682] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77102 payload_rows_remote=67044 dup_rows=21202 dup_rows_remote=18612 dedup_ratio=0.215678 router_tokens_per_expert=[1158, 459, 838, 1184, 612, 803, 762, 928, 1294, 284, 990, 309, 380, 1166, 887, 1100, 731, 997, 783, 551, 765, 689, 413, 1395, 562, 694, 822, 301, 768, 92, 954, 788, 1065, 1010, 1177, 853, 346, 180, 573, 973, 624, 618, 572, 781, 1149, 657, 813, 637, 663, 588, 822, 495, 961, 1214, 842, 733, 1236, 276, 1168, 668, 166, 802, 1040, 991] payload_rows_per_dst=[10136, 8308, 10058, 9918, 9194, 10706, 8584, 10198] dup_rows_per_dst=[3624, 1924, 2590, 2360, 3160, 3424, 1624, 2496] combine_rows_per_src=[10136, 8308, 10058, 9918, 9194, 10706, 8584, 10198] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=276267008 dispatch_meta_bytes=346336 combine_payload_bytes=276267008 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=276613344 ingress_bytes=345473024 link_bytes=345473024 token_bytes=552880352 weight_movement_bytes=69206016 total_nvlink_bytes=622086368 weight_to_token_ratio=0.125174 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.299987176e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=262340608 dispatch_meta_bytes=340336 combine_payload_bytes=262340608 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=262680944 ingress_bytes=366149632 link_bytes=366149632 token_bytes=525021552 weight_movement_bytes=103809024 total_nvlink_bytes=628830576 weight_to_token_ratio=0.197723 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=8832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.377791588e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=275988480 dispatch_meta_bytes=347000 combine_payload_bytes=275988480 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=276335480 ingress_bytes=345194496 link_bytes=345194496 token_bytes=552323960 weight_movement_bytes=69206016 total_nvlink_bytes=621529976 weight_to_token_ratio=0.125300 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.030000 gemm_active_groups=11 gemm_peak_group_rows=9344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.298939098e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=305111040 dispatch_meta_bytes=338128 combine_payload_bytes=305111040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=305449168 ingress_bytes=339714048 link_bytes=339714048 token_bytes=610560208 weight_movement_bytes=34603008 total_nvlink_bytes=645163216 weight_to_token_ratio=0.056674 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=43776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.278316613e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13799475920 prefetch_bytes=726663168 reduce_bytes=795869184 weight_movement_bytes=1522532352 egress_bytes=6903663312 ingress_bytes=8418344960 link_bytes=8418344960 total_nvlink_bytes=15322008272 weight_to_token_ratio=0.110333 prefetch_share_of_ingress=0.086319 dedup_ratio=0.139510 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=23 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027609 gemm_inner_pad_upper_bound=28067 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.377284972e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=11626757688 prefetch_bytes=726663168 reduce_bytes=830472192 weight_movement_bytes=1557135360 egress_bytes=5817024056 ingress_bytes=7366868992 link_bytes=7366868992 total_nvlink_bytes=13183893048 weight_to_token_ratio=0.133927 prefetch_share_of_ingress=0.098639 dedup_ratio=0.236226 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=24 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027935 gemm_inner_pad_upper_bound=28194 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.205258041e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85690 payload_rows_remote=69604 dup_rows=12614 dup_rows_remote=10506 dedup_ratio=0.128316 router_tokens_per_expert=[1, 124, 504, 728, 787, 394, 303, 458, 69, 6390, 180, 0, 0, 360, 62, 72, 46, 136, 0, 221, 674, 246, 483, 0, 1, 139, 6, 20, 678, 5902, 548, 7, 64, 772, 0, 77, 138, 5540, 51, 7291, 68, 5, 25, 2039, 260, 87, 847, 27, 956, 13, 23, 7, 10, 1772, 0, 0, 5, 0, 744, 6021, 18, 31, 33, 2689] payload_rows_per_dst=[13640, 13396, 16086, 13840, 2060, 14224, 5438, 7006] dup_rows_per_dst=[4038, 870, 2108, 762, 144, 4534, 124, 34] combine_rows_per_src=[13640, 13396, 16086, 13840, 2060, 14224, 5438, 7006] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=285097984 dispatch_meta_bytes=320440 combine_payload_bytes=285097984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=285418424 ingress_bytes=319700992 link_bytes=319700992 token_bytes=570516408 weight_movement_bytes=34603008 total_nvlink_bytes=605119416 weight_to_token_ratio=0.060652 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=45056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.203009095e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13151731176 prefetch_bytes=830472192 reduce_bytes=484442112 weight_movement_bytes=1314914304 egress_bytes=6579756520 ingress_bytes=7886888960 link_bytes=7886888960 total_nvlink_bytes=14466645480 weight_to_token_ratio=0.099980 prefetch_share_of_ingress=0.105298 dedup_ratio=0.175322 prefetch_slots=48 unused_prefetch_slots=320 slot_utilization=0.130435 reduce_slots=14 remote_demand=24 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027391 gemm_inner_pad_upper_bound=28448 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.290336009e-04 prefetch_bytes_per_gemm_flop=1.358695652e-05 anomalies=none + [2026-08-26 15:57:03] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 795.6 | throughput per GPU (TFLOP/s/GPU): 160.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.583418E+00 | loss scale: 1.0 | grad norm: 24.333 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75490 payload_rows_remote=65472 dup_rows=22814 dup_rows_remote=19058 dedup_ratio=0.232076 router_tokens_per_expert=[1121, 404, 856, 1138, 614, 1067, 898, 652, 1258, 791, 1191, 319, 230, 1416, 802, 894, 841, 1122, 995, 664, 326, 648, 478, 1080, 444, 672, 821, 454, 885, 120, 779, 582, 1150, 786, 1294, 996, 414, 234, 565, 766, 502, 543, 696, 521, 1117, 683, 750, 493, 749, 622, 709, 529, 1133, 1423, 680, 639, 1122, 344, 1431, 616, 231, 863, 1178, 811] payload_rows_per_dst=[10018, 10522, 9776, 9408, 9130, 10492, 9532, 6612] dup_rows_per_dst=[3756, 3626, 3182, 2348, 3280, 2964, 2730, 928] combine_rows_per_src=[10018, 10522, 9776, 9408, 9130, 10492, 9532, 6612] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75918 payload_rows_remote=67146 dup_rows=22386 dup_rows_remote=19600 dedup_ratio=0.227722 router_tokens_per_expert=[1183, 436, 795, 1142, 712, 897, 924, 763, 1305, 479, 947, 368, 314, 1248, 1004, 1115, 818, 960, 845, 695, 587, 722, 401, 1223, 520, 656, 778, 392, 854, 115, 722, 582, 1049, 665, 1215, 822, 432, 243, 523, 830, 623, 575, 518, 751, 1097, 708, 797, 657, 821, 633, 782, 444, 957, 1352, 867, 807, 1154, 299, 1340, 634, 273, 856, 1069, 857] payload_rows_per_dst=[8914, 9384, 9874, 9420, 8772, 8862, 10178, 10514] dup_rows_per_dst=[3122, 3478, 2628, 2184, 2786, 2590, 3148, 2450] combine_rows_per_src=[8914, 9384, 9874, 9420, 8772, 8862, 10178, 10514] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=274612224 dispatch_meta_bytes=342624 combine_payload_bytes=274612224 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=274954848 ingress_bytes=343818240 link_bytes=343818240 token_bytes=549567072 weight_movement_bytes=69206016 total_nvlink_bytes=618773088 weight_to_token_ratio=0.125928 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=9088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.293760357e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76640 payload_rows_remote=67380 dup_rows=21664 dup_rows_remote=19370 dedup_ratio=0.220378 router_tokens_per_expert=[1208, 462, 892, 1137, 709, 825, 906, 791, 1371, 336, 1000, 377, 305, 1183, 879, 1044, 779, 1105, 794, 652, 610, 716, 392, 1222, 469, 719, 698, 414, 805, 140, 776, 548, 1016, 774, 1256, 921, 380, 195, 545, 844, 583, 594, 616, 643, 1165, 786, 768, 505, 760, 582, 869, 475, 1097, 1380, 811, 732, 1120, 337, 1371, 590, 223, 837, 1157, 956] payload_rows_per_dst=[10272, 8244, 9992, 9260, 8940, 9652, 9598, 10682] dup_rows_per_dst=[3914, 2004, 2548, 2294, 2922, 2820, 2662, 2500] combine_rows_per_src=[10272, 8244, 9992, 9260, 8940, 9652, 9598, 10682] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74650 payload_rows_remote=64048 dup_rows=23654 dup_rows_remote=21036 dedup_ratio=0.240621 router_tokens_per_expert=[1193, 615, 971, 1169, 656, 895, 814, 781, 1371, 372, 1031, 382, 322, 1271, 802, 1102, 746, 1040, 813, 563, 702, 744, 414, 1181, 453, 730, 758, 383, 778, 124, 753, 569, 1058, 888, 1211, 997, 371, 207, 512, 758, 574, 570, 574, 682, 1100, 649, 733, 594, 681, 564, 812, 598, 1032, 1330, 785, 764, 1194, 293, 1380, 624, 236, 843, 1091, 949] payload_rows_per_dst=[7162, 9396, 8892, 10246, 9744, 8662, 9946, 10602] dup_rows_per_dst=[2302, 3910, 1914, 3574, 3860, 2290, 3186, 2618] combine_rows_per_src=[7162, 9396, 8892, 10246, 9744, 8662, 9946, 10602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76404 payload_rows_remote=67448 dup_rows=21900 dup_rows_remote=19136 dedup_ratio=0.222778 router_tokens_per_expert=[1161, 380, 894, 1044, 633, 1043, 969, 621, 1275, 773, 995, 395, 235, 1418, 843, 908, 730, 1041, 757, 668, 398, 613, 433, 1256, 435, 664, 929, 395, 920, 159, 761, 579, 1114, 781, 1314, 1006, 370, 210, 622, 842, 572, 517, 653, 572, 1075, 703, 745, 462, 898, 707, 734, 655, 1035, 1258, 711, 726, 1156, 344, 1352, 616, 257, 884, 1114, 822] payload_rows_per_dst=[9906, 8956, 9284, 9570, 9272, 10162, 9070, 10184] dup_rows_per_dst=[3812, 2764, 2508, 2436, 3246, 2952, 1862, 2320] combine_rows_per_src=[9906, 8956, 9284, 9570, 9272, 10162, 9070, 10184] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75448 payload_rows_remote=65358 dup_rows=22856 dup_rows_remote=19786 dedup_ratio=0.232503 router_tokens_per_expert=[1124, 465, 864, 1202, 687, 847, 830, 794, 1406, 290, 978, 340, 327, 1048, 824, 1064, 804, 971, 741, 613, 664, 739, 372, 1171, 627, 771, 735, 363, 769, 101, 873, 693, 1041, 805, 1196, 881, 460, 245, 502, 804, 675, 694, 582, 781, 1112, 668, 743, 526, 769, 658, 808, 438, 1071, 1275, 821, 740, 1186, 323, 1296, 786, 186, 876, 1128, 979] payload_rows_per_dst=[7840, 9146, 9788, 9956, 9026, 8870, 10090, 10732] dup_rows_per_dst=[2638, 3408, 2362, 3056, 2842, 2692, 3070, 2788] combine_rows_per_src=[7840, 9146, 9788, 9956, 9026, 8870, 10090, 10732] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75490 payload_rows_remote=65472 dup_rows=22814 dup_rows_remote=19058 dedup_ratio=0.232076 router_tokens_per_expert=[1121, 404, 856, 1138, 614, 1067, 898, 652, 1258, 791, 1191, 319, 230, 1416, 802, 894, 841, 1122, 995, 664, 326, 648, 478, 1080, 444, 672, 821, 454, 885, 120, 779, 582, 1150, 786, 1294, 996, 414, 234, 565, 766, 502, 543, 696, 521, 1117, 683, 750, 493, 749, 622, 709, 529, 1133, 1423, 680, 639, 1122, 344, 1431, 616, 231, 863, 1178, 811] payload_rows_per_dst=[10018, 10522, 9776, 9408, 9130, 10492, 9532, 6612] dup_rows_per_dst=[3756, 3626, 3182, 2348, 3280, 2964, 2730, 928] combine_rows_per_src=[10018, 10522, 9776, 9408, 9130, 10492, 9532, 6612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75802 payload_rows_remote=67122 dup_rows=22502 dup_rows_remote=20016 dedup_ratio=0.228902 router_tokens_per_expert=[1138, 374, 845, 1103, 629, 748, 807, 938, 1386, 362, 952, 368, 320, 1394, 872, 963, 850, 1006, 782, 525, 765, 737, 459, 1330, 518, 682, 828, 368, 808, 114, 887, 794, 1067, 855, 1260, 905, 352, 265, 462, 899, 595, 637, 503, 759, 1072, 554, 772, 691, 743, 629, 906, 531, 995, 1182, 833, 693, 1158, 293, 1120, 640, 210, 778, 1096, 1045] payload_rows_per_dst=[8212, 9494, 10346, 9840, 9154, 8680, 9948, 10128] dup_rows_per_dst=[2676, 3740, 2562, 2434, 2976, 2486, 3076, 2552] combine_rows_per_src=[8212, 9494, 10346, 9840, 9154, 8680, 9948, 10128] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=268173312 dispatch_meta_bytes=338120 combine_payload_bytes=268173312 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=268511432 ingress_bytes=371982336 link_bytes=371982336 token_bytes=536684744 weight_movement_bytes=103809024 total_nvlink_bytes=640493768 weight_to_token_ratio=0.193426 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=7552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.399739583e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=275030016 dispatch_meta_bytes=346984 combine_payload_bytes=275030016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275377000 ingress_bytes=309633024 link_bytes=309633024 token_bytes=550407016 weight_movement_bytes=34603008 total_nvlink_bytes=585010024 weight_to_token_ratio=0.062868 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=9984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.165124142e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76450 payload_rows_remote=66920 dup_rows=21854 dup_rows_remote=18698 dedup_ratio=0.222310 router_tokens_per_expert=[741, 1074, 926, 383, 1038, 289, 375, 994, 63, 785, 1609, 382, 527, 844, 746, 1387, 702, 1286, 784, 431, 1316, 369, 1865, 1555, 919, 677, 922, 1044, 124, 447, 226, 118, 507, 360, 345, 153, 899, 1084, 1157, 1145, 2050, 1584, 745, 1122, 496, 1482, 39, 329, 1186, 844, 1230, 17, 193, 1454, 176, 1201, 808, 346, 725, 481, 1024, 355, 344, 323] payload_rows_per_dst=[10856, 9530, 10566, 9274, 9396, 9160, 10144, 7524] dup_rows_per_dst=[3952, 3156, 3456, 3410, 1904, 2230, 2458, 1288] combine_rows_per_src=[10856, 9530, 10566, 9274, 9396, 9160, 10144, 7524] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73658 payload_rows_remote=63854 dup_rows=24646 dup_rows_remote=20074 dedup_ratio=0.250712 router_tokens_per_expert=[651, 940, 778, 395, 982, 284, 293, 1080, 67, 967, 1521, 361, 634, 802, 765, 1465, 821, 1262, 775, 346, 1524, 296, 1784, 1274, 773, 672, 843, 950, 100, 300, 208, 95, 374, 282, 441, 187, 1115, 1180, 1062, 1068, 2401, 1604, 913, 1209, 394, 1336, 37, 388, 1104, 992, 942, 8, 158, 1769, 222, 1171, 781, 349, 868, 541, 1133, 349, 325, 441] payload_rows_per_dst=[8508, 6514, 8198, 10362, 10902, 9174, 10196, 9804] dup_rows_per_dst=[2298, 1692, 1350, 4136, 3558, 4504, 2536, 4572] combine_rows_per_src=[8508, 6514, 8198, 10362, 10902, 9174, 10196, 9804] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=275988480 dispatch_meta_bytes=347000 combine_payload_bytes=275988480 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=276335480 ingress_bytes=345194496 link_bytes=345194496 token_bytes=552323960 weight_movement_bytes=69206016 total_nvlink_bytes=621529976 weight_to_token_ratio=0.125300 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.030000 gemm_active_groups=11 gemm_peak_group_rows=9344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.298939098e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=262340608 dispatch_meta_bytes=340336 combine_payload_bytes=262340608 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=262680944 ingress_bytes=366149632 link_bytes=366149632 token_bytes=525021552 weight_movement_bytes=103809024 total_nvlink_bytes=628830576 weight_to_token_ratio=0.197723 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=8832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.377791588e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=276267008 dispatch_meta_bytes=346336 combine_payload_bytes=276267008 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=276613344 ingress_bytes=345473024 link_bytes=345473024 token_bytes=552880352 weight_movement_bytes=69206016 total_nvlink_bytes=622086368 weight_to_token_ratio=0.125174 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.299987176e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267706368 dispatch_meta_bytes=340576 combine_payload_bytes=267706368 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=268046944 ingress_bytes=336912384 link_bytes=336912384 token_bytes=535753312 weight_movement_bytes=69206016 total_nvlink_bytes=604959328 weight_to_token_ratio=0.129175 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=8320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.267774177e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=274931712 dispatch_meta_bytes=348552 combine_payload_bytes=274931712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275280264 ingress_bytes=309534720 link_bytes=309534720 token_bytes=550211976 weight_movement_bytes=34603008 total_nvlink_bytes=584814984 weight_to_token_ratio=0.062890 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=8960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.164754232e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77102 payload_rows_remote=67044 dup_rows=21202 dup_rows_remote=18612 dedup_ratio=0.215678 router_tokens_per_expert=[1158, 459, 838, 1184, 612, 803, 762, 928, 1294, 284, 990, 309, 380, 1166, 887, 1100, 731, 997, 783, 551, 765, 689, 413, 1395, 562, 694, 822, 301, 768, 92, 954, 788, 1065, 1010, 1177, 853, 346, 180, 573, 973, 624, 618, 572, 781, 1149, 657, 813, 637, 663, 588, 822, 495, 961, 1214, 842, 733, 1236, 276, 1168, 668, 166, 802, 1040, 991] payload_rows_per_dst=[10136, 8308, 10058, 9918, 9194, 10706, 8584, 10198] dup_rows_per_dst=[3624, 1924, 2590, 2360, 3160, 3424, 1624, 2496] combine_rows_per_src=[10136, 8308, 10058, 9918, 9194, 10706, 8584, 10198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74678 payload_rows_remote=65890 dup_rows=23626 dup_rows_remote=20616 dedup_ratio=0.240336 router_tokens_per_expert=[746, 932, 899, 580, 772, 292, 444, 1172, 90, 897, 1268, 393, 594, 816, 913, 1308, 991, 1234, 792, 374, 1430, 268, 1752, 1242, 723, 707, 881, 917, 202, 382, 209, 126, 452, 296, 475, 227, 939, 1367, 1021, 1174, 1988, 1489, 1032, 1351, 401, 1136, 32, 258, 958, 1139, 988, 32, 181, 1602, 195, 1062, 818, 458, 915, 501, 1145, 371, 344, 459] payload_rows_per_dst=[9146, 9402, 9870, 8788, 9722, 8904, 9880, 8966] dup_rows_per_dst=[2528, 3156, 2792, 3010, 2180, 3872, 2434, 3654] combine_rows_per_src=[9146, 9402, 9870, 8788, 9722, 8904, 9880, 8966] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=268173312 dispatch_meta_bytes=338120 combine_payload_bytes=268173312 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=268511432 ingress_bytes=371982336 link_bytes=371982336 token_bytes=536684744 weight_movement_bytes=103809024 total_nvlink_bytes=640493768 weight_to_token_ratio=0.193426 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=7552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.399739583e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76706 payload_rows_remote=66990 dup_rows=21598 dup_rows_remote=19026 dedup_ratio=0.219706 router_tokens_per_expert=[673, 893, 960, 503, 763, 298, 473, 1037, 80, 964, 1198, 286, 648, 955, 949, 1336, 984, 1249, 824, 287, 1529, 172, 1824, 1099, 659, 677, 701, 749, 127, 293, 212, 78, 391, 274, 543, 220, 1037, 1909, 963, 1081, 2248, 1447, 1342, 1416, 440, 1070, 28, 245, 825, 1125, 770, 24, 164, 1729, 245, 1152, 755, 340, 880, 675, 1267, 298, 317, 452] payload_rows_per_dst=[9744, 9936, 9716, 8148, 10300, 10888, 9940, 8034] dup_rows_per_dst=[3056, 2896, 2572, 2492, 2536, 3984, 2128, 1934] combine_rows_per_src=[9744, 9936, 9716, 8148, 10300, 10888, 9940, 8034] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=261545984 dispatch_meta_bytes=335712 combine_payload_bytes=261545984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=261881696 ingress_bytes=296148992 link_bytes=296148992 token_bytes=523427680 weight_movement_bytes=34603008 total_nvlink_bytes=558030688 weight_to_token_ratio=0.066108 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=10112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.114384815e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=274104320 dispatch_meta_bytes=342472 combine_payload_bytes=274104320 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=274446792 ingress_bytes=377913344 link_bytes=377913344 token_bytes=548551112 weight_movement_bytes=103809024 total_nvlink_bytes=652360136 weight_to_token_ratio=0.189242 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.422057489e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=269885440 dispatch_meta_bytes=346024 combine_payload_bytes=269885440 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=270231464 ingress_bytes=339091456 link_bytes=339091456 token_bytes=540116904 weight_movement_bytes=69206016 total_nvlink_bytes=609322920 weight_to_token_ratio=0.128132 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=14336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.275973850e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=274391040 dispatch_meta_bytes=344064 combine_payload_bytes=274391040 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=274735104 ingress_bytes=378200064 link_bytes=378200064 token_bytes=549126144 weight_movement_bytes=103809024 total_nvlink_bytes=652935168 weight_to_token_ratio=0.189044 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=10496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.423136393e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76114 payload_rows_remote=65492 dup_rows=22190 dup_rows_remote=18576 dedup_ratio=0.225728 router_tokens_per_expert=[729, 917, 720, 573, 1116, 269, 362, 1200, 64, 868, 1748, 424, 503, 851, 656, 1503, 728, 1299, 806, 332, 1460, 377, 1775, 1647, 780, 788, 848, 1007, 203, 330, 158, 91, 357, 246, 388, 195, 997, 934, 1012, 982, 1942, 1232, 672, 1130, 357, 1565, 22, 325, 1396, 953, 1293, 3, 127, 1410, 223, 1080, 969, 378, 1072, 551, 1112, 369, 322, 406] payload_rows_per_dst=[10622, 10196, 11390, 8852, 8752, 8060, 9842, 8400] dup_rows_per_dst=[3614, 3752, 4168, 3108, 1470, 1706, 2414, 1958] combine_rows_per_src=[10622, 10196, 11390, 8852, 8752, 8060, 9842, 8400] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74802 payload_rows_remote=65362 dup_rows=23502 dup_rows_remote=21526 dedup_ratio=0.239075 router_tokens_per_expert=[709, 957, 918, 458, 847, 287, 398, 1162, 70, 936, 1373, 377, 529, 868, 984, 1299, 989, 1166, 776, 481, 1488, 258, 1680, 1382, 701, 764, 839, 905, 253, 451, 168, 56, 430, 336, 450, 229, 927, 1302, 1013, 1021, 2057, 1542, 950, 1241, 449, 1281, 26, 274, 1045, 1087, 1001, 10, 242, 1489, 204, 1112, 763, 387, 1011, 551, 1026, 369, 348, 450] payload_rows_per_dst=[9100, 9710, 10250, 8712, 9440, 8076, 10002, 9512] dup_rows_per_dst=[2372, 3162, 2830, 2922, 1976, 3450, 2378, 4412] combine_rows_per_src=[9100, 9710, 10250, 8712, 9440, 8076, 10002, 9512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76450 payload_rows_remote=66920 dup_rows=21854 dup_rows_remote=18698 dedup_ratio=0.222310 router_tokens_per_expert=[741, 1074, 926, 383, 1038, 289, 375, 994, 63, 785, 1609, 382, 527, 844, 746, 1387, 702, 1286, 784, 431, 1316, 369, 1865, 1555, 919, 677, 922, 1044, 124, 447, 226, 118, 507, 360, 345, 153, 899, 1084, 1157, 1145, 2050, 1584, 745, 1122, 496, 1482, 39, 329, 1186, 844, 1230, 17, 193, 1454, 176, 1201, 808, 346, 725, 481, 1024, 355, 344, 323] payload_rows_per_dst=[10856, 9530, 10566, 9274, 9396, 9160, 10144, 7524] dup_rows_per_dst=[3952, 3156, 3456, 3410, 1904, 2230, 2458, 1288] combine_rows_per_src=[10856, 9530, 10566, 9274, 9396, 9160, 10144, 7524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73658 payload_rows_remote=63854 dup_rows=24646 dup_rows_remote=20074 dedup_ratio=0.250712 router_tokens_per_expert=[651, 940, 778, 395, 982, 284, 293, 1080, 67, 967, 1521, 361, 634, 802, 765, 1465, 821, 1262, 775, 346, 1524, 296, 1784, 1274, 773, 672, 843, 950, 100, 300, 208, 95, 374, 282, 441, 187, 1115, 1180, 1062, 1068, 2401, 1604, 913, 1209, 394, 1336, 37, 388, 1104, 992, 942, 8, 158, 1769, 222, 1171, 781, 349, 868, 541, 1133, 349, 325, 441] payload_rows_per_dst=[8508, 6514, 8198, 10362, 10902, 9174, 10196, 9804] dup_rows_per_dst=[2298, 1692, 1350, 4136, 3558, 4504, 2536, 4572] combine_rows_per_src=[8508, 6514, 8198, 10362, 10902, 9174, 10196, 9804] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=274612224 dispatch_meta_bytes=342624 combine_payload_bytes=274612224 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=274954848 ingress_bytes=343818240 link_bytes=343818240 token_bytes=549567072 weight_movement_bytes=69206016 total_nvlink_bytes=618773088 weight_to_token_ratio=0.125928 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=9088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.293760357e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74390 payload_rows_remote=64666 dup_rows=23914 dup_rows_remote=21786 dedup_ratio=0.243266 router_tokens_per_expert=[587, 889, 891, 540, 837, 310, 438, 1072, 75, 1188, 1451, 360, 645, 1005, 938, 1329, 922, 1244, 833, 315, 1494, 194, 1724, 1086, 609, 650, 835, 851, 134, 340, 206, 93, 391, 226, 549, 306, 891, 1529, 1034, 1060, 2281, 1529, 1094, 1559, 433, 1006, 31, 255, 851, 1092, 884, 18, 161, 1539, 226, 1155, 800, 420, 893, 521, 1256, 307, 307, 463] payload_rows_per_dst=[8794, 9300, 7970, 10056, 10550, 8094, 9724, 9902] dup_rows_per_dst=[2334, 2956, 1218, 3816, 3148, 3720, 2128, 4594] combine_rows_per_src=[8794, 9300, 7970, 10056, 10550, 8094, 9724, 9902] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76114 payload_rows_remote=65492 dup_rows=22190 dup_rows_remote=18576 dedup_ratio=0.225728 router_tokens_per_expert=[729, 917, 720, 573, 1116, 269, 362, 1200, 64, 868, 1748, 424, 503, 851, 656, 1503, 728, 1299, 806, 332, 1460, 377, 1775, 1647, 780, 788, 848, 1007, 203, 330, 158, 91, 357, 246, 388, 195, 997, 934, 1012, 982, 1942, 1232, 672, 1130, 357, 1565, 22, 325, 1396, 953, 1293, 3, 127, 1410, 223, 1080, 969, 378, 1072, 551, 1112, 369, 322, 406] payload_rows_per_dst=[10622, 10196, 11390, 8852, 8752, 8060, 9842, 8400] dup_rows_per_dst=[3614, 3752, 4168, 3108, 1470, 1706, 2414, 1958] combine_rows_per_src=[10622, 10196, 11390, 8852, 8752, 8060, 9842, 8400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74916 payload_rows_remote=66296 dup_rows=23388 dup_rows_remote=19526 dedup_ratio=0.237915 router_tokens_per_expert=[654, 983, 919, 601, 774, 353, 488, 918, 86, 898, 1304, 314, 670, 1000, 864, 1337, 912, 1186, 833, 214, 1512, 238, 1858, 1151, 678, 716, 728, 834, 164, 287, 198, 79, 462, 296, 520, 224, 931, 1545, 1117, 1140, 2000, 1578, 1196, 1377, 510, 1252, 30, 298, 776, 1208, 780, 15, 230, 1616, 208, 1217, 764, 384, 855, 528, 1229, 321, 338, 456] payload_rows_per_dst=[9094, 9778, 9516, 8476, 10254, 8620, 9946, 9232] dup_rows_per_dst=[2286, 3168, 2442, 2742, 2216, 3862, 2154, 4518] combine_rows_per_src=[9094, 9778, 9516, 8476, 10254, 8620, 9946, 9232] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75428 payload_rows_remote=64092 dup_rows=22876 dup_rows_remote=18210 dedup_ratio=0.232707 router_tokens_per_expert=[49, 1041, 425, 422, 34, 941, 785, 238, 610, 378, 1184, 1381, 1361, 1101, 209, 1510, 1001, 1575, 476, 735, 138, 122, 1236, 1137, 635, 990, 1034, 692, 424, 611, 984, 231, 373, 1057, 817, 487, 1978, 1756, 1389, 439, 921, 137, 336, 332, 1597, 1862, 166, 1370, 1061, 448, 760, 978, 49, 204, 253, 553, 1093, 417, 122, 546, 69, 1514, 244, 2134] payload_rows_per_dst=[8952, 7974, 10066, 10128, 9216, 8474, 9282, 11336] dup_rows_per_dst=[2874, 1752, 2774, 3796, 3056, 1608, 2350, 4666] combine_rows_per_src=[8952, 7974, 10066, 10128, 9216, 8474, 9282, 11336] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75216 payload_rows_remote=65140 dup_rows=23088 dup_rows_remote=19624 dedup_ratio=0.234863 router_tokens_per_expert=[34, 1143, 325, 492, 13, 927, 887, 186, 518, 331, 1414, 1579, 1216, 1223, 247, 1225, 1037, 1523, 562, 773, 120, 103, 1278, 1172, 537, 1078, 1246, 781, 485, 623, 1156, 121, 308, 963, 490, 342, 2315, 1537, 1329, 418, 1422, 126, 358, 559, 1499, 1807, 198, 1357, 1209, 387, 373, 850, 13, 58, 349, 472, 876, 275, 102, 655, 160, 1661, 176, 2153] payload_rows_per_dst=[10012, 10076, 10280, 9652, 6942, 9874, 8230, 10150] dup_rows_per_dst=[4290, 3464, 2856, 2402, 2174, 4294, 1642, 1966] combine_rows_per_src=[10012, 10076, 10280, 9652, 6942, 9874, 8230, 10150] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267722752 dispatch_meta_bytes=347552 combine_payload_bytes=267722752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268070304 ingress_bytes=302325760 link_bytes=302325760 token_bytes=535793056 weight_movement_bytes=34603008 total_nvlink_bytes=570396064 weight_to_token_ratio=0.064583 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137627496e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=264871936 dispatch_meta_bytes=345808 combine_payload_bytes=264871936 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=265217744 ingress_bytes=299474944 link_bytes=299474944 token_bytes=530089680 weight_movement_bytes=34603008 total_nvlink_bytes=564692688 weight_to_token_ratio=0.065278 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=12288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126900105e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75324 payload_rows_remote=65998 dup_rows=22980 dup_rows_remote=20672 dedup_ratio=0.233765 router_tokens_per_expert=[194, 952, 389, 475, 88, 777, 879, 283, 561, 339, 1182, 1466, 1450, 1596, 565, 1541, 1025, 1289, 429, 803, 100, 332, 1162, 922, 469, 966, 867, 634, 418, 747, 886, 275, 431, 1099, 635, 539, 1747, 1615, 1568, 458, 917, 168, 367, 255, 1626, 1652, 168, 1346, 1300, 400, 816, 767, 102, 399, 366, 662, 1008, 321, 130, 453, 141, 1142, 571, 1922] payload_rows_per_dst=[8590, 9912, 9594, 9326, 9472, 9014, 9770, 9646] dup_rows_per_dst=[2978, 3296, 2530, 2308, 3218, 3984, 2936, 1730] combine_rows_per_src=[8590, 9912, 9594, 9326, 9472, 9014, 9770, 9646] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76660 payload_rows_remote=66926 dup_rows=21644 dup_rows_remote=19454 dedup_ratio=0.220174 router_tokens_per_expert=[296, 690, 362, 419, 108, 889, 764, 275, 582, 302, 1187, 1131, 1173, 1790, 708, 2071, 988, 1416, 246, 783, 101, 381, 1141, 906, 413, 926, 750, 457, 290, 629, 972, 384, 363, 932, 783, 733, 1325, 1806, 1734, 467, 588, 147, 354, 249, 1894, 1602, 147, 1441, 1143, 706, 1121, 811, 166, 388, 258, 819, 952, 374, 77, 583, 71, 1227, 585, 1776] payload_rows_per_dst=[8086, 10548, 9734, 8288, 10030, 9066, 11000, 9908] dup_rows_per_dst=[2170, 3198, 2190, 1354, 3606, 3778, 3966, 1382] combine_rows_per_src=[8086, 10548, 9734, 8288, 10030, 9066, 11000, 9908] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=262520832 dispatch_meta_bytes=329208 combine_payload_bytes=262520832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262850040 ingress_bytes=297123840 link_bytes=297123840 token_bytes=525370872 weight_movement_bytes=34603008 total_nvlink_bytes=559973880 weight_to_token_ratio=0.065864 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=15872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.118053089e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266813440 dispatch_meta_bytes=339056 combine_payload_bytes=266813440 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267152496 ingress_bytes=370622464 link_bytes=370622464 token_bytes=533965936 weight_movement_bytes=103809024 total_nvlink_bytes=637774960 weight_to_token_ratio=0.194411 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394622494e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=269885440 dispatch_meta_bytes=346024 combine_payload_bytes=269885440 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=270231464 ingress_bytes=339091456 link_bytes=339091456 token_bytes=540116904 weight_movement_bytes=69206016 total_nvlink_bytes=609322920 weight_to_token_ratio=0.128132 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=14336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.275973850e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=268255232 dispatch_meta_bytes=336272 combine_payload_bytes=268255232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268591504 ingress_bytes=302858240 link_bytes=302858240 token_bytes=536846736 weight_movement_bytes=34603008 total_nvlink_bytes=571449744 weight_to_token_ratio=0.064456 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=8704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.139631175e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=270327808 dispatch_meta_bytes=346680 combine_payload_bytes=270327808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270674488 ingress_bytes=304930816 link_bytes=304930816 token_bytes=541002296 weight_movement_bytes=34603008 total_nvlink_bytes=575605304 weight_to_token_ratio=0.063961 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=7936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.147430112e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=274128896 dispatch_meta_bytes=345520 combine_payload_bytes=274128896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274474416 ingress_bytes=308731904 link_bytes=308731904 token_bytes=548603312 weight_movement_bytes=34603008 total_nvlink_bytes=583206320 weight_to_token_ratio=0.063075 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.161733300e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267722752 dispatch_meta_bytes=347552 combine_payload_bytes=267722752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268070304 ingress_bytes=302325760 link_bytes=302325760 token_bytes=535793056 weight_movement_bytes=34603008 total_nvlink_bytes=570396064 weight_to_token_ratio=0.064583 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137627496e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=274104320 dispatch_meta_bytes=342472 combine_payload_bytes=274104320 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=274446792 ingress_bytes=377913344 link_bytes=377913344 token_bytes=548551112 weight_movement_bytes=103809024 total_nvlink_bytes=652360136 weight_to_token_ratio=0.189242 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.422057489e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=261545984 dispatch_meta_bytes=335712 combine_payload_bytes=261545984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=261881696 ingress_bytes=296148992 link_bytes=296148992 token_bytes=523427680 weight_movement_bytes=34603008 total_nvlink_bytes=558030688 weight_to_token_ratio=0.066108 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=10112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.114384815e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=264871936 dispatch_meta_bytes=345808 combine_payload_bytes=264871936 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=265217744 ingress_bytes=299474944 link_bytes=299474944 token_bytes=530089680 weight_movement_bytes=34603008 total_nvlink_bytes=564692688 weight_to_token_ratio=0.065278 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=12288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126900105e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=271548416 dispatch_meta_bytes=343288 combine_payload_bytes=271548416 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=271891704 ingress_bytes=409960448 link_bytes=409960448 token_bytes=543440120 weight_movement_bytes=138412032 total_nvlink_bytes=681852152 weight_to_token_ratio=0.254696 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=10240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.542648161e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76706 payload_rows_remote=66990 dup_rows=21598 dup_rows_remote=19026 dedup_ratio=0.219706 router_tokens_per_expert=[673, 893, 960, 503, 763, 298, 473, 1037, 80, 964, 1198, 286, 648, 955, 949, 1336, 984, 1249, 824, 287, 1529, 172, 1824, 1099, 659, 677, 701, 749, 127, 293, 212, 78, 391, 274, 543, 220, 1037, 1909, 963, 1081, 2248, 1447, 1342, 1416, 440, 1070, 28, 245, 825, 1125, 770, 24, 164, 1729, 245, 1152, 755, 340, 880, 675, 1267, 298, 317, 452] payload_rows_per_dst=[9744, 9936, 9716, 8148, 10300, 10888, 9940, 8034] dup_rows_per_dst=[3056, 2896, 2572, 2492, 2536, 3984, 2128, 1934] combine_rows_per_src=[9744, 9936, 9716, 8148, 10300, 10888, 9940, 8034] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75324 payload_rows_remote=65998 dup_rows=22980 dup_rows_remote=20672 dedup_ratio=0.233765 router_tokens_per_expert=[194, 952, 389, 475, 88, 777, 879, 283, 561, 339, 1182, 1466, 1450, 1596, 565, 1541, 1025, 1289, 429, 803, 100, 332, 1162, 922, 469, 966, 867, 634, 418, 747, 886, 275, 431, 1099, 635, 539, 1747, 1615, 1568, 458, 917, 168, 367, 255, 1626, 1652, 168, 1346, 1300, 400, 816, 767, 102, 399, 366, 662, 1008, 321, 130, 453, 141, 1142, 571, 1922] payload_rows_per_dst=[8590, 9912, 9594, 9326, 9472, 9014, 9770, 9646] dup_rows_per_dst=[2978, 3296, 2530, 2308, 3218, 3984, 2936, 1730] combine_rows_per_src=[8590, 9912, 9594, 9326, 9472, 9014, 9770, 9646] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75142 payload_rows_remote=65160 dup_rows=23162 dup_rows_remote=19310 dedup_ratio=0.235616 router_tokens_per_expert=[79, 869, 432, 394, 29, 982, 840, 294, 479, 401, 1237, 1239, 1440, 1032, 346, 1431, 1066, 1423, 338, 767, 82, 146, 1134, 1117, 594, 897, 1127, 848, 404, 602, 897, 265, 417, 1106, 950, 586, 1754, 1847, 1592, 419, 939, 147, 343, 248, 1548, 1844, 170, 1388, 1098, 453, 1020, 926, 50, 324, 305, 590, 1107, 378, 104, 441, 169, 1389, 286, 1983] payload_rows_per_dst=[8588, 7930, 9874, 10382, 9982, 9098, 9652, 9636] dup_rows_per_dst=[2758, 1538, 2272, 3766, 3852, 4156, 2742, 2078] combine_rows_per_src=[8588, 7930, 9874, 10382, 9982, 9098, 9652, 9636] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75752 payload_rows_remote=65158 dup_rows=22552 dup_rows_remote=19000 dedup_ratio=0.229411 router_tokens_per_expert=[236, 824, 396, 377, 95, 887, 754, 267, 557, 379, 1008, 1281, 1388, 1363, 478, 1766, 962, 1280, 249, 737, 112, 312, 1155, 971, 506, 942, 846, 583, 217, 663, 980, 307, 419, 1075, 806, 678, 1629, 1743, 1786, 515, 699, 177, 381, 237, 1869, 1516, 164, 1450, 1041, 603, 1186, 909, 101, 337, 331, 799, 1015, 441, 93, 557, 79, 1155, 497, 1986] payload_rows_per_dst=[8558, 8236, 9536, 9806, 10078, 9028, 10594, 9916] dup_rows_per_dst=[2372, 1896, 2020, 3058, 3966, 3408, 3552, 2280] combine_rows_per_src=[8558, 8236, 9536, 9806, 10078, 9028, 10594, 9916] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266895360 dispatch_meta_bytes=337880 combine_payload_bytes=266895360 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267233240 ingress_bytes=405307392 link_bytes=405307392 token_bytes=534128600 weight_movement_bytes=138412032 total_nvlink_bytes=672540632 weight_to_token_ratio=0.259136 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=12160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.525139086e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75642 payload_rows_remote=65946 dup_rows=22662 dup_rows_remote=20752 dedup_ratio=0.230530 router_tokens_per_expert=[168, 346, 1444, 391, 668, 800, 1218, 1008, 1832, 3101, 247, 383, 794, 674, 1272, 435, 1951, 74, 1904, 424, 920, 500, 1, 452, 1045, 1307, 571, 399, 519, 1165, 1873, 920, 585, 451, 167, 212, 1574, 590, 1848, 718, 64, 1802, 194, 433, 278, 356, 347, 2484, 4, 251, 201, 1053, 777, 860, 170, 186, 461, 286, 1629, 527, 790, 700, 117, 231] payload_rows_per_dst=[10562, 9696, 9040, 10768, 9672, 5692, 9900, 10312] dup_rows_per_dst=[3618, 1910, 3412, 3958, 2618, 1256, 2974, 2916] combine_rows_per_src=[10562, 9696, 9040, 10768, 9672, 5692, 9900, 10312] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76148 payload_rows_remote=65708 dup_rows=22156 dup_rows_remote=19162 dedup_ratio=0.225382 router_tokens_per_expert=[225, 420, 1245, 301, 513, 677, 1248, 905, 1962, 3284, 376, 404, 1423, 489, 1266, 432, 1794, 66, 1953, 492, 919, 589, 3, 481, 1191, 1286, 592, 554, 413, 1088, 1468, 707, 525, 596, 165, 165, 1267, 602, 1789, 732, 174, 2120, 149, 461, 263, 429, 356, 2710, 8, 263, 223, 1115, 681, 849, 148, 150, 269, 225, 1659, 555, 745, 642, 113, 238] payload_rows_per_dst=[8838, 10144, 9208, 8188, 9286, 9710, 10334, 10440] dup_rows_per_dst=[2230, 2560, 3384, 1868, 2396, 3616, 3108, 2994] combine_rows_per_src=[8838, 10144, 9208, 8188, 9286, 9710, 10334, 10440] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=63278 dup_rows=24062 dup_rows_remote=17850 dedup_ratio=0.244771 router_tokens_per_expert=[20, 1250, 371, 507, 15, 974, 823, 290, 525, 394, 1266, 1514, 1734, 871, 85, 925, 1071, 1684, 503, 700, 130, 97, 1134, 945, 828, 1122, 1120, 792, 345, 542, 962, 126, 419, 1096, 724, 451, 2328, 2010, 1305, 422, 1128, 104, 368, 314, 1573, 1908, 159, 1217, 1127, 125, 512, 1075, 42, 140, 287, 482, 1087, 506, 130, 391, 75, 1666, 118, 2198] payload_rows_per_dst=[10964, 12050, 10156, 9426, 6708, 8188, 6598, 10152] dup_rows_per_dst=[6212, 5204, 2710, 2248, 1788, 2208, 1502, 2190] combine_rows_per_src=[10964, 12050, 10156, 9426, 6708, 8188, 6598, 10152] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75864 payload_rows_remote=65896 dup_rows=22440 dup_rows_remote=20008 dedup_ratio=0.228271 router_tokens_per_expert=[316, 674, 1009, 503, 547, 431, 991, 722, 1623, 2343, 164, 415, 958, 725, 1088, 425, 1678, 97, 1823, 518, 643, 361, 21, 526, 1195, 930, 675, 460, 431, 1233, 1644, 1276, 452, 675, 321, 295, 1306, 637, 1886, 667, 387, 1862, 253, 518, 322, 628, 532, 2496, 209, 352, 395, 913, 973, 1067, 400, 500, 364, 295, 1381, 608, 706, 710, 252, 345] payload_rows_per_dst=[8530, 9226, 8430, 9968, 9640, 9990, 10124, 9956] dup_rows_per_dst=[1856, 1570, 2904, 2432, 2838, 4006, 4180, 2654] combine_rows_per_src=[8530, 9226, 8430, 9968, 9640, 9990, 10124, 9956] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75310 payload_rows_remote=66202 dup_rows=22994 dup_rows_remote=19578 dedup_ratio=0.233907 router_tokens_per_expert=[417, 739, 1269, 337, 573, 428, 1122, 734, 1206, 2245, 238, 402, 1309, 709, 847, 459, 1613, 68, 2283, 545, 804, 353, 24, 572, 1082, 1090, 647, 519, 424, 1037, 1501, 950, 598, 772, 262, 201, 1261, 727, 1982, 758, 482, 2099, 148, 548, 349, 726, 555, 2861, 109, 315, 314, 929, 660, 1039, 309, 346, 239, 332, 1552, 588, 680, 518, 150, 197] payload_rows_per_dst=[8966, 8808, 9108, 10610, 9944, 9120, 9332, 9422] dup_rows_per_dst=[2272, 1532, 3416, 4042, 3178, 3262, 3200, 2092] combine_rows_per_src=[8966, 8808, 9108, 10610, 9944, 9120, 9332, 9422] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266887168 dispatch_meta_bytes=336632 combine_payload_bytes=266887168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267223800 ingress_bytes=301490176 link_bytes=301490176 token_bytes=534110968 weight_movement_bytes=34603008 total_nvlink_bytes=568713976 weight_to_token_ratio=0.064786 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134483260e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75216 payload_rows_remote=65140 dup_rows=23088 dup_rows_remote=19624 dedup_ratio=0.234863 router_tokens_per_expert=[34, 1143, 325, 492, 13, 927, 887, 186, 518, 331, 1414, 1579, 1216, 1223, 247, 1225, 1037, 1523, 562, 773, 120, 103, 1278, 1172, 537, 1078, 1246, 781, 485, 623, 1156, 121, 308, 963, 490, 342, 2315, 1537, 1329, 418, 1422, 126, 358, 559, 1499, 1807, 198, 1357, 1209, 387, 373, 850, 13, 58, 349, 472, 876, 275, 102, 655, 160, 1661, 176, 2153] payload_rows_per_dst=[10012, 10076, 10280, 9652, 6942, 9874, 8230, 10150] dup_rows_per_dst=[4290, 3464, 2856, 2402, 2174, 4294, 1642, 1966] combine_rows_per_src=[10012, 10076, 10280, 9652, 6942, 9874, 8230, 10150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75428 payload_rows_remote=64092 dup_rows=22876 dup_rows_remote=18210 dedup_ratio=0.232707 router_tokens_per_expert=[49, 1041, 425, 422, 34, 941, 785, 238, 610, 378, 1184, 1381, 1361, 1101, 209, 1510, 1001, 1575, 476, 735, 138, 122, 1236, 1137, 635, 990, 1034, 692, 424, 611, 984, 231, 373, 1057, 817, 487, 1978, 1756, 1389, 439, 921, 137, 336, 332, 1597, 1862, 166, 1370, 1061, 448, 760, 978, 49, 204, 253, 553, 1093, 417, 122, 546, 69, 1514, 244, 2134] payload_rows_per_dst=[8952, 7974, 10066, 10128, 9216, 8474, 9282, 11336] dup_rows_per_dst=[2874, 1752, 2774, 3796, 3056, 1608, 2350, 4666] combine_rows_per_src=[8952, 7974, 10066, 10128, 9216, 8474, 9282, 11336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75752 payload_rows_remote=65158 dup_rows=22552 dup_rows_remote=19000 dedup_ratio=0.229411 router_tokens_per_expert=[236, 824, 396, 377, 95, 887, 754, 267, 557, 379, 1008, 1281, 1388, 1363, 478, 1766, 962, 1280, 249, 737, 112, 312, 1155, 971, 506, 942, 846, 583, 217, 663, 980, 307, 419, 1075, 806, 678, 1629, 1743, 1786, 515, 699, 177, 381, 237, 1869, 1516, 164, 1450, 1041, 603, 1186, 909, 101, 337, 331, 799, 1015, 441, 93, 557, 79, 1155, 497, 1986] payload_rows_per_dst=[8558, 8236, 9536, 9806, 10078, 9028, 10594, 9916] dup_rows_per_dst=[2372, 1896, 2020, 3058, 3966, 3408, 3552, 2280] combine_rows_per_src=[8558, 8236, 9536, 9806, 10078, 9028, 10594, 9916] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=274391040 dispatch_meta_bytes=344064 combine_payload_bytes=274391040 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=274735104 ingress_bytes=378200064 link_bytes=378200064 token_bytes=549126144 weight_movement_bytes=103809024 total_nvlink_bytes=652935168 weight_to_token_ratio=0.189044 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=10496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.423136393e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 1, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=270114816 dispatch_meta_bytes=346792 combine_payload_bytes=270114816 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=270461608 ingress_bytes=339320832 link_bytes=339320832 token_bytes=540576424 weight_movement_bytes=69206016 total_nvlink_bytes=609782440 weight_to_token_ratio=0.128023 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.276836973e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=269139968 dispatch_meta_bytes=339480 combine_payload_bytes=269139968 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=269479448 ingress_bytes=338345984 link_bytes=338345984 token_bytes=538619416 weight_movement_bytes=69206016 total_nvlink_bytes=607825432 weight_to_token_ratio=0.128488 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=12672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.273168699e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=270327808 dispatch_meta_bytes=346680 combine_payload_bytes=270327808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270674488 ingress_bytes=304930816 link_bytes=304930816 token_bytes=541002296 weight_movement_bytes=34603008 total_nvlink_bytes=575605304 weight_to_token_ratio=0.063961 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=7936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.147430112e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76298 payload_rows_remote=67686 dup_rows=22006 dup_rows_remote=18492 dedup_ratio=0.223857 router_tokens_per_expert=[231, 875, 378, 414, 130, 875, 712, 292, 628, 358, 1160, 1166, 1229, 1490, 813, 1902, 973, 1322, 288, 764, 90, 406, 1143, 870, 412, 917, 797, 539, 261, 643, 996, 388, 369, 985, 827, 813, 1543, 1835, 1780, 499, 599, 166, 380, 276, 1721, 1501, 154, 1266, 1288, 622, 1049, 895, 116, 380, 265, 727, 1012, 391, 130, 535, 49, 1363, 486, 1668] payload_rows_per_dst=[8466, 9088, 9654, 9442, 10310, 8612, 10870, 9856] dup_rows_per_dst=[2434, 2142, 2058, 2922, 3906, 3514, 3618, 1412] combine_rows_per_src=[8466, 9088, 9654, 9442, 10310, 8612, 10870, 9856] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266895360 dispatch_meta_bytes=337880 combine_payload_bytes=266895360 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267233240 ingress_bytes=405307392 link_bytes=405307392 token_bytes=534128600 weight_movement_bytes=138412032 total_nvlink_bytes=672540632 weight_to_token_ratio=0.259136 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=12160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.525139086e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=259186688 dispatch_meta_bytes=324512 combine_payload_bytes=259186688 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=259511200 ingress_bytes=328392704 link_bytes=328392704 token_bytes=518697888 weight_movement_bytes=69206016 total_nvlink_bytes=587903904 weight_to_token_ratio=0.133423 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=14720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.235715307e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=269910016 dispatch_meta_bytes=343616 combine_payload_bytes=269910016 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=270253632 ingress_bytes=373719040 link_bytes=373719040 token_bytes=540163648 weight_movement_bytes=103809024 total_nvlink_bytes=643972672 weight_to_token_ratio=0.192181 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=8960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.406274661e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=271163392 dispatch_meta_bytes=343120 combine_payload_bytes=271163392 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=271506512 ingress_bytes=305766400 link_bytes=305766400 token_bytes=542669904 weight_movement_bytes=34603008 total_nvlink_bytes=577272912 weight_to_token_ratio=0.063764 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=16000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.150574347e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266813440 dispatch_meta_bytes=339056 combine_payload_bytes=266813440 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267152496 ingress_bytes=370622464 link_bytes=370622464 token_bytes=533965936 weight_movement_bytes=103809024 total_nvlink_bytes=637774960 weight_to_token_ratio=0.194411 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394622494e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=262520832 dispatch_meta_bytes=329208 combine_payload_bytes=262520832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262850040 ingress_bytes=297123840 link_bytes=297123840 token_bytes=525370872 weight_movement_bytes=34603008 total_nvlink_bytes=559973880 weight_to_token_ratio=0.065864 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=15872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.118053089e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266887168 dispatch_meta_bytes=336632 combine_payload_bytes=266887168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267223800 ingress_bytes=301490176 link_bytes=301490176 token_bytes=534110968 weight_movement_bytes=34603008 total_nvlink_bytes=568713976 weight_to_token_ratio=0.064786 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134483260e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277241856 dispatch_meta_bytes=344712 combine_payload_bytes=277241856 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=277586568 ingress_bytes=381050880 link_bytes=381050880 token_bytes=554828424 weight_movement_bytes=103809024 total_nvlink_bytes=658637448 weight_to_token_ratio=0.187101 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.433863784e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76660 payload_rows_remote=66926 dup_rows=21644 dup_rows_remote=19454 dedup_ratio=0.220174 router_tokens_per_expert=[296, 690, 362, 419, 108, 889, 764, 275, 582, 302, 1187, 1131, 1173, 1790, 708, 2071, 988, 1416, 246, 783, 101, 381, 1141, 906, 413, 926, 750, 457, 290, 629, 972, 384, 363, 932, 783, 733, 1325, 1806, 1734, 467, 588, 147, 354, 249, 1894, 1602, 147, 1441, 1143, 706, 1121, 811, 166, 388, 258, 819, 952, 374, 77, 583, 71, 1227, 585, 1776] payload_rows_per_dst=[8086, 10548, 9734, 8288, 10030, 9066, 11000, 9908] dup_rows_per_dst=[2170, 3198, 2190, 1354, 3606, 3778, 3966, 1382] combine_rows_per_src=[8086, 10548, 9734, 8288, 10030, 9066, 11000, 9908] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75864 payload_rows_remote=65896 dup_rows=22440 dup_rows_remote=20008 dedup_ratio=0.228271 router_tokens_per_expert=[316, 674, 1009, 503, 547, 431, 991, 722, 1623, 2343, 164, 415, 958, 725, 1088, 425, 1678, 97, 1823, 518, 643, 361, 21, 526, 1195, 930, 675, 460, 431, 1233, 1644, 1276, 452, 675, 321, 295, 1306, 637, 1886, 667, 387, 1862, 253, 518, 322, 628, 532, 2496, 209, 352, 395, 913, 973, 1067, 400, 500, 364, 295, 1381, 608, 706, 710, 252, 345] payload_rows_per_dst=[8530, 9226, 8430, 9968, 9640, 9990, 10124, 9956] dup_rows_per_dst=[1856, 1570, 2904, 2432, 2838, 4006, 4180, 2654] combine_rows_per_src=[8530, 9226, 8430, 9968, 9640, 9990, 10124, 9956] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76932 payload_rows_remote=67728 dup_rows=21372 dup_rows_remote=19190 dedup_ratio=0.217407 router_tokens_per_expert=[270, 452, 1550, 283, 746, 668, 1089, 1137, 1754, 2967, 461, 400, 1393, 601, 1168, 277, 1667, 39, 1767, 542, 996, 479, 6, 619, 1024, 1457, 608, 400, 415, 1097, 1471, 734, 586, 634, 209, 184, 1195, 504, 1634, 747, 326, 2432, 218, 447, 295, 485, 321, 2722, 17, 281, 266, 971, 504, 883, 224, 218, 168, 200, 1627, 539, 653, 779, 125, 221] payload_rows_per_dst=[9598, 9950, 8930, 9468, 9204, 10374, 9844, 9564] dup_rows_per_dst=[2792, 2158, 3300, 2002, 2182, 4118, 2818, 2002] combine_rows_per_src=[9598, 9950, 8930, 9468, 9204, 10374, 9844, 9564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75306 payload_rows_remote=62906 dup_rows=22998 dup_rows_remote=18700 dedup_ratio=0.233948 router_tokens_per_expert=[182, 378, 1491, 250, 630, 904, 1194, 978, 2324, 3177, 314, 401, 1088, 517, 1292, 436, 1737, 65, 2005, 524, 985, 490, 0, 508, 961, 1492, 587, 420, 335, 1267, 1627, 753, 576, 602, 210, 197, 1402, 596, 1469, 656, 118, 2045, 217, 403, 158, 449, 351, 2342, 0, 288, 215, 1141, 569, 866, 136, 149, 414, 115, 1843, 523, 684, 709, 122, 275] payload_rows_per_dst=[12400, 13440, 9252, 9662, 9222, 5696, 5690, 9944] dup_rows_per_dst=[4298, 6072, 3376, 1968, 2360, 1206, 1038, 2680] combine_rows_per_src=[12400, 13440, 9252, 9662, 9222, 5696, 5690, 9944] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76148 payload_rows_remote=65708 dup_rows=22156 dup_rows_remote=19162 dedup_ratio=0.225382 router_tokens_per_expert=[225, 420, 1245, 301, 513, 677, 1248, 905, 1962, 3284, 376, 404, 1423, 489, 1266, 432, 1794, 66, 1953, 492, 919, 589, 3, 481, 1191, 1286, 592, 554, 413, 1088, 1468, 707, 525, 596, 165, 165, 1267, 602, 1789, 732, 174, 2120, 149, 461, 263, 429, 356, 2710, 8, 263, 223, 1115, 681, 849, 148, 150, 269, 225, 1659, 555, 745, 642, 113, 238] payload_rows_per_dst=[8838, 10144, 9208, 8188, 9286, 9710, 10334, 10440] dup_rows_per_dst=[2230, 2560, 3384, 1868, 2396, 3616, 3108, 2994] combine_rows_per_src=[8838, 10144, 9208, 8188, 9286, 9710, 10334, 10440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75642 payload_rows_remote=65946 dup_rows=22662 dup_rows_remote=20752 dedup_ratio=0.230530 router_tokens_per_expert=[168, 346, 1444, 391, 668, 800, 1218, 1008, 1832, 3101, 247, 383, 794, 674, 1272, 435, 1951, 74, 1904, 424, 920, 500, 1, 452, 1045, 1307, 571, 399, 519, 1165, 1873, 920, 585, 451, 167, 212, 1574, 590, 1848, 718, 64, 1802, 194, 433, 278, 356, 347, 2484, 4, 251, 201, 1053, 777, 860, 170, 186, 461, 286, 1629, 527, 790, 700, 117, 231] payload_rows_per_dst=[10562, 9696, 9040, 10768, 9672, 5692, 9900, 10312] dup_rows_per_dst=[3618, 1910, 3412, 3958, 2618, 1256, 2974, 2916] combine_rows_per_src=[10562, 9696, 9040, 10768, 9672, 5692, 9900, 10312] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75836 payload_rows_remote=66214 dup_rows=22468 dup_rows_remote=19162 dedup_ratio=0.228556 router_tokens_per_expert=[367, 740, 1180, 328, 541, 443, 1020, 818, 1512, 2596, 290, 452, 1339, 497, 950, 476, 1665, 64, 2182, 525, 714, 409, 33, 603, 1162, 1098, 665, 584, 445, 995, 1541, 927, 610, 814, 264, 289, 1274, 659, 1686, 730, 527, 2006, 200, 502, 329, 586, 428, 2886, 68, 241, 394, 957, 641, 1046, 236, 285, 221, 237, 1557, 599, 735, 618, 145, 221] payload_rows_per_dst=[8744, 9100, 9092, 9416, 9730, 10546, 9622, 9586] dup_rows_per_dst=[2130, 1932, 3298, 2336, 2922, 4382, 3306, 2162] combine_rows_per_src=[8744, 9100, 9092, 9416, 9730, 10546, 9622, 9586] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74860 payload_rows_remote=65988 dup_rows=23444 dup_rows_remote=21202 dedup_ratio=0.238485 router_tokens_per_expert=[0, 1094, 1379, 730, 1049, 2492, 593, 1152, 393, 673, 2340, 63, 144, 356, 238, 1350, 1668, 340, 8, 72, 340, 742, 229, 289, 464, 333, 2377, 344, 3047, 699, 597, 0, 248, 0, 0, 1128, 1061, 218, 70, 1345, 60, 213, 11, 1110, 447, 0, 2488, 0, 9, 673, 287, 837, 422, 1608, 956, 1334, 440, 1372, 47, 1322, 965, 1879, 1879, 1128] payload_rows_per_dst=[8414, 8872, 9734, 10064, 10072, 11528, 9758, 6418] dup_rows_per_dst=[3580, 2242, 2902, 3576, 3052, 2970, 3740, 1382] combine_rows_per_src=[8414, 8872, 9734, 10064, 10072, 11528, 9758, 6418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=274128896 dispatch_meta_bytes=345520 combine_payload_bytes=274128896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274474416 ingress_bytes=308731904 link_bytes=308731904 token_bytes=548603312 weight_movement_bytes=34603008 total_nvlink_bytes=583206320 weight_to_token_ratio=0.063075 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.161733300e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=269910016 dispatch_meta_bytes=343616 combine_payload_bytes=269910016 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=270253632 ingress_bytes=373719040 link_bytes=373719040 token_bytes=540163648 weight_movement_bytes=103809024 total_nvlink_bytes=643972672 weight_to_token_ratio=0.192181 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=8960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.406274661e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277413888 dispatch_meta_bytes=347672 combine_payload_bytes=277413888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277761560 ingress_bytes=312016896 link_bytes=312016896 token_bytes=555175448 weight_movement_bytes=34603008 total_nvlink_bytes=589778456 weight_to_token_ratio=0.062328 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.174094460e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75306 payload_rows_remote=62906 dup_rows=22998 dup_rows_remote=18700 dedup_ratio=0.233948 router_tokens_per_expert=[182, 378, 1491, 250, 630, 904, 1194, 978, 2324, 3177, 314, 401, 1088, 517, 1292, 436, 1737, 65, 2005, 524, 985, 490, 0, 508, 961, 1492, 587, 420, 335, 1267, 1627, 753, 576, 602, 210, 197, 1402, 596, 1469, 656, 118, 2045, 217, 403, 158, 449, 351, 2342, 0, 288, 215, 1141, 569, 866, 136, 149, 414, 115, 1843, 523, 684, 709, 122, 275] payload_rows_per_dst=[12400, 13440, 9252, 9662, 9222, 5696, 5690, 9944] dup_rows_per_dst=[4298, 6072, 3376, 1968, 2360, 1206, 1038, 2680] combine_rows_per_src=[12400, 13440, 9252, 9662, 9222, 5696, 5690, 9944] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73694 payload_rows_remote=64772 dup_rows=24610 dup_rows_remote=21792 dedup_ratio=0.250346 router_tokens_per_expert=[54, 2082, 1589, 423, 959, 1715, 798, 708, 538, 959, 2117, 125, 265, 509, 407, 1256, 1443, 359, 292, 175, 267, 576, 306, 328, 733, 407, 1268, 719, 1664, 1278, 641, 110, 351, 167, 58, 751, 1273, 462, 265, 1482, 191, 422, 566, 1114, 536, 33, 1714, 131, 431, 526, 308, 701, 488, 1329, 757, 907, 887, 1155, 108, 891, 1036, 2360, 1328, 1354] payload_rows_per_dst=[8720, 9336, 9368, 8922, 10202, 10198, 8122, 8826] dup_rows_per_dst=[4506, 3016, 2844, 2818, 2846, 2544, 2772, 3264] combine_rows_per_src=[8720, 9336, 9368, 8922, 10202, 10198, 8122, 8826] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75202 payload_rows_remote=64324 dup_rows=23102 dup_rows_remote=17814 dedup_ratio=0.235006 router_tokens_per_expert=[0, 1202, 1428, 1100, 889, 1763, 639, 949, 426, 946, 2862, 69, 148, 361, 261, 1073, 1514, 337, 0, 54, 233, 791, 310, 293, 628, 217, 2283, 381, 2945, 745, 570, 0, 206, 0, 0, 1187, 1062, 91, 62, 1182, 52, 157, 12, 929, 410, 0, 2716, 0, 13, 1058, 247, 679, 402, 1631, 901, 1202, 580, 1400, 57, 1311, 1195, 1946, 1625, 1422] payload_rows_per_dst=[7422, 9686, 8842, 7708, 9996, 11528, 9142, 10878] dup_rows_per_dst=[1602, 2606, 2114, 1940, 3514, 2914, 3124, 5288] combine_rows_per_src=[7422, 9686, 8842, 7708, 9996, 11528, 9142, 10878] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73548 payload_rows_remote=63948 dup_rows=24756 dup_rows_remote=22088 dedup_ratio=0.251831 router_tokens_per_expert=[3, 1944, 1788, 718, 970, 1260, 704, 727, 323, 994, 2495, 77, 176, 451, 318, 1376, 1384, 446, 120, 107, 313, 590, 217, 395, 980, 391, 1430, 811, 1370, 1158, 858, 6, 306, 9, 24, 644, 1632, 368, 191, 1238, 102, 257, 194, 877, 665, 0, 1823, 1, 324, 736, 357, 676, 496, 1205, 636, 1344, 978, 1094, 104, 1130, 1224, 2562, 1620, 1435] payload_rows_per_dst=[9098, 9550, 9600, 9952, 9074, 8958, 8670, 8646] dup_rows_per_dst=[4610, 2870, 2668, 4556, 2270, 1620, 2878, 3284] combine_rows_per_src=[9098, 9550, 9600, 9952, 9074, 8958, 8670, 8646] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=271212544 dispatch_meta_bytes=341504 combine_payload_bytes=271212544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271554048 ingress_bytes=305815552 link_bytes=305815552 token_bytes=542766592 weight_movement_bytes=34603008 total_nvlink_bytes=577369600 weight_to_token_ratio=0.063753 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.150759302e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=270286848 dispatch_meta_bytes=348760 combine_payload_bytes=270286848 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=270635608 ingress_bytes=304889856 link_bytes=304889856 token_bytes=540922456 weight_movement_bytes=34603008 total_nvlink_bytes=575525464 weight_to_token_ratio=0.063970 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.147275982e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75438 payload_rows_remote=64640 dup_rows=22866 dup_rows_remote=18334 dedup_ratio=0.232605 router_tokens_per_expert=[348, 807, 1214, 408, 576, 446, 1150, 786, 1296, 2164, 230, 414, 1286, 504, 905, 468, 1611, 83, 2058, 545, 771, 417, 19, 573, 1034, 974, 649, 621, 518, 1071, 1519, 942, 521, 708, 302, 288, 1184, 771, 2092, 690, 379, 1882, 177, 577, 366, 656, 490, 3138, 92, 358, 335, 973, 738, 1234, 229, 294, 312, 247, 1408, 654, 761, 528, 157, 204] payload_rows_per_dst=[8960, 8684, 8822, 9242, 10000, 10798, 9450, 9482] dup_rows_per_dst=[2510, 1522, 3332, 2376, 3112, 4532, 3384, 2098] combine_rows_per_src=[8960, 8684, 8822, 9242, 10000, 10798, 9450, 9482] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=257662976 dispatch_meta_bytes=326424 combine_payload_bytes=257662976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=257989400 ingress_bytes=292265984 link_bytes=292265984 token_bytes=515652376 weight_movement_bytes=34603008 total_nvlink_bytes=550255384 weight_to_token_ratio=0.067105 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.099773368e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265306112 dispatch_meta_bytes=346256 combine_payload_bytes=265306112 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265652368 ingress_bytes=334512128 link_bytes=334512128 token_bytes=530958480 weight_movement_bytes=69206016 total_nvlink_bytes=600164496 weight_to_token_ratio=0.130342 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=14720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258742207e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=261931008 dispatch_meta_bytes=344144 combine_payload_bytes=261931008 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=262275152 ingress_bytes=331137024 link_bytes=331137024 token_bytes=524206160 weight_movement_bytes=69206016 total_nvlink_bytes=593412176 weight_to_token_ratio=0.132021 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.032500 gemm_active_groups=11 gemm_peak_group_rows=17536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.246041963e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=263471104 dispatch_meta_bytes=328552 combine_payload_bytes=263471104 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=263799656 ingress_bytes=436486144 link_bytes=436486144 token_bytes=527270760 weight_movement_bytes=173015040 total_nvlink_bytes=700285800 weight_to_token_ratio=0.328133 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=9472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.642462220e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=269139968 dispatch_meta_bytes=339480 combine_payload_bytes=269139968 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=269479448 ingress_bytes=338345984 link_bytes=338345984 token_bytes=538619416 weight_movement_bytes=69206016 total_nvlink_bytes=607825432 weight_to_token_ratio=0.128488 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=12672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.273168699e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=270114816 dispatch_meta_bytes=346792 combine_payload_bytes=270114816 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=270461608 ingress_bytes=339320832 link_bytes=339320832 token_bytes=540576424 weight_movement_bytes=69206016 total_nvlink_bytes=609782440 weight_to_token_ratio=0.128023 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.276836973e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=271212544 dispatch_meta_bytes=341504 combine_payload_bytes=271212544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271554048 ingress_bytes=305815552 link_bytes=305815552 token_bytes=542766592 weight_movement_bytes=34603008 total_nvlink_bytes=577369600 weight_to_token_ratio=0.063753 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.150759302e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=264765440 dispatch_meta_bytes=331896 combine_payload_bytes=264765440 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=265097336 ingress_bytes=437780480 link_bytes=437780480 token_bytes=529862776 weight_movement_bytes=173015040 total_nvlink_bytes=702877816 weight_to_token_ratio=0.326528 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=16000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.647332702e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75310 payload_rows_remote=66202 dup_rows=22994 dup_rows_remote=19578 dedup_ratio=0.233907 router_tokens_per_expert=[417, 739, 1269, 337, 573, 428, 1122, 734, 1206, 2245, 238, 402, 1309, 709, 847, 459, 1613, 68, 2283, 545, 804, 353, 24, 572, 1082, 1090, 647, 519, 424, 1037, 1501, 950, 598, 772, 262, 201, 1261, 727, 1982, 758, 482, 2099, 148, 548, 349, 726, 555, 2861, 109, 315, 314, 929, 660, 1039, 309, 346, 239, 332, 1552, 588, 680, 518, 150, 197] payload_rows_per_dst=[8966, 8808, 9108, 10610, 9944, 9120, 9332, 9422] dup_rows_per_dst=[2272, 1532, 3416, 4042, 3178, 3262, 3200, 2092] combine_rows_per_src=[8966, 8808, 9108, 10610, 9944, 9120, 9332, 9422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73694 payload_rows_remote=64772 dup_rows=24610 dup_rows_remote=21792 dedup_ratio=0.250346 router_tokens_per_expert=[54, 2082, 1589, 423, 959, 1715, 798, 708, 538, 959, 2117, 125, 265, 509, 407, 1256, 1443, 359, 292, 175, 267, 576, 306, 328, 733, 407, 1268, 719, 1664, 1278, 641, 110, 351, 167, 58, 751, 1273, 462, 265, 1482, 191, 422, 566, 1114, 536, 33, 1714, 131, 431, 526, 308, 701, 488, 1329, 757, 907, 887, 1155, 108, 891, 1036, 2360, 1328, 1354] payload_rows_per_dst=[8720, 9336, 9368, 8922, 10202, 10198, 8122, 8826] dup_rows_per_dst=[4506, 3016, 2844, 2818, 2846, 2544, 2772, 3264] combine_rows_per_src=[8720, 9336, 9368, 8922, 10202, 10198, 8122, 8826] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74862 payload_rows_remote=65712 dup_rows=23442 dup_rows_remote=21178 dedup_ratio=0.238464 router_tokens_per_expert=[0, 1164, 1467, 1165, 853, 1155, 605, 924, 377, 950, 2818, 70, 157, 577, 262, 1025, 1128, 322, 29, 66, 292, 903, 374, 308, 823, 320, 2493, 425, 2820, 732, 484, 0, 270, 2, 4, 1313, 1225, 144, 95, 1499, 49, 200, 7, 958, 477, 0, 2756, 0, 33, 1002, 290, 744, 377, 1779, 663, 1122, 634, 1260, 58, 1188, 1253, 1940, 1447, 1275] payload_rows_per_dst=[8432, 9870, 8614, 8518, 9150, 11518, 9052, 9708] dup_rows_per_dst=[3924, 2602, 2110, 2036, 2264, 3016, 2968, 4522] combine_rows_per_src=[8432, 9870, 8614, 8518, 9150, 11518, 9052, 9708] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=65888 dup_rows=24040 dup_rows_remote=20446 dedup_ratio=0.244548 router_tokens_per_expert=[0, 1139, 1435, 926, 812, 2277, 488, 1050, 517, 1196, 2670, 80, 178, 431, 277, 995, 1906, 351, 2, 10, 209, 1024, 321, 269, 692, 99, 1699, 315, 3110, 538, 415, 0, 253, 0, 0, 1635, 887, 39, 34, 1238, 46, 255, 1, 962, 361, 1, 2954, 1, 4, 1108, 227, 697, 405, 2190, 927, 1026, 458, 1327, 46, 1222, 979, 1751, 1301, 1386] payload_rows_per_dst=[8376, 9598, 10564, 11200, 9812, 8540, 11316, 4858] dup_rows_per_dst=[3594, 2820, 3894, 5138, 2914, 622, 4506, 552] combine_rows_per_src=[8376, 9598, 10564, 11200, 9812, 8540, 11316, 4858] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73226 payload_rows_remote=65174 dup_rows=25078 dup_rows_remote=22382 dedup_ratio=0.255107 router_tokens_per_expert=[20, 1873, 1593, 727, 915, 1163, 685, 823, 589, 1053, 2295, 82, 174, 471, 326, 1248, 1635, 374, 173, 91, 312, 668, 286, 337, 986, 386, 1511, 625, 1625, 1036, 738, 24, 272, 8, 30, 763, 1572, 306, 186, 1143, 81, 373, 168, 978, 582, 6, 2022, 13, 149, 684, 311, 637, 440, 1343, 570, 1240, 993, 1319, 113, 1160, 1329, 2416, 1597, 1504] payload_rows_per_dst=[8364, 9550, 9652, 8226, 9316, 9646, 8052, 10420] dup_rows_per_dst=[3878, 2926, 2932, 2386, 2600, 2050, 2696, 5610] combine_rows_per_src=[8364, 9550, 9652, 8226, 9316, 9646, 8052, 10420] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=269156352 dispatch_meta_bytes=347560 combine_payload_bytes=269156352 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=269503912 ingress_bytes=338362368 link_bytes=338362368 token_bytes=538660264 weight_movement_bytes=69206016 total_nvlink_bytes=607866280 weight_to_token_ratio=0.128478 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=13312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.273230350e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75442 payload_rows_remote=66604 dup_rows=22862 dup_rows_remote=21384 dedup_ratio=0.232564 router_tokens_per_expert=[888, 2161, 672, 156, 86, 367, 2814, 864, 0, 0, 856, 720, 2585, 0, 0, 997, 568, 908, 3946, 91, 428, 1502, 296, 0, 143, 2232, 1271, 138, 58, 2, 2424, 375, 1353, 360, 105, 1920, 367, 315, 1095, 634, 88, 0, 6, 1280, 97, 993, 1239, 727, 915, 1108, 1808, 1162, 133, 485, 0, 1391, 1447, 420, 96, 277, 353, 355, 617, 458] payload_rows_per_dst=[11972, 8838, 13052, 7506, 9734, 8216, 9288, 6836] dup_rows_per_dst=[6014, 1478, 5304, 932, 2564, 3426, 1934, 1210] combine_rows_per_src=[11972, 8838, 13052, 7506, 9734, 8216, 9288, 6836] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75202 payload_rows_remote=64324 dup_rows=23102 dup_rows_remote=17814 dedup_ratio=0.235006 router_tokens_per_expert=[0, 1202, 1428, 1100, 889, 1763, 639, 949, 426, 946, 2862, 69, 148, 361, 261, 1073, 1514, 337, 0, 54, 233, 791, 310, 293, 628, 217, 2283, 381, 2945, 745, 570, 0, 206, 0, 0, 1187, 1062, 91, 62, 1182, 52, 157, 12, 929, 410, 0, 2716, 0, 13, 1058, 247, 679, 402, 1631, 901, 1202, 580, 1400, 57, 1311, 1195, 1946, 1625, 1422] payload_rows_per_dst=[7422, 9686, 8842, 7708, 9996, 11528, 9142, 10878] dup_rows_per_dst=[1602, 2606, 2114, 1940, 3514, 2914, 3124, 5288] combine_rows_per_src=[7422, 9686, 8842, 7708, 9996, 11528, 9142, 10878] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74860 payload_rows_remote=65988 dup_rows=23444 dup_rows_remote=21202 dedup_ratio=0.238485 router_tokens_per_expert=[0, 1094, 1379, 730, 1049, 2492, 593, 1152, 393, 673, 2340, 63, 144, 356, 238, 1350, 1668, 340, 8, 72, 340, 742, 229, 289, 464, 333, 2377, 344, 3047, 699, 597, 0, 248, 0, 0, 1128, 1061, 218, 70, 1345, 60, 213, 11, 1110, 447, 0, 2488, 0, 9, 673, 287, 837, 422, 1608, 956, 1334, 440, 1372, 47, 1322, 965, 1879, 1879, 1128] payload_rows_per_dst=[8414, 8872, 9734, 10064, 10072, 11528, 9758, 6418] dup_rows_per_dst=[3580, 2242, 2902, 3576, 3052, 2970, 3740, 1382] combine_rows_per_src=[8414, 8872, 9734, 10064, 10072, 11528, 9758, 6418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=271163392 dispatch_meta_bytes=343120 combine_payload_bytes=271163392 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=271506512 ingress_bytes=305766400 link_bytes=305766400 token_bytes=542669904 weight_movement_bytes=34603008 total_nvlink_bytes=577272912 weight_to_token_ratio=0.063764 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=16000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.150574347e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265306112 dispatch_meta_bytes=346256 combine_payload_bytes=265306112 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265652368 ingress_bytes=334512128 link_bytes=334512128 token_bytes=530958480 weight_movement_bytes=69206016 total_nvlink_bytes=600164496 weight_to_token_ratio=0.130342 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=14720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258742207e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=65888 dup_rows=24040 dup_rows_remote=20446 dedup_ratio=0.244548 router_tokens_per_expert=[0, 1139, 1435, 926, 812, 2277, 488, 1050, 517, 1196, 2670, 80, 178, 431, 277, 995, 1906, 351, 2, 10, 209, 1024, 321, 269, 692, 99, 1699, 315, 3110, 538, 415, 0, 253, 0, 0, 1635, 887, 39, 34, 1238, 46, 255, 1, 962, 361, 1, 2954, 1, 4, 1108, 227, 697, 405, 2190, 927, 1026, 458, 1327, 46, 1222, 979, 1751, 1301, 1386] payload_rows_per_dst=[8376, 9598, 10564, 11200, 9812, 8540, 11316, 4858] dup_rows_per_dst=[3594, 2820, 3894, 5138, 2914, 622, 4506, 552] combine_rows_per_src=[8376, 9598, 10564, 11200, 9812, 8540, 11316, 4858] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=269156352 dispatch_meta_bytes=347560 combine_payload_bytes=269156352 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=269503912 ingress_bytes=338362368 link_bytes=338362368 token_bytes=538660264 weight_movement_bytes=69206016 total_nvlink_bytes=607866280 weight_to_token_ratio=0.128478 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=13312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.273230350e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72904 payload_rows_remote=63940 dup_rows=25400 dup_rows_remote=23712 dedup_ratio=0.258382 router_tokens_per_expert=[6, 1895, 1630, 761, 963, 1378, 764, 873, 385, 1142, 2247, 109, 261, 437, 304, 1495, 1470, 369, 133, 85, 287, 567, 253, 401, 903, 418, 1574, 743, 1548, 1064, 901, 29, 279, 30, 51, 612, 1399, 327, 241, 1294, 137, 244, 136, 946, 604, 11, 1695, 5, 298, 695, 390, 682, 444, 1289, 720, 1236, 874, 1161, 100, 1147, 1226, 2668, 1547, 1269] payload_rows_per_dst=[8958, 9560, 9736, 8458, 9016, 8964, 8436, 9776] dup_rows_per_dst=[4826, 3200, 2730, 2806, 2206, 1688, 3072, 4872] combine_rows_per_src=[8958, 9560, 9736, 8458, 9016, 8964, 8436, 9776] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=269877248 dispatch_meta_bytes=345336 combine_payload_bytes=269877248 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=270222584 ingress_bytes=408289280 link_bytes=408289280 token_bytes=540099832 weight_movement_bytes=138412032 total_nvlink_bytes=678511864 weight_to_token_ratio=0.256271 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=11904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.536359691e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266952704 dispatch_meta_bytes=350224 combine_payload_bytes=266952704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267302928 ingress_bytes=301555712 link_bytes=301555712 token_bytes=534255632 weight_movement_bytes=34603008 total_nvlink_bytes=568858640 weight_to_token_ratio=0.064769 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134729867e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=263471104 dispatch_meta_bytes=328552 combine_payload_bytes=263471104 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=263799656 ingress_bytes=436486144 link_bytes=436486144 token_bytes=527270760 weight_movement_bytes=173015040 total_nvlink_bytes=700285800 weight_to_token_ratio=0.328133 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=9472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.642462220e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=270286848 dispatch_meta_bytes=348760 combine_payload_bytes=270286848 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=270635608 ingress_bytes=304889856 link_bytes=304889856 token_bytes=540922456 weight_movement_bytes=34603008 total_nvlink_bytes=575525464 weight_to_token_ratio=0.063970 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.147275982e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=261898240 dispatch_meta_bytes=350608 combine_payload_bytes=261898240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262248848 ingress_bytes=296501248 link_bytes=296501248 token_bytes=524147088 weight_movement_bytes=34603008 total_nvlink_bytes=558750096 weight_to_token_ratio=0.066018 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.115710326e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73548 payload_rows_remote=63948 dup_rows=24756 dup_rows_remote=22088 dedup_ratio=0.251831 router_tokens_per_expert=[3, 1944, 1788, 718, 970, 1260, 704, 727, 323, 994, 2495, 77, 176, 451, 318, 1376, 1384, 446, 120, 107, 313, 590, 217, 395, 980, 391, 1430, 811, 1370, 1158, 858, 6, 306, 9, 24, 644, 1632, 368, 191, 1238, 102, 257, 194, 877, 665, 0, 1823, 1, 324, 736, 357, 676, 496, 1205, 636, 1344, 978, 1094, 104, 1130, 1224, 2562, 1620, 1435] payload_rows_per_dst=[9098, 9550, 9600, 9952, 9074, 8958, 8670, 8646] dup_rows_per_dst=[4610, 2870, 2668, 4556, 2270, 1620, 2878, 3284] combine_rows_per_src=[9098, 9550, 9600, 9952, 9074, 8958, 8670, 8646] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76604 payload_rows_remote=67072 dup_rows=21700 dup_rows_remote=19626 dedup_ratio=0.220744 router_tokens_per_expert=[951, 1850, 1119, 258, 303, 555, 2124, 427, 218, 66, 561, 852, 1514, 5, 53, 1158, 982, 813, 2709, 163, 522, 666, 437, 126, 201, 1294, 889, 204, 213, 258, 2385, 359, 1301, 607, 265, 2509, 499, 553, 1616, 585, 192, 114, 61, 1325, 420, 621, 1683, 972, 1095, 1206, 1124, 812, 344, 568, 72, 1758, 1004, 519, 511, 522, 387, 454, 780, 438] payload_rows_per_dst=[8744, 10240, 10170, 9532, 11502, 9634, 8882, 7900] dup_rows_per_dst=[2182, 2862, 2666, 2074, 4368, 4658, 1560, 1330] combine_rows_per_src=[8744, 10240, 10170, 9532, 11502, 9634, 8882, 7900] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77018 payload_rows_remote=64878 dup_rows=21286 dup_rows_remote=16882 dedup_ratio=0.216532 router_tokens_per_expert=[555, 1758, 1027, 235, 169, 316, 2724, 300, 59, 0, 723, 857, 1483, 0, 0, 1174, 1068, 796, 3493, 119, 565, 723, 217, 0, 60, 1989, 846, 115, 175, 176, 3551, 331, 1257, 449, 131, 1822, 656, 312, 1449, 811, 319, 0, 84, 1500, 169, 678, 1546, 649, 982, 1323, 1195, 735, 217, 767, 0, 1317, 1331, 452, 445, 492, 525, 640, 739, 556] payload_rows_per_dst=[10088, 8232, 12140, 9692, 10480, 8588, 8978, 8820] dup_rows_per_dst=[2654, 1786, 4404, 2212, 3294, 3936, 1460, 1540] combine_rows_per_src=[10088, 8232, 12140, 9692, 10480, 8588, 8978, 8820] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76916 payload_rows_remote=67098 dup_rows=21388 dup_rows_remote=18652 dedup_ratio=0.217570 router_tokens_per_expert=[683, 1770, 631, 243, 88, 276, 2736, 582, 1, 0, 800, 650, 1719, 0, 0, 901, 603, 777, 3706, 73, 456, 1361, 229, 0, 146, 2467, 1184, 125, 59, 1, 3408, 241, 1306, 378, 91, 1553, 613, 312, 1120, 904, 56, 0, 13, 1115, 142, 1203, 1614, 893, 1050, 1131, 1459, 1384, 195, 515, 0, 1597, 1394, 444, 222, 323, 467, 411, 730, 601] payload_rows_per_dst=[7230, 10702, 10894, 11136, 9818, 9450, 9304, 8382] dup_rows_per_dst=[1316, 2912, 2514, 4126, 2736, 3816, 2164, 1804] combine_rows_per_src=[7230, 10702, 10894, 11136, 9818, 9450, 9304, 8382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75774 payload_rows_remote=63422 dup_rows=22530 dup_rows_remote=15182 dedup_ratio=0.229187 router_tokens_per_expert=[516, 1797, 659, 277, 55, 383, 2357, 786, 1, 1, 871, 633, 2547, 0, 2, 793, 590, 691, 4011, 46, 386, 1862, 310, 3, 119, 1844, 1165, 128, 68, 33, 3020, 181, 1624, 369, 74, 2013, 511, 335, 1134, 572, 28, 1, 5, 828, 99, 858, 1690, 880, 1206, 999, 1624, 1371, 237, 539, 3, 1435, 1728, 463, 243, 303, 448, 357, 578, 462] payload_rows_per_dst=[12352, 8580, 12276, 7410, 9506, 8308, 9696, 7646] dup_rows_per_dst=[7348, 1116, 3522, 980, 2444, 3340, 2262, 1518] combine_rows_per_src=[12352, 8580, 12276, 7410, 9506, 8308, 9696, 7646] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76156 payload_rows_remote=63812 dup_rows=22148 dup_rows_remote=18314 dedup_ratio=0.225301 router_tokens_per_expert=[596, 1956, 638, 237, 68, 380, 2367, 626, 0, 0, 935, 656, 2343, 0, 0, 972, 504, 916, 3876, 55, 495, 1723, 285, 0, 74, 2475, 1272, 104, 60, 0, 3276, 237, 1278, 434, 85, 1673, 470, 292, 1109, 524, 47, 0, 15, 1148, 106, 1036, 1481, 790, 1035, 1187, 1798, 1248, 191, 501, 0, 1365, 1391, 437, 90, 375, 352, 421, 651, 496] payload_rows_per_dst=[7598, 10966, 7270, 10934, 7170, 7708, 12166, 12344] dup_rows_per_dst=[1404, 3580, 686, 4062, 1214, 1698, 5670, 3834] combine_rows_per_src=[7598, 10966, 7270, 10934, 7170, 7708, 12166, 12344] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266952704 dispatch_meta_bytes=350224 combine_payload_bytes=266952704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267302928 ingress_bytes=301555712 link_bytes=301555712 token_bytes=534255632 weight_movement_bytes=34603008 total_nvlink_bytes=568858640 weight_to_token_ratio=0.064769 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134729867e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=272809984 dispatch_meta_bytes=351952 combine_payload_bytes=272809984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273161936 ingress_bytes=307412992 link_bytes=307412992 token_bytes=545971920 weight_movement_bytes=34603008 total_nvlink_bytes=580574928 weight_to_token_ratio=0.063379 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.156770340e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=269877248 dispatch_meta_bytes=345336 combine_payload_bytes=269877248 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=270222584 ingress_bytes=408289280 link_bytes=408289280 token_bytes=540099832 weight_movement_bytes=138412032 total_nvlink_bytes=678511864 weight_to_token_ratio=0.256271 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=11904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.536359691e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=274726912 dispatch_meta_bytes=346792 combine_payload_bytes=274726912 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=275073704 ingress_bytes=378535936 link_bytes=378535936 token_bytes=549800616 weight_movement_bytes=103809024 total_nvlink_bytes=653609640 weight_to_token_ratio=0.188812 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.424400253e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75802 payload_rows_remote=67122 dup_rows=22502 dup_rows_remote=20016 dedup_ratio=0.228902 router_tokens_per_expert=[1138, 374, 845, 1103, 629, 748, 807, 938, 1386, 362, 952, 368, 320, 1394, 872, 963, 850, 1006, 782, 525, 765, 737, 459, 1330, 518, 682, 828, 368, 808, 114, 887, 794, 1067, 855, 1260, 905, 352, 265, 462, 899, 595, 637, 503, 759, 1072, 554, 772, 691, 743, 629, 906, 531, 995, 1182, 833, 693, 1158, 293, 1120, 640, 210, 778, 1096, 1045] payload_rows_per_dst=[8212, 9494, 10346, 9840, 9154, 8680, 9948, 10128] dup_rows_per_dst=[2676, 3740, 2562, 2434, 2976, 2486, 3076, 2552] combine_rows_per_src=[8212, 9494, 10346, 9840, 9154, 8680, 9948, 10128] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265740288 dispatch_meta_bytes=327040 combine_payload_bytes=265740288 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266067328 ingress_bytes=334946304 link_bytes=334946304 token_bytes=531807616 weight_movement_bytes=69206016 total_nvlink_bytes=601013632 weight_to_token_ratio=0.130134 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260375977e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76916 payload_rows_remote=67098 dup_rows=21388 dup_rows_remote=18652 dedup_ratio=0.217570 router_tokens_per_expert=[683, 1770, 631, 243, 88, 276, 2736, 582, 1, 0, 800, 650, 1719, 0, 0, 901, 603, 777, 3706, 73, 456, 1361, 229, 0, 146, 2467, 1184, 125, 59, 1, 3408, 241, 1306, 378, 91, 1553, 613, 312, 1120, 904, 56, 0, 13, 1115, 142, 1203, 1614, 893, 1050, 1131, 1459, 1384, 195, 515, 0, 1597, 1394, 444, 222, 323, 467, 411, 730, 601] payload_rows_per_dst=[7230, 10702, 10894, 11136, 9818, 9450, 9304, 8382] dup_rows_per_dst=[1316, 2912, 2514, 4126, 2736, 3816, 2164, 1804] combine_rows_per_src=[7230, 10702, 10894, 11136, 9818, 9450, 9304, 8382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=261373952 dispatch_meta_bytes=328504 combine_payload_bytes=261373952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=261702456 ingress_bytes=295976960 link_bytes=295976960 token_bytes=523076408 weight_movement_bytes=34603008 total_nvlink_bytes=557679416 weight_to_token_ratio=0.066153 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.113737472e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76786 payload_rows_remote=66094 dup_rows=21518 dup_rows_remote=18324 dedup_ratio=0.218892 router_tokens_per_expert=[789, 1860, 1004, 206, 128, 540, 2610, 459, 42, 6, 717, 793, 1531, 4, 13, 1125, 790, 929, 3357, 80, 530, 806, 313, 20, 96, 2172, 974, 134, 153, 38, 3518, 260, 1066, 493, 125, 1706, 624, 324, 1323, 664, 161, 27, 38, 1352, 127, 743, 1661, 774, 1007, 1308, 1307, 705, 220, 668, 22, 1462, 1262, 470, 464, 502, 531, 596, 821, 602] payload_rows_per_dst=[8202, 10602, 6434, 10948, 7920, 8968, 10692, 13020] dup_rows_per_dst=[1770, 3080, 502, 3742, 1318, 3722, 3194, 4190] combine_rows_per_src=[8202, 10602, 6434, 10948, 7920, 8968, 10692, 13020] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=261931008 dispatch_meta_bytes=344144 combine_payload_bytes=261931008 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=262275152 ingress_bytes=331137024 link_bytes=331137024 token_bytes=524206160 weight_movement_bytes=69206016 total_nvlink_bytes=593412176 weight_to_token_ratio=0.132021 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.032500 gemm_active_groups=11 gemm_peak_group_rows=17536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.246041963e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75442 payload_rows_remote=66604 dup_rows=22862 dup_rows_remote=21384 dedup_ratio=0.232564 router_tokens_per_expert=[888, 2161, 672, 156, 86, 367, 2814, 864, 0, 0, 856, 720, 2585, 0, 0, 997, 568, 908, 3946, 91, 428, 1502, 296, 0, 143, 2232, 1271, 138, 58, 2, 2424, 375, 1353, 360, 105, 1920, 367, 315, 1095, 634, 88, 0, 6, 1280, 97, 993, 1239, 727, 915, 1108, 1808, 1162, 133, 485, 0, 1391, 1447, 420, 96, 277, 353, 355, 617, 458] payload_rows_per_dst=[11972, 8838, 13052, 7506, 9734, 8216, 9288, 6836] dup_rows_per_dst=[6014, 1478, 5304, 932, 2564, 3426, 1934, 1210] combine_rows_per_src=[11972, 8838, 13052, 7506, 9734, 8216, 9288, 6836] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=274833408 dispatch_meta_bytes=343000 combine_payload_bytes=274833408 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=275176408 ingress_bytes=344039424 link_bytes=344039424 token_bytes=550009816 weight_movement_bytes=69206016 total_nvlink_bytes=619215832 weight_to_token_ratio=0.125827 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=11008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.294592655e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=274726912 dispatch_meta_bytes=346792 combine_payload_bytes=274726912 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=275073704 ingress_bytes=378535936 link_bytes=378535936 token_bytes=549800616 weight_movement_bytes=103809024 total_nvlink_bytes=653609640 weight_to_token_ratio=0.188812 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.424400253e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81462 payload_rows_remote=70716 dup_rows=16842 dup_rows_remote=12900 dedup_ratio=0.171326 router_tokens_per_expert=[617, 91, 0, 3715, 92, 0, 192, 76, 0, 589, 870, 581, 906, 366, 383, 658, 0, 0, 879, 1, 0, 657, 432, 1953, 140, 73, 311, 0, 143, 442, 0, 0, 76, 1291, 130, 1, 0, 0, 0, 4211, 1581, 193, 193, 2263, 5489, 231, 1936, 3540, 351, 122, 0, 1005, 229, 1031, 6075, 0, 4572, 0, 0, 6, 0, 0, 401, 58] payload_rows_per_dst=[11754, 10746, 11194, 12756, 12350, 7430, 5338, 9894] dup_rows_per_dst=[3980, 3942, 3730, 440, 3594, 838, 138, 180] combine_rows_per_src=[11754, 10746, 11194, 12756, 12350, 7430, 5338, 9894] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75774 payload_rows_remote=63422 dup_rows=22530 dup_rows_remote=15182 dedup_ratio=0.229187 router_tokens_per_expert=[516, 1797, 659, 277, 55, 383, 2357, 786, 1, 1, 871, 633, 2547, 0, 2, 793, 590, 691, 4011, 46, 386, 1862, 310, 3, 119, 1844, 1165, 128, 68, 33, 3020, 181, 1624, 369, 74, 2013, 511, 335, 1134, 572, 28, 1, 5, 828, 99, 858, 1690, 880, 1206, 999, 1624, 1371, 237, 539, 3, 1435, 1728, 463, 243, 303, 448, 357, 578, 462] payload_rows_per_dst=[12352, 8580, 12276, 7410, 9506, 8308, 9696, 7646] dup_rows_per_dst=[7348, 1116, 3522, 980, 2444, 3340, 2262, 1518] combine_rows_per_src=[12352, 8580, 12276, 7410, 9506, 8308, 9696, 7646] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79712 payload_rows_remote=68104 dup_rows=18592 dup_rows_remote=16398 dedup_ratio=0.189128 router_tokens_per_expert=[1083, 195, 47, 2738, 220, 65, 320, 228, 79, 773, 791, 739, 898, 781, 664, 790, 2, 39, 925, 113, 136, 772, 546, 1688, 126, 229, 1113, 39, 682, 611, 25, 74, 115, 1369, 354, 133, 85, 116, 78, 2303, 1543, 276, 315, 2309, 4002, 362, 1461, 2019, 386, 841, 153, 1332, 751, 801, 4960, 254, 3762, 165, 80, 199, 591, 2, 395, 109] payload_rows_per_dst=[8364, 11054, 9546, 11608, 9154, 8302, 11762, 9922] dup_rows_per_dst=[1428, 4196, 2934, 2194, 2290, 1892, 2974, 684] combine_rows_per_src=[8364, 11054, 9546, 11608, 9154, 8302, 11762, 9922] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=274931712 dispatch_meta_bytes=348552 combine_payload_bytes=274931712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275280264 ingress_bytes=309534720 link_bytes=309534720 token_bytes=550211976 weight_movement_bytes=34603008 total_nvlink_bytes=584814984 weight_to_token_ratio=0.062890 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=8960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.164754232e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=270721024 dispatch_meta_bytes=337672 combine_payload_bytes=270721024 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=271058696 ingress_bytes=339927040 link_bytes=339927040 token_bytes=541779720 weight_movement_bytes=69206016 total_nvlink_bytes=610985736 weight_to_token_ratio=0.127738 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.279118085e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80064 payload_rows_remote=69202 dup_rows=18240 dup_rows_remote=15400 dedup_ratio=0.185547 router_tokens_per_expert=[709, 78, 0, 3560, 62, 0, 226, 52, 0, 365, 1107, 725, 961, 977, 514, 987, 0, 0, 676, 1, 5, 463, 484, 2814, 118, 62, 1028, 0, 258, 489, 0, 0, 52, 1361, 221, 10, 0, 0, 0, 2679, 1365, 132, 184, 2403, 5270, 116, 1901, 2408, 403, 98, 0, 1496, 268, 576, 6187, 0, 4951, 0, 0, 3, 83, 0, 227, 37] payload_rows_per_dst=[8316, 15024, 10862, 12700, 10400, 6750, 5562, 10450] dup_rows_per_dst=[1058, 8622, 2840, 1750, 3052, 646, 120, 152] combine_rows_per_src=[8316, 15024, 10862, 12700, 10400, 6750, 5562, 10450] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72766 payload_rows_remote=58104 dup_rows=25538 dup_rows_remote=17468 dedup_ratio=0.259786 router_tokens_per_expert=[691, 94, 0, 4163, 87, 0, 177, 78, 0, 480, 619, 454, 896, 540, 412, 611, 0, 0, 589, 1, 0, 489, 418, 1767, 116, 59, 375, 0, 74, 458, 0, 0, 78, 1432, 108, 9, 0, 0, 0, 4412, 1375, 134, 171, 2278, 5612, 161, 1850, 3581, 390, 88, 0, 1297, 213, 949, 6238, 0, 4778, 0, 0, 2, 0, 0, 280, 68] payload_rows_per_dst=[9722, 6464, 5668, 1992, 10620, 14934, 8704, 14662] dup_rows_per_dst=[858, 1560, 860, 172, 1458, 11690, 870, 8070] combine_rows_per_src=[9722, 6464, 5668, 1992, 10620, 14934, 8704, 14662] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=289652736 dispatch_meta_bytes=334464 combine_payload_bytes=289652736 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289987200 ingress_bytes=324255744 link_bytes=324255744 token_bytes=579639936 weight_movement_bytes=34603008 total_nvlink_bytes=614242944 weight_to_token_ratio=0.059697 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.220148260e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76956 payload_rows_remote=67994 dup_rows=21348 dup_rows_remote=17474 dedup_ratio=0.217163 router_tokens_per_expert=[691, 1736, 1050, 190, 203, 544, 2451, 301, 90, 3, 642, 876, 1451, 0, 6, 1146, 923, 708, 3387, 129, 432, 701, 323, 19, 74, 1753, 815, 138, 144, 188, 3157, 240, 1357, 494, 188, 2030, 583, 345, 1551, 810, 199, 2, 79, 1546, 121, 686, 1528, 825, 1031, 1522, 1235, 622, 245, 631, 1, 1432, 1341, 505, 483, 636, 498, 608, 887, 620] payload_rows_per_dst=[7876, 10230, 6076, 10402, 10630, 8962, 9348, 13432] dup_rows_per_dst=[1554, 3100, 394, 2616, 3606, 3874, 1706, 4498] combine_rows_per_src=[7876, 10230, 6076, 10402, 10630, 8962, 9348, 13432] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=259776512 dispatch_meta_bytes=314416 combine_payload_bytes=259776512 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=260090928 ingress_bytes=328982528 link_bytes=328982528 token_bytes=519867440 weight_movement_bytes=69206016 total_nvlink_bytes=589073456 weight_to_token_ratio=0.133122 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=14976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.237934768e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278953984 dispatch_meta_bytes=338008 combine_payload_bytes=278953984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279291992 ingress_bytes=313556992 link_bytes=313556992 token_bytes=558245976 weight_movement_bytes=34603008 total_nvlink_bytes=592848984 weight_to_token_ratio=0.061985 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=35712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.179889717e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=261373952 dispatch_meta_bytes=328504 combine_payload_bytes=261373952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=261702456 ingress_bytes=295976960 link_bytes=295976960 token_bytes=523076408 weight_movement_bytes=34603008 total_nvlink_bytes=557679416 weight_to_token_ratio=0.066153 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.113737472e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=270721024 dispatch_meta_bytes=337672 combine_payload_bytes=270721024 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=271058696 ingress_bytes=339927040 link_bytes=339927040 token_bytes=541779720 weight_movement_bytes=69206016 total_nvlink_bytes=610985736 weight_to_token_ratio=0.127738 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.279118085e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=272809984 dispatch_meta_bytes=351952 combine_payload_bytes=272809984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273161936 ingress_bytes=307412992 link_bytes=307412992 token_bytes=545971920 weight_movement_bytes=34603008 total_nvlink_bytes=580574928 weight_to_token_ratio=0.063379 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.156770340e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278503424 dispatch_meta_bytes=341872 combine_payload_bytes=278503424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278845296 ingress_bytes=313106432 link_bytes=313106432 token_bytes=557348720 weight_movement_bytes=34603008 total_nvlink_bytes=591951728 weight_to_token_ratio=0.062085 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.178194296e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77018 payload_rows_remote=64878 dup_rows=21286 dup_rows_remote=16882 dedup_ratio=0.216532 router_tokens_per_expert=[555, 1758, 1027, 235, 169, 316, 2724, 300, 59, 0, 723, 857, 1483, 0, 0, 1174, 1068, 796, 3493, 119, 565, 723, 217, 0, 60, 1989, 846, 115, 175, 176, 3551, 331, 1257, 449, 131, 1822, 656, 312, 1449, 811, 319, 0, 84, 1500, 169, 678, 1546, 649, 982, 1323, 1195, 735, 217, 767, 0, 1317, 1331, 452, 445, 492, 525, 640, 739, 556] payload_rows_per_dst=[10088, 8232, 12140, 9692, 10480, 8588, 8978, 8820] dup_rows_per_dst=[2654, 1786, 4404, 2212, 3294, 3936, 1460, 1540] combine_rows_per_src=[10088, 8232, 12140, 9692, 10480, 8588, 8978, 8820] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83238 payload_rows_remote=73402 dup_rows=15066 dup_rows_remote=13572 dedup_ratio=0.153259 router_tokens_per_expert=[678, 99, 0, 4029, 92, 0, 192, 83, 0, 474, 667, 462, 855, 603, 400, 501, 0, 0, 722, 0, 1, 526, 406, 1939, 127, 67, 502, 0, 146, 376, 0, 0, 83, 1283, 140, 21, 0, 0, 0, 4138, 1470, 118, 185, 2328, 5421, 237, 1457, 3496, 360, 98, 0, 1880, 260, 970, 6036, 0, 4770, 0, 0, 0, 0, 0, 385, 69] payload_rows_per_dst=[9428, 6414, 10932, 12492, 9836, 9040, 14824, 10272] dup_rows_per_dst=[918, 1510, 3248, 786, 1494, 2550, 4384, 176] combine_rows_per_src=[9428, 6414, 10932, 12492, 9836, 9040, 14824, 10272] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=283451392 dispatch_meta_bytes=338408 combine_payload_bytes=283451392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=283789800 ingress_bytes=318054400 link_bytes=318054400 token_bytes=567241192 weight_movement_bytes=34603008 total_nvlink_bytes=601844200 weight_to_token_ratio=0.061002 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.196813102e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=237993984 dispatch_meta_bytes=302288 combine_payload_bytes=237993984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=238296272 ingress_bytes=272596992 link_bytes=272596992 token_bytes=476290256 weight_movement_bytes=34603008 total_nvlink_bytes=510893264 weight_to_token_ratio=0.072651 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=37376 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.025760535e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79712 payload_rows_remote=68104 dup_rows=18592 dup_rows_remote=16398 dedup_ratio=0.189128 router_tokens_per_expert=[1083, 195, 47, 2738, 220, 65, 320, 228, 79, 773, 791, 739, 898, 781, 664, 790, 2, 39, 925, 113, 136, 772, 546, 1688, 126, 229, 1113, 39, 682, 611, 25, 74, 115, 1369, 354, 133, 85, 116, 78, 2303, 1543, 276, 315, 2309, 4002, 362, 1461, 2019, 386, 841, 153, 1332, 751, 801, 4960, 254, 3762, 165, 80, 199, 591, 2, 395, 109] payload_rows_per_dst=[8364, 11054, 9546, 11608, 9154, 8302, 11762, 9922] dup_rows_per_dst=[1428, 4196, 2934, 2194, 2290, 1892, 2974, 684] combine_rows_per_src=[8364, 11054, 9546, 11608, 9154, 8302, 11762, 9922] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265740288 dispatch_meta_bytes=327040 combine_payload_bytes=265740288 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266067328 ingress_bytes=334946304 link_bytes=334946304 token_bytes=531807616 weight_movement_bytes=69206016 total_nvlink_bytes=601013632 weight_to_token_ratio=0.130134 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260375977e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80430 payload_rows_remote=65526 dup_rows=17874 dup_rows_remote=9788 dedup_ratio=0.181824 router_tokens_per_expert=[767, 115, 0, 3961, 92, 0, 188, 100, 0, 484, 515, 424, 878, 629, 396, 503, 0, 0, 757, 0, 1, 596, 440, 1398, 130, 53, 372, 2, 118, 421, 0, 2, 100, 1384, 90, 22, 0, 3, 2, 3973, 1307, 171, 184, 2540, 5440, 127, 2271, 3493, 337, 95, 1, 1283, 331, 1250, 6272, 1, 4650, 0, 0, 6, 4, 1, 395, 77] payload_rows_per_dst=[14904, 6142, 10402, 12534, 12496, 7442, 6424, 10086] dup_rows_per_dst=[8086, 1516, 2968, 542, 3732, 678, 172, 180] combine_rows_per_src=[14904, 6142, 10402, 12534, 12496, 7442, 6424, 10086] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72766 payload_rows_remote=58104 dup_rows=25538 dup_rows_remote=17468 dedup_ratio=0.259786 router_tokens_per_expert=[691, 94, 0, 4163, 87, 0, 177, 78, 0, 480, 619, 454, 896, 540, 412, 611, 0, 0, 589, 1, 0, 489, 418, 1767, 116, 59, 375, 0, 74, 458, 0, 0, 78, 1432, 108, 9, 0, 0, 0, 4412, 1375, 134, 171, 2278, 5612, 161, 1850, 3581, 390, 88, 0, 1297, 213, 949, 6238, 0, 4778, 0, 0, 2, 0, 0, 280, 68] payload_rows_per_dst=[9722, 6464, 5668, 1992, 10620, 14934, 8704, 14662] dup_rows_per_dst=[858, 1560, 860, 172, 1458, 11690, 870, 8070] combine_rows_per_src=[9722, 6464, 5668, 1992, 10620, 14934, 8704, 14662] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80588 payload_rows_remote=66406 dup_rows=17716 dup_rows_remote=12946 dedup_ratio=0.180216 router_tokens_per_expert=[907, 117, 3, 3507, 97, 8, 243, 89, 1, 492, 823, 606, 687, 980, 519, 830, 0, 1, 622, 17, 20, 528, 431, 2423, 122, 64, 1158, 2, 256, 505, 4, 0, 123, 1386, 255, 12, 3, 26, 24, 2886, 1481, 168, 197, 2233, 5106, 88, 1976, 2630, 393, 109, 13, 1553, 370, 727, 5904, 23, 4964, 7, 23, 28, 28, 3, 282, 69] payload_rows_per_dst=[8744, 7742, 8566, 12406, 7908, 9712, 14182, 11328] dup_rows_per_dst=[1198, 2134, 1592, 1492, 1522, 3464, 4770, 1544] combine_rows_per_src=[8744, 7742, 8566, 12406, 7908, 9712, 14182, 11328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81462 payload_rows_remote=70716 dup_rows=16842 dup_rows_remote=12900 dedup_ratio=0.171326 router_tokens_per_expert=[617, 91, 0, 3715, 92, 0, 192, 76, 0, 589, 870, 581, 906, 366, 383, 658, 0, 0, 879, 1, 0, 657, 432, 1953, 140, 73, 311, 0, 143, 442, 0, 0, 76, 1291, 130, 1, 0, 0, 0, 4211, 1581, 193, 193, 2263, 5489, 231, 1936, 3540, 351, 122, 0, 1005, 229, 1031, 6075, 0, 4572, 0, 0, 6, 0, 0, 401, 58] payload_rows_per_dst=[11754, 10746, 11194, 12756, 12350, 7430, 5338, 9894] dup_rows_per_dst=[3980, 3942, 3730, 440, 3594, 838, 138, 180] combine_rows_per_src=[11754, 10746, 11194, 12756, 12350, 7430, 5338, 9894] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=300654592 dispatch_meta_bytes=347896 combine_payload_bytes=300654592 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=301002488 ingress_bytes=335257600 link_bytes=335257600 token_bytes=601657080 weight_movement_bytes=34603008 total_nvlink_bytes=636260088 weight_to_token_ratio=0.057513 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=26880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261547358e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82630 payload_rows_remote=72744 dup_rows=15674 dup_rows_remote=12624 dedup_ratio=0.159444 router_tokens_per_expert=[910, 134, 1, 3261, 69, 6, 299, 93, 0, 474, 945, 685, 935, 1006, 493, 1081, 7, 2, 659, 27, 21, 508, 419, 2804, 135, 79, 977, 6, 367, 509, 0, 8, 76, 1565, 302, 95, 6, 0, 0, 2155, 1492, 221, 226, 2385, 5139, 108, 2036, 2034, 418, 194, 0, 1636, 319, 629, 5844, 0, 4867, 1, 1, 18, 93, 2, 306, 64] payload_rows_per_dst=[8410, 9040, 10462, 12680, 6922, 9886, 14778, 10452] dup_rows_per_dst=[1136, 2198, 2500, 1760, 1476, 3050, 3302, 252] combine_rows_per_src=[8410, 9040, 10462, 12680, 6922, 9886, 14778, 10452] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81156 payload_rows_remote=67336 dup_rows=17148 dup_rows_remote=14482 dedup_ratio=0.174438 router_tokens_per_expert=[3676, 6872, 0, 180, 542, 0, 58, 31, 0, 363, 58, 234, 139, 301, 563, 443, 373, 1140, 0, 0, 312, 85, 387, 447, 4232, 2731, 1254, 235, 7, 0, 64, 536, 0, 250, 636, 0, 92, 3669, 868, 132, 725, 73, 0, 73, 2758, 6142, 128, 0, 4271, 0, 277, 477, 126, 42, 133, 0, 34, 327, 0, 0, 0, 2120, 192, 344] payload_rows_per_dst=[15556, 13820, 11066, 7740, 10178, 7080, 9744, 5972] dup_rows_per_dst=[7842, 2666, 2886, 1234, 1116, 434, 908, 62] combine_rows_per_src=[15556, 13820, 11066, 7740, 10178, 7080, 9744, 5972] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80430 payload_rows_remote=65526 dup_rows=17874 dup_rows_remote=9788 dedup_ratio=0.181824 router_tokens_per_expert=[767, 115, 0, 3961, 92, 0, 188, 100, 0, 484, 515, 424, 878, 629, 396, 503, 0, 0, 757, 0, 1, 596, 440, 1398, 130, 53, 372, 2, 118, 421, 0, 2, 100, 1384, 90, 22, 0, 3, 2, 3973, 1307, 171, 184, 2540, 5440, 127, 2271, 3493, 337, 95, 1, 1283, 331, 1250, 6272, 1, 4650, 0, 0, 6, 4, 1, 395, 77] payload_rows_per_dst=[14904, 6142, 10402, 12534, 12496, 7442, 6424, 10086] dup_rows_per_dst=[8086, 1516, 2968, 542, 3732, 678, 172, 180] combine_rows_per_src=[14904, 6142, 10402, 12534, 12496, 7442, 6424, 10086] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79664 payload_rows_remote=70006 dup_rows=18640 dup_rows_remote=17042 dedup_ratio=0.189616 router_tokens_per_expert=[2358, 4029, 413, 382, 978, 143, 101, 334, 61, 582, 531, 981, 256, 460, 573, 949, 451, 1293, 338, 231, 558, 140, 769, 1046, 2181, 2401, 1606, 216, 370, 224, 262, 549, 183, 378, 1093, 38, 107, 2724, 1076, 86, 804, 52, 99, 153, 2444, 4611, 503, 172, 2891, 283, 683, 460, 618, 134, 362, 143, 410, 531, 243, 71, 41, 1129, 386, 478] payload_rows_per_dst=[9122, 13460, 10280, 9658, 9492, 7700, 9670, 10282] dup_rows_per_dst=[2768, 4548, 3734, 1598, 1878, 754, 1478, 1882] combine_rows_per_src=[9122, 13460, 10280, 9658, 9492, 7700, 9670, 10282] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74916 payload_rows_remote=66296 dup_rows=23388 dup_rows_remote=19526 dedup_ratio=0.237915 router_tokens_per_expert=[654, 983, 919, 601, 774, 353, 488, 918, 86, 898, 1304, 314, 670, 1000, 864, 1337, 912, 1186, 833, 214, 1512, 238, 1858, 1151, 678, 716, 728, 834, 164, 287, 198, 79, 462, 296, 520, 224, 931, 1545, 1117, 1140, 2000, 1578, 1196, 1377, 510, 1252, 30, 298, 776, 1208, 780, 15, 230, 1616, 208, 1217, 764, 384, 855, 528, 1229, 321, 338, 456] payload_rows_per_dst=[9094, 9778, 9516, 8476, 10254, 8620, 9946, 9232] dup_rows_per_dst=[2286, 3168, 2442, 2742, 2216, 3862, 2154, 4518] combine_rows_per_src=[9094, 9778, 9516, 8476, 10254, 8620, 9946, 9232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=271998976 dispatch_meta_bytes=317408 combine_payload_bytes=271998976 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=272316384 ingress_bytes=410411008 link_bytes=410411008 token_bytes=544315360 weight_movement_bytes=138412032 total_nvlink_bytes=682727392 weight_to_token_ratio=0.254286 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.544343582e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80828 payload_rows_remote=71014 dup_rows=17476 dup_rows_remote=14456 dedup_ratio=0.177775 router_tokens_per_expert=[3027, 5771, 0, 270, 569, 99, 37, 163, 0, 376, 147, 249, 60, 306, 486, 997, 367, 651, 0, 0, 397, 65, 2281, 798, 1858, 3717, 1181, 340, 126, 0, 102, 436, 0, 221, 1562, 0, 62, 1625, 659, 105, 684, 96, 0, 59, 3145, 6674, 357, 0, 5179, 0, 648, 394, 459, 72, 421, 58, 104, 409, 0, 0, 0, 949, 92, 242] payload_rows_per_dst=[14158, 14900, 9814, 9898, 7410, 8090, 12978, 3580] dup_rows_per_dst=[5714, 3690, 3020, 1906, 1058, 592, 1484, 12] combine_rows_per_src=[14158, 14900, 9814, 9898, 7410, 8090, 12978, 3580] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75616 payload_rows_remote=59982 dup_rows=22688 dup_rows_remote=19018 dedup_ratio=0.230794 router_tokens_per_expert=[3895, 6979, 0, 166, 533, 3, 68, 19, 0, 231, 70, 224, 72, 329, 541, 581, 409, 1293, 0, 0, 248, 90, 681, 738, 4364, 2907, 1707, 297, 2, 0, 117, 475, 0, 293, 664, 0, 87, 2697, 875, 32, 623, 37, 0, 59, 2032, 6328, 60, 0, 4278, 0, 367, 485, 243, 83, 157, 40, 46, 381, 0, 0, 0, 1796, 164, 286] payload_rows_per_dst=[1578, 3386, 5580, 13220, 12756, 12820, 10642, 15634] dup_rows_per_dst=[0, 710, 1338, 6518, 4330, 4120, 2002, 3670] combine_rows_per_src=[1578, 3386, 5580, 13220, 12756, 12820, 10642, 15634] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=275808256 dispatch_meta_bytes=327272 combine_payload_bytes=275808256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276135528 ingress_bytes=310411264 link_bytes=310411264 token_bytes=551943784 weight_movement_bytes=34603008 total_nvlink_bytes=586546792 weight_to_token_ratio=0.062693 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=28032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.168052596e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=268394496 dispatch_meta_bytes=301256 combine_payload_bytes=268394496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268695752 ingress_bytes=302997504 link_bytes=302997504 token_bytes=537090248 weight_movement_bytes=34603008 total_nvlink_bytes=571693256 weight_to_token_ratio=0.064427 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=29056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.140155214e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=286744576 dispatch_meta_bytes=348192 combine_payload_bytes=286744576 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=287092768 ingress_bytes=390553600 link_bytes=390553600 token_bytes=573837344 weight_movement_bytes=103809024 total_nvlink_bytes=677646368 weight_to_token_ratio=0.180903 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=21504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.469621755e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278953984 dispatch_meta_bytes=338008 combine_payload_bytes=278953984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279291992 ingress_bytes=313556992 link_bytes=313556992 token_bytes=558245976 weight_movement_bytes=34603008 total_nvlink_bytes=592848984 weight_to_token_ratio=0.061985 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=35712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.179889717e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=237993984 dispatch_meta_bytes=302288 combine_payload_bytes=237993984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=238296272 ingress_bytes=272596992 link_bytes=272596992 token_bytes=476290256 weight_movement_bytes=34603008 total_nvlink_bytes=510893264 weight_to_token_ratio=0.072651 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=37376 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.025760535e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=271998976 dispatch_meta_bytes=317408 combine_payload_bytes=271998976 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=272316384 ingress_bytes=410411008 link_bytes=410411008 token_bytes=544315360 weight_movement_bytes=138412032 total_nvlink_bytes=682727392 weight_to_token_ratio=0.254286 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.544343582e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=289652736 dispatch_meta_bytes=334464 combine_payload_bytes=289652736 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289987200 ingress_bytes=324255744 link_bytes=324255744 token_bytes=579639936 weight_movement_bytes=34603008 total_nvlink_bytes=614242944 weight_to_token_ratio=0.059697 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.220148260e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=297959424 dispatch_meta_bytes=341472 combine_payload_bytes=297959424 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=298300896 ingress_bytes=436371456 link_bytes=436371456 token_bytes=596260320 weight_movement_bytes=138412032 total_nvlink_bytes=734672352 weight_to_token_ratio=0.232134 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=11648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.642030658e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80064 payload_rows_remote=69202 dup_rows=18240 dup_rows_remote=15400 dedup_ratio=0.185547 router_tokens_per_expert=[709, 78, 0, 3560, 62, 0, 226, 52, 0, 365, 1107, 725, 961, 977, 514, 987, 0, 0, 676, 1, 5, 463, 484, 2814, 118, 62, 1028, 0, 258, 489, 0, 0, 52, 1361, 221, 10, 0, 0, 0, 2679, 1365, 132, 184, 2403, 5270, 116, 1901, 2408, 403, 98, 0, 1496, 268, 576, 6187, 0, 4951, 0, 0, 3, 83, 0, 227, 37] payload_rows_per_dst=[8316, 15024, 10862, 12700, 10400, 6750, 5562, 10450] dup_rows_per_dst=[1058, 8622, 2840, 1750, 3052, 646, 120, 152] combine_rows_per_src=[8316, 15024, 10862, 12700, 10400, 6750, 5562, 10450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81622 payload_rows_remote=72796 dup_rows=16682 dup_rows_remote=15220 dedup_ratio=0.169698 router_tokens_per_expert=[3712, 6921, 0, 206, 554, 8, 69, 24, 0, 203, 80, 228, 142, 335, 461, 620, 425, 1065, 0, 0, 275, 95, 898, 675, 3418, 3267, 1457, 314, 0, 0, 139, 489, 0, 349, 608, 1, 92, 2347, 880, 57, 696, 80, 0, 67, 2322, 6499, 79, 0, 4401, 0, 460, 526, 435, 97, 281, 58, 57, 303, 0, 0, 0, 1785, 191, 401] payload_rows_per_dst=[7058, 10546, 10484, 9994, 8826, 8016, 11332, 15366] dup_rows_per_dst=[468, 1974, 3218, 2546, 1462, 1880, 1184, 3950] combine_rows_per_src=[7058, 10546, 10484, 9994, 8826, 8016, 11332, 15366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=290873344 dispatch_meta_bytes=341880 combine_payload_bytes=290873344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291215224 ingress_bytes=325476352 link_bytes=325476352 token_bytes=582088568 weight_movement_bytes=34603008 total_nvlink_bytes=616691576 weight_to_token_ratio=0.059446 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.224741310e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=245686272 dispatch_meta_bytes=316000 combine_payload_bytes=245686272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=246002272 ingress_bytes=280289280 link_bytes=280289280 token_bytes=491688544 weight_movement_bytes=34603008 total_nvlink_bytes=526291552 weight_to_token_ratio=0.070376 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=28032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.054705996e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=271548416 dispatch_meta_bytes=343288 combine_payload_bytes=271548416 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=271891704 ingress_bytes=409960448 link_bytes=409960448 token_bytes=543440120 weight_movement_bytes=138412032 total_nvlink_bytes=681852152 weight_to_token_ratio=0.254696 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=10240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.542648161e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=283451392 dispatch_meta_bytes=338408 combine_payload_bytes=283451392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=283789800 ingress_bytes=318054400 link_bytes=318054400 token_bytes=567241192 weight_movement_bytes=34603008 total_nvlink_bytes=601844200 weight_to_token_ratio=0.061002 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.196813102e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76778 payload_rows_remote=60796 dup_rows=21526 dup_rows_remote=8510 dedup_ratio=0.218974 router_tokens_per_expert=[4211, 7027, 5, 139, 568, 0, 77, 5, 0, 203, 78, 220, 121, 347, 626, 525, 437, 1285, 4, 0, 201, 107, 520, 652, 4424, 2467, 1810, 231, 4, 0, 130, 445, 0, 398, 441, 0, 92, 3068, 965, 22, 687, 22, 1, 54, 1675, 6341, 91, 0, 4203, 4, 312, 525, 154, 84, 33, 4, 136, 302, 2, 0, 0, 2008, 359, 300] payload_rows_per_dst=[15982, 14164, 12024, 5090, 8864, 4774, 9754, 6126] dup_rows_per_dst=[13016, 2758, 3236, 150, 1108, 286, 884, 88] combine_rows_per_src=[15982, 14164, 12024, 5090, 8864, 4774, 9754, 6126] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=298172416 dispatch_meta_bytes=352064 combine_payload_bytes=298172416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=298524480 ingress_bytes=332775424 link_bytes=332775424 token_bytes=596696896 weight_movement_bytes=34603008 total_nvlink_bytes=631299904 weight_to_token_ratio=0.057991 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.252207130e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79664 payload_rows_remote=70006 dup_rows=18640 dup_rows_remote=17042 dedup_ratio=0.189616 router_tokens_per_expert=[2358, 4029, 413, 382, 978, 143, 101, 334, 61, 582, 531, 981, 256, 460, 573, 949, 451, 1293, 338, 231, 558, 140, 769, 1046, 2181, 2401, 1606, 216, 370, 224, 262, 549, 183, 378, 1093, 38, 107, 2724, 1076, 86, 804, 52, 99, 153, 2444, 4611, 503, 172, 2891, 283, 683, 460, 618, 134, 362, 143, 410, 531, 243, 71, 41, 1129, 386, 478] payload_rows_per_dst=[9122, 13460, 10280, 9658, 9492, 7700, 9670, 10282] dup_rows_per_dst=[2768, 4548, 3734, 1598, 1878, 754, 1478, 1882] combine_rows_per_src=[9122, 13460, 10280, 9658, 9492, 7700, 9670, 10282] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75616 payload_rows_remote=59982 dup_rows=22688 dup_rows_remote=19018 dedup_ratio=0.230794 router_tokens_per_expert=[3895, 6979, 0, 166, 533, 3, 68, 19, 0, 231, 70, 224, 72, 329, 541, 581, 409, 1293, 0, 0, 248, 90, 681, 738, 4364, 2907, 1707, 297, 2, 0, 117, 475, 0, 293, 664, 0, 87, 2697, 875, 32, 623, 37, 0, 59, 2032, 6328, 60, 0, 4278, 0, 367, 485, 243, 83, 157, 40, 46, 381, 0, 0, 0, 1796, 164, 286] payload_rows_per_dst=[1578, 3386, 5580, 13220, 12756, 12820, 10642, 15634] dup_rows_per_dst=[0, 710, 1338, 6518, 4330, 4120, 2002, 3670] combine_rows_per_src=[1578, 3386, 5580, 13220, 12756, 12820, 10642, 15634] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77944 payload_rows_remote=66412 dup_rows=20360 dup_rows_remote=18944 dedup_ratio=0.207113 router_tokens_per_expert=[2902, 6218, 62, 253, 613, 67, 75, 84, 10, 412, 186, 366, 89, 353, 463, 1066, 424, 969, 45, 6, 446, 104, 1643, 972, 2452, 3136, 1378, 309, 45, 22, 122, 423, 7, 325, 1212, 2, 79, 2002, 822, 54, 660, 104, 8, 70, 2692, 6161, 355, 31, 4279, 43, 721, 457, 477, 89, 345, 63, 192, 399, 20, 9, 15, 1218, 185, 341] payload_rows_per_dst=[2254, 5130, 7556, 11388, 11758, 13866, 11532, 14460] dup_rows_per_dst=[54, 760, 1662, 4386, 3052, 6296, 1416, 2734] combine_rows_per_src=[2254, 5130, 7556, 11388, 11758, 13866, 11532, 14460] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81722 payload_rows_remote=67680 dup_rows=16582 dup_rows_remote=12950 dedup_ratio=0.168681 router_tokens_per_expert=[589, 330, 3014, 3949, 645, 0, 5984, 2804, 0, 184, 0, 463, 0, 2209, 0, 120, 307, 339, 1654, 102, 884, 60, 352, 134, 58, 92, 1064, 0, 0, 150, 0, 738, 5861, 1524, 0, 92, 405, 3221, 674, 0, 395, 0, 17, 0, 0, 0, 0, 390, 1867, 0, 0, 216, 0, 1048, 292, 0, 718, 0, 120, 3234, 2804, 49, 0, 0] payload_rows_per_dst=[11922, 14042, 6540, 10680, 9116, 12376, 6310, 10736] dup_rows_per_dst=[3552, 3632, 1124, 1422, 2716, 486, 536, 3114] combine_rows_per_src=[11922, 14042, 6540, 10680, 9116, 12376, 6310, 10736] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79406 payload_rows_remote=70194 dup_rows=18898 dup_rows_remote=17144 dedup_ratio=0.192240 router_tokens_per_expert=[547, 586, 3404, 2884, 496, 241, 4810, 2100, 166, 226, 109, 505, 521, 1671, 116, 520, 292, 391, 1238, 133, 833, 58, 754, 293, 79, 240, 1216, 82, 46, 206, 367, 363, 4507, 1369, 88, 227, 230, 1935, 711, 148, 480, 66, 709, 248, 109, 432, 644, 528, 1073, 32, 146, 204, 731, 1407, 598, 382, 902, 455, 101, 2378, 2237, 26, 475, 81] payload_rows_per_dst=[6778, 13172, 11390, 9212, 7280, 13380, 7626, 10568] dup_rows_per_dst=[1306, 3510, 3258, 1754, 2136, 2672, 1520, 2742] combine_rows_per_src=[6778, 13172, 11390, 9212, 7280, 13380, 7626, 10568] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76778 payload_rows_remote=60796 dup_rows=21526 dup_rows_remote=8510 dedup_ratio=0.218974 router_tokens_per_expert=[4211, 7027, 5, 139, 568, 0, 77, 5, 0, 203, 78, 220, 121, 347, 626, 525, 437, 1285, 4, 0, 201, 107, 520, 652, 4424, 2467, 1810, 231, 4, 0, 130, 445, 0, 398, 441, 0, 92, 3068, 965, 22, 687, 22, 1, 54, 1675, 6341, 91, 0, 4203, 4, 312, 525, 154, 84, 33, 4, 136, 302, 2, 0, 0, 2008, 359, 300] payload_rows_per_dst=[15982, 14164, 12024, 5090, 8864, 4774, 9754, 6126] dup_rows_per_dst=[13016, 2758, 3236, 150, 1108, 286, 884, 88] combine_rows_per_src=[15982, 14164, 12024, 5090, 8864, 4774, 9754, 6126] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81156 payload_rows_remote=67336 dup_rows=17148 dup_rows_remote=14482 dedup_ratio=0.174438 router_tokens_per_expert=[3676, 6872, 0, 180, 542, 0, 58, 31, 0, 363, 58, 234, 139, 301, 563, 443, 373, 1140, 0, 0, 312, 85, 387, 447, 4232, 2731, 1254, 235, 7, 0, 64, 536, 0, 250, 636, 0, 92, 3669, 868, 132, 725, 73, 0, 73, 2758, 6142, 128, 0, 4271, 0, 277, 477, 126, 42, 133, 0, 34, 327, 0, 0, 0, 2120, 192, 344] payload_rows_per_dst=[15556, 13820, 11066, 7740, 10178, 7080, 9744, 5972] dup_rows_per_dst=[7842, 2666, 2886, 1234, 1116, 434, 908, 62] combine_rows_per_src=[15556, 13820, 11066, 7740, 10178, 7080, 9744, 5972] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=272023552 dispatch_meta_bytes=341424 combine_payload_bytes=272023552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272364976 ingress_bytes=306626560 link_bytes=306626560 token_bytes=544388528 weight_movement_bytes=34603008 total_nvlink_bytes=578991536 weight_to_token_ratio=0.063563 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=34048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.153811060e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80660 payload_rows_remote=73000 dup_rows=17644 dup_rows_remote=16552 dedup_ratio=0.179484 router_tokens_per_expert=[548, 523, 3728, 3931, 1358, 0, 5942, 4296, 0, 124, 0, 377, 0, 1699, 0, 101, 314, 420, 2160, 93, 835, 52, 400, 102, 22, 62, 1299, 0, 0, 99, 0, 601, 5929, 1126, 0, 62, 187, 1489, 601, 0, 368, 0, 37, 0, 0, 0, 0, 376, 2148, 0, 12, 558, 2, 1508, 165, 0, 1554, 146, 70, 2294, 1394, 40, 0, 0] payload_rows_per_dst=[13948, 13900, 7660, 10760, 4766, 12870, 7674, 9082] dup_rows_per_dst=[6958, 2560, 1092, 1268, 2164, 576, 1112, 1914] combine_rows_per_src=[13948, 13900, 7660, 10760, 4766, 12870, 7674, 9082] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=277217280 dispatch_meta_bytes=322520 combine_payload_bytes=277217280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277539800 ingress_bytes=311820288 link_bytes=311820288 token_bytes=554757080 weight_movement_bytes=34603008 total_nvlink_bytes=589360088 weight_to_token_ratio=0.062375 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=25472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.173354640e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78126 payload_rows_remote=67888 dup_rows=20178 dup_rows_remote=17394 dedup_ratio=0.205261 router_tokens_per_expert=[576, 331, 2962, 3534, 968, 0, 5920, 2782, 0, 172, 0, 413, 0, 2239, 0, 169, 317, 323, 1690, 102, 769, 44, 387, 135, 69, 86, 1009, 0, 0, 155, 0, 666, 5979, 1307, 0, 87, 447, 3430, 578, 0, 415, 0, 19, 0, 0, 0, 0, 408, 1992, 0, 1, 229, 0, 1407, 524, 0, 940, 0, 37, 3049, 2482, 3, 0, 0] payload_rows_per_dst=[3212, 5614, 10274, 9814, 11720, 12996, 14258, 10238] dup_rows_per_dst=[538, 372, 3184, 1224, 5542, 528, 6006, 2784] combine_rows_per_src=[3212, 5614, 10274, 9814, 11720, 12996, 14258, 10238] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=287514624 dispatch_meta_bytes=349352 combine_payload_bytes=287514624 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=287863976 ingress_bytes=356720640 link_bytes=356720640 token_bytes=575378600 weight_movement_bytes=69206016 total_nvlink_bytes=644584616 weight_to_token_ratio=0.120279 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=29056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.342311050e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=298172416 dispatch_meta_bytes=352064 combine_payload_bytes=298172416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=298524480 ingress_bytes=332775424 link_bytes=332775424 token_bytes=596696896 weight_movement_bytes=34603008 total_nvlink_bytes=631299904 weight_to_token_ratio=0.057991 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.252207130e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78146 payload_rows_remote=63902 dup_rows=20158 dup_rows_remote=12806 dedup_ratio=0.205058 router_tokens_per_expert=[2560, 5078, 14, 276, 703, 104, 64, 206, 5, 459, 178, 390, 60, 325, 410, 1069, 390, 726, 1, 1, 492, 114, 1806, 924, 2032, 3630, 1345, 343, 256, 28, 105, 499, 0, 265, 1541, 1, 67, 2217, 748, 109, 653, 81, 1, 89, 3141, 6399, 433, 1, 4504, 2, 666, 482, 482, 85, 341, 86, 205, 421, 1, 0, 0, 1120, 120, 298] payload_rows_per_dst=[2658, 5042, 9462, 10536, 11776, 14244, 11872, 12556] dup_rows_per_dst=[76, 750, 2944, 2442, 3240, 7352, 1424, 1930] combine_rows_per_src=[2658, 5042, 9462, 10536, 11776, 14244, 11872, 12556] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=249020416 dispatch_meta_bytes=277224 combine_payload_bytes=249020416 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=249297640 ingress_bytes=352829440 link_bytes=352829440 token_bytes=498318056 weight_movement_bytes=103809024 total_nvlink_bytes=602127080 weight_to_token_ratio=0.208319 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20992 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.327668778e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76298 payload_rows_remote=67686 dup_rows=22006 dup_rows_remote=18492 dedup_ratio=0.223857 router_tokens_per_expert=[231, 875, 378, 414, 130, 875, 712, 292, 628, 358, 1160, 1166, 1229, 1490, 813, 1902, 973, 1322, 288, 764, 90, 406, 1143, 870, 412, 917, 797, 539, 261, 643, 996, 388, 369, 985, 827, 813, 1543, 1835, 1780, 499, 599, 166, 380, 276, 1721, 1501, 154, 1266, 1288, 622, 1049, 895, 116, 380, 265, 727, 1012, 391, 130, 535, 49, 1363, 486, 1668] payload_rows_per_dst=[8466, 9088, 9654, 9442, 10310, 8612, 10870, 9856] dup_rows_per_dst=[2434, 2142, 2058, 2922, 3906, 3514, 3618, 1412] combine_rows_per_src=[8466, 9088, 9654, 9442, 10310, 8612, 10870, 9856] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=286744576 dispatch_meta_bytes=348192 combine_payload_bytes=286744576 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=287092768 ingress_bytes=390553600 link_bytes=390553600 token_bytes=573837344 weight_movement_bytes=103809024 total_nvlink_bytes=677646368 weight_to_token_ratio=0.180903 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=21504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.469621755e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=245686272 dispatch_meta_bytes=316000 combine_payload_bytes=245686272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=246002272 ingress_bytes=280289280 link_bytes=280289280 token_bytes=491688544 weight_movement_bytes=34603008 total_nvlink_bytes=526291552 weight_to_token_ratio=0.070376 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=28032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.054705996e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=272023552 dispatch_meta_bytes=341424 combine_payload_bytes=272023552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272364976 ingress_bytes=306626560 link_bytes=306626560 token_bytes=544388528 weight_movement_bytes=34603008 total_nvlink_bytes=578991536 weight_to_token_ratio=0.063563 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=34048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.153811060e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=275808256 dispatch_meta_bytes=327272 combine_payload_bytes=275808256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276135528 ingress_bytes=310411264 link_bytes=310411264 token_bytes=551943784 weight_movement_bytes=34603008 total_nvlink_bytes=586546792 weight_to_token_ratio=0.062693 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=28032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.168052596e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=261742592 dispatch_meta_bytes=306832 combine_payload_bytes=261742592 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=262049424 ingress_bytes=365551616 link_bytes=365551616 token_bytes=523792016 weight_movement_bytes=103809024 total_nvlink_bytes=627601040 weight_to_token_ratio=0.198187 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=20224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.375541302e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80828 payload_rows_remote=71014 dup_rows=17476 dup_rows_remote=14456 dedup_ratio=0.177775 router_tokens_per_expert=[3027, 5771, 0, 270, 569, 99, 37, 163, 0, 376, 147, 249, 60, 306, 486, 997, 367, 651, 0, 0, 397, 65, 2281, 798, 1858, 3717, 1181, 340, 126, 0, 102, 436, 0, 221, 1562, 0, 62, 1625, 659, 105, 684, 96, 0, 59, 3145, 6674, 357, 0, 5179, 0, 648, 394, 459, 72, 421, 58, 104, 409, 0, 0, 0, 949, 92, 242] payload_rows_per_dst=[14158, 14900, 9814, 9898, 7410, 8090, 12978, 3580] dup_rows_per_dst=[5714, 3690, 3020, 1906, 1058, 592, 1484, 12] combine_rows_per_src=[14158, 14900, 9814, 9898, 7410, 8090, 12978, 3580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80074 payload_rows_remote=69490 dup_rows=18230 dup_rows_remote=13958 dedup_ratio=0.185445 router_tokens_per_expert=[579, 459, 3259, 3383, 1208, 0, 5564, 2833, 0, 184, 0, 497, 0, 2281, 0, 172, 322, 290, 1847, 131, 890, 58, 361, 145, 69, 92, 1082, 0, 0, 161, 0, 568, 5923, 1278, 0, 92, 355, 2531, 599, 0, 427, 0, 11, 0, 0, 0, 0, 493, 2020, 0, 1, 340, 0, 1500, 673, 0, 1140, 0, 41, 3054, 2233, 6, 0, 0] payload_rows_per_dst=[7928, 10476, 11134, 9412, 10584, 12468, 7846, 10226] dup_rows_per_dst=[2230, 2492, 3472, 1298, 4272, 522, 1222, 2722] combine_rows_per_src=[7928, 10476, 11134, 9412, 10584, 12468, 7846, 10226] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=299008000 dispatch_meta_bytes=358208 combine_payload_bytes=299008000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=299366208 ingress_bytes=333611008 link_bytes=333611008 token_bytes=598374208 weight_movement_bytes=34603008 total_nvlink_bytes=632977216 weight_to_token_ratio=0.057828 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=16768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.255351365e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278069248 dispatch_meta_bytes=341128 combine_payload_bytes=278069248 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=278410376 ingress_bytes=312672256 link_bytes=312672256 token_bytes=556479624 weight_movement_bytes=34603008 total_nvlink_bytes=591082632 weight_to_token_ratio=0.062182 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=20480 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.176560527e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=290873344 dispatch_meta_bytes=341880 combine_payload_bytes=290873344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291215224 ingress_bytes=325476352 link_bytes=325476352 token_bytes=582088568 weight_movement_bytes=34603008 total_nvlink_bytes=616691576 weight_to_token_ratio=0.059446 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.224741310e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72268 payload_rows_remote=57110 dup_rows=26036 dup_rows_remote=12372 dedup_ratio=0.264852 router_tokens_per_expert=[573, 339, 2806, 3504, 864, 1, 6039, 2116, 3, 182, 1, 459, 3, 2344, 0, 142, 307, 348, 1752, 90, 815, 29, 338, 153, 77, 90, 974, 0, 0, 169, 1, 763, 5825, 1323, 1, 92, 420, 3483, 642, 0, 438, 0, 12, 3, 1, 2, 3, 427, 1704, 1, 0, 121, 2, 1533, 563, 3, 938, 2, 31, 3303, 2968, 25, 4, 0] payload_rows_per_dst=[15158, 14048, 6506, 9826, 9388, 1270, 7090, 8982] dup_rows_per_dst=[13664, 3870, 1158, 1330, 2534, 502, 764, 2214] combine_rows_per_src=[15158, 14048, 6506, 9826, 9388, 1270, 7090, 8982] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78126 payload_rows_remote=67888 dup_rows=20178 dup_rows_remote=17394 dedup_ratio=0.205261 router_tokens_per_expert=[576, 331, 2962, 3534, 968, 0, 5920, 2782, 0, 172, 0, 413, 0, 2239, 0, 169, 317, 323, 1690, 102, 769, 44, 387, 135, 69, 86, 1009, 0, 0, 155, 0, 666, 5979, 1307, 0, 87, 447, 3430, 578, 0, 415, 0, 19, 0, 0, 0, 0, 408, 1992, 0, 1, 229, 0, 1407, 524, 0, 940, 0, 37, 3049, 2482, 3, 0, 0] payload_rows_per_dst=[3212, 5614, 10274, 9814, 11720, 12996, 14258, 10238] dup_rows_per_dst=[538, 372, 3184, 1224, 5542, 528, 6006, 2784] combine_rows_per_src=[3212, 5614, 10274, 9814, 11720, 12996, 14258, 10238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79406 payload_rows_remote=70194 dup_rows=18898 dup_rows_remote=17144 dedup_ratio=0.192240 router_tokens_per_expert=[547, 586, 3404, 2884, 496, 241, 4810, 2100, 166, 226, 109, 505, 521, 1671, 116, 520, 292, 391, 1238, 133, 833, 58, 754, 293, 79, 240, 1216, 82, 46, 206, 367, 363, 4507, 1369, 88, 227, 230, 1935, 711, 148, 480, 66, 709, 248, 109, 432, 644, 528, 1073, 32, 146, 204, 731, 1407, 598, 382, 902, 455, 101, 2378, 2237, 26, 475, 81] payload_rows_per_dst=[6778, 13172, 11390, 9212, 7280, 13380, 7626, 10568] dup_rows_per_dst=[1306, 3510, 3258, 1754, 2136, 2672, 1520, 2742] combine_rows_per_src=[6778, 13172, 11390, 9212, 7280, 13380, 7626, 10568] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=284631040 dispatch_meta_bytes=333792 combine_payload_bytes=284631040 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=284964832 ingress_bytes=388440064 link_bytes=388440064 token_bytes=569595872 weight_movement_bytes=103809024 total_nvlink_bytes=673404896 weight_to_token_ratio=0.182250 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=19840 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.461668689e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277241856 dispatch_meta_bytes=344712 combine_payload_bytes=277241856 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=277586568 ingress_bytes=381050880 link_bytes=381050880 token_bytes=554828424 weight_movement_bytes=103809024 total_nvlink_bytes=658637448 weight_to_token_ratio=0.187101 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.433863784e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78512 payload_rows_remote=65092 dup_rows=19792 dup_rows_remote=14172 dedup_ratio=0.201335 router_tokens_per_expert=[592, 551, 3661, 3858, 1323, 15, 5735, 3668, 14, 158, 30, 426, 51, 1675, 4, 230, 328, 344, 1926, 119, 894, 55, 458, 145, 69, 96, 1221, 3, 1, 155, 53, 576, 5685, 1168, 3, 82, 205, 2170, 557, 24, 425, 7, 111, 30, 0, 41, 48, 427, 1695, 7, 21, 470, 25, 1328, 259, 30, 1473, 224, 97, 2275, 1787, 11, 24, 9] payload_rows_per_dst=[4360, 4736, 12106, 10560, 10962, 12904, 13420, 9464] dup_rows_per_dst=[602, 440, 3754, 1504, 4792, 744, 5620, 2336] combine_rows_per_src=[4360, 4736, 12106, 10560, 10962, 12904, 13420, 9464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 2, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81722 payload_rows_remote=67680 dup_rows=16582 dup_rows_remote=12950 dedup_ratio=0.168681 router_tokens_per_expert=[589, 330, 3014, 3949, 645, 0, 5984, 2804, 0, 184, 0, 463, 0, 2209, 0, 120, 307, 339, 1654, 102, 884, 60, 352, 134, 58, 92, 1064, 0, 0, 150, 0, 738, 5861, 1524, 0, 92, 405, 3221, 674, 0, 395, 0, 17, 0, 0, 0, 0, 390, 1867, 0, 0, 216, 0, 1048, 292, 0, 718, 0, 120, 3234, 2804, 49, 0, 0] payload_rows_per_dst=[11922, 14042, 6540, 10680, 9116, 12376, 6310, 10736] dup_rows_per_dst=[3552, 3632, 1124, 1422, 2716, 486, 536, 3114] combine_rows_per_src=[11922, 14042, 6540, 10680, 9116, 12376, 6310, 10736] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77790 payload_rows_remote=64456 dup_rows=20514 dup_rows_remote=19838 dedup_ratio=0.208679 router_tokens_per_expert=[577, 507, 3518, 3903, 1201, 0, 6037, 4050, 5, 134, 0, 381, 1, 2018, 0, 181, 323, 343, 1883, 121, 967, 54, 468, 149, 62, 67, 1443, 0, 4, 130, 1, 542, 5729, 1065, 0, 67, 202, 1474, 542, 1, 394, 0, 81, 0, 12, 1, 100, 380, 2005, 1, 11, 332, 12, 1908, 174, 0, 1497, 30, 72, 2456, 1505, 27, 4, 0] payload_rows_per_dst=[5054, 5136, 11808, 13264, 7776, 13334, 12514, 8904] dup_rows_per_dst=[1094, 304, 3844, 4976, 2556, 676, 4786, 2278] combine_rows_per_src=[5054, 5136, 11808, 13264, 7776, 13334, 12514, 8904] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=233922560 dispatch_meta_bytes=277928 combine_payload_bytes=233922560 prefetch_bytes=34603008 reduce_bytes=173015040 egress_bytes=234200488 ingress_bytes=441540608 link_bytes=441540608 token_bytes=468123048 weight_movement_bytes=207618048 total_nvlink_bytes=675741096 weight_to_token_ratio=0.443512 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=14976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.661481761e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278069248 dispatch_meta_bytes=341128 combine_payload_bytes=278069248 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=278410376 ingress_bytes=312672256 link_bytes=312672256 token_bytes=556479624 weight_movement_bytes=34603008 total_nvlink_bytes=591082632 weight_to_token_ratio=0.062182 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=20480 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.176560527e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=287514624 dispatch_meta_bytes=349352 combine_payload_bytes=287514624 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=287863976 ingress_bytes=356720640 link_bytes=356720640 token_bytes=575378600 weight_movement_bytes=69206016 total_nvlink_bytes=644584616 weight_to_token_ratio=0.120279 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=29056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.342311050e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266616832 dispatch_meta_bytes=317056 combine_payload_bytes=266616832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266933888 ingress_bytes=301219840 link_bytes=301219840 token_bytes=533550720 weight_movement_bytes=34603008 total_nvlink_bytes=568153728 weight_to_token_ratio=0.064854 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133466008e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=277217280 dispatch_meta_bytes=322520 combine_payload_bytes=277217280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277539800 ingress_bytes=311820288 link_bytes=311820288 token_bytes=554757080 weight_movement_bytes=34603008 total_nvlink_bytes=589360088 weight_to_token_ratio=0.062375 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=25472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.173354640e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=264011776 dispatch_meta_bytes=337176 combine_payload_bytes=264011776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264348952 ingress_bytes=298614784 link_bytes=298614784 token_bytes=528360728 weight_movement_bytes=34603008 total_nvlink_bytes=562963736 weight_to_token_ratio=0.065491 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=39680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123663392e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80660 payload_rows_remote=73000 dup_rows=17644 dup_rows_remote=16552 dedup_ratio=0.179484 router_tokens_per_expert=[548, 523, 3728, 3931, 1358, 0, 5942, 4296, 0, 124, 0, 377, 0, 1699, 0, 101, 314, 420, 2160, 93, 835, 52, 400, 102, 22, 62, 1299, 0, 0, 99, 0, 601, 5929, 1126, 0, 62, 187, 1489, 601, 0, 368, 0, 37, 0, 0, 0, 0, 376, 2148, 0, 12, 558, 2, 1508, 165, 0, 1554, 146, 70, 2294, 1394, 40, 0, 0] payload_rows_per_dst=[13948, 13900, 7660, 10760, 4766, 12870, 7674, 9082] dup_rows_per_dst=[6958, 2560, 1092, 1268, 2164, 576, 1112, 1914] combine_rows_per_src=[13948, 13900, 7660, 10760, 4766, 12870, 7674, 9082] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84958 payload_rows_remote=76372 dup_rows=13346 dup_rows_remote=12940 dedup_ratio=0.135763 router_tokens_per_expert=[0, 7427, 878, 2, 0, 69, 664, 4105, 0, 6, 0, 0, 0, 842, 3077, 163, 0, 126, 148, 290, 82, 0, 170, 574, 1452, 337, 0, 352, 228, 71, 345, 0, 123, 7136, 92, 0, 3686, 0, 595, 0, 1272, 0, 0, 0, 2619, 252, 0, 0, 92, 0, 204, 501, 14, 437, 90, 0, 0, 373, 4353, 1561, 265, 2726, 0, 1353] payload_rows_per_dst=[3200, 11582, 16032, 11556, 8586, 12104, 15238, 6660] dup_rows_per_dst=[26, 2046, 1602, 2720, 406, 4392, 1710, 444] combine_rows_per_src=[3200, 11582, 16032, 11556, 8586, 12104, 15238, 6660] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=299008000 dispatch_meta_bytes=358208 combine_payload_bytes=299008000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=299366208 ingress_bytes=333611008 link_bytes=333611008 token_bytes=598374208 weight_movement_bytes=34603008 total_nvlink_bytes=632977216 weight_to_token_ratio=0.057828 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=16768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.255351365e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73512 payload_rows_remote=58002 dup_rows=24792 dup_rows_remote=14752 dedup_ratio=0.252197 router_tokens_per_expert=[0, 7405, 900, 8, 1, 78, 778, 3434, 0, 35, 3, 0, 0, 1051, 3568, 170, 0, 178, 118, 281, 97, 4, 106, 365, 1728, 337, 1, 276, 247, 79, 308, 0, 279, 7106, 93, 2, 3719, 0, 247, 1, 1293, 0, 9, 0, 1967, 247, 0, 4, 92, 4, 206, 520, 5, 534, 96, 4, 0, 366, 4374, 2235, 359, 2256, 0, 1578] payload_rows_per_dst=[15510, 11750, 2092, 11690, 15144, 6488, 2554, 8284] dup_rows_per_dst=[10040, 2416, 206, 3010, 7408, 552, 368, 792] combine_rows_per_src=[15510, 11750, 2092, 11690, 15144, 6488, 2554, 8284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79098 payload_rows_remote=64300 dup_rows=19206 dup_rows_remote=8188 dedup_ratio=0.195374 router_tokens_per_expert=[0, 7433, 857, 0, 0, 68, 706, 3751, 0, 10, 0, 0, 0, 956, 3205, 155, 0, 109, 74, 313, 75, 0, 138, 529, 2040, 330, 0, 368, 235, 68, 252, 0, 202, 7181, 87, 0, 3821, 0, 367, 0, 1360, 0, 0, 0, 2208, 251, 0, 0, 87, 0, 160, 416, 17, 455, 84, 0, 0, 388, 4369, 1821, 278, 2557, 0, 1371] payload_rows_per_dst=[3242, 8192, 16052, 5526, 4592, 11500, 15196, 14798] dup_rows_per_dst=[20, 460, 1290, 1060, 114, 3640, 1604, 11018] combine_rows_per_src=[3242, 8192, 16052, 5526, 4592, 11500, 15196, 14798] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82140 payload_rows_remote=71104 dup_rows=16164 dup_rows_remote=13074 dedup_ratio=0.164429 router_tokens_per_expert=[147, 5736, 2572, 202, 15, 133, 1099, 1688, 12, 757, 225, 58, 102, 1257, 2735, 180, 290, 120, 285, 319, 171, 239, 335, 783, 1366, 370, 274, 573, 251, 103, 385, 57, 392, 5296, 170, 172, 2242, 453, 608, 13, 364, 36, 671, 161, 1889, 329, 65, 556, 98, 524, 193, 486, 93, 596, 395, 708, 125, 408, 3684, 922, 494, 2107, 280, 1783] payload_rows_per_dst=[8006, 11704, 14332, 11036, 7460, 8042, 14286, 7274] dup_rows_per_dst=[1768, 3162, 2224, 3090, 640, 2038, 2492, 750] combine_rows_per_src=[8006, 11704, 14332, 11036, 7460, 8042, 14286, 7274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79896 payload_rows_remote=64646 dup_rows=18408 dup_rows_remote=16406 dedup_ratio=0.187256 router_tokens_per_expert=[11, 7013, 1010, 6, 6, 66, 802, 4255, 0, 83, 25, 0, 6, 806, 2763, 161, 81, 132, 86, 363, 86, 40, 198, 726, 1387, 346, 14, 494, 262, 66, 253, 8, 94, 7081, 88, 29, 2624, 22, 758, 2, 1280, 6, 105, 17, 2429, 269, 9, 67, 79, 45, 145, 546, 24, 555, 92, 59, 0, 383, 4615, 892, 287, 3092, 7, 1896] payload_rows_per_dst=[3738, 7686, 15746, 4538, 6716, 12268, 15250, 13954] dup_rows_per_dst=[64, 226, 1704, 1122, 518, 4606, 2002, 8166] combine_rows_per_src=[3738, 7686, 15746, 4538, 6716, 12268, 15250, 13954] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73722 payload_rows_remote=63632 dup_rows=24582 dup_rows_remote=23052 dedup_ratio=0.250061 router_tokens_per_expert=[0, 7428, 1031, 3, 0, 58, 717, 3219, 0, 26, 0, 0, 0, 1052, 2808, 156, 0, 148, 143, 252, 76, 0, 221, 732, 2893, 273, 0, 246, 224, 69, 311, 0, 302, 7095, 93, 0, 3840, 0, 173, 0, 704, 0, 0, 0, 2523, 226, 0, 0, 92, 0, 212, 467, 24, 527, 92, 0, 0, 351, 4139, 2283, 386, 1768, 0, 1769] payload_rows_per_dst=[15542, 10090, 2844, 12546, 15218, 6492, 2452, 8538] dup_rows_per_dst=[9370, 1530, 300, 3764, 7788, 414, 376, 1040] combine_rows_per_src=[15542, 10090, 2844, 12546, 15218, 6492, 2452, 8538] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72268 payload_rows_remote=57110 dup_rows=26036 dup_rows_remote=12372 dedup_ratio=0.264852 router_tokens_per_expert=[573, 339, 2806, 3504, 864, 1, 6039, 2116, 3, 182, 1, 459, 3, 2344, 0, 142, 307, 348, 1752, 90, 815, 29, 338, 153, 77, 90, 974, 0, 0, 169, 1, 763, 5825, 1323, 1, 92, 420, 3483, 642, 0, 438, 0, 12, 3, 1, 2, 3, 427, 1704, 1, 0, 121, 2, 1533, 563, 3, 938, 2, 31, 3303, 2968, 25, 4, 0] payload_rows_per_dst=[15158, 14048, 6506, 9826, 9388, 1270, 7090, 8982] dup_rows_per_dst=[13664, 3870, 1158, 1330, 2534, 502, 764, 2214] combine_rows_per_src=[15158, 14048, 6506, 9826, 9388, 1270, 7090, 8982] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82140 payload_rows_remote=71104 dup_rows=16164 dup_rows_remote=13074 dedup_ratio=0.164429 router_tokens_per_expert=[147, 5736, 2572, 202, 15, 133, 1099, 1688, 12, 757, 225, 58, 102, 1257, 2735, 180, 290, 120, 285, 319, 171, 239, 335, 783, 1366, 370, 274, 573, 251, 103, 385, 57, 392, 5296, 170, 172, 2242, 453, 608, 13, 364, 36, 671, 161, 1889, 329, 65, 556, 98, 524, 193, 486, 93, 596, 395, 708, 125, 408, 3684, 922, 494, 2107, 280, 1783] payload_rows_per_dst=[8006, 11704, 14332, 11036, 7460, 8042, 14286, 7274] dup_rows_per_dst=[1768, 3162, 2224, 3090, 640, 2038, 2492, 750] combine_rows_per_src=[8006, 11704, 14332, 11036, 7460, 8042, 14286, 7274] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 2, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=312819712 dispatch_meta_bytes=357248 combine_payload_bytes=312819712 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=313176960 ingress_bytes=416628736 link_bytes=416628736 token_bytes=625996672 weight_movement_bytes=103809024 total_nvlink_bytes=729805696 weight_to_token_ratio=0.165830 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=22272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.567740392e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83624 payload_rows_remote=71272 dup_rows=14680 dup_rows_remote=10202 dedup_ratio=0.149333 router_tokens_per_expert=[1, 6979, 1043, 19, 1, 68, 709, 4464, 1, 5, 0, 0, 0, 708, 2816, 142, 98, 89, 60, 328, 71, 1, 212, 770, 1512, 346, 0, 468, 253, 63, 197, 2, 73, 7052, 76, 1, 2509, 0, 983, 0, 1240, 0, 26, 0, 2431, 253, 0, 1, 66, 1, 106, 586, 127, 570, 67, 0, 11, 414, 4808, 692, 341, 3057, 1, 2234] payload_rows_per_dst=[3630, 11560, 15550, 9956, 6650, 12352, 15138, 8788] dup_rows_per_dst=[52, 1898, 1666, 2346, 634, 4478, 2012, 1594] combine_rows_per_src=[3630, 11560, 15550, 9956, 6650, 12352, 15138, 8788] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77270 payload_rows_remote=71170 dup_rows=21034 dup_rows_remote=20470 dedup_ratio=0.213969 router_tokens_per_expert=[0, 7196, 874, 0, 0, 37, 865, 4889, 0, 0, 0, 0, 0, 829, 2560, 107, 28, 77, 60, 309, 52, 0, 169, 824, 1166, 312, 0, 394, 252, 37, 180, 0, 82, 7353, 71, 0, 2108, 0, 958, 0, 1564, 0, 0, 0, 2651, 254, 0, 0, 62, 0, 145, 618, 74, 495, 62, 0, 4, 400, 4831, 550, 221, 3505, 0, 1927] payload_rows_per_dst=[14604, 12138, 6100, 11770, 12102, 8030, 6720, 5806] dup_rows_per_dst=[9492, 1864, 564, 2574, 4328, 908, 906, 398] combine_rows_per_src=[14604, 12138, 6100, 11770, 12102, 8030, 6720, 5806] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266616832 dispatch_meta_bytes=317056 combine_payload_bytes=266616832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266933888 ingress_bytes=301219840 link_bytes=301219840 token_bytes=533550720 weight_movement_bytes=34603008 total_nvlink_bytes=568153728 weight_to_token_ratio=0.064854 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133466008e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=260636672 dispatch_meta_bytes=346736 combine_payload_bytes=260636672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260983408 ingress_bytes=295239680 link_bytes=295239680 token_bytes=521620080 weight_movement_bytes=34603008 total_nvlink_bytes=556223088 weight_to_token_ratio=0.066338 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=23552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.110963147e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79098 payload_rows_remote=64300 dup_rows=19206 dup_rows_remote=8188 dedup_ratio=0.195374 router_tokens_per_expert=[0, 7433, 857, 0, 0, 68, 706, 3751, 0, 10, 0, 0, 0, 956, 3205, 155, 0, 109, 74, 313, 75, 0, 138, 529, 2040, 330, 0, 368, 235, 68, 252, 0, 202, 7181, 87, 0, 3821, 0, 367, 0, 1360, 0, 0, 0, 2208, 251, 0, 0, 87, 0, 160, 416, 17, 455, 84, 0, 0, 388, 4369, 1821, 278, 2557, 0, 1371] payload_rows_per_dst=[3242, 8192, 16052, 5526, 4592, 11500, 15196, 14798] dup_rows_per_dst=[20, 460, 1290, 1060, 114, 3640, 1604, 11018] combine_rows_per_src=[3242, 8192, 16052, 5526, 4592, 11500, 15196, 14798] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=233922560 dispatch_meta_bytes=277928 combine_payload_bytes=233922560 prefetch_bytes=34603008 reduce_bytes=173015040 egress_bytes=234200488 ingress_bytes=441540608 link_bytes=441540608 token_bytes=468123048 weight_movement_bytes=207618048 total_nvlink_bytes=675741096 weight_to_token_ratio=0.443512 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=14976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.661481761e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=237576192 dispatch_meta_bytes=291016 combine_payload_bytes=237576192 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=237867208 ingress_bytes=341385216 link_bytes=341385216 token_bytes=475443400 weight_movement_bytes=103809024 total_nvlink_bytes=579252424 weight_to_token_ratio=0.218341 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.284605084e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=263372800 dispatch_meta_bytes=289952 combine_payload_bytes=263372800 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=263662752 ingress_bytes=367181824 link_bytes=367181824 token_bytes=527035552 weight_movement_bytes=103809024 total_nvlink_bytes=630844576 weight_to_token_ratio=0.196968 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.381675643e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=291241984 dispatch_meta_bytes=336712 combine_payload_bytes=291241984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291578696 ingress_bytes=325844992 link_bytes=325844992 token_bytes=582820680 weight_movement_bytes=34603008 total_nvlink_bytes=617423688 weight_to_token_ratio=0.059372 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=24704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.226128472e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=264790016 dispatch_meta_bytes=324208 combine_payload_bytes=264790016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265114224 ingress_bytes=299393024 link_bytes=299393024 token_bytes=529904240 weight_movement_bytes=34603008 total_nvlink_bytes=564507248 weight_to_token_ratio=0.065300 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=36224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126591846e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=260636672 dispatch_meta_bytes=346736 combine_payload_bytes=260636672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260983408 ingress_bytes=295239680 link_bytes=295239680 token_bytes=521620080 weight_movement_bytes=34603008 total_nvlink_bytes=556223088 weight_to_token_ratio=0.066338 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=23552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.110963147e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77270 payload_rows_remote=71170 dup_rows=21034 dup_rows_remote=20470 dedup_ratio=0.213969 router_tokens_per_expert=[0, 7196, 874, 0, 0, 37, 865, 4889, 0, 0, 0, 0, 0, 829, 2560, 107, 28, 77, 60, 309, 52, 0, 169, 824, 1166, 312, 0, 394, 252, 37, 180, 0, 82, 7353, 71, 0, 2108, 0, 958, 0, 1564, 0, 0, 0, 2651, 254, 0, 0, 62, 0, 145, 618, 74, 495, 62, 0, 4, 400, 4831, 550, 221, 3505, 0, 1927] payload_rows_per_dst=[14604, 12138, 6100, 11770, 12102, 8030, 6720, 5806] dup_rows_per_dst=[9492, 1864, 564, 2574, 4328, 908, 906, 398] combine_rows_per_src=[14604, 12138, 6100, 11770, 12102, 8030, 6720, 5806] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=291930112 dispatch_meta_bytes=325896 combine_payload_bytes=291930112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=292256008 ingress_bytes=326533120 link_bytes=326533120 token_bytes=584186120 weight_movement_bytes=34603008 total_nvlink_bytes=618789128 weight_to_token_ratio=0.059233 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.228717842e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84958 payload_rows_remote=76372 dup_rows=13346 dup_rows_remote=12940 dedup_ratio=0.135763 router_tokens_per_expert=[0, 7427, 878, 2, 0, 69, 664, 4105, 0, 6, 0, 0, 0, 842, 3077, 163, 0, 126, 148, 290, 82, 0, 170, 574, 1452, 337, 0, 352, 228, 71, 345, 0, 123, 7136, 92, 0, 3686, 0, 595, 0, 1272, 0, 0, 0, 2619, 252, 0, 0, 92, 0, 204, 501, 14, 437, 90, 0, 0, 373, 4353, 1561, 265, 2726, 0, 1353] payload_rows_per_dst=[3200, 11582, 16032, 11556, 8586, 12104, 15238, 6660] dup_rows_per_dst=[26, 2046, 1602, 2720, 406, 4392, 1710, 444] combine_rows_per_src=[3200, 11582, 16032, 11556, 8586, 12104, 15238, 6660] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=291512320 dispatch_meta_bytes=366560 combine_payload_bytes=291512320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291878880 ingress_bytes=326115328 link_bytes=326115328 token_bytes=583391200 weight_movement_bytes=34603008 total_nvlink_bytes=617994208 weight_to_token_ratio=0.059314 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=36480 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.227145725e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75438 payload_rows_remote=64640 dup_rows=22866 dup_rows_remote=18334 dedup_ratio=0.232605 router_tokens_per_expert=[348, 807, 1214, 408, 576, 446, 1150, 786, 1296, 2164, 230, 414, 1286, 504, 905, 468, 1611, 83, 2058, 545, 771, 417, 19, 573, 1034, 974, 649, 621, 518, 1071, 1519, 942, 521, 708, 302, 288, 1184, 771, 2092, 690, 379, 1882, 177, 577, 366, 656, 490, 3138, 92, 358, 335, 973, 738, 1234, 229, 294, 312, 247, 1408, 654, 761, 528, 157, 204] payload_rows_per_dst=[8960, 8684, 8822, 9242, 10000, 10798, 9450, 9482] dup_rows_per_dst=[2510, 1522, 3332, 2376, 3112, 4532, 3384, 2098] combine_rows_per_src=[8960, 8684, 8822, 9242, 10000, 10798, 9450, 9482] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=263372800 dispatch_meta_bytes=289952 combine_payload_bytes=263372800 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=263662752 ingress_bytes=367181824 link_bytes=367181824 token_bytes=527035552 weight_movement_bytes=103809024 total_nvlink_bytes=630844576 weight_to_token_ratio=0.196968 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.381675643e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=312819712 dispatch_meta_bytes=357248 combine_payload_bytes=312819712 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=313176960 ingress_bytes=416628736 link_bytes=416628736 token_bytes=625996672 weight_movement_bytes=103809024 total_nvlink_bytes=729805696 weight_to_token_ratio=0.165830 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=22272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.567740392e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79896 payload_rows_remote=64646 dup_rows=18408 dup_rows_remote=16406 dedup_ratio=0.187256 router_tokens_per_expert=[11, 7013, 1010, 6, 6, 66, 802, 4255, 0, 83, 25, 0, 6, 806, 2763, 161, 81, 132, 86, 363, 86, 40, 198, 726, 1387, 346, 14, 494, 262, 66, 253, 8, 94, 7081, 88, 29, 2624, 22, 758, 2, 1280, 6, 105, 17, 2429, 269, 9, 67, 79, 45, 145, 546, 24, 555, 92, 59, 0, 383, 4615, 892, 287, 3092, 7, 1896] payload_rows_per_dst=[3738, 7686, 15746, 4538, 6716, 12268, 15250, 13954] dup_rows_per_dst=[64, 226, 1704, 1122, 518, 4606, 2002, 8166] combine_rows_per_src=[3738, 7686, 15746, 4538, 6716, 12268, 15250, 13954] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=264765440 dispatch_meta_bytes=331896 combine_payload_bytes=264765440 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=265097336 ingress_bytes=437780480 link_bytes=437780480 token_bytes=529862776 weight_movement_bytes=173015040 total_nvlink_bytes=702877816 weight_to_token_ratio=0.326528 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=16000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.647332702e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81430 payload_rows_remote=68760 dup_rows=16874 dup_rows_remote=11706 dedup_ratio=0.171651 router_tokens_per_expert=[1, 0, 0, 193, 0, 0, 0, 0, 361, 130, 2501, 0, 572, 318, 0, 92, 73, 0, 0, 0, 0, 76, 0, 0, 0, 4430, 0, 519, 251, 1391, 2, 7, 893, 4945, 0, 0, 0, 759, 0, 1345, 4328, 803, 4141, 0, 544, 1679, 1223, 0, 0, 143, 0, 1748, 0, 0, 73, 0, 1375, 0, 4647, 5459, 2737, 1211, 0, 182] payload_rows_per_dst=[11226, 12670, 12432, 11632, 6918, 10604, 11326, 4622] dup_rows_per_dst=[1652, 5168, 2468, 1568, 866, 2342, 1896, 914] combine_rows_per_src=[11226, 12670, 12432, 11632, 6918, 10604, 11326, 4622] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73512 payload_rows_remote=58002 dup_rows=24792 dup_rows_remote=14752 dedup_ratio=0.252197 router_tokens_per_expert=[0, 7405, 900, 8, 1, 78, 778, 3434, 0, 35, 3, 0, 0, 1051, 3568, 170, 0, 178, 118, 281, 97, 4, 106, 365, 1728, 337, 1, 276, 247, 79, 308, 0, 279, 7106, 93, 2, 3719, 0, 247, 1, 1293, 0, 9, 0, 1967, 247, 0, 4, 92, 4, 206, 520, 5, 534, 96, 4, 0, 366, 4374, 2235, 359, 2256, 0, 1578] payload_rows_per_dst=[15510, 11750, 2092, 11690, 15144, 6488, 2554, 8284] dup_rows_per_dst=[10040, 2416, 206, 3010, 7408, 552, 368, 792] combine_rows_per_src=[15510, 11750, 2092, 11690, 15144, 6488, 2554, 8284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84828 payload_rows_remote=74976 dup_rows=13476 dup_rows_remote=11648 dedup_ratio=0.137085 router_tokens_per_expert=[0, 0, 0, 193, 0, 1, 0, 0, 404, 153, 1920, 0, 572, 344, 0, 92, 67, 0, 0, 0, 0, 83, 0, 0, 0, 3940, 0, 334, 261, 2051, 0, 0, 758, 4514, 0, 0, 0, 805, 0, 1317, 3430, 886, 4767, 0, 551, 1306, 1023, 0, 0, 152, 0, 1597, 61, 0, 67, 0, 1365, 0, 5641, 5664, 3377, 1273, 0, 183] payload_rows_per_dst=[9922, 11460, 11628, 11234, 9852, 11476, 13192, 6064] dup_rows_per_dst=[0, 3754, 0, 1938, 1828, 2916, 1844, 1196] combine_rows_per_src=[9922, 11460, 11628, 11234, 9852, 11476, 13192, 6064] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80076 payload_rows_remote=67282 dup_rows=18228 dup_rows_remote=14776 dedup_ratio=0.185425 router_tokens_per_expert=[0, 0, 1, 184, 0, 0, 0, 4, 424, 143, 2086, 3, 583, 337, 1, 90, 52, 0, 0, 0, 3, 100, 0, 3, 0, 4177, 1, 588, 260, 1875, 1, 175, 585, 5014, 0, 1, 0, 874, 0, 1318, 3912, 957, 4022, 0, 520, 1740, 1479, 1, 0, 118, 4, 1606, 6, 1, 53, 0, 1356, 5, 4802, 5670, 2849, 986, 0, 182] payload_rows_per_dst=[12794, 12834, 13258, 11850, 5178, 8460, 11408, 4294] dup_rows_per_dst=[3452, 4528, 4096, 2304, 378, 934, 1772, 764] combine_rows_per_src=[12794, 12834, 13258, 11850, 5178, 8460, 11408, 4294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77164 payload_rows_remote=61990 dup_rows=21140 dup_rows_remote=5900 dedup_ratio=0.215047 router_tokens_per_expert=[0, 0, 0, 177, 0, 0, 0, 0, 397, 153, 1733, 0, 566, 337, 0, 86, 59, 0, 0, 0, 0, 78, 0, 0, 0, 3987, 0, 399, 245, 1751, 0, 19, 718, 4711, 0, 0, 0, 763, 0, 1218, 4037, 780, 4835, 0, 564, 1691, 1137, 0, 0, 74, 0, 1709, 110, 0, 59, 0, 1429, 0, 5474, 5589, 2960, 1134, 0, 173] payload_rows_per_dst=[10024, 5300, 11452, 10566, 12640, 8104, 3904, 15174] dup_rows_per_dst=[0, 1244, 0, 1644, 2180, 832, 0, 15240] combine_rows_per_src=[10024, 5300, 11452, 10566, 12640, 8104, 3904, 15174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82126 payload_rows_remote=73834 dup_rows=16178 dup_rows_remote=15098 dedup_ratio=0.164571 router_tokens_per_expert=[0, 7, 0, 201, 32, 2, 0, 68, 419, 177, 1558, 43, 636, 346, 30, 80, 61, 14, 1, 0, 41, 82, 0, 8, 0, 3372, 8, 299, 273, 1621, 35, 66, 1090, 3938, 0, 17, 0, 926, 1, 1384, 2867, 1171, 5723, 33, 570, 941, 1212, 11, 1, 129, 66, 2171, 270, 51, 78, 24, 1393, 65, 4636, 5350, 3721, 1675, 0, 158] payload_rows_per_dst=[12022, 5468, 11092, 9782, 11996, 10578, 8292, 12896] dup_rows_per_dst=[44, 1110, 22, 1566, 2716, 2836, 1080, 6804] combine_rows_per_src=[12022, 5468, 11092, 9782, 11996, 10578, 8292, 12896] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79422 payload_rows_remote=68872 dup_rows=18882 dup_rows_remote=16268 dedup_ratio=0.192078 router_tokens_per_expert=[87, 160, 9, 245, 504, 85, 61, 688, 442, 297, 1778, 481, 757, 518, 380, 158, 50, 145, 5, 35, 539, 120, 1, 440, 44, 3067, 39, 487, 790, 1326, 315, 514, 773, 2875, 0, 215, 15, 945, 162, 1655, 3559, 1673, 3848, 397, 713, 968, 1598, 157, 10, 280, 663, 905, 328, 508, 172, 130, 1718, 421, 3335, 2983, 2218, 1222, 6, 133] payload_rows_per_dst=[10578, 11016, 8066, 10550, 9852, 12524, 10842, 5994] dup_rows_per_dst=[796, 4356, 570, 2614, 2114, 5606, 1820, 1006] combine_rows_per_src=[10578, 11016, 8066, 10550, 9852, 12524, 10842, 5994] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=307101696 dispatch_meta_bytes=346496 combine_payload_bytes=307101696 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=307448192 ingress_bytes=376307712 link_bytes=376307712 token_bytes=614549888 weight_movement_bytes=69206016 total_nvlink_bytes=683755904 weight_to_token_ratio=0.112613 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=12672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.416015625e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81430 payload_rows_remote=68760 dup_rows=16874 dup_rows_remote=11706 dedup_ratio=0.171651 router_tokens_per_expert=[1, 0, 0, 193, 0, 0, 0, 0, 361, 130, 2501, 0, 572, 318, 0, 92, 73, 0, 0, 0, 0, 76, 0, 0, 0, 4430, 0, 519, 251, 1391, 2, 7, 893, 4945, 0, 0, 0, 759, 0, 1345, 4328, 803, 4141, 0, 544, 1679, 1223, 0, 0, 143, 0, 1748, 0, 0, 73, 0, 1375, 0, 4647, 5459, 2737, 1211, 0, 182] payload_rows_per_dst=[11226, 12670, 12432, 11632, 6918, 10604, 11326, 4622] dup_rows_per_dst=[1652, 5168, 2468, 1568, 866, 2342, 1896, 914] combine_rows_per_src=[11226, 12670, 12432, 11632, 6918, 10604, 11326, 4622] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85044 payload_rows_remote=75556 dup_rows=13260 dup_rows_remote=11520 dedup_ratio=0.134888 router_tokens_per_expert=[2, 0, 0, 297, 0, 0, 0, 1, 390, 147, 1515, 1, 529, 320, 1, 79, 68, 0, 14, 0, 0, 71, 2, 0, 1, 3414, 0, 214, 281, 1329, 3, 3, 1189, 3580, 0, 0, 0, 1083, 0, 1393, 2102, 1290, 6064, 0, 633, 632, 957, 0, 0, 119, 1, 2503, 252, 0, 82, 0, 1620, 1, 4532, 5584, 4181, 2538, 0, 134] payload_rows_per_dst=[12724, 4976, 11460, 9458, 11734, 9488, 12322, 12882] dup_rows_per_dst=[4, 988, 18, 1032, 2756, 1740, 2656, 4066] combine_rows_per_src=[12724, 4976, 11460, 9458, 11734, 9488, 12322, 12882] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=264790016 dispatch_meta_bytes=324208 combine_payload_bytes=264790016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265114224 ingress_bytes=299393024 link_bytes=299393024 token_bytes=529904240 weight_movement_bytes=34603008 total_nvlink_bytes=564507248 weight_to_token_ratio=0.065300 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=36224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126591846e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=275587072 dispatch_meta_bytes=328232 combine_payload_bytes=275587072 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=275915304 ingress_bytes=344793088 link_bytes=344793088 token_bytes=551502376 weight_movement_bytes=69206016 total_nvlink_bytes=620708392 weight_to_token_ratio=0.125486 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.030000 gemm_active_groups=11 gemm_peak_group_rows=39680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.297428632e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=253911040 dispatch_meta_bytes=271560 combine_payload_bytes=253911040 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=254182600 ingress_bytes=426926080 link_bytes=426926080 token_bytes=508093640 weight_movement_bytes=173015040 total_nvlink_bytes=681108680 weight_to_token_ratio=0.340518 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=11904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.606488469e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=302424064 dispatch_meta_bytes=355728 combine_payload_bytes=302424064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=302779792 ingress_bytes=337027072 link_bytes=337027072 token_bytes=605203856 weight_movement_bytes=34603008 total_nvlink_bytes=639806864 weight_to_token_ratio=0.057176 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=29824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.268205739e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84994 payload_rows_remote=73252 dup_rows=13310 dup_rows_remote=13310 dedup_ratio=0.135396 router_tokens_per_expert=[0, 0, 0, 286, 0, 0, 0, 0, 346, 117, 1707, 0, 619, 316, 0, 66, 59, 0, 0, 0, 0, 52, 0, 0, 0, 3523, 0, 121, 221, 1418, 0, 0, 1391, 3545, 0, 0, 0, 977, 0, 1464, 1739, 1075, 6247, 0, 584, 644, 890, 0, 0, 122, 0, 2617, 230, 0, 59, 0, 1226, 0, 4947, 5760, 4816, 1844, 0, 124] payload_rows_per_dst=[13066, 10690, 11742, 9584, 13202, 8426, 12860, 5424] dup_rows_per_dst=[0, 2742, 0, 982, 4094, 1438, 3090, 964] combine_rows_per_src=[13066, 10690, 11742, 9584, 13202, 8426, 12860, 5424] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=281640960 dispatch_meta_bytes=321864 combine_payload_bytes=281640960 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281962824 ingress_bytes=316243968 link_bytes=316243968 token_bytes=563603784 weight_movement_bytes=34603008 total_nvlink_bytes=598206792 weight_to_token_ratio=0.061396 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.190000592e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77164 payload_rows_remote=61990 dup_rows=21140 dup_rows_remote=5900 dedup_ratio=0.215047 router_tokens_per_expert=[0, 0, 0, 177, 0, 0, 0, 0, 397, 153, 1733, 0, 566, 337, 0, 86, 59, 0, 0, 0, 0, 78, 0, 0, 0, 3987, 0, 399, 245, 1751, 0, 19, 718, 4711, 0, 0, 0, 763, 0, 1218, 4037, 780, 4835, 0, 564, 1691, 1137, 0, 0, 74, 0, 1709, 110, 0, 59, 0, 1429, 0, 5474, 5589, 2960, 1134, 0, 173] payload_rows_per_dst=[10024, 5300, 11452, 10566, 12640, 8104, 3904, 15174] dup_rows_per_dst=[0, 1244, 0, 1644, 2180, 832, 0, 15240] combine_rows_per_src=[10024, 5300, 11452, 10566, 12640, 8104, 3904, 15174] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=237576192 dispatch_meta_bytes=291016 combine_payload_bytes=237576192 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=237867208 ingress_bytes=341385216 link_bytes=341385216 token_bytes=475443400 weight_movement_bytes=103809024 total_nvlink_bytes=579252424 weight_to_token_ratio=0.218341 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.284605084e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=282099712 dispatch_meta_bytes=340560 combine_payload_bytes=282099712 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=282440272 ingress_bytes=316702720 link_bytes=316702720 token_bytes=564539984 weight_movement_bytes=34603008 total_nvlink_bytes=599142992 weight_to_token_ratio=0.061294 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=29696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.191726839e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84828 payload_rows_remote=74976 dup_rows=13476 dup_rows_remote=11648 dedup_ratio=0.137085 router_tokens_per_expert=[0, 0, 0, 193, 0, 1, 0, 0, 404, 153, 1920, 0, 572, 344, 0, 92, 67, 0, 0, 0, 0, 83, 0, 0, 0, 3940, 0, 334, 261, 2051, 0, 0, 758, 4514, 0, 0, 0, 805, 0, 1317, 3430, 886, 4767, 0, 551, 1306, 1023, 0, 0, 152, 0, 1597, 61, 0, 67, 0, 1365, 0, 5641, 5664, 3377, 1273, 0, 183] payload_rows_per_dst=[9922, 11460, 11628, 11234, 9852, 11476, 13192, 6064] dup_rows_per_dst=[0, 3754, 0, 1938, 1828, 2916, 1844, 1196] combine_rows_per_src=[9922, 11460, 11628, 11234, 9852, 11476, 13192, 6064] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=281640960 dispatch_meta_bytes=321864 combine_payload_bytes=281640960 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281962824 ingress_bytes=316243968 link_bytes=316243968 token_bytes=563603784 weight_movement_bytes=34603008 total_nvlink_bytes=598206792 weight_to_token_ratio=0.061396 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.190000592e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=309477376 dispatch_meta_bytes=348304 combine_payload_bytes=309477376 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=309825680 ingress_bytes=447889408 link_bytes=447889408 token_bytes=619303056 weight_movement_bytes=138412032 total_nvlink_bytes=757715088 weight_to_token_ratio=0.223496 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=15872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.685371784e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=300040192 dispatch_meta_bytes=346248 combine_payload_bytes=300040192 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=300386440 ingress_bytes=369246208 link_bytes=369246208 token_bytes=600426632 weight_movement_bytes=69206016 total_nvlink_bytes=669632648 weight_to_token_ratio=0.115261 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=42112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389443754e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=253911040 dispatch_meta_bytes=271560 combine_payload_bytes=253911040 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=254182600 ingress_bytes=426926080 link_bytes=426926080 token_bytes=508093640 weight_movement_bytes=173015040 total_nvlink_bytes=681108680 weight_to_token_ratio=0.340518 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=11904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.606488469e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72904 payload_rows_remote=63940 dup_rows=25400 dup_rows_remote=23712 dedup_ratio=0.258382 router_tokens_per_expert=[6, 1895, 1630, 761, 963, 1378, 764, 873, 385, 1142, 2247, 109, 261, 437, 304, 1495, 1470, 369, 133, 85, 287, 567, 253, 401, 903, 418, 1574, 743, 1548, 1064, 901, 29, 279, 30, 51, 612, 1399, 327, 241, 1294, 137, 244, 136, 946, 604, 11, 1695, 5, 298, 695, 390, 682, 444, 1289, 720, 1236, 874, 1161, 100, 1147, 1226, 2668, 1547, 1269] payload_rows_per_dst=[8958, 9560, 9736, 8458, 9016, 8964, 8436, 9776] dup_rows_per_dst=[4826, 3200, 2730, 2806, 2206, 1688, 3072, 4872] combine_rows_per_src=[8958, 9560, 9736, 8458, 9016, 8964, 8436, 9776] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82126 payload_rows_remote=73834 dup_rows=16178 dup_rows_remote=15098 dedup_ratio=0.164571 router_tokens_per_expert=[0, 7, 0, 201, 32, 2, 0, 68, 419, 177, 1558, 43, 636, 346, 30, 80, 61, 14, 1, 0, 41, 82, 0, 8, 0, 3372, 8, 299, 273, 1621, 35, 66, 1090, 3938, 0, 17, 0, 926, 1, 1384, 2867, 1171, 5723, 33, 570, 941, 1212, 11, 1, 129, 66, 2171, 270, 51, 78, 24, 1393, 65, 4636, 5350, 3721, 1675, 0, 158] payload_rows_per_dst=[12022, 5468, 11092, 9782, 11996, 10578, 8292, 12896] dup_rows_per_dst=[44, 1110, 22, 1566, 2716, 2836, 1080, 6804] combine_rows_per_src=[12022, 5468, 11092, 9782, 11996, 10578, 8292, 12896] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=307101696 dispatch_meta_bytes=346496 combine_payload_bytes=307101696 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=307448192 ingress_bytes=376307712 link_bytes=376307712 token_bytes=614549888 weight_movement_bytes=69206016 total_nvlink_bytes=683755904 weight_to_token_ratio=0.112613 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=12672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.416015625e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88552 payload_rows_remote=75396 dup_rows=9752 dup_rows_remote=7412 dedup_ratio=0.099202 router_tokens_per_expert=[0, 69, 253, 326, 19, 1522, 0, 446, 0, 0, 65, 0, 92, 0, 0, 0, 1714, 0, 0, 0, 0, 1472, 0, 847, 0, 140, 36, 1482, 0, 0, 0, 69, 5925, 0, 92, 718, 0, 0, 175, 838, 244, 0, 0, 1394, 83, 0, 0, 0, 356, 0, 3618, 91, 5570, 69, 6538, 0, 8, 3793, 4639, 900, 2707, 2508, 0, 334] payload_rows_per_dst=[11222, 13260, 7496, 11616, 13156, 13174, 7884, 10744] dup_rows_per_dst=[1634, 130, 570, 1116, 2340, 1408, 384, 2170] combine_rows_per_src=[11222, 13260, 7496, 11616, 13156, 13174, 7884, 10744] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82968 payload_rows_remote=72298 dup_rows=15336 dup_rows_remote=13736 dedup_ratio=0.156006 router_tokens_per_expert=[1, 77, 247, 348, 42, 1436, 3, 460, 0, 0, 25, 1, 92, 0, 0, 1, 2262, 0, 0, 0, 0, 1363, 0, 1070, 2, 78, 169, 1739, 0, 0, 0, 79, 5600, 6, 92, 649, 0, 4, 190, 743, 267, 4, 5, 1812, 100, 1, 4, 0, 354, 0, 3983, 90, 5484, 77, 6417, 2, 0, 3521, 4054, 801, 2366, 2729, 1, 301] payload_rows_per_dst=[10670, 13026, 14724, 11978, 3582, 13096, 8646, 7246] dup_rows_per_dst=[1600, 46, 5866, 4052, 730, 2258, 366, 418] combine_rows_per_src=[10670, 13026, 14724, 11978, 3582, 13096, 8646, 7246] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76078 payload_rows_remote=63986 dup_rows=22226 dup_rows_remote=18974 dedup_ratio=0.226095 router_tokens_per_expert=[0, 68, 252, 283, 16, 1470, 0, 448, 0, 0, 65, 0, 87, 0, 0, 0, 2259, 0, 0, 0, 0, 1742, 0, 970, 0, 52, 87, 1890, 0, 0, 0, 68, 5907, 9, 87, 621, 0, 0, 164, 804, 232, 0, 0, 1518, 77, 0, 0, 0, 246, 0, 3731, 86, 5812, 68, 6403, 0, 4, 3826, 3980, 876, 2334, 2360, 0, 250] payload_rows_per_dst=[8056, 5448, 9040, 10696, 12906, 3500, 14340, 12092] dup_rows_per_dst=[974, 48, 902, 1458, 2278, 154, 13160, 3252] combine_rows_per_src=[8056, 5448, 9040, 10696, 12906, 3500, 14340, 12092] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84702 payload_rows_remote=72406 dup_rows=13602 dup_rows_remote=9210 dedup_ratio=0.138367 router_tokens_per_expert=[15, 69, 287, 219, 81, 1657, 46, 494, 0, 6, 60, 14, 89, 72, 8, 8, 1995, 3, 0, 0, 10, 1209, 13, 811, 10, 100, 108, 980, 11, 8, 2, 70, 5909, 60, 79, 651, 0, 53, 161, 967, 237, 66, 99, 1139, 82, 3, 77, 14, 386, 6, 3670, 84, 5336, 66, 6223, 24, 53, 3621, 4439, 896, 3451, 2577, 22, 246] payload_rows_per_dst=[11286, 12786, 7610, 10704, 13270, 5212, 12296, 11538] dup_rows_per_dst=[1692, 174, 472, 752, 2490, 678, 4392, 2952] combine_rows_per_src=[11286, 12786, 7610, 10704, 13270, 5212, 12296, 11538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=308822016 dispatch_meta_bytes=331232 combine_payload_bytes=308822016 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=309153248 ingress_bytes=378028032 link_bytes=378028032 token_bytes=617975264 weight_movement_bytes=69206016 total_nvlink_bytes=687181280 weight_to_token_ratio=0.111988 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=31744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.422489051e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84786 payload_rows_remote=71122 dup_rows=13518 dup_rows_remote=13438 dedup_ratio=0.137512 router_tokens_per_expert=[0, 58, 227, 300, 34, 1364, 0, 460, 0, 0, 40, 0, 92, 0, 0, 0, 1950, 0, 0, 0, 0, 1634, 0, 1132, 0, 140, 229, 1445, 0, 0, 0, 58, 6009, 3, 92, 756, 0, 0, 168, 827, 226, 0, 0, 1367, 76, 0, 0, 1, 268, 0, 3914, 92, 5677, 58, 6740, 0, 0, 3757, 4306, 884, 2205, 2229, 0, 334] payload_rows_per_dst=[11014, 13664, 14914, 10994, 4878, 13284, 9626, 6412] dup_rows_per_dst=[1386, 80, 6536, 1362, 746, 1410, 1516, 482] combine_rows_per_src=[11014, 13664, 14914, 10994, 4878, 13284, 9626, 6412] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80076 payload_rows_remote=67282 dup_rows=18228 dup_rows_remote=14776 dedup_ratio=0.185425 router_tokens_per_expert=[0, 0, 1, 184, 0, 0, 0, 4, 424, 143, 2086, 3, 583, 337, 1, 90, 52, 0, 0, 0, 3, 100, 0, 3, 0, 4177, 1, 588, 260, 1875, 1, 175, 585, 5014, 0, 1, 0, 874, 0, 1318, 3912, 957, 4022, 0, 520, 1740, 1479, 1, 0, 118, 4, 1606, 6, 1, 53, 0, 1356, 5, 4802, 5670, 2849, 986, 0, 182] payload_rows_per_dst=[12794, 12834, 13258, 11850, 5178, 8460, 11408, 4294] dup_rows_per_dst=[3452, 4528, 4096, 2304, 378, 934, 1772, 764] combine_rows_per_src=[12794, 12834, 13258, 11850, 5178, 8460, 11408, 4294] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=261898240 dispatch_meta_bytes=350608 combine_payload_bytes=261898240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262248848 ingress_bytes=296501248 link_bytes=296501248 token_bytes=524147088 weight_movement_bytes=34603008 total_nvlink_bytes=558750096 weight_to_token_ratio=0.066018 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.115710326e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 2, 1, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83418 payload_rows_remote=72396 dup_rows=14886 dup_rows_remote=12832 dedup_ratio=0.151428 router_tokens_per_expert=[200, 106, 513, 543, 84, 620, 513, 863, 4, 158, 90, 167, 245, 710, 53, 148, 1173, 7, 45, 0, 174, 879, 146, 989, 104, 358, 813, 1251, 132, 146, 19, 95, 4956, 302, 93, 629, 5, 577, 183, 721, 254, 702, 1217, 1168, 110, 101, 864, 122, 445, 26, 2641, 78, 4556, 68, 5100, 212, 205, 2887, 3620, 941, 2020, 2427, 210, 364] payload_rows_per_dst=[10632, 12398, 6348, 11022, 12618, 13604, 6608, 10188] dup_rows_per_dst=[2026, 952, 478, 2054, 2314, 4584, 332, 2146] combine_rows_per_src=[10632, 12398, 6348, 11022, 12618, 13604, 6608, 10188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=302424064 dispatch_meta_bytes=355728 combine_payload_bytes=302424064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=302779792 ingress_bytes=337027072 link_bytes=337027072 token_bytes=605203856 weight_movement_bytes=34603008 total_nvlink_bytes=639806864 weight_to_token_ratio=0.057176 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=29824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.268205739e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=291315712 dispatch_meta_bytes=338240 combine_payload_bytes=291315712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291653952 ingress_bytes=325918720 link_bytes=325918720 token_bytes=582969664 weight_movement_bytes=34603008 total_nvlink_bytes=617572672 weight_to_token_ratio=0.059356 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=46592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.226405905e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87540 payload_rows_remote=75686 dup_rows=10764 dup_rows_remote=9428 dedup_ratio=0.109497 router_tokens_per_expert=[0, 63, 287, 97, 74, 1979, 0, 486, 0, 0, 39, 0, 67, 0, 0, 0, 2130, 0, 0, 0, 0, 1365, 0, 941, 0, 65, 17, 853, 0, 1, 0, 63, 5996, 2, 67, 664, 0, 1, 138, 1523, 210, 1, 0, 1349, 71, 1, 1, 1, 352, 14, 4069, 79, 4961, 63, 6304, 1, 54, 3559, 3884, 812, 3841, 2413, 1, 193] payload_rows_per_dst=[11272, 12744, 8200, 9468, 13476, 11854, 8702, 11824] dup_rows_per_dst=[1818, 76, 672, 298, 3306, 1336, 454, 2804] combine_rows_per_src=[11272, 12744, 8200, 9468, 13476, 11854, 8702, 11824] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=296132608 dispatch_meta_bytes=344136 combine_payload_bytes=296132608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296476744 ingress_bytes=330735616 link_bytes=330735616 token_bytes=592609352 weight_movement_bytes=34603008 total_nvlink_bytes=627212360 weight_to_token_ratio=0.058391 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=26880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.244531497e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=262086656 dispatch_meta_bytes=331840 combine_payload_bytes=262086656 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=262418496 ingress_bytes=435101696 link_bytes=435101696 token_bytes=524505152 weight_movement_bytes=173015040 total_nvlink_bytes=697520192 weight_to_token_ratio=0.329863 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=19456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.637252653e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87510 payload_rows_remote=76118 dup_rows=10794 dup_rows_remote=7710 dedup_ratio=0.109802 router_tokens_per_expert=[0, 37, 257, 69, 63, 1781, 0, 554, 0, 0, 29, 0, 62, 0, 0, 0, 1958, 0, 0, 0, 0, 1315, 0, 870, 0, 60, 7, 841, 0, 0, 0, 37, 6063, 0, 62, 543, 0, 0, 114, 1420, 155, 0, 0, 735, 52, 0, 0, 0, 250, 0, 3969, 65, 5434, 37, 6645, 0, 53, 3750, 4422, 813, 4340, 2107, 0, 183] payload_rows_per_dst=[11484, 13414, 11392, 10418, 8584, 11920, 8868, 11430] dup_rows_per_dst=[1538, 58, 3084, 316, 1630, 832, 730, 2606] combine_rows_per_src=[11484, 13414, 11392, 10418, 8584, 11920, 8868, 11430] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=275587072 dispatch_meta_bytes=328232 combine_payload_bytes=275587072 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=275915304 ingress_bytes=344793088 link_bytes=344793088 token_bytes=551502376 weight_movement_bytes=69206016 total_nvlink_bytes=620708392 weight_to_token_ratio=0.125486 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.030000 gemm_active_groups=11 gemm_peak_group_rows=39680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.297428632e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76078 payload_rows_remote=63986 dup_rows=22226 dup_rows_remote=18974 dedup_ratio=0.226095 router_tokens_per_expert=[0, 68, 252, 283, 16, 1470, 0, 448, 0, 0, 65, 0, 87, 0, 0, 0, 2259, 0, 0, 0, 0, 1742, 0, 970, 0, 52, 87, 1890, 0, 0, 0, 68, 5907, 9, 87, 621, 0, 0, 164, 804, 232, 0, 0, 1518, 77, 0, 0, 0, 246, 0, 3731, 86, 5812, 68, 6403, 0, 4, 3826, 3980, 876, 2334, 2360, 0, 250] payload_rows_per_dst=[8056, 5448, 9040, 10696, 12906, 3500, 14340, 12092] dup_rows_per_dst=[974, 48, 902, 1458, 2278, 154, 13160, 3252] combine_rows_per_src=[8056, 5448, 9040, 10696, 12906, 3500, 14340, 12092] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=296574976 dispatch_meta_bytes=326464 combine_payload_bytes=296574976 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=296901440 ingress_bytes=400384000 link_bytes=400384000 token_bytes=593476416 weight_movement_bytes=103809024 total_nvlink_bytes=697285440 weight_to_token_ratio=0.174917 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=29696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.506612758e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=291315712 dispatch_meta_bytes=338240 combine_payload_bytes=291315712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291653952 ingress_bytes=325918720 link_bytes=325918720 token_bytes=582969664 weight_movement_bytes=34603008 total_nvlink_bytes=617572672 weight_to_token_ratio=0.059356 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=46592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.226405905e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=296534016 dispatch_meta_bytes=340912 combine_payload_bytes=296534016 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=296874928 ingress_bytes=365740032 link_bytes=365740032 token_bytes=593408944 weight_movement_bytes=69206016 total_nvlink_bytes=662614960 weight_to_token_ratio=0.116624 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=33408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.376250296e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88552 payload_rows_remote=75396 dup_rows=9752 dup_rows_remote=7412 dedup_ratio=0.099202 router_tokens_per_expert=[0, 69, 253, 326, 19, 1522, 0, 446, 0, 0, 65, 0, 92, 0, 0, 0, 1714, 0, 0, 0, 0, 1472, 0, 847, 0, 140, 36, 1482, 0, 0, 0, 69, 5925, 0, 92, 718, 0, 0, 175, 838, 244, 0, 0, 1394, 83, 0, 0, 0, 356, 0, 3618, 91, 5570, 69, 6538, 0, 8, 3793, 4639, 900, 2707, 2508, 0, 334] payload_rows_per_dst=[11222, 13260, 7496, 11616, 13156, 13174, 7884, 10744] dup_rows_per_dst=[1634, 130, 570, 1116, 2340, 1408, 384, 2170] combine_rows_per_src=[11222, 13260, 7496, 11616, 13156, 13174, 7884, 10744] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 2, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=310009856 dispatch_meta_bytes=340456 combine_payload_bytes=310009856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=310350312 ingress_bytes=344612864 link_bytes=344612864 token_bytes=620360168 weight_movement_bytes=34603008 total_nvlink_bytes=654963176 weight_to_token_ratio=0.055779 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=33024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.296750464e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=311779328 dispatch_meta_bytes=335312 combine_payload_bytes=311779328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=312114640 ingress_bytes=346382336 link_bytes=346382336 token_bytes=623893968 weight_movement_bytes=34603008 total_nvlink_bytes=658496976 weight_to_token_ratio=0.055463 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=15488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.303408844e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=262086656 dispatch_meta_bytes=331840 combine_payload_bytes=262086656 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=262418496 ingress_bytes=435101696 link_bytes=435101696 token_bytes=524505152 weight_movement_bytes=173015040 total_nvlink_bytes=697520192 weight_to_token_ratio=0.329863 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=19456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.637252653e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84702 payload_rows_remote=72406 dup_rows=13602 dup_rows_remote=9210 dedup_ratio=0.138367 router_tokens_per_expert=[15, 69, 287, 219, 81, 1657, 46, 494, 0, 6, 60, 14, 89, 72, 8, 8, 1995, 3, 0, 0, 10, 1209, 13, 811, 10, 100, 108, 980, 11, 8, 2, 70, 5909, 60, 79, 651, 0, 53, 161, 967, 237, 66, 99, 1139, 82, 3, 77, 14, 386, 6, 3670, 84, 5336, 66, 6223, 24, 53, 3621, 4439, 896, 3451, 2577, 22, 246] payload_rows_per_dst=[11286, 12786, 7610, 10704, 13270, 5212, 12296, 11538] dup_rows_per_dst=[1692, 174, 472, 752, 2490, 678, 4392, 2952] combine_rows_per_src=[11286, 12786, 7610, 10704, 13270, 5212, 12296, 11538] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=308822016 dispatch_meta_bytes=331232 combine_payload_bytes=308822016 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=309153248 ingress_bytes=378028032 link_bytes=378028032 token_bytes=617975264 weight_movement_bytes=69206016 total_nvlink_bytes=687181280 weight_to_token_ratio=0.111988 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=31744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.422489051e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76956 payload_rows_remote=67994 dup_rows=21348 dup_rows_remote=17474 dedup_ratio=0.217163 router_tokens_per_expert=[691, 1736, 1050, 190, 203, 544, 2451, 301, 90, 3, 642, 876, 1451, 0, 6, 1146, 923, 708, 3387, 129, 432, 701, 323, 19, 74, 1753, 815, 138, 144, 188, 3157, 240, 1357, 494, 188, 2030, 583, 345, 1551, 810, 199, 2, 79, 1546, 121, 686, 1528, 825, 1031, 1522, 1235, 622, 245, 631, 1, 1432, 1341, 505, 483, 636, 498, 608, 887, 620] payload_rows_per_dst=[7876, 10230, 6076, 10402, 10630, 8962, 9348, 13432] dup_rows_per_dst=[1554, 3100, 394, 2616, 3606, 3874, 1706, 4498] combine_rows_per_src=[7876, 10230, 6076, 10402, 10630, 8962, 9348, 13432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88592 payload_rows_remote=74264 dup_rows=9712 dup_rows_remote=8806 dedup_ratio=0.098796 router_tokens_per_expert=[8, 0, 69, 29, 69, 0, 0, 0, 0, 2798, 0, 242, 91, 153, 0, 1498, 0, 6747, 0, 205, 0, 0, 67, 0, 6981, 0, 790, 252, 0, 662, 92, 4612, 6966, 6833, 0, 784, 0, 0, 0, 0, 304, 67, 979, 0, 0, 1532, 252, 0, 55, 0, 0, 1974, 0, 0, 0, 0, 0, 3967, 6, 0, 0, 0, 0, 68] payload_rows_per_dst=[13932, 8358, 13900, 3464, 14328, 11966, 14562, 8082] dup_rows_per_dst=[350, 1206, 138, 128, 906, 3526, 3458, 0] combine_rows_per_src=[13932, 8358, 13900, 3464, 14328, 11966, 14562, 8082] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78794 payload_rows_remote=65036 dup_rows=19510 dup_rows_remote=19180 dedup_ratio=0.198466 router_tokens_per_expert=[4, 0, 77, 10, 78, 0, 0, 0, 0, 2190, 0, 214, 92, 184, 1, 1342, 1, 6821, 2, 199, 0, 0, 53, 2, 6943, 4, 1353, 247, 3, 803, 90, 4409, 6875, 6769, 0, 799, 4, 0, 0, 0, 262, 52, 1189, 4, 0, 1483, 249, 3, 51, 1, 0, 2296, 0, 0, 0, 0, 0, 3879, 57, 0, 1, 3, 0, 53] payload_rows_per_dst=[13758, 15256, 1336, 15614, 14312, 5860, 4674, 7984] dup_rows_per_dst=[330, 6432, 28, 11240, 832, 624, 22, 2] combine_rows_per_src=[13758, 15256, 1336, 15614, 14312, 5860, 4674, 7984] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82732 payload_rows_remote=67620 dup_rows=15572 dup_rows_remote=9034 dedup_ratio=0.158407 router_tokens_per_expert=[2, 0, 68, 0, 68, 0, 0, 0, 0, 2291, 0, 171, 86, 154, 0, 1539, 7, 6886, 0, 211, 0, 0, 59, 0, 7156, 0, 1288, 251, 0, 845, 86, 4351, 6942, 6945, 0, 770, 0, 0, 0, 0, 129, 52, 907, 0, 0, 1510, 251, 0, 53, 0, 0, 2194, 0, 0, 0, 0, 0, 3821, 0, 0, 0, 0, 0, 59] payload_rows_per_dst=[140, 7466, 14192, 4766, 15276, 11068, 14712, 15112] dup_rows_per_dst=[136, 1016, 134, 174, 148, 3332, 4094, 6538] combine_rows_per_src=[140, 7466, 14192, 4766, 15276, 11068, 14712, 15112] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=304185344 dispatch_meta_bytes=332280 combine_payload_bytes=304185344 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=304517624 ingress_bytes=407994368 link_bytes=407994368 token_bytes=608702968 weight_movement_bytes=103809024 total_nvlink_bytes=712511992 weight_to_token_ratio=0.170541 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=30592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.535249961e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83370 payload_rows_remote=69594 dup_rows=14934 dup_rows_remote=11368 dedup_ratio=0.151917 router_tokens_per_expert=[23, 3, 66, 78, 75, 1, 4, 0, 30, 3395, 0, 187, 105, 798, 17, 1334, 2, 6729, 30, 450, 1, 0, 86, 31, 6624, 76, 462, 261, 41, 721, 83, 4323, 6810, 6041, 46, 772, 34, 23, 0, 13, 378, 70, 1439, 59, 0, 1428, 280, 82, 95, 0, 0, 1937, 6, 0, 4, 5, 35, 3401, 5, 2, 17, 66, 0, 68] payload_rows_per_dst=[13288, 10026, 14498, 3182, 2124, 12264, 13776, 14212] dup_rows_per_dst=[438, 1706, 160, 106, 46, 3854, 3566, 5058] combine_rows_per_src=[13288, 10026, 14498, 3182, 2124, 12264, 13776, 14212] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82380 payload_rows_remote=73032 dup_rows=15924 dup_rows_remote=13832 dedup_ratio=0.161987 router_tokens_per_expert=[0, 0, 58, 0, 58, 0, 0, 0, 0, 1813, 0, 243, 92, 143, 0, 1460, 1, 6785, 0, 444, 0, 0, 73, 0, 7000, 0, 1667, 227, 0, 734, 92, 4280, 6946, 6806, 0, 835, 0, 0, 0, 0, 96, 72, 869, 0, 0, 1668, 227, 0, 41, 0, 0, 2542, 0, 0, 0, 0, 0, 3723, 84, 0, 0, 0, 0, 73] payload_rows_per_dst=[13892, 9348, 10554, 15944, 14382, 5338, 5162, 7760] dup_rows_per_dst=[232, 2092, 114, 12056, 900, 526, 4, 0] combine_rows_per_src=[13892, 9348, 10554, 15944, 14382, 5338, 5162, 7760] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=296574976 dispatch_meta_bytes=326464 combine_payload_bytes=296574976 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=296901440 ingress_bytes=400384000 link_bytes=400384000 token_bytes=593476416 weight_movement_bytes=103809024 total_nvlink_bytes=697285440 weight_to_token_ratio=0.174917 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=29696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.506612758e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82968 payload_rows_remote=72298 dup_rows=15336 dup_rows_remote=13736 dedup_ratio=0.156006 router_tokens_per_expert=[1, 77, 247, 348, 42, 1436, 3, 460, 0, 0, 25, 1, 92, 0, 0, 1, 2262, 0, 0, 0, 0, 1363, 0, 1070, 2, 78, 169, 1739, 0, 0, 0, 79, 5600, 6, 92, 649, 0, 4, 190, 743, 267, 4, 5, 1812, 100, 1, 4, 0, 354, 0, 3983, 90, 5484, 77, 6417, 2, 0, 3521, 4054, 801, 2366, 2729, 1, 301] payload_rows_per_dst=[10670, 13026, 14724, 11978, 3582, 13096, 8646, 7246] dup_rows_per_dst=[1600, 46, 5866, 4052, 730, 2258, 366, 418] combine_rows_per_src=[10670, 13026, 14724, 11978, 3582, 13096, 8646, 7246] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82108 payload_rows_remote=74266 dup_rows=16196 dup_rows_remote=14872 dedup_ratio=0.164754 router_tokens_per_expert=[238, 68, 83, 109, 71, 113, 67, 0, 123, 2919, 0, 224, 327, 1639, 272, 1099, 57, 5171, 406, 530, 20, 4, 395, 407, 4586, 692, 733, 241, 172, 1645, 116, 3782, 5449, 2384, 638, 869, 342, 205, 3, 308, 573, 132, 1423, 774, 49, 1551, 392, 899, 450, 103, 24, 2171, 88, 25, 128, 56, 370, 1667, 208, 96, 518, 813, 24, 111] payload_rows_per_dst=[11644, 10198, 13166, 7842, 10212, 12118, 10658, 6270] dup_rows_per_dst=[752, 3008, 814, 1324, 1030, 5064, 2860, 1344] combine_rows_per_src=[11644, 10198, 13166, 7842, 10212, 12118, 10658, 6270] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=299139072 dispatch_meta_bytes=347456 combine_payload_bytes=299139072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=299486528 ingress_bytes=333742080 link_bytes=333742080 token_bytes=598625600 weight_movement_bytes=34603008 total_nvlink_bytes=633228608 weight_to_token_ratio=0.057804 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=21760 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.255844579e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278503424 dispatch_meta_bytes=341872 combine_payload_bytes=278503424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278845296 ingress_bytes=313106432 link_bytes=313106432 token_bytes=557348720 weight_movement_bytes=34603008 total_nvlink_bytes=591951728 weight_to_token_ratio=0.062085 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.178194296e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266387456 dispatch_meta_bytes=336864 combine_payload_bytes=266387456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266724320 ingress_bytes=300990464 link_bytes=300990464 token_bytes=533111776 weight_movement_bytes=34603008 total_nvlink_bytes=567714784 weight_to_token_ratio=0.064908 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=47232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132602884e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84358 payload_rows_remote=71800 dup_rows=13946 dup_rows_remote=9912 dedup_ratio=0.141866 router_tokens_per_expert=[0, 0, 63, 67, 66, 0, 0, 0, 0, 2711, 0, 205, 67, 821, 0, 1374, 0, 6787, 1, 483, 0, 0, 68, 1, 6851, 1, 554, 292, 1, 506, 67, 3998, 6971, 6437, 1, 778, 1, 0, 0, 0, 337, 68, 2303, 0, 0, 1337, 253, 0, 39, 0, 0, 1633, 0, 0, 0, 0, 0, 3941, 0, 0, 0, 0, 0, 69] payload_rows_per_dst=[13942, 8840, 14542, 2834, 4186, 12558, 14016, 13440] dup_rows_per_dst=[392, 1516, 138, 8, 92, 4034, 3030, 4736] combine_rows_per_src=[13942, 8840, 14542, 2834, 4186, 12558, 14016, 13440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=276971520 dispatch_meta_bytes=306616 combine_payload_bytes=276971520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277278136 ingress_bytes=311574528 link_bytes=311574528 token_bytes=554249656 weight_movement_bytes=34603008 total_nvlink_bytes=588852664 weight_to_token_ratio=0.062432 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=28288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.172429865e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84878 payload_rows_remote=70406 dup_rows=13426 dup_rows_remote=13308 dedup_ratio=0.136576 router_tokens_per_expert=[0, 0, 37, 55, 37, 0, 0, 0, 0, 3566, 0, 182, 62, 725, 0, 1383, 0, 6874, 0, 362, 0, 0, 59, 0, 6972, 0, 352, 255, 0, 581, 62, 4300, 7410, 6433, 0, 621, 0, 0, 0, 0, 90, 59, 1494, 0, 0, 1189, 254, 0, 82, 0, 0, 1773, 0, 0, 0, 0, 0, 3824, 0, 0, 0, 0, 0, 59] payload_rows_per_dst=[14820, 10162, 14472, 13466, 14640, 5868, 3684, 7766] dup_rows_per_dst=[258, 1674, 118, 6156, 4890, 304, 26, 0] combine_rows_per_src=[14820, 10162, 14472, 13466, 14640, 5868, 3684, 7766] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82732 payload_rows_remote=67620 dup_rows=15572 dup_rows_remote=9034 dedup_ratio=0.158407 router_tokens_per_expert=[2, 0, 68, 0, 68, 0, 0, 0, 0, 2291, 0, 171, 86, 154, 0, 1539, 7, 6886, 0, 211, 0, 0, 59, 0, 7156, 0, 1288, 251, 0, 845, 86, 4351, 6942, 6945, 0, 770, 0, 0, 0, 0, 129, 52, 907, 0, 0, 1510, 251, 0, 53, 0, 0, 2194, 0, 0, 0, 0, 0, 3821, 0, 0, 0, 0, 0, 59] payload_rows_per_dst=[140, 7466, 14192, 4766, 15276, 11068, 14712, 15112] dup_rows_per_dst=[136, 1016, 134, 174, 148, 3332, 4094, 6538] combine_rows_per_src=[140, 7466, 14192, 4766, 15276, 11068, 14712, 15112] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=296132608 dispatch_meta_bytes=344136 combine_payload_bytes=296132608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296476744 ingress_bytes=330735616 link_bytes=330735616 token_bytes=592609352 weight_movement_bytes=34603008 total_nvlink_bytes=627212360 weight_to_token_ratio=0.058391 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=26880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.244531497e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88592 payload_rows_remote=74264 dup_rows=9712 dup_rows_remote=8806 dedup_ratio=0.098796 router_tokens_per_expert=[8, 0, 69, 29, 69, 0, 0, 0, 0, 2798, 0, 242, 91, 153, 0, 1498, 0, 6747, 0, 205, 0, 0, 67, 0, 6981, 0, 790, 252, 0, 662, 92, 4612, 6966, 6833, 0, 784, 0, 0, 0, 0, 304, 67, 979, 0, 0, 1532, 252, 0, 55, 0, 0, 1974, 0, 0, 0, 0, 0, 3967, 6, 0, 0, 0, 0, 68] payload_rows_per_dst=[13932, 8358, 13900, 3464, 14328, 11966, 14562, 8082] dup_rows_per_dst=[350, 1206, 138, 128, 906, 3526, 3458, 0] combine_rows_per_src=[13932, 8358, 13900, 3464, 14328, 11966, 14562, 8082] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=304193536 dispatch_meta_bytes=356552 combine_payload_bytes=304193536 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=304550088 ingress_bytes=442605568 link_bytes=442605568 token_bytes=608743624 weight_movement_bytes=138412032 total_nvlink_bytes=747155656 weight_to_token_ratio=0.227373 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=17792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.665489120e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=285057024 dispatch_meta_bytes=323848 combine_payload_bytes=285057024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=285380872 ingress_bytes=319660032 link_bytes=319660032 token_bytes=570437896 weight_movement_bytes=34603008 total_nvlink_bytes=605040904 weight_to_token_ratio=0.060660 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=31360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.202854966e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=299139072 dispatch_meta_bytes=347456 combine_payload_bytes=299139072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=299486528 ingress_bytes=333742080 link_bytes=333742080 token_bytes=598625600 weight_movement_bytes=34603008 total_nvlink_bytes=633228608 weight_to_token_ratio=0.057804 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=21760 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.255844579e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=304193536 dispatch_meta_bytes=356552 combine_payload_bytes=304193536 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=304550088 ingress_bytes=442605568 link_bytes=442605568 token_bytes=608743624 weight_movement_bytes=138412032 total_nvlink_bytes=747155656 weight_to_token_ratio=0.227373 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=17792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.665489120e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=294092800 dispatch_meta_bytes=326848 combine_payload_bytes=294092800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=294419648 ingress_bytes=328695808 link_bytes=328695808 token_bytes=588512448 weight_movement_bytes=34603008 total_nvlink_bytes=623115456 weight_to_token_ratio=0.058797 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=20096 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.236855863e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=288382976 dispatch_meta_bytes=334856 combine_payload_bytes=288382976 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=288717832 ingress_bytes=357588992 link_bytes=357588992 token_bytes=577100808 weight_movement_bytes=69206016 total_nvlink_bytes=646306824 weight_to_token_ratio=0.119920 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=44032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.345578589e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83370 payload_rows_remote=69594 dup_rows=14934 dup_rows_remote=11368 dedup_ratio=0.151917 router_tokens_per_expert=[23, 3, 66, 78, 75, 1, 4, 0, 30, 3395, 0, 187, 105, 798, 17, 1334, 2, 6729, 30, 450, 1, 0, 86, 31, 6624, 76, 462, 261, 41, 721, 83, 4323, 6810, 6041, 46, 772, 34, 23, 0, 13, 378, 70, 1439, 59, 0, 1428, 280, 82, 95, 0, 0, 1937, 6, 0, 4, 5, 35, 3401, 5, 2, 17, 66, 0, 68] payload_rows_per_dst=[13288, 10026, 14498, 3182, 2124, 12264, 13776, 14212] dup_rows_per_dst=[438, 1706, 160, 106, 46, 3854, 3566, 5058] combine_rows_per_src=[13288, 10026, 14498, 3182, 2124, 12264, 13776, 14212] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=276971520 dispatch_meta_bytes=306616 combine_payload_bytes=276971520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277278136 ingress_bytes=311574528 link_bytes=311574528 token_bytes=554249656 weight_movement_bytes=34603008 total_nvlink_bytes=588852664 weight_to_token_ratio=0.062432 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=28288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.172429865e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=304185344 dispatch_meta_bytes=332280 combine_payload_bytes=304185344 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=304517624 ingress_bytes=407994368 link_bytes=407994368 token_bytes=608702968 weight_movement_bytes=103809024 total_nvlink_bytes=712511992 weight_to_token_ratio=0.170541 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=30592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.535249961e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86070 payload_rows_remote=76370 dup_rows=12234 dup_rows_remote=10974 dedup_ratio=0.124451 router_tokens_per_expert=[0, 0, 150, 1261, 0, 0, 1050, 0, 0, 1317, 800, 0, 0, 0, 769, 0, 146, 520, 0, 2075, 0, 67, 0, 0, 6114, 0, 0, 366, 0, 0, 0, 0, 768, 490, 0, 0, 1379, 2751, 0, 92, 39, 0, 6015, 0, 0, 3, 73, 0, 491, 0, 92, 0, 6172, 92, 83, 3, 7017, 0, 0, 0, 6804, 679, 1474, 0] payload_rows_per_dst=[10232, 4498, 14184, 12960, 9700, 12180, 12958, 9358] dup_rows_per_dst=[1942, 1274, 5040, 0, 1260, 80, 908, 1730] combine_rows_per_src=[10232, 4498, 14184, 12960, 9700, 12180, 12958, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81668 payload_rows_remote=66556 dup_rows=16636 dup_rows_remote=13026 dedup_ratio=0.169230 router_tokens_per_expert=[0, 1, 118, 828, 0, 5, 1382, 0, 0, 1199, 927, 0, 2, 6, 702, 0, 161, 444, 5, 1494, 0, 53, 0, 0, 6138, 0, 0, 388, 0, 0, 0, 1, 827, 457, 2, 3, 1322, 2929, 0, 90, 92, 3, 5936, 0, 0, 8, 29, 4, 488, 2, 90, 0, 6455, 91, 100, 50, 7027, 0, 2, 1, 6743, 801, 1746, 0] payload_rows_per_dst=[15112, 15918, 3914, 13160, 16056, 11382, 1362, 4764] dup_rows_per_dst=[3610, 3240, 400, 478, 8114, 178, 280, 336] combine_rows_per_src=[15112, 15918, 3914, 13160, 16056, 11382, 1362, 4764] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73482 payload_rows_remote=57554 dup_rows=24822 dup_rows_remote=6408 dedup_ratio=0.252502 router_tokens_per_expert=[0, 0, 74, 1229, 0, 0, 1129, 0, 0, 1261, 803, 0, 0, 0, 793, 0, 152, 502, 0, 1739, 0, 59, 0, 0, 5960, 0, 0, 365, 0, 0, 0, 0, 749, 412, 0, 0, 1513, 2702, 0, 86, 20, 0, 5967, 0, 0, 0, 56, 0, 454, 0, 86, 0, 6475, 86, 78, 0, 7044, 0, 0, 0, 7012, 687, 1659, 0] payload_rows_per_dst=[4706, 4452, 7994, 5006, 9660, 12044, 13692, 15928] dup_rows_per_dst=[158, 1262, 1386, 0, 1264, 42, 2296, 18414] combine_rows_per_src=[4706, 4452, 7994, 5006, 9660, 12044, 13692, 15928] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=312811520 dispatch_meta_bytes=349376 combine_payload_bytes=312811520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=313160896 ingress_bytes=347414528 link_bytes=347414528 token_bytes=625972416 weight_movement_bytes=34603008 total_nvlink_bytes=660575424 weight_to_token_ratio=0.055279 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.307292900e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86862 payload_rows_remote=73842 dup_rows=11442 dup_rows_remote=10424 dedup_ratio=0.116394 router_tokens_per_expert=[5, 9, 91, 1099, 10, 9, 1187, 0, 0, 2074, 953, 2, 74, 107, 707, 3, 144, 507, 46, 2137, 17, 61, 3, 10, 6269, 10, 0, 406, 25, 4, 0, 52, 744, 420, 44, 19, 1661, 1881, 16, 79, 77, 52, 5853, 0, 6, 6, 135, 98, 507, 61, 80, 0, 6103, 113, 88, 67, 6706, 0, 9, 2, 6531, 698, 1075, 0] payload_rows_per_dst=[4628, 6382, 13998, 13512, 8516, 12200, 13020, 14606] dup_rows_per_dst=[192, 1458, 4914, 20, 1212, 254, 1018, 2374] combine_rows_per_src=[4628, 6382, 13998, 13512, 8516, 12200, 13020, 14606] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86900 payload_rows_remote=71038 dup_rows=11404 dup_rows_remote=8526 dedup_ratio=0.116007 router_tokens_per_expert=[0, 0, 143, 1188, 0, 0, 1161, 0, 0, 1182, 855, 0, 0, 0, 715, 0, 160, 412, 0, 1902, 0, 73, 0, 0, 6465, 0, 0, 319, 0, 0, 0, 0, 772, 462, 0, 0, 1510, 2191, 0, 92, 81, 0, 5963, 0, 0, 1, 0, 0, 435, 0, 92, 0, 6566, 92, 76, 13, 7032, 0, 0, 0, 6618, 756, 1825, 0] payload_rows_per_dst=[15202, 15862, 4790, 13568, 10830, 11926, 9952, 4770] dup_rows_per_dst=[3846, 2878, 304, 0, 3124, 164, 696, 392] combine_rows_per_src=[15202, 15862, 4790, 13568, 10830, 11926, 9952, 4770] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82630 payload_rows_remote=72744 dup_rows=15674 dup_rows_remote=12624 dedup_ratio=0.159444 router_tokens_per_expert=[910, 134, 1, 3261, 69, 6, 299, 93, 0, 474, 945, 685, 935, 1006, 493, 1081, 7, 2, 659, 27, 21, 508, 419, 2804, 135, 79, 977, 6, 367, 509, 0, 8, 76, 1565, 302, 95, 6, 0, 0, 2155, 1492, 221, 226, 2385, 5139, 108, 2036, 2034, 418, 194, 0, 1636, 319, 629, 5844, 0, 4867, 1, 1, 18, 93, 2, 306, 64] payload_rows_per_dst=[8410, 9040, 10462, 12680, 6922, 9886, 14778, 10452] dup_rows_per_dst=[1136, 2198, 2500, 1760, 1476, 3050, 3302, 252] combine_rows_per_src=[8410, 9040, 10462, 12680, 6922, 9886, 14778, 10452] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=285057024 dispatch_meta_bytes=323848 combine_payload_bytes=285057024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=285380872 ingress_bytes=319660032 link_bytes=319660032 token_bytes=570437896 weight_movement_bytes=34603008 total_nvlink_bytes=605040904 weight_to_token_ratio=0.060660 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=31360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.202854966e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=290971648 dispatch_meta_bytes=318256 combine_payload_bytes=290971648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291289904 ingress_bytes=325574656 link_bytes=325574656 token_bytes=582261552 weight_movement_bytes=34603008 total_nvlink_bytes=616864560 weight_to_token_ratio=0.059429 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=19840 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.225111220e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78794 payload_rows_remote=65036 dup_rows=19510 dup_rows_remote=19180 dedup_ratio=0.198466 router_tokens_per_expert=[4, 0, 77, 10, 78, 0, 0, 0, 0, 2190, 0, 214, 92, 184, 1, 1342, 1, 6821, 2, 199, 0, 0, 53, 2, 6943, 4, 1353, 247, 3, 803, 90, 4409, 6875, 6769, 0, 799, 4, 0, 0, 0, 262, 52, 1189, 4, 0, 1483, 249, 3, 51, 1, 0, 2296, 0, 0, 0, 0, 0, 3879, 57, 0, 1, 3, 0, 53] payload_rows_per_dst=[13758, 15256, 1336, 15614, 14312, 5860, 4674, 7984] dup_rows_per_dst=[330, 6432, 28, 11240, 832, 624, 22, 2] combine_rows_per_src=[13758, 15256, 1336, 15614, 14312, 5860, 4674, 7984] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=272613376 dispatch_meta_bytes=318328 combine_payload_bytes=272613376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272931704 ingress_bytes=307216384 link_bytes=307216384 token_bytes=545545080 weight_movement_bytes=34603008 total_nvlink_bytes=580148088 weight_to_token_ratio=0.063428 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=29056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.156030520e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82474 payload_rows_remote=70748 dup_rows=15830 dup_rows_remote=15556 dedup_ratio=0.161031 router_tokens_per_expert=[74, 67, 246, 817, 89, 451, 1309, 1, 3, 1138, 1775, 33, 854, 1197, 597, 35, 116, 535, 415, 1685, 99, 66, 19, 76, 4734, 136, 4, 485, 173, 52, 160, 256, 984, 401, 773, 527, 1282, 587, 207, 97, 372, 311, 4834, 0, 204, 195, 256, 1172, 518, 640, 116, 5, 5924, 714, 192, 132, 4257, 15, 242, 245, 4401, 741, 1106, 5] payload_rows_per_dst=[11460, 8294, 11344, 11726, 8046, 12936, 14290, 4378] dup_rows_per_dst=[3162, 2970, 3480, 274, 1670, 1752, 2192, 330] combine_rows_per_src=[11460, 8294, 11344, 11726, 8046, 12936, 14290, 4378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86948 payload_rows_remote=75642 dup_rows=11356 dup_rows_remote=11352 dedup_ratio=0.115519 router_tokens_per_expert=[0, 0, 60, 1686, 0, 0, 770, 0, 0, 2847, 789, 0, 1, 1, 794, 1, 168, 461, 0, 2412, 1, 68, 0, 0, 6562, 65, 0, 318, 0, 0, 0, 1, 627, 387, 0, 1, 1450, 2240, 0, 67, 2, 0, 5579, 0, 0, 1, 72, 1, 547, 1, 67, 0, 5564, 57, 16, 4, 6931, 0, 0, 2, 6741, 759, 1031, 0] payload_rows_per_dst=[4870, 7560, 14266, 13892, 8464, 11306, 11608, 14982] dup_rows_per_dst=[162, 1306, 5436, 0, 1080, 4, 904, 2464] combine_rows_per_src=[4870, 7560, 14266, 13892, 8464, 11306, 11608, 14982] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=235741184 dispatch_meta_bytes=255848 combine_payload_bytes=235741184 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=235997032 ingress_bytes=374153216 link_bytes=374153216 token_bytes=471738216 weight_movement_bytes=138412032 total_nvlink_bytes=610150248 weight_to_token_ratio=0.293409 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=24192 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.407908430e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86422 payload_rows_remote=80016 dup_rows=11882 dup_rows_remote=11434 dedup_ratio=0.120870 router_tokens_per_expert=[0, 0, 60, 1692, 0, 0, 760, 0, 0, 2424, 879, 0, 0, 0, 719, 0, 107, 347, 0, 2595, 0, 59, 0, 0, 6532, 49, 0, 273, 0, 0, 0, 0, 703, 394, 0, 0, 1898, 1597, 0, 62, 0, 0, 5892, 0, 0, 0, 36, 0, 481, 0, 62, 0, 5938, 44, 6, 33, 6999, 0, 0, 0, 6772, 759, 980, 0] payload_rows_per_dst=[15294, 15730, 6406, 13706, 8060, 11856, 12194, 3176] dup_rows_per_dst=[3728, 5220, 448, 2, 1248, 0, 934, 302] combine_rows_per_src=[15294, 15730, 6406, 13706, 8060, 11856, 12194, 3176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=302456832 dispatch_meta_bytes=337064 combine_payload_bytes=302456832 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=302793896 ingress_bytes=371662848 link_bytes=371662848 token_bytes=605250728 weight_movement_bytes=69206016 total_nvlink_bytes=674456744 weight_to_token_ratio=0.114343 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=40832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.398537376e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=290971648 dispatch_meta_bytes=318256 combine_payload_bytes=290971648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291289904 ingress_bytes=325574656 link_bytes=325574656 token_bytes=582261552 weight_movement_bytes=34603008 total_nvlink_bytes=616864560 weight_to_token_ratio=0.059429 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=19840 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.225111220e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=297959424 dispatch_meta_bytes=341472 combine_payload_bytes=297959424 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=298300896 ingress_bytes=436371456 link_bytes=436371456 token_bytes=596260320 weight_movement_bytes=138412032 total_nvlink_bytes=734672352 weight_to_token_ratio=0.232134 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=11648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.642030658e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73482 payload_rows_remote=57554 dup_rows=24822 dup_rows_remote=6408 dedup_ratio=0.252502 router_tokens_per_expert=[0, 0, 74, 1229, 0, 0, 1129, 0, 0, 1261, 803, 0, 0, 0, 793, 0, 152, 502, 0, 1739, 0, 59, 0, 0, 5960, 0, 0, 365, 0, 0, 0, 0, 749, 412, 0, 0, 1513, 2702, 0, 86, 20, 0, 5967, 0, 0, 0, 56, 0, 454, 0, 86, 0, 6475, 86, 78, 0, 7044, 0, 0, 0, 7012, 687, 1659, 0] payload_rows_per_dst=[4706, 4452, 7994, 5006, 9660, 12044, 13692, 15928] dup_rows_per_dst=[158, 1262, 1386, 0, 1264, 42, 2296, 18414] combine_rows_per_src=[4706, 4452, 7994, 5006, 9660, 12044, 13692, 15928] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86070 payload_rows_remote=76370 dup_rows=12234 dup_rows_remote=10974 dedup_ratio=0.124451 router_tokens_per_expert=[0, 0, 150, 1261, 0, 0, 1050, 0, 0, 1317, 800, 0, 0, 0, 769, 0, 146, 520, 0, 2075, 0, 67, 0, 0, 6114, 0, 0, 366, 0, 0, 0, 0, 768, 490, 0, 0, 1379, 2751, 0, 92, 39, 0, 6015, 0, 0, 3, 73, 0, 491, 0, 92, 0, 6172, 92, 83, 3, 7017, 0, 0, 0, 6804, 679, 1474, 0] payload_rows_per_dst=[10232, 4498, 14184, 12960, 9700, 12180, 12958, 9358] dup_rows_per_dst=[1942, 1274, 5040, 0, 1260, 80, 908, 1730] combine_rows_per_src=[10232, 4498, 14184, 12960, 9700, 12180, 12958, 9358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266387456 dispatch_meta_bytes=336864 combine_payload_bytes=266387456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266724320 ingress_bytes=300990464 link_bytes=300990464 token_bytes=533111776 weight_movement_bytes=34603008 total_nvlink_bytes=567714784 weight_to_token_ratio=0.064908 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=47232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132602884e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=289783808 dispatch_meta_bytes=345216 combine_payload_bytes=289783808 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=290129024 ingress_bytes=393592832 link_bytes=393592832 token_bytes=579912832 weight_movement_bytes=103809024 total_nvlink_bytes=683721856 weight_to_token_ratio=0.179008 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=45056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.481058140e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=309829632 dispatch_meta_bytes=347976 combine_payload_bytes=309829632 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=310177608 ingress_bytes=344432640 link_bytes=344432640 token_bytes=620007240 weight_movement_bytes=34603008 total_nvlink_bytes=654610248 weight_to_token_ratio=0.055811 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=45824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.296072295e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=327745536 dispatch_meta_bytes=365800 combine_payload_bytes=327745536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=328111336 ingress_bytes=362348544 link_bytes=362348544 token_bytes=655856872 weight_movement_bytes=34603008 total_nvlink_bytes=690459880 weight_to_token_ratio=0.052760 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=27136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.363488400e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86862 payload_rows_remote=73842 dup_rows=11442 dup_rows_remote=10424 dedup_ratio=0.116394 router_tokens_per_expert=[5, 9, 91, 1099, 10, 9, 1187, 0, 0, 2074, 953, 2, 74, 107, 707, 3, 144, 507, 46, 2137, 17, 61, 3, 10, 6269, 10, 0, 406, 25, 4, 0, 52, 744, 420, 44, 19, 1661, 1881, 16, 79, 77, 52, 5853, 0, 6, 6, 135, 98, 507, 61, 80, 0, 6103, 113, 88, 67, 6706, 0, 9, 2, 6531, 698, 1075, 0] payload_rows_per_dst=[4628, 6382, 13998, 13512, 8516, 12200, 13020, 14606] dup_rows_per_dst=[192, 1458, 4914, 20, 1212, 254, 1018, 2374] combine_rows_per_src=[4628, 6382, 13998, 13512, 8516, 12200, 13020, 14606] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=235741184 dispatch_meta_bytes=255848 combine_payload_bytes=235741184 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=235997032 ingress_bytes=374153216 link_bytes=374153216 token_bytes=471738216 weight_movement_bytes=138412032 total_nvlink_bytes=610150248 weight_to_token_ratio=0.293409 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=24192 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.407908430e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=312811520 dispatch_meta_bytes=349376 combine_payload_bytes=312811520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=313160896 ingress_bytes=347414528 link_bytes=347414528 token_bytes=625972416 weight_movement_bytes=34603008 total_nvlink_bytes=660575424 weight_to_token_ratio=0.055279 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.307292900e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87654 payload_rows_remote=76362 dup_rows=10650 dup_rows_remote=8152 dedup_ratio=0.108337 router_tokens_per_expert=[2653, 0, 0, 93, 788, 5453, 0, 0, 1185, 83, 1357, 0, 0, 0, 0, 0, 0, 92, 154, 0, 0, 92, 0, 0, 284, 0, 0, 0, 0, 92, 0, 83, 0, 0, 6943, 0, 0, 0, 1200, 802, 0, 6922, 865, 7463, 66, 0, 4452, 9, 0, 0, 0, 92, 110, 0, 0, 4998, 332, 0, 0, 318, 593, 218, 252, 1108] payload_rows_per_dst=[6380, 13186, 14194, 15092, 11292, 10136, 10400, 6974] dup_rows_per_dst=[688, 2970, 326, 752, 2498, 648, 0, 2768] combine_rows_per_src=[6380, 13186, 14194, 15092, 11292, 10136, 10400, 6974] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81326 payload_rows_remote=66392 dup_rows=16978 dup_rows_remote=12158 dedup_ratio=0.172709 router_tokens_per_expert=[3083, 1, 6, 92, 763, 5931, 1, 0, 1444, 101, 1156, 0, 1, 0, 0, 0, 0, 92, 177, 5, 0, 90, 3, 1, 258, 0, 0, 3, 0, 92, 0, 101, 0, 0, 6971, 0, 0, 1, 1012, 671, 0, 6952, 881, 7503, 73, 4, 3610, 8, 2, 0, 0, 92, 19, 1, 1, 5005, 354, 6, 0, 350, 631, 171, 248, 1185] payload_rows_per_dst=[14934, 4638, 14298, 15218, 14970, 3478, 10238, 3552] dup_rows_per_dst=[4820, 766, 342, 696, 7940, 74, 2, 2338] combine_rows_per_src=[14934, 4638, 14298, 15218, 14970, 3478, 10238, 3552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=312778752 dispatch_meta_bytes=338056 combine_payload_bytes=312778752 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=313116808 ingress_bytes=381984768 link_bytes=381984768 token_bytes=625895560 weight_movement_bytes=69206016 total_nvlink_bytes=695101576 weight_to_token_ratio=0.110571 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=29184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.437377930e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68630 payload_rows_remote=53402 dup_rows=29674 dup_rows_remote=25360 dedup_ratio=0.301860 router_tokens_per_expert=[2840, 0, 0, 87, 804, 5782, 0, 0, 1392, 77, 1243, 0, 0, 0, 0, 0, 0, 87, 146, 0, 0, 86, 0, 0, 274, 0, 0, 0, 1, 87, 0, 78, 0, 0, 7012, 0, 0, 0, 1120, 750, 0, 7009, 779, 7596, 65, 0, 3984, 1, 0, 0, 0, 87, 65, 0, 0, 4941, 316, 0, 0, 331, 586, 133, 251, 1142] payload_rows_per_dst=[6412, 13542, 638, 878, 2880, 16034, 13018, 15228] dup_rows_per_dst=[652, 3446, 0, 2, 860, 15264, 5136, 4314] combine_rows_per_src=[6412, 13542, 638, 878, 2880, 16034, 13018, 15228] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76088 payload_rows_remote=62950 dup_rows=22216 dup_rows_remote=16202 dedup_ratio=0.225993 router_tokens_per_expert=[2665, 4, 117, 85, 788, 5449, 25, 16, 1468, 109, 1514, 0, 10, 3, 3, 9, 0, 78, 179, 100, 40, 79, 48, 19, 209, 0, 6, 41, 4, 79, 0, 95, 32, 0, 6477, 0, 1, 18, 1344, 774, 0, 6823, 680, 7080, 66, 75, 4454, 80, 14, 18, 13, 79, 83, 12, 34, 4869, 405, 112, 3, 355, 625, 143, 281, 960] payload_rows_per_dst=[6672, 13372, 8606, 7010, 3450, 9134, 13138, 14706] dup_rows_per_dst=[728, 3758, 352, 312, 888, 6148, 6014, 4016] combine_rows_per_src=[6672, 13372, 8606, 7010, 3450, 9134, 13138, 14706] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85270 payload_rows_remote=79452 dup_rows=13034 dup_rows_remote=11802 dedup_ratio=0.132589 router_tokens_per_expert=[3101, 0, 0, 92, 757, 5564, 0, 0, 1661, 75, 1789, 0, 0, 0, 0, 0, 0, 92, 134, 0, 0, 92, 0, 0, 256, 0, 0, 0, 3, 92, 0, 74, 0, 0, 7057, 0, 0, 0, 1395, 768, 0, 6977, 656, 7566, 58, 0, 3549, 4, 0, 0, 0, 92, 139, 0, 0, 4353, 332, 0, 0, 340, 609, 216, 227, 1032] payload_rows_per_dst=[14582, 5818, 14290, 15284, 14820, 8054, 9168, 3254] dup_rows_per_dst=[4446, 1232, 300, 698, 3620, 480, 0, 2258] combine_rows_per_src=[14582, 5818, 14290, 15284, 14820, 8054, 9168, 3254] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=302456832 dispatch_meta_bytes=337064 combine_payload_bytes=302456832 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=302793896 ingress_bytes=371662848 link_bytes=371662848 token_bytes=605250728 weight_movement_bytes=69206016 total_nvlink_bytes=674456744 weight_to_token_ratio=0.114343 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=40832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.398537376e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=325435392 dispatch_meta_bytes=365016 combine_payload_bytes=325435392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=325800408 ingress_bytes=360038400 link_bytes=360038400 token_bytes=651235800 weight_movement_bytes=34603008 total_nvlink_bytes=685838808 weight_to_token_ratio=0.053134 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=24960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.354795514e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81668 payload_rows_remote=66556 dup_rows=16636 dup_rows_remote=13026 dedup_ratio=0.169230 router_tokens_per_expert=[0, 1, 118, 828, 0, 5, 1382, 0, 0, 1199, 927, 0, 2, 6, 702, 0, 161, 444, 5, 1494, 0, 53, 0, 0, 6138, 0, 0, 388, 0, 0, 0, 1, 827, 457, 2, 3, 1322, 2929, 0, 90, 92, 3, 5936, 0, 0, 8, 29, 4, 488, 2, 90, 0, 6455, 91, 100, 50, 7027, 0, 2, 1, 6743, 801, 1746, 0] payload_rows_per_dst=[15112, 15918, 3914, 13160, 16056, 11382, 1362, 4764] dup_rows_per_dst=[3610, 3240, 400, 478, 8114, 178, 280, 336] combine_rows_per_src=[15112, 15918, 3914, 13160, 16056, 11382, 1362, 4764] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78146 payload_rows_remote=63902 dup_rows=20158 dup_rows_remote=12806 dedup_ratio=0.205058 router_tokens_per_expert=[2560, 5078, 14, 276, 703, 104, 64, 206, 5, 459, 178, 390, 60, 325, 410, 1069, 390, 726, 1, 1, 492, 114, 1806, 924, 2032, 3630, 1345, 343, 256, 28, 105, 499, 0, 265, 1541, 1, 67, 2217, 748, 109, 653, 81, 1, 89, 3141, 6399, 433, 1, 4504, 2, 666, 482, 482, 85, 341, 86, 205, 421, 1, 0, 0, 1120, 120, 298] payload_rows_per_dst=[2658, 5042, 9462, 10536, 11776, 14244, 11872, 12556] dup_rows_per_dst=[76, 750, 2944, 2442, 3240, 7352, 1424, 1930] combine_rows_per_src=[2658, 5042, 9462, 10536, 11776, 14244, 11872, 12556] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=271941632 dispatch_meta_bytes=314200 combine_payload_bytes=271941632 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=272255832 ingress_bytes=341147648 link_bytes=341147648 token_bytes=544197464 weight_movement_bytes=69206016 total_nvlink_bytes=613403480 weight_to_token_ratio=0.127171 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=22016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.283711135e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=218734592 dispatch_meta_bytes=315048 combine_payload_bytes=218734592 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=219049640 ingress_bytes=253337600 link_bytes=253337600 token_bytes=437784232 weight_movement_bytes=34603008 total_nvlink_bytes=472387240 weight_to_token_ratio=0.079041 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22528 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.532889935e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81064 payload_rows_remote=70474 dup_rows=17240 dup_rows_remote=16100 dedup_ratio=0.175374 router_tokens_per_expert=[1329, 64, 1378, 88, 759, 5451, 413, 173, 2032, 233, 499, 15, 96, 84, 79, 58, 2, 65, 944, 972, 294, 106, 536, 210, 497, 35, 71, 368, 52, 102, 27, 245, 471, 1, 3391, 1, 22, 276, 1884, 497, 4, 4892, 625, 4468, 94, 776, 3589, 131, 155, 500, 93, 78, 37, 155, 433, 1890, 1348, 1228, 62, 955, 817, 189, 506, 2307] payload_rows_per_dst=[9500, 10888, 13688, 10590, 10538, 9890, 6354, 9616] dup_rows_per_dst=[1998, 3116, 2354, 1140, 2548, 548, 328, 5208] combine_rows_per_src=[9500, 10888, 13688, 10590, 10538, 9890, 6354, 9616] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=325435392 dispatch_meta_bytes=365016 combine_payload_bytes=325435392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=325800408 ingress_bytes=360038400 link_bytes=360038400 token_bytes=651235800 weight_movement_bytes=34603008 total_nvlink_bytes=685838808 weight_to_token_ratio=0.053134 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=24960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.354795514e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86332 payload_rows_remote=72050 dup_rows=11972 dup_rows_remote=11774 dedup_ratio=0.121785 router_tokens_per_expert=[3282, 0, 0, 94, 804, 4890, 0, 0, 1322, 52, 1805, 0, 0, 0, 0, 0, 0, 62, 89, 0, 0, 62, 0, 0, 151, 0, 0, 0, 0, 62, 0, 49, 0, 0, 6856, 0, 0, 0, 1261, 743, 0, 7027, 494, 7402, 26, 0, 4679, 61, 0, 0, 0, 62, 60, 0, 0, 5154, 299, 0, 0, 422, 658, 119, 256, 849] payload_rows_per_dst=[14060, 5246, 14282, 14902, 14488, 10162, 10552, 2640] dup_rows_per_dst=[4080, 1112, 198, 426, 3232, 358, 0, 2566] combine_rows_per_src=[14060, 5246, 14282, 14902, 14488, 10162, 10552, 2640] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68630 payload_rows_remote=53402 dup_rows=29674 dup_rows_remote=25360 dedup_ratio=0.301860 router_tokens_per_expert=[2840, 0, 0, 87, 804, 5782, 0, 0, 1392, 77, 1243, 0, 0, 0, 0, 0, 0, 87, 146, 0, 0, 86, 0, 0, 274, 0, 0, 0, 1, 87, 0, 78, 0, 0, 7012, 0, 0, 0, 1120, 750, 0, 7009, 779, 7596, 65, 0, 3984, 1, 0, 0, 0, 87, 65, 0, 0, 4941, 316, 0, 0, 331, 586, 133, 251, 1142] payload_rows_per_dst=[6412, 13542, 638, 878, 2880, 16034, 13018, 15228] dup_rows_per_dst=[652, 3446, 0, 2, 860, 15264, 5136, 4314] combine_rows_per_src=[6412, 13542, 638, 878, 2880, 16034, 13018, 15228] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86508 payload_rows_remote=78954 dup_rows=11796 dup_rows_remote=11454 dedup_ratio=0.119995 router_tokens_per_expert=[3218, 0, 1, 79, 878, 4313, 0, 0, 1445, 71, 1778, 0, 0, 0, 0, 0, 0, 67, 133, 1, 0, 67, 1, 0, 160, 0, 0, 1, 0, 67, 0, 70, 0, 0, 6908, 4, 0, 0, 1343, 793, 0, 6949, 596, 7354, 58, 1, 4616, 67, 0, 0, 0, 67, 65, 0, 0, 5456, 334, 1, 0, 340, 630, 150, 253, 817] payload_rows_per_dst=[7674, 11986, 14180, 14850, 3332, 7554, 11916, 15016] dup_rows_per_dst=[678, 3228, 256, 454, 948, 342, 2040, 3850] combine_rows_per_src=[7674, 11986, 14180, 14850, 3332, 7554, 11916, 15016] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=257843200 dispatch_meta_bytes=316608 combine_payload_bytes=257843200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258159808 ingress_bytes=292446208 link_bytes=292446208 token_bytes=516003008 weight_movement_bytes=34603008 total_nvlink_bytes=550606016 weight_to_token_ratio=0.067060 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=36736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.100451537e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87654 payload_rows_remote=76362 dup_rows=10650 dup_rows_remote=8152 dedup_ratio=0.108337 router_tokens_per_expert=[2653, 0, 0, 93, 788, 5453, 0, 0, 1185, 83, 1357, 0, 0, 0, 0, 0, 0, 92, 154, 0, 0, 92, 0, 0, 284, 0, 0, 0, 0, 92, 0, 83, 0, 0, 6943, 0, 0, 0, 1200, 802, 0, 6922, 865, 7463, 66, 0, 4452, 9, 0, 0, 0, 92, 110, 0, 0, 4998, 332, 0, 0, 318, 593, 218, 252, 1108] payload_rows_per_dst=[6380, 13186, 14194, 15092, 11292, 10136, 10400, 6974] dup_rows_per_dst=[688, 2970, 326, 752, 2498, 648, 0, 2768] combine_rows_per_src=[6380, 13186, 14194, 15092, 11292, 10136, 10400, 6974] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=272613376 dispatch_meta_bytes=318328 combine_payload_bytes=272613376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272931704 ingress_bytes=307216384 link_bytes=307216384 token_bytes=545545080 weight_movement_bytes=34603008 total_nvlink_bytes=580148088 weight_to_token_ratio=0.063428 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=29056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.156030520e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=295116800 dispatch_meta_bytes=335296 combine_payload_bytes=295116800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=295452096 ingress_bytes=329719808 link_bytes=329719808 token_bytes=590568896 weight_movement_bytes=34603008 total_nvlink_bytes=625171904 weight_to_token_ratio=0.058593 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=43776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.240709093e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=261742592 dispatch_meta_bytes=306832 combine_payload_bytes=261742592 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=262049424 ingress_bytes=365551616 link_bytes=365551616 token_bytes=523792016 weight_movement_bytes=103809024 total_nvlink_bytes=627601040 weight_to_token_ratio=0.198187 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=20224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.375541302e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76088 payload_rows_remote=62950 dup_rows=22216 dup_rows_remote=16202 dedup_ratio=0.225993 router_tokens_per_expert=[2665, 4, 117, 85, 788, 5449, 25, 16, 1468, 109, 1514, 0, 10, 3, 3, 9, 0, 78, 179, 100, 40, 79, 48, 19, 209, 0, 6, 41, 4, 79, 0, 95, 32, 0, 6477, 0, 1, 18, 1344, 774, 0, 6823, 680, 7080, 66, 75, 4454, 80, 14, 18, 13, 79, 83, 12, 34, 4869, 405, 112, 3, 355, 625, 143, 281, 960] payload_rows_per_dst=[6672, 13372, 8606, 7010, 3450, 9134, 13138, 14706] dup_rows_per_dst=[728, 3758, 352, 312, 888, 6148, 6014, 4016] combine_rows_per_src=[6672, 13372, 8606, 7010, 3450, 9134, 13138, 14706] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82888 payload_rows_remote=73732 dup_rows=15416 dup_rows_remote=15416 dedup_ratio=0.156820 router_tokens_per_expert=[5445, 1580, 0, 0, 0, 0, 2430, 91, 0, 2251, 0, 4060, 0, 0, 971, 0, 0, 184, 3551, 0, 0, 350, 0, 0, 0, 0, 2257, 611, 1130, 0, 397, 349, 0, 0, 0, 0, 0, 0, 0, 4578, 0, 0, 406, 93, 1935, 111, 7000, 0, 6514, 53, 0, 160, 0, 92, 0, 0, 83, 0, 0, 0, 1114, 89, 0, 1267] payload_rows_per_dst=[6616, 12102, 13944, 8290, 9156, 7990, 13212, 11578] dup_rows_per_dst=[1586, 2462, 5116, 1198, 0, 1288, 430, 3336] combine_rows_per_src=[6616, 12102, 13944, 8290, 9156, 7990, 13212, 11578] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=288661504 dispatch_meta_bytes=346296 combine_payload_bytes=288661504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289007800 ingress_bytes=323264512 link_bytes=323264512 token_bytes=577669304 weight_movement_bytes=34603008 total_nvlink_bytes=612272312 weight_to_token_ratio=0.059901 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=45056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.216418334e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=295116800 dispatch_meta_bytes=335296 combine_payload_bytes=295116800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=295452096 ingress_bytes=329719808 link_bytes=329719808 token_bytes=590568896 weight_movement_bytes=34603008 total_nvlink_bytes=625171904 weight_to_token_ratio=0.058593 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=43776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.240709093e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=323395584 dispatch_meta_bytes=361632 combine_payload_bytes=323395584 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=323757216 ingress_bytes=496410624 link_bytes=496410624 token_bytes=647152800 weight_movement_bytes=173015040 total_nvlink_bytes=820167840 weight_to_token_ratio=0.267348 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.867953214e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76602 payload_rows_remote=62308 dup_rows=21702 dup_rows_remote=11002 dedup_ratio=0.220764 router_tokens_per_expert=[4970, 2290, 0, 0, 1, 0, 2474, 90, 1, 1705, 1, 3627, 2, 0, 894, 0, 2, 182, 3289, 0, 0, 351, 0, 0, 0, 4, 2417, 583, 886, 0, 411, 369, 6, 3, 1, 0, 4, 0, 0, 4618, 0, 5, 438, 164, 1998, 95, 7028, 3, 6538, 6, 1, 146, 0, 90, 5, 6, 100, 0, 0, 6, 2090, 92, 0, 1160] payload_rows_per_dst=[14294, 14060, 7648, 8120, 9250, 11536, 5358, 6336] dup_rows_per_dst=[10700, 6418, 0, 1220, 14, 2582, 208, 560] combine_rows_per_src=[14294, 14060, 7648, 8120, 9250, 11536, 5358, 6336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81334 payload_rows_remote=66430 dup_rows=16970 dup_rows_remote=11522 dedup_ratio=0.172628 router_tokens_per_expert=[5399, 1367, 0, 0, 0, 0, 2582, 86, 0, 2108, 0, 3779, 0, 0, 992, 0, 0, 173, 3747, 0, 0, 335, 0, 0, 0, 0, 2069, 638, 1259, 0, 336, 352, 0, 0, 0, 0, 0, 0, 0, 4628, 0, 0, 424, 136, 1786, 105, 7155, 0, 6432, 1, 0, 156, 0, 86, 0, 0, 78, 0, 0, 0, 1789, 86, 0, 1068] payload_rows_per_dst=[6748, 6116, 8510, 13064, 14320, 4628, 13044, 14904] dup_rows_per_dst=[1322, 84, 0, 3802, 5734, 274, 306, 5448] combine_rows_per_src=[6748, 6116, 8510, 13064, 14320, 4628, 13044, 14904] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=302006272 dispatch_meta_bytes=356592 combine_payload_bytes=302006272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=302362864 ingress_bytes=336609280 link_bytes=336609280 token_bytes=604369136 weight_movement_bytes=34603008 total_nvlink_bytes=638972144 weight_to_token_ratio=0.057255 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=35968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266633621e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85516 payload_rows_remote=74430 dup_rows=12788 dup_rows_remote=10842 dedup_ratio=0.130086 router_tokens_per_expert=[4843, 1328, 0, 0, 0, 0, 2867, 92, 0, 2314, 0, 3252, 0, 0, 950, 0, 0, 184, 4238, 0, 0, 358, 0, 0, 0, 0, 1668, 578, 1600, 0, 415, 399, 0, 0, 0, 0, 0, 0, 0, 4868, 0, 0, 480, 121, 1777, 111, 7172, 0, 5971, 12, 0, 170, 0, 92, 0, 0, 76, 0, 0, 0, 1889, 92, 0, 1235] payload_rows_per_dst=[13240, 11086, 9560, 8094, 9736, 15636, 12136, 6028] dup_rows_per_dst=[5020, 1946, 0, 1226, 0, 3686, 354, 556] combine_rows_per_src=[13240, 11086, 9560, 8094, 9736, 15636, 12136, 6028] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=218734592 dispatch_meta_bytes=315048 combine_payload_bytes=218734592 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=219049640 ingress_bytes=253337600 link_bytes=253337600 token_bytes=437784232 weight_movement_bytes=34603008 total_nvlink_bytes=472387240 weight_to_token_ratio=0.079041 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22528 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.532889935e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=312778752 dispatch_meta_bytes=338056 combine_payload_bytes=312778752 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=313116808 ingress_bytes=381984768 link_bytes=381984768 token_bytes=625895560 weight_movement_bytes=69206016 total_nvlink_bytes=695101576 weight_to_token_ratio=0.110571 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=29184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.437377930e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80984 payload_rows_remote=67538 dup_rows=17320 dup_rows_remote=16844 dedup_ratio=0.176188 router_tokens_per_expert=[5600, 1276, 1, 0, 10, 9, 1792, 89, 9, 2378, 46, 4172, 13, 8, 1123, 0, 25, 164, 3201, 17, 16, 359, 10, 17, 0, 24, 1858, 660, 1419, 64, 453, 357, 114, 68, 10, 0, 101, 16, 0, 4365, 10, 97, 370, 102, 1905, 156, 6861, 46, 6576, 54, 4, 161, 0, 79, 8, 79, 82, 0, 0, 125, 1367, 80, 4, 1142] payload_rows_per_dst=[5140, 6950, 9698, 13606, 12502, 4960, 13446, 14682] dup_rows_per_dst=[1214, 204, 1138, 4408, 4828, 412, 476, 4640] combine_rows_per_src=[5140, 6950, 9698, 13606, 12502, 4960, 13446, 14682] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=257843200 dispatch_meta_bytes=316608 combine_payload_bytes=257843200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258159808 ingress_bytes=292446208 link_bytes=292446208 token_bytes=516003008 weight_movement_bytes=34603008 total_nvlink_bytes=550606016 weight_to_token_ratio=0.067060 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=36736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.100451537e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=304865280 dispatch_meta_bytes=341088 combine_payload_bytes=304865280 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=305206368 ingress_bytes=374071296 link_bytes=374071296 token_bytes=610071648 weight_movement_bytes=69206016 total_nvlink_bytes=679277664 weight_to_token_ratio=0.113439 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18304 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.407600172e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81326 payload_rows_remote=66392 dup_rows=16978 dup_rows_remote=12158 dedup_ratio=0.172709 router_tokens_per_expert=[3083, 1, 6, 92, 763, 5931, 1, 0, 1444, 101, 1156, 0, 1, 0, 0, 0, 0, 92, 177, 5, 0, 90, 3, 1, 258, 0, 0, 3, 0, 92, 0, 101, 0, 0, 6971, 0, 0, 1, 1012, 671, 0, 6952, 881, 7503, 73, 4, 3610, 8, 2, 0, 0, 92, 19, 1, 1, 5005, 354, 6, 0, 350, 631, 171, 248, 1185] payload_rows_per_dst=[14934, 4638, 14298, 15218, 14970, 3478, 10238, 3552] dup_rows_per_dst=[4820, 766, 342, 696, 7940, 74, 2, 2338] combine_rows_per_src=[14934, 4638, 14298, 15218, 14970, 3478, 10238, 3552] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86434 payload_rows_remote=77770 dup_rows=11870 dup_rows_remote=11870 dedup_ratio=0.120748 router_tokens_per_expert=[5842, 682, 0, 0, 0, 0, 1595, 62, 0, 2545, 0, 4446, 0, 0, 1390, 0, 0, 124, 3872, 0, 0, 336, 0, 0, 0, 0, 1895, 681, 1099, 0, 287, 316, 0, 0, 0, 0, 0, 0, 0, 5223, 0, 0, 182, 70, 2133, 131, 7237, 0, 6806, 29, 0, 125, 0, 62, 0, 0, 52, 0, 0, 0, 856, 62, 0, 1012] payload_rows_per_dst=[13670, 13324, 8664, 7694, 10446, 15248, 13736, 3652] dup_rows_per_dst=[2692, 3438, 0, 862, 0, 4258, 308, 312] combine_rows_per_src=[13670, 13324, 8664, 7694, 10446, 15248, 13736, 3652] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=255213568 dispatch_meta_bytes=293240 combine_payload_bytes=255213568 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=255506808 ingress_bytes=359022592 link_bytes=359022592 token_bytes=510720376 weight_movement_bytes=103809024 total_nvlink_bytes=614529400 weight_to_token_ratio=0.203260 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.350973110e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=272097280 dispatch_meta_bytes=311808 combine_payload_bytes=272097280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272409088 ingress_bytes=306700288 link_bytes=306700288 token_bytes=544506368 weight_movement_bytes=34603008 total_nvlink_bytes=579109376 weight_to_token_ratio=0.063549 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=26112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.154088492e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78358 payload_rows_remote=68888 dup_rows=19946 dup_rows_remote=18212 dedup_ratio=0.202901 router_tokens_per_expert=[3903, 2830, 38, 13, 278, 305, 1252, 150, 221, 1639, 526, 1950, 245, 108, 612, 3, 273, 221, 1899, 251, 233, 391, 112, 321, 53, 244, 1354, 764, 1465, 370, 425, 927, 1346, 915, 132, 6, 1095, 63, 11, 2216, 111, 1182, 1175, 112, 2890, 279, 4377, 679, 4594, 101, 35, 258, 0, 95, 116, 392, 82, 76, 15, 1341, 935, 154, 92, 901] payload_rows_per_dst=[7988, 8874, 11184, 9470, 8774, 14682, 10656, 6730] dup_rows_per_dst=[2888, 1734, 2880, 1734, 2794, 6928, 526, 462] combine_rows_per_src=[7988, 8874, 11184, 9470, 8774, 14682, 10656, 6730] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77790 payload_rows_remote=64456 dup_rows=20514 dup_rows_remote=19838 dedup_ratio=0.208679 router_tokens_per_expert=[577, 507, 3518, 3903, 1201, 0, 6037, 4050, 5, 134, 0, 381, 1, 2018, 0, 181, 323, 343, 1883, 121, 967, 54, 468, 149, 62, 67, 1443, 0, 4, 130, 1, 542, 5729, 1065, 0, 67, 202, 1474, 542, 1, 394, 0, 81, 0, 12, 1, 100, 380, 2005, 1, 11, 332, 12, 1908, 174, 0, 1497, 30, 72, 2456, 1505, 27, 4, 0] payload_rows_per_dst=[5054, 5136, 11808, 13264, 7776, 13334, 12514, 8904] dup_rows_per_dst=[1094, 304, 3844, 4976, 2556, 676, 4786, 2278] combine_rows_per_src=[5054, 5136, 11808, 13264, 7776, 13334, 12514, 8904] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81334 payload_rows_remote=66430 dup_rows=16970 dup_rows_remote=11522 dedup_ratio=0.172628 router_tokens_per_expert=[5399, 1367, 0, 0, 0, 0, 2582, 86, 0, 2108, 0, 3779, 0, 0, 992, 0, 0, 173, 3747, 0, 0, 335, 0, 0, 0, 0, 2069, 638, 1259, 0, 336, 352, 0, 0, 0, 0, 0, 0, 0, 4628, 0, 0, 424, 136, 1786, 105, 7155, 0, 6432, 1, 0, 156, 0, 86, 0, 0, 78, 0, 0, 0, 1789, 86, 0, 1068] payload_rows_per_dst=[6748, 6116, 8510, 13064, 14320, 4628, 13044, 14904] dup_rows_per_dst=[1322, 84, 0, 3802, 5734, 274, 306, 5448] combine_rows_per_src=[6748, 6116, 8510, 13064, 14320, 4628, 13044, 14904] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82888 payload_rows_remote=73732 dup_rows=15416 dup_rows_remote=15416 dedup_ratio=0.156820 router_tokens_per_expert=[5445, 1580, 0, 0, 0, 0, 2430, 91, 0, 2251, 0, 4060, 0, 0, 971, 0, 0, 184, 3551, 0, 0, 350, 0, 0, 0, 0, 2257, 611, 1130, 0, 397, 349, 0, 0, 0, 0, 0, 0, 0, 4578, 0, 0, 406, 93, 1935, 111, 7000, 0, 6514, 53, 0, 160, 0, 92, 0, 0, 83, 0, 0, 0, 1114, 89, 0, 1267] payload_rows_per_dst=[6616, 12102, 13944, 8290, 9156, 7990, 13212, 11578] dup_rows_per_dst=[1586, 2462, 5116, 1198, 0, 1288, 430, 3336] combine_rows_per_src=[6616, 12102, 13944, 8290, 9156, 7990, 13212, 11578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86434 payload_rows_remote=77770 dup_rows=11870 dup_rows_remote=11870 dedup_ratio=0.120748 router_tokens_per_expert=[5842, 682, 0, 0, 0, 0, 1595, 62, 0, 2545, 0, 4446, 0, 0, 1390, 0, 0, 124, 3872, 0, 0, 336, 0, 0, 0, 0, 1895, 681, 1099, 0, 287, 316, 0, 0, 0, 0, 0, 0, 0, 5223, 0, 0, 182, 70, 2133, 131, 7237, 0, 6806, 29, 0, 125, 0, 62, 0, 0, 52, 0, 0, 0, 856, 62, 0, 1012] payload_rows_per_dst=[13670, 13324, 8664, 7694, 10446, 15248, 13736, 3652] dup_rows_per_dst=[2692, 3438, 0, 862, 0, 4258, 308, 312] combine_rows_per_src=[13670, 13324, 8664, 7694, 10446, 15248, 13736, 3652] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=304865280 dispatch_meta_bytes=341088 combine_payload_bytes=304865280 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=305206368 ingress_bytes=374071296 link_bytes=374071296 token_bytes=610071648 weight_movement_bytes=69206016 total_nvlink_bytes=679277664 weight_to_token_ratio=0.113439 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18304 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.407600172e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83646 payload_rows_remote=79452 dup_rows=14658 dup_rows_remote=14656 dedup_ratio=0.149109 router_tokens_per_expert=[5369, 822, 0, 0, 0, 0, 1704, 67, 0, 3274, 0, 4024, 27, 0, 1240, 0, 0, 134, 4066, 2, 0, 403, 6, 1, 0, 0, 1502, 808, 992, 0, 312, 332, 1, 0, 0, 0, 1, 0, 0, 5373, 0, 1, 268, 97, 1564, 167, 7155, 1, 6728, 51, 0, 146, 0, 67, 18, 44, 71, 0, 0, 1, 1232, 67, 0, 1014] payload_rows_per_dst=[4362, 8706, 14420, 12560, 10748, 4194, 13716, 14940] dup_rows_per_dst=[824, 800, 5542, 2956, 2, 2, 392, 4140] combine_rows_per_src=[4362, 8706, 14420, 12560, 10748, 4194, 13716, 14940] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=276635648 dispatch_meta_bytes=337528 combine_payload_bytes=276635648 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=276973176 ingress_bytes=345841664 link_bytes=345841664 token_bytes=553608824 weight_movement_bytes=69206016 total_nvlink_bytes=622814840 weight_to_token_ratio=0.125009 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=46208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.301374339e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=271941632 dispatch_meta_bytes=314200 combine_payload_bytes=271941632 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=272255832 ingress_bytes=341147648 link_bytes=341147648 token_bytes=544197464 weight_movement_bytes=69206016 total_nvlink_bytes=613403480 weight_to_token_ratio=0.127171 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=22016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.283711135e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=318545920 dispatch_meta_bytes=358560 combine_payload_bytes=318545920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=318904480 ingress_bytes=353148928 link_bytes=353148928 token_bytes=637450400 weight_movement_bytes=34603008 total_nvlink_bytes=672053408 weight_to_token_ratio=0.054283 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=27904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.328870985e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80984 payload_rows_remote=67538 dup_rows=17320 dup_rows_remote=16844 dedup_ratio=0.176188 router_tokens_per_expert=[5600, 1276, 1, 0, 10, 9, 1792, 89, 9, 2378, 46, 4172, 13, 8, 1123, 0, 25, 164, 3201, 17, 16, 359, 10, 17, 0, 24, 1858, 660, 1419, 64, 453, 357, 114, 68, 10, 0, 101, 16, 0, 4365, 10, 97, 370, 102, 1905, 156, 6861, 46, 6576, 54, 4, 161, 0, 79, 8, 79, 82, 0, 0, 125, 1367, 80, 4, 1142] payload_rows_per_dst=[5140, 6950, 9698, 13606, 12502, 4960, 13446, 14682] dup_rows_per_dst=[1214, 204, 1138, 4408, 4828, 412, 476, 4640] combine_rows_per_src=[5140, 6950, 9698, 13606, 12502, 4960, 13446, 14682] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=282165248 dispatch_meta_bytes=348400 combine_payload_bytes=282165248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282513648 ingress_bytes=316768256 link_bytes=316768256 token_bytes=564678896 weight_movement_bytes=34603008 total_nvlink_bytes=599281904 weight_to_token_ratio=0.061279 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=15104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.191973445e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86782 payload_rows_remote=73582 dup_rows=11522 dup_rows_remote=11522 dedup_ratio=0.117208 router_tokens_per_expert=[0, 73, 2480, 0, 0, 0, 3606, 0, 0, 0, 161, 1057, 7102, 0, 0, 2349, 0, 0, 0, 0, 0, 1465, 2, 0, 0, 0, 0, 7779, 0, 0, 3738, 0, 0, 0, 161, 0, 0, 6437, 0, 2, 0, 0, 1900, 0, 0, 0, 0, 321, 0, 0, 308, 0, 0, 2787, 0, 92, 0, 6379, 0, 0, 620, 333, 0, 0] payload_rows_per_dst=[10984, 6822, 14426, 7476, 13200, 15888, 9488, 8498] dup_rows_per_dst=[1334, 312, 2712, 0, 0, 4112, 2976, 76] combine_rows_per_src=[10984, 6822, 14426, 7476, 13200, 15888, 9488, 8498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=318545920 dispatch_meta_bytes=358560 combine_payload_bytes=318545920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=318904480 ingress_bytes=353148928 link_bytes=353148928 token_bytes=637450400 weight_movement_bytes=34603008 total_nvlink_bytes=672053408 weight_to_token_ratio=0.054283 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=27904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.328870985e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=325435392 dispatch_meta_bytes=376432 combine_payload_bytes=325435392 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=325811824 ingress_bytes=429244416 link_bytes=429244416 token_bytes=651247216 weight_movement_bytes=103809024 total_nvlink_bytes=755056240 weight_to_token_ratio=0.159400 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=25344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.615212180e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76806 payload_rows_remote=64362 dup_rows=21498 dup_rows_remote=17920 dedup_ratio=0.218689 router_tokens_per_expert=[0, 85, 2760, 0, 0, 0, 4258, 0, 0, 0, 167, 892, 7113, 1, 6, 2370, 0, 0, 0, 0, 0, 1411, 3, 3, 1, 0, 0, 7766, 1, 7, 3627, 0, 0, 1, 167, 0, 0, 6681, 1, 1, 1, 0, 1024, 0, 1, 6, 3, 267, 0, 3, 286, 0, 0, 2715, 3, 92, 1, 6421, 0, 0, 437, 566, 3, 1] payload_rows_per_dst=[12444, 15646, 2834, 15736, 11886, 2590, 13662, 2008] dup_rows_per_dst=[3578, 5452, 0, 7068, 0, 14, 5378, 8] combine_rows_per_src=[12444, 15646, 2834, 15736, 11886, 2590, 13662, 2008] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76814 payload_rows_remote=60548 dup_rows=21490 dup_rows_remote=7254 dedup_ratio=0.218608 router_tokens_per_expert=[0, 11, 2547, 0, 0, 0, 3741, 0, 0, 0, 154, 855, 7158, 0, 0, 2265, 0, 0, 0, 0, 0, 1246, 3, 0, 0, 0, 0, 7787, 0, 0, 3840, 0, 0, 0, 154, 0, 0, 6924, 0, 0, 0, 0, 1542, 0, 0, 0, 0, 366, 0, 0, 309, 0, 0, 2699, 0, 87, 0, 6441, 0, 0, 517, 506, 0, 0] payload_rows_per_dst=[11046, 5914, 6326, 13838, 14234, 3770, 5420, 16266] dup_rows_per_dst=[1552, 208, 514, 3816, 348, 46, 770, 14236] combine_rows_per_src=[11046, 5914, 6326, 13838, 14234, 3770, 5420, 16266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=301391872 dispatch_meta_bytes=340416 combine_payload_bytes=301391872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=301732288 ingress_bytes=370597888 link_bytes=370597888 token_bytes=603124160 weight_movement_bytes=69206016 total_nvlink_bytes=672330176 weight_to_token_ratio=0.114746 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=46720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394530017e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78666 payload_rows_remote=63204 dup_rows=19638 dup_rows_remote=15456 dedup_ratio=0.199768 router_tokens_per_expert=[0, 5, 2427, 0, 0, 0, 4514, 0, 0, 0, 150, 639, 7079, 0, 0, 1954, 1, 0, 0, 0, 0, 1464, 0, 0, 0, 0, 0, 7815, 0, 0, 3377, 0, 0, 0, 150, 0, 0, 6363, 0, 0, 0, 0, 1372, 0, 0, 0, 0, 611, 0, 0, 256, 0, 0, 3449, 0, 92, 0, 6491, 0, 0, 496, 447, 0, 0] payload_rows_per_dst=[11726, 15462, 2930, 15814, 13026, 3920, 13902, 1886] dup_rows_per_dst=[2166, 4182, 0, 6570, 0, 46, 6674, 0] combine_rows_per_src=[11726, 15462, 2930, 15814, 13026, 3920, 13902, 1886] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=264011776 dispatch_meta_bytes=337176 combine_payload_bytes=264011776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264348952 ingress_bytes=298614784 link_bytes=298614784 token_bytes=528360728 weight_movement_bytes=34603008 total_nvlink_bytes=562963736 weight_to_token_ratio=0.065491 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=39680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123663392e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=272097280 dispatch_meta_bytes=311808 combine_payload_bytes=272097280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272409088 ingress_bytes=306700288 link_bytes=306700288 token_bytes=544506368 weight_movement_bytes=34603008 total_nvlink_bytes=579109376 weight_to_token_ratio=0.063549 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=26112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.154088492e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=302006272 dispatch_meta_bytes=356592 combine_payload_bytes=302006272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=302362864 ingress_bytes=336609280 link_bytes=336609280 token_bytes=604369136 weight_movement_bytes=34603008 total_nvlink_bytes=638972144 weight_to_token_ratio=0.057255 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=35968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266633621e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=276635648 dispatch_meta_bytes=337528 combine_payload_bytes=276635648 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=276973176 ingress_bytes=345841664 link_bytes=345841664 token_bytes=553608824 weight_movement_bytes=69206016 total_nvlink_bytes=622814840 weight_to_token_ratio=0.125009 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=46208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.301374339e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=258883584 dispatch_meta_bytes=314640 combine_payload_bytes=258883584 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=259198224 ingress_bytes=362692608 link_bytes=362692608 token_bytes=518081808 weight_movement_bytes=103809024 total_nvlink_bytes=621890832 weight_to_token_ratio=0.200372 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.364783085e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76602 payload_rows_remote=62308 dup_rows=21702 dup_rows_remote=11002 dedup_ratio=0.220764 router_tokens_per_expert=[4970, 2290, 0, 0, 1, 0, 2474, 90, 1, 1705, 1, 3627, 2, 0, 894, 0, 2, 182, 3289, 0, 0, 351, 0, 0, 0, 4, 2417, 583, 886, 0, 411, 369, 6, 3, 1, 0, 4, 0, 0, 4618, 0, 5, 438, 164, 1998, 95, 7028, 3, 6538, 6, 1, 146, 0, 90, 5, 6, 100, 0, 0, 6, 2090, 92, 0, 1160] payload_rows_per_dst=[14294, 14060, 7648, 8120, 9250, 11536, 5358, 6336] dup_rows_per_dst=[10700, 6418, 0, 1220, 14, 2582, 208, 560] combine_rows_per_src=[14294, 14060, 7648, 8120, 9250, 11536, 5358, 6336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79604 payload_rows_remote=75646 dup_rows=18700 dup_rows_remote=18020 dedup_ratio=0.190226 router_tokens_per_expert=[1, 89, 2402, 25, 0, 2, 2945, 1, 0, 0, 144, 1791, 7030, 2, 131, 3519, 8, 1, 3, 13, 7, 1915, 52, 76, 18, 0, 15, 7574, 11, 130, 2722, 8, 7, 12, 156, 0, 5, 5661, 26, 11, 1, 0, 1950, 0, 12, 136, 78, 348, 1, 59, 368, 23, 9, 1677, 103, 79, 18, 6420, 0, 38, 732, 471, 72, 44] payload_rows_per_dst=[9886, 10156, 14538, 5754, 11712, 7832, 3958, 15768] dup_rows_per_dst=[1044, 1018, 3672, 54, 44, 1162, 680, 11026] combine_rows_per_src=[9886, 10156, 14538, 5754, 11712, 7832, 3958, 15768] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=263626752 dispatch_meta_bytes=329128 combine_payload_bytes=263626752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263955880 ingress_bytes=298229760 link_bytes=298229760 token_bytes=527582632 weight_movement_bytes=34603008 total_nvlink_bytes=562185640 weight_to_token_ratio=0.065588 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=27520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.122214577e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=248004608 dispatch_meta_bytes=271208 combine_payload_bytes=248004608 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=248275816 ingress_bytes=317210624 link_bytes=317210624 token_bytes=496280424 weight_movement_bytes=69206016 total_nvlink_bytes=565486440 weight_to_token_ratio=0.139449 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=23040 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.193638041e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79704 payload_rows_remote=68048 dup_rows=18600 dup_rows_remote=14486 dedup_ratio=0.189209 router_tokens_per_expert=[0, 35, 1751, 0, 0, 0, 2879, 0, 0, 0, 103, 1599, 7334, 0, 0, 3431, 0, 0, 0, 0, 0, 3011, 0, 0, 0, 0, 0, 7864, 0, 0, 2641, 0, 0, 0, 100, 0, 0, 5329, 0, 0, 0, 0, 2897, 0, 0, 0, 0, 266, 0, 0, 417, 0, 0, 1678, 0, 62, 0, 6944, 0, 0, 729, 82, 0, 0] payload_rows_per_dst=[8630, 11354, 11656, 7928, 10858, 13330, 14334, 1614] dup_rows_per_dst=[700, 3832, 4114, 1294, 0, 4784, 3868, 8] combine_rows_per_src=[8630, 11354, 11656, 7928, 10858, 13330, 14334, 1614] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87722 payload_rows_remote=80230 dup_rows=10582 dup_rows_remote=10326 dedup_ratio=0.107646 router_tokens_per_expert=[40, 221, 3768, 328, 47, 40, 2013, 9, 3, 74, 157, 444, 4962, 31, 1120, 1899, 272, 13, 26, 120, 83, 652, 240, 991, 38, 58, 95, 4974, 195, 1451, 1955, 82, 177, 64, 133, 6, 100, 5224, 572, 119, 46, 0, 676, 11, 349, 1607, 797, 271, 19, 766, 144, 249, 76, 572, 1288, 109, 132, 3371, 1, 337, 567, 3792, 1055, 121] payload_rows_per_dst=[10810, 7228, 13260, 7492, 12244, 14466, 11468, 10754] dup_rows_per_dst=[2122, 228, 1458, 256, 546, 2996, 1720, 1256] combine_rows_per_src=[10810, 7228, 13260, 7492, 12244, 14466, 11468, 10754] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76814 payload_rows_remote=60548 dup_rows=21490 dup_rows_remote=7254 dedup_ratio=0.218608 router_tokens_per_expert=[0, 11, 2547, 0, 0, 0, 3741, 0, 0, 0, 154, 855, 7158, 0, 0, 2265, 0, 0, 0, 0, 0, 1246, 3, 0, 0, 0, 0, 7787, 0, 0, 3840, 0, 0, 0, 154, 0, 0, 6924, 0, 0, 0, 0, 1542, 0, 0, 0, 0, 366, 0, 0, 309, 0, 0, 2699, 0, 87, 0, 6441, 0, 0, 517, 506, 0, 0] payload_rows_per_dst=[11046, 5914, 6326, 13838, 14234, 3770, 5420, 16266] dup_rows_per_dst=[1552, 208, 514, 3816, 348, 46, 770, 14236] combine_rows_per_src=[11046, 5914, 6326, 13838, 14234, 3770, 5420, 16266] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86782 payload_rows_remote=73582 dup_rows=11522 dup_rows_remote=11522 dedup_ratio=0.117208 router_tokens_per_expert=[0, 73, 2480, 0, 0, 0, 3606, 0, 0, 0, 161, 1057, 7102, 0, 0, 2349, 0, 0, 0, 0, 0, 1465, 2, 0, 0, 0, 0, 7779, 0, 0, 3738, 0, 0, 0, 161, 0, 0, 6437, 0, 2, 0, 0, 1900, 0, 0, 0, 0, 321, 0, 0, 308, 0, 0, 2787, 0, 92, 0, 6379, 0, 0, 620, 333, 0, 0] payload_rows_per_dst=[10984, 6822, 14426, 7476, 13200, 15888, 9488, 8498] dup_rows_per_dst=[1334, 312, 2712, 0, 0, 4112, 2976, 76] combine_rows_per_src=[10984, 6822, 14426, 7476, 13200, 15888, 9488, 8498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=258883584 dispatch_meta_bytes=314640 combine_payload_bytes=258883584 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=259198224 ingress_bytes=362692608 link_bytes=362692608 token_bytes=518081808 weight_movement_bytes=103809024 total_nvlink_bytes=621890832 weight_to_token_ratio=0.200372 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.364783085e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82840 payload_rows_remote=66930 dup_rows=15464 dup_rows_remote=8026 dedup_ratio=0.157308 router_tokens_per_expert=[0, 67, 1608, 0, 0, 0, 3466, 0, 0, 0, 180, 1395, 7310, 0, 1, 3669, 0, 0, 0, 0, 0, 2620, 1, 1, 1, 0, 1, 7832, 0, 1, 2601, 0, 8, 0, 131, 0, 0, 4964, 0, 0, 0, 0, 3564, 0, 0, 1, 0, 277, 0, 0, 372, 0, 3, 1561, 0, 67, 1, 6719, 0, 0, 648, 81, 0, 1] payload_rows_per_dst=[9222, 9554, 14772, 5206, 10190, 15910, 3352, 14634] dup_rows_per_dst=[1060, 936, 5092, 2, 16, 7438, 654, 266] combine_rows_per_src=[9222, 9554, 14772, 5206, 10190, 15910, 3352, 14634] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=309846016 dispatch_meta_bytes=374664 combine_payload_bytes=309846016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=310220680 ingress_bytes=344449024 link_bytes=344449024 token_bytes=620066696 weight_movement_bytes=34603008 total_nvlink_bytes=654669704 weight_to_token_ratio=0.055805 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=26368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.296133947e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83624 payload_rows_remote=71272 dup_rows=14680 dup_rows_remote=10202 dedup_ratio=0.149333 router_tokens_per_expert=[1, 6979, 1043, 19, 1, 68, 709, 4464, 1, 5, 0, 0, 0, 708, 2816, 142, 98, 89, 60, 328, 71, 1, 212, 770, 1512, 346, 0, 468, 253, 63, 197, 2, 73, 7052, 76, 1, 2509, 0, 983, 0, 1240, 0, 26, 0, 2431, 253, 0, 1, 66, 1, 106, 586, 127, 570, 67, 0, 11, 414, 4808, 692, 341, 3057, 1, 2234] payload_rows_per_dst=[3630, 11560, 15550, 9956, 6650, 12352, 15138, 8788] dup_rows_per_dst=[52, 1898, 1666, 2346, 634, 4478, 2012, 1594] combine_rows_per_src=[3630, 11560, 15550, 9956, 6650, 12352, 15138, 8788] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=255213568 dispatch_meta_bytes=293240 combine_payload_bytes=255213568 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=255506808 ingress_bytes=359022592 link_bytes=359022592 token_bytes=510720376 weight_movement_bytes=103809024 total_nvlink_bytes=614529400 weight_to_token_ratio=0.203260 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.350973110e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79604 payload_rows_remote=75646 dup_rows=18700 dup_rows_remote=18020 dedup_ratio=0.190226 router_tokens_per_expert=[1, 89, 2402, 25, 0, 2, 2945, 1, 0, 0, 144, 1791, 7030, 2, 131, 3519, 8, 1, 3, 13, 7, 1915, 52, 76, 18, 0, 15, 7574, 11, 130, 2722, 8, 7, 12, 156, 0, 5, 5661, 26, 11, 1, 0, 1950, 0, 12, 136, 78, 348, 1, 59, 368, 23, 9, 1677, 103, 79, 18, 6420, 0, 38, 732, 471, 72, 44] payload_rows_per_dst=[9886, 10156, 14538, 5754, 11712, 7832, 3958, 15768] dup_rows_per_dst=[1044, 1018, 3672, 54, 44, 1162, 680, 11026] combine_rows_per_src=[9886, 10156, 14538, 5754, 11712, 7832, 3958, 15768] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76070 payload_rows_remote=61066 dup_rows=22234 dup_rows_remote=14942 dedup_ratio=0.226176 router_tokens_per_expert=[36, 0, 0, 0, 0, 89, 314, 0, 0, 76, 231, 0, 0, 58, 3612, 0, 4912, 104, 184, 7133, 6933, 0, 0, 0, 0, 330, 948, 0, 7171, 0, 58, 0, 4140, 0, 39, 0, 220, 1597, 6706, 92, 0, 0, 0, 0, 0, 0, 0, 73, 0, 1327, 0, 862, 0, 0, 0, 0, 0, 0, 0, 238, 959, 0, 370, 340] payload_rows_per_dst=[14290, 15004, 15332, 4068, 3468, 12934, 8096, 2878] dup_rows_per_dst=[0, 7292, 11690, 240, 428, 144, 1504, 936] combine_rows_per_src=[14290, 15004, 15332, 4068, 3468, 12934, 8096, 2878] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=278724608 dispatch_meta_bytes=330136 combine_payload_bytes=278724608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279054744 ingress_bytes=313327616 link_bytes=313327616 token_bytes=557779352 weight_movement_bytes=34603008 total_nvlink_bytes=592382360 weight_to_token_ratio=0.062037 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=33536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.179026594e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89986 payload_rows_remote=78558 dup_rows=8318 dup_rows_remote=6036 dedup_ratio=0.084615 router_tokens_per_expert=[3, 0, 0, 0, 0, 78, 316, 0, 0, 83, 242, 0, 0, 76, 3280, 0, 4499, 81, 184, 7096, 6937, 0, 0, 0, 0, 293, 860, 0, 6961, 0, 69, 0, 4509, 0, 36, 0, 176, 2042, 6829, 92, 0, 0, 0, 0, 0, 0, 0, 67, 0, 1193, 0, 1262, 0, 0, 0, 0, 0, 0, 0, 226, 964, 0, 380, 318] payload_rows_per_dst=[14436, 7230, 530, 15230, 11428, 14192, 11626, 15314] dup_rows_per_dst=[16, 132, 0, 1136, 2282, 134, 2282, 2336] combine_rows_per_src=[14436, 7230, 530, 15230, 11428, 14192, 11626, 15314] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=328622080 dispatch_meta_bytes=362224 combine_payload_bytes=328622080 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=328984304 ingress_bytes=432431104 link_bytes=432431104 token_bytes=657606384 weight_movement_bytes=103809024 total_nvlink_bytes=761415408 weight_to_token_ratio=0.157859 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=24576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.627203431e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=248004608 dispatch_meta_bytes=271208 combine_payload_bytes=248004608 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=248275816 ingress_bytes=317210624 link_bytes=317210624 token_bytes=496280424 weight_movement_bytes=69206016 total_nvlink_bytes=565486440 weight_to_token_ratio=0.139449 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=23040 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.193638041e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=274145280 dispatch_meta_bytes=299824 combine_payload_bytes=274145280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274445104 ingress_bytes=308748288 link_bytes=308748288 token_bytes=548590384 weight_movement_bytes=34603008 total_nvlink_bytes=583193392 weight_to_token_ratio=0.063076 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=28544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.161794951e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80754 payload_rows_remote=65284 dup_rows=17550 dup_rows_remote=15036 dedup_ratio=0.178528 router_tokens_per_expert=[17, 0, 0, 0, 0, 73, 308, 0, 0, 73, 232, 0, 0, 68, 3422, 0, 4665, 34, 173, 7217, 7016, 0, 0, 0, 0, 254, 896, 0, 7174, 0, 68, 0, 4544, 0, 60, 0, 205, 1809, 6711, 87, 0, 0, 0, 0, 0, 0, 0, 59, 0, 1120, 0, 891, 0, 0, 0, 0, 0, 0, 0, 219, 1072, 0, 354, 331] payload_rows_per_dst=[794, 7466, 414, 15600, 15064, 14434, 11512, 15470] dup_rows_per_dst=[2, 124, 0, 1184, 11768, 118, 1840, 2514] combine_rows_per_src=[794, 7466, 414, 15600, 15064, 14434, 11512, 15470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=57756 payload_rows_remote=43544 dup_rows=40548 dup_rows_remote=40506 dedup_ratio=0.412476 router_tokens_per_expert=[39, 0, 3, 0, 6, 30, 422, 0, 0, 100, 165, 0, 0, 77, 3684, 0, 4785, 80, 182, 7093, 6921, 0, 4, 4, 0, 268, 1062, 0, 6887, 1, 80, 0, 4126, 4, 200, 0, 242, 1066, 6627, 92, 0, 2, 0, 4, 0, 3, 0, 53, 0, 1198, 1, 1467, 0, 5, 0, 0, 2, 0, 2, 225, 1184, 6, 399, 351] payload_rows_per_dst=[14212, 14546, 15684, 2816, 2636, 122, 4612, 3128] dup_rows_per_dst=[42, 7280, 30706, 6, 572, 2, 730, 1210] combine_rows_per_src=[14212, 14546, 15684, 2816, 2636, 122, 4612, 3128] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76070 payload_rows_remote=61066 dup_rows=22234 dup_rows_remote=14942 dedup_ratio=0.226176 router_tokens_per_expert=[36, 0, 0, 0, 0, 89, 314, 0, 0, 76, 231, 0, 0, 58, 3612, 0, 4912, 104, 184, 7133, 6933, 0, 0, 0, 0, 330, 948, 0, 7171, 0, 58, 0, 4140, 0, 39, 0, 220, 1597, 6706, 92, 0, 0, 0, 0, 0, 0, 0, 73, 0, 1327, 0, 862, 0, 0, 0, 0, 0, 0, 0, 238, 959, 0, 370, 340] payload_rows_per_dst=[14290, 15004, 15332, 4068, 3468, 12934, 8096, 2878] dup_rows_per_dst=[0, 7292, 11690, 240, 428, 144, 1504, 936] combine_rows_per_src=[14290, 15004, 15332, 4068, 3468, 12934, 8096, 2878] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=321773568 dispatch_meta_bytes=338376 combine_payload_bytes=321773568 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=322111944 ingress_bytes=425582592 link_bytes=425582592 token_bytes=643885512 weight_movement_bytes=103809024 total_nvlink_bytes=747694536 weight_to_token_ratio=0.161223 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=20352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.601433031e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87060 payload_rows_remote=76128 dup_rows=11244 dup_rows_remote=9040 dedup_ratio=0.114380 router_tokens_per_expert=[65, 0, 96, 0, 130, 79, 419, 9, 0, 82, 144, 14, 0, 110, 2482, 1, 4013, 219, 165, 6953, 6639, 2, 84, 12, 0, 607, 833, 0, 6916, 40, 147, 0, 3511, 33, 80, 7, 259, 3199, 6449, 79, 0, 27, 2, 101, 21, 0, 0, 61, 0, 1194, 16, 1231, 1, 113, 0, 0, 56, 3, 69, 260, 1173, 130, 465, 351] payload_rows_per_dst=[12610, 5520, 954, 15426, 12058, 14200, 10932, 15360] dup_rows_per_dst=[302, 146, 10, 1660, 3860, 130, 2204, 2932] combine_rows_per_src=[12610, 5520, 954, 15426, 12058, 14200, 10932, 15360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=267403264 dispatch_meta_bytes=321280 combine_payload_bytes=267403264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267724544 ingress_bytes=302006272 link_bytes=302006272 token_bytes=535127808 weight_movement_bytes=34603008 total_nvlink_bytes=569730816 weight_to_token_ratio=0.064663 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=28672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136425288e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=178356224 dispatch_meta_bytes=336200 combine_payload_bytes=178356224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=178692424 ingress_bytes=212959232 link_bytes=212959232 token_bytes=357048648 weight_movement_bytes=34603008 total_nvlink_bytes=391651656 weight_to_token_ratio=0.096914 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=42112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.013484454e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=301391872 dispatch_meta_bytes=340416 combine_payload_bytes=301391872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=301732288 ingress_bytes=370597888 link_bytes=370597888 token_bytes=603124160 weight_movement_bytes=69206016 total_nvlink_bytes=672330176 weight_to_token_ratio=0.114746 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=46720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394530017e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=291930112 dispatch_meta_bytes=325896 combine_payload_bytes=291930112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=292256008 ingress_bytes=326533120 link_bytes=326533120 token_bytes=584186120 weight_movement_bytes=34603008 total_nvlink_bytes=618789128 weight_to_token_ratio=0.059233 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.228717842e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=309846016 dispatch_meta_bytes=374664 combine_payload_bytes=309846016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=310220680 ingress_bytes=344449024 link_bytes=344449024 token_bytes=620066696 weight_movement_bytes=34603008 total_nvlink_bytes=654669704 weight_to_token_ratio=0.055805 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=26368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.296133947e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=250126336 dispatch_meta_bytes=304032 combine_payload_bytes=250126336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=250430368 ingress_bytes=284729344 link_bytes=284729344 token_bytes=500556704 weight_movement_bytes=34603008 total_nvlink_bytes=535159712 weight_to_token_ratio=0.069129 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.071413599e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76806 payload_rows_remote=64362 dup_rows=21498 dup_rows_remote=17920 dedup_ratio=0.218689 router_tokens_per_expert=[0, 85, 2760, 0, 0, 0, 4258, 0, 0, 0, 167, 892, 7113, 1, 6, 2370, 0, 0, 0, 0, 0, 1411, 3, 3, 1, 0, 0, 7766, 1, 7, 3627, 0, 0, 1, 167, 0, 0, 6681, 1, 1, 1, 0, 1024, 0, 1, 6, 3, 267, 0, 3, 286, 0, 0, 2715, 3, 92, 1, 6421, 0, 0, 437, 566, 3, 1] payload_rows_per_dst=[12444, 15646, 2834, 15736, 11886, 2590, 13662, 2008] dup_rows_per_dst=[3578, 5452, 0, 7068, 0, 14, 5378, 8] combine_rows_per_src=[12444, 15646, 2834, 15736, 11886, 2590, 13662, 2008] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87642 payload_rows_remote=73422 dup_rows=10662 dup_rows_remote=10662 dedup_ratio=0.108459 router_tokens_per_expert=[2, 0, 0, 0, 0, 62, 479, 0, 0, 52, 147, 0, 0, 55, 2664, 0, 4209, 8, 71, 7143, 7031, 0, 0, 0, 0, 470, 555, 0, 7330, 0, 37, 0, 3809, 0, 1, 0, 253, 3543, 6893, 62, 0, 0, 0, 0, 0, 0, 0, 59, 0, 1392, 0, 771, 0, 0, 0, 0, 0, 0, 0, 211, 1122, 0, 299, 422] payload_rows_per_dst=[14752, 14954, 14220, 6018, 9378, 14286, 10790, 3244] dup_rows_per_dst=[120, 5542, 0, 366, 1698, 118, 1954, 864] combine_rows_per_src=[14752, 14954, 14220, 6018, 9378, 14286, 10790, 3244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=250126336 dispatch_meta_bytes=304032 combine_payload_bytes=250126336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=250430368 ingress_bytes=284729344 link_bytes=284729344 token_bytes=500556704 weight_movement_bytes=34603008 total_nvlink_bytes=535159712 weight_to_token_ratio=0.069129 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.071413599e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79704 payload_rows_remote=67552 dup_rows=18600 dup_rows_remote=13360 dedup_ratio=0.189209 router_tokens_per_expert=[88, 2, 1154, 1, 1553, 85, 485, 122, 58, 82, 381, 247, 2, 287, 1724, 33, 2599, 617, 245, 4730, 3314, 7, 962, 157, 0, 1593, 2467, 0, 3508, 503, 1377, 61, 2397, 332, 723, 186, 206, 462, 2490, 134, 2, 291, 62, 1225, 69, 28, 2, 64, 120, 350, 171, 4182, 6, 1260, 0, 4, 553, 108, 785, 306, 1175, 1489, 1111, 415] payload_rows_per_dst=[9878, 5654, 8364, 12152, 7534, 12594, 13040, 10488] dup_rows_per_dst=[2082, 1600, 198, 5240, 1346, 352, 4344, 3438] combine_rows_per_src=[9878, 5654, 8364, 12152, 7534, 12594, 13040, 10488] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80754 payload_rows_remote=65284 dup_rows=17550 dup_rows_remote=15036 dedup_ratio=0.178528 router_tokens_per_expert=[17, 0, 0, 0, 0, 73, 308, 0, 0, 73, 232, 0, 0, 68, 3422, 0, 4665, 34, 173, 7217, 7016, 0, 0, 0, 0, 254, 896, 0, 7174, 0, 68, 0, 4544, 0, 60, 0, 205, 1809, 6711, 87, 0, 0, 0, 0, 0, 0, 0, 59, 0, 1120, 0, 891, 0, 0, 0, 0, 0, 0, 0, 219, 1072, 0, 354, 331] payload_rows_per_dst=[794, 7466, 414, 15600, 15064, 14434, 11512, 15470] dup_rows_per_dst=[2, 124, 0, 1184, 11768, 118, 1840, 2514] combine_rows_per_src=[794, 7466, 414, 15600, 15064, 14434, 11512, 15470] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89986 payload_rows_remote=78558 dup_rows=8318 dup_rows_remote=6036 dedup_ratio=0.084615 router_tokens_per_expert=[3, 0, 0, 0, 0, 78, 316, 0, 0, 83, 242, 0, 0, 76, 3280, 0, 4499, 81, 184, 7096, 6937, 0, 0, 0, 0, 293, 860, 0, 6961, 0, 69, 0, 4509, 0, 36, 0, 176, 2042, 6829, 92, 0, 0, 0, 0, 0, 0, 0, 67, 0, 1193, 0, 1262, 0, 0, 0, 0, 0, 0, 0, 226, 964, 0, 380, 318] payload_rows_per_dst=[14436, 7230, 530, 15230, 11428, 14192, 11626, 15314] dup_rows_per_dst=[16, 132, 0, 1136, 2282, 134, 2282, 2336] combine_rows_per_src=[14436, 7230, 530, 15230, 11428, 14192, 11626, 15314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=311820288 dispatch_meta_bytes=340672 combine_payload_bytes=311820288 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=312160960 ingress_bytes=346423296 link_bytes=346423296 token_bytes=623981248 weight_movement_bytes=34603008 total_nvlink_bytes=658584256 weight_to_token_ratio=0.055455 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=27136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.303562973e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89368 payload_rows_remote=74972 dup_rows=8936 dup_rows_remote=8788 dedup_ratio=0.090902 router_tokens_per_expert=[44, 0, 0, 0, 1, 67, 413, 0, 0, 71, 89, 0, 0, 103, 2644, 0, 4503, 1, 99, 7197, 7042, 0, 1, 0, 0, 657, 488, 0, 7373, 0, 63, 0, 3674, 1, 1, 0, 253, 3396, 6859, 67, 0, 0, 0, 1, 0, 5, 0, 69, 0, 1286, 0, 629, 0, 1, 0, 0, 0, 0, 0, 254, 1090, 1, 379, 330] payload_rows_per_dst=[14624, 5800, 202, 15804, 12012, 14396, 11014, 15516] dup_rows_per_dst=[144, 14, 0, 1358, 2772, 148, 1824, 2676] combine_rows_per_src=[14624, 5800, 202, 15804, 12012, 14396, 11014, 15516] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=263626752 dispatch_meta_bytes=329128 combine_payload_bytes=263626752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263955880 ingress_bytes=298229760 link_bytes=298229760 token_bytes=527582632 weight_movement_bytes=34603008 total_nvlink_bytes=562185640 weight_to_token_ratio=0.065588 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=27520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.122214577e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87060 payload_rows_remote=76128 dup_rows=11244 dup_rows_remote=9040 dedup_ratio=0.114380 router_tokens_per_expert=[65, 0, 96, 0, 130, 79, 419, 9, 0, 82, 144, 14, 0, 110, 2482, 1, 4013, 219, 165, 6953, 6639, 2, 84, 12, 0, 607, 833, 0, 6916, 40, 147, 0, 3511, 33, 80, 7, 259, 3199, 6449, 79, 0, 27, 2, 101, 21, 0, 0, 61, 0, 1194, 16, 1231, 1, 113, 0, 0, 56, 3, 69, 260, 1173, 130, 465, 351] payload_rows_per_dst=[12610, 5520, 954, 15426, 12058, 14200, 10932, 15360] dup_rows_per_dst=[302, 146, 10, 1660, 3860, 130, 2204, 2932] combine_rows_per_src=[12610, 5520, 954, 15426, 12058, 14200, 10932, 15360] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85044 payload_rows_remote=75556 dup_rows=13260 dup_rows_remote=11520 dedup_ratio=0.134888 router_tokens_per_expert=[2, 0, 0, 297, 0, 0, 0, 1, 390, 147, 1515, 1, 529, 320, 1, 79, 68, 0, 14, 0, 0, 71, 2, 0, 1, 3414, 0, 214, 281, 1329, 3, 3, 1189, 3580, 0, 0, 0, 1083, 0, 1393, 2102, 1290, 6064, 0, 633, 632, 957, 0, 0, 119, 1, 2503, 252, 0, 82, 0, 1620, 1, 4532, 5584, 4181, 2538, 0, 134] payload_rows_per_dst=[12724, 4976, 11460, 9458, 11734, 9488, 12322, 12882] dup_rows_per_dst=[4, 988, 18, 1032, 2756, 1740, 2656, 4066] combine_rows_per_src=[12724, 4976, 11460, 9458, 11734, 9488, 12322, 12882] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82456 payload_rows_remote=67774 dup_rows=15848 dup_rows_remote=15014 dedup_ratio=0.161214 router_tokens_per_expert=[0, 0, 0, 600, 0, 0, 0, 0, 0, 0, 0, 92, 0, 92, 709, 0, 0, 0, 2380, 1929, 7347, 637, 0, 92, 7183, 0, 0, 6865, 143, 58, 0, 0, 0, 0, 350, 0, 58, 0, 0, 6419, 0, 676, 0, 0, 0, 0, 92, 0, 0, 0, 0, 236, 2631, 0, 2035, 293, 1522, 92, 11, 0, 0, 5539, 71, 1000] payload_rows_per_dst=[15030, 14682, 8966, 15516, 13538, 1536, 8256, 4932] dup_rows_per_dst=[864, 834, 1110, 10330, 116, 0, 2134, 460] combine_rows_per_src=[15030, 14682, 8966, 15516, 13538, 1536, 8256, 4932] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=276692992 dispatch_meta_bytes=323648 combine_payload_bytes=276692992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=277016640 ingress_bytes=345899008 link_bytes=345899008 token_bytes=553709632 weight_movement_bytes=69206016 total_nvlink_bytes=622915648 weight_to_token_ratio=0.124986 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=31232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.301590120e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=300736512 dispatch_meta_bytes=336336 combine_payload_bytes=300736512 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=301072848 ingress_bytes=439148544 link_bytes=439148544 token_bytes=601809360 weight_movement_bytes=138412032 total_nvlink_bytes=740221392 weight_to_token_ratio=0.229993 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=23296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.652480617e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=267403264 dispatch_meta_bytes=321280 combine_payload_bytes=267403264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267724544 ingress_bytes=302006272 link_bytes=302006272 token_bytes=535127808 weight_movement_bytes=34603008 total_nvlink_bytes=569730816 weight_to_token_ratio=0.064663 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=28672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136425288e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=276692992 dispatch_meta_bytes=323648 combine_payload_bytes=276692992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=277016640 ingress_bytes=345899008 link_bytes=345899008 token_bytes=553709632 weight_movement_bytes=69206016 total_nvlink_bytes=622915648 weight_to_token_ratio=0.124986 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=31232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.301590120e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89620 payload_rows_remote=76342 dup_rows=8684 dup_rows_remote=8546 dedup_ratio=0.088338 router_tokens_per_expert=[0, 0, 0, 588, 0, 0, 0, 0, 0, 0, 0, 92, 0, 91, 812, 0, 0, 0, 2543, 2565, 7309, 634, 0, 87, 7503, 0, 0, 7028, 150, 69, 0, 0, 0, 0, 262, 0, 68, 5, 0, 6373, 0, 611, 0, 0, 0, 0, 92, 0, 0, 0, 0, 210, 2696, 0, 1552, 310, 1452, 92, 0, 0, 0, 5104, 15, 839] payload_rows_per_dst=[15004, 15020, 7330, 5970, 13278, 15228, 7776, 10014] dup_rows_per_dst=[790, 1026, 772, 0, 138, 1184, 1760, 3014] combine_rows_per_src=[15004, 15020, 7330, 5970, 13278, 15228, 7776, 10014] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81226 payload_rows_remote=66424 dup_rows=17078 dup_rows_remote=11320 dedup_ratio=0.173726 router_tokens_per_expert=[0, 0, 0, 592, 0, 0, 0, 0, 0, 0, 0, 87, 0, 87, 840, 0, 0, 0, 2142, 2405, 7337, 651, 0, 86, 7345, 0, 0, 6853, 74, 68, 0, 0, 0, 0, 180, 0, 67, 1, 0, 6370, 0, 725, 0, 0, 0, 0, 87, 0, 0, 0, 0, 220, 2828, 0, 2004, 228, 1466, 86, 5, 0, 0, 5332, 70, 916] payload_rows_per_dst=[1184, 1854, 15290, 13990, 6810, 15200, 12096, 14802] dup_rows_per_dst=[0, 174, 5142, 0, 70, 1114, 4820, 5758] combine_rows_per_src=[1184, 1854, 15290, 13990, 6810, 15200, 12096, 14802] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=307085312 dispatch_meta_bytes=335040 combine_payload_bytes=307085312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307420352 ingress_bytes=341688320 link_bytes=341688320 token_bytes=614505664 weight_movement_bytes=34603008 total_nvlink_bytes=649108672 weight_to_token_ratio=0.056310 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=46848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.285745640e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80662 payload_rows_remote=65734 dup_rows=17642 dup_rows_remote=16772 dedup_ratio=0.179464 router_tokens_per_expert=[0, 0, 4, 618, 2, 6, 0, 0, 0, 0, 0, 92, 2, 92, 833, 2, 0, 0, 1998, 2541, 7269, 671, 2, 92, 7407, 3, 0, 6865, 118, 77, 0, 0, 0, 6, 276, 4, 77, 0, 0, 6338, 3, 655, 1, 6, 1, 0, 90, 0, 0, 0, 2, 234, 3078, 2, 1931, 285, 1464, 91, 1, 0, 6, 4899, 56, 952] payload_rows_per_dst=[14928, 14776, 10800, 15490, 9776, 1502, 8658, 4732] dup_rows_per_dst=[870, 996, 3330, 9518, 104, 10, 2406, 408] combine_rows_per_src=[14928, 14776, 10800, 15490, 9776, 1502, 8658, 4732] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82456 payload_rows_remote=67774 dup_rows=15848 dup_rows_remote=15014 dedup_ratio=0.161214 router_tokens_per_expert=[0, 0, 0, 600, 0, 0, 0, 0, 0, 0, 0, 92, 0, 92, 709, 0, 0, 0, 2380, 1929, 7347, 637, 0, 92, 7183, 0, 0, 6865, 143, 58, 0, 0, 0, 0, 350, 0, 58, 0, 0, 6419, 0, 676, 0, 0, 0, 0, 92, 0, 0, 0, 0, 236, 2631, 0, 2035, 293, 1522, 92, 11, 0, 0, 5539, 71, 1000] payload_rows_per_dst=[15030, 14682, 8966, 15516, 13538, 1536, 8256, 4932] dup_rows_per_dst=[864, 834, 1110, 10330, 116, 0, 2134, 460] combine_rows_per_src=[15030, 14682, 8966, 15516, 13538, 1536, 8256, 4932] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83762 payload_rows_remote=73756 dup_rows=14542 dup_rows_remote=12288 dedup_ratio=0.147929 router_tokens_per_expert=[0, 0, 87, 781, 63, 52, 0, 0, 4, 0, 2, 78, 34, 74, 1347, 62, 1, 3, 2055, 2628, 6937, 652, 24, 79, 7108, 73, 0, 6259, 82, 76, 1, 1, 0, 107, 262, 118, 66, 4, 0, 6096, 93, 634, 9, 126, 3, 0, 102, 0, 22, 12, 54, 256, 4224, 20, 1298, 244, 1396, 86, 77, 0, 136, 4175, 49, 920] payload_rows_per_dst=[1876, 5334, 14600, 10178, 12998, 14808, 10006, 13962] dup_rows_per_dst=[90, 660, 4902, 14, 308, 1342, 2254, 4972] combine_rows_per_src=[1876, 5334, 14600, 10178, 12998, 14808, 10006, 13962] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=312696832 dispatch_meta_bytes=339552 combine_payload_bytes=312696832 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=313036384 ingress_bytes=381902848 link_bytes=381902848 token_bytes=625733216 weight_movement_bytes=69206016 total_nvlink_bytes=694939232 weight_to_token_ratio=0.110600 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=43520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.437069671e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=272072704 dispatch_meta_bytes=310976 combine_payload_bytes=272072704 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=272383680 ingress_bytes=341278720 link_bytes=341278720 token_bytes=544456384 weight_movement_bytes=69206016 total_nvlink_bytes=613662400 weight_to_token_ratio=0.127110 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=17408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.284204348e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=321773568 dispatch_meta_bytes=338376 combine_payload_bytes=321773568 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=322111944 ingress_bytes=425582592 link_bytes=425582592 token_bytes=643885512 weight_movement_bytes=103809024 total_nvlink_bytes=747694536 weight_to_token_ratio=0.161223 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=20352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.601433031e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=311820288 dispatch_meta_bytes=340672 combine_payload_bytes=311820288 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=312160960 ingress_bytes=346423296 link_bytes=346423296 token_bytes=623981248 weight_movement_bytes=34603008 total_nvlink_bytes=658584256 weight_to_token_ratio=0.055455 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=27136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.303562973e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=309477376 dispatch_meta_bytes=348304 combine_payload_bytes=309477376 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=309825680 ingress_bytes=447889408 link_bytes=447889408 token_bytes=619303056 weight_movement_bytes=138412032 total_nvlink_bytes=757715088 weight_to_token_ratio=0.223496 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=15872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.685371784e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=269246464 dispatch_meta_bytes=330024 combine_payload_bytes=269246464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269576488 ingress_bytes=303849472 link_bytes=303849472 token_bytes=538822952 weight_movement_bytes=34603008 total_nvlink_bytes=573425960 weight_to_token_ratio=0.064220 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=40064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.143361101e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=277602304 dispatch_meta_bytes=331152 combine_payload_bytes=277602304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277933456 ingress_bytes=312205312 link_bytes=312205312 token_bytes=555535760 weight_movement_bytes=34603008 total_nvlink_bytes=590138768 weight_to_token_ratio=0.062288 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=38528 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.174803454e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=57756 payload_rows_remote=43544 dup_rows=40548 dup_rows_remote=40506 dedup_ratio=0.412476 router_tokens_per_expert=[39, 0, 3, 0, 6, 30, 422, 0, 0, 100, 165, 0, 0, 77, 3684, 0, 4785, 80, 182, 7093, 6921, 0, 4, 4, 0, 268, 1062, 0, 6887, 1, 80, 0, 4126, 4, 200, 0, 242, 1066, 6627, 92, 0, 2, 0, 4, 0, 3, 0, 53, 0, 1198, 1, 1467, 0, 5, 0, 0, 2, 0, 2, 225, 1184, 6, 399, 351] payload_rows_per_dst=[14212, 14546, 15684, 2816, 2636, 122, 4612, 3128] dup_rows_per_dst=[42, 7280, 30706, 6, 572, 2, 730, 1210] combine_rows_per_src=[14212, 14546, 15684, 2816, 2636, 122, 4612, 3128] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78928 payload_rows_remote=75628 dup_rows=19376 dup_rows_remote=19242 dedup_ratio=0.197103 router_tokens_per_expert=[4, 0, 1097, 2421, 432, 533, 1, 1, 62, 73, 1, 111, 581, 33, 1459, 694, 31, 116, 740, 1070, 3367, 571, 300, 96, 4387, 692, 12, 2120, 164, 161, 57, 24, 0, 1105, 727, 1484, 65, 33, 0, 4233, 996, 735, 81, 1436, 79, 2, 135, 22, 237, 172, 340, 396, 5333, 286, 3465, 469, 1358, 275, 576, 13, 1395, 607, 414, 1272] payload_rows_per_dst=[13100, 8838, 5244, 3300, 13150, 13010, 13292, 8994] dup_rows_per_dst=[2612, 1430, 604, 134, 2144, 2736, 8104, 1612] combine_rows_per_src=[13100, 8838, 5244, 3300, 13150, 13010, 13292, 8994] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89962 payload_rows_remote=79440 dup_rows=8342 dup_rows_remote=6508 dedup_ratio=0.084859 router_tokens_per_expert=[0, 0, 0, 588, 0, 0, 0, 0, 0, 0, 0, 62, 0, 62, 1018, 0, 0, 0, 2468, 2939, 7540, 709, 0, 62, 7579, 0, 0, 6988, 60, 37, 0, 0, 0, 0, 122, 0, 33, 0, 0, 6361, 0, 497, 0, 0, 0, 0, 62, 0, 0, 0, 0, 206, 3217, 0, 786, 159, 1239, 62, 0, 0, 0, 5668, 0, 628] payload_rows_per_dst=[15108, 15080, 10522, 12070, 12966, 13314, 7444, 3458] dup_rows_per_dst=[1148, 1180, 1834, 1548, 66, 874, 1292, 400] combine_rows_per_src=[15108, 15080, 10522, 12070, 12966, 13314, 7444, 3458] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=178356224 dispatch_meta_bytes=336200 combine_payload_bytes=178356224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=178692424 ingress_bytes=212959232 link_bytes=212959232 token_bytes=357048648 weight_movement_bytes=34603008 total_nvlink_bytes=391651656 weight_to_token_ratio=0.096914 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=42112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.013484454e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81226 payload_rows_remote=66424 dup_rows=17078 dup_rows_remote=11320 dedup_ratio=0.173726 router_tokens_per_expert=[0, 0, 0, 592, 0, 0, 0, 0, 0, 0, 0, 87, 0, 87, 840, 0, 0, 0, 2142, 2405, 7337, 651, 0, 86, 7345, 0, 0, 6853, 74, 68, 0, 0, 0, 0, 180, 0, 67, 1, 0, 6370, 0, 725, 0, 0, 0, 0, 87, 0, 0, 0, 0, 220, 2828, 0, 2004, 228, 1466, 86, 5, 0, 0, 5332, 70, 916] payload_rows_per_dst=[1184, 1854, 15290, 13990, 6810, 15200, 12096, 14802] dup_rows_per_dst=[0, 174, 5142, 0, 70, 1114, 4820, 5758] combine_rows_per_src=[1184, 1854, 15290, 13990, 6810, 15200, 12096, 14802] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=302104576 dispatch_meta_bytes=344176 combine_payload_bytes=302104576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=302448752 ingress_bytes=336707584 link_bytes=336707584 token_bytes=604553328 weight_movement_bytes=34603008 total_nvlink_bytes=639156336 weight_to_token_ratio=0.057237 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=26880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.267003531e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78928 payload_rows_remote=75628 dup_rows=19376 dup_rows_remote=19242 dedup_ratio=0.197103 router_tokens_per_expert=[4, 0, 1097, 2421, 432, 533, 1, 1, 62, 73, 1, 111, 581, 33, 1459, 694, 31, 116, 740, 1070, 3367, 571, 300, 96, 4387, 692, 12, 2120, 164, 161, 57, 24, 0, 1105, 727, 1484, 65, 33, 0, 4233, 996, 735, 81, 1436, 79, 2, 135, 22, 237, 172, 340, 396, 5333, 286, 3465, 469, 1358, 275, 576, 13, 1395, 607, 414, 1272] payload_rows_per_dst=[13100, 8838, 5244, 3300, 13150, 13010, 13292, 8994] dup_rows_per_dst=[2612, 1430, 604, 134, 2144, 2736, 8104, 1612] combine_rows_per_src=[13100, 8838, 5244, 3300, 13150, 13010, 13292, 8994] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86950 payload_rows_remote=71512 dup_rows=11354 dup_rows_remote=10346 dedup_ratio=0.115499 router_tokens_per_expert=[0, 0, 1, 663, 0, 1, 0, 0, 0, 0, 1, 70, 0, 69, 924, 0, 0, 0, 2943, 2973, 7454, 649, 1, 67, 7649, 0, 0, 7146, 60, 63, 0, 0, 0, 1, 126, 0, 63, 0, 0, 6231, 0, 487, 0, 1, 0, 0, 86, 0, 0, 0, 2, 210, 2826, 1, 774, 190, 1242, 67, 13, 0, 1, 5438, 16, 643] payload_rows_per_dst=[14606, 15306, 6948, 246, 12716, 15438, 6814, 14876] dup_rows_per_dst=[1094, 1114, 910, 0, 126, 1008, 1192, 5910] combine_rows_per_src=[14606, 15306, 6948, 246, 12716, 15438, 6814, 14876] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87540 payload_rows_remote=75686 dup_rows=10764 dup_rows_remote=9428 dedup_ratio=0.109497 router_tokens_per_expert=[0, 63, 287, 97, 74, 1979, 0, 486, 0, 0, 39, 0, 67, 0, 0, 0, 2130, 0, 0, 0, 0, 1365, 0, 941, 0, 65, 17, 853, 0, 1, 0, 63, 5996, 2, 67, 664, 0, 1, 138, 1523, 210, 1, 0, 1349, 71, 1, 1, 1, 352, 14, 4069, 79, 4961, 63, 6304, 1, 54, 3559, 3884, 812, 3841, 2413, 1, 193] payload_rows_per_dst=[11272, 12744, 8200, 9468, 13476, 11854, 8702, 11824] dup_rows_per_dst=[1818, 76, 672, 298, 3306, 1336, 454, 2804] combine_rows_per_src=[11272, 12744, 8200, 9468, 13476, 11854, 8702, 11824] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89620 payload_rows_remote=76342 dup_rows=8684 dup_rows_remote=8546 dedup_ratio=0.088338 router_tokens_per_expert=[0, 0, 0, 588, 0, 0, 0, 0, 0, 0, 0, 92, 0, 91, 812, 0, 0, 0, 2543, 2565, 7309, 634, 0, 87, 7503, 0, 0, 7028, 150, 69, 0, 0, 0, 0, 262, 0, 68, 5, 0, 6373, 0, 611, 0, 0, 0, 0, 92, 0, 0, 0, 0, 210, 2696, 0, 1552, 310, 1452, 92, 0, 0, 0, 5104, 15, 839] payload_rows_per_dst=[15004, 15020, 7330, 5970, 13278, 15228, 7776, 10014] dup_rows_per_dst=[790, 1026, 772, 0, 138, 1184, 1760, 3014] combine_rows_per_src=[15004, 15020, 7330, 5970, 13278, 15228, 7776, 10014] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83762 payload_rows_remote=73756 dup_rows=14542 dup_rows_remote=12288 dedup_ratio=0.147929 router_tokens_per_expert=[0, 0, 87, 781, 63, 52, 0, 0, 4, 0, 2, 78, 34, 74, 1347, 62, 1, 3, 2055, 2628, 6937, 652, 24, 79, 7108, 73, 0, 6259, 82, 76, 1, 1, 0, 107, 262, 118, 66, 4, 0, 6096, 93, 634, 9, 126, 3, 0, 102, 0, 22, 12, 54, 256, 4224, 20, 1298, 244, 1396, 86, 77, 0, 136, 4175, 49, 920] payload_rows_per_dst=[1876, 5334, 14600, 10178, 12998, 14808, 10006, 13962] dup_rows_per_dst=[90, 660, 4902, 14, 308, 1342, 2254, 4972] combine_rows_per_src=[1876, 5334, 14600, 10178, 12998, 14808, 10006, 13962] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=309772288 dispatch_meta_bytes=379480 combine_payload_bytes=309772288 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=310151768 ingress_bytes=413581312 link_bytes=413581312 token_bytes=619924056 weight_movement_bytes=103809024 total_nvlink_bytes=723733080 weight_to_token_ratio=0.167454 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=19584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.556273181e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=325386240 dispatch_meta_bytes=343792 combine_payload_bytes=325386240 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=325730032 ingress_bytes=429195264 link_bytes=429195264 token_bytes=651116272 weight_movement_bytes=103809024 total_nvlink_bytes=754925296 weight_to_token_ratio=0.159432 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.615027225e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=309772288 dispatch_meta_bytes=379480 combine_payload_bytes=309772288 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=310151768 ingress_bytes=413581312 link_bytes=413581312 token_bytes=619924056 weight_movement_bytes=103809024 total_nvlink_bytes=723733080 weight_to_token_ratio=0.167454 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=19584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.556273181e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88250 payload_rows_remote=75342 dup_rows=10054 dup_rows_remote=9086 dedup_ratio=0.102275 router_tokens_per_expert=[0, 3575, 3002, 1046, 7551, 7624, 0, 0, 0, 0, 0, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2474, 0, 0, 0, 0, 1380, 1638, 92, 0, 0, 0, 0, 0, 112, 0, 0, 6208, 439, 179, 0, 0, 1651, 0, 0, 17, 112, 0, 0, 2345, 322, 0, 0, 0, 89, 2380, 224, 6348, 0, 0, 92, 0, 0, 0] payload_rows_per_dst=[7618, 15248, 10108, 5762, 12908, 15732, 7994, 12880] dup_rows_per_dst=[478, 504, 1990, 458, 968, 2930, 2278, 448] combine_rows_per_src=[7618, 15248, 10108, 5762, 12908, 15732, 7994, 12880] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=292913152 dispatch_meta_bytes=327432 combine_payload_bytes=292913152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293240584 ingress_bytes=327516160 link_bytes=327516160 token_bytes=586153736 weight_movement_bytes=34603008 total_nvlink_bytes=620756744 weight_to_token_ratio=0.059034 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=40576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.232416943e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84616 payload_rows_remote=71280 dup_rows=13688 dup_rows_remote=12248 dedup_ratio=0.139242 router_tokens_per_expert=[0, 3592, 2519, 829, 7691, 7740, 0, 0, 0, 0, 0, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2520, 2, 0, 0, 0, 1367, 1881, 86, 0, 0, 0, 0, 0, 128, 0, 0, 6259, 677, 178, 0, 0, 1710, 0, 0, 30, 14, 0, 0, 2134, 294, 0, 0, 0, 84, 2372, 193, 6515, 0, 0, 86, 0, 0, 0] payload_rows_per_dst=[6280, 15480, 10132, 6352, 2600, 15784, 14652, 13336] dup_rows_per_dst=[416, 502, 2096, 316, 90, 3106, 5722, 1440] combine_rows_per_src=[6280, 15480, 10132, 6352, 2600, 15784, 14652, 13336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=66794 payload_rows_remote=50606 dup_rows=31510 dup_rows_remote=9400 dedup_ratio=0.320536 router_tokens_per_expert=[0, 3606, 2513, 990, 7461, 7691, 0, 0, 0, 0, 1, 247, 0, 1, 4, 1, 6, 0, 0, 0, 4, 2780, 2, 0, 0, 0, 1428, 2031, 90, 0, 6, 0, 0, 0, 187, 0, 0, 6057, 454, 213, 0, 0, 1441, 0, 4, 100, 118, 3, 2, 2516, 335, 0, 0, 7, 110, 2275, 191, 6178, 1, 6, 90, 0, 1, 1] payload_rows_per_dst=[16188, 508, 8588, 12820, 12902, 2896, 8202, 4690] dup_rows_per_dst=[22110, 0, 1560, 3950, 920, 436, 2288, 246] combine_rows_per_src=[16188, 508, 8588, 12820, 12902, 2896, 8202, 4690] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80680 payload_rows_remote=76528 dup_rows=17624 dup_rows_remote=17514 dedup_ratio=0.179281 router_tokens_per_expert=[0, 3832, 2241, 1110, 7503, 7633, 0, 0, 0, 0, 0, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2169, 0, 0, 0, 0, 1394, 2562, 92, 0, 0, 0, 0, 0, 114, 0, 0, 6032, 397, 176, 0, 0, 1675, 0, 0, 76, 65, 0, 0, 2290, 294, 0, 0, 0, 76, 2282, 222, 6599, 0, 0, 92, 0, 0, 0] payload_rows_per_dst=[13440, 4152, 10186, 15902, 12606, 3350, 7662, 13382] dup_rows_per_dst=[4718, 110, 1816, 7200, 832, 282, 2222, 444] combine_rows_per_src=[13440, 4152, 10186, 15902, 12606, 3350, 7662, 13382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=272072704 dispatch_meta_bytes=310976 combine_payload_bytes=272072704 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=272383680 ingress_bytes=341278720 link_bytes=341278720 token_bytes=544456384 weight_movement_bytes=69206016 total_nvlink_bytes=613662400 weight_to_token_ratio=0.127110 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=17408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.284204348e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=312696832 dispatch_meta_bytes=339552 combine_payload_bytes=312696832 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=313036384 ingress_bytes=381902848 link_bytes=381902848 token_bytes=625733216 weight_movement_bytes=69206016 total_nvlink_bytes=694939232 weight_to_token_ratio=0.110600 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=43520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.437069671e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=302104576 dispatch_meta_bytes=344176 combine_payload_bytes=302104576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=302448752 ingress_bytes=336707584 link_bytes=336707584 token_bytes=604553328 weight_movement_bytes=34603008 total_nvlink_bytes=639156336 weight_to_token_ratio=0.057237 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=26880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.267003531e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 1, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88040 payload_rows_remote=79010 dup_rows=10264 dup_rows_remote=8004 dedup_ratio=0.104411 router_tokens_per_expert=[0, 3725, 2569, 1043, 6991, 7480, 20, 0, 1, 6, 24, 258, 17, 16, 66, 17, 141, 29, 6, 0, 85, 2283, 3, 0, 2, 0, 1386, 1045, 79, 0, 96, 1, 5, 5, 302, 0, 3, 6155, 472, 332, 7, 1, 1862, 0, 61, 65, 108, 127, 27, 1767, 240, 0, 2, 146, 106, 3357, 205, 6147, 2, 129, 76, 11, 12, 31] payload_rows_per_dst=[6898, 15226, 10408, 4882, 13486, 15366, 9030, 12744] dup_rows_per_dst=[366, 544, 2136, 336, 1062, 3078, 2260, 482] combine_rows_per_src=[6898, 15226, 10408, 4882, 13486, 15366, 9030, 12744] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=308600832 dispatch_meta_bytes=337712 combine_payload_bytes=308600832 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=308938544 ingress_bytes=377806848 link_bytes=377806848 token_bytes=617539376 weight_movement_bytes=69206016 total_nvlink_bytes=686745392 weight_to_token_ratio=0.112067 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=41728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.421656753e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=291962880 dispatch_meta_bytes=334112 combine_payload_bytes=291962880 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=292296992 ingress_bytes=361168896 link_bytes=361168896 token_bytes=584259872 weight_movement_bytes=69206016 total_nvlink_bytes=653465888 weight_to_token_ratio=0.118451 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=44288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.359049479e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=310009856 dispatch_meta_bytes=340456 combine_payload_bytes=310009856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=310350312 ingress_bytes=344612864 link_bytes=344612864 token_bytes=620360168 weight_movement_bytes=34603008 total_nvlink_bytes=654963176 weight_to_token_ratio=0.055779 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=33024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.296750464e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=207282176 dispatch_meta_bytes=240024 combine_payload_bytes=207282176 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=207522200 ingress_bytes=380297216 link_bytes=380297216 token_bytes=414804376 weight_movement_bytes=173015040 total_nvlink_bytes=587819416 weight_to_token_ratio=0.417100 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=19968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.431027807e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=313458688 dispatch_meta_bytes=376168 combine_payload_bytes=313458688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=313834856 ingress_bytes=348061696 link_bytes=348061696 token_bytes=627293544 weight_movement_bytes=34603008 total_nvlink_bytes=661896552 weight_to_token_ratio=0.055162 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=45184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.309728141e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80662 payload_rows_remote=65734 dup_rows=17642 dup_rows_remote=16772 dedup_ratio=0.179464 router_tokens_per_expert=[0, 0, 4, 618, 2, 6, 0, 0, 0, 0, 0, 92, 2, 92, 833, 2, 0, 0, 1998, 2541, 7269, 671, 2, 92, 7407, 3, 0, 6865, 118, 77, 0, 0, 0, 6, 276, 4, 77, 0, 0, 6338, 3, 655, 1, 6, 1, 0, 90, 0, 0, 0, 2, 234, 3078, 2, 1931, 285, 1464, 91, 1, 0, 6, 4899, 56, 952] payload_rows_per_dst=[14928, 14776, 10800, 15490, 9776, 1502, 8658, 4732] dup_rows_per_dst=[870, 996, 3330, 9518, 104, 10, 2406, 408] combine_rows_per_src=[14928, 14776, 10800, 15490, 9776, 1502, 8658, 4732] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89800 payload_rows_remote=79550 dup_rows=8504 dup_rows_remote=6722 dedup_ratio=0.086507 router_tokens_per_expert=[0, 4212, 2750, 1081, 7510, 7794, 0, 0, 0, 0, 0, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1804, 0, 0, 0, 0, 1380, 435, 62, 0, 0, 0, 0, 0, 85, 0, 0, 6679, 205, 253, 0, 0, 2450, 0, 0, 0, 0, 0, 0, 1722, 16, 0, 0, 0, 62, 4121, 154, 6061, 0, 0, 62, 0, 0, 0] payload_rows_per_dst=[7376, 15588, 10250, 16038, 13682, 4900, 9720, 12246] dup_rows_per_dst=[286, 508, 1782, 2736, 762, 0, 2122, 308] combine_rows_per_src=[7376, 15588, 10250, 16038, 13682, 4900, 9720, 12246] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=323624960 dispatch_meta_bytes=348056 combine_payload_bytes=323624960 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=323973016 ingress_bytes=358227968 link_bytes=358227968 token_bytes=647597976 weight_movement_bytes=34603008 total_nvlink_bytes=682200984 weight_to_token_ratio=0.053433 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=22912 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.347983004e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81338 payload_rows_remote=69058 dup_rows=16966 dup_rows_remote=11734 dedup_ratio=0.172587 router_tokens_per_expert=[15, 1407, 1354, 730, 4811, 4879, 277, 0, 63, 101, 198, 205, 431, 139, 579, 410, 1571, 505, 46, 57, 1056, 3048, 28, 0, 31, 1, 1426, 2791, 135, 4, 844, 67, 48, 79, 775, 53, 18, 3414, 621, 178, 294, 56, 804, 18, 830, 195, 416, 1509, 283, 1066, 755, 0, 146, 1701, 137, 1635, 152, 4727, 103, 994, 88, 109, 91, 648] payload_rows_per_dst=[4508, 12708, 11576, 12280, 9184, 9260, 9116, 12706] dup_rows_per_dst=[244, 1302, 3860, 5232, 1188, 1692, 2330, 1118] combine_rows_per_src=[4508, 12708, 11576, 12280, 9184, 9260, 9116, 12706] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=269246464 dispatch_meta_bytes=330024 combine_payload_bytes=269246464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269576488 ingress_bytes=303849472 link_bytes=303849472 token_bytes=538822952 weight_movement_bytes=34603008 total_nvlink_bytes=573425960 weight_to_token_ratio=0.064220 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=40064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.143361101e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84616 payload_rows_remote=71280 dup_rows=13688 dup_rows_remote=12248 dedup_ratio=0.139242 router_tokens_per_expert=[0, 3592, 2519, 829, 7691, 7740, 0, 0, 0, 0, 0, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2520, 2, 0, 0, 0, 1367, 1881, 86, 0, 0, 0, 0, 0, 128, 0, 0, 6259, 677, 178, 0, 0, 1710, 0, 0, 30, 14, 0, 0, 2134, 294, 0, 0, 0, 84, 2372, 193, 6515, 0, 0, 86, 0, 0, 0] payload_rows_per_dst=[6280, 15480, 10132, 6352, 2600, 15784, 14652, 13336] dup_rows_per_dst=[416, 502, 2096, 316, 90, 3106, 5722, 1440] combine_rows_per_src=[6280, 15480, 10132, 6352, 2600, 15784, 14652, 13336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88362 payload_rows_remote=72508 dup_rows=9942 dup_rows_remote=6304 dedup_ratio=0.101135 router_tokens_per_expert=[0, 4221, 2923, 766, 7470, 7708, 0, 0, 0, 0, 0, 253, 0, 0, 1, 0, 1, 0, 0, 15, 0, 1671, 0, 0, 0, 0, 1390, 562, 67, 0, 1, 0, 0, 0, 79, 0, 0, 6736, 232, 510, 0, 104, 2171, 0, 1, 0, 0, 0, 0, 1831, 12, 0, 0, 1, 90, 4419, 203, 5645, 0, 1, 67, 0, 1, 0] payload_rows_per_dst=[7082, 15418, 10146, 3794, 14224, 15854, 10406, 11438] dup_rows_per_dst=[296, 506, 1670, 246, 890, 3638, 2300, 396] combine_rows_per_src=[7082, 15418, 10146, 3794, 14224, 15854, 10406, 11438] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88250 payload_rows_remote=75342 dup_rows=10054 dup_rows_remote=9086 dedup_ratio=0.102275 router_tokens_per_expert=[0, 3575, 3002, 1046, 7551, 7624, 0, 0, 0, 0, 0, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2474, 0, 0, 0, 0, 1380, 1638, 92, 0, 0, 0, 0, 0, 112, 0, 0, 6208, 439, 179, 0, 0, 1651, 0, 0, 17, 112, 0, 0, 2345, 322, 0, 0, 0, 89, 2380, 224, 6348, 0, 0, 92, 0, 0, 0] payload_rows_per_dst=[7618, 15248, 10108, 5762, 12908, 15732, 7994, 12880] dup_rows_per_dst=[478, 504, 1990, 458, 968, 2930, 2278, 448] combine_rows_per_src=[7618, 15248, 10108, 5762, 12908, 15732, 7994, 12880] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88040 payload_rows_remote=79010 dup_rows=10264 dup_rows_remote=8004 dedup_ratio=0.104411 router_tokens_per_expert=[0, 3725, 2569, 1043, 6991, 7480, 20, 0, 1, 6, 24, 258, 17, 16, 66, 17, 141, 29, 6, 0, 85, 2283, 3, 0, 2, 0, 1386, 1045, 79, 0, 96, 1, 5, 5, 302, 0, 3, 6155, 472, 332, 7, 1, 1862, 0, 61, 65, 108, 127, 27, 1767, 240, 0, 2, 146, 106, 3357, 205, 6147, 2, 129, 76, 11, 12, 31] payload_rows_per_dst=[6898, 15226, 10408, 4882, 13486, 15366, 9030, 12744] dup_rows_per_dst=[366, 544, 2136, 336, 1062, 3078, 2260, 482] combine_rows_per_src=[6898, 15226, 10408, 4882, 13486, 15366, 9030, 12744] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84358 payload_rows_remote=71800 dup_rows=13946 dup_rows_remote=9912 dedup_ratio=0.141866 router_tokens_per_expert=[0, 0, 63, 67, 66, 0, 0, 0, 0, 2711, 0, 205, 67, 821, 0, 1374, 0, 6787, 1, 483, 0, 0, 68, 1, 6851, 1, 554, 292, 1, 506, 67, 3998, 6971, 6437, 1, 778, 1, 0, 0, 0, 337, 68, 2303, 0, 0, 1337, 253, 0, 39, 0, 0, 1633, 0, 0, 0, 0, 0, 3941, 0, 0, 0, 0, 0, 69] payload_rows_per_dst=[13942, 8840, 14542, 2834, 4186, 12558, 14016, 13440] dup_rows_per_dst=[392, 1516, 138, 8, 92, 4034, 3030, 4736] combine_rows_per_src=[13942, 8840, 14542, 2834, 4186, 12558, 14016, 13440] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86118 payload_rows_remote=71416 dup_rows=12186 dup_rows_remote=10324 dedup_ratio=0.123962 router_tokens_per_expert=[11, 0, 0, 0, 0, 0, 0, 3921, 454, 0, 379, 0, 0, 0, 344, 7105, 0, 0, 0, 1520, 271, 476, 0, 575, 0, 0, 0, 0, 0, 0, 0, 0, 2886, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 0, 7769, 0, 0, 293, 0, 0, 0, 0, 3698, 0, 7710, 0, 0, 0, 0, 0, 6998, 168, 432, 0, 92] payload_rows_per_dst=[13192, 14702, 5568, 15420, 11940, 5068, 9900, 10328] dup_rows_per_dst=[4928, 1862, 116, 0, 2524, 208, 1890, 658] combine_rows_per_src=[13192, 14702, 5568, 15420, 11940, 5068, 9900, 10328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=325836800 dispatch_meta_bytes=345088 combine_payload_bytes=325836800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=326181888 ingress_bytes=360439808 link_bytes=360439808 token_bytes=652018688 weight_movement_bytes=34603008 total_nvlink_bytes=686621696 weight_to_token_ratio=0.053071 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=26880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.356305980e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=282861568 dispatch_meta_bytes=323168 combine_payload_bytes=282861568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=283184736 ingress_bytes=317464576 link_bytes=317464576 token_bytes=566046304 weight_movement_bytes=34603008 total_nvlink_bytes=600649312 weight_to_token_ratio=0.061131 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=23424 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.194593641e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=291962880 dispatch_meta_bytes=334112 combine_payload_bytes=291962880 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=292296992 ingress_bytes=361168896 link_bytes=361168896 token_bytes=584259872 weight_movement_bytes=69206016 total_nvlink_bytes=653465888 weight_to_token_ratio=0.118451 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=44288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.359049479e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92754 payload_rows_remote=82458 dup_rows=5550 dup_rows_remote=4150 dedup_ratio=0.056458 router_tokens_per_expert=[45, 0, 0, 0, 0, 0, 0, 4068, 490, 0, 347, 0, 0, 0, 309, 6948, 0, 0, 0, 1959, 283, 522, 0, 518, 0, 0, 0, 0, 0, 0, 0, 0, 2520, 0, 0, 0, 0, 0, 0, 3328, 0, 0, 0, 7784, 0, 0, 306, 0, 0, 0, 0, 4645, 0, 7678, 0, 0, 0, 0, 0, 6725, 175, 410, 0, 92] payload_rows_per_dst=[8144, 11374, 8728, 15356, 10296, 15582, 9290, 13984] dup_rows_per_dst=[82, 1260, 1390, 0, 1400, 598, 0, 820] combine_rows_per_src=[8144, 11374, 8728, 15356, 10296, 15582, 9290, 13984] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=296992768 dispatch_meta_bytes=315248 combine_payload_bytes=296992768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=297308016 ingress_bytes=331595776 link_bytes=331595776 token_bytes=594300784 weight_movement_bytes=34603008 total_nvlink_bytes=628903792 weight_to_token_ratio=0.058225 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=31104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.247768209e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80024 payload_rows_remote=65320 dup_rows=18280 dup_rows_remote=15550 dedup_ratio=0.185954 router_tokens_per_expert=[38, 0, 0, 0, 0, 0, 0, 4198, 492, 0, 266, 0, 0, 0, 424, 6922, 0, 0, 0, 1678, 263, 429, 0, 555, 0, 0, 0, 0, 0, 0, 0, 0, 2595, 0, 0, 0, 0, 0, 0, 3515, 0, 0, 0, 7655, 0, 0, 278, 0, 0, 0, 0, 4496, 0, 7738, 0, 0, 0, 0, 0, 6941, 164, 418, 0, 87] payload_rows_per_dst=[8402, 1962, 15468, 0, 10786, 13226, 15476, 14704] dup_rows_per_dst=[70, 402, 4226, 0, 1434, 426, 8992, 2730] combine_rows_per_src=[8402, 1962, 15468, 0, 10786, 13226, 15476, 14704] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77632 payload_rows_remote=61750 dup_rows=20672 dup_rows_remote=12734 dedup_ratio=0.210286 router_tokens_per_expert=[34, 3, 0, 1, 0, 0, 0, 4225, 477, 7, 259, 0, 0, 0, 378, 7042, 0, 4, 0, 1696, 275, 472, 0, 605, 0, 0, 0, 0, 0, 1, 0, 0, 1930, 4, 1, 0, 0, 0, 0, 4090, 0, 7, 0, 7647, 0, 6, 300, 0, 0, 0, 0, 3944, 4, 7689, 0, 5, 0, 2, 0, 7322, 190, 440, 0, 92] payload_rows_per_dst=[15882, 15110, 5940, 15380, 10762, 614, 12680, 1264] dup_rows_per_dst=[7938, 6276, 164, 0, 1288, 12, 4810, 184] combine_rows_per_src=[15882, 15110, 5940, 15380, 10762, 614, 12680, 1264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86118 payload_rows_remote=71416 dup_rows=12186 dup_rows_remote=10324 dedup_ratio=0.123962 router_tokens_per_expert=[11, 0, 0, 0, 0, 0, 0, 3921, 454, 0, 379, 0, 0, 0, 344, 7105, 0, 0, 0, 1520, 271, 476, 0, 575, 0, 0, 0, 0, 0, 0, 0, 0, 2886, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 0, 7769, 0, 0, 293, 0, 0, 0, 0, 3698, 0, 7710, 0, 0, 0, 0, 0, 6998, 168, 432, 0, 92] payload_rows_per_dst=[13192, 14702, 5568, 15420, 11940, 5068, 9900, 10328] dup_rows_per_dst=[4928, 1862, 116, 0, 2524, 208, 1890, 658] combine_rows_per_src=[13192, 14702, 5568, 15420, 11940, 5068, 9900, 10328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87106 payload_rows_remote=77658 dup_rows=11198 dup_rows_remote=9694 dedup_ratio=0.113912 router_tokens_per_expert=[355, 98, 1, 7, 1, 5, 7, 3956, 519, 152, 791, 7, 25, 0, 742, 6482, 2, 70, 9, 2050, 259, 508, 15, 594, 2, 1, 0, 0, 0, 8, 0, 21, 3364, 109, 61, 0, 1, 0, 0, 2804, 0, 146, 0, 7092, 70, 132, 266, 1, 3, 0, 0, 3962, 50, 7416, 0, 95, 0, 19, 7, 6192, 165, 418, 14, 78] payload_rows_per_dst=[8612, 3636, 14978, 12160, 10820, 14540, 9448, 12912] dup_rows_per_dst=[248, 836, 5000, 4, 1858, 874, 1504, 874] combine_rows_per_src=[8612, 3636, 14978, 12160, 10820, 14540, 9448, 12912] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=337747968 dispatch_meta_bytes=346432 combine_payload_bytes=337747968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=338094400 ingress_bytes=372350976 link_bytes=372350976 token_bytes=675842368 weight_movement_bytes=34603008 total_nvlink_bytes=710445376 weight_to_token_ratio=0.051200 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.401126746e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=308600832 dispatch_meta_bytes=337712 combine_payload_bytes=308600832 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=308938544 ingress_bytes=377806848 link_bytes=377806848 token_bytes=617539376 weight_movement_bytes=69206016 total_nvlink_bytes=686745392 weight_to_token_ratio=0.112067 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=41728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.421656753e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=323624960 dispatch_meta_bytes=348056 combine_payload_bytes=323624960 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=323973016 ingress_bytes=358227968 link_bytes=358227968 token_bytes=647597976 weight_movement_bytes=34603008 total_nvlink_bytes=682200984 weight_to_token_ratio=0.053433 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=22912 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.347983004e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=294092800 dispatch_meta_bytes=326848 combine_payload_bytes=294092800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=294419648 ingress_bytes=328695808 link_bytes=328695808 token_bytes=588512448 weight_movement_bytes=34603008 total_nvlink_bytes=623115456 weight_to_token_ratio=0.058797 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=20096 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.236855863e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267550720 dispatch_meta_bytes=323480 combine_payload_bytes=267550720 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267874200 ingress_bytes=371359744 link_bytes=371359744 token_bytes=535424920 weight_movement_bytes=103809024 total_nvlink_bytes=639233944 weight_to_token_ratio=0.193882 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=41984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397396820e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=252928000 dispatch_meta_bytes=297936 combine_payload_bytes=252928000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=253225936 ingress_bytes=287531008 link_bytes=287531008 token_bytes=506153936 weight_movement_bytes=34603008 total_nvlink_bytes=540756944 weight_to_token_ratio=0.068365 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=30720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.081956035e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=292519936 dispatch_meta_bytes=326960 combine_payload_bytes=292519936 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=292846896 ingress_bytes=361725952 link_bytes=361725952 token_bytes=585366832 weight_movement_bytes=69206016 total_nvlink_bytes=654572848 weight_to_token_ratio=0.118227 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=30976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.361145636e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=66794 payload_rows_remote=50606 dup_rows=31510 dup_rows_remote=9400 dedup_ratio=0.320536 router_tokens_per_expert=[0, 3606, 2513, 990, 7461, 7691, 0, 0, 0, 0, 1, 247, 0, 1, 4, 1, 6, 0, 0, 0, 4, 2780, 2, 0, 0, 0, 1428, 2031, 90, 0, 6, 0, 0, 0, 187, 0, 0, 6057, 454, 213, 0, 0, 1441, 0, 4, 100, 118, 3, 2, 2516, 335, 0, 0, 7, 110, 2275, 191, 6178, 1, 6, 90, 0, 1, 1] payload_rows_per_dst=[16188, 508, 8588, 12820, 12902, 2896, 8202, 4690] dup_rows_per_dst=[22110, 0, 1560, 3950, 920, 436, 2288, 246] combine_rows_per_src=[16188, 508, 8588, 12820, 12902, 2896, 8202, 4690] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 1, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=207282176 dispatch_meta_bytes=240024 combine_payload_bytes=207282176 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=207522200 ingress_bytes=380297216 link_bytes=380297216 token_bytes=414804376 weight_movement_bytes=173015040 total_nvlink_bytes=587819416 weight_to_token_ratio=0.417100 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=19968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.431027807e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93882 payload_rows_remote=85460 dup_rows=4422 dup_rows_remote=4074 dedup_ratio=0.044983 router_tokens_per_expert=[2, 0, 0, 0, 0, 0, 0, 4386, 494, 0, 218, 0, 0, 0, 359, 7002, 0, 0, 0, 2921, 246, 759, 0, 459, 0, 0, 0, 0, 0, 0, 0, 0, 3312, 0, 0, 0, 0, 0, 0, 2033, 0, 0, 0, 7870, 0, 0, 250, 0, 0, 0, 0, 4550, 0, 7847, 0, 0, 0, 0, 0, 5784, 114, 484, 0, 62] payload_rows_per_dst=[8772, 14356, 8422, 15694, 9872, 15746, 9100, 11920] dup_rows_per_dst=[4, 1790, 348, 0, 818, 494, 0, 968] combine_rows_per_src=[8772, 14356, 8422, 15694, 9872, 15746, 9100, 11920] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=318087168 dispatch_meta_bytes=349408 combine_payload_bytes=318087168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=318436576 ingress_bytes=387293184 link_bytes=387293184 token_bytes=636523744 weight_movement_bytes=69206016 total_nvlink_bytes=705729760 weight_to_token_ratio=0.108725 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=31232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.457353072e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78344 payload_rows_remote=69942 dup_rows=19960 dup_rows_remote=19730 dedup_ratio=0.203044 router_tokens_per_expert=[3227, 1422, 61, 119, 31, 84, 179, 1244, 438, 1654, 3870, 87, 115, 20, 487, 4532, 21, 471, 102, 287, 309, 273, 93, 634, 34, 48, 30, 32, 73, 198, 94, 308, 3814, 1377, 227, 3, 37, 0, 0, 1261, 15, 1599, 4, 3192, 799, 1506, 168, 63, 46, 66, 13, 2289, 626, 3499, 0, 1006, 40, 351, 63, 5637, 176, 480, 147, 71] payload_rows_per_dst=[10478, 14926, 3878, 8402, 11044, 10186, 6854, 12576] dup_rows_per_dst=[2256, 7480, 502, 230, 2394, 4506, 1238, 1354] combine_rows_per_src=[10478, 14926, 3878, 8402, 11044, 10186, 6854, 12576] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80024 payload_rows_remote=65320 dup_rows=18280 dup_rows_remote=15550 dedup_ratio=0.185954 router_tokens_per_expert=[38, 0, 0, 0, 0, 0, 0, 4198, 492, 0, 266, 0, 0, 0, 424, 6922, 0, 0, 0, 1678, 263, 429, 0, 555, 0, 0, 0, 0, 0, 0, 0, 0, 2595, 0, 0, 0, 0, 0, 0, 3515, 0, 0, 0, 7655, 0, 0, 278, 0, 0, 0, 0, 4496, 0, 7738, 0, 0, 0, 0, 0, 6941, 164, 418, 0, 87] payload_rows_per_dst=[8402, 1962, 15468, 0, 10786, 13226, 15476, 14704] dup_rows_per_dst=[70, 402, 4226, 0, 1434, 426, 8992, 2730] combine_rows_per_src=[8402, 1962, 15468, 0, 10786, 13226, 15476, 14704] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88232 payload_rows_remote=72512 dup_rows=10072 dup_rows_remote=9568 dedup_ratio=0.102458 router_tokens_per_expert=[3, 0, 0, 0, 0, 4, 0, 4641, 540, 1, 230, 0, 0, 0, 395, 7025, 0, 0, 0, 3274, 283, 883, 0, 323, 1, 0, 0, 0, 0, 1, 0, 0, 3148, 0, 0, 0, 0, 0, 0, 2386, 0, 1, 0, 7859, 0, 1, 251, 0, 0, 0, 0, 3559, 1, 7827, 0, 1, 0, 0, 0, 5902, 138, 407, 0, 67] payload_rows_per_dst=[9286, 2198, 16042, 15656, 9996, 15720, 7120, 12214] dup_rows_per_dst=[10, 134, 7534, 2, 1072, 504, 2, 814] combine_rows_per_src=[9286, 2198, 16042, 15656, 9996, 15720, 7120, 12214] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92754 payload_rows_remote=82458 dup_rows=5550 dup_rows_remote=4150 dedup_ratio=0.056458 router_tokens_per_expert=[45, 0, 0, 0, 0, 0, 0, 4068, 490, 0, 347, 0, 0, 0, 309, 6948, 0, 0, 0, 1959, 283, 522, 0, 518, 0, 0, 0, 0, 0, 0, 0, 0, 2520, 0, 0, 0, 0, 0, 0, 3328, 0, 0, 0, 7784, 0, 0, 306, 0, 0, 0, 0, 4645, 0, 7678, 0, 0, 0, 0, 0, 6725, 175, 410, 0, 92] payload_rows_per_dst=[8144, 11374, 8728, 15356, 10296, 15582, 9290, 13984] dup_rows_per_dst=[82, 1260, 1390, 0, 1400, 598, 0, 820] combine_rows_per_src=[8144, 11374, 8728, 15356, 10296, 15582, 9290, 13984] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87106 payload_rows_remote=77658 dup_rows=11198 dup_rows_remote=9694 dedup_ratio=0.113912 router_tokens_per_expert=[355, 98, 1, 7, 1, 5, 7, 3956, 519, 152, 791, 7, 25, 0, 742, 6482, 2, 70, 9, 2050, 259, 508, 15, 594, 2, 1, 0, 0, 0, 8, 0, 21, 3364, 109, 61, 0, 1, 0, 0, 2804, 0, 146, 0, 7092, 70, 132, 266, 1, 3, 0, 0, 3962, 50, 7416, 0, 95, 0, 19, 7, 6192, 165, 418, 14, 78] payload_rows_per_dst=[8612, 3636, 14978, 12160, 10820, 14540, 9448, 12912] dup_rows_per_dst=[248, 836, 5000, 4, 1858, 874, 1504, 874] combine_rows_per_src=[8612, 3636, 14978, 12160, 10820, 14540, 9448, 12912] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=350044160 dispatch_meta_bytes=358136 combine_payload_bytes=350044160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=350402296 ingress_bytes=384647168 link_bytes=384647168 token_bytes=700446456 weight_movement_bytes=34603008 total_nvlink_bytes=735049464 weight_to_token_ratio=0.049401 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.447396327e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86948 payload_rows_remote=75642 dup_rows=11356 dup_rows_remote=11352 dedup_ratio=0.115519 router_tokens_per_expert=[0, 0, 60, 1686, 0, 0, 770, 0, 0, 2847, 789, 0, 1, 1, 794, 1, 168, 461, 0, 2412, 1, 68, 0, 0, 6562, 65, 0, 318, 0, 0, 0, 1, 627, 387, 0, 1, 1450, 2240, 0, 67, 2, 0, 5579, 0, 0, 1, 72, 1, 547, 1, 67, 0, 5564, 57, 16, 4, 6931, 0, 0, 2, 6741, 759, 1031, 0] payload_rows_per_dst=[4870, 7560, 14266, 13892, 8464, 11306, 11608, 14982] dup_rows_per_dst=[162, 1306, 5436, 0, 1080, 4, 904, 2464] combine_rows_per_src=[4870, 7560, 14266, 13892, 8464, 11306, 11608, 14982] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85726 payload_rows_remote=73490 dup_rows=12578 dup_rows_remote=8862 dedup_ratio=0.127950 router_tokens_per_expert=[0, 0, 432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4469, 0, 5200, 0, 0, 340, 58, 0, 0, 0, 459, 0, 2804, 0, 0, 0, 0, 0, 0, 6036, 6009, 0, 126, 0, 0, 0, 247, 0, 658, 3968, 0, 0, 0, 7258, 5017, 73, 0, 704, 0, 0, 227, 0, 92, 0, 0, 0, 4975, 0, 0, 0, 0, 0] payload_rows_per_dst=[14700, 12236, 13786, 9228, 12088, 2382, 11356, 9950] dup_rows_per_dst=[680, 3716, 4768, 1438, 676, 210, 1090, 0] combine_rows_per_src=[14700, 12236, 13786, 9228, 12088, 2382, 11356, 9950] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=286482432 dispatch_meta_bytes=358688 combine_payload_bytes=286482432 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286841120 ingress_bytes=321085440 link_bytes=321085440 token_bytes=573323552 weight_movement_bytes=34603008 total_nvlink_bytes=607926560 weight_to_token_ratio=0.060355 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=48384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.208218661e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267550720 dispatch_meta_bytes=323480 combine_payload_bytes=267550720 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267874200 ingress_bytes=371359744 link_bytes=371359744 token_bytes=535424920 weight_movement_bytes=103809024 total_nvlink_bytes=639233944 weight_to_token_ratio=0.193882 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=41984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397396820e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=337747968 dispatch_meta_bytes=346432 combine_payload_bytes=337747968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=338094400 ingress_bytes=372350976 link_bytes=372350976 token_bytes=675842368 weight_movement_bytes=34603008 total_nvlink_bytes=710445376 weight_to_token_ratio=0.051200 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.401126746e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=318087168 dispatch_meta_bytes=349408 combine_payload_bytes=318087168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=318436576 ingress_bytes=387293184 link_bytes=387293184 token_bytes=636523744 weight_movement_bytes=69206016 total_nvlink_bytes=705729760 weight_to_token_ratio=0.108725 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=31232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.457353072e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90872 payload_rows_remote=79224 dup_rows=7432 dup_rows_remote=6652 dedup_ratio=0.075602 router_tokens_per_expert=[0, 0, 409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4889, 0, 5331, 0, 0, 318, 69, 0, 0, 0, 689, 0, 2208, 0, 0, 0, 0, 0, 0, 6230, 5676, 0, 194, 0, 0, 0, 344, 0, 927, 3720, 0, 0, 0, 7216, 5072, 67, 0, 704, 0, 0, 252, 0, 92, 0, 0, 0, 4745, 0, 0, 0, 0, 0] payload_rows_per_dst=[14616, 9778, 11372, 13960, 11648, 8440, 11568, 9490] dup_rows_per_dst=[634, 0, 924, 2916, 780, 988, 1190, 0] combine_rows_per_src=[14616, 9778, 11372, 13960, 11648, 8440, 11568, 9490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=297009152 dispatch_meta_bytes=328320 combine_payload_bytes=297009152 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=297337472 ingress_bytes=400818176 link_bytes=400818176 token_bytes=594346624 weight_movement_bytes=103809024 total_nvlink_bytes=698155648 weight_to_token_ratio=0.174661 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=42752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.508246528e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78752 payload_rows_remote=63906 dup_rows=19552 dup_rows_remote=12304 dedup_ratio=0.198893 router_tokens_per_expert=[0, 0, 417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4534, 0, 5259, 0, 0, 331, 68, 0, 0, 0, 552, 0, 2231, 0, 0, 0, 0, 0, 0, 6205, 5811, 0, 116, 0, 0, 0, 275, 0, 895, 3838, 0, 0, 0, 7300, 5342, 59, 0, 739, 0, 0, 251, 0, 87, 0, 0, 0, 4842, 0, 0, 0, 0, 0] payload_rows_per_dst=[834, 9068, 11780, 4462, 10278, 15990, 11494, 14846] dup_rows_per_dst=[0, 0, 806, 0, 642, 9678, 1178, 7248] combine_rows_per_src=[834, 9068, 11780, 4462, 10278, 15990, 11494, 14846] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77388 payload_rows_remote=62724 dup_rows=20916 dup_rows_remote=20206 dedup_ratio=0.212769 router_tokens_per_expert=[0, 0, 440, 1, 0, 0, 1, 0, 5, 3, 0, 0, 4280, 0, 5778, 0, 6, 350, 77, 0, 0, 0, 415, 1, 2632, 0, 3, 0, 0, 0, 0, 6208, 5047, 0, 153, 0, 7, 0, 259, 0, 774, 3690, 0, 3, 7, 7245, 5327, 53, 7, 737, 0, 0, 251, 0, 92, 0, 0, 6, 5290, 3, 0, 0, 0, 1] payload_rows_per_dst=[14664, 15072, 14292, 7238, 7680, 1668, 6180, 10594] dup_rows_per_dst=[710, 12570, 5440, 750, 534, 6, 900, 6] combine_rows_per_src=[14664, 15072, 14292, 7238, 7680, 1668, 6180, 10594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85726 payload_rows_remote=73490 dup_rows=12578 dup_rows_remote=8862 dedup_ratio=0.127950 router_tokens_per_expert=[0, 0, 432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4469, 0, 5200, 0, 0, 340, 58, 0, 0, 0, 459, 0, 2804, 0, 0, 0, 0, 0, 0, 6036, 6009, 0, 126, 0, 0, 0, 247, 0, 658, 3968, 0, 0, 0, 7258, 5017, 73, 0, 704, 0, 0, 227, 0, 92, 0, 0, 0, 4975, 0, 0, 0, 0, 0] payload_rows_per_dst=[14700, 12236, 13786, 9228, 12088, 2382, 11356, 9950] dup_rows_per_dst=[680, 3716, 4768, 1438, 676, 210, 1090, 0] combine_rows_per_src=[14700, 12236, 13786, 9228, 12088, 2382, 11356, 9950] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87044 payload_rows_remote=75036 dup_rows=11260 dup_rows_remote=10296 dedup_ratio=0.114543 router_tokens_per_expert=[21, 2, 477, 25, 14, 0, 10, 0, 132, 221, 1, 1, 4519, 4, 5271, 4, 25, 340, 68, 0, 0, 0, 438, 19, 2278, 25, 18, 0, 1, 7, 0, 5715, 5917, 0, 205, 0, 146, 0, 704, 0, 1436, 3816, 10, 46, 151, 6994, 3785, 61, 157, 527, 0, 20, 433, 0, 78, 0, 0, 149, 4747, 116, 4, 5, 4, 5] payload_rows_per_dst=[6852, 9756, 8676, 13414, 13008, 13476, 12008, 9854] dup_rows_per_dst=[404, 8, 674, 2620, 936, 5394, 964, 260] combine_rows_per_src=[6852, 9756, 8676, 13414, 13008, 13476, 12008, 9854] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=324501504 dispatch_meta_bytes=343504 combine_payload_bytes=324501504 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=324845008 ingress_bytes=393707520 link_bytes=393707520 token_bytes=649346512 weight_movement_bytes=69206016 total_nvlink_bytes=718552528 weight_to_token_ratio=0.106578 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=42112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.481489702e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77632 payload_rows_remote=61750 dup_rows=20672 dup_rows_remote=12734 dedup_ratio=0.210286 router_tokens_per_expert=[34, 3, 0, 1, 0, 0, 0, 4225, 477, 7, 259, 0, 0, 0, 378, 7042, 0, 4, 0, 1696, 275, 472, 0, 605, 0, 0, 0, 0, 0, 1, 0, 0, 1930, 4, 1, 0, 0, 0, 0, 4090, 0, 7, 0, 7647, 0, 6, 300, 0, 0, 0, 0, 3944, 4, 7689, 0, 5, 0, 2, 0, 7322, 190, 440, 0, 92] payload_rows_per_dst=[15882, 15110, 5940, 15380, 10762, 614, 12680, 1264] dup_rows_per_dst=[7938, 6276, 164, 0, 1288, 12, 4810, 184] combine_rows_per_src=[15882, 15110, 5940, 15380, 10762, 614, 12680, 1264] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=309829632 dispatch_meta_bytes=347976 combine_payload_bytes=309829632 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=310177608 ingress_bytes=344432640 link_bytes=344432640 token_bytes=620007240 weight_movement_bytes=34603008 total_nvlink_bytes=654610248 weight_to_token_ratio=0.055811 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=45824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.296072295e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=261758976 dispatch_meta_bytes=304840 combine_payload_bytes=261758976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262063816 ingress_bytes=296361984 link_bytes=296361984 token_bytes=523822792 weight_movement_bytes=34603008 total_nvlink_bytes=558425800 weight_to_token_ratio=0.066059 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=39168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.115186287e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=256917504 dispatch_meta_bytes=331720 combine_payload_bytes=256917504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=257249224 ingress_bytes=291520512 link_bytes=291520512 token_bytes=514166728 weight_movement_bytes=34603008 total_nvlink_bytes=548769736 weight_to_token_ratio=0.067299 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=44544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.096968217e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=301015040 dispatch_meta_bytes=329408 combine_payload_bytes=301015040 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=301344448 ingress_bytes=370221056 link_bytes=370221056 token_bytes=602359488 weight_movement_bytes=69206016 total_nvlink_bytes=671565504 weight_to_token_ratio=0.114892 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=33920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393112028e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=252928000 dispatch_meta_bytes=297936 combine_payload_bytes=252928000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=253225936 ingress_bytes=287531008 link_bytes=287531008 token_bytes=506153936 weight_movement_bytes=34603008 total_nvlink_bytes=540756944 weight_to_token_ratio=0.068365 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=30720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.081956035e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93628 payload_rows_remote=84626 dup_rows=4676 dup_rows_remote=4072 dedup_ratio=0.047567 router_tokens_per_expert=[0, 0, 483, 0, 0, 0, 0, 0, 0, 227, 0, 0, 4959, 0, 4770, 0, 0, 422, 37, 0, 0, 0, 536, 0, 899, 0, 0, 0, 0, 0, 0, 6256, 7324, 0, 168, 0, 0, 0, 445, 0, 808, 4589, 0, 0, 0, 7420, 3808, 59, 0, 489, 0, 0, 254, 0, 62, 0, 0, 0, 5137, 0, 0, 0, 0, 0] payload_rows_per_dst=[14966, 10372, 9002, 13158, 15102, 10290, 10464, 10274] dup_rows_per_dst=[840, 0, 604, 1152, 772, 622, 686, 0] combine_rows_per_src=[14966, 10372, 9002, 13158, 15102, 10290, 10464, 10274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=307347456 dispatch_meta_bytes=341328 combine_payload_bytes=307347456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307688784 ingress_bytes=341950464 link_bytes=341950464 token_bytes=615036240 weight_movement_bytes=34603008 total_nvlink_bytes=649639248 weight_to_token_ratio=0.056262 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=36992 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.286732067e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81966 payload_rows_remote=70962 dup_rows=16338 dup_rows_remote=13938 dedup_ratio=0.166199 router_tokens_per_expert=[166, 86, 697, 219, 29, 41, 142, 89, 1087, 203, 91, 5, 1487, 40, 3532, 47, 353, 340, 85, 19, 0, 64, 221, 96, 4476, 332, 332, 22, 127, 58, 18, 1337, 1022, 37, 114, 0, 1575, 1, 1332, 22, 3091, 3689, 208, 703, 1703, 4959, 1884, 64, 1758, 677, 6, 477, 2119, 3, 75, 6, 34, 1708, 4283, 1455, 114, 72, 48, 72] payload_rows_per_dst=[11516, 5736, 5296, 11004, 7878, 14466, 13430, 12640] dup_rows_per_dst=[1340, 184, 828, 2400, 328, 4450, 3876, 2932] combine_rows_per_src=[11516, 5736, 5296, 11004, 7878, 14466, 13430, 12640] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93670 payload_rows_remote=84024 dup_rows=4634 dup_rows_remote=4036 dedup_ratio=0.047139 router_tokens_per_expert=[0, 0, 429, 0, 0, 3, 0, 0, 1, 237, 0, 0, 4666, 0, 5141, 0, 0, 340, 65, 0, 0, 0, 481, 0, 1048, 0, 1, 0, 0, 7, 0, 6226, 7295, 0, 203, 0, 1, 0, 478, 0, 721, 4332, 0, 0, 1, 7342, 4142, 68, 1, 498, 0, 0, 253, 0, 67, 0, 0, 1, 5104, 0, 0, 0, 0, 0] payload_rows_per_dst=[14866, 9808, 9448, 13346, 15092, 9646, 11254, 10210] dup_rows_per_dst=[682, 0, 608, 1218, 862, 598, 666, 0] combine_rows_per_src=[14866, 9808, 9448, 13346, 15092, 9646, 11254, 10210] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78752 payload_rows_remote=63906 dup_rows=19552 dup_rows_remote=12304 dedup_ratio=0.198893 router_tokens_per_expert=[0, 0, 417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4534, 0, 5259, 0, 0, 331, 68, 0, 0, 0, 552, 0, 2231, 0, 0, 0, 0, 0, 0, 6205, 5811, 0, 116, 0, 0, 0, 275, 0, 895, 3838, 0, 0, 0, 7300, 5342, 59, 0, 739, 0, 0, 251, 0, 87, 0, 0, 0, 4842, 0, 0, 0, 0, 0] payload_rows_per_dst=[834, 9068, 11780, 4462, 10278, 15990, 11494, 14846] dup_rows_per_dst=[0, 0, 806, 0, 642, 9678, 1178, 7248] combine_rows_per_src=[834, 9068, 11780, 4462, 10278, 15990, 11494, 14846] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87044 payload_rows_remote=75036 dup_rows=11260 dup_rows_remote=10296 dedup_ratio=0.114543 router_tokens_per_expert=[21, 2, 477, 25, 14, 0, 10, 0, 132, 221, 1, 1, 4519, 4, 5271, 4, 25, 340, 68, 0, 0, 0, 438, 19, 2278, 25, 18, 0, 1, 7, 0, 5715, 5917, 0, 205, 0, 146, 0, 704, 0, 1436, 3816, 10, 46, 151, 6994, 3785, 61, 157, 527, 0, 20, 433, 0, 78, 0, 0, 149, 4747, 116, 4, 5, 4, 5] payload_rows_per_dst=[6852, 9756, 8676, 13414, 13008, 13476, 12008, 9854] dup_rows_per_dst=[404, 8, 674, 2620, 936, 5394, 964, 260] combine_rows_per_src=[6852, 9756, 8676, 13414, 13008, 13476, 12008, 9854] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90872 payload_rows_remote=79224 dup_rows=7432 dup_rows_remote=6652 dedup_ratio=0.075602 router_tokens_per_expert=[0, 0, 409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4889, 0, 5331, 0, 0, 318, 69, 0, 0, 0, 689, 0, 2208, 0, 0, 0, 0, 0, 0, 6230, 5676, 0, 194, 0, 0, 0, 344, 0, 927, 3720, 0, 0, 0, 7216, 5072, 67, 0, 704, 0, 0, 252, 0, 92, 0, 0, 0, 4745, 0, 0, 0, 0, 0] payload_rows_per_dst=[14616, 9778, 11372, 13960, 11648, 8440, 11568, 9490] dup_rows_per_dst=[634, 0, 924, 2916, 780, 988, 1190, 0] combine_rows_per_src=[14616, 9778, 11372, 13960, 11648, 8440, 11568, 9490] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 2, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=346628096 dispatch_meta_bytes=354792 combine_payload_bytes=346628096 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=346982888 ingress_bytes=415834112 link_bytes=415834112 token_bytes=693610984 weight_movement_bytes=69206016 total_nvlink_bytes=762817000 weight_to_token_ratio=0.099776 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=34432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.564750286e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=290660352 dispatch_meta_bytes=339600 combine_payload_bytes=290660352 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=290999952 ingress_bytes=394469376 link_bytes=394469376 token_bytes=581660304 weight_movement_bytes=103809024 total_nvlink_bytes=685469328 weight_to_token_ratio=0.178470 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=28288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.484356504e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86508 payload_rows_remote=78954 dup_rows=11796 dup_rows_remote=11454 dedup_ratio=0.119995 router_tokens_per_expert=[3218, 0, 1, 79, 878, 4313, 0, 0, 1445, 71, 1778, 0, 0, 0, 0, 0, 0, 67, 133, 1, 0, 67, 1, 0, 160, 0, 0, 1, 0, 67, 0, 70, 0, 0, 6908, 4, 0, 0, 1343, 793, 0, 6949, 596, 7354, 58, 1, 4616, 67, 0, 0, 0, 67, 65, 0, 0, 5456, 334, 1, 0, 340, 630, 150, 253, 817] payload_rows_per_dst=[7674, 11986, 14180, 14850, 3332, 7554, 11916, 15016] dup_rows_per_dst=[678, 3228, 256, 454, 948, 342, 2040, 3850] combine_rows_per_src=[7674, 11986, 14180, 14850, 3332, 7554, 11916, 15016] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90556 payload_rows_remote=75538 dup_rows=7748 dup_rows_remote=7240 dedup_ratio=0.078817 router_tokens_per_expert=[0, 244, 0, 0, 0, 0, 307, 339, 92, 7433, 0, 0, 0, 92, 0, 146, 73, 0, 0, 410, 0, 193, 0, 0, 0, 0, 0, 0, 570, 7255, 0, 0, 0, 0, 0, 0, 0, 7051, 0, 8100, 92, 0, 0, 1360, 8, 0, 92, 0, 319, 0, 0, 0, 0, 5593, 0, 0, 0, 0, 464, 7815, 0, 0, 0, 1104] payload_rows_per_dst=[14218, 15018, 16200, 14796, 0, 15998, 11190, 3136] dup_rows_per_dst=[1664, 508, 1352, 854, 0, 2736, 634, 0] combine_rows_per_src=[14218, 15018, 16200, 14796, 0, 15998, 11190, 3136] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=261758976 dispatch_meta_bytes=304840 combine_payload_bytes=261758976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262063816 ingress_bytes=296361984 link_bytes=296361984 token_bytes=523822792 weight_movement_bytes=34603008 total_nvlink_bytes=558425800 weight_to_token_ratio=0.066059 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=39168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.115186287e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=307347456 dispatch_meta_bytes=341328 combine_payload_bytes=307347456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307688784 ingress_bytes=341950464 link_bytes=341950464 token_bytes=615036240 weight_movement_bytes=34603008 total_nvlink_bytes=649639248 weight_to_token_ratio=0.056262 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=36992 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.286732067e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77388 payload_rows_remote=62724 dup_rows=20916 dup_rows_remote=20206 dedup_ratio=0.212769 router_tokens_per_expert=[0, 0, 440, 1, 0, 0, 1, 0, 5, 3, 0, 0, 4280, 0, 5778, 0, 6, 350, 77, 0, 0, 0, 415, 1, 2632, 0, 3, 0, 0, 0, 0, 6208, 5047, 0, 153, 0, 7, 0, 259, 0, 774, 3690, 0, 3, 7, 7245, 5327, 53, 7, 737, 0, 0, 251, 0, 92, 0, 0, 6, 5290, 3, 0, 0, 0, 1] payload_rows_per_dst=[14664, 15072, 14292, 7238, 7680, 1668, 6180, 10594] dup_rows_per_dst=[710, 12570, 5440, 750, 534, 6, 900, 6] combine_rows_per_src=[14664, 15072, 14292, 7238, 7680, 1668, 6180, 10594] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=324501504 dispatch_meta_bytes=343504 combine_payload_bytes=324501504 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=324845008 ingress_bytes=393707520 link_bytes=393707520 token_bytes=649346512 weight_movement_bytes=69206016 total_nvlink_bytes=718552528 weight_to_token_ratio=0.106578 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=42112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.481489702e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=323395584 dispatch_meta_bytes=361632 combine_payload_bytes=323395584 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=323757216 ingress_bytes=496410624 link_bytes=496410624 token_bytes=647152800 weight_movement_bytes=173015040 total_nvlink_bytes=820167840 weight_to_token_ratio=0.267348 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.867953214e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=309403648 dispatch_meta_bytes=331112 combine_payload_bytes=309403648 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=309734760 ingress_bytes=413212672 link_bytes=413212672 token_bytes=619138408 weight_movement_bytes=103809024 total_nvlink_bytes=722947432 weight_to_token_ratio=0.167667 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=43904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.554886018e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=256917504 dispatch_meta_bytes=331720 combine_payload_bytes=256917504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=257249224 ingress_bytes=291520512 link_bytes=291520512 token_bytes=514166728 weight_movement_bytes=34603008 total_nvlink_bytes=548769736 weight_to_token_ratio=0.067299 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=44544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.096968217e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91252 payload_rows_remote=74992 dup_rows=7052 dup_rows_remote=6474 dedup_ratio=0.071737 router_tokens_per_expert=[0, 217, 0, 0, 0, 0, 311, 422, 62, 7346, 0, 0, 0, 62, 0, 114, 59, 0, 0, 46, 0, 184, 0, 0, 0, 0, 0, 0, 512, 7332, 0, 0, 0, 0, 0, 0, 0, 7121, 0, 8130, 62, 0, 0, 1370, 0, 0, 66, 0, 318, 0, 0, 0, 0, 5868, 0, 0, 0, 0, 829, 7884, 0, 0, 0, 837] payload_rows_per_dst=[14316, 14796, 16260, 14784, 0, 16024, 11740, 3332] dup_rows_per_dst=[1826, 372, 578, 904, 0, 2740, 632, 0] combine_rows_per_src=[14316, 14796, 16260, 14784, 0, 16024, 11740, 3332] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13067929672 prefetch_bytes=726663168 reduce_bytes=519045120 weight_movement_bytes=1245708288 egress_bytes=6537840712 ingress_bytes=7775797248 link_bytes=7775797248 total_nvlink_bytes=14313637960 weight_to_token_ratio=0.095326 prefetch_share_of_ingress=0.093452 dedup_ratio=0.177958 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=15 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028043 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.272160828e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79286 payload_rows_remote=67652 dup_rows=19018 dup_rows_remote=18168 dedup_ratio=0.193461 router_tokens_per_expert=[3, 208, 1035, 1563, 1836, 989, 289, 521, 76, 4646, 33, 8, 3, 671, 138, 114, 78, 309, 0, 355, 1533, 254, 1059, 0, 20, 389, 61, 73, 927, 3405, 1361, 6, 341, 1770, 0, 103, 295, 2433, 192, 6081, 73, 14, 118, 2002, 490, 161, 1816, 83, 2724, 44, 53, 15, 16, 451, 5, 0, 60, 3, 806, 3894, 54, 42, 75, 2975] payload_rows_per_dst=[10136, 10640, 15652, 11634, 4690, 12288, 6418, 7828] dup_rows_per_dst=[7618, 738, 3686, 850, 712, 5014, 198, 202] combine_rows_per_src=[10136, 10640, 15652, 11634, 4690, 12288, 6418, 7828] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90908 payload_rows_remote=90908 dup_rows=7396 dup_rows_remote=7396 dedup_ratio=0.075236 router_tokens_per_expert=[0, 248, 0, 0, 0, 0, 320, 323, 92, 7407, 0, 0, 0, 92, 0, 166, 67, 0, 0, 336, 0, 192, 0, 0, 0, 0, 0, 0, 592, 7272, 0, 0, 0, 0, 0, 0, 0, 6985, 0, 8100, 92, 0, 0, 1103, 9, 0, 100, 0, 344, 0, 0, 0, 0, 6008, 0, 0, 0, 0, 608, 7816, 0, 0, 0, 880] payload_rows_per_dst=[14118, 14980, 16200, 14838, 0, 11316, 12026, 7430] dup_rows_per_dst=[1634, 534, 1190, 890, 0, 1430, 678, 1040] combine_rows_per_src=[14118, 14980, 16200, 14838, 0, 11316, 12026, 7430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=344162304 dispatch_meta_bytes=352240 combine_payload_bytes=344162304 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=344514544 ingress_bytes=482574336 link_bytes=482574336 token_bytes=688676848 weight_movement_bytes=138412032 total_nvlink_bytes=827088880 weight_to_token_ratio=0.200983 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=24576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.815888376e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=51170 payload_rows_remote=34786 dup_rows=47134 dup_rows_remote=30100 dedup_ratio=0.479472 router_tokens_per_expert=[0, 229, 0, 0, 0, 0, 295, 333, 86, 7427, 0, 0, 0, 86, 0, 157, 59, 0, 0, 383, 0, 177, 0, 0, 0, 0, 0, 0, 539, 7347, 0, 0, 0, 0, 0, 0, 0, 7041, 0, 8106, 87, 0, 0, 1370, 7, 0, 104, 0, 338, 0, 0, 0, 0, 5699, 0, 0, 0, 0, 484, 7842, 0, 0, 0, 956] payload_rows_per_dst=[1392, 328, 1238, 1078, 16384, 2944, 11422, 16384] dup_rows_per_dst=[322, 330, 0, 0, 28604, 192, 652, 17034] combine_rows_per_src=[1392, 328, 1238, 1078, 16384, 2944, 11422, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86876 payload_rows_remote=72634 dup_rows=11428 dup_rows_remote=9680 dedup_ratio=0.116252 router_tokens_per_expert=[0, 256, 4, 6, 7, 4, 335, 360, 90, 7417, 0, 0, 0, 90, 1, 170, 53, 0, 0, 370, 7, 183, 6, 0, 0, 1, 0, 0, 555, 7236, 5, 0, 0, 7, 0, 0, 0, 7023, 0, 8095, 92, 0, 0, 1420, 22, 0, 100, 0, 347, 0, 0, 0, 0, 5459, 0, 0, 0, 0, 521, 7813, 0, 0, 0, 1097] payload_rows_per_dst=[14242, 15438, 16204, 10770, 14, 16010, 10962, 3236] dup_rows_per_dst=[1748, 4322, 1224, 600, 0, 2884, 650, 0] combine_rows_per_src=[14242, 15438, 16204, 10770, 14, 16010, 10962, 3236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90556 payload_rows_remote=75538 dup_rows=7748 dup_rows_remote=7240 dedup_ratio=0.078817 router_tokens_per_expert=[0, 244, 0, 0, 0, 0, 307, 339, 92, 7433, 0, 0, 0, 92, 0, 146, 73, 0, 0, 410, 0, 193, 0, 0, 0, 0, 0, 0, 570, 7255, 0, 0, 0, 0, 0, 0, 0, 7051, 0, 8100, 92, 0, 0, 1360, 8, 0, 92, 0, 319, 0, 0, 0, 0, 5593, 0, 0, 0, 0, 464, 7815, 0, 0, 0, 1104] payload_rows_per_dst=[14218, 15018, 16200, 14796, 0, 15998, 11190, 3136] dup_rows_per_dst=[1664, 508, 1352, 854, 0, 2736, 634, 0] combine_rows_per_src=[14218, 15018, 16200, 14796, 0, 15998, 11190, 3136] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=67476 payload_rows_remote=55116 dup_rows=30828 dup_rows_remote=24404 dedup_ratio=0.313599 router_tokens_per_expert=[0, 269, 92, 145, 157, 79, 310, 396, 78, 7093, 5, 0, 0, 106, 13, 116, 63, 25, 0, 296, 141, 183, 113, 0, 1, 38, 0, 0, 578, 6756, 119, 0, 4, 154, 0, 7, 54, 6562, 7, 7944, 78, 3, 5, 2212, 90, 12, 253, 5, 554, 3, 4, 1, 1, 4131, 0, 0, 0, 0, 864, 7501, 1, 0, 8, 1522] payload_rows_per_dst=[1932, 4960, 1420, 10234, 16252, 4924, 12360, 15394] dup_rows_per_dst=[964, 298, 222, 348, 17614, 392, 6424, 4566] combine_rows_per_src=[1932, 4960, 1420, 10234, 16252, 4924, 12360, 15394] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90908 payload_rows_remote=90908 dup_rows=7396 dup_rows_remote=7396 dedup_ratio=0.075236 router_tokens_per_expert=[0, 248, 0, 0, 0, 0, 320, 323, 92, 7407, 0, 0, 0, 92, 0, 166, 67, 0, 0, 336, 0, 192, 0, 0, 0, 0, 0, 0, 592, 7272, 0, 0, 0, 0, 0, 0, 0, 6985, 0, 8100, 92, 0, 0, 1103, 9, 0, 100, 0, 344, 0, 0, 0, 0, 6008, 0, 0, 0, 0, 608, 7816, 0, 0, 0, 880] payload_rows_per_dst=[14118, 14980, 16200, 14838, 0, 11316, 12026, 7430] dup_rows_per_dst=[1634, 534, 1190, 890, 0, 1430, 678, 1040] combine_rows_per_src=[14118, 14980, 16200, 14838, 0, 11316, 12026, 7430] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=307167232 dispatch_meta_bytes=325864 combine_payload_bytes=307167232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307493096 ingress_bytes=341770240 link_bytes=341770240 token_bytes=614660328 weight_movement_bytes=34603008 total_nvlink_bytes=649263336 weight_to_token_ratio=0.056296 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=41600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.286053898e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=372359168 dispatch_meta_bytes=393216 combine_payload_bytes=372359168 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=372752384 ingress_bytes=476168192 link_bytes=476168192 token_bytes=745111552 weight_movement_bytes=103809024 total_nvlink_bytes=848920576 weight_to_token_ratio=0.139320 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=21120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.791782572e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=277102592 dispatch_meta_bytes=343280 combine_payload_bytes=277102592 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=277445872 ingress_bytes=346308608 link_bytes=346308608 token_bytes=554548464 weight_movement_bytes=69206016 total_nvlink_bytes=623754480 weight_to_token_ratio=0.124797 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=42368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.303131412e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83646 payload_rows_remote=79452 dup_rows=14658 dup_rows_remote=14656 dedup_ratio=0.149109 router_tokens_per_expert=[5369, 822, 0, 0, 0, 0, 1704, 67, 0, 3274, 0, 4024, 27, 0, 1240, 0, 0, 134, 4066, 2, 0, 403, 6, 1, 0, 0, 1502, 808, 992, 0, 312, 332, 1, 0, 0, 0, 1, 0, 0, 5373, 0, 1, 268, 97, 1564, 167, 7155, 1, 6728, 51, 0, 146, 0, 67, 18, 44, 71, 0, 0, 1, 1232, 67, 0, 1014] payload_rows_per_dst=[4362, 8706, 14420, 12560, 10748, 4194, 13716, 14940] dup_rows_per_dst=[824, 800, 5542, 2956, 2, 2, 392, 4140] combine_rows_per_src=[4362, 8706, 14420, 12560, 10748, 4194, 13716, 14940] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=142483456 dispatch_meta_bytes=259544 combine_payload_bytes=142483456 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=142743000 ingress_bytes=211689472 link_bytes=211689472 token_bytes=285226456 weight_movement_bytes=69206016 total_nvlink_bytes=354432472 weight_to_token_ratio=0.242635 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=25984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=7.965704407e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86876 payload_rows_remote=72634 dup_rows=11428 dup_rows_remote=9680 dedup_ratio=0.116252 router_tokens_per_expert=[0, 256, 4, 6, 7, 4, 335, 360, 90, 7417, 0, 0, 0, 90, 1, 170, 53, 0, 0, 370, 7, 183, 6, 0, 0, 1, 0, 0, 555, 7236, 5, 0, 0, 7, 0, 0, 0, 7023, 0, 8095, 92, 0, 0, 1420, 22, 0, 100, 0, 347, 0, 0, 0, 0, 5459, 0, 0, 0, 0, 521, 7813, 0, 0, 0, 1097] payload_rows_per_dst=[14242, 15438, 16204, 10770, 14, 16010, 10962, 3236] dup_rows_per_dst=[1748, 4322, 1224, 600, 0, 2884, 650, 0] combine_rows_per_src=[14242, 15438, 16204, 10770, 14, 16010, 10962, 3236] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=225755136 dispatch_meta_bytes=318080 combine_payload_bytes=225755136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=226073216 ingress_bytes=260358144 link_bytes=260358144 token_bytes=451828352 weight_movement_bytes=34603008 total_nvlink_bytes=486431360 weight_to_token_ratio=0.076584 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=39168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.797067353e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=372359168 dispatch_meta_bytes=393216 combine_payload_bytes=372359168 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=372752384 ingress_bytes=476168192 link_bytes=476168192 token_bytes=745111552 weight_movement_bytes=103809024 total_nvlink_bytes=848920576 weight_to_token_ratio=0.139320 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=21120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.791782572e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13733814392 prefetch_bytes=865075200 reduce_bytes=380633088 weight_movement_bytes=1245708288 egress_bytes=6870868088 ingress_bytes=8108654592 link_bytes=8108654592 total_nvlink_bytes=14979522680 weight_to_token_ratio=0.090704 prefetch_share_of_ingress=0.106685 dedup_ratio=0.151874 prefetch_slots=50 unused_prefetch_slots=318 slot_utilization=0.135870 reduce_slots=11 remote_demand=25 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028913 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.326618019e-04 prefetch_bytes_per_gemm_flop=1.415307971e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13140420784 prefetch_bytes=830472192 reduce_bytes=795869184 weight_movement_bytes=1626341376 egress_bytes=6574188720 ingress_bytes=8192573440 link_bytes=8192573440 total_nvlink_bytes=14766762160 weight_to_token_ratio=0.123766 prefetch_share_of_ingress=0.101369 dedup_ratio=0.191568 prefetch_slots=48 unused_prefetch_slots=320 slot_utilization=0.130435 reduce_slots=23 remote_demand=24 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027935 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.340347578e-04 prefetch_bytes_per_gemm_flop=1.358695652e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=11759756584 prefetch_bytes=726663168 reduce_bytes=968884224 weight_movement_bytes=1695547392 egress_bytes=5883462952 ingress_bytes=7571841024 link_bytes=7571841024 total_nvlink_bytes=13455303976 weight_to_token_ratio=0.144182 prefetch_share_of_ingress=0.095969 dedup_ratio=0.226170 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=28 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.238792530e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=297508864 dispatch_meta_bytes=329256 combine_payload_bytes=297508864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=297838120 ingress_bytes=332111872 link_bytes=332111872 token_bytes=595346984 weight_movement_bytes=34603008 total_nvlink_bytes=629949992 weight_to_token_ratio=0.058122 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=35840 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.249710237e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=325435392 dispatch_meta_bytes=376432 combine_payload_bytes=325435392 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=325811824 ingress_bytes=429244416 link_bytes=429244416 token_bytes=651247216 weight_movement_bytes=103809024 total_nvlink_bytes=755056240 weight_to_token_ratio=0.159400 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=25344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.615212180e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13014483064 prefetch_bytes=761266176 reduce_bytes=380633088 weight_movement_bytes=1141899264 egress_bytes=6511124600 ingress_bytes=7645257728 link_bytes=7645257728 total_nvlink_bytes=14156382328 weight_to_token_ratio=0.087741 prefetch_share_of_ingress=0.099574 dedup_ratio=0.185047 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=11 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028043 gemm_inner_pad_upper_bound=28448 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.250803884e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=309403648 dispatch_meta_bytes=331112 combine_payload_bytes=309403648 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=309734760 ingress_bytes=413212672 link_bytes=413212672 token_bytes=619138408 weight_movement_bytes=103809024 total_nvlink_bytes=722947432 weight_to_token_ratio=0.167667 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=43904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.554886018e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91252 payload_rows_remote=74992 dup_rows=7052 dup_rows_remote=6474 dedup_ratio=0.071737 router_tokens_per_expert=[0, 217, 0, 0, 0, 0, 311, 422, 62, 7346, 0, 0, 0, 62, 0, 114, 59, 0, 0, 46, 0, 184, 0, 0, 0, 0, 0, 0, 512, 7332, 0, 0, 0, 0, 0, 0, 0, 7121, 0, 8130, 62, 0, 0, 1370, 0, 0, 66, 0, 318, 0, 0, 0, 0, 5868, 0, 0, 0, 0, 829, 7884, 0, 0, 0, 837] payload_rows_per_dst=[14316, 14796, 16260, 14784, 0, 16024, 11740, 3332] dup_rows_per_dst=[1826, 372, 578, 904, 0, 2740, 632, 0] combine_rows_per_src=[14316, 14796, 16260, 14784, 0, 16024, 11740, 3332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=225755136 dispatch_meta_bytes=318080 combine_payload_bytes=225755136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=226073216 ingress_bytes=260358144 link_bytes=260358144 token_bytes=451828352 weight_movement_bytes=34603008 total_nvlink_bytes=486431360 weight_to_token_ratio=0.076584 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=39168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.797067353e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79286 payload_rows_remote=67652 dup_rows=19018 dup_rows_remote=18168 dedup_ratio=0.193461 router_tokens_per_expert=[3, 208, 1035, 1563, 1836, 989, 289, 521, 76, 4646, 33, 8, 3, 671, 138, 114, 78, 309, 0, 355, 1533, 254, 1059, 0, 20, 389, 61, 73, 927, 3405, 1361, 6, 341, 1770, 0, 103, 295, 2433, 192, 6081, 73, 14, 118, 2002, 490, 161, 1816, 83, 2724, 44, 53, 15, 16, 451, 5, 0, 60, 3, 806, 3894, 54, 42, 75, 2975] payload_rows_per_dst=[10136, 10640, 15652, 11634, 4690, 12288, 6418, 7828] dup_rows_per_dst=[7618, 738, 3686, 850, 712, 5014, 198, 202] combine_rows_per_src=[10136, 10640, 15652, 11634, 4690, 12288, 6418, 7828] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13916168648 prefetch_bytes=588251136 reduce_bytes=968884224 weight_movement_bytes=1557135360 egress_bytes=6962061768 ingress_bytes=8511242240 link_bytes=8511242240 total_nvlink_bytes=15473304008 weight_to_token_ratio=0.111894 prefetch_share_of_ingress=0.069115 dedup_ratio=0.145411 prefetch_slots=34 unused_prefetch_slots=334 slot_utilization=0.092391 reduce_slots=28 remote_demand=17 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029674 gemm_inner_pad_upper_bound=27432 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.392483449e-04 prefetch_bytes_per_gemm_flop=9.624094203e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=11878436448 prefetch_bytes=830472192 reduce_bytes=795869184 weight_movement_bytes=1626341376 egress_bytes=5942849120 ingress_bytes=7561928704 link_bytes=7561928704 total_nvlink_bytes=13504777824 weight_to_token_ratio=0.136915 prefetch_share_of_ingress=0.109823 dedup_ratio=0.221790 prefetch_slots=48 unused_prefetch_slots=320 slot_utilization=0.130435 reduce_slots=23 remote_demand=24 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.237170823e-04 prefetch_bytes_per_gemm_flop=1.358695652e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82840 payload_rows_remote=66930 dup_rows=15464 dup_rows_remote=8026 dedup_ratio=0.157308 router_tokens_per_expert=[0, 67, 1608, 0, 0, 0, 3466, 0, 0, 0, 180, 1395, 7310, 0, 1, 3669, 0, 0, 0, 0, 0, 2620, 1, 1, 1, 0, 1, 7832, 0, 1, 2601, 0, 8, 0, 131, 0, 0, 4964, 0, 0, 0, 0, 3564, 0, 0, 1, 0, 277, 0, 0, 372, 0, 3, 1561, 0, 67, 1, 6719, 0, 0, 648, 81, 0, 1] payload_rows_per_dst=[9222, 9554, 14772, 5206, 10190, 15910, 3352, 14634] dup_rows_per_dst=[1060, 936, 5092, 2, 16, 7438, 654, 266] combine_rows_per_src=[9222, 9554, 14772, 5206, 10190, 15910, 3352, 14634] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=274145280 dispatch_meta_bytes=299824 combine_payload_bytes=274145280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274445104 ingress_bytes=308748288 link_bytes=308748288 token_bytes=548590384 weight_movement_bytes=34603008 total_nvlink_bytes=583193392 weight_to_token_ratio=0.063076 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=28544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.161794951e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89368 payload_rows_remote=74972 dup_rows=8936 dup_rows_remote=8788 dedup_ratio=0.090902 router_tokens_per_expert=[44, 0, 0, 0, 1, 67, 413, 0, 0, 71, 89, 0, 0, 103, 2644, 0, 4503, 1, 99, 7197, 7042, 0, 1, 0, 0, 657, 488, 0, 7373, 0, 63, 0, 3674, 1, 1, 0, 253, 3396, 6859, 67, 0, 0, 0, 1, 0, 5, 0, 69, 0, 1286, 0, 629, 0, 1, 0, 0, 0, 0, 0, 254, 1090, 1, 379, 330] payload_rows_per_dst=[14624, 5800, 202, 15804, 12012, 14396, 11014, 15516] dup_rows_per_dst=[144, 14, 0, 1358, 2772, 148, 1824, 2676] combine_rows_per_src=[14624, 5800, 202, 15804, 12012, 14396, 11014, 15516] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=307085312 dispatch_meta_bytes=335040 combine_payload_bytes=307085312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307420352 ingress_bytes=341688320 link_bytes=341688320 token_bytes=614505664 weight_movement_bytes=34603008 total_nvlink_bytes=649108672 weight_to_token_ratio=0.056310 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=46848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.285745640e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86950 payload_rows_remote=71512 dup_rows=11354 dup_rows_remote=10346 dedup_ratio=0.115499 router_tokens_per_expert=[0, 0, 1, 663, 0, 1, 0, 0, 0, 0, 1, 70, 0, 69, 924, 0, 0, 0, 2943, 2973, 7454, 649, 1, 67, 7649, 0, 0, 7146, 60, 63, 0, 0, 0, 1, 126, 0, 63, 0, 0, 6231, 0, 487, 0, 1, 0, 0, 86, 0, 0, 0, 2, 210, 2826, 1, 774, 190, 1242, 67, 13, 0, 1, 5438, 16, 643] payload_rows_per_dst=[14606, 15306, 6948, 246, 12716, 15438, 6814, 14876] dup_rows_per_dst=[1094, 1114, 910, 0, 126, 1008, 1192, 5910] combine_rows_per_src=[14606, 15306, 6948, 246, 12716, 15438, 6814, 14876] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=292913152 dispatch_meta_bytes=327432 combine_payload_bytes=292913152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293240584 ingress_bytes=327516160 link_bytes=327516160 token_bytes=586153736 weight_movement_bytes=34603008 total_nvlink_bytes=620756744 weight_to_token_ratio=0.059034 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=40576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.232416943e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88362 payload_rows_remote=72508 dup_rows=9942 dup_rows_remote=6304 dedup_ratio=0.101135 router_tokens_per_expert=[0, 4221, 2923, 766, 7470, 7708, 0, 0, 0, 0, 0, 253, 0, 0, 1, 0, 1, 0, 0, 15, 0, 1671, 0, 0, 0, 0, 1390, 562, 67, 0, 1, 0, 0, 0, 79, 0, 0, 6736, 232, 510, 0, 104, 2171, 0, 1, 0, 0, 0, 0, 1831, 12, 0, 0, 1, 90, 4419, 203, 5645, 0, 1, 67, 0, 1, 0] payload_rows_per_dst=[7082, 15418, 10146, 3794, 14224, 15854, 10406, 11438] dup_rows_per_dst=[296, 506, 1670, 246, 890, 3638, 2300, 396] combine_rows_per_src=[7082, 15418, 10146, 3794, 14224, 15854, 10406, 11438] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=296992768 dispatch_meta_bytes=315248 combine_payload_bytes=296992768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=297308016 ingress_bytes=331595776 link_bytes=331595776 token_bytes=594300784 weight_movement_bytes=34603008 total_nvlink_bytes=628903792 weight_to_token_ratio=0.058225 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=31104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.247768209e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88232 payload_rows_remote=72512 dup_rows=10072 dup_rows_remote=9568 dedup_ratio=0.102458 router_tokens_per_expert=[3, 0, 0, 0, 0, 4, 0, 4641, 540, 1, 230, 0, 0, 0, 395, 7025, 0, 0, 0, 3274, 283, 883, 0, 323, 1, 0, 0, 0, 0, 1, 0, 0, 3148, 0, 0, 0, 0, 0, 0, 2386, 0, 1, 0, 7859, 0, 1, 251, 0, 0, 0, 0, 3559, 1, 7827, 0, 1, 0, 0, 0, 5902, 138, 407, 0, 67] payload_rows_per_dst=[9286, 2198, 16042, 15656, 9996, 15720, 7120, 12214] dup_rows_per_dst=[10, 134, 7534, 2, 1072, 504, 2, 814] combine_rows_per_src=[9286, 2198, 16042, 15656, 9996, 15720, 7120, 12214] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=297009152 dispatch_meta_bytes=328320 combine_payload_bytes=297009152 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=297337472 ingress_bytes=400818176 link_bytes=400818176 token_bytes=594346624 weight_movement_bytes=103809024 total_nvlink_bytes=698155648 weight_to_token_ratio=0.174661 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=42752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.508246528e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93670 payload_rows_remote=84024 dup_rows=4634 dup_rows_remote=4036 dedup_ratio=0.047139 router_tokens_per_expert=[0, 0, 429, 0, 0, 3, 0, 0, 1, 237, 0, 0, 4666, 0, 5141, 0, 0, 340, 65, 0, 0, 0, 481, 0, 1048, 0, 1, 0, 0, 7, 0, 6226, 7295, 0, 203, 0, 1, 0, 478, 0, 721, 4332, 0, 0, 1, 7342, 4142, 68, 1, 498, 0, 0, 253, 0, 67, 0, 0, 1, 5104, 0, 0, 0, 0, 0] payload_rows_per_dst=[14866, 9808, 9448, 13346, 15092, 9646, 11254, 10210] dup_rows_per_dst=[682, 0, 608, 1218, 862, 598, 666, 0] combine_rows_per_src=[14866, 9808, 9448, 13346, 15092, 9646, 11254, 10210] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=344162304 dispatch_meta_bytes=352240 combine_payload_bytes=344162304 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=344514544 ingress_bytes=482574336 link_bytes=482574336 token_bytes=688676848 weight_movement_bytes=138412032 total_nvlink_bytes=827088880 weight_to_token_ratio=0.200983 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=24576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.815888376e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87564 payload_rows_remote=84288 dup_rows=10740 dup_rows_remote=10604 dedup_ratio=0.109253 router_tokens_per_expert=[0, 280, 0, 1, 1, 1, 331, 340, 67, 7250, 0, 0, 0, 67, 0, 137, 68, 0, 0, 66, 1, 218, 1, 0, 0, 0, 0, 0, 546, 7316, 0, 0, 0, 1, 0, 0, 0, 7110, 0, 8124, 67, 0, 0, 1570, 2, 0, 67, 0, 319, 0, 0, 0, 0, 5840, 0, 0, 0, 0, 838, 7839, 0, 0, 0, 684] payload_rows_per_dst=[10730, 14642, 6596, 14746, 10080, 3276, 11682, 15812] dup_rows_per_dst=[1444, 400, 282, 978, 3954, 136, 636, 2910] combine_rows_per_src=[10730, 14642, 6596, 14746, 10080, 3276, 11682, 15812] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=11759756584 prefetch_bytes=726663168 reduce_bytes=968884224 weight_movement_bytes=1695547392 egress_bytes=5883462952 ingress_bytes=7571841024 link_bytes=7571841024 total_nvlink_bytes=13455303976 weight_to_token_ratio=0.144182 prefetch_share_of_ingress=0.095969 dedup_ratio=0.226170 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=28 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.238792530e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13067929672 prefetch_bytes=726663168 reduce_bytes=519045120 weight_movement_bytes=1245708288 egress_bytes=6537840712 ingress_bytes=7775797248 link_bytes=7775797248 total_nvlink_bytes=14313637960 weight_to_token_ratio=0.095326 prefetch_share_of_ingress=0.093452 dedup_ratio=0.177958 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=15 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028043 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.272160828e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=11878436448 prefetch_bytes=830472192 reduce_bytes=795869184 weight_movement_bytes=1626341376 egress_bytes=5942849120 ingress_bytes=7561928704 link_bytes=7561928704 total_nvlink_bytes=13504777824 weight_to_token_ratio=0.136915 prefetch_share_of_ingress=0.109823 dedup_ratio=0.221790 prefetch_slots=48 unused_prefetch_slots=320 slot_utilization=0.130435 reduce_slots=23 remote_demand=24 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.237170823e-04 prefetch_bytes_per_gemm_flop=1.358695652e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13014483064 prefetch_bytes=761266176 reduce_bytes=380633088 weight_movement_bytes=1141899264 egress_bytes=6511124600 ingress_bytes=7645257728 link_bytes=7645257728 total_nvlink_bytes=14156382328 weight_to_token_ratio=0.087741 prefetch_share_of_ingress=0.099574 dedup_ratio=0.185047 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=11 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028043 gemm_inner_pad_upper_bound=28448 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.250803884e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=307167232 dispatch_meta_bytes=325864 combine_payload_bytes=307167232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307493096 ingress_bytes=341770240 link_bytes=341770240 token_bytes=614660328 weight_movement_bytes=34603008 total_nvlink_bytes=649263336 weight_to_token_ratio=0.056296 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=41600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.286053898e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=277102592 dispatch_meta_bytes=343280 combine_payload_bytes=277102592 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=277445872 ingress_bytes=346308608 link_bytes=346308608 token_bytes=554548464 weight_movement_bytes=69206016 total_nvlink_bytes=623754480 weight_to_token_ratio=0.124797 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=42368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.303131412e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=345243648 dispatch_meta_bytes=379568 combine_payload_bytes=345243648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=345623216 ingress_bytes=379846656 link_bytes=379846656 token_bytes=690866864 weight_movement_bytes=34603008 total_nvlink_bytes=725469872 weight_to_token_ratio=0.050086 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=32512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.429332386e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13733814392 prefetch_bytes=865075200 reduce_bytes=380633088 weight_movement_bytes=1245708288 egress_bytes=6870868088 ingress_bytes=8108654592 link_bytes=8108654592 total_nvlink_bytes=14979522680 weight_to_token_ratio=0.090704 prefetch_share_of_ingress=0.106685 dedup_ratio=0.151874 prefetch_slots=50 unused_prefetch_slots=318 slot_utilization=0.135870 reduce_slots=11 remote_demand=25 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028913 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.326618019e-04 prefetch_bytes_per_gemm_flop=1.415307971e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13556462120 prefetch_bytes=622854144 reduce_bytes=1141899264 weight_movement_bytes=1764753408 egress_bytes=6782136872 ingress_bytes=8539078656 link_bytes=8539078656 total_nvlink_bytes=15321215528 weight_to_token_ratio=0.130178 prefetch_share_of_ingress=0.072942 dedup_ratio=0.157213 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=33 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029348 gemm_inner_pad_upper_bound=28067 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.397037632e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none - [2026-08-26 15:09:16] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 942.1 | throughput per GPU (TFLOP/s/GPU): 135.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.611391E+00 | loss scale: 1.0 | grad norm: 75.148 | number of skipped iterations: 0 | number of nan iterations: 0 | -[after training is done] datetime: 2026-08-26 15:09:16 -[rank0]:[W826 15:09:17.403746844 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank4]:[W826 15:09:18.343627941 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank5]:[W826 15:09:18.593457450 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank7]:[W826 15:09:18.594923505 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank2]:[W826 15:09:18.710390552 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank6]:[W826 15:09:19.063578490 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank3]:[W826 15:09:19.243741176 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank1]:[W826 15:09:19.566282560 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -W0826 15:09:23.647000 68051 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers -W0826 15:09:23.648000 68051 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 68145 closing signal SIGTERM -W0826 15:09:23.648000 68051 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 68146 closing signal SIGTERM -W0826 15:09:23.649000 68051 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 68147 closing signal SIGTERM -W0826 15:09:23.649000 68051 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 68149 closing signal SIGTERM -W0826 15:09:23.649000 68051 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 68150 closing signal SIGTERM -W0826 15:09:23.649000 68051 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 68151 closing signal SIGTERM +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13140420784 prefetch_bytes=830472192 reduce_bytes=795869184 weight_movement_bytes=1626341376 egress_bytes=6574188720 ingress_bytes=8192573440 link_bytes=8192573440 total_nvlink_bytes=14766762160 weight_to_token_ratio=0.123766 prefetch_share_of_ingress=0.101369 dedup_ratio=0.191568 prefetch_slots=48 unused_prefetch_slots=320 slot_utilization=0.130435 reduce_slots=23 remote_demand=24 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027935 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.340347578e-04 prefetch_bytes_per_gemm_flop=1.358695652e-05 anomalies=none + [2026-08-26 15:57:03] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 794.6 | throughput per GPU (TFLOP/s/GPU): 160.6 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.611391E+00 | loss scale: 1.0 | grad norm: 75.148 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-26 15:57:03 +[rank0]:[W826 15:57:05.213617725 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W826 15:57:05.934858799 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W826 15:57:06.199287523 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W826 15:57:06.433227242 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W826 15:57:06.473741963 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W826 15:57:06.698277456 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W826 15:57:06.783929493 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W826 15:57:07.121550383 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 15:57:13.582000 157880 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 15:57:13.583000 157880 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 157983 closing signal SIGTERM +W0826 15:57:13.584000 157880 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 157986 closing signal SIGTERM Traceback (most recent call last): File "/usr/local/bin/torchrun", line 8, in sys.exit(main()) @@ -7032,4 +7028,4 @@ Traceback (most recent call last): time.sleep(monitor_interval) File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 68051 got signal: 15 +torch.distributed.elastic.multiprocessing.api.SignalException: Process 157880 got signal: 15 diff --git a/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1/mem_samples.csv b/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1/mem_samples.csv index 581a6960f5d652065d3aa2a248fa88eb9192b958..9273a6c6a931c6d6a474a131b6d41810604b558d 100644 --- a/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1/mem_samples.csv +++ b/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1/mem_samples.csv @@ -14,7 +14,7 @@ 5, 0 6, 0 7, 0 -0, 4 +0, 1 1, 1 2, 1 3, 1 @@ -26,7 +26,7 @@ 1, 4 2, 4 3, 4 -4, 4 +4, 1 5, 1 6, 1 7, 1 @@ -38,22 +38,22 @@ 5, 4 6, 4 7, 4 -0, 629 -1, 725 -2, 725 -3, 725 -4, 725 -5, 725 -6, 725 -7, 725 -0, 969 -1, 983 -2, 975 -3, 997 -4, 967 -5, 1007 -6, 975 -7, 963 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 821 +1, 821 +2, 821 +3, 821 +4, 821 +5, 821 +6, 821 +7, 821 0, 1955 1, 1955 2, 1955 @@ -62,38 +62,38 @@ 5, 1955 6, 1955 7, 1955 -0, 3875 -1, 3881 -2, 3897 -3, 3855 -4, 3869 -5, 3873 -6, 3895 -7, 3883 -0, 31783 -1, 31827 -2, 31849 -3, 31805 -4, 31849 -5, 31871 -6, 31871 -7, 31893 -0, 35867 -1, 35991 -2, 35861 -3, 35921 -4, 35873 -5, 35921 -6, 35909 -7, 35991 -0, 45843 -1, 46017 -2, 45833 -3, 45925 -4, 45857 -5, 45869 -6, 45869 -7, 45963 +0, 3747 +1, 3749 +2, 3751 +3, 3751 +4, 3755 +5, 3755 +6, 3755 +7, 3755 +0, 31739 +1, 31739 +2, 31739 +3, 31739 +4, 31739 +5, 31739 +6, 31739 +7, 31739 +0, 35765 +1, 35907 +2, 35715 +3, 35847 +4, 35803 +5, 35803 +6, 35821 +7, 35877 +0, 45779 +1, 45885 +2, 45763 +3, 45837 +4, 45773 +5, 45793 +6, 45793 +7, 45895 0, 46289 1, 46409 2, 46283 @@ -102,38 +102,38 @@ 5, 46339 6, 46331 7, 46409 -0, 46293 +0, 46291 1, 46411 -2, 46287 -3, 46347 -4, 46299 -5, 46345 +2, 46285 +3, 46345 +4, 46297 +5, 46341 6, 46333 -7, 46415 -0, 111863 -1, 111853 -2, 111859 -3, 111859 -4, 111869 -5, 111853 -6, 111841 -7, 111857 -0, 139413 -1, 139405 -2, 139407 -3, 139401 -4, 139417 -5, 139399 -6, 140177 -7, 139405 -0, 139679 -1, 139691 -2, 139657 -3, 139655 -4, 139695 -5, 139655 -6, 140417 -7, 139705 +7, 46411 +0, 103447 +1, 103439 +2, 103441 +3, 103437 +4, 103453 +5, 103433 +6, 103489 +7, 103451 +0, 139025 +1, 139017 +2, 139019 +3, 139013 +4, 139029 +5, 139011 +6, 139789 +7, 139017 +0, 139643 +1, 139675 +2, 139685 +3, 139671 +4, 139685 +5, 139677 +6, 140455 +7, 139715 0, 139801 1, 139793 2, 139795 @@ -150,14 +150,14 @@ 5, 142097 6, 142875 7, 142103 -0, 143115 -1, 143077 -2, 143095 -3, 143075 -4, 143125 -5, 143105 -6, 143863 -7, 143101 +0, 143171 +1, 143153 +2, 143163 +3, 143143 +4, 143143 +5, 143131 +6, 143883 +7, 143111 0, 157149 1, 157133 2, 157143 @@ -193,32 +193,40 @@ 0, 156503 1, 156551 2, 157143 -3, 156809 -4, 156505 -5, 156545 +3, 156873 +4, 157151 +5, 156865 6, 156339 7, 156807 -0, 154621 -1, 154669 -2, 155261 -3, 154539 -4, 154623 -5, 155051 -6, 154457 -7, 154925 +0, 154233 +1, 154171 +2, 155003 +3, 154607 +4, 155011 +5, 154599 +6, 154199 +7, 154913 0, 152157 -1, 152925 -2, 152935 -3, 152143 -4, 152649 -5, 153155 -6, 152321 +1, 9176 +2, 153581 +3, 152859 +4, 152943 +5, 152919 +6, 151743 7, 152925 -0, 1 -1, 4 -2, 4 -3, 4 -4, 4 -5, 4 -6, 142237 -7, 4 +0, 92 +1, 92 +2, 143703 +3, 268 +4, 268 +5, 268 +6, 268 +7, 268 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 diff --git a/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1/train.log b/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1/train.log index 8d5736cb4e457197ca09adb532ea1b8bac0ef6e4..b070e784ff2e49d775b922b5fe8d40f2a70af5bf 100644 --- a/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1/train.log +++ b/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1/train.log @@ -1,5 +1,5 @@ -[15:09:26] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) -[15:09:26] WARN: moonep_backend_token_routing_analysis lacks --moe-debug-drop-logging; probe will have no pre/post rank loads +[15:57:15] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +[15:57:15] WARN: moonep_backend_token_routing_analysis lacks --moe-debug-drop-logging; probe will have no pre/post rank loads ============================================================== DSV2_LITE_L24 / moonep (slots/rank=8 policy=matched) E=64 K=6 H'=1408 m=8 @@ -8,17 +8,10 @@ memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1/memory_snapshots/snapshot.pickle repo=/workspace/Megatron-MoE-Echo @ 35ae6e7f ============================================================== -W0826 15:09:28.183000 71159 torch/distributed/run.py:766] -W0826 15:09:28.183000 71159 torch/distributed/run.py:766] ***************************************** -W0826 15:09:28.183000 71159 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -W0826 15:09:28.183000 71159 torch/distributed/run.py:766] ***************************************** -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +W0826 15:57:17.097000 160561 torch/distributed/run.py:766] +W0826 15:57:17.097000 160561 torch/distributed/run.py:766] ***************************************** +W0826 15:57:17.097000 160561 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 15:57:17.097000 160561 torch/distributed/run.py:766] ***************************************** using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer Number of virtual stages per pipeline stage: None @@ -569,8 +562,6 @@ using torch.bfloat16 for parameters ... -------------------- end of arguments --------------------- INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 > building GPT2BPETokenizer tokenizer ... -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS > initializing torch distributed ... @@ -580,51 +571,60 @@ WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank 124653 is connected to is connected to is connected to is connected to is connected to [Gloo] Rank is connected to 7777777 peer ranks. peer ranks. peer ranks. peer ranks. peer ranks. peer ranks. is connected to Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 7777777 peer ranks. - - - - - +[Gloo] Rank 3[Gloo] Rank is connected to 7 peer ranks. 1Expected number of connected peer ranks is : is connected to 77 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 6 is connected to 7 peer ranks. 5Expected number of connected peer ranks is : is connected to 77 + peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 7[Gloo] Rank is connected to 70 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. -Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 43 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 > initialized tensor model parallel with size 1 > initialized pipeline model parallel with size 1 > setting random seeds to 1234 ... [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0[Gloo] Rank peer ranks. 0Expected number of connected peer ranks is : is connected to 00 + peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[rank3]:[W826 15:09:46.272736027 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank1]:[W826 15:09:46.272757667 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank2]:[W826 15:09:46.278404952 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank5]:[W826 15:09:46.278413537 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank6]:[W826 15:09:46.278423280 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank4]:[W826 15:09:46.278433231 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank7]:[W826 15:09:46.278732807 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[Gloo] Rank [Gloo] Rank 0 is connected to 00 is connected to peer ranks. 0Expected number of connected peer ranks is : peer ranks. 0Expected number of connected peer ranks is : +0 +[rank3]:[W826 15:57:35.729233670 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 15:57:35.729327118 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 15:57:35.729413293 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 15:57:35.729434399 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 15:57:35.729643108 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 15:57:35.729718319 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 15:57:35.730166007 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. > compiling dataset index builder ... make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' make: Nothing to be done for 'default'. make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' ->>> done with dataset index builder. Compilation time: 0.033 seconds +>>> done with dataset index builder. Compilation time: 0.034 seconds WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. > compiling and loading fused kernels ... -[rank0]:[W826 15:09:46.452554987 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. ->>> done with compiling and loading fused kernels. Compilation time: 4.070 seconds -time to initialize megatron (seconds): 8.181 -[after megatron is initialized] datetime: 2026-08-26 15:09:52 +[rank0]:[W826 15:57:35.901732052 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.664 seconds +time to initialize megatron (seconds): 7.131 +[after megatron is initialized] datetime: 2026-08-26 15:57:41 building GPT model ... /workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1542: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. warnings.warn( @@ -664,173 +664,173 @@ INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradi Params for bucket 1 (103022592 elements, 103022592 padded size): module.output_layer.weight Params for bucket 2 (48503296 elements, 48503296 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight module.decoder.layers.23.pre_mlp_layernorm.weight module.decoder.layers.23.mlp.router.weight - module.decoder.layers.23.self_attention.linear_q_proj.weight - module.decoder.layers.23.input_layernorm.weight - module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight - module.decoder.final_layernorm.weight module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight module.decoder.layers.23.self_attention.linear_kv_up_proj.weight module.decoder.layers.23.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.23.self_attention.linear_kv_down_proj.weight + module.decoder.layers.23.self_attention.linear_q_proj.weight module.decoder.layers.23.self_attention.linear_proj.weight + module.decoder.layers.23.input_layernorm.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight Params for bucket 3 (40901632 elements, 40901632 padded size): - module.decoder.layers.22.self_attention.linear_kv_up_proj.weight - module.decoder.layers.22.self_attention.linear_kv_down_proj.weight - module.decoder.layers.22.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.22.mlp.router.weight module.decoder.layers.22.self_attention.linear_q_proj.weight module.decoder.layers.22.self_attention.linear_proj.weight module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.21.mlp.router.weight module.decoder.layers.21.pre_mlp_layernorm.weight - module.decoder.layers.22.mlp.router.weight + module.decoder.layers.22.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.22.pre_mlp_layernorm.weight + module.decoder.layers.22.self_attention.linear_kv_down_proj.weight + module.decoder.layers.22.self_attention.linear_kv_up_proj.weight module.decoder.layers.22.input_layernorm.weight + module.decoder.layers.21.mlp.router.weight module.decoder.layers.21.self_attention.linear_kv_up_proj.weight module.decoder.layers.21.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.21.self_attention.linear_kv_down_proj.weight module.decoder.layers.21.self_attention.linear_q_proj.weight Params for bucket 4 (41163264 elements, 41163264 padded size): - module.decoder.layers.21.self_attention.linear_proj.weight + module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight module.decoder.layers.21.input_layernorm.weight - module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight module.decoder.layers.20.self_attention.linear_kv_up_proj.weight module.decoder.layers.20.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.20.self_attention.linear_kv_down_proj.weight module.decoder.layers.20.self_attention.linear_q_proj.weight module.decoder.layers.20.self_attention.linear_proj.weight + module.decoder.layers.20.input_layernorm.weight module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.21.self_attention.linear_proj.weight module.decoder.layers.20.mlp.router.weight module.decoder.layers.20.pre_mlp_layernorm.weight - module.decoder.layers.20.input_layernorm.weight Params for bucket 5 (42734080 elements, 42734080 padded size): + module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.19.self_attention.linear_kv_up_proj.weight + module.decoder.layers.19.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.19.self_attention.linear_kv_down_proj.weight + module.decoder.layers.19.self_attention.linear_q_proj.weight module.decoder.layers.19.pre_mlp_layernorm.weight module.decoder.layers.19.mlp.router.weight module.decoder.layers.19.self_attention.linear_proj.weight module.decoder.layers.19.input_layernorm.weight module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.19.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.19.self_attention.linear_kv_up_proj.weight - module.decoder.layers.19.self_attention.linear_kv_down_proj.weight - module.decoder.layers.19.self_attention.linear_q_proj.weight Params for bucket 6 (40901632 elements, 40901632 padded size): - module.decoder.layers.18.self_attention.linear_kv_up_proj.weight - module.decoder.layers.18.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.18.self_attention.linear_kv_down_proj.weight - module.decoder.layers.18.self_attention.linear_q_proj.weight + module.decoder.layers.18.mlp.router.weight + module.decoder.layers.18.pre_mlp_layernorm.weight module.decoder.layers.18.input_layernorm.weight - module.decoder.layers.17.pre_mlp_layernorm.weight + module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight module.decoder.layers.17.self_attention.linear_kv_up_proj.weight module.decoder.layers.17.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.17.self_attention.linear_kv_down_proj.weight - module.decoder.layers.17.self_attention.linear_q_proj.weight - module.decoder.layers.18.mlp.router.weight - module.decoder.layers.18.pre_mlp_layernorm.weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.weight + module.decoder.layers.18.self_attention.linear_kv_down_proj.weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.18.self_attention.linear_q_proj.weight module.decoder.layers.18.self_attention.linear_proj.weight - module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight module.decoder.layers.17.mlp.router.weight + module.decoder.layers.17.pre_mlp_layernorm.weight + module.decoder.layers.17.self_attention.linear_q_proj.weight Params for bucket 7 (41163264 elements, 41163264 padded size): - module.decoder.layers.16.mlp.router.weight - module.decoder.layers.16.pre_mlp_layernorm.weight - module.decoder.layers.16.self_attention.linear_q_proj.weight - module.decoder.layers.16.self_attention.linear_proj.weight - module.decoder.layers.16.input_layernorm.weight - module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight module.decoder.layers.17.self_attention.linear_proj.weight - module.decoder.layers.17.input_layernorm.weight - module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.17.input_layernorm.weight module.decoder.layers.16.self_attention.linear_kv_up_proj.weight module.decoder.layers.16.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.16.self_attention.linear_kv_down_proj.weight + module.decoder.layers.16.self_attention.linear_proj.weight + module.decoder.layers.16.mlp.router.weight + module.decoder.layers.16.pre_mlp_layernorm.weight + module.decoder.layers.16.self_attention.linear_q_proj.weight + module.decoder.layers.16.input_layernorm.weight + module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight Params for bucket 8 (42734080 elements, 42734080 padded size): module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.15.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.15.self_attention.linear_kv_up_proj.weight + module.decoder.layers.15.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.15.self_attention.linear_kv_down_proj.weight - module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.15.self_attention.linear_proj.weight module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.15.mlp.router.weight module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.15.mlp.router.weight module.decoder.layers.15.self_attention.linear_q_proj.weight - module.decoder.layers.15.self_attention.linear_proj.weight module.decoder.layers.15.input_layernorm.weight + module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight Params for bucket 9 (40901632 elements, 40901632 padded size): - module.decoder.layers.14.mlp.router.weight - module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.14.self_attention.linear_kv_down_proj.weight + module.decoder.layers.14.self_attention.linear_kv_up_proj.weight module.decoder.layers.14.self_attention.linear_q_proj.weight + module.decoder.layers.14.self_attention.linear_proj.weight module.decoder.layers.14.input_layernorm.weight module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.linear_q_proj.weight + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.pre_mlp_layernorm.weight module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.13.mlp.router.weight module.decoder.layers.13.self_attention.linear_kv_up_proj.weight module.decoder.layers.13.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.13.self_attention.linear_kv_down_proj.weight - module.decoder.layers.13.self_attention.linear_q_proj.weight - module.decoder.layers.14.self_attention.linear_kv_up_proj.weight - module.decoder.layers.14.self_attention.linear_kv_down_proj.weight - module.decoder.layers.14.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.14.self_attention.linear_proj.weight - module.decoder.layers.13.mlp.router.weight - module.decoder.layers.13.pre_mlp_layernorm.weight Params for bucket 10 (41163264 elements, 41163264 padded size): - module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight module.decoder.layers.13.input_layernorm.weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.linear_proj.weight module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight module.decoder.layers.12.self_attention.linear_kv_up_proj.weight module.decoder.layers.12.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.12.self_attention.linear_kv_down_proj.weight module.decoder.layers.12.self_attention.linear_q_proj.weight + module.decoder.layers.12.self_attention.linear_proj.weight module.decoder.layers.12.input_layernorm.weight module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.12.mlp.router.weight - module.decoder.layers.12.pre_mlp_layernorm.weight - module.decoder.layers.12.self_attention.linear_proj.weight Params for bucket 11 (42734080 elements, 42734080 padded size): - module.decoder.layers.11.pre_mlp_layernorm.weight - module.decoder.layers.11.self_attention.linear_q_proj.weight - module.decoder.layers.11.input_layernorm.weight - module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight module.decoder.layers.11.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.11.self_attention.linear_kv_up_proj.weight - module.decoder.layers.11.mlp.router.weight module.decoder.layers.11.self_attention.linear_kv_down_proj.weight module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.linear_q_proj.weight + module.decoder.layers.11.input_layernorm.weight + module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight Params for bucket 12 (40901632 elements, 40901632 padded size): module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_kv_up_proj.weight + module.decoder.layers.9.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.9.self_attention.linear_kv_down_proj.weight + module.decoder.layers.10.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.10.self_attention.linear_kv_up_proj.weight module.decoder.layers.10.self_attention.linear_kv_down_proj.weight - module.decoder.layers.10.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.10.self_attention.linear_q_proj.weight module.decoder.layers.10.input_layernorm.weight module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.9.self_attention.linear_kv_up_proj.weight - module.decoder.layers.9.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.10.pre_mlp_layernorm.weight - module.decoder.layers.10.self_attention.linear_proj.weight module.decoder.layers.9.mlp.router.weight - module.decoder.layers.9.pre_mlp_layernorm.weight module.decoder.layers.9.self_attention.linear_q_proj.weight - module.decoder.layers.9.self_attention.linear_kv_down_proj.weight Params for bucket 13 (41163264 elements, 41163264 padded size): module.decoder.layers.8.mlp.router.weight - module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.9.input_layernorm.weight module.decoder.layers.8.pre_mlp_layernorm.weight module.decoder.layers.8.self_attention.linear_proj.weight module.decoder.layers.9.self_attention.linear_proj.weight module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.9.input_layernorm.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight module.decoder.layers.8.self_attention.linear_kv_up_proj.weight module.decoder.layers.8.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.8.self_attention.linear_kv_down_proj.weight @@ -838,64 +838,65 @@ Params for bucket 13 (41163264 elements, 41163264 padded size): module.decoder.layers.8.input_layernorm.weight module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight Params for bucket 14 (42734080 elements, 42734080 padded size): - module.decoder.layers.7.pre_mlp_layernorm.weight module.decoder.layers.7.mlp.router.weight - module.decoder.layers.7.self_attention.linear_proj.weight - module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight module.decoder.layers.7.self_attention.linear_kv_up_proj.weight + module.decoder.layers.7.pre_mlp_layernorm.weight module.decoder.layers.7.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.7.self_attention.linear_kv_down_proj.weight module.decoder.layers.7.self_attention.linear_q_proj.weight + module.decoder.layers.7.self_attention.linear_proj.weight module.decoder.layers.7.input_layernorm.weight module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight Params for bucket 15 (40901632 elements, 40901632 padded size): module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_kv_up_proj.weight + module.decoder.layers.5.self_attention.linear_q_proj.weight module.decoder.layers.6.self_attention.linear_kv_up_proj.weight - module.decoder.layers.6.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.6.self_attention.linear_kv_down_proj.weight - module.decoder.layers.5.mlp.router.weight - module.decoder.layers.5.self_attention.linear_kv_down_proj.weight - module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.6.self_attention.linear_q_proj.weight - module.decoder.layers.6.self_attention.linear_proj.weight module.decoder.layers.6.input_layernorm.weight module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.5.pre_mlp_layernorm.weight - module.decoder.layers.5.self_attention.linear_kv_up_proj.weight module.decoder.layers.5.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.5.self_attention.linear_q_proj.weight + module.decoder.layers.5.self_attention.linear_kv_down_proj.weight Params for bucket 16 (41163264 elements, 41163264 padded size): - module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.4.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.4.self_attention.linear_kv_down_proj.weight - module.decoder.layers.4.self_attention.linear_q_proj.weight - module.decoder.layers.4.input_layernorm.weight - module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight module.decoder.layers.5.self_attention.linear_proj.weight module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.5.input_layernorm.weight module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.5.input_layernorm.weight module.decoder.layers.4.pre_mlp_layernorm.weight module.decoder.layers.4.self_attention.linear_kv_up_proj.weight + module.decoder.layers.4.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.4.self_attention.linear_kv_down_proj.weight + module.decoder.layers.4.self_attention.linear_q_proj.weight module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.4.input_layernorm.weight + module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight Params for bucket 17 (42734080 elements, 42734080 padded size): + module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.3.self_attention.linear_kv_up_proj.weight + module.decoder.layers.3.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.3.self_attention.linear_kv_down_proj.weight + module.decoder.layers.3.input_layernorm.weight + module.decoder.layers.3.pre_mlp_layernorm.weight module.decoder.layers.3.mlp.router.weight module.decoder.layers.3.self_attention.linear_q_proj.weight module.decoder.layers.3.self_attention.linear_proj.weight - module.decoder.layers.3.input_layernorm.weight module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.3.pre_mlp_layernorm.weight - module.decoder.layers.3.self_attention.linear_kv_up_proj.weight - module.decoder.layers.3.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.3.self_attention.linear_kv_down_proj.weight Params for bucket 18 (40901632 elements, 40901632 padded size): module.decoder.layers.2.self_attention.linear_kv_down_proj.weight - module.decoder.layers.2.self_attention.linear_kv_up_proj.weight module.decoder.layers.2.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.2.self_attention.linear_kv_up_proj.weight + module.decoder.layers.2.self_attention.linear_proj.weight module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight module.decoder.layers.1.pre_mlp_layernorm.weight module.decoder.layers.1.self_attention.linear_kv_up_proj.weight @@ -904,65 +905,64 @@ Params for bucket 18 (40901632 elements, 40901632 padded size): module.decoder.layers.2.mlp.router.weight module.decoder.layers.2.pre_mlp_layernorm.weight module.decoder.layers.2.self_attention.linear_q_proj.weight - module.decoder.layers.2.self_attention.linear_proj.weight module.decoder.layers.2.input_layernorm.weight module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight module.decoder.layers.1.mlp.router.weight module.decoder.layers.1.self_attention.linear_kv_down_proj.weight Params for bucket 19 (71436288 elements, 71436288 padded size): module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.1.input_layernorm.weight module.decoder.layers.0.mlp.linear_fc1.weight module.decoder.layers.0.mlp.linear_fc2.weight - module.decoder.layers.1.input_layernorm.weight Params for bucket 20 (116789760 elements, 116789760 padded size): module.decoder.layers.0.mlp.linear_fc1.layer_norm_weight + module.decoder.layers.0.self_attention.linear_kv_down_proj.weight module.decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.0.self_attention.linear_q_proj.weight module.decoder.layers.0.self_attention.linear_kv_up_proj.weight module.decoder.layers.0.self_attention.linear_proj.weight module.decoder.layers.0.input_layernorm.weight module.embedding.word_embeddings.weight - module.decoder.layers.0.self_attention.linear_kv_down_proj.weight INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 40 Params for bucket 1 (40370176 elements, 40370176 padded size): + module.decoder.layers.23.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc2.weight6 + module.decoder.layers.23.mlp.experts.linear_fc2.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight1 + module.decoder.layers.23.mlp.experts.linear_fc1.weight6 module.decoder.layers.23.mlp.experts.linear_fc2.weight7 - module.decoder.layers.23.mlp.experts.linear_fc2.weight5 module.decoder.layers.23.mlp.experts.linear_fc2.weight3 - module.decoder.layers.23.mlp.experts.linear_fc2.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight5 module.decoder.layers.23.mlp.experts.linear_fc2.weight0 - module.decoder.layers.23.mlp.experts.linear_fc1.weight6 - module.decoder.layers.23.mlp.experts.linear_fc2.weight6 - module.decoder.layers.23.mlp.experts.linear_fc2.weight4 - module.decoder.layers.23.mlp.experts.linear_fc2.weight1 module.decoder.layers.23.mlp.experts.linear_fc1.weight7 module.decoder.layers.23.mlp.experts.linear_fc1.weight5 Params for bucket 2 (40370176 elements, 40370176 padded size): module.decoder.layers.23.mlp.experts.linear_fc1.weight4 module.decoder.layers.23.mlp.experts.linear_fc1.weight3 module.decoder.layers.23.mlp.experts.linear_fc1.weight0 - module.decoder.layers.22.mlp.experts.linear_fc2.weight6 - module.decoder.layers.22.mlp.experts.linear_fc2.weight4 - module.decoder.layers.23.mlp.experts.linear_fc1.weight2 - module.decoder.layers.23.mlp.experts.linear_fc1.weight1 module.decoder.layers.22.mlp.experts.linear_fc2.weight7 module.decoder.layers.22.mlp.experts.linear_fc2.weight5 + module.decoder.layers.23.mlp.experts.linear_fc1.weight1 + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 + module.decoder.layers.22.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc2.weight4 Params for bucket 3 (40370176 elements, 40370176 padded size): - module.decoder.layers.22.mlp.experts.linear_fc2.weight1 - module.decoder.layers.22.mlp.experts.linear_fc1.weight7 - module.decoder.layers.22.mlp.experts.linear_fc1.weight5 module.decoder.layers.22.mlp.experts.linear_fc2.weight3 - module.decoder.layers.22.mlp.experts.linear_fc2.weight2 module.decoder.layers.22.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight2 module.decoder.layers.22.mlp.experts.linear_fc1.weight6 module.decoder.layers.22.mlp.experts.linear_fc1.weight4 module.decoder.layers.22.mlp.experts.linear_fc1.weight3 -Params for bucket 4 (40370176 elements, 40370176 padded size): - module.decoder.layers.22.mlp.experts.linear_fc1.weight2 - module.decoder.layers.22.mlp.experts.linear_fc1.weight1 + module.decoder.layers.22.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight7 + module.decoder.layers.22.mlp.experts.linear_fc1.weight5 +Params for bucket 4 (40370176 elements, 40370176 padded size): module.decoder.layers.21.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight0 module.decoder.layers.21.mlp.experts.linear_fc2.weight4 module.decoder.layers.21.mlp.experts.linear_fc2.weight1 - module.decoder.layers.22.mlp.experts.linear_fc1.weight0 + module.decoder.layers.22.mlp.experts.linear_fc1.weight2 + module.decoder.layers.22.mlp.experts.linear_fc1.weight1 module.decoder.layers.21.mlp.experts.linear_fc2.weight7 module.decoder.layers.21.mlp.experts.linear_fc2.weight5 module.decoder.layers.21.mlp.experts.linear_fc2.weight3 @@ -982,8 +982,8 @@ Params for bucket 6 (40370176 elements, 40370176 padded size): module.decoder.layers.20.mlp.experts.linear_fc2.weight1 module.decoder.layers.20.mlp.experts.linear_fc1.weight7 module.decoder.layers.21.mlp.experts.linear_fc1.weight0 - module.decoder.layers.20.mlp.experts.linear_fc2.weight7 module.decoder.layers.20.mlp.experts.linear_fc2.weight5 + module.decoder.layers.20.mlp.experts.linear_fc2.weight7 module.decoder.layers.20.mlp.experts.linear_fc2.weight3 module.decoder.layers.20.mlp.experts.linear_fc2.weight2 module.decoder.layers.20.mlp.experts.linear_fc2.weight0 @@ -992,90 +992,90 @@ Params for bucket 7 (40370176 elements, 40370176 padded size): module.decoder.layers.20.mlp.experts.linear_fc1.weight5 module.decoder.layers.20.mlp.experts.linear_fc1.weight1 module.decoder.layers.20.mlp.experts.linear_fc1.weight2 - module.decoder.layers.19.mlp.experts.linear_fc2.weight6 module.decoder.layers.20.mlp.experts.linear_fc1.weight3 module.decoder.layers.20.mlp.experts.linear_fc1.weight4 module.decoder.layers.20.mlp.experts.linear_fc1.weight0 module.decoder.layers.19.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc2.weight6 Params for bucket 8 (40370176 elements, 40370176 padded size): - module.decoder.layers.19.mlp.experts.linear_fc2.weight4 - module.decoder.layers.19.mlp.experts.linear_fc2.weight2 - module.decoder.layers.19.mlp.experts.linear_fc2.weight1 - module.decoder.layers.19.mlp.experts.linear_fc1.weight6 - module.decoder.layers.19.mlp.experts.linear_fc1.weight4 module.decoder.layers.19.mlp.experts.linear_fc2.weight5 module.decoder.layers.19.mlp.experts.linear_fc2.weight3 module.decoder.layers.19.mlp.experts.linear_fc2.weight0 module.decoder.layers.19.mlp.experts.linear_fc1.weight7 module.decoder.layers.19.mlp.experts.linear_fc1.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight1 + module.decoder.layers.19.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc1.weight6 + module.decoder.layers.19.mlp.experts.linear_fc1.weight4 Params for bucket 9 (40370176 elements, 40370176 padded size): - module.decoder.layers.19.mlp.experts.linear_fc1.weight3 module.decoder.layers.18.mlp.experts.linear_fc2.weight7 - module.decoder.layers.19.mlp.experts.linear_fc1.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 module.decoder.layers.18.mlp.experts.linear_fc2.weight5 module.decoder.layers.18.mlp.experts.linear_fc2.weight3 - module.decoder.layers.19.mlp.experts.linear_fc1.weight1 - module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 + module.decoder.layers.19.mlp.experts.linear_fc1.weight0 module.decoder.layers.18.mlp.experts.linear_fc2.weight6 module.decoder.layers.18.mlp.experts.linear_fc2.weight4 module.decoder.layers.18.mlp.experts.linear_fc2.weight2 Params for bucket 10 (40370176 elements, 40370176 padded size): - module.decoder.layers.18.mlp.experts.linear_fc1.weight5 - module.decoder.layers.18.mlp.experts.linear_fc1.weight7 module.decoder.layers.18.mlp.experts.linear_fc2.weight0 - module.decoder.layers.18.mlp.experts.linear_fc1.weight2 - module.decoder.layers.18.mlp.experts.linear_fc2.weight1 module.decoder.layers.18.mlp.experts.linear_fc1.weight6 module.decoder.layers.18.mlp.experts.linear_fc1.weight4 module.decoder.layers.18.mlp.experts.linear_fc1.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight5 + module.decoder.layers.18.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight2 Params for bucket 11 (40370176 elements, 40370176 padded size): - module.decoder.layers.18.mlp.experts.linear_fc1.weight1 - module.decoder.layers.17.mlp.experts.linear_fc2.weight6 - module.decoder.layers.17.mlp.experts.linear_fc2.weight3 - module.decoder.layers.17.mlp.experts.linear_fc2.weight2 - module.decoder.layers.17.mlp.experts.linear_fc2.weight0 module.decoder.layers.17.mlp.experts.linear_fc2.weight7 module.decoder.layers.17.mlp.experts.linear_fc2.weight5 module.decoder.layers.18.mlp.experts.linear_fc1.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight3 + module.decoder.layers.17.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight6 module.decoder.layers.17.mlp.experts.linear_fc2.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight2 module.decoder.layers.17.mlp.experts.linear_fc2.weight1 - module.decoder.layers.17.mlp.experts.linear_fc1.weight7 + module.decoder.layers.17.mlp.experts.linear_fc2.weight0 Params for bucket 12 (40370176 elements, 40370176 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight2 + module.decoder.layers.17.mlp.experts.linear_fc1.weight5 + module.decoder.layers.17.mlp.experts.linear_fc1.weight1 module.decoder.layers.17.mlp.experts.linear_fc1.weight6 module.decoder.layers.17.mlp.experts.linear_fc1.weight4 module.decoder.layers.17.mlp.experts.linear_fc1.weight3 module.decoder.layers.17.mlp.experts.linear_fc1.weight0 - module.decoder.layers.17.mlp.experts.linear_fc1.weight2 - module.decoder.layers.17.mlp.experts.linear_fc1.weight5 - module.decoder.layers.17.mlp.experts.linear_fc1.weight1 Params for bucket 13 (40370176 elements, 40370176 padded size): module.decoder.layers.16.mlp.experts.linear_fc2.weight7 module.decoder.layers.16.mlp.experts.linear_fc2.weight5 module.decoder.layers.16.mlp.experts.linear_fc2.weight3 - module.decoder.layers.16.mlp.experts.linear_fc2.weight0 - module.decoder.layers.16.mlp.experts.linear_fc1.weight6 - module.decoder.layers.16.mlp.experts.linear_fc2.weight6 + module.decoder.layers.16.mlp.experts.linear_fc1.weight7 + module.decoder.layers.16.mlp.experts.linear_fc1.weight5 module.decoder.layers.16.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc2.weight6 module.decoder.layers.16.mlp.experts.linear_fc2.weight2 module.decoder.layers.16.mlp.experts.linear_fc2.weight1 - module.decoder.layers.16.mlp.experts.linear_fc1.weight7 - module.decoder.layers.16.mlp.experts.linear_fc1.weight5 + module.decoder.layers.16.mlp.experts.linear_fc2.weight0 + module.decoder.layers.16.mlp.experts.linear_fc1.weight6 Params for bucket 14 (40370176 elements, 40370176 padded size): + module.decoder.layers.16.mlp.experts.linear_fc1.weight2 + module.decoder.layers.16.mlp.experts.linear_fc1.weight1 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 module.decoder.layers.16.mlp.experts.linear_fc1.weight4 module.decoder.layers.16.mlp.experts.linear_fc1.weight0 module.decoder.layers.16.mlp.experts.linear_fc1.weight3 module.decoder.layers.15.mlp.experts.linear_fc2.weight6 module.decoder.layers.15.mlp.experts.linear_fc2.weight4 - module.decoder.layers.16.mlp.experts.linear_fc1.weight1 - module.decoder.layers.16.mlp.experts.linear_fc1.weight2 - module.decoder.layers.15.mlp.experts.linear_fc2.weight7 - module.decoder.layers.15.mlp.experts.linear_fc2.weight5 Params for bucket 15 (40370176 elements, 40370176 padded size): - module.decoder.layers.15.mlp.experts.linear_fc2.weight2 - module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 module.decoder.layers.15.mlp.experts.linear_fc1.weight7 module.decoder.layers.15.mlp.experts.linear_fc1.weight5 - module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 module.decoder.layers.15.mlp.experts.linear_fc2.weight0 module.decoder.layers.15.mlp.experts.linear_fc1.weight6 module.decoder.layers.15.mlp.experts.linear_fc1.weight4 @@ -1107,8 +1107,8 @@ Params for bucket 18 (40370176 elements, 40370176 padded size): module.decoder.layers.13.mlp.experts.linear_fc2.weight1 module.decoder.layers.13.mlp.experts.linear_fc1.weight7 module.decoder.layers.14.mlp.experts.linear_fc1.weight0 - module.decoder.layers.13.mlp.experts.linear_fc2.weight5 module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 module.decoder.layers.13.mlp.experts.linear_fc2.weight3 module.decoder.layers.13.mlp.experts.linear_fc2.weight0 module.decoder.layers.13.mlp.experts.linear_fc1.weight6 @@ -1122,104 +1122,104 @@ Params for bucket 19 (40370176 elements, 40370176 padded size): module.decoder.layers.13.mlp.experts.linear_fc1.weight0 module.decoder.layers.12.mlp.experts.linear_fc2.weight7 Params for bucket 20 (40370176 elements, 40370176 padded size): - module.decoder.layers.12.mlp.experts.linear_fc2.weight2 module.decoder.layers.12.mlp.experts.linear_fc2.weight1 module.decoder.layers.12.mlp.experts.linear_fc2.weight4 - module.decoder.layers.12.mlp.experts.linear_fc1.weight7 - module.decoder.layers.12.mlp.experts.linear_fc1.weight5 - module.decoder.layers.12.mlp.experts.linear_fc2.weight5 - module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 module.decoder.layers.12.mlp.experts.linear_fc2.weight0 module.decoder.layers.12.mlp.experts.linear_fc1.weight6 module.decoder.layers.12.mlp.experts.linear_fc1.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 Params for bucket 21 (40370176 elements, 40370176 padded size): - module.decoder.layers.12.mlp.experts.linear_fc1.weight1 - module.decoder.layers.12.mlp.experts.linear_fc1.weight2 - module.decoder.layers.11.mlp.experts.linear_fc2.weight6 - module.decoder.layers.11.mlp.experts.linear_fc2.weight4 - module.decoder.layers.11.mlp.experts.linear_fc2.weight2 module.decoder.layers.12.mlp.experts.linear_fc1.weight3 module.decoder.layers.12.mlp.experts.linear_fc1.weight0 module.decoder.layers.11.mlp.experts.linear_fc2.weight7 module.decoder.layers.11.mlp.experts.linear_fc2.weight5 module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 Params for bucket 22 (40370176 elements, 40370176 padded size): - module.decoder.layers.11.mlp.experts.linear_fc2.weight1 - module.decoder.layers.11.mlp.experts.linear_fc1.weight5 module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 module.decoder.layers.11.mlp.experts.linear_fc1.weight6 module.decoder.layers.11.mlp.experts.linear_fc2.weight0 module.decoder.layers.11.mlp.experts.linear_fc1.weight4 module.decoder.layers.11.mlp.experts.linear_fc1.weight3 Params for bucket 23 (40370176 elements, 40370176 padded size): module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 module.decoder.layers.10.mlp.experts.linear_fc2.weight6 module.decoder.layers.10.mlp.experts.linear_fc2.weight4 module.decoder.layers.10.mlp.experts.linear_fc2.weight2 module.decoder.layers.10.mlp.experts.linear_fc2.weight1 - module.decoder.layers.10.mlp.experts.linear_fc2.weight0 - module.decoder.layers.10.mlp.experts.linear_fc2.weight5 - module.decoder.layers.10.mlp.experts.linear_fc2.weight7 - module.decoder.layers.11.mlp.experts.linear_fc1.weight0 - module.decoder.layers.10.mlp.experts.linear_fc2.weight3 module.decoder.layers.10.mlp.experts.linear_fc1.weight7 Params for bucket 24 (40370176 elements, 40370176 padded size): module.decoder.layers.10.mlp.experts.linear_fc1.weight6 module.decoder.layers.10.mlp.experts.linear_fc1.weight3 module.decoder.layers.10.mlp.experts.linear_fc1.weight4 module.decoder.layers.10.mlp.experts.linear_fc1.weight0 - module.decoder.layers.10.mlp.experts.linear_fc1.weight5 module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 module.decoder.layers.10.mlp.experts.linear_fc1.weight1 Params for bucket 25 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 module.decoder.layers.9.mlp.experts.linear_fc2.weight6 module.decoder.layers.9.mlp.experts.linear_fc2.weight4 module.decoder.layers.9.mlp.experts.linear_fc2.weight2 module.decoder.layers.9.mlp.experts.linear_fc2.weight1 - module.decoder.layers.9.mlp.experts.linear_fc1.weight7 - module.decoder.layers.9.mlp.experts.linear_fc1.weight5 - module.decoder.layers.9.mlp.experts.linear_fc2.weight7 - module.decoder.layers.9.mlp.experts.linear_fc2.weight3 - module.decoder.layers.9.mlp.experts.linear_fc2.weight5 module.decoder.layers.9.mlp.experts.linear_fc2.weight0 module.decoder.layers.9.mlp.experts.linear_fc1.weight6 Params for bucket 26 (40370176 elements, 40370176 padded size): - module.decoder.layers.9.mlp.experts.linear_fc1.weight1 module.decoder.layers.9.mlp.experts.linear_fc1.weight2 - module.decoder.layers.8.mlp.experts.linear_fc2.weight6 - module.decoder.layers.8.mlp.experts.linear_fc2.weight4 - module.decoder.layers.9.mlp.experts.linear_fc1.weight4 - module.decoder.layers.9.mlp.experts.linear_fc1.weight0 - module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 module.decoder.layers.8.mlp.experts.linear_fc2.weight7 module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 Params for bucket 27 (40370176 elements, 40370176 padded size): - module.decoder.layers.8.mlp.experts.linear_fc2.weight1 - module.decoder.layers.8.mlp.experts.linear_fc2.weight2 - module.decoder.layers.8.mlp.experts.linear_fc1.weight7 - module.decoder.layers.8.mlp.experts.linear_fc1.weight5 module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 module.decoder.layers.8.mlp.experts.linear_fc2.weight0 module.decoder.layers.8.mlp.experts.linear_fc1.weight6 module.decoder.layers.8.mlp.experts.linear_fc1.weight4 module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 Params for bucket 28 (40370176 elements, 40370176 padded size): - module.decoder.layers.8.mlp.experts.linear_fc1.weight2 - module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 module.decoder.layers.7.mlp.experts.linear_fc2.weight7 module.decoder.layers.7.mlp.experts.linear_fc2.weight5 module.decoder.layers.7.mlp.experts.linear_fc2.weight3 module.decoder.layers.7.mlp.experts.linear_fc2.weight2 module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 module.decoder.layers.7.mlp.experts.linear_fc2.weight6 - module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 module.decoder.layers.7.mlp.experts.linear_fc2.weight4 module.decoder.layers.7.mlp.experts.linear_fc2.weight1 Params for bucket 29 (40370176 elements, 40370176 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight4 - module.decoder.layers.7.mlp.experts.linear_fc1.weight3 module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 module.decoder.layers.7.mlp.experts.linear_fc1.weight7 module.decoder.layers.7.mlp.experts.linear_fc1.weight5 module.decoder.layers.7.mlp.experts.linear_fc1.weight2 @@ -1237,8 +1237,8 @@ Params for bucket 30 (40370176 elements, 40370176 padded size): module.decoder.layers.6.mlp.experts.linear_fc2.weight1 module.decoder.layers.6.mlp.experts.linear_fc1.weight7 Params for bucket 31 (40370176 elements, 40370176 padded size): - module.decoder.layers.6.mlp.experts.linear_fc1.weight3 module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 module.decoder.layers.6.mlp.experts.linear_fc1.weight0 module.decoder.layers.5.mlp.experts.linear_fc2.weight7 module.decoder.layers.6.mlp.experts.linear_fc1.weight5 @@ -1252,50 +1252,50 @@ Params for bucket 32 (40370176 elements, 40370176 padded size): module.decoder.layers.5.mlp.experts.linear_fc2.weight0 module.decoder.layers.5.mlp.experts.linear_fc1.weight6 module.decoder.layers.5.mlp.experts.linear_fc1.weight4 - module.decoder.layers.5.mlp.experts.linear_fc2.weight4 module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 module.decoder.layers.5.mlp.experts.linear_fc1.weight7 module.decoder.layers.5.mlp.experts.linear_fc1.weight5 Params for bucket 33 (40370176 elements, 40370176 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight3 - module.decoder.layers.4.mlp.experts.linear_fc2.weight7 module.decoder.layers.5.mlp.experts.linear_fc1.weight0 - module.decoder.layers.4.mlp.experts.linear_fc2.weight5 - module.decoder.layers.4.mlp.experts.linear_fc2.weight3 - module.decoder.layers.5.mlp.experts.linear_fc1.weight1 - module.decoder.layers.5.mlp.experts.linear_fc1.weight2 module.decoder.layers.4.mlp.experts.linear_fc2.weight6 module.decoder.layers.4.mlp.experts.linear_fc2.weight4 module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 Params for bucket 34 (40370176 elements, 40370176 padded size): - module.decoder.layers.4.mlp.experts.linear_fc2.weight0 - module.decoder.layers.4.mlp.experts.linear_fc1.weight7 - module.decoder.layers.4.mlp.experts.linear_fc1.weight5 - module.decoder.layers.4.mlp.experts.linear_fc1.weight2 module.decoder.layers.4.mlp.experts.linear_fc2.weight1 module.decoder.layers.4.mlp.experts.linear_fc1.weight6 module.decoder.layers.4.mlp.experts.linear_fc1.weight4 module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 Params for bucket 35 (40370176 elements, 40370176 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 module.decoder.layers.4.mlp.experts.linear_fc1.weight1 module.decoder.layers.3.mlp.experts.linear_fc2.weight6 module.decoder.layers.3.mlp.experts.linear_fc2.weight4 module.decoder.layers.3.mlp.experts.linear_fc2.weight2 module.decoder.layers.3.mlp.experts.linear_fc2.weight1 - module.decoder.layers.3.mlp.experts.linear_fc2.weight5 - module.decoder.layers.4.mlp.experts.linear_fc1.weight0 - module.decoder.layers.3.mlp.experts.linear_fc2.weight7 - module.decoder.layers.3.mlp.experts.linear_fc2.weight3 - module.decoder.layers.3.mlp.experts.linear_fc2.weight0 - module.decoder.layers.3.mlp.experts.linear_fc1.weight7 Params for bucket 36 (40370176 elements, 40370176 padded size): - module.decoder.layers.3.mlp.experts.linear_fc1.weight6 - module.decoder.layers.3.mlp.experts.linear_fc1.weight4 - module.decoder.layers.3.mlp.experts.linear_fc1.weight3 - module.decoder.layers.3.mlp.experts.linear_fc1.weight0 module.decoder.layers.3.mlp.experts.linear_fc1.weight2 module.decoder.layers.3.mlp.experts.linear_fc1.weight5 module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 Params for bucket 37 (40370176 elements, 40370176 padded size): module.decoder.layers.2.mlp.experts.linear_fc2.weight4 module.decoder.layers.2.mlp.experts.linear_fc2.weight6 @@ -1310,31 +1310,31 @@ Params for bucket 37 (40370176 elements, 40370176 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight5 Params for bucket 38 (40370176 elements, 40370176 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight4 - module.decoder.layers.2.mlp.experts.linear_fc1.weight0 module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 module.decoder.layers.1.mlp.experts.linear_fc2.weight7 module.decoder.layers.1.mlp.experts.linear_fc2.weight5 - module.decoder.layers.2.mlp.experts.linear_fc1.weight2 module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 module.decoder.layers.1.mlp.experts.linear_fc2.weight6 module.decoder.layers.1.mlp.experts.linear_fc2.weight4 Params for bucket 39 (40370176 elements, 40370176 padded size): module.decoder.layers.1.mlp.experts.linear_fc2.weight3 module.decoder.layers.1.mlp.experts.linear_fc2.weight0 - module.decoder.layers.1.mlp.experts.linear_fc1.weight7 - module.decoder.layers.1.mlp.experts.linear_fc1.weight5 - module.decoder.layers.1.mlp.experts.linear_fc2.weight1 module.decoder.layers.1.mlp.experts.linear_fc2.weight2 module.decoder.layers.1.mlp.experts.linear_fc1.weight6 module.decoder.layers.1.mlp.experts.linear_fc1.weight4 module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 Params for bucket 40 (17301504 elements, 17301504 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight2 module.decoder.layers.1.mlp.experts.linear_fc1.weight1 module.decoder.layers.1.mlp.experts.linear_fc1.weight0 -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine -[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 15:09:56 +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 15:57:45 > building train, validation, and test datasets ... > datasets target sizes (minimum size): train: 96 @@ -1342,7 +1342,7 @@ INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine test: 0 INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] > building train, validation, and test datasets for GPT ... -INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers @@ -1355,14 +1355,14 @@ INFO:megatron.core.datasets.gpt_dataset: Load the sample index from eadc3756447e INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-shuffle_index.npy INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 > finished creating GPT datasets ... -[after dataloaders are built] datetime: 2026-08-26 15:09:56 +[after dataloaders are built] datetime: 2026-08-26 15:57:45 done with setup ... (min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (3721.38, 3736.01) - train/valid/test-data-iterators-setup ..........: (59.98, 67.78) -training ... + model-and-optimizer-setup ......................: (3848.59, 3859.22) + train/valid/test-data-iterators-setup ..........: (63.18, 74.62)training ... + Overwriting rerun_state_machine.current_iteration from -1 to 0... -[before the start of training step] datetime: 2026-08-26 15:09:56 +[before the start of training step] datetime: 2026-08-26 15:57:45 INFO:moonep.api:MoonEP comm buffer sizes: config : R=8, S=8192, H=2048, E=64, K=6, NvS=51184, NvS_padded=51200 sm_count : num_sms=32, num_sms_dedup=148 @@ -1371,5635 +1371,5635 @@ INFO:moonep.api:MoonEP comm buffer sizes: local_temp_buffer_size : 2.01 MiB total_size : 204.01 MiB WARNING:megatron.core.rerun_state_machine:Result validation enabled - [2026-08-26 15:10:19] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 22758.9 | throughput per GPU (TFLOP/s/GPU): 5.6 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.092128E+01 | loss scale: 1.0 | grad norm: 30.251 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 15:58:08] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 22484.5 | throughput per GPU (TFLOP/s/GPU): 5.7 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.092128E+01 | loss scale: 1.0 | grad norm: 30.251 | number of skipped iterations: 0 | number of nan iterations: 0 | Number of parameters in transformer block in billions: 13.53 Number of parameters in embedding layers in billions: 0.21 Total number of parameters in billions: 13.74 Number of parameters in most loaded shard in billions: 13.7356 Theoretical memory footprints: weight and optimizer=98244.96 MB [Rank 0] (after 1 iterations) memory (MB) | allocated: 66075.59912109375 | max allocated: 124945.7734375 | reserved: 127502.0 | max reserved: 127502.0 - [2026-08-26 15:10:20] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 801.3 | throughput per GPU (TFLOP/s/GPU): 159.2 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.091997E+01 | loss scale: 1.0 | grad norm: 30.182 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74468 payload_rows_remote=65098 dup_rows=23836 dup_rows_remote=20824 dedup_ratio=0.242472 router_tokens_per_expert=[751, 759, 810, 729, 763, 836, 747, 766, 779, 758, 819, 816, 754, 754, 771, 790, 793, 714, 765, 757, 764, 781, 799, 763, 786, 723, 752, 747, 757, 762, 744, 781, 752, 757, 764, 757, 776, 798, 751, 770, 807, 744, 794, 792, 745, 774, 758, 777, 795, 772, 703, 788, 775, 772, 739, 751, 812, 790, 764, 782, 762, 748, 741, 752] payload_rows_per_dst=[9308, 9454, 9286, 9212, 9246, 9370, 9270, 9322] dup_rows_per_dst=[3014, 3028, 2986, 2892, 3004, 3012, 2920, 2980] combine_rows_per_src=[9308, 9454, 9286, 9212, 9246, 9370, 9270, 9322] + [2026-08-26 15:58:09] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 800.2 | throughput per GPU (TFLOP/s/GPU): 159.4 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.091997E+01 | loss scale: 1.0 | grad norm: 30.182 | number of skipped iterations: 0 | number of nan iterations: 0 | WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74138 payload_rows_remote=64922 dup_rows=24166 dup_rows_remote=21086 dedup_ratio=0.245829 router_tokens_per_expert=[766, 792, 779, 806, 822, 754, 776, 737, 781, 750, 827, 778, 783, 749, 714, 766, 775, 789, 756, 768, 773, 743, 805, 802, 753, 782, 790, 778, 767, 773, 821, 750, 783, 748, 682, 764, 709, 821, 770, 772, 795, 709, 729, 780, 767, 766, 773, 744, 731, 793, 770, 734, 784, 744, 814, 780, 801, 731, 776, 743, 790, 761, 744, 739] payload_rows_per_dst=[9328, 9216, 9376, 9358, 9218, 9196, 9266, 9180] dup_rows_per_dst=[3136, 3080, 3046, 3070, 2880, 2930, 3034, 2990] combine_rows_per_src=[9328, 9216, 9376, 9358, 9218, 9196, 9266, 9180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74468 payload_rows_remote=65098 dup_rows=23836 dup_rows_remote=20824 dedup_ratio=0.242472 router_tokens_per_expert=[751, 759, 810, 729, 763, 836, 747, 766, 779, 758, 819, 816, 754, 754, 771, 790, 793, 714, 765, 757, 764, 781, 799, 763, 786, 723, 752, 747, 757, 762, 744, 781, 752, 757, 764, 757, 776, 798, 751, 770, 807, 744, 794, 792, 745, 774, 758, 777, 795, 772, 703, 788, 775, 772, 739, 751, 812, 790, 764, 782, 762, 748, 741, 752] payload_rows_per_dst=[9308, 9454, 9286, 9212, 9246, 9370, 9270, 9322] dup_rows_per_dst=[3014, 3028, 2986, 2892, 3004, 3012, 2920, 2980] combine_rows_per_src=[9308, 9454, 9286, 9212, 9246, 9370, 9270, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=64978 dup_rows=23860 dup_rows_remote=20740 dedup_ratio=0.242716 router_tokens_per_expert=[763, 802, 813, 769, 785, 774, 791, 783, 798, 769, 769, 785, 726, 766, 783, 788, 799, 774, 744, 785, 792, 785, 770, 763, 758, 721, 835, 750, 783, 810, 742, 779, 803, 725, 774, 723, 788, 729, 758, 720, 735, 815, 751, 781, 755, 719, 789, 771, 804, 744, 723, 757, 761, 809, 716, 763, 757, 770, 722, 727, 769, 772, 817, 751] payload_rows_per_dst=[9466, 9294, 9376, 9216, 9254, 9340, 9288, 9210] dup_rows_per_dst=[3120, 3074, 3048, 3114, 2786, 2892, 2866, 2960] combine_rows_per_src=[9466, 9294, 9376, 9216, 9254, 9340, 9288, 9210] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74240 payload_rows_remote=64792 dup_rows=24064 dup_rows_remote=20924 dedup_ratio=0.244792 router_tokens_per_expert=[722, 789, 748, 730, 800, 759, 765, 760, 791, 795, 762, 768, 798, 794, 796, 740, 766, 815, 801, 764, 785, 778, 756, 807, 782, 766, 760, 741, 791, 775, 765, 765, 767, 768, 682, 748, 714, 764, 721, 735, 745, 851, 779, 738, 759, 767, 737, 742, 733, 808, 753, 811, 756, 758, 737, 825, 732, 782, 806, 842, 767, 770, 771, 750] payload_rows_per_dst=[8862, 8974, 9122, 9286, 9486, 9324, 9738, 9448] dup_rows_per_dst=[2642, 2754, 2798, 2944, 3296, 2972, 3518, 3140] combine_rows_per_src=[8862, 8974, 9122, 9286, 9486, 9324, 9738, 9448] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74176 payload_rows_remote=64892 dup_rows=24128 dup_rows_remote=21062 dedup_ratio=0.245443 router_tokens_per_expert=[782, 754, 773, 735, 755, 868, 802, 770, 801, 774, 745, 748, 721, 751, 730, 809, 776, 726, 786, 784, 812, 744, 798, 749, 779, 751, 787, 793, 787, 760, 770, 786, 748, 800, 812, 713, 742, 732, 777, 731, 805, 790, 774, 797, 776, 755, 761, 731, 748, 778, 758, 758, 778, 758, 794, 706, 725, 786, 753, 801, 767, 764, 755, 773] payload_rows_per_dst=[9392, 9132, 9284, 9354, 9152, 9266, 9254, 9342] dup_rows_per_dst=[3086, 3026, 3066, 3072, 2958, 3112, 2902, 2906] combine_rows_per_src=[9392, 9132, 9284, 9354, 9152, 9266, 9254, 9342] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74196 payload_rows_remote=64900 dup_rows=24108 dup_rows_remote=21060 dedup_ratio=0.245239 router_tokens_per_expert=[761, 800, 809, 777, 773, 790, 787, 758, 753, 737, 800, 764, 746, 756, 838, 765, 779, 721, 725, 775, 795, 749, 776, 792, 786, 729, 738, 762, 801, 801, 777, 778, 832, 744, 777, 820, 752, 750, 780, 721, 763, 745, 746, 783, 819, 750, 745, 763, 730, 731, 795, 780, 763, 782, 776, 746, 771, 753, 723, 743, 758, 778, 789, 746] payload_rows_per_dst=[9400, 9260, 9250, 9296, 9304, 9230, 9276, 9180] dup_rows_per_dst=[3110, 3058, 2974, 3048, 3048, 2998, 2930, 2942] combine_rows_per_src=[9400, 9260, 9250, 9296, 9304, 9230, 9276, 9180] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=64908 dup_rows=24048 dup_rows_remote=21016 dedup_ratio=0.244629 router_tokens_per_expert=[745, 717, 776, 802, 751, 792, 740, 758, 800, 804, 754, 791, 764, 792, 800, 766, 751, 746, 807, 712, 804, 738, 780, 781, 752, 778, 806, 740, 760, 782, 730, 736, 740, 741, 757, 808, 798, 780, 786, 780, 741, 819, 782, 725, 767, 751, 779, 776, 743, 766, 740, 724, 787, 790, 779, 740, 737, 815, 775, 784, 783, 751, 781, 772] payload_rows_per_dst=[9142, 9456, 9284, 9196, 9348, 9302, 9198, 9330] dup_rows_per_dst=[3020, 3086, 2954, 2972, 3032, 2978, 2940, 3066] combine_rows_per_src=[9142, 9456, 9284, 9196, 9348, 9302, 9198, 9330] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265920512 dispatch_meta_bytes=344032 combine_payload_bytes=265920512 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266264544 ingress_bytes=335126528 link_bytes=335126528 token_bytes=532185056 weight_movement_bytes=69206016 total_nvlink_bytes=601391072 weight_to_token_ratio=0.130041 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261054145e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=343688 combine_payload_bytes=266641408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266985096 ingress_bytes=301244416 link_bytes=301244416 token_bytes=533626504 weight_movement_bytes=34603008 total_nvlink_bytes=568229512 weight_to_token_ratio=0.064845 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133558485e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265920512 dispatch_meta_bytes=344032 combine_payload_bytes=265920512 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266264544 ingress_bytes=335126528 link_bytes=335126528 token_bytes=532185056 weight_movement_bytes=69206016 total_nvlink_bytes=601391072 weight_to_token_ratio=0.130041 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261054145e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=342872 combine_payload_bytes=266149888 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266492760 ingress_bytes=404561920 link_bytes=404561920 token_bytes=532642648 weight_movement_bytes=138412032 total_nvlink_bytes=671054680 weight_to_token_ratio=0.259859 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522333935e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74176 payload_rows_remote=64892 dup_rows=24128 dup_rows_remote=21062 dedup_ratio=0.245443 router_tokens_per_expert=[782, 754, 773, 735, 755, 868, 802, 770, 801, 774, 745, 748, 721, 751, 730, 809, 776, 726, 786, 784, 812, 744, 798, 749, 779, 751, 787, 793, 787, 760, 770, 786, 748, 800, 812, 713, 742, 732, 777, 731, 805, 790, 774, 797, 776, 755, 761, 731, 748, 778, 758, 758, 778, 758, 794, 706, 725, 786, 753, 801, 767, 764, 755, 773] payload_rows_per_dst=[9392, 9132, 9284, 9354, 9152, 9266, 9254, 9342] dup_rows_per_dst=[3086, 3026, 3066, 3072, 2958, 3112, 2902, 2906] combine_rows_per_src=[9392, 9132, 9284, 9354, 9152, 9266, 9254, 9342] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265388032 dispatch_meta_bytes=342864 combine_payload_bytes=265388032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265730896 ingress_bytes=299991040 link_bytes=299991040 token_bytes=531118928 weight_movement_bytes=34603008 total_nvlink_bytes=565721936 weight_to_token_ratio=0.065151 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128842132e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265797632 dispatch_meta_bytes=343816 combine_payload_bytes=265797632 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266141448 ingress_bytes=335003648 link_bytes=335003648 token_bytes=531939080 weight_movement_bytes=69206016 total_nvlink_bytes=601145096 weight_to_token_ratio=0.130101 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260591757e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265830400 dispatch_meta_bytes=343840 combine_payload_bytes=265830400 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266174240 ingress_bytes=335036416 link_bytes=335036416 token_bytes=532004640 weight_movement_bytes=69206016 total_nvlink_bytes=601210656 weight_to_token_ratio=0.130085 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260715061e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265863168 dispatch_meta_bytes=343696 combine_payload_bytes=265863168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266206864 ingress_bytes=300466176 link_bytes=300466176 token_bytes=532070032 weight_movement_bytes=34603008 total_nvlink_bytes=566673040 weight_to_token_ratio=0.065035 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130630031e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=64978 dup_rows=23860 dup_rows_remote=20740 dedup_ratio=0.242716 router_tokens_per_expert=[763, 802, 813, 769, 785, 774, 791, 783, 798, 769, 769, 785, 726, 766, 783, 788, 799, 774, 744, 785, 792, 785, 770, 763, 758, 721, 835, 750, 783, 810, 742, 779, 803, 725, 774, 723, 788, 729, 758, 720, 735, 815, 751, 781, 755, 719, 789, 771, 804, 744, 723, 757, 761, 809, 716, 763, 757, 770, 722, 727, 769, 772, 817, 751] payload_rows_per_dst=[9466, 9294, 9376, 9216, 9254, 9340, 9288, 9210] dup_rows_per_dst=[3120, 3074, 3048, 3114, 2786, 2892, 2866, 2960] combine_rows_per_src=[9466, 9294, 9376, 9216, 9254, 9340, 9288, 9210] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74062 payload_rows_remote=64974 dup_rows=24242 dup_rows_remote=21426 dedup_ratio=0.246602 router_tokens_per_expert=[769, 777, 782, 788, 743, 768, 751, 761, 715, 764, 817, 793, 769, 756, 784, 808, 811, 768, 774, 820, 762, 756, 783, 753, 749, 801, 806, 779, 733, 748, 778, 743, 750, 760, 749, 724, 750, 827, 804, 782, 788, 782, 776, 774, 789, 768, 760, 754, 710, 774, 771, 725, 728, 762, 768, 714, 729, 762, 776, 793, 746, 750, 781, 817] payload_rows_per_dst=[9276, 9382, 9406, 9206, 9252, 9216, 9088, 9236] dup_rows_per_dst=[3002, 3030, 3048, 3068, 3040, 3166, 2816, 3072] combine_rows_per_src=[9276, 9382, 9406, 9206, 9252, 9216, 9088, 9236] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74446 payload_rows_remote=65270 dup_rows=23858 dup_rows_remote=20816 dedup_ratio=0.242696 router_tokens_per_expert=[768, 801, 754, 873, 739, 743, 788, 768, 760, 727, 832, 733, 700, 798, 743, 772, 822, 711, 821, 764, 773, 759, 751, 754, 780, 769, 748, 794, 758, 760, 765, 734, 806, 775, 793, 740, 797, 765, 802, 805, 775, 718, 759, 794, 746, 784, 755, 778, 733, 770, 730, 774, 754, 760, 769, 755, 767, 766, 782, 798, 744, 756, 787, 753] payload_rows_per_dst=[9436, 9174, 9312, 9352, 9502, 9176, 9186, 9308] dup_rows_per_dst=[3032, 2956, 2998, 2864, 3064, 3042, 2904, 2998] combine_rows_per_src=[9436, 9174, 9312, 9352, 9502, 9176, 9186, 9308] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265797632 dispatch_meta_bytes=343816 combine_payload_bytes=265797632 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266141448 ingress_bytes=335003648 link_bytes=335003648 token_bytes=531939080 weight_movement_bytes=69206016 total_nvlink_bytes=601145096 weight_to_token_ratio=0.130101 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260591757e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74280 payload_rows_remote=65124 dup_rows=24024 dup_rows_remote=20920 dedup_ratio=0.244385 router_tokens_per_expert=[714, 774, 762, 750, 810, 818, 770, 774, 773, 777, 744, 751, 755, 752, 784, 794, 824, 790, 718, 739, 776, 784, 779, 732, 743, 753, 774, 763, 823, 788, 783, 816, 759, 775, 791, 708, 751, 785, 770, 735, 802, 766, 751, 765, 780, 814, 770, 800, 775, 793, 749, 742, 729, 749, 817, 767, 758, 732, 755, 710, 743, 734, 808, 782] payload_rows_per_dst=[9316, 9156, 9310, 9436, 9230, 9402, 9200, 9230] dup_rows_per_dst=[3028, 3104, 2974, 3050, 2918, 3094, 3042, 2814] combine_rows_per_src=[9316, 9156, 9310, 9436, 9230, 9402, 9200, 9230] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=342872 combine_payload_bytes=266149888 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266492760 ingress_bytes=404561920 link_bytes=404561920 token_bytes=532642648 weight_movement_bytes=138412032 total_nvlink_bytes=671054680 weight_to_token_ratio=0.259859 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522333935e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=345600 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266479104 ingress_bytes=300736512 link_bytes=300736512 token_bytes=532612608 weight_movement_bytes=34603008 total_nvlink_bytes=567215616 weight_to_token_ratio=0.064968 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131647283e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74446 payload_rows_remote=65270 dup_rows=23858 dup_rows_remote=20816 dedup_ratio=0.242696 router_tokens_per_expert=[768, 801, 754, 873, 739, 743, 788, 768, 760, 727, 832, 733, 700, 798, 743, 772, 822, 711, 821, 764, 773, 759, 751, 754, 780, 769, 748, 794, 758, 760, 765, 734, 806, 775, 793, 740, 797, 765, 802, 805, 775, 718, 759, 794, 746, 784, 755, 778, 733, 770, 730, 774, 754, 760, 769, 755, 767, 766, 782, 798, 744, 756, 787, 753] payload_rows_per_dst=[9436, 9174, 9312, 9352, 9502, 9176, 9186, 9308] dup_rows_per_dst=[3032, 2956, 2998, 2864, 3064, 3042, 2904, 2998] combine_rows_per_src=[9436, 9174, 9312, 9352, 9502, 9176, 9186, 9308] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74448 payload_rows_remote=65200 dup_rows=23856 dup_rows_remote=20932 dedup_ratio=0.242676 router_tokens_per_expert=[790, 795, 750, 738, 750, 748, 758, 744, 750, 767, 795, 791, 757, 761, 779, 800, 773, 761, 785, 744, 763, 728, 777, 779, 759, 762, 769, 805, 797, 781, 770, 778, 800, 795, 786, 784, 742, 821, 737, 722, 733, 800, 817, 787, 773, 721, 695, 778, 766, 710, 775, 758, 749, 787, 775, 780, 769, 731, 770, 772, 786, 770, 775, 784] payload_rows_per_dst=[9248, 9682, 9472, 9384, 8968, 9136, 9274, 9284] dup_rows_per_dst=[2924, 3296, 3008, 3032, 2828, 2812, 2926, 3030] combine_rows_per_src=[9248, 9682, 9472, 9384, 8968, 9136, 9274, 9284] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=64908 dup_rows=24048 dup_rows_remote=21016 dedup_ratio=0.244629 router_tokens_per_expert=[745, 717, 776, 802, 751, 792, 740, 758, 800, 804, 754, 791, 764, 792, 800, 766, 751, 746, 807, 712, 804, 738, 780, 781, 752, 778, 806, 740, 760, 782, 730, 736, 740, 741, 757, 808, 798, 780, 786, 780, 741, 819, 782, 725, 767, 751, 779, 776, 743, 766, 740, 724, 787, 790, 779, 740, 737, 815, 775, 784, 783, 751, 781, 772] payload_rows_per_dst=[9142, 9456, 9284, 9196, 9348, 9302, 9198, 9330] dup_rows_per_dst=[3020, 3086, 2954, 2972, 3032, 2978, 2940, 3066] combine_rows_per_src=[9142, 9456, 9284, 9196, 9348, 9302, 9198, 9330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266747904 dispatch_meta_bytes=344176 combine_payload_bytes=266747904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267092080 ingress_bytes=301350912 link_bytes=301350912 token_bytes=533839984 weight_movement_bytes=34603008 total_nvlink_bytes=568442992 weight_to_token_ratio=0.064819 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133959221e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=65034 dup_rows=23906 dup_rows_remote=20782 dedup_ratio=0.243184 router_tokens_per_expert=[749, 829, 788, 739, 749, 734, 775, 806, 784, 766, 757, 750, 697, 777, 750, 756, 794, 746, 744, 815, 827, 742, 726, 775, 789, 715, 750, 749, 773, 729, 845, 732, 800, 770, 793, 776, 772, 780, 760, 778, 712, 754, 798, 796, 780, 779, 778, 720, 788, 714, 776, 795, 772, 752, 852, 796, 775, 783, 733, 712, 809, 776, 765, 751] payload_rows_per_dst=[9114, 9228, 9354, 9184, 9396, 9246, 9512, 9364] dup_rows_per_dst=[2808, 2846, 2984, 2932, 3110, 2864, 3238, 3124] combine_rows_per_src=[9114, 9228, 9354, 9184, 9396, 9246, 9512, 9364] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74426 payload_rows_remote=65084 dup_rows=23878 dup_rows_remote=20968 dedup_ratio=0.242900 router_tokens_per_expert=[759, 767, 796, 757, 792, 774, 736, 784, 793, 731, 733, 753, 769, 787, 770, 715, 759, 800, 791, 766, 761, 738, 761, 775, 753, 813, 796, 759, 753, 765, 716, 815, 749, 765, 748, 801, 797, 757, 752, 757, 773, 801, 748, 805, 802, 771, 792, 729, 738, 748, 769, 749, 754, 781, 793, 779, 786, 762, 731, 809, 761, 804, 738, 766] payload_rows_per_dst=[9284, 9176, 9276, 9332, 9342, 9420, 9244, 9352] dup_rows_per_dst=[3046, 2926, 3026, 3008, 2910, 3022, 2978, 2962] combine_rows_per_src=[9284, 9176, 9276, 9332, 9342, 9420, 9244, 9352] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74164 payload_rows_remote=65036 dup_rows=24140 dup_rows_remote=21236 dedup_ratio=0.245565 router_tokens_per_expert=[726, 756, 715, 751, 777, 748, 836, 775, 772, 763, 737, 761, 751, 796, 746, 776, 769, 777, 733, 785, 749, 730, 722, 751, 766, 783, 726, 769, 773, 759, 770, 768, 788, 745, 759, 755, 739, 774, 799, 805, 806, 801, 793, 760, 740, 744, 807, 759, 806, 799, 784, 764, 797, 777, 810, 753, 789, 745, 763, 766, 813, 791, 761, 744] payload_rows_per_dst=[9228, 9270, 9128, 9210, 9302, 9358, 9418, 9250] dup_rows_per_dst=[2940, 2934, 2904, 3018, 3026, 3062, 3162, 3094] combine_rows_per_src=[9228, 9270, 9128, 9210, 9302, 9358, 9418, 9250] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267345920 dispatch_meta_bytes=344344 combine_payload_bytes=267345920 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267690264 ingress_bytes=371154944 link_bytes=371154944 token_bytes=535036184 weight_movement_bytes=103809024 total_nvlink_bytes=638845208 weight_to_token_ratio=0.194022 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396626174e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74106 payload_rows_remote=64952 dup_rows=24198 dup_rows_remote=21242 dedup_ratio=0.246155 router_tokens_per_expert=[786, 754, 783, 814, 779, 805, 763, 821, 781, 765, 781, 788, 767, 760, 779, 764, 794, 741, 735, 733, 786, 755, 771, 786, 756, 757, 740, 767, 774, 743, 753, 765, 775, 734, 818, 721, 764, 805, 759, 766, 749, 754, 788, 759, 740, 767, 747, 844, 750, 756, 771, 783, 729, 724, 787, 774, 789, 776, 723, 727, 810, 761, 800, 756] payload_rows_per_dst=[9492, 9250, 9214, 9154, 9294, 9270, 9186, 9246] dup_rows_per_dst=[3118, 3120, 2988, 2956, 2990, 3026, 2962, 3038] combine_rows_per_src=[9492, 9250, 9214, 9154, 9294, 9270, 9186, 9246] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266747904 dispatch_meta_bytes=344176 combine_payload_bytes=266747904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267092080 ingress_bytes=301350912 link_bytes=301350912 token_bytes=533839984 weight_movement_bytes=34603008 total_nvlink_bytes=568442992 weight_to_token_ratio=0.064819 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133959221e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267059200 dispatch_meta_bytes=344528 combine_payload_bytes=267059200 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267403728 ingress_bytes=370868224 link_bytes=370868224 token_bytes=534462928 weight_movement_bytes=103809024 total_nvlink_bytes=638271952 weight_to_token_ratio=0.194231 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395547270e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=345600 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266479104 ingress_bytes=300736512 link_bytes=300736512 token_bytes=532612608 weight_movement_bytes=34603008 total_nvlink_bytes=567215616 weight_to_token_ratio=0.064968 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131647283e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267345920 dispatch_meta_bytes=344344 combine_payload_bytes=267345920 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267690264 ingress_bytes=371154944 link_bytes=371154944 token_bytes=535036184 weight_movement_bytes=103809024 total_nvlink_bytes=638845208 weight_to_token_ratio=0.194022 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396626174e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74164 payload_rows_remote=65036 dup_rows=24140 dup_rows_remote=21236 dedup_ratio=0.245565 router_tokens_per_expert=[726, 756, 715, 751, 777, 748, 836, 775, 772, 763, 737, 761, 751, 796, 746, 776, 769, 777, 733, 785, 749, 730, 722, 751, 766, 783, 726, 769, 773, 759, 770, 768, 788, 745, 759, 755, 739, 774, 799, 805, 806, 801, 793, 760, 740, 744, 807, 759, 806, 799, 784, 764, 797, 777, 810, 753, 789, 745, 763, 766, 813, 791, 761, 744] payload_rows_per_dst=[9228, 9270, 9128, 9210, 9302, 9358, 9418, 9250] dup_rows_per_dst=[2940, 2934, 2904, 3018, 3026, 3062, 3162, 3094] combine_rows_per_src=[9228, 9270, 9128, 9210, 9302, 9358, 9418, 9250] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266379264 dispatch_meta_bytes=343264 combine_payload_bytes=266379264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266722528 ingress_bytes=300982272 link_bytes=300982272 token_bytes=533101792 weight_movement_bytes=34603008 total_nvlink_bytes=567704800 weight_to_token_ratio=0.064909 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132572058e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266584064 dispatch_meta_bytes=344208 combine_payload_bytes=266584064 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266928272 ingress_bytes=335790080 link_bytes=335790080 token_bytes=533512336 weight_movement_bytes=69206016 total_nvlink_bytes=602718352 weight_to_token_ratio=0.129718 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263551038e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266387456 dispatch_meta_bytes=345088 combine_payload_bytes=266387456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266732544 ingress_bytes=300990464 link_bytes=300990464 token_bytes=533120000 weight_movement_bytes=34603008 total_nvlink_bytes=567723008 weight_to_token_ratio=0.064907 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132602884e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266043392 dispatch_meta_bytes=344776 combine_payload_bytes=266043392 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266388168 ingress_bytes=335249408 link_bytes=335249408 token_bytes=532431560 weight_movement_bytes=69206016 total_nvlink_bytes=601637576 weight_to_token_ratio=0.129981 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261516533e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74448 payload_rows_remote=65200 dup_rows=23856 dup_rows_remote=20932 dedup_ratio=0.242676 router_tokens_per_expert=[790, 795, 750, 738, 750, 748, 758, 744, 750, 767, 795, 791, 757, 761, 779, 800, 773, 761, 785, 744, 763, 728, 777, 779, 759, 762, 769, 805, 797, 781, 770, 778, 800, 795, 786, 784, 742, 821, 737, 722, 733, 800, 817, 787, 773, 721, 695, 778, 766, 710, 775, 758, 749, 787, 775, 780, 769, 731, 770, 772, 786, 770, 775, 784] payload_rows_per_dst=[9248, 9682, 9472, 9384, 8968, 9136, 9274, 9284] dup_rows_per_dst=[2924, 3296, 3008, 3032, 2828, 2812, 2926, 3030] combine_rows_per_src=[9248, 9682, 9472, 9384, 8968, 9136, 9274, 9284] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74158 payload_rows_remote=64888 dup_rows=24146 dup_rows_remote=21136 dedup_ratio=0.245626 router_tokens_per_expert=[734, 750, 769, 775, 724, 809, 748, 747, 736, 803, 773, 754, 780, 743, 746, 752, 806, 805, 792, 750, 767, 780, 784, 758, 776, 763, 733, 781, 787, 766, 730, 754, 787, 780, 759, 786, 785, 756, 744, 778, 756, 733, 791, 808, 764, 758, 744, 786, 749, 769, 807, 772, 779, 737, 789, 806, 791, 765, 739, 797, 780, 794, 719, 769] payload_rows_per_dst=[9212, 9148, 9292, 9212, 9360, 9270, 9360, 9304] dup_rows_per_dst=[2900, 3026, 3192, 2968, 2990, 3010, 3056, 3004] combine_rows_per_src=[9212, 9148, 9292, 9212, 9360, 9270, 9360, 9304] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265863168 dispatch_meta_bytes=343696 combine_payload_bytes=265863168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266206864 ingress_bytes=300466176 link_bytes=300466176 token_bytes=532070032 weight_movement_bytes=34603008 total_nvlink_bytes=566673040 weight_to_token_ratio=0.065035 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130630031e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74150 payload_rows_remote=64956 dup_rows=24154 dup_rows_remote=21096 dedup_ratio=0.245707 router_tokens_per_expert=[820, 810, 763, 763, 796, 757, 753, 772, 726, 760, 803, 751, 762, 759, 765, 800, 713, 768, 756, 753, 730, 718, 741, 778, 805, 779, 786, 763, 758, 748, 786, 759, 774, 768, 749, 796, 756, 745, 760, 824, 846, 771, 739, 770, 767, 791, 763, 720, 783, 776, 760, 773, 759, 768, 771, 765, 761, 749, 795, 794, 757, 752, 788, 761] payload_rows_per_dst=[9364, 9194, 9098, 9362, 9348, 9334, 9196, 9254] dup_rows_per_dst=[3104, 3058, 2816, 3006, 2996, 3000, 3114, 3060] combine_rows_per_src=[9364, 9194, 9098, 9362, 9348, 9334, 9196, 9254] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65238 dup_rows=23924 dup_rows_remote=20994 dedup_ratio=0.243368 router_tokens_per_expert=[759, 790, 739, 777, 787, 727, 789, 777, 754, 746, 741, 779, 759, 783, 812, 719, 787, 770, 758, 736, 784, 767, 764, 812, 788, 767, 758, 747, 779, 793, 778, 786, 741, 747, 761, 810, 817, 805, 719, 812, 796, 737, 775, 788, 755, 801, 791, 744, 764, 731, 764, 774, 739, 766, 727, 771, 783, 732, 769, 765, 779, 740, 774, 763] payload_rows_per_dst=[9274, 9208, 9346, 9436, 9428, 9292, 9142, 9254] dup_rows_per_dst=[3016, 2978, 3010, 2956, 2996, 3082, 2930, 2956] combine_rows_per_src=[9274, 9208, 9346, 9436, 9428, 9292, 9142, 9254] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267059200 dispatch_meta_bytes=344528 combine_payload_bytes=267059200 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267403728 ingress_bytes=370868224 link_bytes=370868224 token_bytes=534462928 weight_movement_bytes=103809024 total_nvlink_bytes=638271952 weight_to_token_ratio=0.194231 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395547270e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=64932 dup_rows=23982 dup_rows_remote=20940 dedup_ratio=0.243958 router_tokens_per_expert=[749, 796, 780, 751, 784, 801, 771, 777, 732, 756, 748, 767, 763, 788, 825, 751, 760, 793, 780, 757, 737, 772, 782, 762, 799, 771, 748, 755, 787, 718, 749, 775, 746, 775, 771, 774, 818, 780, 755, 810, 769, 774, 750, 765, 759, 842, 801, 711, 770, 789, 784, 709, 772, 733, 731, 799, 758, 798, 757, 771, 766, 724, 801, 706] payload_rows_per_dst=[9390, 9282, 9266, 9206, 9334, 9440, 9190, 9214] dup_rows_per_dst=[3042, 2978, 3020, 2998, 3110, 2950, 2984, 2900] combine_rows_per_src=[9390, 9282, 9266, 9206, 9334, 9440, 9190, 9214] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266387456 dispatch_meta_bytes=345088 combine_payload_bytes=266387456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266732544 ingress_bytes=300990464 link_bytes=300990464 token_bytes=533120000 weight_movement_bytes=34603008 total_nvlink_bytes=567723008 weight_to_token_ratio=0.064907 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132602884e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74158 payload_rows_remote=64888 dup_rows=24146 dup_rows_remote=21136 dedup_ratio=0.245626 router_tokens_per_expert=[734, 750, 769, 775, 724, 809, 748, 747, 736, 803, 773, 754, 780, 743, 746, 752, 806, 805, 792, 750, 767, 780, 784, 758, 776, 763, 733, 781, 787, 766, 730, 754, 787, 780, 759, 786, 785, 756, 744, 778, 756, 733, 791, 808, 764, 758, 744, 786, 749, 769, 807, 772, 779, 737, 789, 806, 791, 765, 739, 797, 780, 794, 719, 769] payload_rows_per_dst=[9212, 9148, 9292, 9212, 9360, 9270, 9360, 9304] dup_rows_per_dst=[2900, 3026, 3192, 2968, 2990, 3010, 3056, 3004] combine_rows_per_src=[9212, 9148, 9292, 9212, 9360, 9270, 9360, 9304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=344208 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266403984 ingress_bytes=300662784 link_bytes=300662784 token_bytes=532463760 weight_movement_bytes=34603008 total_nvlink_bytes=567066768 weight_to_token_ratio=0.064987 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131369851e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265961472 dispatch_meta_bytes=343488 combine_payload_bytes=265961472 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266304960 ingress_bytes=369770496 link_bytes=369770496 token_bytes=532266432 weight_movement_bytes=103809024 total_nvlink_bytes=636075456 weight_to_token_ratio=0.195032 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391416607e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65238 dup_rows=23924 dup_rows_remote=20994 dedup_ratio=0.243368 router_tokens_per_expert=[759, 790, 739, 777, 787, 727, 789, 777, 754, 746, 741, 779, 759, 783, 812, 719, 787, 770, 758, 736, 784, 767, 764, 812, 788, 767, 758, 747, 779, 793, 778, 786, 741, 747, 761, 810, 817, 805, 719, 812, 796, 737, 775, 788, 755, 801, 791, 744, 764, 731, 764, 774, 739, 766, 727, 771, 783, 732, 769, 765, 779, 740, 774, 763] payload_rows_per_dst=[9274, 9208, 9346, 9436, 9428, 9292, 9142, 9254] dup_rows_per_dst=[3016, 2978, 3010, 2956, 2996, 3082, 2930, 2956] combine_rows_per_src=[9274, 9208, 9346, 9436, 9428, 9292, 9142, 9254] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74054 payload_rows_remote=64790 dup_rows=24250 dup_rows_remote=21198 dedup_ratio=0.246684 router_tokens_per_expert=[729, 794, 808, 771, 738, 809, 784, 784, 765, 730, 743, 722, 817, 750, 724, 776, 777, 745, 766, 806, 780, 735, 756, 760, 795, 764, 749, 767, 725, 762, 808, 769, 791, 761, 732, 744, 775, 779, 778, 736, 785, 733, 811, 800, 775, 780, 780, 728, 755, 763, 790, 753, 775, 768, 809, 789, 773, 783, 763, 734, 785, 779, 766, 771] payload_rows_per_dst=[9036, 9190, 9446, 9282, 9276, 9192, 9368, 9264] dup_rows_per_dst=[2850, 2958, 3258, 2996, 2908, 2950, 3278, 3052] combine_rows_per_src=[9036, 9190, 9446, 9282, 9276, 9192, 9368, 9264] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=65012 dup_rows=24028 dup_rows_remote=21080 dedup_ratio=0.244425 router_tokens_per_expert=[775, 778, 744, 777, 777, 768, 810, 763, 759, 771, 815, 748, 782, 729, 767, 817, 776, 785, 765, 724, 741, 776, 791, 748, 796, 740, 765, 757, 801, 750, 794, 788, 759, 701, 757, 743, 789, 774, 791, 744, 783, 792, 792, 773, 819, 767, 724, 779, 774, 768, 771, 774, 754, 695, 807, 765, 754, 741, 764, 763, 789, 780, 741, 748] payload_rows_per_dst=[9248, 9350, 9264, 9400, 9198, 9406, 9174, 9236] dup_rows_per_dst=[3136, 3026, 2948, 2982, 2918, 3052, 3042, 2924] combine_rows_per_src=[9248, 9350, 9264, 9400, 9198, 9406, 9174, 9236] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74362 payload_rows_remote=65148 dup_rows=23942 dup_rows_remote=21106 dedup_ratio=0.243551 router_tokens_per_expert=[760, 780, 789, 776, 798, 764, 780, 739, 809, 795, 786, 757, 750, 802, 764, 787, 754, 747, 727, 784, 778, 748, 796, 792, 758, 790, 748, 757, 776, 773, 740, 776, 739, 723, 783, 763, 765, 738, 757, 757, 790, 774, 791, 794, 718, 745, 731, 766, 758, 759, 822, 787, 724, 768, 771, 758, 757, 797, 807, 740, 813, 735, 789, 753] payload_rows_per_dst=[9302, 9370, 9206, 9254, 9214, 9294, 9354, 9368] dup_rows_per_dst=[3070, 3130, 3046, 2982, 2836, 2924, 2940, 3014] combine_rows_per_src=[9302, 9370, 9206, 9254, 9214, 9294, 9354, 9368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74200 payload_rows_remote=64812 dup_rows=24104 dup_rows_remote=21102 dedup_ratio=0.245199 router_tokens_per_expert=[736, 756, 776, 783, 774, 793, 794, 774, 781, 699, 783, 749, 776, 743, 745, 788, 733, 799, 750, 707, 792, 699, 755, 777, 796, 774, 793, 778, 750, 779, 765, 760, 782, 733, 805, 755, 807, 778, 770, 771, 749, 791, 862, 786, 769, 745, 775, 741, 760, 756, 742, 738, 745, 748, 758, 781, 819, 768, 766, 740, 800, 790, 790, 775] payload_rows_per_dst=[9274, 9154, 9096, 9388, 9282, 9418, 9268, 9320] dup_rows_per_dst=[3098, 2974, 2928, 3002, 3120, 3018, 2788, 3176] combine_rows_per_src=[9274, 9154, 9096, 9388, 9282, 9418, 9268, 9320] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265781248 dispatch_meta_bytes=344096 combine_payload_bytes=265781248 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266125344 ingress_bytes=334987264 link_bytes=334987264 token_bytes=531906592 weight_movement_bytes=69206016 total_nvlink_bytes=601112608 weight_to_token_ratio=0.130109 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260530106e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=344208 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266403984 ingress_bytes=300662784 link_bytes=300662784 token_bytes=532463760 weight_movement_bytes=34603008 total_nvlink_bytes=567066768 weight_to_token_ratio=0.064987 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131369851e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74200 payload_rows_remote=64812 dup_rows=24104 dup_rows_remote=21102 dedup_ratio=0.245199 router_tokens_per_expert=[736, 756, 776, 783, 774, 793, 794, 774, 781, 699, 783, 749, 776, 743, 745, 788, 733, 799, 750, 707, 792, 699, 755, 777, 796, 774, 793, 778, 750, 779, 765, 760, 782, 733, 805, 755, 807, 778, 770, 771, 749, 791, 862, 786, 769, 745, 775, 741, 760, 756, 742, 738, 745, 748, 758, 781, 819, 768, 766, 740, 800, 790, 790, 775] payload_rows_per_dst=[9274, 9154, 9096, 9388, 9282, 9418, 9268, 9320] dup_rows_per_dst=[3098, 2974, 2928, 3002, 3120, 3018, 2788, 3176] combine_rows_per_src=[9274, 9154, 9096, 9388, 9282, 9418, 9268, 9320] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74426 payload_rows_remote=65084 dup_rows=23878 dup_rows_remote=20968 dedup_ratio=0.242900 router_tokens_per_expert=[759, 767, 796, 757, 792, 774, 736, 784, 793, 731, 733, 753, 769, 787, 770, 715, 759, 800, 791, 766, 761, 738, 761, 775, 753, 813, 796, 759, 753, 765, 716, 815, 749, 765, 748, 801, 797, 757, 752, 757, 773, 801, 748, 805, 802, 771, 792, 729, 738, 748, 769, 749, 754, 781, 793, 779, 786, 762, 731, 809, 761, 804, 738, 766] payload_rows_per_dst=[9284, 9176, 9276, 9332, 9342, 9420, 9244, 9352] dup_rows_per_dst=[3046, 2926, 3026, 3008, 2910, 3022, 2978, 2962] combine_rows_per_src=[9284, 9176, 9276, 9332, 9342, 9420, 9244, 9352] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267214848 dispatch_meta_bytes=344928 combine_payload_bytes=267214848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267559776 ingress_bytes=301817856 link_bytes=301817856 token_bytes=534774624 weight_movement_bytes=34603008 total_nvlink_bytes=569377632 weight_to_token_ratio=0.064706 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135716294e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=65012 dup_rows=24028 dup_rows_remote=21080 dedup_ratio=0.244425 router_tokens_per_expert=[775, 778, 744, 777, 777, 768, 810, 763, 759, 771, 815, 748, 782, 729, 767, 817, 776, 785, 765, 724, 741, 776, 791, 748, 796, 740, 765, 757, 801, 750, 794, 788, 759, 701, 757, 743, 789, 774, 791, 744, 783, 792, 792, 773, 819, 767, 724, 779, 774, 768, 771, 774, 754, 695, 807, 765, 754, 741, 764, 763, 789, 780, 741, 748] payload_rows_per_dst=[9248, 9350, 9264, 9400, 9198, 9406, 9174, 9236] dup_rows_per_dst=[3136, 3026, 2948, 2982, 2918, 3052, 3042, 2924] combine_rows_per_src=[9248, 9350, 9264, 9400, 9198, 9406, 9174, 9236] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265379840 dispatch_meta_bytes=343952 combine_payload_bytes=265379840 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265723792 ingress_bytes=334585856 link_bytes=334585856 token_bytes=531103632 weight_movement_bytes=69206016 total_nvlink_bytes=600309648 weight_to_token_ratio=0.130306 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259019640e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266289152 dispatch_meta_bytes=344368 combine_payload_bytes=266289152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266633520 ingress_bytes=300892160 link_bytes=300892160 token_bytes=532922672 weight_movement_bytes=34603008 total_nvlink_bytes=567525680 weight_to_token_ratio=0.064931 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132232974e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266846208 dispatch_meta_bytes=345016 combine_payload_bytes=266846208 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267191224 ingress_bytes=336052224 link_bytes=336052224 token_bytes=534037432 weight_movement_bytes=69206016 total_nvlink_bytes=603243448 weight_to_token_ratio=0.129590 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264537464e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265469952 dispatch_meta_bytes=343656 combine_payload_bytes=265469952 prefetch_bytes=0 reduce_bytes=0 egress_bytes=265813608 ingress_bytes=265469952 link_bytes=265813608 token_bytes=531283560 weight_movement_bytes=0 total_nvlink_bytes=531283560 weight_to_token_ratio=0.000000 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.000235207e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=64932 dup_rows=23982 dup_rows_remote=20940 dedup_ratio=0.243958 router_tokens_per_expert=[749, 796, 780, 751, 784, 801, 771, 777, 732, 756, 748, 767, 763, 788, 825, 751, 760, 793, 780, 757, 737, 772, 782, 762, 799, 771, 748, 755, 787, 718, 749, 775, 746, 775, 771, 774, 818, 780, 755, 810, 769, 774, 750, 765, 759, 842, 801, 711, 770, 789, 784, 709, 772, 733, 731, 799, 758, 798, 757, 771, 766, 724, 801, 706] payload_rows_per_dst=[9390, 9282, 9266, 9206, 9334, 9440, 9190, 9214] dup_rows_per_dst=[3042, 2978, 3020, 2998, 3110, 2950, 2984, 2900] combine_rows_per_src=[9390, 9282, 9266, 9206, 9334, 9440, 9190, 9214] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74524 payload_rows_remote=65110 dup_rows=23780 dup_rows_remote=20764 dedup_ratio=0.241903 router_tokens_per_expert=[749, 785, 759, 751, 761, 747, 775, 762, 769, 721, 757, 767, 773, 746, 752, 811, 762, 737, 769, 758, 708, 748, 782, 775, 797, 816, 790, 828, 761, 771, 760, 759, 791, 779, 746, 795, 760, 733, 787, 743, 782, 789, 822, 796, 741, 771, 769, 745, 785, 797, 757, 795, 750, 780, 740, 764, 779, 744, 765, 758, 749, 778, 786, 770] payload_rows_per_dst=[9266, 9246, 9226, 9428, 9296, 9414, 9306, 9342] dup_rows_per_dst=[2912, 2946, 2852, 3136, 2972, 3016, 3030, 2916] combine_rows_per_src=[9266, 9246, 9226, 9428, 9296, 9414, 9306, 9342] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74290 payload_rows_remote=64964 dup_rows=24014 dup_rows_remote=20976 dedup_ratio=0.244283 router_tokens_per_expert=[784, 796, 745, 768, 746, 782, 776, 770, 803, 773, 793, 715, 773, 740, 807, 778, 771, 790, 743, 741, 799, 767, 769, 773, 725, 739, 742, 751, 767, 759, 796, 761, 734, 776, 754, 786, 781, 754, 772, 786, 801, 732, 774, 790, 785, 780, 772, 752, 764, 758, 750, 769, 758, 769, 751, 723, 764, 801, 747, 753, 821, 795, 780, 778] payload_rows_per_dst=[9324, 9326, 9324, 9208, 9344, 9298, 9064, 9402] dup_rows_per_dst=[3010, 3038, 2982, 2872, 2942, 3074, 3020, 3076] combine_rows_per_src=[9324, 9326, 9324, 9208, 9344, 9298, 9064, 9402] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265961472 dispatch_meta_bytes=343488 combine_payload_bytes=265961472 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266304960 ingress_bytes=369770496 link_bytes=369770496 token_bytes=532266432 weight_movement_bytes=103809024 total_nvlink_bytes=636075456 weight_to_token_ratio=0.195032 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391416607e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73940 payload_rows_remote=64594 dup_rows=24364 dup_rows_remote=21242 dedup_ratio=0.247843 router_tokens_per_expert=[799, 734, 782, 777, 762, 783, 787, 791, 802, 809, 771, 773, 777, 758, 831, 800, 771, 776, 755, 731, 752, 772, 740, 743, 752, 765, 775, 772, 723, 764, 808, 816, 781, 709, 761, 805, 778, 760, 771, 763, 758, 745, 719, 773, 747, 769, 763, 778, 783, 771, 790, 765, 805, 816, 747, 764, 744, 760, 732, 797, 709, 798, 729, 711] payload_rows_per_dst=[9346, 9672, 9152, 9092, 9314, 9056, 9212, 9096] dup_rows_per_dst=[3122, 3400, 2928, 2970, 3070, 2920, 3090, 2864] combine_rows_per_src=[9346, 9672, 9152, 9092, 9314, 9056, 9212, 9096] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266289152 dispatch_meta_bytes=344368 combine_payload_bytes=266289152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266633520 ingress_bytes=300892160 link_bytes=300892160 token_bytes=532922672 weight_movement_bytes=34603008 total_nvlink_bytes=567525680 weight_to_token_ratio=0.064931 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132232974e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266584064 dispatch_meta_bytes=344208 combine_payload_bytes=266584064 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266928272 ingress_bytes=335790080 link_bytes=335790080 token_bytes=533512336 weight_movement_bytes=69206016 total_nvlink_bytes=602718352 weight_to_token_ratio=0.129718 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263551038e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74524 payload_rows_remote=65110 dup_rows=23780 dup_rows_remote=20764 dedup_ratio=0.241903 router_tokens_per_expert=[749, 785, 759, 751, 761, 747, 775, 762, 769, 721, 757, 767, 773, 746, 752, 811, 762, 737, 769, 758, 708, 748, 782, 775, 797, 816, 790, 828, 761, 771, 760, 759, 791, 779, 746, 795, 760, 733, 787, 743, 782, 789, 822, 796, 741, 771, 769, 745, 785, 797, 757, 795, 750, 780, 740, 764, 779, 744, 765, 758, 749, 778, 786, 770] payload_rows_per_dst=[9266, 9246, 9226, 9428, 9296, 9414, 9306, 9342] dup_rows_per_dst=[2912, 2946, 2852, 3136, 2972, 3016, 3030, 2916] combine_rows_per_src=[9266, 9246, 9226, 9428, 9296, 9414, 9306, 9342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=343760 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266436304 ingress_bytes=335298560 link_bytes=335298560 token_bytes=532528848 weight_movement_bytes=69206016 total_nvlink_bytes=601734864 weight_to_token_ratio=0.129957 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261701488e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=264577024 dispatch_meta_bytes=343344 combine_payload_bytes=264577024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264920368 ingress_bytes=299180032 link_bytes=299180032 token_bytes=529497392 weight_movement_bytes=34603008 total_nvlink_bytes=564100400 weight_to_token_ratio=0.065351 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125790374e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74218 payload_rows_remote=64914 dup_rows=24086 dup_rows_remote=21034 dedup_ratio=0.245015 router_tokens_per_expert=[786, 731, 766, 747, 750, 770, 765, 730, 788, 765, 796, 740, 799, 803, 821, 790, 754, 758, 748, 767, 792, 774, 755, 734, 793, 736, 779, 768, 770, 768, 716, 801, 740, 774, 769, 760, 776, 754, 798, 773, 767, 779, 771, 766, 757, 765, 771, 732, 765, 768, 774, 737, 751, 845, 770, 756, 721, 764, 718, 828, 785, 790, 802, 766] payload_rows_per_dst=[9208, 8940, 9618, 9244, 9314, 9320, 9270, 9304] dup_rows_per_dst=[2882, 2768, 3442, 3018, 2974, 2896, 3054, 3052] combine_rows_per_src=[9208, 8940, 9618, 9244, 9314, 9320, 9270, 9304] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74188 payload_rows_remote=65120 dup_rows=24116 dup_rows_remote=21116 dedup_ratio=0.245321 router_tokens_per_expert=[715, 786, 761, 776, 721, 765, 806, 773, 744, 722, 752, 755, 780, 827, 783, 797, 780, 755, 703, 750, 790, 738, 767, 751, 783, 770, 796, 717, 807, 724, 794, 709, 761, 778, 788, 759, 777, 780, 792, 747, 799, 733, 784, 780, 750, 766, 769, 718, 818, 752, 800, 788, 847, 781, 756, 796, 757, 752, 753, 783, 763, 778, 779, 771] payload_rows_per_dst=[9238, 9256, 9068, 9290, 9280, 9294, 9382, 9380] dup_rows_per_dst=[2968, 3064, 3000, 2910, 3084, 2904, 3294, 2892] combine_rows_per_src=[9238, 9256, 9068, 9290, 9280, 9294, 9382, 9380] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266690560 dispatch_meta_bytes=343496 combine_payload_bytes=266690560 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267034056 ingress_bytes=301293568 link_bytes=301293568 token_bytes=533724616 weight_movement_bytes=34603008 total_nvlink_bytes=568327624 weight_to_token_ratio=0.064833 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133743440e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65038 dup_rows=23908 dup_rows_remote=20916 dedup_ratio=0.243205 router_tokens_per_expert=[755, 747, 759, 766, 776, 752, 787, 775, 759, 772, 775, 761, 771, 806, 748, 768, 742, 725, 758, 772, 797, 779, 821, 780, 775, 775, 792, 770, 747, 759, 763, 794, 735, 789, 773, 785, 770, 757, 759, 810, 783, 778, 775, 791, 776, 779, 757, 754, 778, 775, 728, 734, 782, 789, 761, 744, 760, 766, 756, 739, 753, 762, 766, 762] payload_rows_per_dst=[9260, 9332, 9298, 9358, 9318, 9320, 9312, 9198] dup_rows_per_dst=[2974, 2988, 3050, 2992, 3038, 3066, 2870, 2930] combine_rows_per_src=[9260, 9332, 9298, 9358, 9318, 9320, 9312, 9198] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74456 payload_rows_remote=65208 dup_rows=23848 dup_rows_remote=20904 dedup_ratio=0.242594 router_tokens_per_expert=[775, 778, 765, 800, 738, 752, 739, 792, 804, 767, 785, 805, 755, 762, 787, 768, 774, 743, 800, 764, 765, 772, 809, 787, 788, 807, 761, 757, 792, 744, 728, 756, 790, 789, 786, 755, 822, 747, 762, 756, 749, 746, 744, 747, 754, 753, 764, 766, 776, 762, 798, 742, 727, 786, 748, 757, 766, 746, 770, 766, 761, 778, 723, 797] payload_rows_per_dst=[9282, 9364, 9438, 9328, 9380, 9106, 9248, 9310] dup_rows_per_dst=[2996, 3102, 2990, 2938, 3034, 2940, 2944, 2904] combine_rows_per_src=[9282, 9364, 9438, 9328, 9380, 9106, 9248, 9310] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=65046 dup_rows=24028 dup_rows_remote=21004 dedup_ratio=0.244425 router_tokens_per_expert=[768, 788, 784, 758, 788, 810, 799, 724, 774, 736, 747, 754, 747, 776, 725, 825, 767, 790, 825, 750, 757, 749, 768, 711, 804, 793, 691, 719, 788, 822, 755, 769, 714, 781, 762, 729, 818, 807, 763, 753, 770, 779, 746, 785, 803, 782, 795, 767, 771, 777, 756, 747, 754, 708, 796, 750, 753, 752, 779, 750, 785, 799, 744, 816] payload_rows_per_dst=[9326, 9168, 9198, 9328, 9230, 9430, 9248, 9348] dup_rows_per_dst=[3112, 3000, 3036, 2954, 3024, 3024, 2870, 3008] combine_rows_per_src=[9326, 9168, 9198, 9328, 9230, 9430, 9248, 9348] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=343760 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266436304 ingress_bytes=335298560 link_bytes=335298560 token_bytes=532528848 weight_movement_bytes=69206016 total_nvlink_bytes=601734864 weight_to_token_ratio=0.129957 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261701488e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65038 dup_rows=23908 dup_rows_remote=20916 dedup_ratio=0.243205 router_tokens_per_expert=[755, 747, 759, 766, 776, 752, 787, 775, 759, 772, 775, 761, 771, 806, 748, 768, 742, 725, 758, 772, 797, 779, 821, 780, 775, 775, 792, 770, 747, 759, 763, 794, 735, 789, 773, 785, 770, 757, 759, 810, 783, 778, 775, 791, 776, 779, 757, 754, 778, 775, 728, 734, 782, 789, 761, 744, 760, 766, 756, 739, 753, 762, 766, 762] payload_rows_per_dst=[9260, 9332, 9298, 9358, 9318, 9320, 9312, 9198] dup_rows_per_dst=[2974, 2988, 3050, 2992, 3038, 3066, 2870, 2930] combine_rows_per_src=[9260, 9332, 9298, 9358, 9318, 9320, 9312, 9198] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74188 payload_rows_remote=65120 dup_rows=24116 dup_rows_remote=21116 dedup_ratio=0.245321 router_tokens_per_expert=[715, 786, 761, 776, 721, 765, 806, 773, 744, 722, 752, 755, 780, 827, 783, 797, 780, 755, 703, 750, 790, 738, 767, 751, 783, 770, 796, 717, 807, 724, 794, 709, 761, 778, 788, 759, 777, 780, 792, 747, 799, 733, 784, 780, 750, 766, 769, 718, 818, 752, 800, 788, 847, 781, 756, 796, 757, 752, 753, 783, 763, 778, 779, 771] payload_rows_per_dst=[9238, 9256, 9068, 9290, 9280, 9294, 9382, 9380] dup_rows_per_dst=[2968, 3064, 3000, 2910, 3084, 2904, 3294, 2892] combine_rows_per_src=[9238, 9256, 9068, 9290, 9280, 9294, 9382, 9380] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265887744 dispatch_meta_bytes=343792 combine_payload_bytes=265887744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266231536 ingress_bytes=300490752 link_bytes=300490752 token_bytes=532119280 weight_movement_bytes=34603008 total_nvlink_bytes=566722288 weight_to_token_ratio=0.065029 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130722508e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=344944 combine_payload_bytes=266731520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267076464 ingress_bytes=301334528 link_bytes=301334528 token_bytes=533807984 weight_movement_bytes=34603008 total_nvlink_bytes=568410992 weight_to_token_ratio=0.064823 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133897569e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=344200 combine_payload_bytes=266428416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266772616 ingress_bytes=301031424 link_bytes=301031424 token_bytes=533201032 weight_movement_bytes=34603008 total_nvlink_bytes=567804040 weight_to_token_ratio=0.064897 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132757013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267091968 dispatch_meta_bytes=344448 combine_payload_bytes=267091968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267436416 ingress_bytes=301694976 link_bytes=301694976 token_bytes=534528384 weight_movement_bytes=34603008 total_nvlink_bytes=569131392 weight_to_token_ratio=0.064736 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135253906e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74456 payload_rows_remote=65052 dup_rows=23848 dup_rows_remote=20744 dedup_ratio=0.242594 router_tokens_per_expert=[785, 763, 779, 747, 776, 744, 766, 795, 828, 784, 821, 768, 744, 771, 766, 772, 732, 721, 767, 782, 785, 677, 759, 734, 741, 790, 777, 778, 708, 749, 800, 793, 761, 774, 797, 783, 769, 792, 835, 787, 774, 711, 754, 744, 813, 772, 756, 812, 701, 766, 749, 798, 776, 776, 795, 780, 769, 762, 757, 782, 767, 717, 769, 752] payload_rows_per_dst=[9320, 9404, 9052, 9296, 9606, 9246, 9320, 9212] dup_rows_per_dst=[2990, 3104, 2862, 2976, 2990, 3026, 2962, 2938] combine_rows_per_src=[9320, 9404, 9052, 9296, 9606, 9246, 9320, 9212] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74362 payload_rows_remote=65148 dup_rows=23942 dup_rows_remote=21106 dedup_ratio=0.243551 router_tokens_per_expert=[760, 780, 789, 776, 798, 764, 780, 739, 809, 795, 786, 757, 750, 802, 764, 787, 754, 747, 727, 784, 778, 748, 796, 792, 758, 790, 748, 757, 776, 773, 740, 776, 739, 723, 783, 763, 765, 738, 757, 757, 790, 774, 791, 794, 718, 745, 731, 766, 758, 759, 822, 787, 724, 768, 771, 758, 757, 797, 807, 740, 813, 735, 789, 753] payload_rows_per_dst=[9302, 9370, 9206, 9254, 9214, 9294, 9354, 9368] dup_rows_per_dst=[3070, 3130, 3046, 2982, 2836, 2924, 2940, 3014] combine_rows_per_src=[9302, 9370, 9206, 9254, 9214, 9294, 9354, 9368] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=343816 combine_payload_bytes=266395648 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266739464 ingress_bytes=335601664 link_bytes=335601664 token_bytes=533135112 weight_movement_bytes=69206016 total_nvlink_bytes=602341128 weight_to_token_ratio=0.129810 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262842043e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73940 payload_rows_remote=64594 dup_rows=24364 dup_rows_remote=21242 dedup_ratio=0.247843 router_tokens_per_expert=[799, 734, 782, 777, 762, 783, 787, 791, 802, 809, 771, 773, 777, 758, 831, 800, 771, 776, 755, 731, 752, 772, 740, 743, 752, 765, 775, 772, 723, 764, 808, 816, 781, 709, 761, 805, 778, 760, 771, 763, 758, 745, 719, 773, 747, 769, 763, 778, 783, 771, 790, 765, 805, 816, 747, 764, 744, 760, 732, 797, 709, 798, 729, 711] payload_rows_per_dst=[9346, 9672, 9152, 9092, 9314, 9056, 9212, 9096] dup_rows_per_dst=[3122, 3400, 2928, 2970, 3070, 2920, 3090, 2864] combine_rows_per_src=[9346, 9672, 9152, 9092, 9314, 9056, 9212, 9096] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267091968 dispatch_meta_bytes=344448 combine_payload_bytes=267091968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267436416 ingress_bytes=301694976 link_bytes=301694976 token_bytes=534528384 weight_movement_bytes=34603008 total_nvlink_bytes=569131392 weight_to_token_ratio=0.064736 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135253906e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=64638 dup_rows=24056 dup_rows_remote=20644 dedup_ratio=0.244710 router_tokens_per_expert=[768, 717, 737, 781, 778, 711, 841, 797, 809, 783, 762, 768, 788, 758, 756, 785, 789, 750, 750, 739, 758, 723, 781, 747, 739, 814, 739, 790, 781, 777, 778, 771, 731, 809, 788, 761, 721, 738, 803, 742, 766, 783, 776, 754, 784, 781, 755, 751, 789, 758, 753, 751, 791, 786, 757, 747, 772, 773, 742, 789, 794, 762, 804, 776] payload_rows_per_dst=[9610, 8960, 9774, 8952, 9274, 9254, 9270, 9154] dup_rows_per_dst=[3412, 2700, 3432, 2622, 3032, 2970, 2990, 2898] combine_rows_per_src=[9610, 8960, 9774, 8952, 9274, 9254, 9270, 9154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=344944 combine_payload_bytes=266731520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267076464 ingress_bytes=301334528 link_bytes=301334528 token_bytes=533807984 weight_movement_bytes=34603008 total_nvlink_bytes=568410992 weight_to_token_ratio=0.064823 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133897569e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266452992 dispatch_meta_bytes=343184 combine_payload_bytes=266452992 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266796176 ingress_bytes=370262016 link_bytes=370262016 token_bytes=533249168 weight_movement_bytes=103809024 total_nvlink_bytes=637058192 weight_to_token_ratio=0.194673 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393266158e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74212 payload_rows_remote=64944 dup_rows=24092 dup_rows_remote=21080 dedup_ratio=0.245076 router_tokens_per_expert=[753, 742, 740, 775, 733, 685, 812, 806, 798, 764, 815, 720, 776, 843, 795, 774, 759, 771, 776, 755, 742, 790, 757, 762, 777, 822, 788, 740, 758, 800, 740, 755, 775, 816, 738, 774, 766, 769, 794, 715, 800, 797, 755, 773, 705, 768, 802, 740, 709, 765, 754, 765, 769, 788, 749, 776, 763, 811, 737, 772, 777, 745, 763, 799] payload_rows_per_dst=[9200, 9442, 9204, 9308, 9242, 9268, 9230, 9318] dup_rows_per_dst=[2892, 3128, 3020, 3052, 3052, 3012, 2920, 3016] combine_rows_per_src=[9200, 9442, 9204, 9308, 9242, 9268, 9230, 9318] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74456 payload_rows_remote=65052 dup_rows=23848 dup_rows_remote=20744 dedup_ratio=0.242594 router_tokens_per_expert=[785, 763, 779, 747, 776, 744, 766, 795, 828, 784, 821, 768, 744, 771, 766, 772, 732, 721, 767, 782, 785, 677, 759, 734, 741, 790, 777, 778, 708, 749, 800, 793, 761, 774, 797, 783, 769, 792, 835, 787, 774, 711, 754, 744, 813, 772, 756, 812, 701, 766, 749, 798, 776, 776, 795, 780, 769, 762, 757, 782, 767, 717, 769, 752] payload_rows_per_dst=[9320, 9404, 9052, 9296, 9606, 9246, 9320, 9212] dup_rows_per_dst=[2990, 3104, 2862, 2976, 2990, 3026, 2962, 2938] combine_rows_per_src=[9320, 9404, 9052, 9296, 9606, 9246, 9320, 9212] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266846208 dispatch_meta_bytes=345016 combine_payload_bytes=266846208 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267191224 ingress_bytes=336052224 link_bytes=336052224 token_bytes=534037432 weight_movement_bytes=69206016 total_nvlink_bytes=603243448 weight_to_token_ratio=0.129590 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264537464e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=264757248 dispatch_meta_bytes=341128 combine_payload_bytes=264757248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265098376 ingress_bytes=299360256 link_bytes=299360256 token_bytes=529855624 weight_movement_bytes=34603008 total_nvlink_bytes=564458632 weight_to_token_ratio=0.065306 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126468543e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=264577024 dispatch_meta_bytes=343344 combine_payload_bytes=264577024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264920368 ingress_bytes=299180032 link_bytes=299180032 token_bytes=529497392 weight_movement_bytes=34603008 total_nvlink_bytes=564100400 weight_to_token_ratio=0.065351 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125790374e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74612 payload_rows_remote=65282 dup_rows=23692 dup_rows_remote=20708 dedup_ratio=0.241007 router_tokens_per_expert=[728, 776, 769, 797, 764, 753, 775, 746, 818, 761, 784, 773, 756, 783, 787, 779, 758, 743, 733, 748, 740, 744, 731, 745, 794, 764, 763, 790, 800, 723, 788, 761, 716, 750, 801, 733, 811, 745, 743, 753, 790, 771, 751, 771, 790, 801, 828, 734, 823, 785, 820, 749, 761, 780, 750, 769, 755, 787, 799, 782, 776, 770, 720, 764] payload_rows_per_dst=[9266, 9476, 9110, 9398, 9226, 9370, 9436, 9330] dup_rows_per_dst=[2950, 2998, 2774, 2968, 2878, 3102, 3038, 2984] combine_rows_per_src=[9266, 9476, 9110, 9398, 9226, 9370, 9436, 9330] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266010624 dispatch_meta_bytes=344096 combine_payload_bytes=266010624 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266354720 ingress_bytes=335216640 link_bytes=335216640 token_bytes=532365344 weight_movement_bytes=69206016 total_nvlink_bytes=601571360 weight_to_token_ratio=0.129997 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261393229e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=65060 dup_rows=23926 dup_rows_remote=21068 dedup_ratio=0.243388 router_tokens_per_expert=[745, 796, 804, 720, 771, 777, 752, 728, 741, 803, 750, 779, 732, 746, 771, 790, 772, 740, 780, 782, 780, 791, 714, 729, 818, 808, 780, 745, 773, 774, 790, 767, 796, 745, 747, 712, 756, 764, 750, 762, 760, 735, 754, 763, 804, 791, 747, 852, 814, 758, 757, 768, 743, 763, 772, 753, 795, 796, 749, 801, 763, 765, 772, 797] payload_rows_per_dst=[9190, 9220, 9318, 9376, 9206, 9362, 9282, 9424] dup_rows_per_dst=[2996, 3004, 2858, 3134, 2858, 3050, 2974, 3052] combine_rows_per_src=[9190, 9220, 9318, 9376, 9206, 9362, 9282, 9424] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74280 payload_rows_remote=65074 dup_rows=24024 dup_rows_remote=21110 dedup_ratio=0.244385 router_tokens_per_expert=[763, 739, 740, 775, 752, 745, 777, 783, 759, 822, 737, 794, 789, 813, 766, 749, 763, 714, 776, 759, 771, 752, 783, 814, 754, 748, 738, 782, 789, 766, 803, 779, 795, 762, 730, 754, 774, 755, 728, 762, 758, 813, 744, 767, 745, 807, 776, 769, 818, 788, 762, 759, 750, 815, 758, 776, 734, 736, 778, 744, 757, 805, 778, 761] payload_rows_per_dst=[9234, 9476, 9230, 9218, 9206, 9316, 9402, 9198] dup_rows_per_dst=[2914, 2982, 3034, 3100, 2914, 3042, 3050, 2988] combine_rows_per_src=[9234, 9476, 9230, 9218, 9206, 9316, 9402, 9198] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74064 payload_rows_remote=64784 dup_rows=24240 dup_rows_remote=21238 dedup_ratio=0.246582 router_tokens_per_expert=[783, 783, 753, 776, 781, 785, 745, 772, 805, 733, 694, 787, 790, 765, 769, 737, 735, 783, 736, 791, 727, 757, 754, 782, 724, 779, 816, 753, 801, 787, 787, 805, 770, 720, 768, 757, 729, 797, 755, 756, 785, 776, 736, 753, 794, 791, 773, 728, 771, 821, 788, 739, 726, 782, 738, 776, 799, 782, 784, 857, 762, 734, 748, 782] payload_rows_per_dst=[9280, 9114, 9208, 9352, 9178, 9264, 9280, 9388] dup_rows_per_dst=[3076, 3046, 2922, 3152, 2926, 3008, 3002, 3108] combine_rows_per_src=[9280, 9114, 9208, 9352, 9178, 9264, 9280, 9388] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266452992 dispatch_meta_bytes=343184 combine_payload_bytes=266452992 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266796176 ingress_bytes=370262016 link_bytes=370262016 token_bytes=533249168 weight_movement_bytes=103809024 total_nvlink_bytes=637058192 weight_to_token_ratio=0.194673 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393266158e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74390 payload_rows_remote=65070 dup_rows=23914 dup_rows_remote=20842 dedup_ratio=0.243266 router_tokens_per_expert=[753, 727, 763, 752, 778, 769, 760, 743, 744, 852, 785, 736, 772, 780, 782, 727, 761, 788, 742, 775, 751, 776, 773, 765, 775, 784, 759, 739, 740, 795, 821, 783, 754, 771, 778, 817, 799, 784, 758, 776, 763, 730, 769, 774, 762, 758, 759, 764, 758, 764, 750, 768, 782, 712, 746, 747, 765, 823, 833, 754, 785, 782, 771, 746] payload_rows_per_dst=[9124, 9302, 9300, 9320, 9442, 9316, 9114, 9472] dup_rows_per_dst=[2966, 3054, 2962, 3072, 3032, 2842, 2940, 3046] combine_rows_per_src=[9124, 9302, 9300, 9320, 9442, 9316, 9114, 9472] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74064 payload_rows_remote=64784 dup_rows=24240 dup_rows_remote=21238 dedup_ratio=0.246582 router_tokens_per_expert=[783, 783, 753, 776, 781, 785, 745, 772, 805, 733, 694, 787, 790, 765, 769, 737, 735, 783, 736, 791, 727, 757, 754, 782, 724, 779, 816, 753, 801, 787, 787, 805, 770, 720, 768, 757, 729, 797, 755, 756, 785, 776, 736, 753, 794, 791, 773, 728, 771, 821, 788, 739, 726, 782, 738, 776, 799, 782, 784, 857, 762, 734, 748, 782] payload_rows_per_dst=[9280, 9114, 9208, 9352, 9178, 9264, 9280, 9388] dup_rows_per_dst=[3076, 3046, 2922, 3152, 2926, 3008, 3002, 3108] combine_rows_per_src=[9280, 9114, 9208, 9352, 9178, 9264, 9280, 9388] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=267395072 dispatch_meta_bytes=343960 combine_payload_bytes=267395072 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267739032 ingress_bytes=371204096 link_bytes=371204096 token_bytes=535134104 weight_movement_bytes=103809024 total_nvlink_bytes=638943128 weight_to_token_ratio=0.193987 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396811129e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=344736 combine_payload_bytes=266543104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266887840 ingress_bytes=301146112 link_bytes=301146112 token_bytes=533430944 weight_movement_bytes=34603008 total_nvlink_bytes=568033952 weight_to_token_ratio=0.064869 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133188575e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266485760 dispatch_meta_bytes=344512 combine_payload_bytes=266485760 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266830272 ingress_bytes=301088768 link_bytes=301088768 token_bytes=533316032 weight_movement_bytes=34603008 total_nvlink_bytes=567919040 weight_to_token_ratio=0.064883 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132972794e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=64638 dup_rows=24056 dup_rows_remote=20644 dedup_ratio=0.244710 router_tokens_per_expert=[768, 717, 737, 781, 778, 711, 841, 797, 809, 783, 762, 768, 788, 758, 756, 785, 789, 750, 750, 739, 758, 723, 781, 747, 739, 814, 739, 790, 781, 777, 778, 771, 731, 809, 788, 761, 721, 738, 803, 742, 766, 783, 776, 754, 784, 781, 755, 751, 789, 758, 753, 751, 791, 786, 757, 747, 772, 773, 742, 789, 794, 762, 804, 776] payload_rows_per_dst=[9610, 8960, 9774, 8952, 9274, 9254, 9270, 9154] dup_rows_per_dst=[3412, 2700, 3432, 2622, 3032, 2970, 2990, 2898] combine_rows_per_src=[9610, 8960, 9774, 8952, 9274, 9254, 9270, 9154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=65060 dup_rows=23926 dup_rows_remote=21068 dedup_ratio=0.243388 router_tokens_per_expert=[745, 796, 804, 720, 771, 777, 752, 728, 741, 803, 750, 779, 732, 746, 771, 790, 772, 740, 780, 782, 780, 791, 714, 729, 818, 808, 780, 745, 773, 774, 790, 767, 796, 745, 747, 712, 756, 764, 750, 762, 760, 735, 754, 763, 804, 791, 747, 852, 814, 758, 757, 768, 743, 763, 772, 753, 795, 796, 749, 801, 763, 765, 772, 797] payload_rows_per_dst=[9190, 9220, 9318, 9376, 9206, 9362, 9282, 9424] dup_rows_per_dst=[2996, 3004, 2858, 3134, 2858, 3050, 2974, 3052] combine_rows_per_src=[9190, 9220, 9318, 9376, 9206, 9362, 9282, 9424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74316 payload_rows_remote=65030 dup_rows=23988 dup_rows_remote=21046 dedup_ratio=0.244019 router_tokens_per_expert=[765, 775, 773, 765, 758, 741, 772, 754, 776, 750, 797, 760, 780, 744, 749, 758, 793, 762, 807, 802, 755, 812, 817, 762, 773, 805, 798, 803, 745, 749, 764, 787, 750, 711, 772, 735, 744, 782, 808, 716, 743, 807, 790, 769, 791, 736, 763, 765, 764, 781, 749, 777, 778, 773, 736, 767, 785, 772, 769, 762, 751, 760, 792, 703] payload_rows_per_dst=[9282, 9286, 9386, 9446, 9070, 9380, 9292, 9174] dup_rows_per_dst=[2924, 2942, 3234, 3002, 2966, 2948, 2958, 3014] combine_rows_per_src=[9282, 9286, 9386, 9446, 9070, 9380, 9292, 9174] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266526720 dispatch_meta_bytes=343648 combine_payload_bytes=266526720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266870368 ingress_bytes=335732736 link_bytes=335732736 token_bytes=533397088 weight_movement_bytes=69206016 total_nvlink_bytes=602603104 weight_to_token_ratio=0.129746 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263335257e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265355264 dispatch_meta_bytes=344088 combine_payload_bytes=265355264 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265699352 ingress_bytes=369164288 link_bytes=369164288 token_bytes=531054616 weight_movement_bytes=103809024 total_nvlink_bytes=634863640 weight_to_token_ratio=0.195477 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389135496e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=65046 dup_rows=24028 dup_rows_remote=21004 dedup_ratio=0.244425 router_tokens_per_expert=[768, 788, 784, 758, 788, 810, 799, 724, 774, 736, 747, 754, 747, 776, 725, 825, 767, 790, 825, 750, 757, 749, 768, 711, 804, 793, 691, 719, 788, 822, 755, 769, 714, 781, 762, 729, 818, 807, 763, 753, 770, 779, 746, 785, 803, 782, 795, 767, 771, 777, 756, 747, 754, 708, 796, 750, 753, 752, 779, 750, 785, 799, 744, 816] payload_rows_per_dst=[9326, 9168, 9198, 9328, 9230, 9430, 9248, 9348] dup_rows_per_dst=[3112, 3000, 3036, 2954, 3024, 3024, 2870, 3008] combine_rows_per_src=[9326, 9168, 9198, 9328, 9230, 9430, 9248, 9348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74100 payload_rows_remote=64726 dup_rows=24204 dup_rows_remote=21164 dedup_ratio=0.246216 router_tokens_per_expert=[774, 770, 813, 735, 795, 746, 789, 785, 750, 753, 760, 775, 771, 762, 750, 748, 784, 745, 739, 764, 751, 802, 762, 767, 784, 754, 773, 769, 745, 744, 810, 757, 751, 744, 777, 738, 747, 760, 776, 765, 810, 780, 827, 766, 761, 746, 817, 794, 813, 756, 760, 761, 712, 767, 761, 759, 768, 750, 774, 776, 791, 777, 774, 768] payload_rows_per_dst=[9374, 9342, 9394, 9078, 9344, 9126, 9124, 9318] dup_rows_per_dst=[3040, 2962, 3160, 2910, 3132, 2908, 3054, 3038] combine_rows_per_src=[9374, 9342, 9394, 9078, 9344, 9126, 9124, 9318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=65012 dup_rows=23964 dup_rows_remote=20876 dedup_ratio=0.243774 router_tokens_per_expert=[734, 811, 808, 824, 750, 770, 791, 741, 761, 819, 750, 739, 779, 739, 827, 692, 715, 738, 770, 767, 804, 777, 817, 769, 766, 760, 775, 764, 731, 777, 762, 789, 783, 768, 755, 720, 787, 774, 774, 756, 761, 756, 822, 791, 798, 730, 774, 776, 746, 749, 814, 787, 759, 750, 756, 754, 738, 734, 797, 775, 759, 731, 773, 789] payload_rows_per_dst=[9314, 9246, 9368, 9310, 9254, 9328, 9218, 9302] dup_rows_per_dst=[3144, 2966, 2946, 2938, 2980, 3088, 3012, 2890] combine_rows_per_src=[9314, 9246, 9368, 9310, 9254, 9328, 9218, 9302] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74316 payload_rows_remote=65030 dup_rows=23988 dup_rows_remote=21046 dedup_ratio=0.244019 router_tokens_per_expert=[765, 775, 773, 765, 758, 741, 772, 754, 776, 750, 797, 760, 780, 744, 749, 758, 793, 762, 807, 802, 755, 812, 817, 762, 773, 805, 798, 803, 745, 749, 764, 787, 750, 711, 772, 735, 744, 782, 808, 716, 743, 807, 790, 769, 791, 736, 763, 765, 764, 781, 749, 777, 778, 773, 736, 767, 785, 772, 769, 762, 751, 760, 792, 703] payload_rows_per_dst=[9282, 9286, 9386, 9446, 9070, 9380, 9292, 9174] dup_rows_per_dst=[2924, 2942, 3234, 3002, 2966, 2948, 2958, 3014] combine_rows_per_src=[9282, 9286, 9386, 9446, 9070, 9380, 9292, 9174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266485760 dispatch_meta_bytes=344512 combine_payload_bytes=266485760 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266830272 ingress_bytes=301088768 link_bytes=301088768 token_bytes=533316032 weight_movement_bytes=34603008 total_nvlink_bytes=567919040 weight_to_token_ratio=0.064883 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132972794e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344304 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266707184 ingress_bytes=300965888 link_bytes=300965888 token_bytes=533070064 weight_movement_bytes=34603008 total_nvlink_bytes=567673072 weight_to_token_ratio=0.064913 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132510407e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=264757248 dispatch_meta_bytes=341128 combine_payload_bytes=264757248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265098376 ingress_bytes=299360256 link_bytes=299360256 token_bytes=529855624 weight_movement_bytes=34603008 total_nvlink_bytes=564458632 weight_to_token_ratio=0.065306 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126468543e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=344200 combine_payload_bytes=266428416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266772616 ingress_bytes=301031424 link_bytes=301031424 token_bytes=533201032 weight_movement_bytes=34603008 total_nvlink_bytes=567804040 weight_to_token_ratio=0.064897 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132757013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74096 payload_rows_remote=64700 dup_rows=24208 dup_rows_remote=21012 dedup_ratio=0.246257 router_tokens_per_expert=[788, 709, 765, 801, 765, 743, 734, 768, 770, 764, 778, 778, 800, 812, 769, 774, 820, 741, 765, 744, 717, 781, 738, 787, 768, 792, 735, 764, 739, 784, 777, 727, 786, 787, 799, 749, 769, 758, 767, 773, 760, 774, 771, 740, 777, 807, 777, 726, 775, 773, 778, 806, 760, 776, 783, 757, 775, 768, 767, 746, 756, 787, 777, 751] payload_rows_per_dst=[9170, 9394, 9220, 9026, 9250, 9150, 9490, 9396] dup_rows_per_dst=[2976, 3096, 2966, 2808, 3126, 2784, 3256, 3196] combine_rows_per_src=[9170, 9394, 9220, 9026, 9250, 9150, 9490, 9396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265117696 dispatch_meta_bytes=343560 combine_payload_bytes=265117696 prefetch_bytes=0 reduce_bytes=0 egress_bytes=265461256 ingress_bytes=265117696 link_bytes=265461256 token_bytes=530578952 weight_movement_bytes=0 total_nvlink_bytes=530578952 weight_to_token_ratio=0.000000 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.989093349e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266289152 dispatch_meta_bytes=343552 combine_payload_bytes=266289152 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266632704 ingress_bytes=404701184 link_bytes=404701184 token_bytes=532921856 weight_movement_bytes=138412032 total_nvlink_bytes=671333888 weight_to_token_ratio=0.259723 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522857974e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74544 payload_rows_remote=65214 dup_rows=23760 dup_rows_remote=20768 dedup_ratio=0.241699 router_tokens_per_expert=[748, 769, 742, 780, 767, 760, 780, 742, 788, 760, 768, 776, 743, 753, 769, 802, 738, 787, 763, 756, 799, 770, 743, 760, 784, 773, 781, 767, 737, 756, 765, 812, 769, 770, 789, 784, 726, 796, 738, 789, 759, 801, 797, 797, 768, 809, 756, 782, 780, 766, 772, 796, 755, 758, 736, 761, 737, 750, 777, 762, 766, 782, 722, 764] payload_rows_per_dst=[9224, 9380, 9232, 9374, 9330, 9468, 9374, 9162] dup_rows_per_dst=[2952, 2938, 3000, 2976, 2992, 3070, 2874, 2958] combine_rows_per_src=[9224, 9380, 9232, 9374, 9330, 9468, 9374, 9162] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344304 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266707184 ingress_bytes=300965888 link_bytes=300965888 token_bytes=533070064 weight_movement_bytes=34603008 total_nvlink_bytes=567673072 weight_to_token_ratio=0.064913 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132510407e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74300 payload_rows_remote=65202 dup_rows=24004 dup_rows_remote=21120 dedup_ratio=0.244181 router_tokens_per_expert=[804, 727, 769, 760, 782, 756, 795, 756, 766, 746, 754, 736, 792, 786, 785, 763, 758, 706, 775, 751, 778, 726, 743, 754, 781, 784, 752, 840, 779, 825, 759, 779, 812, 789, 746, 794, 779, 744, 722, 805, 780, 758, 737, 786, 802, 759, 737, 789, 748, 752, 756, 789, 749, 748, 776, 750, 771, 767, 739, 799, 741, 794, 767, 800] payload_rows_per_dst=[9218, 9210, 9098, 9530, 9398, 9366, 9230, 9250] dup_rows_per_dst=[3080, 3046, 2884, 3068, 2984, 2930, 2906, 3106] combine_rows_per_src=[9218, 9210, 9098, 9530, 9398, 9366, 9230, 9250] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265011200 dispatch_meta_bytes=342848 combine_payload_bytes=265011200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265354048 ingress_bytes=299614208 link_bytes=299614208 token_bytes=530365248 weight_movement_bytes=34603008 total_nvlink_bytes=564968256 weight_to_token_ratio=0.065244 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127424144e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74644 payload_rows_remote=65324 dup_rows=23660 dup_rows_remote=20758 dedup_ratio=0.240682 router_tokens_per_expert=[796, 750, 759, 757, 779, 793, 755, 752, 704, 827, 725, 763, 790, 774, 770, 774, 704, 762, 747, 770, 752, 772, 806, 763, 766, 766, 781, 780, 734, 765, 757, 762, 774, 805, 758, 767, 742, 736, 771, 794, 764, 787, 807, 758, 747, 804, 786, 770, 761, 809, 767, 769, 754, 769, 764, 789, 790, 789, 774, 769, 760, 777, 739, 747] payload_rows_per_dst=[9374, 9322, 9258, 9320, 9350, 9328, 9342, 9350] dup_rows_per_dst=[2908, 2932, 2894, 2902, 2944, 3118, 3022, 2940] combine_rows_per_src=[9374, 9322, 9258, 9320, 9350, 9328, 9342, 9350] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74118 payload_rows_remote=64824 dup_rows=24186 dup_rows_remote=21104 dedup_ratio=0.246033 router_tokens_per_expert=[756, 784, 775, 772, 757, 730, 770, 776, 751, 739, 784, 748, 738, 747, 715, 805, 765, 757, 798, 766, 733, 806, 751, 811, 804, 813, 744, 772, 762, 767, 798, 796, 773, 758, 740, 770, 734, 812, 810, 776, 742, 731, 754, 804, 752, 776, 751, 754, 783, 768, 807, 746, 749, 779, 789, 767, 734, 787, 754, 787, 760, 774, 794, 747] payload_rows_per_dst=[9186, 9180, 9364, 9378, 9250, 9238, 9294, 9228] dup_rows_per_dst=[3054, 2874, 3010, 3134, 3096, 2890, 3082, 3046] combine_rows_per_src=[9186, 9180, 9364, 9378, 9250, 9238, 9294, 9228] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267116544 dispatch_meta_bytes=343928 combine_payload_bytes=267116544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267460472 ingress_bytes=301719552 link_bytes=301719552 token_bytes=534577016 weight_movement_bytes=34603008 total_nvlink_bytes=569180024 weight_to_token_ratio=0.064730 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135346384e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265011200 dispatch_meta_bytes=342848 combine_payload_bytes=265011200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265354048 ingress_bytes=299614208 link_bytes=299614208 token_bytes=530365248 weight_movement_bytes=34603008 total_nvlink_bytes=564968256 weight_to_token_ratio=0.065244 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127424144e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74100 payload_rows_remote=64726 dup_rows=24204 dup_rows_remote=21164 dedup_ratio=0.246216 router_tokens_per_expert=[774, 770, 813, 735, 795, 746, 789, 785, 750, 753, 760, 775, 771, 762, 750, 748, 784, 745, 739, 764, 751, 802, 762, 767, 784, 754, 773, 769, 745, 744, 810, 757, 751, 744, 777, 738, 747, 760, 776, 765, 810, 780, 827, 766, 761, 746, 817, 794, 813, 756, 760, 761, 712, 767, 761, 759, 768, 750, 774, 776, 791, 777, 774, 768] payload_rows_per_dst=[9374, 9342, 9394, 9078, 9344, 9126, 9124, 9318] dup_rows_per_dst=[3040, 2962, 3160, 2910, 3132, 2908, 3054, 3038] combine_rows_per_src=[9374, 9342, 9394, 9078, 9344, 9126, 9124, 9318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74300 payload_rows_remote=65202 dup_rows=24004 dup_rows_remote=21120 dedup_ratio=0.244181 router_tokens_per_expert=[804, 727, 769, 760, 782, 756, 795, 756, 766, 746, 754, 736, 792, 786, 785, 763, 758, 706, 775, 751, 778, 726, 743, 754, 781, 784, 752, 840, 779, 825, 759, 779, 812, 789, 746, 794, 779, 744, 722, 805, 780, 758, 737, 786, 802, 759, 737, 789, 748, 752, 756, 789, 749, 748, 776, 750, 771, 767, 739, 799, 741, 794, 767, 800] payload_rows_per_dst=[9218, 9210, 9098, 9530, 9398, 9366, 9230, 9250] dup_rows_per_dst=[3080, 3046, 2884, 3068, 2984, 2930, 2906, 3106] combine_rows_per_src=[9218, 9210, 9098, 9530, 9398, 9366, 9230, 9250] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267067392 dispatch_meta_bytes=345288 combine_payload_bytes=267067392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267412680 ingress_bytes=301670400 link_bytes=301670400 token_bytes=534480072 weight_movement_bytes=34603008 total_nvlink_bytes=569083080 weight_to_token_ratio=0.064741 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135161429e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74280 payload_rows_remote=64960 dup_rows=24024 dup_rows_remote=20988 dedup_ratio=0.244385 router_tokens_per_expert=[753, 732, 746, 792, 753, 748, 767, 797, 831, 778, 799, 767, 762, 738, 754, 746, 799, 754, 781, 820, 807, 765, 742, 719, 766, 749, 771, 781, 768, 745, 791, 760, 762, 774, 679, 733, 744, 763, 803, 775, 803, 801, 750, 734, 759, 754, 781, 796, 812, 782, 823, 781, 768, 791, 751, 756, 722, 739, 746, 792, 809, 753, 786, 749] payload_rows_per_dst=[9190, 9308, 9336, 9252, 9196, 9320, 9434, 9244] dup_rows_per_dst=[2986, 3042, 3038, 3010, 2870, 3036, 3094, 2948] combine_rows_per_src=[9190, 9308, 9336, 9252, 9196, 9320, 9434, 9244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74046 payload_rows_remote=64706 dup_rows=24258 dup_rows_remote=21134 dedup_ratio=0.246765 router_tokens_per_expert=[741, 768, 830, 789, 772, 786, 790, 775, 791, 758, 806, 780, 779, 758, 790, 770, 768, 799, 768, 767, 729, 770, 784, 771, 711, 756, 779, 823, 772, 772, 762, 805, 776, 767, 814, 761, 722, 799, 737, 759, 749, 759, 707, 761, 793, 769, 760, 736, 759, 793, 755, 809, 807, 759, 723, 769, 732, 716, 753, 752, 708, 744, 775, 810] payload_rows_per_dst=[9432, 9340, 9318, 9276, 9298, 9108, 9316, 8958] dup_rows_per_dst=[3070, 3124, 2994, 3084, 2972, 2960, 3032, 3022] combine_rows_per_src=[9432, 9340, 9318, 9276, 9298, 9108, 9316, 8958] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267567104 dispatch_meta_bytes=344328 combine_payload_bytes=267567104 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267911432 ingress_bytes=371376128 link_bytes=371376128 token_bytes=535478536 weight_movement_bytes=103809024 total_nvlink_bytes=639287560 weight_to_token_ratio=0.193862 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397458471e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265519104 dispatch_meta_bytes=343712 combine_payload_bytes=265519104 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=265862816 ingress_bytes=300122112 link_bytes=300122112 token_bytes=531381920 weight_movement_bytes=34603008 total_nvlink_bytes=565984928 weight_to_token_ratio=0.065119 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129335346e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74280 payload_rows_remote=65074 dup_rows=24024 dup_rows_remote=21110 dedup_ratio=0.244385 router_tokens_per_expert=[763, 739, 740, 775, 752, 745, 777, 783, 759, 822, 737, 794, 789, 813, 766, 749, 763, 714, 776, 759, 771, 752, 783, 814, 754, 748, 738, 782, 789, 766, 803, 779, 795, 762, 730, 754, 774, 755, 728, 762, 758, 813, 744, 767, 745, 807, 776, 769, 818, 788, 762, 759, 750, 815, 758, 776, 734, 736, 778, 744, 757, 805, 778, 761] payload_rows_per_dst=[9234, 9476, 9230, 9218, 9206, 9316, 9402, 9198] dup_rows_per_dst=[2914, 2982, 3034, 3100, 2914, 3042, 3050, 2988] combine_rows_per_src=[9234, 9476, 9230, 9218, 9206, 9316, 9402, 9198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74436 payload_rows_remote=65116 dup_rows=23868 dup_rows_remote=20816 dedup_ratio=0.242798 router_tokens_per_expert=[729, 761, 765, 768, 787, 755, 808, 753, 799, 820, 801, 807, 730, 736, 739, 773, 820, 813, 775, 775, 760, 786, 776, 769, 783, 798, 747, 769, 732, 737, 736, 731, 731, 814, 778, 762, 807, 755, 777, 781, 764, 730, 755, 773, 734, 794, 778, 740, 758, 759, 773, 781, 762, 766, 765, 788, 728, 751, 770, 774, 772, 766, 731, 797] payload_rows_per_dst=[9320, 9306, 9368, 9132, 9446, 9314, 9290, 9260] dup_rows_per_dst=[3052, 2984, 3180, 2934, 2964, 2822, 3014, 2918] combine_rows_per_src=[9320, 9306, 9368, 9132, 9446, 9314, 9290, 9260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267067392 dispatch_meta_bytes=345288 combine_payload_bytes=267067392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267412680 ingress_bytes=301670400 link_bytes=301670400 token_bytes=534480072 weight_movement_bytes=34603008 total_nvlink_bytes=569083080 weight_to_token_ratio=0.064741 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135161429e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74280 payload_rows_remote=64960 dup_rows=24024 dup_rows_remote=20988 dedup_ratio=0.244385 router_tokens_per_expert=[753, 732, 746, 792, 753, 748, 767, 797, 831, 778, 799, 767, 762, 738, 754, 746, 799, 754, 781, 820, 807, 765, 742, 719, 766, 749, 771, 781, 768, 745, 791, 760, 762, 774, 679, 733, 744, 763, 803, 775, 803, 801, 750, 734, 759, 754, 781, 796, 812, 782, 823, 781, 768, 791, 751, 756, 722, 739, 746, 792, 809, 753, 786, 749] payload_rows_per_dst=[9190, 9308, 9336, 9252, 9196, 9320, 9434, 9244] dup_rows_per_dst=[2986, 3042, 3038, 3010, 2870, 3036, 3094, 2948] combine_rows_per_src=[9190, 9308, 9336, 9252, 9196, 9320, 9434, 9244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265035776 dispatch_meta_bytes=343360 combine_payload_bytes=265035776 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265379136 ingress_bytes=334241792 link_bytes=334241792 token_bytes=530414912 weight_movement_bytes=69206016 total_nvlink_bytes=599620928 weight_to_token_ratio=0.130475 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257724955e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74046 payload_rows_remote=64706 dup_rows=24258 dup_rows_remote=21134 dedup_ratio=0.246765 router_tokens_per_expert=[741, 768, 830, 789, 772, 786, 790, 775, 791, 758, 806, 780, 779, 758, 790, 770, 768, 799, 768, 767, 729, 770, 784, 771, 711, 756, 779, 823, 772, 772, 762, 805, 776, 767, 814, 761, 722, 799, 737, 759, 749, 759, 707, 761, 793, 769, 760, 736, 759, 793, 755, 809, 807, 759, 723, 769, 732, 716, 753, 752, 708, 744, 775, 810] payload_rows_per_dst=[9432, 9340, 9318, 9276, 9298, 9108, 9316, 8958] dup_rows_per_dst=[3070, 3124, 2994, 3084, 2972, 2960, 3032, 3022] combine_rows_per_src=[9432, 9340, 9318, 9276, 9298, 9108, 9316, 8958] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265117696 dispatch_meta_bytes=343560 combine_payload_bytes=265117696 prefetch_bytes=0 reduce_bytes=0 egress_bytes=265461256 ingress_bytes=265117696 link_bytes=265461256 token_bytes=530578952 weight_movement_bytes=0 total_nvlink_bytes=530578952 weight_to_token_ratio=0.000000 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.989093349e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266076160 dispatch_meta_bytes=343792 combine_payload_bytes=266076160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266419952 ingress_bytes=300679168 link_bytes=300679168 token_bytes=532496112 weight_movement_bytes=34603008 total_nvlink_bytes=567099120 weight_to_token_ratio=0.064983 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131431503e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=344736 combine_payload_bytes=266543104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266887840 ingress_bytes=301146112 link_bytes=301146112 token_bytes=533430944 weight_movement_bytes=34603008 total_nvlink_bytes=568033952 weight_to_token_ratio=0.064869 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133188575e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74352 payload_rows_remote=64786 dup_rows=23952 dup_rows_remote=20792 dedup_ratio=0.243652 router_tokens_per_expert=[755, 749, 714, 769, 788, 739, 759, 756, 806, 723, 779, 766, 774, 782, 819, 786, 728, 736, 797, 789, 746, 749, 713, 805, 779, 769, 788, 735, 778, 774, 763, 775, 806, 808, 776, 774, 706, 749, 820, 725, 785, 815, 742, 770, 775, 771, 801, 758, 795, 794, 770, 773, 745, 831, 729, 762, 769, 712, 742, 800, 749, 778, 790, 744] payload_rows_per_dst=[9126, 9460, 9060, 9174, 9310, 9510, 9146, 9566] dup_rows_per_dst=[2932, 3008, 2750, 2924, 3066, 3226, 2886, 3160] combine_rows_per_src=[9126, 9460, 9060, 9174, 9310, 9510, 9146, 9566] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74310 payload_rows_remote=65016 dup_rows=23994 dup_rows_remote=21008 dedup_ratio=0.244080 router_tokens_per_expert=[758, 794, 810, 756, 737, 755, 725, 759, 735, 768, 738, 840, 749, 763, 708, 790, 729, 777, 796, 788, 738, 817, 786, 775, 806, 786, 795, 755, 771, 805, 778, 792, 824, 746, 762, 778, 780, 733, 793, 724, 692, 797, 798, 760, 779, 771, 730, 780, 761, 779, 758, 795, 748, 734, 783, 763, 794, 726, 792, 783, 762, 758, 746, 744] payload_rows_per_dst=[9274, 9232, 9314, 9398, 9294, 9204, 9332, 9262] dup_rows_per_dst=[2914, 2950, 3098, 3178, 2986, 3010, 2910, 2948] combine_rows_per_src=[9274, 9232, 9314, 9398, 9294, 9204, 9332, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266715136 dispatch_meta_bytes=343728 combine_payload_bytes=266715136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267058864 ingress_bytes=301318144 link_bytes=301318144 token_bytes=533774000 weight_movement_bytes=34603008 total_nvlink_bytes=568377008 weight_to_token_ratio=0.064827 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133835918e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266076160 dispatch_meta_bytes=343792 combine_payload_bytes=266076160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266419952 ingress_bytes=300679168 link_bytes=300679168 token_bytes=532496112 weight_movement_bytes=34603008 total_nvlink_bytes=567099120 weight_to_token_ratio=0.064983 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131431503e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74548 payload_rows_remote=65200 dup_rows=23756 dup_rows_remote=20952 dedup_ratio=0.241659 router_tokens_per_expert=[820, 796, 754, 784, 777, 737, 738, 756, 753, 824, 728, 753, 773, 762, 749, 779, 791, 755, 766, 762, 741, 773, 797, 747, 756, 817, 771, 775, 818, 815, 791, 765, 755, 776, 741, 757, 790, 789, 732, 752, 782, 784, 761, 769, 789, 768, 736, 767, 788, 764, 774, 772, 744, 719, 728, 787, 774, 786, 768, 770, 722, 761, 763, 761] payload_rows_per_dst=[9312, 9198, 9310, 9506, 9260, 9318, 9348, 9296] dup_rows_per_dst=[3012, 3044, 2954, 3110, 2924, 2994, 2804, 2914] combine_rows_per_src=[9312, 9198, 9310, 9506, 9260, 9318, 9348, 9296] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265035776 dispatch_meta_bytes=343360 combine_payload_bytes=265035776 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265379136 ingress_bytes=334241792 link_bytes=334241792 token_bytes=530414912 weight_movement_bytes=69206016 total_nvlink_bytes=599620928 weight_to_token_ratio=0.130475 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257724955e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=64810 dup_rows=24048 dup_rows_remote=21056 dedup_ratio=0.244629 router_tokens_per_expert=[776, 776, 731, 763, 743, 799, 766, 743, 810, 795, 781, 753, 718, 794, 782, 761, 786, 752, 769, 771, 768, 724, 770, 755, 755, 755, 812, 806, 758, 768, 763, 802, 733, 794, 766, 770, 759, 748, 781, 787, 781, 750, 776, 751, 785, 764, 816, 734, 841, 761, 784, 805, 787, 783, 770, 765, 739, 784, 739, 733, 730, 776, 720, 735] payload_rows_per_dst=[9254, 9346, 9166, 9446, 9252, 9320, 9496, 8976] dup_rows_per_dst=[2940, 3042, 3024, 2992, 3024, 2994, 3096, 2936] combine_rows_per_src=[9254, 9346, 9166, 9446, 9252, 9320, 9496, 8976] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74420 payload_rows_remote=65086 dup_rows=23884 dup_rows_remote=20794 dedup_ratio=0.242961 router_tokens_per_expert=[763, 808, 823, 745, 755, 803, 825, 775, 701, 705, 763, 745, 737, 744, 754, 789, 743, 780, 830, 778, 756, 753, 790, 782, 795, 703, 804, 776, 744, 807, 720, 768, 761, 787, 748, 759, 812, 764, 760, 761, 756, 744, 812, 726, 804, 770, 778, 730, 700, 771, 741, 785, 752, 743, 738, 796, 784, 775, 805, 753, 802, 798, 795, 778] payload_rows_per_dst=[9550, 9028, 9334, 9376, 9294, 9222, 9090, 9526] dup_rows_per_dst=[3044, 2848, 3090, 2858, 3010, 3018, 2962, 3054] combine_rows_per_src=[9550, 9028, 9334, 9376, 9294, 9222, 9090, 9526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=342312 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265705768 ingress_bytes=299966464 link_bytes=299966464 token_bytes=531069224 weight_movement_bytes=34603008 total_nvlink_bytes=565672232 weight_to_token_ratio=0.065157 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128749655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74548 payload_rows_remote=65200 dup_rows=23756 dup_rows_remote=20952 dedup_ratio=0.241659 router_tokens_per_expert=[820, 796, 754, 784, 777, 737, 738, 756, 753, 824, 728, 753, 773, 762, 749, 779, 791, 755, 766, 762, 741, 773, 797, 747, 756, 817, 771, 775, 818, 815, 791, 765, 755, 776, 741, 757, 790, 789, 732, 752, 782, 784, 761, 769, 789, 768, 736, 767, 788, 764, 774, 772, 744, 719, 728, 787, 774, 786, 768, 770, 722, 761, 763, 761] payload_rows_per_dst=[9312, 9198, 9310, 9506, 9260, 9318, 9348, 9296] dup_rows_per_dst=[3012, 3044, 2954, 3110, 2924, 2994, 2804, 2914] combine_rows_per_src=[9312, 9198, 9310, 9506, 9260, 9318, 9348, 9296] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=344096 combine_payload_bytes=266305536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266649632 ingress_bytes=300908544 link_bytes=300908544 token_bytes=532955168 weight_movement_bytes=34603008 total_nvlink_bytes=567558176 weight_to_token_ratio=0.064927 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132294626e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74420 payload_rows_remote=65086 dup_rows=23884 dup_rows_remote=20794 dedup_ratio=0.242961 router_tokens_per_expert=[763, 808, 823, 745, 755, 803, 825, 775, 701, 705, 763, 745, 737, 744, 754, 789, 743, 780, 830, 778, 756, 753, 790, 782, 795, 703, 804, 776, 744, 807, 720, 768, 761, 787, 748, 759, 812, 764, 760, 761, 756, 744, 812, 726, 804, 770, 778, 730, 700, 771, 741, 785, 752, 743, 738, 796, 784, 775, 805, 753, 802, 798, 795, 778] payload_rows_per_dst=[9550, 9028, 9334, 9376, 9294, 9222, 9090, 9526] dup_rows_per_dst=[3044, 2848, 3090, 2858, 3010, 3018, 2962, 3054] combine_rows_per_src=[9550, 9028, 9334, 9376, 9294, 9222, 9090, 9526] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267059200 dispatch_meta_bytes=344608 combine_payload_bytes=267059200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267403808 ingress_bytes=301662208 link_bytes=301662208 token_bytes=534463008 weight_movement_bytes=34603008 total_nvlink_bytes=569066016 weight_to_token_ratio=0.064744 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135130603e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265461760 dispatch_meta_bytes=343464 combine_payload_bytes=265461760 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265805224 ingress_bytes=369270784 link_bytes=369270784 token_bytes=531266984 weight_movement_bytes=103809024 total_nvlink_bytes=635076008 weight_to_token_ratio=0.195399 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389536231e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74436 payload_rows_remote=65116 dup_rows=23868 dup_rows_remote=20816 dedup_ratio=0.242798 router_tokens_per_expert=[729, 761, 765, 768, 787, 755, 808, 753, 799, 820, 801, 807, 730, 736, 739, 773, 820, 813, 775, 775, 760, 786, 776, 769, 783, 798, 747, 769, 732, 737, 736, 731, 731, 814, 778, 762, 807, 755, 777, 781, 764, 730, 755, 773, 734, 794, 778, 740, 758, 759, 773, 781, 762, 766, 765, 788, 728, 751, 770, 774, 772, 766, 731, 797] payload_rows_per_dst=[9320, 9306, 9368, 9132, 9446, 9314, 9290, 9260] dup_rows_per_dst=[3052, 2984, 3180, 2934, 2964, 2822, 3014, 2918] combine_rows_per_src=[9320, 9306, 9368, 9132, 9446, 9314, 9290, 9260] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74410 payload_rows_remote=65028 dup_rows=23894 dup_rows_remote=20972 dedup_ratio=0.243062 router_tokens_per_expert=[761, 733, 701, 797, 770, 767, 783, 750, 728, 781, 767, 759, 784, 758, 792, 783, 783, 787, 745, 784, 715, 732, 771, 751, 782, 782, 757, 793, 749, 777, 764, 757, 797, 771, 771, 790, 792, 757, 784, 746, 827, 762, 795, 771, 763, 797, 775, 770, 759, 767, 790, 753, 739, 770, 774, 782, 770, 767, 764, 791, 782, 748, 715, 770] payload_rows_per_dst=[9204, 9382, 9250, 9274, 9376, 9404, 9268, 9252] dup_rows_per_dst=[2920, 2922, 2886, 3048, 3040, 3116, 3000, 2962] combine_rows_per_src=[9204, 9382, 9250, 9274, 9376, 9404, 9268, 9252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74544 payload_rows_remote=65214 dup_rows=23760 dup_rows_remote=20768 dedup_ratio=0.241699 router_tokens_per_expert=[748, 769, 742, 780, 767, 760, 780, 742, 788, 760, 768, 776, 743, 753, 769, 802, 738, 787, 763, 756, 799, 770, 743, 760, 784, 773, 781, 767, 737, 756, 765, 812, 769, 770, 789, 784, 726, 796, 738, 789, 759, 801, 797, 797, 768, 809, 756, 782, 780, 766, 772, 796, 755, 758, 736, 761, 737, 750, 777, 762, 766, 782, 722, 764] payload_rows_per_dst=[9224, 9380, 9232, 9374, 9330, 9468, 9374, 9162] dup_rows_per_dst=[2952, 2938, 3000, 2976, 2992, 3070, 2874, 2958] combine_rows_per_src=[9224, 9380, 9232, 9374, 9330, 9468, 9374, 9162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74112 payload_rows_remote=64840 dup_rows=24192 dup_rows_remote=21124 dedup_ratio=0.246094 router_tokens_per_expert=[811, 732, 726, 732, 786, 800, 761, 812, 745, 787, 771, 823, 781, 749, 764, 714, 809, 779, 749, 757, 753, 766, 740, 740, 717, 779, 815, 777, 788, 704, 783, 717, 775, 787, 782, 753, 788, 774, 802, 795, 803, 748, 782, 791, 728, 791, 766, 802, 756, 742, 808, 773, 729, 752, 763, 794, 751, 760, 797, 804, 734, 761, 741, 753] payload_rows_per_dst=[9272, 9512, 9282, 9230, 9130, 9214, 9272, 9200] dup_rows_per_dst=[3068, 3162, 3130, 2930, 2976, 2962, 2962, 3002] combine_rows_per_src=[9272, 9512, 9282, 9230, 9130, 9214, 9272, 9200] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74408 payload_rows_remote=64918 dup_rows=23896 dup_rows_remote=20820 dedup_ratio=0.243083 router_tokens_per_expert=[761, 818, 778, 794, 796, 737, 777, 781, 732, 811, 747, 781, 746, 725, 755, 772, 772, 795, 762, 754, 806, 757, 769, 784, 788, 760, 798, 779, 729, 761, 756, 779, 751, 785, 830, 762, 740, 798, 740, 749, 785, 795, 805, 805, 773, 772, 756, 792, 760, 714, 791, 756, 758, 734, 695, 793, 727, 777, 789, 762, 732, 727, 778, 761] payload_rows_per_dst=[9394, 9218, 9328, 9310, 9344, 9490, 9096, 9228] dup_rows_per_dst=[3090, 2920, 3070, 2990, 2966, 3076, 2906, 2878] combine_rows_per_src=[9394, 9218, 9328, 9310, 9344, 9490, 9096, 9228] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265461760 dispatch_meta_bytes=343464 combine_payload_bytes=265461760 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265805224 ingress_bytes=369270784 link_bytes=369270784 token_bytes=531266984 weight_movement_bytes=103809024 total_nvlink_bytes=635076008 weight_to_token_ratio=0.195399 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389536231e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266592256 dispatch_meta_bytes=343520 combine_payload_bytes=266592256 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266935776 ingress_bytes=370401280 link_bytes=370401280 token_bytes=533528032 weight_movement_bytes=103809024 total_nvlink_bytes=637337056 weight_to_token_ratio=0.194571 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393790197e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267059200 dispatch_meta_bytes=344608 combine_payload_bytes=267059200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267403808 ingress_bytes=301662208 link_bytes=301662208 token_bytes=534463008 weight_movement_bytes=34603008 total_nvlink_bytes=569066016 weight_to_token_ratio=0.064744 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135130603e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266715136 dispatch_meta_bytes=343728 combine_payload_bytes=266715136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267058864 ingress_bytes=301318144 link_bytes=301318144 token_bytes=533774000 weight_movement_bytes=34603008 total_nvlink_bytes=568377008 weight_to_token_ratio=0.064827 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133835918e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74410 payload_rows_remote=65028 dup_rows=23894 dup_rows_remote=20972 dedup_ratio=0.243062 router_tokens_per_expert=[761, 733, 701, 797, 770, 767, 783, 750, 728, 781, 767, 759, 784, 758, 792, 783, 783, 787, 745, 784, 715, 732, 771, 751, 782, 782, 757, 793, 749, 777, 764, 757, 797, 771, 771, 790, 792, 757, 784, 746, 827, 762, 795, 771, 763, 797, 775, 770, 759, 767, 790, 753, 739, 770, 774, 782, 770, 767, 764, 791, 782, 748, 715, 770] payload_rows_per_dst=[9204, 9382, 9250, 9274, 9376, 9404, 9268, 9252] dup_rows_per_dst=[2920, 2922, 2886, 3048, 3040, 3116, 3000, 2962] combine_rows_per_src=[9204, 9382, 9250, 9274, 9376, 9404, 9268, 9252] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=342952 combine_payload_bytes=265904128 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266247080 ingress_bytes=335110144 link_bytes=335110144 token_bytes=532151208 weight_movement_bytes=69206016 total_nvlink_bytes=601357224 weight_to_token_ratio=0.130050 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260992493e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266354688 dispatch_meta_bytes=344000 combine_payload_bytes=266354688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266698688 ingress_bytes=300957696 link_bytes=300957696 token_bytes=533053376 weight_movement_bytes=34603008 total_nvlink_bytes=567656384 weight_to_token_ratio=0.064915 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132479581e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267116544 dispatch_meta_bytes=343928 combine_payload_bytes=267116544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267460472 ingress_bytes=301719552 link_bytes=301719552 token_bytes=534577016 weight_movement_bytes=34603008 total_nvlink_bytes=569180024 weight_to_token_ratio=0.064730 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135346384e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265584640 dispatch_meta_bytes=343856 combine_payload_bytes=265584640 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265928496 ingress_bytes=403996672 link_bytes=403996672 token_bytes=531513136 weight_movement_bytes=138412032 total_nvlink_bytes=669925168 weight_to_token_ratio=0.260411 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520206952e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73996 payload_rows_remote=64426 dup_rows=24308 dup_rows_remote=20874 dedup_ratio=0.247274 router_tokens_per_expert=[749, 780, 793, 787, 760, 774, 776, 778, 765, 783, 733, 794, 745, 772, 774, 779, 764, 747, 766, 780, 759, 753, 790, 732, 758, 743, 788, 752, 766, 798, 774, 747, 788, 763, 792, 752, 768, 778, 789, 794, 805, 748, 767, 766, 741, 790, 755, 773, 749, 764, 758, 742, 795, 767, 839, 715, 755, 749, 752, 764, 827, 730, 761, 757] payload_rows_per_dst=[8812, 9206, 9262, 9302, 9010, 9346, 9488, 9570] dup_rows_per_dst=[2562, 3084, 2920, 3042, 2994, 2944, 3328, 3434] combine_rows_per_src=[8812, 9206, 9262, 9302, 9010, 9346, 9488, 9570] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74156 payload_rows_remote=64748 dup_rows=24148 dup_rows_remote=21168 dedup_ratio=0.245646 router_tokens_per_expert=[789, 794, 762, 799, 794, 807, 744, 786, 796, 725, 777, 770, 797, 730, 770, 738, 748, 790, 826, 751, 755, 756, 727, 746, 855, 768, 787, 815, 694, 743, 798, 770, 762, 781, 765, 778, 796, 775, 743, 794, 811, 706, 735, 766, 803, 747, 784, 704, 750, 735, 755, 726, 823, 761, 774, 766, 787, 799, 780, 734, 761, 776, 708, 760] payload_rows_per_dst=[9384, 9204, 9216, 9382, 9408, 9160, 9162, 9240] dup_rows_per_dst=[3166, 3002, 2982, 3078, 2980, 2952, 3018, 2970] combine_rows_per_src=[9384, 9204, 9216, 9382, 9408, 9160, 9162, 9240] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=342952 combine_payload_bytes=265904128 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266247080 ingress_bytes=335110144 link_bytes=335110144 token_bytes=532151208 weight_movement_bytes=69206016 total_nvlink_bytes=601357224 weight_to_token_ratio=0.130050 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260992493e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74198 payload_rows_remote=64918 dup_rows=24106 dup_rows_remote=21180 dedup_ratio=0.245219 router_tokens_per_expert=[795, 780, 804, 713, 853, 732, 702, 731, 766, 826, 780, 780, 732, 722, 755, 782, 773, 742, 784, 749, 800, 757, 747, 765, 769, 779, 804, 768, 738, 810, 722, 745, 779, 765, 792, 772, 785, 749, 782, 737, 743, 776, 729, 744, 778, 784, 776, 790, 787, 788, 735, 730, 801, 787, 762, 713, 756, 805, 828, 740, 801, 821, 727, 785] payload_rows_per_dst=[9254, 9342, 9256, 9222, 9258, 9250, 9280, 9336] dup_rows_per_dst=[2966, 2944, 2978, 3048, 3064, 2990, 2926, 3190] combine_rows_per_src=[9254, 9342, 9256, 9222, 9258, 9250, 9280, 9336] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266354688 dispatch_meta_bytes=344000 combine_payload_bytes=266354688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266698688 ingress_bytes=300957696 link_bytes=300957696 token_bytes=533053376 weight_movement_bytes=34603008 total_nvlink_bytes=567656384 weight_to_token_ratio=0.064915 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132479581e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=65118 dup_rows=23966 dup_rows_remote=21008 dedup_ratio=0.243795 router_tokens_per_expert=[787, 799, 782, 735, 770, 745, 768, 809, 734, 796, 757, 771, 743, 792, 763, 777, 795, 766, 748, 745, 725, 821, 767, 777, 748, 766, 766, 742, 767, 763, 726, 811, 727, 758, 791, 786, 767, 787, 785, 788, 806, 768, 769, 773, 773, 767, 759, 757, 747, 766, 751, 799, 767, 776, 817, 779, 783, 735, 747, 760, 771, 778, 707, 747] payload_rows_per_dst=[9244, 9330, 9312, 9220, 9298, 9362, 9426, 9146] dup_rows_per_dst=[3146, 2936, 2976, 2958, 3080, 2982, 2978, 2910] combine_rows_per_src=[9244, 9330, 9312, 9220, 9298, 9362, 9426, 9146] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=64788 dup_rows=24136 dup_rows_remote=21162 dedup_ratio=0.245524 router_tokens_per_expert=[760, 781, 796, 832, 809, 786, 753, 799, 765, 809, 704, 753, 795, 777, 750, 744, 772, 746, 751, 780, 782, 777, 785, 784, 783, 777, 743, 800, 771, 779, 780, 757, 786, 746, 766, 762, 775, 756, 823, 772, 760, 731, 773, 735, 793, 745, 723, 804, 747, 755, 767, 744, 756, 776, 748, 777, 681, 721, 761, 787, 776, 769, 777, 780] payload_rows_per_dst=[9400, 9242, 9380, 9312, 9326, 9228, 9196, 9084] dup_rows_per_dst=[3232, 2952, 2974, 3068, 3046, 2900, 2944, 3020] combine_rows_per_src=[9400, 9242, 9380, 9312, 9326, 9228, 9196, 9084] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=263888896 dispatch_meta_bytes=341200 combine_payload_bytes=263888896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264230096 ingress_bytes=298491904 link_bytes=298491904 token_bytes=528118992 weight_movement_bytes=34603008 total_nvlink_bytes=562722000 weight_to_token_ratio=0.065521 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123201004e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74198 payload_rows_remote=64918 dup_rows=24106 dup_rows_remote=21180 dedup_ratio=0.245219 router_tokens_per_expert=[795, 780, 804, 713, 853, 732, 702, 731, 766, 826, 780, 780, 732, 722, 755, 782, 773, 742, 784, 749, 800, 757, 747, 765, 769, 779, 804, 768, 738, 810, 722, 745, 779, 765, 792, 772, 785, 749, 782, 737, 743, 776, 729, 744, 778, 784, 776, 790, 787, 788, 735, 730, 801, 787, 762, 713, 756, 805, 828, 740, 801, 821, 727, 785] payload_rows_per_dst=[9254, 9342, 9256, 9222, 9258, 9250, 9280, 9336] dup_rows_per_dst=[2966, 2944, 2978, 3048, 3064, 2990, 2926, 3190] combine_rows_per_src=[9254, 9342, 9256, 9222, 9258, 9250, 9280, 9336] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=64788 dup_rows=24136 dup_rows_remote=21162 dedup_ratio=0.245524 router_tokens_per_expert=[760, 781, 796, 832, 809, 786, 753, 799, 765, 809, 704, 753, 795, 777, 750, 744, 772, 746, 751, 780, 782, 777, 785, 784, 783, 777, 743, 800, 771, 779, 780, 757, 786, 746, 766, 762, 775, 756, 823, 772, 760, 731, 773, 735, 793, 745, 723, 804, 747, 755, 767, 744, 756, 776, 748, 777, 681, 721, 761, 787, 776, 769, 777, 780] payload_rows_per_dst=[9400, 9242, 9380, 9312, 9326, 9228, 9196, 9084] dup_rows_per_dst=[3232, 2952, 2974, 3068, 3046, 2900, 2944, 3020] combine_rows_per_src=[9400, 9242, 9380, 9312, 9326, 9228, 9196, 9084] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=344392 combine_payload_bytes=265904128 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266248520 ingress_bytes=335110144 link_bytes=335110144 token_bytes=532152648 weight_movement_bytes=69206016 total_nvlink_bytes=601358664 weight_to_token_ratio=0.130049 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260992493e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74480 payload_rows_remote=65136 dup_rows=23824 dup_rows_remote=20802 dedup_ratio=0.242350 router_tokens_per_expert=[751, 717, 739, 779, 760, 819, 765, 737, 762, 773, 773, 748, 740, 783, 787, 817, 761, 722, 756, 818, 802, 749, 760, 757, 754, 806, 759, 777, 762, 761, 797, 775, 761, 756, 804, 736, 743, 742, 809, 753, 742, 740, 792, 803, 766, 808, 770, 760, 755, 803, 739, 758, 765, 720, 816, 747, 779, 772, 748, 807, 741, 782, 782, 787] payload_rows_per_dst=[9208, 9344, 9330, 9316, 9324, 9404, 9232, 9322] dup_rows_per_dst=[2926, 3022, 2920, 3066, 2884, 2958, 2974, 3074] combine_rows_per_src=[9208, 9344, 9330, 9316, 9324, 9404, 9232, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266723328 dispatch_meta_bytes=344504 combine_payload_bytes=266723328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267067832 ingress_bytes=301326336 link_bytes=301326336 token_bytes=533791160 weight_movement_bytes=34603008 total_nvlink_bytes=568394168 weight_to_token_ratio=0.064825 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133866744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74310 payload_rows_remote=65016 dup_rows=23994 dup_rows_remote=21008 dedup_ratio=0.244080 router_tokens_per_expert=[758, 794, 810, 756, 737, 755, 725, 759, 735, 768, 738, 840, 749, 763, 708, 790, 729, 777, 796, 788, 738, 817, 786, 775, 806, 786, 795, 755, 771, 805, 778, 792, 824, 746, 762, 778, 780, 733, 793, 724, 692, 797, 798, 760, 779, 771, 730, 780, 761, 779, 758, 795, 748, 734, 783, 763, 794, 726, 792, 783, 762, 758, 746, 744] payload_rows_per_dst=[9274, 9232, 9314, 9398, 9294, 9204, 9332, 9262] dup_rows_per_dst=[2914, 2950, 3098, 3178, 2986, 3010, 2910, 2948] combine_rows_per_src=[9274, 9232, 9314, 9398, 9294, 9204, 9332, 9262] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265207808 dispatch_meta_bytes=343664 combine_payload_bytes=265207808 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265551472 ingress_bytes=369016832 link_bytes=369016832 token_bytes=530759280 weight_movement_bytes=103809024 total_nvlink_bytes=634568304 weight_to_token_ratio=0.195586 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388580631e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74112 payload_rows_remote=64840 dup_rows=24192 dup_rows_remote=21124 dedup_ratio=0.246094 router_tokens_per_expert=[811, 732, 726, 732, 786, 800, 761, 812, 745, 787, 771, 823, 781, 749, 764, 714, 809, 779, 749, 757, 753, 766, 740, 740, 717, 779, 815, 777, 788, 704, 783, 717, 775, 787, 782, 753, 788, 774, 802, 795, 803, 748, 782, 791, 728, 791, 766, 802, 756, 742, 808, 773, 729, 752, 763, 794, 751, 760, 797, 804, 734, 761, 741, 753] payload_rows_per_dst=[9272, 9512, 9282, 9230, 9130, 9214, 9272, 9200] dup_rows_per_dst=[3068, 3162, 3130, 2930, 2976, 2962, 2962, 3002] combine_rows_per_src=[9272, 9512, 9282, 9230, 9130, 9214, 9272, 9200] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74088 payload_rows_remote=64546 dup_rows=24216 dup_rows_remote=20904 dedup_ratio=0.246338 router_tokens_per_expert=[776, 747, 779, 757, 773, 772, 756, 752, 783, 748, 813, 768, 759, 752, 764, 797, 822, 765, 765, 791, 804, 782, 729, 799, 757, 723, 808, 740, 743, 795, 764, 780, 784, 735, 794, 764, 778, 788, 738, 772, 710, 704, 766, 796, 808, 770, 750, 789, 712, 750, 773, 779, 783, 758, 760, 791, 796, 792, 776, 772, 728, 753, 747, 773] payload_rows_per_dst=[9542, 9136, 9460, 9044, 9288, 9174, 9314, 9130] dup_rows_per_dst=[3312, 2804, 3228, 2804, 3018, 3008, 3042, 3000] combine_rows_per_src=[9542, 9136, 9460, 9044, 9288, 9174, 9314, 9130] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74508 payload_rows_remote=65258 dup_rows=23796 dup_rows_remote=20846 dedup_ratio=0.242065 router_tokens_per_expert=[810, 768, 752, 753, 716, 814, 772, 763, 793, 785, 768, 792, 759, 790, 753, 802, 779, 721, 748, 757, 731, 808, 779, 754, 727, 736, 783, 755, 783, 766, 773, 749, 789, 728, 803, 751, 777, 778, 767, 777, 782, 756, 809, 761, 761, 762, 703, 766, 775, 772, 737, 789, 778, 782, 778, 754, 791, 748, 770, 808, 777, 753, 776, 755] payload_rows_per_dst=[9342, 9418, 9210, 9188, 9386, 9250, 9316, 9398] dup_rows_per_dst=[2954, 3066, 2944, 2956, 2954, 2950, 3014, 2958] combine_rows_per_src=[9342, 9418, 9210, 9188, 9386, 9250, 9316, 9398] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266723328 dispatch_meta_bytes=344504 combine_payload_bytes=266723328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267067832 ingress_bytes=301326336 link_bytes=301326336 token_bytes=533791160 weight_movement_bytes=34603008 total_nvlink_bytes=568394168 weight_to_token_ratio=0.064825 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133866744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265371648 dispatch_meta_bytes=343800 combine_payload_bytes=265371648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265715448 ingress_bytes=299974656 link_bytes=299974656 token_bytes=531087096 weight_movement_bytes=34603008 total_nvlink_bytes=565690104 weight_to_token_ratio=0.065155 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128780481e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=344392 combine_payload_bytes=265904128 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266248520 ingress_bytes=335110144 link_bytes=335110144 token_bytes=532152648 weight_movement_bytes=69206016 total_nvlink_bytes=601358664 weight_to_token_ratio=0.130049 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260992493e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265584640 dispatch_meta_bytes=343856 combine_payload_bytes=265584640 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265928496 ingress_bytes=403996672 link_bytes=403996672 token_bytes=531513136 weight_movement_bytes=138412032 total_nvlink_bytes=669925168 weight_to_token_ratio=0.260411 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520206952e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267296768 dispatch_meta_bytes=344416 combine_payload_bytes=267296768 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267641184 ingress_bytes=301899776 link_bytes=301899776 token_bytes=534937952 weight_movement_bytes=34603008 total_nvlink_bytes=569540960 weight_to_token_ratio=0.064686 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136024552e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74480 payload_rows_remote=65136 dup_rows=23824 dup_rows_remote=20802 dedup_ratio=0.242350 router_tokens_per_expert=[751, 717, 739, 779, 760, 819, 765, 737, 762, 773, 773, 748, 740, 783, 787, 817, 761, 722, 756, 818, 802, 749, 760, 757, 754, 806, 759, 777, 762, 761, 797, 775, 761, 756, 804, 736, 743, 742, 809, 753, 742, 740, 792, 803, 766, 808, 770, 760, 755, 803, 739, 758, 765, 720, 816, 747, 779, 772, 748, 807, 741, 782, 782, 787] payload_rows_per_dst=[9208, 9344, 9330, 9316, 9324, 9404, 9232, 9322] dup_rows_per_dst=[2926, 3022, 2920, 3066, 2884, 2958, 2974, 3074] combine_rows_per_src=[9208, 9344, 9330, 9316, 9324, 9404, 9232, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266797056 dispatch_meta_bytes=343752 combine_payload_bytes=266797056 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267140808 ingress_bytes=336003072 link_bytes=336003072 token_bytes=533937864 weight_movement_bytes=69206016 total_nvlink_bytes=603143880 weight_to_token_ratio=0.129614 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264352509e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=344096 combine_payload_bytes=266305536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266649632 ingress_bytes=300908544 link_bytes=300908544 token_bytes=532955168 weight_movement_bytes=34603008 total_nvlink_bytes=567558176 weight_to_token_ratio=0.064927 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132294626e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=65104 dup_rows=23946 dup_rows_remote=21016 dedup_ratio=0.243591 router_tokens_per_expert=[774, 812, 778, 762, 765, 793, 770, 820, 775, 754, 737, 768, 807, 763, 790, 787, 780, 799, 793, 808, 771, 773, 732, 744, 770, 778, 776, 726, 749, 734, 813, 773, 727, 770, 815, 769, 744, 753, 802, 766, 755, 741, 775, 744, 748, 751, 734, 783, 777, 748, 807, 749, 718, 844, 732, 757, 749, 789, 750, 785, 741, 754, 712, 789] payload_rows_per_dst=[9522, 9260, 9422, 9190, 9216, 9240, 9254, 9254] dup_rows_per_dst=[3026, 3102, 2978, 3038, 3040, 2822, 3010, 2930] combine_rows_per_src=[9522, 9260, 9422, 9190, 9216, 9240, 9254, 9254] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=64796 dup_rows=24010 dup_rows_remote=20938 dedup_ratio=0.244242 router_tokens_per_expert=[758, 780, 755, 731, 802, 738, 748, 769, 775, 753, 803, 736, 771, 773, 781, 736, 758, 714, 716, 803, 764, 786, 794, 766, 799, 784, 767, 731, 763, 793, 734, 789, 809, 763, 799, 777, 763, 782, 799, 793, 717, 789, 747, 794, 755, 744, 788, 751, 759, 739, 761, 834, 727, 763, 769, 835, 762, 799, 799, 785, 780, 744, 717, 739] payload_rows_per_dst=[9218, 9280, 9292, 9290, 9498, 9216, 9262, 9238] dup_rows_per_dst=[2944, 2976, 2910, 3030, 3072, 2954, 3112, 3012] combine_rows_per_src=[9218, 9280, 9292, 9290, 9498, 9216, 9262, 9238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=264380416 dispatch_meta_bytes=341800 combine_payload_bytes=264380416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264722216 ingress_bytes=298983424 link_bytes=298983424 token_bytes=529102632 weight_movement_bytes=34603008 total_nvlink_bytes=563705640 weight_to_token_ratio=0.065399 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125050554e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267296768 dispatch_meta_bytes=344416 combine_payload_bytes=267296768 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267641184 ingress_bytes=301899776 link_bytes=301899776 token_bytes=534937952 weight_movement_bytes=34603008 total_nvlink_bytes=569540960 weight_to_token_ratio=0.064686 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136024552e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266797056 dispatch_meta_bytes=343752 combine_payload_bytes=266797056 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267140808 ingress_bytes=336003072 link_bytes=336003072 token_bytes=533937864 weight_movement_bytes=69206016 total_nvlink_bytes=603143880 weight_to_token_ratio=0.129614 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264352509e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74298 payload_rows_remote=65026 dup_rows=24006 dup_rows_remote=21018 dedup_ratio=0.244202 router_tokens_per_expert=[761, 762, 744, 778, 784, 804, 829, 743, 746, 765, 780, 766, 776, 738, 788, 757, 816, 776, 765, 757, 733, 802, 778, 741, 772, 762, 731, 714, 792, 801, 750, 758, 763, 700, 762, 796, 772, 749, 832, 791, 726, 719, 774, 728, 712, 812, 788, 825, 809, 785, 735, 736, 787, 782, 746, 750, 767, 756, 789, 782, 756, 797, 794, 763] payload_rows_per_dst=[9382, 9248, 9308, 9212, 9272, 9176, 9272, 9428] dup_rows_per_dst=[3028, 2984, 3028, 2948, 3058, 2992, 2988, 2980] combine_rows_per_src=[9382, 9248, 9308, 9212, 9272, 9176, 9272, 9428] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=64886 dup_rows=24136 dup_rows_remote=21076 dedup_ratio=0.245524 router_tokens_per_expert=[787, 742, 797, 818, 726, 775, 724, 766, 756, 834, 773, 751, 738, 758, 766, 737, 772, 714, 742, 777, 765, 760, 779, 759, 797, 747, 795, 765, 762, 808, 738, 759, 736, 773, 779, 758, 773, 823, 762, 749, 790, 757, 791, 764, 768, 789, 801, 785, 795, 755, 720, 711, 739, 789, 740, 819, 755, 775, 776, 802, 735, 771, 785, 800] payload_rows_per_dst=[9264, 9206, 9258, 9282, 9214, 9380, 9230, 9334] dup_rows_per_dst=[3006, 3020, 2878, 3060, 3092, 3110, 2906, 3064] combine_rows_per_src=[9264, 9206, 9258, 9282, 9214, 9380, 9230, 9334] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=65036 dup_rows=24032 dup_rows_remote=21034 dedup_ratio=0.244466 router_tokens_per_expert=[739, 734, 783, 744, 784, 780, 776, 724, 763, 776, 724, 798, 801, 813, 772, 754, 764, 751, 791, 722, 774, 785, 780, 750, 797, 813, 810, 766, 789, 773, 733, 774, 761, 743, 783, 783, 704, 766, 771, 752, 756, 742, 832, 793, 755, 720, 746, 777, 743, 753, 783, 758, 783, 809, 800, 749, 746, 809, 758, 842, 751, 759, 768, 720] payload_rows_per_dst=[9204, 9376, 9236, 9350, 9190, 9292, 9272, 9352] dup_rows_per_dst=[2924, 3026, 2998, 3160, 2936, 2950, 3084, 2954] combine_rows_per_src=[9204, 9376, 9236, 9350, 9190, 9292, 9272, 9352] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266665984 dispatch_meta_bytes=344480 combine_payload_bytes=266665984 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267010464 ingress_bytes=335872000 link_bytes=335872000 token_bytes=533676448 weight_movement_bytes=69206016 total_nvlink_bytes=602882464 weight_to_token_ratio=0.129678 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263859296e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74298 payload_rows_remote=65026 dup_rows=24006 dup_rows_remote=21018 dedup_ratio=0.244202 router_tokens_per_expert=[761, 762, 744, 778, 784, 804, 829, 743, 746, 765, 780, 766, 776, 738, 788, 757, 816, 776, 765, 757, 733, 802, 778, 741, 772, 762, 731, 714, 792, 801, 750, 758, 763, 700, 762, 796, 772, 749, 832, 791, 726, 719, 774, 728, 712, 812, 788, 825, 809, 785, 735, 736, 787, 782, 746, 750, 767, 756, 789, 782, 756, 797, 794, 763] payload_rows_per_dst=[9382, 9248, 9308, 9212, 9272, 9176, 9272, 9428] dup_rows_per_dst=[3028, 2984, 3028, 2948, 3058, 2992, 2988, 2980] combine_rows_per_src=[9382, 9248, 9308, 9212, 9272, 9176, 9272, 9428] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74088 payload_rows_remote=64546 dup_rows=24216 dup_rows_remote=20904 dedup_ratio=0.246338 router_tokens_per_expert=[776, 747, 779, 757, 773, 772, 756, 752, 783, 748, 813, 768, 759, 752, 764, 797, 822, 765, 765, 791, 804, 782, 729, 799, 757, 723, 808, 740, 743, 795, 764, 780, 784, 735, 794, 764, 778, 788, 738, 772, 710, 704, 766, 796, 808, 770, 750, 789, 712, 750, 773, 779, 783, 758, 760, 791, 796, 792, 776, 772, 728, 753, 747, 773] payload_rows_per_dst=[9542, 9136, 9460, 9044, 9288, 9174, 9314, 9130] dup_rows_per_dst=[3312, 2804, 3228, 2804, 3018, 3008, 3042, 3000] combine_rows_per_src=[9542, 9136, 9460, 9044, 9288, 9174, 9314, 9130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=65036 dup_rows=24032 dup_rows_remote=21034 dedup_ratio=0.244466 router_tokens_per_expert=[739, 734, 783, 744, 784, 780, 776, 724, 763, 776, 724, 798, 801, 813, 772, 754, 764, 751, 791, 722, 774, 785, 780, 750, 797, 813, 810, 766, 789, 773, 733, 774, 761, 743, 783, 783, 704, 766, 771, 752, 756, 742, 832, 793, 755, 720, 746, 777, 743, 753, 783, 758, 783, 809, 800, 749, 746, 809, 758, 842, 751, 759, 768, 720] payload_rows_per_dst=[9204, 9376, 9236, 9350, 9190, 9292, 9272, 9352] dup_rows_per_dst=[2924, 3026, 2998, 3160, 2936, 2950, 3084, 2954] combine_rows_per_src=[9204, 9376, 9236, 9350, 9190, 9292, 9272, 9352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=65160 dup_rows=23950 dup_rows_remote=21104 dedup_ratio=0.243632 router_tokens_per_expert=[766, 757, 742, 769, 823, 724, 740, 812, 803, 709, 790, 737, 771, 718, 727, 765, 750, 810, 803, 778, 784, 738, 748, 796, 748, 758, 810, 808, 791, 741, 767, 725, 736, 756, 747, 747, 764, 841, 780, 753, 821, 800, 754, 761, 768, 769, 766, 785, 764, 802, 781, 828, 735, 762, 763, 774, 759, 790, 744, 761, 776, 782, 718, 757] payload_rows_per_dst=[9238, 9194, 9334, 9276, 9280, 9398, 9410, 9224] dup_rows_per_dst=[3028, 2846, 3080, 3020, 2968, 3050, 3008, 2950] combine_rows_per_src=[9238, 9194, 9334, 9276, 9280, 9398, 9410, 9224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266346496 dispatch_meta_bytes=344176 combine_payload_bytes=266346496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266690672 ingress_bytes=300949504 link_bytes=300949504 token_bytes=533037168 weight_movement_bytes=34603008 total_nvlink_bytes=567640176 weight_to_token_ratio=0.064917 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132448755e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=343848 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266116904 ingress_bytes=404185088 link_bytes=404185088 token_bytes=531889960 weight_movement_bytes=138412032 total_nvlink_bytes=670301992 weight_to_token_ratio=0.260227 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520915947e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74156 payload_rows_remote=64748 dup_rows=24148 dup_rows_remote=21168 dedup_ratio=0.245646 router_tokens_per_expert=[789, 794, 762, 799, 794, 807, 744, 786, 796, 725, 777, 770, 797, 730, 770, 738, 748, 790, 826, 751, 755, 756, 727, 746, 855, 768, 787, 815, 694, 743, 798, 770, 762, 781, 765, 778, 796, 775, 743, 794, 811, 706, 735, 766, 803, 747, 784, 704, 750, 735, 755, 726, 823, 761, 774, 766, 787, 799, 780, 734, 761, 776, 708, 760] payload_rows_per_dst=[9384, 9204, 9216, 9382, 9408, 9160, 9162, 9240] dup_rows_per_dst=[3166, 3002, 2982, 3078, 2980, 2952, 3018, 2970] combine_rows_per_src=[9384, 9204, 9216, 9382, 9408, 9160, 9162, 9240] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74160 payload_rows_remote=64594 dup_rows=24144 dup_rows_remote=21050 dedup_ratio=0.245605 router_tokens_per_expert=[837, 766, 797, 780, 750, 748, 794, 778, 723, 763, 799, 774, 788, 779, 786, 751, 737, 755, 791, 765, 766, 813, 792, 716, 804, 745, 758, 769, 789, 732, 775, 742, 760, 745, 849, 775, 778, 751, 776, 757, 716, 804, 761, 759, 784, 748, 745, 751, 735, 737, 783, 728, 770, 781, 728, 749, 743, 787, 808, 776, 787, 758, 788, 773] payload_rows_per_dst=[9566, 9452, 9268, 9208, 9688, 8742, 8858, 9378] dup_rows_per_dst=[3094, 3068, 3110, 3092, 3376, 2644, 2698, 3062] combine_rows_per_src=[9566, 9452, 9268, 9208, 9688, 8742, 8858, 9378] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265404416 dispatch_meta_bytes=342936 combine_payload_bytes=265404416 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265747352 ingress_bytes=334610432 link_bytes=334610432 token_bytes=531151768 weight_movement_bytes=69206016 total_nvlink_bytes=600357784 weight_to_token_ratio=0.130294 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259112117e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74128 payload_rows_remote=64818 dup_rows=24176 dup_rows_remote=21108 dedup_ratio=0.245931 router_tokens_per_expert=[792, 732, 776, 766, 764, 773, 772, 737, 769, 763, 796, 794, 734, 770, 755, 772, 792, 783, 778, 732, 777, 786, 783, 755, 767, 802, 732, 784, 736, 752, 801, 774, 783, 769, 796, 763, 727, 765, 790, 759, 791, 744, 800, 812, 760, 787, 744, 751, 768, 795, 719, 809, 786, 759, 806, 778, 737, 731, 781, 722, 740, 747, 778, 756] payload_rows_per_dst=[9238, 9306, 9332, 9292, 9186, 9310, 9314, 9150] dup_rows_per_dst=[2986, 3000, 3040, 3004, 3118, 3068, 3126, 2834] combine_rows_per_src=[9238, 9306, 9332, 9292, 9186, 9310, 9314, 9150] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=343848 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266116904 ingress_bytes=404185088 link_bytes=404185088 token_bytes=531889960 weight_movement_bytes=138412032 total_nvlink_bytes=670301992 weight_to_token_ratio=0.260227 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520915947e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266387456 dispatch_meta_bytes=344280 combine_payload_bytes=266387456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266731736 ingress_bytes=300990464 link_bytes=300990464 token_bytes=533119192 weight_movement_bytes=34603008 total_nvlink_bytes=567722200 weight_to_token_ratio=0.064907 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132602884e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266346496 dispatch_meta_bytes=344176 combine_payload_bytes=266346496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266690672 ingress_bytes=300949504 link_bytes=300949504 token_bytes=533037168 weight_movement_bytes=34603008 total_nvlink_bytes=567640176 weight_to_token_ratio=0.064917 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132448755e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=264380416 dispatch_meta_bytes=341800 combine_payload_bytes=264380416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264722216 ingress_bytes=298983424 link_bytes=298983424 token_bytes=529102632 weight_movement_bytes=34603008 total_nvlink_bytes=563705640 weight_to_token_ratio=0.065399 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125050554e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=65160 dup_rows=23950 dup_rows_remote=21104 dedup_ratio=0.243632 router_tokens_per_expert=[766, 757, 742, 769, 823, 724, 740, 812, 803, 709, 790, 737, 771, 718, 727, 765, 750, 810, 803, 778, 784, 738, 748, 796, 748, 758, 810, 808, 791, 741, 767, 725, 736, 756, 747, 747, 764, 841, 780, 753, 821, 800, 754, 761, 768, 769, 766, 785, 764, 802, 781, 828, 735, 762, 763, 774, 759, 790, 744, 761, 776, 782, 718, 757] payload_rows_per_dst=[9238, 9194, 9334, 9276, 9280, 9398, 9410, 9224] dup_rows_per_dst=[3028, 2846, 3080, 3020, 2968, 3050, 3008, 2950] combine_rows_per_src=[9238, 9194, 9334, 9276, 9280, 9398, 9410, 9224] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265494528 dispatch_meta_bytes=343704 combine_payload_bytes=265494528 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265838232 ingress_bytes=369303552 link_bytes=369303552 token_bytes=531332760 weight_movement_bytes=103809024 total_nvlink_bytes=635141784 weight_to_token_ratio=0.195375 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389659535e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74176 payload_rows_remote=64588 dup_rows=24128 dup_rows_remote=20816 dedup_ratio=0.245443 router_tokens_per_expert=[743, 808, 791, 790, 776, 791, 770, 757, 726, 818, 746, 716, 748, 755, 751, 772, 757, 768, 730, 772, 778, 771, 826, 789, 791, 768, 762, 759, 775, 759, 777, 757, 782, 810, 747, 745, 764, 785, 708, 774, 763, 763, 760, 775, 790, 782, 767, 798, 780, 748, 744, 754, 769, 802, 770, 766, 796, 751, 763, 785, 767, 762, 718, 767] payload_rows_per_dst=[9362, 9264, 8980, 9248, 9122, 9300, 9312, 9588] dup_rows_per_dst=[3090, 2800, 2720, 3048, 3108, 3096, 2954, 3312] combine_rows_per_src=[9362, 9264, 8980, 9248, 9122, 9300, 9312, 9588] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266895360 dispatch_meta_bytes=345056 combine_payload_bytes=266895360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267240416 ingress_bytes=301498368 link_bytes=301498368 token_bytes=534135776 weight_movement_bytes=34603008 total_nvlink_bytes=568738784 weight_to_token_ratio=0.064783 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134514086e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74450 payload_rows_remote=65208 dup_rows=23854 dup_rows_remote=20904 dedup_ratio=0.242655 router_tokens_per_expert=[785, 731, 775, 780, 714, 803, 757, 803, 719, 771, 771, 790, 711, 781, 779, 748, 698, 790, 790, 734, 791, 770, 776, 714, 781, 770, 747, 772, 782, 749, 783, 831, 754, 776, 777, 781, 729, 747, 758, 774, 781, 786, 761, 789, 728, 786, 788, 804, 783, 772, 773, 782, 777, 771, 799, 760, 766, 778, 782, 761, 782, 808, 688, 755] payload_rows_per_dst=[9376, 9268, 9170, 9374, 9242, 9396, 9346, 9278] dup_rows_per_dst=[2920, 2872, 2956, 3056, 2950, 3050, 3088, 2962] combine_rows_per_src=[9376, 9268, 9170, 9374, 9242, 9396, 9346, 9278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74176 payload_rows_remote=64588 dup_rows=24128 dup_rows_remote=20816 dedup_ratio=0.245443 router_tokens_per_expert=[743, 808, 791, 790, 776, 791, 770, 757, 726, 818, 746, 716, 748, 755, 751, 772, 757, 768, 730, 772, 778, 771, 826, 789, 791, 768, 762, 759, 775, 759, 777, 757, 782, 810, 747, 745, 764, 785, 708, 774, 763, 763, 760, 775, 790, 782, 767, 798, 780, 748, 744, 754, 769, 802, 770, 766, 796, 751, 763, 785, 767, 762, 718, 767] payload_rows_per_dst=[9362, 9264, 8980, 9248, 9122, 9300, 9312, 9588] dup_rows_per_dst=[3090, 2800, 2720, 3048, 3108, 3096, 2954, 3312] combine_rows_per_src=[9362, 9264, 8980, 9248, 9122, 9300, 9312, 9588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265207808 dispatch_meta_bytes=343664 combine_payload_bytes=265207808 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265551472 ingress_bytes=369016832 link_bytes=369016832 token_bytes=530759280 weight_movement_bytes=103809024 total_nvlink_bytes=634568304 weight_to_token_ratio=0.195586 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388580631e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=264577024 dispatch_meta_bytes=342576 combine_payload_bytes=264577024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264919600 ingress_bytes=299180032 link_bytes=299180032 token_bytes=529496624 weight_movement_bytes=34603008 total_nvlink_bytes=564099632 weight_to_token_ratio=0.065351 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125790374e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265494528 dispatch_meta_bytes=343704 combine_payload_bytes=265494528 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265838232 ingress_bytes=369303552 link_bytes=369303552 token_bytes=531332760 weight_movement_bytes=103809024 total_nvlink_bytes=635141784 weight_to_token_ratio=0.195375 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389659535e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=64998 dup_rows=24016 dup_rows_remote=21026 dedup_ratio=0.244303 router_tokens_per_expert=[763, 746, 766, 785, 745, 784, 747, 742, 836, 763, 799, 739, 775, 723, 793, 793, 771, 778, 754, 762, 785, 791, 796, 816, 781, 799, 787, 734, 796, 749, 745, 749, 753, 747, 765, 704, 783, 763, 733, 765, 778, 763, 723, 791, 786, 761, 744, 786, 769, 794, 754, 754, 767, 798, 787, 788, 721, 776, 705, 750, 784, 796, 760, 812] payload_rows_per_dst=[9132, 9382, 9420, 9290, 9070, 9328, 9376, 9290] dup_rows_per_dst=[3024, 3060, 3086, 2990, 2956, 2936, 3046, 2918] combine_rows_per_src=[9132, 9382, 9420, 9290, 9070, 9328, 9376, 9290] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74252 payload_rows_remote=65068 dup_rows=24052 dup_rows_remote=20990 dedup_ratio=0.244670 router_tokens_per_expert=[759, 748, 771, 773, 762, 708, 791, 760, 774, 770, 782, 745, 782, 817, 735, 770, 727, 805, 783, 788, 791, 772, 713, 744, 809, 796, 748, 771, 770, 731, 779, 766, 775, 740, 756, 767, 740, 779, 750, 744, 768, 824, 813, 806, 782, 749, 779, 755, 747, 770, 778, 754, 813, 779, 783, 765, 724, 820, 779, 790, 749, 766, 744, 724] payload_rows_per_dst=[9242, 9366, 9184, 9344, 9160, 9410, 9398, 9148] dup_rows_per_dst=[2902, 2984, 3062, 2996, 2942, 3142, 2980, 3044] combine_rows_per_src=[9242, 9366, 9184, 9344, 9160, 9410, 9398, 9148] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=264552448 dispatch_meta_bytes=341616 combine_payload_bytes=264552448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264894064 ingress_bytes=299155456 link_bytes=299155456 token_bytes=529446512 weight_movement_bytes=34603008 total_nvlink_bytes=564049520 weight_to_token_ratio=0.065357 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125697897e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74160 payload_rows_remote=64594 dup_rows=24144 dup_rows_remote=21050 dedup_ratio=0.245605 router_tokens_per_expert=[837, 766, 797, 780, 750, 748, 794, 778, 723, 763, 799, 774, 788, 779, 786, 751, 737, 755, 791, 765, 766, 813, 792, 716, 804, 745, 758, 769, 789, 732, 775, 742, 760, 745, 849, 775, 778, 751, 776, 757, 716, 804, 761, 759, 784, 748, 745, 751, 735, 737, 783, 728, 770, 781, 728, 749, 743, 787, 808, 776, 787, 758, 788, 773] payload_rows_per_dst=[9566, 9452, 9268, 9208, 9688, 8742, 8858, 9378] dup_rows_per_dst=[3094, 3068, 3110, 3092, 3376, 2644, 2698, 3062] combine_rows_per_src=[9566, 9452, 9268, 9208, 9688, 8742, 8858, 9378] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74194 payload_rows_remote=64974 dup_rows=24110 dup_rows_remote=21048 dedup_ratio=0.245260 router_tokens_per_expert=[746, 786, 742, 711, 731, 735, 743, 762, 799, 786, 802, 743, 754, 749, 760, 741, 780, 734, 776, 768, 825, 749, 729, 759, 755, 808, 752, 796, 773, 782, 786, 813, 770, 767, 768, 793, 769, 737, 733, 759, 800, 761, 790, 762, 807, 814, 745, 819, 758, 762, 801, 823, 746, 713, 759, 779, 753, 786, 783, 784, 741, 761, 764, 770] payload_rows_per_dst=[9056, 9202, 9262, 9440, 9258, 9460, 9220, 9296] dup_rows_per_dst=[2856, 3066, 2978, 3090, 2934, 3136, 3062, 2988] combine_rows_per_src=[9056, 9202, 9262, 9440, 9258, 9460, 9220, 9296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=264552448 dispatch_meta_bytes=341616 combine_payload_bytes=264552448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264894064 ingress_bytes=299155456 link_bytes=299155456 token_bytes=529446512 weight_movement_bytes=34603008 total_nvlink_bytes=564049520 weight_to_token_ratio=0.065357 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125697897e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=64998 dup_rows=24016 dup_rows_remote=21026 dedup_ratio=0.244303 router_tokens_per_expert=[763, 746, 766, 785, 745, 784, 747, 742, 836, 763, 799, 739, 775, 723, 793, 793, 771, 778, 754, 762, 785, 791, 796, 816, 781, 799, 787, 734, 796, 749, 745, 749, 753, 747, 765, 704, 783, 763, 733, 765, 778, 763, 723, 791, 786, 761, 744, 786, 769, 794, 754, 754, 767, 798, 787, 788, 721, 776, 705, 750, 784, 796, 760, 812] payload_rows_per_dst=[9132, 9382, 9420, 9290, 9070, 9328, 9376, 9290] dup_rows_per_dst=[3024, 3060, 3086, 2990, 2956, 2936, 3046, 2918] combine_rows_per_src=[9132, 9382, 9420, 9290, 9070, 9328, 9376, 9290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=65044 dup_rows=24072 dup_rows_remote=21100 dedup_ratio=0.244873 router_tokens_per_expert=[739, 737, 804, 804, 787, 785, 764, 776, 736, 744, 756, 765, 785, 736, 772, 786, 746, 780, 731, 747, 748, 742, 761, 800, 753, 747, 779, 758, 771, 737, 776, 772, 737, 784, 739, 804, 763, 751, 789, 794, 760, 782, 773, 783, 788, 766, 811, 770, 754, 801, 770, 793, 740, 758, 752, 788, 769, 824, 799, 771, 752, 773, 775, 715] payload_rows_per_dst=[9406, 9188, 9288, 9198, 9320, 9228, 9252, 9352] dup_rows_per_dst=[2986, 2972, 2822, 2988, 3002, 3238, 3060, 3004] combine_rows_per_src=[9406, 9188, 9288, 9198, 9320, 9228, 9252, 9352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=344088 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266477592 ingress_bytes=300736512 link_bytes=300736512 token_bytes=532611096 weight_movement_bytes=34603008 total_nvlink_bytes=567214104 weight_to_token_ratio=0.064969 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131647283e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74252 payload_rows_remote=65068 dup_rows=24052 dup_rows_remote=20990 dedup_ratio=0.244670 router_tokens_per_expert=[759, 748, 771, 773, 762, 708, 791, 760, 774, 770, 782, 745, 782, 817, 735, 770, 727, 805, 783, 788, 791, 772, 713, 744, 809, 796, 748, 771, 770, 731, 779, 766, 775, 740, 756, 767, 740, 779, 750, 744, 768, 824, 813, 806, 782, 749, 779, 755, 747, 770, 778, 754, 813, 779, 783, 765, 724, 820, 779, 790, 749, 766, 744, 724] payload_rows_per_dst=[9242, 9366, 9184, 9344, 9160, 9410, 9398, 9148] dup_rows_per_dst=[2902, 2984, 3062, 2996, 2942, 3142, 2980, 3044] combine_rows_per_src=[9242, 9366, 9184, 9344, 9160, 9410, 9398, 9148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=64796 dup_rows=24010 dup_rows_remote=20938 dedup_ratio=0.244242 router_tokens_per_expert=[758, 780, 755, 731, 802, 738, 748, 769, 775, 753, 803, 736, 771, 773, 781, 736, 758, 714, 716, 803, 764, 786, 794, 766, 799, 784, 767, 731, 763, 793, 734, 789, 809, 763, 799, 777, 763, 782, 799, 793, 717, 789, 747, 794, 755, 744, 788, 751, 759, 739, 761, 834, 727, 763, 769, 835, 762, 799, 799, 785, 780, 744, 717, 739] payload_rows_per_dst=[9218, 9280, 9292, 9290, 9498, 9216, 9262, 9238] dup_rows_per_dst=[2944, 2976, 2910, 3030, 3072, 2954, 3112, 3012] combine_rows_per_src=[9218, 9280, 9292, 9290, 9498, 9216, 9262, 9238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=65018 dup_rows=23948 dup_rows_remote=20910 dedup_ratio=0.243612 router_tokens_per_expert=[779, 804, 721, 839, 775, 698, 747, 752, 762, 741, 799, 760, 744, 746, 785, 773, 810, 772, 770, 766, 812, 753, 768, 762, 710, 753, 756, 753, 808, 805, 762, 802, 748, 762, 773, 784, 800, 808, 773, 788, 734, 749, 714, 778, 798, 757, 766, 736, 793, 794, 784, 740, 798, 757, 778, 729, 774, 794, 768, 739, 781, 768, 750, 750] payload_rows_per_dst=[9338, 9260, 9460, 9264, 9388, 9142, 9302, 9202] dup_rows_per_dst=[3038, 2972, 3066, 2982, 2926, 2922, 3044, 2998] combine_rows_per_src=[9338, 9260, 9460, 9264, 9388, 9142, 9302, 9202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266231808 dispatch_meta_bytes=344096 combine_payload_bytes=266231808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266575904 ingress_bytes=300834816 link_bytes=300834816 token_bytes=532807712 weight_movement_bytes=34603008 total_nvlink_bytes=567410720 weight_to_token_ratio=0.064945 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132017193e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74524 payload_rows_remote=65308 dup_rows=23780 dup_rows_remote=20958 dedup_ratio=0.241903 router_tokens_per_expert=[795, 774, 750, 767, 787, 810, 774, 759, 804, 743, 740, 838, 744, 795, 813, 778, 729, 781, 767, 794, 755, 805, 761, 779, 776, 790, 754, 791, 773, 757, 727, 728, 824, 776, 795, 789, 737, 721, 766, 789, 752, 723, 731, 789, 759, 739, 777, 749, 791, 710, 751, 760, 743, 748, 772, 751, 783, 775, 797, 762, 775, 738, 746, 796] payload_rows_per_dst=[9378, 9360, 9428, 9252, 9418, 9216, 9166, 9306] dup_rows_per_dst=[3054, 3150, 2914, 2940, 2976, 2822, 2886, 3038] combine_rows_per_src=[9378, 9360, 9428, 9252, 9418, 9216, 9166, 9306] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267091968 dispatch_meta_bytes=344448 combine_payload_bytes=267091968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267436416 ingress_bytes=301694976 link_bytes=301694976 token_bytes=534528384 weight_movement_bytes=34603008 total_nvlink_bytes=569131392 weight_to_token_ratio=0.064736 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135253906e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266231808 dispatch_meta_bytes=344096 combine_payload_bytes=266231808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266575904 ingress_bytes=300834816 link_bytes=300834816 token_bytes=532807712 weight_movement_bytes=34603008 total_nvlink_bytes=567410720 weight_to_token_ratio=0.064945 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132017193e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74084 payload_rows_remote=64884 dup_rows=24220 dup_rows_remote=21122 dedup_ratio=0.246379 router_tokens_per_expert=[751, 719, 754, 716, 792, 731, 778, 799, 821, 762, 752, 785, 769, 785, 787, 784, 776, 784, 756, 781, 815, 728, 816, 732, 769, 739, 760, 757, 734, 827, 809, 736, 799, 749, 754, 804, 785, 813, 713, 746, 790, 788, 719, 794, 775, 765, 731, 767, 755, 775, 757, 736, 769, 786, 778, 756, 777, 740, 781, 778, 758, 759, 820, 731] payload_rows_per_dst=[9164, 9308, 9336, 9160, 9292, 9348, 9276, 9200] dup_rows_per_dst=[2916, 2944, 3040, 2928, 3024, 3148, 3122, 3098] combine_rows_per_src=[9164, 9308, 9336, 9160, 9292, 9348, 9276, 9200] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=344576 combine_payload_bytes=266420224 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266764800 ingress_bytes=335626240 link_bytes=335626240 token_bytes=533185024 weight_movement_bytes=69206016 total_nvlink_bytes=602391040 weight_to_token_ratio=0.129797 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262934521e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266518528 dispatch_meta_bytes=344232 combine_payload_bytes=266518528 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266862760 ingress_bytes=335724544 link_bytes=335724544 token_bytes=533381288 weight_movement_bytes=69206016 total_nvlink_bytes=602587304 weight_to_token_ratio=0.129750 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263304431e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=264577024 dispatch_meta_bytes=342576 combine_payload_bytes=264577024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264919600 ingress_bytes=299180032 link_bytes=299180032 token_bytes=529496624 weight_movement_bytes=34603008 total_nvlink_bytes=564099632 weight_to_token_ratio=0.065351 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125790374e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=344088 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266477592 ingress_bytes=300736512 link_bytes=300736512 token_bytes=532611096 weight_movement_bytes=34603008 total_nvlink_bytes=567214104 weight_to_token_ratio=0.064969 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131647283e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=65044 dup_rows=24072 dup_rows_remote=21100 dedup_ratio=0.244873 router_tokens_per_expert=[739, 737, 804, 804, 787, 785, 764, 776, 736, 744, 756, 765, 785, 736, 772, 786, 746, 780, 731, 747, 748, 742, 761, 800, 753, 747, 779, 758, 771, 737, 776, 772, 737, 784, 739, 804, 763, 751, 789, 794, 760, 782, 773, 783, 788, 766, 811, 770, 754, 801, 770, 793, 740, 758, 752, 788, 769, 824, 799, 771, 752, 773, 775, 715] payload_rows_per_dst=[9406, 9188, 9288, 9198, 9320, 9228, 9252, 9352] dup_rows_per_dst=[2986, 2972, 2822, 2988, 3002, 3238, 3060, 3004] combine_rows_per_src=[9406, 9188, 9288, 9198, 9320, 9228, 9252, 9352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265404416 dispatch_meta_bytes=342936 combine_payload_bytes=265404416 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265747352 ingress_bytes=334610432 link_bytes=334610432 token_bytes=531151768 weight_movement_bytes=69206016 total_nvlink_bytes=600357784 weight_to_token_ratio=0.130294 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259112117e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=343712 combine_payload_bytes=266313728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266657440 ingress_bytes=300916736 link_bytes=300916736 token_bytes=532971168 weight_movement_bytes=34603008 total_nvlink_bytes=567574176 weight_to_token_ratio=0.064925 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132325452e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=267501568 dispatch_meta_bytes=345064 combine_payload_bytes=267501568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267846632 ingress_bytes=302104576 link_bytes=302104576 token_bytes=535348200 weight_movement_bytes=34603008 total_nvlink_bytes=569951208 weight_to_token_ratio=0.064636 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136795198e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74084 payload_rows_remote=64884 dup_rows=24220 dup_rows_remote=21122 dedup_ratio=0.246379 router_tokens_per_expert=[751, 719, 754, 716, 792, 731, 778, 799, 821, 762, 752, 785, 769, 785, 787, 784, 776, 784, 756, 781, 815, 728, 816, 732, 769, 739, 760, 757, 734, 827, 809, 736, 799, 749, 754, 804, 785, 813, 713, 746, 790, 788, 719, 794, 775, 765, 731, 767, 755, 775, 757, 736, 769, 786, 778, 756, 777, 740, 781, 778, 758, 759, 820, 731] payload_rows_per_dst=[9164, 9308, 9336, 9160, 9292, 9348, 9276, 9200] dup_rows_per_dst=[2916, 2944, 3040, 2928, 3024, 3148, 3122, 3098] combine_rows_per_src=[9164, 9308, 9336, 9160, 9292, 9348, 9276, 9200] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=344576 combine_payload_bytes=266420224 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266764800 ingress_bytes=335626240 link_bytes=335626240 token_bytes=533185024 weight_movement_bytes=69206016 total_nvlink_bytes=602391040 weight_to_token_ratio=0.129797 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262934521e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=65120 dup_rows=23928 dup_rows_remote=20924 dedup_ratio=0.243408 router_tokens_per_expert=[840, 766, 754, 699, 777, 794, 825, 794, 783, 790, 786, 725, 748, 793, 750, 804, 734, 754, 717, 793, 769, 769, 766, 734, 760, 760, 771, 763, 784, 765, 771, 792, 762, 743, 707, 830, 737, 792, 791, 768, 763, 716, 787, 734, 774, 831, 794, 765, 766, 770, 798, 762, 743, 792, 762, 766, 796, 761, 731, 742, 726, 748, 793, 772] payload_rows_per_dst=[9344, 9402, 9206, 9326, 9256, 9316, 9414, 9112] dup_rows_per_dst=[3154, 2956, 2866, 3006, 3004, 3012, 2904, 3026] combine_rows_per_src=[9344, 9402, 9206, 9326, 9256, 9316, 9414, 9112] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74428 payload_rows_remote=64998 dup_rows=23876 dup_rows_remote=20798 dedup_ratio=0.242879 router_tokens_per_expert=[669, 778, 762, 809, 782, 779, 734, 776, 742, 803, 720, 781, 763, 769, 750, 770, 738, 773, 741, 815, 791, 781, 744, 794, 740, 799, 778, 751, 839, 758, 781, 808, 776, 725, 780, 783, 780, 762, 772, 711, 775, 734, 761, 733, 751, 772, 765, 771, 729, 767, 775, 728, 781, 765, 786, 805, 746, 803, 717, 809, 739, 783, 847, 803] payload_rows_per_dst=[9220, 9294, 9418, 9430, 9192, 9192, 9278, 9404] dup_rows_per_dst=[2958, 2902, 2936, 3078, 2986, 2932, 2994, 3090] combine_rows_per_src=[9220, 9294, 9418, 9430, 9192, 9192, 9278, 9404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=64918 dup_rows=24008 dup_rows_remote=20996 dedup_ratio=0.244222 router_tokens_per_expert=[749, 756, 746, 736, 799, 757, 743, 767, 774, 842, 743, 757, 755, 770, 799, 781, 763, 798, 744, 803, 758, 844, 787, 784, 816, 736, 751, 747, 776, 717, 727, 763, 781, 766, 764, 765, 741, 746, 721, 785, 774, 796, 778, 785, 765, 777, 766, 776, 804, 745, 795, 774, 789, 740, 781, 767, 747, 773, 747, 749, 751, 769, 751, 796] payload_rows_per_dst=[9176, 9280, 9432, 9214, 9170, 9384, 9378, 9262] dup_rows_per_dst=[2930, 3162, 3130, 2852, 2968, 3050, 3012, 2904] combine_rows_per_src=[9176, 9280, 9432, 9214, 9170, 9384, 9378, 9262] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265764864 dispatch_meta_bytes=344024 combine_payload_bytes=265764864 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266108888 ingress_bytes=334970880 link_bytes=334970880 token_bytes=531873752 weight_movement_bytes=69206016 total_nvlink_bytes=601079768 weight_to_token_ratio=0.130117 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260468454e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64984 dup_rows=24112 dup_rows_remote=21112 dedup_ratio=0.245280 router_tokens_per_expert=[782, 723, 788, 767, 761, 732, 770, 730, 774, 791, 801, 738, 767, 823, 754, 768, 758, 798, 759, 760, 798, 766, 734, 731, 752, 758, 757, 826, 790, 754, 766, 824, 769, 809, 758, 773, 757, 785, 750, 751, 760, 775, 780, 745, 751, 728, 788, 790, 774, 786, 788, 771, 775, 781, 741, 713, 770, 784, 755, 771, 795, 778, 737, 764] payload_rows_per_dst=[9178, 9392, 9208, 9386, 9330, 9224, 9242, 9232] dup_rows_per_dst=[2928, 3040, 3000, 3068, 2974, 3010, 3016, 3076] combine_rows_per_src=[9178, 9392, 9208, 9386, 9330, 9224, 9242, 9232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=65018 dup_rows=23948 dup_rows_remote=20910 dedup_ratio=0.243612 router_tokens_per_expert=[779, 804, 721, 839, 775, 698, 747, 752, 762, 741, 799, 760, 744, 746, 785, 773, 810, 772, 770, 766, 812, 753, 768, 762, 710, 753, 756, 753, 808, 805, 762, 802, 748, 762, 773, 784, 800, 808, 773, 788, 734, 749, 714, 778, 798, 757, 766, 736, 793, 794, 784, 740, 798, 757, 778, 729, 774, 794, 768, 739, 781, 768, 750, 750] payload_rows_per_dst=[9338, 9260, 9460, 9264, 9388, 9142, 9302, 9202] dup_rows_per_dst=[3038, 2972, 3066, 2982, 2926, 2922, 3044, 2998] combine_rows_per_src=[9338, 9260, 9460, 9264, 9388, 9142, 9302, 9202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74428 payload_rows_remote=64998 dup_rows=23876 dup_rows_remote=20798 dedup_ratio=0.242879 router_tokens_per_expert=[669, 778, 762, 809, 782, 779, 734, 776, 742, 803, 720, 781, 763, 769, 750, 770, 738, 773, 741, 815, 791, 781, 744, 794, 740, 799, 778, 751, 839, 758, 781, 808, 776, 725, 780, 783, 780, 762, 772, 711, 775, 734, 761, 733, 751, 772, 765, 771, 729, 767, 775, 728, 781, 765, 786, 805, 746, 803, 717, 809, 739, 783, 847, 803] payload_rows_per_dst=[9220, 9294, 9418, 9430, 9192, 9192, 9278, 9404] dup_rows_per_dst=[2958, 2902, 2936, 3078, 2986, 2932, 2994, 3090] combine_rows_per_src=[9220, 9294, 9418, 9430, 9192, 9192, 9278, 9404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=64918 dup_rows=24008 dup_rows_remote=20996 dedup_ratio=0.244222 router_tokens_per_expert=[749, 756, 746, 736, 799, 757, 743, 767, 774, 842, 743, 757, 755, 770, 799, 781, 763, 798, 744, 803, 758, 844, 787, 784, 816, 736, 751, 747, 776, 717, 727, 763, 781, 766, 764, 765, 741, 746, 721, 785, 774, 796, 778, 785, 765, 777, 766, 776, 804, 745, 795, 774, 789, 740, 781, 767, 747, 773, 747, 749, 751, 769, 751, 796] payload_rows_per_dst=[9176, 9280, 9432, 9214, 9170, 9384, 9378, 9262] dup_rows_per_dst=[2930, 3162, 3130, 2852, 2968, 3050, 3012, 2904] combine_rows_per_src=[9176, 9280, 9432, 9214, 9170, 9384, 9378, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=343656 combine_payload_bytes=265904128 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266247784 ingress_bytes=404316160 link_bytes=404316160 token_bytes=532151912 weight_movement_bytes=138412032 total_nvlink_bytes=670563944 weight_to_token_ratio=0.260099 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521409160e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64984 dup_rows=24112 dup_rows_remote=21112 dedup_ratio=0.245280 router_tokens_per_expert=[782, 723, 788, 767, 761, 732, 770, 730, 774, 791, 801, 738, 767, 823, 754, 768, 758, 798, 759, 760, 798, 766, 734, 731, 752, 758, 757, 826, 790, 754, 766, 824, 769, 809, 758, 773, 757, 785, 750, 751, 760, 775, 780, 745, 751, 728, 788, 790, 774, 786, 788, 771, 775, 781, 741, 713, 770, 784, 755, 771, 795, 778, 737, 764] payload_rows_per_dst=[9178, 9392, 9208, 9386, 9330, 9224, 9242, 9232] dup_rows_per_dst=[2928, 3040, 3000, 3068, 2974, 3010, 3016, 3076] combine_rows_per_src=[9178, 9392, 9208, 9386, 9330, 9224, 9242, 9232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64888 dup_rows=24068 dup_rows_remote=20982 dedup_ratio=0.244832 router_tokens_per_expert=[763, 773, 738, 759, 806, 791, 818, 767, 772, 784, 752, 745, 808, 791, 806, 759, 763, 768, 783, 790, 776, 735, 809, 786, 762, 743, 763, 779, 741, 781, 792, 761, 717, 793, 757, 766, 814, 757, 764, 787, 779, 766, 727, 771, 723, 781, 725, 731, 775, 749, 796, 775, 768, 750, 753, 744, 714, 803, 747, 760, 794, 790, 766, 746] payload_rows_per_dst=[9404, 9348, 9298, 9196, 9278, 9158, 9274, 9280] dup_rows_per_dst=[3026, 3086, 3122, 3048, 3032, 2848, 2946, 2960] combine_rows_per_src=[9404, 9348, 9298, 9196, 9278, 9158, 9274, 9280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266231808 dispatch_meta_bytes=343184 combine_payload_bytes=266231808 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266574992 ingress_bytes=370040832 link_bytes=370040832 token_bytes=532806800 weight_movement_bytes=103809024 total_nvlink_bytes=636615824 weight_to_token_ratio=0.194834 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392433860e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74450 payload_rows_remote=65208 dup_rows=23854 dup_rows_remote=20904 dedup_ratio=0.242655 router_tokens_per_expert=[785, 731, 775, 780, 714, 803, 757, 803, 719, 771, 771, 790, 711, 781, 779, 748, 698, 790, 790, 734, 791, 770, 776, 714, 781, 770, 747, 772, 782, 749, 783, 831, 754, 776, 777, 781, 729, 747, 758, 774, 781, 786, 761, 789, 728, 786, 788, 804, 783, 772, 773, 782, 777, 771, 799, 760, 766, 778, 782, 761, 782, 808, 688, 755] payload_rows_per_dst=[9376, 9268, 9170, 9374, 9242, 9396, 9346, 9278] dup_rows_per_dst=[2920, 2872, 2956, 3056, 2950, 3050, 3088, 2962] combine_rows_per_src=[9376, 9268, 9170, 9374, 9242, 9396, 9346, 9278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74230 payload_rows_remote=65054 dup_rows=24074 dup_rows_remote=21060 dedup_ratio=0.244893 router_tokens_per_expert=[789, 780, 745, 765, 719, 762, 787, 736, 791, 772, 789, 801, 726, 781, 781, 805, 778, 728, 763, 762, 758, 726, 773, 786, 789, 799, 748, 749, 782, 742, 752, 813, 740, 788, 824, 769, 760, 751, 740, 791, 779, 779, 737, 739, 757, 765, 835, 794, 822, 780, 762, 788, 740, 744, 777, 726, 767, 779, 782, 786, 746, 714, 729, 785] payload_rows_per_dst=[9176, 9442, 9318, 9380, 9236, 9300, 9184, 9194] dup_rows_per_dst=[3014, 3066, 3168, 3224, 2834, 3070, 2716, 2982] combine_rows_per_src=[9176, 9442, 9318, 9380, 9236, 9300, 9184, 9194] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74334 payload_rows_remote=65096 dup_rows=23970 dup_rows_remote=20950 dedup_ratio=0.243835 router_tokens_per_expert=[803, 742, 764, 773, 766, 761, 741, 788, 767, 740, 742, 730, 830, 831, 821, 764, 747, 778, 794, 806, 794, 795, 714, 772, 748, 775, 738, 784, 720, 732, 819, 799, 769, 804, 782, 807, 717, 749, 784, 780, 749, 767, 759, 786, 783, 752, 784, 749, 773, 780, 753, 730, 764, 743, 770, 792, 745, 772, 719, 772, 747, 754, 766, 773] payload_rows_per_dst=[9308, 9358, 9350, 9238, 9434, 9238, 9238, 9170] dup_rows_per_dst=[2968, 3092, 3050, 2992, 2950, 3020, 2972, 2926] combine_rows_per_src=[9308, 9358, 9350, 9238, 9434, 9238, 9238, 9170] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=344176 combine_payload_bytes=266731520 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267075696 ingress_bytes=370540544 link_bytes=370540544 token_bytes=533807216 weight_movement_bytes=103809024 total_nvlink_bytes=637616240 weight_to_token_ratio=0.194469 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394314236e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266231808 dispatch_meta_bytes=343184 combine_payload_bytes=266231808 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266574992 ingress_bytes=370040832 link_bytes=370040832 token_bytes=532806800 weight_movement_bytes=103809024 total_nvlink_bytes=636615824 weight_to_token_ratio=0.194834 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392433860e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266174464 dispatch_meta_bytes=344384 combine_payload_bytes=266174464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266518848 ingress_bytes=300777472 link_bytes=300777472 token_bytes=532693312 weight_movement_bytes=34603008 total_nvlink_bytes=567296320 weight_to_token_ratio=0.064959 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131801413e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=343712 combine_payload_bytes=266313728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266657440 ingress_bytes=300916736 link_bytes=300916736 token_bytes=532971168 weight_movement_bytes=34603008 total_nvlink_bytes=567574176 weight_to_token_ratio=0.064925 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132325452e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64888 dup_rows=24068 dup_rows_remote=20982 dedup_ratio=0.244832 router_tokens_per_expert=[763, 773, 738, 759, 806, 791, 818, 767, 772, 784, 752, 745, 808, 791, 806, 759, 763, 768, 783, 790, 776, 735, 809, 786, 762, 743, 763, 779, 741, 781, 792, 761, 717, 793, 757, 766, 814, 757, 764, 787, 779, 766, 727, 771, 723, 781, 725, 731, 775, 749, 796, 775, 768, 750, 753, 744, 714, 803, 747, 760, 794, 790, 766, 746] payload_rows_per_dst=[9404, 9348, 9298, 9196, 9278, 9158, 9274, 9280] dup_rows_per_dst=[3026, 3086, 3122, 3048, 3032, 2848, 2946, 2960] combine_rows_per_src=[9404, 9348, 9298, 9196, 9278, 9158, 9274, 9280] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=343656 combine_payload_bytes=265904128 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266247784 ingress_bytes=404316160 link_bytes=404316160 token_bytes=532151912 weight_movement_bytes=138412032 total_nvlink_bytes=670563944 weight_to_token_ratio=0.260099 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521409160e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266633216 dispatch_meta_bytes=344184 combine_payload_bytes=266633216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266977400 ingress_bytes=301236224 link_bytes=301236224 token_bytes=533610616 weight_movement_bytes=34603008 total_nvlink_bytes=568213624 weight_to_token_ratio=0.064847 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133527659e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=64576 dup_rows=23950 dup_rows_remote=20666 dedup_ratio=0.243632 router_tokens_per_expert=[741, 780, 733, 792, 711, 782, 777, 790, 745, 735, 775, 770, 777, 799, 786, 835, 749, 747, 767, 754, 744, 727, 750, 747, 771, 752, 776, 758, 791, 780, 763, 778, 774, 769, 773, 734, 774, 760, 792, 802, 762, 760, 762, 787, 691, 773, 712, 775, 807, 795, 783, 763, 785, 779, 791, 796, 781, 765, 812, 781, 738, 769, 768, 757] payload_rows_per_dst=[9188, 8998, 9132, 9318, 9150, 9324, 9466, 9778] dup_rows_per_dst=[2926, 2726, 2838, 3020, 2882, 3142, 3132, 3284] combine_rows_per_src=[9188, 8998, 9132, 9318, 9150, 9324, 9466, 9778] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266174464 dispatch_meta_bytes=344384 combine_payload_bytes=266174464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266518848 ingress_bytes=300777472 link_bytes=300777472 token_bytes=532693312 weight_movement_bytes=34603008 total_nvlink_bytes=567296320 weight_to_token_ratio=0.064959 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131801413e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267091968 dispatch_meta_bytes=344448 combine_payload_bytes=267091968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267436416 ingress_bytes=301694976 link_bytes=301694976 token_bytes=534528384 weight_movement_bytes=34603008 total_nvlink_bytes=569131392 weight_to_token_ratio=0.064736 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135253906e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265781248 dispatch_meta_bytes=343480 combine_payload_bytes=265781248 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266124728 ingress_bytes=334987264 link_bytes=334987264 token_bytes=531905976 weight_movement_bytes=69206016 total_nvlink_bytes=601111992 weight_to_token_ratio=0.130109 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260530106e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=344456 combine_payload_bytes=266461184 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266805640 ingress_bytes=335667200 link_bytes=335667200 token_bytes=533266824 weight_movement_bytes=69206016 total_nvlink_bytes=602472840 weight_to_token_ratio=0.129777 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263088650e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266633216 dispatch_meta_bytes=344184 combine_payload_bytes=266633216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266977400 ingress_bytes=301236224 link_bytes=301236224 token_bytes=533610616 weight_movement_bytes=34603008 total_nvlink_bytes=568213624 weight_to_token_ratio=0.064847 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133527659e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74450 payload_rows_remote=65048 dup_rows=23854 dup_rows_remote=20720 dedup_ratio=0.242655 router_tokens_per_expert=[758, 734, 761, 771, 768, 756, 750, 765, 733, 719, 788, 771, 761, 782, 723, 758, 777, 819, 755, 754, 711, 761, 731, 768, 775, 788, 834, 729, 745, 760, 750, 792, 776, 785, 764, 753, 815, 792, 809, 774, 798, 775, 761, 762, 804, 769, 749, 789, 794, 765, 773, 770, 811, 759, 759, 752, 756, 791, 729, 798, 784, 746, 744, 799] payload_rows_per_dst=[9196, 9256, 9300, 9396, 9402, 9322, 9312, 9266] dup_rows_per_dst=[2930, 2814, 2852, 2950, 3134, 3092, 3054, 3028] combine_rows_per_src=[9196, 9256, 9300, 9396, 9402, 9322, 9312, 9266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=64904 dup_rows=23998 dup_rows_remote=21052 dedup_ratio=0.244120 router_tokens_per_expert=[732, 773, 741, 773, 798, 790, 788, 726, 755, 743, 746, 738, 760, 738, 793, 809, 792, 749, 783, 747, 760, 748, 776, 803, 781, 717, 789, 782, 763, 751, 780, 786, 799, 736, 719, 802, 777, 797, 719, 759, 798, 795, 817, 787, 794, 749, 768, 769, 712, 812, 766, 818, 773, 788, 754, 751, 738, 743, 769, 752, 789, 771, 758, 763] payload_rows_per_dst=[9234, 9262, 9326, 9358, 9202, 9350, 9402, 9172] dup_rows_per_dst=[3008, 2902, 2990, 2940, 3014, 3204, 2946, 2994] combine_rows_per_src=[9234, 9262, 9326, 9358, 9202, 9350, 9402, 9172] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=264503296 dispatch_meta_bytes=340968 combine_payload_bytes=264503296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264844264 ingress_bytes=299106304 link_bytes=299106304 token_bytes=529347560 weight_movement_bytes=34603008 total_nvlink_bytes=563950568 weight_to_token_ratio=0.065369 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125512942e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74488 payload_rows_remote=65158 dup_rows=23816 dup_rows_remote=20960 dedup_ratio=0.242269 router_tokens_per_expert=[749, 769, 777, 733, 786, 785, 803, 805, 775, 703, 810, 788, 813, 782, 770, 779, 769, 776, 754, 765, 760, 789, 778, 774, 763, 748, 757, 758, 778, 759, 767, 763, 795, 829, 787, 787, 772, 767, 750, 751, 738, 705, 770, 772, 794, 779, 758, 790, 743, 762, 759, 772, 763, 805, 761, 737, 727, 741, 740, 734, 770, 831, 749, 729] payload_rows_per_dst=[9350, 9328, 9322, 9330, 9418, 9268, 9294, 9178] dup_rows_per_dst=[3064, 3112, 3008, 2856, 3058, 2944, 2910, 2864] combine_rows_per_src=[9350, 9328, 9322, 9330, 9418, 9268, 9294, 9178] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=64954 dup_rows=23944 dup_rows_remote=20968 dedup_ratio=0.243571 router_tokens_per_expert=[822, 819, 773, 785, 801, 786, 701, 787, 740, 777, 718, 746, 805, 785, 795, 740, 777, 773, 758, 734, 776, 787, 769, 817, 731, 764, 782, 767, 764, 731, 760, 794, 748, 773, 790, 774, 757, 802, 729, 777, 770, 759, 764, 753, 787, 781, 778, 737, 721, 820, 778, 740, 770, 767, 765, 769, 705, 753, 803, 748, 796, 781, 761, 732] payload_rows_per_dst=[9432, 9176, 9406, 9256, 9428, 9152, 9284, 9226] dup_rows_per_dst=[3116, 3036, 2976, 2930, 2872, 3106, 2976, 2932] combine_rows_per_src=[9432, 9176, 9406, 9256, 9428, 9152, 9284, 9226] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74230 payload_rows_remote=65054 dup_rows=24074 dup_rows_remote=21060 dedup_ratio=0.244893 router_tokens_per_expert=[789, 780, 745, 765, 719, 762, 787, 736, 791, 772, 789, 801, 726, 781, 781, 805, 778, 728, 763, 762, 758, 726, 773, 786, 789, 799, 748, 749, 782, 742, 752, 813, 740, 788, 824, 769, 760, 751, 740, 791, 779, 779, 737, 739, 757, 765, 835, 794, 822, 780, 762, 788, 740, 744, 777, 726, 767, 779, 782, 786, 746, 714, 729, 785] payload_rows_per_dst=[9176, 9442, 9318, 9380, 9236, 9300, 9184, 9194] dup_rows_per_dst=[3014, 3066, 3168, 3224, 2834, 3070, 2716, 2982] combine_rows_per_src=[9176, 9442, 9318, 9380, 9236, 9300, 9184, 9194] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=64904 dup_rows=23998 dup_rows_remote=21052 dedup_ratio=0.244120 router_tokens_per_expert=[732, 773, 741, 773, 798, 790, 788, 726, 755, 743, 746, 738, 760, 738, 793, 809, 792, 749, 783, 747, 760, 748, 776, 803, 781, 717, 789, 782, 763, 751, 780, 786, 799, 736, 719, 802, 777, 797, 719, 759, 798, 795, 817, 787, 794, 749, 768, 769, 712, 812, 766, 818, 773, 788, 754, 751, 738, 743, 769, 752, 789, 771, 758, 763] payload_rows_per_dst=[9234, 9262, 9326, 9358, 9202, 9350, 9402, 9172] dup_rows_per_dst=[3008, 2902, 2990, 2940, 3014, 3204, 2946, 2994] combine_rows_per_src=[9234, 9262, 9326, 9358, 9202, 9350, 9402, 9172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265846784 dispatch_meta_bytes=343824 combine_payload_bytes=265846784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266190608 ingress_bytes=300449792 link_bytes=300449792 token_bytes=532037392 weight_movement_bytes=34603008 total_nvlink_bytes=566640400 weight_to_token_ratio=0.065039 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130568379e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266436608 dispatch_meta_bytes=343072 combine_payload_bytes=266436608 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266779680 ingress_bytes=370245632 link_bytes=370245632 token_bytes=533216288 weight_movement_bytes=103809024 total_nvlink_bytes=637025312 weight_to_token_ratio=0.194685 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393204506e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=64872 dup_rows=24046 dup_rows_remote=21038 dedup_ratio=0.244609 router_tokens_per_expert=[778, 779, 763, 784, 747, 773, 731, 757, 744, 764, 762, 763, 738, 843, 737, 764, 741, 743, 751, 790, 758, 763, 770, 754, 751, 774, 797, 779, 767, 731, 783, 781, 786, 771, 787, 778, 805, 736, 789, 779, 777, 779, 799, 784, 790, 758, 732, 778, 729, 780, 771, 785, 734, 745, 761, 766, 786, 785, 755, 828, 727, 770, 765, 777] payload_rows_per_dst=[9236, 9288, 9158, 9284, 9380, 9386, 9142, 9384] dup_rows_per_dst=[2988, 2942, 2982, 3042, 3082, 3008, 3000, 3002] combine_rows_per_src=[9236, 9288, 9158, 9284, 9380, 9386, 9142, 9384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=64954 dup_rows=23944 dup_rows_remote=20968 dedup_ratio=0.243571 router_tokens_per_expert=[822, 819, 773, 785, 801, 786, 701, 787, 740, 777, 718, 746, 805, 785, 795, 740, 777, 773, 758, 734, 776, 787, 769, 817, 731, 764, 782, 767, 764, 731, 760, 794, 748, 773, 790, 774, 757, 802, 729, 777, 770, 759, 764, 753, 787, 781, 778, 737, 721, 820, 778, 740, 770, 767, 765, 769, 705, 753, 803, 748, 796, 781, 761, 732] payload_rows_per_dst=[9432, 9176, 9406, 9256, 9428, 9152, 9284, 9226] dup_rows_per_dst=[3116, 3036, 2976, 2930, 2872, 3106, 2976, 2932] combine_rows_per_src=[9432, 9176, 9406, 9256, 9428, 9152, 9284, 9226] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266887168 dispatch_meta_bytes=344472 combine_payload_bytes=266887168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267231640 ingress_bytes=301490176 link_bytes=301490176 token_bytes=534118808 weight_movement_bytes=34603008 total_nvlink_bytes=568721816 weight_to_token_ratio=0.064785 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134483260e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266051584 dispatch_meta_bytes=343688 combine_payload_bytes=266051584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266395272 ingress_bytes=300654592 link_bytes=300654592 token_bytes=532446856 weight_movement_bytes=34603008 total_nvlink_bytes=567049864 weight_to_token_ratio=0.064989 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131339025e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=65120 dup_rows=23928 dup_rows_remote=20924 dedup_ratio=0.243408 router_tokens_per_expert=[840, 766, 754, 699, 777, 794, 825, 794, 783, 790, 786, 725, 748, 793, 750, 804, 734, 754, 717, 793, 769, 769, 766, 734, 760, 760, 771, 763, 784, 765, 771, 792, 762, 743, 707, 830, 737, 792, 791, 768, 763, 716, 787, 734, 774, 831, 794, 765, 766, 770, 798, 762, 743, 792, 762, 766, 796, 761, 731, 742, 726, 748, 793, 772] payload_rows_per_dst=[9344, 9402, 9206, 9326, 9256, 9316, 9414, 9112] dup_rows_per_dst=[3154, 2956, 2866, 3006, 3004, 3012, 2904, 3026] combine_rows_per_src=[9344, 9402, 9206, 9326, 9256, 9316, 9414, 9112] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74472 payload_rows_remote=65112 dup_rows=23832 dup_rows_remote=20804 dedup_ratio=0.242432 router_tokens_per_expert=[769, 779, 798, 771, 746, 758, 791, 751, 773, 808, 732, 733, 776, 809, 791, 772, 807, 763, 731, 706, 703, 748, 764, 824, 818, 760, 745, 771, 795, 756, 793, 800, 784, 746, 763, 770, 735, 770, 758, 745, 763, 779, 777, 742, 754, 763, 745, 767, 760, 777, 770, 751, 767, 754, 759, 755, 769, 798, 764, 752, 793, 796, 795, 790] payload_rows_per_dst=[9286, 9360, 9238, 9400, 9298, 9214, 9266, 9410] dup_rows_per_dst=[3040, 3028, 2854, 3076, 2844, 2966, 2920, 3104] combine_rows_per_src=[9286, 9360, 9238, 9400, 9298, 9214, 9266, 9410] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=344456 combine_payload_bytes=266461184 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266805640 ingress_bytes=335667200 link_bytes=335667200 token_bytes=533266824 weight_movement_bytes=69206016 total_nvlink_bytes=602472840 weight_to_token_ratio=0.129777 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263088650e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265846784 dispatch_meta_bytes=343824 combine_payload_bytes=265846784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266190608 ingress_bytes=300449792 link_bytes=300449792 token_bytes=532037392 weight_movement_bytes=34603008 total_nvlink_bytes=566640400 weight_to_token_ratio=0.065039 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130568379e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=64872 dup_rows=24046 dup_rows_remote=21038 dedup_ratio=0.244609 router_tokens_per_expert=[778, 779, 763, 784, 747, 773, 731, 757, 744, 764, 762, 763, 738, 843, 737, 764, 741, 743, 751, 790, 758, 763, 770, 754, 751, 774, 797, 779, 767, 731, 783, 781, 786, 771, 787, 778, 805, 736, 789, 779, 777, 779, 799, 784, 790, 758, 732, 778, 729, 780, 771, 785, 734, 745, 761, 766, 786, 785, 755, 828, 727, 770, 765, 777] payload_rows_per_dst=[9236, 9288, 9158, 9284, 9380, 9386, 9142, 9384] dup_rows_per_dst=[2988, 2942, 2982, 3042, 3082, 3008, 3000, 3002] combine_rows_per_src=[9236, 9288, 9158, 9284, 9380, 9386, 9142, 9384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74416 payload_rows_remote=65106 dup_rows=23888 dup_rows_remote=20956 dedup_ratio=0.243001 router_tokens_per_expert=[727, 841, 753, 737, 770, 753, 756, 783, 808, 748, 756, 746, 770, 771, 746, 775, 792, 708, 748, 737, 713, 765, 794, 786, 774, 773, 832, 753, 754, 776, 810, 737, 733, 785, 850, 777, 810, 742, 816, 772, 781, 763, 669, 836, 721, 753, 789, 777, 724, 767, 794, 733, 738, 750, 814, 772, 749, 796, 779, 801, 753, 733, 792, 791] payload_rows_per_dst=[9310, 9230, 9458, 9358, 9512, 9318, 9254, 8976] dup_rows_per_dst=[2932, 3010, 3130, 3092, 3058, 2860, 2930, 2876] combine_rows_per_src=[9310, 9230, 9458, 9358, 9512, 9318, 9254, 8976] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265715712 dispatch_meta_bytes=343640 combine_payload_bytes=265715712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266059352 ingress_bytes=300318720 link_bytes=300318720 token_bytes=531775064 weight_movement_bytes=34603008 total_nvlink_bytes=566378072 weight_to_token_ratio=0.065071 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130075166e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74012 payload_rows_remote=64778 dup_rows=24292 dup_rows_remote=21274 dedup_ratio=0.247111 router_tokens_per_expert=[811, 792, 804, 745, 753, 763, 786, 746, 764, 746, 763, 780, 784, 724, 757, 740, 744, 749, 764, 767, 748, 802, 750, 796, 764, 804, 794, 734, 775, 775, 780, 752, 776, 791, 714, 778, 783, 795, 778, 766, 739, 759, 788, 759, 773, 773, 786, 771, 789, 770, 787, 793, 728, 761, 760, 753, 787, 767, 796, 746, 751, 766, 749, 764] payload_rows_per_dst=[9290, 9138, 9248, 8992, 9376, 9410, 9324, 9234] dup_rows_per_dst=[3042, 3020, 2992, 2834, 3166, 3236, 2984, 3018] combine_rows_per_src=[9290, 9138, 9248, 8992, 9376, 9410, 9324, 9234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266051584 dispatch_meta_bytes=343688 combine_payload_bytes=266051584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266395272 ingress_bytes=300654592 link_bytes=300654592 token_bytes=532446856 weight_movement_bytes=34603008 total_nvlink_bytes=567049864 weight_to_token_ratio=0.064989 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131339025e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=344176 combine_payload_bytes=266731520 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267075696 ingress_bytes=370540544 link_bytes=370540544 token_bytes=533807216 weight_movement_bytes=103809024 total_nvlink_bytes=637616240 weight_to_token_ratio=0.194469 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394314236e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266698752 dispatch_meta_bytes=343664 combine_payload_bytes=266698752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267042416 ingress_bytes=301301760 link_bytes=301301760 token_bytes=533741168 weight_movement_bytes=34603008 total_nvlink_bytes=568344176 weight_to_token_ratio=0.064831 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133774266e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265715712 dispatch_meta_bytes=343640 combine_payload_bytes=265715712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266059352 ingress_bytes=300318720 link_bytes=300318720 token_bytes=531775064 weight_movement_bytes=34603008 total_nvlink_bytes=566378072 weight_to_token_ratio=0.065071 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130075166e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266674176 dispatch_meta_bytes=344248 combine_payload_bytes=266674176 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267018424 ingress_bytes=370483200 link_bytes=370483200 token_bytes=533692600 weight_movement_bytes=103809024 total_nvlink_bytes=637501624 weight_to_token_ratio=0.194511 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394098455e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73904 payload_rows_remote=64742 dup_rows=24400 dup_rows_remote=21306 dedup_ratio=0.248210 router_tokens_per_expert=[769, 760, 785, 798, 799, 784, 771, 757, 733, 774, 742, 792, 821, 756, 813, 760, 779, 773, 738, 757, 767, 775, 795, 749, 777, 750, 755, 801, 765, 767, 778, 732, 817, 784, 725, 753, 759, 743, 752, 795, 792, 779, 769, 760, 824, 766, 766, 764, 782, 770, 760, 734, 740, 755, 731, 726, 770, 781, 798, 730, 744, 779, 745, 787] payload_rows_per_dst=[9342, 9276, 9248, 9224, 9162, 9348, 9014, 9290] dup_rows_per_dst=[3104, 3106, 3018, 3026, 3094, 3092, 2982, 2978] combine_rows_per_src=[9342, 9276, 9248, 9224, 9162, 9348, 9014, 9290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73930 payload_rows_remote=64518 dup_rows=24374 dup_rows_remote=21240 dedup_ratio=0.247945 router_tokens_per_expert=[751, 754, 773, 816, 776, 769, 718, 746, 765, 760, 804, 792, 746, 801, 759, 806, 746, 737, 762, 801, 759, 773, 753, 751, 781, 744, 761, 772, 709, 715, 817, 786, 797, 808, 733, 778, 723, 740, 771, 748, 801, 768, 741, 733, 792, 805, 728, 792, 769, 822, 782, 791, 738, 812, 797, 762, 754, 769, 761, 762, 750, 776, 750, 796] payload_rows_per_dst=[9214, 9348, 9176, 9112, 9216, 9224, 9412, 9228] dup_rows_per_dst=[2992, 3118, 2988, 3058, 2980, 3096, 3134, 3008] combine_rows_per_src=[9214, 9348, 9176, 9112, 9216, 9224, 9412, 9228] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265330688 dispatch_meta_bytes=344208 combine_payload_bytes=265330688 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265674896 ingress_bytes=369139712 link_bytes=369139712 token_bytes=531005584 weight_movement_bytes=103809024 total_nvlink_bytes=634814608 weight_to_token_ratio=0.195495 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389043018e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=65032 dup_rows=24012 dup_rows_remote=21072 dedup_ratio=0.244263 router_tokens_per_expert=[794, 764, 788, 781, 783, 767, 729, 806, 690, 769, 723, 762, 730, 818, 746, 802, 753, 750, 759, 782, 796, 773, 824, 787, 741, 771, 800, 726, 700, 803, 776, 783, 812, 747, 707, 762, 800, 815, 756, 726, 767, 802, 735, 755, 770, 739, 793, 780, 757, 743, 773, 775, 776, 811, 774, 749, 784, 742, 784, 734, 767, 789, 781, 771] payload_rows_per_dst=[9432, 9160, 9468, 9260, 9240, 9262, 9306, 9164] dup_rows_per_dst=[2992, 2920, 2980, 2940, 3010, 3020, 3010, 3140] combine_rows_per_src=[9432, 9160, 9468, 9260, 9240, 9262, 9306, 9164] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74148 payload_rows_remote=64876 dup_rows=24156 dup_rows_remote=21218 dedup_ratio=0.245728 router_tokens_per_expert=[758, 689, 751, 740, 772, 744, 774, 760, 841, 795, 757, 748, 787, 761, 733, 770, 744, 758, 790, 761, 754, 767, 750, 781, 768, 765, 711, 759, 758, 773, 792, 760, 786, 782, 784, 770, 696, 732, 768, 734, 719, 782, 754, 794, 730, 755, 784, 777, 770, 793, 821, 788, 837, 774, 787, 740, 778, 852, 827, 815, 788, 784, 746, 734] payload_rows_per_dst=[9052, 9330, 9272, 9234, 9166, 9242, 9388, 9464] dup_rows_per_dst=[2924, 3054, 2938, 2938, 2938, 2948, 3232, 3184] combine_rows_per_src=[9052, 9330, 9272, 9234, 9166, 9242, 9388, 9464] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74416 payload_rows_remote=65106 dup_rows=23888 dup_rows_remote=20956 dedup_ratio=0.243001 router_tokens_per_expert=[727, 841, 753, 737, 770, 753, 756, 783, 808, 748, 756, 746, 770, 771, 746, 775, 792, 708, 748, 737, 713, 765, 794, 786, 774, 773, 832, 753, 754, 776, 810, 737, 733, 785, 850, 777, 810, 742, 816, 772, 781, 763, 669, 836, 721, 753, 789, 777, 724, 767, 794, 733, 738, 750, 814, 772, 749, 796, 779, 801, 753, 733, 792, 791] payload_rows_per_dst=[9310, 9230, 9458, 9358, 9512, 9318, 9254, 8976] dup_rows_per_dst=[2932, 3010, 3130, 3092, 3058, 2860, 2930, 2876] combine_rows_per_src=[9310, 9230, 9458, 9358, 9512, 9318, 9254, 8976] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=64854 dup_rows=24002 dup_rows_remote=20992 dedup_ratio=0.244161 router_tokens_per_expert=[777, 740, 789, 743, 756, 748, 754, 742, 758, 761, 763, 755, 736, 749, 731, 768, 813, 778, 746, 772, 760, 764, 755, 740, 809, 825, 777, 785, 794, 714, 783, 778, 752, 792, 779, 719, 750, 799, 757, 779, 802, 815, 788, 721, 781, 829, 732, 761, 744, 757, 767, 773, 760, 731, 793, 769, 781, 769, 798, 767, 782, 781, 777, 784] payload_rows_per_dst=[9244, 9144, 9266, 9384, 9220, 9448, 9198, 9398] dup_rows_per_dst=[2854, 2898, 2990, 3146, 3034, 3010, 2990, 3080] combine_rows_per_src=[9244, 9144, 9266, 9384, 9220, 9448, 9198, 9398] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265183232 dispatch_meta_bytes=344192 combine_payload_bytes=265183232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265527424 ingress_bytes=299786240 link_bytes=299786240 token_bytes=530710656 weight_movement_bytes=34603008 total_nvlink_bytes=565313664 weight_to_token_ratio=0.065201 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128071486e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73930 payload_rows_remote=64518 dup_rows=24374 dup_rows_remote=21240 dedup_ratio=0.247945 router_tokens_per_expert=[751, 754, 773, 816, 776, 769, 718, 746, 765, 760, 804, 792, 746, 801, 759, 806, 746, 737, 762, 801, 759, 773, 753, 751, 781, 744, 761, 772, 709, 715, 817, 786, 797, 808, 733, 778, 723, 740, 771, 748, 801, 768, 741, 733, 792, 805, 728, 792, 769, 822, 782, 791, 738, 812, 797, 762, 754, 769, 761, 762, 750, 776, 750, 796] payload_rows_per_dst=[9214, 9348, 9176, 9112, 9216, 9224, 9412, 9228] dup_rows_per_dst=[2992, 3118, 2988, 3058, 2980, 3096, 3134, 3008] combine_rows_per_src=[9214, 9348, 9176, 9112, 9216, 9224, 9412, 9228] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=264265728 dispatch_meta_bytes=343032 combine_payload_bytes=264265728 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=264608760 ingress_bytes=368074752 link_bytes=368074752 token_bytes=528874488 weight_movement_bytes=103809024 total_nvlink_bytes=632683512 weight_to_token_ratio=0.196283 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.385035659e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344416 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266715488 ingress_bytes=370180096 link_bytes=370180096 token_bytes=533086560 weight_movement_bytes=103809024 total_nvlink_bytes=636895584 weight_to_token_ratio=0.194732 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392957899e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74148 payload_rows_remote=64876 dup_rows=24156 dup_rows_remote=21218 dedup_ratio=0.245728 router_tokens_per_expert=[758, 689, 751, 740, 772, 744, 774, 760, 841, 795, 757, 748, 787, 761, 733, 770, 744, 758, 790, 761, 754, 767, 750, 781, 768, 765, 711, 759, 758, 773, 792, 760, 786, 782, 784, 770, 696, 732, 768, 734, 719, 782, 754, 794, 730, 755, 784, 777, 770, 793, 821, 788, 837, 774, 787, 740, 778, 852, 827, 815, 788, 784, 746, 734] payload_rows_per_dst=[9052, 9330, 9272, 9234, 9166, 9242, 9388, 9464] dup_rows_per_dst=[2924, 3054, 2938, 2938, 2938, 2948, 3232, 3184] combine_rows_per_src=[9052, 9330, 9272, 9234, 9166, 9242, 9388, 9464] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74438 payload_rows_remote=65262 dup_rows=23866 dup_rows_remote=20906 dedup_ratio=0.242778 router_tokens_per_expert=[794, 798, 775, 772, 744, 748, 793, 729, 779, 784, 725, 746, 754, 750, 790, 740, 771, 757, 784, 794, 765, 748, 759, 737, 712, 773, 726, 747, 726, 779, 773, 742, 809, 816, 830, 750, 779, 794, 809, 773, 797, 723, 788, 780, 786, 762, 785, 759, 784, 736, 802, 772, 742, 834, 723, 737, 795, 782, 784, 742, 779, 743, 793, 750] payload_rows_per_dst=[9328, 9176, 9260, 9104, 9568, 9410, 9270, 9322] dup_rows_per_dst=[2978, 2960, 2970, 2852, 3152, 2950, 2990, 3014] combine_rows_per_src=[9328, 9176, 9260, 9104, 9568, 9410, 9270, 9322] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265732096 dispatch_meta_bytes=344376 combine_payload_bytes=265732096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266076472 ingress_bytes=300335104 link_bytes=300335104 token_bytes=531808568 weight_movement_bytes=34603008 total_nvlink_bytes=566411576 weight_to_token_ratio=0.065067 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130136817e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266674176 dispatch_meta_bytes=344248 combine_payload_bytes=266674176 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267018424 ingress_bytes=370483200 link_bytes=370483200 token_bytes=533692600 weight_movement_bytes=103809024 total_nvlink_bytes=637501624 weight_to_token_ratio=0.194511 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394098455e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265641984 dispatch_meta_bytes=343384 combine_payload_bytes=265641984 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265985368 ingress_bytes=334848000 link_bytes=334848000 token_bytes=531627352 weight_movement_bytes=69206016 total_nvlink_bytes=600833368 weight_to_token_ratio=0.130178 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260006067e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=64854 dup_rows=24002 dup_rows_remote=20992 dedup_ratio=0.244161 router_tokens_per_expert=[777, 740, 789, 743, 756, 748, 754, 742, 758, 761, 763, 755, 736, 749, 731, 768, 813, 778, 746, 772, 760, 764, 755, 740, 809, 825, 777, 785, 794, 714, 783, 778, 752, 792, 779, 719, 750, 799, 757, 779, 802, 815, 788, 721, 781, 829, 732, 761, 744, 757, 767, 773, 760, 731, 793, 769, 781, 769, 798, 767, 782, 781, 777, 784] payload_rows_per_dst=[9244, 9144, 9266, 9384, 9220, 9448, 9198, 9398] dup_rows_per_dst=[2854, 2898, 2990, 3146, 3034, 3010, 2990, 3080] combine_rows_per_src=[9244, 9144, 9266, 9384, 9220, 9448, 9198, 9398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74450 payload_rows_remote=65048 dup_rows=23854 dup_rows_remote=20720 dedup_ratio=0.242655 router_tokens_per_expert=[758, 734, 761, 771, 768, 756, 750, 765, 733, 719, 788, 771, 761, 782, 723, 758, 777, 819, 755, 754, 711, 761, 731, 768, 775, 788, 834, 729, 745, 760, 750, 792, 776, 785, 764, 753, 815, 792, 809, 774, 798, 775, 761, 762, 804, 769, 749, 789, 794, 765, 773, 770, 811, 759, 759, 752, 756, 791, 729, 798, 784, 746, 744, 799] payload_rows_per_dst=[9196, 9256, 9300, 9396, 9402, 9322, 9312, 9266] dup_rows_per_dst=[2930, 2814, 2852, 2950, 3134, 3092, 3054, 3028] combine_rows_per_src=[9196, 9256, 9300, 9396, 9402, 9322, 9312, 9266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74038 payload_rows_remote=64820 dup_rows=24266 dup_rows_remote=21268 dedup_ratio=0.246847 router_tokens_per_expert=[791, 791, 749, 734, 757, 795, 746, 734, 786, 740, 746, 767, 769, 776, 783, 748, 748, 800, 737, 757, 729, 797, 802, 802, 772, 784, 793, 767, 773, 789, 796, 757, 760, 750, 776, 741, 754, 754, 797, 785, 805, 734, 750, 800, 731, 728, 784, 736, 791, 824, 793, 783, 792, 755, 761, 747, 739, 785, 822, 726, 763, 780, 756, 735] payload_rows_per_dst=[9218, 9610, 9278, 9244, 9286, 8858, 9356, 9188] dup_rows_per_dst=[2998, 3330, 3088, 3206, 2948, 2568, 3104, 3024] combine_rows_per_src=[9218, 9610, 9278, 9244, 9286, 8858, 9356, 9188] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=264265728 dispatch_meta_bytes=343032 combine_payload_bytes=264265728 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=264608760 ingress_bytes=368074752 link_bytes=368074752 token_bytes=528874488 weight_movement_bytes=103809024 total_nvlink_bytes=632683512 weight_to_token_ratio=0.196283 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.385035659e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74200 payload_rows_remote=64920 dup_rows=24104 dup_rows_remote=21140 dedup_ratio=0.245199 router_tokens_per_expert=[750, 763, 766, 808, 761, 791, 771, 753, 741, 790, 776, 755, 760, 753, 787, 755, 746, 728, 769, 808, 789, 751, 763, 771, 774, 771, 737, 763, 754, 762, 780, 739, 749, 773, 754, 809, 750, 749, 745, 772, 762, 767, 817, 801, 763, 769, 800, 768, 780, 780, 797, 755, 788, 744, 834, 735, 749, 721, 768, 786, 778, 726, 756, 822] payload_rows_per_dst=[9270, 9262, 9234, 9248, 9086, 9478, 9342, 9280] dup_rows_per_dst=[3056, 2972, 3016, 2796, 2856, 3392, 3052, 2964] combine_rows_per_src=[9270, 9262, 9234, 9248, 9086, 9478, 9342, 9280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267313152 dispatch_meta_bytes=344672 combine_payload_bytes=267313152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267657824 ingress_bytes=301916160 link_bytes=301916160 token_bytes=534970976 weight_movement_bytes=34603008 total_nvlink_bytes=569573984 weight_to_token_ratio=0.064682 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136086204e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265641984 dispatch_meta_bytes=343384 combine_payload_bytes=265641984 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265985368 ingress_bytes=334848000 link_bytes=334848000 token_bytes=531627352 weight_movement_bytes=69206016 total_nvlink_bytes=600833368 weight_to_token_ratio=0.130178 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260006067e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265732096 dispatch_meta_bytes=344376 combine_payload_bytes=265732096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266076472 ingress_bytes=300335104 link_bytes=300335104 token_bytes=531808568 weight_movement_bytes=34603008 total_nvlink_bytes=566411576 weight_to_token_ratio=0.065067 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130136817e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266436608 dispatch_meta_bytes=343072 combine_payload_bytes=266436608 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266779680 ingress_bytes=370245632 link_bytes=370245632 token_bytes=533216288 weight_movement_bytes=103809024 total_nvlink_bytes=637025312 weight_to_token_ratio=0.194685 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393204506e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74426 payload_rows_remote=65082 dup_rows=23878 dup_rows_remote=20924 dedup_ratio=0.242900 router_tokens_per_expert=[793, 750, 811, 837, 825, 740, 756, 768, 710, 788, 760, 749, 755, 745, 790, 783, 771, 742, 778, 792, 748, 757, 796, 809, 803, 764, 750, 780, 773, 789, 750, 769, 757, 794, 784, 794, 729, 724, 801, 766, 771, 773, 743, 755, 741, 793, 791, 735, 717, 736, 764, 749, 779, 748, 784, 789, 781, 789, 777, 801, 721, 702, 752, 781] payload_rows_per_dst=[9524, 9164, 9358, 9332, 9344, 9234, 9174, 9296] dup_rows_per_dst=[3036, 2996, 3028, 3024, 2954, 2970, 2958, 2912] combine_rows_per_src=[9524, 9164, 9358, 9332, 9344, 9234, 9174, 9296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=65002 dup_rows=23946 dup_rows_remote=20750 dedup_ratio=0.243591 router_tokens_per_expert=[753, 765, 764, 779, 792, 750, 739, 825, 752, 788, 790, 780, 774, 816, 745, 776, 795, 745, 766, 778, 724, 810, 767, 775, 777, 800, 818, 795, 758, 821, 763, 770, 768, 749, 746, 754, 768, 707, 742, 755, 786, 729, 795, 739, 749, 748, 715, 736, 790, 773, 790, 807, 777, 749, 798, 792, 791, 763, 764, 761, 741, 722, 738, 760] payload_rows_per_dst=[9422, 9462, 9276, 9408, 9144, 9142, 9356, 9148] dup_rows_per_dst=[2912, 2980, 3044, 3196, 2834, 2852, 3196, 2932] combine_rows_per_src=[9422, 9462, 9276, 9408, 9144, 9142, 9356, 9148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=344352 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265847072 ingress_bytes=300105728 link_bytes=300105728 token_bytes=531349792 weight_movement_bytes=34603008 total_nvlink_bytes=565952800 weight_to_token_ratio=0.065123 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129273694e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265912320 dispatch_meta_bytes=344240 combine_payload_bytes=265912320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266256560 ingress_bytes=300515328 link_bytes=300515328 token_bytes=532168880 weight_movement_bytes=34603008 total_nvlink_bytes=566771888 weight_to_token_ratio=0.065023 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130814986e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=64970 dup_rows=23924 dup_rows_remote=20830 dedup_ratio=0.243368 router_tokens_per_expert=[746, 765, 713, 813, 766, 736, 796, 768, 783, 770, 740, 742, 747, 785, 763, 734, 766, 796, 737, 760, 821, 749, 785, 778, 774, 738, 760, 754, 794, 793, 863, 776, 775, 737, 758, 780, 787, 732, 764, 792, 784, 758, 751, 760, 799, 770, 787, 780, 750, 747, 768, 784, 758, 763, 760, 768, 749, 767, 793, 787, 702, 742, 812, 777] payload_rows_per_dst=[9262, 9214, 9310, 9410, 9304, 9384, 9250, 9246] dup_rows_per_dst=[2944, 2914, 3074, 3094, 2946, 2994, 2946, 3012] combine_rows_per_src=[9262, 9214, 9310, 9410, 9304, 9384, 9250, 9246] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=343008 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266591200 ingress_bytes=300851200 link_bytes=300851200 token_bytes=532839392 weight_movement_bytes=34603008 total_nvlink_bytes=567442400 weight_to_token_ratio=0.064941 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132078845e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266117120 dispatch_meta_bytes=343200 combine_payload_bytes=266117120 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266460320 ingress_bytes=369926144 link_bytes=369926144 token_bytes=532577440 weight_movement_bytes=103809024 total_nvlink_bytes=636386464 weight_to_token_ratio=0.194918 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392002298e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74404 payload_rows_remote=65180 dup_rows=23900 dup_rows_remote=20860 dedup_ratio=0.243123 router_tokens_per_expert=[763, 758, 735, 778, 734, 762, 753, 742, 766, 743, 753, 745, 786, 749, 806, 784, 796, 774, 757, 771, 775, 810, 733, 731, 745, 785, 779, 761, 739, 819, 810, 797, 771, 705, 786, 758, 780, 817, 763, 750, 753, 771, 749, 771, 765, 747, 734, 769, 820, 780, 786, 785, 798, 743, 781, 770, 769, 776, 767, 786, 757, 764, 762, 780] payload_rows_per_dst=[9168, 9224, 9428, 9390, 9248, 9250, 9370, 9326] dup_rows_per_dst=[2882, 3040, 2866, 3080, 3012, 2868, 3156, 2996] combine_rows_per_src=[9168, 9224, 9428, 9390, 9248, 9250, 9370, 9326] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74410 payload_rows_remote=65086 dup_rows=23894 dup_rows_remote=20854 dedup_ratio=0.243062 router_tokens_per_expert=[779, 676, 746, 768, 810, 775, 786, 778, 767, 753, 735, 743, 750, 793, 834, 783, 747, 771, 774, 767, 800, 773, 777, 773, 774, 796, 764, 770, 745, 762, 780, 725, 776, 767, 792, 795, 754, 762, 771, 719, 798, 803, 770, 768, 754, 737, 790, 725, 745, 816, 812, 703, 780, 825, 782, 796, 787, 736, 704, 772, 750, 754, 763, 772] payload_rows_per_dst=[9294, 9336, 9324, 9288, 9228, 9254, 9426, 9260] dup_rows_per_dst=[2942, 2980, 3040, 2944, 3044, 3036, 3092, 2816] combine_rows_per_src=[9294, 9336, 9324, 9288, 9228, 9254, 9426, 9260] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74038 payload_rows_remote=64820 dup_rows=24266 dup_rows_remote=21268 dedup_ratio=0.246847 router_tokens_per_expert=[791, 791, 749, 734, 757, 795, 746, 734, 786, 740, 746, 767, 769, 776, 783, 748, 748, 800, 737, 757, 729, 797, 802, 802, 772, 784, 793, 767, 773, 789, 796, 757, 760, 750, 776, 741, 754, 754, 797, 785, 805, 734, 750, 800, 731, 728, 784, 736, 791, 824, 793, 783, 792, 755, 761, 747, 739, 785, 822, 726, 763, 780, 756, 735] payload_rows_per_dst=[9218, 9610, 9278, 9244, 9286, 8858, 9356, 9188] dup_rows_per_dst=[2998, 3330, 3088, 3206, 2948, 2568, 3104, 3024] combine_rows_per_src=[9218, 9610, 9278, 9244, 9286, 8858, 9356, 9188] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=65228 dup_rows=23926 dup_rows_remote=20896 dedup_ratio=0.243388 router_tokens_per_expert=[743, 738, 787, 778, 814, 804, 768, 791, 777, 800, 814, 781, 753, 737, 731, 753, 771, 756, 776, 713, 770, 758, 767, 763, 774, 784, 782, 776, 739, 753, 763, 743, 730, 803, 768, 771, 795, 726, 810, 740, 798, 745, 777, 757, 777, 733, 755, 748, 807, 784, 812, 755, 751, 835, 788, 759, 795, 725, 769, 788, 767, 730, 743, 754] payload_rows_per_dst=[9336, 9270, 9246, 9328, 9366, 9150, 9490, 9192] dup_rows_per_dst=[3110, 3022, 2902, 2900, 2920, 3030, 3092, 2950] combine_rows_per_src=[9336, 9270, 9246, 9328, 9366, 9150, 9490, 9192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266575872 dispatch_meta_bytes=344024 combine_payload_bytes=266575872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266919896 ingress_bytes=335781888 link_bytes=335781888 token_bytes=533495768 weight_movement_bytes=69206016 total_nvlink_bytes=602701784 weight_to_token_ratio=0.129722 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263520212e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=65002 dup_rows=23946 dup_rows_remote=20750 dedup_ratio=0.243591 router_tokens_per_expert=[753, 765, 764, 779, 792, 750, 739, 825, 752, 788, 790, 780, 774, 816, 745, 776, 795, 745, 766, 778, 724, 810, 767, 775, 777, 800, 818, 795, 758, 821, 763, 770, 768, 749, 746, 754, 768, 707, 742, 755, 786, 729, 795, 739, 749, 748, 715, 736, 790, 773, 790, 807, 777, 749, 798, 792, 791, 763, 764, 761, 741, 722, 738, 760] payload_rows_per_dst=[9422, 9462, 9276, 9408, 9144, 9142, 9356, 9148] dup_rows_per_dst=[2912, 2980, 3044, 3196, 2834, 2852, 3196, 2932] combine_rows_per_src=[9422, 9462, 9276, 9408, 9144, 9142, 9356, 9148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73904 payload_rows_remote=64742 dup_rows=24400 dup_rows_remote=21306 dedup_ratio=0.248210 router_tokens_per_expert=[769, 760, 785, 798, 799, 784, 771, 757, 733, 774, 742, 792, 821, 756, 813, 760, 779, 773, 738, 757, 767, 775, 795, 749, 777, 750, 755, 801, 765, 767, 778, 732, 817, 784, 725, 753, 759, 743, 752, 795, 792, 779, 769, 760, 824, 766, 766, 764, 782, 770, 760, 734, 740, 755, 731, 726, 770, 781, 798, 730, 744, 779, 745, 787] payload_rows_per_dst=[9342, 9276, 9248, 9224, 9162, 9348, 9014, 9290] dup_rows_per_dst=[3104, 3106, 3018, 3026, 3094, 3092, 2982, 2978] combine_rows_per_src=[9342, 9276, 9248, 9224, 9162, 9348, 9014, 9290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=65030 dup_rows=23992 dup_rows_remote=20962 dedup_ratio=0.244059 router_tokens_per_expert=[817, 804, 737, 739, 775, 689, 819, 775, 820, 749, 801, 790, 801, 758, 793, 738, 807, 750, 759, 736, 759, 821, 721, 737, 776, 782, 752, 789, 775, 752, 761, 813, 775, 760, 782, 748, 772, 754, 760, 734, 793, 716, 759, 761, 712, 797, 760, 754, 807, 745, 727, 819, 759, 759, 724, 717, 788, 794, 783, 778, 819, 737, 777, 787] payload_rows_per_dst=[9282, 9458, 9258, 9562, 9182, 9448, 8690, 9432] dup_rows_per_dst=[3030, 3042, 2920, 3206, 2988, 3154, 2558, 3094] combine_rows_per_src=[9282, 9458, 9258, 9562, 9182, 9448, 8690, 9432] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74404 payload_rows_remote=65180 dup_rows=23900 dup_rows_remote=20860 dedup_ratio=0.243123 router_tokens_per_expert=[763, 758, 735, 778, 734, 762, 753, 742, 766, 743, 753, 745, 786, 749, 806, 784, 796, 774, 757, 771, 775, 810, 733, 731, 745, 785, 779, 761, 739, 819, 810, 797, 771, 705, 786, 758, 780, 817, 763, 750, 753, 771, 749, 771, 765, 747, 734, 769, 820, 780, 786, 785, 798, 743, 781, 770, 769, 776, 767, 786, 757, 764, 762, 780] payload_rows_per_dst=[9168, 9224, 9428, 9390, 9248, 9250, 9370, 9326] dup_rows_per_dst=[2882, 3040, 2866, 3080, 3012, 2868, 3156, 2996] combine_rows_per_src=[9168, 9224, 9428, 9390, 9248, 9250, 9370, 9326] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266117120 dispatch_meta_bytes=343200 combine_payload_bytes=266117120 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266460320 ingress_bytes=369926144 link_bytes=369926144 token_bytes=532577440 weight_movement_bytes=103809024 total_nvlink_bytes=636386464 weight_to_token_ratio=0.194918 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392002298e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74478 payload_rows_remote=65120 dup_rows=23826 dup_rows_remote=20742 dedup_ratio=0.242371 router_tokens_per_expert=[757, 748, 785, 827, 719, 766, 794, 800, 738, 752, 749, 778, 783, 745, 755, 764, 754, 780, 798, 755, 767, 803, 738, 791, 790, 696, 758, 744, 752, 743, 757, 767, 756, 779, 768, 749, 797, 781, 766, 758, 777, 720, 760, 737, 778, 792, 825, 757, 765, 777, 755, 769, 824, 773, 759, 767, 744, 808, 808, 782, 758, 767, 769, 774] payload_rows_per_dst=[9228, 9302, 9278, 9136, 9436, 9326, 9414, 9358] dup_rows_per_dst=[2992, 2858, 3030, 2878, 2936, 2966, 3082, 3084] combine_rows_per_src=[9228, 9302, 9278, 9136, 9436, 9326, 9414, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266977280 dispatch_meta_bytes=344160 combine_payload_bytes=266977280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267321440 ingress_bytes=301580288 link_bytes=301580288 token_bytes=534298720 weight_movement_bytes=34603008 total_nvlink_bytes=568901728 weight_to_token_ratio=0.064763 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134822345e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266592256 dispatch_meta_bytes=343760 combine_payload_bytes=266592256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266936016 ingress_bytes=301195264 link_bytes=301195264 token_bytes=533528272 weight_movement_bytes=34603008 total_nvlink_bytes=568131280 weight_to_token_ratio=0.064857 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133373530e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=344352 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265847072 ingress_bytes=300105728 link_bytes=300105728 token_bytes=531349792 weight_movement_bytes=34603008 total_nvlink_bytes=565952800 weight_to_token_ratio=0.065123 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129273694e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=267173888 dispatch_meta_bytes=344496 combine_payload_bytes=267173888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267518384 ingress_bytes=301776896 link_bytes=301776896 token_bytes=534692272 weight_movement_bytes=34603008 total_nvlink_bytes=569295280 weight_to_token_ratio=0.064716 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135562165e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=343008 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266591200 ingress_bytes=300851200 link_bytes=300851200 token_bytes=532839392 weight_movement_bytes=34603008 total_nvlink_bytes=567442400 weight_to_token_ratio=0.064941 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132078845e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266977280 dispatch_meta_bytes=344160 combine_payload_bytes=266977280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267321440 ingress_bytes=301580288 link_bytes=301580288 token_bytes=534298720 weight_movement_bytes=34603008 total_nvlink_bytes=568901728 weight_to_token_ratio=0.064763 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134822345e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74478 payload_rows_remote=65120 dup_rows=23826 dup_rows_remote=20742 dedup_ratio=0.242371 router_tokens_per_expert=[757, 748, 785, 827, 719, 766, 794, 800, 738, 752, 749, 778, 783, 745, 755, 764, 754, 780, 798, 755, 767, 803, 738, 791, 790, 696, 758, 744, 752, 743, 757, 767, 756, 779, 768, 749, 797, 781, 766, 758, 777, 720, 760, 737, 778, 792, 825, 757, 765, 777, 755, 769, 824, 773, 759, 767, 744, 808, 808, 782, 758, 767, 769, 774] payload_rows_per_dst=[9228, 9302, 9278, 9136, 9436, 9326, 9414, 9358] dup_rows_per_dst=[2992, 2858, 3030, 2878, 2936, 2966, 3082, 3084] combine_rows_per_src=[9228, 9302, 9278, 9136, 9436, 9326, 9414, 9358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=65144 dup_rows=23860 dup_rows_remote=20792 dedup_ratio=0.242716 router_tokens_per_expert=[768, 763, 756, 768, 774, 761, 786, 792, 716, 773, 739, 733, 783, 757, 784, 811, 807, 803, 776, 827, 781, 750, 771, 765, 756, 739, 737, 744, 763, 796, 773, 754, 771, 766, 784, 750, 772, 790, 782, 769, 773, 711, 766, 763, 785, 780, 786, 802, 764, 747, 763, 749, 783, 751, 748, 803, 745, 729, 786, 771, 761, 748, 775, 773] payload_rows_per_dst=[9382, 9268, 9492, 9228, 9300, 9306, 9172, 9296] dup_rows_per_dst=[2954, 2924, 3068, 2896, 3068, 3026, 3044, 2880] combine_rows_per_src=[9382, 9268, 9492, 9228, 9300, 9306, 9172, 9296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74320 payload_rows_remote=65066 dup_rows=23984 dup_rows_remote=21082 dedup_ratio=0.243978 router_tokens_per_expert=[778, 767, 759, 788, 752, 783, 788, 763, 723, 769, 763, 792, 762, 785, 794, 786, 713, 734, 769, 728, 760, 809, 745, 776, 748, 779, 833, 736, 768, 709, 789, 754, 760, 745, 792, 780, 749, 809, 800, 732, 741, 761, 794, 721, 713, 808, 817, 804, 773, 756, 769, 785, 757, 747, 744, 747, 844, 746, 749, 773, 805, 771, 782, 776] payload_rows_per_dst=[9372, 9280, 9196, 9240, 9264, 9316, 9254, 9398] dup_rows_per_dst=[2984, 3068, 2872, 2992, 3070, 3002, 2902, 3094] combine_rows_per_src=[9372, 9280, 9196, 9240, 9264, 9316, 9254, 9398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265183232 dispatch_meta_bytes=344192 combine_payload_bytes=265183232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265527424 ingress_bytes=299786240 link_bytes=299786240 token_bytes=530710656 weight_movement_bytes=34603008 total_nvlink_bytes=565313664 weight_to_token_ratio=0.065201 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128071486e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=343448 combine_payload_bytes=266731520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267074968 ingress_bytes=301334528 link_bytes=301334528 token_bytes=533806488 weight_movement_bytes=34603008 total_nvlink_bytes=568409496 weight_to_token_ratio=0.064823 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133897569e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=343968 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266706848 ingress_bytes=404774912 link_bytes=404774912 token_bytes=533069728 weight_movement_bytes=138412032 total_nvlink_bytes=671481760 weight_to_token_ratio=0.259651 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523135407e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74448 payload_rows_remote=65114 dup_rows=23856 dup_rows_remote=20952 dedup_ratio=0.242676 router_tokens_per_expert=[761, 765, 753, 759, 732, 749, 803, 745, 755, 755, 760, 733, 784, 831, 777, 758, 770, 753, 759, 816, 790, 766, 709, 757, 773, 785, 780, 783, 774, 735, 736, 753, 724, 782, 777, 771, 783, 764, 793, 783, 735, 764, 765, 769, 759, 789, 764, 786, 755, 764, 794, 794, 803, 752, 781, 795, 790, 761, 792, 756, 765, 787, 781, 715] payload_rows_per_dst=[9254, 9336, 9336, 9334, 9272, 9202, 9408, 9306] dup_rows_per_dst=[2880, 2970, 2904, 2904, 3082, 3060, 3068, 2988] combine_rows_per_src=[9254, 9336, 9336, 9334, 9272, 9202, 9408, 9306] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266829824 dispatch_meta_bytes=343744 combine_payload_bytes=266829824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267173568 ingress_bytes=301432832 link_bytes=301432832 token_bytes=534003392 weight_movement_bytes=34603008 total_nvlink_bytes=568606400 weight_to_token_ratio=0.064799 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134267479e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74122 payload_rows_remote=64884 dup_rows=24182 dup_rows_remote=21210 dedup_ratio=0.245992 router_tokens_per_expert=[759, 771, 761, 771, 767, 778, 788, 759, 772, 754, 775, 802, 735, 749, 775, 802, 719, 743, 801, 786, 759, 734, 764, 799, 757, 789, 728, 764, 756, 793, 787, 741, 796, 769, 775, 746, 760, 736, 717, 768, 798, 710, 788, 770, 718, 788, 774, 761, 824, 810, 753, 768, 794, 770, 781, 735, 756, 765, 780, 775, 757, 790, 790, 792] payload_rows_per_dst=[9202, 9320, 9238, 9222, 9234, 9226, 9388, 9292] dup_rows_per_dst=[3106, 3008, 2972, 3008, 2900, 2988, 3082, 3118] combine_rows_per_src=[9202, 9320, 9238, 9222, 9234, 9226, 9388, 9292] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=64940 dup_rows=24118 dup_rows_remote=21184 dedup_ratio=0.245341 router_tokens_per_expert=[767, 749, 768, 775, 746, 765, 788, 736, 794, 801, 777, 803, 791, 776, 743, 767, 714, 724, 763, 845, 765, 744, 766, 775, 791, 839, 785, 787, 783, 762, 723, 738, 780, 733, 748, 783, 742, 814, 741, 780, 732, 747, 757, 767, 780, 754, 782, 771, 750, 784, 785, 770, 792, 798, 764, 800, 782, 761, 750, 767, 766, 737, 736, 749] payload_rows_per_dst=[9198, 9410, 9176, 9312, 9310, 9246, 9286, 9248] dup_rows_per_dst=[2990, 3094, 3016, 3104, 2932, 2934, 3200, 2848] combine_rows_per_src=[9198, 9410, 9176, 9312, 9310, 9246, 9286, 9248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=65030 dup_rows=23992 dup_rows_remote=20962 dedup_ratio=0.244059 router_tokens_per_expert=[817, 804, 737, 739, 775, 689, 819, 775, 820, 749, 801, 790, 801, 758, 793, 738, 807, 750, 759, 736, 759, 821, 721, 737, 776, 782, 752, 789, 775, 752, 761, 813, 775, 760, 782, 748, 772, 754, 760, 734, 793, 716, 759, 761, 712, 797, 760, 754, 807, 745, 727, 819, 759, 759, 724, 717, 788, 794, 783, 778, 819, 737, 777, 787] payload_rows_per_dst=[9282, 9458, 9258, 9562, 9182, 9448, 8690, 9432] dup_rows_per_dst=[3030, 3042, 2920, 3206, 2988, 3154, 2558, 3094] combine_rows_per_src=[9282, 9458, 9258, 9562, 9182, 9448, 8690, 9432] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74320 payload_rows_remote=65066 dup_rows=23984 dup_rows_remote=21082 dedup_ratio=0.243978 router_tokens_per_expert=[778, 767, 759, 788, 752, 783, 788, 763, 723, 769, 763, 792, 762, 785, 794, 786, 713, 734, 769, 728, 760, 809, 745, 776, 748, 779, 833, 736, 768, 709, 789, 754, 760, 745, 792, 780, 749, 809, 800, 732, 741, 761, 794, 721, 713, 808, 817, 804, 773, 756, 769, 785, 757, 747, 744, 747, 844, 746, 749, 773, 805, 771, 782, 776] payload_rows_per_dst=[9372, 9280, 9196, 9240, 9264, 9316, 9254, 9398] dup_rows_per_dst=[2984, 3068, 2872, 2992, 3070, 3002, 2902, 3094] combine_rows_per_src=[9372, 9280, 9196, 9240, 9264, 9316, 9254, 9398] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=64956 dup_rows=24016 dup_rows_remote=21020 dedup_ratio=0.244303 router_tokens_per_expert=[747, 840, 782, 749, 758, 761, 765, 748, 791, 695, 773, 777, 781, 799, 796, 752, 761, 779, 768, 760, 748, 786, 762, 738, 788, 810, 772, 741, 779, 749, 803, 791, 748, 753, 736, 779, 760, 784, 776, 773, 777, 774, 772, 778, 786, 763, 778, 722, 805, 750, 756, 779, 799, 754, 774, 730, 726, 713, 749, 798, 782, 792, 790, 747] payload_rows_per_dst=[9316, 9332, 9188, 9376, 9304, 9364, 9234, 9174] dup_rows_per_dst=[2984, 2996, 3016, 3090, 2914, 2936, 3060, 3020] combine_rows_per_src=[9316, 9332, 9188, 9376, 9304, 9364, 9234, 9174] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=343448 combine_payload_bytes=266731520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267074968 ingress_bytes=301334528 link_bytes=301334528 token_bytes=533806488 weight_movement_bytes=34603008 total_nvlink_bytes=568409496 weight_to_token_ratio=0.064823 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133897569e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266510336 dispatch_meta_bytes=344592 combine_payload_bytes=266510336 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266854928 ingress_bytes=370319360 link_bytes=370319360 token_bytes=533365264 weight_movement_bytes=103809024 total_nvlink_bytes=637174288 weight_to_token_ratio=0.194630 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393481939e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266706944 dispatch_meta_bytes=344264 combine_payload_bytes=266706944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267051208 ingress_bytes=301309952 link_bytes=301309952 token_bytes=533758152 weight_movement_bytes=34603008 total_nvlink_bytes=568361160 weight_to_token_ratio=0.064829 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133805092e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265764864 dispatch_meta_bytes=344376 combine_payload_bytes=265764864 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266109240 ingress_bytes=334970880 link_bytes=334970880 token_bytes=531874104 weight_movement_bytes=69206016 total_nvlink_bytes=601080120 weight_to_token_ratio=0.130117 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260468454e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=64956 dup_rows=24016 dup_rows_remote=21020 dedup_ratio=0.244303 router_tokens_per_expert=[747, 840, 782, 749, 758, 761, 765, 748, 791, 695, 773, 777, 781, 799, 796, 752, 761, 779, 768, 760, 748, 786, 762, 738, 788, 810, 772, 741, 779, 749, 803, 791, 748, 753, 736, 779, 760, 784, 776, 773, 777, 774, 772, 778, 786, 763, 778, 722, 805, 750, 756, 779, 799, 754, 774, 730, 726, 713, 749, 798, 782, 792, 790, 747] payload_rows_per_dst=[9316, 9332, 9188, 9376, 9304, 9364, 9234, 9174] dup_rows_per_dst=[2984, 2996, 3016, 3090, 2914, 2936, 3060, 3020] combine_rows_per_src=[9316, 9332, 9188, 9376, 9304, 9364, 9234, 9174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=64940 dup_rows=24118 dup_rows_remote=21184 dedup_ratio=0.245341 router_tokens_per_expert=[767, 749, 768, 775, 746, 765, 788, 736, 794, 801, 777, 803, 791, 776, 743, 767, 714, 724, 763, 845, 765, 744, 766, 775, 791, 839, 785, 787, 783, 762, 723, 738, 780, 733, 748, 783, 742, 814, 741, 780, 732, 747, 757, 767, 780, 754, 782, 771, 750, 784, 785, 770, 792, 798, 764, 800, 782, 761, 750, 767, 766, 737, 736, 749] payload_rows_per_dst=[9198, 9410, 9176, 9312, 9310, 9246, 9286, 9248] dup_rows_per_dst=[2990, 3094, 3016, 3104, 2932, 2934, 3200, 2848] combine_rows_per_src=[9198, 9410, 9176, 9312, 9310, 9246, 9286, 9248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74122 payload_rows_remote=64884 dup_rows=24182 dup_rows_remote=21210 dedup_ratio=0.245992 router_tokens_per_expert=[759, 771, 761, 771, 767, 778, 788, 759, 772, 754, 775, 802, 735, 749, 775, 802, 719, 743, 801, 786, 759, 734, 764, 799, 757, 789, 728, 764, 756, 793, 787, 741, 796, 769, 775, 746, 760, 736, 717, 768, 798, 710, 788, 770, 718, 788, 774, 761, 824, 810, 753, 768, 794, 770, 781, 735, 756, 765, 780, 775, 757, 790, 790, 792] payload_rows_per_dst=[9202, 9320, 9238, 9222, 9234, 9226, 9388, 9292] dup_rows_per_dst=[3106, 3008, 2972, 3008, 2900, 2988, 3082, 3118] combine_rows_per_src=[9202, 9320, 9238, 9222, 9234, 9226, 9388, 9292] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=344496 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266338736 ingress_bytes=300597248 link_bytes=300597248 token_bytes=532332976 weight_movement_bytes=34603008 total_nvlink_bytes=566935984 weight_to_token_ratio=0.065003 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131123244e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=343968 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266706848 ingress_bytes=404774912 link_bytes=404774912 token_bytes=533069728 weight_movement_bytes=138412032 total_nvlink_bytes=671481760 weight_to_token_ratio=0.259651 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523135407e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266510336 dispatch_meta_bytes=344592 combine_payload_bytes=266510336 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266854928 ingress_bytes=370319360 link_bytes=370319360 token_bytes=533365264 weight_movement_bytes=103809024 total_nvlink_bytes=637174288 weight_to_token_ratio=0.194630 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393481939e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74436 payload_rows_remote=64816 dup_rows=23868 dup_rows_remote=20564 dedup_ratio=0.242798 router_tokens_per_expert=[761, 797, 728, 771, 730, 736, 784, 775, 704, 766, 766, 768, 783, 756, 767, 772, 740, 804, 743, 744, 774, 809, 774, 798, 718, 788, 747, 752, 784, 768, 775, 735, 780, 748, 769, 733, 796, 767, 753, 792, 723, 780, 765, 743, 771, 751, 816, 766, 770, 783, 759, 780, 777, 810, 771, 801, 770, 738, 814, 741, 760, 805, 816, 787] payload_rows_per_dst=[9168, 9026, 9262, 9240, 9300, 9366, 9454, 9620] dup_rows_per_dst=[2996, 2568, 2974, 2894, 2976, 3000, 3156, 3304] combine_rows_per_src=[9168, 9026, 9262, 9240, 9300, 9366, 9454, 9620] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74426 payload_rows_remote=65082 dup_rows=23878 dup_rows_remote=20924 dedup_ratio=0.242900 router_tokens_per_expert=[793, 750, 811, 837, 825, 740, 756, 768, 710, 788, 760, 749, 755, 745, 790, 783, 771, 742, 778, 792, 748, 757, 796, 809, 803, 764, 750, 780, 773, 789, 750, 769, 757, 794, 784, 794, 729, 724, 801, 766, 771, 773, 743, 755, 741, 793, 791, 735, 717, 736, 764, 749, 779, 748, 784, 789, 781, 789, 777, 801, 721, 702, 752, 781] payload_rows_per_dst=[9524, 9164, 9358, 9332, 9344, 9234, 9174, 9296] dup_rows_per_dst=[3036, 2996, 3028, 3024, 2954, 2970, 2958, 2912] combine_rows_per_src=[9524, 9164, 9358, 9332, 9344, 9234, 9174, 9296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=64770 dup_rows=24096 dup_rows_remote=20936 dedup_ratio=0.245117 router_tokens_per_expert=[786, 782, 768, 802, 758, 762, 736, 811, 774, 778, 797, 773, 802, 723, 747, 711, 804, 761, 726, 786, 757, 765, 778, 782, 769, 745, 778, 764, 772, 798, 756, 753, 794, 739, 788, 720, 719, 796, 741, 747, 794, 781, 779, 757, 803, 754, 728, 770, 747, 784, 748, 777, 810, 793, 733, 796, 749, 811, 796, 769, 740, 745, 767, 773] payload_rows_per_dst=[9438, 9226, 9356, 9172, 9358, 9288, 9080, 9290] dup_rows_per_dst=[3160, 3026, 2920, 2910, 3060, 3044, 2966, 3010] combine_rows_per_src=[9438, 9226, 9356, 9172, 9358, 9288, 9080, 9290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=343904 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266403680 ingress_bytes=369868800 link_bytes=369868800 token_bytes=532463456 weight_movement_bytes=103809024 total_nvlink_bytes=636272480 weight_to_token_ratio=0.194960 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391786518e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=64986 dup_rows=24098 dup_rows_remote=21054 dedup_ratio=0.245138 router_tokens_per_expert=[794, 790, 790, 782, 786, 753, 791, 758, 773, 698, 802, 797, 770, 764, 760, 726, 786, 783, 774, 788, 749, 739, 752, 803, 822, 752, 741, 812, 740, 745, 744, 748, 788, 777, 754, 756, 784, 808, 732, 733, 755, 733, 742, 740, 778, 746, 755, 810, 771, 718, 813, 735, 757, 820, 767, 772, 716, 807, 796, 781, 765, 727, 820, 784] payload_rows_per_dst=[9360, 9344, 9282, 9228, 9220, 9154, 9282, 9336] dup_rows_per_dst=[3128, 2836, 3066, 2980, 3044, 2964, 3024, 3056] combine_rows_per_src=[9360, 9344, 9282, 9228, 9220, 9154, 9282, 9336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74436 payload_rows_remote=64816 dup_rows=23868 dup_rows_remote=20564 dedup_ratio=0.242798 router_tokens_per_expert=[761, 797, 728, 771, 730, 736, 784, 775, 704, 766, 766, 768, 783, 756, 767, 772, 740, 804, 743, 744, 774, 809, 774, 798, 718, 788, 747, 752, 784, 768, 775, 735, 780, 748, 769, 733, 796, 767, 753, 792, 723, 780, 765, 743, 771, 751, 816, 766, 770, 783, 759, 780, 777, 810, 771, 801, 770, 738, 814, 741, 760, 805, 816, 787] payload_rows_per_dst=[9168, 9026, 9262, 9240, 9300, 9366, 9454, 9620] dup_rows_per_dst=[2996, 2568, 2974, 2894, 2976, 3000, 3156, 3304] combine_rows_per_src=[9168, 9026, 9262, 9240, 9300, 9366, 9454, 9620] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=344496 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266338736 ingress_bytes=300597248 link_bytes=300597248 token_bytes=532332976 weight_movement_bytes=34603008 total_nvlink_bytes=566935984 weight_to_token_ratio=0.065003 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131123244e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265764864 dispatch_meta_bytes=344376 combine_payload_bytes=265764864 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266109240 ingress_bytes=334970880 link_bytes=334970880 token_bytes=531874104 weight_movement_bytes=69206016 total_nvlink_bytes=601080120 weight_to_token_ratio=0.130117 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260468454e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=65056 dup_rows=23920 dup_rows_remote=20958 dedup_ratio=0.243327 router_tokens_per_expert=[796, 802, 771, 800, 764, 811, 739, 766, 765, 779, 764, 782, 775, 747, 765, 754, 793, 825, 737, 734, 740, 778, 764, 812, 736, 754, 744, 768, 768, 782, 818, 746, 800, 816, 764, 740, 733, 776, 756, 795, 737, 740, 795, 780, 754, 755, 798, 793, 751, 764, 768, 805, 754, 780, 748, 775, 743, 751, 775, 710, 745, 784, 735, 753] payload_rows_per_dst=[9470, 9248, 9400, 9302, 9326, 9300, 9328, 9010] dup_rows_per_dst=[3028, 3014, 2966, 2930, 3034, 3004, 2962, 2982] combine_rows_per_src=[9470, 9248, 9400, 9302, 9326, 9300, 9328, 9010] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266575872 dispatch_meta_bytes=344024 combine_payload_bytes=266575872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266919896 ingress_bytes=335781888 link_bytes=335781888 token_bytes=533495768 weight_movement_bytes=69206016 total_nvlink_bytes=602701784 weight_to_token_ratio=0.129722 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263520212e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265486336 dispatch_meta_bytes=341520 combine_payload_bytes=265486336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265827856 ingress_bytes=300089344 link_bytes=300089344 token_bytes=531314192 weight_movement_bytes=34603008 total_nvlink_bytes=565917200 weight_to_token_ratio=0.065127 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129212042e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265297920 dispatch_meta_bytes=342824 combine_payload_bytes=265297920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265640744 ingress_bytes=299900928 link_bytes=299900928 token_bytes=530938664 weight_movement_bytes=34603008 total_nvlink_bytes=565541672 weight_to_token_ratio=0.065173 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128503048e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74328 payload_rows_remote=65084 dup_rows=23976 dup_rows_remote=20926 dedup_ratio=0.243896 router_tokens_per_expert=[805, 790, 781, 726, 693, 726, 781, 720, 759, 760, 807, 804, 755, 775, 780, 808, 740, 784, 738, 799, 818, 805, 737, 741, 782, 723, 709, 798, 762, 793, 810, 770, 788, 812, 778, 727, 794, 773, 770, 746, 771, 775, 804, 766, 797, 783, 751, 756, 735, 722, 771, 743, 794, 769, 774, 801, 820, 741, 742, 728, 769, 750, 755, 768] payload_rows_per_dst=[9168, 9460, 9258, 9244, 9330, 9354, 9250, 9264] dup_rows_per_dst=[2876, 3036, 3066, 3050, 3046, 3052, 2968, 2882] combine_rows_per_src=[9168, 9460, 9258, 9244, 9330, 9354, 9250, 9264] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266182656 dispatch_meta_bytes=344160 combine_payload_bytes=266182656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266526816 ingress_bytes=300785664 link_bytes=300785664 token_bytes=532709472 weight_movement_bytes=34603008 total_nvlink_bytes=567312480 weight_to_token_ratio=0.064957 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131832238e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=65056 dup_rows=24032 dup_rows_remote=20966 dedup_ratio=0.244466 router_tokens_per_expert=[708, 757, 780, 753, 813, 755, 736, 802, 793, 756, 791, 815, 754, 767, 811, 829, 803, 741, 804, 791, 722, 761, 731, 788, 730, 788, 828, 788, 812, 775, 772, 785, 719, 755, 779, 785, 791, 763, 724, 779, 770, 747, 763, 813, 789, 752, 699, 760, 750, 764, 742, 799, 756, 792, 710, 745, 784, 751, 739, 792, 756, 726, 803, 716] payload_rows_per_dst=[9226, 9520, 9216, 9360, 9258, 9246, 9250, 9196] dup_rows_per_dst=[2982, 3112, 3066, 3196, 2932, 2940, 2866, 2938] combine_rows_per_src=[9226, 9520, 9216, 9360, 9258, 9246, 9250, 9196] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65050 dup_rows=23924 dup_rows_remote=20954 dedup_ratio=0.243368 router_tokens_per_expert=[740, 765, 742, 746, 786, 807, 752, 799, 760, 788, 773, 720, 755, 761, 769, 752, 848, 709, 738, 753, 744, 776, 744, 773, 755, 735, 761, 813, 792, 725, 806, 740, 720, 800, 764, 740, 803, 769, 776, 779, 767, 790, 741, 762, 769, 784, 744, 793, 756, 792, 735, 806, 795, 746, 798, 751, 784, 799, 759, 794, 768, 769, 774, 798] payload_rows_per_dst=[9320, 9208, 9170, 9250, 9338, 9330, 9280, 9484] dup_rows_per_dst=[2954, 2948, 3000, 3004, 2964, 2970, 3078, 3006] combine_rows_per_src=[9320, 9208, 9170, 9250, 9338, 9330, 9280, 9484] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=64770 dup_rows=24096 dup_rows_remote=20936 dedup_ratio=0.245117 router_tokens_per_expert=[786, 782, 768, 802, 758, 762, 736, 811, 774, 778, 797, 773, 802, 723, 747, 711, 804, 761, 726, 786, 757, 765, 778, 782, 769, 745, 778, 764, 772, 798, 756, 753, 794, 739, 788, 720, 719, 796, 741, 747, 794, 781, 779, 757, 803, 754, 728, 770, 747, 784, 748, 777, 810, 793, 733, 796, 749, 811, 796, 769, 740, 745, 767, 773] payload_rows_per_dst=[9438, 9226, 9356, 9172, 9358, 9288, 9080, 9290] dup_rows_per_dst=[3160, 3026, 2920, 2910, 3060, 3044, 2966, 3010] combine_rows_per_src=[9438, 9226, 9356, 9172, 9358, 9288, 9080, 9290] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=65056 dup_rows=23920 dup_rows_remote=20958 dedup_ratio=0.243327 router_tokens_per_expert=[796, 802, 771, 800, 764, 811, 739, 766, 765, 779, 764, 782, 775, 747, 765, 754, 793, 825, 737, 734, 740, 778, 764, 812, 736, 754, 744, 768, 768, 782, 818, 746, 800, 816, 764, 740, 733, 776, 756, 795, 737, 740, 795, 780, 754, 755, 798, 793, 751, 764, 768, 805, 754, 780, 748, 775, 743, 751, 775, 710, 745, 784, 735, 753] payload_rows_per_dst=[9470, 9248, 9400, 9302, 9326, 9300, 9328, 9010] dup_rows_per_dst=[3028, 3014, 2966, 2930, 3034, 3004, 2962, 2982] combine_rows_per_src=[9470, 9248, 9400, 9302, 9326, 9300, 9328, 9010] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74226 payload_rows_remote=64972 dup_rows=24078 dup_rows_remote=21170 dedup_ratio=0.244934 router_tokens_per_expert=[780, 803, 801, 774, 808, 771, 808, 717, 750, 774, 719, 774, 756, 786, 773, 749, 824, 788, 782, 790, 774, 748, 746, 801, 704, 736, 779, 774, 776, 717, 751, 762, 793, 765, 751, 778, 764, 743, 747, 828, 767, 728, 825, 732, 751, 841, 771, 795, 734, 747, 770, 757, 746, 761, 750, 797, 801, 760, 739, 730, 766, 759, 816, 745] payload_rows_per_dst=[9370, 9254, 9394, 9194, 9252, 9322, 9214, 9226] dup_rows_per_dst=[3154, 2908, 3112, 2804, 3086, 3098, 2910, 3006] combine_rows_per_src=[9370, 9254, 9394, 9194, 9252, 9322, 9214, 9226] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265486336 dispatch_meta_bytes=341520 combine_payload_bytes=265486336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265827856 ingress_bytes=300089344 link_bytes=300089344 token_bytes=531314192 weight_movement_bytes=34603008 total_nvlink_bytes=565917200 weight_to_token_ratio=0.065127 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129212042e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266469376 dispatch_meta_bytes=344056 combine_payload_bytes=266469376 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266813432 ingress_bytes=370278400 link_bytes=370278400 token_bytes=533282808 weight_movement_bytes=103809024 total_nvlink_bytes=637091832 weight_to_token_ratio=0.194660 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393327809e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266584064 dispatch_meta_bytes=344040 combine_payload_bytes=266584064 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266928104 ingress_bytes=301187072 link_bytes=301187072 token_bytes=533512168 weight_movement_bytes=34603008 total_nvlink_bytes=568115176 weight_to_token_ratio=0.064859 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133342704e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266469376 dispatch_meta_bytes=344088 combine_payload_bytes=266469376 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266813464 ingress_bytes=335675392 link_bytes=335675392 token_bytes=533282840 weight_movement_bytes=69206016 total_nvlink_bytes=602488856 weight_to_token_ratio=0.129774 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263119476e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266444800 dispatch_meta_bytes=344016 combine_payload_bytes=266444800 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266788816 ingress_bytes=404856832 link_bytes=404856832 token_bytes=533233616 weight_movement_bytes=138412032 total_nvlink_bytes=671645648 weight_to_token_ratio=0.259571 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523443665e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265297920 dispatch_meta_bytes=342824 combine_payload_bytes=265297920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265640744 ingress_bytes=299900928 link_bytes=299900928 token_bytes=530938664 weight_movement_bytes=34603008 total_nvlink_bytes=565541672 weight_to_token_ratio=0.065173 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128503048e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74226 payload_rows_remote=64972 dup_rows=24078 dup_rows_remote=21170 dedup_ratio=0.244934 router_tokens_per_expert=[780, 803, 801, 774, 808, 771, 808, 717, 750, 774, 719, 774, 756, 786, 773, 749, 824, 788, 782, 790, 774, 748, 746, 801, 704, 736, 779, 774, 776, 717, 751, 762, 793, 765, 751, 778, 764, 743, 747, 828, 767, 728, 825, 732, 751, 841, 771, 795, 734, 747, 770, 757, 746, 761, 750, 797, 801, 760, 739, 730, 766, 759, 816, 745] payload_rows_per_dst=[9370, 9254, 9394, 9194, 9252, 9322, 9214, 9226] dup_rows_per_dst=[3154, 2908, 3112, 2804, 3086, 3098, 2910, 3006] combine_rows_per_src=[9370, 9254, 9394, 9194, 9252, 9322, 9214, 9226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65050 dup_rows=23924 dup_rows_remote=20954 dedup_ratio=0.243368 router_tokens_per_expert=[740, 765, 742, 746, 786, 807, 752, 799, 760, 788, 773, 720, 755, 761, 769, 752, 848, 709, 738, 753, 744, 776, 744, 773, 755, 735, 761, 813, 792, 725, 806, 740, 720, 800, 764, 740, 803, 769, 776, 779, 767, 790, 741, 762, 769, 784, 744, 793, 756, 792, 735, 806, 795, 746, 798, 751, 784, 799, 759, 794, 768, 769, 774, 798] payload_rows_per_dst=[9320, 9208, 9170, 9250, 9338, 9330, 9280, 9484] dup_rows_per_dst=[2954, 2948, 3000, 3004, 2964, 2970, 3078, 3006] combine_rows_per_src=[9320, 9208, 9170, 9250, 9338, 9330, 9280, 9484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=65056 dup_rows=24032 dup_rows_remote=20966 dedup_ratio=0.244466 router_tokens_per_expert=[708, 757, 780, 753, 813, 755, 736, 802, 793, 756, 791, 815, 754, 767, 811, 829, 803, 741, 804, 791, 722, 761, 731, 788, 730, 788, 828, 788, 812, 775, 772, 785, 719, 755, 779, 785, 791, 763, 724, 779, 770, 747, 763, 813, 789, 752, 699, 760, 750, 764, 742, 799, 756, 792, 710, 745, 784, 751, 739, 792, 756, 726, 803, 716] payload_rows_per_dst=[9226, 9520, 9216, 9360, 9258, 9246, 9250, 9196] dup_rows_per_dst=[2982, 3112, 3066, 3196, 2932, 2940, 2866, 2938] combine_rows_per_src=[9226, 9520, 9216, 9360, 9258, 9246, 9250, 9196] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74414 payload_rows_remote=65048 dup_rows=23890 dup_rows_remote=20912 dedup_ratio=0.243022 router_tokens_per_expert=[823, 777, 750, 768, 783, 795, 748, 743, 744, 745, 823, 773, 807, 748, 748, 812, 790, 802, 786, 782, 766, 778, 799, 785, 740, 772, 746, 809, 732, 768, 731, 750, 781, 764, 755, 817, 783, 799, 732, 775, 732, 794, 804, 779, 760, 737, 762, 698, 717, 745, 808, 754, 759, 742, 761, 764, 727, 770, 778, 765, 801, 771, 780, 715] payload_rows_per_dst=[9098, 9406, 9362, 9266, 9492, 9108, 9316, 9366] dup_rows_per_dst=[2762, 2994, 3214, 3018, 3116, 2782, 3026, 2978] combine_rows_per_src=[9098, 9406, 9362, 9266, 9492, 9108, 9316, 9366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266469376 dispatch_meta_bytes=344056 combine_payload_bytes=266469376 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266813432 ingress_bytes=370278400 link_bytes=370278400 token_bytes=533282808 weight_movement_bytes=103809024 total_nvlink_bytes=637091832 weight_to_token_ratio=0.194660 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393327809e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=65144 dup_rows=23860 dup_rows_remote=20792 dedup_ratio=0.242716 router_tokens_per_expert=[768, 763, 756, 768, 774, 761, 786, 792, 716, 773, 739, 733, 783, 757, 784, 811, 807, 803, 776, 827, 781, 750, 771, 765, 756, 739, 737, 744, 763, 796, 773, 754, 771, 766, 784, 750, 772, 790, 782, 769, 773, 711, 766, 763, 785, 780, 786, 802, 764, 747, 763, 749, 783, 751, 748, 803, 745, 729, 786, 771, 761, 748, 775, 773] payload_rows_per_dst=[9382, 9268, 9492, 9228, 9300, 9306, 9172, 9296] dup_rows_per_dst=[2954, 2924, 3068, 2896, 3068, 3026, 3044, 2880] combine_rows_per_src=[9382, 9268, 9492, 9228, 9300, 9306, 9172, 9296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74196 payload_rows_remote=64916 dup_rows=24108 dup_rows_remote=21178 dedup_ratio=0.245239 router_tokens_per_expert=[724, 794, 769, 720, 744, 748, 794, 780, 791, 738, 756, 741, 754, 761, 804, 755, 714, 777, 759, 781, 747, 730, 773, 790, 832, 763, 724, 729, 781, 785, 750, 793, 773, 825, 716, 797, 760, 752, 735, 760, 781, 783, 768, 792, 788, 769, 762, 766, 785, 728, 775, 781, 749, 782, 769, 818, 764, 777, 771, 821, 725, 822, 758, 799] payload_rows_per_dst=[9280, 9450, 9290, 9300, 9168, 9046, 9330, 9332] dup_rows_per_dst=[2930, 3176, 3016, 3014, 3068, 2718, 3044, 3142] combine_rows_per_src=[9280, 9450, 9290, 9300, 9168, 9046, 9330, 9332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266125312 dispatch_meta_bytes=344568 combine_payload_bytes=266125312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266469880 ingress_bytes=300728320 link_bytes=300728320 token_bytes=532595192 weight_movement_bytes=34603008 total_nvlink_bytes=567198200 weight_to_token_ratio=0.064971 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131616458e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74558 payload_rows_remote=65132 dup_rows=23746 dup_rows_remote=20618 dedup_ratio=0.241557 router_tokens_per_expert=[714, 763, 816, 749, 768, 781, 788, 806, 832, 753, 772, 754, 752, 782, 755, 777, 728, 763, 825, 773, 721, 767, 766, 710, 796, 769, 804, 725, 785, 768, 755, 770, 797, 780, 785, 785, 775, 832, 781, 742, 812, 786, 793, 752, 790, 755, 740, 819, 752, 770, 734, 728, 798, 766, 743, 783, 800, 763, 715, 743, 704, 731, 763, 748] payload_rows_per_dst=[9396, 9384, 9186, 9382, 9426, 9462, 9254, 9068] dup_rows_per_dst=[2974, 2970, 2920, 2962, 3128, 3032, 2894, 2866] combine_rows_per_src=[9396, 9384, 9186, 9382, 9426, 9462, 9254, 9068] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266444800 dispatch_meta_bytes=344016 combine_payload_bytes=266444800 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266788816 ingress_bytes=404856832 link_bytes=404856832 token_bytes=533233616 weight_movement_bytes=138412032 total_nvlink_bytes=671645648 weight_to_token_ratio=0.259571 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523443665e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74414 payload_rows_remote=65048 dup_rows=23890 dup_rows_remote=20912 dedup_ratio=0.243022 router_tokens_per_expert=[823, 777, 750, 768, 783, 795, 748, 743, 744, 745, 823, 773, 807, 748, 748, 812, 790, 802, 786, 782, 766, 778, 799, 785, 740, 772, 746, 809, 732, 768, 731, 750, 781, 764, 755, 817, 783, 799, 732, 775, 732, 794, 804, 779, 760, 737, 762, 698, 717, 745, 808, 754, 759, 742, 761, 764, 727, 770, 778, 765, 801, 771, 780, 715] payload_rows_per_dst=[9098, 9406, 9362, 9266, 9492, 9108, 9316, 9366] dup_rows_per_dst=[2762, 2994, 3214, 3018, 3116, 2782, 3026, 2978] combine_rows_per_src=[9098, 9406, 9362, 9266, 9492, 9108, 9316, 9366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266469376 dispatch_meta_bytes=344088 combine_payload_bytes=266469376 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266813464 ingress_bytes=335675392 link_bytes=335675392 token_bytes=533282840 weight_movement_bytes=69206016 total_nvlink_bytes=602488856 weight_to_token_ratio=0.129774 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263119476e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74448 payload_rows_remote=65190 dup_rows=23856 dup_rows_remote=20886 dedup_ratio=0.242676 router_tokens_per_expert=[778, 749, 799, 789, 769, 758, 755, 760, 759, 819, 754, 793, 784, 700, 751, 799, 776, 799, 766, 760, 792, 787, 763, 778, 760, 783, 742, 785, 776, 805, 710, 713, 719, 784, 817, 806, 776, 745, 738, 775, 787, 790, 756, 774, 818, 776, 757, 750, 804, 773, 758, 739, 749, 753, 774, 764, 766, 808, 737, 747, 739, 758, 775, 729] payload_rows_per_dst=[9290, 9294, 9330, 9356, 9352, 9464, 9258, 9104] dup_rows_per_dst=[3024, 3024, 3112, 2792, 2968, 2952, 2970, 3014] combine_rows_per_src=[9290, 9294, 9330, 9356, 9352, 9464, 9258, 9104] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266436608 dispatch_meta_bytes=343840 combine_payload_bytes=266436608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266780448 ingress_bytes=301039616 link_bytes=301039616 token_bytes=533217056 weight_movement_bytes=34603008 total_nvlink_bytes=567820064 weight_to_token_ratio=0.064895 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132787839e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266780672 dispatch_meta_bytes=343000 combine_payload_bytes=266780672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267123672 ingress_bytes=301383680 link_bytes=301383680 token_bytes=533904344 weight_movement_bytes=34603008 total_nvlink_bytes=568507352 weight_to_token_ratio=0.064811 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134082524e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74332 payload_rows_remote=65002 dup_rows=23972 dup_rows_remote=20978 dedup_ratio=0.243856 router_tokens_per_expert=[781, 768, 740, 816, 757, 790, 802, 754, 720, 759, 728, 767, 802, 778, 749, 745, 771, 796, 780, 766, 766, 732, 775, 776, 737, 775, 808, 775, 762, 749, 776, 734, 748, 731, 754, 742, 767, 731, 735, 780, 775, 804, 768, 802, 773, 761, 782, 810, 765, 758, 756, 799, 804, 745, 717, 794, 791, 754, 814, 775, 764, 734, 802, 783] payload_rows_per_dst=[9410, 9202, 9330, 9260, 9064, 9426, 9266, 9374] dup_rows_per_dst=[3006, 2894, 2994, 2972, 2912, 3124, 3010, 3060] combine_rows_per_src=[9410, 9202, 9330, 9260, 9064, 9426, 9266, 9374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266829824 dispatch_meta_bytes=343744 combine_payload_bytes=266829824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267173568 ingress_bytes=301432832 link_bytes=301432832 token_bytes=534003392 weight_movement_bytes=34603008 total_nvlink_bytes=568606400 weight_to_token_ratio=0.064799 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134267479e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74052 payload_rows_remote=64554 dup_rows=24252 dup_rows_remote=21100 dedup_ratio=0.246704 router_tokens_per_expert=[749, 816, 765, 798, 789, 750, 780, 771, 768, 769, 749, 691, 738, 789, 795, 805, 756, 731, 713, 750, 741, 725, 800, 747, 818, 744, 851, 784, 794, 772, 795, 767, 772, 715, 777, 745, 797, 768, 759, 781, 758, 812, 795, 742, 785, 751, 765, 775, 787, 792, 771, 780, 752, 772, 761, 764, 772, 814, 746, 775, 785, 752, 718, 704] payload_rows_per_dst=[9290, 9250, 8992, 9498, 9312, 9296, 9250, 9164] dup_rows_per_dst=[3146, 2958, 2934, 3152, 2916, 3070, 3108, 2968] combine_rows_per_src=[9290, 9250, 8992, 9498, 9312, 9296, 9250, 9164] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74516 payload_rows_remote=65082 dup_rows=23788 dup_rows_remote=20722 dedup_ratio=0.241984 router_tokens_per_expert=[820, 763, 732, 790, 745, 751, 822, 741, 762, 755, 775, 799, 749, 708, 782, 765, 753, 785, 744, 798, 791, 740, 742, 764, 751, 775, 753, 804, 765, 778, 804, 735, 760, 831, 772, 785, 727, 741, 769, 782, 776, 798, 834, 777, 707, 784, 781, 793, 744, 800, 763, 735, 805, 769, 767, 778, 763, 742, 781, 762, 787, 714, 756, 728] payload_rows_per_dst=[9284, 9236, 9310, 9312, 9400, 9434, 9300, 9240] dup_rows_per_dst=[3044, 2954, 2924, 3018, 2934, 3066, 3022, 2826] combine_rows_per_src=[9284, 9236, 9310, 9312, 9400, 9434, 9300, 9240] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74196 payload_rows_remote=64916 dup_rows=24108 dup_rows_remote=21178 dedup_ratio=0.245239 router_tokens_per_expert=[724, 794, 769, 720, 744, 748, 794, 780, 791, 738, 756, 741, 754, 761, 804, 755, 714, 777, 759, 781, 747, 730, 773, 790, 832, 763, 724, 729, 781, 785, 750, 793, 773, 825, 716, 797, 760, 752, 735, 760, 781, 783, 768, 792, 788, 769, 762, 766, 785, 728, 775, 781, 749, 782, 769, 818, 764, 777, 771, 821, 725, 822, 758, 799] payload_rows_per_dst=[9280, 9450, 9290, 9300, 9168, 9046, 9330, 9332] dup_rows_per_dst=[2930, 3176, 3016, 3014, 3068, 2718, 3044, 3142] combine_rows_per_src=[9280, 9450, 9290, 9300, 9168, 9046, 9330, 9332] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74448 payload_rows_remote=65190 dup_rows=23856 dup_rows_remote=20886 dedup_ratio=0.242676 router_tokens_per_expert=[778, 749, 799, 789, 769, 758, 755, 760, 759, 819, 754, 793, 784, 700, 751, 799, 776, 799, 766, 760, 792, 787, 763, 778, 760, 783, 742, 785, 776, 805, 710, 713, 719, 784, 817, 806, 776, 745, 738, 775, 787, 790, 756, 774, 818, 776, 757, 750, 804, 773, 758, 739, 749, 753, 774, 764, 766, 808, 737, 747, 739, 758, 775, 729] payload_rows_per_dst=[9290, 9294, 9330, 9356, 9352, 9464, 9258, 9104] dup_rows_per_dst=[3024, 3024, 3112, 2792, 2968, 2952, 2970, 3014] combine_rows_per_src=[9290, 9294, 9330, 9356, 9352, 9464, 9258, 9104] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74366 payload_rows_remote=64900 dup_rows=23938 dup_rows_remote=20884 dedup_ratio=0.243510 router_tokens_per_expert=[771, 788, 802, 780, 772, 784, 762, 764, 784, 769, 771, 803, 809, 772, 787, 765, 746, 754, 768, 794, 754, 758, 754, 690, 743, 787, 757, 752, 713, 753, 795, 781, 769, 782, 768, 736, 817, 754, 756, 780, 771, 757, 780, 711, 755, 807, 744, 747, 774, 755, 737, 780, 756, 795, 826, 738, 808, 754, 797, 781, 760, 727, 769, 779] payload_rows_per_dst=[9434, 9466, 9154, 9186, 9388, 9122, 9264, 9352] dup_rows_per_dst=[3012, 3054, 2882, 2976, 2936, 3022, 3058, 2998] combine_rows_per_src=[9434, 9466, 9154, 9186, 9388, 9122, 9264, 9352] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=343920 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266592112 ingress_bytes=300851200 link_bytes=300851200 token_bytes=532840304 weight_movement_bytes=34603008 total_nvlink_bytes=567443312 weight_to_token_ratio=0.064941 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132078845e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265895936 dispatch_meta_bytes=344376 combine_payload_bytes=265895936 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266240312 ingress_bytes=404307968 link_bytes=404307968 token_bytes=532136248 weight_movement_bytes=138412032 total_nvlink_bytes=670548280 weight_to_token_ratio=0.260106 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521378334e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266436608 dispatch_meta_bytes=343840 combine_payload_bytes=266436608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266780448 ingress_bytes=301039616 link_bytes=301039616 token_bytes=533217056 weight_movement_bytes=34603008 total_nvlink_bytes=567820064 weight_to_token_ratio=0.064895 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132787839e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=267018240 dispatch_meta_bytes=344304 combine_payload_bytes=267018240 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267362544 ingress_bytes=336224256 link_bytes=336224256 token_bytes=534380784 weight_movement_bytes=69206016 total_nvlink_bytes=603586800 weight_to_token_ratio=0.129507 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265184807e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=264413184 dispatch_meta_bytes=342616 combine_payload_bytes=264413184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264755800 ingress_bytes=299016192 link_bytes=299016192 token_bytes=529168984 weight_movement_bytes=34603008 total_nvlink_bytes=563771992 weight_to_token_ratio=0.065391 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125173858e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74366 payload_rows_remote=64900 dup_rows=23938 dup_rows_remote=20884 dedup_ratio=0.243510 router_tokens_per_expert=[771, 788, 802, 780, 772, 784, 762, 764, 784, 769, 771, 803, 809, 772, 787, 765, 746, 754, 768, 794, 754, 758, 754, 690, 743, 787, 757, 752, 713, 753, 795, 781, 769, 782, 768, 736, 817, 754, 756, 780, 771, 757, 780, 711, 755, 807, 744, 747, 774, 755, 737, 780, 756, 795, 826, 738, 808, 754, 797, 781, 760, 727, 769, 779] payload_rows_per_dst=[9434, 9466, 9154, 9186, 9388, 9122, 9264, 9352] dup_rows_per_dst=[3012, 3054, 2882, 2976, 2936, 3022, 3058, 2998] combine_rows_per_src=[9434, 9466, 9154, 9186, 9388, 9122, 9264, 9352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74516 payload_rows_remote=65082 dup_rows=23788 dup_rows_remote=20722 dedup_ratio=0.241984 router_tokens_per_expert=[820, 763, 732, 790, 745, 751, 822, 741, 762, 755, 775, 799, 749, 708, 782, 765, 753, 785, 744, 798, 791, 740, 742, 764, 751, 775, 753, 804, 765, 778, 804, 735, 760, 831, 772, 785, 727, 741, 769, 782, 776, 798, 834, 777, 707, 784, 781, 793, 744, 800, 763, 735, 805, 769, 767, 778, 763, 742, 781, 762, 787, 714, 756, 728] payload_rows_per_dst=[9284, 9236, 9310, 9312, 9400, 9434, 9300, 9240] dup_rows_per_dst=[3044, 2954, 2924, 3018, 2934, 3066, 3022, 2826] combine_rows_per_src=[9284, 9236, 9310, 9312, 9400, 9434, 9300, 9240] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266575872 dispatch_meta_bytes=343216 combine_payload_bytes=266575872 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266919088 ingress_bytes=335781888 link_bytes=335781888 token_bytes=533494960 weight_movement_bytes=69206016 total_nvlink_bytes=602700976 weight_to_token_ratio=0.129722 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263520212e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265895936 dispatch_meta_bytes=344376 combine_payload_bytes=265895936 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266240312 ingress_bytes=404307968 link_bytes=404307968 token_bytes=532136248 weight_movement_bytes=138412032 total_nvlink_bytes=670548280 weight_to_token_ratio=0.260106 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521378334e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74414 payload_rows_remote=65030 dup_rows=23890 dup_rows_remote=20844 dedup_ratio=0.243022 router_tokens_per_expert=[729, 732, 731, 715, 761, 766, 744, 740, 745, 755, 754, 789, 761, 772, 781, 742, 797, 749, 765, 727, 717, 772, 757, 735, 798, 748, 815, 783, 807, 767, 799, 796, 720, 777, 741, 774, 762, 816, 831, 765, 732, 796, 764, 775, 741, 824, 810, 737, 812, 797, 797, 752, 783, 747, 794, 773, 789, 796, 763, 780, 771, 756, 750, 778] payload_rows_per_dst=[8988, 9234, 9176, 9504, 9276, 9418, 9434, 9384] dup_rows_per_dst=[2848, 2964, 2862, 3122, 3038, 2876, 3134, 3046] combine_rows_per_src=[8988, 9234, 9176, 9504, 9276, 9418, 9434, 9384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74110 payload_rows_remote=64712 dup_rows=24194 dup_rows_remote=21104 dedup_ratio=0.246114 router_tokens_per_expert=[811, 790, 747, 803, 804, 766, 762, 749, 766, 740, 802, 758, 750, 776, 749, 774, 777, 744, 773, 796, 768, 722, 788, 733, 758, 820, 748, 750, 816, 766, 740, 715, 742, 786, 807, 750, 772, 786, 801, 800, 746, 803, 786, 741, 743, 787, 758, 738, 781, 728, 818, 723, 776, 745, 712, 784, 753, 746, 804, 763, 790, 769, 791, 762] payload_rows_per_dst=[9412, 9164, 9260, 9216, 9398, 9214, 9142, 9304] dup_rows_per_dst=[3052, 3066, 2942, 3010, 3090, 2990, 2992, 3052] combine_rows_per_src=[9412, 9164, 9260, 9216, 9398, 9214, 9142, 9304] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=267018240 dispatch_meta_bytes=344304 combine_payload_bytes=267018240 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267362544 ingress_bytes=336224256 link_bytes=336224256 token_bytes=534380784 weight_movement_bytes=69206016 total_nvlink_bytes=603586800 weight_to_token_ratio=0.129507 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265184807e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74332 payload_rows_remote=65002 dup_rows=23972 dup_rows_remote=20978 dedup_ratio=0.243856 router_tokens_per_expert=[781, 768, 740, 816, 757, 790, 802, 754, 720, 759, 728, 767, 802, 778, 749, 745, 771, 796, 780, 766, 766, 732, 775, 776, 737, 775, 808, 775, 762, 749, 776, 734, 748, 731, 754, 742, 767, 731, 735, 780, 775, 804, 768, 802, 773, 761, 782, 810, 765, 758, 756, 799, 804, 745, 717, 794, 791, 754, 814, 775, 764, 734, 802, 783] payload_rows_per_dst=[9410, 9202, 9330, 9260, 9064, 9426, 9266, 9374] dup_rows_per_dst=[3006, 2894, 2994, 2972, 2912, 3124, 3010, 3060] combine_rows_per_src=[9410, 9202, 9330, 9260, 9064, 9426, 9266, 9374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=64986 dup_rows=24098 dup_rows_remote=21054 dedup_ratio=0.245138 router_tokens_per_expert=[794, 790, 790, 782, 786, 753, 791, 758, 773, 698, 802, 797, 770, 764, 760, 726, 786, 783, 774, 788, 749, 739, 752, 803, 822, 752, 741, 812, 740, 745, 744, 748, 788, 777, 754, 756, 784, 808, 732, 733, 755, 733, 742, 740, 778, 746, 755, 810, 771, 718, 813, 735, 757, 820, 767, 772, 716, 807, 796, 781, 765, 727, 820, 784] payload_rows_per_dst=[9360, 9344, 9282, 9228, 9220, 9154, 9282, 9336] dup_rows_per_dst=[3128, 2836, 3066, 2980, 3044, 2964, 3024, 3056] combine_rows_per_src=[9360, 9344, 9282, 9228, 9220, 9154, 9282, 9336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74028 payload_rows_remote=64714 dup_rows=24276 dup_rows_remote=21050 dedup_ratio=0.246948 router_tokens_per_expert=[756, 771, 741, 759, 775, 760, 767, 813, 793, 783, 736, 742, 760, 745, 722, 737, 783, 764, 786, 791, 751, 741, 753, 797, 722, 823, 759, 778, 775, 723, 773, 751, 807, 756, 768, 778, 785, 788, 786, 753, 784, 734, 788, 769, 774, 762, 798, 798, 729, 795, 785, 740, 806, 721, 769, 791, 805, 756, 752, 787, 762, 783, 736, 777] payload_rows_per_dst=[9314, 9268, 9388, 9152, 9134, 9374, 9176, 9222] dup_rows_per_dst=[3226, 3080, 3134, 2874, 2928, 3034, 2906, 3094] combine_rows_per_src=[9314, 9268, 9388, 9152, 9134, 9374, 9176, 9222] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265830400 dispatch_meta_bytes=343136 combine_payload_bytes=265830400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266173536 ingress_bytes=300433408 link_bytes=300433408 token_bytes=532003936 weight_movement_bytes=34603008 total_nvlink_bytes=566606944 weight_to_token_ratio=0.065043 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130506727e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74414 payload_rows_remote=65030 dup_rows=23890 dup_rows_remote=20844 dedup_ratio=0.243022 router_tokens_per_expert=[729, 732, 731, 715, 761, 766, 744, 740, 745, 755, 754, 789, 761, 772, 781, 742, 797, 749, 765, 727, 717, 772, 757, 735, 798, 748, 815, 783, 807, 767, 799, 796, 720, 777, 741, 774, 762, 816, 831, 765, 732, 796, 764, 775, 741, 824, 810, 737, 812, 797, 797, 752, 783, 747, 794, 773, 789, 796, 763, 780, 771, 756, 750, 778] payload_rows_per_dst=[8988, 9234, 9176, 9504, 9276, 9418, 9434, 9384] dup_rows_per_dst=[2848, 2964, 2862, 3122, 3038, 2876, 3134, 3046] combine_rows_per_src=[8988, 9234, 9176, 9504, 9276, 9418, 9434, 9384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266575872 dispatch_meta_bytes=343216 combine_payload_bytes=266575872 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266919088 ingress_bytes=335781888 link_bytes=335781888 token_bytes=533494960 weight_movement_bytes=69206016 total_nvlink_bytes=602700976 weight_to_token_ratio=0.129722 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263520212e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74230 payload_rows_remote=65020 dup_rows=24074 dup_rows_remote=20960 dedup_ratio=0.244893 router_tokens_per_expert=[773, 746, 797, 759, 757, 764, 767, 744, 756, 785, 724, 780, 795, 722, 798, 791, 755, 787, 786, 732, 773, 804, 787, 771, 805, 748, 810, 728, 745, 766, 783, 729, 737, 745, 731, 739, 788, 782, 777, 732, 768, 765, 800, 789, 778, 805, 776, 779, 757, 780, 765, 773, 783, 766, 804, 734, 770, 729, 750, 782, 751, 783, 750, 817] payload_rows_per_dst=[9246, 9262, 9306, 9294, 9166, 9412, 9210, 9334] dup_rows_per_dst=[2968, 3040, 3084, 2934, 2896, 3108, 3114, 2930] combine_rows_per_src=[9246, 9262, 9306, 9294, 9166, 9412, 9210, 9334] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=343920 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266592112 ingress_bytes=300851200 link_bytes=300851200 token_bytes=532840304 weight_movement_bytes=34603008 total_nvlink_bytes=567443312 weight_to_token_ratio=0.064941 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132078845e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=343496 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266706376 ingress_bytes=300965888 link_bytes=300965888 token_bytes=533069256 weight_movement_bytes=34603008 total_nvlink_bytes=567672264 weight_to_token_ratio=0.064913 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132510407e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265060352 dispatch_meta_bytes=343264 combine_payload_bytes=265060352 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265403616 ingress_bytes=403472384 link_bytes=403472384 token_bytes=530463968 weight_movement_bytes=138412032 total_nvlink_bytes=668876000 weight_to_token_ratio=0.260926 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.518234099e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74388 payload_rows_remote=65082 dup_rows=23916 dup_rows_remote=20910 dedup_ratio=0.243286 router_tokens_per_expert=[752, 754, 807, 812, 743, 758, 762, 746, 753, 760, 739, 766, 754, 764, 784, 780, 785, 758, 784, 764, 732, 787, 774, 772, 792, 772, 743, 733, 736, 769, 792, 774, 759, 745, 776, 771, 781, 786, 757, 778, 785, 755, 739, 752, 821, 786, 718, 770, 798, 730, 818, 758, 820, 751, 742, 790, 764, 787, 768, 778, 814, 768, 727, 759] payload_rows_per_dst=[9384, 9280, 9306, 9244, 9286, 9228, 9396, 9264] dup_rows_per_dst=[2884, 2920, 3006, 2978, 3020, 3024, 3018, 3066] combine_rows_per_src=[9384, 9280, 9306, 9244, 9286, 9228, 9396, 9264] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74300 payload_rows_remote=65028 dup_rows=24004 dup_rows_remote=21024 dedup_ratio=0.244181 router_tokens_per_expert=[761, 781, 764, 784, 772, 740, 804, 806, 741, 754, 776, 808, 744, 768, 746, 772, 735, 786, 864, 797, 768, 761, 736, 736, 788, 747, 771, 786, 780, 763, 746, 774, 751, 766, 762, 767, 784, 809, 739, 799, 777, 770, 783, 743, 788, 737, 763, 765, 745, 746, 725, 690, 791, 761, 782, 808, 767, 761, 739, 762, 787, 808, 776, 742] payload_rows_per_dst=[9328, 9174, 9310, 9340, 9406, 9272, 9204, 9266] dup_rows_per_dst=[3096, 3044, 3056, 2970, 2948, 2980, 2892, 3018] combine_rows_per_src=[9328, 9174, 9310, 9340, 9406, 9272, 9204, 9266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266182656 dispatch_meta_bytes=344160 combine_payload_bytes=266182656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266526816 ingress_bytes=300785664 link_bytes=300785664 token_bytes=532709472 weight_movement_bytes=34603008 total_nvlink_bytes=567312480 weight_to_token_ratio=0.064957 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131832238e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74580 payload_rows_remote=65174 dup_rows=23724 dup_rows_remote=20658 dedup_ratio=0.241333 router_tokens_per_expert=[741, 806, 708, 764, 796, 776, 750, 761, 766, 736, 766, 771, 794, 783, 755, 794, 771, 775, 796, 770, 856, 802, 789, 736, 760, 784, 733, 765, 799, 802, 801, 792, 765, 745, 777, 774, 779, 756, 764, 794, 771, 746, 736, 788, 779, 736, 793, 754, 793, 751, 733, 763, 754, 763, 797, 775, 763, 711, 773, 703, 716, 758, 786, 758] payload_rows_per_dst=[9270, 9324, 9512, 9406, 9328, 9262, 9346, 9132] dup_rows_per_dst=[2934, 3006, 3078, 3066, 2980, 2944, 2912, 2804] combine_rows_per_src=[9270, 9324, 9512, 9406, 9328, 9262, 9346, 9132] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74028 payload_rows_remote=64714 dup_rows=24276 dup_rows_remote=21050 dedup_ratio=0.246948 router_tokens_per_expert=[756, 771, 741, 759, 775, 760, 767, 813, 793, 783, 736, 742, 760, 745, 722, 737, 783, 764, 786, 791, 751, 741, 753, 797, 722, 823, 759, 778, 775, 723, 773, 751, 807, 756, 768, 778, 785, 788, 786, 753, 784, 734, 788, 769, 774, 762, 798, 798, 729, 795, 785, 740, 806, 721, 769, 791, 805, 756, 752, 787, 762, 783, 736, 777] payload_rows_per_dst=[9314, 9268, 9388, 9152, 9134, 9374, 9176, 9222] dup_rows_per_dst=[3226, 3080, 3134, 2874, 2928, 3034, 2906, 3094] combine_rows_per_src=[9314, 9268, 9388, 9152, 9134, 9374, 9176, 9222] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74230 payload_rows_remote=65020 dup_rows=24074 dup_rows_remote=20960 dedup_ratio=0.244893 router_tokens_per_expert=[773, 746, 797, 759, 757, 764, 767, 744, 756, 785, 724, 780, 795, 722, 798, 791, 755, 787, 786, 732, 773, 804, 787, 771, 805, 748, 810, 728, 745, 766, 783, 729, 737, 745, 731, 739, 788, 782, 777, 732, 768, 765, 800, 789, 778, 805, 776, 779, 757, 780, 765, 773, 783, 766, 804, 734, 770, 729, 750, 782, 751, 783, 750, 817] payload_rows_per_dst=[9246, 9262, 9306, 9294, 9166, 9412, 9210, 9334] dup_rows_per_dst=[2968, 3040, 3084, 2934, 2896, 3108, 3114, 2930] combine_rows_per_src=[9246, 9262, 9306, 9294, 9166, 9412, 9210, 9334] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74478 payload_rows_remote=65188 dup_rows=23826 dup_rows_remote=21086 dedup_ratio=0.242371 router_tokens_per_expert=[782, 757, 793, 762, 761, 808, 764, 787, 730, 768, 787, 780, 739, 740, 736, 735, 798, 760, 806, 736, 821, 779, 750, 824, 795, 717, 734, 790, 709, 812, 746, 802, 814, 796, 716, 797, 727, 751, 749, 748, 721, 811, 765, 777, 782, 759, 766, 769, 718, 753, 732, 766, 795, 812, 769, 786, 808, 692, 801, 778, 788, 748, 779, 771] payload_rows_per_dst=[9362, 9290, 9472, 9168, 9288, 9330, 9298, 9270] dup_rows_per_dst=[3066, 2740, 3076, 3042, 2908, 2970, 2964, 3060] combine_rows_per_src=[9362, 9290, 9472, 9168, 9288, 9330, 9298, 9270] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266575872 dispatch_meta_bytes=343968 combine_payload_bytes=266575872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266919840 ingress_bytes=301178880 link_bytes=301178880 token_bytes=533495712 weight_movement_bytes=34603008 total_nvlink_bytes=568098720 weight_to_token_ratio=0.064861 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133311879e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266354688 dispatch_meta_bytes=344208 combine_payload_bytes=266354688 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266698896 ingress_bytes=370163712 link_bytes=370163712 token_bytes=533053584 weight_movement_bytes=103809024 total_nvlink_bytes=636862608 weight_to_token_ratio=0.194744 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392896248e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266952704 dispatch_meta_bytes=343328 combine_payload_bytes=266952704 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267296032 ingress_bytes=336158720 link_bytes=336158720 token_bytes=534248736 weight_movement_bytes=69206016 total_nvlink_bytes=603454752 weight_to_token_ratio=0.129539 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264938200e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265068544 dispatch_meta_bytes=343056 combine_payload_bytes=265068544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265411600 ingress_bytes=299671552 link_bytes=299671552 token_bytes=530480144 weight_movement_bytes=34603008 total_nvlink_bytes=565083152 weight_to_token_ratio=0.065230 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127639925e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74212 payload_rows_remote=64764 dup_rows=24092 dup_rows_remote=20888 dedup_ratio=0.245076 router_tokens_per_expert=[763, 789, 779, 732, 783, 747, 769, 777, 789, 782, 733, 797, 724, 773, 731, 783, 763, 787, 752, 737, 782, 750, 758, 778, 786, 751, 751, 748, 757, 809, 719, 781, 773, 780, 785, 785, 706, 752, 786, 762, 777, 785, 793, 749, 760, 770, 796, 765, 786, 697, 791, 749, 863, 741, 760, 764, 768, 781, 765, 750, 826, 807, 742, 778] payload_rows_per_dst=[9002, 9286, 9130, 9332, 9332, 9336, 9346, 9448] dup_rows_per_dst=[2848, 2938, 2862, 3222, 2926, 3058, 3034, 3204] combine_rows_per_src=[9002, 9286, 9130, 9332, 9332, 9336, 9346, 9448] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74318 payload_rows_remote=65050 dup_rows=23986 dup_rows_remote=21018 dedup_ratio=0.243998 router_tokens_per_expert=[770, 747, 756, 805, 766, 745, 793, 777, 759, 798, 758, 766, 770, 792, 807, 777, 825, 782, 783, 765, 726, 776, 757, 780, 779, 776, 727, 725, 781, 752, 777, 736, 760, 771, 807, 801, 777, 753, 748, 701, 798, 744, 798, 721, 758, 806, 725, 757, 732, 796, 756, 784, 757, 798, 791, 787, 772, 739, 769, 750, 744, 742, 763, 814] payload_rows_per_dst=[9266, 9362, 9368, 9240, 9268, 9262, 9328, 9224] dup_rows_per_dst=[3052, 3092, 3020, 2866, 2968, 2952, 3074, 2962] combine_rows_per_src=[9266, 9362, 9368, 9240, 9268, 9262, 9328, 9224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=343496 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266706376 ingress_bytes=300965888 link_bytes=300965888 token_bytes=533069256 weight_movement_bytes=34603008 total_nvlink_bytes=567672264 weight_to_token_ratio=0.064913 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132510407e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266321920 dispatch_meta_bytes=343920 combine_payload_bytes=266321920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266665840 ingress_bytes=300924928 link_bytes=300924928 token_bytes=532987760 weight_movement_bytes=34603008 total_nvlink_bytes=567590768 weight_to_token_ratio=0.064923 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132356278e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=267010048 dispatch_meta_bytes=345096 combine_payload_bytes=267010048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267355144 ingress_bytes=301613056 link_bytes=301613056 token_bytes=534365192 weight_movement_bytes=34603008 total_nvlink_bytes=568968200 weight_to_token_ratio=0.064755 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134945648e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265273344 dispatch_meta_bytes=342608 combine_payload_bytes=265273344 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265615952 ingress_bytes=334479360 link_bytes=334479360 token_bytes=530889296 weight_movement_bytes=69206016 total_nvlink_bytes=600095312 weight_to_token_ratio=0.130359 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258618904e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74204 payload_rows_remote=64882 dup_rows=24100 dup_rows_remote=21230 dedup_ratio=0.245158 router_tokens_per_expert=[736, 766, 749, 786, 771, 733, 759, 819, 766, 773, 775, 749, 745, 795, 778, 726, 778, 781, 705, 767, 750, 767, 803, 745, 763, 723, 796, 696, 745, 808, 762, 753, 808, 771, 769, 777, 750, 747, 778, 774, 791, 778, 780, 794, 770, 756, 784, 700, 740, 795, 753, 779, 813, 814, 767, 779, 772, 773, 732, 808, 778, 812, 780, 762] payload_rows_per_dst=[9366, 9136, 9322, 9226, 9364, 9182, 9364, 9244] dup_rows_per_dst=[2872, 3078, 2870, 2866, 2984, 3124, 3116, 3190] combine_rows_per_src=[9366, 9136, 9322, 9226, 9364, 9182, 9364, 9244] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266444800 dispatch_meta_bytes=344272 combine_payload_bytes=266444800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266789072 ingress_bytes=301047808 link_bytes=301047808 token_bytes=533233872 weight_movement_bytes=34603008 total_nvlink_bytes=567836880 weight_to_token_ratio=0.064893 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132818665e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=65054 dup_rows=23950 dup_rows_remote=21006 dedup_ratio=0.243632 router_tokens_per_expert=[802, 804, 777, 791, 773, 763, 742, 722, 764, 739, 781, 775, 765, 781, 745, 790, 832, 708, 793, 792, 736, 764, 761, 737, 806, 766, 787, 733, 759, 785, 792, 763, 805, 742, 752, 838, 767, 763, 828, 755, 765, 734, 764, 822, 736, 768, 763, 770, 777, 731, 754, 754, 716, 782, 758, 797, 749, 791, 746, 766, 779, 745, 782, 725] payload_rows_per_dst=[9324, 9276, 9244, 9408, 9368, 9300, 9182, 9252] dup_rows_per_dst=[3024, 3004, 3002, 2974, 3132, 2944, 2956, 2914] combine_rows_per_src=[9324, 9276, 9244, 9408, 9368, 9300, 9182, 9252] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74446 payload_rows_remote=65060 dup_rows=23858 dup_rows_remote=20890 dedup_ratio=0.242696 router_tokens_per_expert=[788, 778, 783, 785, 762, 758, 740, 792, 750, 754, 748, 751, 732, 755, 754, 735, 794, 750, 753, 750, 804, 729, 744, 814, 769, 758, 807, 757, 798, 750, 768, 770, 763, 786, 792, 759, 756, 785, 716, 777, 752, 745, 773, 741, 836, 809, 748, 784, 754, 786, 776, 759, 797, 751, 751, 706, 783, 775, 797, 842, 761, 751, 769, 792] payload_rows_per_dst=[9378, 9072, 9316, 9386, 9308, 9310, 9296, 9380] dup_rows_per_dst=[2994, 2886, 2960, 2968, 2960, 3066, 2864, 3160] combine_rows_per_src=[9378, 9072, 9316, 9386, 9308, 9310, 9296, 9380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266952704 dispatch_meta_bytes=343328 combine_payload_bytes=266952704 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267296032 ingress_bytes=336158720 link_bytes=336158720 token_bytes=534248736 weight_movement_bytes=69206016 total_nvlink_bytes=603454752 weight_to_token_ratio=0.129539 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264938200e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74478 payload_rows_remote=65188 dup_rows=23826 dup_rows_remote=21086 dedup_ratio=0.242371 router_tokens_per_expert=[782, 757, 793, 762, 761, 808, 764, 787, 730, 768, 787, 780, 739, 740, 736, 735, 798, 760, 806, 736, 821, 779, 750, 824, 795, 717, 734, 790, 709, 812, 746, 802, 814, 796, 716, 797, 727, 751, 749, 748, 721, 811, 765, 777, 782, 759, 766, 769, 718, 753, 732, 766, 795, 812, 769, 786, 808, 692, 801, 778, 788, 748, 779, 771] payload_rows_per_dst=[9362, 9290, 9472, 9168, 9288, 9330, 9298, 9270] dup_rows_per_dst=[3066, 2740, 3076, 3042, 2908, 2970, 2964, 3060] combine_rows_per_src=[9362, 9290, 9472, 9168, 9288, 9330, 9298, 9270] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74300 payload_rows_remote=65028 dup_rows=24004 dup_rows_remote=21024 dedup_ratio=0.244181 router_tokens_per_expert=[761, 781, 764, 784, 772, 740, 804, 806, 741, 754, 776, 808, 744, 768, 746, 772, 735, 786, 864, 797, 768, 761, 736, 736, 788, 747, 771, 786, 780, 763, 746, 774, 751, 766, 762, 767, 784, 809, 739, 799, 777, 770, 783, 743, 788, 737, 763, 765, 745, 746, 725, 690, 791, 761, 782, 808, 767, 761, 739, 762, 787, 808, 776, 742] payload_rows_per_dst=[9328, 9174, 9310, 9340, 9406, 9272, 9204, 9266] dup_rows_per_dst=[3096, 3044, 3056, 2970, 2948, 2980, 2892, 3018] combine_rows_per_src=[9328, 9174, 9310, 9340, 9406, 9272, 9204, 9266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74388 payload_rows_remote=65082 dup_rows=23916 dup_rows_remote=20910 dedup_ratio=0.243286 router_tokens_per_expert=[752, 754, 807, 812, 743, 758, 762, 746, 753, 760, 739, 766, 754, 764, 784, 780, 785, 758, 784, 764, 732, 787, 774, 772, 792, 772, 743, 733, 736, 769, 792, 774, 759, 745, 776, 771, 781, 786, 757, 778, 785, 755, 739, 752, 821, 786, 718, 770, 798, 730, 818, 758, 820, 751, 742, 790, 764, 787, 768, 778, 814, 768, 727, 759] payload_rows_per_dst=[9384, 9280, 9306, 9244, 9286, 9228, 9396, 9264] dup_rows_per_dst=[2884, 2920, 3006, 2978, 3020, 3024, 3018, 3066] combine_rows_per_src=[9384, 9280, 9306, 9244, 9286, 9228, 9396, 9264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74558 payload_rows_remote=65132 dup_rows=23746 dup_rows_remote=20618 dedup_ratio=0.241557 router_tokens_per_expert=[714, 763, 816, 749, 768, 781, 788, 806, 832, 753, 772, 754, 752, 782, 755, 777, 728, 763, 825, 773, 721, 767, 766, 710, 796, 769, 804, 725, 785, 768, 755, 770, 797, 780, 785, 785, 775, 832, 781, 742, 812, 786, 793, 752, 790, 755, 740, 819, 752, 770, 734, 728, 798, 766, 743, 783, 800, 763, 715, 743, 704, 731, 763, 748] payload_rows_per_dst=[9396, 9384, 9186, 9382, 9426, 9462, 9254, 9068] dup_rows_per_dst=[2974, 2970, 2920, 2962, 3128, 3032, 2894, 2866] combine_rows_per_src=[9396, 9384, 9186, 9382, 9426, 9462, 9254, 9068] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=267010048 dispatch_meta_bytes=345096 combine_payload_bytes=267010048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267355144 ingress_bytes=301613056 link_bytes=301613056 token_bytes=534365192 weight_movement_bytes=34603008 total_nvlink_bytes=568968200 weight_to_token_ratio=0.064755 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134945648e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74194 payload_rows_remote=64854 dup_rows=24110 dup_rows_remote=21088 dedup_ratio=0.245260 router_tokens_per_expert=[774, 741, 735, 786, 801, 771, 791, 764, 792, 752, 786, 790, 765, 763, 797, 753, 770, 849, 720, 737, 792, 791, 717, 761, 720, 757, 767, 806, 764, 775, 773, 780, 788, 790, 779, 765, 746, 725, 757, 768, 819, 791, 738, 728, 795, 759, 725, 760, 750, 740, 775, 775, 783, 781, 789, 726, 729, 763, 794, 806, 734, 766, 773, 795] payload_rows_per_dst=[9340, 9674, 9292, 9278, 9310, 9136, 9188, 8976] dup_rows_per_dst=[3022, 3424, 2982, 3006, 3048, 2936, 3050, 2642] combine_rows_per_src=[9340, 9674, 9292, 9278, 9310, 9136, 9188, 8976] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74212 payload_rows_remote=64764 dup_rows=24092 dup_rows_remote=20888 dedup_ratio=0.245076 router_tokens_per_expert=[763, 789, 779, 732, 783, 747, 769, 777, 789, 782, 733, 797, 724, 773, 731, 783, 763, 787, 752, 737, 782, 750, 758, 778, 786, 751, 751, 748, 757, 809, 719, 781, 773, 780, 785, 785, 706, 752, 786, 762, 777, 785, 793, 749, 760, 770, 796, 765, 786, 697, 791, 749, 863, 741, 760, 764, 768, 781, 765, 750, 826, 807, 742, 778] payload_rows_per_dst=[9002, 9286, 9130, 9332, 9332, 9336, 9346, 9448] dup_rows_per_dst=[2848, 2938, 2862, 3222, 2926, 3058, 3034, 3204] combine_rows_per_src=[9002, 9286, 9130, 9332, 9332, 9336, 9346, 9448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266354688 dispatch_meta_bytes=344208 combine_payload_bytes=266354688 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266698896 ingress_bytes=370163712 link_bytes=370163712 token_bytes=533053584 weight_movement_bytes=103809024 total_nvlink_bytes=636862608 weight_to_token_ratio=0.194744 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392896248e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74446 payload_rows_remote=65186 dup_rows=23858 dup_rows_remote=20914 dedup_ratio=0.242696 router_tokens_per_expert=[673, 768, 824, 758, 710, 803, 816, 757, 777, 770, 784, 774, 780, 797, 723, 739, 799, 818, 774, 743, 762, 758, 763, 768, 774, 764, 788, 731, 793, 781, 726, 763, 809, 801, 764, 763, 775, 733, 690, 749, 764, 786, 742, 743, 764, 760, 768, 821, 767, 774, 770, 754, 725, 798, 730, 784, 744, 762, 782, 817, 792, 794, 796, 773] payload_rows_per_dst=[9298, 9274, 9400, 9226, 9324, 9294, 9260, 9370] dup_rows_per_dst=[2920, 3014, 2970, 3014, 2844, 3002, 2944, 3150] combine_rows_per_src=[9298, 9274, 9400, 9226, 9324, 9294, 9260, 9370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=344240 combine_payload_bytes=266461184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266805424 ingress_bytes=301064192 link_bytes=301064192 token_bytes=533266608 weight_movement_bytes=34603008 total_nvlink_bytes=567869616 weight_to_token_ratio=0.064889 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132880317e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266575872 dispatch_meta_bytes=343968 combine_payload_bytes=266575872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266919840 ingress_bytes=301178880 link_bytes=301178880 token_bytes=533495712 weight_movement_bytes=34603008 total_nvlink_bytes=568098720 weight_to_token_ratio=0.064861 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133311879e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=344448 combine_payload_bytes=265756672 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266101120 ingress_bytes=334962688 link_bytes=334962688 token_bytes=531857792 weight_movement_bytes=69206016 total_nvlink_bytes=601063808 weight_to_token_ratio=0.130121 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260437628e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=64926 dup_rows=24032 dup_rows_remote=20976 dedup_ratio=0.244466 router_tokens_per_expert=[805, 729, 795, 813, 779, 813, 791, 765, 802, 807, 771, 725, 795, 776, 801, 724, 727, 801, 741, 718, 764, 776, 748, 806, 803, 819, 748, 740, 757, 750, 745, 772, 774, 761, 737, 752, 747, 739, 803, 741, 775, 765, 777, 771, 760, 790, 759, 753, 776, 764, 788, 760, 762, 746, 763, 781, 803, 709, 715, 738, 788, 779, 768, 802] payload_rows_per_dst=[9462, 9346, 9198, 9282, 9252, 9324, 9218, 9190] dup_rows_per_dst=[3118, 3056, 2964, 2986, 2856, 2976, 3062, 3014] combine_rows_per_src=[9462, 9346, 9198, 9282, 9252, 9324, 9218, 9190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266485760 dispatch_meta_bytes=343800 combine_payload_bytes=266485760 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266829560 ingress_bytes=301088768 link_bytes=301088768 token_bytes=533315320 weight_movement_bytes=34603008 total_nvlink_bytes=567918328 weight_to_token_ratio=0.064883 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132972794e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266780672 dispatch_meta_bytes=343000 combine_payload_bytes=266780672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267123672 ingress_bytes=301383680 link_bytes=301383680 token_bytes=533904344 weight_movement_bytes=34603008 total_nvlink_bytes=568507352 weight_to_token_ratio=0.064811 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134082524e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74446 payload_rows_remote=65060 dup_rows=23858 dup_rows_remote=20890 dedup_ratio=0.242696 router_tokens_per_expert=[788, 778, 783, 785, 762, 758, 740, 792, 750, 754, 748, 751, 732, 755, 754, 735, 794, 750, 753, 750, 804, 729, 744, 814, 769, 758, 807, 757, 798, 750, 768, 770, 763, 786, 792, 759, 756, 785, 716, 777, 752, 745, 773, 741, 836, 809, 748, 784, 754, 786, 776, 759, 797, 751, 751, 706, 783, 775, 797, 842, 761, 751, 769, 792] payload_rows_per_dst=[9378, 9072, 9316, 9386, 9308, 9310, 9296, 9380] dup_rows_per_dst=[2994, 2886, 2960, 2968, 2960, 3066, 2864, 3160] combine_rows_per_src=[9378, 9072, 9316, 9386, 9308, 9310, 9296, 9380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265641984 dispatch_meta_bytes=343768 combine_payload_bytes=265641984 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265985752 ingress_bytes=369451008 link_bytes=369451008 token_bytes=531627736 weight_movement_bytes=103809024 total_nvlink_bytes=635436760 weight_to_token_ratio=0.195266 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390214400e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=65020 dup_rows=24010 dup_rows_remote=21042 dedup_ratio=0.244242 router_tokens_per_expert=[779, 780, 780, 754, 733, 773, 739, 761, 797, 790, 779, 741, 780, 732, 802, 770, 722, 748, 766, 807, 774, 779, 759, 748, 742, 742, 795, 800, 767, 772, 776, 734, 822, 758, 749, 764, 775, 751, 799, 771, 792, 803, 744, 747, 749, 752, 737, 793, 805, 773, 750, 784, 789, 749, 774, 780, 747, 791, 794, 778, 762, 781, 730, 738] payload_rows_per_dst=[9164, 8940, 9564, 9144, 9436, 9380, 9392, 9274] dup_rows_per_dst=[2962, 2646, 3438, 2750, 3192, 2966, 3088, 2968] combine_rows_per_src=[9164, 8940, 9564, 9144, 9436, 9380, 9392, 9274] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=65234 dup_rows=23946 dup_rows_remote=21100 dedup_ratio=0.243591 router_tokens_per_expert=[785, 780, 795, 739, 748, 789, 744, 767, 747, 790, 812, 734, 771, 772, 779, 786, 800, 721, 792, 761, 783, 734, 754, 782, 801, 735, 754, 818, 782, 783, 755, 821, 772, 745, 724, 737, 783, 741, 748, 735, 757, 678, 792, 726, 786, 744, 756, 793, 776, 796, 785, 789, 831, 770, 724, 743, 756, 813, 834, 734, 746, 744, 805, 775] payload_rows_per_dst=[9334, 9396, 9192, 9446, 9124, 9220, 9386, 9260] dup_rows_per_dst=[2960, 2986, 3062, 3052, 2846, 2844, 3042, 3154] combine_rows_per_src=[9334, 9396, 9192, 9446, 9124, 9220, 9386, 9260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265273344 dispatch_meta_bytes=342608 combine_payload_bytes=265273344 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265615952 ingress_bytes=334479360 link_bytes=334479360 token_bytes=530889296 weight_movement_bytes=69206016 total_nvlink_bytes=600095312 weight_to_token_ratio=0.130359 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258618904e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=267001856 dispatch_meta_bytes=344400 combine_payload_bytes=267001856 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267346256 ingress_bytes=336207872 link_bytes=336207872 token_bytes=534348112 weight_movement_bytes=69206016 total_nvlink_bytes=603554128 weight_to_token_ratio=0.129515 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265123155e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266485760 dispatch_meta_bytes=343800 combine_payload_bytes=266485760 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266829560 ingress_bytes=301088768 link_bytes=301088768 token_bytes=533315320 weight_movement_bytes=34603008 total_nvlink_bytes=567918328 weight_to_token_ratio=0.064883 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132972794e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=64926 dup_rows=24032 dup_rows_remote=20976 dedup_ratio=0.244466 router_tokens_per_expert=[805, 729, 795, 813, 779, 813, 791, 765, 802, 807, 771, 725, 795, 776, 801, 724, 727, 801, 741, 718, 764, 776, 748, 806, 803, 819, 748, 740, 757, 750, 745, 772, 774, 761, 737, 752, 747, 739, 803, 741, 775, 765, 777, 771, 760, 790, 759, 753, 776, 764, 788, 760, 762, 746, 763, 781, 803, 709, 715, 738, 788, 779, 768, 802] payload_rows_per_dst=[9462, 9346, 9198, 9282, 9252, 9324, 9218, 9190] dup_rows_per_dst=[3118, 3056, 2964, 2986, 2856, 2976, 3062, 3014] combine_rows_per_src=[9462, 9346, 9198, 9282, 9252, 9324, 9218, 9190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=65054 dup_rows=23950 dup_rows_remote=21006 dedup_ratio=0.243632 router_tokens_per_expert=[802, 804, 777, 791, 773, 763, 742, 722, 764, 739, 781, 775, 765, 781, 745, 790, 832, 708, 793, 792, 736, 764, 761, 737, 806, 766, 787, 733, 759, 785, 792, 763, 805, 742, 752, 838, 767, 763, 828, 755, 765, 734, 764, 822, 736, 768, 763, 770, 777, 731, 754, 754, 716, 782, 758, 797, 749, 791, 746, 766, 779, 745, 782, 725] payload_rows_per_dst=[9324, 9276, 9244, 9408, 9368, 9300, 9182, 9252] dup_rows_per_dst=[3024, 3004, 3002, 2974, 3132, 2944, 2956, 2914] combine_rows_per_src=[9324, 9276, 9244, 9408, 9368, 9300, 9182, 9252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74110 payload_rows_remote=64712 dup_rows=24194 dup_rows_remote=21104 dedup_ratio=0.246114 router_tokens_per_expert=[811, 790, 747, 803, 804, 766, 762, 749, 766, 740, 802, 758, 750, 776, 749, 774, 777, 744, 773, 796, 768, 722, 788, 733, 758, 820, 748, 750, 816, 766, 740, 715, 742, 786, 807, 750, 772, 786, 801, 800, 746, 803, 786, 741, 743, 787, 758, 738, 781, 728, 818, 723, 776, 745, 712, 784, 753, 746, 804, 763, 790, 769, 791, 762] payload_rows_per_dst=[9412, 9164, 9260, 9216, 9398, 9214, 9142, 9304] dup_rows_per_dst=[3052, 3066, 2942, 3010, 3090, 2990, 2992, 3052] combine_rows_per_src=[9412, 9164, 9260, 9216, 9398, 9214, 9142, 9304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74204 payload_rows_remote=64882 dup_rows=24100 dup_rows_remote=21230 dedup_ratio=0.245158 router_tokens_per_expert=[736, 766, 749, 786, 771, 733, 759, 819, 766, 773, 775, 749, 745, 795, 778, 726, 778, 781, 705, 767, 750, 767, 803, 745, 763, 723, 796, 696, 745, 808, 762, 753, 808, 771, 769, 777, 750, 747, 778, 774, 791, 778, 780, 794, 770, 756, 784, 700, 740, 795, 753, 779, 813, 814, 767, 779, 772, 773, 732, 808, 778, 812, 780, 762] payload_rows_per_dst=[9366, 9136, 9322, 9226, 9364, 9182, 9364, 9244] dup_rows_per_dst=[2872, 3078, 2870, 2866, 2984, 3124, 3116, 3190] combine_rows_per_src=[9366, 9136, 9322, 9226, 9364, 9182, 9364, 9244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265936896 dispatch_meta_bytes=343608 combine_payload_bytes=265936896 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266280504 ingress_bytes=335142912 link_bytes=335142912 token_bytes=532217400 weight_movement_bytes=69206016 total_nvlink_bytes=601423416 weight_to_token_ratio=0.130033 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261115797e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=65020 dup_rows=24018 dup_rows_remote=21062 dedup_ratio=0.244324 router_tokens_per_expert=[744, 758, 749, 797, 774, 786, 723, 778, 774, 784, 762, 797, 788, 779, 787, 730, 812, 751, 756, 797, 751, 724, 769, 746, 774, 772, 721, 760, 801, 741, 728, 811, 777, 770, 756, 763, 782, 792, 757, 759, 788, 750, 785, 749, 808, 773, 738, 800, 808, 774, 758, 734, 780, 750, 780, 782, 824, 730, 720, 779, 713, 796, 758, 795] payload_rows_per_dst=[9266, 9486, 9166, 9280, 9354, 9332, 9176, 9226] dup_rows_per_dst=[2956, 3154, 3046, 2938, 2958, 3050, 2918, 2998] combine_rows_per_src=[9266, 9486, 9166, 9280, 9354, 9332, 9176, 9226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=344240 combine_payload_bytes=266461184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266805424 ingress_bytes=301064192 link_bytes=301064192 token_bytes=533266608 weight_movement_bytes=34603008 total_nvlink_bytes=567869616 weight_to_token_ratio=0.064889 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132880317e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=65020 dup_rows=24010 dup_rows_remote=21042 dedup_ratio=0.244242 router_tokens_per_expert=[779, 780, 780, 754, 733, 773, 739, 761, 797, 790, 779, 741, 780, 732, 802, 770, 722, 748, 766, 807, 774, 779, 759, 748, 742, 742, 795, 800, 767, 772, 776, 734, 822, 758, 749, 764, 775, 751, 799, 771, 792, 803, 744, 747, 749, 752, 737, 793, 805, 773, 750, 784, 789, 749, 774, 780, 747, 791, 794, 778, 762, 781, 730, 738] payload_rows_per_dst=[9164, 8940, 9564, 9144, 9436, 9380, 9392, 9274] dup_rows_per_dst=[2962, 2646, 3438, 2750, 3192, 2966, 3088, 2968] combine_rows_per_src=[9164, 8940, 9564, 9144, 9436, 9380, 9392, 9274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74220 payload_rows_remote=64958 dup_rows=24084 dup_rows_remote=21060 dedup_ratio=0.244995 router_tokens_per_expert=[741, 740, 808, 774, 759, 761, 787, 786, 757, 760, 792, 770, 727, 727, 773, 748, 788, 767, 791, 739, 765, 772, 777, 735, 783, 776, 776, 764, 774, 765, 803, 756, 760, 789, 798, 796, 769, 749, 738, 744, 776, 759, 741, 817, 814, 725, 803, 760, 796, 745, 781, 787, 751, 770, 715, 798, 782, 810, 777, 739, 772, 756, 767, 727] payload_rows_per_dst=[9340, 9250, 9256, 9304, 9300, 9312, 9262, 9196] dup_rows_per_dst=[2972, 2858, 3012, 3090, 2986, 3078, 3024, 3064] combine_rows_per_src=[9340, 9250, 9256, 9304, 9300, 9312, 9262, 9196] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265060352 dispatch_meta_bytes=343264 combine_payload_bytes=265060352 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265403616 ingress_bytes=403472384 link_bytes=403472384 token_bytes=530463968 weight_movement_bytes=138412032 total_nvlink_bytes=668876000 weight_to_token_ratio=0.260926 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.518234099e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=344448 combine_payload_bytes=265756672 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266101120 ingress_bytes=334962688 link_bytes=334962688 token_bytes=531857792 weight_movement_bytes=69206016 total_nvlink_bytes=601063808 weight_to_token_ratio=0.130121 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260437628e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=65110 dup_rows=23928 dup_rows_remote=20954 dedup_ratio=0.243408 router_tokens_per_expert=[799, 736, 720, 824, 751, 793, 758, 809, 772, 798, 769, 790, 787, 774, 781, 774, 779, 756, 775, 716, 773, 742, 753, 749, 726, 797, 776, 774, 752, 757, 792, 746, 767, 772, 765, 726, 752, 772, 811, 739, 740, 808, 797, 761, 788, 777, 805, 756, 771, 753, 757, 784, 782, 754, 773, 812, 746, 769, 734, 738, 752, 763, 751, 779] payload_rows_per_dst=[9378, 9408, 9204, 9266, 9176, 9386, 9420, 9138] dup_rows_per_dst=[3002, 3082, 2882, 2974, 3032, 3078, 2952, 2926] combine_rows_per_src=[9378, 9408, 9204, 9266, 9176, 9386, 9420, 9138] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266321920 dispatch_meta_bytes=344328 combine_payload_bytes=266321920 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266666248 ingress_bytes=335527936 link_bytes=335527936 token_bytes=532988168 weight_movement_bytes=69206016 total_nvlink_bytes=602194184 weight_to_token_ratio=0.129845 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262564611e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266321920 dispatch_meta_bytes=344248 combine_payload_bytes=266321920 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266666168 ingress_bytes=335527936 link_bytes=335527936 token_bytes=532988088 weight_movement_bytes=69206016 total_nvlink_bytes=602194104 weight_to_token_ratio=0.129845 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262564611e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=267198464 dispatch_meta_bytes=345336 combine_payload_bytes=267198464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267543800 ingress_bytes=301801472 link_bytes=301801472 token_bytes=534742264 weight_movement_bytes=34603008 total_nvlink_bytes=569345272 weight_to_token_ratio=0.064710 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135654642e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=344072 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266412040 ingress_bytes=300670976 link_bytes=300670976 token_bytes=532480008 weight_movement_bytes=34603008 total_nvlink_bytes=567083016 weight_to_token_ratio=0.064985 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131400677e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74188 payload_rows_remote=64878 dup_rows=24116 dup_rows_remote=21126 dedup_ratio=0.245321 router_tokens_per_expert=[759, 758, 790, 774, 749, 796, 782, 702, 779, 772, 782, 762, 755, 799, 736, 765, 696, 739, 770, 790, 783, 769, 764, 811, 812, 769, 783, 811, 724, 747, 766, 769, 734, 804, 742, 770, 785, 770, 774, 782, 734, 807, 773, 742, 749, 760, 808, 758, 780, 854, 771, 785, 729, 804, 744, 755, 799, 732, 743, 754, 755, 755, 786, 751] payload_rows_per_dst=[9144, 9310, 9342, 9360, 9284, 9280, 9340, 9128] dup_rows_per_dst=[3076, 2990, 2902, 3002, 3038, 2982, 3104, 3022] combine_rows_per_src=[9144, 9310, 9342, 9360, 9284, 9280, 9340, 9128] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266690560 dispatch_meta_bytes=344256 combine_payload_bytes=266690560 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267034816 ingress_bytes=370499584 link_bytes=370499584 token_bytes=533725376 weight_movement_bytes=103809024 total_nvlink_bytes=637534400 weight_to_token_ratio=0.194499 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394160107e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74138 payload_rows_remote=64876 dup_rows=24166 dup_rows_remote=21048 dedup_ratio=0.245829 router_tokens_per_expert=[770, 779, 738, 830, 761, 754, 771, 732, 817, 767, 833, 755, 765, 757, 742, 712, 735, 757, 766, 778, 745, 780, 767, 709, 752, 795, 768, 766, 773, 770, 804, 743, 735, 784, 771, 780, 779, 801, 740, 795, 809, 812, 739, 777, 749, 767, 742, 795, 803, 761, 763, 763, 753, 750, 778, 740, 772, 774, 761, 770, 768, 799, 783, 748] payload_rows_per_dst=[9312, 9228, 9170, 9276, 9394, 9262, 9216, 9280] dup_rows_per_dst=[2958, 3068, 2904, 3066, 2976, 3118, 3006, 3070] combine_rows_per_src=[9312, 9228, 9170, 9276, 9394, 9262, 9216, 9280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74318 payload_rows_remote=65050 dup_rows=23986 dup_rows_remote=21018 dedup_ratio=0.243998 router_tokens_per_expert=[770, 747, 756, 805, 766, 745, 793, 777, 759, 798, 758, 766, 770, 792, 807, 777, 825, 782, 783, 765, 726, 776, 757, 780, 779, 776, 727, 725, 781, 752, 777, 736, 760, 771, 807, 801, 777, 753, 748, 701, 798, 744, 798, 721, 758, 806, 725, 757, 732, 796, 756, 784, 757, 798, 791, 787, 772, 739, 769, 750, 744, 742, 763, 814] payload_rows_per_dst=[9266, 9362, 9368, 9240, 9268, 9262, 9328, 9224] dup_rows_per_dst=[3052, 3092, 3020, 2866, 2968, 2952, 3074, 2962] combine_rows_per_src=[9266, 9362, 9368, 9240, 9268, 9262, 9328, 9224] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74238 payload_rows_remote=65046 dup_rows=24066 dup_rows_remote=21098 dedup_ratio=0.244812 router_tokens_per_expert=[769, 780, 780, 767, 769, 721, 713, 773, 719, 805, 787, 777, 784, 795, 769, 734, 783, 757, 769, 768, 758, 734, 748, 763, 720, 807, 765, 809, 700, 774, 813, 781, 764, 797, 711, 787, 771, 771, 810, 780, 814, 746, 789, 719, 740, 773, 741, 772, 778, 741, 807, 749, 717, 778, 819, 722, 740, 816, 817, 768, 741, 790, 818, 775] payload_rows_per_dst=[9260, 9266, 9192, 9310, 9350, 9172, 9272, 9416] dup_rows_per_dst=[2884, 3074, 2968, 3028, 3032, 3016, 2950, 3114] combine_rows_per_src=[9260, 9266, 9192, 9310, 9350, 9172, 9272, 9416] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=65110 dup_rows=23928 dup_rows_remote=20954 dedup_ratio=0.243408 router_tokens_per_expert=[799, 736, 720, 824, 751, 793, 758, 809, 772, 798, 769, 790, 787, 774, 781, 774, 779, 756, 775, 716, 773, 742, 753, 749, 726, 797, 776, 774, 752, 757, 792, 746, 767, 772, 765, 726, 752, 772, 811, 739, 740, 808, 797, 761, 788, 777, 805, 756, 771, 753, 757, 784, 782, 754, 773, 812, 746, 769, 734, 738, 752, 763, 751, 779] payload_rows_per_dst=[9378, 9408, 9204, 9266, 9176, 9386, 9420, 9138] dup_rows_per_dst=[3002, 3082, 2882, 2974, 3032, 3078, 2952, 2926] combine_rows_per_src=[9378, 9408, 9204, 9266, 9176, 9386, 9420, 9138] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=65020 dup_rows=24018 dup_rows_remote=21062 dedup_ratio=0.244324 router_tokens_per_expert=[744, 758, 749, 797, 774, 786, 723, 778, 774, 784, 762, 797, 788, 779, 787, 730, 812, 751, 756, 797, 751, 724, 769, 746, 774, 772, 721, 760, 801, 741, 728, 811, 777, 770, 756, 763, 782, 792, 757, 759, 788, 750, 785, 749, 808, 773, 738, 800, 808, 774, 758, 734, 780, 750, 780, 782, 824, 730, 720, 779, 713, 796, 758, 795] payload_rows_per_dst=[9266, 9486, 9166, 9280, 9354, 9332, 9176, 9226] dup_rows_per_dst=[2956, 3154, 3046, 2938, 2958, 3050, 2918, 2998] combine_rows_per_src=[9266, 9486, 9166, 9280, 9354, 9332, 9176, 9226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265740288 dispatch_meta_bytes=344016 combine_payload_bytes=265740288 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266084304 ingress_bytes=300343296 link_bytes=300343296 token_bytes=531824592 weight_movement_bytes=34603008 total_nvlink_bytes=566427600 weight_to_token_ratio=0.065065 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130167643e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=64834 dup_rows=23992 dup_rows_remote=20738 dedup_ratio=0.244059 router_tokens_per_expert=[749, 789, 806, 766, 722, 727, 757, 768, 749, 781, 751, 797, 771, 787, 799, 726, 751, 756, 772, 779, 775, 762, 783, 734, 791, 782, 783, 775, 784, 780, 785, 792, 725, 768, 791, 723, 739, 770, 767, 775, 805, 743, 782, 763, 756, 720, 803, 791, 787, 778, 819, 745, 732, 750, 770, 800, 765, 781, 765, 790, 735, 762, 805, 718] payload_rows_per_dst=[9478, 9334, 9638, 9346, 9214, 8978, 9066, 9258] dup_rows_per_dst=[3254, 3116, 3122, 3088, 2918, 2812, 2732, 2950] combine_rows_per_src=[9478, 9334, 9638, 9346, 9214, 8978, 9066, 9258] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265732096 dispatch_meta_bytes=343696 combine_payload_bytes=265732096 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266075792 ingress_bytes=334938112 link_bytes=334938112 token_bytes=531807888 weight_movement_bytes=69206016 total_nvlink_bytes=601013904 weight_to_token_ratio=0.130133 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260345151e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74220 payload_rows_remote=64958 dup_rows=24084 dup_rows_remote=21060 dedup_ratio=0.244995 router_tokens_per_expert=[741, 740, 808, 774, 759, 761, 787, 786, 757, 760, 792, 770, 727, 727, 773, 748, 788, 767, 791, 739, 765, 772, 777, 735, 783, 776, 776, 764, 774, 765, 803, 756, 760, 789, 798, 796, 769, 749, 738, 744, 776, 759, 741, 817, 814, 725, 803, 760, 796, 745, 781, 787, 751, 770, 715, 798, 782, 810, 777, 739, 772, 756, 767, 727] payload_rows_per_dst=[9340, 9250, 9256, 9304, 9300, 9312, 9262, 9196] dup_rows_per_dst=[2972, 2858, 3012, 3090, 2986, 3078, 3024, 3064] combine_rows_per_src=[9340, 9250, 9256, 9304, 9300, 9312, 9262, 9196] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=344576 combine_payload_bytes=266428416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266772992 ingress_bytes=301031424 link_bytes=301031424 token_bytes=533201408 weight_movement_bytes=34603008 total_nvlink_bytes=567804416 weight_to_token_ratio=0.064897 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132757013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266690560 dispatch_meta_bytes=344256 combine_payload_bytes=266690560 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267034816 ingress_bytes=370499584 link_bytes=370499584 token_bytes=533725376 weight_movement_bytes=103809024 total_nvlink_bytes=637534400 weight_to_token_ratio=0.194499 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394160107e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74188 payload_rows_remote=64878 dup_rows=24116 dup_rows_remote=21126 dedup_ratio=0.245321 router_tokens_per_expert=[759, 758, 790, 774, 749, 796, 782, 702, 779, 772, 782, 762, 755, 799, 736, 765, 696, 739, 770, 790, 783, 769, 764, 811, 812, 769, 783, 811, 724, 747, 766, 769, 734, 804, 742, 770, 785, 770, 774, 782, 734, 807, 773, 742, 749, 760, 808, 758, 780, 854, 771, 785, 729, 804, 744, 755, 799, 732, 743, 754, 755, 755, 786, 751] payload_rows_per_dst=[9144, 9310, 9342, 9360, 9284, 9280, 9340, 9128] dup_rows_per_dst=[3076, 2990, 2902, 3002, 3038, 2982, 3104, 3022] combine_rows_per_src=[9144, 9310, 9342, 9360, 9284, 9280, 9340, 9128] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266321920 dispatch_meta_bytes=344328 combine_payload_bytes=266321920 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266666248 ingress_bytes=335527936 link_bytes=335527936 token_bytes=532988168 weight_movement_bytes=69206016 total_nvlink_bytes=602194184 weight_to_token_ratio=0.129845 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262564611e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=64960 dup_rows=23982 dup_rows_remote=20978 dedup_ratio=0.243958 router_tokens_per_expert=[793, 746, 768, 729, 707, 767, 777, 760, 767, 722, 738, 766, 747, 822, 798, 780, 793, 798, 823, 739, 778, 772, 786, 778, 727, 734, 781, 764, 787, 737, 707, 778, 749, 740, 786, 778, 777, 793, 773, 749, 705, 776, 813, 750, 776, 788, 770, 791, 805, 747, 833, 743, 771, 749, 770, 768, 770, 799, 778, 767, 776, 772, 758, 763] payload_rows_per_dst=[9144, 9240, 9362, 9098, 9334, 9394, 9388, 9362] dup_rows_per_dst=[2950, 3040, 3172, 2730, 2956, 3078, 3052, 3004] combine_rows_per_src=[9144, 9240, 9362, 9098, 9334, 9394, 9388, 9362] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74350 payload_rows_remote=65022 dup_rows=23954 dup_rows_remote=20930 dedup_ratio=0.243673 router_tokens_per_expert=[717, 773, 769, 783, 757, 790, 788, 737, 767, 748, 777, 750, 796, 718, 786, 768, 790, 753, 762, 736, 762, 751, 740, 789, 749, 755, 756, 764, 811, 739, 797, 757, 731, 769, 791, 762, 770, 778, 787, 788, 739, 752, 787, 729, 805, 756, 744, 805, 826, 770, 710, 769, 795, 794, 782, 787, 757, 777, 773, 765, 770, 789, 773, 787] payload_rows_per_dst=[9344, 9242, 9242, 9258, 9328, 9232, 9384, 9320] dup_rows_per_dst=[2884, 2978, 2924, 2998, 3024, 3002, 3082, 3062] combine_rows_per_src=[9344, 9242, 9242, 9258, 9328, 9232, 9384, 9320] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266444800 dispatch_meta_bytes=344272 combine_payload_bytes=266444800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266789072 ingress_bytes=301047808 link_bytes=301047808 token_bytes=533233872 weight_movement_bytes=34603008 total_nvlink_bytes=567836880 weight_to_token_ratio=0.064893 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132818665e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74404 payload_rows_remote=65078 dup_rows=23900 dup_rows_remote=20936 dedup_ratio=0.243123 router_tokens_per_expert=[752, 767, 726, 826, 731, 780, 773, 750, 790, 737, 774, 754, 739, 747, 784, 767, 757, 796, 780, 733, 761, 790, 736, 738, 744, 754, 798, 794, 826, 772, 779, 766, 768, 750, 801, 736, 763, 784, 735, 761, 768, 761, 760, 763, 767, 771, 825, 743, 768, 782, 784, 769, 764, 787, 767, 724, 815, 775, 762, 768, 770, 810, 756, 774] payload_rows_per_dst=[9270, 9334, 9200, 9434, 9180, 9292, 9326, 9368] dup_rows_per_dst=[2940, 2850, 2982, 3032, 3016, 3024, 2964, 3092] combine_rows_per_src=[9270, 9334, 9200, 9434, 9180, 9292, 9326, 9368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=344576 combine_payload_bytes=266428416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266772992 ingress_bytes=301031424 link_bytes=301031424 token_bytes=533201408 weight_movement_bytes=34603008 total_nvlink_bytes=567804416 weight_to_token_ratio=0.064897 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132757013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=344072 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266412040 ingress_bytes=300670976 link_bytes=300670976 token_bytes=532480008 weight_movement_bytes=34603008 total_nvlink_bytes=567083016 weight_to_token_ratio=0.064985 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131400677e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265740288 dispatch_meta_bytes=344016 combine_payload_bytes=265740288 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266084304 ingress_bytes=300343296 link_bytes=300343296 token_bytes=531824592 weight_movement_bytes=34603008 total_nvlink_bytes=566427600 weight_to_token_ratio=0.065065 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130167643e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74182 payload_rows_remote=64862 dup_rows=24122 dup_rows_remote=21080 dedup_ratio=0.245382 router_tokens_per_expert=[773, 737, 812, 729, 780, 764, 771, 768, 786, 774, 803, 750, 757, 759, 788, 771, 728, 723, 785, 760, 784, 768, 719, 737, 809, 756, 777, 772, 774, 789, 739, 765, 777, 751, 716, 772, 798, 790, 822, 790, 768, 779, 742, 767, 746, 731, 787, 811, 778, 786, 765, 737, 766, 767, 852, 778, 761, 772, 722, 773, 768, 741, 733, 799] payload_rows_per_dst=[9244, 9300, 9180, 9320, 9302, 9290, 9404, 9142] dup_rows_per_dst=[3024, 3076, 2828, 3042, 3130, 2972, 3054, 2996] combine_rows_per_src=[9244, 9300, 9180, 9320, 9302, 9290, 9404, 9142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265560064 dispatch_meta_bytes=342288 combine_payload_bytes=265560064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265902352 ingress_bytes=300163072 link_bytes=300163072 token_bytes=531462416 weight_movement_bytes=34603008 total_nvlink_bytes=566065424 weight_to_token_ratio=0.065109 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129489475e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266076160 dispatch_meta_bytes=343752 combine_payload_bytes=266076160 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266419912 ingress_bytes=335282176 link_bytes=335282176 token_bytes=532496072 weight_movement_bytes=69206016 total_nvlink_bytes=601702088 weight_to_token_ratio=0.129965 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261639836e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266330112 dispatch_meta_bytes=343808 combine_payload_bytes=266330112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266673920 ingress_bytes=300933120 link_bytes=300933120 token_bytes=533004032 weight_movement_bytes=34603008 total_nvlink_bytes=567607040 weight_to_token_ratio=0.064921 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132387103e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266559488 dispatch_meta_bytes=344056 combine_payload_bytes=266559488 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266903544 ingress_bytes=335765504 link_bytes=335765504 token_bytes=533463032 weight_movement_bytes=69206016 total_nvlink_bytes=602669048 weight_to_token_ratio=0.129730 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263458560e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74740 payload_rows_remote=65304 dup_rows=23564 dup_rows_remote=20558 dedup_ratio=0.239705 router_tokens_per_expert=[766, 764, 747, 784, 717, 795, 748, 743, 795, 749, 797, 742, 777, 782, 793, 786, 755, 820, 765, 763, 710, 742, 766, 777, 763, 803, 789, 790, 783, 758, 747, 758, 718, 728, 772, 762, 754, 794, 730, 759, 760, 771, 761, 781, 733, 778, 728, 780, 776, 761, 810, 773, 798, 831, 784, 749, 772, 773, 804, 745, 801, 749, 766, 777] payload_rows_per_dst=[9266, 9436, 9252, 9358, 9234, 9294, 9478, 9422] dup_rows_per_dst=[2862, 3006, 2944, 3024, 2800, 2890, 3086, 2952] combine_rows_per_src=[9266, 9436, 9252, 9358, 9234, 9294, 9478, 9422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265674752 dispatch_meta_bytes=343768 combine_payload_bytes=265674752 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266018520 ingress_bytes=404086784 link_bytes=404086784 token_bytes=531693272 weight_movement_bytes=138412032 total_nvlink_bytes=670105304 weight_to_token_ratio=0.260323 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520546037e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74562 payload_rows_remote=65132 dup_rows=23742 dup_rows_remote=20800 dedup_ratio=0.241516 router_tokens_per_expert=[785, 761, 767, 743, 746, 740, 782, 785, 770, 776, 779, 793, 802, 758, 768, 740, 820, 740, 762, 799, 777, 753, 750, 747, 771, 743, 783, 792, 746, 748, 802, 765, 799, 793, 745, 793, 754, 790, 744, 721, 808, 768, 748, 840, 757, 730, 768, 767, 726, 731, 768, 769, 776, 812, 741, 769, 798, 801, 746, 748, 753, 750, 789, 757] payload_rows_per_dst=[9156, 9340, 9282, 9474, 9324, 9430, 9272, 9284] dup_rows_per_dst=[3062, 3032, 3014, 2826, 2954, 2942, 2912, 3000] combine_rows_per_src=[9156, 9340, 9282, 9474, 9324, 9430, 9272, 9284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=65234 dup_rows=23946 dup_rows_remote=21100 dedup_ratio=0.243591 router_tokens_per_expert=[785, 780, 795, 739, 748, 789, 744, 767, 747, 790, 812, 734, 771, 772, 779, 786, 800, 721, 792, 761, 783, 734, 754, 782, 801, 735, 754, 818, 782, 783, 755, 821, 772, 745, 724, 737, 783, 741, 748, 735, 757, 678, 792, 726, 786, 744, 756, 793, 776, 796, 785, 789, 831, 770, 724, 743, 756, 813, 834, 734, 746, 744, 805, 775] payload_rows_per_dst=[9334, 9396, 9192, 9446, 9124, 9220, 9386, 9260] dup_rows_per_dst=[2960, 2986, 3062, 3052, 2846, 2844, 3042, 3154] combine_rows_per_src=[9334, 9396, 9192, 9446, 9124, 9220, 9386, 9260] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74494 payload_rows_remote=65222 dup_rows=23810 dup_rows_remote=20880 dedup_ratio=0.242208 router_tokens_per_expert=[767, 774, 780, 743, 771, 784, 797, 776, 692, 754, 742, 750, 754, 792, 758, 766, 778, 745, 711, 743, 785, 762, 768, 809, 739, 794, 808, 790, 761, 782, 718, 780, 796, 783, 770, 757, 745, 747, 765, 758, 739, 758, 792, 770, 755, 840, 752, 809, 761, 773, 751, 763, 800, 745, 779, 762, 821, 772, 757, 763, 771, 781, 783, 761] payload_rows_per_dst=[9320, 9174, 9272, 9314, 9308, 9414, 9294, 9398] dup_rows_per_dst=[3064, 2842, 2930, 3030, 2934, 3016, 2974, 3020] combine_rows_per_src=[9320, 9174, 9272, 9314, 9308, 9414, 9294, 9398] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74182 payload_rows_remote=64862 dup_rows=24122 dup_rows_remote=21080 dedup_ratio=0.245382 router_tokens_per_expert=[773, 737, 812, 729, 780, 764, 771, 768, 786, 774, 803, 750, 757, 759, 788, 771, 728, 723, 785, 760, 784, 768, 719, 737, 809, 756, 777, 772, 774, 789, 739, 765, 777, 751, 716, 772, 798, 790, 822, 790, 768, 779, 742, 767, 746, 731, 787, 811, 778, 786, 765, 737, 766, 767, 852, 778, 761, 772, 722, 773, 768, 741, 733, 799] payload_rows_per_dst=[9244, 9300, 9180, 9320, 9302, 9290, 9404, 9142] dup_rows_per_dst=[3024, 3076, 2828, 3042, 3130, 2972, 3054, 2996] combine_rows_per_src=[9244, 9300, 9180, 9320, 9302, 9290, 9404, 9142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=267485184 dispatch_meta_bytes=343448 combine_payload_bytes=267485184 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267828632 ingress_bytes=371294208 link_bytes=371294208 token_bytes=535313816 weight_movement_bytes=103809024 total_nvlink_bytes=639122840 weight_to_token_ratio=0.193922 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397150213e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=64834 dup_rows=23992 dup_rows_remote=20738 dedup_ratio=0.244059 router_tokens_per_expert=[749, 789, 806, 766, 722, 727, 757, 768, 749, 781, 751, 797, 771, 787, 799, 726, 751, 756, 772, 779, 775, 762, 783, 734, 791, 782, 783, 775, 784, 780, 785, 792, 725, 768, 791, 723, 739, 770, 767, 775, 805, 743, 782, 763, 756, 720, 803, 791, 787, 778, 819, 745, 732, 750, 770, 800, 765, 781, 765, 790, 735, 762, 805, 718] payload_rows_per_dst=[9478, 9334, 9638, 9346, 9214, 8978, 9066, 9258] dup_rows_per_dst=[3254, 3116, 3122, 3088, 2918, 2812, 2732, 2950] combine_rows_per_src=[9478, 9334, 9638, 9346, 9214, 8978, 9066, 9258] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74210 payload_rows_remote=64690 dup_rows=24094 dup_rows_remote=20954 dedup_ratio=0.245097 router_tokens_per_expert=[742, 756, 785, 734, 788, 791, 726, 782, 806, 771, 718, 773, 763, 757, 764, 820, 775, 748, 747, 806, 763, 801, 765, 774, 776, 755, 785, 757, 797, 774, 762, 759, 766, 804, 770, 753, 745, 723, 775, 772, 802, 731, 752, 779, 769, 834, 781, 750, 721, 805, 761, 761, 785, 749, 810, 741, 714, 778, 781, 758, 766, 722, 769, 805] payload_rows_per_dst=[9520, 9110, 9502, 9346, 9248, 9052, 9288, 9144] dup_rows_per_dst=[3140, 2864, 3300, 3026, 2968, 2900, 2978, 2918] combine_rows_per_src=[9520, 9110, 9502, 9346, 9248, 9052, 9288, 9144] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266780672 dispatch_meta_bytes=343728 combine_payload_bytes=266780672 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267124400 ingress_bytes=335986688 link_bytes=335986688 token_bytes=533905072 weight_movement_bytes=69206016 total_nvlink_bytes=603111088 weight_to_token_ratio=0.129622 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264290858e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267149312 dispatch_meta_bytes=344408 combine_payload_bytes=267149312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267493720 ingress_bytes=301752320 link_bytes=301752320 token_bytes=534643032 weight_movement_bytes=34603008 total_nvlink_bytes=569246040 weight_to_token_ratio=0.064722 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135469687e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265674752 dispatch_meta_bytes=343768 combine_payload_bytes=265674752 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266018520 ingress_bytes=404086784 link_bytes=404086784 token_bytes=531693272 weight_movement_bytes=138412032 total_nvlink_bytes=670105304 weight_to_token_ratio=0.260323 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520546037e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74404 payload_rows_remote=65078 dup_rows=23900 dup_rows_remote=20936 dedup_ratio=0.243123 router_tokens_per_expert=[752, 767, 726, 826, 731, 780, 773, 750, 790, 737, 774, 754, 739, 747, 784, 767, 757, 796, 780, 733, 761, 790, 736, 738, 744, 754, 798, 794, 826, 772, 779, 766, 768, 750, 801, 736, 763, 784, 735, 761, 768, 761, 760, 763, 767, 771, 825, 743, 768, 782, 784, 769, 764, 787, 767, 724, 815, 775, 762, 768, 770, 810, 756, 774] payload_rows_per_dst=[9270, 9334, 9200, 9434, 9180, 9292, 9326, 9368] dup_rows_per_dst=[2940, 2850, 2982, 3032, 3016, 3024, 2964, 3092] combine_rows_per_src=[9270, 9334, 9200, 9434, 9180, 9292, 9326, 9368] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=65080 dup_rows=24010 dup_rows_remote=21088 dedup_ratio=0.244242 router_tokens_per_expert=[764, 758, 725, 767, 800, 758, 790, 777, 785, 742, 808, 794, 766, 773, 785, 806, 746, 747, 751, 727, 776, 782, 750, 748, 770, 808, 767, 783, 776, 773, 809, 718, 766, 790, 750, 778, 799, 759, 765, 792, 763, 784, 743, 737, 792, 784, 776, 807, 766, 769, 785, 781, 752, 773, 720, 734, 769, 768, 788, 746, 744, 713, 766, 764] payload_rows_per_dst=[9250, 9224, 9194, 9266, 9464, 9336, 9346, 9214] dup_rows_per_dst=[3028, 3024, 2860, 2952, 3124, 3016, 3084, 2922] combine_rows_per_src=[9250, 9224, 9194, 9266, 9464, 9336, 9346, 9214] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74740 payload_rows_remote=65304 dup_rows=23564 dup_rows_remote=20558 dedup_ratio=0.239705 router_tokens_per_expert=[766, 764, 747, 784, 717, 795, 748, 743, 795, 749, 797, 742, 777, 782, 793, 786, 755, 820, 765, 763, 710, 742, 766, 777, 763, 803, 789, 790, 783, 758, 747, 758, 718, 728, 772, 762, 754, 794, 730, 759, 760, 771, 761, 781, 733, 778, 728, 780, 776, 761, 810, 773, 798, 831, 784, 749, 772, 773, 804, 745, 801, 749, 766, 777] payload_rows_per_dst=[9266, 9436, 9252, 9358, 9234, 9294, 9478, 9422] dup_rows_per_dst=[2862, 3006, 2944, 3024, 2800, 2890, 3086, 2952] combine_rows_per_src=[9266, 9436, 9252, 9358, 9234, 9294, 9478, 9422] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265560064 dispatch_meta_bytes=342288 combine_payload_bytes=265560064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265902352 ingress_bytes=300163072 link_bytes=300163072 token_bytes=531462416 weight_movement_bytes=34603008 total_nvlink_bytes=566065424 weight_to_token_ratio=0.065109 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129489475e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74480 payload_rows_remote=65154 dup_rows=23824 dup_rows_remote=20942 dedup_ratio=0.242350 router_tokens_per_expert=[807, 811, 759, 775, 794, 766, 757, 755, 758, 810, 742, 776, 759, 792, 752, 785, 796, 801, 745, 755, 780, 775, 734, 792, 809, 712, 748, 795, 752, 719, 763, 767, 734, 771, 793, 788, 760, 743, 756, 759, 771, 804, 760, 771, 779, 761, 776, 788, 767, 776, 769, 762, 770, 739, 778, 744, 745, 757, 769, 760, 742, 752, 767, 800] payload_rows_per_dst=[9348, 9360, 9384, 9188, 9326, 9380, 9196, 9298] dup_rows_per_dst=[3100, 2988, 2972, 2942, 2882, 3040, 3014, 2886] combine_rows_per_src=[9348, 9360, 9384, 9188, 9326, 9380, 9196, 9298] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=267198464 dispatch_meta_bytes=345336 combine_payload_bytes=267198464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267543800 ingress_bytes=301801472 link_bytes=301801472 token_bytes=534742264 weight_movement_bytes=34603008 total_nvlink_bytes=569345272 weight_to_token_ratio=0.064710 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135654642e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267149312 dispatch_meta_bytes=344408 combine_payload_bytes=267149312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267493720 ingress_bytes=301752320 link_bytes=301752320 token_bytes=534643032 weight_movement_bytes=34603008 total_nvlink_bytes=569246040 weight_to_token_ratio=0.064722 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135469687e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74386 payload_rows_remote=65052 dup_rows=23918 dup_rows_remote=21044 dedup_ratio=0.243306 router_tokens_per_expert=[746, 743, 725, 754, 776, 784, 806, 720, 763, 778, 768, 801, 755, 765, 767, 786, 791, 724, 775, 753, 736, 798, 833, 748, 768, 761, 786, 756, 826, 755, 791, 745, 770, 769, 741, 748, 766, 726, 786, 776, 829, 765, 778, 773, 748, 766, 784, 794, 756, 784, 741, 838, 740, 736, 757, 752, 790, 798, 726, 815, 769, 831, 712, 705] payload_rows_per_dst=[9216, 9346, 9326, 9324, 9164, 9390, 9334, 9286] dup_rows_per_dst=[2892, 3020, 2990, 3052, 3000, 3084, 2874, 3006] combine_rows_per_src=[9216, 9346, 9326, 9324, 9164, 9390, 9334, 9286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266559488 dispatch_meta_bytes=344056 combine_payload_bytes=266559488 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266903544 ingress_bytes=335765504 link_bytes=335765504 token_bytes=533463032 weight_movement_bytes=69206016 total_nvlink_bytes=602669048 weight_to_token_ratio=0.129730 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263458560e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74452 payload_rows_remote=65070 dup_rows=23852 dup_rows_remote=20866 dedup_ratio=0.242635 router_tokens_per_expert=[787, 751, 734, 761, 775, 776, 746, 722, 735, 806, 717, 747, 778, 818, 797, 760, 752, 714, 801, 765, 802, 764, 742, 805, 779, 747, 752, 796, 815, 742, 784, 769, 767, 796, 753, 774, 763, 722, 774, 757, 780, 762, 781, 729, 773, 790, 771, 754, 788, 733, 805, 748, 753, 707, 812, 766, 795, 768, 770, 764, 817, 748, 769, 824] payload_rows_per_dst=[9158, 9392, 9278, 9382, 9320, 9322, 9224, 9376] dup_rows_per_dst=[2946, 2924, 3012, 2986, 2892, 2958, 3000, 3134] combine_rows_per_src=[9158, 9392, 9278, 9382, 9320, 9322, 9224, 9376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=264970240 dispatch_meta_bytes=342576 combine_payload_bytes=264970240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265312816 ingress_bytes=299573248 link_bytes=299573248 token_bytes=530283056 weight_movement_bytes=34603008 total_nvlink_bytes=564886064 weight_to_token_ratio=0.065254 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127270015e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266567680 dispatch_meta_bytes=344672 combine_payload_bytes=266567680 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266912352 ingress_bytes=370376704 link_bytes=370376704 token_bytes=533480032 weight_movement_bytes=103809024 total_nvlink_bytes=637289056 weight_to_token_ratio=0.194588 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393697719e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=267485184 dispatch_meta_bytes=343448 combine_payload_bytes=267485184 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267828632 ingress_bytes=371294208 link_bytes=371294208 token_bytes=535313816 weight_movement_bytes=103809024 total_nvlink_bytes=639122840 weight_to_token_ratio=0.193922 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397150213e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266452992 dispatch_meta_bytes=344384 combine_payload_bytes=266452992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266797376 ingress_bytes=335659008 link_bytes=335659008 token_bytes=533250368 weight_movement_bytes=69206016 total_nvlink_bytes=602456384 weight_to_token_ratio=0.129781 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263057824e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64770 dup_rows=24112 dup_rows_remote=20910 dedup_ratio=0.245280 router_tokens_per_expert=[777, 772, 730, 766, 776, 765, 739, 787, 801, 807, 788, 773, 777, 783, 802, 781, 738, 722, 772, 785, 761, 803, 785, 749, 754, 744, 771, 766, 718, 767, 821, 777, 755, 731, 775, 795, 782, 781, 806, 714, 733, 784, 799, 781, 817, 719, 762, 742, 786, 764, 779, 713, 760, 755, 772, 744, 768, 788, 792, 778, 726, 774, 746, 774] payload_rows_per_dst=[9286, 9422, 9268, 9230, 9246, 9268, 9158, 9314] dup_rows_per_dst=[2938, 3202, 2962, 3006, 3032, 3006, 2988, 2978] combine_rows_per_src=[9286, 9422, 9268, 9230, 9246, 9268, 9158, 9314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266526720 dispatch_meta_bytes=343744 combine_payload_bytes=266526720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266870464 ingress_bytes=335732736 link_bytes=335732736 token_bytes=533397184 weight_movement_bytes=69206016 total_nvlink_bytes=602603200 weight_to_token_ratio=0.129746 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263335257e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=65066 dup_rows=23930 dup_rows_remote=20842 dedup_ratio=0.243429 router_tokens_per_expert=[754, 735, 742, 764, 765, 738, 766, 772, 819, 705, 782, 776, 764, 787, 759, 756, 751, 748, 793, 764, 779, 773, 762, 786, 788, 779, 743, 764, 752, 801, 723, 796, 732, 827, 760, 776, 718, 800, 759, 817, 738, 773, 805, 751, 784, 786, 760, 801, 795, 755, 739, 705, 813, 748, 759, 742, 772, 763, 810, 806, 751, 757, 807, 757] payload_rows_per_dst=[9148, 9374, 9240, 9392, 9352, 9308, 9176, 9384] dup_rows_per_dst=[2924, 2922, 3072, 2900, 3026, 3088, 2936, 3062] combine_rows_per_src=[9148, 9374, 9240, 9392, 9352, 9308, 9176, 9384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266870784 dispatch_meta_bytes=344384 combine_payload_bytes=266870784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267215168 ingress_bytes=301473792 link_bytes=301473792 token_bytes=534085952 weight_movement_bytes=34603008 total_nvlink_bytes=568688960 weight_to_token_ratio=0.064789 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134421609e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74350 payload_rows_remote=65022 dup_rows=23954 dup_rows_remote=20930 dedup_ratio=0.243673 router_tokens_per_expert=[717, 773, 769, 783, 757, 790, 788, 737, 767, 748, 777, 750, 796, 718, 786, 768, 790, 753, 762, 736, 762, 751, 740, 789, 749, 755, 756, 764, 811, 739, 797, 757, 731, 769, 791, 762, 770, 778, 787, 788, 739, 752, 787, 729, 805, 756, 744, 805, 826, 770, 710, 769, 795, 794, 782, 787, 757, 777, 773, 765, 770, 789, 773, 787] payload_rows_per_dst=[9344, 9242, 9242, 9258, 9328, 9232, 9384, 9320] dup_rows_per_dst=[2884, 2978, 2924, 2998, 3024, 3002, 3082, 3062] combine_rows_per_src=[9344, 9242, 9242, 9258, 9328, 9232, 9384, 9320] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265297920 dispatch_meta_bytes=342720 combine_payload_bytes=265297920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265640640 ingress_bytes=299900928 link_bytes=299900928 token_bytes=530938560 weight_movement_bytes=34603008 total_nvlink_bytes=565541568 weight_to_token_ratio=0.065173 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128503048e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=65074 dup_rows=24032 dup_rows_remote=21216 dedup_ratio=0.244466 router_tokens_per_expert=[716, 770, 771, 761, 774, 745, 755, 809, 809, 740, 787, 754, 764, 821, 735, 765, 772, 762, 745, 762, 745, 764, 719, 738, 797, 776, 733, 749, 810, 773, 758, 769, 824, 775, 736, 782, 768, 756, 736, 759, 728, 773, 759, 782, 823, 795, 809, 763, 769, 765, 739, 788, 760, 743, 807, 804, 750, 805, 771, 797, 760, 713, 768, 797] payload_rows_per_dst=[9164, 9228, 9198, 9230, 9332, 9336, 9418, 9366] dup_rows_per_dst=[3038, 3122, 2816, 3100, 2940, 3128, 2932, 2956] combine_rows_per_src=[9164, 9228, 9198, 9230, 9332, 9336, 9418, 9366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74452 payload_rows_remote=65070 dup_rows=23852 dup_rows_remote=20866 dedup_ratio=0.242635 router_tokens_per_expert=[787, 751, 734, 761, 775, 776, 746, 722, 735, 806, 717, 747, 778, 818, 797, 760, 752, 714, 801, 765, 802, 764, 742, 805, 779, 747, 752, 796, 815, 742, 784, 769, 767, 796, 753, 774, 763, 722, 774, 757, 780, 762, 781, 729, 773, 790, 771, 754, 788, 733, 805, 748, 753, 707, 812, 766, 795, 768, 770, 764, 817, 748, 769, 824] payload_rows_per_dst=[9158, 9392, 9278, 9382, 9320, 9322, 9224, 9376] dup_rows_per_dst=[2946, 2924, 3012, 2986, 2892, 2958, 3000, 3134] combine_rows_per_src=[9158, 9392, 9278, 9382, 9320, 9322, 9224, 9376] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74164 payload_rows_remote=64598 dup_rows=24140 dup_rows_remote=20862 dedup_ratio=0.245565 router_tokens_per_expert=[742, 779, 781, 759, 751, 799, 787, 773, 793, 793, 745, 789, 743, 739, 749, 720, 729, 779, 809, 744, 770, 781, 756, 803, 769, 778, 730, 744, 759, 767, 765, 766, 735, 767, 818, 767, 794, 807, 749, 786, 783, 807, 762, 784, 732, 755, 772, 742, 771, 780, 752, 824, 735, 740, 760, 804, 741, 770, 754, 782, 798, 730, 794, 766] payload_rows_per_dst=[9566, 9402, 9044, 9574, 9016, 9324, 9136, 9102] dup_rows_per_dst=[3278, 3126, 2842, 3292, 2880, 3112, 2810, 2800] combine_rows_per_src=[9566, 9402, 9044, 9574, 9016, 9324, 9136, 9102] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74210 payload_rows_remote=64690 dup_rows=24094 dup_rows_remote=20954 dedup_ratio=0.245097 router_tokens_per_expert=[742, 756, 785, 734, 788, 791, 726, 782, 806, 771, 718, 773, 763, 757, 764, 820, 775, 748, 747, 806, 763, 801, 765, 774, 776, 755, 785, 757, 797, 774, 762, 759, 766, 804, 770, 753, 745, 723, 775, 772, 802, 731, 752, 779, 769, 834, 781, 750, 721, 805, 761, 761, 785, 749, 810, 741, 714, 778, 781, 758, 766, 722, 769, 805] payload_rows_per_dst=[9520, 9110, 9502, 9346, 9248, 9052, 9288, 9144] dup_rows_per_dst=[3140, 2864, 3300, 3026, 2968, 2900, 2978, 2918] combine_rows_per_src=[9520, 9110, 9502, 9346, 9248, 9052, 9288, 9144] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74278 payload_rows_remote=64966 dup_rows=24026 dup_rows_remote=20810 dedup_ratio=0.244405 router_tokens_per_expert=[770, 767, 781, 743, 775, 747, 759, 747, 785, 731, 765, 708, 764, 755, 761, 743, 792, 753, 761, 800, 799, 742, 758, 771, 816, 780, 769, 758, 710, 777, 752, 759, 726, 815, 792, 795, 818, 733, 741, 769, 680, 788, 776, 772, 770, 778, 726, 742, 811, 762, 762, 797, 803, 741, 795, 807, 795, 780, 764, 771, 791, 780, 804, 770] payload_rows_per_dst=[9352, 9148, 9248, 9236, 9384, 9238, 9360, 9312] dup_rows_per_dst=[2826, 2876, 3104, 3006, 2994, 2826, 3178, 3216] combine_rows_per_src=[9352, 9148, 9248, 9236, 9384, 9238, 9360, 9312] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266510336 dispatch_meta_bytes=343632 combine_payload_bytes=266510336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266853968 ingress_bytes=301113344 link_bytes=301113344 token_bytes=533364304 weight_movement_bytes=34603008 total_nvlink_bytes=567967312 weight_to_token_ratio=0.064877 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133065272e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74162 payload_rows_remote=64978 dup_rows=24142 dup_rows_remote=21250 dedup_ratio=0.245585 router_tokens_per_expert=[738, 790, 717, 758, 777, 781, 798, 774, 808, 779, 734, 758, 750, 737, 773, 788, 768, 741, 770, 742, 752, 788, 826, 761, 738, 759, 788, 776, 746, 766, 781, 748, 746, 846, 743, 816, 814, 757, 803, 723, 762, 783, 793, 798, 762, 744, 767, 731, 751, 780, 722, 773, 809, 735, 735, 733, 814, 776, 752, 782, 720, 792, 791, 789] payload_rows_per_dst=[9268, 9226, 9206, 9236, 9414, 9232, 9184, 9396] dup_rows_per_dst=[2998, 3028, 3090, 2968, 3082, 3048, 2892, 3036] combine_rows_per_src=[9268, 9226, 9206, 9236, 9414, 9232, 9184, 9396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266330112 dispatch_meta_bytes=343808 combine_payload_bytes=266330112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266673920 ingress_bytes=300933120 link_bytes=300933120 token_bytes=533004032 weight_movement_bytes=34603008 total_nvlink_bytes=567607040 weight_to_token_ratio=0.064921 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132387103e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12258702816 prefetch_bytes=761266176 reduce_bytes=484442112 weight_movement_bytes=1245708288 egress_bytes=6133306848 ingress_bytes=7371104256 link_bytes=7371104256 total_nvlink_bytes=13504411104 weight_to_token_ratio=0.101618 prefetch_share_of_ingress=0.103277 dedup_ratio=0.243937 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=14 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028913 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.205950952e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=345160 combine_payload_bytes=266543104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266888264 ingress_bytes=301146112 link_bytes=301146112 token_bytes=533431368 weight_movement_bytes=34603008 total_nvlink_bytes=568034376 weight_to_token_ratio=0.064869 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133188575e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266526720 dispatch_meta_bytes=343744 combine_payload_bytes=266526720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266870464 ingress_bytes=335732736 link_bytes=335732736 token_bytes=533397184 weight_movement_bytes=69206016 total_nvlink_bytes=602603200 weight_to_token_ratio=0.129746 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263335257e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74278 payload_rows_remote=64966 dup_rows=24026 dup_rows_remote=20810 dedup_ratio=0.244405 router_tokens_per_expert=[770, 767, 781, 743, 775, 747, 759, 747, 785, 731, 765, 708, 764, 755, 761, 743, 792, 753, 761, 800, 799, 742, 758, 771, 816, 780, 769, 758, 710, 777, 752, 759, 726, 815, 792, 795, 818, 733, 741, 769, 680, 788, 776, 772, 770, 778, 726, 742, 811, 762, 762, 797, 803, 741, 795, 807, 795, 780, 764, 771, 791, 780, 804, 770] payload_rows_per_dst=[9352, 9148, 9248, 9236, 9384, 9238, 9360, 9312] dup_rows_per_dst=[2826, 2876, 3104, 3006, 2994, 2826, 3178, 3216] combine_rows_per_src=[9352, 9148, 9248, 9236, 9384, 9238, 9360, 9312] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74386 payload_rows_remote=65052 dup_rows=23918 dup_rows_remote=21044 dedup_ratio=0.243306 router_tokens_per_expert=[746, 743, 725, 754, 776, 784, 806, 720, 763, 778, 768, 801, 755, 765, 767, 786, 791, 724, 775, 753, 736, 798, 833, 748, 768, 761, 786, 756, 826, 755, 791, 745, 770, 769, 741, 748, 766, 726, 786, 776, 829, 765, 778, 773, 748, 766, 784, 794, 756, 784, 741, 838, 740, 736, 757, 752, 790, 798, 726, 815, 769, 831, 712, 705] payload_rows_per_dst=[9216, 9346, 9326, 9324, 9164, 9390, 9334, 9286] dup_rows_per_dst=[2892, 3020, 2990, 3052, 3000, 3084, 2874, 3006] combine_rows_per_src=[9216, 9346, 9326, 9324, 9164, 9390, 9334, 9286] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=264970240 dispatch_meta_bytes=342576 combine_payload_bytes=264970240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265312816 ingress_bytes=299573248 link_bytes=299573248 token_bytes=530283056 weight_movement_bytes=34603008 total_nvlink_bytes=564886064 weight_to_token_ratio=0.065254 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127270015e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64770 dup_rows=24112 dup_rows_remote=20910 dedup_ratio=0.245280 router_tokens_per_expert=[777, 772, 730, 766, 776, 765, 739, 787, 801, 807, 788, 773, 777, 783, 802, 781, 738, 722, 772, 785, 761, 803, 785, 749, 754, 744, 771, 766, 718, 767, 821, 777, 755, 731, 775, 795, 782, 781, 806, 714, 733, 784, 799, 781, 817, 719, 762, 742, 786, 764, 779, 713, 760, 755, 772, 744, 768, 788, 792, 778, 726, 774, 746, 774] payload_rows_per_dst=[9286, 9422, 9268, 9230, 9246, 9268, 9158, 9314] dup_rows_per_dst=[2938, 3202, 2962, 3006, 3032, 3006, 2988, 2978] combine_rows_per_src=[9286, 9422, 9268, 9230, 9246, 9268, 9158, 9314] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12262011640 prefetch_bytes=553648128 reduce_bytes=899678208 weight_movement_bytes=1453326336 egress_bytes=6134960888 ingress_bytes=7580377088 link_bytes=7580377088 total_nvlink_bytes=13715337976 weight_to_token_ratio=0.118523 prefetch_share_of_ingress=0.073037 dedup_ratio=0.243660 prefetch_slots=32 unused_prefetch_slots=336 slot_utilization=0.086957 reduce_slots=26 remote_demand=16 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=28321 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.240189074e-04 prefetch_bytes_per_gemm_flop=9.057971014e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74592 payload_rows_remote=65242 dup_rows=23712 dup_rows_remote=20832 dedup_ratio=0.241211 router_tokens_per_expert=[818, 792, 756, 776, 777, 753, 775, 759, 719, 744, 780, 789, 793, 767, 768, 728, 783, 794, 761, 795, 783, 751, 781, 704, 760, 741, 749, 760, 808, 778, 731, 739, 729, 781, 794, 813, 756, 744, 735, 763, 749, 751, 776, 788, 798, 793, 765, 775, 758, 777, 800, 706, 790, 824, 788, 767, 749, 778, 740, 783, 764, 768, 756, 782] payload_rows_per_dst=[9334, 9232, 9388, 9248, 9350, 9352, 9406, 9282] dup_rows_per_dst=[3078, 2944, 2916, 2884, 2880, 3038, 3014, 2958] combine_rows_per_src=[9334, 9232, 9388, 9248, 9350, 9352, 9406, 9282] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266100736 dispatch_meta_bytes=343104 combine_payload_bytes=266100736 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266443840 ingress_bytes=404512768 link_bytes=404512768 token_bytes=532544576 weight_movement_bytes=138412032 total_nvlink_bytes=670956608 weight_to_token_ratio=0.259907 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522148980e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266452992 dispatch_meta_bytes=344384 combine_payload_bytes=266452992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266797376 ingress_bytes=335659008 link_bytes=335659008 token_bytes=533250368 weight_movement_bytes=69206016 total_nvlink_bytes=602456384 weight_to_token_ratio=0.129781 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263057824e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265297920 dispatch_meta_bytes=342720 combine_payload_bytes=265297920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265640640 ingress_bytes=299900928 link_bytes=299900928 token_bytes=530938560 weight_movement_bytes=34603008 total_nvlink_bytes=565541568 weight_to_token_ratio=0.065173 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128503048e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=267231232 dispatch_meta_bytes=344296 combine_payload_bytes=267231232 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267575528 ingress_bytes=336437248 link_bytes=336437248 token_bytes=534806760 weight_movement_bytes=69206016 total_nvlink_bytes=604012776 weight_to_token_ratio=0.129404 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265986279e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=64958 dup_rows=23966 dup_rows_remote=20872 dedup_ratio=0.243795 router_tokens_per_expert=[728, 760, 825, 777, 751, 752, 803, 701, 765, 774, 788, 781, 789, 743, 792, 741, 782, 767, 797, 829, 764, 828, 742, 728, 714, 786, 764, 706, 798, 766, 795, 754, 793, 772, 775, 801, 803, 750, 727, 793, 774, 776, 771, 726, 793, 739, 744, 749, 727, 748, 771, 799, 755, 819, 756, 786, 770, 773, 758, 781, 758, 769, 767, 739] payload_rows_per_dst=[9192, 9398, 9380, 9170, 9396, 9194, 9266, 9342] dup_rows_per_dst=[3002, 2948, 3094, 2996, 3032, 2950, 3056, 2888] combine_rows_per_src=[9192, 9398, 9380, 9170, 9396, 9194, 9266, 9342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74144 payload_rows_remote=64830 dup_rows=24160 dup_rows_remote=21170 dedup_ratio=0.245768 router_tokens_per_expert=[793, 732, 743, 780, 757, 735, 762, 745, 785, 788, 783, 720, 759, 765, 793, 758, 831, 747, 733, 785, 747, 778, 776, 818, 732, 765, 755, 764, 789, 780, 797, 770, 763, 736, 734, 788, 754, 773, 835, 758, 742, 777, 797, 730, 770, 777, 789, 768, 747, 792, 794, 739, 813, 730, 750, 775, 783, 771, 751, 771, 775, 782, 771, 752] payload_rows_per_dst=[9170, 9240, 9326, 9314, 9272, 9190, 9354, 9278] dup_rows_per_dst=[2924, 3062, 3104, 2990, 3010, 3110, 2926, 3034] combine_rows_per_src=[9170, 9240, 9326, 9314, 9272, 9190, 9354, 9278] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12233454992 prefetch_bytes=692060160 reduce_bytes=622854144 weight_movement_bytes=1314914304 egress_bytes=6120674704 ingress_bytes=7427694592 link_bytes=7427694592 total_nvlink_bytes=13548369296 weight_to_token_ratio=0.107485 prefetch_share_of_ingress=0.093173 dedup_ratio=0.244450 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=18 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029130 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.215209425e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12258702816 prefetch_bytes=761266176 reduce_bytes=484442112 weight_movement_bytes=1245708288 egress_bytes=6133306848 ingress_bytes=7371104256 link_bytes=7371104256 total_nvlink_bytes=13504411104 weight_to_token_ratio=0.101618 prefetch_share_of_ingress=0.103277 dedup_ratio=0.243937 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=14 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028913 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.205950952e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74164 payload_rows_remote=64598 dup_rows=24140 dup_rows_remote=20862 dedup_ratio=0.245565 router_tokens_per_expert=[742, 779, 781, 759, 751, 799, 787, 773, 793, 793, 745, 789, 743, 739, 749, 720, 729, 779, 809, 744, 770, 781, 756, 803, 769, 778, 730, 744, 759, 767, 765, 766, 735, 767, 818, 767, 794, 807, 749, 786, 783, 807, 762, 784, 732, 755, 772, 742, 771, 780, 752, 824, 735, 740, 760, 804, 741, 770, 754, 782, 798, 730, 794, 766] payload_rows_per_dst=[9566, 9402, 9044, 9574, 9016, 9324, 9136, 9102] dup_rows_per_dst=[3278, 3126, 2842, 3292, 2880, 3112, 2810, 2800] combine_rows_per_src=[9566, 9402, 9044, 9574, 9016, 9324, 9136, 9102] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12262407376 prefetch_bytes=726663168 reduce_bytes=553648128 weight_movement_bytes=1280311296 egress_bytes=6135160016 ingress_bytes=7407558656 link_bytes=7407558656 total_nvlink_bytes=13542718672 weight_to_token_ratio=0.104409 prefetch_share_of_ingress=0.098098 dedup_ratio=0.243728 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=16 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028913 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.211915081e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=343320 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266411288 ingress_bytes=335273984 link_bytes=335273984 token_bytes=532479256 weight_movement_bytes=69206016 total_nvlink_bytes=601685272 weight_to_token_ratio=0.129969 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261609010e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265543680 dispatch_meta_bytes=344000 combine_payload_bytes=265543680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265887680 ingress_bytes=300146688 link_bytes=300146688 token_bytes=531431360 weight_movement_bytes=34603008 total_nvlink_bytes=566034368 weight_to_token_ratio=0.065113 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129427823e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74162 payload_rows_remote=64978 dup_rows=24142 dup_rows_remote=21250 dedup_ratio=0.245585 router_tokens_per_expert=[738, 790, 717, 758, 777, 781, 798, 774, 808, 779, 734, 758, 750, 737, 773, 788, 768, 741, 770, 742, 752, 788, 826, 761, 738, 759, 788, 776, 746, 766, 781, 748, 746, 846, 743, 816, 814, 757, 803, 723, 762, 783, 793, 798, 762, 744, 767, 731, 751, 780, 722, 773, 809, 735, 735, 733, 814, 776, 752, 782, 720, 792, 791, 789] payload_rows_per_dst=[9268, 9226, 9206, 9236, 9414, 9232, 9184, 9396] dup_rows_per_dst=[2998, 3028, 3090, 2968, 3082, 3048, 2892, 3036] combine_rows_per_src=[9268, 9226, 9206, 9236, 9414, 9232, 9184, 9396] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=264593408 dispatch_meta_bytes=341840 combine_payload_bytes=264593408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264935248 ingress_bytes=299196416 link_bytes=299196416 token_bytes=529528656 weight_movement_bytes=34603008 total_nvlink_bytes=564131664 weight_to_token_ratio=0.065347 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125852026e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12258284272 prefetch_bytes=692060160 reduce_bytes=380633088 weight_movement_bytes=1072693248 egress_bytes=6133101296 ingress_bytes=7197876224 link_bytes=7197876224 total_nvlink_bytes=13330977520 weight_to_token_ratio=0.087508 prefetch_share_of_ingress=0.096148 dedup_ratio=0.244418 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=11 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.177609946e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12254326472 prefetch_bytes=657457152 reduce_bytes=934281216 weight_movement_bytes=1591738368 egress_bytes=6131117768 ingress_bytes=7714947072 link_bytes=7715290728 total_nvlink_bytes=13846064840 weight_to_token_ratio=0.129892 prefetch_share_of_ingress=0.085219 dedup_ratio=0.243881 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=27 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.262261647e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266100736 dispatch_meta_bytes=343104 combine_payload_bytes=266100736 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266443840 ingress_bytes=404512768 link_bytes=404512768 token_bytes=532544576 weight_movement_bytes=138412032 total_nvlink_bytes=670956608 weight_to_token_ratio=0.259907 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522148980e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12262011640 prefetch_bytes=553648128 reduce_bytes=899678208 weight_movement_bytes=1453326336 egress_bytes=6134960888 ingress_bytes=7580377088 link_bytes=7580377088 total_nvlink_bytes=13715337976 weight_to_token_ratio=0.118523 prefetch_share_of_ingress=0.073037 dedup_ratio=0.243660 prefetch_slots=32 unused_prefetch_slots=336 slot_utilization=0.086957 reduce_slots=26 remote_demand=16 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=28321 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.240189074e-04 prefetch_bytes_per_gemm_flop=9.057971014e-06 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=344912 combine_payload_bytes=266149888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266494800 ingress_bytes=300752896 link_bytes=300752896 token_bytes=532644688 weight_movement_bytes=34603008 total_nvlink_bytes=567247696 weight_to_token_ratio=0.064965 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131708935e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265543680 dispatch_meta_bytes=344000 combine_payload_bytes=265543680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265887680 ingress_bytes=300146688 link_bytes=300146688 token_bytes=531431360 weight_movement_bytes=34603008 total_nvlink_bytes=566034368 weight_to_token_ratio=0.065113 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129427823e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12226985568 prefetch_bytes=761266176 reduce_bytes=761266176 weight_movement_bytes=1522532352 egress_bytes=6117441120 ingress_bytes=7632076800 link_bytes=7632420360 total_nvlink_bytes=13749517920 weight_to_token_ratio=0.124522 prefetch_share_of_ingress=0.099746 dedup_ratio=0.244912 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=22 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028152 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.248703624e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12233454992 prefetch_bytes=692060160 reduce_bytes=622854144 weight_movement_bytes=1314914304 egress_bytes=6120674704 ingress_bytes=7427694592 link_bytes=7427694592 total_nvlink_bytes=13548369296 weight_to_token_ratio=0.107485 prefetch_share_of_ingress=0.093173 dedup_ratio=0.244450 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=18 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029130 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.215209425e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74480 payload_rows_remote=65154 dup_rows=23824 dup_rows_remote=20942 dedup_ratio=0.242350 router_tokens_per_expert=[807, 811, 759, 775, 794, 766, 757, 755, 758, 810, 742, 776, 759, 792, 752, 785, 796, 801, 745, 755, 780, 775, 734, 792, 809, 712, 748, 795, 752, 719, 763, 767, 734, 771, 793, 788, 760, 743, 756, 759, 771, 804, 760, 771, 779, 761, 776, 788, 767, 776, 769, 762, 770, 739, 778, 744, 745, 757, 769, 760, 742, 752, 767, 800] payload_rows_per_dst=[9348, 9360, 9384, 9188, 9326, 9380, 9196, 9298] dup_rows_per_dst=[3100, 2988, 2972, 2942, 2882, 3040, 3014, 2886] combine_rows_per_src=[9348, 9360, 9384, 9188, 9326, 9380, 9196, 9298] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=64958 dup_rows=23966 dup_rows_remote=20872 dedup_ratio=0.243795 router_tokens_per_expert=[728, 760, 825, 777, 751, 752, 803, 701, 765, 774, 788, 781, 789, 743, 792, 741, 782, 767, 797, 829, 764, 828, 742, 728, 714, 786, 764, 706, 798, 766, 795, 754, 793, 772, 775, 801, 803, 750, 727, 793, 774, 776, 771, 726, 793, 739, 744, 749, 727, 748, 771, 799, 755, 819, 756, 786, 770, 773, 758, 781, 758, 769, 767, 739] payload_rows_per_dst=[9192, 9398, 9380, 9170, 9396, 9194, 9266, 9342] dup_rows_per_dst=[3002, 2948, 3094, 2996, 3032, 2950, 3056, 2888] combine_rows_per_src=[9192, 9398, 9380, 9170, 9396, 9194, 9266, 9342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12254326472 prefetch_bytes=657457152 reduce_bytes=934281216 weight_movement_bytes=1591738368 egress_bytes=6131117768 ingress_bytes=7714947072 link_bytes=7715290728 total_nvlink_bytes=13846064840 weight_to_token_ratio=0.129892 prefetch_share_of_ingress=0.085219 dedup_ratio=0.243881 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=27 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.262261647e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=64938 dup_rows=23962 dup_rows_remote=20826 dedup_ratio=0.243754 router_tokens_per_expert=[740, 818, 770, 766, 719, 803, 756, 771, 753, 776, 720, 725, 725, 762, 755, 788, 747, 738, 809, 774, 796, 783, 757, 754, 746, 748, 698, 757, 782, 761, 748, 793, 753, 785, 790, 764, 791, 759, 799, 776, 812, 788, 763, 756, 716, 782, 766, 767, 771, 764, 817, 799, 773, 797, 780, 769, 759, 813, 721, 771, 777, 781, 768, 787] payload_rows_per_dst=[9274, 9208, 9354, 9154, 9368, 9278, 9404, 9302] dup_rows_per_dst=[3012, 2800, 2962, 2912, 3066, 3022, 3136, 3052] combine_rows_per_src=[9274, 9208, 9354, 9154, 9368, 9278, 9404, 9302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266870784 dispatch_meta_bytes=344384 combine_payload_bytes=266870784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267215168 ingress_bytes=301473792 link_bytes=301473792 token_bytes=534085952 weight_movement_bytes=34603008 total_nvlink_bytes=568688960 weight_to_token_ratio=0.064789 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134421609e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=343320 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266411288 ingress_bytes=335273984 link_bytes=335273984 token_bytes=532479256 weight_movement_bytes=69206016 total_nvlink_bytes=601685272 weight_to_token_ratio=0.129969 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261609010e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265986048 dispatch_meta_bytes=343056 combine_payload_bytes=265986048 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266329104 ingress_bytes=335192064 link_bytes=335192064 token_bytes=532315152 weight_movement_bytes=69206016 total_nvlink_bytes=601521168 weight_to_token_ratio=0.130009 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261300752e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12258284272 prefetch_bytes=692060160 reduce_bytes=380633088 weight_movement_bytes=1072693248 egress_bytes=6133101296 ingress_bytes=7197876224 link_bytes=7197876224 total_nvlink_bytes=13330977520 weight_to_token_ratio=0.087508 prefetch_share_of_ingress=0.096148 dedup_ratio=0.244418 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=11 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.177609946e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12255037232 prefetch_bytes=588251136 reduce_bytes=795869184 weight_movement_bytes=1384120320 egress_bytes=6131476272 ingress_bytes=7507681280 link_bytes=7507681280 total_nvlink_bytes=13639157552 weight_to_token_ratio=0.112943 prefetch_share_of_ingress=0.078353 dedup_ratio=0.244336 prefetch_slots=34 unused_prefetch_slots=334 slot_utilization=0.092391 reduce_slots=23 remote_demand=17 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029022 gemm_inner_pad_upper_bound=28448 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.228295663e-04 prefetch_bytes_per_gemm_flop=9.624094203e-06 anomalies=none - [2026-08-26 15:10:20] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 796.2 | throughput per GPU (TFLOP/s/GPU): 160.2 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.947229E+00 | loss scale: 1.0 | grad norm: 42.080 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74592 payload_rows_remote=65242 dup_rows=23712 dup_rows_remote=20832 dedup_ratio=0.241211 router_tokens_per_expert=[818, 792, 756, 776, 777, 753, 775, 759, 719, 744, 780, 789, 793, 767, 768, 728, 783, 794, 761, 795, 783, 751, 781, 704, 760, 741, 749, 760, 808, 778, 731, 739, 729, 781, 794, 813, 756, 744, 735, 763, 749, 751, 776, 788, 798, 793, 765, 775, 758, 777, 800, 706, 790, 824, 788, 767, 749, 778, 740, 783, 764, 768, 756, 782] payload_rows_per_dst=[9334, 9232, 9388, 9248, 9350, 9352, 9406, 9282] dup_rows_per_dst=[3078, 2944, 2916, 2884, 2880, 3038, 3014, 2958] combine_rows_per_src=[9334, 9232, 9388, 9248, 9350, 9352, 9406, 9282] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=267231232 dispatch_meta_bytes=344296 combine_payload_bytes=267231232 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267575528 ingress_bytes=336437248 link_bytes=336437248 token_bytes=534806760 weight_movement_bytes=69206016 total_nvlink_bytes=604012776 weight_to_token_ratio=0.129404 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265986279e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12262407376 prefetch_bytes=726663168 reduce_bytes=553648128 weight_movement_bytes=1280311296 egress_bytes=6135160016 ingress_bytes=7407558656 link_bytes=7407558656 total_nvlink_bytes=13542718672 weight_to_token_ratio=0.104409 prefetch_share_of_ingress=0.098098 dedup_ratio=0.243728 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=16 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028913 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.211915081e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none + [2026-08-26 15:58:09] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 791.1 | throughput per GPU (TFLOP/s/GPU): 161.3 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.947229E+00 | loss scale: 1.0 | grad norm: 42.080 | number of skipped iterations: 0 | number of nan iterations: 0 | WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74028 payload_rows_remote=64880 dup_rows=24276 dup_rows_remote=21192 dedup_ratio=0.246948 router_tokens_per_expert=[744, 774, 806, 762, 817, 777, 743, 803, 743, 797, 749, 766, 739, 784, 789, 749, 793, 799, 732, 743, 714, 748, 790, 762, 741, 803, 751, 768, 760, 735, 748, 830, 794, 762, 768, 769, 764, 776, 719, 815, 743, 781, 805, 738, 728, 788, 742, 762, 750, 795, 726, 744, 775, 789, 778, 784, 769, 787, 781, 749, 746, 800, 784, 782] payload_rows_per_dst=[9406, 9148, 9210, 9208, 9346, 9176, 9270, 9264] dup_rows_per_dst=[3046, 3084, 2952, 3064, 2988, 2998, 3012, 3132] combine_rows_per_src=[9406, 9148, 9210, 9208, 9346, 9176, 9270, 9264] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74070 payload_rows_remote=64896 dup_rows=24234 dup_rows_remote=21328 dedup_ratio=0.246521 router_tokens_per_expert=[736, 768, 738, 786, 748, 768, 741, 770, 705, 785, 780, 787, 707, 751, 772, 744, 767, 789, 769, 748, 776, 756, 754, 769, 760, 739, 842, 801, 770, 731, 788, 735, 782, 828, 756, 801, 781, 781, 764, 790, 758, 736, 781, 730, 783, 762, 722, 768, 804, 792, 758, 804, 777, 803, 759, 793, 762, 748, 787, 729, 785, 775, 802, 771] payload_rows_per_dst=[9200, 9078, 9196, 9296, 9432, 9174, 9422, 9272] dup_rows_per_dst=[2910, 2984, 3060, 3036, 3134, 2906, 3158, 3046] combine_rows_per_src=[9200, 9078, 9196, 9296, 9432, 9174, 9422, 9272] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74180 payload_rows_remote=64912 dup_rows=24124 dup_rows_remote=21076 dedup_ratio=0.245402 router_tokens_per_expert=[732, 794, 822, 742, 800, 768, 736, 722, 753, 746, 815, 770, 747, 790, 739, 756, 756, 754, 783, 788, 790, 763, 800, 754, 773, 737, 765, 789, 829, 835, 775, 755, 781, 727, 788, 786, 781, 768, 804, 718, 777, 780, 758, 807, 750, 745, 711, 744, 753, 804, 787, 780, 769, 719, 740, 757, 789, 759, 777, 768, 749, 749, 812, 737] payload_rows_per_dst=[9122, 9294, 9300, 9442, 9278, 9190, 9286, 9268] dup_rows_per_dst=[2894, 2938, 3076, 3146, 3142, 2918, 2962, 3048] combine_rows_per_src=[9122, 9294, 9300, 9442, 9278, 9190, 9286, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74308 payload_rows_remote=64950 dup_rows=23996 dup_rows_remote=21048 dedup_ratio=0.244100 router_tokens_per_expert=[758, 780, 775, 758, 797, 752, 743, 789, 764, 794, 799, 801, 713, 761, 774, 798, 807, 758, 830, 764, 794, 721, 750, 773, 746, 775, 774, 741, 709, 802, 721, 767, 764, 810, 762, 806, 795, 773, 742, 783, 761, 796, 783, 778, 784, 736, 742, 728, 775, 751, 742, 750, 800, 793, 718, 803, 761, 744, 761, 758, 764, 762, 776, 763] payload_rows_per_dst=[9358, 9452, 9594, 9156, 9128, 9220, 9148, 9252] dup_rows_per_dst=[2948, 3228, 3218, 2914, 2924, 2994, 2844, 2926] combine_rows_per_src=[9358, 9452, 9594, 9156, 9128, 9220, 9148, 9252] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74180 payload_rows_remote=64918 dup_rows=24124 dup_rows_remote=21094 dedup_ratio=0.245402 router_tokens_per_expert=[820, 767, 730, 745, 803, 779, 774, 771, 751, 771, 768, 743, 784, 780, 776, 749, 786, 753, 786, 727, 757, 788, 794, 755, 724, 791, 768, 780, 782, 770, 767, 763, 809, 746, 780, 776, 798, 721, 738, 744, 713, 767, 756, 726, 764, 811, 828, 754, 739, 852, 755, 718, 750, 783, 773, 779, 790, 767, 766, 755, 777, 776, 761, 778] payload_rows_per_dst=[9232, 9264, 9262, 9306, 9260, 9258, 9268, 9330] dup_rows_per_dst=[3146, 2980, 3030, 2984, 2964, 2980, 3030, 3010] combine_rows_per_src=[9232, 9264, 9262, 9306, 9260, 9258, 9268, 9330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74070 payload_rows_remote=64896 dup_rows=24234 dup_rows_remote=21328 dedup_ratio=0.246521 router_tokens_per_expert=[736, 768, 738, 786, 748, 768, 741, 770, 705, 785, 780, 787, 707, 751, 772, 744, 767, 789, 769, 748, 776, 756, 754, 769, 760, 739, 842, 801, 770, 731, 788, 735, 782, 828, 756, 801, 781, 781, 764, 790, 758, 736, 781, 730, 783, 762, 722, 768, 804, 792, 758, 804, 777, 803, 759, 793, 762, 748, 787, 729, 785, 775, 802, 771] payload_rows_per_dst=[9200, 9078, 9196, 9296, 9432, 9174, 9422, 9272] dup_rows_per_dst=[2910, 2984, 3060, 3036, 3134, 2906, 3158, 3046] combine_rows_per_src=[9200, 9078, 9196, 9296, 9432, 9174, 9422, 9272] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74282 payload_rows_remote=64982 dup_rows=24022 dup_rows_remote=20980 dedup_ratio=0.244364 router_tokens_per_expert=[766, 771, 779, 769, 811, 733, 739, 763, 749, 805, 807, 775, 758, 778, 764, 776, 783, 763, 705, 724, 713, 781, 771, 785, 756, 811, 784, 764, 804, 755, 748, 749, 798, 739, 749, 734, 724, 785, 761, 757, 740, 744, 811, 752, 787, 799, 806, 791, 734, 762, 794, 790, 729, 724, 784, 796, 781, 770, 772, 764, 761, 752, 842, 781] payload_rows_per_dst=[9238, 9332, 9190, 9300, 9170, 9412, 9304, 9336] dup_rows_per_dst=[3024, 3092, 2860, 3042, 2924, 3048, 2922, 3110] combine_rows_per_src=[9238, 9332, 9190, 9300, 9170, 9412, 9304, 9336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74180 payload_rows_remote=64912 dup_rows=24124 dup_rows_remote=21076 dedup_ratio=0.245402 router_tokens_per_expert=[732, 794, 822, 742, 800, 768, 736, 722, 753, 746, 815, 770, 747, 790, 739, 756, 756, 754, 783, 788, 790, 763, 800, 754, 773, 737, 765, 789, 829, 835, 775, 755, 781, 727, 788, 786, 781, 768, 804, 718, 777, 780, 758, 807, 750, 745, 711, 744, 753, 804, 787, 780, 769, 719, 740, 757, 789, 759, 777, 768, 749, 749, 812, 737] payload_rows_per_dst=[9122, 9294, 9300, 9442, 9278, 9190, 9286, 9268] dup_rows_per_dst=[2894, 2938, 3076, 3146, 3142, 2918, 2962, 3048] combine_rows_per_src=[9122, 9294, 9300, 9442, 9278, 9190, 9286, 9268] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74324 payload_rows_remote=64944 dup_rows=23980 dup_rows_remote=21000 dedup_ratio=0.243937 router_tokens_per_expert=[778, 771, 764, 755, 767, 785, 818, 736, 800, 776, 747, 771, 729, 757, 764, 782, 771, 755, 726, 724, 745, 780, 762, 734, 758, 761, 817, 789, 757, 774, 765, 750, 730, 764, 806, 798, 783, 819, 778, 702, 776, 773, 804, 756, 780, 757, 808, 789, 712, 755, 814, 794, 769, 781, 774, 742, 754, 767, 745, 741, 794, 762, 802, 755] payload_rows_per_dst=[9244, 9242, 9104, 9284, 9380, 9454, 9308, 9308] dup_rows_per_dst=[3104, 3010, 2890, 3058, 2980, 3032, 2974, 2932] combine_rows_per_src=[9244, 9242, 9104, 9284, 9380, 9454, 9308, 9308] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265748480 dispatch_meta_bytes=344288 combine_payload_bytes=265748480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266092768 ingress_bytes=300351488 link_bytes=300351488 token_bytes=531841248 weight_movement_bytes=34603008 total_nvlink_bytes=566444256 weight_to_token_ratio=0.065063 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130198469e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265814016 dispatch_meta_bytes=344896 combine_payload_bytes=265814016 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266158912 ingress_bytes=335020032 link_bytes=335020032 token_bytes=531972928 weight_movement_bytes=69206016 total_nvlink_bytes=601178944 weight_to_token_ratio=0.130093 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260653409e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265879552 dispatch_meta_bytes=343952 combine_payload_bytes=265879552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266223504 ingress_bytes=300482560 link_bytes=300482560 token_bytes=532103056 weight_movement_bytes=34603008 total_nvlink_bytes=566706064 weight_to_token_ratio=0.065031 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130691682e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266035200 dispatch_meta_bytes=343992 combine_payload_bytes=266035200 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266379192 ingress_bytes=404447232 link_bytes=404447232 token_bytes=532414392 weight_movement_bytes=138412032 total_nvlink_bytes=670826424 weight_to_token_ratio=0.259970 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521902373e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=344048 combine_payload_bytes=265904128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266248176 ingress_bytes=300507136 link_bytes=300507136 token_bytes=532152304 weight_movement_bytes=34603008 total_nvlink_bytes=566755312 weight_to_token_ratio=0.065025 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130784160e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265814016 dispatch_meta_bytes=344896 combine_payload_bytes=265814016 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266158912 ingress_bytes=335020032 link_bytes=335020032 token_bytes=531972928 weight_movement_bytes=69206016 total_nvlink_bytes=601178944 weight_to_token_ratio=0.130093 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260653409e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266166272 dispatch_meta_bytes=343848 combine_payload_bytes=266166272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266510120 ingress_bytes=300769280 link_bytes=300769280 token_bytes=532676392 weight_movement_bytes=34603008 total_nvlink_bytes=567279400 weight_to_token_ratio=0.064961 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131770587e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266010624 dispatch_meta_bytes=343776 combine_payload_bytes=266010624 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266354400 ingress_bytes=335216640 link_bytes=335216640 token_bytes=532365024 weight_movement_bytes=69206016 total_nvlink_bytes=601571040 weight_to_token_ratio=0.129997 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261393229e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265879552 dispatch_meta_bytes=343952 combine_payload_bytes=265879552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266223504 ingress_bytes=300482560 link_bytes=300482560 token_bytes=532103056 weight_movement_bytes=34603008 total_nvlink_bytes=566706064 weight_to_token_ratio=0.065031 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130691682e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74220 payload_rows_remote=64924 dup_rows=24084 dup_rows_remote=20984 dedup_ratio=0.244995 router_tokens_per_expert=[753, 803, 772, 779, 788, 714, 781, 826, 740, 747, 825, 763, 777, 706, 795, 736, 797, 754, 764, 786, 771, 768, 784, 757, 706, 759, 781, 796, 780, 703, 746, 763, 787, 768, 755, 754, 756, 749, 800, 758, 861, 743, 751, 761, 810, 764, 821, 761, 754, 759, 763, 802, 791, 751, 769, 809, 725, 788, 724, 787, 730, 760, 749, 772] payload_rows_per_dst=[9308, 9226, 9404, 9180, 9254, 9386, 9296, 9166] dup_rows_per_dst=[3124, 2952, 2958, 2888, 3000, 3158, 3100, 2904] combine_rows_per_src=[9308, 9226, 9404, 9180, 9254, 9386, 9296, 9166] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74308 payload_rows_remote=64950 dup_rows=23996 dup_rows_remote=21048 dedup_ratio=0.244100 router_tokens_per_expert=[758, 780, 775, 758, 797, 752, 743, 789, 764, 794, 799, 801, 713, 761, 774, 798, 807, 758, 830, 764, 794, 721, 750, 773, 746, 775, 774, 741, 709, 802, 721, 767, 764, 810, 762, 806, 795, 773, 742, 783, 761, 796, 783, 778, 784, 736, 742, 728, 775, 751, 742, 750, 800, 793, 718, 803, 761, 744, 761, 758, 764, 762, 776, 763] payload_rows_per_dst=[9358, 9452, 9594, 9156, 9128, 9220, 9148, 9252] dup_rows_per_dst=[2948, 3228, 3218, 2914, 2924, 2994, 2844, 2926] combine_rows_per_src=[9358, 9452, 9594, 9156, 9128, 9220, 9148, 9252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266010624 dispatch_meta_bytes=343776 combine_payload_bytes=266010624 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266354400 ingress_bytes=335216640 link_bytes=335216640 token_bytes=532365024 weight_movement_bytes=69206016 total_nvlink_bytes=601571040 weight_to_token_ratio=0.129997 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261393229e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=64970 dup_rows=24062 dup_rows_remote=21124 dedup_ratio=0.244771 router_tokens_per_expert=[766, 752, 700, 752, 760, 726, 759, 770, 763, 746, 767, 757, 754, 732, 763, 755, 787, 790, 771, 787, 744, 725, 791, 758, 801, 772, 754, 793, 789, 780, 841, 789, 734, 763, 738, 741, 768, 805, 789, 782, 789, 751, 775, 769, 789, 735, 737, 760, 786, 771, 752, 806, 781, 773, 769, 794, 761, 793, 749, 796, 754, 754, 770, 824] payload_rows_per_dst=[9140, 9122, 9286, 9496, 9200, 9272, 9354, 9372] dup_rows_per_dst=[2830, 2952, 3020, 3142, 3040, 2938, 3110, 3030] combine_rows_per_src=[9140, 9122, 9286, 9496, 9200, 9272, 9354, 9372] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=65108 dup_rows=23906 dup_rows_remote=20954 dedup_ratio=0.243184 router_tokens_per_expert=[796, 775, 762, 730, 701, 822, 764, 789, 764, 743, 737, 760, 766, 811, 750, 790, 748, 772, 814, 778, 754, 817, 733, 712, 780, 795, 789, 769, 744, 751, 788, 760, 781, 782, 750, 812, 764, 789, 769, 739, 739, 786, 775, 811, 777, 760, 763, 714, 754, 748, 759, 761, 788, 789, 761, 765, 798, 746, 782, 759, 755, 739, 782, 791] payload_rows_per_dst=[9306, 9290, 9196, 9412, 9282, 9366, 9250, 9296] dup_rows_per_dst=[2972, 2952, 3060, 2940, 3090, 2884, 3000, 3008] combine_rows_per_src=[9306, 9290, 9196, 9412, 9282, 9366, 9250, 9296] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=64826 dup_rows=24118 dup_rows_remote=20980 dedup_ratio=0.245341 router_tokens_per_expert=[824, 740, 768, 768, 796, 749, 706, 758, 745, 776, 800, 758, 749, 772, 751, 735, 780, 773, 780, 758, 757, 767, 805, 758, 700, 755, 791, 772, 754, 804, 733, 701, 790, 842, 796, 770, 748, 770, 785, 788, 814, 729, 798, 749, 782, 731, 757, 758, 749, 771, 754, 705, 783, 782, 816, 755, 793, 767, 802, 760, 817, 795, 773, 740] payload_rows_per_dst=[9240, 9174, 9126, 9144, 9496, 9348, 9298, 9360] dup_rows_per_dst=[2978, 2998, 3034, 2876, 3082, 3080, 2932, 3138] combine_rows_per_src=[9240, 9174, 9126, 9144, 9496, 9348, 9298, 9360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265928704 dispatch_meta_bytes=343632 combine_payload_bytes=265928704 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266272336 ingress_bytes=335134720 link_bytes=335134720 token_bytes=532201040 weight_movement_bytes=69206016 total_nvlink_bytes=601407056 weight_to_token_ratio=0.130037 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261084971e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74246 payload_rows_remote=64634 dup_rows=24058 dup_rows_remote=20706 dedup_ratio=0.244731 router_tokens_per_expert=[788, 745, 782, 789, 767, 791, 742, 760, 718, 688, 755, 751, 833, 795, 786, 785, 761, 805, 743, 740, 789, 798, 757, 774, 809, 755, 740, 744, 751, 769, 767, 778, 744, 790, 703, 809, 754, 781, 824, 780, 763, 755, 789, 777, 776, 793, 743, 760, 819, 732, 796, 765, 764, 762, 747, 745, 783, 751, 770, 758, 822, 785, 742, 715] payload_rows_per_dst=[9612, 9232, 9302, 9046, 9428, 9320, 9154, 9152] dup_rows_per_dst=[3352, 3006, 3016, 2806, 3090, 2992, 2870, 2926] combine_rows_per_src=[9612, 9232, 9302, 9046, 9428, 9320, 9154, 9152] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74230 payload_rows_remote=65018 dup_rows=24074 dup_rows_remote=21082 dedup_ratio=0.244893 router_tokens_per_expert=[747, 734, 782, 835, 799, 818, 771, 744, 770, 781, 772, 779, 721, 736, 791, 804, 736, 767, 754, 788, 757, 775, 764, 761, 736, 815, 803, 736, 798, 806, 742, 761, 743, 737, 714, 807, 761, 784, 778, 791, 770, 768, 820, 782, 760, 755, 753, 752, 764, 758, 764, 724, 772, 778, 800, 747, 785, 766, 761, 714, 780, 770, 749, 762] payload_rows_per_dst=[9352, 9308, 9212, 9344, 9188, 9322, 9270, 9234] dup_rows_per_dst=[3108, 3000, 2992, 3050, 3042, 2998, 2944, 2940] combine_rows_per_src=[9352, 9308, 9212, 9344, 9188, 9322, 9270, 9234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266682368 dispatch_meta_bytes=344248 combine_payload_bytes=266682368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267026616 ingress_bytes=301285376 link_bytes=301285376 token_bytes=533708984 weight_movement_bytes=34603008 total_nvlink_bytes=568311992 weight_to_token_ratio=0.064835 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133712614e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=64970 dup_rows=24062 dup_rows_remote=21124 dedup_ratio=0.244771 router_tokens_per_expert=[766, 752, 700, 752, 760, 726, 759, 770, 763, 746, 767, 757, 754, 732, 763, 755, 787, 790, 771, 787, 744, 725, 791, 758, 801, 772, 754, 793, 789, 780, 841, 789, 734, 763, 738, 741, 768, 805, 789, 782, 789, 751, 775, 769, 789, 735, 737, 760, 786, 771, 752, 806, 781, 773, 769, 794, 761, 793, 749, 796, 754, 754, 770, 824] payload_rows_per_dst=[9140, 9122, 9286, 9496, 9200, 9272, 9354, 9372] dup_rows_per_dst=[2830, 2952, 3020, 3142, 3040, 2938, 3110, 3030] combine_rows_per_src=[9140, 9122, 9286, 9496, 9200, 9272, 9354, 9372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=64826 dup_rows=24118 dup_rows_remote=20980 dedup_ratio=0.245341 router_tokens_per_expert=[824, 740, 768, 768, 796, 749, 706, 758, 745, 776, 800, 758, 749, 772, 751, 735, 780, 773, 780, 758, 757, 767, 805, 758, 700, 755, 791, 772, 754, 804, 733, 701, 790, 842, 796, 770, 748, 770, 785, 788, 814, 729, 798, 749, 782, 731, 757, 758, 749, 771, 754, 705, 783, 782, 816, 755, 793, 767, 802, 760, 817, 795, 773, 740] payload_rows_per_dst=[9240, 9174, 9126, 9144, 9496, 9348, 9298, 9360] dup_rows_per_dst=[2978, 2998, 3034, 2876, 3082, 3080, 2932, 3138] combine_rows_per_src=[9240, 9174, 9126, 9144, 9496, 9348, 9298, 9360] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=65048 dup_rows=23928 dup_rows_remote=20826 dedup_ratio=0.243408 router_tokens_per_expert=[758, 772, 734, 757, 748, 784, 757, 801, 775, 780, 781, 767, 783, 772, 731, 725, 771, 759, 779, 776, 769, 753, 775, 808, 792, 780, 792, 758, 749, 717, 813, 814, 768, 743, 749, 828, 727, 727, 736, 763, 826, 724, 744, 754, 755, 760, 763, 785, 771, 779, 771, 753, 752, 767, 784, 824, 817, 750, 784, 742, 757, 763, 798, 758] payload_rows_per_dst=[9302, 9254, 9340, 9328, 9218, 9248, 9340, 9346] dup_rows_per_dst=[2920, 2974, 3040, 3102, 2864, 2974, 3062, 2992] combine_rows_per_src=[9302, 9254, 9340, 9328, 9218, 9248, 9340, 9346] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74482 payload_rows_remote=65202 dup_rows=23822 dup_rows_remote=20844 dedup_ratio=0.242330 router_tokens_per_expert=[734, 769, 748, 679, 810, 746, 756, 758, 845, 724, 745, 800, 799, 766, 768, 797, 748, 799, 771, 789, 802, 764, 752, 779, 788, 759, 782, 775, 767, 790, 757, 758, 776, 755, 770, 779, 756, 794, 766, 733, 769, 744, 743, 825, 796, 781, 748, 774, 789, 763, 734, 813, 750, 802, 750, 758, 769, 717, 778, 793, 759, 732, 752, 760] payload_rows_per_dst=[9166, 9348, 9438, 9388, 9280, 9378, 9330, 9154] dup_rows_per_dst=[2834, 3140, 2970, 2964, 2978, 2982, 2988, 2966] combine_rows_per_src=[9166, 9348, 9438, 9388, 9280, 9378, 9330, 9154] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265928704 dispatch_meta_bytes=343632 combine_payload_bytes=265928704 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266272336 ingress_bytes=335134720 link_bytes=335134720 token_bytes=532201040 weight_movement_bytes=69206016 total_nvlink_bytes=601407056 weight_to_token_ratio=0.130037 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261084971e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266117120 dispatch_meta_bytes=344376 combine_payload_bytes=266117120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266461496 ingress_bytes=300720128 link_bytes=300720128 token_bytes=532578616 weight_movement_bytes=34603008 total_nvlink_bytes=567181624 weight_to_token_ratio=0.064973 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131585632e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266035200 dispatch_meta_bytes=343992 combine_payload_bytes=266035200 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266379192 ingress_bytes=404447232 link_bytes=404447232 token_bytes=532414392 weight_movement_bytes=138412032 total_nvlink_bytes=670826424 weight_to_token_ratio=0.259970 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521902373e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266682368 dispatch_meta_bytes=344248 combine_payload_bytes=266682368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267026616 ingress_bytes=301285376 link_bytes=301285376 token_bytes=533708984 weight_movement_bytes=34603008 total_nvlink_bytes=568311992 weight_to_token_ratio=0.064835 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133712614e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265527296 dispatch_meta_bytes=343224 combine_payload_bytes=265527296 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265870520 ingress_bytes=369336320 link_bytes=369336320 token_bytes=531397816 weight_movement_bytes=103809024 total_nvlink_bytes=635206840 weight_to_token_ratio=0.195351 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389782838e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=264740864 dispatch_meta_bytes=341360 combine_payload_bytes=264740864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265082224 ingress_bytes=299343872 link_bytes=299343872 token_bytes=529823088 weight_movement_bytes=34603008 total_nvlink_bytes=564426096 weight_to_token_ratio=0.065310 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126406891e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=344400 combine_payload_bytes=266313728 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266658128 ingress_bytes=370122752 link_bytes=370122752 token_bytes=532971856 weight_movement_bytes=103809024 total_nvlink_bytes=636780880 weight_to_token_ratio=0.194774 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392742118e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266436608 dispatch_meta_bytes=343496 combine_payload_bytes=266436608 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266780104 ingress_bytes=335642624 link_bytes=335642624 token_bytes=533216712 weight_movement_bytes=69206016 total_nvlink_bytes=602422728 weight_to_token_ratio=0.129790 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262996173e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265527296 dispatch_meta_bytes=343224 combine_payload_bytes=265527296 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265870520 ingress_bytes=369336320 link_bytes=369336320 token_bytes=531397816 weight_movement_bytes=103809024 total_nvlink_bytes=635206840 weight_to_token_ratio=0.195351 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389782838e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266117120 dispatch_meta_bytes=344376 combine_payload_bytes=266117120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266461496 ingress_bytes=300720128 link_bytes=300720128 token_bytes=532578616 weight_movement_bytes=34603008 total_nvlink_bytes=567181624 weight_to_token_ratio=0.064973 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131585632e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267067392 dispatch_meta_bytes=344184 combine_payload_bytes=267067392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267411576 ingress_bytes=301670400 link_bytes=301670400 token_bytes=534478968 weight_movement_bytes=34603008 total_nvlink_bytes=569081976 weight_to_token_ratio=0.064742 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135161429e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=64938 dup_rows=24048 dup_rows_remote=21092 dedup_ratio=0.244629 router_tokens_per_expert=[754, 753, 784, 776, 819, 770, 776, 786, 784, 782, 748, 741, 825, 783, 747, 808, 740, 766, 797, 817, 765, 767, 725, 771, 750, 783, 754, 735, 762, 727, 749, 781, 745, 779, 777, 800, 746, 774, 775, 757, 758, 736, 784, 747, 762, 826, 755, 769, 753, 738, 736, 772, 778, 756, 768, 772, 760, 739, 756, 737, 785, 781, 790, 816] payload_rows_per_dst=[9304, 9342, 9250, 9158, 9344, 9318, 9232, 9308] dup_rows_per_dst=[3132, 3094, 3046, 2924, 2962, 2956, 2914, 3020] combine_rows_per_src=[9304, 9342, 9250, 9158, 9344, 9318, 9232, 9308] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266436608 dispatch_meta_bytes=343496 combine_payload_bytes=266436608 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266780104 ingress_bytes=335642624 link_bytes=335642624 token_bytes=533216712 weight_movement_bytes=69206016 total_nvlink_bytes=602422728 weight_to_token_ratio=0.129790 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262996173e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=64814 dup_rows=24072 dup_rows_remote=21088 dedup_ratio=0.244873 router_tokens_per_expert=[778, 795, 753, 762, 730, 763, 769, 746, 752, 799, 778, 837, 729, 792, 786, 804, 744, 803, 751, 742, 760, 825, 756, 755, 769, 754, 791, 761, 798, 746, 769, 745, 739, 738, 785, 767, 772, 754, 683, 775, 808, 741, 773, 758, 753, 767, 774, 727, 755, 778, 768, 751, 765, 797, 751, 836, 805, 743, 759, 808, 788, 751, 760, 781] payload_rows_per_dst=[9190, 9442, 9172, 9356, 9194, 9162, 9418, 9298] dup_rows_per_dst=[3002, 3112, 3100, 2910, 2832, 3040, 2984, 3092] combine_rows_per_src=[9190, 9442, 9172, 9356, 9194, 9162, 9418, 9298] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74390 payload_rows_remote=65142 dup_rows=23914 dup_rows_remote=21042 dedup_ratio=0.243266 router_tokens_per_expert=[759, 783, 746, 775, 796, 747, 768, 750, 755, 742, 754, 807, 756, 746, 748, 752, 765, 797, 777, 742, 758, 779, 741, 762, 753, 694, 744, 754, 787, 789, 818, 789, 769, 811, 774, 775, 778, 778, 774, 773, 760, 730, 726, 775, 768, 804, 741, 794, 790, 748, 799, 766, 795, 780, 770, 769, 801, 808, 725, 740, 777, 783, 790, 748] payload_rows_per_dst=[9232, 9248, 9376, 9256, 9418, 9260, 9274, 9326] dup_rows_per_dst=[3016, 2872, 2866, 3000, 3046, 2936, 3160, 3018] combine_rows_per_src=[9232, 9248, 9376, 9256, 9418, 9260, 9274, 9326] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74246 payload_rows_remote=64634 dup_rows=24058 dup_rows_remote=20706 dedup_ratio=0.244731 router_tokens_per_expert=[788, 745, 782, 789, 767, 791, 742, 760, 718, 688, 755, 751, 833, 795, 786, 785, 761, 805, 743, 740, 789, 798, 757, 774, 809, 755, 740, 744, 751, 769, 767, 778, 744, 790, 703, 809, 754, 781, 824, 780, 763, 755, 789, 777, 776, 793, 743, 760, 819, 732, 796, 765, 764, 762, 747, 745, 783, 751, 770, 758, 822, 785, 742, 715] payload_rows_per_dst=[9612, 9232, 9302, 9046, 9428, 9320, 9154, 9152] dup_rows_per_dst=[3352, 3006, 3016, 2806, 3090, 2992, 2870, 2926] combine_rows_per_src=[9612, 9232, 9302, 9046, 9428, 9320, 9154, 9152] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266821632 dispatch_meta_bytes=344736 combine_payload_bytes=266821632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267166368 ingress_bytes=301424640 link_bytes=301424640 token_bytes=533988000 weight_movement_bytes=34603008 total_nvlink_bytes=568591008 weight_to_token_ratio=0.064801 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134236654e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265478144 dispatch_meta_bytes=343608 combine_payload_bytes=265478144 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265821752 ingress_bytes=334684160 link_bytes=334684160 token_bytes=531299896 weight_movement_bytes=69206016 total_nvlink_bytes=600505912 weight_to_token_ratio=0.130258 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259389550e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74228 payload_rows_remote=64898 dup_rows=24076 dup_rows_remote=21030 dedup_ratio=0.244914 router_tokens_per_expert=[796, 795, 717, 747, 817, 747, 774, 773, 783, 773, 767, 755, 766, 780, 809, 805, 790, 749, 739, 739, 766, 752, 715, 807, 768, 784, 812, 773, 746, 775, 785, 815, 755, 774, 767, 753, 776, 746, 775, 784, 753, 753, 782, 776, 737, 784, 789, 784, 749, 782, 766, 761, 809, 744, 784, 740, 740, 726, 765, 746, 762, 762, 776, 733] payload_rows_per_dst=[9172, 9468, 9244, 9406, 9172, 9338, 9098, 9330] dup_rows_per_dst=[2946, 3212, 2870, 3076, 2990, 3080, 2856, 3046] combine_rows_per_src=[9172, 9468, 9244, 9406, 9172, 9338, 9098, 9330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74430 payload_rows_remote=65218 dup_rows=23874 dup_rows_remote=21094 dedup_ratio=0.242859 router_tokens_per_expert=[763, 762, 746, 722, 793, 704, 758, 748, 781, 766, 760, 777, 757, 798, 767, 749, 757, 778, 803, 813, 830, 764, 781, 775, 758, 813, 759, 752, 756, 797, 722, 772, 762, 752, 740, 771, 778, 789, 725, 813, 750, 791, 806, 782, 758, 751, 793, 741, 772, 791, 742, 786, 772, 752, 757, 793, 787, 761, 752, 744, 746, 756, 804, 754] payload_rows_per_dst=[9212, 9316, 9468, 9232, 9284, 9326, 9360, 9232] dup_rows_per_dst=[2780, 2994, 3174, 3026, 2936, 3018, 2970, 2976] combine_rows_per_src=[9212, 9316, 9468, 9232, 9284, 9326, 9360, 9232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=64938 dup_rows=24048 dup_rows_remote=21092 dedup_ratio=0.244629 router_tokens_per_expert=[754, 753, 784, 776, 819, 770, 776, 786, 784, 782, 748, 741, 825, 783, 747, 808, 740, 766, 797, 817, 765, 767, 725, 771, 750, 783, 754, 735, 762, 727, 749, 781, 745, 779, 777, 800, 746, 774, 775, 757, 758, 736, 784, 747, 762, 826, 755, 769, 753, 738, 736, 772, 778, 756, 768, 772, 760, 739, 756, 737, 785, 781, 790, 816] payload_rows_per_dst=[9304, 9342, 9250, 9158, 9344, 9318, 9232, 9308] dup_rows_per_dst=[3132, 3094, 3046, 2924, 2962, 2956, 2914, 3020] combine_rows_per_src=[9304, 9342, 9250, 9158, 9344, 9318, 9232, 9308] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74386 payload_rows_remote=64990 dup_rows=23918 dup_rows_remote=20864 dedup_ratio=0.243306 router_tokens_per_expert=[755, 760, 724, 790, 802, 767, 752, 762, 741, 767, 782, 753, 754, 739, 769, 724, 775, 767, 760, 776, 771, 812, 786, 778, 713, 798, 756, 777, 801, 741, 743, 780, 761, 803, 795, 752, 749, 768, 780, 801, 763, 772, 798, 828, 807, 737, 777, 736, 807, 718, 763, 722, 740, 774, 787, 763, 790, 791, 795, 727, 758, 751, 802, 762] payload_rows_per_dst=[9232, 9196, 9396, 9276, 9324, 9360, 9274, 9328] dup_rows_per_dst=[2992, 2862, 3054, 2942, 3094, 3076, 2874, 3024] combine_rows_per_src=[9232, 9196, 9396, 9276, 9324, 9360, 9274, 9328] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74160 payload_rows_remote=64964 dup_rows=24144 dup_rows_remote=21178 dedup_ratio=0.245605 router_tokens_per_expert=[772, 728, 755, 807, 809, 780, 751, 763, 786, 782, 751, 734, 760, 771, 715, 791, 742, 810, 784, 739, 799, 724, 801, 738, 797, 734, 749, 817, 797, 808, 771, 764, 743, 756, 745, 751, 785, 753, 821, 727, 790, 707, 748, 779, 724, 826, 741, 741, 777, 749, 807, 788, 798, 845, 825, 775, 725, 738, 773, 754, 744, 760, 766, 762] payload_rows_per_dst=[9314, 9206, 9296, 9374, 9196, 9184, 9500, 9090] dup_rows_per_dst=[3016, 2974, 2978, 3100, 2966, 2928, 3228, 2954] combine_rows_per_src=[9314, 9206, 9296, 9374, 9196, 9184, 9500, 9090] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74538 payload_rows_remote=65296 dup_rows=23766 dup_rows_remote=20912 dedup_ratio=0.241760 router_tokens_per_expert=[823, 727, 784, 785, 831, 743, 783, 798, 769, 755, 754, 771, 777, 775, 767, 763, 719, 799, 756, 736, 749, 784, 724, 769, 788, 806, 733, 749, 739, 708, 762, 763, 792, 780, 796, 740, 779, 736, 777, 747, 786, 760, 791, 752, 791, 760, 793, 787, 736, 736, 795, 745, 790, 763, 803, 808, 709, 774, 772, 803, 793, 745, 775, 749] payload_rows_per_dst=[9508, 9328, 9182, 9242, 9238, 9372, 9308, 9360] dup_rows_per_dst=[3040, 2934, 2890, 2854, 3056, 3068, 3044, 2880] combine_rows_per_src=[9508, 9328, 9182, 9242, 9238, 9372, 9308, 9360] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265986048 dispatch_meta_bytes=344120 combine_payload_bytes=265986048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266330168 ingress_bytes=300589056 link_bytes=300589056 token_bytes=532316216 weight_movement_bytes=34603008 total_nvlink_bytes=566919224 weight_to_token_ratio=0.065005 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131092418e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74538 payload_rows_remote=65296 dup_rows=23766 dup_rows_remote=20912 dedup_ratio=0.241760 router_tokens_per_expert=[823, 727, 784, 785, 831, 743, 783, 798, 769, 755, 754, 771, 777, 775, 767, 763, 719, 799, 756, 736, 749, 784, 724, 769, 788, 806, 733, 749, 739, 708, 762, 763, 792, 780, 796, 740, 779, 736, 777, 747, 786, 760, 791, 752, 791, 760, 793, 787, 736, 736, 795, 745, 790, 763, 803, 808, 709, 774, 772, 803, 793, 745, 775, 749] payload_rows_per_dst=[9508, 9328, 9182, 9242, 9238, 9372, 9308, 9360] dup_rows_per_dst=[3040, 2934, 2890, 2854, 3056, 3068, 3044, 2880] combine_rows_per_src=[9508, 9328, 9182, 9242, 9238, 9372, 9308, 9360] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266821632 dispatch_meta_bytes=344736 combine_payload_bytes=266821632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267166368 ingress_bytes=301424640 link_bytes=301424640 token_bytes=533988000 weight_movement_bytes=34603008 total_nvlink_bytes=568591008 weight_to_token_ratio=0.064801 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134236654e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265478144 dispatch_meta_bytes=343608 combine_payload_bytes=265478144 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265821752 ingress_bytes=334684160 link_bytes=334684160 token_bytes=531299896 weight_movement_bytes=69206016 total_nvlink_bytes=600505912 weight_to_token_ratio=0.130258 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259389550e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=264740864 dispatch_meta_bytes=341360 combine_payload_bytes=264740864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265082224 ingress_bytes=299343872 link_bytes=299343872 token_bytes=529823088 weight_movement_bytes=34603008 total_nvlink_bytes=564426096 weight_to_token_ratio=0.065310 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126406891e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265822208 dispatch_meta_bytes=343712 combine_payload_bytes=265822208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266165920 ingress_bytes=300425216 link_bytes=300425216 token_bytes=531988128 weight_movement_bytes=34603008 total_nvlink_bytes=566591136 weight_to_token_ratio=0.065045 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130475902e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267132928 dispatch_meta_bytes=345248 combine_payload_bytes=267132928 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267478176 ingress_bytes=370941952 link_bytes=370941952 token_bytes=534611104 weight_movement_bytes=103809024 total_nvlink_bytes=638420128 weight_to_token_ratio=0.194177 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395824702e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265986048 dispatch_meta_bytes=344120 combine_payload_bytes=265986048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266330168 ingress_bytes=300589056 link_bytes=300589056 token_bytes=532316216 weight_movement_bytes=34603008 total_nvlink_bytes=566919224 weight_to_token_ratio=0.065005 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131092418e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74644 payload_rows_remote=65270 dup_rows=23660 dup_rows_remote=20752 dedup_ratio=0.240682 router_tokens_per_expert=[789, 764, 767, 765, 738, 772, 792, 733, 709, 799, 822, 760, 739, 774, 756, 782, 766, 756, 770, 770, 726, 859, 777, 759, 776, 735, 773, 807, 767, 732, 778, 825, 773, 774, 732, 795, 784, 775, 767, 751, 771, 735, 742, 726, 764, 754, 765, 795, 799, 746, 774, 742, 759, 766, 757, 796, 775, 787, 758, 745, 766, 777, 773, 792] payload_rows_per_dst=[9250, 9374, 9354, 9428, 9308, 9234, 9292, 9404] dup_rows_per_dst=[2990, 2908, 3012, 2958, 2994, 2870, 2986, 2942] combine_rows_per_src=[9250, 9374, 9354, 9428, 9308, 9234, 9292, 9404] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=343416 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266542456 ingress_bytes=300802048 link_bytes=300802048 token_bytes=532741496 weight_movement_bytes=34603008 total_nvlink_bytes=567344504 weight_to_token_ratio=0.064953 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131893890e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=344568 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266437112 ingress_bytes=369901568 link_bytes=369901568 token_bytes=532529656 weight_movement_bytes=103809024 total_nvlink_bytes=636338680 weight_to_token_ratio=0.194936 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391909821e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=64998 dup_rows=24098 dup_rows_remote=21094 dedup_ratio=0.245138 router_tokens_per_expert=[814, 738, 806, 797, 744, 741, 791, 773, 770, 782, 776, 790, 717, 754, 759, 781, 764, 772, 759, 736, 769, 771, 753, 783, 821, 770, 790, 777, 761, 807, 738, 755, 768, 768, 798, 793, 779, 750, 716, 767, 767, 738, 760, 744, 759, 727, 773, 774, 740, 778, 764, 754, 735, 812, 758, 765, 759, 830, 762, 736, 785, 753, 771, 810] payload_rows_per_dst=[9320, 9214, 9186, 9432, 9340, 9100, 9208, 9406] dup_rows_per_dst=[3088, 3044, 3028, 3006, 2938, 2984, 3004, 3006] combine_rows_per_src=[9320, 9214, 9186, 9432, 9340, 9100, 9208, 9406] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267452416 dispatch_meta_bytes=344832 combine_payload_bytes=267452416 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267797248 ingress_bytes=336658432 link_bytes=336658432 token_bytes=535249664 weight_movement_bytes=69206016 total_nvlink_bytes=604455680 weight_to_token_ratio=0.129297 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266818576e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267345920 dispatch_meta_bytes=344088 combine_payload_bytes=267345920 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267690008 ingress_bytes=336551936 link_bytes=336551936 token_bytes=535035928 weight_movement_bytes=69206016 total_nvlink_bytes=604241944 weight_to_token_ratio=0.129348 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266417841e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266231808 dispatch_meta_bytes=344368 combine_payload_bytes=266231808 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266576176 ingress_bytes=335437824 link_bytes=335437824 token_bytes=532807984 weight_movement_bytes=69206016 total_nvlink_bytes=602014000 weight_to_token_ratio=0.129889 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262225527e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74184 payload_rows_remote=64872 dup_rows=24120 dup_rows_remote=21238 dedup_ratio=0.245361 router_tokens_per_expert=[801, 760, 719, 774, 750, 762, 764, 792, 802, 781, 763, 767, 804, 765, 748, 779, 766, 734, 762, 753, 753, 749, 765, 747, 792, 745, 772, 762, 745, 708, 807, 790, 738, 808, 796, 773, 724, 779, 773, 785, 782, 790, 759, 767, 761, 772, 777, 745, 784, 794, 775, 806, 742, 797, 794, 760, 789, 762, 791, 753, 750, 751, 765, 729] payload_rows_per_dst=[9232, 9368, 9204, 8918, 9284, 9432, 9434, 9312] dup_rows_per_dst=[3012, 3050, 2854, 2892, 3068, 3306, 3056, 2882] combine_rows_per_src=[9232, 9368, 9204, 8918, 9284, 9432, 9434, 9312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=65056 dup_rows=23962 dup_rows_remote=20996 dedup_ratio=0.243754 router_tokens_per_expert=[747, 754, 759, 756, 706, 790, 817, 729, 747, 735, 756, 765, 720, 766, 738, 755, 813, 807, 763, 727, 794, 759, 791, 787, 776, 834, 763, 760, 800, 800, 760, 740, 726, 812, 755, 767, 708, 790, 773, 775, 770, 790, 784, 768, 785, 786, 725, 788, 794, 775, 765, 722, 798, 820, 751, 791, 766, 749, 757, 785, 774, 757, 744, 788] payload_rows_per_dst=[9286, 9026, 9654, 9232, 9286, 9402, 9382, 9074] dup_rows_per_dst=[2966, 2812, 3294, 2942, 2996, 2990, 3040, 2922] combine_rows_per_src=[9286, 9026, 9654, 9232, 9286, 9402, 9382, 9074] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=65044 dup_rows=23982 dup_rows_remote=21078 dedup_ratio=0.243958 router_tokens_per_expert=[771, 791, 749, 771, 764, 745, 786, 733, 791, 772, 765, 783, 772, 794, 796, 725, 777, 767, 759, 770, 803, 759, 803, 749, 771, 747, 793, 793, 781, 786, 794, 720, 771, 779, 738, 783, 748, 786, 747, 799, 781, 798, 737, 754, 728, 794, 721, 778, 772, 739, 719, 769, 796, 770, 790, 751, 771, 737, 774, 732, 793, 823, 752, 742] payload_rows_per_dst=[9304, 9426, 9306, 9194, 9254, 9278, 9320, 9240] dup_rows_per_dst=[2916, 2970, 3068, 3176, 3048, 2904, 2892, 3008] combine_rows_per_src=[9304, 9426, 9306, 9194, 9254, 9278, 9320, 9240] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74138 payload_rows_remote=64872 dup_rows=24166 dup_rows_remote=21184 dedup_ratio=0.245829 router_tokens_per_expert=[800, 717, 769, 737, 778, 782, 828, 803, 740, 757, 768, 797, 766, 746, 731, 760, 744, 758, 762, 755, 731, 816, 745, 756, 804, 757, 778, 841, 772, 798, 783, 736, 814, 737, 793, 752, 756, 749, 779, 744, 767, 752, 780, 782, 787, 735, 703, 762, 806, 792, 754, 758, 743, 798, 782, 752, 796, 766, 753, 778, 746, 764, 826, 731] payload_rows_per_dst=[9402, 9206, 9158, 9384, 9266, 9226, 9262, 9234] dup_rows_per_dst=[3026, 2924, 2976, 3154, 2982, 2910, 3108, 3086] combine_rows_per_src=[9402, 9206, 9158, 9384, 9266, 9226, 9262, 9234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74644 payload_rows_remote=65270 dup_rows=23660 dup_rows_remote=20752 dedup_ratio=0.240682 router_tokens_per_expert=[789, 764, 767, 765, 738, 772, 792, 733, 709, 799, 822, 760, 739, 774, 756, 782, 766, 756, 770, 770, 726, 859, 777, 759, 776, 735, 773, 807, 767, 732, 778, 825, 773, 774, 732, 795, 784, 775, 767, 751, 771, 735, 742, 726, 764, 754, 765, 795, 799, 746, 774, 742, 759, 766, 757, 796, 775, 787, 758, 745, 766, 777, 773, 792] payload_rows_per_dst=[9250, 9374, 9354, 9428, 9308, 9234, 9292, 9404] dup_rows_per_dst=[2990, 2908, 3012, 2958, 2994, 2870, 2986, 2942] combine_rows_per_src=[9250, 9374, 9354, 9428, 9308, 9234, 9292, 9404] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74430 payload_rows_remote=65218 dup_rows=23874 dup_rows_remote=21094 dedup_ratio=0.242859 router_tokens_per_expert=[763, 762, 746, 722, 793, 704, 758, 748, 781, 766, 760, 777, 757, 798, 767, 749, 757, 778, 803, 813, 830, 764, 781, 775, 758, 813, 759, 752, 756, 797, 722, 772, 762, 752, 740, 771, 778, 789, 725, 813, 750, 791, 806, 782, 758, 751, 793, 741, 772, 791, 742, 786, 772, 752, 757, 793, 787, 761, 752, 744, 746, 756, 804, 754] payload_rows_per_dst=[9212, 9316, 9468, 9232, 9284, 9326, 9360, 9232] dup_rows_per_dst=[2780, 2994, 3174, 3026, 2936, 3018, 2970, 2976] combine_rows_per_src=[9212, 9316, 9468, 9232, 9284, 9326, 9360, 9232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=64998 dup_rows=24098 dup_rows_remote=21094 dedup_ratio=0.245138 router_tokens_per_expert=[814, 738, 806, 797, 744, 741, 791, 773, 770, 782, 776, 790, 717, 754, 759, 781, 764, 772, 759, 736, 769, 771, 753, 783, 821, 770, 790, 777, 761, 807, 738, 755, 768, 768, 798, 793, 779, 750, 716, 767, 767, 738, 760, 744, 759, 727, 773, 774, 740, 778, 764, 754, 735, 812, 758, 765, 759, 830, 762, 736, 785, 753, 771, 810] payload_rows_per_dst=[9320, 9214, 9186, 9432, 9340, 9100, 9208, 9406] dup_rows_per_dst=[3088, 3044, 3028, 3006, 2938, 2984, 3004, 3006] combine_rows_per_src=[9320, 9214, 9186, 9432, 9340, 9100, 9208, 9406] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74184 payload_rows_remote=64872 dup_rows=24120 dup_rows_remote=21238 dedup_ratio=0.245361 router_tokens_per_expert=[801, 760, 719, 774, 750, 762, 764, 792, 802, 781, 763, 767, 804, 765, 748, 779, 766, 734, 762, 753, 753, 749, 765, 747, 792, 745, 772, 762, 745, 708, 807, 790, 738, 808, 796, 773, 724, 779, 773, 785, 782, 790, 759, 767, 761, 772, 777, 745, 784, 794, 775, 806, 742, 797, 794, 760, 789, 762, 791, 753, 750, 751, 765, 729] payload_rows_per_dst=[9232, 9368, 9204, 8918, 9284, 9432, 9434, 9312] dup_rows_per_dst=[3012, 3050, 2854, 2892, 3068, 3306, 3056, 2882] combine_rows_per_src=[9232, 9368, 9204, 8918, 9284, 9432, 9434, 9312] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74124 payload_rows_remote=65020 dup_rows=24180 dup_rows_remote=21278 dedup_ratio=0.245972 router_tokens_per_expert=[769, 741, 777, 782, 776, 748, 777, 775, 800, 771, 802, 815, 807, 795, 769, 765, 707, 803, 769, 774, 753, 682, 764, 751, 760, 827, 820, 778, 746, 740, 757, 790, 804, 734, 804, 755, 740, 757, 778, 780, 713, 746, 757, 763, 811, 765, 770, 773, 728, 711, 768, 751, 787, 794, 743, 732, 726, 774, 777, 770, 808, 798, 785, 760] payload_rows_per_dst=[9208, 9486, 9104, 9332, 9290, 9138, 9182, 9384] dup_rows_per_dst=[3082, 3162, 2902, 3104, 3014, 3058, 2846, 3012] combine_rows_per_src=[9208, 9486, 9104, 9332, 9290, 9138, 9182, 9384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74138 payload_rows_remote=64872 dup_rows=24166 dup_rows_remote=21184 dedup_ratio=0.245829 router_tokens_per_expert=[800, 717, 769, 737, 778, 782, 828, 803, 740, 757, 768, 797, 766, 746, 731, 760, 744, 758, 762, 755, 731, 816, 745, 756, 804, 757, 778, 841, 772, 798, 783, 736, 814, 737, 793, 752, 756, 749, 779, 744, 767, 752, 780, 782, 787, 735, 703, 762, 806, 792, 754, 758, 743, 798, 782, 752, 796, 766, 753, 778, 746, 764, 826, 731] payload_rows_per_dst=[9402, 9206, 9158, 9384, 9266, 9226, 9262, 9234] dup_rows_per_dst=[3026, 2924, 2976, 3154, 2982, 2910, 3108, 3086] combine_rows_per_src=[9402, 9206, 9158, 9384, 9266, 9226, 9262, 9234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74310 payload_rows_remote=65068 dup_rows=23994 dup_rows_remote=20962 dedup_ratio=0.244080 router_tokens_per_expert=[814, 761, 777, 769, 788, 775, 784, 773, 768, 768, 808, 726, 778, 775, 791, 750, 759, 765, 755, 794, 745, 707, 797, 751, 775, 774, 750, 773, 771, 798, 749, 747, 759, 751, 797, 756, 746, 829, 750, 738, 802, 769, 778, 775, 758, 787, 735, 749, 785, 717, 710, 789, 755, 737, 789, 820, 743, 722, 761, 788, 722, 806, 802, 812] payload_rows_per_dst=[9382, 9324, 9274, 9242, 9344, 9274, 9202, 9268] dup_rows_per_dst=[3100, 3004, 2872, 3032, 2908, 3032, 3002, 3044] combine_rows_per_src=[9382, 9324, 9274, 9242, 9344, 9274, 9202, 9268] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=344488 combine_payload_bytes=266420224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266764712 ingress_bytes=301023232 link_bytes=301023232 token_bytes=533184936 weight_movement_bytes=34603008 total_nvlink_bytes=567787944 weight_to_token_ratio=0.064899 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132726188e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265715712 dispatch_meta_bytes=344440 combine_payload_bytes=265715712 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266060152 ingress_bytes=369524736 link_bytes=369524736 token_bytes=531775864 weight_movement_bytes=103809024 total_nvlink_bytes=635584888 weight_to_token_ratio=0.195212 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390491832e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=344488 combine_payload_bytes=266420224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266764712 ingress_bytes=301023232 link_bytes=301023232 token_bytes=533184936 weight_movement_bytes=34603008 total_nvlink_bytes=567787944 weight_to_token_ratio=0.064899 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132726188e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266469376 dispatch_meta_bytes=344208 combine_payload_bytes=266469376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266813584 ingress_bytes=301072384 link_bytes=301072384 token_bytes=533282960 weight_movement_bytes=34603008 total_nvlink_bytes=567885968 weight_to_token_ratio=0.064887 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132911143e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267345920 dispatch_meta_bytes=344088 combine_payload_bytes=267345920 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267690008 ingress_bytes=336551936 link_bytes=336551936 token_bytes=535035928 weight_movement_bytes=69206016 total_nvlink_bytes=604241944 weight_to_token_ratio=0.129348 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266417841e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267132928 dispatch_meta_bytes=345248 combine_payload_bytes=267132928 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267478176 ingress_bytes=370941952 link_bytes=370941952 token_bytes=534611104 weight_movement_bytes=103809024 total_nvlink_bytes=638420128 weight_to_token_ratio=0.194177 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395824702e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266231808 dispatch_meta_bytes=344368 combine_payload_bytes=266231808 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266576176 ingress_bytes=335437824 link_bytes=335437824 token_bytes=532807984 weight_movement_bytes=69206016 total_nvlink_bytes=602014000 weight_to_token_ratio=0.129889 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262225527e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265715712 dispatch_meta_bytes=344224 combine_payload_bytes=265715712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266059936 ingress_bytes=300318720 link_bytes=300318720 token_bytes=531775648 weight_movement_bytes=34603008 total_nvlink_bytes=566378656 weight_to_token_ratio=0.065071 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130075166e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265715712 dispatch_meta_bytes=344440 combine_payload_bytes=265715712 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266060152 ingress_bytes=369524736 link_bytes=369524736 token_bytes=531775864 weight_movement_bytes=103809024 total_nvlink_bytes=635584888 weight_to_token_ratio=0.195212 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390491832e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74184 payload_rows_remote=65016 dup_rows=24120 dup_rows_remote=21102 dedup_ratio=0.245361 router_tokens_per_expert=[808, 778, 822, 780, 772, 761, 766, 770, 796, 764, 764, 773, 739, 765, 745, 747, 802, 780, 747, 788, 750, 750, 777, 733, 744, 741, 793, 789, 726, 775, 740, 722, 770, 747, 761, 770, 803, 772, 795, 780, 761, 780, 768, 748, 794, 767, 794, 792, 760, 764, 748, 805, 778, 796, 793, 739, 795, 733, 746, 763, 726, 732, 795, 770] payload_rows_per_dst=[9462, 9168, 9206, 9146, 9356, 9346, 9326, 9174] dup_rows_per_dst=[3052, 3018, 3048, 2914, 3040, 3062, 3040, 2946] combine_rows_per_src=[9462, 9168, 9206, 9146, 9356, 9346, 9326, 9174] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266321920 dispatch_meta_bytes=345192 combine_payload_bytes=266321920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266667112 ingress_bytes=300924928 link_bytes=300924928 token_bytes=532989032 weight_movement_bytes=34603008 total_nvlink_bytes=567592040 weight_to_token_ratio=0.064923 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132356278e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=65038 dup_rows=23950 dup_rows_remote=20966 dedup_ratio=0.243632 router_tokens_per_expert=[746, 772, 771, 752, 738, 771, 742, 782, 781, 760, 771, 735, 790, 741, 782, 772, 760, 771, 725, 784, 800, 798, 745, 753, 757, 758, 758, 751, 743, 813, 773, 805, 736, 779, 741, 775, 774, 757, 808, 761, 791, 764, 750, 787, 725, 783, 718, 807, 735, 778, 768, 810, 798, 721, 795, 745, 748, 827, 797, 772, 735, 790, 804, 773] payload_rows_per_dst=[9204, 9276, 9304, 9358, 9266, 9234, 9316, 9396] dup_rows_per_dst=[2944, 2988, 2968, 2958, 2996, 3016, 2984, 3096] combine_rows_per_src=[9204, 9276, 9304, 9358, 9266, 9234, 9316, 9396] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266518528 dispatch_meta_bytes=344120 combine_payload_bytes=266518528 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266862648 ingress_bytes=370327552 link_bytes=370327552 token_bytes=533381176 weight_movement_bytes=103809024 total_nvlink_bytes=637190200 weight_to_token_ratio=0.194624 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393512764e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=344472 combine_payload_bytes=266305536 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266650008 ingress_bytes=335511552 link_bytes=335511552 token_bytes=532955544 weight_movement_bytes=69206016 total_nvlink_bytes=602161560 weight_to_token_ratio=0.129853 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262502959e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=344016 combine_payload_bytes=266395648 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266739664 ingress_bytes=335601664 link_bytes=335601664 token_bytes=533135312 weight_movement_bytes=69206016 total_nvlink_bytes=602341328 weight_to_token_ratio=0.129809 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262842043e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74574 payload_rows_remote=65134 dup_rows=23730 dup_rows_remote=20696 dedup_ratio=0.241394 router_tokens_per_expert=[785, 796, 782, 755, 763, 795, 772, 742, 711, 731, 761, 736, 790, 750, 767, 767, 774, 795, 814, 787, 755, 770, 799, 792, 767, 721, 763, 754, 770, 734, 806, 810, 753, 757, 778, 804, 756, 769, 732, 773, 767, 794, 757, 773, 746, 746, 763, 792, 728, 802, 773, 768, 752, 783, 765, 744, 761, 754, 769, 769, 753, 751, 794, 812] payload_rows_per_dst=[9198, 9152, 9118, 9592, 9226, 9342, 9506, 9440] dup_rows_per_dst=[2946, 2836, 2654, 3142, 3020, 3022, 3076, 3034] combine_rows_per_src=[9198, 9152, 9118, 9592, 9226, 9342, 9506, 9440] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=65040 dup_rows=24012 dup_rows_remote=21018 dedup_ratio=0.244263 router_tokens_per_expert=[752, 777, 817, 735, 760, 791, 739, 788, 760, 733, 795, 776, 788, 803, 778, 756, 774, 766, 750, 817, 768, 773, 779, 769, 761, 782, 720, 768, 782, 737, 783, 746, 783, 783, 751, 727, 734, 801, 780, 773, 781, 789, 764, 726, 749, 757, 787, 770, 744, 736, 768, 739, 785, 753, 775, 738, 758, 826, 764, 812, 810, 761, 740, 765] payload_rows_per_dst=[9352, 9310, 9442, 9154, 9260, 9252, 9128, 9394] dup_rows_per_dst=[2966, 3068, 2950, 3004, 3004, 2994, 2948, 3078] combine_rows_per_src=[9352, 9310, 9442, 9154, 9260, 9252, 9128, 9394] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=65056 dup_rows=23962 dup_rows_remote=20996 dedup_ratio=0.243754 router_tokens_per_expert=[747, 754, 759, 756, 706, 790, 817, 729, 747, 735, 756, 765, 720, 766, 738, 755, 813, 807, 763, 727, 794, 759, 791, 787, 776, 834, 763, 760, 800, 800, 760, 740, 726, 812, 755, 767, 708, 790, 773, 775, 770, 790, 784, 768, 785, 786, 725, 788, 794, 775, 765, 722, 798, 820, 751, 791, 766, 749, 757, 785, 774, 757, 744, 788] payload_rows_per_dst=[9286, 9026, 9654, 9232, 9286, 9402, 9382, 9074] dup_rows_per_dst=[2966, 2812, 3294, 2942, 2996, 2990, 3040, 2922] combine_rows_per_src=[9286, 9026, 9654, 9232, 9286, 9402, 9382, 9074] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74184 payload_rows_remote=65016 dup_rows=24120 dup_rows_remote=21102 dedup_ratio=0.245361 router_tokens_per_expert=[808, 778, 822, 780, 772, 761, 766, 770, 796, 764, 764, 773, 739, 765, 745, 747, 802, 780, 747, 788, 750, 750, 777, 733, 744, 741, 793, 789, 726, 775, 740, 722, 770, 747, 761, 770, 803, 772, 795, 780, 761, 780, 768, 748, 794, 767, 794, 792, 760, 764, 748, 805, 778, 796, 793, 739, 795, 733, 746, 763, 726, 732, 795, 770] payload_rows_per_dst=[9462, 9168, 9206, 9146, 9356, 9346, 9326, 9174] dup_rows_per_dst=[3052, 3018, 3048, 2914, 3040, 3062, 3040, 2946] combine_rows_per_src=[9462, 9168, 9206, 9146, 9356, 9346, 9326, 9174] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=64888 dup_rows=24040 dup_rows_remote=20990 dedup_ratio=0.244548 router_tokens_per_expert=[792, 736, 790, 777, 797, 779, 785, 757, 729, 749, 799, 749, 759, 764, 802, 807, 791, 787, 717, 742, 802, 721, 767, 795, 704, 748, 771, 738, 765, 756, 765, 769, 790, 805, 781, 802, 809, 713, 735, 818, 773, 762, 760, 767, 779, 790, 756, 710, 758, 801, 812, 728, 790, 775, 729, 781, 744, 739, 735, 809, 782, 746, 765, 799] payload_rows_per_dst=[9376, 9292, 9266, 9102, 9376, 9270, 9294, 9288] dup_rows_per_dst=[3050, 3024, 2978, 2930, 3130, 2924, 3054, 2950] combine_rows_per_src=[9376, 9292, 9266, 9102, 9376, 9270, 9294, 9288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74004 payload_rows_remote=64672 dup_rows=24300 dup_rows_remote=21186 dedup_ratio=0.247192 router_tokens_per_expert=[746, 804, 750, 715, 757, 759, 767, 801, 809, 750, 768, 779, 819, 760, 773, 787, 765, 712, 805, 761, 775, 759, 773, 748, 749, 773, 762, 806, 744, 756, 761, 727, 759, 741, 791, 758, 836, 752, 786, 800, 740, 786, 781, 800, 804, 766, 741, 788, 761, 785, 789, 767, 755, 720, 744, 752, 792, 739, 745, 762, 775, 762, 786, 769] payload_rows_per_dst=[9200, 9260, 9196, 9172, 9332, 9388, 9290, 9166] dup_rows_per_dst=[2998, 3230, 3000, 2984, 3114, 3024, 2856, 3094] combine_rows_per_src=[9200, 9260, 9196, 9172, 9332, 9388, 9290, 9166] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=65038 dup_rows=23950 dup_rows_remote=20966 dedup_ratio=0.243632 router_tokens_per_expert=[746, 772, 771, 752, 738, 771, 742, 782, 781, 760, 771, 735, 790, 741, 782, 772, 760, 771, 725, 784, 800, 798, 745, 753, 757, 758, 758, 751, 743, 813, 773, 805, 736, 779, 741, 775, 774, 757, 808, 761, 791, 764, 750, 787, 725, 783, 718, 807, 735, 778, 768, 810, 798, 721, 795, 745, 748, 827, 797, 772, 735, 790, 804, 773] payload_rows_per_dst=[9204, 9276, 9304, 9358, 9266, 9234, 9316, 9396] dup_rows_per_dst=[2944, 2988, 2968, 2958, 2996, 3016, 2984, 3096] combine_rows_per_src=[9204, 9276, 9304, 9358, 9266, 9234, 9316, 9396] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74574 payload_rows_remote=65134 dup_rows=23730 dup_rows_remote=20696 dedup_ratio=0.241394 router_tokens_per_expert=[785, 796, 782, 755, 763, 795, 772, 742, 711, 731, 761, 736, 790, 750, 767, 767, 774, 795, 814, 787, 755, 770, 799, 792, 767, 721, 763, 754, 770, 734, 806, 810, 753, 757, 778, 804, 756, 769, 732, 773, 767, 794, 757, 773, 746, 746, 763, 792, 728, 802, 773, 768, 752, 783, 765, 744, 761, 754, 769, 769, 753, 751, 794, 812] payload_rows_per_dst=[9198, 9152, 9118, 9592, 9226, 9342, 9506, 9440] dup_rows_per_dst=[2946, 2836, 2654, 3142, 3020, 3022, 3076, 3034] combine_rows_per_src=[9198, 9152, 9118, 9592, 9226, 9342, 9506, 9440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74116 payload_rows_remote=64702 dup_rows=24188 dup_rows_remote=21198 dedup_ratio=0.246053 router_tokens_per_expert=[757, 752, 736, 791, 797, 771, 776, 733, 712, 760, 778, 787, 772, 766, 748, 818, 784, 764, 769, 804, 764, 788, 724, 805, 738, 787, 739, 820, 809, 730, 805, 740, 755, 791, 749, 754, 715, 792, 735, 805, 757, 792, 815, 739, 749, 727, 760, 774, 780, 766, 792, 777, 760, 775, 783, 769, 731, 747, 783, 750, 759, 806, 769, 772] payload_rows_per_dst=[9234, 9252, 9414, 9344, 9224, 9136, 9330, 9182] dup_rows_per_dst=[2992, 3030, 2990, 2992, 2968, 3090, 3074, 3052] combine_rows_per_src=[9234, 9252, 9414, 9344, 9224, 9136, 9330, 9182] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74226 payload_rows_remote=64932 dup_rows=24078 dup_rows_remote=20960 dedup_ratio=0.244934 router_tokens_per_expert=[758, 779, 788, 809, 732, 787, 817, 779, 791, 728, 756, 777, 784, 730, 762, 805, 744, 817, 799, 783, 806, 767, 758, 765, 786, 794, 831, 759, 777, 782, 751, 726, 793, 744, 747, 725, 739, 769, 758, 743, 799, 779, 735, 774, 746, 740, 770, 755, 766, 719, 768, 735, 757, 779, 712, 788, 758, 754, 732, 756, 805, 785, 774, 821] payload_rows_per_dst=[9378, 9284, 9390, 9294, 9176, 9150, 9244, 9310] dup_rows_per_dst=[3120, 2982, 3088, 3118, 2860, 3046, 2804, 3060] combine_rows_per_src=[9378, 9284, 9390, 9294, 9176, 9150, 9244, 9310] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266403840 dispatch_meta_bytes=344232 combine_payload_bytes=266403840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266748072 ingress_bytes=301006848 link_bytes=301006848 token_bytes=533151912 weight_movement_bytes=34603008 total_nvlink_bytes=567754920 weight_to_token_ratio=0.064903 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132664536e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=343320 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267132184 ingress_bytes=301391872 link_bytes=301391872 token_bytes=533921048 weight_movement_bytes=34603008 total_nvlink_bytes=568524056 weight_to_token_ratio=0.064809 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134113350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266403840 dispatch_meta_bytes=344232 combine_payload_bytes=266403840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266748072 ingress_bytes=301006848 link_bytes=301006848 token_bytes=533151912 weight_movement_bytes=34603008 total_nvlink_bytes=567754920 weight_to_token_ratio=0.064903 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132664536e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266469376 dispatch_meta_bytes=344208 combine_payload_bytes=266469376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266813584 ingress_bytes=301072384 link_bytes=301072384 token_bytes=533282960 weight_movement_bytes=34603008 total_nvlink_bytes=567885968 weight_to_token_ratio=0.064887 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132911143e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=344472 combine_payload_bytes=266305536 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266650008 ingress_bytes=335511552 link_bytes=335511552 token_bytes=532955544 weight_movement_bytes=69206016 total_nvlink_bytes=602161560 weight_to_token_ratio=0.129853 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262502959e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74188 payload_rows_remote=64990 dup_rows=24116 dup_rows_remote=21010 dedup_ratio=0.245321 router_tokens_per_expert=[794, 709, 763, 743, 794, 743, 753, 765, 793, 760, 758, 811, 790, 756, 772, 712, 771, 757, 771, 755, 777, 826, 785, 774, 745, 742, 706, 808, 781, 798, 763, 758, 702, 792, 764, 803, 766, 754, 778, 753, 732, 740, 794, 775, 741, 792, 772, 786, 770, 770, 838, 771, 744, 775, 772, 805, 762, 742, 796, 760, 753, 765, 748, 804] payload_rows_per_dst=[9186, 9198, 9298, 9218, 9296, 9264, 9400, 9328] dup_rows_per_dst=[2942, 3106, 3134, 2984, 2928, 3000, 3090, 2932] combine_rows_per_src=[9186, 9198, 9298, 9218, 9296, 9264, 9400, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265781248 dispatch_meta_bytes=343512 combine_payload_bytes=265781248 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266124760 ingress_bytes=334987264 link_bytes=334987264 token_bytes=531906008 weight_movement_bytes=69206016 total_nvlink_bytes=601112024 weight_to_token_ratio=0.130109 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260530106e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=264896512 dispatch_meta_bytes=343432 combine_payload_bytes=264896512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265239944 ingress_bytes=299499520 link_bytes=299499520 token_bytes=530136456 weight_movement_bytes=34603008 total_nvlink_bytes=564739464 weight_to_token_ratio=0.065272 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126992582e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=343320 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267132184 ingress_bytes=301391872 link_bytes=301391872 token_bytes=533921048 weight_movement_bytes=34603008 total_nvlink_bytes=568524056 weight_to_token_ratio=0.064809 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134113350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=344016 combine_payload_bytes=266395648 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266739664 ingress_bytes=335601664 link_bytes=335601664 token_bytes=533135312 weight_movement_bytes=69206016 total_nvlink_bytes=602341328 weight_to_token_ratio=0.129809 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262842043e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74348 payload_rows_remote=65016 dup_rows=23956 dup_rows_remote=20954 dedup_ratio=0.243693 router_tokens_per_expert=[765, 791, 734, 711, 755, 780, 742, 712, 749, 797, 771, 798, 759, 777, 767, 781, 766, 829, 768, 779, 743, 799, 810, 788, 805, 780, 828, 781, 760, 779, 761, 714, 768, 792, 749, 804, 711, 755, 758, 772, 787, 802, 726, 755, 731, 743, 781, 804, 768, 780, 835, 745, 770, 781, 741, 747, 783, 732, 754, 771, 744, 700, 780, 804] payload_rows_per_dst=[9178, 9312, 9380, 9346, 9286, 9254, 9332, 9260] dup_rows_per_dst=[2802, 3086, 3184, 3070, 2932, 3004, 3002, 2876] combine_rows_per_src=[9178, 9312, 9380, 9346, 9286, 9254, 9332, 9260] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265019392 dispatch_meta_bytes=343600 combine_payload_bytes=265019392 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265362992 ingress_bytes=368828416 link_bytes=368828416 token_bytes=530382384 weight_movement_bytes=103809024 total_nvlink_bytes=634191408 weight_to_token_ratio=0.195725 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.387871636e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265961472 dispatch_meta_bytes=343568 combine_payload_bytes=265961472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266305040 ingress_bytes=300564480 link_bytes=300564480 token_bytes=532266512 weight_movement_bytes=34603008 total_nvlink_bytes=566869520 weight_to_token_ratio=0.065011 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130999941e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=344000 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266543040 ingress_bytes=300802048 link_bytes=300802048 token_bytes=532742080 weight_movement_bytes=34603008 total_nvlink_bytes=567345088 weight_to_token_ratio=0.064953 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131893890e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=343880 combine_payload_bytes=266305536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266649416 ingress_bytes=300908544 link_bytes=300908544 token_bytes=532954952 weight_movement_bytes=34603008 total_nvlink_bytes=567557960 weight_to_token_ratio=0.064927 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132294626e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=65104 dup_rows=24018 dup_rows_remote=20974 dedup_ratio=0.244324 router_tokens_per_expert=[763, 776, 752, 788, 713, 789, 794, 760, 748, 737, 782, 795, 754, 755, 767, 793, 749, 802, 777, 801, 777, 785, 793, 785, 749, 768, 772, 753, 768, 760, 737, 766, 733, 774, 766, 799, 741, 740, 764, 789, 768, 785, 785, 795, 698, 765, 789, 779, 786, 774, 802, 771, 752, 760, 755, 761, 775, 752, 775, 758, 761, 758, 747, 787] payload_rows_per_dst=[9282, 9186, 9352, 9156, 9220, 9578, 9330, 9182] dup_rows_per_dst=[2988, 3076, 2874, 2990, 2992, 3062, 2992, 3044] combine_rows_per_src=[9282, 9186, 9352, 9156, 9220, 9578, 9330, 9182] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=65014 dup_rows=24096 dup_rows_remote=21134 dedup_ratio=0.245117 router_tokens_per_expert=[762, 777, 687, 758, 778, 757, 782, 781, 784, 772, 765, 745, 760, 776, 779, 790, 786, 789, 821, 774, 748, 760, 790, 781, 768, 778, 771, 765, 756, 756, 783, 767, 761, 805, 748, 790, 766, 740, 756, 771, 776, 743, 775, 777, 729, 768, 786, 724, 757, 758, 732, 730, 770, 767, 752, 763, 825, 796, 766, 771, 776, 777, 744, 807] payload_rows_per_dst=[9192, 9310, 9382, 9342, 9262, 9194, 9142, 9384] dup_rows_per_dst=[2972, 3032, 3116, 2946, 3012, 2962, 2916, 3140] combine_rows_per_src=[9192, 9310, 9382, 9342, 9262, 9194, 9142, 9384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=64888 dup_rows=24040 dup_rows_remote=20990 dedup_ratio=0.244548 router_tokens_per_expert=[792, 736, 790, 777, 797, 779, 785, 757, 729, 749, 799, 749, 759, 764, 802, 807, 791, 787, 717, 742, 802, 721, 767, 795, 704, 748, 771, 738, 765, 756, 765, 769, 790, 805, 781, 802, 809, 713, 735, 818, 773, 762, 760, 767, 779, 790, 756, 710, 758, 801, 812, 728, 790, 775, 729, 781, 744, 739, 735, 809, 782, 746, 765, 799] payload_rows_per_dst=[9376, 9292, 9266, 9102, 9376, 9270, 9294, 9288] dup_rows_per_dst=[3050, 3024, 2978, 2930, 3130, 2924, 3054, 2950] combine_rows_per_src=[9376, 9292, 9266, 9102, 9376, 9270, 9294, 9288] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74188 payload_rows_remote=64990 dup_rows=24116 dup_rows_remote=21010 dedup_ratio=0.245321 router_tokens_per_expert=[794, 709, 763, 743, 794, 743, 753, 765, 793, 760, 758, 811, 790, 756, 772, 712, 771, 757, 771, 755, 777, 826, 785, 774, 745, 742, 706, 808, 781, 798, 763, 758, 702, 792, 764, 803, 766, 754, 778, 753, 732, 740, 794, 775, 741, 792, 772, 786, 770, 770, 838, 771, 744, 775, 772, 805, 762, 742, 796, 760, 753, 765, 748, 804] payload_rows_per_dst=[9186, 9198, 9298, 9218, 9296, 9264, 9400, 9328] dup_rows_per_dst=[2942, 3106, 3134, 2984, 2928, 3000, 3090, 2932] combine_rows_per_src=[9186, 9198, 9298, 9218, 9296, 9264, 9400, 9328] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74156 payload_rows_remote=64890 dup_rows=24148 dup_rows_remote=21192 dedup_ratio=0.245646 router_tokens_per_expert=[775, 791, 722, 741, 722, 763, 775, 826, 740, 805, 773, 784, 743, 763, 779, 775, 753, 797, 758, 820, 777, 801, 732, 807, 767, 767, 775, 790, 767, 726, 749, 764, 764, 781, 745, 712, 739, 808, 775, 787, 726, 772, 771, 772, 822, 750, 777, 788, 774, 744, 760, 741, 800, 777, 718, 782, 772, 753, 754, 776, 795, 770, 753, 767] payload_rows_per_dst=[9234, 9302, 9338, 9186, 9266, 9240, 9280, 9310] dup_rows_per_dst=[2996, 3022, 3152, 3024, 2956, 3116, 2912, 2970] combine_rows_per_src=[9234, 9302, 9338, 9186, 9266, 9240, 9280, 9310] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=65104 dup_rows=24018 dup_rows_remote=20974 dedup_ratio=0.244324 router_tokens_per_expert=[763, 776, 752, 788, 713, 789, 794, 760, 748, 737, 782, 795, 754, 755, 767, 793, 749, 802, 777, 801, 777, 785, 793, 785, 749, 768, 772, 753, 768, 760, 737, 766, 733, 774, 766, 799, 741, 740, 764, 789, 768, 785, 785, 795, 698, 765, 789, 779, 786, 774, 802, 771, 752, 760, 755, 761, 775, 752, 775, 758, 761, 758, 747, 787] payload_rows_per_dst=[9282, 9186, 9352, 9156, 9220, 9578, 9330, 9182] dup_rows_per_dst=[2988, 3076, 2874, 2990, 2992, 3062, 2992, 3044] combine_rows_per_src=[9282, 9186, 9352, 9156, 9220, 9578, 9330, 9182] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266297344 dispatch_meta_bytes=344592 combine_payload_bytes=266297344 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266641936 ingress_bytes=335503360 link_bytes=335503360 token_bytes=532939280 weight_movement_bytes=69206016 total_nvlink_bytes=602145296 weight_to_token_ratio=0.129857 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262472133e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74032 payload_rows_remote=64586 dup_rows=24272 dup_rows_remote=21004 dedup_ratio=0.246908 router_tokens_per_expert=[728, 728, 799, 717, 835, 714, 758, 726, 782, 793, 815, 759, 776, 791, 779, 768, 780, 801, 798, 740, 765, 742, 723, 735, 747, 776, 759, 755, 801, 771, 764, 736, 729, 773, 758, 779, 730, 752, 737, 793, 755, 768, 784, 773, 808, 765, 756, 764, 764, 837, 749, 793, 795, 800, 729, 823, 773, 751, 789, 754, 799, 753, 748, 810] payload_rows_per_dst=[9446, 9314, 8894, 9670, 9228, 8874, 9464, 9142] dup_rows_per_dst=[3268, 3222, 2570, 3194, 3064, 2826, 3132, 2996] combine_rows_per_src=[9446, 9314, 8894, 9670, 9228, 8874, 9464, 9142] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74578 payload_rows_remote=65250 dup_rows=23726 dup_rows_remote=20834 dedup_ratio=0.241353 router_tokens_per_expert=[787, 754, 769, 760, 760, 786, 783, 780, 838, 748, 744, 747, 737, 767, 807, 791, 761, 709, 798, 760, 806, 727, 794, 755, 750, 762, 737, 748, 808, 836, 735, 791, 731, 730, 724, 798, 760, 787, 756, 816, 760, 790, 742, 790, 795, 739, 749, 754, 783, 751, 785, 726, 764, 750, 816, 732, 807, 775, 802, 771, 784, 752, 712, 786] payload_rows_per_dst=[9348, 9398, 9328, 9358, 9272, 9238, 9330, 9306] dup_rows_per_dst=[3010, 2960, 2892, 2976, 2932, 3000, 2884, 3072] combine_rows_per_src=[9348, 9398, 9328, 9358, 9272, 9238, 9330, 9306] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74348 payload_rows_remote=65016 dup_rows=23956 dup_rows_remote=20954 dedup_ratio=0.243693 router_tokens_per_expert=[765, 791, 734, 711, 755, 780, 742, 712, 749, 797, 771, 798, 759, 777, 767, 781, 766, 829, 768, 779, 743, 799, 810, 788, 805, 780, 828, 781, 760, 779, 761, 714, 768, 792, 749, 804, 711, 755, 758, 772, 787, 802, 726, 755, 731, 743, 781, 804, 768, 780, 835, 745, 770, 781, 741, 747, 783, 732, 754, 771, 744, 700, 780, 804] payload_rows_per_dst=[9178, 9312, 9380, 9346, 9286, 9254, 9332, 9260] dup_rows_per_dst=[2802, 3086, 3184, 3070, 2932, 3004, 3002, 2876] combine_rows_per_src=[9178, 9312, 9380, 9346, 9286, 9254, 9332, 9260] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74114 payload_rows_remote=65002 dup_rows=24190 dup_rows_remote=21258 dedup_ratio=0.246073 router_tokens_per_expert=[770, 719, 773, 798, 763, 766, 762, 747, 759, 793, 761, 736, 728, 720, 756, 790, 819, 789, 762, 778, 813, 812, 808, 734, 770, 679, 801, 784, 701, 745, 733, 809, 811, 812, 785, 773, 796, 739, 796, 771, 788, 814, 790, 777, 802, 775, 761, 791, 805, 731, 761, 754, 782, 767, 767, 751, 742, 755, 750, 762, 775, 737, 733, 721] payload_rows_per_dst=[9206, 9166, 9432, 9112, 9398, 9398, 9294, 9108] dup_rows_per_dst=[2990, 2920, 3198, 2932, 3168, 3198, 2942, 2842] combine_rows_per_src=[9206, 9166, 9432, 9112, 9398, 9398, 9294, 9108] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=344000 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266543040 ingress_bytes=300802048 link_bytes=300802048 token_bytes=532742080 weight_movement_bytes=34603008 total_nvlink_bytes=567345088 weight_to_token_ratio=0.064953 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131893890e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265781248 dispatch_meta_bytes=343512 combine_payload_bytes=265781248 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266124760 ingress_bytes=334987264 link_bytes=334987264 token_bytes=531906008 weight_movement_bytes=69206016 total_nvlink_bytes=601112024 weight_to_token_ratio=0.130109 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260530106e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266665984 dispatch_meta_bytes=344312 combine_payload_bytes=266665984 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267010296 ingress_bytes=370475008 link_bytes=370475008 token_bytes=533676280 weight_movement_bytes=103809024 total_nvlink_bytes=637485304 weight_to_token_ratio=0.194517 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394067629e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=64908 dup_rows=24050 dup_rows_remote=21102 dedup_ratio=0.244649 router_tokens_per_expert=[753, 771, 807, 758, 735, 767, 737, 804, 760, 792, 786, 780, 747, 770, 766, 746, 776, 758, 754, 740, 735, 763, 798, 742, 785, 743, 774, 800, 753, 755, 714, 746, 763, 766, 742, 817, 791, 756, 783, 777, 777, 756, 760, 763, 770, 745, 756, 756, 789, 767, 796, 749, 736, 775, 784, 781, 783, 797, 742, 777, 830, 803, 744, 806] payload_rows_per_dst=[9178, 9346, 9208, 9188, 9292, 9248, 9296, 9498] dup_rows_per_dst=[3086, 2948, 2924, 2952, 3098, 2918, 3058, 3066] combine_rows_per_src=[9178, 9346, 9208, 9188, 9292, 9248, 9296, 9498] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265789440 dispatch_meta_bytes=344328 combine_payload_bytes=265789440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266133768 ingress_bytes=300392448 link_bytes=300392448 token_bytes=531923208 weight_movement_bytes=34603008 total_nvlink_bytes=566526216 weight_to_token_ratio=0.065053 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130352598e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266297344 dispatch_meta_bytes=344592 combine_payload_bytes=266297344 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266641936 ingress_bytes=335503360 link_bytes=335503360 token_bytes=532939280 weight_movement_bytes=69206016 total_nvlink_bytes=602145296 weight_to_token_ratio=0.129857 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262472133e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266665984 dispatch_meta_bytes=344312 combine_payload_bytes=266665984 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267010296 ingress_bytes=370475008 link_bytes=370475008 token_bytes=533676280 weight_movement_bytes=103809024 total_nvlink_bytes=637485304 weight_to_token_ratio=0.194517 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394067629e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74544 payload_rows_remote=65232 dup_rows=23760 dup_rows_remote=20876 dedup_ratio=0.241699 router_tokens_per_expert=[770, 766, 757, 787, 764, 728, 764, 786, 724, 713, 756, 750, 810, 752, 772, 769, 743, 785, 796, 752, 810, 774, 783, 729, 767, 739, 789, 745, 722, 841, 763, 742, 794, 801, 782, 706, 777, 771, 751, 773, 798, 727, 780, 758, 760, 805, 757, 792, 726, 760, 777, 760, 761, 760, 791, 763, 816, 793, 780, 770, 790, 778, 757, 790] payload_rows_per_dst=[9246, 9272, 9254, 9290, 9244, 9398, 9312, 9528] dup_rows_per_dst=[2998, 2820, 3090, 2926, 3066, 2956, 2884, 3020] combine_rows_per_src=[9246, 9272, 9254, 9290, 9244, 9398, 9312, 9528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265789440 dispatch_meta_bytes=344328 combine_payload_bytes=265789440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266133768 ingress_bytes=300392448 link_bytes=300392448 token_bytes=531923208 weight_movement_bytes=34603008 total_nvlink_bytes=566526216 weight_to_token_ratio=0.065053 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130352598e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267264000 dispatch_meta_bytes=344336 combine_payload_bytes=267264000 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267608336 ingress_bytes=336470016 link_bytes=336470016 token_bytes=534872336 weight_movement_bytes=69206016 total_nvlink_bytes=604078352 weight_to_token_ratio=0.129388 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266109582e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=343880 combine_payload_bytes=266305536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266649416 ingress_bytes=300908544 link_bytes=300908544 token_bytes=532954952 weight_movement_bytes=34603008 total_nvlink_bytes=567557960 weight_to_token_ratio=0.064927 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132294626e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=264544256 dispatch_meta_bytes=342360 combine_payload_bytes=264544256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264886616 ingress_bytes=299147264 link_bytes=299147264 token_bytes=529430872 weight_movement_bytes=34603008 total_nvlink_bytes=564033880 weight_to_token_ratio=0.065359 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125667071e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=345040 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266593232 ingress_bytes=300851200 link_bytes=300851200 token_bytes=532841424 weight_movement_bytes=34603008 total_nvlink_bytes=567444432 weight_to_token_ratio=0.064941 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132078845e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265863168 dispatch_meta_bytes=344040 combine_payload_bytes=265863168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266207208 ingress_bytes=335069184 link_bytes=335069184 token_bytes=532070376 weight_movement_bytes=69206016 total_nvlink_bytes=601276392 weight_to_token_ratio=0.130069 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260838364e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267190272 dispatch_meta_bytes=344432 combine_payload_bytes=267190272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267534704 ingress_bytes=301793280 link_bytes=301793280 token_bytes=534724976 weight_movement_bytes=34603008 total_nvlink_bytes=569327984 weight_to_token_ratio=0.064712 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135623816e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=64790 dup_rows=24136 dup_rows_remote=21130 dedup_ratio=0.245524 router_tokens_per_expert=[771, 748, 765, 740, 763, 736, 761, 773, 798, 824, 770, 795, 784, 761, 725, 799, 767, 790, 804, 768, 754, 783, 780, 745, 739, 785, 767, 725, 775, 781, 767, 802, 750, 791, 738, 730, 813, 749, 783, 784, 814, 827, 803, 761, 759, 746, 758, 768, 739, 675, 765, 749, 788, 739, 737, 762, 740, 801, 813, 736, 739, 796, 809, 745] payload_rows_per_dst=[8950, 9152, 9362, 9770, 9144, 9290, 9122, 9378] dup_rows_per_dst=[2746, 2906, 3020, 3384, 2972, 3156, 2946, 3006] combine_rows_per_src=[8950, 9152, 9362, 9770, 9144, 9290, 9122, 9378] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=64876 dup_rows=24018 dup_rows_remote=20934 dedup_ratio=0.244324 router_tokens_per_expert=[784, 791, 748, 747, 769, 749, 787, 771, 754, 765, 823, 791, 713, 737, 824, 774, 704, 761, 810, 756, 757, 829, 754, 753, 727, 735, 773, 777, 747, 752, 737, 779, 779, 782, 734, 790, 734, 824, 809, 720, 765, 802, 766, 784, 793, 788, 788, 761, 784, 736, 780, 771, 753, 737, 793, 759, 781, 736, 710, 791, 732, 797, 817, 778] payload_rows_per_dst=[9342, 9366, 9250, 9126, 9294, 9410, 9142, 9356] dup_rows_per_dst=[2950, 2996, 2998, 2928, 3050, 3084, 3084, 2928] combine_rows_per_src=[9342, 9366, 9250, 9126, 9294, 9410, 9142, 9356] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=64908 dup_rows=24050 dup_rows_remote=21102 dedup_ratio=0.244649 router_tokens_per_expert=[753, 771, 807, 758, 735, 767, 737, 804, 760, 792, 786, 780, 747, 770, 766, 746, 776, 758, 754, 740, 735, 763, 798, 742, 785, 743, 774, 800, 753, 755, 714, 746, 763, 766, 742, 817, 791, 756, 783, 777, 777, 756, 760, 763, 770, 745, 756, 756, 789, 767, 796, 749, 736, 775, 784, 781, 783, 797, 742, 777, 830, 803, 744, 806] payload_rows_per_dst=[9178, 9346, 9208, 9188, 9292, 9248, 9296, 9498] dup_rows_per_dst=[3086, 2948, 2924, 2952, 3098, 2918, 3058, 3066] combine_rows_per_src=[9178, 9346, 9208, 9188, 9292, 9248, 9296, 9498] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74032 payload_rows_remote=64586 dup_rows=24272 dup_rows_remote=21004 dedup_ratio=0.246908 router_tokens_per_expert=[728, 728, 799, 717, 835, 714, 758, 726, 782, 793, 815, 759, 776, 791, 779, 768, 780, 801, 798, 740, 765, 742, 723, 735, 747, 776, 759, 755, 801, 771, 764, 736, 729, 773, 758, 779, 730, 752, 737, 793, 755, 768, 784, 773, 808, 765, 756, 764, 764, 837, 749, 793, 795, 800, 729, 823, 773, 751, 789, 754, 799, 753, 748, 810] payload_rows_per_dst=[9446, 9314, 8894, 9670, 9228, 8874, 9464, 9142] dup_rows_per_dst=[3268, 3222, 2570, 3194, 3064, 2826, 3132, 2996] combine_rows_per_src=[9446, 9314, 8894, 9670, 9228, 8874, 9464, 9142] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265732096 dispatch_meta_bytes=343240 combine_payload_bytes=265732096 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266075336 ingress_bytes=334938112 link_bytes=334938112 token_bytes=531807432 weight_movement_bytes=69206016 total_nvlink_bytes=601013448 weight_to_token_ratio=0.130134 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260345151e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74520 payload_rows_remote=65162 dup_rows=23784 dup_rows_remote=20680 dedup_ratio=0.241943 router_tokens_per_expert=[761, 782, 746, 736, 811, 813, 771, 764, 731, 758, 736, 776, 773, 718, 758, 785, 761, 827, 771, 814, 763, 714, 756, 826, 749, 755, 760, 760, 757, 753, 720, 719, 733, 777, 767, 775, 757, 794, 783, 845, 777, 738, 784, 761, 824, 752, 780, 763, 792, 801, 752, 806, 753, 761, 801, 776, 775, 747, 748, 724, 784, 749, 792, 757] payload_rows_per_dst=[9340, 9160, 9450, 9134, 9358, 9288, 9470, 9320] dup_rows_per_dst=[3028, 2910, 3014, 2812, 3104, 3070, 3014, 2832] combine_rows_per_src=[9340, 9160, 9450, 9134, 9358, 9288, 9470, 9320] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=64790 dup_rows=24136 dup_rows_remote=21130 dedup_ratio=0.245524 router_tokens_per_expert=[771, 748, 765, 740, 763, 736, 761, 773, 798, 824, 770, 795, 784, 761, 725, 799, 767, 790, 804, 768, 754, 783, 780, 745, 739, 785, 767, 725, 775, 781, 767, 802, 750, 791, 738, 730, 813, 749, 783, 784, 814, 827, 803, 761, 759, 746, 758, 768, 739, 675, 765, 749, 788, 739, 737, 762, 740, 801, 813, 736, 739, 796, 809, 745] payload_rows_per_dst=[8950, 9152, 9362, 9770, 9144, 9290, 9122, 9378] dup_rows_per_dst=[2746, 2906, 3020, 3384, 2972, 3156, 2946, 3006] combine_rows_per_src=[8950, 9152, 9362, 9770, 9144, 9290, 9122, 9378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74128 payload_rows_remote=65012 dup_rows=24176 dup_rows_remote=21192 dedup_ratio=0.245931 router_tokens_per_expert=[738, 809, 780, 794, 791, 809, 787, 794, 788, 771, 791, 716, 745, 808, 824, 720, 788, 734, 741, 766, 771, 730, 769, 751, 730, 731, 761, 806, 772, 793, 774, 737, 766, 755, 747, 806, 784, 766, 797, 771, 786, 746, 794, 675, 789, 791, 803, 751, 779, 775, 837, 749, 711, 723, 758, 783, 792, 773, 715, 809, 796, 752, 709, 745] payload_rows_per_dst=[9332, 9348, 9116, 9302, 9278, 9238, 9206, 9308] dup_rows_per_dst=[3272, 2978, 2984, 2906, 3106, 3032, 3024, 2874] combine_rows_per_src=[9332, 9348, 9116, 9302, 9278, 9238, 9206, 9308] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74544 payload_rows_remote=65232 dup_rows=23760 dup_rows_remote=20876 dedup_ratio=0.241699 router_tokens_per_expert=[770, 766, 757, 787, 764, 728, 764, 786, 724, 713, 756, 750, 810, 752, 772, 769, 743, 785, 796, 752, 810, 774, 783, 729, 767, 739, 789, 745, 722, 841, 763, 742, 794, 801, 782, 706, 777, 771, 751, 773, 798, 727, 780, 758, 760, 805, 757, 792, 726, 760, 777, 760, 761, 760, 791, 763, 816, 793, 780, 770, 790, 778, 757, 790] payload_rows_per_dst=[9246, 9272, 9254, 9290, 9244, 9398, 9312, 9528] dup_rows_per_dst=[2998, 2820, 3090, 2926, 3066, 2956, 2884, 3020] combine_rows_per_src=[9246, 9272, 9254, 9290, 9244, 9398, 9312, 9528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265379840 dispatch_meta_bytes=343680 combine_payload_bytes=265379840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265723520 ingress_bytes=299982848 link_bytes=299982848 token_bytes=531103360 weight_movement_bytes=34603008 total_nvlink_bytes=565706368 weight_to_token_ratio=0.065153 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128811306e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=65010 dup_rows=24040 dup_rows_remote=21066 dedup_ratio=0.244548 router_tokens_per_expert=[777, 754, 768, 775, 773, 717, 780, 764, 724, 797, 806, 708, 774, 791, 780, 834, 739, 786, 719, 768, 809, 763, 754, 781, 725, 792, 760, 745, 738, 779, 727, 757, 726, 764, 757, 748, 764, 786, 745, 792, 775, 786, 826, 780, 773, 729, 737, 723, 783, 799, 807, 773, 747, 822, 755, 801, 768, 813, 764, 791, 767, 750, 756, 781] payload_rows_per_dst=[9254, 9426, 9222, 9118, 9182, 9244, 9460, 9358] dup_rows_per_dst=[2974, 3002, 3068, 2928, 2920, 3012, 3114, 3022] combine_rows_per_src=[9254, 9426, 9222, 9118, 9182, 9244, 9460, 9358] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=65112 dup_rows=23906 dup_rows_remote=20754 dedup_ratio=0.243184 router_tokens_per_expert=[826, 792, 736, 787, 816, 814, 749, 792, 772, 833, 778, 758, 774, 746, 767, 729, 808, 744, 761, 756, 750, 730, 777, 797, 775, 793, 780, 778, 827, 724, 746, 796, 713, 750, 758, 783, 752, 774, 807, 772, 791, 742, 745, 742, 750, 769, 765, 772, 809, 749, 789, 782, 733, 775, 738, 754, 733, 761, 764, 748, 792, 755, 689, 785] payload_rows_per_dst=[9588, 9212, 9414, 9286, 9230, 9206, 9280, 9182] dup_rows_per_dst=[3036, 3102, 2832, 3152, 2988, 2946, 2978, 2872] combine_rows_per_src=[9588, 9212, 9414, 9286, 9230, 9206, 9280, 9182] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265863168 dispatch_meta_bytes=344040 combine_payload_bytes=265863168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266207208 ingress_bytes=335069184 link_bytes=335069184 token_bytes=532070376 weight_movement_bytes=69206016 total_nvlink_bytes=601276392 weight_to_token_ratio=0.130069 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260838364e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74462 payload_rows_remote=65150 dup_rows=23842 dup_rows_remote=20886 dedup_ratio=0.242533 router_tokens_per_expert=[757, 795, 802, 717, 759, 783, 710, 749, 727, 745, 748, 786, 737, 805, 792, 794, 800, 797, 711, 723, 765, 779, 763, 778, 779, 808, 781, 739, 758, 745, 843, 781, 811, 786, 772, 762, 777, 748, 804, 748, 771, 726, 767, 799, 769, 776, 776, 764, 765, 775, 756, 757, 738, 727, 776, 772, 760, 788, 747, 754, 763, 811, 777, 774] payload_rows_per_dst=[9194, 9312, 9300, 9390, 9402, 9322, 9196, 9346] dup_rows_per_dst=[2950, 2956, 2932, 3078, 3014, 2974, 2936, 3002] combine_rows_per_src=[9194, 9312, 9300, 9390, 9402, 9322, 9196, 9346] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=264544256 dispatch_meta_bytes=342360 combine_payload_bytes=264544256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264886616 ingress_bytes=299147264 link_bytes=299147264 token_bytes=529430872 weight_movement_bytes=34603008 total_nvlink_bytes=564033880 weight_to_token_ratio=0.065359 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125667071e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265732096 dispatch_meta_bytes=343240 combine_payload_bytes=265732096 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266075336 ingress_bytes=334938112 link_bytes=334938112 token_bytes=531807432 weight_movement_bytes=69206016 total_nvlink_bytes=601013448 weight_to_token_ratio=0.130134 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260345151e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73902 payload_rows_remote=64770 dup_rows=24402 dup_rows_remote=21424 dedup_ratio=0.248230 router_tokens_per_expert=[759, 735, 780, 785, 811, 714, 756, 774, 792, 780, 771, 811, 799, 755, 768, 750, 799, 807, 774, 815, 747, 786, 773, 720, 790, 734, 774, 818, 730, 806, 729, 778, 802, 768, 748, 753, 774, 733, 779, 787, 768, 737, 772, 787, 750, 744, 757, 766, 771, 745, 755, 755, 736, 779, 730, 784, 748, 826, 797, 777, 734, 763, 739, 768] payload_rows_per_dst=[9242, 9320, 9276, 9254, 9208, 9214, 9132, 9256] dup_rows_per_dst=[2986, 3132, 3166, 3064, 3080, 2948, 2978, 3048] combine_rows_per_src=[9242, 9320, 9276, 9254, 9208, 9214, 9132, 9256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266903552 dispatch_meta_bytes=343368 combine_payload_bytes=266903552 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=267246920 ingress_bytes=405315584 link_bytes=405315584 token_bytes=534150472 weight_movement_bytes=138412032 total_nvlink_bytes=672562504 weight_to_token_ratio=0.259126 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.525169912e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265379840 dispatch_meta_bytes=343680 combine_payload_bytes=265379840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265723520 ingress_bytes=299982848 link_bytes=299982848 token_bytes=531103360 weight_movement_bytes=34603008 total_nvlink_bytes=565706368 weight_to_token_ratio=0.065153 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128811306e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266289152 dispatch_meta_bytes=344816 combine_payload_bytes=266289152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266633968 ingress_bytes=300892160 link_bytes=300892160 token_bytes=532923120 weight_movement_bytes=34603008 total_nvlink_bytes=567526128 weight_to_token_ratio=0.064931 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132232974e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267190272 dispatch_meta_bytes=344432 combine_payload_bytes=267190272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267534704 ingress_bytes=301793280 link_bytes=301793280 token_bytes=534724976 weight_movement_bytes=34603008 total_nvlink_bytes=569327984 weight_to_token_ratio=0.064712 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135623816e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74086 payload_rows_remote=64792 dup_rows=24218 dup_rows_remote=21114 dedup_ratio=0.246358 router_tokens_per_expert=[732, 788, 754, 789, 777, 763, 757, 768, 721, 758, 764, 782, 818, 832, 776, 771, 794, 766, 762, 735, 733, 774, 751, 745, 722, 761, 775, 809, 757, 772, 761, 737, 798, 813, 733, 747, 746, 798, 807, 786, 762, 778, 757, 817, 730, 806, 804, 745, 758, 738, 792, 730, 810, 747, 831, 773, 736, 752, 743, 737, 755, 711, 779, 829] payload_rows_per_dst=[9258, 9396, 9130, 9116, 9348, 9294, 9376, 9168] dup_rows_per_dst=[2998, 3048, 2990, 3072, 3108, 3104, 2982, 2916] combine_rows_per_src=[9258, 9396, 9130, 9116, 9348, 9294, 9376, 9168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266280960 dispatch_meta_bytes=344304 combine_payload_bytes=266280960 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266625264 ingress_bytes=335486976 link_bytes=335486976 token_bytes=532906224 weight_movement_bytes=69206016 total_nvlink_bytes=602112240 weight_to_token_ratio=0.129865 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262410482e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266698752 dispatch_meta_bytes=343464 combine_payload_bytes=266698752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267042216 ingress_bytes=301301760 link_bytes=301301760 token_bytes=533740968 weight_movement_bytes=34603008 total_nvlink_bytes=568343976 weight_to_token_ratio=0.064831 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133774266e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74462 payload_rows_remote=65150 dup_rows=23842 dup_rows_remote=20886 dedup_ratio=0.242533 router_tokens_per_expert=[757, 795, 802, 717, 759, 783, 710, 749, 727, 745, 748, 786, 737, 805, 792, 794, 800, 797, 711, 723, 765, 779, 763, 778, 779, 808, 781, 739, 758, 745, 843, 781, 811, 786, 772, 762, 777, 748, 804, 748, 771, 726, 767, 799, 769, 776, 776, 764, 765, 775, 756, 757, 738, 727, 776, 772, 760, 788, 747, 754, 763, 811, 777, 774] payload_rows_per_dst=[9194, 9312, 9300, 9390, 9402, 9322, 9196, 9346] dup_rows_per_dst=[2950, 2956, 2932, 3078, 3014, 2974, 2936, 3002] combine_rows_per_src=[9194, 9312, 9300, 9390, 9402, 9322, 9196, 9346] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265388032 dispatch_meta_bytes=343624 combine_payload_bytes=265388032 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265731656 ingress_bytes=334594048 link_bytes=334594048 token_bytes=531119688 weight_movement_bytes=69206016 total_nvlink_bytes=600325704 weight_to_token_ratio=0.130302 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259050466e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=65010 dup_rows=24040 dup_rows_remote=21066 dedup_ratio=0.244548 router_tokens_per_expert=[777, 754, 768, 775, 773, 717, 780, 764, 724, 797, 806, 708, 774, 791, 780, 834, 739, 786, 719, 768, 809, 763, 754, 781, 725, 792, 760, 745, 738, 779, 727, 757, 726, 764, 757, 748, 764, 786, 745, 792, 775, 786, 826, 780, 773, 729, 737, 723, 783, 799, 807, 773, 747, 822, 755, 801, 768, 813, 764, 791, 767, 750, 756, 781] payload_rows_per_dst=[9254, 9426, 9222, 9118, 9182, 9244, 9460, 9358] dup_rows_per_dst=[2974, 3002, 3068, 2928, 2920, 3012, 3114, 3022] combine_rows_per_src=[9254, 9426, 9222, 9118, 9182, 9244, 9460, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266854400 dispatch_meta_bytes=344144 combine_payload_bytes=266854400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267198544 ingress_bytes=301457408 link_bytes=301457408 token_bytes=534052944 weight_movement_bytes=34603008 total_nvlink_bytes=568655952 weight_to_token_ratio=0.064793 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134359957e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265297920 dispatch_meta_bytes=344776 combine_payload_bytes=265297920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265642696 ingress_bytes=299900928 link_bytes=299900928 token_bytes=530940616 weight_movement_bytes=34603008 total_nvlink_bytes=565543624 weight_to_token_ratio=0.065173 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128503048e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74246 payload_rows_remote=64750 dup_rows=24058 dup_rows_remote=20918 dedup_ratio=0.244731 router_tokens_per_expert=[748, 734, 819, 781, 732, 763, 773, 771, 744, 817, 735, 760, 768, 774, 769, 772, 750, 784, 737, 721, 781, 760, 771, 740, 795, 771, 765, 772, 748, 748, 775, 754, 771, 761, 762, 794, 746, 814, 770, 763, 785, 782, 763, 729, 774, 752, 758, 826, 801, 788, 769, 743, 743, 780, 759, 758, 824, 785, 748, 831, 781, 732, 761, 767] payload_rows_per_dst=[9294, 9278, 9196, 9336, 9132, 9228, 9286, 9496] dup_rows_per_dst=[2948, 3000, 2892, 2920, 3052, 3110, 2996, 3140] combine_rows_per_src=[9294, 9278, 9196, 9336, 9132, 9228, 9286, 9496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74086 payload_rows_remote=64792 dup_rows=24218 dup_rows_remote=21114 dedup_ratio=0.246358 router_tokens_per_expert=[732, 788, 754, 789, 777, 763, 757, 768, 721, 758, 764, 782, 818, 832, 776, 771, 794, 766, 762, 735, 733, 774, 751, 745, 722, 761, 775, 809, 757, 772, 761, 737, 798, 813, 733, 747, 746, 798, 807, 786, 762, 778, 757, 817, 730, 806, 804, 745, 758, 738, 792, 730, 810, 747, 831, 773, 736, 752, 743, 737, 755, 711, 779, 829] payload_rows_per_dst=[9258, 9396, 9130, 9116, 9348, 9294, 9376, 9168] dup_rows_per_dst=[2998, 3048, 2990, 3072, 3108, 3104, 2982, 2916] combine_rows_per_src=[9258, 9396, 9130, 9116, 9348, 9294, 9376, 9168] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74510 payload_rows_remote=65220 dup_rows=23794 dup_rows_remote=20860 dedup_ratio=0.242045 router_tokens_per_expert=[742, 769, 783, 795, 817, 795, 777, 793, 746, 765, 757, 744, 782, 790, 791, 754, 790, 755, 806, 740, 767, 762, 720, 727, 775, 744, 787, 729, 767, 781, 710, 793, 745, 777, 776, 762, 820, 729, 736, 767, 761, 781, 757, 811, 771, 784, 766, 778, 769, 767, 801, 771, 769, 811, 783, 782, 763, 774, 724, 783, 762, 750, 757, 712] payload_rows_per_dst=[9418, 9326, 9184, 9290, 9290, 9422, 9414, 9166] dup_rows_per_dst=[3124, 2932, 2950, 2882, 2934, 2996, 3092, 2884] combine_rows_per_src=[9418, 9326, 9184, 9290, 9290, 9422, 9414, 9166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265216000 dispatch_meta_bytes=342672 combine_payload_bytes=265216000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265558672 ingress_bytes=299819008 link_bytes=299819008 token_bytes=530774672 weight_movement_bytes=34603008 total_nvlink_bytes=565377680 weight_to_token_ratio=0.065193 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128194790e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74328 payload_rows_remote=65078 dup_rows=23976 dup_rows_remote=20960 dedup_ratio=0.243896 router_tokens_per_expert=[734, 759, 762, 753, 762, 787, 752, 767, 759, 758, 748, 763, 816, 775, 735, 739, 787, 817, 757, 748, 770, 739, 742, 773, 800, 774, 758, 801, 808, 779, 743, 787, 749, 730, 777, 764, 753, 764, 803, 760, 807, 790, 729, 782, 740, 774, 780, 788, 772, 719, 775, 789, 771, 839, 757, 744, 781, 776, 774, 757, 739, 760, 787, 770] payload_rows_per_dst=[9232, 9250, 9250, 9384, 9286, 9350, 9288, 9288] dup_rows_per_dst=[2920, 2936, 3016, 3116, 2914, 3030, 3044, 3000] combine_rows_per_src=[9232, 9250, 9250, 9384, 9286, 9350, 9288, 9288] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74246 payload_rows_remote=64750 dup_rows=24058 dup_rows_remote=20918 dedup_ratio=0.244731 router_tokens_per_expert=[748, 734, 819, 781, 732, 763, 773, 771, 744, 817, 735, 760, 768, 774, 769, 772, 750, 784, 737, 721, 781, 760, 771, 740, 795, 771, 765, 772, 748, 748, 775, 754, 771, 761, 762, 794, 746, 814, 770, 763, 785, 782, 763, 729, 774, 752, 758, 826, 801, 788, 769, 743, 743, 780, 759, 758, 824, 785, 748, 831, 781, 732, 761, 767] payload_rows_per_dst=[9294, 9278, 9196, 9336, 9132, 9228, 9286, 9496] dup_rows_per_dst=[2948, 3000, 2892, 2920, 3052, 3110, 2996, 3140] combine_rows_per_src=[9294, 9278, 9196, 9336, 9132, 9228, 9286, 9496] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266854400 dispatch_meta_bytes=344144 combine_payload_bytes=266854400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267198544 ingress_bytes=301457408 link_bytes=301457408 token_bytes=534052944 weight_movement_bytes=34603008 total_nvlink_bytes=568655952 weight_to_token_ratio=0.064793 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134359957e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73902 payload_rows_remote=64770 dup_rows=24402 dup_rows_remote=21424 dedup_ratio=0.248230 router_tokens_per_expert=[759, 735, 780, 785, 811, 714, 756, 774, 792, 780, 771, 811, 799, 755, 768, 750, 799, 807, 774, 815, 747, 786, 773, 720, 790, 734, 774, 818, 730, 806, 729, 778, 802, 768, 748, 753, 774, 733, 779, 787, 768, 737, 772, 787, 750, 744, 757, 766, 771, 745, 755, 755, 736, 779, 730, 784, 748, 826, 797, 777, 734, 763, 739, 768] payload_rows_per_dst=[9242, 9320, 9276, 9254, 9208, 9214, 9132, 9256] dup_rows_per_dst=[2986, 3132, 3166, 3064, 3080, 2948, 2978, 3048] combine_rows_per_src=[9242, 9320, 9276, 9254, 9208, 9214, 9132, 9256] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=64908 dup_rows=24048 dup_rows_remote=21004 dedup_ratio=0.244629 router_tokens_per_expert=[766, 755, 768, 776, 748, 815, 788, 814, 764, 802, 797, 773, 749, 773, 799, 739, 729, 783, 744, 782, 768, 784, 714, 734, 744, 816, 775, 774, 786, 716, 746, 767, 829, 809, 701, 760, 776, 781, 784, 793, 779, 749, 757, 812, 786, 781, 759, 753, 718, 715, 708, 732, 773, 765, 788, 747, 788, 774, 784, 777, 811, 796, 729, 750] payload_rows_per_dst=[9360, 9348, 9214, 9312, 9338, 9326, 9066, 9292] dup_rows_per_dst=[3100, 3044, 2862, 2936, 3128, 3026, 2826, 3126] combine_rows_per_src=[9360, 9348, 9214, 9312, 9338, 9326, 9066, 9292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74194 payload_rows_remote=64810 dup_rows=24110 dup_rows_remote=20948 dedup_ratio=0.245260 router_tokens_per_expert=[779, 787, 739, 780, 743, 748, 771, 710, 745, 722, 769, 804, 740, 772, 809, 780, 787, 758, 789, 783, 753, 810, 800, 802, 765, 758, 797, 751, 763, 760, 760, 760, 803, 788, 751, 768, 764, 775, 745, 766, 775, 789, 775, 793, 753, 742, 765, 713, 827, 721, 775, 722, 723, 786, 755, 766, 799, 787, 795, 788, 748, 748, 756, 797] payload_rows_per_dst=[9384, 9238, 9718, 9074, 9390, 8958, 9102, 9330] dup_rows_per_dst=[3162, 3102, 3310, 2770, 2942, 2826, 2892, 3106] combine_rows_per_src=[9384, 9238, 9718, 9074, 9390, 8958, 9102, 9330] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74402 payload_rows_remote=64928 dup_rows=23902 dup_rows_remote=20828 dedup_ratio=0.243144 router_tokens_per_expert=[756, 727, 742, 732, 790, 716, 773, 792, 732, 749, 768, 752, 762, 738, 769, 777, 773, 746, 753, 764, 785, 779, 777, 753, 761, 771, 806, 776, 809, 821, 732, 798, 787, 752, 769, 770, 800, 762, 767, 805, 714, 726, 795, 759, 781, 806, 780, 778, 759, 797, 758, 784, 783, 780, 762, 791, 781, 742, 747, 770, 801, 786, 692, 789] payload_rows_per_dst=[9236, 9240, 9272, 9474, 9324, 9232, 9360, 9264] dup_rows_per_dst=[2820, 2854, 2988, 3074, 3100, 3046, 3068, 2952] combine_rows_per_src=[9236, 9240, 9272, 9474, 9324, 9232, 9360, 9264] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266280960 dispatch_meta_bytes=344304 combine_payload_bytes=266280960 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266625264 ingress_bytes=335486976 link_bytes=335486976 token_bytes=532906224 weight_movement_bytes=69206016 total_nvlink_bytes=602112240 weight_to_token_ratio=0.129865 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262410482e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265388032 dispatch_meta_bytes=343624 combine_payload_bytes=265388032 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265731656 ingress_bytes=334594048 link_bytes=334594048 token_bytes=531119688 weight_movement_bytes=69206016 total_nvlink_bytes=600325704 weight_to_token_ratio=0.130302 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259050466e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=267141120 dispatch_meta_bytes=344320 combine_payload_bytes=267141120 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267485440 ingress_bytes=336347136 link_bytes=336347136 token_bytes=534626560 weight_movement_bytes=69206016 total_nvlink_bytes=603832576 weight_to_token_ratio=0.129447 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265647195e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74508 payload_rows_remote=65182 dup_rows=23796 dup_rows_remote=20850 dedup_ratio=0.242065 router_tokens_per_expert=[806, 796, 815, 758, 773, 779, 716, 740, 780, 756, 809, 783, 743, 799, 726, 822, 778, 802, 759, 779, 778, 672, 776, 744, 808, 753, 764, 778, 775, 764, 758, 764, 768, 737, 768, 752, 719, 764, 764, 762, 787, 787, 725, 756, 794, 786, 768, 803, 779, 749, 762, 729, 743, 794, 814, 766, 793, 788, 732, 787, 767, 771, 730, 755] payload_rows_per_dst=[9390, 9342, 9194, 9364, 9230, 9390, 9326, 9272] dup_rows_per_dst=[2976, 3094, 2982, 2964, 2838, 3022, 2946, 2974] combine_rows_per_src=[9390, 9342, 9194, 9364, 9230, 9390, 9326, 9272] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266559488 dispatch_meta_bytes=344152 combine_payload_bytes=266559488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266903640 ingress_bytes=301162496 link_bytes=301162496 token_bytes=533463128 weight_movement_bytes=34603008 total_nvlink_bytes=568066136 weight_to_token_ratio=0.064865 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133250227e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265216000 dispatch_meta_bytes=342672 combine_payload_bytes=265216000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265558672 ingress_bytes=299819008 link_bytes=299819008 token_bytes=530774672 weight_movement_bytes=34603008 total_nvlink_bytes=565377680 weight_to_token_ratio=0.065193 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128194790e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74290 payload_rows_remote=65040 dup_rows=24014 dup_rows_remote=21078 dedup_ratio=0.244283 router_tokens_per_expert=[811, 768, 763, 812, 830, 773, 771, 751, 752, 740, 742, 791, 757, 772, 711, 792, 765, 780, 768, 789, 720, 722, 795, 788, 790, 763, 747, 807, 783, 770, 849, 798, 704, 825, 760, 795, 776, 773, 758, 792, 785, 734, 725, 746, 825, 765, 746, 767, 803, 716, 720, 761, 738, 790, 731, 748, 798, 774, 750, 747, 736, 766, 739, 789] payload_rows_per_dst=[9458, 9184, 9262, 9418, 9376, 9250, 9074, 9268] dup_rows_per_dst=[3100, 2930, 2992, 3196, 2990, 2936, 2940, 2930] combine_rows_per_src=[9458, 9184, 9262, 9418, 9376, 9250, 9074, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265297920 dispatch_meta_bytes=344776 combine_payload_bytes=265297920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265642696 ingress_bytes=299900928 link_bytes=299900928 token_bytes=530940616 weight_movement_bytes=34603008 total_nvlink_bytes=565543624 weight_to_token_ratio=0.065173 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128503048e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265863168 dispatch_meta_bytes=343648 combine_payload_bytes=265863168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266206816 ingress_bytes=300466176 link_bytes=300466176 token_bytes=532069984 weight_movement_bytes=34603008 total_nvlink_bytes=566672992 weight_to_token_ratio=0.065035 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130630031e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265461760 dispatch_meta_bytes=343032 combine_payload_bytes=265461760 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265804792 ingress_bytes=300064768 link_bytes=300064768 token_bytes=531266552 weight_movement_bytes=34603008 total_nvlink_bytes=565869560 weight_to_token_ratio=0.065133 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129119565e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265945088 dispatch_meta_bytes=343024 combine_payload_bytes=265945088 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266288112 ingress_bytes=335151104 link_bytes=335151104 token_bytes=532233200 weight_movement_bytes=69206016 total_nvlink_bytes=601439216 weight_to_token_ratio=0.130029 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261146622e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266403840 dispatch_meta_bytes=344472 combine_payload_bytes=266403840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266748312 ingress_bytes=301006848 link_bytes=301006848 token_bytes=533152152 weight_movement_bytes=34603008 total_nvlink_bytes=567755160 weight_to_token_ratio=0.064903 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132664536e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=64908 dup_rows=24048 dup_rows_remote=21004 dedup_ratio=0.244629 router_tokens_per_expert=[766, 755, 768, 776, 748, 815, 788, 814, 764, 802, 797, 773, 749, 773, 799, 739, 729, 783, 744, 782, 768, 784, 714, 734, 744, 816, 775, 774, 786, 716, 746, 767, 829, 809, 701, 760, 776, 781, 784, 793, 779, 749, 757, 812, 786, 781, 759, 753, 718, 715, 708, 732, 773, 765, 788, 747, 788, 774, 784, 777, 811, 796, 729, 750] payload_rows_per_dst=[9360, 9348, 9214, 9312, 9338, 9326, 9066, 9292] dup_rows_per_dst=[3100, 3044, 2862, 2936, 3128, 3026, 2826, 3126] combine_rows_per_src=[9360, 9348, 9214, 9312, 9338, 9326, 9066, 9292] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74298 payload_rows_remote=65032 dup_rows=24006 dup_rows_remote=20846 dedup_ratio=0.244202 router_tokens_per_expert=[773, 812, 740, 771, 736, 737, 815, 787, 795, 705, 805, 729, 750, 774, 758, 766, 776, 767, 739, 779, 744, 751, 746, 805, 753, 752, 825, 788, 737, 755, 749, 776, 722, 778, 738, 759, 770, 787, 804, 761, 761, 769, 747, 745, 814, 808, 801, 800, 733, 767, 755, 786, 788, 760, 777, 736, 751, 771, 757, 771, 778, 738, 829, 796] payload_rows_per_dst=[9040, 9274, 9242, 8860, 9324, 9576, 9716, 9266] dup_rows_per_dst=[2686, 2890, 2972, 2690, 2940, 3230, 3438, 3160] combine_rows_per_src=[9040, 9274, 9242, 8860, 9324, 9576, 9716, 9266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266985472 dispatch_meta_bytes=344128 combine_payload_bytes=266985472 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267329600 ingress_bytes=405397504 link_bytes=405397504 token_bytes=534315072 weight_movement_bytes=138412032 total_nvlink_bytes=672727104 weight_to_token_ratio=0.259046 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.525478170e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74290 payload_rows_remote=65040 dup_rows=24014 dup_rows_remote=21078 dedup_ratio=0.244283 router_tokens_per_expert=[811, 768, 763, 812, 830, 773, 771, 751, 752, 740, 742, 791, 757, 772, 711, 792, 765, 780, 768, 789, 720, 722, 795, 788, 790, 763, 747, 807, 783, 770, 849, 798, 704, 825, 760, 795, 776, 773, 758, 792, 785, 734, 725, 746, 825, 765, 746, 767, 803, 716, 720, 761, 738, 790, 731, 748, 798, 774, 750, 747, 736, 766, 739, 789] payload_rows_per_dst=[9458, 9184, 9262, 9418, 9376, 9250, 9074, 9268] dup_rows_per_dst=[3100, 2930, 2992, 3196, 2990, 2936, 2940, 2930] combine_rows_per_src=[9458, 9184, 9262, 9418, 9376, 9250, 9074, 9268] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74138 payload_rows_remote=64974 dup_rows=24166 dup_rows_remote=21244 dedup_ratio=0.245829 router_tokens_per_expert=[750, 760, 790, 783, 779, 746, 772, 798, 771, 786, 745, 767, 784, 788, 787, 759, 717, 766, 760, 777, 750, 778, 755, 771, 781, 753, 771, 748, 797, 723, 780, 752, 711, 749, 744, 806, 749, 771, 749, 764, 766, 786, 780, 767, 753, 791, 748, 768, 762, 772, 782, 783, 789, 776, 790, 790, 798, 747, 800, 760, 746, 782, 780, 749] payload_rows_per_dst=[9268, 9328, 9200, 9222, 9164, 9304, 9350, 9302] dup_rows_per_dst=[3088, 3046, 2948, 2988, 2922, 3014, 3138, 3022] combine_rows_per_src=[9268, 9328, 9200, 9222, 9164, 9304, 9350, 9302] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74194 payload_rows_remote=64810 dup_rows=24110 dup_rows_remote=20948 dedup_ratio=0.245260 router_tokens_per_expert=[779, 787, 739, 780, 743, 748, 771, 710, 745, 722, 769, 804, 740, 772, 809, 780, 787, 758, 789, 783, 753, 810, 800, 802, 765, 758, 797, 751, 763, 760, 760, 760, 803, 788, 751, 768, 764, 775, 745, 766, 775, 789, 775, 793, 753, 742, 765, 713, 827, 721, 775, 722, 723, 786, 755, 766, 799, 787, 795, 788, 748, 748, 756, 797] payload_rows_per_dst=[9384, 9238, 9718, 9074, 9390, 8958, 9102, 9330] dup_rows_per_dst=[3162, 3102, 3310, 2770, 2942, 2826, 2892, 3106] combine_rows_per_src=[9384, 9238, 9718, 9074, 9390, 8958, 9102, 9330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=343512 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266714584 ingress_bytes=370180096 link_bytes=370180096 token_bytes=533085656 weight_movement_bytes=103809024 total_nvlink_bytes=636894680 weight_to_token_ratio=0.194732 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392957899e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74086 payload_rows_remote=64766 dup_rows=24218 dup_rows_remote=21112 dedup_ratio=0.246358 router_tokens_per_expert=[774, 763, 774, 762, 779, 782, 747, 799, 786, 762, 769, 818, 791, 778, 724, 770, 780, 766, 791, 762, 804, 796, 761, 753, 754, 783, 774, 767, 773, 807, 779, 722, 792, 775, 726, 756, 723, 824, 778, 777, 766, 778, 741, 792, 789, 728, 734, 746, 738, 778, 762, 821, 768, 750, 757, 769, 763, 745, 796, 748, 782, 711, 706, 783] payload_rows_per_dst=[9286, 9278, 9320, 9346, 9308, 9186, 9228, 9134] dup_rows_per_dst=[3074, 3118, 3106, 2972, 2994, 2962, 3058, 2934] combine_rows_per_src=[9286, 9278, 9320, 9346, 9308, 9186, 9228, 9134] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74298 payload_rows_remote=65032 dup_rows=24006 dup_rows_remote=20846 dedup_ratio=0.244202 router_tokens_per_expert=[773, 812, 740, 771, 736, 737, 815, 787, 795, 705, 805, 729, 750, 774, 758, 766, 776, 767, 739, 779, 744, 751, 746, 805, 753, 752, 825, 788, 737, 755, 749, 776, 722, 778, 738, 759, 770, 787, 804, 761, 761, 769, 747, 745, 814, 808, 801, 800, 733, 767, 755, 786, 788, 760, 777, 736, 751, 771, 757, 771, 778, 738, 829, 796] payload_rows_per_dst=[9040, 9274, 9242, 8860, 9324, 9576, 9716, 9266] dup_rows_per_dst=[2686, 2890, 2972, 2690, 2940, 3230, 3438, 3160] combine_rows_per_src=[9040, 9274, 9242, 8860, 9324, 9576, 9716, 9266] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265863168 dispatch_meta_bytes=343648 combine_payload_bytes=265863168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266206816 ingress_bytes=300466176 link_bytes=300466176 token_bytes=532069984 weight_movement_bytes=34603008 total_nvlink_bytes=566672992 weight_to_token_ratio=0.065035 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130630031e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74508 payload_rows_remote=65182 dup_rows=23796 dup_rows_remote=20850 dedup_ratio=0.242065 router_tokens_per_expert=[806, 796, 815, 758, 773, 779, 716, 740, 780, 756, 809, 783, 743, 799, 726, 822, 778, 802, 759, 779, 778, 672, 776, 744, 808, 753, 764, 778, 775, 764, 758, 764, 768, 737, 768, 752, 719, 764, 764, 762, 787, 787, 725, 756, 794, 786, 768, 803, 779, 749, 762, 729, 743, 794, 814, 766, 793, 788, 732, 787, 767, 771, 730, 755] payload_rows_per_dst=[9390, 9342, 9194, 9364, 9230, 9390, 9326, 9272] dup_rows_per_dst=[2976, 3094, 2982, 2964, 2838, 3022, 2946, 2974] combine_rows_per_src=[9390, 9342, 9194, 9364, 9230, 9390, 9326, 9272] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74032 payload_rows_remote=64828 dup_rows=24272 dup_rows_remote=21378 dedup_ratio=0.246908 router_tokens_per_expert=[751, 794, 792, 779, 782, 800, 751, 721, 787, 730, 746, 726, 773, 745, 768, 774, 798, 821, 763, 755, 747, 794, 745, 728, 808, 766, 814, 804, 862, 793, 792, 773, 745, 776, 747, 720, 787, 760, 778, 783, 785, 772, 748, 743, 790, 767, 731, 754, 808, 734, 775, 732, 761, 804, 773, 782, 763, 757, 774, 790, 730, 752, 723, 726] payload_rows_per_dst=[9256, 9204, 9258, 9524, 9144, 9166, 9266, 9214] dup_rows_per_dst=[3084, 2894, 3044, 3300, 3048, 3014, 3072, 2816] combine_rows_per_src=[9256, 9204, 9258, 9524, 9144, 9166, 9266, 9214] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266403840 dispatch_meta_bytes=344472 combine_payload_bytes=266403840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266748312 ingress_bytes=301006848 link_bytes=301006848 token_bytes=533152152 weight_movement_bytes=34603008 total_nvlink_bytes=567755160 weight_to_token_ratio=0.064903 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132664536e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=64958 dup_rows=24056 dup_rows_remote=20898 dedup_ratio=0.244710 router_tokens_per_expert=[772, 753, 762, 796, 761, 800, 774, 806, 764, 798, 762, 800, 750, 783, 755, 786, 762, 755, 759, 767, 750, 760, 739, 801, 810, 786, 759, 772, 811, 734, 751, 746, 744, 740, 804, 762, 723, 761, 751, 784, 746, 748, 712, 731, 795, 790, 766, 768, 791, 735, 745, 801, 762, 734, 733, 739, 780, 781, 781, 815, 762, 790, 795, 799] payload_rows_per_dst=[9290, 9406, 9364, 9356, 9150, 9278, 9140, 9264] dup_rows_per_dst=[3158, 3114, 3118, 2982, 2988, 2834, 2940, 2922] combine_rows_per_src=[9290, 9406, 9364, 9356, 9150, 9278, 9140, 9264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=65108 dup_rows=23906 dup_rows_remote=20968 dedup_ratio=0.243184 router_tokens_per_expert=[754, 752, 769, 756, 772, 732, 755, 795, 734, 716, 721, 717, 739, 753, 772, 791, 804, 759, 772, 805, 753, 814, 768, 722, 775, 742, 786, 809, 808, 819, 792, 778, 789, 736, 766, 787, 815, 805, 745, 802, 729, 787, 761, 750, 744, 784, 758, 792, 755, 792, 795, 752, 743, 726, 786, 765, 772, 752, 780, 770, 796, 777, 746, 761] payload_rows_per_dst=[9236, 9088, 9392, 9508, 9312, 9262, 9290, 9310] dup_rows_per_dst=[2934, 2798, 3002, 3110, 3178, 2948, 2938, 2998] combine_rows_per_src=[9236, 9088, 9392, 9508, 9312, 9262, 9290, 9310] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=64818 dup_rows=24060 dup_rows_remote=21078 dedup_ratio=0.244751 router_tokens_per_expert=[736, 776, 763, 799, 757, 747, 760, 775, 748, 808, 807, 764, 776, 769, 745, 812, 740, 732, 821, 741, 815, 757, 786, 763, 807, 787, 771, 802, 746, 732, 825, 734, 747, 760, 761, 725, 791, 700, 792, 820, 777, 719, 692, 783, 759, 788, 759, 809, 750, 762, 766, 784, 710, 785, 759, 765, 764, 774, 793, 746, 793, 751, 781, 786] payload_rows_per_dst=[9180, 9314, 9256, 9426, 9244, 9246, 9218, 9360] dup_rows_per_dst=[3046, 3144, 3054, 2982, 2948, 2926, 2944, 3016] combine_rows_per_src=[9180, 9314, 9256, 9426, 9244, 9246, 9218, 9360] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=344872 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266478376 ingress_bytes=300736512 link_bytes=300736512 token_bytes=532611880 weight_movement_bytes=34603008 total_nvlink_bytes=567214888 weight_to_token_ratio=0.064969 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131647283e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265461760 dispatch_meta_bytes=343032 combine_payload_bytes=265461760 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265804792 ingress_bytes=300064768 link_bytes=300064768 token_bytes=531266552 weight_movement_bytes=34603008 total_nvlink_bytes=565869560 weight_to_token_ratio=0.065133 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129119565e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265281536 dispatch_meta_bytes=343512 combine_payload_bytes=265281536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265625048 ingress_bytes=299884544 link_bytes=299884544 token_bytes=530906584 weight_movement_bytes=34603008 total_nvlink_bytes=565509592 weight_to_token_ratio=0.065177 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128441396e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=343512 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266714584 ingress_bytes=370180096 link_bytes=370180096 token_bytes=533085656 weight_movement_bytes=103809024 total_nvlink_bytes=636894680 weight_to_token_ratio=0.194732 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392957899e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=65210 dup_rows=23920 dup_rows_remote=20886 dedup_ratio=0.243327 router_tokens_per_expert=[813, 793, 777, 756, 750, 776, 765, 814, 730, 723, 764, 769, 775, 773, 728, 736, 762, 774, 770, 750, 811, 778, 779, 769, 773, 774, 770, 767, 775, 806, 764, 759, 786, 759, 785, 767, 738, 787, 756, 755, 795, 770, 736, 764, 767, 757, 797, 718, 813, 802, 840, 752, 750, 733, 731, 829, 763, 763, 742, 734, 774, 785, 713, 768] payload_rows_per_dst=[9424, 9140, 9376, 9360, 9350, 9174, 9410, 9150] dup_rows_per_dst=[3064, 2856, 3010, 3016, 2916, 3034, 3090, 2934] combine_rows_per_src=[9424, 9140, 9376, 9360, 9350, 9174, 9410, 9150] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74032 payload_rows_remote=64828 dup_rows=24272 dup_rows_remote=21378 dedup_ratio=0.246908 router_tokens_per_expert=[751, 794, 792, 779, 782, 800, 751, 721, 787, 730, 746, 726, 773, 745, 768, 774, 798, 821, 763, 755, 747, 794, 745, 728, 808, 766, 814, 804, 862, 793, 792, 773, 745, 776, 747, 720, 787, 760, 778, 783, 785, 772, 748, 743, 790, 767, 731, 754, 808, 734, 775, 732, 761, 804, 773, 782, 763, 757, 774, 790, 730, 752, 723, 726] payload_rows_per_dst=[9256, 9204, 9258, 9524, 9144, 9166, 9266, 9214] dup_rows_per_dst=[3084, 2894, 3044, 3300, 3048, 3014, 3072, 2816] combine_rows_per_src=[9256, 9204, 9258, 9524, 9144, 9166, 9266, 9214] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265535488 dispatch_meta_bytes=344824 combine_payload_bytes=265535488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265880312 ingress_bytes=300138496 link_bytes=300138496 token_bytes=531415800 weight_movement_bytes=34603008 total_nvlink_bytes=566018808 weight_to_token_ratio=0.065115 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129396997e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266985472 dispatch_meta_bytes=344128 combine_payload_bytes=266985472 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267329600 ingress_bytes=405397504 link_bytes=405397504 token_bytes=534315072 weight_movement_bytes=138412032 total_nvlink_bytes=672727104 weight_to_token_ratio=0.259046 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.525478170e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74454 payload_rows_remote=64664 dup_rows=23850 dup_rows_remote=20458 dedup_ratio=0.242615 router_tokens_per_expert=[755, 736, 770, 806, 778, 700, 747, 811, 784, 748, 822, 784, 707, 758, 753, 763, 756, 823, 780, 783, 760, 733, 738, 740, 773, 748, 799, 796, 778, 719, 799, 794, 771, 757, 732, 744, 790, 745, 746, 754, 819, 790, 821, 786, 771, 746, 765, 744, 795, 756, 730, 781, 792, 782, 792, 751, 751, 776, 754, 777, 784, 780, 761, 768] payload_rows_per_dst=[9278, 9256, 9240, 9194, 9338, 9026, 9332, 9790] dup_rows_per_dst=[2904, 2982, 2986, 2996, 2870, 2678, 3042, 3392] combine_rows_per_src=[9278, 9256, 9240, 9194, 9338, 9026, 9332, 9790] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=343424 combine_payload_bytes=266067968 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266411392 ingress_bytes=404480000 link_bytes=404480000 token_bytes=532479360 weight_movement_bytes=138412032 total_nvlink_bytes=670891392 weight_to_token_ratio=0.259939 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522025677e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266682368 dispatch_meta_bytes=344304 combine_payload_bytes=266682368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267026672 ingress_bytes=301285376 link_bytes=301285376 token_bytes=533709040 weight_movement_bytes=34603008 total_nvlink_bytes=568312048 weight_to_token_ratio=0.064835 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133712614e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265494528 dispatch_meta_bytes=343584 combine_payload_bytes=265494528 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265838112 ingress_bytes=334700544 link_bytes=334700544 token_bytes=531332640 weight_movement_bytes=69206016 total_nvlink_bytes=600538656 weight_to_token_ratio=0.130250 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259451201e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267100160 dispatch_meta_bytes=344384 combine_payload_bytes=267100160 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267444544 ingress_bytes=336306176 link_bytes=336306176 token_bytes=534544704 weight_movement_bytes=69206016 total_nvlink_bytes=603750720 weight_to_token_ratio=0.129467 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265493065e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265535488 dispatch_meta_bytes=344824 combine_payload_bytes=265535488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265880312 ingress_bytes=300138496 link_bytes=300138496 token_bytes=531415800 weight_movement_bytes=34603008 total_nvlink_bytes=566018808 weight_to_token_ratio=0.065115 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129396997e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=264863744 dispatch_meta_bytes=340488 combine_payload_bytes=264863744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265204232 ingress_bytes=299466752 link_bytes=299466752 token_bytes=530067976 weight_movement_bytes=34603008 total_nvlink_bytes=564670984 weight_to_token_ratio=0.065280 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126869279e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=65210 dup_rows=23920 dup_rows_remote=20886 dedup_ratio=0.243327 router_tokens_per_expert=[813, 793, 777, 756, 750, 776, 765, 814, 730, 723, 764, 769, 775, 773, 728, 736, 762, 774, 770, 750, 811, 778, 779, 769, 773, 774, 770, 767, 775, 806, 764, 759, 786, 759, 785, 767, 738, 787, 756, 755, 795, 770, 736, 764, 767, 757, 797, 718, 813, 802, 840, 752, 750, 733, 731, 829, 763, 763, 742, 734, 774, 785, 713, 768] payload_rows_per_dst=[9424, 9140, 9376, 9360, 9350, 9174, 9410, 9150] dup_rows_per_dst=[3064, 2856, 3010, 3016, 2916, 3034, 3090, 2934] combine_rows_per_src=[9424, 9140, 9376, 9360, 9350, 9174, 9410, 9150] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=64946 dup_rows=24118 dup_rows_remote=21190 dedup_ratio=0.245341 router_tokens_per_expert=[782, 778, 753, 814, 750, 789, 789, 765, 753, 776, 745, 762, 767, 774, 760, 801, 772, 761, 750, 776, 774, 755, 785, 724, 779, 764, 772, 742, 758, 762, 740, 739, 771, 750, 785, 793, 712, 724, 780, 769, 809, 808, 765, 785, 756, 816, 696, 722, 784, 780, 767, 806, 792, 783, 781, 790, 718, 729, 735, 787, 776, 783, 771, 818] payload_rows_per_dst=[9344, 9196, 9290, 9202, 9240, 9188, 9462, 9264] dup_rows_per_dst=[3096, 3080, 2904, 2910, 2928, 3126, 3104, 2970] combine_rows_per_src=[9344, 9196, 9290, 9202, 9240, 9188, 9462, 9264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=65104 dup_rows=23920 dup_rows_remote=20986 dedup_ratio=0.243327 router_tokens_per_expert=[748, 781, 775, 730, 787, 732, 750, 757, 759, 767, 798, 806, 751, 796, 759, 771, 728, 755, 782, 819, 766, 773, 748, 736, 764, 762, 755, 758, 835, 767, 758, 763, 794, 760, 732, 691, 829, 826, 798, 783, 812, 788, 777, 769, 766, 806, 733, 805, 748, 794, 771, 721, 799, 712, 774, 763, 768, 717, 780, 794, 746, 789, 729, 742] payload_rows_per_dst=[9132, 9340, 9280, 9258, 9356, 9454, 9246, 9318] dup_rows_per_dst=[2988, 3074, 2934, 3066, 3070, 3058, 2918, 2812] combine_rows_per_src=[9132, 9340, 9280, 9258, 9356, 9454, 9246, 9318] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=64958 dup_rows=24056 dup_rows_remote=20898 dedup_ratio=0.244710 router_tokens_per_expert=[772, 753, 762, 796, 761, 800, 774, 806, 764, 798, 762, 800, 750, 783, 755, 786, 762, 755, 759, 767, 750, 760, 739, 801, 810, 786, 759, 772, 811, 734, 751, 746, 744, 740, 804, 762, 723, 761, 751, 784, 746, 748, 712, 731, 795, 790, 766, 768, 791, 735, 745, 801, 762, 734, 733, 739, 780, 781, 781, 815, 762, 790, 795, 799] payload_rows_per_dst=[9290, 9406, 9364, 9356, 9150, 9278, 9140, 9264] dup_rows_per_dst=[3158, 3114, 3118, 2982, 2988, 2834, 2940, 2922] combine_rows_per_src=[9290, 9406, 9364, 9356, 9150, 9278, 9140, 9264] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74454 payload_rows_remote=64664 dup_rows=23850 dup_rows_remote=20458 dedup_ratio=0.242615 router_tokens_per_expert=[755, 736, 770, 806, 778, 700, 747, 811, 784, 748, 822, 784, 707, 758, 753, 763, 756, 823, 780, 783, 760, 733, 738, 740, 773, 748, 799, 796, 778, 719, 799, 794, 771, 757, 732, 744, 790, 745, 746, 754, 819, 790, 821, 786, 771, 746, 765, 744, 795, 756, 730, 781, 792, 782, 792, 751, 751, 776, 754, 777, 784, 780, 761, 768] payload_rows_per_dst=[9278, 9256, 9240, 9194, 9338, 9026, 9332, 9790] dup_rows_per_dst=[2904, 2982, 2986, 2996, 2870, 2678, 3042, 3392] combine_rows_per_src=[9278, 9256, 9240, 9194, 9338, 9026, 9332, 9790] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74202 payload_rows_remote=64934 dup_rows=24102 dup_rows_remote=21108 dedup_ratio=0.245178 router_tokens_per_expert=[716, 738, 778, 796, 771, 807, 803, 815, 775, 757, 791, 772, 760, 737, 813, 726, 749, 776, 774, 786, 764, 770, 791, 781, 766, 784, 768, 802, 761, 777, 761, 743, 759, 749, 733, 772, 730, 786, 738, 741, 755, 760, 799, 754, 732, 764, 793, 793, 830, 794, 752, 778, 753, 708, 737, 785, 764, 784, 762, 765, 802, 742, 755, 775] payload_rows_per_dst=[9346, 9268, 9312, 9312, 9096, 9346, 9264, 9258] dup_rows_per_dst=[3102, 2994, 3070, 3012, 2920, 2954, 3010, 3040] combine_rows_per_src=[9346, 9268, 9312, 9312, 9096, 9346, 9264, 9258] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=65108 dup_rows=23906 dup_rows_remote=20968 dedup_ratio=0.243184 router_tokens_per_expert=[754, 752, 769, 756, 772, 732, 755, 795, 734, 716, 721, 717, 739, 753, 772, 791, 804, 759, 772, 805, 753, 814, 768, 722, 775, 742, 786, 809, 808, 819, 792, 778, 789, 736, 766, 787, 815, 805, 745, 802, 729, 787, 761, 750, 744, 784, 758, 792, 755, 792, 795, 752, 743, 726, 786, 765, 772, 752, 780, 770, 796, 777, 746, 761] payload_rows_per_dst=[9236, 9088, 9392, 9508, 9312, 9262, 9290, 9310] dup_rows_per_dst=[2934, 2798, 3002, 3110, 3178, 2948, 2938, 2998] combine_rows_per_src=[9236, 9088, 9392, 9508, 9312, 9262, 9290, 9310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267100160 dispatch_meta_bytes=344384 combine_payload_bytes=267100160 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267444544 ingress_bytes=336306176 link_bytes=336306176 token_bytes=534544704 weight_movement_bytes=69206016 total_nvlink_bytes=603750720 weight_to_token_ratio=0.129467 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265493065e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74314 payload_rows_remote=64950 dup_rows=23990 dup_rows_remote=20946 dedup_ratio=0.244039 router_tokens_per_expert=[781, 749, 792, 775, 786, 732, 793, 796, 782, 763, 773, 834, 731, 771, 739, 748, 775, 785, 781, 782, 743, 732, 769, 789, 761, 787, 710, 800, 783, 710, 746, 761, 813, 719, 787, 763, 747, 730, 770, 807, 753, 788, 808, 763, 785, 786, 829, 776, 795, 793, 750, 790, 800, 735, 790, 768, 700, 709, 762, 720, 759, 762, 798, 738] payload_rows_per_dst=[9364, 9688, 9418, 8980, 9314, 9484, 9010, 9056] dup_rows_per_dst=[3044, 3212, 3200, 2830, 2958, 3092, 2814, 2840] combine_rows_per_src=[9364, 9688, 9418, 8980, 9314, 9484, 9010, 9056] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74410 payload_rows_remote=65082 dup_rows=23894 dup_rows_remote=21030 dedup_ratio=0.243062 router_tokens_per_expert=[759, 771, 743, 757, 749, 769, 772, 764, 766, 781, 768, 753, 768, 771, 746, 771, 800, 771, 731, 720, 767, 748, 733, 753, 770, 763, 730, 791, 786, 810, 782, 774, 771, 780, 784, 776, 806, 782, 743, 756, 817, 747, 768, 733, 755, 724, 817, 764, 762, 763, 736, 777, 767, 788, 743, 760, 764, 777, 728, 841, 789, 791, 786, 820] payload_rows_per_dst=[9196, 9240, 9122, 9444, 9362, 9296, 9328, 9422] dup_rows_per_dst=[2972, 3008, 2924, 2968, 3034, 2954, 2864, 3170] combine_rows_per_src=[9196, 9240, 9122, 9444, 9362, 9296, 9328, 9422] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266018816 dispatch_meta_bytes=344544 combine_payload_bytes=266018816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266363360 ingress_bytes=300621824 link_bytes=300621824 token_bytes=532382176 weight_movement_bytes=34603008 total_nvlink_bytes=566985184 weight_to_token_ratio=0.064997 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131215722e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266665984 dispatch_meta_bytes=344360 combine_payload_bytes=266665984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267010344 ingress_bytes=301268992 link_bytes=301268992 token_bytes=533676328 weight_movement_bytes=34603008 total_nvlink_bytes=568279336 weight_to_token_ratio=0.064839 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133650963e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74456 payload_rows_remote=65110 dup_rows=23848 dup_rows_remote=20938 dedup_ratio=0.242594 router_tokens_per_expert=[760, 714, 759, 737, 778, 770, 790, 771, 700, 769, 765, 787, 741, 776, 800, 729, 763, 777, 762, 750, 769, 772, 774, 792, 804, 756, 755, 741, 783, 754, 728, 807, 824, 768, 755, 764, 780, 770, 787, 740, 783, 765, 802, 807, 786, 760, 775, 802, 733, 781, 804, 804, 763, 789, 775, 724, 779, 770, 752, 760, 733, 798, 758, 728] payload_rows_per_dst=[9282, 9190, 9324, 9346, 9268, 9442, 9380, 9224] dup_rows_per_dst=[2876, 2944, 2994, 2910, 3108, 3118, 2966, 2932] combine_rows_per_src=[9282, 9190, 9324, 9346, 9268, 9442, 9380, 9224] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65014 dup_rows=23944 dup_rows_remote=20950 dedup_ratio=0.243571 router_tokens_per_expert=[777, 751, 767, 743, 717, 747, 727, 758, 740, 756, 743, 774, 818, 780, 743, 771, 757, 777, 798, 741, 787, 754, 740, 741, 766, 779, 766, 761, 744, 780, 736, 807, 790, 741, 763, 780, 763, 757, 756, 812, 802, 783, 748, 778, 751, 780, 788, 740, 793, 814, 775, 743, 799, 784, 798, 771, 738, 778, 828, 721, 763, 788, 780, 801] payload_rows_per_dst=[9122, 9220, 9218, 9366, 9344, 9346, 9362, 9382] dup_rows_per_dst=[2852, 3030, 2972, 2912, 2980, 2994, 3192, 3012] combine_rows_per_src=[9122, 9220, 9218, 9366, 9344, 9346, 9362, 9382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=343424 combine_payload_bytes=266067968 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266411392 ingress_bytes=404480000 link_bytes=404480000 token_bytes=532479360 weight_movement_bytes=138412032 total_nvlink_bytes=670891392 weight_to_token_ratio=0.259939 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522025677e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74202 payload_rows_remote=64934 dup_rows=24102 dup_rows_remote=21108 dedup_ratio=0.245178 router_tokens_per_expert=[716, 738, 778, 796, 771, 807, 803, 815, 775, 757, 791, 772, 760, 737, 813, 726, 749, 776, 774, 786, 764, 770, 791, 781, 766, 784, 768, 802, 761, 777, 761, 743, 759, 749, 733, 772, 730, 786, 738, 741, 755, 760, 799, 754, 732, 764, 793, 793, 830, 794, 752, 778, 753, 708, 737, 785, 764, 784, 762, 765, 802, 742, 755, 775] payload_rows_per_dst=[9346, 9268, 9312, 9312, 9096, 9346, 9264, 9258] dup_rows_per_dst=[3102, 2994, 3070, 3012, 2920, 2954, 3010, 3040] combine_rows_per_src=[9346, 9268, 9312, 9312, 9096, 9346, 9264, 9258] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=264863744 dispatch_meta_bytes=340488 combine_payload_bytes=264863744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265204232 ingress_bytes=299466752 link_bytes=299466752 token_bytes=530067976 weight_movement_bytes=34603008 total_nvlink_bytes=564670984 weight_to_token_ratio=0.065280 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126869279e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266682368 dispatch_meta_bytes=344304 combine_payload_bytes=266682368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267026672 ingress_bytes=301285376 link_bytes=301285376 token_bytes=533709040 weight_movement_bytes=34603008 total_nvlink_bytes=568312048 weight_to_token_ratio=0.064835 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133712614e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266665984 dispatch_meta_bytes=344360 combine_payload_bytes=266665984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267010344 ingress_bytes=301268992 link_bytes=301268992 token_bytes=533676328 weight_movement_bytes=34603008 total_nvlink_bytes=568279336 weight_to_token_ratio=0.064839 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133650963e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265969664 dispatch_meta_bytes=344168 combine_payload_bytes=265969664 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266313832 ingress_bytes=369778688 link_bytes=369778688 token_bytes=532283496 weight_movement_bytes=103809024 total_nvlink_bytes=636092520 weight_to_token_ratio=0.195026 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391447433e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74328 payload_rows_remote=64952 dup_rows=23976 dup_rows_remote=20970 dedup_ratio=0.243896 router_tokens_per_expert=[826, 676, 763, 799, 759, 757, 784, 754, 764, 754, 787, 776, 747, 751, 782, 768, 744, 751, 733, 793, 746, 800, 782, 731, 747, 760, 734, 737, 781, 778, 764, 767, 754, 823, 740, 740, 779, 780, 785, 812, 799, 713, 804, 759, 809, 762, 787, 799, 762, 763, 773, 757, 759, 798, 773, 737, 750, 813, 754, 795, 786, 790, 749, 753] payload_rows_per_dst=[9230, 9148, 9268, 9222, 9416, 9436, 9232, 9376] dup_rows_per_dst=[3006, 2788, 2994, 2914, 3230, 3028, 3010, 3006] combine_rows_per_src=[9230, 9148, 9268, 9222, 9416, 9436, 9232, 9376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266575872 dispatch_meta_bytes=344448 combine_payload_bytes=266575872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266920320 ingress_bytes=335781888 link_bytes=335781888 token_bytes=533496192 weight_movement_bytes=69206016 total_nvlink_bytes=602702208 weight_to_token_ratio=0.129722 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263520212e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266035200 dispatch_meta_bytes=343584 combine_payload_bytes=266035200 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266378784 ingress_bytes=335241216 link_bytes=335241216 token_bytes=532413984 weight_movement_bytes=69206016 total_nvlink_bytes=601620000 weight_to_token_ratio=0.129985 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261485707e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266690560 dispatch_meta_bytes=344192 combine_payload_bytes=266690560 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=267034752 ingress_bytes=405102592 link_bytes=405102592 token_bytes=533725312 weight_movement_bytes=138412032 total_nvlink_bytes=672137344 weight_to_token_ratio=0.259332 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524368440e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266297344 dispatch_meta_bytes=343856 combine_payload_bytes=266297344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266641200 ingress_bytes=300900352 link_bytes=300900352 token_bytes=532938544 weight_movement_bytes=34603008 total_nvlink_bytes=567541552 weight_to_token_ratio=0.064929 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132263800e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265969664 dispatch_meta_bytes=344168 combine_payload_bytes=265969664 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266313832 ingress_bytes=369778688 link_bytes=369778688 token_bytes=532283496 weight_movement_bytes=103809024 total_nvlink_bytes=636092520 weight_to_token_ratio=0.195026 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391447433e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65014 dup_rows=23944 dup_rows_remote=20950 dedup_ratio=0.243571 router_tokens_per_expert=[777, 751, 767, 743, 717, 747, 727, 758, 740, 756, 743, 774, 818, 780, 743, 771, 757, 777, 798, 741, 787, 754, 740, 741, 766, 779, 766, 761, 744, 780, 736, 807, 790, 741, 763, 780, 763, 757, 756, 812, 802, 783, 748, 778, 751, 780, 788, 740, 793, 814, 775, 743, 799, 784, 798, 771, 738, 778, 828, 721, 763, 788, 780, 801] payload_rows_per_dst=[9122, 9220, 9218, 9366, 9344, 9346, 9362, 9382] dup_rows_per_dst=[2852, 3030, 2972, 2912, 2980, 2994, 3192, 3012] combine_rows_per_src=[9122, 9220, 9218, 9366, 9344, 9346, 9362, 9382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266043392 dispatch_meta_bytes=343688 combine_payload_bytes=266043392 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266387080 ingress_bytes=369852416 link_bytes=369852416 token_bytes=532430472 weight_movement_bytes=103809024 total_nvlink_bytes=636239496 weight_to_token_ratio=0.194972 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391724866e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74266 payload_rows_remote=65014 dup_rows=24038 dup_rows_remote=20862 dedup_ratio=0.244527 router_tokens_per_expert=[746, 800, 779, 759, 745, 790, 764, 744, 785, 780, 737, 788, 742, 770, 756, 782, 774, 785, 782, 768, 773, 756, 770, 797, 788, 728, 717, 747, 707, 774, 743, 785, 724, 785, 805, 776, 771, 762, 786, 805, 729, 781, 716, 750, 791, 792, 820, 743, 732, 783, 721, 762, 822, 772, 832, 805, 766, 772, 774, 756, 770, 735, 783, 770] payload_rows_per_dst=[9332, 9286, 9356, 9160, 9252, 9336, 9382, 9162] dup_rows_per_dst=[2922, 2994, 3054, 2818, 3176, 2908, 3076, 3090] combine_rows_per_src=[9332, 9286, 9356, 9160, 9252, 9336, 9382, 9162] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74424 payload_rows_remote=65312 dup_rows=23880 dup_rows_remote=20934 dedup_ratio=0.242920 router_tokens_per_expert=[779, 752, 756, 754, 745, 784, 766, 762, 815, 807, 790, 743, 795, 743, 777, 807, 734, 755, 786, 733, 747, 752, 768, 754, 776, 750, 768, 753, 746, 790, 802, 820, 764, 752, 761, 766, 734, 812, 757, 770, 787, 767, 743, 714, 781, 744, 751, 726, 775, 750, 794, 807, 786, 796, 800, 774, 763, 767, 745, 803, 774, 736, 794, 750] payload_rows_per_dst=[9312, 9462, 9112, 9412, 9234, 9162, 9466, 9264] dup_rows_per_dst=[2884, 3092, 2946, 2998, 2998, 2864, 3098, 3000] combine_rows_per_src=[9312, 9462, 9112, 9412, 9234, 9162, 9466, 9264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266297344 dispatch_meta_bytes=343856 combine_payload_bytes=266297344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266641200 ingress_bytes=300900352 link_bytes=300900352 token_bytes=532938544 weight_movement_bytes=34603008 total_nvlink_bytes=567541552 weight_to_token_ratio=0.064929 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132263800e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=65052 dup_rows=23926 dup_rows_remote=20838 dedup_ratio=0.243388 router_tokens_per_expert=[824, 752, 754, 768, 754, 721, 784, 784, 782, 749, 796, 771, 792, 802, 779, 736, 735, 816, 816, 769, 739, 736, 730, 773, 768, 802, 754, 737, 800, 762, 752, 760, 785, 778, 800, 780, 826, 738, 758, 759, 749, 749, 748, 787, 800, 745, 761, 765, 747, 801, 755, 744, 806, 796, 763, 733, 769, 764, 782, 771, 741, 759, 749, 747] payload_rows_per_dst=[9280, 9326, 9228, 9330, 9318, 9344, 9328, 9224] dup_rows_per_dst=[3002, 3088, 3000, 2940, 3130, 2864, 2962, 2940] combine_rows_per_src=[9280, 9326, 9228, 9330, 9318, 9344, 9328, 9224] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74314 payload_rows_remote=64950 dup_rows=23990 dup_rows_remote=20946 dedup_ratio=0.244039 router_tokens_per_expert=[781, 749, 792, 775, 786, 732, 793, 796, 782, 763, 773, 834, 731, 771, 739, 748, 775, 785, 781, 782, 743, 732, 769, 789, 761, 787, 710, 800, 783, 710, 746, 761, 813, 719, 787, 763, 747, 730, 770, 807, 753, 788, 808, 763, 785, 786, 829, 776, 795, 793, 750, 790, 800, 735, 790, 768, 700, 709, 762, 720, 759, 762, 798, 738] payload_rows_per_dst=[9364, 9688, 9418, 8980, 9314, 9484, 9010, 9056] dup_rows_per_dst=[3044, 3212, 3200, 2830, 2958, 3092, 2814, 2840] combine_rows_per_src=[9364, 9688, 9418, 8980, 9314, 9484, 9010, 9056] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74328 payload_rows_remote=64952 dup_rows=23976 dup_rows_remote=20970 dedup_ratio=0.243896 router_tokens_per_expert=[826, 676, 763, 799, 759, 757, 784, 754, 764, 754, 787, 776, 747, 751, 782, 768, 744, 751, 733, 793, 746, 800, 782, 731, 747, 760, 734, 737, 781, 778, 764, 767, 754, 823, 740, 740, 779, 780, 785, 812, 799, 713, 804, 759, 809, 762, 787, 799, 762, 763, 773, 757, 759, 798, 773, 737, 750, 813, 754, 795, 786, 790, 749, 753] payload_rows_per_dst=[9230, 9148, 9268, 9222, 9416, 9436, 9232, 9376] dup_rows_per_dst=[3006, 2788, 2994, 2914, 3230, 3028, 3010, 3006] combine_rows_per_src=[9230, 9148, 9268, 9222, 9416, 9436, 9232, 9376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74410 payload_rows_remote=65082 dup_rows=23894 dup_rows_remote=21030 dedup_ratio=0.243062 router_tokens_per_expert=[759, 771, 743, 757, 749, 769, 772, 764, 766, 781, 768, 753, 768, 771, 746, 771, 800, 771, 731, 720, 767, 748, 733, 753, 770, 763, 730, 791, 786, 810, 782, 774, 771, 780, 784, 776, 806, 782, 743, 756, 817, 747, 768, 733, 755, 724, 817, 764, 762, 763, 736, 777, 767, 788, 743, 760, 764, 777, 728, 841, 789, 791, 786, 820] payload_rows_per_dst=[9196, 9240, 9122, 9444, 9362, 9296, 9328, 9422] dup_rows_per_dst=[2972, 3008, 2924, 2968, 3034, 2954, 2864, 3170] combine_rows_per_src=[9196, 9240, 9122, 9444, 9362, 9296, 9328, 9422] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74428 payload_rows_remote=65074 dup_rows=23876 dup_rows_remote=20900 dedup_ratio=0.242879 router_tokens_per_expert=[779, 738, 736, 770, 741, 790, 715, 769, 782, 791, 768, 746, 739, 799, 761, 744, 796, 780, 800, 780, 780, 731, 783, 810, 759, 767, 747, 807, 757, 774, 799, 795, 753, 776, 802, 726, 774, 761, 720, 798, 759, 739, 771, 744, 701, 790, 775, 713, 766, 794, 746, 774, 752, 757, 789, 787, 813, 793, 787, 728, 760, 756, 824, 791] payload_rows_per_dst=[9126, 9280, 9532, 9392, 9258, 9102, 9354, 9384] dup_rows_per_dst=[2950, 2980, 2988, 3018, 2962, 2882, 2976, 3120] combine_rows_per_src=[9126, 9280, 9532, 9392, 9258, 9102, 9354, 9384] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266297344 dispatch_meta_bytes=343504 combine_payload_bytes=266297344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266640848 ingress_bytes=300900352 link_bytes=300900352 token_bytes=532938192 weight_movement_bytes=34603008 total_nvlink_bytes=567541200 weight_to_token_ratio=0.064929 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132263800e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74434 payload_rows_remote=65210 dup_rows=23870 dup_rows_remote=20974 dedup_ratio=0.242818 router_tokens_per_expert=[749, 775, 792, 753, 791, 792, 788, 765, 766, 789, 789, 765, 811, 788, 784, 773, 760, 752, 806, 782, 773, 749, 762, 731, 775, 757, 745, 782, 757, 753, 751, 740, 779, 747, 764, 769, 710, 794, 721, 772, 766, 769, 798, 769, 749, 760, 732, 729, 788, 741, 733, 804, 790, 812, 768, 803, 768, 755, 764, 797, 755, 768, 745, 788] payload_rows_per_dst=[9366, 9436, 9258, 9224, 9132, 9332, 9442, 9244] dup_rows_per_dst=[3044, 3094, 2972, 2896, 2980, 2812, 3036, 3036] combine_rows_per_src=[9366, 9436, 9258, 9224, 9132, 9332, 9442, 9244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74542 payload_rows_remote=64974 dup_rows=23762 dup_rows_remote=20620 dedup_ratio=0.241720 router_tokens_per_expert=[748, 787, 781, 755, 755, 790, 795, 759, 748, 785, 794, 752, 747, 761, 726, 786, 772, 755, 787, 774, 785, 735, 764, 754, 733, 745, 784, 780, 813, 732, 747, 816, 774, 757, 802, 772, 756, 779, 775, 748, 783, 800, 756, 763, 783, 761, 739, 782, 739, 765, 747, 781, 751, 822, 699, 806, 764, 767, 775, 770, 780, 751, 805, 755] payload_rows_per_dst=[9568, 9342, 9328, 9512, 9298, 9594, 8836, 9064] dup_rows_per_dst=[3142, 2890, 2924, 3136, 3028, 3208, 2568, 2866] combine_rows_per_src=[9568, 9342, 9328, 9512, 9298, 9594, 8836, 9064] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=267517952 dispatch_meta_bytes=344984 combine_payload_bytes=267517952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267862936 ingress_bytes=302120960 link_bytes=302120960 token_bytes=535380888 weight_movement_bytes=34603008 total_nvlink_bytes=569983896 weight_to_token_ratio=0.064633 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136856850e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74474 payload_rows_remote=65328 dup_rows=23830 dup_rows_remote=21096 dedup_ratio=0.242411 router_tokens_per_expert=[784, 764, 786, 784, 750, 776, 765, 805, 743, 815, 823, 761, 809, 746, 753, 805, 759, 753, 801, 713, 753, 790, 755, 746, 767, 730, 778, 770, 774, 785, 788, 821, 781, 765, 755, 757, 784, 769, 817, 740, 756, 729, 765, 743, 731, 744, 711, 761, 755, 745, 765, 736, 785, 796, 728, 740, 756, 804, 734, 766, 820, 788, 803, 771] payload_rows_per_dst=[9358, 9396, 9234, 9382, 9320, 9146, 9164, 9474] dup_rows_per_dst=[3070, 3114, 2906, 3044, 3016, 2734, 2936, 3010] combine_rows_per_src=[9358, 9396, 9234, 9382, 9320, 9146, 9164, 9474] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=65052 dup_rows=23926 dup_rows_remote=20838 dedup_ratio=0.243388 router_tokens_per_expert=[824, 752, 754, 768, 754, 721, 784, 784, 782, 749, 796, 771, 792, 802, 779, 736, 735, 816, 816, 769, 739, 736, 730, 773, 768, 802, 754, 737, 800, 762, 752, 760, 785, 778, 800, 780, 826, 738, 758, 759, 749, 749, 748, 787, 800, 745, 761, 765, 747, 801, 755, 744, 806, 796, 763, 733, 769, 764, 782, 771, 741, 759, 749, 747] payload_rows_per_dst=[9280, 9326, 9228, 9330, 9318, 9344, 9328, 9224] dup_rows_per_dst=[3002, 3088, 3000, 2940, 3130, 2864, 2962, 2940] combine_rows_per_src=[9280, 9326, 9228, 9330, 9318, 9344, 9328, 9224] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266035200 dispatch_meta_bytes=343584 combine_payload_bytes=266035200 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266378784 ingress_bytes=335241216 link_bytes=335241216 token_bytes=532413984 weight_movement_bytes=69206016 total_nvlink_bytes=601620000 weight_to_token_ratio=0.129985 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261485707e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266043392 dispatch_meta_bytes=343688 combine_payload_bytes=266043392 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266387080 ingress_bytes=369852416 link_bytes=369852416 token_bytes=532430472 weight_movement_bytes=103809024 total_nvlink_bytes=636239496 weight_to_token_ratio=0.194972 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391724866e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266575872 dispatch_meta_bytes=344448 combine_payload_bytes=266575872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266920320 ingress_bytes=335781888 link_bytes=335781888 token_bytes=533496192 weight_movement_bytes=69206016 total_nvlink_bytes=602702208 weight_to_token_ratio=0.129722 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263520212e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266452992 dispatch_meta_bytes=343560 combine_payload_bytes=266452992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266796552 ingress_bytes=335659008 link_bytes=335659008 token_bytes=533249544 weight_movement_bytes=69206016 total_nvlink_bytes=602455560 weight_to_token_ratio=0.129782 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263057824e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=64800 dup_rows=24132 dup_rows_remote=21100 dedup_ratio=0.245483 router_tokens_per_expert=[783, 792, 754, 769, 748, 733, 791, 758, 820, 769, 719, 743, 768, 786, 810, 755, 790, 786, 812, 755, 769, 772, 798, 734, 789, 754, 756, 753, 783, 688, 775, 747, 756, 720, 743, 772, 758, 811, 737, 806, 738, 784, 732, 752, 780, 721, 773, 806, 788, 758, 747, 780, 783, 778, 786, 785, 751, 786, 716, 806, 783, 785, 777, 795] payload_rows_per_dst=[9284, 8540, 9378, 9178, 9244, 9930, 9246, 9372] dup_rows_per_dst=[2972, 2576, 3054, 2912, 2962, 3466, 3158, 3032] combine_rows_per_src=[9284, 8540, 9378, 9178, 9244, 9930, 9246, 9372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=343896 combine_payload_bytes=266543104 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266887000 ingress_bytes=370352128 link_bytes=370352128 token_bytes=533430104 weight_movement_bytes=103809024 total_nvlink_bytes=637239128 weight_to_token_ratio=0.194607 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393605242e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=267100160 dispatch_meta_bytes=344736 combine_payload_bytes=267100160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267444896 ingress_bytes=301703168 link_bytes=301703168 token_bytes=534545056 weight_movement_bytes=34603008 total_nvlink_bytes=569148064 weight_to_token_ratio=0.064734 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135284732e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=267583488 dispatch_meta_bytes=345696 combine_payload_bytes=267583488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267929184 ingress_bytes=302186496 link_bytes=302186496 token_bytes=535512672 weight_movement_bytes=34603008 total_nvlink_bytes=570115680 weight_to_token_ratio=0.064617 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137103456e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266452992 dispatch_meta_bytes=343560 combine_payload_bytes=266452992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266796552 ingress_bytes=335659008 link_bytes=335659008 token_bytes=533249544 weight_movement_bytes=69206016 total_nvlink_bytes=602455560 weight_to_token_ratio=0.129782 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263057824e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=342376 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266475880 ingress_bytes=300736512 link_bytes=300736512 token_bytes=532609384 weight_movement_bytes=34603008 total_nvlink_bytes=567212392 weight_to_token_ratio=0.064969 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131647283e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74474 payload_rows_remote=65328 dup_rows=23830 dup_rows_remote=21096 dedup_ratio=0.242411 router_tokens_per_expert=[784, 764, 786, 784, 750, 776, 765, 805, 743, 815, 823, 761, 809, 746, 753, 805, 759, 753, 801, 713, 753, 790, 755, 746, 767, 730, 778, 770, 774, 785, 788, 821, 781, 765, 755, 757, 784, 769, 817, 740, 756, 729, 765, 743, 731, 744, 711, 761, 755, 745, 765, 736, 785, 796, 728, 740, 756, 804, 734, 766, 820, 788, 803, 771] payload_rows_per_dst=[9358, 9396, 9234, 9382, 9320, 9146, 9164, 9474] dup_rows_per_dst=[3070, 3114, 2906, 3044, 3016, 2734, 2936, 3010] combine_rows_per_src=[9358, 9396, 9234, 9382, 9320, 9146, 9164, 9474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=64992 dup_rows=23998 dup_rows_remote=20908 dedup_ratio=0.244120 router_tokens_per_expert=[714, 778, 716, 762, 757, 789, 774, 781, 749, 715, 713, 756, 778, 780, 812, 766, 749, 787, 765, 739, 814, 737, 786, 808, 735, 826, 772, 781, 768, 778, 787, 781, 794, 749, 774, 784, 789, 774, 783, 755, 773, 753, 783, 767, 774, 755, 763, 748, 746, 781, 754, 772, 748, 761, 754, 756, 813, 809, 754, 811, 767, 740, 773, 742] payload_rows_per_dst=[9204, 9240, 9264, 9438, 9314, 9256, 9266, 9324] dup_rows_per_dst=[2938, 2898, 3106, 3018, 3090, 2976, 2878, 3094] combine_rows_per_src=[9204, 9240, 9264, 9438, 9314, 9256, 9266, 9324] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265420800 dispatch_meta_bytes=343600 combine_payload_bytes=265420800 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265764400 ingress_bytes=369229824 link_bytes=369229824 token_bytes=531185200 weight_movement_bytes=103809024 total_nvlink_bytes=634994224 weight_to_token_ratio=0.195429 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389382102e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=64940 dup_rows=24096 dup_rows_remote=21042 dedup_ratio=0.245117 router_tokens_per_expert=[790, 754, 779, 738, 775, 746, 744, 755, 853, 720, 774, 751, 763, 694, 802, 771, 747, 789, 763, 755, 759, 776, 802, 770, 746, 781, 765, 807, 791, 805, 721, 756, 739, 775, 722, 814, 782, 722, 767, 821, 776, 752, 786, 759, 763, 805, 776, 801, 748, 796, 773, 751, 771, 785, 740, 735, 796, 784, 729, 759, 760, 782, 734, 807] payload_rows_per_dst=[9120, 9292, 9268, 9316, 9288, 9302, 9270, 9352] dup_rows_per_dst=[3042, 2964, 3054, 3028, 2996, 3134, 2928, 2950] combine_rows_per_src=[9120, 9292, 9268, 9316, 9288, 9302, 9270, 9352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=267583488 dispatch_meta_bytes=345696 combine_payload_bytes=267583488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267929184 ingress_bytes=302186496 link_bytes=302186496 token_bytes=535512672 weight_movement_bytes=34603008 total_nvlink_bytes=570115680 weight_to_token_ratio=0.064617 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137103456e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65106 dup_rows=23924 dup_rows_remote=20992 dedup_ratio=0.243368 router_tokens_per_expert=[740, 771, 767, 767, 789, 787, 771, 768, 764, 771, 756, 754, 775, 770, 762, 751, 798, 844, 757, 814, 793, 760, 729, 761, 736, 776, 784, 750, 767, 763, 735, 787, 744, 733, 762, 753, 768, 763, 751, 751, 748, 747, 807, 786, 753, 795, 747, 766, 783, 746, 744, 806, 749, 771, 770, 776, 818, 735, 809, 780, 778, 774, 772, 750] payload_rows_per_dst=[9360, 9274, 9470, 9288, 9104, 9262, 9236, 9386] dup_rows_per_dst=[2960, 2932, 3042, 2908, 2946, 3036, 3054, 3046] combine_rows_per_src=[9360, 9274, 9470, 9288, 9104, 9262, 9236, 9386] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74542 payload_rows_remote=64974 dup_rows=23762 dup_rows_remote=20620 dedup_ratio=0.241720 router_tokens_per_expert=[748, 787, 781, 755, 755, 790, 795, 759, 748, 785, 794, 752, 747, 761, 726, 786, 772, 755, 787, 774, 785, 735, 764, 754, 733, 745, 784, 780, 813, 732, 747, 816, 774, 757, 802, 772, 756, 779, 775, 748, 783, 800, 756, 763, 783, 761, 739, 782, 739, 765, 747, 781, 751, 822, 699, 806, 764, 767, 775, 770, 780, 751, 805, 755] payload_rows_per_dst=[9568, 9342, 9328, 9512, 9298, 9594, 8836, 9064] dup_rows_per_dst=[3142, 2890, 2924, 3136, 3028, 3208, 2568, 2866] combine_rows_per_src=[9568, 9342, 9328, 9512, 9298, 9594, 8836, 9064] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=64800 dup_rows=24132 dup_rows_remote=21100 dedup_ratio=0.245483 router_tokens_per_expert=[783, 792, 754, 769, 748, 733, 791, 758, 820, 769, 719, 743, 768, 786, 810, 755, 790, 786, 812, 755, 769, 772, 798, 734, 789, 754, 756, 753, 783, 688, 775, 747, 756, 720, 743, 772, 758, 811, 737, 806, 738, 784, 732, 752, 780, 721, 773, 806, 788, 758, 747, 780, 783, 778, 786, 785, 751, 786, 716, 806, 783, 785, 777, 795] payload_rows_per_dst=[9284, 8540, 9378, 9178, 9244, 9930, 9246, 9372] dup_rows_per_dst=[2972, 2576, 3054, 2912, 2962, 3466, 3158, 3032] combine_rows_per_src=[9284, 8540, 9378, 9178, 9244, 9930, 9246, 9372] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74428 payload_rows_remote=65074 dup_rows=23876 dup_rows_remote=20900 dedup_ratio=0.242879 router_tokens_per_expert=[779, 738, 736, 770, 741, 790, 715, 769, 782, 791, 768, 746, 739, 799, 761, 744, 796, 780, 800, 780, 780, 731, 783, 810, 759, 767, 747, 807, 757, 774, 799, 795, 753, 776, 802, 726, 774, 761, 720, 798, 759, 739, 771, 744, 701, 790, 775, 713, 766, 794, 746, 774, 752, 757, 789, 787, 813, 793, 787, 728, 760, 756, 824, 791] payload_rows_per_dst=[9126, 9280, 9532, 9392, 9258, 9102, 9354, 9384] dup_rows_per_dst=[2950, 2980, 2988, 3018, 2962, 2882, 2976, 3120] combine_rows_per_src=[9126, 9280, 9532, 9392, 9258, 9102, 9354, 9384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74350 payload_rows_remote=64978 dup_rows=23954 dup_rows_remote=20946 dedup_ratio=0.243673 router_tokens_per_expert=[788, 785, 745, 769, 798, 774, 758, 709, 770, 758, 779, 749, 777, 783, 815, 798, 738, 787, 736, 805, 759, 741, 764, 796, 689, 774, 777, 737, 761, 778, 774, 778, 738, 753, 746, 777, 793, 768, 750, 777, 756, 764, 776, 807, 784, 702, 754, 774, 810, 761, 829, 791, 746, 740, 757, 756, 736, 807, 768, 798, 779, 741, 747, 818] payload_rows_per_dst=[9276, 9332, 9326, 9188, 9252, 9222, 9372, 9382] dup_rows_per_dst=[2976, 3126, 2926, 2948, 2952, 3012, 3008, 3006] combine_rows_per_src=[9276, 9332, 9326, 9188, 9252, 9222, 9372, 9382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74484 payload_rows_remote=65214 dup_rows=23820 dup_rows_remote=20876 dedup_ratio=0.242310 router_tokens_per_expert=[742, 785, 763, 789, 724, 741, 761, 750, 809, 793, 769, 777, 734, 781, 776, 763, 743, 800, 752, 787, 788, 782, 750, 832, 774, 794, 782, 754, 723, 800, 764, 716, 784, 784, 758, 785, 765, 743, 761, 753, 779, 741, 700, 747, 728, 724, 745, 797, 781, 777, 816, 762, 762, 782, 763, 781, 827, 808, 779, 762, 755, 770, 793, 742] payload_rows_per_dst=[9222, 9320, 9450, 9270, 9216, 9186, 9400, 9420] dup_rows_per_dst=[2888, 3084, 3018, 2944, 3050, 2736, 3048, 3052] combine_rows_per_src=[9222, 9320, 9450, 9270, 9216, 9186, 9400, 9420] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266207232 dispatch_meta_bytes=343600 combine_payload_bytes=266207232 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266550832 ingress_bytes=335413248 link_bytes=335413248 token_bytes=532758064 weight_movement_bytes=69206016 total_nvlink_bytes=601964080 weight_to_token_ratio=0.129901 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262133049e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=64842 dup_rows=24008 dup_rows_remote=20790 dedup_ratio=0.244222 router_tokens_per_expert=[795, 783, 749, 787, 793, 757, 736, 769, 750, 783, 778, 778, 772, 756, 805, 812, 812, 772, 790, 781, 752, 769, 756, 744, 752, 743, 714, 784, 769, 765, 772, 809, 785, 726, 762, 766, 796, 760, 735, 765, 774, 708, 797, 714, 733, 771, 731, 770, 771, 775, 786, 747, 775, 807, 759, 826, 772, 745, 765, 755, 785, 765, 785, 754] payload_rows_per_dst=[9454, 9668, 9212, 9314, 9164, 9162, 9350, 8972] dup_rows_per_dst=[3218, 3340, 2880, 3054, 2992, 2834, 2990, 2700] combine_rows_per_src=[9454, 9668, 9212, 9314, 9164, 9162, 9350, 8972] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=343928 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266338168 ingress_bytes=335200256 link_bytes=335200256 token_bytes=532332408 weight_movement_bytes=69206016 total_nvlink_bytes=601538424 weight_to_token_ratio=0.130005 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261331577e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74484 payload_rows_remote=65214 dup_rows=23820 dup_rows_remote=20876 dedup_ratio=0.242310 router_tokens_per_expert=[742, 785, 763, 789, 724, 741, 761, 750, 809, 793, 769, 777, 734, 781, 776, 763, 743, 800, 752, 787, 788, 782, 750, 832, 774, 794, 782, 754, 723, 800, 764, 716, 784, 784, 758, 785, 765, 743, 761, 753, 779, 741, 700, 747, 728, 724, 745, 797, 781, 777, 816, 762, 762, 782, 763, 781, 827, 808, 779, 762, 755, 770, 793, 742] payload_rows_per_dst=[9222, 9320, 9450, 9270, 9216, 9186, 9400, 9420] dup_rows_per_dst=[2888, 3084, 3018, 2944, 3050, 2736, 3048, 3052] combine_rows_per_src=[9222, 9320, 9450, 9270, 9216, 9186, 9400, 9420] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74484 payload_rows_remote=65174 dup_rows=23820 dup_rows_remote=20798 dedup_ratio=0.242310 router_tokens_per_expert=[751, 779, 752, 814, 739, 788, 838, 752, 749, 753, 800, 807, 756, 766, 733, 761, 769, 816, 738, 725, 804, 749, 784, 785, 767, 805, 784, 764, 763, 741, 764, 747, 778, 747, 767, 768, 791, 783, 743, 767, 796, 793, 759, 782, 752, 754, 774, 756, 730, 767, 721, 726, 744, 750, 719, 790, 751, 771, 766, 766, 807, 790, 774, 827] payload_rows_per_dst=[9316, 9362, 9384, 9308, 9286, 9310, 9054, 9464] dup_rows_per_dst=[3110, 2888, 2956, 2962, 3002, 3022, 2840, 3040] combine_rows_per_src=[9316, 9362, 9384, 9308, 9286, 9310, 9054, 9464] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65106 dup_rows=23924 dup_rows_remote=20992 dedup_ratio=0.243368 router_tokens_per_expert=[740, 771, 767, 767, 789, 787, 771, 768, 764, 771, 756, 754, 775, 770, 762, 751, 798, 844, 757, 814, 793, 760, 729, 761, 736, 776, 784, 750, 767, 763, 735, 787, 744, 733, 762, 753, 768, 763, 751, 751, 748, 747, 807, 786, 753, 795, 747, 766, 783, 746, 744, 806, 749, 771, 770, 776, 818, 735, 809, 780, 778, 774, 772, 750] payload_rows_per_dst=[9360, 9274, 9470, 9288, 9104, 9262, 9236, 9386] dup_rows_per_dst=[2960, 2932, 3042, 2908, 2946, 3036, 3054, 3046] combine_rows_per_src=[9360, 9274, 9470, 9288, 9104, 9262, 9236, 9386] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=342376 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266475880 ingress_bytes=300736512 link_bytes=300736512 token_bytes=532609384 weight_movement_bytes=34603008 total_nvlink_bytes=567212392 weight_to_token_ratio=0.064969 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131647283e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265420800 dispatch_meta_bytes=343600 combine_payload_bytes=265420800 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265764400 ingress_bytes=369229824 link_bytes=369229824 token_bytes=531185200 weight_movement_bytes=103809024 total_nvlink_bytes=634994224 weight_to_token_ratio=0.195429 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389382102e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266674176 dispatch_meta_bytes=344392 combine_payload_bytes=266674176 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267018568 ingress_bytes=301277184 link_bytes=301277184 token_bytes=533692744 weight_movement_bytes=34603008 total_nvlink_bytes=568295752 weight_to_token_ratio=0.064837 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133681789e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=343896 combine_payload_bytes=266543104 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266887000 ingress_bytes=370352128 link_bytes=370352128 token_bytes=533430104 weight_movement_bytes=103809024 total_nvlink_bytes=637239128 weight_to_token_ratio=0.194607 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393605242e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=343696 combine_payload_bytes=266149888 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266493584 ingress_bytes=335355904 link_bytes=335355904 token_bytes=532643472 weight_movement_bytes=69206016 total_nvlink_bytes=601849488 weight_to_token_ratio=0.129929 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261917268e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74282 payload_rows_remote=64914 dup_rows=24022 dup_rows_remote=20886 dedup_ratio=0.244364 router_tokens_per_expert=[778, 779, 811, 740, 774, 748, 797, 764, 770, 739, 734, 726, 721, 797, 723, 758, 776, 784, 799, 784, 754, 719, 772, 760, 760, 784, 794, 772, 799, 747, 763, 743, 788, 763, 808, 822, 775, 784, 766, 767, 722, 755, 743, 717, 775, 719, 754, 768, 781, 800, 850, 732, 782, 804, 746, 797, 790, 747, 768, 756, 828, 754, 770, 752] payload_rows_per_dst=[9336, 9084, 9308, 9246, 9392, 9086, 9462, 9368] dup_rows_per_dst=[3046, 2852, 2958, 3078, 2980, 2820, 3152, 3136] combine_rows_per_src=[9336, 9084, 9308, 9246, 9392, 9086, 9462, 9368] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=267116544 dispatch_meta_bytes=344360 combine_payload_bytes=267116544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267460904 ingress_bytes=301719552 link_bytes=301719552 token_bytes=534577448 weight_movement_bytes=34603008 total_nvlink_bytes=569180456 weight_to_token_ratio=0.064730 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135346384e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266674176 dispatch_meta_bytes=344392 combine_payload_bytes=266674176 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267018568 ingress_bytes=301277184 link_bytes=301277184 token_bytes=533692744 weight_movement_bytes=34603008 total_nvlink_bytes=568295752 weight_to_token_ratio=0.064837 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133681789e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265592832 dispatch_meta_bytes=342528 combine_payload_bytes=265592832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265935360 ingress_bytes=300195840 link_bytes=300195840 token_bytes=531528192 weight_movement_bytes=34603008 total_nvlink_bytes=566131200 weight_to_token_ratio=0.065101 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129612778e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74484 payload_rows_remote=65174 dup_rows=23820 dup_rows_remote=20798 dedup_ratio=0.242310 router_tokens_per_expert=[751, 779, 752, 814, 739, 788, 838, 752, 749, 753, 800, 807, 756, 766, 733, 761, 769, 816, 738, 725, 804, 749, 784, 785, 767, 805, 784, 764, 763, 741, 764, 747, 778, 747, 767, 768, 791, 783, 743, 767, 796, 793, 759, 782, 752, 754, 774, 756, 730, 767, 721, 726, 744, 750, 719, 790, 751, 771, 766, 766, 807, 790, 774, 827] payload_rows_per_dst=[9316, 9362, 9384, 9308, 9286, 9310, 9054, 9464] dup_rows_per_dst=[3110, 2888, 2956, 2962, 3002, 3022, 2840, 3040] combine_rows_per_src=[9316, 9362, 9384, 9308, 9286, 9310, 9054, 9464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265887744 dispatch_meta_bytes=343200 combine_payload_bytes=265887744 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266230944 ingress_bytes=335093760 link_bytes=335093760 token_bytes=532118688 weight_movement_bytes=69206016 total_nvlink_bytes=601324704 weight_to_token_ratio=0.130057 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260930842e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266952704 dispatch_meta_bytes=343888 combine_payload_bytes=266952704 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267296592 ingress_bytes=336158720 link_bytes=336158720 token_bytes=534249296 weight_movement_bytes=69206016 total_nvlink_bytes=603455312 weight_to_token_ratio=0.129539 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264938200e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74220 payload_rows_remote=64958 dup_rows=24084 dup_rows_remote=21118 dedup_ratio=0.244995 router_tokens_per_expert=[778, 741, 814, 736, 768, 746, 793, 787, 771, 738, 789, 770, 758, 748, 760, 766, 733, 793, 803, 809, 770, 757, 802, 748, 770, 813, 755, 766, 788, 775, 806, 758, 791, 796, 810, 728, 750, 764, 732, 743, 744, 752, 762, 804, 763, 769, 763, 747, 801, 766, 771, 799, 710, 811, 726, 745, 782, 738, 750, 795, 757, 742, 723, 809] payload_rows_per_dst=[9238, 9192, 9380, 9354, 9262, 9264, 9304, 9226] dup_rows_per_dst=[3088, 3008, 3050, 3108, 2966, 2944, 2954, 2966] combine_rows_per_src=[9238, 9192, 9380, 9354, 9262, 9264, 9304, 9226] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=65092 dup_rows=23946 dup_rows_remote=20992 dedup_ratio=0.243591 router_tokens_per_expert=[803, 752, 735, 760, 772, 796, 752, 763, 751, 780, 811, 792, 707, 787, 790, 735, 733, 764, 791, 783, 753, 773, 755, 758, 762, 752, 776, 756, 771, 739, 769, 761, 777, 774, 763, 757, 756, 808, 811, 760, 721, 775, 790, 732, 800, 812, 800, 763, 759, 775, 798, 746, 801, 738, 785, 792, 740, 719, 780, 763, 778, 760, 793, 744] payload_rows_per_dst=[9274, 9364, 9266, 9304, 9338, 9296, 9336, 9180] dup_rows_per_dst=[2992, 2942, 2954, 2868, 3074, 3090, 3052, 2974] combine_rows_per_src=[9274, 9364, 9266, 9304, 9338, 9296, 9336, 9180] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=64842 dup_rows=24008 dup_rows_remote=20790 dedup_ratio=0.244222 router_tokens_per_expert=[795, 783, 749, 787, 793, 757, 736, 769, 750, 783, 778, 778, 772, 756, 805, 812, 812, 772, 790, 781, 752, 769, 756, 744, 752, 743, 714, 784, 769, 765, 772, 809, 785, 726, 762, 766, 796, 760, 735, 765, 774, 708, 797, 714, 733, 771, 731, 770, 771, 775, 786, 747, 775, 807, 759, 826, 772, 745, 765, 755, 785, 765, 785, 754] payload_rows_per_dst=[9454, 9668, 9212, 9314, 9164, 9162, 9350, 8972] dup_rows_per_dst=[3218, 3340, 2880, 3054, 2992, 2834, 2990, 2700] combine_rows_per_src=[9454, 9668, 9212, 9314, 9164, 9162, 9350, 8972] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74066 payload_rows_remote=64840 dup_rows=24238 dup_rows_remote=21234 dedup_ratio=0.246562 router_tokens_per_expert=[744, 776, 782, 803, 723, 763, 742, 743, 750, 783, 776, 768, 767, 765, 750, 756, 739, 789, 732, 797, 818, 770, 709, 786, 773, 771, 778, 801, 796, 753, 792, 770, 716, 751, 745, 768, 785, 791, 729, 704, 792, 805, 743, 773, 825, 753, 835, 763, 780, 778, 762, 781, 760, 741, 781, 788, 757, 746, 757, 745, 780, 768, 769, 816] payload_rows_per_dst=[9204, 9226, 9270, 9360, 9124, 9390, 9262, 9230] dup_rows_per_dst=[2948, 3004, 3010, 3108, 2854, 3188, 3080, 3046] combine_rows_per_src=[9204, 9226, 9270, 9360, 9124, 9390, 9262, 9230] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74282 payload_rows_remote=64914 dup_rows=24022 dup_rows_remote=20886 dedup_ratio=0.244364 router_tokens_per_expert=[778, 779, 811, 740, 774, 748, 797, 764, 770, 739, 734, 726, 721, 797, 723, 758, 776, 784, 799, 784, 754, 719, 772, 760, 760, 784, 794, 772, 799, 747, 763, 743, 788, 763, 808, 822, 775, 784, 766, 767, 722, 755, 743, 717, 775, 719, 754, 768, 781, 800, 850, 732, 782, 804, 746, 797, 790, 747, 768, 756, 828, 754, 770, 752] payload_rows_per_dst=[9336, 9084, 9308, 9246, 9392, 9086, 9462, 9368] dup_rows_per_dst=[3046, 2852, 2958, 3078, 2980, 2820, 3152, 3136] combine_rows_per_src=[9336, 9084, 9308, 9246, 9392, 9086, 9462, 9368] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74290 payload_rows_remote=64896 dup_rows=24014 dup_rows_remote=21000 dedup_ratio=0.244283 router_tokens_per_expert=[802, 723, 761, 769, 776, 755, 781, 714, 771, 784, 791, 793, 752, 827, 715, 800, 799, 736, 752, 782, 744, 787, 737, 756, 784, 779, 758, 768, 794, 821, 744, 745, 774, 708, 768, 773, 765, 777, 750, 756, 835, 769, 740, 740, 757, 801, 755, 822, 768, 766, 777, 781, 776, 778, 781, 777, 761, 773, 782, 799, 760, 736, 707, 740] payload_rows_per_dst=[9246, 9404, 9214, 9304, 9248, 9292, 9394, 9188] dup_rows_per_dst=[2916, 3062, 2972, 3082, 2894, 3146, 3014, 2928] combine_rows_per_src=[9246, 9404, 9214, 9304, 9248, 9292, 9394, 9188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74166 payload_rows_remote=64924 dup_rows=24138 dup_rows_remote=21092 dedup_ratio=0.245544 router_tokens_per_expert=[753, 803, 789, 719, 765, 770, 774, 829, 794, 748, 741, 768, 771, 757, 802, 754, 730, 755, 757, 780, 769, 727, 735, 790, 770, 784, 777, 735, 713, 785, 809, 771, 783, 781, 766, 806, 745, 806, 759, 753, 782, 780, 793, 756, 719, 771, 744, 781, 752, 773, 760, 808, 782, 773, 755, 725, 785, 787, 784, 754, 773, 752, 756, 784] payload_rows_per_dst=[9304, 9256, 9122, 9242, 9284, 9332, 9316, 9310] dup_rows_per_dst=[3100, 3014, 2964, 3046, 3114, 2920, 2940, 3040] combine_rows_per_src=[9304, 9256, 9122, 9242, 9284, 9332, 9316, 9310] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=344304 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266412272 ingress_bytes=335273984 link_bytes=335273984 token_bytes=532480240 weight_movement_bytes=69206016 total_nvlink_bytes=601686256 weight_to_token_ratio=0.129969 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261609010e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74350 payload_rows_remote=64978 dup_rows=23954 dup_rows_remote=20946 dedup_ratio=0.243673 router_tokens_per_expert=[788, 785, 745, 769, 798, 774, 758, 709, 770, 758, 779, 749, 777, 783, 815, 798, 738, 787, 736, 805, 759, 741, 764, 796, 689, 774, 777, 737, 761, 778, 774, 778, 738, 753, 746, 777, 793, 768, 750, 777, 756, 764, 776, 807, 784, 702, 754, 774, 810, 761, 829, 791, 746, 740, 757, 756, 736, 807, 768, 798, 779, 741, 747, 818] payload_rows_per_dst=[9276, 9332, 9326, 9188, 9252, 9222, 9372, 9382] dup_rows_per_dst=[2976, 3126, 2926, 2948, 2952, 3012, 3008, 3006] combine_rows_per_src=[9276, 9332, 9326, 9188, 9252, 9222, 9372, 9382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74420 payload_rows_remote=65244 dup_rows=23884 dup_rows_remote=21042 dedup_ratio=0.242961 router_tokens_per_expert=[725, 757, 739, 784, 722, 788, 735, 735, 717, 737, 769, 776, 758, 786, 777, 766, 798, 741, 787, 796, 801, 711, 734, 761, 795, 755, 790, 778, 752, 764, 814, 763, 791, 756, 787, 780, 758, 735, 789, 760, 786, 718, 762, 798, 793, 813, 787, 812, 769, 741, 782, 762, 780, 779, 749, 787, 726, 765, 759, 763, 816, 803, 754, 781] payload_rows_per_dst=[9176, 9472, 9268, 9306, 9400, 9442, 9200, 9156] dup_rows_per_dst=[2842, 3198, 2996, 2900, 3014, 3042, 2996, 2896] combine_rows_per_src=[9176, 9472, 9268, 9306, 9400, 9442, 9200, 9156] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266616832 dispatch_meta_bytes=344336 combine_payload_bytes=266616832 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266961168 ingress_bytes=335822848 link_bytes=335822848 token_bytes=533578000 weight_movement_bytes=69206016 total_nvlink_bytes=602784016 weight_to_token_ratio=0.129702 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263674341e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265592832 dispatch_meta_bytes=342528 combine_payload_bytes=265592832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265935360 ingress_bytes=300195840 link_bytes=300195840 token_bytes=531528192 weight_movement_bytes=34603008 total_nvlink_bytes=566131200 weight_to_token_ratio=0.065101 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129612778e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74066 payload_rows_remote=64840 dup_rows=24238 dup_rows_remote=21234 dedup_ratio=0.246562 router_tokens_per_expert=[744, 776, 782, 803, 723, 763, 742, 743, 750, 783, 776, 768, 767, 765, 750, 756, 739, 789, 732, 797, 818, 770, 709, 786, 773, 771, 778, 801, 796, 753, 792, 770, 716, 751, 745, 768, 785, 791, 729, 704, 792, 805, 743, 773, 825, 753, 835, 763, 780, 778, 762, 781, 760, 741, 781, 788, 757, 746, 757, 745, 780, 768, 769, 816] payload_rows_per_dst=[9204, 9226, 9270, 9360, 9124, 9390, 9262, 9230] dup_rows_per_dst=[2948, 3004, 3010, 3108, 2854, 3188, 3080, 3046] combine_rows_per_src=[9204, 9226, 9270, 9360, 9124, 9390, 9262, 9230] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265887744 dispatch_meta_bytes=343200 combine_payload_bytes=265887744 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266230944 ingress_bytes=335093760 link_bytes=335093760 token_bytes=532118688 weight_movement_bytes=69206016 total_nvlink_bytes=601324704 weight_to_token_ratio=0.130057 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260930842e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74166 payload_rows_remote=64924 dup_rows=24138 dup_rows_remote=21092 dedup_ratio=0.245544 router_tokens_per_expert=[753, 803, 789, 719, 765, 770, 774, 829, 794, 748, 741, 768, 771, 757, 802, 754, 730, 755, 757, 780, 769, 727, 735, 790, 770, 784, 777, 735, 713, 785, 809, 771, 783, 781, 766, 806, 745, 806, 759, 753, 782, 780, 793, 756, 719, 771, 744, 781, 752, 773, 760, 808, 782, 773, 755, 725, 785, 787, 784, 754, 773, 752, 756, 784] payload_rows_per_dst=[9304, 9256, 9122, 9242, 9284, 9332, 9316, 9310] dup_rows_per_dst=[3100, 3014, 2964, 3046, 3114, 2920, 2940, 3040] combine_rows_per_src=[9304, 9256, 9122, 9242, 9284, 9332, 9316, 9310] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74346 payload_rows_remote=64990 dup_rows=23958 dup_rows_remote=20858 dedup_ratio=0.243713 router_tokens_per_expert=[765, 786, 719, 768, 778, 748, 758, 756, 736, 818, 756, 753, 724, 792, 757, 747, 782, 750, 797, 746, 812, 776, 780, 784, 760, 797, 768, 738, 783, 787, 732, 773, 766, 764, 775, 755, 756, 737, 753, 841, 751, 757, 762, 807, 792, 762, 762, 835, 764, 791, 762, 790, 749, 752, 735, 757, 798, 816, 740, 718, 782, 795, 767, 735] payload_rows_per_dst=[9108, 9286, 9400, 9264, 9392, 9356, 9218, 9322] dup_rows_per_dst=[3048, 2880, 3054, 3012, 2902, 3100, 2982, 2980] combine_rows_per_src=[9108, 9286, 9400, 9264, 9392, 9356, 9218, 9322] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=343696 combine_payload_bytes=266149888 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266493584 ingress_bytes=335355904 link_bytes=335355904 token_bytes=532643472 weight_movement_bytes=69206016 total_nvlink_bytes=601849488 weight_to_token_ratio=0.129929 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261917268e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265584640 dispatch_meta_bytes=344296 combine_payload_bytes=265584640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265928936 ingress_bytes=300187648 link_bytes=300187648 token_bytes=531513576 weight_movement_bytes=34603008 total_nvlink_bytes=566116584 weight_to_token_ratio=0.065103 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129581952e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265814016 dispatch_meta_bytes=343584 combine_payload_bytes=265814016 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266157600 ingress_bytes=335020032 link_bytes=335020032 token_bytes=531971616 weight_movement_bytes=69206016 total_nvlink_bytes=601177632 weight_to_token_ratio=0.130093 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260653409e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74108 payload_rows_remote=64870 dup_rows=24196 dup_rows_remote=21168 dedup_ratio=0.246134 router_tokens_per_expert=[748, 786, 739, 757, 797, 823, 788, 753, 774, 763, 722, 732, 740, 792, 735, 739, 776, 789, 833, 766, 790, 684, 753, 756, 798, 773, 776, 794, 764, 748, 805, 839, 755, 798, 773, 718, 772, 770, 746, 779, 792, 777, 791, 780, 758, 756, 746, 746, 768, 775, 689, 769, 770, 758, 761, 847, 770, 784, 765, 761, 764, 756, 773, 753] payload_rows_per_dst=[9264, 9136, 8954, 9412, 9494, 9288, 9322, 9238] dup_rows_per_dst=[3118, 2858, 2694, 3182, 3374, 3004, 2938, 3028] combine_rows_per_src=[9264, 9136, 8954, 9412, 9494, 9288, 9322, 9238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74346 payload_rows_remote=64990 dup_rows=23958 dup_rows_remote=20858 dedup_ratio=0.243713 router_tokens_per_expert=[765, 786, 719, 768, 778, 748, 758, 756, 736, 818, 756, 753, 724, 792, 757, 747, 782, 750, 797, 746, 812, 776, 780, 784, 760, 797, 768, 738, 783, 787, 732, 773, 766, 764, 775, 755, 756, 737, 753, 841, 751, 757, 762, 807, 792, 762, 762, 835, 764, 791, 762, 790, 749, 752, 735, 757, 798, 816, 740, 718, 782, 795, 767, 735] payload_rows_per_dst=[9108, 9286, 9400, 9264, 9392, 9356, 9218, 9322] dup_rows_per_dst=[3048, 2880, 3054, 3012, 2902, 3100, 2982, 2980] combine_rows_per_src=[9108, 9286, 9400, 9264, 9392, 9356, 9218, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=267239424 dispatch_meta_bytes=345144 combine_payload_bytes=267239424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267584568 ingress_bytes=301842432 link_bytes=301842432 token_bytes=534823992 weight_movement_bytes=34603008 total_nvlink_bytes=569427000 weight_to_token_ratio=0.064700 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135808771e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265928704 dispatch_meta_bytes=344064 combine_payload_bytes=265928704 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266272768 ingress_bytes=335134720 link_bytes=335134720 token_bytes=532201472 weight_movement_bytes=69206016 total_nvlink_bytes=601407488 weight_to_token_ratio=0.130037 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261084971e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265707520 dispatch_meta_bytes=344152 combine_payload_bytes=265707520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266051672 ingress_bytes=300310528 link_bytes=300310528 token_bytes=531759192 weight_movement_bytes=34603008 total_nvlink_bytes=566362200 weight_to_token_ratio=0.065073 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130044340e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=343392 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266542432 ingress_bytes=335405056 link_bytes=335405056 token_bytes=532741472 weight_movement_bytes=69206016 total_nvlink_bytes=601947488 weight_to_token_ratio=0.129905 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262102223e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=65042 dup_rows=24056 dup_rows_remote=21072 dedup_ratio=0.244710 router_tokens_per_expert=[747, 769, 780, 794, 734, 786, 789, 768, 746, 829, 752, 751, 780, 749, 752, 754, 713, 798, 777, 725, 766, 800, 781, 771, 756, 811, 824, 780, 753, 744, 774, 804, 744, 760, 775, 749, 755, 780, 789, 743, 780, 744, 780, 798, 798, 809, 766, 766, 767, 758, 750, 766, 716, 757, 715, 806, 763, 751, 745, 748, 811, 753, 793, 760] payload_rows_per_dst=[9350, 9240, 9232, 9450, 9206, 9382, 9142, 9246] dup_rows_per_dst=[2984, 2986, 3030, 3042, 2984, 3100, 2928, 3002] combine_rows_per_src=[9350, 9240, 9232, 9450, 9206, 9382, 9142, 9246] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74402 payload_rows_remote=65258 dup_rows=23902 dup_rows_remote=21004 dedup_ratio=0.243144 router_tokens_per_expert=[762, 779, 795, 799, 796, 758, 741, 795, 754, 734, 771, 715, 755, 750, 791, 751, 789, 722, 814, 772, 750, 776, 795, 768, 819, 800, 780, 777, 754, 766, 759, 767, 796, 731, 808, 766, 795, 732, 795, 758, 774, 740, 775, 735, 809, 764, 761, 734, 705, 770, 741, 746, 768, 768, 785, 755, 781, 808, 758, 770, 768, 800, 713, 789] payload_rows_per_dst=[9328, 9144, 9348, 9450, 9302, 9242, 9186, 9402] dup_rows_per_dst=[3122, 2898, 3024, 2994, 3060, 2942, 2890, 2972] combine_rows_per_src=[9328, 9144, 9348, 9450, 9302, 9242, 9186, 9402] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74314 payload_rows_remote=64888 dup_rows=23990 dup_rows_remote=20954 dedup_ratio=0.244039 router_tokens_per_expert=[768, 835, 691, 761, 779, 746, 776, 785, 760, 776, 809, 735, 754, 752, 765, 769, 779, 805, 752, 744, 816, 765, 780, 790, 738, 747, 723, 833, 745, 752, 767, 790, 763, 754, 765, 793, 728, 763, 753, 779, 806, 769, 784, 825, 770, 782, 762, 756, 774, 741, 800, 767, 734, 799, 741, 778, 775, 744, 739, 717, 755, 830, 758, 761] payload_rows_per_dst=[9286, 9388, 9426, 9212, 9256, 9342, 9208, 9196] dup_rows_per_dst=[2996, 2852, 3036, 2978, 2940, 3166, 3060, 2962] combine_rows_per_src=[9286, 9388, 9426, 9212, 9256, 9342, 9208, 9196] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74420 payload_rows_remote=65244 dup_rows=23884 dup_rows_remote=21042 dedup_ratio=0.242961 router_tokens_per_expert=[725, 757, 739, 784, 722, 788, 735, 735, 717, 737, 769, 776, 758, 786, 777, 766, 798, 741, 787, 796, 801, 711, 734, 761, 795, 755, 790, 778, 752, 764, 814, 763, 791, 756, 787, 780, 758, 735, 789, 760, 786, 718, 762, 798, 793, 813, 787, 812, 769, 741, 782, 762, 780, 779, 749, 787, 726, 765, 759, 763, 816, 803, 754, 781] payload_rows_per_dst=[9176, 9472, 9268, 9306, 9400, 9442, 9200, 9156] dup_rows_per_dst=[2842, 3198, 2996, 2900, 3014, 3042, 2996, 2896] combine_rows_per_src=[9176, 9472, 9268, 9306, 9400, 9442, 9200, 9156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74362 payload_rows_remote=64880 dup_rows=23942 dup_rows_remote=20900 dedup_ratio=0.243551 router_tokens_per_expert=[720, 798, 786, 788, 708, 764, 777, 746, 815, 817, 737, 775, 746, 776, 757, 780, 751, 778, 814, 748, 746, 820, 792, 763, 709, 812, 704, 762, 736, 757, 765, 755, 768, 719, 739, 818, 774, 760, 775, 730, 759, 794, 820, 780, 732, 738, 745, 730, 791, 787, 737, 782, 798, 786, 797, 784, 761, 780, 802, 774, 796, 762, 772, 760] payload_rows_per_dst=[9132, 9416, 9348, 9258, 9146, 9210, 9482, 9370] dup_rows_per_dst=[3042, 2990, 3076, 2742, 3020, 2986, 3042, 3044] combine_rows_per_src=[9132, 9416, 9348, 9258, 9146, 9210, 9482, 9370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74108 payload_rows_remote=64870 dup_rows=24196 dup_rows_remote=21168 dedup_ratio=0.246134 router_tokens_per_expert=[748, 786, 739, 757, 797, 823, 788, 753, 774, 763, 722, 732, 740, 792, 735, 739, 776, 789, 833, 766, 790, 684, 753, 756, 798, 773, 776, 794, 764, 748, 805, 839, 755, 798, 773, 718, 772, 770, 746, 779, 792, 777, 791, 780, 758, 756, 746, 746, 768, 775, 689, 769, 770, 758, 761, 847, 770, 784, 765, 761, 764, 756, 773, 753] payload_rows_per_dst=[9264, 9136, 8954, 9412, 9494, 9288, 9322, 9238] dup_rows_per_dst=[3118, 2858, 2694, 3182, 3374, 3004, 2938, 3028] combine_rows_per_src=[9264, 9136, 8954, 9412, 9494, 9288, 9322, 9238] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=344456 combine_payload_bytes=266412032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266756488 ingress_bytes=301015040 link_bytes=301015040 token_bytes=533168520 weight_movement_bytes=34603008 total_nvlink_bytes=567771528 weight_to_token_ratio=0.064901 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132695362e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74290 payload_rows_remote=64896 dup_rows=24014 dup_rows_remote=21000 dedup_ratio=0.244283 router_tokens_per_expert=[802, 723, 761, 769, 776, 755, 781, 714, 771, 784, 791, 793, 752, 827, 715, 800, 799, 736, 752, 782, 744, 787, 737, 756, 784, 779, 758, 768, 794, 821, 744, 745, 774, 708, 768, 773, 765, 777, 750, 756, 835, 769, 740, 740, 757, 801, 755, 822, 768, 766, 777, 781, 776, 778, 781, 777, 761, 773, 782, 799, 760, 736, 707, 740] payload_rows_per_dst=[9246, 9404, 9214, 9304, 9248, 9292, 9394, 9188] dup_rows_per_dst=[2916, 3062, 2972, 3082, 2894, 3146, 3014, 2928] combine_rows_per_src=[9246, 9404, 9214, 9304, 9248, 9292, 9394, 9188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265781248 dispatch_meta_bytes=343368 combine_payload_bytes=265781248 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266124616 ingress_bytes=334987264 link_bytes=334987264 token_bytes=531905864 weight_movement_bytes=69206016 total_nvlink_bytes=601111880 weight_to_token_ratio=0.130110 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260530106e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=267239424 dispatch_meta_bytes=345144 combine_payload_bytes=267239424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267584568 ingress_bytes=301842432 link_bytes=301842432 token_bytes=534823992 weight_movement_bytes=34603008 total_nvlink_bytes=569427000 weight_to_token_ratio=0.064700 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135808771e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74402 payload_rows_remote=65258 dup_rows=23902 dup_rows_remote=21004 dedup_ratio=0.243144 router_tokens_per_expert=[762, 779, 795, 799, 796, 758, 741, 795, 754, 734, 771, 715, 755, 750, 791, 751, 789, 722, 814, 772, 750, 776, 795, 768, 819, 800, 780, 777, 754, 766, 759, 767, 796, 731, 808, 766, 795, 732, 795, 758, 774, 740, 775, 735, 809, 764, 761, 734, 705, 770, 741, 746, 768, 768, 785, 755, 781, 808, 758, 770, 768, 800, 713, 789] payload_rows_per_dst=[9328, 9144, 9348, 9450, 9302, 9242, 9186, 9402] dup_rows_per_dst=[3122, 2898, 3024, 2994, 3060, 2942, 2890, 2972] combine_rows_per_src=[9328, 9144, 9348, 9450, 9302, 9242, 9186, 9402] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74508 payload_rows_remote=64996 dup_rows=23796 dup_rows_remote=20648 dedup_ratio=0.242065 router_tokens_per_expert=[737, 763, 767, 769, 768, 722, 767, 799, 788, 740, 803, 770, 786, 749, 725, 808, 778, 756, 802, 804, 735, 774, 767, 749, 796, 772, 766, 754, 736, 764, 766, 732, 718, 775, 803, 785, 784, 793, 769, 773, 745, 807, 781, 743, 769, 756, 807, 728, 807, 756, 784, 745, 769, 744, 751, 788, 777, 812, 755, 790, 801, 741, 741, 743] payload_rows_per_dst=[9512, 9428, 9442, 9080, 9406, 9040, 9246, 9354] dup_rows_per_dst=[3148, 3248, 2980, 2770, 2994, 2810, 2880, 2966] combine_rows_per_src=[9512, 9428, 9442, 9080, 9406, 9040, 9246, 9354] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74460 payload_rows_remote=65094 dup_rows=23844 dup_rows_remote=20776 dedup_ratio=0.242554 router_tokens_per_expert=[770, 765, 751, 740, 759, 796, 739, 762, 759, 756, 788, 754, 800, 788, 737, 801, 764, 806, 772, 786, 755, 763, 767, 763, 817, 738, 792, 786, 755, 760, 809, 760, 741, 748, 783, 755, 741, 786, 793, 759, 717, 728, 797, 681, 756, 752, 759, 781, 770, 765, 796, 799, 774, 815, 762, 776, 775, 761, 756, 765, 786, 786, 774, 757] payload_rows_per_dst=[9258, 9476, 9336, 9366, 9232, 9144, 9402, 9246] dup_rows_per_dst=[2906, 2890, 3016, 3068, 2980, 2798, 3112, 3074] combine_rows_per_src=[9258, 9476, 9336, 9366, 9232, 9144, 9402, 9246] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265707520 dispatch_meta_bytes=344152 combine_payload_bytes=265707520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266051672 ingress_bytes=300310528 link_bytes=300310528 token_bytes=531759192 weight_movement_bytes=34603008 total_nvlink_bytes=566362200 weight_to_token_ratio=0.065073 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130044340e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74298 payload_rows_remote=65038 dup_rows=24006 dup_rows_remote=21044 dedup_ratio=0.244202 router_tokens_per_expert=[752, 745, 789, 771, 806, 746, 727, 780, 771, 787, 775, 755, 739, 727, 777, 757, 794, 738, 791, 791, 780, 779, 832, 719, 768, 776, 810, 794, 755, 722, 741, 721, 789, 761, 731, 788, 767, 762, 774, 761, 775, 785, 746, 745, 709, 772, 804, 775, 782, 791, 805, 749, 759, 746, 749, 768, 798, 817, 798, 763, 791, 774, 720, 783] payload_rows_per_dst=[9254, 9286, 9434, 9142, 9196, 9260, 9272, 9454] dup_rows_per_dst=[2978, 2890, 3014, 3032, 3070, 2962, 3026, 3034] combine_rows_per_src=[9254, 9286, 9434, 9142, 9196, 9260, 9272, 9454] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265814016 dispatch_meta_bytes=343584 combine_payload_bytes=265814016 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266157600 ingress_bytes=335020032 link_bytes=335020032 token_bytes=531971616 weight_movement_bytes=69206016 total_nvlink_bytes=601177632 weight_to_token_ratio=0.130093 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260653409e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=267296768 dispatch_meta_bytes=345048 combine_payload_bytes=267296768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267641816 ingress_bytes=301899776 link_bytes=301899776 token_bytes=534938584 weight_movement_bytes=34603008 total_nvlink_bytes=569541592 weight_to_token_ratio=0.064686 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136024552e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265781248 dispatch_meta_bytes=343368 combine_payload_bytes=265781248 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266124616 ingress_bytes=334987264 link_bytes=334987264 token_bytes=531905864 weight_movement_bytes=69206016 total_nvlink_bytes=601111880 weight_to_token_ratio=0.130110 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260530106e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265748480 dispatch_meta_bytes=343120 combine_payload_bytes=265748480 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266091600 ingress_bytes=404160512 link_bytes=404160512 token_bytes=531840080 weight_movement_bytes=138412032 total_nvlink_bytes=670252112 weight_to_token_ratio=0.260251 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520823469e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74216 payload_rows_remote=64876 dup_rows=24088 dup_rows_remote=21048 dedup_ratio=0.245036 router_tokens_per_expert=[776, 809, 785, 820, 737, 710, 777, 756, 751, 799, 789, 759, 779, 771, 781, 802, 808, 745, 757, 811, 732, 770, 832, 746, 703, 755, 793, 746, 778, 764, 808, 746, 761, 735, 770, 754, 745, 791, 780, 782, 773, 765, 784, 768, 769, 758, 773, 706, 795, 774, 774, 704, 738, 744, 779, 758, 755, 734, 766, 780, 785, 790, 773, 794] payload_rows_per_dst=[9092, 9538, 9284, 9068, 9360, 9250, 9284, 9340] dup_rows_per_dst=[2742, 3292, 2866, 2932, 3062, 2942, 3212, 3040] combine_rows_per_src=[9092, 9538, 9284, 9068, 9360, 9250, 9284, 9340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74298 payload_rows_remote=65038 dup_rows=24006 dup_rows_remote=21044 dedup_ratio=0.244202 router_tokens_per_expert=[752, 745, 789, 771, 806, 746, 727, 780, 771, 787, 775, 755, 739, 727, 777, 757, 794, 738, 791, 791, 780, 779, 832, 719, 768, 776, 810, 794, 755, 722, 741, 721, 789, 761, 731, 788, 767, 762, 774, 761, 775, 785, 746, 745, 709, 772, 804, 775, 782, 791, 805, 749, 759, 746, 749, 768, 798, 817, 798, 763, 791, 774, 720, 783] payload_rows_per_dst=[9254, 9286, 9434, 9142, 9196, 9260, 9272, 9454] dup_rows_per_dst=[2978, 2890, 3014, 3032, 3070, 2962, 3026, 3034] combine_rows_per_src=[9254, 9286, 9434, 9142, 9196, 9260, 9272, 9454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=342576 combine_payload_bytes=266223616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266566192 ingress_bytes=300826624 link_bytes=300826624 token_bytes=532789808 weight_movement_bytes=34603008 total_nvlink_bytes=567392816 weight_to_token_ratio=0.064947 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131986368e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266625024 dispatch_meta_bytes=343480 combine_payload_bytes=266625024 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266968504 ingress_bytes=335831040 link_bytes=335831040 token_bytes=533593528 weight_movement_bytes=69206016 total_nvlink_bytes=602799544 weight_to_token_ratio=0.129698 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263705167e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74436 payload_rows_remote=65330 dup_rows=23868 dup_rows_remote=20816 dedup_ratio=0.242798 router_tokens_per_expert=[785, 740, 762, 814, 761, 708, 722, 776, 773, 789, 787, 751, 777, 790, 750, 764, 778, 795, 803, 754, 770, 744, 757, 762, 807, 760, 709, 768, 815, 758, 807, 759, 767, 756, 759, 743, 738, 766, 763, 787, 721, 761, 772, 798, 771, 760, 733, 791, 730, 787, 818, 833, 782, 805, 772, 780, 725, 745, 784, 764, 752, 746, 789, 759] payload_rows_per_dst=[9236, 9366, 9352, 9368, 9106, 9212, 9532, 9264] dup_rows_per_dst=[2900, 2996, 2974, 2998, 3052, 3002, 3082, 2864] combine_rows_per_src=[9236, 9366, 9352, 9368, 9106, 9212, 9532, 9264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265732096 dispatch_meta_bytes=343696 combine_payload_bytes=265732096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266075792 ingress_bytes=300335104 link_bytes=300335104 token_bytes=531807888 weight_movement_bytes=34603008 total_nvlink_bytes=566410896 weight_to_token_ratio=0.065067 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130136817e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=344328 combine_payload_bytes=266395648 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266739976 ingress_bytes=335601664 link_bytes=335601664 token_bytes=533135624 weight_movement_bytes=69206016 total_nvlink_bytes=602341640 weight_to_token_ratio=0.129809 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262842043e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74436 payload_rows_remote=65330 dup_rows=23868 dup_rows_remote=20816 dedup_ratio=0.242798 router_tokens_per_expert=[785, 740, 762, 814, 761, 708, 722, 776, 773, 789, 787, 751, 777, 790, 750, 764, 778, 795, 803, 754, 770, 744, 757, 762, 807, 760, 709, 768, 815, 758, 807, 759, 767, 756, 759, 743, 738, 766, 763, 787, 721, 761, 772, 798, 771, 760, 733, 791, 730, 787, 818, 833, 782, 805, 772, 780, 725, 745, 784, 764, 752, 746, 789, 759] payload_rows_per_dst=[9236, 9366, 9352, 9368, 9106, 9212, 9532, 9264] dup_rows_per_dst=[2900, 2996, 2974, 2998, 3052, 3002, 3082, 2864] combine_rows_per_src=[9236, 9366, 9352, 9368, 9106, 9212, 9532, 9264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74434 payload_rows_remote=65136 dup_rows=23870 dup_rows_remote=20900 dedup_ratio=0.242818 router_tokens_per_expert=[748, 750, 762, 709, 760, 753, 769, 772, 783, 836, 756, 705, 787, 734, 762, 771, 771, 772, 736, 751, 766, 749, 761, 834, 749, 771, 767, 720, 796, 786, 762, 822, 750, 706, 762, 810, 759, 776, 786, 787, 770, 796, 760, 765, 744, 788, 754, 761, 795, 749, 810, 823, 796, 736, 809, 768, 751, 771, 806, 785, 721, 756, 768, 764] payload_rows_per_dst=[9190, 9298, 9302, 9382, 9308, 9256, 9464, 9234] dup_rows_per_dst=[2856, 2970, 2978, 2964, 2964, 3020, 3108, 3010] combine_rows_per_src=[9190, 9298, 9302, 9382, 9308, 9256, 9464, 9234] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=65018 dup_rows=24020 dup_rows_remote=21124 dedup_ratio=0.244344 router_tokens_per_expert=[787, 738, 803, 795, 764, 757, 822, 767, 788, 778, 693, 773, 777, 746, 738, 762, 722, 783, 756, 760, 800, 740, 756, 764, 764, 779, 767, 749, 783, 787, 776, 780, 773, 739, 757, 752, 787, 760, 767, 710, 794, 800, 838, 743, 770, 818, 791, 775, 736, 751, 765, 800, 736, 763, 772, 757, 781, 788, 797, 726, 732, 780, 760, 780] payload_rows_per_dst=[9420, 9158, 9266, 9352, 9164, 9486, 9190, 9248] dup_rows_per_dst=[3046, 2952, 2896, 3018, 2926, 3172, 2970, 3040] combine_rows_per_src=[9420, 9158, 9266, 9352, 9164, 9486, 9190, 9248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74508 payload_rows_remote=64996 dup_rows=23796 dup_rows_remote=20648 dedup_ratio=0.242065 router_tokens_per_expert=[737, 763, 767, 769, 768, 722, 767, 799, 788, 740, 803, 770, 786, 749, 725, 808, 778, 756, 802, 804, 735, 774, 767, 749, 796, 772, 766, 754, 736, 764, 766, 732, 718, 775, 803, 785, 784, 793, 769, 773, 745, 807, 781, 743, 769, 756, 807, 728, 807, 756, 784, 745, 769, 744, 751, 788, 777, 812, 755, 790, 801, 741, 741, 743] payload_rows_per_dst=[9512, 9428, 9442, 9080, 9406, 9040, 9246, 9354] dup_rows_per_dst=[3148, 3248, 2980, 2770, 2994, 2810, 2880, 2966] combine_rows_per_src=[9512, 9428, 9442, 9080, 9406, 9040, 9246, 9354] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=267591680 dispatch_meta_bytes=344584 combine_payload_bytes=267591680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267936264 ingress_bytes=302194688 link_bytes=302194688 token_bytes=535527944 weight_movement_bytes=34603008 total_nvlink_bytes=570130952 weight_to_token_ratio=0.064615 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137134282e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74216 payload_rows_remote=64876 dup_rows=24088 dup_rows_remote=21048 dedup_ratio=0.245036 router_tokens_per_expert=[776, 809, 785, 820, 737, 710, 777, 756, 751, 799, 789, 759, 779, 771, 781, 802, 808, 745, 757, 811, 732, 770, 832, 746, 703, 755, 793, 746, 778, 764, 808, 746, 761, 735, 770, 754, 745, 791, 780, 782, 773, 765, 784, 768, 769, 758, 773, 706, 795, 774, 774, 704, 738, 744, 779, 758, 755, 734, 766, 780, 785, 790, 773, 794] payload_rows_per_dst=[9092, 9538, 9284, 9068, 9360, 9250, 9284, 9340] dup_rows_per_dst=[2742, 3292, 2866, 2932, 3062, 2942, 3212, 3040] combine_rows_per_src=[9092, 9538, 9284, 9068, 9360, 9250, 9284, 9340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=65088 dup_rows=24008 dup_rows_remote=20982 dedup_ratio=0.244222 router_tokens_per_expert=[769, 812, 777, 751, 785, 810, 784, 776, 741, 738, 789, 773, 777, 761, 783, 727, 796, 729, 762, 759, 784, 751, 790, 803, 744, 784, 739, 835, 838, 753, 772, 758, 809, 743, 785, 774, 787, 782, 752, 773, 739, 778, 737, 753, 750, 809, 791, 729, 739, 777, 782, 740, 770, 732, 803, 774, 737, 766, 752, 752, 714, 745, 740, 788] payload_rows_per_dst=[9454, 9190, 9356, 9486, 9360, 9144, 9208, 9098] dup_rows_per_dst=[3074, 2988, 2992, 2960, 3050, 3028, 3026, 2890] combine_rows_per_src=[9454, 9190, 9356, 9486, 9360, 9144, 9208, 9098] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74362 payload_rows_remote=64880 dup_rows=23942 dup_rows_remote=20900 dedup_ratio=0.243551 router_tokens_per_expert=[720, 798, 786, 788, 708, 764, 777, 746, 815, 817, 737, 775, 746, 776, 757, 780, 751, 778, 814, 748, 746, 820, 792, 763, 709, 812, 704, 762, 736, 757, 765, 755, 768, 719, 739, 818, 774, 760, 775, 730, 759, 794, 820, 780, 732, 738, 745, 730, 791, 787, 737, 782, 798, 786, 797, 784, 761, 780, 802, 774, 796, 762, 772, 760] payload_rows_per_dst=[9132, 9416, 9348, 9258, 9146, 9210, 9482, 9370] dup_rows_per_dst=[3042, 2990, 3076, 2742, 3020, 2986, 3042, 3044] combine_rows_per_src=[9132, 9416, 9348, 9258, 9146, 9210, 9482, 9370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=344568 combine_payload_bytes=266313728 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266658296 ingress_bytes=335519744 link_bytes=335519744 token_bytes=532972024 weight_movement_bytes=69206016 total_nvlink_bytes=602178040 weight_to_token_ratio=0.129849 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262533785e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74434 payload_rows_remote=65136 dup_rows=23870 dup_rows_remote=20900 dedup_ratio=0.242818 router_tokens_per_expert=[748, 750, 762, 709, 760, 753, 769, 772, 783, 836, 756, 705, 787, 734, 762, 771, 771, 772, 736, 751, 766, 749, 761, 834, 749, 771, 767, 720, 796, 786, 762, 822, 750, 706, 762, 810, 759, 776, 786, 787, 770, 796, 760, 765, 744, 788, 754, 761, 795, 749, 810, 823, 796, 736, 809, 768, 751, 771, 806, 785, 721, 756, 768, 764] payload_rows_per_dst=[9190, 9298, 9302, 9382, 9308, 9256, 9464, 9234] dup_rows_per_dst=[2856, 2970, 2978, 2964, 2964, 3020, 3108, 3010] combine_rows_per_src=[9190, 9298, 9302, 9382, 9308, 9256, 9464, 9234] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=342576 combine_payload_bytes=266223616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266566192 ingress_bytes=300826624 link_bytes=300826624 token_bytes=532789808 weight_movement_bytes=34603008 total_nvlink_bytes=567392816 weight_to_token_ratio=0.064947 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131986368e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74202 payload_rows_remote=64898 dup_rows=24102 dup_rows_remote=21000 dedup_ratio=0.245178 router_tokens_per_expert=[746, 714, 792, 762, 798, 776, 819, 794, 786, 728, 809, 782, 765, 782, 768, 771, 734, 781, 727, 740, 779, 767, 759, 754, 742, 761, 727, 746, 784, 765, 780, 763, 740, 769, 759, 746, 769, 747, 776, 758, 822, 803, 793, 766, 799, 767, 773, 775, 745, 794, 770, 776, 770, 770, 761, 741, 785, 806, 783, 740, 785, 760, 748, 755] payload_rows_per_dst=[9304, 9362, 9152, 9200, 9202, 9372, 9262, 9348] dup_rows_per_dst=[3102, 3020, 2930, 2976, 2926, 3180, 2992, 2976] combine_rows_per_src=[9304, 9362, 9152, 9200, 9202, 9372, 9262, 9348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=267591680 dispatch_meta_bytes=344584 combine_payload_bytes=267591680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267936264 ingress_bytes=302194688 link_bytes=302194688 token_bytes=535527944 weight_movement_bytes=34603008 total_nvlink_bytes=570130952 weight_to_token_ratio=0.064615 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137134282e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74400 payload_rows_remote=65036 dup_rows=23904 dup_rows_remote=20940 dedup_ratio=0.243164 router_tokens_per_expert=[781, 742, 797, 790, 782, 765, 743, 726, 757, 760, 768, 730, 784, 727, 760, 826, 748, 732, 809, 795, 737, 800, 822, 765, 765, 830, 716, 746, 808, 739, 794, 766, 779, 739, 818, 765, 785, 785, 794, 769, 708, 739, 778, 739, 799, 752, 786, 754, 768, 777, 778, 734, 740, 727, 776, 786, 758, 766, 785, 722, 764, 792, 802, 778] payload_rows_per_dst=[9300, 9262, 9396, 9364, 9404, 9176, 9190, 9308] dup_rows_per_dst=[2952, 2962, 3020, 2964, 3064, 2934, 2982, 3026] combine_rows_per_src=[9300, 9262, 9396, 9364, 9404, 9176, 9190, 9308] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266797056 dispatch_meta_bytes=344144 combine_payload_bytes=266797056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267141200 ingress_bytes=301400064 link_bytes=301400064 token_bytes=533938256 weight_movement_bytes=34603008 total_nvlink_bytes=568541264 weight_to_token_ratio=0.064807 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134144176e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=344568 combine_payload_bytes=266313728 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266658296 ingress_bytes=335519744 link_bytes=335519744 token_bytes=532972024 weight_movement_bytes=69206016 total_nvlink_bytes=602178040 weight_to_token_ratio=0.129849 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262533785e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=65136 dup_rows=23860 dup_rows_remote=20836 dedup_ratio=0.242716 router_tokens_per_expert=[743, 760, 775, 750, 720, 701, 808, 768, 750, 754, 803, 780, 742, 837, 762, 779, 766, 778, 790, 780, 756, 768, 782, 766, 834, 750, 750, 786, 761, 737, 783, 731, 757, 766, 811, 735, 754, 794, 766, 772, 801, 750, 793, 782, 784, 731, 763, 762, 733, 803, 752, 754, 787, 812, 800, 713, 751, 783, 768, 771, 779, 773, 772, 730] payload_rows_per_dst=[9156, 9364, 9358, 9398, 9282, 9308, 9340, 9238] dup_rows_per_dst=[2894, 3050, 3014, 2866, 3028, 3024, 2968, 3016] combine_rows_per_src=[9156, 9364, 9358, 9398, 9282, 9308, 9340, 9238] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265732096 dispatch_meta_bytes=343696 combine_payload_bytes=265732096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266075792 ingress_bytes=300335104 link_bytes=300335104 token_bytes=531807888 weight_movement_bytes=34603008 total_nvlink_bytes=566410896 weight_to_token_ratio=0.065067 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130136817e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265748480 dispatch_meta_bytes=343120 combine_payload_bytes=265748480 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266091600 ingress_bytes=404160512 link_bytes=404160512 token_bytes=531840080 weight_movement_bytes=138412032 total_nvlink_bytes=670252112 weight_to_token_ratio=0.260251 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520823469e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=64892 dup_rows=24012 dup_rows_remote=20860 dedup_ratio=0.244263 router_tokens_per_expert=[739, 786, 772, 792, 768, 748, 766, 774, 788, 750, 756, 742, 785, 792, 717, 772, 754, 745, 757, 817, 791, 726, 800, 763, 744, 799, 777, 771, 705, 743, 776, 780, 747, 811, 781, 760, 767, 789, 817, 732, 774, 720, 745, 737, 783, 750, 784, 760, 822, 804, 759, 785, 749, 756, 748, 783, 817, 744, 768, 742, 794, 795, 806, 728] payload_rows_per_dst=[9270, 9256, 9042, 9144, 9472, 9328, 9380, 9400] dup_rows_per_dst=[3020, 2948, 2894, 2874, 3178, 2900, 3046, 3152] combine_rows_per_src=[9270, 9256, 9042, 9144, 9472, 9328, 9380, 9400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=344280 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266944728 ingress_bytes=301203456 link_bytes=301203456 token_bytes=533545176 weight_movement_bytes=34603008 total_nvlink_bytes=568148184 weight_to_token_ratio=0.064855 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133404356e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266797056 dispatch_meta_bytes=344144 combine_payload_bytes=266797056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267141200 ingress_bytes=301400064 link_bytes=301400064 token_bytes=533938256 weight_movement_bytes=34603008 total_nvlink_bytes=568541264 weight_to_token_ratio=0.064807 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134144176e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74106 payload_rows_remote=64886 dup_rows=24198 dup_rows_remote=21110 dedup_ratio=0.246155 router_tokens_per_expert=[747, 778, 836, 774, 774, 767, 753, 818, 771, 760, 731, 769, 786, 705, 721, 764, 762, 766, 774, 777, 830, 770, 777, 723, 807, 753, 782, 781, 749, 783, 753, 758, 780, 751, 792, 750, 800, 793, 763, 725, 806, 772, 742, 737, 781, 756, 794, 753, 775, 735, 785, 768, 769, 729, 830, 755, 743, 777, 768, 801, 740, 794, 726, 763] payload_rows_per_dst=[9334, 9220, 9300, 9310, 9220, 9208, 9312, 9202] dup_rows_per_dst=[3160, 2794, 3058, 3022, 3088, 3074, 2980, 3022] combine_rows_per_src=[9334, 9220, 9300, 9310, 9220, 9208, 9312, 9202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265822208 dispatch_meta_bytes=343592 combine_payload_bytes=265822208 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266165800 ingress_bytes=404234240 link_bytes=404234240 token_bytes=531988008 weight_movement_bytes=138412032 total_nvlink_bytes=670400040 weight_to_token_ratio=0.260179 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521100902e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266387456 dispatch_meta_bytes=343904 combine_payload_bytes=266387456 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266731360 ingress_bytes=335593472 link_bytes=335593472 token_bytes=533118816 weight_movement_bytes=69206016 total_nvlink_bytes=602324832 weight_to_token_ratio=0.129813 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262811218e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265797632 dispatch_meta_bytes=343008 combine_payload_bytes=265797632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266140640 ingress_bytes=300400640 link_bytes=300400640 token_bytes=531938272 weight_movement_bytes=34603008 total_nvlink_bytes=566541280 weight_to_token_ratio=0.065051 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130383424e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266797056 dispatch_meta_bytes=343888 combine_payload_bytes=266797056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267140944 ingress_bytes=301400064 link_bytes=301400064 token_bytes=533938000 weight_movement_bytes=34603008 total_nvlink_bytes=568541008 weight_to_token_ratio=0.064807 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134144176e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74106 payload_rows_remote=64886 dup_rows=24198 dup_rows_remote=21110 dedup_ratio=0.246155 router_tokens_per_expert=[747, 778, 836, 774, 774, 767, 753, 818, 771, 760, 731, 769, 786, 705, 721, 764, 762, 766, 774, 777, 830, 770, 777, 723, 807, 753, 782, 781, 749, 783, 753, 758, 780, 751, 792, 750, 800, 793, 763, 725, 806, 772, 742, 737, 781, 756, 794, 753, 775, 735, 785, 768, 769, 729, 830, 755, 743, 777, 768, 801, 740, 794, 726, 763] payload_rows_per_dst=[9334, 9220, 9300, 9310, 9220, 9208, 9312, 9202] dup_rows_per_dst=[3160, 2794, 3058, 3022, 3088, 3074, 2980, 3022] combine_rows_per_src=[9334, 9220, 9300, 9310, 9220, 9208, 9312, 9202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74122 payload_rows_remote=64686 dup_rows=24182 dup_rows_remote=21154 dedup_ratio=0.245992 router_tokens_per_expert=[788, 726, 761, 794, 767, 781, 772, 787, 775, 789, 792, 753, 847, 813, 740, 723, 743, 797, 749, 733, 760, 790, 759, 766, 779, 748, 754, 753, 741, 714, 862, 754, 762, 762, 780, 791, 813, 800, 816, 742, 728, 751, 781, 723, 739, 774, 772, 770, 766, 777, 770, 762, 801, 707, 775, 734, 791, 787, 765, 751, 771, 755, 776, 750] payload_rows_per_dst=[9368, 9436, 9202, 9136, 9394, 9106, 9194, 9286] dup_rows_per_dst=[2984, 3028, 2992, 3074, 3138, 2970, 2990, 3006] combine_rows_per_src=[9368, 9436, 9202, 9136, 9394, 9106, 9194, 9286] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74424 payload_rows_remote=65082 dup_rows=23880 dup_rows_remote=20892 dedup_ratio=0.242920 router_tokens_per_expert=[776, 752, 799, 806, 733, 756, 777, 751, 748, 777, 729, 735, 750, 791, 754, 799, 770, 745, 761, 796, 775, 747, 806, 765, 793, 750, 787, 774, 715, 764, 806, 794, 786, 742, 807, 776, 737, 778, 745, 779, 727, 754, 744, 767, 778, 753, 782, 754, 767, 771, 786, 768, 763, 712, 793, 769, 796, 729, 837, 769, 782, 813, 709, 798] payload_rows_per_dst=[9290, 9232, 9342, 9320, 9310, 9190, 9260, 9480] dup_rows_per_dst=[3010, 2934, 2988, 3046, 2990, 2928, 2998, 2986] combine_rows_per_src=[9290, 9232, 9342, 9320, 9310, 9190, 9260, 9480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74046 payload_rows_remote=64674 dup_rows=24258 dup_rows_remote=21084 dedup_ratio=0.246765 router_tokens_per_expert=[779, 765, 752, 719, 758, 827, 767, 760, 772, 760, 845, 743, 792, 747, 765, 763, 778, 739, 754, 760, 777, 755, 782, 780, 712, 758, 779, 767, 823, 739, 801, 797, 769, 753, 765, 736, 744, 762, 766, 783, 779, 802, 809, 801, 780, 708, 720, 771, 787, 783, 760, 820, 804, 788, 755, 776, 760, 721, 733, 781, 741, 747, 776, 757] payload_rows_per_dst=[9246, 9240, 9254, 9236, 9206, 9320, 9372, 9172] dup_rows_per_dst=[3008, 3134, 2996, 3116, 2950, 3020, 3174, 2860] combine_rows_per_src=[9246, 9240, 9254, 9236, 9206, 9320, 9372, 9172] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=343984 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266117040 ingress_bytes=300376064 link_bytes=300376064 token_bytes=531890096 weight_movement_bytes=34603008 total_nvlink_bytes=566493104 weight_to_token_ratio=0.065057 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130290947e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74202 payload_rows_remote=64898 dup_rows=24102 dup_rows_remote=21000 dedup_ratio=0.245178 router_tokens_per_expert=[746, 714, 792, 762, 798, 776, 819, 794, 786, 728, 809, 782, 765, 782, 768, 771, 734, 781, 727, 740, 779, 767, 759, 754, 742, 761, 727, 746, 784, 765, 780, 763, 740, 769, 759, 746, 769, 747, 776, 758, 822, 803, 793, 766, 799, 767, 773, 775, 745, 794, 770, 776, 770, 770, 761, 741, 785, 806, 783, 740, 785, 760, 748, 755] payload_rows_per_dst=[9304, 9362, 9152, 9200, 9202, 9372, 9262, 9348] dup_rows_per_dst=[3102, 3020, 2930, 2976, 2926, 3180, 2992, 2976] combine_rows_per_src=[9304, 9362, 9152, 9200, 9202, 9372, 9262, 9348] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=64892 dup_rows=24012 dup_rows_remote=20860 dedup_ratio=0.244263 router_tokens_per_expert=[739, 786, 772, 792, 768, 748, 766, 774, 788, 750, 756, 742, 785, 792, 717, 772, 754, 745, 757, 817, 791, 726, 800, 763, 744, 799, 777, 771, 705, 743, 776, 780, 747, 811, 781, 760, 767, 789, 817, 732, 774, 720, 745, 737, 783, 750, 784, 760, 822, 804, 759, 785, 749, 756, 748, 783, 817, 744, 768, 742, 794, 795, 806, 728] payload_rows_per_dst=[9270, 9256, 9042, 9144, 9472, 9328, 9380, 9400] dup_rows_per_dst=[3020, 2948, 2894, 2874, 3178, 2900, 3046, 3152] combine_rows_per_src=[9270, 9256, 9042, 9144, 9472, 9328, 9380, 9400] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=65088 dup_rows=24008 dup_rows_remote=20982 dedup_ratio=0.244222 router_tokens_per_expert=[769, 812, 777, 751, 785, 810, 784, 776, 741, 738, 789, 773, 777, 761, 783, 727, 796, 729, 762, 759, 784, 751, 790, 803, 744, 784, 739, 835, 838, 753, 772, 758, 809, 743, 785, 774, 787, 782, 752, 773, 739, 778, 737, 753, 750, 809, 791, 729, 739, 777, 782, 740, 770, 732, 803, 774, 737, 766, 752, 752, 714, 745, 740, 788] payload_rows_per_dst=[9454, 9190, 9356, 9486, 9360, 9144, 9208, 9098] dup_rows_per_dst=[3074, 2988, 2992, 2960, 3050, 3028, 3026, 2890] combine_rows_per_src=[9454, 9190, 9356, 9486, 9360, 9144, 9208, 9098] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74122 payload_rows_remote=64686 dup_rows=24182 dup_rows_remote=21154 dedup_ratio=0.245992 router_tokens_per_expert=[788, 726, 761, 794, 767, 781, 772, 787, 775, 789, 792, 753, 847, 813, 740, 723, 743, 797, 749, 733, 760, 790, 759, 766, 779, 748, 754, 753, 741, 714, 862, 754, 762, 762, 780, 791, 813, 800, 816, 742, 728, 751, 781, 723, 739, 774, 772, 770, 766, 777, 770, 762, 801, 707, 775, 734, 791, 787, 765, 751, 771, 755, 776, 750] payload_rows_per_dst=[9368, 9436, 9202, 9136, 9394, 9106, 9194, 9286] dup_rows_per_dst=[2984, 3028, 2992, 3074, 3138, 2970, 2990, 3006] combine_rows_per_src=[9368, 9436, 9202, 9136, 9394, 9106, 9194, 9286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=264953856 dispatch_meta_bytes=343360 combine_payload_bytes=264953856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265297216 ingress_bytes=299556864 link_bytes=299556864 token_bytes=530251072 weight_movement_bytes=34603008 total_nvlink_bytes=564854080 weight_to_token_ratio=0.065258 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127208363e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74470 payload_rows_remote=65192 dup_rows=23834 dup_rows_remote=20628 dedup_ratio=0.242452 router_tokens_per_expert=[743, 719, 764, 770, 744, 769, 795, 775, 797, 787, 781, 774, 748, 797, 796, 744, 811, 803, 726, 796, 768, 797, 787, 785, 755, 773, 833, 750, 762, 721, 778, 759, 747, 745, 763, 742, 765, 795, 755, 748, 752, 778, 783, 751, 743, 822, 760, 777, 792, 818, 748, 773, 737, 815, 731, 745, 742, 781, 777, 717, 781, 801, 729, 732] payload_rows_per_dst=[9278, 9532, 9476, 9350, 9188, 9238, 9116, 9292] dup_rows_per_dst=[3206, 3014, 3070, 2912, 2932, 3094, 2778, 2828] combine_rows_per_src=[9278, 9532, 9476, 9350, 9188, 9238, 9116, 9292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74348 payload_rows_remote=64986 dup_rows=23956 dup_rows_remote=20954 dedup_ratio=0.243693 router_tokens_per_expert=[778, 796, 771, 754, 804, 748, 769, 773, 804, 768, 743, 740, 793, 799, 759, 753, 754, 766, 783, 774, 710, 786, 774, 769, 801, 769, 739, 774, 784, 788, 732, 795, 770, 728, 761, 731, 769, 776, 731, 828, 775, 795, 779, 815, 741, 782, 719, 792, 768, 719, 745, 754, 726, 733, 808, 751, 759, 767, 760, 785, 812, 785, 799, 739] payload_rows_per_dst=[9246, 9294, 9274, 9362, 9230, 9354, 9172, 9416] dup_rows_per_dst=[3140, 3024, 2958, 3002, 2958, 3042, 2836, 2996] combine_rows_per_src=[9246, 9294, 9274, 9362, 9230, 9354, 9172, 9416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266575872 dispatch_meta_bytes=343896 combine_payload_bytes=266575872 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266919768 ingress_bytes=370384896 link_bytes=370384896 token_bytes=533495640 weight_movement_bytes=103809024 total_nvlink_bytes=637304664 weight_to_token_ratio=0.194583 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393728545e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74352 payload_rows_remote=65144 dup_rows=23952 dup_rows_remote=20922 dedup_ratio=0.243652 router_tokens_per_expert=[754, 742, 777, 790, 767, 797, 780, 787, 786, 768, 755, 769, 742, 770, 719, 776, 819, 768, 797, 761, 735, 742, 789, 732, 771, 803, 732, 805, 800, 805, 778, 746, 747, 747, 725, 762, 758, 760, 764, 781, 778, 777, 719, 773, 771, 767, 805, 749, 802, 737, 773, 792, 773, 826, 764, 777, 787, 723, 764, 741, 709, 794, 738, 807] payload_rows_per_dst=[9116, 9218, 9240, 9314, 9214, 9476, 9566, 9208] dup_rows_per_dst=[2920, 2952, 2934, 2840, 2874, 3154, 3248, 3030] combine_rows_per_src=[9116, 9218, 9240, 9314, 9214, 9476, 9566, 9208] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=65016 dup_rows=23982 dup_rows_remote=21004 dedup_ratio=0.243958 router_tokens_per_expert=[776, 777, 763, 781, 745, 769, 774, 739, 757, 719, 738, 738, 772, 780, 765, 809, 828, 793, 768, 785, 781, 744, 751, 762, 798, 786, 712, 820, 786, 725, 767, 758, 792, 762, 772, 718, 807, 736, 767, 770, 785, 804, 776, 724, 780, 744, 782, 747, 780, 796, 789, 764, 720, 770, 750, 746, 786, 772, 812, 766, 821, 744, 736, 768] payload_rows_per_dst=[9206, 9224, 9462, 9274, 9260, 9306, 9234, 9356] dup_rows_per_dst=[3042, 2932, 2962, 3030, 2988, 2978, 2996, 3054] combine_rows_per_src=[9206, 9224, 9462, 9274, 9260, 9306, 9234, 9356] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265797632 dispatch_meta_bytes=343008 combine_payload_bytes=265797632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266140640 ingress_bytes=300400640 link_bytes=300400640 token_bytes=531938272 weight_movement_bytes=34603008 total_nvlink_bytes=566541280 weight_to_token_ratio=0.065051 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130383424e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265822208 dispatch_meta_bytes=343592 combine_payload_bytes=265822208 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266165800 ingress_bytes=404234240 link_bytes=404234240 token_bytes=531988008 weight_movement_bytes=138412032 total_nvlink_bytes=670400040 weight_to_token_ratio=0.260179 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521100902e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74348 payload_rows_remote=64986 dup_rows=23956 dup_rows_remote=20954 dedup_ratio=0.243693 router_tokens_per_expert=[778, 796, 771, 754, 804, 748, 769, 773, 804, 768, 743, 740, 793, 799, 759, 753, 754, 766, 783, 774, 710, 786, 774, 769, 801, 769, 739, 774, 784, 788, 732, 795, 770, 728, 761, 731, 769, 776, 731, 828, 775, 795, 779, 815, 741, 782, 719, 792, 768, 719, 745, 754, 726, 733, 808, 751, 759, 767, 760, 785, 812, 785, 799, 739] payload_rows_per_dst=[9246, 9294, 9274, 9362, 9230, 9354, 9172, 9416] dup_rows_per_dst=[3140, 3024, 2958, 3002, 2958, 3042, 2836, 2996] combine_rows_per_src=[9246, 9294, 9274, 9362, 9230, 9354, 9172, 9416] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=344280 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266944728 ingress_bytes=301203456 link_bytes=301203456 token_bytes=533545176 weight_movement_bytes=34603008 total_nvlink_bytes=568148184 weight_to_token_ratio=0.064855 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133404356e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=264953856 dispatch_meta_bytes=343360 combine_payload_bytes=264953856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265297216 ingress_bytes=299556864 link_bytes=299556864 token_bytes=530251072 weight_movement_bytes=34603008 total_nvlink_bytes=564854080 weight_to_token_ratio=0.065258 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127208363e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74074 payload_rows_remote=64844 dup_rows=24230 dup_rows_remote=21144 dedup_ratio=0.246480 router_tokens_per_expert=[763, 768, 799, 815, 781, 777, 819, 771, 788, 760, 813, 751, 741, 765, 763, 779, 746, 738, 776, 723, 799, 809, 716, 779, 764, 800, 793, 806, 818, 757, 722, 754, 725, 803, 805, 747, 777, 720, 798, 783, 739, 766, 714, 808, 721, 723, 775, 760, 744, 752, 802, 813, 768, 754, 731, 814, 770, 751, 767, 706, 710, 806, 814, 733] payload_rows_per_dst=[9428, 9264, 9204, 9326, 9230, 9168, 9284, 9170] dup_rows_per_dst=[3158, 3056, 2968, 3102, 3086, 2844, 3072, 2944] combine_rows_per_src=[9428, 9264, 9204, 9326, 9230, 9168, 9284, 9170] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=344080 combine_payload_bytes=266305536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266649616 ingress_bytes=300908544 link_bytes=300908544 token_bytes=532955152 weight_movement_bytes=34603008 total_nvlink_bytes=567558160 weight_to_token_ratio=0.064927 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132294626e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=264904704 dispatch_meta_bytes=343032 combine_payload_bytes=264904704 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265247736 ingress_bytes=368713728 link_bytes=368713728 token_bytes=530152440 weight_movement_bytes=103809024 total_nvlink_bytes=633961464 weight_to_token_ratio=0.195810 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.387440075e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=267026432 dispatch_meta_bytes=343280 combine_payload_bytes=267026432 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267369712 ingress_bytes=301629440 link_bytes=301629440 token_bytes=534396144 weight_movement_bytes=34603008 total_nvlink_bytes=568999152 weight_to_token_ratio=0.064752 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135007300e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266182656 dispatch_meta_bytes=343760 combine_payload_bytes=266182656 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266526416 ingress_bytes=369991680 link_bytes=369991680 token_bytes=532709072 weight_movement_bytes=103809024 total_nvlink_bytes=636518096 weight_to_token_ratio=0.194870 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392248905e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266829824 dispatch_meta_bytes=344264 combine_payload_bytes=266829824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267174088 ingress_bytes=301432832 link_bytes=301432832 token_bytes=534003912 weight_movement_bytes=34603008 total_nvlink_bytes=568606920 weight_to_token_ratio=0.064799 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134267479e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=344080 combine_payload_bytes=266305536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266649616 ingress_bytes=300908544 link_bytes=300908544 token_bytes=532955152 weight_movement_bytes=34603008 total_nvlink_bytes=567558160 weight_to_token_ratio=0.064927 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132294626e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74074 payload_rows_remote=64844 dup_rows=24230 dup_rows_remote=21144 dedup_ratio=0.246480 router_tokens_per_expert=[763, 768, 799, 815, 781, 777, 819, 771, 788, 760, 813, 751, 741, 765, 763, 779, 746, 738, 776, 723, 799, 809, 716, 779, 764, 800, 793, 806, 818, 757, 722, 754, 725, 803, 805, 747, 777, 720, 798, 783, 739, 766, 714, 808, 721, 723, 775, 760, 744, 752, 802, 813, 768, 754, 731, 814, 770, 751, 767, 706, 710, 806, 814, 733] payload_rows_per_dst=[9428, 9264, 9204, 9326, 9230, 9168, 9284, 9170] dup_rows_per_dst=[3158, 3056, 2968, 3102, 3086, 2844, 3072, 2944] combine_rows_per_src=[9428, 9264, 9204, 9326, 9230, 9168, 9284, 9170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74106 payload_rows_remote=64938 dup_rows=24198 dup_rows_remote=21024 dedup_ratio=0.246155 router_tokens_per_expert=[761, 716, 755, 799, 757, 734, 830, 744, 773, 779, 750, 785, 747, 757, 785, 795, 739, 811, 737, 784, 768, 742, 821, 776, 726, 740, 769, 767, 780, 766, 757, 779, 772, 784, 721, 766, 729, 775, 782, 811, 801, 738, 763, 754, 820, 754, 705, 803, 707, 786, 769, 813, 791, 796, 749, 768, 798, 765, 781, 762, 777, 768, 762, 753] payload_rows_per_dst=[9262, 9168, 9312, 9226, 9228, 9270, 9340, 9300] dup_rows_per_dst=[2930, 3174, 3044, 2942, 3052, 3006, 3018, 3032] combine_rows_per_src=[9262, 9168, 9312, 9226, 9228, 9270, 9340, 9300] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=64856 dup_rows=24008 dup_rows_remote=20944 dedup_ratio=0.244222 router_tokens_per_expert=[782, 719, 778, 735, 753, 773, 756, 797, 738, 768, 806, 769, 751, 775, 761, 740, 779, 788, 768, 758, 834, 791, 776, 758, 790, 747, 766, 767, 763, 746, 704, 797, 756, 736, 785, 745, 793, 763, 779, 744, 761, 745, 796, 723, 832, 813, 772, 706, 785, 765, 783, 745, 743, 755, 760, 805, 785, 757, 791, 816, 806, 810, 746, 718] payload_rows_per_dst=[9152, 9278, 9440, 9208, 9230, 9290, 9282, 9416] dup_rows_per_dst=[3034, 2938, 3064, 2952, 2972, 3006, 3000, 3042] combine_rows_per_src=[9152, 9278, 9440, 9208, 9230, 9290, 9282, 9416] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74352 payload_rows_remote=65144 dup_rows=23952 dup_rows_remote=20922 dedup_ratio=0.243652 router_tokens_per_expert=[754, 742, 777, 790, 767, 797, 780, 787, 786, 768, 755, 769, 742, 770, 719, 776, 819, 768, 797, 761, 735, 742, 789, 732, 771, 803, 732, 805, 800, 805, 778, 746, 747, 747, 725, 762, 758, 760, 764, 781, 778, 777, 719, 773, 771, 767, 805, 749, 802, 737, 773, 792, 773, 826, 764, 777, 787, 723, 764, 741, 709, 794, 738, 807] payload_rows_per_dst=[9116, 9218, 9240, 9314, 9214, 9476, 9566, 9208] dup_rows_per_dst=[2920, 2952, 2934, 2840, 2874, 3154, 3248, 3030] combine_rows_per_src=[9116, 9218, 9240, 9314, 9214, 9476, 9566, 9208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65396 dup_rows=23908 dup_rows_remote=20960 dedup_ratio=0.243205 router_tokens_per_expert=[784, 746, 754, 774, 718, 785, 755, 744, 769, 760, 791, 746, 746, 771, 807, 711, 834, 760, 795, 742, 794, 806, 745, 789, 797, 828, 780, 754, 795, 768, 770, 800, 763, 714, 752, 717, 779, 795, 787, 801, 825, 761, 853, 725, 801, 743, 779, 738, 706, 717, 738, 775, 753, 733, 775, 777, 756, 713, 764, 775, 779, 791, 785, 764] payload_rows_per_dst=[9230, 9256, 9450, 9420, 9300, 9410, 9000, 9330] dup_rows_per_dst=[2890, 2946, 3080, 3164, 2916, 3040, 2948, 2924] combine_rows_per_src=[9230, 9256, 9450, 9420, 9300, 9410, 9000, 9330] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265601024 dispatch_meta_bytes=343952 combine_payload_bytes=265601024 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265944976 ingress_bytes=369410048 link_bytes=369410048 token_bytes=531546000 weight_movement_bytes=103809024 total_nvlink_bytes=635355024 weight_to_token_ratio=0.195296 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390060271e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74470 payload_rows_remote=65192 dup_rows=23834 dup_rows_remote=20628 dedup_ratio=0.242452 router_tokens_per_expert=[743, 719, 764, 770, 744, 769, 795, 775, 797, 787, 781, 774, 748, 797, 796, 744, 811, 803, 726, 796, 768, 797, 787, 785, 755, 773, 833, 750, 762, 721, 778, 759, 747, 745, 763, 742, 765, 795, 755, 748, 752, 778, 783, 751, 743, 822, 760, 777, 792, 818, 748, 773, 737, 815, 731, 745, 742, 781, 777, 717, 781, 801, 729, 732] payload_rows_per_dst=[9278, 9532, 9476, 9350, 9188, 9238, 9116, 9292] dup_rows_per_dst=[3206, 3014, 3070, 2912, 2932, 3094, 2778, 2828] combine_rows_per_src=[9278, 9532, 9476, 9350, 9188, 9238, 9116, 9292] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74046 payload_rows_remote=64674 dup_rows=24258 dup_rows_remote=21084 dedup_ratio=0.246765 router_tokens_per_expert=[779, 765, 752, 719, 758, 827, 767, 760, 772, 760, 845, 743, 792, 747, 765, 763, 778, 739, 754, 760, 777, 755, 782, 780, 712, 758, 779, 767, 823, 739, 801, 797, 769, 753, 765, 736, 744, 762, 766, 783, 779, 802, 809, 801, 780, 708, 720, 771, 787, 783, 760, 820, 804, 788, 755, 776, 760, 721, 733, 781, 741, 747, 776, 757] payload_rows_per_dst=[9246, 9240, 9254, 9236, 9206, 9320, 9372, 9172] dup_rows_per_dst=[3008, 3134, 2996, 3116, 2950, 3020, 3174, 2860] combine_rows_per_src=[9246, 9240, 9254, 9236, 9206, 9320, 9372, 9172] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74106 payload_rows_remote=64938 dup_rows=24198 dup_rows_remote=21024 dedup_ratio=0.246155 router_tokens_per_expert=[761, 716, 755, 799, 757, 734, 830, 744, 773, 779, 750, 785, 747, 757, 785, 795, 739, 811, 737, 784, 768, 742, 821, 776, 726, 740, 769, 767, 780, 766, 757, 779, 772, 784, 721, 766, 729, 775, 782, 811, 801, 738, 763, 754, 820, 754, 705, 803, 707, 786, 769, 813, 791, 796, 749, 768, 798, 765, 781, 762, 777, 768, 762, 753] payload_rows_per_dst=[9262, 9168, 9312, 9226, 9228, 9270, 9340, 9300] dup_rows_per_dst=[2930, 3174, 3044, 2942, 3052, 3006, 3018, 3032] combine_rows_per_src=[9262, 9168, 9312, 9226, 9228, 9270, 9340, 9300] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=65316 dup_rows=23940 dup_rows_remote=21066 dedup_ratio=0.243530 router_tokens_per_expert=[740, 733, 825, 755, 790, 825, 813, 752, 751, 805, 764, 802, 816, 792, 787, 812, 749, 859, 760, 781, 794, 776, 719, 730, 762, 766, 754, 793, 759, 779, 773, 795, 801, 740, 767, 734, 760, 764, 743, 752, 728, 729, 735, 729, 745, 756, 777, 762, 730, 758, 802, 741, 749, 782, 807, 759, 776, 782, 743, 787, 759, 772, 737, 735] payload_rows_per_dst=[9396, 9526, 9402, 9270, 9186, 9048, 9232, 9304] dup_rows_per_dst=[3070, 3132, 2934, 3092, 2936, 2874, 3024, 2878] combine_rows_per_src=[9396, 9526, 9402, 9270, 9186, 9048, 9232, 9304] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74278 payload_rows_remote=65054 dup_rows=24026 dup_rows_remote=21034 dedup_ratio=0.244405 router_tokens_per_expert=[769, 780, 768, 734, 787, 783, 758, 776, 763, 761, 722, 777, 751, 777, 824, 744, 745, 775, 761, 755, 779, 739, 754, 772, 800, 750, 742, 789, 727, 784, 728, 788, 762, 757, 771, 790, 775, 785, 798, 760, 762, 771, 771, 762, 795, 794, 832, 720, 756, 776, 792, 786, 770, 771, 768, 736, 789, 809, 773, 779, 743, 729, 749, 759] payload_rows_per_dst=[9338, 9334, 9172, 9224, 9330, 9378, 9266, 9236] dup_rows_per_dst=[2972, 2904, 2988, 2992, 3066, 3036, 3044, 3024] combine_rows_per_src=[9338, 9334, 9172, 9224, 9330, 9378, 9266, 9236] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266829824 dispatch_meta_bytes=344264 combine_payload_bytes=266829824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267174088 ingress_bytes=301432832 link_bytes=301432832 token_bytes=534003912 weight_movement_bytes=34603008 total_nvlink_bytes=568606920 weight_to_token_ratio=0.064799 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134267479e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=267026432 dispatch_meta_bytes=343280 combine_payload_bytes=267026432 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267369712 ingress_bytes=301629440 link_bytes=301629440 token_bytes=534396144 weight_movement_bytes=34603008 total_nvlink_bytes=568999152 weight_to_token_ratio=0.064752 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135007300e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=264904704 dispatch_meta_bytes=343032 combine_payload_bytes=264904704 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265247736 ingress_bytes=368713728 link_bytes=368713728 token_bytes=530152440 weight_movement_bytes=103809024 total_nvlink_bytes=633961464 weight_to_token_ratio=0.195810 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.387440075e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265986048 dispatch_meta_bytes=343848 combine_payload_bytes=265986048 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=266329896 ingress_bytes=439001088 link_bytes=439001088 token_bytes=532315944 weight_movement_bytes=173015040 total_nvlink_bytes=705330984 weight_to_token_ratio=0.325023 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.651925752e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=267534336 dispatch_meta_bytes=345528 combine_payload_bytes=267534336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267879864 ingress_bytes=302137344 link_bytes=302137344 token_bytes=535414200 weight_movement_bytes=34603008 total_nvlink_bytes=570017208 weight_to_token_ratio=0.064628 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136918501e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64848 dup_rows=24114 dup_rows_remote=20990 dedup_ratio=0.245300 router_tokens_per_expert=[773, 811, 763, 747, 796, 759, 748, 783, 752, 788, 789, 845, 784, 768, 782, 750, 758, 764, 739, 796, 774, 759, 764, 751, 765, 767, 749, 732, 801, 755, 752, 749, 800, 829, 719, 797, 755, 760, 784, 795, 785, 760, 794, 736, 750, 728, 741, 777, 775, 755, 734, 751, 738, 766, 788, 783, 784, 740, 787, 766, 727, 760, 775, 800] payload_rows_per_dst=[9342, 9468, 9228, 9234, 9306, 9202, 9148, 9262] dup_rows_per_dst=[3124, 3070, 2918, 3014, 3000, 2940, 3032, 3016] combine_rows_per_src=[9342, 9468, 9228, 9234, 9306, 9202, 9148, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74278 payload_rows_remote=65054 dup_rows=24026 dup_rows_remote=21034 dedup_ratio=0.244405 router_tokens_per_expert=[769, 780, 768, 734, 787, 783, 758, 776, 763, 761, 722, 777, 751, 777, 824, 744, 745, 775, 761, 755, 779, 739, 754, 772, 800, 750, 742, 789, 727, 784, 728, 788, 762, 757, 771, 790, 775, 785, 798, 760, 762, 771, 771, 762, 795, 794, 832, 720, 756, 776, 792, 786, 770, 771, 768, 736, 789, 809, 773, 779, 743, 729, 749, 759] payload_rows_per_dst=[9338, 9334, 9172, 9224, 9330, 9378, 9266, 9236] dup_rows_per_dst=[2972, 2904, 2988, 2992, 3066, 3036, 3044, 3024] combine_rows_per_src=[9338, 9334, 9172, 9224, 9330, 9378, 9266, 9236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74412 payload_rows_remote=65074 dup_rows=23892 dup_rows_remote=20974 dedup_ratio=0.243042 router_tokens_per_expert=[743, 798, 771, 735, 722, 755, 746, 756, 776, 754, 797, 832, 794, 787, 760, 787, 776, 788, 698, 705, 804, 766, 792, 763, 745, 772, 784, 796, 769, 728, 756, 798, 788, 736, 767, 759, 784, 753, 755, 738, 788, 812, 708, 767, 813, 741, 818, 796, 770, 741, 759, 769, 785, 807, 748, 784, 776, 760, 754, 752, 778, 764, 777, 752] payload_rows_per_dst=[9184, 9102, 9266, 9372, 9174, 9680, 9296, 9338] dup_rows_per_dst=[2868, 2854, 2918, 2960, 2986, 3314, 3074, 2918] combine_rows_per_src=[9184, 9102, 9266, 9372, 9174, 9680, 9296, 9338] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265650176 dispatch_meta_bytes=343200 combine_payload_bytes=265650176 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265993376 ingress_bytes=369459200 link_bytes=369459200 token_bytes=531643552 weight_movement_bytes=103809024 total_nvlink_bytes=635452576 weight_to_token_ratio=0.195261 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390245226e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=65316 dup_rows=23940 dup_rows_remote=21066 dedup_ratio=0.243530 router_tokens_per_expert=[740, 733, 825, 755, 790, 825, 813, 752, 751, 805, 764, 802, 816, 792, 787, 812, 749, 859, 760, 781, 794, 776, 719, 730, 762, 766, 754, 793, 759, 779, 773, 795, 801, 740, 767, 734, 760, 764, 743, 752, 728, 729, 735, 729, 745, 756, 777, 762, 730, 758, 802, 741, 749, 782, 807, 759, 776, 782, 743, 787, 759, 772, 737, 735] payload_rows_per_dst=[9396, 9526, 9402, 9270, 9186, 9048, 9232, 9304] dup_rows_per_dst=[3070, 3132, 2934, 3092, 2936, 2874, 3024, 2878] combine_rows_per_src=[9396, 9526, 9402, 9270, 9186, 9048, 9232, 9304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=267862016 dispatch_meta_bytes=345424 combine_payload_bytes=267862016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268207440 ingress_bytes=302465024 link_bytes=302465024 token_bytes=536069456 weight_movement_bytes=34603008 total_nvlink_bytes=570672464 weight_to_token_ratio=0.064549 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138151535e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265617408 dispatch_meta_bytes=343352 combine_payload_bytes=265617408 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265960760 ingress_bytes=334823424 link_bytes=334823424 token_bytes=531578168 weight_movement_bytes=69206016 total_nvlink_bytes=600784184 weight_to_token_ratio=0.130190 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259913589e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=344352 combine_payload_bytes=266461184 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266805536 ingress_bytes=335667200 link_bytes=335667200 token_bytes=533266720 weight_movement_bytes=69206016 total_nvlink_bytes=602472736 weight_to_token_ratio=0.129777 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263088650e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=344192 combine_payload_bytes=266543104 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266887296 ingress_bytes=335749120 link_bytes=335749120 token_bytes=533430400 weight_movement_bytes=69206016 total_nvlink_bytes=602636416 weight_to_token_ratio=0.129738 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263396909e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=267534336 dispatch_meta_bytes=345528 combine_payload_bytes=267534336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267879864 ingress_bytes=302137344 link_bytes=302137344 token_bytes=535414200 weight_movement_bytes=34603008 total_nvlink_bytes=570017208 weight_to_token_ratio=0.064628 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136918501e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74514 payload_rows_remote=65202 dup_rows=23790 dup_rows_remote=20858 dedup_ratio=0.242004 router_tokens_per_expert=[717, 780, 750, 726, 727, 746, 810, 754, 774, 758, 804, 784, 784, 780, 784, 780, 761, 770, 753, 742, 779, 748, 757, 788, 798, 748, 739, 837, 780, 809, 740, 776, 743, 796, 728, 746, 772, 785, 796, 756, 762, 754, 727, 737, 722, 769, 780, 746, 778, 759, 786, 795, 769, 739, 753, 816, 779, 754, 808, 785, 799, 817, 771, 742] payload_rows_per_dst=[9176, 9444, 9310, 9416, 9312, 9132, 9312, 9412] dup_rows_per_dst=[2844, 3052, 2886, 3038, 2932, 2862, 3078, 3098] combine_rows_per_src=[9176, 9444, 9310, 9416, 9312, 9132, 9312, 9412] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=267067392 dispatch_meta_bytes=344240 combine_payload_bytes=267067392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267411632 ingress_bytes=301670400 link_bytes=301670400 token_bytes=534479024 weight_movement_bytes=34603008 total_nvlink_bytes=569082032 weight_to_token_ratio=0.064742 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135161429e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74412 payload_rows_remote=65074 dup_rows=23892 dup_rows_remote=20974 dedup_ratio=0.243042 router_tokens_per_expert=[743, 798, 771, 735, 722, 755, 746, 756, 776, 754, 797, 832, 794, 787, 760, 787, 776, 788, 698, 705, 804, 766, 792, 763, 745, 772, 784, 796, 769, 728, 756, 798, 788, 736, 767, 759, 784, 753, 755, 738, 788, 812, 708, 767, 813, 741, 818, 796, 770, 741, 759, 769, 785, 807, 748, 784, 776, 760, 754, 752, 778, 764, 777, 752] payload_rows_per_dst=[9184, 9102, 9266, 9372, 9174, 9680, 9296, 9338] dup_rows_per_dst=[2868, 2854, 2918, 2960, 2986, 3314, 3074, 2918] combine_rows_per_src=[9184, 9102, 9266, 9372, 9174, 9680, 9296, 9338] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64848 dup_rows=24114 dup_rows_remote=20990 dedup_ratio=0.245300 router_tokens_per_expert=[773, 811, 763, 747, 796, 759, 748, 783, 752, 788, 789, 845, 784, 768, 782, 750, 758, 764, 739, 796, 774, 759, 764, 751, 765, 767, 749, 732, 801, 755, 752, 749, 800, 829, 719, 797, 755, 760, 784, 795, 785, 760, 794, 736, 750, 728, 741, 777, 775, 755, 734, 751, 738, 766, 788, 783, 784, 740, 787, 766, 727, 760, 775, 800] payload_rows_per_dst=[9342, 9468, 9228, 9234, 9306, 9202, 9148, 9262] dup_rows_per_dst=[3124, 3070, 2918, 3014, 3000, 2940, 3032, 3016] combine_rows_per_src=[9342, 9468, 9228, 9234, 9306, 9202, 9148, 9262] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65396 dup_rows=23908 dup_rows_remote=20960 dedup_ratio=0.243205 router_tokens_per_expert=[784, 746, 754, 774, 718, 785, 755, 744, 769, 760, 791, 746, 746, 771, 807, 711, 834, 760, 795, 742, 794, 806, 745, 789, 797, 828, 780, 754, 795, 768, 770, 800, 763, 714, 752, 717, 779, 795, 787, 801, 825, 761, 853, 725, 801, 743, 779, 738, 706, 717, 738, 775, 753, 733, 775, 777, 756, 713, 764, 775, 779, 791, 785, 764] payload_rows_per_dst=[9230, 9256, 9450, 9420, 9300, 9410, 9000, 9330] dup_rows_per_dst=[2890, 2946, 3080, 3164, 2916, 3040, 2948, 2924] combine_rows_per_src=[9230, 9256, 9450, 9420, 9300, 9410, 9000, 9330] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74584 payload_rows_remote=65306 dup_rows=23720 dup_rows_remote=20804 dedup_ratio=0.241292 router_tokens_per_expert=[786, 757, 731, 790, 757, 780, 788, 736, 768, 785, 754, 758, 803, 736, 759, 734, 749, 758, 810, 755, 708, 755, 753, 729, 738, 781, 806, 785, 789, 804, 769, 763, 799, 747, 760, 778, 786, 805, 754, 789, 753, 778, 740, 780, 800, 759, 771, 768, 801, 748, 770, 759, 749, 806, 773, 721, 787, 778, 831, 754, 780, 781, 734, 739] payload_rows_per_dst=[9266, 9278, 9102, 9482, 9398, 9378, 9328, 9352] dup_rows_per_dst=[2984, 2916, 2932, 2988, 3038, 2920, 2926, 3016] combine_rows_per_src=[9266, 9278, 9102, 9482, 9398, 9378, 9328, 9352] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=64894 dup_rows=23992 dup_rows_remote=20878 dedup_ratio=0.244059 router_tokens_per_expert=[754, 779, 782, 779, 741, 774, 763, 824, 716, 720, 821, 764, 711, 734, 793, 758, 748, 747, 803, 759, 791, 755, 731, 805, 712, 825, 723, 772, 777, 800, 770, 787, 765, 744, 790, 814, 783, 740, 786, 750, 798, 799, 783, 780, 800, 740, 757, 809, 778, 791, 771, 764, 778, 727, 770, 736, 764, 763, 773, 761, 745, 748, 826, 701] payload_rows_per_dst=[9424, 9216, 9210, 9328, 9288, 9418, 9214, 9214] dup_rows_per_dst=[2968, 2818, 3068, 3004, 3056, 3114, 3016, 2948] combine_rows_per_src=[9424, 9216, 9210, 9328, 9288, 9418, 9214, 9214] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74270 payload_rows_remote=64992 dup_rows=24034 dup_rows_remote=21098 dedup_ratio=0.244486 router_tokens_per_expert=[695, 754, 769, 761, 745, 740, 726, 794, 819, 806, 751, 809, 736, 803, 774, 791, 746, 770, 767, 764, 749, 733, 784, 749, 754, 777, 756, 781, 753, 780, 822, 684, 750, 774, 744, 757, 778, 764, 786, 808, 753, 774, 782, 760, 786, 751, 797, 772, 768, 818, 780, 828, 716, 752, 745, 758, 785, 780, 794, 772, 777, 770, 775, 756] payload_rows_per_dst=[9056, 9450, 9244, 9278, 9270, 9264, 9278, 9430] dup_rows_per_dst=[2912, 3128, 2880, 2936, 3052, 3086, 3052, 2988] combine_rows_per_src=[9056, 9450, 9244, 9278, 9270, 9264, 9278, 9430] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=344192 combine_payload_bytes=266543104 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266887296 ingress_bytes=335749120 link_bytes=335749120 token_bytes=533430400 weight_movement_bytes=69206016 total_nvlink_bytes=602636416 weight_to_token_ratio=0.129738 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263396909e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265617408 dispatch_meta_bytes=343352 combine_payload_bytes=265617408 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265960760 ingress_bytes=334823424 link_bytes=334823424 token_bytes=531578168 weight_movement_bytes=69206016 total_nvlink_bytes=600784184 weight_to_token_ratio=0.130190 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259913589e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74446 payload_rows_remote=65094 dup_rows=23858 dup_rows_remote=20806 dedup_ratio=0.242696 router_tokens_per_expert=[791, 805, 750, 719, 773, 787, 759, 751, 767, 805, 814, 682, 795, 778, 784, 767, 794, 752, 746, 761, 812, 732, 788, 817, 778, 809, 784, 704, 749, 781, 762, 715, 787, 801, 748, 763, 753, 800, 789, 753, 731, 768, 750, 790, 765, 745, 806, 761, 731, 753, 778, 778, 754, 748, 797, 738, 739, 787, 792, 800, 750, 761, 732, 793] payload_rows_per_dst=[9262, 9308, 9352, 9222, 9352, 9324, 9304, 9322] dup_rows_per_dst=[3008, 3076, 3052, 2942, 3036, 2908, 2850, 2986] combine_rows_per_src=[9262, 9308, 9352, 9222, 9352, 9324, 9304, 9322] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=267862016 dispatch_meta_bytes=345424 combine_payload_bytes=267862016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268207440 ingress_bytes=302465024 link_bytes=302465024 token_bytes=536069456 weight_movement_bytes=34603008 total_nvlink_bytes=570672464 weight_to_token_ratio=0.064549 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138151535e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74506 payload_rows_remote=65192 dup_rows=23798 dup_rows_remote=20790 dedup_ratio=0.242086 router_tokens_per_expert=[807, 751, 795, 806, 708, 756, 740, 779, 751, 783, 783, 754, 692, 757, 719, 782, 771, 749, 763, 742, 800, 764, 769, 775, 792, 806, 753, 761, 757, 764, 772, 764, 753, 741, 792, 754, 801, 767, 752, 750, 732, 756, 765, 800, 755, 802, 807, 759, 784, 764, 749, 755, 767, 780, 780, 782, 749, 785, 778, 763, 856, 750, 787, 772] payload_rows_per_dst=[9300, 9204, 9278, 9342, 9338, 9338, 9314, 9392] dup_rows_per_dst=[2984, 2838, 2988, 2996, 2882, 3014, 3008, 3088] combine_rows_per_src=[9300, 9204, 9278, 9342, 9338, 9338, 9314, 9392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=267067392 dispatch_meta_bytes=344240 combine_payload_bytes=267067392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267411632 ingress_bytes=301670400 link_bytes=301670400 token_bytes=534479024 weight_movement_bytes=34603008 total_nvlink_bytes=569082032 weight_to_token_ratio=0.064742 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135161429e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=267493376 dispatch_meta_bytes=344440 combine_payload_bytes=267493376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267837816 ingress_bytes=302096384 link_bytes=302096384 token_bytes=535331192 weight_movement_bytes=34603008 total_nvlink_bytes=569934200 weight_to_token_ratio=0.064639 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136764372e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265805824 dispatch_meta_bytes=343088 combine_payload_bytes=265805824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266148912 ingress_bytes=300408832 link_bytes=300408832 token_bytes=531954736 weight_movement_bytes=34603008 total_nvlink_bytes=566557744 weight_to_token_ratio=0.065049 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130414250e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266207232 dispatch_meta_bytes=344360 combine_payload_bytes=266207232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266551592 ingress_bytes=300810240 link_bytes=300810240 token_bytes=532758824 weight_movement_bytes=34603008 total_nvlink_bytes=567361832 weight_to_token_ratio=0.064951 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131924716e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74472 payload_rows_remote=64742 dup_rows=23832 dup_rows_remote=20510 dedup_ratio=0.242432 router_tokens_per_expert=[764, 804, 823, 783, 802, 734, 798, 800, 750, 780, 726, 772, 764, 794, 755, 801, 722, 747, 738, 770, 792, 786, 774, 786, 759, 789, 765, 753, 838, 744, 768, 720, 768, 737, 744, 799, 775, 755, 775, 763, 732, 806, 795, 786, 734, 731, 767, 741, 763, 800, 770, 787, 796, 774, 767, 773, 777, 695, 724, 790, 767, 729, 790, 741] payload_rows_per_dst=[9730, 9056, 9370, 9234, 9468, 9324, 9312, 8978] dup_rows_per_dst=[3322, 2792, 3068, 3038, 3100, 2860, 2812, 2840] combine_rows_per_src=[9730, 9056, 9370, 9234, 9468, 9324, 9312, 8978] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74146 payload_rows_remote=64848 dup_rows=24158 dup_rows_remote=21114 dedup_ratio=0.245748 router_tokens_per_expert=[755, 775, 771, 792, 707, 749, 793, 758, 781, 784, 757, 780, 778, 744, 746, 786, 727, 771, 838, 757, 754, 777, 720, 770, 738, 796, 775, 761, 743, 798, 785, 750, 770, 788, 801, 766, 750, 747, 768, 781, 762, 749, 770, 754, 785, 777, 722, 748, 795, 810, 774, 749, 755, 803, 738, 781, 784, 780, 787, 744, 807, 785, 753, 753] payload_rows_per_dst=[9218, 9158, 9284, 9254, 9298, 9258, 9380, 9296] dup_rows_per_dst=[2982, 3154, 2944, 3038, 3044, 2876, 3030, 3090] combine_rows_per_src=[9218, 9158, 9284, 9254, 9298, 9258, 9380, 9296] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266625024 dispatch_meta_bytes=343600 combine_payload_bytes=266625024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266968624 ingress_bytes=301228032 link_bytes=301228032 token_bytes=533593648 weight_movement_bytes=34603008 total_nvlink_bytes=568196656 weight_to_token_ratio=0.064849 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133496834e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265617408 dispatch_meta_bytes=343848 combine_payload_bytes=265617408 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265961256 ingress_bytes=334823424 link_bytes=334823424 token_bytes=531578664 weight_movement_bytes=69206016 total_nvlink_bytes=600784680 weight_to_token_ratio=0.130190 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259913589e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74270 payload_rows_remote=64992 dup_rows=24034 dup_rows_remote=21098 dedup_ratio=0.244486 router_tokens_per_expert=[695, 754, 769, 761, 745, 740, 726, 794, 819, 806, 751, 809, 736, 803, 774, 791, 746, 770, 767, 764, 749, 733, 784, 749, 754, 777, 756, 781, 753, 780, 822, 684, 750, 774, 744, 757, 778, 764, 786, 808, 753, 774, 782, 760, 786, 751, 797, 772, 768, 818, 780, 828, 716, 752, 745, 758, 785, 780, 794, 772, 777, 770, 775, 756] payload_rows_per_dst=[9056, 9450, 9244, 9278, 9270, 9264, 9278, 9430] dup_rows_per_dst=[2912, 3128, 2880, 2936, 3052, 3086, 3052, 2988] combine_rows_per_src=[9056, 9450, 9244, 9278, 9270, 9264, 9278, 9430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74318 payload_rows_remote=65068 dup_rows=23986 dup_rows_remote=21060 dedup_ratio=0.243998 router_tokens_per_expert=[780, 772, 750, 765, 767, 748, 779, 792, 770, 770, 769, 796, 723, 816, 780, 722, 764, 741, 733, 803, 739, 729, 787, 797, 784, 773, 775, 813, 772, 801, 745, 784, 774, 760, 746, 756, 716, 749, 787, 722, 747, 766, 792, 760, 727, 774, 836, 782, 789, 762, 747, 762, 795, 802, 778, 796, 769, 760, 801, 774, 750, 733, 760, 741] payload_rows_per_dst=[9314, 9284, 9218, 9416, 9108, 9318, 9410, 9250] dup_rows_per_dst=[2992, 3008, 2968, 3078, 2912, 3050, 3052, 2926] combine_rows_per_src=[9314, 9284, 9218, 9416, 9108, 9318, 9410, 9250] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74472 payload_rows_remote=64742 dup_rows=23832 dup_rows_remote=20510 dedup_ratio=0.242432 router_tokens_per_expert=[764, 804, 823, 783, 802, 734, 798, 800, 750, 780, 726, 772, 764, 794, 755, 801, 722, 747, 738, 770, 792, 786, 774, 786, 759, 789, 765, 753, 838, 744, 768, 720, 768, 737, 744, 799, 775, 755, 775, 763, 732, 806, 795, 786, 734, 731, 767, 741, 763, 800, 770, 787, 796, 774, 767, 773, 777, 695, 724, 790, 767, 729, 790, 741] payload_rows_per_dst=[9730, 9056, 9370, 9234, 9468, 9324, 9312, 8978] dup_rows_per_dst=[3322, 2792, 3068, 3038, 3100, 2860, 2812, 2840] combine_rows_per_src=[9730, 9056, 9370, 9234, 9468, 9324, 9312, 8978] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=65200 dup_rows=24050 dup_rows_remote=21124 dedup_ratio=0.244649 router_tokens_per_expert=[777, 774, 823, 767, 828, 761, 760, 767, 756, 811, 722, 764, 757, 742, 731, 707, 799, 791, 732, 762, 785, 738, 772, 723, 811, 795, 755, 779, 783, 776, 755, 800, 795, 816, 715, 790, 757, 797, 772, 798, 731, 761, 769, 773, 730, 739, 755, 740, 721, 793, 753, 806, 778, 771, 799, 759, 751, 741, 791, 790, 754, 778, 769, 757] payload_rows_per_dst=[9408, 9054, 9296, 9392, 9418, 9084, 9336, 9266] dup_rows_per_dst=[3106, 2926, 2908, 3116, 3062, 2912, 3024, 2996] combine_rows_per_src=[9408, 9054, 9296, 9392, 9418, 9084, 9336, 9266] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74506 payload_rows_remote=65192 dup_rows=23798 dup_rows_remote=20790 dedup_ratio=0.242086 router_tokens_per_expert=[807, 751, 795, 806, 708, 756, 740, 779, 751, 783, 783, 754, 692, 757, 719, 782, 771, 749, 763, 742, 800, 764, 769, 775, 792, 806, 753, 761, 757, 764, 772, 764, 753, 741, 792, 754, 801, 767, 752, 750, 732, 756, 765, 800, 755, 802, 807, 759, 784, 764, 749, 755, 767, 780, 780, 782, 749, 785, 778, 763, 856, 750, 787, 772] payload_rows_per_dst=[9300, 9204, 9278, 9342, 9338, 9338, 9314, 9392] dup_rows_per_dst=[2984, 2838, 2988, 2996, 2882, 3014, 3008, 3088] combine_rows_per_src=[9300, 9204, 9278, 9342, 9338, 9338, 9314, 9392] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74084 payload_rows_remote=64912 dup_rows=24220 dup_rows_remote=21258 dedup_ratio=0.246379 router_tokens_per_expert=[781, 769, 761, 787, 738, 767, 747, 782, 805, 762, 761, 750, 733, 793, 719, 767, 710, 706, 799, 769, 774, 735, 775, 774, 764, 759, 778, 792, 785, 823, 794, 775, 728, 756, 792, 803, 747, 737, 746, 802, 755, 771, 774, 743, 747, 754, 775, 748, 738, 736, 845, 760, 825, 811, 776, 799, 773, 766, 701, 804, 770, 796, 781, 759] payload_rows_per_dst=[9252, 9170, 9152, 9434, 9238, 9172, 9422, 9244] dup_rows_per_dst=[3012, 3010, 2932, 3106, 2984, 2962, 3158, 3056] combine_rows_per_src=[9252, 9170, 9152, 9434, 9238, 9172, 9422, 9244] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65204 dup_rows=23924 dup_rows_remote=21012 dedup_ratio=0.243368 router_tokens_per_expert=[829, 763, 729, 754, 771, 801, 689, 781, 770, 761, 777, 781, 748, 804, 820, 738, 761, 788, 771, 757, 750, 801, 778, 780, 736, 755, 777, 756, 727, 764, 782, 747, 720, 789, 816, 786, 758, 798, 758, 793, 780, 812, 803, 761, 746, 808, 740, 764, 744, 743, 778, 748, 748, 777, 750, 721, 784, 786, 760, 758, 747, 797, 732, 801] payload_rows_per_dst=[9240, 9404, 9272, 9176, 9490, 9262, 9200, 9336] dup_rows_per_dst=[2994, 2994, 3100, 2912, 2946, 3166, 2818, 2994] combine_rows_per_src=[9240, 9404, 9272, 9176, 9490, 9262, 9200, 9336] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266518528 dispatch_meta_bytes=344512 combine_payload_bytes=266518528 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266863040 ingress_bytes=335724544 link_bytes=335724544 token_bytes=533381568 weight_movement_bytes=69206016 total_nvlink_bytes=602587584 weight_to_token_ratio=0.129750 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263304431e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=64894 dup_rows=23992 dup_rows_remote=20878 dedup_ratio=0.244059 router_tokens_per_expert=[754, 779, 782, 779, 741, 774, 763, 824, 716, 720, 821, 764, 711, 734, 793, 758, 748, 747, 803, 759, 791, 755, 731, 805, 712, 825, 723, 772, 777, 800, 770, 787, 765, 744, 790, 814, 783, 740, 786, 750, 798, 799, 783, 780, 800, 740, 757, 809, 778, 791, 771, 764, 778, 727, 770, 736, 764, 763, 773, 761, 745, 748, 826, 701] payload_rows_per_dst=[9424, 9216, 9210, 9328, 9288, 9418, 9214, 9214] dup_rows_per_dst=[2968, 2818, 3068, 3004, 3056, 3114, 3016, 2948] combine_rows_per_src=[9424, 9216, 9210, 9328, 9288, 9418, 9214, 9214] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266625024 dispatch_meta_bytes=343600 combine_payload_bytes=266625024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266968624 ingress_bytes=301228032 link_bytes=301228032 token_bytes=533593648 weight_movement_bytes=34603008 total_nvlink_bytes=568196656 weight_to_token_ratio=0.064849 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133496834e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=267026432 dispatch_meta_bytes=343928 combine_payload_bytes=267026432 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267370360 ingress_bytes=336232448 link_bytes=336232448 token_bytes=534396792 weight_movement_bytes=69206016 total_nvlink_bytes=603602808 weight_to_token_ratio=0.129503 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265215633e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265183232 dispatch_meta_bytes=341008 combine_payload_bytes=265183232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265524240 ingress_bytes=299786240 link_bytes=299786240 token_bytes=530707472 weight_movement_bytes=34603008 total_nvlink_bytes=565310480 weight_to_token_ratio=0.065202 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128071486e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266207232 dispatch_meta_bytes=344360 combine_payload_bytes=266207232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266551592 ingress_bytes=300810240 link_bytes=300810240 token_bytes=532758824 weight_movement_bytes=34603008 total_nvlink_bytes=567361832 weight_to_token_ratio=0.064951 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131924716e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266518528 dispatch_meta_bytes=344512 combine_payload_bytes=266518528 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266863040 ingress_bytes=335724544 link_bytes=335724544 token_bytes=533381568 weight_movement_bytes=69206016 total_nvlink_bytes=602587584 weight_to_token_ratio=0.129750 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263304431e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265805824 dispatch_meta_bytes=343088 combine_payload_bytes=265805824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266148912 ingress_bytes=300408832 link_bytes=300408832 token_bytes=531954736 weight_movement_bytes=34603008 total_nvlink_bytes=566557744 weight_to_token_ratio=0.065049 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130414250e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74146 payload_rows_remote=64848 dup_rows=24158 dup_rows_remote=21114 dedup_ratio=0.245748 router_tokens_per_expert=[755, 775, 771, 792, 707, 749, 793, 758, 781, 784, 757, 780, 778, 744, 746, 786, 727, 771, 838, 757, 754, 777, 720, 770, 738, 796, 775, 761, 743, 798, 785, 750, 770, 788, 801, 766, 750, 747, 768, 781, 762, 749, 770, 754, 785, 777, 722, 748, 795, 810, 774, 749, 755, 803, 738, 781, 784, 780, 787, 744, 807, 785, 753, 753] payload_rows_per_dst=[9218, 9158, 9284, 9254, 9298, 9258, 9380, 9296] dup_rows_per_dst=[2982, 3154, 2944, 3038, 3044, 2876, 3030, 3090] combine_rows_per_src=[9218, 9158, 9284, 9254, 9298, 9258, 9380, 9296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=65200 dup_rows=24050 dup_rows_remote=21124 dedup_ratio=0.244649 router_tokens_per_expert=[777, 774, 823, 767, 828, 761, 760, 767, 756, 811, 722, 764, 757, 742, 731, 707, 799, 791, 732, 762, 785, 738, 772, 723, 811, 795, 755, 779, 783, 776, 755, 800, 795, 816, 715, 790, 757, 797, 772, 798, 731, 761, 769, 773, 730, 739, 755, 740, 721, 793, 753, 806, 778, 771, 799, 759, 751, 741, 791, 790, 754, 778, 769, 757] payload_rows_per_dst=[9408, 9054, 9296, 9392, 9418, 9084, 9336, 9266] dup_rows_per_dst=[3106, 2926, 2908, 3116, 3062, 2912, 3024, 2996] combine_rows_per_src=[9408, 9054, 9296, 9392, 9418, 9084, 9336, 9266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265617408 dispatch_meta_bytes=343848 combine_payload_bytes=265617408 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265961256 ingress_bytes=334823424 link_bytes=334823424 token_bytes=531578664 weight_movement_bytes=69206016 total_nvlink_bytes=600784680 weight_to_token_ratio=0.130190 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259913589e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74126 payload_rows_remote=64660 dup_rows=24178 dup_rows_remote=21016 dedup_ratio=0.245951 router_tokens_per_expert=[808, 754, 710, 754, 725, 714, 744, 747, 783, 804, 785, 762, 794, 743, 783, 777, 745, 793, 792, 816, 774, 796, 800, 798, 778, 804, 732, 794, 760, 754, 739, 710, 760, 758, 754, 794, 776, 780, 810, 795, 751, 716, 774, 713, 747, 763, 794, 758, 752, 775, 764, 743, 790, 785, 741, 755, 814, 790, 785, 788, 778, 746, 765, 766] payload_rows_per_dst=[9052, 9320, 9466, 9168, 9400, 9074, 9246, 9400] dup_rows_per_dst=[2860, 3142, 3162, 2974, 3054, 2958, 2964, 3064] combine_rows_per_src=[9052, 9320, 9466, 9168, 9400, 9074, 9246, 9400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=65114 dup_rows=23920 dup_rows_remote=20858 dedup_ratio=0.243327 router_tokens_per_expert=[770, 735, 772, 759, 755, 752, 753, 743, 754, 808, 750, 769, 793, 756, 757, 722, 746, 729, 814, 797, 801, 768, 741, 811, 779, 758, 741, 818, 759, 759, 772, 818, 804, 714, 752, 773, 753, 810, 778, 740, 776, 800, 763, 790, 768, 718, 775, 763, 790, 783, 785, 775, 741, 760, 789, 743, 749, 800, 759, 827, 718, 812, 759, 726] payload_rows_per_dst=[9146, 9226, 9330, 9332, 9244, 9400, 9270, 9436] dup_rows_per_dst=[2932, 2992, 3084, 3076, 3004, 2906, 3062, 2864] combine_rows_per_src=[9146, 9226, 9330, 9332, 9244, 9400, 9270, 9436] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=267059200 dispatch_meta_bytes=345296 combine_payload_bytes=267059200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267404496 ingress_bytes=301662208 link_bytes=301662208 token_bytes=534463696 weight_movement_bytes=34603008 total_nvlink_bytes=569066704 weight_to_token_ratio=0.064743 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135130603e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265879552 dispatch_meta_bytes=344680 combine_payload_bytes=265879552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266224232 ingress_bytes=300482560 link_bytes=300482560 token_bytes=532103784 weight_movement_bytes=34603008 total_nvlink_bytes=566706792 weight_to_token_ratio=0.065031 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130691682e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74126 payload_rows_remote=64816 dup_rows=24178 dup_rows_remote=21304 dedup_ratio=0.245951 router_tokens_per_expert=[794, 783, 772, 737, 776, 773, 809, 715, 758, 787, 815, 785, 809, 782, 810, 738, 786, 772, 754, 753, 751, 773, 765, 784, 749, 765, 772, 780, 761, 822, 724, 770, 752, 769, 797, 784, 733, 723, 742, 792, 778, 742, 755, 803, 741, 750, 775, 768, 712, 748, 707, 772, 772, 783, 740, 793, 718, 774, 749, 838, 775, 781, 817, 745] payload_rows_per_dst=[9246, 9328, 9250, 9308, 9310, 9234, 9114, 9336] dup_rows_per_dst=[3072, 3240, 3026, 2978, 2874, 2990, 2940, 3058] combine_rows_per_src=[9246, 9328, 9250, 9308, 9310, 9234, 9114, 9336] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=267026432 dispatch_meta_bytes=343928 combine_payload_bytes=267026432 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267370360 ingress_bytes=336232448 link_bytes=336232448 token_bytes=534396792 weight_movement_bytes=69206016 total_nvlink_bytes=603602808 weight_to_token_ratio=0.129503 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265215633e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=267075584 dispatch_meta_bytes=344864 combine_payload_bytes=267075584 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267420448 ingress_bytes=336281600 link_bytes=336281600 token_bytes=534496032 weight_movement_bytes=69206016 total_nvlink_bytes=603702048 weight_to_token_ratio=0.129479 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265400588e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=64644 dup_rows=24060 dup_rows_remote=20728 dedup_ratio=0.244751 router_tokens_per_expert=[767, 801, 792, 788, 770, 762, 780, 797, 757, 747, 845, 762, 765, 777, 718, 775, 782, 775, 737, 792, 704, 809, 720, 803, 752, 759, 747, 792, 745, 763, 762, 728, 774, 820, 800, 788, 731, 780, 795, 735, 752, 771, 741, 765, 736, 756, 796, 733, 769, 726, 779, 785, 749, 724, 801, 740, 777, 766, 801, 787, 773, 807, 753, 769] payload_rows_per_dst=[9600, 9366, 9238, 9130, 9134, 9236, 9296, 9244] dup_rows_per_dst=[3332, 3028, 2926, 2948, 2890, 2970, 3092, 2874] combine_rows_per_src=[9600, 9366, 9238, 9130, 9134, 9236, 9296, 9244] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=264847360 dispatch_meta_bytes=342704 combine_payload_bytes=264847360 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265190064 ingress_bytes=334053376 link_bytes=334053376 token_bytes=530037424 weight_movement_bytes=69206016 total_nvlink_bytes=599243440 weight_to_token_ratio=0.130568 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257015960e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265486336 dispatch_meta_bytes=344480 combine_payload_bytes=265486336 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265830816 ingress_bytes=334692352 link_bytes=334692352 token_bytes=531317152 weight_movement_bytes=69206016 total_nvlink_bytes=600523168 weight_to_token_ratio=0.130254 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259420376e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65204 dup_rows=23924 dup_rows_remote=21012 dedup_ratio=0.243368 router_tokens_per_expert=[829, 763, 729, 754, 771, 801, 689, 781, 770, 761, 777, 781, 748, 804, 820, 738, 761, 788, 771, 757, 750, 801, 778, 780, 736, 755, 777, 756, 727, 764, 782, 747, 720, 789, 816, 786, 758, 798, 758, 793, 780, 812, 803, 761, 746, 808, 740, 764, 744, 743, 778, 748, 748, 777, 750, 721, 784, 786, 760, 758, 747, 797, 732, 801] payload_rows_per_dst=[9240, 9404, 9272, 9176, 9490, 9262, 9200, 9336] dup_rows_per_dst=[2994, 2994, 3100, 2912, 2946, 3166, 2818, 2994] combine_rows_per_src=[9240, 9404, 9272, 9176, 9490, 9262, 9200, 9336] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74026 payload_rows_remote=64632 dup_rows=24278 dup_rows_remote=21216 dedup_ratio=0.246969 router_tokens_per_expert=[794, 822, 772, 749, 729, 807, 722, 731, 729, 810, 735, 769, 789, 768, 782, 723, 740, 776, 805, 776, 766, 783, 795, 745, 818, 756, 771, 767, 784, 742, 750, 782, 782, 805, 800, 747, 790, 775, 768, 764, 780, 749, 748, 764, 783, 758, 774, 724, 767, 783, 766, 761, 788, 755, 780, 751, 751, 766, 741, 759, 811, 772, 782, 721] payload_rows_per_dst=[8974, 9416, 9258, 8824, 9354, 9654, 9152, 9394] dup_rows_per_dst=[2730, 3312, 3114, 2686, 3050, 3284, 3040, 3062] combine_rows_per_src=[8974, 9416, 9258, 8824, 9354, 9654, 9152, 9394] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=64644 dup_rows=24060 dup_rows_remote=20728 dedup_ratio=0.244751 router_tokens_per_expert=[767, 801, 792, 788, 770, 762, 780, 797, 757, 747, 845, 762, 765, 777, 718, 775, 782, 775, 737, 792, 704, 809, 720, 803, 752, 759, 747, 792, 745, 763, 762, 728, 774, 820, 800, 788, 731, 780, 795, 735, 752, 771, 741, 765, 736, 756, 796, 733, 769, 726, 779, 785, 749, 724, 801, 740, 777, 766, 801, 787, 773, 807, 753, 769] payload_rows_per_dst=[9600, 9366, 9238, 9130, 9134, 9236, 9296, 9244] dup_rows_per_dst=[3332, 3028, 2926, 2948, 2890, 2970, 3092, 2874] combine_rows_per_src=[9600, 9366, 9238, 9130, 9134, 9236, 9296, 9244] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74388 payload_rows_remote=65112 dup_rows=23916 dup_rows_remote=20936 dedup_ratio=0.243286 router_tokens_per_expert=[791, 750, 780, 766, 805, 784, 793, 798, 821, 742, 705, 770, 769, 778, 763, 780, 763, 762, 741, 754, 762, 788, 768, 801, 786, 776, 756, 771, 765, 726, 799, 816, 789, 822, 722, 745, 744, 796, 793, 737, 743, 775, 722, 765, 765, 750, 786, 735, 769, 739, 770, 768, 793, 754, 720, 801, 737, 798, 788, 737, 821, 739, 767, 733] payload_rows_per_dst=[9412, 9276, 9300, 9340, 9306, 9232, 9228, 9294] dup_rows_per_dst=[3122, 2980, 2978, 3050, 2990, 2850, 3000, 2946] combine_rows_per_src=[9412, 9276, 9300, 9340, 9306, 9232, 9228, 9294] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74372 payload_rows_remote=65260 dup_rows=23932 dup_rows_remote=21038 dedup_ratio=0.243449 router_tokens_per_expert=[787, 786, 768, 797, 746, 772, 742, 770, 705, 729, 805, 773, 752, 751, 784, 780, 770, 765, 809, 755, 783, 709, 766, 819, 762, 757, 753, 756, 770, 788, 750, 764, 759, 800, 798, 809, 751, 784, 760, 804, 719, 748, 737, 754, 777, 748, 774, 746, 748, 787, 752, 768, 753, 781, 791, 775, 780, 757, 790, 725, 771, 799, 800, 784] payload_rows_per_dst=[9380, 9282, 9312, 9250, 9428, 9112, 9254, 9354] dup_rows_per_dst=[2956, 2876, 3040, 2950, 3102, 2894, 3056, 3058] combine_rows_per_src=[9380, 9282, 9312, 9250, 9428, 9112, 9254, 9354] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74084 payload_rows_remote=64912 dup_rows=24220 dup_rows_remote=21258 dedup_ratio=0.246379 router_tokens_per_expert=[781, 769, 761, 787, 738, 767, 747, 782, 805, 762, 761, 750, 733, 793, 719, 767, 710, 706, 799, 769, 774, 735, 775, 774, 764, 759, 778, 792, 785, 823, 794, 775, 728, 756, 792, 803, 747, 737, 746, 802, 755, 771, 774, 743, 747, 754, 775, 748, 738, 736, 845, 760, 825, 811, 776, 799, 773, 766, 701, 804, 770, 796, 781, 759] payload_rows_per_dst=[9252, 9170, 9152, 9434, 9238, 9172, 9422, 9244] dup_rows_per_dst=[3012, 3010, 2932, 3106, 2984, 2962, 3158, 3056] combine_rows_per_src=[9252, 9170, 9152, 9434, 9238, 9172, 9422, 9244] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74110 payload_rows_remote=64892 dup_rows=24194 dup_rows_remote=21134 dedup_ratio=0.246114 router_tokens_per_expert=[763, 765, 798, 782, 765, 747, 766, 733, 782, 750, 753, 780, 775, 763, 790, 797, 809, 791, 754, 740, 763, 719, 767, 740, 769, 744, 792, 791, 747, 729, 774, 793, 772, 775, 761, 794, 763, 749, 760, 783, 774, 788, 802, 728, 739, 794, 764, 772, 755, 777, 737, 767, 712, 766, 758, 804, 785, 748, 791, 726, 769, 827, 788, 793] payload_rows_per_dst=[9230, 9340, 9224, 9218, 9264, 9268, 9200, 9366] dup_rows_per_dst=[3008, 3040, 2942, 3060, 3050, 3054, 2952, 3088] combine_rows_per_src=[9230, 9340, 9224, 9218, 9264, 9268, 9200, 9366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=65114 dup_rows=23920 dup_rows_remote=20858 dedup_ratio=0.243327 router_tokens_per_expert=[770, 735, 772, 759, 755, 752, 753, 743, 754, 808, 750, 769, 793, 756, 757, 722, 746, 729, 814, 797, 801, 768, 741, 811, 779, 758, 741, 818, 759, 759, 772, 818, 804, 714, 752, 773, 753, 810, 778, 740, 776, 800, 763, 790, 768, 718, 775, 763, 790, 783, 785, 775, 741, 760, 789, 743, 749, 800, 759, 827, 718, 812, 759, 726] payload_rows_per_dst=[9146, 9226, 9330, 9332, 9244, 9400, 9270, 9436] dup_rows_per_dst=[2932, 2992, 3084, 3076, 3004, 2906, 3062, 2864] combine_rows_per_src=[9146, 9226, 9330, 9332, 9244, 9400, 9270, 9436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=264847360 dispatch_meta_bytes=342704 combine_payload_bytes=264847360 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265190064 ingress_bytes=334053376 link_bytes=334053376 token_bytes=530037424 weight_movement_bytes=69206016 total_nvlink_bytes=599243440 weight_to_token_ratio=0.130568 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257015960e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266706944 dispatch_meta_bytes=343888 combine_payload_bytes=266706944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267050832 ingress_bytes=301309952 link_bytes=301309952 token_bytes=533757776 weight_movement_bytes=34603008 total_nvlink_bytes=568360784 weight_to_token_ratio=0.064829 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133805092e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=264781824 dispatch_meta_bytes=341488 combine_payload_bytes=264781824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265123312 ingress_bytes=299384832 link_bytes=299384832 token_bytes=529905136 weight_movement_bytes=34603008 total_nvlink_bytes=564508144 weight_to_token_ratio=0.065300 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126561020e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=267075584 dispatch_meta_bytes=344864 combine_payload_bytes=267075584 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267420448 ingress_bytes=336281600 link_bytes=336281600 token_bytes=534496032 weight_movement_bytes=69206016 total_nvlink_bytes=603702048 weight_to_token_ratio=0.129479 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265400588e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=264732672 dispatch_meta_bytes=343392 combine_payload_bytes=264732672 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265076064 ingress_bytes=333938688 link_bytes=333938688 token_bytes=529808736 weight_movement_bytes=69206016 total_nvlink_bytes=599014752 weight_to_token_ratio=0.130625 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256584399e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265879552 dispatch_meta_bytes=344680 combine_payload_bytes=265879552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266224232 ingress_bytes=300482560 link_bytes=300482560 token_bytes=532103784 weight_movement_bytes=34603008 total_nvlink_bytes=566706792 weight_to_token_ratio=0.065031 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130691682e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74126 payload_rows_remote=64816 dup_rows=24178 dup_rows_remote=21304 dedup_ratio=0.245951 router_tokens_per_expert=[794, 783, 772, 737, 776, 773, 809, 715, 758, 787, 815, 785, 809, 782, 810, 738, 786, 772, 754, 753, 751, 773, 765, 784, 749, 765, 772, 780, 761, 822, 724, 770, 752, 769, 797, 784, 733, 723, 742, 792, 778, 742, 755, 803, 741, 750, 775, 768, 712, 748, 707, 772, 772, 783, 740, 793, 718, 774, 749, 838, 775, 781, 817, 745] payload_rows_per_dst=[9246, 9328, 9250, 9308, 9310, 9234, 9114, 9336] dup_rows_per_dst=[3072, 3240, 3026, 2978, 2874, 2990, 2940, 3058] combine_rows_per_src=[9246, 9328, 9250, 9308, 9310, 9234, 9114, 9336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74388 payload_rows_remote=65112 dup_rows=23916 dup_rows_remote=20936 dedup_ratio=0.243286 router_tokens_per_expert=[791, 750, 780, 766, 805, 784, 793, 798, 821, 742, 705, 770, 769, 778, 763, 780, 763, 762, 741, 754, 762, 788, 768, 801, 786, 776, 756, 771, 765, 726, 799, 816, 789, 822, 722, 745, 744, 796, 793, 737, 743, 775, 722, 765, 765, 750, 786, 735, 769, 739, 770, 768, 793, 754, 720, 801, 737, 798, 788, 737, 821, 739, 767, 733] payload_rows_per_dst=[9412, 9276, 9300, 9340, 9306, 9232, 9228, 9294] dup_rows_per_dst=[3122, 2980, 2978, 3050, 2990, 2850, 3000, 2946] combine_rows_per_src=[9412, 9276, 9300, 9340, 9306, 9232, 9228, 9294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=264781824 dispatch_meta_bytes=341488 combine_payload_bytes=264781824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265123312 ingress_bytes=299384832 link_bytes=299384832 token_bytes=529905136 weight_movement_bytes=34603008 total_nvlink_bytes=564508144 weight_to_token_ratio=0.065300 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126561020e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265486336 dispatch_meta_bytes=344480 combine_payload_bytes=265486336 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265830816 ingress_bytes=334692352 link_bytes=334692352 token_bytes=531317152 weight_movement_bytes=69206016 total_nvlink_bytes=600523168 weight_to_token_ratio=0.130254 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259420376e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74484 payload_rows_remote=65176 dup_rows=23820 dup_rows_remote=20810 dedup_ratio=0.242310 router_tokens_per_expert=[752, 788, 806, 788, 800, 803, 766, 784, 749, 739, 760, 745, 792, 773, 778, 736, 760, 764, 793, 783, 731, 775, 744, 784, 740, 736, 747, 773, 774, 802, 769, 785, 775, 706, 808, 736, 765, 741, 746, 770, 786, 785, 797, 785, 778, 732, 759, 750, 748, 760, 752, 782, 767, 776, 784, 790, 817, 785, 761, 779, 720, 784, 740, 769] payload_rows_per_dst=[9404, 9278, 9278, 9318, 9250, 9338, 9308, 9310] dup_rows_per_dst=[3170, 2866, 2990, 2934, 2844, 3006, 3010, 3000] combine_rows_per_src=[9404, 9278, 9278, 9318, 9250, 9338, 9308, 9310] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74334 payload_rows_remote=65030 dup_rows=23970 dup_rows_remote=21016 dedup_ratio=0.243835 router_tokens_per_expert=[762, 729, 829, 774, 783, 713, 729, 757, 786, 780, 722, 786, 758, 750, 735, 761, 790, 730, 758, 765, 776, 757, 798, 755, 745, 823, 786, 796, 735, 788, 818, 762, 793, 757, 767, 752, 746, 832, 752, 762, 770, 797, 731, 784, 715, 757, 774, 766, 777, 803, 776, 764, 741, 814, 754, 784, 784, 755, 782, 751, 796, 748, 769, 763] payload_rows_per_dst=[9212, 9186, 9304, 9428, 9278, 9262, 9332, 9332] dup_rows_per_dst=[2940, 2970, 2954, 3078, 3044, 2926, 3094, 2964] combine_rows_per_src=[9212, 9186, 9304, 9428, 9278, 9262, 9332, 9332] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266698752 dispatch_meta_bytes=344192 combine_payload_bytes=266698752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267042944 ingress_bytes=301301760 link_bytes=301301760 token_bytes=533741696 weight_movement_bytes=34603008 total_nvlink_bytes=568344704 weight_to_token_ratio=0.064831 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133774266e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74038 payload_rows_remote=64726 dup_rows=24266 dup_rows_remote=21280 dedup_ratio=0.246847 router_tokens_per_expert=[762, 776, 750, 760, 811, 778, 761, 760, 759, 768, 768, 789, 740, 760, 752, 789, 773, 733, 743, 800, 784, 773, 769, 780, 780, 791, 757, 792, 746, 819, 774, 758, 754, 759, 729, 730, 837, 760, 768, 812, 787, 770, 762, 759, 770, 753, 714, 778, 782, 783, 772, 763, 748, 766, 820, 704, 748, 745, 759, 773, 767, 760, 773, 792] payload_rows_per_dst=[9288, 9202, 9158, 9370, 9312, 9300, 9164, 9244] dup_rows_per_dst=[3028, 3048, 3152, 3064, 2986, 2886, 3112, 2990] combine_rows_per_src=[9288, 9202, 9158, 9370, 9312, 9300, 9164, 9244] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267304960 dispatch_meta_bytes=345192 combine_payload_bytes=267304960 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267650152 ingress_bytes=301907968 link_bytes=301907968 token_bytes=534955112 weight_movement_bytes=34603008 total_nvlink_bytes=569558120 weight_to_token_ratio=0.064684 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136055378e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265797632 dispatch_meta_bytes=344104 combine_payload_bytes=265797632 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266141736 ingress_bytes=369606656 link_bytes=369606656 token_bytes=531939368 weight_movement_bytes=103809024 total_nvlink_bytes=635748392 weight_to_token_ratio=0.195152 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390800091e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=64942 dup_rows=23940 dup_rows_remote=20822 dedup_ratio=0.243530 router_tokens_per_expert=[812, 817, 791, 798, 763, 745, 758, 786, 743, 767, 745, 787, 776, 757, 721, 788, 765, 790, 734, 757, 749, 753, 763, 795, 747, 765, 789, 734, 833, 806, 773, 765, 728, 808, 760, 784, 790, 718, 792, 762, 776, 743, 757, 752, 760, 764, 764, 763, 820, 742, 768, 751, 744, 798, 754, 734, 758, 763, 772, 774, 758, 798, 724, 801] payload_rows_per_dst=[9422, 9312, 9502, 9322, 9270, 9180, 9282, 9074] dup_rows_per_dst=[3118, 2856, 3174, 3102, 3014, 2978, 2940, 2758] combine_rows_per_src=[9422, 9312, 9502, 9322, 9270, 9180, 9282, 9074] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266706944 dispatch_meta_bytes=343888 combine_payload_bytes=266706944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267050832 ingress_bytes=301309952 link_bytes=301309952 token_bytes=533757776 weight_movement_bytes=34603008 total_nvlink_bytes=568360784 weight_to_token_ratio=0.064829 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133805092e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344184 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266707064 ingress_bytes=300965888 link_bytes=300965888 token_bytes=533069944 weight_movement_bytes=34603008 total_nvlink_bytes=567672952 weight_to_token_ratio=0.064913 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132510407e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265117696 dispatch_meta_bytes=344024 combine_payload_bytes=265117696 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265461720 ingress_bytes=334323712 link_bytes=334323712 token_bytes=530579416 weight_movement_bytes=69206016 total_nvlink_bytes=599785432 weight_to_token_ratio=0.130435 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258033213e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74110 payload_rows_remote=64892 dup_rows=24194 dup_rows_remote=21134 dedup_ratio=0.246114 router_tokens_per_expert=[763, 765, 798, 782, 765, 747, 766, 733, 782, 750, 753, 780, 775, 763, 790, 797, 809, 791, 754, 740, 763, 719, 767, 740, 769, 744, 792, 791, 747, 729, 774, 793, 772, 775, 761, 794, 763, 749, 760, 783, 774, 788, 802, 728, 739, 794, 764, 772, 755, 777, 737, 767, 712, 766, 758, 804, 785, 748, 791, 726, 769, 827, 788, 793] payload_rows_per_dst=[9230, 9340, 9224, 9218, 9264, 9268, 9200, 9366] dup_rows_per_dst=[3008, 3040, 2942, 3060, 3050, 3054, 2952, 3088] combine_rows_per_src=[9230, 9340, 9224, 9218, 9264, 9268, 9200, 9366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74372 payload_rows_remote=65260 dup_rows=23932 dup_rows_remote=21038 dedup_ratio=0.243449 router_tokens_per_expert=[787, 786, 768, 797, 746, 772, 742, 770, 705, 729, 805, 773, 752, 751, 784, 780, 770, 765, 809, 755, 783, 709, 766, 819, 762, 757, 753, 756, 770, 788, 750, 764, 759, 800, 798, 809, 751, 784, 760, 804, 719, 748, 737, 754, 777, 748, 774, 746, 748, 787, 752, 768, 753, 781, 791, 775, 780, 757, 790, 725, 771, 799, 800, 784] payload_rows_per_dst=[9380, 9282, 9312, 9250, 9428, 9112, 9254, 9354] dup_rows_per_dst=[2956, 2876, 3040, 2950, 3102, 2894, 3056, 3058] combine_rows_per_src=[9380, 9282, 9312, 9250, 9428, 9112, 9254, 9354] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74034 payload_rows_remote=64620 dup_rows=24270 dup_rows_remote=21160 dedup_ratio=0.246887 router_tokens_per_expert=[816, 772, 753, 761, 726, 817, 762, 753, 816, 766, 745, 793, 785, 761, 775, 756, 825, 754, 785, 739, 756, 804, 736, 769, 761, 752, 774, 762, 784, 748, 778, 716, 758, 726, 781, 781, 765, 791, 752, 785, 803, 803, 796, 747, 766, 735, 778, 764, 755, 806, 754, 797, 734, 774, 728, 762, 757, 719, 721, 793, 760, 799, 795, 767] payload_rows_per_dst=[9176, 9384, 9104, 9366, 8920, 9614, 9056, 9414] dup_rows_per_dst=[2856, 3076, 2932, 3210, 2854, 3362, 2870, 3110] combine_rows_per_src=[9176, 9384, 9104, 9366, 8920, 9614, 9056, 9414] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=64942 dup_rows=23940 dup_rows_remote=20822 dedup_ratio=0.243530 router_tokens_per_expert=[812, 817, 791, 798, 763, 745, 758, 786, 743, 767, 745, 787, 776, 757, 721, 788, 765, 790, 734, 757, 749, 753, 763, 795, 747, 765, 789, 734, 833, 806, 773, 765, 728, 808, 760, 784, 790, 718, 792, 762, 776, 743, 757, 752, 760, 764, 764, 763, 820, 742, 768, 751, 744, 798, 754, 734, 758, 763, 772, 774, 758, 798, 724, 801] payload_rows_per_dst=[9422, 9312, 9502, 9322, 9270, 9180, 9282, 9074] dup_rows_per_dst=[3118, 2856, 3174, 3102, 3014, 2978, 2940, 2758] combine_rows_per_src=[9422, 9312, 9502, 9322, 9270, 9180, 9282, 9074] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74314 payload_rows_remote=65114 dup_rows=23990 dup_rows_remote=21106 dedup_ratio=0.244039 router_tokens_per_expert=[727, 759, 774, 780, 803, 786, 771, 828, 736, 752, 811, 769, 734, 717, 797, 726, 781, 814, 693, 757, 783, 755, 800, 765, 712, 773, 757, 774, 730, 777, 768, 769, 795, 764, 782, 761, 799, 730, 787, 784, 814, 772, 762, 786, 814, 736, 755, 678, 757, 742, 820, 754, 812, 756, 768, 818, 784, 789, 738, 729, 787, 758, 768, 775] payload_rows_per_dst=[9436, 9200, 9272, 9148, 9382, 9268, 9352, 9256] dup_rows_per_dst=[3020, 2884, 3024, 2972, 3022, 2966, 3102, 3000] combine_rows_per_src=[9436, 9200, 9272, 9148, 9382, 9268, 9352, 9256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266960896 dispatch_meta_bytes=343944 combine_payload_bytes=266960896 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267304840 ingress_bytes=336166912 link_bytes=336166912 token_bytes=534265736 weight_movement_bytes=69206016 total_nvlink_bytes=603471752 weight_to_token_ratio=0.129535 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264969026e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74392 payload_rows_remote=65118 dup_rows=23912 dup_rows_remote=20960 dedup_ratio=0.243245 router_tokens_per_expert=[725, 779, 749, 757, 774, 755, 784, 797, 735, 769, 801, 748, 785, 769, 753, 774, 760, 734, 771, 804, 823, 786, 747, 811, 762, 735, 802, 784, 800, 723, 736, 771, 737, 761, 741, 775, 787, 746, 774, 782, 779, 784, 805, 711, 820, 724, 813, 806, 760, 751, 764, 812, 766, 747, 799, 743, 724, 771, 784, 764, 725, 723, 787, 784] payload_rows_per_dst=[9356, 9266, 9368, 9274, 9262, 9452, 9234, 9180] dup_rows_per_dst=[2884, 3002, 3104, 2952, 2944, 3032, 3050, 2944] combine_rows_per_src=[9356, 9266, 9368, 9274, 9262, 9452, 9234, 9180] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74476 payload_rows_remote=65178 dup_rows=23828 dup_rows_remote=20872 dedup_ratio=0.242391 router_tokens_per_expert=[709, 733, 790, 792, 754, 748, 788, 805, 795, 759, 779, 756, 774, 792, 763, 772, 780, 782, 790, 783, 776, 778, 754, 765, 743, 767, 791, 762, 745, 775, 769, 787, 780, 736, 798, 732, 788, 809, 768, 791, 771, 776, 777, 696, 794, 770, 786, 757, 764, 749, 765, 735, 738, 756, 747, 796, 776, 768, 776, 723, 780, 771, 764, 759] payload_rows_per_dst=[9262, 9372, 9396, 9320, 9408, 9298, 9168, 9252] dup_rows_per_dst=[2976, 3008, 3020, 2958, 2996, 2956, 2932, 2982] combine_rows_per_src=[9262, 9372, 9396, 9320, 9408, 9298, 9168, 9252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344184 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266707064 ingress_bytes=300965888 link_bytes=300965888 token_bytes=533069944 weight_movement_bytes=34603008 total_nvlink_bytes=567672952 weight_to_token_ratio=0.064913 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132510407e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266002432 dispatch_meta_bytes=343056 combine_payload_bytes=266002432 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266345488 ingress_bytes=335208448 link_bytes=335208448 token_bytes=532347920 weight_movement_bytes=69206016 total_nvlink_bytes=601553936 weight_to_token_ratio=0.130001 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261362403e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265797632 dispatch_meta_bytes=344104 combine_payload_bytes=265797632 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266141736 ingress_bytes=369606656 link_bytes=369606656 token_bytes=531939368 weight_movement_bytes=103809024 total_nvlink_bytes=635748392 weight_to_token_ratio=0.195152 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390800091e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267304960 dispatch_meta_bytes=345192 combine_payload_bytes=267304960 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267650152 ingress_bytes=301907968 link_bytes=301907968 token_bytes=534955112 weight_movement_bytes=34603008 total_nvlink_bytes=569558120 weight_to_token_ratio=0.064684 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136055378e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=264683520 dispatch_meta_bytes=343120 combine_payload_bytes=264683520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265026640 ingress_bytes=299286528 link_bytes=299286528 token_bytes=529710160 weight_movement_bytes=34603008 total_nvlink_bytes=564313168 weight_to_token_ratio=0.065324 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126191110e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74314 payload_rows_remote=65114 dup_rows=23990 dup_rows_remote=21106 dedup_ratio=0.244039 router_tokens_per_expert=[727, 759, 774, 780, 803, 786, 771, 828, 736, 752, 811, 769, 734, 717, 797, 726, 781, 814, 693, 757, 783, 755, 800, 765, 712, 773, 757, 774, 730, 777, 768, 769, 795, 764, 782, 761, 799, 730, 787, 784, 814, 772, 762, 786, 814, 736, 755, 678, 757, 742, 820, 754, 812, 756, 768, 818, 784, 789, 738, 729, 787, 758, 768, 775] payload_rows_per_dst=[9436, 9200, 9272, 9148, 9382, 9268, 9352, 9256] dup_rows_per_dst=[3020, 2884, 3024, 2972, 3022, 2966, 3102, 3000] combine_rows_per_src=[9436, 9200, 9272, 9148, 9382, 9268, 9352, 9256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74038 payload_rows_remote=64726 dup_rows=24266 dup_rows_remote=21280 dedup_ratio=0.246847 router_tokens_per_expert=[762, 776, 750, 760, 811, 778, 761, 760, 759, 768, 768, 789, 740, 760, 752, 789, 773, 733, 743, 800, 784, 773, 769, 780, 780, 791, 757, 792, 746, 819, 774, 758, 754, 759, 729, 730, 837, 760, 768, 812, 787, 770, 762, 759, 770, 753, 714, 778, 782, 783, 772, 763, 748, 766, 820, 704, 748, 745, 759, 773, 767, 760, 773, 792] payload_rows_per_dst=[9288, 9202, 9158, 9370, 9312, 9300, 9164, 9244] dup_rows_per_dst=[3028, 3048, 3152, 3064, 2986, 2886, 3112, 2990] combine_rows_per_src=[9288, 9202, 9158, 9370, 9312, 9300, 9164, 9244] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266002432 dispatch_meta_bytes=343056 combine_payload_bytes=266002432 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266345488 ingress_bytes=335208448 link_bytes=335208448 token_bytes=532347920 weight_movement_bytes=69206016 total_nvlink_bytes=601553936 weight_to_token_ratio=0.130001 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261362403e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74484 payload_rows_remote=65176 dup_rows=23820 dup_rows_remote=20810 dedup_ratio=0.242310 router_tokens_per_expert=[752, 788, 806, 788, 800, 803, 766, 784, 749, 739, 760, 745, 792, 773, 778, 736, 760, 764, 793, 783, 731, 775, 744, 784, 740, 736, 747, 773, 774, 802, 769, 785, 775, 706, 808, 736, 765, 741, 746, 770, 786, 785, 797, 785, 778, 732, 759, 750, 748, 760, 752, 782, 767, 776, 784, 790, 817, 785, 761, 779, 720, 784, 740, 769] payload_rows_per_dst=[9404, 9278, 9278, 9318, 9250, 9338, 9308, 9310] dup_rows_per_dst=[3170, 2866, 2990, 2934, 2844, 3006, 3010, 3000] combine_rows_per_src=[9404, 9278, 9278, 9318, 9250, 9338, 9308, 9310] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266706944 dispatch_meta_bytes=344880 combine_payload_bytes=266706944 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267051824 ingress_bytes=335912960 link_bytes=335912960 token_bytes=533758768 weight_movement_bytes=69206016 total_nvlink_bytes=602964784 weight_to_token_ratio=0.129658 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264013425e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=65044 dup_rows=24096 dup_rows_remote=21038 dedup_ratio=0.245117 router_tokens_per_expert=[809, 784, 769, 753, 781, 757, 740, 803, 760, 788, 773, 789, 759, 777, 815, 758, 778, 760, 763, 771, 773, 791, 810, 765, 768, 785, 792, 735, 740, 716, 781, 771, 785, 771, 761, 777, 736, 761, 734, 786, 759, 756, 743, 768, 750, 775, 742, 762, 837, 749, 783, 733, 783, 763, 744, 733, 798, 797, 771, 754, 755, 758, 779, 735] payload_rows_per_dst=[9310, 9348, 9362, 9312, 9164, 9074, 9236, 9402] dup_rows_per_dst=[3082, 3090, 3060, 2864, 3058, 3036, 3014, 2892] combine_rows_per_src=[9310, 9348, 9362, 9312, 9164, 9074, 9236, 9402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74336 payload_rows_remote=65024 dup_rows=23968 dup_rows_remote=20892 dedup_ratio=0.243815 router_tokens_per_expert=[826, 717, 748, 740, 786, 768, 758, 796, 805, 806, 780, 764, 713, 751, 784, 785, 794, 776, 814, 749, 698, 790, 763, 732, 763, 746, 767, 835, 807, 748, 792, 732, 757, 754, 784, 818, 736, 782, 773, 729, 773, 791, 745, 783, 713, 746, 734, 779, 773, 780, 768, 814, 777, 749, 784, 749, 791, 785, 751, 749, 785, 742, 764, 761] payload_rows_per_dst=[9294, 9332, 9280, 9304, 9280, 9266, 9312, 9268] dup_rows_per_dst=[2984, 3044, 2952, 3076, 2986, 2862, 3076, 2988] combine_rows_per_src=[9294, 9332, 9280, 9304, 9280, 9266, 9312, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=65054 dup_rows=23930 dup_rows_remote=20840 dedup_ratio=0.243429 router_tokens_per_expert=[780, 779, 784, 809, 762, 762, 749, 812, 762, 739, 738, 755, 793, 713, 741, 756, 764, 820, 788, 775, 746, 778, 754, 780, 749, 799, 753, 709, 780, 710, 756, 755, 789, 797, 768, 725, 784, 798, 752, 759, 773, 739, 760, 742, 809, 800, 798, 767, 772, 767, 784, 757, 824, 758, 789, 780, 755, 739, 775, 746, 746, 793, 778, 779] payload_rows_per_dst=[9382, 9138, 9320, 9138, 9384, 9326, 9448, 9238] dup_rows_per_dst=[3092, 2856, 3090, 2884, 2960, 3050, 3014, 2984] combine_rows_per_src=[9382, 9138, 9320, 9138, 9384, 9326, 9448, 9238] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266723328 dispatch_meta_bytes=344312 combine_payload_bytes=266723328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267067640 ingress_bytes=301326336 link_bytes=301326336 token_bytes=533790968 weight_movement_bytes=34603008 total_nvlink_bytes=568393976 weight_to_token_ratio=0.064825 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133866744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266969088 dispatch_meta_bytes=344200 combine_payload_bytes=266969088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267313288 ingress_bytes=301572096 link_bytes=301572096 token_bytes=534282376 weight_movement_bytes=34603008 total_nvlink_bytes=568885384 weight_to_token_ratio=0.064765 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134791519e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266706944 dispatch_meta_bytes=344880 combine_payload_bytes=266706944 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267051824 ingress_bytes=335912960 link_bytes=335912960 token_bytes=533758768 weight_movement_bytes=69206016 total_nvlink_bytes=602964784 weight_to_token_ratio=0.129658 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264013425e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265117696 dispatch_meta_bytes=344024 combine_payload_bytes=265117696 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265461720 ingress_bytes=334323712 link_bytes=334323712 token_bytes=530579416 weight_movement_bytes=69206016 total_nvlink_bytes=599785432 weight_to_token_ratio=0.130435 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258033213e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266960896 dispatch_meta_bytes=343944 combine_payload_bytes=266960896 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267304840 ingress_bytes=336166912 link_bytes=336166912 token_bytes=534265736 weight_movement_bytes=69206016 total_nvlink_bytes=603471752 weight_to_token_ratio=0.129535 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264969026e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=344328 combine_payload_bytes=266420224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266764552 ingress_bytes=301023232 link_bytes=301023232 token_bytes=533184776 weight_movement_bytes=34603008 total_nvlink_bytes=567787784 weight_to_token_ratio=0.064899 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132726188e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=343576 combine_payload_bytes=266461184 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266804760 ingress_bytes=335667200 link_bytes=335667200 token_bytes=533265944 weight_movement_bytes=69206016 total_nvlink_bytes=602471960 weight_to_token_ratio=0.129778 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263088650e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74316 payload_rows_remote=64816 dup_rows=23988 dup_rows_remote=20922 dedup_ratio=0.244019 router_tokens_per_expert=[753, 777, 776, 811, 778, 786, 753, 749, 777, 709, 774, 804, 799, 747, 762, 754, 773, 731, 822, 719, 770, 773, 737, 772, 747, 759, 747, 764, 775, 785, 773, 824, 765, 753, 772, 785, 753, 745, 767, 788, 786, 812, 778, 775, 723, 755, 755, 765, 726, 770, 770, 778, 762, 696, 788, 770, 833, 734, 772, 773, 790, 807, 775, 751] payload_rows_per_dst=[9172, 9014, 9258, 9560, 9280, 9260, 9272, 9500] dup_rows_per_dst=[2878, 2846, 2958, 3180, 3068, 3038, 2954, 3066] combine_rows_per_src=[9172, 9014, 9258, 9560, 9280, 9260, 9272, 9500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74476 payload_rows_remote=65178 dup_rows=23828 dup_rows_remote=20872 dedup_ratio=0.242391 router_tokens_per_expert=[709, 733, 790, 792, 754, 748, 788, 805, 795, 759, 779, 756, 774, 792, 763, 772, 780, 782, 790, 783, 776, 778, 754, 765, 743, 767, 791, 762, 745, 775, 769, 787, 780, 736, 798, 732, 788, 809, 768, 791, 771, 776, 777, 696, 794, 770, 786, 757, 764, 749, 765, 735, 738, 756, 747, 796, 776, 768, 776, 723, 780, 771, 764, 759] payload_rows_per_dst=[9262, 9372, 9396, 9320, 9408, 9298, 9168, 9252] dup_rows_per_dst=[2976, 3008, 3020, 2958, 2996, 2956, 2932, 2982] combine_rows_per_src=[9262, 9372, 9396, 9320, 9408, 9298, 9168, 9252] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74278 payload_rows_remote=65056 dup_rows=24026 dup_rows_remote=21154 dedup_ratio=0.244405 router_tokens_per_expert=[742, 740, 778, 757, 742, 745, 770, 773, 806, 791, 829, 767, 783, 760, 779, 756, 753, 782, 797, 745, 737, 759, 770, 764, 787, 754, 746, 773, 761, 724, 747, 746, 804, 820, 776, 813, 772, 760, 771, 779, 751, 767, 768, 720, 782, 764, 713, 768, 743, 776, 798, 804, 795, 774, 780, 745, 775, 743, 781, 768, 811, 767, 772, 729] payload_rows_per_dst=[9222, 9416, 9174, 9192, 9382, 9204, 9430, 9258] dup_rows_per_dst=[2872, 3126, 3040, 2884, 3208, 2862, 3000, 3034] combine_rows_per_src=[9222, 9416, 9174, 9192, 9382, 9204, 9430, 9258] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74174 payload_rows_remote=64706 dup_rows=24130 dup_rows_remote=20972 dedup_ratio=0.245463 router_tokens_per_expert=[758, 773, 764, 792, 751, 754, 788, 755, 807, 711, 790, 835, 771, 846, 747, 806, 718, 759, 713, 736, 745, 729, 744, 769, 769, 793, 769, 829, 755, 721, 775, 741, 723, 785, 738, 789, 818, 743, 799, 774, 769, 761, 817, 807, 754, 746, 781, 730, 752, 758, 719, 755, 787, 764, 771, 765, 788, 762, 800, 762, 760, 814, 771, 777] payload_rows_per_dst=[9214, 9468, 8946, 9266, 9348, 9354, 9250, 9328] dup_rows_per_dst=[3056, 3158, 2880, 3038, 2990, 2976, 2892, 3140] combine_rows_per_src=[9214, 9468, 8946, 9266, 9348, 9354, 9250, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74392 payload_rows_remote=65118 dup_rows=23912 dup_rows_remote=20960 dedup_ratio=0.243245 router_tokens_per_expert=[725, 779, 749, 757, 774, 755, 784, 797, 735, 769, 801, 748, 785, 769, 753, 774, 760, 734, 771, 804, 823, 786, 747, 811, 762, 735, 802, 784, 800, 723, 736, 771, 737, 761, 741, 775, 787, 746, 774, 782, 779, 784, 805, 711, 820, 724, 813, 806, 760, 751, 764, 812, 766, 747, 799, 743, 724, 771, 784, 764, 725, 723, 787, 784] payload_rows_per_dst=[9356, 9266, 9368, 9274, 9262, 9452, 9234, 9180] dup_rows_per_dst=[2884, 3002, 3104, 2952, 2944, 3032, 3050, 2944] combine_rows_per_src=[9356, 9266, 9368, 9274, 9262, 9452, 9234, 9180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74316 payload_rows_remote=64816 dup_rows=23988 dup_rows_remote=20922 dedup_ratio=0.244019 router_tokens_per_expert=[753, 777, 776, 811, 778, 786, 753, 749, 777, 709, 774, 804, 799, 747, 762, 754, 773, 731, 822, 719, 770, 773, 737, 772, 747, 759, 747, 764, 775, 785, 773, 824, 765, 753, 772, 785, 753, 745, 767, 788, 786, 812, 778, 775, 723, 755, 755, 765, 726, 770, 770, 778, 762, 696, 788, 770, 833, 734, 772, 773, 790, 807, 775, 751] payload_rows_per_dst=[9172, 9014, 9258, 9560, 9280, 9260, 9272, 9500] dup_rows_per_dst=[2878, 2846, 2958, 3180, 3068, 3038, 2954, 3066] combine_rows_per_src=[9172, 9014, 9258, 9560, 9280, 9260, 9272, 9500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=343664 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266681968 ingress_bytes=335544320 link_bytes=335544320 token_bytes=533020272 weight_movement_bytes=69206016 total_nvlink_bytes=602226288 weight_to_token_ratio=0.129837 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262626263e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=343576 combine_payload_bytes=266461184 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266804760 ingress_bytes=335667200 link_bytes=335667200 token_bytes=533265944 weight_movement_bytes=69206016 total_nvlink_bytes=602471960 weight_to_token_ratio=0.129778 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263088650e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266969088 dispatch_meta_bytes=344200 combine_payload_bytes=266969088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267313288 ingress_bytes=301572096 link_bytes=301572096 token_bytes=534282376 weight_movement_bytes=34603008 total_nvlink_bytes=568885384 weight_to_token_ratio=0.064765 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134791519e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266469376 dispatch_meta_bytes=344840 combine_payload_bytes=266469376 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266814216 ingress_bytes=404881408 link_bytes=404881408 token_bytes=533283592 weight_movement_bytes=138412032 total_nvlink_bytes=671695624 weight_to_token_ratio=0.259547 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523536143e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266723328 dispatch_meta_bytes=344312 combine_payload_bytes=266723328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267067640 ingress_bytes=301326336 link_bytes=301326336 token_bytes=533790968 weight_movement_bytes=34603008 total_nvlink_bytes=568393976 weight_to_token_ratio=0.064825 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133866744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265486336 dispatch_meta_bytes=342952 combine_payload_bytes=265486336 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265829288 ingress_bytes=334692352 link_bytes=334692352 token_bytes=531315624 weight_movement_bytes=69206016 total_nvlink_bytes=600521640 weight_to_token_ratio=0.130254 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259420376e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74494 payload_rows_remote=65274 dup_rows=23810 dup_rows_remote=20878 dedup_ratio=0.242208 router_tokens_per_expert=[759, 773, 775, 807, 774, 730, 787, 720, 785, 755, 810, 765, 788, 761, 765, 774, 792, 777, 725, 799, 728, 749, 766, 777, 764, 729, 789, 751, 750, 789, 748, 756, 787, 801, 752, 753, 765, 792, 771, 773, 745, 723, 805, 765, 726, 781, 788, 752, 790, 785, 759, 771, 730, 775, 793, 807, 760, 798, 784, 770, 760, 729, 771, 774] payload_rows_per_dst=[9326, 9364, 9332, 9220, 9376, 9252, 9328, 9296] dup_rows_per_dst=[2924, 3042, 2894, 2932, 3012, 2918, 3092, 2996] combine_rows_per_src=[9326, 9364, 9332, 9220, 9376, 9252, 9328, 9296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74174 payload_rows_remote=64706 dup_rows=24130 dup_rows_remote=20972 dedup_ratio=0.245463 router_tokens_per_expert=[758, 773, 764, 792, 751, 754, 788, 755, 807, 711, 790, 835, 771, 846, 747, 806, 718, 759, 713, 736, 745, 729, 744, 769, 769, 793, 769, 829, 755, 721, 775, 741, 723, 785, 738, 789, 818, 743, 799, 774, 769, 761, 817, 807, 754, 746, 781, 730, 752, 758, 719, 755, 787, 764, 771, 765, 788, 762, 800, 762, 760, 814, 771, 777] payload_rows_per_dst=[9214, 9468, 8946, 9266, 9348, 9354, 9250, 9328] dup_rows_per_dst=[3056, 3158, 2880, 3038, 2990, 2976, 2892, 3140] combine_rows_per_src=[9214, 9468, 8946, 9266, 9348, 9354, 9250, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=65044 dup_rows=24096 dup_rows_remote=21038 dedup_ratio=0.245117 router_tokens_per_expert=[809, 784, 769, 753, 781, 757, 740, 803, 760, 788, 773, 789, 759, 777, 815, 758, 778, 760, 763, 771, 773, 791, 810, 765, 768, 785, 792, 735, 740, 716, 781, 771, 785, 771, 761, 777, 736, 761, 734, 786, 759, 756, 743, 768, 750, 775, 742, 762, 837, 749, 783, 733, 783, 763, 744, 733, 798, 797, 771, 754, 755, 758, 779, 735] payload_rows_per_dst=[9310, 9348, 9362, 9312, 9164, 9074, 9236, 9402] dup_rows_per_dst=[3082, 3090, 3060, 2864, 3058, 3036, 3014, 2892] combine_rows_per_src=[9310, 9348, 9362, 9312, 9164, 9074, 9236, 9402] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74468 payload_rows_remote=65134 dup_rows=23836 dup_rows_remote=20792 dedup_ratio=0.242472 router_tokens_per_expert=[740, 793, 750, 789, 763, 756, 753, 740, 729, 728, 737, 788, 745, 751, 815, 805, 743, 760, 768, 756, 760, 782, 750, 814, 824, 765, 759, 772, 754, 762, 766, 776, 763, 795, 792, 759, 812, 767, 743, 737, 774, 807, 748, 777, 742, 767, 797, 777, 747, 766, 763, 765, 786, 737, 710, 802, 796, 750, 784, 754, 830, 762, 782, 768] payload_rows_per_dst=[9258, 9252, 9250, 9360, 9344, 9334, 9270, 9400] dup_rows_per_dst=[2910, 2944, 3016, 2996, 2992, 3044, 2882, 3052] combine_rows_per_src=[9258, 9252, 9250, 9360, 9344, 9334, 9270, 9400] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266469376 dispatch_meta_bytes=344840 combine_payload_bytes=266469376 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266814216 ingress_bytes=404881408 link_bytes=404881408 token_bytes=533283592 weight_movement_bytes=138412032 total_nvlink_bytes=671695624 weight_to_token_ratio=0.259547 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523536143e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74336 payload_rows_remote=65024 dup_rows=23968 dup_rows_remote=20892 dedup_ratio=0.243815 router_tokens_per_expert=[826, 717, 748, 740, 786, 768, 758, 796, 805, 806, 780, 764, 713, 751, 784, 785, 794, 776, 814, 749, 698, 790, 763, 732, 763, 746, 767, 835, 807, 748, 792, 732, 757, 754, 784, 818, 736, 782, 773, 729, 773, 791, 745, 783, 713, 746, 734, 779, 773, 780, 768, 814, 777, 749, 784, 749, 791, 785, 751, 749, 785, 742, 764, 761] payload_rows_per_dst=[9294, 9332, 9280, 9304, 9280, 9266, 9312, 9268] dup_rows_per_dst=[2984, 3044, 2952, 3076, 2986, 2862, 3076, 2988] combine_rows_per_src=[9294, 9332, 9280, 9304, 9280, 9266, 9312, 9268] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265035776 dispatch_meta_bytes=342712 combine_payload_bytes=265035776 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265378488 ingress_bytes=368844800 link_bytes=368844800 token_bytes=530414264 weight_movement_bytes=103809024 total_nvlink_bytes=634223288 weight_to_token_ratio=0.195713 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.387933288e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=344328 combine_payload_bytes=266420224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266764552 ingress_bytes=301023232 link_bytes=301023232 token_bytes=533184776 weight_movement_bytes=34603008 total_nvlink_bytes=567787784 weight_to_token_ratio=0.064899 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132726188e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74420 payload_rows_remote=65120 dup_rows=23884 dup_rows_remote=20980 dedup_ratio=0.242961 router_tokens_per_expert=[715, 792, 744, 782, 790, 809, 751, 755, 731, 782, 755, 778, 741, 750, 771, 803, 778, 764, 781, 790, 783, 739, 731, 736, 765, 781, 783, 782, 769, 792, 742, 750, 756, 758, 752, 763, 783, 766, 765, 765, 747, 825, 754, 741, 751, 746, 788, 777, 758, 760, 791, 807, 700, 841, 754, 757, 761, 811, 785, 803, 789, 747, 769, 767] payload_rows_per_dst=[9170, 9220, 9300, 9318, 9224, 9318, 9398, 9472] dup_rows_per_dst=[3106, 3002, 2904, 3010, 2992, 2940, 2938, 2992] combine_rows_per_src=[9170, 9220, 9300, 9318, 9224, 9318, 9398, 9472] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=65030 dup_rows=24012 dup_rows_remote=21014 dedup_ratio=0.244263 router_tokens_per_expert=[771, 765, 758, 705, 817, 775, 741, 781, 760, 799, 756, 765, 725, 744, 733, 829, 772, 743, 744, 798, 805, 780, 736, 723, 770, 750, 757, 814, 764, 742, 781, 745, 752, 785, 790, 765, 719, 796, 744, 779, 779, 810, 745, 733, 741, 812, 723, 744, 832, 766, 789, 808, 793, 714, 815, 773, 784, 783, 775, 809, 750, 776, 732, 788] payload_rows_per_dst=[9250, 9268, 9252, 9316, 9262, 9184, 9436, 9324] dup_rows_per_dst=[2976, 2954, 2950, 2930, 2998, 2990, 3144, 3070] combine_rows_per_src=[9250, 9268, 9252, 9316, 9262, 9184, 9436, 9324] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=267362304 dispatch_meta_bytes=344608 combine_payload_bytes=267362304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267706912 ingress_bytes=301965312 link_bytes=301965312 token_bytes=535069216 weight_movement_bytes=34603008 total_nvlink_bytes=569672224 weight_to_token_ratio=0.064670 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136271159e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=65086 dup_rows=24032 dup_rows_remote=21112 dedup_ratio=0.244466 router_tokens_per_expert=[753, 761, 756, 816, 754, 759, 779, 745, 747, 771, 785, 781, 750, 739, 788, 796, 751, 778, 784, 750, 753, 774, 848, 748, 775, 843, 767, 711, 781, 748, 759, 767, 792, 742, 822, 750, 758, 786, 781, 760, 753, 751, 744, 741, 808, 764, 712, 760, 755, 772, 777, 749, 749, 708, 770, 773, 781, 780, 808, 752, 835, 757, 760, 785] payload_rows_per_dst=[9226, 9266, 9276, 9284, 9330, 9162, 9186, 9542] dup_rows_per_dst=[3020, 3048, 3096, 3018, 3052, 2904, 2920, 2974] combine_rows_per_src=[9226, 9266, 9276, 9284, 9330, 9162, 9186, 9542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74468 payload_rows_remote=65134 dup_rows=23836 dup_rows_remote=20792 dedup_ratio=0.242472 router_tokens_per_expert=[740, 793, 750, 789, 763, 756, 753, 740, 729, 728, 737, 788, 745, 751, 815, 805, 743, 760, 768, 756, 760, 782, 750, 814, 824, 765, 759, 772, 754, 762, 766, 776, 763, 795, 792, 759, 812, 767, 743, 737, 774, 807, 748, 777, 742, 767, 797, 777, 747, 766, 763, 765, 786, 737, 710, 802, 796, 750, 784, 754, 830, 762, 782, 768] payload_rows_per_dst=[9258, 9252, 9250, 9360, 9344, 9334, 9270, 9400] dup_rows_per_dst=[2910, 2944, 3016, 2996, 2992, 3044, 2882, 3052] combine_rows_per_src=[9258, 9252, 9250, 9360, 9344, 9334, 9270, 9400] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=343704 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267132568 ingress_bytes=370597888 link_bytes=370597888 token_bytes=533921432 weight_movement_bytes=103809024 total_nvlink_bytes=637730456 weight_to_token_ratio=0.194428 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394530017e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=343664 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266681968 ingress_bytes=335544320 link_bytes=335544320 token_bytes=533020272 weight_movement_bytes=69206016 total_nvlink_bytes=602226288 weight_to_token_ratio=0.129837 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262626263e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12259773720 prefetch_bytes=726663168 reduce_bytes=519045120 weight_movement_bytes=1245708288 egress_bytes=6133845272 ingress_bytes=7371636736 link_bytes=7371636736 total_nvlink_bytes=13505482008 weight_to_token_ratio=0.101609 prefetch_share_of_ingress=0.098576 dedup_ratio=0.244296 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=15 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028043 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.206038068e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=344400 combine_payload_bytes=266731520 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267075920 ingress_bytes=335937536 link_bytes=335937536 token_bytes=533807440 weight_movement_bytes=69206016 total_nvlink_bytes=603013456 weight_to_token_ratio=0.129646 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264105903e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74230 payload_rows_remote=65024 dup_rows=24074 dup_rows_remote=21146 dedup_ratio=0.244893 router_tokens_per_expert=[781, 749, 795, 770, 742, 751, 782, 755, 795, 797, 797, 785, 812, 776, 786, 772, 811, 740, 791, 747, 757, 793, 755, 765, 783, 800, 757, 800, 764, 795, 767, 779, 761, 742, 775, 759, 786, 765, 742, 797, 778, 725, 782, 789, 749, 740, 772, 765, 745, 760, 817, 749, 717, 780, 737, 718, 762, 743, 756, 753, 799, 740, 726, 774] payload_rows_per_dst=[9274, 9198, 9260, 9360, 9312, 9290, 9330, 9206] dup_rows_per_dst=[2976, 2942, 3130, 3130, 2910, 2914, 3144, 2928] combine_rows_per_src=[9274, 9198, 9260, 9360, 9312, 9290, 9330, 9206] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344176 combine_payload_bytes=266362880 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266707056 ingress_bytes=335568896 link_bytes=335568896 token_bytes=533069936 weight_movement_bytes=69206016 total_nvlink_bytes=602275952 weight_to_token_ratio=0.129825 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262718740e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74494 payload_rows_remote=65274 dup_rows=23810 dup_rows_remote=20878 dedup_ratio=0.242208 router_tokens_per_expert=[759, 773, 775, 807, 774, 730, 787, 720, 785, 755, 810, 765, 788, 761, 765, 774, 792, 777, 725, 799, 728, 749, 766, 777, 764, 729, 789, 751, 750, 789, 748, 756, 787, 801, 752, 753, 765, 792, 771, 773, 745, 723, 805, 765, 726, 781, 788, 752, 790, 785, 759, 771, 730, 775, 793, 807, 760, 798, 784, 770, 760, 729, 771, 774] payload_rows_per_dst=[9326, 9364, 9332, 9220, 9376, 9252, 9328, 9296] dup_rows_per_dst=[2924, 3042, 2894, 2932, 3012, 2918, 3092, 2996] combine_rows_per_src=[9326, 9364, 9332, 9220, 9376, 9252, 9328, 9296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12259773720 prefetch_bytes=726663168 reduce_bytes=519045120 weight_movement_bytes=1245708288 egress_bytes=6133845272 ingress_bytes=7371636736 link_bytes=7371636736 total_nvlink_bytes=13505482008 weight_to_token_ratio=0.101609 prefetch_share_of_ingress=0.098576 dedup_ratio=0.244296 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=15 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028043 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.206038068e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74430 payload_rows_remote=65034 dup_rows=23874 dup_rows_remote=20874 dedup_ratio=0.242859 router_tokens_per_expert=[756, 799, 837, 771, 792, 717, 746, 780, 764, 753, 766, 811, 821, 785, 755, 812, 768, 799, 749, 760, 740, 728, 747, 818, 779, 796, 768, 762, 701, 769, 783, 746, 752, 748, 720, 730, 759, 730, 782, 749, 754, 758, 781, 734, 792, 752, 817, 772, 776, 757, 791, 766, 730, 827, 802, 757, 771, 777, 761, 760, 747, 772, 764, 786] payload_rows_per_dst=[9396, 9444, 9354, 9208, 9080, 9620, 9394, 8934] dup_rows_per_dst=[3000, 3156, 2798, 3000, 2860, 3294, 3018, 2748] combine_rows_per_src=[9396, 9444, 9354, 9208, 9080, 9620, 9394, 8934] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=344400 combine_payload_bytes=266731520 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267075920 ingress_bytes=335937536 link_bytes=335937536 token_bytes=533807440 weight_movement_bytes=69206016 total_nvlink_bytes=603013456 weight_to_token_ratio=0.129646 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264105903e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266592256 dispatch_meta_bytes=344792 combine_payload_bytes=266592256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266937048 ingress_bytes=301195264 link_bytes=301195264 token_bytes=533529304 weight_movement_bytes=34603008 total_nvlink_bytes=568132312 weight_to_token_ratio=0.064857 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133373530e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12264870264 prefetch_bytes=657457152 reduce_bytes=484442112 weight_movement_bytes=1141899264 egress_bytes=6136394104 ingress_bytes=7270375424 link_bytes=7270375424 total_nvlink_bytes=13406769528 weight_to_token_ratio=0.093103 prefetch_share_of_ingress=0.090430 dedup_ratio=0.244031 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=14 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029022 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.189471192e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=343704 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267132568 ingress_bytes=370597888 link_bytes=370597888 token_bytes=533921432 weight_movement_bytes=103809024 total_nvlink_bytes=637730456 weight_to_token_ratio=0.194428 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394530017e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=344680 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266682984 ingress_bytes=335544320 link_bytes=335544320 token_bytes=533021288 weight_movement_bytes=69206016 total_nvlink_bytes=602227304 weight_to_token_ratio=0.129837 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262626263e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12251498072 prefetch_bytes=692060160 reduce_bytes=622854144 weight_movement_bytes=1314914304 egress_bytes=6129706584 ingress_bytes=7436705792 link_bytes=7436705792 total_nvlink_bytes=13566412376 weight_to_token_ratio=0.107327 prefetch_share_of_ingress=0.093060 dedup_ratio=0.244757 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=18 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.216683704e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74440 payload_rows_remote=65244 dup_rows=23864 dup_rows_remote=20986 dedup_ratio=0.242757 router_tokens_per_expert=[728, 733, 784, 728, 791, 779, 806, 771, 787, 745, 705, 790, 777, 744, 701, 779, 745, 793, 830, 747, 778, 778, 734, 787, 717, 750, 761, 715, 744, 787, 802, 761, 786, 777, 803, 742, 768, 768, 759, 794, 779, 782, 745, 776, 752, 796, 786, 788, 713, 748, 781, 759, 755, 777, 805, 792, 783, 783, 771, 756, 759, 828, 771, 793] payload_rows_per_dst=[9266, 9208, 9404, 9196, 9302, 9354, 9322, 9388] dup_rows_per_dst=[2974, 2848, 2980, 2878, 3092, 3054, 2938, 3100] combine_rows_per_src=[9266, 9208, 9404, 9196, 9302, 9354, 9322, 9388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=267362304 dispatch_meta_bytes=344608 combine_payload_bytes=267362304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267706912 ingress_bytes=301965312 link_bytes=301965312 token_bytes=535069216 weight_movement_bytes=34603008 total_nvlink_bytes=569672224 weight_to_token_ratio=0.064670 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136271159e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=65030 dup_rows=24012 dup_rows_remote=21014 dedup_ratio=0.244263 router_tokens_per_expert=[771, 765, 758, 705, 817, 775, 741, 781, 760, 799, 756, 765, 725, 744, 733, 829, 772, 743, 744, 798, 805, 780, 736, 723, 770, 750, 757, 814, 764, 742, 781, 745, 752, 785, 790, 765, 719, 796, 744, 779, 779, 810, 745, 733, 741, 812, 723, 744, 832, 766, 789, 808, 793, 714, 815, 773, 784, 783, 775, 809, 750, 776, 732, 788] payload_rows_per_dst=[9250, 9268, 9252, 9316, 9262, 9184, 9436, 9324] dup_rows_per_dst=[2976, 2954, 2950, 2930, 2998, 2990, 3144, 3070] combine_rows_per_src=[9250, 9268, 9252, 9316, 9262, 9184, 9436, 9324] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266379264 dispatch_meta_bytes=343632 combine_payload_bytes=266379264 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266722896 ingress_bytes=404791296 link_bytes=404791296 token_bytes=533102160 weight_movement_bytes=138412032 total_nvlink_bytes=671514192 weight_to_token_ratio=0.259635 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523197058e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=65086 dup_rows=24032 dup_rows_remote=21112 dedup_ratio=0.244466 router_tokens_per_expert=[753, 761, 756, 816, 754, 759, 779, 745, 747, 771, 785, 781, 750, 739, 788, 796, 751, 778, 784, 750, 753, 774, 848, 748, 775, 843, 767, 711, 781, 748, 759, 767, 792, 742, 822, 750, 758, 786, 781, 760, 753, 751, 744, 741, 808, 764, 712, 760, 755, 772, 777, 749, 749, 708, 770, 773, 781, 780, 808, 752, 835, 757, 760, 785] payload_rows_per_dst=[9226, 9266, 9276, 9284, 9330, 9162, 9186, 9542] dup_rows_per_dst=[3020, 3048, 3096, 3018, 3052, 2904, 2920, 2974] combine_rows_per_src=[9226, 9266, 9276, 9284, 9330, 9162, 9186, 9542] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12264870264 prefetch_bytes=657457152 reduce_bytes=484442112 weight_movement_bytes=1141899264 egress_bytes=6136394104 ingress_bytes=7270375424 link_bytes=7270375424 total_nvlink_bytes=13406769528 weight_to_token_ratio=0.093103 prefetch_share_of_ingress=0.090430 dedup_ratio=0.244031 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=14 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029022 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.189471192e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12235805256 prefetch_bytes=726663168 reduce_bytes=692060160 weight_movement_bytes=1418723328 egress_bytes=6121853512 ingress_bytes=7532675072 link_bytes=7532675072 total_nvlink_bytes=13654528584 weight_to_token_ratio=0.115949 prefetch_share_of_ingress=0.096468 dedup_ratio=0.244628 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=20 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028370 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.232384777e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344176 combine_payload_bytes=266362880 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266707056 ingress_bytes=335568896 link_bytes=335568896 token_bytes=533069936 weight_movement_bytes=69206016 total_nvlink_bytes=602275952 weight_to_token_ratio=0.129825 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262718740e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74472 payload_rows_remote=64928 dup_rows=23832 dup_rows_remote=20834 dedup_ratio=0.242432 router_tokens_per_expert=[747, 751, 770, 771, 800, 759, 766, 807, 757, 761, 730, 776, 790, 765, 797, 762, 799, 827, 739, 795, 763, 787, 776, 785, 712, 749, 771, 762, 729, 726, 822, 773, 765, 744, 774, 789, 760, 755, 767, 784, 768, 791, 798, 755, 770, 773, 796, 765, 796, 742, 766, 742, 707, 759, 774, 788, 789, 779, 737, 755, 732, 758, 760, 790] payload_rows_per_dst=[9276, 9328, 9544, 9216, 9284, 9486, 9170, 9168] dup_rows_per_dst=[3066, 2948, 2998, 2872, 2992, 2946, 2978, 3032] combine_rows_per_src=[9276, 9328, 9544, 9216, 9284, 9486, 9170, 9168] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=267239424 dispatch_meta_bytes=344920 combine_payload_bytes=267239424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267584344 ingress_bytes=301842432 link_bytes=301842432 token_bytes=534823768 weight_movement_bytes=34603008 total_nvlink_bytes=569426776 weight_to_token_ratio=0.064700 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135808771e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74614 payload_rows_remote=65202 dup_rows=23690 dup_rows_remote=20652 dedup_ratio=0.240987 router_tokens_per_expert=[800, 721, 742, 794, 740, 784, 733, 775, 734, 780, 742, 788, 764, 805, 726, 754, 770, 808, 734, 789, 819, 719, 784, 815, 768, 769, 788, 771, 726, 788, 769, 761, 782, 795, 769, 752, 739, 731, 771, 775, 744, 765, 779, 776, 778, 768, 805, 775, 764, 776, 797, 742, 781, 797, 784, 784, 749, 768, 776, 760, 717, 738, 754, 801] payload_rows_per_dst=[9260, 9248, 9468, 9344, 9298, 9360, 9412, 9224] dup_rows_per_dst=[2918, 2938, 3008, 2936, 2930, 3020, 3038, 2902] combine_rows_per_src=[9260, 9248, 9468, 9344, 9298, 9360, 9412, 9224] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266592256 dispatch_meta_bytes=344792 combine_payload_bytes=266592256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266937048 ingress_bytes=301195264 link_bytes=301195264 token_bytes=533529304 weight_movement_bytes=34603008 total_nvlink_bytes=568132312 weight_to_token_ratio=0.064857 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133373530e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74440 payload_rows_remote=65244 dup_rows=23864 dup_rows_remote=20986 dedup_ratio=0.242757 router_tokens_per_expert=[728, 733, 784, 728, 791, 779, 806, 771, 787, 745, 705, 790, 777, 744, 701, 779, 745, 793, 830, 747, 778, 778, 734, 787, 717, 750, 761, 715, 744, 787, 802, 761, 786, 777, 803, 742, 768, 768, 759, 794, 779, 782, 745, 776, 752, 796, 786, 788, 713, 748, 781, 759, 755, 777, 805, 792, 783, 783, 771, 756, 759, 828, 771, 793] payload_rows_per_dst=[9266, 9208, 9404, 9196, 9302, 9354, 9322, 9388] dup_rows_per_dst=[2974, 2848, 2980, 2878, 3092, 3054, 2938, 3100] combine_rows_per_src=[9266, 9208, 9404, 9196, 9302, 9354, 9322, 9388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12251498072 prefetch_bytes=692060160 reduce_bytes=622854144 weight_movement_bytes=1314914304 egress_bytes=6129706584 ingress_bytes=7436705792 link_bytes=7436705792 total_nvlink_bytes=13566412376 weight_to_token_ratio=0.107327 prefetch_share_of_ingress=0.093060 dedup_ratio=0.244757 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=18 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.216683704e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265945088 dispatch_meta_bytes=343048 combine_payload_bytes=265945088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266288136 ingress_bytes=300548096 link_bytes=300548096 token_bytes=532233224 weight_movement_bytes=34603008 total_nvlink_bytes=566836232 weight_to_token_ratio=0.065015 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130938289e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12267554288 prefetch_bytes=795869184 reduce_bytes=622854144 weight_movement_bytes=1418723328 egress_bytes=6137734640 ingress_bytes=7548542976 link_bytes=7548542976 total_nvlink_bytes=13686277616 weight_to_token_ratio=0.115648 prefetch_share_of_ingress=0.105433 dedup_ratio=0.243694 prefetch_slots=46 unused_prefetch_slots=322 slot_utilization=0.125000 reduce_slots=18 remote_demand=23 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027717 gemm_inner_pad_upper_bound=29210 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.234980848e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=267067392 dispatch_meta_bytes=343416 combine_payload_bytes=267067392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267410808 ingress_bytes=301670400 link_bytes=301670400 token_bytes=534478200 weight_movement_bytes=34603008 total_nvlink_bytes=569081208 weight_to_token_ratio=0.064742 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135161429e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74370 payload_rows_remote=64832 dup_rows=23934 dup_rows_remote=20842 dedup_ratio=0.243469 router_tokens_per_expert=[753, 801, 768, 740, 741, 834, 766, 724, 803, 829, 817, 758, 783, 770, 775, 758, 799, 786, 704, 802, 783, 737, 747, 751, 727, 774, 748, 787, 732, 776, 770, 805, 782, 801, 789, 778, 734, 741, 748, 752, 785, 770, 793, 741, 785, 750, 740, 757, 759, 718, 772, 746, 754, 769, 768, 763, 796, 787, 781, 768, 757, 750, 807, 763] payload_rows_per_dst=[9538, 9504, 9184, 9412, 9302, 9184, 9174, 9072] dup_rows_per_dst=[3092, 3098, 3034, 3054, 2948, 2814, 2924, 2970] combine_rows_per_src=[9538, 9504, 9184, 9412, 9302, 9184, 9174, 9072] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=267239424 dispatch_meta_bytes=344920 combine_payload_bytes=267239424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267584344 ingress_bytes=301842432 link_bytes=301842432 token_bytes=534823768 weight_movement_bytes=34603008 total_nvlink_bytes=569426776 weight_to_token_ratio=0.064700 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135808771e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12254394568 prefetch_bytes=692060160 reduce_bytes=657457152 weight_movement_bytes=1349517312 egress_bytes=6131153096 ingress_bytes=7472758784 link_bytes=7472758784 total_nvlink_bytes=13603911880 weight_to_token_ratio=0.110125 prefetch_share_of_ingress=0.092611 dedup_ratio=0.244128 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=19 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.222582161e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12262685504 prefetch_bytes=622854144 reduce_bytes=865075200 weight_movement_bytes=1487929344 egress_bytes=6135298880 ingress_bytes=7615315968 link_bytes=7615315968 total_nvlink_bytes=13750614848 weight_to_token_ratio=0.121338 prefetch_share_of_ingress=0.081790 dedup_ratio=0.243745 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=25 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027500 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.245905257e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265551872 dispatch_meta_bytes=342696 combine_payload_bytes=265551872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265894568 ingress_bytes=300154880 link_bytes=300154880 token_bytes=531446440 weight_movement_bytes=34603008 total_nvlink_bytes=566049448 weight_to_token_ratio=0.065111 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129458649e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74614 payload_rows_remote=65202 dup_rows=23690 dup_rows_remote=20652 dedup_ratio=0.240987 router_tokens_per_expert=[800, 721, 742, 794, 740, 784, 733, 775, 734, 780, 742, 788, 764, 805, 726, 754, 770, 808, 734, 789, 819, 719, 784, 815, 768, 769, 788, 771, 726, 788, 769, 761, 782, 795, 769, 752, 739, 731, 771, 775, 744, 765, 779, 776, 778, 768, 805, 775, 764, 776, 797, 742, 781, 797, 784, 784, 749, 768, 776, 760, 717, 738, 754, 801] payload_rows_per_dst=[9260, 9248, 9468, 9344, 9298, 9360, 9412, 9224] dup_rows_per_dst=[2918, 2938, 3008, 2936, 2930, 3020, 3038, 2902] combine_rows_per_src=[9260, 9248, 9468, 9344, 9298, 9360, 9412, 9224] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12267554288 prefetch_bytes=795869184 reduce_bytes=622854144 weight_movement_bytes=1418723328 egress_bytes=6137734640 ingress_bytes=7548542976 link_bytes=7548542976 total_nvlink_bytes=13686277616 weight_to_token_ratio=0.115648 prefetch_share_of_ingress=0.105433 dedup_ratio=0.243694 prefetch_slots=46 unused_prefetch_slots=322 slot_utilization=0.125000 reduce_slots=18 remote_demand=23 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027717 gemm_inner_pad_upper_bound=29210 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.234980848e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12241040960 prefetch_bytes=553648128 reduce_bytes=1003487232 weight_movement_bytes=1557135360 egress_bytes=6124467776 ingress_bytes=7673708544 link_bytes=7673708544 total_nvlink_bytes=13798176320 weight_to_token_ratio=0.127206 prefetch_share_of_ingress=0.072149 dedup_ratio=0.243948 prefetch_slots=32 unused_prefetch_slots=336 slot_utilization=0.086957 reduce_slots=29 remote_demand=16 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028370 gemm_inner_pad_upper_bound=28321 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.255458586e-04 prefetch_bytes_per_gemm_flop=9.057971014e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=267067392 dispatch_meta_bytes=343416 combine_payload_bytes=267067392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267410808 ingress_bytes=301670400 link_bytes=301670400 token_bytes=534478200 weight_movement_bytes=34603008 total_nvlink_bytes=569081208 weight_to_token_ratio=0.064742 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135161429e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12262685504 prefetch_bytes=622854144 reduce_bytes=865075200 weight_movement_bytes=1487929344 egress_bytes=6135298880 ingress_bytes=7615315968 link_bytes=7615315968 total_nvlink_bytes=13750614848 weight_to_token_ratio=0.121338 prefetch_share_of_ingress=0.081790 dedup_ratio=0.243745 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=25 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027500 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.245905257e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none - [2026-08-26 15:10:21] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 795.2 | throughput per GPU (TFLOP/s/GPU): 160.4 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.476632E+00 | loss scale: 1.0 | grad norm: 7.104 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74464 payload_rows_remote=65208 dup_rows=23840 dup_rows_remote=20938 dedup_ratio=0.242513 router_tokens_per_expert=[749, 771, 742, 765, 830, 777, 776, 764, 789, 759, 811, 777, 749, 745, 757, 805, 743, 773, 745, 757, 763, 770, 781, 790, 766, 726, 734, 738, 790, 769, 794, 800, 793, 736, 768, 817, 772, 728, 762, 763, 793, 764, 761, 770, 733, 770, 738, 750, 758, 763, 788, 773, 767, 776, 778, 744, 805, 789, 779, 752, 779, 780, 776, 722] payload_rows_per_dst=[9336, 9332, 9312, 9340, 9274, 9256, 9270, 9344] dup_rows_per_dst=[3012, 3052, 2932, 2894, 3004, 2902, 3024, 3020] combine_rows_per_src=[9336, 9332, 9312, 9340, 9274, 9256, 9270, 9344] + [2026-08-26 15:58:10] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 793.6 | throughput per GPU (TFLOP/s/GPU): 160.8 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.476632E+00 | loss scale: 1.0 | grad norm: 7.104 | number of skipped iterations: 0 | number of nan iterations: 0 | WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74368 payload_rows_remote=65004 dup_rows=23936 dup_rows_remote=20840 dedup_ratio=0.243490 router_tokens_per_expert=[789, 759, 764, 763, 726, 717, 772, 820, 754, 774, 804, 780, 804, 780, 757, 777, 758, 762, 748, 732, 810, 783, 811, 769, 779, 781, 817, 773, 685, 843, 788, 729, 817, 770, 806, 752, 763, 770, 744, 784, 737, 790, 755, 753, 740, 734, 778, 774, 750, 753, 744, 772, 766, 735, 758, 764, 783, 791, 786, 734, 762, 722, 772, 785] payload_rows_per_dst=[9236, 9364, 9402, 9414, 9346, 9110, 9190, 9306] dup_rows_per_dst=[2984, 3096, 2944, 2976, 3066, 3012, 2894, 2964] combine_rows_per_src=[9236, 9364, 9402, 9414, 9346, 9110, 9190, 9306] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74250 payload_rows_remote=65032 dup_rows=24054 dup_rows_remote=21120 dedup_ratio=0.244690 router_tokens_per_expert=[756, 733, 753, 797, 773, 751, 730, 746, 768, 788, 770, 763, 736, 755, 829, 825, 771, 775, 776, 767, 807, 772, 822, 749, 781, 730, 732, 790, 800, 754, 713, 776, 788, 769, 764, 761, 784, 768, 763, 739, 743, 776, 792, 777, 786, 773, 766, 734, 762, 769, 739, 788, 730, 765, 728, 799, 760, 772, 761, 760, 790, 780, 780, 798] payload_rows_per_dst=[9230, 9386, 9372, 9218, 9228, 9326, 9146, 9344] dup_rows_per_dst=[2848, 3082, 3106, 2934, 3044, 2968, 3014, 3058] combine_rows_per_src=[9230, 9386, 9372, 9218, 9228, 9326, 9146, 9344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74464 payload_rows_remote=65208 dup_rows=23840 dup_rows_remote=20938 dedup_ratio=0.242513 router_tokens_per_expert=[749, 771, 742, 765, 830, 777, 776, 764, 789, 759, 811, 777, 749, 745, 757, 805, 743, 773, 745, 757, 763, 770, 781, 790, 766, 726, 734, 738, 790, 769, 794, 800, 793, 736, 768, 817, 772, 728, 762, 763, 793, 764, 761, 770, 733, 770, 738, 750, 758, 763, 788, 773, 767, 776, 778, 744, 805, 789, 779, 752, 779, 780, 776, 722] payload_rows_per_dst=[9336, 9332, 9312, 9340, 9274, 9256, 9270, 9344] dup_rows_per_dst=[3012, 3052, 2932, 2894, 3004, 2902, 3024, 3020] combine_rows_per_src=[9336, 9332, 9312, 9340, 9274, 9256, 9270, 9344] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64972 dup_rows=24112 dup_rows_remote=21216 dedup_ratio=0.245280 router_tokens_per_expert=[755, 777, 795, 757, 769, 777, 754, 737, 787, 795, 813, 728, 791, 759, 735, 776, 745, 775, 725, 754, 728, 791, 770, 770, 746, 777, 728, 769, 774, 748, 789, 725, 756, 771, 788, 774, 823, 782, 761, 778, 798, 771, 757, 721, 806, 779, 750, 799, 788, 783, 719, 764, 729, 783, 813, 753, 761, 780, 797, 774, 784, 775, 736, 780] payload_rows_per_dst=[9290, 9332, 9220, 9176, 9326, 9302, 9218, 9328] dup_rows_per_dst=[2952, 3036, 2896, 2936, 3140, 3060, 3046, 3046] combine_rows_per_src=[9290, 9332, 9220, 9176, 9326, 9302, 9218, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=64976 dup_rows=24056 dup_rows_remote=21042 dedup_ratio=0.244710 router_tokens_per_expert=[774, 764, 747, 793, 718, 757, 771, 744, 742, 798, 773, 735, 778, 756, 750, 786, 729, 732, 748, 771, 795, 793, 795, 753, 764, 780, 783, 806, 758, 790, 771, 785, 756, 775, 764, 723, 768, 764, 761, 802, 752, 782, 807, 754, 759, 771, 798, 778, 788, 753, 771, 754, 779, 741, 789, 768, 741, 727, 779, 820, 769, 787, 802, 731] payload_rows_per_dst=[9206, 9296, 9234, 9374, 9206, 9352, 9272, 9308] dup_rows_per_dst=[2930, 2940, 2998, 3100, 3020, 3050, 3014, 3004] combine_rows_per_src=[9206, 9296, 9234, 9374, 9206, 9352, 9272, 9308] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74250 payload_rows_remote=65032 dup_rows=24054 dup_rows_remote=21120 dedup_ratio=0.244690 router_tokens_per_expert=[756, 733, 753, 797, 773, 751, 730, 746, 768, 788, 770, 763, 736, 755, 829, 825, 771, 775, 776, 767, 807, 772, 822, 749, 781, 730, 732, 790, 800, 754, 713, 776, 788, 769, 764, 761, 784, 768, 763, 739, 743, 776, 792, 777, 786, 773, 766, 734, 762, 769, 739, 788, 730, 765, 728, 799, 760, 772, 761, 760, 790, 780, 780, 798] payload_rows_per_dst=[9230, 9386, 9372, 9218, 9228, 9326, 9146, 9344] dup_rows_per_dst=[2848, 3082, 3106, 2934, 3044, 2968, 3014, 3058] combine_rows_per_src=[9230, 9386, 9372, 9218, 9228, 9326, 9146, 9344] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74414 payload_rows_remote=65080 dup_rows=23890 dup_rows_remote=20998 dedup_ratio=0.243022 router_tokens_per_expert=[770, 754, 792, 776, 725, 732, 732, 777, 834, 829, 772, 744, 764, 800, 810, 813, 781, 746, 774, 726, 772, 769, 781, 766, 748, 710, 766, 807, 801, 751, 771, 793, 772, 779, 806, 747, 768, 807, 797, 783, 738, 794, 785, 760, 788, 737, 797, 736, 708, 764, 748, 758, 743, 759, 735, 788, 750, 734, 791, 778, 729, 779, 690, 818] payload_rows_per_dst=[9214, 9202, 9162, 9456, 9100, 9554, 9392, 9334] dup_rows_per_dst=[2902, 3048, 2904, 3146, 2866, 3180, 2952, 2892] combine_rows_per_src=[9214, 9202, 9162, 9456, 9100, 9554, 9392, 9334] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74416 payload_rows_remote=64848 dup_rows=23888 dup_rows_remote=20642 dedup_ratio=0.243001 router_tokens_per_expert=[757, 792, 725, 770, 748, 770, 811, 767, 804, 798, 744, 789, 782, 794, 739, 789, 756, 752, 786, 779, 816, 736, 816, 829, 798, 704, 760, 780, 755, 784, 785, 789, 787, 782, 780, 769, 784, 801, 766, 757, 778, 711, 699, 763, 735, 788, 714, 730, 732, 767, 792, 780, 790, 830, 765, 736, 740, 752, 740, 783, 686, 761, 760, 790] payload_rows_per_dst=[9568, 9220, 9334, 9290, 9398, 9092, 9352, 9162] dup_rows_per_dst=[3246, 2752, 3206, 3020, 3026, 2744, 3032, 2862] combine_rows_per_src=[9568, 9220, 9334, 9290, 9398, 9092, 9352, 9162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266256384 dispatch_meta_bytes=343376 combine_payload_bytes=266256384 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266599760 ingress_bytes=404668416 link_bytes=404668416 token_bytes=532856144 weight_movement_bytes=138412032 total_nvlink_bytes=671268176 weight_to_token_ratio=0.259755 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522734671e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267091968 dispatch_meta_bytes=344584 combine_payload_bytes=267091968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267436552 ingress_bytes=301694976 link_bytes=301694976 token_bytes=534528520 weight_movement_bytes=34603008 total_nvlink_bytes=569131528 weight_to_token_ratio=0.064736 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135253906e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64958 dup_rows=24112 dup_rows_remote=21086 dedup_ratio=0.245280 router_tokens_per_expert=[777, 780, 759, 786, 787, 761, 732, 793, 784, 748, 763, 767, 772, 760, 756, 733, 777, 757, 807, 754, 760, 727, 796, 763, 714, 795, 766, 736, 768, 737, 761, 738, 724, 735, 802, 732, 815, 752, 793, 777, 759, 776, 801, 792, 778, 716, 807, 769, 821, 800, 762, 782, 809, 782, 768, 720, 837, 707, 765, 789, 783, 780, 737, 768] payload_rows_per_dst=[9362, 9220, 9214, 9188, 9234, 9330, 9418, 9226] dup_rows_per_dst=[2988, 2946, 3068, 2842, 3026, 3066, 3070, 3106] combine_rows_per_src=[9362, 9220, 9214, 9188, 9234, 9330, 9418, 9226] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266256384 dispatch_meta_bytes=343376 combine_payload_bytes=266256384 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266599760 ingress_bytes=404668416 link_bytes=404668416 token_bytes=532856144 weight_movement_bytes=138412032 total_nvlink_bytes=671268176 weight_to_token_ratio=0.259755 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522734671e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=64976 dup_rows=24056 dup_rows_remote=21042 dedup_ratio=0.244710 router_tokens_per_expert=[774, 764, 747, 793, 718, 757, 771, 744, 742, 798, 773, 735, 778, 756, 750, 786, 729, 732, 748, 771, 795, 793, 795, 753, 764, 780, 783, 806, 758, 790, 771, 785, 756, 775, 764, 723, 768, 764, 761, 802, 752, 782, 807, 754, 759, 771, 798, 778, 788, 753, 771, 754, 779, 741, 789, 768, 741, 727, 779, 820, 769, 787, 802, 731] payload_rows_per_dst=[9206, 9296, 9234, 9374, 9206, 9352, 9272, 9308] dup_rows_per_dst=[2930, 2940, 2998, 3100, 3020, 3050, 3014, 3004] combine_rows_per_src=[9206, 9296, 9234, 9374, 9206, 9352, 9272, 9308] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344608 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266715680 ingress_bytes=300974080 link_bytes=300974080 token_bytes=533086752 weight_movement_bytes=34603008 total_nvlink_bytes=567689760 weight_to_token_ratio=0.064911 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132541233e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266125312 dispatch_meta_bytes=344752 combine_payload_bytes=266125312 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266470064 ingress_bytes=335331328 link_bytes=335331328 token_bytes=532595376 weight_movement_bytes=69206016 total_nvlink_bytes=601801392 weight_to_token_ratio=0.129941 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261824791e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266567680 dispatch_meta_bytes=344312 combine_payload_bytes=266567680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266911992 ingress_bytes=301170688 link_bytes=301170688 token_bytes=533479672 weight_movement_bytes=34603008 total_nvlink_bytes=568082680 weight_to_token_ratio=0.064863 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133281053e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74416 payload_rows_remote=64848 dup_rows=23888 dup_rows_remote=20642 dedup_ratio=0.243001 router_tokens_per_expert=[757, 792, 725, 770, 748, 770, 811, 767, 804, 798, 744, 789, 782, 794, 739, 789, 756, 752, 786, 779, 816, 736, 816, 829, 798, 704, 760, 780, 755, 784, 785, 789, 787, 782, 780, 769, 784, 801, 766, 757, 778, 711, 699, 763, 735, 788, 714, 730, 732, 767, 792, 780, 790, 830, 765, 736, 740, 752, 740, 783, 686, 761, 760, 790] payload_rows_per_dst=[9568, 9220, 9334, 9290, 9398, 9092, 9352, 9162] dup_rows_per_dst=[3246, 2752, 3206, 3020, 3026, 2744, 3032, 2862] combine_rows_per_src=[9568, 9220, 9334, 9290, 9398, 9092, 9352, 9162] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=344176 combine_payload_bytes=266067968 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266412144 ingress_bytes=369876992 link_bytes=369876992 token_bytes=532480112 weight_movement_bytes=103809024 total_nvlink_bytes=636289136 weight_to_token_ratio=0.194954 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391817343e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266141696 dispatch_meta_bytes=344072 combine_payload_bytes=266141696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266485768 ingress_bytes=300744704 link_bytes=300744704 token_bytes=532627464 weight_movement_bytes=34603008 total_nvlink_bytes=567230472 weight_to_token_ratio=0.064967 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131678109e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344608 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266715680 ingress_bytes=300974080 link_bytes=300974080 token_bytes=533086752 weight_movement_bytes=34603008 total_nvlink_bytes=567689760 weight_to_token_ratio=0.064911 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132541233e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74368 payload_rows_remote=64956 dup_rows=23936 dup_rows_remote=20954 dedup_ratio=0.243490 router_tokens_per_expert=[769, 773, 757, 736, 755, 730, 740, 768, 806, 791, 762, 773, 754, 793, 813, 784, 790, 778, 790, 708, 761, 812, 782, 754, 757, 776, 761, 795, 786, 816, 732, 794, 731, 810, 777, 761, 794, 767, 743, 804, 776, 803, 751, 779, 745, 770, 810, 763, 790, 747, 797, 752, 768, 725, 704, 770, 731, 732, 767, 787, 729, 764, 751, 758] payload_rows_per_dst=[9148, 9400, 9330, 9412, 9322, 9412, 9174, 9170] dup_rows_per_dst=[2908, 3152, 3020, 3022, 3052, 2982, 2932, 2868] combine_rows_per_src=[9148, 9400, 9330, 9412, 9322, 9412, 9174, 9170] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=65122 dup_rows=23962 dup_rows_remote=21060 dedup_ratio=0.243754 router_tokens_per_expert=[809, 790, 759, 772, 785, 796, 778, 765, 738, 731, 748, 785, 754, 735, 796, 774, 787, 785, 777, 792, 767, 764, 791, 764, 817, 781, 764, 785, 772, 769, 741, 749, 730, 749, 786, 827, 774, 762, 723, 823, 767, 742, 767, 735, 780, 740, 733, 740, 738, 758, 776, 769, 766, 779, 749, 743, 756, 800, 770, 755, 817, 739, 777, 762] payload_rows_per_dst=[9514, 9220, 9268, 9256, 9332, 9196, 9250, 9306] dup_rows_per_dst=[2994, 2902, 3186, 3100, 3016, 2812, 2906, 3046] combine_rows_per_src=[9514, 9220, 9268, 9256, 9332, 9196, 9250, 9306] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74534 payload_rows_remote=65300 dup_rows=23770 dup_rows_remote=20852 dedup_ratio=0.241801 router_tokens_per_expert=[753, 754, 756, 810, 798, 737, 807, 763, 800, 783, 792, 805, 779, 724, 756, 732, 790, 740, 808, 753, 756, 778, 732, 813, 742, 775, 724, 766, 783, 791, 760, 735, 725, 763, 779, 746, 791, 757, 764, 760, 754, 786, 742, 785, 752, 787, 774, 744, 784, 787, 728, 793, 768, 779, 720, 771, 750, 819, 770, 807, 776, 792, 746, 758] payload_rows_per_dst=[9368, 9264, 9384, 9234, 9238, 9264, 9348, 9434] dup_rows_per_dst=[2988, 3078, 2956, 2918, 2932, 2984, 2912, 3002] combine_rows_per_src=[9368, 9264, 9384, 9234, 9238, 9264, 9348, 9434] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266567680 dispatch_meta_bytes=344312 combine_payload_bytes=266567680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266911992 ingress_bytes=301170688 link_bytes=301170688 token_bytes=533479672 weight_movement_bytes=34603008 total_nvlink_bytes=568082680 weight_to_token_ratio=0.064863 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133281053e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265617408 dispatch_meta_bytes=341960 combine_payload_bytes=265617408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265959368 ingress_bytes=300220416 link_bytes=300220416 token_bytes=531576776 weight_movement_bytes=34603008 total_nvlink_bytes=566179784 weight_to_token_ratio=0.065095 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129705256e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=64872 dup_rows=24062 dup_rows_remote=21022 dedup_ratio=0.244771 router_tokens_per_expert=[791, 770, 751, 796, 760, 748, 767, 778, 792, 811, 787, 796, 766, 779, 779, 821, 721, 759, 740, 759, 775, 737, 796, 723, 754, 800, 768, 739, 747, 752, 771, 765, 776, 752, 789, 738, 763, 778, 797, 732, 745, 768, 715, 738, 788, 741, 797, 809, 780, 822, 741, 755, 723, 731, 785, 789, 762, 770, 763, 818, 768, 791, 767, 763] payload_rows_per_dst=[9318, 9146, 9410, 9286, 8982, 9258, 9472, 9370] dup_rows_per_dst=[3004, 2946, 3156, 2924, 2734, 2944, 3314, 3040] combine_rows_per_src=[9318, 9146, 9410, 9286, 8982, 9258, 9472, 9370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64958 dup_rows=24112 dup_rows_remote=21086 dedup_ratio=0.245280 router_tokens_per_expert=[777, 780, 759, 786, 787, 761, 732, 793, 784, 748, 763, 767, 772, 760, 756, 733, 777, 757, 807, 754, 760, 727, 796, 763, 714, 795, 766, 736, 768, 737, 761, 738, 724, 735, 802, 732, 815, 752, 793, 777, 759, 776, 801, 792, 778, 716, 807, 769, 821, 800, 762, 782, 809, 782, 768, 720, 837, 707, 765, 789, 783, 780, 737, 768] payload_rows_per_dst=[9362, 9220, 9214, 9188, 9234, 9330, 9418, 9226] dup_rows_per_dst=[2988, 2946, 3068, 2842, 3026, 3066, 3070, 3106] combine_rows_per_src=[9362, 9220, 9214, 9188, 9234, 9330, 9418, 9226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=65122 dup_rows=23962 dup_rows_remote=21060 dedup_ratio=0.243754 router_tokens_per_expert=[809, 790, 759, 772, 785, 796, 778, 765, 738, 731, 748, 785, 754, 735, 796, 774, 787, 785, 777, 792, 767, 764, 791, 764, 817, 781, 764, 785, 772, 769, 741, 749, 730, 749, 786, 827, 774, 762, 723, 823, 767, 742, 767, 735, 780, 740, 733, 740, 738, 758, 776, 769, 766, 779, 749, 743, 756, 800, 770, 755, 817, 739, 777, 762] payload_rows_per_dst=[9514, 9220, 9268, 9256, 9332, 9196, 9250, 9306] dup_rows_per_dst=[2994, 2902, 3186, 3100, 3016, 2812, 2906, 3046] combine_rows_per_src=[9514, 9220, 9268, 9256, 9332, 9196, 9250, 9306] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74368 payload_rows_remote=64956 dup_rows=23936 dup_rows_remote=20954 dedup_ratio=0.243490 router_tokens_per_expert=[769, 773, 757, 736, 755, 730, 740, 768, 806, 791, 762, 773, 754, 793, 813, 784, 790, 778, 790, 708, 761, 812, 782, 754, 757, 776, 761, 795, 786, 816, 732, 794, 731, 810, 777, 761, 794, 767, 743, 804, 776, 803, 751, 779, 745, 770, 810, 763, 790, 747, 797, 752, 768, 725, 704, 770, 731, 732, 767, 787, 729, 764, 751, 758] payload_rows_per_dst=[9148, 9400, 9330, 9412, 9322, 9412, 9174, 9170] dup_rows_per_dst=[2908, 3152, 3020, 3022, 3052, 2982, 2932, 2868] combine_rows_per_src=[9148, 9400, 9330, 9412, 9322, 9412, 9174, 9170] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74406 payload_rows_remote=64980 dup_rows=23898 dup_rows_remote=20848 dedup_ratio=0.243103 router_tokens_per_expert=[765, 738, 755, 813, 736, 782, 786, 771, 798, 810, 751, 762, 785, 819, 726, 753, 736, 707, 775, 729, 721, 746, 824, 791, 737, 768, 787, 776, 737, 775, 798, 728, 836, 812, 754, 763, 753, 730, 755, 758, 770, 790, 758, 803, 767, 748, 799, 728, 813, 800, 758, 777, 776, 811, 743, 760, 768, 777, 760, 697, 801, 749, 786, 767] payload_rows_per_dst=[9288, 9316, 9178, 9250, 9272, 9348, 9426, 9328] dup_rows_per_dst=[3004, 3092, 2880, 2962, 3050, 2978, 3050, 2882] combine_rows_per_src=[9288, 9316, 9178, 9250, 9272, 9348, 9426, 9328] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74328 payload_rows_remote=65018 dup_rows=23976 dup_rows_remote=21064 dedup_ratio=0.243896 router_tokens_per_expert=[750, 785, 788, 787, 727, 766, 804, 793, 742, 771, 788, 778, 766, 744, 788, 762, 744, 775, 763, 813, 775, 780, 683, 778, 765, 732, 759, 737, 785, 790, 789, 809, 755, 803, 743, 801, 753, 793, 760, 786, 772, 753, 728, 755, 830, 783, 770, 766, 760, 721, 778, 767, 741, 774, 777, 766, 797, 712, 747, 747, 774, 771, 756, 797] payload_rows_per_dst=[9372, 9366, 9310, 9222, 9290, 9310, 9248, 9210] dup_rows_per_dst=[3028, 2912, 2912, 3110, 3098, 3004, 2920, 2992] combine_rows_per_src=[9372, 9366, 9310, 9222, 9290, 9310, 9248, 9210] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=343640 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266403416 ingress_bytes=335265792 link_bytes=335265792 token_bytes=532463192 weight_movement_bytes=69206016 total_nvlink_bytes=601669208 weight_to_token_ratio=0.129973 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261578184e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74534 payload_rows_remote=65300 dup_rows=23770 dup_rows_remote=20852 dedup_ratio=0.241801 router_tokens_per_expert=[753, 754, 756, 810, 798, 737, 807, 763, 800, 783, 792, 805, 779, 724, 756, 732, 790, 740, 808, 753, 756, 778, 732, 813, 742, 775, 724, 766, 783, 791, 760, 735, 725, 763, 779, 746, 791, 757, 764, 760, 754, 786, 742, 785, 752, 787, 774, 744, 784, 787, 728, 793, 768, 779, 720, 771, 750, 819, 770, 807, 776, 792, 746, 758] payload_rows_per_dst=[9368, 9264, 9384, 9234, 9238, 9264, 9348, 9434] dup_rows_per_dst=[2988, 3078, 2956, 2918, 2932, 2984, 2912, 3002] combine_rows_per_src=[9368, 9264, 9384, 9234, 9238, 9264, 9348, 9434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=64872 dup_rows=24062 dup_rows_remote=21022 dedup_ratio=0.244771 router_tokens_per_expert=[791, 770, 751, 796, 760, 748, 767, 778, 792, 811, 787, 796, 766, 779, 779, 821, 721, 759, 740, 759, 775, 737, 796, 723, 754, 800, 768, 739, 747, 752, 771, 765, 776, 752, 789, 738, 763, 778, 797, 732, 745, 768, 715, 738, 788, 741, 797, 809, 780, 822, 741, 755, 723, 731, 785, 789, 762, 770, 763, 818, 768, 791, 767, 763] payload_rows_per_dst=[9318, 9146, 9410, 9286, 8982, 9258, 9472, 9370] dup_rows_per_dst=[3004, 2946, 3156, 2924, 2734, 2944, 3314, 3040] combine_rows_per_src=[9318, 9146, 9410, 9286, 8982, 9258, 9472, 9370] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266739712 dispatch_meta_bytes=344728 combine_payload_bytes=266739712 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267084440 ingress_bytes=405151744 link_bytes=405151744 token_bytes=533824152 weight_movement_bytes=138412032 total_nvlink_bytes=672236184 weight_to_token_ratio=0.259284 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524553395e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267468800 dispatch_meta_bytes=344608 combine_payload_bytes=267468800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267813408 ingress_bytes=302071808 link_bytes=302071808 token_bytes=535282208 weight_movement_bytes=34603008 total_nvlink_bytes=569885216 weight_to_token_ratio=0.064644 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136671895e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74266 payload_rows_remote=64882 dup_rows=24038 dup_rows_remote=20970 dedup_ratio=0.244527 router_tokens_per_expert=[787, 728, 734, 783, 792, 762, 790, 773, 786, 712, 820, 751, 762, 774, 774, 727, 731, 771, 722, 749, 822, 727, 763, 755, 798, 765, 800, 750, 793, 780, 780, 797, 781, 775, 768, 775, 807, 795, 788, 737, 774, 772, 762, 771, 833, 782, 731, 764, 778, 726, 723, 774, 744, 772, 796, 750, 795, 747, 762, 747, 764, 799, 767, 735] payload_rows_per_dst=[9204, 9352, 9198, 9440, 9384, 9340, 9088, 9260] dup_rows_per_dst=[3094, 2860, 2882, 3086, 3068, 3038, 3038, 2972] combine_rows_per_src=[9204, 9352, 9198, 9440, 9384, 9340, 9088, 9260] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265715712 dispatch_meta_bytes=343576 combine_payload_bytes=265715712 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266059288 ingress_bytes=334921728 link_bytes=334921728 token_bytes=531775000 weight_movement_bytes=69206016 total_nvlink_bytes=600981016 weight_to_token_ratio=0.130142 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260283499e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=344328 combine_payload_bytes=266313728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266658056 ingress_bytes=300916736 link_bytes=300916736 token_bytes=532971784 weight_movement_bytes=34603008 total_nvlink_bytes=567574792 weight_to_token_ratio=0.064925 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132325452e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74406 payload_rows_remote=64980 dup_rows=23898 dup_rows_remote=20848 dedup_ratio=0.243103 router_tokens_per_expert=[765, 738, 755, 813, 736, 782, 786, 771, 798, 810, 751, 762, 785, 819, 726, 753, 736, 707, 775, 729, 721, 746, 824, 791, 737, 768, 787, 776, 737, 775, 798, 728, 836, 812, 754, 763, 753, 730, 755, 758, 770, 790, 758, 803, 767, 748, 799, 728, 813, 800, 758, 777, 776, 811, 743, 760, 768, 777, 760, 697, 801, 749, 786, 767] payload_rows_per_dst=[9288, 9316, 9178, 9250, 9272, 9348, 9426, 9328] dup_rows_per_dst=[3004, 3092, 2880, 2962, 3050, 2978, 3050, 2882] combine_rows_per_src=[9288, 9316, 9178, 9250, 9272, 9348, 9426, 9328] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=343408 combine_payload_bytes=265756672 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266100080 ingress_bytes=334962688 link_bytes=334962688 token_bytes=531856752 weight_movement_bytes=69206016 total_nvlink_bytes=601062768 weight_to_token_ratio=0.130122 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260437628e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74548 payload_rows_remote=65158 dup_rows=23756 dup_rows_remote=20754 dedup_ratio=0.241659 router_tokens_per_expert=[789, 778, 769, 762, 796, 793, 755, 765, 774, 793, 795, 752, 743, 765, 791, 735, 767, 777, 756, 809, 783, 770, 772, 773, 781, 776, 727, 764, 749, 728, 773, 792, 779, 769, 770, 740, 725, 810, 774, 724, 768, 737, 785, 796, 747, 770, 766, 827, 759, 732, 802, 797, 788, 754, 788, 744, 761, 712, 795, 759, 701, 815, 748, 758] payload_rows_per_dst=[9410, 9438, 9340, 9250, 9238, 9390, 9298, 9184] dup_rows_per_dst=[3004, 2858, 3074, 2930, 2944, 3002, 3030, 2914] combine_rows_per_src=[9410, 9438, 9340, 9250, 9238, 9390, 9298, 9184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=343640 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266403416 ingress_bytes=335265792 link_bytes=335265792 token_bytes=532463192 weight_movement_bytes=69206016 total_nvlink_bytes=601669208 weight_to_token_ratio=0.129973 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261578184e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=344176 combine_payload_bytes=266067968 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266412144 ingress_bytes=369876992 link_bytes=369876992 token_bytes=532480112 weight_movement_bytes=103809024 total_nvlink_bytes=636289136 weight_to_token_ratio=0.194954 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391817343e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74160 payload_rows_remote=64994 dup_rows=24144 dup_rows_remote=21238 dedup_ratio=0.245605 router_tokens_per_expert=[733, 786, 757, 698, 719, 761, 808, 745, 738, 772, 773, 776, 772, 802, 769, 776, 780, 749, 741, 772, 768, 779, 784, 744, 791, 723, 742, 734, 781, 760, 774, 736, 759, 743, 761, 751, 782, 748, 784, 784, 743, 745, 837, 795, 816, 780, 808, 788, 771, 800, 778, 718, 742, 739, 797, 768, 818, 749, 757, 792, 765, 826, 762, 803] payload_rows_per_dst=[9166, 9228, 9296, 9194, 9350, 9330, 9236, 9360] dup_rows_per_dst=[2906, 3070, 2938, 2888, 3184, 3104, 2990, 3064] combine_rows_per_src=[9166, 9228, 9296, 9194, 9350, 9330, 9236, 9360] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266158080 dispatch_meta_bytes=343312 combine_payload_bytes=266158080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266501392 ingress_bytes=300761088 link_bytes=300761088 token_bytes=532659472 weight_movement_bytes=34603008 total_nvlink_bytes=567262480 weight_to_token_ratio=0.064963 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131739761e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=344328 combine_payload_bytes=266313728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266658056 ingress_bytes=300916736 link_bytes=300916736 token_bytes=532971784 weight_movement_bytes=34603008 total_nvlink_bytes=567574792 weight_to_token_ratio=0.064925 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132325452e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267468800 dispatch_meta_bytes=344608 combine_payload_bytes=267468800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267813408 ingress_bytes=302071808 link_bytes=302071808 token_bytes=535282208 weight_movement_bytes=34603008 total_nvlink_bytes=569885216 weight_to_token_ratio=0.064644 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136671895e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265715712 dispatch_meta_bytes=343576 combine_payload_bytes=265715712 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266059288 ingress_bytes=334921728 link_bytes=334921728 token_bytes=531775000 weight_movement_bytes=69206016 total_nvlink_bytes=600981016 weight_to_token_ratio=0.130142 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260283499e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266215424 dispatch_meta_bytes=344928 combine_payload_bytes=266215424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266560352 ingress_bytes=300818432 link_bytes=300818432 token_bytes=532775776 weight_movement_bytes=34603008 total_nvlink_bytes=567378784 weight_to_token_ratio=0.064949 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131955542e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65018 dup_rows=23908 dup_rows_remote=20986 dedup_ratio=0.243205 router_tokens_per_expert=[774, 772, 788, 770, 742, 780, 781, 764, 829, 722, 771, 813, 744, 757, 748, 766, 786, 782, 788, 760, 725, 753, 724, 751, 748, 791, 775, 810, 828, 788, 762, 821, 750, 731, 850, 753, 767, 724, 770, 818, 738, 815, 743, 786, 799, 718, 739, 739, 726, 720, 766, 764, 784, 736, 770, 790, 776, 727, 737, 816, 778, 769, 768, 772] payload_rows_per_dst=[9374, 9378, 9244, 9528, 9286, 9228, 9114, 9244] dup_rows_per_dst=[2968, 2922, 2894, 3118, 3040, 2926, 2998, 3042] combine_rows_per_src=[9374, 9378, 9244, 9528, 9286, 9228, 9114, 9244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74548 payload_rows_remote=65158 dup_rows=23756 dup_rows_remote=20754 dedup_ratio=0.241659 router_tokens_per_expert=[789, 778, 769, 762, 796, 793, 755, 765, 774, 793, 795, 752, 743, 765, 791, 735, 767, 777, 756, 809, 783, 770, 772, 773, 781, 776, 727, 764, 749, 728, 773, 792, 779, 769, 770, 740, 725, 810, 774, 724, 768, 737, 785, 796, 747, 770, 766, 827, 759, 732, 802, 797, 788, 754, 788, 744, 761, 712, 795, 759, 701, 815, 748, 758] payload_rows_per_dst=[9410, 9438, 9340, 9250, 9238, 9390, 9298, 9184] dup_rows_per_dst=[3004, 2858, 3074, 2930, 2944, 3002, 3030, 2914] combine_rows_per_src=[9410, 9438, 9340, 9250, 9238, 9390, 9298, 9184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74458 payload_rows_remote=65282 dup_rows=23846 dup_rows_remote=20862 dedup_ratio=0.242574 router_tokens_per_expert=[718, 737, 718, 744, 792, 802, 821, 783, 760, 748, 795, 782, 751, 783, 811, 753, 784, 749, 744, 798, 778, 777, 746, 781, 750, 752, 779, 843, 769, 768, 793, 793, 749, 775, 755, 780, 801, 796, 773, 805, 745, 734, 743, 732, 744, 786, 738, 730, 734, 758, 744, 727, 760, 770, 824, 763, 741, 786, 783, 764, 769, 783, 749, 809] payload_rows_per_dst=[9314, 9390, 9254, 9340, 9382, 9148, 9176, 9454] dup_rows_per_dst=[2916, 2976, 3060, 3154, 3086, 2756, 2984, 2914] combine_rows_per_src=[9314, 9390, 9254, 9340, 9382, 9148, 9176, 9454] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=344016 combine_payload_bytes=266313728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266657744 ingress_bytes=300916736 link_bytes=300916736 token_bytes=532971472 weight_movement_bytes=34603008 total_nvlink_bytes=567574480 weight_to_token_ratio=0.064925 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132325452e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74586 payload_rows_remote=65302 dup_rows=23718 dup_rows_remote=20728 dedup_ratio=0.241272 router_tokens_per_expert=[801, 762, 786, 764, 811, 728, 747, 768, 758, 781, 778, 810, 752, 797, 792, 746, 740, 798, 743, 757, 788, 728, 776, 807, 758, 799, 746, 758, 754, 826, 764, 760, 754, 744, 784, 796, 759, 780, 746, 768, 782, 747, 761, 744, 752, 737, 760, 811, 741, 780, 810, 777, 771, 748, 783, 747, 742, 808, 774, 750, 751, 767, 725, 770] payload_rows_per_dst=[9332, 9366, 9284, 9412, 9256, 9238, 9368, 9330] dup_rows_per_dst=[3002, 3062, 2990, 2918, 3006, 2950, 2946, 2844] combine_rows_per_src=[9332, 9366, 9284, 9412, 9256, 9238, 9368, 9330] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=64884 dup_rows=24028 dup_rows_remote=20862 dedup_ratio=0.244425 router_tokens_per_expert=[748, 803, 787, 780, 811, 775, 799, 762, 791, 760, 754, 771, 780, 755, 779, 788, 733, 755, 762, 737, 797, 738, 743, 722, 745, 788, 807, 833, 759, 754, 774, 819, 749, 764, 766, 740, 802, 773, 721, 745, 761, 767, 771, 757, 767, 779, 780, 750, 782, 737, 765, 753, 768, 794, 776, 763, 747, 722, 835, 804, 708, 782, 769, 746] payload_rows_per_dst=[9404, 9362, 9120, 9392, 9178, 9330, 9330, 9160] dup_rows_per_dst=[3126, 2994, 2854, 3166, 2942, 2934, 2946, 3066] combine_rows_per_src=[9404, 9362, 9120, 9392, 9178, 9330, 9330, 9160] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74106 payload_rows_remote=64734 dup_rows=24198 dup_rows_remote=20958 dedup_ratio=0.246155 router_tokens_per_expert=[769, 765, 750, 784, 792, 771, 791, 729, 743, 745, 744, 756, 729, 760, 787, 775, 788, 721, 752, 769, 748, 747, 790, 742, 745, 729, 773, 738, 763, 744, 760, 733, 800, 815, 820, 830, 768, 712, 761, 805, 759, 806, 778, 748, 796, 786, 771, 792, 776, 765, 786, 782, 762, 720, 760, 814, 782, 801, 787, 798, 786, 742, 784, 728] payload_rows_per_dst=[8924, 9186, 9390, 9054, 9128, 9470, 9582, 9372] dup_rows_per_dst=[2802, 2892, 3142, 2898, 2784, 3160, 3280, 3240] combine_rows_per_src=[8924, 9186, 9390, 9054, 9128, 9470, 9582, 9372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74586 payload_rows_remote=65302 dup_rows=23718 dup_rows_remote=20728 dedup_ratio=0.241272 router_tokens_per_expert=[801, 762, 786, 764, 811, 728, 747, 768, 758, 781, 778, 810, 752, 797, 792, 746, 740, 798, 743, 757, 788, 728, 776, 807, 758, 799, 746, 758, 754, 826, 764, 760, 754, 744, 784, 796, 759, 780, 746, 768, 782, 747, 761, 744, 752, 737, 760, 811, 741, 780, 810, 777, 771, 748, 783, 747, 742, 808, 774, 750, 751, 767, 725, 770] payload_rows_per_dst=[9332, 9366, 9284, 9412, 9256, 9238, 9368, 9330] dup_rows_per_dst=[3002, 3062, 2990, 2918, 3006, 2950, 2946, 2844] combine_rows_per_src=[9332, 9366, 9284, 9412, 9256, 9238, 9368, 9330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74266 payload_rows_remote=64882 dup_rows=24038 dup_rows_remote=20970 dedup_ratio=0.244527 router_tokens_per_expert=[787, 728, 734, 783, 792, 762, 790, 773, 786, 712, 820, 751, 762, 774, 774, 727, 731, 771, 722, 749, 822, 727, 763, 755, 798, 765, 800, 750, 793, 780, 780, 797, 781, 775, 768, 775, 807, 795, 788, 737, 774, 772, 762, 771, 833, 782, 731, 764, 778, 726, 723, 774, 744, 772, 796, 750, 795, 747, 762, 747, 764, 799, 767, 735] payload_rows_per_dst=[9204, 9352, 9198, 9440, 9384, 9340, 9088, 9260] dup_rows_per_dst=[3094, 2860, 2882, 3086, 3068, 3038, 3038, 2972] combine_rows_per_src=[9204, 9352, 9198, 9440, 9384, 9340, 9088, 9260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266887168 dispatch_meta_bytes=343648 combine_payload_bytes=266887168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267230816 ingress_bytes=301490176 link_bytes=301490176 token_bytes=534117984 weight_movement_bytes=34603008 total_nvlink_bytes=568720992 weight_to_token_ratio=0.064785 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134483260e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266215424 dispatch_meta_bytes=344928 combine_payload_bytes=266215424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266560352 ingress_bytes=300818432 link_bytes=300818432 token_bytes=532775776 weight_movement_bytes=34603008 total_nvlink_bytes=567378784 weight_to_token_ratio=0.064949 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131955542e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=344016 combine_payload_bytes=266313728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266657744 ingress_bytes=300916736 link_bytes=300916736 token_bytes=532971472 weight_movement_bytes=34603008 total_nvlink_bytes=567574480 weight_to_token_ratio=0.064925 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132325452e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74134 payload_rows_remote=64848 dup_rows=24170 dup_rows_remote=21080 dedup_ratio=0.245870 router_tokens_per_expert=[784, 787, 721, 762, 825, 759, 783, 779, 761, 766, 753, 753, 738, 741, 733, 795, 735, 729, 773, 785, 766, 815, 799, 776, 781, 795, 716, 754, 729, 776, 743, 752, 744, 804, 724, 827, 769, 767, 793, 760, 773, 772, 766, 833, 729, 760, 796, 805, 748, 694, 800, 778, 713, 788, 742, 815, 775, 749, 756, 761, 748, 781, 793, 825] payload_rows_per_dst=[9320, 9166, 9284, 9206, 9286, 9388, 9232, 9252] dup_rows_per_dst=[3080, 2914, 3072, 2886, 3090, 3080, 2924, 3124] combine_rows_per_src=[9320, 9166, 9284, 9206, 9286, 9388, 9232, 9252] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267395072 dispatch_meta_bytes=344576 combine_payload_bytes=267395072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267739648 ingress_bytes=301998080 link_bytes=301998080 token_bytes=535134720 weight_movement_bytes=34603008 total_nvlink_bytes=569737728 weight_to_token_ratio=0.064662 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136394462e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74540 payload_rows_remote=65202 dup_rows=23764 dup_rows_remote=20774 dedup_ratio=0.241740 router_tokens_per_expert=[803, 738, 755, 756, 795, 796, 768, 732, 774, 749, 705, 782, 798, 799, 753, 817, 762, 764, 787, 818, 799, 772, 726, 723, 782, 752, 783, 719, 810, 743, 755, 726, 786, 804, 820, 756, 793, 842, 774, 754, 744, 805, 786, 756, 734, 749, 783, 795, 728, 774, 790, 764, 733, 759, 759, 741, 794, 767, 689, 760, 720, 747, 811, 794] payload_rows_per_dst=[9338, 9368, 9358, 9280, 9450, 9384, 9140, 9222] dup_rows_per_dst=[2990, 3032, 2944, 2772, 3208, 2920, 2956, 2942] combine_rows_per_src=[9338, 9368, 9358, 9280, 9450, 9384, 9140, 9222] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267476992 dispatch_meta_bytes=344120 combine_payload_bytes=267476992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267821112 ingress_bytes=302080000 link_bytes=302080000 token_bytes=535298104 weight_movement_bytes=34603008 total_nvlink_bytes=569901112 weight_to_token_ratio=0.064643 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136702721e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265764864 dispatch_meta_bytes=342984 combine_payload_bytes=265764864 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266107848 ingress_bytes=369573888 link_bytes=369573888 token_bytes=531872712 weight_movement_bytes=103809024 total_nvlink_bytes=635681736 weight_to_token_ratio=0.195176 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390676787e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265150464 dispatch_meta_bytes=342768 combine_payload_bytes=265150464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265493232 ingress_bytes=299753472 link_bytes=299753472 token_bytes=530643696 weight_movement_bytes=34603008 total_nvlink_bytes=565246704 weight_to_token_ratio=0.065209 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127948183e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267476992 dispatch_meta_bytes=344120 combine_payload_bytes=267476992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267821112 ingress_bytes=302080000 link_bytes=302080000 token_bytes=535298104 weight_movement_bytes=34603008 total_nvlink_bytes=569901112 weight_to_token_ratio=0.064643 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136702721e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265617408 dispatch_meta_bytes=343712 combine_payload_bytes=265617408 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265961120 ingress_bytes=369426432 link_bytes=369426432 token_bytes=531578528 weight_movement_bytes=103809024 total_nvlink_bytes=635387552 weight_to_token_ratio=0.195284 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390121922e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74458 payload_rows_remote=65282 dup_rows=23846 dup_rows_remote=20862 dedup_ratio=0.242574 router_tokens_per_expert=[718, 737, 718, 744, 792, 802, 821, 783, 760, 748, 795, 782, 751, 783, 811, 753, 784, 749, 744, 798, 778, 777, 746, 781, 750, 752, 779, 843, 769, 768, 793, 793, 749, 775, 755, 780, 801, 796, 773, 805, 745, 734, 743, 732, 744, 786, 738, 730, 734, 758, 744, 727, 760, 770, 824, 763, 741, 786, 783, 764, 769, 783, 749, 809] payload_rows_per_dst=[9314, 9390, 9254, 9340, 9382, 9148, 9176, 9454] dup_rows_per_dst=[2916, 2976, 3060, 3154, 3086, 2756, 2984, 2914] combine_rows_per_src=[9314, 9390, 9254, 9340, 9382, 9148, 9176, 9454] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74138 payload_rows_remote=64816 dup_rows=24166 dup_rows_remote=21092 dedup_ratio=0.245829 router_tokens_per_expert=[797, 829, 776, 742, 739, 739, 731, 767, 793, 790, 763, 784, 776, 781, 815, 778, 781, 753, 733, 761, 752, 794, 759, 795, 729, 770, 763, 755, 780, 761, 779, 759, 810, 766, 792, 705, 733, 723, 782, 792, 772, 749, 801, 764, 764, 770, 759, 819, 829, 718, 707, 761, 805, 755, 732, 737, 762, 795, 804, 766, 750, 795, 728, 783] payload_rows_per_dst=[9224, 9468, 9214, 9226, 9202, 9322, 9130, 9352] dup_rows_per_dst=[3016, 3092, 3042, 2966, 3004, 3074, 2958, 3014] combine_rows_per_src=[9224, 9468, 9214, 9226, 9202, 9322, 9130, 9352] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74540 payload_rows_remote=65202 dup_rows=23764 dup_rows_remote=20774 dedup_ratio=0.241740 router_tokens_per_expert=[803, 738, 755, 756, 795, 796, 768, 732, 774, 749, 705, 782, 798, 799, 753, 817, 762, 764, 787, 818, 799, 772, 726, 723, 782, 752, 783, 719, 810, 743, 755, 726, 786, 804, 820, 756, 793, 842, 774, 754, 744, 805, 786, 756, 734, 749, 783, 795, 728, 774, 790, 764, 733, 759, 759, 741, 794, 767, 689, 760, 720, 747, 811, 794] payload_rows_per_dst=[9338, 9368, 9358, 9280, 9450, 9384, 9140, 9222] dup_rows_per_dst=[2990, 3032, 2944, 2772, 3208, 2920, 2956, 2942] combine_rows_per_src=[9338, 9368, 9358, 9280, 9450, 9384, 9140, 9222] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267395072 dispatch_meta_bytes=344576 combine_payload_bytes=267395072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267739648 ingress_bytes=301998080 link_bytes=301998080 token_bytes=535134720 weight_movement_bytes=34603008 total_nvlink_bytes=569737728 weight_to_token_ratio=0.064662 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136394462e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=343408 combine_payload_bytes=265756672 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266100080 ingress_bytes=334962688 link_bytes=334962688 token_bytes=531856752 weight_movement_bytes=69206016 total_nvlink_bytes=601062768 weight_to_token_ratio=0.130122 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260437628e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74412 payload_rows_remote=65276 dup_rows=23892 dup_rows_remote=21038 dedup_ratio=0.243042 router_tokens_per_expert=[763, 811, 748, 778, 746, 790, 773, 776, 761, 774, 706, 749, 759, 701, 754, 791, 744, 777, 789, 762, 746, 809, 779, 782, 775, 743, 766, 792, 779, 758, 777, 841, 788, 760, 766, 788, 716, 777, 787, 726, 780, 763, 794, 766, 760, 781, 775, 757, 757, 747, 774, 810, 761, 711, 723, 766, 746, 773, 807, 772, 794, 785, 721, 822] payload_rows_per_dst=[9336, 9136, 9408, 9398, 9256, 9356, 9178, 9344] dup_rows_per_dst=[3034, 2854, 2968, 3064, 2960, 2996, 2920, 3096] combine_rows_per_src=[9336, 9136, 9408, 9398, 9256, 9356, 9178, 9344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267067392 dispatch_meta_bytes=343904 combine_payload_bytes=267067392 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267411296 ingress_bytes=370876416 link_bytes=370876416 token_bytes=534478688 weight_movement_bytes=103809024 total_nvlink_bytes=638287712 weight_to_token_ratio=0.194225 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395578095e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74138 payload_rows_remote=64816 dup_rows=24166 dup_rows_remote=21092 dedup_ratio=0.245829 router_tokens_per_expert=[797, 829, 776, 742, 739, 739, 731, 767, 793, 790, 763, 784, 776, 781, 815, 778, 781, 753, 733, 761, 752, 794, 759, 795, 729, 770, 763, 755, 780, 761, 779, 759, 810, 766, 792, 705, 733, 723, 782, 792, 772, 749, 801, 764, 764, 770, 759, 819, 829, 718, 707, 761, 805, 755, 732, 737, 762, 795, 804, 766, 750, 795, 728, 783] payload_rows_per_dst=[9224, 9468, 9214, 9226, 9202, 9322, 9130, 9352] dup_rows_per_dst=[3016, 3092, 3042, 2966, 3004, 3074, 2958, 3014] combine_rows_per_src=[9224, 9468, 9214, 9226, 9202, 9322, 9130, 9352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=65098 dup_rows=23982 dup_rows_remote=20894 dedup_ratio=0.243958 router_tokens_per_expert=[741, 782, 783, 775, 780, 751, 767, 763, 760, 860, 738, 697, 794, 822, 757, 754, 771, 801, 772, 714, 790, 756, 767, 738, 765, 787, 797, 744, 751, 724, 757, 756, 739, 738, 784, 770, 762, 779, 773, 772, 785, 796, 740, 737, 792, 764, 778, 774, 742, 837, 789, 707, 775, 800, 756, 750, 799, 747, 769, 790, 762, 774, 774, 784] payload_rows_per_dst=[9272, 9336, 9286, 9274, 9272, 9320, 9224, 9338] dup_rows_per_dst=[3012, 3028, 2932, 2888, 2962, 3012, 3088, 3060] combine_rows_per_src=[9272, 9336, 9286, 9274, 9272, 9320, 9224, 9338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267370496 dispatch_meta_bytes=345256 combine_payload_bytes=267370496 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267715752 ingress_bytes=336576512 link_bytes=336576512 token_bytes=535086248 weight_movement_bytes=69206016 total_nvlink_bytes=604292264 weight_to_token_ratio=0.129336 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266510318e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74616 payload_rows_remote=65232 dup_rows=23688 dup_rows_remote=20810 dedup_ratio=0.240967 router_tokens_per_expert=[791, 759, 773, 751, 800, 780, 746, 792, 746, 769, 782, 750, 830, 751, 787, 797, 797, 745, 780, 757, 735, 795, 741, 781, 732, 762, 774, 738, 740, 800, 780, 813, 753, 766, 776, 810, 802, 748, 782, 748, 792, 747, 779, 821, 741, 746, 752, 775, 732, 743, 725, 771, 778, 800, 733, 752, 752, 767, 779, 735, 742, 775, 786, 770] payload_rows_per_dst=[9404, 9376, 9384, 9342, 9326, 9360, 9074, 9350] dup_rows_per_dst=[2980, 3048, 2878, 2936, 3044, 2946, 2994, 2862] combine_rows_per_src=[9404, 9376, 9384, 9342, 9326, 9360, 9074, 9350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=64980 dup_rows=24062 dup_rows_remote=21220 dedup_ratio=0.244771 router_tokens_per_expert=[782, 771, 777, 718, 746, 770, 775, 755, 796, 718, 731, 788, 759, 751, 804, 750, 749, 829, 747, 762, 755, 794, 756, 759, 762, 825, 750, 780, 735, 807, 814, 803, 803, 761, 762, 755, 827, 726, 766, 744, 781, 801, 733, 762, 784, 717, 774, 819, 764, 767, 809, 759, 762, 759, 781, 770, 757, 691, 784, 768, 778, 747, 771, 752] payload_rows_per_dst=[9142, 9034, 9284, 9416, 9284, 9330, 9490, 9262] dup_rows_per_dst=[2844, 2818, 3018, 3136, 3086, 3028, 3290, 2842] combine_rows_per_src=[9142, 9034, 9284, 9416, 9284, 9330, 9490, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=65144 dup_rows=23920 dup_rows_remote=21084 dedup_ratio=0.243327 router_tokens_per_expert=[786, 749, 778, 809, 744, 774, 801, 753, 785, 774, 826, 795, 855, 730, 778, 705, 737, 760, 764, 760, 795, 754, 813, 744, 755, 723, 784, 771, 746, 747, 765, 747, 750, 760, 769, 742, 752, 781, 771, 770, 752, 789, 796, 825, 764, 736, 793, 745, 773, 781, 802, 724, 765, 754, 746, 753, 788, 749, 737, 742, 754, 860, 758, 764] payload_rows_per_dst=[9382, 9434, 9272, 9240, 9152, 9358, 9208, 9338] dup_rows_per_dst=[3006, 3062, 2982, 2836, 3038, 3042, 2988, 2966] combine_rows_per_src=[9382, 9434, 9272, 9240, 9152, 9358, 9208, 9338] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74134 payload_rows_remote=64848 dup_rows=24170 dup_rows_remote=21080 dedup_ratio=0.245870 router_tokens_per_expert=[784, 787, 721, 762, 825, 759, 783, 779, 761, 766, 753, 753, 738, 741, 733, 795, 735, 729, 773, 785, 766, 815, 799, 776, 781, 795, 716, 754, 729, 776, 743, 752, 744, 804, 724, 827, 769, 767, 793, 760, 773, 772, 766, 833, 729, 760, 796, 805, 748, 694, 800, 778, 713, 788, 742, 815, 775, 749, 756, 761, 748, 781, 793, 825] payload_rows_per_dst=[9320, 9166, 9284, 9206, 9286, 9388, 9232, 9252] dup_rows_per_dst=[3080, 2914, 3072, 2886, 3090, 3080, 2924, 3124] combine_rows_per_src=[9320, 9166, 9284, 9206, 9286, 9388, 9232, 9252] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265486336 dispatch_meta_bytes=343632 combine_payload_bytes=265486336 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265829968 ingress_bytes=334692352 link_bytes=334692352 token_bytes=531316304 weight_movement_bytes=69206016 total_nvlink_bytes=600522320 weight_to_token_ratio=0.130254 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259420376e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=64980 dup_rows=24062 dup_rows_remote=21220 dedup_ratio=0.244771 router_tokens_per_expert=[782, 771, 777, 718, 746, 770, 775, 755, 796, 718, 731, 788, 759, 751, 804, 750, 749, 829, 747, 762, 755, 794, 756, 759, 762, 825, 750, 780, 735, 807, 814, 803, 803, 761, 762, 755, 827, 726, 766, 744, 781, 801, 733, 762, 784, 717, 774, 819, 764, 767, 809, 759, 762, 759, 781, 770, 757, 691, 784, 768, 778, 747, 771, 752] payload_rows_per_dst=[9142, 9034, 9284, 9416, 9284, 9330, 9490, 9262] dup_rows_per_dst=[2844, 2818, 3018, 3136, 3086, 3028, 3290, 2842] combine_rows_per_src=[9142, 9034, 9284, 9416, 9284, 9330, 9490, 9262] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74616 payload_rows_remote=65232 dup_rows=23688 dup_rows_remote=20810 dedup_ratio=0.240967 router_tokens_per_expert=[791, 759, 773, 751, 800, 780, 746, 792, 746, 769, 782, 750, 830, 751, 787, 797, 797, 745, 780, 757, 735, 795, 741, 781, 732, 762, 774, 738, 740, 800, 780, 813, 753, 766, 776, 810, 802, 748, 782, 748, 792, 747, 779, 821, 741, 746, 752, 775, 732, 743, 725, 771, 778, 800, 733, 752, 752, 767, 779, 735, 742, 775, 786, 770] payload_rows_per_dst=[9404, 9376, 9384, 9342, 9326, 9360, 9074, 9350] dup_rows_per_dst=[2980, 3048, 2878, 2936, 3044, 2946, 2994, 2862] combine_rows_per_src=[9404, 9376, 9384, 9342, 9326, 9360, 9074, 9350] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74270 payload_rows_remote=65090 dup_rows=24034 dup_rows_remote=20960 dedup_ratio=0.244486 router_tokens_per_expert=[741, 769, 783, 760, 809, 766, 795, 783, 747, 771, 767, 732, 756, 777, 788, 750, 758, 777, 778, 782, 761, 731, 742, 753, 765, 757, 752, 781, 770, 758, 734, 739, 786, 733, 706, 817, 768, 765, 775, 777, 770, 742, 772, 793, 714, 797, 759, 754, 791, 774, 782, 729, 804, 796, 795, 763, 799, 786, 773, 812, 790, 746, 777, 775] payload_rows_per_dst=[9324, 9276, 9312, 9166, 9180, 9276, 9350, 9386] dup_rows_per_dst=[3088, 2900, 2852, 2946, 3074, 2926, 3118, 3130] combine_rows_per_src=[9324, 9276, 9312, 9166, 9180, 9276, 9350, 9386] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267067392 dispatch_meta_bytes=343904 combine_payload_bytes=267067392 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267411296 ingress_bytes=370876416 link_bytes=370876416 token_bytes=534478688 weight_movement_bytes=103809024 total_nvlink_bytes=638287712 weight_to_token_ratio=0.194225 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395578095e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267370496 dispatch_meta_bytes=345256 combine_payload_bytes=267370496 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267715752 ingress_bytes=336576512 link_bytes=336576512 token_bytes=535086248 weight_movement_bytes=69206016 total_nvlink_bytes=604292264 weight_to_token_ratio=0.129336 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266510318e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266829824 dispatch_meta_bytes=344912 combine_payload_bytes=266829824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267174736 ingress_bytes=301432832 link_bytes=301432832 token_bytes=534004560 weight_movement_bytes=34603008 total_nvlink_bytes=568607568 weight_to_token_ratio=0.064799 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134267479e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266158080 dispatch_meta_bytes=344800 combine_payload_bytes=266158080 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266502880 ingress_bytes=335364096 link_bytes=335364096 token_bytes=532660960 weight_movement_bytes=69206016 total_nvlink_bytes=601866976 weight_to_token_ratio=0.129925 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261948094e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267190272 dispatch_meta_bytes=344168 combine_payload_bytes=267190272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267534440 ingress_bytes=301793280 link_bytes=301793280 token_bytes=534724712 weight_movement_bytes=34603008 total_nvlink_bytes=569327720 weight_to_token_ratio=0.064712 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135623816e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=65098 dup_rows=23982 dup_rows_remote=20894 dedup_ratio=0.243958 router_tokens_per_expert=[741, 782, 783, 775, 780, 751, 767, 763, 760, 860, 738, 697, 794, 822, 757, 754, 771, 801, 772, 714, 790, 756, 767, 738, 765, 787, 797, 744, 751, 724, 757, 756, 739, 738, 784, 770, 762, 779, 773, 772, 785, 796, 740, 737, 792, 764, 778, 774, 742, 837, 789, 707, 775, 800, 756, 750, 799, 747, 769, 790, 762, 774, 774, 784] payload_rows_per_dst=[9272, 9336, 9286, 9274, 9272, 9320, 9224, 9338] dup_rows_per_dst=[3012, 3028, 2932, 2888, 2962, 3012, 3088, 3060] combine_rows_per_src=[9272, 9336, 9286, 9274, 9272, 9320, 9224, 9338] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266608640 dispatch_meta_bytes=344200 combine_payload_bytes=266608640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266952840 ingress_bytes=301211648 link_bytes=301211648 token_bytes=533561480 weight_movement_bytes=34603008 total_nvlink_bytes=568164488 weight_to_token_ratio=0.064853 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133435182e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74336 payload_rows_remote=65102 dup_rows=23968 dup_rows_remote=21042 dedup_ratio=0.243815 router_tokens_per_expert=[764, 821, 777, 768, 807, 701, 779, 760, 764, 764, 765, 739, 756, 746, 756, 780, 787, 777, 792, 786, 824, 780, 752, 781, 789, 794, 743, 771, 743, 789, 809, 785, 734, 755, 739, 795, 753, 774, 785, 769, 815, 766, 747, 720, 730, 752, 789, 761, 767, 754, 802, 736, 812, 794, 745, 753, 787, 746, 728, 741, 765, 730, 795, 764] payload_rows_per_dst=[9300, 9200, 9388, 9376, 9298, 9234, 9358, 9182] dup_rows_per_dst=[3054, 2940, 3170, 3070, 2910, 2926, 2968, 2930] combine_rows_per_src=[9300, 9200, 9388, 9376, 9298, 9234, 9358, 9182] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=343968 combine_payload_bytes=266641408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266985376 ingress_bytes=301244416 link_bytes=301244416 token_bytes=533626784 weight_movement_bytes=34603008 total_nvlink_bytes=568229792 weight_to_token_ratio=0.064845 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133558485e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267190272 dispatch_meta_bytes=344168 combine_payload_bytes=267190272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267534440 ingress_bytes=301793280 link_bytes=301793280 token_bytes=534724712 weight_movement_bytes=34603008 total_nvlink_bytes=569327720 weight_to_token_ratio=0.064712 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135623816e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266158080 dispatch_meta_bytes=344800 combine_payload_bytes=266158080 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266502880 ingress_bytes=335364096 link_bytes=335364096 token_bytes=532660960 weight_movement_bytes=69206016 total_nvlink_bytes=601866976 weight_to_token_ratio=0.129925 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261948094e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74128 payload_rows_remote=64886 dup_rows=24176 dup_rows_remote=21178 dedup_ratio=0.245931 router_tokens_per_expert=[761, 745, 760, 757, 757, 791, 759, 790, 772, 793, 762, 806, 773, 762, 778, 775, 748, 734, 821, 726, 746, 765, 777, 771, 757, 776, 729, 763, 832, 728, 736, 747, 780, 747, 761, 788, 799, 758, 794, 805, 736, 746, 789, 772, 760, 770, 798, 775, 799, 724, 745, 773, 813, 788, 784, 721, 763, 753, 768, 779, 759, 743, 779, 786] payload_rows_per_dst=[9242, 9308, 9378, 9332, 9348, 9142, 9254, 9124] dup_rows_per_dst=[2998, 3134, 3094, 3138, 3116, 2816, 3040, 2840] combine_rows_per_src=[9242, 9308, 9378, 9332, 9348, 9142, 9254, 9124] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266657792 dispatch_meta_bytes=344576 combine_payload_bytes=266657792 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267002368 ingress_bytes=335863808 link_bytes=335863808 token_bytes=533660160 weight_movement_bytes=69206016 total_nvlink_bytes=602866176 weight_to_token_ratio=0.129682 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263828470e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74526 payload_rows_remote=65146 dup_rows=23778 dup_rows_remote=20810 dedup_ratio=0.241882 router_tokens_per_expert=[777, 799, 723, 800, 791, 754, 779, 734, 743, 750, 742, 792, 761, 796, 813, 777, 755, 759, 716, 747, 794, 712, 764, 807, 787, 744, 806, 751, 788, 835, 757, 771, 747, 759, 756, 773, 761, 786, 754, 779, 790, 735, 774, 751, 764, 786, 756, 745, 755, 772, 809, 750, 744, 724, 781, 793, 767, 807, 785, 797, 736, 794, 743, 755] payload_rows_per_dst=[9358, 9380, 9202, 9396, 9258, 9274, 9308, 9350] dup_rows_per_dst=[2956, 2968, 2906, 3082, 2972, 2928, 2948, 3018] combine_rows_per_src=[9358, 9380, 9202, 9396, 9258, 9274, 9308, 9350] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266829824 dispatch_meta_bytes=344912 combine_payload_bytes=266829824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267174736 ingress_bytes=301432832 link_bytes=301432832 token_bytes=534004560 weight_movement_bytes=34603008 total_nvlink_bytes=568607568 weight_to_token_ratio=0.064799 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134267479e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265617408 dispatch_meta_bytes=343712 combine_payload_bytes=265617408 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265961120 ingress_bytes=369426432 link_bytes=369426432 token_bytes=531578528 weight_movement_bytes=103809024 total_nvlink_bytes=635387552 weight_to_token_ratio=0.195284 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390121922e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74526 payload_rows_remote=65146 dup_rows=23778 dup_rows_remote=20810 dedup_ratio=0.241882 router_tokens_per_expert=[777, 799, 723, 800, 791, 754, 779, 734, 743, 750, 742, 792, 761, 796, 813, 777, 755, 759, 716, 747, 794, 712, 764, 807, 787, 744, 806, 751, 788, 835, 757, 771, 747, 759, 756, 773, 761, 786, 754, 779, 790, 735, 774, 751, 764, 786, 756, 745, 755, 772, 809, 750, 744, 724, 781, 793, 767, 807, 785, 797, 736, 794, 743, 755] payload_rows_per_dst=[9358, 9380, 9202, 9396, 9258, 9274, 9308, 9350] dup_rows_per_dst=[2956, 2968, 2906, 3082, 2972, 2928, 2948, 3018] combine_rows_per_src=[9358, 9380, 9202, 9396, 9258, 9274, 9308, 9350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74336 payload_rows_remote=65102 dup_rows=23968 dup_rows_remote=21042 dedup_ratio=0.243815 router_tokens_per_expert=[764, 821, 777, 768, 807, 701, 779, 760, 764, 764, 765, 739, 756, 746, 756, 780, 787, 777, 792, 786, 824, 780, 752, 781, 789, 794, 743, 771, 743, 789, 809, 785, 734, 755, 739, 795, 753, 774, 785, 769, 815, 766, 747, 720, 730, 752, 789, 761, 767, 754, 802, 736, 812, 794, 745, 753, 787, 746, 728, 741, 765, 730, 795, 764] payload_rows_per_dst=[9300, 9200, 9388, 9376, 9298, 9234, 9358, 9182] dup_rows_per_dst=[3054, 2940, 3170, 3070, 2910, 2926, 2968, 2930] combine_rows_per_src=[9300, 9200, 9388, 9376, 9298, 9234, 9358, 9182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74352 payload_rows_remote=65052 dup_rows=23952 dup_rows_remote=20968 dedup_ratio=0.243652 router_tokens_per_expert=[751, 810, 808, 737, 831, 781, 749, 788, 782, 732, 756, 768, 780, 714, 773, 752, 744, 740, 791, 782, 740, 779, 788, 744, 751, 747, 794, 720, 788, 785, 750, 769, 772, 738, 760, 745, 777, 788, 758, 789, 744, 772, 730, 780, 774, 759, 783, 772, 803, 772, 744, 734, 772, 802, 773, 742, 835, 791, 783, 786, 735, 757, 783, 775] payload_rows_per_dst=[9422, 9218, 9190, 9204, 9238, 9346, 9300, 9434] dup_rows_per_dst=[3088, 2896, 3026, 3004, 3016, 2882, 2984, 3056] combine_rows_per_src=[9422, 9218, 9190, 9204, 9238, 9346, 9300, 9434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=65056 dup_rows=23962 dup_rows_remote=21020 dedup_ratio=0.243754 router_tokens_per_expert=[782, 790, 748, 767, 783, 741, 718, 801, 744, 751, 765, 785, 771, 746, 757, 769, 759, 726, 771, 776, 798, 761, 808, 715, 771, 769, 768, 750, 798, 808, 741, 757, 775, 799, 803, 770, 736, 780, 716, 827, 749, 786, 758, 719, 759, 728, 768, 792, 781, 767, 765, 770, 786, 775, 829, 774, 717, 750, 718, 787, 789, 796, 751, 838] payload_rows_per_dst=[9294, 9212, 9286, 9206, 9396, 9180, 9480, 9288] dup_rows_per_dst=[2966, 2964, 2942, 3118, 3016, 2938, 3014, 3004] combine_rows_per_src=[9294, 9212, 9286, 9206, 9396, 9180, 9480, 9288] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=64884 dup_rows=23948 dup_rows_remote=20804 dedup_ratio=0.243612 router_tokens_per_expert=[800, 750, 732, 775, 765, 774, 742, 815, 780, 796, 741, 765, 687, 782, 767, 782, 822, 806, 756, 729, 762, 741, 737, 782, 749, 771, 768, 783, 701, 775, 814, 776, 775, 756, 765, 757, 797, 737, 786, 796, 755, 748, 782, 756, 800, 714, 801, 751, 779, 782, 748, 758, 750, 736, 742, 748, 790, 766, 808, 775, 814, 782, 777, 796] payload_rows_per_dst=[9064, 9296, 9276, 9272, 9474, 9252, 9250, 9472] dup_rows_per_dst=[2952, 2904, 2994, 3002, 3154, 2962, 2836, 3144] combine_rows_per_src=[9064, 9296, 9276, 9272, 9474, 9252, 9250, 9472] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74452 payload_rows_remote=65298 dup_rows=23852 dup_rows_remote=21036 dedup_ratio=0.242635 router_tokens_per_expert=[758, 694, 816, 764, 779, 805, 787, 769, 792, 751, 758, 750, 759, 737, 775, 768, 785, 802, 765, 741, 783, 794, 809, 779, 766, 784, 740, 720, 757, 754, 713, 751, 738, 775, 739, 774, 745, 753, 738, 795, 771, 784, 762, 768, 756, 790, 860, 792, 783, 779, 792, 732, 802, 745, 747, 786, 781, 740, 738, 795, 858, 720, 769, 740] payload_rows_per_dst=[9282, 9240, 9426, 9154, 9216, 9604, 9314, 9216] dup_rows_per_dst=[3062, 2940, 3090, 2816, 2898, 2962, 3018, 3066] combine_rows_per_src=[9282, 9240, 9426, 9154, 9216, 9604, 9314, 9216] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=65056 dup_rows=23962 dup_rows_remote=21020 dedup_ratio=0.243754 router_tokens_per_expert=[782, 790, 748, 767, 783, 741, 718, 801, 744, 751, 765, 785, 771, 746, 757, 769, 759, 726, 771, 776, 798, 761, 808, 715, 771, 769, 768, 750, 798, 808, 741, 757, 775, 799, 803, 770, 736, 780, 716, 827, 749, 786, 758, 719, 759, 728, 768, 792, 781, 767, 765, 770, 786, 775, 829, 774, 717, 750, 718, 787, 789, 796, 751, 838] payload_rows_per_dst=[9294, 9212, 9286, 9206, 9396, 9180, 9480, 9288] dup_rows_per_dst=[2966, 2964, 2942, 3118, 3016, 2938, 3014, 3004] combine_rows_per_src=[9294, 9212, 9286, 9206, 9396, 9180, 9480, 9288] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=344256 combine_payload_bytes=265773056 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266117312 ingress_bytes=404185088 link_bytes=404185088 token_bytes=531890368 weight_movement_bytes=138412032 total_nvlink_bytes=670302400 weight_to_token_ratio=0.260227 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520915947e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74260 payload_rows_remote=65076 dup_rows=24044 dup_rows_remote=21070 dedup_ratio=0.244588 router_tokens_per_expert=[743, 783, 783, 762, 767, 764, 753, 751, 774, 761, 818, 744, 733, 790, 738, 743, 743, 777, 774, 726, 732, 782, 726, 837, 735, 744, 767, 750, 809, 728, 782, 761, 788, 749, 797, 783, 725, 734, 737, 766, 780, 769, 789, 753, 762, 824, 775, 788, 770, 752, 787, 757, 777, 795, 759, 758, 788, 765, 768, 813, 778, 799, 777, 810] payload_rows_per_dst=[9252, 9318, 9236, 9164, 9184, 9360, 9264, 9482] dup_rows_per_dst=[2960, 2884, 2958, 2988, 2974, 3120, 3046, 3114] combine_rows_per_src=[9252, 9318, 9236, 9164, 9184, 9360, 9264, 9482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266838016 dispatch_meta_bytes=343824 combine_payload_bytes=266838016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267181840 ingress_bytes=301441024 link_bytes=301441024 token_bytes=534019856 weight_movement_bytes=34603008 total_nvlink_bytes=568622864 weight_to_token_ratio=0.064797 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134298305e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74352 payload_rows_remote=65052 dup_rows=23952 dup_rows_remote=20968 dedup_ratio=0.243652 router_tokens_per_expert=[751, 810, 808, 737, 831, 781, 749, 788, 782, 732, 756, 768, 780, 714, 773, 752, 744, 740, 791, 782, 740, 779, 788, 744, 751, 747, 794, 720, 788, 785, 750, 769, 772, 738, 760, 745, 777, 788, 758, 789, 744, 772, 730, 780, 774, 759, 783, 772, 803, 772, 744, 734, 772, 802, 773, 742, 835, 791, 783, 786, 735, 757, 783, 775] payload_rows_per_dst=[9422, 9218, 9190, 9204, 9238, 9346, 9300, 9434] dup_rows_per_dst=[3088, 2896, 3026, 3004, 3016, 2882, 2984, 3056] combine_rows_per_src=[9422, 9218, 9190, 9204, 9238, 9346, 9300, 9434] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266452992 dispatch_meta_bytes=344080 combine_payload_bytes=266452992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266797072 ingress_bytes=301056000 link_bytes=301056000 token_bytes=533250064 weight_movement_bytes=34603008 total_nvlink_bytes=567853072 weight_to_token_ratio=0.064891 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132849491e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266657792 dispatch_meta_bytes=344576 combine_payload_bytes=266657792 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267002368 ingress_bytes=335863808 link_bytes=335863808 token_bytes=533660160 weight_movement_bytes=69206016 total_nvlink_bytes=602866176 weight_to_token_ratio=0.129682 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263828470e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266469376 dispatch_meta_bytes=344304 combine_payload_bytes=266469376 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266813680 ingress_bytes=335675392 link_bytes=335675392 token_bytes=533283056 weight_movement_bytes=69206016 total_nvlink_bytes=602489072 weight_to_token_ratio=0.129774 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263119476e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265764864 dispatch_meta_bytes=342752 combine_payload_bytes=265764864 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266107616 ingress_bytes=404176896 link_bytes=404176896 token_bytes=531872480 weight_movement_bytes=138412032 total_nvlink_bytes=670284512 weight_to_token_ratio=0.260235 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520885121e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=267460608 dispatch_meta_bytes=345336 combine_payload_bytes=267460608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267805944 ingress_bytes=302063616 link_bytes=302063616 token_bytes=535266552 weight_movement_bytes=34603008 total_nvlink_bytes=569869560 weight_to_token_ratio=0.064646 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136641069e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74130 payload_rows_remote=64732 dup_rows=24174 dup_rows_remote=20994 dedup_ratio=0.245911 router_tokens_per_expert=[765, 752, 763, 746, 758, 748, 746, 762, 796, 751, 744, 755, 785, 818, 775, 781, 725, 768, 772, 757, 791, 738, 767, 794, 755, 787, 776, 735, 764, 770, 793, 757, 774, 756, 763, 765, 830, 768, 764, 759, 822, 809, 765, 819, 798, 773, 761, 742, 744, 733, 764, 749, 758, 717, 802, 791, 764, 727, 779, 741, 796, 730, 769, 826] payload_rows_per_dst=[9200, 9294, 9306, 9266, 9342, 9398, 9060, 9264] dup_rows_per_dst=[2880, 3116, 2918, 3008, 3016, 3180, 3056, 3000] combine_rows_per_src=[9200, 9294, 9306, 9266, 9342, 9398, 9060, 9264] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266469376 dispatch_meta_bytes=344304 combine_payload_bytes=266469376 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266813680 ingress_bytes=335675392 link_bytes=335675392 token_bytes=533283056 weight_movement_bytes=69206016 total_nvlink_bytes=602489072 weight_to_token_ratio=0.129774 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263119476e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266551296 dispatch_meta_bytes=344584 combine_payload_bytes=266551296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266895880 ingress_bytes=301154304 link_bytes=301154304 token_bytes=533447176 weight_movement_bytes=34603008 total_nvlink_bytes=568050184 weight_to_token_ratio=0.064867 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133219401e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266452992 dispatch_meta_bytes=344080 combine_payload_bytes=266452992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266797072 ingress_bytes=301056000 link_bytes=301056000 token_bytes=533250064 weight_movement_bytes=34603008 total_nvlink_bytes=567853072 weight_to_token_ratio=0.064891 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132849491e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265142272 dispatch_meta_bytes=342904 combine_payload_bytes=265142272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265485176 ingress_bytes=299745280 link_bytes=299745280 token_bytes=530627448 weight_movement_bytes=34603008 total_nvlink_bytes=565230456 weight_to_token_ratio=0.065211 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127917357e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74270 payload_rows_remote=65090 dup_rows=24034 dup_rows_remote=20960 dedup_ratio=0.244486 router_tokens_per_expert=[741, 769, 783, 760, 809, 766, 795, 783, 747, 771, 767, 732, 756, 777, 788, 750, 758, 777, 778, 782, 761, 731, 742, 753, 765, 757, 752, 781, 770, 758, 734, 739, 786, 733, 706, 817, 768, 765, 775, 777, 770, 742, 772, 793, 714, 797, 759, 754, 791, 774, 782, 729, 804, 796, 795, 763, 799, 786, 773, 812, 790, 746, 777, 775] payload_rows_per_dst=[9324, 9276, 9312, 9166, 9180, 9276, 9350, 9386] dup_rows_per_dst=[3088, 2900, 2852, 2946, 3074, 2926, 3118, 3130] combine_rows_per_src=[9324, 9276, 9312, 9166, 9180, 9276, 9350, 9386] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=65098 dup_rows=23920 dup_rows_remote=20912 dedup_ratio=0.243327 router_tokens_per_expert=[773, 712, 755, 781, 759, 763, 799, 752, 772, 735, 798, 750, 800, 780, 758, 762, 778, 789, 752, 790, 796, 748, 767, 726, 725, 751, 765, 727, 777, 816, 804, 782, 809, 792, 755, 708, 792, 770, 767, 757, 767, 750, 795, 804, 792, 801, 769, 756, 752, 799, 730, 789, 756, 754, 741, 782, 789, 778, 746, 769, 738, 743, 781, 779] payload_rows_per_dst=[9286, 9660, 9222, 9312, 9360, 9302, 9290, 8952] dup_rows_per_dst=[3008, 3284, 2992, 3140, 2940, 3034, 2926, 2596] combine_rows_per_src=[9286, 9660, 9222, 9312, 9360, 9302, 9290, 8952] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=64820 dup_rows=24132 dup_rows_remote=21020 dedup_ratio=0.245483 router_tokens_per_expert=[747, 755, 776, 783, 757, 756, 822, 750, 759, 774, 727, 750, 766, 769, 752, 788, 751, 740, 772, 773, 746, 762, 733, 823, 797, 735, 759, 828, 764, 733, 770, 770, 735, 775, 766, 794, 747, 752, 768, 787, 804, 745, 761, 794, 826, 726, 765, 754, 748, 750, 753, 762, 780, 837, 801, 801, 789, 770, 760, 740, 771, 734, 768, 802] payload_rows_per_dst=[9338, 9258, 9236, 9264, 9236, 9330, 9352, 9158] dup_rows_per_dst=[2954, 2912, 2964, 3048, 3012, 3020, 3112, 3110] combine_rows_per_src=[9338, 9258, 9236, 9264, 9236, 9330, 9352, 9158] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=64918 dup_rows=24096 dup_rows_remote=21088 dedup_ratio=0.245117 router_tokens_per_expert=[735, 772, 787, 816, 741, 763, 782, 796, 796, 798, 767, 770, 818, 703, 737, 760, 816, 760, 776, 758, 845, 786, 777, 797, 812, 798, 772, 754, 755, 739, 787, 799, 746, 761, 801, 799, 773, 715, 759, 760, 724, 771, 731, 773, 780, 720, 810, 729, 766, 718, 753, 776, 765, 786, 785, 766, 751, 750, 752, 762, 721, 758, 764, 755] payload_rows_per_dst=[9396, 9290, 9398, 9328, 9194, 9294, 9242, 9066] dup_rows_per_dst=[2988, 3008, 3232, 3104, 3034, 2782, 2988, 2960] combine_rows_per_src=[9396, 9290, 9398, 9328, 9194, 9294, 9242, 9066] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74130 payload_rows_remote=64732 dup_rows=24174 dup_rows_remote=20994 dedup_ratio=0.245911 router_tokens_per_expert=[765, 752, 763, 746, 758, 748, 746, 762, 796, 751, 744, 755, 785, 818, 775, 781, 725, 768, 772, 757, 791, 738, 767, 794, 755, 787, 776, 735, 764, 770, 793, 757, 774, 756, 763, 765, 830, 768, 764, 759, 822, 809, 765, 819, 798, 773, 761, 742, 744, 733, 764, 749, 758, 717, 802, 791, 764, 727, 779, 741, 796, 730, 769, 826] payload_rows_per_dst=[9200, 9294, 9306, 9266, 9342, 9398, 9060, 9264] dup_rows_per_dst=[2880, 3116, 2918, 3008, 3016, 3180, 3056, 3000] combine_rows_per_src=[9200, 9294, 9306, 9266, 9342, 9398, 9060, 9264] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64658 dup_rows=24112 dup_rows_remote=20844 dedup_ratio=0.245280 router_tokens_per_expert=[738, 781, 735, 811, 775, 732, 750, 813, 790, 720, 779, 744, 792, 782, 811, 754, 753, 708, 785, 736, 724, 792, 777, 774, 819, 774, 764, 783, 751, 764, 797, 775, 743, 728, 782, 746, 795, 757, 778, 758, 800, 758, 815, 694, 742, 784, 796, 735, 780, 758, 783, 760, 710, 809, 787, 746, 748, 724, 792, 768, 793, 804, 802, 794] payload_rows_per_dst=[9282, 9166, 9270, 9062, 9276, 9326, 9276, 9534] dup_rows_per_dst=[2882, 3072, 2934, 2702, 3046, 3112, 3096, 3268] combine_rows_per_src=[9282, 9166, 9270, 9062, 9276, 9326, 9276, 9534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=64850 dup_rows=24060 dup_rows_remote=21062 dedup_ratio=0.244751 router_tokens_per_expert=[751, 756, 753, 772, 771, 781, 815, 784, 764, 777, 773, 778, 787, 772, 789, 683, 744, 734, 794, 769, 780, 835, 762, 778, 762, 803, 740, 789, 788, 754, 789, 759, 784, 730, 731, 731, 792, 753, 766, 753, 774, 770, 752, 772, 753, 755, 748, 759, 758, 762, 730, 754, 790, 758, 791, 787, 777, 754, 770, 807, 775, 808, 748, 774] payload_rows_per_dst=[9316, 9218, 9394, 9358, 9100, 9180, 9302, 9376] dup_rows_per_dst=[3050, 3028, 2998, 3010, 2980, 2986, 2958, 3050] combine_rows_per_src=[9316, 9218, 9394, 9358, 9100, 9180, 9302, 9376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266608640 dispatch_meta_bytes=344200 combine_payload_bytes=266608640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266952840 ingress_bytes=301211648 link_bytes=301211648 token_bytes=533561480 weight_movement_bytes=34603008 total_nvlink_bytes=568164488 weight_to_token_ratio=0.064853 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133435182e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74738 payload_rows_remote=65288 dup_rows=23566 dup_rows_remote=20586 dedup_ratio=0.239726 router_tokens_per_expert=[759, 745, 740, 771, 743, 731, 809, 767, 818, 766, 757, 780, 763, 745, 794, 798, 796, 761, 728, 766, 767, 794, 778, 785, 766, 756, 800, 791, 792, 785, 724, 801, 750, 793, 729, 753, 820, 784, 802, 743, 745, 794, 738, 750, 799, 775, 784, 750, 739, 740, 816, 813, 714, 781, 780, 736, 741, 822, 740, 764, 728, 718, 765, 770] payload_rows_per_dst=[9352, 9456, 9326, 9450, 9324, 9272, 9284, 9274] dup_rows_per_dst=[2778, 2986, 3024, 2980, 3024, 2998, 2954, 2822] combine_rows_per_src=[9352, 9456, 9326, 9450, 9324, 9272, 9284, 9274] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=64850 dup_rows=24060 dup_rows_remote=21062 dedup_ratio=0.244751 router_tokens_per_expert=[751, 756, 753, 772, 771, 781, 815, 784, 764, 777, 773, 778, 787, 772, 789, 683, 744, 734, 794, 769, 780, 835, 762, 778, 762, 803, 740, 789, 788, 754, 789, 759, 784, 730, 731, 731, 792, 753, 766, 753, 774, 770, 752, 772, 753, 755, 748, 759, 758, 762, 730, 754, 790, 758, 791, 787, 777, 754, 770, 807, 775, 808, 748, 774] payload_rows_per_dst=[9316, 9218, 9394, 9358, 9100, 9180, 9302, 9376] dup_rows_per_dst=[3050, 3028, 2998, 3010, 2980, 2986, 2958, 3050] combine_rows_per_src=[9316, 9218, 9394, 9358, 9100, 9180, 9302, 9376] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=344040 combine_payload_bytes=266641408 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266985448 ingress_bytes=335847424 link_bytes=335847424 token_bytes=533626856 weight_movement_bytes=69206016 total_nvlink_bytes=602832872 weight_to_token_ratio=0.129690 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263766819e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74118 payload_rows_remote=64900 dup_rows=24186 dup_rows_remote=21154 dedup_ratio=0.246033 router_tokens_per_expert=[812, 748, 772, 754, 776, 774, 776, 760, 769, 793, 794, 731, 755, 776, 765, 776, 756, 759, 779, 737, 773, 744, 763, 775, 775, 796, 782, 787, 763, 751, 722, 759, 793, 775, 808, 744, 750, 768, 737, 750, 758, 769, 773, 757, 781, 742, 768, 750, 805, 754, 767, 792, 787, 765, 794, 765, 759, 769, 772, 776, 745, 763, 777, 787] payload_rows_per_dst=[9240, 9314, 9252, 9226, 9218, 9222, 9320, 9326] dup_rows_per_dst=[3104, 3004, 2920, 3044, 3032, 2974, 3138, 2970] combine_rows_per_src=[9240, 9314, 9252, 9226, 9218, 9222, 9320, 9326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=343360 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265846080 ingress_bytes=300105728 link_bytes=300105728 token_bytes=531348800 weight_movement_bytes=34603008 total_nvlink_bytes=565951808 weight_to_token_ratio=0.065123 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129273694e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 1, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=344024 combine_payload_bytes=265904128 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266248152 ingress_bytes=335110144 link_bytes=335110144 token_bytes=532152280 weight_movement_bytes=69206016 total_nvlink_bytes=601358296 weight_to_token_ratio=0.130049 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260992493e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=64820 dup_rows=24132 dup_rows_remote=21020 dedup_ratio=0.245483 router_tokens_per_expert=[747, 755, 776, 783, 757, 756, 822, 750, 759, 774, 727, 750, 766, 769, 752, 788, 751, 740, 772, 773, 746, 762, 733, 823, 797, 735, 759, 828, 764, 733, 770, 770, 735, 775, 766, 794, 747, 752, 768, 787, 804, 745, 761, 794, 826, 726, 765, 754, 748, 750, 753, 762, 780, 837, 801, 801, 789, 770, 760, 740, 771, 734, 768, 802] payload_rows_per_dst=[9338, 9258, 9236, 9264, 9236, 9330, 9352, 9158] dup_rows_per_dst=[2954, 2912, 2964, 3048, 3012, 3020, 3112, 3110] combine_rows_per_src=[9338, 9258, 9236, 9264, 9236, 9330, 9352, 9158] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265142272 dispatch_meta_bytes=342904 combine_payload_bytes=265142272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265485176 ingress_bytes=299745280 link_bytes=299745280 token_bytes=530627448 weight_movement_bytes=34603008 total_nvlink_bytes=565230456 weight_to_token_ratio=0.065211 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127917357e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=264839168 dispatch_meta_bytes=342008 combine_payload_bytes=264839168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265181176 ingress_bytes=299442176 link_bytes=299442176 token_bytes=530020344 weight_movement_bytes=34603008 total_nvlink_bytes=564623352 weight_to_token_ratio=0.065286 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126776801e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267419648 dispatch_meta_bytes=343496 combine_payload_bytes=267419648 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=267763144 ingress_bytes=440434688 link_bytes=440434688 token_bytes=535182792 weight_movement_bytes=173015040 total_nvlink_bytes=708197832 weight_to_token_ratio=0.323282 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.657320273e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74316 payload_rows_remote=64990 dup_rows=23988 dup_rows_remote=20872 dedup_ratio=0.244019 router_tokens_per_expert=[746, 770, 745, 757, 762, 739, 783, 750, 770, 774, 755, 759, 749, 801, 733, 722, 800, 841, 754, 786, 748, 787, 774, 781, 805, 763, 818, 785, 762, 771, 740, 755, 802, 775, 728, 765, 769, 758, 777, 757, 795, 805, 806, 790, 788, 729, 771, 737, 759, 770, 735, 778, 768, 785, 700, 735, 745, 786, 775, 755, 770, 782, 775, 797] payload_rows_per_dst=[9166, 9168, 9446, 9408, 9254, 9326, 9210, 9338] dup_rows_per_dst=[2938, 2958, 3096, 2990, 3008, 3116, 2850, 3032] combine_rows_per_src=[9166, 9168, 9446, 9408, 9254, 9326, 9210, 9338] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265625600 dispatch_meta_bytes=343648 combine_payload_bytes=265625600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265969248 ingress_bytes=300228608 link_bytes=300228608 token_bytes=531594848 weight_movement_bytes=34603008 total_nvlink_bytes=566197856 weight_to_token_ratio=0.065093 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129736082e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265830400 dispatch_meta_bytes=344216 combine_payload_bytes=265830400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266174616 ingress_bytes=300433408 link_bytes=300433408 token_bytes=532005016 weight_movement_bytes=34603008 total_nvlink_bytes=566608024 weight_to_token_ratio=0.065043 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130506727e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=343360 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265846080 ingress_bytes=300105728 link_bytes=300105728 token_bytes=531348800 weight_movement_bytes=34603008 total_nvlink_bytes=565951808 weight_to_token_ratio=0.065123 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129273694e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=343448 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266542488 ingress_bytes=300802048 link_bytes=300802048 token_bytes=532741528 weight_movement_bytes=34603008 total_nvlink_bytes=567344536 weight_to_token_ratio=0.064953 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131893890e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267419648 dispatch_meta_bytes=343496 combine_payload_bytes=267419648 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=267763144 ingress_bytes=440434688 link_bytes=440434688 token_bytes=535182792 weight_movement_bytes=173015040 total_nvlink_bytes=708197832 weight_to_token_ratio=0.323282 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.657320273e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74260 payload_rows_remote=65076 dup_rows=24044 dup_rows_remote=21070 dedup_ratio=0.244588 router_tokens_per_expert=[743, 783, 783, 762, 767, 764, 753, 751, 774, 761, 818, 744, 733, 790, 738, 743, 743, 777, 774, 726, 732, 782, 726, 837, 735, 744, 767, 750, 809, 728, 782, 761, 788, 749, 797, 783, 725, 734, 737, 766, 780, 769, 789, 753, 762, 824, 775, 788, 770, 752, 787, 757, 777, 795, 759, 758, 788, 765, 768, 813, 778, 799, 777, 810] payload_rows_per_dst=[9252, 9318, 9236, 9164, 9184, 9360, 9264, 9482] dup_rows_per_dst=[2960, 2884, 2958, 2988, 2974, 3120, 3046, 3114] combine_rows_per_src=[9252, 9318, 9236, 9164, 9184, 9360, 9264, 9482] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74348 payload_rows_remote=64966 dup_rows=23956 dup_rows_remote=20794 dedup_ratio=0.243693 router_tokens_per_expert=[730, 813, 765, 827, 808, 771, 775, 775, 764, 729, 790, 761, 740, 795, 733, 768, 738, 778, 768, 788, 731, 723, 809, 731, 780, 773, 776, 808, 750, 786, 777, 796, 786, 784, 771, 783, 760, 778, 791, 782, 746, 725, 802, 745, 757, 804, 792, 744, 750, 752, 774, 754, 783, 782, 761, 727, 773, 776, 761, 743, 746, 743, 767, 754] payload_rows_per_dst=[9382, 9278, 9248, 9362, 9358, 9216, 9294, 9210] dup_rows_per_dst=[3162, 2904, 2884, 3092, 3112, 3014, 2872, 2916] combine_rows_per_src=[9382, 9278, 9248, 9362, 9358, 9216, 9294, 9210] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74478 payload_rows_remote=65240 dup_rows=23826 dup_rows_remote=20942 dedup_ratio=0.242371 router_tokens_per_expert=[754, 725, 771, 794, 787, 764, 784, 775, 789, 694, 788, 706, 763, 775, 768, 778, 710, 727, 774, 734, 765, 792, 735, 791, 773, 799, 759, 803, 776, 730, 775, 779, 807, 812, 801, 784, 744, 785, 767, 806, 744, 718, 745, 724, 780, 744, 794, 773, 763, 758, 789, 753, 769, 760, 778, 795, 761, 787, 745, 803, 788, 788, 801, 749] payload_rows_per_dst=[9286, 9238, 9262, 9330, 9524, 9176, 9318, 9344] dup_rows_per_dst=[3022, 2884, 2794, 3058, 3088, 2868, 3012, 3100] combine_rows_per_src=[9286, 9238, 9262, 9330, 9524, 9176, 9318, 9344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74446 payload_rows_remote=65062 dup_rows=23858 dup_rows_remote=20780 dedup_ratio=0.242696 router_tokens_per_expert=[782, 769, 761, 749, 732, 796, 758, 761, 761, 722, 763, 718, 738, 813, 806, 755, 711, 737, 750, 781, 793, 791, 788, 742, 818, 764, 805, 788, 771, 837, 765, 752, 797, 723, 777, 762, 759, 805, 787, 776, 760, 774, 764, 779, 801, 701, 730, 755, 828, 781, 749, 799, 776, 806, 794, 698, 749, 789, 772, 765, 796, 696, 753, 774] payload_rows_per_dst=[9278, 9268, 9240, 9452, 9278, 9274, 9384, 9272] dup_rows_per_dst=[2938, 2884, 2946, 3148, 3094, 2854, 3078, 2916] combine_rows_per_src=[9278, 9268, 9240, 9452, 9278, 9274, 9384, 9272] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74478 payload_rows_remote=65240 dup_rows=23826 dup_rows_remote=20942 dedup_ratio=0.242371 router_tokens_per_expert=[754, 725, 771, 794, 787, 764, 784, 775, 789, 694, 788, 706, 763, 775, 768, 778, 710, 727, 774, 734, 765, 792, 735, 791, 773, 799, 759, 803, 776, 730, 775, 779, 807, 812, 801, 784, 744, 785, 767, 806, 744, 718, 745, 724, 780, 744, 794, 773, 763, 758, 789, 753, 769, 760, 778, 795, 761, 787, 745, 803, 788, 788, 801, 749] payload_rows_per_dst=[9286, 9238, 9262, 9330, 9524, 9176, 9318, 9344] dup_rows_per_dst=[3022, 2884, 2794, 3058, 3088, 2868, 3012, 3100] combine_rows_per_src=[9286, 9238, 9262, 9330, 9524, 9176, 9318, 9344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266551296 dispatch_meta_bytes=344584 combine_payload_bytes=266551296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266895880 ingress_bytes=301154304 link_bytes=301154304 token_bytes=533447176 weight_movement_bytes=34603008 total_nvlink_bytes=568050184 weight_to_token_ratio=0.064867 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133219401e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=64932 dup_rows=23962 dup_rows_remote=20846 dedup_ratio=0.243754 router_tokens_per_expert=[778, 741, 816, 736, 763, 806, 777, 748, 778, 799, 797, 754, 794, 786, 731, 791, 762, 811, 805, 747, 765, 754, 784, 794, 731, 787, 765, 721, 756, 730, 752, 785, 768, 739, 778, 800, 778, 806, 799, 763, 761, 757, 752, 819, 742, 794, 777, 805, 761, 772, 722, 749, 713, 760, 742, 754, 804, 773, 734, 777, 705, 770, 773, 761] payload_rows_per_dst=[9302, 9338, 9416, 8920, 9344, 9442, 9170, 9410] dup_rows_per_dst=[2946, 3122, 3028, 2802, 2916, 3054, 2978, 3116] combine_rows_per_src=[9302, 9338, 9416, 8920, 9344, 9442, 9170, 9410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266100736 dispatch_meta_bytes=343040 combine_payload_bytes=266100736 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266443776 ingress_bytes=335306752 link_bytes=335306752 token_bytes=532544512 weight_movement_bytes=69206016 total_nvlink_bytes=601750528 weight_to_token_ratio=0.129953 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261732313e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74316 payload_rows_remote=64990 dup_rows=23988 dup_rows_remote=20872 dedup_ratio=0.244019 router_tokens_per_expert=[746, 770, 745, 757, 762, 739, 783, 750, 770, 774, 755, 759, 749, 801, 733, 722, 800, 841, 754, 786, 748, 787, 774, 781, 805, 763, 818, 785, 762, 771, 740, 755, 802, 775, 728, 765, 769, 758, 777, 757, 795, 805, 806, 790, 788, 729, 771, 737, 759, 770, 735, 778, 768, 785, 700, 735, 745, 786, 775, 755, 770, 782, 775, 797] payload_rows_per_dst=[9166, 9168, 9446, 9408, 9254, 9326, 9210, 9338] dup_rows_per_dst=[2938, 2958, 3096, 2990, 3008, 3116, 2850, 3032] combine_rows_per_src=[9166, 9168, 9446, 9408, 9254, 9326, 9210, 9338] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74218 payload_rows_remote=64956 dup_rows=24086 dup_rows_remote=21020 dedup_ratio=0.245015 router_tokens_per_expert=[764, 723, 782, 766, 802, 769, 794, 755, 753, 770, 792, 709, 759, 787, 806, 734, 781, 746, 765, 728, 785, 743, 792, 773, 734, 739, 770, 805, 763, 779, 794, 780, 787, 792, 789, 754, 746, 739, 757, 762, 792, 771, 777, 800, 810, 777, 747, 754, 794, 770, 709, 746, 732, 743, 809, 740, 751, 743, 779, 775, 805, 779, 810, 771] payload_rows_per_dst=[9314, 9278, 9270, 9262, 9284, 9330, 9094, 9386] dup_rows_per_dst=[2996, 2942, 2956, 3066, 2968, 3126, 2992, 3040] combine_rows_per_src=[9314, 9278, 9270, 9262, 9284, 9330, 9094, 9386] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266100736 dispatch_meta_bytes=343040 combine_payload_bytes=266100736 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266443776 ingress_bytes=335306752 link_bytes=335306752 token_bytes=532544512 weight_movement_bytes=69206016 total_nvlink_bytes=601750528 weight_to_token_ratio=0.129953 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261732313e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=65014 dup_rows=23974 dup_rows_remote=20982 dedup_ratio=0.243876 router_tokens_per_expert=[781, 780, 758, 744, 799, 818, 762, 782, 735, 737, 759, 753, 793, 783, 791, 742, 765, 714, 798, 769, 777, 772, 778, 781, 759, 760, 762, 805, 817, 769, 771, 798, 783, 791, 764, 813, 771, 723, 782, 783, 741, 789, 751, 782, 778, 714, 747, 742, 752, 789, 786, 738, 804, 794, 731, 764, 743, 740, 735, 768, 787, 785, 750, 720] payload_rows_per_dst=[9370, 9164, 9316, 9432, 9314, 9162, 9388, 9184] dup_rows_per_dst=[3078, 3022, 2992, 3050, 3106, 2926, 2928, 2872] combine_rows_per_src=[9370, 9164, 9316, 9432, 9314, 9162, 9388, 9184] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74460 payload_rows_remote=65234 dup_rows=23844 dup_rows_remote=20936 dedup_ratio=0.242554 router_tokens_per_expert=[778, 784, 744, 761, 783, 788, 770, 757, 785, 808, 758, 816, 785, 747, 791, 749, 746, 772, 766, 753, 748, 772, 694, 803, 790, 780, 760, 752, 747, 736, 761, 769, 724, 771, 730, 787, 749, 742, 776, 788, 798, 788, 767, 761, 718, 725, 776, 832, 764, 734, 811, 783, 806, 815, 779, 762, 715, 777, 813, 789, 743, 730, 772, 774] payload_rows_per_dst=[9284, 9326, 9258, 9256, 9226, 9322, 9476, 9312] dup_rows_per_dst=[3046, 3152, 2850, 2934, 2908, 3008, 3032, 2914] combine_rows_per_src=[9284, 9326, 9258, 9256, 9226, 9322, 9476, 9312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266493952 dispatch_meta_bytes=343368 combine_payload_bytes=266493952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266837320 ingress_bytes=301096960 link_bytes=301096960 token_bytes=533331272 weight_movement_bytes=34603008 total_nvlink_bytes=567934280 weight_to_token_ratio=0.064881 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133003620e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=267223040 dispatch_meta_bytes=344728 combine_payload_bytes=267223040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267567768 ingress_bytes=301826048 link_bytes=301826048 token_bytes=534790808 weight_movement_bytes=34603008 total_nvlink_bytes=569393816 weight_to_token_ratio=0.064704 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135747120e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74446 payload_rows_remote=65062 dup_rows=23858 dup_rows_remote=20780 dedup_ratio=0.242696 router_tokens_per_expert=[782, 769, 761, 749, 732, 796, 758, 761, 761, 722, 763, 718, 738, 813, 806, 755, 711, 737, 750, 781, 793, 791, 788, 742, 818, 764, 805, 788, 771, 837, 765, 752, 797, 723, 777, 762, 759, 805, 787, 776, 760, 774, 764, 779, 801, 701, 730, 755, 828, 781, 749, 799, 776, 806, 794, 698, 749, 789, 772, 765, 796, 696, 753, 774] payload_rows_per_dst=[9278, 9268, 9240, 9452, 9278, 9274, 9384, 9272] dup_rows_per_dst=[2938, 2884, 2946, 3148, 3094, 2854, 3078, 2916] combine_rows_per_src=[9278, 9268, 9240, 9452, 9278, 9274, 9384, 9272] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265961472 dispatch_meta_bytes=343112 combine_payload_bytes=265961472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266304584 ingress_bytes=300564480 link_bytes=300564480 token_bytes=532266056 weight_movement_bytes=34603008 total_nvlink_bytes=566869064 weight_to_token_ratio=0.065011 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130999941e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=343448 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266542488 ingress_bytes=300802048 link_bytes=300802048 token_bytes=532741528 weight_movement_bytes=34603008 total_nvlink_bytes=567344536 weight_to_token_ratio=0.064953 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131893890e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=343904 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266403680 ingress_bytes=369868800 link_bytes=369868800 token_bytes=532463456 weight_movement_bytes=103809024 total_nvlink_bytes=636272480 weight_to_token_ratio=0.194960 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391786518e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74406 payload_rows_remote=65264 dup_rows=23898 dup_rows_remote=20916 dedup_ratio=0.243103 router_tokens_per_expert=[782, 734, 804, 738, 778, 729, 778, 782, 780, 799, 759, 785, 789, 784, 764, 738, 755, 795, 712, 786, 816, 768, 797, 744, 783, 751, 766, 756, 801, 742, 802, 817, 809, 767, 776, 808, 748, 778, 767, 757, 755, 744, 746, 754, 757, 741, 821, 744, 754, 772, 766, 776, 746, 795, 750, 772, 807, 729, 725, 760, 746, 734, 778, 756] payload_rows_per_dst=[9254, 9420, 9342, 9322, 9524, 9142, 9260, 9142] dup_rows_per_dst=[2996, 2976, 3004, 3114, 2896, 2982, 3002, 2928] combine_rows_per_src=[9254, 9420, 9342, 9322, 9524, 9142, 9260, 9142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74118 payload_rows_remote=64900 dup_rows=24186 dup_rows_remote=21154 dedup_ratio=0.246033 router_tokens_per_expert=[812, 748, 772, 754, 776, 774, 776, 760, 769, 793, 794, 731, 755, 776, 765, 776, 756, 759, 779, 737, 773, 744, 763, 775, 775, 796, 782, 787, 763, 751, 722, 759, 793, 775, 808, 744, 750, 768, 737, 750, 758, 769, 773, 757, 781, 742, 768, 750, 805, 754, 767, 792, 787, 765, 794, 765, 759, 769, 772, 776, 745, 763, 777, 787] payload_rows_per_dst=[9240, 9314, 9252, 9226, 9218, 9222, 9320, 9326] dup_rows_per_dst=[3104, 3004, 2920, 3044, 3032, 2974, 3138, 2970] combine_rows_per_src=[9240, 9314, 9252, 9226, 9218, 9222, 9320, 9326] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266297344 dispatch_meta_bytes=343984 combine_payload_bytes=266297344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266641328 ingress_bytes=300900352 link_bytes=300900352 token_bytes=532938672 weight_movement_bytes=34603008 total_nvlink_bytes=567541680 weight_to_token_ratio=0.064929 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132263800e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=267198464 dispatch_meta_bytes=344680 combine_payload_bytes=267198464 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267543144 ingress_bytes=405610496 link_bytes=405610496 token_bytes=534741608 weight_movement_bytes=138412032 total_nvlink_bytes=673153640 weight_to_token_ratio=0.258839 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.526279642e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266493952 dispatch_meta_bytes=343368 combine_payload_bytes=266493952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266837320 ingress_bytes=301096960 link_bytes=301096960 token_bytes=533331272 weight_movement_bytes=34603008 total_nvlink_bytes=567934280 weight_to_token_ratio=0.064881 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133003620e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=267321344 dispatch_meta_bytes=344720 combine_payload_bytes=267321344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267666064 ingress_bytes=301924352 link_bytes=301924352 token_bytes=534987408 weight_movement_bytes=34603008 total_nvlink_bytes=569590416 weight_to_token_ratio=0.064680 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136117030e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=64906 dup_rows=24016 dup_rows_remote=21058 dedup_ratio=0.244303 router_tokens_per_expert=[741, 830, 766, 766, 765, 776, 758, 756, 832, 719, 745, 771, 776, 748, 773, 749, 746, 823, 728, 779, 768, 726, 821, 762, 739, 767, 767, 846, 787, 791, 714, 746, 778, 826, 755, 811, 736, 765, 715, 794, 773, 775, 733, 756, 798, 786, 798, 741, 805, 775, 753, 766, 769, 765, 784, 780, 753, 749, 742, 748, 734, 805, 733, 770] payload_rows_per_dst=[9382, 9400, 9290, 9214, 9110, 9332, 9374, 9186] dup_rows_per_dst=[2958, 3160, 3144, 2982, 2882, 2988, 3020, 2882] combine_rows_per_src=[9382, 9400, 9290, 9214, 9110, 9332, 9374, 9186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265830400 dispatch_meta_bytes=344216 combine_payload_bytes=265830400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266174616 ingress_bytes=300433408 link_bytes=300433408 token_bytes=532005016 weight_movement_bytes=34603008 total_nvlink_bytes=566608024 weight_to_token_ratio=0.065043 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130506727e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=64988 dup_rows=24002 dup_rows_remote=20834 dedup_ratio=0.244161 router_tokens_per_expert=[722, 731, 758, 803, 753, 744, 760, 819, 745, 786, 746, 751, 740, 749, 759, 780, 763, 772, 754, 776, 771, 763, 735, 784, 758, 818, 767, 766, 779, 819, 759, 797, 747, 790, 797, 787, 758, 718, 771, 786, 749, 771, 691, 734, 767, 760, 788, 813, 777, 777, 805, 772, 760, 792, 743, 815, 786, 801, 785, 751, 714, 772, 801, 747] payload_rows_per_dst=[9204, 9294, 9270, 9398, 9296, 9198, 9314, 9328] dup_rows_per_dst=[2976, 2818, 2966, 3128, 3012, 2948, 3168, 2986] combine_rows_per_src=[9204, 9294, 9270, 9398, 9296, 9198, 9314, 9328] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64726 dup_rows=24112 dup_rows_remote=20872 dedup_ratio=0.245280 router_tokens_per_expert=[753, 745, 724, 778, 769, 733, 766, 808, 746, 776, 776, 758, 747, 770, 771, 790, 778, 774, 764, 751, 717, 779, 749, 776, 798, 784, 755, 736, 799, 763, 810, 757, 732, 747, 817, 749, 760, 792, 825, 802, 746, 740, 742, 804, 835, 789, 804, 750, 741, 725, 787, 762, 779, 756, 774, 781, 724, 780, 784, 760, 806, 709, 767, 783] payload_rows_per_dst=[9184, 9244, 9146, 9130, 9118, 9582, 9322, 9466] dup_rows_per_dst=[2968, 3006, 3030, 2770, 2850, 3350, 2898, 3240] combine_rows_per_src=[9184, 9244, 9146, 9130, 9118, 9582, 9322, 9466] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74370 payload_rows_remote=65030 dup_rows=23934 dup_rows_remote=20838 dedup_ratio=0.243469 router_tokens_per_expert=[768, 723, 775, 760, 792, 765, 791, 785, 774, 817, 799, 786, 708, 776, 774, 784, 780, 754, 796, 759, 758, 725, 792, 800, 738, 838, 778, 799, 708, 761, 764, 777, 783, 750, 788, 792, 681, 787, 781, 764, 780, 730, 731, 737, 768, 755, 813, 768, 743, 764, 755, 771, 758, 728, 796, 757, 765, 764, 768, 784, 801, 783, 776, 727] payload_rows_per_dst=[9270, 9340, 9288, 9330, 9298, 9216, 9312, 9316] dup_rows_per_dst=[3048, 3096, 3040, 2996, 2954, 2948, 2832, 3020] combine_rows_per_src=[9270, 9340, 9288, 9330, 9298, 9216, 9312, 9316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265854976 dispatch_meta_bytes=343856 combine_payload_bytes=265854976 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266198832 ingress_bytes=335060992 link_bytes=335060992 token_bytes=532053808 weight_movement_bytes=69206016 total_nvlink_bytes=601259824 weight_to_token_ratio=0.130073 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260807538e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64726 dup_rows=24112 dup_rows_remote=20872 dedup_ratio=0.245280 router_tokens_per_expert=[753, 745, 724, 778, 769, 733, 766, 808, 746, 776, 776, 758, 747, 770, 771, 790, 778, 774, 764, 751, 717, 779, 749, 776, 798, 784, 755, 736, 799, 763, 810, 757, 732, 747, 817, 749, 760, 792, 825, 802, 746, 740, 742, 804, 835, 789, 804, 750, 741, 725, 787, 762, 779, 756, 774, 781, 724, 780, 784, 760, 806, 709, 767, 783] payload_rows_per_dst=[9184, 9244, 9146, 9130, 9118, 9582, 9322, 9466] dup_rows_per_dst=[2968, 3006, 3030, 2770, 2850, 3350, 2898, 3240] combine_rows_per_src=[9184, 9244, 9146, 9130, 9118, 9582, 9322, 9466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74406 payload_rows_remote=65264 dup_rows=23898 dup_rows_remote=20916 dedup_ratio=0.243103 router_tokens_per_expert=[782, 734, 804, 738, 778, 729, 778, 782, 780, 799, 759, 785, 789, 784, 764, 738, 755, 795, 712, 786, 816, 768, 797, 744, 783, 751, 766, 756, 801, 742, 802, 817, 809, 767, 776, 808, 748, 778, 767, 757, 755, 744, 746, 754, 757, 741, 821, 744, 754, 772, 766, 776, 746, 795, 750, 772, 807, 729, 725, 760, 746, 734, 778, 756] payload_rows_per_dst=[9254, 9420, 9342, 9322, 9524, 9142, 9260, 9142] dup_rows_per_dst=[2996, 2976, 3004, 3114, 2896, 2982, 3002, 2928] combine_rows_per_src=[9254, 9420, 9342, 9322, 9524, 9142, 9260, 9142] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=64996 dup_rows=24002 dup_rows_remote=21068 dedup_ratio=0.244161 router_tokens_per_expert=[822, 760, 774, 783, 739, 770, 740, 758, 777, 759, 761, 751, 802, 756, 771, 760, 744, 761, 778, 751, 715, 776, 749, 801, 780, 773, 720, 758, 745, 811, 764, 769, 754, 846, 809, 780, 810, 749, 760, 788, 722, 748, 800, 775, 765, 753, 762, 771, 693, 793, 775, 792, 782, 732, 761, 767, 781, 732, 733, 761, 794, 798, 794, 794] payload_rows_per_dst=[9274, 9278, 9236, 9306, 9448, 9230, 9180, 9350] dup_rows_per_dst=[3018, 2996, 2914, 2934, 3144, 2962, 3010, 3024] combine_rows_per_src=[9274, 9278, 9236, 9306, 9448, 9230, 9180, 9350] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74268 payload_rows_remote=65000 dup_rows=24036 dup_rows_remote=21062 dedup_ratio=0.244507 router_tokens_per_expert=[769, 791, 776, 795, 782, 766, 771, 767, 791, 734, 792, 799, 744, 753, 776, 756, 763, 735, 796, 779, 766, 753, 743, 786, 798, 794, 698, 800, 811, 751, 769, 759, 789, 736, 744, 778, 761, 716, 766, 750, 783, 794, 787, 753, 763, 787, 706, 780, 807, 773, 749, 766, 773, 789, 756, 791, 744, 787, 752, 780, 741, 771, 756, 761] payload_rows_per_dst=[9308, 9260, 9268, 9394, 9186, 9382, 9314, 9156] dup_rows_per_dst=[3126, 3030, 2974, 2966, 2894, 2924, 3094, 3028] combine_rows_per_src=[9308, 9260, 9268, 9394, 9186, 9382, 9314, 9156] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74402 payload_rows_remote=65146 dup_rows=23902 dup_rows_remote=20848 dedup_ratio=0.243144 router_tokens_per_expert=[735, 789, 775, 742, 779, 757, 804, 763, 769, 755, 813, 801, 768, 773, 780, 740, 745, 759, 742, 739, 819, 748, 786, 761, 734, 761, 752, 785, 780, 777, 754, 771, 742, 771, 761, 797, 805, 759, 761, 759, 748, 777, 713, 815, 739, 759, 781, 740, 769, 815, 804, 822, 772, 755, 785, 774, 831, 805, 726, 769, 742, 723, 769, 708] payload_rows_per_dst=[9360, 9364, 9226, 9242, 9256, 9194, 9500, 9260] dup_rows_per_dst=[2928, 3034, 2972, 2986, 3054, 2950, 3092, 2886] combine_rows_per_src=[9360, 9364, 9226, 9242, 9256, 9194, 9500, 9260] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266190848 dispatch_meta_bytes=343288 combine_payload_bytes=266190848 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266534136 ingress_bytes=369999872 link_bytes=369999872 token_bytes=532724984 weight_movement_bytes=103809024 total_nvlink_bytes=636534008 weight_to_token_ratio=0.194864 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392279731e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265117696 dispatch_meta_bytes=342392 combine_payload_bytes=265117696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265460088 ingress_bytes=299720704 link_bytes=299720704 token_bytes=530577784 weight_movement_bytes=34603008 total_nvlink_bytes=565180792 weight_to_token_ratio=0.065218 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127824880e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=64988 dup_rows=24002 dup_rows_remote=20834 dedup_ratio=0.244161 router_tokens_per_expert=[722, 731, 758, 803, 753, 744, 760, 819, 745, 786, 746, 751, 740, 749, 759, 780, 763, 772, 754, 776, 771, 763, 735, 784, 758, 818, 767, 766, 779, 819, 759, 797, 747, 790, 797, 787, 758, 718, 771, 786, 749, 771, 691, 734, 767, 760, 788, 813, 777, 777, 805, 772, 760, 792, 743, 815, 786, 801, 785, 751, 714, 772, 801, 747] payload_rows_per_dst=[9204, 9294, 9270, 9398, 9296, 9198, 9314, 9328] dup_rows_per_dst=[2976, 2818, 2966, 3128, 3012, 2948, 3168, 2986] combine_rows_per_src=[9204, 9294, 9270, 9398, 9296, 9198, 9314, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74268 payload_rows_remote=65000 dup_rows=24036 dup_rows_remote=21062 dedup_ratio=0.244507 router_tokens_per_expert=[769, 791, 776, 795, 782, 766, 771, 767, 791, 734, 792, 799, 744, 753, 776, 756, 763, 735, 796, 779, 766, 753, 743, 786, 798, 794, 698, 800, 811, 751, 769, 759, 789, 736, 744, 778, 761, 716, 766, 750, 783, 794, 787, 753, 763, 787, 706, 780, 807, 773, 749, 766, 773, 789, 756, 791, 744, 787, 752, 780, 741, 771, 756, 761] payload_rows_per_dst=[9308, 9260, 9268, 9394, 9186, 9382, 9314, 9156] dup_rows_per_dst=[3126, 3030, 2974, 2966, 2894, 2924, 3094, 3028] combine_rows_per_src=[9308, 9260, 9268, 9394, 9186, 9382, 9314, 9156] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=343472 combine_payload_bytes=266362880 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266706352 ingress_bytes=335568896 link_bytes=335568896 token_bytes=533069232 weight_movement_bytes=69206016 total_nvlink_bytes=602275248 weight_to_token_ratio=0.129826 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262718740e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265117696 dispatch_meta_bytes=342392 combine_payload_bytes=265117696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265460088 ingress_bytes=299720704 link_bytes=299720704 token_bytes=530577784 weight_movement_bytes=34603008 total_nvlink_bytes=565180792 weight_to_token_ratio=0.065218 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127824880e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=267321344 dispatch_meta_bytes=344720 combine_payload_bytes=267321344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267666064 ingress_bytes=301924352 link_bytes=301924352 token_bytes=534987408 weight_movement_bytes=34603008 total_nvlink_bytes=569590416 weight_to_token_ratio=0.064680 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136117030e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=344256 combine_payload_bytes=266223616 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266567872 ingress_bytes=370032640 link_bytes=370032640 token_bytes=532791488 weight_movement_bytes=103809024 total_nvlink_bytes=636600512 weight_to_token_ratio=0.194840 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392403034e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=65054 dup_rows=23940 dup_rows_remote=20800 dedup_ratio=0.243530 router_tokens_per_expert=[736, 779, 727, 785, 737, 722, 746, 805, 772, 771, 778, 800, 768, 742, 816, 770, 777, 735, 743, 755, 777, 719, 745, 741, 763, 713, 775, 748, 777, 743, 778, 758, 765, 740, 804, 786, 763, 772, 789, 757, 761, 783, 790, 750, 764, 811, 755, 811, 784, 785, 824, 765, 783, 767, 765, 757, 775, 793, 767, 794, 772, 779, 781, 759] payload_rows_per_dst=[9208, 9518, 9110, 9232, 9282, 9310, 9360, 9344] dup_rows_per_dst=[2866, 2916, 2874, 2878, 3070, 3140, 3100, 3096] combine_rows_per_src=[9208, 9518, 9110, 9232, 9282, 9310, 9360, 9344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74460 payload_rows_remote=65234 dup_rows=23844 dup_rows_remote=20936 dedup_ratio=0.242554 router_tokens_per_expert=[778, 784, 744, 761, 783, 788, 770, 757, 785, 808, 758, 816, 785, 747, 791, 749, 746, 772, 766, 753, 748, 772, 694, 803, 790, 780, 760, 752, 747, 736, 761, 769, 724, 771, 730, 787, 749, 742, 776, 788, 798, 788, 767, 761, 718, 725, 776, 832, 764, 734, 811, 783, 806, 815, 779, 762, 715, 777, 813, 789, 743, 730, 772, 774] payload_rows_per_dst=[9284, 9326, 9258, 9256, 9226, 9322, 9476, 9312] dup_rows_per_dst=[3046, 3152, 2850, 2934, 2908, 3008, 3032, 2914] combine_rows_per_src=[9284, 9326, 9258, 9256, 9226, 9322, 9476, 9312] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266240000 dispatch_meta_bytes=344248 combine_payload_bytes=266240000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266584248 ingress_bytes=300843008 link_bytes=300843008 token_bytes=532824248 weight_movement_bytes=34603008 total_nvlink_bytes=567427256 weight_to_token_ratio=0.064943 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132048019e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266838016 dispatch_meta_bytes=343976 combine_payload_bytes=266838016 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267181992 ingress_bytes=336044032 link_bytes=336044032 token_bytes=534020008 weight_movement_bytes=69206016 total_nvlink_bytes=603226024 weight_to_token_ratio=0.129594 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264506639e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74280 payload_rows_remote=64914 dup_rows=24024 dup_rows_remote=21000 dedup_ratio=0.244385 router_tokens_per_expert=[727, 786, 774, 781, 726, 798, 786, 750, 737, 808, 737, 775, 742, 758, 771, 746, 752, 796, 783, 746, 774, 747, 797, 782, 791, 736, 786, 777, 766, 795, 803, 757, 775, 732, 744, 751, 778, 802, 782, 784, 752, 761, 760, 762, 774, 773, 761, 805, 747, 805, 755, 764, 800, 814, 763, 731, 812, 777, 786, 745, 731, 759, 742, 735] payload_rows_per_dst=[9366, 9184, 9456, 9302, 9284, 9328, 9130, 9230] dup_rows_per_dst=[3024, 2964, 3172, 2986, 3090, 2968, 2876, 2944] combine_rows_per_src=[9366, 9184, 9456, 9302, 9284, 9328, 9130, 9230] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266190848 dispatch_meta_bytes=343288 combine_payload_bytes=266190848 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266534136 ingress_bytes=369999872 link_bytes=369999872 token_bytes=532724984 weight_movement_bytes=103809024 total_nvlink_bytes=636534008 weight_to_token_ratio=0.194864 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392279731e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=64980 dup_rows=23860 dup_rows_remote=20862 dedup_ratio=0.242716 router_tokens_per_expert=[769, 746, 731, 774, 822, 790, 751, 766, 750, 733, 772, 750, 749, 789, 736, 817, 823, 739, 754, 787, 830, 762, 792, 768, 755, 765, 760, 738, 790, 747, 783, 754, 761, 767, 752, 735, 768, 734, 755, 775, 748, 804, 794, 748, 820, 744, 758, 752, 788, 791, 791, 784, 804, 802, 744, 727, 759, 768, 744, 732, 795, 797, 750, 769] payload_rows_per_dst=[9314, 9212, 9446, 9270, 9232, 9278, 9464, 9228] dup_rows_per_dst=[2984, 2980, 3064, 2914, 2862, 3058, 2998, 3000] combine_rows_per_src=[9314, 9212, 9446, 9270, 9232, 9278, 9464, 9228] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=343416 combine_payload_bytes=266461184 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266804600 ingress_bytes=370270208 link_bytes=370270208 token_bytes=533265784 weight_movement_bytes=103809024 total_nvlink_bytes=637074808 weight_to_token_ratio=0.194667 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393296984e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74408 payload_rows_remote=65018 dup_rows=23896 dup_rows_remote=20790 dedup_ratio=0.243083 router_tokens_per_expert=[797, 765, 744, 764, 744, 757, 718, 789, 781, 735, 769, 793, 730, 778, 774, 826, 781, 759, 796, 792, 771, 797, 747, 735, 808, 761, 774, 765, 757, 735, 828, 805, 733, 767, 746, 807, 739, 779, 730, 777, 757, 806, 786, 740, 805, 769, 750, 721, 748, 759, 812, 751, 777, 745, 763, 758, 751, 825, 744, 804, 771, 740, 755, 762] payload_rows_per_dst=[9310, 9180, 9322, 9442, 9252, 9222, 9290, 9390] dup_rows_per_dst=[2846, 3100, 3034, 3024, 2904, 2854, 3028, 3106] combine_rows_per_src=[9310, 9180, 9322, 9442, 9252, 9222, 9290, 9390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=267198464 dispatch_meta_bytes=344680 combine_payload_bytes=267198464 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267543144 ingress_bytes=405610496 link_bytes=405610496 token_bytes=534741608 weight_movement_bytes=138412032 total_nvlink_bytes=673153640 weight_to_token_ratio=0.258839 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.526279642e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265887744 dispatch_meta_bytes=343656 combine_payload_bytes=265887744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266231400 ingress_bytes=300490752 link_bytes=300490752 token_bytes=532119144 weight_movement_bytes=34603008 total_nvlink_bytes=566722152 weight_to_token_ratio=0.065029 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130722508e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=64968 dup_rows=24020 dup_rows_remote=20906 dedup_ratio=0.244344 router_tokens_per_expert=[785, 807, 795, 749, 780, 803, 749, 762, 793, 727, 801, 817, 763, 766, 750, 785, 763, 746, 782, 743, 776, 792, 762, 797, 752, 828, 806, 757, 737, 752, 796, 787, 758, 749, 752, 745, 733, 766, 767, 837, 745, 859, 773, 719, 777, 746, 807, 768, 750, 755, 795, 731, 743, 745, 795, 757, 685, 775, 738, 766, 734, 791, 750, 733] payload_rows_per_dst=[9322, 9262, 9322, 9316, 9330, 9324, 9276, 9132] dup_rows_per_dst=[3138, 3142, 3000, 3114, 2884, 3064, 2866, 2812] combine_rows_per_src=[9322, 9262, 9322, 9316, 9330, 9324, 9276, 9132] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74166 payload_rows_remote=64862 dup_rows=24138 dup_rows_remote=21092 dedup_ratio=0.245544 router_tokens_per_expert=[727, 744, 736, 795, 746, 741, 725, 746, 766, 742, 789, 774, 757, 797, 755, 795, 850, 759, 777, 775, 775, 765, 812, 784, 745, 760, 780, 750, 805, 747, 788, 728, 793, 773, 763, 784, 826, 810, 786, 718, 767, 762, 723, 746, 770, 759, 741, 785, 797, 748, 753, 777, 748, 794, 785, 730, 758, 765, 778, 784, 803, 736, 806, 749] payload_rows_per_dst=[9088, 9304, 9434, 9258, 9298, 9220, 9236, 9328] dup_rows_per_dst=[2832, 3046, 3160, 2948, 3208, 2886, 3028, 3030] combine_rows_per_src=[9088, 9304, 9434, 9258, 9298, 9220, 9236, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=65054 dup_rows=23940 dup_rows_remote=20800 dedup_ratio=0.243530 router_tokens_per_expert=[736, 779, 727, 785, 737, 722, 746, 805, 772, 771, 778, 800, 768, 742, 816, 770, 777, 735, 743, 755, 777, 719, 745, 741, 763, 713, 775, 748, 777, 743, 778, 758, 765, 740, 804, 786, 763, 772, 789, 757, 761, 783, 790, 750, 764, 811, 755, 811, 784, 785, 824, 765, 783, 767, 765, 757, 775, 793, 767, 794, 772, 779, 781, 759] payload_rows_per_dst=[9208, 9518, 9110, 9232, 9282, 9310, 9360, 9344] dup_rows_per_dst=[2866, 2916, 2874, 2878, 3070, 3140, 3100, 3096] combine_rows_per_src=[9208, 9518, 9110, 9232, 9282, 9310, 9360, 9344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74408 payload_rows_remote=65018 dup_rows=23896 dup_rows_remote=20790 dedup_ratio=0.243083 router_tokens_per_expert=[797, 765, 744, 764, 744, 757, 718, 789, 781, 735, 769, 793, 730, 778, 774, 826, 781, 759, 796, 792, 771, 797, 747, 735, 808, 761, 774, 765, 757, 735, 828, 805, 733, 767, 746, 807, 739, 779, 730, 777, 757, 806, 786, 740, 805, 769, 750, 721, 748, 759, 812, 751, 777, 745, 763, 758, 751, 825, 744, 804, 771, 740, 755, 762] payload_rows_per_dst=[9310, 9180, 9322, 9442, 9252, 9222, 9290, 9390] dup_rows_per_dst=[2846, 3100, 3034, 3024, 2904, 2854, 3028, 3106] combine_rows_per_src=[9310, 9180, 9322, 9442, 9252, 9222, 9290, 9390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=64968 dup_rows=24020 dup_rows_remote=20906 dedup_ratio=0.244344 router_tokens_per_expert=[785, 807, 795, 749, 780, 803, 749, 762, 793, 727, 801, 817, 763, 766, 750, 785, 763, 746, 782, 743, 776, 792, 762, 797, 752, 828, 806, 757, 737, 752, 796, 787, 758, 749, 752, 745, 733, 766, 767, 837, 745, 859, 773, 719, 777, 746, 807, 768, 750, 755, 795, 731, 743, 745, 795, 757, 685, 775, 738, 766, 734, 791, 750, 733] payload_rows_per_dst=[9322, 9262, 9322, 9316, 9330, 9324, 9276, 9132] dup_rows_per_dst=[3138, 3142, 3000, 3114, 2884, 3064, 2866, 2812] combine_rows_per_src=[9322, 9262, 9322, 9316, 9330, 9324, 9276, 9132] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266158080 dispatch_meta_bytes=343368 combine_payload_bytes=266158080 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266501448 ingress_bytes=335364096 link_bytes=335364096 token_bytes=532659528 weight_movement_bytes=69206016 total_nvlink_bytes=601865544 weight_to_token_ratio=0.129925 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261948094e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74470 payload_rows_remote=65240 dup_rows=23834 dup_rows_remote=20942 dedup_ratio=0.242452 router_tokens_per_expert=[700, 761, 826, 763, 772, 787, 784, 757, 741, 795, 775, 718, 791, 706, 786, 754, 776, 759, 745, 743, 766, 772, 726, 774, 772, 784, 754, 796, 788, 730, 747, 765, 773, 762, 785, 729, 743, 819, 787, 816, 757, 757, 765, 807, 848, 755, 714, 768, 791, 738, 767, 755, 769, 788, 761, 773, 768, 744, 748, 802, 769, 774, 771, 836] payload_rows_per_dst=[9280, 9232, 9230, 9292, 9424, 9400, 9296, 9316] dup_rows_per_dst=[3020, 2900, 2892, 2980, 3004, 2942, 2988, 3108] combine_rows_per_src=[9280, 9232, 9230, 9292, 9424, 9400, 9296, 9316] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74524 payload_rows_remote=65292 dup_rows=23780 dup_rows_remote=20894 dedup_ratio=0.241903 router_tokens_per_expert=[779, 812, 771, 733, 794, 788, 795, 757, 781, 748, 753, 758, 725, 742, 765, 757, 761, 741, 788, 813, 765, 744, 798, 759, 745, 806, 753, 818, 768, 735, 772, 743, 771, 761, 741, 801, 739, 749, 766, 731, 816, 791, 718, 781, 746, 755, 810, 763, 735, 751, 833, 788, 787, 784, 759, 778, 740, 758, 699, 785, 782, 779, 808, 780] payload_rows_per_dst=[9370, 9200, 9342, 9272, 9232, 9440, 9400, 9268] dup_rows_per_dst=[3088, 2858, 2996, 3008, 2886, 2920, 3030, 2994] combine_rows_per_src=[9370, 9200, 9342, 9272, 9232, 9440, 9400, 9268] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=343232 combine_payload_bytes=266313728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266656960 ingress_bytes=300916736 link_bytes=300916736 token_bytes=532970688 weight_movement_bytes=34603008 total_nvlink_bytes=567573696 weight_to_token_ratio=0.064925 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132325452e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265674752 dispatch_meta_bytes=343816 combine_payload_bytes=265674752 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266018568 ingress_bytes=334880768 link_bytes=334880768 token_bytes=531693320 weight_movement_bytes=69206016 total_nvlink_bytes=600899336 weight_to_token_ratio=0.130162 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260129370e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=343232 combine_payload_bytes=266313728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266656960 ingress_bytes=300916736 link_bytes=300916736 token_bytes=532970688 weight_movement_bytes=34603008 total_nvlink_bytes=567573696 weight_to_token_ratio=0.064925 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132325452e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=343416 combine_payload_bytes=266461184 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266804600 ingress_bytes=370270208 link_bytes=370270208 token_bytes=533265784 weight_movement_bytes=103809024 total_nvlink_bytes=637074808 weight_to_token_ratio=0.194667 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393296984e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=64980 dup_rows=23860 dup_rows_remote=20862 dedup_ratio=0.242716 router_tokens_per_expert=[769, 746, 731, 774, 822, 790, 751, 766, 750, 733, 772, 750, 749, 789, 736, 817, 823, 739, 754, 787, 830, 762, 792, 768, 755, 765, 760, 738, 790, 747, 783, 754, 761, 767, 752, 735, 768, 734, 755, 775, 748, 804, 794, 748, 820, 744, 758, 752, 788, 791, 791, 784, 804, 802, 744, 727, 759, 768, 744, 732, 795, 797, 750, 769] payload_rows_per_dst=[9314, 9212, 9446, 9270, 9232, 9278, 9464, 9228] dup_rows_per_dst=[2984, 2980, 3064, 2914, 2862, 3058, 2998, 3000] combine_rows_per_src=[9314, 9212, 9446, 9270, 9232, 9278, 9464, 9228] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=343496 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266452424 ingress_bytes=300711936 link_bytes=300711936 token_bytes=532561352 weight_movement_bytes=34603008 total_nvlink_bytes=567164360 weight_to_token_ratio=0.064975 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131554806e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265674752 dispatch_meta_bytes=343816 combine_payload_bytes=265674752 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266018568 ingress_bytes=334880768 link_bytes=334880768 token_bytes=531693320 weight_movement_bytes=69206016 total_nvlink_bytes=600899336 weight_to_token_ratio=0.130162 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260129370e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74402 payload_rows_remote=65146 dup_rows=23902 dup_rows_remote=20848 dedup_ratio=0.243144 router_tokens_per_expert=[735, 789, 775, 742, 779, 757, 804, 763, 769, 755, 813, 801, 768, 773, 780, 740, 745, 759, 742, 739, 819, 748, 786, 761, 734, 761, 752, 785, 780, 777, 754, 771, 742, 771, 761, 797, 805, 759, 761, 759, 748, 777, 713, 815, 739, 759, 781, 740, 769, 815, 804, 822, 772, 755, 785, 774, 831, 805, 726, 769, 742, 723, 769, 708] payload_rows_per_dst=[9360, 9364, 9226, 9242, 9256, 9194, 9500, 9260] dup_rows_per_dst=[2928, 3034, 2972, 2986, 3054, 2950, 3092, 2886] combine_rows_per_src=[9360, 9364, 9226, 9242, 9256, 9194, 9500, 9260] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267223040 dispatch_meta_bytes=344728 combine_payload_bytes=267223040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267567768 ingress_bytes=301826048 link_bytes=301826048 token_bytes=534790808 weight_movement_bytes=34603008 total_nvlink_bytes=569393816 weight_to_token_ratio=0.064704 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135747120e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74202 payload_rows_remote=65018 dup_rows=24102 dup_rows_remote=21188 dedup_ratio=0.245178 router_tokens_per_expert=[798, 739, 790, 761, 749, 809, 801, 761, 752, 757, 760, 761, 773, 747, 797, 738, 760, 783, 737, 800, 763, 731, 800, 772, 768, 785, 767, 757, 725, 730, 732, 750, 775, 787, 802, 787, 795, 759, 786, 769, 736, 771, 735, 762, 783, 783, 731, 748, 748, 774, 799, 751, 759, 772, 759, 795, 762, 710, 791, 797, 819, 775, 787, 792] payload_rows_per_dst=[9382, 9168, 9252, 9108, 9422, 9184, 9246, 9440] dup_rows_per_dst=[3034, 3002, 3040, 2920, 3098, 2914, 3068, 3026] combine_rows_per_src=[9382, 9168, 9252, 9108, 9422, 9184, 9246, 9440] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267436032 dispatch_meta_bytes=344744 combine_payload_bytes=267436032 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267780776 ingress_bytes=371245056 link_bytes=371245056 token_bytes=535216808 weight_movement_bytes=103809024 total_nvlink_bytes=639025832 weight_to_token_ratio=0.193957 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396965258e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74270 payload_rows_remote=64846 dup_rows=24034 dup_rows_remote=20820 dedup_ratio=0.244486 router_tokens_per_expert=[775, 750, 749, 741, 761, 779, 783, 806, 801, 775, 749, 775, 828, 790, 738, 768, 767, 745, 780, 708, 676, 791, 727, 759, 774, 729, 764, 784, 760, 790, 766, 795, 794, 778, 750, 791, 810, 781, 782, 832, 772, 801, 791, 713, 755, 733, 782, 781, 812, 804, 713, 736, 807, 800, 740, 735, 724, 760, 767, 799, 781, 735, 752, 758] payload_rows_per_dst=[9424, 9452, 9128, 9312, 9268, 9362, 9130, 9194] dup_rows_per_dst=[3214, 3002, 2950, 3044, 3168, 2884, 2814, 2958] combine_rows_per_src=[9424, 9452, 9128, 9312, 9268, 9362, 9130, 9194] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266158080 dispatch_meta_bytes=343368 combine_payload_bytes=266158080 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266501448 ingress_bytes=335364096 link_bytes=335364096 token_bytes=532659528 weight_movement_bytes=69206016 total_nvlink_bytes=601865544 weight_to_token_ratio=0.129925 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261948094e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=344824 combine_payload_bytes=266313728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266658552 ingress_bytes=300916736 link_bytes=300916736 token_bytes=532972280 weight_movement_bytes=34603008 total_nvlink_bytes=567575288 weight_to_token_ratio=0.064925 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132325452e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=64608 dup_rows=24018 dup_rows_remote=20878 dedup_ratio=0.244324 router_tokens_per_expert=[785, 780, 745, 749, 812, 789, 797, 768, 768, 752, 723, 777, 783, 746, 773, 738, 735, 716, 760, 765, 780, 765, 758, 725, 780, 806, 756, 766, 779, 784, 770, 786, 753, 793, 758, 780, 735, 772, 757, 757, 769, 774, 749, 763, 742, 771, 784, 760, 793, 760, 735, 761, 735, 767, 735, 800, 740, 833, 785, 812, 835, 780, 756, 792] payload_rows_per_dst=[9362, 9078, 9134, 9314, 8920, 9472, 9328, 9678] dup_rows_per_dst=[2968, 3042, 2874, 3006, 2784, 3298, 2906, 3140] combine_rows_per_src=[9362, 9078, 9134, 9314, 8920, 9472, 9328, 9678] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74240 payload_rows_remote=64912 dup_rows=24064 dup_rows_remote=21156 dedup_ratio=0.244792 router_tokens_per_expert=[771, 732, 730, 806, 783, 776, 773, 788, 761, 739, 811, 748, 775, 789, 774, 796, 743, 752, 766, 762, 751, 763, 780, 791, 709, 749, 766, 811, 766, 779, 806, 749, 761, 773, 807, 771, 755, 789, 803, 758, 753, 786, 734, 786, 780, 729, 721, 812, 748, 777, 796, 775, 770, 720, 771, 761, 760, 770, 737, 783, 775, 740, 788, 768] payload_rows_per_dst=[9222, 9290, 9222, 9214, 9404, 9334, 9328, 9226] dup_rows_per_dst=[3096, 3096, 2994, 3056, 3030, 2868, 2908, 3016] combine_rows_per_src=[9222, 9290, 9222, 9214, 9404, 9334, 9328, 9226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266838016 dispatch_meta_bytes=343976 combine_payload_bytes=266838016 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267181992 ingress_bytes=336044032 link_bytes=336044032 token_bytes=534020008 weight_movement_bytes=69206016 total_nvlink_bytes=603226024 weight_to_token_ratio=0.129594 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264506639e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265609216 dispatch_meta_bytes=342664 combine_payload_bytes=265609216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265951880 ingress_bytes=300212224 link_bytes=300212224 token_bytes=531561096 weight_movement_bytes=34603008 total_nvlink_bytes=566164104 weight_to_token_ratio=0.065097 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129674430e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74524 payload_rows_remote=65052 dup_rows=23780 dup_rows_remote=20682 dedup_ratio=0.241903 router_tokens_per_expert=[762, 776, 751, 809, 769, 773, 805, 792, 765, 776, 727, 769, 752, 708, 737, 757, 754, 822, 762, 768, 789, 815, 778, 739, 801, 748, 789, 796, 815, 796, 774, 766, 785, 785, 778, 724, 808, 781, 760, 771, 729, 739, 724, 795, 720, 757, 759, 780, 748, 753, 756, 784, 804, 732, 796, 778, 723, 712, 796, 755, 775, 772, 775, 758] payload_rows_per_dst=[9368, 9166, 9434, 9472, 9378, 9182, 9324, 9200] dup_rows_per_dst=[3106, 2816, 3020, 3098, 3006, 2824, 2978, 2932] combine_rows_per_src=[9368, 9166, 9434, 9472, 9378, 9182, 9324, 9200] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65154 dup_rows=23944 dup_rows_remote=20938 dedup_ratio=0.243571 router_tokens_per_expert=[765, 794, 761, 769, 722, 750, 768, 827, 752, 798, 752, 804, 774, 724, 752, 750, 780, 755, 768, 743, 741, 773, 788, 822, 778, 726, 772, 785, 756, 729, 732, 753, 759, 774, 797, 733, 795, 759, 789, 769, 836, 737, 793, 750, 773, 773, 773, 783, 778, 781, 750, 765, 789, 770, 732, 768, 747, 782, 802, 772, 795, 769, 776, 720] payload_rows_per_dst=[9322, 9206, 9396, 9146, 9346, 9374, 9174, 9396] dup_rows_per_dst=[2990, 3006, 2944, 2916, 3004, 3062, 3092, 2930] combine_rows_per_src=[9322, 9206, 9396, 9146, 9346, 9374, 9174, 9396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=64608 dup_rows=24018 dup_rows_remote=20878 dedup_ratio=0.244324 router_tokens_per_expert=[785, 780, 745, 749, 812, 789, 797, 768, 768, 752, 723, 777, 783, 746, 773, 738, 735, 716, 760, 765, 780, 765, 758, 725, 780, 806, 756, 766, 779, 784, 770, 786, 753, 793, 758, 780, 735, 772, 757, 757, 769, 774, 749, 763, 742, 771, 784, 760, 793, 760, 735, 761, 735, 767, 735, 800, 740, 833, 785, 812, 835, 780, 756, 792] payload_rows_per_dst=[9362, 9078, 9134, 9314, 8920, 9472, 9328, 9678] dup_rows_per_dst=[2968, 3042, 2874, 3006, 2784, 3298, 2906, 3140] combine_rows_per_src=[9362, 9078, 9134, 9314, 8920, 9472, 9328, 9678] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74202 payload_rows_remote=65018 dup_rows=24102 dup_rows_remote=21188 dedup_ratio=0.245178 router_tokens_per_expert=[798, 739, 790, 761, 749, 809, 801, 761, 752, 757, 760, 761, 773, 747, 797, 738, 760, 783, 737, 800, 763, 731, 800, 772, 768, 785, 767, 757, 725, 730, 732, 750, 775, 787, 802, 787, 795, 759, 786, 769, 736, 771, 735, 762, 783, 783, 731, 748, 748, 774, 799, 751, 759, 772, 759, 795, 762, 710, 791, 797, 819, 775, 787, 792] payload_rows_per_dst=[9382, 9168, 9252, 9108, 9422, 9184, 9246, 9440] dup_rows_per_dst=[3034, 3002, 3040, 2920, 3098, 2914, 3068, 3026] combine_rows_per_src=[9382, 9168, 9252, 9108, 9422, 9184, 9246, 9440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265879552 dispatch_meta_bytes=344272 combine_payload_bytes=265879552 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266223824 ingress_bytes=335085568 link_bytes=335085568 token_bytes=532103376 weight_movement_bytes=69206016 total_nvlink_bytes=601309392 weight_to_token_ratio=0.130061 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260900016e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74524 payload_rows_remote=65052 dup_rows=23780 dup_rows_remote=20682 dedup_ratio=0.241903 router_tokens_per_expert=[762, 776, 751, 809, 769, 773, 805, 792, 765, 776, 727, 769, 752, 708, 737, 757, 754, 822, 762, 768, 789, 815, 778, 739, 801, 748, 789, 796, 815, 796, 774, 766, 785, 785, 778, 724, 808, 781, 760, 771, 729, 739, 724, 795, 720, 757, 759, 780, 748, 753, 756, 784, 804, 732, 796, 778, 723, 712, 796, 755, 775, 772, 775, 758] payload_rows_per_dst=[9368, 9166, 9434, 9472, 9378, 9182, 9324, 9200] dup_rows_per_dst=[3106, 2816, 3020, 3098, 3006, 2824, 2978, 2932] combine_rows_per_src=[9368, 9166, 9434, 9472, 9378, 9182, 9324, 9200] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=65016 dup_rows=24002 dup_rows_remote=20954 dedup_ratio=0.244161 router_tokens_per_expert=[731, 780, 773, 841, 784, 725, 750, 732, 780, 761, 759, 787, 783, 722, 758, 775, 801, 817, 748, 749, 759, 766, 743, 784, 795, 779, 742, 831, 782, 743, 730, 764, 761, 761, 744, 737, 741, 775, 770, 735, 792, 797, 735, 815, 763, 797, 755, 826, 764, 814, 743, 751, 752, 780, 759, 799, 772, 743, 766, 757, 769, 766, 773, 766] payload_rows_per_dst=[9272, 9196, 9286, 9310, 9204, 9446, 9292, 9296] dup_rows_per_dst=[2960, 3054, 3048, 3022, 2844, 3114, 3032, 2928] combine_rows_per_src=[9272, 9196, 9286, 9310, 9204, 9446, 9292, 9296] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74060 payload_rows_remote=64714 dup_rows=24244 dup_rows_remote=21200 dedup_ratio=0.246623 router_tokens_per_expert=[766, 744, 699, 757, 785, 781, 755, 780, 811, 749, 763, 789, 768, 721, 768, 710, 793, 764, 769, 784, 780, 792, 699, 772, 803, 789, 751, 784, 786, 837, 744, 765, 799, 757, 772, 802, 772, 755, 778, 760, 745, 765, 737, 806, 772, 737, 781, 734, 786, 774, 779, 785, 787, 741, 737, 783, 794, 747, 769, 784, 779, 781, 765, 731] payload_rows_per_dst=[9168, 9270, 9208, 9352, 9346, 9160, 9284, 9272] dup_rows_per_dst=[2966, 2888, 3098, 3166, 3044, 2994, 3060, 3028] combine_rows_per_src=[9168, 9270, 9208, 9352, 9346, 9160, 9284, 9272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266870784 dispatch_meta_bytes=344368 combine_payload_bytes=266870784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267215152 ingress_bytes=301473792 link_bytes=301473792 token_bytes=534085936 weight_movement_bytes=34603008 total_nvlink_bytes=568688944 weight_to_token_ratio=0.064789 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134421609e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=264634368 dispatch_meta_bytes=341944 combine_payload_bytes=264634368 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=264976312 ingress_bytes=333840384 link_bytes=333840384 token_bytes=529610680 weight_movement_bytes=69206016 total_nvlink_bytes=598816696 weight_to_token_ratio=0.130673 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256214489e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=344824 combine_payload_bytes=266313728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266658552 ingress_bytes=300916736 link_bytes=300916736 token_bytes=532972280 weight_movement_bytes=34603008 total_nvlink_bytes=567575288 weight_to_token_ratio=0.064925 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132325452e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266452992 dispatch_meta_bytes=342936 combine_payload_bytes=266452992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266795928 ingress_bytes=335659008 link_bytes=335659008 token_bytes=533248920 weight_movement_bytes=69206016 total_nvlink_bytes=602454936 weight_to_token_ratio=0.129782 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263057824e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74240 payload_rows_remote=64912 dup_rows=24064 dup_rows_remote=21156 dedup_ratio=0.244792 router_tokens_per_expert=[771, 732, 730, 806, 783, 776, 773, 788, 761, 739, 811, 748, 775, 789, 774, 796, 743, 752, 766, 762, 751, 763, 780, 791, 709, 749, 766, 811, 766, 779, 806, 749, 761, 773, 807, 771, 755, 789, 803, 758, 753, 786, 734, 786, 780, 729, 721, 812, 748, 777, 796, 775, 770, 720, 771, 761, 760, 770, 737, 783, 775, 740, 788, 768] payload_rows_per_dst=[9222, 9290, 9222, 9214, 9404, 9334, 9328, 9226] dup_rows_per_dst=[3096, 3096, 2994, 3056, 3030, 2868, 2908, 3016] combine_rows_per_src=[9222, 9290, 9222, 9214, 9404, 9334, 9328, 9226] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266870784 dispatch_meta_bytes=344368 combine_payload_bytes=266870784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267215152 ingress_bytes=301473792 link_bytes=301473792 token_bytes=534085936 weight_movement_bytes=34603008 total_nvlink_bytes=568688944 weight_to_token_ratio=0.064789 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134421609e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=343880 combine_payload_bytes=266305536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266649416 ingress_bytes=300908544 link_bytes=300908544 token_bytes=532954952 weight_movement_bytes=34603008 total_nvlink_bytes=567557960 weight_to_token_ratio=0.064927 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132294626e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74366 payload_rows_remote=65126 dup_rows=23938 dup_rows_remote=20942 dedup_ratio=0.243510 router_tokens_per_expert=[744, 822, 799, 751, 767, 707, 749, 743, 758, 764, 751, 776, 777, 777, 770, 758, 764, 726, 784, 794, 816, 751, 764, 743, 720, 761, 753, 799, 761, 726, 773, 733, 729, 790, 799, 784, 755, 728, 807, 765, 778, 786, 763, 722, 765, 756, 771, 777, 791, 817, 807, 802, 770, 795, 796, 738, 758, 770, 782, 791, 783, 756, 801, 739] payload_rows_per_dst=[9276, 9342, 9256, 9146, 9308, 9240, 9436, 9362] dup_rows_per_dst=[2888, 2920, 3028, 2906, 3006, 2996, 3196, 2998] combine_rows_per_src=[9276, 9342, 9256, 9146, 9308, 9240, 9436, 9362] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265068544 dispatch_meta_bytes=343656 combine_payload_bytes=265068544 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265412200 ingress_bytes=334274560 link_bytes=334274560 token_bytes=530480744 weight_movement_bytes=69206016 total_nvlink_bytes=599686760 weight_to_token_ratio=0.130459 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257848258e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74524 payload_rows_remote=65292 dup_rows=23780 dup_rows_remote=20894 dedup_ratio=0.241903 router_tokens_per_expert=[779, 812, 771, 733, 794, 788, 795, 757, 781, 748, 753, 758, 725, 742, 765, 757, 761, 741, 788, 813, 765, 744, 798, 759, 745, 806, 753, 818, 768, 735, 772, 743, 771, 761, 741, 801, 739, 749, 766, 731, 816, 791, 718, 781, 746, 755, 810, 763, 735, 751, 833, 788, 787, 784, 759, 778, 740, 758, 699, 785, 782, 779, 808, 780] payload_rows_per_dst=[9370, 9200, 9342, 9272, 9232, 9440, 9400, 9268] dup_rows_per_dst=[3088, 2858, 2996, 3008, 2886, 2920, 3030, 2994] combine_rows_per_src=[9370, 9200, 9342, 9272, 9232, 9440, 9400, 9268] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74220 payload_rows_remote=65078 dup_rows=24084 dup_rows_remote=21138 dedup_ratio=0.244995 router_tokens_per_expert=[722, 733, 783, 781, 751, 756, 754, 764, 774, 787, 817, 821, 771, 756, 728, 803, 754, 709, 760, 791, 746, 808, 749, 752, 751, 767, 761, 748, 800, 778, 748, 749, 804, 724, 776, 806, 802, 770, 759, 796, 758, 764, 765, 761, 752, 774, 730, 735, 768, 774, 750, 735, 836, 775, 807, 805, 772, 746, 782, 753, 782, 770, 776, 773] payload_rows_per_dst=[9142, 9630, 9278, 9264, 9386, 9172, 9288, 9060] dup_rows_per_dst=[2946, 3396, 3076, 2940, 3088, 2906, 2996, 2736] combine_rows_per_src=[9142, 9630, 9278, 9264, 9386, 9172, 9288, 9060] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74344 payload_rows_remote=65040 dup_rows=23960 dup_rows_remote=20920 dedup_ratio=0.243734 router_tokens_per_expert=[744, 798, 778, 743, 781, 761, 700, 766, 812, 745, 803, 780, 783, 770, 809, 779, 755, 759, 744, 781, 749, 746, 798, 729, 695, 741, 755, 743, 780, 793, 755, 795, 777, 800, 775, 761, 798, 802, 815, 722, 748, 755, 771, 753, 767, 819, 749, 796, 771, 739, 773, 777, 738, 784, 790, 800, 791, 732, 743, 739, 789, 822, 745, 741] payload_rows_per_dst=[9110, 9498, 9204, 9318, 9388, 9280, 9304, 9242] dup_rows_per_dst=[3032, 3064, 2918, 2796, 3112, 3036, 3040, 2962] combine_rows_per_src=[9110, 9498, 9204, 9318, 9388, 9280, 9304, 9242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265879552 dispatch_meta_bytes=344272 combine_payload_bytes=265879552 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266223824 ingress_bytes=335085568 link_bytes=335085568 token_bytes=532103376 weight_movement_bytes=69206016 total_nvlink_bytes=601309392 weight_to_token_ratio=0.130061 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260900016e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266756096 dispatch_meta_bytes=344272 combine_payload_bytes=266756096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267100368 ingress_bytes=301359104 link_bytes=301359104 token_bytes=533856464 weight_movement_bytes=34603008 total_nvlink_bytes=568459472 weight_to_token_ratio=0.064817 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133990047e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=64880 dup_rows=24060 dup_rows_remote=20926 dedup_ratio=0.244751 router_tokens_per_expert=[740, 777, 735, 707, 736, 797, 774, 774, 720, 769, 744, 761, 748, 756, 748, 754, 770, 794, 769, 758, 812, 809, 804, 758, 744, 748, 775, 809, 763, 755, 755, 797, 751, 746, 778, 815, 760, 754, 741, 730, 740, 748, 691, 780, 791, 790, 777, 816, 819, 772, 744, 792, 810, 763, 784, 751, 765, 780, 751, 826, 760, 755, 848, 764] payload_rows_per_dst=[9104, 8736, 9770, 9352, 9216, 9318, 9384, 9364] dup_rows_per_dst=[2976, 2600, 3442, 2940, 2934, 2948, 3086, 3134] combine_rows_per_src=[9104, 8736, 9770, 9352, 9216, 9318, 9384, 9364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267436032 dispatch_meta_bytes=344744 combine_payload_bytes=267436032 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267780776 ingress_bytes=371245056 link_bytes=371245056 token_bytes=535216808 weight_movement_bytes=103809024 total_nvlink_bytes=639025832 weight_to_token_ratio=0.193957 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396965258e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266559488 dispatch_meta_bytes=344864 combine_payload_bytes=266559488 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266904352 ingress_bytes=370368512 link_bytes=370368512 token_bytes=533463840 weight_movement_bytes=103809024 total_nvlink_bytes=637272864 weight_to_token_ratio=0.194594 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393666894e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74462 payload_rows_remote=64960 dup_rows=23842 dup_rows_remote=20804 dedup_ratio=0.242533 router_tokens_per_expert=[776, 783, 772, 733, 757, 776, 777, 761, 758, 822, 778, 718, 772, 767, 770, 736, 723, 774, 741, 744, 765, 756, 754, 785, 792, 807, 713, 780, 828, 784, 800, 766, 775, 745, 780, 761, 770, 803, 791, 756, 750, 741, 723, 772, 796, 745, 747, 747, 783, 760, 800, 796, 741, 747, 762, 757, 753, 765, 773, 787, 806, 749, 808, 795] payload_rows_per_dst=[9366, 9250, 9250, 9502, 9264, 9170, 9264, 9396] dup_rows_per_dst=[2904, 2992, 2834, 3038, 3098, 2872, 3028, 3076] combine_rows_per_src=[9366, 9250, 9250, 9502, 9264, 9170, 9264, 9396] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=65132 dup_rows=23950 dup_rows_remote=20968 dedup_ratio=0.243632 router_tokens_per_expert=[784, 781, 755, 763, 749, 723, 775, 793, 791, 749, 763, 749, 731, 766, 767, 756, 762, 793, 730, 767, 758, 736, 752, 804, 814, 783, 772, 730, 769, 762, 749, 748, 772, 778, 744, 781, 749, 718, 746, 800, 750, 755, 766, 801, 766, 783, 804, 754, 785, 759, 778, 768, 732, 751, 824, 772, 798, 773, 813, 769, 779, 780, 774, 806] payload_rows_per_dst=[9288, 9254, 9222, 9300, 9190, 9338, 9302, 9460] dup_rows_per_dst=[2958, 2890, 2982, 2954, 2986, 3020, 3036, 3124] combine_rows_per_src=[9288, 9254, 9222, 9300, 9190, 9338, 9302, 9460] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74432 payload_rows_remote=65146 dup_rows=23872 dup_rows_remote=20784 dedup_ratio=0.242839 router_tokens_per_expert=[782, 800, 756, 784, 781, 777, 758, 744, 760, 786, 804, 774, 743, 735, 790, 795, 714, 712, 761, 726, 757, 809, 810, 774, 761, 774, 751, 802, 761, 798, 803, 761, 817, 762, 753, 726, 760, 764, 775, 793, 781, 754, 764, 808, 765, 767, 767, 795, 752, 726, 720, 724, 759, 735, 761, 800, 783, 767, 764, 749, 732, 822, 737, 827] payload_rows_per_dst=[9344, 9286, 9202, 9350, 9294, 9448, 9170, 9338] dup_rows_per_dst=[3020, 3088, 2924, 3072, 3006, 2954, 2784, 3024] combine_rows_per_src=[9344, 9286, 9202, 9350, 9294, 9448, 9170, 9338] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74086 payload_rows_remote=64786 dup_rows=24218 dup_rows_remote=21396 dedup_ratio=0.246358 router_tokens_per_expert=[814, 755, 752, 671, 755, 783, 760, 791, 834, 773, 807, 754, 791, 813, 791, 752, 739, 742, 739, 719, 781, 805, 789, 794, 723, 781, 763, 757, 776, 773, 753, 768, 751, 745, 774, 767, 732, 801, 745, 746, 765, 746, 783, 779, 784, 748, 739, 793, 801, 766, 776, 762, 762, 755, 751, 784, 767, 783, 774, 791, 759, 767, 770, 788] payload_rows_per_dst=[9126, 9476, 9266, 9162, 9300, 9192, 9264, 9300] dup_rows_per_dst=[3036, 3154, 2950, 3026, 2822, 3082, 3050, 3098] combine_rows_per_src=[9126, 9476, 9266, 9162, 9300, 9192, 9264, 9300] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74344 payload_rows_remote=65040 dup_rows=23960 dup_rows_remote=20920 dedup_ratio=0.243734 router_tokens_per_expert=[744, 798, 778, 743, 781, 761, 700, 766, 812, 745, 803, 780, 783, 770, 809, 779, 755, 759, 744, 781, 749, 746, 798, 729, 695, 741, 755, 743, 780, 793, 755, 795, 777, 800, 775, 761, 798, 802, 815, 722, 748, 755, 771, 753, 767, 819, 749, 796, 771, 739, 773, 777, 738, 784, 790, 800, 791, 732, 743, 739, 789, 822, 745, 741] payload_rows_per_dst=[9110, 9498, 9204, 9318, 9388, 9280, 9304, 9242] dup_rows_per_dst=[3032, 3064, 2918, 2796, 3112, 3036, 3040, 2962] combine_rows_per_src=[9110, 9498, 9204, 9318, 9388, 9280, 9304, 9242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=64880 dup_rows=24060 dup_rows_remote=20926 dedup_ratio=0.244751 router_tokens_per_expert=[740, 777, 735, 707, 736, 797, 774, 774, 720, 769, 744, 761, 748, 756, 748, 754, 770, 794, 769, 758, 812, 809, 804, 758, 744, 748, 775, 809, 763, 755, 755, 797, 751, 746, 778, 815, 760, 754, 741, 730, 740, 748, 691, 780, 791, 790, 777, 816, 819, 772, 744, 792, 810, 763, 784, 751, 765, 780, 751, 826, 760, 755, 848, 764] payload_rows_per_dst=[9104, 8736, 9770, 9352, 9216, 9318, 9384, 9364] dup_rows_per_dst=[2976, 2600, 3442, 2940, 2934, 2948, 3086, 3134] combine_rows_per_src=[9104, 8736, 9770, 9352, 9216, 9318, 9384, 9364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266403840 dispatch_meta_bytes=343840 combine_payload_bytes=266403840 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266747680 ingress_bytes=370212864 link_bytes=370212864 token_bytes=533151520 weight_movement_bytes=103809024 total_nvlink_bytes=636960544 weight_to_token_ratio=0.194708 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393081203e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74366 payload_rows_remote=65126 dup_rows=23938 dup_rows_remote=20942 dedup_ratio=0.243510 router_tokens_per_expert=[744, 822, 799, 751, 767, 707, 749, 743, 758, 764, 751, 776, 777, 777, 770, 758, 764, 726, 784, 794, 816, 751, 764, 743, 720, 761, 753, 799, 761, 726, 773, 733, 729, 790, 799, 784, 755, 728, 807, 765, 778, 786, 763, 722, 765, 756, 771, 777, 791, 817, 807, 802, 770, 795, 796, 738, 758, 770, 782, 791, 783, 756, 801, 739] payload_rows_per_dst=[9276, 9342, 9256, 9146, 9308, 9240, 9436, 9362] dup_rows_per_dst=[2888, 2920, 3028, 2906, 3006, 2996, 3196, 2998] combine_rows_per_src=[9276, 9342, 9256, 9146, 9308, 9240, 9436, 9362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74462 payload_rows_remote=64960 dup_rows=23842 dup_rows_remote=20804 dedup_ratio=0.242533 router_tokens_per_expert=[776, 783, 772, 733, 757, 776, 777, 761, 758, 822, 778, 718, 772, 767, 770, 736, 723, 774, 741, 744, 765, 756, 754, 785, 792, 807, 713, 780, 828, 784, 800, 766, 775, 745, 780, 761, 770, 803, 791, 756, 750, 741, 723, 772, 796, 745, 747, 747, 783, 760, 800, 796, 741, 747, 762, 757, 753, 765, 773, 787, 806, 749, 808, 795] payload_rows_per_dst=[9366, 9250, 9250, 9502, 9264, 9170, 9264, 9396] dup_rows_per_dst=[2904, 2992, 2834, 3038, 3098, 2872, 3028, 3076] combine_rows_per_src=[9366, 9250, 9250, 9502, 9264, 9170, 9264, 9396] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=65132 dup_rows=23950 dup_rows_remote=20968 dedup_ratio=0.243632 router_tokens_per_expert=[784, 781, 755, 763, 749, 723, 775, 793, 791, 749, 763, 749, 731, 766, 767, 756, 762, 793, 730, 767, 758, 736, 752, 804, 814, 783, 772, 730, 769, 762, 749, 748, 772, 778, 744, 781, 749, 718, 746, 800, 750, 755, 766, 801, 766, 783, 804, 754, 785, 759, 778, 768, 732, 751, 824, 772, 798, 773, 813, 769, 779, 780, 774, 806] payload_rows_per_dst=[9288, 9254, 9222, 9300, 9190, 9338, 9302, 9460] dup_rows_per_dst=[2958, 2890, 2982, 2954, 2986, 3020, 3036, 3124] combine_rows_per_src=[9288, 9254, 9222, 9300, 9190, 9338, 9302, 9460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266838016 dispatch_meta_bytes=343720 combine_payload_bytes=266838016 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267181736 ingress_bytes=336044032 link_bytes=336044032 token_bytes=534019752 weight_movement_bytes=69206016 total_nvlink_bytes=603225768 weight_to_token_ratio=0.129594 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264506639e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265748480 dispatch_meta_bytes=343224 combine_payload_bytes=265748480 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266091704 ingress_bytes=334954496 link_bytes=334954496 token_bytes=531840184 weight_movement_bytes=69206016 total_nvlink_bytes=601046200 weight_to_token_ratio=0.130126 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260406802e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266756096 dispatch_meta_bytes=344272 combine_payload_bytes=266756096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267100368 ingress_bytes=301359104 link_bytes=301359104 token_bytes=533856464 weight_movement_bytes=34603008 total_nvlink_bytes=568459472 weight_to_token_ratio=0.064817 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133990047e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266076160 dispatch_meta_bytes=343056 combine_payload_bytes=266076160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266419216 ingress_bytes=300679168 link_bytes=300679168 token_bytes=532495376 weight_movement_bytes=34603008 total_nvlink_bytes=567098384 weight_to_token_ratio=0.064983 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131431503e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74060 payload_rows_remote=64714 dup_rows=24244 dup_rows_remote=21200 dedup_ratio=0.246623 router_tokens_per_expert=[766, 744, 699, 757, 785, 781, 755, 780, 811, 749, 763, 789, 768, 721, 768, 710, 793, 764, 769, 784, 780, 792, 699, 772, 803, 789, 751, 784, 786, 837, 744, 765, 799, 757, 772, 802, 772, 755, 778, 760, 745, 765, 737, 806, 772, 737, 781, 734, 786, 774, 779, 785, 787, 741, 737, 783, 794, 747, 769, 784, 779, 781, 765, 731] payload_rows_per_dst=[9168, 9270, 9208, 9352, 9346, 9160, 9284, 9272] dup_rows_per_dst=[2966, 2888, 3098, 3166, 3044, 2994, 3060, 3028] combine_rows_per_src=[9168, 9270, 9208, 9352, 9346, 9160, 9284, 9272] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266780672 dispatch_meta_bytes=344400 combine_payload_bytes=266780672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267125072 ingress_bytes=301383680 link_bytes=301383680 token_bytes=533905744 weight_movement_bytes=34603008 total_nvlink_bytes=568508752 weight_to_token_ratio=0.064811 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134082524e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266838016 dispatch_meta_bytes=343720 combine_payload_bytes=266838016 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267181736 ingress_bytes=336044032 link_bytes=336044032 token_bytes=534019752 weight_movement_bytes=69206016 total_nvlink_bytes=603225768 weight_to_token_ratio=0.129594 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264506639e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=344728 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265708184 ingress_bytes=299966464 link_bytes=299966464 token_bytes=531071640 weight_movement_bytes=34603008 total_nvlink_bytes=565674648 weight_to_token_ratio=0.065157 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128749655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266403840 dispatch_meta_bytes=343840 combine_payload_bytes=266403840 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266747680 ingress_bytes=370212864 link_bytes=370212864 token_bytes=533151520 weight_movement_bytes=103809024 total_nvlink_bytes=636960544 weight_to_token_ratio=0.194708 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393081203e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74346 payload_rows_remote=64924 dup_rows=23958 dup_rows_remote=20788 dedup_ratio=0.243713 router_tokens_per_expert=[733, 801, 763, 768, 802, 753, 760, 812, 762, 788, 782, 770, 775, 813, 777, 774, 733, 773, 741, 796, 722, 775, 761, 751, 784, 774, 716, 713, 775, 784, 786, 732, 770, 763, 754, 771, 786, 777, 788, 735, 740, 789, 795, 767, 775, 839, 771, 820, 754, 750, 778, 747, 771, 774, 685, 809, 778, 755, 756, 775, 735, 758, 794, 744] payload_rows_per_dst=[9272, 9506, 9162, 9280, 9264, 9422, 9226, 9214] dup_rows_per_dst=[3112, 2976, 2942, 2848, 3024, 3170, 2910, 2976] combine_rows_per_src=[9272, 9506, 9162, 9280, 9264, 9422, 9226, 9214] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74150 payload_rows_remote=64600 dup_rows=24154 dup_rows_remote=20828 dedup_ratio=0.245707 router_tokens_per_expert=[736, 760, 801, 783, 755, 734, 759, 766, 768, 776, 789, 783, 752, 774, 821, 809, 746, 757, 757, 706, 783, 784, 726, 769, 763, 766, 678, 809, 767, 729, 768, 795, 795, 722, 760, 791, 750, 793, 772, 791, 756, 766, 815, 755, 789, 789, 778, 752, 748, 752, 752, 760, 760, 769, 835, 771, 794, 788, 789, 732, 771, 794, 767, 727] payload_rows_per_dst=[9550, 9444, 9114, 9316, 9370, 9306, 9104, 8946] dup_rows_per_dst=[3326, 3310, 2942, 3126, 3010, 3104, 2856, 2480] combine_rows_per_src=[9550, 9444, 9114, 9316, 9370, 9306, 9104, 8946] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74462 payload_rows_remote=65198 dup_rows=23842 dup_rows_remote=20906 dedup_ratio=0.242533 router_tokens_per_expert=[756, 753, 770, 740, 829, 787, 745, 787, 770, 772, 759, 773, 788, 757, 793, 737, 782, 782, 777, 771, 778, 753, 761, 744, 770, 728, 736, 818, 774, 760, 810, 755, 800, 751, 756, 727, 794, 761, 748, 727, 751, 806, 782, 776, 805, 761, 780, 782, 790, 772, 780, 715, 780, 752, 764, 747, 798, 757, 789, 785, 745, 755, 757, 744] payload_rows_per_dst=[9394, 9280, 9230, 9316, 9174, 9464, 9264, 9340] dup_rows_per_dst=[2940, 3018, 3066, 2986, 2954, 3022, 2936, 2920] combine_rows_per_src=[9394, 9280, 9230, 9316, 9174, 9464, 9264, 9340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265928704 dispatch_meta_bytes=342848 combine_payload_bytes=265928704 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266271552 ingress_bytes=369737728 link_bytes=369737728 token_bytes=532200256 weight_movement_bytes=103809024 total_nvlink_bytes=636009280 weight_to_token_ratio=0.195056 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391293304e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265068544 dispatch_meta_bytes=343656 combine_payload_bytes=265068544 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265412200 ingress_bytes=334274560 link_bytes=334274560 token_bytes=530480744 weight_movement_bytes=69206016 total_nvlink_bytes=599686760 weight_to_token_ratio=0.130459 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257848258e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=264601600 dispatch_meta_bytes=341712 combine_payload_bytes=264601600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264943312 ingress_bytes=299204608 link_bytes=299204608 token_bytes=529544912 weight_movement_bytes=34603008 total_nvlink_bytes=564147920 weight_to_token_ratio=0.065345 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125882852e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74412 payload_rows_remote=65196 dup_rows=23892 dup_rows_remote=20910 dedup_ratio=0.243042 router_tokens_per_expert=[761, 812, 790, 791, 798, 818, 741, 745, 714, 812, 742, 769, 760, 813, 761, 728, 801, 780, 787, 788, 773, 809, 753, 786, 731, 772, 773, 739, 738, 781, 803, 765, 831, 786, 749, 760, 786, 740, 734, 716, 754, 721, 767, 755, 779, 802, 752, 746, 733, 760, 756, 770, 759, 790, 781, 731, 759, 783, 776, 731, 816, 791, 767, 737] payload_rows_per_dst=[9432, 9216, 9466, 9204, 9240, 9272, 9240, 9342] dup_rows_per_dst=[3080, 2982, 3088, 3000, 2964, 2880, 2920, 2978] combine_rows_per_src=[9432, 9216, 9466, 9204, 9240, 9272, 9240, 9342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=267051008 dispatch_meta_bytes=344416 combine_payload_bytes=267051008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267395424 ingress_bytes=301654016 link_bytes=301654016 token_bytes=534446432 weight_movement_bytes=34603008 total_nvlink_bytes=569049440 weight_to_token_ratio=0.064746 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135099777e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=64676 dup_rows=24020 dup_rows_remote=20548 dedup_ratio=0.244344 router_tokens_per_expert=[769, 834, 777, 758, 778, 703, 753, 759, 758, 789, 770, 781, 758, 768, 748, 745, 746, 757, 792, 707, 795, 775, 772, 774, 744, 743, 823, 740, 784, 730, 748, 788, 799, 744, 745, 774, 751, 760, 800, 747, 784, 787, 792, 789, 802, 774, 771, 777, 736, 747, 771, 782, 730, 742, 791, 772, 780, 802, 798, 790, 759, 765, 748, 777] payload_rows_per_dst=[8906, 9200, 9082, 9508, 9300, 9264, 9416, 9608] dup_rows_per_dst=[2656, 2942, 2866, 3072, 2940, 2948, 3124, 3472] combine_rows_per_src=[8906, 9200, 9082, 9508, 9300, 9264, 9416, 9608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74346 payload_rows_remote=64924 dup_rows=23958 dup_rows_remote=20788 dedup_ratio=0.243713 router_tokens_per_expert=[733, 801, 763, 768, 802, 753, 760, 812, 762, 788, 782, 770, 775, 813, 777, 774, 733, 773, 741, 796, 722, 775, 761, 751, 784, 774, 716, 713, 775, 784, 786, 732, 770, 763, 754, 771, 786, 777, 788, 735, 740, 789, 795, 767, 775, 839, 771, 820, 754, 750, 778, 747, 771, 774, 685, 809, 778, 755, 756, 775, 735, 758, 794, 744] payload_rows_per_dst=[9272, 9506, 9162, 9280, 9264, 9422, 9226, 9214] dup_rows_per_dst=[3112, 2976, 2942, 2848, 3024, 3170, 2910, 2976] combine_rows_per_src=[9272, 9506, 9162, 9280, 9264, 9422, 9226, 9214] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74268 payload_rows_remote=64950 dup_rows=24036 dup_rows_remote=21046 dedup_ratio=0.244507 router_tokens_per_expert=[815, 813, 746, 762, 765, 794, 831, 807, 740, 795, 783, 771, 746, 750, 749, 772, 734, 801, 809, 740, 776, 767, 802, 769, 790, 810, 784, 746, 771, 770, 711, 772, 716, 735, 742, 763, 798, 752, 790, 778, 794, 808, 758, 702, 792, 755, 759, 757, 785, 723, 765, 755, 737, 795, 804, 756, 720, 746, 832, 728, 752, 775, 757, 732] payload_rows_per_dst=[9490, 9228, 9328, 9318, 9210, 9262, 9252, 9180] dup_rows_per_dst=[3176, 2984, 3068, 2990, 2938, 2988, 2988, 2904] combine_rows_per_src=[9490, 9228, 9328, 9318, 9210, 9262, 9252, 9180] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74212 payload_rows_remote=64902 dup_rows=24092 dup_rows_remote=20958 dedup_ratio=0.245076 router_tokens_per_expert=[769, 754, 789, 770, 758, 785, 780, 736, 783, 770, 737, 759, 748, 808, 798, 788, 762, 786, 790, 786, 786, 794, 779, 739, 729, 792, 762, 760, 818, 770, 770, 757, 766, 760, 761, 691, 793, 800, 754, 814, 777, 723, 732, 763, 753, 737, 802, 744, 764, 762, 791, 756, 787, 770, 758, 775, 715, 800, 770, 743, 772, 727, 770, 810] payload_rows_per_dst=[9244, 9340, 9310, 9252, 9244, 9206, 9322, 9294] dup_rows_per_dst=[3038, 3042, 3134, 3064, 3034, 2856, 3004, 2920] combine_rows_per_src=[9244, 9340, 9310, 9252, 9244, 9206, 9322, 9294] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74412 payload_rows_remote=65196 dup_rows=23892 dup_rows_remote=20910 dedup_ratio=0.243042 router_tokens_per_expert=[761, 812, 790, 791, 798, 818, 741, 745, 714, 812, 742, 769, 760, 813, 761, 728, 801, 780, 787, 788, 773, 809, 753, 786, 731, 772, 773, 739, 738, 781, 803, 765, 831, 786, 749, 760, 786, 740, 734, 716, 754, 721, 767, 755, 779, 802, 752, 746, 733, 760, 756, 770, 759, 790, 781, 731, 759, 783, 776, 731, 816, 791, 767, 737] payload_rows_per_dst=[9432, 9216, 9466, 9204, 9240, 9272, 9240, 9342] dup_rows_per_dst=[3080, 2982, 3088, 3000, 2964, 2880, 2920, 2978] combine_rows_per_src=[9432, 9216, 9466, 9204, 9240, 9272, 9240, 9342] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74462 payload_rows_remote=65198 dup_rows=23842 dup_rows_remote=20906 dedup_ratio=0.242533 router_tokens_per_expert=[756, 753, 770, 740, 829, 787, 745, 787, 770, 772, 759, 773, 788, 757, 793, 737, 782, 782, 777, 771, 778, 753, 761, 744, 770, 728, 736, 818, 774, 760, 810, 755, 800, 751, 756, 727, 794, 761, 748, 727, 751, 806, 782, 776, 805, 761, 780, 782, 790, 772, 780, 715, 780, 752, 764, 747, 798, 757, 789, 785, 745, 755, 757, 744] payload_rows_per_dst=[9394, 9280, 9230, 9316, 9174, 9464, 9264, 9340] dup_rows_per_dst=[2940, 3018, 3066, 2986, 2954, 3022, 2936, 2920] combine_rows_per_src=[9394, 9280, 9230, 9316, 9174, 9464, 9264, 9340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=64964 dup_rows=23920 dup_rows_remote=20972 dedup_ratio=0.243327 router_tokens_per_expert=[775, 780, 739, 762, 753, 781, 816, 738, 769, 792, 788, 744, 735, 797, 811, 756, 793, 793, 838, 744, 726, 722, 826, 788, 711, 760, 840, 724, 762, 800, 792, 717, 754, 761, 772, 760, 788, 742, 783, 824, 770, 792, 776, 758, 735, 795, 712, 767, 794, 704, 818, 785, 798, 733, 706, 767, 752, 778, 782, 786, 747, 721, 768, 752] payload_rows_per_dst=[9384, 9328, 9406, 9218, 9420, 9244, 9238, 9146] dup_rows_per_dst=[2904, 3056, 3054, 2994, 2948, 2966, 2972, 3026] combine_rows_per_src=[9384, 9328, 9406, 9218, 9420, 9244, 9238, 9146] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74212 payload_rows_remote=64902 dup_rows=24092 dup_rows_remote=20958 dedup_ratio=0.245076 router_tokens_per_expert=[769, 754, 789, 770, 758, 785, 780, 736, 783, 770, 737, 759, 748, 808, 798, 788, 762, 786, 790, 786, 786, 794, 779, 739, 729, 792, 762, 760, 818, 770, 770, 757, 766, 760, 761, 691, 793, 800, 754, 814, 777, 723, 732, 763, 753, 737, 802, 744, 764, 762, 791, 756, 787, 770, 758, 775, 715, 800, 770, 743, 772, 727, 770, 810] payload_rows_per_dst=[9244, 9340, 9310, 9252, 9244, 9206, 9322, 9294] dup_rows_per_dst=[3038, 3042, 3134, 3064, 3034, 2856, 3004, 2920] combine_rows_per_src=[9244, 9340, 9310, 9252, 9244, 9206, 9322, 9294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=267042816 dispatch_meta_bytes=344424 combine_payload_bytes=267042816 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267387240 ingress_bytes=336248832 link_bytes=336248832 token_bytes=534430056 weight_movement_bytes=69206016 total_nvlink_bytes=603636072 weight_to_token_ratio=0.129495 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265277285e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265928704 dispatch_meta_bytes=342848 combine_payload_bytes=265928704 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266271552 ingress_bytes=369737728 link_bytes=369737728 token_bytes=532200256 weight_movement_bytes=103809024 total_nvlink_bytes=636009280 weight_to_token_ratio=0.195056 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391293304e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=264912896 dispatch_meta_bytes=340896 combine_payload_bytes=264912896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265253792 ingress_bytes=299515904 link_bytes=299515904 token_bytes=530166688 weight_movement_bytes=34603008 total_nvlink_bytes=564769696 weight_to_token_ratio=0.065268 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127054234e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74086 payload_rows_remote=64786 dup_rows=24218 dup_rows_remote=21396 dedup_ratio=0.246358 router_tokens_per_expert=[814, 755, 752, 671, 755, 783, 760, 791, 834, 773, 807, 754, 791, 813, 791, 752, 739, 742, 739, 719, 781, 805, 789, 794, 723, 781, 763, 757, 776, 773, 753, 768, 751, 745, 774, 767, 732, 801, 745, 746, 765, 746, 783, 779, 784, 748, 739, 793, 801, 766, 776, 762, 762, 755, 751, 784, 767, 783, 774, 791, 759, 767, 770, 788] payload_rows_per_dst=[9126, 9476, 9266, 9162, 9300, 9192, 9264, 9300] dup_rows_per_dst=[3036, 3154, 2950, 3026, 2822, 3082, 3050, 3098] combine_rows_per_src=[9126, 9476, 9266, 9162, 9300, 9192, 9264, 9300] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266035200 dispatch_meta_bytes=343984 combine_payload_bytes=266035200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266379184 ingress_bytes=300638208 link_bytes=300638208 token_bytes=532414384 weight_movement_bytes=34603008 total_nvlink_bytes=567017392 weight_to_token_ratio=0.064993 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131277373e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265838592 dispatch_meta_bytes=343440 combine_payload_bytes=265838592 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266182032 ingress_bytes=335044608 link_bytes=335044608 token_bytes=532020624 weight_movement_bytes=69206016 total_nvlink_bytes=601226640 weight_to_token_ratio=0.130081 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260745887e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=267042816 dispatch_meta_bytes=344424 combine_payload_bytes=267042816 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267387240 ingress_bytes=336248832 link_bytes=336248832 token_bytes=534430056 weight_movement_bytes=69206016 total_nvlink_bytes=603636072 weight_to_token_ratio=0.129495 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265277285e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=267051008 dispatch_meta_bytes=344416 combine_payload_bytes=267051008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267395424 ingress_bytes=301654016 link_bytes=301654016 token_bytes=534446432 weight_movement_bytes=34603008 total_nvlink_bytes=569049440 weight_to_token_ratio=0.064746 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135099777e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74084 payload_rows_remote=64884 dup_rows=24220 dup_rows_remote=21224 dedup_ratio=0.246379 router_tokens_per_expert=[776, 766, 747, 810, 723, 775, 801, 739, 770, 758, 748, 714, 802, 783, 771, 785, 774, 791, 769, 746, 779, 766, 803, 770, 848, 756, 756, 794, 759, 759, 793, 777, 784, 764, 743, 761, 730, 772, 835, 741, 757, 730, 720, 782, 779, 760, 789, 781, 809, 757, 760, 765, 763, 775, 754, 745, 736, 774, 734, 762, 826, 783, 748, 725] payload_rows_per_dst=[9224, 9238, 9340, 9320, 9276, 9200, 9316, 9170] dup_rows_per_dst=[3050, 3024, 3056, 3164, 2984, 2996, 2940, 3006] combine_rows_per_src=[9224, 9238, 9340, 9320, 9276, 9200, 9316, 9170] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=343744 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266436288 ingress_bytes=300695552 link_bytes=300695552 token_bytes=532528832 weight_movement_bytes=34603008 total_nvlink_bytes=567131840 weight_to_token_ratio=0.064979 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131493154e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74068 payload_rows_remote=64816 dup_rows=24236 dup_rows_remote=21212 dedup_ratio=0.246541 router_tokens_per_expert=[786, 780, 739, 718, 788, 771, 749, 802, 765, 808, 762, 725, 762, 773, 761, 752, 771, 773, 754, 763, 746, 711, 756, 777, 767, 779, 783, 745, 775, 785, 721, 772, 769, 774, 796, 770, 813, 746, 778, 778, 772, 801, 769, 811, 737, 786, 778, 817, 786, 770, 757, 784, 791, 756, 831, 771, 720, 736, 764, 744, 795, 706, 770, 757] payload_rows_per_dst=[9252, 9290, 9146, 9150, 9462, 9338, 9284, 9146] dup_rows_per_dst=[3024, 2916, 2956, 3104, 3188, 3002, 3208, 2838] combine_rows_per_src=[9252, 9290, 9146, 9150, 9462, 9338, 9284, 9146] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265838592 dispatch_meta_bytes=343440 combine_payload_bytes=265838592 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266182032 ingress_bytes=335044608 link_bytes=335044608 token_bytes=532020624 weight_movement_bytes=69206016 total_nvlink_bytes=601226640 weight_to_token_ratio=0.130081 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260745887e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=65088 dup_rows=23928 dup_rows_remote=20810 dedup_ratio=0.243408 router_tokens_per_expert=[764, 768, 771, 757, 736, 774, 800, 818, 773, 754, 788, 736, 788, 785, 734, 785, 739, 818, 802, 813, 771, 744, 771, 754, 749, 751, 726, 735, 772, 801, 766, 736, 756, 781, 765, 739, 733, 746, 777, 767, 760, 793, 764, 741, 761, 768, 741, 771, 731, 753, 772, 769, 845, 796, 759, 778, 762, 783, 791, 760, 772, 754, 814, 771] payload_rows_per_dst=[9346, 9356, 9400, 9150, 9184, 9248, 9288, 9404] dup_rows_per_dst=[3030, 2930, 3024, 2922, 2944, 2950, 3118, 3010] combine_rows_per_src=[9346, 9356, 9400, 9150, 9184, 9248, 9288, 9404] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265764864 dispatch_meta_bytes=344432 combine_payload_bytes=265764864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266109296 ingress_bytes=300367872 link_bytes=300367872 token_bytes=531874160 weight_movement_bytes=34603008 total_nvlink_bytes=566477168 weight_to_token_ratio=0.065059 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130260121e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=344728 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265708184 ingress_bytes=299966464 link_bytes=299966464 token_bytes=531071640 weight_movement_bytes=34603008 total_nvlink_bytes=565674648 weight_to_token_ratio=0.065157 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128749655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265486336 dispatch_meta_bytes=344112 combine_payload_bytes=265486336 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265830448 ingress_bytes=369295360 link_bytes=369295360 token_bytes=531316784 weight_movement_bytes=103809024 total_nvlink_bytes=635125808 weight_to_token_ratio=0.195381 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389628709e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74320 payload_rows_remote=65080 dup_rows=23984 dup_rows_remote=21014 dedup_ratio=0.243978 router_tokens_per_expert=[736, 767, 786, 772, 811, 758, 801, 740, 762, 767, 781, 756, 806, 771, 760, 702, 805, 723, 808, 770, 736, 731, 769, 739, 775, 782, 775, 758, 756, 754, 813, 753, 755, 781, 797, 789, 745, 740, 796, 728, 757, 732, 744, 765, 771, 798, 792, 781, 754, 805, 789, 722, 767, 802, 795, 704, 816, 735, 763, 826, 761, 791, 784, 744] payload_rows_per_dst=[9328, 9240, 9258, 9280, 9250, 9274, 9288, 9402] dup_rows_per_dst=[3014, 2970, 2904, 3052, 3012, 3006, 2988, 3038] combine_rows_per_src=[9328, 9240, 9258, 9280, 9250, 9274, 9288, 9402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=343592 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266944040 ingress_bytes=301203456 link_bytes=301203456 token_bytes=533544488 weight_movement_bytes=34603008 total_nvlink_bytes=568147496 weight_to_token_ratio=0.064855 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133404356e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74084 payload_rows_remote=64884 dup_rows=24220 dup_rows_remote=21224 dedup_ratio=0.246379 router_tokens_per_expert=[776, 766, 747, 810, 723, 775, 801, 739, 770, 758, 748, 714, 802, 783, 771, 785, 774, 791, 769, 746, 779, 766, 803, 770, 848, 756, 756, 794, 759, 759, 793, 777, 784, 764, 743, 761, 730, 772, 835, 741, 757, 730, 720, 782, 779, 760, 789, 781, 809, 757, 760, 765, 763, 775, 754, 745, 736, 774, 734, 762, 826, 783, 748, 725] payload_rows_per_dst=[9224, 9238, 9340, 9320, 9276, 9200, 9316, 9170] dup_rows_per_dst=[3050, 3024, 3056, 3164, 2984, 2996, 2940, 3006] combine_rows_per_src=[9224, 9238, 9340, 9320, 9276, 9200, 9316, 9170] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73972 payload_rows_remote=64714 dup_rows=24332 dup_rows_remote=21360 dedup_ratio=0.247518 router_tokens_per_expert=[762, 734, 791, 739, 701, 809, 754, 784, 790, 755, 802, 766, 792, 759, 773, 826, 751, 785, 780, 773, 775, 789, 764, 785, 759, 754, 758, 804, 774, 750, 799, 787, 755, 770, 769, 785, 734, 760, 746, 790, 767, 754, 786, 769, 764, 761, 766, 741, 766, 769, 769, 736, 767, 785, 722, 782, 751, 758, 752, 758, 748, 808, 780, 760] payload_rows_per_dst=[9194, 9130, 9398, 9254, 9262, 9194, 9282, 9258] dup_rows_per_dst=[2954, 3030, 3200, 3116, 2956, 3022, 3082, 2972] combine_rows_per_src=[9194, 9130, 9398, 9254, 9262, 9194, 9282, 9258] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74460 payload_rows_remote=65004 dup_rows=23844 dup_rows_remote=20794 dedup_ratio=0.242554 router_tokens_per_expert=[803, 750, 730, 811, 770, 741, 767, 741, 756, 752, 772, 771, 761, 773, 784, 771, 748, 741, 726, 749, 758, 776, 791, 780, 764, 742, 729, 782, 794, 797, 825, 820, 772, 736, 757, 806, 738, 769, 752, 757, 750, 822, 748, 762, 797, 765, 741, 758, 759, 757, 730, 705, 812, 753, 783, 756, 767, 829, 769, 815, 754, 783, 775, 800] payload_rows_per_dst=[9302, 9356, 9194, 9456, 9306, 9268, 9162, 9416] dup_rows_per_dst=[2924, 2924, 2944, 3050, 2868, 3018, 2948, 3168] combine_rows_per_src=[9302, 9356, 9194, 9456, 9306, 9268, 9162, 9416] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74448 payload_rows_remote=65190 dup_rows=23856 dup_rows_remote=21006 dedup_ratio=0.242676 router_tokens_per_expert=[734, 751, 796, 757, 777, 736, 737, 774, 759, 781, 763, 775, 751, 781, 781, 748, 733, 742, 775, 767, 720, 780, 757, 780, 759, 765, 814, 758, 764, 740, 773, 798, 800, 765, 767, 825, 763, 766, 786, 814, 778, 744, 727, 731, 719, 783, 785, 779, 790, 803, 774, 768, 799, 764, 751, 742, 778, 765, 768, 742, 747, 803, 813, 787] payload_rows_per_dst=[9206, 9262, 9258, 9316, 9460, 9156, 9402, 9388] dup_rows_per_dst=[2918, 3016, 2850, 3026, 3112, 2936, 2980, 3018] combine_rows_per_src=[9206, 9262, 9258, 9316, 9460, 9156, 9402, 9388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74320 payload_rows_remote=65080 dup_rows=23984 dup_rows_remote=21014 dedup_ratio=0.243978 router_tokens_per_expert=[736, 767, 786, 772, 811, 758, 801, 740, 762, 767, 781, 756, 806, 771, 760, 702, 805, 723, 808, 770, 736, 731, 769, 739, 775, 782, 775, 758, 756, 754, 813, 753, 755, 781, 797, 789, 745, 740, 796, 728, 757, 732, 744, 765, 771, 798, 792, 781, 754, 805, 789, 722, 767, 802, 795, 704, 816, 735, 763, 826, 761, 791, 784, 744] payload_rows_per_dst=[9328, 9240, 9258, 9280, 9250, 9274, 9288, 9402] dup_rows_per_dst=[3014, 2970, 2904, 3052, 3012, 3006, 2988, 3038] combine_rows_per_src=[9328, 9240, 9258, 9280, 9250, 9274, 9288, 9402] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=65088 dup_rows=23928 dup_rows_remote=20810 dedup_ratio=0.243408 router_tokens_per_expert=[764, 768, 771, 757, 736, 774, 800, 818, 773, 754, 788, 736, 788, 785, 734, 785, 739, 818, 802, 813, 771, 744, 771, 754, 749, 751, 726, 735, 772, 801, 766, 736, 756, 781, 765, 739, 733, 746, 777, 767, 760, 793, 764, 741, 761, 768, 741, 771, 731, 753, 772, 769, 845, 796, 759, 778, 762, 783, 791, 760, 772, 754, 814, 771] payload_rows_per_dst=[9346, 9356, 9400, 9150, 9184, 9248, 9288, 9404] dup_rows_per_dst=[3030, 2930, 3024, 2922, 2944, 2950, 3118, 3010] combine_rows_per_src=[9346, 9356, 9400, 9150, 9184, 9248, 9288, 9404] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74124 payload_rows_remote=64814 dup_rows=24180 dup_rows_remote=21250 dedup_ratio=0.245972 router_tokens_per_expert=[763, 805, 748, 772, 717, 781, 769, 725, 749, 764, 744, 819, 804, 789, 832, 749, 762, 806, 762, 710, 758, 790, 731, 779, 803, 777, 774, 729, 790, 736, 776, 758, 745, 750, 760, 760, 815, 768, 772, 750, 774, 745, 764, 799, 765, 765, 811, 760, 765, 799, 724, 765, 823, 737, 769, 757, 770, 773, 789, 775, 749, 738, 783, 762] payload_rows_per_dst=[9240, 9390, 9164, 9158, 9310, 9324, 9276, 9262] dup_rows_per_dst=[2920, 3110, 3032, 3128, 2930, 3042, 3002, 3016] combine_rows_per_src=[9240, 9390, 9164, 9158, 9310, 9324, 9276, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266567680 dispatch_meta_bytes=344376 combine_payload_bytes=266567680 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266912056 ingress_bytes=370376704 link_bytes=370376704 token_bytes=533479736 weight_movement_bytes=103809024 total_nvlink_bytes=637288760 weight_to_token_ratio=0.194589 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393697719e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74448 payload_rows_remote=65190 dup_rows=23856 dup_rows_remote=21006 dedup_ratio=0.242676 router_tokens_per_expert=[734, 751, 796, 757, 777, 736, 737, 774, 759, 781, 763, 775, 751, 781, 781, 748, 733, 742, 775, 767, 720, 780, 757, 780, 759, 765, 814, 758, 764, 740, 773, 798, 800, 765, 767, 825, 763, 766, 786, 814, 778, 744, 727, 731, 719, 783, 785, 779, 790, 803, 774, 768, 799, 764, 751, 742, 778, 765, 768, 742, 747, 803, 813, 787] payload_rows_per_dst=[9206, 9262, 9258, 9316, 9460, 9156, 9402, 9388] dup_rows_per_dst=[2918, 3016, 2850, 3026, 3112, 2936, 2980, 3018] combine_rows_per_src=[9206, 9262, 9258, 9316, 9460, 9156, 9402, 9388] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265764864 dispatch_meta_bytes=344432 combine_payload_bytes=265764864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266109296 ingress_bytes=300367872 link_bytes=300367872 token_bytes=531874160 weight_movement_bytes=34603008 total_nvlink_bytes=566477168 weight_to_token_ratio=0.065059 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130260121e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265068544 dispatch_meta_bytes=344296 combine_payload_bytes=265068544 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265412840 ingress_bytes=334274560 link_bytes=334274560 token_bytes=530481384 weight_movement_bytes=69206016 total_nvlink_bytes=599687400 weight_to_token_ratio=0.130459 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257848258e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=64964 dup_rows=23920 dup_rows_remote=20972 dedup_ratio=0.243327 router_tokens_per_expert=[775, 780, 739, 762, 753, 781, 816, 738, 769, 792, 788, 744, 735, 797, 811, 756, 793, 793, 838, 744, 726, 722, 826, 788, 711, 760, 840, 724, 762, 800, 792, 717, 754, 761, 772, 760, 788, 742, 783, 824, 770, 792, 776, 758, 735, 795, 712, 767, 794, 704, 818, 785, 798, 733, 706, 767, 752, 778, 782, 786, 747, 721, 768, 752] payload_rows_per_dst=[9384, 9328, 9406, 9218, 9420, 9244, 9238, 9146] dup_rows_per_dst=[2904, 3056, 3054, 2994, 2948, 2966, 2972, 3026] combine_rows_per_src=[9384, 9328, 9406, 9218, 9420, 9244, 9238, 9146] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266256384 dispatch_meta_bytes=343192 combine_payload_bytes=266256384 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266599576 ingress_bytes=335462400 link_bytes=335462400 token_bytes=532855960 weight_movement_bytes=69206016 total_nvlink_bytes=602061976 weight_to_token_ratio=0.129878 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262318004e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=267018240 dispatch_meta_bytes=344784 combine_payload_bytes=267018240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267363024 ingress_bytes=301621248 link_bytes=301621248 token_bytes=534381264 weight_movement_bytes=34603008 total_nvlink_bytes=568984272 weight_to_token_ratio=0.064753 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134976474e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74308 payload_rows_remote=64964 dup_rows=23996 dup_rows_remote=20960 dedup_ratio=0.244100 router_tokens_per_expert=[790, 727, 743, 794, 775, 757, 803, 750, 783, 737, 772, 778, 777, 774, 774, 769, 802, 760, 746, 804, 734, 800, 764, 769, 779, 794, 757, 753, 771, 756, 756, 788, 795, 754, 753, 789, 748, 829, 721, 699, 762, 823, 714, 770, 787, 765, 806, 763, 766, 710, 782, 729, 778, 776, 767, 787, 752, 770, 788, 740, 781, 788, 763, 761] payload_rows_per_dst=[9260, 9344, 9304, 9244, 9232, 9344, 9232, 9348] dup_rows_per_dst=[3018, 2984, 3054, 3064, 2944, 3036, 2958, 2938] combine_rows_per_src=[9260, 9344, 9304, 9244, 9232, 9344, 9232, 9348] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74196 payload_rows_remote=64732 dup_rows=24108 dup_rows_remote=20918 dedup_ratio=0.245239 router_tokens_per_expert=[759, 773, 775, 731, 742, 755, 782, 813, 707, 809, 824, 726, 766, 761, 767, 769, 754, 739, 772, 762, 769, 798, 747, 800, 766, 761, 781, 742, 777, 784, 824, 753, 760, 716, 788, 779, 763, 707, 734, 747, 737, 791, 771, 786, 747, 769, 775, 810, 776, 741, 741, 766, 770, 788, 759, 775, 789, 792, 774, 779, 769, 780, 817, 768] payload_rows_per_dst=[9464, 9336, 9276, 9012, 9250, 9512, 9300, 9046] dup_rows_per_dst=[3190, 2992, 3006, 2900, 3200, 3158, 2932, 2730] combine_rows_per_src=[9464, 9336, 9276, 9012, 9250, 9512, 9300, 9046] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265478144 dispatch_meta_bytes=344256 combine_payload_bytes=265478144 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265822400 ingress_bytes=300081152 link_bytes=300081152 token_bytes=531300544 weight_movement_bytes=34603008 total_nvlink_bytes=565903552 weight_to_token_ratio=0.065129 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129181216e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266567680 dispatch_meta_bytes=344376 combine_payload_bytes=266567680 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266912056 ingress_bytes=370376704 link_bytes=370376704 token_bytes=533479736 weight_movement_bytes=103809024 total_nvlink_bytes=637288760 weight_to_token_ratio=0.194589 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393697719e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=343592 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266944040 ingress_bytes=301203456 link_bytes=301203456 token_bytes=533544488 weight_movement_bytes=34603008 total_nvlink_bytes=568147496 weight_to_token_ratio=0.064855 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133404356e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=267018240 dispatch_meta_bytes=344784 combine_payload_bytes=267018240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267363024 ingress_bytes=301621248 link_bytes=301621248 token_bytes=534381264 weight_movement_bytes=34603008 total_nvlink_bytes=568984272 weight_to_token_ratio=0.064753 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134976474e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74546 payload_rows_remote=65252 dup_rows=23758 dup_rows_remote=20776 dedup_ratio=0.241679 router_tokens_per_expert=[737, 732, 772, 743, 766, 771, 788, 739, 804, 859, 729, 763, 730, 780, 751, 749, 774, 772, 717, 818, 761, 765, 807, 834, 762, 753, 777, 796, 789, 755, 771, 780, 824, 703, 753, 731, 771, 751, 743, 813, 789, 763, 739, 755, 722, 769, 771, 759, 814, 793, 771, 725, 759, 737, 752, 787, 771, 793, 773, 755, 763, 839, 761, 759] payload_rows_per_dst=[9194, 9318, 9464, 9340, 9230, 9246, 9294, 9460] dup_rows_per_dst=[2902, 3012, 3032, 3026, 2948, 2888, 2982, 2968] combine_rows_per_src=[9194, 9318, 9464, 9340, 9230, 9246, 9294, 9460] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65226 dup_rows=23944 dup_rows_remote=21088 dedup_ratio=0.243571 router_tokens_per_expert=[802, 775, 758, 789, 804, 816, 754, 770, 733, 771, 771, 738, 780, 809, 758, 771, 753, 763, 758, 794, 769, 749, 795, 759, 794, 791, 737, 740, 764, 792, 771, 763, 708, 769, 794, 842, 767, 757, 753, 732, 762, 800, 770, 754, 789, 782, 778, 770, 727, 758, 746, 788, 799, 763, 802, 758, 781, 742, 777, 740, 716, 743, 754, 740] payload_rows_per_dst=[9458, 9254, 9262, 9310, 9274, 9202, 9466, 9134] dup_rows_per_dst=[3078, 3008, 3018, 2994, 2970, 2830, 3190, 2856] combine_rows_per_src=[9458, 9254, 9262, 9310, 9274, 9202, 9466, 9134] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=65124 dup_rows=24072 dup_rows_remote=21120 dedup_ratio=0.244873 router_tokens_per_expert=[763, 792, 758, 730, 791, 762, 754, 780, 783, 734, 766, 750, 767, 824, 765, 780, 753, 800, 767, 779, 756, 786, 772, 795, 748, 734, 762, 745, 787, 743, 775, 736, 779, 753, 771, 746, 732, 820, 739, 732, 801, 746, 764, 786, 819, 793, 795, 776, 809, 720, 803, 747, 736, 724, 765, 723, 797, 830, 769, 743, 789, 744, 743, 821] payload_rows_per_dst=[9282, 9268, 9408, 9108, 9162, 9432, 9134, 9438] dup_rows_per_dst=[2978, 3070, 3008, 2952, 2982, 3128, 2920, 3034] combine_rows_per_src=[9282, 9268, 9408, 9108, 9162, 9432, 9134, 9438] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74430 payload_rows_remote=65010 dup_rows=23874 dup_rows_remote=20918 dedup_ratio=0.242859 router_tokens_per_expert=[770, 766, 735, 760, 777, 746, 812, 733, 761, 752, 806, 738, 770, 773, 772, 774, 740, 712, 781, 802, 806, 762, 793, 792, 785, 753, 795, 774, 781, 744, 795, 781, 755, 789, 768, 752, 771, 769, 767, 770, 748, 772, 743, 773, 711, 792, 742, 760, 760, 792, 779, 791, 796, 766, 775, 761, 759, 773, 762, 784, 761, 757, 734, 779] payload_rows_per_dst=[9244, 9416, 9420, 9286, 9284, 9134, 9410, 9236] dup_rows_per_dst=[2954, 2876, 2956, 3130, 2998, 2948, 3030, 2982] combine_rows_per_src=[9244, 9416, 9420, 9286, 9284, 9134, 9410, 9236] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74260 payload_rows_remote=64972 dup_rows=24044 dup_rows_remote=21032 dedup_ratio=0.244588 router_tokens_per_expert=[717, 752, 736, 768, 769, 755, 723, 747, 761, 740, 752, 794, 768, 739, 770, 782, 784, 737, 758, 807, 802, 777, 789, 762, 751, 764, 738, 753, 765, 726, 802, 765, 745, 807, 749, 738, 796, 787, 768, 760, 750, 797, 782, 788, 754, 746, 807, 780, 726, 789, 750, 746, 778, 764, 778, 797, 822, 782, 789, 785, 756, 770, 795, 818] payload_rows_per_dst=[9070, 9194, 9344, 9190, 9288, 9402, 9348, 9424] dup_rows_per_dst=[2864, 3018, 3088, 2938, 3012, 3006, 2908, 3210] combine_rows_per_src=[9070, 9194, 9344, 9190, 9288, 9402, 9348, 9424] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74544 payload_rows_remote=65304 dup_rows=23760 dup_rows_remote=20782 dedup_ratio=0.241699 router_tokens_per_expert=[799, 759, 738, 774, 761, 743, 782, 726, 728, 717, 825, 750, 752, 793, 743, 801, 790, 740, 786, 779, 778, 744, 728, 773, 790, 736, 755, 731, 791, 773, 738, 824, 785, 762, 771, 775, 768, 776, 779, 774, 761, 752, 794, 821, 747, 756, 771, 751, 795, 769, 801, 814, 739, 786, 733, 725, 734, 795, 817, 729, 789, 788, 756, 792] payload_rows_per_dst=[9262, 9240, 9396, 9240, 9272, 9290, 9352, 9492] dup_rows_per_dst=[2902, 2978, 2840, 3036, 3108, 3016, 2972, 2908] combine_rows_per_src=[9262, 9240, 9396, 9240, 9272, 9290, 9352, 9492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=343744 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266436288 ingress_bytes=300695552 link_bytes=300695552 token_bytes=532528832 weight_movement_bytes=34603008 total_nvlink_bytes=567131840 weight_to_token_ratio=0.064979 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131493154e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265142272 dispatch_meta_bytes=342600 combine_payload_bytes=265142272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265484872 ingress_bytes=299745280 link_bytes=299745280 token_bytes=530627144 weight_movement_bytes=34603008 total_nvlink_bytes=565230152 weight_to_token_ratio=0.065212 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127917357e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=343696 combine_payload_bytes=266092544 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266436240 ingress_bytes=369901568 link_bytes=369901568 token_bytes=532528784 weight_movement_bytes=103809024 total_nvlink_bytes=636337808 weight_to_token_ratio=0.194936 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391909821e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267272192 dispatch_meta_bytes=344112 combine_payload_bytes=267272192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267616304 ingress_bytes=301875200 link_bytes=301875200 token_bytes=534888496 weight_movement_bytes=34603008 total_nvlink_bytes=569491504 weight_to_token_ratio=0.064692 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135932075e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74544 payload_rows_remote=65304 dup_rows=23760 dup_rows_remote=20782 dedup_ratio=0.241699 router_tokens_per_expert=[799, 759, 738, 774, 761, 743, 782, 726, 728, 717, 825, 750, 752, 793, 743, 801, 790, 740, 786, 779, 778, 744, 728, 773, 790, 736, 755, 731, 791, 773, 738, 824, 785, 762, 771, 775, 768, 776, 779, 774, 761, 752, 794, 821, 747, 756, 771, 751, 795, 769, 801, 814, 739, 786, 733, 725, 734, 795, 817, 729, 789, 788, 756, 792] payload_rows_per_dst=[9262, 9240, 9396, 9240, 9272, 9290, 9352, 9492] dup_rows_per_dst=[2902, 2978, 2840, 3036, 3108, 3016, 2972, 2908] combine_rows_per_src=[9262, 9240, 9396, 9240, 9272, 9290, 9352, 9492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74308 payload_rows_remote=64964 dup_rows=23996 dup_rows_remote=20960 dedup_ratio=0.244100 router_tokens_per_expert=[790, 727, 743, 794, 775, 757, 803, 750, 783, 737, 772, 778, 777, 774, 774, 769, 802, 760, 746, 804, 734, 800, 764, 769, 779, 794, 757, 753, 771, 756, 756, 788, 795, 754, 753, 789, 748, 829, 721, 699, 762, 823, 714, 770, 787, 765, 806, 763, 766, 710, 782, 729, 778, 776, 767, 787, 752, 770, 788, 740, 781, 788, 763, 761] payload_rows_per_dst=[9260, 9344, 9304, 9244, 9232, 9344, 9232, 9348] dup_rows_per_dst=[3018, 2984, 3054, 3064, 2944, 3036, 2958, 2938] combine_rows_per_src=[9260, 9344, 9304, 9244, 9232, 9344, 9232, 9348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65226 dup_rows=23944 dup_rows_remote=21088 dedup_ratio=0.243571 router_tokens_per_expert=[802, 775, 758, 789, 804, 816, 754, 770, 733, 771, 771, 738, 780, 809, 758, 771, 753, 763, 758, 794, 769, 749, 795, 759, 794, 791, 737, 740, 764, 792, 771, 763, 708, 769, 794, 842, 767, 757, 753, 732, 762, 800, 770, 754, 789, 782, 778, 770, 727, 758, 746, 788, 799, 763, 802, 758, 781, 742, 777, 740, 716, 743, 754, 740] payload_rows_per_dst=[9458, 9254, 9262, 9310, 9274, 9202, 9466, 9134] dup_rows_per_dst=[3078, 3008, 3018, 2994, 2970, 2830, 3190, 2856] combine_rows_per_src=[9458, 9254, 9262, 9310, 9274, 9202, 9466, 9134] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=65124 dup_rows=24072 dup_rows_remote=21120 dedup_ratio=0.244873 router_tokens_per_expert=[763, 792, 758, 730, 791, 762, 754, 780, 783, 734, 766, 750, 767, 824, 765, 780, 753, 800, 767, 779, 756, 786, 772, 795, 748, 734, 762, 745, 787, 743, 775, 736, 779, 753, 771, 746, 732, 820, 739, 732, 801, 746, 764, 786, 819, 793, 795, 776, 809, 720, 803, 747, 736, 724, 765, 723, 797, 830, 769, 743, 789, 744, 743, 821] payload_rows_per_dst=[9282, 9268, 9408, 9108, 9162, 9432, 9134, 9438] dup_rows_per_dst=[2978, 3070, 3008, 2952, 2982, 3128, 2920, 3034] combine_rows_per_src=[9282, 9268, 9408, 9108, 9162, 9432, 9134, 9438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267485184 dispatch_meta_bytes=344344 combine_payload_bytes=267485184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267829528 ingress_bytes=302088192 link_bytes=302088192 token_bytes=535314712 weight_movement_bytes=34603008 total_nvlink_bytes=569917720 weight_to_token_ratio=0.064640 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136733546e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74430 payload_rows_remote=65010 dup_rows=23874 dup_rows_remote=20918 dedup_ratio=0.242859 router_tokens_per_expert=[770, 766, 735, 760, 777, 746, 812, 733, 761, 752, 806, 738, 770, 773, 772, 774, 740, 712, 781, 802, 806, 762, 793, 792, 785, 753, 795, 774, 781, 744, 795, 781, 755, 789, 768, 752, 771, 769, 767, 770, 748, 772, 743, 773, 711, 792, 742, 760, 760, 792, 779, 791, 796, 766, 775, 761, 759, 773, 762, 784, 761, 757, 734, 779] payload_rows_per_dst=[9244, 9416, 9420, 9286, 9284, 9134, 9410, 9236] dup_rows_per_dst=[2954, 2876, 2956, 3130, 2998, 2948, 3030, 2982] combine_rows_per_src=[9244, 9416, 9420, 9286, 9284, 9134, 9410, 9236] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=343696 combine_payload_bytes=266092544 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266436240 ingress_bytes=369901568 link_bytes=369901568 token_bytes=532528784 weight_movement_bytes=103809024 total_nvlink_bytes=636337808 weight_to_token_ratio=0.194936 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391909821e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267165696 dispatch_meta_bytes=345256 combine_payload_bytes=267165696 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=267510952 ingress_bytes=405577728 link_bytes=405577728 token_bytes=534676648 weight_movement_bytes=138412032 total_nvlink_bytes=673088680 weight_to_token_ratio=0.258871 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.526156339e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74124 payload_rows_remote=64814 dup_rows=24180 dup_rows_remote=21250 dedup_ratio=0.245972 router_tokens_per_expert=[763, 805, 748, 772, 717, 781, 769, 725, 749, 764, 744, 819, 804, 789, 832, 749, 762, 806, 762, 710, 758, 790, 731, 779, 803, 777, 774, 729, 790, 736, 776, 758, 745, 750, 760, 760, 815, 768, 772, 750, 774, 745, 764, 799, 765, 765, 811, 760, 765, 799, 724, 765, 823, 737, 769, 757, 770, 773, 789, 775, 749, 738, 783, 762] payload_rows_per_dst=[9240, 9390, 9164, 9158, 9310, 9324, 9276, 9262] dup_rows_per_dst=[2920, 3110, 3032, 3128, 2930, 3042, 3002, 3016] combine_rows_per_src=[9240, 9390, 9164, 9158, 9310, 9324, 9276, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74194 payload_rows_remote=64636 dup_rows=24110 dup_rows_remote=20830 dedup_ratio=0.245260 router_tokens_per_expert=[759, 758, 784, 738, 756, 772, 758, 813, 757, 743, 797, 740, 809, 767, 731, 794, 774, 749, 756, 789, 780, 763, 774, 747, 799, 723, 782, 794, 791, 782, 766, 769, 776, 777, 784, 773, 736, 800, 777, 777, 790, 811, 809, 768, 709, 788, 784, 746, 717, 761, 790, 733, 734, 812, 795, 712, 772, 735, 789, 694, 765, 769, 804, 751] payload_rows_per_dst=[9558, 9270, 8900, 9332, 9454, 9390, 9204, 9086] dup_rows_per_dst=[3280, 3054, 2810, 3114, 3146, 3012, 2904, 2790] combine_rows_per_src=[9558, 9270, 8900, 9332, 9454, 9390, 9204, 9086] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266747904 dispatch_meta_bytes=344976 combine_payload_bytes=266747904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267092880 ingress_bytes=301350912 link_bytes=301350912 token_bytes=533840784 weight_movement_bytes=34603008 total_nvlink_bytes=568443792 weight_to_token_ratio=0.064819 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133959221e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74218 payload_rows_remote=64902 dup_rows=24086 dup_rows_remote=21054 dedup_ratio=0.245015 router_tokens_per_expert=[797, 761, 766, 765, 742, 734, 762, 801, 759, 752, 745, 768, 758, 804, 767, 785, 762, 721, 733, 717, 791, 759, 808, 727, 763, 754, 751, 820, 827, 774, 745, 761, 771, 797, 795, 731, 717, 772, 771, 747, 816, 747, 820, 782, 751, 742, 826, 729, 786, 764, 791, 771, 783, 755, 786, 738, 740, 742, 814, 749, 838, 779, 760, 763] payload_rows_per_dst=[9250, 9294, 9158, 9342, 9236, 9358, 9316, 9264] dup_rows_per_dst=[3006, 2982, 2878, 3048, 2966, 3068, 3032, 3106] combine_rows_per_src=[9250, 9294, 9158, 9342, 9236, 9358, 9316, 9264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266280960 dispatch_meta_bytes=343712 combine_payload_bytes=266280960 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266624672 ingress_bytes=335486976 link_bytes=335486976 token_bytes=532905632 weight_movement_bytes=69206016 total_nvlink_bytes=602111648 weight_to_token_ratio=0.129865 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262410482e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266125312 dispatch_meta_bytes=344016 combine_payload_bytes=266125312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266469328 ingress_bytes=300728320 link_bytes=300728320 token_bytes=532594640 weight_movement_bytes=34603008 total_nvlink_bytes=567197648 weight_to_token_ratio=0.064971 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131616458e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267485184 dispatch_meta_bytes=344344 combine_payload_bytes=267485184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267829528 ingress_bytes=302088192 link_bytes=302088192 token_bytes=535314712 weight_movement_bytes=34603008 total_nvlink_bytes=569917720 weight_to_token_ratio=0.064640 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136733546e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74546 payload_rows_remote=65252 dup_rows=23758 dup_rows_remote=20776 dedup_ratio=0.241679 router_tokens_per_expert=[737, 732, 772, 743, 766, 771, 788, 739, 804, 859, 729, 763, 730, 780, 751, 749, 774, 772, 717, 818, 761, 765, 807, 834, 762, 753, 777, 796, 789, 755, 771, 780, 824, 703, 753, 731, 771, 751, 743, 813, 789, 763, 739, 755, 722, 769, 771, 759, 814, 793, 771, 725, 759, 737, 752, 787, 771, 793, 773, 755, 763, 839, 761, 759] payload_rows_per_dst=[9194, 9318, 9464, 9340, 9230, 9246, 9294, 9460] dup_rows_per_dst=[2902, 3012, 3032, 3026, 2948, 2888, 2982, 2968] combine_rows_per_src=[9194, 9318, 9464, 9340, 9230, 9246, 9294, 9460] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74194 payload_rows_remote=64636 dup_rows=24110 dup_rows_remote=20830 dedup_ratio=0.245260 router_tokens_per_expert=[759, 758, 784, 738, 756, 772, 758, 813, 757, 743, 797, 740, 809, 767, 731, 794, 774, 749, 756, 789, 780, 763, 774, 747, 799, 723, 782, 794, 791, 782, 766, 769, 776, 777, 784, 773, 736, 800, 777, 777, 790, 811, 809, 768, 709, 788, 784, 746, 717, 761, 790, 733, 734, 812, 795, 712, 772, 735, 789, 694, 765, 769, 804, 751] payload_rows_per_dst=[9558, 9270, 8900, 9332, 9454, 9390, 9204, 9086] dup_rows_per_dst=[3280, 3054, 2810, 3114, 3146, 3012, 2904, 2790] combine_rows_per_src=[9558, 9270, 8900, 9332, 9454, 9390, 9204, 9086] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=65082 dup_rows=23964 dup_rows_remote=21172 dedup_ratio=0.243774 router_tokens_per_expert=[766, 772, 739, 795, 801, 792, 732, 787, 777, 733, 766, 769, 778, 780, 770, 781, 799, 775, 775, 739, 724, 774, 754, 760, 751, 710, 778, 799, 753, 752, 782, 775, 758, 776, 769, 758, 794, 781, 792, 782, 753, 786, 783, 771, 729, 696, 768, 739, 788, 784, 792, 757, 798, 778, 758, 749, 782, 766, 716, 783, 770, 789, 789, 780] payload_rows_per_dst=[9346, 9292, 9184, 9196, 9404, 9258, 9376, 9284] dup_rows_per_dst=[3022, 3016, 3016, 3004, 3016, 2792, 3032, 3066] combine_rows_per_src=[9346, 9292, 9184, 9196, 9404, 9258, 9376, 9284] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267272192 dispatch_meta_bytes=344112 combine_payload_bytes=267272192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267616304 ingress_bytes=301875200 link_bytes=301875200 token_bytes=534888496 weight_movement_bytes=34603008 total_nvlink_bytes=569491504 weight_to_token_ratio=0.064692 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135932075e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74170 payload_rows_remote=64990 dup_rows=24134 dup_rows_remote=21234 dedup_ratio=0.245504 router_tokens_per_expert=[771, 785, 793, 746, 741, 744, 767, 799, 763, 803, 788, 754, 737, 737, 756, 716, 782, 755, 709, 799, 789, 742, 773, 798, 736, 777, 760, 772, 769, 755, 781, 729, 805, 807, 731, 776, 753, 806, 746, 763, 754, 794, 768, 785, 767, 749, 812, 833, 784, 773, 797, 788, 771, 743, 823, 758, 741, 712, 769, 778, 802, 728, 769, 741] payload_rows_per_dst=[9266, 9108, 9280, 9352, 9300, 9346, 9338, 9180] dup_rows_per_dst=[2770, 3000, 3014, 3062, 3074, 3178, 3136, 2900] combine_rows_per_src=[9266, 9108, 9280, 9352, 9300, 9346, 9338, 9180] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74346 payload_rows_remote=64978 dup_rows=23958 dup_rows_remote=21066 dedup_ratio=0.243713 router_tokens_per_expert=[786, 737, 776, 778, 793, 771, 793, 791, 816, 774, 763, 735, 802, 768, 741, 778, 746, 748, 746, 764, 788, 775, 767, 731, 746, 709, 773, 786, 787, 790, 756, 783, 720, 782, 780, 787, 756, 775, 773, 768, 783, 754, 797, 749, 669, 774, 724, 780, 759, 759, 750, 811, 780, 780, 776, 781, 783, 800, 760, 731, 777, 764, 774, 799] payload_rows_per_dst=[9456, 9240, 9220, 9368, 9250, 9150, 9308, 9354] dup_rows_per_dst=[2994, 3114, 2910, 2892, 3032, 2910, 3084, 3022] combine_rows_per_src=[9456, 9240, 9220, 9368, 9250, 9150, 9308, 9354] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265478144 dispatch_meta_bytes=344256 combine_payload_bytes=265478144 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265822400 ingress_bytes=300081152 link_bytes=300081152 token_bytes=531300544 weight_movement_bytes=34603008 total_nvlink_bytes=565903552 weight_to_token_ratio=0.065129 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129181216e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=264749056 dispatch_meta_bytes=341864 combine_payload_bytes=264749056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265090920 ingress_bytes=299352064 link_bytes=299352064 token_bytes=529839976 weight_movement_bytes=34603008 total_nvlink_bytes=564442984 weight_to_token_ratio=0.065308 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126437717e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74310 payload_rows_remote=65036 dup_rows=23994 dup_rows_remote=21100 dedup_ratio=0.244080 router_tokens_per_expert=[793, 813, 788, 777, 756, 768, 789, 705, 757, 751, 746, 755, 724, 756, 782, 743, 780, 768, 802, 782, 830, 723, 738, 790, 780, 735, 748, 812, 776, 806, 759, 829, 755, 751, 756, 755, 793, 758, 776, 740, 749, 740, 802, 780, 776, 776, 787, 788, 790, 758, 754, 767, 772, 807, 753, 740, 761, 783, 709, 732, 736, 820, 767, 760] payload_rows_per_dst=[9336, 9122, 9378, 9484, 9274, 9232, 9298, 9186] dup_rows_per_dst=[3042, 2906, 3048, 3006, 2894, 3164, 2984, 2950] combine_rows_per_src=[9336, 9122, 9378, 9484, 9274, 9232, 9298, 9186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74146 payload_rows_remote=64920 dup_rows=24158 dup_rows_remote=21182 dedup_ratio=0.245748 router_tokens_per_expert=[760, 745, 777, 776, 780, 783, 739, 758, 742, 751, 769, 790, 773, 806, 740, 793, 745, 766, 787, 733, 813, 789, 713, 755, 774, 759, 801, 741, 769, 765, 727, 747, 822, 798, 763, 783, 812, 768, 772, 741, 733, 761, 785, 759, 768, 738, 776, 729, 764, 731, 803, 770, 804, 767, 770, 771, 799, 771, 795, 773, 740, 788, 760, 772] payload_rows_per_dst=[9174, 9344, 9226, 9204, 9402, 9126, 9298, 9372] dup_rows_per_dst=[3062, 2984, 2976, 2962, 3116, 2972, 3062, 3024] combine_rows_per_src=[9174, 9344, 9226, 9204, 9402, 9126, 9298, 9372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266575872 dispatch_meta_bytes=345016 combine_payload_bytes=266575872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266920888 ingress_bytes=301178880 link_bytes=301178880 token_bytes=533496760 weight_movement_bytes=34603008 total_nvlink_bytes=568099768 weight_to_token_ratio=0.064861 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133311879e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74262 payload_rows_remote=65008 dup_rows=24042 dup_rows_remote=20996 dedup_ratio=0.244568 router_tokens_per_expert=[763, 816, 783, 762, 732, 741, 737, 796, 742, 795, 785, 754, 729, 808, 754, 783, 770, 737, 795, 783, 749, 759, 797, 714, 751, 778, 745, 802, 763, 730, 787, 728, 747, 760, 743, 782, 809, 764, 726, 772, 825, 786, 749, 792, 722, 740, 782, 761, 750, 791, 760, 793, 749, 775, 780, 797, 782, 762, 815, 763, 813, 759, 748, 787] payload_rows_per_dst=[9310, 9254, 9250, 9162, 9296, 9302, 9310, 9378] dup_rows_per_dst=[2950, 3046, 2958, 3006, 2910, 3012, 3080, 3080] combine_rows_per_src=[9310, 9254, 9250, 9162, 9296, 9302, 9310, 9378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265838592 dispatch_meta_bytes=343824 combine_payload_bytes=265838592 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=266182416 ingress_bytes=438853632 link_bytes=438853632 token_bytes=532021008 weight_movement_bytes=173015040 total_nvlink_bytes=705036048 weight_to_token_ratio=0.325203 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.651370887e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74170 payload_rows_remote=64990 dup_rows=24134 dup_rows_remote=21234 dedup_ratio=0.245504 router_tokens_per_expert=[771, 785, 793, 746, 741, 744, 767, 799, 763, 803, 788, 754, 737, 737, 756, 716, 782, 755, 709, 799, 789, 742, 773, 798, 736, 777, 760, 772, 769, 755, 781, 729, 805, 807, 731, 776, 753, 806, 746, 763, 754, 794, 768, 785, 767, 749, 812, 833, 784, 773, 797, 788, 771, 743, 823, 758, 741, 712, 769, 778, 802, 728, 769, 741] payload_rows_per_dst=[9266, 9108, 9280, 9352, 9300, 9346, 9338, 9180] dup_rows_per_dst=[2770, 3000, 3014, 3062, 3074, 3178, 3136, 2900] combine_rows_per_src=[9266, 9108, 9280, 9352, 9300, 9346, 9338, 9180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=65082 dup_rows=23964 dup_rows_remote=21172 dedup_ratio=0.243774 router_tokens_per_expert=[766, 772, 739, 795, 801, 792, 732, 787, 777, 733, 766, 769, 778, 780, 770, 781, 799, 775, 775, 739, 724, 774, 754, 760, 751, 710, 778, 799, 753, 752, 782, 775, 758, 776, 769, 758, 794, 781, 792, 782, 753, 786, 783, 771, 729, 696, 768, 739, 788, 784, 792, 757, 798, 778, 758, 749, 782, 766, 716, 783, 770, 789, 789, 780] payload_rows_per_dst=[9346, 9292, 9184, 9196, 9404, 9258, 9376, 9284] dup_rows_per_dst=[3022, 3016, 3016, 3004, 3016, 2792, 3032, 3066] combine_rows_per_src=[9346, 9292, 9184, 9196, 9404, 9258, 9376, 9284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266272768 dispatch_meta_bytes=344016 combine_payload_bytes=266272768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266616784 ingress_bytes=300875776 link_bytes=300875776 token_bytes=532889552 weight_movement_bytes=34603008 total_nvlink_bytes=567492560 weight_to_token_ratio=0.064935 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132171323e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74346 payload_rows_remote=64978 dup_rows=23958 dup_rows_remote=21066 dedup_ratio=0.243713 router_tokens_per_expert=[786, 737, 776, 778, 793, 771, 793, 791, 816, 774, 763, 735, 802, 768, 741, 778, 746, 748, 746, 764, 788, 775, 767, 731, 746, 709, 773, 786, 787, 790, 756, 783, 720, 782, 780, 787, 756, 775, 773, 768, 783, 754, 797, 749, 669, 774, 724, 780, 759, 759, 750, 811, 780, 780, 776, 781, 783, 800, 760, 731, 777, 764, 774, 799] payload_rows_per_dst=[9456, 9240, 9220, 9368, 9250, 9150, 9308, 9354] dup_rows_per_dst=[2994, 3114, 2910, 2892, 3032, 2910, 3084, 3022] combine_rows_per_src=[9456, 9240, 9220, 9368, 9250, 9150, 9308, 9354] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74146 payload_rows_remote=64920 dup_rows=24158 dup_rows_remote=21182 dedup_ratio=0.245748 router_tokens_per_expert=[760, 745, 777, 776, 780, 783, 739, 758, 742, 751, 769, 790, 773, 806, 740, 793, 745, 766, 787, 733, 813, 789, 713, 755, 774, 759, 801, 741, 769, 765, 727, 747, 822, 798, 763, 783, 812, 768, 772, 741, 733, 761, 785, 759, 768, 738, 776, 729, 764, 731, 803, 770, 804, 767, 770, 771, 799, 771, 795, 773, 740, 788, 760, 772] payload_rows_per_dst=[9174, 9344, 9226, 9204, 9402, 9126, 9298, 9372] dup_rows_per_dst=[3062, 2984, 2976, 2962, 3116, 2972, 3062, 3024] combine_rows_per_src=[9174, 9344, 9226, 9204, 9402, 9126, 9298, 9372] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=344896 combine_payload_bytes=266199040 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266543936 ingress_bytes=335405056 link_bytes=335405056 token_bytes=532742976 weight_movement_bytes=69206016 total_nvlink_bytes=601948992 weight_to_token_ratio=0.129905 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262102223e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74260 payload_rows_remote=64972 dup_rows=24044 dup_rows_remote=21032 dedup_ratio=0.244588 router_tokens_per_expert=[717, 752, 736, 768, 769, 755, 723, 747, 761, 740, 752, 794, 768, 739, 770, 782, 784, 737, 758, 807, 802, 777, 789, 762, 751, 764, 738, 753, 765, 726, 802, 765, 745, 807, 749, 738, 796, 787, 768, 760, 750, 797, 782, 788, 754, 746, 807, 780, 726, 789, 750, 746, 778, 764, 778, 797, 822, 782, 789, 785, 756, 770, 795, 818] payload_rows_per_dst=[9070, 9194, 9344, 9190, 9288, 9402, 9348, 9424] dup_rows_per_dst=[2864, 3018, 3088, 2938, 3012, 3006, 2908, 3210] combine_rows_per_src=[9070, 9194, 9344, 9190, 9288, 9402, 9348, 9424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266575872 dispatch_meta_bytes=345016 combine_payload_bytes=266575872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266920888 ingress_bytes=301178880 link_bytes=301178880 token_bytes=533496760 weight_movement_bytes=34603008 total_nvlink_bytes=568099768 weight_to_token_ratio=0.064861 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133311879e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=64978 dup_rows=24056 dup_rows_remote=21006 dedup_ratio=0.244710 router_tokens_per_expert=[782, 761, 753, 754, 819, 761, 781, 744, 777, 753, 760, 750, 744, 765, 794, 792, 789, 792, 766, 747, 781, 696, 747, 785, 756, 770, 759, 745, 754, 808, 770, 746, 813, 771, 701, 712, 766, 767, 740, 761, 767, 781, 759, 795, 722, 758, 750, 812, 738, 838, 801, 794, 771, 795, 733, 769, 783, 778, 808, 769, 774, 790, 758, 777] payload_rows_per_dst=[9270, 9528, 9572, 9322, 9196, 9278, 8904, 9178] dup_rows_per_dst=[3050, 3074, 3236, 2894, 2938, 3158, 2746, 2960] combine_rows_per_src=[9270, 9528, 9572, 9322, 9196, 9278, 8904, 9178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74032 payload_rows_remote=64744 dup_rows=24272 dup_rows_remote=21280 dedup_ratio=0.246908 router_tokens_per_expert=[729, 783, 803, 750, 746, 766, 750, 778, 710, 782, 780, 771, 818, 768, 785, 756, 788, 819, 773, 798, 789, 790, 738, 755, 753, 793, 777, 786, 743, 757, 806, 777, 750, 761, 759, 742, 833, 767, 794, 742, 741, 760, 752, 746, 753, 764, 742, 754, 776, 803, 740, 720, 720, 758, 843, 780, 753, 800, 783, 732, 750, 757, 768, 792] payload_rows_per_dst=[9208, 9214, 9342, 9256, 9330, 9164, 9288, 9230] dup_rows_per_dst=[3002, 3126, 3158, 3128, 2966, 2860, 2992, 3040] combine_rows_per_src=[9208, 9214, 9342, 9256, 9330, 9164, 9288, 9230] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=344176 combine_payload_bytes=266149888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266494064 ingress_bytes=300752896 link_bytes=300752896 token_bytes=532643952 weight_movement_bytes=34603008 total_nvlink_bytes=567246960 weight_to_token_ratio=0.064965 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131708935e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266387456 dispatch_meta_bytes=344544 combine_payload_bytes=266387456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266732000 ingress_bytes=300990464 link_bytes=300990464 token_bytes=533119456 weight_movement_bytes=34603008 total_nvlink_bytes=567722464 weight_to_token_ratio=0.064907 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132602884e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265912320 dispatch_meta_bytes=344408 combine_payload_bytes=265912320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266256728 ingress_bytes=300515328 link_bytes=300515328 token_bytes=532169048 weight_movement_bytes=34603008 total_nvlink_bytes=566772056 weight_to_token_ratio=0.065023 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130814986e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266272768 dispatch_meta_bytes=344016 combine_payload_bytes=266272768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266616784 ingress_bytes=300875776 link_bytes=300875776 token_bytes=532889552 weight_movement_bytes=34603008 total_nvlink_bytes=567492560 weight_to_token_ratio=0.064935 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132171323e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74218 payload_rows_remote=64902 dup_rows=24086 dup_rows_remote=21054 dedup_ratio=0.245015 router_tokens_per_expert=[797, 761, 766, 765, 742, 734, 762, 801, 759, 752, 745, 768, 758, 804, 767, 785, 762, 721, 733, 717, 791, 759, 808, 727, 763, 754, 751, 820, 827, 774, 745, 761, 771, 797, 795, 731, 717, 772, 771, 747, 816, 747, 820, 782, 751, 742, 826, 729, 786, 764, 791, 771, 783, 755, 786, 738, 740, 742, 814, 749, 838, 779, 760, 763] payload_rows_per_dst=[9250, 9294, 9158, 9342, 9236, 9358, 9316, 9264] dup_rows_per_dst=[3006, 2982, 2878, 3048, 2966, 3068, 3032, 3106] combine_rows_per_src=[9250, 9294, 9158, 9342, 9236, 9358, 9316, 9264] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=64978 dup_rows=24056 dup_rows_remote=21006 dedup_ratio=0.244710 router_tokens_per_expert=[782, 761, 753, 754, 819, 761, 781, 744, 777, 753, 760, 750, 744, 765, 794, 792, 789, 792, 766, 747, 781, 696, 747, 785, 756, 770, 759, 745, 754, 808, 770, 746, 813, 771, 701, 712, 766, 767, 740, 761, 767, 781, 759, 795, 722, 758, 750, 812, 738, 838, 801, 794, 771, 795, 733, 769, 783, 778, 808, 769, 774, 790, 758, 777] payload_rows_per_dst=[9270, 9528, 9572, 9322, 9196, 9278, 8904, 9178] dup_rows_per_dst=[3050, 3074, 3236, 2894, 2938, 3158, 2746, 2960] combine_rows_per_src=[9270, 9528, 9572, 9322, 9196, 9278, 8904, 9178] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74482 payload_rows_remote=65156 dup_rows=23822 dup_rows_remote=20860 dedup_ratio=0.242330 router_tokens_per_expert=[799, 770, 738, 778, 788, 727, 771, 789, 749, 752, 761, 760, 783, 807, 763, 760, 774, 806, 766, 770, 743, 740, 772, 835, 782, 737, 763, 772, 750, 778, 785, 711, 731, 789, 783, 755, 739, 752, 785, 766, 736, 756, 789, 705, 774, 753, 840, 791, 784, 762, 786, 766, 776, 783, 762, 770, 752, 817, 758, 778, 748, 779, 752, 756] payload_rows_per_dst=[9374, 9294, 9428, 9184, 9228, 9326, 9348, 9300] dup_rows_per_dst=[2946, 2976, 2984, 2972, 2972, 2962, 3030, 2980] combine_rows_per_src=[9374, 9294, 9428, 9184, 9228, 9326, 9348, 9300] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265838592 dispatch_meta_bytes=343824 combine_payload_bytes=265838592 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=266182416 ingress_bytes=438853632 link_bytes=438853632 token_bytes=532021008 weight_movement_bytes=173015040 total_nvlink_bytes=705036048 weight_to_token_ratio=0.325203 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.651370887e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=64644 dup_rows=24098 dup_rows_remote=20916 dedup_ratio=0.245138 router_tokens_per_expert=[774, 779, 772, 812, 768, 803, 796, 786, 769, 744, 813, 776, 784, 746, 742, 773, 725, 781, 774, 797, 746, 786, 765, 790, 818, 767, 739, 718, 798, 765, 744, 774, 779, 746, 804, 801, 781, 758, 756, 742, 720, 788, 731, 773, 763, 782, 787, 776, 747, 740, 725, 760, 747, 737, 741, 708, 766, 744, 776, 778, 800, 794, 816, 762] payload_rows_per_dst=[9306, 9238, 8672, 9092, 9330, 9780, 9226, 9562] dup_rows_per_dst=[3090, 2882, 2536, 2852, 3090, 3580, 2886, 3182] combine_rows_per_src=[9306, 9238, 8672, 9092, 9330, 9780, 9226, 9562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266125312 dispatch_meta_bytes=344016 combine_payload_bytes=266125312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266469328 ingress_bytes=300728320 link_bytes=300728320 token_bytes=532594640 weight_movement_bytes=34603008 total_nvlink_bytes=567197648 weight_to_token_ratio=0.064971 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131616458e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74368 payload_rows_remote=65088 dup_rows=23936 dup_rows_remote=21000 dedup_ratio=0.243490 router_tokens_per_expert=[724, 780, 761, 783, 797, 746, 784, 754, 756, 798, 788, 724, 805, 775, 753, 709, 768, 758, 755, 724, 786, 738, 788, 774, 789, 754, 737, 804, 767, 742, 755, 761, 797, 743, 765, 803, 816, 766, 809, 756, 793, 741, 788, 690, 785, 773, 749, 752, 798, 759, 753, 741, 744, 815, 829, 773, 788, 803, 794, 786, 762, 765, 736, 743] payload_rows_per_dst=[9292, 9280, 9260, 9258, 9428, 9196, 9344, 9310] dup_rows_per_dst=[2966, 2936, 2922, 2960, 3082, 2946, 3080, 3044] combine_rows_per_src=[9292, 9280, 9260, 9258, 9428, 9196, 9344, 9310] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=343936 combine_payload_bytes=266149888 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266493824 ingress_bytes=335355904 link_bytes=335355904 token_bytes=532643712 weight_movement_bytes=69206016 total_nvlink_bytes=601849728 weight_to_token_ratio=0.129929 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261917268e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265191424 dispatch_meta_bytes=344096 combine_payload_bytes=265191424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265535520 ingress_bytes=299794432 link_bytes=299794432 token_bytes=530726944 weight_movement_bytes=34603008 total_nvlink_bytes=565329952 weight_to_token_ratio=0.065199 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128102312e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=64644 dup_rows=24098 dup_rows_remote=20916 dedup_ratio=0.245138 router_tokens_per_expert=[774, 779, 772, 812, 768, 803, 796, 786, 769, 744, 813, 776, 784, 746, 742, 773, 725, 781, 774, 797, 746, 786, 765, 790, 818, 767, 739, 718, 798, 765, 744, 774, 779, 746, 804, 801, 781, 758, 756, 742, 720, 788, 731, 773, 763, 782, 787, 776, 747, 740, 725, 760, 747, 737, 741, 708, 766, 744, 776, 778, 800, 794, 816, 762] payload_rows_per_dst=[9306, 9238, 8672, 9092, 9330, 9780, 9226, 9562] dup_rows_per_dst=[3090, 2882, 2536, 2852, 3090, 3580, 2886, 3182] combine_rows_per_src=[9306, 9238, 8672, 9092, 9330, 9780, 9226, 9562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=344352 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266944800 ingress_bytes=335806464 link_bytes=335806464 token_bytes=533545248 weight_movement_bytes=69206016 total_nvlink_bytes=602751264 weight_to_token_ratio=0.129710 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263612689e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74482 payload_rows_remote=65156 dup_rows=23822 dup_rows_remote=20860 dedup_ratio=0.242330 router_tokens_per_expert=[799, 770, 738, 778, 788, 727, 771, 789, 749, 752, 761, 760, 783, 807, 763, 760, 774, 806, 766, 770, 743, 740, 772, 835, 782, 737, 763, 772, 750, 778, 785, 711, 731, 789, 783, 755, 739, 752, 785, 766, 736, 756, 789, 705, 774, 753, 840, 791, 784, 762, 786, 766, 776, 783, 762, 770, 752, 817, 758, 778, 748, 779, 752, 756] payload_rows_per_dst=[9374, 9294, 9428, 9184, 9228, 9326, 9348, 9300] dup_rows_per_dst=[2946, 2976, 2984, 2972, 2972, 2962, 3030, 2980] combine_rows_per_src=[9374, 9294, 9428, 9184, 9228, 9326, 9348, 9300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74474 payload_rows_remote=65214 dup_rows=23830 dup_rows_remote=20824 dedup_ratio=0.242411 router_tokens_per_expert=[802, 764, 805, 765, 788, 757, 734, 743, 811, 775, 788, 765, 782, 758, 781, 747, 716, 757, 838, 783, 773, 744, 780, 772, 769, 793, 749, 749, 772, 768, 799, 734, 786, 813, 796, 799, 714, 758, 787, 793, 767, 709, 755, 797, 731, 752, 773, 700, 773, 780, 776, 759, 712, 756, 801, 781, 779, 776, 756, 800, 753, 751, 764, 744] payload_rows_per_dst=[9380, 9330, 9330, 9260, 9402, 9162, 9292, 9318] dup_rows_per_dst=[2936, 3084, 2996, 3006, 3090, 2806, 2984, 2928] combine_rows_per_src=[9380, 9330, 9330, 9260, 9402, 9162, 9292, 9318] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266878976 dispatch_meta_bytes=344064 combine_payload_bytes=266878976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267223040 ingress_bytes=301481984 link_bytes=301481984 token_bytes=534102016 weight_movement_bytes=34603008 total_nvlink_bytes=568705024 weight_to_token_ratio=0.064787 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134452435e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74152 payload_rows_remote=64964 dup_rows=24152 dup_rows_remote=21220 dedup_ratio=0.245687 router_tokens_per_expert=[747, 732, 765, 783, 808, 724, 775, 768, 774, 733, 800, 789, 794, 770, 782, 775, 746, 744, 798, 817, 803, 771, 795, 752, 799, 772, 764, 719, 798, 759, 747, 821, 732, 805, 739, 770, 782, 753, 748, 731, 779, 750, 743, 726, 751, 793, 734, 768, 734, 776, 742, 746, 778, 780, 790, 747, 703, 763, 806, 796, 806, 793, 793, 771] payload_rows_per_dst=[9182, 9362, 9338, 9380, 9188, 9208, 9214, 9280] dup_rows_per_dst=[3022, 3072, 3114, 2978, 2932, 2880, 2972, 3182] combine_rows_per_src=[9182, 9362, 9338, 9380, 9188, 9208, 9214, 9280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=264781824 dispatch_meta_bytes=342240 combine_payload_bytes=264781824 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265124064 ingress_bytes=333987840 link_bytes=333987840 token_bytes=529905888 weight_movement_bytes=69206016 total_nvlink_bytes=599111904 weight_to_token_ratio=0.130601 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256769354e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=65110 dup_rows=23974 dup_rows_remote=20996 dedup_ratio=0.243876 router_tokens_per_expert=[772, 769, 814, 745, 741, 743, 791, 740, 685, 777, 793, 800, 747, 807, 779, 820, 808, 745, 719, 767, 770, 757, 733, 800, 782, 741, 753, 800, 799, 778, 764, 805, 774, 783, 751, 783, 698, 727, 796, 770, 753, 765, 789, 750, 740, 759, 778, 753, 763, 745, 769, 778, 828, 799, 743, 776, 780, 770, 724, 839, 741, 740, 755, 789] payload_rows_per_dst=[9276, 9340, 9220, 9342, 9174, 9194, 9466, 9318] dup_rows_per_dst=[2954, 3076, 2978, 3102, 2990, 2980, 2936, 2958] combine_rows_per_src=[9276, 9340, 9220, 9342, 9174, 9194, 9466, 9318] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74368 payload_rows_remote=65088 dup_rows=23936 dup_rows_remote=21000 dedup_ratio=0.243490 router_tokens_per_expert=[724, 780, 761, 783, 797, 746, 784, 754, 756, 798, 788, 724, 805, 775, 753, 709, 768, 758, 755, 724, 786, 738, 788, 774, 789, 754, 737, 804, 767, 742, 755, 761, 797, 743, 765, 803, 816, 766, 809, 756, 793, 741, 788, 690, 785, 773, 749, 752, 798, 759, 753, 741, 744, 815, 829, 773, 788, 803, 794, 786, 762, 765, 736, 743] payload_rows_per_dst=[9292, 9280, 9260, 9258, 9428, 9196, 9344, 9310] dup_rows_per_dst=[2966, 2936, 2922, 2960, 3082, 2946, 3080, 3044] combine_rows_per_src=[9292, 9280, 9260, 9258, 9428, 9196, 9344, 9310] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74310 payload_rows_remote=65036 dup_rows=23994 dup_rows_remote=21100 dedup_ratio=0.244080 router_tokens_per_expert=[793, 813, 788, 777, 756, 768, 789, 705, 757, 751, 746, 755, 724, 756, 782, 743, 780, 768, 802, 782, 830, 723, 738, 790, 780, 735, 748, 812, 776, 806, 759, 829, 755, 751, 756, 755, 793, 758, 776, 740, 749, 740, 802, 780, 776, 776, 787, 788, 790, 758, 754, 767, 772, 807, 753, 740, 761, 783, 709, 732, 736, 820, 767, 760] payload_rows_per_dst=[9336, 9122, 9378, 9484, 9274, 9232, 9298, 9186] dup_rows_per_dst=[3042, 2906, 3048, 3006, 2894, 3164, 2984, 2950] combine_rows_per_src=[9336, 9122, 9378, 9484, 9274, 9232, 9298, 9186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266878976 dispatch_meta_bytes=344064 combine_payload_bytes=266878976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267223040 ingress_bytes=301481984 link_bytes=301481984 token_bytes=534102016 weight_movement_bytes=34603008 total_nvlink_bytes=568705024 weight_to_token_ratio=0.064787 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134452435e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74350 payload_rows_remote=65178 dup_rows=23954 dup_rows_remote=20972 dedup_ratio=0.243673 router_tokens_per_expert=[743, 800, 751, 760, 775, 734, 723, 755, 753, 803, 782, 818, 779, 786, 762, 791, 811, 763, 779, 809, 769, 790, 785, 745, 777, 752, 754, 783, 778, 801, 774, 735, 790, 817, 694, 780, 805, 760, 742, 771, 776, 772, 738, 748, 802, 764, 752, 747, 791, 727, 746, 807, 703, 754, 756, 793, 762, 763, 739, 772, 796, 769, 766, 730] payload_rows_per_dst=[9226, 9376, 9410, 9356, 9338, 9194, 9172, 9278] dup_rows_per_dst=[2856, 3172, 3092, 2952, 2980, 3004, 2982, 2916] combine_rows_per_src=[9226, 9376, 9410, 9356, 9338, 9194, 9172, 9278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74204 payload_rows_remote=64922 dup_rows=24100 dup_rows_remote=21012 dedup_ratio=0.245158 router_tokens_per_expert=[766, 792, 762, 772, 744, 783, 761, 787, 803, 772, 740, 742, 748, 787, 714, 741, 806, 771, 801, 777, 769, 758, 744, 743, 761, 842, 817, 795, 775, 760, 773, 762, 770, 764, 751, 768, 763, 777, 726, 732, 735, 774, 736, 844, 783, 788, 724, 746, 765, 811, 775, 788, 734, 765, 753, 732, 786, 797, 818, 739, 771, 750, 758, 761] payload_rows_per_dst=[9282, 9172, 9644, 9398, 9196, 9284, 9266, 8962] dup_rows_per_dst=[3088, 2922, 3382, 3136, 2906, 2976, 2980, 2710] combine_rows_per_src=[9282, 9172, 9644, 9398, 9196, 9284, 9266, 8962] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=264781824 dispatch_meta_bytes=342240 combine_payload_bytes=264781824 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265124064 ingress_bytes=333987840 link_bytes=333987840 token_bytes=529905888 weight_movement_bytes=69206016 total_nvlink_bytes=599111904 weight_to_token_ratio=0.130601 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256769354e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=267116544 dispatch_meta_bytes=344152 combine_payload_bytes=267116544 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267460696 ingress_bytes=336322560 link_bytes=336322560 token_bytes=534577240 weight_movement_bytes=69206016 total_nvlink_bytes=603783256 weight_to_token_ratio=0.129459 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265554717e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=344736 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266437280 ingress_bytes=300695552 link_bytes=300695552 token_bytes=532529824 weight_movement_bytes=34603008 total_nvlink_bytes=567132832 weight_to_token_ratio=0.064979 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131493154e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74032 payload_rows_remote=64744 dup_rows=24272 dup_rows_remote=21280 dedup_ratio=0.246908 router_tokens_per_expert=[729, 783, 803, 750, 746, 766, 750, 778, 710, 782, 780, 771, 818, 768, 785, 756, 788, 819, 773, 798, 789, 790, 738, 755, 753, 793, 777, 786, 743, 757, 806, 777, 750, 761, 759, 742, 833, 767, 794, 742, 741, 760, 752, 746, 753, 764, 742, 754, 776, 803, 740, 720, 720, 758, 843, 780, 753, 800, 783, 732, 750, 757, 768, 792] payload_rows_per_dst=[9208, 9214, 9342, 9256, 9330, 9164, 9288, 9230] dup_rows_per_dst=[3002, 3126, 3158, 3128, 2966, 2860, 2992, 3040] combine_rows_per_src=[9208, 9214, 9342, 9256, 9330, 9164, 9288, 9230] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266690560 dispatch_meta_bytes=344424 combine_payload_bytes=266690560 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267034984 ingress_bytes=335896576 link_bytes=335896576 token_bytes=533725544 weight_movement_bytes=69206016 total_nvlink_bytes=602931560 weight_to_token_ratio=0.129666 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263951774e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=344352 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266944800 ingress_bytes=335806464 link_bytes=335806464 token_bytes=533545248 weight_movement_bytes=69206016 total_nvlink_bytes=602751264 weight_to_token_ratio=0.129710 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263612689e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74204 payload_rows_remote=64922 dup_rows=24100 dup_rows_remote=21012 dedup_ratio=0.245158 router_tokens_per_expert=[766, 792, 762, 772, 744, 783, 761, 787, 803, 772, 740, 742, 748, 787, 714, 741, 806, 771, 801, 777, 769, 758, 744, 743, 761, 842, 817, 795, 775, 760, 773, 762, 770, 764, 751, 768, 763, 777, 726, 732, 735, 774, 736, 844, 783, 788, 724, 746, 765, 811, 775, 788, 734, 765, 753, 732, 786, 797, 818, 739, 771, 750, 758, 761] payload_rows_per_dst=[9282, 9172, 9644, 9398, 9196, 9284, 9266, 8962] dup_rows_per_dst=[3088, 2922, 3382, 3136, 2906, 2976, 2980, 2710] combine_rows_per_src=[9282, 9172, 9644, 9398, 9196, 9284, 9266, 8962] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74098 payload_rows_remote=64890 dup_rows=24206 dup_rows_remote=21172 dedup_ratio=0.246236 router_tokens_per_expert=[763, 769, 814, 770, 774, 753, 759, 755, 790, 766, 744, 746, 750, 781, 782, 776, 798, 769, 731, 754, 758, 747, 797, 757, 754, 779, 758, 762, 808, 771, 755, 811, 781, 771, 776, 761, 781, 781, 758, 770, 764, 806, 777, 774, 717, 782, 767, 734, 757, 773, 796, 780, 716, 773, 769, 780, 792, 760, 758, 753, 751, 739, 763, 791] payload_rows_per_dst=[9328, 9284, 9212, 9298, 9300, 9208, 9356, 9112] dup_rows_per_dst=[2986, 2986, 3010, 3098, 3058, 3034, 2932, 3102] combine_rows_per_src=[9328, 9284, 9212, 9298, 9300, 9208, 9356, 9112] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265191424 dispatch_meta_bytes=344096 combine_payload_bytes=265191424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265535520 ingress_bytes=299794432 link_bytes=299794432 token_bytes=530726944 weight_movement_bytes=34603008 total_nvlink_bytes=565329952 weight_to_token_ratio=0.065199 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128102312e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266690560 dispatch_meta_bytes=344424 combine_payload_bytes=266690560 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267034984 ingress_bytes=335896576 link_bytes=335896576 token_bytes=533725544 weight_movement_bytes=69206016 total_nvlink_bytes=602931560 weight_to_token_ratio=0.129666 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263951774e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266387456 dispatch_meta_bytes=344544 combine_payload_bytes=266387456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266732000 ingress_bytes=300990464 link_bytes=300990464 token_bytes=533119456 weight_movement_bytes=34603008 total_nvlink_bytes=567722464 weight_to_token_ratio=0.064907 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132602884e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266969088 dispatch_meta_bytes=344600 combine_payload_bytes=266969088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267313688 ingress_bytes=301572096 link_bytes=301572096 token_bytes=534282776 weight_movement_bytes=34603008 total_nvlink_bytes=568885784 weight_to_token_ratio=0.064765 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134791519e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74194 payload_rows_remote=64928 dup_rows=24110 dup_rows_remote=21108 dedup_ratio=0.245260 router_tokens_per_expert=[781, 788, 771, 768, 777, 756, 794, 792, 764, 741, 754, 741, 801, 812, 752, 769, 770, 776, 789, 753, 777, 761, 750, 761, 782, 752, 748, 711, 773, 744, 803, 723, 770, 726, 754, 787, 787, 800, 762, 783, 787, 757, 766, 823, 818, 791, 769, 750, 717, 749, 799, 758, 763, 740, 764, 761, 740, 775, 804, 766, 739, 774, 800, 739] payload_rows_per_dst=[9398, 9266, 9188, 9094, 9284, 9442, 9190, 9332] dup_rows_per_dst=[3056, 3002, 3086, 2978, 3054, 3080, 2912, 2942] combine_rows_per_src=[9398, 9266, 9188, 9094, 9284, 9442, 9190, 9332] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265920512 dispatch_meta_bytes=343736 combine_payload_bytes=265920512 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266264248 ingress_bytes=369729536 link_bytes=369729536 token_bytes=532184760 weight_movement_bytes=103809024 total_nvlink_bytes=635993784 weight_to_token_ratio=0.195062 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391262478e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265789440 dispatch_meta_bytes=344248 combine_payload_bytes=265789440 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266133688 ingress_bytes=334995456 link_bytes=334995456 token_bytes=531923128 weight_movement_bytes=69206016 total_nvlink_bytes=601129144 weight_to_token_ratio=0.130105 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260560932e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=64964 dup_rows=24012 dup_rows_remote=20950 dedup_ratio=0.244263 router_tokens_per_expert=[745, 773, 794, 765, 701, 765, 784, 741, 769, 737, 781, 773, 785, 754, 779, 786, 773, 746, 783, 729, 764, 768, 752, 750, 762, 781, 730, 797, 781, 752, 790, 758, 794, 745, 788, 797, 724, 763, 743, 798, 771, 811, 757, 792, 757, 784, 781, 727, 762, 806, 755, 764, 808, 763, 739, 788, 769, 801, 740, 789, 807, 801, 739, 741] payload_rows_per_dst=[9194, 8932, 9158, 9278, 9282, 9344, 9776, 9328] dup_rows_per_dst=[2942, 2648, 2972, 3024, 3022, 3000, 3342, 3062] combine_rows_per_src=[9194, 8932, 9158, 9278, 9282, 9344, 9776, 9328] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74432 payload_rows_remote=65048 dup_rows=23872 dup_rows_remote=20868 dedup_ratio=0.242839 router_tokens_per_expert=[815, 774, 789, 725, 747, 748, 755, 747, 725, 763, 777, 746, 780, 786, 736, 740, 781, 756, 717, 833, 797, 728, 765, 800, 770, 782, 799, 770, 789, 773, 749, 762, 769, 743, 770, 793, 734, 782, 764, 762, 821, 764, 783, 779, 759, 751, 719, 735, 778, 790, 806, 740, 790, 764, 758, 758, 767, 735, 770, 794, 780, 780, 775, 815] payload_rows_per_dst=[9222, 9132, 9402, 9384, 9234, 9226, 9364, 9468] dup_rows_per_dst=[2978, 2974, 2952, 3004, 3000, 2996, 3004, 2964] combine_rows_per_src=[9222, 9132, 9402, 9384, 9234, 9226, 9364, 9468] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74528 payload_rows_remote=65282 dup_rows=23776 dup_rows_remote=20934 dedup_ratio=0.241862 router_tokens_per_expert=[759, 764, 766, 773, 795, 728, 753, 815, 763, 760, 794, 740, 757, 777, 788, 755, 805, 799, 730, 771, 814, 756, 723, 792, 713, 775, 698, 805, 783, 783, 748, 736, 716, 780, 754, 709, 740, 764, 823, 758, 807, 819, 753, 798, 748, 773, 764, 798, 710, 799, 786, 707, 779, 807, 731, 779, 767, 788, 825, 744, 795, 739, 782, 792] payload_rows_per_dst=[9382, 9264, 9320, 9204, 9246, 9476, 9226, 9410] dup_rows_per_dst=[2924, 3004, 3060, 2878, 2842, 3044, 2970, 3054] combine_rows_per_src=[9382, 9264, 9320, 9204, 9246, 9476, 9226, 9410] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74446 payload_rows_remote=65134 dup_rows=23858 dup_rows_remote=21036 dedup_ratio=0.242696 router_tokens_per_expert=[762, 765, 755, 778, 816, 768, 777, 799, 791, 771, 771, 774, 726, 756, 793, 789, 784, 768, 746, 713, 733, 792, 747, 784, 774, 799, 758, 777, 774, 764, 782, 752, 747, 724, 783, 743, 784, 784, 808, 838, 753, 757, 758, 762, 769, 774, 763, 744, 772, 716, 763, 740, 754, 728, 761, 742, 774, 798, 815, 778, 814, 781, 730, 757] payload_rows_per_dst=[9322, 9370, 9312, 9304, 9334, 9280, 9142, 9382] dup_rows_per_dst=[3118, 2972, 2822, 3056, 3088, 2880, 2810, 3112] combine_rows_per_src=[9322, 9370, 9312, 9304, 9334, 9280, 9142, 9382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74194 payload_rows_remote=64928 dup_rows=24110 dup_rows_remote=21108 dedup_ratio=0.245260 router_tokens_per_expert=[781, 788, 771, 768, 777, 756, 794, 792, 764, 741, 754, 741, 801, 812, 752, 769, 770, 776, 789, 753, 777, 761, 750, 761, 782, 752, 748, 711, 773, 744, 803, 723, 770, 726, 754, 787, 787, 800, 762, 783, 787, 757, 766, 823, 818, 791, 769, 750, 717, 749, 799, 758, 763, 740, 764, 761, 740, 775, 804, 766, 739, 774, 800, 739] payload_rows_per_dst=[9398, 9266, 9188, 9094, 9284, 9442, 9190, 9332] dup_rows_per_dst=[3056, 3002, 3086, 2978, 3054, 3080, 2912, 2942] combine_rows_per_src=[9398, 9266, 9188, 9094, 9284, 9442, 9190, 9332] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74098 payload_rows_remote=64890 dup_rows=24206 dup_rows_remote=21172 dedup_ratio=0.246236 router_tokens_per_expert=[763, 769, 814, 770, 774, 753, 759, 755, 790, 766, 744, 746, 750, 781, 782, 776, 798, 769, 731, 754, 758, 747, 797, 757, 754, 779, 758, 762, 808, 771, 755, 811, 781, 771, 776, 761, 781, 781, 758, 770, 764, 806, 777, 774, 717, 782, 767, 734, 757, 773, 796, 780, 716, 773, 769, 780, 792, 760, 758, 753, 751, 739, 763, 791] payload_rows_per_dst=[9328, 9284, 9212, 9298, 9300, 9208, 9356, 9112] dup_rows_per_dst=[2986, 2986, 3010, 3098, 3058, 3034, 2932, 3102] combine_rows_per_src=[9328, 9284, 9212, 9298, 9300, 9208, 9356, 9112] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265945088 dispatch_meta_bytes=344144 combine_payload_bytes=265945088 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266289232 ingress_bytes=335151104 link_bytes=335151104 token_bytes=532234320 weight_movement_bytes=69206016 total_nvlink_bytes=601440336 weight_to_token_ratio=0.130029 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261146622e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266436608 dispatch_meta_bytes=343664 combine_payload_bytes=266436608 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266780272 ingress_bytes=335642624 link_bytes=335642624 token_bytes=533216880 weight_movement_bytes=69206016 total_nvlink_bytes=602422896 weight_to_token_ratio=0.129790 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262996173e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74432 payload_rows_remote=65048 dup_rows=23872 dup_rows_remote=20868 dedup_ratio=0.242839 router_tokens_per_expert=[815, 774, 789, 725, 747, 748, 755, 747, 725, 763, 777, 746, 780, 786, 736, 740, 781, 756, 717, 833, 797, 728, 765, 800, 770, 782, 799, 770, 789, 773, 749, 762, 769, 743, 770, 793, 734, 782, 764, 762, 821, 764, 783, 779, 759, 751, 719, 735, 778, 790, 806, 740, 790, 764, 758, 758, 767, 735, 770, 794, 780, 780, 775, 815] payload_rows_per_dst=[9222, 9132, 9402, 9384, 9234, 9226, 9364, 9468] dup_rows_per_dst=[2978, 2974, 2952, 3004, 3000, 2996, 3004, 2964] combine_rows_per_src=[9222, 9132, 9402, 9384, 9234, 9226, 9364, 9468] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=343656 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266436200 ingress_bytes=369901568 link_bytes=369901568 token_bytes=532528744 weight_movement_bytes=103809024 total_nvlink_bytes=636337768 weight_to_token_ratio=0.194936 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391909821e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=267395072 dispatch_meta_bytes=344864 combine_payload_bytes=267395072 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267739936 ingress_bytes=336601088 link_bytes=336601088 token_bytes=535135008 weight_movement_bytes=69206016 total_nvlink_bytes=604341024 weight_to_token_ratio=0.129324 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266602796e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=344680 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267133544 ingress_bytes=301391872 link_bytes=301391872 token_bytes=533922408 weight_movement_bytes=34603008 total_nvlink_bytes=568525416 weight_to_token_ratio=0.064809 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134113350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74350 payload_rows_remote=65178 dup_rows=23954 dup_rows_remote=20972 dedup_ratio=0.243673 router_tokens_per_expert=[743, 800, 751, 760, 775, 734, 723, 755, 753, 803, 782, 818, 779, 786, 762, 791, 811, 763, 779, 809, 769, 790, 785, 745, 777, 752, 754, 783, 778, 801, 774, 735, 790, 817, 694, 780, 805, 760, 742, 771, 776, 772, 738, 748, 802, 764, 752, 747, 791, 727, 746, 807, 703, 754, 756, 793, 762, 763, 739, 772, 796, 769, 766, 730] payload_rows_per_dst=[9226, 9376, 9410, 9356, 9338, 9194, 9172, 9278] dup_rows_per_dst=[2856, 3172, 3092, 2952, 2980, 3004, 2982, 2916] combine_rows_per_src=[9226, 9376, 9410, 9356, 9338, 9194, 9172, 9278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265789440 dispatch_meta_bytes=344248 combine_payload_bytes=265789440 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266133688 ingress_bytes=334995456 link_bytes=334995456 token_bytes=531923128 weight_movement_bytes=69206016 total_nvlink_bytes=601129144 weight_to_token_ratio=0.130105 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260560932e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74446 payload_rows_remote=65134 dup_rows=23858 dup_rows_remote=21036 dedup_ratio=0.242696 router_tokens_per_expert=[762, 765, 755, 778, 816, 768, 777, 799, 791, 771, 771, 774, 726, 756, 793, 789, 784, 768, 746, 713, 733, 792, 747, 784, 774, 799, 758, 777, 774, 764, 782, 752, 747, 724, 783, 743, 784, 784, 808, 838, 753, 757, 758, 762, 769, 774, 763, 744, 772, 716, 763, 740, 754, 728, 761, 742, 774, 798, 815, 778, 814, 781, 730, 757] payload_rows_per_dst=[9322, 9370, 9312, 9304, 9334, 9280, 9142, 9382] dup_rows_per_dst=[3118, 2972, 2822, 3056, 3088, 2880, 2810, 3112] combine_rows_per_src=[9322, 9370, 9312, 9304, 9334, 9280, 9142, 9382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74152 payload_rows_remote=64964 dup_rows=24152 dup_rows_remote=21220 dedup_ratio=0.245687 router_tokens_per_expert=[747, 732, 765, 783, 808, 724, 775, 768, 774, 733, 800, 789, 794, 770, 782, 775, 746, 744, 798, 817, 803, 771, 795, 752, 799, 772, 764, 719, 798, 759, 747, 821, 732, 805, 739, 770, 782, 753, 748, 731, 779, 750, 743, 726, 751, 793, 734, 768, 734, 776, 742, 746, 778, 780, 790, 747, 703, 763, 806, 796, 806, 793, 793, 771] payload_rows_per_dst=[9182, 9362, 9338, 9380, 9188, 9208, 9214, 9280] dup_rows_per_dst=[3022, 3072, 3114, 2978, 2932, 2880, 2972, 3182] combine_rows_per_src=[9182, 9362, 9338, 9380, 9188, 9208, 9214, 9280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=65006 dup_rows=23964 dup_rows_remote=20990 dedup_ratio=0.243774 router_tokens_per_expert=[744, 801, 749, 778, 790, 755, 734, 741, 795, 757, 803, 806, 762, 760, 775, 741, 788, 782, 761, 772, 799, 731, 778, 715, 768, 768, 803, 792, 804, 786, 767, 714, 757, 781, 747, 745, 749, 767, 810, 775, 802, 756, 782, 763, 758, 823, 772, 787, 757, 776, 743, 784, 767, 807, 748, 772, 805, 729, 738, 778, 714, 744, 760, 737] payload_rows_per_dst=[9248, 9340, 9302, 9338, 9316, 9332, 9334, 9130] dup_rows_per_dst=[2936, 3058, 2950, 3066, 2946, 3154, 2974, 2880] combine_rows_per_src=[9248, 9340, 9302, 9338, 9316, 9332, 9334, 9130] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=64986 dup_rows=24062 dup_rows_remote=20946 dedup_ratio=0.244771 router_tokens_per_expert=[761, 779, 727, 778, 748, 769, 745, 806, 758, 772, 824, 792, 753, 750, 839, 806, 770, 745, 745, 726, 734, 744, 804, 774, 774, 774, 819, 776, 748, 804, 772, 770, 769, 798, 751, 765, 772, 787, 801, 779, 757, 750, 794, 750, 799, 741, 762, 706, 770, 800, 734, 756, 748, 754, 751, 750, 777, 776, 771, 781, 729, 791, 746, 751] payload_rows_per_dst=[9256, 9556, 9306, 9300, 9302, 9228, 9296, 8998] dup_rows_per_dst=[3116, 3366, 3002, 3096, 2940, 2868, 2830, 2844] combine_rows_per_src=[9256, 9556, 9306, 9300, 9302, 9228, 9296, 8998] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265945088 dispatch_meta_bytes=344144 combine_payload_bytes=265945088 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266289232 ingress_bytes=335151104 link_bytes=335151104 token_bytes=532234320 weight_movement_bytes=69206016 total_nvlink_bytes=601440336 weight_to_token_ratio=0.130029 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261146622e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74440 payload_rows_remote=65134 dup_rows=23864 dup_rows_remote=20856 dedup_ratio=0.242757 router_tokens_per_expert=[789, 722, 780, 773, 743, 767, 750, 754, 801, 736, 750, 747, 766, 726, 768, 793, 807, 769, 744, 749, 717, 804, 761, 771, 786, 724, 759, 772, 742, 793, 739, 781, 792, 750, 779, 793, 778, 773, 787, 800, 816, 752, 779, 783, 747, 782, 759, 739, 792, 746, 767, 711, 800, 783, 781, 787, 785, 779, 784, 756, 793, 782, 761, 753] payload_rows_per_dst=[9246, 9194, 9190, 9332, 9422, 9306, 9366, 9384] dup_rows_per_dst=[2910, 2980, 3054, 2860, 3082, 3008, 2968, 3002] combine_rows_per_src=[9246, 9194, 9190, 9332, 9422, 9306, 9366, 9384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266436608 dispatch_meta_bytes=343664 combine_payload_bytes=266436608 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266780272 ingress_bytes=335642624 link_bytes=335642624 token_bytes=533216880 weight_movement_bytes=69206016 total_nvlink_bytes=602422896 weight_to_token_ratio=0.129790 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262996173e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266969088 dispatch_meta_bytes=344600 combine_payload_bytes=266969088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267313688 ingress_bytes=301572096 link_bytes=301572096 token_bytes=534282776 weight_movement_bytes=34603008 total_nvlink_bytes=568885784 weight_to_token_ratio=0.064765 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134791519e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=344680 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267133544 ingress_bytes=301391872 link_bytes=301391872 token_bytes=533922408 weight_movement_bytes=34603008 total_nvlink_bytes=568525416 weight_to_token_ratio=0.064809 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134113350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=344736 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266437280 ingress_bytes=300695552 link_bytes=300695552 token_bytes=532529824 weight_movement_bytes=34603008 total_nvlink_bytes=567132832 weight_to_token_ratio=0.064979 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131493154e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74130 payload_rows_remote=64866 dup_rows=24174 dup_rows_remote=21144 dedup_ratio=0.245911 router_tokens_per_expert=[796, 763, 753, 729, 766, 733, 776, 746, 740, 798, 805, 764, 772, 782, 754, 732, 774, 775, 759, 719, 762, 741, 811, 748, 833, 759, 747, 752, 782, 767, 773, 777, 808, 753, 790, 768, 736, 794, 792, 752, 738, 763, 800, 729, 764, 788, 763, 749, 842, 733, 773, 777, 801, 792, 771, 757, 787, 761, 792, 729, 764, 756, 784, 758] payload_rows_per_dst=[9212, 9264, 9228, 9228, 9348, 9186, 9406, 9258] dup_rows_per_dst=[2912, 3030, 2950, 3152, 3038, 3002, 3086, 3004] combine_rows_per_src=[9212, 9264, 9228, 9228, 9348, 9186, 9406, 9258] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266264576 dispatch_meta_bytes=343984 combine_payload_bytes=266264576 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266608560 ingress_bytes=335470592 link_bytes=335470592 token_bytes=532873136 weight_movement_bytes=69206016 total_nvlink_bytes=602079152 weight_to_token_ratio=0.129873 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262348830e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266182656 dispatch_meta_bytes=343728 combine_payload_bytes=266182656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266526384 ingress_bytes=300785664 link_bytes=300785664 token_bytes=532709040 weight_movement_bytes=34603008 total_nvlink_bytes=567312048 weight_to_token_ratio=0.064957 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131832238e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=343960 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267132824 ingress_bytes=301391872 link_bytes=301391872 token_bytes=533921688 weight_movement_bytes=34603008 total_nvlink_bytes=568524696 weight_to_token_ratio=0.064809 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134113350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74112 payload_rows_remote=64842 dup_rows=24192 dup_rows_remote=21222 dedup_ratio=0.246094 router_tokens_per_expert=[798, 739, 760, 745, 740, 777, 794, 748, 741, 792, 774, 792, 793, 798, 780, 798, 781, 791, 810, 782, 800, 719, 750, 806, 783, 739, 737, 796, 770, 747, 794, 754, 771, 786, 754, 751, 772, 735, 755, 758, 722, 727, 758, 742, 739, 767, 781, 759, 797, 786, 789, 782, 777, 810, 753, 777, 793, 754, 769, 774, 781, 737, 729, 739] payload_rows_per_dst=[9310, 9408, 9264, 9270, 9156, 9182, 9422, 9100] dup_rows_per_dst=[2892, 3128, 3214, 2970, 3008, 2808, 3120, 3052] combine_rows_per_src=[9310, 9408, 9264, 9270, 9156, 9182, 9422, 9100] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74230 payload_rows_remote=65024 dup_rows=24074 dup_rows_remote=21056 dedup_ratio=0.244893 router_tokens_per_expert=[774, 767, 789, 755, 779, 765, 710, 764, 724, 824, 777, 716, 753, 760, 784, 754, 796, 786, 765, 800, 776, 846, 786, 821, 755, 737, 731, 728, 795, 775, 736, 773, 744, 755, 803, 776, 752, 757, 767, 758, 741, 772, 751, 752, 797, 780, 797, 725, 753, 767, 784, 753, 747, 831, 775, 770, 788, 795, 772, 784, 740, 750, 769, 746] payload_rows_per_dst=[9262, 9264, 9546, 9092, 9206, 9306, 9300, 9254] dup_rows_per_dst=[2944, 2920, 3206, 2968, 3018, 2924, 3060, 3034] combine_rows_per_src=[9262, 9264, 9546, 9092, 9206, 9306, 9300, 9254] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74300 payload_rows_remote=64926 dup_rows=24004 dup_rows_remote=20770 dedup_ratio=0.244181 router_tokens_per_expert=[764, 799, 791, 759, 800, 776, 735, 756, 783, 758, 784, 805, 830, 735, 771, 757, 741, 773, 763, 741, 711, 764, 782, 774, 808, 774, 770, 791, 730, 767, 756, 782, 761, 789, 739, 745, 763, 729, 784, 730, 767, 778, 752, 784, 738, 776, 765, 801, 779, 727, 805, 820, 757, 771, 758, 763, 793, 719, 793, 711, 822, 763, 750, 790] payload_rows_per_dst=[9378, 9186, 9070, 9146, 9412, 9438, 9296, 9374] dup_rows_per_dst=[2982, 2980, 2772, 2820, 3058, 3094, 3064, 3234] combine_rows_per_src=[9378, 9186, 9070, 9146, 9412, 9438, 9296, 9374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74382 payload_rows_remote=65048 dup_rows=23922 dup_rows_remote=21008 dedup_ratio=0.243347 router_tokens_per_expert=[795, 734, 755, 782, 725, 776, 750, 790, 772, 777, 773, 790, 742, 727, 765, 756, 819, 733, 775, 780, 752, 739, 745, 781, 772, 791, 745, 803, 727, 810, 770, 815, 713, 739, 727, 772, 756, 805, 736, 734, 776, 788, 770, 762, 778, 739, 805, 802, 728, 798, 772, 770, 801, 759, 826, 775, 724, 796, 807, 785, 730, 751, 788, 774] payload_rows_per_dst=[9232, 9326, 9334, 9356, 9098, 9320, 9382, 9334] dup_rows_per_dst=[2982, 2878, 2914, 3110, 2866, 3120, 3076, 2976] combine_rows_per_src=[9232, 9326, 9334, 9356, 9098, 9320, 9382, 9334] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74130 payload_rows_remote=64866 dup_rows=24174 dup_rows_remote=21144 dedup_ratio=0.245911 router_tokens_per_expert=[796, 763, 753, 729, 766, 733, 776, 746, 740, 798, 805, 764, 772, 782, 754, 732, 774, 775, 759, 719, 762, 741, 811, 748, 833, 759, 747, 752, 782, 767, 773, 777, 808, 753, 790, 768, 736, 794, 792, 752, 738, 763, 800, 729, 764, 788, 763, 749, 842, 733, 773, 777, 801, 792, 771, 757, 787, 761, 792, 729, 764, 756, 784, 758] payload_rows_per_dst=[9212, 9264, 9228, 9228, 9348, 9186, 9406, 9258] dup_rows_per_dst=[2912, 3030, 2950, 3152, 3038, 3002, 3086, 3004] combine_rows_per_src=[9212, 9264, 9228, 9228, 9348, 9186, 9406, 9258] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74440 payload_rows_remote=65134 dup_rows=23864 dup_rows_remote=20856 dedup_ratio=0.242757 router_tokens_per_expert=[789, 722, 780, 773, 743, 767, 750, 754, 801, 736, 750, 747, 766, 726, 768, 793, 807, 769, 744, 749, 717, 804, 761, 771, 786, 724, 759, 772, 742, 793, 739, 781, 792, 750, 779, 793, 778, 773, 787, 800, 816, 752, 779, 783, 747, 782, 759, 739, 792, 746, 767, 711, 800, 783, 781, 787, 785, 779, 784, 756, 793, 782, 761, 753] payload_rows_per_dst=[9246, 9194, 9190, 9332, 9422, 9306, 9366, 9384] dup_rows_per_dst=[2910, 2980, 3054, 2860, 3082, 3008, 2968, 3002] combine_rows_per_src=[9246, 9194, 9190, 9332, 9422, 9306, 9366, 9384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265691136 dispatch_meta_bytes=344040 combine_payload_bytes=265691136 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266035176 ingress_bytes=369500160 link_bytes=369500160 token_bytes=531726312 weight_movement_bytes=103809024 total_nvlink_bytes=635535336 weight_to_token_ratio=0.195230 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390399355e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265592832 dispatch_meta_bytes=344256 combine_payload_bytes=265592832 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=265937088 ingress_bytes=300195840 link_bytes=300195840 token_bytes=531529920 weight_movement_bytes=34603008 total_nvlink_bytes=566132928 weight_to_token_ratio=0.065101 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129612778e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=344320 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266682624 ingress_bytes=300941312 link_bytes=300941312 token_bytes=533020928 weight_movement_bytes=34603008 total_nvlink_bytes=567623936 weight_to_token_ratio=0.064919 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132417929e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74112 payload_rows_remote=64842 dup_rows=24192 dup_rows_remote=21222 dedup_ratio=0.246094 router_tokens_per_expert=[798, 739, 760, 745, 740, 777, 794, 748, 741, 792, 774, 792, 793, 798, 780, 798, 781, 791, 810, 782, 800, 719, 750, 806, 783, 739, 737, 796, 770, 747, 794, 754, 771, 786, 754, 751, 772, 735, 755, 758, 722, 727, 758, 742, 739, 767, 781, 759, 797, 786, 789, 782, 777, 810, 753, 777, 793, 754, 769, 774, 781, 737, 729, 739] payload_rows_per_dst=[9310, 9408, 9264, 9270, 9156, 9182, 9422, 9100] dup_rows_per_dst=[2892, 3128, 3214, 2970, 3008, 2808, 3120, 3052] combine_rows_per_src=[9310, 9408, 9264, 9270, 9156, 9182, 9422, 9100] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265936896 dispatch_meta_bytes=342784 combine_payload_bytes=265936896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266279680 ingress_bytes=300539904 link_bytes=300539904 token_bytes=532216576 weight_movement_bytes=34603008 total_nvlink_bytes=566819584 weight_to_token_ratio=0.065017 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130907463e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=65006 dup_rows=23964 dup_rows_remote=20990 dedup_ratio=0.243774 router_tokens_per_expert=[744, 801, 749, 778, 790, 755, 734, 741, 795, 757, 803, 806, 762, 760, 775, 741, 788, 782, 761, 772, 799, 731, 778, 715, 768, 768, 803, 792, 804, 786, 767, 714, 757, 781, 747, 745, 749, 767, 810, 775, 802, 756, 782, 763, 758, 823, 772, 787, 757, 776, 743, 784, 767, 807, 748, 772, 805, 729, 738, 778, 714, 744, 760, 737] payload_rows_per_dst=[9248, 9340, 9302, 9338, 9316, 9332, 9334, 9130] dup_rows_per_dst=[2936, 3058, 2950, 3066, 2946, 3154, 2974, 2880] combine_rows_per_src=[9248, 9340, 9302, 9338, 9316, 9332, 9334, 9130] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266436608 dispatch_meta_bytes=344224 combine_payload_bytes=266436608 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266780832 ingress_bytes=335642624 link_bytes=335642624 token_bytes=533217440 weight_movement_bytes=69206016 total_nvlink_bytes=602423456 weight_to_token_ratio=0.129789 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262996173e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74382 payload_rows_remote=65048 dup_rows=23922 dup_rows_remote=21008 dedup_ratio=0.243347 router_tokens_per_expert=[795, 734, 755, 782, 725, 776, 750, 790, 772, 777, 773, 790, 742, 727, 765, 756, 819, 733, 775, 780, 752, 739, 745, 781, 772, 791, 745, 803, 727, 810, 770, 815, 713, 739, 727, 772, 756, 805, 736, 734, 776, 788, 770, 762, 778, 739, 805, 802, 728, 798, 772, 770, 801, 759, 826, 775, 724, 796, 807, 785, 730, 751, 788, 774] payload_rows_per_dst=[9232, 9326, 9334, 9356, 9098, 9320, 9382, 9334] dup_rows_per_dst=[2982, 2878, 2914, 3110, 2866, 3120, 3076, 2976] combine_rows_per_src=[9232, 9326, 9334, 9356, 9098, 9320, 9382, 9334] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74528 payload_rows_remote=65282 dup_rows=23776 dup_rows_remote=20934 dedup_ratio=0.241862 router_tokens_per_expert=[759, 764, 766, 773, 795, 728, 753, 815, 763, 760, 794, 740, 757, 777, 788, 755, 805, 799, 730, 771, 814, 756, 723, 792, 713, 775, 698, 805, 783, 783, 748, 736, 716, 780, 754, 709, 740, 764, 823, 758, 807, 819, 753, 798, 748, 773, 764, 798, 710, 799, 786, 707, 779, 807, 731, 779, 767, 788, 825, 744, 795, 739, 782, 792] payload_rows_per_dst=[9382, 9264, 9320, 9204, 9246, 9476, 9226, 9410] dup_rows_per_dst=[2924, 3004, 3060, 2878, 2842, 3044, 2970, 3054] combine_rows_per_src=[9382, 9264, 9320, 9204, 9246, 9476, 9226, 9410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73978 payload_rows_remote=64976 dup_rows=24326 dup_rows_remote=21368 dedup_ratio=0.247457 router_tokens_per_expert=[755, 758, 752, 758, 735, 756, 783, 801, 780, 757, 762, 786, 748, 743, 792, 795, 778, 816, 774, 805, 727, 776, 774, 783, 741, 741, 751, 829, 745, 749, 793, 763, 761, 777, 801, 759, 795, 774, 824, 765, 797, 763, 758, 743, 817, 769, 804, 767, 720, 755, 754, 774, 730, 761, 764, 722, 751, 754, 806, 772, 727, 761, 761, 760] payload_rows_per_dst=[9226, 9250, 9324, 9220, 9430, 9344, 9002, 9182] dup_rows_per_dst=[2970, 3076, 3142, 3004, 3082, 3092, 2958, 3002] combine_rows_per_src=[9226, 9250, 9324, 9220, 9430, 9344, 9002, 9182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=343960 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267132824 ingress_bytes=301391872 link_bytes=301391872 token_bytes=533921688 weight_movement_bytes=34603008 total_nvlink_bytes=568524696 weight_to_token_ratio=0.064809 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134113350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64642 dup_rows=24114 dup_rows_remote=20804 dedup_ratio=0.245300 router_tokens_per_expert=[722, 790, 819, 752, 774, 791, 766, 712, 735, 744, 834, 773, 774, 767, 759, 773, 767, 746, 772, 755, 782, 806, 755, 772, 833, 750, 806, 771, 755, 790, 818, 775, 776, 750, 785, 770, 785, 760, 772, 767, 777, 730, 754, 785, 737, 789, 739, 751, 715, 744, 759, 762, 762, 770, 739, 792, 772, 766, 764, 756, 813, 768, 775, 730] payload_rows_per_dst=[9548, 9306, 9298, 9526, 9222, 9110, 8878, 9302] dup_rows_per_dst=[3310, 3042, 3012, 3070, 3108, 3014, 2572, 2986] combine_rows_per_src=[9548, 9306, 9298, 9526, 9222, 9110, 8878, 9302] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74426 payload_rows_remote=65180 dup_rows=23878 dup_rows_remote=20890 dedup_ratio=0.242900 router_tokens_per_expert=[771, 743, 775, 772, 779, 782, 727, 790, 797, 750, 768, 731, 812, 802, 741, 743, 779, 735, 818, 736, 735, 758, 783, 779, 783, 791, 774, 787, 780, 798, 737, 808, 774, 753, 742, 765, 712, 755, 792, 795, 738, 774, 750, 817, 746, 816, 719, 757, 757, 777, 757, 749, 762, 761, 791, 777, 809, 805, 786, 784, 728, 743, 712, 785] payload_rows_per_dst=[9276, 9282, 9280, 9460, 9294, 9246, 9370, 9218] dup_rows_per_dst=[3002, 3006, 2966, 3056, 2882, 2988, 2892, 3086] combine_rows_per_src=[9276, 9282, 9280, 9460, 9294, 9246, 9370, 9218] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265691136 dispatch_meta_bytes=344040 combine_payload_bytes=265691136 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266035176 ingress_bytes=369500160 link_bytes=369500160 token_bytes=531726312 weight_movement_bytes=103809024 total_nvlink_bytes=635535336 weight_to_token_ratio=0.195230 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390399355e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265592832 dispatch_meta_bytes=344256 combine_payload_bytes=265592832 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=265937088 ingress_bytes=300195840 link_bytes=300195840 token_bytes=531529920 weight_movement_bytes=34603008 total_nvlink_bytes=566132928 weight_to_token_ratio=0.065101 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129612778e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266264576 dispatch_meta_bytes=343984 combine_payload_bytes=266264576 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266608560 ingress_bytes=335470592 link_bytes=335470592 token_bytes=532873136 weight_movement_bytes=69206016 total_nvlink_bytes=602079152 weight_to_token_ratio=0.129873 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262348830e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266436608 dispatch_meta_bytes=344224 combine_payload_bytes=266436608 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266780832 ingress_bytes=335642624 link_bytes=335642624 token_bytes=533217440 weight_movement_bytes=69206016 total_nvlink_bytes=602423456 weight_to_token_ratio=0.129789 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262996173e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=267395072 dispatch_meta_bytes=344864 combine_payload_bytes=267395072 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267739936 ingress_bytes=336601088 link_bytes=336601088 token_bytes=535135008 weight_movement_bytes=69206016 total_nvlink_bytes=604341024 weight_to_token_ratio=0.129324 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266602796e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74476 payload_rows_remote=65124 dup_rows=23828 dup_rows_remote=20824 dedup_ratio=0.242391 router_tokens_per_expert=[726, 768, 741, 795, 750, 794, 765, 747, 786, 721, 795, 777, 797, 756, 769, 777, 734, 738, 793, 759, 784, 772, 738, 763, 809, 787, 725, 748, 789, 785, 750, 765, 806, 760, 721, 765, 742, 791, 782, 774, 813, 816, 747, 770, 794, 770, 745, 757, 829, 760, 816, 782, 731, 752, 794, 752, 783, 780, 732, 739, 764, 760, 771, 751] payload_rows_per_dst=[9242, 9352, 9338, 9380, 9190, 9318, 9458, 9198] dup_rows_per_dst=[2930, 3004, 2824, 2936, 3092, 3106, 2974, 2962] combine_rows_per_src=[9242, 9352, 9338, 9380, 9190, 9318, 9458, 9198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266141696 dispatch_meta_bytes=345376 combine_payload_bytes=266141696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266487072 ingress_bytes=300744704 link_bytes=300744704 token_bytes=532628768 weight_movement_bytes=34603008 total_nvlink_bytes=567231776 weight_to_token_ratio=0.064966 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131678109e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=264773632 dispatch_meta_bytes=341784 combine_payload_bytes=264773632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265115416 ingress_bytes=299376640 link_bytes=299376640 token_bytes=529889048 weight_movement_bytes=34603008 total_nvlink_bytes=564492056 weight_to_token_ratio=0.065302 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126530195e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266977280 dispatch_meta_bytes=344280 combine_payload_bytes=266977280 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267321560 ingress_bytes=405389312 link_bytes=405389312 token_bytes=534298840 weight_movement_bytes=138412032 total_nvlink_bytes=672710872 weight_to_token_ratio=0.259054 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.525447345e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=65112 dup_rows=23940 dup_rows_remote=21068 dedup_ratio=0.243530 router_tokens_per_expert=[801, 728, 802, 829, 820, 801, 766, 775, 795, 771, 721, 709, 753, 774, 751, 764, 792, 766, 822, 745, 774, 765, 733, 755, 788, 749, 722, 785, 756, 772, 746, 744, 772, 753, 735, 758, 755, 783, 765, 799, 792, 727, 785, 714, 804, 786, 775, 764, 778, 756, 716, 797, 751, 762, 755, 781, 776, 732, 787, 769, 757, 796, 807, 791] payload_rows_per_dst=[9542, 9278, 9220, 9252, 9270, 9286, 9182, 9334] dup_rows_per_dst=[3102, 2798, 3084, 2872, 2970, 3008, 3010, 3096] combine_rows_per_src=[9542, 9278, 9220, 9252, 9270, 9286, 9182, 9334] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74224 payload_rows_remote=64894 dup_rows=24080 dup_rows_remote=21012 dedup_ratio=0.244954 router_tokens_per_expert=[803, 740, 779, 761, 790, 719, 783, 806, 730, 764, 761, 761, 784, 763, 797, 806, 766, 781, 803, 777, 740, 776, 758, 756, 771, 719, 758, 782, 783, 750, 779, 693, 705, 785, 797, 761, 796, 784, 797, 774, 782, 750, 759, 749, 853, 797, 783, 749, 761, 797, 777, 729, 748, 745, 751, 782, 761, 794, 777, 750, 751, 761, 761, 747] payload_rows_per_dst=[9336, 9224, 9248, 9158, 9330, 9368, 9236, 9324] dup_rows_per_dst=[3026, 3108, 3066, 2912, 3068, 3076, 2944, 2880] combine_rows_per_src=[9336, 9224, 9248, 9158, 9330, 9368, 9236, 9324] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=64782 dup_rows=24132 dup_rows_remote=20882 dedup_ratio=0.245483 router_tokens_per_expert=[747, 772, 749, 806, 800, 756, 787, 804, 813, 744, 768, 768, 714, 749, 790, 730, 720, 761, 756, 743, 791, 783, 788, 795, 767, 796, 749, 798, 774, 770, 790, 762, 748, 759, 793, 754, 764, 770, 791, 800, 767, 780, 795, 779, 753, 751, 727, 799, 793, 812, 771, 748, 726, 764, 726, 758, 747, 761, 767, 767, 780, 747, 768, 747] payload_rows_per_dst=[9204, 9214, 9226, 9328, 9294, 9040, 9476, 9390] dup_rows_per_dst=[2792, 2938, 3048, 3084, 3064, 2790, 3166, 3250] combine_rows_per_src=[9204, 9214, 9226, 9328, 9294, 9040, 9476, 9390] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74218 payload_rows_remote=65046 dup_rows=24086 dup_rows_remote=21272 dedup_ratio=0.245015 router_tokens_per_expert=[786, 804, 754, 756, 742, 775, 786, 786, 754, 740, 818, 782, 745, 791, 773, 748, 752, 713, 799, 696, 769, 795, 699, 770, 812, 796, 754, 752, 820, 779, 805, 799, 740, 739, 755, 749, 773, 720, 828, 734, 788, 784, 726, 730, 786, 769, 781, 776, 770, 815, 712, 818, 771, 801, 789, 763, 740, 818, 762, 756, 754, 774, 727, 754] payload_rows_per_dst=[9284, 9364, 9172, 9456, 9110, 9276, 9276, 9280] dup_rows_per_dst=[3094, 2938, 2814, 3178, 2966, 3004, 3202, 2890] combine_rows_per_src=[9284, 9364, 9172, 9456, 9110, 9276, 9276, 9280] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74476 payload_rows_remote=65124 dup_rows=23828 dup_rows_remote=20824 dedup_ratio=0.242391 router_tokens_per_expert=[726, 768, 741, 795, 750, 794, 765, 747, 786, 721, 795, 777, 797, 756, 769, 777, 734, 738, 793, 759, 784, 772, 738, 763, 809, 787, 725, 748, 789, 785, 750, 765, 806, 760, 721, 765, 742, 791, 782, 774, 813, 816, 747, 770, 794, 770, 745, 757, 829, 760, 816, 782, 731, 752, 794, 752, 783, 780, 732, 739, 764, 760, 771, 751] payload_rows_per_dst=[9242, 9352, 9338, 9380, 9190, 9318, 9458, 9198] dup_rows_per_dst=[2930, 3004, 2824, 2936, 3092, 3106, 2974, 2962] combine_rows_per_src=[9242, 9352, 9338, 9380, 9190, 9318, 9458, 9198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266747904 dispatch_meta_bytes=343792 combine_payload_bytes=266747904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267091696 ingress_bytes=301350912 link_bytes=301350912 token_bytes=533839600 weight_movement_bytes=34603008 total_nvlink_bytes=568442608 weight_to_token_ratio=0.064819 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133959221e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266698752 dispatch_meta_bytes=344720 combine_payload_bytes=266698752 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267043472 ingress_bytes=370507776 link_bytes=370507776 token_bytes=533742224 weight_movement_bytes=103809024 total_nvlink_bytes=637551248 weight_to_token_ratio=0.194493 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394190933e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265805824 dispatch_meta_bytes=343624 combine_payload_bytes=265805824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266149448 ingress_bytes=300408832 link_bytes=300408832 token_bytes=531955272 weight_movement_bytes=34603008 total_nvlink_bytes=566558280 weight_to_token_ratio=0.065049 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130414250e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74426 payload_rows_remote=65180 dup_rows=23878 dup_rows_remote=20890 dedup_ratio=0.242900 router_tokens_per_expert=[771, 743, 775, 772, 779, 782, 727, 790, 797, 750, 768, 731, 812, 802, 741, 743, 779, 735, 818, 736, 735, 758, 783, 779, 783, 791, 774, 787, 780, 798, 737, 808, 774, 753, 742, 765, 712, 755, 792, 795, 738, 774, 750, 817, 746, 816, 719, 757, 757, 777, 757, 749, 762, 761, 791, 777, 809, 805, 786, 784, 728, 743, 712, 785] payload_rows_per_dst=[9276, 9282, 9280, 9460, 9294, 9246, 9370, 9218] dup_rows_per_dst=[3002, 3006, 2966, 3056, 2882, 2988, 2892, 3086] combine_rows_per_src=[9276, 9282, 9280, 9460, 9294, 9246, 9370, 9218] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=65112 dup_rows=23940 dup_rows_remote=21068 dedup_ratio=0.243530 router_tokens_per_expert=[801, 728, 802, 829, 820, 801, 766, 775, 795, 771, 721, 709, 753, 774, 751, 764, 792, 766, 822, 745, 774, 765, 733, 755, 788, 749, 722, 785, 756, 772, 746, 744, 772, 753, 735, 758, 755, 783, 765, 799, 792, 727, 785, 714, 804, 786, 775, 764, 778, 756, 716, 797, 751, 762, 755, 781, 776, 732, 787, 769, 757, 796, 807, 791] payload_rows_per_dst=[9542, 9278, 9220, 9252, 9270, 9286, 9182, 9334] dup_rows_per_dst=[3102, 2798, 3084, 2872, 2970, 3008, 3010, 3096] combine_rows_per_src=[9542, 9278, 9220, 9252, 9270, 9286, 9182, 9334] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265347072 dispatch_meta_bytes=342656 combine_payload_bytes=265347072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265689728 ingress_bytes=299950080 link_bytes=299950080 token_bytes=531036800 weight_movement_bytes=34603008 total_nvlink_bytes=565639808 weight_to_token_ratio=0.065161 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128688003e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73978 payload_rows_remote=64976 dup_rows=24326 dup_rows_remote=21368 dedup_ratio=0.247457 router_tokens_per_expert=[755, 758, 752, 758, 735, 756, 783, 801, 780, 757, 762, 786, 748, 743, 792, 795, 778, 816, 774, 805, 727, 776, 774, 783, 741, 741, 751, 829, 745, 749, 793, 763, 761, 777, 801, 759, 795, 774, 824, 765, 797, 763, 758, 743, 817, 769, 804, 767, 720, 755, 754, 774, 730, 761, 764, 722, 751, 754, 806, 772, 727, 761, 761, 760] payload_rows_per_dst=[9226, 9250, 9324, 9220, 9430, 9344, 9002, 9182] dup_rows_per_dst=[2970, 3076, 3142, 3004, 3082, 3092, 2958, 3002] combine_rows_per_src=[9226, 9250, 9324, 9220, 9430, 9344, 9002, 9182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74218 payload_rows_remote=65046 dup_rows=24086 dup_rows_remote=21272 dedup_ratio=0.245015 router_tokens_per_expert=[786, 804, 754, 756, 742, 775, 786, 786, 754, 740, 818, 782, 745, 791, 773, 748, 752, 713, 799, 696, 769, 795, 699, 770, 812, 796, 754, 752, 820, 779, 805, 799, 740, 739, 755, 749, 773, 720, 828, 734, 788, 784, 726, 730, 786, 769, 781, 776, 770, 815, 712, 818, 771, 801, 789, 763, 740, 818, 762, 756, 754, 774, 727, 754] payload_rows_per_dst=[9284, 9364, 9172, 9456, 9110, 9276, 9276, 9280] dup_rows_per_dst=[3094, 2938, 2814, 3178, 2966, 3004, 3202, 2890] combine_rows_per_src=[9284, 9364, 9172, 9456, 9110, 9276, 9276, 9280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74230 payload_rows_remote=65024 dup_rows=24074 dup_rows_remote=21056 dedup_ratio=0.244893 router_tokens_per_expert=[774, 767, 789, 755, 779, 765, 710, 764, 724, 824, 777, 716, 753, 760, 784, 754, 796, 786, 765, 800, 776, 846, 786, 821, 755, 737, 731, 728, 795, 775, 736, 773, 744, 755, 803, 776, 752, 757, 767, 758, 741, 772, 751, 752, 797, 780, 797, 725, 753, 767, 784, 753, 747, 831, 775, 770, 788, 795, 772, 784, 740, 750, 769, 746] payload_rows_per_dst=[9262, 9264, 9546, 9092, 9206, 9306, 9300, 9254] dup_rows_per_dst=[2944, 2920, 3206, 2968, 3018, 2924, 3060, 3034] combine_rows_per_src=[9262, 9264, 9546, 9092, 9206, 9306, 9300, 9254] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74616 payload_rows_remote=65424 dup_rows=23688 dup_rows_remote=20794 dedup_ratio=0.240967 router_tokens_per_expert=[811, 760, 728, 785, 795, 791, 763, 733, 796, 761, 758, 779, 739, 809, 739, 780, 718, 785, 744, 809, 697, 785, 783, 778, 752, 752, 783, 779, 785, 781, 797, 779, 752, 789, 724, 798, 777, 763, 743, 742, 798, 768, 744, 766, 811, 753, 799, 740, 751, 726, 754, 799, 763, 790, 707, 753, 772, 749, 780, 804, 767, 774, 799, 763] payload_rows_per_dst=[9322, 9350, 9226, 9476, 9306, 9290, 9192, 9454] dup_rows_per_dst=[3010, 2972, 2972, 2940, 2870, 3068, 2894, 2962] combine_rows_per_src=[9322, 9350, 9226, 9476, 9306, 9290, 9192, 9454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266977280 dispatch_meta_bytes=344280 combine_payload_bytes=266977280 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267321560 ingress_bytes=405389312 link_bytes=405389312 token_bytes=534298840 weight_movement_bytes=138412032 total_nvlink_bytes=672710872 weight_to_token_ratio=0.259054 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.525447345e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74214 payload_rows_remote=64906 dup_rows=24090 dup_rows_remote=21120 dedup_ratio=0.245056 router_tokens_per_expert=[764, 790, 776, 741, 795, 728, 740, 755, 776, 803, 730, 795, 791, 747, 811, 760, 772, 759, 732, 755, 793, 764, 774, 787, 760, 776, 732, 766, 742, 807, 788, 787, 786, 777, 738, 761, 759, 784, 751, 742, 782, 837, 803, 761, 789, 758, 753, 790, 792, 721, 756, 744, 768, 757, 777, 764, 768, 786, 788, 697, 794, 759, 749, 765] payload_rows_per_dst=[9308, 9418, 9362, 9148, 9228, 9350, 9176, 9224] dup_rows_per_dst=[2970, 3054, 3190, 2860, 3034, 3012, 2982, 2988] combine_rows_per_src=[9308, 9418, 9362, 9148, 9228, 9350, 9176, 9224] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74326 payload_rows_remote=64982 dup_rows=23978 dup_rows_remote=20966 dedup_ratio=0.243917 router_tokens_per_expert=[726, 771, 749, 780, 749, 755, 825, 745, 759, 775, 784, 816, 748, 748, 748, 727, 754, 783, 750, 746, 770, 764, 753, 737, 739, 814, 788, 762, 747, 757, 800, 720, 763, 769, 735, 711, 777, 804, 782, 747, 812, 759, 772, 786, 768, 785, 731, 765, 779, 765, 796, 797, 775, 764, 803, 782, 784, 823, 744, 817, 719, 784, 761, 804] payload_rows_per_dst=[9214, 9240, 9154, 9308, 9246, 9344, 9410, 9410] dup_rows_per_dst=[2986, 2970, 2960, 2946, 2930, 3012, 3112, 3062] combine_rows_per_src=[9214, 9240, 9154, 9308, 9246, 9344, 9410, 9410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266698752 dispatch_meta_bytes=344720 combine_payload_bytes=266698752 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267043472 ingress_bytes=370507776 link_bytes=370507776 token_bytes=533742224 weight_movement_bytes=103809024 total_nvlink_bytes=637551248 weight_to_token_ratio=0.194493 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394190933e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=345272 combine_payload_bytes=266428416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266773688 ingress_bytes=301031424 link_bytes=301031424 token_bytes=533202104 weight_movement_bytes=34603008 total_nvlink_bytes=567805112 weight_to_token_ratio=0.064897 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132757013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266747904 dispatch_meta_bytes=343792 combine_payload_bytes=266747904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267091696 ingress_bytes=301350912 link_bytes=301350912 token_bytes=533839600 weight_movement_bytes=34603008 total_nvlink_bytes=568442608 weight_to_token_ratio=0.064819 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133959221e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74450 payload_rows_remote=65184 dup_rows=23854 dup_rows_remote=20852 dedup_ratio=0.242655 router_tokens_per_expert=[759, 767, 793, 760, 795, 774, 709, 729, 711, 792, 786, 784, 777, 760, 758, 766, 784, 811, 752, 751, 735, 767, 792, 810, 726, 798, 798, 787, 800, 734, 725, 788, 693, 787, 799, 787, 731, 768, 782, 785, 760, 727, 785, 738, 774, 746, 828, 744, 773, 779, 772, 736, 725, 779, 781, 770, 798, 758, 787, 784, 742, 799, 743, 814] payload_rows_per_dst=[9184, 9266, 9466, 9324, 9254, 9286, 9228, 9442] dup_rows_per_dst=[2988, 3002, 2938, 2988, 3010, 2918, 3002, 3008] combine_rows_per_src=[9184, 9266, 9466, 9324, 9254, 9286, 9228, 9442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=344320 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266682624 ingress_bytes=300941312 link_bytes=300941312 token_bytes=533020928 weight_movement_bytes=34603008 total_nvlink_bytes=567623936 weight_to_token_ratio=0.064919 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132417929e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266141696 dispatch_meta_bytes=345376 combine_payload_bytes=266141696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266487072 ingress_bytes=300744704 link_bytes=300744704 token_bytes=532628768 weight_movement_bytes=34603008 total_nvlink_bytes=567231776 weight_to_token_ratio=0.064966 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131678109e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=267976704 dispatch_meta_bytes=344872 combine_payload_bytes=267976704 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=268321576 ingress_bytes=337182720 link_bytes=337182720 token_bytes=536298280 weight_movement_bytes=69206016 total_nvlink_bytes=605504296 weight_to_token_ratio=0.129044 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.268791430e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265854976 dispatch_meta_bytes=344104 combine_payload_bytes=265854976 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266199080 ingress_bytes=335060992 link_bytes=335060992 token_bytes=532054056 weight_movement_bytes=69206016 total_nvlink_bytes=601260072 weight_to_token_ratio=0.130073 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260807538e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266166272 dispatch_meta_bytes=343792 combine_payload_bytes=266166272 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266510064 ingress_bytes=369975296 link_bytes=369975296 token_bytes=532676336 weight_movement_bytes=103809024 total_nvlink_bytes=636485360 weight_to_token_ratio=0.194882 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392187253e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=64994 dup_rows=23966 dup_rows_remote=20860 dedup_ratio=0.243795 router_tokens_per_expert=[736, 770, 803, 779, 797, 751, 735, 775, 776, 797, 763, 746, 764, 752, 822, 769, 754, 763, 748, 800, 774, 756, 757, 787, 790, 800, 738, 804, 844, 763, 720, 766, 755, 772, 763, 725, 759, 754, 762, 761, 747, 771, 770, 775, 764, 818, 777, 766, 785, 764, 727, 732, 790, 797, 768, 739, 717, 741, 726, 770, 807, 805, 758, 788] payload_rows_per_dst=[9296, 9294, 9286, 9344, 9250, 9352, 9232, 9284] dup_rows_per_dst=[2996, 3084, 2992, 3106, 2852, 3024, 2972, 2940] combine_rows_per_src=[9296, 9294, 9286, 9344, 9250, 9352, 9232, 9284] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74200 payload_rows_remote=64932 dup_rows=24104 dup_rows_remote=21048 dedup_ratio=0.245199 router_tokens_per_expert=[726, 778, 764, 774, 799, 797, 778, 741, 790, 729, 777, 748, 819, 773, 762, 686, 753, 755, 733, 778, 800, 820, 740, 754, 786, 776, 766, 775, 756, 749, 763, 773, 754, 759, 777, 775, 788, 794, 768, 747, 698, 739, 737, 800, 767, 765, 794, 771, 753, 770, 795, 786, 795, 795, 803, 777, 766, 761, 761, 723, 789, 770, 802, 755] payload_rows_per_dst=[9378, 9148, 9326, 9212, 9268, 9196, 9446, 9226] dup_rows_per_dst=[2936, 3020, 2940, 3076, 3056, 2946, 3102, 3028] combine_rows_per_src=[9378, 9148, 9326, 9212, 9268, 9196, 9446, 9226] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=65256 dup_rows=23948 dup_rows_remote=21010 dedup_ratio=0.243612 router_tokens_per_expert=[784, 733, 748, 733, 750, 798, 733, 785, 752, 757, 755, 756, 754, 767, 759, 785, 791, 757, 794, 769, 807, 760, 769, 747, 805, 762, 766, 776, 789, 753, 753, 789, 765, 813, 767, 793, 808, 771, 745, 775, 722, 768, 779, 751, 821, 789, 782, 773, 754, 733, 788, 787, 780, 759, 831, 745, 763, 732, 715, 728, 800, 712, 776, 791] payload_rows_per_dst=[9244, 9164, 9256, 9422, 9456, 9366, 9348, 9100] dup_rows_per_dst=[2884, 3006, 3012, 3084, 3018, 2996, 3010, 2938] combine_rows_per_src=[9244, 9164, 9256, 9422, 9456, 9366, 9348, 9100] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=64958 dup_rows=24062 dup_rows_remote=21010 dedup_ratio=0.244771 router_tokens_per_expert=[758, 783, 798, 751, 810, 787, 778, 752, 749, 734, 797, 797, 744, 740, 728, 791, 785, 803, 772, 753, 789, 766, 753, 747, 765, 765, 753, 758, 749, 791, 788, 754, 767, 809, 755, 751, 789, 749, 789, 790, 780, 783, 817, 793, 705, 766, 741, 731, 799, 751, 736, 772, 739, 779, 764, 755, 745, 800, 783, 796, 759, 752, 753, 766] payload_rows_per_dst=[9360, 9240, 9284, 9254, 9314, 9270, 9216, 9304] dup_rows_per_dst=[3074, 2920, 3052, 2992, 3084, 2962, 2974, 3004] combine_rows_per_src=[9360, 9240, 9284, 9254, 9314, 9270, 9216, 9304] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74450 payload_rows_remote=65184 dup_rows=23854 dup_rows_remote=20852 dedup_ratio=0.242655 router_tokens_per_expert=[759, 767, 793, 760, 795, 774, 709, 729, 711, 792, 786, 784, 777, 760, 758, 766, 784, 811, 752, 751, 735, 767, 792, 810, 726, 798, 798, 787, 800, 734, 725, 788, 693, 787, 799, 787, 731, 768, 782, 785, 760, 727, 785, 738, 774, 746, 828, 744, 773, 779, 772, 736, 725, 779, 781, 770, 798, 758, 787, 784, 742, 799, 743, 814] payload_rows_per_dst=[9184, 9266, 9466, 9324, 9254, 9286, 9228, 9442] dup_rows_per_dst=[2988, 3002, 2938, 2988, 3010, 2918, 3002, 3008] combine_rows_per_src=[9184, 9266, 9466, 9324, 9254, 9286, 9228, 9442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266993664 dispatch_meta_bytes=344144 combine_payload_bytes=266993664 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267337808 ingress_bytes=301596672 link_bytes=301596672 token_bytes=534331472 weight_movement_bytes=34603008 total_nvlink_bytes=568934480 weight_to_token_ratio=0.064759 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134883996e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266215424 dispatch_meta_bytes=343416 combine_payload_bytes=266215424 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266558840 ingress_bytes=335421440 link_bytes=335421440 token_bytes=532774264 weight_movement_bytes=69206016 total_nvlink_bytes=601980280 weight_to_token_ratio=0.129897 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262163875e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265961472 dispatch_meta_bytes=343920 combine_payload_bytes=265961472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266305392 ingress_bytes=300564480 link_bytes=300564480 token_bytes=532266864 weight_movement_bytes=34603008 total_nvlink_bytes=566869872 weight_to_token_ratio=0.065011 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130999941e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74616 payload_rows_remote=65424 dup_rows=23688 dup_rows_remote=20794 dedup_ratio=0.240967 router_tokens_per_expert=[811, 760, 728, 785, 795, 791, 763, 733, 796, 761, 758, 779, 739, 809, 739, 780, 718, 785, 744, 809, 697, 785, 783, 778, 752, 752, 783, 779, 785, 781, 797, 779, 752, 789, 724, 798, 777, 763, 743, 742, 798, 768, 744, 766, 811, 753, 799, 740, 751, 726, 754, 799, 763, 790, 707, 753, 772, 749, 780, 804, 767, 774, 799, 763] payload_rows_per_dst=[9322, 9350, 9226, 9476, 9306, 9290, 9192, 9454] dup_rows_per_dst=[3010, 2972, 2972, 2940, 2870, 3068, 2894, 2962] combine_rows_per_src=[9322, 9350, 9226, 9476, 9306, 9290, 9192, 9454] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=65230 dup_rows=23860 dup_rows_remote=20804 dedup_ratio=0.242716 router_tokens_per_expert=[794, 752, 744, 796, 775, 766, 817, 768, 752, 731, 741, 766, 813, 708, 792, 745, 800, 772, 768, 753, 781, 790, 829, 780, 797, 767, 830, 758, 766, 767, 833, 737, 714, 774, 739, 740, 737, 763, 767, 750, 754, 728, 798, 754, 785, 797, 754, 765, 748, 723, 803, 781, 757, 749, 763, 757, 773, 784, 795, 786, 739, 721, 819, 747] payload_rows_per_dst=[9428, 9194, 9454, 9412, 9130, 9214, 9282, 9330] dup_rows_per_dst=[2996, 2902, 3092, 3098, 2838, 3056, 2880, 2998] combine_rows_per_src=[9428, 9194, 9454, 9412, 9130, 9214, 9282, 9330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74326 payload_rows_remote=64982 dup_rows=23978 dup_rows_remote=20966 dedup_ratio=0.243917 router_tokens_per_expert=[726, 771, 749, 780, 749, 755, 825, 745, 759, 775, 784, 816, 748, 748, 748, 727, 754, 783, 750, 746, 770, 764, 753, 737, 739, 814, 788, 762, 747, 757, 800, 720, 763, 769, 735, 711, 777, 804, 782, 747, 812, 759, 772, 786, 768, 785, 731, 765, 779, 765, 796, 797, 775, 764, 803, 782, 784, 823, 744, 817, 719, 784, 761, 804] payload_rows_per_dst=[9214, 9240, 9154, 9308, 9246, 9344, 9410, 9410] dup_rows_per_dst=[2986, 2970, 2960, 2946, 2930, 3012, 3112, 3062] combine_rows_per_src=[9214, 9240, 9154, 9308, 9246, 9344, 9410, 9410] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=267288576 dispatch_meta_bytes=345064 combine_payload_bytes=267288576 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267633640 ingress_bytes=336494592 link_bytes=336494592 token_bytes=534922216 weight_movement_bytes=69206016 total_nvlink_bytes=604128232 weight_to_token_ratio=0.129376 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266202060e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=64958 dup_rows=24062 dup_rows_remote=21010 dedup_ratio=0.244771 router_tokens_per_expert=[758, 783, 798, 751, 810, 787, 778, 752, 749, 734, 797, 797, 744, 740, 728, 791, 785, 803, 772, 753, 789, 766, 753, 747, 765, 765, 753, 758, 749, 791, 788, 754, 767, 809, 755, 751, 789, 749, 789, 790, 780, 783, 817, 793, 705, 766, 741, 731, 799, 751, 736, 772, 739, 779, 764, 755, 745, 800, 783, 796, 759, 752, 753, 766] payload_rows_per_dst=[9360, 9240, 9284, 9254, 9314, 9270, 9216, 9304] dup_rows_per_dst=[3074, 2920, 3052, 2992, 3084, 2962, 2974, 3004] combine_rows_per_src=[9360, 9240, 9284, 9254, 9314, 9270, 9216, 9304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=64994 dup_rows=23966 dup_rows_remote=20860 dedup_ratio=0.243795 router_tokens_per_expert=[736, 770, 803, 779, 797, 751, 735, 775, 776, 797, 763, 746, 764, 752, 822, 769, 754, 763, 748, 800, 774, 756, 757, 787, 790, 800, 738, 804, 844, 763, 720, 766, 755, 772, 763, 725, 759, 754, 762, 761, 747, 771, 770, 775, 764, 818, 777, 766, 785, 764, 727, 732, 790, 797, 768, 739, 717, 741, 726, 770, 807, 805, 758, 788] payload_rows_per_dst=[9296, 9294, 9286, 9344, 9250, 9352, 9232, 9284] dup_rows_per_dst=[2996, 3084, 2992, 3106, 2852, 3024, 2972, 2940] combine_rows_per_src=[9296, 9294, 9286, 9344, 9250, 9352, 9232, 9284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74224 payload_rows_remote=64894 dup_rows=24080 dup_rows_remote=21012 dedup_ratio=0.244954 router_tokens_per_expert=[803, 740, 779, 761, 790, 719, 783, 806, 730, 764, 761, 761, 784, 763, 797, 806, 766, 781, 803, 777, 740, 776, 758, 756, 771, 719, 758, 782, 783, 750, 779, 693, 705, 785, 797, 761, 796, 784, 797, 774, 782, 750, 759, 749, 853, 797, 783, 749, 761, 797, 777, 729, 748, 745, 751, 782, 761, 794, 777, 750, 751, 761, 761, 747] payload_rows_per_dst=[9336, 9224, 9248, 9158, 9330, 9368, 9236, 9324] dup_rows_per_dst=[3026, 3108, 3066, 2912, 3068, 3076, 2944, 2880] combine_rows_per_src=[9336, 9224, 9248, 9158, 9330, 9368, 9236, 9324] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74132 payload_rows_remote=65006 dup_rows=24172 dup_rows_remote=21210 dedup_ratio=0.245890 router_tokens_per_expert=[836, 761, 744, 766, 816, 729, 766, 801, 813, 738, 822, 784, 747, 757, 778, 785, 787, 746, 825, 754, 799, 796, 778, 732, 783, 731, 776, 731, 794, 747, 756, 776, 814, 792, 772, 760, 744, 743, 770, 750, 737, 809, 724, 796, 751, 723, 750, 746, 785, 772, 751, 758, 752, 753, 708, 765, 688, 771, 822, 750, 766, 748, 796, 832] payload_rows_per_dst=[9348, 9360, 9332, 9214, 9296, 9184, 9126, 9272] dup_rows_per_dst=[3090, 3088, 3102, 2974, 2994, 2888, 2962, 3074] combine_rows_per_src=[9348, 9360, 9332, 9214, 9296, 9184, 9126, 9272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266166272 dispatch_meta_bytes=343792 combine_payload_bytes=266166272 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266510064 ingress_bytes=369975296 link_bytes=369975296 token_bytes=532676336 weight_movement_bytes=103809024 total_nvlink_bytes=636485360 weight_to_token_ratio=0.194882 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392187253e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74222 payload_rows_remote=64914 dup_rows=24082 dup_rows_remote=20958 dedup_ratio=0.244975 router_tokens_per_expert=[728, 750, 775, 719, 734, 779, 747, 776, 794, 751, 779, 815, 778, 783, 766, 759, 802, 739, 754, 793, 758, 816, 785, 772, 746, 776, 736, 772, 806, 776, 740, 772, 750, 770, 777, 784, 784, 762, 753, 718, 772, 737, 754, 775, 759, 767, 780, 794, 786, 742, 792, 700, 799, 812, 773, 776, 791, 779, 800, 779, 770, 768, 725, 748] payload_rows_per_dst=[9308, 9490, 9358, 9122, 9386, 9252, 9146, 9160] dup_rows_per_dst=[3124, 3138, 3082, 2710, 3004, 3022, 3020, 2982] combine_rows_per_src=[9308, 9490, 9358, 9122, 9386, 9252, 9146, 9160] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=343872 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266411840 ingress_bytes=335273984 link_bytes=335273984 token_bytes=532479808 weight_movement_bytes=69206016 total_nvlink_bytes=601685824 weight_to_token_ratio=0.129969 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261609010e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266993664 dispatch_meta_bytes=344144 combine_payload_bytes=266993664 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267337808 ingress_bytes=301596672 link_bytes=301596672 token_bytes=534331472 weight_movement_bytes=34603008 total_nvlink_bytes=568934480 weight_to_token_ratio=0.064759 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134883996e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74536 payload_rows_remote=65042 dup_rows=23768 dup_rows_remote=20682 dedup_ratio=0.241781 router_tokens_per_expert=[760, 698, 773, 766, 779, 782, 728, 781, 784, 838, 771, 783, 748, 842, 734, 790, 738, 775, 784, 792, 796, 797, 772, 772, 749, 759, 755, 814, 740, 773, 722, 748, 783, 702, 768, 728, 765, 772, 758, 780, 796, 754, 810, 774, 758, 756, 743, 706, 782, 812, 780, 780, 765, 741, 770, 769, 763, 759, 775, 790, 774, 768, 767, 761] payload_rows_per_dst=[9208, 9494, 9332, 9234, 9226, 9254, 9476, 9312] dup_rows_per_dst=[2926, 3086, 3120, 2886, 2886, 2940, 2922, 3002] combine_rows_per_src=[9208, 9494, 9332, 9234, 9226, 9254, 9476, 9312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266215424 dispatch_meta_bytes=343416 combine_payload_bytes=266215424 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266558840 ingress_bytes=335421440 link_bytes=335421440 token_bytes=532774264 weight_movement_bytes=69206016 total_nvlink_bytes=601980280 weight_to_token_ratio=0.129897 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262163875e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265805824 dispatch_meta_bytes=343624 combine_payload_bytes=265805824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266149448 ingress_bytes=300408832 link_bytes=300408832 token_bytes=531955272 weight_movement_bytes=34603008 total_nvlink_bytes=566558280 weight_to_token_ratio=0.065049 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130414250e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266264576 dispatch_meta_bytes=344864 combine_payload_bytes=266264576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266609440 ingress_bytes=300867584 link_bytes=300867584 token_bytes=532874016 weight_movement_bytes=34603008 total_nvlink_bytes=567477024 weight_to_token_ratio=0.064937 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132140497e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=267976704 dispatch_meta_bytes=344872 combine_payload_bytes=267976704 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=268321576 ingress_bytes=337182720 link_bytes=337182720 token_bytes=536298280 weight_movement_bytes=69206016 total_nvlink_bytes=605504296 weight_to_token_ratio=0.129044 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.268791430e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267182080 dispatch_meta_bytes=344136 combine_payload_bytes=267182080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267526216 ingress_bytes=301785088 link_bytes=301785088 token_bytes=534708296 weight_movement_bytes=34603008 total_nvlink_bytes=569311304 weight_to_token_ratio=0.064714 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135592990e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=65032 dup_rows=24008 dup_rows_remote=21072 dedup_ratio=0.244222 router_tokens_per_expert=[818, 717, 752, 773, 710, 760, 747, 743, 753, 834, 724, 766, 742, 762, 739, 744, 788, 786, 815, 799, 779, 749, 726, 761, 824, 782, 775, 774, 823, 763, 752, 758, 762, 847, 804, 807, 734, 753, 756, 793, 810, 798, 792, 764, 768, 808, 776, 789, 740, 747, 762, 740, 749, 708, 731, 779, 720, 797, 796, 713, 750, 771, 760, 790] payload_rows_per_dst=[9132, 9250, 9250, 9156, 9398, 9502, 9344, 9264] dup_rows_per_dst=[2908, 2878, 3038, 2912, 3170, 3108, 3058, 2936] combine_rows_per_src=[9132, 9250, 9250, 9156, 9398, 9502, 9344, 9264] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265887744 dispatch_meta_bytes=343488 combine_payload_bytes=265887744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266231232 ingress_bytes=300490752 link_bytes=300490752 token_bytes=532118976 weight_movement_bytes=34603008 total_nvlink_bytes=566721984 weight_to_token_ratio=0.065029 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130722508e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=342896 combine_payload_bytes=266412032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266754928 ingress_bytes=301015040 link_bytes=301015040 token_bytes=533166960 weight_movement_bytes=34603008 total_nvlink_bytes=567769968 weight_to_token_ratio=0.064901 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132695362e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=65230 dup_rows=23860 dup_rows_remote=20804 dedup_ratio=0.242716 router_tokens_per_expert=[794, 752, 744, 796, 775, 766, 817, 768, 752, 731, 741, 766, 813, 708, 792, 745, 800, 772, 768, 753, 781, 790, 829, 780, 797, 767, 830, 758, 766, 767, 833, 737, 714, 774, 739, 740, 737, 763, 767, 750, 754, 728, 798, 754, 785, 797, 754, 765, 748, 723, 803, 781, 757, 749, 763, 757, 773, 784, 795, 786, 739, 721, 819, 747] payload_rows_per_dst=[9428, 9194, 9454, 9412, 9130, 9214, 9282, 9330] dup_rows_per_dst=[2996, 2902, 3092, 3098, 2838, 3056, 2880, 2998] combine_rows_per_src=[9428, 9194, 9454, 9412, 9130, 9214, 9282, 9330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344416 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266715488 ingress_bytes=370180096 link_bytes=370180096 token_bytes=533086560 weight_movement_bytes=103809024 total_nvlink_bytes=636895584 weight_to_token_ratio=0.194732 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392957899e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74134 payload_rows_remote=64894 dup_rows=24170 dup_rows_remote=21190 dedup_ratio=0.245870 router_tokens_per_expert=[768, 771, 791, 774, 774, 781, 809, 743, 797, 735, 761, 772, 753, 803, 772, 758, 780, 723, 779, 754, 748, 756, 781, 789, 766, 789, 810, 735, 779, 766, 730, 748, 765, 714, 765, 740, 798, 758, 814, 764, 786, 789, 771, 713, 787, 784, 805, 777, 811, 776, 730, 780, 739, 755, 756, 741, 761, 713, 786, 741, 764, 816, 791, 767] payload_rows_per_dst=[9348, 9236, 9240, 9334, 9256, 9356, 9138, 9226] dup_rows_per_dst=[3074, 3066, 2980, 2912, 2980, 3068, 3038, 3052] combine_rows_per_src=[9348, 9236, 9240, 9334, 9256, 9356, 9138, 9226] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74478 payload_rows_remote=65202 dup_rows=23826 dup_rows_remote=20818 dedup_ratio=0.242371 router_tokens_per_expert=[773, 782, 749, 796, 724, 748, 798, 734, 768, 735, 758, 768, 807, 762, 755, 739, 799, 751, 759, 790, 780, 746, 691, 776, 790, 785, 767, 746, 753, 751, 785, 765, 817, 750, 777, 764, 794, 805, 772, 798, 797, 785, 753, 737, 771, 784, 771, 759, 759, 781, 775, 768, 775, 736, 741, 748, 775, 763, 822, 770, 781, 798, 709, 787] payload_rows_per_dst=[9288, 9292, 9170, 9276, 9464, 9334, 9286, 9368] dup_rows_per_dst=[2920, 2892, 3014, 3008, 3090, 2980, 2880, 3042] combine_rows_per_src=[9288, 9292, 9170, 9276, 9464, 9334, 9286, 9368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74352 payload_rows_remote=65142 dup_rows=23952 dup_rows_remote=20978 dedup_ratio=0.243652 router_tokens_per_expert=[748, 754, 799, 750, 783, 806, 774, 722, 752, 731, 737, 792, 729, 766, 774, 756, 773, 740, 732, 815, 804, 713, 777, 698, 761, 778, 779, 834, 778, 770, 776, 809, 740, 735, 820, 759, 750, 748, 758, 782, 769, 755, 799, 794, 763, 772, 760, 754, 761, 806, 791, 750, 756, 774, 761, 798, 757, 762, 769, 791, 786, 825, 724, 773] payload_rows_per_dst=[9350, 9212, 9232, 9462, 9210, 9264, 9360, 9262] dup_rows_per_dst=[2922, 2862, 2872, 3108, 2974, 3068, 3034, 3112] combine_rows_per_src=[9350, 9212, 9232, 9462, 9210, 9264, 9360, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74200 payload_rows_remote=64932 dup_rows=24104 dup_rows_remote=21048 dedup_ratio=0.245199 router_tokens_per_expert=[726, 778, 764, 774, 799, 797, 778, 741, 790, 729, 777, 748, 819, 773, 762, 686, 753, 755, 733, 778, 800, 820, 740, 754, 786, 776, 766, 775, 756, 749, 763, 773, 754, 759, 777, 775, 788, 794, 768, 747, 698, 739, 737, 800, 767, 765, 794, 771, 753, 770, 795, 786, 795, 795, 803, 777, 766, 761, 761, 723, 789, 770, 802, 755] payload_rows_per_dst=[9378, 9148, 9326, 9212, 9268, 9196, 9446, 9226] dup_rows_per_dst=[2936, 3020, 2940, 3076, 3056, 2946, 3102, 3028] combine_rows_per_src=[9378, 9148, 9326, 9212, 9268, 9196, 9446, 9226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74184 payload_rows_remote=65012 dup_rows=24120 dup_rows_remote=21114 dedup_ratio=0.245361 router_tokens_per_expert=[753, 778, 719, 795, 712, 763, 773, 784, 771, 755, 818, 802, 815, 811, 733, 827, 797, 765, 723, 759, 793, 780, 786, 757, 749, 774, 773, 767, 755, 755, 778, 731, 801, 778, 764, 801, 800, 815, 744, 794, 736, 686, 789, 787, 717, 703, 771, 760, 716, 720, 784, 776, 761, 791, 762, 779, 788, 779, 780, 770, 773, 736, 730, 810] payload_rows_per_dst=[9212, 9566, 9340, 9252, 9374, 9056, 9172, 9212] dup_rows_per_dst=[2942, 3098, 2980, 2912, 3220, 2842, 3006, 3120] combine_rows_per_src=[9212, 9566, 9340, 9252, 9374, 9056, 9172, 9212] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267182080 dispatch_meta_bytes=344136 combine_payload_bytes=267182080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267526216 ingress_bytes=301785088 link_bytes=301785088 token_bytes=534708296 weight_movement_bytes=34603008 total_nvlink_bytes=569311304 weight_to_token_ratio=0.064714 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135592990e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=65032 dup_rows=24008 dup_rows_remote=21072 dedup_ratio=0.244222 router_tokens_per_expert=[818, 717, 752, 773, 710, 760, 747, 743, 753, 834, 724, 766, 742, 762, 739, 744, 788, 786, 815, 799, 779, 749, 726, 761, 824, 782, 775, 774, 823, 763, 752, 758, 762, 847, 804, 807, 734, 753, 756, 793, 810, 798, 792, 764, 768, 808, 776, 789, 740, 747, 762, 740, 749, 708, 731, 779, 720, 797, 796, 713, 750, 771, 760, 790] payload_rows_per_dst=[9132, 9250, 9250, 9156, 9398, 9502, 9344, 9264] dup_rows_per_dst=[2908, 2878, 3038, 2912, 3170, 3108, 3058, 2936] combine_rows_per_src=[9132, 9250, 9250, 9156, 9398, 9502, 9344, 9264] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74134 payload_rows_remote=64894 dup_rows=24170 dup_rows_remote=21190 dedup_ratio=0.245870 router_tokens_per_expert=[768, 771, 791, 774, 774, 781, 809, 743, 797, 735, 761, 772, 753, 803, 772, 758, 780, 723, 779, 754, 748, 756, 781, 789, 766, 789, 810, 735, 779, 766, 730, 748, 765, 714, 765, 740, 798, 758, 814, 764, 786, 789, 771, 713, 787, 784, 805, 777, 811, 776, 730, 780, 739, 755, 756, 741, 761, 713, 786, 741, 764, 816, 791, 767] payload_rows_per_dst=[9348, 9236, 9240, 9334, 9256, 9356, 9138, 9226] dup_rows_per_dst=[3074, 3066, 2980, 2912, 2980, 3068, 3038, 3052] combine_rows_per_src=[9348, 9236, 9240, 9334, 9256, 9356, 9138, 9226] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74536 payload_rows_remote=65042 dup_rows=23768 dup_rows_remote=20682 dedup_ratio=0.241781 router_tokens_per_expert=[760, 698, 773, 766, 779, 782, 728, 781, 784, 838, 771, 783, 748, 842, 734, 790, 738, 775, 784, 792, 796, 797, 772, 772, 749, 759, 755, 814, 740, 773, 722, 748, 783, 702, 768, 728, 765, 772, 758, 780, 796, 754, 810, 774, 758, 756, 743, 706, 782, 812, 780, 780, 765, 741, 770, 769, 763, 759, 775, 790, 774, 768, 767, 761] payload_rows_per_dst=[9208, 9494, 9332, 9234, 9226, 9254, 9476, 9312] dup_rows_per_dst=[2926, 3086, 3120, 2886, 2886, 2940, 2922, 3002] combine_rows_per_src=[9208, 9494, 9332, 9234, 9226, 9254, 9476, 9312] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267067392 dispatch_meta_bytes=344080 combine_payload_bytes=267067392 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267411472 ingress_bytes=336273408 link_bytes=336273408 token_bytes=534478864 weight_movement_bytes=69206016 total_nvlink_bytes=603684880 weight_to_token_ratio=0.129483 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265369762e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266821632 dispatch_meta_bytes=344480 combine_payload_bytes=266821632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267166112 ingress_bytes=301424640 link_bytes=301424640 token_bytes=533987744 weight_movement_bytes=34603008 total_nvlink_bytes=568590752 weight_to_token_ratio=0.064801 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134236654e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74132 payload_rows_remote=65006 dup_rows=24172 dup_rows_remote=21210 dedup_ratio=0.245890 router_tokens_per_expert=[836, 761, 744, 766, 816, 729, 766, 801, 813, 738, 822, 784, 747, 757, 778, 785, 787, 746, 825, 754, 799, 796, 778, 732, 783, 731, 776, 731, 794, 747, 756, 776, 814, 792, 772, 760, 744, 743, 770, 750, 737, 809, 724, 796, 751, 723, 750, 746, 785, 772, 751, 758, 752, 753, 708, 765, 688, 771, 822, 750, 766, 748, 796, 832] payload_rows_per_dst=[9348, 9360, 9332, 9214, 9296, 9184, 9126, 9272] dup_rows_per_dst=[3090, 3088, 3102, 2974, 2994, 2888, 2962, 3074] combine_rows_per_src=[9348, 9360, 9332, 9214, 9296, 9184, 9126, 9272] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74214 payload_rows_remote=65046 dup_rows=24090 dup_rows_remote=21186 dedup_ratio=0.245056 router_tokens_per_expert=[810, 763, 770, 751, 736, 849, 782, 722, 796, 695, 755, 772, 749, 797, 775, 799, 769, 783, 748, 791, 747, 734, 827, 780, 809, 766, 790, 769, 754, 778, 754, 727, 764, 739, 723, 749, 771, 775, 727, 775, 738, 782, 739, 752, 691, 768, 761, 805, 778, 820, 768, 819, 750, 775, 774, 750, 740, 828, 750, 800, 760, 799, 762, 773] payload_rows_per_dst=[9300, 9246, 9248, 9272, 9228, 9168, 9356, 9396] dup_rows_per_dst=[3066, 3030, 3110, 3022, 2818, 2904, 3112, 3028] combine_rows_per_src=[9300, 9246, 9248, 9272, 9228, 9168, 9356, 9396] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344416 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266715488 ingress_bytes=370180096 link_bytes=370180096 token_bytes=533086560 weight_movement_bytes=103809024 total_nvlink_bytes=636895584 weight_to_token_ratio=0.194732 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392957899e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74070 payload_rows_remote=64678 dup_rows=24234 dup_rows_remote=21260 dedup_ratio=0.246521 router_tokens_per_expert=[755, 718, 773, 776, 788, 767, 804, 756, 788, 823, 748, 781, 769, 745, 797, 776, 734, 763, 755, 750, 781, 754, 808, 739, 784, 755, 832, 752, 749, 749, 764, 774, 760, 800, 795, 829, 759, 749, 751, 753, 759, 745, 734, 772, 771, 747, 725, 784, 780, 824, 758, 735, 776, 835, 789, 762, 749, 728, 772, 704, 801, 780, 762, 757] payload_rows_per_dst=[9392, 9364, 9092, 9270, 9334, 9104, 9394, 9120] dup_rows_per_dst=[2974, 3128, 2984, 3048, 3058, 2984, 3124, 2934] combine_rows_per_src=[9392, 9364, 9092, 9270, 9334, 9104, 9394, 9120] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73986 payload_rows_remote=64660 dup_rows=24318 dup_rows_remote=21150 dedup_ratio=0.247375 router_tokens_per_expert=[786, 773, 777, 771, 751, 762, 840, 747, 773, 775, 742, 819, 785, 776, 769, 808, 746, 754, 776, 743, 778, 768, 788, 737, 785, 722, 790, 748, 729, 843, 732, 751, 760, 814, 760, 795, 782, 789, 718, 756, 817, 714, 772, 797, 732, 713, 772, 743, 779, 810, 784, 766, 743, 790, 703, 755, 767, 770, 773, 759, 717, 776, 789, 793] payload_rows_per_dst=[9260, 9326, 9156, 9204, 9264, 9192, 9248, 9336] dup_rows_per_dst=[3154, 3168, 3024, 2996, 3084, 2928, 3012, 2952] combine_rows_per_src=[9260, 9326, 9156, 9204, 9264, 9192, 9248, 9336] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265805824 dispatch_meta_bytes=344336 combine_payload_bytes=265805824 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266150160 ingress_bytes=404217856 link_bytes=404217856 token_bytes=531955984 weight_movement_bytes=138412032 total_nvlink_bytes=670368016 weight_to_token_ratio=0.260195 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521039250e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=342896 combine_payload_bytes=266412032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266754928 ingress_bytes=301015040 link_bytes=301015040 token_bytes=533166960 weight_movement_bytes=34603008 total_nvlink_bytes=567769968 weight_to_token_ratio=0.064901 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132695362e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267067392 dispatch_meta_bytes=344080 combine_payload_bytes=267067392 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267411472 ingress_bytes=336273408 link_bytes=336273408 token_bytes=534478864 weight_movement_bytes=69206016 total_nvlink_bytes=603684880 weight_to_token_ratio=0.129483 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265369762e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265961472 dispatch_meta_bytes=343920 combine_payload_bytes=265961472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266305392 ingress_bytes=300564480 link_bytes=300564480 token_bytes=532266864 weight_movement_bytes=34603008 total_nvlink_bytes=566869872 weight_to_token_ratio=0.065011 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130999941e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266289152 dispatch_meta_bytes=344504 combine_payload_bytes=266289152 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=266633656 ingress_bytes=439304192 link_bytes=439304192 token_bytes=532922808 weight_movement_bytes=173015040 total_nvlink_bytes=705937848 weight_to_token_ratio=0.324653 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.653066308e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266264576 dispatch_meta_bytes=344864 combine_payload_bytes=266264576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266609440 ingress_bytes=300867584 link_bytes=300867584 token_bytes=532874016 weight_movement_bytes=34603008 total_nvlink_bytes=567477024 weight_to_token_ratio=0.064937 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132140497e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=344928 combine_payload_bytes=266428416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266773344 ingress_bytes=301031424 link_bytes=301031424 token_bytes=533201760 weight_movement_bytes=34603008 total_nvlink_bytes=567804768 weight_to_token_ratio=0.064897 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132757013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74204 payload_rows_remote=65022 dup_rows=24100 dup_rows_remote=21112 dedup_ratio=0.245158 router_tokens_per_expert=[809, 770, 751, 779, 721, 761, 761, 750, 759, 767, 739, 812, 741, 719, 735, 763, 769, 737, 749, 759, 764, 807, 754, 808, 728, 834, 768, 764, 748, 792, 785, 797, 805, 783, 789, 769, 807, 814, 723, 767, 804, 767, 805, 795, 755, 735, 769, 767, 787, 798, 805, 729, 738, 743, 773, 759, 733, 758, 758, 780, 758, 738, 791, 750] payload_rows_per_dst=[9062, 9082, 9448, 9346, 9178, 9674, 9232, 9182] dup_rows_per_dst=[2894, 2986, 3094, 3086, 2758, 3298, 2996, 2988] combine_rows_per_src=[9062, 9082, 9448, 9346, 9178, 9674, 9232, 9182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=264921088 dispatch_meta_bytes=343752 combine_payload_bytes=264921088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265264840 ingress_bytes=299524096 link_bytes=299524096 token_bytes=530185928 weight_movement_bytes=34603008 total_nvlink_bytes=564788936 weight_to_token_ratio=0.065266 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127085060e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=64974 dup_rows=23992 dup_rows_remote=21038 dedup_ratio=0.244059 router_tokens_per_expert=[743, 720, 750, 789, 800, 766, 794, 784, 800, 730, 743, 812, 740, 812, 736, 740, 773, 774, 799, 755, 750, 749, 759, 766, 770, 782, 789, 765, 752, 763, 746, 779, 783, 746, 787, 765, 744, 789, 735, 770, 746, 758, 783, 781, 775, 789, 747, 811, 768, 769, 762, 802, 800, 758, 828, 768, 752, 799, 749, 738, 779, 748, 771, 722] payload_rows_per_dst=[9356, 9250, 9278, 9338, 9256, 9234, 9370, 9230] dup_rows_per_dst=[2936, 2976, 2972, 2954, 2982, 3146, 3140, 2886] combine_rows_per_src=[9356, 9250, 9278, 9338, 9256, 9234, 9370, 9230] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74036 payload_rows_remote=64770 dup_rows=24268 dup_rows_remote=21306 dedup_ratio=0.246867 router_tokens_per_expert=[836, 778, 761, 776, 764, 806, 737, 752, 782, 750, 781, 745, 772, 800, 744, 741, 769, 796, 735, 789, 800, 755, 733, 781, 794, 784, 737, 813, 724, 779, 768, 779, 764, 813, 754, 776, 762, 741, 761, 743, 791, 754, 774, 758, 751, 720, 802, 798, 784, 770, 759, 722, 780, 738, 809, 796, 751, 737, 791, 779, 727, 772, 739, 775] payload_rows_per_dst=[9340, 9226, 9190, 9266, 9266, 9296, 9268, 9184] dup_rows_per_dst=[3080, 3004, 3126, 3090, 2962, 3000, 3048, 2958] combine_rows_per_src=[9340, 9226, 9190, 9266, 9266, 9296, 9268, 9184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=264847360 dispatch_meta_bytes=343240 combine_payload_bytes=264847360 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=265190600 ingress_bytes=299450368 link_bytes=299450368 token_bytes=530037960 weight_movement_bytes=34603008 total_nvlink_bytes=564640968 weight_to_token_ratio=0.065284 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126807627e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266330112 dispatch_meta_bytes=344536 combine_payload_bytes=266330112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266674648 ingress_bytes=300933120 link_bytes=300933120 token_bytes=533004760 weight_movement_bytes=34603008 total_nvlink_bytes=567607768 weight_to_token_ratio=0.064921 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132387103e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74214 payload_rows_remote=65046 dup_rows=24090 dup_rows_remote=21186 dedup_ratio=0.245056 router_tokens_per_expert=[810, 763, 770, 751, 736, 849, 782, 722, 796, 695, 755, 772, 749, 797, 775, 799, 769, 783, 748, 791, 747, 734, 827, 780, 809, 766, 790, 769, 754, 778, 754, 727, 764, 739, 723, 749, 771, 775, 727, 775, 738, 782, 739, 752, 691, 768, 761, 805, 778, 820, 768, 819, 750, 775, 774, 750, 740, 828, 750, 800, 760, 799, 762, 773] payload_rows_per_dst=[9300, 9246, 9248, 9272, 9228, 9168, 9356, 9396] dup_rows_per_dst=[3066, 3030, 3110, 3022, 2818, 2904, 3112, 3028] combine_rows_per_src=[9300, 9246, 9248, 9272, 9228, 9168, 9356, 9396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74200 payload_rows_remote=64948 dup_rows=24104 dup_rows_remote=21120 dedup_ratio=0.245199 router_tokens_per_expert=[774, 823, 776, 746, 768, 812, 767, 771, 760, 769, 746, 743, 778, 716, 789, 714, 786, 702, 773, 783, 742, 787, 811, 734, 745, 764, 755, 795, 784, 743, 782, 775, 816, 792, 790, 734, 766, 786, 774, 769, 730, 768, 778, 813, 737, 766, 744, 741, 789, 786, 758, 760, 735, 772, 783, 809, 767, 740, 784, 735, 773, 787, 780, 777] payload_rows_per_dst=[9360, 9130, 9252, 9298, 9358, 9180, 9308, 9314] dup_rows_per_dst=[3114, 2900, 2984, 2988, 3096, 2974, 3076, 2972] combine_rows_per_src=[9360, 9130, 9252, 9298, 9358, 9180, 9308, 9314] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=344048 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266477552 ingress_bytes=300736512 link_bytes=300736512 token_bytes=532611056 weight_movement_bytes=34603008 total_nvlink_bytes=567214064 weight_to_token_ratio=0.064969 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131647283e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265297920 dispatch_meta_bytes=344304 combine_payload_bytes=265297920 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265642224 ingress_bytes=334503936 link_bytes=334503936 token_bytes=530940144 weight_movement_bytes=69206016 total_nvlink_bytes=600146160 weight_to_token_ratio=0.130346 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258711381e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74204 payload_rows_remote=65022 dup_rows=24100 dup_rows_remote=21112 dedup_ratio=0.245158 router_tokens_per_expert=[809, 770, 751, 779, 721, 761, 761, 750, 759, 767, 739, 812, 741, 719, 735, 763, 769, 737, 749, 759, 764, 807, 754, 808, 728, 834, 768, 764, 748, 792, 785, 797, 805, 783, 789, 769, 807, 814, 723, 767, 804, 767, 805, 795, 755, 735, 769, 767, 787, 798, 805, 729, 738, 743, 773, 759, 733, 758, 758, 780, 758, 738, 791, 750] payload_rows_per_dst=[9062, 9082, 9448, 9346, 9178, 9674, 9232, 9182] dup_rows_per_dst=[2894, 2986, 3094, 3086, 2758, 3298, 2996, 2988] combine_rows_per_src=[9062, 9082, 9448, 9346, 9178, 9674, 9232, 9182] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73986 payload_rows_remote=64660 dup_rows=24318 dup_rows_remote=21150 dedup_ratio=0.247375 router_tokens_per_expert=[786, 773, 777, 771, 751, 762, 840, 747, 773, 775, 742, 819, 785, 776, 769, 808, 746, 754, 776, 743, 778, 768, 788, 737, 785, 722, 790, 748, 729, 843, 732, 751, 760, 814, 760, 795, 782, 789, 718, 756, 817, 714, 772, 797, 732, 713, 772, 743, 779, 810, 784, 766, 743, 790, 703, 755, 767, 770, 773, 759, 717, 776, 789, 793] payload_rows_per_dst=[9260, 9326, 9156, 9204, 9264, 9192, 9248, 9336] dup_rows_per_dst=[3154, 3168, 3024, 2996, 3084, 2928, 3012, 2952] combine_rows_per_src=[9260, 9326, 9156, 9204, 9264, 9192, 9248, 9336] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74184 payload_rows_remote=65012 dup_rows=24120 dup_rows_remote=21114 dedup_ratio=0.245361 router_tokens_per_expert=[753, 778, 719, 795, 712, 763, 773, 784, 771, 755, 818, 802, 815, 811, 733, 827, 797, 765, 723, 759, 793, 780, 786, 757, 749, 774, 773, 767, 755, 755, 778, 731, 801, 778, 764, 801, 800, 815, 744, 794, 736, 686, 789, 787, 717, 703, 771, 760, 716, 720, 784, 776, 761, 791, 762, 779, 788, 779, 780, 770, 773, 736, 730, 810] payload_rows_per_dst=[9212, 9566, 9340, 9252, 9374, 9056, 9172, 9212] dup_rows_per_dst=[2942, 3098, 2980, 2912, 3220, 2842, 3006, 3120] combine_rows_per_src=[9212, 9566, 9340, 9252, 9374, 9056, 9172, 9212] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=64974 dup_rows=23992 dup_rows_remote=21038 dedup_ratio=0.244059 router_tokens_per_expert=[743, 720, 750, 789, 800, 766, 794, 784, 800, 730, 743, 812, 740, 812, 736, 740, 773, 774, 799, 755, 750, 749, 759, 766, 770, 782, 789, 765, 752, 763, 746, 779, 783, 746, 787, 765, 744, 789, 735, 770, 746, 758, 783, 781, 775, 789, 747, 811, 768, 769, 762, 802, 800, 758, 828, 768, 752, 799, 749, 738, 779, 748, 771, 722] payload_rows_per_dst=[9356, 9250, 9278, 9338, 9256, 9234, 9370, 9230] dup_rows_per_dst=[2936, 2976, 2972, 2954, 2982, 3146, 3140, 2886] combine_rows_per_src=[9356, 9250, 9278, 9338, 9256, 9234, 9370, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74158 payload_rows_remote=64850 dup_rows=24146 dup_rows_remote=21144 dedup_ratio=0.245626 router_tokens_per_expert=[800, 768, 760, 748, 789, 730, 744, 819, 757, 803, 770, 732, 789, 758, 751, 740, 776, 780, 789, 798, 767, 743, 721, 732, 768, 741, 787, 786, 749, 765, 775, 768, 794, 822, 735, 776, 764, 783, 770, 732, 786, 751, 764, 782, 746, 751, 761, 780, 741, 748, 742, 781, 761, 782, 838, 762, 791, 756, 783, 741, 784, 751, 808, 783] payload_rows_per_dst=[9332, 9252, 9194, 9282, 9252, 9220, 9308, 9318] dup_rows_per_dst=[2984, 2948, 3018, 2996, 3100, 3022, 3002, 3076] combine_rows_per_src=[9332, 9252, 9194, 9282, 9252, 9220, 9308, 9318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74352 payload_rows_remote=65142 dup_rows=23952 dup_rows_remote=20978 dedup_ratio=0.243652 router_tokens_per_expert=[748, 754, 799, 750, 783, 806, 774, 722, 752, 731, 737, 792, 729, 766, 774, 756, 773, 740, 732, 815, 804, 713, 777, 698, 761, 778, 779, 834, 778, 770, 776, 809, 740, 735, 820, 759, 750, 748, 758, 782, 769, 755, 799, 794, 763, 772, 760, 754, 761, 806, 791, 750, 756, 774, 761, 798, 757, 762, 769, 791, 786, 825, 724, 773] payload_rows_per_dst=[9350, 9212, 9232, 9462, 9210, 9264, 9360, 9262] dup_rows_per_dst=[2922, 2862, 2872, 3108, 2974, 3068, 3034, 3112] combine_rows_per_src=[9350, 9212, 9232, 9462, 9210, 9264, 9360, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=344928 combine_payload_bytes=266428416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266773344 ingress_bytes=301031424 link_bytes=301031424 token_bytes=533201760 weight_movement_bytes=34603008 total_nvlink_bytes=567804768 weight_to_token_ratio=0.064897 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132757013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=64988 dup_rows=24098 dup_rows_remote=21064 dedup_ratio=0.245138 router_tokens_per_expert=[725, 749, 776, 763, 731, 759, 786, 827, 783, 736, 773, 762, 797, 847, 759, 760, 774, 764, 752, 779, 753, 749, 808, 770, 747, 764, 735, 763, 775, 758, 756, 784, 793, 786, 751, 778, 760, 775, 791, 774, 767, 739, 753, 773, 779, 732, 788, 795, 733, 768, 743, 721, 742, 829, 777, 762, 761, 765, 761, 777, 793, 772, 803, 747] payload_rows_per_dst=[9282, 9382, 9242, 9154, 9366, 9218, 9276, 9286] dup_rows_per_dst=[2950, 3052, 3056, 3010, 3050, 3034, 2874, 3072] combine_rows_per_src=[9282, 9382, 9242, 9154, 9366, 9218, 9276, 9286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=65068 dup_rows=23966 dup_rows_remote=20890 dedup_ratio=0.243795 router_tokens_per_expert=[752, 745, 768, 736, 780, 777, 766, 769, 750, 767, 781, 774, 762, 767, 757, 815, 757, 747, 797, 743, 738, 787, 746, 767, 779, 831, 734, 786, 784, 762, 779, 725, 745, 789, 803, 754, 765, 798, 765, 752, 731, 782, 779, 739, 747, 779, 753, 776, 797, 800, 769, 764, 774, 749, 775, 792, 774, 778, 750, 786, 760, 791, 738, 770] payload_rows_per_dst=[9264, 9270, 9202, 9348, 9294, 9206, 9366, 9388] dup_rows_per_dst=[2922, 3076, 2962, 3012, 3048, 2966, 3074, 2906] combine_rows_per_src=[9264, 9270, 9202, 9348, 9294, 9206, 9366, 9388] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65094 dup_rows=23944 dup_rows_remote=20982 dedup_ratio=0.243571 router_tokens_per_expert=[743, 765, 766, 783, 747, 771, 785, 748, 784, 750, 757, 759, 803, 812, 732, 761, 746, 726, 770, 775, 792, 816, 703, 723, 728, 747, 732, 779, 712, 764, 808, 733, 748, 777, 770, 783, 731, 760, 815, 816, 778, 734, 768, 799, 765, 753, 800, 809, 773, 759, 783, 807, 766, 736, 746, 783, 755, 779, 800, 785, 797, 791, 786, 780] payload_rows_per_dst=[9266, 9378, 9126, 9328, 9420, 9352, 9060, 9430] dup_rows_per_dst=[2962, 2938, 2976, 2952, 3040, 3060, 2900, 3116] combine_rows_per_src=[9266, 9378, 9126, 9328, 9420, 9352, 9060, 9430] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266027008 dispatch_meta_bytes=344272 combine_payload_bytes=266027008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266371280 ingress_bytes=300630016 link_bytes=300630016 token_bytes=532398288 weight_movement_bytes=34603008 total_nvlink_bytes=567001296 weight_to_token_ratio=0.064995 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131246548e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74148 payload_rows_remote=64734 dup_rows=24156 dup_rows_remote=20992 dedup_ratio=0.245728 router_tokens_per_expert=[715, 724, 782, 773, 772, 787, 777, 726, 779, 730, 784, 815, 761, 769, 760, 762, 782, 783, 790, 806, 738, 765, 829, 784, 779, 716, 713, 819, 773, 758, 733, 725, 808, 768, 778, 760, 794, 740, 785, 774, 743, 748, 716, 795, 774, 767, 786, 736, 771, 791, 784, 770, 797, 777, 765, 706, 776, 753, 837, 745, 779, 771, 785, 764] payload_rows_per_dst=[9070, 9290, 9416, 9144, 9328, 9098, 9388, 9414] dup_rows_per_dst=[2982, 3030, 3096, 2888, 3086, 2826, 3084, 3164] combine_rows_per_src=[9070, 9290, 9416, 9144, 9328, 9098, 9388, 9414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=344048 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266477552 ingress_bytes=300736512 link_bytes=300736512 token_bytes=532611056 weight_movement_bytes=34603008 total_nvlink_bytes=567214064 weight_to_token_ratio=0.064969 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131647283e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265625600 dispatch_meta_bytes=343976 combine_payload_bytes=265625600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265969576 ingress_bytes=300228608 link_bytes=300228608 token_bytes=531595176 weight_movement_bytes=34603008 total_nvlink_bytes=566198184 weight_to_token_ratio=0.065093 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129736082e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266821632 dispatch_meta_bytes=344480 combine_payload_bytes=266821632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267166112 ingress_bytes=301424640 link_bytes=301424640 token_bytes=533987744 weight_movement_bytes=34603008 total_nvlink_bytes=568590752 weight_to_token_ratio=0.064801 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134236654e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266330112 dispatch_meta_bytes=344536 combine_payload_bytes=266330112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266674648 ingress_bytes=300933120 link_bytes=300933120 token_bytes=533004760 weight_movement_bytes=34603008 total_nvlink_bytes=567607768 weight_to_token_ratio=0.064921 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132387103e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=264847360 dispatch_meta_bytes=343240 combine_payload_bytes=264847360 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=265190600 ingress_bytes=299450368 link_bytes=299450368 token_bytes=530037960 weight_movement_bytes=34603008 total_nvlink_bytes=564640968 weight_to_token_ratio=0.065284 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126807627e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266289152 dispatch_meta_bytes=344504 combine_payload_bytes=266289152 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=266633656 ingress_bytes=439304192 link_bytes=439304192 token_bytes=532922808 weight_movement_bytes=173015040 total_nvlink_bytes=705937848 weight_to_token_ratio=0.324653 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.653066308e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266190848 dispatch_meta_bytes=344208 combine_payload_bytes=266190848 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266535056 ingress_bytes=369999872 link_bytes=369999872 token_bytes=532725904 weight_movement_bytes=103809024 total_nvlink_bytes=636534928 weight_to_token_ratio=0.194864 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392279731e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266518528 dispatch_meta_bytes=343832 combine_payload_bytes=266518528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266862360 ingress_bytes=301121536 link_bytes=301121536 token_bytes=533380888 weight_movement_bytes=34603008 total_nvlink_bytes=567983896 weight_to_token_ratio=0.064875 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133096098e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265150464 dispatch_meta_bytes=342904 combine_payload_bytes=265150464 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265493368 ingress_bytes=334356480 link_bytes=334356480 token_bytes=530643832 weight_movement_bytes=69206016 total_nvlink_bytes=599849848 weight_to_token_ratio=0.130419 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258156516e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266625024 dispatch_meta_bytes=344304 combine_payload_bytes=266625024 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266969328 ingress_bytes=335831040 link_bytes=335831040 token_bytes=533594352 weight_movement_bytes=69206016 total_nvlink_bytes=602800368 weight_to_token_ratio=0.129698 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263705167e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74542 payload_rows_remote=65204 dup_rows=23762 dup_rows_remote=20894 dedup_ratio=0.241720 router_tokens_per_expert=[810, 784, 705, 780, 799, 803, 778, 754, 785, 747, 750, 734, 759, 758, 734, 784, 731, 784, 796, 745, 754, 795, 784, 757, 825, 769, 764, 743, 752, 738, 766, 746, 785, 818, 775, 781, 784, 760, 764, 783, 812, 797, 806, 775, 789, 724, 777, 741, 783, 772, 721, 721, 829, 701, 732, 824, 758, 781, 729, 771, 748, 774, 764, 760] payload_rows_per_dst=[9368, 9234, 9266, 9338, 9446, 9358, 9264, 9268] dup_rows_per_dst=[3058, 2868, 3026, 2868, 3054, 3084, 2902, 2902] combine_rows_per_src=[9368, 9234, 9266, 9338, 9446, 9358, 9264, 9268] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=64980 dup_rows=24016 dup_rows_remote=20994 dedup_ratio=0.244303 router_tokens_per_expert=[769, 758, 813, 791, 718, 754, 747, 772, 743, 744, 727, 801, 732, 794, 764, 766, 830, 777, 761, 780, 752, 752, 757, 790, 784, 796, 806, 737, 741, 763, 785, 761, 789, 788, 750, 768, 765, 827, 724, 754, 795, 751, 732, 797, 785, 773, 801, 757, 789, 745, 782, 735, 725, 766, 789, 777, 773, 769, 774, 763, 686, 800, 791, 767] payload_rows_per_dst=[9314, 9226, 9334, 9374, 9308, 9312, 9132, 9288] dup_rows_per_dst=[2930, 2916, 3064, 2972, 3022, 3070, 3084, 2958] combine_rows_per_src=[9314, 9226, 9334, 9374, 9308, 9312, 9132, 9288] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12262195000 prefetch_bytes=622854144 reduce_bytes=761266176 weight_movement_bytes=1384120320 egress_bytes=6135054136 ingress_bytes=7511261184 link_bytes=7511261184 total_nvlink_bytes=13646315320 weight_to_token_ratio=0.112877 prefetch_share_of_ingress=0.082923 dedup_ratio=0.243981 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=22 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.228881353e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=64988 dup_rows=24098 dup_rows_remote=21064 dedup_ratio=0.245138 router_tokens_per_expert=[725, 749, 776, 763, 731, 759, 786, 827, 783, 736, 773, 762, 797, 847, 759, 760, 774, 764, 752, 779, 753, 749, 808, 770, 747, 764, 735, 763, 775, 758, 756, 784, 793, 786, 751, 778, 760, 775, 791, 774, 767, 739, 753, 773, 779, 732, 788, 795, 733, 768, 743, 721, 742, 829, 777, 762, 761, 765, 761, 777, 793, 772, 803, 747] payload_rows_per_dst=[9282, 9382, 9242, 9154, 9366, 9218, 9276, 9286] dup_rows_per_dst=[2950, 3052, 3056, 3010, 3050, 3034, 2874, 3072] combine_rows_per_src=[9282, 9382, 9242, 9154, 9366, 9218, 9276, 9286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12261720312 prefetch_bytes=726663168 reduce_bytes=622854144 weight_movement_bytes=1349517312 egress_bytes=6134817016 ingress_bytes=7476420608 link_bytes=7476420608 total_nvlink_bytes=13611237624 weight_to_token_ratio=0.110059 prefetch_share_of_ingress=0.097194 dedup_ratio=0.244081 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=18 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028043 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.223181254e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12266946016 prefetch_bytes=657457152 reduce_bytes=761266176 weight_movement_bytes=1418723328 egress_bytes=6137429472 ingress_bytes=7548239872 link_bytes=7548239872 total_nvlink_bytes=13685669344 weight_to_token_ratio=0.115654 prefetch_share_of_ingress=0.087101 dedup_ratio=0.243662 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=22 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029239 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.234931259e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74188 payload_rows_remote=64968 dup_rows=24116 dup_rows_remote=21106 dedup_ratio=0.245321 router_tokens_per_expert=[763, 769, 753, 788, 756, 812, 804, 765, 712, 780, 792, 761, 734, 807, 761, 793, 764, 738, 756, 764, 762, 792, 762, 777, 785, 763, 773, 749, 792, 743, 757, 748, 731, 723, 778, 761, 773, 765, 801, 801, 785, 762, 782, 746, 834, 755, 848, 736, 768, 741, 710, 767, 749, 779, 759, 741, 835, 801, 775, 799, 713, 719, 777, 763] payload_rows_per_dst=[9316, 9260, 9220, 9214, 9322, 9384, 9160, 9312] dup_rows_per_dst=[3104, 3020, 3010, 3006, 2944, 3112, 2868, 3052] combine_rows_per_src=[9316, 9260, 9220, 9214, 9322, 9384, 9160, 9312] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=267075584 dispatch_meta_bytes=344392 combine_payload_bytes=267075584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267419976 ingress_bytes=301678592 link_bytes=301678592 token_bytes=534495560 weight_movement_bytes=34603008 total_nvlink_bytes=569098568 weight_to_token_ratio=0.064740 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135192255e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266158080 dispatch_meta_bytes=343896 combine_payload_bytes=266158080 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266501976 ingress_bytes=335364096 link_bytes=335364096 token_bytes=532660056 weight_movement_bytes=69206016 total_nvlink_bytes=601866072 weight_to_token_ratio=0.129925 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261948094e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74148 payload_rows_remote=64734 dup_rows=24156 dup_rows_remote=20992 dedup_ratio=0.245728 router_tokens_per_expert=[715, 724, 782, 773, 772, 787, 777, 726, 779, 730, 784, 815, 761, 769, 760, 762, 782, 783, 790, 806, 738, 765, 829, 784, 779, 716, 713, 819, 773, 758, 733, 725, 808, 768, 778, 760, 794, 740, 785, 774, 743, 748, 716, 795, 774, 767, 786, 736, 771, 791, 784, 770, 797, 777, 765, 706, 776, 753, 837, 745, 779, 771, 785, 764] payload_rows_per_dst=[9070, 9290, 9416, 9144, 9328, 9098, 9388, 9414] dup_rows_per_dst=[2982, 3030, 3096, 2888, 3086, 2826, 3084, 3164] combine_rows_per_src=[9070, 9290, 9416, 9144, 9328, 9098, 9388, 9414] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=65068 dup_rows=23966 dup_rows_remote=20890 dedup_ratio=0.243795 router_tokens_per_expert=[752, 745, 768, 736, 780, 777, 766, 769, 750, 767, 781, 774, 762, 767, 757, 815, 757, 747, 797, 743, 738, 787, 746, 767, 779, 831, 734, 786, 784, 762, 779, 725, 745, 789, 803, 754, 765, 798, 765, 752, 731, 782, 779, 739, 747, 779, 753, 776, 797, 800, 769, 764, 774, 749, 775, 792, 774, 778, 750, 786, 760, 791, 738, 770] payload_rows_per_dst=[9264, 9270, 9202, 9348, 9294, 9206, 9366, 9388] dup_rows_per_dst=[2922, 3076, 2962, 3012, 3048, 2966, 3074, 2906] combine_rows_per_src=[9264, 9270, 9202, 9348, 9294, 9206, 9366, 9388] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74158 payload_rows_remote=64850 dup_rows=24146 dup_rows_remote=21144 dedup_ratio=0.245626 router_tokens_per_expert=[800, 768, 760, 748, 789, 730, 744, 819, 757, 803, 770, 732, 789, 758, 751, 740, 776, 780, 789, 798, 767, 743, 721, 732, 768, 741, 787, 786, 749, 765, 775, 768, 794, 822, 735, 776, 764, 783, 770, 732, 786, 751, 764, 782, 746, 751, 761, 780, 741, 748, 742, 781, 761, 782, 838, 762, 791, 756, 783, 741, 784, 751, 808, 783] payload_rows_per_dst=[9332, 9252, 9194, 9282, 9252, 9220, 9308, 9318] dup_rows_per_dst=[2984, 2948, 3018, 2996, 3100, 3022, 3002, 3076] combine_rows_per_src=[9332, 9252, 9194, 9282, 9252, 9220, 9308, 9318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12233130616 prefetch_bytes=657457152 reduce_bytes=795869184 weight_movement_bytes=1453326336 egress_bytes=6120514168 ingress_bytes=7565942784 link_bytes=7565942784 total_nvlink_bytes=13686456952 weight_to_token_ratio=0.118802 prefetch_share_of_ingress=0.086897 dedup_ratio=0.244675 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=23 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.237827547e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74542 payload_rows_remote=65204 dup_rows=23762 dup_rows_remote=20894 dedup_ratio=0.241720 router_tokens_per_expert=[810, 784, 705, 780, 799, 803, 778, 754, 785, 747, 750, 734, 759, 758, 734, 784, 731, 784, 796, 745, 754, 795, 784, 757, 825, 769, 764, 743, 752, 738, 766, 746, 785, 818, 775, 781, 784, 760, 764, 783, 812, 797, 806, 775, 789, 724, 777, 741, 783, 772, 721, 721, 829, 701, 732, 824, 758, 781, 729, 771, 748, 774, 764, 760] payload_rows_per_dst=[9368, 9234, 9266, 9338, 9446, 9358, 9264, 9268] dup_rows_per_dst=[3058, 2868, 3026, 2868, 3054, 3084, 2902, 2902] combine_rows_per_src=[9368, 9234, 9266, 9338, 9446, 9358, 9264, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74036 payload_rows_remote=64770 dup_rows=24268 dup_rows_remote=21306 dedup_ratio=0.246867 router_tokens_per_expert=[836, 778, 761, 776, 764, 806, 737, 752, 782, 750, 781, 745, 772, 800, 744, 741, 769, 796, 735, 789, 800, 755, 733, 781, 794, 784, 737, 813, 724, 779, 768, 779, 764, 813, 754, 776, 762, 741, 761, 743, 791, 754, 774, 758, 751, 720, 802, 798, 784, 770, 759, 722, 780, 738, 809, 796, 751, 737, 791, 779, 727, 772, 739, 775] payload_rows_per_dst=[9340, 9226, 9190, 9266, 9266, 9296, 9268, 9184] dup_rows_per_dst=[3080, 3004, 3126, 3090, 2962, 3000, 3048, 2958] combine_rows_per_src=[9340, 9226, 9190, 9266, 9266, 9296, 9268, 9184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266190848 dispatch_meta_bytes=344208 combine_payload_bytes=266190848 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266535056 ingress_bytes=369999872 link_bytes=369999872 token_bytes=532725904 weight_movement_bytes=103809024 total_nvlink_bytes=636534928 weight_to_token_ratio=0.194864 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392279731e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74058 payload_rows_remote=64720 dup_rows=24246 dup_rows_remote=21156 dedup_ratio=0.246643 router_tokens_per_expert=[756, 816, 727, 788, 770, 761, 802, 781, 734, 767, 756, 756, 747, 781, 753, 723, 794, 769, 733, 807, 748, 728, 787, 733, 786, 743, 786, 682, 754, 744, 787, 766, 772, 750, 789, 704, 765, 767, 800, 809, 752, 762, 796, 756, 767, 796, 769, 799, 823, 762, 776, 808, 757, 768, 781, 786, 785, 699, 804, 774, 771, 808, 752, 780] payload_rows_per_dst=[9338, 9368, 9288, 9408, 8926, 9250, 9438, 9042] dup_rows_per_dst=[3090, 3112, 2910, 3290, 2758, 3144, 3084, 2858] combine_rows_per_src=[9338, 9368, 9288, 9408, 8926, 9250, 9438, 9042] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12267437256 prefetch_bytes=622854144 reduce_bytes=795869184 weight_movement_bytes=1418723328 egress_bytes=6137674952 ingress_bytes=7548485632 link_bytes=7548485632 total_nvlink_bytes=13686160584 weight_to_token_ratio=0.115650 prefetch_share_of_ingress=0.082514 dedup_ratio=0.243349 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=23 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028370 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.234971467e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12252499184 prefetch_bytes=692060160 reduce_bytes=622854144 weight_movement_bytes=1314914304 egress_bytes=6130207984 ingress_bytes=7437205504 link_bytes=7437205504 total_nvlink_bytes=13567413488 weight_to_token_ratio=0.107318 prefetch_share_of_ingress=0.093054 dedup_ratio=0.244641 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=18 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.216765459e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=344296 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266453224 ingress_bytes=335314944 link_bytes=335314944 token_bytes=532562152 weight_movement_bytes=69206016 total_nvlink_bytes=601768168 weight_to_token_ratio=0.129949 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261763139e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74058 payload_rows_remote=64720 dup_rows=24246 dup_rows_remote=21156 dedup_ratio=0.246643 router_tokens_per_expert=[756, 816, 727, 788, 770, 761, 802, 781, 734, 767, 756, 756, 747, 781, 753, 723, 794, 769, 733, 807, 748, 728, 787, 733, 786, 743, 786, 682, 754, 744, 787, 766, 772, 750, 789, 704, 765, 767, 800, 809, 752, 762, 796, 756, 767, 796, 769, 799, 823, 762, 776, 808, 757, 768, 781, 786, 785, 699, 804, 774, 771, 808, 752, 780] payload_rows_per_dst=[9338, 9368, 9288, 9408, 8926, 9250, 9438, 9042] dup_rows_per_dst=[3090, 3112, 2910, 3290, 2758, 3144, 3084, 2858] combine_rows_per_src=[9338, 9368, 9288, 9408, 8926, 9250, 9438, 9042] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=267075584 dispatch_meta_bytes=344392 combine_payload_bytes=267075584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267419976 ingress_bytes=301678592 link_bytes=301678592 token_bytes=534495560 weight_movement_bytes=34603008 total_nvlink_bytes=569098568 weight_to_token_ratio=0.064740 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135192255e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265297920 dispatch_meta_bytes=344304 combine_payload_bytes=265297920 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265642224 ingress_bytes=334503936 link_bytes=334503936 token_bytes=530940144 weight_movement_bytes=69206016 total_nvlink_bytes=600146160 weight_to_token_ratio=0.130346 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258711381e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12262195000 prefetch_bytes=622854144 reduce_bytes=761266176 weight_movement_bytes=1384120320 egress_bytes=6135054136 ingress_bytes=7511261184 link_bytes=7511261184 total_nvlink_bytes=13646315320 weight_to_token_ratio=0.112877 prefetch_share_of_ingress=0.082923 dedup_ratio=0.243981 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=22 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.228881353e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265150464 dispatch_meta_bytes=342904 combine_payload_bytes=265150464 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265493368 ingress_bytes=334356480 link_bytes=334356480 token_bytes=530643832 weight_movement_bytes=69206016 total_nvlink_bytes=599849848 weight_to_token_ratio=0.130419 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258156516e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266518528 dispatch_meta_bytes=343832 combine_payload_bytes=266518528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266862360 ingress_bytes=301121536 link_bytes=301121536 token_bytes=533380888 weight_movement_bytes=34603008 total_nvlink_bytes=567983896 weight_to_token_ratio=0.064875 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133096098e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265625600 dispatch_meta_bytes=343976 combine_payload_bytes=265625600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265969576 ingress_bytes=300228608 link_bytes=300228608 token_bytes=531595176 weight_movement_bytes=34603008 total_nvlink_bytes=566198184 weight_to_token_ratio=0.065093 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129736082e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265093120 dispatch_meta_bytes=343504 combine_payload_bytes=265093120 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265436624 ingress_bytes=334299136 link_bytes=334299136 token_bytes=530529744 weight_movement_bytes=69206016 total_nvlink_bytes=599735760 weight_to_token_ratio=0.130447 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257940735e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12262822600 prefetch_bytes=761266176 reduce_bytes=415236096 weight_movement_bytes=1176502272 egress_bytes=6135370440 ingress_bytes=7303954432 link_bytes=7303954432 total_nvlink_bytes=13439324872 weight_to_token_ratio=0.095941 prefetch_share_of_ingress=0.104227 dedup_ratio=0.243957 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=12 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028152 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.194964892e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12233130616 prefetch_bytes=657457152 reduce_bytes=795869184 weight_movement_bytes=1453326336 egress_bytes=6120514168 ingress_bytes=7565942784 link_bytes=7565942784 total_nvlink_bytes=13686456952 weight_to_token_ratio=0.118802 prefetch_share_of_ingress=0.086897 dedup_ratio=0.244675 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=23 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.237827547e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12266946016 prefetch_bytes=657457152 reduce_bytes=761266176 weight_movement_bytes=1418723328 egress_bytes=6137429472 ingress_bytes=7548239872 link_bytes=7548239872 total_nvlink_bytes=13685669344 weight_to_token_ratio=0.115654 prefetch_share_of_ingress=0.087101 dedup_ratio=0.243662 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=22 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029239 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.234931259e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12261720312 prefetch_bytes=726663168 reduce_bytes=622854144 weight_movement_bytes=1349517312 egress_bytes=6134817016 ingress_bytes=7476420608 link_bytes=7476420608 total_nvlink_bytes=13611237624 weight_to_token_ratio=0.110059 prefetch_share_of_ingress=0.097194 dedup_ratio=0.244081 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=18 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028043 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.223181254e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265093120 dispatch_meta_bytes=343504 combine_payload_bytes=265093120 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265436624 ingress_bytes=334299136 link_bytes=334299136 token_bytes=530529744 weight_movement_bytes=69206016 total_nvlink_bytes=599735760 weight_to_token_ratio=0.130447 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257940735e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12267437256 prefetch_bytes=622854144 reduce_bytes=795869184 weight_movement_bytes=1418723328 egress_bytes=6137674952 ingress_bytes=7548485632 link_bytes=7548485632 total_nvlink_bytes=13686160584 weight_to_token_ratio=0.115650 prefetch_share_of_ingress=0.082514 dedup_ratio=0.243349 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=23 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028370 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.234971467e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12233329296 prefetch_bytes=726663168 reduce_bytes=692060160 weight_movement_bytes=1418723328 egress_bytes=6120614544 ingress_bytes=7531438080 link_bytes=7531438080 total_nvlink_bytes=13652052624 weight_to_token_ratio=0.115972 prefetch_share_of_ingress=0.096484 dedup_ratio=0.244823 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=20 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027826 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.232182398e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none - [2026-08-26 15:10:22] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 789.7 | throughput per GPU (TFLOP/s/GPU): 161.6 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.098793E+01 | loss scale: 1.0 | grad norm: 435.853 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74432 payload_rows_remote=65038 dup_rows=23872 dup_rows_remote=20734 dedup_ratio=0.242839 router_tokens_per_expert=[733, 778, 772, 776, 729, 737, 761, 768, 781, 751, 773, 771, 793, 768, 771, 719, 810, 792, 781, 768, 759, 846, 744, 766, 796, 751, 810, 771, 767, 755, 771, 744, 801, 767, 735, 766, 776, 742, 741, 758, 790, 792, 743, 716, 793, 771, 785, 750, 759, 750, 774, 789, 827, 759, 765, 752, 817, 768, 764, 794, 742, 770, 739, 745] payload_rows_per_dst=[9194, 9164, 9394, 9384, 9296, 9324, 9328, 9348] dup_rows_per_dst=[2914, 3090, 3138, 2946, 2876, 2956, 3022, 2930] combine_rows_per_src=[9194, 9164, 9394, 9384, 9296, 9324, 9328, 9348] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74176 payload_rows_remote=64644 dup_rows=24128 dup_rows_remote=20738 dedup_ratio=0.245443 router_tokens_per_expert=[814, 758, 812, 743, 732, 739, 779, 759, 745, 712, 752, 790, 812, 792, 832, 777, 706, 727, 769, 730, 770, 722, 760, 727, 779, 751, 754, 741, 725, 807, 760, 758, 760, 739, 796, 761, 723, 753, 789, 777, 784, 790, 832, 801, 775, 812, 767, 751, 749, 768, 785, 733, 799, 770, 788, 769, 823, 785, 771, 788, 758, 778, 772, 772] payload_rows_per_dst=[9254, 9132, 9024, 9166, 9234, 9440, 9394, 9532] dup_rows_per_dst=[3018, 2826, 2798, 2984, 2962, 3222, 2928, 3390] combine_rows_per_src=[9254, 9132, 9024, 9166, 9234, 9440, 9394, 9532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=64980 dup_rows=24016 dup_rows_remote=20994 dedup_ratio=0.244303 router_tokens_per_expert=[769, 758, 813, 791, 718, 754, 747, 772, 743, 744, 727, 801, 732, 794, 764, 766, 830, 777, 761, 780, 752, 752, 757, 790, 784, 796, 806, 737, 741, 763, 785, 761, 789, 788, 750, 768, 765, 827, 724, 754, 795, 751, 732, 797, 785, 773, 801, 757, 789, 745, 782, 735, 725, 766, 789, 777, 773, 769, 774, 763, 686, 800, 791, 767] payload_rows_per_dst=[9314, 9226, 9334, 9374, 9308, 9312, 9132, 9288] dup_rows_per_dst=[2930, 2916, 3064, 2972, 3022, 3070, 3084, 2958] combine_rows_per_src=[9314, 9226, 9334, 9374, 9308, 9312, 9132, 9288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266158080 dispatch_meta_bytes=343896 combine_payload_bytes=266158080 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266501976 ingress_bytes=335364096 link_bytes=335364096 token_bytes=532660056 weight_movement_bytes=69206016 total_nvlink_bytes=601866072 weight_to_token_ratio=0.129925 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261948094e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12252499184 prefetch_bytes=692060160 reduce_bytes=622854144 weight_movement_bytes=1314914304 egress_bytes=6130207984 ingress_bytes=7437205504 link_bytes=7437205504 total_nvlink_bytes=13567413488 weight_to_token_ratio=0.107318 prefetch_share_of_ingress=0.093054 dedup_ratio=0.244641 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=18 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.216765459e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none + [2026-08-26 15:58:11] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 793.7 | throughput per GPU (TFLOP/s/GPU): 160.7 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.098793E+01 | loss scale: 1.0 | grad norm: 435.853 | number of skipped iterations: 0 | number of nan iterations: 0 | WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74470 payload_rows_remote=65184 dup_rows=23834 dup_rows_remote=20870 dedup_ratio=0.242452 router_tokens_per_expert=[799, 805, 774, 730, 767, 836, 752, 791, 770, 755, 816, 735, 758, 777, 746, 768, 796, 732, 748, 766, 769, 785, 770, 784, 729, 778, 726, 786, 760, 736, 760, 805, 765, 773, 752, 738, 753, 773, 759, 796, 750, 816, 772, 742, 802, 767, 714, 743, 724, 736, 763, 775, 793, 848, 784, 763, 789, 755, 774, 769, 724, 771, 767, 793] payload_rows_per_dst=[9378, 9286, 9306, 9250, 9250, 9286, 9312, 9402] dup_rows_per_dst=[3130, 2964, 2994, 2910, 2968, 2926, 3060, 2882] combine_rows_per_src=[9378, 9286, 9306, 9250, 9250, 9286, 9312, 9402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74432 payload_rows_remote=65038 dup_rows=23872 dup_rows_remote=20734 dedup_ratio=0.242839 router_tokens_per_expert=[733, 778, 772, 776, 729, 737, 761, 768, 781, 751, 773, 771, 793, 768, 771, 719, 810, 792, 781, 768, 759, 846, 744, 766, 796, 751, 810, 771, 767, 755, 771, 744, 801, 767, 735, 766, 776, 742, 741, 758, 790, 792, 743, 716, 793, 771, 785, 750, 759, 750, 774, 789, 827, 759, 765, 752, 817, 768, 764, 794, 742, 770, 739, 745] payload_rows_per_dst=[9194, 9164, 9394, 9384, 9296, 9324, 9328, 9348] dup_rows_per_dst=[2914, 3090, 3138, 2946, 2876, 2956, 3022, 2930] combine_rows_per_src=[9194, 9164, 9394, 9384, 9296, 9324, 9328, 9348] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74154 payload_rows_remote=64890 dup_rows=24150 dup_rows_remote=21142 dedup_ratio=0.245667 router_tokens_per_expert=[772, 747, 781, 773, 733, 763, 744, 735, 744, 742, 786, 805, 806, 768, 844, 805, 757, 754, 751, 791, 778, 725, 759, 757, 697, 730, 785, 796, 803, 723, 783, 787, 784, 806, 793, 735, 792, 767, 740, 801, 770, 791, 795, 761, 770, 741, 760, 748, 779, 812, 753, 762, 813, 781, 779, 771, 718, 748, 768, 746, 763, 771, 734, 776] payload_rows_per_dst=[9136, 9480, 9180, 9180, 9328, 9264, 9382, 9204] dup_rows_per_dst=[2960, 3120, 2964, 3028, 3108, 3008, 3118, 2844] combine_rows_per_src=[9136, 9480, 9180, 9180, 9328, 9264, 9382, 9204] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74176 payload_rows_remote=64644 dup_rows=24128 dup_rows_remote=20738 dedup_ratio=0.245443 router_tokens_per_expert=[814, 758, 812, 743, 732, 739, 779, 759, 745, 712, 752, 790, 812, 792, 832, 777, 706, 727, 769, 730, 770, 722, 760, 727, 779, 751, 754, 741, 725, 807, 760, 758, 760, 739, 796, 761, 723, 753, 789, 777, 784, 790, 832, 801, 775, 812, 767, 751, 749, 768, 785, 733, 799, 770, 788, 769, 823, 785, 771, 788, 758, 778, 772, 772] payload_rows_per_dst=[9254, 9132, 9024, 9166, 9234, 9440, 9394, 9532] dup_rows_per_dst=[3018, 2826, 2798, 2984, 2962, 3222, 2928, 3390] combine_rows_per_src=[9254, 9132, 9024, 9166, 9234, 9440, 9394, 9532] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74156 payload_rows_remote=64914 dup_rows=24148 dup_rows_remote=21040 dedup_ratio=0.245646 router_tokens_per_expert=[818, 694, 803, 808, 744, 780, 761, 714, 771, 761, 803, 735, 753, 775, 760, 771, 741, 724, 760, 785, 788, 749, 753, 775, 717, 800, 763, 772, 780, 808, 772, 763, 790, 769, 753, 717, 781, 806, 775, 740, 816, 757, 713, 762, 756, 797, 739, 831, 832, 771, 815, 791, 750, 741, 757, 768, 800, 742, 761, 786, 706, 807, 745, 777] payload_rows_per_dst=[9246, 9262, 9258, 9242, 9230, 9306, 9342, 9270] dup_rows_per_dst=[2998, 2996, 2892, 3108, 3032, 3036, 3108, 2978] combine_rows_per_src=[9246, 9262, 9258, 9242, 9230, 9306, 9342, 9270] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=65140 dup_rows=23926 dup_rows_remote=21004 dedup_ratio=0.243388 router_tokens_per_expert=[763, 770, 806, 740, 761, 758, 779, 781, 738, 802, 733, 780, 736, 730, 778, 813, 806, 776, 756, 859, 794, 789, 784, 770, 794, 736, 778, 740, 755, 743, 796, 779, 806, 763, 752, 726, 795, 764, 725, 749, 747, 775, 758, 705, 806, 797, 767, 763, 767, 803, 750, 745, 807, 727, 767, 739, 782, 752, 802, 754, 762, 770, 753, 751] payload_rows_per_dst=[9290, 9272, 9546, 9216, 9238, 9206, 9288, 9322] dup_rows_per_dst=[3026, 2948, 3122, 3026, 2922, 3030, 2922, 2930] combine_rows_per_src=[9290, 9272, 9546, 9216, 9238, 9206, 9288, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74326 payload_rows_remote=64990 dup_rows=23978 dup_rows_remote=20788 dedup_ratio=0.243917 router_tokens_per_expert=[774, 782, 791, 792, 725, 798, 724, 724, 784, 795, 738, 776, 763, 775, 823, 758, 821, 765, 778, 790, 758, 720, 771, 803, 790, 776, 734, 750, 751, 765, 745, 726, 769, 783, 784, 780, 760, 767, 746, 758, 772, 790, 793, 808, 736, 745, 745, 778, 735, 786, 839, 781, 796, 710, 765, 767, 763, 774, 775, 794, 743, 725, 755, 765] payload_rows_per_dst=[9336, 9342, 9168, 9508, 9432, 9214, 9078, 9248] dup_rows_per_dst=[3190, 3084, 2964, 3110, 3148, 2834, 2708, 2940] combine_rows_per_src=[9336, 9342, 9168, 9508, 9432, 9214, 9078, 9248] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=64966 dup_rows=24046 dup_rows_remote=21024 dedup_ratio=0.244609 router_tokens_per_expert=[813, 748, 796, 752, 804, 702, 799, 703, 787, 778, 800, 754, 777, 780, 774, 719, 781, 769, 744, 793, 777, 818, 798, 798, 756, 795, 745, 809, 756, 756, 757, 749, 737, 745, 778, 738, 783, 779, 782, 798, 794, 787, 762, 756, 727, 774, 760, 766, 770, 765, 768, 797, 769, 769, 764, 755, 741, 779, 764, 778, 756, 748, 719, 757] payload_rows_per_dst=[9370, 9264, 9498, 9170, 9320, 9214, 9292, 9130] dup_rows_per_dst=[2864, 3074, 3058, 3076, 2960, 3038, 3022, 2954] combine_rows_per_src=[9370, 9264, 9498, 9170, 9320, 9214, 9292, 9130] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=343088 combine_payload_bytes=266395648 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266738736 ingress_bytes=335601664 link_bytes=335601664 token_bytes=533134384 weight_movement_bytes=69206016 total_nvlink_bytes=602340400 weight_to_token_ratio=0.129810 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262842043e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=264781824 dispatch_meta_bytes=341528 combine_payload_bytes=264781824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265123352 ingress_bytes=299384832 link_bytes=299384832 token_bytes=529905176 weight_movement_bytes=34603008 total_nvlink_bytes=564508184 weight_to_token_ratio=0.065300 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126561020e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266993664 dispatch_meta_bytes=344216 combine_payload_bytes=266993664 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267337880 ingress_bytes=370802688 link_bytes=370802688 token_bytes=534331544 weight_movement_bytes=103809024 total_nvlink_bytes=638140568 weight_to_token_ratio=0.194278 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395300663e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265789440 dispatch_meta_bytes=344128 combine_payload_bytes=265789440 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266133568 ingress_bytes=334995456 link_bytes=334995456 token_bytes=531923008 weight_movement_bytes=69206016 total_nvlink_bytes=601129024 weight_to_token_ratio=0.130105 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260560932e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=343088 combine_payload_bytes=266395648 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266738736 ingress_bytes=335601664 link_bytes=335601664 token_bytes=533134384 weight_movement_bytes=69206016 total_nvlink_bytes=602340400 weight_to_token_ratio=0.129810 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262842043e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265789440 dispatch_meta_bytes=344128 combine_payload_bytes=265789440 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266133568 ingress_bytes=334995456 link_bytes=334995456 token_bytes=531923008 weight_movement_bytes=69206016 total_nvlink_bytes=601129024 weight_to_token_ratio=0.130105 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260560932e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=264781824 dispatch_meta_bytes=341528 combine_payload_bytes=264781824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265123352 ingress_bytes=299384832 link_bytes=299384832 token_bytes=529905176 weight_movement_bytes=34603008 total_nvlink_bytes=564508184 weight_to_token_ratio=0.065300 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126561020e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265887744 dispatch_meta_bytes=343816 combine_payload_bytes=265887744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266231560 ingress_bytes=300490752 link_bytes=300490752 token_bytes=532119304 weight_movement_bytes=34603008 total_nvlink_bytes=566722312 weight_to_token_ratio=0.065029 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130722508e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266813440 dispatch_meta_bytes=344576 combine_payload_bytes=266813440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267158016 ingress_bytes=301416448 link_bytes=301416448 token_bytes=533971456 weight_movement_bytes=34603008 total_nvlink_bytes=568574464 weight_to_token_ratio=0.064803 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134205828e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=343112 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266542152 ingress_bytes=300802048 link_bytes=300802048 token_bytes=532741192 weight_movement_bytes=34603008 total_nvlink_bytes=567344200 weight_to_token_ratio=0.064953 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131893890e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266100736 dispatch_meta_bytes=343960 combine_payload_bytes=266100736 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266444696 ingress_bytes=369909760 link_bytes=369909760 token_bytes=532545432 weight_movement_bytes=103809024 total_nvlink_bytes=636354456 weight_to_token_ratio=0.194930 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391940647e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74326 payload_rows_remote=64990 dup_rows=23978 dup_rows_remote=20788 dedup_ratio=0.243917 router_tokens_per_expert=[774, 782, 791, 792, 725, 798, 724, 724, 784, 795, 738, 776, 763, 775, 823, 758, 821, 765, 778, 790, 758, 720, 771, 803, 790, 776, 734, 750, 751, 765, 745, 726, 769, 783, 784, 780, 760, 767, 746, 758, 772, 790, 793, 808, 736, 745, 745, 778, 735, 786, 839, 781, 796, 710, 765, 767, 763, 774, 775, 794, 743, 725, 755, 765] payload_rows_per_dst=[9336, 9342, 9168, 9508, 9432, 9214, 9078, 9248] dup_rows_per_dst=[3190, 3084, 2964, 3110, 3148, 2834, 2708, 2940] combine_rows_per_src=[9336, 9342, 9168, 9508, 9432, 9214, 9078, 9248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=64938 dup_rows=23992 dup_rows_remote=20916 dedup_ratio=0.244059 router_tokens_per_expert=[817, 792, 791, 759, 801, 822, 740, 793, 830, 754, 764, 755, 741, 760, 761, 765, 797, 741, 794, 790, 769, 791, 796, 747, 769, 759, 758, 756, 781, 758, 751, 757, 769, 768, 763, 767, 780, 772, 743, 763, 763, 781, 726, 770, 777, 789, 783, 768, 750, 808, 711, 731, 725, 801, 756, 763, 713, 753, 789, 746, 801, 742, 755, 767] payload_rows_per_dst=[9444, 9228, 9374, 9194, 9272, 9392, 9210, 9198] dup_rows_per_dst=[3186, 3032, 3076, 2984, 2978, 2922, 2880, 2934] combine_rows_per_src=[9444, 9228, 9374, 9194, 9272, 9392, 9210, 9198] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=64802 dup_rows=24136 dup_rows_remote=20994 dedup_ratio=0.245524 router_tokens_per_expert=[712, 735, 777, 734, 742, 744, 770, 782, 741, 756, 763, 783, 815, 802, 779, 766, 789, 740, 747, 810, 754, 735, 782, 733, 775, 782, 765, 763, 752, 792, 769, 831, 747, 833, 783, 785, 730, 780, 763, 741, 735, 796, 754, 792, 785, 765, 776, 778, 779, 806, 772, 782, 733, 787, 724, 796, 729, 739, 789, 773, 779, 767, 758, 776] payload_rows_per_dst=[9206, 9262, 8782, 9368, 9306, 9556, 9322, 9366] dup_rows_per_dst=[2786, 2990, 2692, 3108, 3018, 3292, 3108, 3142] combine_rows_per_src=[9206, 9262, 8782, 9368, 9306, 9556, 9322, 9366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=65140 dup_rows=23926 dup_rows_remote=21004 dedup_ratio=0.243388 router_tokens_per_expert=[763, 770, 806, 740, 761, 758, 779, 781, 738, 802, 733, 780, 736, 730, 778, 813, 806, 776, 756, 859, 794, 789, 784, 770, 794, 736, 778, 740, 755, 743, 796, 779, 806, 763, 752, 726, 795, 764, 725, 749, 747, 775, 758, 705, 806, 797, 767, 763, 767, 803, 750, 745, 807, 727, 767, 739, 782, 752, 802, 754, 762, 770, 753, 751] payload_rows_per_dst=[9290, 9272, 9546, 9216, 9238, 9206, 9288, 9322] dup_rows_per_dst=[3026, 2948, 3122, 3026, 2922, 3030, 2922, 2930] combine_rows_per_src=[9290, 9272, 9546, 9216, 9238, 9206, 9288, 9322] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74318 payload_rows_remote=65098 dup_rows=23986 dup_rows_remote=20964 dedup_ratio=0.243998 router_tokens_per_expert=[770, 774, 740, 753, 740, 787, 742, 778, 746, 756, 774, 767, 755, 810, 743, 770, 780, 811, 748, 776, 757, 800, 800, 783, 717, 724, 817, 756, 780, 761, 792, 759, 822, 767, 804, 752, 776, 802, 748, 756, 747, 752, 756, 768, 802, 752, 744, 762, 746, 750, 800, 791, 795, 756, 778, 746, 722, 822, 759, 787, 779, 735, 766, 744] payload_rows_per_dst=[9276, 9220, 9386, 9200, 9402, 9172, 9386, 9276] dup_rows_per_dst=[2892, 3022, 3124, 3012, 3052, 2994, 2938, 2952] combine_rows_per_src=[9276, 9220, 9386, 9200, 9402, 9172, 9386, 9276] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=64938 dup_rows=23992 dup_rows_remote=20916 dedup_ratio=0.244059 router_tokens_per_expert=[817, 792, 791, 759, 801, 822, 740, 793, 830, 754, 764, 755, 741, 760, 761, 765, 797, 741, 794, 790, 769, 791, 796, 747, 769, 759, 758, 756, 781, 758, 751, 757, 769, 768, 763, 767, 780, 772, 743, 763, 763, 781, 726, 770, 777, 789, 783, 768, 750, 808, 711, 731, 725, 801, 756, 763, 713, 753, 789, 746, 801, 742, 755, 767] payload_rows_per_dst=[9444, 9228, 9374, 9194, 9272, 9392, 9210, 9198] dup_rows_per_dst=[3186, 3032, 3076, 2984, 2978, 2922, 2880, 2934] combine_rows_per_src=[9444, 9228, 9374, 9194, 9272, 9392, 9210, 9198] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=64802 dup_rows=24136 dup_rows_remote=20994 dedup_ratio=0.245524 router_tokens_per_expert=[712, 735, 777, 734, 742, 744, 770, 782, 741, 756, 763, 783, 815, 802, 779, 766, 789, 740, 747, 810, 754, 735, 782, 733, 775, 782, 765, 763, 752, 792, 769, 831, 747, 833, 783, 785, 730, 780, 763, 741, 735, 796, 754, 792, 785, 765, 776, 778, 779, 806, 772, 782, 733, 787, 724, 796, 729, 739, 789, 773, 779, 767, 758, 776] payload_rows_per_dst=[9206, 9262, 8782, 9368, 9306, 9556, 9322, 9366] dup_rows_per_dst=[2786, 2990, 2692, 3108, 3018, 3292, 3108, 3142] combine_rows_per_src=[9206, 9262, 8782, 9368, 9306, 9556, 9322, 9366] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=65138 dup_rows=23950 dup_rows_remote=21076 dedup_ratio=0.243632 router_tokens_per_expert=[769, 743, 772, 781, 800, 789, 765, 741, 793, 780, 773, 787, 772, 673, 772, 794, 746, 769, 798, 748, 745, 794, 761, 816, 797, 778, 743, 771, 777, 753, 778, 804, 722, 761, 785, 705, 764, 759, 798, 772, 743, 766, 710, 773, 779, 751, 748, 775, 802, 780, 795, 814, 728, 766, 757, 780, 768, 770, 727, 766, 777, 815, 801, 713] payload_rows_per_dst=[9320, 9280, 9354, 9390, 9236, 9216, 9328, 9230] dup_rows_per_dst=[3000, 3008, 3000, 3012, 2896, 2874, 3116, 3044] combine_rows_per_src=[9320, 9280, 9354, 9390, 9236, 9216, 9328, 9230] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265986048 dispatch_meta_bytes=343416 combine_payload_bytes=265986048 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266329464 ingress_bytes=369795072 link_bytes=369795072 token_bytes=532315512 weight_movement_bytes=103809024 total_nvlink_bytes=636124536 weight_to_token_ratio=0.195014 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391509085e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265428992 dispatch_meta_bytes=343184 combine_payload_bytes=265428992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265772176 ingress_bytes=300032000 link_bytes=300032000 token_bytes=531201168 weight_movement_bytes=34603008 total_nvlink_bytes=565804176 weight_to_token_ratio=0.065141 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128996261e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74298 payload_rows_remote=64994 dup_rows=24006 dup_rows_remote=21062 dedup_ratio=0.244202 router_tokens_per_expert=[767, 733, 753, 771, 764, 779, 762, 780, 808, 780, 792, 776, 744, 812, 787, 828, 737, 743, 796, 768, 768, 790, 754, 804, 802, 773, 737, 734, 793, 780, 751, 754, 784, 764, 754, 794, 718, 797, 746, 779, 772, 769, 778, 747, 760, 740, 797, 779, 786, 776, 757, 745, 733, 780, 786, 812, 731, 736, 740, 796, 697, 791, 771, 717] payload_rows_per_dst=[9310, 9364, 9334, 9304, 9432, 9304, 9230, 9020] dup_rows_per_dst=[2908, 3290, 2986, 2944, 2840, 2980, 3120, 2938] combine_rows_per_src=[9310, 9364, 9334, 9304, 9432, 9304, 9230, 9020] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74266 payload_rows_remote=64792 dup_rows=24038 dup_rows_remote=20992 dedup_ratio=0.244527 router_tokens_per_expert=[807, 790, 733, 789, 753, 766, 763, 758, 736, 768, 781, 760, 817, 727, 785, 735, 764, 768, 767, 768, 781, 750, 762, 780, 765, 796, 781, 770, 748, 739, 804, 765, 778, 781, 772, 792, 773, 729, 751, 699, 783, 736, 735, 777, 730, 775, 813, 777, 793, 746, 771, 761, 806, 745, 823, 754, 763, 786, 769, 791, 805, 731, 767, 764] payload_rows_per_dst=[9474, 9236, 9330, 9162, 9428, 9166, 9066, 9404] dup_rows_per_dst=[3046, 2982, 2950, 2972, 3154, 3086, 2900, 2948] combine_rows_per_src=[9474, 9236, 9330, 9162, 9428, 9166, 9066, 9404] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=344248 combine_payload_bytes=266641408 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266985656 ingress_bytes=370450432 link_bytes=370450432 token_bytes=533627064 weight_movement_bytes=103809024 total_nvlink_bytes=637436088 weight_to_token_ratio=0.194535 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393975152e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74240 payload_rows_remote=65102 dup_rows=24064 dup_rows_remote=21106 dedup_ratio=0.244792 router_tokens_per_expert=[779, 737, 802, 773, 748, 734, 716, 743, 746, 772, 750, 759, 759, 760, 796, 759, 786, 763, 770, 811, 808, 789, 803, 744, 772, 765, 771, 800, 784, 767, 766, 763, 760, 769, 737, 760, 697, 780, 800, 745, 761, 768, 760, 761, 755, 773, 793, 798, 769, 764, 797, 772, 731, 790, 781, 787, 790, 770, 776, 755, 736, 791, 776, 755] payload_rows_per_dst=[9222, 9194, 9504, 9332, 9138, 9314, 9288, 9248] dup_rows_per_dst=[2842, 3008, 3044, 3044, 2958, 3024, 3094, 3050] combine_rows_per_src=[9222, 9194, 9504, 9332, 9138, 9314, 9288, 9248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=343112 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266542152 ingress_bytes=300802048 link_bytes=300802048 token_bytes=532741192 weight_movement_bytes=34603008 total_nvlink_bytes=567344200 weight_to_token_ratio=0.064953 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131893890e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266813440 dispatch_meta_bytes=344576 combine_payload_bytes=266813440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267158016 ingress_bytes=301416448 link_bytes=301416448 token_bytes=533971456 weight_movement_bytes=34603008 total_nvlink_bytes=568574464 weight_to_token_ratio=0.064803 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134205828e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74588 payload_rows_remote=65328 dup_rows=23716 dup_rows_remote=20658 dedup_ratio=0.241252 router_tokens_per_expert=[738, 787, 743, 744, 816, 793, 775, 800, 796, 771, 746, 746, 762, 771, 734, 768, 783, 792, 795, 785, 761, 749, 796, 723, 787, 775, 785, 738, 759, 770, 784, 741, 763, 728, 733, 747, 779, 751, 776, 820, 779, 804, 708, 739, 744, 745, 725, 762, 782, 807, 787, 722, 783, 758, 753, 767, 770, 744, 804, 854, 768, 766, 760, 811] payload_rows_per_dst=[9456, 9284, 9332, 9266, 9278, 9124, 9260, 9588] dup_rows_per_dst=[2936, 2904, 3036, 3012, 2916, 2888, 3058, 2966] combine_rows_per_src=[9456, 9284, 9332, 9266, 9278, 9124, 9260, 9588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266805248 dispatch_meta_bytes=344856 combine_payload_bytes=266805248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267150104 ingress_bytes=301408256 link_bytes=301408256 token_bytes=533955352 weight_movement_bytes=34603008 total_nvlink_bytes=568558360 weight_to_token_ratio=0.064805 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134175002e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265986048 dispatch_meta_bytes=343416 combine_payload_bytes=265986048 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266329464 ingress_bytes=369795072 link_bytes=369795072 token_bytes=532315512 weight_movement_bytes=103809024 total_nvlink_bytes=636124536 weight_to_token_ratio=0.195014 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391509085e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265428992 dispatch_meta_bytes=343184 combine_payload_bytes=265428992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265772176 ingress_bytes=300032000 link_bytes=300032000 token_bytes=531201168 weight_movement_bytes=34603008 total_nvlink_bytes=565804176 weight_to_token_ratio=0.065141 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128996261e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265388032 dispatch_meta_bytes=343136 combine_payload_bytes=265388032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265731168 ingress_bytes=299991040 link_bytes=299991040 token_bytes=531119200 weight_movement_bytes=34603008 total_nvlink_bytes=565722208 weight_to_token_ratio=0.065151 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128842132e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266805248 dispatch_meta_bytes=344856 combine_payload_bytes=266805248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267150104 ingress_bytes=301408256 link_bytes=301408256 token_bytes=533955352 weight_movement_bytes=34603008 total_nvlink_bytes=568558360 weight_to_token_ratio=0.064805 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134175002e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266215424 dispatch_meta_bytes=344224 combine_payload_bytes=266215424 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266559648 ingress_bytes=335421440 link_bytes=335421440 token_bytes=532775072 weight_movement_bytes=69206016 total_nvlink_bytes=601981088 weight_to_token_ratio=0.129897 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262163875e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266657792 dispatch_meta_bytes=344832 combine_payload_bytes=266657792 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267002624 ingress_bytes=301260800 link_bytes=301260800 token_bytes=533660416 weight_movement_bytes=34603008 total_nvlink_bytes=568263424 weight_to_token_ratio=0.064841 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133620137e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267583488 dispatch_meta_bytes=343944 combine_payload_bytes=267583488 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267927432 ingress_bytes=336789504 link_bytes=336789504 token_bytes=535510920 weight_movement_bytes=69206016 total_nvlink_bytes=604716936 weight_to_token_ratio=0.129234 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.267311790e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74250 payload_rows_remote=64848 dup_rows=24054 dup_rows_remote=20880 dedup_ratio=0.244690 router_tokens_per_expert=[781, 730, 780, 744, 782, 774, 725, 735, 764, 771, 792, 790, 763, 755, 791, 780, 759, 758, 768, 772, 781, 778, 768, 791, 753, 777, 736, 755, 746, 717, 757, 765, 749, 825, 726, 772, 771, 803, 814, 756, 757, 738, 785, 777, 788, 753, 788, 782, 754, 759, 731, 736, 767, 783, 790, 795, 740, 781, 746, 776, 784, 806, 799, 783] payload_rows_per_dst=[8998, 9200, 8766, 9506, 9802, 9312, 9264, 9402] dup_rows_per_dst=[2920, 2968, 2564, 3190, 3394, 2878, 2966, 3174] combine_rows_per_src=[8998, 9200, 8766, 9506, 9802, 9312, 9264, 9402] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74460 payload_rows_remote=64958 dup_rows=23844 dup_rows_remote=20750 dedup_ratio=0.242554 router_tokens_per_expert=[771, 717, 838, 736, 711, 794, 773, 763, 754, 771, 772, 789, 742, 759, 785, 809, 812, 801, 805, 780, 726, 814, 790, 770, 767, 739, 743, 789, 773, 789, 758, 734, 721, 770, 744, 803, 762, 790, 753, 765, 794, 757, 790, 784, 755, 722, 723, 767, 760, 768, 800, 740, 757, 778, 776, 784, 742, 766, 753, 735, 759, 798, 761, 801] payload_rows_per_dst=[9328, 9354, 9502, 9284, 9238, 9164, 9298, 9292] dup_rows_per_dst=[2878, 3008, 3094, 2900, 2978, 3020, 3028, 2938] combine_rows_per_src=[9328, 9354, 9502, 9284, 9238, 9164, 9298, 9292] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=65136 dup_rows=23906 dup_rows_remote=21076 dedup_ratio=0.243184 router_tokens_per_expert=[769, 806, 792, 816, 772, 769, 795, 714, 832, 754, 742, 764, 744, 733, 754, 723, 761, 748, 814, 772, 791, 786, 806, 738, 762, 769, 807, 754, 795, 790, 790, 796, 719, 797, 754, 763, 776, 737, 693, 778, 791, 777, 761, 749, 779, 734, 755, 758, 767, 787, 729, 815, 777, 779, 767, 733, 773, 775, 750, 727, 772, 790, 750, 782] payload_rows_per_dst=[9440, 9262, 9424, 9304, 9228, 9246, 9266, 9228] dup_rows_per_dst=[3026, 2830, 3008, 3222, 2806, 2962, 3042, 3010] combine_rows_per_src=[9440, 9262, 9424, 9304, 9228, 9246, 9266, 9228] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74460 payload_rows_remote=64958 dup_rows=23844 dup_rows_remote=20750 dedup_ratio=0.242554 router_tokens_per_expert=[771, 717, 838, 736, 711, 794, 773, 763, 754, 771, 772, 789, 742, 759, 785, 809, 812, 801, 805, 780, 726, 814, 790, 770, 767, 739, 743, 789, 773, 789, 758, 734, 721, 770, 744, 803, 762, 790, 753, 765, 794, 757, 790, 784, 755, 722, 723, 767, 760, 768, 800, 740, 757, 778, 776, 784, 742, 766, 753, 735, 759, 798, 761, 801] payload_rows_per_dst=[9328, 9354, 9502, 9284, 9238, 9164, 9298, 9292] dup_rows_per_dst=[2878, 3008, 3094, 2900, 2978, 3020, 3028, 2938] combine_rows_per_src=[9328, 9354, 9502, 9284, 9238, 9164, 9298, 9292] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74346 payload_rows_remote=65034 dup_rows=23958 dup_rows_remote=21016 dedup_ratio=0.243713 router_tokens_per_expert=[737, 796, 795, 817, 773, 725, 805, 788, 790, 784, 730, 805, 766, 801, 748, 759, 725, 755, 791, 756, 796, 759, 777, 747, 761, 713, 808, 762, 781, 759, 805, 775, 743, 764, 764, 778, 772, 787, 751, 730, 764, 800, 801, 731, 731, 778, 730, 792, 722, 775, 771, 746, 749, 779, 778, 776, 717, 813, 760, 791, 735, 799, 765, 771] payload_rows_per_dst=[9404, 9346, 9198, 9344, 9264, 9312, 9228, 9250] dup_rows_per_dst=[3068, 3020, 3014, 2984, 2914, 2942, 2964, 3052] combine_rows_per_src=[9404, 9346, 9198, 9344, 9264, 9312, 9228, 9250] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265617408 dispatch_meta_bytes=342912 combine_payload_bytes=265617408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265960320 ingress_bytes=300220416 link_bytes=300220416 token_bytes=531577728 weight_movement_bytes=34603008 total_nvlink_bytes=566180736 weight_to_token_ratio=0.065095 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129705256e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=342832 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266410800 ingress_bytes=369876992 link_bytes=369876992 token_bytes=532478768 weight_movement_bytes=103809024 total_nvlink_bytes=636287792 weight_to_token_ratio=0.194954 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391817343e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74266 payload_rows_remote=64792 dup_rows=24038 dup_rows_remote=20992 dedup_ratio=0.244527 router_tokens_per_expert=[807, 790, 733, 789, 753, 766, 763, 758, 736, 768, 781, 760, 817, 727, 785, 735, 764, 768, 767, 768, 781, 750, 762, 780, 765, 796, 781, 770, 748, 739, 804, 765, 778, 781, 772, 792, 773, 729, 751, 699, 783, 736, 735, 777, 730, 775, 813, 777, 793, 746, 771, 761, 806, 745, 823, 754, 763, 786, 769, 791, 805, 731, 767, 764] payload_rows_per_dst=[9474, 9236, 9330, 9162, 9428, 9166, 9066, 9404] dup_rows_per_dst=[3046, 2982, 2950, 2972, 3154, 3086, 2900, 2948] combine_rows_per_src=[9474, 9236, 9330, 9162, 9428, 9166, 9066, 9404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74240 payload_rows_remote=65102 dup_rows=24064 dup_rows_remote=21106 dedup_ratio=0.244792 router_tokens_per_expert=[779, 737, 802, 773, 748, 734, 716, 743, 746, 772, 750, 759, 759, 760, 796, 759, 786, 763, 770, 811, 808, 789, 803, 744, 772, 765, 771, 800, 784, 767, 766, 763, 760, 769, 737, 760, 697, 780, 800, 745, 761, 768, 760, 761, 755, 773, 793, 798, 769, 764, 797, 772, 731, 790, 781, 787, 790, 770, 776, 755, 736, 791, 776, 755] payload_rows_per_dst=[9222, 9194, 9504, 9332, 9138, 9314, 9288, 9248] dup_rows_per_dst=[2842, 3008, 3044, 3044, 2958, 3024, 3094, 3050] combine_rows_per_src=[9222, 9194, 9504, 9332, 9138, 9314, 9288, 9248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74250 payload_rows_remote=64848 dup_rows=24054 dup_rows_remote=20880 dedup_ratio=0.244690 router_tokens_per_expert=[781, 730, 780, 744, 782, 774, 725, 735, 764, 771, 792, 790, 763, 755, 791, 780, 759, 758, 768, 772, 781, 778, 768, 791, 753, 777, 736, 755, 746, 717, 757, 765, 749, 825, 726, 772, 771, 803, 814, 756, 757, 738, 785, 777, 788, 753, 788, 782, 754, 759, 731, 736, 767, 783, 790, 795, 740, 781, 746, 776, 784, 806, 799, 783] payload_rows_per_dst=[8998, 9200, 8766, 9506, 9802, 9312, 9264, 9402] dup_rows_per_dst=[2920, 2968, 2564, 3190, 3394, 2878, 2966, 3174] combine_rows_per_src=[8998, 9200, 8766, 9506, 9802, 9312, 9264, 9402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266797056 dispatch_meta_bytes=344848 combine_payload_bytes=266797056 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267141904 ingress_bytes=336003072 link_bytes=336003072 token_bytes=533938960 weight_movement_bytes=69206016 total_nvlink_bytes=603144976 weight_to_token_ratio=0.129614 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264352509e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74260 payload_rows_remote=65028 dup_rows=24044 dup_rows_remote=20944 dedup_ratio=0.244588 router_tokens_per_expert=[746, 766, 793, 758, 753, 789, 778, 762, 747, 765, 784, 806, 737, 789, 793, 788, 827, 746, 762, 784, 763, 756, 738, 755, 778, 807, 772, 734, 748, 752, 791, 776, 774, 809, 772, 719, 758, 772, 803, 759, 718, 774, 785, 784, 778, 739, 787, 789, 751, 736, 735, 824, 736, 799, 783, 770, 755, 778, 750, 740, 785, 731, 762, 754] payload_rows_per_dst=[9356, 9382, 9218, 9348, 9232, 9250, 9254, 9220] dup_rows_per_dst=[2934, 3036, 3044, 2968, 3100, 3058, 3014, 2890] combine_rows_per_src=[9356, 9382, 9218, 9348, 9232, 9250, 9254, 9220] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74438 payload_rows_remote=65134 dup_rows=23866 dup_rows_remote=20876 dedup_ratio=0.242778 router_tokens_per_expert=[785, 733, 749, 772, 770, 776, 806, 756, 764, 749, 804, 762, 756, 749, 773, 770, 741, 799, 775, 769, 774, 785, 713, 772, 742, 767, 745, 803, 765, 770, 738, 750, 758, 748, 775, 797, 779, 749, 709, 771, 753, 760, 780, 828, 773, 835, 814, 734, 807, 758, 795, 750, 781, 774, 789, 768, 792, 790, 798, 767, 707, 731, 765, 735] payload_rows_per_dst=[9304, 9246, 9258, 9290, 9262, 9524, 9294, 9260] dup_rows_per_dst=[2990, 3008, 2998, 2870, 2910, 3034, 3146, 2910] combine_rows_per_src=[9304, 9246, 9258, 9290, 9262, 9524, 9294, 9260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74346 payload_rows_remote=65034 dup_rows=23958 dup_rows_remote=21016 dedup_ratio=0.243713 router_tokens_per_expert=[737, 796, 795, 817, 773, 725, 805, 788, 790, 784, 730, 805, 766, 801, 748, 759, 725, 755, 791, 756, 796, 759, 777, 747, 761, 713, 808, 762, 781, 759, 805, 775, 743, 764, 764, 778, 772, 787, 751, 730, 764, 800, 801, 731, 731, 778, 730, 792, 722, 775, 771, 746, 749, 779, 778, 776, 717, 813, 760, 791, 735, 799, 765, 771] payload_rows_per_dst=[9404, 9346, 9198, 9344, 9264, 9312, 9228, 9250] dup_rows_per_dst=[3068, 3020, 3014, 2984, 2914, 2942, 2964, 3052] combine_rows_per_src=[9404, 9346, 9198, 9344, 9264, 9312, 9228, 9250] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74200 payload_rows_remote=65028 dup_rows=24104 dup_rows_remote=21146 dedup_ratio=0.245199 router_tokens_per_expert=[776, 760, 750, 775, 786, 778, 729, 785, 764, 771, 780, 777, 806, 779, 735, 756, 815, 741, 851, 755, 840, 854, 784, 795, 769, 803, 753, 710, 758, 759, 742, 771, 795, 722, 752, 740, 710, 754, 725, 732, 753, 784, 819, 691, 744, 814, 773, 725, 760, 784, 800, 739, 763, 782, 742, 788, 725, 761, 758, 751, 755, 765, 834, 805] payload_rows_per_dst=[9306, 9390, 9548, 9172, 9000, 9236, 9316, 9232] dup_rows_per_dst=[2972, 2946, 3322, 2958, 2860, 2970, 3000, 3076] combine_rows_per_src=[9306, 9390, 9548, 9172, 9000, 9236, 9316, 9232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=342832 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266410800 ingress_bytes=369876992 link_bytes=369876992 token_bytes=532478768 weight_movement_bytes=103809024 total_nvlink_bytes=636287792 weight_to_token_ratio=0.194954 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391817343e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74480 payload_rows_remote=65190 dup_rows=23824 dup_rows_remote=20890 dedup_ratio=0.242350 router_tokens_per_expert=[783, 780, 748, 779, 787, 785, 765, 763, 758, 799, 756, 755, 795, 795, 724, 772, 799, 809, 734, 793, 759, 750, 797, 732, 735, 773, 769, 758, 745, 752, 716, 734, 751, 771, 747, 738, 776, 797, 780, 798, 808, 770, 763, 817, 767, 749, 777, 747, 775, 713, 778, 789, 760, 744, 752, 801, 789, 810, 729, 774, 777, 782, 815, 709] payload_rows_per_dst=[9308, 9288, 9316, 9244, 9272, 9410, 9290, 9352] dup_rows_per_dst=[3072, 3020, 3030, 2720, 3044, 2986, 2934, 3018] combine_rows_per_src=[9308, 9288, 9316, 9244, 9272, 9410, 9290, 9352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266379264 dispatch_meta_bytes=344200 combine_payload_bytes=266379264 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266723464 ingress_bytes=335585280 link_bytes=335585280 token_bytes=533102728 weight_movement_bytes=69206016 total_nvlink_bytes=602308744 weight_to_token_ratio=0.129817 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262780392e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265388032 dispatch_meta_bytes=343136 combine_payload_bytes=265388032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265731168 ingress_bytes=299991040 link_bytes=299991040 token_bytes=531119200 weight_movement_bytes=34603008 total_nvlink_bytes=565722208 weight_to_token_ratio=0.065151 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128842132e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266354688 dispatch_meta_bytes=343888 combine_payload_bytes=266354688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266698576 ingress_bytes=300957696 link_bytes=300957696 token_bytes=533053264 weight_movement_bytes=34603008 total_nvlink_bytes=567656272 weight_to_token_ratio=0.064915 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132479581e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265617408 dispatch_meta_bytes=342912 combine_payload_bytes=265617408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265960320 ingress_bytes=300220416 link_bytes=300220416 token_bytes=531577728 weight_movement_bytes=34603008 total_nvlink_bytes=566180736 weight_to_token_ratio=0.065095 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129705256e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266657792 dispatch_meta_bytes=344832 combine_payload_bytes=266657792 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267002624 ingress_bytes=301260800 link_bytes=301260800 token_bytes=533660416 weight_movement_bytes=34603008 total_nvlink_bytes=568263424 weight_to_token_ratio=0.064841 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133620137e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=344040 combine_payload_bytes=266788864 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267132904 ingress_bytes=301391872 link_bytes=301391872 token_bytes=533921768 weight_movement_bytes=34603008 total_nvlink_bytes=568524776 weight_to_token_ratio=0.064809 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134113350e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266379264 dispatch_meta_bytes=344200 combine_payload_bytes=266379264 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266723464 ingress_bytes=335585280 link_bytes=335585280 token_bytes=533102728 weight_movement_bytes=69206016 total_nvlink_bytes=602308744 weight_to_token_ratio=0.129817 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262780392e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266354688 dispatch_meta_bytes=344696 combine_payload_bytes=266354688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266699384 ingress_bytes=300957696 link_bytes=300957696 token_bytes=533054072 weight_movement_bytes=34603008 total_nvlink_bytes=567657080 weight_to_token_ratio=0.064915 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132479581e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=64628 dup_rows=24016 dup_rows_remote=20678 dedup_ratio=0.244303 router_tokens_per_expert=[784, 805, 781, 751, 780, 756, 736, 789, 772, 781, 733, 787, 797, 793, 757, 763, 767, 750, 756, 776, 806, 746, 775, 787, 764, 739, 793, 789, 731, 755, 794, 756, 780, 788, 789, 799, 751, 783, 801, 760, 766, 776, 754, 734, 775, 722, 762, 739, 752, 767, 789, 778, 747, 735, 758, 778, 783, 767, 738, 770, 800, 769, 759, 734] payload_rows_per_dst=[9340, 9306, 9344, 9252, 9434, 8716, 9236, 9660] dup_rows_per_dst=[2974, 3060, 2982, 2990, 3068, 2582, 3022, 3338] combine_rows_per_src=[9340, 9306, 9344, 9252, 9434, 8716, 9236, 9660] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267018240 dispatch_meta_bytes=344320 combine_payload_bytes=267018240 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267362560 ingress_bytes=301621248 link_bytes=301621248 token_bytes=534380800 weight_movement_bytes=34603008 total_nvlink_bytes=568983808 weight_to_token_ratio=0.064753 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134976474e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74270 payload_rows_remote=65088 dup_rows=24034 dup_rows_remote=21118 dedup_ratio=0.244486 router_tokens_per_expert=[813, 817, 768, 806, 772, 757, 773, 749, 732, 793, 757, 787, 727, 771, 777, 787, 764, 694, 787, 723, 764, 761, 784, 772, 761, 733, 755, 742, 776, 753, 755, 782, 764, 794, 732, 822, 710, 756, 776, 781, 770, 775, 747, 779, 786, 806, 758, 768, 744, 785, 792, 788, 747, 815, 847, 745, 761, 811, 764, 714, 801, 745, 730, 747] payload_rows_per_dst=[9516, 9226, 9182, 9194, 9280, 9270, 9364, 9238] dup_rows_per_dst=[2994, 3036, 2916, 2920, 2990, 3108, 3162, 2908] combine_rows_per_src=[9516, 9226, 9182, 9194, 9280, 9270, 9364, 9238] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74182 payload_rows_remote=65046 dup_rows=24122 dup_rows_remote=21084 dedup_ratio=0.245382 router_tokens_per_expert=[761, 785, 745, 743, 781, 826, 763, 742, 782, 756, 783, 732, 709, 785, 788, 752, 756, 770, 781, 748, 797, 790, 756, 800, 799, 749, 760, 740, 778, 788, 750, 785, 777, 751, 769, 776, 757, 756, 754, 802, 829, 752, 773, 772, 758, 771, 763, 777, 730, 771, 779, 789, 734, 769, 740, 791, 782, 766, 754, 765, 732, 745, 784, 804] payload_rows_per_dst=[9218, 9136, 9380, 9224, 9228, 9360, 9320, 9316] dup_rows_per_dst=[3074, 3038, 3016, 3074, 3056, 3030, 2886, 2948] combine_rows_per_src=[9218, 9136, 9380, 9224, 9228, 9360, 9320, 9316] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=264716288 dispatch_meta_bytes=341224 combine_payload_bytes=264716288 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265057512 ingress_bytes=299319296 link_bytes=299319296 token_bytes=529773800 weight_movement_bytes=34603008 total_nvlink_bytes=564376808 weight_to_token_ratio=0.065317 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126314414e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=65026 dup_rows=23992 dup_rows_remote=21032 dedup_ratio=0.244059 router_tokens_per_expert=[756, 801, 772, 771, 719, 804, 753, 779, 732, 769, 801, 732, 784, 771, 764, 700, 754, 801, 747, 722, 724, 767, 769, 778, 793, 777, 832, 791, 713, 789, 745, 749, 746, 805, 751, 751, 754, 730, 784, 802, 790, 741, 737, 803, 793, 773, 768, 762, 755, 807, 762, 765, 820, 807, 754, 783, 787, 784, 764, 735, 771, 733, 803, 773] payload_rows_per_dst=[9338, 9162, 9208, 9258, 9286, 9344, 9442, 9274] dup_rows_per_dst=[2972, 2944, 2916, 3120, 2960, 2990, 3064, 3026] combine_rows_per_src=[9338, 9162, 9208, 9258, 9286, 9344, 9442, 9274] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=344824 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266945272 ingress_bytes=301203456 link_bytes=301203456 token_bytes=533545720 weight_movement_bytes=34603008 total_nvlink_bytes=568148728 weight_to_token_ratio=0.064855 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133404356e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74438 payload_rows_remote=65134 dup_rows=23866 dup_rows_remote=20876 dedup_ratio=0.242778 router_tokens_per_expert=[785, 733, 749, 772, 770, 776, 806, 756, 764, 749, 804, 762, 756, 749, 773, 770, 741, 799, 775, 769, 774, 785, 713, 772, 742, 767, 745, 803, 765, 770, 738, 750, 758, 748, 775, 797, 779, 749, 709, 771, 753, 760, 780, 828, 773, 835, 814, 734, 807, 758, 795, 750, 781, 774, 789, 768, 792, 790, 798, 767, 707, 731, 765, 735] payload_rows_per_dst=[9304, 9246, 9258, 9290, 9262, 9524, 9294, 9260] dup_rows_per_dst=[2990, 3008, 2998, 2870, 2910, 3034, 3146, 2910] combine_rows_per_src=[9304, 9246, 9258, 9290, 9262, 9524, 9294, 9260] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74458 payload_rows_remote=65006 dup_rows=23846 dup_rows_remote=20756 dedup_ratio=0.242574 router_tokens_per_expert=[738, 811, 743, 824, 726, 733, 783, 756, 785, 802, 738, 725, 757, 741, 790, 767, 794, 828, 811, 790, 761, 769, 771, 726, 756, 750, 744, 762, 762, 721, 810, 765, 779, 726, 782, 761, 743, 769, 766, 746, 826, 799, 784, 775, 736, 763, 820, 768, 748, 820, 784, 765, 782, 791, 759, 739, 759, 784, 726, 751, 769, 747, 773, 773] payload_rows_per_dst=[9334, 9256, 9462, 9156, 9236, 9452, 9350, 9212] dup_rows_per_dst=[2894, 2954, 3038, 2984, 2908, 3090, 3026, 2952] combine_rows_per_src=[9334, 9256, 9462, 9156, 9236, 9452, 9350, 9212] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74270 payload_rows_remote=65088 dup_rows=24034 dup_rows_remote=21118 dedup_ratio=0.244486 router_tokens_per_expert=[813, 817, 768, 806, 772, 757, 773, 749, 732, 793, 757, 787, 727, 771, 777, 787, 764, 694, 787, 723, 764, 761, 784, 772, 761, 733, 755, 742, 776, 753, 755, 782, 764, 794, 732, 822, 710, 756, 776, 781, 770, 775, 747, 779, 786, 806, 758, 768, 744, 785, 792, 788, 747, 815, 847, 745, 761, 811, 764, 714, 801, 745, 730, 747] payload_rows_per_dst=[9516, 9226, 9182, 9194, 9280, 9270, 9364, 9238] dup_rows_per_dst=[2994, 3036, 2916, 2920, 2990, 3108, 3162, 2908] combine_rows_per_src=[9516, 9226, 9182, 9194, 9280, 9270, 9364, 9238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=64628 dup_rows=24016 dup_rows_remote=20678 dedup_ratio=0.244303 router_tokens_per_expert=[784, 805, 781, 751, 780, 756, 736, 789, 772, 781, 733, 787, 797, 793, 757, 763, 767, 750, 756, 776, 806, 746, 775, 787, 764, 739, 793, 789, 731, 755, 794, 756, 780, 788, 789, 799, 751, 783, 801, 760, 766, 776, 754, 734, 775, 722, 762, 739, 752, 767, 789, 778, 747, 735, 758, 778, 783, 767, 738, 770, 800, 769, 759, 734] payload_rows_per_dst=[9340, 9306, 9344, 9252, 9434, 8716, 9236, 9660] dup_rows_per_dst=[2974, 3060, 2982, 2990, 3068, 2582, 3022, 3338] combine_rows_per_src=[9340, 9306, 9344, 9252, 9434, 8716, 9236, 9660] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=344520 combine_payload_bytes=266428416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266772936 ingress_bytes=301031424 link_bytes=301031424 token_bytes=533201352 weight_movement_bytes=34603008 total_nvlink_bytes=567804360 weight_to_token_ratio=0.064897 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132757013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74458 payload_rows_remote=65006 dup_rows=23846 dup_rows_remote=20756 dedup_ratio=0.242574 router_tokens_per_expert=[738, 811, 743, 824, 726, 733, 783, 756, 785, 802, 738, 725, 757, 741, 790, 767, 794, 828, 811, 790, 761, 769, 771, 726, 756, 750, 744, 762, 762, 721, 810, 765, 779, 726, 782, 761, 743, 769, 766, 746, 826, 799, 784, 775, 736, 763, 820, 768, 748, 820, 784, 765, 782, 791, 759, 739, 759, 784, 726, 751, 769, 747, 773, 773] payload_rows_per_dst=[9334, 9256, 9462, 9156, 9236, 9452, 9350, 9212] dup_rows_per_dst=[2894, 2954, 3038, 2984, 2908, 3090, 3026, 2952] combine_rows_per_src=[9334, 9256, 9462, 9156, 9236, 9452, 9350, 9212] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=65088 dup_rows=23928 dup_rows_remote=20892 dedup_ratio=0.243408 router_tokens_per_expert=[806, 782, 757, 740, 759, 761, 804, 744, 790, 754, 774, 721, 764, 798, 791, 746, 726, 746, 762, 798, 752, 739, 765, 810, 737, 784, 757, 784, 749, 760, 779, 792, 761, 784, 751, 790, 779, 746, 822, 752, 796, 770, 771, 776, 802, 736, 739, 839, 776, 770, 718, 776, 764, 818, 795, 820, 762, 711, 753, 737, 707, 759, 788, 753] payload_rows_per_dst=[9288, 9326, 9330, 9486, 9100, 9584, 9152, 9110] dup_rows_per_dst=[3036, 3118, 3028, 2982, 2900, 3138, 2896, 2830] combine_rows_per_src=[9288, 9326, 9330, 9486, 9100, 9584, 9152, 9110] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74218 payload_rows_remote=64960 dup_rows=24086 dup_rows_remote=21102 dedup_ratio=0.245015 router_tokens_per_expert=[786, 732, 753, 766, 763, 745, 808, 751, 766, 793, 754, 758, 777, 760, 794, 754, 781, 720, 726, 761, 783, 742, 750, 747, 745, 730, 751, 769, 804, 811, 748, 763, 831, 761, 801, 776, 727, 803, 732, 768, 786, 753, 766, 783, 759, 781, 820, 785, 790, 756, 763, 779, 783, 744, 767, 788, 823, 785, 786, 717, 769, 721, 746, 812] payload_rows_per_dst=[9166, 9180, 9176, 9258, 9360, 9428, 9284, 9366] dup_rows_per_dst=[3042, 3132, 2844, 2984, 3038, 3038, 3056, 2952] combine_rows_per_src=[9166, 9180, 9176, 9258, 9360, 9428, 9284, 9366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74260 payload_rows_remote=65028 dup_rows=24044 dup_rows_remote=20944 dedup_ratio=0.244588 router_tokens_per_expert=[746, 766, 793, 758, 753, 789, 778, 762, 747, 765, 784, 806, 737, 789, 793, 788, 827, 746, 762, 784, 763, 756, 738, 755, 778, 807, 772, 734, 748, 752, 791, 776, 774, 809, 772, 719, 758, 772, 803, 759, 718, 774, 785, 784, 778, 739, 787, 789, 751, 736, 735, 824, 736, 799, 783, 770, 755, 778, 750, 740, 785, 731, 762, 754] payload_rows_per_dst=[9356, 9382, 9218, 9348, 9232, 9250, 9254, 9220] dup_rows_per_dst=[2934, 3036, 3044, 2968, 3100, 3058, 3014, 2890] combine_rows_per_src=[9356, 9382, 9218, 9348, 9232, 9250, 9254, 9220] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74308 payload_rows_remote=65186 dup_rows=23996 dup_rows_remote=21074 dedup_ratio=0.244100 router_tokens_per_expert=[762, 792, 731, 793, 728, 749, 746, 758, 754, 810, 752, 761, 775, 769, 824, 770, 785, 737, 846, 776, 845, 737, 776, 739, 774, 766, 802, 764, 785, 780, 745, 795, 764, 788, 806, 713, 821, 814, 764, 760, 729, 778, 765, 760, 793, 765, 747, 750, 793, 696, 726, 800, 795, 720, 729, 763, 789, 748, 749, 775, 765, 744, 763, 754] payload_rows_per_dst=[9218, 9334, 9446, 9382, 9390, 9166, 9122, 9250] dup_rows_per_dst=[2900, 3096, 3036, 3040, 3070, 3008, 2922, 2924] combine_rows_per_src=[9218, 9334, 9446, 9382, 9390, 9166, 9122, 9250] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266346496 dispatch_meta_bytes=344232 combine_payload_bytes=266346496 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266690728 ingress_bytes=370155520 link_bytes=370155520 token_bytes=533037224 weight_movement_bytes=103809024 total_nvlink_bytes=636846248 weight_to_token_ratio=0.194750 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392865422e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=344824 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266945272 ingress_bytes=301203456 link_bytes=301203456 token_bytes=533545720 weight_movement_bytes=34603008 total_nvlink_bytes=568148728 weight_to_token_ratio=0.064855 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133404356e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=344040 combine_payload_bytes=266788864 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267132904 ingress_bytes=301391872 link_bytes=301391872 token_bytes=533921768 weight_movement_bytes=34603008 total_nvlink_bytes=568524776 weight_to_token_ratio=0.064809 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134113350e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266264576 dispatch_meta_bytes=343048 combine_payload_bytes=266264576 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266607624 ingress_bytes=335470592 link_bytes=335470592 token_bytes=532872200 weight_movement_bytes=69206016 total_nvlink_bytes=602078216 weight_to_token_ratio=0.129874 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262348830e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=264716288 dispatch_meta_bytes=341224 combine_payload_bytes=264716288 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265057512 ingress_bytes=299319296 link_bytes=299319296 token_bytes=529773800 weight_movement_bytes=34603008 total_nvlink_bytes=564376808 weight_to_token_ratio=0.065317 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126314414e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266264576 dispatch_meta_bytes=343048 combine_payload_bytes=266264576 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266607624 ingress_bytes=335470592 link_bytes=335470592 token_bytes=532872200 weight_movement_bytes=69206016 total_nvlink_bytes=602078216 weight_to_token_ratio=0.129874 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262348830e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=343920 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266944368 ingress_bytes=335806464 link_bytes=335806464 token_bytes=533544816 weight_movement_bytes=69206016 total_nvlink_bytes=602750832 weight_to_token_ratio=0.129710 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263612689e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266076160 dispatch_meta_bytes=344248 combine_payload_bytes=266076160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266420408 ingress_bytes=300679168 link_bytes=300679168 token_bytes=532496568 weight_movement_bytes=34603008 total_nvlink_bytes=567099576 weight_to_token_ratio=0.064983 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131431503e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74334 payload_rows_remote=64812 dup_rows=23970 dup_rows_remote=20748 dedup_ratio=0.243835 router_tokens_per_expert=[775, 759, 741, 749, 726, 801, 789, 766, 779, 781, 744, 737, 771, 801, 737, 776, 807, 742, 789, 735, 766, 822, 735, 762, 804, 758, 722, 772, 747, 799, 759, 784, 761, 783, 769, 752, 722, 725, 756, 768, 768, 777, 791, 772, 772, 772, 765, 755, 797, 769, 806, 779, 753, 794, 753, 811, 767, 769, 769, 721, 817, 761, 744, 799] payload_rows_per_dst=[9114, 9054, 9226, 9446, 9182, 9258, 9532, 9522] dup_rows_per_dst=[2864, 2748, 2986, 2948, 2890, 3086, 3226, 3222] combine_rows_per_src=[9114, 9054, 9226, 9446, 9182, 9258, 9532, 9522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266354688 dispatch_meta_bytes=343888 combine_payload_bytes=266354688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266698576 ingress_bytes=300957696 link_bytes=300957696 token_bytes=533053264 weight_movement_bytes=34603008 total_nvlink_bytes=567656272 weight_to_token_ratio=0.064915 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132479581e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267001856 dispatch_meta_bytes=345040 combine_payload_bytes=267001856 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267346896 ingress_bytes=370810880 link_bytes=370810880 token_bytes=534348752 weight_movement_bytes=103809024 total_nvlink_bytes=638157776 weight_to_token_ratio=0.194272 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395331489e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74404 payload_rows_remote=65082 dup_rows=23900 dup_rows_remote=21024 dedup_ratio=0.243123 router_tokens_per_expert=[788, 778, 739, 777, 747, 807, 739, 750, 788, 778, 733, 780, 782, 765, 828, 833, 729, 774, 754, 731, 781, 744, 756, 830, 771, 812, 764, 726, 794, 811, 722, 802, 765, 728, 760, 802, 746, 736, 803, 751, 749, 807, 753, 729, 753, 801, 802, 803, 786, 781, 775, 753, 743, 736, 762, 765, 755, 759, 804, 757, 783, 726, 739, 727] payload_rows_per_dst=[9228, 9392, 9322, 9428, 9204, 9318, 9284, 9228] dup_rows_per_dst=[3022, 3182, 2876, 2976, 2978, 3076, 2918, 2872] combine_rows_per_src=[9228, 9392, 9322, 9428, 9204, 9318, 9284, 9228] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74460 payload_rows_remote=65192 dup_rows=23844 dup_rows_remote=20752 dedup_ratio=0.242554 router_tokens_per_expert=[817, 790, 732, 792, 781, 794, 761, 812, 784, 775, 781, 777, 748, 766, 746, 803, 776, 726, 783, 772, 766, 751, 775, 747, 800, 778, 713, 721, 769, 782, 817, 776, 764, 790, 752, 773, 751, 803, 813, 742, 724, 821, 754, 774, 765, 722, 725, 710, 774, 768, 772, 786, 783, 781, 775, 763, 690, 756, 812, 782, 760, 766, 768, 722] payload_rows_per_dst=[9468, 9268, 9284, 9378, 9358, 9122, 9326, 9256] dup_rows_per_dst=[3090, 3092, 2908, 2934, 3018, 2868, 3078, 2856] combine_rows_per_src=[9468, 9268, 9284, 9378, 9358, 9122, 9326, 9256] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265469952 dispatch_meta_bytes=342240 combine_payload_bytes=265469952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265812192 ingress_bytes=300072960 link_bytes=300072960 token_bytes=531282144 weight_movement_bytes=34603008 total_nvlink_bytes=565885152 weight_to_token_ratio=0.065131 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129150391e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266575872 dispatch_meta_bytes=344424 combine_payload_bytes=266575872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266920296 ingress_bytes=335781888 link_bytes=335781888 token_bytes=533496168 weight_movement_bytes=69206016 total_nvlink_bytes=602702184 weight_to_token_ratio=0.129722 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263520212e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=64942 dup_rows=24060 dup_rows_remote=20966 dedup_ratio=0.244751 router_tokens_per_expert=[744, 761, 745, 753, 772, 790, 754, 756, 778, 775, 800, 763, 736, 769, 793, 825, 743, 779, 742, 777, 764, 744, 758, 778, 834, 734, 758, 802, 746, 779, 786, 728, 794, 769, 792, 760, 800, 728, 804, 751, 719, 780, 792, 738, 761, 751, 777, 762, 746, 763, 782, 794, 741, 776, 734, 785, 796, 744, 753, 721, 830, 764, 831, 748] payload_rows_per_dst=[9236, 9448, 9214, 9396, 9302, 9096, 9210, 9342] dup_rows_per_dst=[2914, 3030, 2956, 2938, 3094, 3064, 3032, 3032] combine_rows_per_src=[9236, 9448, 9214, 9396, 9302, 9096, 9210, 9342] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=65088 dup_rows=23928 dup_rows_remote=20892 dedup_ratio=0.243408 router_tokens_per_expert=[806, 782, 757, 740, 759, 761, 804, 744, 790, 754, 774, 721, 764, 798, 791, 746, 726, 746, 762, 798, 752, 739, 765, 810, 737, 784, 757, 784, 749, 760, 779, 792, 761, 784, 751, 790, 779, 746, 822, 752, 796, 770, 771, 776, 802, 736, 739, 839, 776, 770, 718, 776, 764, 818, 795, 820, 762, 711, 753, 737, 707, 759, 788, 753] payload_rows_per_dst=[9288, 9326, 9330, 9486, 9100, 9584, 9152, 9110] dup_rows_per_dst=[3036, 3118, 3028, 2982, 2900, 3138, 2896, 2830] combine_rows_per_src=[9288, 9326, 9330, 9486, 9100, 9584, 9152, 9110] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74412 payload_rows_remote=65316 dup_rows=23892 dup_rows_remote=20986 dedup_ratio=0.243042 router_tokens_per_expert=[761, 718, 745, 744, 746, 802, 801, 779, 816, 764, 793, 765, 767, 755, 779, 714, 769, 764, 756, 752, 751, 787, 800, 812, 813, 736, 782, 742, 771, 760, 795, 752, 775, 759, 788, 758, 812, 765, 775, 765, 773, 707, 758, 767, 743, 764, 743, 746, 782, 785, 755, 797, 797, 761, 776, 771, 767, 737, 780, 795, 790, 725, 767, 778] payload_rows_per_dst=[9226, 9342, 9318, 9304, 9460, 9096, 9382, 9284] dup_rows_per_dst=[2966, 2964, 3064, 2998, 2934, 2906, 3066, 2994] combine_rows_per_src=[9226, 9342, 9318, 9304, 9460, 9096, 9382, 9284] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74404 payload_rows_remote=65082 dup_rows=23900 dup_rows_remote=21024 dedup_ratio=0.243123 router_tokens_per_expert=[788, 778, 739, 777, 747, 807, 739, 750, 788, 778, 733, 780, 782, 765, 828, 833, 729, 774, 754, 731, 781, 744, 756, 830, 771, 812, 764, 726, 794, 811, 722, 802, 765, 728, 760, 802, 746, 736, 803, 751, 749, 807, 753, 729, 753, 801, 802, 803, 786, 781, 775, 753, 743, 736, 762, 765, 755, 759, 804, 757, 783, 726, 739, 727] payload_rows_per_dst=[9228, 9392, 9322, 9428, 9204, 9318, 9284, 9228] dup_rows_per_dst=[3022, 3182, 2876, 2976, 2978, 3076, 2918, 2872] combine_rows_per_src=[9228, 9392, 9322, 9428, 9204, 9318, 9284, 9228] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74334 payload_rows_remote=64812 dup_rows=23970 dup_rows_remote=20748 dedup_ratio=0.243835 router_tokens_per_expert=[775, 759, 741, 749, 726, 801, 789, 766, 779, 781, 744, 737, 771, 801, 737, 776, 807, 742, 789, 735, 766, 822, 735, 762, 804, 758, 722, 772, 747, 799, 759, 784, 761, 783, 769, 752, 722, 725, 756, 768, 768, 777, 791, 772, 772, 772, 765, 755, 797, 769, 806, 779, 753, 794, 753, 811, 767, 769, 769, 721, 817, 761, 744, 799] payload_rows_per_dst=[9114, 9054, 9226, 9446, 9182, 9258, 9532, 9522] dup_rows_per_dst=[2864, 2748, 2986, 2948, 2890, 3086, 3226, 3222] combine_rows_per_src=[9114, 9054, 9226, 9446, 9182, 9258, 9532, 9522] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=267026432 dispatch_meta_bytes=343776 combine_payload_bytes=267026432 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267370208 ingress_bytes=336232448 link_bytes=336232448 token_bytes=534396640 weight_movement_bytes=69206016 total_nvlink_bytes=603602656 weight_to_token_ratio=0.129503 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265215633e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74412 payload_rows_remote=65316 dup_rows=23892 dup_rows_remote=20986 dedup_ratio=0.243042 router_tokens_per_expert=[761, 718, 745, 744, 746, 802, 801, 779, 816, 764, 793, 765, 767, 755, 779, 714, 769, 764, 756, 752, 751, 787, 800, 812, 813, 736, 782, 742, 771, 760, 795, 752, 775, 759, 788, 758, 812, 765, 775, 765, 773, 707, 758, 767, 743, 764, 743, 746, 782, 785, 755, 797, 797, 761, 776, 771, 767, 737, 780, 795, 790, 725, 767, 778] payload_rows_per_dst=[9226, 9342, 9318, 9304, 9460, 9096, 9382, 9284] dup_rows_per_dst=[2966, 2964, 3064, 2998, 2934, 2906, 3066, 2994] combine_rows_per_src=[9226, 9342, 9318, 9304, 9460, 9096, 9382, 9284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74532 payload_rows_remote=65184 dup_rows=23772 dup_rows_remote=20666 dedup_ratio=0.241821 router_tokens_per_expert=[811, 787, 785, 776, 753, 758, 799, 755, 812, 773, 715, 714, 771, 771, 767, 767, 777, 772, 756, 841, 785, 746, 763, 788, 774, 772, 809, 755, 755, 737, 759, 738, 784, 718, 774, 761, 771, 777, 804, 737, 755, 737, 775, 767, 745, 752, 736, 742, 713, 746, 810, 764, 775, 797, 807, 809, 780, 781, 750, 773, 775, 763, 742, 791] payload_rows_per_dst=[9348, 9226, 9470, 9320, 9284, 9626, 8910, 9348] dup_rows_per_dst=[3106, 2992, 3002, 2878, 2908, 3200, 2724, 2962] combine_rows_per_src=[9348, 9226, 9470, 9320, 9284, 9626, 8910, 9348] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74460 payload_rows_remote=65140 dup_rows=23844 dup_rows_remote=21018 dedup_ratio=0.242554 router_tokens_per_expert=[752, 733, 785, 808, 780, 784, 752, 773, 772, 788, 792, 763, 778, 735, 786, 778, 731, 778, 753, 751, 772, 760, 780, 769, 730, 737, 768, 798, 759, 737, 747, 797, 809, 752, 790, 780, 789, 808, 732, 767, 805, 762, 737, 813, 736, 726, 774, 754, 796, 734, 731, 767, 830, 731, 754, 791, 759, 773, 809, 745, 720, 825, 734, 793] payload_rows_per_dst=[9234, 9410, 9250, 9320, 9402, 9210, 9320, 9314] dup_rows_per_dst=[3100, 2974, 2938, 2826, 3052, 3004, 2948, 3002] combine_rows_per_src=[9234, 9410, 9250, 9320, 9402, 9210, 9320, 9314] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266002432 dispatch_meta_bytes=343632 combine_payload_bytes=266002432 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266346064 ingress_bytes=369811456 link_bytes=369811456 token_bytes=532348496 weight_movement_bytes=103809024 total_nvlink_bytes=636157520 weight_to_token_ratio=0.195002 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391570737e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266575872 dispatch_meta_bytes=344424 combine_payload_bytes=266575872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266920296 ingress_bytes=335781888 link_bytes=335781888 token_bytes=533496168 weight_movement_bytes=69206016 total_nvlink_bytes=602702184 weight_to_token_ratio=0.129722 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263520212e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74200 payload_rows_remote=64958 dup_rows=24104 dup_rows_remote=20998 dedup_ratio=0.245199 router_tokens_per_expert=[789, 784, 785, 799, 777, 724, 792, 744, 764, 751, 723, 741, 780, 819, 732, 781, 726, 762, 808, 802, 812, 752, 775, 808, 790, 742, 840, 724, 754, 750, 771, 736, 754, 772, 827, 760, 799, 739, 720, 748, 742, 778, 790, 768, 760, 777, 815, 757, 771, 738, 785, 751, 782, 810, 778, 759, 717, 773, 753, 754, 782, 739, 767, 750] payload_rows_per_dst=[9400, 9266, 9304, 9250, 9266, 9254, 9242, 9218] dup_rows_per_dst=[2988, 2916, 3186, 2964, 2972, 3120, 3106, 2852] combine_rows_per_src=[9400, 9266, 9304, 9250, 9266, 9254, 9242, 9218] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=65026 dup_rows=23992 dup_rows_remote=21032 dedup_ratio=0.244059 router_tokens_per_expert=[756, 801, 772, 771, 719, 804, 753, 779, 732, 769, 801, 732, 784, 771, 764, 700, 754, 801, 747, 722, 724, 767, 769, 778, 793, 777, 832, 791, 713, 789, 745, 749, 746, 805, 751, 751, 754, 730, 784, 802, 790, 741, 737, 803, 793, 773, 768, 762, 755, 807, 762, 765, 820, 807, 754, 783, 787, 784, 764, 735, 771, 733, 803, 773] payload_rows_per_dst=[9338, 9162, 9208, 9258, 9286, 9344, 9442, 9274] dup_rows_per_dst=[2972, 2944, 2916, 3120, 2960, 2990, 3064, 3026] combine_rows_per_src=[9338, 9162, 9208, 9258, 9286, 9344, 9442, 9274] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=343920 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266944368 ingress_bytes=335806464 link_bytes=335806464 token_bytes=533544816 weight_movement_bytes=69206016 total_nvlink_bytes=602750832 weight_to_token_ratio=0.129710 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263612689e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265469952 dispatch_meta_bytes=342240 combine_payload_bytes=265469952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265812192 ingress_bytes=300072960 link_bytes=300072960 token_bytes=531282144 weight_movement_bytes=34603008 total_nvlink_bytes=565885152 weight_to_token_ratio=0.065131 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129150391e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=267534336 dispatch_meta_bytes=345208 combine_payload_bytes=267534336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267879544 ingress_bytes=302137344 link_bytes=302137344 token_bytes=535413880 weight_movement_bytes=34603008 total_nvlink_bytes=570016888 weight_to_token_ratio=0.064629 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136918501e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74416 payload_rows_remote=64894 dup_rows=23888 dup_rows_remote=20742 dedup_ratio=0.243001 router_tokens_per_expert=[770, 763, 794, 757, 723, 723, 769, 763, 739, 785, 794, 800, 825, 744, 749, 777, 761, 816, 738, 797, 771, 781, 783, 749, 799, 766, 803, 818, 745, 770, 792, 786, 746, 776, 770, 745, 785, 767, 775, 733, 721, 780, 758, 779, 747, 812, 712, 753, 762, 733, 755, 729, 800, 792, 728, 759, 821, 802, 757, 770, 768, 772, 734, 761] payload_rows_per_dst=[9230, 9356, 9168, 9274, 9330, 9226, 9310, 9522] dup_rows_per_dst=[2894, 3022, 2870, 2986, 2864, 2972, 3134, 3146] combine_rows_per_src=[9230, 9356, 9168, 9274, 9330, 9226, 9310, 9522] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266813440 dispatch_meta_bytes=344632 combine_payload_bytes=266813440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267158072 ingress_bytes=301416448 link_bytes=301416448 token_bytes=533971512 weight_movement_bytes=34603008 total_nvlink_bytes=568574520 weight_to_token_ratio=0.064803 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134205828e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=64970 dup_rows=23948 dup_rows_remote=20838 dedup_ratio=0.243612 router_tokens_per_expert=[809, 764, 721, 756, 752, 749, 727, 766, 768, 792, 766, 778, 751, 794, 770, 745, 791, 784, 794, 785, 782, 767, 758, 787, 805, 727, 750, 828, 774, 756, 831, 704, 780, 804, 765, 763, 762, 774, 771, 759, 814, 758, 762, 763, 756, 765, 751, 782, 786, 759, 736, 761, 700, 749, 749, 802, 741, 786, 797, 743, 781, 778, 720, 804] payload_rows_per_dst=[9114, 9274, 9386, 9372, 9436, 9326, 9236, 9212] dup_rows_per_dst=[2974, 3054, 3110, 2978, 2920, 2976, 2848, 3088] combine_rows_per_src=[9114, 9274, 9386, 9372, 9436, 9326, 9236, 9212] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=343824 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266411792 ingress_bytes=335273984 link_bytes=335273984 token_bytes=532479760 weight_movement_bytes=69206016 total_nvlink_bytes=601685776 weight_to_token_ratio=0.129969 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261609010e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266993664 dispatch_meta_bytes=343400 combine_payload_bytes=266993664 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267337064 ingress_bytes=336199680 link_bytes=336199680 token_bytes=534330728 weight_movement_bytes=69206016 total_nvlink_bytes=603536744 weight_to_token_ratio=0.129519 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265092330e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266813440 dispatch_meta_bytes=344632 combine_payload_bytes=266813440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267158072 ingress_bytes=301416448 link_bytes=301416448 token_bytes=533971512 weight_movement_bytes=34603008 total_nvlink_bytes=568574520 weight_to_token_ratio=0.064803 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134205828e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64940 dup_rows=24112 dup_rows_remote=21072 dedup_ratio=0.245280 router_tokens_per_expert=[767, 735, 770, 792, 762, 758, 784, 746, 770, 777, 752, 776, 789, 749, 742, 791, 807, 762, 715, 763, 840, 799, 814, 784, 757, 739, 759, 747, 772, 745, 764, 819, 777, 755, 762, 749, 773, 733, 773, 728, 742, 731, 797, 769, 804, 774, 746, 725, 750, 742, 771, 797, 793, 826, 748, 787, 792, 761, 729, 718, 797, 777, 809, 771] payload_rows_per_dst=[9214, 9252, 9422, 9224, 9168, 9210, 9402, 9300] dup_rows_per_dst=[3014, 3040, 3146, 2980, 2932, 2966, 3026, 3008] combine_rows_per_src=[9214, 9252, 9422, 9224, 9168, 9210, 9402, 9300] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265805824 dispatch_meta_bytes=342544 combine_payload_bytes=265805824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266148368 ingress_bytes=300408832 link_bytes=300408832 token_bytes=531954192 weight_movement_bytes=34603008 total_nvlink_bytes=566557200 weight_to_token_ratio=0.065049 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130414250e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266117120 dispatch_meta_bytes=343232 combine_payload_bytes=266117120 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266460352 ingress_bytes=404529152 link_bytes=404529152 token_bytes=532577472 weight_movement_bytes=138412032 total_nvlink_bytes=670989504 weight_to_token_ratio=0.259891 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522210632e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74158 payload_rows_remote=64794 dup_rows=24146 dup_rows_remote=21186 dedup_ratio=0.245626 router_tokens_per_expert=[773, 744, 769, 749, 723, 769, 753, 753, 768, 795, 765, 771, 793, 805, 760, 765, 747, 754, 774, 812, 780, 792, 775, 766, 791, 780, 796, 714, 783, 754, 772, 769, 800, 779, 769, 797, 788, 751, 712, 766, 788, 719, 787, 749, 748, 770, 777, 791, 785, 755, 754, 787, 742, 790, 772, 761, 756, 766, 760, 763, 805, 718, 735, 798] payload_rows_per_dst=[9114, 9402, 9266, 9200, 9364, 9254, 9262, 9296] dup_rows_per_dst=[2952, 3042, 3134, 3118, 2960, 3004, 3030, 2906] combine_rows_per_src=[9114, 9402, 9266, 9200, 9364, 9254, 9262, 9296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=343824 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266411792 ingress_bytes=335273984 link_bytes=335273984 token_bytes=532479760 weight_movement_bytes=69206016 total_nvlink_bytes=601685776 weight_to_token_ratio=0.129969 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261609010e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266346496 dispatch_meta_bytes=344232 combine_payload_bytes=266346496 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266690728 ingress_bytes=370155520 link_bytes=370155520 token_bytes=533037224 weight_movement_bytes=103809024 total_nvlink_bytes=636846248 weight_to_token_ratio=0.194750 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392865422e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74532 payload_rows_remote=65184 dup_rows=23772 dup_rows_remote=20666 dedup_ratio=0.241821 router_tokens_per_expert=[811, 787, 785, 776, 753, 758, 799, 755, 812, 773, 715, 714, 771, 771, 767, 767, 777, 772, 756, 841, 785, 746, 763, 788, 774, 772, 809, 755, 755, 737, 759, 738, 784, 718, 774, 761, 771, 777, 804, 737, 755, 737, 775, 767, 745, 752, 736, 742, 713, 746, 810, 764, 775, 797, 807, 809, 780, 781, 750, 773, 775, 763, 742, 791] payload_rows_per_dst=[9348, 9226, 9470, 9320, 9284, 9626, 8910, 9348] dup_rows_per_dst=[3106, 2992, 3002, 2878, 2908, 3200, 2724, 2962] combine_rows_per_src=[9348, 9226, 9470, 9320, 9284, 9626, 8910, 9348] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74476 payload_rows_remote=65026 dup_rows=23828 dup_rows_remote=20626 dedup_ratio=0.242391 router_tokens_per_expert=[737, 797, 766, 752, 768, 759, 792, 744, 811, 775, 747, 748, 761, 768, 763, 777, 745, 756, 766, 721, 770, 774, 778, 766, 770, 743, 729, 727, 782, 797, 774, 792, 777, 781, 751, 761, 771, 778, 736, 744, 743, 814, 794, 827, 814, 810, 791, 733, 813, 764, 768, 787, 771, 789, 753, 743, 733, 784, 788, 751, 737, 759, 778, 754] payload_rows_per_dst=[9252, 9298, 9216, 9318, 9266, 9450, 9394, 9282] dup_rows_per_dst=[2978, 3002, 2936, 2910, 2932, 3202, 2982, 2886] combine_rows_per_src=[9252, 9298, 9216, 9318, 9266, 9450, 9394, 9282] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=344048 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266338288 ingress_bytes=335200256 link_bytes=335200256 token_bytes=532332528 weight_movement_bytes=69206016 total_nvlink_bytes=601538544 weight_to_token_ratio=0.130005 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261331577e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74226 payload_rows_remote=64974 dup_rows=24078 dup_rows_remote=21134 dedup_ratio=0.244934 router_tokens_per_expert=[780, 782, 775, 728, 762, 785, 794, 782, 779, 763, 753, 745, 752, 779, 729, 758, 803, 755, 792, 721, 760, 782, 794, 762, 722, 765, 794, 773, 760, 755, 783, 746, 781, 787, 719, 779, 796, 795, 758, 774, 736, 750, 766, 753, 823, 761, 772, 756, 744, 796, 736, 810, 749, 786, 789, 750, 769, 777, 812, 769, 779, 743, 776, 748] payload_rows_per_dst=[9348, 9194, 9228, 9252, 9290, 9190, 9368, 9356] dup_rows_per_dst=[3028, 2922, 3110, 2944, 3088, 3044, 2952, 2990] combine_rows_per_src=[9348, 9194, 9228, 9252, 9290, 9190, 9368, 9356] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265396224 dispatch_meta_bytes=343920 combine_payload_bytes=265396224 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265740144 ingress_bytes=334602240 link_bytes=334602240 token_bytes=531136368 weight_movement_bytes=69206016 total_nvlink_bytes=600342384 weight_to_token_ratio=0.130298 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259081291e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266993664 dispatch_meta_bytes=343400 combine_payload_bytes=266993664 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267337064 ingress_bytes=336199680 link_bytes=336199680 token_bytes=534330728 weight_movement_bytes=69206016 total_nvlink_bytes=603536744 weight_to_token_ratio=0.129519 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265092330e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=64970 dup_rows=23948 dup_rows_remote=20838 dedup_ratio=0.243612 router_tokens_per_expert=[809, 764, 721, 756, 752, 749, 727, 766, 768, 792, 766, 778, 751, 794, 770, 745, 791, 784, 794, 785, 782, 767, 758, 787, 805, 727, 750, 828, 774, 756, 831, 704, 780, 804, 765, 763, 762, 774, 771, 759, 814, 758, 762, 763, 756, 765, 751, 782, 786, 759, 736, 761, 700, 749, 749, 802, 741, 786, 797, 743, 781, 778, 720, 804] payload_rows_per_dst=[9114, 9274, 9386, 9372, 9436, 9326, 9236, 9212] dup_rows_per_dst=[2974, 3054, 3110, 2978, 2920, 2976, 2848, 3088] combine_rows_per_src=[9114, 9274, 9386, 9372, 9436, 9326, 9236, 9212] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74416 payload_rows_remote=64894 dup_rows=23888 dup_rows_remote=20742 dedup_ratio=0.243001 router_tokens_per_expert=[770, 763, 794, 757, 723, 723, 769, 763, 739, 785, 794, 800, 825, 744, 749, 777, 761, 816, 738, 797, 771, 781, 783, 749, 799, 766, 803, 818, 745, 770, 792, 786, 746, 776, 770, 745, 785, 767, 775, 733, 721, 780, 758, 779, 747, 812, 712, 753, 762, 733, 755, 729, 800, 792, 728, 759, 821, 802, 757, 770, 768, 772, 734, 761] payload_rows_per_dst=[9230, 9356, 9168, 9274, 9330, 9226, 9310, 9522] dup_rows_per_dst=[2894, 3022, 2870, 2986, 2864, 2972, 3134, 3146] combine_rows_per_src=[9230, 9356, 9168, 9274, 9330, 9226, 9310, 9522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74476 payload_rows_remote=65026 dup_rows=23828 dup_rows_remote=20626 dedup_ratio=0.242391 router_tokens_per_expert=[737, 797, 766, 752, 768, 759, 792, 744, 811, 775, 747, 748, 761, 768, 763, 777, 745, 756, 766, 721, 770, 774, 778, 766, 770, 743, 729, 727, 782, 797, 774, 792, 777, 781, 751, 761, 771, 778, 736, 744, 743, 814, 794, 827, 814, 810, 791, 733, 813, 764, 768, 787, 771, 789, 753, 743, 733, 784, 788, 751, 737, 759, 778, 754] payload_rows_per_dst=[9252, 9298, 9216, 9318, 9266, 9450, 9394, 9282] dup_rows_per_dst=[2978, 3002, 2936, 2910, 2932, 3202, 2982, 2886] combine_rows_per_src=[9252, 9298, 9216, 9318, 9266, 9450, 9394, 9282] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74336 payload_rows_remote=64796 dup_rows=23968 dup_rows_remote=20656 dedup_ratio=0.243815 router_tokens_per_expert=[761, 770, 757, 714, 748, 814, 774, 791, 801, 726, 776, 754, 775, 734, 767, 799, 807, 782, 789, 801, 769, 786, 764, 780, 732, 812, 759, 792, 811, 772, 726, 743, 778, 749, 765, 759, 793, 758, 793, 764, 750, 794, 760, 774, 749, 785, 751, 736, 787, 766, 768, 737, 781, 773, 773, 731, 764, 759, 765, 761, 749, 744, 796, 754] payload_rows_per_dst=[9540, 9314, 9160, 9410, 9244, 9284, 9150, 9234] dup_rows_per_dst=[3312, 2974, 2802, 2958, 2976, 2914, 3082, 2950] combine_rows_per_src=[9540, 9314, 9160, 9410, 9244, 9284, 9150, 9234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74226 payload_rows_remote=64974 dup_rows=24078 dup_rows_remote=21134 dedup_ratio=0.244934 router_tokens_per_expert=[780, 782, 775, 728, 762, 785, 794, 782, 779, 763, 753, 745, 752, 779, 729, 758, 803, 755, 792, 721, 760, 782, 794, 762, 722, 765, 794, 773, 760, 755, 783, 746, 781, 787, 719, 779, 796, 795, 758, 774, 736, 750, 766, 753, 823, 761, 772, 756, 744, 796, 736, 810, 749, 786, 789, 750, 769, 777, 812, 769, 779, 743, 776, 748] payload_rows_per_dst=[9348, 9194, 9228, 9252, 9290, 9190, 9368, 9356] dup_rows_per_dst=[3028, 2922, 3110, 2944, 3088, 3044, 2952, 2990] combine_rows_per_src=[9348, 9194, 9228, 9252, 9290, 9190, 9368, 9356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=65112 dup_rows=23982 dup_rows_remote=21064 dedup_ratio=0.243958 router_tokens_per_expert=[760, 764, 797, 760, 750, 783, 754, 802, 773, 730, 764, 773, 744, 746, 776, 773, 803, 766, 786, 756, 749, 769, 798, 748, 768, 806, 787, 761, 754, 800, 756, 758, 725, 761, 796, 824, 761, 812, 809, 779, 776, 760, 750, 775, 744, 774, 767, 797, 775, 725, 748, 787, 743, 731, 777, 778, 753, 777, 741, 736, 767, 771, 725, 794] payload_rows_per_dst=[9320, 9248, 9334, 9378, 9416, 9206, 9210, 9210] dup_rows_per_dst=[3020, 2910, 3016, 3002, 3118, 3080, 2918, 2918] combine_rows_per_src=[9320, 9248, 9334, 9378, 9416, 9206, 9210, 9210] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=64942 dup_rows=24060 dup_rows_remote=20966 dedup_ratio=0.244751 router_tokens_per_expert=[744, 761, 745, 753, 772, 790, 754, 756, 778, 775, 800, 763, 736, 769, 793, 825, 743, 779, 742, 777, 764, 744, 758, 778, 834, 734, 758, 802, 746, 779, 786, 728, 794, 769, 792, 760, 800, 728, 804, 751, 719, 780, 792, 738, 761, 751, 777, 762, 746, 763, 782, 794, 741, 776, 734, 785, 796, 744, 753, 721, 830, 764, 831, 748] payload_rows_per_dst=[9236, 9448, 9214, 9396, 9302, 9096, 9210, 9342] dup_rows_per_dst=[2914, 3030, 2956, 2938, 3094, 3064, 3032, 3032] combine_rows_per_src=[9236, 9448, 9214, 9396, 9302, 9096, 9210, 9342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266117120 dispatch_meta_bytes=343232 combine_payload_bytes=266117120 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266460352 ingress_bytes=404529152 link_bytes=404529152 token_bytes=532577472 weight_movement_bytes=138412032 total_nvlink_bytes=670989504 weight_to_token_ratio=0.259891 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522210632e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266346496 dispatch_meta_bytes=342608 combine_payload_bytes=266346496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266689104 ingress_bytes=300949504 link_bytes=300949504 token_bytes=533035600 weight_movement_bytes=34603008 total_nvlink_bytes=567638608 weight_to_token_ratio=0.064917 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132448755e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74318 payload_rows_remote=64796 dup_rows=23986 dup_rows_remote=20626 dedup_ratio=0.243998 router_tokens_per_expert=[728, 766, 771, 772, 776, 771, 788, 765, 798, 785, 758, 760, 795, 778, 761, 749, 791, 796, 825, 735, 765, 765, 794, 765, 804, 787, 774, 731, 749, 742, 723, 752, 759, 773, 729, 748, 726, 776, 827, 800, 788, 765, 722, 782, 785, 777, 697, 799, 750, 793, 736, 721, 775, 799, 747, 746, 793, 800, 783, 766, 763, 766, 753, 789] payload_rows_per_dst=[9314, 9324, 9168, 9162, 9296, 9034, 9498, 9522] dup_rows_per_dst=[2960, 3044, 2826, 2886, 3088, 2740, 3082, 3360] combine_rows_per_src=[9314, 9324, 9168, 9162, 9296, 9034, 9498, 9522] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=344432 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266477936 ingress_bytes=335339520 link_bytes=335339520 token_bytes=532611440 weight_movement_bytes=69206016 total_nvlink_bytes=601817456 weight_to_token_ratio=0.129937 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261855617e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265805824 dispatch_meta_bytes=342544 combine_payload_bytes=265805824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266148368 ingress_bytes=300408832 link_bytes=300408832 token_bytes=531954192 weight_movement_bytes=34603008 total_nvlink_bytes=566557200 weight_to_token_ratio=0.065049 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130414250e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74250 payload_rows_remote=64842 dup_rows=24054 dup_rows_remote=21008 dedup_ratio=0.244690 router_tokens_per_expert=[713, 743, 774, 783, 785, 764, 729, 757, 715, 748, 761, 794, 788, 776, 782, 757, 774, 734, 762, 763, 794, 809, 768, 823, 787, 761, 778, 782, 786, 795, 754, 830, 812, 757, 765, 761, 803, 782, 787, 725, 756, 732, 815, 789, 777, 757, 750, 767, 758, 768, 787, 770, 742, 793, 762, 716, 768, 764, 770, 753, 754, 720, 772, 751] payload_rows_per_dst=[9152, 9236, 9408, 9424, 9394, 9288, 9166, 9182] dup_rows_per_dst=[2944, 3006, 3046, 3122, 2990, 2998, 3026, 2922] combine_rows_per_src=[9152, 9236, 9408, 9424, 9394, 9288, 9166, 9182] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266698752 dispatch_meta_bytes=344704 combine_payload_bytes=266698752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267043456 ingress_bytes=301301760 link_bytes=301301760 token_bytes=533742208 weight_movement_bytes=34603008 total_nvlink_bytes=568345216 weight_to_token_ratio=0.064831 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133774266e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265404416 dispatch_meta_bytes=341688 combine_payload_bytes=265404416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265746104 ingress_bytes=300007424 link_bytes=300007424 token_bytes=531150520 weight_movement_bytes=34603008 total_nvlink_bytes=565753528 weight_to_token_ratio=0.065147 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128903784e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265404416 dispatch_meta_bytes=341808 combine_payload_bytes=265404416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265746224 ingress_bytes=300007424 link_bytes=300007424 token_bytes=531150640 weight_movement_bytes=34603008 total_nvlink_bytes=565753648 weight_to_token_ratio=0.065147 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128903784e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74266 payload_rows_remote=64934 dup_rows=24038 dup_rows_remote=20998 dedup_ratio=0.244527 router_tokens_per_expert=[734, 741, 743, 745, 715, 804, 755, 764, 780, 745, 796, 751, 816, 781, 757, 760, 759, 781, 745, 787, 756, 793, 767, 777, 800, 793, 776, 728, 726, 726, 828, 769, 732, 780, 765, 774, 822, 783, 750, 759, 797, 729, 764, 762, 725, 790, 789, 784, 762, 781, 789, 820, 786, 737, 729, 745, 789, 777, 768, 752, 748, 818, 771, 777] payload_rows_per_dst=[9132, 9332, 9294, 9280, 9326, 9336, 9226, 9340] dup_rows_per_dst=[2870, 3040, 3036, 3012, 3004, 2944, 3072, 3060] combine_rows_per_src=[9132, 9332, 9294, 9280, 9326, 9336, 9226, 9340] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74310 payload_rows_remote=64944 dup_rows=23994 dup_rows_remote=21006 dedup_ratio=0.244080 router_tokens_per_expert=[740, 795, 752, 745, 777, 749, 790, 750, 755, 785, 771, 735, 707, 800, 837, 784, 748, 760, 771, 753, 762, 756, 802, 747, 729, 757, 736, 776, 789, 798, 756, 796, 784, 765, 813, 739, 751, 765, 797, 763, 768, 773, 799, 762, 759, 774, 774, 845, 756, 763, 753, 796, 737, 769, 792, 783, 767, 785, 721, 764, 770, 773, 747, 737] payload_rows_per_dst=[9244, 9322, 9210, 9292, 9366, 9426, 9258, 9192] dup_rows_per_dst=[2952, 3026, 2988, 2982, 2988, 3082, 3040, 2936] combine_rows_per_src=[9244, 9322, 9210, 9292, 9366, 9426, 9258, 9192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74336 payload_rows_remote=64796 dup_rows=23968 dup_rows_remote=20656 dedup_ratio=0.243815 router_tokens_per_expert=[761, 770, 757, 714, 748, 814, 774, 791, 801, 726, 776, 754, 775, 734, 767, 799, 807, 782, 789, 801, 769, 786, 764, 780, 732, 812, 759, 792, 811, 772, 726, 743, 778, 749, 765, 759, 793, 758, 793, 764, 750, 794, 760, 774, 749, 785, 751, 736, 787, 766, 768, 737, 781, 773, 773, 731, 764, 759, 765, 761, 749, 744, 796, 754] payload_rows_per_dst=[9540, 9314, 9160, 9410, 9244, 9284, 9150, 9234] dup_rows_per_dst=[3312, 2974, 2802, 2958, 2976, 2914, 3082, 2950] combine_rows_per_src=[9540, 9314, 9160, 9410, 9244, 9284, 9150, 9234] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265592832 dispatch_meta_bytes=343400 combine_payload_bytes=265592832 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265936232 ingress_bytes=369401856 link_bytes=369401856 token_bytes=531529064 weight_movement_bytes=103809024 total_nvlink_bytes=635338088 weight_to_token_ratio=0.195303 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390029445e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=344432 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266477936 ingress_bytes=335339520 link_bytes=335339520 token_bytes=532611440 weight_movement_bytes=69206016 total_nvlink_bytes=601817456 weight_to_token_ratio=0.129937 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261855617e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266002432 dispatch_meta_bytes=343632 combine_payload_bytes=266002432 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266346064 ingress_bytes=369811456 link_bytes=369811456 token_bytes=532348496 weight_movement_bytes=103809024 total_nvlink_bytes=636157520 weight_to_token_ratio=0.195002 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391570737e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266698752 dispatch_meta_bytes=344704 combine_payload_bytes=266698752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267043456 ingress_bytes=301301760 link_bytes=301301760 token_bytes=533742208 weight_movement_bytes=34603008 total_nvlink_bytes=568345216 weight_to_token_ratio=0.064831 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133774266e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74170 payload_rows_remote=64874 dup_rows=24134 dup_rows_remote=21188 dedup_ratio=0.245504 router_tokens_per_expert=[772, 796, 785, 775, 764, 777, 728, 761, 792, 729, 760, 749, 795, 785, 790, 749, 758, 806, 766, 772, 820, 777, 760, 836, 703, 766, 766, 738, 809, 736, 708, 802, 773, 776, 781, 786, 781, 769, 798, 770, 778, 801, 707, 785, 797, 767, 749, 737, 751, 773, 728, 731, 734, 772, 725, 761, 797, 767, 814, 747, 763, 808, 749, 747] payload_rows_per_dst=[9296, 9284, 9464, 9160, 9254, 9296, 9116, 9300] dup_rows_per_dst=[3020, 3014, 3126, 2896, 3214, 2946, 2834, 3084] combine_rows_per_src=[9296, 9284, 9464, 9160, 9254, 9296, 9116, 9300] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=65030 dup_rows=23982 dup_rows_remote=21082 dedup_ratio=0.243958 router_tokens_per_expert=[772, 720, 738, 753, 770, 788, 761, 752, 833, 741, 751, 762, 754, 792, 792, 815, 788, 802, 744, 802, 726, 770, 756, 763, 780, 741, 789, 771, 753, 720, 757, 785, 721, 724, 772, 710, 815, 703, 780, 742, 719, 762, 830, 800, 770, 785, 773, 753, 772, 799, 754, 757, 770, 805, 782, 773, 776, 773, 763, 804, 752, 744, 854, 774] payload_rows_per_dst=[9182, 9438, 9334, 9292, 9042, 9334, 9340, 9360] dup_rows_per_dst=[2926, 3042, 2968, 2900, 2892, 3050, 3084, 3120] combine_rows_per_src=[9182, 9438, 9334, 9292, 9042, 9334, 9340, 9360] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265969664 dispatch_meta_bytes=343728 combine_payload_bytes=265969664 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266313392 ingress_bytes=335175680 link_bytes=335175680 token_bytes=532283056 weight_movement_bytes=69206016 total_nvlink_bytes=601489072 weight_to_token_ratio=0.130017 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261239100e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266010624 dispatch_meta_bytes=343800 combine_payload_bytes=266010624 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266354424 ingress_bytes=335216640 link_bytes=335216640 token_bytes=532365048 weight_movement_bytes=69206016 total_nvlink_bytes=601571064 weight_to_token_ratio=0.129997 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261393229e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74170 payload_rows_remote=64874 dup_rows=24134 dup_rows_remote=21188 dedup_ratio=0.245504 router_tokens_per_expert=[772, 796, 785, 775, 764, 777, 728, 761, 792, 729, 760, 749, 795, 785, 790, 749, 758, 806, 766, 772, 820, 777, 760, 836, 703, 766, 766, 738, 809, 736, 708, 802, 773, 776, 781, 786, 781, 769, 798, 770, 778, 801, 707, 785, 797, 767, 749, 737, 751, 773, 728, 731, 734, 772, 725, 761, 797, 767, 814, 747, 763, 808, 749, 747] payload_rows_per_dst=[9296, 9284, 9464, 9160, 9254, 9296, 9116, 9300] dup_rows_per_dst=[3020, 3014, 3126, 2896, 3214, 2946, 2834, 3084] combine_rows_per_src=[9296, 9284, 9464, 9160, 9254, 9296, 9116, 9300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74250 payload_rows_remote=64842 dup_rows=24054 dup_rows_remote=21008 dedup_ratio=0.244690 router_tokens_per_expert=[713, 743, 774, 783, 785, 764, 729, 757, 715, 748, 761, 794, 788, 776, 782, 757, 774, 734, 762, 763, 794, 809, 768, 823, 787, 761, 778, 782, 786, 795, 754, 830, 812, 757, 765, 761, 803, 782, 787, 725, 756, 732, 815, 789, 777, 757, 750, 767, 758, 768, 787, 770, 742, 793, 762, 716, 768, 764, 770, 753, 754, 720, 772, 751] payload_rows_per_dst=[9152, 9236, 9408, 9424, 9394, 9288, 9166, 9182] dup_rows_per_dst=[2944, 3006, 3046, 3122, 2990, 2998, 3026, 2922] combine_rows_per_src=[9152, 9236, 9408, 9424, 9394, 9288, 9166, 9182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74318 payload_rows_remote=64796 dup_rows=23986 dup_rows_remote=20626 dedup_ratio=0.243998 router_tokens_per_expert=[728, 766, 771, 772, 776, 771, 788, 765, 798, 785, 758, 760, 795, 778, 761, 749, 791, 796, 825, 735, 765, 765, 794, 765, 804, 787, 774, 731, 749, 742, 723, 752, 759, 773, 729, 748, 726, 776, 827, 800, 788, 765, 722, 782, 785, 777, 697, 799, 750, 793, 736, 721, 775, 799, 747, 746, 793, 800, 783, 766, 763, 766, 753, 789] payload_rows_per_dst=[9314, 9324, 9168, 9162, 9296, 9034, 9498, 9522] dup_rows_per_dst=[2960, 3044, 2826, 2886, 3088, 2740, 3082, 3360] combine_rows_per_src=[9314, 9324, 9168, 9162, 9296, 9034, 9498, 9522] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265404416 dispatch_meta_bytes=341808 combine_payload_bytes=265404416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265746224 ingress_bytes=300007424 link_bytes=300007424 token_bytes=531150640 weight_movement_bytes=34603008 total_nvlink_bytes=565753648 weight_to_token_ratio=0.065147 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128903784e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265723904 dispatch_meta_bytes=344248 combine_payload_bytes=265723904 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266068152 ingress_bytes=334929920 link_bytes=334929920 token_bytes=531792056 weight_movement_bytes=69206016 total_nvlink_bytes=600998072 weight_to_token_ratio=0.130137 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260314325e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=64728 dup_rows=23930 dup_rows_remote=20472 dedup_ratio=0.243429 router_tokens_per_expert=[793, 777, 769, 761, 761, 797, 739, 768, 782, 759, 771, 787, 816, 783, 749, 756, 808, 769, 794, 732, 762, 711, 788, 748, 797, 793, 746, 754, 820, 779, 808, 775, 756, 744, 799, 741, 781, 781, 765, 755, 809, 744, 808, 737, 801, 785, 744, 794, 772, 786, 730, 741, 768, 798, 754, 762, 714, 730, 760, 748, 719, 786, 785, 703] payload_rows_per_dst=[9646, 9088, 9316, 9422, 9118, 9386, 9284, 9114] dup_rows_per_dst=[3458, 2820, 2922, 3122, 2850, 3044, 2938, 2776] combine_rows_per_src=[9646, 9088, 9316, 9422, 9118, 9386, 9284, 9114] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74280 payload_rows_remote=64968 dup_rows=24024 dup_rows_remote=20996 dedup_ratio=0.244385 router_tokens_per_expert=[750, 762, 799, 763, 699, 802, 778, 751, 764, 757, 805, 790, 750, 759, 767, 752, 730, 753, 736, 811, 770, 788, 757, 754, 813, 806, 761, 780, 760, 803, 749, 704, 787, 776, 761, 786, 784, 773, 784, 790, 767, 780, 769, 775, 794, 765, 764, 786, 724, 770, 815, 858, 727, 756, 775, 745, 769, 734, 803, 781, 760, 743, 722, 706] payload_rows_per_dst=[9256, 9278, 9236, 9292, 9326, 9378, 9312, 9202] dup_rows_per_dst=[2952, 3010, 2962, 3060, 3156, 3022, 3028, 2834] combine_rows_per_src=[9256, 9278, 9236, 9292, 9326, 9378, 9312, 9202] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=64894 dup_rows=23992 dup_rows_remote=20892 dedup_ratio=0.244059 router_tokens_per_expert=[786, 758, 770, 781, 776, 771, 762, 762, 745, 784, 787, 737, 786, 817, 766, 715, 730, 760, 827, 796, 761, 768, 742, 749, 792, 762, 795, 813, 777, 782, 759, 731, 737, 790, 743, 773, 742, 730, 763, 797, 780, 768, 745, 747, 726, 810, 788, 754, 758, 793, 745, 772, 789, 794, 813, 719, 805, 753, 821, 717, 732, 765, 764, 772] payload_rows_per_dst=[9298, 8768, 9610, 9468, 9248, 9264, 9238, 9418] dup_rows_per_dst=[3034, 2456, 3206, 3152, 2864, 3274, 2906, 3100] combine_rows_per_src=[9298, 8768, 9610, 9468, 9248, 9264, 9238, 9418] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344448 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266707328 ingress_bytes=335568896 link_bytes=335568896 token_bytes=533070208 weight_movement_bytes=69206016 total_nvlink_bytes=602276224 weight_to_token_ratio=0.129825 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262718740e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=65030 dup_rows=23982 dup_rows_remote=21082 dedup_ratio=0.243958 router_tokens_per_expert=[772, 720, 738, 753, 770, 788, 761, 752, 833, 741, 751, 762, 754, 792, 792, 815, 788, 802, 744, 802, 726, 770, 756, 763, 780, 741, 789, 771, 753, 720, 757, 785, 721, 724, 772, 710, 815, 703, 780, 742, 719, 762, 830, 800, 770, 785, 773, 753, 772, 799, 754, 757, 770, 805, 782, 773, 776, 773, 763, 804, 752, 744, 854, 774] payload_rows_per_dst=[9182, 9438, 9334, 9292, 9042, 9334, 9340, 9360] dup_rows_per_dst=[2926, 3042, 2968, 2900, 2892, 3050, 3084, 3120] combine_rows_per_src=[9182, 9438, 9334, 9292, 9042, 9334, 9340, 9360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74158 payload_rows_remote=64794 dup_rows=24146 dup_rows_remote=21186 dedup_ratio=0.245626 router_tokens_per_expert=[773, 744, 769, 749, 723, 769, 753, 753, 768, 795, 765, 771, 793, 805, 760, 765, 747, 754, 774, 812, 780, 792, 775, 766, 791, 780, 796, 714, 783, 754, 772, 769, 800, 779, 769, 797, 788, 751, 712, 766, 788, 719, 787, 749, 748, 770, 777, 791, 785, 755, 754, 787, 742, 790, 772, 761, 756, 766, 760, 763, 805, 718, 735, 798] payload_rows_per_dst=[9114, 9402, 9266, 9200, 9364, 9254, 9262, 9296] dup_rows_per_dst=[2952, 3042, 3134, 3118, 2960, 3004, 3030, 2906] combine_rows_per_src=[9114, 9402, 9266, 9200, 9364, 9254, 9262, 9296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265723904 dispatch_meta_bytes=344248 combine_payload_bytes=265723904 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266068152 ingress_bytes=334929920 link_bytes=334929920 token_bytes=531792056 weight_movement_bytes=69206016 total_nvlink_bytes=600998072 weight_to_token_ratio=0.130137 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260314325e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265592832 dispatch_meta_bytes=343400 combine_payload_bytes=265592832 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265936232 ingress_bytes=369401856 link_bytes=369401856 token_bytes=531529064 weight_movement_bytes=103809024 total_nvlink_bytes=635338088 weight_to_token_ratio=0.195303 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390029445e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265404416 dispatch_meta_bytes=341688 combine_payload_bytes=265404416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265746104 ingress_bytes=300007424 link_bytes=300007424 token_bytes=531150520 weight_movement_bytes=34603008 total_nvlink_bytes=565753528 weight_to_token_ratio=0.065147 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128903784e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=65104 dup_rows=24020 dup_rows_remote=21032 dedup_ratio=0.244344 router_tokens_per_expert=[756, 769, 711, 766, 782, 775, 764, 769, 796, 766, 791, 749, 804, 756, 758, 746, 759, 790, 742, 746, 785, 740, 740, 782, 761, 765, 758, 777, 819, 766, 823, 749, 759, 702, 802, 768, 794, 823, 743, 761, 688, 826, 784, 803, 742, 758, 817, 788, 708, 767, 788, 787, 744, 764, 760, 775, 720, 781, 759, 737, 810, 767, 786, 781] payload_rows_per_dst=[9348, 9300, 9180, 9314, 9362, 9354, 9186, 9240] dup_rows_per_dst=[2836, 3032, 2988, 3122, 2942, 3058, 3000, 3042] combine_rows_per_src=[9348, 9300, 9180, 9314, 9362, 9354, 9186, 9240] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=343856 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266452784 ingress_bytes=300711936 link_bytes=300711936 token_bytes=532561712 weight_movement_bytes=34603008 total_nvlink_bytes=567164720 weight_to_token_ratio=0.064975 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131554806e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265805824 dispatch_meta_bytes=343144 combine_payload_bytes=265805824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266148968 ingress_bytes=300408832 link_bytes=300408832 token_bytes=531954792 weight_movement_bytes=34603008 total_nvlink_bytes=566557800 weight_to_token_ratio=0.065049 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130414250e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=64818 dup_rows=24018 dup_rows_remote=20874 dedup_ratio=0.244324 router_tokens_per_expert=[753, 755, 797, 728, 759, 728, 759, 791, 768, 774, 770, 799, 794, 857, 788, 756, 758, 761, 749, 751, 758, 777, 769, 739, 755, 746, 816, 830, 780, 771, 789, 761, 788, 741, 776, 752, 740, 788, 767, 765, 753, 759, 781, 750, 773, 714, 771, 769, 766, 767, 771, 768, 771, 769, 728, 786, 757, 767, 734, 789, 783, 766, 792, 765] payload_rows_per_dst=[9210, 9468, 9176, 9388, 9300, 9164, 9270, 9310] dup_rows_per_dst=[2930, 3144, 2948, 3108, 2934, 2976, 2982, 2996] combine_rows_per_src=[9210, 9468, 9176, 9388, 9300, 9164, 9270, 9310] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74222 payload_rows_remote=64898 dup_rows=24082 dup_rows_remote=20930 dedup_ratio=0.244975 router_tokens_per_expert=[760, 752, 722, 765, 757, 776, 810, 773, 739, 761, 806, 803, 807, 802, 794, 730, 773, 803, 758, 783, 743, 778, 759, 745, 732, 706, 760, 750, 753, 773, 755, 753, 740, 807, 845, 812, 779, 762, 753, 740, 790, 755, 719, 742, 753, 730, 774, 789, 736, 731, 786, 815, 809, 791, 820, 740, 785, 751, 772, 803, 778, 747, 753, 764] payload_rows_per_dst=[9218, 9408, 9234, 9126, 9324, 9214, 9370, 9328] dup_rows_per_dst=[3012, 3076, 3050, 2838, 3152, 2890, 3086, 2978] combine_rows_per_src=[9218, 9408, 9234, 9126, 9324, 9214, 9370, 9328] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=64728 dup_rows=23930 dup_rows_remote=20472 dedup_ratio=0.243429 router_tokens_per_expert=[793, 777, 769, 761, 761, 797, 739, 768, 782, 759, 771, 787, 816, 783, 749, 756, 808, 769, 794, 732, 762, 711, 788, 748, 797, 793, 746, 754, 820, 779, 808, 775, 756, 744, 799, 741, 781, 781, 765, 755, 809, 744, 808, 737, 801, 785, 744, 794, 772, 786, 730, 741, 768, 798, 754, 762, 714, 730, 760, 748, 719, 786, 785, 703] payload_rows_per_dst=[9646, 9088, 9316, 9422, 9118, 9386, 9284, 9114] dup_rows_per_dst=[3458, 2820, 2922, 3122, 2850, 3044, 2938, 2776] combine_rows_per_src=[9646, 9088, 9316, 9422, 9118, 9386, 9284, 9114] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266665984 dispatch_meta_bytes=344544 combine_payload_bytes=266665984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267010528 ingress_bytes=301268992 link_bytes=301268992 token_bytes=533676512 weight_movement_bytes=34603008 total_nvlink_bytes=568279520 weight_to_token_ratio=0.064839 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133650963e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344448 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266707328 ingress_bytes=335568896 link_bytes=335568896 token_bytes=533070208 weight_movement_bytes=69206016 total_nvlink_bytes=602276224 weight_to_token_ratio=0.129825 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262718740e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265125888 dispatch_meta_bytes=340800 combine_payload_bytes=265125888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265466688 ingress_bytes=299728896 link_bytes=299728896 token_bytes=530592576 weight_movement_bytes=34603008 total_nvlink_bytes=565195584 weight_to_token_ratio=0.065216 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127855705e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=343856 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266452784 ingress_bytes=300711936 link_bytes=300711936 token_bytes=532561712 weight_movement_bytes=34603008 total_nvlink_bytes=567164720 weight_to_token_ratio=0.064975 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131554806e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265396224 dispatch_meta_bytes=343920 combine_payload_bytes=265396224 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265740144 ingress_bytes=334602240 link_bytes=334602240 token_bytes=531136368 weight_movement_bytes=69206016 total_nvlink_bytes=600342384 weight_to_token_ratio=0.130298 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259081291e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74420 payload_rows_remote=65174 dup_rows=23884 dup_rows_remote=21016 dedup_ratio=0.242961 router_tokens_per_expert=[776, 746, 763, 718, 806, 798, 796, 748, 739, 841, 750, 741, 802, 811, 781, 791, 725, 751, 721, 784, 785, 773, 770, 777, 771, 731, 775, 740, 764, 797, 795, 746, 799, 797, 791, 770, 796, 751, 756, 789, 782, 773, 717, 774, 731, 765, 762, 753, 801, 801, 750, 772, 745, 784, 765, 804, 736, 764, 764, 758, 745, 753, 716, 776] payload_rows_per_dst=[9322, 9416, 9256, 9310, 9418, 9246, 9322, 9130] dup_rows_per_dst=[2980, 3096, 2916, 2928, 3080, 2868, 3122, 2894] combine_rows_per_src=[9322, 9416, 9256, 9310, 9418, 9246, 9322, 9130] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=64902 dup_rows=24060 dup_rows_remote=20982 dedup_ratio=0.244751 router_tokens_per_expert=[825, 783, 751, 721, 777, 782, 776, 771, 767, 780, 801, 779, 776, 789, 765, 761, 799, 741, 729, 722, 765, 756, 763, 739, 773, 790, 794, 756, 763, 799, 783, 752, 740, 754, 772, 765, 799, 746, 776, 762, 768, 763, 793, 744, 756, 800, 733, 790, 760, 758, 727, 775, 763, 789, 764, 768, 759, 739, 758, 798, 773, 804, 753, 775] payload_rows_per_dst=[9264, 9418, 9144, 9342, 9250, 9248, 9278, 9300] dup_rows_per_dst=[3108, 3018, 2884, 3078, 2978, 3046, 2930, 3018] combine_rows_per_src=[9264, 9418, 9144, 9342, 9250, 9248, 9278, 9300] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265494528 dispatch_meta_bytes=342768 combine_payload_bytes=265494528 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=265837296 ingress_bytes=438509568 link_bytes=438509568 token_bytes=531331824 weight_movement_bytes=173015040 total_nvlink_bytes=704346864 weight_to_token_ratio=0.325625 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.650076201e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74420 payload_rows_remote=65174 dup_rows=23884 dup_rows_remote=21016 dedup_ratio=0.242961 router_tokens_per_expert=[776, 746, 763, 718, 806, 798, 796, 748, 739, 841, 750, 741, 802, 811, 781, 791, 725, 751, 721, 784, 785, 773, 770, 777, 771, 731, 775, 740, 764, 797, 795, 746, 799, 797, 791, 770, 796, 751, 756, 789, 782, 773, 717, 774, 731, 765, 762, 753, 801, 801, 750, 772, 745, 784, 765, 804, 736, 764, 764, 758, 745, 753, 716, 776] payload_rows_per_dst=[9322, 9416, 9256, 9310, 9418, 9246, 9322, 9130] dup_rows_per_dst=[2980, 3096, 2916, 2928, 3080, 2868, 3122, 2894] combine_rows_per_src=[9322, 9416, 9256, 9310, 9418, 9246, 9322, 9130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=64894 dup_rows=23992 dup_rows_remote=20892 dedup_ratio=0.244059 router_tokens_per_expert=[786, 758, 770, 781, 776, 771, 762, 762, 745, 784, 787, 737, 786, 817, 766, 715, 730, 760, 827, 796, 761, 768, 742, 749, 792, 762, 795, 813, 777, 782, 759, 731, 737, 790, 743, 773, 742, 730, 763, 797, 780, 768, 745, 747, 726, 810, 788, 754, 758, 793, 745, 772, 789, 794, 813, 719, 805, 753, 821, 717, 732, 765, 764, 772] payload_rows_per_dst=[9298, 8768, 9610, 9468, 9248, 9264, 9238, 9418] dup_rows_per_dst=[3034, 2456, 3206, 3152, 2864, 3274, 2906, 3100] combine_rows_per_src=[9298, 8768, 9610, 9468, 9248, 9264, 9238, 9418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=65104 dup_rows=24020 dup_rows_remote=21032 dedup_ratio=0.244344 router_tokens_per_expert=[756, 769, 711, 766, 782, 775, 764, 769, 796, 766, 791, 749, 804, 756, 758, 746, 759, 790, 742, 746, 785, 740, 740, 782, 761, 765, 758, 777, 819, 766, 823, 749, 759, 702, 802, 768, 794, 823, 743, 761, 688, 826, 784, 803, 742, 758, 817, 788, 708, 767, 788, 787, 744, 764, 760, 775, 720, 781, 759, 737, 810, 767, 786, 781] payload_rows_per_dst=[9348, 9300, 9180, 9314, 9362, 9354, 9186, 9240] dup_rows_per_dst=[2836, 3032, 2988, 3122, 2942, 3058, 3000, 3042] combine_rows_per_src=[9348, 9300, 9180, 9314, 9362, 9354, 9186, 9240] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265822208 dispatch_meta_bytes=343312 combine_payload_bytes=265822208 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266165520 ingress_bytes=300425216 link_bytes=300425216 token_bytes=531987728 weight_movement_bytes=34603008 total_nvlink_bytes=566590736 weight_to_token_ratio=0.065045 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130475902e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265125888 dispatch_meta_bytes=340800 combine_payload_bytes=265125888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265466688 ingress_bytes=299728896 link_bytes=299728896 token_bytes=530592576 weight_movement_bytes=34603008 total_nvlink_bytes=565195584 weight_to_token_ratio=0.065216 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127855705e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266952704 dispatch_meta_bytes=344760 combine_payload_bytes=266952704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267297464 ingress_bytes=301555712 link_bytes=301555712 token_bytes=534250168 weight_movement_bytes=34603008 total_nvlink_bytes=568853176 weight_to_token_ratio=0.064769 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134729867e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=64902 dup_rows=24060 dup_rows_remote=20982 dedup_ratio=0.244751 router_tokens_per_expert=[825, 783, 751, 721, 777, 782, 776, 771, 767, 780, 801, 779, 776, 789, 765, 761, 799, 741, 729, 722, 765, 756, 763, 739, 773, 790, 794, 756, 763, 799, 783, 752, 740, 754, 772, 765, 799, 746, 776, 762, 768, 763, 793, 744, 756, 800, 733, 790, 760, 758, 727, 775, 763, 789, 764, 768, 759, 739, 758, 798, 773, 804, 753, 775] payload_rows_per_dst=[9264, 9418, 9144, 9342, 9250, 9248, 9278, 9300] dup_rows_per_dst=[3108, 3018, 2884, 3078, 2978, 3046, 2930, 3018] combine_rows_per_src=[9264, 9418, 9144, 9342, 9250, 9248, 9278, 9300] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74062 payload_rows_remote=64768 dup_rows=24242 dup_rows_remote=21172 dedup_ratio=0.246602 router_tokens_per_expert=[784, 764, 811, 777, 718, 814, 751, 695, 792, 778, 835, 753, 784, 786, 773, 789, 756, 730, 758, 736, 787, 759, 770, 777, 768, 748, 773, 730, 766, 765, 703, 804, 779, 769, 789, 732, 721, 811, 771, 770, 786, 764, 793, 751, 767, 749, 786, 721, 769, 751, 800, 799, 761, 795, 771, 736, 794, 761, 768, 765, 763, 776, 798, 752] payload_rows_per_dst=[9278, 9450, 9224, 9140, 9136, 9288, 9294, 9252] dup_rows_per_dst=[2950, 3130, 2922, 2974, 3148, 2946, 3070, 3102] combine_rows_per_src=[9278, 9450, 9224, 9140, 9136, 9288, 9294, 9252] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74588 payload_rows_remote=65244 dup_rows=23716 dup_rows_remote=20650 dedup_ratio=0.241252 router_tokens_per_expert=[787, 744, 747, 746, 762, 774, 721, 774, 755, 789, 774, 750, 765, 795, 748, 767, 732, 730, 727, 776, 747, 763, 840, 738, 771, 736, 780, 822, 791, 777, 799, 762, 814, 822, 746, 743, 778, 745, 743, 764, 773, 775, 786, 736, 800, 749, 763, 768, 759, 800, 775, 767, 792, 799, 753, 780, 774, 769, 745, 762, 811, 755, 719, 798] payload_rows_per_dst=[9230, 9162, 9210, 9474, 9322, 9324, 9522, 9344] dup_rows_per_dst=[2880, 2964, 2896, 3002, 2988, 2886, 3034, 3066] combine_rows_per_src=[9230, 9162, 9210, 9474, 9322, 9324, 9522, 9344] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265838592 dispatch_meta_bytes=343536 combine_payload_bytes=265838592 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266182128 ingress_bytes=300441600 link_bytes=300441600 token_bytes=532020720 weight_movement_bytes=34603008 total_nvlink_bytes=566623728 weight_to_token_ratio=0.065041 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130537553e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74204 payload_rows_remote=64826 dup_rows=24100 dup_rows_remote=20962 dedup_ratio=0.245158 router_tokens_per_expert=[738, 779, 774, 741, 797, 769, 748, 775, 775, 737, 795, 719, 740, 794, 795, 786, 784, 806, 795, 753, 792, 768, 770, 815, 756, 726, 739, 770, 780, 758, 740, 739, 786, 783, 717, 719, 813, 747, 784, 735, 768, 797, 757, 773, 827, 779, 770, 763, 779, 754, 752, 779, 780, 773, 771, 795, 762, 789, 757, 788, 773, 732, 759, 738] payload_rows_per_dst=[9378, 9322, 9426, 9068, 9210, 9270, 9262, 9268] dup_rows_per_dst=[3138, 2802, 3140, 2948, 2958, 3198, 2988, 2928] combine_rows_per_src=[9378, 9322, 9426, 9068, 9210, 9270, 9262, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74310 payload_rows_remote=64944 dup_rows=23994 dup_rows_remote=21006 dedup_ratio=0.244080 router_tokens_per_expert=[740, 795, 752, 745, 777, 749, 790, 750, 755, 785, 771, 735, 707, 800, 837, 784, 748, 760, 771, 753, 762, 756, 802, 747, 729, 757, 736, 776, 789, 798, 756, 796, 784, 765, 813, 739, 751, 765, 797, 763, 768, 773, 799, 762, 759, 774, 774, 845, 756, 763, 753, 796, 737, 769, 792, 783, 767, 785, 721, 764, 770, 773, 747, 737] payload_rows_per_dst=[9244, 9322, 9210, 9292, 9366, 9426, 9258, 9192] dup_rows_per_dst=[2952, 3026, 2988, 2982, 2988, 3082, 3040, 2936] combine_rows_per_src=[9244, 9322, 9210, 9292, 9366, 9426, 9258, 9192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266952704 dispatch_meta_bytes=344760 combine_payload_bytes=266952704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267297464 ingress_bytes=301555712 link_bytes=301555712 token_bytes=534250168 weight_movement_bytes=34603008 total_nvlink_bytes=568853176 weight_to_token_ratio=0.064769 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134729867e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265805824 dispatch_meta_bytes=343144 combine_payload_bytes=265805824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266148968 ingress_bytes=300408832 link_bytes=300408832 token_bytes=531954792 weight_movement_bytes=34603008 total_nvlink_bytes=566557800 weight_to_token_ratio=0.065049 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130414250e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266665984 dispatch_meta_bytes=344544 combine_payload_bytes=266665984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267010528 ingress_bytes=301268992 link_bytes=301268992 token_bytes=533676512 weight_movement_bytes=34603008 total_nvlink_bytes=568279520 weight_to_token_ratio=0.064839 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133650963e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74394 payload_rows_remote=65164 dup_rows=23910 dup_rows_remote=20916 dedup_ratio=0.243225 router_tokens_per_expert=[751, 731, 794, 753, 758, 748, 761, 758, 783, 731, 791, 807, 760, 775, 797, 740, 756, 777, 777, 777, 749, 747, 754, 775, 786, 722, 825, 694, 714, 820, 706, 753, 770, 772, 745, 752, 797, 761, 771, 754, 745, 806, 759, 744, 818, 775, 772, 785, 773, 773, 781, 786, 762, 800, 806, 774, 757, 805, 750, 822, 736, 785, 759, 787] payload_rows_per_dst=[9134, 9466, 9230, 9136, 9296, 9378, 9450, 9304] dup_rows_per_dst=[2974, 2902, 2994, 2904, 2948, 3030, 3060, 3098] combine_rows_per_src=[9134, 9466, 9230, 9136, 9296, 9378, 9450, 9304] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265289728 dispatch_meta_bytes=343760 combine_payload_bytes=265289728 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265633488 ingress_bytes=369098752 link_bytes=369098752 token_bytes=530923216 weight_movement_bytes=103809024 total_nvlink_bytes=634732240 weight_to_token_ratio=0.195525 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388888889e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267239424 dispatch_meta_bytes=343576 combine_payload_bytes=267239424 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267583000 ingress_bytes=336445440 link_bytes=336445440 token_bytes=534822424 weight_movement_bytes=69206016 total_nvlink_bytes=604028440 weight_to_token_ratio=0.129400 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266017105e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=65020 dup_rows=24018 dup_rows_remote=21080 dedup_ratio=0.244324 router_tokens_per_expert=[803, 765, 770, 741, 780, 811, 787, 728, 779, 791, 752, 762, 741, 773, 750, 754, 745, 787, 739, 732, 752, 769, 811, 767, 799, 767, 803, 752, 751, 753, 715, 817, 720, 784, 752, 806, 775, 749, 784, 786, 780, 729, 775, 820, 751, 745, 797, 784, 775, 747, 781, 768, 744, 799, 741, 777, 817, 725, 736, 774, 772, 763, 750, 800] payload_rows_per_dst=[9390, 9266, 9272, 9316, 9222, 9320, 9232, 9268] dup_rows_per_dst=[2980, 2938, 2932, 2998, 3090, 3042, 3032, 3006] combine_rows_per_src=[9390, 9266, 9272, 9316, 9222, 9320, 9232, 9268] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74222 payload_rows_remote=64874 dup_rows=24082 dup_rows_remote=21154 dedup_ratio=0.244975 router_tokens_per_expert=[762, 776, 736, 757, 762, 787, 764, 784, 742, 775, 787, 781, 756, 738, 747, 767, 795, 769, 760, 743, 740, 731, 767, 795, 757, 806, 821, 734, 767, 737, 725, 776, 772, 817, 774, 781, 739, 771, 722, 762, 748, 853, 740, 786, 748, 796, 797, 755, 774, 744, 783, 800, 792, 765, 782, 773, 760, 765, 747, 758, 775, 756, 801, 772] payload_rows_per_dst=[9240, 9220, 9286, 9210, 9348, 9400, 9270, 9248] dup_rows_per_dst=[3016, 2966, 2914, 3036, 2928, 3046, 3156, 3020] combine_rows_per_src=[9240, 9220, 9286, 9210, 9348, 9400, 9270, 9248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74204 payload_rows_remote=64826 dup_rows=24100 dup_rows_remote=20962 dedup_ratio=0.245158 router_tokens_per_expert=[738, 779, 774, 741, 797, 769, 748, 775, 775, 737, 795, 719, 740, 794, 795, 786, 784, 806, 795, 753, 792, 768, 770, 815, 756, 726, 739, 770, 780, 758, 740, 739, 786, 783, 717, 719, 813, 747, 784, 735, 768, 797, 757, 773, 827, 779, 770, 763, 779, 754, 752, 779, 780, 773, 771, 795, 762, 789, 757, 788, 773, 732, 759, 738] payload_rows_per_dst=[9378, 9322, 9426, 9068, 9210, 9270, 9262, 9268] dup_rows_per_dst=[3138, 2802, 3140, 2948, 2958, 3198, 2988, 2928] combine_rows_per_src=[9378, 9322, 9426, 9068, 9210, 9270, 9262, 9268] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266911744 dispatch_meta_bytes=344320 combine_payload_bytes=266911744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267256064 ingress_bytes=301514752 link_bytes=301514752 token_bytes=534167808 weight_movement_bytes=34603008 total_nvlink_bytes=568770816 weight_to_token_ratio=0.064779 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134575738e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265838592 dispatch_meta_bytes=343536 combine_payload_bytes=265838592 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266182128 ingress_bytes=300441600 link_bytes=300441600 token_bytes=532020720 weight_movement_bytes=34603008 total_nvlink_bytes=566623728 weight_to_token_ratio=0.065041 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130537553e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265289728 dispatch_meta_bytes=343760 combine_payload_bytes=265289728 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265633488 ingress_bytes=369098752 link_bytes=369098752 token_bytes=530923216 weight_movement_bytes=103809024 total_nvlink_bytes=634732240 weight_to_token_ratio=0.195525 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388888889e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265527296 dispatch_meta_bytes=343152 combine_payload_bytes=265527296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265870448 ingress_bytes=300130304 link_bytes=300130304 token_bytes=531397744 weight_movement_bytes=34603008 total_nvlink_bytes=566000752 weight_to_token_ratio=0.065117 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129366171e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74318 payload_rows_remote=65002 dup_rows=23986 dup_rows_remote=20976 dedup_ratio=0.243998 router_tokens_per_expert=[761, 722, 775, 783, 795, 748, 757, 761, 783, 823, 799, 755, 738, 761, 791, 782, 748, 826, 786, 760, 752, 725, 821, 765, 758, 760, 778, 777, 753, 754, 760, 786, 821, 737, 728, 735, 763, 746, 728, 778, 764, 751, 791, 761, 759, 806, 767, 764, 772, 736, 761, 785, 752, 810, 771, 834, 723, 783, 757, 784, 802, 759, 737, 744] payload_rows_per_dst=[9284, 9448, 9208, 9208, 9210, 9316, 9388, 9256] dup_rows_per_dst=[2920, 3016, 3158, 3044, 2862, 3010, 3054, 2922] combine_rows_per_src=[9284, 9448, 9208, 9208, 9210, 9316, 9388, 9256] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74446 payload_rows_remote=65256 dup_rows=23858 dup_rows_remote=20994 dedup_ratio=0.242696 router_tokens_per_expert=[762, 738, 747, 770, 779, 766, 708, 762, 790, 799, 750, 714, 844, 765, 774, 769, 748, 761, 789, 757, 729, 782, 779, 777, 729, 773, 758, 739, 752, 737, 748, 791, 781, 782, 750, 740, 778, 728, 743, 773, 780, 808, 783, 794, 733, 793, 791, 782, 778, 768, 815, 709, 770, 743, 755, 751, 810, 789, 753, 830, 730, 836, 838, 752] payload_rows_per_dst=[9178, 9390, 9234, 9190, 9176, 9470, 9258, 9550] dup_rows_per_dst=[2886, 3020, 3010, 2864, 2974, 3058, 2920, 3126] combine_rows_per_src=[9178, 9390, 9234, 9190, 9176, 9470, 9258, 9550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266010624 dispatch_meta_bytes=343800 combine_payload_bytes=266010624 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266354424 ingress_bytes=335216640 link_bytes=335216640 token_bytes=532365048 weight_movement_bytes=69206016 total_nvlink_bytes=601571064 weight_to_token_ratio=0.129997 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261393229e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266321920 dispatch_meta_bytes=344400 combine_payload_bytes=266321920 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266666320 ingress_bytes=335527936 link_bytes=335527936 token_bytes=532988240 weight_movement_bytes=69206016 total_nvlink_bytes=602194256 weight_to_token_ratio=0.129845 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262564611e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74318 payload_rows_remote=65002 dup_rows=23986 dup_rows_remote=20976 dedup_ratio=0.243998 router_tokens_per_expert=[761, 722, 775, 783, 795, 748, 757, 761, 783, 823, 799, 755, 738, 761, 791, 782, 748, 826, 786, 760, 752, 725, 821, 765, 758, 760, 778, 777, 753, 754, 760, 786, 821, 737, 728, 735, 763, 746, 728, 778, 764, 751, 791, 761, 759, 806, 767, 764, 772, 736, 761, 785, 752, 810, 771, 834, 723, 783, 757, 784, 802, 759, 737, 744] payload_rows_per_dst=[9284, 9448, 9208, 9208, 9210, 9316, 9388, 9256] dup_rows_per_dst=[2920, 3016, 3158, 3044, 2862, 3010, 3054, 2922] combine_rows_per_src=[9284, 9448, 9208, 9208, 9210, 9316, 9388, 9256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74588 payload_rows_remote=65244 dup_rows=23716 dup_rows_remote=20650 dedup_ratio=0.241252 router_tokens_per_expert=[787, 744, 747, 746, 762, 774, 721, 774, 755, 789, 774, 750, 765, 795, 748, 767, 732, 730, 727, 776, 747, 763, 840, 738, 771, 736, 780, 822, 791, 777, 799, 762, 814, 822, 746, 743, 778, 745, 743, 764, 773, 775, 786, 736, 800, 749, 763, 768, 759, 800, 775, 767, 792, 799, 753, 780, 774, 769, 745, 762, 811, 755, 719, 798] payload_rows_per_dst=[9230, 9162, 9210, 9474, 9322, 9324, 9522, 9344] dup_rows_per_dst=[2880, 2964, 2896, 3002, 2988, 2886, 3034, 3066] combine_rows_per_src=[9230, 9162, 9210, 9474, 9322, 9324, 9522, 9344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74394 payload_rows_remote=65164 dup_rows=23910 dup_rows_remote=20916 dedup_ratio=0.243225 router_tokens_per_expert=[751, 731, 794, 753, 758, 748, 761, 758, 783, 731, 791, 807, 760, 775, 797, 740, 756, 777, 777, 777, 749, 747, 754, 775, 786, 722, 825, 694, 714, 820, 706, 753, 770, 772, 745, 752, 797, 761, 771, 754, 745, 806, 759, 744, 818, 775, 772, 785, 773, 773, 781, 786, 762, 800, 806, 774, 757, 805, 750, 822, 736, 785, 759, 787] payload_rows_per_dst=[9134, 9466, 9230, 9136, 9296, 9378, 9450, 9304] dup_rows_per_dst=[2974, 2902, 2994, 2904, 2948, 3030, 3060, 3098] combine_rows_per_src=[9134, 9466, 9230, 9136, 9296, 9378, 9450, 9304] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265723904 dispatch_meta_bytes=344112 combine_payload_bytes=265723904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266068016 ingress_bytes=300326912 link_bytes=300326912 token_bytes=531791920 weight_movement_bytes=34603008 total_nvlink_bytes=566394928 weight_to_token_ratio=0.065069 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130105992e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265527296 dispatch_meta_bytes=343152 combine_payload_bytes=265527296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265870448 ingress_bytes=300130304 link_bytes=300130304 token_bytes=531397744 weight_movement_bytes=34603008 total_nvlink_bytes=566000752 weight_to_token_ratio=0.065117 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129366171e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=343912 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266592104 ingress_bytes=370057216 link_bytes=370057216 token_bytes=532840296 weight_movement_bytes=103809024 total_nvlink_bytes=636649320 weight_to_token_ratio=0.194822 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392495512e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74446 payload_rows_remote=65256 dup_rows=23858 dup_rows_remote=20994 dedup_ratio=0.242696 router_tokens_per_expert=[762, 738, 747, 770, 779, 766, 708, 762, 790, 799, 750, 714, 844, 765, 774, 769, 748, 761, 789, 757, 729, 782, 779, 777, 729, 773, 758, 739, 752, 737, 748, 791, 781, 782, 750, 740, 778, 728, 743, 773, 780, 808, 783, 794, 733, 793, 791, 782, 778, 768, 815, 709, 770, 743, 755, 751, 810, 789, 753, 830, 730, 836, 838, 752] payload_rows_per_dst=[9178, 9390, 9234, 9190, 9176, 9470, 9258, 9550] dup_rows_per_dst=[2886, 3020, 3010, 2864, 2974, 3058, 2920, 3126] combine_rows_per_src=[9178, 9390, 9234, 9190, 9176, 9470, 9258, 9550] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74366 payload_rows_remote=64920 dup_rows=23938 dup_rows_remote=20900 dedup_ratio=0.243510 router_tokens_per_expert=[751, 751, 798, 773, 767, 746, 743, 765, 816, 816, 743, 793, 756, 743, 775, 769, 764, 781, 761, 737, 766, 742, 740, 807, 774, 758, 740, 784, 755, 768, 742, 783, 790, 801, 782, 720, 773, 770, 754, 770, 750, 758, 748, 767, 727, 806, 799, 727, 790, 782, 806, 808, 766, 745, 740, 805, 751, 777, 751, 788, 742, 781, 770, 801] payload_rows_per_dst=[9210, 9344, 9260, 9254, 9318, 9220, 9446, 9314] dup_rows_per_dst=[2978, 3078, 2936, 2954, 3002, 2944, 3038, 3008] combine_rows_per_src=[9210, 9344, 9260, 9254, 9318, 9220, 9446, 9314] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74098 payload_rows_remote=64788 dup_rows=24206 dup_rows_remote=21214 dedup_ratio=0.246236 router_tokens_per_expert=[768, 732, 758, 752, 742, 765, 735, 772, 724, 758, 748, 807, 741, 741, 791, 765, 801, 800, 781, 763, 762, 766, 743, 750, 774, 789, 767, 770, 815, 722, 738, 782, 744, 777, 792, 768, 742, 780, 719, 824, 795, 774, 799, 794, 793, 791, 767, 801, 770, 779, 801, 743, 766, 772, 764, 731, 737, 814, 810, 746, 768, 745, 776, 748] payload_rows_per_dst=[9086, 9160, 9322, 9310, 9252, 9470, 9188, 9310] dup_rows_per_dst=[2962, 2924, 3052, 3006, 2948, 3258, 3064, 2992] combine_rows_per_src=[9086, 9160, 9322, 9310, 9252, 9470, 9188, 9310] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267288576 dispatch_meta_bytes=345000 combine_payload_bytes=267288576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267633576 ingress_bytes=301891584 link_bytes=301891584 token_bytes=534922152 weight_movement_bytes=34603008 total_nvlink_bytes=569525160 weight_to_token_ratio=0.064688 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135993726e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=343912 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266592104 ingress_bytes=370057216 link_bytes=370057216 token_bytes=532840296 weight_movement_bytes=103809024 total_nvlink_bytes=636649320 weight_to_token_ratio=0.194822 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392495512e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267239424 dispatch_meta_bytes=343576 combine_payload_bytes=267239424 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267583000 ingress_bytes=336445440 link_bytes=336445440 token_bytes=534822424 weight_movement_bytes=69206016 total_nvlink_bytes=604028440 weight_to_token_ratio=0.129400 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266017105e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73710 payload_rows_remote=64134 dup_rows=24594 dup_rows_remote=21166 dedup_ratio=0.250183 router_tokens_per_expert=[806, 814, 756, 799, 789, 785, 775, 793, 748, 762, 772, 779, 770, 728, 757, 746, 761, 805, 734, 757, 744, 771, 757, 794, 778, 708, 751, 763, 751, 748, 776, 801, 763, 716, 744, 800, 786, 745, 737, 803, 787, 762, 757, 790, 756, 830, 761, 746, 813, 756, 755, 820, 774, 798, 708, 764, 736, 735, 790, 782, 773, 781, 769, 737] payload_rows_per_dst=[9576, 9166, 9350, 9400, 9422, 9080, 8758, 8958] dup_rows_per_dst=[3428, 2958, 3414, 3314, 3198, 2780, 2686, 2816] combine_rows_per_src=[9576, 9166, 9350, 9400, 9422, 9080, 8758, 8958] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74222 payload_rows_remote=64898 dup_rows=24082 dup_rows_remote=20930 dedup_ratio=0.244975 router_tokens_per_expert=[760, 752, 722, 765, 757, 776, 810, 773, 739, 761, 806, 803, 807, 802, 794, 730, 773, 803, 758, 783, 743, 778, 759, 745, 732, 706, 760, 750, 753, 773, 755, 753, 740, 807, 845, 812, 779, 762, 753, 740, 790, 755, 719, 742, 753, 730, 774, 789, 736, 731, 786, 815, 809, 791, 820, 740, 785, 751, 772, 803, 778, 747, 753, 764] payload_rows_per_dst=[9218, 9408, 9234, 9126, 9324, 9214, 9370, 9328] dup_rows_per_dst=[3012, 3076, 3050, 2838, 3152, 2890, 3086, 2978] combine_rows_per_src=[9218, 9408, 9234, 9126, 9324, 9214, 9370, 9328] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=64998 dup_rows=23966 dup_rows_remote=20984 dedup_ratio=0.243795 router_tokens_per_expert=[737, 768, 778, 751, 775, 796, 768, 769, 720, 791, 796, 773, 728, 790, 801, 775, 780, 748, 782, 749, 770, 761, 812, 759, 783, 742, 722, 840, 768, 795, 794, 763, 775, 756, 786, 719, 731, 773, 804, 738, 770, 738, 719, 782, 798, 788, 744, 763, 808, 758, 791, 781, 754, 758, 740, 795, 766, 758, 746, 757, 768, 766, 785, 753] payload_rows_per_dst=[9304, 9242, 9340, 9388, 9240, 9300, 9388, 9136] dup_rows_per_dst=[2980, 3106, 2982, 3026, 2924, 2904, 2982, 3062] combine_rows_per_src=[9304, 9242, 9340, 9388, 9240, 9300, 9388, 9136] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265912320 dispatch_meta_bytes=343280 combine_payload_bytes=265912320 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266255600 ingress_bytes=369721344 link_bytes=369721344 token_bytes=532167920 weight_movement_bytes=103809024 total_nvlink_bytes=635976944 weight_to_token_ratio=0.195068 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391231652e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265371648 dispatch_meta_bytes=344008 combine_payload_bytes=265371648 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265715656 ingress_bytes=334577664 link_bytes=334577664 token_bytes=531087304 weight_movement_bytes=69206016 total_nvlink_bytes=600293320 weight_to_token_ratio=0.130310 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258988814e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266911744 dispatch_meta_bytes=344320 combine_payload_bytes=266911744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267256064 ingress_bytes=301514752 link_bytes=301514752 token_bytes=534167808 weight_movement_bytes=34603008 total_nvlink_bytes=568770816 weight_to_token_ratio=0.064779 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134575738e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267288576 dispatch_meta_bytes=345000 combine_payload_bytes=267288576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267633576 ingress_bytes=301891584 link_bytes=301891584 token_bytes=534922152 weight_movement_bytes=34603008 total_nvlink_bytes=569525160 weight_to_token_ratio=0.064688 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135993726e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74422 payload_rows_remote=64990 dup_rows=23882 dup_rows_remote=20744 dedup_ratio=0.242940 router_tokens_per_expert=[735, 810, 689, 785, 761, 761, 773, 772, 792, 808, 815, 762, 810, 786, 732, 780, 828, 789, 757, 781, 754, 776, 744, 755, 802, 763, 739, 838, 736, 740, 739, 706, 819, 798, 729, 726, 797, 789, 777, 764, 804, 767, 800, 755, 755, 737, 776, 736, 752, 762, 756, 728, 745, 752, 753, 733, 805, 763, 769, 807, 768, 767, 733, 812] payload_rows_per_dst=[9248, 9432, 9338, 9252, 9414, 9222, 9104, 9412] dup_rows_per_dst=[2924, 3138, 3030, 2874, 2984, 3038, 2858, 3036] combine_rows_per_src=[9248, 9432, 9338, 9252, 9414, 9222, 9104, 9412] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74138 payload_rows_remote=64920 dup_rows=24166 dup_rows_remote=21022 dedup_ratio=0.245829 router_tokens_per_expert=[742, 788, 762, 771, 780, 762, 821, 744, 739, 730, 750, 798, 797, 777, 778, 795, 755, 748, 771, 805, 781, 723, 729, 744, 780, 735, 758, 749, 767, 779, 741, 799, 761, 764, 769, 777, 803, 757, 786, 764, 787, 754, 742, 758, 807, 773, 733, 767, 818, 735, 772, 736, 773, 771, 789, 797, 765, 785, 744, 799, 776, 752, 772, 768] payload_rows_per_dst=[9328, 9200, 9258, 9214, 9218, 9322, 9356, 9242] dup_rows_per_dst=[3012, 3128, 2854, 3002, 3144, 2920, 3026, 3080] combine_rows_per_src=[9328, 9200, 9258, 9214, 9218, 9322, 9356, 9242] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266231808 dispatch_meta_bytes=343928 combine_payload_bytes=266231808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266575736 ingress_bytes=300834816 link_bytes=300834816 token_bytes=532807544 weight_movement_bytes=34603008 total_nvlink_bytes=567410552 weight_to_token_ratio=0.064945 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132017193e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73710 payload_rows_remote=64134 dup_rows=24594 dup_rows_remote=21166 dedup_ratio=0.250183 router_tokens_per_expert=[806, 814, 756, 799, 789, 785, 775, 793, 748, 762, 772, 779, 770, 728, 757, 746, 761, 805, 734, 757, 744, 771, 757, 794, 778, 708, 751, 763, 751, 748, 776, 801, 763, 716, 744, 800, 786, 745, 737, 803, 787, 762, 757, 790, 756, 830, 761, 746, 813, 756, 755, 820, 774, 798, 708, 764, 736, 735, 790, 782, 773, 781, 769, 737] payload_rows_per_dst=[9576, 9166, 9350, 9400, 9422, 9080, 8758, 8958] dup_rows_per_dst=[3428, 2958, 3414, 3314, 3198, 2780, 2686, 2816] combine_rows_per_src=[9576, 9166, 9350, 9400, 9422, 9080, 8758, 8958] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74514 payload_rows_remote=65140 dup_rows=23790 dup_rows_remote=20666 dedup_ratio=0.242004 router_tokens_per_expert=[762, 773, 751, 752, 767, 732, 763, 742, 739, 788, 764, 788, 765, 769, 767, 773, 788, 785, 789, 736, 751, 736, 747, 801, 760, 774, 781, 805, 767, 760, 741, 786, 719, 815, 741, 717, 813, 742, 744, 759, 779, 801, 798, 715, 817, 793, 749, 797, 717, 761, 784, 774, 779, 787, 727, 780, 778, 783, 744, 774, 746, 792, 830, 795] payload_rows_per_dst=[9160, 9278, 9322, 9362, 9244, 9374, 9330, 9444] dup_rows_per_dst=[2924, 3028, 2944, 2986, 2856, 3124, 2888, 3040] combine_rows_per_src=[9160, 9278, 9322, 9362, 9244, 9374, 9330, 9444] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265912320 dispatch_meta_bytes=343280 combine_payload_bytes=265912320 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266255600 ingress_bytes=369721344 link_bytes=369721344 token_bytes=532167920 weight_movement_bytes=103809024 total_nvlink_bytes=635976944 weight_to_token_ratio=0.195068 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391231652e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=64894 dup_rows=24046 dup_rows_remote=21002 dedup_ratio=0.244609 router_tokens_per_expert=[784, 791, 772, 765, 773, 727, 754, 742, 790, 793, 745, 759, 790, 812, 755, 772, 744, 804, 776, 766, 750, 755, 790, 799, 783, 765, 774, 791, 780, 763, 787, 761, 781, 724, 776, 799, 797, 772, 765, 731, 759, 755, 730, 750, 744, 755, 777, 761, 759, 763, 762, 760, 817, 822, 785, 746, 813, 803, 728, 712, 742, 730, 767, 755] payload_rows_per_dst=[9240, 9390, 9300, 9364, 9284, 9138, 9340, 9202] dup_rows_per_dst=[2976, 3042, 3068, 3044, 3006, 2924, 3088, 2898] combine_rows_per_src=[9240, 9390, 9300, 9364, 9284, 9138, 9340, 9202] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=342936 combine_payload_bytes=266199040 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266541976 ingress_bytes=404611072 link_bytes=404611072 token_bytes=532741016 weight_movement_bytes=138412032 total_nvlink_bytes=671153048 weight_to_token_ratio=0.259811 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522518890e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265912320 dispatch_meta_bytes=343768 combine_payload_bytes=265912320 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266256088 ingress_bytes=335118336 link_bytes=335118336 token_bytes=532168408 weight_movement_bytes=69206016 total_nvlink_bytes=601374424 weight_to_token_ratio=0.130045 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261023319e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=262692864 dispatch_meta_bytes=341200 combine_payload_bytes=262692864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263034064 ingress_bytes=297295872 link_bytes=297295872 token_bytes=525726928 weight_movement_bytes=34603008 total_nvlink_bytes=560329936 weight_to_token_ratio=0.065819 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.118700432e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266813440 dispatch_meta_bytes=343224 combine_payload_bytes=266813440 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267156664 ingress_bytes=336019456 link_bytes=336019456 token_bytes=533970104 weight_movement_bytes=69206016 total_nvlink_bytes=603176120 weight_to_token_ratio=0.129607 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264414161e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74020 payload_rows_remote=64654 dup_rows=24284 dup_rows_remote=21076 dedup_ratio=0.247030 router_tokens_per_expert=[786, 842, 747, 786, 772, 829, 750, 792, 762, 764, 776, 821, 763, 718, 744, 753, 747, 779, 789, 776, 742, 794, 810, 762, 784, 780, 772, 732, 762, 760, 740, 806, 722, 756, 743, 801, 759, 748, 742, 744, 791, 747, 749, 821, 757, 820, 783, 746, 748, 771, 795, 726, 805, 722, 775, 761, 758, 774, 756, 763, 740, 753, 743, 793] payload_rows_per_dst=[9106, 9264, 9272, 8972, 9272, 9278, 9490, 9366] dup_rows_per_dst=[2916, 2938, 3078, 2786, 2930, 3198, 3230, 3208] combine_rows_per_src=[9106, 9264, 9272, 8972, 9272, 9278, 9490, 9366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265822208 dispatch_meta_bytes=343312 combine_payload_bytes=265822208 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266165520 ingress_bytes=300425216 link_bytes=300425216 token_bytes=531987728 weight_movement_bytes=34603008 total_nvlink_bytes=566590736 weight_to_token_ratio=0.065045 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130475902e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=342936 combine_payload_bytes=266199040 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266541976 ingress_bytes=404611072 link_bytes=404611072 token_bytes=532741016 weight_movement_bytes=138412032 total_nvlink_bytes=671153048 weight_to_token_ratio=0.259811 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522518890e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74514 payload_rows_remote=65140 dup_rows=23790 dup_rows_remote=20666 dedup_ratio=0.242004 router_tokens_per_expert=[762, 773, 751, 752, 767, 732, 763, 742, 739, 788, 764, 788, 765, 769, 767, 773, 788, 785, 789, 736, 751, 736, 747, 801, 760, 774, 781, 805, 767, 760, 741, 786, 719, 815, 741, 717, 813, 742, 744, 759, 779, 801, 798, 715, 817, 793, 749, 797, 717, 761, 784, 774, 779, 787, 727, 780, 778, 783, 744, 774, 746, 792, 830, 795] payload_rows_per_dst=[9160, 9278, 9322, 9362, 9244, 9374, 9330, 9444] dup_rows_per_dst=[2924, 3028, 2944, 2986, 2856, 3124, 2888, 3040] combine_rows_per_src=[9160, 9278, 9322, 9362, 9244, 9374, 9330, 9444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74098 payload_rows_remote=64788 dup_rows=24206 dup_rows_remote=21214 dedup_ratio=0.246236 router_tokens_per_expert=[768, 732, 758, 752, 742, 765, 735, 772, 724, 758, 748, 807, 741, 741, 791, 765, 801, 800, 781, 763, 762, 766, 743, 750, 774, 789, 767, 770, 815, 722, 738, 782, 744, 777, 792, 768, 742, 780, 719, 824, 795, 774, 799, 794, 793, 791, 767, 801, 770, 779, 801, 743, 766, 772, 764, 731, 737, 814, 810, 746, 768, 745, 776, 748] payload_rows_per_dst=[9086, 9160, 9322, 9310, 9252, 9470, 9188, 9310] dup_rows_per_dst=[2962, 2924, 3052, 3006, 2948, 3258, 3064, 2992] combine_rows_per_src=[9086, 9160, 9322, 9310, 9252, 9470, 9188, 9310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=64998 dup_rows=23966 dup_rows_remote=20984 dedup_ratio=0.243795 router_tokens_per_expert=[737, 768, 778, 751, 775, 796, 768, 769, 720, 791, 796, 773, 728, 790, 801, 775, 780, 748, 782, 749, 770, 761, 812, 759, 783, 742, 722, 840, 768, 795, 794, 763, 775, 756, 786, 719, 731, 773, 804, 738, 770, 738, 719, 782, 798, 788, 744, 763, 808, 758, 791, 781, 754, 758, 740, 795, 766, 758, 746, 757, 768, 766, 785, 753] payload_rows_per_dst=[9304, 9242, 9340, 9388, 9240, 9300, 9388, 9136] dup_rows_per_dst=[2980, 3106, 2982, 3026, 2924, 2904, 2982, 3062] combine_rows_per_src=[9304, 9242, 9340, 9388, 9240, 9300, 9388, 9136] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=64894 dup_rows=24046 dup_rows_remote=21002 dedup_ratio=0.244609 router_tokens_per_expert=[784, 791, 772, 765, 773, 727, 754, 742, 790, 793, 745, 759, 790, 812, 755, 772, 744, 804, 776, 766, 750, 755, 790, 799, 783, 765, 774, 791, 780, 763, 787, 761, 781, 724, 776, 799, 797, 772, 765, 731, 759, 755, 730, 750, 744, 755, 777, 761, 759, 763, 762, 760, 817, 822, 785, 746, 813, 803, 728, 712, 742, 730, 767, 755] payload_rows_per_dst=[9240, 9390, 9300, 9364, 9284, 9138, 9340, 9202] dup_rows_per_dst=[2976, 3042, 3068, 3044, 3006, 2924, 3088, 2898] combine_rows_per_src=[9240, 9390, 9300, 9364, 9284, 9138, 9340, 9202] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74250 payload_rows_remote=65024 dup_rows=24054 dup_rows_remote=20984 dedup_ratio=0.244690 router_tokens_per_expert=[750, 772, 785, 754, 755, 785, 772, 750, 720, 723, 762, 738, 776, 782, 758, 819, 734, 729, 760, 798, 762, 836, 737, 778, 757, 741, 759, 756, 760, 776, 760, 767, 787, 800, 785, 749, 819, 767, 769, 783, 775, 797, 770, 713, 792, 753, 792, 799, 806, 752, 717, 775, 804, 797, 726, 771, 713, 783, 726, 816, 762, 780, 783, 780] payload_rows_per_dst=[9306, 9262, 9318, 9162, 9400, 9326, 9226, 9250] dup_rows_per_dst=[2940, 2894, 2950, 2990, 3118, 3056, 3070, 3036] combine_rows_per_src=[9306, 9262, 9318, 9162, 9400, 9326, 9226, 9250] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265805824 dispatch_meta_bytes=343584 combine_payload_bytes=265805824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266149408 ingress_bytes=300408832 link_bytes=300408832 token_bytes=531955232 weight_movement_bytes=34603008 total_nvlink_bytes=566558240 weight_to_token_ratio=0.065049 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130414250e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74350 payload_rows_remote=65080 dup_rows=23954 dup_rows_remote=20902 dedup_ratio=0.243673 router_tokens_per_expert=[764, 784, 724, 773, 793, 791, 746, 735, 799, 805, 780, 781, 779, 706, 777, 789, 752, 805, 740, 766, 778, 762, 806, 752, 743, 825, 758, 762, 796, 799, 740, 739, 789, 736, 778, 784, 764, 757, 787, 731, 773, 764, 768, 811, 750, 749, 752, 786, 796, 795, 757, 803, 744, 754, 776, 762, 746, 727, 731, 743, 736, 769, 808, 777] payload_rows_per_dst=[9216, 9344, 9270, 9310, 9352, 9232, 9358, 9268] dup_rows_per_dst=[3004, 3088, 3052, 3014, 2900, 3074, 3016, 2806] combine_rows_per_src=[9216, 9344, 9270, 9310, 9352, 9232, 9358, 9268] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266813440 dispatch_meta_bytes=343224 combine_payload_bytes=266813440 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267156664 ingress_bytes=336019456 link_bytes=336019456 token_bytes=533970104 weight_movement_bytes=69206016 total_nvlink_bytes=603176120 weight_to_token_ratio=0.129607 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264414161e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265371648 dispatch_meta_bytes=344008 combine_payload_bytes=265371648 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265715656 ingress_bytes=334577664 link_bytes=334577664 token_bytes=531087304 weight_movement_bytes=69206016 total_nvlink_bytes=600293320 weight_to_token_ratio=0.130310 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258988814e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=264822784 dispatch_meta_bytes=342920 combine_payload_bytes=264822784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265165704 ingress_bytes=299425792 link_bytes=299425792 token_bytes=529988488 weight_movement_bytes=34603008 total_nvlink_bytes=564591496 weight_to_token_ratio=0.065290 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126715150e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=64732 dup_rows=23966 dup_rows_remote=20658 dedup_ratio=0.243795 router_tokens_per_expert=[794, 805, 753, 801, 789, 806, 780, 767, 765, 757, 745, 764, 747, 801, 745, 716, 768, 764, 776, 765, 750, 751, 770, 769, 692, 766, 839, 787, 797, 736, 765, 780, 753, 768, 758, 772, 750, 820, 749, 766, 755, 759, 710, 827, 763, 786, 734, 792, 742, 750, 779, 796, 747, 780, 761, 761, 764, 798, 746, 744, 787, 766, 773, 786] payload_rows_per_dst=[9606, 9180, 9336, 9286, 9334, 9102, 9314, 9180] dup_rows_per_dst=[3308, 2900, 2890, 3038, 2938, 2826, 3082, 2984] combine_rows_per_src=[9606, 9180, 9336, 9286, 9334, 9102, 9314, 9180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74222 payload_rows_remote=64874 dup_rows=24082 dup_rows_remote=21154 dedup_ratio=0.244975 router_tokens_per_expert=[762, 776, 736, 757, 762, 787, 764, 784, 742, 775, 787, 781, 756, 738, 747, 767, 795, 769, 760, 743, 740, 731, 767, 795, 757, 806, 821, 734, 767, 737, 725, 776, 772, 817, 774, 781, 739, 771, 722, 762, 748, 853, 740, 786, 748, 796, 797, 755, 774, 744, 783, 800, 792, 765, 782, 773, 760, 765, 747, 758, 775, 756, 801, 772] payload_rows_per_dst=[9240, 9220, 9286, 9210, 9348, 9400, 9270, 9248] dup_rows_per_dst=[3016, 2966, 2914, 3036, 2928, 3046, 3156, 3020] combine_rows_per_src=[9240, 9220, 9286, 9210, 9348, 9400, 9270, 9248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266231808 dispatch_meta_bytes=343928 combine_payload_bytes=266231808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266575736 ingress_bytes=300834816 link_bytes=300834816 token_bytes=532807544 weight_movement_bytes=34603008 total_nvlink_bytes=567410552 weight_to_token_ratio=0.064945 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132017193e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265805824 dispatch_meta_bytes=343584 combine_payload_bytes=265805824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266149408 ingress_bytes=300408832 link_bytes=300408832 token_bytes=531955232 weight_movement_bytes=34603008 total_nvlink_bytes=566558240 weight_to_token_ratio=0.065049 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130414250e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=344032 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266682336 ingress_bytes=300941312 link_bytes=300941312 token_bytes=533020640 weight_movement_bytes=34603008 total_nvlink_bytes=567623648 weight_to_token_ratio=0.064919 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132417929e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74268 payload_rows_remote=65152 dup_rows=24036 dup_rows_remote=21176 dedup_ratio=0.244507 router_tokens_per_expert=[719, 783, 733, 788, 781, 821, 735, 701, 741, 709, 794, 720, 781, 734, 765, 744, 761, 752, 786, 759, 772, 754, 816, 792, 753, 754, 786, 782, 824, 769, 759, 788, 793, 771, 821, 807, 768, 775, 798, 781, 737, 727, 704, 837, 811, 762, 783, 779, 751, 735, 756, 764, 749, 742, 780, 823, 784, 789, 699, 745, 779, 735, 806, 805] payload_rows_per_dst=[9250, 9116, 9226, 9338, 9482, 9344, 9182, 9330] dup_rows_per_dst=[2872, 2860, 3158, 3092, 3146, 2936, 3018, 2954] combine_rows_per_src=[9250, 9116, 9226, 9338, 9482, 9344, 9182, 9330] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74080 payload_rows_remote=64922 dup_rows=24224 dup_rows_remote=21164 dedup_ratio=0.246419 router_tokens_per_expert=[775, 785, 807, 778, 787, 819, 752, 753, 768, 771, 778, 778, 779, 767, 783, 725, 748, 785, 794, 791, 768, 723, 761, 774, 760, 772, 724, 775, 784, 785, 792, 742, 761, 786, 763, 767, 748, 757, 756, 771, 795, 728, 803, 781, 773, 804, 751, 768, 758, 758, 741, 752, 759, 777, 756, 772, 777, 702, 760, 804, 777, 737, 740, 787] payload_rows_per_dst=[9314, 9184, 9238, 9288, 9158, 9424, 9264, 9210] dup_rows_per_dst=[3198, 3114, 3050, 2980, 3060, 2982, 2882, 2958] combine_rows_per_src=[9314, 9184, 9238, 9288, 9158, 9424, 9264, 9210] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266567680 dispatch_meta_bytes=343928 combine_payload_bytes=266567680 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266911608 ingress_bytes=301170688 link_bytes=301170688 token_bytes=533479288 weight_movement_bytes=34603008 total_nvlink_bytes=568082296 weight_to_token_ratio=0.064863 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133281053e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=64732 dup_rows=23966 dup_rows_remote=20658 dedup_ratio=0.243795 router_tokens_per_expert=[794, 805, 753, 801, 789, 806, 780, 767, 765, 757, 745, 764, 747, 801, 745, 716, 768, 764, 776, 765, 750, 751, 770, 769, 692, 766, 839, 787, 797, 736, 765, 780, 753, 768, 758, 772, 750, 820, 749, 766, 755, 759, 710, 827, 763, 786, 734, 792, 742, 750, 779, 796, 747, 780, 761, 761, 764, 798, 746, 744, 787, 766, 773, 786] payload_rows_per_dst=[9606, 9180, 9336, 9286, 9334, 9102, 9314, 9180] dup_rows_per_dst=[3308, 2900, 2890, 3038, 2938, 2826, 3082, 2984] combine_rows_per_src=[9606, 9180, 9336, 9286, 9334, 9102, 9314, 9180] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74160 payload_rows_remote=64884 dup_rows=24144 dup_rows_remote=21100 dedup_ratio=0.245605 router_tokens_per_expert=[801, 773, 785, 755, 764, 782, 772, 764, 791, 772, 763, 810, 800, 742, 771, 726, 785, 736, 810, 773, 763, 812, 770, 750, 759, 792, 748, 811, 740, 775, 785, 752, 802, 751, 755, 748, 826, 728, 735, 818, 759, 792, 759, 796, 781, 744, 754, 775, 756, 768, 735, 782, 745, 758, 775, 752, 785, 743, 786, 731, 782, 722, 776, 701] payload_rows_per_dst=[9318, 9308, 9322, 9380, 9226, 9276, 9224, 9106] dup_rows_per_dst=[3074, 3042, 3076, 2944, 3100, 3044, 2918, 2946] combine_rows_per_src=[9318, 9308, 9322, 9380, 9226, 9276, 9224, 9106] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265142272 dispatch_meta_bytes=341560 combine_payload_bytes=265142272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265483832 ingress_bytes=299745280 link_bytes=299745280 token_bytes=530626104 weight_movement_bytes=34603008 total_nvlink_bytes=565229112 weight_to_token_ratio=0.065212 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127917357e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265723904 dispatch_meta_bytes=344112 combine_payload_bytes=265723904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266068016 ingress_bytes=300326912 link_bytes=300326912 token_bytes=531791920 weight_movement_bytes=34603008 total_nvlink_bytes=566394928 weight_to_token_ratio=0.065069 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130105992e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266862592 dispatch_meta_bytes=345312 combine_payload_bytes=266862592 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267207904 ingress_bytes=301465600 link_bytes=301465600 token_bytes=534070496 weight_movement_bytes=34603008 total_nvlink_bytes=568673504 weight_to_token_ratio=0.064791 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134390783e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265920512 dispatch_meta_bytes=344344 combine_payload_bytes=265920512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266264856 ingress_bytes=300523520 link_bytes=300523520 token_bytes=532185368 weight_movement_bytes=34603008 total_nvlink_bytes=566788376 weight_to_token_ratio=0.065021 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130845812e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74160 payload_rows_remote=64884 dup_rows=24144 dup_rows_remote=21100 dedup_ratio=0.245605 router_tokens_per_expert=[801, 773, 785, 755, 764, 782, 772, 764, 791, 772, 763, 810, 800, 742, 771, 726, 785, 736, 810, 773, 763, 812, 770, 750, 759, 792, 748, 811, 740, 775, 785, 752, 802, 751, 755, 748, 826, 728, 735, 818, 759, 792, 759, 796, 781, 744, 754, 775, 756, 768, 735, 782, 745, 758, 775, 752, 785, 743, 786, 731, 782, 722, 776, 701] payload_rows_per_dst=[9318, 9308, 9322, 9380, 9226, 9276, 9224, 9106] dup_rows_per_dst=[3074, 3042, 3076, 2944, 3100, 3044, 2918, 2946] combine_rows_per_src=[9318, 9308, 9322, 9380, 9226, 9276, 9224, 9106] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74020 payload_rows_remote=64654 dup_rows=24284 dup_rows_remote=21076 dedup_ratio=0.247030 router_tokens_per_expert=[786, 842, 747, 786, 772, 829, 750, 792, 762, 764, 776, 821, 763, 718, 744, 753, 747, 779, 789, 776, 742, 794, 810, 762, 784, 780, 772, 732, 762, 760, 740, 806, 722, 756, 743, 801, 759, 748, 742, 744, 791, 747, 749, 821, 757, 820, 783, 746, 748, 771, 795, 726, 805, 722, 775, 761, 758, 774, 756, 763, 740, 753, 743, 793] payload_rows_per_dst=[9106, 9264, 9272, 8972, 9272, 9278, 9490, 9366] dup_rows_per_dst=[2916, 2938, 3078, 2786, 2930, 3198, 3230, 3208] combine_rows_per_src=[9106, 9264, 9272, 8972, 9272, 9278, 9490, 9366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74534 payload_rows_remote=65250 dup_rows=23770 dup_rows_remote=20804 dedup_ratio=0.241801 router_tokens_per_expert=[771, 787, 769, 752, 786, 790, 774, 795, 774, 766, 757, 750, 844, 762, 813, 746, 767, 774, 732, 740, 746, 758, 774, 746, 793, 796, 730, 785, 761, 849, 762, 779, 768, 763, 766, 787, 741, 759, 754, 754, 790, 766, 737, 763, 751, 780, 738, 751, 747, 784, 766, 736, 770, 759, 785, 778, 744, 759, 775, 756, 835, 783, 749, 730] payload_rows_per_dst=[9368, 9384, 9138, 9472, 9302, 9226, 9284, 9360] dup_rows_per_dst=[3080, 3040, 2936, 3038, 2882, 2926, 2966, 2902] combine_rows_per_src=[9368, 9384, 9138, 9472, 9302, 9226, 9284, 9360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74350 payload_rows_remote=65080 dup_rows=23954 dup_rows_remote=20902 dedup_ratio=0.243673 router_tokens_per_expert=[764, 784, 724, 773, 793, 791, 746, 735, 799, 805, 780, 781, 779, 706, 777, 789, 752, 805, 740, 766, 778, 762, 806, 752, 743, 825, 758, 762, 796, 799, 740, 739, 789, 736, 778, 784, 764, 757, 787, 731, 773, 764, 768, 811, 750, 749, 752, 786, 796, 795, 757, 803, 744, 754, 776, 762, 746, 727, 731, 743, 736, 769, 808, 777] payload_rows_per_dst=[9216, 9344, 9270, 9310, 9352, 9232, 9358, 9268] dup_rows_per_dst=[3004, 3088, 3052, 3014, 2900, 3074, 3016, 2806] combine_rows_per_src=[9216, 9344, 9270, 9310, 9352, 9232, 9358, 9268] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74262 payload_rows_remote=65024 dup_rows=24042 dup_rows_remote=21056 dedup_ratio=0.244568 router_tokens_per_expert=[737, 754, 767, 764, 759, 768, 762, 780, 788, 716, 735, 776, 758, 797, 722, 759, 757, 770, 696, 758, 805, 788, 805, 751, 761, 747, 781, 747, 789, 755, 771, 761, 759, 752, 757, 750, 769, 802, 708, 793, 742, 788, 743, 819, 782, 759, 775, 788, 807, 783, 769, 753, 793, 764, 758, 737, 774, 785, 805, 814, 766, 784, 793, 797] payload_rows_per_dst=[9222, 9162, 9352, 9238, 9184, 9360, 9288, 9456] dup_rows_per_dst=[2960, 2940, 2908, 2986, 2996, 3032, 3040, 3180] combine_rows_per_src=[9222, 9162, 9352, 9238, 9184, 9360, 9288, 9456] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265142272 dispatch_meta_bytes=341560 combine_payload_bytes=265142272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265483832 ingress_bytes=299745280 link_bytes=299745280 token_bytes=530626104 weight_movement_bytes=34603008 total_nvlink_bytes=565229112 weight_to_token_ratio=0.065212 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127917357e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265764864 dispatch_meta_bytes=343936 combine_payload_bytes=265764864 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266108800 ingress_bytes=334970880 link_bytes=334970880 token_bytes=531873664 weight_movement_bytes=69206016 total_nvlink_bytes=601079680 weight_to_token_ratio=0.130117 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260468454e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74162 payload_rows_remote=64846 dup_rows=24142 dup_rows_remote=21084 dedup_ratio=0.245585 router_tokens_per_expert=[739, 800, 792, 768, 747, 767, 760, 787, 775, 734, 798, 778, 752, 768, 786, 758, 800, 806, 761, 787, 754, 755, 776, 761, 765, 734, 735, 722, 784, 729, 788, 785, 787, 784, 718, 759, 749, 774, 795, 764, 792, 764, 768, 774, 738, 750, 771, 743, 794, 814, 796, 783, 766, 763, 776, 826, 755, 764, 751, 754, 764, 764, 726, 775] payload_rows_per_dst=[9246, 9320, 9314, 9052, 9166, 9202, 9546, 9316] dup_rows_per_dst=[3074, 2978, 3086, 2764, 3032, 2998, 3152, 3058] combine_rows_per_src=[9246, 9320, 9314, 9052, 9166, 9202, 9546, 9316] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74534 payload_rows_remote=65250 dup_rows=23770 dup_rows_remote=20804 dedup_ratio=0.241801 router_tokens_per_expert=[771, 787, 769, 752, 786, 790, 774, 795, 774, 766, 757, 750, 844, 762, 813, 746, 767, 774, 732, 740, 746, 758, 774, 746, 793, 796, 730, 785, 761, 849, 762, 779, 768, 763, 766, 787, 741, 759, 754, 754, 790, 766, 737, 763, 751, 780, 738, 751, 747, 784, 766, 736, 770, 759, 785, 778, 744, 759, 775, 756, 835, 783, 749, 730] payload_rows_per_dst=[9368, 9384, 9138, 9472, 9302, 9226, 9284, 9360] dup_rows_per_dst=[3080, 3040, 2936, 3038, 2882, 2926, 2966, 2902] combine_rows_per_src=[9368, 9384, 9138, 9472, 9302, 9226, 9284, 9360] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74472 payload_rows_remote=65038 dup_rows=23832 dup_rows_remote=20744 dedup_ratio=0.242432 router_tokens_per_expert=[810, 719, 769, 808, 757, 712, 758, 796, 744, 736, 792, 774, 764, 756, 748, 747, 773, 785, 796, 775, 802, 762, 834, 734, 797, 769, 756, 801, 781, 759, 776, 762, 807, 807, 735, 757, 750, 786, 763, 766, 846, 755, 727, 760, 753, 747, 743, 749, 718, 785, 777, 807, 774, 742, 749, 773, 740, 756, 749, 854, 757, 788, 751, 729] payload_rows_per_dst=[9292, 9238, 9434, 9468, 9336, 9228, 9226, 9250] dup_rows_per_dst=[2966, 2884, 3088, 2934, 3006, 2932, 3024, 2998] combine_rows_per_src=[9292, 9238, 9434, 9468, 9336, 9228, 9226, 9250] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=344320 combine_payload_bytes=266338304 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266682624 ingress_bytes=335544320 link_bytes=335544320 token_bytes=533020928 weight_movement_bytes=69206016 total_nvlink_bytes=602226944 weight_to_token_ratio=0.129837 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262626263e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=264822784 dispatch_meta_bytes=342920 combine_payload_bytes=264822784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265165704 ingress_bytes=299425792 link_bytes=299425792 token_bytes=529988488 weight_movement_bytes=34603008 total_nvlink_bytes=564591496 weight_to_token_ratio=0.065290 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126715150e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265609216 dispatch_meta_bytes=343720 combine_payload_bytes=265609216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265952936 ingress_bytes=300212224 link_bytes=300212224 token_bytes=531562152 weight_movement_bytes=34603008 total_nvlink_bytes=566165160 weight_to_token_ratio=0.065097 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129674430e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74170 payload_rows_remote=64986 dup_rows=24134 dup_rows_remote=21096 dedup_ratio=0.245504 router_tokens_per_expert=[770, 752, 749, 788, 768, 724, 780, 781, 741, 770, 732, 777, 713, 748, 776, 797, 791, 747, 784, 813, 765, 839, 797, 726, 739, 767, 749, 750, 771, 739, 800, 787, 738, 777, 770, 743, 768, 768, 755, 792, 818, 805, 787, 815, 791, 783, 733, 748, 776, 774, 749, 735, 793, 753, 750, 716, 778, 751, 808, 754, 777, 790, 776, 751] payload_rows_per_dst=[9220, 9154, 9494, 9256, 9184, 9388, 9236, 9238] dup_rows_per_dst=[3004, 2954, 3030, 2948, 3038, 3172, 2856, 3132] combine_rows_per_src=[9220, 9154, 9494, 9256, 9184, 9388, 9236, 9238] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=65038 dup_rows=24098 dup_rows_remote=21204 dedup_ratio=0.245138 router_tokens_per_expert=[786, 763, 815, 819, 797, 737, 747, 791, 743, 753, 767, 731, 757, 800, 756, 724, 796, 771, 727, 810, 721, 751, 775, 752, 772, 801, 797, 770, 757, 800, 750, 800, 834, 722, 776, 755, 750, 767, 764, 778, 729, 733, 747, 767, 757, 774, 785, 767, 761, 796, 802, 765, 804, 784, 785, 731, 766, 715, 770, 748, 785, 763, 780, 756] payload_rows_per_dst=[9440, 9168, 9162, 9378, 9308, 9274, 9336, 9140] dup_rows_per_dst=[3070, 2894, 3044, 3116, 2984, 2844, 3120, 3026] combine_rows_per_src=[9440, 9168, 9162, 9378, 9308, 9274, 9336, 9140] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74138 payload_rows_remote=64920 dup_rows=24166 dup_rows_remote=21022 dedup_ratio=0.245829 router_tokens_per_expert=[742, 788, 762, 771, 780, 762, 821, 744, 739, 730, 750, 798, 797, 777, 778, 795, 755, 748, 771, 805, 781, 723, 729, 744, 780, 735, 758, 749, 767, 779, 741, 799, 761, 764, 769, 777, 803, 757, 786, 764, 787, 754, 742, 758, 807, 773, 733, 767, 818, 735, 772, 736, 773, 771, 789, 797, 765, 785, 744, 799, 776, 752, 772, 768] payload_rows_per_dst=[9328, 9200, 9258, 9214, 9218, 9322, 9356, 9242] dup_rows_per_dst=[3012, 3128, 2854, 3002, 3144, 2920, 3026, 3080] combine_rows_per_src=[9328, 9200, 9258, 9214, 9218, 9322, 9356, 9242] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=267264000 dispatch_meta_bytes=344216 combine_payload_bytes=267264000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267608216 ingress_bytes=301867008 link_bytes=301867008 token_bytes=534872216 weight_movement_bytes=34603008 total_nvlink_bytes=569475224 weight_to_token_ratio=0.064694 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135901249e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=343128 combine_payload_bytes=266395648 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266738776 ingress_bytes=335601664 link_bytes=335601664 token_bytes=533134424 weight_movement_bytes=69206016 total_nvlink_bytes=602340440 weight_to_token_ratio=0.129810 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262842043e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=65046 dup_rows=23948 dup_rows_remote=20852 dedup_ratio=0.243612 router_tokens_per_expert=[780, 788, 803, 753, 735, 780, 779, 758, 757, 725, 764, 801, 783, 730, 698, 802, 725, 741, 755, 785, 794, 755, 714, 845, 770, 727, 777, 767, 816, 807, 802, 794, 804, 807, 742, 785, 756, 751, 774, 738, 756, 769, 766, 780, 753, 799, 810, 753, 719, 760, 790, 742, 761, 749, 770, 781, 791, 781, 719, 769, 788, 742, 801, 736] payload_rows_per_dst=[9310, 9522, 9252, 9332, 9294, 9106, 9302, 9238] dup_rows_per_dst=[3096, 2998, 2976, 3134, 3020, 2866, 2842, 3016] combine_rows_per_src=[9310, 9522, 9252, 9332, 9294, 9106, 9302, 9238] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74692 payload_rows_remote=65534 dup_rows=23612 dup_rows_remote=20880 dedup_ratio=0.240194 router_tokens_per_expert=[804, 749, 783, 783, 764, 751, 785, 748, 742, 785, 762, 825, 772, 745, 799, 788, 787, 755, 773, 794, 767, 778, 748, 748, 774, 750, 722, 791, 776, 761, 770, 781, 770, 798, 776, 741, 776, 723, 744, 801, 718, 778, 724, 746, 760, 738, 719, 762, 745, 828, 759, 745, 785, 775, 817, 791, 742, 786, 747, 785, 780, 798, 754, 781] payload_rows_per_dst=[9362, 9414, 9376, 9306, 9186, 9158, 9488, 9402] dup_rows_per_dst=[2972, 3022, 2924, 2944, 3072, 2732, 3002, 2944] combine_rows_per_src=[9362, 9414, 9376, 9306, 9186, 9158, 9488, 9402] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266567680 dispatch_meta_bytes=343928 combine_payload_bytes=266567680 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266911608 ingress_bytes=301170688 link_bytes=301170688 token_bytes=533479288 weight_movement_bytes=34603008 total_nvlink_bytes=568082296 weight_to_token_ratio=0.064863 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133281053e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=344320 combine_payload_bytes=266338304 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266682624 ingress_bytes=335544320 link_bytes=335544320 token_bytes=533020928 weight_movement_bytes=69206016 total_nvlink_bytes=602226944 weight_to_token_ratio=0.129837 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262626263e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=65038 dup_rows=24098 dup_rows_remote=21204 dedup_ratio=0.245138 router_tokens_per_expert=[786, 763, 815, 819, 797, 737, 747, 791, 743, 753, 767, 731, 757, 800, 756, 724, 796, 771, 727, 810, 721, 751, 775, 752, 772, 801, 797, 770, 757, 800, 750, 800, 834, 722, 776, 755, 750, 767, 764, 778, 729, 733, 747, 767, 757, 774, 785, 767, 761, 796, 802, 765, 804, 784, 785, 731, 766, 715, 770, 748, 785, 763, 780, 756] payload_rows_per_dst=[9440, 9168, 9162, 9378, 9308, 9274, 9336, 9140] dup_rows_per_dst=[3070, 2894, 3044, 3116, 2984, 2844, 3120, 3026] combine_rows_per_src=[9440, 9168, 9162, 9378, 9308, 9274, 9336, 9140] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266182656 dispatch_meta_bytes=344328 combine_payload_bytes=266182656 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266526984 ingress_bytes=369991680 link_bytes=369991680 token_bytes=532709640 weight_movement_bytes=103809024 total_nvlink_bytes=636518664 weight_to_token_ratio=0.194870 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392248905e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=344968 combine_payload_bytes=266395648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266740616 ingress_bytes=300998656 link_bytes=300998656 token_bytes=533136264 weight_movement_bytes=34603008 total_nvlink_bytes=567739272 weight_to_token_ratio=0.064905 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132633710e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74486 payload_rows_remote=65174 dup_rows=23818 dup_rows_remote=20854 dedup_ratio=0.242289 router_tokens_per_expert=[761, 733, 795, 738, 784, 733, 772, 841, 756, 756, 761, 784, 750, 761, 721, 778, 789, 767, 793, 754, 748, 740, 813, 773, 760, 766, 782, 752, 755, 775, 815, 733, 786, 747, 772, 812, 725, 825, 739, 786, 766, 788, 747, 751, 760, 748, 755, 774, 779, 793, 812, 780, 793, 756, 766, 757, 761, 735, 738, 774, 802, 748, 748, 790] payload_rows_per_dst=[9270, 9302, 9368, 9312, 9352, 9216, 9414, 9252] dup_rows_per_dst=[3044, 2832, 2986, 2964, 3032, 2962, 3058, 2940] combine_rows_per_src=[9270, 9302, 9368, 9312, 9352, 9216, 9414, 9252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265912320 dispatch_meta_bytes=343768 combine_payload_bytes=265912320 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266256088 ingress_bytes=335118336 link_bytes=335118336 token_bytes=532168408 weight_movement_bytes=69206016 total_nvlink_bytes=601374424 weight_to_token_ratio=0.130045 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261023319e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74162 payload_rows_remote=64846 dup_rows=24142 dup_rows_remote=21084 dedup_ratio=0.245585 router_tokens_per_expert=[739, 800, 792, 768, 747, 767, 760, 787, 775, 734, 798, 778, 752, 768, 786, 758, 800, 806, 761, 787, 754, 755, 776, 761, 765, 734, 735, 722, 784, 729, 788, 785, 787, 784, 718, 759, 749, 774, 795, 764, 792, 764, 768, 774, 738, 750, 771, 743, 794, 814, 796, 783, 766, 763, 776, 826, 755, 764, 751, 754, 764, 764, 726, 775] payload_rows_per_dst=[9246, 9320, 9314, 9052, 9166, 9202, 9546, 9316] dup_rows_per_dst=[3074, 2978, 3086, 2764, 3032, 2998, 3152, 3058] combine_rows_per_src=[9246, 9320, 9314, 9052, 9166, 9202, 9546, 9316] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=343592 combine_payload_bytes=266428416 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266772008 ingress_bytes=370237440 link_bytes=370237440 token_bytes=533200424 weight_movement_bytes=103809024 total_nvlink_bytes=637009448 weight_to_token_ratio=0.194690 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393173680e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=268427264 dispatch_meta_bytes=345656 combine_payload_bytes=268427264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268772920 ingress_bytes=303030272 link_bytes=303030272 token_bytes=537200184 weight_movement_bytes=34603008 total_nvlink_bytes=571803192 weight_to_token_ratio=0.064414 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.140278518e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73948 payload_rows_remote=64602 dup_rows=24356 dup_rows_remote=21034 dedup_ratio=0.247762 router_tokens_per_expert=[735, 745, 782, 811, 762, 820, 794, 805, 784, 734, 744, 733, 806, 771, 754, 704, 797, 768, 770, 782, 725, 765, 728, 763, 771, 810, 740, 715, 753, 726, 762, 741, 755, 747, 767, 783, 777, 721, 811, 765, 717, 791, 785, 860, 767, 782, 776, 767, 762, 778, 753, 848, 737, 768, 781, 769, 775, 757, 739, 771, 774, 786, 773, 810] payload_rows_per_dst=[9034, 9096, 9144, 9336, 9272, 9366, 9354, 9346] dup_rows_per_dst=[2924, 2964, 2842, 3128, 2980, 3126, 3070, 3322] combine_rows_per_src=[9034, 9096, 9144, 9336, 9272, 9366, 9354, 9346] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74692 payload_rows_remote=65534 dup_rows=23612 dup_rows_remote=20880 dedup_ratio=0.240194 router_tokens_per_expert=[804, 749, 783, 783, 764, 751, 785, 748, 742, 785, 762, 825, 772, 745, 799, 788, 787, 755, 773, 794, 767, 778, 748, 748, 774, 750, 722, 791, 776, 761, 770, 781, 770, 798, 776, 741, 776, 723, 744, 801, 718, 778, 724, 746, 760, 738, 719, 762, 745, 828, 759, 745, 785, 775, 817, 791, 742, 786, 747, 785, 780, 798, 754, 781] payload_rows_per_dst=[9362, 9414, 9376, 9306, 9186, 9158, 9488, 9402] dup_rows_per_dst=[2972, 3022, 2924, 2944, 3072, 2732, 3002, 2944] combine_rows_per_src=[9362, 9414, 9376, 9306, 9186, 9158, 9488, 9402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=344968 combine_payload_bytes=266395648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266740616 ingress_bytes=300998656 link_bytes=300998656 token_bytes=533136264 weight_movement_bytes=34603008 total_nvlink_bytes=567739272 weight_to_token_ratio=0.064905 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132633710e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265609216 dispatch_meta_bytes=343720 combine_payload_bytes=265609216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265952936 ingress_bytes=300212224 link_bytes=300212224 token_bytes=531562152 weight_movement_bytes=34603008 total_nvlink_bytes=566165160 weight_to_token_ratio=0.065097 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129674430e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74422 payload_rows_remote=65104 dup_rows=23882 dup_rows_remote=20976 dedup_ratio=0.242940 router_tokens_per_expert=[743, 721, 772, 773, 732, 789, 738, 779, 787, 783, 771, 757, 787, 806, 785, 804, 769, 748, 751, 767, 811, 777, 761, 720, 781, 808, 782, 783, 803, 774, 755, 774, 736, 771, 747, 796, 735, 795, 755, 733, 771, 780, 763, 741, 760, 767, 772, 783, 779, 765, 752, 764, 780, 769, 721, 782, 757, 789, 748, 759, 747, 776, 817, 751] payload_rows_per_dst=[9146, 9452, 9210, 9454, 9250, 9360, 9318, 9232] dup_rows_per_dst=[2948, 3108, 2998, 3066, 2886, 2914, 2906, 3056] combine_rows_per_src=[9146, 9452, 9210, 9454, 9250, 9360, 9318, 9232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266952704 dispatch_meta_bytes=344112 combine_payload_bytes=266952704 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267296816 ingress_bytes=370761728 link_bytes=370761728 token_bytes=534249520 weight_movement_bytes=103809024 total_nvlink_bytes=638058544 weight_to_token_ratio=0.194308 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395146534e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74574 payload_rows_remote=65164 dup_rows=23730 dup_rows_remote=20822 dedup_ratio=0.241394 router_tokens_per_expert=[800, 799, 766, 751, 780, 742, 728, 798, 771, 794, 757, 767, 723, 751, 773, 727, 777, 757, 759, 730, 744, 779, 796, 817, 792, 799, 799, 747, 746, 745, 769, 735, 754, 760, 722, 772, 771, 785, 710, 829, 756, 749, 831, 740, 776, 765, 818, 787, 772, 775, 802, 729, 787, 763, 787, 783, 780, 793, 731, 768, 696, 775, 770, 798] payload_rows_per_dst=[9368, 9262, 9410, 9334, 9232, 9382, 9390, 9196] dup_rows_per_dst=[2960, 2864, 2908, 2930, 2974, 3062, 3006, 3026] combine_rows_per_src=[9368, 9262, 9410, 9334, 9232, 9382, 9390, 9196] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74472 payload_rows_remote=65038 dup_rows=23832 dup_rows_remote=20744 dedup_ratio=0.242432 router_tokens_per_expert=[810, 719, 769, 808, 757, 712, 758, 796, 744, 736, 792, 774, 764, 756, 748, 747, 773, 785, 796, 775, 802, 762, 834, 734, 797, 769, 756, 801, 781, 759, 776, 762, 807, 807, 735, 757, 750, 786, 763, 766, 846, 755, 727, 760, 753, 747, 743, 749, 718, 785, 777, 807, 774, 742, 749, 773, 740, 756, 749, 854, 757, 788, 751, 729] payload_rows_per_dst=[9292, 9238, 9434, 9468, 9336, 9228, 9226, 9250] dup_rows_per_dst=[2966, 2884, 3088, 2934, 3006, 2932, 3024, 2998] combine_rows_per_src=[9292, 9238, 9434, 9468, 9336, 9228, 9226, 9250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=268427264 dispatch_meta_bytes=345656 combine_payload_bytes=268427264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268772920 ingress_bytes=303030272 link_bytes=303030272 token_bytes=537200184 weight_movement_bytes=34603008 total_nvlink_bytes=571803192 weight_to_token_ratio=0.064414 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.140278518e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74486 payload_rows_remote=65174 dup_rows=23818 dup_rows_remote=20854 dedup_ratio=0.242289 router_tokens_per_expert=[761, 733, 795, 738, 784, 733, 772, 841, 756, 756, 761, 784, 750, 761, 721, 778, 789, 767, 793, 754, 748, 740, 813, 773, 760, 766, 782, 752, 755, 775, 815, 733, 786, 747, 772, 812, 725, 825, 739, 786, 766, 788, 747, 751, 760, 748, 755, 774, 779, 793, 812, 780, 793, 756, 766, 757, 761, 735, 738, 774, 802, 748, 748, 790] payload_rows_per_dst=[9270, 9302, 9368, 9312, 9352, 9216, 9414, 9252] dup_rows_per_dst=[3044, 2832, 2986, 2964, 3032, 2962, 3058, 2940] combine_rows_per_src=[9270, 9302, 9368, 9312, 9352, 9216, 9414, 9252] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=264609792 dispatch_meta_bytes=342544 combine_payload_bytes=264609792 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264952336 ingress_bytes=299212800 link_bytes=299212800 token_bytes=529562128 weight_movement_bytes=34603008 total_nvlink_bytes=564165136 weight_to_token_ratio=0.065343 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125913678e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74578 payload_rows_remote=65412 dup_rows=23726 dup_rows_remote=20744 dedup_ratio=0.241353 router_tokens_per_expert=[807, 816, 746, 739, 752, 751, 806, 776, 750, 784, 808, 815, 776, 799, 778, 761, 764, 737, 743, 828, 790, 720, 817, 752, 804, 759, 785, 751, 803, 752, 732, 764, 741, 759, 735, 775, 768, 752, 762, 782, 701, 768, 788, 752, 817, 775, 750, 782, 751, 783, 754, 745, 749, 821, 754, 754, 725, 747, 771, 745, 786, 769, 786, 740] payload_rows_per_dst=[9316, 9458, 9316, 9404, 9166, 9342, 9302, 9274] dup_rows_per_dst=[3070, 3084, 2986, 2896, 2982, 2924, 2920, 2864] combine_rows_per_src=[9316, 9458, 9316, 9404, 9166, 9342, 9302, 9274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74080 payload_rows_remote=64922 dup_rows=24224 dup_rows_remote=21164 dedup_ratio=0.246419 router_tokens_per_expert=[775, 785, 807, 778, 787, 819, 752, 753, 768, 771, 778, 778, 779, 767, 783, 725, 748, 785, 794, 791, 768, 723, 761, 774, 760, 772, 724, 775, 784, 785, 792, 742, 761, 786, 763, 767, 748, 757, 756, 771, 795, 728, 803, 781, 773, 804, 751, 768, 758, 758, 741, 752, 759, 777, 756, 772, 777, 702, 760, 804, 777, 737, 740, 787] payload_rows_per_dst=[9314, 9184, 9238, 9288, 9158, 9424, 9264, 9210] dup_rows_per_dst=[3198, 3114, 3050, 2980, 3060, 2982, 2882, 2958] combine_rows_per_src=[9314, 9184, 9238, 9288, 9158, 9424, 9264, 9210] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266665984 dispatch_meta_bytes=344320 combine_payload_bytes=266665984 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267010304 ingress_bytes=335872000 link_bytes=335872000 token_bytes=533676288 weight_movement_bytes=69206016 total_nvlink_bytes=602882304 weight_to_token_ratio=0.129678 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263859296e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74574 payload_rows_remote=65180 dup_rows=23730 dup_rows_remote=20874 dedup_ratio=0.241394 router_tokens_per_expert=[783, 759, 749, 733, 802, 739, 780, 778, 738, 772, 779, 760, 774, 759, 759, 784, 778, 705, 780, 739, 739, 780, 787, 746, 772, 747, 746, 800, 761, 820, 753, 778, 789, 768, 795, 776, 767, 773, 758, 814, 795, 709, 766, 769, 765, 813, 812, 724, 794, 757, 771, 782, 753, 737, 741, 785, 788, 764, 754, 747, 783, 785, 786, 753] payload_rows_per_dst=[9394, 9488, 9016, 9476, 9444, 9582, 8880, 9294] dup_rows_per_dst=[2856, 3096, 2774, 2866, 3028, 3342, 2742, 3026] combine_rows_per_src=[9394, 9488, 9016, 9476, 9444, 9582, 8880, 9294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=343128 combine_payload_bytes=266395648 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266738776 ingress_bytes=335601664 link_bytes=335601664 token_bytes=533134424 weight_movement_bytes=69206016 total_nvlink_bytes=602340440 weight_to_token_ratio=0.129810 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262842043e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74212 payload_rows_remote=64854 dup_rows=24092 dup_rows_remote=21208 dedup_ratio=0.245076 router_tokens_per_expert=[748, 747, 743, 783, 783, 747, 798, 737, 706, 768, 778, 794, 827, 777, 740, 731, 779, 781, 758, 809, 746, 813, 771, 790, 736, 781, 756, 790, 717, 770, 761, 738, 773, 798, 773, 757, 767, 773, 749, 769, 774, 781, 746, 819, 739, 769, 729, 762, 795, 791, 741, 772, 771, 761, 735, 793, 720, 834, 722, 821, 764, 751, 784, 816] payload_rows_per_dst=[9168, 9358, 9286, 9230, 9294, 9160, 9302, 9414] dup_rows_per_dst=[3004, 2884, 3208, 2868, 3024, 3078, 3016, 3010] combine_rows_per_src=[9168, 9358, 9286, 9230, 9294, 9160, 9302, 9414] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266911744 dispatch_meta_bytes=343944 combine_payload_bytes=266911744 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267255688 ingress_bytes=336117760 link_bytes=336117760 token_bytes=534167432 weight_movement_bytes=69206016 total_nvlink_bytes=603373448 weight_to_token_ratio=0.129559 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264784071e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74290 payload_rows_remote=64998 dup_rows=24014 dup_rows_remote=20916 dedup_ratio=0.244283 router_tokens_per_expert=[756, 797, 732, 769, 747, 807, 796, 800, 744, 792, 768, 739, 749, 741, 772, 751, 745, 758, 753, 796, 742, 771, 788, 763, 776, 750, 770, 766, 760, 749, 804, 814, 770, 738, 763, 748, 747, 800, 737, 818, 781, 772, 825, 734, 804, 768, 775, 736, 818, 786, 731, 754, 780, 771, 796, 801, 764, 727, 792, 711, 795, 733, 764, 748] payload_rows_per_dst=[9368, 9190, 9304, 9324, 9232, 9292, 9466, 9114] dup_rows_per_dst=[3040, 2922, 2928, 3054, 3010, 3098, 3008, 2954] combine_rows_per_src=[9368, 9190, 9304, 9324, 9232, 9292, 9466, 9114] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266952704 dispatch_meta_bytes=344112 combine_payload_bytes=266952704 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267296816 ingress_bytes=370761728 link_bytes=370761728 token_bytes=534249520 weight_movement_bytes=103809024 total_nvlink_bytes=638058544 weight_to_token_ratio=0.194308 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395146534e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73948 payload_rows_remote=64602 dup_rows=24356 dup_rows_remote=21034 dedup_ratio=0.247762 router_tokens_per_expert=[735, 745, 782, 811, 762, 820, 794, 805, 784, 734, 744, 733, 806, 771, 754, 704, 797, 768, 770, 782, 725, 765, 728, 763, 771, 810, 740, 715, 753, 726, 762, 741, 755, 747, 767, 783, 777, 721, 811, 765, 717, 791, 785, 860, 767, 782, 776, 767, 762, 778, 753, 848, 737, 768, 781, 769, 775, 757, 739, 771, 774, 786, 773, 810] payload_rows_per_dst=[9034, 9096, 9144, 9336, 9272, 9366, 9354, 9346] dup_rows_per_dst=[2924, 2964, 2842, 3128, 2980, 3126, 3070, 3322] combine_rows_per_src=[9034, 9096, 9144, 9336, 9272, 9366, 9354, 9346] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265920512 dispatch_meta_bytes=344344 combine_payload_bytes=265920512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266264856 ingress_bytes=300523520 link_bytes=300523520 token_bytes=532185368 weight_movement_bytes=34603008 total_nvlink_bytes=566788376 weight_to_token_ratio=0.065021 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130845812e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=267927552 dispatch_meta_bytes=344624 combine_payload_bytes=267927552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268272176 ingress_bytes=302530560 link_bytes=302530560 token_bytes=536199728 weight_movement_bytes=34603008 total_nvlink_bytes=570802736 weight_to_token_ratio=0.064534 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138398142e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266977280 dispatch_meta_bytes=344216 combine_payload_bytes=266977280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267321496 ingress_bytes=301580288 link_bytes=301580288 token_bytes=534298776 weight_movement_bytes=34603008 total_nvlink_bytes=568901784 weight_to_token_ratio=0.064763 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134822345e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74290 payload_rows_remote=64998 dup_rows=24014 dup_rows_remote=20916 dedup_ratio=0.244283 router_tokens_per_expert=[756, 797, 732, 769, 747, 807, 796, 800, 744, 792, 768, 739, 749, 741, 772, 751, 745, 758, 753, 796, 742, 771, 788, 763, 776, 750, 770, 766, 760, 749, 804, 814, 770, 738, 763, 748, 747, 800, 737, 818, 781, 772, 825, 734, 804, 768, 775, 736, 818, 786, 731, 754, 780, 771, 796, 801, 764, 727, 792, 711, 795, 733, 764, 748] payload_rows_per_dst=[9368, 9190, 9304, 9324, 9232, 9292, 9466, 9114] dup_rows_per_dst=[3040, 2922, 2928, 3054, 3010, 3098, 3008, 2954] combine_rows_per_src=[9368, 9190, 9304, 9324, 9232, 9292, 9466, 9114] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265641984 dispatch_meta_bytes=344248 combine_payload_bytes=265641984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265986232 ingress_bytes=300244992 link_bytes=300244992 token_bytes=531628216 weight_movement_bytes=34603008 total_nvlink_bytes=566231224 weight_to_token_ratio=0.065089 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129797733e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=264609792 dispatch_meta_bytes=342544 combine_payload_bytes=264609792 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264952336 ingress_bytes=299212800 link_bytes=299212800 token_bytes=529562128 weight_movement_bytes=34603008 total_nvlink_bytes=564165136 weight_to_token_ratio=0.065343 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125913678e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74386 payload_rows_remote=65200 dup_rows=23918 dup_rows_remote=20896 dedup_ratio=0.243306 router_tokens_per_expert=[739, 759, 747, 776, 752, 830, 764, 794, 794, 772, 761, 772, 760, 795, 758, 748, 773, 803, 801, 766, 757, 782, 750, 802, 767, 747, 778, 736, 747, 782, 808, 764, 773, 747, 778, 726, 787, 777, 751, 771, 771, 807, 755, 745, 760, 765, 799, 782, 767, 762, 754, 777, 777, 770, 729, 768, 762, 773, 745, 752, 761, 741, 747, 789] payload_rows_per_dst=[9326, 9282, 9418, 9310, 9338, 9282, 9186, 9244] dup_rows_per_dst=[2996, 3038, 3050, 2948, 2882, 3086, 3022, 2896] combine_rows_per_src=[9326, 9282, 9418, 9310, 9338, 9282, 9186, 9244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74574 payload_rows_remote=65164 dup_rows=23730 dup_rows_remote=20822 dedup_ratio=0.241394 router_tokens_per_expert=[800, 799, 766, 751, 780, 742, 728, 798, 771, 794, 757, 767, 723, 751, 773, 727, 777, 757, 759, 730, 744, 779, 796, 817, 792, 799, 799, 747, 746, 745, 769, 735, 754, 760, 722, 772, 771, 785, 710, 829, 756, 749, 831, 740, 776, 765, 818, 787, 772, 775, 802, 729, 787, 763, 787, 783, 780, 793, 731, 768, 696, 775, 770, 798] payload_rows_per_dst=[9368, 9262, 9410, 9334, 9232, 9382, 9390, 9196] dup_rows_per_dst=[2960, 2864, 2908, 2930, 2974, 3062, 3006, 3026] combine_rows_per_src=[9368, 9262, 9410, 9334, 9232, 9382, 9390, 9196] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74386 payload_rows_remote=65096 dup_rows=23918 dup_rows_remote=20994 dedup_ratio=0.243306 router_tokens_per_expert=[797, 792, 770, 803, 760, 795, 767, 766, 765, 770, 767, 739, 799, 776, 727, 747, 725, 744, 766, 795, 729, 732, 803, 759, 735, 793, 766, 779, 766, 765, 766, 737, 799, 794, 778, 796, 784, 757, 837, 752, 717, 756, 781, 754, 716, 724, 788, 751, 786, 763, 742, 759, 788, 765, 755, 772, 728, 821, 806, 799, 769, 780, 762, 773] payload_rows_per_dst=[9450, 9202, 9160, 9290, 9452, 9220, 9270, 9342] dup_rows_per_dst=[3050, 2978, 2946, 2924, 3142, 2754, 2990, 3134] combine_rows_per_src=[9450, 9202, 9160, 9290, 9452, 9220, 9270, 9342] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74278 payload_rows_remote=64896 dup_rows=24026 dup_rows_remote=21030 dedup_ratio=0.244405 router_tokens_per_expert=[792, 802, 740, 778, 748, 776, 762, 730, 740, 806, 763, 787, 787, 768, 747, 792, 774, 785, 779, 798, 768, 781, 758, 753, 751, 719, 725, 773, 727, 776, 746, 820, 740, 811, 761, 759, 757, 726, 838, 740, 781, 767, 751, 765, 762, 762, 757, 745, 776, 725, 809, 747, 833, 790, 738, 774, 763, 794, 764, 731, 801, 737, 793, 804] payload_rows_per_dst=[9302, 9218, 9466, 9116, 9266, 9172, 9356, 9382] dup_rows_per_dst=[2954, 2874, 3168, 2958, 2994, 3054, 3028, 2996] combine_rows_per_src=[9302, 9218, 9466, 9116, 9266, 9172, 9356, 9382] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266231808 dispatch_meta_bytes=343656 combine_payload_bytes=266231808 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266575464 ingress_bytes=335437824 link_bytes=335437824 token_bytes=532807272 weight_movement_bytes=69206016 total_nvlink_bytes=602013288 weight_to_token_ratio=0.129889 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262225527e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74170 payload_rows_remote=64986 dup_rows=24134 dup_rows_remote=21096 dedup_ratio=0.245504 router_tokens_per_expert=[770, 752, 749, 788, 768, 724, 780, 781, 741, 770, 732, 777, 713, 748, 776, 797, 791, 747, 784, 813, 765, 839, 797, 726, 739, 767, 749, 750, 771, 739, 800, 787, 738, 777, 770, 743, 768, 768, 755, 792, 818, 805, 787, 815, 791, 783, 733, 748, 776, 774, 749, 735, 793, 753, 750, 716, 778, 751, 808, 754, 777, 790, 776, 751] payload_rows_per_dst=[9220, 9154, 9494, 9256, 9184, 9388, 9236, 9238] dup_rows_per_dst=[3004, 2954, 3030, 2948, 3038, 3172, 2856, 3132] combine_rows_per_src=[9220, 9154, 9494, 9256, 9184, 9388, 9236, 9238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=267059200 dispatch_meta_bytes=344384 combine_payload_bytes=267059200 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267403584 ingress_bytes=336265216 link_bytes=336265216 token_bytes=534462784 weight_movement_bytes=69206016 total_nvlink_bytes=603668800 weight_to_token_ratio=0.129487 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265338936e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74386 payload_rows_remote=65200 dup_rows=23918 dup_rows_remote=20896 dedup_ratio=0.243306 router_tokens_per_expert=[739, 759, 747, 776, 752, 830, 764, 794, 794, 772, 761, 772, 760, 795, 758, 748, 773, 803, 801, 766, 757, 782, 750, 802, 767, 747, 778, 736, 747, 782, 808, 764, 773, 747, 778, 726, 787, 777, 751, 771, 771, 807, 755, 745, 760, 765, 799, 782, 767, 762, 754, 777, 777, 770, 729, 768, 762, 773, 745, 752, 761, 741, 747, 789] payload_rows_per_dst=[9326, 9282, 9418, 9310, 9338, 9282, 9186, 9244] dup_rows_per_dst=[2996, 3038, 3050, 2948, 2882, 3086, 3022, 2896] combine_rows_per_src=[9326, 9282, 9418, 9310, 9338, 9282, 9186, 9244] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74316 payload_rows_remote=65130 dup_rows=23988 dup_rows_remote=21058 dedup_ratio=0.244019 router_tokens_per_expert=[785, 753, 761, 776, 759, 783, 752, 789, 740, 825, 795, 820, 780, 761, 781, 794, 751, 674, 782, 803, 746, 768, 774, 760, 805, 741, 745, 722, 733, 801, 769, 776, 789, 781, 784, 775, 741, 782, 776, 809, 777, 781, 746, 777, 737, 760, 750, 794, 743, 747, 784, 770, 767, 732, 736, 752, 778, 782, 788, 773, 765, 796, 722, 754] payload_rows_per_dst=[9336, 9460, 9186, 9154, 9340, 9354, 9186, 9300] dup_rows_per_dst=[2980, 3132, 2930, 3030, 3134, 2890, 2876, 3016] combine_rows_per_src=[9336, 9460, 9186, 9154, 9340, 9354, 9186, 9300] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266633216 dispatch_meta_bytes=344360 combine_payload_bytes=266633216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266977576 ingress_bytes=301236224 link_bytes=301236224 token_bytes=533610792 weight_movement_bytes=34603008 total_nvlink_bytes=568213800 weight_to_token_ratio=0.064847 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133527659e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265814016 dispatch_meta_bytes=343704 combine_payload_bytes=265814016 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266157720 ingress_bytes=369623040 link_bytes=369623040 token_bytes=531971736 weight_movement_bytes=103809024 total_nvlink_bytes=635780760 weight_to_token_ratio=0.195140 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390861742e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74188 payload_rows_remote=64940 dup_rows=24116 dup_rows_remote=21120 dedup_ratio=0.245321 router_tokens_per_expert=[735, 733, 756, 772, 817, 773, 779, 773, 750, 759, 781, 778, 792, 765, 737, 748, 741, 734, 787, 764, 791, 767, 771, 793, 723, 822, 780, 778, 732, 793, 797, 825, 797, 754, 754, 713, 732, 762, 813, 797, 729, 780, 794, 749, 767, 816, 748, 717, 768, 774, 786, 797, 715, 763, 741, 805, 789, 712, 772, 746, 755, 748, 823, 790] payload_rows_per_dst=[9272, 9344, 9264, 9332, 9248, 9138, 9274, 9316] dup_rows_per_dst=[3004, 2876, 3032, 3168, 2996, 3062, 3024, 2954] combine_rows_per_src=[9272, 9344, 9264, 9332, 9248, 9138, 9274, 9316] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=64996 dup_rows=24010 dup_rows_remote=21078 dedup_ratio=0.244242 router_tokens_per_expert=[783, 786, 757, 747, 765, 735, 778, 764, 785, 760, 762, 775, 758, 770, 785, 774, 752, 770, 760, 810, 766, 760, 770, 811, 769, 764, 779, 774, 723, 836, 738, 781, 789, 744, 773, 687, 792, 747, 808, 737, 754, 802, 777, 780, 768, 758, 824, 795, 725, 757, 776, 773, 767, 776, 738, 774, 757, 781, 730, 722, 788, 755, 790, 761] payload_rows_per_dst=[9298, 9500, 9344, 9440, 9230, 9158, 9140, 9184] dup_rows_per_dst=[2932, 3222, 3054, 2888, 3094, 2974, 2862, 2984] combine_rows_per_src=[9298, 9500, 9344, 9440, 9230, 9158, 9140, 9184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266911744 dispatch_meta_bytes=343944 combine_payload_bytes=266911744 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267255688 ingress_bytes=336117760 link_bytes=336117760 token_bytes=534167432 weight_movement_bytes=69206016 total_nvlink_bytes=603373448 weight_to_token_ratio=0.129559 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264784071e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74550 payload_rows_remote=65414 dup_rows=23754 dup_rows_remote=20830 dedup_ratio=0.241638 router_tokens_per_expert=[739, 783, 773, 779, 766, 788, 720, 790, 784, 732, 767, 718, 777, 741, 752, 759, 763, 747, 804, 821, 757, 722, 797, 728, 765, 794, 753, 778, 724, 774, 757, 765, 751, 736, 799, 724, 769, 788, 764, 784, 767, 768, 747, 793, 763, 803, 777, 824, 791, 776, 730, 755, 818, 761, 788, 768, 817, 742, 742, 782, 743, 741, 822, 802] payload_rows_per_dst=[9292, 9136, 9428, 9278, 9272, 9396, 9380, 9368] dup_rows_per_dst=[2984, 2924, 2850, 2942, 2958, 3088, 2994, 3014] combine_rows_per_src=[9292, 9136, 9428, 9278, 9272, 9396, 9380, 9368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=267059200 dispatch_meta_bytes=344384 combine_payload_bytes=267059200 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267403584 ingress_bytes=336265216 link_bytes=336265216 token_bytes=534462784 weight_movement_bytes=69206016 total_nvlink_bytes=603668800 weight_to_token_ratio=0.129487 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265338936e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266772480 dispatch_meta_bytes=344752 combine_payload_bytes=266772480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267117232 ingress_bytes=301375488 link_bytes=301375488 token_bytes=533889712 weight_movement_bytes=34603008 total_nvlink_bytes=568492720 weight_to_token_ratio=0.064813 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134051699e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74022 payload_rows_remote=64720 dup_rows=24282 dup_rows_remote=21260 dedup_ratio=0.247009 router_tokens_per_expert=[805, 734, 768, 750, 770, 751, 759, 810, 752, 793, 761, 794, 766, 728, 743, 789, 790, 773, 784, 778, 777, 754, 807, 733, 717, 783, 747, 717, 749, 800, 768, 768, 775, 726, 798, 797, 768, 762, 810, 771, 774, 796, 770, 746, 730, 800, 791, 755, 746, 787, 755, 748, 731, 756, 756, 804, 769, 748, 743, 727, 810, 822, 786, 777] payload_rows_per_dst=[9316, 9244, 9264, 9158, 9246, 9302, 9180, 9312] dup_rows_per_dst=[2978, 3008, 3128, 2940, 3168, 3022, 2986, 3052] combine_rows_per_src=[9316, 9244, 9264, 9158, 9246, 9302, 9180, 9312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266633216 dispatch_meta_bytes=344360 combine_payload_bytes=266633216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266977576 ingress_bytes=301236224 link_bytes=301236224 token_bytes=533610792 weight_movement_bytes=34603008 total_nvlink_bytes=568213800 weight_to_token_ratio=0.064847 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133527659e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74278 payload_rows_remote=64896 dup_rows=24026 dup_rows_remote=21030 dedup_ratio=0.244405 router_tokens_per_expert=[792, 802, 740, 778, 748, 776, 762, 730, 740, 806, 763, 787, 787, 768, 747, 792, 774, 785, 779, 798, 768, 781, 758, 753, 751, 719, 725, 773, 727, 776, 746, 820, 740, 811, 761, 759, 757, 726, 838, 740, 781, 767, 751, 765, 762, 762, 757, 745, 776, 725, 809, 747, 833, 790, 738, 774, 763, 794, 764, 731, 801, 737, 793, 804] payload_rows_per_dst=[9302, 9218, 9466, 9116, 9266, 9172, 9356, 9382] dup_rows_per_dst=[2954, 2874, 3168, 2958, 2994, 3054, 3028, 2996] combine_rows_per_src=[9302, 9218, 9466, 9116, 9266, 9172, 9356, 9382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266182656 dispatch_meta_bytes=344328 combine_payload_bytes=266182656 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266526984 ingress_bytes=369991680 link_bytes=369991680 token_bytes=532709640 weight_movement_bytes=103809024 total_nvlink_bytes=636518664 weight_to_token_ratio=0.194870 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392248905e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=344240 combine_payload_bytes=265994240 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266338480 ingress_bytes=369803264 link_bytes=369803264 token_bytes=532332720 weight_movement_bytes=103809024 total_nvlink_bytes=636141744 weight_to_token_ratio=0.195008 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391539911e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=344296 combine_payload_bytes=266223616 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266567912 ingress_bytes=404635648 link_bytes=404635648 token_bytes=532791528 weight_movement_bytes=138412032 total_nvlink_bytes=671203560 weight_to_token_ratio=0.259786 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522611368e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267935744 dispatch_meta_bytes=344976 combine_payload_bytes=267935744 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=268280720 ingress_bytes=371744768 link_bytes=371744768 token_bytes=536216464 weight_movement_bytes=103809024 total_nvlink_bytes=640025488 weight_to_token_ratio=0.193595 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.398845634e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74428 payload_rows_remote=65040 dup_rows=23876 dup_rows_remote=20880 dedup_ratio=0.242879 router_tokens_per_expert=[795, 756, 748, 763, 770, 798, 758, 781, 711, 783, 756, 742, 772, 778, 793, 743, 787, 771, 794, 731, 792, 762, 715, 791, 767, 767, 785, 854, 753, 714, 788, 764, 731, 796, 807, 820, 802, 791, 733, 764, 743, 791, 764, 762, 734, 793, 725, 774, 770, 794, 811, 767, 753, 793, 715, 758, 754, 780, 785, 751, 754, 756, 758, 741] payload_rows_per_dst=[9264, 9270, 9300, 9388, 9400, 9234, 9310, 9262] dup_rows_per_dst=[3074, 2886, 2986, 2996, 3088, 2938, 3012, 2896] combine_rows_per_src=[9264, 9270, 9300, 9388, 9400, 9234, 9310, 9262] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74092 payload_rows_remote=64704 dup_rows=24212 dup_rows_remote=21142 dedup_ratio=0.246297 router_tokens_per_expert=[803, 758, 762, 753, 767, 755, 749, 793, 766, 774, 778, 762, 704, 781, 768, 821, 780, 762, 853, 735, 719, 802, 802, 784, 767, 831, 755, 765, 751, 746, 785, 760, 755, 749, 773, 719, 737, 735, 750, 761, 719, 718, 794, 768, 795, 741, 798, 757, 807, 789, 696, 782, 792, 745, 748, 805, 769, 763, 785, 764, 781, 782, 801, 783] payload_rows_per_dst=[9238, 9242, 8904, 9226, 9590, 9142, 9362, 9388] dup_rows_per_dst=[3042, 3066, 2704, 3094, 3234, 3038, 2964, 3070] combine_rows_per_src=[9238, 9242, 8904, 9226, 9590, 9142, 9362, 9388] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265093120 dispatch_meta_bytes=343920 combine_payload_bytes=265093120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265437040 ingress_bytes=299696128 link_bytes=299696128 token_bytes=530530160 weight_movement_bytes=34603008 total_nvlink_bytes=565133168 weight_to_token_ratio=0.065223 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127732402e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266403840 dispatch_meta_bytes=343680 combine_payload_bytes=266403840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266747520 ingress_bytes=301006848 link_bytes=301006848 token_bytes=533151360 weight_movement_bytes=34603008 total_nvlink_bytes=567754368 weight_to_token_ratio=0.064903 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132664536e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=64926 dup_rows=23982 dup_rows_remote=20916 dedup_ratio=0.243958 router_tokens_per_expert=[747, 785, 766, 778, 754, 812, 802, 768, 755, 773, 773, 773, 788, 778, 763, 762, 764, 785, 800, 803, 777, 776, 713, 813, 773, 727, 710, 740, 766, 813, 787, 786, 765, 751, 751, 748, 780, 747, 750, 778, 735, 770, 826, 755, 738, 753, 749, 707, 706, 788, 813, 755, 785, 785, 790, 781, 759, 773, 804, 752, 797, 736, 740, 775] payload_rows_per_dst=[9420, 9306, 9396, 9266, 9140, 9162, 9354, 9278] dup_rows_per_dst=[3004, 3024, 3066, 2938, 3000, 2904, 3052, 2994] combine_rows_per_src=[9420, 9306, 9396, 9266, 9140, 9162, 9354, 9278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74022 payload_rows_remote=64720 dup_rows=24282 dup_rows_remote=21260 dedup_ratio=0.247009 router_tokens_per_expert=[805, 734, 768, 750, 770, 751, 759, 810, 752, 793, 761, 794, 766, 728, 743, 789, 790, 773, 784, 778, 777, 754, 807, 733, 717, 783, 747, 717, 749, 800, 768, 768, 775, 726, 798, 797, 768, 762, 810, 771, 774, 796, 770, 746, 730, 800, 791, 755, 746, 787, 755, 748, 731, 756, 756, 804, 769, 748, 743, 727, 810, 822, 786, 777] payload_rows_per_dst=[9316, 9244, 9264, 9158, 9246, 9302, 9180, 9312] dup_rows_per_dst=[2978, 3008, 3128, 2940, 3168, 3022, 2986, 3052] combine_rows_per_src=[9316, 9244, 9264, 9158, 9246, 9302, 9180, 9312] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74268 payload_rows_remote=65016 dup_rows=24036 dup_rows_remote=21112 dedup_ratio=0.244507 router_tokens_per_expert=[730, 735, 768, 760, 762, 746, 763, 726, 741, 754, 795, 782, 800, 779, 785, 761, 751, 800, 714, 780, 739, 758, 748, 768, 766, 803, 752, 773, 745, 780, 751, 790, 767, 762, 749, 786, 751, 801, 783, 773, 770, 784, 755, 750, 801, 775, 790, 760, 778, 729, 791, 773, 760, 752, 755, 750, 813, 807, 734, 787, 771, 782, 782, 826] payload_rows_per_dst=[9106, 9314, 9202, 9292, 9322, 9380, 9252, 9400] dup_rows_per_dst=[2874, 3080, 2914, 3028, 3022, 2990, 2924, 3204] combine_rows_per_src=[9106, 9314, 9202, 9292, 9322, 9380, 9252, 9400] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265027584 dispatch_meta_bytes=343384 combine_payload_bytes=265027584 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265370968 ingress_bytes=334233600 link_bytes=334233600 token_bytes=530398552 weight_movement_bytes=69206016 total_nvlink_bytes=599604568 weight_to_token_ratio=0.130479 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257694129e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265814016 dispatch_meta_bytes=343704 combine_payload_bytes=265814016 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266157720 ingress_bytes=369623040 link_bytes=369623040 token_bytes=531971736 weight_movement_bytes=103809024 total_nvlink_bytes=635780760 weight_to_token_ratio=0.195140 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390861742e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74316 payload_rows_remote=65130 dup_rows=23988 dup_rows_remote=21058 dedup_ratio=0.244019 router_tokens_per_expert=[785, 753, 761, 776, 759, 783, 752, 789, 740, 825, 795, 820, 780, 761, 781, 794, 751, 674, 782, 803, 746, 768, 774, 760, 805, 741, 745, 722, 733, 801, 769, 776, 789, 781, 784, 775, 741, 782, 776, 809, 777, 781, 746, 777, 737, 760, 750, 794, 743, 747, 784, 770, 767, 732, 736, 752, 778, 782, 788, 773, 765, 796, 722, 754] payload_rows_per_dst=[9336, 9460, 9186, 9154, 9340, 9354, 9186, 9300] dup_rows_per_dst=[2980, 3132, 2930, 3030, 3134, 2890, 2876, 3016] combine_rows_per_src=[9336, 9460, 9186, 9154, 9340, 9354, 9186, 9300] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74304 payload_rows_remote=65032 dup_rows=24000 dup_rows_remote=21046 dedup_ratio=0.244141 router_tokens_per_expert=[785, 790, 734, 783, 748, 777, 728, 755, 754, 801, 762, 752, 773, 810, 729, 740, 768, 789, 817, 783, 748, 789, 776, 812, 772, 745, 724, 747, 817, 780, 758, 825, 777, 789, 759, 778, 766, 712, 771, 761, 738, 763, 763, 806, 763, 837, 799, 731, 728, 778, 774, 761, 795, 739, 769, 728, 809, 762, 793, 726, 755, 748, 764, 739] payload_rows_per_dst=[9286, 9294, 9482, 9286, 9272, 9300, 9224, 9160] dup_rows_per_dst=[2914, 2948, 3082, 3050, 2954, 3100, 2920, 3032] combine_rows_per_src=[9286, 9294, 9482, 9286, 9272, 9300, 9224, 9160] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74216 payload_rows_remote=64728 dup_rows=24088 dup_rows_remote=21008 dedup_ratio=0.245036 router_tokens_per_expert=[784, 768, 768, 769, 758, 774, 758, 768, 816, 745, 750, 780, 783, 789, 790, 806, 755, 764, 755, 770, 775, 760, 769, 745, 727, 768, 747, 749, 724, 789, 742, 746, 811, 782, 788, 765, 804, 790, 778, 814, 805, 725, 717, 717, 780, 823, 798, 777, 762, 777, 784, 733, 825, 723, 770, 757, 767, 769, 728, 788, 753, 726, 771, 754] payload_rows_per_dst=[9488, 9450, 9300, 9320, 9036, 9258, 9360, 9004] dup_rows_per_dst=[3080, 3088, 2886, 3204, 2814, 3026, 3162, 2828] combine_rows_per_src=[9488, 9450, 9300, 9320, 9036, 9258, 9360, 9004] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65096 dup_rows=23908 dup_rows_remote=20994 dedup_ratio=0.243205 router_tokens_per_expert=[786, 775, 764, 754, 785, 741, 789, 774, 779, 729, 719, 791, 795, 781, 745, 768, 764, 755, 770, 744, 708, 807, 779, 803, 750, 785, 780, 832, 769, 791, 780, 779, 741, 787, 728, 769, 737, 752, 783, 763, 789, 738, 802, 734, 778, 758, 782, 834, 756, 805, 750, 790, 774, 760, 755, 785, 777, 758, 724, 758, 780, 780, 717, 737] payload_rows_per_dst=[9290, 9300, 9232, 9436, 9194, 9396, 9384, 9164] dup_rows_per_dst=[3046, 2914, 3028, 3096, 2926, 3034, 2966, 2898] combine_rows_per_src=[9290, 9300, 9232, 9436, 9194, 9396, 9384, 9164] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265936896 dispatch_meta_bytes=343368 combine_payload_bytes=265936896 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266280264 ingress_bytes=335142912 link_bytes=335142912 token_bytes=532217160 weight_movement_bytes=69206016 total_nvlink_bytes=601423176 weight_to_token_ratio=0.130033 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261115797e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265093120 dispatch_meta_bytes=343920 combine_payload_bytes=265093120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265437040 ingress_bytes=299696128 link_bytes=299696128 token_bytes=530530160 weight_movement_bytes=34603008 total_nvlink_bytes=565133168 weight_to_token_ratio=0.065223 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127732402e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74428 payload_rows_remote=65040 dup_rows=23876 dup_rows_remote=20880 dedup_ratio=0.242879 router_tokens_per_expert=[795, 756, 748, 763, 770, 798, 758, 781, 711, 783, 756, 742, 772, 778, 793, 743, 787, 771, 794, 731, 792, 762, 715, 791, 767, 767, 785, 854, 753, 714, 788, 764, 731, 796, 807, 820, 802, 791, 733, 764, 743, 791, 764, 762, 734, 793, 725, 774, 770, 794, 811, 767, 753, 793, 715, 758, 754, 780, 785, 751, 754, 756, 758, 741] payload_rows_per_dst=[9264, 9270, 9300, 9388, 9400, 9234, 9310, 9262] dup_rows_per_dst=[3074, 2886, 2986, 2996, 3088, 2938, 3012, 2896] combine_rows_per_src=[9264, 9270, 9300, 9388, 9400, 9234, 9310, 9262] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=344512 combine_payload_bytes=266305536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266650048 ingress_bytes=300908544 link_bytes=300908544 token_bytes=532955584 weight_movement_bytes=34603008 total_nvlink_bytes=567558592 weight_to_token_ratio=0.064927 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132294626e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74578 payload_rows_remote=65412 dup_rows=23726 dup_rows_remote=20744 dedup_ratio=0.241353 router_tokens_per_expert=[807, 816, 746, 739, 752, 751, 806, 776, 750, 784, 808, 815, 776, 799, 778, 761, 764, 737, 743, 828, 790, 720, 817, 752, 804, 759, 785, 751, 803, 752, 732, 764, 741, 759, 735, 775, 768, 752, 762, 782, 701, 768, 788, 752, 817, 775, 750, 782, 751, 783, 754, 745, 749, 821, 754, 754, 725, 747, 771, 745, 786, 769, 786, 740] payload_rows_per_dst=[9316, 9458, 9316, 9404, 9166, 9342, 9302, 9274] dup_rows_per_dst=[3070, 3084, 2986, 2896, 2982, 2924, 2920, 2864] combine_rows_per_src=[9316, 9458, 9316, 9404, 9166, 9342, 9302, 9274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266772480 dispatch_meta_bytes=344752 combine_payload_bytes=266772480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267117232 ingress_bytes=301375488 link_bytes=301375488 token_bytes=533889712 weight_movement_bytes=34603008 total_nvlink_bytes=568492720 weight_to_token_ratio=0.064813 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134051699e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65096 dup_rows=23908 dup_rows_remote=20994 dedup_ratio=0.243205 router_tokens_per_expert=[786, 775, 764, 754, 785, 741, 789, 774, 779, 729, 719, 791, 795, 781, 745, 768, 764, 755, 770, 744, 708, 807, 779, 803, 750, 785, 780, 832, 769, 791, 780, 779, 741, 787, 728, 769, 737, 752, 783, 763, 789, 738, 802, 734, 778, 758, 782, 834, 756, 805, 750, 790, 774, 760, 755, 785, 777, 758, 724, 758, 780, 780, 717, 737] payload_rows_per_dst=[9290, 9300, 9232, 9436, 9194, 9396, 9384, 9164] dup_rows_per_dst=[3046, 2914, 3028, 3096, 2926, 3034, 2966, 2898] combine_rows_per_src=[9290, 9300, 9232, 9436, 9194, 9396, 9384, 9164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74216 payload_rows_remote=64728 dup_rows=24088 dup_rows_remote=21008 dedup_ratio=0.245036 router_tokens_per_expert=[784, 768, 768, 769, 758, 774, 758, 768, 816, 745, 750, 780, 783, 789, 790, 806, 755, 764, 755, 770, 775, 760, 769, 745, 727, 768, 747, 749, 724, 789, 742, 746, 811, 782, 788, 765, 804, 790, 778, 814, 805, 725, 717, 717, 780, 823, 798, 777, 762, 777, 784, 733, 825, 723, 770, 757, 767, 769, 728, 788, 753, 726, 771, 754] payload_rows_per_dst=[9488, 9450, 9300, 9320, 9036, 9258, 9360, 9004] dup_rows_per_dst=[3080, 3088, 2886, 3204, 2814, 3026, 3162, 2828] combine_rows_per_src=[9488, 9450, 9300, 9320, 9036, 9258, 9360, 9004] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266403840 dispatch_meta_bytes=343680 combine_payload_bytes=266403840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266747520 ingress_bytes=301006848 link_bytes=301006848 token_bytes=533151360 weight_movement_bytes=34603008 total_nvlink_bytes=567754368 weight_to_token_ratio=0.064903 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132664536e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=267927552 dispatch_meta_bytes=344624 combine_payload_bytes=267927552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268272176 ingress_bytes=302530560 link_bytes=302530560 token_bytes=536199728 weight_movement_bytes=34603008 total_nvlink_bytes=570802736 weight_to_token_ratio=0.064534 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138398142e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74092 payload_rows_remote=64704 dup_rows=24212 dup_rows_remote=21142 dedup_ratio=0.246297 router_tokens_per_expert=[803, 758, 762, 753, 767, 755, 749, 793, 766, 774, 778, 762, 704, 781, 768, 821, 780, 762, 853, 735, 719, 802, 802, 784, 767, 831, 755, 765, 751, 746, 785, 760, 755, 749, 773, 719, 737, 735, 750, 761, 719, 718, 794, 768, 795, 741, 798, 757, 807, 789, 696, 782, 792, 745, 748, 805, 769, 763, 785, 764, 781, 782, 801, 783] payload_rows_per_dst=[9238, 9242, 8904, 9226, 9590, 9142, 9362, 9388] dup_rows_per_dst=[3042, 3066, 2704, 3094, 3234, 3038, 2964, 3070] combine_rows_per_src=[9238, 9242, 8904, 9226, 9590, 9142, 9362, 9388] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74468 payload_rows_remote=65110 dup_rows=23836 dup_rows_remote=20810 dedup_ratio=0.242472 router_tokens_per_expert=[766, 769, 760, 799, 730, 747, 770, 727, 754, 768, 765, 779, 738, 727, 741, 752, 788, 773, 735, 758, 767, 736, 789, 721, 780, 779, 755, 764, 755, 777, 739, 772, 764, 786, 790, 781, 792, 771, 761, 735, 830, 779, 775, 789, 738, 771, 789, 721, 722, 793, 771, 818, 797, 801, 808, 757, 757, 787, 793, 825, 792, 761, 763, 755] payload_rows_per_dst=[9238, 9268, 9238, 9280, 9344, 9358, 9366, 9376] dup_rows_per_dst=[2898, 2780, 2896, 2962, 3016, 3026, 3168, 3090] combine_rows_per_src=[9238, 9268, 9238, 9280, 9344, 9358, 9366, 9376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344312 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266715384 ingress_bytes=300974080 link_bytes=300974080 token_bytes=533086456 weight_movement_bytes=34603008 total_nvlink_bytes=567689464 weight_to_token_ratio=0.064911 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132541233e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265125888 dispatch_meta_bytes=342944 combine_payload_bytes=265125888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265468832 ingress_bytes=299728896 link_bytes=299728896 token_bytes=530594720 weight_movement_bytes=34603008 total_nvlink_bytes=565197728 weight_to_token_ratio=0.065216 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127855705e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74510 payload_rows_remote=65070 dup_rows=23794 dup_rows_remote=20888 dedup_ratio=0.242045 router_tokens_per_expert=[770, 757, 731, 781, 754, 785, 774, 760, 785, 740, 784, 762, 751, 786, 756, 749, 776, 782, 774, 774, 717, 756, 765, 763, 768, 769, 788, 810, 742, 759, 779, 758, 766, 750, 759, 782, 772, 750, 784, 729, 788, 756, 801, 740, 762, 822, 766, 774, 772, 799, 766, 770, 758, 786, 778, 777, 776, 777, 782, 765, 743, 764, 753, 780] payload_rows_per_dst=[9292, 9282, 9238, 9440, 9160, 9400, 9378, 9320] dup_rows_per_dst=[2932, 2944, 2976, 2906, 3024, 3018, 3034, 2960] combine_rows_per_src=[9292, 9282, 9238, 9440, 9160, 9400, 9378, 9320] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266633216 dispatch_meta_bytes=344360 combine_payload_bytes=266633216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266977576 ingress_bytes=301236224 link_bytes=301236224 token_bytes=533610792 weight_movement_bytes=34603008 total_nvlink_bytes=568213800 weight_to_token_ratio=0.064847 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133527659e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74402 payload_rows_remote=64622 dup_rows=23902 dup_rows_remote=20404 dedup_ratio=0.243144 router_tokens_per_expert=[793, 752, 750, 787, 744, 723, 776, 739, 780, 778, 758, 801, 745, 747, 762, 750, 777, 793, 776, 776, 705, 789, 797, 774, 737, 788, 771, 792, 739, 768, 792, 735, 815, 735, 698, 793, 795, 783, 778, 778, 756, 789, 767, 820, 754, 799, 760, 743, 742, 786, 782, 779, 796, 748, 788, 755, 786, 799, 737, 791, 729, 766, 790, 721] payload_rows_per_dst=[9260, 9210, 9366, 9284, 8778, 9406, 9318, 9780] dup_rows_per_dst=[2868, 3032, 3008, 2942, 2532, 2970, 3052, 3498] combine_rows_per_src=[9260, 9210, 9366, 9284, 8778, 9406, 9318, 9780] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266690560 dispatch_meta_bytes=343680 combine_payload_bytes=266690560 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267034240 ingress_bytes=301293568 link_bytes=301293568 token_bytes=533724800 weight_movement_bytes=34603008 total_nvlink_bytes=568327808 weight_to_token_ratio=0.064833 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133743440e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266526720 dispatch_meta_bytes=343832 combine_payload_bytes=266526720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266870552 ingress_bytes=335732736 link_bytes=335732736 token_bytes=533397272 weight_movement_bytes=69206016 total_nvlink_bytes=602603288 weight_to_token_ratio=0.129746 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263335257e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74520 payload_rows_remote=65104 dup_rows=23784 dup_rows_remote=20716 dedup_ratio=0.241943 router_tokens_per_expert=[800, 731, 774, 837, 798, 763, 792, 736, 740, 733, 753, 738, 718, 775, 786, 742, 767, 772, 757, 743, 805, 769, 825, 804, 713, 779, 789, 786, 769, 808, 752, 764, 797, 815, 744, 745, 771, 778, 764, 723, 797, 717, 775, 775, 796, 721, 747, 791, 795, 778, 777, 778, 749, 746, 740, 791, 768, 769, 730, 777, 768, 783, 779, 750] payload_rows_per_dst=[9440, 9206, 9416, 9244, 9298, 9272, 9366, 9278] dup_rows_per_dst=[3022, 2764, 3068, 3076, 2976, 2966, 2942, 2970] combine_rows_per_src=[9440, 9206, 9416, 9244, 9298, 9272, 9366, 9278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265125888 dispatch_meta_bytes=342944 combine_payload_bytes=265125888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265468832 ingress_bytes=299728896 link_bytes=299728896 token_bytes=530594720 weight_movement_bytes=34603008 total_nvlink_bytes=565197728 weight_to_token_ratio=0.065216 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127855705e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74440 payload_rows_remote=65050 dup_rows=23864 dup_rows_remote=20758 dedup_ratio=0.242757 router_tokens_per_expert=[771, 790, 753, 817, 747, 736, 723, 789, 767, 756, 813, 800, 780, 771, 792, 779, 785, 754, 786, 756, 824, 768, 766, 843, 753, 768, 801, 773, 709, 791, 721, 751, 765, 759, 753, 741, 733, 757, 744, 750, 792, 725, 784, 800, 770, 751, 721, 744, 797, 755, 773, 796, 811, 725, 786, 805, 739, 781, 722, 771, 763, 760, 780, 766] payload_rows_per_dst=[9190, 9480, 9448, 9196, 9220, 9220, 9390, 9296] dup_rows_per_dst=[3062, 3036, 3116, 2938, 2784, 2954, 3106, 2868] combine_rows_per_src=[9190, 9480, 9448, 9196, 9220, 9220, 9390, 9296] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=264691712 dispatch_meta_bytes=340104 combine_payload_bytes=264691712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265031816 ingress_bytes=299294720 link_bytes=299294720 token_bytes=529723528 weight_movement_bytes=34603008 total_nvlink_bytes=564326536 weight_to_token_ratio=0.065323 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126221936e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=64926 dup_rows=23982 dup_rows_remote=20916 dedup_ratio=0.243958 router_tokens_per_expert=[747, 785, 766, 778, 754, 812, 802, 768, 755, 773, 773, 773, 788, 778, 763, 762, 764, 785, 800, 803, 777, 776, 713, 813, 773, 727, 710, 740, 766, 813, 787, 786, 765, 751, 751, 748, 780, 747, 750, 778, 735, 770, 826, 755, 738, 753, 749, 707, 706, 788, 813, 755, 785, 785, 790, 781, 759, 773, 804, 752, 797, 736, 740, 775] payload_rows_per_dst=[9420, 9306, 9396, 9266, 9140, 9162, 9354, 9278] dup_rows_per_dst=[3004, 3024, 3066, 2938, 3000, 2904, 3052, 2994] combine_rows_per_src=[9420, 9306, 9396, 9266, 9140, 9162, 9354, 9278] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74532 payload_rows_remote=65164 dup_rows=23772 dup_rows_remote=20734 dedup_ratio=0.241821 router_tokens_per_expert=[788, 748, 751, 783, 735, 759, 791, 771, 727, 764, 766, 778, 782, 750, 765, 832, 777, 733, 796, 792, 793, 787, 778, 742, 749, 748, 748, 802, 762, 761, 759, 821, 790, 768, 774, 785, 778, 754, 782, 772, 772, 762, 721, 732, 776, 758, 838, 738, 760, 784, 766, 750, 791, 773, 770, 772, 776, 729, 754, 724, 778, 757, 772, 758] payload_rows_per_dst=[9318, 9394, 9382, 9298, 9368, 9310, 9242, 9220] dup_rows_per_dst=[2934, 2934, 3014, 3002, 3038, 2884, 3090, 2876] combine_rows_per_src=[9318, 9394, 9382, 9298, 9368, 9310, 9242, 9220] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=64940 dup_rows=23930 dup_rows_remote=20912 dedup_ratio=0.243429 router_tokens_per_expert=[757, 810, 765, 792, 772, 765, 771, 755, 752, 783, 783, 769, 793, 718, 780, 759, 745, 799, 755, 776, 735, 799, 740, 789, 784, 761, 795, 739, 772, 763, 813, 775, 785, 817, 766, 795, 748, 767, 770, 775, 745, 728, 726, 700, 769, 774, 745, 794, 758, 737, 739, 743, 780, 766, 743, 774, 758, 775, 737, 840, 822, 792, 779, 741] payload_rows_per_dst=[9434, 9392, 9430, 9232, 9366, 9240, 8908, 9372] dup_rows_per_dst=[3018, 3028, 3068, 2958, 3080, 3012, 2728, 3038] combine_rows_per_src=[9434, 9392, 9430, 9232, 9366, 9240, 8908, 9372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266665984 dispatch_meta_bytes=343280 combine_payload_bytes=266665984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267009264 ingress_bytes=301268992 link_bytes=301268992 token_bytes=533675248 weight_movement_bytes=34603008 total_nvlink_bytes=568278256 weight_to_token_ratio=0.064839 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133650963e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74116 payload_rows_remote=64822 dup_rows=24188 dup_rows_remote=21140 dedup_ratio=0.246053 router_tokens_per_expert=[769, 784, 745, 766, 747, 767, 793, 794, 797, 773, 802, 778, 738, 807, 723, 753, 784, 747, 746, 751, 778, 767, 762, 700, 764, 764, 716, 780, 766, 802, 749, 763, 794, 803, 785, 714, 772, 799, 790, 757, 729, 773, 758, 807, 758, 738, 784, 758, 779, 751, 797, 759, 747, 752, 810, 762, 777, 802, 773, 770, 752, 799, 732, 796] payload_rows_per_dst=[9344, 9294, 9200, 9216, 9262, 9256, 9222, 9322] dup_rows_per_dst=[2986, 3048, 2870, 2992, 3166, 2954, 3092, 3080] combine_rows_per_src=[9344, 9294, 9200, 9216, 9262, 9256, 9222, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266690560 dispatch_meta_bytes=343680 combine_payload_bytes=266690560 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267034240 ingress_bytes=301293568 link_bytes=301293568 token_bytes=533724800 weight_movement_bytes=34603008 total_nvlink_bytes=568327808 weight_to_token_ratio=0.064833 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133743440e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265027584 dispatch_meta_bytes=343384 combine_payload_bytes=265027584 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265370968 ingress_bytes=334233600 link_bytes=334233600 token_bytes=530398552 weight_movement_bytes=69206016 total_nvlink_bytes=599604568 weight_to_token_ratio=0.130479 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257694129e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266444800 dispatch_meta_bytes=343232 combine_payload_bytes=266444800 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266788032 ingress_bytes=404856832 link_bytes=404856832 token_bytes=533232832 weight_movement_bytes=138412032 total_nvlink_bytes=671644864 weight_to_token_ratio=0.259571 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523443665e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74510 payload_rows_remote=65070 dup_rows=23794 dup_rows_remote=20888 dedup_ratio=0.242045 router_tokens_per_expert=[770, 757, 731, 781, 754, 785, 774, 760, 785, 740, 784, 762, 751, 786, 756, 749, 776, 782, 774, 774, 717, 756, 765, 763, 768, 769, 788, 810, 742, 759, 779, 758, 766, 750, 759, 782, 772, 750, 784, 729, 788, 756, 801, 740, 762, 822, 766, 774, 772, 799, 766, 770, 758, 786, 778, 777, 776, 777, 782, 765, 743, 764, 753, 780] payload_rows_per_dst=[9292, 9282, 9238, 9440, 9160, 9400, 9378, 9320] dup_rows_per_dst=[2932, 2944, 2976, 2906, 3024, 3018, 3034, 2960] combine_rows_per_src=[9292, 9282, 9238, 9440, 9160, 9400, 9378, 9320] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265936896 dispatch_meta_bytes=343368 combine_payload_bytes=265936896 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266280264 ingress_bytes=335142912 link_bytes=335142912 token_bytes=532217160 weight_movement_bytes=69206016 total_nvlink_bytes=601423176 weight_to_token_ratio=0.130033 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261115797e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74188 payload_rows_remote=64940 dup_rows=24116 dup_rows_remote=21120 dedup_ratio=0.245321 router_tokens_per_expert=[735, 733, 756, 772, 817, 773, 779, 773, 750, 759, 781, 778, 792, 765, 737, 748, 741, 734, 787, 764, 791, 767, 771, 793, 723, 822, 780, 778, 732, 793, 797, 825, 797, 754, 754, 713, 732, 762, 813, 797, 729, 780, 794, 749, 767, 816, 748, 717, 768, 774, 786, 797, 715, 763, 741, 805, 789, 712, 772, 746, 755, 748, 823, 790] payload_rows_per_dst=[9272, 9344, 9264, 9332, 9248, 9138, 9274, 9316] dup_rows_per_dst=[3004, 2876, 3032, 3168, 2996, 3062, 3024, 2954] combine_rows_per_src=[9272, 9344, 9264, 9332, 9248, 9138, 9274, 9316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74116 payload_rows_remote=64822 dup_rows=24188 dup_rows_remote=21140 dedup_ratio=0.246053 router_tokens_per_expert=[769, 784, 745, 766, 747, 767, 793, 794, 797, 773, 802, 778, 738, 807, 723, 753, 784, 747, 746, 751, 778, 767, 762, 700, 764, 764, 716, 780, 766, 802, 749, 763, 794, 803, 785, 714, 772, 799, 790, 757, 729, 773, 758, 807, 758, 738, 784, 758, 779, 751, 797, 759, 747, 752, 810, 762, 777, 802, 773, 770, 752, 799, 732, 796] payload_rows_per_dst=[9344, 9294, 9200, 9216, 9262, 9256, 9222, 9322] dup_rows_per_dst=[2986, 3048, 2870, 2992, 3166, 2954, 3092, 3080] combine_rows_per_src=[9344, 9294, 9200, 9216, 9262, 9256, 9222, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=64940 dup_rows=23930 dup_rows_remote=20912 dedup_ratio=0.243429 router_tokens_per_expert=[757, 810, 765, 792, 772, 765, 771, 755, 752, 783, 783, 769, 793, 718, 780, 759, 745, 799, 755, 776, 735, 799, 740, 789, 784, 761, 795, 739, 772, 763, 813, 775, 785, 817, 766, 795, 748, 767, 770, 775, 745, 728, 726, 700, 769, 774, 745, 794, 758, 737, 739, 743, 780, 766, 743, 774, 758, 775, 737, 840, 822, 792, 779, 741] payload_rows_per_dst=[9434, 9392, 9430, 9232, 9366, 9240, 8908, 9372] dup_rows_per_dst=[3018, 3028, 3068, 2958, 3080, 3012, 2728, 3038] combine_rows_per_src=[9434, 9392, 9430, 9232, 9366, 9240, 8908, 9372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266526720 dispatch_meta_bytes=343832 combine_payload_bytes=266526720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266870552 ingress_bytes=335732736 link_bytes=335732736 token_bytes=533397272 weight_movement_bytes=69206016 total_nvlink_bytes=602603288 weight_to_token_ratio=0.129746 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263335257e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=344240 combine_payload_bytes=265994240 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266338480 ingress_bytes=369803264 link_bytes=369803264 token_bytes=532332720 weight_movement_bytes=103809024 total_nvlink_bytes=636141744 weight_to_token_ratio=0.195008 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391539911e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265510912 dispatch_meta_bytes=343848 combine_payload_bytes=265510912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265854760 ingress_bytes=300113920 link_bytes=300113920 token_bytes=531365672 weight_movement_bytes=34603008 total_nvlink_bytes=565968680 weight_to_token_ratio=0.065121 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129304520e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74452 payload_rows_remote=65098 dup_rows=23852 dup_rows_remote=20780 dedup_ratio=0.242635 router_tokens_per_expert=[744, 738, 757, 775, 776, 734, 804, 785, 772, 728, 758, 747, 792, 722, 777, 761, 740, 781, 793, 761, 726, 773, 773, 765, 751, 768, 802, 734, 734, 776, 769, 773, 764, 774, 755, 774, 790, 816, 797, 766, 788, 768, 752, 741, 815, 778, 797, 774, 765, 816, 766, 791, 777, 850, 744, 787, 762, 763, 780, 753, 757, 730, 708, 765] payload_rows_per_dst=[9234, 9150, 9252, 9268, 9518, 9354, 9490, 9186] dup_rows_per_dst=[2992, 2964, 2972, 2946, 2954, 3072, 3102, 2850] combine_rows_per_src=[9234, 9150, 9252, 9268, 9518, 9354, 9490, 9186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266911744 dispatch_meta_bytes=343592 combine_payload_bytes=266911744 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267255336 ingress_bytes=336117760 link_bytes=336117760 token_bytes=534167080 weight_movement_bytes=69206016 total_nvlink_bytes=603373096 weight_to_token_ratio=0.129559 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264784071e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74402 payload_rows_remote=64622 dup_rows=23902 dup_rows_remote=20404 dedup_ratio=0.243144 router_tokens_per_expert=[793, 752, 750, 787, 744, 723, 776, 739, 780, 778, 758, 801, 745, 747, 762, 750, 777, 793, 776, 776, 705, 789, 797, 774, 737, 788, 771, 792, 739, 768, 792, 735, 815, 735, 698, 793, 795, 783, 778, 778, 756, 789, 767, 820, 754, 799, 760, 743, 742, 786, 782, 779, 796, 748, 788, 755, 786, 799, 737, 791, 729, 766, 790, 721] payload_rows_per_dst=[9260, 9210, 9366, 9284, 8778, 9406, 9318, 9780] dup_rows_per_dst=[2868, 3032, 3008, 2942, 2532, 2970, 3052, 3498] combine_rows_per_src=[9260, 9210, 9366, 9284, 8778, 9406, 9318, 9780] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74438 payload_rows_remote=65122 dup_rows=23866 dup_rows_remote=20864 dedup_ratio=0.242778 router_tokens_per_expert=[733, 817, 796, 763, 715, 811, 748, 783, 754, 747, 802, 766, 747, 774, 769, 814, 771, 719, 752, 758, 759, 737, 811, 769, 795, 772, 798, 761, 785, 782, 796, 799, 689, 753, 742, 743, 732, 813, 776, 767, 759, 741, 823, 776, 809, 784, 743, 772, 733, 753, 816, 782, 714, 768, 767, 826, 730, 766, 732, 785, 743, 783, 780, 749] payload_rows_per_dst=[9382, 9340, 9230, 9426, 9142, 9384, 9316, 9218] dup_rows_per_dst=[2950, 3006, 2922, 3150, 2888, 3030, 3002, 2918] combine_rows_per_src=[9382, 9340, 9230, 9426, 9142, 9384, 9316, 9218] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=343408 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266337648 ingress_bytes=300597248 link_bytes=300597248 token_bytes=532331888 weight_movement_bytes=34603008 total_nvlink_bytes=566934896 weight_to_token_ratio=0.065003 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131123244e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74062 payload_rows_remote=64808 dup_rows=24242 dup_rows_remote=21376 dedup_ratio=0.246602 router_tokens_per_expert=[774, 774, 755, 806, 737, 703, 785, 732, 803, 828, 765, 803, 799, 762, 794, 750, 775, 758, 741, 741, 791, 779, 757, 779, 750, 746, 759, 756, 792, 771, 709, 777, 825, 776, 759, 787, 796, 761, 767, 762, 812, 746, 781, 757, 792, 794, 745, 716, 788, 800, 746, 774, 791, 747, 754, 778, 725, 751, 739, 766, 747, 765, 787, 767] payload_rows_per_dst=[9090, 9514, 9260, 9254, 9308, 9220, 9302, 9114] dup_rows_per_dst=[3042, 3094, 2982, 2866, 3158, 3066, 3054, 2980] combine_rows_per_src=[9090, 9514, 9260, 9254, 9308, 9220, 9302, 9114] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265510912 dispatch_meta_bytes=343848 combine_payload_bytes=265510912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265854760 ingress_bytes=300113920 link_bytes=300113920 token_bytes=531365672 weight_movement_bytes=34603008 total_nvlink_bytes=565968680 weight_to_token_ratio=0.065121 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129304520e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=65098 dup_rows=23926 dup_rows_remote=20866 dedup_ratio=0.243388 router_tokens_per_expert=[802, 791, 772, 765, 758, 740, 767, 761, 821, 776, 778, 765, 798, 765, 798, 730, 760, 726, 822, 743, 740, 777, 731, 726, 765, 778, 762, 747, 773, 786, 778, 789, 721, 774, 763, 732, 766, 785, 739, 744, 786, 750, 762, 821, 787, 781, 770, 799, 769, 804, 797, 751, 689, 745, 780, 816, 738, 775, 793, 791, 745, 767, 765, 757] payload_rows_per_dst=[9346, 9386, 9234, 9298, 9188, 9340, 9306, 9280] dup_rows_per_dst=[2966, 3076, 2816, 3058, 2774, 3172, 3004, 3060] combine_rows_per_src=[9346, 9386, 9234, 9298, 9188, 9340, 9306, 9280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74520 payload_rows_remote=65104 dup_rows=23784 dup_rows_remote=20716 dedup_ratio=0.241943 router_tokens_per_expert=[800, 731, 774, 837, 798, 763, 792, 736, 740, 733, 753, 738, 718, 775, 786, 742, 767, 772, 757, 743, 805, 769, 825, 804, 713, 779, 789, 786, 769, 808, 752, 764, 797, 815, 744, 745, 771, 778, 764, 723, 797, 717, 775, 775, 796, 721, 747, 791, 795, 778, 777, 778, 749, 746, 740, 791, 768, 769, 730, 777, 768, 783, 779, 750] payload_rows_per_dst=[9440, 9206, 9416, 9244, 9298, 9272, 9366, 9278] dup_rows_per_dst=[3022, 2764, 3068, 3076, 2976, 2966, 2942, 2970] combine_rows_per_src=[9440, 9206, 9416, 9244, 9298, 9272, 9366, 9278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=264691712 dispatch_meta_bytes=340104 combine_payload_bytes=264691712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265031816 ingress_bytes=299294720 link_bytes=299294720 token_bytes=529723528 weight_movement_bytes=34603008 total_nvlink_bytes=564326536 weight_to_token_ratio=0.065323 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126221936e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=343512 combine_payload_bytes=266641408 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266984920 ingress_bytes=370450432 link_bytes=370450432 token_bytes=533626328 weight_movement_bytes=103809024 total_nvlink_bytes=637435352 weight_to_token_ratio=0.194535 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393975152e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266739712 dispatch_meta_bytes=343944 combine_payload_bytes=266739712 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=267083656 ingress_bytes=405151744 link_bytes=405151744 token_bytes=533823368 weight_movement_bytes=138412032 total_nvlink_bytes=672235400 weight_to_token_ratio=0.259284 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524553395e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74062 payload_rows_remote=64808 dup_rows=24242 dup_rows_remote=21376 dedup_ratio=0.246602 router_tokens_per_expert=[774, 774, 755, 806, 737, 703, 785, 732, 803, 828, 765, 803, 799, 762, 794, 750, 775, 758, 741, 741, 791, 779, 757, 779, 750, 746, 759, 756, 792, 771, 709, 777, 825, 776, 759, 787, 796, 761, 767, 762, 812, 746, 781, 757, 792, 794, 745, 716, 788, 800, 746, 774, 791, 747, 754, 778, 725, 751, 739, 766, 747, 765, 787, 767] payload_rows_per_dst=[9090, 9514, 9260, 9254, 9308, 9220, 9302, 9114] dup_rows_per_dst=[3042, 3094, 2982, 2866, 3158, 3066, 3054, 2980] combine_rows_per_src=[9090, 9514, 9260, 9254, 9308, 9220, 9302, 9114] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74304 payload_rows_remote=65032 dup_rows=24000 dup_rows_remote=21046 dedup_ratio=0.244141 router_tokens_per_expert=[785, 790, 734, 783, 748, 777, 728, 755, 754, 801, 762, 752, 773, 810, 729, 740, 768, 789, 817, 783, 748, 789, 776, 812, 772, 745, 724, 747, 817, 780, 758, 825, 777, 789, 759, 778, 766, 712, 771, 761, 738, 763, 763, 806, 763, 837, 799, 731, 728, 778, 774, 761, 795, 739, 769, 728, 809, 762, 793, 726, 755, 748, 764, 739] payload_rows_per_dst=[9286, 9294, 9482, 9286, 9272, 9300, 9224, 9160] dup_rows_per_dst=[2914, 2948, 3082, 3050, 2954, 3100, 2920, 3032] combine_rows_per_src=[9286, 9294, 9482, 9286, 9272, 9300, 9224, 9160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266665984 dispatch_meta_bytes=343280 combine_payload_bytes=266665984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267009264 ingress_bytes=301268992 link_bytes=301268992 token_bytes=533675248 weight_movement_bytes=34603008 total_nvlink_bytes=568278256 weight_to_token_ratio=0.064839 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133650963e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=65090 dup_rows=23860 dup_rows_remote=20910 dedup_ratio=0.242716 router_tokens_per_expert=[756, 744, 778, 766, 741, 809, 813, 782, 760, 744, 741, 782, 807, 738, 811, 769, 738, 744, 771, 763, 764, 748, 794, 740, 709, 806, 734, 739, 765, 735, 749, 778, 768, 785, 772, 799, 767, 772, 756, 769, 781, 756, 786, 815, 768, 704, 789, 797, 769, 770, 777, 762, 765, 780, 838, 739, 793, 722, 744, 759, 793, 830, 800, 709] payload_rows_per_dst=[9380, 9354, 9198, 9176, 9366, 9342, 9342, 9286] dup_rows_per_dst=[2998, 2950, 2926, 2854, 3010, 3050, 3058, 3014] combine_rows_per_src=[9380, 9354, 9198, 9176, 9366, 9342, 9342, 9286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265453568 dispatch_meta_bytes=344736 combine_payload_bytes=265453568 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=265798304 ingress_bytes=300056576 link_bytes=300056576 token_bytes=531251872 weight_movement_bytes=34603008 total_nvlink_bytes=565854880 weight_to_token_ratio=0.065135 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129088739e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=65042 dup_rows=24008 dup_rows_remote=21038 dedup_ratio=0.244222 router_tokens_per_expert=[732, 711, 756, 751, 747, 782, 794, 743, 763, 784, 809, 802, 784, 717, 743, 745, 767, 759, 778, 762, 762, 752, 793, 739, 778, 783, 781, 819, 765, 787, 808, 784, 696, 794, 796, 780, 730, 748, 758, 752, 767, 739, 770, 808, 826, 805, 716, 780, 764, 779, 743, 726, 774, 790, 739, 778, 787, 760, 800, 787, 741, 774, 804, 761] payload_rows_per_dst=[9124, 9362, 9254, 9470, 9178, 9318, 9280, 9310] dup_rows_per_dst=[2908, 2932, 2970, 3140, 2930, 3104, 2906, 3118] combine_rows_per_src=[9124, 9362, 9254, 9470, 9178, 9318, 9280, 9310] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=343856 combine_payload_bytes=266641408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266985264 ingress_bytes=301244416 link_bytes=301244416 token_bytes=533626672 weight_movement_bytes=34603008 total_nvlink_bytes=568229680 weight_to_token_ratio=0.064845 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133558485e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74438 payload_rows_remote=65122 dup_rows=23866 dup_rows_remote=20864 dedup_ratio=0.242778 router_tokens_per_expert=[733, 817, 796, 763, 715, 811, 748, 783, 754, 747, 802, 766, 747, 774, 769, 814, 771, 719, 752, 758, 759, 737, 811, 769, 795, 772, 798, 761, 785, 782, 796, 799, 689, 753, 742, 743, 732, 813, 776, 767, 759, 741, 823, 776, 809, 784, 743, 772, 733, 753, 816, 782, 714, 768, 767, 826, 730, 766, 732, 785, 743, 783, 780, 749] payload_rows_per_dst=[9382, 9340, 9230, 9426, 9142, 9384, 9316, 9218] dup_rows_per_dst=[2950, 3006, 2922, 3150, 2888, 3030, 3002, 2918] combine_rows_per_src=[9382, 9340, 9230, 9426, 9142, 9384, 9316, 9218] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74266 payload_rows_remote=64814 dup_rows=24038 dup_rows_remote=20870 dedup_ratio=0.244527 router_tokens_per_expert=[746, 729, 767, 771, 805, 780, 771, 693, 777, 755, 724, 782, 759, 746, 766, 778, 769, 787, 809, 809, 783, 750, 750, 768, 759, 775, 743, 766, 785, 742, 730, 741, 819, 789, 781, 779, 792, 772, 770, 808, 736, 790, 767, 800, 752, 761, 800, 757, 795, 768, 782, 777, 732, 786, 740, 761, 734, 758, 770, 777, 800, 794, 765, 725] payload_rows_per_dst=[9204, 9178, 9454, 9158, 9452, 9284, 9236, 9300] dup_rows_per_dst=[2920, 2996, 2996, 2924, 3168, 3042, 3046, 2946] combine_rows_per_src=[9204, 9178, 9454, 9158, 9452, 9284, 9236, 9300] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74240 payload_rows_remote=64824 dup_rows=24064 dup_rows_remote=21038 dedup_ratio=0.244792 router_tokens_per_expert=[808, 771, 789, 761, 776, 747, 690, 749, 763, 768, 794, 704, 790, 798, 728, 791, 761, 711, 793, 758, 807, 832, 731, 736, 753, 789, 800, 758, 758, 782, 779, 751, 782, 798, 795, 793, 752, 821, 760, 789, 780, 785, 833, 742, 752, 822, 754, 761, 795, 787, 737, 778, 757, 742, 741, 763, 749, 702, 744, 755, 761, 750, 759, 787] payload_rows_per_dst=[9416, 9336, 9318, 9288, 9600, 9188, 8974, 9120] dup_rows_per_dst=[3026, 3018, 3044, 3052, 3318, 2972, 2740, 2894] combine_rows_per_src=[9416, 9336, 9318, 9288, 9600, 9188, 8974, 9120] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=344320 combine_payload_bytes=266412032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266756352 ingress_bytes=301015040 link_bytes=301015040 token_bytes=533168384 weight_movement_bytes=34603008 total_nvlink_bytes=567771392 weight_to_token_ratio=0.064901 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132695362e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=65090 dup_rows=23860 dup_rows_remote=20910 dedup_ratio=0.242716 router_tokens_per_expert=[756, 744, 778, 766, 741, 809, 813, 782, 760, 744, 741, 782, 807, 738, 811, 769, 738, 744, 771, 763, 764, 748, 794, 740, 709, 806, 734, 739, 765, 735, 749, 778, 768, 785, 772, 799, 767, 772, 756, 769, 781, 756, 786, 815, 768, 704, 789, 797, 769, 770, 777, 762, 765, 780, 838, 739, 793, 722, 744, 759, 793, 830, 800, 709] payload_rows_per_dst=[9380, 9354, 9198, 9176, 9366, 9342, 9342, 9286] dup_rows_per_dst=[2998, 2950, 2926, 2854, 3010, 3050, 3058, 3014] combine_rows_per_src=[9380, 9354, 9198, 9176, 9366, 9342, 9342, 9286] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266739712 dispatch_meta_bytes=343944 combine_payload_bytes=266739712 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=267083656 ingress_bytes=405151744 link_bytes=405151744 token_bytes=533823368 weight_movement_bytes=138412032 total_nvlink_bytes=672235400 weight_to_token_ratio=0.259284 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524553395e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74326 payload_rows_remote=64998 dup_rows=23978 dup_rows_remote=20802 dedup_ratio=0.243917 router_tokens_per_expert=[744, 745, 773, 819, 769, 768, 789, 792, 784, 733, 766, 743, 752, 807, 713, 738, 729, 798, 756, 771, 747, 765, 808, 752, 726, 773, 780, 773, 805, 693, 773, 804, 825, 765, 780, 801, 790, 746, 797, 744, 781, 763, 768, 812, 791, 767, 797, 773, 792, 763, 737, 766, 785, 756, 778, 735, 766, 728, 736, 690, 776, 754, 797, 805] payload_rows_per_dst=[9422, 9118, 9296, 9336, 9398, 9328, 9224, 9204] dup_rows_per_dst=[2976, 2954, 2956, 2918, 3098, 3176, 3000, 2900] combine_rows_per_src=[9422, 9118, 9296, 9336, 9398, 9328, 9224, 9204] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344312 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266715384 ingress_bytes=300974080 link_bytes=300974080 token_bytes=533086456 weight_movement_bytes=34603008 total_nvlink_bytes=567689464 weight_to_token_ratio=0.064911 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132541233e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266608640 dispatch_meta_bytes=344000 combine_payload_bytes=266608640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266952640 ingress_bytes=301211648 link_bytes=301211648 token_bytes=533561280 weight_movement_bytes=34603008 total_nvlink_bytes=568164288 weight_to_token_ratio=0.064853 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133435182e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265478144 dispatch_meta_bytes=342736 combine_payload_bytes=265478144 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265820880 ingress_bytes=369287168 link_bytes=369287168 token_bytes=531299024 weight_movement_bytes=103809024 total_nvlink_bytes=635108048 weight_to_token_ratio=0.195387 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389597883e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74094 payload_rows_remote=64808 dup_rows=24210 dup_rows_remote=21110 dedup_ratio=0.246277 router_tokens_per_expert=[771, 791, 732, 780, 789, 786, 765, 771, 764, 743, 769, 784, 758, 718, 772, 790, 772, 772, 767, 751, 743, 731, 715, 762, 792, 798, 797, 819, 765, 742, 776, 704, 759, 800, 766, 755, 781, 824, 814, 757, 763, 789, 747, 789, 812, 763, 731, 768, 770, 781, 828, 755, 733, 798, 764, 769, 784, 748, 731, 790, 772, 743, 712, 767] payload_rows_per_dst=[9312, 9176, 9110, 9286, 9428, 9304, 9314, 9164] dup_rows_per_dst=[3058, 3020, 2916, 3100, 3084, 3020, 3082, 2930] combine_rows_per_src=[9312, 9176, 9110, 9286, 9428, 9304, 9314, 9164] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=65098 dup_rows=23926 dup_rows_remote=20866 dedup_ratio=0.243388 router_tokens_per_expert=[802, 791, 772, 765, 758, 740, 767, 761, 821, 776, 778, 765, 798, 765, 798, 730, 760, 726, 822, 743, 740, 777, 731, 726, 765, 778, 762, 747, 773, 786, 778, 789, 721, 774, 763, 732, 766, 785, 739, 744, 786, 750, 762, 821, 787, 781, 770, 799, 769, 804, 797, 751, 689, 745, 780, 816, 738, 775, 793, 791, 745, 767, 765, 757] payload_rows_per_dst=[9346, 9386, 9234, 9298, 9188, 9340, 9306, 9280] dup_rows_per_dst=[2966, 3076, 2816, 3058, 2774, 3172, 3004, 3060] combine_rows_per_src=[9346, 9386, 9234, 9298, 9188, 9340, 9306, 9280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74326 payload_rows_remote=64998 dup_rows=23978 dup_rows_remote=20802 dedup_ratio=0.243917 router_tokens_per_expert=[744, 745, 773, 819, 769, 768, 789, 792, 784, 733, 766, 743, 752, 807, 713, 738, 729, 798, 756, 771, 747, 765, 808, 752, 726, 773, 780, 773, 805, 693, 773, 804, 825, 765, 780, 801, 790, 746, 797, 744, 781, 763, 768, 812, 791, 767, 797, 773, 792, 763, 737, 766, 785, 756, 778, 735, 766, 728, 736, 690, 776, 754, 797, 805] payload_rows_per_dst=[9422, 9118, 9296, 9336, 9398, 9328, 9224, 9204] dup_rows_per_dst=[2976, 2954, 2956, 2918, 3098, 3176, 3000, 2900] combine_rows_per_src=[9422, 9118, 9296, 9336, 9398, 9328, 9224, 9204] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74234 payload_rows_remote=64896 dup_rows=24070 dup_rows_remote=21084 dedup_ratio=0.244853 router_tokens_per_expert=[798, 792, 789, 764, 760, 730, 761, 789, 805, 803, 746, 779, 791, 782, 725, 769, 754, 812, 738, 760, 773, 749, 761, 789, 747, 736, 761, 743, 755, 743, 751, 787, 778, 743, 772, 759, 745, 779, 764, 757, 785, 776, 802, 738, 727, 725, 819, 796, 788, 785, 786, 818, 719, 723, 758, 785, 809, 808, 781, 744, 775, 763, 724, 779] payload_rows_per_dst=[9230, 9424, 9212, 9172, 9238, 9314, 9338, 9306] dup_rows_per_dst=[3136, 2976, 3060, 2874, 2956, 3022, 2986, 3060] combine_rows_per_src=[9230, 9424, 9212, 9172, 9238, 9314, 9338, 9306] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=65042 dup_rows=24008 dup_rows_remote=21038 dedup_ratio=0.244222 router_tokens_per_expert=[732, 711, 756, 751, 747, 782, 794, 743, 763, 784, 809, 802, 784, 717, 743, 745, 767, 759, 778, 762, 762, 752, 793, 739, 778, 783, 781, 819, 765, 787, 808, 784, 696, 794, 796, 780, 730, 748, 758, 752, 767, 739, 770, 808, 826, 805, 716, 780, 764, 779, 743, 726, 774, 790, 739, 778, 787, 760, 800, 787, 741, 774, 804, 761] payload_rows_per_dst=[9124, 9362, 9254, 9470, 9178, 9318, 9280, 9310] dup_rows_per_dst=[2908, 2932, 2970, 3140, 2930, 3104, 2906, 3118] combine_rows_per_src=[9124, 9362, 9254, 9470, 9178, 9318, 9280, 9310] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265519104 dispatch_meta_bytes=343448 combine_payload_bytes=265519104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265862552 ingress_bytes=300122112 link_bytes=300122112 token_bytes=531381656 weight_movement_bytes=34603008 total_nvlink_bytes=565984664 weight_to_token_ratio=0.065119 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129335346e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266608640 dispatch_meta_bytes=344000 combine_payload_bytes=266608640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266952640 ingress_bytes=301211648 link_bytes=301211648 token_bytes=533561280 weight_movement_bytes=34603008 total_nvlink_bytes=568164288 weight_to_token_ratio=0.064853 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133435182e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74366 payload_rows_remote=65038 dup_rows=23938 dup_rows_remote=20990 dedup_ratio=0.243510 router_tokens_per_expert=[744, 763, 795, 793, 751, 784, 809, 769, 760, 774, 758, 755, 762, 815, 762, 800, 778, 767, 791, 819, 782, 750, 755, 783, 716, 778, 790, 802, 715, 768, 734, 766, 743, 779, 806, 744, 759, 714, 742, 757, 769, 791, 762, 726, 804, 784, 783, 810, 793, 736, 741, 750, 805, 725, 716, 788, 769, 753, 771, 764, 787, 774, 752, 767] payload_rows_per_dst=[9002, 9002, 9490, 9252, 9502, 9496, 9294, 9328] dup_rows_per_dst=[2684, 2732, 3084, 2886, 3314, 3298, 2992, 2948] combine_rows_per_src=[9002, 9002, 9490, 9252, 9502, 9496, 9294, 9328] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=343856 combine_payload_bytes=266641408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266985264 ingress_bytes=301244416 link_bytes=301244416 token_bytes=533626672 weight_movement_bytes=34603008 total_nvlink_bytes=568229680 weight_to_token_ratio=0.064845 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133558485e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74094 payload_rows_remote=64808 dup_rows=24210 dup_rows_remote=21110 dedup_ratio=0.246277 router_tokens_per_expert=[771, 791, 732, 780, 789, 786, 765, 771, 764, 743, 769, 784, 758, 718, 772, 790, 772, 772, 767, 751, 743, 731, 715, 762, 792, 798, 797, 819, 765, 742, 776, 704, 759, 800, 766, 755, 781, 824, 814, 757, 763, 789, 747, 789, 812, 763, 731, 768, 770, 781, 828, 755, 733, 798, 764, 769, 784, 748, 731, 790, 772, 743, 712, 767] payload_rows_per_dst=[9312, 9176, 9110, 9286, 9428, 9304, 9314, 9164] dup_rows_per_dst=[3058, 3020, 2916, 3100, 3084, 3020, 3082, 2930] combine_rows_per_src=[9312, 9176, 9110, 9286, 9428, 9304, 9314, 9164] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266231808 dispatch_meta_bytes=343200 combine_payload_bytes=266231808 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266575008 ingress_bytes=370040832 link_bytes=370040832 token_bytes=532806816 weight_movement_bytes=103809024 total_nvlink_bytes=636615840 weight_to_token_ratio=0.194834 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392433860e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265814016 dispatch_meta_bytes=343920 combine_payload_bytes=265814016 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266157936 ingress_bytes=369623040 link_bytes=369623040 token_bytes=531971952 weight_movement_bytes=103809024 total_nvlink_bytes=635780976 weight_to_token_ratio=0.195140 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390861742e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74532 payload_rows_remote=65164 dup_rows=23772 dup_rows_remote=20734 dedup_ratio=0.241821 router_tokens_per_expert=[788, 748, 751, 783, 735, 759, 791, 771, 727, 764, 766, 778, 782, 750, 765, 832, 777, 733, 796, 792, 793, 787, 778, 742, 749, 748, 748, 802, 762, 761, 759, 821, 790, 768, 774, 785, 778, 754, 782, 772, 772, 762, 721, 732, 776, 758, 838, 738, 760, 784, 766, 750, 791, 773, 770, 772, 776, 729, 754, 724, 778, 757, 772, 758] payload_rows_per_dst=[9318, 9394, 9382, 9298, 9368, 9310, 9242, 9220] dup_rows_per_dst=[2934, 2934, 3014, 3002, 3038, 2884, 3090, 2876] combine_rows_per_src=[9318, 9394, 9382, 9298, 9368, 9310, 9242, 9220] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=344320 combine_payload_bytes=266412032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266756352 ingress_bytes=301015040 link_bytes=301015040 token_bytes=533168384 weight_movement_bytes=34603008 total_nvlink_bytes=567771392 weight_to_token_ratio=0.064901 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132695362e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74474 payload_rows_remote=64990 dup_rows=23830 dup_rows_remote=20704 dedup_ratio=0.242411 router_tokens_per_expert=[773, 739, 743, 776, 822, 769, 728, 785, 762, 795, 767, 829, 797, 786, 795, 774, 790, 747, 731, 753, 784, 776, 789, 782, 748, 749, 736, 764, 750, 723, 814, 724, 728, 784, 800, 727, 770, 764, 792, 735, 741, 714, 759, 753, 756, 824, 754, 744, 747, 754, 739, 766, 754, 807, 824, 803, 806, 784, 780, 792, 726, 716, 786, 823] payload_rows_per_dst=[9272, 9484, 9284, 9116, 9258, 9276, 9360, 9424] dup_rows_per_dst=[2998, 3126, 3020, 2900, 2942, 2814, 3028, 3002] combine_rows_per_src=[9272, 9484, 9284, 9116, 9258, 9276, 9360, 9424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265453568 dispatch_meta_bytes=343672 combine_payload_bytes=265453568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265797240 ingress_bytes=300056576 link_bytes=300056576 token_bytes=531250808 weight_movement_bytes=34603008 total_nvlink_bytes=565853816 weight_to_token_ratio=0.065135 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129088739e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=64964 dup_rows=24018 dup_rows_remote=20960 dedup_ratio=0.244324 router_tokens_per_expert=[787, 759, 693, 729, 793, 717, 794, 753, 784, 822, 774, 750, 747, 761, 785, 799, 755, 788, 751, 767, 759, 800, 783, 787, 727, 800, 788, 768, 764, 772, 785, 738, 733, 732, 740, 782, 805, 819, 785, 756, 800, 820, 777, 765, 784, 758, 748, 732, 749, 790, 799, 761, 770, 781, 734, 796, 739, 765, 747, 774, 742, 803, 783, 704] payload_rows_per_dst=[9130, 9384, 9322, 9298, 9276, 9302, 9360, 9214] dup_rows_per_dst=[2920, 3060, 3058, 2986, 3028, 3066, 3000, 2900] combine_rows_per_src=[9130, 9384, 9322, 9298, 9276, 9302, 9360, 9214] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=344112 combine_payload_bytes=266395648 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266739760 ingress_bytes=370204672 link_bytes=370204672 token_bytes=533135408 weight_movement_bytes=103809024 total_nvlink_bytes=636944432 weight_to_token_ratio=0.194714 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393050377e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74234 payload_rows_remote=64896 dup_rows=24070 dup_rows_remote=21084 dedup_ratio=0.244853 router_tokens_per_expert=[798, 792, 789, 764, 760, 730, 761, 789, 805, 803, 746, 779, 791, 782, 725, 769, 754, 812, 738, 760, 773, 749, 761, 789, 747, 736, 761, 743, 755, 743, 751, 787, 778, 743, 772, 759, 745, 779, 764, 757, 785, 776, 802, 738, 727, 725, 819, 796, 788, 785, 786, 818, 719, 723, 758, 785, 809, 808, 781, 744, 775, 763, 724, 779] payload_rows_per_dst=[9230, 9424, 9212, 9172, 9238, 9314, 9338, 9306] dup_rows_per_dst=[3136, 2976, 3060, 2874, 2956, 3022, 2986, 3060] combine_rows_per_src=[9230, 9424, 9212, 9172, 9238, 9314, 9338, 9306] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74146 payload_rows_remote=65004 dup_rows=24158 dup_rows_remote=21158 dedup_ratio=0.245748 router_tokens_per_expert=[746, 752, 808, 785, 817, 788, 770, 797, 787, 742, 750, 778, 750, 811, 765, 746, 754, 751, 816, 746, 764, 725, 738, 755, 729, 776, 750, 796, 799, 797, 783, 754, 762, 739, 713, 764, 782, 774, 763, 774, 763, 747, 767, 731, 718, 763, 787, 760, 800, 821, 796, 792, 731, 792, 714, 792, 793, 799, 770, 805, 760, 750, 767, 738] payload_rows_per_dst=[9438, 9234, 9162, 9298, 9142, 9136, 9392, 9344] dup_rows_per_dst=[3088, 3024, 2936, 3070, 3000, 2936, 3084, 3020] combine_rows_per_src=[9438, 9234, 9162, 9298, 9142, 9136, 9392, 9344] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266911744 dispatch_meta_bytes=343592 combine_payload_bytes=266911744 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267255336 ingress_bytes=336117760 link_bytes=336117760 token_bytes=534167080 weight_movement_bytes=69206016 total_nvlink_bytes=603373096 weight_to_token_ratio=0.129559 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264784071e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74326 payload_rows_remote=64968 dup_rows=23978 dup_rows_remote=21012 dedup_ratio=0.243917 router_tokens_per_expert=[764, 752, 757, 745, 786, 762, 719, 795, 708, 749, 778, 759, 779, 807, 761, 753, 757, 737, 800, 839, 758, 729, 746, 735, 768, 767, 779, 714, 758, 765, 736, 779, 758, 792, 790, 785, 758, 781, 774, 784, 742, 814, 748, 760, 763, 724, 803, 815, 779, 742, 766, 821, 791, 774, 766, 768, 786, 768, 810, 752, 764, 772, 766, 795] payload_rows_per_dst=[9358, 9330, 9652, 9430, 9108, 8882, 9232, 9334] dup_rows_per_dst=[2966, 3072, 3254, 3110, 2928, 2606, 2950, 3092] combine_rows_per_src=[9358, 9330, 9652, 9430, 9108, 8882, 9232, 9334] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=343696 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266436240 ingress_bytes=300695552 link_bytes=300695552 token_bytes=532528784 weight_movement_bytes=34603008 total_nvlink_bytes=567131792 weight_to_token_ratio=0.064979 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131493154e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74474 payload_rows_remote=64990 dup_rows=23830 dup_rows_remote=20704 dedup_ratio=0.242411 router_tokens_per_expert=[773, 739, 743, 776, 822, 769, 728, 785, 762, 795, 767, 829, 797, 786, 795, 774, 790, 747, 731, 753, 784, 776, 789, 782, 748, 749, 736, 764, 750, 723, 814, 724, 728, 784, 800, 727, 770, 764, 792, 735, 741, 714, 759, 753, 756, 824, 754, 744, 747, 754, 739, 766, 754, 807, 824, 803, 806, 784, 780, 792, 726, 716, 786, 823] payload_rows_per_dst=[9272, 9484, 9284, 9116, 9258, 9276, 9360, 9424] dup_rows_per_dst=[2998, 3126, 3020, 2900, 2942, 2814, 3028, 3002] combine_rows_per_src=[9272, 9484, 9284, 9116, 9258, 9276, 9360, 9424] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74372 payload_rows_remote=65168 dup_rows=23932 dup_rows_remote=20984 dedup_ratio=0.243449 router_tokens_per_expert=[783, 780, 768, 824, 789, 786, 770, 778, 795, 755, 767, 775, 778, 730, 798, 765, 713, 783, 794, 724, 776, 749, 782, 732, 771, 799, 784, 796, 745, 788, 776, 760, 816, 775, 762, 716, 753, 792, 829, 741, 765, 762, 787, 742, 795, 754, 733, 738, 733, 777, 762, 782, 732, 721, 766, 755, 808, 790, 807, 724, 779, 765, 750, 728] payload_rows_per_dst=[9518, 9346, 9178, 9450, 9246, 9204, 9140, 9290] dup_rows_per_dst=[3038, 2980, 2928, 2988, 3122, 2948, 2916, 3012] combine_rows_per_src=[9518, 9346, 9178, 9450, 9246, 9204, 9140, 9290] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265814016 dispatch_meta_bytes=343920 combine_payload_bytes=265814016 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266157936 ingress_bytes=369623040 link_bytes=369623040 token_bytes=531971952 weight_movement_bytes=103809024 total_nvlink_bytes=635780976 weight_to_token_ratio=0.195140 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390861742e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=342776 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266541816 ingress_bytes=404611072 link_bytes=404611072 token_bytes=532740856 weight_movement_bytes=138412032 total_nvlink_bytes=671152888 weight_to_token_ratio=0.259811 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522518890e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74366 payload_rows_remote=65038 dup_rows=23938 dup_rows_remote=20990 dedup_ratio=0.243510 router_tokens_per_expert=[744, 763, 795, 793, 751, 784, 809, 769, 760, 774, 758, 755, 762, 815, 762, 800, 778, 767, 791, 819, 782, 750, 755, 783, 716, 778, 790, 802, 715, 768, 734, 766, 743, 779, 806, 744, 759, 714, 742, 757, 769, 791, 762, 726, 804, 784, 783, 810, 793, 736, 741, 750, 805, 725, 716, 788, 769, 753, 771, 764, 787, 774, 752, 767] payload_rows_per_dst=[9002, 9002, 9490, 9252, 9502, 9496, 9294, 9328] dup_rows_per_dst=[2684, 2732, 3084, 2886, 3314, 3298, 2992, 2948] combine_rows_per_src=[9002, 9002, 9490, 9252, 9502, 9496, 9294, 9328] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266256384 dispatch_meta_bytes=344648 combine_payload_bytes=266256384 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266601032 ingress_bytes=300859392 link_bytes=300859392 token_bytes=532857416 weight_movement_bytes=34603008 total_nvlink_bytes=567460424 weight_to_token_ratio=0.064939 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132109671e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65100 dup_rows=23924 dup_rows_remote=20876 dedup_ratio=0.243368 router_tokens_per_expert=[815, 773, 781, 734, 772, 745, 769, 803, 765, 731, 736, 797, 760, 800, 754, 784, 803, 749, 760, 781, 797, 738, 733, 751, 757, 738, 736, 780, 795, 781, 766, 811, 767, 742, 754, 738, 751, 740, 802, 766, 764, 794, 781, 754, 772, 796, 733, 766, 780, 745, 806, 770, 740, 775, 763, 780, 803, 792, 754, 761, 755, 750, 779, 784] payload_rows_per_dst=[9366, 9266, 9296, 9280, 9230, 9328, 9286, 9328] dup_rows_per_dst=[3018, 2988, 2928, 3048, 2890, 2992, 3032, 3028] combine_rows_per_src=[9366, 9266, 9296, 9280, 9230, 9328, 9286, 9328] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74372 payload_rows_remote=65168 dup_rows=23932 dup_rows_remote=20984 dedup_ratio=0.243449 router_tokens_per_expert=[783, 780, 768, 824, 789, 786, 770, 778, 795, 755, 767, 775, 778, 730, 798, 765, 713, 783, 794, 724, 776, 749, 782, 732, 771, 799, 784, 796, 745, 788, 776, 760, 816, 775, 762, 716, 753, 792, 829, 741, 765, 762, 787, 742, 795, 754, 733, 738, 733, 777, 762, 782, 732, 721, 766, 755, 808, 790, 807, 724, 779, 765, 750, 728] payload_rows_per_dst=[9518, 9346, 9178, 9450, 9246, 9204, 9140, 9290] dup_rows_per_dst=[3038, 2980, 2928, 2988, 3122, 2948, 2916, 3012] combine_rows_per_src=[9518, 9346, 9178, 9450, 9246, 9204, 9140, 9290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74502 payload_rows_remote=65168 dup_rows=23802 dup_rows_remote=20934 dedup_ratio=0.242126 router_tokens_per_expert=[779, 801, 786, 814, 781, 768, 790, 786, 770, 748, 775, 790, 775, 771, 769, 762, 738, 741, 720, 780, 733, 709, 832, 796, 777, 803, 723, 770, 783, 742, 775, 755, 776, 786, 728, 785, 767, 738, 762, 750, 808, 770, 782, 727, 793, 770, 771, 725, 766, 747, 787, 707, 768, 781, 776, 769, 788, 740, 801, 777, 785, 785, 737, 758] payload_rows_per_dst=[9480, 9304, 9238, 9298, 9194, 9312, 9334, 9342] dup_rows_per_dst=[3130, 3016, 2860, 2958, 2990, 2980, 2868, 3000] combine_rows_per_src=[9480, 9304, 9238, 9298, 9194, 9312, 9334, 9342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=64964 dup_rows=24018 dup_rows_remote=20960 dedup_ratio=0.244324 router_tokens_per_expert=[787, 759, 693, 729, 793, 717, 794, 753, 784, 822, 774, 750, 747, 761, 785, 799, 755, 788, 751, 767, 759, 800, 783, 787, 727, 800, 788, 768, 764, 772, 785, 738, 733, 732, 740, 782, 805, 819, 785, 756, 800, 820, 777, 765, 784, 758, 748, 732, 749, 790, 799, 761, 770, 781, 734, 796, 739, 765, 747, 774, 742, 803, 783, 704] payload_rows_per_dst=[9130, 9384, 9322, 9298, 9276, 9302, 9360, 9214] dup_rows_per_dst=[2920, 3060, 3058, 2986, 3028, 3066, 3000, 2900] combine_rows_per_src=[9130, 9384, 9322, 9298, 9276, 9302, 9360, 9214] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=343920 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266452848 ingress_bytes=300711936 link_bytes=300711936 token_bytes=532561776 weight_movement_bytes=34603008 total_nvlink_bytes=567164784 weight_to_token_ratio=0.064975 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131554806e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74362 payload_rows_remote=64962 dup_rows=23942 dup_rows_remote=20872 dedup_ratio=0.243551 router_tokens_per_expert=[771, 755, 790, 720, 826, 752, 784, 763, 799, 781, 795, 766, 773, 788, 759, 772, 750, 781, 827, 724, 778, 847, 787, 820, 749, 781, 785, 775, 774, 780, 713, 771, 743, 736, 743, 735, 769, 785, 799, 777, 780, 768, 789, 738, 709, 755, 705, 752, 745, 763, 759, 763, 767, 825, 806, 766, 755, 782, 737, 764, 757, 750, 756, 738] payload_rows_per_dst=[9098, 9210, 9398, 9454, 9302, 9286, 9214, 9400] dup_rows_per_dst=[2940, 2916, 3050, 3134, 3052, 2990, 2790, 3070] combine_rows_per_src=[9098, 9210, 9398, 9454, 9302, 9286, 9214, 9400] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=342776 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266541816 ingress_bytes=404611072 link_bytes=404611072 token_bytes=532740856 weight_movement_bytes=138412032 total_nvlink_bytes=671152888 weight_to_token_ratio=0.259811 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522518890e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65100 dup_rows=23924 dup_rows_remote=20876 dedup_ratio=0.243368 router_tokens_per_expert=[815, 773, 781, 734, 772, 745, 769, 803, 765, 731, 736, 797, 760, 800, 754, 784, 803, 749, 760, 781, 797, 738, 733, 751, 757, 738, 736, 780, 795, 781, 766, 811, 767, 742, 754, 738, 751, 740, 802, 766, 764, 794, 781, 754, 772, 796, 733, 766, 780, 745, 806, 770, 740, 775, 763, 780, 803, 792, 754, 761, 755, 750, 779, 784] payload_rows_per_dst=[9366, 9266, 9296, 9280, 9230, 9328, 9286, 9328] dup_rows_per_dst=[3018, 2988, 2928, 3048, 2890, 2992, 3032, 3028] combine_rows_per_src=[9366, 9266, 9296, 9280, 9230, 9328, 9286, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=344112 combine_payload_bytes=266395648 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266739760 ingress_bytes=370204672 link_bytes=370204672 token_bytes=533135408 weight_movement_bytes=103809024 total_nvlink_bytes=636944432 weight_to_token_ratio=0.194714 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393050377e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74266 payload_rows_remote=64814 dup_rows=24038 dup_rows_remote=20870 dedup_ratio=0.244527 router_tokens_per_expert=[746, 729, 767, 771, 805, 780, 771, 693, 777, 755, 724, 782, 759, 746, 766, 778, 769, 787, 809, 809, 783, 750, 750, 768, 759, 775, 743, 766, 785, 742, 730, 741, 819, 789, 781, 779, 792, 772, 770, 808, 736, 790, 767, 800, 752, 761, 800, 757, 795, 768, 782, 777, 732, 786, 740, 761, 734, 758, 770, 777, 800, 794, 765, 725] payload_rows_per_dst=[9204, 9178, 9454, 9158, 9452, 9284, 9236, 9300] dup_rows_per_dst=[2920, 2996, 2996, 2924, 3168, 3042, 3046, 2946] combine_rows_per_src=[9204, 9178, 9454, 9158, 9452, 9284, 9236, 9300] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266928128 dispatch_meta_bytes=344608 combine_payload_bytes=266928128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267272736 ingress_bytes=301531136 link_bytes=301531136 token_bytes=534200864 weight_movement_bytes=34603008 total_nvlink_bytes=568803872 weight_to_token_ratio=0.064775 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134637390e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266928128 dispatch_meta_bytes=344408 combine_payload_bytes=266928128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267272536 ingress_bytes=301531136 link_bytes=301531136 token_bytes=534200664 weight_movement_bytes=34603008 total_nvlink_bytes=568803672 weight_to_token_ratio=0.064775 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134637390e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=343696 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266436240 ingress_bytes=300695552 link_bytes=300695552 token_bytes=532528784 weight_movement_bytes=34603008 total_nvlink_bytes=567131792 weight_to_token_ratio=0.064979 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131493154e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74566 payload_rows_remote=65298 dup_rows=23738 dup_rows_remote=20832 dedup_ratio=0.241475 router_tokens_per_expert=[770, 799, 781, 750, 773, 755, 764, 797, 795, 754, 751, 740, 777, 711, 789, 770, 771, 783, 733, 765, 761, 806, 796, 770, 762, 744, 780, 745, 776, 757, 789, 770, 818, 754, 784, 777, 748, 769, 772, 733, 727, 742, 753, 699, 740, 772, 751, 777, 794, 800, 772, 749, 811, 768, 781, 765, 738, 769, 742, 795, 817, 785, 778, 788] payload_rows_per_dst=[9394, 9268, 9352, 9288, 9432, 9096, 9346, 9390] dup_rows_per_dst=[2984, 2906, 3018, 2958, 2878, 2826, 3134, 3034] combine_rows_per_src=[9394, 9268, 9352, 9288, 9432, 9096, 9346, 9390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265478144 dispatch_meta_bytes=342736 combine_payload_bytes=265478144 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265820880 ingress_bytes=369287168 link_bytes=369287168 token_bytes=531299024 weight_movement_bytes=103809024 total_nvlink_bytes=635108048 weight_to_token_ratio=0.195387 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389597883e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266649600 dispatch_meta_bytes=343904 combine_payload_bytes=266649600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266993504 ingress_bytes=301252608 link_bytes=301252608 token_bytes=533643104 weight_movement_bytes=34603008 total_nvlink_bytes=568246112 weight_to_token_ratio=0.064843 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133589311e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266084352 dispatch_meta_bytes=343336 combine_payload_bytes=266084352 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266427688 ingress_bytes=300687360 link_bytes=300687360 token_bytes=532512040 weight_movement_bytes=34603008 total_nvlink_bytes=567115048 weight_to_token_ratio=0.064981 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131462328e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74224 payload_rows_remote=65016 dup_rows=24080 dup_rows_remote=21076 dedup_ratio=0.244954 router_tokens_per_expert=[833, 801, 791, 778, 769, 738, 789, 773, 777, 802, 808, 755, 773, 830, 780, 750, 758, 768, 772, 768, 763, 760, 732, 785, 759, 726, 772, 742, 761, 766, 800, 755, 796, 802, 716, 718, 791, 741, 733, 808, 784, 734, 780, 760, 780, 764, 804, 758, 787, 760, 752, 732, 772, 747, 778, 734, 768, 739, 736, 820, 709, 774, 762, 779] payload_rows_per_dst=[9404, 9470, 9208, 9198, 9252, 9240, 9170, 9282] dup_rows_per_dst=[3140, 3080, 3004, 2964, 2958, 3088, 2954, 2892] combine_rows_per_src=[9404, 9470, 9208, 9198, 9252, 9240, 9170, 9282] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74502 payload_rows_remote=65168 dup_rows=23802 dup_rows_remote=20934 dedup_ratio=0.242126 router_tokens_per_expert=[779, 801, 786, 814, 781, 768, 790, 786, 770, 748, 775, 790, 775, 771, 769, 762, 738, 741, 720, 780, 733, 709, 832, 796, 777, 803, 723, 770, 783, 742, 775, 755, 776, 786, 728, 785, 767, 738, 762, 750, 808, 770, 782, 727, 793, 770, 771, 725, 766, 747, 787, 707, 768, 781, 776, 769, 788, 740, 801, 777, 785, 785, 737, 758] payload_rows_per_dst=[9480, 9304, 9238, 9298, 9194, 9312, 9334, 9342] dup_rows_per_dst=[3130, 3016, 2860, 2958, 2990, 2980, 2868, 3000] combine_rows_per_src=[9480, 9304, 9238, 9298, 9194, 9312, 9334, 9342] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74280 payload_rows_remote=64970 dup_rows=24024 dup_rows_remote=21056 dedup_ratio=0.244385 router_tokens_per_expert=[770, 755, 740, 776, 814, 791, 805, 785, 759, 714, 771, 780, 766, 780, 785, 781, 765, 759, 780, 731, 775, 734, 748, 772, 762, 758, 847, 736, 768, 754, 734, 762, 767, 722, 826, 721, 775, 763, 754, 811, 746, 725, 791, 800, 766, 763, 786, 799, 720, 759, 846, 768, 717, 770, 797, 778, 755, 779, 772, 797, 747, 783, 763, 729] payload_rows_per_dst=[9358, 9276, 9122, 9284, 9310, 9312, 9316, 9302] dup_rows_per_dst=[3114, 2996, 3006, 2958, 2968, 3040, 2994, 2948] combine_rows_per_src=[9358, 9276, 9122, 9284, 9310, 9312, 9316, 9302] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74262 payload_rows_remote=64972 dup_rows=24042 dup_rows_remote=21110 dedup_ratio=0.244568 router_tokens_per_expert=[691, 773, 766, 782, 804, 800, 765, 730, 757, 743, 809, 743, 791, 784, 801, 740, 825, 794, 738, 762, 790, 795, 779, 777, 744, 737, 757, 787, 773, 793, 793, 767, 767, 749, 765, 745, 776, 764, 778, 741, 772, 735, 748, 747, 786, 776, 748, 796, 773, 743, 742, 783, 783, 773, 781, 731, 781, 757, 795, 742, 732, 831, 745, 777] payload_rows_per_dst=[9290, 9364, 9388, 9458, 9178, 9104, 9228, 9252] dup_rows_per_dst=[2932, 2994, 3132, 3018, 2992, 3112, 2990, 2872] combine_rows_per_src=[9290, 9364, 9388, 9458, 9178, 9104, 9228, 9252] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=344368 combine_payload_bytes=266305536 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266649904 ingress_bytes=335511552 link_bytes=335511552 token_bytes=532955440 weight_movement_bytes=69206016 total_nvlink_bytes=602161456 weight_to_token_ratio=0.129853 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262502959e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74566 payload_rows_remote=65298 dup_rows=23738 dup_rows_remote=20832 dedup_ratio=0.241475 router_tokens_per_expert=[770, 799, 781, 750, 773, 755, 764, 797, 795, 754, 751, 740, 777, 711, 789, 770, 771, 783, 733, 765, 761, 806, 796, 770, 762, 744, 780, 745, 776, 757, 789, 770, 818, 754, 784, 777, 748, 769, 772, 733, 727, 742, 753, 699, 740, 772, 751, 777, 794, 800, 772, 749, 811, 768, 781, 765, 738, 769, 742, 795, 817, 785, 778, 788] payload_rows_per_dst=[9394, 9268, 9352, 9288, 9432, 9096, 9346, 9390] dup_rows_per_dst=[2984, 2906, 3018, 2958, 2878, 2826, 3134, 3034] combine_rows_per_src=[9394, 9268, 9352, 9288, 9432, 9096, 9346, 9390] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=65016 dup_rows=23974 dup_rows_remote=20872 dedup_ratio=0.243876 router_tokens_per_expert=[770, 750, 736, 768, 755, 766, 765, 750, 758, 770, 734, 832, 732, 819, 757, 740, 748, 747, 733, 795, 768, 743, 773, 748, 742, 769, 797, 800, 731, 784, 734, 770, 745, 790, 757, 802, 765, 765, 775, 797, 783, 762, 739, 806, 781, 779, 788, 770, 755, 771, 823, 762, 794, 775, 733, 808, 754, 741, 782, 776, 747, 811, 793, 739] payload_rows_per_dst=[9242, 9346, 9252, 9322, 9336, 9314, 9268, 9250] dup_rows_per_dst=[2878, 2938, 2858, 2932, 3056, 3102, 3174, 3036] combine_rows_per_src=[9242, 9346, 9252, 9322, 9336, 9314, 9268, 9250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=267460608 dispatch_meta_bytes=344520 combine_payload_bytes=267460608 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267805128 ingress_bytes=371269632 link_bytes=371269632 token_bytes=535265736 weight_movement_bytes=103809024 total_nvlink_bytes=639074760 weight_to_token_ratio=0.193939 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397057736e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266117120 dispatch_meta_bytes=344104 combine_payload_bytes=266117120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266461224 ingress_bytes=300720128 link_bytes=300720128 token_bytes=532578344 weight_movement_bytes=34603008 total_nvlink_bytes=567181352 weight_to_token_ratio=0.064973 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131585632e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266928128 dispatch_meta_bytes=344408 combine_payload_bytes=266928128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267272536 ingress_bytes=301531136 link_bytes=301531136 token_bytes=534200664 weight_movement_bytes=34603008 total_nvlink_bytes=568803672 weight_to_token_ratio=0.064775 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134637390e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73970 payload_rows_remote=64708 dup_rows=24334 dup_rows_remote=21230 dedup_ratio=0.247538 router_tokens_per_expert=[733, 750, 749, 793, 853, 730, 770, 803, 761, 748, 771, 746, 742, 759, 758, 791, 768, 754, 772, 749, 800, 765, 791, 796, 806, 729, 766, 784, 734, 795, 772, 775, 766, 759, 725, 787, 760, 717, 791, 770, 764, 809, 746, 788, 760, 817, 786, 797, 808, 771, 790, 757, 745, 771, 758, 733, 728, 774, 757, 804, 728, 760, 786, 727] payload_rows_per_dst=[9222, 8992, 9482, 9242, 9202, 9324, 9244, 9262] dup_rows_per_dst=[3044, 2894, 3174, 3078, 3010, 3006, 3024, 3104] combine_rows_per_src=[9222, 8992, 9482, 9242, 9202, 9324, 9244, 9262] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=65008 dup_rows=23926 dup_rows_remote=21044 dedup_ratio=0.243388 router_tokens_per_expert=[741, 800, 750, 824, 716, 755, 761, 720, 764, 761, 734, 791, 786, 765, 805, 770, 759, 808, 780, 755, 769, 711, 798, 828, 787, 772, 786, 767, 763, 746, 735, 770, 695, 824, 755, 766, 786, 738, 792, 771, 749, 804, 725, 815, 780, 769, 781, 741, 753, 767, 760, 783, 811, 751, 775, 771, 754, 746, 728, 802, 753, 799, 765, 766] payload_rows_per_dst=[9142, 9238, 9354, 9370, 9248, 9312, 9386, 9328] dup_rows_per_dst=[2992, 3114, 3062, 2882, 3006, 3016, 2956, 2898] combine_rows_per_src=[9142, 9238, 9354, 9370, 9248, 9312, 9386, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74362 payload_rows_remote=64962 dup_rows=23942 dup_rows_remote=20872 dedup_ratio=0.243551 router_tokens_per_expert=[771, 755, 790, 720, 826, 752, 784, 763, 799, 781, 795, 766, 773, 788, 759, 772, 750, 781, 827, 724, 778, 847, 787, 820, 749, 781, 785, 775, 774, 780, 713, 771, 743, 736, 743, 735, 769, 785, 799, 777, 780, 768, 789, 738, 709, 755, 705, 752, 745, 763, 759, 763, 767, 825, 806, 766, 755, 782, 737, 764, 757, 750, 756, 738] payload_rows_per_dst=[9098, 9210, 9398, 9454, 9302, 9286, 9214, 9400] dup_rows_per_dst=[2940, 2916, 3050, 3134, 3052, 2990, 2790, 3070] combine_rows_per_src=[9098, 9210, 9398, 9454, 9302, 9286, 9214, 9400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=65016 dup_rows=23974 dup_rows_remote=20872 dedup_ratio=0.243876 router_tokens_per_expert=[770, 750, 736, 768, 755, 766, 765, 750, 758, 770, 734, 832, 732, 819, 757, 740, 748, 747, 733, 795, 768, 743, 773, 748, 742, 769, 797, 800, 731, 784, 734, 770, 745, 790, 757, 802, 765, 765, 775, 797, 783, 762, 739, 806, 781, 779, 788, 770, 755, 771, 823, 762, 794, 775, 733, 808, 754, 741, 782, 776, 747, 811, 793, 739] payload_rows_per_dst=[9242, 9346, 9252, 9322, 9336, 9314, 9268, 9250] dup_rows_per_dst=[2878, 2938, 2858, 2932, 3056, 3102, 3174, 3036] combine_rows_per_src=[9242, 9346, 9252, 9322, 9336, 9314, 9268, 9250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74372 payload_rows_remote=65054 dup_rows=23932 dup_rows_remote=20958 dedup_ratio=0.243449 router_tokens_per_expert=[787, 752, 776, 802, 752, 765, 754, 795, 772, 766, 810, 746, 770, 762, 756, 826, 728, 778, 776, 731, 764, 765, 818, 789, 799, 779, 792, 768, 754, 767, 735, 780, 728, 785, 796, 715, 747, 755, 767, 789, 769, 817, 741, 741, 763, 792, 743, 768, 785, 805, 726, 793, 789, 795, 689, 764, 786, 751, 760, 750, 770, 734, 775, 750] payload_rows_per_dst=[9338, 9422, 9300, 9290, 9214, 9268, 9318, 9222] dup_rows_per_dst=[3028, 2994, 2998, 3058, 2950, 3000, 2974, 2930] combine_rows_per_src=[9338, 9422, 9300, 9290, 9214, 9268, 9318, 9222] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266125312 dispatch_meta_bytes=344328 combine_payload_bytes=266125312 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266469640 ingress_bytes=335331328 link_bytes=335331328 token_bytes=532594952 weight_movement_bytes=69206016 total_nvlink_bytes=601800968 weight_to_token_ratio=0.129941 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261824791e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=267460608 dispatch_meta_bytes=344520 combine_payload_bytes=267460608 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267805128 ingress_bytes=371269632 link_bytes=371269632 token_bytes=535265736 weight_movement_bytes=103809024 total_nvlink_bytes=639074760 weight_to_token_ratio=0.193939 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397057736e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74224 payload_rows_remote=65016 dup_rows=24080 dup_rows_remote=21076 dedup_ratio=0.244954 router_tokens_per_expert=[833, 801, 791, 778, 769, 738, 789, 773, 777, 802, 808, 755, 773, 830, 780, 750, 758, 768, 772, 768, 763, 760, 732, 785, 759, 726, 772, 742, 761, 766, 800, 755, 796, 802, 716, 718, 791, 741, 733, 808, 784, 734, 780, 760, 780, 764, 804, 758, 787, 760, 752, 732, 772, 747, 778, 734, 768, 739, 736, 820, 709, 774, 762, 779] payload_rows_per_dst=[9404, 9470, 9208, 9198, 9252, 9240, 9170, 9282] dup_rows_per_dst=[3140, 3080, 3004, 2964, 2958, 3088, 2954, 2892] combine_rows_per_src=[9404, 9470, 9208, 9198, 9252, 9240, 9170, 9282] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266084352 dispatch_meta_bytes=343336 combine_payload_bytes=266084352 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266427688 ingress_bytes=300687360 link_bytes=300687360 token_bytes=532512040 weight_movement_bytes=34603008 total_nvlink_bytes=567115048 weight_to_token_ratio=0.064981 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131462328e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74146 payload_rows_remote=65004 dup_rows=24158 dup_rows_remote=21158 dedup_ratio=0.245748 router_tokens_per_expert=[746, 752, 808, 785, 817, 788, 770, 797, 787, 742, 750, 778, 750, 811, 765, 746, 754, 751, 816, 746, 764, 725, 738, 755, 729, 776, 750, 796, 799, 797, 783, 754, 762, 739, 713, 764, 782, 774, 763, 774, 763, 747, 767, 731, 718, 763, 787, 760, 800, 821, 796, 792, 731, 792, 714, 792, 793, 799, 770, 805, 760, 750, 767, 738] payload_rows_per_dst=[9438, 9234, 9162, 9298, 9142, 9136, 9392, 9344] dup_rows_per_dst=[3088, 3024, 2936, 3070, 3000, 2936, 3084, 3020] combine_rows_per_src=[9438, 9234, 9162, 9298, 9142, 9136, 9392, 9344] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=343552 combine_payload_bytes=266305536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266649088 ingress_bytes=300908544 link_bytes=300908544 token_bytes=532954624 weight_movement_bytes=34603008 total_nvlink_bytes=567557632 weight_to_token_ratio=0.064927 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132294626e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=65008 dup_rows=23926 dup_rows_remote=21044 dedup_ratio=0.243388 router_tokens_per_expert=[741, 800, 750, 824, 716, 755, 761, 720, 764, 761, 734, 791, 786, 765, 805, 770, 759, 808, 780, 755, 769, 711, 798, 828, 787, 772, 786, 767, 763, 746, 735, 770, 695, 824, 755, 766, 786, 738, 792, 771, 749, 804, 725, 815, 780, 769, 781, 741, 753, 767, 760, 783, 811, 751, 775, 771, 754, 746, 728, 802, 753, 799, 765, 766] payload_rows_per_dst=[9142, 9238, 9354, 9370, 9248, 9312, 9386, 9328] dup_rows_per_dst=[2992, 3114, 3062, 2882, 3006, 3016, 2956, 2898] combine_rows_per_src=[9142, 9238, 9354, 9370, 9248, 9312, 9386, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=344048 combine_payload_bytes=266461184 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266805232 ingress_bytes=370270208 link_bytes=370270208 token_bytes=533266416 weight_movement_bytes=103809024 total_nvlink_bytes=637075440 weight_to_token_ratio=0.194666 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393296984e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74034 payload_rows_remote=64808 dup_rows=24270 dup_rows_remote=21076 dedup_ratio=0.246887 router_tokens_per_expert=[758, 794, 742, 762, 729, 774, 817, 796, 799, 755, 797, 786, 782, 728, 792, 771, 795, 757, 752, 740, 778, 804, 803, 780, 815, 766, 716, 743, 703, 755, 796, 771, 774, 794, 785, 749, 768, 769, 785, 739, 746, 783, 799, 761, 778, 785, 732, 730, 751, 790, 729, 760, 724, 756, 759, 803, 771, 818, 782, 736, 741, 765, 800, 734] payload_rows_per_dst=[9268, 9226, 9374, 9192, 9342, 9114, 9250, 9268] dup_rows_per_dst=[3076, 3194, 3044, 2938, 2984, 3114, 2894, 3026] combine_rows_per_src=[9268, 9226, 9374, 9192, 9342, 9114, 9250, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=344368 combine_payload_bytes=266305536 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266649904 ingress_bytes=335511552 link_bytes=335511552 token_bytes=532955440 weight_movement_bytes=69206016 total_nvlink_bytes=602161456 weight_to_token_ratio=0.129853 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262502959e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265043968 dispatch_meta_bytes=343752 combine_payload_bytes=265043968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265387720 ingress_bytes=299646976 link_bytes=299646976 token_bytes=530431688 weight_movement_bytes=34603008 total_nvlink_bytes=565034696 weight_to_token_ratio=0.065236 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127547447e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266256384 dispatch_meta_bytes=344648 combine_payload_bytes=266256384 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266601032 ingress_bytes=300859392 link_bytes=300859392 token_bytes=532857416 weight_movement_bytes=34603008 total_nvlink_bytes=567460424 weight_to_token_ratio=0.064939 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132109671e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266272768 dispatch_meta_bytes=344208 combine_payload_bytes=266272768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266616976 ingress_bytes=300875776 link_bytes=300875776 token_bytes=532889744 weight_movement_bytes=34603008 total_nvlink_bytes=567492752 weight_to_token_ratio=0.064935 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132171323e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74138 payload_rows_remote=64944 dup_rows=24166 dup_rows_remote=21158 dedup_ratio=0.245829 router_tokens_per_expert=[759, 750, 760, 793, 766, 790, 750, 750, 831, 743, 796, 764, 786, 752, 741, 748, 775, 774, 767, 760, 753, 729, 770, 773, 755, 772, 758, 811, 764, 763, 827, 772, 794, 774, 781, 739, 782, 783, 770, 765, 749, 767, 737, 797, 777, 783, 778, 765, 793, 788, 762, 781, 715, 752, 757, 772, 755, 748, 776, 794, 753, 803, 708, 752] payload_rows_per_dst=[9190, 9314, 9194, 9410, 9272, 9278, 9270, 9210] dup_rows_per_dst=[3046, 3008, 3008, 3034, 3104, 3028, 2970, 2968] combine_rows_per_src=[9190, 9314, 9194, 9410, 9272, 9278, 9270, 9210] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74308 payload_rows_remote=65032 dup_rows=23996 dup_rows_remote=21142 dedup_ratio=0.244100 router_tokens_per_expert=[765, 754, 773, 744, 741, 806, 779, 766, 817, 774, 730, 775, 765, 787, 756, 746, 779, 726, 728, 735, 786, 769, 778, 752, 750, 777, 750, 770, 806, 785, 801, 807, 745, 717, 752, 816, 745, 717, 791, 782, 773, 684, 839, 805, 750, 769, 765, 743, 757, 765, 734, 777, 755, 776, 809, 764, 739, 790, 771, 768, 806, 816, 792, 763] payload_rows_per_dst=[9254, 9252, 9208, 9364, 9276, 9266, 9264, 9424] dup_rows_per_dst=[3002, 3048, 2898, 3128, 2854, 2990, 3010, 3066] combine_rows_per_src=[9254, 9252, 9208, 9364, 9276, 9266, 9264, 9424] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74372 payload_rows_remote=65054 dup_rows=23932 dup_rows_remote=20958 dedup_ratio=0.243449 router_tokens_per_expert=[787, 752, 776, 802, 752, 765, 754, 795, 772, 766, 810, 746, 770, 762, 756, 826, 728, 778, 776, 731, 764, 765, 818, 789, 799, 779, 792, 768, 754, 767, 735, 780, 728, 785, 796, 715, 747, 755, 767, 789, 769, 817, 741, 741, 763, 792, 743, 768, 785, 805, 726, 793, 789, 795, 689, 764, 786, 751, 760, 750, 770, 734, 775, 750] payload_rows_per_dst=[9338, 9422, 9300, 9290, 9214, 9268, 9318, 9222] dup_rows_per_dst=[3028, 2994, 2998, 3058, 2950, 3000, 2974, 2930] combine_rows_per_src=[9338, 9422, 9300, 9290, 9214, 9268, 9318, 9222] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266010624 dispatch_meta_bytes=344408 combine_payload_bytes=266010624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266355032 ingress_bytes=300613632 link_bytes=300613632 token_bytes=532365656 weight_movement_bytes=34603008 total_nvlink_bytes=566968664 weight_to_token_ratio=0.064999 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131184896e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74200 payload_rows_remote=64890 dup_rows=24104 dup_rows_remote=21162 dedup_ratio=0.245199 router_tokens_per_expert=[739, 786, 747, 729, 751, 779, 799, 760, 728, 777, 820, 760, 760, 771, 745, 740, 716, 770, 729, 792, 732, 792, 815, 794, 759, 771, 750, 739, 797, 756, 776, 758, 784, 791, 778, 740, 747, 761, 777, 793, 735, 776, 790, 818, 764, 771, 777, 776, 772, 764, 821, 770, 740, 767, 738, 804, 774, 765, 771, 767, 796, 785, 735, 768] payload_rows_per_dst=[9310, 9168, 9268, 9250, 9296, 9316, 9282, 9310] dup_rows_per_dst=[2942, 2962, 3054, 2962, 3046, 3098, 3028, 3012] combine_rows_per_src=[9310, 9168, 9268, 9250, 9296, 9316, 9282, 9310] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74034 payload_rows_remote=64808 dup_rows=24270 dup_rows_remote=21076 dedup_ratio=0.246887 router_tokens_per_expert=[758, 794, 742, 762, 729, 774, 817, 796, 799, 755, 797, 786, 782, 728, 792, 771, 795, 757, 752, 740, 778, 804, 803, 780, 815, 766, 716, 743, 703, 755, 796, 771, 774, 794, 785, 749, 768, 769, 785, 739, 746, 783, 799, 761, 778, 785, 732, 730, 751, 790, 729, 760, 724, 756, 759, 803, 771, 818, 782, 736, 741, 765, 800, 734] payload_rows_per_dst=[9268, 9226, 9374, 9192, 9342, 9114, 9250, 9268] dup_rows_per_dst=[3076, 3194, 3044, 2938, 2984, 3114, 2894, 3026] combine_rows_per_src=[9268, 9226, 9374, 9192, 9342, 9114, 9250, 9268] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74506 payload_rows_remote=65180 dup_rows=23798 dup_rows_remote=20758 dedup_ratio=0.242086 router_tokens_per_expert=[761, 760, 750, 775, 797, 748, 772, 796, 792, 783, 738, 739, 798, 791, 779, 776, 754, 764, 758, 807, 750, 776, 725, 753, 736, 818, 736, 750, 752, 778, 784, 809, 758, 765, 749, 785, 734, 750, 775, 742, 788, 813, 716, 789, 786, 791, 758, 742, 759, 737, 781, 815, 779, 730, 770, 782, 753, 763, 797, 777, 723, 775, 801, 764] payload_rows_per_dst=[9318, 9392, 9236, 9302, 9288, 9326, 9336, 9308] dup_rows_per_dst=[3000, 3000, 2938, 3024, 2828, 3040, 2970, 2998] combine_rows_per_src=[9318, 9392, 9236, 9302, 9288, 9326, 9336, 9308] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344696 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266715768 ingress_bytes=300974080 link_bytes=300974080 token_bytes=533086840 weight_movement_bytes=34603008 total_nvlink_bytes=567689848 weight_to_token_ratio=0.064911 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132541233e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265453568 dispatch_meta_bytes=343536 combine_payload_bytes=265453568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265797104 ingress_bytes=334659584 link_bytes=334659584 token_bytes=531250672 weight_movement_bytes=69206016 total_nvlink_bytes=600456688 weight_to_token_ratio=0.130270 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259297072e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73970 payload_rows_remote=64708 dup_rows=24334 dup_rows_remote=21230 dedup_ratio=0.247538 router_tokens_per_expert=[733, 750, 749, 793, 853, 730, 770, 803, 761, 748, 771, 746, 742, 759, 758, 791, 768, 754, 772, 749, 800, 765, 791, 796, 806, 729, 766, 784, 734, 795, 772, 775, 766, 759, 725, 787, 760, 717, 791, 770, 764, 809, 746, 788, 760, 817, 786, 797, 808, 771, 790, 757, 745, 771, 758, 733, 728, 774, 757, 804, 728, 760, 786, 727] payload_rows_per_dst=[9222, 8992, 9482, 9242, 9202, 9324, 9244, 9262] dup_rows_per_dst=[3044, 2894, 3174, 3078, 3010, 3006, 3024, 3104] combine_rows_per_src=[9222, 8992, 9482, 9242, 9202, 9324, 9244, 9262] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=64812 dup_rows=24072 dup_rows_remote=20946 dedup_ratio=0.244873 router_tokens_per_expert=[757, 766, 794, 736, 764, 766, 779, 830, 820, 746, 787, 757, 756, 772, 756, 722, 721, 749, 779, 785, 804, 802, 759, 817, 747, 779, 752, 741, 785, 783, 794, 743, 775, 787, 762, 761, 766, 754, 738, 773, 773, 766, 833, 807, 802, 728, 728, 728, 792, 761, 767, 771, 749, 770, 788, 732, 801, 786, 738, 730, 769, 757, 744, 768] payload_rows_per_dst=[8710, 9410, 9382, 9330, 9464, 9322, 9194, 9420] dup_rows_per_dst=[2590, 3140, 3050, 2918, 3174, 3008, 3066, 3126] combine_rows_per_src=[8710, 9410, 9382, 9330, 9464, 9322, 9194, 9420] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=344048 combine_payload_bytes=266461184 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266805232 ingress_bytes=370270208 link_bytes=370270208 token_bytes=533266416 weight_movement_bytes=103809024 total_nvlink_bytes=637075440 weight_to_token_ratio=0.194666 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393296984e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74506 payload_rows_remote=65180 dup_rows=23798 dup_rows_remote=20758 dedup_ratio=0.242086 router_tokens_per_expert=[761, 760, 750, 775, 797, 748, 772, 796, 792, 783, 738, 739, 798, 791, 779, 776, 754, 764, 758, 807, 750, 776, 725, 753, 736, 818, 736, 750, 752, 778, 784, 809, 758, 765, 749, 785, 734, 750, 775, 742, 788, 813, 716, 789, 786, 791, 758, 742, 759, 737, 781, 815, 779, 730, 770, 782, 753, 763, 797, 777, 723, 775, 801, 764] payload_rows_per_dst=[9318, 9392, 9236, 9302, 9288, 9326, 9336, 9308] dup_rows_per_dst=[3000, 3000, 2938, 3024, 2828, 3040, 2970, 2998] combine_rows_per_src=[9318, 9392, 9236, 9302, 9288, 9326, 9336, 9308] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74550 payload_rows_remote=65106 dup_rows=23754 dup_rows_remote=20704 dedup_ratio=0.241638 router_tokens_per_expert=[763, 735, 762, 781, 790, 748, 760, 732, 767, 746, 762, 772, 746, 765, 740, 741, 746, 734, 811, 802, 760, 762, 781, 770, 726, 796, 737, 790, 786, 707, 778, 773, 792, 745, 754, 779, 841, 817, 790, 761, 823, 767, 772, 783, 759, 754, 754, 728, 770, 803, 767, 772, 826, 759, 764, 786, 792, 768, 733, 769, 775, 780, 744, 756] payload_rows_per_dst=[9232, 9168, 9376, 9340, 9440, 9290, 9444, 9260] dup_rows_per_dst=[2910, 2910, 2956, 2846, 3118, 2990, 3050, 2974] combine_rows_per_src=[9232, 9168, 9376, 9340, 9440, 9290, 9444, 9260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265789440 dispatch_meta_bytes=344208 combine_payload_bytes=265789440 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266133648 ingress_bytes=334995456 link_bytes=334995456 token_bytes=531923088 weight_movement_bytes=69206016 total_nvlink_bytes=601129104 weight_to_token_ratio=0.130105 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260560932e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74138 payload_rows_remote=64944 dup_rows=24166 dup_rows_remote=21158 dedup_ratio=0.245829 router_tokens_per_expert=[759, 750, 760, 793, 766, 790, 750, 750, 831, 743, 796, 764, 786, 752, 741, 748, 775, 774, 767, 760, 753, 729, 770, 773, 755, 772, 758, 811, 764, 763, 827, 772, 794, 774, 781, 739, 782, 783, 770, 765, 749, 767, 737, 797, 777, 783, 778, 765, 793, 788, 762, 781, 715, 752, 757, 772, 755, 748, 776, 794, 753, 803, 708, 752] payload_rows_per_dst=[9190, 9314, 9194, 9410, 9272, 9278, 9270, 9210] dup_rows_per_dst=[3046, 3008, 3008, 3034, 3104, 3028, 2970, 2968] combine_rows_per_src=[9190, 9314, 9194, 9410, 9272, 9278, 9270, 9210] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265043968 dispatch_meta_bytes=343752 combine_payload_bytes=265043968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265387720 ingress_bytes=299646976 link_bytes=299646976 token_bytes=530431688 weight_movement_bytes=34603008 total_nvlink_bytes=565034696 weight_to_token_ratio=0.065236 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127547447e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74280 payload_rows_remote=64970 dup_rows=24024 dup_rows_remote=21056 dedup_ratio=0.244385 router_tokens_per_expert=[770, 755, 740, 776, 814, 791, 805, 785, 759, 714, 771, 780, 766, 780, 785, 781, 765, 759, 780, 731, 775, 734, 748, 772, 762, 758, 847, 736, 768, 754, 734, 762, 767, 722, 826, 721, 775, 763, 754, 811, 746, 725, 791, 800, 766, 763, 786, 799, 720, 759, 846, 768, 717, 770, 797, 778, 755, 779, 772, 797, 747, 783, 763, 729] payload_rows_per_dst=[9358, 9276, 9122, 9284, 9310, 9312, 9316, 9302] dup_rows_per_dst=[3114, 2996, 3006, 2958, 2968, 3040, 2994, 2948] combine_rows_per_src=[9358, 9276, 9122, 9284, 9310, 9312, 9316, 9302] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74648 payload_rows_remote=65446 dup_rows=23656 dup_rows_remote=20740 dedup_ratio=0.240641 router_tokens_per_expert=[759, 797, 751, 776, 790, 751, 788, 789, 773, 767, 764, 787, 813, 756, 797, 750, 773, 744, 793, 729, 734, 803, 744, 746, 800, 773, 754, 738, 772, 757, 728, 737, 817, 753, 734, 716, 794, 769, 777, 779, 788, 778, 763, 785, 769, 771, 743, 749, 737, 740, 772, 773, 777, 781, 795, 807, 735, 771, 760, 785, 772, 778, 758, 793] payload_rows_per_dst=[9340, 9466, 9266, 9202, 9342, 9288, 9390, 9354] dup_rows_per_dst=[3062, 2948, 2866, 2916, 2936, 3004, 2974, 2950] combine_rows_per_src=[9340, 9466, 9266, 9202, 9342, 9288, 9390, 9354] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265789440 dispatch_meta_bytes=344208 combine_payload_bytes=265789440 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266133648 ingress_bytes=334995456 link_bytes=334995456 token_bytes=531923088 weight_movement_bytes=69206016 total_nvlink_bytes=601129104 weight_to_token_ratio=0.130105 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260560932e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265453568 dispatch_meta_bytes=343536 combine_payload_bytes=265453568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265797104 ingress_bytes=334659584 link_bytes=334659584 token_bytes=531250672 weight_movement_bytes=69206016 total_nvlink_bytes=600456688 weight_to_token_ratio=0.130270 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259297072e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266977280 dispatch_meta_bytes=343752 combine_payload_bytes=266977280 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267321032 ingress_bytes=336183296 link_bytes=336183296 token_bytes=534298312 weight_movement_bytes=69206016 total_nvlink_bytes=603504328 weight_to_token_ratio=0.129527 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265030678e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266674176 dispatch_meta_bytes=343240 combine_payload_bytes=266674176 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267017416 ingress_bytes=370483200 link_bytes=370483200 token_bytes=533691592 weight_movement_bytes=103809024 total_nvlink_bytes=637500616 weight_to_token_ratio=0.194511 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394098455e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74018 payload_rows_remote=64862 dup_rows=24286 dup_rows_remote=21358 dedup_ratio=0.247050 router_tokens_per_expert=[760, 730, 827, 817, 694, 745, 787, 818, 747, 771, 715, 801, 753, 709, 759, 787, 755, 759, 782, 780, 768, 765, 744, 727, 811, 783, 771, 822, 760, 818, 779, 763, 764, 782, 782, 794, 742, 721, 809, 772, 761, 809, 762, 776, 742, 752, 774, 772, 816, 763, 737, 749, 782, 822, 756, 751, 765, 761, 734, 758, 762, 733, 788, 754] payload_rows_per_dst=[9322, 9156, 9158, 9458, 9212, 9238, 9308, 9166] dup_rows_per_dst=[3034, 2928, 3002, 3156, 3120, 3058, 3044, 2944] combine_rows_per_src=[9322, 9156, 9158, 9458, 9212, 9238, 9308, 9166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266010624 dispatch_meta_bytes=344408 combine_payload_bytes=266010624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266355032 ingress_bytes=300613632 link_bytes=300613632 token_bytes=532365656 weight_movement_bytes=34603008 total_nvlink_bytes=566968664 weight_to_token_ratio=0.064999 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131184896e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265469952 dispatch_meta_bytes=343032 combine_payload_bytes=265469952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265812984 ingress_bytes=300072960 link_bytes=300072960 token_bytes=531282936 weight_movement_bytes=34603008 total_nvlink_bytes=565885944 weight_to_token_ratio=0.065131 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129150391e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=268066816 dispatch_meta_bytes=344744 combine_payload_bytes=268066816 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=268411560 ingress_bytes=302669824 link_bytes=302669824 token_bytes=536478376 weight_movement_bytes=34603008 total_nvlink_bytes=571081384 weight_to_token_ratio=0.064500 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138922181e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74518 payload_rows_remote=65272 dup_rows=23786 dup_rows_remote=20840 dedup_ratio=0.241964 router_tokens_per_expert=[774, 760, 792, 759, 825, 771, 788, 797, 758, 768, 767, 796, 832, 778, 736, 786, 768, 743, 791, 727, 763, 790, 752, 762, 772, 759, 765, 842, 695, 736, 784, 756, 726, 774, 731, 770, 764, 760, 771, 798, 739, 781, 805, 727, 723, 732, 768, 761, 784, 800, 770, 776, 752, 799, 754, 776, 806, 728, 790, 803, 790, 766, 683, 753] payload_rows_per_dst=[9422, 9400, 9246, 9354, 9234, 9146, 9412, 9304] dup_rows_per_dst=[3110, 3042, 2946, 2864, 2954, 2926, 3010, 2934] combine_rows_per_src=[9422, 9400, 9246, 9354, 9234, 9146, 9412, 9304] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74418 payload_rows_remote=65132 dup_rows=23886 dup_rows_remote=20886 dedup_ratio=0.242981 router_tokens_per_expert=[740, 782, 760, 755, 813, 809, 790, 758, 746, 738, 768, 726, 819, 755, 742, 786, 763, 736, 772, 783, 793, 811, 773, 793, 750, 797, 722, 758, 794, 736, 758, 746, 761, 777, 755, 774, 805, 757, 755, 759, 813, 728, 781, 753, 750, 816, 752, 833, 800, 729, 796, 747, 795, 733, 815, 765, 750, 793, 749, 770, 743, 692, 744, 790] payload_rows_per_dst=[9340, 9282, 9446, 9154, 9286, 9406, 9326, 9178] dup_rows_per_dst=[3074, 2878, 3002, 2968, 3000, 3046, 3034, 2884] combine_rows_per_src=[9340, 9282, 9446, 9154, 9286, 9406, 9326, 9178] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74550 payload_rows_remote=65106 dup_rows=23754 dup_rows_remote=20704 dedup_ratio=0.241638 router_tokens_per_expert=[763, 735, 762, 781, 790, 748, 760, 732, 767, 746, 762, 772, 746, 765, 740, 741, 746, 734, 811, 802, 760, 762, 781, 770, 726, 796, 737, 790, 786, 707, 778, 773, 792, 745, 754, 779, 841, 817, 790, 761, 823, 767, 772, 783, 759, 754, 754, 728, 770, 803, 767, 772, 826, 759, 764, 786, 792, 768, 733, 769, 775, 780, 744, 756] payload_rows_per_dst=[9232, 9168, 9376, 9340, 9440, 9290, 9444, 9260] dup_rows_per_dst=[2910, 2910, 2956, 2846, 3118, 2990, 3050, 2974] combine_rows_per_src=[9232, 9168, 9376, 9340, 9440, 9290, 9444, 9260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266117120 dispatch_meta_bytes=344104 combine_payload_bytes=266117120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266461224 ingress_bytes=300720128 link_bytes=300720128 token_bytes=532578344 weight_movement_bytes=34603008 total_nvlink_bytes=567181352 weight_to_token_ratio=0.064973 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131585632e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74524 payload_rows_remote=65146 dup_rows=23780 dup_rows_remote=20818 dedup_ratio=0.241903 router_tokens_per_expert=[781, 780, 781, 780, 797, 760, 710, 774, 753, 766, 772, 795, 828, 751, 775, 787, 725, 773, 744, 784, 773, 794, 756, 793, 756, 754, 749, 743, 779, 781, 788, 794, 718, 804, 787, 785, 763, 753, 794, 750, 765, 794, 734, 791, 798, 717, 768, 756, 777, 765, 781, 739, 714, 843, 775, 739, 798, 726, 746, 743, 717, 788, 793, 755] payload_rows_per_dst=[9378, 9424, 9348, 9308, 9322, 9310, 9284, 9150] dup_rows_per_dst=[2962, 3068, 2936, 2980, 2986, 2936, 2930, 2982] combine_rows_per_src=[9378, 9424, 9348, 9308, 9322, 9310, 9284, 9150] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267354112 dispatch_meta_bytes=344448 combine_payload_bytes=267354112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267698560 ingress_bytes=301957120 link_bytes=301957120 token_bytes=535052672 weight_movement_bytes=34603008 total_nvlink_bytes=569655680 weight_to_token_ratio=0.064672 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136240333e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74018 payload_rows_remote=64862 dup_rows=24286 dup_rows_remote=21358 dedup_ratio=0.247050 router_tokens_per_expert=[760, 730, 827, 817, 694, 745, 787, 818, 747, 771, 715, 801, 753, 709, 759, 787, 755, 759, 782, 780, 768, 765, 744, 727, 811, 783, 771, 822, 760, 818, 779, 763, 764, 782, 782, 794, 742, 721, 809, 772, 761, 809, 762, 776, 742, 752, 774, 772, 816, 763, 737, 749, 782, 822, 756, 751, 765, 761, 734, 758, 762, 733, 788, 754] payload_rows_per_dst=[9322, 9156, 9158, 9458, 9212, 9238, 9308, 9166] dup_rows_per_dst=[3034, 2928, 3002, 3156, 3120, 3058, 3044, 2944] combine_rows_per_src=[9322, 9156, 9158, 9458, 9212, 9238, 9308, 9166] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266780672 dispatch_meta_bytes=344072 combine_payload_bytes=266780672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267124744 ingress_bytes=301383680 link_bytes=301383680 token_bytes=533905416 weight_movement_bytes=34603008 total_nvlink_bytes=568508424 weight_to_token_ratio=0.064811 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134082524e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74280 payload_rows_remote=65088 dup_rows=24024 dup_rows_remote=21064 dedup_ratio=0.244385 router_tokens_per_expert=[768, 761, 760, 754, 775, 775, 752, 797, 797, 756, 772, 774, 736, 760, 755, 816, 777, 750, 697, 795, 807, 781, 800, 754, 778, 764, 765, 751, 701, 752, 745, 776, 766, 746, 756, 737, 782, 831, 727, 742, 761, 750, 785, 756, 765, 766, 759, 734, 783, 787, 816, 754, 814, 783, 770, 780, 745, 796, 767, 763, 780, 745, 790, 815] payload_rows_per_dst=[9232, 9354, 9310, 9144, 9306, 9192, 9424, 9318] dup_rows_per_dst=[3052, 2978, 3012, 2920, 2868, 2960, 3150, 3084] combine_rows_per_src=[9232, 9354, 9310, 9144, 9306, 9192, 9424, 9318] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=64654 dup_rows=24132 dup_rows_remote=21028 dedup_ratio=0.245483 router_tokens_per_expert=[791, 794, 746, 812, 752, 740, 769, 753, 791, 782, 814, 761, 769, 754, 703, 815, 759, 770, 763, 766, 789, 786, 772, 810, 726, 798, 788, 791, 793, 747, 756, 772, 747, 773, 723, 769, 791, 754, 746, 784, 775, 750, 758, 773, 749, 733, 752, 782, 805, 791, 725, 779, 771, 786, 812, 731, 810, 775, 755, 705, 749, 759, 757, 751] payload_rows_per_dst=[8952, 8864, 9224, 9636, 9184, 9466, 9328, 9518] dup_rows_per_dst=[2792, 2726, 2980, 3296, 2958, 3172, 3104, 3104] combine_rows_per_src=[8952, 8864, 9224, 9636, 9184, 9466, 9328, 9518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265674752 dispatch_meta_bytes=344880 combine_payload_bytes=265674752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266019632 ingress_bytes=300277760 link_bytes=300277760 token_bytes=531694384 weight_movement_bytes=34603008 total_nvlink_bytes=566297392 weight_to_token_ratio=0.065081 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129921037e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=64812 dup_rows=24072 dup_rows_remote=20946 dedup_ratio=0.244873 router_tokens_per_expert=[757, 766, 794, 736, 764, 766, 779, 830, 820, 746, 787, 757, 756, 772, 756, 722, 721, 749, 779, 785, 804, 802, 759, 817, 747, 779, 752, 741, 785, 783, 794, 743, 775, 787, 762, 761, 766, 754, 738, 773, 773, 766, 833, 807, 802, 728, 728, 728, 792, 761, 767, 771, 749, 770, 788, 732, 801, 786, 738, 730, 769, 757, 744, 768] payload_rows_per_dst=[8710, 9410, 9382, 9330, 9464, 9322, 9194, 9420] dup_rows_per_dst=[2590, 3140, 3050, 2918, 3174, 3008, 3066, 3126] combine_rows_per_src=[8710, 9410, 9382, 9330, 9464, 9322, 9194, 9420] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65094 dup_rows=23924 dup_rows_remote=20990 dedup_ratio=0.243368 router_tokens_per_expert=[759, 793, 793, 770, 803, 801, 746, 755, 749, 801, 737, 777, 740, 763, 794, 758, 795, 743, 742, 751, 760, 752, 747, 750, 735, 770, 742, 800, 776, 778, 753, 756, 772, 800, 797, 797, 737, 757, 790, 742, 769, 753, 817, 781, 787, 772, 771, 780, 778, 737, 763, 761, 795, 733, 724, 764, 735, 790, 812, 798, 755, 734, 759, 803] payload_rows_per_dst=[9370, 9270, 9212, 9286, 9326, 9340, 9208, 9368] dup_rows_per_dst=[3070, 2968, 2868, 2934, 3058, 3120, 2902, 3004] combine_rows_per_src=[9370, 9270, 9212, 9286, 9326, 9340, 9208, 9368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266674176 dispatch_meta_bytes=343240 combine_payload_bytes=266674176 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267017416 ingress_bytes=370483200 link_bytes=370483200 token_bytes=533691592 weight_movement_bytes=103809024 total_nvlink_bytes=637500616 weight_to_token_ratio=0.194511 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394098455e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74280 payload_rows_remote=65088 dup_rows=24024 dup_rows_remote=21064 dedup_ratio=0.244385 router_tokens_per_expert=[768, 761, 760, 754, 775, 775, 752, 797, 797, 756, 772, 774, 736, 760, 755, 816, 777, 750, 697, 795, 807, 781, 800, 754, 778, 764, 765, 751, 701, 752, 745, 776, 766, 746, 756, 737, 782, 831, 727, 742, 761, 750, 785, 756, 765, 766, 759, 734, 783, 787, 816, 754, 814, 783, 770, 780, 745, 796, 767, 763, 780, 745, 790, 815] payload_rows_per_dst=[9232, 9354, 9310, 9144, 9306, 9192, 9424, 9318] dup_rows_per_dst=[3052, 2978, 3012, 2920, 2868, 2960, 3150, 3084] combine_rows_per_src=[9232, 9354, 9310, 9144, 9306, 9192, 9424, 9318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266838016 dispatch_meta_bytes=343856 combine_payload_bytes=266838016 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267181872 ingress_bytes=370647040 link_bytes=370647040 token_bytes=534019888 weight_movement_bytes=103809024 total_nvlink_bytes=637828912 weight_to_token_ratio=0.194392 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394714972e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265674752 dispatch_meta_bytes=344880 combine_payload_bytes=265674752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266019632 ingress_bytes=300277760 link_bytes=300277760 token_bytes=531694384 weight_movement_bytes=34603008 total_nvlink_bytes=566297392 weight_to_token_ratio=0.065081 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129921037e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=65030 dup_rows=24056 dup_rows_remote=21114 dedup_ratio=0.244710 router_tokens_per_expert=[771, 744, 798, 740, 794, 764, 780, 798, 814, 783, 753, 777, 750, 750, 772, 753, 739, 773, 779, 724, 786, 778, 784, 781, 772, 761, 789, 802, 824, 744, 780, 749, 771, 723, 748, 748, 759, 745, 750, 739, 747, 796, 747, 797, 787, 792, 747, 796, 761, 762, 779, 782, 786, 747, 709, 754, 757, 772, 794, 785, 812, 730, 769, 755] payload_rows_per_dst=[9338, 9278, 9282, 9364, 9088, 9326, 9218, 9354] dup_rows_per_dst=[3040, 3026, 3006, 3078, 2878, 3092, 2942, 2994] combine_rows_per_src=[9338, 9278, 9282, 9364, 9088, 9326, 9218, 9354] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74518 payload_rows_remote=65272 dup_rows=23786 dup_rows_remote=20840 dedup_ratio=0.241964 router_tokens_per_expert=[774, 760, 792, 759, 825, 771, 788, 797, 758, 768, 767, 796, 832, 778, 736, 786, 768, 743, 791, 727, 763, 790, 752, 762, 772, 759, 765, 842, 695, 736, 784, 756, 726, 774, 731, 770, 764, 760, 771, 798, 739, 781, 805, 727, 723, 732, 768, 761, 784, 800, 770, 776, 752, 799, 754, 776, 806, 728, 790, 803, 790, 766, 683, 753] payload_rows_per_dst=[9422, 9400, 9246, 9354, 9234, 9146, 9412, 9304] dup_rows_per_dst=[3110, 3042, 2946, 2864, 2954, 2926, 3010, 2934] combine_rows_per_src=[9422, 9400, 9246, 9354, 9234, 9146, 9412, 9304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265469952 dispatch_meta_bytes=343032 combine_payload_bytes=265469952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265812984 ingress_bytes=300072960 link_bytes=300072960 token_bytes=531282936 weight_movement_bytes=34603008 total_nvlink_bytes=565885944 weight_to_token_ratio=0.065131 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129150391e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74308 payload_rows_remote=65032 dup_rows=23996 dup_rows_remote=21142 dedup_ratio=0.244100 router_tokens_per_expert=[765, 754, 773, 744, 741, 806, 779, 766, 817, 774, 730, 775, 765, 787, 756, 746, 779, 726, 728, 735, 786, 769, 778, 752, 750, 777, 750, 770, 806, 785, 801, 807, 745, 717, 752, 816, 745, 717, 791, 782, 773, 684, 839, 805, 750, 769, 765, 743, 757, 765, 734, 777, 755, 776, 809, 764, 739, 790, 771, 768, 806, 816, 792, 763] payload_rows_per_dst=[9254, 9252, 9208, 9364, 9276, 9266, 9264, 9424] dup_rows_per_dst=[3002, 3048, 2898, 3128, 2854, 2990, 3010, 3066] combine_rows_per_src=[9254, 9252, 9208, 9364, 9276, 9266, 9264, 9424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=344608 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266945056 ingress_bytes=335806464 link_bytes=335806464 token_bytes=533545504 weight_movement_bytes=69206016 total_nvlink_bytes=602751520 weight_to_token_ratio=0.129710 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263612689e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=264822784 dispatch_meta_bytes=342728 combine_payload_bytes=264822784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265165512 ingress_bytes=299425792 link_bytes=299425792 token_bytes=529988296 weight_movement_bytes=34603008 total_nvlink_bytes=564591304 weight_to_token_ratio=0.065290 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126715150e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65094 dup_rows=23924 dup_rows_remote=20990 dedup_ratio=0.243368 router_tokens_per_expert=[759, 793, 793, 770, 803, 801, 746, 755, 749, 801, 737, 777, 740, 763, 794, 758, 795, 743, 742, 751, 760, 752, 747, 750, 735, 770, 742, 800, 776, 778, 753, 756, 772, 800, 797, 797, 737, 757, 790, 742, 769, 753, 817, 781, 787, 772, 771, 780, 778, 737, 763, 761, 795, 733, 724, 764, 735, 790, 812, 798, 755, 734, 759, 803] payload_rows_per_dst=[9370, 9270, 9212, 9286, 9326, 9340, 9208, 9368] dup_rows_per_dst=[3070, 2968, 2868, 2934, 3058, 3120, 2902, 3004] combine_rows_per_src=[9370, 9270, 9212, 9286, 9326, 9340, 9208, 9368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344576 combine_payload_bytes=266362880 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266707456 ingress_bytes=370171904 link_bytes=370171904 token_bytes=533070336 weight_movement_bytes=103809024 total_nvlink_bytes=636879360 weight_to_token_ratio=0.194738 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392927073e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74106 payload_rows_remote=64768 dup_rows=24198 dup_rows_remote=21048 dedup_ratio=0.246155 router_tokens_per_expert=[793, 803, 774, 795, 750, 743, 758, 738, 737, 758, 773, 801, 779, 819, 804, 773, 745, 734, 745, 828, 779, 757, 755, 739, 759, 727, 770, 747, 753, 730, 795, 728, 776, 785, 758, 787, 743, 777, 703, 801, 755, 796, 772, 757, 756, 793, 795, 792, 775, 782, 778, 783, 788, 773, 772, 744, 795, 739, 795, 748, 725, 795, 769, 756] payload_rows_per_dst=[9284, 9338, 9158, 9192, 9214, 9268, 9386, 9266] dup_rows_per_dst=[3024, 3150, 3006, 2826, 3046, 3164, 3004, 2978] combine_rows_per_src=[9284, 9338, 9158, 9192, 9214, 9268, 9386, 9266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267354112 dispatch_meta_bytes=344448 combine_payload_bytes=267354112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267698560 ingress_bytes=301957120 link_bytes=301957120 token_bytes=535052672 weight_movement_bytes=34603008 total_nvlink_bytes=569655680 weight_to_token_ratio=0.064672 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136240333e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344696 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266715768 ingress_bytes=300974080 link_bytes=300974080 token_bytes=533086840 weight_movement_bytes=34603008 total_nvlink_bytes=567689848 weight_to_token_ratio=0.064911 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132541233e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73862 payload_rows_remote=64468 dup_rows=24442 dup_rows_remote=21236 dedup_ratio=0.248637 router_tokens_per_expert=[814, 801, 796, 805, 776, 776, 774, 752, 767, 775, 758, 700, 767, 793, 753, 784, 820, 735, 739, 770, 759, 754, 793, 725, 761, 766, 796, 732, 752, 811, 745, 729, 766, 768, 796, 726, 791, 811, 755, 755, 780, 767, 739, 789, 760, 753, 770, 784, 781, 696, 785, 764, 805, 769, 774, 777, 773, 756, 788, 742, 735, 802, 727, 790] payload_rows_per_dst=[9394, 9202, 9214, 9270, 9306, 9198, 9096, 9182] dup_rows_per_dst=[3206, 2992, 3168, 3106, 3030, 2926, 2970, 3044] combine_rows_per_src=[9394, 9202, 9214, 9270, 9306, 9198, 9096, 9182] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266625024 dispatch_meta_bytes=344336 combine_payload_bytes=266625024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266969360 ingress_bytes=301228032 link_bytes=301228032 token_bytes=533594384 weight_movement_bytes=34603008 total_nvlink_bytes=568197392 weight_to_token_ratio=0.064849 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133496834e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265289728 dispatch_meta_bytes=343264 combine_payload_bytes=265289728 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265632992 ingress_bytes=369098752 link_bytes=369098752 token_bytes=530922720 weight_movement_bytes=103809024 total_nvlink_bytes=634731744 weight_to_token_ratio=0.195526 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388888889e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=64654 dup_rows=24132 dup_rows_remote=21028 dedup_ratio=0.245483 router_tokens_per_expert=[791, 794, 746, 812, 752, 740, 769, 753, 791, 782, 814, 761, 769, 754, 703, 815, 759, 770, 763, 766, 789, 786, 772, 810, 726, 798, 788, 791, 793, 747, 756, 772, 747, 773, 723, 769, 791, 754, 746, 784, 775, 750, 758, 773, 749, 733, 752, 782, 805, 791, 725, 779, 771, 786, 812, 731, 810, 775, 755, 705, 749, 759, 757, 751] payload_rows_per_dst=[8952, 8864, 9224, 9636, 9184, 9466, 9328, 9518] dup_rows_per_dst=[2792, 2726, 2980, 3296, 2958, 3172, 3104, 3104] combine_rows_per_src=[8952, 8864, 9224, 9636, 9184, 9466, 9328, 9518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64946 dup_rows=24114 dup_rows_remote=21078 dedup_ratio=0.245300 router_tokens_per_expert=[791, 773, 764, 773, 772, 739, 756, 768, 767, 773, 757, 776, 728, 749, 760, 734, 782, 772, 745, 739, 801, 786, 782, 773, 781, 742, 806, 787, 779, 748, 766, 733, 776, 747, 770, 787, 742, 735, 793, 749, 769, 725, 783, 759, 796, 810, 757, 741, 769, 776, 779, 770, 834, 764, 756, 801, 730, 737, 810, 796, 772, 779, 774, 764] payload_rows_per_dst=[9320, 9120, 9300, 9274, 9202, 9244, 9366, 9364] dup_rows_per_dst=[2952, 2968, 3060, 3010, 2996, 3036, 3132, 2960] combine_rows_per_src=[9320, 9120, 9300, 9274, 9202, 9244, 9366, 9364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74332 payload_rows_remote=65104 dup_rows=23972 dup_rows_remote=21016 dedup_ratio=0.243856 router_tokens_per_expert=[792, 765, 738, 794, 750, 732, 760, 802, 775, 754, 838, 800, 768, 774, 789, 754, 775, 732, 721, 772, 763, 796, 800, 771, 730, 769, 761, 726, 738, 767, 785, 781, 812, 790, 757, 761, 759, 796, 750, 745, 771, 779, 794, 753, 752, 774, 757, 751, 768, 759, 752, 755, 744, 779, 758, 777, 798, 782, 752, 747, 773, 766, 782, 787] payload_rows_per_dst=[9382, 9318, 9284, 9274, 9228, 9278, 9228, 9340] dup_rows_per_dst=[2884, 3186, 2976, 2840, 3112, 2984, 2956, 3034] combine_rows_per_src=[9382, 9318, 9284, 9274, 9228, 9278, 9228, 9340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=264060928 dispatch_meta_bytes=342816 combine_payload_bytes=264060928 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=264403744 ingress_bytes=402472960 link_bytes=402472960 token_bytes=528464672 weight_movement_bytes=138412032 total_nvlink_bytes=666876704 weight_to_token_ratio=0.261914 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.514473347e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=64926 dup_rows=23962 dup_rows_remote=20956 dedup_ratio=0.243754 router_tokens_per_expert=[766, 744, 786, 751, 789, 782, 754, 795, 751, 724, 782, 820, 743, 776, 787, 761, 818, 749, 769, 765, 777, 771, 801, 761, 783, 779, 805, 745, 761, 763, 752, 827, 808, 793, 823, 758, 772, 793, 778, 734, 735, 791, 777, 790, 760, 746, 792, 768, 749, 759, 758, 734, 763, 745, 737, 695, 768, 755, 774, 744, 747, 772, 757, 740] payload_rows_per_dst=[9256, 9342, 9416, 9420, 9332, 9262, 9140, 9174] dup_rows_per_dst=[3078, 2946, 3006, 3010, 3186, 3056, 2740, 2940] combine_rows_per_src=[9256, 9342, 9416, 9420, 9332, 9262, 9140, 9174] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=65148 dup_rows=23930 dup_rows_remote=21066 dedup_ratio=0.243429 router_tokens_per_expert=[813, 752, 746, 723, 749, 814, 755, 758, 753, 801, 729, 788, 771, 725, 790, 793, 777, 779, 796, 775, 780, 782, 775, 745, 756, 815, 799, 708, 831, 759, 732, 764, 784, 810, 723, 748, 734, 759, 735, 752, 736, 758, 791, 742, 757, 792, 771, 764, 768, 751, 755, 743, 816, 771, 802, 791, 751, 743, 750, 771, 769, 793, 822, 767] payload_rows_per_dst=[9294, 9334, 9330, 9332, 9226, 9226, 9290, 9342] dup_rows_per_dst=[2926, 2966, 3088, 2996, 2864, 2996, 3104, 2990] combine_rows_per_src=[9294, 9334, 9330, 9332, 9226, 9226, 9290, 9342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=264822784 dispatch_meta_bytes=342728 combine_payload_bytes=264822784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265165512 ingress_bytes=299425792 link_bytes=299425792 token_bytes=529988296 weight_movement_bytes=34603008 total_nvlink_bytes=564591304 weight_to_token_ratio=0.065290 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126715150e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74418 payload_rows_remote=65132 dup_rows=23886 dup_rows_remote=20886 dedup_ratio=0.242981 router_tokens_per_expert=[740, 782, 760, 755, 813, 809, 790, 758, 746, 738, 768, 726, 819, 755, 742, 786, 763, 736, 772, 783, 793, 811, 773, 793, 750, 797, 722, 758, 794, 736, 758, 746, 761, 777, 755, 774, 805, 757, 755, 759, 813, 728, 781, 753, 750, 816, 752, 833, 800, 729, 796, 747, 795, 733, 815, 765, 750, 793, 749, 770, 743, 692, 744, 790] payload_rows_per_dst=[9340, 9282, 9446, 9154, 9286, 9406, 9326, 9178] dup_rows_per_dst=[3074, 2878, 3002, 2968, 3000, 3046, 3034, 2884] combine_rows_per_src=[9340, 9282, 9446, 9154, 9286, 9406, 9326, 9178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266018816 dispatch_meta_bytes=344096 combine_payload_bytes=266018816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266362912 ingress_bytes=300621824 link_bytes=300621824 token_bytes=532381728 weight_movement_bytes=34603008 total_nvlink_bytes=566984736 weight_to_token_ratio=0.064997 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131215722e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74246 payload_rows_remote=64984 dup_rows=24058 dup_rows_remote=20994 dedup_ratio=0.244731 router_tokens_per_expert=[774, 773, 740, 750, 777, 778, 764, 788, 745, 740, 791, 777, 758, 827, 778, 747, 754, 748, 765, 735, 771, 790, 762, 757, 755, 817, 749, 769, 760, 719, 789, 760, 768, 815, 750, 786, 789, 771, 785, 747, 821, 803, 783, 768, 763, 744, 807, 701, 741, 730, 737, 761, 806, 807, 842, 779, 728, 778, 767, 743, 766, 804, 715, 740] payload_rows_per_dst=[9304, 9262, 9164, 9224, 9454, 9420, 9252, 9166] dup_rows_per_dst=[2984, 3064, 3000, 3012, 2968, 2960, 3154, 2916] combine_rows_per_src=[9304, 9262, 9164, 9224, 9454, 9420, 9252, 9166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74118 payload_rows_remote=64932 dup_rows=24186 dup_rows_remote=21340 dedup_ratio=0.246033 router_tokens_per_expert=[804, 772, 843, 779, 829, 762, 765, 766, 819, 776, 734, 801, 815, 746, 807, 771, 768, 743, 813, 775, 776, 742, 692, 729, 785, 748, 738, 749, 725, 719, 790, 762, 749, 789, 769, 781, 785, 824, 805, 801, 803, 764, 771, 763, 724, 742, 708, 787, 742, 797, 777, 749, 786, 774, 746, 744, 774, 771, 789, 744, 755, 720, 722, 754] payload_rows_per_dst=[9464, 9370, 9198, 9186, 9396, 9192, 9174, 9138] dup_rows_per_dst=[3176, 3168, 2878, 2846, 3210, 2932, 3056, 2920] combine_rows_per_src=[9464, 9370, 9198, 9186, 9396, 9192, 9174, 9138] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266665984 dispatch_meta_bytes=344480 combine_payload_bytes=266665984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267010464 ingress_bytes=301268992 link_bytes=301268992 token_bytes=533676448 weight_movement_bytes=34603008 total_nvlink_bytes=568279456 weight_to_token_ratio=0.064839 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133650963e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265936896 dispatch_meta_bytes=343528 combine_payload_bytes=265936896 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266280424 ingress_bytes=335142912 link_bytes=335142912 token_bytes=532217320 weight_movement_bytes=69206016 total_nvlink_bytes=601423336 weight_to_token_ratio=0.130033 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261115797e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=65030 dup_rows=24056 dup_rows_remote=21114 dedup_ratio=0.244710 router_tokens_per_expert=[771, 744, 798, 740, 794, 764, 780, 798, 814, 783, 753, 777, 750, 750, 772, 753, 739, 773, 779, 724, 786, 778, 784, 781, 772, 761, 789, 802, 824, 744, 780, 749, 771, 723, 748, 748, 759, 745, 750, 739, 747, 796, 747, 797, 787, 792, 747, 796, 761, 762, 779, 782, 786, 747, 709, 754, 757, 772, 794, 785, 812, 730, 769, 755] payload_rows_per_dst=[9338, 9278, 9282, 9364, 9088, 9326, 9218, 9354] dup_rows_per_dst=[3040, 3026, 3006, 3078, 2878, 3092, 2942, 2994] combine_rows_per_src=[9338, 9278, 9282, 9364, 9088, 9326, 9218, 9354] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73862 payload_rows_remote=64468 dup_rows=24442 dup_rows_remote=21236 dedup_ratio=0.248637 router_tokens_per_expert=[814, 801, 796, 805, 776, 776, 774, 752, 767, 775, 758, 700, 767, 793, 753, 784, 820, 735, 739, 770, 759, 754, 793, 725, 761, 766, 796, 732, 752, 811, 745, 729, 766, 768, 796, 726, 791, 811, 755, 755, 780, 767, 739, 789, 760, 753, 770, 784, 781, 696, 785, 764, 805, 769, 774, 777, 773, 756, 788, 742, 735, 802, 727, 790] payload_rows_per_dst=[9394, 9202, 9214, 9270, 9306, 9198, 9096, 9182] dup_rows_per_dst=[3206, 2992, 3168, 3106, 3030, 2926, 2970, 3044] combine_rows_per_src=[9394, 9202, 9214, 9270, 9306, 9198, 9096, 9182] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266846208 dispatch_meta_bytes=344856 combine_payload_bytes=266846208 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267191064 ingress_bytes=336052224 link_bytes=336052224 token_bytes=534037272 weight_movement_bytes=69206016 total_nvlink_bytes=603243288 weight_to_token_ratio=0.129590 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264537464e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74106 payload_rows_remote=64768 dup_rows=24198 dup_rows_remote=21048 dedup_ratio=0.246155 router_tokens_per_expert=[793, 803, 774, 795, 750, 743, 758, 738, 737, 758, 773, 801, 779, 819, 804, 773, 745, 734, 745, 828, 779, 757, 755, 739, 759, 727, 770, 747, 753, 730, 795, 728, 776, 785, 758, 787, 743, 777, 703, 801, 755, 796, 772, 757, 756, 793, 795, 792, 775, 782, 778, 783, 788, 773, 772, 744, 795, 739, 795, 748, 725, 795, 769, 756] payload_rows_per_dst=[9284, 9338, 9158, 9192, 9214, 9268, 9386, 9266] dup_rows_per_dst=[3024, 3150, 3006, 2826, 3046, 3164, 3004, 2978] combine_rows_per_src=[9284, 9338, 9158, 9192, 9214, 9268, 9386, 9266] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64946 dup_rows=24114 dup_rows_remote=21078 dedup_ratio=0.245300 router_tokens_per_expert=[791, 773, 764, 773, 772, 739, 756, 768, 767, 773, 757, 776, 728, 749, 760, 734, 782, 772, 745, 739, 801, 786, 782, 773, 781, 742, 806, 787, 779, 748, 766, 733, 776, 747, 770, 787, 742, 735, 793, 749, 769, 725, 783, 759, 796, 810, 757, 741, 769, 776, 779, 770, 834, 764, 756, 801, 730, 737, 810, 796, 772, 779, 774, 764] payload_rows_per_dst=[9320, 9120, 9300, 9274, 9202, 9244, 9366, 9364] dup_rows_per_dst=[2952, 2968, 3060, 3010, 2996, 3036, 3132, 2960] combine_rows_per_src=[9320, 9120, 9300, 9274, 9202, 9244, 9366, 9364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266780672 dispatch_meta_bytes=344072 combine_payload_bytes=266780672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267124744 ingress_bytes=301383680 link_bytes=301383680 token_bytes=533905416 weight_movement_bytes=34603008 total_nvlink_bytes=568508424 weight_to_token_ratio=0.064811 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134082524e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266174464 dispatch_meta_bytes=343912 combine_payload_bytes=266174464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266518376 ingress_bytes=300777472 link_bytes=300777472 token_bytes=532692840 weight_movement_bytes=34603008 total_nvlink_bytes=567295848 weight_to_token_ratio=0.064959 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131801413e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74220 payload_rows_remote=64790 dup_rows=24084 dup_rows_remote=20994 dedup_ratio=0.244995 router_tokens_per_expert=[780, 806, 755, 755, 779, 817, 768, 797, 755, 757, 820, 770, 793, 797, 766, 796, 756, 775, 790, 764, 805, 757, 809, 744, 749, 734, 738, 791, 754, 751, 795, 771, 757, 721, 759, 816, 751, 759, 784, 731, 758, 759, 726, 751, 784, 765, 736, 735, 751, 773, 781, 745, 772, 801, 811, 714, 769, 779, 763, 742, 760, 777, 750, 778] payload_rows_per_dst=[9430, 9356, 9388, 9238, 9198, 9122, 9266, 9222] dup_rows_per_dst=[3090, 3152, 3012, 2928, 2952, 2906, 3030, 3014] combine_rows_per_src=[9430, 9356, 9388, 9238, 9198, 9122, 9266, 9222] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265961472 dispatch_meta_bytes=345088 combine_payload_bytes=265961472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266306560 ingress_bytes=300564480 link_bytes=300564480 token_bytes=532268032 weight_movement_bytes=34603008 total_nvlink_bytes=566871040 weight_to_token_ratio=0.065010 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130999941e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74238 payload_rows_remote=64480 dup_rows=24066 dup_rows_remote=20682 dedup_ratio=0.244812 router_tokens_per_expert=[737, 824, 780, 752, 785, 834, 769, 761, 756, 770, 766, 786, 757, 733, 845, 747, 723, 766, 768, 770, 759, 759, 790, 759, 778, 770, 754, 722, 777, 817, 761, 748, 777, 777, 755, 783, 781, 802, 764, 809, 735, 769, 748, 780, 744, 744, 703, 803, 784, 764, 741, 751, 787, 756, 750, 769, 796, 757, 751, 769, 729, 764, 792, 795] payload_rows_per_dst=[9282, 9362, 9200, 9162, 9444, 8796, 9234, 9758] dup_rows_per_dst=[3202, 2958, 2988, 2906, 3082, 2576, 2970, 3384] combine_rows_per_src=[9282, 9362, 9200, 9162, 9444, 8796, 9234, 9758] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74118 payload_rows_remote=64932 dup_rows=24186 dup_rows_remote=21340 dedup_ratio=0.246033 router_tokens_per_expert=[804, 772, 843, 779, 829, 762, 765, 766, 819, 776, 734, 801, 815, 746, 807, 771, 768, 743, 813, 775, 776, 742, 692, 729, 785, 748, 738, 749, 725, 719, 790, 762, 749, 789, 769, 781, 785, 824, 805, 801, 803, 764, 771, 763, 724, 742, 708, 787, 742, 797, 777, 749, 786, 774, 746, 744, 774, 771, 789, 744, 755, 720, 722, 754] payload_rows_per_dst=[9464, 9370, 9198, 9186, 9396, 9192, 9174, 9138] dup_rows_per_dst=[3176, 3168, 2878, 2846, 3210, 2932, 3056, 2920] combine_rows_per_src=[9464, 9370, 9198, 9186, 9396, 9192, 9174, 9138] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344576 combine_payload_bytes=266362880 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266707456 ingress_bytes=370171904 link_bytes=370171904 token_bytes=533070336 weight_movement_bytes=103809024 total_nvlink_bytes=636879360 weight_to_token_ratio=0.194738 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392927073e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=264060928 dispatch_meta_bytes=342816 combine_payload_bytes=264060928 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=264403744 ingress_bytes=402472960 link_bytes=402472960 token_bytes=528464672 weight_movement_bytes=138412032 total_nvlink_bytes=666876704 weight_to_token_ratio=0.261914 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.514473347e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265289728 dispatch_meta_bytes=343264 combine_payload_bytes=265289728 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265632992 ingress_bytes=369098752 link_bytes=369098752 token_bytes=530922720 weight_movement_bytes=103809024 total_nvlink_bytes=634731744 weight_to_token_ratio=0.195526 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388888889e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266018816 dispatch_meta_bytes=344096 combine_payload_bytes=266018816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266362912 ingress_bytes=300621824 link_bytes=300621824 token_bytes=532381728 weight_movement_bytes=34603008 total_nvlink_bytes=566984736 weight_to_token_ratio=0.064997 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131215722e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74476 payload_rows_remote=65094 dup_rows=23828 dup_rows_remote=20862 dedup_ratio=0.242391 router_tokens_per_expert=[751, 797, 740, 743, 805, 752, 745, 739, 752, 747, 733, 796, 760, 752, 782, 751, 810, 807, 768, 766, 778, 758, 824, 746, 800, 751, 757, 781, 800, 713, 723, 749, 737, 799, 759, 759, 796, 763, 771, 752, 785, 752, 743, 804, 770, 731, 793, 796, 761, 700, 765, 810, 727, 807, 764, 798, 771, 777, 762, 817, 786, 747, 769, 805] payload_rows_per_dst=[9214, 9206, 9456, 9212, 9266, 9382, 9284, 9456] dup_rows_per_dst=[2930, 2940, 3058, 2936, 3006, 2966, 2980, 3012] combine_rows_per_src=[9214, 9206, 9456, 9212, 9266, 9382, 9284, 9456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12259328616 prefetch_bytes=657457152 reduce_bytes=1038090240 weight_movement_bytes=1695547392 egress_bytes=6133621352 ingress_bytes=7821254656 link_bytes=7821254656 total_nvlink_bytes=13954876008 weight_to_token_ratio=0.138307 prefetch_share_of_ingress=0.084060 dedup_ratio=0.244174 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=30 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.279597896e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=65112 dup_rows=23940 dup_rows_remote=20934 dedup_ratio=0.243530 router_tokens_per_expert=[755, 789, 740, 802, 745, 733, 774, 744, 727, 730, 774, 759, 757, 790, 755, 791, 763, 805, 802, 739, 766, 774, 770, 790, 731, 790, 725, 741, 805, 795, 751, 790, 781, 741, 760, 766, 777, 723, 781, 757, 794, 791, 796, 783, 782, 734, 781, 749, 755, 759, 776, 820, 716, 775, 766, 762, 740, 769, 807, 757, 774, 769, 780, 829] payload_rows_per_dst=[9252, 9218, 9354, 9300, 9248, 9344, 9252, 9396] dup_rows_per_dst=[2912, 2948, 3064, 2956, 2924, 3076, 3006, 3054] combine_rows_per_src=[9252, 9218, 9354, 9300, 9248, 9344, 9252, 9396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265379840 dispatch_meta_bytes=343136 combine_payload_bytes=265379840 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265722976 ingress_bytes=369188864 link_bytes=369188864 token_bytes=531102816 weight_movement_bytes=103809024 total_nvlink_bytes=634911840 weight_to_token_ratio=0.195459 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389227973e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=264110080 dispatch_meta_bytes=340648 combine_payload_bytes=264110080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264450728 ingress_bytes=298713088 link_bytes=298713088 token_bytes=528560808 weight_movement_bytes=34603008 total_nvlink_bytes=563163816 weight_to_token_ratio=0.065466 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.124033302e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265961472 dispatch_meta_bytes=345088 combine_payload_bytes=265961472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266306560 ingress_bytes=300564480 link_bytes=300564480 token_bytes=532268032 weight_movement_bytes=34603008 total_nvlink_bytes=566871040 weight_to_token_ratio=0.065010 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130999941e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74370 payload_rows_remote=65072 dup_rows=23934 dup_rows_remote=20918 dedup_ratio=0.243469 router_tokens_per_expert=[783, 780, 784, 754, 820, 784, 789, 747, 756, 819, 763, 775, 694, 743, 777, 735, 778, 744, 781, 775, 752, 777, 749, 801, 794, 807, 778, 780, 770, 757, 776, 792, 792, 756, 748, 758, 746, 772, 789, 757, 780, 738, 762, 771, 761, 723, 766, 778, 776, 809, 737, 774, 767, 775, 816, 832, 753, 704, 759, 739, 728, 736, 732, 804] payload_rows_per_dst=[9414, 9186, 9298, 9412, 9322, 9190, 9466, 9082] dup_rows_per_dst=[3068, 2938, 3016, 3096, 2914, 2968, 3106, 2828] combine_rows_per_src=[9414, 9186, 9298, 9412, 9322, 9190, 9466, 9082] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64904 dup_rows=24114 dup_rows_remote=21160 dedup_ratio=0.245300 router_tokens_per_expert=[767, 764, 775, 773, 791, 733, 765, 713, 796, 806, 782, 723, 770, 775, 776, 746, 798, 751, 772, 785, 753, 748, 829, 821, 752, 753, 795, 778, 804, 810, 767, 760, 730, 736, 764, 789, 790, 789, 733, 789, 757, 772, 786, 796, 763, 752, 795, 805, 731, 779, 743, 734, 760, 754, 788, 769, 732, 773, 721, 750, 751, 763, 753, 774] payload_rows_per_dst=[9212, 9350, 9300, 9306, 9286, 9388, 9158, 9190] dup_rows_per_dst=[2950, 2998, 3214, 3132, 2954, 3064, 2958, 2844] combine_rows_per_src=[9212, 9350, 9300, 9306, 9286, 9388, 9158, 9190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12217453680 prefetch_bytes=795869184 reduce_bytes=242221056 weight_movement_bytes=1038090240 egress_bytes=6112668784 ingress_bytes=7142875136 link_bytes=7142875136 total_nvlink_bytes=13255543920 weight_to_token_ratio=0.084968 prefetch_share_of_ingress=0.111421 dedup_ratio=0.244770 prefetch_slots=46 unused_prefetch_slots=322 slot_utilization=0.125000 reduce_slots=7 remote_demand=23 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029022 gemm_inner_pad_upper_bound=29210 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.168611482e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=65148 dup_rows=23930 dup_rows_remote=21066 dedup_ratio=0.243429 router_tokens_per_expert=[813, 752, 746, 723, 749, 814, 755, 758, 753, 801, 729, 788, 771, 725, 790, 793, 777, 779, 796, 775, 780, 782, 775, 745, 756, 815, 799, 708, 831, 759, 732, 764, 784, 810, 723, 748, 734, 759, 735, 752, 736, 758, 791, 742, 757, 792, 771, 764, 768, 751, 755, 743, 816, 771, 802, 791, 751, 743, 750, 771, 769, 793, 822, 767] payload_rows_per_dst=[9294, 9334, 9330, 9332, 9226, 9226, 9290, 9342] dup_rows_per_dst=[2926, 2966, 3088, 2996, 2864, 2996, 3104, 2990] combine_rows_per_src=[9294, 9334, 9330, 9332, 9226, 9226, 9290, 9342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266625024 dispatch_meta_bytes=343824 combine_payload_bytes=266625024 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266968848 ingress_bytes=335831040 link_bytes=335831040 token_bytes=533593872 weight_movement_bytes=69206016 total_nvlink_bytes=602799888 weight_to_token_ratio=0.129698 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263705167e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266534912 dispatch_meta_bytes=343960 combine_payload_bytes=266534912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266878872 ingress_bytes=301137920 link_bytes=301137920 token_bytes=533413784 weight_movement_bytes=34603008 total_nvlink_bytes=568016792 weight_to_token_ratio=0.064871 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133157749e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265846784 dispatch_meta_bytes=344256 combine_payload_bytes=265846784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266191040 ingress_bytes=300449792 link_bytes=300449792 token_bytes=532037824 weight_movement_bytes=34603008 total_nvlink_bytes=566640832 weight_to_token_ratio=0.065039 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130568379e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74332 payload_rows_remote=65104 dup_rows=23972 dup_rows_remote=21016 dedup_ratio=0.243856 router_tokens_per_expert=[792, 765, 738, 794, 750, 732, 760, 802, 775, 754, 838, 800, 768, 774, 789, 754, 775, 732, 721, 772, 763, 796, 800, 771, 730, 769, 761, 726, 738, 767, 785, 781, 812, 790, 757, 761, 759, 796, 750, 745, 771, 779, 794, 753, 752, 774, 757, 751, 768, 759, 752, 755, 744, 779, 758, 777, 798, 782, 752, 747, 773, 766, 782, 787] payload_rows_per_dst=[9382, 9318, 9284, 9274, 9228, 9278, 9228, 9340] dup_rows_per_dst=[2884, 3186, 2976, 2840, 3112, 2984, 2956, 3034] combine_rows_per_src=[9382, 9318, 9284, 9274, 9228, 9278, 9228, 9340] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74220 payload_rows_remote=64790 dup_rows=24084 dup_rows_remote=20994 dedup_ratio=0.244995 router_tokens_per_expert=[780, 806, 755, 755, 779, 817, 768, 797, 755, 757, 820, 770, 793, 797, 766, 796, 756, 775, 790, 764, 805, 757, 809, 744, 749, 734, 738, 791, 754, 751, 795, 771, 757, 721, 759, 816, 751, 759, 784, 731, 758, 759, 726, 751, 784, 765, 736, 735, 751, 773, 781, 745, 772, 801, 811, 714, 769, 779, 763, 742, 760, 777, 750, 778] payload_rows_per_dst=[9430, 9356, 9388, 9238, 9198, 9122, 9266, 9222] dup_rows_per_dst=[3090, 3152, 3012, 2928, 2952, 2906, 3030, 3014] combine_rows_per_src=[9430, 9356, 9388, 9238, 9198, 9122, 9266, 9222] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74246 payload_rows_remote=64984 dup_rows=24058 dup_rows_remote=20994 dedup_ratio=0.244731 router_tokens_per_expert=[774, 773, 740, 750, 777, 778, 764, 788, 745, 740, 791, 777, 758, 827, 778, 747, 754, 748, 765, 735, 771, 790, 762, 757, 755, 817, 749, 769, 760, 719, 789, 760, 768, 815, 750, 786, 789, 771, 785, 747, 821, 803, 783, 768, 763, 744, 807, 701, 741, 730, 737, 761, 806, 807, 842, 779, 728, 778, 767, 743, 766, 804, 715, 740] payload_rows_per_dst=[9304, 9262, 9164, 9224, 9454, 9420, 9252, 9166] dup_rows_per_dst=[2984, 3064, 3000, 3012, 2968, 2960, 3154, 2916] combine_rows_per_src=[9304, 9262, 9164, 9224, 9454, 9420, 9252, 9166] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74476 payload_rows_remote=65094 dup_rows=23828 dup_rows_remote=20862 dedup_ratio=0.242391 router_tokens_per_expert=[751, 797, 740, 743, 805, 752, 745, 739, 752, 747, 733, 796, 760, 752, 782, 751, 810, 807, 768, 766, 778, 758, 824, 746, 800, 751, 757, 781, 800, 713, 723, 749, 737, 799, 759, 759, 796, 763, 771, 752, 785, 752, 743, 804, 770, 731, 793, 796, 761, 700, 765, 810, 727, 807, 764, 798, 771, 777, 762, 817, 786, 747, 769, 805] payload_rows_per_dst=[9214, 9206, 9456, 9212, 9266, 9382, 9284, 9456] dup_rows_per_dst=[2930, 2940, 3058, 2936, 3006, 2966, 2980, 3012] combine_rows_per_src=[9214, 9206, 9456, 9212, 9266, 9382, 9284, 9456] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266698752 dispatch_meta_bytes=344184 combine_payload_bytes=266698752 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267042936 ingress_bytes=301301760 link_bytes=301301760 token_bytes=533741688 weight_movement_bytes=34603008 total_nvlink_bytes=568344696 weight_to_token_ratio=0.064831 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133774266e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12217453680 prefetch_bytes=795869184 reduce_bytes=242221056 weight_movement_bytes=1038090240 egress_bytes=6112668784 ingress_bytes=7142875136 link_bytes=7142875136 total_nvlink_bytes=13255543920 weight_to_token_ratio=0.084968 prefetch_share_of_ingress=0.111421 dedup_ratio=0.244770 prefetch_slots=46 unused_prefetch_slots=322 slot_utilization=0.125000 reduce_slots=7 remote_demand=23 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029022 gemm_inner_pad_upper_bound=29210 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.168611482e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74406 payload_rows_remote=65150 dup_rows=23898 dup_rows_remote=20794 dedup_ratio=0.243103 router_tokens_per_expert=[743, 760, 781, 709, 758, 719, 783, 767, 730, 803, 751, 717, 780, 779, 783, 812, 729, 786, 754, 758, 746, 778, 722, 760, 750, 777, 759, 795, 796, 764, 753, 786, 784, 765, 774, 776, 788, 762, 790, 755, 786, 808, 787, 739, 800, 790, 827, 808, 748, 742, 725, 774, 780, 820, 771, 770, 745, 776, 771, 744, 798, 773, 755, 733] payload_rows_per_dst=[9264, 9212, 9248, 9256, 9352, 9510, 9334, 9230] dup_rows_per_dst=[2776, 3098, 2818, 3104, 3036, 3180, 2926, 2960] combine_rows_per_src=[9264, 9212, 9248, 9256, 9352, 9510, 9334, 9230] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12261994256 prefetch_bytes=761266176 reduce_bytes=588251136 weight_movement_bytes=1349517312 egress_bytes=6134951696 ingress_bytes=7476559872 link_bytes=7476559872 total_nvlink_bytes=13611511568 weight_to_token_ratio=0.110057 prefetch_share_of_ingress=0.101820 dedup_ratio=0.243594 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=17 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.223204039e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12256003456 prefetch_bytes=519045120 reduce_bytes=795869184 weight_movement_bytes=1314914304 egress_bytes=6131959168 ingress_bytes=7438958592 link_bytes=7438958592 total_nvlink_bytes=13570917760 weight_to_token_ratio=0.107287 prefetch_share_of_ingress=0.069774 dedup_ratio=0.244439 prefetch_slots=30 unused_prefetch_slots=338 slot_utilization=0.081522 reduce_slots=23 remote_demand=15 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029022 gemm_inner_pad_upper_bound=28194 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.217052274e-04 prefetch_bytes_per_gemm_flop=8.491847826e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265379840 dispatch_meta_bytes=343136 combine_payload_bytes=265379840 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265722976 ingress_bytes=369188864 link_bytes=369188864 token_bytes=531102816 weight_movement_bytes=103809024 total_nvlink_bytes=634911840 weight_to_token_ratio=0.195459 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389227973e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266665984 dispatch_meta_bytes=344480 combine_payload_bytes=266665984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267010464 ingress_bytes=301268992 link_bytes=301268992 token_bytes=533676448 weight_movement_bytes=34603008 total_nvlink_bytes=568279456 weight_to_token_ratio=0.064839 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133650963e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266625024 dispatch_meta_bytes=343824 combine_payload_bytes=266625024 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266968848 ingress_bytes=335831040 link_bytes=335831040 token_bytes=533593872 weight_movement_bytes=69206016 total_nvlink_bytes=602799888 weight_to_token_ratio=0.129698 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263705167e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266174464 dispatch_meta_bytes=343912 combine_payload_bytes=266174464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266518376 ingress_bytes=300777472 link_bytes=300777472 token_bytes=532692840 weight_movement_bytes=34603008 total_nvlink_bytes=567295848 weight_to_token_ratio=0.064959 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131801413e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266854400 dispatch_meta_bytes=343776 combine_payload_bytes=266854400 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267198176 ingress_bytes=405266432 link_bytes=405266432 token_bytes=534052576 weight_movement_bytes=138412032 total_nvlink_bytes=672464608 weight_to_token_ratio=0.259173 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524984957e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266534912 dispatch_meta_bytes=343960 combine_payload_bytes=266534912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266878872 ingress_bytes=301137920 link_bytes=301137920 token_bytes=533413784 weight_movement_bytes=34603008 total_nvlink_bytes=568016792 weight_to_token_ratio=0.064871 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133157749e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12266699520 prefetch_bytes=761266176 reduce_bytes=588251136 weight_movement_bytes=1349517312 egress_bytes=6137305856 ingress_bytes=7478910976 link_bytes=7478910976 total_nvlink_bytes=13616216832 weight_to_token_ratio=0.110015 prefetch_share_of_ingress=0.101788 dedup_ratio=0.243613 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=17 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027826 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.223588691e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12259328616 prefetch_bytes=657457152 reduce_bytes=1038090240 weight_movement_bytes=1695547392 egress_bytes=6133621352 ingress_bytes=7821254656 link_bytes=7821254656 total_nvlink_bytes=13954876008 weight_to_token_ratio=0.138307 prefetch_share_of_ingress=0.084060 dedup_ratio=0.244174 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=30 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.279597896e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12261855048 prefetch_bytes=657457152 reduce_bytes=484442112 weight_movement_bytes=1141899264 egress_bytes=6134886216 ingress_bytes=7268868096 link_bytes=7268868096 total_nvlink_bytes=13403754312 weight_to_token_ratio=0.093126 prefetch_share_of_ingress=0.090448 dedup_ratio=0.243975 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=14 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.189224585e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266846208 dispatch_meta_bytes=344856 combine_payload_bytes=266846208 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267191064 ingress_bytes=336052224 link_bytes=336052224 token_bytes=534037272 weight_movement_bytes=69206016 total_nvlink_bytes=603243288 weight_to_token_ratio=0.129590 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264537464e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12268323528 prefetch_bytes=622854144 reduce_bytes=1072693248 weight_movement_bytes=1695547392 egress_bytes=6138118856 ingress_bytes=7825752064 link_bytes=7825752064 total_nvlink_bytes=13963870920 weight_to_token_ratio=0.138205 prefetch_share_of_ingress=0.079590 dedup_ratio=0.243605 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=31 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027935 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.280333695e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=64928 dup_rows=23920 dup_rows_remote=20836 dedup_ratio=0.243327 router_tokens_per_expert=[761, 776, 788, 756, 752, 756, 800, 798, 772, 755, 777, 772, 803, 793, 797, 766, 708, 780, 773, 760, 726, 764, 741, 764, 758, 783, 760, 812, 737, 773, 835, 817, 772, 744, 757, 729, 728, 786, 748, 760, 805, 771, 767, 771, 779, 749, 824, 776, 743, 781, 766, 809, 824, 736, 746, 754, 774, 766, 737, 759, 713, 761, 733, 771] payload_rows_per_dst=[9456, 9492, 9264, 9192, 9248, 9362, 9226, 9144] dup_rows_per_dst=[3084, 3072, 2862, 3098, 2800, 3122, 2998, 2884] combine_rows_per_src=[9456, 9492, 9264, 9192, 9248, 9362, 9226, 9144] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=65112 dup_rows=23940 dup_rows_remote=20934 dedup_ratio=0.243530 router_tokens_per_expert=[755, 789, 740, 802, 745, 733, 774, 744, 727, 730, 774, 759, 757, 790, 755, 791, 763, 805, 802, 739, 766, 774, 770, 790, 731, 790, 725, 741, 805, 795, 751, 790, 781, 741, 760, 766, 777, 723, 781, 757, 794, 791, 796, 783, 782, 734, 781, 749, 755, 759, 776, 820, 716, 775, 766, 762, 740, 769, 807, 757, 774, 769, 780, 829] payload_rows_per_dst=[9252, 9218, 9354, 9300, 9248, 9344, 9252, 9396] dup_rows_per_dst=[2912, 2948, 3064, 2956, 2924, 3076, 3006, 3054] combine_rows_per_src=[9252, 9218, 9354, 9300, 9248, 9344, 9252, 9396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266854400 dispatch_meta_bytes=343776 combine_payload_bytes=266854400 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267198176 ingress_bytes=405266432 link_bytes=405266432 token_bytes=534052576 weight_movement_bytes=138412032 total_nvlink_bytes=672464608 weight_to_token_ratio=0.259173 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524984957e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12261994256 prefetch_bytes=761266176 reduce_bytes=588251136 weight_movement_bytes=1349517312 egress_bytes=6134951696 ingress_bytes=7476559872 link_bytes=7476559872 total_nvlink_bytes=13611511568 weight_to_token_ratio=0.110057 prefetch_share_of_ingress=0.101820 dedup_ratio=0.243594 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=17 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.223204039e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265945088 dispatch_meta_bytes=343056 combine_payload_bytes=265945088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266288144 ingress_bytes=300548096 link_bytes=300548096 token_bytes=532233232 weight_movement_bytes=34603008 total_nvlink_bytes=566836240 weight_to_token_ratio=0.065015 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130938289e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266698752 dispatch_meta_bytes=344184 combine_payload_bytes=266698752 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267042936 ingress_bytes=301301760 link_bytes=301301760 token_bytes=533741688 weight_movement_bytes=34603008 total_nvlink_bytes=568344696 weight_to_token_ratio=0.064831 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133774266e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12261855048 prefetch_bytes=657457152 reduce_bytes=484442112 weight_movement_bytes=1141899264 egress_bytes=6134886216 ingress_bytes=7268868096 link_bytes=7268868096 total_nvlink_bytes=13403754312 weight_to_token_ratio=0.093126 prefetch_share_of_ingress=0.090448 dedup_ratio=0.243975 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=14 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.189224585e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64904 dup_rows=24114 dup_rows_remote=21160 dedup_ratio=0.245300 router_tokens_per_expert=[767, 764, 775, 773, 791, 733, 765, 713, 796, 806, 782, 723, 770, 775, 776, 746, 798, 751, 772, 785, 753, 748, 829, 821, 752, 753, 795, 778, 804, 810, 767, 760, 730, 736, 764, 789, 790, 789, 733, 789, 757, 772, 786, 796, 763, 752, 795, 805, 731, 779, 743, 734, 760, 754, 788, 769, 732, 773, 721, 750, 751, 763, 753, 774] payload_rows_per_dst=[9212, 9350, 9300, 9306, 9286, 9388, 9158, 9190] dup_rows_per_dst=[2950, 2998, 3214, 3132, 2954, 3064, 2958, 2844] combine_rows_per_src=[9212, 9350, 9300, 9306, 9286, 9388, 9158, 9190] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12232651112 prefetch_bytes=692060160 reduce_bytes=657457152 weight_movement_bytes=1349517312 egress_bytes=6120272232 ingress_bytes=7461896192 link_bytes=7461896192 total_nvlink_bytes=13582168424 weight_to_token_ratio=0.110321 prefetch_share_of_ingress=0.092746 dedup_ratio=0.244256 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=19 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.220804984e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12268323528 prefetch_bytes=622854144 reduce_bytes=1072693248 weight_movement_bytes=1695547392 egress_bytes=6138118856 ingress_bytes=7825752064 link_bytes=7825752064 total_nvlink_bytes=13963870920 weight_to_token_ratio=0.138205 prefetch_share_of_ingress=0.079590 dedup_ratio=0.243605 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=31 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027935 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.280333695e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none - [2026-08-26 15:10:23] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 791.1 | throughput per GPU (TFLOP/s/GPU): 161.3 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.388426E+00 | loss scale: 1.0 | grad norm: 5.302 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74408 payload_rows_remote=65018 dup_rows=23896 dup_rows_remote=20912 dedup_ratio=0.243083 router_tokens_per_expert=[800, 743, 741, 746, 748, 760, 725, 721, 785, 764, 737, 771, 767, 799, 773, 762, 787, 751, 748, 742, 776, 786, 747, 752, 760, 760, 779, 786, 776, 822, 786, 757, 814, 752, 755, 805, 778, 755, 772, 775, 814, 775, 760, 778, 776, 730, 750, 804, 800, 750, 746, 779, 738, 814, 785, 797, 752, 796, 751, 777, 740, 753, 757, 767] payload_rows_per_dst=[9134, 9268, 9228, 9360, 9424, 9390, 9396, 9208] dup_rows_per_dst=[2834, 3048, 2950, 3092, 2988, 2984, 3022, 2978] combine_rows_per_src=[9134, 9268, 9228, 9360, 9424, 9390, 9396, 9208] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=65024 dup_rows=23926 dup_rows_remote=20970 dedup_ratio=0.243388 router_tokens_per_expert=[753, 776, 760, 787, 796, 757, 783, 752, 781, 723, 726, 761, 810, 776, 823, 755, 728, 733, 743, 764, 741, 773, 749, 793, 785, 729, 814, 792, 744, 743, 741, 769, 789, 755, 766, 731, 777, 806, 778, 789, 792, 823, 769, 741, 778, 744, 779, 774, 776, 768, 754, 748, 773, 752, 805, 769, 759, 785, 789, 742, 785, 749, 733, 814] payload_rows_per_dst=[9260, 9354, 9158, 9240, 9324, 9408, 9336, 9298] dup_rows_per_dst=[3068, 2956, 2890, 2994, 3058, 2992, 2954, 3014] combine_rows_per_src=[9260, 9354, 9158, 9240, 9324, 9408, 9336, 9298] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265846784 dispatch_meta_bytes=344256 combine_payload_bytes=265846784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266191040 ingress_bytes=300449792 link_bytes=300449792 token_bytes=532037824 weight_movement_bytes=34603008 total_nvlink_bytes=566640832 weight_to_token_ratio=0.065039 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130568379e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12256003456 prefetch_bytes=519045120 reduce_bytes=795869184 weight_movement_bytes=1314914304 egress_bytes=6131959168 ingress_bytes=7438958592 link_bytes=7438958592 total_nvlink_bytes=13570917760 weight_to_token_ratio=0.107287 prefetch_share_of_ingress=0.069774 dedup_ratio=0.244439 prefetch_slots=30 unused_prefetch_slots=338 slot_utilization=0.081522 reduce_slots=23 remote_demand=15 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029022 gemm_inner_pad_upper_bound=28194 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.217052274e-04 prefetch_bytes_per_gemm_flop=8.491847826e-06 anomalies=none + [2026-08-26 15:58:12] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 793.1 | throughput per GPU (TFLOP/s/GPU): 160.9 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.388426E+00 | loss scale: 1.0 | grad norm: 5.302 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74156 payload_rows_remote=64632 dup_rows=24148 dup_rows_remote=20844 dedup_ratio=0.245646 router_tokens_per_expert=[764, 785, 752, 731, 770, 770, 737, 780, 773, 775, 762, 756, 760, 744, 781, 765, 785, 790, 761, 783, 748, 784, 700, 833, 738, 787, 693, 726, 789, 772, 788, 773, 738, 774, 769, 811, 775, 799, 765, 748, 787, 792, 728, 792, 742, 800, 792, 756, 804, 781, 783, 759, 762, 759, 756, 743, 783, 778, 810, 787, 723, 763, 775, 763] payload_rows_per_dst=[9524, 9380, 9552, 9306, 8894, 8976, 9308, 9216] dup_rows_per_dst=[3304, 3092, 3104, 3082, 2622, 2890, 3034, 3020] combine_rows_per_src=[9524, 9380, 9552, 9306, 8894, 8976, 9308, 9216] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74136 payload_rows_remote=64804 dup_rows=24168 dup_rows_remote=21056 dedup_ratio=0.245850 router_tokens_per_expert=[786, 762, 843, 766, 781, 763, 773, 743, 785, 780, 755, 767, 769, 811, 761, 776, 760, 756, 779, 751, 783, 777, 818, 762, 761, 744, 811, 750, 748, 763, 800, 729, 737, 814, 714, 812, 776, 800, 817, 752, 751, 774, 767, 753, 773, 808, 749, 775, 738, 764, 768, 756, 745, 773, 705, 788, 771, 774, 769, 736, 744, 747, 766, 723] payload_rows_per_dst=[9344, 9346, 9354, 9248, 9332, 9250, 9096, 9166] dup_rows_per_dst=[3090, 3062, 3018, 2964, 3112, 3050, 2978, 2894] combine_rows_per_src=[9344, 9346, 9354, 9248, 9332, 9250, 9096, 9166] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=64900 dup_rows=24096 dup_rows_remote=21086 dedup_ratio=0.245117 router_tokens_per_expert=[719, 768, 773, 771, 790, 751, 743, 736, 734, 717, 750, 727, 811, 796, 767, 770, 714, 797, 745, 745, 765, 804, 747, 758, 779, 784, 798, 776, 785, 741, 753, 743, 766, 841, 801, 811, 798, 776, 746, 759, 814, 766, 800, 767, 802, 740, 813, 714, 733, 743, 775, 741, 757, 810, 796, 795, 751, 749, 762, 772, 777, 780, 746, 794] payload_rows_per_dst=[9174, 9280, 9202, 9308, 9400, 9366, 9244, 9234] dup_rows_per_dst=[2928, 2864, 2948, 3010, 3196, 3066, 3056, 3028] combine_rows_per_src=[9174, 9280, 9202, 9308, 9400, 9366, 9244, 9234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74408 payload_rows_remote=65018 dup_rows=23896 dup_rows_remote=20912 dedup_ratio=0.243083 router_tokens_per_expert=[800, 743, 741, 746, 748, 760, 725, 721, 785, 764, 737, 771, 767, 799, 773, 762, 787, 751, 748, 742, 776, 786, 747, 752, 760, 760, 779, 786, 776, 822, 786, 757, 814, 752, 755, 805, 778, 755, 772, 775, 814, 775, 760, 778, 776, 730, 750, 804, 800, 750, 746, 779, 738, 814, 785, 797, 752, 796, 751, 777, 740, 753, 757, 767] payload_rows_per_dst=[9134, 9268, 9228, 9360, 9424, 9390, 9396, 9208] dup_rows_per_dst=[2834, 3048, 2950, 3092, 2988, 2984, 3022, 2978] combine_rows_per_src=[9134, 9268, 9228, 9360, 9424, 9390, 9396, 9208] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74170 payload_rows_remote=64822 dup_rows=24134 dup_rows_remote=20972 dedup_ratio=0.245504 router_tokens_per_expert=[750, 745, 761, 769, 787, 772, 742, 775, 819, 794, 743, 744, 743, 770, 740, 743, 786, 769, 761, 775, 743, 735, 798, 757, 810, 723, 794, 787, 809, 726, 744, 772, 781, 802, 753, 742, 806, 749, 775, 785, 733, 786, 724, 758, 749, 767, 743, 788, 742, 749, 772, 773, 772, 821, 769, 772, 791, 782, 754, 769, 792, 786, 790, 791] payload_rows_per_dst=[9190, 9192, 9224, 9324, 9422, 9140, 9330, 9348] dup_rows_per_dst=[3012, 3000, 3024, 3006, 2964, 2956, 3010, 3162] combine_rows_per_src=[9190, 9192, 9224, 9324, 9422, 9140, 9330, 9348] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=64842 dup_rows=24050 dup_rows_remote=20954 dedup_ratio=0.244649 router_tokens_per_expert=[810, 719, 756, 761, 762, 784, 730, 745, 763, 805, 777, 763, 761, 696, 790, 748, 777, 776, 788, 752, 803, 768, 823, 767, 763, 717, 811, 807, 753, 718, 728, 758, 807, 755, 767, 784, 760, 760, 750, 768, 753, 776, 813, 786, 791, 797, 762, 743, 762, 761, 771, 720, 796, 711, 821, 738, 786, 780, 814, 804, 762, 746, 739, 790] payload_rows_per_dst=[9212, 9242, 9412, 9160, 9312, 9332, 9218, 9366] dup_rows_per_dst=[2922, 2964, 3096, 2950, 2990, 3110, 2942, 3076] combine_rows_per_src=[9212, 9242, 9412, 9160, 9312, 9332, 9218, 9366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74156 payload_rows_remote=64632 dup_rows=24148 dup_rows_remote=20844 dedup_ratio=0.245646 router_tokens_per_expert=[764, 785, 752, 731, 770, 770, 737, 780, 773, 775, 762, 756, 760, 744, 781, 765, 785, 790, 761, 783, 748, 784, 700, 833, 738, 787, 693, 726, 789, 772, 788, 773, 738, 774, 769, 811, 775, 799, 765, 748, 787, 792, 728, 792, 742, 800, 792, 756, 804, 781, 783, 759, 762, 759, 756, 743, 783, 778, 810, 787, 723, 763, 775, 763] payload_rows_per_dst=[9524, 9380, 9552, 9306, 8894, 8976, 9308, 9216] dup_rows_per_dst=[3304, 3092, 3104, 3082, 2622, 2890, 3034, 3020] combine_rows_per_src=[9524, 9380, 9552, 9306, 8894, 8976, 9308, 9216] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=343720 combine_payload_bytes=266313728 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266657448 ingress_bytes=370122752 link_bytes=370122752 token_bytes=532971176 weight_movement_bytes=103809024 total_nvlink_bytes=636780200 weight_to_token_ratio=0.194774 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392742118e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74136 payload_rows_remote=64804 dup_rows=24168 dup_rows_remote=21056 dedup_ratio=0.245850 router_tokens_per_expert=[786, 762, 843, 766, 781, 763, 773, 743, 785, 780, 755, 767, 769, 811, 761, 776, 760, 756, 779, 751, 783, 777, 818, 762, 761, 744, 811, 750, 748, 763, 800, 729, 737, 814, 714, 812, 776, 800, 817, 752, 751, 774, 767, 753, 773, 808, 749, 775, 738, 764, 768, 756, 745, 773, 705, 788, 771, 774, 769, 736, 744, 747, 766, 723] payload_rows_per_dst=[9344, 9346, 9354, 9248, 9332, 9250, 9096, 9166] dup_rows_per_dst=[3090, 3062, 3018, 2964, 3112, 3050, 2978, 2894] combine_rows_per_src=[9344, 9346, 9354, 9248, 9332, 9250, 9096, 9166] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=343976 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266682280 ingress_bytes=300941312 link_bytes=300941312 token_bytes=533020584 weight_movement_bytes=34603008 total_nvlink_bytes=567623592 weight_to_token_ratio=0.064919 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132417929e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265830400 dispatch_meta_bytes=343944 combine_payload_bytes=265830400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266174344 ingress_bytes=300433408 link_bytes=300433408 token_bytes=532004744 weight_movement_bytes=34603008 total_nvlink_bytes=566607752 weight_to_token_ratio=0.065043 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130506727e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74506 payload_rows_remote=65206 dup_rows=23798 dup_rows_remote=20718 dedup_ratio=0.242086 router_tokens_per_expert=[801, 758, 781, 746, 782, 708, 760, 818, 754, 746, 755, 765, 760, 766, 804, 778, 759, 732, 759, 749, 804, 738, 784, 747, 740, 754, 764, 780, 762, 774, 754, 802, 796, 741, 755, 737, 777, 799, 766, 762, 760, 749, 763, 849, 768, 758, 795, 755, 741, 756, 761, 781, 782, 793, 801, 775, 802, 786, 796, 763, 766, 792, 715, 728] payload_rows_per_dst=[9344, 9340, 9268, 9260, 9322, 9394, 9300, 9278] dup_rows_per_dst=[2964, 2916, 2876, 3000, 2944, 3000, 3080, 3018] combine_rows_per_src=[9344, 9340, 9268, 9260, 9322, 9394, 9300, 9278] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265510912 dispatch_meta_bytes=343176 combine_payload_bytes=265510912 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265854088 ingress_bytes=369319936 link_bytes=369319936 token_bytes=531365000 weight_movement_bytes=103809024 total_nvlink_bytes=635174024 weight_to_token_ratio=0.195363 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389721186e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265592832 dispatch_meta_bytes=343184 combine_payload_bytes=265592832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265936016 ingress_bytes=300195840 link_bytes=300195840 token_bytes=531528848 weight_movement_bytes=34603008 total_nvlink_bytes=566131856 weight_to_token_ratio=0.065101 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129612778e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=65024 dup_rows=23926 dup_rows_remote=20970 dedup_ratio=0.243388 router_tokens_per_expert=[753, 776, 760, 787, 796, 757, 783, 752, 781, 723, 726, 761, 810, 776, 823, 755, 728, 733, 743, 764, 741, 773, 749, 793, 785, 729, 814, 792, 744, 743, 741, 769, 789, 755, 766, 731, 777, 806, 778, 789, 792, 823, 769, 741, 778, 744, 779, 774, 776, 768, 754, 748, 773, 752, 805, 769, 759, 785, 789, 742, 785, 749, 733, 814] payload_rows_per_dst=[9260, 9354, 9158, 9240, 9324, 9408, 9336, 9298] dup_rows_per_dst=[3068, 2956, 2890, 2994, 3058, 2992, 2954, 3014] combine_rows_per_src=[9260, 9354, 9158, 9240, 9324, 9408, 9336, 9298] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=264732672 dispatch_meta_bytes=341904 combine_payload_bytes=264732672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265074576 ingress_bytes=299335680 link_bytes=299335680 token_bytes=529807248 weight_movement_bytes=34603008 total_nvlink_bytes=564410256 weight_to_token_ratio=0.065312 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126376065e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265437184 dispatch_meta_bytes=343440 combine_payload_bytes=265437184 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265780624 ingress_bytes=369246208 link_bytes=369246208 token_bytes=531217808 weight_movement_bytes=103809024 total_nvlink_bytes=635026832 weight_to_token_ratio=0.195417 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389443754e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265830400 dispatch_meta_bytes=343944 combine_payload_bytes=265830400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266174344 ingress_bytes=300433408 link_bytes=300433408 token_bytes=532004744 weight_movement_bytes=34603008 total_nvlink_bytes=566607752 weight_to_token_ratio=0.065043 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130506727e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=343720 combine_payload_bytes=266313728 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266657448 ingress_bytes=370122752 link_bytes=370122752 token_bytes=532971176 weight_movement_bytes=103809024 total_nvlink_bytes=636780200 weight_to_token_ratio=0.194774 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392742118e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265510912 dispatch_meta_bytes=343176 combine_payload_bytes=265510912 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265854088 ingress_bytes=369319936 link_bytes=369319936 token_bytes=531365000 weight_movement_bytes=103809024 total_nvlink_bytes=635174024 weight_to_token_ratio=0.195363 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389721186e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265592832 dispatch_meta_bytes=343184 combine_payload_bytes=265592832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265936016 ingress_bytes=300195840 link_bytes=300195840 token_bytes=531528848 weight_movement_bytes=34603008 total_nvlink_bytes=566131856 weight_to_token_ratio=0.065101 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129612778e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=343976 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266682280 ingress_bytes=300941312 link_bytes=300941312 token_bytes=533020584 weight_movement_bytes=34603008 total_nvlink_bytes=567623592 weight_to_token_ratio=0.064919 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132417929e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74506 payload_rows_remote=65206 dup_rows=23798 dup_rows_remote=20718 dedup_ratio=0.242086 router_tokens_per_expert=[801, 758, 781, 746, 782, 708, 760, 818, 754, 746, 755, 765, 760, 766, 804, 778, 759, 732, 759, 749, 804, 738, 784, 747, 740, 754, 764, 780, 762, 774, 754, 802, 796, 741, 755, 737, 777, 799, 766, 762, 760, 749, 763, 849, 768, 758, 795, 755, 741, 756, 761, 781, 782, 793, 801, 775, 802, 786, 796, 763, 766, 792, 715, 728] payload_rows_per_dst=[9344, 9340, 9268, 9260, 9322, 9394, 9300, 9278] dup_rows_per_dst=[2964, 2916, 2876, 3000, 2944, 3000, 3080, 3018] combine_rows_per_src=[9344, 9340, 9268, 9260, 9322, 9394, 9300, 9278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74132 payload_rows_remote=64780 dup_rows=24172 dup_rows_remote=21010 dedup_ratio=0.245890 router_tokens_per_expert=[785, 752, 788, 785, 746, 763, 769, 753, 769, 766, 793, 777, 716, 769, 755, 759, 781, 824, 779, 789, 760, 793, 768, 796, 798, 729, 752, 752, 779, 799, 720, 777, 766, 773, 790, 796, 767, 807, 790, 768, 748, 752, 793, 800, 765, 749, 731, 827, 746, 752, 767, 787, 794, 717, 774, 728, 791, 754, 763, 734, 747, 707, 731, 797] payload_rows_per_dst=[9272, 9218, 9432, 9214, 9352, 9346, 9160, 9138] dup_rows_per_dst=[3010, 2990, 3148, 2998, 3162, 2984, 2970, 2910] combine_rows_per_src=[9272, 9218, 9432, 9214, 9352, 9346, 9160, 9138] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=64886 dup_rows=24136 dup_rows_remote=21154 dedup_ratio=0.245524 router_tokens_per_expert=[803, 741, 795, 788, 719, 772, 743, 732, 791, 810, 760, 737, 809, 787, 787, 736, 782, 789, 762, 756, 740, 788, 805, 753, 805, 774, 743, 800, 739, 768, 793, 831, 741, 719, 774, 745, 768, 717, 749, 802, 751, 794, 714, 776, 768, 790, 785, 728, 765, 783, 773, 713, 768, 789, 782, 769, 764, 744, 799, 735, 781, 785, 789, 754] payload_rows_per_dst=[9282, 9484, 9222, 9336, 9082, 9392, 9334, 9036] dup_rows_per_dst=[2982, 3110, 2894, 3170, 2970, 3020, 3034, 2956] combine_rows_per_src=[9282, 9484, 9222, 9336, 9082, 9392, 9334, 9036] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=65072 dup_rows=24010 dup_rows_remote=21056 dedup_ratio=0.244242 router_tokens_per_expert=[784, 782, 790, 714, 746, 784, 788, 752, 734, 810, 774, 744, 731, 788, 808, 761, 776, 770, 806, 776, 769, 816, 746, 760, 750, 773, 750, 771, 751, 762, 774, 726, 763, 757, 761, 770, 728, 789, 777, 759, 764, 772, 771, 752, 758, 738, 764, 769, 756, 779, 781, 785, 785, 809, 782, 768, 775, 755, 757, 765, 807, 795, 737, 758] payload_rows_per_dst=[9350, 9304, 9444, 9132, 9234, 9222, 9316, 9292] dup_rows_per_dst=[2930, 2996, 2994, 2982, 2974, 2954, 3174, 3006] combine_rows_per_src=[9350, 9304, 9444, 9132, 9234, 9222, 9316, 9292] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74066 payload_rows_remote=64958 dup_rows=24238 dup_rows_remote=21330 dedup_ratio=0.246562 router_tokens_per_expert=[772, 736, 753, 761, 712, 770, 775, 706, 787, 754, 740, 757, 724, 741, 734, 771, 764, 792, 769, 781, 776, 804, 786, 773, 719, 744, 819, 769, 765, 758, 771, 772, 849, 800, 763, 804, 756, 737, 783, 764, 764, 739, 713, 815, 754, 780, 786, 762, 831, 765, 751, 751, 790, 812, 784, 768, 779, 764, 756, 754, 810, 756, 785, 772] payload_rows_per_dst=[9052, 9108, 9408, 9204, 9408, 9196, 9340, 9350] dup_rows_per_dst=[2918, 2908, 3082, 3030, 3104, 3030, 3164, 3002] combine_rows_per_src=[9052, 9108, 9408, 9204, 9408, 9196, 9340, 9350] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74016 payload_rows_remote=64720 dup_rows=24288 dup_rows_remote=21258 dedup_ratio=0.247070 router_tokens_per_expert=[831, 814, 811, 755, 796, 753, 756, 810, 710, 753, 765, 784, 717, 744, 764, 803, 792, 723, 770, 732, 743, 768, 721, 797, 820, 789, 787, 765, 752, 723, 770, 757, 737, 745, 708, 795, 790, 777, 788, 752, 758, 769, 751, 763, 743, 777, 789, 807, 733, 739, 769, 814, 753, 785, 807, 774, 771, 724, 770, 763, 797, 782, 771, 776] payload_rows_per_dst=[9534, 9102, 9190, 9296, 9198, 9200, 9296, 9200] dup_rows_per_dst=[3118, 2978, 2902, 3030, 2986, 3114, 3052, 3108] combine_rows_per_src=[9534, 9102, 9190, 9296, 9198, 9200, 9296, 9200] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265338880 dispatch_meta_bytes=343160 combine_payload_bytes=265338880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265682040 ingress_bytes=299941888 link_bytes=299941888 token_bytes=531020920 weight_movement_bytes=34603008 total_nvlink_bytes=565623928 weight_to_token_ratio=0.065163 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128657177e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=65008 dup_rows=24016 dup_rows_remote=21098 dedup_ratio=0.244303 router_tokens_per_expert=[710, 762, 792, 797, 823, 833, 767, 737, 787, 788, 811, 784, 793, 781, 761, 765, 806, 747, 761, 735, 731, 730, 820, 769, 757, 770, 719, 769, 743, 733, 739, 749, 766, 812, 780, 755, 749, 748, 800, 783, 754, 774, 745, 807, 711, 753, 759, 751, 796, 727, 749, 777, 768, 798, 725, 776, 782, 763, 799, 766, 825, 770, 746, 769] payload_rows_per_dst=[9352, 9426, 9280, 9034, 9390, 9126, 9264, 9416] dup_rows_per_dst=[3090, 3114, 2918, 2924, 2996, 2982, 2968, 3024] combine_rows_per_src=[9352, 9426, 9280, 9034, 9390, 9126, 9264, 9416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65090 dup_rows=23944 dup_rows_remote=20876 dedup_ratio=0.243571 router_tokens_per_expert=[720, 777, 809, 789, 769, 845, 781, 767, 769, 763, 740, 807, 821, 705, 786, 727, 709, 754, 766, 769, 770, 762, 768, 771, 767, 785, 787, 802, 740, 779, 750, 734, 717, 767, 807, 745, 771, 757, 741, 832, 770, 784, 744, 785, 797, 806, 747, 771, 772, 756, 760, 754, 776, 726, 780, 730, 763, 786, 728, 780, 752, 753, 792, 815] payload_rows_per_dst=[9208, 9208, 9260, 9548, 9266, 9418, 9182, 9270] dup_rows_per_dst=[2852, 3028, 2878, 3194, 3008, 2990, 2926, 3068] combine_rows_per_src=[9208, 9208, 9260, 9548, 9266, 9418, 9182, 9270] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=344160 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266117216 ingress_bytes=334979072 link_bytes=334979072 token_bytes=531890272 weight_movement_bytes=69206016 total_nvlink_bytes=601096288 weight_to_token_ratio=0.130113 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260499280e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74066 payload_rows_remote=64958 dup_rows=24238 dup_rows_remote=21330 dedup_ratio=0.246562 router_tokens_per_expert=[772, 736, 753, 761, 712, 770, 775, 706, 787, 754, 740, 757, 724, 741, 734, 771, 764, 792, 769, 781, 776, 804, 786, 773, 719, 744, 819, 769, 765, 758, 771, 772, 849, 800, 763, 804, 756, 737, 783, 764, 764, 739, 713, 815, 754, 780, 786, 762, 831, 765, 751, 751, 790, 812, 784, 768, 779, 764, 756, 754, 810, 756, 785, 772] payload_rows_per_dst=[9052, 9108, 9408, 9204, 9408, 9196, 9340, 9350] dup_rows_per_dst=[2918, 2908, 3082, 3030, 3104, 3030, 3164, 3002] combine_rows_per_src=[9052, 9108, 9408, 9204, 9408, 9196, 9340, 9350] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267083776 dispatch_meta_bytes=343696 combine_payload_bytes=267083776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267427472 ingress_bytes=301686784 link_bytes=301686784 token_bytes=534511248 weight_movement_bytes=34603008 total_nvlink_bytes=569114256 weight_to_token_ratio=0.064738 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135223080e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266534912 dispatch_meta_bytes=344512 combine_payload_bytes=266534912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266879424 ingress_bytes=301137920 link_bytes=301137920 token_bytes=533414336 weight_movement_bytes=34603008 total_nvlink_bytes=568017344 weight_to_token_ratio=0.064871 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133157749e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=345152 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266413120 ingress_bytes=300670976 link_bytes=300670976 token_bytes=532481088 weight_movement_bytes=34603008 total_nvlink_bytes=567084096 weight_to_token_ratio=0.064984 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131400677e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65090 dup_rows=23944 dup_rows_remote=20876 dedup_ratio=0.243571 router_tokens_per_expert=[720, 777, 809, 789, 769, 845, 781, 767, 769, 763, 740, 807, 821, 705, 786, 727, 709, 754, 766, 769, 770, 762, 768, 771, 767, 785, 787, 802, 740, 779, 750, 734, 717, 767, 807, 745, 771, 757, 741, 832, 770, 784, 744, 785, 797, 806, 747, 771, 772, 756, 760, 754, 776, 726, 780, 730, 763, 786, 728, 780, 752, 753, 792, 815] payload_rows_per_dst=[9208, 9208, 9260, 9548, 9266, 9418, 9182, 9270] dup_rows_per_dst=[2852, 3028, 2878, 3194, 3008, 2990, 2926, 3068] combine_rows_per_src=[9208, 9208, 9260, 9548, 9266, 9418, 9182, 9270] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265093120 dispatch_meta_bytes=343912 combine_payload_bytes=265093120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265437032 ingress_bytes=299696128 link_bytes=299696128 token_bytes=530530152 weight_movement_bytes=34603008 total_nvlink_bytes=565133160 weight_to_token_ratio=0.065223 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127732402e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=65008 dup_rows=24016 dup_rows_remote=21098 dedup_ratio=0.244303 router_tokens_per_expert=[710, 762, 792, 797, 823, 833, 767, 737, 787, 788, 811, 784, 793, 781, 761, 765, 806, 747, 761, 735, 731, 730, 820, 769, 757, 770, 719, 769, 743, 733, 739, 749, 766, 812, 780, 755, 749, 748, 800, 783, 754, 774, 745, 807, 711, 753, 759, 751, 796, 727, 749, 777, 768, 798, 725, 776, 782, 763, 799, 766, 825, 770, 746, 769] payload_rows_per_dst=[9352, 9426, 9280, 9034, 9390, 9126, 9264, 9416] dup_rows_per_dst=[3090, 3114, 2918, 2924, 2996, 2982, 2968, 3024] combine_rows_per_src=[9352, 9426, 9280, 9034, 9390, 9126, 9264, 9416] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=64886 dup_rows=24136 dup_rows_remote=21154 dedup_ratio=0.245524 router_tokens_per_expert=[803, 741, 795, 788, 719, 772, 743, 732, 791, 810, 760, 737, 809, 787, 787, 736, 782, 789, 762, 756, 740, 788, 805, 753, 805, 774, 743, 800, 739, 768, 793, 831, 741, 719, 774, 745, 768, 717, 749, 802, 751, 794, 714, 776, 768, 790, 785, 728, 765, 783, 773, 713, 768, 789, 782, 769, 764, 744, 799, 735, 781, 785, 789, 754] payload_rows_per_dst=[9282, 9484, 9222, 9336, 9082, 9392, 9334, 9036] dup_rows_per_dst=[2982, 3110, 2894, 3170, 2970, 3020, 3034, 2956] combine_rows_per_src=[9282, 9484, 9222, 9336, 9082, 9392, 9334, 9036] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74132 payload_rows_remote=64780 dup_rows=24172 dup_rows_remote=21010 dedup_ratio=0.245890 router_tokens_per_expert=[785, 752, 788, 785, 746, 763, 769, 753, 769, 766, 793, 777, 716, 769, 755, 759, 781, 824, 779, 789, 760, 793, 768, 796, 798, 729, 752, 752, 779, 799, 720, 777, 766, 773, 790, 796, 767, 807, 790, 768, 748, 752, 793, 800, 765, 749, 731, 827, 746, 752, 767, 787, 794, 717, 774, 728, 791, 754, 763, 734, 747, 707, 731, 797] payload_rows_per_dst=[9272, 9218, 9432, 9214, 9352, 9346, 9160, 9138] dup_rows_per_dst=[3010, 2990, 3148, 2998, 3162, 2984, 2970, 2910] combine_rows_per_src=[9272, 9218, 9432, 9214, 9352, 9346, 9160, 9138] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266608640 dispatch_meta_bytes=343864 combine_payload_bytes=266608640 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266952504 ingress_bytes=405020672 link_bytes=405020672 token_bytes=533561144 weight_movement_bytes=138412032 total_nvlink_bytes=671973176 weight_to_token_ratio=0.259412 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524060182e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266272768 dispatch_meta_bytes=344424 combine_payload_bytes=266272768 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266617192 ingress_bytes=370081792 link_bytes=370081792 token_bytes=532889960 weight_movement_bytes=103809024 total_nvlink_bytes=636698984 weight_to_token_ratio=0.194804 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392587989e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=65064 dup_rows=24072 dup_rows_remote=21094 dedup_ratio=0.244873 router_tokens_per_expert=[735, 725, 752, 796, 785, 767, 778, 766, 775, 775, 817, 846, 748, 755, 723, 775, 767, 781, 846, 761, 766, 783, 769, 798, 767, 786, 743, 766, 799, 807, 775, 753, 822, 749, 812, 785, 748, 760, 740, 756, 775, 732, 755, 764, 738, 778, 773, 758, 775, 719, 743, 777, 751, 766, 770, 787, 774, 752, 724, 739, 721, 729, 758, 837] payload_rows_per_dst=[9246, 9338, 9438, 9380, 9330, 9168, 9232, 9100] dup_rows_per_dst=[2962, 3090, 3104, 3012, 3014, 2978, 2944, 2968] combine_rows_per_src=[9246, 9338, 9438, 9380, 9330, 9168, 9232, 9100] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=344160 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266117216 ingress_bytes=334979072 link_bytes=334979072 token_bytes=531890272 weight_movement_bytes=69206016 total_nvlink_bytes=601096288 weight_to_token_ratio=0.130113 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260499280e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266608640 dispatch_meta_bytes=343864 combine_payload_bytes=266608640 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266952504 ingress_bytes=405020672 link_bytes=405020672 token_bytes=533561144 weight_movement_bytes=138412032 total_nvlink_bytes=671973176 weight_to_token_ratio=0.259412 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524060182e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=345152 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266413120 ingress_bytes=300670976 link_bytes=300670976 token_bytes=532481088 weight_movement_bytes=34603008 total_nvlink_bytes=567084096 weight_to_token_ratio=0.064984 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131400677e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74150 payload_rows_remote=64972 dup_rows=24154 dup_rows_remote=21298 dedup_ratio=0.245707 router_tokens_per_expert=[792, 786, 811, 778, 749, 768, 733, 783, 761, 774, 754, 711, 747, 811, 796, 766, 774, 785, 740, 801, 714, 760, 816, 810, 778, 743, 780, 735, 777, 775, 783, 766, 768, 737, 765, 746, 734, 752, 758, 757, 730, 747, 750, 784, 782, 762, 769, 768, 772, 762, 747, 830, 817, 820, 763, 749, 774, 793, 757, 838, 738, 741, 757, 728] payload_rows_per_dst=[9380, 9186, 9328, 9232, 9178, 9264, 9404, 9178] dup_rows_per_dst=[3020, 3054, 3072, 3042, 2856, 2920, 3116, 3074] combine_rows_per_src=[9380, 9186, 9328, 9232, 9178, 9264, 9404, 9178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74506 payload_rows_remote=65274 dup_rows=23798 dup_rows_remote=20792 dedup_ratio=0.242086 router_tokens_per_expert=[752, 770, 768, 723, 780, 789, 757, 780, 757, 776, 770, 784, 773, 739, 765, 750, 761, 791, 736, 750, 795, 795, 733, 773, 753, 735, 776, 785, 749, 742, 778, 788, 765, 783, 813, 771, 750, 763, 760, 802, 811, 769, 732, 790, 756, 713, 733, 745, 731, 782, 742, 771, 727, 807, 795, 785, 775, 837, 785, 733, 803, 742, 824, 784] payload_rows_per_dst=[9232, 9270, 9308, 9242, 9434, 9182, 9402, 9436] dup_rows_per_dst=[3006, 2958, 2968, 2962, 2980, 2916, 2878, 3130] combine_rows_per_src=[9232, 9270, 9308, 9242, 9434, 9182, 9402, 9436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266125312 dispatch_meta_bytes=345080 combine_payload_bytes=266125312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266470392 ingress_bytes=300728320 link_bytes=300728320 token_bytes=532595704 weight_movement_bytes=34603008 total_nvlink_bytes=567198712 weight_to_token_ratio=0.064970 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131616458e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74220 payload_rows_remote=64948 dup_rows=24084 dup_rows_remote=21232 dedup_ratio=0.244995 router_tokens_per_expert=[799, 776, 737, 761, 781, 774, 774, 775, 771, 747, 797, 748, 806, 758, 810, 728, 804, 776, 736, 783, 729, 774, 777, 747, 771, 803, 746, 735, 786, 744, 782, 739, 743, 749, 812, 736, 783, 740, 770, 754, 760, 779, 763, 780, 731, 778, 816, 758, 757, 761, 771, 748, 741, 795, 756, 733, 784, 758, 780, 837, 827, 742, 754, 782] payload_rows_per_dst=[9290, 9278, 9186, 9344, 9174, 9256, 9272, 9420] dup_rows_per_dst=[3064, 3052, 3066, 2868, 3000, 3074, 2852, 3108] combine_rows_per_src=[9290, 9278, 9186, 9344, 9174, 9256, 9272, 9420] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=65012 dup_rows=24016 dup_rows_remote=21048 dedup_ratio=0.244303 router_tokens_per_expert=[749, 778, 791, 804, 833, 788, 759, 794, 762, 754, 746, 781, 776, 775, 774, 754, 799, 730, 743, 766, 769, 776, 742, 760, 768, 835, 723, 742, 778, 750, 747, 774, 757, 780, 780, 774, 817, 805, 752, 774, 786, 790, 756, 719, 807, 780, 727, 783, 748, 756, 763, 753, 760, 738, 765, 759, 783, 747, 741, 759, 757, 808, 747, 761] payload_rows_per_dst=[9436, 9276, 9258, 9252, 9386, 9248, 9220, 9212] dup_rows_per_dst=[3156, 2968, 2912, 2982, 3092, 3048, 2864, 2994] combine_rows_per_src=[9436, 9276, 9258, 9252, 9386, 9248, 9220, 9212] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265338880 dispatch_meta_bytes=343160 combine_payload_bytes=265338880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265682040 ingress_bytes=299941888 link_bytes=299941888 token_bytes=531020920 weight_movement_bytes=34603008 total_nvlink_bytes=565623928 weight_to_token_ratio=0.065163 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128657177e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=65064 dup_rows=24072 dup_rows_remote=21094 dedup_ratio=0.244873 router_tokens_per_expert=[735, 725, 752, 796, 785, 767, 778, 766, 775, 775, 817, 846, 748, 755, 723, 775, 767, 781, 846, 761, 766, 783, 769, 798, 767, 786, 743, 766, 799, 807, 775, 753, 822, 749, 812, 785, 748, 760, 740, 756, 775, 732, 755, 764, 738, 778, 773, 758, 775, 719, 743, 777, 751, 766, 770, 787, 774, 752, 724, 739, 721, 729, 758, 837] payload_rows_per_dst=[9246, 9338, 9438, 9380, 9330, 9168, 9232, 9100] dup_rows_per_dst=[2962, 3090, 3104, 3012, 3014, 2978, 2944, 2968] combine_rows_per_src=[9246, 9338, 9438, 9380, 9330, 9168, 9232, 9100] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74304 payload_rows_remote=64886 dup_rows=24000 dup_rows_remote=20866 dedup_ratio=0.244141 router_tokens_per_expert=[746, 818, 749, 758, 778, 743, 747, 768, 820, 725, 761, 766, 742, 769, 784, 761, 751, 766, 800, 726, 770, 771, 748, 738, 786, 788, 747, 771, 775, 786, 812, 811, 775, 794, 780, 696, 727, 767, 747, 791, 732, 767, 741, 774, 847, 752, 807, 766, 727, 740, 788, 801, 789, 846, 693, 809, 767, 795, 771, 789, 743, 764, 751, 735] payload_rows_per_dst=[9246, 9218, 9280, 9418, 9256, 9370, 9332, 9184] dup_rows_per_dst=[2968, 3038, 2860, 3134, 2898, 3002, 3054, 3046] combine_rows_per_src=[9246, 9218, 9280, 9418, 9256, 9370, 9332, 9184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266502144 dispatch_meta_bytes=344632 combine_payload_bytes=266502144 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266846776 ingress_bytes=301105152 link_bytes=301105152 token_bytes=533348920 weight_movement_bytes=34603008 total_nvlink_bytes=567951928 weight_to_token_ratio=0.064879 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133034446e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266289152 dispatch_meta_bytes=344240 combine_payload_bytes=266289152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266633392 ingress_bytes=300892160 link_bytes=300892160 token_bytes=532922544 weight_movement_bytes=34603008 total_nvlink_bytes=567525552 weight_to_token_ratio=0.064931 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132232974e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266027008 dispatch_meta_bytes=344720 combine_payload_bytes=266027008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266371728 ingress_bytes=300630016 link_bytes=300630016 token_bytes=532398736 weight_movement_bytes=34603008 total_nvlink_bytes=567001744 weight_to_token_ratio=0.064995 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131246548e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267362304 dispatch_meta_bytes=344264 combine_payload_bytes=267362304 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267706568 ingress_bytes=336568320 link_bytes=336568320 token_bytes=535068872 weight_movement_bytes=69206016 total_nvlink_bytes=604274888 weight_to_token_ratio=0.129340 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266479492e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64956 dup_rows=24114 dup_rows_remote=21056 dedup_ratio=0.245300 router_tokens_per_expert=[762, 812, 745, 812, 793, 783, 757, 749, 805, 789, 708, 762, 801, 719, 801, 772, 756, 752, 773, 781, 708, 747, 839, 760, 811, 766, 779, 775, 806, 782, 717, 755, 793, 800, 732, 796, 751, 819, 698, 798, 792, 764, 787, 740, 808, 786, 745, 755, 743, 756, 786, 766, 746, 764, 707, 741, 752, 740, 783, 776, 742, 778, 771, 760] payload_rows_per_dst=[9260, 9328, 9360, 9050, 9242, 9428, 9288, 9234] dup_rows_per_dst=[2932, 2992, 2872, 2858, 3138, 3154, 3110, 3058] combine_rows_per_src=[9260, 9328, 9360, 9050, 9242, 9428, 9288, 9234] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74026 payload_rows_remote=64854 dup_rows=24278 dup_rows_remote=21228 dedup_ratio=0.246969 router_tokens_per_expert=[744, 727, 765, 751, 737, 769, 767, 743, 742, 738, 763, 721, 794, 792, 736, 808, 775, 767, 751, 782, 730, 771, 743, 792, 801, 789, 774, 727, 720, 768, 760, 807, 815, 765, 797, 805, 750, 749, 787, 747, 787, 770, 716, 776, 780, 775, 837, 759, 773, 758, 784, 749, 751, 794, 780, 740, 725, 770, 800, 772, 823, 792, 789, 783] payload_rows_per_dst=[9076, 9224, 9172, 9294, 9302, 9290, 9264, 9404] dup_rows_per_dst=[2930, 2964, 3050, 2998, 3128, 3110, 2994, 3104] combine_rows_per_src=[9076, 9224, 9172, 9294, 9302, 9290, 9264, 9404] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=343008 combine_payload_bytes=265773056 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266116064 ingress_bytes=404185088 link_bytes=404185088 token_bytes=531889120 weight_movement_bytes=138412032 total_nvlink_bytes=670301152 weight_to_token_ratio=0.260227 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520915947e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=65012 dup_rows=24016 dup_rows_remote=21048 dedup_ratio=0.244303 router_tokens_per_expert=[749, 778, 791, 804, 833, 788, 759, 794, 762, 754, 746, 781, 776, 775, 774, 754, 799, 730, 743, 766, 769, 776, 742, 760, 768, 835, 723, 742, 778, 750, 747, 774, 757, 780, 780, 774, 817, 805, 752, 774, 786, 790, 756, 719, 807, 780, 727, 783, 748, 756, 763, 753, 760, 738, 765, 759, 783, 747, 741, 759, 757, 808, 747, 761] payload_rows_per_dst=[9436, 9276, 9258, 9252, 9386, 9248, 9220, 9212] dup_rows_per_dst=[3156, 2968, 2912, 2982, 3092, 3048, 2864, 2994] combine_rows_per_src=[9436, 9276, 9258, 9252, 9386, 9248, 9220, 9212] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74506 payload_rows_remote=65274 dup_rows=23798 dup_rows_remote=20792 dedup_ratio=0.242086 router_tokens_per_expert=[752, 770, 768, 723, 780, 789, 757, 780, 757, 776, 770, 784, 773, 739, 765, 750, 761, 791, 736, 750, 795, 795, 733, 773, 753, 735, 776, 785, 749, 742, 778, 788, 765, 783, 813, 771, 750, 763, 760, 802, 811, 769, 732, 790, 756, 713, 733, 745, 731, 782, 742, 771, 727, 807, 795, 785, 775, 837, 785, 733, 803, 742, 824, 784] payload_rows_per_dst=[9232, 9270, 9308, 9242, 9434, 9182, 9402, 9436] dup_rows_per_dst=[3006, 2958, 2968, 2962, 2980, 2916, 2878, 3130] combine_rows_per_src=[9232, 9270, 9308, 9242, 9434, 9182, 9402, 9436] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74150 payload_rows_remote=64972 dup_rows=24154 dup_rows_remote=21298 dedup_ratio=0.245707 router_tokens_per_expert=[792, 786, 811, 778, 749, 768, 733, 783, 761, 774, 754, 711, 747, 811, 796, 766, 774, 785, 740, 801, 714, 760, 816, 810, 778, 743, 780, 735, 777, 775, 783, 766, 768, 737, 765, 746, 734, 752, 758, 757, 730, 747, 750, 784, 782, 762, 769, 768, 772, 762, 747, 830, 817, 820, 763, 749, 774, 793, 757, 838, 738, 741, 757, 728] payload_rows_per_dst=[9380, 9186, 9328, 9232, 9178, 9264, 9404, 9178] dup_rows_per_dst=[3020, 3054, 3072, 3042, 2856, 2920, 3116, 3074] combine_rows_per_src=[9380, 9186, 9328, 9232, 9178, 9264, 9404, 9178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266502144 dispatch_meta_bytes=344632 combine_payload_bytes=266502144 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266846776 ingress_bytes=301105152 link_bytes=301105152 token_bytes=533348920 weight_movement_bytes=34603008 total_nvlink_bytes=567951928 weight_to_token_ratio=0.064879 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133034446e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=65050 dup_rows=24056 dup_rows_remote=21072 dedup_ratio=0.244710 router_tokens_per_expert=[820, 815, 759, 767, 783, 770, 742, 763, 749, 771, 782, 788, 748, 766, 750, 753, 803, 794, 778, 790, 739, 797, 745, 780, 826, 729, 765, 793, 749, 774, 792, 781, 756, 753, 759, 767, 742, 780, 758, 776, 785, 743, 772, 790, 805, 756, 722, 769, 706, 758, 799, 770, 759, 766, 730, 769, 779, 832, 785, 751, 740, 731, 736, 747] payload_rows_per_dst=[9394, 9174, 9458, 9360, 9198, 9240, 9170, 9254] dup_rows_per_dst=[3044, 3040, 2994, 3058, 2984, 3044, 2944, 2948] combine_rows_per_src=[9394, 9174, 9458, 9360, 9198, 9240, 9170, 9254] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266027008 dispatch_meta_bytes=344720 combine_payload_bytes=266027008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266371728 ingress_bytes=300630016 link_bytes=300630016 token_bytes=532398736 weight_movement_bytes=34603008 total_nvlink_bytes=567001744 weight_to_token_ratio=0.064995 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131246548e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=343008 combine_payload_bytes=265773056 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266116064 ingress_bytes=404185088 link_bytes=404185088 token_bytes=531889120 weight_movement_bytes=138412032 total_nvlink_bytes=670301152 weight_to_token_ratio=0.260227 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520915947e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=344048 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266403824 ingress_bytes=300662784 link_bytes=300662784 token_bytes=532463600 weight_movement_bytes=34603008 total_nvlink_bytes=567066608 weight_to_token_ratio=0.064987 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131369851e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265641984 dispatch_meta_bytes=344328 combine_payload_bytes=265641984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265986312 ingress_bytes=300244992 link_bytes=300244992 token_bytes=531628296 weight_movement_bytes=34603008 total_nvlink_bytes=566231304 weight_to_token_ratio=0.065089 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129797733e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266289152 dispatch_meta_bytes=344240 combine_payload_bytes=266289152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266633392 ingress_bytes=300892160 link_bytes=300892160 token_bytes=532922544 weight_movement_bytes=34603008 total_nvlink_bytes=567525552 weight_to_token_ratio=0.064931 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132232974e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74464 payload_rows_remote=65130 dup_rows=23840 dup_rows_remote=20814 dedup_ratio=0.242513 router_tokens_per_expert=[783, 770, 803, 751, 779, 742, 783, 769, 771, 728, 786, 755, 765, 785, 749, 779, 760, 757, 788, 746, 768, 847, 763, 809, 782, 719, 774, 807, 765, 753, 728, 759, 723, 700, 775, 801, 801, 768, 757, 807, 751, 769, 747, 776, 728, 794, 779, 752, 780, 785, 755, 757, 763, 747, 797, 774, 777, 777, 760, 806, 783, 739, 758, 743] payload_rows_per_dst=[9334, 9346, 9388, 9120, 9258, 9308, 9384, 9326] dup_rows_per_dst=[3026, 3002, 3088, 2942, 3006, 2884, 2932, 2960] combine_rows_per_src=[9334, 9346, 9388, 9120, 9258, 9308, 9384, 9326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266444800 dispatch_meta_bytes=344488 combine_payload_bytes=266444800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266789288 ingress_bytes=301047808 link_bytes=301047808 token_bytes=533234088 weight_movement_bytes=34603008 total_nvlink_bytes=567837096 weight_to_token_ratio=0.064893 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132818665e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266772480 dispatch_meta_bytes=343776 combine_payload_bytes=266772480 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267116256 ingress_bytes=405184512 link_bytes=405184512 token_bytes=533888736 weight_movement_bytes=138412032 total_nvlink_bytes=672300768 weight_to_token_ratio=0.259253 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524676699e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74418 payload_rows_remote=65322 dup_rows=23886 dup_rows_remote=21082 dedup_ratio=0.242981 router_tokens_per_expert=[762, 761, 815, 802, 788, 773, 778, 720, 804, 797, 768, 796, 759, 771, 748, 752, 818, 780, 735, 800, 733, 753, 758, 751, 789, 838, 781, 773, 809, 771, 755, 774, 790, 776, 743, 717, 685, 778, 805, 741, 766, 700, 738, 746, 747, 779, 752, 722, 796, 768, 794, 748, 743, 781, 785, 758, 757, 790, 795, 762, 756, 801, 741, 780] payload_rows_per_dst=[9416, 9358, 9354, 9442, 9192, 9096, 9300, 9260] dup_rows_per_dst=[2982, 3032, 2902, 3138, 2878, 2804, 3046, 3104] combine_rows_per_src=[9416, 9358, 9354, 9442, 9192, 9096, 9300, 9260] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=64922 dup_rows=24016 dup_rows_remote=20900 dedup_ratio=0.244303 router_tokens_per_expert=[774, 818, 745, 779, 767, 764, 815, 774, 731, 789, 781, 801, 705, 828, 808, 798, 724, 777, 709, 759, 704, 731, 791, 731, 765, 759, 747, 791, 812, 776, 787, 741, 766, 812, 766, 767, 719, 758, 824, 757, 759, 759, 770, 791, 731, 775, 724, 767, 794, 787, 747, 778, 810, 794, 792, 765, 762, 756, 760, 744, 721, 782, 764, 770] payload_rows_per_dst=[9372, 9366, 9100, 9290, 9284, 9186, 9482, 9208] dup_rows_per_dst=[3100, 3116, 2752, 3066, 3054, 2966, 3052, 2910] combine_rows_per_src=[9372, 9366, 9100, 9290, 9284, 9186, 9482, 9208] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267362304 dispatch_meta_bytes=344264 combine_payload_bytes=267362304 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267706568 ingress_bytes=336568320 link_bytes=336568320 token_bytes=535068872 weight_movement_bytes=69206016 total_nvlink_bytes=604274888 weight_to_token_ratio=0.129340 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266479492e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266125312 dispatch_meta_bytes=345080 combine_payload_bytes=266125312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266470392 ingress_bytes=300728320 link_bytes=300728320 token_bytes=532595704 weight_movement_bytes=34603008 total_nvlink_bytes=567198712 weight_to_token_ratio=0.064970 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131616458e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74182 payload_rows_remote=64998 dup_rows=24122 dup_rows_remote=21104 dedup_ratio=0.245382 router_tokens_per_expert=[779, 756, 785, 791, 795, 765, 757, 799, 794, 779, 758, 714, 780, 782, 792, 801, 796, 736, 808, 781, 733, 770, 793, 744, 800, 801, 786, 750, 775, 772, 772, 768, 769, 709, 753, 749, 772, 771, 764, 748, 745, 779, 749, 765, 820, 757, 762, 736, 745, 753, 766, 771, 774, 772, 768, 752, 734, 739, 787, 786, 770, 749, 766, 760] payload_rows_per_dst=[9350, 9310, 9320, 9428, 9196, 9230, 9184, 9164] dup_rows_per_dst=[3104, 3090, 3002, 3020, 2874, 2996, 3018, 3018] combine_rows_per_src=[9350, 9310, 9320, 9428, 9196, 9230, 9184, 9164] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=64964 dup_rows=23944 dup_rows_remote=20770 dedup_ratio=0.243571 router_tokens_per_expert=[757, 829, 786, 741, 815, 775, 773, 769, 760, 771, 811, 747, 748, 744, 730, 764, 811, 681, 747, 783, 836, 770, 817, 753, 739, 764, 770, 793, 758, 830, 833, 798, 738, 741, 711, 794, 814, 747, 726, 819, 798, 756, 787, 763, 749, 787, 808, 782, 746, 729, 746, 701, 786, 709, 827, 777, 759, 745, 731, 716, 761, 771, 764, 761] payload_rows_per_dst=[9410, 9264, 9338, 9396, 9294, 9370, 9138, 9150] dup_rows_per_dst=[3080, 2886, 3058, 3174, 2886, 3090, 2904, 2866] combine_rows_per_src=[9410, 9264, 9338, 9396, 9294, 9370, 9138, 9150] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267558912 dispatch_meta_bytes=345616 combine_payload_bytes=267558912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267904528 ingress_bytes=302161920 link_bytes=302161920 token_bytes=535463440 weight_movement_bytes=34603008 total_nvlink_bytes=570066448 weight_to_token_ratio=0.064623 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137010979e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265920512 dispatch_meta_bytes=343288 combine_payload_bytes=265920512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266263800 ingress_bytes=300523520 link_bytes=300523520 token_bytes=532184312 weight_movement_bytes=34603008 total_nvlink_bytes=566787320 weight_to_token_ratio=0.065021 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130845812e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74182 payload_rows_remote=64998 dup_rows=24122 dup_rows_remote=21104 dedup_ratio=0.245382 router_tokens_per_expert=[779, 756, 785, 791, 795, 765, 757, 799, 794, 779, 758, 714, 780, 782, 792, 801, 796, 736, 808, 781, 733, 770, 793, 744, 800, 801, 786, 750, 775, 772, 772, 768, 769, 709, 753, 749, 772, 771, 764, 748, 745, 779, 749, 765, 820, 757, 762, 736, 745, 753, 766, 771, 774, 772, 768, 752, 734, 739, 787, 786, 770, 749, 766, 760] payload_rows_per_dst=[9350, 9310, 9320, 9428, 9196, 9230, 9184, 9164] dup_rows_per_dst=[3104, 3090, 3002, 3020, 2874, 2996, 3018, 3018] combine_rows_per_src=[9350, 9310, 9320, 9428, 9196, 9230, 9184, 9164] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74156 payload_rows_remote=64516 dup_rows=24148 dup_rows_remote=20772 dedup_ratio=0.245646 router_tokens_per_expert=[773, 746, 763, 833, 759, 772, 768, 814, 726, 750, 728, 769, 781, 764, 738, 784, 774, 775, 779, 738, 768, 798, 762, 800, 788, 762, 794, 790, 797, 779, 761, 760, 777, 750, 746, 779, 773, 729, 786, 759, 760, 797, 734, 761, 746, 736, 811, 804, 757, 776, 747, 774, 709, 762, 793, 714, 772, 784, 763, 767, 773, 725, 797, 798] payload_rows_per_dst=[8978, 9072, 9368, 8756, 9602, 9384, 9356, 9640] dup_rows_per_dst=[2682, 3008, 3074, 2668, 3184, 3054, 3102, 3376] combine_rows_per_src=[8978, 9072, 9368, 8756, 9602, 9384, 9356, 9640] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266231808 dispatch_meta_bytes=344408 combine_payload_bytes=266231808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266576216 ingress_bytes=300834816 link_bytes=300834816 token_bytes=532808024 weight_movement_bytes=34603008 total_nvlink_bytes=567411032 weight_to_token_ratio=0.064945 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132017193e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74218 payload_rows_remote=64926 dup_rows=24086 dup_rows_remote=21140 dedup_ratio=0.245015 router_tokens_per_expert=[765, 782, 827, 775, 780, 721, 778, 771, 749, 745, 801, 773, 781, 752, 737, 829, 766, 772, 731, 732, 755, 804, 790, 769, 762, 769, 773, 765, 827, 808, 749, 833, 780, 765, 787, 730, 767, 785, 720, 762, 782, 786, 717, 763, 791, 803, 743, 772, 797, 790, 763, 763, 794, 697, 746, 720, 786, 752, 794, 754, 723, 729, 746, 774] payload_rows_per_dst=[9350, 9312, 9292, 9386, 9178, 9290, 9178, 9232] dup_rows_per_dst=[3048, 3022, 2946, 3186, 3014, 3024, 2962, 2884] combine_rows_per_src=[9350, 9312, 9292, 9386, 9178, 9290, 9178, 9232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=342936 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266435480 ingress_bytes=404504576 link_bytes=404504576 token_bytes=532528024 weight_movement_bytes=138412032 total_nvlink_bytes=670940056 weight_to_token_ratio=0.259915 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522118154e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=64922 dup_rows=24016 dup_rows_remote=20900 dedup_ratio=0.244303 router_tokens_per_expert=[774, 818, 745, 779, 767, 764, 815, 774, 731, 789, 781, 801, 705, 828, 808, 798, 724, 777, 709, 759, 704, 731, 791, 731, 765, 759, 747, 791, 812, 776, 787, 741, 766, 812, 766, 767, 719, 758, 824, 757, 759, 759, 770, 791, 731, 775, 724, 767, 794, 787, 747, 778, 810, 794, 792, 765, 762, 756, 760, 744, 721, 782, 764, 770] payload_rows_per_dst=[9372, 9366, 9100, 9290, 9284, 9186, 9482, 9208] dup_rows_per_dst=[3100, 3116, 2752, 3066, 3054, 2966, 3052, 2910] combine_rows_per_src=[9372, 9366, 9100, 9290, 9284, 9186, 9482, 9208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74136 payload_rows_remote=64734 dup_rows=24168 dup_rows_remote=21074 dedup_ratio=0.245850 router_tokens_per_expert=[762, 767, 804, 773, 773, 814, 750, 720, 747, 801, 816, 776, 755, 745, 750, 790, 754, 745, 794, 765, 719, 772, 766, 750, 774, 784, 806, 746, 806, 772, 770, 767, 799, 802, 758, 822, 754, 792, 773, 748, 783, 777, 747, 792, 746, 715, 783, 773, 744, 777, 733, 739, 724, 751, 820, 734, 709, 745, 790, 795, 767, 772, 755, 800] payload_rows_per_dst=[9240, 9328, 9218, 9366, 9402, 9282, 9042, 9258] dup_rows_per_dst=[3086, 3032, 2912, 3084, 3094, 2950, 3002, 3008] combine_rows_per_src=[9240, 9328, 9218, 9366, 9402, 9282, 9042, 9258] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267558912 dispatch_meta_bytes=345616 combine_payload_bytes=267558912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267904528 ingress_bytes=302161920 link_bytes=302161920 token_bytes=535463440 weight_movement_bytes=34603008 total_nvlink_bytes=570066448 weight_to_token_ratio=0.064623 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137010979e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74464 payload_rows_remote=65130 dup_rows=23840 dup_rows_remote=20814 dedup_ratio=0.242513 router_tokens_per_expert=[783, 770, 803, 751, 779, 742, 783, 769, 771, 728, 786, 755, 765, 785, 749, 779, 760, 757, 788, 746, 768, 847, 763, 809, 782, 719, 774, 807, 765, 753, 728, 759, 723, 700, 775, 801, 801, 768, 757, 807, 751, 769, 747, 776, 728, 794, 779, 752, 780, 785, 755, 757, 763, 747, 797, 774, 777, 777, 760, 806, 783, 739, 758, 743] payload_rows_per_dst=[9334, 9346, 9388, 9120, 9258, 9308, 9384, 9326] dup_rows_per_dst=[3026, 3002, 3088, 2942, 3006, 2884, 2932, 2960] combine_rows_per_src=[9334, 9346, 9388, 9120, 9258, 9308, 9384, 9326] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=65050 dup_rows=24056 dup_rows_remote=21072 dedup_ratio=0.244710 router_tokens_per_expert=[820, 815, 759, 767, 783, 770, 742, 763, 749, 771, 782, 788, 748, 766, 750, 753, 803, 794, 778, 790, 739, 797, 745, 780, 826, 729, 765, 793, 749, 774, 792, 781, 756, 753, 759, 767, 742, 780, 758, 776, 785, 743, 772, 790, 805, 756, 722, 769, 706, 758, 799, 770, 759, 766, 730, 769, 779, 832, 785, 751, 740, 731, 736, 747] payload_rows_per_dst=[9394, 9174, 9458, 9360, 9198, 9240, 9170, 9254] dup_rows_per_dst=[3044, 3040, 2994, 3058, 2984, 3044, 2944, 2948] combine_rows_per_src=[9394, 9174, 9458, 9360, 9198, 9240, 9170, 9254] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=342936 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266435480 ingress_bytes=404504576 link_bytes=404504576 token_bytes=532528024 weight_movement_bytes=138412032 total_nvlink_bytes=670940056 weight_to_token_ratio=0.259915 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522118154e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266231808 dispatch_meta_bytes=344408 combine_payload_bytes=266231808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266576216 ingress_bytes=300834816 link_bytes=300834816 token_bytes=532808024 weight_movement_bytes=34603008 total_nvlink_bytes=567411032 weight_to_token_ratio=0.064945 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132017193e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=264257536 dispatch_meta_bytes=341152 combine_payload_bytes=264257536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264598688 ingress_bytes=298860544 link_bytes=298860544 token_bytes=528856224 weight_movement_bytes=34603008 total_nvlink_bytes=563459232 weight_to_token_ratio=0.065430 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.124588167e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74116 payload_rows_remote=64712 dup_rows=24188 dup_rows_remote=21164 dedup_ratio=0.246053 router_tokens_per_expert=[758, 789, 776, 729, 760, 751, 747, 740, 793, 767, 770, 783, 797, 765, 701, 815, 759, 769, 776, 736, 752, 749, 736, 774, 773, 776, 783, 779, 757, 748, 782, 763, 783, 795, 772, 781, 734, 760, 727, 746, 738, 805, 738, 780, 796, 792, 792, 773, 765, 765, 746, 799, 758, 769, 749, 795, 805, 718, 788, 781, 797, 766, 808, 778] payload_rows_per_dst=[9132, 9280, 9156, 9294, 9204, 9404, 9294, 9352] dup_rows_per_dst=[2968, 3102, 2946, 3028, 2992, 3024, 2998, 3130] combine_rows_per_src=[9132, 9280, 9156, 9294, 9204, 9404, 9294, 9352] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265936896 dispatch_meta_bytes=344264 combine_payload_bytes=265936896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266281160 ingress_bytes=300539904 link_bytes=300539904 token_bytes=532218056 weight_movement_bytes=34603008 total_nvlink_bytes=566821064 weight_to_token_ratio=0.065017 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130907463e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74336 payload_rows_remote=65172 dup_rows=23968 dup_rows_remote=21128 dedup_ratio=0.243815 router_tokens_per_expert=[758, 741, 789, 792, 742, 834, 750, 778, 775, 765, 753, 752, 738, 723, 772, 724, 755, 819, 774, 738, 799, 735, 767, 762, 820, 825, 750, 782, 769, 806, 741, 741, 747, 788, 781, 759, 765, 792, 750, 712, 765, 797, 762, 778, 774, 767, 727, 765, 758, 749, 763, 790, 806, 799, 802, 769, 726, 785, 755, 775, 735, 828, 722, 792] payload_rows_per_dst=[9286, 9164, 9304, 9370, 9294, 9360, 9368, 9190] dup_rows_per_dst=[3082, 2840, 2994, 3098, 2894, 2910, 3104, 3046] combine_rows_per_src=[9286, 9164, 9304, 9370, 9294, 9360, 9368, 9190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266772480 dispatch_meta_bytes=343776 combine_payload_bytes=266772480 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267116256 ingress_bytes=405184512 link_bytes=405184512 token_bytes=533888736 weight_movement_bytes=138412032 total_nvlink_bytes=672300768 weight_to_token_ratio=0.259253 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524676699e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74332 payload_rows_remote=64992 dup_rows=23972 dup_rows_remote=20958 dedup_ratio=0.243856 router_tokens_per_expert=[743, 737, 717, 801, 774, 760, 739, 773, 750, 754, 772, 793, 795, 783, 742, 764, 787, 744, 806, 739, 778, 758, 723, 761, 766, 755, 724, 759, 792, 787, 814, 764, 772, 753, 753, 725, 792, 796, 801, 754, 785, 765, 820, 789, 771, 806, 744, 802, 786, 794, 769, 720, 754, 781, 790, 783, 791, 736, 761, 741, 751, 816, 745, 752] payload_rows_per_dst=[9212, 9302, 9220, 9338, 9326, 9396, 9340, 9198] dup_rows_per_dst=[2876, 3004, 2972, 2984, 2966, 3168, 3014, 2988] combine_rows_per_src=[9212, 9302, 9220, 9338, 9326, 9396, 9340, 9198] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266444800 dispatch_meta_bytes=344488 combine_payload_bytes=266444800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266789288 ingress_bytes=301047808 link_bytes=301047808 token_bytes=533234088 weight_movement_bytes=34603008 total_nvlink_bytes=567837096 weight_to_token_ratio=0.064893 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132818665e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265060352 dispatch_meta_bytes=343504 combine_payload_bytes=265060352 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=265403856 ingress_bytes=403472384 link_bytes=403472384 token_bytes=530464208 weight_movement_bytes=138412032 total_nvlink_bytes=668876240 weight_to_token_ratio=0.260926 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.518234099e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265920512 dispatch_meta_bytes=343288 combine_payload_bytes=265920512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266263800 ingress_bytes=300523520 link_bytes=300523520 token_bytes=532184312 weight_movement_bytes=34603008 total_nvlink_bytes=566787320 weight_to_token_ratio=0.065021 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130845812e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74118 payload_rows_remote=64912 dup_rows=24186 dup_rows_remote=21122 dedup_ratio=0.246033 router_tokens_per_expert=[738, 775, 783, 774, 789, 758, 773, 735, 742, 790, 796, 774, 758, 771, 759, 792, 785, 759, 785, 739, 805, 758, 781, 810, 756, 786, 751, 732, 727, 809, 712, 817, 749, 756, 768, 749, 769, 775, 762, 815, 780, 766, 769, 766, 784, 773, 773, 775, 769, 756, 753, 724, 791, 716, 801, 759, 799, 737, 775, 723, 749, 829, 743, 780] payload_rows_per_dst=[9206, 9346, 9402, 9206, 9304, 9198, 9238, 9218] dup_rows_per_dst=[3064, 3102, 3232, 2982, 2982, 2956, 2900, 2968] combine_rows_per_src=[9206, 9346, 9402, 9206, 9304, 9198, 9238, 9218] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265150464 dispatch_meta_bytes=343232 combine_payload_bytes=265150464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265493696 ingress_bytes=299753472 link_bytes=299753472 token_bytes=530644160 weight_movement_bytes=34603008 total_nvlink_bytes=565247168 weight_to_token_ratio=0.065209 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127948183e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74116 payload_rows_remote=64712 dup_rows=24188 dup_rows_remote=21164 dedup_ratio=0.246053 router_tokens_per_expert=[758, 789, 776, 729, 760, 751, 747, 740, 793, 767, 770, 783, 797, 765, 701, 815, 759, 769, 776, 736, 752, 749, 736, 774, 773, 776, 783, 779, 757, 748, 782, 763, 783, 795, 772, 781, 734, 760, 727, 746, 738, 805, 738, 780, 796, 792, 792, 773, 765, 765, 746, 799, 758, 769, 749, 795, 805, 718, 788, 781, 797, 766, 808, 778] payload_rows_per_dst=[9132, 9280, 9156, 9294, 9204, 9404, 9294, 9352] dup_rows_per_dst=[2968, 3102, 2946, 3028, 2992, 3024, 2998, 3130] combine_rows_per_src=[9132, 9280, 9156, 9294, 9204, 9404, 9294, 9352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265879552 dispatch_meta_bytes=344136 combine_payload_bytes=265879552 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266223688 ingress_bytes=335085568 link_bytes=335085568 token_bytes=532103240 weight_movement_bytes=69206016 total_nvlink_bytes=601309256 weight_to_token_ratio=0.130061 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260900016e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74176 payload_rows_remote=65052 dup_rows=24128 dup_rows_remote=21160 dedup_ratio=0.245443 router_tokens_per_expert=[775, 751, 770, 774, 843, 756, 776, 793, 787, 762, 806, 790, 784, 741, 814, 734, 776, 747, 768, 786, 767, 761, 737, 759, 760, 710, 773, 785, 737, 750, 780, 751, 813, 806, 745, 724, 751, 741, 721, 782, 754, 789, 806, 755, 744, 770, 757, 764, 798, 760, 739, 753, 785, 779, 794, 757, 775, 780, 794, 761, 780, 761, 765, 746] payload_rows_per_dst=[9294, 9348, 9216, 9124, 9218, 9294, 9334, 9348] dup_rows_per_dst=[3182, 3088, 2986, 2968, 2948, 2984, 2996, 2976] combine_rows_per_src=[9294, 9348, 9216, 9124, 9218, 9294, 9334, 9348] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74332 payload_rows_remote=64992 dup_rows=23972 dup_rows_remote=20958 dedup_ratio=0.243856 router_tokens_per_expert=[743, 737, 717, 801, 774, 760, 739, 773, 750, 754, 772, 793, 795, 783, 742, 764, 787, 744, 806, 739, 778, 758, 723, 761, 766, 755, 724, 759, 792, 787, 814, 764, 772, 753, 753, 725, 792, 796, 801, 754, 785, 765, 820, 789, 771, 806, 744, 802, 786, 794, 769, 720, 754, 781, 790, 783, 791, 736, 761, 741, 751, 816, 745, 752] payload_rows_per_dst=[9212, 9302, 9220, 9338, 9326, 9396, 9340, 9198] dup_rows_per_dst=[2876, 3004, 2972, 2984, 2966, 3168, 3014, 2988] combine_rows_per_src=[9212, 9302, 9220, 9338, 9326, 9396, 9340, 9198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266944512 dispatch_meta_bytes=345200 combine_payload_bytes=266944512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267289712 ingress_bytes=301547520 link_bytes=301547520 token_bytes=534234224 weight_movement_bytes=34603008 total_nvlink_bytes=568837232 weight_to_token_ratio=0.064771 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134699041e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266207232 dispatch_meta_bytes=343800 combine_payload_bytes=266207232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266551032 ingress_bytes=300810240 link_bytes=300810240 token_bytes=532758264 weight_movement_bytes=34603008 total_nvlink_bytes=567361272 weight_to_token_ratio=0.064951 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131924716e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74112 payload_rows_remote=64820 dup_rows=24192 dup_rows_remote=21140 dedup_ratio=0.246094 router_tokens_per_expert=[736, 723, 764, 753, 779, 811, 730, 740, 754, 777, 790, 808, 739, 727, 776, 726, 754, 782, 772, 814, 736, 744, 756, 771, 749, 769, 735, 830, 767, 782, 756, 772, 785, 766, 785, 780, 758, 788, 788, 763, 746, 780, 742, 752, 816, 772, 748, 790, 820, 802, 765, 813, 722, 772, 742, 765, 791, 805, 755, 775, 748, 747, 785, 764] payload_rows_per_dst=[9096, 9126, 9226, 9216, 9542, 9274, 9340, 9292] dup_rows_per_dst=[2842, 3068, 3032, 2852, 3136, 3014, 3196, 3052] combine_rows_per_src=[9096, 9126, 9226, 9216, 9542, 9274, 9340, 9292] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=65020 dup_rows=23962 dup_rows_remote=21076 dedup_ratio=0.243754 router_tokens_per_expert=[746, 829, 805, 754, 778, 784, 770, 746, 779, 756, 766, 774, 762, 727, 769, 756, 753, 761, 779, 721, 788, 759, 764, 779, 786, 760, 750, 800, 779, 792, 788, 782, 780, 779, 760, 772, 780, 758, 741, 723, 731, 752, 808, 753, 761, 736, 828, 761, 765, 728, 804, 761, 756, 757, 801, 759, 791, 777, 781, 743, 721, 754, 807, 782] payload_rows_per_dst=[9388, 9226, 9322, 9374, 9208, 9302, 9190, 9332] dup_rows_per_dst=[3036, 2952, 2886, 3100, 2978, 2958, 3072, 2980] combine_rows_per_src=[9388, 9226, 9322, 9374, 9208, 9302, 9190, 9332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74336 payload_rows_remote=65172 dup_rows=23968 dup_rows_remote=21128 dedup_ratio=0.243815 router_tokens_per_expert=[758, 741, 789, 792, 742, 834, 750, 778, 775, 765, 753, 752, 738, 723, 772, 724, 755, 819, 774, 738, 799, 735, 767, 762, 820, 825, 750, 782, 769, 806, 741, 741, 747, 788, 781, 759, 765, 792, 750, 712, 765, 797, 762, 778, 774, 767, 727, 765, 758, 749, 763, 790, 806, 799, 802, 769, 726, 785, 755, 775, 735, 828, 722, 792] payload_rows_per_dst=[9286, 9164, 9304, 9370, 9294, 9360, 9368, 9190] dup_rows_per_dst=[3082, 2840, 2994, 3098, 2894, 2910, 3104, 3046] combine_rows_per_src=[9286, 9164, 9304, 9370, 9294, 9360, 9368, 9190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265060352 dispatch_meta_bytes=343504 combine_payload_bytes=265060352 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=265403856 ingress_bytes=403472384 link_bytes=403472384 token_bytes=530464208 weight_movement_bytes=138412032 total_nvlink_bytes=668876240 weight_to_token_ratio=0.260926 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.518234099e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=65184 dup_rows=23926 dup_rows_remote=20918 dedup_ratio=0.243388 router_tokens_per_expert=[791, 738, 773, 755, 770, 738, 744, 743, 808, 736, 809, 789, 770, 761, 763, 781, 768, 745, 755, 770, 763, 758, 788, 793, 776, 816, 769, 782, 792, 766, 735, 778, 765, 760, 768, 749, 800, 786, 723, 750, 813, 728, 814, 790, 719, 738, 751, 777, 806, 798, 809, 767, 754, 772, 709, 734, 758, 773, 788, 771, 757, 751, 769, 782] payload_rows_per_dst=[9216, 9328, 9274, 9350, 9194, 9308, 9370, 9338] dup_rows_per_dst=[2888, 3106, 3006, 3078, 3008, 2952, 2928, 2960] combine_rows_per_src=[9216, 9328, 9274, 9350, 9194, 9308, 9370, 9338] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266452992 dispatch_meta_bytes=344848 combine_payload_bytes=266452992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266797840 ingress_bytes=335659008 link_bytes=335659008 token_bytes=533250832 weight_movement_bytes=69206016 total_nvlink_bytes=602456848 weight_to_token_ratio=0.129781 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263057824e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266207232 dispatch_meta_bytes=343800 combine_payload_bytes=266207232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266551032 ingress_bytes=300810240 link_bytes=300810240 token_bytes=532758264 weight_movement_bytes=34603008 total_nvlink_bytes=567361272 weight_to_token_ratio=0.064951 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131924716e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74118 payload_rows_remote=64912 dup_rows=24186 dup_rows_remote=21122 dedup_ratio=0.246033 router_tokens_per_expert=[738, 775, 783, 774, 789, 758, 773, 735, 742, 790, 796, 774, 758, 771, 759, 792, 785, 759, 785, 739, 805, 758, 781, 810, 756, 786, 751, 732, 727, 809, 712, 817, 749, 756, 768, 749, 769, 775, 762, 815, 780, 766, 769, 766, 784, 773, 773, 775, 769, 756, 753, 724, 791, 716, 801, 759, 799, 737, 775, 723, 749, 829, 743, 780] payload_rows_per_dst=[9206, 9346, 9402, 9206, 9304, 9198, 9238, 9218] dup_rows_per_dst=[3064, 3102, 3232, 2982, 2982, 2956, 2900, 2968] combine_rows_per_src=[9206, 9346, 9402, 9206, 9304, 9198, 9238, 9218] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266321920 dispatch_meta_bytes=344384 combine_payload_bytes=266321920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266666304 ingress_bytes=300924928 link_bytes=300924928 token_bytes=532988224 weight_movement_bytes=34603008 total_nvlink_bytes=567591232 weight_to_token_ratio=0.064923 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132356278e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=343840 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265846560 ingress_bytes=334708736 link_bytes=334708736 token_bytes=531349280 weight_movement_bytes=69206016 total_nvlink_bytes=600555296 weight_to_token_ratio=0.130246 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259482027e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74136 payload_rows_remote=64734 dup_rows=24168 dup_rows_remote=21074 dedup_ratio=0.245850 router_tokens_per_expert=[762, 767, 804, 773, 773, 814, 750, 720, 747, 801, 816, 776, 755, 745, 750, 790, 754, 745, 794, 765, 719, 772, 766, 750, 774, 784, 806, 746, 806, 772, 770, 767, 799, 802, 758, 822, 754, 792, 773, 748, 783, 777, 747, 792, 746, 715, 783, 773, 744, 777, 733, 739, 724, 751, 820, 734, 709, 745, 790, 795, 767, 772, 755, 800] payload_rows_per_dst=[9240, 9328, 9218, 9366, 9402, 9282, 9042, 9258] dup_rows_per_dst=[3086, 3032, 2912, 3084, 3094, 2950, 3002, 3008] combine_rows_per_src=[9240, 9328, 9218, 9366, 9402, 9282, 9042, 9258] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266944512 dispatch_meta_bytes=345200 combine_payload_bytes=266944512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267289712 ingress_bytes=301547520 link_bytes=301547520 token_bytes=534234224 weight_movement_bytes=34603008 total_nvlink_bytes=568837232 weight_to_token_ratio=0.064771 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134699041e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74156 payload_rows_remote=64216 dup_rows=24148 dup_rows_remote=20504 dedup_ratio=0.245646 router_tokens_per_expert=[800, 796, 774, 775, 806, 767, 724, 798, 772, 739, 733, 766, 750, 743, 747, 838, 745, 806, 689, 742, 774, 801, 742, 739, 724, 748, 818, 796, 792, 748, 788, 767, 756, 754, 721, 761, 784, 774, 752, 785, 781, 766, 790, 788, 766, 797, 743, 799, 729, 763, 775, 789, 712, 747, 798, 789, 773, 792, 802, 734, 737, 743, 802, 803] payload_rows_per_dst=[9940, 9364, 9214, 9304, 9172, 8744, 9216, 9202] dup_rows_per_dst=[3644, 3068, 2862, 3058, 3002, 2612, 2988, 2914] combine_rows_per_src=[9940, 9364, 9214, 9304, 9172, 8744, 9216, 9202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266993664 dispatch_meta_bytes=344408 combine_payload_bytes=266993664 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267338072 ingress_bytes=301596672 link_bytes=301596672 token_bytes=534331736 weight_movement_bytes=34603008 total_nvlink_bytes=568934744 weight_to_token_ratio=0.064759 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134883996e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=263028736 dispatch_meta_bytes=338880 combine_payload_bytes=263028736 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263367616 ingress_bytes=297631744 link_bytes=297631744 token_bytes=526396352 weight_movement_bytes=34603008 total_nvlink_bytes=560999360 weight_to_token_ratio=0.065736 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.119964291e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=64948 dup_rows=23940 dup_rows_remote=20878 dedup_ratio=0.243530 router_tokens_per_expert=[758, 746, 716, 742, 780, 713, 750, 792, 792, 791, 771, 777, 775, 768, 786, 757, 740, 798, 758, 765, 717, 787, 756, 766, 742, 780, 796, 745, 752, 726, 739, 774, 740, 782, 760, 769, 809, 738, 843, 740, 780, 778, 762, 768, 755, 814, 810, 772, 712, 781, 780, 782, 793, 798, 810, 816, 704, 776, 774, 766, 743, 754, 809, 779] payload_rows_per_dst=[9134, 9352, 9220, 9196, 9382, 9416, 9408, 9256] dup_rows_per_dst=[2860, 3082, 2954, 2912, 2980, 3062, 3136, 2954] combine_rows_per_src=[9134, 9352, 9220, 9196, 9382, 9416, 9408, 9256] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266321920 dispatch_meta_bytes=344384 combine_payload_bytes=266321920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266666304 ingress_bytes=300924928 link_bytes=300924928 token_bytes=532988224 weight_movement_bytes=34603008 total_nvlink_bytes=567591232 weight_to_token_ratio=0.064923 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132356278e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=64924 dup_rows=24018 dup_rows_remote=20978 dedup_ratio=0.244324 router_tokens_per_expert=[776, 737, 760, 741, 739, 745, 773, 799, 800, 749, 755, 787, 782, 761, 766, 801, 749, 747, 796, 768, 822, 739, 780, 740, 779, 775, 765, 800, 765, 790, 758, 842, 768, 729, 785, 776, 754, 772, 767, 767, 804, 765, 793, 778, 776, 840, 767, 732, 717, 736, 800, 685, 773, 773, 713, 767, 780, 770, 766, 768, 760, 749, 801, 735] payload_rows_per_dst=[9144, 9362, 9238, 9498, 9286, 9390, 9084, 9284] dup_rows_per_dst=[2996, 3040, 3044, 3050, 2950, 3120, 2844, 2974] combine_rows_per_src=[9144, 9362, 9238, 9498, 9286, 9390, 9084, 9284] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74180 payload_rows_remote=64904 dup_rows=24124 dup_rows_remote=21170 dedup_ratio=0.245402 router_tokens_per_expert=[760, 743, 755, 798, 803, 775, 765, 802, 781, 785, 775, 756, 774, 755, 768, 757, 793, 776, 785, 773, 761, 786, 813, 749, 790, 729, 767, 813, 711, 758, 764, 789, 757, 773, 783, 754, 730, 756, 732, 769, 779, 744, 777, 750, 778, 787, 813, 769, 812, 764, 797, 775, 756, 730, 756, 725, 766, 766, 705, 710, 842, 748, 762, 778] payload_rows_per_dst=[9372, 9336, 9312, 9198, 9126, 9394, 9276, 9166] dup_rows_per_dst=[3030, 2966, 3160, 3044, 2982, 3000, 2954, 2988] combine_rows_per_src=[9372, 9336, 9312, 9198, 9126, 9394, 9276, 9166] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265879552 dispatch_meta_bytes=344136 combine_payload_bytes=265879552 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266223688 ingress_bytes=335085568 link_bytes=335085568 token_bytes=532103240 weight_movement_bytes=69206016 total_nvlink_bytes=601309256 weight_to_token_ratio=0.130061 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260900016e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265150464 dispatch_meta_bytes=343232 combine_payload_bytes=265150464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265493696 ingress_bytes=299753472 link_bytes=299753472 token_bytes=530644160 weight_movement_bytes=34603008 total_nvlink_bytes=565247168 weight_to_token_ratio=0.065209 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127948183e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266027008 dispatch_meta_bytes=343304 combine_payload_bytes=266027008 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266370312 ingress_bytes=335233024 link_bytes=335233024 token_bytes=532397320 weight_movement_bytes=69206016 total_nvlink_bytes=601603336 weight_to_token_ratio=0.129989 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261454881e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74110 payload_rows_remote=64800 dup_rows=24194 dup_rows_remote=21206 dedup_ratio=0.246114 router_tokens_per_expert=[744, 747, 744, 779, 762, 786, 731, 802, 707, 720, 829, 774, 798, 776, 797, 730, 776, 742, 743, 812, 768, 780, 798, 770, 805, 785, 757, 757, 744, 782, 739, 780, 752, 752, 789, 766, 782, 766, 771, 785, 775, 809, 767, 749, 797, 785, 758, 745, 765, 757, 801, 767, 770, 780, 762, 746, 768, 768, 739, 771, 732, 769, 768, 777] payload_rows_per_dst=[9206, 9198, 9374, 9310, 9236, 9304, 9210, 9272] dup_rows_per_dst=[2984, 3064, 3004, 2988, 3090, 3066, 3086, 2912] combine_rows_per_src=[9206, 9198, 9374, 9310, 9236, 9304, 9210, 9272] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74180 payload_rows_remote=64904 dup_rows=24124 dup_rows_remote=21170 dedup_ratio=0.245402 router_tokens_per_expert=[760, 743, 755, 798, 803, 775, 765, 802, 781, 785, 775, 756, 774, 755, 768, 757, 793, 776, 785, 773, 761, 786, 813, 749, 790, 729, 767, 813, 711, 758, 764, 789, 757, 773, 783, 754, 730, 756, 732, 769, 779, 744, 777, 750, 778, 787, 813, 769, 812, 764, 797, 775, 756, 730, 756, 725, 766, 766, 705, 710, 842, 748, 762, 778] payload_rows_per_dst=[9372, 9336, 9312, 9198, 9126, 9394, 9276, 9166] dup_rows_per_dst=[3030, 2966, 3160, 3044, 2982, 3000, 2954, 2988] combine_rows_per_src=[9372, 9336, 9312, 9198, 9126, 9394, 9276, 9166] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265846784 dispatch_meta_bytes=344296 combine_payload_bytes=265846784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266191080 ingress_bytes=300449792 link_bytes=300449792 token_bytes=532037864 weight_movement_bytes=34603008 total_nvlink_bytes=566640872 weight_to_token_ratio=0.065039 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130568379e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265928704 dispatch_meta_bytes=343608 combine_payload_bytes=265928704 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266272312 ingress_bytes=335134720 link_bytes=335134720 token_bytes=532201016 weight_movement_bytes=69206016 total_nvlink_bytes=601407032 weight_to_token_ratio=0.130037 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261084971e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=64818 dup_rows=24056 dup_rows_remote=20916 dedup_ratio=0.244710 router_tokens_per_expert=[751, 739, 774, 765, 711, 743, 765, 790, 743, 771, 795, 779, 811, 767, 754, 732, 782, 813, 745, 806, 784, 780, 810, 765, 790, 779, 793, 763, 751, 797, 778, 743, 742, 767, 746, 757, 738, 791, 783, 748, 821, 799, 778, 777, 749, 747, 764, 824, 735, 737, 758, 732, 721, 772, 738, 786, 771, 761, 733, 782, 779, 782, 775, 790] payload_rows_per_dst=[9176, 9258, 9430, 9328, 9154, 9344, 9206, 9352] dup_rows_per_dst=[2900, 3046, 3140, 3060, 2990, 3174, 2752, 2994] combine_rows_per_src=[9176, 9258, 9430, 9328, 9154, 9344, 9206, 9352] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74020 payload_rows_remote=64618 dup_rows=24284 dup_rows_remote=21240 dedup_ratio=0.247030 router_tokens_per_expert=[800, 726, 758, 704, 769, 743, 754, 716, 824, 754, 745, 756, 785, 796, 772, 777, 756, 716, 800, 803, 753, 811, 741, 736, 790, 706, 755, 729, 786, 820, 834, 769, 739, 766, 795, 806, 772, 755, 799, 780, 732, 794, 726, 801, 768, 789, 785, 767, 750, 786, 781, 783, 774, 782, 712, 763, 775, 734, 772, 787, 754, 783, 792, 766] payload_rows_per_dst=[9122, 8930, 9152, 9082, 9430, 9272, 9630, 9402] dup_rows_per_dst=[2818, 2778, 3050, 2890, 3282, 3080, 3342, 3044] combine_rows_per_src=[9122, 8930, 9152, 9082, 9430, 9272, 9630, 9402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=64924 dup_rows=24018 dup_rows_remote=20978 dedup_ratio=0.244324 router_tokens_per_expert=[776, 737, 760, 741, 739, 745, 773, 799, 800, 749, 755, 787, 782, 761, 766, 801, 749, 747, 796, 768, 822, 739, 780, 740, 779, 775, 765, 800, 765, 790, 758, 842, 768, 729, 785, 776, 754, 772, 767, 767, 804, 765, 793, 778, 776, 840, 767, 732, 717, 736, 800, 685, 773, 773, 713, 767, 780, 770, 766, 768, 760, 749, 801, 735] payload_rows_per_dst=[9144, 9362, 9238, 9498, 9286, 9390, 9084, 9284] dup_rows_per_dst=[2996, 3040, 3044, 3050, 2950, 3120, 2844, 2974] combine_rows_per_src=[9144, 9362, 9238, 9498, 9286, 9390, 9084, 9284] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265420800 dispatch_meta_bytes=344024 combine_payload_bytes=265420800 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265764824 ingress_bytes=369229824 link_bytes=369229824 token_bytes=531185624 weight_movement_bytes=103809024 total_nvlink_bytes=634994648 weight_to_token_ratio=0.195429 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389382102e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=64818 dup_rows=24056 dup_rows_remote=20916 dedup_ratio=0.244710 router_tokens_per_expert=[751, 739, 774, 765, 711, 743, 765, 790, 743, 771, 795, 779, 811, 767, 754, 732, 782, 813, 745, 806, 784, 780, 810, 765, 790, 779, 793, 763, 751, 797, 778, 743, 742, 767, 746, 757, 738, 791, 783, 748, 821, 799, 778, 777, 749, 747, 764, 824, 735, 737, 758, 732, 721, 772, 738, 786, 771, 761, 733, 782, 779, 782, 775, 790] payload_rows_per_dst=[9176, 9258, 9430, 9328, 9154, 9344, 9206, 9352] dup_rows_per_dst=[2900, 3046, 3140, 3060, 2990, 3174, 2752, 2994] combine_rows_per_src=[9176, 9258, 9430, 9328, 9154, 9344, 9206, 9352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74252 payload_rows_remote=65052 dup_rows=24052 dup_rows_remote=21072 dedup_ratio=0.244670 router_tokens_per_expert=[770, 783, 761, 785, 705, 753, 719, 739, 735, 810, 752, 741, 814, 848, 706, 746, 751, 785, 742, 799, 741, 778, 774, 748, 789, 805, 750, 775, 807, 759, 799, 762, 779, 756, 728, 766, 808, 784, 758, 711, 775, 791, 805, 766, 821, 782, 792, 788, 797, 746, 751, 751, 782, 780, 736, 710, 746, 794, 735, 764, 785, 818, 768, 748] payload_rows_per_dst=[9076, 9358, 9222, 9440, 9200, 9466, 9214, 9276] dup_rows_per_dst=[2954, 2946, 3014, 3052, 2980, 3174, 2892, 3040] combine_rows_per_src=[9076, 9358, 9222, 9440, 9200, 9466, 9214, 9276] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266027008 dispatch_meta_bytes=343304 combine_payload_bytes=266027008 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266370312 ingress_bytes=335233024 link_bytes=335233024 token_bytes=532397320 weight_movement_bytes=69206016 total_nvlink_bytes=601603336 weight_to_token_ratio=0.129989 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261454881e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265846784 dispatch_meta_bytes=344296 combine_payload_bytes=265846784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266191080 ingress_bytes=300449792 link_bytes=300449792 token_bytes=532037864 weight_movement_bytes=34603008 total_nvlink_bytes=566640872 weight_to_token_ratio=0.065039 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130568379e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74408 payload_rows_remote=64948 dup_rows=23896 dup_rows_remote=20930 dedup_ratio=0.243083 router_tokens_per_expert=[755, 782, 785, 766, 792, 786, 788, 757, 782, 743, 768, 708, 770, 771, 793, 768, 719, 773, 756, 752, 760, 752, 763, 810, 760, 773, 761, 781, 761, 736, 792, 802, 802, 772, 809, 724, 759, 782, 763, 777, 793, 735, 768, 776, 811, 753, 732, 741, 764, 777, 770, 765, 734, 761, 772, 761, 744, 736, 784, 749, 800, 797, 800, 776] payload_rows_per_dst=[9460, 9278, 9398, 9164, 9344, 9144, 9174, 9446] dup_rows_per_dst=[2966, 3090, 3102, 2838, 3082, 2858, 3034, 2926] combine_rows_per_src=[9460, 9278, 9398, 9164, 9344, 9144, 9174, 9446] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=264675328 dispatch_meta_bytes=343432 combine_payload_bytes=264675328 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265018760 ingress_bytes=333881344 link_bytes=333881344 token_bytes=529694088 weight_movement_bytes=69206016 total_nvlink_bytes=598900104 weight_to_token_ratio=0.130653 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256368618e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74156 payload_rows_remote=64216 dup_rows=24148 dup_rows_remote=20504 dedup_ratio=0.245646 router_tokens_per_expert=[800, 796, 774, 775, 806, 767, 724, 798, 772, 739, 733, 766, 750, 743, 747, 838, 745, 806, 689, 742, 774, 801, 742, 739, 724, 748, 818, 796, 792, 748, 788, 767, 756, 754, 721, 761, 784, 774, 752, 785, 781, 766, 790, 788, 766, 797, 743, 799, 729, 763, 775, 789, 712, 747, 798, 789, 773, 792, 802, 734, 737, 743, 802, 803] payload_rows_per_dst=[9940, 9364, 9214, 9304, 9172, 8744, 9216, 9202] dup_rows_per_dst=[3644, 3068, 2862, 3058, 3002, 2612, 2988, 2914] combine_rows_per_src=[9940, 9364, 9214, 9304, 9172, 8744, 9216, 9202] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74250 payload_rows_remote=65044 dup_rows=24054 dup_rows_remote=21062 dedup_ratio=0.244690 router_tokens_per_expert=[738, 788, 781, 755, 793, 760, 765, 762, 785, 785, 760, 722, 767, 812, 736, 755, 764, 781, 774, 758, 805, 781, 766, 742, 773, 764, 776, 757, 811, 754, 739, 732, 822, 776, 781, 787, 782, 753, 829, 805, 731, 754, 762, 729, 776, 756, 803, 788, 735, 754, 818, 791, 746, 742, 738, 805, 749, 751, 696, 775, 787, 767, 762, 761] payload_rows_per_dst=[9266, 9186, 9304, 9244, 9474, 9206, 9340, 9230] dup_rows_per_dst=[3018, 3058, 3038, 2968, 3196, 2992, 2918, 2866] combine_rows_per_src=[9266, 9186, 9304, 9244, 9474, 9206, 9340, 9230] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=65184 dup_rows=23926 dup_rows_remote=20918 dedup_ratio=0.243388 router_tokens_per_expert=[791, 738, 773, 755, 770, 738, 744, 743, 808, 736, 809, 789, 770, 761, 763, 781, 768, 745, 755, 770, 763, 758, 788, 793, 776, 816, 769, 782, 792, 766, 735, 778, 765, 760, 768, 749, 800, 786, 723, 750, 813, 728, 814, 790, 719, 738, 751, 777, 806, 798, 809, 767, 754, 772, 709, 734, 758, 773, 788, 771, 757, 751, 769, 782] payload_rows_per_dst=[9216, 9328, 9274, 9350, 9194, 9308, 9370, 9338] dup_rows_per_dst=[2888, 3106, 3006, 3078, 3008, 2952, 2928, 2960] combine_rows_per_src=[9216, 9328, 9274, 9350, 9194, 9308, 9370, 9338] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265494528 dispatch_meta_bytes=342936 combine_payload_bytes=265494528 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265837464 ingress_bytes=334700544 link_bytes=334700544 token_bytes=531331992 weight_movement_bytes=69206016 total_nvlink_bytes=600538008 weight_to_token_ratio=0.130250 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259451201e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74468 payload_rows_remote=65262 dup_rows=23836 dup_rows_remote=20938 dedup_ratio=0.242472 router_tokens_per_expert=[808, 810, 810, 745, 771, 721, 721, 752, 760, 804, 745, 758, 788, 800, 755, 754, 757, 750, 769, 763, 770, 826, 795, 741, 767, 823, 737, 756, 755, 810, 731, 721, 766, 780, 717, 759, 693, 788, 821, 750, 780, 791, 797, 791, 776, 727, 771, 797, 746, 715, 743, 748, 780, 735, 804, 781, 781, 759, 764, 791, 757, 777, 776, 818] payload_rows_per_dst=[9244, 9310, 9328, 9330, 9228, 9376, 9206, 9446] dup_rows_per_dst=[3032, 3018, 3014, 2870, 2920, 3084, 2898, 3000] combine_rows_per_src=[9244, 9310, 9328, 9330, 9228, 9376, 9206, 9446] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74304 payload_rows_remote=65078 dup_rows=24000 dup_rows_remote=21128 dedup_ratio=0.244141 router_tokens_per_expert=[751, 753, 764, 754, 831, 789, 789, 809, 763, 755, 736, 759, 791, 774, 731, 740, 755, 803, 724, 811, 750, 727, 770, 759, 778, 764, 790, 783, 759, 755, 737, 820, 802, 732, 774, 731, 751, 779, 724, 759, 762, 757, 727, 797, 797, 808, 755, 786, 753, 767, 776, 776, 793, 795, 754, 803, 779, 738, 762, 754, 840, 730, 769, 748] payload_rows_per_dst=[9394, 9226, 9216, 9326, 9222, 9354, 9304, 9262] dup_rows_per_dst=[3086, 2872, 2982, 3046, 2882, 3024, 3130, 2978] combine_rows_per_src=[9394, 9226, 9216, 9326, 9222, 9354, 9304, 9262] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=263028736 dispatch_meta_bytes=338880 combine_payload_bytes=263028736 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263367616 ingress_bytes=297631744 link_bytes=297631744 token_bytes=526396352 weight_movement_bytes=34603008 total_nvlink_bytes=560999360 weight_to_token_ratio=0.065736 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.119964291e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265928704 dispatch_meta_bytes=343608 combine_payload_bytes=265928704 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266272312 ingress_bytes=335134720 link_bytes=335134720 token_bytes=532201016 weight_movement_bytes=69206016 total_nvlink_bytes=601407032 weight_to_token_ratio=0.130037 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261084971e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266452992 dispatch_meta_bytes=344496 combine_payload_bytes=266452992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266797488 ingress_bytes=335659008 link_bytes=335659008 token_bytes=533250480 weight_movement_bytes=69206016 total_nvlink_bytes=602456496 weight_to_token_ratio=0.129781 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263057824e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266027008 dispatch_meta_bytes=343512 combine_payload_bytes=266027008 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266370520 ingress_bytes=369836032 link_bytes=369836032 token_bytes=532397528 weight_movement_bytes=103809024 total_nvlink_bytes=636206552 weight_to_token_ratio=0.194984 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391663214e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74336 payload_rows_remote=65072 dup_rows=23968 dup_rows_remote=20988 dedup_ratio=0.243815 router_tokens_per_expert=[751, 809, 704, 765, 781, 768, 788, 746, 860, 775, 762, 750, 767, 780, 779, 832, 790, 747, 748, 782, 770, 795, 801, 756, 788, 790, 759, 748, 742, 766, 783, 746, 775, 755, 774, 768, 715, 730, 780, 732, 822, 778, 784, 771, 722, 795, 771, 798, 723, 778, 800, 749, 756, 751, 735, 748, 769, 765, 730, 758, 792, 789, 767, 744] payload_rows_per_dst=[9210, 9404, 9308, 9264, 9182, 9486, 9192, 9290] dup_rows_per_dst=[3014, 3206, 3070, 2980, 2876, 2996, 2888, 2938] combine_rows_per_src=[9210, 9404, 9308, 9264, 9182, 9486, 9192, 9290] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=344424 combine_payload_bytes=266420224 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266764648 ingress_bytes=370229248 link_bytes=370229248 token_bytes=533184872 weight_movement_bytes=103809024 total_nvlink_bytes=636993896 weight_to_token_ratio=0.194696 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393142854e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266993664 dispatch_meta_bytes=344408 combine_payload_bytes=266993664 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267338072 ingress_bytes=301596672 link_bytes=301596672 token_bytes=534331736 weight_movement_bytes=34603008 total_nvlink_bytes=568934744 weight_to_token_ratio=0.064759 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134883996e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74468 payload_rows_remote=65262 dup_rows=23836 dup_rows_remote=20938 dedup_ratio=0.242472 router_tokens_per_expert=[808, 810, 810, 745, 771, 721, 721, 752, 760, 804, 745, 758, 788, 800, 755, 754, 757, 750, 769, 763, 770, 826, 795, 741, 767, 823, 737, 756, 755, 810, 731, 721, 766, 780, 717, 759, 693, 788, 821, 750, 780, 791, 797, 791, 776, 727, 771, 797, 746, 715, 743, 748, 780, 735, 804, 781, 781, 759, 764, 791, 757, 777, 776, 818] payload_rows_per_dst=[9244, 9310, 9328, 9330, 9228, 9376, 9206, 9446] dup_rows_per_dst=[3032, 3018, 3014, 2870, 2920, 3084, 2898, 3000] combine_rows_per_src=[9244, 9310, 9328, 9330, 9228, 9376, 9206, 9446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74250 payload_rows_remote=65044 dup_rows=24054 dup_rows_remote=21062 dedup_ratio=0.244690 router_tokens_per_expert=[738, 788, 781, 755, 793, 760, 765, 762, 785, 785, 760, 722, 767, 812, 736, 755, 764, 781, 774, 758, 805, 781, 766, 742, 773, 764, 776, 757, 811, 754, 739, 732, 822, 776, 781, 787, 782, 753, 829, 805, 731, 754, 762, 729, 776, 756, 803, 788, 735, 754, 818, 791, 746, 742, 738, 805, 749, 751, 696, 775, 787, 767, 762, 761] payload_rows_per_dst=[9266, 9186, 9304, 9244, 9474, 9206, 9340, 9230] dup_rows_per_dst=[3018, 3058, 3038, 2968, 3196, 2992, 2918, 2866] combine_rows_per_src=[9266, 9186, 9304, 9244, 9474, 9206, 9340, 9230] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74128 payload_rows_remote=64776 dup_rows=24176 dup_rows_remote=21036 dedup_ratio=0.245931 router_tokens_per_expert=[799, 783, 796, 757, 818, 763, 775, 709, 754, 771, 691, 757, 803, 777, 708, 775, 770, 754, 751, 791, 760, 756, 758, 779, 803, 772, 806, 732, 711, 755, 784, 804, 727, 811, 797, 750, 790, 777, 789, 757, 789, 780, 764, 810, 753, 783, 757, 757, 791, 747, 757, 796, 753, 791, 739, 741, 726, 840, 720, 756, 750, 765, 830, 737] payload_rows_per_dst=[9222, 9176, 9250, 9272, 9082, 9262, 9512, 9352] dup_rows_per_dst=[2898, 2896, 2988, 3098, 2860, 3124, 3172, 3140] combine_rows_per_src=[9222, 9176, 9250, 9272, 9082, 9262, 9512, 9352] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266559488 dispatch_meta_bytes=344824 combine_payload_bytes=266559488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266904312 ingress_bytes=301162496 link_bytes=301162496 token_bytes=533463800 weight_movement_bytes=34603008 total_nvlink_bytes=568066808 weight_to_token_ratio=0.064865 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133250227e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267313152 dispatch_meta_bytes=344800 combine_payload_bytes=267313152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267657952 ingress_bytes=301916160 link_bytes=301916160 token_bytes=534971104 weight_movement_bytes=34603008 total_nvlink_bytes=569574112 weight_to_token_ratio=0.064682 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136086204e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74316 payload_rows_remote=65010 dup_rows=23988 dup_rows_remote=20906 dedup_ratio=0.244019 router_tokens_per_expert=[789, 725, 764, 755, 764, 743, 815, 742, 764, 774, 783, 779, 710, 795, 748, 815, 728, 776, 750, 776, 812, 765, 795, 792, 785, 717, 756, 793, 793, 734, 758, 790, 777, 786, 789, 803, 762, 764, 773, 751, 762, 763, 741, 745, 720, 775, 762, 735, 767, 734, 791, 785, 802, 745, 765, 809, 786, 773, 769, 734, 794, 813, 772, 720] payload_rows_per_dst=[9216, 9242, 9306, 9320, 9370, 9198, 9392, 9272] dup_rows_per_dst=[2978, 3094, 3082, 2932, 3040, 2808, 3004, 3050] combine_rows_per_src=[9216, 9242, 9306, 9320, 9370, 9198, 9392, 9272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266534912 dispatch_meta_bytes=344240 combine_payload_bytes=266534912 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266879152 ingress_bytes=335740928 link_bytes=335740928 token_bytes=533414064 weight_movement_bytes=69206016 total_nvlink_bytes=602620080 weight_to_token_ratio=0.129742 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263366083e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74316 payload_rows_remote=65010 dup_rows=23988 dup_rows_remote=20906 dedup_ratio=0.244019 router_tokens_per_expert=[789, 725, 764, 755, 764, 743, 815, 742, 764, 774, 783, 779, 710, 795, 748, 815, 728, 776, 750, 776, 812, 765, 795, 792, 785, 717, 756, 793, 793, 734, 758, 790, 777, 786, 789, 803, 762, 764, 773, 751, 762, 763, 741, 745, 720, 775, 762, 735, 767, 734, 791, 785, 802, 745, 765, 809, 786, 773, 769, 734, 794, 813, 772, 720] payload_rows_per_dst=[9216, 9242, 9306, 9320, 9370, 9198, 9392, 9272] dup_rows_per_dst=[2978, 3094, 3082, 2932, 3040, 2808, 3004, 3050] combine_rows_per_src=[9216, 9242, 9306, 9320, 9370, 9198, 9392, 9272] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74304 payload_rows_remote=65078 dup_rows=24000 dup_rows_remote=21128 dedup_ratio=0.244141 router_tokens_per_expert=[751, 753, 764, 754, 831, 789, 789, 809, 763, 755, 736, 759, 791, 774, 731, 740, 755, 803, 724, 811, 750, 727, 770, 759, 778, 764, 790, 783, 759, 755, 737, 820, 802, 732, 774, 731, 751, 779, 724, 759, 762, 757, 727, 797, 797, 808, 755, 786, 753, 767, 776, 776, 793, 795, 754, 803, 779, 738, 762, 754, 840, 730, 769, 748] payload_rows_per_dst=[9394, 9226, 9216, 9326, 9222, 9354, 9304, 9262] dup_rows_per_dst=[3086, 2872, 2982, 3046, 2882, 3024, 3130, 2978] combine_rows_per_src=[9394, 9226, 9216, 9326, 9222, 9354, 9304, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74352 payload_rows_remote=64918 dup_rows=23952 dup_rows_remote=20970 dedup_ratio=0.243652 router_tokens_per_expert=[768, 782, 770, 794, 772, 768, 730, 710, 761, 770, 757, 763, 776, 786, 794, 752, 760, 753, 766, 765, 738, 786, 745, 761, 797, 733, 802, 798, 764, 820, 782, 781, 775, 835, 768, 728, 760, 782, 746, 814, 805, 763, 784, 790, 734, 729, 752, 747, 781, 789, 774, 757, 746, 719, 749, 746, 783, 803, 724, 796, 788, 774, 745, 762] payload_rows_per_dst=[9256, 9324, 9148, 9440, 9434, 9274, 9136, 9340] dup_rows_per_dst=[2932, 2994, 3000, 3114, 2982, 2934, 2986, 3010] combine_rows_per_src=[9256, 9324, 9148, 9440, 9434, 9274, 9136, 9340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267313152 dispatch_meta_bytes=344800 combine_payload_bytes=267313152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267657952 ingress_bytes=301916160 link_bytes=301916160 token_bytes=534971104 weight_movement_bytes=34603008 total_nvlink_bytes=569574112 weight_to_token_ratio=0.064682 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136086204e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=344424 combine_payload_bytes=266420224 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266764648 ingress_bytes=370229248 link_bytes=370229248 token_bytes=533184872 weight_movement_bytes=103809024 total_nvlink_bytes=636993896 weight_to_token_ratio=0.194696 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393142854e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265322496 dispatch_meta_bytes=343248 combine_payload_bytes=265322496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265665744 ingress_bytes=299925504 link_bytes=299925504 token_bytes=530988240 weight_movement_bytes=34603008 total_nvlink_bytes=565591248 weight_to_token_ratio=0.065167 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128595526e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74408 payload_rows_remote=64948 dup_rows=23896 dup_rows_remote=20930 dedup_ratio=0.243083 router_tokens_per_expert=[755, 782, 785, 766, 792, 786, 788, 757, 782, 743, 768, 708, 770, 771, 793, 768, 719, 773, 756, 752, 760, 752, 763, 810, 760, 773, 761, 781, 761, 736, 792, 802, 802, 772, 809, 724, 759, 782, 763, 777, 793, 735, 768, 776, 811, 753, 732, 741, 764, 777, 770, 765, 734, 761, 772, 761, 744, 736, 784, 749, 800, 797, 800, 776] payload_rows_per_dst=[9460, 9278, 9398, 9164, 9344, 9144, 9174, 9446] dup_rows_per_dst=[2966, 3090, 3102, 2838, 3082, 2858, 3034, 2926] combine_rows_per_src=[9460, 9278, 9398, 9164, 9344, 9144, 9174, 9446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=64966 dup_rows=23966 dup_rows_remote=20812 dedup_ratio=0.243795 router_tokens_per_expert=[787, 743, 789, 791, 806, 754, 798, 779, 770, 773, 787, 786, 786, 769, 743, 755, 765, 775, 772, 766, 760, 819, 832, 746, 731, 764, 713, 761, 773, 788, 750, 738, 749, 767, 764, 758, 711, 762, 790, 766, 798, 748, 771, 755, 769, 770, 776, 775, 711, 784, 767, 775, 802, 759, 798, 759, 736, 725, 774, 814, 747, 796, 761, 746] payload_rows_per_dst=[9372, 9366, 9456, 9514, 9186, 9284, 8870, 9290] dup_rows_per_dst=[3154, 2972, 3014, 3112, 2948, 3040, 2818, 2908] combine_rows_per_src=[9372, 9366, 9456, 9514, 9186, 9284, 8870, 9290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266280960 dispatch_meta_bytes=343664 combine_payload_bytes=266280960 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266624624 ingress_bytes=300883968 link_bytes=300883968 token_bytes=532905584 weight_movement_bytes=34603008 total_nvlink_bytes=567508592 weight_to_token_ratio=0.064933 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132202148e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266559488 dispatch_meta_bytes=344824 combine_payload_bytes=266559488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266904312 ingress_bytes=301162496 link_bytes=301162496 token_bytes=533463800 weight_movement_bytes=34603008 total_nvlink_bytes=568066808 weight_to_token_ratio=0.064865 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133250227e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=343552 combine_payload_bytes=265904128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266247680 ingress_bytes=300507136 link_bytes=300507136 token_bytes=532151808 weight_movement_bytes=34603008 total_nvlink_bytes=566754816 weight_to_token_ratio=0.065025 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130784160e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266100736 dispatch_meta_bytes=343112 combine_payload_bytes=266100736 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266443848 ingress_bytes=369909760 link_bytes=369909760 token_bytes=532544584 weight_movement_bytes=103809024 total_nvlink_bytes=636353608 weight_to_token_ratio=0.194930 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391940647e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74644 payload_rows_remote=65404 dup_rows=23660 dup_rows_remote=20690 dedup_ratio=0.240682 router_tokens_per_expert=[796, 826, 778, 801, 756, 823, 809, 748, 727, 771, 767, 782, 818, 725, 764, 772, 799, 730, 749, 790, 778, 737, 761, 729, 769, 757, 757, 756, 755, 776, 776, 759, 731, 704, 823, 776, 792, 726, 752, 760, 755, 758, 769, 796, 805, 829, 734, 733, 804, 748, 782, 735, 747, 777, 780, 794, 747, 788, 729, 786, 713, 778, 773, 787] payload_rows_per_dst=[9538, 9350, 9218, 9240, 9222, 9324, 9400, 9352] dup_rows_per_dst=[3136, 2902, 2928, 2970, 2906, 3034, 2934, 2850] combine_rows_per_src=[9538, 9350, 9218, 9240, 9222, 9324, 9400, 9352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74210 payload_rows_remote=65086 dup_rows=24094 dup_rows_remote=21080 dedup_ratio=0.245097 router_tokens_per_expert=[797, 787, 808, 790, 792, 755, 793, 751, 766, 747, 739, 775, 795, 750, 814, 750, 747, 769, 782, 748, 729, 717, 753, 767, 758, 783, 806, 763, 763, 763, 757, 787, 804, 819, 784, 784, 807, 753, 725, 781, 737, 776, 764, 771, 699, 772, 767, 820, 750, 820, 719, 742, 769, 759, 749, 761, 777, 808, 781, 745, 728, 772, 753, 755] payload_rows_per_dst=[9500, 9234, 9174, 9300, 9418, 9232, 9124, 9228] dup_rows_per_dst=[3046, 3038, 2850, 3060, 3096, 2980, 3014, 3010] combine_rows_per_src=[9500, 9234, 9174, 9300, 9418, 9232, 9124, 9228] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74730 payload_rows_remote=65432 dup_rows=23574 dup_rows_remote=20612 dedup_ratio=0.239807 router_tokens_per_expert=[782, 731, 772, 839, 754, 728, 791, 739, 762, 773, 773, 746, 731, 766, 747, 739, 783, 795, 759, 777, 773, 787, 810, 777, 767, 730, 775, 797, 771, 735, 765, 814, 729, 715, 751, 789, 764, 757, 789, 769, 757, 843, 760, 710, 761, 748, 781, 770, 784, 777, 719, 792, 774, 777, 804, 770, 791, 766, 779, 776, 747, 818, 746, 751] payload_rows_per_dst=[9336, 9184, 9492, 9314, 9284, 9298, 9376, 9446] dup_rows_per_dst=[2936, 2890, 3030, 2994, 2842, 2962, 3018, 2902] combine_rows_per_src=[9336, 9184, 9492, 9314, 9284, 9298, 9376, 9446] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74252 payload_rows_remote=65052 dup_rows=24052 dup_rows_remote=21072 dedup_ratio=0.244670 router_tokens_per_expert=[770, 783, 761, 785, 705, 753, 719, 739, 735, 810, 752, 741, 814, 848, 706, 746, 751, 785, 742, 799, 741, 778, 774, 748, 789, 805, 750, 775, 807, 759, 799, 762, 779, 756, 728, 766, 808, 784, 758, 711, 775, 791, 805, 766, 821, 782, 792, 788, 797, 746, 751, 751, 782, 780, 736, 710, 746, 794, 735, 764, 785, 818, 768, 748] payload_rows_per_dst=[9076, 9358, 9222, 9440, 9200, 9466, 9214, 9276] dup_rows_per_dst=[2954, 2946, 3014, 3052, 2980, 3174, 2892, 3040] combine_rows_per_src=[9076, 9358, 9222, 9440, 9200, 9466, 9214, 9276] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=64670 dup_rows=24072 dup_rows_remote=20880 dedup_ratio=0.244873 router_tokens_per_expert=[748, 737, 799, 755, 781, 712, 772, 815, 796, 728, 763, 816, 772, 742, 736, 744, 786, 716, 746, 758, 778, 794, 750, 745, 725, 780, 761, 804, 720, 763, 802, 765, 787, 790, 737, 764, 767, 795, 781, 776, 755, 745, 783, 704, 764, 785, 765, 750, 802, 759, 768, 755, 798, 787, 754, 854, 768, 781, 800, 765, 764, 804, 780, 756] payload_rows_per_dst=[9098, 9262, 8982, 9294, 9456, 9272, 9306, 9562] dup_rows_per_dst=[2830, 3000, 2764, 2946, 3180, 2902, 3258, 3192] combine_rows_per_src=[9098, 9262, 8982, 9294, 9456, 9272, 9306, 9562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=267894784 dispatch_meta_bytes=344376 combine_payload_bytes=267894784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268239160 ingress_bytes=302497792 link_bytes=302497792 token_bytes=536133944 weight_movement_bytes=34603008 total_nvlink_bytes=570736952 weight_to_token_ratio=0.064542 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138274838e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74414 payload_rows_remote=65116 dup_rows=23890 dup_rows_remote=20800 dedup_ratio=0.243022 router_tokens_per_expert=[835, 709, 726, 769, 801, 762, 750, 769, 811, 768, 750, 776, 753, 770, 745, 821, 853, 736, 826, 830, 783, 809, 743, 729, 767, 771, 754, 801, 740, 769, 788, 762, 767, 739, 723, 757, 732, 763, 734, 785, 730, 758, 810, 763, 771, 753, 787, 781, 774, 796, 784, 765, 776, 807, 781, 798, 716, 766, 758, 766, 754, 739, 725, 718] payload_rows_per_dst=[9330, 9298, 9510, 9314, 9134, 9298, 9452, 9078] dup_rows_per_dst=[2912, 3090, 3108, 2990, 2866, 3008, 3110, 2806] combine_rows_per_src=[9330, 9298, 9510, 9314, 9134, 9298, 9452, 9078] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266027008 dispatch_meta_bytes=343512 combine_payload_bytes=266027008 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266370520 ingress_bytes=369836032 link_bytes=369836032 token_bytes=532397528 weight_movement_bytes=103809024 total_nvlink_bytes=636206552 weight_to_token_ratio=0.194984 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391663214e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74210 payload_rows_remote=65086 dup_rows=24094 dup_rows_remote=21080 dedup_ratio=0.245097 router_tokens_per_expert=[797, 787, 808, 790, 792, 755, 793, 751, 766, 747, 739, 775, 795, 750, 814, 750, 747, 769, 782, 748, 729, 717, 753, 767, 758, 783, 806, 763, 763, 763, 757, 787, 804, 819, 784, 784, 807, 753, 725, 781, 737, 776, 764, 771, 699, 772, 767, 820, 750, 820, 719, 742, 769, 759, 749, 761, 777, 808, 781, 745, 728, 772, 753, 755] payload_rows_per_dst=[9500, 9234, 9174, 9300, 9418, 9232, 9124, 9228] dup_rows_per_dst=[3046, 3038, 2850, 3060, 3096, 2980, 3014, 3010] combine_rows_per_src=[9500, 9234, 9174, 9300, 9418, 9232, 9124, 9228] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=268009472 dispatch_meta_bytes=344176 combine_payload_bytes=268009472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268353648 ingress_bytes=302612480 link_bytes=302612480 token_bytes=536363120 weight_movement_bytes=34603008 total_nvlink_bytes=570966128 weight_to_token_ratio=0.064514 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138706400e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266452992 dispatch_meta_bytes=344496 combine_payload_bytes=266452992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266797488 ingress_bytes=335659008 link_bytes=335659008 token_bytes=533250480 weight_movement_bytes=69206016 total_nvlink_bytes=602456496 weight_to_token_ratio=0.129781 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263057824e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74644 payload_rows_remote=65404 dup_rows=23660 dup_rows_remote=20690 dedup_ratio=0.240682 router_tokens_per_expert=[796, 826, 778, 801, 756, 823, 809, 748, 727, 771, 767, 782, 818, 725, 764, 772, 799, 730, 749, 790, 778, 737, 761, 729, 769, 757, 757, 756, 755, 776, 776, 759, 731, 704, 823, 776, 792, 726, 752, 760, 755, 758, 769, 796, 805, 829, 734, 733, 804, 748, 782, 735, 747, 777, 780, 794, 747, 788, 729, 786, 713, 778, 773, 787] payload_rows_per_dst=[9538, 9350, 9218, 9240, 9222, 9324, 9400, 9352] dup_rows_per_dst=[3136, 2902, 2928, 2970, 2906, 3034, 2934, 2850] combine_rows_per_src=[9538, 9350, 9218, 9240, 9222, 9324, 9400, 9352] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266715136 dispatch_meta_bytes=343664 combine_payload_bytes=266715136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267058800 ingress_bytes=301318144 link_bytes=301318144 token_bytes=533773936 weight_movement_bytes=34603008 total_nvlink_bytes=568376944 weight_to_token_ratio=0.064827 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133835918e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=64670 dup_rows=24072 dup_rows_remote=20880 dedup_ratio=0.244873 router_tokens_per_expert=[748, 737, 799, 755, 781, 712, 772, 815, 796, 728, 763, 816, 772, 742, 736, 744, 786, 716, 746, 758, 778, 794, 750, 745, 725, 780, 761, 804, 720, 763, 802, 765, 787, 790, 737, 764, 767, 795, 781, 776, 755, 745, 783, 704, 764, 785, 765, 750, 802, 759, 768, 755, 798, 787, 754, 854, 768, 781, 800, 765, 764, 804, 780, 756] payload_rows_per_dst=[9098, 9262, 8982, 9294, 9456, 9272, 9306, 9562] dup_rows_per_dst=[2830, 3000, 2764, 2946, 3180, 2902, 3258, 3192] combine_rows_per_src=[9098, 9262, 8982, 9294, 9456, 9272, 9306, 9562] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74252 payload_rows_remote=64890 dup_rows=24052 dup_rows_remote=20950 dedup_ratio=0.244670 router_tokens_per_expert=[755, 725, 804, 778, 755, 746, 799, 754, 789, 782, 752, 717, 764, 747, 829, 782, 764, 778, 786, 793, 779, 797, 744, 791, 765, 746, 742, 736, 752, 762, 763, 733, 773, 749, 760, 758, 795, 743, 776, 749, 762, 749, 764, 843, 762, 761, 792, 790, 735, 776, 791, 766, 782, 784, 786, 753, 759, 783, 712, 758, 792, 788, 793, 759] payload_rows_per_dst=[9250, 9332, 9362, 9124, 9216, 9306, 9416, 9246] dup_rows_per_dst=[2982, 2992, 3102, 2874, 2990, 3140, 2930, 3042] combine_rows_per_src=[9250, 9332, 9362, 9124, 9216, 9306, 9416, 9246] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266592256 dispatch_meta_bytes=344664 combine_payload_bytes=266592256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266936920 ingress_bytes=301195264 link_bytes=301195264 token_bytes=533529176 weight_movement_bytes=34603008 total_nvlink_bytes=568132184 weight_to_token_ratio=0.064857 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133373530e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=267894784 dispatch_meta_bytes=344376 combine_payload_bytes=267894784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268239160 ingress_bytes=302497792 link_bytes=302497792 token_bytes=536133944 weight_movement_bytes=34603008 total_nvlink_bytes=570736952 weight_to_token_ratio=0.064542 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138274838e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74166 payload_rows_remote=64964 dup_rows=24138 dup_rows_remote=21270 dedup_ratio=0.245544 router_tokens_per_expert=[766, 815, 741, 750, 801, 814, 721, 769, 753, 761, 776, 786, 792, 793, 815, 779, 779, 774, 782, 801, 789, 690, 746, 742, 779, 797, 747, 771, 765, 783, 770, 744, 715, 786, 741, 691, 782, 727, 803, 790, 754, 721, 786, 792, 745, 789, 775, 824, 810, 806, 751, 814, 745, 733, 733, 764, 757, 759, 767, 733, 792, 795, 734, 747] payload_rows_per_dst=[9224, 9428, 9248, 9268, 9202, 9318, 9290, 9188] dup_rows_per_dst=[3130, 3082, 2958, 3044, 2868, 3054, 3022, 2980] combine_rows_per_src=[9224, 9428, 9248, 9268, 9202, 9318, 9290, 9188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266592256 dispatch_meta_bytes=344664 combine_payload_bytes=266592256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266936920 ingress_bytes=301195264 link_bytes=301195264 token_bytes=533529176 weight_movement_bytes=34603008 total_nvlink_bytes=568132184 weight_to_token_ratio=0.064857 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133373530e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=268009472 dispatch_meta_bytes=344176 combine_payload_bytes=268009472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268353648 ingress_bytes=302612480 link_bytes=302612480 token_bytes=536363120 weight_movement_bytes=34603008 total_nvlink_bytes=570966128 weight_to_token_ratio=0.064514 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138706400e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74346 payload_rows_remote=64836 dup_rows=23958 dup_rows_remote=20768 dedup_ratio=0.243713 router_tokens_per_expert=[792, 755, 772, 782, 800, 796, 749, 784, 824, 779, 798, 770, 769, 737, 815, 755, 777, 763, 765, 725, 796, 722, 741, 742, 725, 773, 782, 739, 776, 747, 759, 771, 798, 787, 737, 764, 737, 778, 774, 715, 767, 799, 814, 759, 790, 729, 782, 741, 788, 755, 781, 803, 709, 733, 785, 798, 770, 758, 773, 741, 788, 758, 758, 803] payload_rows_per_dst=[9510, 9502, 9166, 9178, 9330, 9244, 9116, 9300] dup_rows_per_dst=[3190, 3092, 2896, 2980, 3074, 2894, 2834, 2998] combine_rows_per_src=[9510, 9502, 9166, 9178, 9330, 9244, 9116, 9300] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=264888320 dispatch_meta_bytes=342200 combine_payload_bytes=264888320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265230520 ingress_bytes=299491328 link_bytes=299491328 token_bytes=530118840 weight_movement_bytes=34603008 total_nvlink_bytes=564721848 weight_to_token_ratio=0.065274 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126961756e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=64966 dup_rows=23966 dup_rows_remote=20812 dedup_ratio=0.243795 router_tokens_per_expert=[787, 743, 789, 791, 806, 754, 798, 779, 770, 773, 787, 786, 786, 769, 743, 755, 765, 775, 772, 766, 760, 819, 832, 746, 731, 764, 713, 761, 773, 788, 750, 738, 749, 767, 764, 758, 711, 762, 790, 766, 798, 748, 771, 755, 769, 770, 776, 775, 711, 784, 767, 775, 802, 759, 798, 759, 736, 725, 774, 814, 747, 796, 761, 746] payload_rows_per_dst=[9372, 9366, 9456, 9514, 9186, 9284, 8870, 9290] dup_rows_per_dst=[3154, 2972, 3014, 3112, 2948, 3040, 2818, 2908] combine_rows_per_src=[9372, 9366, 9456, 9514, 9186, 9284, 8870, 9290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266715136 dispatch_meta_bytes=343664 combine_payload_bytes=266715136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267058800 ingress_bytes=301318144 link_bytes=301318144 token_bytes=533773936 weight_movement_bytes=34603008 total_nvlink_bytes=568376944 weight_to_token_ratio=0.064827 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133835918e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265789440 dispatch_meta_bytes=343360 combine_payload_bytes=265789440 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266132800 ingress_bytes=369598464 link_bytes=369598464 token_bytes=531922240 weight_movement_bytes=103809024 total_nvlink_bytes=635731264 weight_to_token_ratio=0.195158 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390769265e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74352 payload_rows_remote=64918 dup_rows=23952 dup_rows_remote=20970 dedup_ratio=0.243652 router_tokens_per_expert=[768, 782, 770, 794, 772, 768, 730, 710, 761, 770, 757, 763, 776, 786, 794, 752, 760, 753, 766, 765, 738, 786, 745, 761, 797, 733, 802, 798, 764, 820, 782, 781, 775, 835, 768, 728, 760, 782, 746, 814, 805, 763, 784, 790, 734, 729, 752, 747, 781, 789, 774, 757, 746, 719, 749, 746, 783, 803, 724, 796, 788, 774, 745, 762] payload_rows_per_dst=[9256, 9324, 9148, 9440, 9434, 9274, 9136, 9340] dup_rows_per_dst=[2932, 2994, 3000, 3114, 2982, 2934, 2986, 3010] combine_rows_per_src=[9256, 9324, 9148, 9440, 9434, 9274, 9136, 9340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=344936 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266437480 ingress_bytes=300695552 link_bytes=300695552 token_bytes=532530024 weight_movement_bytes=34603008 total_nvlink_bytes=567133032 weight_to_token_ratio=0.064979 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131493154e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265568256 dispatch_meta_bytes=342416 combine_payload_bytes=265568256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265910672 ingress_bytes=300171264 link_bytes=300171264 token_bytes=531478928 weight_movement_bytes=34603008 total_nvlink_bytes=566081936 weight_to_token_ratio=0.065107 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129520301e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74078 payload_rows_remote=64794 dup_rows=24226 dup_rows_remote=21038 dedup_ratio=0.246440 router_tokens_per_expert=[753, 742, 748, 771, 763, 771, 743, 751, 802, 814, 740, 782, 786, 760, 783, 759, 754, 773, 822, 711, 752, 758, 722, 759, 772, 763, 810, 749, 741, 819, 807, 775, 770, 725, 760, 771, 769, 800, 758, 760, 752, 802, 753, 761, 732, 713, 760, 804, 780, 750, 806, 813, 788, 745, 783, 773, 750, 784, 751, 793, 766, 775, 793, 757] payload_rows_per_dst=[9202, 9346, 9172, 9284, 9320, 9198, 9308, 9248] dup_rows_per_dst=[2882, 3106, 2930, 3188, 2906, 2956, 3168, 3090] combine_rows_per_src=[9202, 9346, 9172, 9284, 9320, 9198, 9308, 9248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64992 dup_rows=24068 dup_rows_remote=21018 dedup_ratio=0.244832 router_tokens_per_expert=[802, 757, 773, 764, 746, 738, 765, 776, 800, 764, 795, 755, 740, 789, 761, 770, 747, 789, 740, 759, 744, 798, 747, 771, 713, 745, 774, 775, 739, 722, 808, 756, 763, 843, 812, 788, 786, 793, 738, 806, 772, 692, 756, 824, 794, 788, 781, 707, 778, 774, 762, 743, 857, 717, 791, 725, 754, 780, 746, 792, 762, 781, 756, 769] payload_rows_per_dst=[9260, 9318, 9258, 9210, 9462, 9228, 9244, 9256] dup_rows_per_dst=[2982, 3030, 2932, 2854, 3196, 3000, 3050, 3024] combine_rows_per_src=[9260, 9318, 9258, 9210, 9462, 9228, 9244, 9256] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74266 payload_rows_remote=64900 dup_rows=24038 dup_rows_remote=20974 dedup_ratio=0.244527 router_tokens_per_expert=[771, 728, 789, 798, 795, 732, 782, 747, 794, 771, 742, 805, 751, 765, 793, 804, 774, 751, 763, 702, 762, 743, 786, 783, 759, 744, 777, 760, 806, 768, 772, 778, 767, 736, 766, 775, 731, 764, 738, 782, 768, 781, 775, 759, 764, 776, 793, 799, 761, 752, 731, 794, 811, 753, 779, 754, 781, 742, 796, 726, 767, 738, 807, 791] payload_rows_per_dst=[9294, 9332, 9236, 9228, 9216, 9366, 9270, 9324] dup_rows_per_dst=[2990, 3118, 2892, 3100, 2902, 3064, 3000, 2972] combine_rows_per_src=[9294, 9332, 9236, 9228, 9216, 9366, 9270, 9324] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265396224 dispatch_meta_bytes=343328 combine_payload_bytes=265396224 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265739552 ingress_bytes=334602240 link_bytes=334602240 token_bytes=531135776 weight_movement_bytes=69206016 total_nvlink_bytes=600341792 weight_to_token_ratio=0.130298 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259081291e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74368 payload_rows_remote=64906 dup_rows=23936 dup_rows_remote=20892 dedup_ratio=0.243490 router_tokens_per_expert=[790, 786, 764, 752, 826, 769, 717, 744, 812, 750, 727, 764, 741, 759, 788, 751, 765, 796, 752, 741, 778, 762, 781, 768, 742, 765, 790, 786, 745, 783, 736, 767, 771, 727, 752, 804, 735, 756, 756, 750, 807, 754, 762, 781, 778, 783, 776, 790, 803, 778, 744, 762, 806, 751, 812, 788, 780, 764, 706, 754, 788, 776, 796, 765] payload_rows_per_dst=[9264, 8776, 9670, 9290, 9252, 9364, 9290, 9462] dup_rows_per_dst=[3032, 2538, 3486, 2906, 2850, 3130, 2950, 3044] combine_rows_per_src=[9264, 8776, 9670, 9290, 9252, 9364, 9290, 9462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74280 payload_rows_remote=65064 dup_rows=24024 dup_rows_remote=21060 dedup_ratio=0.244385 router_tokens_per_expert=[775, 763, 715, 771, 791, 758, 761, 793, 769, 722, 802, 739, 793, 778, 731, 756, 754, 780, 801, 749, 774, 781, 752, 803, 791, 762, 786, 768, 719, 744, 745, 788, 798, 759, 788, 740, 779, 789, 763, 784, 752, 772, 766, 787, 742, 823, 778, 769, 738, 801, 740, 788, 782, 742, 771, 729, 781, 779, 794, 763, 759, 736, 793, 753] payload_rows_per_dst=[9264, 9216, 9320, 9206, 9390, 9252, 9278, 9354] dup_rows_per_dst=[2990, 2964, 3068, 3000, 3010, 3126, 2904, 2962] combine_rows_per_src=[9264, 9216, 9320, 9206, 9390, 9252, 9278, 9354] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266100736 dispatch_meta_bytes=343112 combine_payload_bytes=266100736 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266443848 ingress_bytes=369909760 link_bytes=369909760 token_bytes=532544584 weight_movement_bytes=103809024 total_nvlink_bytes=636353608 weight_to_token_ratio=0.194930 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391940647e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64992 dup_rows=24068 dup_rows_remote=21018 dedup_ratio=0.244832 router_tokens_per_expert=[802, 757, 773, 764, 746, 738, 765, 776, 800, 764, 795, 755, 740, 789, 761, 770, 747, 789, 740, 759, 744, 798, 747, 771, 713, 745, 774, 775, 739, 722, 808, 756, 763, 843, 812, 788, 786, 793, 738, 806, 772, 692, 756, 824, 794, 788, 781, 707, 778, 774, 762, 743, 857, 717, 791, 725, 754, 780, 746, 792, 762, 781, 756, 769] payload_rows_per_dst=[9260, 9318, 9258, 9210, 9462, 9228, 9244, 9256] dup_rows_per_dst=[2982, 3030, 2932, 2854, 3196, 3000, 3050, 3024] combine_rows_per_src=[9260, 9318, 9258, 9210, 9462, 9228, 9244, 9256] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=343552 combine_payload_bytes=265904128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266247680 ingress_bytes=300507136 link_bytes=300507136 token_bytes=532151808 weight_movement_bytes=34603008 total_nvlink_bytes=566754816 weight_to_token_ratio=0.065025 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130784160e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265830400 dispatch_meta_bytes=343496 combine_payload_bytes=265830400 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266173896 ingress_bytes=335036416 link_bytes=335036416 token_bytes=532004296 weight_movement_bytes=69206016 total_nvlink_bytes=601210312 weight_to_token_ratio=0.130085 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260715061e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74078 payload_rows_remote=64794 dup_rows=24226 dup_rows_remote=21038 dedup_ratio=0.246440 router_tokens_per_expert=[753, 742, 748, 771, 763, 771, 743, 751, 802, 814, 740, 782, 786, 760, 783, 759, 754, 773, 822, 711, 752, 758, 722, 759, 772, 763, 810, 749, 741, 819, 807, 775, 770, 725, 760, 771, 769, 800, 758, 760, 752, 802, 753, 761, 732, 713, 760, 804, 780, 750, 806, 813, 788, 745, 783, 773, 750, 784, 751, 793, 766, 775, 793, 757] payload_rows_per_dst=[9202, 9346, 9172, 9284, 9320, 9198, 9308, 9248] dup_rows_per_dst=[2882, 3106, 2930, 3188, 2906, 2956, 3168, 3090] combine_rows_per_src=[9202, 9346, 9172, 9284, 9320, 9198, 9308, 9248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266502144 dispatch_meta_bytes=344496 combine_payload_bytes=266502144 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266846640 ingress_bytes=335708160 link_bytes=335708160 token_bytes=533348784 weight_movement_bytes=69206016 total_nvlink_bytes=602554800 weight_to_token_ratio=0.129758 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263242779e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74368 payload_rows_remote=64906 dup_rows=23936 dup_rows_remote=20892 dedup_ratio=0.243490 router_tokens_per_expert=[790, 786, 764, 752, 826, 769, 717, 744, 812, 750, 727, 764, 741, 759, 788, 751, 765, 796, 752, 741, 778, 762, 781, 768, 742, 765, 790, 786, 745, 783, 736, 767, 771, 727, 752, 804, 735, 756, 756, 750, 807, 754, 762, 781, 778, 783, 776, 790, 803, 778, 744, 762, 806, 751, 812, 788, 780, 764, 706, 754, 788, 776, 796, 765] payload_rows_per_dst=[9264, 8776, 9670, 9290, 9252, 9364, 9290, 9462] dup_rows_per_dst=[3032, 2538, 3486, 2906, 2850, 3130, 2950, 3044] combine_rows_per_src=[9264, 8776, 9670, 9290, 9252, 9364, 9290, 9462] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266207232 dispatch_meta_bytes=344040 combine_payload_bytes=266207232 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266551272 ingress_bytes=370016256 link_bytes=370016256 token_bytes=532758504 weight_movement_bytes=103809024 total_nvlink_bytes=636567528 weight_to_token_ratio=0.194852 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392341383e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74000 payload_rows_remote=64806 dup_rows=24304 dup_rows_remote=21308 dedup_ratio=0.247233 router_tokens_per_expert=[747, 791, 759, 733, 799, 775, 741, 757, 804, 757, 765, 808, 791, 799, 745, 800, 791, 758, 725, 727, 767, 749, 793, 785, 786, 773, 748, 778, 771, 799, 786, 711, 766, 800, 772, 810, 790, 761, 769, 705, 772, 731, 761, 755, 826, 722, 760, 781, 822, 744, 761, 746, 802, 774, 793, 783, 768, 746, 739, 742, 721, 727, 798, 787] payload_rows_per_dst=[9152, 9444, 9194, 9280, 9340, 9148, 9394, 9048] dup_rows_per_dst=[3052, 3094, 2996, 3024, 3006, 3068, 3056, 3008] combine_rows_per_src=[9152, 9444, 9194, 9280, 9340, 9148, 9394, 9048] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=64900 dup_rows=23998 dup_rows_remote=20886 dedup_ratio=0.244120 router_tokens_per_expert=[781, 767, 730, 770, 817, 753, 760, 768, 795, 767, 776, 807, 767, 766, 746, 759, 750, 762, 759, 743, 722, 768, 755, 755, 787, 750, 724, 777, 744, 768, 754, 800, 822, 769, 778, 800, 731, 778, 792, 789, 776, 787, 790, 744, 711, 782, 779, 761, 763, 761, 762, 747, 800, 779, 771, 762, 764, 843, 808, 757, 766, 750, 740, 743] payload_rows_per_dst=[9258, 9328, 9150, 9266, 9406, 9302, 9276, 9320] dup_rows_per_dst=[3034, 3038, 2878, 2942, 3112, 2958, 3014, 3022] combine_rows_per_src=[9258, 9328, 9150, 9266, 9406, 9302, 9276, 9320] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265396224 dispatch_meta_bytes=343328 combine_payload_bytes=265396224 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265739552 ingress_bytes=334602240 link_bytes=334602240 token_bytes=531135776 weight_movement_bytes=69206016 total_nvlink_bytes=600341792 weight_to_token_ratio=0.130298 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259081291e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266207232 dispatch_meta_bytes=344040 combine_payload_bytes=266207232 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266551272 ingress_bytes=370016256 link_bytes=370016256 token_bytes=532758504 weight_movement_bytes=103809024 total_nvlink_bytes=636567528 weight_to_token_ratio=0.194852 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392341383e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265830400 dispatch_meta_bytes=343496 combine_payload_bytes=265830400 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266173896 ingress_bytes=335036416 link_bytes=335036416 token_bytes=532004296 weight_movement_bytes=69206016 total_nvlink_bytes=601210312 weight_to_token_ratio=0.130085 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260715061e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74290 payload_rows_remote=64988 dup_rows=24014 dup_rows_remote=20990 dedup_ratio=0.244283 router_tokens_per_expert=[742, 743, 777, 754, 793, 755, 780, 799, 743, 774, 744, 753, 746, 734, 755, 746, 788, 776, 712, 778, 737, 808, 801, 779, 756, 766, 753, 758, 777, 783, 763, 765, 778, 792, 776, 819, 785, 755, 784, 736, 776, 730, 774, 785, 755, 779, 755, 780, 763, 815, 816, 746, 743, 788, 756, 776, 758, 761, 755, 763, 791, 793, 788, 743] payload_rows_per_dst=[9302, 9170, 9242, 9250, 9404, 9410, 9218, 9294] dup_rows_per_dst=[3024, 2982, 3082, 2992, 3150, 2858, 3078, 2848] combine_rows_per_src=[9302, 9170, 9242, 9250, 9404, 9410, 9218, 9294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265854976 dispatch_meta_bytes=343192 combine_payload_bytes=265854976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266198168 ingress_bytes=300457984 link_bytes=300457984 token_bytes=532053144 weight_movement_bytes=34603008 total_nvlink_bytes=566656152 weight_to_token_ratio=0.065037 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130599205e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74346 payload_rows_remote=64836 dup_rows=23958 dup_rows_remote=20768 dedup_ratio=0.243713 router_tokens_per_expert=[792, 755, 772, 782, 800, 796, 749, 784, 824, 779, 798, 770, 769, 737, 815, 755, 777, 763, 765, 725, 796, 722, 741, 742, 725, 773, 782, 739, 776, 747, 759, 771, 798, 787, 737, 764, 737, 778, 774, 715, 767, 799, 814, 759, 790, 729, 782, 741, 788, 755, 781, 803, 709, 733, 785, 798, 770, 758, 773, 741, 788, 758, 758, 803] payload_rows_per_dst=[9510, 9502, 9166, 9178, 9330, 9244, 9116, 9300] dup_rows_per_dst=[3190, 3092, 2896, 2980, 3074, 2894, 2834, 2998] combine_rows_per_src=[9510, 9502, 9166, 9178, 9330, 9244, 9116, 9300] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74166 payload_rows_remote=64964 dup_rows=24138 dup_rows_remote=21270 dedup_ratio=0.245544 router_tokens_per_expert=[766, 815, 741, 750, 801, 814, 721, 769, 753, 761, 776, 786, 792, 793, 815, 779, 779, 774, 782, 801, 789, 690, 746, 742, 779, 797, 747, 771, 765, 783, 770, 744, 715, 786, 741, 691, 782, 727, 803, 790, 754, 721, 786, 792, 745, 789, 775, 824, 810, 806, 751, 814, 745, 733, 733, 764, 757, 759, 767, 733, 792, 795, 734, 747] payload_rows_per_dst=[9224, 9428, 9248, 9268, 9202, 9318, 9290, 9188] dup_rows_per_dst=[3130, 3082, 2958, 3044, 2868, 3054, 3022, 2980] combine_rows_per_src=[9224, 9428, 9248, 9268, 9202, 9318, 9290, 9188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74240 payload_rows_remote=65030 dup_rows=24064 dup_rows_remote=21048 dedup_ratio=0.244792 router_tokens_per_expert=[760, 808, 792, 771, 796, 752, 786, 766, 752, 742, 744, 782, 789, 811, 806, 824, 727, 721, 763, 812, 769, 806, 746, 778, 744, 735, 780, 787, 776, 756, 749, 780, 791, 742, 801, 745, 727, 791, 754, 779, 763, 764, 758, 826, 745, 780, 758, 719, 722, 728, 762, 762, 791, 819, 805, 707, 793, 735, 767, 805, 722, 781, 761, 739] payload_rows_per_dst=[9366, 9404, 9236, 9168, 9344, 9210, 9232, 9280] dup_rows_per_dst=[3096, 3096, 3008, 3046, 2916, 3016, 2960, 2926] combine_rows_per_src=[9366, 9404, 9236, 9168, 9344, 9210, 9232, 9280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266502144 dispatch_meta_bytes=344496 combine_payload_bytes=266502144 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266846640 ingress_bytes=335708160 link_bytes=335708160 token_bytes=533348784 weight_movement_bytes=69206016 total_nvlink_bytes=602554800 weight_to_token_ratio=0.129758 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263242779e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265445376 dispatch_meta_bytes=344456 combine_payload_bytes=265445376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265789832 ingress_bytes=300048384 link_bytes=300048384 token_bytes=531235208 weight_movement_bytes=34603008 total_nvlink_bytes=565838216 weight_to_token_ratio=0.065137 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129057913e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74402 payload_rows_remote=65046 dup_rows=23902 dup_rows_remote=20918 dedup_ratio=0.243144 router_tokens_per_expert=[810, 792, 799, 796, 765, 734, 802, 770, 764, 777, 761, 767, 763, 759, 790, 789, 780, 793, 800, 808, 740, 777, 755, 744, 759, 795, 734, 771, 747, 727, 778, 744, 779, 785, 743, 699, 737, 801, 778, 781, 767, 806, 719, 794, 754, 722, 739, 768, 738, 749, 759, 788, 774, 784, 785, 778, 800, 742, 753, 755, 746, 788, 784, 767] payload_rows_per_dst=[9404, 9356, 9284, 9286, 9220, 9310, 9300, 9242] dup_rows_per_dst=[3132, 2984, 3110, 2824, 2986, 2828, 3010, 3028] combine_rows_per_src=[9404, 9356, 9284, 9286, 9220, 9310, 9300, 9242] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265568256 dispatch_meta_bytes=342416 combine_payload_bytes=265568256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265910672 ingress_bytes=300171264 link_bytes=300171264 token_bytes=531478928 weight_movement_bytes=34603008 total_nvlink_bytes=566081936 weight_to_token_ratio=0.065107 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129520301e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265830400 dispatch_meta_bytes=343144 combine_payload_bytes=265830400 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266173544 ingress_bytes=335036416 link_bytes=335036416 token_bytes=532003944 weight_movement_bytes=69206016 total_nvlink_bytes=601209960 weight_to_token_ratio=0.130086 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260715061e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=64886 dup_rows=24096 dup_rows_remote=21094 dedup_ratio=0.245117 router_tokens_per_expert=[800, 728, 760, 757, 801, 738, 744, 760, 761, 733, 772, 810, 753, 789, 747, 738, 753, 805, 764, 746, 811, 802, 782, 776, 720, 793, 746, 757, 775, 820, 779, 772, 789, 742, 769, 779, 760, 750, 818, 757, 720, 766, 793, 774, 768, 748, 803, 776, 784, 750, 756, 743, 789, 776, 764, 770, 795, 744, 785, 761, 743, 810, 715, 763] payload_rows_per_dst=[9254, 9172, 9404, 9322, 9294, 9260, 9260, 9242] dup_rows_per_dst=[2922, 3034, 3074, 3002, 3034, 3036, 3004, 2990] combine_rows_per_src=[9254, 9172, 9404, 9322, 9294, 9260, 9260, 9242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266190848 dispatch_meta_bytes=343912 combine_payload_bytes=266190848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266534760 ingress_bytes=300793856 link_bytes=300793856 token_bytes=532725608 weight_movement_bytes=34603008 total_nvlink_bytes=567328616 weight_to_token_ratio=0.064955 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131863064e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=64950 dup_rows=23998 dup_rows_remote=20930 dedup_ratio=0.244120 router_tokens_per_expert=[745, 783, 765, 743, 751, 780, 756, 741, 729, 799, 738, 724, 805, 783, 799, 756, 807, 766, 703, 802, 734, 753, 732, 739, 776, 744, 761, 792, 761, 738, 784, 808, 863, 764, 792, 818, 786, 766, 765, 765, 741, 737, 752, 778, 762, 749, 774, 768, 823, 805, 754, 771, 759, 731, 786, 783, 789, 757, 754, 762, 744, 752, 826, 779] payload_rows_per_dst=[9228, 9292, 9162, 9258, 9420, 9232, 9356, 9358] dup_rows_per_dst=[2900, 2974, 2910, 3070, 3218, 2890, 3068, 2968] combine_rows_per_src=[9228, 9292, 9162, 9258, 9420, 9232, 9356, 9358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=344936 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266437480 ingress_bytes=300695552 link_bytes=300695552 token_bytes=532530024 weight_movement_bytes=34603008 total_nvlink_bytes=567133032 weight_to_token_ratio=0.064979 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131493154e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344312 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266707192 ingress_bytes=300965888 link_bytes=300965888 token_bytes=533070072 weight_movement_bytes=34603008 total_nvlink_bytes=567673080 weight_to_token_ratio=0.064913 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132510407e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=343856 combine_payload_bytes=266428416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266772272 ingress_bytes=301031424 link_bytes=301031424 token_bytes=533200688 weight_movement_bytes=34603008 total_nvlink_bytes=567803696 weight_to_token_ratio=0.064897 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132757013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=343920 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266116976 ingress_bytes=300376064 link_bytes=300376064 token_bytes=531890032 weight_movement_bytes=34603008 total_nvlink_bytes=566493040 weight_to_token_ratio=0.065057 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130290947e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74240 payload_rows_remote=65030 dup_rows=24064 dup_rows_remote=21048 dedup_ratio=0.244792 router_tokens_per_expert=[760, 808, 792, 771, 796, 752, 786, 766, 752, 742, 744, 782, 789, 811, 806, 824, 727, 721, 763, 812, 769, 806, 746, 778, 744, 735, 780, 787, 776, 756, 749, 780, 791, 742, 801, 745, 727, 791, 754, 779, 763, 764, 758, 826, 745, 780, 758, 719, 722, 728, 762, 762, 791, 819, 805, 707, 793, 735, 767, 805, 722, 781, 761, 739] payload_rows_per_dst=[9366, 9404, 9236, 9168, 9344, 9210, 9232, 9280] dup_rows_per_dst=[3096, 3096, 3008, 3046, 2916, 3016, 2960, 2926] combine_rows_per_src=[9366, 9404, 9236, 9168, 9344, 9210, 9232, 9280] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=64886 dup_rows=24096 dup_rows_remote=21094 dedup_ratio=0.245117 router_tokens_per_expert=[800, 728, 760, 757, 801, 738, 744, 760, 761, 733, 772, 810, 753, 789, 747, 738, 753, 805, 764, 746, 811, 802, 782, 776, 720, 793, 746, 757, 775, 820, 779, 772, 789, 742, 769, 779, 760, 750, 818, 757, 720, 766, 793, 774, 768, 748, 803, 776, 784, 750, 756, 743, 789, 776, 764, 770, 795, 744, 785, 761, 743, 810, 715, 763] payload_rows_per_dst=[9254, 9172, 9404, 9322, 9294, 9260, 9260, 9242] dup_rows_per_dst=[2922, 3034, 3074, 3002, 3034, 3036, 3004, 2990] combine_rows_per_src=[9254, 9172, 9404, 9322, 9294, 9260, 9260, 9242] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=64918 dup_rows=24072 dup_rows_remote=21022 dedup_ratio=0.244873 router_tokens_per_expert=[699, 776, 777, 812, 746, 750, 774, 748, 751, 759, 762, 788, 705, 788, 765, 822, 764, 763, 779, 774, 793, 768, 781, 821, 761, 759, 737, 742, 715, 774, 716, 755, 762, 754, 722, 792, 812, 744, 762, 776, 770, 766, 843, 765, 768, 764, 741, 766, 784, 800, 763, 782, 770, 801, 722, 831, 755, 791, 773, 766, 780, 766, 770, 767] payload_rows_per_dst=[9262, 9306, 9114, 9302, 9040, 9516, 9378, 9314] dup_rows_per_dst=[2902, 2974, 2914, 3046, 2814, 3244, 3128, 3050] combine_rows_per_src=[9262, 9306, 9114, 9302, 9040, 9516, 9378, 9314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74402 payload_rows_remote=65046 dup_rows=23902 dup_rows_remote=20918 dedup_ratio=0.243144 router_tokens_per_expert=[810, 792, 799, 796, 765, 734, 802, 770, 764, 777, 761, 767, 763, 759, 790, 789, 780, 793, 800, 808, 740, 777, 755, 744, 759, 795, 734, 771, 747, 727, 778, 744, 779, 785, 743, 699, 737, 801, 778, 781, 767, 806, 719, 794, 754, 722, 739, 768, 738, 749, 759, 788, 774, 784, 785, 778, 800, 742, 753, 755, 746, 788, 784, 767] payload_rows_per_dst=[9404, 9356, 9284, 9286, 9220, 9310, 9300, 9242] dup_rows_per_dst=[3132, 2984, 3110, 2824, 2986, 2828, 3010, 3028] combine_rows_per_src=[9404, 9356, 9284, 9286, 9220, 9310, 9300, 9242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74432 payload_rows_remote=65118 dup_rows=23872 dup_rows_remote=20786 dedup_ratio=0.242839 router_tokens_per_expert=[816, 759, 780, 736, 806, 748, 699, 765, 725, 760, 789, 783, 738, 767, 765, 793, 778, 737, 771, 756, 768, 808, 822, 760, 810, 780, 776, 793, 777, 753, 766, 785, 784, 769, 761, 757, 733, 734, 794, 750, 711, 767, 811, 751, 775, 723, 825, 706, 784, 763, 791, 747, 714, 745, 793, 755, 769, 768, 779, 777, 770, 798, 789, 790] payload_rows_per_dst=[9306, 9248, 9314, 9428, 9258, 9208, 9230, 9440] dup_rows_per_dst=[2912, 2992, 3086, 3052, 2906, 2930, 2954, 3040] combine_rows_per_src=[9306, 9248, 9314, 9428, 9258, 9208, 9230, 9440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74410 payload_rows_remote=64920 dup_rows=23894 dup_rows_remote=20788 dedup_ratio=0.243062 router_tokens_per_expert=[816, 756, 748, 781, 780, 768, 731, 716, 774, 735, 771, 774, 797, 752, 760, 729, 746, 757, 744, 806, 747, 805, 800, 742, 724, 807, 763, 764, 740, 730, 810, 789, 786, 794, 785, 782, 783, 818, 750, 800, 750, 771, 800, 738, 758, 741, 742, 785, 770, 763, 757, 769, 764, 819, 750, 782, 775, 738, 766, 756, 803, 753, 786, 756] payload_rows_per_dst=[9228, 9196, 9364, 9256, 9490, 9204, 9328, 9344] dup_rows_per_dst=[2964, 2988, 2930, 2998, 3106, 2966, 3020, 2922] combine_rows_per_src=[9228, 9196, 9364, 9256, 9490, 9204, 9328, 9344] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266035200 dispatch_meta_bytes=343520 combine_payload_bytes=266035200 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266378720 ingress_bytes=404447232 link_bytes=404447232 token_bytes=532413920 weight_movement_bytes=138412032 total_nvlink_bytes=670825952 weight_to_token_ratio=0.259971 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521902373e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74432 payload_rows_remote=65118 dup_rows=23872 dup_rows_remote=20786 dedup_ratio=0.242839 router_tokens_per_expert=[816, 759, 780, 736, 806, 748, 699, 765, 725, 760, 789, 783, 738, 767, 765, 793, 778, 737, 771, 756, 768, 808, 822, 760, 810, 780, 776, 793, 777, 753, 766, 785, 784, 769, 761, 757, 733, 734, 794, 750, 711, 767, 811, 751, 775, 723, 825, 706, 784, 763, 791, 747, 714, 745, 793, 755, 769, 768, 779, 777, 770, 798, 789, 790] payload_rows_per_dst=[9306, 9248, 9314, 9428, 9258, 9208, 9230, 9440] dup_rows_per_dst=[2912, 2992, 3086, 3052, 2906, 2930, 2954, 3040] combine_rows_per_src=[9306, 9248, 9314, 9428, 9258, 9208, 9230, 9440] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=343920 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266116976 ingress_bytes=300376064 link_bytes=300376064 token_bytes=531890032 weight_movement_bytes=34603008 total_nvlink_bytes=566493040 weight_to_token_ratio=0.065057 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130290947e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=343760 combine_payload_bytes=265904128 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266247888 ingress_bytes=335110144 link_bytes=335110144 token_bytes=532152016 weight_movement_bytes=69206016 total_nvlink_bytes=601358032 weight_to_token_ratio=0.130049 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260992493e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344312 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266707192 ingress_bytes=300965888 link_bytes=300965888 token_bytes=533070072 weight_movement_bytes=34603008 total_nvlink_bytes=567673080 weight_to_token_ratio=0.064913 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132510407e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74366 payload_rows_remote=64880 dup_rows=23938 dup_rows_remote=20772 dedup_ratio=0.243510 router_tokens_per_expert=[774, 819, 762, 763, 787, 795, 810, 741, 798, 733, 741, 751, 771, 757, 750, 734, 747, 745, 747, 719, 786, 733, 790, 785, 771, 759, 761, 755, 792, 763, 725, 804, 761, 730, 763, 770, 768, 774, 788, 776, 745, 785, 752, 767, 740, 807, 771, 746, 800, 756, 810, 798, 791, 743, 758, 786, 760, 775, 807, 777, 797, 792, 731, 760] payload_rows_per_dst=[9486, 9534, 9320, 9254, 9110, 9552, 9150, 8960] dup_rows_per_dst=[3166, 3186, 2982, 2856, 2952, 3208, 2800, 2788] combine_rows_per_src=[9486, 9534, 9320, 9254, 9110, 9552, 9150, 8960] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74290 payload_rows_remote=64988 dup_rows=24014 dup_rows_remote=20990 dedup_ratio=0.244283 router_tokens_per_expert=[742, 743, 777, 754, 793, 755, 780, 799, 743, 774, 744, 753, 746, 734, 755, 746, 788, 776, 712, 778, 737, 808, 801, 779, 756, 766, 753, 758, 777, 783, 763, 765, 778, 792, 776, 819, 785, 755, 784, 736, 776, 730, 774, 785, 755, 779, 755, 780, 763, 815, 816, 746, 743, 788, 756, 776, 758, 761, 755, 763, 791, 793, 788, 743] payload_rows_per_dst=[9302, 9170, 9242, 9250, 9404, 9410, 9218, 9294] dup_rows_per_dst=[3024, 2982, 3082, 2992, 3150, 2858, 3078, 2848] combine_rows_per_src=[9302, 9170, 9242, 9250, 9404, 9410, 9218, 9294] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=64900 dup_rows=23998 dup_rows_remote=20886 dedup_ratio=0.244120 router_tokens_per_expert=[781, 767, 730, 770, 817, 753, 760, 768, 795, 767, 776, 807, 767, 766, 746, 759, 750, 762, 759, 743, 722, 768, 755, 755, 787, 750, 724, 777, 744, 768, 754, 800, 822, 769, 778, 800, 731, 778, 792, 789, 776, 787, 790, 744, 711, 782, 779, 761, 763, 761, 762, 747, 800, 779, 771, 762, 764, 843, 808, 757, 766, 750, 740, 743] payload_rows_per_dst=[9258, 9328, 9150, 9266, 9406, 9302, 9276, 9320] dup_rows_per_dst=[3034, 3038, 2878, 2942, 3112, 2958, 3014, 3022] combine_rows_per_src=[9258, 9328, 9150, 9266, 9406, 9302, 9276, 9320] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=343760 combine_payload_bytes=265904128 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266247888 ingress_bytes=335110144 link_bytes=335110144 token_bytes=532152016 weight_movement_bytes=69206016 total_nvlink_bytes=601358032 weight_to_token_ratio=0.130049 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260992493e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=343856 combine_payload_bytes=266428416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266772272 ingress_bytes=301031424 link_bytes=301031424 token_bytes=533200688 weight_movement_bytes=34603008 total_nvlink_bytes=567803696 weight_to_token_ratio=0.064897 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132757013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266723328 dispatch_meta_bytes=343616 combine_payload_bytes=266723328 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=267066944 ingress_bytes=405135360 link_bytes=405135360 token_bytes=533790272 weight_movement_bytes=138412032 total_nvlink_bytes=672202304 weight_to_token_ratio=0.259300 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524491744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74428 payload_rows_remote=65088 dup_rows=23876 dup_rows_remote=20934 dedup_ratio=0.242879 router_tokens_per_expert=[773, 766, 764, 774, 765, 719, 808, 709, 742, 717, 764, 764, 794, 768, 759, 771, 736, 751, 794, 800, 754, 763, 803, 735, 767, 745, 784, 754, 804, 758, 776, 753, 795, 796, 772, 797, 775, 790, 781, 776, 796, 717, 754, 752, 731, 760, 755, 799, 769, 733, 864, 765, 773, 775, 774, 774, 772, 746, 776, 764, 800, 752, 772, 763] payload_rows_per_dst=[9246, 9282, 9294, 9340, 9394, 9256, 9362, 9254] dup_rows_per_dst=[2910, 2876, 2978, 2942, 3170, 2872, 3092, 3036] combine_rows_per_src=[9246, 9282, 9294, 9340, 9394, 9256, 9362, 9254] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265912320 dispatch_meta_bytes=342832 combine_payload_bytes=265912320 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266255152 ingress_bytes=369721344 link_bytes=369721344 token_bytes=532167472 weight_movement_bytes=103809024 total_nvlink_bytes=635976496 weight_to_token_ratio=0.195068 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391231652e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265748480 dispatch_meta_bytes=342608 combine_payload_bytes=265748480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266091088 ingress_bytes=300351488 link_bytes=300351488 token_bytes=531839568 weight_movement_bytes=34603008 total_nvlink_bytes=566442576 weight_to_token_ratio=0.065063 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130198469e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=344088 combine_payload_bytes=266600448 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266944536 ingress_bytes=370409472 link_bytes=370409472 token_bytes=533544984 weight_movement_bytes=103809024 total_nvlink_bytes=637354008 weight_to_token_ratio=0.194565 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393821023e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74262 payload_rows_remote=65050 dup_rows=24042 dup_rows_remote=21080 dedup_ratio=0.244568 router_tokens_per_expert=[804, 760, 759, 725, 736, 756, 769, 766, 794, 773, 782, 784, 743, 710, 796, 773, 761, 772, 775, 758, 743, 744, 782, 755, 789, 786, 733, 771, 776, 768, 869, 768, 764, 753, 787, 719, 753, 749, 789, 786, 759, 738, 739, 764, 799, 751, 771, 766, 762, 769, 763, 822, 783, 751, 796, 785, 784, 742, 802, 775, 770, 766, 733, 782] payload_rows_per_dst=[9136, 9350, 9194, 9418, 9266, 9212, 9374, 9312] dup_rows_per_dst=[3014, 2960, 2986, 3102, 2934, 2962, 3088, 2996] combine_rows_per_src=[9136, 9350, 9194, 9418, 9266, 9212, 9374, 9312] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74110 payload_rows_remote=64858 dup_rows=24194 dup_rows_remote=21128 dedup_ratio=0.246114 router_tokens_per_expert=[754, 771, 774, 738, 778, 800, 791, 735, 784, 764, 693, 792, 731, 778, 827, 790, 781, 773, 778, 765, 735, 722, 770, 781, 812, 777, 791, 745, 755, 785, 786, 760, 790, 795, 752, 752, 761, 820, 772, 792, 772, 794, 770, 770, 771, 727, 761, 753, 732, 728, 734, 756, 745, 759, 772, 739, 722, 786, 831, 771, 764, 813, 762, 770] payload_rows_per_dst=[9254, 9252, 9248, 9400, 9376, 9246, 8954, 9380] dup_rows_per_dst=[3028, 3066, 2962, 3022, 3092, 2990, 2976, 3058] combine_rows_per_src=[9254, 9252, 9248, 9400, 9376, 9246, 8954, 9380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74304 payload_rows_remote=65074 dup_rows=24000 dup_rows_remote=20968 dedup_ratio=0.244141 router_tokens_per_expert=[731, 758, 747, 719, 782, 799, 765, 764, 784, 753, 797, 779, 748, 760, 748, 741, 784, 792, 762, 792, 768, 799, 767, 740, 765, 785, 801, 795, 721, 763, 748, 811, 753, 795, 781, 765, 794, 815, 752, 755, 817, 788, 775, 745, 786, 769, 713, 716, 751, 768, 755, 773, 795, 748, 756, 785, 758, 756, 747, 817, 763, 770, 727, 796] payload_rows_per_dst=[9252, 9250, 9384, 9356, 9322, 9284, 9230, 9226] dup_rows_per_dst=[2878, 2970, 3024, 3022, 3098, 2934, 3032, 3042] combine_rows_per_src=[9252, 9250, 9384, 9356, 9322, 9284, 9230, 9226] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266190848 dispatch_meta_bytes=343912 combine_payload_bytes=266190848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266534760 ingress_bytes=300793856 link_bytes=300793856 token_bytes=532725608 weight_movement_bytes=34603008 total_nvlink_bytes=567328616 weight_to_token_ratio=0.064955 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131863064e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265830400 dispatch_meta_bytes=343144 combine_payload_bytes=265830400 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266173544 ingress_bytes=335036416 link_bytes=335036416 token_bytes=532003944 weight_movement_bytes=69206016 total_nvlink_bytes=601209960 weight_to_token_ratio=0.130086 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260715061e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266444800 dispatch_meta_bytes=344520 combine_payload_bytes=266444800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266789320 ingress_bytes=301047808 link_bytes=301047808 token_bytes=533234120 weight_movement_bytes=34603008 total_nvlink_bytes=567837128 weight_to_token_ratio=0.064893 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132818665e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74428 payload_rows_remote=65088 dup_rows=23876 dup_rows_remote=20934 dedup_ratio=0.242879 router_tokens_per_expert=[773, 766, 764, 774, 765, 719, 808, 709, 742, 717, 764, 764, 794, 768, 759, 771, 736, 751, 794, 800, 754, 763, 803, 735, 767, 745, 784, 754, 804, 758, 776, 753, 795, 796, 772, 797, 775, 790, 781, 776, 796, 717, 754, 752, 731, 760, 755, 799, 769, 733, 864, 765, 773, 775, 774, 774, 772, 746, 776, 764, 800, 752, 772, 763] payload_rows_per_dst=[9246, 9282, 9294, 9340, 9394, 9256, 9362, 9254] dup_rows_per_dst=[2910, 2876, 2978, 2942, 3170, 2872, 3092, 3036] combine_rows_per_src=[9246, 9282, 9294, 9340, 9394, 9256, 9362, 9254] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265658368 dispatch_meta_bytes=343944 combine_payload_bytes=265658368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266002312 ingress_bytes=300261376 link_bytes=300261376 token_bytes=531660680 weight_movement_bytes=34603008 total_nvlink_bytes=566263688 weight_to_token_ratio=0.065085 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129859385e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74404 payload_rows_remote=65148 dup_rows=23900 dup_rows_remote=20856 dedup_ratio=0.243123 router_tokens_per_expert=[823, 790, 784, 718, 817, 778, 753, 748, 772, 816, 772, 739, 738, 740, 757, 770, 747, 763, 783, 768, 773, 771, 786, 790, 754, 773, 779, 724, 763, 808, 783, 850, 756, 747, 754, 736, 709, 745, 742, 754, 768, 804, 764, 766, 811, 740, 793, 788, 746, 782, 802, 725, 757, 774, 763, 756, 814, 815, 728, 782, 770, 709, 747, 775] payload_rows_per_dst=[9388, 9170, 9336, 9398, 9072, 9422, 9362, 9256] dup_rows_per_dst=[3034, 3038, 3026, 3026, 2732, 3046, 2954, 3044] combine_rows_per_src=[9388, 9170, 9336, 9398, 9072, 9422, 9362, 9256] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=344168 combine_payload_bytes=266543104 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266887272 ingress_bytes=335749120 link_bytes=335749120 token_bytes=533430376 weight_movement_bytes=69206016 total_nvlink_bytes=602636392 weight_to_token_ratio=0.129738 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263396909e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74110 payload_rows_remote=64858 dup_rows=24194 dup_rows_remote=21128 dedup_ratio=0.246114 router_tokens_per_expert=[754, 771, 774, 738, 778, 800, 791, 735, 784, 764, 693, 792, 731, 778, 827, 790, 781, 773, 778, 765, 735, 722, 770, 781, 812, 777, 791, 745, 755, 785, 786, 760, 790, 795, 752, 752, 761, 820, 772, 792, 772, 794, 770, 770, 771, 727, 761, 753, 732, 728, 734, 756, 745, 759, 772, 739, 722, 786, 831, 771, 764, 813, 762, 770] payload_rows_per_dst=[9254, 9252, 9248, 9400, 9376, 9246, 8954, 9380] dup_rows_per_dst=[3028, 3066, 2962, 3022, 3092, 2990, 2976, 3058] combine_rows_per_src=[9254, 9252, 9248, 9400, 9376, 9246, 8954, 9380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=64954 dup_rows=23974 dup_rows_remote=20990 dedup_ratio=0.243876 router_tokens_per_expert=[767, 813, 762, 761, 771, 750, 770, 772, 781, 770, 766, 708, 795, 765, 740, 770, 817, 757, 780, 796, 756, 762, 776, 736, 792, 787, 750, 779, 762, 786, 776, 747, 766, 776, 759, 749, 768, 791, 778, 791, 758, 805, 767, 774, 753, 737, 763, 760, 775, 789, 779, 745, 764, 738, 770, 732, 797, 770, 822, 752, 742, 734, 799, 729] payload_rows_per_dst=[9288, 9334, 9376, 9232, 9292, 9304, 9218, 9286] dup_rows_per_dst=[3044, 2856, 2984, 3126, 3064, 2930, 2966, 3004] combine_rows_per_src=[9288, 9334, 9376, 9232, 9292, 9304, 9218, 9286] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=344088 combine_payload_bytes=266600448 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266944536 ingress_bytes=370409472 link_bytes=370409472 token_bytes=533544984 weight_movement_bytes=103809024 total_nvlink_bytes=637354008 weight_to_token_ratio=0.194565 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393821023e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266846208 dispatch_meta_bytes=344016 combine_payload_bytes=266846208 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267190224 ingress_bytes=336052224 link_bytes=336052224 token_bytes=534036432 weight_movement_bytes=69206016 total_nvlink_bytes=603242448 weight_to_token_ratio=0.129590 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264537464e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=64850 dup_rows=23998 dup_rows_remote=20842 dedup_ratio=0.244120 router_tokens_per_expert=[766, 781, 813, 758, 792, 759, 744, 733, 770, 748, 738, 770, 818, 734, 740, 783, 733, 759, 873, 746, 788, 776, 751, 746, 739, 772, 765, 734, 755, 756, 748, 827, 783, 809, 799, 819, 780, 776, 790, 750, 731, 788, 765, 728, 766, 768, 775, 745, 730, 747, 719, 808, 766, 793, 724, 766, 790, 772, 780, 804, 760, 768, 744, 794] payload_rows_per_dst=[9390, 9212, 9366, 9234, 9456, 9154, 9182, 9312] dup_rows_per_dst=[2902, 2990, 2978, 2958, 3156, 2978, 2924, 3112] combine_rows_per_src=[9390, 9212, 9366, 9234, 9456, 9154, 9182, 9312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266444800 dispatch_meta_bytes=344520 combine_payload_bytes=266444800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266789320 ingress_bytes=301047808 link_bytes=301047808 token_bytes=533234120 weight_movement_bytes=34603008 total_nvlink_bytes=567837128 weight_to_token_ratio=0.064893 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132818665e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=64592 dup_rows=24028 dup_rows_remote=20678 dedup_ratio=0.244425 router_tokens_per_expert=[766, 787, 737, 765, 737, 792, 800, 782, 722, 768, 741, 732, 775, 757, 790, 778, 748, 764, 799, 770, 778, 765, 750, 714, 806, 742, 774, 798, 784, 769, 787, 767, 788, 793, 775, 779, 780, 786, 756, 800, 795, 850, 725, 773, 756, 757, 790, 705, 735, 776, 777, 796, 760, 776, 741, 759, 790, 762, 755, 729, 806, 742, 722, 774] payload_rows_per_dst=[9684, 9184, 9238, 8968, 9420, 9300, 9324, 9158] dup_rows_per_dst=[3350, 2972, 2908, 2784, 3094, 3002, 3064, 2854] combine_rows_per_src=[9684, 9184, 9238, 8968, 9420, 9300, 9324, 9158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=344168 combine_payload_bytes=266543104 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266887272 ingress_bytes=335749120 link_bytes=335749120 token_bytes=533430376 weight_movement_bytes=69206016 total_nvlink_bytes=602636392 weight_to_token_ratio=0.129738 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263396909e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74366 payload_rows_remote=64880 dup_rows=23938 dup_rows_remote=20772 dedup_ratio=0.243510 router_tokens_per_expert=[774, 819, 762, 763, 787, 795, 810, 741, 798, 733, 741, 751, 771, 757, 750, 734, 747, 745, 747, 719, 786, 733, 790, 785, 771, 759, 761, 755, 792, 763, 725, 804, 761, 730, 763, 770, 768, 774, 788, 776, 745, 785, 752, 767, 740, 807, 771, 746, 800, 756, 810, 798, 791, 743, 758, 786, 760, 775, 807, 777, 797, 792, 731, 760] payload_rows_per_dst=[9486, 9534, 9320, 9254, 9110, 9552, 9150, 8960] dup_rows_per_dst=[3166, 3186, 2982, 2856, 2952, 3208, 2800, 2788] combine_rows_per_src=[9486, 9534, 9320, 9254, 9110, 9552, 9150, 8960] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74410 payload_rows_remote=64920 dup_rows=23894 dup_rows_remote=20788 dedup_ratio=0.243062 router_tokens_per_expert=[816, 756, 748, 781, 780, 768, 731, 716, 774, 735, 771, 774, 797, 752, 760, 729, 746, 757, 744, 806, 747, 805, 800, 742, 724, 807, 763, 764, 740, 730, 810, 789, 786, 794, 785, 782, 783, 818, 750, 800, 750, 771, 800, 738, 758, 741, 742, 785, 770, 763, 757, 769, 764, 819, 750, 782, 775, 738, 766, 756, 803, 753, 786, 756] payload_rows_per_dst=[9228, 9196, 9364, 9256, 9490, 9204, 9328, 9344] dup_rows_per_dst=[2964, 2988, 2930, 2998, 3106, 2966, 3020, 2922] combine_rows_per_src=[9228, 9196, 9364, 9256, 9490, 9204, 9328, 9344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266846208 dispatch_meta_bytes=344016 combine_payload_bytes=266846208 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267190224 ingress_bytes=336052224 link_bytes=336052224 token_bytes=534036432 weight_movement_bytes=69206016 total_nvlink_bytes=603242448 weight_to_token_ratio=0.129590 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264537464e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265658368 dispatch_meta_bytes=343944 combine_payload_bytes=265658368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266002312 ingress_bytes=300261376 link_bytes=300261376 token_bytes=531660680 weight_movement_bytes=34603008 total_nvlink_bytes=566263688 weight_to_token_ratio=0.065085 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129859385e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74412 payload_rows_remote=65062 dup_rows=23892 dup_rows_remote=20886 dedup_ratio=0.243042 router_tokens_per_expert=[767, 741, 744, 776, 755, 793, 751, 771, 785, 755, 728, 758, 764, 782, 735, 782, 725, 746, 792, 763, 782, 767, 826, 779, 760, 801, 731, 718, 819, 759, 781, 809, 760, 765, 720, 749, 774, 788, 759, 758, 748, 798, 775, 819, 755, 791, 771, 773, 774, 737, 799, 787, 715, 756, 773, 722, 804, 750, 822, 765, 787, 788, 721, 804] payload_rows_per_dst=[9264, 9250, 9362, 9350, 9174, 9400, 9194, 9418] dup_rows_per_dst=[2932, 2928, 2998, 3006, 2972, 3060, 2932, 3064] combine_rows_per_src=[9264, 9250, 9362, 9350, 9174, 9400, 9194, 9418] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266051584 dispatch_meta_bytes=343776 combine_payload_bytes=266051584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266395360 ingress_bytes=300654592 link_bytes=300654592 token_bytes=532446944 weight_movement_bytes=34603008 total_nvlink_bytes=567049952 weight_to_token_ratio=0.064989 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131339025e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265625600 dispatch_meta_bytes=342768 combine_payload_bytes=265625600 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265968368 ingress_bytes=334831616 link_bytes=334831616 token_bytes=531593968 weight_movement_bytes=69206016 total_nvlink_bytes=600799984 weight_to_token_ratio=0.130186 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259944415e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=264568832 dispatch_meta_bytes=341080 combine_payload_bytes=264568832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264909912 ingress_bytes=299171840 link_bytes=299171840 token_bytes=529478744 weight_movement_bytes=34603008 total_nvlink_bytes=564081752 weight_to_token_ratio=0.065353 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125759549e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266493952 dispatch_meta_bytes=343792 combine_payload_bytes=266493952 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266837744 ingress_bytes=335699968 link_bytes=335699968 token_bytes=533331696 weight_movement_bytes=69206016 total_nvlink_bytes=602537712 weight_to_token_ratio=0.129762 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263211954e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74196 payload_rows_remote=64928 dup_rows=24108 dup_rows_remote=21084 dedup_ratio=0.245239 router_tokens_per_expert=[764, 769, 786, 761, 766, 777, 764, 744, 768, 771, 747, 829, 787, 788, 762, 779, 776, 758, 791, 753, 754, 756, 752, 781, 765, 762, 797, 778, 729, 778, 762, 761, 755, 734, 778, 784, 772, 762, 721, 766, 794, 805, 758, 747, 777, 718, 787, 760, 783, 766, 748, 778, 789, 797, 784, 819, 766, 760, 738, 747, 750, 765, 767, 762] payload_rows_per_dst=[9248, 9402, 9252, 9340, 9112, 9268, 9406, 9168] dup_rows_per_dst=[3014, 3060, 2990, 2924, 3032, 3024, 3122, 2942] combine_rows_per_src=[9248, 9402, 9252, 9340, 9112, 9268, 9406, 9168] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74406 payload_rows_remote=64912 dup_rows=23898 dup_rows_remote=20732 dedup_ratio=0.243103 router_tokens_per_expert=[728, 793, 767, 740, 779, 763, 715, 774, 769, 757, 752, 768, 741, 791, 769, 761, 805, 762, 784, 813, 746, 759, 777, 716, 776, 738, 752, 780, 788, 718, 731, 684, 793, 751, 748, 803, 766, 736, 774, 756, 790, 748, 788, 764, 756, 744, 740, 737, 820, 808, 774, 794, 737, 760, 840, 797, 810, 822, 788, 799, 764, 815, 779, 755] payload_rows_per_dst=[9270, 9292, 9282, 9090, 9298, 9186, 9494, 9494] dup_rows_per_dst=[2848, 2924, 3042, 2844, 2956, 2948, 3166, 3170] combine_rows_per_src=[9270, 9292, 9282, 9090, 9298, 9186, 9494, 9494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=65030 dup_rows=23966 dup_rows_remote=20904 dedup_ratio=0.243795 router_tokens_per_expert=[736, 726, 734, 772, 719, 769, 766, 774, 759, 748, 742, 788, 749, 786, 833, 800, 767, 756, 775, 766, 787, 793, 757, 725, 750, 765, 762, 791, 788, 776, 776, 807, 738, 723, 745, 813, 744, 797, 738, 789, 726, 779, 762, 764, 733, 789, 769, 750, 783, 784, 812, 799, 790, 781, 771, 751, 802, 773, 742, 764, 758, 790, 825, 726] payload_rows_per_dst=[9080, 9374, 9340, 9376, 9184, 9336, 9340, 9308] dup_rows_per_dst=[2912, 3036, 2902, 3042, 2782, 3028, 3202, 3062] combine_rows_per_src=[9080, 9374, 9340, 9376, 9184, 9336, 9340, 9308] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74222 payload_rows_remote=64914 dup_rows=24082 dup_rows_remote=21024 dedup_ratio=0.244975 router_tokens_per_expert=[755, 775, 709, 756, 779, 735, 772, 770, 779, 766, 776, 803, 729, 761, 788, 781, 710, 765, 806, 765, 828, 724, 770, 775, 793, 798, 745, 781, 780, 794, 764, 751, 786, 787, 746, 749, 785, 817, 759, 793, 784, 763, 770, 765, 754, 782, 704, 737, 786, 720, 792, 761, 734, 758, 736, 792, 792, 788, 784, 783, 774, 711, 790, 787] payload_rows_per_dst=[9158, 9308, 9278, 9308, 9334, 9258, 9238, 9340] dup_rows_per_dst=[2944, 3058, 3008, 3104, 3110, 2860, 2920, 3078] combine_rows_per_src=[9158, 9308, 9278, 9308, 9334, 9258, 9238, 9340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265748480 dispatch_meta_bytes=342608 combine_payload_bytes=265748480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266091088 ingress_bytes=300351488 link_bytes=300351488 token_bytes=531839568 weight_movement_bytes=34603008 total_nvlink_bytes=566442576 weight_to_token_ratio=0.065063 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130198469e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74406 payload_rows_remote=64912 dup_rows=23898 dup_rows_remote=20732 dedup_ratio=0.243103 router_tokens_per_expert=[728, 793, 767, 740, 779, 763, 715, 774, 769, 757, 752, 768, 741, 791, 769, 761, 805, 762, 784, 813, 746, 759, 777, 716, 776, 738, 752, 780, 788, 718, 731, 684, 793, 751, 748, 803, 766, 736, 774, 756, 790, 748, 788, 764, 756, 744, 740, 737, 820, 808, 774, 794, 737, 760, 840, 797, 810, 822, 788, 799, 764, 815, 779, 755] payload_rows_per_dst=[9270, 9292, 9282, 9090, 9298, 9186, 9494, 9494] dup_rows_per_dst=[2848, 2924, 3042, 2844, 2956, 2948, 3166, 3170] combine_rows_per_src=[9270, 9292, 9282, 9090, 9298, 9186, 9494, 9494] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265912320 dispatch_meta_bytes=342832 combine_payload_bytes=265912320 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266255152 ingress_bytes=369721344 link_bytes=369721344 token_bytes=532167472 weight_movement_bytes=103809024 total_nvlink_bytes=635976496 weight_to_token_ratio=0.195068 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391231652e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65042 dup_rows=23944 dup_rows_remote=21008 dedup_ratio=0.243571 router_tokens_per_expert=[781, 718, 777, 793, 745, 767, 779, 806, 776, 797, 714, 757, 735, 735, 785, 779, 741, 772, 768, 729, 738, 826, 779, 774, 782, 765, 796, 776, 750, 791, 739, 804, 776, 741, 753, 791, 766, 798, 724, 764, 774, 778, 763, 800, 727, 790, 761, 751, 789, 781, 798, 729, 742, 772, 725, 818, 777, 802, 816, 757, 756, 752, 748, 759] payload_rows_per_dst=[9298, 9226, 9318, 9284, 9224, 9318, 9334, 9358] dup_rows_per_dst=[3034, 2930, 2936, 3122, 3002, 2970, 2974, 2976] combine_rows_per_src=[9298, 9226, 9318, 9284, 9224, 9318, 9334, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=65128 dup_rows=23930 dup_rows_remote=21094 dedup_ratio=0.243429 router_tokens_per_expert=[792, 740, 848, 760, 779, 846, 772, 786, 779, 730, 792, 785, 803, 765, 738, 788, 751, 796, 727, 798, 755, 760, 730, 781, 739, 791, 783, 737, 778, 772, 779, 814, 751, 754, 755, 766, 738, 762, 767, 748, 710, 738, 791, 740, 757, 759, 733, 809, 742, 770, 801, 730, 763, 813, 768, 745, 779, 764, 749, 807, 745, 778, 742, 784] payload_rows_per_dst=[9386, 9302, 9268, 9452, 9246, 9134, 9222, 9364] dup_rows_per_dst=[3260, 3058, 2928, 2934, 2836, 2940, 3042, 2932] combine_rows_per_src=[9386, 9302, 9268, 9452, 9246, 9134, 9222, 9364] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265945088 dispatch_meta_bytes=344048 combine_payload_bytes=265945088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266289136 ingress_bytes=300548096 link_bytes=300548096 token_bytes=532234224 weight_movement_bytes=34603008 total_nvlink_bytes=566837232 weight_to_token_ratio=0.065015 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130938289e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=65030 dup_rows=23966 dup_rows_remote=20904 dedup_ratio=0.243795 router_tokens_per_expert=[736, 726, 734, 772, 719, 769, 766, 774, 759, 748, 742, 788, 749, 786, 833, 800, 767, 756, 775, 766, 787, 793, 757, 725, 750, 765, 762, 791, 788, 776, 776, 807, 738, 723, 745, 813, 744, 797, 738, 789, 726, 779, 762, 764, 733, 789, 769, 750, 783, 784, 812, 799, 790, 781, 771, 751, 802, 773, 742, 764, 758, 790, 825, 726] payload_rows_per_dst=[9080, 9374, 9340, 9376, 9184, 9336, 9340, 9308] dup_rows_per_dst=[2912, 3036, 2902, 3042, 2782, 3028, 3202, 3062] combine_rows_per_src=[9080, 9374, 9340, 9376, 9184, 9336, 9340, 9308] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74412 payload_rows_remote=65062 dup_rows=23892 dup_rows_remote=20886 dedup_ratio=0.243042 router_tokens_per_expert=[767, 741, 744, 776, 755, 793, 751, 771, 785, 755, 728, 758, 764, 782, 735, 782, 725, 746, 792, 763, 782, 767, 826, 779, 760, 801, 731, 718, 819, 759, 781, 809, 760, 765, 720, 749, 774, 788, 759, 758, 748, 798, 775, 819, 755, 791, 771, 773, 774, 737, 799, 787, 715, 756, 773, 722, 804, 750, 822, 765, 787, 788, 721, 804] payload_rows_per_dst=[9264, 9250, 9362, 9350, 9174, 9400, 9194, 9418] dup_rows_per_dst=[2932, 2928, 2998, 3006, 2972, 3060, 2932, 3064] combine_rows_per_src=[9264, 9250, 9362, 9350, 9174, 9400, 9194, 9418] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265887744 dispatch_meta_bytes=343752 combine_payload_bytes=265887744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266231496 ingress_bytes=300490752 link_bytes=300490752 token_bytes=532119240 weight_movement_bytes=34603008 total_nvlink_bytes=566722248 weight_to_token_ratio=0.065029 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130722508e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74424 payload_rows_remote=65212 dup_rows=23880 dup_rows_remote=20966 dedup_ratio=0.242920 router_tokens_per_expert=[743, 774, 753, 750, 750, 737, 736, 725, 734, 783, 780, 781, 716, 778, 767, 755, 788, 763, 776, 804, 843, 782, 788, 774, 810, 777, 733, 786, 747, 760, 768, 764, 768, 770, 753, 748, 800, 725, 809, 755, 782, 791, 747, 756, 792, 784, 798, 773, 805, 793, 725, 776, 766, 770, 745, 773, 729, 768, 732, 798, 780, 780, 746, 790] payload_rows_per_dst=[9212, 9554, 9234, 9250, 9292, 9344, 9272, 9266] dup_rows_per_dst=[2914, 3186, 2890, 2896, 3014, 3102, 3034, 2844] combine_rows_per_src=[9212, 9554, 9234, 9250, 9292, 9344, 9272, 9266] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265879552 dispatch_meta_bytes=342576 combine_payload_bytes=265879552 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266222128 ingress_bytes=335085568 link_bytes=335085568 token_bytes=532101680 weight_movement_bytes=69206016 total_nvlink_bytes=601307696 weight_to_token_ratio=0.130062 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260900016e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65042 dup_rows=23944 dup_rows_remote=21008 dedup_ratio=0.243571 router_tokens_per_expert=[781, 718, 777, 793, 745, 767, 779, 806, 776, 797, 714, 757, 735, 735, 785, 779, 741, 772, 768, 729, 738, 826, 779, 774, 782, 765, 796, 776, 750, 791, 739, 804, 776, 741, 753, 791, 766, 798, 724, 764, 774, 778, 763, 800, 727, 790, 761, 751, 789, 781, 798, 729, 742, 772, 725, 818, 777, 802, 816, 757, 756, 752, 748, 759] payload_rows_per_dst=[9298, 9226, 9318, 9284, 9224, 9318, 9334, 9358] dup_rows_per_dst=[3034, 2930, 2936, 3122, 3002, 2970, 2974, 2976] combine_rows_per_src=[9298, 9226, 9318, 9284, 9224, 9318, 9334, 9358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=343736 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266706616 ingress_bytes=335568896 link_bytes=335568896 token_bytes=533069496 weight_movement_bytes=69206016 total_nvlink_bytes=602275512 weight_to_token_ratio=0.129826 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262718740e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266493952 dispatch_meta_bytes=343792 combine_payload_bytes=266493952 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266837744 ingress_bytes=335699968 link_bytes=335699968 token_bytes=533331696 weight_movement_bytes=69206016 total_nvlink_bytes=602537712 weight_to_token_ratio=0.129762 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263211954e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=64592 dup_rows=24028 dup_rows_remote=20678 dedup_ratio=0.244425 router_tokens_per_expert=[766, 787, 737, 765, 737, 792, 800, 782, 722, 768, 741, 732, 775, 757, 790, 778, 748, 764, 799, 770, 778, 765, 750, 714, 806, 742, 774, 798, 784, 769, 787, 767, 788, 793, 775, 779, 780, 786, 756, 800, 795, 850, 725, 773, 756, 757, 790, 705, 735, 776, 777, 796, 760, 776, 741, 759, 790, 762, 755, 729, 806, 742, 722, 774] payload_rows_per_dst=[9684, 9184, 9238, 8968, 9420, 9300, 9324, 9158] dup_rows_per_dst=[3350, 2972, 2908, 2784, 3094, 3002, 3064, 2854] combine_rows_per_src=[9684, 9184, 9238, 8968, 9420, 9300, 9324, 9158] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=64850 dup_rows=23998 dup_rows_remote=20842 dedup_ratio=0.244120 router_tokens_per_expert=[766, 781, 813, 758, 792, 759, 744, 733, 770, 748, 738, 770, 818, 734, 740, 783, 733, 759, 873, 746, 788, 776, 751, 746, 739, 772, 765, 734, 755, 756, 748, 827, 783, 809, 799, 819, 780, 776, 790, 750, 731, 788, 765, 728, 766, 768, 775, 745, 730, 747, 719, 808, 766, 793, 724, 766, 790, 772, 780, 804, 760, 768, 744, 794] payload_rows_per_dst=[9390, 9212, 9366, 9234, 9456, 9154, 9182, 9312] dup_rows_per_dst=[2902, 2990, 2978, 2958, 3156, 2978, 2924, 3112] combine_rows_per_src=[9390, 9212, 9366, 9234, 9456, 9154, 9182, 9312] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74048 payload_rows_remote=64850 dup_rows=24256 dup_rows_remote=21260 dedup_ratio=0.246745 router_tokens_per_expert=[726, 780, 789, 745, 752, 810, 763, 792, 756, 750, 741, 771, 744, 726, 795, 744, 778, 780, 759, 753, 753, 771, 849, 785, 777, 713, 712, 797, 809, 793, 778, 787, 728, 767, 766, 746, 798, 761, 781, 768, 717, 728, 732, 766, 812, 813, 730, 799, 739, 770, 752, 794, 769, 799, 777, 779, 724, 773, 770, 780, 801, 780, 785, 770] payload_rows_per_dst=[9302, 9108, 9384, 9286, 9192, 9198, 9318, 9260] dup_rows_per_dst=[3012, 2946, 3072, 3046, 3038, 2996, 3040, 3106] combine_rows_per_src=[9302, 9108, 9384, 9286, 9192, 9198, 9318, 9260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=343736 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266706616 ingress_bytes=335568896 link_bytes=335568896 token_bytes=533069496 weight_movement_bytes=69206016 total_nvlink_bytes=602275512 weight_to_token_ratio=0.129826 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262718740e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=64818 dup_rows=24132 dup_rows_remote=21156 dedup_ratio=0.245483 router_tokens_per_expert=[712, 785, 757, 726, 798, 765, 794, 782, 743, 808, 810, 732, 799, 753, 746, 723, 766, 757, 796, 785, 784, 788, 773, 812, 745, 794, 786, 749, 781, 809, 747, 754, 741, 741, 760, 755, 761, 779, 737, 762, 809, 779, 775, 747, 765, 786, 774, 729, 763, 730, 778, 776, 764, 750, 758, 787, 741, 761, 769, 814, 781, 753, 808, 760] payload_rows_per_dst=[9228, 9160, 9406, 9354, 9184, 9250, 9220, 9370] dup_rows_per_dst=[3010, 3068, 3116, 2976, 2888, 3078, 2992, 3004] combine_rows_per_src=[9228, 9160, 9406, 9354, 9184, 9250, 9220, 9370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265887744 dispatch_meta_bytes=343752 combine_payload_bytes=265887744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266231496 ingress_bytes=300490752 link_bytes=300490752 token_bytes=532119240 weight_movement_bytes=34603008 total_nvlink_bytes=566722248 weight_to_token_ratio=0.065029 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130722508e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=344200 combine_payload_bytes=266412032 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266756232 ingress_bytes=335618048 link_bytes=335618048 token_bytes=533168264 weight_movement_bytes=69206016 total_nvlink_bytes=602374280 weight_to_token_ratio=0.129801 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262903695e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266764288 dispatch_meta_bytes=344888 combine_payload_bytes=266764288 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267109176 ingress_bytes=301367296 link_bytes=301367296 token_bytes=533873464 weight_movement_bytes=34603008 total_nvlink_bytes=568476472 weight_to_token_ratio=0.064815 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134020873e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=267108352 dispatch_meta_bytes=344712 combine_payload_bytes=267108352 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267453064 ingress_bytes=336314368 link_bytes=336314368 token_bytes=534561416 weight_movement_bytes=69206016 total_nvlink_bytes=603767432 weight_to_token_ratio=0.129463 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265523891e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265494528 dispatch_meta_bytes=343896 combine_payload_bytes=265494528 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265838424 ingress_bytes=334700544 link_bytes=334700544 token_bytes=531332952 weight_movement_bytes=69206016 total_nvlink_bytes=600538968 weight_to_token_ratio=0.130250 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259451201e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74048 payload_rows_remote=64850 dup_rows=24256 dup_rows_remote=21260 dedup_ratio=0.246745 router_tokens_per_expert=[726, 780, 789, 745, 752, 810, 763, 792, 756, 750, 741, 771, 744, 726, 795, 744, 778, 780, 759, 753, 753, 771, 849, 785, 777, 713, 712, 797, 809, 793, 778, 787, 728, 767, 766, 746, 798, 761, 781, 768, 717, 728, 732, 766, 812, 813, 730, 799, 739, 770, 752, 794, 769, 799, 777, 779, 724, 773, 770, 780, 801, 780, 785, 770] payload_rows_per_dst=[9302, 9108, 9384, 9286, 9192, 9198, 9318, 9260] dup_rows_per_dst=[3012, 2946, 3072, 3046, 3038, 2996, 3040, 3106] combine_rows_per_src=[9302, 9108, 9384, 9286, 9192, 9198, 9318, 9260] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=65122 dup_rows=23950 dup_rows_remote=21034 dedup_ratio=0.243632 router_tokens_per_expert=[728, 715, 775, 786, 804, 762, 759, 718, 800, 768, 779, 792, 751, 749, 778, 770, 789, 798, 735, 792, 762, 768, 788, 783, 764, 824, 813, 804, 747, 757, 770, 773, 729, 804, 747, 770, 755, 776, 779, 766, 767, 779, 782, 751, 783, 754, 776, 782, 790, 742, 752, 766, 757, 764, 752, 751, 710, 789, 710, 746, 803, 764, 752, 803] payload_rows_per_dst=[9240, 9316, 9364, 9422, 9258, 9332, 9232, 9190] dup_rows_per_dst=[2854, 3058, 3066, 3082, 2994, 3016, 2916, 2964] combine_rows_per_src=[9240, 9316, 9364, 9422, 9258, 9332, 9232, 9190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=65120 dup_rows=23946 dup_rows_remote=20992 dedup_ratio=0.243591 router_tokens_per_expert=[786, 711, 763, 773, 816, 749, 758, 728, 754, 741, 768, 716, 738, 760, 750, 799, 759, 822, 693, 746, 782, 806, 831, 760, 782, 788, 772, 809, 785, 777, 789, 802, 787, 797, 778, 776, 738, 786, 786, 770, 805, 711, 758, 793, 743, 758, 716, 794, 753, 776, 800, 760, 769, 732, 770, 787, 751, 718, 755, 760, 729, 788, 780, 815] payload_rows_per_dst=[9010, 9150, 9090, 9444, 9416, 9516, 9494, 9238] dup_rows_per_dst=[2794, 2902, 2804, 3164, 3020, 3144, 3164, 2954] combine_rows_per_src=[9010, 9150, 9090, 9444, 9416, 9516, 9494, 9238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74098 payload_rows_remote=64744 dup_rows=24206 dup_rows_remote=21150 dedup_ratio=0.246236 router_tokens_per_expert=[743, 769, 735, 767, 788, 747, 779, 792, 782, 780, 767, 797, 796, 738, 745, 800, 746, 783, 779, 783, 786, 780, 783, 813, 765, 772, 780, 762, 756, 738, 777, 749, 787, 767, 743, 769, 769, 814, 768, 804, 746, 730, 737, 761, 790, 782, 798, 788, 753, 702, 768, 807, 751, 758, 737, 741, 758, 768, 743, 755, 823, 723, 757, 778] payload_rows_per_dst=[9256, 9354, 9294, 9296, 9366, 9242, 9080, 9210] dup_rows_per_dst=[2984, 3056, 3212, 2902, 3076, 3022, 2954, 3000] combine_rows_per_src=[9256, 9354, 9294, 9296, 9366, 9242, 9080, 9210] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=264568832 dispatch_meta_bytes=341080 combine_payload_bytes=264568832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264909912 ingress_bytes=299171840 link_bytes=299171840 token_bytes=529478744 weight_movement_bytes=34603008 total_nvlink_bytes=564081752 weight_to_token_ratio=0.065353 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125759549e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265625600 dispatch_meta_bytes=342768 combine_payload_bytes=265625600 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265968368 ingress_bytes=334831616 link_bytes=334831616 token_bytes=531593968 weight_movement_bytes=69206016 total_nvlink_bytes=600799984 weight_to_token_ratio=0.130186 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259944415e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265625600 dispatch_meta_bytes=344440 combine_payload_bytes=265625600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265970040 ingress_bytes=300228608 link_bytes=300228608 token_bytes=531595640 weight_movement_bytes=34603008 total_nvlink_bytes=566198648 weight_to_token_ratio=0.065093 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129736082e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74030 payload_rows_remote=64844 dup_rows=24274 dup_rows_remote=21350 dedup_ratio=0.246928 router_tokens_per_expert=[793, 782, 716, 806, 755, 775, 786, 793, 759, 769, 732, 767, 801, 756, 803, 723, 703, 747, 790, 713, 782, 760, 781, 779, 718, 738, 749, 767, 751, 760, 816, 764, 775, 770, 789, 814, 776, 802, 783, 761, 792, 750, 774, 786, 770, 739, 721, 821, 719, 775, 732, 792, 758, 778, 805, 738, 773, 738, 759, 815, 801, 704, 809, 799] payload_rows_per_dst=[9266, 9216, 9186, 9162, 9386, 9240, 9276, 9298] dup_rows_per_dst=[3146, 3004, 2924, 2964, 3154, 3066, 2918, 3098] combine_rows_per_src=[9266, 9216, 9186, 9162, 9386, 9240, 9276, 9298] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65182 dup_rows=23944 dup_rows_remote=21006 dedup_ratio=0.243571 router_tokens_per_expert=[767, 748, 801, 752, 759, 782, 751, 731, 763, 730, 750, 728, 736, 776, 775, 777, 765, 788, 735, 764, 757, 761, 776, 756, 753, 758, 787, 748, 801, 769, 787, 760, 768, 772, 733, 767, 728, 791, 743, 756, 802, 771, 852, 783, 779, 791, 785, 789, 692, 756, 789, 805, 769, 786, 727, 816, 797, 767, 765, 767, 838, 780, 766, 731] payload_rows_per_dst=[9326, 9232, 9280, 9218, 9178, 9500, 9254, 9372] dup_rows_per_dst=[2856, 2838, 2924, 3108, 2938, 3204, 3026, 3050] combine_rows_per_src=[9326, 9232, 9280, 9218, 9178, 9500, 9254, 9372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265625600 dispatch_meta_bytes=344440 combine_payload_bytes=265625600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265970040 ingress_bytes=300228608 link_bytes=300228608 token_bytes=531595640 weight_movement_bytes=34603008 total_nvlink_bytes=566198648 weight_to_token_ratio=0.065093 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129736082e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=65120 dup_rows=23946 dup_rows_remote=20992 dedup_ratio=0.243591 router_tokens_per_expert=[786, 711, 763, 773, 816, 749, 758, 728, 754, 741, 768, 716, 738, 760, 750, 799, 759, 822, 693, 746, 782, 806, 831, 760, 782, 788, 772, 809, 785, 777, 789, 802, 787, 797, 778, 776, 738, 786, 786, 770, 805, 711, 758, 793, 743, 758, 716, 794, 753, 776, 800, 760, 769, 732, 770, 787, 751, 718, 755, 760, 729, 788, 780, 815] payload_rows_per_dst=[9010, 9150, 9090, 9444, 9416, 9516, 9494, 9238] dup_rows_per_dst=[2794, 2902, 2804, 3164, 3020, 3144, 3164, 2954] combine_rows_per_src=[9010, 9150, 9090, 9444, 9416, 9516, 9494, 9238] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=64818 dup_rows=24132 dup_rows_remote=21156 dedup_ratio=0.245483 router_tokens_per_expert=[712, 785, 757, 726, 798, 765, 794, 782, 743, 808, 810, 732, 799, 753, 746, 723, 766, 757, 796, 785, 784, 788, 773, 812, 745, 794, 786, 749, 781, 809, 747, 754, 741, 741, 760, 755, 761, 779, 737, 762, 809, 779, 775, 747, 765, 786, 774, 729, 763, 730, 778, 776, 764, 750, 758, 787, 741, 761, 769, 814, 781, 753, 808, 760] payload_rows_per_dst=[9228, 9160, 9406, 9354, 9184, 9250, 9220, 9370] dup_rows_per_dst=[3010, 3068, 3116, 2976, 2888, 3078, 2992, 3004] combine_rows_per_src=[9228, 9160, 9406, 9354, 9184, 9250, 9220, 9370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265191424 dispatch_meta_bytes=343576 combine_payload_bytes=265191424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265535000 ingress_bytes=299794432 link_bytes=299794432 token_bytes=530726424 weight_movement_bytes=34603008 total_nvlink_bytes=565329432 weight_to_token_ratio=0.065199 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128102312e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74140 payload_rows_remote=64862 dup_rows=24164 dup_rows_remote=21050 dedup_ratio=0.245809 router_tokens_per_expert=[715, 766, 766, 744, 773, 779, 771, 759, 797, 770, 754, 762, 812, 750, 778, 766, 762, 788, 724, 737, 736, 770, 778, 834, 766, 738, 716, 788, 800, 795, 775, 787, 797, 764, 769, 789, 796, 771, 734, 791, 800, 760, 797, 763, 776, 735, 812, 742, 759, 781, 796, 782, 746, 767, 779, 744, 773, 750, 754, 750, 766, 748, 755, 750] payload_rows_per_dst=[9278, 9660, 9284, 9152, 9380, 8956, 9274, 9156] dup_rows_per_dst=[3114, 3306, 3014, 2892, 3042, 2840, 3184, 2772] combine_rows_per_src=[9278, 9660, 9284, 9152, 9380, 8956, 9274, 9156] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266739712 dispatch_meta_bytes=344624 combine_payload_bytes=266739712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267084336 ingress_bytes=301342720 link_bytes=301342720 token_bytes=533824048 weight_movement_bytes=34603008 total_nvlink_bytes=568427056 weight_to_token_ratio=0.064821 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133928395e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74030 payload_rows_remote=64844 dup_rows=24274 dup_rows_remote=21350 dedup_ratio=0.246928 router_tokens_per_expert=[793, 782, 716, 806, 755, 775, 786, 793, 759, 769, 732, 767, 801, 756, 803, 723, 703, 747, 790, 713, 782, 760, 781, 779, 718, 738, 749, 767, 751, 760, 816, 764, 775, 770, 789, 814, 776, 802, 783, 761, 792, 750, 774, 786, 770, 739, 721, 821, 719, 775, 732, 792, 758, 778, 805, 738, 773, 738, 759, 815, 801, 704, 809, 799] payload_rows_per_dst=[9266, 9216, 9186, 9162, 9386, 9240, 9276, 9298] dup_rows_per_dst=[3146, 3004, 2924, 2964, 3154, 3066, 2918, 3098] combine_rows_per_src=[9266, 9216, 9186, 9162, 9386, 9240, 9276, 9298] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=344448 combine_payload_bytes=266731520 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267075968 ingress_bytes=370540544 link_bytes=370540544 token_bytes=533807488 weight_movement_bytes=103809024 total_nvlink_bytes=637616512 weight_to_token_ratio=0.194469 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394314236e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265494528 dispatch_meta_bytes=343896 combine_payload_bytes=265494528 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265838424 ingress_bytes=334700544 link_bytes=334700544 token_bytes=531332952 weight_movement_bytes=69206016 total_nvlink_bytes=600538968 weight_to_token_ratio=0.130250 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259451201e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74424 payload_rows_remote=65212 dup_rows=23880 dup_rows_remote=20966 dedup_ratio=0.242920 router_tokens_per_expert=[743, 774, 753, 750, 750, 737, 736, 725, 734, 783, 780, 781, 716, 778, 767, 755, 788, 763, 776, 804, 843, 782, 788, 774, 810, 777, 733, 786, 747, 760, 768, 764, 768, 770, 753, 748, 800, 725, 809, 755, 782, 791, 747, 756, 792, 784, 798, 773, 805, 793, 725, 776, 766, 770, 745, 773, 729, 768, 732, 798, 780, 780, 746, 790] payload_rows_per_dst=[9212, 9554, 9234, 9250, 9292, 9344, 9272, 9266] dup_rows_per_dst=[2914, 3186, 2890, 2896, 3014, 3102, 3034, 2844] combine_rows_per_src=[9212, 9554, 9234, 9250, 9292, 9344, 9272, 9266] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=65128 dup_rows=23930 dup_rows_remote=21094 dedup_ratio=0.243429 router_tokens_per_expert=[792, 740, 848, 760, 779, 846, 772, 786, 779, 730, 792, 785, 803, 765, 738, 788, 751, 796, 727, 798, 755, 760, 730, 781, 739, 791, 783, 737, 778, 772, 779, 814, 751, 754, 755, 766, 738, 762, 767, 748, 710, 738, 791, 740, 757, 759, 733, 809, 742, 770, 801, 730, 763, 813, 768, 745, 779, 764, 749, 807, 745, 778, 742, 784] payload_rows_per_dst=[9386, 9302, 9268, 9452, 9246, 9134, 9222, 9364] dup_rows_per_dst=[3260, 3058, 2928, 2934, 2836, 2940, 3042, 2932] combine_rows_per_src=[9386, 9302, 9268, 9452, 9246, 9134, 9222, 9364] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74406 payload_rows_remote=65072 dup_rows=23898 dup_rows_remote=20918 dedup_ratio=0.243103 router_tokens_per_expert=[746, 770, 762, 800, 752, 765, 806, 692, 809, 779, 763, 763, 724, 732, 781, 788, 801, 800, 813, 813, 771, 792, 767, 775, 777, 746, 769, 712, 763, 797, 804, 800, 803, 749, 746, 738, 776, 723, 785, 787, 738, 745, 800, 799, 793, 748, 774, 760, 753, 782, 793, 785, 801, 718, 775, 695, 759, 778, 762, 759, 737, 753, 738, 768] payload_rows_per_dst=[9146, 9308, 9542, 9398, 9280, 9334, 9212, 9186] dup_rows_per_dst=[3040, 2970, 3122, 2938, 2934, 2980, 2992, 2922] combine_rows_per_src=[9146, 9308, 9542, 9398, 9280, 9334, 9212, 9186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=344448 combine_payload_bytes=266731520 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267075968 ingress_bytes=370540544 link_bytes=370540544 token_bytes=533807488 weight_movement_bytes=103809024 total_nvlink_bytes=637616512 weight_to_token_ratio=0.194469 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394314236e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74124 payload_rows_remote=64756 dup_rows=24180 dup_rows_remote=21072 dedup_ratio=0.245972 router_tokens_per_expert=[766, 752, 797, 725, 764, 756, 790, 747, 784, 741, 772, 774, 785, 785, 817, 727, 856, 797, 756, 757, 791, 695, 745, 742, 752, 813, 775, 783, 747, 804, 772, 792, 749, 748, 770, 775, 787, 718, 801, 743, 781, 753, 766, 731, 744, 759, 733, 797, 739, 783, 772, 802, 764, 750, 781, 775, 789, 788, 805, 741, 771, 761, 763, 754] payload_rows_per_dst=[9174, 9308, 9216, 9368, 9264, 9198, 9264, 9332] dup_rows_per_dst=[3020, 3062, 3062, 3108, 2918, 2930, 3068, 3012] combine_rows_per_src=[9174, 9308, 9216, 9368, 9264, 9198, 9264, 9332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265191424 dispatch_meta_bytes=343576 combine_payload_bytes=265191424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265535000 ingress_bytes=299794432 link_bytes=299794432 token_bytes=530726424 weight_movement_bytes=34603008 total_nvlink_bytes=565329432 weight_to_token_ratio=0.065199 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128102312e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265601024 dispatch_meta_bytes=344776 combine_payload_bytes=265601024 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265945800 ingress_bytes=369410048 link_bytes=369410048 token_bytes=531546824 weight_movement_bytes=103809024 total_nvlink_bytes=635355848 weight_to_token_ratio=0.195296 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390060271e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266985472 dispatch_meta_bytes=344752 combine_payload_bytes=266985472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267330224 ingress_bytes=301588480 link_bytes=301588480 token_bytes=534315696 weight_movement_bytes=34603008 total_nvlink_bytes=568918704 weight_to_token_ratio=0.064761 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134853170e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265674752 dispatch_meta_bytes=343648 combine_payload_bytes=265674752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266018400 ingress_bytes=300277760 link_bytes=300277760 token_bytes=531693152 weight_movement_bytes=34603008 total_nvlink_bytes=566296160 weight_to_token_ratio=0.065081 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129921037e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265240576 dispatch_meta_bytes=343312 combine_payload_bytes=265240576 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=265583888 ingress_bytes=403652608 link_bytes=403652608 token_bytes=530824464 weight_movement_bytes=138412032 total_nvlink_bytes=669236496 weight_to_token_ratio=0.260749 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.518912267e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74406 payload_rows_remote=65072 dup_rows=23898 dup_rows_remote=20918 dedup_ratio=0.243103 router_tokens_per_expert=[746, 770, 762, 800, 752, 765, 806, 692, 809, 779, 763, 763, 724, 732, 781, 788, 801, 800, 813, 813, 771, 792, 767, 775, 777, 746, 769, 712, 763, 797, 804, 800, 803, 749, 746, 738, 776, 723, 785, 787, 738, 745, 800, 799, 793, 748, 774, 760, 753, 782, 793, 785, 801, 718, 775, 695, 759, 778, 762, 759, 737, 753, 738, 768] payload_rows_per_dst=[9146, 9308, 9542, 9398, 9280, 9334, 9212, 9186] dup_rows_per_dst=[3040, 2970, 3122, 2938, 2934, 2980, 2992, 2922] combine_rows_per_src=[9146, 9308, 9542, 9398, 9280, 9334, 9212, 9186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=65044 dup_rows=23964 dup_rows_remote=20982 dedup_ratio=0.243774 router_tokens_per_expert=[766, 828, 752, 814, 844, 757, 764, 737, 744, 743, 797, 805, 742, 743, 790, 756, 789, 781, 795, 787, 848, 754, 763, 731, 745, 752, 788, 756, 775, 749, 798, 726, 750, 766, 797, 779, 741, 717, 784, 764, 754, 738, 731, 784, 789, 753, 766, 701, 771, 751, 751, 778, 759, 762, 771, 796, 770, 751, 755, 767, 768, 801, 788, 780] payload_rows_per_dst=[9482, 9280, 9372, 9186, 9228, 9156, 9296, 9340] dup_rows_per_dst=[3042, 2960, 3124, 2992, 2968, 2876, 2982, 3020] combine_rows_per_src=[9482, 9280, 9372, 9186, 9228, 9156, 9296, 9340] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74214 payload_rows_remote=64756 dup_rows=24090 dup_rows_remote=21024 dedup_ratio=0.245056 router_tokens_per_expert=[760, 780, 763, 782, 755, 780, 773, 767, 740, 745, 737, 773, 818, 727, 747, 746, 761, 752, 768, 797, 731, 754, 764, 748, 792, 703, 766, 765, 834, 790, 738, 793, 772, 709, 753, 766, 767, 764, 764, 791, 750, 807, 798, 776, 797, 758, 817, 803, 722, 750, 781, 822, 747, 745, 741, 744, 797, 769, 789, 805, 763, 791, 760, 785] payload_rows_per_dst=[9222, 9214, 9242, 9108, 9396, 9404, 9170, 9458] dup_rows_per_dst=[3098, 2852, 2908, 3016, 3008, 3208, 2934, 3066] combine_rows_per_src=[9222, 9214, 9242, 9108, 9396, 9404, 9170, 9458] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74488 payload_rows_remote=65352 dup_rows=23816 dup_rows_remote=21004 dedup_ratio=0.242269 router_tokens_per_expert=[736, 783, 815, 769, 760, 729, 768, 800, 710, 759, 757, 717, 757, 762, 787, 725, 794, 723, 771, 769, 800, 796, 712, 741, 784, 761, 776, 775, 764, 801, 753, 764, 790, 786, 779, 744, 783, 771, 756, 768, 774, 797, 769, 754, 804, 796, 766, 780, 771, 815, 727, 762, 801, 759, 767, 767, 761, 759, 745, 824, 762, 772, 774, 751] payload_rows_per_dst=[9356, 9136, 9212, 9370, 9378, 9410, 9284, 9342] dup_rows_per_dst=[2964, 2812, 3000, 2986, 2976, 3070, 3054, 2954] combine_rows_per_src=[9356, 9136, 9212, 9370, 9378, 9410, 9284, 9342] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=65044 dup_rows=23964 dup_rows_remote=20982 dedup_ratio=0.243774 router_tokens_per_expert=[766, 828, 752, 814, 844, 757, 764, 737, 744, 743, 797, 805, 742, 743, 790, 756, 789, 781, 795, 787, 848, 754, 763, 731, 745, 752, 788, 756, 775, 749, 798, 726, 750, 766, 797, 779, 741, 717, 784, 764, 754, 738, 731, 784, 789, 753, 766, 701, 771, 751, 751, 778, 759, 762, 771, 796, 770, 751, 755, 767, 768, 801, 788, 780] payload_rows_per_dst=[9482, 9280, 9372, 9186, 9228, 9156, 9296, 9340] dup_rows_per_dst=[3042, 2960, 3124, 2992, 2968, 2876, 2982, 3020] combine_rows_per_src=[9482, 9280, 9372, 9186, 9228, 9156, 9296, 9340] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=267108352 dispatch_meta_bytes=344712 combine_payload_bytes=267108352 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267453064 ingress_bytes=336314368 link_bytes=336314368 token_bytes=534561416 weight_movement_bytes=69206016 total_nvlink_bytes=603767432 weight_to_token_ratio=0.129463 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265523891e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266764288 dispatch_meta_bytes=344888 combine_payload_bytes=266764288 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267109176 ingress_bytes=301367296 link_bytes=301367296 token_bytes=533873464 weight_movement_bytes=34603008 total_nvlink_bytes=568476472 weight_to_token_ratio=0.064815 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134020873e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266534912 dispatch_meta_bytes=343960 combine_payload_bytes=266534912 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266878872 ingress_bytes=335740928 link_bytes=335740928 token_bytes=533413784 weight_movement_bytes=69206016 total_nvlink_bytes=602619800 weight_to_token_ratio=0.129742 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263366083e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74298 payload_rows_remote=65164 dup_rows=24006 dup_rows_remote=21008 dedup_ratio=0.244202 router_tokens_per_expert=[781, 800, 814, 754, 752, 771, 750, 766, 747, 776, 767, 762, 776, 804, 780, 732, 765, 761, 783, 815, 737, 753, 740, 761, 747, 779, 750, 719, 816, 795, 804, 796, 752, 752, 733, 760, 739, 763, 791, 776, 753, 780, 765, 799, 774, 735, 775, 736, 804, 750, 733, 740, 810, 792, 767, 766, 787, 781, 757, 732, 770, 799, 741, 787] payload_rows_per_dst=[9326, 9248, 9320, 9376, 9134, 9304, 9288, 9302] dup_rows_per_dst=[3050, 3040, 2910, 3036, 2998, 2930, 3036, 3006] combine_rows_per_src=[9326, 9248, 9320, 9376, 9134, 9304, 9288, 9302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=65070 dup_rows=24132 dup_rows_remote=21230 dedup_ratio=0.245483 router_tokens_per_expert=[763, 810, 797, 735, 800, 733, 741, 749, 762, 766, 759, 804, 764, 759, 725, 786, 790, 733, 706, 764, 759, 746, 767, 737, 778, 822, 773, 787, 759, 770, 778, 806, 784, 786, 782, 748, 765, 746, 761, 793, 749, 764, 735, 791, 751, 739, 761, 748, 757, 768, 783, 771, 775, 811, 790, 780, 779, 809, 757, 737, 746, 833, 742, 783] payload_rows_per_dst=[9270, 9272, 9102, 9410, 9290, 9132, 9434, 9262] dup_rows_per_dst=[2986, 2978, 2902, 3136, 3040, 2944, 3036, 3110] combine_rows_per_src=[9270, 9272, 9102, 9410, 9290, 9132, 9434, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74366 payload_rows_remote=64920 dup_rows=23938 dup_rows_remote=20714 dedup_ratio=0.243510 router_tokens_per_expert=[809, 769, 778, 753, 778, 781, 749, 772, 757, 738, 778, 815, 737, 698, 748, 775, 758, 746, 754, 738, 772, 797, 747, 850, 775, 774, 788, 819, 787, 812, 764, 747, 796, 725, 770, 746, 755, 790, 771, 804, 786, 798, 763, 748, 790, 742, 767, 762, 703, 795, 778, 703, 764, 749, 764, 769, 773, 785, 797, 769, 785, 724, 774, 744] payload_rows_per_dst=[9446, 9242, 9332, 9536, 9260, 9188, 9094, 9268] dup_rows_per_dst=[3224, 2918, 2992, 3136, 2914, 2946, 2774, 3034] combine_rows_per_src=[9446, 9242, 9332, 9536, 9260, 9188, 9094, 9268] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74214 payload_rows_remote=64756 dup_rows=24090 dup_rows_remote=21024 dedup_ratio=0.245056 router_tokens_per_expert=[760, 780, 763, 782, 755, 780, 773, 767, 740, 745, 737, 773, 818, 727, 747, 746, 761, 752, 768, 797, 731, 754, 764, 748, 792, 703, 766, 765, 834, 790, 738, 793, 772, 709, 753, 766, 767, 764, 764, 791, 750, 807, 798, 776, 797, 758, 817, 803, 722, 750, 781, 822, 747, 745, 741, 744, 797, 769, 789, 805, 763, 791, 760, 785] payload_rows_per_dst=[9222, 9214, 9242, 9108, 9396, 9404, 9170, 9458] dup_rows_per_dst=[3098, 2852, 2908, 3016, 3008, 3208, 2934, 3066] combine_rows_per_src=[9222, 9214, 9242, 9108, 9396, 9404, 9170, 9458] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267681792 dispatch_meta_bytes=345424 combine_payload_bytes=267681792 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268027216 ingress_bytes=302284800 link_bytes=302284800 token_bytes=535709008 weight_movement_bytes=34603008 total_nvlink_bytes=570312016 weight_to_token_ratio=0.064593 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137473366e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=344104 combine_payload_bytes=266420224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266764328 ingress_bytes=301023232 link_bytes=301023232 token_bytes=533184552 weight_movement_bytes=34603008 total_nvlink_bytes=567787560 weight_to_token_ratio=0.064899 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132726188e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74124 payload_rows_remote=64756 dup_rows=24180 dup_rows_remote=21072 dedup_ratio=0.245972 router_tokens_per_expert=[766, 752, 797, 725, 764, 756, 790, 747, 784, 741, 772, 774, 785, 785, 817, 727, 856, 797, 756, 757, 791, 695, 745, 742, 752, 813, 775, 783, 747, 804, 772, 792, 749, 748, 770, 775, 787, 718, 801, 743, 781, 753, 766, 731, 744, 759, 733, 797, 739, 783, 772, 802, 764, 750, 781, 775, 789, 788, 805, 741, 771, 761, 763, 754] payload_rows_per_dst=[9174, 9308, 9216, 9368, 9264, 9198, 9264, 9332] dup_rows_per_dst=[3020, 3062, 3062, 3108, 2918, 2930, 3068, 3012] combine_rows_per_src=[9174, 9308, 9216, 9368, 9264, 9198, 9264, 9332] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=65070 dup_rows=24132 dup_rows_remote=21230 dedup_ratio=0.245483 router_tokens_per_expert=[763, 810, 797, 735, 800, 733, 741, 749, 762, 766, 759, 804, 764, 759, 725, 786, 790, 733, 706, 764, 759, 746, 767, 737, 778, 822, 773, 787, 759, 770, 778, 806, 784, 786, 782, 748, 765, 746, 761, 793, 749, 764, 735, 791, 751, 739, 761, 748, 757, 768, 783, 771, 775, 811, 790, 780, 779, 809, 757, 737, 746, 833, 742, 783] payload_rows_per_dst=[9270, 9272, 9102, 9410, 9290, 9132, 9434, 9262] dup_rows_per_dst=[2986, 2978, 2902, 3136, 3040, 2944, 3036, 3110] combine_rows_per_src=[9270, 9272, 9102, 9410, 9290, 9132, 9434, 9262] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265240576 dispatch_meta_bytes=343120 combine_payload_bytes=265240576 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265583696 ingress_bytes=369049600 link_bytes=369049600 token_bytes=530824272 weight_movement_bytes=103809024 total_nvlink_bytes=634633296 weight_to_token_ratio=0.195562 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388703934e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265240576 dispatch_meta_bytes=343312 combine_payload_bytes=265240576 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=265583888 ingress_bytes=403652608 link_bytes=403652608 token_bytes=530824464 weight_movement_bytes=138412032 total_nvlink_bytes=669236496 weight_to_token_ratio=0.260749 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.518912267e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74054 payload_rows_remote=64704 dup_rows=24250 dup_rows_remote=21248 dedup_ratio=0.246684 router_tokens_per_expert=[711, 771, 764, 766, 813, 767, 740, 789, 775, 776, 812, 791, 774, 767, 775, 770, 748, 757, 779, 773, 794, 745, 795, 744, 820, 761, 778, 770, 758, 736, 736, 817, 843, 762, 742, 777, 747, 731, 722, 773, 686, 785, 803, 780, 778, 749, 751, 799, 820, 782, 751, 790, 796, 790, 797, 760, 708, 755, 751, 729, 761, 730, 766, 766] payload_rows_per_dst=[9282, 9324, 9306, 9350, 9106, 9180, 9434, 9072] dup_rows_per_dst=[2960, 3156, 2964, 3002, 3088, 3082, 3138, 2860] combine_rows_per_src=[9282, 9324, 9306, 9350, 9106, 9180, 9434, 9072] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74140 payload_rows_remote=64862 dup_rows=24164 dup_rows_remote=21050 dedup_ratio=0.245809 router_tokens_per_expert=[715, 766, 766, 744, 773, 779, 771, 759, 797, 770, 754, 762, 812, 750, 778, 766, 762, 788, 724, 737, 736, 770, 778, 834, 766, 738, 716, 788, 800, 795, 775, 787, 797, 764, 769, 789, 796, 771, 734, 791, 800, 760, 797, 763, 776, 735, 812, 742, 759, 781, 796, 782, 746, 767, 779, 744, 773, 750, 754, 750, 766, 748, 755, 750] payload_rows_per_dst=[9278, 9660, 9284, 9152, 9380, 8956, 9274, 9156] dup_rows_per_dst=[3114, 3306, 3014, 2892, 3042, 2840, 3184, 2772] combine_rows_per_src=[9278, 9660, 9284, 9152, 9380, 8956, 9274, 9156] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65182 dup_rows=23944 dup_rows_remote=21006 dedup_ratio=0.243571 router_tokens_per_expert=[767, 748, 801, 752, 759, 782, 751, 731, 763, 730, 750, 728, 736, 776, 775, 777, 765, 788, 735, 764, 757, 761, 776, 756, 753, 758, 787, 748, 801, 769, 787, 760, 768, 772, 733, 767, 728, 791, 743, 756, 802, 771, 852, 783, 779, 791, 785, 789, 692, 756, 789, 805, 769, 786, 727, 816, 797, 767, 765, 767, 838, 780, 766, 731] payload_rows_per_dst=[9326, 9232, 9280, 9218, 9178, 9500, 9254, 9372] dup_rows_per_dst=[2856, 2838, 2924, 3108, 2938, 3204, 3026, 3050] combine_rows_per_src=[9326, 9232, 9280, 9218, 9178, 9500, 9254, 9372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=64944 dup_rows=24020 dup_rows_remote=20888 dedup_ratio=0.244344 router_tokens_per_expert=[765, 752, 806, 756, 771, 761, 783, 758, 761, 724, 735, 784, 753, 775, 760, 720, 747, 807, 742, 754, 793, 761, 768, 761, 725, 776, 819, 804, 759, 785, 740, 765, 798, 733, 800, 759, 724, 754, 785, 739, 778, 771, 761, 808, 768, 798, 794, 758, 743, 808, 761, 767, 797, 764, 762, 774, 715, 722, 805, 765, 810, 798, 782, 781] payload_rows_per_dst=[9298, 9100, 9292, 9420, 9174, 9340, 9278, 9382] dup_rows_per_dst=[3006, 2924, 2974, 2926, 3010, 3132, 3074, 2974] combine_rows_per_src=[9298, 9100, 9292, 9420, 9174, 9340, 9278, 9382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265240576 dispatch_meta_bytes=343120 combine_payload_bytes=265240576 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265583696 ingress_bytes=369049600 link_bytes=369049600 token_bytes=530824272 weight_movement_bytes=103809024 total_nvlink_bytes=634633296 weight_to_token_ratio=0.195562 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388703934e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267681792 dispatch_meta_bytes=345424 combine_payload_bytes=267681792 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268027216 ingress_bytes=302284800 link_bytes=302284800 token_bytes=535709008 weight_movement_bytes=34603008 total_nvlink_bytes=570312016 weight_to_token_ratio=0.064593 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137473366e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266911744 dispatch_meta_bytes=344688 combine_payload_bytes=266911744 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267256432 ingress_bytes=336117760 link_bytes=336117760 token_bytes=534168176 weight_movement_bytes=69206016 total_nvlink_bytes=603374192 weight_to_token_ratio=0.129558 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264784071e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266526720 dispatch_meta_bytes=345200 combine_payload_bytes=266526720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266871920 ingress_bytes=301129728 link_bytes=301129728 token_bytes=533398640 weight_movement_bytes=34603008 total_nvlink_bytes=568001648 weight_to_token_ratio=0.064873 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133126924e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265912320 dispatch_meta_bytes=342536 combine_payload_bytes=265912320 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266254856 ingress_bytes=335118336 link_bytes=335118336 token_bytes=532167176 weight_movement_bytes=69206016 total_nvlink_bytes=601373192 weight_to_token_ratio=0.130046 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261023319e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265027584 dispatch_meta_bytes=343808 combine_payload_bytes=265027584 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265371392 ingress_bytes=334233600 link_bytes=334233600 token_bytes=530398976 weight_movement_bytes=69206016 total_nvlink_bytes=599604992 weight_to_token_ratio=0.130479 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257694129e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=64944 dup_rows=24020 dup_rows_remote=20888 dedup_ratio=0.244344 router_tokens_per_expert=[765, 752, 806, 756, 771, 761, 783, 758, 761, 724, 735, 784, 753, 775, 760, 720, 747, 807, 742, 754, 793, 761, 768, 761, 725, 776, 819, 804, 759, 785, 740, 765, 798, 733, 800, 759, 724, 754, 785, 739, 778, 771, 761, 808, 768, 798, 794, 758, 743, 808, 761, 767, 797, 764, 762, 774, 715, 722, 805, 765, 810, 798, 782, 781] payload_rows_per_dst=[9298, 9100, 9292, 9420, 9174, 9340, 9278, 9382] dup_rows_per_dst=[3006, 2924, 2974, 2926, 3010, 3132, 3074, 2974] combine_rows_per_src=[9298, 9100, 9292, 9420, 9174, 9340, 9278, 9382] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74128 payload_rows_remote=64842 dup_rows=24176 dup_rows_remote=21092 dedup_ratio=0.245931 router_tokens_per_expert=[743, 782, 760, 777, 790, 763, 759, 752, 798, 766, 739, 807, 776, 782, 793, 724, 744, 755, 725, 716, 750, 767, 768, 719, 761, 773, 761, 813, 737, 798, 800, 761, 762, 801, 807, 766, 749, 827, 792, 793, 793, 769, 781, 720, 758, 797, 767, 758, 771, 802, 774, 751, 760, 736, 741, 817, 778, 772, 757, 764, 768, 740, 758, 764] payload_rows_per_dst=[9198, 9286, 9118, 9336, 9386, 9280, 9318, 9206] dup_rows_per_dst=[3054, 3084, 2770, 3072, 3208, 3006, 2986, 2996] combine_rows_per_src=[9198, 9286, 9118, 9336, 9386, 9280, 9318, 9206] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74488 payload_rows_remote=65222 dup_rows=23816 dup_rows_remote=20964 dedup_ratio=0.242269 router_tokens_per_expert=[786, 805, 754, 721, 752, 822, 815, 772, 787, 815, 746, 748, 770, 770, 803, 739, 771, 781, 762, 763, 800, 809, 776, 785, 707, 743, 781, 748, 787, 783, 761, 738, 737, 767, 758, 725, 815, 794, 794, 751, 764, 779, 734, 756, 734, 794, 789, 747, 776, 774, 727, 738, 763, 824, 732, 725, 798, 734, 773, 765, 770, 717, 810, 788] payload_rows_per_dst=[9388, 9386, 9446, 9156, 9250, 9240, 9266, 9356] dup_rows_per_dst=[3066, 2970, 3048, 2940, 3032, 2954, 2852, 2954] combine_rows_per_src=[9388, 9386, 9446, 9156, 9250, 9240, 9266, 9356] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265674752 dispatch_meta_bytes=343648 combine_payload_bytes=265674752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266018400 ingress_bytes=300277760 link_bytes=300277760 token_bytes=531693152 weight_movement_bytes=34603008 total_nvlink_bytes=566296160 weight_to_token_ratio=0.065081 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129921037e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266985472 dispatch_meta_bytes=344752 combine_payload_bytes=266985472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267330224 ingress_bytes=301588480 link_bytes=301588480 token_bytes=534315696 weight_movement_bytes=34603008 total_nvlink_bytes=568918704 weight_to_token_ratio=0.064761 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134853170e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266010624 dispatch_meta_bytes=343328 combine_payload_bytes=266010624 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266353952 ingress_bytes=335216640 link_bytes=335216640 token_bytes=532364576 weight_movement_bytes=69206016 total_nvlink_bytes=601570592 weight_to_token_ratio=0.129997 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261393229e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74240 payload_rows_remote=64736 dup_rows=24064 dup_rows_remote=20898 dedup_ratio=0.244792 router_tokens_per_expert=[768, 752, 805, 761, 787, 729, 761, 742, 802, 796, 788, 751, 791, 744, 776, 794, 820, 746, 739, 781, 794, 748, 779, 738, 784, 762, 732, 788, 770, 794, 775, 759, 789, 735, 756, 763, 720, 766, 754, 781, 836, 727, 743, 798, 768, 718, 748, 816, 734, 782, 808, 783, 798, 754, 759, 761, 757, 756, 749, 773, 761, 731, 812, 760] payload_rows_per_dst=[8896, 9410, 9602, 9056, 9096, 9334, 9342, 9504] dup_rows_per_dst=[2630, 3074, 3372, 2800, 3000, 2974, 3048, 3166] combine_rows_per_src=[8896, 9410, 9602, 9056, 9096, 9334, 9342, 9504] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265592832 dispatch_meta_bytes=343736 combine_payload_bytes=265592832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265936568 ingress_bytes=300195840 link_bytes=300195840 token_bytes=531529400 weight_movement_bytes=34603008 total_nvlink_bytes=566132408 weight_to_token_ratio=0.065101 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129612778e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267149312 dispatch_meta_bytes=344744 combine_payload_bytes=267149312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267494056 ingress_bytes=301752320 link_bytes=301752320 token_bytes=534643368 weight_movement_bytes=34603008 total_nvlink_bytes=569246376 weight_to_token_ratio=0.064722 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135469687e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74054 payload_rows_remote=64704 dup_rows=24250 dup_rows_remote=21248 dedup_ratio=0.246684 router_tokens_per_expert=[711, 771, 764, 766, 813, 767, 740, 789, 775, 776, 812, 791, 774, 767, 775, 770, 748, 757, 779, 773, 794, 745, 795, 744, 820, 761, 778, 770, 758, 736, 736, 817, 843, 762, 742, 777, 747, 731, 722, 773, 686, 785, 803, 780, 778, 749, 751, 799, 820, 782, 751, 790, 796, 790, 797, 760, 708, 755, 751, 729, 761, 730, 766, 766] payload_rows_per_dst=[9282, 9324, 9306, 9350, 9106, 9180, 9434, 9072] dup_rows_per_dst=[2960, 3156, 2964, 3002, 3088, 3082, 3138, 2860] combine_rows_per_src=[9282, 9324, 9306, 9350, 9106, 9180, 9434, 9072] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74128 payload_rows_remote=64842 dup_rows=24176 dup_rows_remote=21092 dedup_ratio=0.245931 router_tokens_per_expert=[743, 782, 760, 777, 790, 763, 759, 752, 798, 766, 739, 807, 776, 782, 793, 724, 744, 755, 725, 716, 750, 767, 768, 719, 761, 773, 761, 813, 737, 798, 800, 761, 762, 801, 807, 766, 749, 827, 792, 793, 793, 769, 781, 720, 758, 797, 767, 758, 771, 802, 774, 751, 760, 736, 741, 817, 778, 772, 757, 764, 768, 740, 758, 764] payload_rows_per_dst=[9198, 9286, 9118, 9336, 9386, 9280, 9318, 9206] dup_rows_per_dst=[3054, 3084, 2770, 3072, 3208, 3006, 2986, 2996] combine_rows_per_src=[9198, 9286, 9118, 9336, 9386, 9280, 9318, 9206] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=64984 dup_rows=23974 dup_rows_remote=20944 dedup_ratio=0.243876 router_tokens_per_expert=[777, 756, 755, 769, 766, 754, 793, 761, 738, 761, 774, 736, 797, 802, 837, 750, 720, 746, 785, 726, 739, 791, 772, 757, 742, 730, 751, 761, 726, 719, 804, 778, 727, 729, 767, 776, 813, 776, 807, 793, 780, 756, 785, 769, 807, 792, 745, 742, 777, 770, 789, 755, 810, 766, 761, 768, 776, 781, 803, 738, 800, 762, 811, 748] payload_rows_per_dst=[9194, 9296, 9186, 9176, 9346, 9390, 9348, 9394] dup_rows_per_dst=[3068, 3094, 2886, 2846, 3030, 2962, 3044, 3044] combine_rows_per_src=[9194, 9296, 9186, 9176, 9346, 9390, 9348, 9394] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=65166 dup_rows=24010 dup_rows_remote=21090 dedup_ratio=0.244242 router_tokens_per_expert=[826, 772, 816, 784, 784, 782, 782, 777, 803, 783, 793, 768, 787, 725, 736, 742, 763, 723, 750, 785, 757, 754, 743, 749, 744, 738, 774, 767, 748, 738, 769, 741, 837, 773, 779, 765, 767, 776, 799, 754, 758, 783, 721, 761, 756, 782, 803, 740, 750, 788, 808, 748, 774, 723, 787, 766, 770, 789, 717, 844, 720, 742, 770, 799] payload_rows_per_dst=[9518, 9236, 9128, 9214, 9380, 9250, 9292, 9276] dup_rows_per_dst=[3128, 3038, 2920, 2824, 3120, 2958, 2996, 3026] combine_rows_per_src=[9518, 9236, 9128, 9214, 9380, 9250, 9292, 9276] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265158656 dispatch_meta_bytes=342536 combine_payload_bytes=265158656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265501192 ingress_bytes=299761664 link_bytes=299761664 token_bytes=530659848 weight_movement_bytes=34603008 total_nvlink_bytes=565262856 weight_to_token_ratio=0.065208 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127979009e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265027584 dispatch_meta_bytes=343808 combine_payload_bytes=265027584 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265371392 ingress_bytes=334233600 link_bytes=334233600 token_bytes=530398976 weight_movement_bytes=69206016 total_nvlink_bytes=599604992 weight_to_token_ratio=0.130479 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257694129e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74366 payload_rows_remote=64920 dup_rows=23938 dup_rows_remote=20714 dedup_ratio=0.243510 router_tokens_per_expert=[809, 769, 778, 753, 778, 781, 749, 772, 757, 738, 778, 815, 737, 698, 748, 775, 758, 746, 754, 738, 772, 797, 747, 850, 775, 774, 788, 819, 787, 812, 764, 747, 796, 725, 770, 746, 755, 790, 771, 804, 786, 798, 763, 748, 790, 742, 767, 762, 703, 795, 778, 703, 764, 749, 764, 769, 773, 785, 797, 769, 785, 724, 774, 744] payload_rows_per_dst=[9446, 9242, 9332, 9536, 9260, 9188, 9094, 9268] dup_rows_per_dst=[3224, 2918, 2992, 3136, 2914, 2946, 2774, 3034] combine_rows_per_src=[9446, 9242, 9332, 9536, 9260, 9188, 9094, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74214 payload_rows_remote=65048 dup_rows=24090 dup_rows_remote=21164 dedup_ratio=0.245056 router_tokens_per_expert=[748, 714, 775, 760, 727, 773, 779, 770, 778, 737, 802, 747, 740, 779, 775, 803, 742, 758, 757, 784, 732, 758, 806, 738, 747, 797, 778, 808, 729, 769, 708, 786, 787, 760, 756, 750, 768, 757, 820, 779, 751, 717, 767, 790, 795, 794, 772, 803, 817, 787, 730, 795, 766, 775, 758, 743, 742, 773, 804, 797, 767, 762, 804, 762] payload_rows_per_dst=[9166, 9342, 9484, 9530, 9274, 9272, 9068, 9078] dup_rows_per_dst=[2926, 3062, 3138, 3216, 3080, 3024, 2802, 2842] combine_rows_per_src=[9166, 9342, 9484, 9530, 9274, 9272, 9068, 9078] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74130 payload_rows_remote=64914 dup_rows=24174 dup_rows_remote=21198 dedup_ratio=0.245911 router_tokens_per_expert=[770, 796, 745, 768, 794, 743, 742, 761, 732, 778, 750, 754, 797, 766, 819, 802, 764, 794, 770, 718, 753, 768, 754, 758, 747, 750, 721, 727, 777, 789, 839, 746, 741, 725, 763, 787, 799, 758, 755, 769, 750, 797, 810, 819, 828, 744, 757, 736, 791, 790, 750, 717, 743, 771, 778, 780, 748, 774, 763, 765, 836, 786, 732, 798] payload_rows_per_dst=[9270, 9326, 9184, 9216, 9228, 9290, 9302, 9314] dup_rows_per_dst=[2968, 3070, 2974, 2976, 2966, 3192, 2938, 3090] combine_rows_per_src=[9270, 9326, 9184, 9216, 9228, 9290, 9302, 9314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267149312 dispatch_meta_bytes=344744 combine_payload_bytes=267149312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267494056 ingress_bytes=301752320 link_bytes=301752320 token_bytes=534643368 weight_movement_bytes=34603008 total_nvlink_bytes=569246376 weight_to_token_ratio=0.064722 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135469687e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74298 payload_rows_remote=65164 dup_rows=24006 dup_rows_remote=21008 dedup_ratio=0.244202 router_tokens_per_expert=[781, 800, 814, 754, 752, 771, 750, 766, 747, 776, 767, 762, 776, 804, 780, 732, 765, 761, 783, 815, 737, 753, 740, 761, 747, 779, 750, 719, 816, 795, 804, 796, 752, 752, 733, 760, 739, 763, 791, 776, 753, 780, 765, 799, 774, 735, 775, 736, 804, 750, 733, 740, 810, 792, 767, 766, 787, 781, 757, 732, 770, 799, 741, 787] payload_rows_per_dst=[9326, 9248, 9320, 9376, 9134, 9304, 9288, 9302] dup_rows_per_dst=[3050, 3040, 2910, 3036, 2998, 2930, 3036, 3006] combine_rows_per_src=[9326, 9248, 9320, 9376, 9134, 9304, 9288, 9302] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74510 payload_rows_remote=65298 dup_rows=23794 dup_rows_remote=20908 dedup_ratio=0.242045 router_tokens_per_expert=[756, 771, 745, 760, 723, 796, 742, 793, 775, 740, 735, 723, 762, 764, 754, 758, 728, 784, 767, 724, 795, 755, 786, 767, 757, 760, 794, 753, 781, 787, 771, 778, 748, 811, 788, 756, 763, 711, 781, 812, 778, 751, 778, 792, 749, 744, 725, 732, 729, 780, 780, 747, 800, 790, 800, 813, 800, 785, 819, 810, 771, 753, 773, 799] payload_rows_per_dst=[9278, 9146, 9224, 9426, 9332, 9212, 9408, 9484] dup_rows_per_dst=[2894, 2876, 2988, 2936, 3008, 2886, 3070, 3136] combine_rows_per_src=[9278, 9146, 9224, 9426, 9332, 9212, 9408, 9484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265158656 dispatch_meta_bytes=342536 combine_payload_bytes=265158656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265501192 ingress_bytes=299761664 link_bytes=299761664 token_bytes=530659848 weight_movement_bytes=34603008 total_nvlink_bytes=565262856 weight_to_token_ratio=0.065208 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127979009e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265592832 dispatch_meta_bytes=343736 combine_payload_bytes=265592832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265936568 ingress_bytes=300195840 link_bytes=300195840 token_bytes=531529400 weight_movement_bytes=34603008 total_nvlink_bytes=566132408 weight_to_token_ratio=0.065101 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129612778e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266174464 dispatch_meta_bytes=343712 combine_payload_bytes=266174464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266518176 ingress_bytes=300777472 link_bytes=300777472 token_bytes=532692640 weight_movement_bytes=34603008 total_nvlink_bytes=567295648 weight_to_token_ratio=0.064959 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131801413e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266919936 dispatch_meta_bytes=345024 combine_payload_bytes=266919936 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267264960 ingress_bytes=301522944 link_bytes=301522944 token_bytes=534184896 weight_movement_bytes=34603008 total_nvlink_bytes=568787904 weight_to_token_ratio=0.064777 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134606564e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74426 payload_rows_remote=65176 dup_rows=23878 dup_rows_remote=20922 dedup_ratio=0.242900 router_tokens_per_expert=[841, 724, 787, 750, 775, 768, 760, 739, 784, 751, 699, 783, 799, 782, 765, 749, 766, 746, 802, 787, 763, 773, 758, 757, 773, 763, 751, 784, 770, 744, 741, 784, 787, 782, 742, 795, 781, 822, 736, 770, 783, 768, 747, 788, 756, 790, 800, 768, 771, 743, 789, 771, 757, 733, 813, 726, 753, 734, 755, 769, 805, 733, 768, 799] payload_rows_per_dst=[9274, 9232, 9322, 9306, 9410, 9344, 9250, 9288] dup_rows_per_dst=[3014, 2992, 2982, 2914, 3020, 3056, 2956, 2944] combine_rows_per_src=[9274, 9232, 9322, 9306, 9410, 9344, 9250, 9288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266436608 dispatch_meta_bytes=344848 combine_payload_bytes=266436608 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266781456 ingress_bytes=370245632 link_bytes=370245632 token_bytes=533218064 weight_movement_bytes=103809024 total_nvlink_bytes=637027088 weight_to_token_ratio=0.194684 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393204506e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265887744 dispatch_meta_bytes=344448 combine_payload_bytes=265887744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266232192 ingress_bytes=300490752 link_bytes=300490752 token_bytes=532119936 weight_movement_bytes=34603008 total_nvlink_bytes=566722944 weight_to_token_ratio=0.065029 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130722508e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74510 payload_rows_remote=65298 dup_rows=23794 dup_rows_remote=20908 dedup_ratio=0.242045 router_tokens_per_expert=[756, 771, 745, 760, 723, 796, 742, 793, 775, 740, 735, 723, 762, 764, 754, 758, 728, 784, 767, 724, 795, 755, 786, 767, 757, 760, 794, 753, 781, 787, 771, 778, 748, 811, 788, 756, 763, 711, 781, 812, 778, 751, 778, 792, 749, 744, 725, 732, 729, 780, 780, 747, 800, 790, 800, 813, 800, 785, 819, 810, 771, 753, 773, 799] payload_rows_per_dst=[9278, 9146, 9224, 9426, 9332, 9212, 9408, 9484] dup_rows_per_dst=[2894, 2876, 2988, 2936, 3008, 2886, 3070, 3136] combine_rows_per_src=[9278, 9146, 9224, 9426, 9332, 9212, 9408, 9484] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=65000 dup_rows=24020 dup_rows_remote=21000 dedup_ratio=0.244344 router_tokens_per_expert=[790, 774, 758, 777, 787, 814, 747, 737, 748, 764, 736, 730, 784, 803, 805, 782, 765, 754, 753, 753, 746, 748, 775, 780, 790, 735, 768, 711, 779, 773, 792, 775, 793, 775, 758, 805, 778, 817, 765, 773, 752, 796, 744, 766, 725, 802, 780, 719, 771, 769, 763, 781, 789, 743, 804, 775, 766, 749, 760, 730, 737, 773, 789, 772] payload_rows_per_dst=[9338, 9284, 9200, 9238, 9450, 9174, 9360, 9240] dup_rows_per_dst=[3030, 3020, 2948, 3008, 3078, 2994, 3030, 2912] combine_rows_per_src=[9338, 9284, 9200, 9238, 9450, 9174, 9360, 9240] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265912320 dispatch_meta_bytes=342536 combine_payload_bytes=265912320 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266254856 ingress_bytes=335118336 link_bytes=335118336 token_bytes=532167176 weight_movement_bytes=69206016 total_nvlink_bytes=601373192 weight_to_token_ratio=0.130046 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261023319e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266911744 dispatch_meta_bytes=344688 combine_payload_bytes=266911744 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267256432 ingress_bytes=336117760 link_bytes=336117760 token_bytes=534168176 weight_movement_bytes=69206016 total_nvlink_bytes=603374192 weight_to_token_ratio=0.129558 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264784071e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=267460608 dispatch_meta_bytes=344824 combine_payload_bytes=267460608 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267805432 ingress_bytes=371269632 link_bytes=371269632 token_bytes=535266040 weight_movement_bytes=103809024 total_nvlink_bytes=639075064 weight_to_token_ratio=0.193939 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397057736e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74426 payload_rows_remote=65176 dup_rows=23878 dup_rows_remote=20922 dedup_ratio=0.242900 router_tokens_per_expert=[841, 724, 787, 750, 775, 768, 760, 739, 784, 751, 699, 783, 799, 782, 765, 749, 766, 746, 802, 787, 763, 773, 758, 757, 773, 763, 751, 784, 770, 744, 741, 784, 787, 782, 742, 795, 781, 822, 736, 770, 783, 768, 747, 788, 756, 790, 800, 768, 771, 743, 789, 771, 757, 733, 813, 726, 753, 734, 755, 769, 805, 733, 768, 799] payload_rows_per_dst=[9274, 9232, 9322, 9306, 9410, 9344, 9250, 9288] dup_rows_per_dst=[3014, 2992, 2982, 2914, 3020, 3056, 2956, 2944] combine_rows_per_src=[9274, 9232, 9322, 9306, 9410, 9344, 9250, 9288] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74498 payload_rows_remote=65170 dup_rows=23806 dup_rows_remote=20820 dedup_ratio=0.242167 router_tokens_per_expert=[720, 773, 831, 779, 793, 720, 788, 776, 770, 750, 773, 756, 741, 761, 806, 748, 734, 767, 756, 756, 824, 748, 791, 783, 751, 750, 816, 811, 761, 769, 732, 785, 756, 783, 753, 767, 741, 744, 738, 735, 828, 795, 754, 774, 770, 827, 769, 803, 755, 732, 773, 787, 731, 779, 780, 755, 785, 727, 800, 770, 771, 783, 755, 713] payload_rows_per_dst=[9174, 9244, 9360, 9330, 9240, 9464, 9358, 9328] dup_rows_per_dst=[2982, 2966, 2958, 3020, 2882, 3006, 3006, 2986] combine_rows_per_src=[9174, 9244, 9360, 9330, 9240, 9464, 9358, 9328] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266960896 dispatch_meta_bytes=344392 combine_payload_bytes=266960896 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267305288 ingress_bytes=370769920 link_bytes=370769920 token_bytes=534266184 weight_movement_bytes=103809024 total_nvlink_bytes=638075208 weight_to_token_ratio=0.194302 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395177360e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74130 payload_rows_remote=64914 dup_rows=24174 dup_rows_remote=21198 dedup_ratio=0.245911 router_tokens_per_expert=[770, 796, 745, 768, 794, 743, 742, 761, 732, 778, 750, 754, 797, 766, 819, 802, 764, 794, 770, 718, 753, 768, 754, 758, 747, 750, 721, 727, 777, 789, 839, 746, 741, 725, 763, 787, 799, 758, 755, 769, 750, 797, 810, 819, 828, 744, 757, 736, 791, 790, 750, 717, 743, 771, 778, 780, 748, 774, 763, 765, 836, 786, 732, 798] payload_rows_per_dst=[9270, 9326, 9184, 9216, 9228, 9290, 9302, 9314] dup_rows_per_dst=[2968, 3070, 2974, 2976, 2966, 3192, 2938, 3090] combine_rows_per_src=[9270, 9326, 9184, 9216, 9228, 9290, 9302, 9314] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266240000 dispatch_meta_bytes=344000 combine_payload_bytes=266240000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266584000 ingress_bytes=300843008 link_bytes=300843008 token_bytes=532824000 weight_movement_bytes=34603008 total_nvlink_bytes=567427008 weight_to_token_ratio=0.064943 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132048019e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=65000 dup_rows=24020 dup_rows_remote=21000 dedup_ratio=0.244344 router_tokens_per_expert=[790, 774, 758, 777, 787, 814, 747, 737, 748, 764, 736, 730, 784, 803, 805, 782, 765, 754, 753, 753, 746, 748, 775, 780, 790, 735, 768, 711, 779, 773, 792, 775, 793, 775, 758, 805, 778, 817, 765, 773, 752, 796, 744, 766, 725, 802, 780, 719, 771, 769, 763, 781, 789, 743, 804, 775, 766, 749, 760, 730, 737, 773, 789, 772] payload_rows_per_dst=[9338, 9284, 9200, 9238, 9450, 9174, 9360, 9240] dup_rows_per_dst=[3030, 3020, 2948, 3008, 3078, 2994, 3030, 2912] combine_rows_per_src=[9338, 9284, 9200, 9238, 9450, 9174, 9360, 9240] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74100 payload_rows_remote=64862 dup_rows=24204 dup_rows_remote=21140 dedup_ratio=0.246216 router_tokens_per_expert=[720, 772, 791, 771, 781, 759, 767, 783, 747, 755, 759, 723, 799, 738, 776, 780, 773, 773, 725, 801, 739, 791, 810, 728, 780, 775, 704, 746, 739, 791, 787, 764, 765, 765, 758, 780, 784, 758, 765, 776, 745, 827, 760, 826, 784, 761, 778, 774, 749, 717, 746, 775, 815, 767, 768, 763, 790, 800, 793, 773, 776, 731, 761, 775] payload_rows_per_dst=[9258, 9182, 9226, 9200, 9238, 9428, 9244, 9324] dup_rows_per_dst=[3030, 2972, 3054, 2972, 3064, 3082, 2956, 3074] combine_rows_per_src=[9258, 9182, 9226, 9200, 9238, 9428, 9244, 9324] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74414 payload_rows_remote=65170 dup_rows=23890 dup_rows_remote=20994 dedup_ratio=0.243022 router_tokens_per_expert=[791, 798, 811, 780, 758, 837, 771, 787, 819, 787, 774, 749, 769, 782, 773, 767, 710, 743, 804, 771, 773, 734, 751, 784, 753, 778, 767, 753, 749, 767, 778, 770, 765, 795, 772, 773, 766, 766, 752, 769, 720, 798, 751, 753, 723, 752, 816, 775, 807, 757, 778, 749, 720, 746, 749, 767, 766, 702, 758, 775, 782, 772, 836, 704] payload_rows_per_dst=[9552, 9470, 9244, 9232, 9286, 9152, 9228, 9250] dup_rows_per_dst=[3114, 2970, 2896, 2998, 3030, 3024, 2918, 2940] combine_rows_per_src=[9552, 9470, 9244, 9232, 9286, 9152, 9228, 9250] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266936320 dispatch_meta_bytes=343960 combine_payload_bytes=266936320 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267280280 ingress_bytes=336142336 link_bytes=336142336 token_bytes=534216600 weight_movement_bytes=69206016 total_nvlink_bytes=603422616 weight_to_token_ratio=0.129547 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264876549e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265887744 dispatch_meta_bytes=344448 combine_payload_bytes=265887744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266232192 ingress_bytes=300490752 link_bytes=300490752 token_bytes=532119936 weight_movement_bytes=34603008 total_nvlink_bytes=566722944 weight_to_token_ratio=0.065029 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130722508e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74214 payload_rows_remote=65048 dup_rows=24090 dup_rows_remote=21164 dedup_ratio=0.245056 router_tokens_per_expert=[748, 714, 775, 760, 727, 773, 779, 770, 778, 737, 802, 747, 740, 779, 775, 803, 742, 758, 757, 784, 732, 758, 806, 738, 747, 797, 778, 808, 729, 769, 708, 786, 787, 760, 756, 750, 768, 757, 820, 779, 751, 717, 767, 790, 795, 794, 772, 803, 817, 787, 730, 795, 766, 775, 758, 743, 742, 773, 804, 797, 767, 762, 804, 762] payload_rows_per_dst=[9166, 9342, 9484, 9530, 9274, 9272, 9068, 9078] dup_rows_per_dst=[2926, 3062, 3138, 3216, 3080, 3024, 2802, 2842] combine_rows_per_src=[9166, 9342, 9484, 9530, 9274, 9272, 9068, 9078] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74432 payload_rows_remote=65140 dup_rows=23872 dup_rows_remote=20858 dedup_ratio=0.242839 router_tokens_per_expert=[746, 817, 776, 759, 764, 790, 804, 735, 764, 781, 734, 778, 766, 733, 754, 770, 788, 754, 757, 751, 783, 810, 780, 722, 761, 750, 785, 754, 793, 761, 791, 758, 785, 748, 757, 786, 815, 755, 770, 795, 753, 738, 791, 741, 713, 781, 751, 772, 772, 765, 737, 788, 733, 771, 736, 807, 763, 760, 786, 799, 779, 799, 748, 789] payload_rows_per_dst=[9372, 9144, 9338, 9292, 9344, 9214, 9346, 9382] dup_rows_per_dst=[3010, 3016, 2952, 3014, 3078, 2866, 2872, 3064] combine_rows_per_src=[9372, 9144, 9338, 9292, 9344, 9214, 9346, 9382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74476 payload_rows_remote=65134 dup_rows=23828 dup_rows_remote=20806 dedup_ratio=0.242391 router_tokens_per_expert=[721, 776, 798, 739, 803, 749, 735, 786, 766, 792, 728, 749, 775, 768, 814, 741, 813, 735, 776, 810, 739, 783, 782, 792, 714, 807, 778, 733, 744, 764, 755, 739, 813, 790, 748, 750, 754, 775, 767, 778, 764, 788, 772, 739, 775, 755, 776, 744, 734, 814, 727, 752, 801, 804, 790, 761, 788, 759, 777, 762, 723, 815, 791, 762] payload_rows_per_dst=[9342, 9264, 9398, 9368, 9344, 9294, 9076, 9390] dup_rows_per_dst=[3022, 2892, 3098, 3184, 3006, 2932, 2730, 2964] combine_rows_per_src=[9342, 9264, 9398, 9368, 9344, 9294, 9076, 9390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266960896 dispatch_meta_bytes=344392 combine_payload_bytes=266960896 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267305288 ingress_bytes=370769920 link_bytes=370769920 token_bytes=534266184 weight_movement_bytes=103809024 total_nvlink_bytes=638075208 weight_to_token_ratio=0.194302 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395177360e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=64984 dup_rows=23974 dup_rows_remote=20944 dedup_ratio=0.243876 router_tokens_per_expert=[777, 756, 755, 769, 766, 754, 793, 761, 738, 761, 774, 736, 797, 802, 837, 750, 720, 746, 785, 726, 739, 791, 772, 757, 742, 730, 751, 761, 726, 719, 804, 778, 727, 729, 767, 776, 813, 776, 807, 793, 780, 756, 785, 769, 807, 792, 745, 742, 777, 770, 789, 755, 810, 766, 761, 768, 776, 781, 803, 738, 800, 762, 811, 748] payload_rows_per_dst=[9194, 9296, 9186, 9176, 9346, 9390, 9348, 9394] dup_rows_per_dst=[3068, 3094, 2886, 2846, 3030, 2962, 3044, 3044] combine_rows_per_src=[9194, 9296, 9186, 9176, 9346, 9390, 9348, 9394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266936320 dispatch_meta_bytes=343960 combine_payload_bytes=266936320 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267280280 ingress_bytes=336142336 link_bytes=336142336 token_bytes=534216600 weight_movement_bytes=69206016 total_nvlink_bytes=603422616 weight_to_token_ratio=0.129547 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264876549e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266240000 dispatch_meta_bytes=344000 combine_payload_bytes=266240000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266584000 ingress_bytes=300843008 link_bytes=300843008 token_bytes=532824000 weight_movement_bytes=34603008 total_nvlink_bytes=567427008 weight_to_token_ratio=0.064943 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132048019e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265674752 dispatch_meta_bytes=344008 combine_payload_bytes=265674752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266018760 ingress_bytes=300277760 link_bytes=300277760 token_bytes=531693512 weight_movement_bytes=34603008 total_nvlink_bytes=566296520 weight_to_token_ratio=0.065081 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129921037e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74348 payload_rows_remote=65054 dup_rows=23956 dup_rows_remote=20908 dedup_ratio=0.243693 router_tokens_per_expert=[783, 766, 774, 771, 743, 752, 740, 798, 800, 762, 807, 765, 724, 777, 802, 777, 841, 783, 751, 759, 735, 725, 793, 720, 794, 757, 730, 740, 744, 757, 773, 762, 752, 775, 778, 756, 757, 773, 746, 764, 776, 820, 757, 766, 707, 746, 802, 797, 801, 771, 777, 809, 780, 796, 751, 783, 762, 785, 749, 736, 746, 771, 772, 786] payload_rows_per_dst=[9298, 9356, 9244, 9124, 9236, 9294, 9554, 9242] dup_rows_per_dst=[2956, 3072, 2970, 2990, 2966, 3048, 2982, 2972] combine_rows_per_src=[9298, 9356, 9244, 9124, 9236, 9294, 9554, 9242] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266936320 dispatch_meta_bytes=344656 combine_payload_bytes=266936320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267280976 ingress_bytes=301539328 link_bytes=301539328 token_bytes=534217296 weight_movement_bytes=34603008 total_nvlink_bytes=568820304 weight_to_token_ratio=0.064773 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134668215e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=64996 dup_rows=23926 dup_rows_remote=20884 dedup_ratio=0.243388 router_tokens_per_expert=[744, 789, 762, 760, 820, 774, 767, 759, 788, 782, 707, 726, 819, 778, 705, 764, 768, 808, 844, 796, 783, 764, 747, 807, 736, 793, 790, 740, 790, 757, 717, 755, 754, 732, 784, 741, 750, 756, 721, 799, 776, 725, 797, 765, 781, 768, 741, 778, 785, 783, 794, 792, 824, 711, 753, 770, 753, 707, 723, 781, 789, 793, 757, 830] payload_rows_per_dst=[9394, 9200, 9520, 9228, 9200, 9250, 9382, 9204] dup_rows_per_dst=[2956, 2938, 3114, 2928, 2874, 3012, 3042, 3062] combine_rows_per_src=[9394, 9200, 9520, 9228, 9200, 9250, 9382, 9204] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266813440 dispatch_meta_bytes=343992 combine_payload_bytes=266813440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267157432 ingress_bytes=301416448 link_bytes=301416448 token_bytes=533970872 weight_movement_bytes=34603008 total_nvlink_bytes=568573880 weight_to_token_ratio=0.064803 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134205828e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=343760 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267132624 ingress_bytes=301391872 link_bytes=301391872 token_bytes=533921488 weight_movement_bytes=34603008 total_nvlink_bytes=568524496 weight_to_token_ratio=0.064809 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134113350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=64826 dup_rows=24050 dup_rows_remote=20902 dedup_ratio=0.244649 router_tokens_per_expert=[760, 794, 738, 770, 760, 839, 696, 771, 768, 809, 810, 803, 801, 723, 797, 777, 758, 749, 744, 757, 828, 784, 774, 803, 794, 712, 810, 751, 781, 770, 746, 782, 744, 745, 765, 791, 781, 760, 804, 760, 779, 753, 766, 722, 772, 760, 765, 786, 756, 733, 786, 739, 784, 763, 768, 750, 747, 772, 786, 773, 776, 768, 737, 702] payload_rows_per_dst=[9276, 9428, 9310, 9300, 9382, 9264, 9184, 9110] dup_rows_per_dst=[2980, 3148, 3084, 2992, 2918, 2942, 2974, 3012] combine_rows_per_src=[9276, 9428, 9310, 9300, 9382, 9264, 9184, 9110] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266436608 dispatch_meta_bytes=344848 combine_payload_bytes=266436608 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266781456 ingress_bytes=370245632 link_bytes=370245632 token_bytes=533218064 weight_movement_bytes=103809024 total_nvlink_bytes=637027088 weight_to_token_ratio=0.194684 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393204506e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266174464 dispatch_meta_bytes=343712 combine_payload_bytes=266174464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266518176 ingress_bytes=300777472 link_bytes=300777472 token_bytes=532692640 weight_movement_bytes=34603008 total_nvlink_bytes=567295648 weight_to_token_ratio=0.064959 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131801413e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=343848 combine_payload_bytes=266461184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266805032 ingress_bytes=301064192 link_bytes=301064192 token_bytes=533266216 weight_movement_bytes=34603008 total_nvlink_bytes=567869224 weight_to_token_ratio=0.064889 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132880317e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=64996 dup_rows=23926 dup_rows_remote=20884 dedup_ratio=0.243388 router_tokens_per_expert=[744, 789, 762, 760, 820, 774, 767, 759, 788, 782, 707, 726, 819, 778, 705, 764, 768, 808, 844, 796, 783, 764, 747, 807, 736, 793, 790, 740, 790, 757, 717, 755, 754, 732, 784, 741, 750, 756, 721, 799, 776, 725, 797, 765, 781, 768, 741, 778, 785, 783, 794, 792, 824, 711, 753, 770, 753, 707, 723, 781, 789, 793, 757, 830] payload_rows_per_dst=[9394, 9200, 9520, 9228, 9200, 9250, 9382, 9204] dup_rows_per_dst=[2956, 2938, 3114, 2928, 2874, 3012, 3042, 3062] combine_rows_per_src=[9394, 9200, 9520, 9228, 9200, 9250, 9382, 9204] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=65100 dup_rows=24062 dup_rows_remote=21088 dedup_ratio=0.244771 router_tokens_per_expert=[765, 763, 797, 815, 744, 768, 796, 770, 785, 783, 803, 836, 775, 795, 750, 763, 788, 746, 810, 728, 782, 825, 779, 790, 767, 745, 823, 793, 716, 757, 774, 704, 757, 745, 767, 783, 813, 754, 740, 726, 745, 743, 808, 764, 720, 757, 766, 765, 830, 776, 802, 759, 736, 776, 777, 770, 787, 745, 726, 755, 716, 687, 794, 728] payload_rows_per_dst=[9354, 9268, 9148, 9296, 9274, 9322, 9438, 9142] dup_rows_per_dst=[3082, 2940, 2972, 2862, 3032, 3186, 3014, 2974] combine_rows_per_src=[9354, 9268, 9148, 9296, 9274, 9322, 9438, 9142] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=64826 dup_rows=24050 dup_rows_remote=20902 dedup_ratio=0.244649 router_tokens_per_expert=[760, 794, 738, 770, 760, 839, 696, 771, 768, 809, 810, 803, 801, 723, 797, 777, 758, 749, 744, 757, 828, 784, 774, 803, 794, 712, 810, 751, 781, 770, 746, 782, 744, 745, 765, 791, 781, 760, 804, 760, 779, 753, 766, 722, 772, 760, 765, 786, 756, 733, 786, 739, 784, 763, 768, 750, 747, 772, 786, 773, 776, 768, 737, 702] payload_rows_per_dst=[9276, 9428, 9310, 9300, 9382, 9264, 9184, 9110] dup_rows_per_dst=[2980, 3148, 3084, 2992, 2918, 2942, 2974, 3012] combine_rows_per_src=[9276, 9428, 9310, 9300, 9382, 9264, 9184, 9110] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74004 payload_rows_remote=64786 dup_rows=24300 dup_rows_remote=21310 dedup_ratio=0.247192 router_tokens_per_expert=[772, 737, 779, 734, 733, 753, 765, 755, 769, 771, 773, 788, 768, 783, 703, 739, 771, 790, 762, 735, 808, 768, 772, 755, 801, 750, 727, 783, 773, 853, 792, 763, 769, 760, 734, 775, 797, 784, 764, 721, 753, 773, 750, 762, 733, 778, 788, 740, 780, 785, 811, 759, 777, 800, 762, 801, 812, 802, 772, 729, 783, 780, 721, 772] payload_rows_per_dst=[9080, 9244, 9336, 9366, 9218, 9116, 9348, 9296] dup_rows_per_dst=[2976, 2944, 2986, 3118, 2990, 3038, 3202, 3046] combine_rows_per_src=[9080, 9244, 9336, 9366, 9218, 9116, 9348, 9296] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=343520 combine_payload_bytes=266223616 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266567136 ingress_bytes=335429632 link_bytes=335429632 token_bytes=532790752 weight_movement_bytes=69206016 total_nvlink_bytes=601996768 weight_to_token_ratio=0.129893 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262194701e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74432 payload_rows_remote=65140 dup_rows=23872 dup_rows_remote=20858 dedup_ratio=0.242839 router_tokens_per_expert=[746, 817, 776, 759, 764, 790, 804, 735, 764, 781, 734, 778, 766, 733, 754, 770, 788, 754, 757, 751, 783, 810, 780, 722, 761, 750, 785, 754, 793, 761, 791, 758, 785, 748, 757, 786, 815, 755, 770, 795, 753, 738, 791, 741, 713, 781, 751, 772, 772, 765, 737, 788, 733, 771, 736, 807, 763, 760, 786, 799, 779, 799, 748, 789] payload_rows_per_dst=[9372, 9144, 9338, 9292, 9344, 9214, 9346, 9382] dup_rows_per_dst=[3010, 3016, 2952, 3014, 3078, 2866, 2872, 3064] combine_rows_per_src=[9372, 9144, 9338, 9292, 9344, 9214, 9346, 9382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265527296 dispatch_meta_bytes=342912 combine_payload_bytes=265527296 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265870208 ingress_bytes=369336320 link_bytes=369336320 token_bytes=531397504 weight_movement_bytes=103809024 total_nvlink_bytes=635206528 weight_to_token_ratio=0.195351 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389782838e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74122 payload_rows_remote=64980 dup_rows=24182 dup_rows_remote=21224 dedup_ratio=0.245992 router_tokens_per_expert=[733, 760, 771, 730, 755, 725, 775, 778, 761, 790, 800, 772, 791, 755, 731, 805, 765, 732, 767, 745, 760, 739, 806, 736, 785, 757, 756, 789, 753, 803, 788, 742, 781, 797, 730, 757, 807, 762, 742, 809, 762, 781, 773, 759, 780, 738, 788, 729, 767, 818, 760, 764, 732, 795, 820, 778, 780, 721, 746, 763, 791, 825, 779, 763] payload_rows_per_dst=[9196, 9378, 9142, 9280, 9278, 9202, 9414, 9232] dup_rows_per_dst=[2858, 3032, 2958, 3066, 3092, 3018, 3054, 3104] combine_rows_per_src=[9196, 9378, 9142, 9280, 9278, 9202, 9414, 9232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266649600 dispatch_meta_bytes=344752 combine_payload_bytes=266649600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266994352 ingress_bytes=301252608 link_bytes=301252608 token_bytes=533643952 weight_movement_bytes=34603008 total_nvlink_bytes=568246960 weight_to_token_ratio=0.064843 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133589311e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266813440 dispatch_meta_bytes=343992 combine_payload_bytes=266813440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267157432 ingress_bytes=301416448 link_bytes=301416448 token_bytes=533970872 weight_movement_bytes=34603008 total_nvlink_bytes=568573880 weight_to_token_ratio=0.064803 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134205828e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74476 payload_rows_remote=65134 dup_rows=23828 dup_rows_remote=20806 dedup_ratio=0.242391 router_tokens_per_expert=[721, 776, 798, 739, 803, 749, 735, 786, 766, 792, 728, 749, 775, 768, 814, 741, 813, 735, 776, 810, 739, 783, 782, 792, 714, 807, 778, 733, 744, 764, 755, 739, 813, 790, 748, 750, 754, 775, 767, 778, 764, 788, 772, 739, 775, 755, 776, 744, 734, 814, 727, 752, 801, 804, 790, 761, 788, 759, 777, 762, 723, 815, 791, 762] payload_rows_per_dst=[9342, 9264, 9398, 9368, 9344, 9294, 9076, 9390] dup_rows_per_dst=[3022, 2892, 3098, 3184, 3006, 2932, 2730, 2964] combine_rows_per_src=[9342, 9264, 9398, 9368, 9344, 9294, 9076, 9390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=64732 dup_rows=24012 dup_rows_remote=20676 dedup_ratio=0.244263 router_tokens_per_expert=[751, 753, 710, 733, 802, 774, 779, 798, 748, 776, 786, 750, 759, 823, 794, 788, 760, 813, 746, 758, 760, 785, 779, 795, 774, 769, 786, 793, 738, 732, 770, 755, 787, 728, 788, 760, 715, 753, 736, 746, 759, 772, 753, 804, 783, 731, 792, 748, 714, 787, 741, 737, 784, 790, 840, 784, 740, 735, 806, 761, 784, 744, 816, 797] payload_rows_per_dst=[9560, 9400, 9354, 9242, 9528, 9032, 9024, 9152] dup_rows_per_dst=[3336, 3078, 3038, 2992, 3094, 2840, 2734, 2900] combine_rows_per_src=[9560, 9400, 9354, 9242, 9528, 9032, 9024, 9152] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74262 payload_rows_remote=64900 dup_rows=24042 dup_rows_remote=20970 dedup_ratio=0.244568 router_tokens_per_expert=[737, 746, 776, 768, 754, 756, 766, 710, 746, 758, 783, 759, 781, 811, 779, 736, 744, 787, 754, 753, 732, 812, 803, 773, 776, 768, 760, 802, 774, 763, 783, 791, 745, 727, 786, 765, 807, 717, 728, 730, 774, 785, 794, 825, 759, 786, 753, 786, 812, 732, 789, 770, 756, 792, 785, 769, 743, 745, 793, 778, 751, 740, 777, 812] payload_rows_per_dst=[9156, 9354, 9240, 9362, 9158, 9330, 9374, 9288] dup_rows_per_dst=[2870, 2952, 3076, 3072, 2852, 3194, 3036, 2990] combine_rows_per_src=[9156, 9354, 9240, 9362, 9158, 9330, 9374, 9288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266649600 dispatch_meta_bytes=344752 combine_payload_bytes=266649600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266994352 ingress_bytes=301252608 link_bytes=301252608 token_bytes=533643952 weight_movement_bytes=34603008 total_nvlink_bytes=568246960 weight_to_token_ratio=0.064843 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133589311e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265527296 dispatch_meta_bytes=342912 combine_payload_bytes=265527296 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265870208 ingress_bytes=369336320 link_bytes=369336320 token_bytes=531397504 weight_movement_bytes=103809024 total_nvlink_bytes=635206528 weight_to_token_ratio=0.195351 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389782838e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=344384 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265707840 ingress_bytes=299966464 link_bytes=299966464 token_bytes=531071296 weight_movement_bytes=34603008 total_nvlink_bytes=565674304 weight_to_token_ratio=0.065157 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128749655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=64928 dup_rows=24010 dup_rows_remote=20902 dedup_ratio=0.244242 router_tokens_per_expert=[761, 769, 768, 808, 770, 723, 756, 787, 810, 752, 798, 779, 778, 762, 765, 780, 741, 770, 721, 727, 734, 783, 781, 751, 763, 761, 768, 760, 766, 754, 761, 781, 734, 756, 762, 812, 758, 771, 805, 780, 768, 773, 809, 809, 768, 753, 754, 803, 744, 750, 740, 759, 759, 756, 735, 765, 782, 748, 797, 767, 780, 802, 768, 797] payload_rows_per_dst=[9292, 9360, 9214, 9220, 9262, 9366, 9114, 9466] dup_rows_per_dst=[2992, 3088, 2802, 3008, 3094, 3108, 2902, 3016] combine_rows_per_src=[9292, 9360, 9214, 9220, 9262, 9366, 9114, 9466] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74100 payload_rows_remote=64862 dup_rows=24204 dup_rows_remote=21140 dedup_ratio=0.246216 router_tokens_per_expert=[720, 772, 791, 771, 781, 759, 767, 783, 747, 755, 759, 723, 799, 738, 776, 780, 773, 773, 725, 801, 739, 791, 810, 728, 780, 775, 704, 746, 739, 791, 787, 764, 765, 765, 758, 780, 784, 758, 765, 776, 745, 827, 760, 826, 784, 761, 778, 774, 749, 717, 746, 775, 815, 767, 768, 763, 790, 800, 793, 773, 776, 731, 761, 775] payload_rows_per_dst=[9258, 9182, 9226, 9200, 9238, 9428, 9244, 9324] dup_rows_per_dst=[3030, 2972, 3054, 2972, 3064, 3082, 2956, 3074] combine_rows_per_src=[9258, 9182, 9226, 9200, 9238, 9428, 9244, 9324] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74174 payload_rows_remote=64806 dup_rows=24130 dup_rows_remote=21032 dedup_ratio=0.245463 router_tokens_per_expert=[755, 776, 770, 774, 787, 791, 752, 730, 752, 717, 800, 752, 763, 712, 779, 831, 728, 800, 825, 742, 787, 758, 828, 737, 783, 773, 753, 786, 782, 775, 758, 772, 703, 710, 782, 774, 736, 795, 752, 774, 798, 755, 794, 740, 756, 768, 731, 764, 806, 794, 788, 802, 751, 796, 745, 751, 799, 735, 761, 774, 757, 739, 795, 799] payload_rows_per_dst=[9270, 9196, 9294, 9342, 9124, 9280, 9368, 9300] dup_rows_per_dst=[3000, 3016, 3116, 3022, 2928, 2932, 3098, 3018] combine_rows_per_src=[9270, 9196, 9294, 9342, 9124, 9280, 9368, 9300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266158080 dispatch_meta_bytes=344816 combine_payload_bytes=266158080 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266502896 ingress_bytes=369967104 link_bytes=369967104 token_bytes=532660976 weight_movement_bytes=103809024 total_nvlink_bytes=636470000 weight_to_token_ratio=0.194888 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392156428e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265142272 dispatch_meta_bytes=341632 combine_payload_bytes=265142272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265483904 ingress_bytes=299745280 link_bytes=299745280 token_bytes=530626176 weight_movement_bytes=34603008 total_nvlink_bytes=565229184 weight_to_token_ratio=0.065212 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127917357e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=343760 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267132624 ingress_bytes=301391872 link_bytes=301391872 token_bytes=533921488 weight_movement_bytes=34603008 total_nvlink_bytes=568524496 weight_to_token_ratio=0.064809 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134113350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74138 payload_rows_remote=64990 dup_rows=24166 dup_rows_remote=21152 dedup_ratio=0.245829 router_tokens_per_expert=[786, 788, 698, 763, 734, 782, 752, 792, 788, 751, 756, 762, 794, 743, 740, 747, 750, 750, 751, 822, 819, 790, 749, 779, 757, 796, 695, 757, 764, 764, 746, 777, 799, 763, 740, 752, 780, 793, 746, 759, 784, 765, 794, 782, 804, 767, 785, 749, 762, 795, 773, 758, 725, 760, 760, 757, 744, 793, 826, 795, 822, 784, 782, 712] payload_rows_per_dst=[9296, 9148, 9318, 9294, 9236, 9310, 9202, 9334] dup_rows_per_dst=[2894, 3014, 3102, 2818, 3028, 3150, 2978, 3182] combine_rows_per_src=[9296, 9148, 9318, 9294, 9236, 9310, 9202, 9334] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265830400 dispatch_meta_bytes=343480 combine_payload_bytes=265830400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266173880 ingress_bytes=300433408 link_bytes=300433408 token_bytes=532004280 weight_movement_bytes=34603008 total_nvlink_bytes=566607288 weight_to_token_ratio=0.065043 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130506727e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265945088 dispatch_meta_bytes=343320 combine_payload_bytes=265945088 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266288408 ingress_bytes=335151104 link_bytes=335151104 token_bytes=532233496 weight_movement_bytes=69206016 total_nvlink_bytes=601439512 weight_to_token_ratio=0.130029 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261146622e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265674752 dispatch_meta_bytes=344008 combine_payload_bytes=265674752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266018760 ingress_bytes=300277760 link_bytes=300277760 token_bytes=531693512 weight_movement_bytes=34603008 total_nvlink_bytes=566296520 weight_to_token_ratio=0.065081 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129921037e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74174 payload_rows_remote=64806 dup_rows=24130 dup_rows_remote=21032 dedup_ratio=0.245463 router_tokens_per_expert=[755, 776, 770, 774, 787, 791, 752, 730, 752, 717, 800, 752, 763, 712, 779, 831, 728, 800, 825, 742, 787, 758, 828, 737, 783, 773, 753, 786, 782, 775, 758, 772, 703, 710, 782, 774, 736, 795, 752, 774, 798, 755, 794, 740, 756, 768, 731, 764, 806, 794, 788, 802, 751, 796, 745, 751, 799, 735, 761, 774, 757, 739, 795, 799] payload_rows_per_dst=[9270, 9196, 9294, 9342, 9124, 9280, 9368, 9300] dup_rows_per_dst=[3000, 3016, 3116, 3022, 2928, 2932, 3098, 3018] combine_rows_per_src=[9270, 9196, 9294, 9342, 9124, 9280, 9368, 9300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74138 payload_rows_remote=64990 dup_rows=24166 dup_rows_remote=21152 dedup_ratio=0.245829 router_tokens_per_expert=[786, 788, 698, 763, 734, 782, 752, 792, 788, 751, 756, 762, 794, 743, 740, 747, 750, 750, 751, 822, 819, 790, 749, 779, 757, 796, 695, 757, 764, 764, 746, 777, 799, 763, 740, 752, 780, 793, 746, 759, 784, 765, 794, 782, 804, 767, 785, 749, 762, 795, 773, 758, 725, 760, 760, 757, 744, 793, 826, 795, 822, 784, 782, 712] payload_rows_per_dst=[9296, 9148, 9318, 9294, 9236, 9310, 9202, 9334] dup_rows_per_dst=[2894, 3014, 3102, 2818, 3028, 3150, 2978, 3182] combine_rows_per_src=[9296, 9148, 9318, 9294, 9236, 9310, 9202, 9334] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74418 payload_rows_remote=65084 dup_rows=23886 dup_rows_remote=20902 dedup_ratio=0.242981 router_tokens_per_expert=[789, 773, 759, 768, 738, 790, 761, 776, 772, 715, 778, 787, 756, 797, 736, 802, 761, 768, 763, 763, 760, 771, 777, 754, 742, 815, 737, 783, 757, 745, 799, 709, 793, 738, 725, 754, 764, 807, 760, 804, 790, 791, 788, 730, 760, 768, 756, 785, 769, 784, 794, 773, 764, 766, 784, 745, 767, 786, 826, 773, 753, 748, 761, 745] payload_rows_per_dst=[9294, 9212, 9120, 9336, 9318, 9354, 9450, 9334] dup_rows_per_dst=[3014, 2914, 2956, 2932, 2972, 2982, 3132, 2984] combine_rows_per_src=[9294, 9212, 9120, 9336, 9318, 9354, 9450, 9334] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=64988 dup_rows=23966 dup_rows_remote=20808 dedup_ratio=0.243795 router_tokens_per_expert=[762, 769, 771, 753, 707, 845, 810, 777, 781, 773, 755, 766, 768, 758, 752, 759, 806, 752, 780, 721, 756, 778, 798, 711, 771, 734, 790, 749, 778, 749, 792, 754, 768, 754, 798, 800, 797, 750, 764, 823, 776, 757, 769, 762, 755, 821, 763, 798, 765, 758, 786, 731, 768, 759, 749, 760, 784, 737, 753, 789, 723, 780, 772, 758] payload_rows_per_dst=[9384, 9240, 9244, 9276, 9350, 9432, 9210, 9202] dup_rows_per_dst=[3004, 2984, 2960, 2958, 3158, 2970, 2942, 2990] combine_rows_per_src=[9384, 9240, 9244, 9276, 9350, 9432, 9210, 9202] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265445376 dispatch_meta_bytes=343352 combine_payload_bytes=265445376 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265788728 ingress_bytes=369254400 link_bytes=369254400 token_bytes=531234104 weight_movement_bytes=103809024 total_nvlink_bytes=635043128 weight_to_token_ratio=0.195411 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389474580e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74262 payload_rows_remote=64900 dup_rows=24042 dup_rows_remote=20970 dedup_ratio=0.244568 router_tokens_per_expert=[737, 746, 776, 768, 754, 756, 766, 710, 746, 758, 783, 759, 781, 811, 779, 736, 744, 787, 754, 753, 732, 812, 803, 773, 776, 768, 760, 802, 774, 763, 783, 791, 745, 727, 786, 765, 807, 717, 728, 730, 774, 785, 794, 825, 759, 786, 753, 786, 812, 732, 789, 770, 756, 792, 785, 769, 743, 745, 793, 778, 751, 740, 777, 812] payload_rows_per_dst=[9156, 9354, 9240, 9362, 9158, 9330, 9374, 9288] dup_rows_per_dst=[2870, 2952, 3076, 3072, 2852, 3194, 3036, 2990] combine_rows_per_src=[9156, 9354, 9240, 9362, 9158, 9330, 9374, 9288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74142 payload_rows_remote=64808 dup_rows=24162 dup_rows_remote=21158 dedup_ratio=0.245789 router_tokens_per_expert=[725, 734, 763, 769, 731, 763, 768, 773, 779, 780, 724, 735, 804, 774, 776, 827, 763, 789, 767, 785, 705, 804, 774, 782, 754, 734, 754, 787, 798, 783, 809, 775, 741, 773, 782, 733, 778, 745, 787, 732, 759, 780, 739, 821, 731, 787, 737, 823, 771, 814, 757, 732, 808, 786, 808, 748, 783, 742, 789, 763, 733, 748, 749, 785] payload_rows_per_dst=[9096, 9368, 9334, 9388, 9196, 9284, 9280, 9196] dup_rows_per_dst=[2956, 3030, 3004, 3000, 2946, 3070, 3168, 2988] combine_rows_per_src=[9096, 9368, 9334, 9388, 9196, 9284, 9280, 9196] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=64858 dup_rows=24046 dup_rows_remote=21024 dedup_ratio=0.244609 router_tokens_per_expert=[758, 711, 790, 765, 804, 748, 762, 804, 779, 797, 782, 744, 811, 770, 767, 760, 762, 690, 805, 743, 797, 772, 792, 778, 770, 777, 770, 760, 790, 780, 745, 771, 764, 779, 761, 723, 813, 776, 727, 784, 803, 768, 766, 724, 785, 796, 753, 770, 747, 727, 775, 776, 789, 754, 743, 813, 749, 750, 707, 800, 740, 825, 775, 736] payload_rows_per_dst=[9400, 9340, 9580, 9218, 9218, 9338, 8918, 9246] dup_rows_per_dst=[3022, 3108, 3344, 2942, 3036, 2992, 2684, 2918] combine_rows_per_src=[9400, 9340, 9580, 9218, 9218, 9338, 8918, 9246] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=344568 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266543608 ingress_bytes=370008064 link_bytes=370008064 token_bytes=532742648 weight_movement_bytes=103809024 total_nvlink_bytes=636551672 weight_to_token_ratio=0.194858 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392310557e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266584064 dispatch_meta_bytes=343944 combine_payload_bytes=266584064 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266928008 ingress_bytes=335790080 link_bytes=335790080 token_bytes=533512072 weight_movement_bytes=69206016 total_nvlink_bytes=602718088 weight_to_token_ratio=0.129718 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263551038e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74142 payload_rows_remote=64808 dup_rows=24162 dup_rows_remote=21158 dedup_ratio=0.245789 router_tokens_per_expert=[725, 734, 763, 769, 731, 763, 768, 773, 779, 780, 724, 735, 804, 774, 776, 827, 763, 789, 767, 785, 705, 804, 774, 782, 754, 734, 754, 787, 798, 783, 809, 775, 741, 773, 782, 733, 778, 745, 787, 732, 759, 780, 739, 821, 731, 787, 737, 823, 771, 814, 757, 732, 808, 786, 808, 748, 783, 742, 789, 763, 733, 748, 749, 785] payload_rows_per_dst=[9096, 9368, 9334, 9388, 9196, 9284, 9280, 9196] dup_rows_per_dst=[2956, 3030, 3004, 3000, 2946, 3070, 3168, 2988] combine_rows_per_src=[9096, 9368, 9334, 9388, 9196, 9284, 9280, 9196] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265830400 dispatch_meta_bytes=343480 combine_payload_bytes=265830400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266173880 ingress_bytes=300433408 link_bytes=300433408 token_bytes=532004280 weight_movement_bytes=34603008 total_nvlink_bytes=566607288 weight_to_token_ratio=0.065043 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130506727e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=64732 dup_rows=24012 dup_rows_remote=20676 dedup_ratio=0.244263 router_tokens_per_expert=[751, 753, 710, 733, 802, 774, 779, 798, 748, 776, 786, 750, 759, 823, 794, 788, 760, 813, 746, 758, 760, 785, 779, 795, 774, 769, 786, 793, 738, 732, 770, 755, 787, 728, 788, 760, 715, 753, 736, 746, 759, 772, 753, 804, 783, 731, 792, 748, 714, 787, 741, 737, 784, 790, 840, 784, 740, 735, 806, 761, 784, 744, 816, 797] payload_rows_per_dst=[9560, 9400, 9354, 9242, 9528, 9032, 9024, 9152] dup_rows_per_dst=[3336, 3078, 3038, 2992, 3094, 2840, 2734, 2900] combine_rows_per_src=[9560, 9400, 9354, 9242, 9528, 9032, 9024, 9152] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=65134 dup_rows=23928 dup_rows_remote=20876 dedup_ratio=0.243408 router_tokens_per_expert=[782, 726, 725, 743, 792, 793, 801, 756, 776, 795, 795, 752, 735, 807, 778, 716, 713, 794, 750, 809, 787, 794, 739, 789, 831, 763, 763, 781, 726, 738, 765, 766, 746, 755, 790, 737, 749, 785, 747, 775, 791, 763, 747, 754, 785, 774, 779, 754, 782, 783, 786, 793, 791, 781, 788, 788, 733, 810, 757, 787, 753, 739, 741, 729] payload_rows_per_dst=[9206, 9372, 9318, 9306, 9160, 9242, 9512, 9260] dup_rows_per_dst=[3030, 2936, 3032, 2960, 3008, 3052, 3072, 2838] combine_rows_per_src=[9206, 9372, 9318, 9306, 9160, 9242, 9512, 9260] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74004 payload_rows_remote=64786 dup_rows=24300 dup_rows_remote=21310 dedup_ratio=0.247192 router_tokens_per_expert=[772, 737, 779, 734, 733, 753, 765, 755, 769, 771, 773, 788, 768, 783, 703, 739, 771, 790, 762, 735, 808, 768, 772, 755, 801, 750, 727, 783, 773, 853, 792, 763, 769, 760, 734, 775, 797, 784, 764, 721, 753, 773, 750, 762, 733, 778, 788, 740, 780, 785, 811, 759, 777, 800, 762, 801, 812, 802, 772, 729, 783, 780, 721, 772] payload_rows_per_dst=[9080, 9244, 9336, 9366, 9218, 9116, 9348, 9296] dup_rows_per_dst=[2976, 2944, 2986, 3118, 2990, 3038, 3202, 3046] combine_rows_per_src=[9080, 9244, 9336, 9366, 9218, 9116, 9348, 9296] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74400 payload_rows_remote=65094 dup_rows=23904 dup_rows_remote=20850 dedup_ratio=0.243164 router_tokens_per_expert=[777, 763, 789, 811, 750, 779, 793, 784, 767, 721, 767, 740, 774, 795, 796, 739, 752, 813, 771, 779, 767, 779, 779, 772, 788, 731, 718, 793, 771, 744, 775, 758, 800, 747, 777, 789, 785, 754, 775, 779, 752, 756, 779, 707, 757, 770, 787, 757, 721, 790, 765, 793, 790, 760, 755, 806, 775, 748, 724, 787, 749, 769, 773, 741] payload_rows_per_dst=[9426, 9276, 9290, 9306, 9340, 9220, 9306, 9236] dup_rows_per_dst=[3066, 2922, 3134, 2850, 3072, 2910, 3054, 2896] combine_rows_per_src=[9426, 9276, 9290, 9306, 9340, 9220, 9306, 9236] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74114 payload_rows_remote=64770 dup_rows=24190 dup_rows_remote=21094 dedup_ratio=0.246073 router_tokens_per_expert=[820, 743, 754, 728, 764, 751, 781, 728, 780, 792, 787, 775, 773, 728, 767, 772, 720, 755, 776, 737, 772, 794, 817, 737, 755, 738, 763, 776, 813, 802, 806, 767, 788, 708, 765, 747, 713, 797, 752, 772, 786, 741, 791, 742, 771, 744, 714, 750, 789, 766, 845, 830, 754, 783, 752, 748, 807, 800, 785, 772, 770, 770, 714, 815] payload_rows_per_dst=[9176, 9386, 9184, 9344, 9112, 9152, 9364, 9396] dup_rows_per_dst=[2962, 2962, 3032, 3096, 2972, 2926, 3170, 3070] combine_rows_per_src=[9176, 9386, 9184, 9344, 9112, 9152, 9364, 9396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266190848 dispatch_meta_bytes=343184 combine_payload_bytes=266190848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266534032 ingress_bytes=300793856 link_bytes=300793856 token_bytes=532724880 weight_movement_bytes=34603008 total_nvlink_bytes=567327888 weight_to_token_ratio=0.064955 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131863064e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=65134 dup_rows=23928 dup_rows_remote=20876 dedup_ratio=0.243408 router_tokens_per_expert=[782, 726, 725, 743, 792, 793, 801, 756, 776, 795, 795, 752, 735, 807, 778, 716, 713, 794, 750, 809, 787, 794, 739, 789, 831, 763, 763, 781, 726, 738, 765, 766, 746, 755, 790, 737, 749, 785, 747, 775, 791, 763, 747, 754, 785, 774, 779, 754, 782, 783, 786, 793, 791, 781, 788, 788, 733, 810, 757, 787, 753, 739, 741, 729] payload_rows_per_dst=[9206, 9372, 9318, 9306, 9160, 9242, 9512, 9260] dup_rows_per_dst=[3030, 2936, 3032, 2960, 3008, 3052, 3072, 2838] combine_rows_per_src=[9206, 9372, 9318, 9306, 9160, 9242, 9512, 9260] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265453568 dispatch_meta_bytes=343864 combine_payload_bytes=265453568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265797432 ingress_bytes=334659584 link_bytes=334659584 token_bytes=531251000 weight_movement_bytes=69206016 total_nvlink_bytes=600457016 weight_to_token_ratio=0.130270 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259297072e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265658368 dispatch_meta_bytes=343528 combine_payload_bytes=265658368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266001896 ingress_bytes=300261376 link_bytes=300261376 token_bytes=531660264 weight_movement_bytes=34603008 total_nvlink_bytes=566263272 weight_to_token_ratio=0.065085 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129859385e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265297920 dispatch_meta_bytes=343456 combine_payload_bytes=265297920 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265641376 ingress_bytes=369106944 link_bytes=369106944 token_bytes=530939296 weight_movement_bytes=103809024 total_nvlink_bytes=634748320 weight_to_token_ratio=0.195520 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388919715e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74400 payload_rows_remote=65094 dup_rows=23904 dup_rows_remote=20850 dedup_ratio=0.243164 router_tokens_per_expert=[777, 763, 789, 811, 750, 779, 793, 784, 767, 721, 767, 740, 774, 795, 796, 739, 752, 813, 771, 779, 767, 779, 779, 772, 788, 731, 718, 793, 771, 744, 775, 758, 800, 747, 777, 789, 785, 754, 775, 779, 752, 756, 779, 707, 757, 770, 787, 757, 721, 790, 765, 793, 790, 760, 755, 806, 775, 748, 724, 787, 749, 769, 773, 741] payload_rows_per_dst=[9426, 9276, 9290, 9306, 9340, 9220, 9306, 9236] dup_rows_per_dst=[3066, 2922, 3134, 2850, 3072, 2910, 3054, 2896] combine_rows_per_src=[9426, 9276, 9290, 9306, 9340, 9220, 9306, 9236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=344040 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267132904 ingress_bytes=301391872 link_bytes=301391872 token_bytes=533921768 weight_movement_bytes=34603008 total_nvlink_bytes=568524776 weight_to_token_ratio=0.064809 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134113350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265142272 dispatch_meta_bytes=341632 combine_payload_bytes=265142272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265483904 ingress_bytes=299745280 link_bytes=299745280 token_bytes=530626176 weight_movement_bytes=34603008 total_nvlink_bytes=565229184 weight_to_token_ratio=0.065212 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127917357e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74400 payload_rows_remote=65100 dup_rows=23904 dup_rows_remote=20922 dedup_ratio=0.243164 router_tokens_per_expert=[761, 701, 784, 806, 757, 762, 788, 790, 805, 737, 753, 768, 773, 782, 730, 793, 790, 742, 754, 759, 777, 790, 739, 751, 765, 811, 750, 751, 721, 760, 740, 759, 799, 734, 742, 782, 737, 770, 809, 761, 851, 764, 829, 755, 747, 785, 707, 756, 791, 786, 785, 769, 785, 797, 797, 742, 733, 748, 793, 797, 722, 753, 795, 782] payload_rows_per_dst=[9278, 9300, 9204, 9212, 9330, 9326, 9418, 9332] dup_rows_per_dst=[3020, 2982, 3000, 2902, 2938, 3062, 3086, 2914] combine_rows_per_src=[9278, 9300, 9204, 9212, 9330, 9326, 9418, 9332] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=344040 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267132904 ingress_bytes=301391872 link_bytes=301391872 token_bytes=533921768 weight_movement_bytes=34603008 total_nvlink_bytes=568524776 weight_to_token_ratio=0.064809 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134113350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=344384 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265707840 ingress_bytes=299966464 link_bytes=299966464 token_bytes=531071296 weight_movement_bytes=34603008 total_nvlink_bytes=565674304 weight_to_token_ratio=0.065157 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128749655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74170 payload_rows_remote=64710 dup_rows=24134 dup_rows_remote=20908 dedup_ratio=0.245504 router_tokens_per_expert=[752, 709, 715, 774, 763, 788, 743, 801, 796, 827, 745, 749, 818, 766, 774, 801, 739, 727, 782, 740, 757, 766, 765, 779, 777, 785, 815, 761, 767, 803, 758, 760, 743, 762, 770, 747, 777, 755, 775, 765, 774, 758, 759, 797, 752, 791, 739, 800, 747, 764, 732, 756, 766, 754, 768, 795, 766, 803, 783, 773, 795, 730, 770, 784] payload_rows_per_dst=[9146, 9136, 9178, 9292, 9390, 9344, 9224, 9460] dup_rows_per_dst=[2944, 2888, 2932, 2894, 3064, 3172, 3014, 3226] combine_rows_per_src=[9146, 9136, 9178, 9292, 9390, 9344, 9224, 9460] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266625024 dispatch_meta_bytes=343776 combine_payload_bytes=266625024 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266968800 ingress_bytes=335831040 link_bytes=335831040 token_bytes=533593824 weight_movement_bytes=69206016 total_nvlink_bytes=602799840 weight_to_token_ratio=0.129698 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263705167e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74170 payload_rows_remote=64710 dup_rows=24134 dup_rows_remote=20908 dedup_ratio=0.245504 router_tokens_per_expert=[752, 709, 715, 774, 763, 788, 743, 801, 796, 827, 745, 749, 818, 766, 774, 801, 739, 727, 782, 740, 757, 766, 765, 779, 777, 785, 815, 761, 767, 803, 758, 760, 743, 762, 770, 747, 777, 755, 775, 765, 774, 758, 759, 797, 752, 791, 739, 800, 747, 764, 732, 756, 766, 754, 768, 795, 766, 803, 783, 773, 795, 730, 770, 784] payload_rows_per_dst=[9146, 9136, 9178, 9292, 9390, 9344, 9224, 9460] dup_rows_per_dst=[2944, 2888, 2932, 2894, 3064, 3172, 3014, 3226] combine_rows_per_src=[9146, 9136, 9178, 9292, 9390, 9344, 9224, 9460] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74114 payload_rows_remote=64770 dup_rows=24190 dup_rows_remote=21094 dedup_ratio=0.246073 router_tokens_per_expert=[820, 743, 754, 728, 764, 751, 781, 728, 780, 792, 787, 775, 773, 728, 767, 772, 720, 755, 776, 737, 772, 794, 817, 737, 755, 738, 763, 776, 813, 802, 806, 767, 788, 708, 765, 747, 713, 797, 752, 772, 786, 741, 791, 742, 771, 744, 714, 750, 789, 766, 845, 830, 754, 783, 752, 748, 807, 800, 785, 772, 770, 770, 714, 815] payload_rows_per_dst=[9176, 9386, 9184, 9344, 9112, 9152, 9364, 9396] dup_rows_per_dst=[2962, 2962, 3032, 3096, 2972, 2926, 3170, 3070] combine_rows_per_src=[9176, 9386, 9184, 9344, 9112, 9152, 9364, 9396] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266649600 dispatch_meta_bytes=344088 combine_payload_bytes=266649600 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266993688 ingress_bytes=405061632 link_bytes=405061632 token_bytes=533643288 weight_movement_bytes=138412032 total_nvlink_bytes=672055320 weight_to_token_ratio=0.259372 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524214311e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265052160 dispatch_meta_bytes=342472 combine_payload_bytes=265052160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265394632 ingress_bytes=299655168 link_bytes=299655168 token_bytes=530446792 weight_movement_bytes=34603008 total_nvlink_bytes=565049800 weight_to_token_ratio=0.065234 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127578273e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73924 payload_rows_remote=64738 dup_rows=24380 dup_rows_remote=21362 dedup_ratio=0.248006 router_tokens_per_expert=[721, 790, 775, 838, 759, 764, 789, 726, 724, 773, 780, 720, 758, 795, 744, 752, 810, 792, 771, 771, 754, 806, 768, 759, 755, 781, 751, 742, 790, 769, 753, 759, 776, 775, 778, 739, 751, 746, 724, 813, 712, 757, 811, 748, 773, 777, 795, 826, 783, 752, 777, 730, 736, 816, 763, 822, 791, 780, 735, 792, 721, 711, 803, 800] payload_rows_per_dst=[9240, 9250, 9274, 9146, 9186, 9292, 9318, 9218] dup_rows_per_dst=[3084, 2842, 3188, 3054, 3018, 3106, 3040, 3048] combine_rows_per_src=[9240, 9250, 9274, 9146, 9186, 9292, 9318, 9218] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=65124 dup_rows=24032 dup_rows_remote=21214 dedup_ratio=0.244466 router_tokens_per_expert=[752, 773, 785, 763, 757, 787, 794, 759, 763, 737, 777, 791, 769, 775, 769, 795, 740, 745, 777, 754, 719, 720, 761, 767, 820, 784, 783, 778, 759, 773, 811, 771, 785, 744, 773, 751, 765, 745, 792, 781, 762, 755, 793, 710, 767, 751, 762, 777, 736, 787, 791, 855, 767, 748, 789, 791, 715, 790, 779, 808, 751, 726, 740, 758] payload_rows_per_dst=[9280, 9276, 9148, 9400, 9274, 9262, 9466, 9166] dup_rows_per_dst=[3060, 3076, 2818, 3158, 2998, 2892, 3062, 2968] combine_rows_per_src=[9280, 9276, 9148, 9400, 9274, 9262, 9466, 9166] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=64858 dup_rows=24046 dup_rows_remote=21024 dedup_ratio=0.244609 router_tokens_per_expert=[758, 711, 790, 765, 804, 748, 762, 804, 779, 797, 782, 744, 811, 770, 767, 760, 762, 690, 805, 743, 797, 772, 792, 778, 770, 777, 770, 760, 790, 780, 745, 771, 764, 779, 761, 723, 813, 776, 727, 784, 803, 768, 766, 724, 785, 796, 753, 770, 747, 727, 775, 776, 789, 754, 743, 813, 749, 750, 707, 800, 740, 825, 775, 736] payload_rows_per_dst=[9400, 9340, 9580, 9218, 9218, 9338, 8918, 9246] dup_rows_per_dst=[3022, 3108, 3344, 2942, 3036, 2992, 2684, 2918] combine_rows_per_src=[9400, 9340, 9580, 9218, 9218, 9338, 8918, 9246] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265297920 dispatch_meta_bytes=343456 combine_payload_bytes=265297920 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265641376 ingress_bytes=369106944 link_bytes=369106944 token_bytes=530939296 weight_movement_bytes=103809024 total_nvlink_bytes=634748320 weight_to_token_ratio=0.195520 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388919715e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=64910 dup_rows=23906 dup_rows_remote=20714 dedup_ratio=0.243184 router_tokens_per_expert=[733, 763, 770, 795, 769, 761, 800, 757, 770, 807, 786, 767, 776, 737, 792, 765, 723, 771, 719, 738, 786, 807, 760, 775, 753, 862, 777, 748, 788, 782, 773, 744, 799, 770, 739, 748, 777, 786, 782, 769, 751, 749, 769, 785, 764, 772, 783, 779, 789, 769, 757, 772, 762, 733, 748, 781, 751, 770, 755, 741, 762, 779, 733, 774] payload_rows_per_dst=[9488, 9412, 9176, 9976, 9360, 8736, 9164, 9086] dup_rows_per_dst=[3192, 3132, 2986, 3572, 2986, 2474, 2674, 2890] combine_rows_per_src=[9488, 9412, 9176, 9976, 9360, 8736, 9164, 9086] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266649600 dispatch_meta_bytes=344088 combine_payload_bytes=266649600 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266993688 ingress_bytes=405061632 link_bytes=405061632 token_bytes=533643288 weight_movement_bytes=138412032 total_nvlink_bytes=672055320 weight_to_token_ratio=0.259372 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524214311e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265052160 dispatch_meta_bytes=342472 combine_payload_bytes=265052160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265394632 ingress_bytes=299655168 link_bytes=299655168 token_bytes=530446792 weight_movement_bytes=34603008 total_nvlink_bytes=565049800 weight_to_token_ratio=0.065234 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127578273e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74196 payload_rows_remote=64872 dup_rows=24108 dup_rows_remote=21104 dedup_ratio=0.245239 router_tokens_per_expert=[728, 774, 736, 762, 759, 785, 760, 782, 793, 781, 781, 784, 754, 741, 751, 764, 742, 742, 768, 779, 728, 781, 768, 783, 740, 788, 747, 762, 735, 757, 805, 830, 750, 763, 754, 800, 754, 763, 741, 797, 772, 800, 791, 779, 761, 735, 754, 722, 800, 775, 771, 776, 751, 757, 772, 792, 765, 718, 811, 807, 779, 791, 745, 816] payload_rows_per_dst=[9228, 9236, 9240, 9324, 9220, 9264, 9264, 9420] dup_rows_per_dst=[2944, 3062, 2942, 3004, 3024, 2964, 3124, 3044] combine_rows_per_src=[9228, 9236, 9240, 9324, 9220, 9264, 9264, 9420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265166848 dispatch_meta_bytes=344400 combine_payload_bytes=265166848 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265511248 ingress_bytes=334372864 link_bytes=334372864 token_bytes=530678096 weight_movement_bytes=69206016 total_nvlink_bytes=599884112 weight_to_token_ratio=0.130411 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258218168e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74282 payload_rows_remote=65026 dup_rows=24022 dup_rows_remote=20938 dedup_ratio=0.244364 router_tokens_per_expert=[803, 745, 771, 776, 800, 755, 738, 752, 746, 757, 744, 721, 759, 737, 780, 794, 763, 739, 773, 770, 813, 800, 744, 759, 782, 761, 744, 757, 775, 786, 768, 748, 772, 788, 734, 760, 790, 797, 782, 798, 768, 744, 781, 786, 810, 756, 769, 756, 755, 757, 756, 775, 793, 798, 722, 784, 747, 731, 744, 788, 768, 803, 797, 783] payload_rows_per_dst=[9332, 9150, 9256, 9300, 9388, 9256, 9288, 9312] dup_rows_per_dst=[2948, 2926, 3066, 2942, 3054, 3084, 2992, 3010] combine_rows_per_src=[9332, 9150, 9256, 9300, 9388, 9256, 9288, 9312] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=64988 dup_rows=23966 dup_rows_remote=20808 dedup_ratio=0.243795 router_tokens_per_expert=[762, 769, 771, 753, 707, 845, 810, 777, 781, 773, 755, 766, 768, 758, 752, 759, 806, 752, 780, 721, 756, 778, 798, 711, 771, 734, 790, 749, 778, 749, 792, 754, 768, 754, 798, 800, 797, 750, 764, 823, 776, 757, 769, 762, 755, 821, 763, 798, 765, 758, 786, 731, 768, 759, 749, 760, 784, 737, 753, 789, 723, 780, 772, 758] payload_rows_per_dst=[9384, 9240, 9244, 9276, 9350, 9432, 9210, 9202] dup_rows_per_dst=[3004, 2984, 2960, 2958, 3158, 2970, 2942, 2990] combine_rows_per_src=[9384, 9240, 9244, 9276, 9350, 9432, 9210, 9202] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74132 payload_rows_remote=64890 dup_rows=24172 dup_rows_remote=21252 dedup_ratio=0.245890 router_tokens_per_expert=[774, 752, 775, 737, 796, 775, 794, 793, 784, 792, 727, 811, 708, 822, 728, 787, 777, 828, 745, 717, 809, 770, 737, 782, 767, 771, 787, 750, 774, 752, 775, 797, 758, 754, 746, 783, 745, 762, 817, 783, 797, 758, 807, 802, 762, 706, 683, 756, 767, 821, 746, 722, 769, 775, 720, 761, 786, 733, 734, 766, 761, 781, 761, 837] payload_rows_per_dst=[9382, 9256, 9338, 9254, 9268, 9154, 9242, 9238] dup_rows_per_dst=[3010, 3062, 2992, 3092, 3028, 2988, 2920, 3080] combine_rows_per_src=[9382, 9256, 9338, 9254, 9268, 9154, 9242, 9238] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266747904 dispatch_meta_bytes=345352 combine_payload_bytes=266747904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267093256 ingress_bytes=301350912 link_bytes=301350912 token_bytes=533841160 weight_movement_bytes=34603008 total_nvlink_bytes=568444168 weight_to_token_ratio=0.064819 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133959221e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265871360 dispatch_meta_bytes=342496 combine_payload_bytes=265871360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266213856 ingress_bytes=300474368 link_bytes=300474368 token_bytes=532085216 weight_movement_bytes=34603008 total_nvlink_bytes=566688224 weight_to_token_ratio=0.065033 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130660857e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74132 payload_rows_remote=64890 dup_rows=24172 dup_rows_remote=21252 dedup_ratio=0.245890 router_tokens_per_expert=[774, 752, 775, 737, 796, 775, 794, 793, 784, 792, 727, 811, 708, 822, 728, 787, 777, 828, 745, 717, 809, 770, 737, 782, 767, 771, 787, 750, 774, 752, 775, 797, 758, 754, 746, 783, 745, 762, 817, 783, 797, 758, 807, 802, 762, 706, 683, 756, 767, 821, 746, 722, 769, 775, 720, 761, 786, 733, 734, 766, 761, 781, 761, 837] payload_rows_per_dst=[9382, 9256, 9338, 9254, 9268, 9154, 9242, 9238] dup_rows_per_dst=[3010, 3062, 2992, 3092, 3028, 2988, 2920, 3080] combine_rows_per_src=[9382, 9256, 9338, 9254, 9268, 9154, 9242, 9238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265715712 dispatch_meta_bytes=343904 combine_payload_bytes=265715712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266059616 ingress_bytes=300318720 link_bytes=300318720 token_bytes=531775328 weight_movement_bytes=34603008 total_nvlink_bytes=566378336 weight_to_token_ratio=0.065071 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130075166e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266346496 dispatch_meta_bytes=343856 combine_payload_bytes=266346496 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266690352 ingress_bytes=335552512 link_bytes=335552512 token_bytes=533036848 weight_movement_bytes=69206016 total_nvlink_bytes=602242864 weight_to_token_ratio=0.129833 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262657088e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265658368 dispatch_meta_bytes=343528 combine_payload_bytes=265658368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266001896 ingress_bytes=300261376 link_bytes=300261376 token_bytes=531660264 weight_movement_bytes=34603008 total_nvlink_bytes=566263272 weight_to_token_ratio=0.065085 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129859385e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=64974 dup_rows=24012 dup_rows_remote=21018 dedup_ratio=0.244263 router_tokens_per_expert=[757, 823, 788, 790, 818, 723, 753, 728, 768, 722, 742, 810, 773, 752, 800, 789, 786, 699, 762, 789, 747, 789, 750, 803, 793, 726, 740, 770, 754, 765, 725, 747, 746, 766, 741, 780, 782, 784, 770, 767, 751, 790, 737, 752, 787, 799, 803, 772, 759, 739, 771, 791, 775, 786, 771, 751, 749, 784, 791, 809, 778, 784, 738, 768] payload_rows_per_dst=[9296, 9318, 9274, 9124, 9284, 9350, 9314, 9332] dup_rows_per_dst=[3064, 2994, 2976, 2916, 2988, 3032, 2972, 3070] combine_rows_per_src=[9296, 9318, 9274, 9124, 9284, 9350, 9314, 9332] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266346496 dispatch_meta_bytes=343856 combine_payload_bytes=266346496 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266690352 ingress_bytes=335552512 link_bytes=335552512 token_bytes=533036848 weight_movement_bytes=69206016 total_nvlink_bytes=602242864 weight_to_token_ratio=0.129833 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262657088e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266190848 dispatch_meta_bytes=343184 combine_payload_bytes=266190848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266534032 ingress_bytes=300793856 link_bytes=300793856 token_bytes=532724880 weight_movement_bytes=34603008 total_nvlink_bytes=567327888 weight_to_token_ratio=0.064955 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131863064e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265789440 dispatch_meta_bytes=344568 combine_payload_bytes=265789440 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266134008 ingress_bytes=334995456 link_bytes=334995456 token_bytes=531923448 weight_movement_bytes=69206016 total_nvlink_bytes=601129464 weight_to_token_ratio=0.130105 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260560932e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=65054 dup_rows=24010 dup_rows_remote=20948 dedup_ratio=0.244242 router_tokens_per_expert=[784, 758, 768, 756, 762, 749, 785, 772, 791, 813, 761, 759, 755, 809, 776, 779, 803, 806, 785, 757, 768, 720, 783, 805, 721, 724, 751, 733, 773, 769, 716, 680, 765, 746, 769, 767, 751, 761, 775, 747, 778, 819, 718, 804, 781, 759, 811, 748, 810, 768, 730, 759, 767, 812, 796, 789, 730, 752, 767, 801, 773, 785, 789, 754] payload_rows_per_dst=[9270, 9456, 9080, 8976, 9158, 9588, 9526, 9240] dup_rows_per_dst=[2998, 3030, 2836, 2758, 2892, 3386, 3048, 3062] combine_rows_per_src=[9270, 9456, 9080, 8976, 9158, 9588, 9526, 9240] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265789440 dispatch_meta_bytes=344568 combine_payload_bytes=265789440 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266134008 ingress_bytes=334995456 link_bytes=334995456 token_bytes=531923448 weight_movement_bytes=69206016 total_nvlink_bytes=601129464 weight_to_token_ratio=0.130105 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260560932e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=65174 dup_rows=24028 dup_rows_remote=21148 dedup_ratio=0.244425 router_tokens_per_expert=[788, 753, 765, 796, 731, 805, 784, 743, 718, 818, 728, 750, 778, 742, 752, 745, 747, 786, 789, 745, 772, 779, 793, 753, 764, 795, 783, 758, 800, 789, 783, 786, 739, 751, 728, 772, 747, 719, 762, 773, 794, 729, 768, 802, 746, 785, 761, 798, 762, 748, 743, 782, 866, 801, 743, 753, 750, 794, 776, 816, 762, 736, 760, 768] payload_rows_per_dst=[9310, 9138, 9258, 9416, 9102, 9376, 9346, 9330] dup_rows_per_dst=[3020, 2924, 3070, 3100, 2880, 2990, 3050, 2994] combine_rows_per_src=[9310, 9138, 9258, 9416, 9102, 9376, 9346, 9330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=64822 dup_rows=24056 dup_rows_remote=20910 dedup_ratio=0.244710 router_tokens_per_expert=[725, 741, 748, 771, 728, 795, 697, 800, 727, 749, 754, 741, 782, 790, 734, 787, 823, 781, 731, 773, 828, 787, 762, 801, 768, 748, 775, 752, 774, 795, 742, 791, 760, 812, 725, 797, 760, 754, 778, 802, 780, 743, 778, 778, 785, 789, 764, 800, 763, 810, 773, 777, 728, 773, 779, 784, 749, 741, 739, 797, 784, 744, 749, 757] payload_rows_per_dst=[9098, 9244, 9426, 9254, 9360, 9360, 9330, 9176] dup_rows_per_dst=[2912, 2884, 3146, 3036, 3016, 3074, 3044, 2944] combine_rows_per_src=[9098, 9244, 9426, 9254, 9360, 9360, 9330, 9176] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74196 payload_rows_remote=64872 dup_rows=24108 dup_rows_remote=21104 dedup_ratio=0.245239 router_tokens_per_expert=[728, 774, 736, 762, 759, 785, 760, 782, 793, 781, 781, 784, 754, 741, 751, 764, 742, 742, 768, 779, 728, 781, 768, 783, 740, 788, 747, 762, 735, 757, 805, 830, 750, 763, 754, 800, 754, 763, 741, 797, 772, 800, 791, 779, 761, 735, 754, 722, 800, 775, 771, 776, 751, 757, 772, 792, 765, 718, 811, 807, 779, 791, 745, 816] payload_rows_per_dst=[9228, 9236, 9240, 9324, 9220, 9264, 9264, 9420] dup_rows_per_dst=[2944, 3062, 2942, 3004, 3024, 2964, 3124, 3044] combine_rows_per_src=[9228, 9236, 9240, 9324, 9220, 9264, 9264, 9420] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=343968 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266477472 ingress_bytes=300736512 link_bytes=300736512 token_bytes=532610976 weight_movement_bytes=34603008 total_nvlink_bytes=567213984 weight_to_token_ratio=0.064969 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131647283e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=64822 dup_rows=24056 dup_rows_remote=20910 dedup_ratio=0.244710 router_tokens_per_expert=[725, 741, 748, 771, 728, 795, 697, 800, 727, 749, 754, 741, 782, 790, 734, 787, 823, 781, 731, 773, 828, 787, 762, 801, 768, 748, 775, 752, 774, 795, 742, 791, 760, 812, 725, 797, 760, 754, 778, 802, 780, 743, 778, 778, 785, 789, 764, 800, 763, 810, 773, 777, 728, 773, 779, 784, 749, 741, 739, 797, 784, 744, 749, 757] payload_rows_per_dst=[9098, 9244, 9426, 9254, 9360, 9360, 9330, 9176] dup_rows_per_dst=[2912, 2884, 3146, 3036, 3016, 3074, 3044, 2944] combine_rows_per_src=[9098, 9244, 9426, 9254, 9360, 9360, 9330, 9176] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=344008 combine_payload_bytes=266461184 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266805192 ingress_bytes=370270208 link_bytes=370270208 token_bytes=533266376 weight_movement_bytes=103809024 total_nvlink_bytes=637075400 weight_to_token_ratio=0.194666 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393296984e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=64910 dup_rows=23906 dup_rows_remote=20714 dedup_ratio=0.243184 router_tokens_per_expert=[733, 763, 770, 795, 769, 761, 800, 757, 770, 807, 786, 767, 776, 737, 792, 765, 723, 771, 719, 738, 786, 807, 760, 775, 753, 862, 777, 748, 788, 782, 773, 744, 799, 770, 739, 748, 777, 786, 782, 769, 751, 749, 769, 785, 764, 772, 783, 779, 789, 769, 757, 772, 762, 733, 748, 781, 751, 770, 755, 741, 762, 779, 733, 774] payload_rows_per_dst=[9488, 9412, 9176, 9976, 9360, 8736, 9164, 9086] dup_rows_per_dst=[3192, 3132, 2986, 3572, 2986, 2474, 2674, 2890] combine_rows_per_src=[9488, 9412, 9176, 9976, 9360, 8736, 9164, 9086] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74114 payload_rows_remote=64658 dup_rows=24190 dup_rows_remote=21042 dedup_ratio=0.246073 router_tokens_per_expert=[811, 725, 754, 783, 764, 747, 762, 764, 752, 774, 782, 776, 738, 744, 759, 798, 762, 733, 730, 745, 780, 774, 756, 731, 759, 785, 792, 723, 792, 768, 781, 789, 828, 780, 739, 755, 766, 760, 786, 768, 764, 812, 783, 791, 768, 770, 792, 768, 767, 819, 772, 745, 782, 759, 795, 778, 735, 784, 767, 751, 698, 791, 775, 771] payload_rows_per_dst=[9456, 9480, 9214, 9056, 9698, 9026, 8934, 9250] dup_rows_per_dst=[3148, 3230, 3050, 2938, 3352, 2784, 2794, 2894] combine_rows_per_src=[9456, 9480, 9214, 9056, 9698, 9026, 8934, 9250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=344008 combine_payload_bytes=266461184 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266805192 ingress_bytes=370270208 link_bytes=370270208 token_bytes=533266376 weight_movement_bytes=103809024 total_nvlink_bytes=637075400 weight_to_token_ratio=0.194666 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393296984e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74414 payload_rows_remote=65124 dup_rows=23890 dup_rows_remote=20782 dedup_ratio=0.243022 router_tokens_per_expert=[830, 732, 798, 730, 797, 779, 791, 713, 784, 797, 769, 802, 783, 740, 759, 771, 751, 745, 731, 738, 746, 770, 773, 757, 765, 768, 808, 812, 797, 757, 713, 779, 798, 761, 774, 719, 780, 768, 754, 764, 755, 808, 799, 787, 774, 773, 785, 772, 792, 769, 746, 757, 740, 769, 770, 737, 742, 785, 805, 765, 784, 756, 734, 745] payload_rows_per_dst=[9280, 9398, 9186, 9290, 9300, 9400, 9256, 9304] dup_rows_per_dst=[3060, 3012, 2836, 3108, 2936, 3106, 2904, 2928] combine_rows_per_src=[9280, 9398, 9186, 9290, 9300, 9400, 9256, 9304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266952704 dispatch_meta_bytes=345288 combine_payload_bytes=266952704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267297992 ingress_bytes=301555712 link_bytes=301555712 token_bytes=534250696 weight_movement_bytes=34603008 total_nvlink_bytes=568853704 weight_to_token_ratio=0.064769 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134729867e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265510912 dispatch_meta_bytes=342928 combine_payload_bytes=265510912 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265853840 ingress_bytes=334716928 link_bytes=334716928 token_bytes=531364752 weight_movement_bytes=69206016 total_nvlink_bytes=600570768 weight_to_token_ratio=0.130242 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259512853e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74604 payload_rows_remote=65304 dup_rows=23700 dup_rows_remote=20700 dedup_ratio=0.241089 router_tokens_per_expert=[836, 756, 726, 756, 788, 758, 770, 712, 769, 791, 791, 749, 775, 739, 725, 768, 785, 805, 771, 777, 804, 797, 740, 757, 769, 775, 757, 689, 734, 740, 787, 813, 813, 763, 752, 796, 775, 755, 768, 716, 757, 755, 790, 772, 771, 771, 734, 800, 766, 732, 813, 807, 804, 721, 784, 795, 759, 753, 774, 788, 733, 780, 775, 771] payload_rows_per_dst=[9314, 9298, 9434, 9240, 9332, 9300, 9362, 9324] dup_rows_per_dst=[2890, 2916, 3038, 2888, 2944, 3000, 3082, 2942] combine_rows_per_src=[9314, 9298, 9434, 9240, 9332, 9300, 9362, 9324] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265715712 dispatch_meta_bytes=343904 combine_payload_bytes=265715712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266059616 ingress_bytes=300318720 link_bytes=300318720 token_bytes=531775328 weight_movement_bytes=34603008 total_nvlink_bytes=566378336 weight_to_token_ratio=0.065071 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130075166e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73924 payload_rows_remote=64738 dup_rows=24380 dup_rows_remote=21362 dedup_ratio=0.248006 router_tokens_per_expert=[721, 790, 775, 838, 759, 764, 789, 726, 724, 773, 780, 720, 758, 795, 744, 752, 810, 792, 771, 771, 754, 806, 768, 759, 755, 781, 751, 742, 790, 769, 753, 759, 776, 775, 778, 739, 751, 746, 724, 813, 712, 757, 811, 748, 773, 777, 795, 826, 783, 752, 777, 730, 736, 816, 763, 822, 791, 780, 735, 792, 721, 711, 803, 800] payload_rows_per_dst=[9240, 9250, 9274, 9146, 9186, 9292, 9318, 9218] dup_rows_per_dst=[3084, 2842, 3188, 3054, 3018, 3106, 3040, 3048] combine_rows_per_src=[9240, 9250, 9274, 9146, 9186, 9292, 9318, 9218] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=264839168 dispatch_meta_bytes=342800 combine_payload_bytes=264839168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265181968 ingress_bytes=299442176 link_bytes=299442176 token_bytes=530021136 weight_movement_bytes=34603008 total_nvlink_bytes=564624144 weight_to_token_ratio=0.065286 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126776801e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74514 payload_rows_remote=65274 dup_rows=23790 dup_rows_remote=20824 dedup_ratio=0.242004 router_tokens_per_expert=[776, 766, 776, 784, 702, 775, 796, 730, 770, 758, 767, 769, 775, 748, 790, 807, 756, 769, 751, 777, 764, 830, 766, 806, 712, 755, 771, 733, 766, 744, 743, 737, 741, 784, 727, 734, 815, 783, 803, 798, 738, 802, 774, 790, 793, 806, 741, 725, 801, 774, 779, 681, 736, 822, 755, 755, 769, 806, 806, 786, 774, 781, 757, 747] payload_rows_per_dst=[9278, 9372, 9412, 9104, 9300, 9382, 9240, 9426] dup_rows_per_dst=[2932, 2996, 3026, 2818, 3070, 2956, 2966, 3026] combine_rows_per_src=[9278, 9372, 9412, 9104, 9300, 9382, 9240, 9426] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265510912 dispatch_meta_bytes=342928 combine_payload_bytes=265510912 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265853840 ingress_bytes=334716928 link_bytes=334716928 token_bytes=531364752 weight_movement_bytes=69206016 total_nvlink_bytes=600570768 weight_to_token_ratio=0.130242 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259512853e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265871360 dispatch_meta_bytes=342496 combine_payload_bytes=265871360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266213856 ingress_bytes=300474368 link_bytes=300474368 token_bytes=532085216 weight_movement_bytes=34603008 total_nvlink_bytes=566688224 weight_to_token_ratio=0.065033 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130660857e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266747904 dispatch_meta_bytes=343624 combine_payload_bytes=266747904 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267091528 ingress_bytes=370556928 link_bytes=370556928 token_bytes=533839432 weight_movement_bytes=103809024 total_nvlink_bytes=637648456 weight_to_token_ratio=0.194457 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394375888e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267485184 dispatch_meta_bytes=344016 combine_payload_bytes=267485184 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267829200 ingress_bytes=371294208 link_bytes=371294208 token_bytes=535314384 weight_movement_bytes=103809024 total_nvlink_bytes=639123408 weight_to_token_ratio=0.193922 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397150213e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265166848 dispatch_meta_bytes=344400 combine_payload_bytes=265166848 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265511248 ingress_bytes=334372864 link_bytes=334372864 token_bytes=530678096 weight_movement_bytes=69206016 total_nvlink_bytes=599884112 weight_to_token_ratio=0.130411 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258218168e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74450 payload_rows_remote=65118 dup_rows=23854 dup_rows_remote=20932 dedup_ratio=0.242655 router_tokens_per_expert=[796, 731, 774, 759, 754, 771, 770, 753, 765, 779, 785, 759, 756, 750, 772, 761, 788, 750, 780, 772, 759, 742, 731, 772, 798, 794, 750, 824, 705, 753, 756, 775, 768, 773, 772, 780, 758, 766, 757, 764, 742, 792, 810, 738, 756, 790, 786, 797, 775, 737, 766, 832, 784, 729, 778, 811, 746, 767, 773, 789, 731, 752, 787, 762] payload_rows_per_dst=[9280, 9332, 9244, 9314, 9234, 9362, 9386, 9298] dup_rows_per_dst=[2936, 2922, 2944, 2996, 3042, 3060, 3038, 2916] combine_rows_per_src=[9280, 9332, 9244, 9314, 9234, 9362, 9386, 9298] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=267362304 dispatch_meta_bytes=344392 combine_payload_bytes=267362304 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267706696 ingress_bytes=336568320 link_bytes=336568320 token_bytes=535069000 weight_movement_bytes=69206016 total_nvlink_bytes=604275016 weight_to_token_ratio=0.129340 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266479492e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74456 payload_rows_remote=65186 dup_rows=23848 dup_rows_remote=20752 dedup_ratio=0.242594 router_tokens_per_expert=[762, 772, 761, 805, 789, 797, 752, 751, 797, 755, 761, 760, 744, 758, 760, 740, 766, 747, 808, 715, 751, 750, 773, 769, 798, 760, 783, 772, 781, 756, 769, 760, 744, 750, 723, 753, 771, 761, 753, 753, 803, 723, 796, 799, 804, 780, 757, 799, 740, 786, 806, 784, 795, 759, 794, 770, 753, 780, 759, 724, 754, 755, 786, 816] payload_rows_per_dst=[9334, 9322, 9226, 9280, 9120, 9456, 9448, 9270] dup_rows_per_dst=[3044, 2828, 2932, 2938, 2896, 3000, 3114, 3096] combine_rows_per_src=[9334, 9322, 9226, 9280, 9120, 9456, 9448, 9270] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74418 payload_rows_remote=65288 dup_rows=23886 dup_rows_remote=20984 dedup_ratio=0.242981 router_tokens_per_expert=[741, 764, 750, 772, 783, 761, 781, 772, 736, 764, 775, 765, 777, 772, 770, 768, 718, 753, 791, 757, 794, 764, 757, 804, 803, 801, 741, 783, 784, 766, 759, 786, 706, 772, 740, 738, 749, 783, 778, 750, 740, 778, 755, 719, 800, 825, 813, 759, 801, 794, 736, 789, 769, 771, 774, 794, 746, 756, 758, 742, 791, 757, 798, 759] payload_rows_per_dst=[9322, 9294, 9340, 9386, 9130, 9358, 9390, 9198] dup_rows_per_dst=[2926, 2960, 2936, 3060, 2902, 3020, 3066, 3016] combine_rows_per_src=[9322, 9294, 9340, 9386, 9130, 9358, 9390, 9198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74102 payload_rows_remote=64652 dup_rows=24202 dup_rows_remote=21020 dedup_ratio=0.246195 router_tokens_per_expert=[759, 739, 765, 754, 775, 751, 744, 782, 723, 729, 743, 745, 775, 835, 743, 757, 807, 789, 759, 815, 776, 801, 779, 790, 806, 766, 774, 782, 732, 784, 730, 791, 764, 764, 739, 789, 761, 764, 759, 776, 712, 776, 754, 767, 784, 786, 796, 746, 762, 783, 759, 791, 779, 736, 792, 752, 772, 747, 814, 783, 779, 743, 784, 739] payload_rows_per_dst=[9212, 9194, 9450, 9276, 9230, 9186, 9344, 9210] dup_rows_per_dst=[2926, 2906, 3182, 3054, 3002, 3056, 2964, 3112] combine_rows_per_src=[9212, 9194, 9450, 9276, 9230, 9186, 9344, 9210] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74414 payload_rows_remote=65124 dup_rows=23890 dup_rows_remote=20782 dedup_ratio=0.243022 router_tokens_per_expert=[830, 732, 798, 730, 797, 779, 791, 713, 784, 797, 769, 802, 783, 740, 759, 771, 751, 745, 731, 738, 746, 770, 773, 757, 765, 768, 808, 812, 797, 757, 713, 779, 798, 761, 774, 719, 780, 768, 754, 764, 755, 808, 799, 787, 774, 773, 785, 772, 792, 769, 746, 757, 740, 769, 770, 737, 742, 785, 805, 765, 784, 756, 734, 745] payload_rows_per_dst=[9280, 9398, 9186, 9290, 9300, 9400, 9256, 9304] dup_rows_per_dst=[3060, 3012, 2836, 3108, 2936, 3106, 2904, 2928] combine_rows_per_src=[9280, 9398, 9186, 9290, 9300, 9400, 9256, 9304] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266723328 dispatch_meta_bytes=344200 combine_payload_bytes=266723328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267067528 ingress_bytes=301326336 link_bytes=301326336 token_bytes=533790856 weight_movement_bytes=34603008 total_nvlink_bytes=568393864 weight_to_token_ratio=0.064825 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133866744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74102 payload_rows_remote=64652 dup_rows=24202 dup_rows_remote=21020 dedup_ratio=0.246195 router_tokens_per_expert=[759, 739, 765, 754, 775, 751, 744, 782, 723, 729, 743, 745, 775, 835, 743, 757, 807, 789, 759, 815, 776, 801, 779, 790, 806, 766, 774, 782, 732, 784, 730, 791, 764, 764, 739, 789, 761, 764, 759, 776, 712, 776, 754, 767, 784, 786, 796, 746, 762, 783, 759, 791, 779, 736, 792, 752, 772, 747, 814, 783, 779, 743, 784, 739] payload_rows_per_dst=[9212, 9194, 9450, 9276, 9230, 9186, 9344, 9210] dup_rows_per_dst=[2926, 2906, 3182, 3054, 3002, 3056, 2964, 3112] combine_rows_per_src=[9212, 9194, 9450, 9276, 9230, 9186, 9344, 9210] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267001856 dispatch_meta_bytes=343752 combine_payload_bytes=267001856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267345608 ingress_bytes=301604864 link_bytes=301604864 token_bytes=534347464 weight_movement_bytes=34603008 total_nvlink_bytes=568950472 weight_to_token_ratio=0.064758 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134914822e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74114 payload_rows_remote=64658 dup_rows=24190 dup_rows_remote=21042 dedup_ratio=0.246073 router_tokens_per_expert=[811, 725, 754, 783, 764, 747, 762, 764, 752, 774, 782, 776, 738, 744, 759, 798, 762, 733, 730, 745, 780, 774, 756, 731, 759, 785, 792, 723, 792, 768, 781, 789, 828, 780, 739, 755, 766, 760, 786, 768, 764, 812, 783, 791, 768, 770, 792, 768, 767, 819, 772, 745, 782, 759, 795, 778, 735, 784, 767, 751, 698, 791, 775, 771] payload_rows_per_dst=[9456, 9480, 9214, 9056, 9698, 9026, 8934, 9250] dup_rows_per_dst=[3148, 3230, 3050, 2938, 3352, 2784, 2794, 2894] combine_rows_per_src=[9456, 9480, 9214, 9056, 9698, 9026, 8934, 9250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=64830 dup_rows=24096 dup_rows_remote=20920 dedup_ratio=0.245117 router_tokens_per_expert=[785, 824, 713, 776, 796, 770, 821, 750, 750, 717, 799, 799, 743, 743, 762, 787, 778, 782, 832, 754, 741, 772, 790, 771, 781, 739, 755, 779, 792, 745, 735, 728, 715, 811, 754, 777, 754, 731, 754, 796, 776, 794, 736, 757, 784, 749, 793, 754, 789, 803, 764, 768, 782, 773, 817, 762, 775, 739, 741, 726, 754, 759, 772, 784] payload_rows_per_dst=[9378, 9146, 9488, 9082, 9220, 9226, 9470, 9198] dup_rows_per_dst=[3176, 2970, 3228, 2896, 2964, 2954, 3006, 2902] combine_rows_per_src=[9378, 9146, 9488, 9082, 9220, 9226, 9470, 9198] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267001856 dispatch_meta_bytes=343752 combine_payload_bytes=267001856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267345608 ingress_bytes=301604864 link_bytes=301604864 token_bytes=534347464 weight_movement_bytes=34603008 total_nvlink_bytes=568950472 weight_to_token_ratio=0.064758 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134914822e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=65128 dup_rows=24002 dup_rows_remote=21126 dedup_ratio=0.244161 router_tokens_per_expert=[801, 773, 799, 729, 796, 792, 818, 759, 803, 774, 746, 750, 761, 791, 772, 816, 735, 792, 773, 761, 794, 795, 770, 780, 778, 771, 772, 723, 765, 720, 748, 748, 761, 748, 754, 754, 779, 769, 726, 784, 736, 812, 805, 825, 777, 776, 748, 761, 713, 749, 795, 742, 763, 752, 812, 754, 758, 723, 805, 751, 730, 758, 793, 734] payload_rows_per_dst=[9402, 9290, 9380, 9174, 9138, 9394, 9254, 9270] dup_rows_per_dst=[3132, 3136, 3020, 2876, 3012, 3086, 2906, 2834] combine_rows_per_src=[9402, 9290, 9380, 9174, 9138, 9394, 9254, 9270] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=267419648 dispatch_meta_bytes=345088 combine_payload_bytes=267419648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267764736 ingress_bytes=302022656 link_bytes=302022656 token_bytes=535184384 weight_movement_bytes=34603008 total_nvlink_bytes=569787392 weight_to_token_ratio=0.064656 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136486940e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73818 payload_rows_remote=64530 dup_rows=24486 dup_rows_remote=21518 dedup_ratio=0.249084 router_tokens_per_expert=[741, 727, 792, 779, 754, 777, 777, 737, 728, 820, 786, 792, 723, 753, 782, 786, 764, 783, 754, 815, 771, 745, 777, 793, 709, 751, 775, 785, 741, 794, 754, 790, 794, 794, 751, 803, 740, 751, 753, 752, 783, 730, 783, 755, 756, 776, 799, 746, 779, 779, 781, 790, 771, 778, 785, 785, 729, 740, 740, 755, 826, 757, 755, 781] payload_rows_per_dst=[9130, 9284, 9204, 9190, 9226, 9288, 9384, 9112] dup_rows_per_dst=[3038, 3056, 3200, 3008, 3050, 2968, 3112, 3054] combine_rows_per_src=[9130, 9284, 9204, 9190, 9226, 9288, 9384, 9112] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=65174 dup_rows=24028 dup_rows_remote=21148 dedup_ratio=0.244425 router_tokens_per_expert=[788, 753, 765, 796, 731, 805, 784, 743, 718, 818, 728, 750, 778, 742, 752, 745, 747, 786, 789, 745, 772, 779, 793, 753, 764, 795, 783, 758, 800, 789, 783, 786, 739, 751, 728, 772, 747, 719, 762, 773, 794, 729, 768, 802, 746, 785, 761, 798, 762, 748, 743, 782, 866, 801, 743, 753, 750, 794, 776, 816, 762, 736, 760, 768] payload_rows_per_dst=[9310, 9138, 9258, 9416, 9102, 9376, 9346, 9330] dup_rows_per_dst=[3020, 2924, 3070, 3100, 2880, 2990, 3050, 2994] combine_rows_per_src=[9310, 9138, 9258, 9416, 9102, 9376, 9346, 9330] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266747904 dispatch_meta_bytes=343624 combine_payload_bytes=266747904 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267091528 ingress_bytes=370556928 link_bytes=370556928 token_bytes=533839432 weight_movement_bytes=103809024 total_nvlink_bytes=637648456 weight_to_token_ratio=0.194457 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394375888e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=264814592 dispatch_meta_bytes=342688 combine_payload_bytes=264814592 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265157280 ingress_bytes=299417600 link_bytes=299417600 token_bytes=529971872 weight_movement_bytes=34603008 total_nvlink_bytes=564574880 weight_to_token_ratio=0.065292 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126684324e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74146 payload_rows_remote=64872 dup_rows=24158 dup_rows_remote=21150 dedup_ratio=0.245748 router_tokens_per_expert=[768, 764, 749, 727, 774, 737, 752, 776, 810, 816, 760, 783, 741, 775, 725, 792, 725, 786, 766, 792, 738, 792, 738, 783, 795, 747, 781, 752, 783, 782, 796, 769, 780, 751, 776, 770, 767, 742, 775, 757, 781, 792, 775, 821, 673, 745, 770, 744, 797, 755, 742, 777, 770, 768, 778, 754, 816, 750, 726, 750, 788, 787, 790, 811] payload_rows_per_dst=[9116, 9380, 9246, 9388, 9184, 9226, 9274, 9332] dup_rows_per_dst=[2978, 3024, 2994, 3022, 3052, 2976, 3008, 3104] combine_rows_per_src=[9116, 9380, 9246, 9388, 9184, 9226, 9274, 9332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265543680 dispatch_meta_bytes=343000 combine_payload_bytes=265543680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265886680 ingress_bytes=300146688 link_bytes=300146688 token_bytes=531430360 weight_movement_bytes=34603008 total_nvlink_bytes=566033368 weight_to_token_ratio=0.065113 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129427823e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=264839168 dispatch_meta_bytes=342800 combine_payload_bytes=264839168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265181968 ingress_bytes=299442176 link_bytes=299442176 token_bytes=530021136 weight_movement_bytes=34603008 total_nvlink_bytes=564624144 weight_to_token_ratio=0.065286 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126776801e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266764288 dispatch_meta_bytes=345016 combine_payload_bytes=266764288 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267109304 ingress_bytes=335970304 link_bytes=335970304 token_bytes=533873592 weight_movement_bytes=69206016 total_nvlink_bytes=603079608 weight_to_token_ratio=0.129630 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264229206e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=264314880 dispatch_meta_bytes=344192 combine_payload_bytes=264314880 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=264659072 ingress_bytes=333520896 link_bytes=333520896 token_bytes=528973952 weight_movement_bytes=69206016 total_nvlink_bytes=598179968 weight_to_token_ratio=0.130831 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.255012281e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74224 payload_rows_remote=64980 dup_rows=24080 dup_rows_remote=21100 dedup_ratio=0.244954 router_tokens_per_expert=[754, 727, 715, 791, 744, 779, 758, 743, 801, 757, 779, 767, 775, 745, 754, 734, 759, 796, 770, 783, 791, 795, 783, 794, 776, 787, 751, 770, 772, 791, 767, 778, 828, 742, 774, 774, 751, 791, 725, 740, 760, 757, 806, 730, 791, 740, 776, 752, 711, 786, 822, 750, 776, 767, 815, 790, 786, 778, 767, 774, 744, 772, 736, 755] payload_rows_per_dst=[9094, 9244, 9530, 9320, 9208, 9278, 9328, 9222] dup_rows_per_dst=[2928, 2980, 3012, 3064, 3042, 2946, 3106, 3002] combine_rows_per_src=[9094, 9244, 9530, 9320, 9208, 9278, 9328, 9222] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266952704 dispatch_meta_bytes=345288 combine_payload_bytes=266952704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267297992 ingress_bytes=301555712 link_bytes=301555712 token_bytes=534250696 weight_movement_bytes=34603008 total_nvlink_bytes=568853704 weight_to_token_ratio=0.064769 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134729867e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74154 payload_rows_remote=64432 dup_rows=24150 dup_rows_remote=20778 dedup_ratio=0.245667 router_tokens_per_expert=[745, 756, 698, 796, 747, 831, 737, 710, 805, 725, 733, 762, 782, 855, 802, 757, 786, 748, 801, 716, 776, 779, 764, 769, 779, 783, 799, 785, 797, 778, 755, 797, 729, 780, 791, 770, 754, 795, 760, 725, 761, 771, 786, 750, 777, 793, 750, 741, 805, 783, 754, 803, 743, 747, 729, 771, 762, 773, 774, 731, 807, 744, 779, 761] payload_rows_per_dst=[9114, 9354, 8984, 9360, 9458, 9298, 8864, 9722] dup_rows_per_dst=[2926, 3072, 2686, 3186, 3358, 2960, 2590, 3372] combine_rows_per_src=[9114, 9354, 8984, 9360, 9458, 9298, 8864, 9722] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265715712 dispatch_meta_bytes=344088 combine_payload_bytes=265715712 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266059800 ingress_bytes=369524736 link_bytes=369524736 token_bytes=531775512 weight_movement_bytes=103809024 total_nvlink_bytes=635584536 weight_to_token_ratio=0.195212 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390491832e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74154 payload_rows_remote=64432 dup_rows=24150 dup_rows_remote=20778 dedup_ratio=0.245667 router_tokens_per_expert=[745, 756, 698, 796, 747, 831, 737, 710, 805, 725, 733, 762, 782, 855, 802, 757, 786, 748, 801, 716, 776, 779, 764, 769, 779, 783, 799, 785, 797, 778, 755, 797, 729, 780, 791, 770, 754, 795, 760, 725, 761, 771, 786, 750, 777, 793, 750, 741, 805, 783, 754, 803, 743, 747, 729, 771, 762, 773, 774, 731, 807, 744, 779, 761] payload_rows_per_dst=[9114, 9354, 8984, 9360, 9458, 9298, 8864, 9722] dup_rows_per_dst=[2926, 3072, 2686, 3186, 3358, 2960, 2590, 3372] combine_rows_per_src=[9114, 9354, 8984, 9360, 9458, 9298, 8864, 9722] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74446 payload_rows_remote=65052 dup_rows=23858 dup_rows_remote=20868 dedup_ratio=0.242696 router_tokens_per_expert=[761, 783, 735, 766, 761, 814, 843, 790, 737, 782, 768, 780, 791, 793, 814, 781, 780, 734, 744, 790, 775, 799, 800, 764, 739, 756, 750, 743, 711, 745, 809, 818, 790, 802, 749, 761, 760, 811, 776, 743, 775, 752, 797, 796, 743, 745, 719, 786, 723, 758, 761, 799, 781, 723, 746, 741, 733, 768, 782, 764, 797, 728, 755, 732] payload_rows_per_dst=[9376, 9438, 9362, 9222, 9394, 9306, 9188, 9160] dup_rows_per_dst=[3130, 3054, 3010, 2920, 2990, 2920, 2876, 2958] combine_rows_per_src=[9376, 9438, 9362, 9222, 9394, 9306, 9188, 9160] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=65104 dup_rows=24098 dup_rows_remote=21214 dedup_ratio=0.245138 router_tokens_per_expert=[772, 805, 764, 750, 768, 764, 765, 792, 815, 788, 753, 780, 745, 744, 756, 761, 752, 717, 724, 773, 781, 771, 734, 741, 749, 816, 769, 768, 739, 729, 796, 809, 763, 762, 793, 776, 787, 755, 786, 765, 753, 745, 786, 806, 775, 751, 761, 735, 809, 753, 807, 780, 766, 784, 757, 832, 794, 760, 781, 705, 773, 730, 729, 803] payload_rows_per_dst=[9360, 9194, 9102, 9258, 9292, 9282, 9462, 9256] dup_rows_per_dst=[3000, 3090, 2884, 3092, 3082, 2942, 3114, 2894] combine_rows_per_src=[9360, 9194, 9102, 9258, 9292, 9282, 9462, 9256] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=64956 dup_rows=24072 dup_rows_remote=21046 dedup_ratio=0.244873 router_tokens_per_expert=[773, 768, 780, 733, 752, 775, 777, 767, 720, 751, 769, 822, 751, 790, 759, 738, 784, 767, 793, 717, 838, 770, 810, 791, 791, 786, 775, 738, 791, 744, 769, 704, 767, 742, 751, 807, 749, 721, 791, 765, 752, 778, 773, 782, 740, 740, 779, 838, 782, 781, 725, 778, 743, 737, 731, 789, 739, 779, 823, 757, 756, 805, 758, 801] payload_rows_per_dst=[9276, 9266, 9406, 9204, 9310, 9152, 9238, 9380] dup_rows_per_dst=[3026, 3026, 3104, 2992, 3138, 2836, 2894, 3056] combine_rows_per_src=[9276, 9266, 9406, 9204, 9310, 9152, 9238, 9380] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266158080 dispatch_meta_bytes=344320 combine_payload_bytes=266158080 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266502400 ingress_bytes=335364096 link_bytes=335364096 token_bytes=532660480 weight_movement_bytes=69206016 total_nvlink_bytes=601866496 weight_to_token_ratio=0.129925 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261948094e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=65128 dup_rows=24002 dup_rows_remote=21126 dedup_ratio=0.244161 router_tokens_per_expert=[801, 773, 799, 729, 796, 792, 818, 759, 803, 774, 746, 750, 761, 791, 772, 816, 735, 792, 773, 761, 794, 795, 770, 780, 778, 771, 772, 723, 765, 720, 748, 748, 761, 748, 754, 754, 779, 769, 726, 784, 736, 812, 805, 825, 777, 776, 748, 761, 713, 749, 795, 742, 763, 752, 812, 754, 758, 723, 805, 751, 730, 758, 793, 734] payload_rows_per_dst=[9402, 9290, 9380, 9174, 9138, 9394, 9254, 9270] dup_rows_per_dst=[3132, 3136, 3020, 2876, 3012, 3086, 2906, 2834] combine_rows_per_src=[9402, 9290, 9380, 9174, 9138, 9394, 9254, 9270] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=65104 dup_rows=24098 dup_rows_remote=21214 dedup_ratio=0.245138 router_tokens_per_expert=[772, 805, 764, 750, 768, 764, 765, 792, 815, 788, 753, 780, 745, 744, 756, 761, 752, 717, 724, 773, 781, 771, 734, 741, 749, 816, 769, 768, 739, 729, 796, 809, 763, 762, 793, 776, 787, 755, 786, 765, 753, 745, 786, 806, 775, 751, 761, 735, 809, 753, 807, 780, 766, 784, 757, 832, 794, 760, 781, 705, 773, 730, 729, 803] payload_rows_per_dst=[9360, 9194, 9102, 9258, 9292, 9282, 9462, 9256] dup_rows_per_dst=[3000, 3090, 2884, 3092, 3082, 2942, 3114, 2894] combine_rows_per_src=[9360, 9194, 9102, 9258, 9292, 9282, 9462, 9256] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=263913472 dispatch_meta_bytes=340840 combine_payload_bytes=263913472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264254312 ingress_bytes=298516480 link_bytes=298516480 token_bytes=528167784 weight_movement_bytes=34603008 total_nvlink_bytes=562770792 weight_to_token_ratio=0.065515 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123293482e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=64830 dup_rows=24096 dup_rows_remote=20920 dedup_ratio=0.245117 router_tokens_per_expert=[785, 824, 713, 776, 796, 770, 821, 750, 750, 717, 799, 799, 743, 743, 762, 787, 778, 782, 832, 754, 741, 772, 790, 771, 781, 739, 755, 779, 792, 745, 735, 728, 715, 811, 754, 777, 754, 731, 754, 796, 776, 794, 736, 757, 784, 749, 793, 754, 789, 803, 764, 768, 782, 773, 817, 762, 775, 739, 741, 726, 754, 759, 772, 784] payload_rows_per_dst=[9378, 9146, 9488, 9082, 9220, 9226, 9470, 9198] dup_rows_per_dst=[3176, 2970, 3228, 2896, 2964, 2954, 3006, 2902] combine_rows_per_src=[9378, 9146, 9488, 9082, 9220, 9226, 9470, 9198] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74144 payload_rows_remote=64886 dup_rows=24160 dup_rows_remote=21078 dedup_ratio=0.245768 router_tokens_per_expert=[777, 768, 809, 775, 738, 728, 810, 769, 728, 761, 755, 745, 764, 762, 751, 761, 742, 789, 730, 750, 790, 754, 796, 756, 798, 753, 784, 780, 839, 744, 731, 724, 747, 778, 729, 797, 773, 750, 758, 809, 772, 749, 770, 768, 798, 759, 774, 780, 809, 751, 780, 810, 748, 808, 808, 786, 761, 784, 785, 728, 772, 746, 737, 767] payload_rows_per_dst=[9324, 9118, 9260, 9302, 9262, 9258, 9468, 9152] dup_rows_per_dst=[3024, 2936, 2954, 3004, 3020, 3082, 3132, 3008] combine_rows_per_src=[9324, 9118, 9260, 9302, 9262, 9258, 9468, 9152] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266764288 dispatch_meta_bytes=345016 combine_payload_bytes=266764288 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267109304 ingress_bytes=335970304 link_bytes=335970304 token_bytes=533873592 weight_movement_bytes=69206016 total_nvlink_bytes=603079608 weight_to_token_ratio=0.129630 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264229206e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=263913472 dispatch_meta_bytes=340840 combine_payload_bytes=263913472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264254312 ingress_bytes=298516480 link_bytes=298516480 token_bytes=528167784 weight_movement_bytes=34603008 total_nvlink_bytes=562770792 weight_to_token_ratio=0.065515 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123293482e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266452992 dispatch_meta_bytes=343680 combine_payload_bytes=266452992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266796672 ingress_bytes=301056000 link_bytes=301056000 token_bytes=533249664 weight_movement_bytes=34603008 total_nvlink_bytes=567852672 weight_to_token_ratio=0.064891 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132849491e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74492 payload_rows_remote=65172 dup_rows=23812 dup_rows_remote=20842 dedup_ratio=0.242228 router_tokens_per_expert=[779, 770, 782, 793, 790, 740, 768, 759, 718, 800, 761, 752, 784, 751, 703, 817, 755, 749, 773, 748, 768, 761, 765, 804, 760, 782, 769, 786, 810, 758, 734, 746, 761, 839, 738, 752, 742, 774, 763, 768, 743, 771, 770, 787, 781, 737, 780, 780, 806, 713, 738, 751, 814, 775, 784, 740, 746, 752, 761, 790, 768, 832, 798, 763] payload_rows_per_dst=[9340, 9276, 9204, 9320, 9244, 9384, 9342, 9382] dup_rows_per_dst=[3022, 2896, 3042, 2970, 3030, 2914, 2900, 3038] combine_rows_per_src=[9340, 9276, 9204, 9320, 9244, 9384, 9342, 9382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266665984 dispatch_meta_bytes=345272 combine_payload_bytes=266665984 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267011256 ingress_bytes=335872000 link_bytes=335872000 token_bytes=533677240 weight_movement_bytes=69206016 total_nvlink_bytes=602883256 weight_to_token_ratio=0.129678 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263859296e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74418 payload_rows_remote=65288 dup_rows=23886 dup_rows_remote=20984 dedup_ratio=0.242981 router_tokens_per_expert=[741, 764, 750, 772, 783, 761, 781, 772, 736, 764, 775, 765, 777, 772, 770, 768, 718, 753, 791, 757, 794, 764, 757, 804, 803, 801, 741, 783, 784, 766, 759, 786, 706, 772, 740, 738, 749, 783, 778, 750, 740, 778, 755, 719, 800, 825, 813, 759, 801, 794, 736, 789, 769, 771, 774, 794, 746, 756, 758, 742, 791, 757, 798, 759] payload_rows_per_dst=[9322, 9294, 9340, 9386, 9130, 9358, 9390, 9198] dup_rows_per_dst=[2926, 2960, 2936, 3060, 2902, 3020, 3066, 3016] combine_rows_per_src=[9322, 9294, 9340, 9386, 9130, 9358, 9390, 9198] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74144 payload_rows_remote=64886 dup_rows=24160 dup_rows_remote=21078 dedup_ratio=0.245768 router_tokens_per_expert=[777, 768, 809, 775, 738, 728, 810, 769, 728, 761, 755, 745, 764, 762, 751, 761, 742, 789, 730, 750, 790, 754, 796, 756, 798, 753, 784, 780, 839, 744, 731, 724, 747, 778, 729, 797, 773, 750, 758, 809, 772, 749, 770, 768, 798, 759, 774, 780, 809, 751, 780, 810, 748, 808, 808, 786, 761, 784, 785, 728, 772, 746, 737, 767] payload_rows_per_dst=[9324, 9118, 9260, 9302, 9262, 9258, 9468, 9152] dup_rows_per_dst=[3024, 2936, 2954, 3004, 3020, 3082, 3132, 3008] combine_rows_per_src=[9324, 9118, 9260, 9302, 9262, 9258, 9468, 9152] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=65132 dup_rows=23962 dup_rows_remote=20920 dedup_ratio=0.243754 router_tokens_per_expert=[764, 773, 783, 826, 724, 751, 791, 777, 750, 761, 775, 756, 784, 742, 772, 777, 739, 792, 776, 756, 756, 760, 790, 786, 772, 771, 756, 780, 812, 767, 751, 767, 816, 775, 776, 763, 736, 755, 748, 742, 789, 759, 740, 762, 828, 740, 783, 751, 784, 732, 770, 747, 786, 796, 739, 772, 758, 765, 758, 766, 774, 796, 744, 765] payload_rows_per_dst=[9356, 9218, 9364, 9320, 9236, 9360, 9210, 9278] dup_rows_per_dst=[3022, 3016, 2946, 3032, 2986, 2944, 3042, 2974] combine_rows_per_src=[9356, 9218, 9364, 9320, 9236, 9360, 9210, 9278] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265543680 dispatch_meta_bytes=343000 combine_payload_bytes=265543680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265886680 ingress_bytes=300146688 link_bytes=300146688 token_bytes=531430360 weight_movement_bytes=34603008 total_nvlink_bytes=566033368 weight_to_token_ratio=0.065113 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129427823e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=343856 combine_payload_bytes=265773056 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266116912 ingress_bytes=404185088 link_bytes=404185088 token_bytes=531889968 weight_movement_bytes=138412032 total_nvlink_bytes=670302000 weight_to_token_ratio=0.260227 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520915947e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=344008 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266403784 ingress_bytes=300662784 link_bytes=300662784 token_bytes=532463560 weight_movement_bytes=34603008 total_nvlink_bytes=567066568 weight_to_token_ratio=0.064987 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131369851e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12250644616 prefetch_bytes=761266176 reduce_bytes=346030080 weight_movement_bytes=1107296256 egress_bytes=6129279112 ingress_bytes=7228661760 link_bytes=7228661760 total_nvlink_bytes=13357940872 weight_to_token_ratio=0.090387 prefetch_share_of_ingress=0.105312 dedup_ratio=0.244650 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=10 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029348 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.182646619e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266944512 dispatch_meta_bytes=344056 combine_payload_bytes=266944512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267288568 ingress_bytes=301547520 link_bytes=301547520 token_bytes=534233080 weight_movement_bytes=34603008 total_nvlink_bytes=568836088 weight_to_token_ratio=0.064771 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134699041e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=343856 combine_payload_bytes=265773056 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266116912 ingress_bytes=404185088 link_bytes=404185088 token_bytes=531889968 weight_movement_bytes=138412032 total_nvlink_bytes=670302000 weight_to_token_ratio=0.260227 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520915947e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12233467688 prefetch_bytes=692060160 reduce_bytes=588251136 weight_movement_bytes=1280311296 egress_bytes=6120679208 ingress_bytes=7393099776 link_bytes=7393099776 total_nvlink_bytes=13513778984 weight_to_token_ratio=0.104656 prefetch_share_of_ingress=0.093609 dedup_ratio=0.244264 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=17 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029130 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.209549533e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266780672 dispatch_meta_bytes=344208 combine_payload_bytes=266780672 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267124880 ingress_bytes=370589696 link_bytes=370589696 token_bytes=533905552 weight_movement_bytes=103809024 total_nvlink_bytes=637714576 weight_to_token_ratio=0.194433 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394499191e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74488 payload_rows_remote=65186 dup_rows=23816 dup_rows_remote=20840 dedup_ratio=0.242269 router_tokens_per_expert=[767, 768, 775, 725, 743, 794, 787, 783, 755, 745, 777, 786, 708, 807, 775, 786, 747, 793, 766, 732, 771, 731, 740, 825, 782, 753, 782, 763, 785, 790, 780, 757, 744, 750, 716, 829, 772, 741, 793, 795, 787, 792, 760, 787, 756, 737, 811, 777, 749, 787, 753, 780, 771, 778, 727, 758, 727, 783, 779, 753, 785, 817, 709, 771] payload_rows_per_dst=[9270, 9302, 9242, 9374, 9360, 9368, 9270, 9302] dup_rows_per_dst=[3014, 2976, 2968, 3010, 2920, 3046, 2936, 2946] combine_rows_per_src=[9270, 9302, 9242, 9374, 9360, 9368, 9270, 9302] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74174 payload_rows_remote=64708 dup_rows=24130 dup_rows_remote=20982 dedup_ratio=0.245463 router_tokens_per_expert=[753, 785, 754, 772, 739, 762, 746, 742, 702, 761, 788, 753, 781, 751, 766, 822, 782, 787, 788, 825, 790, 798, 745, 737, 741, 778, 744, 723, 780, 760, 762, 774, 760, 762, 785, 723, 743, 760, 801, 772, 787, 796, 777, 759, 756, 769, 818, 744, 811, 748, 741, 757, 775, 739, 841, 760, 742, 787, 766, 772, 760, 800, 792, 758] payload_rows_per_dst=[9212, 9180, 9184, 9298, 9172, 9264, 9398, 9466] dup_rows_per_dst=[2894, 3068, 2886, 3000, 3040, 2946, 3148, 3148] combine_rows_per_src=[9212, 9180, 9184, 9298, 9172, 9264, 9398, 9466] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=267419648 dispatch_meta_bytes=345088 combine_payload_bytes=267419648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267764736 ingress_bytes=302022656 link_bytes=302022656 token_bytes=535184384 weight_movement_bytes=34603008 total_nvlink_bytes=569787392 weight_to_token_ratio=0.064656 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136486940e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12246151200 prefetch_bytes=588251136 reduce_bytes=1038090240 weight_movement_bytes=1626341376 egress_bytes=6127030304 ingress_bytes=7745462272 link_bytes=7745462272 total_nvlink_bytes=13872492576 weight_to_token_ratio=0.132804 prefetch_share_of_ingress=0.075948 dedup_ratio=0.244592 prefetch_slots=34 unused_prefetch_slots=334 slot_utilization=0.092391 reduce_slots=30 remote_demand=17 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=28448 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.267197868e-04 prefetch_bytes_per_gemm_flop=9.624094203e-06 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266780672 dispatch_meta_bytes=344208 combine_payload_bytes=266780672 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267124880 ingress_bytes=370589696 link_bytes=370589696 token_bytes=533905552 weight_movement_bytes=103809024 total_nvlink_bytes=637714576 weight_to_token_ratio=0.194433 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394499191e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74150 payload_rows_remote=64666 dup_rows=24154 dup_rows_remote=21130 dedup_ratio=0.245707 router_tokens_per_expert=[782, 808, 818, 755, 783, 744, 693, 751, 724, 812, 776, 740, 794, 859, 769, 796, 758, 764, 810, 785, 769, 736, 822, 810, 783, 791, 728, 780, 749, 778, 757, 784, 800, 751, 759, 777, 755, 744, 771, 737, 727, 756, 820, 767, 776, 746, 772, 749, 764, 799, 746, 751, 779, 750, 757, 736, 778, 803, 763, 745, 773, 757, 738, 698] payload_rows_per_dst=[9200, 9448, 9484, 9234, 9204, 9218, 9186, 9176] dup_rows_per_dst=[3068, 3092, 3024, 3066, 2984, 3008, 2978, 2934] combine_rows_per_src=[9200, 9448, 9484, 9234, 9204, 9218, 9186, 9176] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12259426776 prefetch_bytes=692060160 reduce_bytes=830472192 weight_movement_bytes=1522532352 egress_bytes=6133670360 ingress_bytes=7648288768 link_bytes=7648288768 total_nvlink_bytes=13781959128 weight_to_token_ratio=0.124193 prefetch_share_of_ingress=0.090486 dedup_ratio=0.244186 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=24 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.251299778e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=267001856 dispatch_meta_bytes=344104 combine_payload_bytes=267001856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267345960 ingress_bytes=301604864 link_bytes=301604864 token_bytes=534347816 weight_movement_bytes=34603008 total_nvlink_bytes=568950824 weight_to_token_ratio=0.064757 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134914822e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74492 payload_rows_remote=65172 dup_rows=23812 dup_rows_remote=20842 dedup_ratio=0.242228 router_tokens_per_expert=[779, 770, 782, 793, 790, 740, 768, 759, 718, 800, 761, 752, 784, 751, 703, 817, 755, 749, 773, 748, 768, 761, 765, 804, 760, 782, 769, 786, 810, 758, 734, 746, 761, 839, 738, 752, 742, 774, 763, 768, 743, 771, 770, 787, 781, 737, 780, 780, 806, 713, 738, 751, 814, 775, 784, 740, 746, 752, 761, 790, 768, 832, 798, 763] payload_rows_per_dst=[9340, 9276, 9204, 9320, 9244, 9384, 9342, 9382] dup_rows_per_dst=[3022, 2896, 3042, 2970, 3030, 2914, 2900, 3038] combine_rows_per_src=[9340, 9276, 9204, 9320, 9244, 9384, 9342, 9382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74150 payload_rows_remote=64666 dup_rows=24154 dup_rows_remote=21130 dedup_ratio=0.245707 router_tokens_per_expert=[782, 808, 818, 755, 783, 744, 693, 751, 724, 812, 776, 740, 794, 859, 769, 796, 758, 764, 810, 785, 769, 736, 822, 810, 783, 791, 728, 780, 749, 778, 757, 784, 800, 751, 759, 777, 755, 744, 771, 737, 727, 756, 820, 767, 776, 746, 772, 749, 764, 799, 746, 751, 779, 750, 757, 736, 778, 803, 763, 745, 773, 757, 738, 698] payload_rows_per_dst=[9200, 9448, 9484, 9234, 9204, 9218, 9186, 9176] dup_rows_per_dst=[3068, 3092, 3024, 3066, 2984, 3008, 2978, 2934] combine_rows_per_src=[9200, 9448, 9484, 9234, 9204, 9218, 9186, 9176] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265043968 dispatch_meta_bytes=342760 combine_payload_bytes=265043968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265386728 ingress_bytes=299646976 link_bytes=299646976 token_bytes=530430696 weight_movement_bytes=34603008 total_nvlink_bytes=565033704 weight_to_token_ratio=0.065236 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127547447e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12256593296 prefetch_bytes=761266176 reduce_bytes=380633088 weight_movement_bytes=1141899264 egress_bytes=6132254096 ingress_bytes=7266238464 link_bytes=7266238464 total_nvlink_bytes=13398492560 weight_to_token_ratio=0.093166 prefetch_share_of_ingress=0.104768 dedup_ratio=0.244306 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=11 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.188794363e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=64956 dup_rows=24072 dup_rows_remote=21046 dedup_ratio=0.244873 router_tokens_per_expert=[773, 768, 780, 733, 752, 775, 777, 767, 720, 751, 769, 822, 751, 790, 759, 738, 784, 767, 793, 717, 838, 770, 810, 791, 791, 786, 775, 738, 791, 744, 769, 704, 767, 742, 751, 807, 749, 721, 791, 765, 752, 778, 773, 782, 740, 740, 779, 838, 782, 781, 725, 778, 743, 737, 731, 789, 739, 779, 823, 757, 756, 805, 758, 801] payload_rows_per_dst=[9276, 9266, 9406, 9204, 9310, 9152, 9238, 9380] dup_rows_per_dst=[3026, 3026, 3104, 2992, 3138, 2836, 2894, 3056] combine_rows_per_src=[9276, 9266, 9406, 9204, 9310, 9152, 9238, 9380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266944512 dispatch_meta_bytes=344056 combine_payload_bytes=266944512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267288568 ingress_bytes=301547520 link_bytes=301547520 token_bytes=534233080 weight_movement_bytes=34603008 total_nvlink_bytes=568836088 weight_to_token_ratio=0.064771 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134699041e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265043968 dispatch_meta_bytes=342760 combine_payload_bytes=265043968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265386728 ingress_bytes=299646976 link_bytes=299646976 token_bytes=530430696 weight_movement_bytes=34603008 total_nvlink_bytes=565033704 weight_to_token_ratio=0.065236 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127547447e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=264871936 dispatch_meta_bytes=343184 combine_payload_bytes=264871936 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265215120 ingress_bytes=403283968 link_bytes=403283968 token_bytes=530087056 weight_movement_bytes=138412032 total_nvlink_bytes=668499088 weight_to_token_ratio=0.261112 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.517525105e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12233030848 prefetch_bytes=622854144 reduce_bytes=795869184 weight_movement_bytes=1418723328 egress_bytes=6120463552 ingress_bytes=7531290624 link_bytes=7531290624 total_nvlink_bytes=13651754176 weight_to_token_ratio=0.115975 prefetch_share_of_ingress=0.082702 dedup_ratio=0.244611 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=23 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028043 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.232158274e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74446 payload_rows_remote=65052 dup_rows=23858 dup_rows_remote=20868 dedup_ratio=0.242696 router_tokens_per_expert=[761, 783, 735, 766, 761, 814, 843, 790, 737, 782, 768, 780, 791, 793, 814, 781, 780, 734, 744, 790, 775, 799, 800, 764, 739, 756, 750, 743, 711, 745, 809, 818, 790, 802, 749, 761, 760, 811, 776, 743, 775, 752, 797, 796, 743, 745, 719, 786, 723, 758, 761, 799, 781, 723, 746, 741, 733, 768, 782, 764, 797, 728, 755, 732] payload_rows_per_dst=[9376, 9438, 9362, 9222, 9394, 9306, 9188, 9160] dup_rows_per_dst=[3130, 3054, 3010, 2920, 2990, 2920, 2876, 2958] combine_rows_per_src=[9376, 9438, 9362, 9222, 9394, 9306, 9188, 9160] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12256593296 prefetch_bytes=761266176 reduce_bytes=380633088 weight_movement_bytes=1141899264 egress_bytes=6132254096 ingress_bytes=7266238464 link_bytes=7266238464 total_nvlink_bytes=13398492560 weight_to_token_ratio=0.093166 prefetch_share_of_ingress=0.104768 dedup_ratio=0.244306 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=11 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.188794363e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64962 dup_rows=24068 dup_rows_remote=20992 dedup_ratio=0.244832 router_tokens_per_expert=[763, 785, 776, 711, 744, 770, 782, 733, 759, 775, 757, 731, 783, 734, 798, 777, 762, 766, 682, 748, 805, 771, 765, 788, 797, 779, 731, 757, 776, 789, 793, 772, 811, 728, 792, 743, 784, 753, 770, 737, 815, 805, 810, 799, 788, 755, 780, 749, 740, 825, 762, 800, 775, 752, 775, 746, 746, 717, 753, 810, 778, 735, 746, 814] payload_rows_per_dst=[9154, 9214, 9222, 9290, 9382, 9418, 9274, 9282] dup_rows_per_dst=[2974, 3014, 2952, 3098, 2854, 3184, 3076, 2916] combine_rows_per_src=[9154, 9214, 9222, 9290, 9382, 9418, 9274, 9282] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=344008 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266403784 ingress_bytes=300662784 link_bytes=300662784 token_bytes=532463560 weight_movement_bytes=34603008 total_nvlink_bytes=567066568 weight_to_token_ratio=0.064987 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131369851e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12246151200 prefetch_bytes=588251136 reduce_bytes=1038090240 weight_movement_bytes=1626341376 egress_bytes=6127030304 ingress_bytes=7745462272 link_bytes=7745462272 total_nvlink_bytes=13872492576 weight_to_token_ratio=0.132804 prefetch_share_of_ingress=0.075948 dedup_ratio=0.244592 prefetch_slots=34 unused_prefetch_slots=334 slot_utilization=0.092391 reduce_slots=30 remote_demand=17 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=28448 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.267197868e-04 prefetch_bytes_per_gemm_flop=9.624094203e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12233030848 prefetch_bytes=622854144 reduce_bytes=795869184 weight_movement_bytes=1418723328 egress_bytes=6120463552 ingress_bytes=7531290624 link_bytes=7531290624 total_nvlink_bytes=13651754176 weight_to_token_ratio=0.115975 prefetch_share_of_ingress=0.082702 dedup_ratio=0.244611 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=23 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028043 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.232158274e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12244255600 prefetch_bytes=692060160 reduce_bytes=761266176 weight_movement_bytes=1453326336 egress_bytes=6126084976 ingress_bytes=7571496960 link_bytes=7571496960 total_nvlink_bytes=13697581936 weight_to_token_ratio=0.118695 prefetch_share_of_ingress=0.091403 dedup_ratio=0.244925 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=22 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027065 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.238736239e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266452992 dispatch_meta_bytes=343680 combine_payload_bytes=266452992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266796672 ingress_bytes=301056000 link_bytes=301056000 token_bytes=533249664 weight_movement_bytes=34603008 total_nvlink_bytes=567852672 weight_to_token_ratio=0.064891 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132849491e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266084352 dispatch_meta_bytes=343816 combine_payload_bytes=266084352 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266428168 ingress_bytes=300687360 link_bytes=300687360 token_bytes=532512520 weight_movement_bytes=34603008 total_nvlink_bytes=567115528 weight_to_token_ratio=0.064981 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131462328e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12233467688 prefetch_bytes=692060160 reduce_bytes=588251136 weight_movement_bytes=1280311296 egress_bytes=6120679208 ingress_bytes=7393099776 link_bytes=7393099776 total_nvlink_bytes=13513778984 weight_to_token_ratio=0.104656 prefetch_share_of_ingress=0.093609 dedup_ratio=0.244264 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=17 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029130 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.209549533e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12250644616 prefetch_bytes=761266176 reduce_bytes=346030080 weight_movement_bytes=1107296256 egress_bytes=6129279112 ingress_bytes=7228661760 link_bytes=7228661760 total_nvlink_bytes=13357940872 weight_to_token_ratio=0.090387 prefetch_share_of_ingress=0.105312 dedup_ratio=0.244650 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=10 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029348 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.182646619e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12262441760 prefetch_bytes=692060160 reduce_bytes=761266176 weight_movement_bytes=1453326336 egress_bytes=6135178016 ingress_bytes=7580590080 link_bytes=7580590080 total_nvlink_bytes=13715768096 weight_to_token_ratio=0.118519 prefetch_share_of_ingress=0.091294 dedup_ratio=0.244013 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=22 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.240223921e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none - [2026-08-26 15:10:24] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 788.2 | throughput per GPU (TFLOP/s/GPU): 161.9 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 8.993162E+00 | loss scale: 1.0 | grad norm: 3.651 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74092 payload_rows_remote=64762 dup_rows=24212 dup_rows_remote=21154 dedup_ratio=0.246297 router_tokens_per_expert=[790, 711, 776, 798, 781, 784, 776, 764, 753, 819, 787, 798, 778, 749, 763, 747, 764, 802, 731, 771, 808, 740, 773, 781, 783, 731, 727, 793, 733, 791, 734, 710, 761, 747, 811, 775, 795, 762, 727, 747, 752, 786, 743, 765, 774, 775, 723, 777, 739, 802, 792, 787, 764, 780, 764, 772, 783, 727, 794, 778, 827, 741, 777, 759] payload_rows_per_dst=[9290, 9330, 9280, 9072, 9312, 9198, 9280, 9330] dup_rows_per_dst=[3070, 3058, 3060, 2932, 2938, 2992, 3120, 3042] combine_rows_per_src=[9290, 9330, 9280, 9072, 9312, 9198, 9280, 9330] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74372 payload_rows_remote=65164 dup_rows=23932 dup_rows_remote=20974 dedup_ratio=0.243449 router_tokens_per_expert=[796, 763, 763, 752, 748, 756, 771, 776, 717, 810, 749, 803, 769, 768, 782, 743, 771, 757, 728, 779, 781, 751, 783, 796, 782, 757, 771, 789, 781, 793, 786, 798, 776, 756, 737, 807, 744, 753, 731, 752, 749, 804, 761, 791, 765, 757, 788, 741, 771, 790, 752, 767, 789, 808, 786, 786, 778, 714, 762, 795, 735, 748, 749, 741] payload_rows_per_dst=[8918, 9244, 9096, 9754, 9402, 9272, 9478, 9208] dup_rows_per_dst=[2724, 2916, 2788, 3368, 3118, 3040, 3020, 2958] combine_rows_per_src=[8918, 9244, 9096, 9754, 9402, 9272, 9478, 9208] + [2026-08-26 15:58:12] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 799.9 | throughput per GPU (TFLOP/s/GPU): 159.5 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 8.993162E+00 | loss scale: 1.0 | grad norm: 3.651 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74056 payload_rows_remote=64696 dup_rows=24248 dup_rows_remote=21168 dedup_ratio=0.246663 router_tokens_per_expert=[771, 744, 811, 756, 798, 768, 758, 731, 787, 812, 741, 781, 737, 781, 771, 785, 826, 734, 761, 789, 730, 775, 770, 782, 782, 801, 725, 755, 721, 806, 803, 744, 767, 752, 791, 765, 778, 784, 746, 784, 801, 797, 752, 719, 776, 775, 775, 783, 760, 770, 755, 742, 769, 771, 758, 748, 746, 743, 751, 815, 782, 764, 768, 729] payload_rows_per_dst=[9360, 9286, 9240, 9290, 9338, 9264, 9052, 9226] dup_rows_per_dst=[3080, 3138, 3106, 2984, 2996, 3122, 2852, 2970] combine_rows_per_src=[9360, 9286, 9240, 9290, 9338, 9264, 9052, 9226] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=65100 dup_rows=23906 dup_rows_remote=20858 dedup_ratio=0.243184 router_tokens_per_expert=[744, 803, 749, 781, 753, 754, 783, 764, 725, 752, 761, 732, 765, 817, 782, 776, 785, 752, 759, 785, 771, 736, 788, 752, 748, 796, 760, 760, 735, 724, 763, 811, 841, 700, 782, 792, 783, 731, 738, 776, 781, 803, 704, 769, 736, 832, 815, 733, 774, 771, 775, 754, 787, 812, 773, 788, 759, 741, 767, 750, 771, 790, 768, 790] payload_rows_per_dst=[9274, 9152, 9334, 9280, 9352, 9298, 9436, 9272] dup_rows_per_dst=[2988, 3068, 2922, 2914, 2934, 3048, 3032, 3000] combine_rows_per_src=[9274, 9152, 9334, 9280, 9352, 9298, 9436, 9272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74372 payload_rows_remote=65164 dup_rows=23932 dup_rows_remote=20974 dedup_ratio=0.243449 router_tokens_per_expert=[796, 763, 763, 752, 748, 756, 771, 776, 717, 810, 749, 803, 769, 768, 782, 743, 771, 757, 728, 779, 781, 751, 783, 796, 782, 757, 771, 789, 781, 793, 786, 798, 776, 756, 737, 807, 744, 753, 731, 752, 749, 804, 761, 791, 765, 757, 788, 741, 771, 790, 752, 767, 789, 808, 786, 786, 778, 714, 762, 795, 735, 748, 749, 741] payload_rows_per_dst=[8918, 9244, 9096, 9754, 9402, 9272, 9478, 9208] dup_rows_per_dst=[2724, 2916, 2788, 3368, 3118, 3040, 3020, 2958] combine_rows_per_src=[8918, 9244, 9096, 9754, 9402, 9272, 9478, 9208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74092 payload_rows_remote=64762 dup_rows=24212 dup_rows_remote=21154 dedup_ratio=0.246297 router_tokens_per_expert=[790, 711, 776, 798, 781, 784, 776, 764, 753, 819, 787, 798, 778, 749, 763, 747, 764, 802, 731, 771, 808, 740, 773, 781, 783, 731, 727, 793, 733, 791, 734, 710, 761, 747, 811, 775, 795, 762, 727, 747, 752, 786, 743, 765, 774, 775, 723, 777, 739, 802, 792, 787, 764, 780, 764, 772, 783, 727, 794, 778, 827, 741, 777, 759] payload_rows_per_dst=[9290, 9330, 9280, 9072, 9312, 9198, 9280, 9330] dup_rows_per_dst=[3070, 3058, 3060, 2932, 2938, 2992, 3120, 3042] combine_rows_per_src=[9290, 9330, 9280, 9072, 9312, 9198, 9280, 9330] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=64944 dup_rows=24136 dup_rows_remote=21154 dedup_ratio=0.245524 router_tokens_per_expert=[782, 754, 721, 754, 801, 810, 763, 797, 761, 754, 732, 773, 747, 747, 747, 815, 784, 793, 795, 764, 774, 801, 805, 766, 781, 776, 760, 770, 765, 752, 756, 743, 737, 768, 777, 782, 804, 751, 750, 780, 720, 772, 761, 776, 752, 771, 789, 730, 751, 779, 818, 778, 753, 800, 746, 767, 759, 740, 743, 755, 750, 829, 770, 751] payload_rows_per_dst=[9308, 9158, 9430, 9224, 9264, 9152, 9378, 9254] dup_rows_per_dst=[3056, 2994, 3134, 2982, 3034, 2990, 3006, 2940] combine_rows_per_src=[9308, 9158, 9430, 9224, 9264, 9152, 9378, 9254] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74056 payload_rows_remote=64696 dup_rows=24248 dup_rows_remote=21168 dedup_ratio=0.246663 router_tokens_per_expert=[771, 744, 811, 756, 798, 768, 758, 731, 787, 812, 741, 781, 737, 781, 771, 785, 826, 734, 761, 789, 730, 775, 770, 782, 782, 801, 725, 755, 721, 806, 803, 744, 767, 752, 791, 765, 778, 784, 746, 784, 801, 797, 752, 719, 776, 775, 775, 783, 760, 770, 755, 742, 769, 771, 758, 748, 746, 743, 751, 815, 782, 764, 768, 729] payload_rows_per_dst=[9360, 9286, 9240, 9290, 9338, 9264, 9052, 9226] dup_rows_per_dst=[3080, 3138, 3106, 2984, 2996, 3122, 2852, 2970] combine_rows_per_src=[9360, 9286, 9240, 9290, 9338, 9264, 9052, 9226] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74262 payload_rows_remote=64940 dup_rows=24042 dup_rows_remote=21078 dedup_ratio=0.244568 router_tokens_per_expert=[719, 788, 784, 747, 786, 787, 784, 781, 802, 800, 791, 814, 734, 758, 717, 779, 785, 738, 719, 805, 805, 768, 778, 745, 819, 735, 766, 763, 753, 716, 736, 740, 748, 709, 727, 772, 763, 756, 755, 807, 823, 769, 763, 781, 777, 713, 815, 775, 751, 749, 744, 781, 772, 776, 804, 766, 770, 841, 745, 813, 730, 778, 740, 797] payload_rows_per_dst=[9276, 9316, 9322, 9198, 9190, 9374, 9258, 9328] dup_rows_per_dst=[3076, 3074, 2964, 2858, 2884, 3058, 3028, 3100] combine_rows_per_src=[9276, 9316, 9322, 9198, 9190, 9374, 9258, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74370 payload_rows_remote=65160 dup_rows=23934 dup_rows_remote=20920 dedup_ratio=0.243469 router_tokens_per_expert=[798, 779, 747, 803, 830, 805, 756, 795, 773, 758, 763, 784, 766, 722, 762, 780, 752, 753, 732, 712, 759, 765, 799, 755, 804, 705, 767, 780, 735, 726, 768, 822, 738, 768, 790, 711, 790, 772, 769, 768, 783, 775, 806, 762, 776, 736, 753, 777, 732, 746, 752, 797, 785, 734, 763, 803, 761, 813, 770, 778, 744, 786, 786, 773] payload_rows_per_dst=[9468, 9246, 9174, 9288, 9298, 9304, 9210, 9382] dup_rows_per_dst=[3158, 2970, 2880, 2926, 2914, 3032, 3014, 3040] combine_rows_per_src=[9468, 9246, 9174, 9288, 9298, 9304, 9210, 9382] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74388 payload_rows_remote=65066 dup_rows=23916 dup_rows_remote=20908 dedup_ratio=0.243286 router_tokens_per_expert=[759, 786, 762, 775, 787, 748, 755, 757, 790, 819, 819, 715, 765, 742, 774, 753, 751, 808, 836, 787, 763, 755, 827, 760, 790, 757, 717, 791, 773, 787, 767, 735, 799, 809, 784, 749, 751, 765, 737, 771, 738, 742, 740, 801, 771, 787, 770, 731, 731, 763, 768, 760, 755, 788, 758, 747, 783, 728, 780, 788, 763, 781, 768, 736] payload_rows_per_dst=[9354, 9420, 9400, 9230, 9322, 9190, 9150, 9322] dup_rows_per_dst=[2904, 2934, 3174, 3004, 3008, 2970, 2990, 2932] combine_rows_per_src=[9354, 9420, 9400, 9230, 9322, 9190, 9150, 9322] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=264994816 dispatch_meta_bytes=343456 combine_payload_bytes=264994816 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265338272 ingress_bytes=334200832 link_bytes=334200832 token_bytes=530333088 weight_movement_bytes=69206016 total_nvlink_bytes=599539104 weight_to_token_ratio=0.130495 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257570825e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266649600 dispatch_meta_bytes=343832 combine_payload_bytes=266649600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266993432 ingress_bytes=301252608 link_bytes=301252608 token_bytes=533643032 weight_movement_bytes=34603008 total_nvlink_bytes=568246040 weight_to_token_ratio=0.064843 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133589311e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265265152 dispatch_meta_bytes=343664 combine_payload_bytes=265265152 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265608816 ingress_bytes=334471168 link_bytes=334471168 token_bytes=530873968 weight_movement_bytes=69206016 total_nvlink_bytes=600079984 weight_to_token_ratio=0.130362 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258588078e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266911744 dispatch_meta_bytes=344552 combine_payload_bytes=266911744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267256296 ingress_bytes=301514752 link_bytes=301514752 token_bytes=534168040 weight_movement_bytes=34603008 total_nvlink_bytes=568771048 weight_to_token_ratio=0.064779 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134575738e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266010624 dispatch_meta_bytes=344392 combine_payload_bytes=266010624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266355016 ingress_bytes=300613632 link_bytes=300613632 token_bytes=532365640 weight_movement_bytes=34603008 total_nvlink_bytes=566968648 weight_to_token_ratio=0.064999 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131184896e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=264994816 dispatch_meta_bytes=343456 combine_payload_bytes=264994816 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265338272 ingress_bytes=334200832 link_bytes=334200832 token_bytes=530333088 weight_movement_bytes=69206016 total_nvlink_bytes=599539104 weight_to_token_ratio=0.130495 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257570825e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266649600 dispatch_meta_bytes=343832 combine_payload_bytes=266649600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266993432 ingress_bytes=301252608 link_bytes=301252608 token_bytes=533643032 weight_movement_bytes=34603008 total_nvlink_bytes=568246040 weight_to_token_ratio=0.064843 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133589311e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=344072 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266338312 ingress_bytes=335200256 link_bytes=335200256 token_bytes=532332552 weight_movement_bytes=69206016 total_nvlink_bytes=601538568 weight_to_token_ratio=0.130005 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261331577e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266895360 dispatch_meta_bytes=344320 combine_payload_bytes=266895360 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267239680 ingress_bytes=405307392 link_bytes=405307392 token_bytes=534135040 weight_movement_bytes=138412032 total_nvlink_bytes=672547072 weight_to_token_ratio=0.259133 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.525139086e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266510336 dispatch_meta_bytes=343896 combine_payload_bytes=266510336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266854232 ingress_bytes=301113344 link_bytes=301113344 token_bytes=533364568 weight_movement_bytes=34603008 total_nvlink_bytes=567967576 weight_to_token_ratio=0.064877 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133065272e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74370 payload_rows_remote=65160 dup_rows=23934 dup_rows_remote=20920 dedup_ratio=0.243469 router_tokens_per_expert=[798, 779, 747, 803, 830, 805, 756, 795, 773, 758, 763, 784, 766, 722, 762, 780, 752, 753, 732, 712, 759, 765, 799, 755, 804, 705, 767, 780, 735, 726, 768, 822, 738, 768, 790, 711, 790, 772, 769, 768, 783, 775, 806, 762, 776, 736, 753, 777, 732, 746, 752, 797, 785, 734, 763, 803, 761, 813, 770, 778, 744, 786, 786, 773] payload_rows_per_dst=[9468, 9246, 9174, 9288, 9298, 9304, 9210, 9382] dup_rows_per_dst=[3158, 2970, 2880, 2926, 2914, 3032, 3014, 3040] combine_rows_per_src=[9468, 9246, 9174, 9288, 9298, 9304, 9210, 9382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74362 payload_rows_remote=64970 dup_rows=23942 dup_rows_remote=20832 dedup_ratio=0.243551 router_tokens_per_expert=[778, 750, 785, 759, 791, 772, 766, 721, 797, 808, 786, 761, 783, 761, 756, 806, 747, 785, 731, 781, 749, 732, 767, 801, 732, 795, 763, 745, 742, 759, 779, 799, 722, 742, 752, 701, 793, 785, 729, 845, 776, 735, 806, 828, 765, 761, 783, 763, 762, 763, 798, 779, 737, 768, 790, 750, 735, 766, 746, 784, 743, 798, 811, 749] payload_rows_per_dst=[9272, 9262, 9274, 9288, 9226, 9296, 9352, 9392] dup_rows_per_dst=[2972, 2968, 2960, 2912, 2940, 3016, 3064, 3110] combine_rows_per_src=[9272, 9262, 9274, 9288, 9226, 9296, 9352, 9392] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=65030 dup_rows=23964 dup_rows_remote=20860 dedup_ratio=0.243774 router_tokens_per_expert=[752, 787, 756, 750, 787, 774, 756, 730, 796, 741, 816, 801, 740, 773, 748, 792, 776, 786, 793, 783, 802, 751, 737, 788, 762, 747, 773, 770, 778, 778, 735, 756, 740, 786, 754, 755, 763, 781, 747, 777, 774, 728, 796, 771, 757, 722, 811, 797, 725, 861, 787, 789, 749, 747, 742, 786, 765, 755, 792, 743, 758, 732, 764, 784] payload_rows_per_dst=[9268, 9310, 9382, 9226, 9226, 9368, 9312, 9248] dup_rows_per_dst=[2916, 3104, 3050, 2972, 2980, 2944, 3060, 2938] combine_rows_per_src=[9268, 9310, 9382, 9226, 9226, 9368, 9312, 9248] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64792 dup_rows=24068 dup_rows_remote=21060 dedup_ratio=0.244832 router_tokens_per_expert=[786, 767, 789, 737, 732, 729, 739, 797, 776, 813, 728, 719, 768, 747, 814, 781, 756, 769, 748, 738, 755, 754, 765, 806, 811, 765, 759, 792, 806, 761, 775, 767, 769, 732, 752, 763, 787, 801, 753, 806, 725, 741, 774, 728, 802, 796, 786, 756, 815, 758, 724, 763, 819, 783, 752, 786, 804, 704, 771, 765, 782, 784, 767, 755] payload_rows_per_dst=[9444, 9238, 9196, 9178, 9286, 9262, 9390, 9242] dup_rows_per_dst=[3008, 3060, 2986, 2990, 3040, 2952, 3010, 3022] combine_rows_per_src=[9444, 9238, 9196, 9178, 9286, 9262, 9390, 9242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74282 payload_rows_remote=65038 dup_rows=24022 dup_rows_remote=21030 dedup_ratio=0.244364 router_tokens_per_expert=[746, 772, 749, 802, 784, 772, 762, 826, 754, 777, 764, 792, 789, 778, 751, 767, 765, 767, 748, 744, 758, 761, 768, 813, 715, 778, 782, 744, 751, 773, 776, 762, 752, 790, 792, 808, 749, 752, 797, 749, 775, 763, 769, 726, 751, 776, 760, 798, 741, 736, 796, 747, 722, 761, 813, 761, 756, 696, 781, 786, 771, 802, 791, 795] payload_rows_per_dst=[9360, 9382, 9210, 9312, 9334, 9244, 9162, 9278] dup_rows_per_dst=[3066, 2962, 3038, 2850, 3044, 2992, 2992, 3078] combine_rows_per_src=[9360, 9382, 9210, 9312, 9334, 9244, 9162, 9278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74262 payload_rows_remote=64940 dup_rows=24042 dup_rows_remote=21078 dedup_ratio=0.244568 router_tokens_per_expert=[719, 788, 784, 747, 786, 787, 784, 781, 802, 800, 791, 814, 734, 758, 717, 779, 785, 738, 719, 805, 805, 768, 778, 745, 819, 735, 766, 763, 753, 716, 736, 740, 748, 709, 727, 772, 763, 756, 755, 807, 823, 769, 763, 781, 777, 713, 815, 775, 751, 749, 744, 781, 772, 776, 804, 766, 770, 841, 745, 813, 730, 778, 740, 797] payload_rows_per_dst=[9276, 9316, 9322, 9198, 9190, 9374, 9258, 9328] dup_rows_per_dst=[3076, 3074, 2964, 2858, 2884, 3058, 3028, 3100] combine_rows_per_src=[9276, 9316, 9322, 9198, 9190, 9374, 9258, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=65030 dup_rows=23964 dup_rows_remote=20860 dedup_ratio=0.243774 router_tokens_per_expert=[752, 787, 756, 750, 787, 774, 756, 730, 796, 741, 816, 801, 740, 773, 748, 792, 776, 786, 793, 783, 802, 751, 737, 788, 762, 747, 773, 770, 778, 778, 735, 756, 740, 786, 754, 755, 763, 781, 747, 777, 774, 728, 796, 771, 757, 722, 811, 797, 725, 861, 787, 789, 749, 747, 742, 786, 765, 755, 792, 743, 758, 732, 764, 784] payload_rows_per_dst=[9268, 9310, 9382, 9226, 9226, 9368, 9312, 9248] dup_rows_per_dst=[2916, 3104, 3050, 2972, 2980, 2944, 3060, 2938] combine_rows_per_src=[9268, 9310, 9382, 9226, 9226, 9368, 9312, 9248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74362 payload_rows_remote=64970 dup_rows=23942 dup_rows_remote=20832 dedup_ratio=0.243551 router_tokens_per_expert=[778, 750, 785, 759, 791, 772, 766, 721, 797, 808, 786, 761, 783, 761, 756, 806, 747, 785, 731, 781, 749, 732, 767, 801, 732, 795, 763, 745, 742, 759, 779, 799, 722, 742, 752, 701, 793, 785, 729, 845, 776, 735, 806, 828, 765, 761, 783, 763, 762, 763, 798, 779, 737, 768, 790, 750, 735, 766, 746, 784, 743, 798, 811, 749] payload_rows_per_dst=[9272, 9262, 9274, 9288, 9226, 9296, 9352, 9392] dup_rows_per_dst=[2972, 2968, 2960, 2912, 2940, 3016, 3064, 3110] combine_rows_per_src=[9272, 9262, 9274, 9288, 9226, 9296, 9352, 9392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74574 payload_rows_remote=65206 dup_rows=23730 dup_rows_remote=20736 dedup_ratio=0.241394 router_tokens_per_expert=[739, 776, 746, 778, 723, 752, 790, 801, 765, 727, 782, 728, 748, 772, 778, 754, 823, 764, 790, 718, 749, 806, 740, 713, 860, 756, 747, 779, 770, 778, 796, 805, 739, 762, 797, 772, 767, 745, 784, 743, 751, 723, 810, 814, 734, 733, 780, 776, 794, 755, 788, 763, 763, 769, 745, 804, 730, 775, 727, 797, 814, 773, 754, 818] payload_rows_per_dst=[9308, 9242, 9210, 9452, 9236, 9354, 9368, 9404] dup_rows_per_dst=[2902, 2866, 2996, 3130, 2982, 2888, 2994, 2972] combine_rows_per_src=[9308, 9242, 9210, 9452, 9236, 9354, 9368, 9404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=64960 dup_rows=23998 dup_rows_remote=20934 dedup_ratio=0.244120 router_tokens_per_expert=[797, 777, 761, 703, 803, 713, 783, 777, 789, 775, 772, 765, 718, 752, 781, 791, 758, 763, 792, 791, 753, 773, 776, 732, 804, 801, 779, 767, 758, 739, 815, 742, 770, 823, 804, 746, 774, 740, 779, 769, 772, 756, 809, 788, 778, 774, 772, 747, 792, 737, 769, 749, 837, 766, 702, 752, 810, 754, 731, 774, 698, 761, 766, 753] payload_rows_per_dst=[9210, 9308, 9284, 9374, 9346, 9358, 9274, 9152] dup_rows_per_dst=[3018, 2978, 2992, 3036, 3064, 3034, 2934, 2942] combine_rows_per_src=[9210, 9308, 9284, 9374, 9346, 9358, 9274, 9152] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74476 payload_rows_remote=65114 dup_rows=23828 dup_rows_remote=20812 dedup_ratio=0.242391 router_tokens_per_expert=[752, 749, 764, 724, 793, 748, 753, 759, 767, 794, 765, 724, 743, 767, 791, 801, 786, 735, 819, 786, 762, 741, 758, 827, 799, 776, 794, 792, 738, 762, 737, 791, 743, 785, 782, 753, 758, 783, 736, 797, 784, 728, 812, 732, 796, 806, 816, 758, 823, 783, 739, 770, 762, 763, 773, 719, 804, 749, 821, 746, 717, 735, 744, 738] payload_rows_per_dst=[9232, 9316, 9332, 9362, 9304, 9442, 9298, 9190] dup_rows_per_dst=[2852, 2988, 3096, 3016, 2970, 3022, 2966, 2918] combine_rows_per_src=[9232, 9316, 9332, 9362, 9304, 9442, 9298, 9190] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=65118 dup_rows=23946 dup_rows_remote=20888 dedup_ratio=0.243591 router_tokens_per_expert=[791, 769, 767, 824, 791, 763, 788, 745, 739, 765, 749, 774, 805, 767, 773, 788, 770, 770, 750, 783, 772, 779, 787, 738, 762, 789, 795, 716, 718, 813, 784, 726, 752, 776, 759, 789, 788, 796, 764, 739, 758, 759, 725, 792, 762, 763, 745, 762, 721, 760, 813, 734, 749, 806, 719, 762, 753, 770, 802, 762, 792, 790, 760, 780] payload_rows_per_dst=[9392, 9374, 9240, 9240, 9308, 9182, 9176, 9446] dup_rows_per_dst=[3084, 2946, 3058, 2966, 3018, 2950, 2952, 2972] combine_rows_per_src=[9392, 9374, 9240, 9240, 9308, 9182, 9176, 9446] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74476 payload_rows_remote=65114 dup_rows=23828 dup_rows_remote=20812 dedup_ratio=0.242391 router_tokens_per_expert=[752, 749, 764, 724, 793, 748, 753, 759, 767, 794, 765, 724, 743, 767, 791, 801, 786, 735, 819, 786, 762, 741, 758, 827, 799, 776, 794, 792, 738, 762, 737, 791, 743, 785, 782, 753, 758, 783, 736, 797, 784, 728, 812, 732, 796, 806, 816, 758, 823, 783, 739, 770, 762, 763, 773, 719, 804, 749, 821, 746, 717, 735, 744, 738] payload_rows_per_dst=[9232, 9316, 9332, 9362, 9304, 9442, 9298, 9190] dup_rows_per_dst=[2852, 2988, 3096, 3016, 2970, 3022, 2966, 2918] combine_rows_per_src=[9232, 9316, 9332, 9362, 9304, 9442, 9298, 9190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=64960 dup_rows=23998 dup_rows_remote=20934 dedup_ratio=0.244120 router_tokens_per_expert=[797, 777, 761, 703, 803, 713, 783, 777, 789, 775, 772, 765, 718, 752, 781, 791, 758, 763, 792, 791, 753, 773, 776, 732, 804, 801, 779, 767, 758, 739, 815, 742, 770, 823, 804, 746, 774, 740, 779, 769, 772, 756, 809, 788, 778, 774, 772, 747, 792, 737, 769, 749, 837, 766, 702, 752, 810, 754, 731, 774, 698, 761, 766, 753] payload_rows_per_dst=[9210, 9308, 9284, 9374, 9346, 9358, 9274, 9152] dup_rows_per_dst=[3018, 2978, 2992, 3036, 3064, 3034, 2934, 2942] combine_rows_per_src=[9210, 9308, 9284, 9374, 9346, 9358, 9274, 9152] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74282 payload_rows_remote=65038 dup_rows=24022 dup_rows_remote=21030 dedup_ratio=0.244364 router_tokens_per_expert=[746, 772, 749, 802, 784, 772, 762, 826, 754, 777, 764, 792, 789, 778, 751, 767, 765, 767, 748, 744, 758, 761, 768, 813, 715, 778, 782, 744, 751, 773, 776, 762, 752, 790, 792, 808, 749, 752, 797, 749, 775, 763, 769, 726, 751, 776, 760, 798, 741, 736, 796, 747, 722, 761, 813, 761, 756, 696, 781, 786, 771, 802, 791, 795] payload_rows_per_dst=[9360, 9382, 9210, 9312, 9334, 9244, 9162, 9278] dup_rows_per_dst=[3066, 2962, 3038, 2850, 3044, 2992, 2992, 3078] combine_rows_per_src=[9360, 9382, 9210, 9312, 9334, 9244, 9162, 9278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266117120 dispatch_meta_bytes=343208 combine_payload_bytes=266117120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266460328 ingress_bytes=300720128 link_bytes=300720128 token_bytes=532577448 weight_movement_bytes=34603008 total_nvlink_bytes=567180456 weight_to_token_ratio=0.064973 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131585632e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=343560 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266706440 ingress_bytes=370171904 link_bytes=370171904 token_bytes=533069320 weight_movement_bytes=103809024 total_nvlink_bytes=636878344 weight_to_token_ratio=0.194738 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392927073e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265388032 dispatch_meta_bytes=343408 combine_payload_bytes=265388032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265731440 ingress_bytes=299991040 link_bytes=299991040 token_bytes=531119472 weight_movement_bytes=34603008 total_nvlink_bytes=565722480 weight_to_token_ratio=0.065151 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128842132e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266895360 dispatch_meta_bytes=344320 combine_payload_bytes=266895360 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267239680 ingress_bytes=405307392 link_bytes=405307392 token_bytes=534135040 weight_movement_bytes=138412032 total_nvlink_bytes=672547072 weight_to_token_ratio=0.259133 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.525139086e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266723328 dispatch_meta_bytes=344024 combine_payload_bytes=266723328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267067352 ingress_bytes=301326336 link_bytes=301326336 token_bytes=533790680 weight_movement_bytes=34603008 total_nvlink_bytes=568393688 weight_to_token_ratio=0.064825 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133866744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266706944 dispatch_meta_bytes=343704 combine_payload_bytes=266706944 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=267050648 ingress_bytes=405118976 link_bytes=405118976 token_bytes=533757592 weight_movement_bytes=138412032 total_nvlink_bytes=672169624 weight_to_token_ratio=0.259316 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524430092e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266076160 dispatch_meta_bytes=343576 combine_payload_bytes=266076160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266419736 ingress_bytes=300679168 link_bytes=300679168 token_bytes=532495896 weight_movement_bytes=34603008 total_nvlink_bytes=567098904 weight_to_token_ratio=0.064983 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131431503e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=343560 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266706440 ingress_bytes=370171904 link_bytes=370171904 token_bytes=533069320 weight_movement_bytes=103809024 total_nvlink_bytes=636878344 weight_to_token_ratio=0.194738 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392927073e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=344272 combine_payload_bytes=266395648 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266739920 ingress_bytes=335601664 link_bytes=335601664 token_bytes=533135568 weight_movement_bytes=69206016 total_nvlink_bytes=602341584 weight_to_token_ratio=0.129809 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262842043e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267083776 dispatch_meta_bytes=343768 combine_payload_bytes=267083776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267427544 ingress_bytes=301686784 link_bytes=301686784 token_bytes=534511320 weight_movement_bytes=34603008 total_nvlink_bytes=569114328 weight_to_token_ratio=0.064738 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135223080e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266076160 dispatch_meta_bytes=343576 combine_payload_bytes=266076160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266419736 ingress_bytes=300679168 link_bytes=300679168 token_bytes=532495896 weight_movement_bytes=34603008 total_nvlink_bytes=567098904 weight_to_token_ratio=0.064983 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131431503e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266706944 dispatch_meta_bytes=343704 combine_payload_bytes=266706944 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=267050648 ingress_bytes=405118976 link_bytes=405118976 token_bytes=533757592 weight_movement_bytes=138412032 total_nvlink_bytes=672169624 weight_to_token_ratio=0.259316 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524430092e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=344072 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266338312 ingress_bytes=335200256 link_bytes=335200256 token_bytes=532332552 weight_movement_bytes=69206016 total_nvlink_bytes=601538568 weight_to_token_ratio=0.130005 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261331577e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74218 payload_rows_remote=64552 dup_rows=24086 dup_rows_remote=20658 dedup_ratio=0.245015 router_tokens_per_expert=[773, 775, 803, 761, 715, 757, 775, 779, 738, 777, 746, 779, 761, 773, 787, 763, 736, 801, 753, 742, 829, 783, 791, 784, 786, 812, 776, 751, 743, 735, 765, 799, 791, 773, 772, 761, 774, 768, 771, 742, 716, 747, 732, 849, 758, 754, 776, 735, 764, 828, 758, 822, 737, 699, 752, 743, 763, 752, 789, 814, 760, 766, 748, 790] payload_rows_per_dst=[8870, 9282, 9332, 9314, 9270, 9230, 9254, 9666] dup_rows_per_dst=[2676, 2936, 3066, 2898, 3034, 3026, 3022, 3428] combine_rows_per_src=[8870, 9282, 9332, 9314, 9270, 9230, 9254, 9666] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74472 payload_rows_remote=65042 dup_rows=23832 dup_rows_remote=20630 dedup_ratio=0.242432 router_tokens_per_expert=[753, 743, 770, 800, 791, 745, 789, 761, 820, 794, 762, 760, 787, 802, 751, 736, 800, 781, 779, 799, 732, 763, 760, 793, 730, 779, 758, 762, 799, 782, 722, 792, 755, 757, 769, 775, 752, 750, 780, 794, 815, 763, 736, 723, 748, 750, 746, 790, 764, 787, 764, 728, 765, 759, 774, 797, 808, 785, 729, 768, 735, 729, 767, 795] payload_rows_per_dst=[9430, 9336, 9226, 9340, 9514, 9214, 9118, 9294] dup_rows_per_dst=[3202, 3098, 2860, 2898, 3200, 2928, 2708, 2938] combine_rows_per_src=[9430, 9336, 9226, 9340, 9514, 9214, 9118, 9294] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74350 payload_rows_remote=65214 dup_rows=23954 dup_rows_remote=21062 dedup_ratio=0.243673 router_tokens_per_expert=[786, 747, 752, 782, 784, 767, 828, 770, 725, 736, 814, 737, 751, 751, 754, 746, 794, 779, 796, 797, 771, 756, 750, 754, 786, 776, 794, 803, 765, 738, 737, 723, 723, 784, 783, 792, 706, 772, 812, 764, 770, 745, 762, 752, 782, 783, 769, 792, 767, 808, 763, 795, 774, 774, 834, 767, 752, 731, 750, 774, 751, 741, 774, 757] payload_rows_per_dst=[9480, 9136, 9314, 9224, 9330, 9224, 9462, 9180] dup_rows_per_dst=[2952, 2892, 3080, 3020, 2942, 3086, 3102, 2880] combine_rows_per_src=[9480, 9136, 9314, 9224, 9330, 9224, 9462, 9180] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=64938 dup_rows=23946 dup_rows_remote=20842 dedup_ratio=0.243591 router_tokens_per_expert=[774, 707, 772, 773, 752, 764, 786, 788, 765, 730, 801, 749, 745, 778, 815, 801, 790, 801, 796, 744, 803, 771, 788, 769, 829, 754, 768, 743, 797, 793, 741, 761, 781, 757, 769, 794, 751, 750, 799, 781, 766, 762, 736, 737, 775, 786, 744, 775, 756, 788, 741, 752, 792, 775, 747, 806, 768, 733, 767, 745, 758, 770, 721, 722] payload_rows_per_dst=[9230, 9346, 9420, 9272, 9366, 9244, 9346, 9134] dup_rows_per_dst=[3002, 3022, 3104, 3100, 2998, 2918, 2968, 2834] combine_rows_per_src=[9230, 9346, 9420, 9272, 9366, 9244, 9346, 9134] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=65030 dup_rows=23928 dup_rows_remote=20972 dedup_ratio=0.243408 router_tokens_per_expert=[819, 733, 730, 782, 847, 798, 819, 713, 790, 799, 765, 734, 757, 765, 804, 813, 736, 795, 769, 728, 736, 778, 728, 827, 778, 765, 745, 797, 752, 765, 759, 756, 723, 800, 757, 767, 757, 791, 745, 710, 801, 767, 757, 756, 762, 741, 812, 771, 765, 772, 777, 765, 771, 743, 777, 781, 744, 724, 746, 776, 723, 820, 793, 776] payload_rows_per_dst=[9356, 9352, 9298, 9312, 9172, 9316, 9346, 9224] dup_rows_per_dst=[3126, 3102, 2896, 2922, 2928, 3018, 2956, 2980] combine_rows_per_src=[9356, 9352, 9298, 9312, 9172, 9316, 9346, 9224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74230 payload_rows_remote=64846 dup_rows=24074 dup_rows_remote=21014 dedup_ratio=0.244893 router_tokens_per_expert=[765, 775, 745, 761, 760, 726, 756, 769, 747, 753, 807, 762, 775, 797, 760, 760, 766, 768, 780, 780, 749, 788, 735, 786, 771, 782, 762, 759, 822, 749, 761, 778, 812, 732, 729, 735, 751, 751, 810, 744, 744, 785, 779, 759, 822, 768, 781, 784, 806, 757, 784, 739, 782, 747, 789, 754, 753, 778, 782, 801, 755, 754, 766, 765] payload_rows_per_dst=[9102, 9328, 9264, 9252, 9324, 9384, 9274, 9302] dup_rows_per_dst=[3012, 2994, 3040, 3116, 2804, 3060, 3042, 3006] combine_rows_per_src=[9102, 9328, 9264, 9252, 9324, 9384, 9274, 9302] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74132 payload_rows_remote=64896 dup_rows=24172 dup_rows_remote=21168 dedup_ratio=0.245890 router_tokens_per_expert=[752, 758, 751, 782, 769, 793, 755, 755, 765, 758, 799, 802, 754, 772, 713, 785, 755, 772, 751, 778, 797, 763, 765, 741, 746, 742, 758, 851, 789, 792, 777, 762, 729, 703, 796, 739, 802, 790, 738, 823, 739, 775, 772, 779, 800, 753, 763, 781, 804, 755, 827, 769, 752, 759, 835, 766, 749, 730, 754, 735, 765, 724, 755, 789] payload_rows_per_dst=[9182, 9314, 9272, 9304, 9236, 9296, 9446, 9082] dup_rows_per_dst=[3048, 2982, 2972, 3130, 3004, 3028, 3088, 2920] combine_rows_per_src=[9182, 9314, 9272, 9304, 9236, 9296, 9446, 9082] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74252 payload_rows_remote=65038 dup_rows=24052 dup_rows_remote=21064 dedup_ratio=0.244670 router_tokens_per_expert=[856, 814, 775, 795, 780, 754, 759, 785, 729, 762, 783, 790, 719, 796, 733, 780, 738, 792, 755, 744, 746, 777, 777, 751, 806, 773, 783, 728, 782, 730, 777, 722, 772, 751, 774, 733, 737, 796, 768, 760, 736, 769, 756, 815, 794, 756, 754, 786, 736, 764, 812, 732, 782, 781, 715, 764, 844, 736, 754, 793, 772, 764, 768, 787] payload_rows_per_dst=[9494, 9226, 9292, 9214, 9166, 9306, 9224, 9330] dup_rows_per_dst=[3142, 2958, 2868, 2988, 3016, 3026, 2948, 3106] combine_rows_per_src=[9494, 9226, 9292, 9214, 9166, 9306, 9224, 9330] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74252 payload_rows_remote=65038 dup_rows=24052 dup_rows_remote=21064 dedup_ratio=0.244670 router_tokens_per_expert=[856, 814, 775, 795, 780, 754, 759, 785, 729, 762, 783, 790, 719, 796, 733, 780, 738, 792, 755, 744, 746, 777, 777, 751, 806, 773, 783, 728, 782, 730, 777, 722, 772, 751, 774, 733, 737, 796, 768, 760, 736, 769, 756, 815, 794, 756, 754, 786, 736, 764, 812, 732, 782, 781, 715, 764, 844, 736, 754, 793, 772, 764, 768, 787] payload_rows_per_dst=[9494, 9226, 9292, 9214, 9166, 9306, 9224, 9330] dup_rows_per_dst=[3142, 2958, 2868, 2988, 3016, 3026, 2948, 3106] combine_rows_per_src=[9494, 9226, 9292, 9214, 9166, 9306, 9224, 9330] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74574 payload_rows_remote=65206 dup_rows=23730 dup_rows_remote=20736 dedup_ratio=0.241394 router_tokens_per_expert=[739, 776, 746, 778, 723, 752, 790, 801, 765, 727, 782, 728, 748, 772, 778, 754, 823, 764, 790, 718, 749, 806, 740, 713, 860, 756, 747, 779, 770, 778, 796, 805, 739, 762, 797, 772, 767, 745, 784, 743, 751, 723, 810, 814, 734, 733, 780, 776, 794, 755, 788, 763, 763, 769, 745, 804, 730, 775, 727, 797, 814, 773, 754, 818] payload_rows_per_dst=[9308, 9242, 9210, 9452, 9236, 9354, 9368, 9404] dup_rows_per_dst=[2902, 2866, 2996, 3130, 2982, 2888, 2994, 2972] combine_rows_per_src=[9308, 9242, 9210, 9452, 9236, 9354, 9368, 9404] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74230 payload_rows_remote=64846 dup_rows=24074 dup_rows_remote=21014 dedup_ratio=0.244893 router_tokens_per_expert=[765, 775, 745, 761, 760, 726, 756, 769, 747, 753, 807, 762, 775, 797, 760, 760, 766, 768, 780, 780, 749, 788, 735, 786, 771, 782, 762, 759, 822, 749, 761, 778, 812, 732, 729, 735, 751, 751, 810, 744, 744, 785, 779, 759, 822, 768, 781, 784, 806, 757, 784, 739, 782, 747, 789, 754, 753, 778, 782, 801, 755, 754, 766, 765] payload_rows_per_dst=[9102, 9328, 9264, 9252, 9324, 9384, 9274, 9302] dup_rows_per_dst=[3012, 2994, 3040, 3116, 2804, 3060, 3042, 3006] combine_rows_per_src=[9102, 9328, 9264, 9252, 9324, 9384, 9274, 9302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=65118 dup_rows=23946 dup_rows_remote=20888 dedup_ratio=0.243591 router_tokens_per_expert=[791, 769, 767, 824, 791, 763, 788, 745, 739, 765, 749, 774, 805, 767, 773, 788, 770, 770, 750, 783, 772, 779, 787, 738, 762, 789, 795, 716, 718, 813, 784, 726, 752, 776, 759, 789, 788, 796, 764, 739, 758, 759, 725, 792, 762, 763, 745, 762, 721, 760, 813, 734, 749, 806, 719, 762, 753, 770, 802, 762, 792, 790, 760, 780] payload_rows_per_dst=[9392, 9374, 9240, 9240, 9308, 9182, 9176, 9446] dup_rows_per_dst=[3084, 2946, 3058, 2966, 3018, 2950, 2952, 2972] combine_rows_per_src=[9392, 9374, 9240, 9240, 9308, 9182, 9176, 9446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=342688 combine_payload_bytes=266412032 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266754720 ingress_bytes=335618048 link_bytes=335618048 token_bytes=533166752 weight_movement_bytes=69206016 total_nvlink_bytes=602372768 weight_to_token_ratio=0.129802 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262903695e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=264404992 dispatch_meta_bytes=340840 combine_payload_bytes=264404992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264745832 ingress_bytes=299008000 link_bytes=299008000 token_bytes=529150824 weight_movement_bytes=34603008 total_nvlink_bytes=563753832 weight_to_token_ratio=0.065393 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125143032e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265986048 dispatch_meta_bytes=343120 combine_payload_bytes=265986048 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266329168 ingress_bytes=369795072 link_bytes=369795072 token_bytes=532315216 weight_movement_bytes=103809024 total_nvlink_bytes=636124240 weight_to_token_ratio=0.195014 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391509085e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=342688 combine_payload_bytes=266412032 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266754720 ingress_bytes=335618048 link_bytes=335618048 token_bytes=533166752 weight_movement_bytes=69206016 total_nvlink_bytes=602372768 weight_to_token_ratio=0.129802 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262903695e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267116544 dispatch_meta_bytes=345104 combine_payload_bytes=267116544 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267461648 ingress_bytes=336322560 link_bytes=336322560 token_bytes=534578192 weight_movement_bytes=69206016 total_nvlink_bytes=603784208 weight_to_token_ratio=0.129459 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265554717e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344008 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266706888 ingress_bytes=335568896 link_bytes=335568896 token_bytes=533069768 weight_movement_bytes=69206016 total_nvlink_bytes=602275784 weight_to_token_ratio=0.129825 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262718740e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265609216 dispatch_meta_bytes=343440 combine_payload_bytes=265609216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265952656 ingress_bytes=300212224 link_bytes=300212224 token_bytes=531561872 weight_movement_bytes=34603008 total_nvlink_bytes=566164880 weight_to_token_ratio=0.065097 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129674430e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265814016 dispatch_meta_bytes=344256 combine_payload_bytes=265814016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266158272 ingress_bytes=300417024 link_bytes=300417024 token_bytes=531972288 weight_movement_bytes=34603008 total_nvlink_bytes=566575296 weight_to_token_ratio=0.065047 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130445076e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=344408 combine_payload_bytes=266395648 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266740056 ingress_bytes=335601664 link_bytes=335601664 token_bytes=533135704 weight_movement_bytes=69206016 total_nvlink_bytes=602341720 weight_to_token_ratio=0.129809 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262842043e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265609216 dispatch_meta_bytes=343440 combine_payload_bytes=265609216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265952656 ingress_bytes=300212224 link_bytes=300212224 token_bytes=531561872 weight_movement_bytes=34603008 total_nvlink_bytes=566164880 weight_to_token_ratio=0.065097 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129674430e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267083776 dispatch_meta_bytes=343768 combine_payload_bytes=267083776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267427544 ingress_bytes=301686784 link_bytes=301686784 token_bytes=534511320 weight_movement_bytes=34603008 total_nvlink_bytes=569114328 weight_to_token_ratio=0.064738 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135223080e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74144 payload_rows_remote=64498 dup_rows=24160 dup_rows_remote=20790 dedup_ratio=0.245768 router_tokens_per_expert=[776, 769, 769, 790, 788, 775, 760, 782, 758, 809, 764, 786, 784, 811, 767, 756, 795, 763, 773, 760, 771, 764, 767, 744, 759, 703, 769, 779, 765, 724, 784, 772, 771, 741, 765, 801, 750, 749, 784, 824, 785, 730, 781, 782, 750, 740, 806, 717, 801, 782, 778, 761, 755, 797, 742, 780, 762, 734, 730, 745, 804, 755, 765, 749] payload_rows_per_dst=[9386, 9402, 9260, 9050, 9250, 9316, 8834, 9646] dup_rows_per_dst=[3032, 3068, 3014, 2788, 3106, 3138, 2644, 3370] combine_rows_per_src=[9386, 9402, 9260, 9050, 9250, 9316, 8834, 9646] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74170 payload_rows_remote=64974 dup_rows=24134 dup_rows_remote=21224 dedup_ratio=0.245504 router_tokens_per_expert=[750, 737, 735, 752, 744, 741, 755, 784, 761, 722, 811, 772, 802, 766, 772, 778, 768, 740, 707, 765, 753, 758, 774, 788, 768, 770, 790, 731, 778, 758, 776, 803, 782, 740, 802, 750, 756, 760, 768, 779, 753, 753, 799, 773, 737, 799, 737, 751, 776, 773, 837, 818, 802, 770, 815, 751, 817, 758, 802, 804, 763, 743, 722, 753] payload_rows_per_dst=[9126, 9298, 9196, 9300, 9284, 9180, 9506, 9280] dup_rows_per_dst=[2870, 3070, 2910, 3048, 2990, 3024, 3178, 3044] combine_rows_per_src=[9126, 9298, 9196, 9300, 9284, 9180, 9506, 9280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266723328 dispatch_meta_bytes=344024 combine_payload_bytes=266723328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267067352 ingress_bytes=301326336 link_bytes=301326336 token_bytes=533790680 weight_movement_bytes=34603008 total_nvlink_bytes=568393688 weight_to_token_ratio=0.064825 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133866744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74134 payload_rows_remote=64804 dup_rows=24170 dup_rows_remote=21138 dedup_ratio=0.245870 router_tokens_per_expert=[792, 823, 765, 769, 749, 755, 744, 783, 713, 807, 805, 764, 762, 745, 765, 784, 790, 802, 791, 725, 773, 720, 730, 818, 784, 814, 757, 778, 824, 759, 780, 781, 791, 757, 763, 783, 814, 755, 758, 758, 763, 765, 772, 741, 797, 727, 782, 794, 787, 763, 748, 773, 743, 765, 752, 771, 779, 764, 768, 752, 729, 729, 724, 734] payload_rows_per_dst=[9330, 9258, 9604, 9352, 8978, 9244, 9236, 9132] dup_rows_per_dst=[3032, 3044, 3228, 3274, 2832, 3038, 2896, 2826] combine_rows_per_src=[9330, 9258, 9604, 9352, 8978, 9244, 9236, 9132] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74144 payload_rows_remote=64498 dup_rows=24160 dup_rows_remote=20790 dedup_ratio=0.245768 router_tokens_per_expert=[776, 769, 769, 790, 788, 775, 760, 782, 758, 809, 764, 786, 784, 811, 767, 756, 795, 763, 773, 760, 771, 764, 767, 744, 759, 703, 769, 779, 765, 724, 784, 772, 771, 741, 765, 801, 750, 749, 784, 824, 785, 730, 781, 782, 750, 740, 806, 717, 801, 782, 778, 761, 755, 797, 742, 780, 762, 734, 730, 745, 804, 755, 765, 749] payload_rows_per_dst=[9386, 9402, 9260, 9050, 9250, 9316, 8834, 9646] dup_rows_per_dst=[3032, 3068, 3014, 2788, 3106, 3138, 2644, 3370] combine_rows_per_src=[9386, 9402, 9260, 9050, 9250, 9316, 8834, 9646] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74344 payload_rows_remote=65006 dup_rows=23960 dup_rows_remote=20994 dedup_ratio=0.243734 router_tokens_per_expert=[783, 756, 819, 785, 774, 763, 760, 768, 816, 759, 781, 721, 775, 786, 765, 749, 750, 792, 766, 772, 742, 708, 760, 785, 768, 743, 793, 779, 744, 749, 772, 776, 766, 739, 803, 766, 807, 768, 769, 756, 769, 758, 735, 769, 782, 728, 781, 783, 748, 769, 814, 750, 730, 792, 831, 741, 739, 800, 758, 777, 777, 769, 766, 753] payload_rows_per_dst=[9374, 9338, 9214, 9306, 9340, 9236, 9266, 9270] dup_rows_per_dst=[3042, 2966, 2936, 2942, 3008, 2974, 3084, 3008] combine_rows_per_src=[9374, 9338, 9214, 9306, 9340, 9236, 9266, 9270] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74126 payload_rows_remote=64736 dup_rows=24178 dup_rows_remote=21102 dedup_ratio=0.245951 router_tokens_per_expert=[800, 783, 801, 746, 743, 776, 801, 771, 769, 783, 777, 766, 693, 757, 763, 790, 725, 781, 717, 754, 765, 754, 748, 805, 756, 824, 782, 752, 747, 803, 813, 763, 768, 797, 772, 783, 747, 757, 775, 825, 780, 735, 729, 764, 723, 752, 762, 763, 782, 782, 794, 739, 780, 814, 791, 751, 761, 785, 796, 765, 750, 799, 690, 733] payload_rows_per_dst=[9368, 9214, 9156, 9368, 9312, 9120, 9390, 9198] dup_rows_per_dst=[3074, 2982, 2942, 3112, 3136, 2896, 3076, 2960] combine_rows_per_src=[9368, 9214, 9156, 9368, 9312, 9120, 9390, 9198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74262 payload_rows_remote=64940 dup_rows=24042 dup_rows_remote=21104 dedup_ratio=0.244568 router_tokens_per_expert=[797, 738, 741, 796, 796, 738, 777, 767, 787, 732, 723, 789, 758, 737, 788, 775, 764, 795, 766, 791, 794, 773, 771, 764, 770, 771, 722, 758, 715, 822, 757, 737, 793, 744, 783, 759, 781, 776, 792, 753, 748, 744, 766, 782, 790, 812, 754, 734, 749, 807, 774, 769, 803, 783, 780, 803, 729, 761, 820, 759, 771, 745, 756, 723] payload_rows_per_dst=[9224, 9304, 9356, 9182, 9280, 9322, 9390, 9204] dup_rows_per_dst=[3076, 2874, 3080, 2922, 3082, 2938, 3146, 2924] combine_rows_per_src=[9224, 9304, 9356, 9182, 9280, 9322, 9390, 9204] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74252 payload_rows_remote=64828 dup_rows=24052 dup_rows_remote=20946 dedup_ratio=0.244670 router_tokens_per_expert=[756, 754, 757, 755, 766, 737, 767, 743, 752, 751, 729, 788, 800, 774, 751, 737, 800, 735, 685, 775, 793, 802, 755, 763, 741, 801, 744, 792, 778, 781, 758, 802, 831, 748, 788, 799, 773, 778, 761, 787, 795, 751, 769, 772, 763, 763, 804, 803, 712, 801, 774, 800, 776, 778, 754, 725, 775, 743, 752, 753, 738, 778, 818, 768] payload_rows_per_dst=[9164, 9212, 9218, 9358, 9424, 9406, 9222, 9248] dup_rows_per_dst=[2906, 2952, 2998, 3036, 3106, 3034, 3018, 3002] combine_rows_per_src=[9164, 9212, 9218, 9358, 9424, 9406, 9222, 9248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74246 payload_rows_remote=64980 dup_rows=24058 dup_rows_remote=21058 dedup_ratio=0.244731 router_tokens_per_expert=[766, 810, 804, 728, 728, 751, 760, 803, 758, 768, 734, 752, 774, 795, 772, 808, 776, 747, 778, 737, 704, 782, 764, 808, 810, 785, 744, 759, 743, 774, 740, 778, 767, 735, 769, 753, 768, 785, 762, 749, 783, 798, 775, 801, 741, 791, 749, 781, 768, 758, 737, 797, 769, 761, 794, 825, 770, 767, 752, 768, 783, 776, 707, 773] payload_rows_per_dst=[9284, 9252, 9202, 9266, 9274, 9342, 9326, 9300] dup_rows_per_dst=[3016, 3070, 2990, 3000, 2902, 3096, 3092, 2892] combine_rows_per_src=[9284, 9252, 9202, 9266, 9274, 9342, 9326, 9300] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74262 payload_rows_remote=64940 dup_rows=24042 dup_rows_remote=21104 dedup_ratio=0.244568 router_tokens_per_expert=[797, 738, 741, 796, 796, 738, 777, 767, 787, 732, 723, 789, 758, 737, 788, 775, 764, 795, 766, 791, 794, 773, 771, 764, 770, 771, 722, 758, 715, 822, 757, 737, 793, 744, 783, 759, 781, 776, 792, 753, 748, 744, 766, 782, 790, 812, 754, 734, 749, 807, 774, 769, 803, 783, 780, 803, 729, 761, 820, 759, 771, 745, 756, 723] payload_rows_per_dst=[9224, 9304, 9356, 9182, 9280, 9322, 9390, 9204] dup_rows_per_dst=[3076, 2874, 3080, 2922, 3082, 2938, 3146, 2924] combine_rows_per_src=[9224, 9304, 9356, 9182, 9280, 9322, 9390, 9204] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265437184 dispatch_meta_bytes=343768 combine_payload_bytes=265437184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265780952 ingress_bytes=300040192 link_bytes=300040192 token_bytes=531218136 weight_movement_bytes=34603008 total_nvlink_bytes=565821144 weight_to_token_ratio=0.065139 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129027087e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=264183808 dispatch_meta_bytes=341152 combine_payload_bytes=264183808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264524960 ingress_bytes=298786816 link_bytes=298786816 token_bytes=528708768 weight_movement_bytes=34603008 total_nvlink_bytes=563311776 weight_to_token_ratio=0.065448 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.124310734e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=65030 dup_rows=23928 dup_rows_remote=20972 dedup_ratio=0.243408 router_tokens_per_expert=[819, 733, 730, 782, 847, 798, 819, 713, 790, 799, 765, 734, 757, 765, 804, 813, 736, 795, 769, 728, 736, 778, 728, 827, 778, 765, 745, 797, 752, 765, 759, 756, 723, 800, 757, 767, 757, 791, 745, 710, 801, 767, 757, 756, 762, 741, 812, 771, 765, 772, 777, 765, 771, 743, 777, 781, 744, 724, 746, 776, 723, 820, 793, 776] payload_rows_per_dst=[9356, 9352, 9298, 9312, 9172, 9316, 9346, 9224] dup_rows_per_dst=[3126, 3102, 2896, 2922, 2928, 3018, 2956, 2980] combine_rows_per_src=[9356, 9352, 9298, 9312, 9172, 9316, 9346, 9224] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=344792 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266478296 ingress_bytes=300736512 link_bytes=300736512 token_bytes=532611800 weight_movement_bytes=34603008 total_nvlink_bytes=567214808 weight_to_token_ratio=0.064969 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131647283e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265437184 dispatch_meta_bytes=343768 combine_payload_bytes=265437184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265780952 ingress_bytes=300040192 link_bytes=300040192 token_bytes=531218136 weight_movement_bytes=34603008 total_nvlink_bytes=565821144 weight_to_token_ratio=0.065139 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129027087e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266264576 dispatch_meta_bytes=344000 combine_payload_bytes=266264576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266608576 ingress_bytes=300867584 link_bytes=300867584 token_bytes=532873152 weight_movement_bytes=34603008 total_nvlink_bytes=567476160 weight_to_token_ratio=0.064937 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132140497e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265158656 dispatch_meta_bytes=343352 combine_payload_bytes=265158656 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265502008 ingress_bytes=368967680 link_bytes=368967680 token_bytes=530660664 weight_movement_bytes=103809024 total_nvlink_bytes=634469688 weight_to_token_ratio=0.195622 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388395676e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=344176 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266338416 ingress_bytes=300597248 link_bytes=300597248 token_bytes=532332656 weight_movement_bytes=34603008 total_nvlink_bytes=566935664 weight_to_token_ratio=0.065003 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131123244e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=64938 dup_rows=23946 dup_rows_remote=20842 dedup_ratio=0.243591 router_tokens_per_expert=[774, 707, 772, 773, 752, 764, 786, 788, 765, 730, 801, 749, 745, 778, 815, 801, 790, 801, 796, 744, 803, 771, 788, 769, 829, 754, 768, 743, 797, 793, 741, 761, 781, 757, 769, 794, 751, 750, 799, 781, 766, 762, 736, 737, 775, 786, 744, 775, 756, 788, 741, 752, 792, 775, 747, 806, 768, 733, 767, 745, 758, 770, 721, 722] payload_rows_per_dst=[9230, 9346, 9420, 9272, 9366, 9244, 9346, 9134] dup_rows_per_dst=[3002, 3022, 3104, 3100, 2998, 2918, 2968, 2834] combine_rows_per_src=[9230, 9346, 9420, 9272, 9366, 9244, 9346, 9134] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265535488 dispatch_meta_bytes=343096 combine_payload_bytes=265535488 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=265878584 ingress_bytes=403947520 link_bytes=403947520 token_bytes=531414072 weight_movement_bytes=138412032 total_nvlink_bytes=669826104 weight_to_token_ratio=0.260460 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520021997e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266158080 dispatch_meta_bytes=344152 combine_payload_bytes=266158080 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266502232 ingress_bytes=335364096 link_bytes=335364096 token_bytes=532660312 weight_movement_bytes=69206016 total_nvlink_bytes=601866328 weight_to_token_ratio=0.129925 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261948094e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=344176 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266338416 ingress_bytes=300597248 link_bytes=300597248 token_bytes=532332656 weight_movement_bytes=34603008 total_nvlink_bytes=566935664 weight_to_token_ratio=0.065003 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131123244e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344008 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266706888 ingress_bytes=335568896 link_bytes=335568896 token_bytes=533069768 weight_movement_bytes=69206016 total_nvlink_bytes=602275784 weight_to_token_ratio=0.129825 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262718740e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74426 payload_rows_remote=64958 dup_rows=23878 dup_rows_remote=20776 dedup_ratio=0.242900 router_tokens_per_expert=[767, 766, 753, 763, 788, 785, 735, 815, 762, 762, 784, 803, 772, 731, 745, 788, 747, 743, 768, 761, 788, 764, 760, 747, 794, 761, 769, 770, 786, 778, 826, 768, 769, 726, 782, 808, 778, 778, 795, 762, 780, 755, 798, 815, 760, 759, 759, 736, 777, 732, 749, 762, 732, 747, 711, 768, 730, 790, 751, 755, 781, 798, 782, 778] payload_rows_per_dst=[9310, 9274, 9196, 9448, 9408, 9200, 9122, 9468] dup_rows_per_dst=[3034, 3020, 2960, 3054, 2988, 2884, 2836, 3102] combine_rows_per_src=[9310, 9274, 9196, 9448, 9408, 9200, 9122, 9468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265986048 dispatch_meta_bytes=343120 combine_payload_bytes=265986048 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266329168 ingress_bytes=369795072 link_bytes=369795072 token_bytes=532315216 weight_movement_bytes=103809024 total_nvlink_bytes=636124240 weight_to_token_ratio=0.195014 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391509085e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74486 payload_rows_remote=65186 dup_rows=23818 dup_rows_remote=20750 dedup_ratio=0.242289 router_tokens_per_expert=[781, 755, 747, 797, 791, 774, 743, 734, 733, 810, 770, 712, 818, 762, 735, 749, 717, 748, 748, 774, 801, 781, 759, 776, 785, 771, 762, 790, 748, 775, 786, 756, 743, 817, 752, 735, 789, 773, 794, 760, 790, 780, 797, 767, 782, 755, 761, 764, 768, 780, 746, 759, 781, 822, 755, 744, 775, 778, 822, 747, 732, 729, 792, 775] payload_rows_per_dst=[9300, 9708, 9006, 9150, 9250, 9386, 9336, 9350] dup_rows_per_dst=[3068, 3356, 2672, 2840, 2952, 3058, 2922, 2950] combine_rows_per_src=[9300, 9708, 9006, 9150, 9250, 9386, 9336, 9350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74426 payload_rows_remote=64958 dup_rows=23878 dup_rows_remote=20776 dedup_ratio=0.242900 router_tokens_per_expert=[767, 766, 753, 763, 788, 785, 735, 815, 762, 762, 784, 803, 772, 731, 745, 788, 747, 743, 768, 761, 788, 764, 760, 747, 794, 761, 769, 770, 786, 778, 826, 768, 769, 726, 782, 808, 778, 778, 795, 762, 780, 755, 798, 815, 760, 759, 759, 736, 777, 732, 749, 762, 732, 747, 711, 768, 730, 790, 751, 755, 781, 798, 782, 778] payload_rows_per_dst=[9310, 9274, 9196, 9448, 9408, 9200, 9122, 9468] dup_rows_per_dst=[3034, 3020, 2960, 3054, 2988, 2884, 2836, 3102] combine_rows_per_src=[9310, 9274, 9196, 9448, 9408, 9200, 9122, 9468] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74320 payload_rows_remote=64902 dup_rows=23984 dup_rows_remote=21158 dedup_ratio=0.243978 router_tokens_per_expert=[778, 758, 749, 710, 770, 756, 754, 803, 758, 757, 728, 779, 797, 758, 804, 741, 745, 735, 790, 798, 736, 780, 814, 809, 786, 771, 743, 779, 750, 772, 806, 732, 746, 763, 785, 771, 757, 754, 787, 727, 773, 775, 792, 782, 801, 771, 751, 805, 812, 772, 785, 760, 769, 811, 813, 821, 735, 713, 777, 752, 722, 730, 717, 777] payload_rows_per_dst=[9248, 9418, 9318, 9216, 9232, 9338, 9476, 9074] dup_rows_per_dst=[2908, 2826, 3096, 3062, 2948, 3162, 3210, 2772] combine_rows_per_src=[9248, 9418, 9318, 9216, 9232, 9338, 9476, 9074] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74310 payload_rows_remote=64966 dup_rows=23994 dup_rows_remote=20992 dedup_ratio=0.244080 router_tokens_per_expert=[759, 784, 773, 764, 779, 732, 733, 777, 723, 802, 755, 780, 769, 813, 771, 735, 781, 795, 809, 764, 723, 748, 770, 783, 766, 747, 779, 775, 739, 750, 782, 774, 794, 775, 797, 785, 732, 774, 736, 742, 737, 754, 754, 795, 755, 764, 769, 785, 768, 731, 785, 785, 757, 776, 787, 740, 778, 792, 752, 788, 821, 769, 776, 765] payload_rows_per_dst=[9198, 9340, 9344, 9204, 9316, 9276, 9304, 9328] dup_rows_per_dst=[3004, 2956, 3002, 3020, 2954, 2950, 2954, 3154] combine_rows_per_src=[9198, 9340, 9344, 9204, 9316, 9276, 9304, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74300 payload_rows_remote=64914 dup_rows=24004 dup_rows_remote=21012 dedup_ratio=0.244181 router_tokens_per_expert=[763, 741, 714, 772, 784, 797, 760, 792, 768, 772, 726, 744, 747, 745, 787, 756, 760, 745, 742, 758, 791, 807, 782, 777, 717, 752, 798, 775, 761, 790, 757, 776, 817, 795, 753, 823, 793, 755, 787, 815, 776, 716, 780, 749, 721, 793, 728, 746, 773, 779, 762, 785, 741, 785, 769, 795, 787, 764, 819, 783, 776, 734, 745, 752] payload_rows_per_dst=[9340, 9190, 9286, 9248, 9428, 9064, 9386, 9358] dup_rows_per_dst=[2906, 2900, 3038, 3004, 3248, 2954, 2992, 2962] combine_rows_per_src=[9340, 9190, 9286, 9248, 9428, 9064, 9386, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74362 payload_rows_remote=65068 dup_rows=23942 dup_rows_remote=20906 dedup_ratio=0.243551 router_tokens_per_expert=[747, 742, 755, 779, 775, 792, 782, 794, 746, 793, 789, 746, 781, 722, 746, 770, 767, 806, 778, 777, 726, 768, 748, 749, 786, 798, 744, 794, 720, 780, 798, 807, 773, 782, 777, 757, 755, 744, 755, 790, 793, 829, 722, 755, 781, 795, 738, 752, 751, 743, 772, 788, 769, 765, 739, 780, 778, 756, 773, 737, 795, 733, 825, 745] payload_rows_per_dst=[9306, 9284, 9284, 9386, 9286, 9294, 9270, 9252] dup_rows_per_dst=[3026, 2902, 2954, 3068, 2980, 3036, 2944, 3032] combine_rows_per_src=[9306, 9284, 9284, 9386, 9286, 9294, 9270, 9252] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=65004 dup_rows=24016 dup_rows_remote=20986 dedup_ratio=0.244303 router_tokens_per_expert=[725, 727, 809, 809, 735, 763, 743, 762, 709, 734, 740, 754, 743, 813, 758, 751, 731, 773, 786, 791, 790, 819, 826, 756, 765, 793, 729, 791, 741, 797, 816, 755, 766, 739, 785, 804, 778, 752, 790, 743, 758, 772, 757, 788, 753, 765, 781, 731, 797, 761, 735, 814, 774, 745, 769, 787, 786, 777, 808, 753, 763, 744, 768, 775] payload_rows_per_dst=[9246, 9172, 9384, 9340, 9284, 9304, 9324, 9234] dup_rows_per_dst=[2900, 2832, 3160, 3034, 3030, 2906, 3040, 3114] combine_rows_per_src=[9246, 9172, 9384, 9340, 9284, 9304, 9324, 9234] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74362 payload_rows_remote=65068 dup_rows=23942 dup_rows_remote=20906 dedup_ratio=0.243551 router_tokens_per_expert=[747, 742, 755, 779, 775, 792, 782, 794, 746, 793, 789, 746, 781, 722, 746, 770, 767, 806, 778, 777, 726, 768, 748, 749, 786, 798, 744, 794, 720, 780, 798, 807, 773, 782, 777, 757, 755, 744, 755, 790, 793, 829, 722, 755, 781, 795, 738, 752, 751, 743, 772, 788, 769, 765, 739, 780, 778, 756, 773, 737, 795, 733, 825, 745] payload_rows_per_dst=[9306, 9284, 9284, 9386, 9286, 9294, 9270, 9252] dup_rows_per_dst=[3026, 2902, 2954, 3068, 2980, 3036, 2944, 3032] combine_rows_per_src=[9306, 9284, 9284, 9386, 9286, 9294, 9270, 9252] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74688 payload_rows_remote=65360 dup_rows=23616 dup_rows_remote=20714 dedup_ratio=0.240234 router_tokens_per_expert=[798, 781, 759, 791, 782, 778, 783, 783, 738, 791, 758, 758, 778, 739, 725, 776, 798, 775, 801, 792, 762, 795, 758, 821, 778, 814, 754, 753, 755, 752, 737, 772, 797, 768, 742, 714, 780, 772, 700, 727, 739, 805, 854, 801, 746, 819, 769, 713, 789, 758, 742, 751, 742, 760, 771, 781, 743, 751, 742, 761, 766, 783, 775, 756] payload_rows_per_dst=[9422, 9246, 9564, 9328, 9158, 9446, 9294, 9230] dup_rows_per_dst=[3088, 2880, 3040, 2902, 2842, 3046, 2894, 2924] combine_rows_per_src=[9422, 9246, 9564, 9328, 9158, 9446, 9294, 9230] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=267001856 dispatch_meta_bytes=343744 combine_payload_bytes=267001856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267345600 ingress_bytes=301604864 link_bytes=301604864 token_bytes=534347456 weight_movement_bytes=34603008 total_nvlink_bytes=568950464 weight_to_token_ratio=0.064758 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134914822e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=342936 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266410904 ingress_bytes=300670976 link_bytes=300670976 token_bytes=532478872 weight_movement_bytes=34603008 total_nvlink_bytes=567081880 weight_to_token_ratio=0.064985 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131400677e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74126 payload_rows_remote=64736 dup_rows=24178 dup_rows_remote=21102 dedup_ratio=0.245951 router_tokens_per_expert=[800, 783, 801, 746, 743, 776, 801, 771, 769, 783, 777, 766, 693, 757, 763, 790, 725, 781, 717, 754, 765, 754, 748, 805, 756, 824, 782, 752, 747, 803, 813, 763, 768, 797, 772, 783, 747, 757, 775, 825, 780, 735, 729, 764, 723, 752, 762, 763, 782, 782, 794, 739, 780, 814, 791, 751, 761, 785, 796, 765, 750, 799, 690, 733] payload_rows_per_dst=[9368, 9214, 9156, 9368, 9312, 9120, 9390, 9198] dup_rows_per_dst=[3074, 2982, 2942, 3112, 3136, 2896, 3076, 2960] combine_rows_per_src=[9368, 9214, 9156, 9368, 9312, 9120, 9390, 9198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265838592 dispatch_meta_bytes=344240 combine_payload_bytes=265838592 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266182832 ingress_bytes=300441600 link_bytes=300441600 token_bytes=532021424 weight_movement_bytes=34603008 total_nvlink_bytes=566624432 weight_to_token_ratio=0.065041 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130537553e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=267001856 dispatch_meta_bytes=343744 combine_payload_bytes=267001856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267345600 ingress_bytes=301604864 link_bytes=301604864 token_bytes=534347456 weight_movement_bytes=34603008 total_nvlink_bytes=568950464 weight_to_token_ratio=0.064758 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134914822e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74170 payload_rows_remote=64974 dup_rows=24134 dup_rows_remote=21224 dedup_ratio=0.245504 router_tokens_per_expert=[750, 737, 735, 752, 744, 741, 755, 784, 761, 722, 811, 772, 802, 766, 772, 778, 768, 740, 707, 765, 753, 758, 774, 788, 768, 770, 790, 731, 778, 758, 776, 803, 782, 740, 802, 750, 756, 760, 768, 779, 753, 753, 799, 773, 737, 799, 737, 751, 776, 773, 837, 818, 802, 770, 815, 751, 817, 758, 802, 804, 763, 743, 722, 753] payload_rows_per_dst=[9126, 9298, 9196, 9300, 9284, 9180, 9506, 9280] dup_rows_per_dst=[2870, 3070, 2910, 3048, 2990, 3024, 3178, 3044] combine_rows_per_src=[9126, 9298, 9196, 9300, 9284, 9180, 9506, 9280] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266100736 dispatch_meta_bytes=343832 combine_payload_bytes=266100736 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266444568 ingress_bytes=335306752 link_bytes=335306752 token_bytes=532545304 weight_movement_bytes=69206016 total_nvlink_bytes=601751320 weight_to_token_ratio=0.129953 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261732313e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266256384 dispatch_meta_bytes=343960 combine_payload_bytes=266256384 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266600344 ingress_bytes=300859392 link_bytes=300859392 token_bytes=532856728 weight_movement_bytes=34603008 total_nvlink_bytes=567459736 weight_to_token_ratio=0.064939 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132109671e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265887744 dispatch_meta_bytes=343704 combine_payload_bytes=265887744 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266231448 ingress_bytes=335093760 link_bytes=335093760 token_bytes=532119192 weight_movement_bytes=69206016 total_nvlink_bytes=601325208 weight_to_token_ratio=0.130057 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260930842e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266518528 dispatch_meta_bytes=343896 combine_payload_bytes=266518528 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266862424 ingress_bytes=335724544 link_bytes=335724544 token_bytes=533380952 weight_movement_bytes=69206016 total_nvlink_bytes=602586968 weight_to_token_ratio=0.129750 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263304431e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266256384 dispatch_meta_bytes=343960 combine_payload_bytes=266256384 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266600344 ingress_bytes=300859392 link_bytes=300859392 token_bytes=532856728 weight_movement_bytes=34603008 total_nvlink_bytes=567459736 weight_to_token_ratio=0.064939 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132109671e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=267714560 dispatch_meta_bytes=344296 combine_payload_bytes=267714560 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=268058856 ingress_bytes=336920576 link_bytes=336920576 token_bytes=535773416 weight_movement_bytes=69206016 total_nvlink_bytes=604979432 weight_to_token_ratio=0.129170 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.267805003e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265158656 dispatch_meta_bytes=343352 combine_payload_bytes=265158656 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265502008 ingress_bytes=368967680 link_bytes=368967680 token_bytes=530660664 weight_movement_bytes=103809024 total_nvlink_bytes=634469688 weight_to_token_ratio=0.195622 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388395676e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=344792 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266478296 ingress_bytes=300736512 link_bytes=300736512 token_bytes=532611800 weight_movement_bytes=34603008 total_nvlink_bytes=567214808 weight_to_token_ratio=0.064969 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131647283e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74410 payload_rows_remote=64996 dup_rows=23894 dup_rows_remote=20786 dedup_ratio=0.243062 router_tokens_per_expert=[759, 783, 742, 731, 769, 763, 780, 796, 814, 776, 786, 795, 758, 792, 736, 763, 788, 740, 762, 752, 773, 770, 764, 711, 781, 773, 736, 772, 777, 765, 760, 803, 757, 755, 806, 753, 769, 792, 776, 740, 769, 789, 781, 742, 772, 743, 787, 768, 743, 775, 751, 807, 780, 823, 788, 770, 768, 797, 756, 708, 751, 711, 772, 783] payload_rows_per_dst=[9414, 9456, 9328, 9214, 9334, 9076, 9380, 9208] dup_rows_per_dst=[3108, 3134, 2868, 2996, 2962, 2960, 2982, 2884] combine_rows_per_src=[9414, 9456, 9328, 9214, 9334, 9076, 9380, 9208] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74246 payload_rows_remote=64578 dup_rows=24058 dup_rows_remote=20864 dedup_ratio=0.244731 router_tokens_per_expert=[761, 768, 786, 794, 792, 782, 778, 805, 762, 785, 716, 774, 764, 748, 761, 739, 756, 751, 798, 818, 818, 752, 767, 759, 766, 768, 790, 732, 792, 733, 818, 784, 816, 723, 717, 775, 772, 759, 786, 778, 769, 730, 780, 750, 723, 786, 779, 765, 740, 783, 756, 751, 770, 741, 769, 801, 802, 758, 788, 775, 762, 738, 754, 739] payload_rows_per_dst=[9102, 9126, 9250, 9308, 9408, 9106, 9278, 9668] dup_rows_per_dst=[2800, 2972, 2966, 3058, 3066, 3058, 2944, 3194] combine_rows_per_src=[9102, 9126, 9250, 9308, 9408, 9106, 9278, 9668] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=64934 dup_rows=23974 dup_rows_remote=20856 dedup_ratio=0.243876 router_tokens_per_expert=[761, 751, 729, 745, 780, 741, 751, 815, 825, 758, 778, 802, 765, 787, 756, 786, 781, 775, 738, 781, 769, 781, 787, 725, 797, 762, 790, 773, 751, 685, 745, 774, 789, 746, 736, 725, 755, 759, 719, 805, 774, 777, 772, 768, 756, 722, 794, 761, 803, 800, 773, 775, 761, 765, 779, 811, 773, 796, 745, 817, 753, 767, 712, 820] payload_rows_per_dst=[9182, 9396, 9270, 9188, 9214, 9306, 9480, 9294] dup_rows_per_dst=[2964, 3118, 3004, 2966, 2854, 2942, 3054, 3072] combine_rows_per_src=[9182, 9396, 9270, 9188, 9214, 9306, 9480, 9294] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74410 payload_rows_remote=64996 dup_rows=23894 dup_rows_remote=20786 dedup_ratio=0.243062 router_tokens_per_expert=[759, 783, 742, 731, 769, 763, 780, 796, 814, 776, 786, 795, 758, 792, 736, 763, 788, 740, 762, 752, 773, 770, 764, 711, 781, 773, 736, 772, 777, 765, 760, 803, 757, 755, 806, 753, 769, 792, 776, 740, 769, 789, 781, 742, 772, 743, 787, 768, 743, 775, 751, 807, 780, 823, 788, 770, 768, 797, 756, 708, 751, 711, 772, 783] payload_rows_per_dst=[9414, 9456, 9328, 9214, 9334, 9076, 9380, 9208] dup_rows_per_dst=[3108, 3134, 2868, 2996, 2962, 2960, 2982, 2884] combine_rows_per_src=[9414, 9456, 9328, 9214, 9334, 9076, 9380, 9208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74174 payload_rows_remote=65030 dup_rows=24130 dup_rows_remote=21126 dedup_ratio=0.245463 router_tokens_per_expert=[778, 771, 778, 790, 751, 798, 768, 756, 800, 732, 747, 760, 736, 817, 791, 796, 788, 776, 739, 760, 735, 759, 765, 707, 763, 758, 787, 756, 782, 742, 754, 743, 760, 767, 766, 743, 817, 755, 731, 770, 798, 792, 793, 801, 731, 849, 815, 799, 765, 790, 771, 738, 769, 752, 713, 776, 751, 773, 766, 746, 772, 782, 746, 772] payload_rows_per_dst=[9380, 9272, 9082, 9208, 9326, 9512, 9144, 9250] dup_rows_per_dst=[3000, 3086, 2976, 2962, 2892, 3244, 3004, 2966] combine_rows_per_src=[9380, 9272, 9082, 9208, 9326, 9512, 9144, 9250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=65024 dup_rows=24056 dup_rows_remote=21194 dedup_ratio=0.244710 router_tokens_per_expert=[789, 734, 761, 757, 698, 748, 790, 784, 702, 798, 771, 755, 739, 784, 758, 749, 820, 784, 778, 790, 778, 779, 781, 798, 754, 794, 761, 790, 768, 769, 758, 795, 791, 778, 765, 704, 779, 777, 747, 753, 755, 780, 757, 774, 713, 747, 780, 737, 792, 789, 766, 734, 755, 797, 775, 795, 817, 743, 804, 755, 762, 779, 761, 777] payload_rows_per_dst=[9138, 9292, 9380, 9328, 9204, 9224, 9362, 9320] dup_rows_per_dst=[2984, 2820, 3236, 3050, 2984, 2862, 3044, 3076] combine_rows_per_src=[9138, 9292, 9380, 9328, 9204, 9224, 9362, 9320] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=64864 dup_rows=24040 dup_rows_remote=21076 dedup_ratio=0.244548 router_tokens_per_expert=[788, 737, 760, 773, 784, 784, 715, 782, 745, 816, 734, 743, 757, 772, 762, 740, 793, 764, 722, 731, 751, 781, 775, 776, 791, 785, 772, 808, 786, 771, 789, 835, 784, 765, 757, 774, 778, 764, 761, 799, 770, 780, 770, 749, 803, 723, 762, 781, 761, 775, 730, 776, 776, 777, 749, 774, 779, 738, 765, 803, 754, 788, 755, 710] payload_rows_per_dst=[9196, 9162, 9148, 9574, 9400, 9306, 9248, 9230] dup_rows_per_dst=[3050, 2976, 3038, 3100, 2964, 2970, 2988, 2954] combine_rows_per_src=[9196, 9162, 9148, 9574, 9400, 9306, 9248, 9230] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74280 payload_rows_remote=65000 dup_rows=24024 dup_rows_remote=20932 dedup_ratio=0.244385 router_tokens_per_expert=[744, 791, 762, 797, 773, 816, 835, 752, 806, 745, 793, 792, 781, 754, 759, 753, 781, 780, 718, 777, 807, 795, 741, 787, 774, 727, 769, 800, 762, 750, 790, 740, 748, 778, 803, 800, 717, 775, 795, 772, 728, 775, 761, 779, 768, 773, 775, 778, 776, 795, 751, 732, 787, 807, 763, 708, 727, 740, 763, 759, 786, 740, 733, 709] payload_rows_per_dst=[9408, 9396, 9280, 9214, 9372, 9208, 9308, 9094] dup_rows_per_dst=[3132, 2970, 3092, 3010, 3004, 3066, 2930, 2820] combine_rows_per_src=[9408, 9396, 9280, 9214, 9372, 9208, 9308, 9094] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=343128 combine_payload_bytes=266223616 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266566744 ingress_bytes=335429632 link_bytes=335429632 token_bytes=532790360 weight_movement_bytes=69206016 total_nvlink_bytes=601996376 weight_to_token_ratio=0.129894 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262194701e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=264511488 dispatch_meta_bytes=341768 combine_payload_bytes=264511488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264853256 ingress_bytes=299114496 link_bytes=299114496 token_bytes=529364744 weight_movement_bytes=34603008 total_nvlink_bytes=563967752 weight_to_token_ratio=0.065367 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125543768e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265969664 dispatch_meta_bytes=343160 combine_payload_bytes=265969664 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266312824 ingress_bytes=300572672 link_bytes=300572672 token_bytes=532282488 weight_movement_bytes=34603008 total_nvlink_bytes=566885496 weight_to_token_ratio=0.065009 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131030767e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=64864 dup_rows=24040 dup_rows_remote=21076 dedup_ratio=0.244548 router_tokens_per_expert=[788, 737, 760, 773, 784, 784, 715, 782, 745, 816, 734, 743, 757, 772, 762, 740, 793, 764, 722, 731, 751, 781, 775, 776, 791, 785, 772, 808, 786, 771, 789, 835, 784, 765, 757, 774, 778, 764, 761, 799, 770, 780, 770, 749, 803, 723, 762, 781, 761, 775, 730, 776, 776, 777, 749, 774, 779, 738, 765, 803, 754, 788, 755, 710] payload_rows_per_dst=[9196, 9162, 9148, 9574, 9400, 9306, 9248, 9230] dup_rows_per_dst=[3050, 2976, 3038, 3100, 2964, 2970, 2988, 2954] combine_rows_per_src=[9196, 9162, 9148, 9574, 9400, 9306, 9248, 9230] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=65024 dup_rows=24056 dup_rows_remote=21194 dedup_ratio=0.244710 router_tokens_per_expert=[789, 734, 761, 757, 698, 748, 790, 784, 702, 798, 771, 755, 739, 784, 758, 749, 820, 784, 778, 790, 778, 779, 781, 798, 754, 794, 761, 790, 768, 769, 758, 795, 791, 778, 765, 704, 779, 777, 747, 753, 755, 780, 757, 774, 713, 747, 780, 737, 792, 789, 766, 734, 755, 797, 775, 795, 817, 743, 804, 755, 762, 779, 761, 777] payload_rows_per_dst=[9138, 9292, 9380, 9328, 9204, 9224, 9362, 9320] dup_rows_per_dst=[2984, 2820, 3236, 3050, 2984, 2862, 3044, 3076] combine_rows_per_src=[9138, 9292, 9380, 9328, 9204, 9224, 9362, 9320] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74300 payload_rows_remote=64914 dup_rows=24004 dup_rows_remote=21012 dedup_ratio=0.244181 router_tokens_per_expert=[763, 741, 714, 772, 784, 797, 760, 792, 768, 772, 726, 744, 747, 745, 787, 756, 760, 745, 742, 758, 791, 807, 782, 777, 717, 752, 798, 775, 761, 790, 757, 776, 817, 795, 753, 823, 793, 755, 787, 815, 776, 716, 780, 749, 721, 793, 728, 746, 773, 779, 762, 785, 741, 785, 769, 795, 787, 764, 819, 783, 776, 734, 745, 752] payload_rows_per_dst=[9340, 9190, 9286, 9248, 9428, 9064, 9386, 9358] dup_rows_per_dst=[2906, 2900, 3038, 3004, 3248, 2954, 2992, 2962] combine_rows_per_src=[9340, 9190, 9286, 9248, 9428, 9064, 9386, 9358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74150 payload_rows_remote=64988 dup_rows=24154 dup_rows_remote=21188 dedup_ratio=0.245707 router_tokens_per_expert=[766, 801, 788, 734, 781, 801, 796, 756, 762, 736, 753, 749, 732, 797, 741, 794, 751, 771, 767, 787, 790, 774, 757, 796, 757, 731, 763, 732, 775, 762, 752, 792, 795, 730, 757, 777, 811, 745, 752, 793, 786, 785, 736, 785, 728, 809, 788, 815, 716, 809, 747, 761, 772, 733, 751, 790, 730, 734, 757, 782, 795, 805, 763, 771] payload_rows_per_dst=[9322, 9178, 9290, 9162, 9258, 9372, 9250, 9318] dup_rows_per_dst=[3124, 2950, 3096, 2966, 3062, 3092, 2908, 2956] combine_rows_per_src=[9322, 9178, 9290, 9162, 9258, 9372, 9250, 9318] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=343128 combine_payload_bytes=266223616 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266566744 ingress_bytes=335429632 link_bytes=335429632 token_bytes=532790360 weight_movement_bytes=69206016 total_nvlink_bytes=601996376 weight_to_token_ratio=0.129894 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262194701e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266240000 dispatch_meta_bytes=343728 combine_payload_bytes=266240000 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266583728 ingress_bytes=335446016 link_bytes=335446016 token_bytes=532823728 weight_movement_bytes=69206016 total_nvlink_bytes=602029744 weight_to_token_ratio=0.129885 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262256353e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265969664 dispatch_meta_bytes=343160 combine_payload_bytes=265969664 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266312824 ingress_bytes=300572672 link_bytes=300572672 token_bytes=532282488 weight_movement_bytes=34603008 total_nvlink_bytes=566885496 weight_to_token_ratio=0.065009 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131030767e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74310 payload_rows_remote=64966 dup_rows=23994 dup_rows_remote=20992 dedup_ratio=0.244080 router_tokens_per_expert=[759, 784, 773, 764, 779, 732, 733, 777, 723, 802, 755, 780, 769, 813, 771, 735, 781, 795, 809, 764, 723, 748, 770, 783, 766, 747, 779, 775, 739, 750, 782, 774, 794, 775, 797, 785, 732, 774, 736, 742, 737, 754, 754, 795, 755, 764, 769, 785, 768, 731, 785, 785, 757, 776, 787, 740, 778, 792, 752, 788, 821, 769, 776, 765] payload_rows_per_dst=[9198, 9340, 9344, 9204, 9316, 9276, 9304, 9328] dup_rows_per_dst=[3004, 2956, 3002, 3020, 2954, 2950, 2954, 3154] combine_rows_per_src=[9198, 9340, 9344, 9204, 9316, 9276, 9304, 9328] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265682944 dispatch_meta_bytes=343760 combine_payload_bytes=265682944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266026704 ingress_bytes=300285952 link_bytes=300285952 token_bytes=531709648 weight_movement_bytes=34603008 total_nvlink_bytes=566312656 weight_to_token_ratio=0.065079 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129951862e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=344872 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266683176 ingress_bytes=335544320 link_bytes=335544320 token_bytes=533021480 weight_movement_bytes=69206016 total_nvlink_bytes=602227496 weight_to_token_ratio=0.129837 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262626263e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265887744 dispatch_meta_bytes=343704 combine_payload_bytes=265887744 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266231448 ingress_bytes=335093760 link_bytes=335093760 token_bytes=532119192 weight_movement_bytes=69206016 total_nvlink_bytes=601325208 weight_to_token_ratio=0.130057 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260930842e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265682944 dispatch_meta_bytes=343760 combine_payload_bytes=265682944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266026704 ingress_bytes=300285952 link_bytes=300285952 token_bytes=531709648 weight_movement_bytes=34603008 total_nvlink_bytes=566312656 weight_to_token_ratio=0.065079 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129951862e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344624 combine_payload_bytes=266362880 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266707504 ingress_bytes=370171904 link_bytes=370171904 token_bytes=533070384 weight_movement_bytes=103809024 total_nvlink_bytes=636879408 weight_to_token_ratio=0.194738 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392927073e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266190848 dispatch_meta_bytes=344704 combine_payload_bytes=266190848 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266535552 ingress_bytes=335396864 link_bytes=335396864 token_bytes=532726400 weight_movement_bytes=69206016 total_nvlink_bytes=601932416 weight_to_token_ratio=0.129909 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262071398e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74170 payload_rows_remote=64756 dup_rows=24134 dup_rows_remote=21026 dedup_ratio=0.245504 router_tokens_per_expert=[772, 730, 772, 792, 761, 773, 777, 760, 755, 750, 759, 748, 803, 782, 733, 787, 765, 806, 754, 786, 827, 717, 774, 758, 773, 729, 759, 767, 743, 747, 770, 827, 725, 771, 775, 796, 762, 798, 819, 806, 765, 731, 749, 798, 800, 760, 721, 756, 740, 746, 770, 746, 767, 746, 779, 765, 770, 808, 773, 797, 755, 777, 749, 776] payload_rows_per_dst=[9260, 9252, 9050, 9206, 9274, 9370, 9344, 9414] dup_rows_per_dst=[3014, 2982, 2926, 3024, 2800, 3220, 3060, 3108] combine_rows_per_src=[9260, 9252, 9050, 9206, 9274, 9370, 9344, 9414] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74314 payload_rows_remote=65172 dup_rows=23990 dup_rows_remote=21026 dedup_ratio=0.244039 router_tokens_per_expert=[795, 739, 793, 752, 784, 791, 781, 768, 730, 731, 774, 752, 769, 777, 732, 788, 786, 763, 773, 778, 782, 763, 772, 725, 824, 779, 745, 800, 759, 743, 755, 812, 786, 752, 751, 769, 768, 761, 735, 761, 818, 795, 756, 730, 744, 726, 729, 822, 824, 800, 772, 795, 758, 761, 752, 750, 782, 757, 809, 761, 747, 749, 752, 765] payload_rows_per_dst=[9434, 9142, 9290, 9266, 9226, 9288, 9474, 9194] dup_rows_per_dst=[2972, 2964, 2994, 3168, 2940, 2952, 2950, 3050] combine_rows_per_src=[9434, 9142, 9290, 9266, 9226, 9288, 9474, 9194] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266100736 dispatch_meta_bytes=343832 combine_payload_bytes=266100736 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266444568 ingress_bytes=335306752 link_bytes=335306752 token_bytes=532545304 weight_movement_bytes=69206016 total_nvlink_bytes=601751320 weight_to_token_ratio=0.129953 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261732313e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74152 payload_rows_remote=64678 dup_rows=24152 dup_rows_remote=21124 dedup_ratio=0.245687 router_tokens_per_expert=[808, 761, 778, 794, 724, 791, 751, 762, 752, 802, 715, 772, 828, 788, 789, 748, 786, 727, 781, 789, 791, 780, 809, 752, 715, 751, 777, 803, 778, 757, 758, 768, 777, 756, 787, 757, 780, 828, 767, 758, 756, 785, 719, 734, 756, 773, 787, 767, 789, 790, 755, 779, 752, 787, 727, 752, 719, 731, 795, 785, 792, 733, 745, 749] payload_rows_per_dst=[9474, 9472, 9392, 9158, 9026, 9180, 9268, 9182] dup_rows_per_dst=[3028, 3320, 3098, 3006, 2816, 2974, 2994, 2916] combine_rows_per_src=[9474, 9472, 9392, 9158, 9026, 9180, 9268, 9182] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265240576 dispatch_meta_bytes=343128 combine_payload_bytes=265240576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265583704 ingress_bytes=299843584 link_bytes=299843584 token_bytes=530824280 weight_movement_bytes=34603008 total_nvlink_bytes=565427288 weight_to_token_ratio=0.065187 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128287267e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74150 payload_rows_remote=64984 dup_rows=24154 dup_rows_remote=21056 dedup_ratio=0.245707 router_tokens_per_expert=[718, 751, 808, 787, 795, 747, 738, 709, 750, 757, 793, 774, 728, 807, 746, 757, 753, 782, 756, 762, 776, 733, 798, 772, 755, 816, 778, 815, 765, 764, 778, 713, 774, 760, 822, 779, 800, 802, 787, 774, 757, 745, 802, 756, 733, 750, 734, 782, 764, 760, 777, 789, 797, 756, 744, 740, 765, 740, 762, 781, 783, 798, 795, 763] payload_rows_per_dst=[9174, 9232, 9166, 9306, 9464, 9124, 9352, 9332] dup_rows_per_dst=[2932, 2992, 3098, 3062, 3132, 2994, 2902, 3042] combine_rows_per_src=[9174, 9232, 9166, 9306, 9464, 9124, 9352, 9332] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266944512 dispatch_meta_bytes=344792 combine_payload_bytes=266944512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267289304 ingress_bytes=301547520 link_bytes=301547520 token_bytes=534233816 weight_movement_bytes=34603008 total_nvlink_bytes=568836824 weight_to_token_ratio=0.064771 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134699041e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74170 payload_rows_remote=64756 dup_rows=24134 dup_rows_remote=21026 dedup_ratio=0.245504 router_tokens_per_expert=[772, 730, 772, 792, 761, 773, 777, 760, 755, 750, 759, 748, 803, 782, 733, 787, 765, 806, 754, 786, 827, 717, 774, 758, 773, 729, 759, 767, 743, 747, 770, 827, 725, 771, 775, 796, 762, 798, 819, 806, 765, 731, 749, 798, 800, 760, 721, 756, 740, 746, 770, 746, 767, 746, 779, 765, 770, 808, 773, 797, 755, 777, 749, 776] payload_rows_per_dst=[9260, 9252, 9050, 9206, 9274, 9370, 9344, 9414] dup_rows_per_dst=[3014, 2982, 2926, 3024, 2800, 3220, 3060, 3108] combine_rows_per_src=[9260, 9252, 9050, 9206, 9274, 9370, 9344, 9414] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74240 payload_rows_remote=64764 dup_rows=24064 dup_rows_remote=20906 dedup_ratio=0.244792 router_tokens_per_expert=[750, 740, 730, 775, 747, 746, 775, 765, 749, 777, 760, 787, 788, 796, 777, 737, 764, 756, 743, 750, 766, 770, 794, 768, 735, 791, 795, 779, 808, 742, 775, 760, 757, 802, 784, 748, 808, 804, 807, 807, 785, 731, 733, 761, 755, 805, 748, 790, 788, 770, 761, 759, 726, 749, 747, 760, 773, 753, 780, 733, 778, 772, 778, 805] payload_rows_per_dst=[9176, 9376, 9182, 9336, 9476, 9246, 9184, 9264] dup_rows_per_dst=[2880, 2966, 3040, 3034, 3158, 2970, 2936, 3080] combine_rows_per_src=[9176, 9376, 9182, 9336, 9476, 9246, 9184, 9264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74314 payload_rows_remote=65172 dup_rows=23990 dup_rows_remote=21026 dedup_ratio=0.244039 router_tokens_per_expert=[795, 739, 793, 752, 784, 791, 781, 768, 730, 731, 774, 752, 769, 777, 732, 788, 786, 763, 773, 778, 782, 763, 772, 725, 824, 779, 745, 800, 759, 743, 755, 812, 786, 752, 751, 769, 768, 761, 735, 761, 818, 795, 756, 730, 744, 726, 729, 822, 824, 800, 772, 795, 758, 761, 752, 750, 782, 757, 809, 761, 747, 749, 752, 765] payload_rows_per_dst=[9434, 9142, 9290, 9266, 9226, 9288, 9474, 9194] dup_rows_per_dst=[2972, 2964, 2994, 3168, 2940, 2952, 2950, 3050] combine_rows_per_src=[9434, 9142, 9290, 9266, 9226, 9288, 9474, 9194] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74274 payload_rows_remote=65032 dup_rows=24030 dup_rows_remote=21084 dedup_ratio=0.244446 router_tokens_per_expert=[776, 763, 772, 794, 754, 719, 786, 760, 799, 771, 751, 783, 765, 737, 762, 742, 768, 733, 731, 780, 754, 814, 785, 789, 773, 719, 731, 792, 758, 761, 739, 774, 808, 769, 779, 750, 817, 815, 783, 750, 763, 783, 787, 764, 732, 746, 746, 773, 772, 738, 813, 754, 764, 797, 781, 766, 776, 750, 794, 734, 797, 799, 756, 761] payload_rows_per_dst=[9216, 9282, 9262, 9188, 9392, 9242, 9386, 9306] dup_rows_per_dst=[3032, 2938, 3046, 2906, 3150, 2946, 2984, 3028] combine_rows_per_src=[9216, 9282, 9262, 9188, 9392, 9242, 9386, 9306] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74174 payload_rows_remote=65030 dup_rows=24130 dup_rows_remote=21126 dedup_ratio=0.245463 router_tokens_per_expert=[778, 771, 778, 790, 751, 798, 768, 756, 800, 732, 747, 760, 736, 817, 791, 796, 788, 776, 739, 760, 735, 759, 765, 707, 763, 758, 787, 756, 782, 742, 754, 743, 760, 767, 766, 743, 817, 755, 731, 770, 798, 792, 793, 801, 731, 849, 815, 799, 765, 790, 771, 738, 769, 752, 713, 776, 751, 773, 766, 746, 772, 782, 746, 772] payload_rows_per_dst=[9380, 9272, 9082, 9208, 9326, 9512, 9144, 9250] dup_rows_per_dst=[3000, 3086, 2976, 2962, 2892, 3244, 3004, 2966] combine_rows_per_src=[9380, 9272, 9082, 9208, 9326, 9512, 9144, 9250] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74092 payload_rows_remote=64824 dup_rows=24212 dup_rows_remote=21116 dedup_ratio=0.246297 router_tokens_per_expert=[764, 746, 783, 762, 776, 771, 770, 765, 784, 741, 757, 748, 772, 748, 795, 777, 763, 812, 791, 771, 728, 728, 741, 810, 772, 765, 744, 799, 744, 748, 816, 794, 792, 781, 769, 819, 751, 759, 763, 749, 767, 807, 754, 755, 779, 768, 825, 753, 745, 752, 764, 790, 822, 751, 752, 764, 713, 766, 753, 764, 718, 760, 806, 756] payload_rows_per_dst=[9256, 9252, 9188, 9268, 9410, 9244, 9268, 9206] dup_rows_per_dst=[3018, 2992, 3100, 3096, 2956, 3172, 3012, 2866] combine_rows_per_src=[9256, 9252, 9188, 9268, 9410, 9244, 9268, 9206] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=264921088 dispatch_meta_bytes=343208 combine_payload_bytes=264921088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265264296 ingress_bytes=299524096 link_bytes=299524096 token_bytes=530185384 weight_movement_bytes=34603008 total_nvlink_bytes=564788392 weight_to_token_ratio=0.065266 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127085060e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74240 payload_rows_remote=64764 dup_rows=24064 dup_rows_remote=20906 dedup_ratio=0.244792 router_tokens_per_expert=[750, 740, 730, 775, 747, 746, 775, 765, 749, 777, 760, 787, 788, 796, 777, 737, 764, 756, 743, 750, 766, 770, 794, 768, 735, 791, 795, 779, 808, 742, 775, 760, 757, 802, 784, 748, 808, 804, 807, 807, 785, 731, 733, 761, 755, 805, 748, 790, 788, 770, 761, 759, 726, 749, 747, 760, 773, 753, 780, 733, 778, 772, 778, 805] payload_rows_per_dst=[9176, 9376, 9182, 9336, 9476, 9246, 9184, 9264] dup_rows_per_dst=[2880, 2966, 3040, 3034, 3158, 2970, 2936, 3080] combine_rows_per_src=[9176, 9376, 9182, 9336, 9476, 9246, 9184, 9264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265240576 dispatch_meta_bytes=343128 combine_payload_bytes=265240576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265583704 ingress_bytes=299843584 link_bytes=299843584 token_bytes=530824280 weight_movement_bytes=34603008 total_nvlink_bytes=565427288 weight_to_token_ratio=0.065187 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128287267e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74344 payload_rows_remote=65088 dup_rows=23960 dup_rows_remote=21026 dedup_ratio=0.243734 router_tokens_per_expert=[772, 766, 806, 778, 774, 775, 753, 795, 736, 734, 736, 791, 770, 779, 757, 768, 784, 790, 772, 801, 783, 786, 741, 779, 794, 771, 792, 727, 723, 765, 790, 778, 791, 737, 776, 731, 790, 737, 761, 719, 816, 759, 790, 767, 734, 790, 762, 773, 745, 762, 774, 750, 752, 753, 784, 775, 747, 814, 758, 751, 740, 796, 770, 782] payload_rows_per_dst=[9412, 9194, 9432, 9228, 9190, 9346, 9256, 9286] dup_rows_per_dst=[3026, 2948, 3040, 3052, 2894, 3036, 2934, 3030] combine_rows_per_src=[9412, 9194, 9432, 9228, 9190, 9346, 9256, 9286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266174464 dispatch_meta_bytes=344160 combine_payload_bytes=266174464 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266518624 ingress_bytes=369983488 link_bytes=369983488 token_bytes=532693088 weight_movement_bytes=103809024 total_nvlink_bytes=636502112 weight_to_token_ratio=0.194876 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392218079e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265273344 dispatch_meta_bytes=342680 combine_payload_bytes=265273344 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=265616024 ingress_bytes=403685376 link_bytes=403685376 token_bytes=530889368 weight_movement_bytes=138412032 total_nvlink_bytes=669301400 weight_to_token_ratio=0.260717 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519035571e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74092 payload_rows_remote=64824 dup_rows=24212 dup_rows_remote=21116 dedup_ratio=0.246297 router_tokens_per_expert=[764, 746, 783, 762, 776, 771, 770, 765, 784, 741, 757, 748, 772, 748, 795, 777, 763, 812, 791, 771, 728, 728, 741, 810, 772, 765, 744, 799, 744, 748, 816, 794, 792, 781, 769, 819, 751, 759, 763, 749, 767, 807, 754, 755, 779, 768, 825, 753, 745, 752, 764, 790, 822, 751, 752, 764, 713, 766, 753, 764, 718, 760, 806, 756] payload_rows_per_dst=[9256, 9252, 9188, 9268, 9410, 9244, 9268, 9206] dup_rows_per_dst=[3018, 2992, 3100, 3096, 2956, 3172, 3012, 2866] combine_rows_per_src=[9256, 9252, 9188, 9268, 9410, 9244, 9268, 9206] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74280 payload_rows_remote=65000 dup_rows=24024 dup_rows_remote=20932 dedup_ratio=0.244385 router_tokens_per_expert=[744, 791, 762, 797, 773, 816, 835, 752, 806, 745, 793, 792, 781, 754, 759, 753, 781, 780, 718, 777, 807, 795, 741, 787, 774, 727, 769, 800, 762, 750, 790, 740, 748, 778, 803, 800, 717, 775, 795, 772, 728, 775, 761, 779, 768, 773, 775, 778, 776, 795, 751, 732, 787, 807, 763, 708, 727, 740, 763, 759, 786, 740, 733, 709] payload_rows_per_dst=[9408, 9396, 9280, 9214, 9372, 9208, 9308, 9094] dup_rows_per_dst=[3132, 2970, 3092, 3010, 3004, 3066, 2930, 2820] combine_rows_per_src=[9408, 9396, 9280, 9214, 9372, 9208, 9308, 9094] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266944512 dispatch_meta_bytes=344792 combine_payload_bytes=266944512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267289304 ingress_bytes=301547520 link_bytes=301547520 token_bytes=534233816 weight_movement_bytes=34603008 total_nvlink_bytes=568836824 weight_to_token_ratio=0.064771 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134699041e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344464 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266715536 ingress_bytes=300974080 link_bytes=300974080 token_bytes=533086608 weight_movement_bytes=34603008 total_nvlink_bytes=567689616 weight_to_token_ratio=0.064911 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132541233e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344624 combine_payload_bytes=266362880 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266707504 ingress_bytes=370171904 link_bytes=370171904 token_bytes=533070384 weight_movement_bytes=103809024 total_nvlink_bytes=636879408 weight_to_token_ratio=0.194738 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392927073e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265273344 dispatch_meta_bytes=342680 combine_payload_bytes=265273344 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=265616024 ingress_bytes=403685376 link_bytes=403685376 token_bytes=530889368 weight_movement_bytes=138412032 total_nvlink_bytes=669301400 weight_to_token_ratio=0.260717 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519035571e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=344456 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266944904 ingress_bytes=301203456 link_bytes=301203456 token_bytes=533545352 weight_movement_bytes=34603008 total_nvlink_bytes=568148360 weight_to_token_ratio=0.064855 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133404356e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265519104 dispatch_meta_bytes=343760 combine_payload_bytes=265519104 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265862864 ingress_bytes=334725120 link_bytes=334725120 token_bytes=531381968 weight_movement_bytes=69206016 total_nvlink_bytes=600587984 weight_to_token_ratio=0.130238 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259543679e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74006 payload_rows_remote=64580 dup_rows=24298 dup_rows_remote=21154 dedup_ratio=0.247172 router_tokens_per_expert=[768, 774, 753, 780, 785, 760, 771, 793, 754, 746, 776, 753, 791, 721, 792, 740, 794, 744, 737, 791, 750, 753, 731, 841, 800, 740, 788, 746, 759, 746, 778, 752, 747, 774, 717, 724, 744, 820, 769, 775, 784, 787, 789, 728, 745, 794, 795, 806, 778, 713, 758, 744, 744, 783, 759, 787, 776, 765, 763, 777, 801, 764, 818, 817] payload_rows_per_dst=[9294, 9098, 9270, 9234, 9136, 9392, 9156, 9426] dup_rows_per_dst=[3074, 2952, 3094, 2990, 3004, 3064, 2976, 3144] combine_rows_per_src=[9294, 9098, 9270, 9234, 9136, 9392, 9156, 9426] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74426 payload_rows_remote=65048 dup_rows=23878 dup_rows_remote=20798 dedup_ratio=0.242900 router_tokens_per_expert=[780, 805, 806, 733, 754, 746, 705, 745, 808, 755, 729, 797, 799, 775, 783, 783, 772, 745, 811, 787, 731, 728, 752, 772, 813, 778, 751, 819, 759, 718, 784, 727, 768, 801, 762, 788, 790, 773, 798, 727, 747, 779, 792, 763, 756, 756, 745, 717, 763, 759, 772, 753, 795, 810, 745, 866, 781, 721, 758, 748, 772, 762, 758, 777] payload_rows_per_dst=[9144, 9378, 9284, 9260, 9392, 9250, 9440, 9278] dup_rows_per_dst=[3004, 3080, 2912, 3038, 3022, 2860, 3086, 2876] combine_rows_per_src=[9144, 9378, 9284, 9260, 9392, 9250, 9440, 9278] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=264519680 dispatch_meta_bytes=342936 combine_payload_bytes=264519680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264862616 ingress_bytes=299122688 link_bytes=299122688 token_bytes=529382296 weight_movement_bytes=34603008 total_nvlink_bytes=563985304 weight_to_token_ratio=0.065365 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125574594e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266240000 dispatch_meta_bytes=343728 combine_payload_bytes=266240000 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266583728 ingress_bytes=335446016 link_bytes=335446016 token_bytes=532823728 weight_movement_bytes=69206016 total_nvlink_bytes=602029744 weight_to_token_ratio=0.129885 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262256353e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74088 payload_rows_remote=64692 dup_rows=24216 dup_rows_remote=21106 dedup_ratio=0.246338 router_tokens_per_expert=[782, 744, 787, 771, 733, 760, 752, 773, 769, 741, 765, 749, 782, 728, 797, 802, 790, 790, 761, 792, 731, 779, 769, 737, 737, 761, 768, 781, 789, 783, 762, 753, 729, 748, 818, 783, 748, 793, 765, 782, 751, 808, 759, 791, 747, 748, 764, 780, 810, 760, 746, 726, 747, 739, 780, 795, 769, 737, 816, 779, 777, 798, 791, 750] payload_rows_per_dst=[9396, 9258, 9228, 9248, 9298, 9154, 9202, 9304] dup_rows_per_dst=[3110, 3010, 3070, 3020, 3064, 2932, 2880, 3130] combine_rows_per_src=[9396, 9258, 9228, 9248, 9298, 9154, 9202, 9304] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266436608 dispatch_meta_bytes=343384 combine_payload_bytes=266436608 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266779992 ingress_bytes=404848640 link_bytes=404848640 token_bytes=533216600 weight_movement_bytes=138412032 total_nvlink_bytes=671628632 weight_to_token_ratio=0.259579 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523412839e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74274 payload_rows_remote=65042 dup_rows=24030 dup_rows_remote=21172 dedup_ratio=0.244446 router_tokens_per_expert=[775, 797, 783, 733, 779, 768, 759, 795, 800, 750, 790, 792, 752, 750, 782, 747, 809, 762, 769, 734, 769, 762, 721, 719, 763, 796, 752, 727, 752, 773, 750, 794, 765, 717, 785, 789, 806, 814, 764, 775, 696, 821, 778, 768, 788, 748, 777, 785, 754, 768, 767, 755, 764, 772, 801, 792, 766, 758, 764, 769, 745, 737, 760, 800] payload_rows_per_dst=[9378, 9226, 9232, 9228, 9362, 9298, 9324, 9226] dup_rows_per_dst=[3000, 3100, 2858, 2986, 3068, 3024, 3022, 2972] combine_rows_per_src=[9378, 9226, 9232, 9228, 9362, 9298, 9324, 9226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74006 payload_rows_remote=64580 dup_rows=24298 dup_rows_remote=21154 dedup_ratio=0.247172 router_tokens_per_expert=[768, 774, 753, 780, 785, 760, 771, 793, 754, 746, 776, 753, 791, 721, 792, 740, 794, 744, 737, 791, 750, 753, 731, 841, 800, 740, 788, 746, 759, 746, 778, 752, 747, 774, 717, 724, 744, 820, 769, 775, 784, 787, 789, 728, 745, 794, 795, 806, 778, 713, 758, 744, 744, 783, 759, 787, 776, 765, 763, 777, 801, 764, 818, 817] payload_rows_per_dst=[9294, 9098, 9270, 9234, 9136, 9392, 9156, 9426] dup_rows_per_dst=[3074, 2952, 3094, 2990, 3004, 3064, 2976, 3144] combine_rows_per_src=[9294, 9098, 9270, 9234, 9136, 9392, 9156, 9426] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=65088 dup_rows=24048 dup_rows_remote=21198 dedup_ratio=0.244629 router_tokens_per_expert=[727, 807, 780, 727, 787, 779, 761, 809, 785, 762, 820, 785, 759, 754, 755, 805, 844, 754, 782, 801, 738, 714, 784, 745, 751, 755, 766, 767, 769, 775, 771, 764, 755, 767, 739, 775, 768, 714, 722, 769, 814, 738, 811, 762, 734, 773, 760, 767, 766, 754, 767, 774, 782, 793, 766, 755, 728, 768, 779, 758, 754, 775, 835, 748] payload_rows_per_dst=[9326, 9254, 9370, 9176, 9168, 9342, 9378, 9242] dup_rows_per_dst=[3028, 3196, 2954, 3060, 2850, 2976, 2936, 3048] combine_rows_per_src=[9326, 9254, 9370, 9176, 9168, 9342, 9378, 9242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74426 payload_rows_remote=65048 dup_rows=23878 dup_rows_remote=20798 dedup_ratio=0.242900 router_tokens_per_expert=[780, 805, 806, 733, 754, 746, 705, 745, 808, 755, 729, 797, 799, 775, 783, 783, 772, 745, 811, 787, 731, 728, 752, 772, 813, 778, 751, 819, 759, 718, 784, 727, 768, 801, 762, 788, 790, 773, 798, 727, 747, 779, 792, 763, 756, 756, 745, 717, 763, 759, 772, 753, 795, 810, 745, 866, 781, 721, 758, 748, 772, 762, 758, 777] payload_rows_per_dst=[9144, 9378, 9284, 9260, 9392, 9250, 9440, 9278] dup_rows_per_dst=[3004, 3080, 2912, 3038, 3022, 2860, 3086, 2876] combine_rows_per_src=[9144, 9378, 9284, 9260, 9392, 9250, 9440, 9278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=264978432 dispatch_meta_bytes=343192 combine_payload_bytes=264978432 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265321624 ingress_bytes=299581440 link_bytes=299581440 token_bytes=530300056 weight_movement_bytes=34603008 total_nvlink_bytes=564903064 weight_to_token_ratio=0.065252 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127300840e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74382 payload_rows_remote=64986 dup_rows=23922 dup_rows_remote=21006 dedup_ratio=0.243347 router_tokens_per_expert=[742, 764, 782, 761, 719, 730, 750, 743, 763, 766, 752, 770, 762, 776, 764, 752, 750, 853, 741, 783, 801, 769, 775, 785, 775, 781, 723, 755, 751, 800, 780, 782, 765, 728, 778, 745, 755, 780, 769, 789, 780, 745, 744, 777, 774, 767, 778, 791, 780, 792, 762, 779, 735, 797, 736, 789, 804, 824, 769, 763, 747, 726, 763, 821] payload_rows_per_dst=[9142, 9266, 9366, 9334, 9218, 9396, 9310, 9350] dup_rows_per_dst=[2840, 2944, 3148, 2960, 3000, 2916, 3030, 3084] combine_rows_per_src=[9142, 9266, 9366, 9334, 9218, 9396, 9310, 9350] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=65088 dup_rows=24048 dup_rows_remote=21198 dedup_ratio=0.244629 router_tokens_per_expert=[727, 807, 780, 727, 787, 779, 761, 809, 785, 762, 820, 785, 759, 754, 755, 805, 844, 754, 782, 801, 738, 714, 784, 745, 751, 755, 766, 767, 769, 775, 771, 764, 755, 767, 739, 775, 768, 714, 722, 769, 814, 738, 811, 762, 734, 773, 760, 767, 766, 754, 767, 774, 782, 793, 766, 755, 728, 768, 779, 758, 754, 775, 835, 748] payload_rows_per_dst=[9326, 9254, 9370, 9176, 9168, 9342, 9378, 9242] dup_rows_per_dst=[3028, 3196, 2954, 3060, 2850, 2976, 2936, 3048] combine_rows_per_src=[9326, 9254, 9370, 9176, 9168, 9342, 9378, 9242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=264519680 dispatch_meta_bytes=342936 combine_payload_bytes=264519680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264862616 ingress_bytes=299122688 link_bytes=299122688 token_bytes=529382296 weight_movement_bytes=34603008 total_nvlink_bytes=563985304 weight_to_token_ratio=0.065365 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125574594e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74140 payload_rows_remote=64886 dup_rows=24164 dup_rows_remote=21120 dedup_ratio=0.245809 router_tokens_per_expert=[723, 761, 759, 746, 770, 788, 727, 804, 764, 754, 789, 795, 717, 796, 766, 781, 780, 803, 779, 747, 767, 750, 756, 728, 788, 760, 756, 772, 741, 770, 762, 800, 737, 771, 780, 776, 780, 821, 803, 761, 827, 727, 786, 794, 773, 759, 795, 707, 809, 780, 769, 748, 781, 782, 769, 739, 750, 773, 772, 788, 701, 734, 796, 765] payload_rows_per_dst=[9166, 9276, 9192, 9254, 9446, 9298, 9334, 9174] dup_rows_per_dst=[2990, 3048, 3028, 3044, 3012, 3038, 3020, 2984] combine_rows_per_src=[9166, 9276, 9192, 9254, 9446, 9298, 9334, 9174] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74344 payload_rows_remote=65088 dup_rows=23960 dup_rows_remote=21026 dedup_ratio=0.243734 router_tokens_per_expert=[772, 766, 806, 778, 774, 775, 753, 795, 736, 734, 736, 791, 770, 779, 757, 768, 784, 790, 772, 801, 783, 786, 741, 779, 794, 771, 792, 727, 723, 765, 790, 778, 791, 737, 776, 731, 790, 737, 761, 719, 816, 759, 790, 767, 734, 790, 762, 773, 745, 762, 774, 750, 752, 753, 784, 775, 747, 814, 758, 751, 740, 796, 770, 782] payload_rows_per_dst=[9412, 9194, 9432, 9228, 9190, 9346, 9256, 9286] dup_rows_per_dst=[3026, 2948, 3040, 3052, 2894, 3036, 2934, 3030] combine_rows_per_src=[9412, 9194, 9432, 9228, 9190, 9346, 9256, 9286] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=264978432 dispatch_meta_bytes=343192 combine_payload_bytes=264978432 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265321624 ingress_bytes=299581440 link_bytes=299581440 token_bytes=530300056 weight_movement_bytes=34603008 total_nvlink_bytes=564903064 weight_to_token_ratio=0.065252 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127300840e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74220 payload_rows_remote=64974 dup_rows=24084 dup_rows_remote=21120 dedup_ratio=0.244995 router_tokens_per_expert=[779, 794, 767, 803, 753, 767, 751, 792, 791, 765, 724, 726, 758, 814, 743, 788, 764, 738, 787, 789, 766, 760, 763, 782, 760, 817, 787, 743, 777, 806, 773, 773, 748, 727, 776, 789, 760, 798, 756, 778, 765, 792, 838, 722, 794, 761, 764, 746, 718, 761, 763, 771, 767, 821, 778, 726, 742, 778, 756, 761, 769, 724, 784, 719] payload_rows_per_dst=[9346, 9308, 9236, 9370, 9260, 9286, 9246, 9168] dup_rows_per_dst=[3066, 2910, 3062, 3102, 3004, 3078, 2964, 2898] combine_rows_per_src=[9346, 9308, 9236, 9370, 9260, 9286, 9246, 9168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=344856 combine_payload_bytes=266412032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266756888 ingress_bytes=301015040 link_bytes=301015040 token_bytes=533168920 weight_movement_bytes=34603008 total_nvlink_bytes=567771928 weight_to_token_ratio=0.064901 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132695362e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74150 payload_rows_remote=64984 dup_rows=24154 dup_rows_remote=21056 dedup_ratio=0.245707 router_tokens_per_expert=[718, 751, 808, 787, 795, 747, 738, 709, 750, 757, 793, 774, 728, 807, 746, 757, 753, 782, 756, 762, 776, 733, 798, 772, 755, 816, 778, 815, 765, 764, 778, 713, 774, 760, 822, 779, 800, 802, 787, 774, 757, 745, 802, 756, 733, 750, 734, 782, 764, 760, 777, 789, 797, 756, 744, 740, 765, 740, 762, 781, 783, 798, 795, 763] payload_rows_per_dst=[9174, 9232, 9166, 9306, 9464, 9124, 9352, 9332] dup_rows_per_dst=[2932, 2992, 3098, 3062, 3132, 2994, 2902, 3042] combine_rows_per_src=[9174, 9232, 9166, 9306, 9464, 9124, 9352, 9332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266436608 dispatch_meta_bytes=343384 combine_payload_bytes=266436608 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266779992 ingress_bytes=404848640 link_bytes=404848640 token_bytes=533216600 weight_movement_bytes=138412032 total_nvlink_bytes=671628632 weight_to_token_ratio=0.259579 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523412839e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266182656 dispatch_meta_bytes=343968 combine_payload_bytes=266182656 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266526624 ingress_bytes=335388672 link_bytes=335388672 token_bytes=532709280 weight_movement_bytes=69206016 total_nvlink_bytes=601915296 weight_to_token_ratio=0.129913 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262040572e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=345144 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266945592 ingress_bytes=301203456 link_bytes=301203456 token_bytes=533546040 weight_movement_bytes=34603008 total_nvlink_bytes=568149048 weight_to_token_ratio=0.064855 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133404356e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=344024 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266117080 ingress_bytes=300376064 link_bytes=300376064 token_bytes=531890136 weight_movement_bytes=34603008 total_nvlink_bytes=566493144 weight_to_token_ratio=0.065057 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130290947e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266182656 dispatch_meta_bytes=343968 combine_payload_bytes=266182656 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266526624 ingress_bytes=335388672 link_bytes=335388672 token_bytes=532709280 weight_movement_bytes=69206016 total_nvlink_bytes=601915296 weight_to_token_ratio=0.129913 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262040572e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=344376 combine_payload_bytes=266133504 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266477880 ingress_bytes=369942528 link_bytes=369942528 token_bytes=532611384 weight_movement_bytes=103809024 total_nvlink_bytes=636420408 weight_to_token_ratio=0.194906 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392063950e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74058 payload_rows_remote=64786 dup_rows=24246 dup_rows_remote=21208 dedup_ratio=0.246643 router_tokens_per_expert=[785, 739, 789, 788, 796, 718, 757, 747, 766, 774, 733, 762, 756, 781, 788, 762, 727, 811, 776, 793, 787, 778, 768, 803, 782, 791, 767, 729, 750, 748, 779, 796, 801, 763, 732, 824, 744, 776, 767, 764, 771, 726, 785, 767, 819, 747, 770, 725, 822, 735, 787, 783, 771, 751, 752, 777, 764, 791, 772, 708, 744, 737, 803, 748] payload_rows_per_dst=[9272, 9628, 9318, 9190, 9334, 9214, 9304, 8798] dup_rows_per_dst=[3038, 3378, 3202, 2988, 3008, 3006, 3052, 2574] combine_rows_per_src=[9272, 9628, 9318, 9190, 9334, 9214, 9304, 8798] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266174464 dispatch_meta_bytes=344160 combine_payload_bytes=266174464 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266518624 ingress_bytes=369983488 link_bytes=369983488 token_bytes=532693088 weight_movement_bytes=103809024 total_nvlink_bytes=636502112 weight_to_token_ratio=0.194876 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392218079e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=64884 dup_rows=23944 dup_rows_remote=20578 dedup_ratio=0.243571 router_tokens_per_expert=[746, 767, 791, 741, 744, 771, 767, 777, 712, 775, 764, 734, 803, 762, 798, 752, 781, 782, 741, 743, 802, 759, 761, 777, 777, 814, 728, 751, 798, 729, 784, 761, 777, 762, 759, 755, 787, 748, 773, 787, 792, 748, 775, 768, 787, 817, 750, 786, 764, 764, 758, 792, 749, 748, 710, 755, 811, 747, 810, 752, 744, 787, 807, 791] payload_rows_per_dst=[9176, 9332, 9172, 9150, 9296, 9272, 9486, 9476] dup_rows_per_dst=[3032, 2868, 2776, 2954, 3054, 2832, 3062, 3366] combine_rows_per_src=[9176, 9332, 9172, 9150, 9296, 9272, 9486, 9476] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74316 payload_rows_remote=65016 dup_rows=23988 dup_rows_remote=21014 dedup_ratio=0.244019 router_tokens_per_expert=[787, 799, 760, 767, 794, 781, 789, 746, 753, 769, 770, 807, 727, 786, 737, 788, 768, 760, 776, 781, 767, 769, 798, 751, 755, 765, 790, 791, 775, 779, 746, 754, 746, 796, 762, 771, 748, 714, 808, 755, 771, 768, 763, 759, 730, 767, 752, 733, 753, 771, 801, 807, 760, 777, 763, 791, 766, 756, 745, 749, 754, 782, 776, 773] payload_rows_per_dst=[9398, 9300, 9350, 9292, 9200, 9174, 9316, 9286] dup_rows_per_dst=[3048, 2974, 2990, 3018, 3000, 2912, 3130, 2916] combine_rows_per_src=[9398, 9300, 9350, 9292, 9200, 9174, 9316, 9286] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=344456 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266944904 ingress_bytes=301203456 link_bytes=301203456 token_bytes=533545352 weight_movement_bytes=34603008 total_nvlink_bytes=568148360 weight_to_token_ratio=0.064855 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133404356e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=343976 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265707432 ingress_bytes=299966464 link_bytes=299966464 token_bytes=531070888 weight_movement_bytes=34603008 total_nvlink_bytes=565673896 weight_to_token_ratio=0.065157 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128749655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74420 payload_rows_remote=65274 dup_rows=23884 dup_rows_remote=20876 dedup_ratio=0.242961 router_tokens_per_expert=[774, 795, 734, 775, 733, 717, 760, 798, 758, 801, 810, 749, 797, 722, 792, 763, 754, 756, 780, 775, 778, 760, 752, 760, 742, 792, 776, 749, 792, 767, 765, 814, 750, 732, 764, 797, 784, 784, 823, 759, 748, 791, 778, 752, 768, 810, 674, 756, 792, 732, 808, 786, 776, 813, 754, 754, 767, 745, 753, 788, 760, 717, 737, 810] payload_rows_per_dst=[9240, 9382, 9230, 9288, 9378, 9146, 9486, 9270] dup_rows_per_dst=[2932, 3002, 3000, 3106, 3008, 3008, 2944, 2884] combine_rows_per_src=[9240, 9382, 9230, 9288, 9378, 9146, 9486, 9270] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74316 payload_rows_remote=65016 dup_rows=23988 dup_rows_remote=21014 dedup_ratio=0.244019 router_tokens_per_expert=[787, 799, 760, 767, 794, 781, 789, 746, 753, 769, 770, 807, 727, 786, 737, 788, 768, 760, 776, 781, 767, 769, 798, 751, 755, 765, 790, 791, 775, 779, 746, 754, 746, 796, 762, 771, 748, 714, 808, 755, 771, 768, 763, 759, 730, 767, 752, 733, 753, 771, 801, 807, 760, 777, 763, 791, 766, 756, 745, 749, 754, 782, 776, 773] payload_rows_per_dst=[9398, 9300, 9350, 9292, 9200, 9174, 9316, 9286] dup_rows_per_dst=[3048, 2974, 2990, 3018, 3000, 2912, 3130, 2916] combine_rows_per_src=[9398, 9300, 9350, 9292, 9200, 9174, 9316, 9286] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265764864 dispatch_meta_bytes=341848 combine_payload_bytes=265764864 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266106712 ingress_bytes=334970880 link_bytes=334970880 token_bytes=531871576 weight_movement_bytes=69206016 total_nvlink_bytes=601077592 weight_to_token_ratio=0.130118 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260468454e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=344120 combine_payload_bytes=266305536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266649656 ingress_bytes=300908544 link_bytes=300908544 token_bytes=532955192 weight_movement_bytes=34603008 total_nvlink_bytes=567558200 weight_to_token_ratio=0.064927 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132294626e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74058 payload_rows_remote=64786 dup_rows=24246 dup_rows_remote=21208 dedup_ratio=0.246643 router_tokens_per_expert=[785, 739, 789, 788, 796, 718, 757, 747, 766, 774, 733, 762, 756, 781, 788, 762, 727, 811, 776, 793, 787, 778, 768, 803, 782, 791, 767, 729, 750, 748, 779, 796, 801, 763, 732, 824, 744, 776, 767, 764, 771, 726, 785, 767, 819, 747, 770, 725, 822, 735, 787, 783, 771, 751, 752, 777, 764, 791, 772, 708, 744, 737, 803, 748] payload_rows_per_dst=[9272, 9628, 9318, 9190, 9334, 9214, 9304, 8798] dup_rows_per_dst=[3038, 3378, 3202, 2988, 3008, 3006, 3052, 2574] combine_rows_per_src=[9272, 9628, 9318, 9190, 9334, 9214, 9304, 8798] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=64976 dup_rows=24032 dup_rows_remote=21100 dedup_ratio=0.244466 router_tokens_per_expert=[768, 753, 758, 790, 755, 792, 783, 833, 741, 821, 744, 737, 769, 719, 747, 787, 762, 776, 765, 765, 741, 745, 788, 772, 813, 738, 770, 733, 770, 773, 779, 821, 769, 759, 780, 761, 724, 763, 746, 750, 698, 788, 746, 813, 779, 794, 801, 783, 761, 735, 754, 775, 760, 705, 793, 780, 765, 758, 819, 760, 783, 773, 759, 810] payload_rows_per_dst=[9446, 9156, 9296, 9350, 9142, 9322, 9230, 9330] dup_rows_per_dst=[3018, 2974, 2932, 3044, 2962, 3082, 2896, 3124] combine_rows_per_src=[9446, 9156, 9296, 9350, 9142, 9322, 9230, 9330] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74210 payload_rows_remote=64936 dup_rows=24094 dup_rows_remote=21082 dedup_ratio=0.245097 router_tokens_per_expert=[739, 734, 769, 786, 748, 765, 761, 745, 746, 818, 707, 767, 734, 762, 723, 770, 786, 804, 794, 796, 758, 762, 747, 814, 712, 812, 741, 805, 756, 762, 731, 767, 752, 811, 782, 780, 763, 734, 724, 760, 774, 776, 785, 834, 770, 772, 792, 756, 778, 794, 722, 738, 778, 767, 770, 796, 814, 775, 775, 787, 771, 744, 761, 796] payload_rows_per_dst=[9172, 9098, 9424, 9156, 9226, 9454, 9274, 9406] dup_rows_per_dst=[2922, 2956, 3098, 3016, 2986, 3064, 3012, 3040] combine_rows_per_src=[9172, 9098, 9424, 9156, 9226, 9454, 9274, 9406] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74108 payload_rows_remote=64802 dup_rows=24196 dup_rows_remote=21214 dedup_ratio=0.246134 router_tokens_per_expert=[782, 774, 724, 801, 758, 744, 827, 775, 779, 744, 771, 725, 754, 744, 780, 740, 790, 762, 745, 794, 734, 721, 772, 781, 808, 824, 762, 809, 722, 746, 761, 763, 790, 754, 751, 763, 731, 793, 823, 739, 785, 763, 797, 788, 761, 795, 759, 747, 751, 773, 765, 744, 748, 746, 797, 750, 793, 777, 754, 826, 769, 770, 788, 746] payload_rows_per_dst=[9398, 9106, 9182, 9240, 9306, 9348, 9200, 9328] dup_rows_per_dst=[2972, 2968, 3016, 3150, 2982, 3042, 2948, 3118] combine_rows_per_src=[9398, 9106, 9182, 9240, 9306, 9348, 9200, 9328] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74240 payload_rows_remote=64956 dup_rows=24064 dup_rows_remote=20986 dedup_ratio=0.244792 router_tokens_per_expert=[763, 758, 763, 794, 732, 804, 791, 715, 742, 750, 765, 781, 747, 756, 758, 792, 761, 785, 782, 736, 776, 774, 787, 758, 761, 778, 715, 777, 750, 800, 781, 819, 774, 776, 794, 776, 754, 800, 758, 779, 768, 775, 755, 820, 769, 767, 766, 794, 789, 740, 726, 771, 747, 761, 711, 737, 786, 757, 778, 770, 793, 769, 778, 763] payload_rows_per_dst=[9240, 9304, 9284, 9284, 9348, 9290, 9154, 9336] dup_rows_per_dst=[3000, 2878, 3034, 3078, 3074, 3138, 2810, 3052] combine_rows_per_src=[9240, 9304, 9284, 9284, 9348, 9290, 9154, 9336] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74420 payload_rows_remote=65274 dup_rows=23884 dup_rows_remote=20876 dedup_ratio=0.242961 router_tokens_per_expert=[774, 795, 734, 775, 733, 717, 760, 798, 758, 801, 810, 749, 797, 722, 792, 763, 754, 756, 780, 775, 778, 760, 752, 760, 742, 792, 776, 749, 792, 767, 765, 814, 750, 732, 764, 797, 784, 784, 823, 759, 748, 791, 778, 752, 768, 810, 674, 756, 792, 732, 808, 786, 776, 813, 754, 754, 767, 745, 753, 788, 760, 717, 737, 810] payload_rows_per_dst=[9240, 9382, 9230, 9288, 9378, 9146, 9486, 9270] dup_rows_per_dst=[2932, 3002, 3000, 3106, 3008, 3008, 2944, 2884] combine_rows_per_src=[9240, 9382, 9230, 9288, 9378, 9146, 9486, 9270] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=343976 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265707432 ingress_bytes=299966464 link_bytes=299966464 token_bytes=531070888 weight_movement_bytes=34603008 total_nvlink_bytes=565673896 weight_to_token_ratio=0.065157 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128749655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266141696 dispatch_meta_bytes=344304 combine_payload_bytes=266141696 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266486000 ingress_bytes=335347712 link_bytes=335347712 token_bytes=532627696 weight_movement_bytes=69206016 total_nvlink_bytes=601833712 weight_to_token_ratio=0.129933 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261886443e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74220 payload_rows_remote=64974 dup_rows=24084 dup_rows_remote=21120 dedup_ratio=0.244995 router_tokens_per_expert=[779, 794, 767, 803, 753, 767, 751, 792, 791, 765, 724, 726, 758, 814, 743, 788, 764, 738, 787, 789, 766, 760, 763, 782, 760, 817, 787, 743, 777, 806, 773, 773, 748, 727, 776, 789, 760, 798, 756, 778, 765, 792, 838, 722, 794, 761, 764, 746, 718, 761, 763, 771, 767, 821, 778, 726, 742, 778, 756, 761, 769, 724, 784, 719] payload_rows_per_dst=[9346, 9308, 9236, 9370, 9260, 9286, 9246, 9168] dup_rows_per_dst=[3066, 2910, 3062, 3102, 3004, 3078, 2964, 2898] combine_rows_per_src=[9346, 9308, 9236, 9370, 9260, 9286, 9246, 9168] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74274 payload_rows_remote=65042 dup_rows=24030 dup_rows_remote=21172 dedup_ratio=0.244446 router_tokens_per_expert=[775, 797, 783, 733, 779, 768, 759, 795, 800, 750, 790, 792, 752, 750, 782, 747, 809, 762, 769, 734, 769, 762, 721, 719, 763, 796, 752, 727, 752, 773, 750, 794, 765, 717, 785, 789, 806, 814, 764, 775, 696, 821, 778, 768, 788, 748, 777, 785, 754, 768, 767, 755, 764, 772, 801, 792, 766, 758, 764, 769, 745, 737, 760, 800] payload_rows_per_dst=[9378, 9226, 9232, 9228, 9362, 9298, 9324, 9226] dup_rows_per_dst=[3000, 3100, 2858, 2986, 3068, 3024, 3022, 2972] combine_rows_per_src=[9378, 9226, 9232, 9228, 9362, 9298, 9324, 9226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267362304 dispatch_meta_bytes=344600 combine_payload_bytes=267362304 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267706904 ingress_bytes=336568320 link_bytes=336568320 token_bytes=535069208 weight_movement_bytes=69206016 total_nvlink_bytes=604275224 weight_to_token_ratio=0.129340 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266479492e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=344120 combine_payload_bytes=266305536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266649656 ingress_bytes=300908544 link_bytes=300908544 token_bytes=532955192 weight_movement_bytes=34603008 total_nvlink_bytes=567558200 weight_to_token_ratio=0.064927 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132294626e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265977856 dispatch_meta_bytes=344072 combine_payload_bytes=265977856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266321928 ingress_bytes=300580864 link_bytes=300580864 token_bytes=532299784 weight_movement_bytes=34603008 total_nvlink_bytes=566902792 weight_to_token_ratio=0.065007 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131061592e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265428992 dispatch_meta_bytes=344064 combine_payload_bytes=265428992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265773056 ingress_bytes=300032000 link_bytes=300032000 token_bytes=531202048 weight_movement_bytes=34603008 total_nvlink_bytes=565805056 weight_to_token_ratio=0.065141 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128996261e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74580 payload_rows_remote=65096 dup_rows=23724 dup_rows_remote=20720 dedup_ratio=0.241333 router_tokens_per_expert=[747, 787, 740, 754, 785, 744, 724, 755, 776, 755, 750, 757, 749, 773, 745, 783, 752, 764, 761, 741, 775, 816, 738, 783, 781, 788, 752, 776, 773, 743, 701, 816, 756, 806, 780, 749, 772, 751, 758, 790, 801, 810, 785, 744, 761, 744, 751, 807, 788, 758, 759, 747, 742, 790, 786, 797, 784, 787, 762, 771, 850, 747, 795, 740] payload_rows_per_dst=[9180, 9248, 9152, 9472, 9314, 9174, 9556, 9484] dup_rows_per_dst=[2892, 2928, 2874, 3022, 3010, 2768, 3226, 3004] combine_rows_per_src=[9180, 9248, 9152, 9472, 9314, 9174, 9556, 9484] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=343768 combine_payload_bytes=266059776 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266403544 ingress_bytes=404471808 link_bytes=404471808 token_bytes=532463320 weight_movement_bytes=138412032 total_nvlink_bytes=670875352 weight_to_token_ratio=0.259947 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521994851e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74458 payload_rows_remote=65140 dup_rows=23846 dup_rows_remote=20806 dedup_ratio=0.242574 router_tokens_per_expert=[778, 737, 759, 786, 794, 736, 764, 751, 783, 766, 816, 734, 776, 804, 730, 770, 785, 765, 737, 782, 785, 783, 769, 787, 745, 775, 759, 754, 706, 822, 798, 771, 755, 777, 788, 820, 766, 722, 751, 787, 819, 747, 824, 743, 806, 746, 769, 761, 797, 750, 760, 731, 781, 754, 738, 757, 743, 763, 813, 740, 785, 747, 687, 818] payload_rows_per_dst=[9276, 9318, 9382, 9250, 9328, 9432, 9188, 9284] dup_rows_per_dst=[2934, 3040, 3004, 3010, 3004, 2998, 2948, 2908] combine_rows_per_src=[9276, 9318, 9382, 9250, 9328, 9432, 9188, 9284] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267362304 dispatch_meta_bytes=344600 combine_payload_bytes=267362304 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267706904 ingress_bytes=336568320 link_bytes=336568320 token_bytes=535069208 weight_movement_bytes=69206016 total_nvlink_bytes=604275224 weight_to_token_ratio=0.129340 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266479492e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=344376 combine_payload_bytes=266133504 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266477880 ingress_bytes=369942528 link_bytes=369942528 token_bytes=532611384 weight_movement_bytes=103809024 total_nvlink_bytes=636420408 weight_to_token_ratio=0.194906 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392063950e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266633216 dispatch_meta_bytes=343264 combine_payload_bytes=266633216 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266976480 ingress_bytes=370442240 link_bytes=370442240 token_bytes=533609696 weight_movement_bytes=103809024 total_nvlink_bytes=637418720 weight_to_token_ratio=0.194541 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393944326e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266813440 dispatch_meta_bytes=343784 combine_payload_bytes=266813440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267157224 ingress_bytes=301416448 link_bytes=301416448 token_bytes=533970664 weight_movement_bytes=34603008 total_nvlink_bytes=568573672 weight_to_token_ratio=0.064803 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134205828e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74226 payload_rows_remote=64872 dup_rows=24078 dup_rows_remote=20938 dedup_ratio=0.244934 router_tokens_per_expert=[780, 782, 780, 763, 796, 764, 744, 759, 770, 772, 792, 778, 738, 748, 738, 803, 723, 743, 761, 751, 795, 753, 786, 752, 802, 761, 790, 812, 804, 807, 722, 765, 773, 780, 750, 747, 763, 748, 775, 788, 751, 715, 774, 816, 758, 779, 729, 758, 826, 772, 757, 748, 759, 734, 770, 790, 814, 772, 752, 756, 775, 734, 785, 770] payload_rows_per_dst=[9354, 9246, 9216, 9440, 9356, 9162, 9234, 9218] dup_rows_per_dst=[3140, 3106, 2934, 3086, 2996, 2818, 3078, 2920] combine_rows_per_src=[9354, 9246, 9216, 9440, 9356, 9162, 9234, 9218] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74324 payload_rows_remote=64994 dup_rows=23980 dup_rows_remote=20954 dedup_ratio=0.243937 router_tokens_per_expert=[810, 772, 737, 799, 780, 796, 770, 702, 739, 745, 776, 723, 799, 786, 784, 778, 803, 793, 765, 759, 773, 734, 764, 787, 778, 812, 794, 751, 745, 779, 801, 784, 739, 707, 728, 777, 795, 768, 764, 767, 821, 822, 763, 733, 775, 761, 791, 766, 771, 726, 769, 730, 761, 724, 791, 776, 765, 766, 786, 781, 803, 759, 741, 708] payload_rows_per_dst=[9300, 9364, 9330, 9460, 9232, 9356, 9128, 9154] dup_rows_per_dst=[3032, 2896, 3026, 3028, 2858, 3108, 2968, 3064] combine_rows_per_src=[9300, 9364, 9330, 9460, 9232, 9356, 9128, 9154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74548 payload_rows_remote=65234 dup_rows=23756 dup_rows_remote=20782 dedup_ratio=0.241659 router_tokens_per_expert=[804, 785, 755, 766, 720, 755, 727, 754, 776, 787, 769, 777, 812, 751, 800, 721, 735, 827, 742, 775, 786, 796, 760, 766, 719, 762, 753, 770, 766, 763, 745, 781, 767, 793, 748, 788, 788, 749, 758, 753, 805, 774, 783, 775, 714, 791, 755, 820, 765, 782, 800, 765, 766, 727, 730, 799, 778, 784, 779, 759, 757, 765, 784, 746] payload_rows_per_dst=[9200, 9326, 9358, 9174, 9314, 9480, 9364, 9332] dup_rows_per_dst=[2932, 3060, 3016, 2944, 2974, 2954, 2904, 2972] combine_rows_per_src=[9200, 9326, 9358, 9174, 9314, 9480, 9364, 9332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=344856 combine_payload_bytes=266412032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266756888 ingress_bytes=301015040 link_bytes=301015040 token_bytes=533168920 weight_movement_bytes=34603008 total_nvlink_bytes=567771928 weight_to_token_ratio=0.064901 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132695362e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74580 payload_rows_remote=65096 dup_rows=23724 dup_rows_remote=20720 dedup_ratio=0.241333 router_tokens_per_expert=[747, 787, 740, 754, 785, 744, 724, 755, 776, 755, 750, 757, 749, 773, 745, 783, 752, 764, 761, 741, 775, 816, 738, 783, 781, 788, 752, 776, 773, 743, 701, 816, 756, 806, 780, 749, 772, 751, 758, 790, 801, 810, 785, 744, 761, 744, 751, 807, 788, 758, 759, 747, 742, 790, 786, 797, 784, 787, 762, 771, 850, 747, 795, 740] payload_rows_per_dst=[9180, 9248, 9152, 9472, 9314, 9174, 9556, 9484] dup_rows_per_dst=[2892, 2928, 2874, 3022, 3010, 2768, 3226, 3004] combine_rows_per_src=[9180, 9248, 9152, 9472, 9314, 9174, 9556, 9484] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265715712 dispatch_meta_bytes=343240 combine_payload_bytes=265715712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266058952 ingress_bytes=300318720 link_bytes=300318720 token_bytes=531774664 weight_movement_bytes=34603008 total_nvlink_bytes=566377672 weight_to_token_ratio=0.065071 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130075166e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=65046 dup_rows=23964 dup_rows_remote=21092 dedup_ratio=0.243774 router_tokens_per_expert=[802, 826, 789, 771, 775, 771, 740, 783, 777, 757, 734, 786, 740, 799, 775, 716, 768, 813, 818, 751, 754, 774, 785, 776, 783, 760, 733, 823, 750, 761, 782, 691, 755, 757, 766, 785, 750, 765, 800, 773, 772, 764, 755, 767, 753, 755, 751, 781, 771, 786, 723, 772, 729, 762, 762, 782, 770, 756, 760, 797, 764, 765, 819, 722] payload_rows_per_dst=[9522, 9162, 9382, 9294, 9294, 9210, 9210, 9266] dup_rows_per_dst=[2992, 3006, 3096, 2872, 3008, 2986, 2964, 3040] combine_rows_per_src=[9522, 9162, 9382, 9294, 9294, 9210, 9210, 9266] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266215424 dispatch_meta_bytes=343792 combine_payload_bytes=266215424 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266559216 ingress_bytes=335421440 link_bytes=335421440 token_bytes=532774640 weight_movement_bytes=69206016 total_nvlink_bytes=601980656 weight_to_token_ratio=0.129897 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262163875e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=64962 dup_rows=24020 dup_rows_remote=20938 dedup_ratio=0.244344 router_tokens_per_expert=[772, 736, 770, 781, 739, 738, 789, 815, 784, 754, 741, 758, 766, 795, 754, 804, 791, 755, 760, 832, 763, 709, 769, 743, 746, 763, 739, 742, 754, 790, 812, 754, 758, 792, 740, 828, 710, 751, 820, 761, 771, 755, 800, 809, 789, 759, 744, 775, 790, 747, 748, 799, 717, 790, 768, 780, 778, 750, 762, 754, 747, 789, 785, 768] payload_rows_per_dst=[9336, 9236, 9240, 9242, 9304, 9322, 9332, 9272] dup_rows_per_dst=[2944, 3076, 3004, 2958, 3016, 3082, 2946, 2994] combine_rows_per_src=[9336, 9236, 9240, 9242, 9304, 9322, 9332, 9272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266633216 dispatch_meta_bytes=343264 combine_payload_bytes=266633216 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266976480 ingress_bytes=370442240 link_bytes=370442240 token_bytes=533609696 weight_movement_bytes=103809024 total_nvlink_bytes=637418720 weight_to_token_ratio=0.194541 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393944326e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74458 payload_rows_remote=65140 dup_rows=23846 dup_rows_remote=20806 dedup_ratio=0.242574 router_tokens_per_expert=[778, 737, 759, 786, 794, 736, 764, 751, 783, 766, 816, 734, 776, 804, 730, 770, 785, 765, 737, 782, 785, 783, 769, 787, 745, 775, 759, 754, 706, 822, 798, 771, 755, 777, 788, 820, 766, 722, 751, 787, 819, 747, 824, 743, 806, 746, 769, 761, 797, 750, 760, 731, 781, 754, 738, 757, 743, 763, 813, 740, 785, 747, 687, 818] payload_rows_per_dst=[9276, 9318, 9382, 9250, 9328, 9432, 9188, 9284] dup_rows_per_dst=[2934, 3040, 3004, 3010, 3004, 2998, 2948, 2908] combine_rows_per_src=[9276, 9318, 9382, 9250, 9328, 9432, 9188, 9284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74432 payload_rows_remote=65162 dup_rows=23872 dup_rows_remote=20884 dedup_ratio=0.242839 router_tokens_per_expert=[765, 787, 746, 791, 760, 731, 750, 805, 770, 826, 764, 757, 747, 750, 773, 759, 765, 748, 770, 775, 798, 776, 752, 761, 743, 795, 699, 790, 752, 743, 748, 756, 769, 727, 800, 778, 763, 738, 792, 781, 794, 801, 769, 752, 766, 809, 784, 752, 752, 775, 733, 807, 772, 770, 778, 742, 766, 758, 801, 761, 807, 736, 802, 765] payload_rows_per_dst=[9222, 9346, 9330, 9148, 9312, 9438, 9270, 9366] dup_rows_per_dst=[3048, 2946, 2960, 2904, 2984, 3016, 2988, 3026] combine_rows_per_src=[9222, 9346, 9330, 9148, 9312, 9438, 9270, 9366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74548 payload_rows_remote=65234 dup_rows=23756 dup_rows_remote=20782 dedup_ratio=0.241659 router_tokens_per_expert=[804, 785, 755, 766, 720, 755, 727, 754, 776, 787, 769, 777, 812, 751, 800, 721, 735, 827, 742, 775, 786, 796, 760, 766, 719, 762, 753, 770, 766, 763, 745, 781, 767, 793, 748, 788, 788, 749, 758, 753, 805, 774, 783, 775, 714, 791, 755, 820, 765, 782, 800, 765, 766, 727, 730, 799, 778, 784, 779, 759, 757, 765, 784, 746] payload_rows_per_dst=[9200, 9326, 9358, 9174, 9314, 9480, 9364, 9332] dup_rows_per_dst=[2932, 3060, 3016, 2944, 2974, 2954, 2904, 2972] combine_rows_per_src=[9200, 9326, 9358, 9174, 9314, 9480, 9364, 9332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=65046 dup_rows=23964 dup_rows_remote=21092 dedup_ratio=0.243774 router_tokens_per_expert=[802, 826, 789, 771, 775, 771, 740, 783, 777, 757, 734, 786, 740, 799, 775, 716, 768, 813, 818, 751, 754, 774, 785, 776, 783, 760, 733, 823, 750, 761, 782, 691, 755, 757, 766, 785, 750, 765, 800, 773, 772, 764, 755, 767, 753, 755, 751, 781, 771, 786, 723, 772, 729, 762, 762, 782, 770, 756, 760, 797, 764, 765, 819, 722] payload_rows_per_dst=[9522, 9162, 9382, 9294, 9294, 9210, 9210, 9266] dup_rows_per_dst=[2992, 3006, 3096, 2872, 3008, 2986, 2964, 3040] combine_rows_per_src=[9522, 9162, 9382, 9294, 9294, 9210, 9210, 9266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74210 payload_rows_remote=64936 dup_rows=24094 dup_rows_remote=21082 dedup_ratio=0.245097 router_tokens_per_expert=[739, 734, 769, 786, 748, 765, 761, 745, 746, 818, 707, 767, 734, 762, 723, 770, 786, 804, 794, 796, 758, 762, 747, 814, 712, 812, 741, 805, 756, 762, 731, 767, 752, 811, 782, 780, 763, 734, 724, 760, 774, 776, 785, 834, 770, 772, 792, 756, 778, 794, 722, 738, 778, 767, 770, 796, 814, 775, 775, 787, 771, 744, 761, 796] payload_rows_per_dst=[9172, 9098, 9424, 9156, 9226, 9454, 9274, 9406] dup_rows_per_dst=[2922, 2956, 3098, 3016, 2986, 3064, 3012, 3040] combine_rows_per_src=[9172, 9098, 9424, 9156, 9226, 9454, 9274, 9406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=64976 dup_rows=24032 dup_rows_remote=21100 dedup_ratio=0.244466 router_tokens_per_expert=[768, 753, 758, 790, 755, 792, 783, 833, 741, 821, 744, 737, 769, 719, 747, 787, 762, 776, 765, 765, 741, 745, 788, 772, 813, 738, 770, 733, 770, 773, 779, 821, 769, 759, 780, 761, 724, 763, 746, 750, 698, 788, 746, 813, 779, 794, 801, 783, 761, 735, 754, 775, 760, 705, 793, 780, 765, 758, 819, 760, 783, 773, 759, 810] payload_rows_per_dst=[9446, 9156, 9296, 9350, 9142, 9322, 9230, 9330] dup_rows_per_dst=[3018, 2974, 2932, 3044, 2962, 3082, 2896, 3124] combine_rows_per_src=[9446, 9156, 9296, 9350, 9142, 9322, 9230, 9330] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74048 payload_rows_remote=64644 dup_rows=24256 dup_rows_remote=21114 dedup_ratio=0.246745 router_tokens_per_expert=[779, 749, 756, 766, 745, 790, 825, 784, 787, 771, 773, 790, 727, 725, 786, 712, 795, 776, 765, 778, 820, 770, 757, 786, 787, 765, 780, 760, 828, 753, 781, 731, 732, 772, 792, 765, 775, 711, 756, 744, 759, 791, 779, 767, 737, 755, 751, 801, 721, 740, 759, 743, 760, 795, 813, 737, 785, 778, 794, 757, 761, 800, 791, 734] payload_rows_per_dst=[9294, 8820, 9328, 9442, 9338, 9188, 9234, 9404] dup_rows_per_dst=[3094, 2542, 3020, 3282, 3182, 3092, 2902, 3142] combine_rows_per_src=[9294, 8820, 9328, 9442, 9338, 9188, 9234, 9404] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266084352 dispatch_meta_bytes=343600 combine_payload_bytes=266084352 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266427952 ingress_bytes=404496384 link_bytes=404496384 token_bytes=532512304 weight_movement_bytes=138412032 total_nvlink_bytes=670924336 weight_to_token_ratio=0.259923 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522087328e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266813440 dispatch_meta_bytes=343784 combine_payload_bytes=266813440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267157224 ingress_bytes=301416448 link_bytes=301416448 token_bytes=533970664 weight_movement_bytes=34603008 total_nvlink_bytes=568573672 weight_to_token_ratio=0.064803 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134205828e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266903552 dispatch_meta_bytes=344184 combine_payload_bytes=266903552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267247736 ingress_bytes=301506560 link_bytes=301506560 token_bytes=534151288 weight_movement_bytes=34603008 total_nvlink_bytes=568754296 weight_to_token_ratio=0.064781 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134544912e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267198464 dispatch_meta_bytes=344064 combine_payload_bytes=267198464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267542528 ingress_bytes=301801472 link_bytes=301801472 token_bytes=534740992 weight_movement_bytes=34603008 total_nvlink_bytes=569344000 weight_to_token_ratio=0.064710 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135654642e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=64864 dup_rows=24040 dup_rows_remote=21004 dedup_ratio=0.244548 router_tokens_per_expert=[781, 739, 803, 799, 779, 819, 714, 746, 794, 835, 756, 781, 774, 762, 803, 713, 767, 763, 805, 777, 758, 746, 733, 858, 749, 745, 756, 787, 742, 737, 748, 756, 746, 746, 740, 775, 744, 752, 788, 761, 769, 754, 791, 831, 754, 775, 759, 751, 774, 785, 741, 766, 724, 783, 817, 776, 770, 747, 774, 776, 730, 774, 772, 782] payload_rows_per_dst=[9338, 9400, 9326, 9114, 9188, 9358, 9266, 9274] dup_rows_per_dst=[3022, 3036, 3088, 2926, 2916, 3010, 3066, 2976] combine_rows_per_src=[9338, 9400, 9326, 9114, 9188, 9358, 9266, 9274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74164 payload_rows_remote=64726 dup_rows=24140 dup_rows_remote=20828 dedup_ratio=0.245565 router_tokens_per_expert=[788, 807, 782, 768, 783, 752, 749, 761, 776, 787, 812, 746, 791, 768, 761, 774, 761, 753, 755, 752, 744, 761, 702, 787, 712, 781, 747, 819, 807, 754, 783, 729, 723, 758, 787, 763, 798, 748, 795, 757, 751, 746, 798, 802, 764, 806, 793, 719, 754, 750, 764, 786, 778, 811, 748, 769, 737, 753, 796, 769, 783, 754, 756, 784] payload_rows_per_dst=[9438, 9424, 9058, 9334, 9206, 9232, 9218, 9254] dup_rows_per_dst=[3312, 3080, 2902, 2930, 3052, 2842, 3012, 3010] combine_rows_per_src=[9438, 9424, 9058, 9334, 9206, 9232, 9218, 9254] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=344552 combine_payload_bytes=266428416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266772968 ingress_bytes=301031424 link_bytes=301031424 token_bytes=533201384 weight_movement_bytes=34603008 total_nvlink_bytes=567804392 weight_to_token_ratio=0.064897 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132757013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266141696 dispatch_meta_bytes=344304 combine_payload_bytes=266141696 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266486000 ingress_bytes=335347712 link_bytes=335347712 token_bytes=532627696 weight_movement_bytes=69206016 total_nvlink_bytes=601833712 weight_to_token_ratio=0.129933 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261886443e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74048 payload_rows_remote=64644 dup_rows=24256 dup_rows_remote=21114 dedup_ratio=0.246745 router_tokens_per_expert=[779, 749, 756, 766, 745, 790, 825, 784, 787, 771, 773, 790, 727, 725, 786, 712, 795, 776, 765, 778, 820, 770, 757, 786, 787, 765, 780, 760, 828, 753, 781, 731, 732, 772, 792, 765, 775, 711, 756, 744, 759, 791, 779, 767, 737, 755, 751, 801, 721, 740, 759, 743, 760, 795, 813, 737, 785, 778, 794, 757, 761, 800, 791, 734] payload_rows_per_dst=[9294, 8820, 9328, 9442, 9338, 9188, 9234, 9404] dup_rows_per_dst=[3094, 2542, 3020, 3282, 3182, 3092, 2902, 3142] combine_rows_per_src=[9294, 8820, 9328, 9442, 9338, 9188, 9234, 9404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265117696 dispatch_meta_bytes=342216 combine_payload_bytes=265117696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265459912 ingress_bytes=299720704 link_bytes=299720704 token_bytes=530577608 weight_movement_bytes=34603008 total_nvlink_bytes=565180616 weight_to_token_ratio=0.065218 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127824880e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266084352 dispatch_meta_bytes=343600 combine_payload_bytes=266084352 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266427952 ingress_bytes=404496384 link_bytes=404496384 token_bytes=532512304 weight_movement_bytes=138412032 total_nvlink_bytes=670924336 weight_to_token_ratio=0.259923 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522087328e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265977856 dispatch_meta_bytes=344072 combine_payload_bytes=265977856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266321928 ingress_bytes=300580864 link_bytes=300580864 token_bytes=532299784 weight_movement_bytes=34603008 total_nvlink_bytes=566902792 weight_to_token_ratio=0.065007 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131061592e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=264781824 dispatch_meta_bytes=343032 combine_payload_bytes=264781824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265124856 ingress_bytes=299384832 link_bytes=299384832 token_bytes=529906680 weight_movement_bytes=34603008 total_nvlink_bytes=564509688 weight_to_token_ratio=0.065300 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126561020e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265682944 dispatch_meta_bytes=343472 combine_payload_bytes=265682944 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266026416 ingress_bytes=369491968 link_bytes=369491968 token_bytes=531709360 weight_movement_bytes=103809024 total_nvlink_bytes=635518384 weight_to_token_ratio=0.195236 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390368529e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74164 payload_rows_remote=64726 dup_rows=24140 dup_rows_remote=20828 dedup_ratio=0.245565 router_tokens_per_expert=[788, 807, 782, 768, 783, 752, 749, 761, 776, 787, 812, 746, 791, 768, 761, 774, 761, 753, 755, 752, 744, 761, 702, 787, 712, 781, 747, 819, 807, 754, 783, 729, 723, 758, 787, 763, 798, 748, 795, 757, 751, 746, 798, 802, 764, 806, 793, 719, 754, 750, 764, 786, 778, 811, 748, 769, 737, 753, 796, 769, 783, 754, 756, 784] payload_rows_per_dst=[9438, 9424, 9058, 9334, 9206, 9232, 9218, 9254] dup_rows_per_dst=[3312, 3080, 2902, 2930, 3052, 2842, 3012, 3010] combine_rows_per_src=[9438, 9424, 9058, 9334, 9206, 9232, 9218, 9254] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74676 payload_rows_remote=65204 dup_rows=23628 dup_rows_remote=20530 dedup_ratio=0.240356 router_tokens_per_expert=[736, 726, 751, 748, 749, 763, 734, 739, 732, 763, 785, 745, 764, 782, 787, 802, 756, 776, 729, 792, 873, 790, 771, 798, 809, 785, 781, 703, 764, 782, 755, 832, 745, 798, 814, 759, 757, 771, 781, 756, 797, 731, 770, 778, 814, 760, 717, 802, 712, 721, 757, 777, 806, 791, 771, 707, 786, 742, 719, 806, 779, 737, 789, 800] payload_rows_per_dst=[9080, 9376, 9472, 9364, 9436, 9368, 9218, 9362] dup_rows_per_dst=[2812, 2944, 3098, 3058, 2926, 2970, 2866, 2954] combine_rows_per_src=[9080, 9376, 9472, 9364, 9436, 9368, 9218, 9362] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74406 payload_rows_remote=65066 dup_rows=23898 dup_rows_remote=20842 dedup_ratio=0.243103 router_tokens_per_expert=[787, 760, 785, 770, 745, 794, 753, 774, 783, 829, 772, 769, 738, 751, 757, 782, 766, 748, 735, 718, 737, 774, 725, 798, 767, 757, 756, 741, 748, 771, 815, 797, 738, 819, 758, 780, 756, 759, 761, 720, 773, 755, 793, 769, 720, 777, 771, 811, 785, 762, 793, 772, 749, 775, 779, 783, 786, 767, 785, 779, 750, 777, 774, 774] payload_rows_per_dst=[9348, 9322, 9160, 9278, 9268, 9340, 9340, 9350] dup_rows_per_dst=[2988, 3040, 2842, 3026, 2914, 2998, 3056, 3034] combine_rows_per_src=[9348, 9322, 9160, 9278, 9268, 9340, 9340, 9350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=64864 dup_rows=24040 dup_rows_remote=21004 dedup_ratio=0.244548 router_tokens_per_expert=[781, 739, 803, 799, 779, 819, 714, 746, 794, 835, 756, 781, 774, 762, 803, 713, 767, 763, 805, 777, 758, 746, 733, 858, 749, 745, 756, 787, 742, 737, 748, 756, 746, 746, 740, 775, 744, 752, 788, 761, 769, 754, 791, 831, 754, 775, 759, 751, 774, 785, 741, 766, 724, 783, 817, 776, 770, 747, 774, 776, 730, 774, 772, 782] payload_rows_per_dst=[9338, 9400, 9326, 9114, 9188, 9358, 9266, 9274] dup_rows_per_dst=[3022, 3036, 3088, 2926, 2916, 3010, 3066, 2976] combine_rows_per_src=[9338, 9400, 9326, 9114, 9188, 9358, 9266, 9274] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74282 payload_rows_remote=64914 dup_rows=24022 dup_rows_remote=20928 dedup_ratio=0.244364 router_tokens_per_expert=[733, 794, 755, 773, 716, 757, 793, 810, 766, 826, 757, 715, 809, 743, 786, 787, 780, 767, 770, 827, 780, 799, 823, 730, 767, 755, 749, 741, 743, 806, 780, 758, 789, 746, 785, 789, 773, 779, 821, 749, 798, 802, 742, 796, 739, 725, 767, 762, 771, 743, 752, 735, 750, 816, 764, 745, 751, 736, 775, 719, 734, 751, 774, 779] payload_rows_per_dst=[9236, 9238, 9424, 9256, 9368, 9322, 9312, 9126] dup_rows_per_dst=[3026, 3140, 3128, 2942, 3094, 2940, 2840, 2912] combine_rows_per_src=[9236, 9238, 9424, 9256, 9368, 9322, 9312, 9126] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265117696 dispatch_meta_bytes=342216 combine_payload_bytes=265117696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265459912 ingress_bytes=299720704 link_bytes=299720704 token_bytes=530577608 weight_movement_bytes=34603008 total_nvlink_bytes=565180616 weight_to_token_ratio=0.065218 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127824880e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=267075584 dispatch_meta_bytes=342936 combine_payload_bytes=267075584 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267418520 ingress_bytes=370884608 link_bytes=370884608 token_bytes=534494104 weight_movement_bytes=103809024 total_nvlink_bytes=638303128 weight_to_token_ratio=0.194219 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395608921e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73942 payload_rows_remote=64714 dup_rows=24362 dup_rows_remote=21292 dedup_ratio=0.247823 router_tokens_per_expert=[764, 794, 758, 746, 757, 732, 745, 813, 774, 768, 803, 762, 805, 791, 763, 763, 731, 809, 772, 770, 756, 762, 766, 747, 764, 769, 785, 786, 777, 746, 744, 778, 737, 796, 800, 763, 757, 764, 735, 773, 795, 839, 757, 741, 746, 756, 732, 731, 797, 789, 760, 785, 808, 756, 750, 775, 740, 741, 790, 796, 760, 768, 776, 739] payload_rows_per_dst=[9216, 9354, 9242, 9228, 9250, 9166, 9290, 9196] dup_rows_per_dst=[3002, 3104, 2984, 3070, 3000, 3028, 3150, 3024] combine_rows_per_src=[9216, 9354, 9242, 9228, 9250, 9166, 9290, 9196] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=64858 dup_rows=24048 dup_rows_remote=20996 dedup_ratio=0.244629 router_tokens_per_expert=[720, 740, 752, 796, 756, 725, 783, 777, 770, 772, 817, 769, 798, 787, 765, 764, 707, 747, 773, 770, 804, 778, 785, 752, 764, 759, 758, 760, 781, 759, 744, 702, 731, 781, 757, 749, 749, 783, 785, 748, 796, 761, 769, 777, 788, 764, 736, 834, 809, 822, 718, 814, 767, 801, 809, 727, 762, 735, 746, 758, 769, 781, 770, 822] payload_rows_per_dst=[9094, 9344, 9226, 9160, 9230, 9398, 9454, 9350] dup_rows_per_dst=[3004, 3140, 3006, 2894, 2936, 3052, 3080, 2936] combine_rows_per_src=[9094, 9344, 9226, 9160, 9230, 9398, 9454, 9350] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74432 payload_rows_remote=65162 dup_rows=23872 dup_rows_remote=20884 dedup_ratio=0.242839 router_tokens_per_expert=[765, 787, 746, 791, 760, 731, 750, 805, 770, 826, 764, 757, 747, 750, 773, 759, 765, 748, 770, 775, 798, 776, 752, 761, 743, 795, 699, 790, 752, 743, 748, 756, 769, 727, 800, 778, 763, 738, 792, 781, 794, 801, 769, 752, 766, 809, 784, 752, 752, 775, 733, 807, 772, 770, 778, 742, 766, 758, 801, 761, 807, 736, 802, 765] payload_rows_per_dst=[9222, 9346, 9330, 9148, 9312, 9438, 9270, 9366] dup_rows_per_dst=[3048, 2946, 2960, 2904, 2984, 3016, 2988, 3026] combine_rows_per_src=[9222, 9346, 9330, 9148, 9312, 9438, 9270, 9366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=65006 dup_rows=24048 dup_rows_remote=21038 dedup_ratio=0.244629 router_tokens_per_expert=[783, 805, 741, 788, 736, 769, 757, 798, 769, 773, 754, 747, 720, 745, 744, 777, 780, 738, 780, 771, 759, 768, 788, 766, 754, 786, 759, 799, 761, 764, 802, 777, 783, 775, 721, 801, 748, 792, 797, 780, 781, 762, 719, 789, 815, 779, 776, 734, 791, 789, 828, 795, 794, 763, 735, 758, 767, 672, 747, 751, 758, 789, 763, 742] payload_rows_per_dst=[9400, 9146, 8982, 9396, 9322, 9172, 9588, 9250] dup_rows_per_dst=[2954, 2912, 2796, 3140, 2988, 2964, 3284, 3010] combine_rows_per_src=[9400, 9146, 8982, 9396, 9322, 9172, 9588, 9250] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265887744 dispatch_meta_bytes=343368 combine_payload_bytes=265887744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266231112 ingress_bytes=300490752 link_bytes=300490752 token_bytes=532118856 weight_movement_bytes=34603008 total_nvlink_bytes=566721864 weight_to_token_ratio=0.065029 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130722508e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73942 payload_rows_remote=64714 dup_rows=24362 dup_rows_remote=21292 dedup_ratio=0.247823 router_tokens_per_expert=[764, 794, 758, 746, 757, 732, 745, 813, 774, 768, 803, 762, 805, 791, 763, 763, 731, 809, 772, 770, 756, 762, 766, 747, 764, 769, 785, 786, 777, 746, 744, 778, 737, 796, 800, 763, 757, 764, 735, 773, 795, 839, 757, 741, 746, 756, 732, 731, 797, 789, 760, 785, 808, 756, 750, 775, 740, 741, 790, 796, 760, 768, 776, 739] payload_rows_per_dst=[9216, 9354, 9242, 9228, 9250, 9166, 9290, 9196] dup_rows_per_dst=[3002, 3104, 2984, 3070, 3000, 3028, 3150, 3024] combine_rows_per_src=[9216, 9354, 9242, 9228, 9250, 9166, 9290, 9196] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74324 payload_rows_remote=64994 dup_rows=23980 dup_rows_remote=20954 dedup_ratio=0.243937 router_tokens_per_expert=[810, 772, 737, 799, 780, 796, 770, 702, 739, 745, 776, 723, 799, 786, 784, 778, 803, 793, 765, 759, 773, 734, 764, 787, 778, 812, 794, 751, 745, 779, 801, 784, 739, 707, 728, 777, 795, 768, 764, 767, 821, 822, 763, 733, 775, 761, 791, 766, 771, 726, 769, 730, 761, 724, 791, 776, 765, 766, 786, 781, 803, 759, 741, 708] payload_rows_per_dst=[9300, 9364, 9330, 9460, 9232, 9356, 9128, 9154] dup_rows_per_dst=[3032, 2896, 3026, 3028, 2858, 3108, 2968, 3064] combine_rows_per_src=[9300, 9364, 9330, 9460, 9232, 9356, 9128, 9154] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74428 payload_rows_remote=65142 dup_rows=23876 dup_rows_remote=20872 dedup_ratio=0.242879 router_tokens_per_expert=[804, 758, 777, 748, 752, 778, 744, 759, 770, 759, 784, 778, 746, 781, 780, 747, 772, 758, 770, 739, 744, 792, 816, 777, 770, 755, 789, 756, 758, 778, 812, 775, 750, 819, 793, 751, 773, 802, 778, 796, 763, 768, 798, 790, 783, 766, 733, 756, 755, 742, 716, 771, 730, 687, 748, 769, 788, 761, 765, 761, 781, 780, 786, 767] payload_rows_per_dst=[9250, 9286, 9282, 9328, 9484, 9372, 9068, 9358] dup_rows_per_dst=[2990, 3004, 3054, 3058, 3040, 2942, 2768, 3020] combine_rows_per_src=[9250, 9286, 9282, 9328, 9484, 9372, 9068, 9358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266510336 dispatch_meta_bytes=343632 combine_payload_bytes=266510336 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266853968 ingress_bytes=335716352 link_bytes=335716352 token_bytes=533364304 weight_movement_bytes=69206016 total_nvlink_bytes=602570320 weight_to_token_ratio=0.129754 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263273605e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74526 payload_rows_remote=65222 dup_rows=23778 dup_rows_remote=20852 dedup_ratio=0.241882 router_tokens_per_expert=[808, 743, 749, 736, 786, 779, 736, 778, 764, 763, 815, 721, 795, 759, 759, 766, 801, 768, 755, 789, 769, 714, 768, 752, 746, 758, 769, 743, 751, 746, 763, 806, 789, 783, 817, 760, 787, 721, 767, 761, 765, 803, 761, 792, 771, 770, 707, 767, 756, 805, 764, 792, 743, 733, 772, 793, 770, 773, 773, 811, 802, 802, 744, 743] payload_rows_per_dst=[9304, 9698, 9358, 8806, 9366, 9270, 9322, 9402] dup_rows_per_dst=[2926, 3306, 2874, 2638, 3004, 3002, 2994, 3034] combine_rows_per_src=[9304, 9698, 9358, 8806, 9366, 9270, 9322, 9402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265682944 dispatch_meta_bytes=343472 combine_payload_bytes=265682944 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266026416 ingress_bytes=369491968 link_bytes=369491968 token_bytes=531709360 weight_movement_bytes=103809024 total_nvlink_bytes=635518384 weight_to_token_ratio=0.195236 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390368529e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265887744 dispatch_meta_bytes=343368 combine_payload_bytes=265887744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266231112 ingress_bytes=300490752 link_bytes=300490752 token_bytes=532118856 weight_movement_bytes=34603008 total_nvlink_bytes=566721864 weight_to_token_ratio=0.065029 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130722508e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265658368 dispatch_meta_bytes=343416 combine_payload_bytes=265658368 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266001784 ingress_bytes=334864384 link_bytes=334864384 token_bytes=531660152 weight_movement_bytes=69206016 total_nvlink_bytes=600866168 weight_to_token_ratio=0.130170 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260067718e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265068544 dispatch_meta_bytes=344024 combine_payload_bytes=265068544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265412568 ingress_bytes=299671552 link_bytes=299671552 token_bytes=530481112 weight_movement_bytes=34603008 total_nvlink_bytes=565084120 weight_to_token_ratio=0.065229 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127639925e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266903552 dispatch_meta_bytes=344184 combine_payload_bytes=266903552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267247736 ingress_bytes=301506560 link_bytes=301506560 token_bytes=534151288 weight_movement_bytes=34603008 total_nvlink_bytes=568754296 weight_to_token_ratio=0.064781 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134544912e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=65006 dup_rows=24048 dup_rows_remote=21038 dedup_ratio=0.244629 router_tokens_per_expert=[783, 805, 741, 788, 736, 769, 757, 798, 769, 773, 754, 747, 720, 745, 744, 777, 780, 738, 780, 771, 759, 768, 788, 766, 754, 786, 759, 799, 761, 764, 802, 777, 783, 775, 721, 801, 748, 792, 797, 780, 781, 762, 719, 789, 815, 779, 776, 734, 791, 789, 828, 795, 794, 763, 735, 758, 767, 672, 747, 751, 758, 789, 763, 742] payload_rows_per_dst=[9400, 9146, 8982, 9396, 9322, 9172, 9588, 9250] dup_rows_per_dst=[2954, 2912, 2796, 3140, 2988, 2964, 3284, 3010] combine_rows_per_src=[9400, 9146, 8982, 9396, 9322, 9172, 9588, 9250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266215424 dispatch_meta_bytes=343792 combine_payload_bytes=266215424 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266559216 ingress_bytes=335421440 link_bytes=335421440 token_bytes=532774640 weight_movement_bytes=69206016 total_nvlink_bytes=601980656 weight_to_token_ratio=0.129897 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262163875e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=267149312 dispatch_meta_bytes=344296 combine_payload_bytes=267149312 prefetch_bytes=0 reduce_bytes=0 egress_bytes=267493608 ingress_bytes=267149312 link_bytes=267493608 token_bytes=534642920 weight_movement_bytes=0 total_nvlink_bytes=534642920 weight_to_token_ratio=0.000000 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.006556912e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266264576 dispatch_meta_bytes=344176 combine_payload_bytes=266264576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266608752 ingress_bytes=300867584 link_bytes=300867584 token_bytes=532873328 weight_movement_bytes=34603008 total_nvlink_bytes=567476336 weight_to_token_ratio=0.064937 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132140497e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266821632 dispatch_meta_bytes=344056 combine_payload_bytes=266821632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267165688 ingress_bytes=301424640 link_bytes=301424640 token_bytes=533987320 weight_movement_bytes=34603008 total_nvlink_bytes=568590328 weight_to_token_ratio=0.064801 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134236654e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74526 payload_rows_remote=65222 dup_rows=23778 dup_rows_remote=20852 dedup_ratio=0.241882 router_tokens_per_expert=[808, 743, 749, 736, 786, 779, 736, 778, 764, 763, 815, 721, 795, 759, 759, 766, 801, 768, 755, 789, 769, 714, 768, 752, 746, 758, 769, 743, 751, 746, 763, 806, 789, 783, 817, 760, 787, 721, 767, 761, 765, 803, 761, 792, 771, 770, 707, 767, 756, 805, 764, 792, 743, 733, 772, 793, 770, 773, 773, 811, 802, 802, 744, 743] payload_rows_per_dst=[9304, 9698, 9358, 8806, 9366, 9270, 9322, 9402] dup_rows_per_dst=[2926, 3306, 2874, 2638, 3004, 3002, 2994, 3034] combine_rows_per_src=[9304, 9698, 9358, 8806, 9366, 9270, 9322, 9402] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65078 dup_rows=23908 dup_rows_remote=20896 dedup_ratio=0.243205 router_tokens_per_expert=[732, 784, 723, 715, 768, 724, 738, 736, 766, 773, 744, 772, 789, 768, 761, 735, 772, 783, 779, 764, 748, 760, 772, 787, 743, 758, 841, 796, 757, 747, 818, 797, 738, 778, 757, 733, 788, 803, 743, 808, 791, 746, 798, 771, 809, 755, 809, 787, 790, 749, 776, 744, 713, 747, 796, 803, 777, 787, 775, 758, 788, 759, 788, 738] payload_rows_per_dst=[9102, 9228, 9318, 9434, 9330, 9426, 9200, 9358] dup_rows_per_dst=[2738, 2988, 3012, 3080, 2966, 3106, 3036, 2982] combine_rows_per_src=[9102, 9228, 9318, 9434, 9330, 9426, 9200, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73964 payload_rows_remote=64752 dup_rows=24340 dup_rows_remote=21302 dedup_ratio=0.247599 router_tokens_per_expert=[757, 769, 822, 769, 764, 771, 770, 822, 713, 758, 769, 737, 756, 766, 796, 781, 764, 813, 768, 791, 782, 775, 789, 754, 781, 795, 752, 787, 783, 750, 782, 715, 759, 758, 784, 757, 776, 783, 718, 726, 741, 789, 789, 764, 770, 748, 717, 779, 758, 783, 758, 734, 746, 791, 758, 797, 752, 790, 739, 814, 731, 791, 797, 754] payload_rows_per_dst=[9356, 9140, 9348, 9242, 9114, 9268, 9212, 9284] dup_rows_per_dst=[3132, 3012, 3124, 3048, 3008, 2926, 3038, 3052] combine_rows_per_src=[9356, 9140, 9348, 9242, 9114, 9268, 9212, 9284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74428 payload_rows_remote=65142 dup_rows=23876 dup_rows_remote=20872 dedup_ratio=0.242879 router_tokens_per_expert=[804, 758, 777, 748, 752, 778, 744, 759, 770, 759, 784, 778, 746, 781, 780, 747, 772, 758, 770, 739, 744, 792, 816, 777, 770, 755, 789, 756, 758, 778, 812, 775, 750, 819, 793, 751, 773, 802, 778, 796, 763, 768, 798, 790, 783, 766, 733, 756, 755, 742, 716, 771, 730, 687, 748, 769, 788, 761, 765, 761, 781, 780, 786, 767] payload_rows_per_dst=[9250, 9286, 9282, 9328, 9484, 9372, 9068, 9358] dup_rows_per_dst=[2990, 3004, 3054, 3058, 3040, 2942, 2768, 3020] combine_rows_per_src=[9250, 9286, 9282, 9328, 9484, 9372, 9068, 9358] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74280 payload_rows_remote=65092 dup_rows=24024 dup_rows_remote=21016 dedup_ratio=0.244385 router_tokens_per_expert=[734, 749, 774, 796, 786, 794, 757, 771, 731, 746, 747, 755, 802, 769, 747, 794, 786, 816, 781, 772, 760, 751, 750, 793, 817, 789, 854, 769, 799, 818, 827, 774, 734, 773, 763, 754, 777, 773, 780, 744, 767, 765, 733, 747, 762, 746, 794, 747, 753, 768, 744, 783, 743, 749, 728, 782, 756, 738, 782, 712, 736, 738, 809, 764] payload_rows_per_dst=[9322, 9184, 9388, 9604, 9188, 9184, 9208, 9202] dup_rows_per_dst=[3000, 2998, 3030, 3290, 3008, 2938, 2892, 2868] combine_rows_per_src=[9322, 9184, 9388, 9604, 9188, 9184, 9208, 9202] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=64990 dup_rows=23924 dup_rows_remote=20928 dedup_ratio=0.243368 router_tokens_per_expert=[767, 776, 746, 764, 731, 773, 781, 791, 752, 745, 775, 786, 796, 740, 776, 781, 778, 741, 788, 760, 750, 780, 761, 791, 806, 795, 796, 691, 804, 703, 757, 729, 789, 764, 734, 757, 791, 772, 753, 826, 772, 787, 775, 784, 750, 760, 782, 783, 820, 782, 751, 756, 767, 752, 812, 783, 759, 704, 788, 772, 746, 754, 756, 761] payload_rows_per_dst=[9236, 9242, 9382, 9184, 9326, 9390, 9422, 9198] dup_rows_per_dst=[3022, 3060, 2916, 2978, 3046, 2996, 3024, 2882] combine_rows_per_src=[9236, 9242, 9382, 9184, 9326, 9390, 9422, 9198] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74406 payload_rows_remote=65066 dup_rows=23898 dup_rows_remote=20842 dedup_ratio=0.243103 router_tokens_per_expert=[787, 760, 785, 770, 745, 794, 753, 774, 783, 829, 772, 769, 738, 751, 757, 782, 766, 748, 735, 718, 737, 774, 725, 798, 767, 757, 756, 741, 748, 771, 815, 797, 738, 819, 758, 780, 756, 759, 761, 720, 773, 755, 793, 769, 720, 777, 771, 811, 785, 762, 793, 772, 749, 775, 779, 783, 786, 767, 785, 779, 750, 777, 774, 774] payload_rows_per_dst=[9348, 9322, 9160, 9278, 9268, 9340, 9340, 9350] dup_rows_per_dst=[2988, 3040, 2842, 3026, 2914, 2998, 3056, 3034] combine_rows_per_src=[9348, 9322, 9160, 9278, 9268, 9340, 9340, 9350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74542 payload_rows_remote=65270 dup_rows=23762 dup_rows_remote=20956 dedup_ratio=0.241720 router_tokens_per_expert=[824, 786, 777, 737, 798, 798, 779, 787, 753, 764, 759, 729, 737, 770, 758, 780, 774, 803, 786, 777, 760, 789, 748, 783, 747, 706, 726, 773, 765, 764, 802, 756, 761, 739, 759, 821, 719, 754, 705, 799, 806, 741, 816, 777, 757, 748, 763, 766, 748, 750, 775, 780, 793, 730, 755, 793, 774, 777, 816, 753, 793, 750, 784, 755] payload_rows_per_dst=[9452, 9092, 9412, 9272, 9304, 9320, 9290, 9400] dup_rows_per_dst=[3120, 3008, 3028, 2806, 2810, 3028, 2958, 3004] combine_rows_per_src=[9452, 9092, 9412, 9272, 9304, 9320, 9290, 9400] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266559488 dispatch_meta_bytes=343896 combine_payload_bytes=266559488 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266903384 ingress_bytes=370368512 link_bytes=370368512 token_bytes=533462872 weight_movement_bytes=103809024 total_nvlink_bytes=637271896 weight_to_token_ratio=0.194595 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393666894e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266616832 dispatch_meta_bytes=344432 combine_payload_bytes=266616832 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266961264 ingress_bytes=335822848 link_bytes=335822848 token_bytes=533578096 weight_movement_bytes=69206016 total_nvlink_bytes=602784112 weight_to_token_ratio=0.129702 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263674341e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74676 payload_rows_remote=65204 dup_rows=23628 dup_rows_remote=20530 dedup_ratio=0.240356 router_tokens_per_expert=[736, 726, 751, 748, 749, 763, 734, 739, 732, 763, 785, 745, 764, 782, 787, 802, 756, 776, 729, 792, 873, 790, 771, 798, 809, 785, 781, 703, 764, 782, 755, 832, 745, 798, 814, 759, 757, 771, 781, 756, 797, 731, 770, 778, 814, 760, 717, 802, 712, 721, 757, 777, 806, 791, 771, 707, 786, 742, 719, 806, 779, 737, 789, 800] payload_rows_per_dst=[9080, 9376, 9472, 9364, 9436, 9368, 9218, 9362] dup_rows_per_dst=[2812, 2944, 3098, 3058, 2926, 2970, 2866, 2954] combine_rows_per_src=[9080, 9376, 9472, 9364, 9436, 9368, 9218, 9362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265224192 dispatch_meta_bytes=344216 combine_payload_bytes=265224192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265568408 ingress_bytes=299827200 link_bytes=299827200 token_bytes=530792600 weight_movement_bytes=34603008 total_nvlink_bytes=565395608 weight_to_token_ratio=0.065191 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128225616e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=267149312 dispatch_meta_bytes=344296 combine_payload_bytes=267149312 prefetch_bytes=0 reduce_bytes=0 egress_bytes=267493608 ingress_bytes=267149312 link_bytes=267493608 token_bytes=534642920 weight_movement_bytes=0 total_nvlink_bytes=534642920 weight_to_token_ratio=0.000000 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.006556912e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=64850 dup_rows=24048 dup_rows_remote=21030 dedup_ratio=0.244629 router_tokens_per_expert=[774, 777, 781, 801, 795, 816, 729, 718, 751, 778, 792, 731, 756, 712, 695, 762, 746, 790, 753, 741, 754, 757, 729, 890, 742, 811, 759, 721, 795, 809, 753, 796, 723, 834, 747, 710, 740, 771, 773, 769, 755, 808, 786, 738, 788, 807, 780, 790, 780, 703, 740, 797, 762, 792, 800, 747, 803, 748, 762, 774, 735, 787, 832, 757] payload_rows_per_dst=[9296, 9110, 9304, 9238, 9162, 9454, 9286, 9406] dup_rows_per_dst=[3086, 2844, 3016, 3106, 2972, 3050, 2956, 3018] combine_rows_per_src=[9296, 9110, 9304, 9238, 9162, 9454, 9286, 9406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266821632 dispatch_meta_bytes=344056 combine_payload_bytes=266821632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267165688 ingress_bytes=301424640 link_bytes=301424640 token_bytes=533987320 weight_movement_bytes=34603008 total_nvlink_bytes=568590328 weight_to_token_ratio=0.064801 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134236654e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266616832 dispatch_meta_bytes=344432 combine_payload_bytes=266616832 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266961264 ingress_bytes=335822848 link_bytes=335822848 token_bytes=533578096 weight_movement_bytes=69206016 total_nvlink_bytes=602784112 weight_to_token_ratio=0.129702 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263674341e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74274 payload_rows_remote=64690 dup_rows=24030 dup_rows_remote=20782 dedup_ratio=0.244446 router_tokens_per_expert=[814, 757, 743, 688, 784, 746, 810, 770, 755, 766, 771, 778, 822, 726, 765, 750, 822, 777, 766, 797, 787, 731, 725, 782, 772, 778, 800, 761, 774, 733, 801, 761, 725, 766, 792, 813, 793, 741, 791, 733, 742, 758, 792, 756, 770, 752, 778, 727, 751, 755, 826, 746, 768, 846, 763, 771, 741, 770, 739, 754, 757, 755, 799, 770] payload_rows_per_dst=[9584, 9362, 9584, 9082, 9328, 8848, 9302, 9184] dup_rows_per_dst=[3248, 3172, 3230, 2842, 2996, 2674, 2972, 2896] combine_rows_per_src=[9584, 9362, 9584, 9082, 9328, 8848, 9302, 9184] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=343672 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266542712 ingress_bytes=335405056 link_bytes=335405056 token_bytes=532741752 weight_movement_bytes=69206016 total_nvlink_bytes=601947768 weight_to_token_ratio=0.129905 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262102223e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74324 payload_rows_remote=65092 dup_rows=23980 dup_rows_remote=21074 dedup_ratio=0.243937 router_tokens_per_expert=[716, 761, 734, 765, 802, 772, 778, 816, 759, 771, 746, 746, 756, 763, 798, 730, 776, 753, 768, 746, 784, 770, 772, 751, 790, 749, 718, 803, 784, 775, 775, 810, 780, 788, 721, 825, 795, 756, 747, 761, 816, 759, 791, 760, 750, 765, 767, 816, 783, 759, 810, 795, 784, 771, 720, 751, 729, 776, 739, 756, 762, 760, 757, 766] payload_rows_per_dst=[9250, 9232, 9288, 9392, 9278, 9320, 9280, 9284] dup_rows_per_dst=[3038, 2906, 2952, 3016, 3068, 3128, 3066, 2806] combine_rows_per_src=[9250, 9232, 9288, 9392, 9278, 9320, 9280, 9284] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266510336 dispatch_meta_bytes=343632 combine_payload_bytes=266510336 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266853968 ingress_bytes=335716352 link_bytes=335716352 token_bytes=533364304 weight_movement_bytes=69206016 total_nvlink_bytes=602570320 weight_to_token_ratio=0.129754 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263273605e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=64850 dup_rows=24048 dup_rows_remote=21030 dedup_ratio=0.244629 router_tokens_per_expert=[774, 777, 781, 801, 795, 816, 729, 718, 751, 778, 792, 731, 756, 712, 695, 762, 746, 790, 753, 741, 754, 757, 729, 890, 742, 811, 759, 721, 795, 809, 753, 796, 723, 834, 747, 710, 740, 771, 773, 769, 755, 808, 786, 738, 788, 807, 780, 790, 780, 703, 740, 797, 762, 792, 800, 747, 803, 748, 762, 774, 735, 787, 832, 757] payload_rows_per_dst=[9296, 9110, 9304, 9238, 9162, 9454, 9286, 9406] dup_rows_per_dst=[3086, 2844, 3016, 3106, 2972, 3050, 2956, 3018] combine_rows_per_src=[9296, 9110, 9304, 9238, 9162, 9454, 9286, 9406] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=267345920 dispatch_meta_bytes=344904 combine_payload_bytes=267345920 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267690824 ingress_bytes=336551936 link_bytes=336551936 token_bytes=535036744 weight_movement_bytes=69206016 total_nvlink_bytes=604242760 weight_to_token_ratio=0.129348 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266417841e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=267075584 dispatch_meta_bytes=342936 combine_payload_bytes=267075584 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267418520 ingress_bytes=370884608 link_bytes=370884608 token_bytes=534494104 weight_movement_bytes=103809024 total_nvlink_bytes=638303128 weight_to_token_ratio=0.194219 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395608921e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=264970240 dispatch_meta_bytes=341888 combine_payload_bytes=264970240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265312128 ingress_bytes=299573248 link_bytes=299573248 token_bytes=530282368 weight_movement_bytes=34603008 total_nvlink_bytes=564885376 weight_to_token_ratio=0.065254 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127270015e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265625600 dispatch_meta_bytes=343520 combine_payload_bytes=265625600 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265969120 ingress_bytes=334831616 link_bytes=334831616 token_bytes=531594720 weight_movement_bytes=69206016 total_nvlink_bytes=600800736 weight_to_token_ratio=0.130186 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259944415e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266616832 dispatch_meta_bytes=344664 combine_payload_bytes=266616832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266961496 ingress_bytes=301219840 link_bytes=301219840 token_bytes=533578328 weight_movement_bytes=34603008 total_nvlink_bytes=568181336 weight_to_token_ratio=0.064851 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133466008e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74462 payload_rows_remote=65018 dup_rows=23842 dup_rows_remote=20810 dedup_ratio=0.242533 router_tokens_per_expert=[718, 776, 769, 744, 760, 770, 785, 753, 759, 785, 834, 762, 734, 765, 778, 770, 726, 777, 765, 778, 791, 743, 761, 744, 784, 751, 801, 751, 777, 751, 748, 796, 753, 750, 727, 758, 753, 765, 767, 742, 824, 787, 776, 768, 763, 767, 781, 773, 765, 803, 758, 794, 779, 798, 787, 754, 737, 771, 753, 751, 822, 752, 764, 804] payload_rows_per_dst=[9256, 9256, 9192, 9348, 9162, 9422, 9444, 9382] dup_rows_per_dst=[2894, 3118, 2978, 2970, 2868, 3056, 3032, 2926] combine_rows_per_src=[9256, 9256, 9192, 9348, 9162, 9422, 9444, 9382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74324 payload_rows_remote=65092 dup_rows=23980 dup_rows_remote=21074 dedup_ratio=0.243937 router_tokens_per_expert=[716, 761, 734, 765, 802, 772, 778, 816, 759, 771, 746, 746, 756, 763, 798, 730, 776, 753, 768, 746, 784, 770, 772, 751, 790, 749, 718, 803, 784, 775, 775, 810, 780, 788, 721, 825, 795, 756, 747, 761, 816, 759, 791, 760, 750, 765, 767, 816, 783, 759, 810, 795, 784, 771, 720, 751, 729, 776, 739, 756, 762, 760, 757, 766] payload_rows_per_dst=[9250, 9232, 9288, 9392, 9278, 9320, 9280, 9284] dup_rows_per_dst=[3038, 2906, 2952, 3016, 3068, 3128, 3066, 2806] combine_rows_per_src=[9250, 9232, 9288, 9392, 9278, 9320, 9280, 9284] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74368 payload_rows_remote=65066 dup_rows=23936 dup_rows_remote=21072 dedup_ratio=0.243490 router_tokens_per_expert=[752, 746, 767, 728, 762, 713, 765, 744, 785, 736, 752, 767, 788, 783, 797, 814, 752, 788, 743, 751, 789, 703, 755, 739, 788, 802, 774, 810, 795, 780, 809, 732, 801, 766, 732, 743, 745, 747, 785, 764, 766, 807, 770, 817, 784, 711, 783, 747, 772, 757, 785, 773, 791, 765, 733, 756, 785, 752, 778, 749, 799, 823, 778, 779] payload_rows_per_dst=[9126, 9414, 9078, 9354, 9302, 9398, 9238, 9458] dup_rows_per_dst=[2828, 3030, 2962, 3226, 2864, 2972, 3026, 3028] combine_rows_per_src=[9126, 9414, 9078, 9354, 9302, 9398, 9238, 9458] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74522 payload_rows_remote=65316 dup_rows=23782 dup_rows_remote=20704 dedup_ratio=0.241923 router_tokens_per_expert=[733, 731, 782, 807, 756, 752, 752, 768, 748, 764, 772, 782, 762, 782, 779, 737, 762, 764, 779, 757, 776, 783, 793, 728, 794, 788, 728, 734, 748, 792, 756, 769, 774, 790, 807, 747, 805, 749, 764, 739, 782, 741, 786, 758, 796, 791, 755, 771, 804, 744, 741, 774, 763, 723, 788, 777, 781, 785, 770, 749, 760, 803, 764, 813] payload_rows_per_dst=[9226, 9302, 9206, 9370, 9418, 9308, 9242, 9450] dup_rows_per_dst=[2936, 2950, 3078, 2848, 2932, 3052, 2986, 3000] combine_rows_per_src=[9226, 9302, 9206, 9370, 9418, 9308, 9242, 9450] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74274 payload_rows_remote=64690 dup_rows=24030 dup_rows_remote=20782 dedup_ratio=0.244446 router_tokens_per_expert=[814, 757, 743, 688, 784, 746, 810, 770, 755, 766, 771, 778, 822, 726, 765, 750, 822, 777, 766, 797, 787, 731, 725, 782, 772, 778, 800, 761, 774, 733, 801, 761, 725, 766, 792, 813, 793, 741, 791, 733, 742, 758, 792, 756, 770, 752, 778, 727, 751, 755, 826, 746, 768, 846, 763, 771, 741, 770, 739, 754, 757, 755, 799, 770] payload_rows_per_dst=[9584, 9362, 9584, 9082, 9328, 8848, 9302, 9184] dup_rows_per_dst=[3248, 3172, 3230, 2842, 2996, 2674, 2972, 2896] combine_rows_per_src=[9584, 9362, 9584, 9082, 9328, 8848, 9302, 9184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74194 payload_rows_remote=64824 dup_rows=24110 dup_rows_remote=21062 dedup_ratio=0.245260 router_tokens_per_expert=[727, 793, 733, 739, 811, 759, 769, 740, 759, 786, 796, 770, 765, 741, 764, 691, 771, 762, 734, 738, 809, 736, 718, 805, 788, 802, 773, 748, 782, 786, 762, 752, 750, 795, 782, 807, 776, 801, 776, 774, 769, 732, 767, 796, 784, 796, 761, 804, 697, 760, 787, 769, 773, 812, 755, 750, 774, 775, 729, 762, 801, 744, 787, 798] payload_rows_per_dst=[9274, 9138, 9168, 9360, 9402, 9370, 9224, 9258] dup_rows_per_dst=[2868, 3006, 2978, 3026, 3120, 3048, 2982, 3082] combine_rows_per_src=[9274, 9138, 9168, 9360, 9402, 9370, 9224, 9258] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73964 payload_rows_remote=64752 dup_rows=24340 dup_rows_remote=21302 dedup_ratio=0.247599 router_tokens_per_expert=[757, 769, 822, 769, 764, 771, 770, 822, 713, 758, 769, 737, 756, 766, 796, 781, 764, 813, 768, 791, 782, 775, 789, 754, 781, 795, 752, 787, 783, 750, 782, 715, 759, 758, 784, 757, 776, 783, 718, 726, 741, 789, 789, 764, 770, 748, 717, 779, 758, 783, 758, 734, 746, 791, 758, 797, 752, 790, 739, 814, 731, 791, 797, 754] payload_rows_per_dst=[9356, 9140, 9348, 9242, 9114, 9268, 9212, 9284] dup_rows_per_dst=[3132, 3012, 3124, 3048, 3008, 2926, 3038, 3052] combine_rows_per_src=[9356, 9140, 9348, 9242, 9114, 9268, 9212, 9284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74460 payload_rows_remote=65202 dup_rows=23844 dup_rows_remote=20976 dedup_ratio=0.242554 router_tokens_per_expert=[814, 767, 746, 758, 775, 807, 775, 755, 786, 763, 778, 763, 769, 805, 748, 820, 767, 785, 756, 784, 743, 766, 770, 774, 748, 758, 780, 756, 737, 772, 765, 747, 728, 752, 777, 810, 789, 733, 803, 824, 770, 802, 750, 770, 760, 715, 766, 769, 773, 757, 789, 816, 766, 748, 733, 752, 751, 779, 756, 730, 790, 737, 727, 793] payload_rows_per_dst=[9372, 9428, 9276, 9258, 9318, 9316, 9298, 9194] dup_rows_per_dst=[3022, 3036, 3014, 2868, 3114, 2888, 2970, 2932] combine_rows_per_src=[9372, 9428, 9276, 9258, 9318, 9316, 9298, 9194] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266510336 dispatch_meta_bytes=344552 combine_payload_bytes=266510336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266854888 ingress_bytes=301113344 link_bytes=301113344 token_bytes=533365224 weight_movement_bytes=34603008 total_nvlink_bytes=567968232 weight_to_token_ratio=0.064877 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133065272e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=267534336 dispatch_meta_bytes=344080 combine_payload_bytes=267534336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267878416 ingress_bytes=302137344 link_bytes=302137344 token_bytes=535412752 weight_movement_bytes=34603008 total_nvlink_bytes=570015760 weight_to_token_ratio=0.064629 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136918501e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=343312 combine_payload_bytes=266313728 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266657040 ingress_bytes=335519744 link_bytes=335519744 token_bytes=532970768 weight_movement_bytes=69206016 total_nvlink_bytes=602176784 weight_to_token_ratio=0.129850 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262533785e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74320 payload_rows_remote=64742 dup_rows=23984 dup_rows_remote=20656 dedup_ratio=0.243978 router_tokens_per_expert=[780, 800, 796, 756, 779, 704, 729, 758, 770, 733, 787, 729, 738, 752, 817, 765, 773, 750, 741, 792, 778, 781, 773, 786, 763, 758, 781, 782, 720, 740, 801, 745, 768, 768, 793, 780, 745, 733, 770, 803, 762, 771, 728, 819, 766, 794, 811, 721, 810, 790, 746, 814, 794, 781, 758, 767, 816, 767, 710, 769, 756, 797, 748, 740] payload_rows_per_dst=[9186, 9282, 9336, 9208, 9222, 9404, 9104, 9578] dup_rows_per_dst=[2868, 2900, 3012, 3002, 3004, 3034, 2836, 3328] combine_rows_per_src=[9186, 9282, 9336, 9208, 9222, 9404, 9104, 9578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266616832 dispatch_meta_bytes=344664 combine_payload_bytes=266616832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266961496 ingress_bytes=301219840 link_bytes=301219840 token_bytes=533578328 weight_movement_bytes=34603008 total_nvlink_bytes=568181336 weight_to_token_ratio=0.064851 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133466008e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65078 dup_rows=23908 dup_rows_remote=20896 dedup_ratio=0.243205 router_tokens_per_expert=[732, 784, 723, 715, 768, 724, 738, 736, 766, 773, 744, 772, 789, 768, 761, 735, 772, 783, 779, 764, 748, 760, 772, 787, 743, 758, 841, 796, 757, 747, 818, 797, 738, 778, 757, 733, 788, 803, 743, 808, 791, 746, 798, 771, 809, 755, 809, 787, 790, 749, 776, 744, 713, 747, 796, 803, 777, 787, 775, 758, 788, 759, 788, 738] payload_rows_per_dst=[9102, 9228, 9318, 9434, 9330, 9426, 9200, 9358] dup_rows_per_dst=[2738, 2988, 3012, 3080, 2966, 3106, 3036, 2982] combine_rows_per_src=[9102, 9228, 9318, 9434, 9330, 9426, 9200, 9358] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=264970240 dispatch_meta_bytes=341888 combine_payload_bytes=264970240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265312128 ingress_bytes=299573248 link_bytes=299573248 token_bytes=530282368 weight_movement_bytes=34603008 total_nvlink_bytes=564885376 weight_to_token_ratio=0.065254 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127270015e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74438 payload_rows_remote=65146 dup_rows=23866 dup_rows_remote=20900 dedup_ratio=0.242778 router_tokens_per_expert=[729, 791, 774, 765, 768, 748, 782, 704, 741, 709, 826, 757, 783, 768, 781, 764, 785, 793, 777, 734, 704, 812, 734, 758, 809, 772, 776, 752, 789, 776, 808, 739, 743, 760, 769, 776, 759, 748, 780, 807, 805, 790, 782, 776, 818, 734, 730, 766, 769, 745, 745, 744, 731, 786, 819, 798, 761, 764, 757, 766, 757, 801, 801, 757] payload_rows_per_dst=[9204, 9292, 9234, 9406, 9328, 9382, 9246, 9346] dup_rows_per_dst=[2918, 2966, 2960, 3036, 2956, 3020, 3028, 2982] combine_rows_per_src=[9204, 9292, 9234, 9406, 9328, 9382, 9246, 9346] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266510336 dispatch_meta_bytes=344552 combine_payload_bytes=266510336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266854888 ingress_bytes=301113344 link_bytes=301113344 token_bytes=533365224 weight_movement_bytes=34603008 total_nvlink_bytes=567968232 weight_to_token_ratio=0.064877 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133065272e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74214 payload_rows_remote=64868 dup_rows=24090 dup_rows_remote=20986 dedup_ratio=0.245056 router_tokens_per_expert=[754, 788, 779, 755, 808, 786, 760, 794, 785, 752, 794, 764, 743, 786, 732, 784, 765, 704, 752, 753, 795, 760, 750, 759, 732, 761, 755, 756, 778, 736, 783, 769, 775, 766, 783, 786, 808, 804, 769, 770, 783, 763, 732, 780, 790, 782, 806, 806, 724, 759, 752, 745, 778, 783, 741, 781, 760, 787, 748, 765, 784, 736, 765, 769] payload_rows_per_dst=[9346, 9380, 9264, 9232, 9382, 9184, 9212, 9214] dup_rows_per_dst=[3104, 3170, 2884, 2836, 3138, 3030, 2914, 3014] combine_rows_per_src=[9346, 9380, 9264, 9232, 9382, 9184, 9212, 9214] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265519104 dispatch_meta_bytes=343544 combine_payload_bytes=265519104 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265862648 ingress_bytes=369328128 link_bytes=369328128 token_bytes=531381752 weight_movement_bytes=103809024 total_nvlink_bytes=635190776 weight_to_token_ratio=0.195357 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389752012e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265224192 dispatch_meta_bytes=344216 combine_payload_bytes=265224192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265568408 ingress_bytes=299827200 link_bytes=299827200 token_bytes=530792600 weight_movement_bytes=34603008 total_nvlink_bytes=565395608 weight_to_token_ratio=0.065191 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128225616e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74320 payload_rows_remote=64742 dup_rows=23984 dup_rows_remote=20656 dedup_ratio=0.243978 router_tokens_per_expert=[780, 800, 796, 756, 779, 704, 729, 758, 770, 733, 787, 729, 738, 752, 817, 765, 773, 750, 741, 792, 778, 781, 773, 786, 763, 758, 781, 782, 720, 740, 801, 745, 768, 768, 793, 780, 745, 733, 770, 803, 762, 771, 728, 819, 766, 794, 811, 721, 810, 790, 746, 814, 794, 781, 758, 767, 816, 767, 710, 769, 756, 797, 748, 740] payload_rows_per_dst=[9186, 9282, 9336, 9208, 9222, 9404, 9104, 9578] dup_rows_per_dst=[2868, 2900, 3012, 3002, 3004, 3034, 2836, 3328] combine_rows_per_src=[9186, 9282, 9336, 9208, 9222, 9404, 9104, 9578] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=267067392 dispatch_meta_bytes=344712 combine_payload_bytes=267067392 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267412104 ingress_bytes=370876416 link_bytes=370876416 token_bytes=534479496 weight_movement_bytes=103809024 total_nvlink_bytes=638288520 weight_to_token_ratio=0.194225 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395578095e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266559488 dispatch_meta_bytes=343896 combine_payload_bytes=266559488 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266903384 ingress_bytes=370368512 link_bytes=370368512 token_bytes=533462872 weight_movement_bytes=103809024 total_nvlink_bytes=637271896 weight_to_token_ratio=0.194595 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393666894e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265699328 dispatch_meta_bytes=343416 combine_payload_bytes=265699328 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266042744 ingress_bytes=369508352 link_bytes=369508352 token_bytes=531742072 weight_movement_bytes=103809024 total_nvlink_bytes=635551096 weight_to_token_ratio=0.195224 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390430181e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74394 payload_rows_remote=65092 dup_rows=23910 dup_rows_remote=20870 dedup_ratio=0.243225 router_tokens_per_expert=[749, 759, 807, 781, 760, 746, 766, 736, 748, 800, 779, 826, 803, 724, 725, 796, 749, 782, 819, 780, 748, 764, 794, 717, 749, 758, 777, 778, 816, 754, 747, 802, 720, 767, 750, 739, 734, 735, 771, 811, 833, 788, 790, 789, 773, 777, 712, 746, 759, 766, 810, 744, 791, 794, 771, 736, 789, 783, 770, 725, 793, 743, 764, 740] payload_rows_per_dst=[9278, 9368, 9338, 9306, 9178, 9390, 9302, 9234] dup_rows_per_dst=[2930, 3034, 2968, 3056, 2876, 3026, 3040, 2980] combine_rows_per_src=[9278, 9368, 9338, 9306, 9178, 9390, 9302, 9234] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265183232 dispatch_meta_bytes=341592 combine_payload_bytes=265183232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265524824 ingress_bytes=299786240 link_bytes=299786240 token_bytes=530708056 weight_movement_bytes=34603008 total_nvlink_bytes=565311064 weight_to_token_ratio=0.065202 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128071486e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266838016 dispatch_meta_bytes=344184 combine_payload_bytes=266838016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267182200 ingress_bytes=301441024 link_bytes=301441024 token_bytes=534020216 weight_movement_bytes=34603008 total_nvlink_bytes=568623224 weight_to_token_ratio=0.064797 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134298305e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74438 payload_rows_remote=65146 dup_rows=23866 dup_rows_remote=20900 dedup_ratio=0.242778 router_tokens_per_expert=[729, 791, 774, 765, 768, 748, 782, 704, 741, 709, 826, 757, 783, 768, 781, 764, 785, 793, 777, 734, 704, 812, 734, 758, 809, 772, 776, 752, 789, 776, 808, 739, 743, 760, 769, 776, 759, 748, 780, 807, 805, 790, 782, 776, 818, 734, 730, 766, 769, 745, 745, 744, 731, 786, 819, 798, 761, 764, 757, 766, 757, 801, 801, 757] payload_rows_per_dst=[9204, 9292, 9234, 9406, 9328, 9382, 9246, 9346] dup_rows_per_dst=[2918, 2966, 2960, 3036, 2956, 3020, 3028, 2982] combine_rows_per_src=[9204, 9292, 9234, 9406, 9328, 9382, 9246, 9346] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=65046 dup_rows=23926 dup_rows_remote=20890 dedup_ratio=0.243388 router_tokens_per_expert=[844, 770, 753, 757, 814, 715, 754, 782, 795, 776, 755, 721, 774, 749, 783, 813, 799, 767, 791, 753, 769, 729, 768, 779, 767, 774, 726, 747, 757, 782, 771, 761, 773, 756, 754, 767, 801, 756, 773, 804, 812, 744, 768, 774, 788, 796, 759, 745, 825, 741, 750, 764, 762, 768, 789, 782, 763, 755, 764, 752, 756, 750, 755, 711] payload_rows_per_dst=[9368, 9300, 9300, 9264, 9332, 9378, 9262, 9174] dup_rows_per_dst=[3010, 3032, 3010, 2906, 3036, 2994, 3100, 2838] combine_rows_per_src=[9368, 9300, 9300, 9264, 9332, 9378, 9262, 9174] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64978 dup_rows=24114 dup_rows_remote=21156 dedup_ratio=0.245300 router_tokens_per_expert=[807, 752, 806, 803, 737, 771, 763, 803, 751, 783, 731, 743, 782, 769, 779, 755, 800, 782, 751, 742, 772, 763, 730, 745, 786, 766, 834, 787, 744, 752, 761, 734, 761, 735, 777, 717, 797, 744, 767, 794, 739, 774, 807, 742, 754, 783, 720, 767, 793, 783, 743, 795, 787, 761, 800, 762, 760, 783, 795, 740, 785, 738, 799, 766] payload_rows_per_dst=[9362, 9104, 9212, 9342, 9248, 9188, 9336, 9398] dup_rows_per_dst=[3122, 3082, 2958, 2986, 2936, 2984, 3112, 2934] combine_rows_per_src=[9362, 9104, 9212, 9342, 9248, 9188, 9336, 9398] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=65098 dup_rows=23948 dup_rows_remote=21144 dedup_ratio=0.243612 router_tokens_per_expert=[762, 805, 779, 767, 748, 748, 774, 781, 754, 822, 770, 802, 741, 795, 736, 755, 777, 790, 745, 777, 766, 781, 781, 788, 776, 756, 771, 769, 765, 819, 761, 755, 782, 830, 811, 777, 744, 746, 799, 763, 767, 741, 790, 750, 755, 753, 743, 732, 804, 734, 740, 797, 761, 725, 791, 776, 747, 761, 733, 782, 745, 771, 747, 739] payload_rows_per_dst=[9272, 9320, 9400, 9228, 9442, 9258, 9306, 9130] dup_rows_per_dst=[3056, 3030, 3010, 3116, 3062, 2804, 2950, 2920] combine_rows_per_src=[9272, 9320, 9400, 9228, 9442, 9258, 9306, 9130] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74214 payload_rows_remote=64868 dup_rows=24090 dup_rows_remote=20986 dedup_ratio=0.245056 router_tokens_per_expert=[754, 788, 779, 755, 808, 786, 760, 794, 785, 752, 794, 764, 743, 786, 732, 784, 765, 704, 752, 753, 795, 760, 750, 759, 732, 761, 755, 756, 778, 736, 783, 769, 775, 766, 783, 786, 808, 804, 769, 770, 783, 763, 732, 780, 790, 782, 806, 806, 724, 759, 752, 745, 778, 783, 741, 781, 760, 787, 748, 765, 784, 736, 765, 769] payload_rows_per_dst=[9346, 9380, 9264, 9232, 9382, 9184, 9212, 9214] dup_rows_per_dst=[3104, 3170, 2884, 2836, 3138, 3030, 2914, 3014] combine_rows_per_src=[9346, 9380, 9264, 9232, 9382, 9184, 9212, 9214] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74462 payload_rows_remote=65018 dup_rows=23842 dup_rows_remote=20810 dedup_ratio=0.242533 router_tokens_per_expert=[718, 776, 769, 744, 760, 770, 785, 753, 759, 785, 834, 762, 734, 765, 778, 770, 726, 777, 765, 778, 791, 743, 761, 744, 784, 751, 801, 751, 777, 751, 748, 796, 753, 750, 727, 758, 753, 765, 767, 742, 824, 787, 776, 768, 763, 767, 781, 773, 765, 803, 758, 794, 779, 798, 787, 754, 737, 771, 753, 751, 822, 752, 764, 804] payload_rows_per_dst=[9256, 9256, 9192, 9348, 9162, 9422, 9444, 9382] dup_rows_per_dst=[2894, 3118, 2978, 2970, 2868, 3056, 3032, 2926] combine_rows_per_src=[9256, 9256, 9192, 9348, 9162, 9422, 9444, 9382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266616832 dispatch_meta_bytes=343848 combine_payload_bytes=266616832 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266960680 ingress_bytes=335822848 link_bytes=335822848 token_bytes=533577512 weight_movement_bytes=69206016 total_nvlink_bytes=602783528 weight_to_token_ratio=0.129702 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263674341e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=64874 dup_rows=24118 dup_rows_remote=21050 dedup_ratio=0.245341 router_tokens_per_expert=[774, 820, 747, 769, 737, 721, 852, 720, 736, 774, 729, 746, 759, 744, 741, 793, 801, 789, 785, 799, 755, 732, 770, 778, 758, 774, 770, 796, 788, 783, 769, 752, 784, 742, 756, 768, 766, 768, 787, 711, 751, 774, 827, 751, 722, 789, 780, 720, 777, 733, 816, 770, 807, 779, 780, 816, 780, 741, 772, 721, 800, 755, 782, 766] payload_rows_per_dst=[9250, 9164, 9330, 9312, 9198, 9290, 9400, 9242] dup_rows_per_dst=[3030, 2880, 3088, 3068, 2966, 2938, 3156, 2992] combine_rows_per_src=[9250, 9164, 9330, 9312, 9198, 9290, 9400, 9242] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=343744 combine_payload_bytes=266428416 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266772160 ingress_bytes=335634432 link_bytes=335634432 token_bytes=533200576 weight_movement_bytes=69206016 total_nvlink_bytes=602406592 weight_to_token_ratio=0.129794 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262965347e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=344536 combine_payload_bytes=266149888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266494424 ingress_bytes=300752896 link_bytes=300752896 token_bytes=532644312 weight_movement_bytes=34603008 total_nvlink_bytes=567247320 weight_to_token_ratio=0.064965 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131708935e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74316 payload_rows_remote=64952 dup_rows=23988 dup_rows_remote=20918 dedup_ratio=0.244019 router_tokens_per_expert=[720, 735, 786, 766, 755, 748, 781, 767, 797, 795, 747, 746, 760, 752, 806, 737, 799, 738, 778, 818, 768, 760, 761, 764, 772, 798, 748, 754, 779, 743, 806, 753, 734, 832, 761, 765, 768, 767, 749, 813, 737, 805, 792, 767, 762, 790, 749, 754, 775, 728, 813, 741, 717, 799, 789, 786, 737, 746, 750, 825, 793, 770, 735, 766] payload_rows_per_dst=[9218, 9250, 9368, 9338, 9298, 9288, 9192, 9364] dup_rows_per_dst=[2898, 3030, 2956, 2968, 3126, 3012, 2928, 3070] combine_rows_per_src=[9218, 9250, 9368, 9338, 9298, 9288, 9192, 9364] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74238 payload_rows_remote=64982 dup_rows=24066 dup_rows_remote=21096 dedup_ratio=0.244812 router_tokens_per_expert=[790, 755, 787, 749, 740, 737, 788, 765, 742, 774, 737, 773, 766, 780, 750, 791, 738, 787, 800, 741, 763, 789, 774, 791, 757, 726, 745, 802, 751, 786, 766, 772, 727, 785, 726, 759, 758, 818, 760, 734, 817, 772, 785, 761, 772, 766, 731, 777, 824, 764, 737, 762, 774, 734, 784, 784, 739, 802, 787, 769, 766, 773, 815, 778] payload_rows_per_dst=[9208, 9256, 9328, 9222, 9232, 9330, 9290, 9372] dup_rows_per_dst=[3014, 2970, 3038, 2988, 2902, 3032, 3036, 3086] combine_rows_per_src=[9208, 9256, 9328, 9222, 9232, 9330, 9290, 9372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266838016 dispatch_meta_bytes=344184 combine_payload_bytes=266838016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267182200 ingress_bytes=301441024 link_bytes=301441024 token_bytes=534020216 weight_movement_bytes=34603008 total_nvlink_bytes=568623224 weight_to_token_ratio=0.064797 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134298305e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=343744 combine_payload_bytes=266428416 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266772160 ingress_bytes=335634432 link_bytes=335634432 token_bytes=533200576 weight_movement_bytes=69206016 total_nvlink_bytes=602406592 weight_to_token_ratio=0.129794 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262965347e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74522 payload_rows_remote=65316 dup_rows=23782 dup_rows_remote=20704 dedup_ratio=0.241923 router_tokens_per_expert=[733, 731, 782, 807, 756, 752, 752, 768, 748, 764, 772, 782, 762, 782, 779, 737, 762, 764, 779, 757, 776, 783, 793, 728, 794, 788, 728, 734, 748, 792, 756, 769, 774, 790, 807, 747, 805, 749, 764, 739, 782, 741, 786, 758, 796, 791, 755, 771, 804, 744, 741, 774, 763, 723, 788, 777, 781, 785, 770, 749, 760, 803, 764, 813] payload_rows_per_dst=[9226, 9302, 9206, 9370, 9418, 9308, 9242, 9450] dup_rows_per_dst=[2936, 2950, 3078, 2848, 2932, 3052, 2986, 3000] combine_rows_per_src=[9226, 9302, 9206, 9370, 9418, 9308, 9242, 9450] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=344968 combine_payload_bytes=266641408 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266986376 ingress_bytes=335847424 link_bytes=335847424 token_bytes=533627784 weight_movement_bytes=69206016 total_nvlink_bytes=602833800 weight_to_token_ratio=0.129690 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263766819e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265699328 dispatch_meta_bytes=343416 combine_payload_bytes=265699328 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266042744 ingress_bytes=369508352 link_bytes=369508352 token_bytes=531742072 weight_movement_bytes=103809024 total_nvlink_bytes=635551096 weight_to_token_ratio=0.195224 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390430181e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=343312 combine_payload_bytes=266313728 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266657040 ingress_bytes=335519744 link_bytes=335519744 token_bytes=532970768 weight_movement_bytes=69206016 total_nvlink_bytes=602176784 weight_to_token_ratio=0.129850 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262533785e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74008 payload_rows_remote=64496 dup_rows=24296 dup_rows_remote=21130 dedup_ratio=0.247152 router_tokens_per_expert=[771, 736, 729, 820, 752, 734, 740, 813, 767, 754, 771, 764, 754, 800, 739, 771, 777, 741, 779, 735, 804, 783, 773, 782, 748, 777, 802, 749, 714, 782, 747, 751, 768, 779, 788, 803, 753, 806, 788, 738, 774, 783, 780, 764, 781, 762, 801, 772, 797, 766, 783, 838, 758, 749, 736, 793, 745, 730, 747, 770, 741, 744, 765, 791] payload_rows_per_dst=[9512, 9240, 9346, 9330, 9084, 9298, 9326, 8872] dup_rows_per_dst=[3166, 3034, 3002, 3152, 3020, 3136, 3080, 2706] combine_rows_per_src=[9512, 9240, 9346, 9330, 9084, 9298, 9326, 8872] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74316 payload_rows_remote=64952 dup_rows=23988 dup_rows_remote=20918 dedup_ratio=0.244019 router_tokens_per_expert=[720, 735, 786, 766, 755, 748, 781, 767, 797, 795, 747, 746, 760, 752, 806, 737, 799, 738, 778, 818, 768, 760, 761, 764, 772, 798, 748, 754, 779, 743, 806, 753, 734, 832, 761, 765, 768, 767, 749, 813, 737, 805, 792, 767, 762, 790, 749, 754, 775, 728, 813, 741, 717, 799, 789, 786, 737, 746, 750, 825, 793, 770, 735, 766] payload_rows_per_dst=[9218, 9250, 9368, 9338, 9298, 9288, 9192, 9364] dup_rows_per_dst=[2898, 3030, 2956, 2968, 3126, 3012, 2928, 3070] combine_rows_per_src=[9218, 9250, 9368, 9338, 9298, 9288, 9192, 9364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265723904 dispatch_meta_bytes=343696 combine_payload_bytes=265723904 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266067600 ingress_bytes=334929920 link_bytes=334929920 token_bytes=531791504 weight_movement_bytes=69206016 total_nvlink_bytes=600997520 weight_to_token_ratio=0.130137 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260314325e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=267534336 dispatch_meta_bytes=344080 combine_payload_bytes=267534336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267878416 ingress_bytes=302137344 link_bytes=302137344 token_bytes=535412752 weight_movement_bytes=34603008 total_nvlink_bytes=570015760 weight_to_token_ratio=0.064629 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136918501e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=64894 dup_rows=24056 dup_rows_remote=21188 dedup_ratio=0.244710 router_tokens_per_expert=[777, 777, 750, 792, 760, 757, 778, 732, 750, 723, 754, 772, 766, 806, 747, 778, 800, 746, 756, 722, 775, 756, 816, 794, 805, 776, 781, 746, 727, 775, 767, 745, 754, 787, 765, 776, 764, 765, 771, 770, 771, 766, 797, 824, 785, 742, 786, 743, 740, 758, 753, 760, 739, 813, 779, 769, 792, 785, 790, 716, 752, 765, 804, 765] payload_rows_per_dst=[9248, 9220, 9248, 9244, 9248, 9406, 9354, 9280] dup_rows_per_dst=[2998, 2972, 3082, 3000, 3056, 3022, 2868, 3058] combine_rows_per_src=[9248, 9220, 9248, 9244, 9248, 9406, 9354, 9280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=264175616 dispatch_meta_bytes=342504 combine_payload_bytes=264175616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264518120 ingress_bytes=298778624 link_bytes=298778624 token_bytes=528693736 weight_movement_bytes=34603008 total_nvlink_bytes=563296744 weight_to_token_ratio=0.065450 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.124279908e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266043392 dispatch_meta_bytes=343480 combine_payload_bytes=266043392 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266386872 ingress_bytes=335249408 link_bytes=335249408 token_bytes=532430264 weight_movement_bytes=69206016 total_nvlink_bytes=601636280 weight_to_token_ratio=0.129981 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261516533e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266166272 dispatch_meta_bytes=344312 combine_payload_bytes=266166272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266510584 ingress_bytes=300769280 link_bytes=300769280 token_bytes=532676856 weight_movement_bytes=34603008 total_nvlink_bytes=567279864 weight_to_token_ratio=0.064961 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131770587e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74238 payload_rows_remote=64982 dup_rows=24066 dup_rows_remote=21096 dedup_ratio=0.244812 router_tokens_per_expert=[790, 755, 787, 749, 740, 737, 788, 765, 742, 774, 737, 773, 766, 780, 750, 791, 738, 787, 800, 741, 763, 789, 774, 791, 757, 726, 745, 802, 751, 786, 766, 772, 727, 785, 726, 759, 758, 818, 760, 734, 817, 772, 785, 761, 772, 766, 731, 777, 824, 764, 737, 762, 774, 734, 784, 784, 739, 802, 787, 769, 766, 773, 815, 778] payload_rows_per_dst=[9208, 9256, 9328, 9222, 9232, 9330, 9290, 9372] dup_rows_per_dst=[3014, 2970, 3038, 2988, 2902, 3032, 3036, 3086] combine_rows_per_src=[9208, 9256, 9328, 9222, 9232, 9330, 9290, 9372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74226 payload_rows_remote=64882 dup_rows=24078 dup_rows_remote=20878 dedup_ratio=0.244934 router_tokens_per_expert=[795, 731, 752, 782, 783, 768, 786, 802, 800, 769, 772, 756, 779, 773, 775, 770, 787, 756, 724, 720, 766, 790, 777, 766, 741, 740, 802, 743, 761, 801, 731, 697, 835, 813, 747, 792, 768, 761, 734, 822, 772, 773, 759, 774, 760, 747, 808, 764, 750, 724, 777, 760, 789, 726, 755, 733, 787, 765, 797, 789, 734, 811, 774, 757] payload_rows_per_dst=[9382, 9340, 9250, 9170, 9344, 9200, 9100, 9440] dup_rows_per_dst=[3016, 3048, 2922, 2862, 3200, 3114, 2928, 2988] combine_rows_per_src=[9382, 9340, 9250, 9170, 9344, 9200, 9100, 9440] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74240 payload_rows_remote=64806 dup_rows=24064 dup_rows_remote=21086 dedup_ratio=0.244792 router_tokens_per_expert=[721, 769, 789, 753, 734, 773, 763, 781, 816, 740, 802, 818, 776, 782, 721, 724, 803, 763, 761, 810, 715, 797, 780, 777, 739, 795, 815, 789, 741, 745, 820, 742, 738, 774, 746, 739, 775, 803, 769, 785, 779, 744, 779, 726, 733, 773, 774, 763, 740, 800, 751, 764, 783, 762, 715, 755, 772, 766, 767, 798, 769, 772, 806, 778] payload_rows_per_dst=[9240, 9402, 9434, 9300, 9274, 9136, 9144, 9310] dup_rows_per_dst=[2926, 2956, 2978, 3072, 2984, 3006, 2996, 3146] combine_rows_per_src=[9240, 9402, 9434, 9300, 9274, 9136, 9144, 9310] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74154 payload_rows_remote=64820 dup_rows=24150 dup_rows_remote=21104 dedup_ratio=0.245667 router_tokens_per_expert=[782, 770, 778, 803, 773, 799, 740, 727, 691, 804, 812, 763, 806, 778, 770, 707, 762, 740, 777, 747, 722, 749, 760, 738, 745, 785, 783, 817, 803, 720, 740, 786, 746, 767, 749, 794, 734, 761, 778, 808, 818, 760, 757, 821, 756, 742, 753, 783, 776, 768, 747, 783, 811, 756, 834, 783, 726, 770, 792, 739, 734, 745, 798, 786] payload_rows_per_dst=[9250, 9204, 9064, 9348, 9270, 9334, 9470, 9214] dup_rows_per_dst=[3094, 3058, 2926, 3010, 3004, 3046, 3046, 2966] combine_rows_per_src=[9250, 9204, 9064, 9348, 9270, 9334, 9470, 9214] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74008 payload_rows_remote=64496 dup_rows=24296 dup_rows_remote=21130 dedup_ratio=0.247152 router_tokens_per_expert=[771, 736, 729, 820, 752, 734, 740, 813, 767, 754, 771, 764, 754, 800, 739, 771, 777, 741, 779, 735, 804, 783, 773, 782, 748, 777, 802, 749, 714, 782, 747, 751, 768, 779, 788, 803, 753, 806, 788, 738, 774, 783, 780, 764, 781, 762, 801, 772, 797, 766, 783, 838, 758, 749, 736, 793, 745, 730, 747, 770, 741, 744, 765, 791] payload_rows_per_dst=[9512, 9240, 9346, 9330, 9084, 9298, 9326, 8872] dup_rows_per_dst=[3166, 3034, 3002, 3152, 3020, 3136, 3080, 2706] combine_rows_per_src=[9512, 9240, 9346, 9330, 9084, 9298, 9326, 8872] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=343040 combine_payload_bytes=265756672 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266099712 ingress_bytes=334962688 link_bytes=334962688 token_bytes=531856384 weight_movement_bytes=69206016 total_nvlink_bytes=601062400 weight_to_token_ratio=0.130122 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260437628e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74394 payload_rows_remote=65092 dup_rows=23910 dup_rows_remote=20870 dedup_ratio=0.243225 router_tokens_per_expert=[749, 759, 807, 781, 760, 746, 766, 736, 748, 800, 779, 826, 803, 724, 725, 796, 749, 782, 819, 780, 748, 764, 794, 717, 749, 758, 777, 778, 816, 754, 747, 802, 720, 767, 750, 739, 734, 735, 771, 811, 833, 788, 790, 789, 773, 777, 712, 746, 759, 766, 810, 744, 791, 794, 771, 736, 789, 783, 770, 725, 793, 743, 764, 740] payload_rows_per_dst=[9278, 9368, 9338, 9306, 9178, 9390, 9302, 9234] dup_rows_per_dst=[2930, 3034, 2968, 3056, 2876, 3026, 3040, 2980] combine_rows_per_src=[9278, 9368, 9338, 9306, 9178, 9390, 9302, 9234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265805824 dispatch_meta_bytes=344328 combine_payload_bytes=265805824 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266150152 ingress_bytes=369614848 link_bytes=369614848 token_bytes=531955976 weight_movement_bytes=103809024 total_nvlink_bytes=635765000 weight_to_token_ratio=0.195146 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390830917e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74170 payload_rows_remote=64492 dup_rows=24134 dup_rows_remote=20808 dedup_ratio=0.245504 router_tokens_per_expert=[765, 720, 761, 801, 764, 769, 751, 794, 773, 784, 717, 729, 760, 774, 771, 726, 766, 766, 786, 776, 763, 742, 773, 832, 777, 761, 781, 750, 771, 747, 797, 737, 743, 803, 756, 749, 811, 766, 780, 778, 776, 789, 732, 735, 757, 750, 804, 772, 743, 767, 759, 835, 764, 769, 782, 771, 770, 749, 779, 801, 794, 718, 798, 768] payload_rows_per_dst=[9212, 9146, 9152, 8914, 9372, 9316, 9380, 9678] dup_rows_per_dst=[2994, 2922, 2928, 2820, 2942, 3122, 3080, 3326] combine_rows_per_src=[9212, 9146, 9152, 8914, 9372, 9316, 9380, 9678] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74250 payload_rows_remote=64854 dup_rows=24054 dup_rows_remote=21150 dedup_ratio=0.244690 router_tokens_per_expert=[773, 756, 772, 745, 715, 760, 761, 785, 741, 755, 768, 796, 762, 786, 816, 738, 798, 750, 743, 775, 821, 724, 800, 770, 781, 798, 795, 772, 728, 764, 776, 736, 729, 818, 761, 826, 772, 703, 768, 754, 753, 724, 776, 746, 745, 783, 737, 708, 788, 773, 819, 767, 781, 733, 843, 769, 743, 785, 802, 812, 729, 783, 819, 743] payload_rows_per_dst=[9134, 9322, 9272, 9396, 9298, 9050, 9432, 9346] dup_rows_per_dst=[3000, 3002, 3090, 2904, 2964, 2894, 3114, 3086] combine_rows_per_src=[9134, 9322, 9272, 9396, 9298, 9050, 9432, 9346] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265445376 dispatch_meta_bytes=343568 combine_payload_bytes=265445376 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265788944 ingress_bytes=334651392 link_bytes=334651392 token_bytes=531234320 weight_movement_bytes=69206016 total_nvlink_bytes=600440336 weight_to_token_ratio=0.130274 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259266246e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74442 payload_rows_remote=65090 dup_rows=23862 dup_rows_remote=20876 dedup_ratio=0.242737 router_tokens_per_expert=[778, 799, 793, 771, 742, 746, 771, 744, 764, 790, 752, 760, 789, 781, 766, 767, 797, 769, 771, 759, 774, 762, 778, 735, 754, 724, 768, 761, 801, 792, 764, 764, 730, 793, 780, 765, 823, 775, 772, 762, 741, 753, 790, 763, 779, 729, 767, 744, 808, 762, 760, 798, 767, 736, 798, 793, 804, 742, 751, 736, 750, 776, 774, 745] payload_rows_per_dst=[9320, 9352, 9312, 9320, 9428, 9216, 9308, 9186] dup_rows_per_dst=[2968, 2986, 2978, 2936, 2972, 2916, 3136, 2970] combine_rows_per_src=[9320, 9352, 9312, 9320, 9428, 9216, 9308, 9186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266166272 dispatch_meta_bytes=344312 combine_payload_bytes=266166272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266510584 ingress_bytes=300769280 link_bytes=300769280 token_bytes=532676856 weight_movement_bytes=34603008 total_nvlink_bytes=567279864 weight_to_token_ratio=0.064961 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131770587e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=343040 combine_payload_bytes=265756672 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266099712 ingress_bytes=334962688 link_bytes=334962688 token_bytes=531856384 weight_movement_bytes=69206016 total_nvlink_bytes=601062400 weight_to_token_ratio=0.130122 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260437628e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64978 dup_rows=24114 dup_rows_remote=21156 dedup_ratio=0.245300 router_tokens_per_expert=[807, 752, 806, 803, 737, 771, 763, 803, 751, 783, 731, 743, 782, 769, 779, 755, 800, 782, 751, 742, 772, 763, 730, 745, 786, 766, 834, 787, 744, 752, 761, 734, 761, 735, 777, 717, 797, 744, 767, 794, 739, 774, 807, 742, 754, 783, 720, 767, 793, 783, 743, 795, 787, 761, 800, 762, 760, 783, 795, 740, 785, 738, 799, 766] payload_rows_per_dst=[9362, 9104, 9212, 9342, 9248, 9188, 9336, 9398] dup_rows_per_dst=[3122, 3082, 2958, 2986, 2936, 2984, 3112, 2934] combine_rows_per_src=[9362, 9104, 9212, 9342, 9248, 9188, 9336, 9398] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=343696 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265846416 ingress_bytes=334708736 link_bytes=334708736 token_bytes=531349136 weight_movement_bytes=69206016 total_nvlink_bytes=600555152 weight_to_token_ratio=0.130246 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259482027e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=264175616 dispatch_meta_bytes=342504 combine_payload_bytes=264175616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264518120 ingress_bytes=298778624 link_bytes=298778624 token_bytes=528693736 weight_movement_bytes=34603008 total_nvlink_bytes=563296744 weight_to_token_ratio=0.065450 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.124279908e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266616832 dispatch_meta_bytes=343848 combine_payload_bytes=266616832 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266960680 ingress_bytes=335822848 link_bytes=335822848 token_bytes=533577512 weight_movement_bytes=69206016 total_nvlink_bytes=602783528 weight_to_token_ratio=0.129702 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263674341e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=264159232 dispatch_meta_bytes=341200 combine_payload_bytes=264159232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264500432 ingress_bytes=298762240 link_bytes=298762240 token_bytes=528659664 weight_movement_bytes=34603008 total_nvlink_bytes=563262672 weight_to_token_ratio=0.065454 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.124218257e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74170 payload_rows_remote=64492 dup_rows=24134 dup_rows_remote=20808 dedup_ratio=0.245504 router_tokens_per_expert=[765, 720, 761, 801, 764, 769, 751, 794, 773, 784, 717, 729, 760, 774, 771, 726, 766, 766, 786, 776, 763, 742, 773, 832, 777, 761, 781, 750, 771, 747, 797, 737, 743, 803, 756, 749, 811, 766, 780, 778, 776, 789, 732, 735, 757, 750, 804, 772, 743, 767, 759, 835, 764, 769, 782, 771, 770, 749, 779, 801, 794, 718, 798, 768] payload_rows_per_dst=[9212, 9146, 9152, 8914, 9372, 9316, 9380, 9678] dup_rows_per_dst=[2994, 2922, 2928, 2820, 2942, 3122, 3080, 3326] combine_rows_per_src=[9212, 9146, 9152, 8914, 9372, 9316, 9380, 9678] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74252 payload_rows_remote=64954 dup_rows=24052 dup_rows_remote=21024 dedup_ratio=0.244670 router_tokens_per_expert=[812, 804, 752, 767, 719, 770, 739, 800, 745, 770, 717, 762, 798, 800, 758, 762, 790, 790, 788, 758, 746, 758, 752, 826, 810, 823, 804, 751, 775, 700, 789, 798, 746, 763, 798, 763, 795, 763, 727, 780, 732, 778, 732, 779, 771, 773, 756, 771, 801, 796, 766, 769, 783, 767, 780, 712, 783, 762, 715, 735, 751, 779, 734, 759] payload_rows_per_dst=[9298, 9496, 9368, 9402, 9316, 9198, 9046, 9128] dup_rows_per_dst=[3028, 3304, 3048, 3098, 2954, 2986, 2726, 2908] combine_rows_per_src=[9298, 9496, 9368, 9402, 9316, 9198, 9046, 9128] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265641984 dispatch_meta_bytes=344016 combine_payload_bytes=265641984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265986000 ingress_bytes=300244992 link_bytes=300244992 token_bytes=531627984 weight_movement_bytes=34603008 total_nvlink_bytes=566230992 weight_to_token_ratio=0.065089 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129797733e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266608640 dispatch_meta_bytes=343864 combine_payload_bytes=266608640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266952504 ingress_bytes=301211648 link_bytes=301211648 token_bytes=533561144 weight_movement_bytes=34603008 total_nvlink_bytes=568164152 weight_to_token_ratio=0.064853 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133435182e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=344536 combine_payload_bytes=266149888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266494424 ingress_bytes=300752896 link_bytes=300752896 token_bytes=532644312 weight_movement_bytes=34603008 total_nvlink_bytes=567247320 weight_to_token_ratio=0.064965 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131708935e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74406 payload_rows_remote=64956 dup_rows=23898 dup_rows_remote=20802 dedup_ratio=0.243103 router_tokens_per_expert=[766, 748, 763, 787, 733, 753, 742, 745, 718, 741, 744, 756, 789, 762, 798, 770, 758, 773, 788, 760, 784, 774, 750, 780, 749, 768, 790, 789, 742, 765, 779, 743, 758, 789, 736, 775, 779, 810, 765, 769, 791, 803, 770, 797, 755, 761, 723, 725, 735, 808, 791, 809, 790, 762, 771, 807, 731, 764, 773, 751, 792, 794, 797, 764] payload_rows_per_dst=[9308, 9218, 9334, 9244, 9332, 9216, 9450, 9304] dup_rows_per_dst=[2766, 2938, 3000, 3006, 3030, 3034, 3096, 3028] combine_rows_per_src=[9308, 9218, 9334, 9244, 9332, 9216, 9450, 9304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=264159232 dispatch_meta_bytes=341200 combine_payload_bytes=264159232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264500432 ingress_bytes=298762240 link_bytes=298762240 token_bytes=528659664 weight_movement_bytes=34603008 total_nvlink_bytes=563262672 weight_to_token_ratio=0.065454 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.124218257e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266051584 dispatch_meta_bytes=343912 combine_payload_bytes=266051584 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266395496 ingress_bytes=335257600 link_bytes=335257600 token_bytes=532447080 weight_movement_bytes=69206016 total_nvlink_bytes=601653096 weight_to_token_ratio=0.129977 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261547358e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74442 payload_rows_remote=65090 dup_rows=23862 dup_rows_remote=20876 dedup_ratio=0.242737 router_tokens_per_expert=[778, 799, 793, 771, 742, 746, 771, 744, 764, 790, 752, 760, 789, 781, 766, 767, 797, 769, 771, 759, 774, 762, 778, 735, 754, 724, 768, 761, 801, 792, 764, 764, 730, 793, 780, 765, 823, 775, 772, 762, 741, 753, 790, 763, 779, 729, 767, 744, 808, 762, 760, 798, 767, 736, 798, 793, 804, 742, 751, 736, 750, 776, 774, 745] payload_rows_per_dst=[9320, 9352, 9312, 9320, 9428, 9216, 9308, 9186] dup_rows_per_dst=[2968, 2986, 2978, 2936, 2972, 2916, 3136, 2970] combine_rows_per_src=[9320, 9352, 9312, 9320, 9428, 9216, 9308, 9186] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74506 payload_rows_remote=65228 dup_rows=23798 dup_rows_remote=20808 dedup_ratio=0.242086 router_tokens_per_expert=[776, 765, 722, 784, 768, 801, 772, 783, 736, 729, 758, 773, 712, 796, 818, 777, 749, 763, 732, 745, 740, 752, 773, 815, 789, 788, 785, 801, 767, 751, 756, 765, 770, 812, 738, 797, 755, 767, 746, 749, 764, 696, 780, 828, 789, 766, 790, 746, 785, 759, 785, 772, 793, 752, 753, 729, 773, 755, 760, 806, 809, 735, 760, 792] payload_rows_per_dst=[9346, 9194, 9216, 9422, 9278, 9402, 9292, 9356] dup_rows_per_dst=[2996, 3004, 2922, 2982, 2990, 2916, 2964, 3024] combine_rows_per_src=[9346, 9194, 9216, 9422, 9278, 9402, 9292, 9356] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64894 dup_rows=24112 dup_rows_remote=21108 dedup_ratio=0.245280 router_tokens_per_expert=[828, 730, 769, 802, 749, 752, 774, 745, 764, 730, 781, 786, 745, 785, 749, 735, 751, 760, 780, 796, 779, 790, 777, 718, 774, 788, 774, 768, 786, 781, 726, 814, 731, 722, 777, 752, 775, 778, 792, 741, 747, 802, 788, 759, 764, 780, 787, 811, 752, 776, 796, 797, 765, 754, 761, 805, 741, 731, 781, 763, 738, 815, 736, 749] payload_rows_per_dst=[9246, 9246, 9298, 9322, 9260, 9294, 9370, 9156] dup_rows_per_dst=[3052, 2904, 3004, 3100, 2876, 3182, 3042, 2952] combine_rows_per_src=[9246, 9246, 9298, 9322, 9260, 9294, 9370, 9156] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=267173888 dispatch_meta_bytes=344144 combine_payload_bytes=267173888 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267518032 ingress_bytes=336379904 link_bytes=336379904 token_bytes=534691920 weight_movement_bytes=69206016 total_nvlink_bytes=603897936 weight_to_token_ratio=0.129432 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265770498e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=343032 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266402808 ingress_bytes=335265792 link_bytes=335265792 token_bytes=532462584 weight_movement_bytes=69206016 total_nvlink_bytes=601668600 weight_to_token_ratio=0.129973 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261578184e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=65058 dup_rows=23998 dup_rows_remote=21138 dedup_ratio=0.244120 router_tokens_per_expert=[765, 778, 788, 777, 778, 752, 748, 762, 771, 778, 780, 773, 801, 766, 747, 788, 781, 803, 779, 761, 786, 735, 756, 748, 809, 811, 775, 732, 749, 757, 825, 760, 742, 783, 731, 772, 828, 748, 730, 792, 768, 767, 743, 707, 742, 794, 779, 754, 771, 795, 746, 751, 792, 771, 764, 762, 736, 769, 751, 793, 800, 735, 747, 770] payload_rows_per_dst=[9326, 9348, 9290, 9308, 9260, 9248, 9258, 9268] dup_rows_per_dst=[2970, 3060, 3008, 3128, 2992, 2860, 3046, 2934] combine_rows_per_src=[9326, 9348, 9290, 9308, 9260, 9248, 9258, 9268] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74252 payload_rows_remote=64954 dup_rows=24052 dup_rows_remote=21024 dedup_ratio=0.244670 router_tokens_per_expert=[812, 804, 752, 767, 719, 770, 739, 800, 745, 770, 717, 762, 798, 800, 758, 762, 790, 790, 788, 758, 746, 758, 752, 826, 810, 823, 804, 751, 775, 700, 789, 798, 746, 763, 798, 763, 795, 763, 727, 780, 732, 778, 732, 779, 771, 773, 756, 771, 801, 796, 766, 769, 783, 767, 780, 712, 783, 762, 715, 735, 751, 779, 734, 759] payload_rows_per_dst=[9298, 9496, 9368, 9402, 9316, 9198, 9046, 9128] dup_rows_per_dst=[3028, 3304, 3048, 3098, 2954, 2986, 2726, 2908] combine_rows_per_src=[9298, 9496, 9368, 9402, 9316, 9198, 9046, 9128] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=64894 dup_rows=24056 dup_rows_remote=21188 dedup_ratio=0.244710 router_tokens_per_expert=[777, 777, 750, 792, 760, 757, 778, 732, 750, 723, 754, 772, 766, 806, 747, 778, 800, 746, 756, 722, 775, 756, 816, 794, 805, 776, 781, 746, 727, 775, 767, 745, 754, 787, 765, 776, 764, 765, 771, 770, 771, 766, 797, 824, 785, 742, 786, 743, 740, 758, 753, 760, 739, 813, 779, 769, 792, 785, 790, 716, 752, 765, 804, 765] payload_rows_per_dst=[9248, 9220, 9248, 9244, 9248, 9406, 9354, 9280] dup_rows_per_dst=[2998, 2972, 3082, 3000, 3056, 3022, 2868, 3058] combine_rows_per_src=[9248, 9220, 9248, 9244, 9248, 9406, 9354, 9280] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74024 payload_rows_remote=64672 dup_rows=24280 dup_rows_remote=21132 dedup_ratio=0.246989 router_tokens_per_expert=[762, 781, 745, 788, 792, 765, 731, 767, 720, 751, 747, 776, 769, 745, 746, 786, 747, 856, 792, 758, 748, 798, 783, 785, 727, 805, 755, 756, 742, 771, 775, 785, 740, 815, 741, 787, 766, 682, 777, 806, 797, 755, 759, 751, 735, 767, 716, 789, 803, 769, 745, 761, 776, 803, 783, 733, 808, 825, 754, 725, 815, 819, 744, 752] payload_rows_per_dst=[8768, 9166, 9408, 9250, 9398, 9292, 9390, 9352] dup_rows_per_dst=[2880, 2914, 3126, 2966, 3076, 3214, 2956, 3148] combine_rows_per_src=[8768, 9166, 9408, 9250, 9398, 9292, 9390, 9352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265805824 dispatch_meta_bytes=344008 combine_payload_bytes=265805824 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266149832 ingress_bytes=369614848 link_bytes=369614848 token_bytes=531955656 weight_movement_bytes=103809024 total_nvlink_bytes=635764680 weight_to_token_ratio=0.195146 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390830917e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266608640 dispatch_meta_bytes=343864 combine_payload_bytes=266608640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266952504 ingress_bytes=301211648 link_bytes=301211648 token_bytes=533561144 weight_movement_bytes=34603008 total_nvlink_bytes=568164152 weight_to_token_ratio=0.064853 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133435182e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=267173888 dispatch_meta_bytes=344144 combine_payload_bytes=267173888 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267518032 ingress_bytes=336379904 link_bytes=336379904 token_bytes=534691920 weight_movement_bytes=69206016 total_nvlink_bytes=603897936 weight_to_token_ratio=0.129432 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265770498e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74252 payload_rows_remote=64876 dup_rows=24052 dup_rows_remote=21126 dedup_ratio=0.244670 router_tokens_per_expert=[745, 757, 795, 824, 766, 762, 783, 805, 750, 769, 793, 753, 693, 774, 813, 748, 783, 780, 773, 740, 811, 768, 770, 798, 739, 753, 783, 766, 730, 762, 809, 809, 754, 732, 800, 834, 753, 772, 750, 756, 795, 748, 751, 768, 779, 835, 766, 757, 759, 743, 756, 762, 742, 758, 782, 753, 771, 729, 722, 788, 795, 712, 771, 755] payload_rows_per_dst=[9466, 9166, 9326, 9376, 9252, 9332, 9190, 9144] dup_rows_per_dst=[3008, 3020, 3120, 2926, 3050, 3066, 2920, 2942] combine_rows_per_src=[9466, 9166, 9326, 9376, 9252, 9332, 9190, 9144] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74250 payload_rows_remote=65022 dup_rows=24054 dup_rows_remote=21070 dedup_ratio=0.244690 router_tokens_per_expert=[775, 745, 781, 815, 776, 805, 767, 759, 761, 820, 746, 746, 701, 765, 804, 763, 803, 780, 748, 763, 754, 742, 770, 802, 822, 765, 785, 782, 790, 753, 775, 756, 844, 777, 772, 739, 754, 808, 766, 777, 791, 727, 745, 743, 774, 792, 756, 733, 738, 778, 787, 741, 698, 804, 728, 769, 723, 756, 796, 781, 768, 759, 748, 761] payload_rows_per_dst=[9376, 9228, 9368, 9342, 9400, 9184, 9110, 9242] dup_rows_per_dst=[3070, 2984, 2956, 3114, 3074, 2938, 2976, 2942] combine_rows_per_src=[9376, 9228, 9368, 9342, 9400, 9184, 9110, 9242] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74240 payload_rows_remote=64806 dup_rows=24064 dup_rows_remote=21086 dedup_ratio=0.244792 router_tokens_per_expert=[721, 769, 789, 753, 734, 773, 763, 781, 816, 740, 802, 818, 776, 782, 721, 724, 803, 763, 761, 810, 715, 797, 780, 777, 739, 795, 815, 789, 741, 745, 820, 742, 738, 774, 746, 739, 775, 803, 769, 785, 779, 744, 779, 726, 733, 773, 774, 763, 740, 800, 751, 764, 783, 762, 715, 755, 772, 766, 767, 798, 769, 772, 806, 778] payload_rows_per_dst=[9240, 9402, 9434, 9300, 9274, 9136, 9144, 9310] dup_rows_per_dst=[2926, 2956, 2978, 3072, 2984, 3006, 2996, 3146] combine_rows_per_src=[9240, 9402, 9434, 9300, 9274, 9136, 9144, 9310] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=344784 combine_payload_bytes=266477568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266822352 ingress_bytes=301080576 link_bytes=301080576 token_bytes=533299920 weight_movement_bytes=34603008 total_nvlink_bytes=567902928 weight_to_token_ratio=0.064885 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132941969e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74024 payload_rows_remote=64672 dup_rows=24280 dup_rows_remote=21132 dedup_ratio=0.246989 router_tokens_per_expert=[762, 781, 745, 788, 792, 765, 731, 767, 720, 751, 747, 776, 769, 745, 746, 786, 747, 856, 792, 758, 748, 798, 783, 785, 727, 805, 755, 756, 742, 771, 775, 785, 740, 815, 741, 787, 766, 682, 777, 806, 797, 755, 759, 751, 735, 767, 716, 789, 803, 769, 745, 761, 776, 803, 783, 733, 808, 825, 754, 725, 815, 819, 744, 752] payload_rows_per_dst=[8768, 9166, 9408, 9250, 9398, 9292, 9390, 9352] dup_rows_per_dst=[2880, 2914, 3126, 2966, 3076, 3214, 2956, 3148] combine_rows_per_src=[8768, 9166, 9408, 9250, 9398, 9292, 9390, 9352] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266051584 dispatch_meta_bytes=343912 combine_payload_bytes=266051584 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266395496 ingress_bytes=335257600 link_bytes=335257600 token_bytes=532447080 weight_movement_bytes=69206016 total_nvlink_bytes=601653096 weight_to_token_ratio=0.129977 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261547358e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265805824 dispatch_meta_bytes=344328 combine_payload_bytes=265805824 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266150152 ingress_bytes=369614848 link_bytes=369614848 token_bytes=531955976 weight_movement_bytes=103809024 total_nvlink_bytes=635765000 weight_to_token_ratio=0.195146 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390830917e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=264896512 dispatch_meta_bytes=343216 combine_payload_bytes=264896512 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265239728 ingress_bytes=368705536 link_bytes=368705536 token_bytes=530136240 weight_movement_bytes=103809024 total_nvlink_bytes=633945264 weight_to_token_ratio=0.195816 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.387409249e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=64766 dup_rows=24118 dup_rows_remote=21080 dedup_ratio=0.245341 router_tokens_per_expert=[771, 765, 770, 779, 815, 749, 756, 772, 772, 774, 739, 720, 750, 710, 750, 704, 788, 741, 795, 774, 825, 722, 742, 791, 789, 796, 763, 776, 786, 794, 780, 784, 766, 800, 789, 793, 794, 798, 748, 739, 762, 758, 778, 745, 795, 770, 760, 765, 758, 746, 778, 776, 758, 779, 758, 773, 774, 747, 767, 750, 758, 767, 761, 800] payload_rows_per_dst=[9420, 9370, 8938, 9456, 9332, 9178, 9434, 9058] dup_rows_per_dst=[3038, 3152, 2850, 2964, 3122, 3088, 3086, 2818] combine_rows_per_src=[9420, 9370, 8938, 9456, 9332, 9178, 9434, 9058] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74178 payload_rows_remote=64974 dup_rows=24126 dup_rows_remote=21242 dedup_ratio=0.245422 router_tokens_per_expert=[785, 754, 766, 780, 729, 773, 751, 759, 776, 777, 746, 811, 737, 760, 766, 787, 733, 794, 770, 764, 755, 789, 738, 751, 762, 780, 788, 729, 752, 748, 814, 813, 746, 797, 786, 746, 730, 756, 785, 774, 773, 780, 777, 748, 805, 777, 753, 802, 778, 761, 740, 723, 760, 743, 772, 767, 800, 768, 808, 771, 790, 760, 770, 769] payload_rows_per_dst=[9216, 9256, 9224, 9382, 9252, 9258, 9204, 9386] dup_rows_per_dst=[2978, 3064, 2964, 2990, 2988, 3172, 2884, 3086] combine_rows_per_src=[9216, 9256, 9224, 9382, 9252, 9258, 9204, 9386] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265732096 dispatch_meta_bytes=344008 combine_payload_bytes=265732096 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266076104 ingress_bytes=334938112 link_bytes=334938112 token_bytes=531808200 weight_movement_bytes=69206016 total_nvlink_bytes=601014216 weight_to_token_ratio=0.130133 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260345151e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265445376 dispatch_meta_bytes=343568 combine_payload_bytes=265445376 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265788944 ingress_bytes=334651392 link_bytes=334651392 token_bytes=531234320 weight_movement_bytes=69206016 total_nvlink_bytes=600440336 weight_to_token_ratio=0.130274 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259266246e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=264896512 dispatch_meta_bytes=343216 combine_payload_bytes=264896512 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265239728 ingress_bytes=368705536 link_bytes=368705536 token_bytes=530136240 weight_movement_bytes=103809024 total_nvlink_bytes=633945264 weight_to_token_ratio=0.195816 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.387409249e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265281536 dispatch_meta_bytes=343384 combine_payload_bytes=265281536 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265624920 ingress_bytes=369090560 link_bytes=369090560 token_bytes=530906456 weight_movement_bytes=103809024 total_nvlink_bytes=634715480 weight_to_token_ratio=0.195532 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388858063e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74250 payload_rows_remote=65022 dup_rows=24054 dup_rows_remote=21070 dedup_ratio=0.244690 router_tokens_per_expert=[775, 745, 781, 815, 776, 805, 767, 759, 761, 820, 746, 746, 701, 765, 804, 763, 803, 780, 748, 763, 754, 742, 770, 802, 822, 765, 785, 782, 790, 753, 775, 756, 844, 777, 772, 739, 754, 808, 766, 777, 791, 727, 745, 743, 774, 792, 756, 733, 738, 778, 787, 741, 698, 804, 728, 769, 723, 756, 796, 781, 768, 759, 748, 761] payload_rows_per_dst=[9376, 9228, 9368, 9342, 9400, 9184, 9110, 9242] dup_rows_per_dst=[3070, 2984, 2956, 3114, 3074, 2938, 2976, 2942] combine_rows_per_src=[9376, 9228, 9368, 9342, 9400, 9184, 9110, 9242] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=64898 dup_rows=24118 dup_rows_remote=21030 dedup_ratio=0.245341 router_tokens_per_expert=[749, 762, 791, 777, 798, 784, 753, 819, 780, 783, 788, 758, 728, 822, 753, 811, 791, 795, 769, 750, 791, 710, 758, 757, 757, 780, 781, 723, 740, 772, 790, 726, 737, 818, 829, 760, 732, 758, 775, 779, 740, 761, 747, 786, 788, 737, 753, 774, 792, 785, 777, 775, 734, 749, 796, 759, 714, 761, 800, 737, 833, 769, 724, 727] payload_rows_per_dst=[9394, 9254, 9324, 9198, 9288, 9212, 9314, 9202] dup_rows_per_dst=[3072, 3192, 2918, 2940, 3088, 2960, 3020, 2928] combine_rows_per_src=[9394, 9254, 9324, 9198, 9288, 9212, 9314, 9202] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266330112 dispatch_meta_bytes=344368 combine_payload_bytes=266330112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266674480 ingress_bytes=300933120 link_bytes=300933120 token_bytes=533004592 weight_movement_bytes=34603008 total_nvlink_bytes=567607600 weight_to_token_ratio=0.064921 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132387103e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74632 payload_rows_remote=65258 dup_rows=23672 dup_rows_remote=20632 dedup_ratio=0.240804 router_tokens_per_expert=[768, 717, 740, 772, 762, 757, 780, 798, 782, 762, 788, 776, 795, 792, 776, 811, 783, 783, 776, 753, 748, 803, 782, 779, 779, 760, 795, 764, 771, 722, 741, 782, 746, 806, 739, 748, 769, 765, 769, 780, 814, 741, 792, 743, 734, 735, 776, 780, 761, 763, 763, 790, 706, 766, 725, 753, 734, 785, 779, 777, 781, 754, 810, 771] payload_rows_per_dst=[9310, 9436, 9374, 9294, 9292, 9274, 9260, 9392] dup_rows_per_dst=[2878, 3128, 3040, 2934, 2952, 2956, 2794, 2990] combine_rows_per_src=[9310, 9436, 9374, 9294, 9292, 9274, 9260, 9392] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265822208 dispatch_meta_bytes=343712 combine_payload_bytes=265822208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266165920 ingress_bytes=300425216 link_bytes=300425216 token_bytes=531988128 weight_movement_bytes=34603008 total_nvlink_bytes=566591136 weight_to_token_ratio=0.065045 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130475902e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=344864 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266478368 ingress_bytes=300736512 link_bytes=300736512 token_bytes=532611872 weight_movement_bytes=34603008 total_nvlink_bytes=567214880 weight_to_token_ratio=0.064969 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131647283e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74222 payload_rows_remote=65008 dup_rows=24082 dup_rows_remote=21184 dedup_ratio=0.244975 router_tokens_per_expert=[797, 767, 760, 788, 809, 783, 746, 763, 791, 741, 771, 742, 786, 777, 770, 719, 786, 834, 760, 795, 755, 762, 759, 771, 776, 750, 806, 752, 752, 783, 786, 729, 738, 758, 731, 734, 750, 736, 765, 729, 771, 724, 748, 778, 758, 759, 733, 785, 824, 731, 780, 741, 799, 804, 800, 782, 776, 826, 800, 748, 790, 759, 825, 704] payload_rows_per_dst=[9346, 9196, 9412, 9196, 9084, 9214, 9376, 9398] dup_rows_per_dst=[3080, 2998, 3032, 3072, 2798, 2898, 3146, 3058] combine_rows_per_src=[9346, 9196, 9412, 9196, 9084, 9214, 9376, 9398] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=64766 dup_rows=24118 dup_rows_remote=21080 dedup_ratio=0.245341 router_tokens_per_expert=[771, 765, 770, 779, 815, 749, 756, 772, 772, 774, 739, 720, 750, 710, 750, 704, 788, 741, 795, 774, 825, 722, 742, 791, 789, 796, 763, 776, 786, 794, 780, 784, 766, 800, 789, 793, 794, 798, 748, 739, 762, 758, 778, 745, 795, 770, 760, 765, 758, 746, 778, 776, 758, 779, 758, 773, 774, 747, 767, 750, 758, 767, 761, 800] payload_rows_per_dst=[9420, 9370, 8938, 9456, 9332, 9178, 9434, 9058] dup_rows_per_dst=[3038, 3152, 2850, 2964, 3122, 3088, 3086, 2818] combine_rows_per_src=[9420, 9370, 8938, 9456, 9332, 9178, 9434, 9058] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74406 payload_rows_remote=64956 dup_rows=23898 dup_rows_remote=20802 dedup_ratio=0.243103 router_tokens_per_expert=[766, 748, 763, 787, 733, 753, 742, 745, 718, 741, 744, 756, 789, 762, 798, 770, 758, 773, 788, 760, 784, 774, 750, 780, 749, 768, 790, 789, 742, 765, 779, 743, 758, 789, 736, 775, 779, 810, 765, 769, 791, 803, 770, 797, 755, 761, 723, 725, 735, 808, 791, 809, 790, 762, 771, 807, 731, 764, 773, 751, 792, 794, 797, 764] payload_rows_per_dst=[9308, 9218, 9334, 9244, 9332, 9216, 9450, 9304] dup_rows_per_dst=[2766, 2938, 3000, 3006, 3030, 3034, 3096, 3028] combine_rows_per_src=[9308, 9218, 9334, 9244, 9332, 9216, 9450, 9304] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=64802 dup_rows=23944 dup_rows_remote=20638 dedup_ratio=0.243571 router_tokens_per_expert=[742, 753, 758, 761, 739, 758, 784, 762, 737, 821, 800, 739, 803, 777, 747, 791, 770, 785, 753, 763, 719, 740, 771, 725, 763, 748, 732, 795, 766, 765, 720, 772, 725, 750, 792, 772, 748, 763, 775, 716, 724, 760, 771, 760, 785, 800, 770, 781, 763, 798, 785, 769, 798, 784, 799, 787, 774, 841, 801, 792, 777, 782, 795, 756] payload_rows_per_dst=[9180, 9218, 9170, 9218, 9362, 9186, 9468, 9558] dup_rows_per_dst=[2848, 2850, 2882, 2904, 3082, 2896, 3176, 3306] combine_rows_per_src=[9180, 9218, 9170, 9218, 9362, 9186, 9468, 9558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266330112 dispatch_meta_bytes=344368 combine_payload_bytes=266330112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266674480 ingress_bytes=300933120 link_bytes=300933120 token_bytes=533004592 weight_movement_bytes=34603008 total_nvlink_bytes=567607600 weight_to_token_ratio=0.064921 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132387103e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265822208 dispatch_meta_bytes=343712 combine_payload_bytes=265822208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266165920 ingress_bytes=300425216 link_bytes=300425216 token_bytes=531988128 weight_movement_bytes=34603008 total_nvlink_bytes=566591136 weight_to_token_ratio=0.065045 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130475902e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74522 payload_rows_remote=65258 dup_rows=23782 dup_rows_remote=20840 dedup_ratio=0.241923 router_tokens_per_expert=[786, 759, 796, 806, 772, 739, 795, 786, 739, 761, 810, 729, 735, 726, 733, 750, 743, 757, 790, 769, 782, 748, 796, 800, 753, 722, 765, 767, 746, 766, 795, 789, 752, 757, 790, 788, 729, 767, 745, 752, 704, 779, 820, 803, 791, 784, 755, 797, 762, 773, 744, 797, 786, 740, 779, 796, 787, 809, 754, 772, 757, 751, 799, 723] payload_rows_per_dst=[9500, 9070, 9358, 9264, 9304, 9346, 9372, 9308] dup_rows_per_dst=[2978, 2896, 3012, 2942, 2856, 3120, 2982, 2996] combine_rows_per_src=[9500, 9070, 9358, 9264, 9304, 9346, 9372, 9308] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=267296768 dispatch_meta_bytes=343560 combine_payload_bytes=267296768 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267640328 ingress_bytes=336502784 link_bytes=336502784 token_bytes=534937096 weight_movement_bytes=69206016 total_nvlink_bytes=604143112 weight_to_token_ratio=0.129372 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266232885e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266272768 dispatch_meta_bytes=344768 combine_payload_bytes=266272768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266617536 ingress_bytes=300875776 link_bytes=300875776 token_bytes=532890304 weight_movement_bytes=34603008 total_nvlink_bytes=567493312 weight_to_token_ratio=0.064935 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132171323e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=64924 dup_rows=23924 dup_rows_remote=20896 dedup_ratio=0.243368 router_tokens_per_expert=[771, 727, 780, 766, 752, 756, 762, 749, 759, 783, 739, 806, 763, 793, 828, 771, 791, 797, 754, 759, 786, 760, 751, 752, 764, 791, 757, 744, 781, 764, 804, 719, 742, 739, 767, 815, 751, 750, 775, 777, 791, 785, 752, 772, 728, 773, 767, 780, 820, 722, 756, 761, 774, 783, 816, 798, 779, 758, 774, 749, 794, 747, 757, 721] payload_rows_per_dst=[9140, 9456, 9374, 9224, 9246, 9318, 9374, 9248] dup_rows_per_dst=[2986, 3028, 2926, 3024, 2986, 2978, 3086, 2910] combine_rows_per_src=[9140, 9456, 9374, 9224, 9246, 9318, 9374, 9248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64894 dup_rows=24112 dup_rows_remote=21108 dedup_ratio=0.245280 router_tokens_per_expert=[828, 730, 769, 802, 749, 752, 774, 745, 764, 730, 781, 786, 745, 785, 749, 735, 751, 760, 780, 796, 779, 790, 777, 718, 774, 788, 774, 768, 786, 781, 726, 814, 731, 722, 777, 752, 775, 778, 792, 741, 747, 802, 788, 759, 764, 780, 787, 811, 752, 776, 796, 797, 765, 754, 761, 805, 741, 731, 781, 763, 738, 815, 736, 749] payload_rows_per_dst=[9246, 9246, 9298, 9322, 9260, 9294, 9370, 9156] dup_rows_per_dst=[3052, 2904, 3004, 3100, 2876, 3182, 3042, 2952] combine_rows_per_src=[9246, 9246, 9298, 9322, 9260, 9294, 9370, 9156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=64802 dup_rows=23944 dup_rows_remote=20638 dedup_ratio=0.243571 router_tokens_per_expert=[742, 753, 758, 761, 739, 758, 784, 762, 737, 821, 800, 739, 803, 777, 747, 791, 770, 785, 753, 763, 719, 740, 771, 725, 763, 748, 732, 795, 766, 765, 720, 772, 725, 750, 792, 772, 748, 763, 775, 716, 724, 760, 771, 760, 785, 800, 770, 781, 763, 798, 785, 769, 798, 784, 799, 787, 774, 841, 801, 792, 777, 782, 795, 756] payload_rows_per_dst=[9180, 9218, 9170, 9218, 9362, 9186, 9468, 9558] dup_rows_per_dst=[2848, 2850, 2882, 2904, 3082, 2896, 3176, 3306] combine_rows_per_src=[9180, 9218, 9170, 9218, 9362, 9186, 9468, 9558] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265281536 dispatch_meta_bytes=343384 combine_payload_bytes=265281536 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265624920 ingress_bytes=369090560 link_bytes=369090560 token_bytes=530906456 weight_movement_bytes=103809024 total_nvlink_bytes=634715480 weight_to_token_ratio=0.195532 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388858063e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=343032 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266402808 ingress_bytes=335265792 link_bytes=335265792 token_bytes=532462584 weight_movement_bytes=69206016 total_nvlink_bytes=601668600 weight_to_token_ratio=0.129973 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261578184e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74028 payload_rows_remote=64790 dup_rows=24276 dup_rows_remote=21202 dedup_ratio=0.246948 router_tokens_per_expert=[780, 753, 723, 785, 808, 771, 799, 737, 794, 739, 793, 760, 764, 769, 795, 771, 748, 762, 742, 747, 780, 749, 684, 750, 738, 764, 781, 743, 724, 747, 759, 774, 755, 762, 774, 803, 768, 784, 787, 812, 786, 714, 725, 806, 749, 723, 759, 802, 716, 779, 787, 783, 780, 767, 797, 823, 775, 812, 823, 768, 773, 801, 779, 747] payload_rows_per_dst=[9238, 9346, 9344, 9574, 9404, 9098, 9022, 9002] dup_rows_per_dst=[3074, 3090, 3130, 3244, 3086, 2964, 2892, 2796] combine_rows_per_src=[9238, 9346, 9344, 9574, 9404, 9098, 9022, 9002] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74268 payload_rows_remote=65070 dup_rows=24036 dup_rows_remote=21140 dedup_ratio=0.244507 router_tokens_per_expert=[799, 748, 777, 776, 780, 788, 747, 766, 717, 753, 788, 766, 822, 751, 745, 792, 735, 758, 756, 753, 770, 706, 799, 761, 791, 723, 787, 738, 745, 810, 793, 757, 804, 746, 711, 756, 777, 785, 785, 755, 821, 788, 751, 769, 753, 787, 808, 783, 774, 790, 757, 748, 743, 749, 765, 721, 768, 820, 770, 755, 791, 811, 792, 722] payload_rows_per_dst=[9412, 9242, 9112, 9252, 9274, 9400, 9198, 9378] dup_rows_per_dst=[2950, 3026, 2964, 3036, 2964, 3120, 2896, 3080] combine_rows_per_src=[9412, 9242, 9112, 9252, 9274, 9400, 9198, 9378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=267296768 dispatch_meta_bytes=344392 combine_payload_bytes=267296768 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267641160 ingress_bytes=336502784 link_bytes=336502784 token_bytes=534937928 weight_movement_bytes=69206016 total_nvlink_bytes=604143944 weight_to_token_ratio=0.129372 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266232885e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265805824 dispatch_meta_bytes=344008 combine_payload_bytes=265805824 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266149832 ingress_bytes=369614848 link_bytes=369614848 token_bytes=531955656 weight_movement_bytes=103809024 total_nvlink_bytes=635764680 weight_to_token_ratio=0.195146 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390830917e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265428992 dispatch_meta_bytes=341760 combine_payload_bytes=265428992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265770752 ingress_bytes=334635008 link_bytes=334635008 token_bytes=531199744 weight_movement_bytes=69206016 total_nvlink_bytes=600405760 weight_to_token_ratio=0.130282 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259204595e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=64924 dup_rows=23924 dup_rows_remote=20896 dedup_ratio=0.243368 router_tokens_per_expert=[771, 727, 780, 766, 752, 756, 762, 749, 759, 783, 739, 806, 763, 793, 828, 771, 791, 797, 754, 759, 786, 760, 751, 752, 764, 791, 757, 744, 781, 764, 804, 719, 742, 739, 767, 815, 751, 750, 775, 777, 791, 785, 752, 772, 728, 773, 767, 780, 820, 722, 756, 761, 774, 783, 816, 798, 779, 758, 774, 749, 794, 747, 757, 721] payload_rows_per_dst=[9140, 9456, 9374, 9224, 9246, 9318, 9374, 9248] dup_rows_per_dst=[2986, 3028, 2926, 3024, 2986, 2978, 3086, 2910] combine_rows_per_src=[9140, 9456, 9374, 9224, 9246, 9318, 9374, 9248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265379840 dispatch_meta_bytes=343968 combine_payload_bytes=265379840 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265723808 ingress_bytes=334585856 link_bytes=334585856 token_bytes=531103648 weight_movement_bytes=69206016 total_nvlink_bytes=600309664 weight_to_token_ratio=0.130306 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259019640e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=64908 dup_rows=24048 dup_rows_remote=21050 dedup_ratio=0.244629 router_tokens_per_expert=[751, 737, 718, 816, 789, 727, 785, 748, 750, 788, 788, 730, 778, 779, 827, 789, 776, 748, 732, 767, 790, 788, 773, 753, 738, 727, 772, 790, 770, 750, 788, 733, 741, 782, 813, 801, 785, 738, 776, 737, 798, 758, 768, 756, 791, 749, 751, 806, 778, 789, 788, 757, 762, 768, 802, 740, 768, 760, 743, 765, 792, 754, 735, 806] payload_rows_per_dst=[9202, 9382, 9214, 9186, 9348, 9304, 9398, 9222] dup_rows_per_dst=[2940, 3076, 3040, 2950, 2998, 3050, 2970, 3024] combine_rows_per_src=[9202, 9382, 9214, 9186, 9348, 9304, 9398, 9222] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=267296768 dispatch_meta_bytes=344392 combine_payload_bytes=267296768 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267641160 ingress_bytes=336502784 link_bytes=336502784 token_bytes=534937928 weight_movement_bytes=69206016 total_nvlink_bytes=604143944 weight_to_token_ratio=0.129372 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266232885e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265928704 dispatch_meta_bytes=343280 combine_payload_bytes=265928704 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266271984 ingress_bytes=335134720 link_bytes=335134720 token_bytes=532200688 weight_movement_bytes=69206016 total_nvlink_bytes=601406704 weight_to_token_ratio=0.130037 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261084971e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265863168 dispatch_meta_bytes=343832 combine_payload_bytes=265863168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266207000 ingress_bytes=300466176 link_bytes=300466176 token_bytes=532070168 weight_movement_bytes=34603008 total_nvlink_bytes=566673176 weight_to_token_ratio=0.065035 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130630031e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=65040 dup_rows=23930 dup_rows_remote=20960 dedup_ratio=0.243429 router_tokens_per_expert=[782, 827, 764, 839, 775, 765, 769, 783, 784, 727, 751, 791, 767, 770, 719, 761, 773, 711, 727, 796, 781, 765, 812, 787, 724, 750, 802, 733, 743, 797, 798, 792, 749, 723, 752, 825, 738, 732, 775, 728, 786, 763, 738, 813, 758, 752, 768, 791, 749, 754, 777, 792, 762, 742, 794, 766, 772, 783, 776, 794, 759, 761, 732, 783] payload_rows_per_dst=[9504, 9258, 9334, 9214, 9132, 9304, 9402, 9226] dup_rows_per_dst=[3104, 2882, 2970, 3064, 2912, 3034, 2870, 3094] combine_rows_per_src=[9504, 9258, 9334, 9214, 9132, 9304, 9402, 9226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266526720 dispatch_meta_bytes=344840 combine_payload_bytes=266526720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266871560 ingress_bytes=335732736 link_bytes=335732736 token_bytes=533398280 weight_movement_bytes=69206016 total_nvlink_bytes=602604296 weight_to_token_ratio=0.129745 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263335257e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=65010 dup_rows=24032 dup_rows_remote=21054 dedup_ratio=0.244466 router_tokens_per_expert=[796, 769, 792, 731, 798, 747, 780, 806, 809, 791, 755, 731, 746, 735, 798, 743, 821, 782, 718, 781, 750, 814, 740, 778, 772, 792, 740, 756, 759, 762, 766, 746, 767, 833, 784, 783, 742, 760, 748, 779, 812, 747, 789, 703, 741, 761, 835, 732, 722, 713, 756, 788, 837, 784, 746, 811, 829, 743, 742, 746, 760, 766, 748, 741] payload_rows_per_dst=[9398, 9238, 9334, 9250, 9302, 9262, 9306, 9182] dup_rows_per_dst=[3040, 2978, 3034, 2936, 3090, 2978, 3008, 2968] combine_rows_per_src=[9398, 9238, 9334, 9250, 9302, 9262, 9306, 9182] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74028 payload_rows_remote=64790 dup_rows=24276 dup_rows_remote=21202 dedup_ratio=0.246948 router_tokens_per_expert=[780, 753, 723, 785, 808, 771, 799, 737, 794, 739, 793, 760, 764, 769, 795, 771, 748, 762, 742, 747, 780, 749, 684, 750, 738, 764, 781, 743, 724, 747, 759, 774, 755, 762, 774, 803, 768, 784, 787, 812, 786, 714, 725, 806, 749, 723, 759, 802, 716, 779, 787, 783, 780, 767, 797, 823, 775, 812, 823, 768, 773, 801, 779, 747] payload_rows_per_dst=[9238, 9346, 9344, 9574, 9404, 9098, 9022, 9002] dup_rows_per_dst=[3074, 3090, 3130, 3244, 3086, 2964, 2892, 2796] combine_rows_per_src=[9238, 9346, 9344, 9574, 9404, 9098, 9022, 9002] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265928704 dispatch_meta_bytes=343280 combine_payload_bytes=265928704 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266271984 ingress_bytes=335134720 link_bytes=335134720 token_bytes=532200688 weight_movement_bytes=69206016 total_nvlink_bytes=601406704 weight_to_token_ratio=0.130037 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261084971e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74178 payload_rows_remote=64974 dup_rows=24126 dup_rows_remote=21242 dedup_ratio=0.245422 router_tokens_per_expert=[785, 754, 766, 780, 729, 773, 751, 759, 776, 777, 746, 811, 737, 760, 766, 787, 733, 794, 770, 764, 755, 789, 738, 751, 762, 780, 788, 729, 752, 748, 814, 813, 746, 797, 786, 746, 730, 756, 785, 774, 773, 780, 777, 748, 805, 777, 753, 802, 778, 761, 740, 723, 760, 743, 772, 767, 800, 768, 808, 771, 790, 760, 770, 769] payload_rows_per_dst=[9216, 9256, 9224, 9382, 9252, 9258, 9204, 9386] dup_rows_per_dst=[2978, 3064, 2964, 2990, 2988, 3172, 2884, 3086] combine_rows_per_src=[9216, 9256, 9224, 9382, 9252, 9258, 9204, 9386] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=64964 dup_rows=23928 dup_rows_remote=20820 dedup_ratio=0.243408 router_tokens_per_expert=[748, 748, 789, 750, 757, 786, 768, 752, 789, 782, 813, 784, 766, 771, 748, 792, 801, 767, 781, 717, 817, 747, 711, 752, 771, 778, 813, 774, 729, 690, 752, 751, 757, 774, 795, 737, 735, 805, 789, 751, 777, 773, 736, 764, 735, 786, 745, 757, 771, 719, 783, 808, 792, 769, 800, 773, 800, 737, 801, 745, 815, 807, 777, 745] payload_rows_per_dst=[9218, 9262, 9228, 9206, 9322, 9332, 9396, 9412] dup_rows_per_dst=[2912, 2952, 2988, 2910, 2964, 3060, 3034, 3108] combine_rows_per_src=[9218, 9262, 9228, 9206, 9322, 9332, 9396, 9412] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266403840 dispatch_meta_bytes=344000 combine_payload_bytes=266403840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266747840 ingress_bytes=301006848 link_bytes=301006848 token_bytes=533151680 weight_movement_bytes=34603008 total_nvlink_bytes=567754688 weight_to_token_ratio=0.064903 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132664536e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265863168 dispatch_meta_bytes=343832 combine_payload_bytes=265863168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266207000 ingress_bytes=300466176 link_bytes=300466176 token_bytes=532070168 weight_movement_bytes=34603008 total_nvlink_bytes=566673176 weight_to_token_ratio=0.065035 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130630031e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64910 dup_rows=24068 dup_rows_remote=21166 dedup_ratio=0.244832 router_tokens_per_expert=[767, 781, 786, 803, 772, 752, 731, 752, 777, 768, 799, 762, 734, 787, 712, 778, 808, 823, 740, 791, 781, 789, 756, 750, 747, 729, 757, 786, 779, 796, 738, 782, 803, 746, 745, 729, 734, 783, 769, 750, 729, 787, 808, 738, 770, 792, 791, 791, 738, 770, 761, 770, 743, 760, 818, 759, 768, 783, 790, 757, 792, 747, 726, 792] payload_rows_per_dst=[9384, 9298, 9282, 9326, 9100, 9356, 9244, 9246] dup_rows_per_dst=[2904, 2936, 3194, 2902, 3018, 3056, 2994, 3064] combine_rows_per_src=[9384, 9298, 9282, 9326, 9100, 9356, 9244, 9246] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74056 payload_rows_remote=64892 dup_rows=24248 dup_rows_remote=21226 dedup_ratio=0.246663 router_tokens_per_expert=[797, 757, 830, 776, 751, 738, 729, 786, 745, 765, 771, 731, 777, 790, 762, 752, 785, 726, 750, 735, 758, 783, 772, 792, 749, 743, 766, 740, 777, 758, 751, 766, 756, 765, 783, 743, 739, 775, 760, 802, 803, 778, 730, 782, 807, 782, 760, 805, 770, 756, 790, 757, 778, 757, 832, 747, 769, 773, 743, 813, 793, 772, 740, 784] payload_rows_per_dst=[9330, 9164, 9170, 9134, 9212, 9354, 9360, 9332] dup_rows_per_dst=[2998, 3022, 3032, 2966, 3034, 3140, 3014, 3042] combine_rows_per_src=[9330, 9164, 9170, 9134, 9212, 9354, 9360, 9332] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266280960 dispatch_meta_bytes=344256 combine_payload_bytes=266280960 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266625216 ingress_bytes=370089984 link_bytes=370089984 token_bytes=532906176 weight_movement_bytes=103809024 total_nvlink_bytes=636715200 weight_to_token_ratio=0.194798 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392618815e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74130 payload_rows_remote=64812 dup_rows=24174 dup_rows_remote=21112 dedup_ratio=0.245911 router_tokens_per_expert=[780, 761, 780, 761, 786, 801, 764, 789, 710, 766, 771, 774, 766, 762, 754, 824, 774, 768, 780, 703, 770, 804, 759, 797, 775, 788, 745, 756, 709, 777, 816, 676, 780, 763, 728, 776, 801, 757, 797, 788, 793, 775, 749, 771, 768, 729, 736, 805, 759, 763, 798, 751, 759, 746, 769, 779, 788, 757, 768, 765, 721, 766, 786, 815] payload_rows_per_dst=[9348, 9270, 9284, 9138, 9318, 9244, 9180, 9348] dup_rows_per_dst=[3096, 2984, 3026, 2946, 3062, 3008, 3068, 2984] combine_rows_per_src=[9348, 9270, 9284, 9138, 9318, 9244, 9180, 9348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74632 payload_rows_remote=65258 dup_rows=23672 dup_rows_remote=20632 dedup_ratio=0.240804 router_tokens_per_expert=[768, 717, 740, 772, 762, 757, 780, 798, 782, 762, 788, 776, 795, 792, 776, 811, 783, 783, 776, 753, 748, 803, 782, 779, 779, 760, 795, 764, 771, 722, 741, 782, 746, 806, 739, 748, 769, 765, 769, 780, 814, 741, 792, 743, 734, 735, 776, 780, 761, 763, 763, 790, 706, 766, 725, 753, 734, 785, 779, 777, 781, 754, 810, 771] payload_rows_per_dst=[9310, 9436, 9374, 9294, 9292, 9274, 9260, 9392] dup_rows_per_dst=[2878, 3128, 3040, 2934, 2952, 2956, 2794, 2990] combine_rows_per_src=[9310, 9436, 9374, 9294, 9292, 9274, 9260, 9392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=64964 dup_rows=23928 dup_rows_remote=20820 dedup_ratio=0.243408 router_tokens_per_expert=[748, 748, 789, 750, 757, 786, 768, 752, 789, 782, 813, 784, 766, 771, 748, 792, 801, 767, 781, 717, 817, 747, 711, 752, 771, 778, 813, 774, 729, 690, 752, 751, 757, 774, 795, 737, 735, 805, 789, 751, 777, 773, 736, 764, 735, 786, 745, 757, 771, 719, 783, 808, 792, 769, 800, 773, 800, 737, 801, 745, 815, 807, 777, 745] payload_rows_per_dst=[9218, 9262, 9228, 9206, 9322, 9332, 9396, 9412] dup_rows_per_dst=[2912, 2952, 2988, 2910, 2964, 3060, 3034, 3108] combine_rows_per_src=[9218, 9262, 9228, 9206, 9322, 9332, 9396, 9412] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265379840 dispatch_meta_bytes=343968 combine_payload_bytes=265379840 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265723808 ingress_bytes=334585856 link_bytes=334585856 token_bytes=531103648 weight_movement_bytes=69206016 total_nvlink_bytes=600309664 weight_to_token_ratio=0.130306 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259019640e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=344864 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266478368 ingress_bytes=300736512 link_bytes=300736512 token_bytes=532611872 weight_movement_bytes=34603008 total_nvlink_bytes=567214880 weight_to_token_ratio=0.064969 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131647283e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=343136 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266435680 ingress_bytes=369901568 link_bytes=369901568 token_bytes=532528224 weight_movement_bytes=103809024 total_nvlink_bytes=636337248 weight_to_token_ratio=0.194936 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391909821e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=64968 dup_rows=23940 dup_rows_remote=20964 dedup_ratio=0.243530 router_tokens_per_expert=[764, 758, 741, 810, 790, 797, 756, 768, 768, 731, 743, 826, 758, 753, 750, 763, 747, 796, 755, 742, 812, 766, 780, 756, 722, 758, 764, 754, 754, 742, 807, 803, 716, 747, 715, 755, 724, 779, 736, 736, 766, 738, 751, 806, 753, 832, 775, 739, 758, 790, 841, 807, 746, 765, 745, 764, 769, 764, 802, 776, 795, 763, 839, 826] payload_rows_per_dst=[9396, 9228, 9372, 9780, 9300, 9224, 8890, 9174] dup_rows_per_dst=[2976, 2952, 2936, 3354, 3068, 3096, 2772, 2786] combine_rows_per_src=[9396, 9228, 9372, 9780, 9300, 9224, 8890, 9174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=64968 dup_rows=24098 dup_rows_remote=21068 dedup_ratio=0.245138 router_tokens_per_expert=[761, 769, 753, 742, 806, 773, 731, 774, 774, 762, 758, 829, 744, 722, 743, 759, 775, 744, 786, 760, 782, 797, 739, 774, 785, 747, 749, 784, 761, 752, 783, 793, 753, 813, 750, 774, 735, 762, 750, 755, 744, 777, 796, 758, 799, 764, 745, 798, 767, 762, 808, 756, 768, 757, 790, 726, 786, 761, 811, 758, 779, 779, 785, 775] payload_rows_per_dst=[9300, 9068, 9314, 9364, 9212, 9282, 9238, 9428] dup_rows_per_dst=[2918, 3114, 3000, 2944, 2972, 3080, 3030, 3040] combine_rows_per_src=[9300, 9068, 9314, 9364, 9212, 9282, 9238, 9428] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265871360 dispatch_meta_bytes=344304 combine_payload_bytes=265871360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266215664 ingress_bytes=300474368 link_bytes=300474368 token_bytes=532087024 weight_movement_bytes=34603008 total_nvlink_bytes=566690032 weight_to_token_ratio=0.065033 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130660857e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265797632 dispatch_meta_bytes=344472 combine_payload_bytes=265797632 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266142104 ingress_bytes=369606656 link_bytes=369606656 token_bytes=531939736 weight_movement_bytes=103809024 total_nvlink_bytes=635748760 weight_to_token_ratio=0.195152 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390800091e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265469952 dispatch_meta_bytes=343696 combine_payload_bytes=265469952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265813648 ingress_bytes=300072960 link_bytes=300072960 token_bytes=531283600 weight_movement_bytes=34603008 total_nvlink_bytes=565886608 weight_to_token_ratio=0.065131 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129150391e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74114 payload_rows_remote=64994 dup_rows=24190 dup_rows_remote=21128 dedup_ratio=0.246073 router_tokens_per_expert=[722, 777, 745, 792, 779, 782, 792, 752, 749, 774, 769, 791, 806, 760, 745, 767, 753, 748, 719, 815, 777, 772, 742, 765, 761, 759, 807, 764, 806, 743, 724, 763, 758, 757, 757, 779, 732, 760, 735, 782, 794, 744, 734, 754, 775, 745, 767, 741, 783, 806, 778, 784, 785, 738, 794, 781, 804, 742, 789, 792, 802, 772, 797, 771] payload_rows_per_dst=[9248, 9270, 9120, 9300, 9152, 9250, 9358, 9416] dup_rows_per_dst=[3034, 3052, 3062, 2954, 2968, 2858, 3140, 3122] combine_rows_per_src=[9248, 9270, 9120, 9300, 9152, 9250, 9358, 9416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=267296768 dispatch_meta_bytes=343560 combine_payload_bytes=267296768 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267640328 ingress_bytes=336502784 link_bytes=336502784 token_bytes=534937096 weight_movement_bytes=69206016 total_nvlink_bytes=604143112 weight_to_token_ratio=0.129372 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266232885e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=343136 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266435680 ingress_bytes=369901568 link_bytes=369901568 token_bytes=532528224 weight_movement_bytes=103809024 total_nvlink_bytes=636337248 weight_to_token_ratio=0.194936 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391909821e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74056 payload_rows_remote=64892 dup_rows=24248 dup_rows_remote=21226 dedup_ratio=0.246663 router_tokens_per_expert=[797, 757, 830, 776, 751, 738, 729, 786, 745, 765, 771, 731, 777, 790, 762, 752, 785, 726, 750, 735, 758, 783, 772, 792, 749, 743, 766, 740, 777, 758, 751, 766, 756, 765, 783, 743, 739, 775, 760, 802, 803, 778, 730, 782, 807, 782, 760, 805, 770, 756, 790, 757, 778, 757, 832, 747, 769, 773, 743, 813, 793, 772, 740, 784] payload_rows_per_dst=[9330, 9164, 9170, 9134, 9212, 9354, 9360, 9332] dup_rows_per_dst=[2998, 3022, 3032, 2966, 3034, 3140, 3014, 3042] combine_rows_per_src=[9330, 9164, 9170, 9134, 9212, 9354, 9360, 9332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=343728 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266452656 ingress_bytes=335314944 link_bytes=335314944 token_bytes=532561584 weight_movement_bytes=69206016 total_nvlink_bytes=601767600 weight_to_token_ratio=0.129949 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261763139e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=344144 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266453072 ingress_bytes=335314944 link_bytes=335314944 token_bytes=532562000 weight_movement_bytes=69206016 total_nvlink_bytes=601768016 weight_to_token_ratio=0.129949 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261763139e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74278 payload_rows_remote=65012 dup_rows=24026 dup_rows_remote=21082 dedup_ratio=0.244405 router_tokens_per_expert=[749, 708, 734, 812, 796, 756, 781, 757, 841, 803, 781, 719, 792, 787, 757, 750, 772, 786, 773, 742, 763, 756, 758, 752, 786, 749, 728, 767, 769, 778, 748, 754, 747, 835, 810, 745, 785, 787, 786, 742, 809, 713, 764, 724, 770, 764, 815, 746, 759, 821, 752, 782, 793, 808, 769, 770, 741, 774, 763, 797, 758, 754, 727, 738] payload_rows_per_dst=[9242, 9396, 9220, 9162, 9368, 9266, 9418, 9206] dup_rows_per_dst=[2944, 3064, 2984, 2996, 3106, 2944, 3090, 2898] combine_rows_per_src=[9242, 9396, 9220, 9162, 9368, 9266, 9418, 9206] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=64968 dup_rows=23940 dup_rows_remote=20964 dedup_ratio=0.243530 router_tokens_per_expert=[764, 758, 741, 810, 790, 797, 756, 768, 768, 731, 743, 826, 758, 753, 750, 763, 747, 796, 755, 742, 812, 766, 780, 756, 722, 758, 764, 754, 754, 742, 807, 803, 716, 747, 715, 755, 724, 779, 736, 736, 766, 738, 751, 806, 753, 832, 775, 739, 758, 790, 841, 807, 746, 765, 745, 764, 769, 764, 802, 776, 795, 763, 839, 826] payload_rows_per_dst=[9396, 9228, 9372, 9780, 9300, 9224, 8890, 9174] dup_rows_per_dst=[2976, 2952, 2936, 3354, 3068, 3096, 2772, 2786] combine_rows_per_src=[9396, 9228, 9372, 9780, 9300, 9224, 8890, 9174] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74268 payload_rows_remote=65070 dup_rows=24036 dup_rows_remote=21140 dedup_ratio=0.244507 router_tokens_per_expert=[799, 748, 777, 776, 780, 788, 747, 766, 717, 753, 788, 766, 822, 751, 745, 792, 735, 758, 756, 753, 770, 706, 799, 761, 791, 723, 787, 738, 745, 810, 793, 757, 804, 746, 711, 756, 777, 785, 785, 755, 821, 788, 751, 769, 753, 787, 808, 783, 774, 790, 757, 748, 743, 749, 765, 721, 768, 820, 770, 755, 791, 811, 792, 722] payload_rows_per_dst=[9412, 9242, 9112, 9252, 9274, 9400, 9198, 9378] dup_rows_per_dst=[2950, 3026, 2964, 3036, 2964, 3120, 2896, 3080] combine_rows_per_src=[9412, 9242, 9112, 9252, 9274, 9400, 9198, 9378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74130 payload_rows_remote=64812 dup_rows=24174 dup_rows_remote=21112 dedup_ratio=0.245911 router_tokens_per_expert=[780, 761, 780, 761, 786, 801, 764, 789, 710, 766, 771, 774, 766, 762, 754, 824, 774, 768, 780, 703, 770, 804, 759, 797, 775, 788, 745, 756, 709, 777, 816, 676, 780, 763, 728, 776, 801, 757, 797, 788, 793, 775, 749, 771, 768, 729, 736, 805, 759, 763, 798, 751, 759, 746, 769, 779, 788, 757, 768, 765, 721, 766, 786, 815] payload_rows_per_dst=[9348, 9270, 9284, 9138, 9318, 9244, 9180, 9348] dup_rows_per_dst=[3096, 2984, 3026, 2946, 3062, 3008, 3068, 2984] combine_rows_per_src=[9348, 9270, 9284, 9138, 9318, 9244, 9180, 9348] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266215424 dispatch_meta_bytes=344488 combine_payload_bytes=266215424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266559912 ingress_bytes=300818432 link_bytes=300818432 token_bytes=532775336 weight_movement_bytes=34603008 total_nvlink_bytes=567378344 weight_to_token_ratio=0.064949 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131955542e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=65122 dup_rows=23974 dup_rows_remote=20814 dedup_ratio=0.243876 router_tokens_per_expert=[787, 778, 745, 807, 765, 756, 746, 753, 799, 721, 744, 752, 743, 709, 779, 712, 756, 800, 776, 791, 798, 851, 761, 808, 748, 781, 771, 813, 806, 721, 779, 771, 797, 803, 773, 765, 745, 785, 746, 768, 747, 769, 763, 773, 744, 755, 773, 773, 783, 790, 770, 744, 781, 728, 756, 756, 794, 749, 813, 750, 764, 758, 747, 763] payload_rows_per_dst=[9298, 9178, 9440, 9222, 9336, 9272, 9376, 9208] dup_rows_per_dst=[2976, 2740, 2970, 2980, 3044, 3008, 3096, 3160] combine_rows_per_src=[9298, 9178, 9440, 9222, 9336, 9272, 9376, 9208] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266289152 dispatch_meta_bytes=344376 combine_payload_bytes=266289152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266633528 ingress_bytes=300892160 link_bytes=300892160 token_bytes=532922680 weight_movement_bytes=34603008 total_nvlink_bytes=567525688 weight_to_token_ratio=0.064931 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132232974e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265797632 dispatch_meta_bytes=344472 combine_payload_bytes=265797632 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266142104 ingress_bytes=369606656 link_bytes=369606656 token_bytes=531939736 weight_movement_bytes=103809024 total_nvlink_bytes=635748760 weight_to_token_ratio=0.195152 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390800091e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=65002 dup_rows=24062 dup_rows_remote=21152 dedup_ratio=0.244771 router_tokens_per_expert=[752, 759, 753, 805, 785, 783, 787, 748, 769, 779, 797, 762, 808, 819, 754, 765, 778, 734, 774, 776, 807, 805, 794, 816, 746, 738, 712, 753, 782, 760, 765, 819, 752, 746, 763, 801, 753, 787, 769, 752, 809, 714, 764, 742, 780, 739, 763, 772, 791, 765, 743, 759, 787, 778, 742, 793, 777, 770, 762, 752, 767, 727, 759, 690] payload_rows_per_dst=[9256, 9418, 9446, 9240, 9228, 9264, 9248, 9142] dup_rows_per_dst=[3088, 3088, 3122, 2910, 3018, 2902, 3068, 2866] combine_rows_per_src=[9256, 9418, 9446, 9240, 9228, 9264, 9248, 9142] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=64786 dup_rows=24132 dup_rows_remote=21106 dedup_ratio=0.245483 router_tokens_per_expert=[731, 763, 787, 731, 814, 756, 758, 790, 760, 824, 751, 751, 785, 803, 721, 811, 795, 771, 749, 830, 741, 790, 743, 769, 821, 788, 779, 778, 815, 822, 722, 811, 850, 741, 769, 724, 742, 757, 787, 711, 750, 783, 727, 845, 761, 756, 757, 765, 745, 778, 723, 763, 729, 738, 718, 723, 772, 751, 763, 792, 773, 744, 730, 825] payload_rows_per_dst=[9298, 9386, 9192, 9426, 9164, 9318, 9062, 9326] dup_rows_per_dst=[2962, 3026, 3184, 3246, 2998, 2970, 2772, 2974] combine_rows_per_src=[9298, 9386, 9192, 9426, 9164, 9318, 9062, 9326] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=64996 dup_rows=24072 dup_rows_remote=21100 dedup_ratio=0.244873 router_tokens_per_expert=[751, 746, 750, 774, 806, 752, 748, 736, 738, 754, 762, 791, 731, 769, 758, 778, 761, 773, 774, 809, 782, 746, 768, 734, 756, 798, 772, 783, 755, 832, 746, 797, 757, 767, 750, 766, 776, 760, 777, 751, 778, 788, 732, 753, 752, 772, 798, 805, 771, 780, 791, 765, 744, 809, 798, 780, 792, 729, 739, 740, 784, 736, 819, 763] payload_rows_per_dst=[9200, 9138, 9278, 9380, 9236, 9322, 9434, 9244] dup_rows_per_dst=[2926, 3024, 3016, 3098, 2972, 3034, 3042, 2960] combine_rows_per_src=[9200, 9138, 9278, 9380, 9236, 9322, 9434, 9244] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=343728 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266452656 ingress_bytes=335314944 link_bytes=335314944 token_bytes=532561584 weight_movement_bytes=69206016 total_nvlink_bytes=601767600 weight_to_token_ratio=0.129949 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261763139e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266526720 dispatch_meta_bytes=344840 combine_payload_bytes=266526720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266871560 ingress_bytes=335732736 link_bytes=335732736 token_bytes=533398280 weight_movement_bytes=69206016 total_nvlink_bytes=602604296 weight_to_token_ratio=0.129745 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263335257e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266289152 dispatch_meta_bytes=344376 combine_payload_bytes=266289152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266633528 ingress_bytes=300892160 link_bytes=300892160 token_bytes=532922680 weight_movement_bytes=34603008 total_nvlink_bytes=567525688 weight_to_token_ratio=0.064931 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132232974e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265469952 dispatch_meta_bytes=343696 combine_payload_bytes=265469952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265813648 ingress_bytes=300072960 link_bytes=300072960 token_bytes=531283600 weight_movement_bytes=34603008 total_nvlink_bytes=565886608 weight_to_token_ratio=0.065131 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129150391e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=65122 dup_rows=23974 dup_rows_remote=20814 dedup_ratio=0.243876 router_tokens_per_expert=[787, 778, 745, 807, 765, 756, 746, 753, 799, 721, 744, 752, 743, 709, 779, 712, 756, 800, 776, 791, 798, 851, 761, 808, 748, 781, 771, 813, 806, 721, 779, 771, 797, 803, 773, 765, 745, 785, 746, 768, 747, 769, 763, 773, 744, 755, 773, 773, 783, 790, 770, 744, 781, 728, 756, 756, 794, 749, 813, 750, 764, 758, 747, 763] payload_rows_per_dst=[9298, 9178, 9440, 9222, 9336, 9272, 9376, 9208] dup_rows_per_dst=[2976, 2740, 2970, 2980, 3044, 3008, 3096, 3160] combine_rows_per_src=[9298, 9178, 9440, 9222, 9336, 9272, 9376, 9208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=65040 dup_rows=23930 dup_rows_remote=20960 dedup_ratio=0.243429 router_tokens_per_expert=[782, 827, 764, 839, 775, 765, 769, 783, 784, 727, 751, 791, 767, 770, 719, 761, 773, 711, 727, 796, 781, 765, 812, 787, 724, 750, 802, 733, 743, 797, 798, 792, 749, 723, 752, 825, 738, 732, 775, 728, 786, 763, 738, 813, 758, 752, 768, 791, 749, 754, 777, 792, 762, 742, 794, 766, 772, 783, 776, 794, 759, 761, 732, 783] payload_rows_per_dst=[9504, 9258, 9334, 9214, 9132, 9304, 9402, 9226] dup_rows_per_dst=[3104, 2882, 2970, 3064, 2912, 3034, 2870, 3094] combine_rows_per_src=[9504, 9258, 9334, 9214, 9132, 9304, 9402, 9226] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266739712 dispatch_meta_bytes=343744 combine_payload_bytes=266739712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267083456 ingress_bytes=301342720 link_bytes=301342720 token_bytes=533823168 weight_movement_bytes=34603008 total_nvlink_bytes=568426176 weight_to_token_ratio=0.064821 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133928395e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=64986 dup_rows=23964 dup_rows_remote=20954 dedup_ratio=0.243774 router_tokens_per_expert=[738, 745, 728, 738, 777, 730, 766, 798, 732, 749, 745, 786, 745, 792, 766, 785, 727, 796, 773, 784, 780, 715, 738, 792, 802, 776, 754, 785, 732, 805, 795, 792, 790, 719, 796, 755, 780, 730, 804, 811, 801, 756, 749, 791, 797, 751, 760, 764, 784, 829, 779, 750, 769, 742, 756, 765, 737, 805, 771, 763, 799, 747, 745, 791] payload_rows_per_dst=[9354, 9332, 9242, 9210, 9172, 9376, 9368, 9286] dup_rows_per_dst=[3010, 3088, 2968, 2998, 2928, 2962, 2980, 3030] combine_rows_per_src=[9354, 9332, 9242, 9210, 9172, 9376, 9368, 9286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74420 payload_rows_remote=65056 dup_rows=23884 dup_rows_remote=20960 dedup_ratio=0.242961 router_tokens_per_expert=[823, 778, 722, 796, 758, 767, 754, 769, 742, 778, 746, 798, 734, 782, 747, 802, 772, 764, 757, 761, 754, 763, 744, 784, 753, 730, 792, 793, 763, 714, 747, 740, 766, 768, 801, 770, 807, 804, 793, 767, 761, 766, 770, 775, 753, 766, 748, 783, 755, 777, 779, 784, 773, 769, 786, 721, 773, 786, 768, 783, 787, 739, 769, 778] payload_rows_per_dst=[9392, 9238, 9232, 9114, 9426, 9256, 9364, 9398] dup_rows_per_dst=[2942, 3020, 2966, 2950, 3126, 2988, 2924, 2968] combine_rows_per_src=[9392, 9238, 9232, 9114, 9426, 9256, 9364, 9398] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=344616 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266592808 ingress_bytes=300851200 link_bytes=300851200 token_bytes=532841000 weight_movement_bytes=34603008 total_nvlink_bytes=567444008 weight_to_token_ratio=0.064941 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132078845e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=343568 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265707024 ingress_bytes=299966464 link_bytes=299966464 token_bytes=531070480 weight_movement_bytes=34603008 total_nvlink_bytes=565673488 weight_to_token_ratio=0.065157 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128749655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=344384 combine_payload_bytes=266223616 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266568000 ingress_bytes=370032640 link_bytes=370032640 token_bytes=532791616 weight_movement_bytes=103809024 total_nvlink_bytes=636600640 weight_to_token_ratio=0.194840 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392403034e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74024 payload_rows_remote=64884 dup_rows=24280 dup_rows_remote=21304 dedup_ratio=0.246989 router_tokens_per_expert=[744, 759, 766, 804, 753, 775, 799, 709, 787, 847, 753, 739, 816, 780, 763, 769, 727, 733, 734, 773, 783, 793, 782, 733, 765, 757, 796, 809, 755, 762, 772, 798, 757, 748, 778, 760, 801, 800, 752, 752, 745, 750, 766, 772, 808, 784, 775, 808, 785, 703, 750, 755, 766, 780, 788, 727, 776, 746, 724, 789, 752, 790, 803, 727] payload_rows_per_dst=[9240, 9402, 9140, 9264, 9264, 9316, 9200, 9198] dup_rows_per_dst=[2978, 3106, 2976, 3164, 3032, 3100, 2908, 3016] combine_rows_per_src=[9240, 9402, 9140, 9264, 9264, 9316, 9200, 9198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266739712 dispatch_meta_bytes=343744 combine_payload_bytes=266739712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267083456 ingress_bytes=301342720 link_bytes=301342720 token_bytes=533823168 weight_movement_bytes=34603008 total_nvlink_bytes=568426176 weight_to_token_ratio=0.064821 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133928395e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266403840 dispatch_meta_bytes=344000 combine_payload_bytes=266403840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266747840 ingress_bytes=301006848 link_bytes=301006848 token_bytes=533151680 weight_movement_bytes=34603008 total_nvlink_bytes=567754688 weight_to_token_ratio=0.064903 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132664536e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=64786 dup_rows=24132 dup_rows_remote=21106 dedup_ratio=0.245483 router_tokens_per_expert=[731, 763, 787, 731, 814, 756, 758, 790, 760, 824, 751, 751, 785, 803, 721, 811, 795, 771, 749, 830, 741, 790, 743, 769, 821, 788, 779, 778, 815, 822, 722, 811, 850, 741, 769, 724, 742, 757, 787, 711, 750, 783, 727, 845, 761, 756, 757, 765, 745, 778, 723, 763, 729, 738, 718, 723, 772, 751, 763, 792, 773, 744, 730, 825] payload_rows_per_dst=[9298, 9386, 9192, 9426, 9164, 9318, 9062, 9326] dup_rows_per_dst=[2962, 3026, 3184, 3246, 2998, 2970, 2772, 2974] combine_rows_per_src=[9298, 9386, 9192, 9426, 9164, 9318, 9062, 9326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266182656 dispatch_meta_bytes=343760 combine_payload_bytes=266182656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266526416 ingress_bytes=300785664 link_bytes=300785664 token_bytes=532709072 weight_movement_bytes=34603008 total_nvlink_bytes=567312080 weight_to_token_ratio=0.064957 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131832238e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266469376 dispatch_meta_bytes=344064 combine_payload_bytes=266469376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266813440 ingress_bytes=301072384 link_bytes=301072384 token_bytes=533282816 weight_movement_bytes=34603008 total_nvlink_bytes=567885824 weight_to_token_ratio=0.064887 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132911143e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74104 payload_rows_remote=64794 dup_rows=24200 dup_rows_remote=21210 dedup_ratio=0.246175 router_tokens_per_expert=[739, 779, 776, 791, 781, 732, 794, 796, 774, 765, 740, 803, 772, 757, 754, 763, 724, 781, 769, 789, 763, 768, 791, 754, 757, 730, 793, 752, 747, 784, 753, 790, 791, 752, 775, 775, 750, 729, 774, 767, 759, 751, 789, 758, 738, 814, 777, 764, 778, 754, 756, 802, 772, 790, 784, 739, 712, 793, 737, 779, 794, 806, 746, 786] payload_rows_per_dst=[9316, 9262, 9268, 9258, 9250, 9310, 9294, 9146] dup_rows_per_dst=[3060, 2994, 3010, 2954, 2976, 2990, 3056, 3160] combine_rows_per_src=[9316, 9262, 9268, 9258, 9250, 9310, 9294, 9146] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=64986 dup_rows=23964 dup_rows_remote=20954 dedup_ratio=0.243774 router_tokens_per_expert=[738, 745, 728, 738, 777, 730, 766, 798, 732, 749, 745, 786, 745, 792, 766, 785, 727, 796, 773, 784, 780, 715, 738, 792, 802, 776, 754, 785, 732, 805, 795, 792, 790, 719, 796, 755, 780, 730, 804, 811, 801, 756, 749, 791, 797, 751, 760, 764, 784, 829, 779, 750, 769, 742, 756, 765, 737, 805, 771, 763, 799, 747, 745, 791] payload_rows_per_dst=[9354, 9332, 9242, 9210, 9172, 9376, 9368, 9286] dup_rows_per_dst=[3010, 3088, 2968, 2998, 2928, 2962, 2980, 3030] combine_rows_per_src=[9354, 9332, 9242, 9210, 9172, 9376, 9368, 9286] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=64968 dup_rows=24098 dup_rows_remote=21068 dedup_ratio=0.245138 router_tokens_per_expert=[761, 769, 753, 742, 806, 773, 731, 774, 774, 762, 758, 829, 744, 722, 743, 759, 775, 744, 786, 760, 782, 797, 739, 774, 785, 747, 749, 784, 761, 752, 783, 793, 753, 813, 750, 774, 735, 762, 750, 755, 744, 777, 796, 758, 799, 764, 745, 798, 767, 762, 808, 756, 768, 757, 790, 726, 786, 761, 811, 758, 779, 779, 785, 775] payload_rows_per_dst=[9300, 9068, 9314, 9364, 9212, 9282, 9238, 9428] dup_rows_per_dst=[2918, 3114, 3000, 2944, 2972, 3080, 3030, 3040] combine_rows_per_src=[9300, 9068, 9314, 9364, 9212, 9282, 9238, 9428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=64996 dup_rows=24072 dup_rows_remote=21100 dedup_ratio=0.244873 router_tokens_per_expert=[751, 746, 750, 774, 806, 752, 748, 736, 738, 754, 762, 791, 731, 769, 758, 778, 761, 773, 774, 809, 782, 746, 768, 734, 756, 798, 772, 783, 755, 832, 746, 797, 757, 767, 750, 766, 776, 760, 777, 751, 778, 788, 732, 753, 752, 772, 798, 805, 771, 780, 791, 765, 744, 809, 798, 780, 792, 729, 739, 740, 784, 736, 819, 763] payload_rows_per_dst=[9200, 9138, 9278, 9380, 9236, 9322, 9434, 9244] dup_rows_per_dst=[2926, 3024, 3016, 3098, 2972, 3034, 3042, 2960] combine_rows_per_src=[9200, 9138, 9278, 9380, 9236, 9322, 9434, 9244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=343568 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265707024 ingress_bytes=299966464 link_bytes=299966464 token_bytes=531070480 weight_movement_bytes=34603008 total_nvlink_bytes=565673488 weight_to_token_ratio=0.065157 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128749655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74158 payload_rows_remote=64934 dup_rows=24146 dup_rows_remote=21118 dedup_ratio=0.245626 router_tokens_per_expert=[763, 790, 817, 773, 757, 789, 789, 744, 739, 737, 772, 748, 804, 756, 771, 737, 794, 722, 764, 744, 788, 755, 759, 769, 789, 745, 757, 758, 780, 756, 782, 717, 755, 750, 784, 776, 780, 770, 732, 774, 792, 799, 818, 786, 786, 731, 774, 751, 739, 779, 771, 776, 816, 776, 800, 747, 725, 768, 782, 771, 772, 764, 789, 754] payload_rows_per_dst=[9138, 9502, 9204, 9084, 9206, 9372, 9428, 9224] dup_rows_per_dst=[2850, 3082, 2986, 2918, 3036, 3100, 3146, 3028] combine_rows_per_src=[9138, 9502, 9204, 9084, 9206, 9372, 9428, 9224] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265764864 dispatch_meta_bytes=344752 combine_payload_bytes=265764864 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266109616 ingress_bytes=334970880 link_bytes=334970880 token_bytes=531874480 weight_movement_bytes=69206016 total_nvlink_bytes=601080496 weight_to_token_ratio=0.130117 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260468454e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74130 payload_rows_remote=64940 dup_rows=24174 dup_rows_remote=21324 dedup_ratio=0.245911 router_tokens_per_expert=[771, 789, 776, 776, 795, 754, 734, 781, 768, 738, 787, 748, 768, 766, 773, 837, 750, 767, 732, 774, 786, 744, 833, 825, 786, 742, 796, 761, 788, 763, 720, 664, 758, 801, 773, 807, 775, 774, 741, 761, 766, 783, 758, 739, 751, 733, 739, 772, 771, 757, 760, 782, 778, 758, 775, 833, 794, 785, 750, 727, 735, 779, 772, 773] payload_rows_per_dst=[9250, 9352, 9310, 9190, 9310, 9194, 9362, 9162] dup_rows_per_dst=[3102, 3018, 3112, 2850, 3070, 2888, 3066, 3068] combine_rows_per_src=[9250, 9352, 9310, 9190, 9310, 9194, 9362, 9162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265396224 dispatch_meta_bytes=344016 combine_payload_bytes=265396224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265740240 ingress_bytes=299999232 link_bytes=299999232 token_bytes=531136464 weight_movement_bytes=34603008 total_nvlink_bytes=565739472 weight_to_token_ratio=0.065149 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128872958e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74240 payload_rows_remote=65032 dup_rows=24064 dup_rows_remote=21114 dedup_ratio=0.244792 router_tokens_per_expert=[773, 735, 758, 751, 765, 778, 756, 762, 743, 774, 798, 770, 734, 765, 768, 762, 757, 776, 783, 751, 739, 792, 712, 771, 732, 738, 785, 818, 793, 782, 803, 783, 765, 776, 738, 774, 736, 757, 769, 764, 763, 763, 777, 777, 770, 759, 768, 761, 769, 754, 755, 782, 799, 728, 747, 766, 796, 790, 802, 771, 792, 795, 765, 817] payload_rows_per_dst=[9230, 9258, 9204, 9304, 9208, 9274, 9214, 9548] dup_rows_per_dst=[2926, 2970, 2958, 3164, 2950, 3002, 2986, 3108] combine_rows_per_src=[9230, 9258, 9204, 9304, 9208, 9274, 9214, 9548] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74368 payload_rows_remote=64994 dup_rows=23936 dup_rows_remote=20964 dedup_ratio=0.243490 router_tokens_per_expert=[771, 829, 741, 739, 750, 807, 781, 740, 755, 799, 776, 793, 745, 778, 777, 750, 775, 773, 721, 773, 736, 773, 744, 772, 760, 773, 760, 798, 765, 777, 770, 802, 753, 766, 768, 797, 747, 770, 777, 766, 783, 773, 810, 739, 771, 826, 705, 749, 761, 725, 749, 787, 774, 768, 758, 752, 778, 762, 788, 766, 785, 811, 762, 723] payload_rows_per_dst=[9302, 9374, 9168, 9336, 9344, 9330, 9264, 9250] dup_rows_per_dst=[3014, 2972, 2966, 3074, 2944, 2982, 2884, 3100] combine_rows_per_src=[9302, 9374, 9168, 9336, 9344, 9330, 9264, 9250] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266182656 dispatch_meta_bytes=343760 combine_payload_bytes=266182656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266526416 ingress_bytes=300785664 link_bytes=300785664 token_bytes=532709072 weight_movement_bytes=34603008 total_nvlink_bytes=567312080 weight_to_token_ratio=0.064957 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131832238e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74130 payload_rows_remote=64940 dup_rows=24174 dup_rows_remote=21324 dedup_ratio=0.245911 router_tokens_per_expert=[771, 789, 776, 776, 795, 754, 734, 781, 768, 738, 787, 748, 768, 766, 773, 837, 750, 767, 732, 774, 786, 744, 833, 825, 786, 742, 796, 761, 788, 763, 720, 664, 758, 801, 773, 807, 775, 774, 741, 761, 766, 783, 758, 739, 751, 733, 739, 772, 771, 757, 760, 782, 778, 758, 775, 833, 794, 785, 750, 727, 735, 779, 772, 773] payload_rows_per_dst=[9250, 9352, 9310, 9190, 9310, 9194, 9362, 9162] dup_rows_per_dst=[3102, 3018, 3112, 2850, 3070, 2888, 3066, 3068] combine_rows_per_src=[9250, 9352, 9310, 9190, 9310, 9194, 9362, 9162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74158 payload_rows_remote=64934 dup_rows=24146 dup_rows_remote=21118 dedup_ratio=0.245626 router_tokens_per_expert=[763, 790, 817, 773, 757, 789, 789, 744, 739, 737, 772, 748, 804, 756, 771, 737, 794, 722, 764, 744, 788, 755, 759, 769, 789, 745, 757, 758, 780, 756, 782, 717, 755, 750, 784, 776, 780, 770, 732, 774, 792, 799, 818, 786, 786, 731, 774, 751, 739, 779, 771, 776, 816, 776, 800, 747, 725, 768, 782, 771, 772, 764, 789, 754] payload_rows_per_dst=[9138, 9502, 9204, 9084, 9206, 9372, 9428, 9224] dup_rows_per_dst=[2850, 3082, 2986, 2918, 3036, 3100, 3146, 3028] combine_rows_per_src=[9138, 9502, 9204, 9084, 9206, 9372, 9428, 9224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=344384 combine_payload_bytes=266223616 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266568000 ingress_bytes=370032640 link_bytes=370032640 token_bytes=532791616 weight_movement_bytes=103809024 total_nvlink_bytes=636600640 weight_to_token_ratio=0.194840 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392403034e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=344144 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266453072 ingress_bytes=335314944 link_bytes=335314944 token_bytes=532562000 weight_movement_bytes=69206016 total_nvlink_bytes=601768016 weight_to_token_ratio=0.129949 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261763139e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265969664 dispatch_meta_bytes=344208 combine_payload_bytes=265969664 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266313872 ingress_bytes=369778688 link_bytes=369778688 token_bytes=532283536 weight_movement_bytes=103809024 total_nvlink_bytes=636092560 weight_to_token_ratio=0.195026 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391447433e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74114 payload_rows_remote=64994 dup_rows=24190 dup_rows_remote=21128 dedup_ratio=0.246073 router_tokens_per_expert=[722, 777, 745, 792, 779, 782, 792, 752, 749, 774, 769, 791, 806, 760, 745, 767, 753, 748, 719, 815, 777, 772, 742, 765, 761, 759, 807, 764, 806, 743, 724, 763, 758, 757, 757, 779, 732, 760, 735, 782, 794, 744, 734, 754, 775, 745, 767, 741, 783, 806, 778, 784, 785, 738, 794, 781, 804, 742, 789, 792, 802, 772, 797, 771] payload_rows_per_dst=[9248, 9270, 9120, 9300, 9152, 9250, 9358, 9416] dup_rows_per_dst=[3034, 3052, 3062, 2954, 2968, 2858, 3140, 3122] combine_rows_per_src=[9248, 9270, 9120, 9300, 9152, 9250, 9358, 9416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=64858 dup_rows=24020 dup_rows_remote=20770 dedup_ratio=0.244344 router_tokens_per_expert=[774, 816, 819, 785, 779, 775, 748, 808, 788, 747, 721, 716, 822, 803, 730, 761, 744, 808, 793, 807, 741, 797, 765, 758, 735, 778, 780, 737, 770, 756, 779, 763, 753, 760, 782, 757, 792, 745, 749, 781, 760, 780, 800, 755, 745, 727, 745, 769, 768, 738, 768, 755, 703, 785, 784, 778, 769, 776, 767, 777, 775, 799, 745, 762] payload_rows_per_dst=[9426, 9236, 9686, 9278, 9350, 9012, 9262, 9034] dup_rows_per_dst=[3250, 2940, 3448, 2864, 3206, 2832, 2896, 2584] combine_rows_per_src=[9426, 9236, 9686, 9278, 9350, 9012, 9262, 9034] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74090 payload_rows_remote=64756 dup_rows=24214 dup_rows_remote=21190 dedup_ratio=0.246318 router_tokens_per_expert=[715, 731, 745, 725, 754, 782, 790, 780, 765, 774, 742, 797, 745, 822, 742, 791, 794, 720, 798, 791, 774, 732, 735, 775, 762, 766, 759, 742, 776, 786, 768, 782, 785, 787, 779, 745, 851, 733, 783, 739, 763, 769, 782, 773, 791, 773, 738, 742, 764, 791, 787, 739, 799, 790, 761, 748, 784, 827, 773, 725, 720, 778, 806, 767] payload_rows_per_dst=[9096, 9346, 9252, 9302, 9294, 9304, 9334, 9162] dup_rows_per_dst=[2948, 3010, 2986, 2980, 3110, 2958, 3024, 3198] combine_rows_per_src=[9096, 9346, 9252, 9302, 9294, 9304, 9334, 9162] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=345056 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266339296 ingress_bytes=404406272 link_bytes=404406272 token_bytes=532333536 weight_movement_bytes=138412032 total_nvlink_bytes=670745568 weight_to_token_ratio=0.260010 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521748244e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344584 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266715656 ingress_bytes=300974080 link_bytes=300974080 token_bytes=533086728 weight_movement_bytes=34603008 total_nvlink_bytes=567689736 weight_to_token_ratio=0.064911 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132541233e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266215424 dispatch_meta_bytes=343832 combine_payload_bytes=266215424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266559256 ingress_bytes=300818432 link_bytes=300818432 token_bytes=532774680 weight_movement_bytes=34603008 total_nvlink_bytes=567377688 weight_to_token_ratio=0.064949 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131955542e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74148 payload_rows_remote=65032 dup_rows=24156 dup_rows_remote=21184 dedup_ratio=0.245728 router_tokens_per_expert=[788, 787, 739, 739, 801, 783, 781, 778, 772, 761, 735, 757, 790, 764, 756, 776, 777, 772, 782, 760, 747, 697, 766, 743, 789, 797, 749, 751, 762, 745, 736, 811, 774, 778, 716, 728, 786, 802, 762, 738, 770, 779, 772, 814, 735, 813, 811, 752, 753, 735, 778, 788, 748, 759, 758, 785, 766, 763, 789, 725, 792, 795, 803, 794] payload_rows_per_dst=[9374, 9232, 9116, 9244, 9226, 9388, 9246, 9322] dup_rows_per_dst=[3018, 2990, 2972, 3036, 2942, 3104, 2962, 3132] combine_rows_per_src=[9374, 9232, 9116, 9244, 9226, 9388, 9246, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265969664 dispatch_meta_bytes=344208 combine_payload_bytes=265969664 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266313872 ingress_bytes=369778688 link_bytes=369778688 token_bytes=532283536 weight_movement_bytes=103809024 total_nvlink_bytes=636092560 weight_to_token_ratio=0.195026 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391447433e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74368 payload_rows_remote=64994 dup_rows=23936 dup_rows_remote=20964 dedup_ratio=0.243490 router_tokens_per_expert=[771, 829, 741, 739, 750, 807, 781, 740, 755, 799, 776, 793, 745, 778, 777, 750, 775, 773, 721, 773, 736, 773, 744, 772, 760, 773, 760, 798, 765, 777, 770, 802, 753, 766, 768, 797, 747, 770, 777, 766, 783, 773, 810, 739, 771, 826, 705, 749, 761, 725, 749, 787, 774, 768, 758, 752, 778, 762, 788, 766, 785, 811, 762, 723] payload_rows_per_dst=[9302, 9374, 9168, 9336, 9344, 9330, 9264, 9250] dup_rows_per_dst=[3014, 2972, 2966, 3074, 2944, 2982, 2884, 3100] combine_rows_per_src=[9302, 9374, 9168, 9336, 9344, 9330, 9264, 9250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266215424 dispatch_meta_bytes=344488 combine_payload_bytes=266215424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266559912 ingress_bytes=300818432 link_bytes=300818432 token_bytes=532775336 weight_movement_bytes=34603008 total_nvlink_bytes=567378344 weight_to_token_ratio=0.064949 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131955542e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265658368 dispatch_meta_bytes=342512 combine_payload_bytes=265658368 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266000880 ingress_bytes=334864384 link_bytes=334864384 token_bytes=531659248 weight_movement_bytes=69206016 total_nvlink_bytes=600865264 weight_to_token_ratio=0.130170 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260067718e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265240576 dispatch_meta_bytes=343784 combine_payload_bytes=265240576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265584360 ingress_bytes=299843584 link_bytes=299843584 token_bytes=530824936 weight_movement_bytes=34603008 total_nvlink_bytes=565427944 weight_to_token_ratio=0.065187 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128287267e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74334 payload_rows_remote=65000 dup_rows=23970 dup_rows_remote=20854 dedup_ratio=0.243835 router_tokens_per_expert=[799, 743, 771, 740, 794, 757, 770, 796, 764, 751, 756, 732, 712, 789, 743, 789, 780, 776, 789, 774, 761, 749, 743, 730, 726, 790, 757, 787, 822, 755, 757, 780, 772, 773, 722, 760, 754, 767, 823, 775, 772, 778, 782, 763, 766, 799, 780, 785, 763, 791, 737, 768, 796, 798, 744, 735, 775, 763, 709, 765, 755, 845, 747, 808] payload_rows_per_dst=[9406, 9168, 9270, 9364, 9286, 9334, 9220, 9286] dup_rows_per_dst=[2934, 2904, 2934, 2984, 3006, 3116, 3044, 3048] combine_rows_per_src=[9406, 9168, 9270, 9364, 9286, 9334, 9220, 9286] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=64858 dup_rows=24020 dup_rows_remote=20770 dedup_ratio=0.244344 router_tokens_per_expert=[774, 816, 819, 785, 779, 775, 748, 808, 788, 747, 721, 716, 822, 803, 730, 761, 744, 808, 793, 807, 741, 797, 765, 758, 735, 778, 780, 737, 770, 756, 779, 763, 753, 760, 782, 757, 792, 745, 749, 781, 760, 780, 800, 755, 745, 727, 745, 769, 768, 738, 768, 755, 703, 785, 784, 778, 769, 776, 767, 777, 775, 799, 745, 762] payload_rows_per_dst=[9426, 9236, 9686, 9278, 9350, 9012, 9262, 9034] dup_rows_per_dst=[3250, 2940, 3448, 2864, 3206, 2832, 2896, 2584] combine_rows_per_src=[9426, 9236, 9686, 9278, 9350, 9012, 9262, 9034] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74240 payload_rows_remote=65032 dup_rows=24064 dup_rows_remote=21114 dedup_ratio=0.244792 router_tokens_per_expert=[773, 735, 758, 751, 765, 778, 756, 762, 743, 774, 798, 770, 734, 765, 768, 762, 757, 776, 783, 751, 739, 792, 712, 771, 732, 738, 785, 818, 793, 782, 803, 783, 765, 776, 738, 774, 736, 757, 769, 764, 763, 763, 777, 777, 770, 759, 768, 761, 769, 754, 755, 782, 799, 728, 747, 766, 796, 790, 802, 771, 792, 795, 765, 817] payload_rows_per_dst=[9230, 9258, 9204, 9304, 9208, 9274, 9214, 9548] dup_rows_per_dst=[2926, 2970, 2958, 3164, 2950, 3002, 2986, 3108] combine_rows_per_src=[9230, 9258, 9204, 9304, 9208, 9274, 9214, 9548] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74420 payload_rows_remote=65056 dup_rows=23884 dup_rows_remote=20960 dedup_ratio=0.242961 router_tokens_per_expert=[823, 778, 722, 796, 758, 767, 754, 769, 742, 778, 746, 798, 734, 782, 747, 802, 772, 764, 757, 761, 754, 763, 744, 784, 753, 730, 792, 793, 763, 714, 747, 740, 766, 768, 801, 770, 807, 804, 793, 767, 761, 766, 770, 775, 753, 766, 748, 783, 755, 777, 779, 784, 773, 769, 786, 721, 773, 786, 768, 783, 787, 739, 769, 778] payload_rows_per_dst=[9392, 9238, 9232, 9114, 9426, 9256, 9364, 9398] dup_rows_per_dst=[2942, 3020, 2966, 2950, 3126, 2988, 2924, 2968] combine_rows_per_src=[9392, 9238, 9232, 9114, 9426, 9256, 9364, 9398] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344864 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266715936 ingress_bytes=300974080 link_bytes=300974080 token_bytes=533087008 weight_movement_bytes=34603008 total_nvlink_bytes=567690016 weight_to_token_ratio=0.064911 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132541233e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74092 payload_rows_remote=64804 dup_rows=24212 dup_rows_remote=21280 dedup_ratio=0.246297 router_tokens_per_expert=[781, 782, 775, 774, 746, 789, 791, 727, 746, 775, 755, 796, 686, 790, 780, 767, 803, 796, 752, 781, 777, 782, 768, 766, 768, 767, 738, 740, 827, 752, 779, 725, 778, 779, 770, 767, 753, 753, 811, 758, 813, 795, 766, 772, 738, 782, 735, 772, 758, 777, 748, 801, 736, 739, 767, 793, 757, 725, 761, 751, 754, 798, 767, 797] payload_rows_per_dst=[8506, 9196, 9210, 9980, 9324, 9284, 9304, 9288] dup_rows_per_dst=[2400, 2994, 3122, 3636, 3014, 3180, 2934, 2932] combine_rows_per_src=[8506, 9196, 9210, 9980, 9324, 9284, 9304, 9288] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74082 payload_rows_remote=64910 dup_rows=24222 dup_rows_remote=21022 dedup_ratio=0.246399 router_tokens_per_expert=[728, 780, 776, 821, 774, 764, 786, 767, 742, 770, 736, 751, 795, 782, 824, 760, 740, 806, 742, 747, 747, 790, 746, 759, 800, 744, 772, 780, 775, 757, 800, 758, 802, 773, 793, 746, 728, 745, 737, 774, 758, 777, 747, 761, 785, 741, 773, 756, 757, 782, 797, 753, 772, 763, 761, 812, 791, 800, 734, 807, 739, 718, 765, 786] payload_rows_per_dst=[9274, 9320, 9242, 9172, 9256, 9146, 9418, 9254] dup_rows_per_dst=[3118, 3000, 2912, 3200, 2940, 3050, 2976, 3026] combine_rows_per_src=[9274, 9320, 9242, 9172, 9256, 9146, 9418, 9254] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266215424 dispatch_meta_bytes=343832 combine_payload_bytes=266215424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266559256 ingress_bytes=300818432 link_bytes=300818432 token_bytes=532774680 weight_movement_bytes=34603008 total_nvlink_bytes=567377688 weight_to_token_ratio=0.064949 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131955542e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266240000 dispatch_meta_bytes=343416 combine_payload_bytes=266240000 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266583416 ingress_bytes=335446016 link_bytes=335446016 token_bytes=532823416 weight_movement_bytes=69206016 total_nvlink_bytes=602029432 weight_to_token_ratio=0.129885 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262256353e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=64992 dup_rows=23982 dup_rows_remote=20990 dedup_ratio=0.243958 router_tokens_per_expert=[758, 786, 829, 740, 740, 751, 751, 782, 782, 757, 747, 765, 765, 785, 717, 793, 731, 772, 780, 815, 757, 758, 759, 767, 764, 745, 802, 741, 730, 779, 800, 751, 774, 780, 832, 774, 749, 760, 738, 754, 772, 777, 707, 802, 791, 779, 738, 772, 748, 817, 794, 784, 726, 819, 783, 766, 763, 755, 743, 791, 765, 770, 772, 758] payload_rows_per_dst=[9256, 9212, 9320, 9296, 9330, 9238, 9424, 9246] dup_rows_per_dst=[3018, 3010, 2958, 2928, 2992, 3038, 3050, 2988] combine_rows_per_src=[9256, 9212, 9320, 9296, 9330, 9238, 9424, 9246] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74466 payload_rows_remote=65232 dup_rows=23838 dup_rows_remote=20874 dedup_ratio=0.242493 router_tokens_per_expert=[763, 815, 789, 793, 741, 786, 757, 807, 778, 762, 785, 811, 751, 735, 729, 748, 788, 765, 801, 769, 803, 815, 780, 746, 729, 714, 768, 745, 712, 742, 743, 740, 788, 758, 770, 823, 749, 772, 734, 714, 769, 800, 791, 809, 734, 757, 748, 722, 780, 771, 761, 777, 761, 789, 719, 747, 804, 840, 806, 751, 783, 807, 731, 777] payload_rows_per_dst=[9344, 9234, 9516, 9060, 9326, 9282, 9242, 9462] dup_rows_per_dst=[3158, 2964, 3018, 2726, 2890, 2978, 2968, 3136] combine_rows_per_src=[9344, 9234, 9516, 9060, 9326, 9282, 9242, 9462] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265658368 dispatch_meta_bytes=342512 combine_payload_bytes=265658368 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266000880 ingress_bytes=334864384 link_bytes=334864384 token_bytes=531659248 weight_movement_bytes=69206016 total_nvlink_bytes=600865264 weight_to_token_ratio=0.130170 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260067718e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74082 payload_rows_remote=64910 dup_rows=24222 dup_rows_remote=21022 dedup_ratio=0.246399 router_tokens_per_expert=[728, 780, 776, 821, 774, 764, 786, 767, 742, 770, 736, 751, 795, 782, 824, 760, 740, 806, 742, 747, 747, 790, 746, 759, 800, 744, 772, 780, 775, 757, 800, 758, 802, 773, 793, 746, 728, 745, 737, 774, 758, 777, 747, 761, 785, 741, 773, 756, 757, 782, 797, 753, 772, 763, 761, 812, 791, 800, 734, 807, 739, 718, 765, 786] payload_rows_per_dst=[9274, 9320, 9242, 9172, 9256, 9146, 9418, 9254] dup_rows_per_dst=[3118, 3000, 2912, 3200, 2940, 3050, 2976, 3026] combine_rows_per_src=[9274, 9320, 9242, 9172, 9256, 9146, 9418, 9254] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344584 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266715656 ingress_bytes=300974080 link_bytes=300974080 token_bytes=533086728 weight_movement_bytes=34603008 total_nvlink_bytes=567689736 weight_to_token_ratio=0.064911 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132541233e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74092 payload_rows_remote=64804 dup_rows=24212 dup_rows_remote=21280 dedup_ratio=0.246297 router_tokens_per_expert=[781, 782, 775, 774, 746, 789, 791, 727, 746, 775, 755, 796, 686, 790, 780, 767, 803, 796, 752, 781, 777, 782, 768, 766, 768, 767, 738, 740, 827, 752, 779, 725, 778, 779, 770, 767, 753, 753, 811, 758, 813, 795, 766, 772, 738, 782, 735, 772, 758, 777, 748, 801, 736, 739, 767, 793, 757, 725, 761, 751, 754, 798, 767, 797] payload_rows_per_dst=[8506, 9196, 9210, 9980, 9324, 9284, 9304, 9288] dup_rows_per_dst=[2400, 2994, 3122, 3636, 3014, 3180, 2934, 2932] combine_rows_per_src=[8506, 9196, 9210, 9980, 9324, 9284, 9304, 9288] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266469376 dispatch_meta_bytes=344064 combine_payload_bytes=266469376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266813440 ingress_bytes=301072384 link_bytes=301072384 token_bytes=533282816 weight_movement_bytes=34603008 total_nvlink_bytes=567885824 weight_to_token_ratio=0.064887 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132911143e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265437184 dispatch_meta_bytes=344336 combine_payload_bytes=265437184 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265781520 ingress_bytes=334643200 link_bytes=334643200 token_bytes=531218704 weight_movement_bytes=69206016 total_nvlink_bytes=600424720 weight_to_token_ratio=0.130278 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259235421e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265871360 dispatch_meta_bytes=343728 combine_payload_bytes=265871360 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266215088 ingress_bytes=369680384 link_bytes=369680384 token_bytes=532086448 weight_movement_bytes=103809024 total_nvlink_bytes=635895472 weight_to_token_ratio=0.195098 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391077523e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266207232 dispatch_meta_bytes=343928 combine_payload_bytes=266207232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266551160 ingress_bytes=300810240 link_bytes=300810240 token_bytes=532758392 weight_movement_bytes=34603008 total_nvlink_bytes=567361400 weight_to_token_ratio=0.064951 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131924716e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=267190272 dispatch_meta_bytes=344424 combine_payload_bytes=267190272 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267534696 ingress_bytes=336396288 link_bytes=336396288 token_bytes=534724968 weight_movement_bytes=69206016 total_nvlink_bytes=603930984 weight_to_token_ratio=0.129424 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265832150e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74538 payload_rows_remote=65234 dup_rows=23766 dup_rows_remote=20878 dedup_ratio=0.241760 router_tokens_per_expert=[813, 770, 760, 764, 741, 794, 778, 760, 746, 767, 760, 762, 797, 793, 772, 808, 759, 776, 721, 789, 790, 735, 751, 775, 774, 734, 766, 819, 745, 793, 804, 702, 762, 726, 821, 744, 775, 762, 790, 726, 822, 762, 775, 744, 736, 737, 777, 751, 788, 759, 766, 739, 773, 761, 745, 762, 816, 776, 809, 739, 797, 753, 744, 797] payload_rows_per_dst=[9344, 9396, 9304, 9346, 9314, 9238, 9230, 9366] dup_rows_per_dst=[3016, 3014, 2888, 2928, 2898, 2970, 2956, 3096] combine_rows_per_src=[9344, 9396, 9304, 9346, 9314, 9238, 9230, 9366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74086 payload_rows_remote=64828 dup_rows=24218 dup_rows_remote=21074 dedup_ratio=0.246358 router_tokens_per_expert=[818, 767, 786, 774, 777, 746, 809, 752, 797, 775, 758, 730, 783, 743, 789, 763, 775, 784, 759, 777, 749, 780, 760, 737, 723, 734, 821, 748, 743, 772, 785, 755, 811, 796, 781, 790, 745, 758, 734, 780, 777, 771, 754, 818, 768, 778, 784, 751, 758, 756, 766, 787, 750, 797, 743, 800, 741, 743, 744, 777, 755, 743, 758, 769] payload_rows_per_dst=[9446, 9258, 9236, 9100, 9362, 9258, 9284, 9142] dup_rows_per_dst=[3012, 3018, 3006, 3062, 3028, 3144, 3030, 2918] combine_rows_per_src=[9446, 9258, 9236, 9100, 9362, 9258, 9284, 9142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74024 payload_rows_remote=64884 dup_rows=24280 dup_rows_remote=21304 dedup_ratio=0.246989 router_tokens_per_expert=[744, 759, 766, 804, 753, 775, 799, 709, 787, 847, 753, 739, 816, 780, 763, 769, 727, 733, 734, 773, 783, 793, 782, 733, 765, 757, 796, 809, 755, 762, 772, 798, 757, 748, 778, 760, 801, 800, 752, 752, 745, 750, 766, 772, 808, 784, 775, 808, 785, 703, 750, 755, 766, 780, 788, 727, 776, 746, 724, 789, 752, 790, 803, 727] payload_rows_per_dst=[9240, 9402, 9140, 9264, 9264, 9316, 9200, 9198] dup_rows_per_dst=[2978, 3106, 2976, 3164, 3032, 3100, 2908, 3016] combine_rows_per_src=[9240, 9402, 9140, 9264, 9264, 9316, 9200, 9198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74478 payload_rows_remote=65148 dup_rows=23826 dup_rows_remote=20942 dedup_ratio=0.242371 router_tokens_per_expert=[751, 780, 770, 775, 785, 775, 764, 768, 828, 729, 758, 788, 728, 809, 784, 766, 739, 786, 805, 764, 771, 777, 800, 731, 802, 776, 795, 772, 775, 737, 735, 719, 783, 727, 772, 740, 760, 723, 790, 797, 769, 759, 700, 777, 771, 784, 784, 747, 784, 748, 790, 717, 766, 740, 786, 776, 751, 809, 780, 819, 771, 770, 745, 775] payload_rows_per_dst=[9298, 9364, 9282, 9298, 9236, 9256, 9330, 9414] dup_rows_per_dst=[3038, 3016, 3064, 2924, 2948, 2926, 2884, 3026] combine_rows_per_src=[9298, 9364, 9282, 9298, 9236, 9256, 9330, 9414] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74088 payload_rows_remote=64364 dup_rows=24216 dup_rows_remote=20740 dedup_ratio=0.246338 router_tokens_per_expert=[782, 793, 743, 773, 765, 792, 786, 792, 784, 774, 827, 789, 804, 789, 803, 763, 773, 782, 788, 725, 790, 741, 753, 769, 762, 728, 746, 740, 751, 741, 764, 721, 729, 787, 751, 752, 784, 742, 740, 835, 784, 822, 714, 764, 772, 790, 726, 768, 729, 744, 770, 728, 767, 748, 820, 760, 793, 734, 809, 774, 774, 764, 788, 757] payload_rows_per_dst=[9724, 9218, 9132, 9106, 9316, 9122, 9238, 9232] dup_rows_per_dst=[3476, 2956, 2904, 2800, 3114, 2968, 3108, 2890] combine_rows_per_src=[9724, 9218, 9132, 9106, 9316, 9122, 9238, 9232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=267198464 dispatch_meta_bytes=344448 combine_payload_bytes=267198464 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267542912 ingress_bytes=371007488 link_bytes=371007488 token_bytes=534741376 weight_movement_bytes=103809024 total_nvlink_bytes=638550400 weight_to_token_ratio=0.194129 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396071309e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74090 payload_rows_remote=64756 dup_rows=24214 dup_rows_remote=21190 dedup_ratio=0.246318 router_tokens_per_expert=[715, 731, 745, 725, 754, 782, 790, 780, 765, 774, 742, 797, 745, 822, 742, 791, 794, 720, 798, 791, 774, 732, 735, 775, 762, 766, 759, 742, 776, 786, 768, 782, 785, 787, 779, 745, 851, 733, 783, 739, 763, 769, 782, 773, 791, 773, 738, 742, 764, 791, 787, 739, 799, 790, 761, 748, 784, 827, 773, 725, 720, 778, 806, 767] payload_rows_per_dst=[9096, 9346, 9252, 9302, 9294, 9304, 9334, 9162] dup_rows_per_dst=[2948, 3010, 2986, 2980, 3110, 2958, 3024, 3198] combine_rows_per_src=[9096, 9346, 9252, 9302, 9294, 9304, 9334, 9162] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74260 payload_rows_remote=65002 dup_rows=24044 dup_rows_remote=20992 dedup_ratio=0.244588 router_tokens_per_expert=[830, 758, 813, 757, 803, 782, 774, 769, 770, 758, 745, 768, 785, 775, 787, 785, 799, 735, 725, 779, 754, 767, 773, 756, 743, 783, 757, 785, 759, 719, 763, 728, 804, 772, 790, 774, 745, 766, 765, 778, 737, 760, 784, 759, 783, 738, 735, 787, 807, 748, 728, 813, 776, 758, 754, 774, 775, 743, 788, 731, 781, 760, 750, 805] payload_rows_per_dst=[9334, 9370, 9232, 9392, 9292, 9100, 9282, 9258] dup_rows_per_dst=[2916, 2976, 2944, 3004, 3052, 2876, 3224, 3052] combine_rows_per_src=[9334, 9370, 9232, 9392, 9292, 9100, 9282, 9258] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265871360 dispatch_meta_bytes=343728 combine_payload_bytes=265871360 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266215088 ingress_bytes=369680384 link_bytes=369680384 token_bytes=532086448 weight_movement_bytes=103809024 total_nvlink_bytes=635895472 weight_to_token_ratio=0.195098 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391077523e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265437184 dispatch_meta_bytes=344336 combine_payload_bytes=265437184 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265781520 ingress_bytes=334643200 link_bytes=334643200 token_bytes=531218704 weight_movement_bytes=69206016 total_nvlink_bytes=600424720 weight_to_token_ratio=0.130278 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259235421e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74310 payload_rows_remote=65128 dup_rows=23994 dup_rows_remote=21062 dedup_ratio=0.244080 router_tokens_per_expert=[768, 787, 745, 766, 818, 763, 731, 756, 758, 796, 756, 755, 807, 750, 800, 805, 745, 763, 786, 781, 799, 757, 755, 786, 764, 756, 758, 826, 789, 748, 690, 726, 786, 769, 746, 758, 777, 801, 748, 721, 760, 786, 776, 749, 762, 813, 812, 753, 797, 808, 776, 713, 741, 802, 777, 747, 763, 754, 761, 784, 742, 769, 747, 764] payload_rows_per_dst=[9302, 9352, 9332, 9182, 9234, 9380, 9276, 9252] dup_rows_per_dst=[2966, 3102, 3012, 2932, 2978, 3042, 3046, 2916] combine_rows_per_src=[9302, 9352, 9332, 9182, 9234, 9380, 9276, 9252] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265535488 dispatch_meta_bytes=343608 combine_payload_bytes=265535488 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265879096 ingress_bytes=334741504 link_bytes=334741504 token_bytes=531414584 weight_movement_bytes=69206016 total_nvlink_bytes=600620600 weight_to_token_ratio=0.130230 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259605331e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74386 payload_rows_remote=64868 dup_rows=23918 dup_rows_remote=20798 dedup_ratio=0.243306 router_tokens_per_expert=[756, 761, 777, 763, 754, 787, 777, 766, 760, 769, 743, 734, 763, 739, 771, 733, 756, 779, 770, 730, 741, 783, 770, 808, 754, 768, 789, 762, 791, 778, 787, 726, 799, 796, 767, 822, 775, 814, 768, 778, 731, 754, 767, 743, 756, 799, 768, 820, 740, 799, 759, 773, 786, 766, 757, 795, 786, 809, 750, 745, 767, 735, 749, 734] payload_rows_per_dst=[9254, 9122, 9266, 9260, 9518, 9220, 9382, 9364] dup_rows_per_dst=[3028, 2902, 3008, 3050, 3120, 3056, 2968, 2786] combine_rows_per_src=[9254, 9122, 9266, 9260, 9518, 9220, 9382, 9364] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74466 payload_rows_remote=65232 dup_rows=23838 dup_rows_remote=20874 dedup_ratio=0.242493 router_tokens_per_expert=[763, 815, 789, 793, 741, 786, 757, 807, 778, 762, 785, 811, 751, 735, 729, 748, 788, 765, 801, 769, 803, 815, 780, 746, 729, 714, 768, 745, 712, 742, 743, 740, 788, 758, 770, 823, 749, 772, 734, 714, 769, 800, 791, 809, 734, 757, 748, 722, 780, 771, 761, 777, 761, 789, 719, 747, 804, 840, 806, 751, 783, 807, 731, 777] payload_rows_per_dst=[9344, 9234, 9516, 9060, 9326, 9282, 9242, 9462] dup_rows_per_dst=[3158, 2964, 3018, 2726, 2890, 2978, 2968, 3136] combine_rows_per_src=[9344, 9234, 9516, 9060, 9326, 9282, 9242, 9462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265764864 dispatch_meta_bytes=344752 combine_payload_bytes=265764864 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266109616 ingress_bytes=334970880 link_bytes=334970880 token_bytes=531874480 weight_movement_bytes=69206016 total_nvlink_bytes=601080496 weight_to_token_ratio=0.130117 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260468454e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266846208 dispatch_meta_bytes=344360 combine_payload_bytes=266846208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267190568 ingress_bytes=301449216 link_bytes=301449216 token_bytes=534036776 weight_movement_bytes=34603008 total_nvlink_bytes=568639784 weight_to_token_ratio=0.064795 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134329131e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=263634944 dispatch_meta_bytes=340416 combine_payload_bytes=263634944 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=263975360 ingress_bytes=332840960 link_bytes=332840960 token_bytes=527610304 weight_movement_bytes=69206016 total_nvlink_bytes=596816320 weight_to_token_ratio=0.131169 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.252453737e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74462 payload_rows_remote=65148 dup_rows=23842 dup_rows_remote=20874 dedup_ratio=0.242533 router_tokens_per_expert=[767, 741, 747, 789, 750, 800, 723, 762, 751, 753, 754, 742, 782, 739, 786, 834, 759, 733, 745, 736, 760, 779, 812, 748, 765, 752, 746, 787, 798, 775, 797, 754, 724, 715, 750, 820, 781, 749, 807, 800, 804, 732, 775, 767, 767, 752, 783, 740, 776, 777, 763, 759, 780, 786, 778, 736, 774, 803, 746, 767, 809, 813, 768, 785] payload_rows_per_dst=[9154, 9314, 9222, 9376, 9322, 9278, 9318, 9478] dup_rows_per_dst=[3004, 2968, 2922, 2972, 2970, 2962, 2992, 3052] combine_rows_per_src=[9154, 9314, 9222, 9376, 9322, 9278, 9318, 9478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74086 payload_rows_remote=64828 dup_rows=24218 dup_rows_remote=21074 dedup_ratio=0.246358 router_tokens_per_expert=[818, 767, 786, 774, 777, 746, 809, 752, 797, 775, 758, 730, 783, 743, 789, 763, 775, 784, 759, 777, 749, 780, 760, 737, 723, 734, 821, 748, 743, 772, 785, 755, 811, 796, 781, 790, 745, 758, 734, 780, 777, 771, 754, 818, 768, 778, 784, 751, 758, 756, 766, 787, 750, 797, 743, 800, 741, 743, 744, 777, 755, 743, 758, 769] payload_rows_per_dst=[9446, 9258, 9236, 9100, 9362, 9258, 9284, 9142] dup_rows_per_dst=[3012, 3018, 3006, 3062, 3028, 3144, 3030, 2918] combine_rows_per_src=[9446, 9258, 9236, 9100, 9362, 9258, 9284, 9142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=64992 dup_rows=23982 dup_rows_remote=20990 dedup_ratio=0.243958 router_tokens_per_expert=[758, 786, 829, 740, 740, 751, 751, 782, 782, 757, 747, 765, 765, 785, 717, 793, 731, 772, 780, 815, 757, 758, 759, 767, 764, 745, 802, 741, 730, 779, 800, 751, 774, 780, 832, 774, 749, 760, 738, 754, 772, 777, 707, 802, 791, 779, 738, 772, 748, 817, 794, 784, 726, 819, 783, 766, 763, 755, 743, 791, 765, 770, 772, 758] payload_rows_per_dst=[9256, 9212, 9320, 9296, 9330, 9238, 9424, 9246] dup_rows_per_dst=[3018, 3010, 2958, 2928, 2992, 3038, 3050, 2988] combine_rows_per_src=[9256, 9212, 9320, 9296, 9330, 9238, 9424, 9246] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=267190272 dispatch_meta_bytes=344424 combine_payload_bytes=267190272 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267534696 ingress_bytes=336396288 link_bytes=336396288 token_bytes=534724968 weight_movement_bytes=69206016 total_nvlink_bytes=603930984 weight_to_token_ratio=0.129424 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265832150e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265535488 dispatch_meta_bytes=343608 combine_payload_bytes=265535488 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265879096 ingress_bytes=334741504 link_bytes=334741504 token_bytes=531414584 weight_movement_bytes=69206016 total_nvlink_bytes=600620600 weight_to_token_ratio=0.130230 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259605331e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74310 payload_rows_remote=65128 dup_rows=23994 dup_rows_remote=21062 dedup_ratio=0.244080 router_tokens_per_expert=[768, 787, 745, 766, 818, 763, 731, 756, 758, 796, 756, 755, 807, 750, 800, 805, 745, 763, 786, 781, 799, 757, 755, 786, 764, 756, 758, 826, 789, 748, 690, 726, 786, 769, 746, 758, 777, 801, 748, 721, 760, 786, 776, 749, 762, 813, 812, 753, 797, 808, 776, 713, 741, 802, 777, 747, 763, 754, 761, 784, 742, 769, 747, 764] payload_rows_per_dst=[9302, 9352, 9332, 9182, 9234, 9380, 9276, 9252] dup_rows_per_dst=[2966, 3102, 3012, 2932, 2978, 3042, 3046, 2916] combine_rows_per_src=[9302, 9352, 9332, 9182, 9234, 9380, 9276, 9252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266207232 dispatch_meta_bytes=343928 combine_payload_bytes=266207232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266551160 ingress_bytes=300810240 link_bytes=300810240 token_bytes=532758392 weight_movement_bytes=34603008 total_nvlink_bytes=567361400 weight_to_token_ratio=0.064951 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131924716e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74260 payload_rows_remote=65002 dup_rows=24044 dup_rows_remote=20992 dedup_ratio=0.244588 router_tokens_per_expert=[830, 758, 813, 757, 803, 782, 774, 769, 770, 758, 745, 768, 785, 775, 787, 785, 799, 735, 725, 779, 754, 767, 773, 756, 743, 783, 757, 785, 759, 719, 763, 728, 804, 772, 790, 774, 745, 766, 765, 778, 737, 760, 784, 759, 783, 738, 735, 787, 807, 748, 728, 813, 776, 758, 754, 774, 775, 743, 788, 731, 781, 760, 750, 805] payload_rows_per_dst=[9334, 9370, 9232, 9392, 9292, 9100, 9282, 9258] dup_rows_per_dst=[2916, 2976, 2944, 3004, 3052, 2876, 3224, 3052] combine_rows_per_src=[9334, 9370, 9232, 9392, 9292, 9100, 9282, 9258] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265240576 dispatch_meta_bytes=343784 combine_payload_bytes=265240576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265584360 ingress_bytes=299843584 link_bytes=299843584 token_bytes=530824936 weight_movement_bytes=34603008 total_nvlink_bytes=565427944 weight_to_token_ratio=0.065187 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128287267e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=343976 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266592168 ingress_bytes=335454208 link_bytes=335454208 token_bytes=532840360 weight_movement_bytes=69206016 total_nvlink_bytes=602046376 weight_to_token_ratio=0.129881 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262287178e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64962 dup_rows=24068 dup_rows_remote=21024 dedup_ratio=0.244832 router_tokens_per_expert=[785, 733, 746, 773, 757, 771, 746, 762, 767, 751, 785, 785, 795, 749, 771, 735, 784, 797, 736, 770, 777, 746, 791, 780, 732, 845, 770, 804, 755, 738, 811, 756, 815, 760, 709, 756, 744, 758, 788, 715, 723, 756, 832, 751, 762, 793, 740, 808, 769, 783, 766, 778, 765, 772, 739, 787, 794, 752, 741, 741, 784, 763, 790, 815] payload_rows_per_dst=[9240, 9266, 9276, 9356, 9196, 9322, 9274, 9306] dup_rows_per_dst=[2906, 3010, 3086, 3066, 2894, 3008, 3044, 3054] combine_rows_per_src=[9240, 9266, 9276, 9356, 9196, 9322, 9274, 9306] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74148 payload_rows_remote=65032 dup_rows=24156 dup_rows_remote=21184 dedup_ratio=0.245728 router_tokens_per_expert=[788, 787, 739, 739, 801, 783, 781, 778, 772, 761, 735, 757, 790, 764, 756, 776, 777, 772, 782, 760, 747, 697, 766, 743, 789, 797, 749, 751, 762, 745, 736, 811, 774, 778, 716, 728, 786, 802, 762, 738, 770, 779, 772, 814, 735, 813, 811, 752, 753, 735, 778, 788, 748, 759, 758, 785, 766, 763, 789, 725, 792, 795, 803, 794] payload_rows_per_dst=[9374, 9232, 9116, 9244, 9226, 9388, 9246, 9322] dup_rows_per_dst=[3018, 2990, 2972, 3036, 2942, 3104, 2962, 3132] combine_rows_per_src=[9374, 9232, 9116, 9244, 9226, 9388, 9246, 9322] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266764288 dispatch_meta_bytes=344760 combine_payload_bytes=266764288 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267109048 ingress_bytes=301367296 link_bytes=301367296 token_bytes=533873336 weight_movement_bytes=34603008 total_nvlink_bytes=568476344 weight_to_token_ratio=0.064815 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134020873e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265699328 dispatch_meta_bytes=342664 combine_payload_bytes=265699328 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266041992 ingress_bytes=369508352 link_bytes=369508352 token_bytes=531741320 weight_movement_bytes=103809024 total_nvlink_bytes=635550344 weight_to_token_ratio=0.195225 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390430181e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=65118 dup_rows=23964 dup_rows_remote=20990 dedup_ratio=0.243774 router_tokens_per_expert=[776, 795, 782, 765, 788, 806, 707, 774, 786, 750, 791, 764, 792, 719, 735, 741, 780, 769, 800, 764, 738, 754, 739, 754, 749, 772, 747, 746, 763, 775, 801, 796, 754, 841, 764, 762, 728, 768, 736, 800, 786, 788, 757, 783, 769, 759, 758, 798, 794, 737, 779, 748, 766, 759, 735, 785, 767, 791, 744, 788, 770, 796, 746, 778] payload_rows_per_dst=[9362, 9190, 9222, 9392, 9340, 9310, 9166, 9358] dup_rows_per_dst=[3024, 2966, 2974, 2906, 2966, 3086, 3040, 3002] combine_rows_per_src=[9362, 9190, 9222, 9392, 9340, 9310, 9166, 9358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266846208 dispatch_meta_bytes=344088 combine_payload_bytes=266846208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267190296 ingress_bytes=301449216 link_bytes=301449216 token_bytes=534036504 weight_movement_bytes=34603008 total_nvlink_bytes=568639512 weight_to_token_ratio=0.064795 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134329131e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12222339232 prefetch_bytes=761266176 reduce_bytes=519045120 weight_movement_bytes=1280311296 egress_bytes=6115113120 ingress_bytes=7387537408 link_bytes=7387537408 total_nvlink_bytes=13502650528 weight_to_token_ratio=0.104752 prefetch_share_of_ingress=0.103047 dedup_ratio=0.244646 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=15 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029022 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.208639501e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12248722656 prefetch_bytes=657457152 reduce_bytes=622854144 weight_movement_bytes=1280311296 egress_bytes=6128315616 ingress_bytes=7400718336 link_bytes=7400718336 total_nvlink_bytes=13529033952 weight_to_token_ratio=0.104526 prefetch_share_of_ingress=0.088837 dedup_ratio=0.244325 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=18 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.210795969e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74708 payload_rows_remote=65370 dup_rows=23596 dup_rows_remote=20616 dedup_ratio=0.240031 router_tokens_per_expert=[785, 799, 771, 796, 761, 765, 745, 799, 810, 804, 821, 719, 787, 774, 737, 779, 777, 769, 768, 772, 722, 756, 760, 758, 734, 759, 799, 802, 734, 744, 781, 749, 751, 736, 788, 787, 744, 778, 809, 785, 838, 741, 765, 763, 741, 796, 791, 724, 769, 728, 720, 784, 758, 756, 757, 762, 752, 794, 790, 768, 788, 760, 736, 757] payload_rows_per_dst=[9420, 9454, 9186, 9288, 9438, 9338, 9256, 9328] dup_rows_per_dst=[3022, 3008, 2978, 2916, 2918, 2980, 2812, 2962] combine_rows_per_src=[9420, 9454, 9186, 9288, 9438, 9338, 9256, 9328] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266723328 dispatch_meta_bytes=344432 combine_payload_bytes=266723328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267067760 ingress_bytes=301326336 link_bytes=301326336 token_bytes=533791088 weight_movement_bytes=34603008 total_nvlink_bytes=568394096 weight_to_token_ratio=0.064825 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133866744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12259359688 prefetch_bytes=761266176 reduce_bytes=484442112 weight_movement_bytes=1245708288 egress_bytes=6133636040 ingress_bytes=7371431936 link_bytes=7371431936 total_nvlink_bytes=13505067976 weight_to_token_ratio=0.101613 prefetch_share_of_ingress=0.103272 dedup_ratio=0.243881 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=14 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.206004562e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=64930 dup_rows=24040 dup_rows_remote=21054 dedup_ratio=0.244548 router_tokens_per_expert=[792, 763, 778, 745, 743, 778, 804, 757, 767, 777, 809, 740, 738, 767, 804, 765, 733, 735, 797, 754, 765, 823, 780, 780, 768, 749, 788, 765, 764, 758, 731, 707, 784, 777, 765, 786, 727, 756, 753, 800, 756, 767, 781, 745, 815, 738, 737, 790, 767, 770, 723, 766, 798, 763, 796, 758, 795, 743, 780, 775, 734, 790, 782, 811] payload_rows_per_dst=[9334, 9228, 9460, 9190, 9278, 9266, 9302, 9206] dup_rows_per_dst=[2986, 3158, 3244, 2870, 2986, 2972, 2980, 2844] combine_rows_per_src=[9334, 9228, 9460, 9190, 9278, 9266, 9302, 9206] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74478 payload_rows_remote=65148 dup_rows=23826 dup_rows_remote=20942 dedup_ratio=0.242371 router_tokens_per_expert=[751, 780, 770, 775, 785, 775, 764, 768, 828, 729, 758, 788, 728, 809, 784, 766, 739, 786, 805, 764, 771, 777, 800, 731, 802, 776, 795, 772, 775, 737, 735, 719, 783, 727, 772, 740, 760, 723, 790, 797, 769, 759, 700, 777, 771, 784, 784, 747, 784, 748, 790, 717, 766, 740, 786, 776, 751, 809, 780, 819, 771, 770, 745, 775] payload_rows_per_dst=[9298, 9364, 9282, 9298, 9236, 9256, 9330, 9414] dup_rows_per_dst=[3038, 3016, 3064, 2924, 2948, 2926, 2884, 3026] combine_rows_per_src=[9298, 9364, 9282, 9298, 9236, 9256, 9330, 9414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=343976 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266592168 ingress_bytes=335454208 link_bytes=335454208 token_bytes=532840360 weight_movement_bytes=69206016 total_nvlink_bytes=602046376 weight_to_token_ratio=0.129881 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262287178e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74508 payload_rows_remote=65290 dup_rows=23796 dup_rows_remote=20748 dedup_ratio=0.242065 router_tokens_per_expert=[756, 724, 794, 785, 830, 724, 772, 775, 801, 735, 746, 776, 770, 780, 801, 777, 753, 769, 767, 752, 821, 749, 773, 758, 764, 794, 757, 780, 741, 789, 780, 728, 764, 748, 746, 730, 757, 752, 750, 760, 767, 745, 813, 810, 763, 787, 791, 789, 743, 765, 765, 770, 793, 765, 764, 767, 761, 743, 744, 753, 754, 800, 778, 794] payload_rows_per_dst=[9466, 9390, 9340, 9218, 9130, 9382, 9300, 9282] dup_rows_per_dst=[2854, 2982, 2944, 3048, 2884, 3148, 2964, 2972] combine_rows_per_src=[9466, 9390, 9340, 9218, 9130, 9382, 9300, 9282] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=267755520 dispatch_meta_bytes=343944 combine_payload_bytes=267755520 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=268099464 ingress_bytes=336961536 link_bytes=336961536 token_bytes=535854984 weight_movement_bytes=69206016 total_nvlink_bytes=605061000 weight_to_token_ratio=0.129151 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.267959132e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12261245704 prefetch_bytes=692060160 reduce_bytes=795869184 weight_movement_bytes=1487929344 egress_bytes=6134579976 ingress_bytes=7614595072 link_bytes=7614595072 total_nvlink_bytes=13749175048 weight_to_token_ratio=0.121352 prefetch_share_of_ingress=0.090886 dedup_ratio=0.244073 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=23 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028043 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.245787314e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74462 payload_rows_remote=65148 dup_rows=23842 dup_rows_remote=20874 dedup_ratio=0.242533 router_tokens_per_expert=[767, 741, 747, 789, 750, 800, 723, 762, 751, 753, 754, 742, 782, 739, 786, 834, 759, 733, 745, 736, 760, 779, 812, 748, 765, 752, 746, 787, 798, 775, 797, 754, 724, 715, 750, 820, 781, 749, 807, 800, 804, 732, 775, 767, 767, 752, 783, 740, 776, 777, 763, 759, 780, 786, 778, 736, 774, 803, 746, 767, 809, 813, 768, 785] payload_rows_per_dst=[9154, 9314, 9222, 9376, 9322, 9278, 9318, 9478] dup_rows_per_dst=[3004, 2968, 2922, 2972, 2970, 2962, 2992, 3052] combine_rows_per_src=[9154, 9314, 9222, 9376, 9322, 9278, 9318, 9478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266084352 dispatch_meta_bytes=343944 combine_payload_bytes=266084352 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266428296 ingress_bytes=300687360 link_bytes=300687360 token_bytes=532512648 weight_movement_bytes=34603008 total_nvlink_bytes=567115656 weight_to_token_ratio=0.064981 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131462328e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344864 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266715936 ingress_bytes=300974080 link_bytes=300974080 token_bytes=533087008 weight_movement_bytes=34603008 total_nvlink_bytes=567690016 weight_to_token_ratio=0.064911 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132541233e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74708 payload_rows_remote=65370 dup_rows=23596 dup_rows_remote=20616 dedup_ratio=0.240031 router_tokens_per_expert=[785, 799, 771, 796, 761, 765, 745, 799, 810, 804, 821, 719, 787, 774, 737, 779, 777, 769, 768, 772, 722, 756, 760, 758, 734, 759, 799, 802, 734, 744, 781, 749, 751, 736, 788, 787, 744, 778, 809, 785, 838, 741, 765, 763, 741, 796, 791, 724, 769, 728, 720, 784, 758, 756, 757, 762, 752, 794, 790, 768, 788, 760, 736, 757] payload_rows_per_dst=[9420, 9454, 9186, 9288, 9438, 9338, 9256, 9328] dup_rows_per_dst=[3022, 3008, 2978, 2916, 2918, 2980, 2812, 2962] combine_rows_per_src=[9420, 9454, 9186, 9288, 9438, 9338, 9256, 9328] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265953280 dispatch_meta_bytes=343936 combine_payload_bytes=265953280 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266297216 ingress_bytes=300556288 link_bytes=300556288 token_bytes=532250496 weight_movement_bytes=34603008 total_nvlink_bytes=566853504 weight_to_token_ratio=0.065013 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130969115e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74386 payload_rows_remote=64868 dup_rows=23918 dup_rows_remote=20798 dedup_ratio=0.243306 router_tokens_per_expert=[756, 761, 777, 763, 754, 787, 777, 766, 760, 769, 743, 734, 763, 739, 771, 733, 756, 779, 770, 730, 741, 783, 770, 808, 754, 768, 789, 762, 791, 778, 787, 726, 799, 796, 767, 822, 775, 814, 768, 778, 731, 754, 767, 743, 756, 799, 768, 820, 740, 799, 759, 773, 786, 766, 757, 795, 786, 809, 750, 745, 767, 735, 749, 734] payload_rows_per_dst=[9254, 9122, 9266, 9260, 9518, 9220, 9382, 9364] dup_rows_per_dst=[3028, 2902, 3008, 3050, 3120, 3056, 2968, 2786] combine_rows_per_src=[9254, 9122, 9266, 9260, 9518, 9220, 9382, 9364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12222339232 prefetch_bytes=761266176 reduce_bytes=519045120 weight_movement_bytes=1280311296 egress_bytes=6115113120 ingress_bytes=7387537408 link_bytes=7387537408 total_nvlink_bytes=13502650528 weight_to_token_ratio=0.104752 prefetch_share_of_ingress=0.103047 dedup_ratio=0.244646 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=15 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029022 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.208639501e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12254527248 prefetch_bytes=657457152 reduce_bytes=795869184 weight_movement_bytes=1453326336 egress_bytes=6131220240 ingress_bytes=7576633344 link_bytes=7576633344 total_nvlink_bytes=13707853584 weight_to_token_ratio=0.118595 prefetch_share_of_ingress=0.086774 dedup_ratio=0.244189 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=23 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.239576578e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266846208 dispatch_meta_bytes=344088 combine_payload_bytes=266846208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267190296 ingress_bytes=301449216 link_bytes=301449216 token_bytes=534036504 weight_movement_bytes=34603008 total_nvlink_bytes=568639512 weight_to_token_ratio=0.064795 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134329131e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=267755520 dispatch_meta_bytes=343944 combine_payload_bytes=267755520 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=268099464 ingress_bytes=336961536 link_bytes=336961536 token_bytes=535854984 weight_movement_bytes=69206016 total_nvlink_bytes=605061000 weight_to_token_ratio=0.129151 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.267959132e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12223492848 prefetch_bytes=657457152 reduce_bytes=519045120 weight_movement_bytes=1176502272 egress_bytes=6115693296 ingress_bytes=7284301824 link_bytes=7284646120 total_nvlink_bytes=13399995120 weight_to_token_ratio=0.096249 prefetch_share_of_ingress=0.090257 dedup_ratio=0.244885 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=15 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029239 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.191805952e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74508 payload_rows_remote=65290 dup_rows=23796 dup_rows_remote=20748 dedup_ratio=0.242065 router_tokens_per_expert=[756, 724, 794, 785, 830, 724, 772, 775, 801, 735, 746, 776, 770, 780, 801, 777, 753, 769, 767, 752, 821, 749, 773, 758, 764, 794, 757, 780, 741, 789, 780, 728, 764, 748, 746, 730, 757, 752, 750, 760, 767, 745, 813, 810, 763, 787, 791, 789, 743, 765, 765, 770, 793, 765, 764, 767, 761, 743, 744, 753, 754, 800, 778, 794] payload_rows_per_dst=[9466, 9390, 9340, 9218, 9130, 9382, 9300, 9282] dup_rows_per_dst=[2854, 2982, 2944, 3048, 2884, 3148, 2964, 2972] combine_rows_per_src=[9466, 9390, 9340, 9218, 9130, 9382, 9300, 9282] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265699328 dispatch_meta_bytes=342664 combine_payload_bytes=265699328 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266041992 ingress_bytes=369508352 link_bytes=369508352 token_bytes=531741320 weight_movement_bytes=103809024 total_nvlink_bytes=635550344 weight_to_token_ratio=0.195225 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390430181e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266846208 dispatch_meta_bytes=344360 combine_payload_bytes=266846208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267190568 ingress_bytes=301449216 link_bytes=301449216 token_bytes=534036776 weight_movement_bytes=34603008 total_nvlink_bytes=568639784 weight_to_token_ratio=0.064795 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134329131e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=267427840 dispatch_meta_bytes=344152 combine_payload_bytes=267427840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267771992 ingress_bytes=302030848 link_bytes=302030848 token_bytes=535199832 weight_movement_bytes=34603008 total_nvlink_bytes=569802840 weight_to_token_ratio=0.064654 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136517766e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12259359688 prefetch_bytes=761266176 reduce_bytes=484442112 weight_movement_bytes=1245708288 egress_bytes=6133636040 ingress_bytes=7371431936 link_bytes=7371431936 total_nvlink_bytes=13505067976 weight_to_token_ratio=0.101613 prefetch_share_of_ingress=0.103272 dedup_ratio=0.243881 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=14 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.206004562e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74538 payload_rows_remote=65234 dup_rows=23766 dup_rows_remote=20878 dedup_ratio=0.241760 router_tokens_per_expert=[813, 770, 760, 764, 741, 794, 778, 760, 746, 767, 760, 762, 797, 793, 772, 808, 759, 776, 721, 789, 790, 735, 751, 775, 774, 734, 766, 819, 745, 793, 804, 702, 762, 726, 821, 744, 775, 762, 790, 726, 822, 762, 775, 744, 736, 737, 777, 751, 788, 759, 766, 739, 773, 761, 745, 762, 816, 776, 809, 739, 797, 753, 744, 797] payload_rows_per_dst=[9344, 9396, 9304, 9346, 9314, 9238, 9230, 9366] dup_rows_per_dst=[3016, 3014, 2888, 2928, 2898, 2970, 2956, 3096] combine_rows_per_src=[9344, 9396, 9304, 9346, 9314, 9238, 9230, 9366] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12254462064 prefetch_bytes=622854144 reduce_bytes=830472192 weight_movement_bytes=1453326336 egress_bytes=6131187824 ingress_bytes=7576600576 link_bytes=7576600576 total_nvlink_bytes=13707788400 weight_to_token_ratio=0.118596 prefetch_share_of_ingress=0.082208 dedup_ratio=0.244229 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=24 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.239571217e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12223492848 prefetch_bytes=657457152 reduce_bytes=519045120 weight_movement_bytes=1176502272 egress_bytes=6115693296 ingress_bytes=7284301824 link_bytes=7284646120 total_nvlink_bytes=13399995120 weight_to_token_ratio=0.096249 prefetch_share_of_ingress=0.090257 dedup_ratio=0.244885 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=15 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029239 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.191805952e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12248722656 prefetch_bytes=657457152 reduce_bytes=622854144 weight_movement_bytes=1280311296 egress_bytes=6128315616 ingress_bytes=7400718336 link_bytes=7400718336 total_nvlink_bytes=13529033952 weight_to_token_ratio=0.104526 prefetch_share_of_ingress=0.088837 dedup_ratio=0.244325 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=18 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.210795969e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=267427840 dispatch_meta_bytes=344152 combine_payload_bytes=267427840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267771992 ingress_bytes=302030848 link_bytes=302030848 token_bytes=535199832 weight_movement_bytes=34603008 total_nvlink_bytes=569802840 weight_to_token_ratio=0.064654 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136517766e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=267198464 dispatch_meta_bytes=344448 combine_payload_bytes=267198464 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267542912 ingress_bytes=371007488 link_bytes=371007488 token_bytes=534741376 weight_movement_bytes=103809024 total_nvlink_bytes=638550400 weight_to_token_ratio=0.194129 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396071309e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12257940016 prefetch_bytes=657457152 reduce_bytes=899678208 weight_movement_bytes=1557135360 egress_bytes=6132929072 ingress_bytes=7682146304 link_bytes=7682146304 total_nvlink_bytes=13815075376 weight_to_token_ratio=0.127031 prefetch_share_of_ingress=0.085582 dedup_ratio=0.244379 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=26 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.256839047e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64962 dup_rows=24068 dup_rows_remote=21024 dedup_ratio=0.244832 router_tokens_per_expert=[785, 733, 746, 773, 757, 771, 746, 762, 767, 751, 785, 785, 795, 749, 771, 735, 784, 797, 736, 770, 777, 746, 791, 780, 732, 845, 770, 804, 755, 738, 811, 756, 815, 760, 709, 756, 744, 758, 788, 715, 723, 756, 832, 751, 762, 793, 740, 808, 769, 783, 766, 778, 765, 772, 739, 787, 794, 752, 741, 741, 784, 763, 790, 815] payload_rows_per_dst=[9240, 9266, 9276, 9356, 9196, 9322, 9274, 9306] dup_rows_per_dst=[2906, 3010, 3086, 3066, 2894, 3008, 3044, 3054] combine_rows_per_src=[9240, 9266, 9276, 9356, 9196, 9322, 9274, 9306] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266084352 dispatch_meta_bytes=343944 combine_payload_bytes=266084352 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266428296 ingress_bytes=300687360 link_bytes=300687360 token_bytes=532512648 weight_movement_bytes=34603008 total_nvlink_bytes=567115656 weight_to_token_ratio=0.064981 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131462328e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12254527248 prefetch_bytes=657457152 reduce_bytes=795869184 weight_movement_bytes=1453326336 egress_bytes=6131220240 ingress_bytes=7576633344 link_bytes=7576633344 total_nvlink_bytes=13707853584 weight_to_token_ratio=0.118595 prefetch_share_of_ingress=0.086774 dedup_ratio=0.244189 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=23 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.239576578e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none - [2026-08-26 15:10:24] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 789.4 | throughput per GPU (TFLOP/s/GPU): 161.6 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.785748E+00 | loss scale: 1.0 | grad norm: 4.541 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74164 payload_rows_remote=64990 dup_rows=24140 dup_rows_remote=21106 dedup_ratio=0.245565 router_tokens_per_expert=[797, 755, 767, 753, 767, 729, 753, 788, 770, 792, 733, 776, 791, 744, 725, 773, 772, 799, 733, 708, 789, 736, 778, 749, 763, 758, 741, 816, 775, 747, 789, 733, 783, 778, 751, 810, 782, 763, 769, 791, 767, 760, 749, 796, 750, 775, 769, 771, 797, 775, 798, 781, 750, 783, 773, 841, 779, 725, 789, 736, 763, 748, 714, 837] payload_rows_per_dst=[9222, 9174, 9258, 9204, 9366, 9242, 9520, 9178] dup_rows_per_dst=[2996, 3034, 2870, 3040, 3088, 3032, 3076, 3004] combine_rows_per_src=[9222, 9174, 9258, 9204, 9366, 9242, 9520, 9178] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=64642 dup_rows=24028 dup_rows_remote=20644 dedup_ratio=0.244425 router_tokens_per_expert=[752, 787, 721, 764, 766, 771, 751, 791, 700, 791, 786, 776, 782, 806, 807, 749, 755, 766, 800, 763, 791, 775, 785, 781, 758, 765, 749, 765, 802, 790, 754, 744, 755, 775, 789, 770, 778, 783, 737, 795, 710, 859, 773, 781, 698, 729, 783, 774, 752, 735, 711, 781, 782, 777, 767, 723, 747, 756, 765, 774, 801, 840, 791, 718] payload_rows_per_dst=[9146, 9324, 9498, 8708, 9216, 9526, 9224, 9634] dup_rows_per_dst=[2794, 3010, 3200, 2498, 2972, 3106, 3064, 3384] combine_rows_per_src=[9146, 9324, 9498, 8708, 9216, 9526, 9224, 9634] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=65118 dup_rows=23964 dup_rows_remote=20990 dedup_ratio=0.243774 router_tokens_per_expert=[776, 795, 782, 765, 788, 806, 707, 774, 786, 750, 791, 764, 792, 719, 735, 741, 780, 769, 800, 764, 738, 754, 739, 754, 749, 772, 747, 746, 763, 775, 801, 796, 754, 841, 764, 762, 728, 768, 736, 800, 786, 788, 757, 783, 769, 759, 758, 798, 794, 737, 779, 748, 766, 759, 735, 785, 767, 791, 744, 788, 770, 796, 746, 778] payload_rows_per_dst=[9362, 9190, 9222, 9392, 9340, 9310, 9166, 9358] dup_rows_per_dst=[3024, 2966, 2974, 2906, 2966, 3086, 3040, 3002] combine_rows_per_src=[9362, 9190, 9222, 9392, 9340, 9310, 9166, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266723328 dispatch_meta_bytes=344432 combine_payload_bytes=266723328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267067760 ingress_bytes=301326336 link_bytes=301326336 token_bytes=533791088 weight_movement_bytes=34603008 total_nvlink_bytes=568394096 weight_to_token_ratio=0.064825 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133866744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12261245704 prefetch_bytes=692060160 reduce_bytes=795869184 weight_movement_bytes=1487929344 egress_bytes=6134579976 ingress_bytes=7614595072 link_bytes=7614595072 total_nvlink_bytes=13749175048 weight_to_token_ratio=0.121352 prefetch_share_of_ingress=0.090886 dedup_ratio=0.244073 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=23 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028043 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.245787314e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none + [2026-08-26 15:58:13] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 794.8 | throughput per GPU (TFLOP/s/GPU): 160.5 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.785748E+00 | loss scale: 1.0 | grad norm: 4.541 | number of skipped iterations: 0 | number of nan iterations: 0 | WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74274 payload_rows_remote=65028 dup_rows=24030 dup_rows_remote=21012 dedup_ratio=0.244446 router_tokens_per_expert=[761, 767, 738, 748, 760, 730, 747, 800, 844, 811, 784, 805, 728, 755, 756, 776, 749, 715, 760, 770, 761, 756, 773, 787, 803, 790, 830, 752, 765, 765, 777, 773, 798, 794, 737, 733, 778, 787, 832, 788, 733, 803, 770, 789, 756, 733, 801, 747, 728, 780, 761, 761, 737, 740, 776, 728, 755, 803, 794, 726, 747, 773, 771, 757] payload_rows_per_dst=[9250, 9384, 9262, 9424, 9282, 9246, 9134, 9292] dup_rows_per_dst=[2852, 3134, 2880, 3086, 3212, 3018, 2888, 2960] combine_rows_per_src=[9250, 9384, 9262, 9424, 9282, 9246, 9134, 9292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=64642 dup_rows=24028 dup_rows_remote=20644 dedup_ratio=0.244425 router_tokens_per_expert=[752, 787, 721, 764, 766, 771, 751, 791, 700, 791, 786, 776, 782, 806, 807, 749, 755, 766, 800, 763, 791, 775, 785, 781, 758, 765, 749, 765, 802, 790, 754, 744, 755, 775, 789, 770, 778, 783, 737, 795, 710, 859, 773, 781, 698, 729, 783, 774, 752, 735, 711, 781, 782, 777, 767, 723, 747, 756, 765, 774, 801, 840, 791, 718] payload_rows_per_dst=[9146, 9324, 9498, 8708, 9216, 9526, 9224, 9634] dup_rows_per_dst=[2794, 3010, 3200, 2498, 2972, 3106, 3064, 3384] combine_rows_per_src=[9146, 9324, 9498, 8708, 9216, 9526, 9224, 9634] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74494 payload_rows_remote=65166 dup_rows=23810 dup_rows_remote=20706 dedup_ratio=0.242208 router_tokens_per_expert=[798, 794, 761, 726, 797, 753, 775, 790, 766, 795, 743, 753, 735, 791, 770, 724, 769, 766, 718, 841, 779, 794, 762, 726, 770, 800, 818, 781, 763, 777, 819, 752, 751, 814, 748, 765, 766, 732, 774, 738, 746, 780, 752, 750, 720, 774, 780, 806, 759, 802, 790, 762, 780, 760, 763, 730, 757, 765, 744, 741, 795, 802, 761, 739] payload_rows_per_dst=[9328, 9262, 9308, 9464, 9174, 9326, 9332, 9300] dup_rows_per_dst=[3104, 2848, 3002, 3096, 3002, 2890, 2960, 2908] combine_rows_per_src=[9328, 9262, 9308, 9464, 9174, 9326, 9332, 9300] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=64882 dup_rows=24046 dup_rows_remote=20978 dedup_ratio=0.244609 router_tokens_per_expert=[747, 773, 731, 804, 795, 791, 819, 815, 778, 784, 718, 772, 756, 826, 714, 799, 780, 760, 761, 731, 770, 764, 783, 774, 784, 738, 774, 819, 779, 796, 723, 809, 783, 750, 792, 760, 770, 751, 757, 800, 749, 793, 766, 757, 789, 785, 732, 740, 762, 775, 758, 761, 743, 798, 744, 782, 753, 757, 710, 773, 725, 750, 784, 736] payload_rows_per_dst=[9410, 9234, 9248, 9376, 9264, 9228, 9324, 9174] dup_rows_per_dst=[3140, 3060, 2998, 3068, 3062, 2994, 2922, 2802] combine_rows_per_src=[9410, 9234, 9248, 9376, 9264, 9228, 9324, 9174] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74104 payload_rows_remote=64790 dup_rows=24200 dup_rows_remote=21312 dedup_ratio=0.246175 router_tokens_per_expert=[810, 806, 755, 824, 777, 767, 745, 843, 770, 766, 799, 789, 815, 705, 756, 778, 775, 754, 767, 768, 740, 761, 802, 734, 759, 801, 724, 810, 759, 777, 766, 778, 762, 794, 761, 784, 790, 750, 759, 724, 752, 804, 771, 775, 778, 728, 711, 824, 794, 737, 774, 746, 821, 731, 766, 727, 755, 754, 759, 718, 784, 750, 766, 723] payload_rows_per_dst=[9466, 9228, 9314, 9314, 9192, 9260, 9226, 9104] dup_rows_per_dst=[3188, 3128, 2888, 3034, 3056, 3026, 2966, 2914] combine_rows_per_src=[9466, 9228, 9314, 9314, 9192, 9260, 9226, 9104] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74494 payload_rows_remote=65166 dup_rows=23810 dup_rows_remote=20706 dedup_ratio=0.242208 router_tokens_per_expert=[798, 794, 761, 726, 797, 753, 775, 790, 766, 795, 743, 753, 735, 791, 770, 724, 769, 766, 718, 841, 779, 794, 762, 726, 770, 800, 818, 781, 763, 777, 819, 752, 751, 814, 748, 765, 766, 732, 774, 738, 746, 780, 752, 750, 720, 774, 780, 806, 759, 802, 790, 762, 780, 760, 763, 730, 757, 765, 744, 741, 795, 802, 761, 739] payload_rows_per_dst=[9328, 9262, 9308, 9464, 9174, 9326, 9332, 9300] dup_rows_per_dst=[3104, 2848, 3002, 3096, 3002, 2890, 2960, 2908] combine_rows_per_src=[9328, 9262, 9308, 9464, 9174, 9326, 9332, 9300] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=344384 combine_payload_bytes=266199040 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266543424 ingress_bytes=370008064 link_bytes=370008064 token_bytes=532742464 weight_movement_bytes=103809024 total_nvlink_bytes=636551488 weight_to_token_ratio=0.194858 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392310557e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74402 payload_rows_remote=65050 dup_rows=23902 dup_rows_remote=20976 dedup_ratio=0.243144 router_tokens_per_expert=[680, 768, 785, 754, 783, 788, 765, 710, 758, 756, 804, 726, 772, 811, 742, 753, 802, 726, 746, 754, 767, 749, 760, 770, 756, 785, 842, 815, 821, 747, 762, 805, 741, 732, 785, 771, 772, 775, 749, 781, 783, 755, 803, 803, 740, 741, 816, 736, 799, 750, 761, 723, 797, 802, 761, 746, 775, 792, 744, 808, 779, 711, 749, 810] payload_rows_per_dst=[9218, 9302, 9274, 9484, 9228, 9358, 9352, 9186] dup_rows_per_dst=[2848, 2942, 2874, 3182, 2984, 2996, 2926, 3150] combine_rows_per_src=[9218, 9302, 9274, 9484, 9228, 9358, 9352, 9186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74164 payload_rows_remote=64990 dup_rows=24140 dup_rows_remote=21106 dedup_ratio=0.245565 router_tokens_per_expert=[797, 755, 767, 753, 767, 729, 753, 788, 770, 792, 733, 776, 791, 744, 725, 773, 772, 799, 733, 708, 789, 736, 778, 749, 763, 758, 741, 816, 775, 747, 789, 733, 783, 778, 751, 810, 782, 763, 769, 791, 767, 760, 749, 796, 750, 775, 769, 771, 797, 775, 798, 781, 750, 783, 773, 841, 779, 725, 789, 736, 763, 748, 714, 837] payload_rows_per_dst=[9222, 9174, 9258, 9204, 9366, 9242, 9520, 9178] dup_rows_per_dst=[2996, 3034, 2870, 3040, 3088, 3032, 3076, 3004] combine_rows_per_src=[9222, 9174, 9258, 9204, 9366, 9242, 9520, 9178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=264773632 dispatch_meta_bytes=341144 combine_payload_bytes=264773632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265114776 ingress_bytes=299376640 link_bytes=299376640 token_bytes=529888408 weight_movement_bytes=34603008 total_nvlink_bytes=564491416 weight_to_token_ratio=0.065302 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126530195e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266354688 dispatch_meta_bytes=344160 combine_payload_bytes=266354688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266698848 ingress_bytes=300957696 link_bytes=300957696 token_bytes=533053536 weight_movement_bytes=34603008 total_nvlink_bytes=567656544 weight_to_token_ratio=0.064915 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132479581e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266354688 dispatch_meta_bytes=344160 combine_payload_bytes=266354688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266698848 ingress_bytes=300957696 link_bytes=300957696 token_bytes=533053536 weight_movement_bytes=34603008 total_nvlink_bytes=567656544 weight_to_token_ratio=0.064915 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132479581e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=264773632 dispatch_meta_bytes=341144 combine_payload_bytes=264773632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265114776 ingress_bytes=299376640 link_bytes=299376640 token_bytes=529888408 weight_movement_bytes=34603008 total_nvlink_bytes=564491416 weight_to_token_ratio=0.065302 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126530195e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266919936 dispatch_meta_bytes=343488 combine_payload_bytes=266919936 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267263424 ingress_bytes=336125952 link_bytes=336125952 token_bytes=534183360 weight_movement_bytes=69206016 total_nvlink_bytes=603389376 weight_to_token_ratio=0.129555 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264814897e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=343440 combine_payload_bytes=265756672 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266100112 ingress_bytes=369565696 link_bytes=369565696 token_bytes=531856784 weight_movement_bytes=103809024 total_nvlink_bytes=635665808 weight_to_token_ratio=0.195182 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390645962e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74522 payload_rows_remote=65282 dup_rows=23782 dup_rows_remote=20816 dedup_ratio=0.241923 router_tokens_per_expert=[746, 814, 748, 794, 767, 749, 779, 774, 755, 768, 736, 807, 762, 743, 795, 754, 792, 787, 778, 751, 808, 792, 746, 761, 755, 794, 757, 745, 812, 771, 785, 744, 808, 804, 718, 734, 777, 760, 707, 795, 774, 766, 758, 765, 723, 714, 764, 764, 761, 796, 761, 772, 736, 797, 805, 767, 760, 772, 768, 801, 761, 756, 797, 742] payload_rows_per_dst=[9386, 9304, 9422, 9374, 9240, 9088, 9382, 9326] dup_rows_per_dst=[2956, 2936, 3008, 2952, 2966, 2968, 3008, 2988] combine_rows_per_src=[9386, 9304, 9422, 9374, 9240, 9088, 9382, 9326] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265379840 dispatch_meta_bytes=344408 combine_payload_bytes=265379840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265724248 ingress_bytes=299982848 link_bytes=299982848 token_bytes=531104088 weight_movement_bytes=34603008 total_nvlink_bytes=565707096 weight_to_token_ratio=0.065153 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128811306e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266919936 dispatch_meta_bytes=343488 combine_payload_bytes=266919936 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267263424 ingress_bytes=336125952 link_bytes=336125952 token_bytes=534183360 weight_movement_bytes=69206016 total_nvlink_bytes=603389376 weight_to_token_ratio=0.129555 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264814897e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74402 payload_rows_remote=65050 dup_rows=23902 dup_rows_remote=20976 dedup_ratio=0.243144 router_tokens_per_expert=[680, 768, 785, 754, 783, 788, 765, 710, 758, 756, 804, 726, 772, 811, 742, 753, 802, 726, 746, 754, 767, 749, 760, 770, 756, 785, 842, 815, 821, 747, 762, 805, 741, 732, 785, 771, 772, 775, 749, 781, 783, 755, 803, 803, 740, 741, 816, 736, 799, 750, 761, 723, 797, 802, 761, 746, 775, 792, 744, 808, 779, 711, 749, 810] payload_rows_per_dst=[9218, 9302, 9274, 9484, 9228, 9358, 9352, 9186] dup_rows_per_dst=[2848, 2942, 2874, 3182, 2984, 2996, 2926, 3150] combine_rows_per_src=[9218, 9302, 9274, 9484, 9228, 9358, 9352, 9186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266444800 dispatch_meta_bytes=344104 combine_payload_bytes=266444800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266788904 ingress_bytes=301047808 link_bytes=301047808 token_bytes=533233704 weight_movement_bytes=34603008 total_nvlink_bytes=567836712 weight_to_token_ratio=0.064893 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132818665e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=344384 combine_payload_bytes=266199040 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266543424 ingress_bytes=370008064 link_bytes=370008064 token_bytes=532742464 weight_movement_bytes=103809024 total_nvlink_bytes=636551488 weight_to_token_ratio=0.194858 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392310557e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74522 payload_rows_remote=65282 dup_rows=23782 dup_rows_remote=20816 dedup_ratio=0.241923 router_tokens_per_expert=[746, 814, 748, 794, 767, 749, 779, 774, 755, 768, 736, 807, 762, 743, 795, 754, 792, 787, 778, 751, 808, 792, 746, 761, 755, 794, 757, 745, 812, 771, 785, 744, 808, 804, 718, 734, 777, 760, 707, 795, 774, 766, 758, 765, 723, 714, 764, 764, 761, 796, 761, 772, 736, 797, 805, 767, 760, 772, 768, 801, 761, 756, 797, 742] payload_rows_per_dst=[9386, 9304, 9422, 9374, 9240, 9088, 9382, 9326] dup_rows_per_dst=[2956, 2936, 3008, 2952, 2966, 2968, 3008, 2988] combine_rows_per_src=[9386, 9304, 9422, 9374, 9240, 9088, 9382, 9326] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74108 payload_rows_remote=64722 dup_rows=24196 dup_rows_remote=21152 dedup_ratio=0.246134 router_tokens_per_expert=[784, 804, 784, 780, 767, 732, 730, 770, 802, 770, 784, 758, 815, 751, 781, 754, 818, 776, 714, 808, 757, 768, 773, 776, 757, 778, 759, 767, 747, 757, 710, 758, 735, 768, 744, 758, 799, 798, 799, 747, 759, 785, 793, 772, 807, 765, 798, 803, 779, 769, 770, 765, 731, 727, 742, 753, 795, 796, 771, 742, 698, 759, 754, 782] payload_rows_per_dst=[9276, 9386, 9286, 9084, 9344, 9340, 9138, 9254] dup_rows_per_dst=[3026, 3044, 3094, 2982, 2952, 3224, 2934, 2940] combine_rows_per_src=[9276, 9386, 9286, 9084, 9344, 9340, 9138, 9254] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74362 payload_rows_remote=65002 dup_rows=23942 dup_rows_remote=20890 dedup_ratio=0.243551 router_tokens_per_expert=[774, 803, 772, 757, 789, 778, 813, 813, 782, 755, 773, 777, 767, 777, 736, 784, 760, 714, 798, 789, 749, 761, 779, 747, 739, 810, 773, 732, 794, 785, 770, 787, 753, 783, 750, 783, 748, 751, 758, 815, 728, 764, 798, 754, 728, 742, 755, 745, 765, 763, 777, 736, 775, 761, 779, 778, 799, 734, 799, 765, 729, 773, 787, 740] payload_rows_per_dst=[9010, 9154, 9198, 9444, 9374, 9278, 9544, 9360] dup_rows_per_dst=[2946, 2888, 2894, 2936, 2960, 2900, 3366, 3052] combine_rows_per_src=[9010, 9154, 9198, 9444, 9374, 9278, 9544, 9360] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74064 payload_rows_remote=64950 dup_rows=24240 dup_rows_remote=21324 dedup_ratio=0.246582 router_tokens_per_expert=[790, 784, 785, 728, 812, 685, 766, 776, 774, 789, 810, 742, 781, 735, 757, 780, 795, 723, 782, 818, 821, 756, 786, 792, 751, 716, 783, 804, 731, 747, 790, 819, 753, 811, 774, 755, 783, 746, 772, 798, 772, 755, 737, 751, 734, 787, 756, 723, 758, 739, 739, 773, 803, 770, 774, 781, 755, 761, 782, 742, 768, 760, 782, 750] payload_rows_per_dst=[9240, 9258, 9364, 9268, 9356, 9114, 9266, 9198] dup_rows_per_dst=[3012, 3078, 3182, 3014, 3028, 2916, 3008, 3002] combine_rows_per_src=[9240, 9258, 9364, 9268, 9356, 9114, 9266, 9198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74362 payload_rows_remote=65002 dup_rows=23942 dup_rows_remote=20890 dedup_ratio=0.243551 router_tokens_per_expert=[774, 803, 772, 757, 789, 778, 813, 813, 782, 755, 773, 777, 767, 777, 736, 784, 760, 714, 798, 789, 749, 761, 779, 747, 739, 810, 773, 732, 794, 785, 770, 787, 753, 783, 750, 783, 748, 751, 758, 815, 728, 764, 798, 754, 728, 742, 755, 745, 765, 763, 777, 736, 775, 761, 779, 778, 799, 734, 799, 765, 729, 773, 787, 740] payload_rows_per_dst=[9010, 9154, 9198, 9444, 9374, 9278, 9544, 9360] dup_rows_per_dst=[2946, 2888, 2894, 2936, 2960, 2900, 3366, 3052] combine_rows_per_src=[9010, 9154, 9198, 9444, 9374, 9278, 9544, 9360] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267395072 dispatch_meta_bytes=344392 combine_payload_bytes=267395072 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267739464 ingress_bytes=336601088 link_bytes=336601088 token_bytes=535134536 weight_movement_bytes=69206016 total_nvlink_bytes=604340552 weight_to_token_ratio=0.129325 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266602796e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=65284 dup_rows=23920 dup_rows_remote=20876 dedup_ratio=0.243327 router_tokens_per_expert=[782, 760, 743, 764, 738, 753, 765, 766, 765, 802, 766, 795, 790, 721, 772, 766, 748, 793, 795, 817, 742, 791, 768, 753, 757, 753, 741, 760, 754, 798, 792, 717, 744, 796, 766, 763, 732, 778, 795, 761, 729, 729, 761, 761, 797, 785, 775, 813, 774, 715, 788, 786, 817, 765, 783, 762, 755, 810, 767, 799, 756, 809, 725, 729] payload_rows_per_dst=[9190, 9364, 9420, 9100, 9346, 9338, 9352, 9274] dup_rows_per_dst=[2952, 2990, 2994, 3044, 2924, 2962, 3028, 3026] combine_rows_per_src=[9190, 9364, 9420, 9100, 9346, 9338, 9352, 9274] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265101312 dispatch_meta_bytes=343496 combine_payload_bytes=265101312 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265444808 ingress_bytes=368910336 link_bytes=368910336 token_bytes=530546120 weight_movement_bytes=103809024 total_nvlink_bytes=634355144 weight_to_token_ratio=0.195664 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388179895e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=343568 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266591760 ingress_bytes=300851200 link_bytes=300851200 token_bytes=532839952 weight_movement_bytes=34603008 total_nvlink_bytes=567442960 weight_to_token_ratio=0.064941 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132078845e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74442 payload_rows_remote=65258 dup_rows=23862 dup_rows_remote=20816 dedup_ratio=0.242737 router_tokens_per_expert=[790, 773, 745, 838, 767, 736, 768, 741, 743, 790, 721, 763, 782, 777, 771, 784, 724, 790, 773, 737, 799, 755, 759, 757, 779, 792, 759, 757, 739, 782, 794, 746, 784, 812, 757, 766, 765, 804, 758, 794, 757, 753, 733, 729, 758, 773, 776, 774, 761, 748, 797, 795, 794, 736, 742, 742, 811, 736, 794, 773, 777, 827, 761, 734] payload_rows_per_dst=[9324, 9332, 9188, 9320, 9438, 9210, 9184, 9446] dup_rows_per_dst=[2992, 2930, 3000, 2976, 3042, 2896, 3046, 2980] combine_rows_per_src=[9324, 9332, 9188, 9320, 9438, 9210, 9184, 9446] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74370 payload_rows_remote=65040 dup_rows=23934 dup_rows_remote=20908 dedup_ratio=0.243469 router_tokens_per_expert=[725, 782, 789, 797, 755, 779, 728, 784, 782, 746, 790, 749, 771, 784, 784, 780, 759, 838, 736, 781, 763, 718, 804, 779, 757, 742, 752, 776, 797, 727, 768, 764, 754, 756, 785, 771, 749, 759, 745, 788, 721, 787, 823, 768, 798, 807, 759, 792, 757, 731, 741, 754, 810, 786, 757, 766, 777, 768, 731, 760, 785, 773, 769, 739] payload_rows_per_dst=[9170, 9360, 9330, 9250, 9280, 9362, 9334, 9284] dup_rows_per_dst=[3108, 3012, 3026, 2916, 2934, 3148, 2870, 2920] combine_rows_per_src=[9170, 9360, 9330, 9250, 9280, 9362, 9334, 9284] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266444800 dispatch_meta_bytes=344104 combine_payload_bytes=266444800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266788904 ingress_bytes=301047808 link_bytes=301047808 token_bytes=533233704 weight_movement_bytes=34603008 total_nvlink_bytes=567836712 weight_to_token_ratio=0.064893 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132818665e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=65284 dup_rows=23920 dup_rows_remote=20876 dedup_ratio=0.243327 router_tokens_per_expert=[782, 760, 743, 764, 738, 753, 765, 766, 765, 802, 766, 795, 790, 721, 772, 766, 748, 793, 795, 817, 742, 791, 768, 753, 757, 753, 741, 760, 754, 798, 792, 717, 744, 796, 766, 763, 732, 778, 795, 761, 729, 729, 761, 761, 797, 785, 775, 813, 774, 715, 788, 786, 817, 765, 783, 762, 755, 810, 767, 799, 756, 809, 725, 729] payload_rows_per_dst=[9190, 9364, 9420, 9100, 9346, 9338, 9352, 9274] dup_rows_per_dst=[2952, 2990, 2994, 3044, 2924, 2962, 3028, 3026] combine_rows_per_src=[9190, 9364, 9420, 9100, 9346, 9338, 9352, 9274] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74404 payload_rows_remote=64964 dup_rows=23900 dup_rows_remote=20798 dedup_ratio=0.243123 router_tokens_per_expert=[791, 774, 766, 731, 807, 802, 770, 714, 769, 761, 856, 748, 788, 759, 767, 743, 780, 774, 767, 735, 788, 798, 726, 753, 747, 758, 744, 767, 735, 754, 806, 793, 792, 782, 811, 774, 776, 754, 773, 771, 771, 759, 784, 733, 761, 778, 793, 778, 731, 751, 773, 786, 729, 765, 719, 788, 758, 748, 762, 816, 748, 758, 762, 797] payload_rows_per_dst=[9440, 9242, 9150, 9420, 9168, 9430, 9182, 9372] dup_rows_per_dst=[3102, 3140, 2860, 3202, 2884, 2884, 2902, 2926] combine_rows_per_src=[9440, 9242, 9150, 9420, 9168, 9430, 9182, 9372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74108 payload_rows_remote=64722 dup_rows=24196 dup_rows_remote=21152 dedup_ratio=0.246134 router_tokens_per_expert=[784, 804, 784, 780, 767, 732, 730, 770, 802, 770, 784, 758, 815, 751, 781, 754, 818, 776, 714, 808, 757, 768, 773, 776, 757, 778, 759, 767, 747, 757, 710, 758, 735, 768, 744, 758, 799, 798, 799, 747, 759, 785, 793, 772, 807, 765, 798, 803, 779, 769, 770, 765, 731, 727, 742, 753, 795, 796, 771, 742, 698, 759, 754, 782] payload_rows_per_dst=[9276, 9386, 9286, 9084, 9344, 9340, 9138, 9254] dup_rows_per_dst=[3026, 3044, 3094, 2982, 2952, 3224, 2934, 2940] combine_rows_per_src=[9276, 9386, 9286, 9084, 9344, 9340, 9138, 9254] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266035200 dispatch_meta_bytes=345096 combine_payload_bytes=266035200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266380296 ingress_bytes=300638208 link_bytes=300638208 token_bytes=532415496 weight_movement_bytes=34603008 total_nvlink_bytes=567018504 weight_to_token_ratio=0.064992 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131277373e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=343568 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266591760 ingress_bytes=300851200 link_bytes=300851200 token_bytes=532839952 weight_movement_bytes=34603008 total_nvlink_bytes=567442960 weight_to_token_ratio=0.064941 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132078845e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267296768 dispatch_meta_bytes=344296 combine_payload_bytes=267296768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267641064 ingress_bytes=301899776 link_bytes=301899776 token_bytes=534937832 weight_movement_bytes=34603008 total_nvlink_bytes=569540840 weight_to_token_ratio=0.064686 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136024552e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267403264 dispatch_meta_bytes=344640 combine_payload_bytes=267403264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267747904 ingress_bytes=302006272 link_bytes=302006272 token_bytes=535151168 weight_movement_bytes=34603008 total_nvlink_bytes=569754176 weight_to_token_ratio=0.064660 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136425288e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266403840 dispatch_meta_bytes=343792 combine_payload_bytes=266403840 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266747632 ingress_bytes=370212864 link_bytes=370212864 token_bytes=533151472 weight_movement_bytes=103809024 total_nvlink_bytes=636960496 weight_to_token_ratio=0.194708 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393081203e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=343048 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266435592 ingress_bytes=335298560 link_bytes=335298560 token_bytes=532528136 weight_movement_bytes=69206016 total_nvlink_bytes=601734152 weight_to_token_ratio=0.129957 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261701488e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265101312 dispatch_meta_bytes=343496 combine_payload_bytes=265101312 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265444808 ingress_bytes=368910336 link_bytes=368910336 token_bytes=530546120 weight_movement_bytes=103809024 total_nvlink_bytes=634355144 weight_to_token_ratio=0.195664 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388179895e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74170 payload_rows_remote=64928 dup_rows=24134 dup_rows_remote=21102 dedup_ratio=0.245504 router_tokens_per_expert=[764, 801, 768, 762, 840, 756, 794, 773, 731, 750, 745, 800, 764, 742, 765, 766, 770, 727, 764, 787, 822, 774, 734, 781, 813, 753, 788, 742, 775, 745, 745, 813, 745, 809, 735, 794, 782, 742, 773, 757, 829, 786, 752, 753, 764, 777, 720, 770, 747, 744, 721, 745, 788, 781, 794, 755, 772, 748, 761, 795, 796, 763, 746, 754] payload_rows_per_dst=[9326, 9182, 9318, 9424, 9242, 9330, 9116, 9232] dup_rows_per_dst=[3190, 2944, 3000, 2924, 3032, 2972, 3034, 3038] combine_rows_per_src=[9326, 9182, 9318, 9424, 9242, 9330, 9116, 9232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74026 payload_rows_remote=64814 dup_rows=24278 dup_rows_remote=21306 dedup_ratio=0.246969 router_tokens_per_expert=[792, 769, 768, 739, 830, 787, 793, 791, 742, 749, 782, 765, 821, 749, 798, 741, 798, 746, 777, 740, 734, 778, 777, 744, 782, 778, 784, 720, 788, 694, 751, 752, 739, 785, 739, 742, 801, 751, 790, 775, 753, 752, 769, 773, 747, 783, 776, 739, 773, 808, 775, 774, 738, 804, 762, 776, 803, 738, 777, 779, 778, 772, 756, 766] payload_rows_per_dst=[9392, 9228, 9194, 9116, 9274, 9212, 9320, 9290] dup_rows_per_dst=[3146, 3066, 2994, 2982, 2970, 2972, 3100, 3048] combine_rows_per_src=[9392, 9228, 9194, 9116, 9274, 9212, 9320, 9290] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74040 payload_rows_remote=64662 dup_rows=24264 dup_rows_remote=21060 dedup_ratio=0.246826 router_tokens_per_expert=[765, 806, 739, 745, 819, 746, 755, 765, 753, 775, 752, 773, 707, 780, 802, 788, 741, 777, 769, 815, 800, 744, 772, 769, 805, 758, 754, 779, 799, 762, 767, 774, 746, 755, 813, 754, 769, 774, 806, 729, 748, 759, 752, 738, 764, 772, 756, 773, 746, 754, 785, 756, 764, 762, 796, 799, 766, 730, 715, 789, 744, 788, 813, 782] payload_rows_per_dst=[9028, 9222, 9172, 9150, 9230, 9180, 9680, 9378] dup_rows_per_dst=[2902, 3038, 2914, 2880, 3000, 3028, 3298, 3204] combine_rows_per_src=[9028, 9222, 9172, 9150, 9230, 9180, 9680, 9378] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74318 payload_rows_remote=65040 dup_rows=23986 dup_rows_remote=20964 dedup_ratio=0.243998 router_tokens_per_expert=[766, 815, 767, 774, 766, 764, 772, 773, 790, 760, 801, 765, 713, 822, 727, 772, 722, 798, 715, 765, 775, 773, 766, 826, 733, 756, 786, 754, 817, 766, 747, 774, 773, 775, 773, 776, 728, 849, 799, 766, 772, 783, 767, 798, 758, 741, 754, 777, 774, 806, 801, 744, 727, 768, 763, 736, 785, 780, 710, 764, 703, 747, 771, 764] payload_rows_per_dst=[9376, 9278, 9314, 9178, 9364, 9288, 9296, 9224] dup_rows_per_dst=[3018, 3022, 2966, 3088, 3114, 3012, 2942, 2824] combine_rows_per_src=[9376, 9278, 9314, 9178, 9364, 9288, 9296, 9224] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265945088 dispatch_meta_bytes=344120 combine_payload_bytes=265945088 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266289208 ingress_bytes=335151104 link_bytes=335151104 token_bytes=532234296 weight_movement_bytes=69206016 total_nvlink_bytes=601440312 weight_to_token_ratio=0.130029 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261146622e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74026 payload_rows_remote=64814 dup_rows=24278 dup_rows_remote=21306 dedup_ratio=0.246969 router_tokens_per_expert=[792, 769, 768, 739, 830, 787, 793, 791, 742, 749, 782, 765, 821, 749, 798, 741, 798, 746, 777, 740, 734, 778, 777, 744, 782, 778, 784, 720, 788, 694, 751, 752, 739, 785, 739, 742, 801, 751, 790, 775, 753, 752, 769, 773, 747, 783, 776, 739, 773, 808, 775, 774, 738, 804, 762, 776, 803, 738, 777, 779, 778, 772, 756, 766] payload_rows_per_dst=[9392, 9228, 9194, 9116, 9274, 9212, 9320, 9290] dup_rows_per_dst=[3146, 3066, 2994, 2982, 2970, 2972, 3100, 3048] combine_rows_per_src=[9392, 9228, 9194, 9116, 9274, 9212, 9320, 9290] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=264855552 dispatch_meta_bytes=342888 combine_payload_bytes=264855552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265198440 ingress_bytes=299458560 link_bytes=299458560 token_bytes=530053992 weight_movement_bytes=34603008 total_nvlink_bytes=564657000 weight_to_token_ratio=0.065282 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126838453e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74442 payload_rows_remote=65258 dup_rows=23862 dup_rows_remote=20816 dedup_ratio=0.242737 router_tokens_per_expert=[790, 773, 745, 838, 767, 736, 768, 741, 743, 790, 721, 763, 782, 777, 771, 784, 724, 790, 773, 737, 799, 755, 759, 757, 779, 792, 759, 757, 739, 782, 794, 746, 784, 812, 757, 766, 765, 804, 758, 794, 757, 753, 733, 729, 758, 773, 776, 774, 761, 748, 797, 795, 794, 736, 742, 742, 811, 736, 794, 773, 777, 827, 761, 734] payload_rows_per_dst=[9324, 9332, 9188, 9320, 9438, 9210, 9184, 9446] dup_rows_per_dst=[2992, 2930, 3000, 2976, 3042, 2896, 3046, 2980] combine_rows_per_src=[9324, 9332, 9188, 9320, 9438, 9210, 9184, 9446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74212 payload_rows_remote=65080 dup_rows=24092 dup_rows_remote=21234 dedup_ratio=0.245076 router_tokens_per_expert=[734, 750, 805, 784, 787, 783, 815, 748, 764, 751, 793, 767, 789, 774, 707, 742, 792, 731, 765, 768, 768, 825, 728, 747, 749, 777, 749, 789, 778, 775, 803, 776, 736, 771, 779, 790, 757, 758, 756, 785, 774, 821, 760, 772, 759, 735, 764, 798, 727, 753, 703, 799, 728, 788, 777, 720, 831, 769, 743, 785, 732, 755, 823, 791] payload_rows_per_dst=[9306, 9268, 9208, 9400, 9306, 9274, 9132, 9318] dup_rows_per_dst=[3106, 2906, 3040, 2992, 2958, 3092, 2858, 3140] combine_rows_per_src=[9306, 9268, 9208, 9400, 9306, 9274, 9132, 9318] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74204 payload_rows_remote=64820 dup_rows=24100 dup_rows_remote=21162 dedup_ratio=0.245158 router_tokens_per_expert=[752, 748, 790, 829, 758, 791, 751, 820, 780, 778, 739, 789, 740, 746, 766, 754, 810, 753, 725, 762, 763, 814, 774, 819, 755, 759, 759, 777, 759, 798, 783, 771, 737, 810, 757, 762, 760, 722, 762, 764, 770, 798, 758, 800, 751, 778, 815, 724, 796, 745, 737, 789, 819, 746, 760, 752, 724, 741, 755, 788, 725, 756, 801, 738] payload_rows_per_dst=[9348, 9270, 9356, 9384, 9202, 9294, 9256, 9094] dup_rows_per_dst=[3130, 2914, 3084, 2938, 2946, 3094, 3032, 2962] combine_rows_per_src=[9348, 9270, 9356, 9384, 9202, 9294, 9256, 9094] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265945088 dispatch_meta_bytes=344120 combine_payload_bytes=265945088 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266289208 ingress_bytes=335151104 link_bytes=335151104 token_bytes=532234296 weight_movement_bytes=69206016 total_nvlink_bytes=601440312 weight_to_token_ratio=0.130029 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261146622e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74408 payload_rows_remote=65204 dup_rows=23896 dup_rows_remote=20930 dedup_ratio=0.243083 router_tokens_per_expert=[753, 778, 763, 750, 747, 797, 764, 733, 742, 791, 726, 772, 740, 787, 746, 762, 773, 797, 793, 790, 788, 737, 757, 750, 783, 755, 783, 743, 772, 804, 752, 811, 786, 853, 743, 732, 765, 737, 731, 793, 764, 780, 753, 811, 838, 823, 794, 776, 737, 762, 722, 758, 737, 801, 743, 812, 714, 791, 781, 768, 736, 739, 750, 783] payload_rows_per_dst=[9204, 9374, 9434, 9426, 9242, 9370, 9114, 9244] dup_rows_per_dst=[2966, 3050, 3036, 2980, 2938, 3016, 3030, 2880] combine_rows_per_src=[9204, 9374, 9434, 9426, 9242, 9370, 9114, 9244] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=65058 dup_rows=23940 dup_rows_remote=21044 dedup_ratio=0.243530 router_tokens_per_expert=[730, 785, 777, 763, 746, 771, 768, 778, 774, 771, 763, 767, 761, 746, 792, 776, 749, 745, 774, 778, 761, 744, 774, 776, 733, 774, 766, 755, 751, 831, 782, 797, 750, 727, 762, 783, 783, 802, 790, 801, 783, 735, 769, 791, 786, 737, 776, 769, 764, 750, 720, 783, 733, 757, 765, 785, 786, 736, 759, 766, 770, 827, 762, 787] payload_rows_per_dst=[9288, 9262, 9306, 9232, 9348, 9358, 9234, 9336] dup_rows_per_dst=[2948, 3038, 2896, 3146, 3048, 2934, 2880, 3050] combine_rows_per_src=[9288, 9262, 9306, 9232, 9348, 9358, 9234, 9336] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266403840 dispatch_meta_bytes=344016 combine_payload_bytes=266403840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266747856 ingress_bytes=301006848 link_bytes=301006848 token_bytes=533151696 weight_movement_bytes=34603008 total_nvlink_bytes=567754704 weight_to_token_ratio=0.064903 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132664536e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265478144 dispatch_meta_bytes=344480 combine_payload_bytes=265478144 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265822624 ingress_bytes=334684160 link_bytes=334684160 token_bytes=531300768 weight_movement_bytes=69206016 total_nvlink_bytes=600506784 weight_to_token_ratio=0.130258 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259389550e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74408 payload_rows_remote=65204 dup_rows=23896 dup_rows_remote=20930 dedup_ratio=0.243083 router_tokens_per_expert=[753, 778, 763, 750, 747, 797, 764, 733, 742, 791, 726, 772, 740, 787, 746, 762, 773, 797, 793, 790, 788, 737, 757, 750, 783, 755, 783, 743, 772, 804, 752, 811, 786, 853, 743, 732, 765, 737, 731, 793, 764, 780, 753, 811, 838, 823, 794, 776, 737, 762, 722, 758, 737, 801, 743, 812, 714, 791, 781, 768, 736, 739, 750, 783] payload_rows_per_dst=[9204, 9374, 9434, 9426, 9242, 9370, 9114, 9244] dup_rows_per_dst=[2966, 3050, 3036, 2980, 2938, 3016, 3030, 2880] combine_rows_per_src=[9204, 9374, 9434, 9426, 9242, 9370, 9114, 9244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=264855552 dispatch_meta_bytes=342888 combine_payload_bytes=264855552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265198440 ingress_bytes=299458560 link_bytes=299458560 token_bytes=530053992 weight_movement_bytes=34603008 total_nvlink_bytes=564657000 weight_to_token_ratio=0.065282 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126838453e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74318 payload_rows_remote=65040 dup_rows=23986 dup_rows_remote=20964 dedup_ratio=0.243998 router_tokens_per_expert=[766, 815, 767, 774, 766, 764, 772, 773, 790, 760, 801, 765, 713, 822, 727, 772, 722, 798, 715, 765, 775, 773, 766, 826, 733, 756, 786, 754, 817, 766, 747, 774, 773, 775, 773, 776, 728, 849, 799, 766, 772, 783, 767, 798, 758, 741, 754, 777, 774, 806, 801, 744, 727, 768, 763, 736, 785, 780, 710, 764, 703, 747, 771, 764] payload_rows_per_dst=[9376, 9278, 9314, 9178, 9364, 9288, 9296, 9224] dup_rows_per_dst=[3018, 3022, 2966, 3088, 3114, 3012, 2942, 2824] combine_rows_per_src=[9376, 9278, 9314, 9178, 9364, 9288, 9296, 9224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267296768 dispatch_meta_bytes=344296 combine_payload_bytes=267296768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267641064 ingress_bytes=301899776 link_bytes=301899776 token_bytes=534937832 weight_movement_bytes=34603008 total_nvlink_bytes=569540840 weight_to_token_ratio=0.064686 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136024552e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266567680 dispatch_meta_bytes=345256 combine_payload_bytes=266567680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266912936 ingress_bytes=301170688 link_bytes=301170688 token_bytes=533480616 weight_movement_bytes=34603008 total_nvlink_bytes=568083624 weight_to_token_ratio=0.064863 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133281053e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=343928 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265846648 ingress_bytes=334708736 link_bytes=334708736 token_bytes=531349368 weight_movement_bytes=69206016 total_nvlink_bytes=600555384 weight_to_token_ratio=0.130246 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259482027e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267075584 dispatch_meta_bytes=344536 combine_payload_bytes=267075584 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267420120 ingress_bytes=336281600 link_bytes=336281600 token_bytes=534495704 weight_movement_bytes=69206016 total_nvlink_bytes=603701720 weight_to_token_ratio=0.129479 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265400588e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=344408 combine_payload_bytes=266477568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266821976 ingress_bytes=335683584 link_bytes=335683584 token_bytes=533299544 weight_movement_bytes=69206016 total_nvlink_bytes=602505560 weight_to_token_ratio=0.129770 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263150302e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266403840 dispatch_meta_bytes=344016 combine_payload_bytes=266403840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266747856 ingress_bytes=301006848 link_bytes=301006848 token_bytes=533151696 weight_movement_bytes=34603008 total_nvlink_bytes=567754704 weight_to_token_ratio=0.064903 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132664536e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74412 payload_rows_remote=65034 dup_rows=23892 dup_rows_remote=20906 dedup_ratio=0.243042 router_tokens_per_expert=[780, 707, 735, 741, 765, 784, 806, 755, 735, 778, 765, 762, 822, 762, 779, 781, 762, 783, 757, 736, 786, 750, 808, 813, 782, 778, 740, 797, 785, 749, 787, 788, 792, 769, 751, 732, 768, 788, 788, 794, 753, 751, 768, 758, 784, 757, 761, 758, 744, 754, 756, 756, 769, 774, 774, 739, 722, 732, 812, 791, 774, 737, 807, 781] payload_rows_per_dst=[9260, 9348, 9336, 9296, 9378, 9278, 9166, 9350] dup_rows_per_dst=[2886, 3020, 3054, 3116, 2986, 2902, 2966, 2962] combine_rows_per_src=[9260, 9348, 9336, 9296, 9378, 9278, 9166, 9350] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267075584 dispatch_meta_bytes=344536 combine_payload_bytes=267075584 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267420120 ingress_bytes=336281600 link_bytes=336281600 token_bytes=534495704 weight_movement_bytes=69206016 total_nvlink_bytes=603701720 weight_to_token_ratio=0.129479 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265400588e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74436 payload_rows_remote=65190 dup_rows=23868 dup_rows_remote=20896 dedup_ratio=0.242798 router_tokens_per_expert=[813, 804, 765, 771, 747, 763, 779, 775, 767, 785, 781, 744, 740, 782, 794, 767, 805, 800, 719, 745, 776, 761, 748, 750, 750, 801, 836, 760, 778, 738, 764, 794, 756, 790, 762, 832, 739, 724, 792, 733, 772, 725, 768, 787, 759, 761, 762, 775, 779, 761, 762, 763, 765, 749, 772, 773, 788, 721, 755, 743, 811, 747, 764, 760] payload_rows_per_dst=[9354, 9300, 9284, 9400, 9258, 9246, 9314, 9280] dup_rows_per_dst=[3080, 3020, 2924, 3042, 2998, 2972, 2934, 2898] combine_rows_per_src=[9354, 9300, 9284, 9400, 9258, 9246, 9314, 9280] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=64924 dup_rows=24118 dup_rows_remote=21194 dedup_ratio=0.245341 router_tokens_per_expert=[777, 801, 821, 786, 726, 759, 813, 775, 752, 743, 775, 792, 761, 762, 772, 790, 771, 773, 762, 763, 775, 774, 744, 742, 759, 804, 763, 770, 750, 800, 744, 772, 717, 748, 766, 824, 733, 745, 717, 798, 715, 727, 752, 794, 765, 727, 736, 829, 763, 805, 831, 743, 788, 761, 810, 794, 746, 730, 754, 757, 813, 744, 776, 773] payload_rows_per_dst=[8868, 9076, 9166, 9324, 9346, 9624, 9520, 9262] dup_rows_per_dst=[2758, 2822, 2980, 3062, 2990, 3356, 3226, 2924] combine_rows_per_src=[8868, 9076, 9166, 9324, 9346, 9624, 9520, 9262] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266379264 dispatch_meta_bytes=343760 combine_payload_bytes=266379264 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266723024 ingress_bytes=370188288 link_bytes=370188288 token_bytes=533102288 weight_movement_bytes=103809024 total_nvlink_bytes=636911312 weight_to_token_ratio=0.194726 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392988725e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74388 payload_rows_remote=65034 dup_rows=23916 dup_rows_remote=20948 dedup_ratio=0.243286 router_tokens_per_expert=[718, 732, 759, 787, 787, 783, 759, 778, 791, 762, 785, 779, 758, 723, 784, 779, 775, 737, 750, 804, 741, 786, 732, 722, 704, 815, 765, 753, 757, 771, 818, 788, 783, 792, 858, 778, 769, 789, 716, 767, 773, 739, 786, 743, 745, 773, 782, 718, 758, 780, 787, 747, 753, 757, 778, 733, 786, 785, 792, 758, 777, 803, 783, 782] payload_rows_per_dst=[9334, 9354, 9130, 9280, 9422, 9156, 9272, 9440] dup_rows_per_dst=[2872, 2968, 2964, 3062, 3082, 2962, 2914, 3092] combine_rows_per_src=[9334, 9354, 9130, 9280, 9422, 9156, 9272, 9440] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74436 payload_rows_remote=65190 dup_rows=23868 dup_rows_remote=20896 dedup_ratio=0.242798 router_tokens_per_expert=[813, 804, 765, 771, 747, 763, 779, 775, 767, 785, 781, 744, 740, 782, 794, 767, 805, 800, 719, 745, 776, 761, 748, 750, 750, 801, 836, 760, 778, 738, 764, 794, 756, 790, 762, 832, 739, 724, 792, 733, 772, 725, 768, 787, 759, 761, 762, 775, 779, 761, 762, 763, 765, 749, 772, 773, 788, 721, 755, 743, 811, 747, 764, 760] payload_rows_per_dst=[9354, 9300, 9284, 9400, 9258, 9246, 9314, 9280] dup_rows_per_dst=[3080, 3020, 2924, 3042, 2998, 2972, 2934, 2898] combine_rows_per_src=[9354, 9300, 9284, 9400, 9258, 9246, 9314, 9280] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265928704 dispatch_meta_bytes=344472 combine_payload_bytes=265928704 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266273176 ingress_bytes=335134720 link_bytes=335134720 token_bytes=532201880 weight_movement_bytes=69206016 total_nvlink_bytes=601407896 weight_to_token_ratio=0.130037 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261084971e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74480 payload_rows_remote=65250 dup_rows=23824 dup_rows_remote=20876 dedup_ratio=0.242350 router_tokens_per_expert=[798, 831, 802, 748, 748, 752, 744, 768, 774, 784, 765, 783, 809, 754, 802, 795, 767, 800, 845, 779, 770, 771, 741, 770, 770, 750, 748, 744, 771, 727, 808, 743, 793, 762, 723, 818, 776, 737, 740, 754, 744, 707, 815, 775, 800, 714, 789, 739, 759, 802, 779, 739, 774, 734, 776, 726, 773, 814, 797, 719, 775, 738, 736, 764] payload_rows_per_dst=[9454, 9436, 9360, 9158, 9284, 9190, 9230, 9368] dup_rows_per_dst=[2928, 3096, 3126, 2964, 2922, 2976, 2948, 2864] combine_rows_per_src=[9454, 9436, 9360, 9158, 9284, 9190, 9230, 9368] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=65154 dup_rows=23906 dup_rows_remote=20904 dedup_ratio=0.243184 router_tokens_per_expert=[786, 771, 773, 764, 782, 797, 768, 804, 778, 774, 736, 809, 766, 729, 748, 783, 747, 774, 754, 783, 798, 756, 716, 768, 729, 778, 780, 765, 803, 734, 762, 772, 732, 782, 747, 795, 799, 776, 741, 756, 742, 769, 794, 760, 766, 780, 753, 813, 780, 737, 785, 787, 781, 751, 767, 761, 759, 772, 804, 779, 733, 761, 718, 785] payload_rows_per_dst=[9438, 9216, 9246, 9244, 9258, 9374, 9364, 9258] dup_rows_per_dst=[3052, 3030, 2946, 3002, 2998, 2980, 2934, 2964] combine_rows_per_src=[9438, 9216, 9246, 9244, 9258, 9374, 9364, 9258] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=64960 dup_rows=24018 dup_rows_remote=21146 dedup_ratio=0.244324 router_tokens_per_expert=[783, 739, 794, 807, 766, 801, 761, 799, 780, 786, 810, 777, 848, 775, 731, 760, 813, 755, 756, 748, 723, 762, 777, 765, 773, 732, 724, 797, 759, 738, 748, 781, 773, 808, 795, 723, 713, 813, 774, 746, 758, 782, 742, 748, 764, 780, 773, 745, 742, 796, 761, 763, 776, 750, 751, 760, 752, 846, 773, 744, 775, 735, 811, 712] payload_rows_per_dst=[9388, 9432, 9326, 9176, 9272, 9228, 9168, 9296] dup_rows_per_dst=[3112, 3102, 2872, 2928, 3018, 2956, 3030, 3000] combine_rows_per_src=[9388, 9432, 9326, 9176, 9272, 9228, 9168, 9296] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74212 payload_rows_remote=65080 dup_rows=24092 dup_rows_remote=21234 dedup_ratio=0.245076 router_tokens_per_expert=[734, 750, 805, 784, 787, 783, 815, 748, 764, 751, 793, 767, 789, 774, 707, 742, 792, 731, 765, 768, 768, 825, 728, 747, 749, 777, 749, 789, 778, 775, 803, 776, 736, 771, 779, 790, 757, 758, 756, 785, 774, 821, 760, 772, 759, 735, 764, 798, 727, 753, 703, 799, 728, 788, 777, 720, 831, 769, 743, 785, 732, 755, 823, 791] payload_rows_per_dst=[9306, 9268, 9208, 9400, 9306, 9274, 9132, 9318] dup_rows_per_dst=[3106, 2906, 3040, 2992, 2958, 3092, 2858, 3140] combine_rows_per_src=[9306, 9268, 9208, 9400, 9306, 9274, 9132, 9318] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266379264 dispatch_meta_bytes=343928 combine_payload_bytes=266379264 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266723192 ingress_bytes=404791296 link_bytes=404791296 token_bytes=533102456 weight_movement_bytes=138412032 total_nvlink_bytes=671514488 weight_to_token_ratio=0.259635 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523197058e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267018240 dispatch_meta_bytes=344344 combine_payload_bytes=267018240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267362584 ingress_bytes=301621248 link_bytes=301621248 token_bytes=534380824 weight_movement_bytes=34603008 total_nvlink_bytes=568983832 weight_to_token_ratio=0.064753 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134976474e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74434 payload_rows_remote=65120 dup_rows=23870 dup_rows_remote=20804 dedup_ratio=0.242818 router_tokens_per_expert=[740, 765, 754, 790, 793, 769, 754, 768, 766, 782, 790, 795, 773, 761, 775, 810, 802, 751, 775, 812, 774, 782, 758, 759, 774, 743, 748, 764, 754, 749, 807, 795, 815, 779, 772, 785, 747, 760, 800, 723, 751, 724, 758, 796, 749, 736, 746, 770, 716, 748, 731, 753, 736, 753, 805, 802, 761, 793, 773, 769, 803, 740, 753, 773] payload_rows_per_dst=[9314, 9428, 9464, 9236, 9390, 9192, 9138, 9272] dup_rows_per_dst=[3066, 2978, 2984, 3032, 2972, 2868, 2950, 3020] combine_rows_per_src=[9314, 9428, 9464, 9236, 9390, 9192, 9138, 9272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267018240 dispatch_meta_bytes=344344 combine_payload_bytes=267018240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267362584 ingress_bytes=301621248 link_bytes=301621248 token_bytes=534380824 weight_movement_bytes=34603008 total_nvlink_bytes=568983832 weight_to_token_ratio=0.064753 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134976474e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74388 payload_rows_remote=65034 dup_rows=23916 dup_rows_remote=20948 dedup_ratio=0.243286 router_tokens_per_expert=[718, 732, 759, 787, 787, 783, 759, 778, 791, 762, 785, 779, 758, 723, 784, 779, 775, 737, 750, 804, 741, 786, 732, 722, 704, 815, 765, 753, 757, 771, 818, 788, 783, 792, 858, 778, 769, 789, 716, 767, 773, 739, 786, 743, 745, 773, 782, 718, 758, 780, 787, 747, 753, 757, 778, 733, 786, 785, 792, 758, 777, 803, 783, 782] payload_rows_per_dst=[9334, 9354, 9130, 9280, 9422, 9156, 9272, 9440] dup_rows_per_dst=[2872, 2968, 2964, 3062, 3082, 2962, 2914, 3092] combine_rows_per_src=[9334, 9354, 9130, 9280, 9422, 9156, 9272, 9440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=64960 dup_rows=24018 dup_rows_remote=21146 dedup_ratio=0.244324 router_tokens_per_expert=[783, 739, 794, 807, 766, 801, 761, 799, 780, 786, 810, 777, 848, 775, 731, 760, 813, 755, 756, 748, 723, 762, 777, 765, 773, 732, 724, 797, 759, 738, 748, 781, 773, 808, 795, 723, 713, 813, 774, 746, 758, 782, 742, 748, 764, 780, 773, 745, 742, 796, 761, 763, 776, 750, 751, 760, 752, 846, 773, 744, 775, 735, 811, 712] payload_rows_per_dst=[9388, 9432, 9326, 9176, 9272, 9228, 9168, 9296] dup_rows_per_dst=[3112, 3102, 2872, 2928, 3018, 2956, 3030, 3000] combine_rows_per_src=[9388, 9432, 9326, 9176, 9272, 9228, 9168, 9296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265928704 dispatch_meta_bytes=344472 combine_payload_bytes=265928704 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266273176 ingress_bytes=335134720 link_bytes=335134720 token_bytes=532201880 weight_movement_bytes=69206016 total_nvlink_bytes=601407896 weight_to_token_ratio=0.130037 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261084971e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267264000 dispatch_meta_bytes=344504 combine_payload_bytes=267264000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267608504 ingress_bytes=301867008 link_bytes=301867008 token_bytes=534872504 weight_movement_bytes=34603008 total_nvlink_bytes=569475512 weight_to_token_ratio=0.064694 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135901249e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266870784 dispatch_meta_bytes=344232 combine_payload_bytes=266870784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267215016 ingress_bytes=301473792 link_bytes=301473792 token_bytes=534085800 weight_movement_bytes=34603008 total_nvlink_bytes=568688808 weight_to_token_ratio=0.064789 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134421609e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=343696 combine_payload_bytes=266731520 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267075216 ingress_bytes=335937536 link_bytes=335937536 token_bytes=533806736 weight_movement_bytes=69206016 total_nvlink_bytes=603012752 weight_to_token_ratio=0.129646 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264105903e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266379264 dispatch_meta_bytes=343928 combine_payload_bytes=266379264 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266723192 ingress_bytes=404791296 link_bytes=404791296 token_bytes=533102456 weight_movement_bytes=138412032 total_nvlink_bytes=671514488 weight_to_token_ratio=0.259635 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523197058e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266076160 dispatch_meta_bytes=344424 combine_payload_bytes=266076160 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266420584 ingress_bytes=335282176 link_bytes=335282176 token_bytes=532496744 weight_movement_bytes=69206016 total_nvlink_bytes=601702760 weight_to_token_ratio=0.129965 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261639836e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74196 payload_rows_remote=65046 dup_rows=24108 dup_rows_remote=21164 dedup_ratio=0.245239 router_tokens_per_expert=[766, 708, 750, 776, 775, 792, 770, 806, 733, 765, 706, 776, 780, 775, 729, 759, 804, 791, 812, 752, 785, 774, 791, 757, 756, 742, 781, 832, 769, 752, 765, 800, 755, 793, 759, 750, 724, 757, 751, 758, 791, 746, 747, 758, 761, 766, 763, 780, 756, 796, 778, 760, 746, 786, 757, 802, 768, 781, 777, 778, 758, 795, 766, 760] payload_rows_per_dst=[9228, 9140, 9434, 9442, 9150, 9284, 9268, 9250] dup_rows_per_dst=[3058, 2906, 3098, 2952, 2944, 2940, 3094, 3116] combine_rows_per_src=[9228, 9140, 9434, 9442, 9150, 9284, 9268, 9250] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266567680 dispatch_meta_bytes=345256 combine_payload_bytes=266567680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266912936 ingress_bytes=301170688 link_bytes=301170688 token_bytes=533480616 weight_movement_bytes=34603008 total_nvlink_bytes=568083624 weight_to_token_ratio=0.064863 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133281053e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=343696 combine_payload_bytes=266731520 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267075216 ingress_bytes=335937536 link_bytes=335937536 token_bytes=533806736 weight_movement_bytes=69206016 total_nvlink_bytes=603012752 weight_to_token_ratio=0.129646 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264105903e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73924 payload_rows_remote=64568 dup_rows=24380 dup_rows_remote=21280 dedup_ratio=0.248006 router_tokens_per_expert=[789, 759, 787, 831, 776, 817, 732, 773, 742, 767, 738, 722, 792, 767, 811, 753, 715, 780, 823, 733, 767, 798, 755, 717, 748, 747, 760, 790, 751, 742, 734, 728, 770, 738, 747, 801, 739, 727, 767, 756, 749, 777, 765, 780, 801, 791, 770, 772, 766, 779, 777, 748, 822, 804, 770, 786, 765, 783, 789, 769, 764, 752, 802, 782] payload_rows_per_dst=[9378, 9142, 8782, 9474, 9118, 9186, 9488, 9356] dup_rows_per_dst=[3150, 2958, 2712, 3248, 2972, 2956, 3284, 3100] combine_rows_per_src=[9378, 9142, 8782, 9474, 9118, 9186, 9488, 9356] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=344840 combine_payload_bytes=266428416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266773256 ingress_bytes=301031424 link_bytes=301031424 token_bytes=533201672 weight_movement_bytes=34603008 total_nvlink_bytes=567804680 weight_to_token_ratio=0.064897 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132757013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=65034 dup_rows=24028 dup_rows_remote=21008 dedup_ratio=0.244425 router_tokens_per_expert=[722, 757, 745, 741, 803, 751, 724, 696, 745, 761, 770, 755, 792, 751, 775, 782, 804, 761, 755, 774, 821, 785, 795, 821, 783, 766, 768, 753, 750, 792, 764, 808, 756, 780, 793, 770, 783, 758, 760, 731, 744, 780, 734, 786, 754, 781, 764, 764, 768, 739, 759, 765, 740, 795, 756, 764, 785, 745, 802, 717, 761, 800, 816, 832] payload_rows_per_dst=[9044, 9242, 9526, 9300, 9248, 9218, 9290, 9408] dup_rows_per_dst=[2834, 3020, 3106, 3068, 3014, 2996, 2882, 3108] combine_rows_per_src=[9044, 9242, 9526, 9300, 9248, 9218, 9290, 9408] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74064 payload_rows_remote=64852 dup_rows=24240 dup_rows_remote=21278 dedup_ratio=0.246582 router_tokens_per_expert=[796, 762, 785, 738, 796, 787, 804, 811, 770, 769, 749, 732, 797, 750, 783, 808, 754, 784, 789, 741, 777, 766, 812, 737, 769, 755, 780, 751, 728, 728, 774, 791, 782, 742, 811, 769, 739, 784, 759, 757, 735, 775, 735, 782, 765, 781, 758, 756, 752, 711, 742, 824, 757, 741, 743, 820, 781, 769, 788, 761, 761, 746, 794, 759] payload_rows_per_dst=[9350, 9262, 9246, 9206, 9324, 9212, 9196, 9268] dup_rows_per_dst=[3208, 3054, 3074, 2946, 2962, 2962, 2984, 3050] combine_rows_per_src=[9350, 9262, 9246, 9206, 9324, 9212, 9196, 9268] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=264470528 dispatch_meta_bytes=343392 combine_payload_bytes=264470528 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=264813920 ingress_bytes=368279552 link_bytes=368279552 token_bytes=529284448 weight_movement_bytes=103809024 total_nvlink_bytes=633093472 weight_to_token_ratio=0.196131 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.385806305e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73924 payload_rows_remote=64568 dup_rows=24380 dup_rows_remote=21280 dedup_ratio=0.248006 router_tokens_per_expert=[789, 759, 787, 831, 776, 817, 732, 773, 742, 767, 738, 722, 792, 767, 811, 753, 715, 780, 823, 733, 767, 798, 755, 717, 748, 747, 760, 790, 751, 742, 734, 728, 770, 738, 747, 801, 739, 727, 767, 756, 749, 777, 765, 780, 801, 791, 770, 772, 766, 779, 777, 748, 822, 804, 770, 786, 765, 783, 789, 769, 764, 752, 802, 782] payload_rows_per_dst=[9378, 9142, 8782, 9474, 9118, 9186, 9488, 9356] dup_rows_per_dst=[3150, 2958, 2712, 3248, 2972, 2956, 3284, 3100] combine_rows_per_src=[9378, 9142, 8782, 9474, 9118, 9186, 9488, 9356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=344840 combine_payload_bytes=266428416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266773256 ingress_bytes=301031424 link_bytes=301031424 token_bytes=533201672 weight_movement_bytes=34603008 total_nvlink_bytes=567804680 weight_to_token_ratio=0.064897 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132757013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265633792 dispatch_meta_bytes=344520 combine_payload_bytes=265633792 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265978312 ingress_bytes=334839808 link_bytes=334839808 token_bytes=531612104 weight_movement_bytes=69206016 total_nvlink_bytes=600818120 weight_to_token_ratio=0.130181 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259975241e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74224 payload_rows_remote=64948 dup_rows=24080 dup_rows_remote=21186 dedup_ratio=0.244954 router_tokens_per_expert=[767, 772, 710, 747, 737, 786, 790, 732, 748, 794, 789, 793, 749, 798, 756, 778, 774, 786, 767, 764, 783, 746, 803, 760, 759, 773, 742, 705, 796, 768, 748, 779, 763, 788, 786, 822, 774, 747, 774, 768, 811, 775, 781, 797, 786, 792, 776, 779, 770, 721, 758, 762, 778, 758, 717, 821, 798, 704, 795, 772, 735, 751, 739, 755] payload_rows_per_dst=[9180, 9280, 9282, 9230, 9392, 9442, 9276, 9142] dup_rows_per_dst=[2902, 3130, 3084, 2910, 3052, 3152, 2894, 2956] combine_rows_per_src=[9180, 9280, 9282, 9230, 9392, 9442, 9276, 9142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74386 payload_rows_remote=65126 dup_rows=23918 dup_rows_remote=20970 dedup_ratio=0.243306 router_tokens_per_expert=[746, 778, 793, 701, 771, 701, 793, 766, 781, 773, 766, 760, 729, 794, 772, 780, 792, 779, 768, 744, 770, 742, 774, 752, 833, 761, 748, 740, 756, 750, 741, 775, 784, 772, 778, 779, 784, 757, 761, 830, 747, 763, 792, 751, 763, 755, 782, 791, 791, 751, 735, 761, 745, 776, 730, 775, 770, 799, 806, 799, 818, 758, 758, 762] payload_rows_per_dst=[9166, 9316, 9240, 9260, 9442, 9266, 9260, 9436] dup_rows_per_dst=[2932, 2994, 3002, 2948, 3048, 3022, 2868, 3104] combine_rows_per_src=[9166, 9316, 9240, 9260, 9442, 9266, 9260, 9436] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74486 payload_rows_remote=65052 dup_rows=23818 dup_rows_remote=20794 dedup_ratio=0.242289 router_tokens_per_expert=[823, 750, 752, 752, 780, 780, 734, 783, 761, 803, 764, 766, 788, 780, 774, 735, 772, 775, 753, 761, 766, 781, 810, 811, 767, 732, 769, 733, 773, 749, 747, 802, 760, 737, 767, 783, 738, 773, 803, 780, 775, 760, 707, 766, 778, 787, 764, 781, 781, 804, 775, 765, 750, 729, 725, 752, 750, 788, 761, 752, 770, 778, 826, 761] payload_rows_per_dst=[9336, 9336, 9434, 9236, 9372, 9244, 9264, 9264] dup_rows_per_dst=[2972, 3006, 3024, 2908, 2910, 2992, 2898, 3108] combine_rows_per_src=[9336, 9336, 9434, 9236, 9372, 9244, 9264, 9264] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=64934 dup_rows=23924 dup_rows_remote=20842 dedup_ratio=0.243368 router_tokens_per_expert=[702, 748, 742, 770, 782, 809, 758, 744, 804, 737, 785, 784, 746, 771, 740, 797, 774, 780, 801, 739, 776, 783, 759, 802, 735, 751, 726, 776, 802, 771, 780, 741, 782, 803, 754, 767, 776, 809, 733, 753, 786, 769, 783, 741, 742, 785, 771, 760, 788, 735, 790, 754, 792, 795, 779, 792, 792, 801, 726, 734, 769, 749, 754, 773] payload_rows_per_dst=[9446, 9320, 9300, 9284, 9242, 9350, 9432, 9006] dup_rows_per_dst=[3082, 3012, 3128, 2880, 3108, 3060, 3018, 2636] combine_rows_per_src=[9446, 9320, 9300, 9284, 9242, 9350, 9432, 9006] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74480 payload_rows_remote=65250 dup_rows=23824 dup_rows_remote=20876 dedup_ratio=0.242350 router_tokens_per_expert=[798, 831, 802, 748, 748, 752, 744, 768, 774, 784, 765, 783, 809, 754, 802, 795, 767, 800, 845, 779, 770, 771, 741, 770, 770, 750, 748, 744, 771, 727, 808, 743, 793, 762, 723, 818, 776, 737, 740, 754, 744, 707, 815, 775, 800, 714, 789, 739, 759, 802, 779, 739, 774, 734, 776, 726, 773, 814, 797, 719, 775, 738, 736, 764] payload_rows_per_dst=[9454, 9436, 9360, 9158, 9284, 9190, 9230, 9368] dup_rows_per_dst=[2928, 3096, 3126, 2964, 2922, 2976, 2948, 2864] combine_rows_per_src=[9454, 9436, 9360, 9158, 9284, 9190, 9230, 9368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266379264 dispatch_meta_bytes=344168 combine_payload_bytes=266379264 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266723432 ingress_bytes=370188288 link_bytes=370188288 token_bytes=533102696 weight_movement_bytes=103809024 total_nvlink_bytes=636911720 weight_to_token_ratio=0.194726 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392988725e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=65034 dup_rows=24028 dup_rows_remote=21008 dedup_ratio=0.244425 router_tokens_per_expert=[722, 757, 745, 741, 803, 751, 724, 696, 745, 761, 770, 755, 792, 751, 775, 782, 804, 761, 755, 774, 821, 785, 795, 821, 783, 766, 768, 753, 750, 792, 764, 808, 756, 780, 793, 770, 783, 758, 760, 731, 744, 780, 734, 786, 754, 781, 764, 764, 768, 739, 759, 765, 740, 795, 756, 764, 785, 745, 802, 717, 761, 800, 816, 832] payload_rows_per_dst=[9044, 9242, 9526, 9300, 9248, 9218, 9290, 9408] dup_rows_per_dst=[2834, 3020, 3106, 3068, 3014, 2996, 2882, 3108] combine_rows_per_src=[9044, 9242, 9526, 9300, 9248, 9218, 9290, 9408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74486 payload_rows_remote=65052 dup_rows=23818 dup_rows_remote=20794 dedup_ratio=0.242289 router_tokens_per_expert=[823, 750, 752, 752, 780, 780, 734, 783, 761, 803, 764, 766, 788, 780, 774, 735, 772, 775, 753, 761, 766, 781, 810, 811, 767, 732, 769, 733, 773, 749, 747, 802, 760, 737, 767, 783, 738, 773, 803, 780, 775, 760, 707, 766, 778, 787, 764, 781, 781, 804, 775, 765, 750, 729, 725, 752, 750, 788, 761, 752, 770, 778, 826, 761] payload_rows_per_dst=[9336, 9336, 9434, 9236, 9372, 9244, 9264, 9264] dup_rows_per_dst=[2972, 3006, 3024, 2908, 2910, 2992, 2898, 3108] combine_rows_per_src=[9336, 9336, 9434, 9236, 9372, 9244, 9264, 9264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=264470528 dispatch_meta_bytes=343392 combine_payload_bytes=264470528 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=264813920 ingress_bytes=368279552 link_bytes=368279552 token_bytes=529284448 weight_movement_bytes=103809024 total_nvlink_bytes=633093472 weight_to_token_ratio=0.196131 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.385806305e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265633792 dispatch_meta_bytes=344520 combine_payload_bytes=265633792 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265978312 ingress_bytes=334839808 link_bytes=334839808 token_bytes=531612104 weight_movement_bytes=69206016 total_nvlink_bytes=600818120 weight_to_token_ratio=0.130181 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259975241e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266027008 dispatch_meta_bytes=344536 combine_payload_bytes=266027008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266371544 ingress_bytes=300630016 link_bytes=300630016 token_bytes=532398552 weight_movement_bytes=34603008 total_nvlink_bytes=567001560 weight_to_token_ratio=0.064995 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131246548e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266756096 dispatch_meta_bytes=344384 combine_payload_bytes=266756096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267100480 ingress_bytes=301359104 link_bytes=301359104 token_bytes=533856576 weight_movement_bytes=34603008 total_nvlink_bytes=568459584 weight_to_token_ratio=0.064817 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133990047e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74174 payload_rows_remote=65084 dup_rows=24130 dup_rows_remote=21120 dedup_ratio=0.245463 router_tokens_per_expert=[804, 743, 806, 779, 740, 813, 730, 747, 758, 747, 750, 778, 774, 787, 736, 786, 752, 764, 762, 799, 794, 792, 782, 737, 796, 820, 743, 803, 764, 753, 739, 785, 767, 775, 771, 747, 756, 770, 705, 759, 761, 762, 795, 707, 803, 736, 767, 792, 773, 761, 815, 733, 781, 769, 729, 774, 765, 765, 769, 797, 807, 726, 787, 765] payload_rows_per_dst=[9328, 9298, 9228, 9374, 9090, 9260, 9274, 9322] dup_rows_per_dst=[2996, 2934, 3136, 3032, 3010, 2986, 2996, 3040] combine_rows_per_src=[9328, 9298, 9228, 9374, 9090, 9260, 9274, 9322] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266452992 dispatch_meta_bytes=343384 combine_payload_bytes=266452992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266796376 ingress_bytes=335659008 link_bytes=335659008 token_bytes=533249368 weight_movement_bytes=69206016 total_nvlink_bytes=602455384 weight_to_token_ratio=0.129782 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263057824e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265969664 dispatch_meta_bytes=343104 combine_payload_bytes=265969664 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266312768 ingress_bytes=300572672 link_bytes=300572672 token_bytes=532282432 weight_movement_bytes=34603008 total_nvlink_bytes=566885440 weight_to_token_ratio=0.065009 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131030767e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=65084 dup_rows=23992 dup_rows_remote=21046 dedup_ratio=0.244059 router_tokens_per_expert=[807, 774, 757, 776, 713, 783, 756, 793, 781, 791, 767, 789, 752, 748, 769, 765, 784, 750, 788, 768, 803, 760, 790, 804, 751, 715, 730, 791, 733, 774, 780, 777, 813, 749, 802, 771, 737, 801, 797, 772, 773, 798, 773, 772, 746, 731, 753, 792, 778, 757, 764, 783, 752, 740, 769, 729, 770, 789, 727, 765, 797, 779, 738, 716] payload_rows_per_dst=[9202, 9352, 9502, 9126, 9404, 9330, 9168, 9228] dup_rows_per_dst=[3116, 2972, 2992, 2976, 3080, 2934, 2976, 2946] combine_rows_per_src=[9202, 9352, 9502, 9126, 9404, 9330, 9168, 9228] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267264000 dispatch_meta_bytes=344504 combine_payload_bytes=267264000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267608504 ingress_bytes=301867008 link_bytes=301867008 token_bytes=534872504 weight_movement_bytes=34603008 total_nvlink_bytes=569475512 weight_to_token_ratio=0.064694 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135901249e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266379264 dispatch_meta_bytes=344168 combine_payload_bytes=266379264 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266723432 ingress_bytes=370188288 link_bytes=370188288 token_bytes=533102696 weight_movement_bytes=103809024 total_nvlink_bytes=636911720 weight_to_token_ratio=0.194726 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392988725e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266452992 dispatch_meta_bytes=343384 combine_payload_bytes=266452992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266796376 ingress_bytes=335659008 link_bytes=335659008 token_bytes=533249368 weight_movement_bytes=69206016 total_nvlink_bytes=602455384 weight_to_token_ratio=0.129782 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263057824e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74174 payload_rows_remote=65084 dup_rows=24130 dup_rows_remote=21120 dedup_ratio=0.245463 router_tokens_per_expert=[804, 743, 806, 779, 740, 813, 730, 747, 758, 747, 750, 778, 774, 787, 736, 786, 752, 764, 762, 799, 794, 792, 782, 737, 796, 820, 743, 803, 764, 753, 739, 785, 767, 775, 771, 747, 756, 770, 705, 759, 761, 762, 795, 707, 803, 736, 767, 792, 773, 761, 815, 733, 781, 769, 729, 774, 765, 765, 769, 797, 807, 726, 787, 765] payload_rows_per_dst=[9328, 9298, 9228, 9374, 9090, 9260, 9274, 9322] dup_rows_per_dst=[2996, 2934, 3136, 3032, 3010, 2986, 2996, 3040] combine_rows_per_src=[9328, 9298, 9228, 9374, 9090, 9260, 9274, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73932 payload_rows_remote=64778 dup_rows=24372 dup_rows_remote=21318 dedup_ratio=0.247925 router_tokens_per_expert=[760, 768, 809, 758, 793, 790, 774, 813, 789, 741, 790, 782, 741, 763, 767, 745, 802, 762, 761, 737, 737, 766, 761, 713, 763, 781, 771, 830, 796, 809, 751, 775, 751, 766, 732, 782, 746, 783, 723, 763, 786, 716, 775, 810, 750, 749, 753, 765, 771, 821, 776, 799, 769, 760, 745, 749, 813, 796, 719, 760, 697, 825, 775, 729] payload_rows_per_dst=[9284, 9132, 9158, 9408, 9268, 9154, 9320, 9208] dup_rows_per_dst=[3246, 3104, 2920, 3144, 2824, 3054, 3060, 3020] combine_rows_per_src=[9284, 9132, 9158, 9408, 9268, 9154, 9320, 9208] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266584064 dispatch_meta_bytes=344816 combine_payload_bytes=266584064 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266928880 ingress_bytes=301187072 link_bytes=301187072 token_bytes=533512944 weight_movement_bytes=34603008 total_nvlink_bytes=568115952 weight_to_token_ratio=0.064859 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133342704e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73998 payload_rows_remote=64828 dup_rows=24306 dup_rows_remote=21304 dedup_ratio=0.247253 router_tokens_per_expert=[781, 775, 757, 769, 794, 766, 800, 780, 744, 804, 775, 771, 750, 756, 747, 739, 803, 759, 732, 747, 791, 762, 771, 783, 791, 817, 729, 737, 703, 727, 756, 736, 797, 757, 749, 783, 788, 768, 792, 753, 777, 772, 747, 810, 747, 756, 750, 728, 711, 786, 775, 760, 821, 763, 778, 807, 736, 787, 789, 775, 808, 754, 762, 814] payload_rows_per_dst=[9374, 9170, 9314, 9116, 9342, 9094, 9284, 9304] dup_rows_per_dst=[3070, 3002, 2982, 2876, 3032, 3080, 3118, 3146] combine_rows_per_src=[9374, 9170, 9314, 9116, 9342, 9094, 9284, 9304] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266584064 dispatch_meta_bytes=344520 combine_payload_bytes=266584064 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266928584 ingress_bytes=370393088 link_bytes=370393088 token_bytes=533512648 weight_movement_bytes=103809024 total_nvlink_bytes=637321672 weight_to_token_ratio=0.194577 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393759371e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73932 payload_rows_remote=64778 dup_rows=24372 dup_rows_remote=21318 dedup_ratio=0.247925 router_tokens_per_expert=[760, 768, 809, 758, 793, 790, 774, 813, 789, 741, 790, 782, 741, 763, 767, 745, 802, 762, 761, 737, 737, 766, 761, 713, 763, 781, 771, 830, 796, 809, 751, 775, 751, 766, 732, 782, 746, 783, 723, 763, 786, 716, 775, 810, 750, 749, 753, 765, 771, 821, 776, 799, 769, 760, 745, 749, 813, 796, 719, 760, 697, 825, 775, 729] payload_rows_per_dst=[9284, 9132, 9158, 9408, 9268, 9154, 9320, 9208] dup_rows_per_dst=[3246, 3104, 2920, 3144, 2824, 3054, 3060, 3020] combine_rows_per_src=[9284, 9132, 9158, 9408, 9268, 9154, 9320, 9208] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=65084 dup_rows=23992 dup_rows_remote=21046 dedup_ratio=0.244059 router_tokens_per_expert=[807, 774, 757, 776, 713, 783, 756, 793, 781, 791, 767, 789, 752, 748, 769, 765, 784, 750, 788, 768, 803, 760, 790, 804, 751, 715, 730, 791, 733, 774, 780, 777, 813, 749, 802, 771, 737, 801, 797, 772, 773, 798, 773, 772, 746, 731, 753, 792, 778, 757, 764, 783, 752, 740, 769, 729, 770, 789, 727, 765, 797, 779, 738, 716] payload_rows_per_dst=[9202, 9352, 9502, 9126, 9404, 9330, 9168, 9228] dup_rows_per_dst=[3116, 2972, 2992, 2976, 3080, 2934, 2976, 2946] combine_rows_per_src=[9202, 9352, 9502, 9126, 9404, 9330, 9168, 9228] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74234 payload_rows_remote=65022 dup_rows=24070 dup_rows_remote=21262 dedup_ratio=0.244853 router_tokens_per_expert=[719, 747, 784, 740, 747, 758, 754, 727, 816, 771, 722, 727, 769, 767, 755, 768, 759, 749, 805, 850, 778, 730, 712, 781, 792, 712, 782, 745, 806, 806, 785, 779, 791, 734, 775, 813, 777, 771, 754, 752, 773, 764, 773, 826, 760, 862, 734, 801, 754, 729, 760, 771, 777, 740, 755, 724, 790, 765, 797, 793, 712, 783, 813, 787] payload_rows_per_dst=[9114, 9236, 9274, 9332, 9294, 9450, 9212, 9322] dup_rows_per_dst=[2838, 2954, 3054, 3082, 3040, 3136, 2808, 3158] combine_rows_per_src=[9114, 9236, 9274, 9332, 9294, 9450, 9212, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265330688 dispatch_meta_bytes=344384 combine_payload_bytes=265330688 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265675072 ingress_bytes=369139712 link_bytes=369139712 token_bytes=531005760 weight_movement_bytes=103809024 total_nvlink_bytes=634814784 weight_to_token_ratio=0.195495 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389043018e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=65024 dup_rows=23928 dup_rows_remote=20906 dedup_ratio=0.243408 router_tokens_per_expert=[778, 763, 746, 791, 753, 789, 757, 763, 767, 785, 788, 755, 781, 755, 734, 693, 753, 811, 790, 764, 776, 782, 794, 766, 787, 783, 810, 772, 783, 771, 731, 750, 761, 802, 813, 753, 743, 793, 729, 776, 755, 765, 774, 778, 745, 763, 766, 761, 717, 760, 769, 798, 770, 812, 757, 722, 760, 770, 792, 761, 746, 774, 768, 778] payload_rows_per_dst=[9302, 9204, 9444, 9352, 9334, 9240, 9236, 9264] dup_rows_per_dst=[2978, 2912, 3028, 3022, 3006, 2974, 2974, 3034] combine_rows_per_src=[9302, 9204, 9444, 9352, 9334, 9240, 9236, 9264] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74026 payload_rows_remote=64690 dup_rows=24278 dup_rows_remote=21038 dedup_ratio=0.246969 router_tokens_per_expert=[720, 701, 817, 771, 787, 748, 771, 737, 749, 743, 804, 731, 771, 739, 790, 785, 768, 731, 803, 785, 803, 762, 728, 742, 800, 770, 781, 772, 832, 832, 805, 755, 784, 738, 765, 732, 727, 795, 757, 782, 765, 767, 723, 732, 772, 753, 804, 759, 745, 746, 757, 796, 760, 758, 791, 745, 775, 786, 789, 800, 748, 788, 776, 804] payload_rows_per_dst=[9336, 9338, 9294, 9308, 9052, 9164, 9430, 9104] dup_rows_per_dst=[3240, 3140, 3026, 3052, 2868, 2896, 3258, 2798] combine_rows_per_src=[9336, 9338, 9294, 9308, 9052, 9164, 9430, 9104] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266584064 dispatch_meta_bytes=344520 combine_payload_bytes=266584064 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266928584 ingress_bytes=370393088 link_bytes=370393088 token_bytes=533512648 weight_movement_bytes=103809024 total_nvlink_bytes=637321672 weight_to_token_ratio=0.194577 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393759371e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64968 dup_rows=24114 dup_rows_remote=21184 dedup_ratio=0.245300 router_tokens_per_expert=[741, 764, 710, 750, 795, 769, 757, 757, 760, 772, 767, 810, 801, 756, 790, 817, 783, 760, 741, 769, 760, 751, 738, 774, 783, 779, 759, 778, 789, 751, 733, 750, 762, 772, 780, 791, 800, 729, 768, 803, 780, 772, 765, 762, 727, 756, 816, 801, 745, 792, 796, 775, 770, 738, 690, 732, 800, 751, 731, 781, 773, 767, 792, 821] payload_rows_per_dst=[9090, 9452, 9222, 9272, 9220, 9370, 9172, 9392] dup_rows_per_dst=[2996, 3094, 2930, 2972, 3190, 2988, 2904, 3040] combine_rows_per_src=[9090, 9452, 9222, 9272, 9220, 9370, 9172, 9392] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265535488 dispatch_meta_bytes=344528 combine_payload_bytes=265535488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265880016 ingress_bytes=300138496 link_bytes=300138496 token_bytes=531415504 weight_movement_bytes=34603008 total_nvlink_bytes=566018512 weight_to_token_ratio=0.065115 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129396997e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73998 payload_rows_remote=64828 dup_rows=24306 dup_rows_remote=21304 dedup_ratio=0.247253 router_tokens_per_expert=[781, 775, 757, 769, 794, 766, 800, 780, 744, 804, 775, 771, 750, 756, 747, 739, 803, 759, 732, 747, 791, 762, 771, 783, 791, 817, 729, 737, 703, 727, 756, 736, 797, 757, 749, 783, 788, 768, 792, 753, 777, 772, 747, 810, 747, 756, 750, 728, 711, 786, 775, 760, 821, 763, 778, 807, 736, 787, 789, 775, 808, 754, 762, 814] payload_rows_per_dst=[9374, 9170, 9314, 9116, 9342, 9094, 9284, 9304] dup_rows_per_dst=[3070, 3002, 2982, 2876, 3032, 3080, 3118, 3146] combine_rows_per_src=[9374, 9170, 9314, 9116, 9342, 9094, 9284, 9304] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265330688 dispatch_meta_bytes=344384 combine_payload_bytes=265330688 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265675072 ingress_bytes=369139712 link_bytes=369139712 token_bytes=531005760 weight_movement_bytes=103809024 total_nvlink_bytes=634814784 weight_to_token_ratio=0.195495 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389043018e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74224 payload_rows_remote=64948 dup_rows=24080 dup_rows_remote=21186 dedup_ratio=0.244954 router_tokens_per_expert=[767, 772, 710, 747, 737, 786, 790, 732, 748, 794, 789, 793, 749, 798, 756, 778, 774, 786, 767, 764, 783, 746, 803, 760, 759, 773, 742, 705, 796, 768, 748, 779, 763, 788, 786, 822, 774, 747, 774, 768, 811, 775, 781, 797, 786, 792, 776, 779, 770, 721, 758, 762, 778, 758, 717, 821, 798, 704, 795, 772, 735, 751, 739, 755] payload_rows_per_dst=[9180, 9280, 9282, 9230, 9392, 9442, 9276, 9142] dup_rows_per_dst=[2902, 3130, 3084, 2910, 3052, 3152, 2894, 2956] combine_rows_per_src=[9180, 9280, 9282, 9230, 9392, 9442, 9276, 9142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266330112 dispatch_meta_bytes=345136 combine_payload_bytes=266330112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266675248 ingress_bytes=300933120 link_bytes=300933120 token_bytes=533005360 weight_movement_bytes=34603008 total_nvlink_bytes=567608368 weight_to_token_ratio=0.064921 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132387103e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=264970240 dispatch_meta_bytes=342912 combine_payload_bytes=264970240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265313152 ingress_bytes=299573248 link_bytes=299573248 token_bytes=530283392 weight_movement_bytes=34603008 total_nvlink_bytes=564886400 weight_to_token_ratio=0.065254 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127270015e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=343720 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266682024 ingress_bytes=335544320 link_bytes=335544320 token_bytes=533020328 weight_movement_bytes=69206016 total_nvlink_bytes=602226344 weight_to_token_ratio=0.129837 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262626263e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74326 payload_rows_remote=65088 dup_rows=23978 dup_rows_remote=20866 dedup_ratio=0.243917 router_tokens_per_expert=[761, 751, 776, 749, 709, 749, 796, 768, 745, 785, 754, 795, 785, 706, 765, 786, 750, 774, 717, 813, 788, 727, 771, 742, 726, 781, 800, 730, 790, 726, 797, 783, 756, 773, 790, 772, 763, 833, 751, 737, 791, 773, 777, 789, 785, 796, 752, 757, 788, 784, 805, 770, 743, 768, 803, 731, 738, 802, 737, 798, 757, 767, 762, 809] payload_rows_per_dst=[9272, 9254, 9228, 9358, 9238, 9328, 9274, 9374] dup_rows_per_dst=[2846, 2988, 2936, 2908, 3112, 3112, 3110, 2966] combine_rows_per_src=[9272, 9254, 9228, 9358, 9238, 9328, 9274, 9374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=64786 dup_rows=24018 dup_rows_remote=20926 dedup_ratio=0.244324 router_tokens_per_expert=[766, 759, 736, 797, 760, 786, 736, 729, 727, 753, 767, 783, 751, 787, 755, 742, 718, 786, 761, 740, 774, 782, 786, 756, 737, 771, 762, 754, 786, 758, 756, 803, 772, 768, 805, 798, 795, 800, 729, 749, 800, 772, 747, 756, 781, 734, 774, 765, 800, 790, 755, 779, 774, 798, 753, 726, 801, 781, 779, 771, 819, 764, 760, 793] payload_rows_per_dst=[9162, 9158, 9252, 9170, 9410, 9316, 9318, 9500] dup_rows_per_dst=[2976, 2972, 2954, 3084, 3020, 2926, 2994, 3092] combine_rows_per_src=[9162, 9158, 9252, 9170, 9410, 9316, 9318, 9500] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=344608 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266453536 ingress_bytes=300711936 link_bytes=300711936 token_bytes=532562464 weight_movement_bytes=34603008 total_nvlink_bytes=567165472 weight_to_token_ratio=0.064975 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131554806e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266027008 dispatch_meta_bytes=344536 combine_payload_bytes=266027008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266371544 ingress_bytes=300630016 link_bytes=300630016 token_bytes=532398552 weight_movement_bytes=34603008 total_nvlink_bytes=567001560 weight_to_token_ratio=0.064995 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131246548e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265535488 dispatch_meta_bytes=344528 combine_payload_bytes=265535488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265880016 ingress_bytes=300138496 link_bytes=300138496 token_bytes=531415504 weight_movement_bytes=34603008 total_nvlink_bytes=566018512 weight_to_token_ratio=0.065115 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129396997e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74522 payload_rows_remote=65160 dup_rows=23782 dup_rows_remote=20832 dedup_ratio=0.241923 router_tokens_per_expert=[721, 768, 753, 774, 783, 816, 812, 783, 779, 713, 810, 782, 761, 767, 756, 764, 727, 759, 751, 773, 770, 731, 777, 774, 724, 758, 740, 787, 766, 797, 804, 743, 740, 751, 788, 732, 777, 740, 767, 795, 762, 805, 789, 731, 750, 767, 789, 763, 721, 750, 752, 808, 790, 769, 819, 780, 780, 785, 753, 786, 770, 786, 727, 807] payload_rows_per_dst=[9394, 9294, 9236, 9260, 9298, 9362, 9362, 9316] dup_rows_per_dst=[3026, 2970, 2888, 2978, 2882, 2950, 3016, 3072] combine_rows_per_src=[9394, 9294, 9236, 9260, 9298, 9362, 9362, 9316] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=64786 dup_rows=24018 dup_rows_remote=20926 dedup_ratio=0.244324 router_tokens_per_expert=[766, 759, 736, 797, 760, 786, 736, 729, 727, 753, 767, 783, 751, 787, 755, 742, 718, 786, 761, 740, 774, 782, 786, 756, 737, 771, 762, 754, 786, 758, 756, 803, 772, 768, 805, 798, 795, 800, 729, 749, 800, 772, 747, 756, 781, 734, 774, 765, 800, 790, 755, 779, 774, 798, 753, 726, 801, 781, 779, 771, 819, 764, 760, 793] payload_rows_per_dst=[9162, 9158, 9252, 9170, 9410, 9316, 9318, 9500] dup_rows_per_dst=[2976, 2972, 2954, 3084, 3020, 2926, 2994, 3092] combine_rows_per_src=[9162, 9158, 9252, 9170, 9410, 9316, 9318, 9500] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=343816 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266944264 ingress_bytes=335806464 link_bytes=335806464 token_bytes=533544712 weight_movement_bytes=69206016 total_nvlink_bytes=602750728 weight_to_token_ratio=0.129710 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263612689e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=342848 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265706304 ingress_bytes=403775488 link_bytes=403775488 token_bytes=531069760 weight_movement_bytes=138412032 total_nvlink_bytes=669481792 weight_to_token_ratio=0.260629 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519374655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74400 payload_rows_remote=65230 dup_rows=23904 dup_rows_remote=20912 dedup_ratio=0.243164 router_tokens_per_expert=[779, 791, 797, 744, 759, 737, 741, 813, 767, 728, 732, 808, 743, 720, 808, 775, 733, 771, 800, 760, 750, 794, 809, 814, 778, 726, 778, 775, 739, 789, 758, 734, 813, 740, 741, 757, 779, 776, 765, 745, 750, 779, 802, 757, 709, 826, 820, 752, 736, 777, 786, 762, 774, 781, 817, 739, 743, 779, 747, 728, 770, 811, 751, 790] payload_rows_per_dst=[9350, 9170, 9392, 9254, 9290, 9380, 9266, 9298] dup_rows_per_dst=[2972, 2992, 3070, 2900, 2942, 3010, 3078, 2940] combine_rows_per_src=[9350, 9170, 9392, 9254, 9290, 9380, 9266, 9298] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74522 payload_rows_remote=65160 dup_rows=23782 dup_rows_remote=20832 dedup_ratio=0.241923 router_tokens_per_expert=[721, 768, 753, 774, 783, 816, 812, 783, 779, 713, 810, 782, 761, 767, 756, 764, 727, 759, 751, 773, 770, 731, 777, 774, 724, 758, 740, 787, 766, 797, 804, 743, 740, 751, 788, 732, 777, 740, 767, 795, 762, 805, 789, 731, 750, 767, 789, 763, 721, 750, 752, 808, 790, 769, 819, 780, 780, 785, 753, 786, 770, 786, 727, 807] payload_rows_per_dst=[9394, 9294, 9236, 9260, 9298, 9362, 9362, 9316] dup_rows_per_dst=[3026, 2970, 2888, 2978, 2882, 2950, 3016, 3072] combine_rows_per_src=[9394, 9294, 9236, 9260, 9298, 9362, 9362, 9316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74750 payload_rows_remote=65304 dup_rows=23554 dup_rows_remote=20528 dedup_ratio=0.239604 router_tokens_per_expert=[765, 784, 767, 767, 807, 761, 786, 717, 770, 728, 768, 786, 717, 772, 761, 764, 739, 794, 762, 725, 740, 757, 772, 776, 749, 765, 756, 751, 730, 758, 788, 801, 774, 734, 813, 771, 747, 746, 761, 794, 749, 831, 761, 766, 756, 758, 783, 759, 788, 771, 816, 757, 767, 806, 802, 729, 757, 793, 769, 786, 794, 752, 812, 767] payload_rows_per_dst=[9356, 9254, 9212, 9326, 9336, 9320, 9446, 9500] dup_rows_per_dst=[2952, 2878, 2918, 2870, 2944, 3006, 3026, 2960] combine_rows_per_src=[9356, 9254, 9212, 9326, 9336, 9320, 9446, 9500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266895360 dispatch_meta_bytes=343968 combine_payload_bytes=266895360 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267239328 ingress_bytes=336101376 link_bytes=336101376 token_bytes=534134688 weight_movement_bytes=69206016 total_nvlink_bytes=603340704 weight_to_token_ratio=0.129567 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264722420e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74316 payload_rows_remote=64744 dup_rows=23988 dup_rows_remote=20746 dedup_ratio=0.244019 router_tokens_per_expert=[763, 798, 786, 827, 815, 779, 763, 806, 727, 725, 796, 772, 793, 772, 791, 746, 764, 754, 784, 767, 762, 742, 775, 760, 774, 780, 780, 752, 774, 744, 780, 710, 806, 731, 775, 757, 788, 772, 749, 766, 760, 763, 785, 761, 767, 757, 747, 760, 734, 772, 740, 778, 753, 764, 750, 737, 775, 801, 765, 750, 822, 774, 755, 777] payload_rows_per_dst=[9572, 9546, 9402, 9312, 9242, 9190, 9192, 8860] dup_rows_per_dst=[3242, 3226, 3004, 3022, 2900, 3010, 2864, 2720] combine_rows_per_src=[9572, 9546, 9402, 9312, 9242, 9190, 9192, 8860] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65058 dup_rows=23908 dup_rows_remote=20866 dedup_ratio=0.243205 router_tokens_per_expert=[776, 712, 739, 755, 759, 738, 815, 778, 785, 793, 750, 755, 804, 768, 749, 797, 727, 795, 790, 750, 784, 783, 782, 752, 763, 803, 793, 760, 779, 773, 783, 736, 775, 779, 746, 762, 746, 773, 766, 780, 709, 767, 811, 783, 768, 782, 772, 777, 783, 736, 794, 737, 751, 747, 792, 762, 798, 787, 781, 781, 732, 785, 704, 760] payload_rows_per_dst=[9220, 9362, 9326, 9338, 9276, 9296, 9284, 9294] dup_rows_per_dst=[2924, 3040, 3000, 3042, 2978, 3042, 2920, 2962] combine_rows_per_src=[9220, 9362, 9326, 9338, 9276, 9296, 9284, 9294] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=267182080 dispatch_meta_bytes=344568 combine_payload_bytes=267182080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267526648 ingress_bytes=301785088 link_bytes=301785088 token_bytes=534708728 weight_movement_bytes=34603008 total_nvlink_bytes=569311736 weight_to_token_ratio=0.064714 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135592990e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266895360 dispatch_meta_bytes=343968 combine_payload_bytes=266895360 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267239328 ingress_bytes=336101376 link_bytes=336101376 token_bytes=534134688 weight_movement_bytes=69206016 total_nvlink_bytes=603340704 weight_to_token_ratio=0.129567 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264722420e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=342848 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265706304 ingress_bytes=403775488 link_bytes=403775488 token_bytes=531069760 weight_movement_bytes=138412032 total_nvlink_bytes=669481792 weight_to_token_ratio=0.260629 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519374655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74290 payload_rows_remote=65000 dup_rows=24014 dup_rows_remote=20940 dedup_ratio=0.244283 router_tokens_per_expert=[809, 739, 760, 769, 768, 732, 767, 753, 744, 709, 752, 805, 812, 807, 777, 768, 797, 728, 772, 772, 793, 740, 815, 765, 797, 763, 755, 773, 769, 752, 773, 776, 803, 714, 778, 763, 806, 749, 769, 731, 748, 756, 801, 768, 780, 753, 778, 762, 755, 797, 764, 764, 834, 754, 735, 752, 750, 763, 771, 721, 797, 790, 753, 782] payload_rows_per_dst=[9232, 9264, 9290, 9440, 9180, 9302, 9272, 9310] dup_rows_per_dst=[2962, 3084, 3074, 2876, 3046, 2990, 3038, 2944] combine_rows_per_src=[9232, 9264, 9290, 9440, 9180, 9302, 9272, 9310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74400 payload_rows_remote=65230 dup_rows=23904 dup_rows_remote=20912 dedup_ratio=0.243164 router_tokens_per_expert=[779, 791, 797, 744, 759, 737, 741, 813, 767, 728, 732, 808, 743, 720, 808, 775, 733, 771, 800, 760, 750, 794, 809, 814, 778, 726, 778, 775, 739, 789, 758, 734, 813, 740, 741, 757, 779, 776, 765, 745, 750, 779, 802, 757, 709, 826, 820, 752, 736, 777, 786, 762, 774, 781, 817, 739, 743, 779, 747, 728, 770, 811, 751, 790] payload_rows_per_dst=[9350, 9170, 9392, 9254, 9290, 9380, 9266, 9298] dup_rows_per_dst=[2972, 2992, 3070, 2900, 2942, 3010, 3078, 2940] combine_rows_per_src=[9350, 9170, 9392, 9254, 9290, 9380, 9266, 9298] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=267485184 dispatch_meta_bytes=343328 combine_payload_bytes=267485184 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267828512 ingress_bytes=336691200 link_bytes=336691200 token_bytes=535313696 weight_movement_bytes=69206016 total_nvlink_bytes=604519712 weight_to_token_ratio=0.129281 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266941880e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74234 payload_rows_remote=65022 dup_rows=24070 dup_rows_remote=21262 dedup_ratio=0.244853 router_tokens_per_expert=[719, 747, 784, 740, 747, 758, 754, 727, 816, 771, 722, 727, 769, 767, 755, 768, 759, 749, 805, 850, 778, 730, 712, 781, 792, 712, 782, 745, 806, 806, 785, 779, 791, 734, 775, 813, 777, 771, 754, 752, 773, 764, 773, 826, 760, 862, 734, 801, 754, 729, 760, 771, 777, 740, 755, 724, 790, 765, 797, 793, 712, 783, 813, 787] payload_rows_per_dst=[9114, 9236, 9274, 9332, 9294, 9450, 9212, 9322] dup_rows_per_dst=[2838, 2954, 3054, 3082, 3040, 3136, 2808, 3158] combine_rows_per_src=[9114, 9236, 9274, 9332, 9294, 9450, 9212, 9322] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265191424 dispatch_meta_bytes=341960 combine_payload_bytes=265191424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265533384 ingress_bytes=299794432 link_bytes=299794432 token_bytes=530724808 weight_movement_bytes=34603008 total_nvlink_bytes=565327816 weight_to_token_ratio=0.065200 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128102312e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265191424 dispatch_meta_bytes=341960 combine_payload_bytes=265191424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265533384 ingress_bytes=299794432 link_bytes=299794432 token_bytes=530724808 weight_movement_bytes=34603008 total_nvlink_bytes=565327816 weight_to_token_ratio=0.065200 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128102312e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=343696 combine_payload_bytes=266477568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266821264 ingress_bytes=301080576 link_bytes=301080576 token_bytes=533298832 weight_movement_bytes=34603008 total_nvlink_bytes=567901840 weight_to_token_ratio=0.064885 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132941969e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=64876 dup_rows=23930 dup_rows_remote=20868 dedup_ratio=0.243429 router_tokens_per_expert=[779, 765, 746, 740, 720, 743, 764, 725, 761, 787, 712, 763, 801, 709, 760, 754, 790, 719, 752, 799, 750, 779, 771, 783, 764, 809, 777, 765, 835, 757, 716, 793, 746, 774, 815, 811, 782, 748, 796, 808, 732, 774, 763, 788, 728, 753, 764, 800, 725, 769, 780, 780, 787, 748, 768, 757, 779, 760, 800, 797, 795, 809, 745, 783] payload_rows_per_dst=[9204, 9096, 9262, 9328, 9498, 9268, 9262, 9456] dup_rows_per_dst=[2760, 2998, 3024, 3104, 3062, 2936, 2966, 3080] combine_rows_per_src=[9204, 9096, 9262, 9328, 9498, 9268, 9262, 9456] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74164 payload_rows_remote=64860 dup_rows=24140 dup_rows_remote=21008 dedup_ratio=0.245565 router_tokens_per_expert=[792, 760, 779, 746, 745, 791, 738, 781, 758, 801, 790, 779, 768, 775, 787, 794, 721, 751, 800, 760, 756, 774, 791, 750, 785, 769, 818, 771, 741, 776, 798, 758, 759, 725, 768, 725, 716, 792, 754, 734, 732, 776, 747, 753, 748, 734, 799, 786, 781, 741, 809, 794, 775, 797, 766, 779, 785, 765, 745, 740, 780, 768, 781, 795] payload_rows_per_dst=[9270, 9272, 9248, 8896, 9130, 9608, 9436, 9304] dup_rows_per_dst=[2950, 3232, 2998, 2772, 2816, 3188, 3052, 3132] combine_rows_per_src=[9270, 9272, 9248, 8896, 9130, 9608, 9436, 9304] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266240000 dispatch_meta_bytes=343760 combine_payload_bytes=266240000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266583760 ingress_bytes=300843008 link_bytes=300843008 token_bytes=532823760 weight_movement_bytes=34603008 total_nvlink_bytes=567426768 weight_to_token_ratio=0.064943 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132048019e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74362 payload_rows_remote=65044 dup_rows=23942 dup_rows_remote=20930 dedup_ratio=0.243551 router_tokens_per_expert=[782, 721, 799, 784, 744, 759, 814, 739, 758, 764, 762, 776, 770, 832, 756, 777, 821, 831, 756, 721, 759, 746, 782, 739, 799, 798, 798, 761, 730, 794, 769, 763, 748, 743, 738, 695, 736, 816, 743, 773, 758, 843, 740, 775, 741, 741, 773, 794, 749, 788, 726, 740, 782, 748, 801, 721, 735, 773, 744, 783, 805, 823, 746, 827] payload_rows_per_dst=[9206, 9366, 9250, 9360, 9206, 9318, 9234, 9422] dup_rows_per_dst=[3078, 3024, 3060, 3064, 2778, 3012, 2876, 3050] combine_rows_per_src=[9206, 9366, 9250, 9360, 9206, 9318, 9234, 9422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=267182080 dispatch_meta_bytes=344568 combine_payload_bytes=267182080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267526648 ingress_bytes=301785088 link_bytes=301785088 token_bytes=534708728 weight_movement_bytes=34603008 total_nvlink_bytes=569311736 weight_to_token_ratio=0.064714 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135592990e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266330112 dispatch_meta_bytes=345136 combine_payload_bytes=266330112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266675248 ingress_bytes=300933120 link_bytes=300933120 token_bytes=533005360 weight_movement_bytes=34603008 total_nvlink_bytes=567608368 weight_to_token_ratio=0.064921 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132387103e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74164 payload_rows_remote=64860 dup_rows=24140 dup_rows_remote=21008 dedup_ratio=0.245565 router_tokens_per_expert=[792, 760, 779, 746, 745, 791, 738, 781, 758, 801, 790, 779, 768, 775, 787, 794, 721, 751, 800, 760, 756, 774, 791, 750, 785, 769, 818, 771, 741, 776, 798, 758, 759, 725, 768, 725, 716, 792, 754, 734, 732, 776, 747, 753, 748, 734, 799, 786, 781, 741, 809, 794, 775, 797, 766, 779, 785, 765, 745, 740, 780, 768, 781, 795] payload_rows_per_dst=[9270, 9272, 9248, 8896, 9130, 9608, 9436, 9304] dup_rows_per_dst=[2950, 3232, 2998, 2772, 2816, 3188, 3052, 3132] combine_rows_per_src=[9270, 9272, 9248, 8896, 9130, 9608, 9436, 9304] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265732096 dispatch_meta_bytes=342976 combine_payload_bytes=265732096 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266075072 ingress_bytes=334938112 link_bytes=334938112 token_bytes=531807168 weight_movement_bytes=69206016 total_nvlink_bytes=601013184 weight_to_token_ratio=0.130134 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260345151e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265666560 dispatch_meta_bytes=343472 combine_payload_bytes=265666560 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266010032 ingress_bytes=334872576 link_bytes=334872576 token_bytes=531676592 weight_movement_bytes=69206016 total_nvlink_bytes=600882608 weight_to_token_ratio=0.130166 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260098544e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74352 payload_rows_remote=65120 dup_rows=23952 dup_rows_remote=21002 dedup_ratio=0.243652 router_tokens_per_expert=[772, 762, 750, 741, 848, 755, 738, 797, 772, 799, 749, 800, 765, 762, 723, 721, 780, 772, 790, 797, 734, 728, 789, 762, 791, 777, 797, 810, 739, 764, 845, 765, 699, 771, 755, 711, 785, 739, 769, 775, 761, 787, 741, 724, 773, 799, 765, 779, 835, 764, 812, 790, 769, 778, 749, 774, 751, 800, 754, 777, 769, 730, 676, 797] payload_rows_per_dst=[9296, 9232, 9342, 9514, 9154, 9252, 9378, 9184] dup_rows_per_dst=[3030, 2950, 2962, 3062, 2854, 3006, 3164, 2924] combine_rows_per_src=[9296, 9232, 9342, 9514, 9154, 9252, 9378, 9184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74362 payload_rows_remote=65044 dup_rows=23942 dup_rows_remote=20930 dedup_ratio=0.243551 router_tokens_per_expert=[782, 721, 799, 784, 744, 759, 814, 739, 758, 764, 762, 776, 770, 832, 756, 777, 821, 831, 756, 721, 759, 746, 782, 739, 799, 798, 798, 761, 730, 794, 769, 763, 748, 743, 738, 695, 736, 816, 743, 773, 758, 843, 740, 775, 741, 741, 773, 794, 749, 788, 726, 740, 782, 748, 801, 721, 735, 773, 744, 783, 805, 823, 746, 827] payload_rows_per_dst=[9206, 9366, 9250, 9360, 9206, 9318, 9234, 9422] dup_rows_per_dst=[3078, 3024, 3060, 3064, 2778, 3012, 2876, 3050] combine_rows_per_src=[9206, 9366, 9250, 9360, 9206, 9318, 9234, 9422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=343896 combine_payload_bytes=266420224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266764120 ingress_bytes=301023232 link_bytes=301023232 token_bytes=533184344 weight_movement_bytes=34603008 total_nvlink_bytes=567787352 weight_to_token_ratio=0.064899 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132726188e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74146 payload_rows_remote=64736 dup_rows=24158 dup_rows_remote=21042 dedup_ratio=0.245748 router_tokens_per_expert=[807, 781, 791, 720, 782, 770, 815, 777, 731, 657, 742, 740, 797, 778, 794, 764, 746, 786, 779, 779, 813, 747, 750, 782, 776, 816, 745, 760, 776, 783, 785, 749, 730, 803, 778, 788, 775, 751, 786, 762, 807, 761, 735, 759, 729, 736, 713, 785, 795, 802, 732, 776, 774, 752, 785, 847, 779, 756, 785, 745, 784, 785, 735, 704] payload_rows_per_dst=[9360, 9076, 9326, 9316, 9352, 9104, 9410, 9202] dup_rows_per_dst=[3126, 2930, 3038, 3064, 2994, 2946, 3116, 2944] combine_rows_per_src=[9360, 9076, 9326, 9316, 9352, 9104, 9410, 9202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74498 payload_rows_remote=65264 dup_rows=23806 dup_rows_remote=20722 dedup_ratio=0.242167 router_tokens_per_expert=[803, 752, 721, 708, 774, 770, 813, 818, 740, 792, 782, 712, 761, 757, 747, 773, 771, 793, 759, 804, 770, 788, 757, 750, 770, 815, 735, 728, 731, 777, 796, 773, 742, 775, 812, 824, 700, 752, 787, 779, 788, 788, 759, 743, 823, 744, 762, 791, 739, 757, 729, 778, 828, 764, 737, 791, 747, 762, 799, 760, 773, 717, 782, 780] payload_rows_per_dst=[9234, 9424, 9388, 9322, 9518, 9312, 9142, 9158] dup_rows_per_dst=[3084, 2962, 2996, 2966, 3172, 3084, 2718, 2824] combine_rows_per_src=[9234, 9424, 9388, 9322, 9518, 9312, 9142, 9158] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265666560 dispatch_meta_bytes=343472 combine_payload_bytes=265666560 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266010032 ingress_bytes=334872576 link_bytes=334872576 token_bytes=531676592 weight_movement_bytes=69206016 total_nvlink_bytes=600882608 weight_to_token_ratio=0.130166 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260098544e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74220 payload_rows_remote=65048 dup_rows=24084 dup_rows_remote=21056 dedup_ratio=0.244995 router_tokens_per_expert=[790, 734, 730, 810, 797, 754, 739, 791, 738, 783, 795, 785, 825, 744, 769, 754, 792, 764, 799, 732, 723, 740, 769, 773, 727, 733, 769, 754, 726, 835, 775, 781, 791, 778, 792, 784, 784, 751, 781, 788, 762, 756, 731, 778, 767, 762, 728, 750, 752, 739, 811, 764, 788, 709, 737, 787, 777, 759, 785, 796, 820, 789, 741, 785] payload_rows_per_dst=[9268, 9388, 9138, 9172, 9432, 9230, 9214, 9378] dup_rows_per_dst=[3022, 2998, 3046, 3028, 3066, 2838, 2960, 3126] combine_rows_per_src=[9268, 9388, 9138, 9172, 9432, 9230, 9214, 9378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=344488 combine_payload_bytes=266731520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267076008 ingress_bytes=301334528 link_bytes=301334528 token_bytes=533807528 weight_movement_bytes=34603008 total_nvlink_bytes=568410536 weight_to_token_ratio=0.064823 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133897569e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74220 payload_rows_remote=64906 dup_rows=24084 dup_rows_remote=21060 dedup_ratio=0.244995 router_tokens_per_expert=[752, 771, 789, 763, 693, 737, 796, 744, 725, 773, 804, 799, 723, 805, 750, 765, 788, 760, 720, 798, 737, 801, 780, 785, 780, 789, 750, 768, 793, 813, 788, 755, 796, 773, 718, 799, 758, 802, 748, 784, 714, 740, 777, 774, 753, 752, 767, 797, 751, 798, 816, 801, 766, 754, 782, 779, 733, 798, 725, 784, 750, 766, 749, 754] payload_rows_per_dst=[9206, 9294, 9314, 9290, 9352, 9188, 9364, 9212] dup_rows_per_dst=[2884, 2994, 3024, 3182, 3004, 2960, 3130, 2906] combine_rows_per_src=[9206, 9294, 9314, 9290, 9352, 9188, 9364, 9212] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=343896 combine_payload_bytes=266420224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266764120 ingress_bytes=301023232 link_bytes=301023232 token_bytes=533184344 weight_movement_bytes=34603008 total_nvlink_bytes=567787352 weight_to_token_ratio=0.064899 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132726188e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265158656 dispatch_meta_bytes=343112 combine_payload_bytes=265158656 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265501768 ingress_bytes=368967680 link_bytes=368967680 token_bytes=530660424 weight_movement_bytes=103809024 total_nvlink_bytes=634469448 weight_to_token_ratio=0.195622 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388395676e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74352 payload_rows_remote=65120 dup_rows=23952 dup_rows_remote=21002 dedup_ratio=0.243652 router_tokens_per_expert=[772, 762, 750, 741, 848, 755, 738, 797, 772, 799, 749, 800, 765, 762, 723, 721, 780, 772, 790, 797, 734, 728, 789, 762, 791, 777, 797, 810, 739, 764, 845, 765, 699, 771, 755, 711, 785, 739, 769, 775, 761, 787, 741, 724, 773, 799, 765, 779, 835, 764, 812, 790, 769, 778, 749, 774, 751, 800, 754, 777, 769, 730, 676, 797] payload_rows_per_dst=[9296, 9232, 9342, 9514, 9154, 9252, 9378, 9184] dup_rows_per_dst=[3030, 2950, 2962, 3062, 2854, 3006, 3164, 2924] combine_rows_per_src=[9296, 9232, 9342, 9514, 9154, 9252, 9378, 9184] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=267321344 dispatch_meta_bytes=343944 combine_payload_bytes=267321344 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267665288 ingress_bytes=336527360 link_bytes=336527360 token_bytes=534986632 weight_movement_bytes=69206016 total_nvlink_bytes=604192648 weight_to_token_ratio=0.129360 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266325363e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74750 payload_rows_remote=65304 dup_rows=23554 dup_rows_remote=20528 dedup_ratio=0.239604 router_tokens_per_expert=[765, 784, 767, 767, 807, 761, 786, 717, 770, 728, 768, 786, 717, 772, 761, 764, 739, 794, 762, 725, 740, 757, 772, 776, 749, 765, 756, 751, 730, 758, 788, 801, 774, 734, 813, 771, 747, 746, 761, 794, 749, 831, 761, 766, 756, 758, 783, 759, 788, 771, 816, 757, 767, 806, 802, 729, 757, 793, 769, 786, 794, 752, 812, 767] payload_rows_per_dst=[9356, 9254, 9212, 9326, 9336, 9320, 9446, 9500] dup_rows_per_dst=[2952, 2878, 2918, 2870, 2944, 3006, 3026, 2960] combine_rows_per_src=[9356, 9254, 9212, 9326, 9336, 9320, 9446, 9500] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=267321344 dispatch_meta_bytes=343944 combine_payload_bytes=267321344 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267665288 ingress_bytes=336527360 link_bytes=336527360 token_bytes=534986632 weight_movement_bytes=69206016 total_nvlink_bytes=604192648 weight_to_token_ratio=0.129360 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266325363e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64880 dup_rows=24068 dup_rows_remote=20948 dedup_ratio=0.244832 router_tokens_per_expert=[765, 825, 756, 748, 747, 764, 787, 753, 791, 750, 771, 764, 733, 778, 748, 746, 796, 745, 764, 746, 778, 798, 743, 717, 782, 752, 793, 794, 777, 752, 774, 724, 740, 784, 807, 736, 811, 765, 789, 806, 828, 746, 780, 749, 741, 775, 751, 759, 703, 794, 748, 794, 721, 746, 805, 744, 812, 786, 785, 797, 770, 772, 756, 791] payload_rows_per_dst=[9250, 9260, 9186, 9310, 9356, 9230, 9192, 9452] dup_rows_per_dst=[3040, 2902, 2988, 2986, 3120, 3028, 2918, 3086] combine_rows_per_src=[9250, 9260, 9186, 9310, 9356, 9230, 9192, 9452] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266436608 dispatch_meta_bytes=344416 combine_payload_bytes=266436608 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266781024 ingress_bytes=370245632 link_bytes=370245632 token_bytes=533217632 weight_movement_bytes=103809024 total_nvlink_bytes=637026656 weight_to_token_ratio=0.194684 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393204506e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74332 payload_rows_remote=64820 dup_rows=23972 dup_rows_remote=20824 dedup_ratio=0.243856 router_tokens_per_expert=[781, 761, 750, 778, 780, 764, 778, 708, 727, 745, 780, 764, 749, 761, 798, 802, 768, 795, 805, 795, 721, 771, 775, 748, 792, 805, 821, 768, 827, 747, 781, 734, 766, 764, 735, 760, 814, 762, 783, 780, 795, 774, 705, 721, 819, 802, 771, 780, 796, 773, 815, 788, 704, 741, 743, 784, 774, 706, 766, 768, 797, 721, 707, 759] payload_rows_per_dst=[9278, 9246, 8954, 9268, 9370, 9288, 9416, 9512] dup_rows_per_dst=[2922, 2954, 2738, 3026, 2984, 3068, 3132, 3148] combine_rows_per_src=[9278, 9246, 8954, 9268, 9370, 9288, 9416, 9512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74188 payload_rows_remote=64882 dup_rows=24116 dup_rows_remote=21102 dedup_ratio=0.245321 router_tokens_per_expert=[814, 746, 791, 746, 772, 791, 765, 794, 749, 757, 767, 808, 758, 739, 754, 794, 801, 772, 784, 733, 777, 787, 760, 765, 761, 704, 785, 786, 713, 808, 759, 788, 748, 798, 758, 692, 785, 796, 770, 802, 795, 736, 763, 758, 746, 764, 816, 782, 757, 788, 758, 775, 780, 767, 774, 727, 756, 755, 730, 790, 741, 796, 734, 787] payload_rows_per_dst=[9270, 9268, 9304, 9202, 9254, 9306, 9262, 9322] dup_rows_per_dst=[3168, 2984, 3054, 3006, 3044, 3014, 2990, 2856] combine_rows_per_src=[9270, 9268, 9304, 9202, 9254, 9306, 9262, 9322] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265854976 dispatch_meta_bytes=343864 combine_payload_bytes=265854976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266198840 ingress_bytes=300457984 link_bytes=300457984 token_bytes=532053816 weight_movement_bytes=34603008 total_nvlink_bytes=566656824 weight_to_token_ratio=0.065037 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130599205e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=344488 combine_payload_bytes=266731520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267076008 ingress_bytes=301334528 link_bytes=301334528 token_bytes=533807528 weight_movement_bytes=34603008 total_nvlink_bytes=568410536 weight_to_token_ratio=0.064823 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133897569e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=267485184 dispatch_meta_bytes=343328 combine_payload_bytes=267485184 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267828512 ingress_bytes=336691200 link_bytes=336691200 token_bytes=535313696 weight_movement_bytes=69206016 total_nvlink_bytes=604519712 weight_to_token_ratio=0.129281 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266941880e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74332 payload_rows_remote=64820 dup_rows=23972 dup_rows_remote=20824 dedup_ratio=0.243856 router_tokens_per_expert=[781, 761, 750, 778, 780, 764, 778, 708, 727, 745, 780, 764, 749, 761, 798, 802, 768, 795, 805, 795, 721, 771, 775, 748, 792, 805, 821, 768, 827, 747, 781, 734, 766, 764, 735, 760, 814, 762, 783, 780, 795, 774, 705, 721, 819, 802, 771, 780, 796, 773, 815, 788, 704, 741, 743, 784, 774, 706, 766, 768, 797, 721, 707, 759] payload_rows_per_dst=[9278, 9246, 8954, 9268, 9370, 9288, 9416, 9512] dup_rows_per_dst=[2922, 2954, 2738, 3026, 2984, 3068, 3132, 3148] combine_rows_per_src=[9278, 9246, 8954, 9268, 9370, 9288, 9416, 9512] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265748480 dispatch_meta_bytes=343312 combine_payload_bytes=265748480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266091792 ingress_bytes=300351488 link_bytes=300351488 token_bytes=531840272 weight_movement_bytes=34603008 total_nvlink_bytes=566443280 weight_to_token_ratio=0.065063 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130198469e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=342576 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265845296 ingress_bytes=300105728 link_bytes=300105728 token_bytes=531348016 weight_movement_bytes=34603008 total_nvlink_bytes=565951024 weight_to_token_ratio=0.065123 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129273694e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74434 payload_rows_remote=65154 dup_rows=23870 dup_rows_remote=20970 dedup_ratio=0.242818 router_tokens_per_expert=[784, 753, 746, 796, 780, 781, 786, 824, 739, 763, 667, 772, 767, 786, 779, 817, 797, 758, 755, 798, 742, 766, 760, 806, 724, 804, 833, 745, 774, 712, 701, 792, 767, 752, 724, 769, 772, 798, 742, 792, 780, 749, 800, 765, 744, 800, 770, 785, 778, 766, 737, 784, 793, 759, 774, 773, 742, 741, 788, 780, 773, 761, 758, 729] payload_rows_per_dst=[9402, 9280, 9348, 9200, 9330, 9368, 9270, 9236] dup_rows_per_dst=[3098, 2900, 3016, 2970, 2902, 3018, 3058, 2908] combine_rows_per_src=[9402, 9280, 9348, 9200, 9330, 9368, 9270, 9236] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74188 payload_rows_remote=64882 dup_rows=24116 dup_rows_remote=21102 dedup_ratio=0.245321 router_tokens_per_expert=[814, 746, 791, 746, 772, 791, 765, 794, 749, 757, 767, 808, 758, 739, 754, 794, 801, 772, 784, 733, 777, 787, 760, 765, 761, 704, 785, 786, 713, 808, 759, 788, 748, 798, 758, 692, 785, 796, 770, 802, 795, 736, 763, 758, 746, 764, 816, 782, 757, 788, 758, 775, 780, 767, 774, 727, 756, 755, 730, 790, 741, 796, 734, 787] payload_rows_per_dst=[9270, 9268, 9304, 9202, 9254, 9306, 9262, 9322] dup_rows_per_dst=[3168, 2984, 3054, 3006, 3044, 3014, 2990, 2856] combine_rows_per_src=[9270, 9268, 9304, 9202, 9254, 9306, 9262, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=343936 combine_payload_bytes=265756672 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266100608 ingress_bytes=334962688 link_bytes=334962688 token_bytes=531857280 weight_movement_bytes=69206016 total_nvlink_bytes=601063296 weight_to_token_ratio=0.130121 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260437628e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74156 payload_rows_remote=64952 dup_rows=24148 dup_rows_remote=21192 dedup_ratio=0.245646 router_tokens_per_expert=[784, 833, 774, 752, 766, 774, 771, 768, 744, 750, 801, 836, 790, 758, 742, 804, 783, 716, 754, 785, 757, 781, 773, 761, 759, 777, 784, 753, 726, 791, 800, 768, 769, 758, 779, 813, 747, 774, 766, 736, 765, 741, 755, 741, 804, 779, 725, 756, 761, 797, 723, 766, 780, 764, 756, 733, 769, 735, 770, 774, 750, 803, 779, 769] payload_rows_per_dst=[9408, 9382, 9200, 9272, 9228, 9190, 9204, 9272] dup_rows_per_dst=[3036, 3068, 3020, 3044, 3056, 2942, 2956, 3026] combine_rows_per_src=[9408, 9382, 9200, 9272, 9228, 9190, 9204, 9272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74368 payload_rows_remote=65074 dup_rows=23936 dup_rows_remote=20992 dedup_ratio=0.243490 router_tokens_per_expert=[717, 752, 783, 786, 747, 750, 768, 773, 734, 774, 759, 791, 781, 744, 782, 760, 745, 754, 775, 783, 747, 804, 834, 751, 749, 761, 840, 792, 773, 759, 772, 723, 767, 781, 773, 753, 776, 778, 801, 776, 771, 775, 811, 765, 726, 698, 747, 813, 717, 787, 793, 801, 767, 762, 809, 756, 745, 727, 775, 760, 753, 742, 832, 752] payload_rows_per_dst=[9294, 9286, 9432, 9286, 9194, 9222, 9404, 9250] dup_rows_per_dst=[2944, 2964, 3256, 3052, 2914, 2904, 2980, 2922] combine_rows_per_src=[9294, 9286, 9432, 9286, 9194, 9222, 9404, 9250] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=342576 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265845296 ingress_bytes=300105728 link_bytes=300105728 token_bytes=531348016 weight_movement_bytes=34603008 total_nvlink_bytes=565951024 weight_to_token_ratio=0.065123 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129273694e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74654 payload_rows_remote=65222 dup_rows=23650 dup_rows_remote=20610 dedup_ratio=0.240580 router_tokens_per_expert=[785, 782, 725, 741, 751, 753, 751, 782, 794, 764, 725, 742, 799, 731, 786, 754, 788, 779, 790, 779, 750, 808, 778, 734, 775, 805, 795, 758, 801, 763, 789, 750, 802, 793, 759, 770, 788, 755, 779, 752, 778, 740, 758, 809, 765, 799, 787, 770, 765, 780, 749, 756, 779, 733, 756, 716, 770, 735, 780, 751, 736, 749, 811, 775] payload_rows_per_dst=[9274, 9252, 9442, 9432, 9434, 9400, 9154, 9266] dup_rows_per_dst=[2866, 2938, 2970, 3040, 2962, 3012, 2914, 2948] combine_rows_per_src=[9274, 9252, 9442, 9432, 9434, 9400, 9154, 9266] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266870784 dispatch_meta_bytes=344496 combine_payload_bytes=266870784 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267215280 ingress_bytes=370679808 link_bytes=370679808 token_bytes=534086064 weight_movement_bytes=103809024 total_nvlink_bytes=637895088 weight_to_token_ratio=0.194368 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394838275e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=343936 combine_payload_bytes=265756672 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266100608 ingress_bytes=334962688 link_bytes=334962688 token_bytes=531857280 weight_movement_bytes=69206016 total_nvlink_bytes=601063296 weight_to_token_ratio=0.130121 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260437628e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74184 payload_rows_remote=65040 dup_rows=24120 dup_rows_remote=21150 dedup_ratio=0.245361 router_tokens_per_expert=[715, 782, 745, 759, 763, 767, 772, 783, 753, 728, 785, 748, 800, 764, 821, 740, 744, 775, 786, 782, 743, 741, 764, 722, 817, 764, 825, 727, 797, 768, 789, 762, 796, 793, 798, 775, 750, 741, 770, 779, 762, 756, 764, 770, 758, 798, 790, 760, 788, 803, 774, 704, 745, 733, 771, 754, 773, 774, 760, 777, 801, 794, 765, 745] payload_rows_per_dst=[9212, 9288, 9144, 9424, 9334, 9360, 9118, 9304] dup_rows_per_dst=[2960, 2990, 2970, 3074, 3070, 2956, 3026, 3074] combine_rows_per_src=[9212, 9288, 9144, 9424, 9334, 9360, 9118, 9304] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=344264 combine_payload_bytes=266543104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266887368 ingress_bytes=301146112 link_bytes=301146112 token_bytes=533430472 weight_movement_bytes=34603008 total_nvlink_bytes=568033480 weight_to_token_ratio=0.064869 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133188575e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266043392 dispatch_meta_bytes=344576 combine_payload_bytes=266043392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266387968 ingress_bytes=300646400 link_bytes=300646400 token_bytes=532431360 weight_movement_bytes=34603008 total_nvlink_bytes=567034368 weight_to_token_ratio=0.064991 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131308199e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74434 payload_rows_remote=65154 dup_rows=23870 dup_rows_remote=20970 dedup_ratio=0.242818 router_tokens_per_expert=[784, 753, 746, 796, 780, 781, 786, 824, 739, 763, 667, 772, 767, 786, 779, 817, 797, 758, 755, 798, 742, 766, 760, 806, 724, 804, 833, 745, 774, 712, 701, 792, 767, 752, 724, 769, 772, 798, 742, 792, 780, 749, 800, 765, 744, 800, 770, 785, 778, 766, 737, 784, 793, 759, 774, 773, 742, 741, 788, 780, 773, 761, 758, 729] payload_rows_per_dst=[9402, 9280, 9348, 9200, 9330, 9368, 9270, 9236] dup_rows_per_dst=[3098, 2900, 3016, 2970, 2902, 3018, 3058, 2908] combine_rows_per_src=[9402, 9280, 9348, 9200, 9330, 9368, 9270, 9236] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74146 payload_rows_remote=64736 dup_rows=24158 dup_rows_remote=21042 dedup_ratio=0.245748 router_tokens_per_expert=[807, 781, 791, 720, 782, 770, 815, 777, 731, 657, 742, 740, 797, 778, 794, 764, 746, 786, 779, 779, 813, 747, 750, 782, 776, 816, 745, 760, 776, 783, 785, 749, 730, 803, 778, 788, 775, 751, 786, 762, 807, 761, 735, 759, 729, 736, 713, 785, 795, 802, 732, 776, 774, 752, 785, 847, 779, 756, 785, 745, 784, 785, 735, 704] payload_rows_per_dst=[9360, 9076, 9326, 9316, 9352, 9104, 9410, 9202] dup_rows_per_dst=[3126, 2930, 3038, 3064, 2994, 2946, 3116, 2944] combine_rows_per_src=[9360, 9076, 9326, 9316, 9352, 9104, 9410, 9202] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65160 dup_rows=23924 dup_rows_remote=20916 dedup_ratio=0.243368 router_tokens_per_expert=[746, 748, 738, 748, 747, 774, 803, 782, 775, 780, 776, 747, 784, 820, 776, 794, 800, 780, 779, 722, 776, 768, 794, 809, 744, 775, 758, 770, 723, 751, 740, 743, 802, 694, 753, 771, 771, 745, 783, 795, 781, 727, 795, 738, 757, 757, 838, 735, 803, 750, 792, 756, 739, 835, 777, 758, 754, 756, 735, 776, 758, 796, 781, 774] payload_rows_per_dst=[9174, 9434, 9352, 9116, 9220, 9364, 9396, 9324] dup_rows_per_dst=[2998, 3070, 3104, 2892, 3008, 2892, 3024, 2936] combine_rows_per_src=[9174, 9434, 9352, 9116, 9220, 9364, 9396, 9324] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74200 payload_rows_remote=64856 dup_rows=24104 dup_rows_remote=21042 dedup_ratio=0.245199 router_tokens_per_expert=[732, 758, 741, 790, 722, 769, 819, 780, 785, 726, 759, 813, 719, 769, 782, 786, 788, 803, 816, 759, 768, 723, 789, 724, 794, 753, 781, 787, 758, 787, 742, 794, 778, 763, 678, 780, 798, 784, 763, 763, 847, 731, 746, 795, 745, 764, 738, 757, 753, 781, 744, 740, 789, 805, 766, 725, 747, 791, 772, 785, 816, 770, 790, 732] payload_rows_per_dst=[8926, 9168, 9352, 9176, 9340, 9388, 9506, 9344] dup_rows_per_dst=[2684, 2844, 3104, 2988, 3140, 3148, 3134, 3062] combine_rows_per_src=[8926, 9168, 9352, 9176, 9340, 9388, 9506, 9344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=344264 combine_payload_bytes=266543104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266887368 ingress_bytes=301146112 link_bytes=301146112 token_bytes=533430472 weight_movement_bytes=34603008 total_nvlink_bytes=568033480 weight_to_token_ratio=0.064869 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133188575e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65160 dup_rows=23924 dup_rows_remote=20916 dedup_ratio=0.243368 router_tokens_per_expert=[746, 748, 738, 748, 747, 774, 803, 782, 775, 780, 776, 747, 784, 820, 776, 794, 800, 780, 779, 722, 776, 768, 794, 809, 744, 775, 758, 770, 723, 751, 740, 743, 802, 694, 753, 771, 771, 745, 783, 795, 781, 727, 795, 738, 757, 757, 838, 735, 803, 750, 792, 756, 739, 835, 777, 758, 754, 756, 735, 776, 758, 796, 781, 774] payload_rows_per_dst=[9174, 9434, 9352, 9116, 9220, 9364, 9396, 9324] dup_rows_per_dst=[2998, 3070, 3104, 2892, 3008, 2892, 3024, 2936] combine_rows_per_src=[9174, 9434, 9352, 9116, 9220, 9364, 9396, 9324] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267149312 dispatch_meta_bytes=343328 combine_payload_bytes=267149312 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267492640 ingress_bytes=336355328 link_bytes=336355328 token_bytes=534641952 weight_movement_bytes=69206016 total_nvlink_bytes=603847968 weight_to_token_ratio=0.129444 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265678020e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74126 payload_rows_remote=64854 dup_rows=24178 dup_rows_remote=21384 dedup_ratio=0.245951 router_tokens_per_expert=[770, 783, 814, 790, 767, 767, 777, 757, 798, 747, 768, 720, 824, 740, 791, 772, 734, 782, 762, 733, 766, 761, 782, 760, 771, 758, 746, 791, 800, 765, 803, 774, 768, 836, 748, 787, 790, 743, 759, 778, 735, 743, 769, 749, 760, 752, 784, 741, 738, 767, 786, 778, 752, 766, 759, 729, 745, 749, 766, 776, 775, 784, 769, 798] payload_rows_per_dst=[9334, 9234, 9218, 9294, 9298, 9272, 9182, 9294] dup_rows_per_dst=[3116, 3086, 2942, 3122, 3120, 2794, 2968, 3030] combine_rows_per_src=[9334, 9234, 9218, 9294, 9298, 9272, 9182, 9294] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266403840 dispatch_meta_bytes=344760 combine_payload_bytes=266403840 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266748600 ingress_bytes=335609856 link_bytes=335609856 token_bytes=533152440 weight_movement_bytes=69206016 total_nvlink_bytes=602358456 weight_to_token_ratio=0.129805 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262872869e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265158656 dispatch_meta_bytes=343112 combine_payload_bytes=265158656 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265501768 ingress_bytes=368967680 link_bytes=368967680 token_bytes=530660424 weight_movement_bytes=103809024 total_nvlink_bytes=634469448 weight_to_token_ratio=0.195622 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388395676e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266870784 dispatch_meta_bytes=344496 combine_payload_bytes=266870784 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267215280 ingress_bytes=370679808 link_bytes=370679808 token_bytes=534086064 weight_movement_bytes=103809024 total_nvlink_bytes=637895088 weight_to_token_ratio=0.194368 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394838275e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74200 payload_rows_remote=64856 dup_rows=24104 dup_rows_remote=21042 dedup_ratio=0.245199 router_tokens_per_expert=[732, 758, 741, 790, 722, 769, 819, 780, 785, 726, 759, 813, 719, 769, 782, 786, 788, 803, 816, 759, 768, 723, 789, 724, 794, 753, 781, 787, 758, 787, 742, 794, 778, 763, 678, 780, 798, 784, 763, 763, 847, 731, 746, 795, 745, 764, 738, 757, 753, 781, 744, 740, 789, 805, 766, 725, 747, 791, 772, 785, 816, 770, 790, 732] payload_rows_per_dst=[8926, 9168, 9352, 9176, 9340, 9388, 9506, 9344] dup_rows_per_dst=[2684, 2844, 3104, 2988, 3140, 3148, 3134, 3062] combine_rows_per_src=[8926, 9168, 9352, 9176, 9340, 9388, 9506, 9344] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266895360 dispatch_meta_bytes=344304 combine_payload_bytes=266895360 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267239664 ingress_bytes=336101376 link_bytes=336101376 token_bytes=534135024 weight_movement_bytes=69206016 total_nvlink_bytes=603341040 weight_to_token_ratio=0.129567 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264722420e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265641984 dispatch_meta_bytes=344952 combine_payload_bytes=265641984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265986936 ingress_bytes=300244992 link_bytes=300244992 token_bytes=531628920 weight_movement_bytes=34603008 total_nvlink_bytes=566231928 weight_to_token_ratio=0.065089 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129797733e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74532 payload_rows_remote=65164 dup_rows=23772 dup_rows_remote=20878 dedup_ratio=0.241821 router_tokens_per_expert=[753, 756, 738, 810, 798, 783, 770, 792, 772, 787, 715, 779, 770, 780, 752, 797, 765, 753, 760, 788, 722, 788, 791, 773, 750, 798, 765, 800, 782, 766, 810, 751, 753, 757, 805, 772, 736, 744, 768, 753, 776, 710, 757, 783, 781, 776, 777, 758, 743, 783, 755, 780, 767, 740, 802, 761, 767, 771, 736, 782, 751, 727, 772, 795] payload_rows_per_dst=[9280, 9240, 9308, 9446, 9188, 9400, 9368, 9302] dup_rows_per_dst=[3120, 3064, 2972, 2998, 2988, 2836, 2894, 2900] combine_rows_per_src=[9280, 9240, 9308, 9446, 9188, 9400, 9368, 9302] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265650176 dispatch_meta_bytes=343592 combine_payload_bytes=265650176 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265993768 ingress_bytes=334856192 link_bytes=334856192 token_bytes=531643944 weight_movement_bytes=69206016 total_nvlink_bytes=600849960 weight_to_token_ratio=0.130174 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260036892e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65240 dup_rows=23944 dup_rows_remote=20994 dedup_ratio=0.243571 router_tokens_per_expert=[788, 779, 711, 770, 763, 807, 785, 763, 748, 774, 763, 760, 775, 748, 748, 719, 774, 770, 771, 814, 755, 790, 785, 815, 783, 785, 742, 756, 784, 751, 741, 802, 767, 788, 782, 775, 781, 812, 779, 780, 786, 743, 734, 770, 772, 775, 768, 737, 749, 781, 739, 754, 730, 754, 724, 749, 727, 797, 764, 767, 797, 788, 778, 786] payload_rows_per_dst=[9342, 9120, 9386, 9338, 9406, 9232, 9142, 9394] dup_rows_per_dst=[2990, 2950, 3162, 2950, 3122, 2938, 2818, 3014] combine_rows_per_src=[9342, 9120, 9386, 9338, 9406, 9232, 9142, 9394] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74126 payload_rows_remote=64854 dup_rows=24178 dup_rows_remote=21384 dedup_ratio=0.245951 router_tokens_per_expert=[770, 783, 814, 790, 767, 767, 777, 757, 798, 747, 768, 720, 824, 740, 791, 772, 734, 782, 762, 733, 766, 761, 782, 760, 771, 758, 746, 791, 800, 765, 803, 774, 768, 836, 748, 787, 790, 743, 759, 778, 735, 743, 769, 749, 760, 752, 784, 741, 738, 767, 786, 778, 752, 766, 759, 729, 745, 749, 766, 776, 775, 784, 769, 798] payload_rows_per_dst=[9334, 9234, 9218, 9294, 9298, 9272, 9182, 9294] dup_rows_per_dst=[3116, 3086, 2942, 3122, 3120, 2794, 2968, 3030] combine_rows_per_src=[9334, 9234, 9218, 9294, 9298, 9272, 9182, 9294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74238 payload_rows_remote=64784 dup_rows=24066 dup_rows_remote=21044 dedup_ratio=0.244812 router_tokens_per_expert=[801, 806, 764, 803, 739, 760, 793, 758, 771, 764, 745, 769, 793, 779, 787, 791, 746, 719, 759, 777, 756, 798, 777, 815, 781, 791, 758, 734, 779, 755, 729, 771, 801, 740, 728, 753, 765, 793, 757, 737, 774, 799, 755, 793, 721, 789, 757, 786, 801, 793, 806, 761, 718, 725, 810, 745, 770, 759, 741, 750, 769, 727, 780, 781] payload_rows_per_dst=[9454, 9438, 9314, 9212, 9288, 9246, 9196, 9090] dup_rows_per_dst=[3022, 3048, 2980, 2984, 2860, 3102, 3122, 2948] combine_rows_per_src=[9454, 9438, 9314, 9212, 9288, 9246, 9196, 9090] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267223040 dispatch_meta_bytes=344936 combine_payload_bytes=267223040 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267567976 ingress_bytes=336429056 link_bytes=336429056 token_bytes=534791016 weight_movement_bytes=69206016 total_nvlink_bytes=603997032 weight_to_token_ratio=0.129408 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265955453e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265641984 dispatch_meta_bytes=344952 combine_payload_bytes=265641984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265986936 ingress_bytes=300244992 link_bytes=300244992 token_bytes=531628920 weight_movement_bytes=34603008 total_nvlink_bytes=566231928 weight_to_token_ratio=0.065089 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129797733e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=65058 dup_rows=24040 dup_rows_remote=21026 dedup_ratio=0.244548 router_tokens_per_expert=[761, 771, 778, 749, 736, 715, 791, 824, 795, 813, 781, 748, 767, 774, 797, 781, 759, 790, 772, 816, 724, 813, 742, 740, 799, 770, 783, 701, 760, 735, 744, 818, 774, 754, 788, 833, 772, 725, 733, 714, 777, 749, 792, 762, 779, 789, 749, 775, 777, 738, 752, 738, 715, 743, 792, 783, 774, 808, 780, 788, 755, 773, 753, 771] payload_rows_per_dst=[9302, 9370, 9356, 9206, 9236, 9252, 9214, 9328] dup_rows_per_dst=[2948, 3142, 2956, 3014, 2950, 3092, 2862, 3076] combine_rows_per_src=[9302, 9370, 9356, 9206, 9236, 9252, 9214, 9328] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265355264 dispatch_meta_bytes=343312 combine_payload_bytes=265355264 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265698576 ingress_bytes=403767296 link_bytes=403767296 token_bytes=531053840 weight_movement_bytes=138412032 total_nvlink_bytes=669465872 weight_to_token_ratio=0.260637 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519343829e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266911744 dispatch_meta_bytes=344168 combine_payload_bytes=266911744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267255912 ingress_bytes=301514752 link_bytes=301514752 token_bytes=534167656 weight_movement_bytes=34603008 total_nvlink_bytes=568770664 weight_to_token_ratio=0.064779 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134575738e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74108 payload_rows_remote=65048 dup_rows=24196 dup_rows_remote=21142 dedup_ratio=0.246134 router_tokens_per_expert=[830, 750, 759, 738, 761, 767, 792, 791, 734, 774, 795, 792, 767, 740, 782, 777, 795, 795, 754, 718, 763, 731, 757, 744, 781, 773, 745, 779, 745, 816, 775, 779, 759, 785, 817, 759, 750, 775, 737, 745, 756, 790, 727, 800, 750, 780, 788, 799, 768, 774, 761, 794, 763, 792, 834, 768, 736, 778, 731, 717, 723, 797, 715, 785] payload_rows_per_dst=[9344, 9308, 9060, 9344, 9234, 9286, 9424, 9108] dup_rows_per_dst=[3032, 3014, 3054, 3042, 3020, 3094, 3084, 2856] combine_rows_per_src=[9344, 9308, 9060, 9344, 9234, 9286, 9424, 9108] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65240 dup_rows=23944 dup_rows_remote=20994 dedup_ratio=0.243571 router_tokens_per_expert=[788, 779, 711, 770, 763, 807, 785, 763, 748, 774, 763, 760, 775, 748, 748, 719, 774, 770, 771, 814, 755, 790, 785, 815, 783, 785, 742, 756, 784, 751, 741, 802, 767, 788, 782, 775, 781, 812, 779, 780, 786, 743, 734, 770, 772, 775, 768, 737, 749, 781, 739, 754, 730, 754, 724, 749, 727, 797, 764, 767, 797, 788, 778, 786] payload_rows_per_dst=[9342, 9120, 9386, 9338, 9406, 9232, 9142, 9394] dup_rows_per_dst=[2990, 2950, 3162, 2950, 3122, 2938, 2818, 3014] combine_rows_per_src=[9342, 9120, 9386, 9338, 9406, 9232, 9142, 9394] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74310 payload_rows_remote=65118 dup_rows=23994 dup_rows_remote=21072 dedup_ratio=0.244080 router_tokens_per_expert=[746, 771, 772, 802, 740, 776, 809, 789, 762, 738, 756, 798, 813, 732, 776, 764, 717, 730, 723, 767, 789, 789, 741, 814, 762, 795, 755, 761, 725, 770, 783, 744, 779, 730, 732, 766, 693, 768, 825, 764, 764, 711, 749, 739, 792, 768, 792, 797, 783, 746, 827, 769, 739, 755, 773, 803, 776, 782, 802, 803, 782, 769, 778, 787] payload_rows_per_dst=[9416, 9278, 9102, 9238, 9192, 9216, 9350, 9518] dup_rows_per_dst=[2994, 3000, 3038, 2952, 2922, 3008, 3040, 3040] combine_rows_per_src=[9416, 9278, 9102, 9238, 9192, 9216, 9350, 9518] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74156 payload_rows_remote=64952 dup_rows=24148 dup_rows_remote=21192 dedup_ratio=0.245646 router_tokens_per_expert=[784, 833, 774, 752, 766, 774, 771, 768, 744, 750, 801, 836, 790, 758, 742, 804, 783, 716, 754, 785, 757, 781, 773, 761, 759, 777, 784, 753, 726, 791, 800, 768, 769, 758, 779, 813, 747, 774, 766, 736, 765, 741, 755, 741, 804, 779, 725, 756, 761, 797, 723, 766, 780, 764, 756, 733, 769, 735, 770, 774, 750, 803, 779, 769] payload_rows_per_dst=[9408, 9382, 9200, 9272, 9228, 9190, 9204, 9272] dup_rows_per_dst=[3036, 3068, 3020, 3044, 3056, 2942, 2956, 3026] combine_rows_per_src=[9408, 9382, 9200, 9272, 9228, 9190, 9204, 9272] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74026 payload_rows_remote=64668 dup_rows=24278 dup_rows_remote=21032 dedup_ratio=0.246969 router_tokens_per_expert=[770, 761, 763, 738, 748, 741, 755, 865, 730, 786, 753, 799, 744, 704, 834, 717, 763, 746, 751, 711, 744, 796, 773, 787, 765, 735, 746, 781, 755, 770, 785, 785, 779, 815, 758, 760, 805, 769, 780, 739, 791, 749, 777, 790, 794, 756, 817, 791, 749, 760, 768, 782, 810, 769, 792, 805, 739, 739, 739, 725, 785, 799, 746, 774] payload_rows_per_dst=[8918, 9132, 9436, 9276, 9132, 9316, 9458, 9358] dup_rows_per_dst=[2860, 2910, 3210, 3058, 2766, 3068, 3160, 3246] combine_rows_per_src=[8918, 9132, 9436, 9276, 9132, 9316, 9458, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265355264 dispatch_meta_bytes=343312 combine_payload_bytes=265355264 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265698576 ingress_bytes=403767296 link_bytes=403767296 token_bytes=531053840 weight_movement_bytes=138412032 total_nvlink_bytes=669465872 weight_to_token_ratio=0.260637 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519343829e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=344336 combine_payload_bytes=266477568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266821904 ingress_bytes=335683584 link_bytes=335683584 token_bytes=533299472 weight_movement_bytes=69206016 total_nvlink_bytes=602505488 weight_to_token_ratio=0.129770 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263150302e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74310 payload_rows_remote=65118 dup_rows=23994 dup_rows_remote=21072 dedup_ratio=0.244080 router_tokens_per_expert=[746, 771, 772, 802, 740, 776, 809, 789, 762, 738, 756, 798, 813, 732, 776, 764, 717, 730, 723, 767, 789, 789, 741, 814, 762, 795, 755, 761, 725, 770, 783, 744, 779, 730, 732, 766, 693, 768, 825, 764, 764, 711, 749, 739, 792, 768, 792, 797, 783, 746, 827, 769, 739, 755, 773, 803, 776, 782, 802, 803, 782, 769, 778, 787] payload_rows_per_dst=[9416, 9278, 9102, 9238, 9192, 9216, 9350, 9518] dup_rows_per_dst=[2994, 3000, 3038, 2952, 2922, 3008, 3040, 3040] combine_rows_per_src=[9416, 9278, 9102, 9238, 9192, 9216, 9350, 9518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74464 payload_rows_remote=65140 dup_rows=23840 dup_rows_remote=20886 dedup_ratio=0.242513 router_tokens_per_expert=[816, 755, 771, 742, 794, 757, 769, 823, 793, 728, 794, 780, 773, 769, 826, 733, 717, 732, 748, 810, 730, 755, 780, 745, 759, 797, 752, 772, 811, 757, 763, 792, 776, 775, 739, 833, 786, 784, 795, 759, 790, 744, 765, 793, 715, 779, 765, 788, 741, 756, 784, 717, 753, 753, 744, 723, 755, 728, 767, 789, 778, 787, 776, 772] payload_rows_per_dst=[9454, 9290, 9184, 9380, 9404, 9324, 9092, 9336] dup_rows_per_dst=[3000, 3102, 2850, 3026, 3090, 2954, 2850, 2968] combine_rows_per_src=[9454, 9290, 9184, 9380, 9404, 9324, 9092, 9336] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266436608 dispatch_meta_bytes=344760 combine_payload_bytes=266436608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266781368 ingress_bytes=301039616 link_bytes=301039616 token_bytes=533217976 weight_movement_bytes=34603008 total_nvlink_bytes=567820984 weight_to_token_ratio=0.064895 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132787839e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267223040 dispatch_meta_bytes=344936 combine_payload_bytes=267223040 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267567976 ingress_bytes=336429056 link_bytes=336429056 token_bytes=534791016 weight_movement_bytes=69206016 total_nvlink_bytes=603997032 weight_to_token_ratio=0.129408 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265955453e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74026 payload_rows_remote=64668 dup_rows=24278 dup_rows_remote=21032 dedup_ratio=0.246969 router_tokens_per_expert=[770, 761, 763, 738, 748, 741, 755, 865, 730, 786, 753, 799, 744, 704, 834, 717, 763, 746, 751, 711, 744, 796, 773, 787, 765, 735, 746, 781, 755, 770, 785, 785, 779, 815, 758, 760, 805, 769, 780, 739, 791, 749, 777, 790, 794, 756, 817, 791, 749, 760, 768, 782, 810, 769, 792, 805, 739, 739, 739, 725, 785, 799, 746, 774] payload_rows_per_dst=[8918, 9132, 9436, 9276, 9132, 9316, 9458, 9358] dup_rows_per_dst=[2860, 2910, 3210, 3058, 2766, 3068, 3160, 3246] combine_rows_per_src=[8918, 9132, 9436, 9276, 9132, 9316, 9458, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266723328 dispatch_meta_bytes=344760 combine_payload_bytes=266723328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267068088 ingress_bytes=301326336 link_bytes=301326336 token_bytes=533791416 weight_movement_bytes=34603008 total_nvlink_bytes=568394424 weight_to_token_ratio=0.064825 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133866744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266043392 dispatch_meta_bytes=344576 combine_payload_bytes=266043392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266387968 ingress_bytes=300646400 link_bytes=300646400 token_bytes=532431360 weight_movement_bytes=34603008 total_nvlink_bytes=567034368 weight_to_token_ratio=0.064991 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131308199e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74464 payload_rows_remote=65140 dup_rows=23840 dup_rows_remote=20886 dedup_ratio=0.242513 router_tokens_per_expert=[816, 755, 771, 742, 794, 757, 769, 823, 793, 728, 794, 780, 773, 769, 826, 733, 717, 732, 748, 810, 730, 755, 780, 745, 759, 797, 752, 772, 811, 757, 763, 792, 776, 775, 739, 833, 786, 784, 795, 759, 790, 744, 765, 793, 715, 779, 765, 788, 741, 756, 784, 717, 753, 753, 744, 723, 755, 728, 767, 789, 778, 787, 776, 772] payload_rows_per_dst=[9454, 9290, 9184, 9380, 9404, 9324, 9092, 9336] dup_rows_per_dst=[3000, 3102, 2850, 3026, 3090, 2954, 2850, 2968] combine_rows_per_src=[9454, 9290, 9184, 9380, 9404, 9324, 9092, 9336] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=64788 dup_rows=24072 dup_rows_remote=20908 dedup_ratio=0.244873 router_tokens_per_expert=[778, 754, 748, 774, 735, 766, 783, 786, 779, 792, 778, 768, 841, 789, 739, 818, 785, 746, 770, 742, 778, 741, 789, 765, 771, 752, 797, 800, 796, 789, 746, 773, 705, 720, 754, 796, 814, 751, 735, 806, 742, 758, 794, 747, 785, 750, 725, 779, 748, 785, 771, 780, 746, 759, 794, 794, 815, 765, 716, 760, 712, 779, 729, 770] payload_rows_per_dst=[9318, 9444, 9256, 9348, 9256, 9204, 9288, 9118] dup_rows_per_dst=[2930, 3164, 2976, 3100, 2906, 2956, 3066, 2974] combine_rows_per_src=[9318, 9444, 9256, 9348, 9256, 9204, 9288, 9118] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74362 payload_rows_remote=65036 dup_rows=23942 dup_rows_remote=20924 dedup_ratio=0.243551 router_tokens_per_expert=[760, 742, 766, 792, 762, 756, 759, 736, 746, 761, 794, 753, 784, 757, 775, 792, 744, 808, 728, 778, 776, 782, 781, 763, 778, 778, 780, 798, 784, 746, 777, 742, 755, 755, 734, 755, 758, 771, 803, 771, 797, 753, 777, 764, 743, 792, 785, 716, 752, 778, 772, 747, 803, 796, 772, 752, 792, 742, 817, 757, 786, 771, 741, 767] payload_rows_per_dst=[9188, 9282, 9370, 9372, 9260, 9254, 9326, 9310] dup_rows_per_dst=[2958, 3042, 2950, 2994, 2944, 3000, 3018, 3036] combine_rows_per_src=[9188, 9282, 9370, 9372, 9260, 9254, 9326, 9310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266813440 dispatch_meta_bytes=344104 combine_payload_bytes=266813440 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267157544 ingress_bytes=370622464 link_bytes=370622464 token_bytes=533970984 weight_movement_bytes=103809024 total_nvlink_bytes=637780008 weight_to_token_ratio=0.194409 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394622494e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=264880128 dispatch_meta_bytes=342800 combine_payload_bytes=264880128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265222928 ingress_bytes=299483136 link_bytes=299483136 token_bytes=530103056 weight_movement_bytes=34603008 total_nvlink_bytes=564706064 weight_to_token_ratio=0.065276 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126930930e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74434 payload_rows_remote=64886 dup_rows=23870 dup_rows_remote=20734 dedup_ratio=0.242818 router_tokens_per_expert=[815, 772, 776, 805, 778, 778, 765, 809, 803, 789, 791, 744, 749, 761, 718, 789, 701, 758, 780, 747, 776, 733, 772, 781, 768, 756, 736, 749, 714, 738, 816, 754, 794, 749, 809, 769, 753, 777, 748, 735, 728, 798, 779, 785, 772, 720, 782, 780, 764, 807, 742, 803, 775, 772, 813, 770, 746, 750, 730, 759, 765, 793, 748, 816] payload_rows_per_dst=[9548, 9242, 9106, 9238, 9356, 9270, 9360, 9314] dup_rows_per_dst=[3136, 2958, 2990, 2824, 2994, 2936, 3132, 2900] combine_rows_per_src=[9548, 9242, 9106, 9238, 9356, 9270, 9360, 9314] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266813440 dispatch_meta_bytes=344104 combine_payload_bytes=266813440 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267157544 ingress_bytes=370622464 link_bytes=370622464 token_bytes=533970984 weight_movement_bytes=103809024 total_nvlink_bytes=637780008 weight_to_token_ratio=0.194409 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394622494e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265371648 dispatch_meta_bytes=342784 combine_payload_bytes=265371648 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265714432 ingress_bytes=369180672 link_bytes=369180672 token_bytes=531086080 weight_movement_bytes=103809024 total_nvlink_bytes=634895104 weight_to_token_ratio=0.195466 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389197147e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266387456 dispatch_meta_bytes=343840 combine_payload_bytes=266387456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266731296 ingress_bytes=300990464 link_bytes=300990464 token_bytes=533118752 weight_movement_bytes=34603008 total_nvlink_bytes=567721760 weight_to_token_ratio=0.064907 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132602884e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74560 payload_rows_remote=65424 dup_rows=23744 dup_rows_remote=20838 dedup_ratio=0.241536 router_tokens_per_expert=[802, 784, 786, 792, 754, 769, 784, 749, 774, 788, 762, 766, 747, 793, 777, 732, 762, 773, 779, 803, 765, 794, 799, 758, 780, 730, 753, 766, 765, 751, 728, 748, 752, 804, 762, 731, 781, 759, 807, 787, 721, 805, 781, 752, 829, 757, 806, 774, 746, 783, 784, 754, 767, 748, 705, 759, 802, 762, 808, 742, 724, 772, 747, 728] payload_rows_per_dst=[9394, 9250, 9468, 9136, 9474, 9444, 9176, 9218] dup_rows_per_dst=[3046, 3028, 2998, 2906, 2892, 3006, 2916, 2952] combine_rows_per_src=[9394, 9250, 9468, 9136, 9474, 9444, 9176, 9218] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=342480 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266115536 ingress_bytes=334979072 link_bytes=334979072 token_bytes=531888592 weight_movement_bytes=69206016 total_nvlink_bytes=601094608 weight_to_token_ratio=0.130114 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260499280e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74400 payload_rows_remote=65042 dup_rows=23904 dup_rows_remote=20894 dedup_ratio=0.243164 router_tokens_per_expert=[749, 800, 760, 725, 771, 747, 787, 757, 822, 794, 810, 745, 789, 723, 771, 705, 747, 769, 745, 798, 705, 774, 842, 804, 775, 767, 800, 761, 750, 742, 808, 777, 777, 742, 824, 800, 758, 767, 776, 783, 805, 792, 775, 735, 736, 765, 711, 775, 728, 784, 762, 729, 780, 797, 729, 755, 807, 752, 790, 723, 777, 739, 760, 800] payload_rows_per_dst=[9238, 9356, 9358, 9362, 9350, 9220, 9190, 9326] dup_rows_per_dst=[2954, 2962, 3010, 2998, 3104, 2968, 2938, 2970] combine_rows_per_src=[9238, 9356, 9358, 9362, 9350, 9220, 9190, 9326] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74112 payload_rows_remote=64786 dup_rows=24192 dup_rows_remote=21142 dedup_ratio=0.246094 router_tokens_per_expert=[787, 721, 749, 813, 768, 759, 773, 811, 770, 767, 733, 746, 772, 761, 738, 784, 736, 748, 725, 797, 753, 744, 799, 769, 745, 761, 778, 811, 765, 753, 767, 763, 769, 780, 785, 785, 770, 780, 740, 779, 743, 756, 766, 808, 790, 746, 792, 791, 772, 767, 764, 800, 797, 740, 764, 763, 758, 791, 738, 730, 771, 765, 807, 779] payload_rows_per_dst=[9254, 9216, 9178, 9266, 9326, 9290, 9326, 9256] dup_rows_per_dst=[3108, 2926, 2964, 3020, 3050, 3094, 3008, 3022] combine_rows_per_src=[9254, 9216, 9178, 9266, 9326, 9290, 9326, 9256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=64788 dup_rows=24072 dup_rows_remote=20908 dedup_ratio=0.244873 router_tokens_per_expert=[778, 754, 748, 774, 735, 766, 783, 786, 779, 792, 778, 768, 841, 789, 739, 818, 785, 746, 770, 742, 778, 741, 789, 765, 771, 752, 797, 800, 796, 789, 746, 773, 705, 720, 754, 796, 814, 751, 735, 806, 742, 758, 794, 747, 785, 750, 725, 779, 748, 785, 771, 780, 746, 759, 794, 794, 815, 765, 716, 760, 712, 779, 729, 770] payload_rows_per_dst=[9318, 9444, 9256, 9348, 9256, 9204, 9288, 9118] dup_rows_per_dst=[2930, 3164, 2976, 3100, 2906, 2956, 3066, 2974] combine_rows_per_src=[9318, 9444, 9256, 9348, 9256, 9204, 9288, 9118] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74532 payload_rows_remote=65164 dup_rows=23772 dup_rows_remote=20878 dedup_ratio=0.241821 router_tokens_per_expert=[753, 756, 738, 810, 798, 783, 770, 792, 772, 787, 715, 779, 770, 780, 752, 797, 765, 753, 760, 788, 722, 788, 791, 773, 750, 798, 765, 800, 782, 766, 810, 751, 753, 757, 805, 772, 736, 744, 768, 753, 776, 710, 757, 783, 781, 776, 777, 758, 743, 783, 755, 780, 767, 740, 802, 761, 767, 771, 736, 782, 751, 727, 772, 795] payload_rows_per_dst=[9280, 9240, 9308, 9446, 9188, 9400, 9368, 9302] dup_rows_per_dst=[3120, 3064, 2972, 2998, 2988, 2836, 2894, 2900] combine_rows_per_src=[9280, 9240, 9308, 9446, 9188, 9400, 9368, 9302] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74112 payload_rows_remote=64786 dup_rows=24192 dup_rows_remote=21142 dedup_ratio=0.246094 router_tokens_per_expert=[787, 721, 749, 813, 768, 759, 773, 811, 770, 767, 733, 746, 772, 761, 738, 784, 736, 748, 725, 797, 753, 744, 799, 769, 745, 761, 778, 811, 765, 753, 767, 763, 769, 780, 785, 785, 770, 780, 740, 779, 743, 756, 766, 808, 790, 746, 792, 791, 772, 767, 764, 800, 797, 740, 764, 763, 758, 791, 738, 730, 771, 765, 807, 779] payload_rows_per_dst=[9254, 9216, 9178, 9266, 9326, 9290, 9326, 9256] dup_rows_per_dst=[3108, 2926, 2964, 3020, 3050, 3094, 3008, 3022] combine_rows_per_src=[9254, 9216, 9178, 9266, 9326, 9290, 9326, 9256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=342480 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266115536 ingress_bytes=334979072 link_bytes=334979072 token_bytes=531888592 weight_movement_bytes=69206016 total_nvlink_bytes=601094608 weight_to_token_ratio=0.130114 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260499280e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=267976704 dispatch_meta_bytes=345048 combine_payload_bytes=267976704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268321752 ingress_bytes=302579712 link_bytes=302579712 token_bytes=536298456 weight_movement_bytes=34603008 total_nvlink_bytes=570901464 weight_to_token_ratio=0.064522 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138583097e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74480 payload_rows_remote=65108 dup_rows=23824 dup_rows_remote=20786 dedup_ratio=0.242350 router_tokens_per_expert=[746, 762, 777, 738, 759, 768, 748, 773, 783, 768, 738, 767, 760, 716, 755, 800, 773, 755, 762, 786, 812, 722, 750, 753, 731, 740, 790, 811, 783, 763, 799, 801, 733, 747, 791, 736, 799, 787, 761, 795, 756, 823, 762, 803, 759, 759, 785, 761, 800, 742, 780, 741, 746, 803, 763, 726, 774, 776, 831, 810, 762, 759, 759, 734] payload_rows_per_dst=[9268, 9000, 9254, 9594, 9320, 9452, 9220, 9372] dup_rows_per_dst=[2874, 2726, 2972, 3206, 3062, 2964, 2982, 3038] combine_rows_per_src=[9268, 9000, 9254, 9594, 9320, 9452, 9220, 9372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74412 payload_rows_remote=64992 dup_rows=23892 dup_rows_remote=20880 dedup_ratio=0.243042 router_tokens_per_expert=[782, 793, 787, 774, 790, 763, 792, 814, 772, 755, 781, 749, 767, 793, 775, 761, 737, 759, 772, 742, 752, 764, 715, 726, 748, 787, 763, 745, 765, 708, 782, 714, 704, 742, 798, 860, 803, 732, 750, 754, 737, 783, 775, 794, 772, 792, 799, 764, 772, 765, 802, 729, 757, 808, 790, 779, 808, 712, 770, 736, 779, 751, 791, 817] payload_rows_per_dst=[9534, 9272, 9160, 9086, 9376, 9420, 9294, 9270] dup_rows_per_dst=[3056, 3034, 2774, 2938, 2910, 3012, 3110, 3058] combine_rows_per_src=[9534, 9272, 9160, 9086, 9376, 9420, 9294, 9270] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=343744 combine_payload_bytes=266412032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266755776 ingress_bytes=301015040 link_bytes=301015040 token_bytes=533167808 weight_movement_bytes=34603008 total_nvlink_bytes=567770816 weight_to_token_ratio=0.064901 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132695362e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265371648 dispatch_meta_bytes=342784 combine_payload_bytes=265371648 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265714432 ingress_bytes=369180672 link_bytes=369180672 token_bytes=531086080 weight_movement_bytes=103809024 total_nvlink_bytes=634895104 weight_to_token_ratio=0.195466 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389197147e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74480 payload_rows_remote=65108 dup_rows=23824 dup_rows_remote=20786 dedup_ratio=0.242350 router_tokens_per_expert=[746, 762, 777, 738, 759, 768, 748, 773, 783, 768, 738, 767, 760, 716, 755, 800, 773, 755, 762, 786, 812, 722, 750, 753, 731, 740, 790, 811, 783, 763, 799, 801, 733, 747, 791, 736, 799, 787, 761, 795, 756, 823, 762, 803, 759, 759, 785, 761, 800, 742, 780, 741, 746, 803, 763, 726, 774, 776, 831, 810, 762, 759, 759, 734] payload_rows_per_dst=[9268, 9000, 9254, 9594, 9320, 9452, 9220, 9372] dup_rows_per_dst=[2874, 2726, 2972, 3206, 3062, 2964, 2982, 3038] combine_rows_per_src=[9268, 9000, 9254, 9594, 9320, 9452, 9220, 9372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=343712 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265707168 ingress_bytes=334569472 link_bytes=334569472 token_bytes=531070624 weight_movement_bytes=69206016 total_nvlink_bytes=600276640 weight_to_token_ratio=0.130314 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258957988e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74412 payload_rows_remote=64992 dup_rows=23892 dup_rows_remote=20880 dedup_ratio=0.243042 router_tokens_per_expert=[782, 793, 787, 774, 790, 763, 792, 814, 772, 755, 781, 749, 767, 793, 775, 761, 737, 759, 772, 742, 752, 764, 715, 726, 748, 787, 763, 745, 765, 708, 782, 714, 704, 742, 798, 860, 803, 732, 750, 754, 737, 783, 775, 794, 772, 792, 799, 764, 772, 765, 802, 729, 757, 808, 790, 779, 808, 712, 770, 736, 779, 751, 791, 817] payload_rows_per_dst=[9534, 9272, 9160, 9086, 9376, 9420, 9294, 9270] dup_rows_per_dst=[3056, 3034, 2774, 2938, 2910, 3012, 3110, 3058] combine_rows_per_src=[9534, 9272, 9160, 9086, 9376, 9420, 9294, 9270] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=64930 dup_rows=23940 dup_rows_remote=20822 dedup_ratio=0.243530 router_tokens_per_expert=[773, 752, 744, 734, 691, 734, 751, 747, 737, 794, 791, 814, 780, 797, 780, 783, 827, 791, 719, 747, 766, 778, 794, 770, 762, 809, 773, 775, 751, 768, 729, 740, 755, 765, 757, 743, 746, 783, 784, 784, 781, 793, 766, 731, 779, 799, 827, 765, 743, 771, 787, 769, 771, 747, 822, 776, 749, 725, 759, 801, 757, 737, 807, 772] payload_rows_per_dst=[9082, 9434, 9386, 9272, 9312, 9352, 9342, 9184] dup_rows_per_dst=[2770, 3118, 2998, 2942, 2922, 3130, 3030, 3030] combine_rows_per_src=[9082, 9434, 9386, 9272, 9312, 9352, 9342, 9184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73960 payload_rows_remote=64618 dup_rows=24344 dup_rows_remote=21280 dedup_ratio=0.247640 router_tokens_per_expert=[748, 764, 737, 756, 769, 795, 767, 778, 763, 747, 784, 762, 766, 788, 770, 793, 763, 812, 771, 765, 723, 774, 726, 807, 747, 773, 750, 758, 774, 804, 716, 742, 745, 794, 750, 731, 766, 751, 784, 801, 734, 798, 784, 743, 765, 790, 760, 725, 737, 753, 788, 757, 803, 764, 814, 787, 757, 792, 782, 769, 797, 781, 806, 752] payload_rows_per_dst=[9266, 9272, 9176, 9140, 9214, 9226, 9342, 9324] dup_rows_per_dst=[2962, 3074, 3106, 2988, 3030, 2972, 3064, 3148] combine_rows_per_src=[9266, 9272, 9176, 9140, 9214, 9226, 9342, 9324] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266911744 dispatch_meta_bytes=344168 combine_payload_bytes=266911744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267255912 ingress_bytes=301514752 link_bytes=301514752 token_bytes=534167656 weight_movement_bytes=34603008 total_nvlink_bytes=568770664 weight_to_token_ratio=0.064779 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134575738e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=65130 dup_rows=24020 dup_rows_remote=21032 dedup_ratio=0.244344 router_tokens_per_expert=[799, 725, 753, 770, 776, 763, 735, 744, 747, 742, 819, 757, 771, 818, 792, 771, 745, 768, 767, 773, 760, 759, 761, 762, 748, 788, 798, 734, 787, 783, 812, 720, 771, 777, 820, 742, 736, 785, 806, 751, 770, 749, 795, 817, 779, 751, 739, 751, 723, 772, 812, 784, 773, 759, 739, 752, 757, 785, 766, 790, 746, 785, 761, 762] payload_rows_per_dst=[9154, 9334, 9322, 9350, 9340, 9256, 9232, 9296] dup_rows_per_dst=[2988, 3102, 3102, 2990, 3036, 2890, 2944, 2968] combine_rows_per_src=[9154, 9334, 9322, 9350, 9340, 9256, 9232, 9296] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266682368 dispatch_meta_bytes=343576 combine_payload_bytes=266682368 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267025944 ingress_bytes=370491392 link_bytes=370491392 token_bytes=533708312 weight_movement_bytes=103809024 total_nvlink_bytes=637517336 weight_to_token_ratio=0.194505 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394129281e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266207232 dispatch_meta_bytes=343488 combine_payload_bytes=266207232 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266550720 ingress_bytes=335413248 link_bytes=335413248 token_bytes=532757952 weight_movement_bytes=69206016 total_nvlink_bytes=601963968 weight_to_token_ratio=0.129901 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262133049e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265953280 dispatch_meta_bytes=343008 combine_payload_bytes=265953280 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266296288 ingress_bytes=369762304 link_bytes=369762304 token_bytes=532249568 weight_movement_bytes=103809024 total_nvlink_bytes=636058592 weight_to_token_ratio=0.195038 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391385782e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266772480 dispatch_meta_bytes=344648 combine_payload_bytes=266772480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267117128 ingress_bytes=301375488 link_bytes=301375488 token_bytes=533889608 weight_movement_bytes=34603008 total_nvlink_bytes=568492616 weight_to_token_ratio=0.064813 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134051699e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266682368 dispatch_meta_bytes=343576 combine_payload_bytes=266682368 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267025944 ingress_bytes=370491392 link_bytes=370491392 token_bytes=533708312 weight_movement_bytes=103809024 total_nvlink_bytes=637517336 weight_to_token_ratio=0.194505 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394129281e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=264675328 dispatch_meta_bytes=343592 combine_payload_bytes=264675328 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265018920 ingress_bytes=333881344 link_bytes=333881344 token_bytes=529694248 weight_movement_bytes=69206016 total_nvlink_bytes=598900264 weight_to_token_ratio=0.130653 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256368618e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=65130 dup_rows=24020 dup_rows_remote=21032 dedup_ratio=0.244344 router_tokens_per_expert=[799, 725, 753, 770, 776, 763, 735, 744, 747, 742, 819, 757, 771, 818, 792, 771, 745, 768, 767, 773, 760, 759, 761, 762, 748, 788, 798, 734, 787, 783, 812, 720, 771, 777, 820, 742, 736, 785, 806, 751, 770, 749, 795, 817, 779, 751, 739, 751, 723, 772, 812, 784, 773, 759, 739, 752, 757, 785, 766, 790, 746, 785, 761, 762] payload_rows_per_dst=[9154, 9334, 9322, 9350, 9340, 9256, 9232, 9296] dup_rows_per_dst=[2988, 3102, 3102, 2990, 3036, 2890, 2944, 2968] combine_rows_per_src=[9154, 9334, 9322, 9350, 9340, 9256, 9232, 9296] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=64978 dup_rows=24020 dup_rows_remote=21082 dedup_ratio=0.244344 router_tokens_per_expert=[781, 739, 756, 767, 801, 806, 763, 805, 831, 794, 718, 764, 775, 748, 792, 764, 783, 762, 781, 748, 780, 728, 800, 791, 797, 761, 789, 714, 753, 740, 800, 768, 759, 762, 768, 747, 768, 746, 730, 801, 762, 753, 775, 755, 784, 749, 760, 755, 774, 738, 795, 796, 767, 756, 761, 797, 745, 767, 764, 721, 818, 770, 766, 744] payload_rows_per_dst=[9388, 9300, 9274, 9306, 9228, 9196, 9356, 9236] dup_rows_per_dst=[3048, 3072, 3072, 2938, 2934, 2990, 3012, 2954] combine_rows_per_src=[9388, 9300, 9274, 9306, 9228, 9196, 9356, 9236] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74112 payload_rows_remote=64826 dup_rows=24192 dup_rows_remote=21102 dedup_ratio=0.246094 router_tokens_per_expert=[800, 781, 833, 708, 764, 782, 825, 753, 761, 747, 788, 726, 782, 758, 716, 753, 762, 750, 773, 783, 810, 734, 759, 817, 773, 774, 771, 774, 756, 730, 761, 789, 716, 737, 762, 797, 722, 794, 795, 789, 754, 785, 732, 790, 799, 813, 767, 795, 754, 738, 793, 793, 800, 789, 740, 771, 719, 792, 771, 771, 753, 769, 754, 705] payload_rows_per_dst=[9394, 9174, 9286, 9188, 9216, 9358, 9318, 9178] dup_rows_per_dst=[3098, 2888, 3090, 3068, 3008, 3112, 3038, 2890] combine_rows_per_src=[9394, 9174, 9286, 9188, 9216, 9358, 9318, 9178] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74394 payload_rows_remote=64948 dup_rows=23910 dup_rows_remote=20960 dedup_ratio=0.243225 router_tokens_per_expert=[773, 734, 783, 801, 752, 786, 777, 805, 790, 783, 763, 756, 778, 790, 815, 777, 735, 782, 769, 735, 733, 829, 774, 809, 774, 729, 781, 772, 795, 749, 768, 781, 783, 749, 733, 780, 809, 786, 764, 794, 745, 777, 777, 766, 709, 721, 755, 737, 729, 757, 756, 741, 761, 749, 751, 766, 758, 753, 749, 806, 780, 766, 787, 780] payload_rows_per_dst=[9306, 9380, 9268, 9238, 9446, 9228, 9184, 9344] dup_rows_per_dst=[3116, 3124, 3064, 3060, 2950, 2746, 2836, 3014] combine_rows_per_src=[9306, 9380, 9268, 9238, 9446, 9228, 9184, 9344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=64930 dup_rows=23940 dup_rows_remote=20822 dedup_ratio=0.243530 router_tokens_per_expert=[773, 752, 744, 734, 691, 734, 751, 747, 737, 794, 791, 814, 780, 797, 780, 783, 827, 791, 719, 747, 766, 778, 794, 770, 762, 809, 773, 775, 751, 768, 729, 740, 755, 765, 757, 743, 746, 783, 784, 784, 781, 793, 766, 731, 779, 799, 827, 765, 743, 771, 787, 769, 771, 747, 822, 776, 749, 725, 759, 801, 757, 737, 807, 772] payload_rows_per_dst=[9082, 9434, 9386, 9272, 9312, 9352, 9342, 9184] dup_rows_per_dst=[2770, 3118, 2998, 2942, 2922, 3130, 3030, 3030] combine_rows_per_src=[9082, 9434, 9386, 9272, 9312, 9352, 9342, 9184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74362 payload_rows_remote=65036 dup_rows=23942 dup_rows_remote=20924 dedup_ratio=0.243551 router_tokens_per_expert=[760, 742, 766, 792, 762, 756, 759, 736, 746, 761, 794, 753, 784, 757, 775, 792, 744, 808, 728, 778, 776, 782, 781, 763, 778, 778, 780, 798, 784, 746, 777, 742, 755, 755, 734, 755, 758, 771, 803, 771, 797, 753, 777, 764, 743, 792, 785, 716, 752, 778, 772, 747, 803, 796, 772, 752, 792, 742, 817, 757, 786, 771, 741, 767] payload_rows_per_dst=[9188, 9282, 9370, 9372, 9260, 9254, 9326, 9310] dup_rows_per_dst=[2958, 3042, 2950, 2994, 2944, 3000, 3018, 3036] combine_rows_per_src=[9188, 9282, 9370, 9372, 9260, 9254, 9326, 9310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74394 payload_rows_remote=64948 dup_rows=23910 dup_rows_remote=20960 dedup_ratio=0.243225 router_tokens_per_expert=[773, 734, 783, 801, 752, 786, 777, 805, 790, 783, 763, 756, 778, 790, 815, 777, 735, 782, 769, 735, 733, 829, 774, 809, 774, 729, 781, 772, 795, 749, 768, 781, 783, 749, 733, 780, 809, 786, 764, 794, 745, 777, 777, 766, 709, 721, 755, 737, 729, 757, 756, 741, 761, 749, 751, 766, 758, 753, 749, 806, 780, 766, 787, 780] payload_rows_per_dst=[9306, 9380, 9268, 9238, 9446, 9228, 9184, 9344] dup_rows_per_dst=[3116, 3124, 3064, 3060, 2950, 2746, 2836, 3014] combine_rows_per_src=[9306, 9380, 9268, 9238, 9446, 9228, 9184, 9344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266772480 dispatch_meta_bytes=344648 combine_payload_bytes=266772480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267117128 ingress_bytes=301375488 link_bytes=301375488 token_bytes=533889608 weight_movement_bytes=34603008 total_nvlink_bytes=568492616 weight_to_token_ratio=0.064813 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134051699e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=344240 combine_payload_bytes=266149888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266494128 ingress_bytes=300752896 link_bytes=300752896 token_bytes=532644016 weight_movement_bytes=34603008 total_nvlink_bytes=567247024 weight_to_token_ratio=0.064965 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131708935e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265527296 dispatch_meta_bytes=343712 combine_payload_bytes=265527296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265871008 ingress_bytes=300130304 link_bytes=300130304 token_bytes=531398304 weight_movement_bytes=34603008 total_nvlink_bytes=566001312 weight_to_token_ratio=0.065117 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129366171e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74154 payload_rows_remote=65036 dup_rows=24150 dup_rows_remote=21152 dedup_ratio=0.245667 router_tokens_per_expert=[742, 775, 743, 788, 775, 780, 704, 780, 738, 785, 770, 809, 808, 754, 753, 805, 772, 768, 794, 789, 771, 738, 714, 791, 783, 755, 735, 770, 774, 746, 764, 749, 785, 766, 782, 767, 791, 772, 755, 772, 786, 750, 751, 746, 738, 769, 760, 758, 781, 812, 733, 785, 804, 790, 794, 792, 781, 754, 725, 795, 760, 753, 722, 801] payload_rows_per_dst=[9216, 9272, 9212, 9202, 9426, 9118, 9490, 9218] dup_rows_per_dst=[2958, 3172, 3062, 2950, 2954, 2998, 3092, 2964] combine_rows_per_src=[9216, 9272, 9212, 9202, 9426, 9118, 9490, 9218] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265953280 dispatch_meta_bytes=343008 combine_payload_bytes=265953280 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266296288 ingress_bytes=369762304 link_bytes=369762304 token_bytes=532249568 weight_movement_bytes=103809024 total_nvlink_bytes=636058592 weight_to_token_ratio=0.195038 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391385782e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73988 payload_rows_remote=64452 dup_rows=24316 dup_rows_remote=21104 dedup_ratio=0.247355 router_tokens_per_expert=[784, 775, 755, 761, 805, 734, 765, 786, 760, 749, 754, 782, 835, 775, 795, 744, 802, 752, 725, 758, 733, 762, 757, 798, 798, 753, 760, 793, 762, 724, 767, 784, 797, 729, 792, 733, 725, 782, 739, 757, 800, 769, 748, 713, 762, 745, 751, 808, 800, 796, 784, 766, 794, 788, 757, 775, 740, 764, 757, 777, 811, 788, 752, 766] payload_rows_per_dst=[9254, 9002, 9202, 9300, 8980, 9370, 9344, 9536] dup_rows_per_dst=[3076, 2888, 2972, 2982, 2798, 3152, 3236, 3212] combine_rows_per_src=[9254, 9002, 9202, 9300, 8980, 9370, 9344, 9536] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=65090 dup_rows=24062 dup_rows_remote=21126 dedup_ratio=0.244771 router_tokens_per_expert=[796, 786, 755, 784, 748, 737, 785, 778, 755, 765, 750, 808, 749, 749, 750, 728, 786, 796, 740, 787, 772, 768, 781, 751, 724, 767, 767, 754, 742, 815, 764, 766, 817, 788, 760, 747, 798, 780, 794, 750, 790, 791, 785, 788, 783, 809, 726, 795, 814, 744, 724, 726, 765, 744, 759, 768, 777, 810, 744, 801, 741, 757, 754, 720] payload_rows_per_dst=[9288, 9176, 9330, 9142, 9428, 9478, 9152, 9248] dup_rows_per_dst=[3050, 2932, 3032, 3056, 3040, 3056, 2936, 2960] combine_rows_per_src=[9288, 9176, 9330, 9142, 9428, 9478, 9152, 9248] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266027008 dispatch_meta_bytes=343632 combine_payload_bytes=266027008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266370640 ingress_bytes=300630016 link_bytes=300630016 token_bytes=532397648 weight_movement_bytes=34603008 total_nvlink_bytes=567000656 weight_to_token_ratio=0.064995 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131246548e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74154 payload_rows_remote=65036 dup_rows=24150 dup_rows_remote=21152 dedup_ratio=0.245667 router_tokens_per_expert=[742, 775, 743, 788, 775, 780, 704, 780, 738, 785, 770, 809, 808, 754, 753, 805, 772, 768, 794, 789, 771, 738, 714, 791, 783, 755, 735, 770, 774, 746, 764, 749, 785, 766, 782, 767, 791, 772, 755, 772, 786, 750, 751, 746, 738, 769, 760, 758, 781, 812, 733, 785, 804, 790, 794, 792, 781, 754, 725, 795, 760, 753, 722, 801] payload_rows_per_dst=[9216, 9272, 9212, 9202, 9426, 9118, 9490, 9218] dup_rows_per_dst=[2958, 3172, 3062, 2950, 2954, 2998, 3092, 2964] combine_rows_per_src=[9216, 9272, 9212, 9202, 9426, 9118, 9490, 9218] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74348 payload_rows_remote=65068 dup_rows=23956 dup_rows_remote=20912 dedup_ratio=0.243693 router_tokens_per_expert=[765, 751, 804, 808, 766, 768, 715, 748, 738, 793, 787, 813, 782, 802, 711, 736, 796, 781, 773, 752, 766, 707, 794, 762, 777, 818, 785, 773, 805, 751, 799, 792, 779, 768, 754, 788, 768, 813, 819, 746, 731, 765, 770, 792, 751, 752, 760, 761, 765, 783, 771, 718, 732, 769, 742, 778, 772, 775, 792, 799, 723, 735, 722, 741] payload_rows_per_dst=[9304, 9280, 9300, 9464, 9356, 9214, 9258, 9172] dup_rows_per_dst=[2946, 3044, 2962, 3136, 3114, 2950, 2858, 2946] combine_rows_per_src=[9304, 9280, 9300, 9464, 9356, 9214, 9258, 9172] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266387456 dispatch_meta_bytes=343840 combine_payload_bytes=266387456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266731296 ingress_bytes=300990464 link_bytes=300990464 token_bytes=533118752 weight_movement_bytes=34603008 total_nvlink_bytes=567721760 weight_to_token_ratio=0.064907 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132602884e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74216 payload_rows_remote=65000 dup_rows=24088 dup_rows_remote=21102 dedup_ratio=0.245036 router_tokens_per_expert=[777, 780, 771, 709, 796, 739, 756, 736, 800, 767, 769, 766, 762, 771, 761, 799, 770, 733, 748, 756, 746, 748, 731, 785, 807, 742, 712, 789, 783, 781, 801, 748, 817, 778, 760, 796, 807, 782, 795, 712, 807, 740, 774, 781, 728, 723, 772, 759, 742, 821, 765, 806, 779, 756, 809, 765, 764, 744, 821, 779, 782, 751, 749, 749] payload_rows_per_dst=[9216, 9358, 9206, 9410, 9382, 9186, 9168, 9290] dup_rows_per_dst=[2986, 3038, 3022, 3110, 3032, 2982, 2930, 2988] combine_rows_per_src=[9216, 9358, 9206, 9410, 9382, 9186, 9168, 9290] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=263995392 dispatch_meta_bytes=342224 combine_payload_bytes=263995392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264337616 ingress_bytes=298598400 link_bytes=298598400 token_bytes=528333008 weight_movement_bytes=34603008 total_nvlink_bytes=562936016 weight_to_token_ratio=0.065495 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123601740e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266387456 dispatch_meta_bytes=344752 combine_payload_bytes=266387456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266732208 ingress_bytes=300990464 link_bytes=300990464 token_bytes=533119664 weight_movement_bytes=34603008 total_nvlink_bytes=567722672 weight_to_token_ratio=0.064907 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132602884e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266518528 dispatch_meta_bytes=343920 combine_payload_bytes=266518528 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266862448 ingress_bytes=370327552 link_bytes=370327552 token_bytes=533380976 weight_movement_bytes=103809024 total_nvlink_bytes=637190000 weight_to_token_ratio=0.194625 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393512764e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266240000 dispatch_meta_bytes=344408 combine_payload_bytes=266240000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266584408 ingress_bytes=300843008 link_bytes=300843008 token_bytes=532824408 weight_movement_bytes=34603008 total_nvlink_bytes=567427416 weight_to_token_ratio=0.064943 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132048019e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=65042 dup_rows=23998 dup_rows_remote=21056 dedup_ratio=0.244120 router_tokens_per_expert=[763, 780, 789, 797, 779, 768, 787, 733, 790, 773, 790, 773, 757, 794, 778, 785, 783, 759, 759, 784, 751, 790, 760, 745, 776, 763, 759, 787, 756, 746, 801, 715, 771, 749, 761, 756, 731, 807, 794, 745, 748, 736, 767, 818, 805, 721, 723, 770, 744, 758, 778, 771, 775, 713, 828, 813, 779, 775, 750, 779, 765, 755, 770, 727] payload_rows_per_dst=[9386, 9438, 9266, 9264, 9298, 9212, 9248, 9194] dup_rows_per_dst=[3006, 3042, 2996, 2942, 2930, 2964, 3112, 3006] combine_rows_per_src=[9386, 9438, 9266, 9264, 9298, 9212, 9248, 9194] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=263995392 dispatch_meta_bytes=342224 combine_payload_bytes=263995392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264337616 ingress_bytes=298598400 link_bytes=298598400 token_bytes=528333008 weight_movement_bytes=34603008 total_nvlink_bytes=562936016 weight_to_token_ratio=0.065495 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123601740e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74216 payload_rows_remote=65000 dup_rows=24088 dup_rows_remote=21102 dedup_ratio=0.245036 router_tokens_per_expert=[777, 780, 771, 709, 796, 739, 756, 736, 800, 767, 769, 766, 762, 771, 761, 799, 770, 733, 748, 756, 746, 748, 731, 785, 807, 742, 712, 789, 783, 781, 801, 748, 817, 778, 760, 796, 807, 782, 795, 712, 807, 740, 774, 781, 728, 723, 772, 759, 742, 821, 765, 806, 779, 756, 809, 765, 764, 744, 821, 779, 782, 751, 749, 749] payload_rows_per_dst=[9216, 9358, 9206, 9410, 9382, 9186, 9168, 9290] dup_rows_per_dst=[2986, 3038, 3022, 3110, 3032, 2982, 2930, 2988] combine_rows_per_src=[9216, 9358, 9206, 9410, 9382, 9186, 9168, 9290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266608640 dispatch_meta_bytes=344864 combine_payload_bytes=266608640 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266953504 ingress_bytes=370417664 link_bytes=370417664 token_bytes=533562144 weight_movement_bytes=103809024 total_nvlink_bytes=637371168 weight_to_token_ratio=0.194558 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393851849e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=65064 dup_rows=23920 dup_rows_remote=20956 dedup_ratio=0.243327 router_tokens_per_expert=[829, 767, 762, 732, 798, 730, 758, 763, 781, 774, 753, 768, 814, 767, 725, 782, 744, 766, 806, 779, 759, 757, 763, 768, 743, 749, 732, 785, 769, 735, 774, 767, 755, 745, 768, 751, 795, 756, 805, 773, 775, 774, 725, 798, 796, 806, 739, 776, 759, 771, 736, 755, 781, 785, 799, 762, 771, 764, 718, 782, 763, 822, 762, 786] payload_rows_per_dst=[9244, 9350, 9320, 9176, 9286, 9376, 9316, 9316] dup_rows_per_dst=[3034, 2978, 2964, 2932, 3010, 3002, 2980, 3020] combine_rows_per_src=[9244, 9350, 9320, 9176, 9286, 9376, 9316, 9316] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74160 payload_rows_remote=64860 dup_rows=24144 dup_rows_remote=21174 dedup_ratio=0.245605 router_tokens_per_expert=[768, 775, 803, 741, 766, 784, 780, 753, 782, 728, 714, 778, 769, 793, 818, 785, 750, 764, 797, 818, 769, 816, 796, 719, 735, 755, 756, 784, 770, 784, 772, 763, 766, 750, 753, 757, 820, 761, 748, 780, 748, 753, 787, 766, 741, 800, 733, 795, 741, 748, 778, 722, 798, 751, 795, 759, 730, 743, 822, 753, 788, 738, 752, 791] payload_rows_per_dst=[9226, 9272, 9352, 9270, 9300, 9260, 9244, 9236] dup_rows_per_dst=[3114, 3062, 3106, 2968, 2970, 2986, 2940, 2998] combine_rows_per_src=[9226, 9272, 9352, 9270, 9300, 9260, 9244, 9236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=65042 dup_rows=23998 dup_rows_remote=21056 dedup_ratio=0.244120 router_tokens_per_expert=[763, 780, 789, 797, 779, 768, 787, 733, 790, 773, 790, 773, 757, 794, 778, 785, 783, 759, 759, 784, 751, 790, 760, 745, 776, 763, 759, 787, 756, 746, 801, 715, 771, 749, 761, 756, 731, 807, 794, 745, 748, 736, 767, 818, 805, 721, 723, 770, 744, 758, 778, 771, 775, 713, 828, 813, 779, 775, 750, 779, 765, 755, 770, 727] payload_rows_per_dst=[9386, 9438, 9266, 9264, 9298, 9212, 9248, 9194] dup_rows_per_dst=[3006, 3042, 2996, 2942, 2930, 2964, 3112, 3006] combine_rows_per_src=[9386, 9438, 9266, 9264, 9298, 9212, 9248, 9194] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74348 payload_rows_remote=65068 dup_rows=23956 dup_rows_remote=20912 dedup_ratio=0.243693 router_tokens_per_expert=[765, 751, 804, 808, 766, 768, 715, 748, 738, 793, 787, 813, 782, 802, 711, 736, 796, 781, 773, 752, 766, 707, 794, 762, 777, 818, 785, 773, 805, 751, 799, 792, 779, 768, 754, 788, 768, 813, 819, 746, 731, 765, 770, 792, 751, 752, 760, 761, 765, 783, 771, 718, 732, 769, 742, 778, 772, 775, 792, 799, 723, 735, 722, 741] payload_rows_per_dst=[9304, 9280, 9300, 9464, 9356, 9214, 9258, 9172] dup_rows_per_dst=[2946, 3044, 2962, 3136, 3114, 2950, 2858, 2946] combine_rows_per_src=[9304, 9280, 9300, 9464, 9356, 9214, 9258, 9172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73960 payload_rows_remote=64618 dup_rows=24344 dup_rows_remote=21280 dedup_ratio=0.247640 router_tokens_per_expert=[748, 764, 737, 756, 769, 795, 767, 778, 763, 747, 784, 762, 766, 788, 770, 793, 763, 812, 771, 765, 723, 774, 726, 807, 747, 773, 750, 758, 774, 804, 716, 742, 745, 794, 750, 731, 766, 751, 784, 801, 734, 798, 784, 743, 765, 790, 760, 725, 737, 753, 788, 757, 803, 764, 814, 787, 757, 792, 782, 769, 797, 781, 806, 752] payload_rows_per_dst=[9266, 9272, 9176, 9140, 9214, 9226, 9342, 9324] dup_rows_per_dst=[2962, 3074, 3106, 2988, 3030, 2972, 3064, 3148] combine_rows_per_src=[9266, 9272, 9176, 9140, 9214, 9226, 9342, 9324] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74160 payload_rows_remote=64860 dup_rows=24144 dup_rows_remote=21174 dedup_ratio=0.245605 router_tokens_per_expert=[768, 775, 803, 741, 766, 784, 780, 753, 782, 728, 714, 778, 769, 793, 818, 785, 750, 764, 797, 818, 769, 816, 796, 719, 735, 755, 756, 784, 770, 784, 772, 763, 766, 750, 753, 757, 820, 761, 748, 780, 748, 753, 787, 766, 741, 800, 733, 795, 741, 748, 778, 722, 798, 751, 795, 759, 730, 743, 822, 753, 788, 738, 752, 791] payload_rows_per_dst=[9226, 9272, 9352, 9270, 9300, 9260, 9244, 9236] dup_rows_per_dst=[3114, 3062, 3106, 2968, 2970, 2986, 2940, 2998] combine_rows_per_src=[9226, 9272, 9352, 9270, 9300, 9260, 9244, 9236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266240000 dispatch_meta_bytes=344408 combine_payload_bytes=266240000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266584408 ingress_bytes=300843008 link_bytes=300843008 token_bytes=532824408 weight_movement_bytes=34603008 total_nvlink_bytes=567427416 weight_to_token_ratio=0.064943 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132048019e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266502144 dispatch_meta_bytes=344080 combine_payload_bytes=266502144 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266846224 ingress_bytes=301105152 link_bytes=301105152 token_bytes=533348368 weight_movement_bytes=34603008 total_nvlink_bytes=567951376 weight_to_token_ratio=0.064879 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133034446e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74428 payload_rows_remote=65180 dup_rows=23876 dup_rows_remote=21110 dedup_ratio=0.242879 router_tokens_per_expert=[743, 822, 762, 783, 752, 723, 764, 763, 814, 792, 769, 772, 760, 770, 805, 812, 783, 728, 777, 730, 796, 740, 796, 781, 713, 758, 792, 760, 760, 768, 750, 733, 769, 765, 788, 793, 749, 750, 808, 785, 702, 768, 793, 733, 752, 754, 755, 750, 768, 792, 747, 773, 765, 844, 759, 801, 769, 774, 779, 754, 747, 769, 749, 777] payload_rows_per_dst=[9304, 9424, 9338, 9186, 9332, 9248, 9398, 9198] dup_rows_per_dst=[2920, 3164, 2924, 2882, 3082, 2766, 3100, 3038] combine_rows_per_src=[9304, 9424, 9338, 9186, 9332, 9248, 9398, 9198] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=344392 combine_payload_bytes=266412032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266756424 ingress_bytes=301015040 link_bytes=301015040 token_bytes=533168456 weight_movement_bytes=34603008 total_nvlink_bytes=567771464 weight_to_token_ratio=0.064901 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132695362e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266518528 dispatch_meta_bytes=343920 combine_payload_bytes=266518528 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266862448 ingress_bytes=370327552 link_bytes=370327552 token_bytes=533380976 weight_movement_bytes=103809024 total_nvlink_bytes=637190000 weight_to_token_ratio=0.194625 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393512764e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=64834 dup_rows=23926 dup_rows_remote=20780 dedup_ratio=0.243388 router_tokens_per_expert=[802, 810, 754, 725, 780, 814, 793, 746, 767, 746, 760, 794, 794, 710, 714, 736, 816, 771, 759, 757, 758, 740, 744, 813, 755, 779, 760, 792, 778, 768, 750, 796, 774, 742, 812, 793, 770, 727, 784, 769, 778, 793, 762, 755, 730, 737, 775, 729, 798, 784, 757, 773, 795, 753, 768, 797, 739, 763, 749, 777, 774, 772, 801, 741] payload_rows_per_dst=[9138, 9202, 9242, 9410, 9320, 9404, 9118, 9544] dup_rows_per_dst=[3012, 2840, 2946, 3074, 3022, 3012, 2874, 3146] combine_rows_per_src=[9138, 9202, 9242, 9410, 9320, 9404, 9118, 9544] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74428 payload_rows_remote=65180 dup_rows=23876 dup_rows_remote=21110 dedup_ratio=0.242879 router_tokens_per_expert=[743, 822, 762, 783, 752, 723, 764, 763, 814, 792, 769, 772, 760, 770, 805, 812, 783, 728, 777, 730, 796, 740, 796, 781, 713, 758, 792, 760, 760, 768, 750, 733, 769, 765, 788, 793, 749, 750, 808, 785, 702, 768, 793, 733, 752, 754, 755, 750, 768, 792, 747, 773, 765, 844, 759, 801, 769, 774, 779, 754, 747, 769, 749, 777] payload_rows_per_dst=[9304, 9424, 9338, 9186, 9332, 9248, 9398, 9198] dup_rows_per_dst=[2920, 3164, 2924, 2882, 3082, 2766, 3100, 3038] combine_rows_per_src=[9304, 9424, 9338, 9186, 9332, 9248, 9398, 9198] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265666560 dispatch_meta_bytes=344136 combine_payload_bytes=265666560 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266010696 ingress_bytes=369475584 link_bytes=369475584 token_bytes=531677256 weight_movement_bytes=103809024 total_nvlink_bytes=635486280 weight_to_token_ratio=0.195248 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390306877e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74336 payload_rows_remote=65098 dup_rows=23968 dup_rows_remote=21028 dedup_ratio=0.243815 router_tokens_per_expert=[743, 737, 728, 784, 824, 795, 833, 787, 752, 761, 754, 758, 706, 772, 798, 788, 757, 737, 781, 811, 766, 773, 783, 749, 804, 767, 790, 787, 780, 755, 761, 782, 743, 745, 748, 775, 761, 769, 748, 756, 765, 770, 762, 744, 773, 769, 763, 734, 769, 768, 772, 779, 792, 761, 761, 756, 735, 785, 798, 791, 759, 734, 817, 747] payload_rows_per_dst=[9398, 9238, 9322, 9362, 9116, 9208, 9314, 9378] dup_rows_per_dst=[3064, 2940, 2992, 3090, 2974, 2952, 3002, 2954] combine_rows_per_src=[9398, 9238, 9322, 9362, 9116, 9208, 9314, 9378] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=264675328 dispatch_meta_bytes=343592 combine_payload_bytes=264675328 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265018920 ingress_bytes=333881344 link_bytes=333881344 token_bytes=529694248 weight_movement_bytes=69206016 total_nvlink_bytes=598900264 weight_to_token_ratio=0.130653 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256368618e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74226 payload_rows_remote=64724 dup_rows=24078 dup_rows_remote=20948 dedup_ratio=0.244934 router_tokens_per_expert=[814, 767, 759, 763, 784, 761, 812, 822, 775, 791, 763, 767, 787, 801, 758, 784, 751, 761, 761, 740, 748, 723, 738, 788, 748, 793, 721, 770, 810, 762, 739, 760, 789, 790, 777, 754, 745, 737, 783, 772, 777, 749, 782, 756, 747, 764, 761, 734, 815, 754, 798, 731, 814, 784, 773, 785, 746, 761, 754, 776, 727, 817, 725, 754] payload_rows_per_dst=[9502, 9502, 9142, 9180, 9204, 9198, 9348, 9150] dup_rows_per_dst=[3130, 3122, 2902, 3026, 3022, 2942, 2964, 2970] combine_rows_per_src=[9502, 9502, 9142, 9180, 9204, 9198, 9348, 9150] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265560064 dispatch_meta_bytes=342456 combine_payload_bytes=265560064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265902520 ingress_bytes=300163072 link_bytes=300163072 token_bytes=531462584 weight_movement_bytes=34603008 total_nvlink_bytes=566065592 weight_to_token_ratio=0.065109 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129489475e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266977280 dispatch_meta_bytes=345160 combine_payload_bytes=266977280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267322440 ingress_bytes=301580288 link_bytes=301580288 token_bytes=534299720 weight_movement_bytes=34603008 total_nvlink_bytes=568902728 weight_to_token_ratio=0.064763 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134822345e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=344504 combine_payload_bytes=266641408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266985912 ingress_bytes=301244416 link_bytes=301244416 token_bytes=533627320 weight_movement_bytes=34603008 total_nvlink_bytes=568230328 weight_to_token_ratio=0.064845 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133558485e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74230 payload_rows_remote=64880 dup_rows=24074 dup_rows_remote=20946 dedup_ratio=0.244893 router_tokens_per_expert=[762, 765, 735, 805, 733, 781, 744, 764, 765, 715, 765, 771, 769, 781, 724, 753, 792, 781, 762, 785, 790, 758, 758, 799, 771, 771, 738, 772, 767, 753, 749, 757, 775, 756, 758, 775, 761, 780, 801, 787, 739, 788, 746, 821, 770, 794, 757, 739, 777, 782, 792, 782, 782, 758, 785, 781, 779, 802, 750, 779, 745, 792, 749, 735] payload_rows_per_dst=[9188, 9224, 9428, 9246, 9318, 9276, 9350, 9200] dup_rows_per_dst=[2990, 2862, 3022, 2910, 3068, 3032, 3128, 3062] combine_rows_per_src=[9188, 9224, 9428, 9246, 9318, 9276, 9350, 9200] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265560064 dispatch_meta_bytes=342456 combine_payload_bytes=265560064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265902520 ingress_bytes=300163072 link_bytes=300163072 token_bytes=531462584 weight_movement_bytes=34603008 total_nvlink_bytes=566065592 weight_to_token_ratio=0.065109 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129489475e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74226 payload_rows_remote=64724 dup_rows=24078 dup_rows_remote=20948 dedup_ratio=0.244934 router_tokens_per_expert=[814, 767, 759, 763, 784, 761, 812, 822, 775, 791, 763, 767, 787, 801, 758, 784, 751, 761, 761, 740, 748, 723, 738, 788, 748, 793, 721, 770, 810, 762, 739, 760, 789, 790, 777, 754, 745, 737, 783, 772, 777, 749, 782, 756, 747, 764, 761, 734, 815, 754, 798, 731, 814, 784, 773, 785, 746, 761, 754, 776, 727, 817, 725, 754] payload_rows_per_dst=[9502, 9502, 9142, 9180, 9204, 9198, 9348, 9150] dup_rows_per_dst=[3130, 3122, 2902, 3026, 3022, 2942, 2964, 2970] combine_rows_per_src=[9502, 9502, 9142, 9180, 9204, 9198, 9348, 9150] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74424 payload_rows_remote=65016 dup_rows=23880 dup_rows_remote=20960 dedup_ratio=0.242920 router_tokens_per_expert=[786, 729, 759, 784, 767, 731, 771, 745, 771, 738, 796, 748, 822, 792, 710, 725, 792, 769, 782, 740, 759, 730, 779, 813, 785, 825, 784, 777, 838, 748, 736, 772, 749, 771, 814, 744, 770, 695, 740, 773, 803, 796, 832, 788, 746, 757, 774, 807, 778, 754, 767, 782, 736, 740, 781, 806, 780, 758, 735, 725, 799, 731, 751, 767] payload_rows_per_dst=[9184, 9212, 9408, 9490, 9102, 9550, 9270, 9208] dup_rows_per_dst=[2960, 2992, 2920, 3040, 3010, 3056, 3018, 2884] combine_rows_per_src=[9184, 9212, 9408, 9490, 9102, 9550, 9270, 9208] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265109504 dispatch_meta_bytes=342688 combine_payload_bytes=265109504 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265452192 ingress_bytes=334315520 link_bytes=334315520 token_bytes=530561696 weight_movement_bytes=69206016 total_nvlink_bytes=599767712 weight_to_token_ratio=0.130439 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258002387e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265748480 dispatch_meta_bytes=343304 combine_payload_bytes=265748480 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266091784 ingress_bytes=369557504 link_bytes=369557504 token_bytes=531840264 weight_movement_bytes=103809024 total_nvlink_bytes=635649288 weight_to_token_ratio=0.195188 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390615136e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=64904 dup_rows=24046 dup_rows_remote=21108 dedup_ratio=0.244609 router_tokens_per_expert=[809, 787, 755, 768, 762, 741, 746, 732, 738, 772, 748, 811, 765, 789, 785, 742, 753, 689, 779, 749, 804, 775, 808, 739, 752, 753, 781, 711, 786, 787, 814, 762, 803, 715, 778, 784, 760, 777, 744, 778, 800, 753, 760, 832, 768, 744, 814, 762, 742, 796, 741, 784, 755, 717, 820, 789, 799, 758, 772, 792, 821, 734, 718, 750] payload_rows_per_dst=[9214, 9330, 9192, 9354, 9236, 9350, 9300, 9282] dup_rows_per_dst=[2986, 2970, 3000, 2938, 3042, 3116, 2988, 3006] combine_rows_per_src=[9214, 9330, 9192, 9354, 9236, 9350, 9300, 9282] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=64844 dup_rows=24040 dup_rows_remote=21004 dedup_ratio=0.244548 router_tokens_per_expert=[765, 772, 760, 748, 771, 766, 760, 815, 798, 751, 761, 758, 768, 813, 747, 745, 791, 787, 821, 825, 757, 770, 767, 745, 781, 783, 753, 772, 729, 734, 751, 755, 773, 794, 795, 757, 761, 740, 824, 784, 785, 786, 776, 731, 735, 740, 786, 758, 764, 760, 769, 731, 737, 762, 801, 742, 799, 734, 751, 845, 754, 729, 768, 762] payload_rows_per_dst=[9236, 9292, 9434, 9226, 9420, 9230, 9158, 9268] dup_rows_per_dst=[3078, 2990, 3092, 2890, 3036, 2964, 2974, 3016] combine_rows_per_src=[9236, 9292, 9434, 9226, 9420, 9230, 9158, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74336 payload_rows_remote=65098 dup_rows=23968 dup_rows_remote=21028 dedup_ratio=0.243815 router_tokens_per_expert=[743, 737, 728, 784, 824, 795, 833, 787, 752, 761, 754, 758, 706, 772, 798, 788, 757, 737, 781, 811, 766, 773, 783, 749, 804, 767, 790, 787, 780, 755, 761, 782, 743, 745, 748, 775, 761, 769, 748, 756, 765, 770, 762, 744, 773, 769, 763, 734, 769, 768, 772, 779, 792, 761, 761, 756, 735, 785, 798, 791, 759, 734, 817, 747] payload_rows_per_dst=[9398, 9238, 9322, 9362, 9116, 9208, 9314, 9378] dup_rows_per_dst=[3064, 2940, 2992, 3090, 2974, 2952, 3002, 2954] combine_rows_per_src=[9398, 9238, 9322, 9362, 9116, 9208, 9314, 9378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=64844 dup_rows=24040 dup_rows_remote=21004 dedup_ratio=0.244548 router_tokens_per_expert=[765, 772, 760, 748, 771, 766, 760, 815, 798, 751, 761, 758, 768, 813, 747, 745, 791, 787, 821, 825, 757, 770, 767, 745, 781, 783, 753, 772, 729, 734, 751, 755, 773, 794, 795, 757, 761, 740, 824, 784, 785, 786, 776, 731, 735, 740, 786, 758, 764, 760, 769, 731, 737, 762, 801, 742, 799, 734, 751, 845, 754, 729, 768, 762] payload_rows_per_dst=[9236, 9292, 9434, 9226, 9420, 9230, 9158, 9268] dup_rows_per_dst=[3078, 2990, 3092, 2890, 3036, 2964, 2974, 3016] combine_rows_per_src=[9236, 9292, 9434, 9226, 9420, 9230, 9158, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74212 payload_rows_remote=64996 dup_rows=24092 dup_rows_remote=21086 dedup_ratio=0.245076 router_tokens_per_expert=[789, 739, 741, 781, 774, 749, 779, 758, 750, 768, 759, 782, 762, 765, 764, 768, 766, 742, 803, 763, 759, 747, 811, 747, 787, 774, 762, 757, 805, 739, 772, 757, 787, 726, 807, 778, 741, 787, 750, 730, 748, 746, 790, 777, 746, 764, 794, 746, 728, 814, 801, 785, 701, 810, 814, 822, 727, 779, 778, 785, 803, 790, 720, 759] payload_rows_per_dst=[9236, 9214, 9182, 9290, 9216, 9216, 9470, 9388] dup_rows_per_dst=[2984, 3022, 3094, 3016, 2996, 3006, 3080, 2894] combine_rows_per_src=[9236, 9214, 9182, 9290, 9216, 9216, 9470, 9388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265109504 dispatch_meta_bytes=342688 combine_payload_bytes=265109504 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265452192 ingress_bytes=334315520 link_bytes=334315520 token_bytes=530561696 weight_movement_bytes=69206016 total_nvlink_bytes=599767712 weight_to_token_ratio=0.130439 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258002387e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=343904 combine_payload_bytes=266305536 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266649440 ingress_bytes=335511552 link_bytes=335511552 token_bytes=532954976 weight_movement_bytes=69206016 total_nvlink_bytes=602160992 weight_to_token_ratio=0.129853 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262502959e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=65090 dup_rows=24062 dup_rows_remote=21126 dedup_ratio=0.244771 router_tokens_per_expert=[796, 786, 755, 784, 748, 737, 785, 778, 755, 765, 750, 808, 749, 749, 750, 728, 786, 796, 740, 787, 772, 768, 781, 751, 724, 767, 767, 754, 742, 815, 764, 766, 817, 788, 760, 747, 798, 780, 794, 750, 790, 791, 785, 788, 783, 809, 726, 795, 814, 744, 724, 726, 765, 744, 759, 768, 777, 810, 744, 801, 741, 757, 754, 720] payload_rows_per_dst=[9288, 9176, 9330, 9142, 9428, 9478, 9152, 9248] dup_rows_per_dst=[3050, 2932, 3032, 3056, 3040, 3056, 2936, 2960] combine_rows_per_src=[9288, 9176, 9330, 9142, 9428, 9478, 9152, 9248] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265846784 dispatch_meta_bytes=344048 combine_payload_bytes=265846784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266190832 ingress_bytes=300449792 link_bytes=300449792 token_bytes=532037616 weight_movement_bytes=34603008 total_nvlink_bytes=566640624 weight_to_token_ratio=0.065039 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130568379e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=344504 combine_payload_bytes=266641408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266985912 ingress_bytes=301244416 link_bytes=301244416 token_bytes=533627320 weight_movement_bytes=34603008 total_nvlink_bytes=568230328 weight_to_token_ratio=0.064845 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133558485e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74016 payload_rows_remote=64428 dup_rows=24288 dup_rows_remote=21000 dedup_ratio=0.247070 router_tokens_per_expert=[786, 731, 739, 793, 739, 762, 734, 766, 796, 783, 777, 778, 780, 803, 741, 786, 711, 762, 759, 768, 765, 774, 756, 741, 744, 735, 776, 791, 819, 755, 725, 723, 743, 748, 810, 769, 786, 743, 750, 778, 714, 808, 785, 759, 789, 811, 760, 777, 769, 798, 754, 776, 786, 784, 738, 815, 812, 740, 796, 792, 827, 734, 753, 750] payload_rows_per_dst=[9138, 9104, 9206, 9238, 9234, 9044, 9464, 9588] dup_rows_per_dst=[2962, 3002, 2866, 3054, 3032, 2894, 3190, 3288] combine_rows_per_src=[9138, 9104, 9206, 9238, 9234, 9044, 9464, 9588] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265601024 dispatch_meta_bytes=343392 combine_payload_bytes=265601024 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=265944416 ingress_bytes=300204032 link_bytes=300204032 token_bytes=531545440 weight_movement_bytes=34603008 total_nvlink_bytes=566148448 weight_to_token_ratio=0.065099 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129643604e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74212 payload_rows_remote=64996 dup_rows=24092 dup_rows_remote=21086 dedup_ratio=0.245076 router_tokens_per_expert=[789, 739, 741, 781, 774, 749, 779, 758, 750, 768, 759, 782, 762, 765, 764, 768, 766, 742, 803, 763, 759, 747, 811, 747, 787, 774, 762, 757, 805, 739, 772, 757, 787, 726, 807, 778, 741, 787, 750, 730, 748, 746, 790, 777, 746, 764, 794, 746, 728, 814, 801, 785, 701, 810, 814, 822, 727, 779, 778, 785, 803, 790, 720, 759] payload_rows_per_dst=[9236, 9214, 9182, 9290, 9216, 9216, 9470, 9388] dup_rows_per_dst=[2984, 3022, 3094, 3016, 2996, 3006, 3080, 2894] combine_rows_per_src=[9236, 9214, 9182, 9290, 9216, 9216, 9470, 9388] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=65060 dup_rows=24132 dup_rows_remote=21138 dedup_ratio=0.245483 router_tokens_per_expert=[775, 782, 786, 781, 765, 819, 818, 735, 745, 747, 737, 743, 791, 733, 803, 754, 825, 753, 738, 779, 787, 741, 781, 757, 751, 732, 749, 771, 751, 759, 759, 782, 763, 724, 800, 804, 819, 764, 783, 737, 775, 755, 736, 764, 769, 763, 760, 739, 765, 801, 766, 822, 751, 738, 777, 798, 722, 786, 765, 784, 759, 787, 798, 749] payload_rows_per_dst=[9480, 9112, 9318, 9164, 9312, 9176, 9298, 9312] dup_rows_per_dst=[3042, 2994, 3004, 2944, 3076, 2946, 3138, 2988] combine_rows_per_src=[9480, 9112, 9318, 9164, 9312, 9176, 9298, 9312] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=64662 dup_rows=24098 dup_rows_remote=21042 dedup_ratio=0.245138 router_tokens_per_expert=[762, 745, 778, 785, 770, 826, 784, 817, 752, 788, 769, 753, 787, 769, 789, 801, 769, 796, 737, 754, 760, 725, 745, 771, 757, 847, 794, 757, 801, 692, 736, 808, 764, 769, 755, 801, 785, 752, 747, 743, 757, 788, 784, 704, 754, 800, 719, 746, 782, 767, 767, 792, 753, 755, 829, 740, 762, 769, 740, 788, 778, 731, 741, 766] payload_rows_per_dst=[9544, 9298, 9596, 9286, 9256, 9080, 8976, 9170] dup_rows_per_dst=[3056, 3054, 3240, 3098, 2976, 3022, 2672, 2980] combine_rows_per_src=[9544, 9298, 9596, 9286, 9256, 9080, 8976, 9170] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266608640 dispatch_meta_bytes=344864 combine_payload_bytes=266608640 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266953504 ingress_bytes=370417664 link_bytes=370417664 token_bytes=533562144 weight_movement_bytes=103809024 total_nvlink_bytes=637371168 weight_to_token_ratio=0.194558 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393851849e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=263897088 dispatch_meta_bytes=341712 combine_payload_bytes=263897088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264238800 ingress_bytes=298500096 link_bytes=298500096 token_bytes=528135888 weight_movement_bytes=34603008 total_nvlink_bytes=562738896 weight_to_token_ratio=0.065519 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123231830e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265601024 dispatch_meta_bytes=343392 combine_payload_bytes=265601024 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=265944416 ingress_bytes=300204032 link_bytes=300204032 token_bytes=531545440 weight_movement_bytes=34603008 total_nvlink_bytes=566148448 weight_to_token_ratio=0.065099 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129643604e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=344328 combine_payload_bytes=266223616 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266567944 ingress_bytes=335429632 link_bytes=335429632 token_bytes=532791560 weight_movement_bytes=69206016 total_nvlink_bytes=601997576 weight_to_token_ratio=0.129893 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262194701e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266485760 dispatch_meta_bytes=344792 combine_payload_bytes=266485760 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=266830552 ingress_bytes=439500800 link_bytes=439500800 token_bytes=533316312 weight_movement_bytes=173015040 total_nvlink_bytes=706331352 weight_to_token_ratio=0.324414 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.653806128e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74530 payload_rows_remote=65210 dup_rows=23774 dup_rows_remote=20784 dedup_ratio=0.241842 router_tokens_per_expert=[778, 798, 743, 753, 792, 730, 702, 776, 820, 767, 757, 792, 778, 807, 769, 772, 775, 728, 782, 738, 751, 761, 711, 768, 807, 723, 744, 760, 819, 771, 773, 763, 811, 775, 765, 750, 798, 777, 795, 796, 765, 780, 748, 813, 751, 778, 765, 772, 762, 750, 764, 792, 803, 806, 718, 760, 795, 742, 743, 747, 719, 743, 771, 790] payload_rows_per_dst=[9230, 9466, 9172, 9334, 9464, 9328, 9320, 9216] dup_rows_per_dst=[2914, 3058, 2856, 2986, 3070, 3016, 2990, 2884] combine_rows_per_src=[9230, 9466, 9172, 9334, 9464, 9328, 9320, 9216] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=263897088 dispatch_meta_bytes=341712 combine_payload_bytes=263897088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264238800 ingress_bytes=298500096 link_bytes=298500096 token_bytes=528135888 weight_movement_bytes=34603008 total_nvlink_bytes=562738896 weight_to_token_ratio=0.065519 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123231830e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=64662 dup_rows=24098 dup_rows_remote=21042 dedup_ratio=0.245138 router_tokens_per_expert=[762, 745, 778, 785, 770, 826, 784, 817, 752, 788, 769, 753, 787, 769, 789, 801, 769, 796, 737, 754, 760, 725, 745, 771, 757, 847, 794, 757, 801, 692, 736, 808, 764, 769, 755, 801, 785, 752, 747, 743, 757, 788, 784, 704, 754, 800, 719, 746, 782, 767, 767, 792, 753, 755, 829, 740, 762, 769, 740, 788, 778, 731, 741, 766] payload_rows_per_dst=[9544, 9298, 9596, 9286, 9256, 9080, 8976, 9170] dup_rows_per_dst=[3056, 3054, 3240, 3098, 2976, 3022, 2672, 2980] combine_rows_per_src=[9544, 9298, 9596, 9286, 9256, 9080, 8976, 9170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=267100160 dispatch_meta_bytes=343976 combine_payload_bytes=267100160 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267444136 ingress_bytes=336306176 link_bytes=336306176 token_bytes=534544296 weight_movement_bytes=69206016 total_nvlink_bytes=603750312 weight_to_token_ratio=0.129467 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265493065e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=65042 dup_rows=23974 dup_rows_remote=21010 dedup_ratio=0.243876 router_tokens_per_expert=[743, 780, 806, 739, 740, 794, 756, 761, 796, 754, 821, 766, 810, 731, 780, 760, 742, 789, 808, 771, 706, 734, 779, 797, 771, 766, 776, 778, 764, 764, 795, 731, 727, 731, 763, 766, 815, 760, 793, 773, 778, 820, 833, 734, 767, 839, 767, 751, 756, 725, 740, 732, 762, 764, 761, 774, 783, 784, 712, 767, 777, 768, 734, 788] payload_rows_per_dst=[9318, 9324, 9288, 9312, 9252, 9450, 9118, 9268] dup_rows_per_dst=[2920, 3112, 2964, 2978, 3004, 3128, 2910, 2958] combine_rows_per_src=[9318, 9324, 9288, 9312, 9252, 9450, 9118, 9268] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=264855552 dispatch_meta_bytes=342816 combine_payload_bytes=264855552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265198368 ingress_bytes=299458560 link_bytes=299458560 token_bytes=530053920 weight_movement_bytes=34603008 total_nvlink_bytes=564656928 weight_to_token_ratio=0.065282 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126838453e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=64950 dup_rows=23962 dup_rows_remote=20954 dedup_ratio=0.243754 router_tokens_per_expert=[710, 785, 765, 691, 755, 747, 753, 760, 752, 829, 750, 783, 767, 768, 752, 765, 783, 763, 788, 731, 778, 751, 747, 795, 748, 795, 757, 816, 812, 759, 762, 751, 742, 750, 779, 732, 766, 774, 781, 737, 790, 803, 775, 779, 845, 781, 794, 815, 761, 810, 826, 736, 765, 758, 757, 733, 792, 756, 758, 731, 802, 758, 754, 744] payload_rows_per_dst=[9128, 9302, 9186, 9392, 9192, 9548, 9294, 9300] dup_rows_per_dst=[2804, 3030, 3086, 3008, 2930, 3216, 2998, 2890] combine_rows_per_src=[9128, 9302, 9186, 9392, 9192, 9548, 9294, 9300] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=65136 dup_rows=23860 dup_rows_remote=20874 dedup_ratio=0.242716 router_tokens_per_expert=[799, 767, 759, 759, 799, 809, 802, 780, 789, 771, 759, 751, 730, 755, 730, 821, 768, 724, 789, 749, 735, 779, 792, 787, 781, 744, 781, 748, 756, 764, 766, 762, 757, 736, 809, 747, 776, 789, 751, 782, 775, 799, 794, 741, 748, 723, 751, 786, 745, 742, 779, 835, 759, 764, 782, 787, 750, 736, 777, 763, 797, 758, 768, 741] payload_rows_per_dst=[9390, 9290, 9250, 9284, 9308, 9298, 9372, 9252] dup_rows_per_dst=[3158, 2922, 2996, 2920, 2986, 2936, 3014, 2928] combine_rows_per_src=[9390, 9290, 9250, 9284, 9308, 9298, 9372, 9252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=65060 dup_rows=24132 dup_rows_remote=21138 dedup_ratio=0.245483 router_tokens_per_expert=[775, 782, 786, 781, 765, 819, 818, 735, 745, 747, 737, 743, 791, 733, 803, 754, 825, 753, 738, 779, 787, 741, 781, 757, 751, 732, 749, 771, 751, 759, 759, 782, 763, 724, 800, 804, 819, 764, 783, 737, 775, 755, 736, 764, 769, 763, 760, 739, 765, 801, 766, 822, 751, 738, 777, 798, 722, 786, 765, 784, 759, 787, 798, 749] payload_rows_per_dst=[9480, 9112, 9318, 9164, 9312, 9176, 9298, 9312] dup_rows_per_dst=[3042, 2994, 3004, 2944, 3076, 2946, 3138, 2988] combine_rows_per_src=[9480, 9112, 9318, 9164, 9312, 9176, 9298, 9312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=344208 combine_payload_bytes=266412032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266756240 ingress_bytes=301015040 link_bytes=301015040 token_bytes=533168272 weight_movement_bytes=34603008 total_nvlink_bytes=567771280 weight_to_token_ratio=0.064901 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132695362e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74230 payload_rows_remote=64880 dup_rows=24074 dup_rows_remote=20946 dedup_ratio=0.244893 router_tokens_per_expert=[762, 765, 735, 805, 733, 781, 744, 764, 765, 715, 765, 771, 769, 781, 724, 753, 792, 781, 762, 785, 790, 758, 758, 799, 771, 771, 738, 772, 767, 753, 749, 757, 775, 756, 758, 775, 761, 780, 801, 787, 739, 788, 746, 821, 770, 794, 757, 739, 777, 782, 792, 782, 782, 758, 785, 781, 779, 802, 750, 779, 745, 792, 749, 735] payload_rows_per_dst=[9188, 9224, 9428, 9246, 9318, 9276, 9350, 9200] dup_rows_per_dst=[2990, 2862, 3022, 2910, 3068, 3032, 3128, 3062] combine_rows_per_src=[9188, 9224, 9428, 9246, 9318, 9276, 9350, 9200] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266035200 dispatch_meta_bytes=343616 combine_payload_bytes=266035200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266378816 ingress_bytes=300638208 link_bytes=300638208 token_bytes=532414016 weight_movement_bytes=34603008 total_nvlink_bytes=567017024 weight_to_token_ratio=0.064993 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131277373e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266797056 dispatch_meta_bytes=344040 combine_payload_bytes=266797056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267141096 ingress_bytes=301400064 link_bytes=301400064 token_bytes=533938152 weight_movement_bytes=34603008 total_nvlink_bytes=568541160 weight_to_token_ratio=0.064807 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134144176e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=64700 dup_rows=23992 dup_rows_remote=20674 dedup_ratio=0.244059 router_tokens_per_expert=[791, 745, 796, 781, 766, 779, 753, 723, 753, 745, 798, 763, 739, 785, 786, 784, 806, 776, 798, 804, 748, 782, 761, 782, 722, 792, 743, 782, 796, 777, 743, 765, 803, 729, 744, 811, 769, 755, 783, 735, 721, 790, 733, 817, 742, 752, 781, 759, 779, 803, 758, 776, 755, 744, 790, 771, 750, 771, 721, 765, 743, 776, 783, 779] payload_rows_per_dst=[8978, 9150, 8976, 9216, 9364, 9344, 9672, 9612] dup_rows_per_dst=[2870, 2866, 2768, 2914, 3020, 3136, 3100, 3318] combine_rows_per_src=[8978, 9150, 8976, 9216, 9364, 9344, 9672, 9612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=65136 dup_rows=23860 dup_rows_remote=20874 dedup_ratio=0.242716 router_tokens_per_expert=[799, 767, 759, 759, 799, 809, 802, 780, 789, 771, 759, 751, 730, 755, 730, 821, 768, 724, 789, 749, 735, 779, 792, 787, 781, 744, 781, 748, 756, 764, 766, 762, 757, 736, 809, 747, 776, 789, 751, 782, 775, 799, 794, 741, 748, 723, 751, 786, 745, 742, 779, 835, 759, 764, 782, 787, 750, 736, 777, 763, 797, 758, 768, 741] payload_rows_per_dst=[9390, 9290, 9250, 9284, 9308, 9298, 9372, 9252] dup_rows_per_dst=[3158, 2922, 2996, 2920, 2986, 2936, 3014, 2928] combine_rows_per_src=[9390, 9290, 9250, 9284, 9308, 9298, 9372, 9252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=65044 dup_rows=24008 dup_rows_remote=21090 dedup_ratio=0.244222 router_tokens_per_expert=[823, 763, 813, 837, 767, 789, 739, 767, 739, 789, 776, 798, 746, 792, 803, 828, 746, 799, 762, 785, 751, 794, 740, 751, 784, 747, 777, 726, 766, 775, 750, 770, 701, 782, 775, 732, 743, 700, 754, 735, 734, 758, 796, 734, 774, 744, 750, 795, 758, 716, 783, 782, 772, 769, 759, 753, 785, 770, 755, 808, 742, 795, 800, 806] payload_rows_per_dst=[9530, 9424, 9274, 9192, 8988, 9252, 9224, 9412] dup_rows_per_dst=[3066, 3118, 2982, 2998, 2856, 2918, 2960, 3110] combine_rows_per_src=[9530, 9424, 9274, 9192, 8988, 9252, 9224, 9412] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74324 payload_rows_remote=65108 dup_rows=23980 dup_rows_remote=20918 dedup_ratio=0.243937 router_tokens_per_expert=[771, 716, 753, 782, 767, 764, 802, 789, 772, 811, 770, 751, 731, 770, 770, 764, 796, 757, 844, 744, 781, 765, 782, 783, 756, 818, 738, 743, 731, 784, 796, 756, 807, 767, 767, 816, 800, 760, 807, 760, 808, 789, 734, 754, 748, 771, 735, 752, 756, 777, 756, 747, 755, 781, 742, 784, 762, 761, 730, 749, 738, 747, 735, 800] payload_rows_per_dst=[9272, 9216, 9404, 9256, 9484, 9242, 9250, 9200] dup_rows_per_dst=[3016, 3062, 3100, 2988, 3084, 2940, 2946, 2844] combine_rows_per_src=[9272, 9216, 9404, 9256, 9484, 9242, 9250, 9200] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=264855552 dispatch_meta_bytes=342816 combine_payload_bytes=264855552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265198368 ingress_bytes=299458560 link_bytes=299458560 token_bytes=530053920 weight_movement_bytes=34603008 total_nvlink_bytes=564656928 weight_to_token_ratio=0.065282 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126838453e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=344208 combine_payload_bytes=266412032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266756240 ingress_bytes=301015040 link_bytes=301015040 token_bytes=533168272 weight_movement_bytes=34603008 total_nvlink_bytes=567771280 weight_to_token_ratio=0.064901 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132695362e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266035200 dispatch_meta_bytes=343616 combine_payload_bytes=266035200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266378816 ingress_bytes=300638208 link_bytes=300638208 token_bytes=532414016 weight_movement_bytes=34603008 total_nvlink_bytes=567017024 weight_to_token_ratio=0.064993 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131277373e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=64700 dup_rows=23992 dup_rows_remote=20674 dedup_ratio=0.244059 router_tokens_per_expert=[791, 745, 796, 781, 766, 779, 753, 723, 753, 745, 798, 763, 739, 785, 786, 784, 806, 776, 798, 804, 748, 782, 761, 782, 722, 792, 743, 782, 796, 777, 743, 765, 803, 729, 744, 811, 769, 755, 783, 735, 721, 790, 733, 817, 742, 752, 781, 759, 779, 803, 758, 776, 755, 744, 790, 771, 750, 771, 721, 765, 743, 776, 783, 779] payload_rows_per_dst=[8978, 9150, 8976, 9216, 9364, 9344, 9672, 9612] dup_rows_per_dst=[2870, 2866, 2768, 2914, 3020, 3136, 3100, 3318] combine_rows_per_src=[8978, 9150, 8976, 9216, 9364, 9344, 9672, 9612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266485760 dispatch_meta_bytes=344792 combine_payload_bytes=266485760 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=266830552 ingress_bytes=439500800 link_bytes=439500800 token_bytes=533316312 weight_movement_bytes=173015040 total_nvlink_bytes=706331352 weight_to_token_ratio=0.324414 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.653806128e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74136 payload_rows_remote=64748 dup_rows=24168 dup_rows_remote=20982 dedup_ratio=0.245850 router_tokens_per_expert=[743, 785, 775, 757, 788, 819, 791, 774, 800, 739, 807, 812, 787, 787, 738, 743, 730, 781, 719, 797, 763, 771, 765, 747, 741, 798, 768, 782, 742, 748, 752, 750, 744, 808, 783, 786, 761, 804, 762, 800, 810, 746, 786, 778, 751, 798, 794, 774, 692, 767, 790, 714, 767, 799, 758, 726, 716, 754, 789, 792, 798, 733, 722, 751] payload_rows_per_dst=[9388, 9322, 9250, 9150, 9336, 9346, 9134, 9210] dup_rows_per_dst=[3186, 3122, 2878, 2902, 3160, 3128, 2892, 2900] combine_rows_per_src=[9388, 9322, 9250, 9150, 9336, 9346, 9134, 9210] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265748480 dispatch_meta_bytes=343304 combine_payload_bytes=265748480 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266091784 ingress_bytes=369557504 link_bytes=369557504 token_bytes=531840264 weight_movement_bytes=103809024 total_nvlink_bytes=635649288 weight_to_token_ratio=0.195188 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390615136e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265011200 dispatch_meta_bytes=341496 combine_payload_bytes=265011200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265352696 ingress_bytes=299614208 link_bytes=299614208 token_bytes=530363896 weight_movement_bytes=34603008 total_nvlink_bytes=564966904 weight_to_token_ratio=0.065244 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127424144e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266797056 dispatch_meta_bytes=344040 combine_payload_bytes=266797056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267141096 ingress_bytes=301400064 link_bytes=301400064 token_bytes=533938152 weight_movement_bytes=34603008 total_nvlink_bytes=568541160 weight_to_token_ratio=0.064807 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134144176e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=64826 dup_rows=24096 dup_rows_remote=21164 dedup_ratio=0.245117 router_tokens_per_expert=[717, 756, 748, 785, 782, 816, 803, 707, 791, 743, 756, 762, 754, 762, 758, 754, 786, 812, 773, 827, 771, 798, 752, 737, 775, 764, 781, 783, 809, 781, 780, 731, 745, 756, 746, 746, 754, 796, 785, 745, 763, 773, 786, 782, 716, 774, 769, 827, 770, 799, 764, 795, 756, 771, 770, 732, 728, 804, 755, 714, 784, 776, 746, 771] payload_rows_per_dst=[9112, 9232, 9410, 9336, 9180, 9330, 9382, 9226] dup_rows_per_dst=[3116, 2928, 3102, 3072, 2966, 3050, 2932, 2930] combine_rows_per_src=[9112, 9232, 9410, 9336, 9180, 9330, 9382, 9226] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=344536 combine_payload_bytes=266420224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266764760 ingress_bytes=301023232 link_bytes=301023232 token_bytes=533184984 weight_movement_bytes=34603008 total_nvlink_bytes=567787992 weight_to_token_ratio=0.064899 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132726188e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266682368 dispatch_meta_bytes=344104 combine_payload_bytes=266682368 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267026472 ingress_bytes=335888384 link_bytes=335888384 token_bytes=533708840 weight_movement_bytes=69206016 total_nvlink_bytes=602914856 weight_to_token_ratio=0.129670 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263920948e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265011200 dispatch_meta_bytes=341496 combine_payload_bytes=265011200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265352696 ingress_bytes=299614208 link_bytes=299614208 token_bytes=530363896 weight_movement_bytes=34603008 total_nvlink_bytes=564966904 weight_to_token_ratio=0.065244 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127424144e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265527296 dispatch_meta_bytes=343960 combine_payload_bytes=265527296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265871256 ingress_bytes=300130304 link_bytes=300130304 token_bytes=531398552 weight_movement_bytes=34603008 total_nvlink_bytes=566001560 weight_to_token_ratio=0.065117 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129366171e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74136 payload_rows_remote=64748 dup_rows=24168 dup_rows_remote=20982 dedup_ratio=0.245850 router_tokens_per_expert=[743, 785, 775, 757, 788, 819, 791, 774, 800, 739, 807, 812, 787, 787, 738, 743, 730, 781, 719, 797, 763, 771, 765, 747, 741, 798, 768, 782, 742, 748, 752, 750, 744, 808, 783, 786, 761, 804, 762, 800, 810, 746, 786, 778, 751, 798, 794, 774, 692, 767, 790, 714, 767, 799, 758, 726, 716, 754, 789, 792, 798, 733, 722, 751] payload_rows_per_dst=[9388, 9322, 9250, 9150, 9336, 9346, 9134, 9210] dup_rows_per_dst=[3186, 3122, 2878, 2902, 3160, 3128, 2892, 2900] combine_rows_per_src=[9388, 9322, 9250, 9150, 9336, 9346, 9134, 9210] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74434 payload_rows_remote=65008 dup_rows=23870 dup_rows_remote=20772 dedup_ratio=0.242818 router_tokens_per_expert=[791, 780, 718, 789, 766, 757, 805, 760, 767, 780, 791, 716, 765, 714, 742, 743, 755, 815, 787, 794, 772, 782, 769, 788, 776, 810, 762, 752, 817, 759, 809, 776, 799, 777, 773, 772, 753, 775, 750, 770, 775, 794, 768, 717, 739, 830, 775, 734, 758, 735, 757, 743, 773, 747, 786, 753, 717, 811, 757, 772, 789, 768, 729, 749] payload_rows_per_dst=[9288, 9118, 9426, 9416, 9414, 9240, 9272, 9260] dup_rows_per_dst=[3044, 2918, 3098, 3106, 2924, 3024, 2832, 2924] combine_rows_per_src=[9288, 9118, 9426, 9416, 9414, 9240, 9272, 9260] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265207808 dispatch_meta_bytes=342920 combine_payload_bytes=265207808 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265550728 ingress_bytes=334413824 link_bytes=334413824 token_bytes=530758536 weight_movement_bytes=69206016 total_nvlink_bytes=599964552 weight_to_token_ratio=0.130391 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258372297e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=64932 dup_rows=23966 dup_rows_remote=20936 dedup_ratio=0.243795 router_tokens_per_expert=[750, 762, 793, 758, 739, 758, 754, 791, 795, 787, 799, 765, 775, 745, 808, 764, 812, 756, 765, 761, 800, 733, 760, 788, 760, 820, 756, 727, 777, 783, 756, 777, 734, 783, 774, 776, 768, 763, 808, 812, 786, 745, 768, 746, 759, 727, 750, 765, 780, 781, 738, 805, 765, 741, 763, 754, 746, 778, 772, 743, 782, 757, 746, 763] payload_rows_per_dst=[9256, 9370, 9272, 9390, 9406, 9174, 9250, 9220] dup_rows_per_dst=[2954, 3106, 3078, 2922, 3030, 2918, 3004, 2954] combine_rows_per_src=[9256, 9370, 9272, 9390, 9406, 9174, 9250, 9220] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=64846 dup_rows=24040 dup_rows_remote=21012 dedup_ratio=0.244548 router_tokens_per_expert=[800, 742, 753, 739, 786, 760, 763, 771, 782, 775, 750, 807, 755, 773, 781, 753, 743, 776, 736, 773, 768, 783, 790, 774, 745, 758, 787, 772, 808, 790, 807, 756, 786, 748, 727, 757, 786, 738, 756, 748, 767, 736, 761, 782, 748, 811, 777, 749, 789, 797, 781, 776, 772, 747, 772, 793, 728, 776, 780, 739, 768, 794, 792, 715] payload_rows_per_dst=[9202, 9318, 9292, 9418, 9152, 9244, 9376, 9262] dup_rows_per_dst=[3026, 3034, 2994, 3028, 2940, 3018, 3078, 2922] combine_rows_per_src=[9202, 9318, 9292, 9418, 9152, 9244, 9376, 9262] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266272768 dispatch_meta_bytes=343120 combine_payload_bytes=266272768 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266615888 ingress_bytes=370081792 link_bytes=370081792 token_bytes=532888656 weight_movement_bytes=103809024 total_nvlink_bytes=636697680 weight_to_token_ratio=0.194804 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392587989e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74324 payload_rows_remote=65108 dup_rows=23980 dup_rows_remote=20918 dedup_ratio=0.243937 router_tokens_per_expert=[771, 716, 753, 782, 767, 764, 802, 789, 772, 811, 770, 751, 731, 770, 770, 764, 796, 757, 844, 744, 781, 765, 782, 783, 756, 818, 738, 743, 731, 784, 796, 756, 807, 767, 767, 816, 800, 760, 807, 760, 808, 789, 734, 754, 748, 771, 735, 752, 756, 777, 756, 747, 755, 781, 742, 784, 762, 761, 730, 749, 738, 747, 735, 800] payload_rows_per_dst=[9272, 9216, 9404, 9256, 9484, 9242, 9250, 9200] dup_rows_per_dst=[3016, 3062, 3100, 2988, 3084, 2940, 2946, 2844] combine_rows_per_src=[9272, 9216, 9404, 9256, 9484, 9242, 9250, 9200] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74530 payload_rows_remote=65210 dup_rows=23774 dup_rows_remote=20784 dedup_ratio=0.241842 router_tokens_per_expert=[778, 798, 743, 753, 792, 730, 702, 776, 820, 767, 757, 792, 778, 807, 769, 772, 775, 728, 782, 738, 751, 761, 711, 768, 807, 723, 744, 760, 819, 771, 773, 763, 811, 775, 765, 750, 798, 777, 795, 796, 765, 780, 748, 813, 751, 778, 765, 772, 762, 750, 764, 792, 803, 806, 718, 760, 795, 742, 743, 747, 719, 743, 771, 790] payload_rows_per_dst=[9230, 9466, 9172, 9334, 9464, 9328, 9320, 9216] dup_rows_per_dst=[2914, 3058, 2856, 2986, 3070, 3016, 2990, 2884] combine_rows_per_src=[9230, 9466, 9172, 9334, 9464, 9328, 9320, 9216] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265961472 dispatch_meta_bytes=343472 combine_payload_bytes=265961472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266304944 ingress_bytes=300564480 link_bytes=300564480 token_bytes=532266416 weight_movement_bytes=34603008 total_nvlink_bytes=566869424 weight_to_token_ratio=0.065011 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130999941e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=64932 dup_rows=23966 dup_rows_remote=20936 dedup_ratio=0.243795 router_tokens_per_expert=[750, 762, 793, 758, 739, 758, 754, 791, 795, 787, 799, 765, 775, 745, 808, 764, 812, 756, 765, 761, 800, 733, 760, 788, 760, 820, 756, 727, 777, 783, 756, 777, 734, 783, 774, 776, 768, 763, 808, 812, 786, 745, 768, 746, 759, 727, 750, 765, 780, 781, 738, 805, 765, 741, 763, 754, 746, 778, 772, 743, 782, 757, 746, 763] payload_rows_per_dst=[9256, 9370, 9272, 9390, 9406, 9174, 9250, 9220] dup_rows_per_dst=[2954, 3106, 3078, 2922, 3030, 2918, 3004, 2954] combine_rows_per_src=[9256, 9370, 9272, 9390, 9406, 9174, 9250, 9220] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265207808 dispatch_meta_bytes=342920 combine_payload_bytes=265207808 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265550728 ingress_bytes=334413824 link_bytes=334413824 token_bytes=530758536 weight_movement_bytes=69206016 total_nvlink_bytes=599964552 weight_to_token_ratio=0.130391 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258372297e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74188 payload_rows_remote=64984 dup_rows=24116 dup_rows_remote=21160 dedup_ratio=0.245321 router_tokens_per_expert=[775, 741, 790, 778, 780, 727, 791, 755, 742, 708, 772, 801, 777, 752, 756, 810, 794, 784, 769, 784, 768, 758, 757, 747, 761, 762, 796, 825, 733, 740, 730, 806, 707, 772, 799, 774, 819, 806, 773, 752, 768, 800, 742, 747, 737, 781, 734, 771, 807, 737, 756, 749, 790, 740, 788, 771, 756, 770, 758, 778, 762, 768, 796, 775] payload_rows_per_dst=[9348, 9212, 9258, 9206, 9342, 9204, 9316, 9302] dup_rows_per_dst=[2926, 3024, 3064, 3100, 3062, 2956, 2960, 3024] combine_rows_per_src=[9348, 9212, 9258, 9206, 9342, 9204, 9316, 9302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266272768 dispatch_meta_bytes=343120 combine_payload_bytes=266272768 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266615888 ingress_bytes=370081792 link_bytes=370081792 token_bytes=532888656 weight_movement_bytes=103809024 total_nvlink_bytes=636697680 weight_to_token_ratio=0.194804 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392587989e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=64904 dup_rows=24098 dup_rows_remote=20918 dedup_ratio=0.245138 router_tokens_per_expert=[789, 784, 753, 806, 795, 786, 740, 756, 773, 793, 784, 755, 767, 763, 752, 750, 816, 760, 787, 769, 773, 765, 729, 768, 728, 745, 732, 795, 762, 751, 738, 749, 782, 820, 771, 799, 779, 795, 783, 725, 739, 732, 767, 762, 758, 772, 759, 813, 717, 762, 753, 768, 767, 797, 782, 751, 756, 820, 768, 801, 778, 767, 748, 748] payload_rows_per_dst=[8982, 9478, 9330, 9216, 9296, 9304, 9298, 9302] dup_rows_per_dst=[2872, 3120, 3024, 2784, 3002, 3082, 3034, 3180] combine_rows_per_src=[8982, 9478, 9330, 9216, 9296, 9304, 9298, 9302] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265609216 dispatch_meta_bytes=343432 combine_payload_bytes=265609216 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265952648 ingress_bytes=369418240 link_bytes=369418240 token_bytes=531561864 weight_movement_bytes=103809024 total_nvlink_bytes=635370888 weight_to_token_ratio=0.195291 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390091097e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74188 payload_rows_remote=64984 dup_rows=24116 dup_rows_remote=21160 dedup_ratio=0.245321 router_tokens_per_expert=[775, 741, 790, 778, 780, 727, 791, 755, 742, 708, 772, 801, 777, 752, 756, 810, 794, 784, 769, 784, 768, 758, 757, 747, 761, 762, 796, 825, 733, 740, 730, 806, 707, 772, 799, 774, 819, 806, 773, 752, 768, 800, 742, 747, 737, 781, 734, 771, 807, 737, 756, 749, 790, 740, 788, 771, 756, 770, 758, 778, 762, 768, 796, 775] payload_rows_per_dst=[9348, 9212, 9258, 9206, 9342, 9204, 9316, 9302] dup_rows_per_dst=[2926, 3024, 3064, 3100, 3062, 2956, 2960, 3024] combine_rows_per_src=[9348, 9212, 9258, 9206, 9342, 9204, 9316, 9302] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74412 payload_rows_remote=64978 dup_rows=23892 dup_rows_remote=20866 dedup_ratio=0.243042 router_tokens_per_expert=[790, 760, 753, 756, 825, 807, 737, 762, 732, 795, 753, 759, 843, 792, 791, 765, 800, 719, 765, 750, 771, 746, 795, 753, 820, 751, 803, 755, 719, 757, 780, 738, 753, 735, 759, 749, 782, 789, 763, 773, 784, 759, 781, 784, 742, 754, 767, 756, 765, 699, 776, 767, 797, 787, 771, 774, 797, 777, 786, 716, 716, 744, 825, 783] payload_rows_per_dst=[9332, 9434, 9286, 9302, 9310, 9242, 9272, 9234] dup_rows_per_dst=[3048, 3026, 2912, 2944, 2896, 3012, 3000, 3054] combine_rows_per_src=[9332, 9434, 9286, 9302, 9310, 9242, 9272, 9234] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266682368 dispatch_meta_bytes=344104 combine_payload_bytes=266682368 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267026472 ingress_bytes=335888384 link_bytes=335888384 token_bytes=533708840 weight_movement_bytes=69206016 total_nvlink_bytes=602914856 weight_to_token_ratio=0.129670 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263920948e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=267100160 dispatch_meta_bytes=343976 combine_payload_bytes=267100160 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267444136 ingress_bytes=336306176 link_bytes=336306176 token_bytes=534544296 weight_movement_bytes=69206016 total_nvlink_bytes=603750312 weight_to_token_ratio=0.129467 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265493065e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74414 payload_rows_remote=65124 dup_rows=23890 dup_rows_remote=20786 dedup_ratio=0.243022 router_tokens_per_expert=[763, 819, 735, 716, 785, 820, 772, 787, 770, 783, 751, 696, 791, 796, 786, 793, 710, 732, 814, 763, 797, 777, 783, 785, 731, 752, 729, 746, 781, 757, 758, 782, 793, 793, 810, 780, 769, 713, 836, 756, 769, 800, 786, 763, 798, 763, 789, 749, 821, 720, 776, 754, 776, 761, 750, 763, 797, 754, 778, 713, 749, 737, 716, 760] payload_rows_per_dst=[9290, 9278, 9330, 9462, 9198, 9354, 9350, 9152] dup_rows_per_dst=[3104, 3054, 2992, 3078, 2834, 3080, 2892, 2856] combine_rows_per_src=[9290, 9278, 9330, 9462, 9198, 9354, 9350, 9152] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265846784 dispatch_meta_bytes=343288 combine_payload_bytes=265846784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266190072 ingress_bytes=300449792 link_bytes=300449792 token_bytes=532036856 weight_movement_bytes=34603008 total_nvlink_bytes=566639864 weight_to_token_ratio=0.065039 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130568379e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74278 payload_rows_remote=64982 dup_rows=24026 dup_rows_remote=21050 dedup_ratio=0.244405 router_tokens_per_expert=[738, 776, 739, 765, 771, 777, 799, 747, 762, 751, 744, 767, 726, 774, 753, 765, 760, 757, 782, 749, 744, 785, 720, 777, 766, 788, 770, 776, 783, 808, 771, 769, 813, 792, 807, 754, 753, 780, 768, 756, 740, 754, 755, 782, 835, 765, 779, 762, 769, 742, 778, 791, 776, 757, 763, 760, 759, 753, 743, 691, 801, 799, 765, 851] payload_rows_per_dst=[9220, 9160, 9176, 9362, 9410, 9294, 9296, 9360] dup_rows_per_dst=[3004, 2924, 2972, 3100, 3036, 3050, 2976, 2964] combine_rows_per_src=[9220, 9160, 9176, 9362, 9410, 9294, 9296, 9360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265961472 dispatch_meta_bytes=343472 combine_payload_bytes=265961472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266304944 ingress_bytes=300564480 link_bytes=300564480 token_bytes=532266416 weight_movement_bytes=34603008 total_nvlink_bytes=566869424 weight_to_token_ratio=0.065011 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130999941e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266174464 dispatch_meta_bytes=344576 combine_payload_bytes=266174464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266519040 ingress_bytes=300777472 link_bytes=300777472 token_bytes=532693504 weight_movement_bytes=34603008 total_nvlink_bytes=567296512 weight_to_token_ratio=0.064959 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131801413e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=343376 combine_payload_bytes=266149888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266493264 ingress_bytes=300752896 link_bytes=300752896 token_bytes=532643152 weight_movement_bytes=34603008 total_nvlink_bytes=567246160 weight_to_token_ratio=0.064965 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131708935e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265846784 dispatch_meta_bytes=343288 combine_payload_bytes=265846784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266190072 ingress_bytes=300449792 link_bytes=300449792 token_bytes=532036856 weight_movement_bytes=34603008 total_nvlink_bytes=566639864 weight_to_token_ratio=0.065039 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130568379e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266166272 dispatch_meta_bytes=344128 combine_payload_bytes=266166272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266510400 ingress_bytes=300769280 link_bytes=300769280 token_bytes=532676672 weight_movement_bytes=34603008 total_nvlink_bytes=567279680 weight_to_token_ratio=0.064961 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131770587e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74414 payload_rows_remote=65124 dup_rows=23890 dup_rows_remote=20786 dedup_ratio=0.243022 router_tokens_per_expert=[763, 819, 735, 716, 785, 820, 772, 787, 770, 783, 751, 696, 791, 796, 786, 793, 710, 732, 814, 763, 797, 777, 783, 785, 731, 752, 729, 746, 781, 757, 758, 782, 793, 793, 810, 780, 769, 713, 836, 756, 769, 800, 786, 763, 798, 763, 789, 749, 821, 720, 776, 754, 776, 761, 750, 763, 797, 754, 778, 713, 749, 737, 716, 760] payload_rows_per_dst=[9290, 9278, 9330, 9462, 9198, 9354, 9350, 9152] dup_rows_per_dst=[3104, 3054, 2992, 3078, 2834, 3080, 2892, 2856] combine_rows_per_src=[9290, 9278, 9330, 9462, 9198, 9354, 9350, 9152] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74574 payload_rows_remote=65218 dup_rows=23730 dup_rows_remote=20678 dedup_ratio=0.241394 router_tokens_per_expert=[702, 791, 731, 801, 737, 754, 759, 788, 732, 776, 718, 791, 853, 741, 769, 773, 846, 767, 781, 765, 802, 727, 748, 768, 790, 753, 746, 753, 766, 751, 741, 765, 791, 826, 758, 754, 744, 774, 755, 772, 741, 729, 784, 772, 791, 809, 723, 787, 808, 796, 749, 777, 779, 798, 729, 802, 759, 791, 721, 753, 731, 786, 818, 760] payload_rows_per_dst=[9234, 9318, 9356, 9244, 9380, 9364, 9406, 9272] dup_rows_per_dst=[2892, 2988, 3052, 2886, 2968, 2908, 3070, 2966] combine_rows_per_src=[9234, 9318, 9356, 9244, 9380, 9364, 9406, 9272] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266747904 dispatch_meta_bytes=343640 combine_payload_bytes=266747904 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267091544 ingress_bytes=405159936 link_bytes=405159936 token_bytes=533839448 weight_movement_bytes=138412032 total_nvlink_bytes=672251480 weight_to_token_ratio=0.259277 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524584221e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64782 dup_rows=24112 dup_rows_remote=21120 dedup_ratio=0.245280 router_tokens_per_expert=[807, 769, 835, 774, 748, 791, 779, 797, 739, 703, 757, 764, 830, 798, 769, 743, 790, 769, 795, 779, 791, 737, 713, 753, 758, 779, 741, 712, 796, 790, 813, 785, 778, 778, 760, 765, 762, 768, 785, 805, 766, 754, 703, 795, 775, 748, 772, 754, 763, 771, 770, 766, 749, 766, 720, 775, 762, 747, 736, 748, 769, 822, 765, 751] payload_rows_per_dst=[9424, 9166, 9236, 9332, 9410, 9228, 9208, 9188] dup_rows_per_dst=[3176, 3040, 3018, 3016, 2992, 2906, 2952, 3012] combine_rows_per_src=[9424, 9166, 9236, 9332, 9410, 9228, 9208, 9188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74366 payload_rows_remote=65056 dup_rows=23938 dup_rows_remote=20976 dedup_ratio=0.243510 router_tokens_per_expert=[781, 770, 757, 770, 728, 763, 852, 805, 790, 746, 756, 725, 740, 733, 813, 738, 739, 824, 798, 740, 736, 742, 745, 825, 756, 749, 791, 808, 785, 772, 750, 725, 741, 785, 777, 755, 747, 797, 724, 758, 789, 737, 779, 754, 786, 766, 776, 773, 744, 764, 772, 788, 765, 750, 761, 793, 794, 747, 770, 785, 748, 773, 787, 815] payload_rows_per_dst=[9416, 9104, 9310, 9310, 9254, 9356, 9290, 9326] dup_rows_per_dst=[3036, 2978, 2988, 2962, 2914, 2964, 2984, 3112] combine_rows_per_src=[9416, 9104, 9310, 9310, 9254, 9356, 9290, 9326] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=267132928 dispatch_meta_bytes=343584 combine_payload_bytes=267132928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267476512 ingress_bytes=301735936 link_bytes=301735936 token_bytes=534609440 weight_movement_bytes=34603008 total_nvlink_bytes=569212448 weight_to_token_ratio=0.064726 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135408035e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74412 payload_rows_remote=64978 dup_rows=23892 dup_rows_remote=20866 dedup_ratio=0.243042 router_tokens_per_expert=[790, 760, 753, 756, 825, 807, 737, 762, 732, 795, 753, 759, 843, 792, 791, 765, 800, 719, 765, 750, 771, 746, 795, 753, 820, 751, 803, 755, 719, 757, 780, 738, 753, 735, 759, 749, 782, 789, 763, 773, 784, 759, 781, 784, 742, 754, 767, 756, 765, 699, 776, 767, 797, 787, 771, 774, 797, 777, 786, 716, 716, 744, 825, 783] payload_rows_per_dst=[9332, 9434, 9286, 9302, 9310, 9242, 9272, 9234] dup_rows_per_dst=[3048, 3026, 2912, 2944, 2896, 3012, 3000, 3054] combine_rows_per_src=[9332, 9434, 9286, 9302, 9310, 9242, 9272, 9234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265347072 dispatch_meta_bytes=343608 combine_payload_bytes=265347072 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265690680 ingress_bytes=403759104 link_bytes=403759104 token_bytes=531037752 weight_movement_bytes=138412032 total_nvlink_bytes=669449784 weight_to_token_ratio=0.260644 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519313003e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=64826 dup_rows=24096 dup_rows_remote=21164 dedup_ratio=0.245117 router_tokens_per_expert=[717, 756, 748, 785, 782, 816, 803, 707, 791, 743, 756, 762, 754, 762, 758, 754, 786, 812, 773, 827, 771, 798, 752, 737, 775, 764, 781, 783, 809, 781, 780, 731, 745, 756, 746, 746, 754, 796, 785, 745, 763, 773, 786, 782, 716, 774, 769, 827, 770, 799, 764, 795, 756, 771, 770, 732, 728, 804, 755, 714, 784, 776, 746, 771] payload_rows_per_dst=[9112, 9232, 9410, 9336, 9180, 9330, 9382, 9226] dup_rows_per_dst=[3116, 2928, 3102, 3072, 2966, 3050, 2932, 2930] combine_rows_per_src=[9112, 9232, 9410, 9336, 9180, 9330, 9382, 9226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64782 dup_rows=24112 dup_rows_remote=21120 dedup_ratio=0.245280 router_tokens_per_expert=[807, 769, 835, 774, 748, 791, 779, 797, 739, 703, 757, 764, 830, 798, 769, 743, 790, 769, 795, 779, 791, 737, 713, 753, 758, 779, 741, 712, 796, 790, 813, 785, 778, 778, 760, 765, 762, 768, 785, 805, 766, 754, 703, 795, 775, 748, 772, 754, 763, 771, 770, 766, 749, 766, 720, 775, 762, 747, 736, 748, 769, 822, 765, 751] payload_rows_per_dst=[9424, 9166, 9236, 9332, 9410, 9228, 9208, 9188] dup_rows_per_dst=[3176, 3040, 3018, 3016, 2992, 2906, 2952, 3012] combine_rows_per_src=[9424, 9166, 9236, 9332, 9410, 9228, 9208, 9188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74432 payload_rows_remote=65228 dup_rows=23872 dup_rows_remote=20952 dedup_ratio=0.242839 router_tokens_per_expert=[775, 767, 778, 805, 732, 764, 729, 749, 742, 776, 773, 771, 743, 723, 822, 775, 772, 784, 811, 745, 777, 804, 769, 766, 780, 752, 729, 761, 750, 812, 738, 782, 789, 784, 788, 740, 743, 778, 767, 774, 737, 762, 745, 748, 763, 816, 740, 751, 729, 745, 784, 765, 822, 771, 802, 806, 767, 763, 811, 740, 758, 763, 769, 776] payload_rows_per_dst=[9206, 9294, 9368, 9262, 9396, 9204, 9448, 9254] dup_rows_per_dst=[2992, 2956, 3088, 2946, 2930, 2920, 3000, 3040] combine_rows_per_src=[9206, 9294, 9368, 9262, 9396, 9204, 9448, 9254] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266747904 dispatch_meta_bytes=343640 combine_payload_bytes=266747904 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267091544 ingress_bytes=405159936 link_bytes=405159936 token_bytes=533839448 weight_movement_bytes=138412032 total_nvlink_bytes=672251480 weight_to_token_ratio=0.259277 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524584221e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=267132928 dispatch_meta_bytes=343584 combine_payload_bytes=267132928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267476512 ingress_bytes=301735936 link_bytes=301735936 token_bytes=534609440 weight_movement_bytes=34603008 total_nvlink_bytes=569212448 weight_to_token_ratio=0.064726 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135408035e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=343376 combine_payload_bytes=266149888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266493264 ingress_bytes=300752896 link_bytes=300752896 token_bytes=532643152 weight_movement_bytes=34603008 total_nvlink_bytes=567246160 weight_to_token_ratio=0.064965 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131708935e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73998 payload_rows_remote=64254 dup_rows=24306 dup_rows_remote=20772 dedup_ratio=0.247253 router_tokens_per_expert=[766, 780, 784, 779, 782, 766, 781, 757, 744, 730, 775, 833, 778, 771, 743, 794, 789, 763, 762, 789, 701, 789, 791, 771, 736, 745, 725, 747, 750, 729, 780, 749, 793, 814, 722, 737, 772, 785, 771, 738, 736, 744, 793, 765, 809, 774, 799, 775, 777, 776, 797, 792, 797, 780, 776, 781, 779, 742, 746, 782, 703, 748, 784, 786] payload_rows_per_dst=[9182, 9078, 9312, 8854, 8980, 9450, 9398, 9744] dup_rows_per_dst=[3010, 2930, 3010, 2610, 2790, 3268, 3154, 3534] combine_rows_per_src=[9182, 9078, 9312, 8854, 8980, 9450, 9398, 9744] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266469376 dispatch_meta_bytes=344128 combine_payload_bytes=266469376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266813504 ingress_bytes=301072384 link_bytes=301072384 token_bytes=533282880 weight_movement_bytes=34603008 total_nvlink_bytes=567885888 weight_to_token_ratio=0.064887 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132911143e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74432 payload_rows_remote=65228 dup_rows=23872 dup_rows_remote=20952 dedup_ratio=0.242839 router_tokens_per_expert=[775, 767, 778, 805, 732, 764, 729, 749, 742, 776, 773, 771, 743, 723, 822, 775, 772, 784, 811, 745, 777, 804, 769, 766, 780, 752, 729, 761, 750, 812, 738, 782, 789, 784, 788, 740, 743, 778, 767, 774, 737, 762, 745, 748, 763, 816, 740, 751, 729, 745, 784, 765, 822, 771, 802, 806, 767, 763, 811, 740, 758, 763, 769, 776] payload_rows_per_dst=[9206, 9294, 9368, 9262, 9396, 9204, 9448, 9254] dup_rows_per_dst=[2992, 2956, 3088, 2946, 2930, 2920, 3000, 3040] combine_rows_per_src=[9206, 9294, 9368, 9262, 9396, 9204, 9448, 9254] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64790 dup_rows=24112 dup_rows_remote=21032 dedup_ratio=0.245280 router_tokens_per_expert=[779, 805, 742, 768, 740, 733, 788, 760, 705, 762, 811, 831, 790, 776, 763, 803, 770, 769, 750, 808, 769, 754, 806, 715, 772, 768, 769, 779, 811, 736, 821, 792, 757, 837, 805, 816, 789, 739, 763, 774, 745, 760, 764, 749, 730, 799, 749, 723, 759, 731, 789, 779, 798, 740, 759, 752, 778, 721, 764, 759, 732, 732, 728, 787] payload_rows_per_dst=[9262, 9402, 9294, 9346, 9418, 9106, 9248, 9116] dup_rows_per_dst=[2968, 3080, 2988, 3150, 3142, 2932, 2966, 2886] combine_rows_per_src=[9262, 9402, 9294, 9346, 9418, 9106, 9248, 9116] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74300 payload_rows_remote=65020 dup_rows=24004 dup_rows_remote=20846 dedup_ratio=0.244181 router_tokens_per_expert=[783, 764, 729, 764, 774, 817, 784, 804, 716, 767, 797, 790, 755, 750, 788, 732, 803, 757, 781, 784, 792, 775, 733, 788, 767, 732, 776, 783, 746, 800, 768, 756, 757, 744, 786, 784, 818, 756, 768, 783, 782, 782, 747, 835, 756, 750, 784, 852, 745, 759, 746, 765, 773, 784, 740, 747, 732, 721, 758, 732, 752, 750, 756, 753] payload_rows_per_dst=[9280, 9242, 9374, 9226, 9378, 9474, 9228, 9098] dup_rows_per_dst=[3158, 3018, 3052, 3030, 3014, 3102, 2820, 2810] combine_rows_per_src=[9280, 9242, 9374, 9226, 9378, 9474, 9228, 9098] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265527296 dispatch_meta_bytes=343960 combine_payload_bytes=265527296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265871256 ingress_bytes=300130304 link_bytes=300130304 token_bytes=531398552 weight_movement_bytes=34603008 total_nvlink_bytes=566001560 weight_to_token_ratio=0.065117 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129366171e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=263184384 dispatch_meta_bytes=340104 combine_payload_bytes=263184384 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263524488 ingress_bytes=297787392 link_bytes=297787392 token_bytes=526708872 weight_movement_bytes=34603008 total_nvlink_bytes=561311880 weight_to_token_ratio=0.065697 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.120549982e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74470 payload_rows_remote=65154 dup_rows=23834 dup_rows_remote=20944 dedup_ratio=0.242452 router_tokens_per_expert=[790, 786, 766, 777, 813, 786, 749, 788, 774, 740, 762, 779, 753, 811, 781, 783, 722, 760, 783, 737, 789, 720, 797, 765, 777, 787, 766, 768, 766, 789, 789, 795, 755, 801, 801, 743, 720, 747, 784, 787, 783, 765, 710, 751, 763, 769, 809, 790, 715, 800, 766, 750, 762, 777, 789, 744, 753, 763, 758, 762, 761, 758, 745, 723] payload_rows_per_dst=[9456, 9428, 9216, 9392, 9186, 9326, 9316, 9150] dup_rows_per_dst=[3054, 2938, 2930, 3082, 3090, 2954, 2890, 2896] combine_rows_per_src=[9456, 9428, 9216, 9392, 9186, 9326, 9316, 9150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265347072 dispatch_meta_bytes=343608 combine_payload_bytes=265347072 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265690680 ingress_bytes=403759104 link_bytes=403759104 token_bytes=531037752 weight_movement_bytes=138412032 total_nvlink_bytes=669449784 weight_to_token_ratio=0.260644 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519313003e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=267173888 dispatch_meta_bytes=344720 combine_payload_bytes=267173888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267518608 ingress_bytes=301776896 link_bytes=301776896 token_bytes=534692496 weight_movement_bytes=34603008 total_nvlink_bytes=569295504 weight_to_token_ratio=0.064716 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135562165e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265379840 dispatch_meta_bytes=343288 combine_payload_bytes=265379840 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265723128 ingress_bytes=334585856 link_bytes=334585856 token_bytes=531102968 weight_movement_bytes=69206016 total_nvlink_bytes=600308984 weight_to_token_ratio=0.130306 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259019640e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=263184384 dispatch_meta_bytes=340104 combine_payload_bytes=263184384 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263524488 ingress_bytes=297787392 link_bytes=297787392 token_bytes=526708872 weight_movement_bytes=34603008 total_nvlink_bytes=561311880 weight_to_token_ratio=0.065697 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.120549982e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266870784 dispatch_meta_bytes=344392 combine_payload_bytes=266870784 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267215176 ingress_bytes=301473792 link_bytes=301473792 token_bytes=534085960 weight_movement_bytes=34603008 total_nvlink_bytes=568688968 weight_to_token_ratio=0.064789 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134421609e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74300 payload_rows_remote=65020 dup_rows=24004 dup_rows_remote=20846 dedup_ratio=0.244181 router_tokens_per_expert=[783, 764, 729, 764, 774, 817, 784, 804, 716, 767, 797, 790, 755, 750, 788, 732, 803, 757, 781, 784, 792, 775, 733, 788, 767, 732, 776, 783, 746, 800, 768, 756, 757, 744, 786, 784, 818, 756, 768, 783, 782, 782, 747, 835, 756, 750, 784, 852, 745, 759, 746, 765, 773, 784, 740, 747, 732, 721, 758, 732, 752, 750, 756, 753] payload_rows_per_dst=[9280, 9242, 9374, 9226, 9378, 9474, 9228, 9098] dup_rows_per_dst=[3158, 3018, 3052, 3030, 3014, 3102, 2820, 2810] combine_rows_per_src=[9280, 9242, 9374, 9226, 9378, 9474, 9228, 9098] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74158 payload_rows_remote=65076 dup_rows=24146 dup_rows_remote=21116 dedup_ratio=0.245626 router_tokens_per_expert=[751, 768, 794, 767, 757, 765, 760, 793, 777, 791, 732, 742, 763, 808, 771, 760, 775, 768, 767, 783, 723, 699, 780, 761, 837, 763, 775, 778, 794, 784, 805, 779, 755, 774, 770, 781, 758, 812, 799, 769, 749, 765, 735, 789, 800, 798, 726, 716, 764, 796, 746, 711, 702, 825, 783, 737, 749, 715, 774, 775, 754, 814, 747, 794] payload_rows_per_dst=[9270, 9374, 9082, 9498, 9310, 9234, 9164, 9226] dup_rows_per_dst=[3040, 2914, 3030, 3132, 3126, 2922, 2964, 3018] combine_rows_per_src=[9270, 9374, 9082, 9498, 9310, 9234, 9164, 9226] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266321920 dispatch_meta_bytes=343464 combine_payload_bytes=266321920 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266665384 ingress_bytes=335527936 link_bytes=335527936 token_bytes=532987304 weight_movement_bytes=69206016 total_nvlink_bytes=602193320 weight_to_token_ratio=0.129846 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262564611e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74348 payload_rows_remote=64986 dup_rows=23956 dup_rows_remote=20878 dedup_ratio=0.243693 router_tokens_per_expert=[772, 795, 785, 760, 760, 797, 748, 742, 768, 775, 754, 786, 771, 770, 758, 765, 768, 766, 723, 739, 791, 767, 798, 779, 778, 768, 781, 819, 761, 751, 783, 768, 812, 755, 742, 789, 788, 780, 741, 813, 747, 754, 753, 798, 768, 752, 740, 749, 758, 815, 737, 778, 777, 765, 828, 752, 746, 745, 742, 747, 763, 774, 783, 715] payload_rows_per_dst=[9304, 9376, 9266, 9296, 9362, 9222, 9400, 9122] dup_rows_per_dst=[3014, 2918, 2996, 3122, 3078, 2900, 3020, 2908] combine_rows_per_src=[9304, 9376, 9266, 9296, 9362, 9222, 9400, 9122] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64790 dup_rows=24112 dup_rows_remote=21032 dedup_ratio=0.245280 router_tokens_per_expert=[779, 805, 742, 768, 740, 733, 788, 760, 705, 762, 811, 831, 790, 776, 763, 803, 770, 769, 750, 808, 769, 754, 806, 715, 772, 768, 769, 779, 811, 736, 821, 792, 757, 837, 805, 816, 789, 739, 763, 774, 745, 760, 764, 749, 730, 799, 749, 723, 759, 731, 789, 779, 798, 740, 759, 752, 778, 721, 764, 759, 732, 732, 728, 787] payload_rows_per_dst=[9262, 9402, 9294, 9346, 9418, 9106, 9248, 9116] dup_rows_per_dst=[2968, 3080, 2988, 3150, 3142, 2932, 2966, 2886] combine_rows_per_src=[9262, 9402, 9294, 9346, 9418, 9106, 9248, 9116] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=64978 dup_rows=23962 dup_rows_remote=20968 dedup_ratio=0.243754 router_tokens_per_expert=[753, 749, 766, 783, 762, 747, 736, 758, 744, 763, 793, 765, 797, 746, 784, 786, 761, 751, 777, 782, 752, 777, 797, 752, 806, 845, 725, 777, 749, 761, 742, 774, 745, 799, 735, 733, 761, 730, 792, 757, 735, 749, 754, 756, 766, 843, 789, 753, 785, 789, 729, 750, 750, 765, 802, 774, 772, 796, 767, 801, 736, 807, 801, 771] payload_rows_per_dst=[9238, 9326, 9332, 9364, 9196, 9240, 9242, 9404] dup_rows_per_dst=[2870, 3030, 2966, 2994, 2908, 3050, 3046, 3098] combine_rows_per_src=[9238, 9326, 9332, 9364, 9196, 9240, 9242, 9404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266551296 dispatch_meta_bytes=344768 combine_payload_bytes=266551296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266896064 ingress_bytes=301154304 link_bytes=301154304 token_bytes=533447360 weight_movement_bytes=34603008 total_nvlink_bytes=568050368 weight_to_token_ratio=0.064867 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133219401e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266182656 dispatch_meta_bytes=343456 combine_payload_bytes=266182656 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266526112 ingress_bytes=335388672 link_bytes=335388672 token_bytes=532708768 weight_movement_bytes=69206016 total_nvlink_bytes=601914784 weight_to_token_ratio=0.129913 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262040572e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74278 payload_rows_remote=64982 dup_rows=24026 dup_rows_remote=21050 dedup_ratio=0.244405 router_tokens_per_expert=[738, 776, 739, 765, 771, 777, 799, 747, 762, 751, 744, 767, 726, 774, 753, 765, 760, 757, 782, 749, 744, 785, 720, 777, 766, 788, 770, 776, 783, 808, 771, 769, 813, 792, 807, 754, 753, 780, 768, 756, 740, 754, 755, 782, 835, 765, 779, 762, 769, 742, 778, 791, 776, 757, 763, 760, 759, 753, 743, 691, 801, 799, 765, 851] payload_rows_per_dst=[9220, 9160, 9176, 9362, 9410, 9294, 9296, 9360] dup_rows_per_dst=[3004, 2924, 2972, 3100, 3036, 3050, 2976, 2964] combine_rows_per_src=[9220, 9160, 9176, 9362, 9410, 9294, 9296, 9360] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74462 payload_rows_remote=65050 dup_rows=23842 dup_rows_remote=20908 dedup_ratio=0.242533 router_tokens_per_expert=[778, 744, 739, 784, 744, 763, 750, 806, 742, 766, 767, 706, 702, 781, 775, 815, 771, 778, 803, 747, 781, 790, 762, 739, 776, 796, 769, 755, 765, 792, 803, 749, 773, 760, 783, 805, 732, 787, 805, 727, 789, 780, 781, 794, 721, 766, 757, 785, 746, 762, 812, 819, 734, 768, 744, 777, 737, 781, 751, 775, 743, 756, 777, 787] payload_rows_per_dst=[9268, 9178, 9298, 9390, 9300, 9412, 9326, 9290] dup_rows_per_dst=[2948, 2930, 3044, 3020, 3044, 2934, 2998, 2924] combine_rows_per_src=[9268, 9178, 9298, 9390, 9300, 9412, 9326, 9290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74348 payload_rows_remote=64986 dup_rows=23956 dup_rows_remote=20878 dedup_ratio=0.243693 router_tokens_per_expert=[772, 795, 785, 760, 760, 797, 748, 742, 768, 775, 754, 786, 771, 770, 758, 765, 768, 766, 723, 739, 791, 767, 798, 779, 778, 768, 781, 819, 761, 751, 783, 768, 812, 755, 742, 789, 788, 780, 741, 813, 747, 754, 753, 798, 768, 752, 740, 749, 758, 815, 737, 778, 777, 765, 828, 752, 746, 745, 742, 747, 763, 774, 783, 715] payload_rows_per_dst=[9304, 9376, 9266, 9296, 9362, 9222, 9400, 9122] dup_rows_per_dst=[3014, 2918, 2996, 3122, 3078, 2900, 3020, 2908] combine_rows_per_src=[9304, 9376, 9266, 9296, 9362, 9222, 9400, 9122] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266321920 dispatch_meta_bytes=343464 combine_payload_bytes=266321920 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266665384 ingress_bytes=335527936 link_bytes=335527936 token_bytes=532987304 weight_movement_bytes=69206016 total_nvlink_bytes=602193320 weight_to_token_ratio=0.129846 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262564611e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266551296 dispatch_meta_bytes=344768 combine_payload_bytes=266551296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266896064 ingress_bytes=301154304 link_bytes=301154304 token_bytes=533447360 weight_movement_bytes=34603008 total_nvlink_bytes=568050368 weight_to_token_ratio=0.064867 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133219401e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265379840 dispatch_meta_bytes=343288 combine_payload_bytes=265379840 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265723128 ingress_bytes=334585856 link_bytes=334585856 token_bytes=531102968 weight_movement_bytes=69206016 total_nvlink_bytes=600308984 weight_to_token_ratio=0.130306 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259019640e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74530 payload_rows_remote=65096 dup_rows=23774 dup_rows_remote=20696 dedup_ratio=0.241842 router_tokens_per_expert=[758, 774, 782, 772, 708, 782, 754, 796, 797, 788, 792, 731, 765, 726, 800, 788, 753, 763, 761, 724, 746, 770, 793, 789, 805, 756, 759, 690, 789, 814, 762, 758, 792, 801, 788, 747, 772, 766, 722, 804, 766, 726, 763, 795, 727, 753, 742, 767, 782, 793, 790, 753, 792, 767, 722, 788, 793, 767, 786, 779, 810, 789, 721, 744] payload_rows_per_dst=[9272, 9396, 9332, 9274, 9186, 9298, 9338, 9434] dup_rows_per_dst=[2980, 2978, 2866, 2992, 2838, 3006, 3036, 3078] combine_rows_per_src=[9272, 9396, 9332, 9274, 9186, 9298, 9338, 9434] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=343784 combine_payload_bytes=266149888 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266493672 ingress_bytes=335355904 link_bytes=335355904 token_bytes=532643560 weight_movement_bytes=69206016 total_nvlink_bytes=601849576 weight_to_token_ratio=0.129929 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261917268e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=65142 dup_rows=23906 dup_rows_remote=20816 dedup_ratio=0.243184 router_tokens_per_expert=[805, 787, 750, 749, 794, 787, 765, 743, 776, 733, 787, 737, 794, 779, 779, 788, 744, 774, 765, 707, 725, 714, 765, 790, 769, 743, 771, 806, 703, 714, 785, 766, 753, 742, 823, 788, 788, 800, 807, 768, 780, 787, 772, 713, 784, 796, 748, 804, 765, 785, 766, 760, 740, 795, 772, 753, 762, 752, 774, 802, 802, 801, 775, 701] payload_rows_per_dst=[9356, 9256, 9096, 9202, 9396, 9384, 9370, 9338] dup_rows_per_dst=[3004, 3090, 2872, 2912, 3142, 2984, 2902, 3000] combine_rows_per_src=[9356, 9256, 9096, 9202, 9396, 9384, 9370, 9338] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74234 payload_rows_remote=64946 dup_rows=24070 dup_rows_remote=21022 dedup_ratio=0.244853 router_tokens_per_expert=[771, 774, 770, 750, 796, 763, 749, 715, 777, 746, 741, 757, 779, 780, 768, 714, 793, 755, 792, 784, 798, 821, 767, 727, 766, 793, 795, 761, 777, 755, 783, 789, 765, 818, 770, 753, 800, 719, 800, 812, 745, 780, 756, 756, 762, 756, 761, 728, 781, 777, 797, 715, 796, 798, 774, 747, 744, 744, 768, 780, 760, 784, 774, 726] payload_rows_per_dst=[9288, 9418, 9362, 9442, 9292, 9240, 8930, 9262] dup_rows_per_dst=[3048, 3266, 2972, 3216, 2978, 2848, 2844, 2898] combine_rows_per_src=[9288, 9418, 9362, 9442, 9292, 9240, 8930, 9262] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266166272 dispatch_meta_bytes=344128 combine_payload_bytes=266166272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266510400 ingress_bytes=300769280 link_bytes=300769280 token_bytes=532676672 weight_movement_bytes=34603008 total_nvlink_bytes=567279680 weight_to_token_ratio=0.064961 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131770587e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=64996 dup_rows=24008 dup_rows_remote=20856 dedup_ratio=0.244222 router_tokens_per_expert=[747, 779, 772, 753, 753, 752, 787, 720, 805, 764, 756, 791, 804, 731, 735, 744, 738, 795, 777, 786, 762, 826, 779, 754, 732, 807, 759, 737, 771, 800, 736, 758, 785, 746, 770, 752, 725, 778, 758, 773, 830, 723, 751, 744, 820, 771, 757, 738, 765, 739, 807, 780, 728, 826, 786, 795, 770, 745, 817, 743, 766, 797, 781, 776] payload_rows_per_dst=[9190, 9262, 9390, 9250, 9300, 9264, 9300, 9340] dup_rows_per_dst=[2936, 2998, 3044, 2950, 2874, 3004, 3152, 3050] combine_rows_per_src=[9190, 9262, 9390, 9250, 9300, 9264, 9300, 9340] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266444800 dispatch_meta_bytes=343832 combine_payload_bytes=266444800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266788632 ingress_bytes=301047808 link_bytes=301047808 token_bytes=533233432 weight_movement_bytes=34603008 total_nvlink_bytes=567836440 weight_to_token_ratio=0.064893 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132818665e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266182656 dispatch_meta_bytes=343456 combine_payload_bytes=266182656 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266526112 ingress_bytes=335388672 link_bytes=335388672 token_bytes=532708768 weight_movement_bytes=69206016 total_nvlink_bytes=601914784 weight_to_token_ratio=0.129913 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262040572e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266633216 dispatch_meta_bytes=343168 combine_payload_bytes=266633216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266976384 ingress_bytes=301236224 link_bytes=301236224 token_bytes=533609600 weight_movement_bytes=34603008 total_nvlink_bytes=568212608 weight_to_token_ratio=0.064847 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133527659e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266821632 dispatch_meta_bytes=343832 combine_payload_bytes=266821632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267165464 ingress_bytes=301424640 link_bytes=301424640 token_bytes=533987096 weight_movement_bytes=34603008 total_nvlink_bytes=568590104 weight_to_token_ratio=0.064801 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134236654e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266018816 dispatch_meta_bytes=343872 combine_payload_bytes=266018816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266362688 ingress_bytes=300621824 link_bytes=300621824 token_bytes=532381504 weight_movement_bytes=34603008 total_nvlink_bytes=566984512 weight_to_token_ratio=0.064997 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131215722e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=343408 combine_payload_bytes=266223616 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266567024 ingress_bytes=370032640 link_bytes=370032640 token_bytes=532790640 weight_movement_bytes=103809024 total_nvlink_bytes=636599664 weight_to_token_ratio=0.194840 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392403034e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74234 payload_rows_remote=64946 dup_rows=24070 dup_rows_remote=21022 dedup_ratio=0.244853 router_tokens_per_expert=[771, 774, 770, 750, 796, 763, 749, 715, 777, 746, 741, 757, 779, 780, 768, 714, 793, 755, 792, 784, 798, 821, 767, 727, 766, 793, 795, 761, 777, 755, 783, 789, 765, 818, 770, 753, 800, 719, 800, 812, 745, 780, 756, 756, 762, 756, 761, 728, 781, 777, 797, 715, 796, 798, 774, 747, 744, 744, 768, 780, 760, 784, 774, 726] payload_rows_per_dst=[9288, 9418, 9362, 9442, 9292, 9240, 8930, 9262] dup_rows_per_dst=[3048, 3266, 2972, 3216, 2978, 2848, 2844, 2898] combine_rows_per_src=[9288, 9418, 9362, 9442, 9292, 9240, 8930, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=65142 dup_rows=23906 dup_rows_remote=20816 dedup_ratio=0.243184 router_tokens_per_expert=[805, 787, 750, 749, 794, 787, 765, 743, 776, 733, 787, 737, 794, 779, 779, 788, 744, 774, 765, 707, 725, 714, 765, 790, 769, 743, 771, 806, 703, 714, 785, 766, 753, 742, 823, 788, 788, 800, 807, 768, 780, 787, 772, 713, 784, 796, 748, 804, 765, 785, 766, 760, 740, 795, 772, 753, 762, 752, 774, 802, 802, 801, 775, 701] payload_rows_per_dst=[9356, 9256, 9096, 9202, 9396, 9384, 9370, 9338] dup_rows_per_dst=[3004, 3090, 2872, 2912, 3142, 2984, 2902, 3000] combine_rows_per_src=[9356, 9256, 9096, 9202, 9396, 9384, 9370, 9338] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=65012 dup_rows=24098 dup_rows_remote=21064 dedup_ratio=0.245138 router_tokens_per_expert=[781, 742, 731, 784, 759, 773, 786, 759, 758, 772, 729, 746, 793, 761, 734, 761, 754, 741, 741, 759, 802, 729, 808, 780, 804, 780, 776, 751, 726, 800, 807, 757, 805, 743, 777, 766, 791, 788, 757, 750, 784, 804, 784, 748, 778, 740, 768, 824, 722, 785, 799, 765, 817, 741, 815, 717, 741, 787, 799, 753, 776, 756, 771, 717] payload_rows_per_dst=[9152, 9298, 9194, 9356, 9298, 9348, 9294, 9266] dup_rows_per_dst=[3078, 2810, 3034, 3046, 3056, 3112, 3028, 2934] combine_rows_per_src=[9152, 9298, 9194, 9356, 9298, 9348, 9294, 9266] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74274 payload_rows_remote=65026 dup_rows=24030 dup_rows_remote=20966 dedup_ratio=0.244446 router_tokens_per_expert=[776, 772, 764, 742, 743, 744, 815, 842, 735, 753, 764, 764, 778, 716, 768, 782, 776, 744, 796, 784, 764, 742, 823, 770, 773, 735, 705, 784, 743, 719, 767, 791, 790, 749, 743, 803, 754, 752, 774, 791, 790, 754, 797, 787, 735, 753, 801, 709, 799, 791, 778, 787, 764, 799, 773, 743, 747, 744, 797, 785, 796, 748, 754, 791] payload_rows_per_dst=[9332, 9252, 9370, 9110, 9248, 9258, 9400, 9304] dup_rows_per_dst=[3064, 2868, 3028, 2924, 3064, 2994, 3068, 3020] combine_rows_per_src=[9332, 9252, 9370, 9110, 9248, 9258, 9400, 9304] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74078 payload_rows_remote=64824 dup_rows=24226 dup_rows_remote=21260 dedup_ratio=0.246440 router_tokens_per_expert=[755, 820, 750, 762, 775, 813, 766, 753, 729, 734, 777, 760, 802, 817, 770, 763, 797, 748, 729, 765, 787, 795, 737, 843, 786, 795, 748, 737, 728, 813, 762, 741, 771, 747, 813, 745, 770, 776, 723, 799, 746, 766, 740, 799, 724, 798, 763, 773, 769, 816, 787, 730, 787, 748, 759, 763, 747, 787, 750, 762, 775, 737, 782, 743] payload_rows_per_dst=[9304, 9290, 9326, 9254, 9266, 9260, 9200, 9178] dup_rows_per_dst=[3084, 3014, 3076, 2966, 3022, 2958, 3118, 2988] combine_rows_per_src=[9304, 9290, 9326, 9254, 9266, 9260, 9200, 9178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74470 payload_rows_remote=65154 dup_rows=23834 dup_rows_remote=20944 dedup_ratio=0.242452 router_tokens_per_expert=[790, 786, 766, 777, 813, 786, 749, 788, 774, 740, 762, 779, 753, 811, 781, 783, 722, 760, 783, 737, 789, 720, 797, 765, 777, 787, 766, 768, 766, 789, 789, 795, 755, 801, 801, 743, 720, 747, 784, 787, 783, 765, 710, 751, 763, 769, 809, 790, 715, 800, 766, 750, 762, 777, 789, 744, 753, 763, 758, 762, 761, 758, 745, 723] payload_rows_per_dst=[9456, 9428, 9216, 9392, 9186, 9326, 9316, 9150] dup_rows_per_dst=[3054, 2938, 2930, 3082, 3090, 2954, 2890, 2896] combine_rows_per_src=[9456, 9428, 9216, 9392, 9186, 9326, 9316, 9150] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266289152 dispatch_meta_bytes=344304 combine_payload_bytes=266289152 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266633456 ingress_bytes=335495168 link_bytes=335495168 token_bytes=532922608 weight_movement_bytes=69206016 total_nvlink_bytes=602128624 weight_to_token_ratio=0.129861 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262441308e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=65094 dup_rows=23966 dup_rows_remote=21108 dedup_ratio=0.243795 router_tokens_per_expert=[802, 795, 735, 760, 778, 782, 765, 793, 764, 787, 731, 768, 767, 794, 785, 790, 773, 759, 757, 790, 725, 743, 785, 789, 727, 742, 795, 785, 770, 760, 803, 771, 748, 742, 826, 750, 766, 783, 738, 759, 784, 728, 751, 797, 715, 806, 742, 728, 742, 748, 799, 745, 783, 764, 770, 794, 786, 780, 778, 778, 780, 762, 777, 733] payload_rows_per_dst=[9460, 9298, 9228, 9330, 9166, 9244, 9230, 9382] dup_rows_per_dst=[2960, 3074, 3014, 2976, 3058, 2858, 3060, 2966] combine_rows_per_src=[9460, 9298, 9228, 9330, 9166, 9244, 9230, 9382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266346496 dispatch_meta_bytes=343968 combine_payload_bytes=266346496 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266690464 ingress_bytes=404758528 link_bytes=404758528 token_bytes=533036960 weight_movement_bytes=138412032 total_nvlink_bytes=671448992 weight_to_token_ratio=0.259667 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523073755e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74274 payload_rows_remote=65026 dup_rows=24030 dup_rows_remote=20966 dedup_ratio=0.244446 router_tokens_per_expert=[776, 772, 764, 742, 743, 744, 815, 842, 735, 753, 764, 764, 778, 716, 768, 782, 776, 744, 796, 784, 764, 742, 823, 770, 773, 735, 705, 784, 743, 719, 767, 791, 790, 749, 743, 803, 754, 752, 774, 791, 790, 754, 797, 787, 735, 753, 801, 709, 799, 791, 778, 787, 764, 799, 773, 743, 747, 744, 797, 785, 796, 748, 754, 791] payload_rows_per_dst=[9332, 9252, 9370, 9110, 9248, 9258, 9400, 9304] dup_rows_per_dst=[3064, 2868, 3028, 2924, 3064, 2994, 3068, 3020] combine_rows_per_src=[9332, 9252, 9370, 9110, 9248, 9258, 9400, 9304] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=64916 dup_rows=23948 dup_rows_remote=20768 dedup_ratio=0.243612 router_tokens_per_expert=[787, 794, 817, 745, 769, 824, 817, 716, 765, 744, 746, 787, 775, 724, 773, 763, 784, 768, 730, 792, 762, 771, 796, 784, 773, 700, 764, 755, 740, 764, 726, 799, 768, 763, 782, 784, 740, 780, 765, 761, 761, 764, 777, 778, 761, 767, 754, 793, 806, 761, 751, 756, 776, 746, 719, 771, 742, 763, 814, 738, 749, 805, 807, 796] payload_rows_per_dst=[9364, 9230, 9126, 9070, 9288, 9256, 9582, 9440] dup_rows_per_dst=[3006, 2880, 3036, 2792, 3030, 2814, 3210, 3180] combine_rows_per_src=[9364, 9230, 9126, 9070, 9288, 9256, 9582, 9440] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265519104 dispatch_meta_bytes=344336 combine_payload_bytes=265519104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265863440 ingress_bytes=300122112 link_bytes=300122112 token_bytes=531382544 weight_movement_bytes=34603008 total_nvlink_bytes=565985552 weight_to_token_ratio=0.065119 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129335346e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266018816 dispatch_meta_bytes=343872 combine_payload_bytes=266018816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266362688 ingress_bytes=300621824 link_bytes=300621824 token_bytes=532381504 weight_movement_bytes=34603008 total_nvlink_bytes=566984512 weight_to_token_ratio=0.064997 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131215722e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266821632 dispatch_meta_bytes=343832 combine_payload_bytes=266821632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267165464 ingress_bytes=301424640 link_bytes=301424640 token_bytes=533987096 weight_movement_bytes=34603008 total_nvlink_bytes=568590104 weight_to_token_ratio=0.064801 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134236654e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266289152 dispatch_meta_bytes=344304 combine_payload_bytes=266289152 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266633456 ingress_bytes=335495168 link_bytes=335495168 token_bytes=532922608 weight_movement_bytes=69206016 total_nvlink_bytes=602128624 weight_to_token_ratio=0.129861 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262441308e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=65068 dup_rows=24032 dup_rows_remote=21042 dedup_ratio=0.244466 router_tokens_per_expert=[756, 719, 780, 741, 780, 836, 760, 745, 805, 729, 742, 778, 780, 754, 778, 731, 772, 782, 740, 757, 787, 756, 769, 766, 751, 809, 808, 747, 779, 762, 772, 747, 745, 716, 765, 823, 786, 735, 790, 811, 731, 786, 837, 804, 740, 804, 796, 735, 731, 758, 743, 801, 794, 814, 783, 733, 717, 755, 743, 747, 764, 797, 776, 774] payload_rows_per_dst=[9204, 9204, 9174, 9348, 9338, 9392, 9374, 9238] dup_rows_per_dst=[3030, 2990, 3084, 3002, 3004, 3074, 2940, 2908] combine_rows_per_src=[9204, 9204, 9174, 9348, 9338, 9392, 9374, 9238] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74138 payload_rows_remote=64834 dup_rows=24166 dup_rows_remote=21032 dedup_ratio=0.245829 router_tokens_per_expert=[785, 807, 805, 734, 743, 747, 785, 768, 799, 771, 773, 785, 747, 788, 740, 786, 820, 768, 777, 722, 789, 802, 737, 756, 753, 713, 736, 812, 755, 787, 813, 778, 736, 726, 745, 773, 774, 748, 772, 769, 751, 761, 769, 820, 766, 791, 745, 757, 729, 756, 763, 714, 809, 755, 751, 812, 746, 804, 770, 762, 792, 755, 798, 752] payload_rows_per_dst=[9304, 9358, 9336, 9254, 9084, 9272, 9214, 9316] dup_rows_per_dst=[3134, 3020, 3006, 3040, 2912, 3048, 2964, 3042] combine_rows_per_src=[9304, 9358, 9336, 9254, 9084, 9272, 9214, 9316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265519104 dispatch_meta_bytes=344336 combine_payload_bytes=265519104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265863440 ingress_bytes=300122112 link_bytes=300122112 token_bytes=531382544 weight_movement_bytes=34603008 total_nvlink_bytes=565985552 weight_to_token_ratio=0.065119 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129335346e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266870784 dispatch_meta_bytes=344392 combine_payload_bytes=266870784 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267215176 ingress_bytes=301473792 link_bytes=301473792 token_bytes=534085960 weight_movement_bytes=34603008 total_nvlink_bytes=568688968 weight_to_token_ratio=0.064789 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134421609e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=65024 dup_rows=24098 dup_rows_remote=21126 dedup_ratio=0.245138 router_tokens_per_expert=[715, 784, 726, 747, 757, 839, 779, 763, 767, 744, 752, 756, 788, 697, 765, 780, 786, 778, 791, 760, 734, 758, 816, 779, 792, 735, 795, 727, 780, 748, 772, 775, 801, 797, 762, 788, 804, 759, 759, 818, 742, 796, 750, 789, 828, 760, 761, 826, 789, 784, 755, 791, 716, 745, 755, 742, 771, 768, 741, 755, 756, 751, 756, 752] payload_rows_per_dst=[9178, 9224, 9314, 9228, 9440, 9432, 9182, 9208] dup_rows_per_dst=[3042, 2874, 3090, 3020, 3136, 3072, 2972, 2892] combine_rows_per_src=[9178, 9224, 9314, 9228, 9440, 9432, 9182, 9208] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266625024 dispatch_meta_bytes=344808 combine_payload_bytes=266625024 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266969832 ingress_bytes=335831040 link_bytes=335831040 token_bytes=533594856 weight_movement_bytes=69206016 total_nvlink_bytes=602800872 weight_to_token_ratio=0.129698 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263705167e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266346496 dispatch_meta_bytes=343968 combine_payload_bytes=266346496 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266690464 ingress_bytes=404758528 link_bytes=404758528 token_bytes=533036960 weight_movement_bytes=138412032 total_nvlink_bytes=671448992 weight_to_token_ratio=0.259667 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523073755e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265895936 dispatch_meta_bytes=342736 combine_payload_bytes=265895936 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266238672 ingress_bytes=300498944 link_bytes=300498944 token_bytes=532134608 weight_movement_bytes=34603008 total_nvlink_bytes=566737616 weight_to_token_ratio=0.065027 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130753334e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266518528 dispatch_meta_bytes=344440 combine_payload_bytes=266518528 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266862968 ingress_bytes=335724544 link_bytes=335724544 token_bytes=533381496 weight_movement_bytes=69206016 total_nvlink_bytes=602587512 weight_to_token_ratio=0.129750 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263304431e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265560064 dispatch_meta_bytes=343464 combine_payload_bytes=265560064 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265903528 ingress_bytes=334766080 link_bytes=334766080 token_bytes=531463592 weight_movement_bytes=69206016 total_nvlink_bytes=600669608 weight_to_token_ratio=0.130218 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259697808e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=344600 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266682904 ingress_bytes=300941312 link_bytes=300941312 token_bytes=533021208 weight_movement_bytes=34603008 total_nvlink_bytes=567624216 weight_to_token_ratio=0.064919 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132417929e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=65068 dup_rows=24032 dup_rows_remote=21042 dedup_ratio=0.244466 router_tokens_per_expert=[756, 719, 780, 741, 780, 836, 760, 745, 805, 729, 742, 778, 780, 754, 778, 731, 772, 782, 740, 757, 787, 756, 769, 766, 751, 809, 808, 747, 779, 762, 772, 747, 745, 716, 765, 823, 786, 735, 790, 811, 731, 786, 837, 804, 740, 804, 796, 735, 731, 758, 743, 801, 794, 814, 783, 733, 717, 755, 743, 747, 764, 797, 776, 774] payload_rows_per_dst=[9204, 9204, 9174, 9348, 9338, 9392, 9374, 9238] dup_rows_per_dst=[3030, 2990, 3084, 3002, 3004, 3074, 2940, 2908] combine_rows_per_src=[9204, 9204, 9174, 9348, 9338, 9392, 9374, 9238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74138 payload_rows_remote=64834 dup_rows=24166 dup_rows_remote=21032 dedup_ratio=0.245829 router_tokens_per_expert=[785, 807, 805, 734, 743, 747, 785, 768, 799, 771, 773, 785, 747, 788, 740, 786, 820, 768, 777, 722, 789, 802, 737, 756, 753, 713, 736, 812, 755, 787, 813, 778, 736, 726, 745, 773, 774, 748, 772, 769, 751, 761, 769, 820, 766, 791, 745, 757, 729, 756, 763, 714, 809, 755, 751, 812, 746, 804, 770, 762, 792, 755, 798, 752] payload_rows_per_dst=[9304, 9358, 9336, 9254, 9084, 9272, 9214, 9316] dup_rows_per_dst=[3134, 3020, 3006, 3040, 2912, 3048, 2964, 3042] combine_rows_per_src=[9304, 9358, 9336, 9254, 9084, 9272, 9214, 9316] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=65076 dup_rows=24072 dup_rows_remote=21168 dedup_ratio=0.244873 router_tokens_per_expert=[798, 732, 752, 778, 808, 778, 760, 727, 742, 842, 782, 782, 745, 781, 830, 767, 788, 762, 760, 732, 730, 735, 781, 742, 779, 743, 778, 775, 815, 770, 796, 817, 731, 724, 773, 732, 788, 732, 759, 806, 726, 762, 767, 806, 808, 733, 746, 784, 788, 768, 747, 754, 754, 781, 726, 722, 748, 824, 763, 773, 807, 735, 824, 754] payload_rows_per_dst=[9302, 9388, 9156, 9354, 9222, 9296, 9180, 9334] dup_rows_per_dst=[2964, 3154, 2904, 3192, 2868, 2968, 2900, 3122] combine_rows_per_src=[9302, 9388, 9156, 9354, 9222, 9296, 9180, 9334] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74212 payload_rows_remote=64822 dup_rows=24092 dup_rows_remote=20986 dedup_ratio=0.245076 router_tokens_per_expert=[784, 755, 745, 746, 785, 803, 784, 746, 765, 767, 823, 794, 804, 719, 780, 757, 730, 809, 790, 779, 757, 794, 766, 805, 730, 741, 763, 833, 789, 745, 786, 738, 805, 732, 778, 810, 718, 776, 832, 797, 782, 727, 810, 781, 696, 785, 766, 755, 723, 719, 804, 722, 692, 752, 797, 781, 781, 759, 748, 764, 761, 746, 810, 731] payload_rows_per_dst=[9234, 9368, 9360, 9256, 9390, 9176, 9176, 9252] dup_rows_per_dst=[3062, 3050, 3100, 2994, 3106, 3028, 2804, 2948] combine_rows_per_src=[9234, 9368, 9360, 9256, 9390, 9176, 9176, 9252] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74546 payload_rows_remote=65178 dup_rows=23758 dup_rows_remote=20678 dedup_ratio=0.241679 router_tokens_per_expert=[746, 767, 737, 752, 725, 759, 766, 778, 716, 779, 745, 776, 733, 780, 801, 766, 771, 766, 796, 736, 753, 817, 789, 760, 777, 777, 774, 781, 760, 784, 800, 771, 788, 790, 728, 729, 781, 750, 790, 775, 787, 780, 793, 736, 733, 811, 762, 785, 781, 784, 756, 817, 814, 776, 812, 718, 744, 769, 751, 763, 753, 713, 775, 770] payload_rows_per_dst=[9186, 9280, 9400, 9368, 9296, 9402, 9420, 9194] dup_rows_per_dst=[2874, 2912, 2976, 3080, 2966, 2972, 3096, 2882] combine_rows_per_src=[9186, 9280, 9400, 9368, 9296, 9402, 9420, 9194] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74498 payload_rows_remote=65142 dup_rows=23806 dup_rows_remote=20694 dedup_ratio=0.242167 router_tokens_per_expert=[752, 778, 822, 781, 710, 793, 765, 812, 780, 771, 770, 793, 732, 771, 749, 711, 791, 795, 750, 761, 751, 789, 794, 765, 740, 735, 795, 723, 745, 718, 773, 732, 772, 791, 727, 761, 754, 798, 767, 756, 821, 770, 754, 792, 773, 753, 759, 812, 759, 821, 768, 771, 731, 760, 765, 804, 751, 771, 768, 756, 791, 769, 801, 759] payload_rows_per_dst=[9446, 9258, 9342, 9120, 9248, 9356, 9338, 9390] dup_rows_per_dst=[2980, 2896, 3050, 2802, 3004, 3112, 3020, 2942] combine_rows_per_src=[9446, 9258, 9342, 9120, 9248, 9356, 9338, 9390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74212 payload_rows_remote=64822 dup_rows=24092 dup_rows_remote=20986 dedup_ratio=0.245076 router_tokens_per_expert=[784, 755, 745, 746, 785, 803, 784, 746, 765, 767, 823, 794, 804, 719, 780, 757, 730, 809, 790, 779, 757, 794, 766, 805, 730, 741, 763, 833, 789, 745, 786, 738, 805, 732, 778, 810, 718, 776, 832, 797, 782, 727, 810, 781, 696, 785, 766, 755, 723, 719, 804, 722, 692, 752, 797, 781, 781, 759, 748, 764, 761, 746, 810, 731] payload_rows_per_dst=[9234, 9368, 9360, 9256, 9390, 9176, 9176, 9252] dup_rows_per_dst=[3062, 3050, 3100, 2994, 3106, 3028, 2804, 2948] combine_rows_per_src=[9234, 9368, 9360, 9256, 9390, 9176, 9176, 9252] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=64996 dup_rows=24008 dup_rows_remote=20856 dedup_ratio=0.244222 router_tokens_per_expert=[747, 779, 772, 753, 753, 752, 787, 720, 805, 764, 756, 791, 804, 731, 735, 744, 738, 795, 777, 786, 762, 826, 779, 754, 732, 807, 759, 737, 771, 800, 736, 758, 785, 746, 770, 752, 725, 778, 758, 773, 830, 723, 751, 744, 820, 771, 757, 738, 765, 739, 807, 780, 728, 826, 786, 795, 770, 745, 817, 743, 766, 797, 781, 776] payload_rows_per_dst=[9190, 9262, 9390, 9250, 9300, 9264, 9300, 9340] dup_rows_per_dst=[2936, 2998, 3044, 2950, 2874, 3004, 3152, 3050] combine_rows_per_src=[9190, 9262, 9390, 9250, 9300, 9264, 9300, 9340] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266551296 dispatch_meta_bytes=344976 combine_payload_bytes=266551296 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266896272 ingress_bytes=335757312 link_bytes=335757312 token_bytes=533447568 weight_movement_bytes=69206016 total_nvlink_bytes=602653584 weight_to_token_ratio=0.129733 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263427734e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265510912 dispatch_meta_bytes=343232 combine_payload_bytes=265510912 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265854144 ingress_bytes=334716928 link_bytes=334716928 token_bytes=531365056 weight_movement_bytes=69206016 total_nvlink_bytes=600571072 weight_to_token_ratio=0.130242 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259512853e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266518528 dispatch_meta_bytes=344440 combine_payload_bytes=266518528 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266862968 ingress_bytes=335724544 link_bytes=335724544 token_bytes=533381496 weight_movement_bytes=69206016 total_nvlink_bytes=602587512 weight_to_token_ratio=0.129750 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263304431e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74012 payload_rows_remote=64750 dup_rows=24292 dup_rows_remote=21216 dedup_ratio=0.247111 router_tokens_per_expert=[726, 784, 761, 814, 740, 756, 769, 808, 774, 766, 782, 781, 721, 764, 802, 803, 743, 805, 774, 729, 722, 794, 783, 729, 774, 782, 785, 786, 816, 772, 772, 773, 801, 733, 760, 768, 762, 799, 749, 791, 717, 770, 777, 808, 781, 749, 772, 730, 712, 777, 772, 705, 750, 768, 785, 792, 782, 786, 755, 784, 738, 759, 749, 781] payload_rows_per_dst=[9154, 9154, 9158, 9502, 9200, 9398, 9184, 9262] dup_rows_per_dst=[3010, 2832, 3000, 3146, 3126, 3210, 2892, 3076] combine_rows_per_src=[9154, 9154, 9158, 9502, 9200, 9398, 9184, 9262] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74216 payload_rows_remote=64898 dup_rows=24088 dup_rows_remote=20978 dedup_ratio=0.245036 router_tokens_per_expert=[744, 743, 764, 784, 759, 766, 802, 753, 815, 786, 756, 775, 749, 784, 801, 748, 766, 721, 798, 739, 760, 760, 810, 766, 792, 781, 727, 770, 753, 796, 819, 784, 757, 768, 790, 734, 779, 787, 777, 754, 717, 763, 745, 778, 736, 717, 772, 784, 792, 815, 759, 770, 781, 772, 778, 729, 766, 783, 810, 770, 731, 747, 771, 749] payload_rows_per_dst=[9242, 9318, 9208, 9456, 9274, 9146, 9322, 9250] dup_rows_per_dst=[2988, 3110, 3032, 2988, 3018, 2878, 3070, 3004] combine_rows_per_src=[9242, 9318, 9208, 9456, 9274, 9146, 9322, 9250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265560064 dispatch_meta_bytes=343464 combine_payload_bytes=265560064 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265903528 ingress_bytes=334766080 link_bytes=334766080 token_bytes=531463592 weight_movement_bytes=69206016 total_nvlink_bytes=600669608 weight_to_token_ratio=0.130218 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259697808e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266551296 dispatch_meta_bytes=344976 combine_payload_bytes=266551296 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266896272 ingress_bytes=335757312 link_bytes=335757312 token_bytes=533447568 weight_movement_bytes=69206016 total_nvlink_bytes=602653584 weight_to_token_ratio=0.129733 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263427734e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266969088 dispatch_meta_bytes=343424 combine_payload_bytes=266969088 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267312512 ingress_bytes=336175104 link_bytes=336175104 token_bytes=534281600 weight_movement_bytes=69206016 total_nvlink_bytes=603487616 weight_to_token_ratio=0.129531 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264999852e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74106 payload_rows_remote=64970 dup_rows=24198 dup_rows_remote=21234 dedup_ratio=0.246155 router_tokens_per_expert=[772, 776, 751, 748, 750, 744, 731, 778, 792, 782, 808, 788, 747, 778, 759, 780, 804, 780, 843, 735, 768, 769, 692, 785, 775, 775, 787, 782, 771, 769, 766, 738, 695, 755, 768, 777, 748, 700, 796, 766, 751, 800, 795, 784, 796, 816, 729, 717, 729, 823, 767, 774, 759, 777, 797, 745, 783, 769, 755, 801, 734, 772, 788, 763] payload_rows_per_dst=[9136, 9530, 9504, 9284, 9282, 9348, 9128, 8894] dup_rows_per_dst=[2964, 3108, 3358, 3042, 2852, 3028, 3044, 2802] combine_rows_per_src=[9136, 9530, 9504, 9284, 9282, 9348, 9128, 8894] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266821632 dispatch_meta_bytes=343344 combine_payload_bytes=266821632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267164976 ingress_bytes=301424640 link_bytes=301424640 token_bytes=533986608 weight_movement_bytes=34603008 total_nvlink_bytes=568589616 weight_to_token_ratio=0.064801 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134236654e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=65186 dup_rows=23948 dup_rows_remote=21116 dedup_ratio=0.243612 router_tokens_per_expert=[780, 816, 812, 765, 787, 796, 781, 773, 771, 774, 774, 748, 759, 830, 747, 744, 740, 760, 781, 772, 752, 791, 788, 745, 764, 792, 782, 762, 745, 691, 817, 680, 761, 809, 789, 794, 763, 766, 771, 749, 739, 776, 789, 729, 774, 750, 758, 782, 747, 756, 728, 778, 730, 789, 756, 717, 769, 802, 772, 781, 765, 780, 807, 757] payload_rows_per_dst=[9576, 9196, 9304, 9114, 9360, 9226, 9170, 9410] dup_rows_per_dst=[3044, 3098, 2954, 2952, 3044, 2968, 2832, 3056] combine_rows_per_src=[9576, 9196, 9304, 9114, 9360, 9226, 9170, 9410] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=343408 combine_payload_bytes=266223616 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266567024 ingress_bytes=370032640 link_bytes=370032640 token_bytes=532790640 weight_movement_bytes=103809024 total_nvlink_bytes=636599664 weight_to_token_ratio=0.194840 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392403034e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266821632 dispatch_meta_bytes=343344 combine_payload_bytes=266821632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267164976 ingress_bytes=301424640 link_bytes=301424640 token_bytes=533986608 weight_movement_bytes=34603008 total_nvlink_bytes=568589616 weight_to_token_ratio=0.064801 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134236654e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265510912 dispatch_meta_bytes=343232 combine_payload_bytes=265510912 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265854144 ingress_bytes=334716928 link_bytes=334716928 token_bytes=531365056 weight_movement_bytes=69206016 total_nvlink_bytes=600571072 weight_to_token_ratio=0.130242 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259512853e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265216000 dispatch_meta_bytes=343864 combine_payload_bytes=265216000 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265559864 ingress_bytes=369025024 link_bytes=369025024 token_bytes=530775864 weight_movement_bytes=103809024 total_nvlink_bytes=634584888 weight_to_token_ratio=0.195580 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388611456e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265822208 dispatch_meta_bytes=343504 combine_payload_bytes=265822208 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266165712 ingress_bytes=335028224 link_bytes=335028224 token_bytes=531987920 weight_movement_bytes=69206016 total_nvlink_bytes=601193936 weight_to_token_ratio=0.130089 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260684235e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266117120 dispatch_meta_bytes=344816 combine_payload_bytes=266117120 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266461936 ingress_bytes=335323136 link_bytes=335323136 token_bytes=532579056 weight_movement_bytes=69206016 total_nvlink_bytes=601785072 weight_to_token_ratio=0.129945 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261793965e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267001856 dispatch_meta_bytes=345208 combine_payload_bytes=267001856 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267347064 ingress_bytes=370810880 link_bytes=370810880 token_bytes=534348920 weight_movement_bytes=103809024 total_nvlink_bytes=638157944 weight_to_token_ratio=0.194272 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395331489e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74216 payload_rows_remote=64898 dup_rows=24088 dup_rows_remote=20978 dedup_ratio=0.245036 router_tokens_per_expert=[744, 743, 764, 784, 759, 766, 802, 753, 815, 786, 756, 775, 749, 784, 801, 748, 766, 721, 798, 739, 760, 760, 810, 766, 792, 781, 727, 770, 753, 796, 819, 784, 757, 768, 790, 734, 779, 787, 777, 754, 717, 763, 745, 778, 736, 717, 772, 784, 792, 815, 759, 770, 781, 772, 778, 729, 766, 783, 810, 770, 731, 747, 771, 749] payload_rows_per_dst=[9242, 9318, 9208, 9456, 9274, 9146, 9322, 9250] dup_rows_per_dst=[2988, 3110, 3032, 2988, 3018, 2878, 3070, 3004] combine_rows_per_src=[9242, 9318, 9208, 9456, 9274, 9146, 9322, 9250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74106 payload_rows_remote=64970 dup_rows=24198 dup_rows_remote=21234 dedup_ratio=0.246155 router_tokens_per_expert=[772, 776, 751, 748, 750, 744, 731, 778, 792, 782, 808, 788, 747, 778, 759, 780, 804, 780, 843, 735, 768, 769, 692, 785, 775, 775, 787, 782, 771, 769, 766, 738, 695, 755, 768, 777, 748, 700, 796, 766, 751, 800, 795, 784, 796, 816, 729, 717, 729, 823, 767, 774, 759, 777, 797, 745, 783, 769, 755, 801, 734, 772, 788, 763] payload_rows_per_dst=[9136, 9530, 9504, 9284, 9282, 9348, 9128, 8894] dup_rows_per_dst=[2964, 3108, 3358, 3042, 2852, 3028, 3044, 2802] combine_rows_per_src=[9136, 9530, 9504, 9284, 9282, 9348, 9128, 8894] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74608 payload_rows_remote=65496 dup_rows=23696 dup_rows_remote=20906 dedup_ratio=0.241048 router_tokens_per_expert=[746, 778, 748, 776, 774, 812, 805, 757, 775, 791, 719, 811, 757, 758, 790, 718, 782, 771, 716, 757, 714, 767, 700, 744, 778, 767, 726, 758, 776, 785, 788, 757, 806, 786, 784, 775, 755, 764, 743, 792, 768, 794, 782, 765, 772, 776, 721, 773, 825, 797, 765, 757, 800, 773, 817, 758, 810, 727, 776, 743, 787, 752, 752, 756] payload_rows_per_dst=[9314, 9308, 9112, 9372, 9388, 9338, 9466, 9310] dup_rows_per_dst=[3078, 2930, 2790, 2898, 3022, 2964, 3118, 2896] combine_rows_per_src=[9314, 9308, 9112, 9372, 9388, 9338, 9466, 9310] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=65110 dup_rows=24016 dup_rows_remote=21128 dedup_ratio=0.244303 router_tokens_per_expert=[767, 731, 753, 777, 750, 756, 749, 796, 748, 760, 777, 770, 803, 807, 782, 778, 737, 795, 786, 753, 776, 751, 758, 713, 783, 756, 773, 815, 746, 754, 786, 812, 729, 771, 733, 788, 758, 741, 772, 741, 745, 772, 772, 772, 766, 786, 742, 778, 770, 772, 779, 788, 763, 777, 747, 767, 780, 811, 774, 786, 766, 758, 762, 788] payload_rows_per_dst=[9350, 9338, 9200, 9376, 9178, 9216, 9274, 9356] dup_rows_per_dst=[2808, 3112, 2938, 3074, 2888, 3050, 3052, 3094] combine_rows_per_src=[9350, 9338, 9200, 9376, 9178, 9216, 9274, 9356] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74484 payload_rows_remote=65160 dup_rows=23820 dup_rows_remote=20966 dedup_ratio=0.242310 router_tokens_per_expert=[781, 804, 772, 718, 757, 723, 803, 749, 784, 757, 758, 758, 754, 743, 759, 745, 776, 748, 799, 769, 777, 742, 785, 781, 789, 749, 710, 788, 727, 753, 771, 802, 747, 782, 817, 809, 786, 757, 771, 741, 779, 755, 730, 748, 791, 732, 758, 740, 754, 826, 755, 773, 768, 797, 795, 729, 813, 814, 771, 782, 777, 833, 762, 729] payload_rows_per_dst=[9224, 9152, 9322, 9324, 9402, 9210, 9410, 9440] dup_rows_per_dst=[2990, 2964, 3032, 2854, 3018, 2856, 2984, 3122] combine_rows_per_src=[9224, 9152, 9322, 9324, 9402, 9210, 9410, 9440] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74352 payload_rows_remote=64870 dup_rows=23952 dup_rows_remote=20964 dedup_ratio=0.243652 router_tokens_per_expert=[790, 747, 762, 775, 753, 766, 784, 784, 784, 739, 802, 765, 794, 766, 771, 753, 776, 749, 825, 818, 768, 749, 753, 743, 772, 776, 777, 720, 732, 742, 774, 756, 785, 815, 760, 756, 756, 804, 760, 726, 786, 788, 788, 802, 752, 763, 781, 775, 783, 751, 719, 768, 764, 768, 802, 734, 717, 785, 827, 742, 735, 782, 778, 735] payload_rows_per_dst=[9306, 9286, 9318, 9166, 9254, 9482, 9264, 9276] dup_rows_per_dst=[3016, 3062, 3044, 2932, 3070, 2988, 2914, 2926] combine_rows_per_src=[9306, 9286, 9318, 9166, 9254, 9482, 9264, 9276] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=65024 dup_rows=24098 dup_rows_remote=21126 dedup_ratio=0.245138 router_tokens_per_expert=[715, 784, 726, 747, 757, 839, 779, 763, 767, 744, 752, 756, 788, 697, 765, 780, 786, 778, 791, 760, 734, 758, 816, 779, 792, 735, 795, 727, 780, 748, 772, 775, 801, 797, 762, 788, 804, 759, 759, 818, 742, 796, 750, 789, 828, 760, 761, 826, 789, 784, 755, 791, 716, 745, 755, 742, 771, 768, 741, 755, 756, 751, 756, 752] payload_rows_per_dst=[9178, 9224, 9314, 9228, 9440, 9432, 9182, 9208] dup_rows_per_dst=[3042, 2874, 3090, 3020, 3136, 3072, 2972, 2892] combine_rows_per_src=[9178, 9224, 9314, 9228, 9440, 9432, 9182, 9208] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=268271616 dispatch_meta_bytes=345608 combine_payload_bytes=268271616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268617224 ingress_bytes=302874624 link_bytes=302874624 token_bytes=536888840 weight_movement_bytes=34603008 total_nvlink_bytes=571491848 weight_to_token_ratio=0.064451 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.139692827e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266690560 dispatch_meta_bytes=344952 combine_payload_bytes=266690560 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267035512 ingress_bytes=301293568 link_bytes=301293568 token_bytes=533726072 weight_movement_bytes=34603008 total_nvlink_bytes=568329080 weight_to_token_ratio=0.064833 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133743440e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=65110 dup_rows=24016 dup_rows_remote=21128 dedup_ratio=0.244303 router_tokens_per_expert=[767, 731, 753, 777, 750, 756, 749, 796, 748, 760, 777, 770, 803, 807, 782, 778, 737, 795, 786, 753, 776, 751, 758, 713, 783, 756, 773, 815, 746, 754, 786, 812, 729, 771, 733, 788, 758, 741, 772, 741, 745, 772, 772, 772, 766, 786, 742, 778, 770, 772, 779, 788, 763, 777, 747, 767, 780, 811, 774, 786, 766, 758, 762, 788] payload_rows_per_dst=[9350, 9338, 9200, 9376, 9178, 9216, 9274, 9356] dup_rows_per_dst=[2808, 3112, 2938, 3074, 2888, 3050, 3052, 3094] combine_rows_per_src=[9350, 9338, 9200, 9376, 9178, 9216, 9274, 9356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74096 payload_rows_remote=64802 dup_rows=24208 dup_rows_remote=21208 dedup_ratio=0.246257 router_tokens_per_expert=[788, 783, 738, 797, 804, 790, 769, 754, 760, 793, 734, 781, 780, 744, 824, 726, 781, 772, 750, 777, 777, 779, 759, 772, 798, 782, 761, 738, 742, 774, 768, 780, 751, 769, 788, 745, 719, 805, 795, 775, 787, 725, 773, 785, 743, 815, 771, 775, 779, 735, 752, 762, 760, 768, 726, 729, 787, 751, 743, 771, 762, 753, 797, 781] payload_rows_per_dst=[9292, 9044, 9190, 9402, 9196, 9048, 9630, 9294] dup_rows_per_dst=[3154, 2824, 2872, 3204, 3098, 2678, 3378, 3000] combine_rows_per_src=[9292, 9044, 9190, 9402, 9196, 9048, 9630, 9294] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=65138 dup_rows=23962 dup_rows_remote=20994 dedup_ratio=0.243754 router_tokens_per_expert=[713, 751, 782, 805, 745, 710, 780, 773, 809, 762, 738, 791, 744, 718, 778, 751, 774, 773, 801, 759, 786, 786, 774, 738, 818, 741, 710, 767, 769, 712, 738, 750, 734, 814, 740, 763, 769, 790, 784, 843, 766, 796, 783, 778, 769, 769, 730, 763, 753, 825, 789, 772, 741, 783, 792, 762, 756, 745, 801, 818, 777, 796, 756, 749] payload_rows_per_dst=[9204, 9224, 9382, 9194, 9518, 9278, 9340, 9202] dup_rows_per_dst=[2968, 2904, 3000, 2816, 3154, 3030, 3094, 2996] combine_rows_per_src=[9204, 9224, 9382, 9194, 9518, 9278, 9340, 9202] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74238 payload_rows_remote=64818 dup_rows=24066 dup_rows_remote=21058 dedup_ratio=0.244812 router_tokens_per_expert=[783, 790, 798, 812, 770, 747, 778, 759, 734, 768, 772, 783, 796, 804, 759, 798, 757, 753, 758, 759, 733, 779, 836, 747, 785, 746, 760, 793, 790, 758, 759, 773, 764, 697, 761, 757, 791, 784, 791, 730, 737, 776, 766, 749, 797, 755, 745, 806, 745, 747, 762, 750, 770, 791, 767, 758, 769, 748, 753, 755, 734, 836, 775, 749] payload_rows_per_dst=[9382, 9420, 9284, 9258, 9176, 9266, 9222, 9230] dup_rows_per_dst=[3092, 3008, 2960, 3070, 2974, 2996, 2958, 3008] combine_rows_per_src=[9382, 9420, 9284, 9258, 9176, 9266, 9222, 9230] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265822208 dispatch_meta_bytes=343504 combine_payload_bytes=265822208 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266165712 ingress_bytes=335028224 link_bytes=335028224 token_bytes=531987920 weight_movement_bytes=69206016 total_nvlink_bytes=601193936 weight_to_token_ratio=0.130089 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260684235e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=268271616 dispatch_meta_bytes=345608 combine_payload_bytes=268271616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268617224 ingress_bytes=302874624 link_bytes=302874624 token_bytes=536888840 weight_movement_bytes=34603008 total_nvlink_bytes=571491848 weight_to_token_ratio=0.064451 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.139692827e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=65134 dup_rows=24008 dup_rows_remote=21044 dedup_ratio=0.244222 router_tokens_per_expert=[764, 738, 756, 826, 761, 738, 767, 758, 768, 748, 777, 800, 730, 795, 784, 757, 770, 787, 745, 784, 813, 749, 734, 784, 779, 799, 777, 809, 787, 764, 739, 757, 747, 766, 749, 751, 797, 769, 775, 786, 786, 744, 764, 768, 778, 783, 724, 763, 737, 763, 807, 736, 762, 770, 711, 777, 764, 798, 749, 787, 770, 800, 727, 800] payload_rows_per_dst=[9270, 9328, 9276, 9372, 9272, 9246, 9162, 9370] dup_rows_per_dst=[2946, 2990, 3056, 3050, 3008, 2974, 2964, 3020] combine_rows_per_src=[9270, 9328, 9276, 9372, 9272, 9246, 9162, 9370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266117120 dispatch_meta_bytes=344816 combine_payload_bytes=266117120 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266461936 ingress_bytes=335323136 link_bytes=335323136 token_bytes=532579056 weight_movement_bytes=69206016 total_nvlink_bytes=601785072 weight_to_token_ratio=0.129945 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261793965e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266895360 dispatch_meta_bytes=344504 combine_payload_bytes=266895360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267239864 ingress_bytes=301498368 link_bytes=301498368 token_bytes=534135224 weight_movement_bytes=34603008 total_nvlink_bytes=568738232 weight_to_token_ratio=0.064783 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134514086e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265707520 dispatch_meta_bytes=343336 combine_payload_bytes=265707520 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266050856 ingress_bytes=334913536 link_bytes=334913536 token_bytes=531758376 weight_movement_bytes=69206016 total_nvlink_bytes=600964392 weight_to_token_ratio=0.130146 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260252673e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=344600 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266682904 ingress_bytes=300941312 link_bytes=300941312 token_bytes=533021208 weight_movement_bytes=34603008 total_nvlink_bytes=567624216 weight_to_token_ratio=0.064919 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132417929e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266690560 dispatch_meta_bytes=344952 combine_payload_bytes=266690560 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267035512 ingress_bytes=301293568 link_bytes=301293568 token_bytes=533726072 weight_movement_bytes=34603008 total_nvlink_bytes=568329080 weight_to_token_ratio=0.064833 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133743440e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265428992 dispatch_meta_bytes=344040 combine_payload_bytes=265428992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265773032 ingress_bytes=334635008 link_bytes=334635008 token_bytes=531202024 weight_movement_bytes=69206016 total_nvlink_bytes=600408040 weight_to_token_ratio=0.130282 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259204595e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266805248 dispatch_meta_bytes=344528 combine_payload_bytes=266805248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267149776 ingress_bytes=301408256 link_bytes=301408256 token_bytes=533955024 weight_movement_bytes=34603008 total_nvlink_bytes=568558032 weight_to_token_ratio=0.064805 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134175002e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265494528 dispatch_meta_bytes=343504 combine_payload_bytes=265494528 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=265838032 ingress_bytes=403906560 link_bytes=403906560 token_bytes=531332560 weight_movement_bytes=138412032 total_nvlink_bytes=669744592 weight_to_token_ratio=0.260500 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519867868e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=344712 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267133576 ingress_bytes=301391872 link_bytes=301391872 token_bytes=533922440 weight_movement_bytes=34603008 total_nvlink_bytes=568525448 weight_to_token_ratio=0.064809 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134113350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74238 payload_rows_remote=64818 dup_rows=24066 dup_rows_remote=21058 dedup_ratio=0.244812 router_tokens_per_expert=[783, 790, 798, 812, 770, 747, 778, 759, 734, 768, 772, 783, 796, 804, 759, 798, 757, 753, 758, 759, 733, 779, 836, 747, 785, 746, 760, 793, 790, 758, 759, 773, 764, 697, 761, 757, 791, 784, 791, 730, 737, 776, 766, 749, 797, 755, 745, 806, 745, 747, 762, 750, 770, 791, 767, 758, 769, 748, 753, 755, 734, 836, 775, 749] payload_rows_per_dst=[9382, 9420, 9284, 9258, 9176, 9266, 9222, 9230] dup_rows_per_dst=[3092, 3008, 2960, 3070, 2974, 2996, 2958, 3008] combine_rows_per_src=[9382, 9420, 9284, 9258, 9176, 9266, 9222, 9230] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65150 dup_rows=23944 dup_rows_remote=21030 dedup_ratio=0.243571 router_tokens_per_expert=[774, 712, 722, 753, 859, 784, 777, 764, 788, 751, 770, 714, 782, 770, 746, 771, 766, 768, 735, 753, 713, 763, 789, 775, 750, 768, 776, 788, 789, 740, 758, 782, 767, 763, 739, 782, 805, 776, 775, 767, 780, 748, 737, 753, 770, 755, 813, 825, 765, 745, 732, 780, 727, 760, 793, 775, 801, 746, 796, 771, 827, 814, 743, 772] payload_rows_per_dst=[9318, 9214, 9210, 9294, 9350, 9406, 9214, 9354] dup_rows_per_dst=[2972, 2970, 2914, 3008, 2998, 2956, 2940, 3186] combine_rows_per_src=[9318, 9214, 9210, 9294, 9350, 9406, 9214, 9354] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=65138 dup_rows=23962 dup_rows_remote=20994 dedup_ratio=0.243754 router_tokens_per_expert=[713, 751, 782, 805, 745, 710, 780, 773, 809, 762, 738, 791, 744, 718, 778, 751, 774, 773, 801, 759, 786, 786, 774, 738, 818, 741, 710, 767, 769, 712, 738, 750, 734, 814, 740, 763, 769, 790, 784, 843, 766, 796, 783, 778, 769, 769, 730, 763, 753, 825, 789, 772, 741, 783, 792, 762, 756, 745, 801, 818, 777, 796, 756, 749] payload_rows_per_dst=[9204, 9224, 9382, 9194, 9518, 9278, 9340, 9202] dup_rows_per_dst=[2968, 2904, 3000, 2816, 3154, 3030, 3094, 2996] combine_rows_per_src=[9204, 9224, 9382, 9194, 9518, 9278, 9340, 9202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74138 payload_rows_remote=64878 dup_rows=24166 dup_rows_remote=21138 dedup_ratio=0.245829 router_tokens_per_expert=[773, 763, 765, 688, 778, 760, 742, 794, 733, 782, 811, 775, 800, 767, 767, 840, 791, 791, 775, 791, 789, 742, 811, 799, 805, 755, 758, 775, 757, 770, 742, 782, 775, 781, 787, 764, 769, 759, 740, 723, 773, 814, 721, 787, 779, 731, 736, 778, 806, 743, 754, 747, 766, 732, 777, 765, 700, 823, 751, 795, 764, 774, 756, 711] payload_rows_per_dst=[9178, 9494, 9424, 9260, 9170, 9196, 9196, 9220] dup_rows_per_dst=[2948, 3056, 3154, 3028, 3026, 3042, 2984, 2928] combine_rows_per_src=[9178, 9494, 9424, 9260, 9170, 9196, 9196, 9220] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=65084 dup_rows=24056 dup_rows_remote=21126 dedup_ratio=0.244710 router_tokens_per_expert=[814, 741, 761, 747, 769, 766, 713, 746, 750, 788, 772, 785, 777, 746, 782, 796, 766, 801, 774, 763, 788, 739, 737, 792, 769, 752, 785, 852, 774, 762, 746, 836, 732, 808, 757, 765, 785, 769, 710, 760, 738, 798, 741, 754, 810, 784, 725, 697, 798, 759, 745, 776, 734, 721, 756, 805, 802, 753, 750, 771, 782, 802, 770, 806] payload_rows_per_dst=[9204, 9310, 9334, 9348, 9308, 9164, 9182, 9398] dup_rows_per_dst=[2910, 3082, 2986, 3204, 2864, 2930, 3006, 3074] combine_rows_per_src=[9204, 9310, 9334, 9348, 9308, 9164, 9182, 9398] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74540 payload_rows_remote=65420 dup_rows=23764 dup_rows_remote=20844 dedup_ratio=0.241740 router_tokens_per_expert=[740, 730, 777, 764, 804, 770, 775, 769, 778, 795, 797, 747, 797, 770, 768, 788, 750, 762, 788, 797, 774, 755, 718, 766, 763, 758, 798, 749, 798, 759, 750, 750, 780, 730, 691, 770, 795, 734, 731, 789, 807, 792, 782, 758, 801, 784, 807, 828, 836, 750, 708, 715, 819, 728, 724, 742, 798, 728, 816, 742, 773, 743, 750, 797] payload_rows_per_dst=[9314, 9404, 9316, 9268, 9120, 9580, 9164, 9374] dup_rows_per_dst=[2944, 3076, 2904, 2982, 2920, 3138, 2880, 2920] combine_rows_per_src=[9314, 9404, 9316, 9268, 9120, 9580, 9164, 9374] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=65084 dup_rows=24056 dup_rows_remote=21126 dedup_ratio=0.244710 router_tokens_per_expert=[814, 741, 761, 747, 769, 766, 713, 746, 750, 788, 772, 785, 777, 746, 782, 796, 766, 801, 774, 763, 788, 739, 737, 792, 769, 752, 785, 852, 774, 762, 746, 836, 732, 808, 757, 765, 785, 769, 710, 760, 738, 798, 741, 754, 810, 784, 725, 697, 798, 759, 745, 776, 734, 721, 756, 805, 802, 753, 750, 771, 782, 802, 770, 806] payload_rows_per_dst=[9204, 9310, 9334, 9348, 9308, 9164, 9182, 9398] dup_rows_per_dst=[2910, 3082, 2986, 3204, 2864, 2930, 3006, 3074] combine_rows_per_src=[9204, 9310, 9334, 9348, 9308, 9164, 9182, 9398] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74138 payload_rows_remote=64878 dup_rows=24166 dup_rows_remote=21138 dedup_ratio=0.245829 router_tokens_per_expert=[773, 763, 765, 688, 778, 760, 742, 794, 733, 782, 811, 775, 800, 767, 767, 840, 791, 791, 775, 791, 789, 742, 811, 799, 805, 755, 758, 775, 757, 770, 742, 782, 775, 781, 787, 764, 769, 759, 740, 723, 773, 814, 721, 787, 779, 731, 736, 778, 806, 743, 754, 747, 766, 732, 777, 765, 700, 823, 751, 795, 764, 774, 756, 711] payload_rows_per_dst=[9178, 9494, 9424, 9260, 9170, 9196, 9196, 9220] dup_rows_per_dst=[2948, 3056, 3154, 3028, 3026, 3042, 2984, 2928] combine_rows_per_src=[9178, 9494, 9424, 9260, 9170, 9196, 9196, 9220] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=267960320 dispatch_meta_bytes=345056 combine_payload_bytes=267960320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268305376 ingress_bytes=302563328 link_bytes=302563328 token_bytes=536265696 weight_movement_bytes=34603008 total_nvlink_bytes=570868704 weight_to_token_ratio=0.064526 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138521445e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266854400 dispatch_meta_bytes=344720 combine_payload_bytes=266854400 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267199120 ingress_bytes=336060416 link_bytes=336060416 token_bytes=534053520 weight_movement_bytes=69206016 total_nvlink_bytes=603259536 weight_to_token_ratio=0.129586 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264568290e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=65186 dup_rows=23948 dup_rows_remote=21116 dedup_ratio=0.243612 router_tokens_per_expert=[780, 816, 812, 765, 787, 796, 781, 773, 771, 774, 774, 748, 759, 830, 747, 744, 740, 760, 781, 772, 752, 791, 788, 745, 764, 792, 782, 762, 745, 691, 817, 680, 761, 809, 789, 794, 763, 766, 771, 749, 739, 776, 789, 729, 774, 750, 758, 782, 747, 756, 728, 778, 730, 789, 756, 717, 769, 802, 772, 781, 765, 780, 807, 757] payload_rows_per_dst=[9576, 9196, 9304, 9114, 9360, 9226, 9170, 9410] dup_rows_per_dst=[3044, 3098, 2954, 2952, 3044, 2968, 2832, 3056] combine_rows_per_src=[9576, 9196, 9304, 9114, 9360, 9226, 9170, 9410] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265494528 dispatch_meta_bytes=343504 combine_payload_bytes=265494528 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=265838032 ingress_bytes=403906560 link_bytes=403906560 token_bytes=531332560 weight_movement_bytes=138412032 total_nvlink_bytes=669744592 weight_to_token_ratio=0.260500 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519867868e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74596 payload_rows_remote=65108 dup_rows=23708 dup_rows_remote=20658 dedup_ratio=0.241170 router_tokens_per_expert=[786, 845, 769, 811, 769, 792, 750, 740, 786, 745, 810, 731, 797, 771, 807, 831, 715, 787, 738, 751, 790, 770, 776, 819, 729, 781, 763, 751, 822, 727, 784, 734, 730, 782, 746, 742, 788, 791, 770, 747, 773, 804, 758, 750, 732, 713, 800, 725, 729, 805, 802, 726, 756, 767, 822, 764, 792, 747, 795, 782, 733, 744, 737, 723] payload_rows_per_dst=[9448, 9204, 9362, 8900, 9646, 9200, 9348, 9488] dup_rows_per_dst=[3076, 2920, 2906, 2640, 3188, 2934, 2994, 3050] combine_rows_per_src=[9448, 9204, 9362, 8900, 9646, 9200, 9348, 9488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74328 payload_rows_remote=65020 dup_rows=23976 dup_rows_remote=21008 dedup_ratio=0.243896 router_tokens_per_expert=[810, 763, 764, 758, 781, 783, 791, 710, 769, 758, 751, 788, 806, 772, 773, 791, 764, 759, 760, 766, 780, 761, 733, 757, 748, 784, 742, 762, 785, 783, 756, 789, 708, 800, 799, 769, 766, 741, 773, 791, 791, 765, 784, 784, 751, 766, 770, 749, 717, 799, 750, 758, 803, 801, 758, 752, 785, 759, 743, 765, 793, 731, 764, 770] payload_rows_per_dst=[9276, 9434, 9184, 9350, 9292, 9278, 9308, 9206] dup_rows_per_dst=[3044, 2982, 2976, 2948, 3002, 3042, 2968, 3014] combine_rows_per_src=[9276, 9434, 9184, 9350, 9292, 9278, 9308, 9206] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=64790 dup_rows=24136 dup_rows_remote=21134 dedup_ratio=0.245524 router_tokens_per_expert=[775, 761, 781, 782, 773, 758, 775, 745, 782, 818, 768, 770, 744, 775, 754, 799, 783, 747, 773, 772, 807, 778, 755, 758, 803, 782, 787, 752, 761, 748, 777, 763, 803, 719, 792, 801, 732, 781, 702, 769, 779, 755, 793, 763, 762, 801, 757, 758, 773, 809, 761, 750, 787, 740, 760, 751, 836, 739, 728, 723, 766, 743, 724, 789] payload_rows_per_dst=[9378, 9434, 9242, 9360, 9234, 9186, 9156, 9178] dup_rows_per_dst=[3002, 3144, 3112, 2986, 2964, 2992, 3018, 2918] combine_rows_per_src=[9378, 9434, 9242, 9360, 9234, 9186, 9156, 9178] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266584064 dispatch_meta_bytes=344840 combine_payload_bytes=266584064 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266928904 ingress_bytes=335790080 link_bytes=335790080 token_bytes=533512968 weight_movement_bytes=69206016 total_nvlink_bytes=602718984 weight_to_token_ratio=0.129718 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263551038e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65058 dup_rows=23944 dup_rows_remote=21062 dedup_ratio=0.243571 router_tokens_per_expert=[744, 763, 769, 788, 776, 787, 729, 783, 734, 777, 746, 782, 741, 743, 783, 786, 763, 744, 799, 787, 755, 718, 760, 778, 745, 793, 729, 834, 722, 796, 800, 791, 803, 756, 765, 760, 753, 756, 719, 781, 809, 814, 710, 783, 786, 764, 729, 810, 744, 747, 786, 781, 796, 757, 766, 735, 742, 770, 747, 779, 815, 761, 769, 814] payload_rows_per_dst=[9182, 9302, 9234, 9386, 9218, 9338, 9300, 9400] dup_rows_per_dst=[3096, 2882, 2974, 3034, 2968, 3072, 2924, 2994] combine_rows_per_src=[9182, 9302, 9234, 9386, 9218, 9338, 9300, 9400] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266854400 dispatch_meta_bytes=344720 combine_payload_bytes=266854400 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267199120 ingress_bytes=336060416 link_bytes=336060416 token_bytes=534053520 weight_movement_bytes=69206016 total_nvlink_bytes=603259536 weight_to_token_ratio=0.129586 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264568290e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266805248 dispatch_meta_bytes=344528 combine_payload_bytes=266805248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267149776 ingress_bytes=301408256 link_bytes=301408256 token_bytes=533955024 weight_movement_bytes=34603008 total_nvlink_bytes=568558032 weight_to_token_ratio=0.064805 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134175002e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265740288 dispatch_meta_bytes=344064 combine_payload_bytes=265740288 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266084352 ingress_bytes=300343296 link_bytes=300343296 token_bytes=531824640 weight_movement_bytes=34603008 total_nvlink_bytes=566427648 weight_to_token_ratio=0.065065 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130167643e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267001856 dispatch_meta_bytes=345208 combine_payload_bytes=267001856 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267347064 ingress_bytes=370810880 link_bytes=370810880 token_bytes=534348920 weight_movement_bytes=103809024 total_nvlink_bytes=638157944 weight_to_token_ratio=0.194272 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395331489e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266584064 dispatch_meta_bytes=344840 combine_payload_bytes=266584064 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266928904 ingress_bytes=335790080 link_bytes=335790080 token_bytes=533512968 weight_movement_bytes=69206016 total_nvlink_bytes=602718984 weight_to_token_ratio=0.129718 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263551038e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=267960320 dispatch_meta_bytes=345056 combine_payload_bytes=267960320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268305376 ingress_bytes=302563328 link_bytes=302563328 token_bytes=536265696 weight_movement_bytes=34603008 total_nvlink_bytes=570868704 weight_to_token_ratio=0.064526 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138521445e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266682368 dispatch_meta_bytes=343064 combine_payload_bytes=266682368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267025432 ingress_bytes=301285376 link_bytes=301285376 token_bytes=533707800 weight_movement_bytes=34603008 total_nvlink_bytes=568310808 weight_to_token_ratio=0.064835 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133712614e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265379840 dispatch_meta_bytes=343696 combine_payload_bytes=265379840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265723536 ingress_bytes=299982848 link_bytes=299982848 token_bytes=531103376 weight_movement_bytes=34603008 total_nvlink_bytes=565706384 weight_to_token_ratio=0.065153 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128811306e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266321920 dispatch_meta_bytes=344112 combine_payload_bytes=266321920 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266666032 ingress_bytes=335527936 link_bytes=335527936 token_bytes=532987952 weight_movement_bytes=69206016 total_nvlink_bytes=602193968 weight_to_token_ratio=0.129845 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262564611e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12258170824 prefetch_bytes=726663168 reduce_bytes=519045120 weight_movement_bytes=1245708288 egress_bytes=6133045192 ingress_bytes=7370833920 link_bytes=7370833920 total_nvlink_bytes=13503879112 weight_to_token_ratio=0.101623 prefetch_share_of_ingress=0.098586 dedup_ratio=0.244368 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=15 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.205906723e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65058 dup_rows=23944 dup_rows_remote=21062 dedup_ratio=0.243571 router_tokens_per_expert=[744, 763, 769, 788, 776, 787, 729, 783, 734, 777, 746, 782, 741, 743, 783, 786, 763, 744, 799, 787, 755, 718, 760, 778, 745, 793, 729, 834, 722, 796, 800, 791, 803, 756, 765, 760, 753, 756, 719, 781, 809, 814, 710, 783, 786, 764, 729, 810, 744, 747, 786, 781, 796, 757, 766, 735, 742, 770, 747, 779, 815, 761, 769, 814] payload_rows_per_dst=[9182, 9302, 9234, 9386, 9218, 9338, 9300, 9400] dup_rows_per_dst=[3096, 2882, 2974, 3034, 2968, 3072, 2924, 2994] combine_rows_per_src=[9182, 9302, 9234, 9386, 9218, 9338, 9300, 9400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12262493752 prefetch_bytes=657457152 reduce_bytes=622854144 weight_movement_bytes=1280311296 egress_bytes=6135205432 ingress_bytes=7407599616 link_bytes=7407599616 total_nvlink_bytes=13542805048 weight_to_token_ratio=0.104409 prefetch_share_of_ingress=0.088754 dedup_ratio=0.243987 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=18 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028913 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.211921782e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12215802008 prefetch_bytes=692060160 reduce_bytes=622854144 weight_movement_bytes=1314914304 egress_bytes=6111844504 ingress_bytes=7418871808 link_bytes=7418871808 total_nvlink_bytes=13530716312 weight_to_token_ratio=0.107640 prefetch_share_of_ingress=0.093284 dedup_ratio=0.244988 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=18 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029348 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.213765972e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=65054 dup_rows=24048 dup_rows_remote=21086 dedup_ratio=0.244629 router_tokens_per_expert=[794, 809, 790, 751, 729, 775, 753, 788, 801, 750, 761, 773, 780, 729, 756, 721, 771, 768, 718, 745, 795, 785, 746, 754, 820, 799, 810, 783, 787, 739, 761, 762, 764, 727, 757, 728, 756, 765, 775, 739, 842, 761, 819, 783, 775, 785, 766, 713, 769, 771, 789, 783, 771, 771, 799, 783, 769, 746, 754, 737, 793, 754, 739, 766] payload_rows_per_dst=[9360, 9228, 9202, 9378, 9140, 9458, 9358, 9132] dup_rows_per_dst=[3018, 2914, 2962, 3144, 2882, 3030, 3114, 2984] combine_rows_per_src=[9360, 9228, 9202, 9378, 9140, 9458, 9358, 9132] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=64790 dup_rows=24136 dup_rows_remote=21134 dedup_ratio=0.245524 router_tokens_per_expert=[775, 761, 781, 782, 773, 758, 775, 745, 782, 818, 768, 770, 744, 775, 754, 799, 783, 747, 773, 772, 807, 778, 755, 758, 803, 782, 787, 752, 761, 748, 777, 763, 803, 719, 792, 801, 732, 781, 702, 769, 779, 755, 793, 763, 762, 801, 757, 758, 773, 809, 761, 750, 787, 740, 760, 751, 836, 739, 728, 723, 766, 743, 724, 789] payload_rows_per_dst=[9378, 9434, 9242, 9360, 9234, 9186, 9156, 9178] dup_rows_per_dst=[3002, 3144, 3112, 2986, 2964, 2992, 3018, 2918] combine_rows_per_src=[9378, 9434, 9242, 9360, 9234, 9186, 9156, 9178] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74080 payload_rows_remote=64810 dup_rows=24224 dup_rows_remote=21298 dedup_ratio=0.246419 router_tokens_per_expert=[732, 729, 722, 746, 745, 799, 757, 761, 770, 821, 766, 766, 808, 748, 760, 793, 757, 757, 766, 781, 804, 766, 782, 738, 783, 777, 831, 784, 760, 749, 769, 757, 748, 784, 766, 755, 745, 743, 748, 809, 801, 749, 796, 776, 780, 777, 790, 740, 738, 789, 770, 774, 756, 789, 738, 783, 754, 746, 805, 741, 774, 728, 800, 776] payload_rows_per_dst=[9116, 9298, 9274, 9286, 9270, 9300, 9242, 9294] dup_rows_per_dst=[2866, 3166, 3028, 3134, 2926, 3118, 3032, 2954] combine_rows_per_src=[9116, 9298, 9274, 9286, 9270, 9300, 9242, 9294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74382 payload_rows_remote=65150 dup_rows=23922 dup_rows_remote=20948 dedup_ratio=0.243347 router_tokens_per_expert=[779, 730, 788, 752, 772, 729, 810, 804, 735, 782, 715, 762, 717, 744, 793, 754, 824, 809, 711, 798, 833, 771, 740, 778, 806, 722, 771, 773, 729, 760, 752, 790, 783, 790, 790, 759, 816, 760, 757, 746, 763, 750, 773, 752, 753, 754, 796, 790, 742, 753, 772, 758, 818, 775, 797, 762, 780, 755, 783, 790, 738, 727, 775, 762] payload_rows_per_dst=[9332, 9210, 9424, 9232, 9316, 9278, 9340, 9250] dup_rows_per_dst=[2996, 2794, 3104, 2974, 3086, 2984, 3014, 2970] combine_rows_per_src=[9332, 9210, 9424, 9232, 9316, 9278, 9340, 9250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=344480 combine_payload_bytes=266477568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266822048 ingress_bytes=335683584 link_bytes=335683584 token_bytes=533299616 weight_movement_bytes=69206016 total_nvlink_bytes=602505632 weight_to_token_ratio=0.129769 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263150302e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=65054 dup_rows=24048 dup_rows_remote=21086 dedup_ratio=0.244629 router_tokens_per_expert=[794, 809, 790, 751, 729, 775, 753, 788, 801, 750, 761, 773, 780, 729, 756, 721, 771, 768, 718, 745, 795, 785, 746, 754, 820, 799, 810, 783, 787, 739, 761, 762, 764, 727, 757, 728, 756, 765, 775, 739, 842, 761, 819, 783, 775, 785, 766, 713, 769, 771, 789, 783, 771, 771, 799, 783, 769, 746, 754, 737, 793, 754, 739, 766] payload_rows_per_dst=[9360, 9228, 9202, 9378, 9140, 9458, 9358, 9132] dup_rows_per_dst=[3018, 2914, 2962, 3144, 2882, 3030, 3114, 2984] combine_rows_per_src=[9360, 9228, 9202, 9378, 9140, 9458, 9358, 9132] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12215802008 prefetch_bytes=692060160 reduce_bytes=622854144 weight_movement_bytes=1314914304 egress_bytes=6111844504 ingress_bytes=7418871808 link_bytes=7418871808 total_nvlink_bytes=13530716312 weight_to_token_ratio=0.107640 prefetch_share_of_ingress=0.093284 dedup_ratio=0.244988 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=18 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029348 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.213765972e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12246061576 prefetch_bytes=622854144 reduce_bytes=761266176 weight_movement_bytes=1384120320 egress_bytes=6126981640 ingress_bytes=7503200256 link_bytes=7503200256 total_nvlink_bytes=13630181896 weight_to_token_ratio=0.113026 prefetch_share_of_ingress=0.083012 dedup_ratio=0.244239 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=22 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028043 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.227562544e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=344560 combine_payload_bytes=266461184 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266805744 ingress_bytes=335667200 link_bytes=335667200 token_bytes=533266928 weight_movement_bytes=69206016 total_nvlink_bytes=602472944 weight_to_token_ratio=0.129777 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263088650e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265379840 dispatch_meta_bytes=343696 combine_payload_bytes=265379840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265723536 ingress_bytes=299982848 link_bytes=299982848 token_bytes=531103376 weight_movement_bytes=34603008 total_nvlink_bytes=565706384 weight_to_token_ratio=0.065153 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128811306e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265461760 dispatch_meta_bytes=344432 combine_payload_bytes=265461760 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265806192 ingress_bytes=300064768 link_bytes=300064768 token_bytes=531267952 weight_movement_bytes=34603008 total_nvlink_bytes=565870960 weight_to_token_ratio=0.065133 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129119565e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12256558176 prefetch_bytes=588251136 reduce_bytes=1245708288 weight_movement_bytes=1833959424 egress_bytes=6132235360 ingress_bytes=7958282240 link_bytes=7958282240 total_nvlink_bytes=14090517600 weight_to_token_ratio=0.149631 prefetch_share_of_ingress=0.073917 dedup_ratio=0.244287 prefetch_slots=34 unused_prefetch_slots=334 slot_utilization=0.092391 reduce_slots=36 remote_demand=17 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=28448 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.302016321e-04 prefetch_bytes_per_gemm_flop=9.624094203e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=344560 combine_payload_bytes=266461184 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266805744 ingress_bytes=335667200 link_bytes=335667200 token_bytes=533266928 weight_movement_bytes=69206016 total_nvlink_bytes=602472944 weight_to_token_ratio=0.129777 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263088650e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74382 payload_rows_remote=65150 dup_rows=23922 dup_rows_remote=20948 dedup_ratio=0.243347 router_tokens_per_expert=[779, 730, 788, 752, 772, 729, 810, 804, 735, 782, 715, 762, 717, 744, 793, 754, 824, 809, 711, 798, 833, 771, 740, 778, 806, 722, 771, 773, 729, 760, 752, 790, 783, 790, 790, 759, 816, 760, 757, 746, 763, 750, 773, 752, 753, 754, 796, 790, 742, 753, 772, 758, 818, 775, 797, 762, 780, 755, 783, 790, 738, 727, 775, 762] payload_rows_per_dst=[9332, 9210, 9424, 9232, 9316, 9278, 9340, 9250] dup_rows_per_dst=[2996, 2794, 3104, 2974, 3086, 2984, 3014, 2970] combine_rows_per_src=[9332, 9210, 9424, 9232, 9316, 9278, 9340, 9250] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=65134 dup_rows=24008 dup_rows_remote=21044 dedup_ratio=0.244222 router_tokens_per_expert=[764, 738, 756, 826, 761, 738, 767, 758, 768, 748, 777, 800, 730, 795, 784, 757, 770, 787, 745, 784, 813, 749, 734, 784, 779, 799, 777, 809, 787, 764, 739, 757, 747, 766, 749, 751, 797, 769, 775, 786, 786, 744, 764, 768, 778, 783, 724, 763, 737, 763, 807, 736, 762, 770, 711, 777, 764, 798, 749, 787, 770, 800, 727, 800] payload_rows_per_dst=[9270, 9328, 9276, 9372, 9272, 9246, 9162, 9370] dup_rows_per_dst=[2946, 2990, 3056, 3050, 3008, 2974, 2964, 3020] combine_rows_per_src=[9270, 9328, 9276, 9372, 9272, 9246, 9162, 9370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12254608120 prefetch_bytes=657457152 reduce_bytes=761266176 weight_movement_bytes=1418723328 egress_bytes=6131260152 ingress_bytes=7542071296 link_bytes=7542071296 total_nvlink_bytes=13673331448 weight_to_token_ratio=0.115771 prefetch_share_of_ingress=0.087172 dedup_ratio=0.244249 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=22 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028370 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.233922048e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12262674320 prefetch_bytes=761266176 reduce_bytes=484442112 weight_movement_bytes=1245708288 egress_bytes=6135295888 ingress_bytes=7373086720 link_bytes=7373086720 total_nvlink_bytes=13508382608 weight_to_token_ratio=0.101585 prefetch_share_of_ingress=0.103249 dedup_ratio=0.244102 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=14 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.206275293e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266854400 dispatch_meta_bytes=344392 combine_payload_bytes=266854400 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267198792 ingress_bytes=301457408 link_bytes=301457408 token_bytes=534053192 weight_movement_bytes=34603008 total_nvlink_bytes=568656200 weight_to_token_ratio=0.064793 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134359957e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=344712 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267133576 ingress_bytes=301391872 link_bytes=301391872 token_bytes=533922440 weight_movement_bytes=34603008 total_nvlink_bytes=568525448 weight_to_token_ratio=0.064809 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134113350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12262674320 prefetch_bytes=761266176 reduce_bytes=484442112 weight_movement_bytes=1245708288 egress_bytes=6135295888 ingress_bytes=7373086720 link_bytes=7373086720 total_nvlink_bytes=13508382608 weight_to_token_ratio=0.101585 prefetch_share_of_ingress=0.103249 dedup_ratio=0.244102 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=14 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.206275293e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12246061576 prefetch_bytes=622854144 reduce_bytes=761266176 weight_movement_bytes=1384120320 egress_bytes=6126981640 ingress_bytes=7503200256 link_bytes=7503200256 total_nvlink_bytes=13630181896 weight_to_token_ratio=0.113026 prefetch_share_of_ingress=0.083012 dedup_ratio=0.244239 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=22 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028043 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.227562544e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12254608120 prefetch_bytes=657457152 reduce_bytes=761266176 weight_movement_bytes=1418723328 egress_bytes=6131260152 ingress_bytes=7542071296 link_bytes=7542071296 total_nvlink_bytes=13673331448 weight_to_token_ratio=0.115771 prefetch_share_of_ingress=0.087172 dedup_ratio=0.244249 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=22 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028370 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.233922048e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12263505976 prefetch_bytes=761266176 reduce_bytes=449839104 weight_movement_bytes=1211105280 egress_bytes=6135709752 ingress_bytes=7338901504 link_bytes=7338901504 total_nvlink_bytes=13474611256 weight_to_token_ratio=0.098757 prefetch_share_of_ingress=0.103730 dedup_ratio=0.243743 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=13 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029565 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.200682414e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74328 payload_rows_remote=65020 dup_rows=23976 dup_rows_remote=21008 dedup_ratio=0.243896 router_tokens_per_expert=[810, 763, 764, 758, 781, 783, 791, 710, 769, 758, 751, 788, 806, 772, 773, 791, 764, 759, 760, 766, 780, 761, 733, 757, 748, 784, 742, 762, 785, 783, 756, 789, 708, 800, 799, 769, 766, 741, 773, 791, 791, 765, 784, 784, 751, 766, 770, 749, 717, 799, 750, 758, 803, 801, 758, 752, 785, 759, 743, 765, 793, 731, 764, 770] payload_rows_per_dst=[9276, 9434, 9184, 9350, 9292, 9278, 9308, 9206] dup_rows_per_dst=[3044, 2982, 2976, 2948, 3002, 3042, 2968, 3014] combine_rows_per_src=[9276, 9434, 9184, 9350, 9292, 9278, 9308, 9206] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266321920 dispatch_meta_bytes=344112 combine_payload_bytes=266321920 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266666032 ingress_bytes=335527936 link_bytes=335527936 token_bytes=532987952 weight_movement_bytes=69206016 total_nvlink_bytes=602193968 weight_to_token_ratio=0.129845 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262564611e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12262493752 prefetch_bytes=657457152 reduce_bytes=622854144 weight_movement_bytes=1280311296 egress_bytes=6135205432 ingress_bytes=7407599616 link_bytes=7407599616 total_nvlink_bytes=13542805048 weight_to_token_ratio=0.104409 prefetch_share_of_ingress=0.088754 dedup_ratio=0.243987 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=18 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028913 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.211921782e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none - [2026-08-26 15:10:25] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 793.2 | throughput per GPU (TFLOP/s/GPU): 160.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.619553E+00 | loss scale: 1.0 | grad norm: 2.956 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65196 dup_rows=23908 dup_rows_remote=20818 dedup_ratio=0.243205 router_tokens_per_expert=[785, 785, 789, 772, 763, 798, 769, 761, 766, 819, 750, 810, 779, 787, 746, 761, 787, 756, 730, 724, 751, 795, 749, 767, 781, 762, 751, 728, 719, 760, 832, 778, 784, 786, 778, 729, 778, 762, 777, 751, 760, 779, 814, 747, 786, 765, 771, 685, 759, 744, 818, 742, 753, 767, 731, 746, 770, 764, 791, 784, 771, 792, 782, 776] payload_rows_per_dst=[9384, 9282, 9256, 9250, 9200, 9368, 9228, 9428] dup_rows_per_dst=[3060, 3154, 2862, 2972, 3090, 2846, 2892, 3032] combine_rows_per_src=[9384, 9282, 9256, 9250, 9200, 9368, 9228, 9428] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64870 dup_rows=24068 dup_rows_remote=20988 dedup_ratio=0.244832 router_tokens_per_expert=[739, 733, 796, 737, 745, 817, 757, 781, 769, 726, 764, 736, 746, 790, 763, 786, 803, 754, 800, 790, 743, 784, 778, 771, 771, 783, 769, 812, 738, 779, 776, 729, 823, 798, 776, 726, 724, 776, 764, 773, 784, 747, 767, 772, 766, 746, 766, 765, 732, 703, 794, 768, 774, 804, 851, 759, 763, 752, 749, 797, 722, 791, 793, 762] payload_rows_per_dst=[9210, 9180, 9366, 9252, 9294, 9314, 9382, 9238] dup_rows_per_dst=[3000, 2980, 3080, 3062, 3026, 2912, 2988, 3020] combine_rows_per_src=[9210, 9180, 9366, 9252, 9294, 9314, 9382, 9238] + [2026-08-26 15:58:14] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 796.0 | throughput per GPU (TFLOP/s/GPU): 160.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.619553E+00 | loss scale: 1.0 | grad norm: 2.956 | number of skipped iterations: 0 | number of nan iterations: 0 | WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74440 payload_rows_remote=65128 dup_rows=23864 dup_rows_remote=20864 dedup_ratio=0.242757 router_tokens_per_expert=[763, 781, 809, 747, 784, 777, 761, 767, 755, 793, 712, 784, 802, 782, 775, 753, 738, 790, 804, 800, 767, 765, 799, 753, 792, 810, 779, 739, 690, 772, 761, 751, 752, 760, 731, 754, 797, 736, 746, 746, 745, 760, 741, 773, 830, 769, 728, 840, 756, 764, 799, 748, 777, 767, 717, 751, 763, 728, 781, 769, 775, 794, 804, 796] payload_rows_per_dst=[9310, 9312, 9372, 9284, 9206, 9356, 9254, 9346] dup_rows_per_dst=[3068, 3000, 3060, 2904, 2838, 3016, 2904, 3074] combine_rows_per_src=[9310, 9312, 9372, 9284, 9206, 9356, 9254, 9346] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74462 payload_rows_remote=65192 dup_rows=23842 dup_rows_remote=20932 dedup_ratio=0.242533 router_tokens_per_expert=[787, 755, 755, 760, 741, 761, 790, 776, 769, 779, 793, 745, 781, 779, 739, 790, 746, 783, 734, 766, 720, 750, 818, 798, 744, 804, 762, 796, 765, 814, 765, 708, 758, 760, 798, 784, 745, 755, 763, 767, 730, 784, 806, 733, 785, 778, 719, 755, 775, 782, 782, 769, 807, 788, 771, 729, 807, 771, 752, 778, 788, 711, 780, 769] payload_rows_per_dst=[9342, 9310, 9320, 9262, 9296, 9270, 9368, 9294] dup_rows_per_dst=[2908, 3040, 2910, 3054, 2964, 2910, 3038, 3018] combine_rows_per_src=[9342, 9310, 9320, 9262, 9296, 9270, 9368, 9294] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=64966 dup_rows=24098 dup_rows_remote=21146 dedup_ratio=0.245138 router_tokens_per_expert=[763, 751, 772, 740, 765, 743, 770, 749, 752, 754, 765, 816, 773, 815, 772, 795, 793, 784, 794, 789, 749, 789, 717, 794, 765, 763, 774, 784, 749, 780, 809, 784, 780, 725, 766, 768, 827, 768, 761, 756, 760, 773, 748, 781, 747, 700, 763, 809, 785, 782, 800, 778, 752, 719, 770, 757, 771, 793, 732, 764, 754, 761, 748, 742] payload_rows_per_dst=[9092, 9066, 9004, 9442, 9558, 9554, 9250, 9240] dup_rows_per_dst=[2874, 2814, 2830, 3090, 3348, 3192, 2998, 2952] combine_rows_per_src=[9092, 9066, 9004, 9442, 9558, 9554, 9250, 9240] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64870 dup_rows=24068 dup_rows_remote=20988 dedup_ratio=0.244832 router_tokens_per_expert=[739, 733, 796, 737, 745, 817, 757, 781, 769, 726, 764, 736, 746, 790, 763, 786, 803, 754, 800, 790, 743, 784, 778, 771, 771, 783, 769, 812, 738, 779, 776, 729, 823, 798, 776, 726, 724, 776, 764, 773, 784, 747, 767, 772, 766, 746, 766, 765, 732, 703, 794, 768, 774, 804, 851, 759, 763, 752, 749, 797, 722, 791, 793, 762] payload_rows_per_dst=[9210, 9180, 9366, 9252, 9294, 9314, 9382, 9238] dup_rows_per_dst=[3000, 2980, 3080, 3062, 3026, 2912, 2988, 3020] combine_rows_per_src=[9210, 9180, 9366, 9252, 9294, 9314, 9382, 9238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74290 payload_rows_remote=64896 dup_rows=24014 dup_rows_remote=21020 dedup_ratio=0.244283 router_tokens_per_expert=[748, 775, 729, 790, 783, 824, 766, 779, 781, 774, 788, 740, 773, 761, 766, 788, 788, 793, 785, 788, 767, 772, 766, 793, 729, 788, 827, 754, 785, 780, 741, 803, 753, 750, 800, 734, 745, 773, 759, 760, 737, 791, 752, 764, 764, 757, 737, 749, 803, 776, 714, 832, 776, 728, 776, 793, 740, 749, 726, 727, 793, 747, 770, 753] payload_rows_per_dst=[9394, 9360, 9474, 9370, 9212, 9216, 9202, 9062] dup_rows_per_dst=[2994, 3242, 3274, 3044, 2936, 2886, 2934, 2704] combine_rows_per_src=[9394, 9360, 9474, 9370, 9212, 9216, 9202, 9062] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74556 payload_rows_remote=65236 dup_rows=23748 dup_rows_remote=20826 dedup_ratio=0.241577 router_tokens_per_expert=[813, 769, 782, 764, 783, 747, 775, 788, 711, 755, 759, 795, 775, 780, 777, 788, 744, 728, 787, 753, 760, 758, 735, 782, 741, 797, 730, 769, 749, 778, 786, 771, 792, 714, 769, 756, 728, 718, 776, 783, 727, 763, 741, 779, 783, 757, 777, 743, 841, 810, 762, 802, 767, 810, 759, 774, 724, 800, 817, 726, 790, 836, 713, 786] payload_rows_per_dst=[9406, 9322, 9190, 9320, 9276, 9198, 9530, 9314] dup_rows_per_dst=[3036, 2958, 2904, 2922, 2796, 2942, 3120, 3070] combine_rows_per_src=[9406, 9322, 9190, 9320, 9276, 9198, 9530, 9314] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74082 payload_rows_remote=64842 dup_rows=24222 dup_rows_remote=21246 dedup_ratio=0.246399 router_tokens_per_expert=[787, 728, 753, 727, 802, 788, 782, 746, 819, 790, 763, 716, 779, 714, 799, 762, 820, 753, 750, 802, 699, 779, 782, 816, 765, 758, 758, 754, 745, 749, 765, 744, 791, 716, 789, 766, 761, 804, 751, 754, 789, 744, 771, 782, 753, 756, 816, 751, 767, 773, 751, 727, 769, 787, 801, 733, 739, 744, 811, 821, 746, 815, 777, 803] payload_rows_per_dst=[9246, 9242, 9284, 9158, 9248, 9240, 9240, 9424] dup_rows_per_dst=[2980, 3042, 3118, 2918, 3016, 3084, 2976, 3088] combine_rows_per_src=[9246, 9242, 9284, 9158, 9248, 9240, 9240, 9424] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=64966 dup_rows=24098 dup_rows_remote=21146 dedup_ratio=0.245138 router_tokens_per_expert=[763, 751, 772, 740, 765, 743, 770, 749, 752, 754, 765, 816, 773, 815, 772, 795, 793, 784, 794, 789, 749, 789, 717, 794, 765, 763, 774, 784, 749, 780, 809, 784, 780, 725, 766, 768, 827, 768, 761, 756, 760, 773, 748, 781, 747, 700, 763, 809, 785, 782, 800, 778, 752, 719, 770, 757, 771, 793, 732, 764, 754, 761, 748, 742] payload_rows_per_dst=[9092, 9066, 9004, 9442, 9558, 9554, 9250, 9240] dup_rows_per_dst=[2874, 2814, 2830, 3090, 3348, 3192, 2998, 2952] combine_rows_per_src=[9092, 9066, 9004, 9442, 9558, 9554, 9250, 9240] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74462 payload_rows_remote=65192 dup_rows=23842 dup_rows_remote=20932 dedup_ratio=0.242533 router_tokens_per_expert=[787, 755, 755, 760, 741, 761, 790, 776, 769, 779, 793, 745, 781, 779, 739, 790, 746, 783, 734, 766, 720, 750, 818, 798, 744, 804, 762, 796, 765, 814, 765, 708, 758, 760, 798, 784, 745, 755, 763, 767, 730, 784, 806, 733, 785, 778, 719, 755, 775, 782, 782, 769, 807, 788, 771, 729, 807, 771, 752, 778, 788, 711, 780, 769] payload_rows_per_dst=[9342, 9310, 9320, 9262, 9296, 9270, 9368, 9294] dup_rows_per_dst=[2908, 3040, 2910, 3054, 2964, 2910, 3038, 3018] combine_rows_per_src=[9342, 9310, 9320, 9262, 9296, 9270, 9368, 9294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266764288 dispatch_meta_bytes=343968 combine_payload_bytes=266764288 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267108256 ingress_bytes=335970304 link_bytes=335970304 token_bytes=533872544 weight_movement_bytes=69206016 total_nvlink_bytes=603078560 weight_to_token_ratio=0.129630 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264229206e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265707520 dispatch_meta_bytes=343432 combine_payload_bytes=265707520 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266050952 ingress_bytes=334913536 link_bytes=334913536 token_bytes=531758472 weight_movement_bytes=69206016 total_nvlink_bytes=600964488 weight_to_token_ratio=0.130146 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260252673e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74082 payload_rows_remote=64842 dup_rows=24222 dup_rows_remote=21246 dedup_ratio=0.246399 router_tokens_per_expert=[787, 728, 753, 727, 802, 788, 782, 746, 819, 790, 763, 716, 779, 714, 799, 762, 820, 753, 750, 802, 699, 779, 782, 816, 765, 758, 758, 754, 745, 749, 765, 744, 791, 716, 789, 766, 761, 804, 751, 754, 789, 744, 771, 782, 753, 756, 816, 751, 767, 773, 751, 727, 769, 787, 801, 733, 739, 744, 811, 821, 746, 815, 777, 803] payload_rows_per_dst=[9246, 9242, 9284, 9158, 9248, 9240, 9240, 9424] dup_rows_per_dst=[2980, 3042, 3118, 2918, 3016, 3084, 2976, 3088] combine_rows_per_src=[9246, 9242, 9284, 9158, 9248, 9240, 9240, 9424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265814016 dispatch_meta_bytes=343664 combine_payload_bytes=265814016 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266157680 ingress_bytes=335020032 link_bytes=335020032 token_bytes=531971696 weight_movement_bytes=69206016 total_nvlink_bytes=601177712 weight_to_token_ratio=0.130093 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260653409e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267206656 dispatch_meta_bytes=344248 combine_payload_bytes=267206656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267550904 ingress_bytes=301809664 link_bytes=301809664 token_bytes=534757560 weight_movement_bytes=34603008 total_nvlink_bytes=569360568 weight_to_token_ratio=0.064708 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135685468e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267042816 dispatch_meta_bytes=344056 combine_payload_bytes=267042816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267386872 ingress_bytes=301645824 link_bytes=301645824 token_bytes=534429688 weight_movement_bytes=34603008 total_nvlink_bytes=569032696 weight_to_token_ratio=0.064748 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135068951e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266764288 dispatch_meta_bytes=343968 combine_payload_bytes=266764288 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267108256 ingress_bytes=335970304 link_bytes=335970304 token_bytes=533872544 weight_movement_bytes=69206016 total_nvlink_bytes=603078560 weight_to_token_ratio=0.129630 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264229206e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267026432 dispatch_meta_bytes=344496 combine_payload_bytes=267026432 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267370928 ingress_bytes=301629440 link_bytes=301629440 token_bytes=534397360 weight_movement_bytes=34603008 total_nvlink_bytes=569000368 weight_to_token_ratio=0.064751 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135007300e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266100736 dispatch_meta_bytes=344448 combine_payload_bytes=266100736 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266445184 ingress_bytes=335306752 link_bytes=335306752 token_bytes=532545920 weight_movement_bytes=69206016 total_nvlink_bytes=601751936 weight_to_token_ratio=0.129953 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261732313e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267206656 dispatch_meta_bytes=344248 combine_payload_bytes=267206656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267550904 ingress_bytes=301809664 link_bytes=301809664 token_bytes=534757560 weight_movement_bytes=34603008 total_nvlink_bytes=569360568 weight_to_token_ratio=0.064708 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135685468e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65196 dup_rows=23908 dup_rows_remote=20818 dedup_ratio=0.243205 router_tokens_per_expert=[785, 785, 789, 772, 763, 798, 769, 761, 766, 819, 750, 810, 779, 787, 746, 761, 787, 756, 730, 724, 751, 795, 749, 767, 781, 762, 751, 728, 719, 760, 832, 778, 784, 786, 778, 729, 778, 762, 777, 751, 760, 779, 814, 747, 786, 765, 771, 685, 759, 744, 818, 742, 753, 767, 731, 746, 770, 764, 791, 784, 771, 792, 782, 776] payload_rows_per_dst=[9384, 9282, 9256, 9250, 9200, 9368, 9228, 9428] dup_rows_per_dst=[3060, 3154, 2862, 2972, 3090, 2846, 2892, 3032] combine_rows_per_src=[9384, 9282, 9256, 9250, 9200, 9368, 9228, 9428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267026432 dispatch_meta_bytes=344496 combine_payload_bytes=267026432 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267370928 ingress_bytes=301629440 link_bytes=301629440 token_bytes=534397360 weight_movement_bytes=34603008 total_nvlink_bytes=569000368 weight_to_token_ratio=0.064751 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135007300e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265592832 dispatch_meta_bytes=344352 combine_payload_bytes=265592832 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265937184 ingress_bytes=369401856 link_bytes=369401856 token_bytes=531530016 weight_movement_bytes=103809024 total_nvlink_bytes=635339040 weight_to_token_ratio=0.195302 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390029445e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74290 payload_rows_remote=64896 dup_rows=24014 dup_rows_remote=21020 dedup_ratio=0.244283 router_tokens_per_expert=[748, 775, 729, 790, 783, 824, 766, 779, 781, 774, 788, 740, 773, 761, 766, 788, 788, 793, 785, 788, 767, 772, 766, 793, 729, 788, 827, 754, 785, 780, 741, 803, 753, 750, 800, 734, 745, 773, 759, 760, 737, 791, 752, 764, 764, 757, 737, 749, 803, 776, 714, 832, 776, 728, 776, 793, 740, 749, 726, 727, 793, 747, 770, 753] payload_rows_per_dst=[9394, 9360, 9474, 9370, 9212, 9216, 9202, 9062] dup_rows_per_dst=[2994, 3242, 3274, 3044, 2936, 2886, 2934, 2704] combine_rows_per_src=[9394, 9360, 9474, 9370, 9212, 9216, 9202, 9062] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74140 payload_rows_remote=64912 dup_rows=24164 dup_rows_remote=21224 dedup_ratio=0.245809 router_tokens_per_expert=[833, 783, 731, 743, 795, 804, 804, 794, 738, 748, 800, 757, 737, 799, 766, 721, 761, 770, 708, 749, 747, 796, 774, 779, 769, 778, 761, 789, 750, 761, 766, 757, 771, 788, 774, 833, 761, 802, 709, 698, 776, 755, 767, 765, 770, 793, 744, 771, 753, 725, 776, 747, 759, 780, 807, 789, 789, 742, 773, 827, 735, 764, 771, 770] payload_rows_per_dst=[9458, 9184, 9228, 9262, 9224, 9280, 9232, 9272] dup_rows_per_dst=[3116, 2948, 2940, 3000, 3048, 3002, 3040, 3070] combine_rows_per_src=[9458, 9184, 9228, 9262, 9224, 9280, 9232, 9272] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74368 payload_rows_remote=65044 dup_rows=23936 dup_rows_remote=20964 dedup_ratio=0.243490 router_tokens_per_expert=[761, 780, 775, 770, 769, 755, 802, 739, 759, 788, 788, 737, 777, 744, 802, 752, 760, 769, 725, 792, 741, 776, 759, 782, 791, 782, 808, 783, 735, 793, 795, 771, 723, 761, 766, 799, 744, 797, 792, 766, 774, 729, 756, 785, 754, 730, 779, 757, 772, 740, 756, 805, 785, 729, 797, 724, 760, 741, 753, 785, 701, 817, 811, 804] payload_rows_per_dst=[9268, 9366, 9156, 9388, 9324, 9238, 9284, 9344] dup_rows_per_dst=[3034, 2928, 3052, 3128, 2972, 2890, 2932, 3000] combine_rows_per_src=[9268, 9366, 9156, 9388, 9324, 9238, 9284, 9344] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74472 payload_rows_remote=65292 dup_rows=23832 dup_rows_remote=20830 dedup_ratio=0.242432 router_tokens_per_expert=[768, 726, 735, 749, 751, 806, 743, 767, 767, 761, 795, 789, 736, 764, 769, 778, 788, 821, 749, 746, 752, 755, 874, 784, 746, 797, 734, 752, 772, 789, 779, 767, 748, 776, 743, 744, 733, 752, 814, 783, 762, 779, 764, 748, 746, 783, 748, 761, 740, 752, 777, 744, 791, 766, 784, 831, 743, 733, 762, 768, 798, 822, 800, 748] payload_rows_per_dst=[9260, 9372, 9412, 9324, 9234, 9180, 9328, 9362] dup_rows_per_dst=[2830, 2946, 3126, 2948, 2952, 3002, 3042, 2986] combine_rows_per_src=[9260, 9372, 9412, 9324, 9234, 9180, 9328, 9362] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74418 payload_rows_remote=65126 dup_rows=23886 dup_rows_remote=20900 dedup_ratio=0.242981 router_tokens_per_expert=[815, 752, 802, 753, 772, 770, 831, 732, 812, 735, 763, 801, 750, 698, 800, 780, 744, 814, 801, 767, 769, 800, 775, 781, 814, 741, 780, 702, 788, 738, 750, 789, 801, 754, 777, 754, 755, 772, 782, 766, 782, 756, 772, 766, 757, 819, 696, 762, 764, 770, 769, 767, 767, 760, 757, 790, 769, 730, 744, 784, 717, 736, 752, 786] payload_rows_per_dst=[9360, 9292, 9446, 9284, 9290, 9214, 9308, 9224] dup_rows_per_dst=[3094, 2986, 3056, 2920, 3032, 3006, 2980, 2812] combine_rows_per_src=[9360, 9292, 9446, 9284, 9290, 9214, 9308, 9224] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74238 payload_rows_remote=64972 dup_rows=24066 dup_rows_remote=21036 dedup_ratio=0.244812 router_tokens_per_expert=[764, 749, 824, 753, 771, 761, 754, 673, 751, 760, 763, 707, 780, 762, 799, 797, 765, 754, 791, 739, 792, 789, 768, 812, 780, 751, 767, 766, 767, 784, 780, 745, 768, 783, 775, 766, 764, 776, 775, 810, 787, 813, 780, 792, 785, 797, 767, 755, 749, 760, 724, 761, 743, 765, 817, 776, 752, 741, 762, 763, 776, 726, 756, 770] payload_rows_per_dst=[9054, 9046, 9300, 9332, 9526, 9480, 9234, 9266] dup_rows_per_dst=[2840, 2976, 2980, 3090, 3122, 3050, 2978, 3030] combine_rows_per_src=[9054, 9046, 9300, 9332, 9526, 9480, 9234, 9266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74140 payload_rows_remote=64912 dup_rows=24164 dup_rows_remote=21224 dedup_ratio=0.245809 router_tokens_per_expert=[833, 783, 731, 743, 795, 804, 804, 794, 738, 748, 800, 757, 737, 799, 766, 721, 761, 770, 708, 749, 747, 796, 774, 779, 769, 778, 761, 789, 750, 761, 766, 757, 771, 788, 774, 833, 761, 802, 709, 698, 776, 755, 767, 765, 770, 793, 744, 771, 753, 725, 776, 747, 759, 780, 807, 789, 789, 742, 773, 827, 735, 764, 771, 770] payload_rows_per_dst=[9458, 9184, 9228, 9262, 9224, 9280, 9232, 9272] dup_rows_per_dst=[3116, 2948, 2940, 3000, 3048, 3002, 3040, 3070] combine_rows_per_src=[9458, 9184, 9228, 9262, 9224, 9280, 9232, 9272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=64982 dup_rows=23974 dup_rows_remote=20950 dedup_ratio=0.243876 router_tokens_per_expert=[763, 792, 760, 766, 791, 740, 786, 756, 793, 787, 788, 783, 780, 759, 786, 770, 782, 812, 729, 743, 783, 784, 703, 704, 789, 757, 729, 729, 769, 776, 757, 773, 770, 752, 725, 742, 775, 788, 748, 772, 792, 749, 751, 775, 732, 753, 771, 791, 788, 778, 773, 772, 819, 773, 771, 821, 803, 767, 768, 752, 734, 772, 801, 755] payload_rows_per_dst=[9348, 9470, 9242, 9236, 9174, 9192, 9400, 9268] dup_rows_per_dst=[3024, 3022, 2838, 2922, 2970, 3036, 3126, 3036] combine_rows_per_src=[9348, 9470, 9242, 9236, 9174, 9192, 9400, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=65032 dup_rows=24068 dup_rows_remote=21190 dedup_ratio=0.244832 router_tokens_per_expert=[806, 786, 783, 741, 746, 729, 749, 763, 807, 802, 830, 745, 722, 758, 767, 779, 801, 811, 804, 744, 778, 772, 735, 763, 718, 744, 742, 741, 741, 790, 793, 759, 764, 792, 762, 750, 756, 747, 798, 809, 795, 782, 730, 792, 798, 786, 764, 759, 758, 729, 774, 781, 757, 725, 773, 744, 794, 781, 778, 770, 769, 742, 787, 757] payload_rows_per_dst=[9252, 9304, 9396, 9194, 9272, 9304, 9204, 9310] dup_rows_per_dst=[2954, 3116, 3020, 2862, 3084, 3108, 2878, 3046] combine_rows_per_src=[9252, 9304, 9396, 9194, 9272, 9304, 9204, 9310] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=64952 dup_rows=23964 dup_rows_remote=20938 dedup_ratio=0.243774 router_tokens_per_expert=[763, 797, 741, 751, 768, 797, 816, 773, 789, 812, 791, 742, 776, 759, 745, 760, 733, 767, 770, 741, 736, 765, 729, 815, 807, 780, 784, 770, 778, 787, 736, 765, 754, 732, 757, 736, 768, 768, 807, 718, 791, 765, 787, 798, 740, 756, 771, 753, 773, 761, 777, 792, 734, 767, 782, 783, 761, 762, 764, 797, 755, 777, 774, 749] payload_rows_per_dst=[9358, 9412, 9152, 9388, 9226, 9340, 9132, 9332] dup_rows_per_dst=[3054, 2936, 2960, 3026, 2854, 2982, 3206, 2946] combine_rows_per_src=[9358, 9412, 9152, 9388, 9226, 9340, 9132, 9332] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265814016 dispatch_meta_bytes=343664 combine_payload_bytes=265814016 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266157680 ingress_bytes=335020032 link_bytes=335020032 token_bytes=531971696 weight_movement_bytes=69206016 total_nvlink_bytes=601177712 weight_to_token_ratio=0.130093 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260653409e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267042816 dispatch_meta_bytes=344056 combine_payload_bytes=267042816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267386872 ingress_bytes=301645824 link_bytes=301645824 token_bytes=534429688 weight_movement_bytes=34603008 total_nvlink_bytes=569032696 weight_to_token_ratio=0.064748 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135068951e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266756096 dispatch_meta_bytes=344104 combine_payload_bytes=266756096 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267100200 ingress_bytes=335962112 link_bytes=335962112 token_bytes=533856296 weight_movement_bytes=69206016 total_nvlink_bytes=603062312 weight_to_token_ratio=0.129634 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264198380e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265879552 dispatch_meta_bytes=344544 combine_payload_bytes=265879552 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266224096 ingress_bytes=335085568 link_bytes=335085568 token_bytes=532103648 weight_movement_bytes=69206016 total_nvlink_bytes=601309664 weight_to_token_ratio=0.130061 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260900016e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=344032 combine_payload_bytes=266420224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266764256 ingress_bytes=301023232 link_bytes=301023232 token_bytes=533184480 weight_movement_bytes=34603008 total_nvlink_bytes=567787488 weight_to_token_ratio=0.064899 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132726188e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74238 payload_rows_remote=64972 dup_rows=24066 dup_rows_remote=21036 dedup_ratio=0.244812 router_tokens_per_expert=[764, 749, 824, 753, 771, 761, 754, 673, 751, 760, 763, 707, 780, 762, 799, 797, 765, 754, 791, 739, 792, 789, 768, 812, 780, 751, 767, 766, 767, 784, 780, 745, 768, 783, 775, 766, 764, 776, 775, 810, 787, 813, 780, 792, 785, 797, 767, 755, 749, 760, 724, 761, 743, 765, 817, 776, 752, 741, 762, 763, 776, 726, 756, 770] payload_rows_per_dst=[9054, 9046, 9300, 9332, 9526, 9480, 9234, 9266] dup_rows_per_dst=[2840, 2976, 2980, 3090, 3122, 3050, 2978, 3030] combine_rows_per_src=[9054, 9046, 9300, 9332, 9526, 9480, 9234, 9266] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267436032 dispatch_meta_bytes=344488 combine_payload_bytes=267436032 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267780520 ingress_bytes=302039040 link_bytes=302039040 token_bytes=535216552 weight_movement_bytes=34603008 total_nvlink_bytes=569819560 weight_to_token_ratio=0.064652 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136548591e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266756096 dispatch_meta_bytes=344104 combine_payload_bytes=266756096 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267100200 ingress_bytes=335962112 link_bytes=335962112 token_bytes=533856296 weight_movement_bytes=69206016 total_nvlink_bytes=603062312 weight_to_token_ratio=0.129634 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264198380e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266166272 dispatch_meta_bytes=343728 combine_payload_bytes=266166272 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266510000 ingress_bytes=335372288 link_bytes=335372288 token_bytes=532676272 weight_movement_bytes=69206016 total_nvlink_bytes=601882288 weight_to_token_ratio=0.129921 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261978920e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=65032 dup_rows=24068 dup_rows_remote=21190 dedup_ratio=0.244832 router_tokens_per_expert=[806, 786, 783, 741, 746, 729, 749, 763, 807, 802, 830, 745, 722, 758, 767, 779, 801, 811, 804, 744, 778, 772, 735, 763, 718, 744, 742, 741, 741, 790, 793, 759, 764, 792, 762, 750, 756, 747, 798, 809, 795, 782, 730, 792, 798, 786, 764, 759, 758, 729, 774, 781, 757, 725, 773, 744, 794, 781, 778, 770, 769, 742, 787, 757] payload_rows_per_dst=[9252, 9304, 9396, 9194, 9272, 9304, 9204, 9310] dup_rows_per_dst=[2954, 3116, 3020, 2862, 3084, 3108, 2878, 3046] combine_rows_per_src=[9252, 9304, 9396, 9194, 9272, 9304, 9204, 9310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74472 payload_rows_remote=65292 dup_rows=23832 dup_rows_remote=20830 dedup_ratio=0.242432 router_tokens_per_expert=[768, 726, 735, 749, 751, 806, 743, 767, 767, 761, 795, 789, 736, 764, 769, 778, 788, 821, 749, 746, 752, 755, 874, 784, 746, 797, 734, 752, 772, 789, 779, 767, 748, 776, 743, 744, 733, 752, 814, 783, 762, 779, 764, 748, 746, 783, 748, 761, 740, 752, 777, 744, 791, 766, 784, 831, 743, 733, 762, 768, 798, 822, 800, 748] payload_rows_per_dst=[9260, 9372, 9412, 9324, 9234, 9180, 9328, 9362] dup_rows_per_dst=[2830, 2946, 3126, 2948, 2952, 3002, 3042, 2986] combine_rows_per_src=[9260, 9372, 9412, 9324, 9234, 9180, 9328, 9362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266043392 dispatch_meta_bytes=343560 combine_payload_bytes=266043392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266386952 ingress_bytes=300646400 link_bytes=300646400 token_bytes=532430344 weight_movement_bytes=34603008 total_nvlink_bytes=567033352 weight_to_token_ratio=0.064991 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131308199e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266125312 dispatch_meta_bytes=344032 combine_payload_bytes=266125312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266469344 ingress_bytes=300728320 link_bytes=300728320 token_bytes=532594656 weight_movement_bytes=34603008 total_nvlink_bytes=567197664 weight_to_token_ratio=0.064971 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131616458e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344888 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266715960 ingress_bytes=335577088 link_bytes=335577088 token_bytes=533087032 weight_movement_bytes=69206016 total_nvlink_bytes=602293048 weight_to_token_ratio=0.129821 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262749566e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266043392 dispatch_meta_bytes=343560 combine_payload_bytes=266043392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266386952 ingress_bytes=300646400 link_bytes=300646400 token_bytes=532430344 weight_movement_bytes=34603008 total_nvlink_bytes=567033352 weight_to_token_ratio=0.064991 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131308199e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74420 payload_rows_remote=65276 dup_rows=23884 dup_rows_remote=20906 dedup_ratio=0.242961 router_tokens_per_expert=[748, 750, 814, 778, 775, 789, 795, 750, 739, 764, 768, 789, 795, 763, 805, 805, 786, 743, 711, 785, 781, 779, 732, 756, 809, 768, 784, 793, 696, 782, 804, 703, 776, 756, 739, 726, 826, 763, 762, 713, 789, 802, 764, 769, 753, 760, 786, 763, 732, 719, 820, 786, 791, 781, 781, 679, 777, 768, 754, 771, 810, 750, 774, 773] payload_rows_per_dst=[9340, 9496, 9194, 9304, 9144, 9314, 9268, 9360] dup_rows_per_dst=[3058, 2960, 2952, 2974, 2978, 3058, 2910, 2994] combine_rows_per_src=[9340, 9496, 9194, 9304, 9144, 9314, 9268, 9360] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=64936 dup_rows=23992 dup_rows_remote=20948 dedup_ratio=0.244059 router_tokens_per_expert=[757, 774, 764, 773, 734, 760, 750, 724, 730, 839, 794, 762, 755, 755, 774, 778, 771, 798, 776, 787, 760, 767, 757, 794, 720, 761, 774, 765, 794, 790, 781, 818, 782, 750, 772, 790, 755, 795, 774, 754, 767, 731, 767, 812, 732, 774, 795, 752, 744, 746, 703, 762, 758, 808, 799, 737, 750, 806, 781, 797, 787, 734, 768, 734] payload_rows_per_dst=[9206, 9296, 9376, 9312, 9382, 9258, 9144, 9338] dup_rows_per_dst=[2866, 3078, 3044, 3094, 2962, 3002, 2970, 2976] combine_rows_per_src=[9206, 9296, 9376, 9312, 9382, 9258, 9144, 9338] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=64946 dup_rows=24002 dup_rows_remote=20996 dedup_ratio=0.244161 router_tokens_per_expert=[755, 763, 790, 783, 711, 750, 758, 762, 762, 771, 806, 734, 817, 774, 779, 727, 799, 770, 758, 758, 779, 810, 769, 768, 791, 773, 734, 790, 733, 764, 735, 787, 775, 774, 760, 781, 777, 739, 755, 700, 781, 773, 757, 786, 776, 746, 743, 819, 804, 783, 804, 780, 796, 778, 779, 742, 793, 777, 748, 757, 730, 768, 716, 795] payload_rows_per_dst=[9174, 9362, 9388, 9180, 9156, 9356, 9438, 9248] dup_rows_per_dst=[2970, 2978, 3034, 3034, 2966, 3006, 3094, 2920] combine_rows_per_src=[9174, 9362, 9388, 9180, 9156, 9356, 9438, 9248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74178 payload_rows_remote=64804 dup_rows=24126 dup_rows_remote=20950 dedup_ratio=0.245422 router_tokens_per_expert=[769, 773, 756, 727, 800, 780, 822, 731, 731, 736, 781, 827, 781, 782, 742, 814, 753, 773, 780, 775, 749, 802, 754, 772, 757, 779, 760, 769, 829, 799, 740, 739, 764, 758, 726, 802, 760, 743, 795, 738, 767, 774, 756, 750, 738, 778, 806, 786, 776, 794, 735, 809, 729, 766, 779, 773, 753, 778, 763, 667, 765, 805, 796, 741] payload_rows_per_dst=[9286, 8998, 9190, 9090, 9556, 9302, 9382, 9374] dup_rows_per_dst=[3030, 2722, 3050, 2866, 3284, 3046, 2952, 3176] combine_rows_per_src=[9286, 8998, 9190, 9090, 9556, 9302, 9382, 9374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=64982 dup_rows=23974 dup_rows_remote=20950 dedup_ratio=0.243876 router_tokens_per_expert=[763, 792, 760, 766, 791, 740, 786, 756, 793, 787, 788, 783, 780, 759, 786, 770, 782, 812, 729, 743, 783, 784, 703, 704, 789, 757, 729, 729, 769, 776, 757, 773, 770, 752, 725, 742, 775, 788, 748, 772, 792, 749, 751, 775, 732, 753, 771, 791, 788, 778, 773, 772, 819, 773, 771, 821, 803, 767, 768, 752, 734, 772, 801, 755] payload_rows_per_dst=[9348, 9470, 9242, 9236, 9174, 9192, 9400, 9268] dup_rows_per_dst=[3024, 3022, 2838, 2922, 2970, 3036, 3126, 3036] combine_rows_per_src=[9348, 9470, 9242, 9236, 9174, 9192, 9400, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267436032 dispatch_meta_bytes=344488 combine_payload_bytes=267436032 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267780520 ingress_bytes=302039040 link_bytes=302039040 token_bytes=535216552 weight_movement_bytes=34603008 total_nvlink_bytes=569819560 weight_to_token_ratio=0.064652 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136548591e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74404 payload_rows_remote=65156 dup_rows=23900 dup_rows_remote=20836 dedup_ratio=0.243123 router_tokens_per_expert=[787, 801, 785, 727, 747, 806, 744, 741, 784, 754, 740, 764, 820, 759, 768, 767, 779, 797, 740, 767, 795, 763, 741, 724, 732, 718, 801, 762, 754, 828, 735, 769, 744, 749, 809, 755, 748, 712, 796, 767, 745, 746, 764, 800, 766, 736, 798, 762, 754, 749, 752, 806, 787, 776, 806, 735, 750, 789, 806, 763, 795, 816, 782, 790] payload_rows_per_dst=[9348, 9248, 9268, 9262, 9110, 9312, 9378, 9478] dup_rows_per_dst=[2928, 3064, 2944, 2936, 3050, 2922, 2952, 3104] combine_rows_per_src=[9348, 9248, 9268, 9262, 9110, 9312, 9378, 9478] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65058 dup_rows=23908 dup_rows_remote=20994 dedup_ratio=0.243205 router_tokens_per_expert=[747, 804, 759, 761, 711, 777, 753, 762, 817, 735, 786, 759, 813, 764, 763, 769, 745, 742, 797, 778, 766, 842, 760, 761, 749, 839, 809, 746, 779, 780, 772, 753, 771, 829, 754, 744, 774, 701, 782, 774, 781, 755, 757, 716, 786, 798, 742, 768, 762, 729, 760, 786, 755, 762, 748, 824, 794, 783, 762, 751, 767, 749, 765, 725] payload_rows_per_dst=[9190, 9348, 9332, 9324, 9332, 9282, 9338, 9250] dup_rows_per_dst=[2958, 3064, 3050, 3130, 2926, 2924, 2914, 2942] combine_rows_per_src=[9190, 9348, 9332, 9324, 9332, 9282, 9338, 9250] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74344 payload_rows_remote=64894 dup_rows=23960 dup_rows_remote=20904 dedup_ratio=0.243734 router_tokens_per_expert=[780, 791, 777, 730, 773, 816, 765, 816, 790, 788, 723, 742, 732, 769, 771, 754, 781, 747, 767, 781, 767, 718, 825, 770, 807, 740, 819, 805, 734, 792, 756, 800, 755, 730, 772, 776, 737, 762, 726, 742, 802, 764, 732, 761, 804, 777, 804, 795, 726, 835, 796, 799, 749, 743, 728, 786, 732, 744, 787, 722, 737, 755, 777, 771] payload_rows_per_dst=[9362, 9246, 9324, 9450, 9122, 9446, 9330, 9064] dup_rows_per_dst=[3134, 2892, 2988, 3056, 2878, 3032, 2994, 2986] combine_rows_per_src=[9362, 9246, 9324, 9450, 9122, 9446, 9330, 9064] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=64936 dup_rows=23992 dup_rows_remote=20948 dedup_ratio=0.244059 router_tokens_per_expert=[757, 774, 764, 773, 734, 760, 750, 724, 730, 839, 794, 762, 755, 755, 774, 778, 771, 798, 776, 787, 760, 767, 757, 794, 720, 761, 774, 765, 794, 790, 781, 818, 782, 750, 772, 790, 755, 795, 774, 754, 767, 731, 767, 812, 732, 774, 795, 752, 744, 746, 703, 762, 758, 808, 799, 737, 750, 806, 781, 797, 787, 734, 768, 734] payload_rows_per_dst=[9206, 9296, 9376, 9312, 9382, 9258, 9144, 9338] dup_rows_per_dst=[2866, 3078, 3044, 3094, 2962, 3002, 2970, 2976] combine_rows_per_src=[9206, 9296, 9376, 9312, 9382, 9258, 9144, 9338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74368 payload_rows_remote=65044 dup_rows=23936 dup_rows_remote=20964 dedup_ratio=0.243490 router_tokens_per_expert=[761, 780, 775, 770, 769, 755, 802, 739, 759, 788, 788, 737, 777, 744, 802, 752, 760, 769, 725, 792, 741, 776, 759, 782, 791, 782, 808, 783, 735, 793, 795, 771, 723, 761, 766, 799, 744, 797, 792, 766, 774, 729, 756, 785, 754, 730, 779, 757, 772, 740, 756, 805, 785, 729, 797, 724, 760, 741, 753, 785, 701, 817, 811, 804] payload_rows_per_dst=[9268, 9366, 9156, 9388, 9324, 9238, 9284, 9344] dup_rows_per_dst=[3034, 2928, 3052, 3128, 2972, 2890, 2932, 3000] combine_rows_per_src=[9268, 9366, 9156, 9388, 9324, 9238, 9284, 9344] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74282 payload_rows_remote=64840 dup_rows=24022 dup_rows_remote=21010 dedup_ratio=0.244364 router_tokens_per_expert=[763, 829, 753, 768, 761, 800, 795, 749, 745, 770, 781, 782, 784, 770, 759, 790, 769, 764, 764, 727, 739, 798, 775, 758, 747, 761, 765, 777, 783, 776, 766, 771, 787, 825, 766, 776, 775, 757, 741, 802, 749, 784, 756, 756, 767, 724, 853, 728, 804, 756, 747, 773, 761, 808, 736, 735, 777, 781, 763, 748, 734, 707, 777, 760] payload_rows_per_dst=[9442, 9306, 9136, 9300, 9392, 9176, 9318, 9212] dup_rows_per_dst=[3012, 3246, 3034, 2992, 3066, 2868, 2922, 2882] combine_rows_per_src=[9442, 9306, 9136, 9300, 9392, 9176, 9318, 9212] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266878976 dispatch_meta_bytes=343968 combine_payload_bytes=266878976 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267222944 ingress_bytes=336084992 link_bytes=336084992 token_bytes=534101920 weight_movement_bytes=69206016 total_nvlink_bytes=603307936 weight_to_token_ratio=0.129575 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264660768e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265977856 dispatch_meta_bytes=343536 combine_payload_bytes=265977856 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266321392 ingress_bytes=404389888 link_bytes=404389888 token_bytes=532299248 weight_movement_bytes=138412032 total_nvlink_bytes=670711280 weight_to_token_ratio=0.260027 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521686592e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267370496 dispatch_meta_bytes=344728 combine_payload_bytes=267370496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267715224 ingress_bytes=301973504 link_bytes=301973504 token_bytes=535085720 weight_movement_bytes=34603008 total_nvlink_bytes=569688728 weight_to_token_ratio=0.064668 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136301985e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74344 payload_rows_remote=64894 dup_rows=23960 dup_rows_remote=20904 dedup_ratio=0.243734 router_tokens_per_expert=[780, 791, 777, 730, 773, 816, 765, 816, 790, 788, 723, 742, 732, 769, 771, 754, 781, 747, 767, 781, 767, 718, 825, 770, 807, 740, 819, 805, 734, 792, 756, 800, 755, 730, 772, 776, 737, 762, 726, 742, 802, 764, 732, 761, 804, 777, 804, 795, 726, 835, 796, 799, 749, 743, 728, 786, 732, 744, 787, 722, 737, 755, 777, 771] payload_rows_per_dst=[9362, 9246, 9324, 9450, 9122, 9446, 9330, 9064] dup_rows_per_dst=[3134, 2892, 2988, 3056, 2878, 3032, 2994, 2986] combine_rows_per_src=[9362, 9246, 9324, 9450, 9122, 9446, 9330, 9064] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65058 dup_rows=23908 dup_rows_remote=20994 dedup_ratio=0.243205 router_tokens_per_expert=[747, 804, 759, 761, 711, 777, 753, 762, 817, 735, 786, 759, 813, 764, 763, 769, 745, 742, 797, 778, 766, 842, 760, 761, 749, 839, 809, 746, 779, 780, 772, 753, 771, 829, 754, 744, 774, 701, 782, 774, 781, 755, 757, 716, 786, 798, 742, 768, 762, 729, 760, 786, 755, 762, 748, 824, 794, 783, 762, 751, 767, 749, 765, 725] payload_rows_per_dst=[9190, 9348, 9332, 9324, 9332, 9282, 9338, 9250] dup_rows_per_dst=[2958, 3064, 3050, 3130, 2926, 2924, 2914, 2942] combine_rows_per_src=[9190, 9348, 9332, 9324, 9332, 9282, 9338, 9250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=344032 combine_payload_bytes=266420224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266764256 ingress_bytes=301023232 link_bytes=301023232 token_bytes=533184480 weight_movement_bytes=34603008 total_nvlink_bytes=567787488 weight_to_token_ratio=0.064899 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132726188e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74178 payload_rows_remote=64804 dup_rows=24126 dup_rows_remote=20950 dedup_ratio=0.245422 router_tokens_per_expert=[769, 773, 756, 727, 800, 780, 822, 731, 731, 736, 781, 827, 781, 782, 742, 814, 753, 773, 780, 775, 749, 802, 754, 772, 757, 779, 760, 769, 829, 799, 740, 739, 764, 758, 726, 802, 760, 743, 795, 738, 767, 774, 756, 750, 738, 778, 806, 786, 776, 794, 735, 809, 729, 766, 779, 773, 753, 778, 763, 667, 765, 805, 796, 741] payload_rows_per_dst=[9286, 8998, 9190, 9090, 9556, 9302, 9382, 9374] dup_rows_per_dst=[3030, 2722, 3050, 2866, 3284, 3046, 2952, 3176] combine_rows_per_src=[9286, 8998, 9190, 9090, 9556, 9302, 9382, 9374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265584640 dispatch_meta_bytes=343400 combine_payload_bytes=265584640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265928040 ingress_bytes=300187648 link_bytes=300187648 token_bytes=531512680 weight_movement_bytes=34603008 total_nvlink_bytes=566115688 weight_to_token_ratio=0.065103 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129581952e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=64946 dup_rows=24002 dup_rows_remote=20996 dedup_ratio=0.244161 router_tokens_per_expert=[755, 763, 790, 783, 711, 750, 758, 762, 762, 771, 806, 734, 817, 774, 779, 727, 799, 770, 758, 758, 779, 810, 769, 768, 791, 773, 734, 790, 733, 764, 735, 787, 775, 774, 760, 781, 777, 739, 755, 700, 781, 773, 757, 786, 776, 746, 743, 819, 804, 783, 804, 780, 796, 778, 779, 742, 793, 777, 748, 757, 730, 768, 716, 795] payload_rows_per_dst=[9174, 9362, 9388, 9180, 9156, 9356, 9438, 9248] dup_rows_per_dst=[2970, 2978, 3034, 3034, 2966, 3006, 3094, 2920] combine_rows_per_src=[9174, 9362, 9388, 9180, 9156, 9356, 9438, 9248] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266018816 dispatch_meta_bytes=343768 combine_payload_bytes=266018816 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266362584 ingress_bytes=335224832 link_bytes=335224832 token_bytes=532381400 weight_movement_bytes=69206016 total_nvlink_bytes=601587416 weight_to_token_ratio=0.129993 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261424055e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265437184 dispatch_meta_bytes=343016 combine_payload_bytes=265437184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265780200 ingress_bytes=300040192 link_bytes=300040192 token_bytes=531217384 weight_movement_bytes=34603008 total_nvlink_bytes=565820392 weight_to_token_ratio=0.065139 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129027087e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=344208 combine_payload_bytes=266477568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266821776 ingress_bytes=301080576 link_bytes=301080576 token_bytes=533299344 weight_movement_bytes=34603008 total_nvlink_bytes=567902352 weight_to_token_ratio=0.064885 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132941969e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266166272 dispatch_meta_bytes=343728 combine_payload_bytes=266166272 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266510000 ingress_bytes=335372288 link_bytes=335372288 token_bytes=532676272 weight_movement_bytes=69206016 total_nvlink_bytes=601882288 weight_to_token_ratio=0.129921 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261978920e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266878976 dispatch_meta_bytes=343968 combine_payload_bytes=266878976 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267222944 ingress_bytes=336084992 link_bytes=336084992 token_bytes=534101920 weight_movement_bytes=69206016 total_nvlink_bytes=603307936 weight_to_token_ratio=0.129575 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264660768e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265805824 dispatch_meta_bytes=343192 combine_payload_bytes=265805824 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266149016 ingress_bytes=335011840 link_bytes=335011840 token_bytes=531954840 weight_movement_bytes=69206016 total_nvlink_bytes=601160856 weight_to_token_ratio=0.130098 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260622583e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74438 payload_rows_remote=65224 dup_rows=23866 dup_rows_remote=20810 dedup_ratio=0.242778 router_tokens_per_expert=[787, 792, 805, 793, 778, 773, 777, 790, 730, 742, 778, 763, 754, 801, 796, 723, 817, 731, 778, 773, 794, 765, 767, 782, 763, 744, 789, 753, 808, 741, 823, 730, 826, 743, 774, 777, 739, 728, 773, 775, 754, 778, 765, 756, 811, 770, 760, 746, 781, 767, 737, 748, 782, 743, 727, 721, 730, 795, 740, 797, 738, 822, 733, 776] payload_rows_per_dst=[9484, 9278, 9372, 9352, 9214, 9272, 9128, 9338] dup_rows_per_dst=[3106, 2896, 3042, 2950, 3056, 3008, 2884, 2924] combine_rows_per_src=[9484, 9278, 9372, 9352, 9214, 9272, 9128, 9338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=344208 combine_payload_bytes=266477568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266821776 ingress_bytes=301080576 link_bytes=301080576 token_bytes=533299344 weight_movement_bytes=34603008 total_nvlink_bytes=567902352 weight_to_token_ratio=0.064885 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132941969e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265437184 dispatch_meta_bytes=343016 combine_payload_bytes=265437184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265780200 ingress_bytes=300040192 link_bytes=300040192 token_bytes=531217384 weight_movement_bytes=34603008 total_nvlink_bytes=565820392 weight_to_token_ratio=0.065139 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129027087e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266018816 dispatch_meta_bytes=343768 combine_payload_bytes=266018816 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266362584 ingress_bytes=335224832 link_bytes=335224832 token_bytes=532381400 weight_movement_bytes=69206016 total_nvlink_bytes=601587416 weight_to_token_ratio=0.129993 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261424055e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=65038 dup_rows=24046 dup_rows_remote=21098 dedup_ratio=0.244609 router_tokens_per_expert=[749, 708, 738, 762, 742, 774, 763, 808, 772, 778, 729, 776, 794, 713, 760, 762, 709, 793, 779, 788, 797, 796, 721, 758, 799, 755, 773, 772, 769, 731, 769, 748, 711, 784, 776, 826, 714, 731, 814, 801, 808, 801, 755, 796, 805, 753, 728, 775, 802, 762, 800, 821, 752, 773, 821, 779, 753, 785, 752, 754, 764, 775, 777, 719] payload_rows_per_dst=[9214, 9220, 9328, 9228, 9210, 9340, 9484, 9234] dup_rows_per_dst=[2874, 2948, 2954, 3004, 3104, 3102, 3136, 2924] combine_rows_per_src=[9214, 9220, 9328, 9228, 9210, 9340, 9484, 9234] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=65074 dup_rows=23930 dup_rows_remote=20886 dedup_ratio=0.243429 router_tokens_per_expert=[800, 796, 781, 797, 749, 758, 755, 746, 801, 759, 713, 758, 724, 773, 797, 798, 761, 772, 818, 769, 744, 783, 731, 794, 759, 744, 784, 780, 760, 808, 779, 771, 746, 760, 757, 812, 779, 744, 744, 799, 761, 824, 775, 772, 789, 746, 762, 803, 736, 731, 750, 787, 739, 751, 800, 730, 724, 764, 732, 770, 750, 804, 804, 745] payload_rows_per_dst=[9318, 9230, 9300, 9414, 9278, 9416, 9158, 9260] dup_rows_per_dst=[3046, 3016, 3044, 2956, 3004, 3048, 2890, 2926] combine_rows_per_src=[9318, 9230, 9300, 9414, 9278, 9416, 9158, 9260] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74624 payload_rows_remote=65144 dup_rows=23680 dup_rows_remote=20586 dedup_ratio=0.240885 router_tokens_per_expert=[797, 727, 824, 746, 724, 756, 771, 740, 711, 827, 756, 754, 771, 741, 745, 748, 768, 784, 751, 763, 753, 792, 741, 758, 766, 784, 731, 761, 793, 765, 793, 793, 766, 744, 757, 803, 784, 771, 779, 761, 779, 808, 769, 770, 790, 791, 764, 816, 738, 732, 768, 789, 797, 829, 735, 748, 781, 778, 793, 772, 776, 739, 785, 706] payload_rows_per_dst=[9270, 9240, 9366, 9298, 9264, 9480, 9332, 9374] dup_rows_per_dst=[2900, 2866, 2854, 3074, 3066, 3094, 2940, 2886] combine_rows_per_src=[9270, 9240, 9366, 9298, 9264, 9480, 9332, 9374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74140 payload_rows_remote=64732 dup_rows=24164 dup_rows_remote=20986 dedup_ratio=0.245809 router_tokens_per_expert=[739, 796, 752, 788, 761, 812, 742, 790, 826, 783, 756, 783, 772, 804, 748, 706, 797, 791, 793, 751, 793, 787, 725, 741, 762, 762, 738, 734, 776, 756, 754, 738, 765, 792, 754, 743, 820, 715, 785, 793, 760, 751, 765, 774, 776, 779, 760, 742, 789, 759, 791, 756, 746, 755, 753, 792, 727, 775, 797, 794, 782, 740, 795, 771] payload_rows_per_dst=[9248, 9376, 8986, 9116, 9332, 9166, 9508, 9408] dup_rows_per_dst=[2888, 2980, 2806, 2924, 3112, 3048, 3228, 3178] combine_rows_per_src=[9248, 9376, 8986, 9116, 9332, 9166, 9508, 9408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74298 payload_rows_remote=64972 dup_rows=24006 dup_rows_remote=20966 dedup_ratio=0.244202 router_tokens_per_expert=[794, 751, 769, 777, 779, 726, 791, 814, 812, 783, 760, 718, 742, 753, 742, 763, 758, 786, 759, 756, 789, 743, 796, 775, 765, 777, 785, 827, 793, 791, 766, 743, 751, 779, 733, 744, 725, 748, 733, 793, 794, 735, 805, 731, 757, 783, 709, 767, 773, 759, 756, 775, 816, 756, 796, 752, 738, 803, 753, 832, 764, 770, 781, 758] payload_rows_per_dst=[9346, 9138, 9274, 9400, 9172, 9300, 9326, 9342] dup_rows_per_dst=[3056, 3008, 3050, 3094, 2840, 2862, 3040, 3056] combine_rows_per_src=[9346, 9138, 9274, 9400, 9172, 9300, 9326, 9342] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=65038 dup_rows=24046 dup_rows_remote=21098 dedup_ratio=0.244609 router_tokens_per_expert=[749, 708, 738, 762, 742, 774, 763, 808, 772, 778, 729, 776, 794, 713, 760, 762, 709, 793, 779, 788, 797, 796, 721, 758, 799, 755, 773, 772, 769, 731, 769, 748, 711, 784, 776, 826, 714, 731, 814, 801, 808, 801, 755, 796, 805, 753, 728, 775, 802, 762, 800, 821, 752, 773, 821, 779, 753, 785, 752, 754, 764, 775, 777, 719] payload_rows_per_dst=[9214, 9220, 9328, 9228, 9210, 9340, 9484, 9234] dup_rows_per_dst=[2874, 2948, 2954, 3004, 3104, 3102, 3136, 2924] combine_rows_per_src=[9214, 9220, 9328, 9228, 9210, 9340, 9484, 9234] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74282 payload_rows_remote=64840 dup_rows=24022 dup_rows_remote=21010 dedup_ratio=0.244364 router_tokens_per_expert=[763, 829, 753, 768, 761, 800, 795, 749, 745, 770, 781, 782, 784, 770, 759, 790, 769, 764, 764, 727, 739, 798, 775, 758, 747, 761, 765, 777, 783, 776, 766, 771, 787, 825, 766, 776, 775, 757, 741, 802, 749, 784, 756, 756, 767, 724, 853, 728, 804, 756, 747, 773, 761, 808, 736, 735, 777, 781, 763, 748, 734, 707, 777, 760] payload_rows_per_dst=[9442, 9306, 9136, 9300, 9392, 9176, 9318, 9212] dup_rows_per_dst=[3012, 3246, 3034, 2992, 3066, 2868, 2922, 2882] combine_rows_per_src=[9442, 9306, 9136, 9300, 9392, 9176, 9318, 9212] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=65130 dup_rows=24020 dup_rows_remote=21038 dedup_ratio=0.244344 router_tokens_per_expert=[722, 762, 790, 793, 784, 750, 751, 801, 744, 751, 760, 796, 744, 763, 733, 785, 787, 785, 815, 818, 783, 780, 806, 763, 812, 723, 739, 770, 775, 822, 735, 692, 785, 768, 770, 744, 823, 779, 767, 761, 762, 734, 815, 761, 755, 806, 773, 799, 704, 741, 769, 790, 756, 750, 727, 759, 777, 755, 720, 802, 746, 749, 760, 811] payload_rows_per_dst=[9318, 9160, 9432, 9154, 9382, 9320, 9118, 9400] dup_rows_per_dst=[2988, 2992, 3242, 2982, 3012, 3090, 2874, 2840] combine_rows_per_src=[9318, 9160, 9432, 9154, 9382, 9320, 9118, 9400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74222 payload_rows_remote=64980 dup_rows=24082 dup_rows_remote=21076 dedup_ratio=0.244975 router_tokens_per_expert=[737, 793, 757, 753, 757, 735, 749, 788, 741, 743, 809, 785, 758, 770, 738, 750, 762, 779, 754, 800, 744, 717, 797, 758, 802, 768, 805, 762, 776, 778, 764, 777, 814, 757, 774, 728, 741, 787, 765, 763, 748, 808, 763, 796, 824, 800, 780, 762, 727, 780, 755, 797, 758, 764, 762, 786, 773, 770, 746, 722, 771, 789, 760, 776] payload_rows_per_dst=[9242, 9608, 9244, 9374, 9284, 8972, 9268, 9230] dup_rows_per_dst=[3006, 3348, 3010, 2984, 2974, 2786, 2990, 2984] combine_rows_per_src=[9242, 9608, 9244, 9374, 9284, 8972, 9268, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=344544 combine_payload_bytes=266395648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266740192 ingress_bytes=300998656 link_bytes=300998656 token_bytes=533135840 weight_movement_bytes=34603008 total_nvlink_bytes=567738848 weight_to_token_ratio=0.064905 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132633710e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=343840 combine_payload_bytes=266543104 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266886944 ingress_bytes=370352128 link_bytes=370352128 token_bytes=533430048 weight_movement_bytes=103809024 total_nvlink_bytes=637239072 weight_to_token_ratio=0.194607 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393605242e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267157504 dispatch_meta_bytes=344136 combine_payload_bytes=267157504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267501640 ingress_bytes=301760512 link_bytes=301760512 token_bytes=534659144 weight_movement_bytes=34603008 total_nvlink_bytes=569262152 weight_to_token_ratio=0.064720 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135500513e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266829824 dispatch_meta_bytes=342920 combine_payload_bytes=266829824 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267172744 ingress_bytes=370638848 link_bytes=370638848 token_bytes=534002568 weight_movement_bytes=103809024 total_nvlink_bytes=637811592 weight_to_token_ratio=0.194398 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394684146e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265142272 dispatch_meta_bytes=342872 combine_payload_bytes=265142272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265485144 ingress_bytes=299745280 link_bytes=299745280 token_bytes=530627416 weight_movement_bytes=34603008 total_nvlink_bytes=565230424 weight_to_token_ratio=0.065211 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127917357e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74420 payload_rows_remote=65276 dup_rows=23884 dup_rows_remote=20906 dedup_ratio=0.242961 router_tokens_per_expert=[748, 750, 814, 778, 775, 789, 795, 750, 739, 764, 768, 789, 795, 763, 805, 805, 786, 743, 711, 785, 781, 779, 732, 756, 809, 768, 784, 793, 696, 782, 804, 703, 776, 756, 739, 726, 826, 763, 762, 713, 789, 802, 764, 769, 753, 760, 786, 763, 732, 719, 820, 786, 791, 781, 781, 679, 777, 768, 754, 771, 810, 750, 774, 773] payload_rows_per_dst=[9340, 9496, 9194, 9304, 9144, 9314, 9268, 9360] dup_rows_per_dst=[3058, 2960, 2952, 2974, 2978, 3058, 2910, 2994] combine_rows_per_src=[9340, 9496, 9194, 9304, 9144, 9314, 9268, 9360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=65130 dup_rows=24020 dup_rows_remote=21038 dedup_ratio=0.244344 router_tokens_per_expert=[722, 762, 790, 793, 784, 750, 751, 801, 744, 751, 760, 796, 744, 763, 733, 785, 787, 785, 815, 818, 783, 780, 806, 763, 812, 723, 739, 770, 775, 822, 735, 692, 785, 768, 770, 744, 823, 779, 767, 761, 762, 734, 815, 761, 755, 806, 773, 799, 704, 741, 769, 790, 756, 750, 727, 759, 777, 755, 720, 802, 746, 749, 760, 811] payload_rows_per_dst=[9318, 9160, 9432, 9154, 9382, 9320, 9118, 9400] dup_rows_per_dst=[2988, 2992, 3242, 2982, 3012, 3090, 2874, 2840] combine_rows_per_src=[9318, 9160, 9432, 9154, 9382, 9320, 9118, 9400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74298 payload_rows_remote=64972 dup_rows=24006 dup_rows_remote=20966 dedup_ratio=0.244202 router_tokens_per_expert=[794, 751, 769, 777, 779, 726, 791, 814, 812, 783, 760, 718, 742, 753, 742, 763, 758, 786, 759, 756, 789, 743, 796, 775, 765, 777, 785, 827, 793, 791, 766, 743, 751, 779, 733, 744, 725, 748, 733, 793, 794, 735, 805, 731, 757, 783, 709, 767, 773, 759, 756, 775, 816, 756, 796, 752, 738, 803, 753, 832, 764, 770, 781, 758] payload_rows_per_dst=[9346, 9138, 9274, 9400, 9172, 9300, 9326, 9342] dup_rows_per_dst=[3056, 3008, 3050, 3094, 2840, 2862, 3040, 3056] combine_rows_per_src=[9346, 9138, 9274, 9400, 9172, 9300, 9326, 9342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266158080 dispatch_meta_bytes=344224 combine_payload_bytes=266158080 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266502304 ingress_bytes=335364096 link_bytes=335364096 token_bytes=532660384 weight_movement_bytes=69206016 total_nvlink_bytes=601866400 weight_to_token_ratio=0.129925 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261948094e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74140 payload_rows_remote=64732 dup_rows=24164 dup_rows_remote=20986 dedup_ratio=0.245809 router_tokens_per_expert=[739, 796, 752, 788, 761, 812, 742, 790, 826, 783, 756, 783, 772, 804, 748, 706, 797, 791, 793, 751, 793, 787, 725, 741, 762, 762, 738, 734, 776, 756, 754, 738, 765, 792, 754, 743, 820, 715, 785, 793, 760, 751, 765, 774, 776, 779, 760, 742, 789, 759, 791, 756, 746, 755, 753, 792, 727, 775, 797, 794, 782, 740, 795, 771] payload_rows_per_dst=[9248, 9376, 8986, 9116, 9332, 9166, 9508, 9408] dup_rows_per_dst=[2888, 2980, 2806, 2924, 3112, 3048, 3228, 3178] combine_rows_per_src=[9248, 9376, 8986, 9116, 9332, 9166, 9508, 9408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74624 payload_rows_remote=65144 dup_rows=23680 dup_rows_remote=20586 dedup_ratio=0.240885 router_tokens_per_expert=[797, 727, 824, 746, 724, 756, 771, 740, 711, 827, 756, 754, 771, 741, 745, 748, 768, 784, 751, 763, 753, 792, 741, 758, 766, 784, 731, 761, 793, 765, 793, 793, 766, 744, 757, 803, 784, 771, 779, 761, 779, 808, 769, 770, 790, 791, 764, 816, 738, 732, 768, 789, 797, 829, 735, 748, 781, 778, 793, 772, 776, 739, 785, 706] payload_rows_per_dst=[9270, 9240, 9366, 9298, 9264, 9480, 9332, 9374] dup_rows_per_dst=[2900, 2866, 2854, 3074, 3066, 3094, 2940, 2886] combine_rows_per_src=[9270, 9240, 9366, 9298, 9264, 9480, 9332, 9374] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266125312 dispatch_meta_bytes=343752 combine_payload_bytes=266125312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266469064 ingress_bytes=300728320 link_bytes=300728320 token_bytes=532594376 weight_movement_bytes=34603008 total_nvlink_bytes=567197384 weight_to_token_ratio=0.064971 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131616458e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265584640 dispatch_meta_bytes=343400 combine_payload_bytes=265584640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265928040 ingress_bytes=300187648 link_bytes=300187648 token_bytes=531512680 weight_movement_bytes=34603008 total_nvlink_bytes=566115688 weight_to_token_ratio=0.065103 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129581952e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=344544 combine_payload_bytes=266395648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266740192 ingress_bytes=300998656 link_bytes=300998656 token_bytes=533135840 weight_movement_bytes=34603008 total_nvlink_bytes=567738848 weight_to_token_ratio=0.064905 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132633710e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267370496 dispatch_meta_bytes=344728 combine_payload_bytes=267370496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267715224 ingress_bytes=301973504 link_bytes=301973504 token_bytes=535085720 weight_movement_bytes=34603008 total_nvlink_bytes=569688728 weight_to_token_ratio=0.064668 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136301985e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266772480 dispatch_meta_bytes=344672 combine_payload_bytes=266772480 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267117152 ingress_bytes=335978496 link_bytes=335978496 token_bytes=533889632 weight_movement_bytes=69206016 total_nvlink_bytes=603095648 weight_to_token_ratio=0.129626 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264260032e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74386 payload_rows_remote=65044 dup_rows=23918 dup_rows_remote=20904 dedup_ratio=0.243306 router_tokens_per_expert=[746, 750, 745, 770, 811, 752, 794, 757, 680, 811, 780, 788, 748, 781, 715, 749, 741, 745, 745, 724, 762, 781, 781, 809, 726, 766, 752, 767, 786, 735, 748, 772, 789, 800, 779, 758, 777, 735, 764, 776, 783, 788, 739, 795, 785, 756, 774, 761, 797, 766, 786, 793, 777, 793, 764, 786, 784, 781, 754, 814, 777, 769, 786, 749] payload_rows_per_dst=[9270, 9252, 9208, 9268, 9342, 9260, 9422, 9364] dup_rows_per_dst=[2980, 2852, 2968, 2836, 3014, 3102, 3102, 3064] combine_rows_per_src=[9270, 9252, 9208, 9268, 9342, 9260, 9422, 9364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266125312 dispatch_meta_bytes=343752 combine_payload_bytes=266125312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266469064 ingress_bytes=300728320 link_bytes=300728320 token_bytes=532594376 weight_movement_bytes=34603008 total_nvlink_bytes=567197384 weight_to_token_ratio=0.064971 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131616458e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265142272 dispatch_meta_bytes=342872 combine_payload_bytes=265142272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265485144 ingress_bytes=299745280 link_bytes=299745280 token_bytes=530627416 weight_movement_bytes=34603008 total_nvlink_bytes=565230424 weight_to_token_ratio=0.065211 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127917357e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266829824 dispatch_meta_bytes=342920 combine_payload_bytes=266829824 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267172744 ingress_bytes=370638848 link_bytes=370638848 token_bytes=534002568 weight_movement_bytes=103809024 total_nvlink_bytes=637811592 weight_to_token_ratio=0.194398 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394684146e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74464 payload_rows_remote=65212 dup_rows=23840 dup_rows_remote=20792 dedup_ratio=0.242513 router_tokens_per_expert=[804, 751, 820, 790, 745, 764, 763, 757, 775, 759, 760, 782, 767, 782, 779, 746, 813, 754, 741, 770, 752, 785, 741, 744, 751, 785, 737, 798, 780, 763, 793, 754, 753, 751, 749, 760, 745, 715, 807, 772, 747, 785, 770, 752, 740, 752, 751, 768, 798, 781, 770, 758, 764, 753, 812, 707, 852, 800, 778, 748, 761, 795, 790, 763] payload_rows_per_dst=[9396, 9252, 9326, 9324, 9182, 9158, 9420, 9406] dup_rows_per_dst=[2992, 3048, 2874, 2998, 2922, 2972, 2866, 3168] combine_rows_per_src=[9396, 9252, 9326, 9324, 9182, 9158, 9420, 9406] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74528 payload_rows_remote=65226 dup_rows=23776 dup_rows_remote=20872 dedup_ratio=0.241862 router_tokens_per_expert=[801, 731, 792, 783, 791, 756, 743, 802, 738, 793, 702, 746, 763, 815, 781, 810, 776, 780, 748, 791, 776, 755, 733, 744, 744, 809, 769, 783, 742, 768, 756, 787, 730, 784, 795, 755, 791, 764, 722, 777, 763, 745, 762, 786, 724, 783, 770, 782, 752, 853, 783, 772, 759, 800, 773, 740, 757, 741, 747, 774, 794, 747, 764, 755] payload_rows_per_dst=[9384, 9250, 9302, 9304, 9322, 9296, 9370, 9300] dup_rows_per_dst=[3014, 3046, 2904, 3012, 2914, 2934, 3094, 2858] combine_rows_per_src=[9384, 9250, 9302, 9304, 9322, 9296, 9370, 9300] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74450 payload_rows_remote=65178 dup_rows=23854 dup_rows_remote=20826 dedup_ratio=0.242655 router_tokens_per_expert=[723, 762, 763, 777, 813, 756, 791, 802, 745, 721, 759, 816, 747, 760, 803, 793, 734, 786, 792, 832, 796, 727, 778, 739, 752, 747, 785, 783, 757, 769, 795, 745, 767, 776, 792, 738, 748, 756, 783, 760, 762, 762, 777, 787, 738, 783, 771, 770, 759, 742, 737, 739, 737, 752, 783, 771, 763, 778, 757, 751, 809, 795, 790, 771] payload_rows_per_dst=[9330, 9210, 9418, 9308, 9296, 9272, 9190, 9426] dup_rows_per_dst=[3044, 3078, 2950, 2958, 2944, 3028, 2850, 3002] combine_rows_per_src=[9330, 9210, 9418, 9308, 9296, 9272, 9190, 9426] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74150 payload_rows_remote=64864 dup_rows=24154 dup_rows_remote=21072 dedup_ratio=0.245707 router_tokens_per_expert=[751, 766, 779, 797, 790, 819, 813, 757, 769, 766, 769, 753, 740, 751, 728, 768, 772, 764, 777, 758, 712, 780, 799, 797, 755, 779, 724, 806, 748, 720, 734, 758, 782, 754, 773, 758, 765, 742, 791, 778, 761, 749, 735, 727, 756, 775, 764, 768, 765, 863, 793, 784, 772, 764, 810, 750, 767, 761, 791, 783, 743, 764, 787, 778] payload_rows_per_dst=[9126, 9350, 9288, 9170, 9322, 9146, 9462, 9286] dup_rows_per_dst=[2968, 3072, 3030, 2878, 2964, 2956, 3204, 3082] combine_rows_per_src=[9126, 9350, 9288, 9170, 9322, 9146, 9462, 9286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74368 payload_rows_remote=65140 dup_rows=23936 dup_rows_remote=21058 dedup_ratio=0.243490 router_tokens_per_expert=[778, 732, 780, 804, 771, 810, 731, 777, 742, 773, 779, 823, 741, 790, 782, 784, 806, 776, 733, 755, 755, 759, 752, 783, 762, 758, 802, 809, 764, 781, 735, 709, 776, 761, 805, 736, 747, 729, 743, 750, 768, 795, 762, 840, 785, 765, 743, 806, 752, 768, 777, 765, 753, 797, 735, 706, 775, 746, 763, 762, 787, 780, 755, 784] payload_rows_per_dst=[9376, 9428, 9188, 9206, 9114, 9526, 9228, 9302] dup_rows_per_dst=[2990, 3000, 3050, 3034, 2980, 3002, 2878, 3002] combine_rows_per_src=[9376, 9428, 9188, 9206, 9114, 9526, 9228, 9302] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=343792 combine_payload_bytes=266420224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266764016 ingress_bytes=301023232 link_bytes=301023232 token_bytes=533184240 weight_movement_bytes=34603008 total_nvlink_bytes=567787248 weight_to_token_ratio=0.064899 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132726188e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74222 payload_rows_remote=64980 dup_rows=24082 dup_rows_remote=21076 dedup_ratio=0.244975 router_tokens_per_expert=[737, 793, 757, 753, 757, 735, 749, 788, 741, 743, 809, 785, 758, 770, 738, 750, 762, 779, 754, 800, 744, 717, 797, 758, 802, 768, 805, 762, 776, 778, 764, 777, 814, 757, 774, 728, 741, 787, 765, 763, 748, 808, 763, 796, 824, 800, 780, 762, 727, 780, 755, 797, 758, 764, 762, 786, 773, 770, 746, 722, 771, 789, 760, 776] payload_rows_per_dst=[9242, 9608, 9244, 9374, 9284, 8972, 9268, 9230] dup_rows_per_dst=[3006, 3348, 3010, 2984, 2974, 2786, 2990, 2984] combine_rows_per_src=[9242, 9608, 9244, 9374, 9284, 8972, 9268, 9230] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74464 payload_rows_remote=65212 dup_rows=23840 dup_rows_remote=20792 dedup_ratio=0.242513 router_tokens_per_expert=[804, 751, 820, 790, 745, 764, 763, 757, 775, 759, 760, 782, 767, 782, 779, 746, 813, 754, 741, 770, 752, 785, 741, 744, 751, 785, 737, 798, 780, 763, 793, 754, 753, 751, 749, 760, 745, 715, 807, 772, 747, 785, 770, 752, 740, 752, 751, 768, 798, 781, 770, 758, 764, 753, 812, 707, 852, 800, 778, 748, 761, 795, 790, 763] payload_rows_per_dst=[9396, 9252, 9326, 9324, 9182, 9158, 9420, 9406] dup_rows_per_dst=[2992, 3048, 2874, 2998, 2922, 2972, 2866, 3168] combine_rows_per_src=[9396, 9252, 9326, 9324, 9182, 9158, 9420, 9406] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74328 payload_rows_remote=65110 dup_rows=23976 dup_rows_remote=20946 dedup_ratio=0.243896 router_tokens_per_expert=[700, 779, 768, 761, 770, 788, 786, 772, 755, 751, 763, 753, 777, 725, 762, 777, 755, 781, 764, 791, 783, 735, 719, 728, 809, 715, 752, 802, 753, 758, 870, 738, 802, 692, 763, 785, 742, 787, 785, 788, 798, 766, 753, 734, 849, 801, 766, 755, 753, 784, 719, 743, 755, 813, 833, 768, 788, 774, 764, 757, 769, 747, 778, 801] payload_rows_per_dst=[9218, 9252, 9510, 9558, 9376, 9326, 9088, 9000] dup_rows_per_dst=[3030, 2874, 3236, 3092, 3136, 3118, 2768, 2722] combine_rows_per_src=[9218, 9252, 9510, 9558, 9376, 9326, 9088, 9000] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=267108352 dispatch_meta_bytes=344016 combine_payload_bytes=267108352 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267452368 ingress_bytes=301711360 link_bytes=301711360 token_bytes=534560720 weight_movement_bytes=34603008 total_nvlink_bytes=569163728 weight_to_token_ratio=0.064732 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135315558e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=267165696 dispatch_meta_bytes=344392 combine_payload_bytes=267165696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267510088 ingress_bytes=301768704 link_bytes=301768704 token_bytes=534675784 weight_movement_bytes=34603008 total_nvlink_bytes=569278792 weight_to_token_ratio=0.064718 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135531339e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74438 payload_rows_remote=65224 dup_rows=23866 dup_rows_remote=20810 dedup_ratio=0.242778 router_tokens_per_expert=[787, 792, 805, 793, 778, 773, 777, 790, 730, 742, 778, 763, 754, 801, 796, 723, 817, 731, 778, 773, 794, 765, 767, 782, 763, 744, 789, 753, 808, 741, 823, 730, 826, 743, 774, 777, 739, 728, 773, 775, 754, 778, 765, 756, 811, 770, 760, 746, 781, 767, 737, 748, 782, 743, 727, 721, 730, 795, 740, 797, 738, 822, 733, 776] payload_rows_per_dst=[9484, 9278, 9372, 9352, 9214, 9272, 9128, 9338] dup_rows_per_dst=[3106, 2896, 3042, 2950, 3056, 3008, 2884, 2924] combine_rows_per_src=[9484, 9278, 9372, 9352, 9214, 9272, 9128, 9338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74368 payload_rows_remote=65140 dup_rows=23936 dup_rows_remote=21058 dedup_ratio=0.243490 router_tokens_per_expert=[778, 732, 780, 804, 771, 810, 731, 777, 742, 773, 779, 823, 741, 790, 782, 784, 806, 776, 733, 755, 755, 759, 752, 783, 762, 758, 802, 809, 764, 781, 735, 709, 776, 761, 805, 736, 747, 729, 743, 750, 768, 795, 762, 840, 785, 765, 743, 806, 752, 768, 777, 765, 753, 797, 735, 706, 775, 746, 763, 762, 787, 780, 755, 784] payload_rows_per_dst=[9376, 9428, 9188, 9206, 9114, 9526, 9228, 9302] dup_rows_per_dst=[2990, 3000, 3050, 3034, 2980, 3002, 2878, 3002] combine_rows_per_src=[9376, 9428, 9188, 9206, 9114, 9526, 9228, 9302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74036 payload_rows_remote=64756 dup_rows=24268 dup_rows_remote=21190 dedup_ratio=0.246867 router_tokens_per_expert=[748, 753, 737, 779, 782, 728, 809, 757, 747, 773, 779, 761, 780, 790, 765, 775, 748, 752, 741, 765, 770, 768, 741, 741, 780, 747, 758, 772, 778, 774, 786, 784, 799, 789, 796, 770, 748, 776, 772, 788, 775, 707, 802, 767, 776, 789, 745, 743, 809, 764, 781, 770, 705, 769, 790, 793, 773, 781, 775, 731, 797, 769, 757, 778] payload_rows_per_dst=[9228, 9326, 9084, 9280, 9380, 9222, 9242, 9274] dup_rows_per_dst=[2958, 3014, 2968, 3078, 3096, 2986, 3120, 3048] combine_rows_per_src=[9228, 9326, 9084, 9280, 9380, 9222, 9242, 9274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74150 payload_rows_remote=64864 dup_rows=24154 dup_rows_remote=21072 dedup_ratio=0.245707 router_tokens_per_expert=[751, 766, 779, 797, 790, 819, 813, 757, 769, 766, 769, 753, 740, 751, 728, 768, 772, 764, 777, 758, 712, 780, 799, 797, 755, 779, 724, 806, 748, 720, 734, 758, 782, 754, 773, 758, 765, 742, 791, 778, 761, 749, 735, 727, 756, 775, 764, 768, 765, 863, 793, 784, 772, 764, 810, 750, 767, 761, 791, 783, 743, 764, 787, 778] payload_rows_per_dst=[9126, 9350, 9288, 9170, 9322, 9146, 9462, 9286] dup_rows_per_dst=[2968, 3072, 3030, 2878, 2964, 2956, 3204, 3082] combine_rows_per_src=[9126, 9350, 9288, 9170, 9322, 9146, 9462, 9286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74450 payload_rows_remote=65178 dup_rows=23854 dup_rows_remote=20826 dedup_ratio=0.242655 router_tokens_per_expert=[723, 762, 763, 777, 813, 756, 791, 802, 745, 721, 759, 816, 747, 760, 803, 793, 734, 786, 792, 832, 796, 727, 778, 739, 752, 747, 785, 783, 757, 769, 795, 745, 767, 776, 792, 738, 748, 756, 783, 760, 762, 762, 777, 787, 738, 783, 771, 770, 759, 742, 737, 739, 737, 752, 783, 771, 763, 778, 757, 751, 809, 795, 790, 771] payload_rows_per_dst=[9330, 9210, 9418, 9308, 9296, 9272, 9190, 9426] dup_rows_per_dst=[3044, 3078, 2950, 2958, 2944, 3028, 2850, 3002] combine_rows_per_src=[9330, 9210, 9418, 9308, 9296, 9272, 9190, 9426] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266969088 dispatch_meta_bytes=344016 combine_payload_bytes=266969088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267313104 ingress_bytes=301572096 link_bytes=301572096 token_bytes=534282192 weight_movement_bytes=34603008 total_nvlink_bytes=568885200 weight_to_token_ratio=0.064765 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134791519e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265682944 dispatch_meta_bytes=343744 combine_payload_bytes=265682944 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266026688 ingress_bytes=334888960 link_bytes=334888960 token_bytes=531709632 weight_movement_bytes=69206016 total_nvlink_bytes=600915648 weight_to_token_ratio=0.130158 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260160196e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266690560 dispatch_meta_bytes=344224 combine_payload_bytes=266690560 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267034784 ingress_bytes=405102592 link_bytes=405102592 token_bytes=533725344 weight_movement_bytes=138412032 total_nvlink_bytes=672137376 weight_to_token_ratio=0.259332 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524368440e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267157504 dispatch_meta_bytes=344136 combine_payload_bytes=267157504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267501640 ingress_bytes=301760512 link_bytes=301760512 token_bytes=534659144 weight_movement_bytes=34603008 total_nvlink_bytes=569262152 weight_to_token_ratio=0.064720 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135500513e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266813440 dispatch_meta_bytes=344792 combine_payload_bytes=266813440 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267158232 ingress_bytes=370622464 link_bytes=370622464 token_bytes=533971672 weight_movement_bytes=103809024 total_nvlink_bytes=637780696 weight_to_token_ratio=0.194409 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394622494e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=267108352 dispatch_meta_bytes=344016 combine_payload_bytes=267108352 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267452368 ingress_bytes=301711360 link_bytes=301711360 token_bytes=534560720 weight_movement_bytes=34603008 total_nvlink_bytes=569163728 weight_to_token_ratio=0.064732 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135315558e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266158080 dispatch_meta_bytes=344224 combine_payload_bytes=266158080 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266502304 ingress_bytes=335364096 link_bytes=335364096 token_bytes=532660384 weight_movement_bytes=69206016 total_nvlink_bytes=601866400 weight_to_token_ratio=0.129925 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261948094e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265240576 dispatch_meta_bytes=343784 combine_payload_bytes=265240576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265584360 ingress_bytes=299843584 link_bytes=299843584 token_bytes=530824936 weight_movement_bytes=34603008 total_nvlink_bytes=565427944 weight_to_token_ratio=0.065187 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128287267e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74290 payload_rows_remote=65030 dup_rows=24014 dup_rows_remote=21036 dedup_ratio=0.244283 router_tokens_per_expert=[717, 746, 784, 770, 768, 781, 757, 763, 793, 742, 735, 717, 712, 746, 786, 772, 746, 764, 826, 794, 714, 758, 767, 731, 793, 772, 745, 786, 791, 807, 772, 810, 766, 721, 780, 816, 802, 771, 746, 717, 799, 781, 799, 738, 775, 772, 767, 792, 760, 763, 796, 789, 805, 783, 807, 697, 773, 749, 741, 784, 781, 794, 767, 756] payload_rows_per_dst=[9248, 9146, 9272, 9306, 9260, 9408, 9320, 9330] dup_rows_per_dst=[2924, 2860, 2928, 3246, 2978, 3038, 3080, 2960] combine_rows_per_src=[9248, 9146, 9272, 9306, 9260, 9408, 9320, 9330] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265682944 dispatch_meta_bytes=343744 combine_payload_bytes=265682944 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266026688 ingress_bytes=334888960 link_bytes=334888960 token_bytes=531709632 weight_movement_bytes=69206016 total_nvlink_bytes=600915648 weight_to_token_ratio=0.130158 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260160196e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266969088 dispatch_meta_bytes=344016 combine_payload_bytes=266969088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267313104 ingress_bytes=301572096 link_bytes=301572096 token_bytes=534282192 weight_movement_bytes=34603008 total_nvlink_bytes=568885200 weight_to_token_ratio=0.064765 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134791519e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74270 payload_rows_remote=65126 dup_rows=24034 dup_rows_remote=21052 dedup_ratio=0.244486 router_tokens_per_expert=[778, 795, 824, 806, 765, 763, 761, 740, 765, 786, 765, 785, 726, 768, 724, 744, 772, 767, 810, 800, 757, 766, 806, 761, 727, 795, 761, 758, 712, 796, 825, 747, 765, 766, 772, 767, 763, 770, 763, 726, 708, 751, 821, 782, 772, 804, 761, 792, 718, 720, 714, 788, 810, 783, 730, 775, 766, 805, 767, 807, 762, 717, 766, 786] payload_rows_per_dst=[9392, 9144, 9312, 9268, 9240, 9372, 9168, 9374] dup_rows_per_dst=[3072, 2982, 3166, 2974, 2944, 3010, 2908, 2978] combine_rows_per_src=[9392, 9144, 9312, 9268, 9240, 9372, 9168, 9374] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74142 payload_rows_remote=64986 dup_rows=24162 dup_rows_remote=21220 dedup_ratio=0.245789 router_tokens_per_expert=[801, 785, 789, 779, 756, 784, 777, 750, 772, 762, 743, 799, 715, 767, 755, 756, 707, 766, 748, 724, 798, 768, 740, 798, 749, 781, 764, 786, 787, 744, 738, 776, 767, 762, 772, 798, 806, 794, 741, 772, 813, 743, 772, 788, 776, 745, 799, 783, 748, 760, 687, 798, 798, 742, 794, 707, 751, 782, 800, 821, 781, 740, 794, 754] payload_rows_per_dst=[9346, 9226, 9156, 9270, 9270, 9326, 9144, 9404] dup_rows_per_dst=[3096, 2912, 2942, 2980, 3154, 3112, 2924, 3042] combine_rows_per_src=[9346, 9226, 9156, 9270, 9270, 9326, 9144, 9404] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74298 payload_rows_remote=65058 dup_rows=24006 dup_rows_remote=21026 dedup_ratio=0.244202 router_tokens_per_expert=[797, 804, 773, 822, 788, 801, 771, 769, 713, 776, 758, 706, 820, 747, 770, 754, 773, 756, 754, 741, 760, 763, 827, 727, 740, 788, 752, 760, 823, 781, 776, 735, 784, 753, 761, 780, 765, 797, 746, 793, 730, 769, 730, 799, 771, 771, 789, 751, 740, 837, 766, 742, 728, 771, 748, 798, 788, 733, 722, 783, 813, 753, 758, 758] payload_rows_per_dst=[9490, 9248, 9290, 9300, 9228, 9240, 9282, 9220] dup_rows_per_dst=[3160, 2840, 2912, 3010, 3130, 2980, 2978, 2996] combine_rows_per_src=[9490, 9248, 9290, 9300, 9228, 9240, 9282, 9220] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344264 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266707144 ingress_bytes=300965888 link_bytes=300965888 token_bytes=533070024 weight_movement_bytes=34603008 total_nvlink_bytes=567673032 weight_to_token_ratio=0.064913 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132510407e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74166 payload_rows_remote=64882 dup_rows=24138 dup_rows_remote=21086 dedup_ratio=0.245544 router_tokens_per_expert=[742, 756, 762, 775, 795, 709, 783, 780, 782, 759, 753, 761, 755, 761, 742, 770, 772, 770, 797, 746, 816, 737, 746, 771, 786, 791, 801, 776, 729, 804, 739, 787, 800, 752, 780, 732, 727, 814, 742, 719, 793, 779, 785, 745, 755, 777, 713, 801, 818, 777, 729, 776, 813, 806, 740, 796, 779, 693, 788, 732, 789, 771, 793, 785] payload_rows_per_dst=[8830, 9610, 9244, 9360, 9226, 9254, 9358, 9284] dup_rows_per_dst=[2452, 3478, 2902, 2990, 3070, 3042, 3152, 3052] combine_rows_per_src=[8830, 9610, 9244, 9360, 9226, 9254, 9358, 9284] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74074 payload_rows_remote=64696 dup_rows=24230 dup_rows_remote=21168 dedup_ratio=0.246480 router_tokens_per_expert=[739, 762, 812, 797, 759, 777, 724, 742, 791, 760, 813, 741, 736, 782, 758, 787, 753, 778, 741, 774, 778, 766, 773, 773, 794, 781, 788, 776, 764, 759, 778, 764, 748, 783, 747, 750, 772, 727, 790, 715, 724, 777, 767, 785, 739, 767, 790, 786, 771, 791, 751, 760, 811, 795, 781, 760, 788, 738, 773, 747, 772, 784, 772, 771] payload_rows_per_dst=[9260, 9246, 9234, 9372, 9136, 9260, 9378, 9188] dup_rows_per_dst=[2964, 3090, 3038, 3036, 2928, 3010, 3062, 3102] combine_rows_per_src=[9260, 9246, 9234, 9372, 9136, 9260, 9378, 9188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74270 payload_rows_remote=65126 dup_rows=24034 dup_rows_remote=21052 dedup_ratio=0.244486 router_tokens_per_expert=[778, 795, 824, 806, 765, 763, 761, 740, 765, 786, 765, 785, 726, 768, 724, 744, 772, 767, 810, 800, 757, 766, 806, 761, 727, 795, 761, 758, 712, 796, 825, 747, 765, 766, 772, 767, 763, 770, 763, 726, 708, 751, 821, 782, 772, 804, 761, 792, 718, 720, 714, 788, 810, 783, 730, 775, 766, 805, 767, 807, 762, 717, 766, 786] payload_rows_per_dst=[9392, 9144, 9312, 9268, 9240, 9372, 9168, 9374] dup_rows_per_dst=[3072, 2982, 3166, 2974, 2944, 3010, 2908, 2978] combine_rows_per_src=[9392, 9144, 9312, 9268, 9240, 9372, 9168, 9374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266756096 dispatch_meta_bytes=344712 combine_payload_bytes=266756096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267100808 ingress_bytes=301359104 link_bytes=301359104 token_bytes=533856904 weight_movement_bytes=34603008 total_nvlink_bytes=568459912 weight_to_token_ratio=0.064817 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133990047e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74230 payload_rows_remote=64676 dup_rows=24074 dup_rows_remote=20730 dedup_ratio=0.244893 router_tokens_per_expert=[823, 756, 755, 777, 754, 828, 779, 745, 772, 747, 788, 748, 797, 783, 750, 767, 784, 761, 795, 757, 764, 766, 781, 780, 750, 735, 740, 761, 733, 757, 755, 760, 769, 743, 764, 769, 753, 784, 799, 777, 745, 755, 729, 772, 780, 748, 773, 757, 769, 803, 740, 746, 789, 748, 789, 786, 727, 784, 823, 798, 771, 814, 743, 757] payload_rows_per_dst=[9554, 9342, 9360, 9076, 9244, 9262, 9174, 9218] dup_rows_per_dst=[3344, 2962, 3044, 2924, 3072, 2810, 2918, 3000] combine_rows_per_src=[9554, 9342, 9360, 9076, 9244, 9262, 9174, 9218] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266182656 dispatch_meta_bytes=344824 combine_payload_bytes=266182656 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266527480 ingress_bytes=335388672 link_bytes=335388672 token_bytes=532710136 weight_movement_bytes=69206016 total_nvlink_bytes=601916152 weight_to_token_ratio=0.129913 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262040572e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74386 payload_rows_remote=65044 dup_rows=23918 dup_rows_remote=20904 dedup_ratio=0.243306 router_tokens_per_expert=[746, 750, 745, 770, 811, 752, 794, 757, 680, 811, 780, 788, 748, 781, 715, 749, 741, 745, 745, 724, 762, 781, 781, 809, 726, 766, 752, 767, 786, 735, 748, 772, 789, 800, 779, 758, 777, 735, 764, 776, 783, 788, 739, 795, 785, 756, 774, 761, 797, 766, 786, 793, 777, 793, 764, 786, 784, 781, 754, 814, 777, 769, 786, 749] payload_rows_per_dst=[9270, 9252, 9208, 9268, 9342, 9260, 9422, 9364] dup_rows_per_dst=[2980, 2852, 2968, 2836, 3014, 3102, 3102, 3064] combine_rows_per_src=[9270, 9252, 9208, 9268, 9342, 9260, 9422, 9364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74074 payload_rows_remote=64696 dup_rows=24230 dup_rows_remote=21168 dedup_ratio=0.246480 router_tokens_per_expert=[739, 762, 812, 797, 759, 777, 724, 742, 791, 760, 813, 741, 736, 782, 758, 787, 753, 778, 741, 774, 778, 766, 773, 773, 794, 781, 788, 776, 764, 759, 778, 764, 748, 783, 747, 750, 772, 727, 790, 715, 724, 777, 767, 785, 739, 767, 790, 786, 771, 791, 751, 760, 811, 795, 781, 760, 788, 738, 773, 747, 772, 784, 772, 771] payload_rows_per_dst=[9260, 9246, 9234, 9372, 9136, 9260, 9378, 9188] dup_rows_per_dst=[2964, 3090, 3038, 3036, 2928, 3010, 3062, 3102] combine_rows_per_src=[9260, 9246, 9234, 9372, 9136, 9260, 9378, 9188] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=65162 dup_rows=23930 dup_rows_remote=20950 dedup_ratio=0.243429 router_tokens_per_expert=[720, 747, 743, 772, 790, 746, 768, 800, 735, 777, 758, 780, 803, 719, 721, 816, 762, 762, 793, 806, 790, 781, 785, 773, 752, 793, 778, 719, 775, 780, 762, 737, 795, 778, 761, 784, 755, 803, 772, 764, 765, 767, 745, 743, 762, 791, 746, 771, 823, 747, 765, 829, 782, 740, 787, 756, 754, 744, 767, 792, 769, 773, 737, 742] payload_rows_per_dst=[9258, 9230, 9448, 9212, 9366, 9266, 9366, 9228] dup_rows_per_dst=[2914, 2988, 3056, 2980, 3058, 2914, 3092, 2928] combine_rows_per_src=[9258, 9230, 9448, 9212, 9366, 9266, 9366, 9228] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74328 payload_rows_remote=65110 dup_rows=23976 dup_rows_remote=20946 dedup_ratio=0.243896 router_tokens_per_expert=[700, 779, 768, 761, 770, 788, 786, 772, 755, 751, 763, 753, 777, 725, 762, 777, 755, 781, 764, 791, 783, 735, 719, 728, 809, 715, 752, 802, 753, 758, 870, 738, 802, 692, 763, 785, 742, 787, 785, 788, 798, 766, 753, 734, 849, 801, 766, 755, 753, 784, 719, 743, 755, 813, 833, 768, 788, 774, 764, 757, 769, 747, 778, 801] payload_rows_per_dst=[9218, 9252, 9510, 9558, 9376, 9326, 9088, 9000] dup_rows_per_dst=[3030, 2874, 3236, 3092, 3136, 3118, 2768, 2722] combine_rows_per_src=[9218, 9252, 9510, 9558, 9376, 9326, 9088, 9000] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74166 payload_rows_remote=64882 dup_rows=24138 dup_rows_remote=21086 dedup_ratio=0.245544 router_tokens_per_expert=[742, 756, 762, 775, 795, 709, 783, 780, 782, 759, 753, 761, 755, 761, 742, 770, 772, 770, 797, 746, 816, 737, 746, 771, 786, 791, 801, 776, 729, 804, 739, 787, 800, 752, 780, 732, 727, 814, 742, 719, 793, 779, 785, 745, 755, 777, 713, 801, 818, 777, 729, 776, 813, 806, 740, 796, 779, 693, 788, 732, 789, 771, 793, 785] payload_rows_per_dst=[8830, 9610, 9244, 9360, 9226, 9254, 9358, 9284] dup_rows_per_dst=[2452, 3478, 2902, 2990, 3070, 3042, 3152, 3052] combine_rows_per_src=[8830, 9610, 9244, 9360, 9226, 9254, 9358, 9284] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=344336 combine_payload_bytes=266477568 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266821904 ingress_bytes=335683584 link_bytes=335683584 token_bytes=533299472 weight_movement_bytes=69206016 total_nvlink_bytes=602505488 weight_to_token_ratio=0.129770 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263150302e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=343872 combine_payload_bytes=265756672 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266100544 ingress_bytes=334962688 link_bytes=334962688 token_bytes=531857216 weight_movement_bytes=69206016 total_nvlink_bytes=601063232 weight_to_token_ratio=0.130121 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260437628e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74298 payload_rows_remote=65058 dup_rows=24006 dup_rows_remote=21026 dedup_ratio=0.244202 router_tokens_per_expert=[797, 804, 773, 822, 788, 801, 771, 769, 713, 776, 758, 706, 820, 747, 770, 754, 773, 756, 754, 741, 760, 763, 827, 727, 740, 788, 752, 760, 823, 781, 776, 735, 784, 753, 761, 780, 765, 797, 746, 793, 730, 769, 730, 799, 771, 771, 789, 751, 740, 837, 766, 742, 728, 771, 748, 798, 788, 733, 722, 783, 813, 753, 758, 758] payload_rows_per_dst=[9490, 9248, 9290, 9300, 9228, 9240, 9282, 9220] dup_rows_per_dst=[3160, 2840, 2912, 3010, 3130, 2980, 2978, 2996] combine_rows_per_src=[9490, 9248, 9290, 9300, 9228, 9240, 9282, 9220] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=264912896 dispatch_meta_bytes=341624 combine_payload_bytes=264912896 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265254520 ingress_bytes=334118912 link_bytes=334118912 token_bytes=530167416 weight_movement_bytes=69206016 total_nvlink_bytes=599373432 weight_to_token_ratio=0.130536 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257262567e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=343792 combine_payload_bytes=266420224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266764016 ingress_bytes=301023232 link_bytes=301023232 token_bytes=533184240 weight_movement_bytes=34603008 total_nvlink_bytes=567787248 weight_to_token_ratio=0.064899 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132726188e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=264994816 dispatch_meta_bytes=343456 combine_payload_bytes=264994816 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265338272 ingress_bytes=334200832 link_bytes=334200832 token_bytes=530333088 weight_movement_bytes=69206016 total_nvlink_bytes=599539104 weight_to_token_ratio=0.130495 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257570825e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266756096 dispatch_meta_bytes=344712 combine_payload_bytes=266756096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267100808 ingress_bytes=301359104 link_bytes=301359104 token_bytes=533856904 weight_movement_bytes=34603008 total_nvlink_bytes=568459912 weight_to_token_ratio=0.064817 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133990047e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266903552 dispatch_meta_bytes=344448 combine_payload_bytes=266903552 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267248000 ingress_bytes=336109568 link_bytes=336109568 token_bytes=534151552 weight_movement_bytes=69206016 total_nvlink_bytes=603357568 weight_to_token_ratio=0.129563 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264753245e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266690560 dispatch_meta_bytes=344224 combine_payload_bytes=266690560 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267034784 ingress_bytes=405102592 link_bytes=405102592 token_bytes=533725344 weight_movement_bytes=138412032 total_nvlink_bytes=672137376 weight_to_token_ratio=0.259332 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524368440e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=65274 dup_rows=23860 dup_rows_remote=20956 dedup_ratio=0.242716 router_tokens_per_expert=[780, 741, 800, 728, 777, 760, 763, 782, 772, 734, 763, 784, 769, 745, 785, 755, 799, 739, 807, 776, 791, 759, 815, 755, 732, 787, 711, 730, 816, 763, 781, 783, 740, 765, 757, 730, 752, 738, 775, 780, 764, 779, 782, 797, 770, 773, 752, 771, 727, 780, 761, 745, 775, 767, 770, 761, 783, 769, 756, 796, 817, 791, 773, 774] payload_rows_per_dst=[9272, 9272, 9470, 9350, 9170, 9358, 9224, 9328] dup_rows_per_dst=[2990, 2942, 3012, 2856, 2904, 3018, 2948, 3190] combine_rows_per_src=[9272, 9272, 9470, 9350, 9170, 9358, 9224, 9328] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=343872 combine_payload_bytes=265756672 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266100544 ingress_bytes=334962688 link_bytes=334962688 token_bytes=531857216 weight_movement_bytes=69206016 total_nvlink_bytes=601063232 weight_to_token_ratio=0.130121 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260437628e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74402 payload_rows_remote=64938 dup_rows=23902 dup_rows_remote=20844 dedup_ratio=0.243144 router_tokens_per_expert=[761, 790, 717, 751, 762, 779, 745, 812, 781, 776, 816, 808, 720, 809, 775, 776, 746, 715, 740, 757, 772, 768, 770, 782, 766, 792, 808, 820, 711, 788, 779, 763, 762, 720, 773, 763, 717, 776, 751, 773, 791, 838, 763, 762, 734, 742, 784, 780, 742, 793, 818, 717, 799, 781, 766, 744, 779, 768, 744, 740, 760, 785, 786, 746] payload_rows_per_dst=[9198, 9464, 9150, 9388, 9212, 9350, 9360, 9280] dup_rows_per_dst=[3036, 3058, 2950, 3066, 2858, 3038, 2960, 2936] combine_rows_per_src=[9198, 9464, 9150, 9388, 9212, 9350, 9360, 9280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=344336 combine_payload_bytes=266477568 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266821904 ingress_bytes=335683584 link_bytes=335683584 token_bytes=533299472 weight_movement_bytes=69206016 total_nvlink_bytes=602505488 weight_to_token_ratio=0.129770 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263150302e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74496 payload_rows_remote=65208 dup_rows=23808 dup_rows_remote=20784 dedup_ratio=0.242188 router_tokens_per_expert=[784, 728, 756, 779, 762, 736, 775, 758, 733, 832, 771, 745, 744, 772, 746, 814, 770, 733, 765, 814, 796, 734, 788, 756, 779, 742, 729, 734, 800, 799, 781, 762, 790, 771, 749, 746, 741, 781, 791, 722, 798, 772, 738, 762, 806, 781, 750, 813, 737, 763, 763, 765, 777, 790, 744, 778, 817, 791, 803, 743, 749, 725, 782, 797] payload_rows_per_dst=[9222, 9326, 9288, 9316, 9258, 9322, 9332, 9432] dup_rows_per_dst=[2934, 2988, 3024, 2936, 2924, 3118, 2902, 2982] combine_rows_per_src=[9222, 9326, 9288, 9316, 9258, 9322, 9332, 9432] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=267362304 dispatch_meta_bytes=344920 combine_payload_bytes=267362304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267707224 ingress_bytes=301965312 link_bytes=301965312 token_bytes=535069528 weight_movement_bytes=34603008 total_nvlink_bytes=569672536 weight_to_token_ratio=0.064670 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136271159e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74516 payload_rows_remote=65250 dup_rows=23788 dup_rows_remote=20830 dedup_ratio=0.241984 router_tokens_per_expert=[787, 780, 811, 750, 768, 723, 801, 789, 735, 760, 786, 760, 754, 774, 746, 753, 756, 732, 760, 754, 772, 754, 745, 762, 802, 786, 760, 797, 777, 739, 734, 811, 761, 741, 758, 729, 765, 795, 794, 794, 808, 764, 790, 753, 748, 754, 753, 742, 774, 760, 713, 763, 802, 761, 816, 790, 787, 740, 806, 777, 828, 809, 710, 749] payload_rows_per_dst=[9376, 9230, 9198, 9376, 9324, 9266, 9344, 9402] dup_rows_per_dst=[3042, 2906, 2872, 3036, 2950, 2958, 3014, 3010] combine_rows_per_src=[9376, 9230, 9198, 9376, 9324, 9266, 9344, 9402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=64956 dup_rows=24012 dup_rows_remote=21078 dedup_ratio=0.244263 router_tokens_per_expert=[769, 760, 787, 796, 780, 756, 775, 825, 761, 752, 827, 773, 736, 756, 760, 779, 755, 771, 766, 748, 768, 758, 755, 762, 760, 804, 765, 750, 783, 736, 839, 771, 794, 789, 711, 782, 753, 764, 747, 750, 783, 751, 790, 766, 764, 806, 757, 757, 830, 745, 767, 758, 727, 729, 763, 751, 763, 750, 775, 780, 766, 803, 752, 746] payload_rows_per_dst=[9232, 8798, 9210, 9346, 9696, 9342, 9332, 9336] dup_rows_per_dst=[2914, 2624, 2956, 3036, 3350, 3040, 3158, 2934] combine_rows_per_src=[9232, 8798, 9210, 9346, 9696, 9342, 9332, 9336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265986048 dispatch_meta_bytes=343128 combine_payload_bytes=265986048 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266329176 ingress_bytes=335192064 link_bytes=335192064 token_bytes=532315224 weight_movement_bytes=69206016 total_nvlink_bytes=601521240 weight_to_token_ratio=0.130009 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261300752e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74318 payload_rows_remote=65072 dup_rows=23986 dup_rows_remote=21006 dedup_ratio=0.243998 router_tokens_per_expert=[795, 765, 780, 754, 724, 760, 758, 762, 746, 758, 765, 769, 800, 804, 762, 805, 815, 762, 825, 772, 748, 809, 764, 765, 777, 721, 771, 748, 746, 744, 774, 776, 748, 776, 771, 705, 805, 801, 808, 775, 821, 757, 733, 744, 738, 783, 778, 716, 707, 740, 722, 726, 764, 751, 822, 764, 786, 804, 802, 767, 807, 787, 755, 765] payload_rows_per_dst=[9246, 9454, 9502, 9246, 9318, 9138, 9090, 9324] dup_rows_per_dst=[2980, 3114, 3282, 2870, 3060, 2852, 2902, 2926] combine_rows_per_src=[9246, 9454, 9502, 9246, 9318, 9138, 9090, 9324] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74290 payload_rows_remote=65030 dup_rows=24014 dup_rows_remote=21036 dedup_ratio=0.244283 router_tokens_per_expert=[717, 746, 784, 770, 768, 781, 757, 763, 793, 742, 735, 717, 712, 746, 786, 772, 746, 764, 826, 794, 714, 758, 767, 731, 793, 772, 745, 786, 791, 807, 772, 810, 766, 721, 780, 816, 802, 771, 746, 717, 799, 781, 799, 738, 775, 772, 767, 792, 760, 763, 796, 789, 805, 783, 807, 697, 773, 749, 741, 784, 781, 794, 767, 756] payload_rows_per_dst=[9248, 9146, 9272, 9306, 9260, 9408, 9320, 9330] dup_rows_per_dst=[2924, 2860, 2928, 3246, 2978, 3038, 3080, 2960] combine_rows_per_src=[9248, 9146, 9272, 9306, 9260, 9408, 9320, 9330] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=65014 dup_rows=23948 dup_rows_remote=20934 dedup_ratio=0.243612 router_tokens_per_expert=[763, 756, 795, 777, 763, 812, 797, 776, 852, 783, 808, 799, 803, 809, 761, 788, 737, 799, 762, 753, 736, 764, 797, 746, 746, 760, 752, 762, 708, 801, 800, 802, 742, 764, 727, 770, 745, 710, 775, 779, 826, 778, 742, 776, 712, 770, 770, 747, 808, 742, 783, 802, 752, 765, 733, 793, 743, 750, 755, 792, 710, 711, 747, 766] payload_rows_per_dst=[9328, 9594, 9212, 9292, 9256, 9294, 9342, 9038] dup_rows_per_dst=[3150, 3212, 2976, 2970, 2768, 2948, 3014, 2910] combine_rows_per_src=[9328, 9594, 9212, 9292, 9256, 9294, 9342, 9038] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74402 payload_rows_remote=64938 dup_rows=23902 dup_rows_remote=20844 dedup_ratio=0.243144 router_tokens_per_expert=[761, 790, 717, 751, 762, 779, 745, 812, 781, 776, 816, 808, 720, 809, 775, 776, 746, 715, 740, 757, 772, 768, 770, 782, 766, 792, 808, 820, 711, 788, 779, 763, 762, 720, 773, 763, 717, 776, 751, 773, 791, 838, 763, 762, 734, 742, 784, 780, 742, 793, 818, 717, 799, 781, 766, 744, 779, 768, 744, 740, 760, 785, 786, 746] payload_rows_per_dst=[9198, 9464, 9150, 9388, 9212, 9350, 9360, 9280] dup_rows_per_dst=[3036, 3058, 2950, 3066, 2858, 3038, 2960, 2936] combine_rows_per_src=[9198, 9464, 9150, 9388, 9212, 9350, 9360, 9280] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=267091968 dispatch_meta_bytes=343968 combine_payload_bytes=267091968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267435936 ingress_bytes=301694976 link_bytes=301694976 token_bytes=534527904 weight_movement_bytes=34603008 total_nvlink_bytes=569130912 weight_to_token_ratio=0.064736 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135253906e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74482 payload_rows_remote=65148 dup_rows=23822 dup_rows_remote=20936 dedup_ratio=0.242330 router_tokens_per_expert=[740, 771, 745, 787, 747, 793, 790, 819, 782, 804, 756, 733, 757, 787, 757, 785, 718, 776, 769, 768, 745, 791, 777, 757, 797, 772, 781, 762, 772, 724, 728, 774, 772, 747, 787, 775, 745, 758, 751, 793, 752, 726, 797, 777, 773, 746, 800, 760, 765, 753, 854, 766, 740, 758, 783, 772, 777, 789, 784, 783, 773, 725, 765, 742] payload_rows_per_dst=[9378, 9294, 9234, 9334, 9332, 9278, 9356, 9276] dup_rows_per_dst=[3006, 3028, 2968, 2886, 2924, 2984, 3026, 3000] combine_rows_per_src=[9378, 9294, 9234, 9334, 9332, 9278, 9356, 9276] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74230 payload_rows_remote=64676 dup_rows=24074 dup_rows_remote=20730 dedup_ratio=0.244893 router_tokens_per_expert=[823, 756, 755, 777, 754, 828, 779, 745, 772, 747, 788, 748, 797, 783, 750, 767, 784, 761, 795, 757, 764, 766, 781, 780, 750, 735, 740, 761, 733, 757, 755, 760, 769, 743, 764, 769, 753, 784, 799, 777, 745, 755, 729, 772, 780, 748, 773, 757, 769, 803, 740, 746, 789, 748, 789, 786, 727, 784, 823, 798, 771, 814, 743, 757] payload_rows_per_dst=[9554, 9342, 9360, 9076, 9244, 9262, 9174, 9218] dup_rows_per_dst=[3344, 2962, 3044, 2924, 3072, 2810, 2918, 3000] combine_rows_per_src=[9554, 9342, 9360, 9076, 9244, 9262, 9174, 9218] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=267264000 dispatch_meta_bytes=344320 combine_payload_bytes=267264000 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267608320 ingress_bytes=336470016 link_bytes=336470016 token_bytes=534872320 weight_movement_bytes=69206016 total_nvlink_bytes=604078336 weight_to_token_ratio=0.129388 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266109582e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=64956 dup_rows=24012 dup_rows_remote=21078 dedup_ratio=0.244263 router_tokens_per_expert=[769, 760, 787, 796, 780, 756, 775, 825, 761, 752, 827, 773, 736, 756, 760, 779, 755, 771, 766, 748, 768, 758, 755, 762, 760, 804, 765, 750, 783, 736, 839, 771, 794, 789, 711, 782, 753, 764, 747, 750, 783, 751, 790, 766, 764, 806, 757, 757, 830, 745, 767, 758, 727, 729, 763, 751, 763, 750, 775, 780, 766, 803, 752, 746] payload_rows_per_dst=[9232, 8798, 9210, 9346, 9696, 9342, 9332, 9336] dup_rows_per_dst=[2914, 2624, 2956, 3036, 3350, 3040, 3158, 2934] combine_rows_per_src=[9232, 8798, 9210, 9346, 9696, 9342, 9332, 9336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74516 payload_rows_remote=65250 dup_rows=23788 dup_rows_remote=20830 dedup_ratio=0.241984 router_tokens_per_expert=[787, 780, 811, 750, 768, 723, 801, 789, 735, 760, 786, 760, 754, 774, 746, 753, 756, 732, 760, 754, 772, 754, 745, 762, 802, 786, 760, 797, 777, 739, 734, 811, 761, 741, 758, 729, 765, 795, 794, 794, 808, 764, 790, 753, 748, 754, 753, 742, 774, 760, 713, 763, 802, 761, 816, 790, 787, 740, 806, 777, 828, 809, 710, 749] payload_rows_per_dst=[9376, 9230, 9198, 9376, 9324, 9266, 9344, 9402] dup_rows_per_dst=[3042, 2906, 2872, 3036, 2950, 2958, 3014, 3010] combine_rows_per_src=[9376, 9230, 9198, 9376, 9324, 9266, 9344, 9402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266534912 dispatch_meta_bytes=344312 combine_payload_bytes=266534912 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266879224 ingress_bytes=335740928 link_bytes=335740928 token_bytes=533414136 weight_movement_bytes=69206016 total_nvlink_bytes=602620152 weight_to_token_ratio=0.129742 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263366083e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266297344 dispatch_meta_bytes=343792 combine_payload_bytes=266297344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266641136 ingress_bytes=300900352 link_bytes=300900352 token_bytes=532938480 weight_movement_bytes=34603008 total_nvlink_bytes=567541488 weight_to_token_ratio=0.064929 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132263800e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=344136 combine_payload_bytes=266059776 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266403912 ingress_bytes=369868800 link_bytes=369868800 token_bytes=532463688 weight_movement_bytes=103809024 total_nvlink_bytes=636272712 weight_to_token_ratio=0.194960 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391786518e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344264 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266707144 ingress_bytes=300965888 link_bytes=300965888 token_bytes=533070024 weight_movement_bytes=34603008 total_nvlink_bytes=567673032 weight_to_token_ratio=0.064913 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132510407e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74064 payload_rows_remote=64912 dup_rows=24240 dup_rows_remote=21166 dedup_ratio=0.246582 router_tokens_per_expert=[769, 761, 781, 800, 777, 736, 782, 769, 760, 744, 798, 770, 787, 805, 764, 731, 753, 805, 764, 794, 776, 763, 766, 806, 783, 739, 773, 719, 734, 792, 734, 768, 765, 758, 775, 778, 805, 735, 741, 756, 812, 814, 747, 760, 808, 774, 771, 765, 751, 760, 796, 783, 701, 790, 743, 739, 778, 731, 782, 775, 742, 753, 758, 803] payload_rows_per_dst=[9308, 9202, 9436, 9194, 9152, 9394, 9130, 9248] dup_rows_per_dst=[3042, 3116, 3018, 2890, 3074, 3108, 2996, 2996] combine_rows_per_src=[9308, 9202, 9436, 9194, 9152, 9394, 9130, 9248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265986048 dispatch_meta_bytes=343128 combine_payload_bytes=265986048 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266329176 ingress_bytes=335192064 link_bytes=335192064 token_bytes=532315224 weight_movement_bytes=69206016 total_nvlink_bytes=601521240 weight_to_token_ratio=0.130009 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261300752e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266846208 dispatch_meta_bytes=344336 combine_payload_bytes=266846208 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267190544 ingress_bytes=336052224 link_bytes=336052224 token_bytes=534036752 weight_movement_bytes=69206016 total_nvlink_bytes=603242768 weight_to_token_ratio=0.129590 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264537464e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=264912896 dispatch_meta_bytes=341624 combine_payload_bytes=264912896 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265254520 ingress_bytes=334118912 link_bytes=334118912 token_bytes=530167416 weight_movement_bytes=69206016 total_nvlink_bytes=599373432 weight_to_token_ratio=0.130536 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257262567e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265879552 dispatch_meta_bytes=344312 combine_payload_bytes=265879552 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266223864 ingress_bytes=369688576 link_bytes=369688576 token_bytes=532103416 weight_movement_bytes=103809024 total_nvlink_bytes=635912440 weight_to_token_ratio=0.195092 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391108349e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74392 payload_rows_remote=65184 dup_rows=23912 dup_rows_remote=21020 dedup_ratio=0.243245 router_tokens_per_expert=[770, 766, 765, 765, 770, 764, 779, 768, 794, 721, 743, 768, 796, 741, 746, 750, 747, 769, 764, 766, 735, 754, 762, 753, 796, 776, 821, 784, 717, 754, 794, 778, 719, 737, 769, 765, 796, 790, 779, 749, 749, 841, 784, 796, 804, 732, 770, 774, 738, 763, 750, 718, 797, 731, 768, 751, 801, 816, 760, 759, 777, 803, 815, 775] payload_rows_per_dst=[9258, 9220, 9208, 9384, 9276, 9366, 9160, 9520] dup_rows_per_dst=[3036, 2898, 2892, 3056, 2932, 3134, 2872, 3092] combine_rows_per_src=[9258, 9220, 9208, 9384, 9276, 9366, 9160, 9520] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74412 payload_rows_remote=65122 dup_rows=23892 dup_rows_remote=20924 dedup_ratio=0.243042 router_tokens_per_expert=[763, 762, 755, 764, 780, 748, 786, 788, 761, 735, 761, 799, 766, 776, 769, 785, 767, 740, 750, 758, 747, 754, 750, 752, 744, 802, 774, 747, 746, 779, 744, 770, 785, 777, 745, 760, 776, 783, 793, 760, 729, 759, 788, 742, 773, 794, 765, 779, 799, 764, 825, 782, 761, 767, 754, 795, 781, 738, 792, 734, 812, 724, 784, 810] payload_rows_per_dst=[9312, 9288, 9202, 9238, 9266, 9290, 9474, 9342] dup_rows_per_dst=[2980, 3016, 2834, 2974, 3092, 2968, 3020, 3008] combine_rows_per_src=[9312, 9288, 9202, 9238, 9266, 9290, 9474, 9342] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74198 payload_rows_remote=64990 dup_rows=24106 dup_rows_remote=21232 dedup_ratio=0.245219 router_tokens_per_expert=[792, 774, 777, 762, 769, 741, 791, 751, 800, 742, 697, 724, 790, 812, 759, 765, 766, 775, 758, 798, 754, 748, 783, 764, 762, 816, 765, 735, 807, 798, 767, 796, 786, 781, 781, 774, 804, 797, 777, 804, 739, 760, 758, 732, 792, 769, 725, 812, 780, 728, 703, 767, 768, 774, 735, 786, 814, 746, 759, 722, 746, 756, 749, 790] payload_rows_per_dst=[9278, 9192, 9252, 9318, 9528, 9192, 9208, 9230] dup_rows_per_dst=[3036, 2986, 3040, 3174, 3080, 2982, 2874, 2934] combine_rows_per_src=[9278, 9192, 9252, 9318, 9528, 9192, 9208, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=344136 combine_payload_bytes=266059776 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266403912 ingress_bytes=369868800 link_bytes=369868800 token_bytes=532463688 weight_movement_bytes=103809024 total_nvlink_bytes=636272712 weight_to_token_ratio=0.194960 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391786518e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74262 payload_rows_remote=64896 dup_rows=24042 dup_rows_remote=21024 dedup_ratio=0.244568 router_tokens_per_expert=[761, 745, 789, 730, 771, 754, 782, 756, 801, 769, 775, 784, 774, 711, 782, 796, 751, 777, 785, 771, 735, 772, 766, 769, 758, 783, 743, 789, 781, 751, 771, 796, 748, 738, 718, 787, 771, 767, 742, 776, 769, 762, 732, 794, 768, 776, 770, 840, 809, 808, 799, 709, 732, 743, 779, 745, 806, 774, 748, 805, 701, 799, 806, 753] payload_rows_per_dst=[9204, 9366, 9256, 9290, 9248, 9328, 9258, 9312] dup_rows_per_dst=[2972, 3018, 2996, 3054, 2846, 3094, 2990, 3072] combine_rows_per_src=[9204, 9366, 9256, 9290, 9248, 9328, 9258, 9312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=267264000 dispatch_meta_bytes=344320 combine_payload_bytes=267264000 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267608320 ingress_bytes=336470016 link_bytes=336470016 token_bytes=534872320 weight_movement_bytes=69206016 total_nvlink_bytes=604078336 weight_to_token_ratio=0.129388 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266109582e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74392 payload_rows_remote=65184 dup_rows=23912 dup_rows_remote=21020 dedup_ratio=0.243245 router_tokens_per_expert=[770, 766, 765, 765, 770, 764, 779, 768, 794, 721, 743, 768, 796, 741, 746, 750, 747, 769, 764, 766, 735, 754, 762, 753, 796, 776, 821, 784, 717, 754, 794, 778, 719, 737, 769, 765, 796, 790, 779, 749, 749, 841, 784, 796, 804, 732, 770, 774, 738, 763, 750, 718, 797, 731, 768, 751, 801, 816, 760, 759, 777, 803, 815, 775] payload_rows_per_dst=[9258, 9220, 9208, 9384, 9276, 9366, 9160, 9520] dup_rows_per_dst=[3036, 2898, 2892, 3056, 2932, 3134, 2872, 3092] combine_rows_per_src=[9258, 9220, 9208, 9384, 9276, 9366, 9160, 9520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265814016 dispatch_meta_bytes=343680 combine_payload_bytes=265814016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266157696 ingress_bytes=300417024 link_bytes=300417024 token_bytes=531971712 weight_movement_bytes=34603008 total_nvlink_bytes=566574720 weight_to_token_ratio=0.065047 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130445076e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65054 dup_rows=23908 dup_rows_remote=20988 dedup_ratio=0.243205 router_tokens_per_expert=[797, 753, 745, 783, 794, 723, 714, 801, 756, 780, 743, 770, 752, 732, 764, 750, 771, 752, 722, 815, 764, 746, 749, 758, 735, 770, 776, 812, 801, 744, 811, 793, 778, 765, 789, 739, 757, 813, 757, 808, 749, 757, 772, 792, 785, 754, 756, 764, 793, 759, 759, 765, 794, 782, 780, 778, 762, 742, 762, 787, 762, 763, 767, 786] payload_rows_per_dst=[9220, 9200, 9310, 9350, 9108, 9300, 9566, 9342] dup_rows_per_dst=[3000, 2894, 2844, 3006, 2950, 3086, 3208, 2920] combine_rows_per_src=[9220, 9200, 9310, 9350, 9108, 9300, 9566, 9342] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74198 payload_rows_remote=64990 dup_rows=24106 dup_rows_remote=21232 dedup_ratio=0.245219 router_tokens_per_expert=[792, 774, 777, 762, 769, 741, 791, 751, 800, 742, 697, 724, 790, 812, 759, 765, 766, 775, 758, 798, 754, 748, 783, 764, 762, 816, 765, 735, 807, 798, 767, 796, 786, 781, 781, 774, 804, 797, 777, 804, 739, 760, 758, 732, 792, 769, 725, 812, 780, 728, 703, 767, 768, 774, 735, 786, 814, 746, 759, 722, 746, 756, 749, 790] payload_rows_per_dst=[9278, 9192, 9252, 9318, 9528, 9192, 9208, 9230] dup_rows_per_dst=[3036, 2986, 3040, 3174, 3080, 2982, 2874, 2934] combine_rows_per_src=[9278, 9192, 9252, 9318, 9528, 9192, 9208, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74758 payload_rows_remote=65358 dup_rows=23546 dup_rows_remote=20596 dedup_ratio=0.239522 router_tokens_per_expert=[767, 767, 763, 749, 757, 736, 789, 775, 775, 762, 800, 795, 793, 754, 747, 744, 785, 709, 740, 803, 746, 783, 734, 765, 784, 791, 735, 792, 764, 783, 738, 758, 769, 727, 738, 767, 803, 771, 814, 739, 723, 753, 782, 795, 752, 765, 786, 774, 767, 743, 754, 738, 757, 888, 783, 781, 758, 822, 758, 758, 774, 820, 741, 769] payload_rows_per_dst=[9400, 9522, 9628, 9366, 9188, 9030, 9444, 9180] dup_rows_per_dst=[2950, 3078, 3112, 2964, 2948, 2620, 2978, 2896] combine_rows_per_src=[9400, 9522, 9628, 9366, 9188, 9030, 9444, 9180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=65274 dup_rows=23860 dup_rows_remote=20956 dedup_ratio=0.242716 router_tokens_per_expert=[780, 741, 800, 728, 777, 760, 763, 782, 772, 734, 763, 784, 769, 745, 785, 755, 799, 739, 807, 776, 791, 759, 815, 755, 732, 787, 711, 730, 816, 763, 781, 783, 740, 765, 757, 730, 752, 738, 775, 780, 764, 779, 782, 797, 770, 773, 752, 771, 727, 780, 761, 745, 775, 767, 770, 761, 783, 769, 756, 796, 817, 791, 773, 774] payload_rows_per_dst=[9272, 9272, 9470, 9350, 9170, 9358, 9224, 9328] dup_rows_per_dst=[2990, 2942, 3012, 2856, 2904, 3018, 2948, 3190] combine_rows_per_src=[9272, 9272, 9470, 9350, 9170, 9358, 9224, 9328] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266993664 dispatch_meta_bytes=344816 combine_payload_bytes=266993664 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267338480 ingress_bytes=301596672 link_bytes=301596672 token_bytes=534332144 weight_movement_bytes=34603008 total_nvlink_bytes=568935152 weight_to_token_ratio=0.064759 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134883996e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74262 payload_rows_remote=64896 dup_rows=24042 dup_rows_remote=21024 dedup_ratio=0.244568 router_tokens_per_expert=[761, 745, 789, 730, 771, 754, 782, 756, 801, 769, 775, 784, 774, 711, 782, 796, 751, 777, 785, 771, 735, 772, 766, 769, 758, 783, 743, 789, 781, 751, 771, 796, 748, 738, 718, 787, 771, 767, 742, 776, 769, 762, 732, 794, 768, 776, 770, 840, 809, 808, 799, 709, 732, 743, 779, 745, 806, 774, 748, 805, 701, 799, 806, 753] payload_rows_per_dst=[9204, 9366, 9256, 9290, 9248, 9328, 9258, 9312] dup_rows_per_dst=[2972, 3018, 2996, 3054, 2846, 3094, 2990, 3072] combine_rows_per_src=[9204, 9366, 9256, 9290, 9248, 9328, 9258, 9312] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74514 payload_rows_remote=65318 dup_rows=23790 dup_rows_remote=20940 dedup_ratio=0.242004 router_tokens_per_expert=[790, 768, 765, 746, 759, 809, 752, 765, 745, 777, 804, 757, 797, 752, 784, 738, 748, 771, 752, 803, 748, 731, 767, 818, 743, 741, 770, 750, 768, 760, 745, 746, 799, 798, 804, 771, 799, 737, 773, 827, 801, 763, 773, 780, 771, 800, 732, 786, 740, 725, 759, 749, 781, 777, 762, 770, 806, 754, 757, 739, 757, 718, 787, 788] payload_rows_per_dst=[9342, 9306, 9276, 9196, 9398, 9412, 9274, 9310] dup_rows_per_dst=[2966, 3002, 3000, 2850, 3218, 3000, 2852, 2902] combine_rows_per_src=[9342, 9306, 9276, 9196, 9398, 9412, 9274, 9310] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266739712 dispatch_meta_bytes=344184 combine_payload_bytes=266739712 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267083896 ingress_bytes=335945728 link_bytes=335945728 token_bytes=533823608 weight_movement_bytes=69206016 total_nvlink_bytes=603029624 weight_to_token_ratio=0.129642 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264136729e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65054 dup_rows=23908 dup_rows_remote=20988 dedup_ratio=0.243205 router_tokens_per_expert=[797, 753, 745, 783, 794, 723, 714, 801, 756, 780, 743, 770, 752, 732, 764, 750, 771, 752, 722, 815, 764, 746, 749, 758, 735, 770, 776, 812, 801, 744, 811, 793, 778, 765, 789, 739, 757, 813, 757, 808, 749, 757, 772, 792, 785, 754, 756, 764, 793, 759, 759, 765, 794, 782, 780, 778, 762, 742, 762, 787, 762, 763, 767, 786] payload_rows_per_dst=[9220, 9200, 9310, 9350, 9108, 9300, 9566, 9342] dup_rows_per_dst=[3000, 2894, 2844, 3006, 2950, 3086, 3208, 2920] combine_rows_per_src=[9220, 9200, 9310, 9350, 9108, 9300, 9566, 9342] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=344888 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266543928 ingress_bytes=300802048 link_bytes=300802048 token_bytes=532742968 weight_movement_bytes=34603008 total_nvlink_bytes=567345976 weight_to_token_ratio=0.064953 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131893890e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74318 payload_rows_remote=65072 dup_rows=23986 dup_rows_remote=21006 dedup_ratio=0.243998 router_tokens_per_expert=[795, 765, 780, 754, 724, 760, 758, 762, 746, 758, 765, 769, 800, 804, 762, 805, 815, 762, 825, 772, 748, 809, 764, 765, 777, 721, 771, 748, 746, 744, 774, 776, 748, 776, 771, 705, 805, 801, 808, 775, 821, 757, 733, 744, 738, 783, 778, 716, 707, 740, 722, 726, 764, 751, 822, 764, 786, 804, 802, 767, 807, 787, 755, 765] payload_rows_per_dst=[9246, 9454, 9502, 9246, 9318, 9138, 9090, 9324] dup_rows_per_dst=[2980, 3114, 3282, 2870, 3060, 2852, 2902, 2926] combine_rows_per_src=[9246, 9454, 9502, 9246, 9318, 9138, 9090, 9324] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=344168 combine_payload_bytes=266461184 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266805352 ingress_bytes=370270208 link_bytes=370270208 token_bytes=533266536 weight_movement_bytes=103809024 total_nvlink_bytes=637075560 weight_to_token_ratio=0.194666 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393296984e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74412 payload_rows_remote=65122 dup_rows=23892 dup_rows_remote=20924 dedup_ratio=0.243042 router_tokens_per_expert=[763, 762, 755, 764, 780, 748, 786, 788, 761, 735, 761, 799, 766, 776, 769, 785, 767, 740, 750, 758, 747, 754, 750, 752, 744, 802, 774, 747, 746, 779, 744, 770, 785, 777, 745, 760, 776, 783, 793, 760, 729, 759, 788, 742, 773, 794, 765, 779, 799, 764, 825, 782, 761, 767, 754, 795, 781, 738, 792, 734, 812, 724, 784, 810] payload_rows_per_dst=[9312, 9288, 9202, 9238, 9266, 9290, 9474, 9342] dup_rows_per_dst=[2980, 3016, 2834, 2974, 3092, 2968, 3020, 3008] combine_rows_per_src=[9312, 9288, 9202, 9238, 9266, 9290, 9474, 9342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=267706368 dispatch_meta_bytes=343816 combine_payload_bytes=267706368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268050184 ingress_bytes=302309376 link_bytes=302309376 token_bytes=535756552 weight_movement_bytes=34603008 total_nvlink_bytes=570359560 weight_to_token_ratio=0.064587 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137565844e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74324 payload_rows_remote=65302 dup_rows=23980 dup_rows_remote=21078 dedup_ratio=0.243937 router_tokens_per_expert=[754, 779, 764, 773, 733, 763, 787, 757, 745, 742, 782, 792, 773, 806, 720, 805, 771, 792, 762, 793, 787, 762, 770, 756, 741, 791, 744, 792, 741, 813, 772, 799, 686, 734, 738, 762, 787, 750, 761, 744, 738, 802, 756, 754, 743, 725, 805, 804, 787, 767, 746, 749, 801, 820, 757, 767, 769, 743, 789, 826, 778, 775, 753, 775] payload_rows_per_dst=[9276, 9366, 9344, 9366, 9022, 9286, 9356, 9308] dup_rows_per_dst=[2944, 2964, 3042, 3020, 2902, 2968, 3032, 3108] combine_rows_per_src=[9276, 9366, 9344, 9366, 9022, 9286, 9356, 9308] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265814016 dispatch_meta_bytes=343680 combine_payload_bytes=265814016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266157696 ingress_bytes=300417024 link_bytes=300417024 token_bytes=531971712 weight_movement_bytes=34603008 total_nvlink_bytes=566574720 weight_to_token_ratio=0.065047 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130445076e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=267362304 dispatch_meta_bytes=344920 combine_payload_bytes=267362304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267707224 ingress_bytes=301965312 link_bytes=301965312 token_bytes=535069528 weight_movement_bytes=34603008 total_nvlink_bytes=569672536 weight_to_token_ratio=0.064670 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136271159e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=64928 dup_rows=24050 dup_rows_remote=20976 dedup_ratio=0.244649 router_tokens_per_expert=[769, 787, 761, 794, 804, 793, 810, 756, 730, 781, 775, 793, 750, 780, 789, 802, 738, 755, 766, 790, 744, 738, 745, 768, 802, 775, 776, 774, 735, 769, 786, 787, 764, 749, 737, 714, 769, 736, 745, 728, 776, 761, 735, 752, 764, 784, 765, 771, 785, 774, 768, 779, 787, 740, 755, 788, 785, 753, 777, 777, 775, 743, 787, 807] payload_rows_per_dst=[9432, 9326, 9202, 9250, 9124, 9210, 9302, 9408] dup_rows_per_dst=[3116, 3074, 2886, 3158, 2760, 3006, 3050, 3000] combine_rows_per_src=[9432, 9326, 9202, 9250, 9124, 9210, 9302, 9408] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=267542528 dispatch_meta_bytes=345032 combine_payload_bytes=267542528 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267887560 ingress_bytes=336748544 link_bytes=336748544 token_bytes=535430088 weight_movement_bytes=69206016 total_nvlink_bytes=604636104 weight_to_token_ratio=0.129253 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.267157661e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266534912 dispatch_meta_bytes=344312 combine_payload_bytes=266534912 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266879224 ingress_bytes=335740928 link_bytes=335740928 token_bytes=533414136 weight_movement_bytes=69206016 total_nvlink_bytes=602620152 weight_to_token_ratio=0.129742 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263366083e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267476992 dispatch_meta_bytes=345520 combine_payload_bytes=267476992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267822512 ingress_bytes=302080000 link_bytes=302080000 token_bytes=535299504 weight_movement_bytes=34603008 total_nvlink_bytes=569902512 weight_to_token_ratio=0.064642 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136702721e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=344168 combine_payload_bytes=266461184 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266805352 ingress_bytes=370270208 link_bytes=370270208 token_bytes=533266536 weight_movement_bytes=103809024 total_nvlink_bytes=637075560 weight_to_token_ratio=0.194666 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393296984e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266739712 dispatch_meta_bytes=344184 combine_payload_bytes=266739712 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267083896 ingress_bytes=335945728 link_bytes=335945728 token_bytes=533823608 weight_movement_bytes=69206016 total_nvlink_bytes=603029624 weight_to_token_ratio=0.129642 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264136729e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74452 payload_rows_remote=65142 dup_rows=23852 dup_rows_remote=20926 dedup_ratio=0.242635 router_tokens_per_expert=[791, 800, 731, 781, 754, 820, 752, 735, 774, 825, 753, 795, 771, 746, 780, 771, 741, 776, 784, 719, 782, 789, 801, 726, 734, 778, 813, 725, 746, 769, 783, 745, 734, 781, 770, 756, 777, 761, 768, 756, 756, 771, 712, 802, 798, 793, 810, 760, 760, 749, 788, 742, 797, 806, 763, 770, 737, 707, 754, 836, 753, 735, 813, 747] payload_rows_per_dst=[9330, 9254, 9310, 9204, 9254, 9394, 9402, 9304] dup_rows_per_dst=[2998, 3176, 2926, 2982, 2952, 3010, 2948, 2860] combine_rows_per_src=[9330, 9254, 9310, 9204, 9254, 9394, 9402, 9304] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74612 payload_rows_remote=65124 dup_rows=23692 dup_rows_remote=20688 dedup_ratio=0.241007 router_tokens_per_expert=[762, 768, 777, 790, 754, 785, 762, 782, 802, 792, 798, 742, 763, 722, 794, 720, 773, 757, 744, 779, 769, 765, 783, 732, 787, 809, 799, 701, 760, 759, 792, 773, 751, 758, 774, 780, 737, 784, 789, 766, 798, 747, 782, 811, 803, 800, 758, 747, 780, 790, 752, 751, 781, 752, 764, 747, 745, 787, 733, 749, 779, 747, 783, 732] payload_rows_per_dst=[9350, 9312, 9302, 9390, 9276, 9488, 9222, 9272] dup_rows_per_dst=[3010, 2954, 2902, 2970, 3002, 3004, 3012, 2838] combine_rows_per_src=[9350, 9312, 9302, 9390, 9276, 9488, 9222, 9272] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74130 payload_rows_remote=64820 dup_rows=24174 dup_rows_remote=21158 dedup_ratio=0.245911 router_tokens_per_expert=[714, 769, 788, 750, 768, 776, 807, 797, 725, 822, 759, 702, 825, 733, 762, 797, 774, 767, 783, 775, 767, 727, 752, 764, 780, 809, 751, 729, 734, 776, 772, 779, 749, 771, 817, 769, 781, 723, 765, 727, 776, 749, 767, 790, 791, 762, 769, 773, 772, 752, 757, 775, 819, 749, 779, 760, 754, 787, 803, 760, 775, 760, 793, 745] payload_rows_per_dst=[9328, 9260, 9246, 9232, 9218, 9252, 9310, 9284] dup_rows_per_dst=[3010, 2990, 2972, 3028, 2986, 3102, 3016, 3070] combine_rows_per_src=[9328, 9260, 9246, 9232, 9218, 9252, 9310, 9284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265945088 dispatch_meta_bytes=343616 combine_payload_bytes=265945088 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266288704 ingress_bytes=335151104 link_bytes=335151104 token_bytes=532233792 weight_movement_bytes=69206016 total_nvlink_bytes=601439808 weight_to_token_ratio=0.130029 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261146622e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74180 payload_rows_remote=64880 dup_rows=24124 dup_rows_remote=21092 dedup_ratio=0.245402 router_tokens_per_expert=[805, 795, 757, 763, 760, 721, 751, 717, 801, 813, 775, 756, 795, 766, 751, 780, 716, 776, 777, 751, 730, 753, 731, 794, 776, 768, 755, 748, 759, 810, 734, 777, 787, 801, 721, 812, 756, 779, 756, 803, 782, 764, 754, 742, 796, 786, 753, 769, 790, 737, 793, 776, 764, 773, 754, 777, 776, 751, 788, 776, 774, 749, 775, 777] payload_rows_per_dst=[8882, 8904, 9606, 9220, 9732, 9240, 9296, 9300] dup_rows_per_dst=[2480, 2662, 3226, 3034, 3606, 3020, 3064, 3032] combine_rows_per_src=[8882, 8904, 9606, 9220, 9732, 9240, 9296, 9300] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74130 payload_rows_remote=64820 dup_rows=24174 dup_rows_remote=21158 dedup_ratio=0.245911 router_tokens_per_expert=[714, 769, 788, 750, 768, 776, 807, 797, 725, 822, 759, 702, 825, 733, 762, 797, 774, 767, 783, 775, 767, 727, 752, 764, 780, 809, 751, 729, 734, 776, 772, 779, 749, 771, 817, 769, 781, 723, 765, 727, 776, 749, 767, 790, 791, 762, 769, 773, 772, 752, 757, 775, 819, 749, 779, 760, 754, 787, 803, 760, 775, 760, 793, 745] payload_rows_per_dst=[9328, 9260, 9246, 9232, 9218, 9252, 9310, 9284] dup_rows_per_dst=[3010, 2990, 2972, 3028, 2986, 3102, 3016, 3070] combine_rows_per_src=[9328, 9260, 9246, 9232, 9218, 9252, 9310, 9284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74422 payload_rows_remote=64854 dup_rows=23882 dup_rows_remote=20656 dedup_ratio=0.242940 router_tokens_per_expert=[781, 787, 747, 803, 796, 772, 772, 773, 746, 734, 780, 771, 760, 804, 795, 779, 791, 751, 785, 762, 740, 722, 777, 782, 760, 761, 788, 798, 746, 741, 752, 748, 778, 747, 815, 767, 764, 769, 741, 751, 785, 712, 751, 749, 761, 765, 777, 789, 757, 770, 777, 759, 764, 770, 814, 783, 781, 769, 773, 776, 786, 710, 789, 749] payload_rows_per_dst=[9568, 9526, 9190, 9060, 9272, 9100, 9318, 9388] dup_rows_per_dst=[3226, 3010, 3030, 2898, 2992, 2888, 2968, 2870] combine_rows_per_src=[9568, 9526, 9190, 9060, 9272, 9100, 9318, 9388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74064 payload_rows_remote=64912 dup_rows=24240 dup_rows_remote=21166 dedup_ratio=0.246582 router_tokens_per_expert=[769, 761, 781, 800, 777, 736, 782, 769, 760, 744, 798, 770, 787, 805, 764, 731, 753, 805, 764, 794, 776, 763, 766, 806, 783, 739, 773, 719, 734, 792, 734, 768, 765, 758, 775, 778, 805, 735, 741, 756, 812, 814, 747, 760, 808, 774, 771, 765, 751, 760, 796, 783, 701, 790, 743, 739, 778, 731, 782, 775, 742, 753, 758, 803] payload_rows_per_dst=[9308, 9202, 9436, 9194, 9152, 9394, 9130, 9248] dup_rows_per_dst=[3042, 3116, 3018, 2890, 3074, 3108, 2996, 2996] combine_rows_per_src=[9308, 9202, 9436, 9194, 9152, 9394, 9130, 9248] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266821632 dispatch_meta_bytes=344272 combine_payload_bytes=266821632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267165904 ingress_bytes=301424640 link_bytes=301424640 token_bytes=533987536 weight_movement_bytes=34603008 total_nvlink_bytes=568590544 weight_to_token_ratio=0.064801 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134236654e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=64928 dup_rows=24050 dup_rows_remote=20976 dedup_ratio=0.244649 router_tokens_per_expert=[769, 787, 761, 794, 804, 793, 810, 756, 730, 781, 775, 793, 750, 780, 789, 802, 738, 755, 766, 790, 744, 738, 745, 768, 802, 775, 776, 774, 735, 769, 786, 787, 764, 749, 737, 714, 769, 736, 745, 728, 776, 761, 735, 752, 764, 784, 765, 771, 785, 774, 768, 779, 787, 740, 755, 788, 785, 753, 777, 777, 775, 743, 787, 807] payload_rows_per_dst=[9432, 9326, 9202, 9250, 9124, 9210, 9302, 9408] dup_rows_per_dst=[3116, 3074, 2886, 3158, 2760, 3006, 3050, 3000] combine_rows_per_src=[9432, 9326, 9202, 9250, 9124, 9210, 9302, 9408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74382 payload_rows_remote=65078 dup_rows=23922 dup_rows_remote=20734 dedup_ratio=0.243347 router_tokens_per_expert=[808, 767, 801, 752, 774, 764, 727, 784, 775, 809, 783, 762, 773, 789, 785, 787, 778, 767, 729, 778, 763, 749, 743, 753, 771, 779, 769, 768, 820, 783, 769, 787, 775, 729, 792, 760, 747, 757, 759, 784, 780, 775, 804, 751, 766, 770, 782, 740, 749, 742, 761, 723, 750, 792, 771, 716, 764, 756, 798, 776, 765, 797, 701, 774] payload_rows_per_dst=[9282, 9478, 9224, 9304, 9332, 9290, 9158, 9314] dup_rows_per_dst=[3072, 3048, 2896, 3188, 2874, 3046, 2850, 2948] combine_rows_per_src=[9282, 9478, 9224, 9304, 9332, 9290, 9158, 9314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=343912 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265846632 ingress_bytes=334708736 link_bytes=334708736 token_bytes=531349352 weight_movement_bytes=69206016 total_nvlink_bytes=600555368 weight_to_token_ratio=0.130246 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259482027e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266747904 dispatch_meta_bytes=343248 combine_payload_bytes=266747904 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267091152 ingress_bytes=370556928 link_bytes=370556928 token_bytes=533839056 weight_movement_bytes=103809024 total_nvlink_bytes=637648080 weight_to_token_ratio=0.194458 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394375888e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265748480 dispatch_meta_bytes=343888 combine_payload_bytes=265748480 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266092368 ingress_bytes=300351488 link_bytes=300351488 token_bytes=531840848 weight_movement_bytes=34603008 total_nvlink_bytes=566443856 weight_to_token_ratio=0.065063 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130198469e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74758 payload_rows_remote=65358 dup_rows=23546 dup_rows_remote=20596 dedup_ratio=0.239522 router_tokens_per_expert=[767, 767, 763, 749, 757, 736, 789, 775, 775, 762, 800, 795, 793, 754, 747, 744, 785, 709, 740, 803, 746, 783, 734, 765, 784, 791, 735, 792, 764, 783, 738, 758, 769, 727, 738, 767, 803, 771, 814, 739, 723, 753, 782, 795, 752, 765, 786, 774, 767, 743, 754, 738, 757, 888, 783, 781, 758, 822, 758, 758, 774, 820, 741, 769] payload_rows_per_dst=[9400, 9522, 9628, 9366, 9188, 9030, 9444, 9180] dup_rows_per_dst=[2950, 3078, 3112, 2964, 2948, 2620, 2978, 2896] combine_rows_per_src=[9400, 9522, 9628, 9366, 9188, 9030, 9444, 9180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74180 payload_rows_remote=64880 dup_rows=24124 dup_rows_remote=21092 dedup_ratio=0.245402 router_tokens_per_expert=[805, 795, 757, 763, 760, 721, 751, 717, 801, 813, 775, 756, 795, 766, 751, 780, 716, 776, 777, 751, 730, 753, 731, 794, 776, 768, 755, 748, 759, 810, 734, 777, 787, 801, 721, 812, 756, 779, 756, 803, 782, 764, 754, 742, 796, 786, 753, 769, 790, 737, 793, 776, 764, 773, 754, 777, 776, 751, 788, 776, 774, 749, 775, 777] payload_rows_per_dst=[8882, 8904, 9606, 9220, 9732, 9240, 9296, 9300] dup_rows_per_dst=[2480, 2662, 3226, 3034, 3606, 3020, 3064, 3032] combine_rows_per_src=[8882, 8904, 9606, 9220, 9732, 9240, 9296, 9300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74612 payload_rows_remote=65124 dup_rows=23692 dup_rows_remote=20688 dedup_ratio=0.241007 router_tokens_per_expert=[762, 768, 777, 790, 754, 785, 762, 782, 802, 792, 798, 742, 763, 722, 794, 720, 773, 757, 744, 779, 769, 765, 783, 732, 787, 809, 799, 701, 760, 759, 792, 773, 751, 758, 774, 780, 737, 784, 789, 766, 798, 747, 782, 811, 803, 800, 758, 747, 780, 790, 752, 751, 781, 752, 764, 747, 745, 787, 733, 749, 779, 747, 783, 732] payload_rows_per_dst=[9350, 9312, 9302, 9390, 9276, 9488, 9222, 9272] dup_rows_per_dst=[3010, 2954, 2902, 2970, 3002, 3004, 3012, 2838] combine_rows_per_src=[9350, 9312, 9302, 9390, 9276, 9488, 9222, 9272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265641984 dispatch_meta_bytes=342040 combine_payload_bytes=265641984 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265984024 ingress_bytes=334848000 link_bytes=334848000 token_bytes=531626008 weight_movement_bytes=69206016 total_nvlink_bytes=600832024 weight_to_token_ratio=0.130178 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260006067e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74270 payload_rows_remote=64880 dup_rows=24034 dup_rows_remote=20988 dedup_ratio=0.244486 router_tokens_per_expert=[709, 730, 782, 791, 741, 794, 786, 772, 711, 775, 754, 758, 773, 751, 782, 772, 775, 775, 810, 791, 757, 748, 755, 795, 727, 752, 754, 763, 773, 771, 770, 782, 794, 784, 733, 777, 792, 803, 783, 752, 768, 779, 742, 811, 742, 761, 771, 789, 819, 765, 782, 722, 763, 785, 799, 763, 803, 735, 729, 770, 804, 767, 760, 726] payload_rows_per_dst=[9234, 9176, 9282, 9256, 9390, 9330, 9312, 9290] dup_rows_per_dst=[2976, 2976, 3130, 2928, 3046, 2996, 3084, 2898] combine_rows_per_src=[9234, 9176, 9282, 9256, 9390, 9330, 9312, 9290] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265945088 dispatch_meta_bytes=343616 combine_payload_bytes=265945088 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266288704 ingress_bytes=335151104 link_bytes=335151104 token_bytes=532233792 weight_movement_bytes=69206016 total_nvlink_bytes=601439808 weight_to_token_ratio=0.130029 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261146622e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265879552 dispatch_meta_bytes=344312 combine_payload_bytes=265879552 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266223864 ingress_bytes=369688576 link_bytes=369688576 token_bytes=532103416 weight_movement_bytes=103809024 total_nvlink_bytes=635912440 weight_to_token_ratio=0.195092 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391108349e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74524 payload_rows_remote=65208 dup_rows=23780 dup_rows_remote=20760 dedup_ratio=0.241903 router_tokens_per_expert=[755, 760, 763, 747, 813, 778, 717, 734, 748, 799, 807, 819, 704, 726, 788, 777, 766, 789, 772, 753, 819, 726, 783, 779, 752, 760, 729, 729, 802, 782, 760, 754, 762, 793, 776, 790, 781, 721, 759, 798, 766, 744, 747, 772, 745, 773, 813, 817, 818, 678, 805, 785, 768, 791, 715, 754, 794, 773, 754, 831, 725, 740, 772, 802] payload_rows_per_dst=[9258, 9316, 9362, 9156, 9370, 9324, 9348, 9390] dup_rows_per_dst=[2876, 3020, 3012, 2980, 2990, 3030, 2880, 2992] combine_rows_per_src=[9258, 9316, 9362, 9156, 9370, 9324, 9348, 9390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266559488 dispatch_meta_bytes=343248 combine_payload_bytes=266559488 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266902736 ingress_bytes=335765504 link_bytes=335765504 token_bytes=533462224 weight_movement_bytes=69206016 total_nvlink_bytes=602668240 weight_to_token_ratio=0.129730 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263458560e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=267706368 dispatch_meta_bytes=343816 combine_payload_bytes=267706368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268050184 ingress_bytes=302309376 link_bytes=302309376 token_bytes=535756552 weight_movement_bytes=34603008 total_nvlink_bytes=570359560 weight_to_token_ratio=0.064587 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137565844e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265748480 dispatch_meta_bytes=343888 combine_payload_bytes=265748480 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266092368 ingress_bytes=300351488 link_bytes=300351488 token_bytes=531840848 weight_movement_bytes=34603008 total_nvlink_bytes=566443856 weight_to_token_ratio=0.065063 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130198469e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266747904 dispatch_meta_bytes=343248 combine_payload_bytes=266747904 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267091152 ingress_bytes=370556928 link_bytes=370556928 token_bytes=533839056 weight_movement_bytes=103809024 total_nvlink_bytes=637648080 weight_to_token_ratio=0.194458 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394375888e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74540 payload_rows_remote=65394 dup_rows=23764 dup_rows_remote=20866 dedup_ratio=0.241740 router_tokens_per_expert=[745, 789, 768, 812, 759, 785, 748, 740, 775, 758, 760, 765, 803, 782, 745, 734, 792, 782, 713, 764, 762, 748, 715, 746, 733, 777, 788, 752, 753, 797, 752, 826, 770, 791, 739, 761, 799, 794, 799, 755, 764, 779, 801, 790, 781, 797, 741, 737, 783, 757, 806, 761, 774, 755, 761, 709, 749, 789, 798, 741, 777, 734, 836, 756] payload_rows_per_dst=[9340, 9280, 9146, 9394, 9474, 9346, 9238, 9322] dup_rows_per_dst=[2952, 2964, 2898, 2962, 2942, 3034, 2974, 3038] combine_rows_per_src=[9340, 9280, 9146, 9394, 9474, 9346, 9238, 9322] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265748480 dispatch_meta_bytes=343472 combine_payload_bytes=265748480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266091952 ingress_bytes=300351488 link_bytes=300351488 token_bytes=531840432 weight_movement_bytes=34603008 total_nvlink_bytes=566443440 weight_to_token_ratio=0.065063 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130198469e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267091968 dispatch_meta_bytes=343872 combine_payload_bytes=267091968 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267435840 ingress_bytes=370900992 link_bytes=370900992 token_bytes=534527808 weight_movement_bytes=103809024 total_nvlink_bytes=638336832 weight_to_token_ratio=0.194207 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395670573e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74646 payload_rows_remote=65332 dup_rows=23658 dup_rows_remote=20664 dedup_ratio=0.240662 router_tokens_per_expert=[736, 754, 806, 786, 750, 804, 791, 760, 726, 765, 745, 789, 739, 795, 783, 800, 787, 770, 783, 752, 746, 768, 778, 746, 745, 777, 758, 788, 722, 720, 790, 781, 709, 758, 741, 836, 741, 760, 737, 792, 769, 756, 737, 765, 782, 772, 761, 779, 794, 788, 762, 732, 775, 786, 788, 729, 819, 764, 770, 795, 775, 810, 765, 765] payload_rows_per_dst=[9402, 9356, 9240, 9306, 9212, 9264, 9314, 9552] dup_rows_per_dst=[2972, 2928, 3020, 2856, 2936, 2978, 2994, 2974] combine_rows_per_src=[9402, 9356, 9240, 9306, 9212, 9264, 9314, 9552] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=64946 dup_rows=24046 dup_rows_remote=21060 dedup_ratio=0.244609 router_tokens_per_expert=[789, 757, 764, 797, 807, 735, 778, 759, 753, 745, 787, 803, 776, 819, 764, 726, 736, 763, 774, 806, 803, 756, 762, 793, 790, 743, 781, 772, 805, 768, 804, 723, 733, 715, 728, 767, 724, 802, 761, 753, 826, 734, 743, 750, 736, 777, 775, 808, 757, 783, 757, 773, 758, 770, 788, 789, 774, 763, 759, 786, 773, 736, 731, 785] payload_rows_per_dst=[9346, 9274, 9316, 9304, 9176, 9312, 9312, 9218] dup_rows_per_dst=[3026, 3072, 3070, 3068, 2790, 2986, 3038, 2996] combine_rows_per_src=[9346, 9274, 9316, 9304, 9176, 9312, 9312, 9218] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=65064 dup_rows=23962 dup_rows_remote=20876 dedup_ratio=0.243754 router_tokens_per_expert=[821, 753, 730, 769, 741, 721, 764, 778, 782, 806, 783, 776, 776, 754, 802, 788, 746, 731, 812, 768, 753, 740, 789, 752, 727, 748, 737, 805, 754, 760, 756, 736, 753, 765, 759, 762, 739, 814, 724, 751, 765, 774, 749, 790, 773, 795, 763, 751, 815, 789, 735, 804, 769, 754, 830, 789, 750, 775, 767, 761, 812, 768, 805, 744] payload_rows_per_dst=[9220, 9370, 9236, 9164, 9280, 9226, 9568, 9278] dup_rows_per_dst=[2934, 3124, 2986, 2882, 2854, 2918, 3178, 3086] combine_rows_per_src=[9220, 9370, 9236, 9164, 9280, 9226, 9568, 9278] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=64868 dup_rows=23926 dup_rows_remote=20866 dedup_ratio=0.243388 router_tokens_per_expert=[754, 754, 797, 813, 762, 773, 728, 786, 769, 748, 782, 754, 800, 803, 754, 778, 846, 780, 740, 761, 752, 762, 789, 804, 760, 745, 769, 693, 818, 748, 737, 812, 772, 747, 793, 777, 740, 785, 724, 732, 737, 811, 706, 763, 785, 729, 700, 779, 769, 798, 762, 772, 802, 761, 763, 778, 802, 775, 809, 777, 748, 769, 754, 762] payload_rows_per_dst=[9510, 9226, 9384, 9570, 9394, 9230, 8980, 9084] dup_rows_per_dst=[3060, 2916, 3084, 3292, 3054, 2860, 2732, 2928] combine_rows_per_src=[9510, 9226, 9384, 9570, 9394, 9230, 8980, 9084] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267853824 dispatch_meta_bytes=345040 combine_payload_bytes=267853824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268198864 ingress_bytes=302456832 link_bytes=302456832 token_bytes=536052688 weight_movement_bytes=34603008 total_nvlink_bytes=570655696 weight_to_token_ratio=0.064552 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138120709e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74524 payload_rows_remote=65208 dup_rows=23780 dup_rows_remote=20760 dedup_ratio=0.241903 router_tokens_per_expert=[755, 760, 763, 747, 813, 778, 717, 734, 748, 799, 807, 819, 704, 726, 788, 777, 766, 789, 772, 753, 819, 726, 783, 779, 752, 760, 729, 729, 802, 782, 760, 754, 762, 793, 776, 790, 781, 721, 759, 798, 766, 744, 747, 772, 745, 773, 813, 817, 818, 678, 805, 785, 768, 791, 715, 754, 794, 773, 754, 831, 725, 740, 772, 802] payload_rows_per_dst=[9258, 9316, 9362, 9156, 9370, 9324, 9348, 9390] dup_rows_per_dst=[2876, 3020, 3012, 2980, 2990, 3030, 2880, 2992] combine_rows_per_src=[9258, 9316, 9362, 9156, 9370, 9324, 9348, 9390] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266018816 dispatch_meta_bytes=344024 combine_payload_bytes=266018816 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266362840 ingress_bytes=335224832 link_bytes=335224832 token_bytes=532381656 weight_movement_bytes=69206016 total_nvlink_bytes=601587672 weight_to_token_ratio=0.129993 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261424055e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267599872 dispatch_meta_bytes=343984 combine_payload_bytes=267599872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267943856 ingress_bytes=336805888 link_bytes=336805888 token_bytes=535543728 weight_movement_bytes=69206016 total_nvlink_bytes=604749744 weight_to_token_ratio=0.129226 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.267373441e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74324 payload_rows_remote=65302 dup_rows=23980 dup_rows_remote=21078 dedup_ratio=0.243937 router_tokens_per_expert=[754, 779, 764, 773, 733, 763, 787, 757, 745, 742, 782, 792, 773, 806, 720, 805, 771, 792, 762, 793, 787, 762, 770, 756, 741, 791, 744, 792, 741, 813, 772, 799, 686, 734, 738, 762, 787, 750, 761, 744, 738, 802, 756, 754, 743, 725, 805, 804, 787, 767, 746, 749, 801, 820, 757, 767, 769, 743, 789, 826, 778, 775, 753, 775] payload_rows_per_dst=[9276, 9366, 9344, 9366, 9022, 9286, 9356, 9308] dup_rows_per_dst=[2944, 2964, 3042, 3020, 2902, 2968, 3032, 3108] combine_rows_per_src=[9276, 9366, 9344, 9366, 9022, 9286, 9356, 9308] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74262 payload_rows_remote=65074 dup_rows=24042 dup_rows_remote=21044 dedup_ratio=0.244568 router_tokens_per_expert=[802, 749, 743, 756, 762, 789, 732, 766, 757, 802, 747, 772, 775, 758, 774, 768, 760, 764, 739, 744, 753, 749, 800, 760, 775, 771, 710, 755, 770, 771, 796, 745, 766, 761, 788, 807, 765, 772, 769, 770, 761, 747, 835, 768, 744, 786, 778, 816, 767, 771, 768, 742, 784, 792, 791, 761, 799, 774, 772, 752, 764, 753, 764, 751] payload_rows_per_dst=[9204, 9250, 9254, 9188, 9372, 9450, 9312, 9232] dup_rows_per_dst=[2994, 3056, 2884, 2998, 3024, 3020, 3040, 3026] combine_rows_per_src=[9204, 9250, 9254, 9188, 9372, 9450, 9312, 9232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266502144 dispatch_meta_bytes=343760 combine_payload_bytes=266502144 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266845904 ingress_bytes=370311168 link_bytes=370311168 token_bytes=533348048 weight_movement_bytes=103809024 total_nvlink_bytes=637157072 weight_to_token_ratio=0.194637 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393451113e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74422 payload_rows_remote=64854 dup_rows=23882 dup_rows_remote=20656 dedup_ratio=0.242940 router_tokens_per_expert=[781, 787, 747, 803, 796, 772, 772, 773, 746, 734, 780, 771, 760, 804, 795, 779, 791, 751, 785, 762, 740, 722, 777, 782, 760, 761, 788, 798, 746, 741, 752, 748, 778, 747, 815, 767, 764, 769, 741, 751, 785, 712, 751, 749, 761, 765, 777, 789, 757, 770, 777, 759, 764, 770, 814, 783, 781, 769, 773, 776, 786, 710, 789, 749] payload_rows_per_dst=[9568, 9526, 9190, 9060, 9272, 9100, 9318, 9388] dup_rows_per_dst=[3226, 3010, 3030, 2898, 2992, 2888, 2968, 2870] combine_rows_per_src=[9568, 9526, 9190, 9060, 9272, 9100, 9318, 9388] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=64950 dup_rows=24046 dup_rows_remote=20904 dedup_ratio=0.244609 router_tokens_per_expert=[757, 746, 754, 704, 795, 806, 767, 748, 748, 728, 740, 765, 753, 777, 776, 780, 805, 811, 796, 758, 767, 780, 785, 817, 763, 778, 791, 786, 760, 776, 824, 712, 776, 749, 790, 753, 773, 780, 808, 796, 754, 744, 778, 749, 803, 733, 768, 765, 785, 756, 730, 772, 738, 747, 768, 832, 773, 740, 790, 760, 746, 738, 744, 761] payload_rows_per_dst=[9248, 9236, 9444, 9366, 9308, 9224, 9276, 9156] dup_rows_per_dst=[2906, 2898, 3194, 3014, 3142, 2964, 2980, 2948] combine_rows_per_src=[9248, 9236, 9444, 9366, 9308, 9224, 9276, 9156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267599872 dispatch_meta_bytes=343984 combine_payload_bytes=267599872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267943856 ingress_bytes=336805888 link_bytes=336805888 token_bytes=535543728 weight_movement_bytes=69206016 total_nvlink_bytes=604749744 weight_to_token_ratio=0.129226 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.267373441e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=65064 dup_rows=23962 dup_rows_remote=20876 dedup_ratio=0.243754 router_tokens_per_expert=[821, 753, 730, 769, 741, 721, 764, 778, 782, 806, 783, 776, 776, 754, 802, 788, 746, 731, 812, 768, 753, 740, 789, 752, 727, 748, 737, 805, 754, 760, 756, 736, 753, 765, 759, 762, 739, 814, 724, 751, 765, 774, 749, 790, 773, 795, 763, 751, 815, 789, 735, 804, 769, 754, 830, 789, 750, 775, 767, 761, 812, 768, 805, 744] payload_rows_per_dst=[9220, 9370, 9236, 9164, 9280, 9226, 9568, 9278] dup_rows_per_dst=[2934, 3124, 2986, 2882, 2854, 2918, 3178, 3086] combine_rows_per_src=[9220, 9370, 9236, 9164, 9280, 9226, 9568, 9278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=64946 dup_rows=24046 dup_rows_remote=21060 dedup_ratio=0.244609 router_tokens_per_expert=[789, 757, 764, 797, 807, 735, 778, 759, 753, 745, 787, 803, 776, 819, 764, 726, 736, 763, 774, 806, 803, 756, 762, 793, 790, 743, 781, 772, 805, 768, 804, 723, 733, 715, 728, 767, 724, 802, 761, 753, 826, 734, 743, 750, 736, 777, 775, 808, 757, 783, 757, 773, 758, 770, 788, 789, 774, 763, 759, 786, 773, 736, 731, 785] payload_rows_per_dst=[9346, 9274, 9316, 9304, 9176, 9312, 9312, 9218] dup_rows_per_dst=[3026, 3072, 3070, 3068, 2790, 2986, 3038, 2996] combine_rows_per_src=[9346, 9274, 9316, 9304, 9176, 9312, 9312, 9218] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267091968 dispatch_meta_bytes=343872 combine_payload_bytes=267091968 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267435840 ingress_bytes=370900992 link_bytes=370900992 token_bytes=534527808 weight_movement_bytes=103809024 total_nvlink_bytes=638336832 weight_to_token_ratio=0.194207 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395670573e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267476992 dispatch_meta_bytes=345520 combine_payload_bytes=267476992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267822512 ingress_bytes=302080000 link_bytes=302080000 token_bytes=535299504 weight_movement_bytes=34603008 total_nvlink_bytes=569902512 weight_to_token_ratio=0.064642 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136702721e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265699328 dispatch_meta_bytes=342936 combine_payload_bytes=265699328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266042264 ingress_bytes=300302336 link_bytes=300302336 token_bytes=531741592 weight_movement_bytes=34603008 total_nvlink_bytes=566344600 weight_to_token_ratio=0.065075 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130013514e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=64972 dup_rows=24016 dup_rows_remote=20906 dedup_ratio=0.244303 router_tokens_per_expert=[783, 799, 792, 778, 739, 744, 731, 758, 761, 764, 791, 777, 766, 784, 759, 811, 778, 749, 740, 772, 774, 706, 764, 764, 753, 729, 811, 791, 774, 771, 721, 768, 743, 735, 719, 789, 789, 760, 769, 728, 812, 767, 771, 802, 805, 789, 774, 762, 754, 787, 785, 779, 778, 759, 744, 736, 812, 757, 726, 808, 771, 736, 808, 796] payload_rows_per_dst=[9276, 9316, 9150, 9352, 9140, 9508, 9228, 9318] dup_rows_per_dst=[2972, 3110, 2944, 2884, 2924, 3056, 3016, 3110] combine_rows_per_src=[9276, 9316, 9150, 9352, 9140, 9508, 9228, 9318] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=344472 combine_payload_bytes=266543104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266887576 ingress_bytes=301146112 link_bytes=301146112 token_bytes=533430680 weight_movement_bytes=34603008 total_nvlink_bytes=568033688 weight_to_token_ratio=0.064869 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133188575e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265641984 dispatch_meta_bytes=342040 combine_payload_bytes=265641984 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265984024 ingress_bytes=334848000 link_bytes=334848000 token_bytes=531626008 weight_movement_bytes=69206016 total_nvlink_bytes=600832024 weight_to_token_ratio=0.130178 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260006067e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266502144 dispatch_meta_bytes=343760 combine_payload_bytes=266502144 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266845904 ingress_bytes=370311168 link_bytes=370311168 token_bytes=533348048 weight_movement_bytes=103809024 total_nvlink_bytes=637157072 weight_to_token_ratio=0.194637 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393451113e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74098 payload_rows_remote=64836 dup_rows=24206 dup_rows_remote=21112 dedup_ratio=0.246236 router_tokens_per_expert=[752, 754, 772, 762, 798, 789, 739, 799, 792, 800, 766, 791, 741, 720, 771, 772, 731, 818, 732, 771, 799, 761, 772, 794, 760, 806, 759, 756, 712, 757, 787, 769, 839, 753, 754, 752, 739, 733, 795, 768, 777, 782, 743, 764, 781, 795, 744, 772, 763, 772, 758, 755, 738, 746, 781, 760, 773, 723, 775, 770, 787, 795, 778, 785] payload_rows_per_dst=[9320, 9318, 9262, 9212, 9232, 9276, 9208, 9270] dup_rows_per_dst=[3010, 2988, 3094, 3000, 3034, 3040, 2938, 3102] combine_rows_per_src=[9320, 9318, 9262, 9212, 9232, 9276, 9208, 9270] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=64908 dup_rows=24050 dup_rows_remote=20910 dedup_ratio=0.244649 router_tokens_per_expert=[778, 798, 719, 790, 815, 754, 759, 721, 741, 811, 752, 696, 787, 739, 775, 744, 766, 765, 792, 780, 736, 732, 769, 762, 751, 786, 807, 745, 832, 810, 810, 729, 755, 790, 771, 749, 789, 795, 790, 775, 776, 776, 787, 797, 793, 761, 766, 787, 832, 795, 749, 718, 756, 774, 754, 732, 787, 725, 755, 739, 780, 730, 751, 767] payload_rows_per_dst=[9316, 9152, 9234, 9472, 9298, 9346, 9290, 9146] dup_rows_per_dst=[2952, 2938, 2970, 3068, 3130, 3140, 2930, 2922] combine_rows_per_src=[9316, 9152, 9234, 9472, 9298, 9346, 9290, 9146] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266035200 dispatch_meta_bytes=343416 combine_payload_bytes=266035200 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266378616 ingress_bytes=335241216 link_bytes=335241216 token_bytes=532413816 weight_movement_bytes=69206016 total_nvlink_bytes=601619832 weight_to_token_ratio=0.129985 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261485707e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266018816 dispatch_meta_bytes=344024 combine_payload_bytes=266018816 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266362840 ingress_bytes=335224832 link_bytes=335224832 token_bytes=532381656 weight_movement_bytes=69206016 total_nvlink_bytes=601587672 weight_to_token_ratio=0.129993 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261424055e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266125312 dispatch_meta_bytes=343512 combine_payload_bytes=266125312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266468824 ingress_bytes=300728320 link_bytes=300728320 token_bytes=532594136 weight_movement_bytes=34603008 total_nvlink_bytes=567197144 weight_to_token_ratio=0.064971 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131616458e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74438 payload_rows_remote=65144 dup_rows=23866 dup_rows_remote=20940 dedup_ratio=0.242778 router_tokens_per_expert=[813, 745, 742, 764, 801, 788, 776, 757, 742, 789, 796, 750, 782, 798, 830, 759, 723, 766, 759, 747, 782, 747, 766, 742, 753, 783, 771, 780, 785, 778, 776, 759, 776, 798, 787, 741, 755, 777, 748, 731, 829, 743, 798, 746, 725, 735, 772, 752, 789, 763, 757, 747, 754, 763, 751, 786, 795, 745, 753, 784, 755, 795, 766, 787] payload_rows_per_dst=[9322, 9484, 9170, 9308, 9254, 9246, 9294, 9360] dup_rows_per_dst=[3050, 3008, 2894, 3062, 2972, 2954, 2926, 3000] combine_rows_per_src=[9322, 9484, 9170, 9308, 9254, 9246, 9294, 9360] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74066 payload_rows_remote=64670 dup_rows=24238 dup_rows_remote=21172 dedup_ratio=0.246562 router_tokens_per_expert=[759, 802, 776, 762, 795, 770, 806, 775, 757, 734, 790, 759, 757, 752, 741, 722, 771, 745, 766, 794, 736, 802, 785, 755, 761, 834, 781, 732, 788, 751, 791, 798, 774, 775, 750, 753, 699, 768, 775, 762, 798, 780, 753, 811, 771, 819, 759, 807, 753, 761, 753, 751, 749, 816, 706, 739, 791, 776, 786, 779, 744, 746, 749, 752] payload_rows_per_dst=[9288, 9108, 9318, 9076, 9078, 9452, 9350, 9396] dup_rows_per_dst=[2928, 2916, 2990, 2814, 3034, 3202, 3288, 3066] combine_rows_per_src=[9288, 9108, 9318, 9076, 9078, 9452, 9350, 9396] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74270 payload_rows_remote=64880 dup_rows=24034 dup_rows_remote=20988 dedup_ratio=0.244486 router_tokens_per_expert=[709, 730, 782, 791, 741, 794, 786, 772, 711, 775, 754, 758, 773, 751, 782, 772, 775, 775, 810, 791, 757, 748, 755, 795, 727, 752, 754, 763, 773, 771, 770, 782, 794, 784, 733, 777, 792, 803, 783, 752, 768, 779, 742, 811, 742, 761, 771, 789, 819, 765, 782, 722, 763, 785, 799, 763, 803, 735, 729, 770, 804, 767, 760, 726] payload_rows_per_dst=[9234, 9176, 9282, 9256, 9390, 9330, 9312, 9290] dup_rows_per_dst=[2976, 2976, 3130, 2928, 3046, 2996, 3084, 2898] combine_rows_per_src=[9234, 9176, 9282, 9256, 9390, 9330, 9312, 9290] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=64972 dup_rows=24016 dup_rows_remote=20906 dedup_ratio=0.244303 router_tokens_per_expert=[783, 799, 792, 778, 739, 744, 731, 758, 761, 764, 791, 777, 766, 784, 759, 811, 778, 749, 740, 772, 774, 706, 764, 764, 753, 729, 811, 791, 774, 771, 721, 768, 743, 735, 719, 789, 789, 760, 769, 728, 812, 767, 771, 802, 805, 789, 774, 762, 754, 787, 785, 779, 778, 759, 744, 736, 812, 757, 726, 808, 771, 736, 808, 796] payload_rows_per_dst=[9276, 9316, 9150, 9352, 9140, 9508, 9228, 9318] dup_rows_per_dst=[2972, 3110, 2944, 2884, 2924, 3056, 3016, 3110] combine_rows_per_src=[9276, 9316, 9150, 9352, 9140, 9508, 9228, 9318] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265568256 dispatch_meta_bytes=343792 combine_payload_bytes=265568256 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265912048 ingress_bytes=334774272 link_bytes=334774272 token_bytes=531480304 weight_movement_bytes=69206016 total_nvlink_bytes=600686320 weight_to_token_ratio=0.130214 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259728634e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265863168 dispatch_meta_bytes=343272 combine_payload_bytes=265863168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266206440 ingress_bytes=335069184 link_bytes=335069184 token_bytes=532069608 weight_movement_bytes=69206016 total_nvlink_bytes=601275624 weight_to_token_ratio=0.130069 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260838364e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64882 dup_rows=24114 dup_rows_remote=21104 dedup_ratio=0.245300 router_tokens_per_expert=[813, 792, 765, 794, 806, 711, 755, 711, 823, 752, 772, 789, 781, 724, 707, 804, 751, 767, 763, 807, 775, 751, 806, 744, 751, 819, 755, 757, 736, 711, 743, 786, 786, 796, 803, 805, 755, 800, 772, 808, 757, 810, 757, 795, 725, 735, 742, 743, 763, 733, 788, 799, 771, 761, 732, 798, 795, 743, 780, 718, 761, 778, 774, 748] payload_rows_per_dst=[9308, 9464, 9314, 9226, 9402, 9002, 9200, 9274] dup_rows_per_dst=[3010, 3184, 2994, 3072, 3062, 2782, 3090, 2920] combine_rows_per_src=[9308, 9464, 9314, 9226, 9402, 9002, 9200, 9274] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266829824 dispatch_meta_bytes=344336 combine_payload_bytes=266829824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267174160 ingress_bytes=301432832 link_bytes=301432832 token_bytes=534003984 weight_movement_bytes=34603008 total_nvlink_bytes=568606992 weight_to_token_ratio=0.064799 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134267479e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=264888320 dispatch_meta_bytes=343368 combine_payload_bytes=264888320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265231688 ingress_bytes=299491328 link_bytes=299491328 token_bytes=530120008 weight_movement_bytes=34603008 total_nvlink_bytes=564723016 weight_to_token_ratio=0.065274 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126961756e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74430 payload_rows_remote=65130 dup_rows=23874 dup_rows_remote=20796 dedup_ratio=0.242859 router_tokens_per_expert=[737, 792, 805, 786, 772, 776, 775, 756, 769, 772, 767, 782, 760, 763, 772, 707, 769, 773, 769, 790, 740, 794, 779, 752, 720, 738, 812, 734, 826, 761, 810, 788, 789, 742, 744, 733, 757, 798, 793, 791, 760, 760, 770, 765, 727, 790, 749, 735, 757, 745, 787, 777, 775, 760, 758, 786, 728, 755, 757, 778, 793, 797, 806, 744] payload_rows_per_dst=[9286, 9288, 9314, 9300, 9382, 9234, 9326, 9300] dup_rows_per_dst=[3112, 2896, 3018, 3078, 2912, 2878, 2964, 3016] combine_rows_per_src=[9286, 9288, 9314, 9300, 9382, 9234, 9326, 9300] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=64868 dup_rows=23926 dup_rows_remote=20866 dedup_ratio=0.243388 router_tokens_per_expert=[754, 754, 797, 813, 762, 773, 728, 786, 769, 748, 782, 754, 800, 803, 754, 778, 846, 780, 740, 761, 752, 762, 789, 804, 760, 745, 769, 693, 818, 748, 737, 812, 772, 747, 793, 777, 740, 785, 724, 732, 737, 811, 706, 763, 785, 729, 700, 779, 769, 798, 762, 772, 802, 761, 763, 778, 802, 775, 809, 777, 748, 769, 754, 762] payload_rows_per_dst=[9510, 9226, 9384, 9570, 9394, 9230, 8980, 9084] dup_rows_per_dst=[3060, 2916, 3084, 3292, 3054, 2860, 2732, 2928] combine_rows_per_src=[9510, 9226, 9384, 9570, 9394, 9230, 8980, 9084] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=65120 dup_rows=23992 dup_rows_remote=20880 dedup_ratio=0.244059 router_tokens_per_expert=[793, 751, 788, 752, 793, 772, 777, 765, 750, 782, 755, 808, 751, 757, 737, 800, 734, 737, 752, 763, 769, 788, 756, 792, 768, 758, 810, 708, 749, 765, 766, 739, 776, 730, 745, 796, 792, 731, 770, 812, 749, 759, 787, 786, 816, 784, 792, 799, 751, 758, 768, 781, 765, 731, 805, 784, 751, 765, 767, 731, 787, 751, 760, 788] payload_rows_per_dst=[9384, 9342, 9246, 9150, 9192, 9454, 9322, 9222] dup_rows_per_dst=[2998, 2938, 2936, 2976, 3112, 3090, 2964, 2978] combine_rows_per_src=[9384, 9342, 9246, 9150, 9192, 9454, 9322, 9222] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74066 payload_rows_remote=64670 dup_rows=24238 dup_rows_remote=21172 dedup_ratio=0.246562 router_tokens_per_expert=[759, 802, 776, 762, 795, 770, 806, 775, 757, 734, 790, 759, 757, 752, 741, 722, 771, 745, 766, 794, 736, 802, 785, 755, 761, 834, 781, 732, 788, 751, 791, 798, 774, 775, 750, 753, 699, 768, 775, 762, 798, 780, 753, 811, 771, 819, 759, 807, 753, 761, 753, 751, 749, 816, 706, 739, 791, 776, 786, 779, 744, 746, 749, 752] payload_rows_per_dst=[9288, 9108, 9318, 9076, 9078, 9452, 9350, 9396] dup_rows_per_dst=[2928, 2916, 2990, 2814, 3034, 3202, 3288, 3066] combine_rows_per_src=[9288, 9108, 9318, 9076, 9078, 9452, 9350, 9396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=64908 dup_rows=24050 dup_rows_remote=20910 dedup_ratio=0.244649 router_tokens_per_expert=[778, 798, 719, 790, 815, 754, 759, 721, 741, 811, 752, 696, 787, 739, 775, 744, 766, 765, 792, 780, 736, 732, 769, 762, 751, 786, 807, 745, 832, 810, 810, 729, 755, 790, 771, 749, 789, 795, 790, 775, 776, 776, 787, 797, 793, 761, 766, 787, 832, 795, 749, 718, 756, 774, 754, 732, 787, 725, 755, 739, 780, 730, 751, 767] payload_rows_per_dst=[9316, 9152, 9234, 9472, 9298, 9346, 9290, 9146] dup_rows_per_dst=[2952, 2938, 2970, 3068, 3130, 3140, 2930, 2922] combine_rows_per_src=[9316, 9152, 9234, 9472, 9298, 9346, 9290, 9146] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265748480 dispatch_meta_bytes=343472 combine_payload_bytes=265748480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266091952 ingress_bytes=300351488 link_bytes=300351488 token_bytes=531840432 weight_movement_bytes=34603008 total_nvlink_bytes=566443440 weight_to_token_ratio=0.065063 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130198469e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266125312 dispatch_meta_bytes=343512 combine_payload_bytes=266125312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266468824 ingress_bytes=300728320 link_bytes=300728320 token_bytes=532594136 weight_movement_bytes=34603008 total_nvlink_bytes=567197144 weight_to_token_ratio=0.064971 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131616458e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=64922 dup_rows=24032 dup_rows_remote=20896 dedup_ratio=0.244466 router_tokens_per_expert=[725, 752, 734, 749, 817, 767, 766, 777, 783, 794, 760, 805, 824, 780, 738, 759, 767, 794, 798, 762, 742, 737, 775, 795, 784, 766, 766, 793, 842, 786, 781, 730, 798, 744, 764, 798, 765, 749, 767, 755, 726, 790, 809, 746, 747, 747, 712, 742, 785, 745, 743, 742, 720, 772, 736, 771, 793, 790, 794, 743, 736, 754, 777, 844] payload_rows_per_dst=[9308, 9350, 9244, 9340, 9244, 9144, 9166, 9476] dup_rows_per_dst=[2866, 3136, 3096, 3156, 3036, 2894, 2862, 2986] combine_rows_per_src=[9308, 9350, 9244, 9340, 9244, 9144, 9166, 9476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=343944 combine_payload_bytes=265756672 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266100616 ingress_bytes=369565696 link_bytes=369565696 token_bytes=531857288 weight_movement_bytes=103809024 total_nvlink_bytes=635666312 weight_to_token_ratio=0.195182 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390645962e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266772480 dispatch_meta_bytes=343704 combine_payload_bytes=266772480 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267116184 ingress_bytes=335978496 link_bytes=335978496 token_bytes=533888664 weight_movement_bytes=69206016 total_nvlink_bytes=603094680 weight_to_token_ratio=0.129626 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264260032e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265699328 dispatch_meta_bytes=342936 combine_payload_bytes=265699328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266042264 ingress_bytes=300302336 link_bytes=300302336 token_bytes=531741592 weight_movement_bytes=34603008 total_nvlink_bytes=566344600 weight_to_token_ratio=0.065075 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130013514e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=344000 combine_payload_bytes=266731520 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267075520 ingress_bytes=335937536 link_bytes=335937536 token_bytes=533807040 weight_movement_bytes=69206016 total_nvlink_bytes=603013056 weight_to_token_ratio=0.129646 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264105903e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265863168 dispatch_meta_bytes=343272 combine_payload_bytes=265863168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266206440 ingress_bytes=335069184 link_bytes=335069184 token_bytes=532069608 weight_movement_bytes=69206016 total_nvlink_bytes=601275624 weight_to_token_ratio=0.130069 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260838364e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=264888320 dispatch_meta_bytes=343368 combine_payload_bytes=264888320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265231688 ingress_bytes=299491328 link_bytes=299491328 token_bytes=530120008 weight_movement_bytes=34603008 total_nvlink_bytes=564723016 weight_to_token_ratio=0.065274 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126961756e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74348 payload_rows_remote=65080 dup_rows=23956 dup_rows_remote=21050 dedup_ratio=0.243693 router_tokens_per_expert=[772, 775, 742, 770, 781, 786, 792, 761, 790, 771, 848, 760, 738, 736, 776, 729, 730, 770, 720, 755, 754, 762, 836, 760, 737, 799, 747, 817, 742, 778, 750, 753, 767, 786, 765, 770, 773, 732, 792, 801, 755, 775, 717, 745, 742, 768, 784, 749, 751, 819, 775, 744, 800, 799, 792, 799, 717, 780, 771, 792, 778, 769, 735, 773] payload_rows_per_dst=[9352, 9216, 9268, 9226, 9372, 9238, 9408, 9268] dup_rows_per_dst=[3006, 3080, 2906, 3020, 3000, 2832, 3150, 2962] combine_rows_per_src=[9352, 9216, 9268, 9226, 9372, 9238, 9408, 9268] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74472 payload_rows_remote=65230 dup_rows=23832 dup_rows_remote=21030 dedup_ratio=0.242432 router_tokens_per_expert=[730, 785, 769, 761, 734, 803, 796, 779, 798, 732, 743, 769, 785, 830, 791, 777, 768, 753, 821, 765, 763, 769, 749, 766, 785, 752, 776, 770, 767, 760, 757, 797, 812, 745, 789, 732, 800, 788, 769, 799, 756, 794, 746, 740, 761, 746, 794, 814, 820, 803, 722, 746, 728, 717, 749, 737, 738, 739, 720, 780, 748, 746, 772, 802] payload_rows_per_dst=[9404, 9350, 9314, 9278, 9370, 9354, 9242, 9160] dup_rows_per_dst=[2910, 3100, 2994, 3050, 3098, 2948, 2802, 2930] combine_rows_per_src=[9404, 9350, 9314, 9278, 9370, 9354, 9242, 9160] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74426 payload_rows_remote=65120 dup_rows=23878 dup_rows_remote=20804 dedup_ratio=0.242900 router_tokens_per_expert=[765, 784, 767, 746, 766, 768, 768, 749, 758, 741, 794, 759, 730, 796, 768, 809, 747, 767, 748, 785, 709, 753, 814, 809, 796, 737, 710, 766, 746, 777, 780, 758, 787, 786, 746, 771, 767, 806, 790, 734, 799, 772, 784, 768, 777, 755, 802, 733, 748, 779, 756, 784, 717, 831, 753, 760, 804, 742, 769, 763, 761, 820, 720, 798] payload_rows_per_dst=[9260, 9378, 9272, 9208, 9332, 9306, 9338, 9332] dup_rows_per_dst=[2966, 2932, 2992, 2932, 3042, 3074, 2918, 3022] combine_rows_per_src=[9260, 9378, 9272, 9208, 9332, 9306, 9338, 9332] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74230 payload_rows_remote=64834 dup_rows=24074 dup_rows_remote=20912 dedup_ratio=0.244893 router_tokens_per_expert=[732, 741, 785, 796, 776, 774, 778, 788, 772, 761, 783, 762, 779, 750, 773, 767, 768, 792, 755, 741, 781, 769, 792, 713, 715, 760, 731, 777, 732, 769, 754, 801, 764, 796, 726, 740, 788, 789, 804, 778, 795, 799, 776, 793, 776, 773, 756, 745, 792, 779, 763, 785, 743, 751, 728, 850, 754, 787, 725, 763, 753, 732, 810, 772] payload_rows_per_dst=[9306, 9280, 9270, 9120, 9188, 9276, 9394, 9396] dup_rows_per_dst=[3034, 3014, 2936, 2958, 2816, 3150, 3004, 3162] combine_rows_per_src=[9306, 9280, 9270, 9120, 9188, 9276, 9394, 9396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265920512 dispatch_meta_bytes=343272 combine_payload_bytes=265920512 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266263784 ingress_bytes=335126528 link_bytes=335126528 token_bytes=532184296 weight_movement_bytes=69206016 total_nvlink_bytes=601390312 weight_to_token_ratio=0.130041 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261054145e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=64922 dup_rows=24032 dup_rows_remote=20896 dedup_ratio=0.244466 router_tokens_per_expert=[725, 752, 734, 749, 817, 767, 766, 777, 783, 794, 760, 805, 824, 780, 738, 759, 767, 794, 798, 762, 742, 737, 775, 795, 784, 766, 766, 793, 842, 786, 781, 730, 798, 744, 764, 798, 765, 749, 767, 755, 726, 790, 809, 746, 747, 747, 712, 742, 785, 745, 743, 742, 720, 772, 736, 771, 793, 790, 794, 743, 736, 754, 777, 844] payload_rows_per_dst=[9308, 9350, 9244, 9340, 9244, 9144, 9166, 9476] dup_rows_per_dst=[2866, 3136, 3096, 3156, 3036, 2894, 2862, 2986] combine_rows_per_src=[9308, 9350, 9244, 9340, 9244, 9144, 9166, 9476] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266567680 dispatch_meta_bytes=344520 combine_payload_bytes=266567680 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266912200 ingress_bytes=301170688 link_bytes=301170688 token_bytes=533479880 weight_movement_bytes=34603008 total_nvlink_bytes=568082888 weight_to_token_ratio=0.064863 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133281053e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=64950 dup_rows=24046 dup_rows_remote=20904 dedup_ratio=0.244609 router_tokens_per_expert=[757, 746, 754, 704, 795, 806, 767, 748, 748, 728, 740, 765, 753, 777, 776, 780, 805, 811, 796, 758, 767, 780, 785, 817, 763, 778, 791, 786, 760, 776, 824, 712, 776, 749, 790, 753, 773, 780, 808, 796, 754, 744, 778, 749, 803, 733, 768, 765, 785, 756, 730, 772, 738, 747, 768, 832, 773, 740, 790, 760, 746, 738, 744, 761] payload_rows_per_dst=[9248, 9236, 9444, 9366, 9308, 9224, 9276, 9156] dup_rows_per_dst=[2906, 2898, 3194, 3014, 3142, 2964, 2980, 2948] combine_rows_per_src=[9248, 9236, 9444, 9366, 9308, 9224, 9276, 9156] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=267182080 dispatch_meta_bytes=345040 combine_payload_bytes=267182080 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267527120 ingress_bytes=336388096 link_bytes=336388096 token_bytes=534709200 weight_movement_bytes=69206016 total_nvlink_bytes=603915216 weight_to_token_ratio=0.129427 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265801324e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=343696 combine_payload_bytes=266731520 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267075216 ingress_bytes=335937536 link_bytes=335937536 token_bytes=533806736 weight_movement_bytes=69206016 total_nvlink_bytes=603012752 weight_to_token_ratio=0.129646 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264105903e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=65028 dup_rows=23974 dup_rows_remote=21058 dedup_ratio=0.243876 router_tokens_per_expert=[731, 725, 732, 772, 767, 729, 752, 748, 740, 792, 719, 773, 748, 787, 767, 768, 756, 759, 797, 809, 777, 807, 777, 827, 757, 783, 736, 753, 808, 748, 775, 784, 757, 755, 772, 754, 775, 758, 754, 760, 730, 791, 781, 784, 782, 800, 731, 806, 756, 750, 781, 789, 712, 773, 835, 813, 791, 748, 792, 757, 756, 753, 792, 761] payload_rows_per_dst=[9302, 9062, 9462, 9514, 9278, 9066, 9344, 9302] dup_rows_per_dst=[2916, 2936, 3156, 3170, 2940, 2832, 3026, 2998] combine_rows_per_src=[9302, 9062, 9462, 9514, 9278, 9066, 9344, 9302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74346 payload_rows_remote=65098 dup_rows=23958 dup_rows_remote=21000 dedup_ratio=0.243713 router_tokens_per_expert=[732, 774, 809, 766, 766, 778, 765, 756, 762, 761, 752, 786, 720, 810, 766, 738, 762, 809, 769, 746, 768, 772, 725, 755, 757, 795, 772, 757, 741, 770, 732, 779, 756, 727, 764, 742, 790, 801, 822, 740, 795, 765, 769, 767, 831, 826, 796, 774, 767, 760, 745, 780, 817, 769, 770, 774, 760, 765, 741, 743, 742, 785, 754, 765] payload_rows_per_dst=[9248, 9360, 9270, 9248, 9322, 9504, 9276, 9118] dup_rows_per_dst=[3044, 2830, 2942, 2958, 2962, 3142, 3088, 2992] combine_rows_per_src=[9248, 9360, 9270, 9248, 9322, 9504, 9276, 9118] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265560064 dispatch_meta_bytes=342984 combine_payload_bytes=265560064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265903048 ingress_bytes=300163072 link_bytes=300163072 token_bytes=531463112 weight_movement_bytes=34603008 total_nvlink_bytes=566066120 weight_to_token_ratio=0.065109 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129489475e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64882 dup_rows=24114 dup_rows_remote=21104 dedup_ratio=0.245300 router_tokens_per_expert=[813, 792, 765, 794, 806, 711, 755, 711, 823, 752, 772, 789, 781, 724, 707, 804, 751, 767, 763, 807, 775, 751, 806, 744, 751, 819, 755, 757, 736, 711, 743, 786, 786, 796, 803, 805, 755, 800, 772, 808, 757, 810, 757, 795, 725, 735, 742, 743, 763, 733, 788, 799, 771, 761, 732, 798, 795, 743, 780, 718, 761, 778, 774, 748] payload_rows_per_dst=[9308, 9464, 9314, 9226, 9402, 9002, 9200, 9274] dup_rows_per_dst=[3010, 3184, 2994, 3072, 3062, 2782, 3090, 2920] combine_rows_per_src=[9308, 9464, 9314, 9226, 9402, 9002, 9200, 9274] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=65088 dup_rows=24050 dup_rows_remote=21118 dedup_ratio=0.244649 router_tokens_per_expert=[796, 811, 742, 810, 805, 743, 762, 777, 776, 724, 768, 772, 726, 827, 754, 852, 754, 763, 809, 806, 806, 727, 687, 739, 762, 758, 771, 746, 803, 763, 791, 774, 750, 758, 730, 777, 765, 749, 765, 755, 749, 760, 854, 761, 808, 810, 758, 785, 780, 752, 773, 758, 770, 723, 744, 793, 748, 767, 735, 747, 727, 775, 782, 740] payload_rows_per_dst=[9374, 9322, 9268, 9310, 9166, 9480, 9170, 9164] dup_rows_per_dst=[3118, 3076, 2914, 3026, 2932, 3090, 3016, 2878] combine_rows_per_src=[9374, 9322, 9268, 9310, 9166, 9480, 9170, 9164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74426 payload_rows_remote=65120 dup_rows=23878 dup_rows_remote=20804 dedup_ratio=0.242900 router_tokens_per_expert=[765, 784, 767, 746, 766, 768, 768, 749, 758, 741, 794, 759, 730, 796, 768, 809, 747, 767, 748, 785, 709, 753, 814, 809, 796, 737, 710, 766, 746, 777, 780, 758, 787, 786, 746, 771, 767, 806, 790, 734, 799, 772, 784, 768, 777, 755, 802, 733, 748, 779, 756, 784, 717, 831, 753, 760, 804, 742, 769, 763, 761, 820, 720, 798] payload_rows_per_dst=[9260, 9378, 9272, 9208, 9332, 9306, 9338, 9332] dup_rows_per_dst=[2966, 2932, 2992, 2932, 3042, 3074, 2918, 3022] combine_rows_per_src=[9260, 9378, 9272, 9208, 9332, 9306, 9338, 9332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74230 payload_rows_remote=64834 dup_rows=24074 dup_rows_remote=20912 dedup_ratio=0.244893 router_tokens_per_expert=[732, 741, 785, 796, 776, 774, 778, 788, 772, 761, 783, 762, 779, 750, 773, 767, 768, 792, 755, 741, 781, 769, 792, 713, 715, 760, 731, 777, 732, 769, 754, 801, 764, 796, 726, 740, 788, 789, 804, 778, 795, 799, 776, 793, 776, 773, 756, 745, 792, 779, 763, 785, 743, 751, 728, 850, 754, 787, 725, 763, 753, 732, 810, 772] payload_rows_per_dst=[9306, 9280, 9270, 9120, 9188, 9276, 9394, 9396] dup_rows_per_dst=[3034, 3014, 2936, 2958, 2816, 3150, 3004, 3162] combine_rows_per_src=[9306, 9280, 9270, 9120, 9188, 9276, 9394, 9396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266035200 dispatch_meta_bytes=343416 combine_payload_bytes=266035200 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266378616 ingress_bytes=335241216 link_bytes=335241216 token_bytes=532413816 weight_movement_bytes=69206016 total_nvlink_bytes=601619832 weight_to_token_ratio=0.129985 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261485707e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64820 dup_rows=24114 dup_rows_remote=21082 dedup_ratio=0.245300 router_tokens_per_expert=[821, 749, 749, 778, 751, 803, 770, 745, 756, 780, 800, 748, 789, 759, 793, 776, 758, 768, 788, 759, 786, 728, 760, 793, 762, 759, 802, 744, 780, 739, 762, 779, 711, 767, 780, 742, 763, 753, 788, 787, 840, 784, 780, 760, 747, 780, 754, 782, 733, 759, 733, 782, 769, 741, 811, 765, 775, 748, 788, 745, 768, 745, 773, 765] payload_rows_per_dst=[9374, 9370, 9292, 9204, 9194, 9286, 9276, 9194] dup_rows_per_dst=[2958, 3032, 2988, 3050, 2988, 3168, 2910, 3020] combine_rows_per_src=[9374, 9370, 9292, 9204, 9194, 9286, 9276, 9194] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265920512 dispatch_meta_bytes=343272 combine_payload_bytes=265920512 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266263784 ingress_bytes=335126528 link_bytes=335126528 token_bytes=532184296 weight_movement_bytes=69206016 total_nvlink_bytes=601390312 weight_to_token_ratio=0.130041 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261054145e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=344392 combine_payload_bytes=266641408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266985800 ingress_bytes=301244416 link_bytes=301244416 token_bytes=533627208 weight_movement_bytes=34603008 total_nvlink_bytes=568230216 weight_to_token_ratio=0.064845 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133558485e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=343944 combine_payload_bytes=265756672 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266100616 ingress_bytes=369565696 link_bytes=369565696 token_bytes=531857288 weight_movement_bytes=103809024 total_nvlink_bytes=635666312 weight_to_token_ratio=0.195182 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390645962e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=344824 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266945272 ingress_bytes=301203456 link_bytes=301203456 token_bytes=533545720 weight_movement_bytes=34603008 total_nvlink_bytes=568148728 weight_to_token_ratio=0.064855 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133404356e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266354688 dispatch_meta_bytes=344344 combine_payload_bytes=266354688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266699032 ingress_bytes=300957696 link_bytes=300957696 token_bytes=533053720 weight_movement_bytes=34603008 total_nvlink_bytes=567656728 weight_to_token_ratio=0.064915 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132479581e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74120 payload_rows_remote=64754 dup_rows=24184 dup_rows_remote=21076 dedup_ratio=0.246012 router_tokens_per_expert=[778, 758, 721, 786, 721, 775, 755, 790, 768, 788, 768, 787, 751, 773, 754, 770, 751, 802, 827, 792, 765, 755, 763, 782, 806, 752, 703, 776, 796, 762, 775, 798, 785, 738, 835, 812, 767, 750, 726, 771, 731, 782, 797, 765, 714, 830, 768, 794, 769, 735, 768, 724, 756, 758, 768, 725, 748, 775, 742, 778, 770, 763, 792, 768] payload_rows_per_dst=[9228, 9224, 9366, 9300, 9344, 9310, 9170, 9178] dup_rows_per_dst=[2940, 3094, 3108, 3036, 3024, 3052, 2836, 3094] combine_rows_per_src=[9228, 9224, 9366, 9300, 9344, 9310, 9170, 9178] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74408 payload_rows_remote=65186 dup_rows=23896 dup_rows_remote=20936 dedup_ratio=0.243083 router_tokens_per_expert=[756, 755, 786, 739, 787, 779, 762, 744, 758, 766, 768, 801, 773, 784, 743, 772, 766, 766, 725, 730, 806, 809, 816, 811, 778, 752, 755, 795, 788, 786, 796, 740, 807, 784, 791, 769, 770, 721, 759, 751, 754, 755, 738, 784, 749, 777, 769, 765, 739, 783, 808, 769, 794, 749, 776, 734, 745, 783, 743, 787, 749, 755, 752, 751] payload_rows_per_dst=[9242, 9360, 9404, 9336, 9364, 9222, 9250, 9230] dup_rows_per_dst=[2974, 2970, 3054, 3044, 2940, 2960, 3054, 2900] combine_rows_per_src=[9242, 9360, 9404, 9336, 9364, 9222, 9250, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265560064 dispatch_meta_bytes=342984 combine_payload_bytes=265560064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265903048 ingress_bytes=300163072 link_bytes=300163072 token_bytes=531463112 weight_movement_bytes=34603008 total_nvlink_bytes=566066120 weight_to_token_ratio=0.065109 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129489475e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=344392 combine_payload_bytes=266641408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266985800 ingress_bytes=301244416 link_bytes=301244416 token_bytes=533627208 weight_movement_bytes=34603008 total_nvlink_bytes=568230216 weight_to_token_ratio=0.064845 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133558485e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=65044 dup_rows=23948 dup_rows_remote=21066 dedup_ratio=0.243612 router_tokens_per_expert=[814, 739, 795, 760, 792, 754, 783, 758, 802, 741, 796, 779, 760, 745, 794, 768, 770, 762, 759, 771, 755, 728, 735, 754, 808, 783, 785, 778, 772, 797, 804, 783, 744, 739, 786, 755, 785, 773, 772, 762, 774, 775, 806, 747, 779, 754, 745, 760, 743, 790, 778, 770, 790, 738, 731, 757, 752, 795, 784, 690, 764, 721, 782, 787] payload_rows_per_dst=[9418, 9308, 9202, 9420, 9220, 9284, 9312, 9192] dup_rows_per_dst=[2972, 3062, 2866, 3200, 3012, 2996, 2882, 2958] combine_rows_per_src=[9418, 9308, 9202, 9420, 9220, 9284, 9312, 9192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74430 payload_rows_remote=64968 dup_rows=23874 dup_rows_remote=20604 dedup_ratio=0.242859 router_tokens_per_expert=[777, 796, 754, 757, 744, 742, 722, 777, 775, 779, 789, 806, 787, 780, 757, 750, 750, 813, 803, 760, 830, 736, 758, 793, 738, 779, 790, 773, 715, 796, 741, 761, 790, 774, 766, 727, 735, 816, 770, 730, 788, 752, 751, 768, 769, 778, 764, 758, 800, 734, 768, 795, 742, 780, 801, 809, 740, 765, 732, 733, 789, 775, 761, 764] payload_rows_per_dst=[9194, 9124, 9422, 8920, 9452, 9364, 9492, 9462] dup_rows_per_dst=[2818, 2900, 3102, 2652, 3186, 2980, 2966, 3270] combine_rows_per_src=[9194, 9124, 9422, 8920, 9452, 9364, 9492, 9462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=343696 combine_payload_bytes=266731520 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267075216 ingress_bytes=335937536 link_bytes=335937536 token_bytes=533806736 weight_movement_bytes=69206016 total_nvlink_bytes=603012752 weight_to_token_ratio=0.129646 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264105903e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=343608 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265846328 ingress_bytes=334708736 link_bytes=334708736 token_bytes=531349048 weight_movement_bytes=69206016 total_nvlink_bytes=600555064 weight_to_token_ratio=0.130246 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259482027e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265232384 dispatch_meta_bytes=343320 combine_payload_bytes=265232384 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265575704 ingress_bytes=334438400 link_bytes=334438400 token_bytes=530808088 weight_movement_bytes=69206016 total_nvlink_bytes=600014104 weight_to_token_ratio=0.130379 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258464775e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=267001856 dispatch_meta_bytes=344488 combine_payload_bytes=267001856 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267346344 ingress_bytes=336207872 link_bytes=336207872 token_bytes=534348200 weight_movement_bytes=69206016 total_nvlink_bytes=603554216 weight_to_token_ratio=0.129515 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265123155e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=65060 dup_rows=24050 dup_rows_remote=20986 dedup_ratio=0.244649 router_tokens_per_expert=[787, 766, 778, 770, 798, 743, 768, 767, 718, 761, 770, 805, 762, 762, 759, 809, 754, 740, 768, 741, 763, 783, 779, 712, 772, 725, 774, 786, 732, 768, 790, 782, 771, 765, 745, 763, 747, 795, 712, 784, 781, 739, 766, 752, 777, 728, 813, 809, 787, 780, 755, 778, 753, 804, 795, 725, 794, 805, 778, 767, 749, 775, 784, 784] payload_rows_per_dst=[9406, 9274, 9196, 9194, 9214, 9258, 9278, 9434] dup_rows_per_dst=[2948, 3018, 2884, 3064, 2950, 3072, 3076, 3038] combine_rows_per_src=[9406, 9274, 9196, 9194, 9214, 9258, 9278, 9434] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64820 dup_rows=24114 dup_rows_remote=21082 dedup_ratio=0.245300 router_tokens_per_expert=[821, 749, 749, 778, 751, 803, 770, 745, 756, 780, 800, 748, 789, 759, 793, 776, 758, 768, 788, 759, 786, 728, 760, 793, 762, 759, 802, 744, 780, 739, 762, 779, 711, 767, 780, 742, 763, 753, 788, 787, 840, 784, 780, 760, 747, 780, 754, 782, 733, 759, 733, 782, 769, 741, 811, 765, 775, 748, 788, 745, 768, 745, 773, 765] payload_rows_per_dst=[9374, 9370, 9292, 9204, 9194, 9286, 9276, 9194] dup_rows_per_dst=[2958, 3032, 2988, 3050, 2988, 3168, 2910, 3020] combine_rows_per_src=[9374, 9370, 9292, 9204, 9194, 9286, 9276, 9194] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=65120 dup_rows=23992 dup_rows_remote=20880 dedup_ratio=0.244059 router_tokens_per_expert=[793, 751, 788, 752, 793, 772, 777, 765, 750, 782, 755, 808, 751, 757, 737, 800, 734, 737, 752, 763, 769, 788, 756, 792, 768, 758, 810, 708, 749, 765, 766, 739, 776, 730, 745, 796, 792, 731, 770, 812, 749, 759, 787, 786, 816, 784, 792, 799, 751, 758, 768, 781, 765, 731, 805, 784, 751, 765, 767, 731, 787, 751, 760, 788] payload_rows_per_dst=[9384, 9342, 9246, 9150, 9192, 9454, 9322, 9222] dup_rows_per_dst=[2998, 2938, 2936, 2976, 3112, 3090, 2964, 2978] combine_rows_per_src=[9384, 9342, 9246, 9150, 9192, 9454, 9322, 9222] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=344440 combine_payload_bytes=266420224 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266764664 ingress_bytes=301023232 link_bytes=301023232 token_bytes=533184888 weight_movement_bytes=34603008 total_nvlink_bytes=567787896 weight_to_token_ratio=0.064899 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132726188e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=342288 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266451216 ingress_bytes=300711936 link_bytes=300711936 token_bytes=532560144 weight_movement_bytes=34603008 total_nvlink_bytes=567163152 weight_to_token_ratio=0.064975 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131554806e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64852 dup_rows=24114 dup_rows_remote=21112 dedup_ratio=0.245300 router_tokens_per_expert=[765, 807, 769, 763, 750, 727, 830, 755, 725, 745, 803, 758, 745, 735, 761, 725, 811, 756, 779, 747, 821, 749, 792, 752, 783, 745, 762, 784, 759, 783, 735, 762, 764, 742, 815, 770, 770, 750, 804, 744, 733, 771, 730, 750, 821, 748, 768, 731, 792, 827, 791, 761, 760, 806, 750, 825, 782, 751, 731, 777, 780, 780, 760, 785] payload_rows_per_dst=[9338, 9188, 9248, 9296, 9318, 9020, 9510, 9272] dup_rows_per_dst=[3002, 2982, 2990, 3252, 3000, 2762, 3106, 3020] combine_rows_per_src=[9338, 9188, 9248, 9296, 9318, 9020, 9510, 9272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=344000 combine_payload_bytes=266731520 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267075520 ingress_bytes=335937536 link_bytes=335937536 token_bytes=533807040 weight_movement_bytes=69206016 total_nvlink_bytes=603013056 weight_to_token_ratio=0.129646 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264105903e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74430 payload_rows_remote=64968 dup_rows=23874 dup_rows_remote=20604 dedup_ratio=0.242859 router_tokens_per_expert=[777, 796, 754, 757, 744, 742, 722, 777, 775, 779, 789, 806, 787, 780, 757, 750, 750, 813, 803, 760, 830, 736, 758, 793, 738, 779, 790, 773, 715, 796, 741, 761, 790, 774, 766, 727, 735, 816, 770, 730, 788, 752, 751, 768, 769, 778, 764, 758, 800, 734, 768, 795, 742, 780, 801, 809, 740, 765, 732, 733, 789, 775, 761, 764] payload_rows_per_dst=[9194, 9124, 9422, 8920, 9452, 9364, 9492, 9462] dup_rows_per_dst=[2818, 2900, 3102, 2652, 3186, 2980, 2966, 3270] combine_rows_per_src=[9194, 9124, 9422, 8920, 9452, 9364, 9492, 9462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=65060 dup_rows=24050 dup_rows_remote=20986 dedup_ratio=0.244649 router_tokens_per_expert=[787, 766, 778, 770, 798, 743, 768, 767, 718, 761, 770, 805, 762, 762, 759, 809, 754, 740, 768, 741, 763, 783, 779, 712, 772, 725, 774, 786, 732, 768, 790, 782, 771, 765, 745, 763, 747, 795, 712, 784, 781, 739, 766, 752, 777, 728, 813, 809, 787, 780, 755, 778, 753, 804, 795, 725, 794, 805, 778, 767, 749, 775, 784, 784] payload_rows_per_dst=[9406, 9274, 9196, 9194, 9214, 9258, 9278, 9434] dup_rows_per_dst=[2948, 3018, 2884, 3064, 2950, 3072, 3076, 3038] combine_rows_per_src=[9406, 9274, 9196, 9194, 9214, 9258, 9278, 9434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74414 payload_rows_remote=64950 dup_rows=23890 dup_rows_remote=20754 dedup_ratio=0.243022 router_tokens_per_expert=[758, 749, 784, 725, 741, 842, 756, 763, 779, 784, 802, 743, 822, 781, 793, 796, 794, 746, 799, 761, 773, 799, 766, 695, 816, 769, 760, 784, 780, 747, 773, 754, 789, 775, 747, 785, 734, 769, 801, 774, 780, 752, 729, 774, 736, 711, 766, 748, 708, 785, 808, 757, 748, 779, 748, 787, 755, 814, 759, 761, 766, 780, 763, 730] payload_rows_per_dst=[9180, 9464, 9330, 9390, 9306, 9142, 9290, 9312] dup_rows_per_dst=[3056, 3136, 2936, 2976, 3042, 2850, 2950, 2944] combine_rows_per_src=[9180, 9464, 9330, 9390, 9306, 9142, 9290, 9312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74408 payload_rows_remote=65186 dup_rows=23896 dup_rows_remote=20936 dedup_ratio=0.243083 router_tokens_per_expert=[756, 755, 786, 739, 787, 779, 762, 744, 758, 766, 768, 801, 773, 784, 743, 772, 766, 766, 725, 730, 806, 809, 816, 811, 778, 752, 755, 795, 788, 786, 796, 740, 807, 784, 791, 769, 770, 721, 759, 751, 754, 755, 738, 784, 749, 777, 769, 765, 739, 783, 808, 769, 794, 749, 776, 734, 745, 783, 743, 787, 749, 755, 752, 751] payload_rows_per_dst=[9242, 9360, 9404, 9336, 9364, 9222, 9250, 9230] dup_rows_per_dst=[2974, 2970, 3054, 3044, 2940, 2960, 3054, 2900] combine_rows_per_src=[9242, 9360, 9404, 9336, 9364, 9222, 9250, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265633792 dispatch_meta_bytes=343856 combine_payload_bytes=265633792 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265977648 ingress_bytes=369442816 link_bytes=369442816 token_bytes=531611440 weight_movement_bytes=103809024 total_nvlink_bytes=635420464 weight_to_token_ratio=0.195272 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390183574e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74542 payload_rows_remote=65134 dup_rows=23762 dup_rows_remote=20718 dedup_ratio=0.241720 router_tokens_per_expert=[779, 739, 784, 753, 769, 755, 766, 776, 784, 736, 735, 789, 743, 766, 817, 729, 757, 800, 807, 774, 736, 767, 815, 743, 782, 720, 744, 789, 740, 784, 816, 783, 797, 778, 742, 760, 757, 792, 815, 785, 746, 793, 773, 752, 747, 772, 770, 779, 803, 733, 758, 729, 700, 756, 761, 808, 761, 779, 811, 777, 748, 775, 750, 768] payload_rows_per_dst=[9306, 9234, 9350, 9306, 9408, 9318, 9262, 9358] dup_rows_per_dst=[2936, 2964, 3048, 3010, 3044, 2946, 2834, 2980] combine_rows_per_src=[9306, 9234, 9350, 9306, 9408, 9318, 9262, 9358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=65028 dup_rows=23974 dup_rows_remote=21058 dedup_ratio=0.243876 router_tokens_per_expert=[731, 725, 732, 772, 767, 729, 752, 748, 740, 792, 719, 773, 748, 787, 767, 768, 756, 759, 797, 809, 777, 807, 777, 827, 757, 783, 736, 753, 808, 748, 775, 784, 757, 755, 772, 754, 775, 758, 754, 760, 730, 791, 781, 784, 782, 800, 731, 806, 756, 750, 781, 789, 712, 773, 835, 813, 791, 748, 792, 757, 756, 753, 792, 761] payload_rows_per_dst=[9302, 9062, 9462, 9514, 9278, 9066, 9344, 9302] dup_rows_per_dst=[2916, 2936, 3156, 3170, 2940, 2832, 3026, 2998] combine_rows_per_src=[9302, 9062, 9462, 9514, 9278, 9066, 9344, 9302] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266485760 dispatch_meta_bytes=344184 combine_payload_bytes=266485760 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266829944 ingress_bytes=335691776 link_bytes=335691776 token_bytes=533315704 weight_movement_bytes=69206016 total_nvlink_bytes=602521720 weight_to_token_ratio=0.129766 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263181128e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=343608 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265846328 ingress_bytes=334708736 link_bytes=334708736 token_bytes=531349048 weight_movement_bytes=69206016 total_nvlink_bytes=600555064 weight_to_token_ratio=0.130246 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259482027e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=64998 dup_rows=24096 dup_rows_remote=21130 dedup_ratio=0.245117 router_tokens_per_expert=[711, 775, 771, 727, 763, 837, 760, 780, 753, 779, 744, 754, 746, 811, 757, 767, 743, 732, 693, 776, 770, 836, 777, 761, 746, 774, 743, 776, 734, 734, 739, 749, 781, 749, 804, 789, 770, 753, 744, 728, 767, 788, 795, 793, 785, 758, 789, 779, 794, 769, 757, 747, 774, 804, 809, 780, 773, 761, 803, 794, 753, 757, 793, 794] payload_rows_per_dst=[9266, 9272, 9210, 9126, 9242, 9320, 9346, 9426] dup_rows_per_dst=[2982, 2950, 2966, 2864, 2994, 3188, 3122, 3030] combine_rows_per_src=[9266, 9272, 9210, 9126, 9242, 9320, 9346, 9426] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=343408 combine_payload_bytes=266788864 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267132272 ingress_bytes=370597888 link_bytes=370597888 token_bytes=533921136 weight_movement_bytes=103809024 total_nvlink_bytes=637730160 weight_to_token_ratio=0.194428 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394530017e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266354688 dispatch_meta_bytes=344344 combine_payload_bytes=266354688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266699032 ingress_bytes=300957696 link_bytes=300957696 token_bytes=533053720 weight_movement_bytes=34603008 total_nvlink_bytes=567656728 weight_to_token_ratio=0.064915 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132479581e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74458 payload_rows_remote=65108 dup_rows=23846 dup_rows_remote=20922 dedup_ratio=0.242574 router_tokens_per_expert=[765, 761, 787, 744, 753, 769, 713, 785, 800, 756, 765, 812, 788, 712, 801, 739, 778, 692, 777, 757, 749, 776, 777, 808, 794, 777, 757, 766, 761, 789, 720, 761, 761, 774, 769, 767, 818, 778, 719, 772, 770, 755, 808, 751, 797, 736, 779, 741, 788, 768, 747, 748, 757, 743, 785, 775, 782, 773, 776, 808, 784, 776, 755, 803] payload_rows_per_dst=[9206, 9374, 9284, 9270, 9262, 9350, 9322, 9390] dup_rows_per_dst=[2948, 2972, 2944, 2980, 3054, 2924, 2900, 3124] combine_rows_per_src=[9206, 9374, 9284, 9270, 9262, 9350, 9322, 9390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=342288 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266451216 ingress_bytes=300711936 link_bytes=300711936 token_bytes=532560144 weight_movement_bytes=34603008 total_nvlink_bytes=567163152 weight_to_token_ratio=0.064975 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131554806e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74134 payload_rows_remote=65034 dup_rows=24170 dup_rows_remote=21436 dedup_ratio=0.245870 router_tokens_per_expert=[790, 757, 756, 780, 753, 827, 816, 769, 781, 756, 769, 786, 731, 722, 736, 813, 774, 803, 747, 741, 798, 747, 764, 774, 784, 754, 748, 749, 733, 800, 776, 772, 750, 798, 744, 746, 751, 791, 768, 784, 761, 811, 789, 781, 782, 807, 778, 808, 737, 731, 717, 733, 761, 749, 725, 764, 760, 739, 746, 800, 794, 780, 768, 793] payload_rows_per_dst=[9382, 9224, 9250, 9200, 9290, 9400, 9100, 9288] dup_rows_per_dst=[3114, 2964, 3046, 3032, 2974, 3234, 2734, 3072] combine_rows_per_src=[9382, 9224, 9250, 9200, 9290, 9400, 9100, 9288] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74508 payload_rows_remote=65150 dup_rows=23796 dup_rows_remote=20764 dedup_ratio=0.242065 router_tokens_per_expert=[767, 732, 742, 784, 699, 822, 756, 703, 783, 779, 786, 710, 772, 745, 753, 768, 766, 742, 723, 777, 764, 726, 790, 728, 816, 777, 795, 807, 788, 736, 793, 826, 798, 754, 788, 789, 754, 769, 773, 792, 783, 759, 731, 784, 731, 720, 811, 774, 776, 744, 724, 787, 803, 792, 797, 773, 760, 768, 764, 761, 783, 763, 818, 774] payload_rows_per_dst=[9160, 9222, 9188, 9498, 9422, 9294, 9366, 9358] dup_rows_per_dst=[2850, 2970, 2844, 3178, 3004, 2892, 3026, 3032] combine_rows_per_src=[9160, 9222, 9188, 9498, 9422, 9294, 9366, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266485760 dispatch_meta_bytes=344184 combine_payload_bytes=266485760 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266829944 ingress_bytes=335691776 link_bytes=335691776 token_bytes=533315704 weight_movement_bytes=69206016 total_nvlink_bytes=602521720 weight_to_token_ratio=0.129766 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263181128e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266035200 dispatch_meta_bytes=342816 combine_payload_bytes=266035200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266378016 ingress_bytes=300638208 link_bytes=300638208 token_bytes=532413216 weight_movement_bytes=34603008 total_nvlink_bytes=567016224 weight_to_token_ratio=0.064993 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131277373e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=267001856 dispatch_meta_bytes=344488 combine_payload_bytes=267001856 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267346344 ingress_bytes=336207872 link_bytes=336207872 token_bytes=534348200 weight_movement_bytes=69206016 total_nvlink_bytes=603554216 weight_to_token_ratio=0.129515 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265123155e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=65088 dup_rows=24050 dup_rows_remote=21118 dedup_ratio=0.244649 router_tokens_per_expert=[796, 811, 742, 810, 805, 743, 762, 777, 776, 724, 768, 772, 726, 827, 754, 852, 754, 763, 809, 806, 806, 727, 687, 739, 762, 758, 771, 746, 803, 763, 791, 774, 750, 758, 730, 777, 765, 749, 765, 755, 749, 760, 854, 761, 808, 810, 758, 785, 780, 752, 773, 758, 770, 723, 744, 793, 748, 767, 735, 747, 727, 775, 782, 740] payload_rows_per_dst=[9374, 9322, 9268, 9310, 9166, 9480, 9170, 9164] dup_rows_per_dst=[3118, 3076, 2914, 3026, 2932, 3090, 3016, 2878] combine_rows_per_src=[9374, 9322, 9268, 9310, 9166, 9480, 9170, 9164] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266231808 dispatch_meta_bytes=344512 combine_payload_bytes=266231808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266576320 ingress_bytes=300834816 link_bytes=300834816 token_bytes=532808128 weight_movement_bytes=34603008 total_nvlink_bytes=567411136 weight_to_token_ratio=0.064945 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132017193e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266682368 dispatch_meta_bytes=344120 combine_payload_bytes=266682368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267026488 ingress_bytes=301285376 link_bytes=301285376 token_bytes=533708856 weight_movement_bytes=34603008 total_nvlink_bytes=568311864 weight_to_token_ratio=0.064835 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133712614e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74140 payload_rows_remote=64866 dup_rows=24164 dup_rows_remote=21080 dedup_ratio=0.245809 router_tokens_per_expert=[735, 760, 754, 786, 748, 716, 794, 761, 769, 771, 790, 774, 735, 738, 727, 753, 802, 771, 777, 762, 751, 747, 786, 766, 763, 777, 736, 856, 727, 784, 779, 757, 784, 782, 761, 725, 763, 776, 819, 780, 772, 771, 781, 748, 766, 741, 739, 780, 796, 786, 760, 768, 762, 738, 777, 766, 763, 844, 762, 739, 772, 814, 765, 800] payload_rows_per_dst=[9200, 9104, 9180, 9274, 9304, 9300, 9352, 9426] dup_rows_per_dst=[2908, 3010, 3144, 3084, 3076, 2896, 2954, 3092] combine_rows_per_src=[9200, 9104, 9180, 9274, 9304, 9300, 9352, 9426] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266379264 dispatch_meta_bytes=345880 combine_payload_bytes=266379264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266725144 ingress_bytes=300982272 link_bytes=300982272 token_bytes=533104408 weight_movement_bytes=34603008 total_nvlink_bytes=567707416 weight_to_token_ratio=0.064909 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132572058e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266854400 dispatch_meta_bytes=343656 combine_payload_bytes=266854400 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=267198056 ingress_bytes=439869440 link_bytes=439869440 token_bytes=534052456 weight_movement_bytes=173015040 total_nvlink_bytes=707067496 weight_to_token_ratio=0.323966 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.655193290e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74414 payload_rows_remote=64950 dup_rows=23890 dup_rows_remote=20754 dedup_ratio=0.243022 router_tokens_per_expert=[758, 749, 784, 725, 741, 842, 756, 763, 779, 784, 802, 743, 822, 781, 793, 796, 794, 746, 799, 761, 773, 799, 766, 695, 816, 769, 760, 784, 780, 747, 773, 754, 789, 775, 747, 785, 734, 769, 801, 774, 780, 752, 729, 774, 736, 711, 766, 748, 708, 785, 808, 757, 748, 779, 748, 787, 755, 814, 759, 761, 766, 780, 763, 730] payload_rows_per_dst=[9180, 9464, 9330, 9390, 9306, 9142, 9290, 9312] dup_rows_per_dst=[3056, 3136, 2936, 2976, 3042, 2850, 2950, 2944] combine_rows_per_src=[9180, 9464, 9330, 9390, 9306, 9142, 9290, 9312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74166 payload_rows_remote=64586 dup_rows=24138 dup_rows_remote=20908 dedup_ratio=0.245544 router_tokens_per_expert=[751, 793, 756, 796, 774, 723, 791, 758, 778, 782, 763, 780, 784, 784, 756, 750, 744, 785, 752, 773, 799, 743, 785, 770, 738, 771, 777, 734, 767, 754, 746, 739, 774, 766, 747, 739, 778, 819, 738, 753, 780, 816, 702, 757, 777, 762, 712, 792, 797, 811, 801, 797, 769, 790, 781, 741, 760, 767, 775, 802, 749, 744, 785, 775] payload_rows_per_dst=[9580, 9302, 9436, 9172, 9078, 9296, 9572, 8730] dup_rows_per_dst=[3230, 3142, 3148, 2944, 2804, 2954, 3174, 2742] combine_rows_per_src=[9580, 9302, 9436, 9172, 9078, 9296, 9572, 8730] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=344824 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266945272 ingress_bytes=301203456 link_bytes=301203456 token_bytes=533545720 weight_movement_bytes=34603008 total_nvlink_bytes=568148728 weight_to_token_ratio=0.064855 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133404356e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74508 payload_rows_remote=65150 dup_rows=23796 dup_rows_remote=20764 dedup_ratio=0.242065 router_tokens_per_expert=[767, 732, 742, 784, 699, 822, 756, 703, 783, 779, 786, 710, 772, 745, 753, 768, 766, 742, 723, 777, 764, 726, 790, 728, 816, 777, 795, 807, 788, 736, 793, 826, 798, 754, 788, 789, 754, 769, 773, 792, 783, 759, 731, 784, 731, 720, 811, 774, 776, 744, 724, 787, 803, 792, 797, 773, 760, 768, 764, 761, 783, 763, 818, 774] payload_rows_per_dst=[9160, 9222, 9188, 9498, 9422, 9294, 9366, 9358] dup_rows_per_dst=[2850, 2970, 2844, 3178, 3004, 2892, 3026, 3032] combine_rows_per_src=[9160, 9222, 9188, 9498, 9422, 9294, 9366, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=65002 dup_rows=24016 dup_rows_remote=21004 dedup_ratio=0.244303 router_tokens_per_expert=[768, 763, 764, 760, 737, 792, 713, 756, 769, 854, 761, 728, 781, 749, 741, 766, 772, 811, 764, 759, 789, 800, 733, 754, 747, 751, 764, 756, 831, 780, 730, 782, 810, 749, 759, 710, 779, 723, 808, 767, 778, 775, 793, 769, 823, 728, 780, 747, 785, 743, 776, 803, 753, 752, 750, 765, 744, 765, 791, 751, 818, 777, 783, 773] payload_rows_per_dst=[9242, 9286, 9358, 9292, 9222, 9258, 9250, 9380] dup_rows_per_dst=[2864, 3012, 3006, 2990, 2988, 3128, 3004, 3024] combine_rows_per_src=[9242, 9286, 9358, 9292, 9222, 9258, 9250, 9380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74140 payload_rows_remote=64866 dup_rows=24164 dup_rows_remote=21080 dedup_ratio=0.245809 router_tokens_per_expert=[735, 760, 754, 786, 748, 716, 794, 761, 769, 771, 790, 774, 735, 738, 727, 753, 802, 771, 777, 762, 751, 747, 786, 766, 763, 777, 736, 856, 727, 784, 779, 757, 784, 782, 761, 725, 763, 776, 819, 780, 772, 771, 781, 748, 766, 741, 739, 780, 796, 786, 760, 768, 762, 738, 777, 766, 763, 844, 762, 739, 772, 814, 765, 800] payload_rows_per_dst=[9200, 9104, 9180, 9274, 9304, 9300, 9352, 9426] dup_rows_per_dst=[2908, 3010, 3144, 3084, 3076, 2896, 2954, 3092] combine_rows_per_src=[9200, 9104, 9180, 9274, 9304, 9300, 9352, 9426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=264544256 dispatch_meta_bytes=341976 combine_payload_bytes=264544256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264886232 ingress_bytes=299147264 link_bytes=299147264 token_bytes=529430488 weight_movement_bytes=34603008 total_nvlink_bytes=564033496 weight_to_token_ratio=0.065359 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125667071e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74458 payload_rows_remote=65108 dup_rows=23846 dup_rows_remote=20922 dedup_ratio=0.242574 router_tokens_per_expert=[765, 761, 787, 744, 753, 769, 713, 785, 800, 756, 765, 812, 788, 712, 801, 739, 778, 692, 777, 757, 749, 776, 777, 808, 794, 777, 757, 766, 761, 789, 720, 761, 761, 774, 769, 767, 818, 778, 719, 772, 770, 755, 808, 751, 797, 736, 779, 741, 788, 768, 747, 748, 757, 743, 785, 775, 782, 773, 776, 808, 784, 776, 755, 803] payload_rows_per_dst=[9206, 9374, 9284, 9270, 9262, 9350, 9322, 9390] dup_rows_per_dst=[2948, 2972, 2944, 2980, 3054, 2924, 2900, 3124] combine_rows_per_src=[9206, 9374, 9284, 9270, 9262, 9350, 9322, 9390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=64918 dup_rows=24072 dup_rows_remote=21002 dedup_ratio=0.244873 router_tokens_per_expert=[770, 758, 761, 732, 770, 794, 746, 704, 799, 769, 696, 753, 738, 769, 775, 770, 748, 788, 793, 808, 734, 797, 788, 702, 789, 776, 781, 760, 770, 744, 758, 761, 764, 790, 813, 777, 740, 814, 745, 749, 697, 763, 779, 736, 788, 743, 781, 804, 772, 776, 818, 784, 780, 752, 762, 765, 769, 791, 823, 741, 762, 817, 764, 792] payload_rows_per_dst=[9100, 9254, 9318, 9268, 9314, 9264, 9340, 9374] dup_rows_per_dst=[2970, 2884, 2998, 3010, 3070, 2918, 3078, 3144] combine_rows_per_src=[9100, 9254, 9318, 9268, 9314, 9264, 9340, 9374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265691136 dispatch_meta_bytes=343784 combine_payload_bytes=265691136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266034920 ingress_bytes=300294144 link_bytes=300294144 token_bytes=531726056 weight_movement_bytes=34603008 total_nvlink_bytes=566329064 weight_to_token_ratio=0.065077 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129982688e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64852 dup_rows=24114 dup_rows_remote=21112 dedup_ratio=0.245300 router_tokens_per_expert=[765, 807, 769, 763, 750, 727, 830, 755, 725, 745, 803, 758, 745, 735, 761, 725, 811, 756, 779, 747, 821, 749, 792, 752, 783, 745, 762, 784, 759, 783, 735, 762, 764, 742, 815, 770, 770, 750, 804, 744, 733, 771, 730, 750, 821, 748, 768, 731, 792, 827, 791, 761, 760, 806, 750, 825, 782, 751, 731, 777, 780, 780, 760, 785] payload_rows_per_dst=[9338, 9188, 9248, 9296, 9318, 9020, 9510, 9272] dup_rows_per_dst=[3002, 2982, 2990, 3252, 3000, 2762, 3106, 3020] combine_rows_per_src=[9338, 9188, 9248, 9296, 9318, 9020, 9510, 9272] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266035200 dispatch_meta_bytes=342816 combine_payload_bytes=266035200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266378016 ingress_bytes=300638208 link_bytes=300638208 token_bytes=532413216 weight_movement_bytes=34603008 total_nvlink_bytes=567016224 weight_to_token_ratio=0.064993 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131277373e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74546 payload_rows_remote=65262 dup_rows=23758 dup_rows_remote=20714 dedup_ratio=0.241679 router_tokens_per_expert=[778, 749, 738, 756, 766, 770, 743, 857, 796, 783, 763, 825, 804, 747, 733, 730, 731, 773, 797, 798, 755, 781, 800, 729, 745, 774, 755, 784, 758, 762, 778, 779, 809, 765, 792, 793, 768, 770, 755, 765, 759, 755, 775, 731, 779, 739, 777, 758, 730, 748, 721, 733, 739, 824, 771, 812, 784, 758, 828, 761, 755, 770, 739, 752] payload_rows_per_dst=[9338, 9346, 9284, 9326, 9436, 9240, 9330, 9246] dup_rows_per_dst=[2976, 3016, 3044, 2944, 2998, 2906, 2826, 3048] combine_rows_per_src=[9338, 9346, 9284, 9326, 9436, 9240, 9330, 9246] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=343680 combine_payload_bytes=265904128 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266247808 ingress_bytes=335110144 link_bytes=335110144 token_bytes=532151936 weight_movement_bytes=69206016 total_nvlink_bytes=601357952 weight_to_token_ratio=0.130049 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260992493e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74428 payload_rows_remote=65134 dup_rows=23876 dup_rows_remote=20894 dedup_ratio=0.242879 router_tokens_per_expert=[752, 746, 800, 785, 787, 831, 783, 792, 786, 793, 801, 727, 772, 789, 771, 737, 770, 774, 805, 751, 748, 766, 778, 744, 749, 776, 748, 754, 769, 790, 776, 748, 770, 731, 747, 741, 789, 778, 782, 804, 749, 758, 759, 790, 773, 755, 775, 779, 753, 761, 781, 703, 720, 741, 814, 760, 794, 755, 746, 760, 752, 772, 800, 762] payload_rows_per_dst=[9514, 9302, 9330, 9202, 9296, 9294, 9228, 9262] dup_rows_per_dst=[3038, 3050, 2942, 3018, 2988, 2982, 2838, 3020] combine_rows_per_src=[9514, 9302, 9330, 9202, 9296, 9294, 9228, 9262] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265633792 dispatch_meta_bytes=343856 combine_payload_bytes=265633792 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265977648 ingress_bytes=369442816 link_bytes=369442816 token_bytes=531611440 weight_movement_bytes=103809024 total_nvlink_bytes=635420464 weight_to_token_ratio=0.195272 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390183574e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74102 payload_rows_remote=64820 dup_rows=24202 dup_rows_remote=21184 dedup_ratio=0.246195 router_tokens_per_expert=[811, 767, 770, 814, 765, 778, 766, 748, 756, 728, 759, 758, 738, 720, 765, 744, 778, 789, 740, 769, 791, 749, 803, 777, 777, 721, 794, 782, 778, 719, 752, 716, 788, 778, 760, 757, 750, 788, 816, 796, 745, 751, 740, 746, 788, 729, 793, 765, 739, 777, 811, 798, 748, 751, 771, 755, 750, 775, 763, 742, 851, 826, 808, 775] payload_rows_per_dst=[9286, 9078, 9270, 9140, 9476, 9138, 9282, 9432] dup_rows_per_dst=[3152, 2858, 3122, 2938, 2990, 2976, 3018, 3148] combine_rows_per_src=[9286, 9078, 9270, 9140, 9476, 9138, 9282, 9432] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74352 payload_rows_remote=64958 dup_rows=23952 dup_rows_remote=20864 dedup_ratio=0.243652 router_tokens_per_expert=[735, 772, 781, 776, 791, 801, 783, 766, 774, 763, 761, 815, 751, 758, 732, 765, 750, 799, 752, 725, 763, 802, 783, 773, 751, 815, 805, 736, 816, 785, 804, 730, 761, 746, 808, 779, 737, 800, 719, 727, 789, 772, 708, 735, 771, 747, 775, 749, 721, 806, 770, 784, 719, 776, 739, 841, 770, 744, 802, 776, 781, 759, 758, 770] payload_rows_per_dst=[9070, 9176, 9292, 9344, 9286, 9214, 9576, 9394] dup_rows_per_dst=[2920, 3062, 2948, 2978, 2922, 2878, 3156, 3088] combine_rows_per_src=[9070, 9176, 9292, 9344, 9286, 9214, 9576, 9394] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=267313152 dispatch_meta_bytes=343904 combine_payload_bytes=267313152 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267657056 ingress_bytes=371122176 link_bytes=371122176 token_bytes=534970208 weight_movement_bytes=103809024 total_nvlink_bytes=638779232 weight_to_token_ratio=0.194046 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396502871e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266854400 dispatch_meta_bytes=343656 combine_payload_bytes=266854400 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=267198056 ingress_bytes=439869440 link_bytes=439869440 token_bytes=534052456 weight_movement_bytes=173015040 total_nvlink_bytes=707067496 weight_to_token_ratio=0.323966 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.655193290e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=344024 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266592216 ingress_bytes=300851200 link_bytes=300851200 token_bytes=532840408 weight_movement_bytes=34603008 total_nvlink_bytes=567443416 weight_to_token_ratio=0.064941 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132078845e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265691136 dispatch_meta_bytes=343784 combine_payload_bytes=265691136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266034920 ingress_bytes=300294144 link_bytes=300294144 token_bytes=531726056 weight_movement_bytes=34603008 total_nvlink_bytes=566329064 weight_to_token_ratio=0.065077 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129982688e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266682368 dispatch_meta_bytes=344120 combine_payload_bytes=266682368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267026488 ingress_bytes=301285376 link_bytes=301285376 token_bytes=533708856 weight_movement_bytes=34603008 total_nvlink_bytes=568311864 weight_to_token_ratio=0.064835 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133712614e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=344112 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267132976 ingress_bytes=301391872 link_bytes=301391872 token_bytes=533921840 weight_movement_bytes=34603008 total_nvlink_bytes=568524848 weight_to_token_ratio=0.064809 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134113350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74470 payload_rows_remote=65014 dup_rows=23834 dup_rows_remote=20738 dedup_ratio=0.242452 router_tokens_per_expert=[751, 729, 750, 777, 759, 732, 796, 758, 786, 787, 765, 743, 798, 784, 754, 749, 755, 750, 764, 764, 796, 808, 764, 757, 787, 789, 807, 769, 790, 770, 784, 780, 792, 773, 761, 792, 795, 798, 760, 743, 739, 760, 766, 783, 753, 750, 779, 783, 754, 790, 787, 719, 739, 725, 762, 768, 785, 756, 767, 728, 783, 762, 755, 793] payload_rows_per_dst=[9206, 9234, 9298, 9456, 9382, 9330, 9250, 9314] dup_rows_per_dst=[2898, 3098, 3018, 3096, 3046, 2896, 2838, 2944] combine_rows_per_src=[9206, 9234, 9298, 9456, 9382, 9330, 9250, 9314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74542 payload_rows_remote=65134 dup_rows=23762 dup_rows_remote=20718 dedup_ratio=0.241720 router_tokens_per_expert=[779, 739, 784, 753, 769, 755, 766, 776, 784, 736, 735, 789, 743, 766, 817, 729, 757, 800, 807, 774, 736, 767, 815, 743, 782, 720, 744, 789, 740, 784, 816, 783, 797, 778, 742, 760, 757, 792, 815, 785, 746, 793, 773, 752, 747, 772, 770, 779, 803, 733, 758, 729, 700, 756, 761, 808, 761, 779, 811, 777, 748, 775, 750, 768] payload_rows_per_dst=[9306, 9234, 9350, 9306, 9408, 9318, 9262, 9358] dup_rows_per_dst=[2936, 2964, 3048, 3010, 3044, 2946, 2834, 2980] combine_rows_per_src=[9306, 9234, 9350, 9306, 9408, 9318, 9262, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=267313152 dispatch_meta_bytes=343904 combine_payload_bytes=267313152 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267657056 ingress_bytes=371122176 link_bytes=371122176 token_bytes=534970208 weight_movement_bytes=103809024 total_nvlink_bytes=638779232 weight_to_token_ratio=0.194046 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396502871e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=344016 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265846736 ingress_bytes=300105728 link_bytes=300105728 token_bytes=531349456 weight_movement_bytes=34603008 total_nvlink_bytes=565952464 weight_to_token_ratio=0.065123 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129273694e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=65002 dup_rows=24016 dup_rows_remote=21004 dedup_ratio=0.244303 router_tokens_per_expert=[768, 763, 764, 760, 737, 792, 713, 756, 769, 854, 761, 728, 781, 749, 741, 766, 772, 811, 764, 759, 789, 800, 733, 754, 747, 751, 764, 756, 831, 780, 730, 782, 810, 749, 759, 710, 779, 723, 808, 767, 778, 775, 793, 769, 823, 728, 780, 747, 785, 743, 776, 803, 753, 752, 750, 765, 744, 765, 791, 751, 818, 777, 783, 773] payload_rows_per_dst=[9242, 9286, 9358, 9292, 9222, 9258, 9250, 9380] dup_rows_per_dst=[2864, 3012, 3006, 2990, 2988, 3128, 3004, 3024] combine_rows_per_src=[9242, 9286, 9358, 9292, 9222, 9258, 9250, 9380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=343288 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266411256 ingress_bytes=335273984 link_bytes=335273984 token_bytes=532479224 weight_movement_bytes=69206016 total_nvlink_bytes=601685240 weight_to_token_ratio=0.129969 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261609010e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74518 payload_rows_remote=65146 dup_rows=23786 dup_rows_remote=20788 dedup_ratio=0.241964 router_tokens_per_expert=[792, 766, 796, 785, 753, 756, 794, 719, 796, 750, 692, 790, 747, 779, 780, 746, 758, 753, 790, 780, 780, 765, 770, 763, 750, 791, 753, 772, 745, 725, 790, 786, 772, 761, 820, 752, 769, 784, 764, 763, 751, 736, 826, 801, 791, 804, 761, 739, 762, 755, 741, 747, 742, 805, 779, 808, 785, 728, 804, 745, 767, 791, 741, 746] payload_rows_per_dst=[9374, 9344, 9328, 9214, 9372, 9342, 9262, 9282] dup_rows_per_dst=[2948, 2816, 2990, 3010, 2998, 3076, 3016, 2932] combine_rows_per_src=[9374, 9344, 9328, 9214, 9372, 9342, 9262, 9282] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266297344 dispatch_meta_bytes=343008 combine_payload_bytes=266297344 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266640352 ingress_bytes=335503360 link_bytes=335503360 token_bytes=532937696 weight_movement_bytes=69206016 total_nvlink_bytes=602143712 weight_to_token_ratio=0.129858 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262472133e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74166 payload_rows_remote=64586 dup_rows=24138 dup_rows_remote=20908 dedup_ratio=0.245544 router_tokens_per_expert=[751, 793, 756, 796, 774, 723, 791, 758, 778, 782, 763, 780, 784, 784, 756, 750, 744, 785, 752, 773, 799, 743, 785, 770, 738, 771, 777, 734, 767, 754, 746, 739, 774, 766, 747, 739, 778, 819, 738, 753, 780, 816, 702, 757, 777, 762, 712, 792, 797, 811, 801, 797, 769, 790, 781, 741, 760, 767, 775, 802, 749, 744, 785, 775] payload_rows_per_dst=[9580, 9302, 9436, 9172, 9078, 9296, 9572, 8730] dup_rows_per_dst=[3230, 3142, 3148, 2944, 2804, 2954, 3174, 2742] combine_rows_per_src=[9580, 9302, 9436, 9172, 9078, 9296, 9572, 8730] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=344024 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266592216 ingress_bytes=300851200 link_bytes=300851200 token_bytes=532840408 weight_movement_bytes=34603008 total_nvlink_bytes=567443416 weight_to_token_ratio=0.064941 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132078845e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=64918 dup_rows=23930 dup_rows_remote=20874 dedup_ratio=0.243429 router_tokens_per_expert=[750, 773, 790, 810, 782, 760, 776, 789, 778, 817, 775, 738, 732, 756, 736, 716, 735, 837, 764, 781, 803, 783, 795, 758, 776, 747, 787, 761, 766, 739, 791, 793, 772, 730, 766, 735, 760, 695, 747, 742, 769, 798, 694, 724, 802, 761, 789, 770, 801, 783, 735, 724, 749, 786, 740, 770, 776, 757, 816, 835, 833, 769, 744, 786] payload_rows_per_dst=[9348, 9260, 9456, 9302, 9116, 9216, 9280, 9396] dup_rows_per_dst=[3112, 2836, 3056, 3018, 2778, 2998, 2896, 3236] combine_rows_per_src=[9348, 9260, 9456, 9302, 9116, 9216, 9280, 9396] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266838016 dispatch_meta_bytes=343736 combine_payload_bytes=266838016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267181752 ingress_bytes=301441024 link_bytes=301441024 token_bytes=534019768 weight_movement_bytes=34603008 total_nvlink_bytes=568622776 weight_to_token_ratio=0.064797 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134298305e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74394 payload_rows_remote=65282 dup_rows=23910 dup_rows_remote=20946 dedup_ratio=0.243225 router_tokens_per_expert=[759, 759, 780, 782, 773, 766, 777, 783, 758, 767, 778, 784, 774, 724, 771, 725, 760, 752, 802, 769, 769, 787, 713, 808, 756, 791, 728, 801, 760, 780, 812, 766, 751, 725, 761, 763, 775, 746, 778, 784, 727, 772, 794, 759, 744, 777, 730, 735, 787, 756, 798, 761, 740, 779, 786, 745, 770, 850, 787, 723, 796, 777, 804, 758] payload_rows_per_dst=[9352, 9250, 9344, 9376, 9168, 9112, 9322, 9470] dup_rows_per_dst=[3006, 2912, 2976, 3012, 2998, 2964, 2982, 3060] combine_rows_per_src=[9352, 9250, 9344, 9376, 9168, 9112, 9322, 9470] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74518 payload_rows_remote=65082 dup_rows=23786 dup_rows_remote=20492 dedup_ratio=0.241964 router_tokens_per_expert=[759, 779, 760, 802, 808, 804, 769, 792, 766, 784, 811, 796, 734, 801, 806, 763, 743, 777, 749, 755, 764, 780, 802, 759, 738, 739, 771, 798, 752, 804, 746, 804, 697, 769, 772, 782, 772, 769, 682, 765, 774, 717, 782, 779, 758, 771, 763, 741, 731, 802, 796, 764, 806, 717, 745, 784, 765, 768, 785, 758, 760, 801, 725, 737] payload_rows_per_dst=[9436, 9570, 9234, 9426, 9120, 9556, 9282, 8894] dup_rows_per_dst=[3294, 3078, 2842, 2876, 2770, 3326, 3008, 2592] combine_rows_per_src=[9436, 9570, 9234, 9426, 9120, 9556, 9282, 8894] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=343408 combine_payload_bytes=266788864 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267132272 ingress_bytes=370597888 link_bytes=370597888 token_bytes=533921136 weight_movement_bytes=103809024 total_nvlink_bytes=637730160 weight_to_token_ratio=0.194428 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394530017e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74352 payload_rows_remote=64958 dup_rows=23952 dup_rows_remote=20864 dedup_ratio=0.243652 router_tokens_per_expert=[735, 772, 781, 776, 791, 801, 783, 766, 774, 763, 761, 815, 751, 758, 732, 765, 750, 799, 752, 725, 763, 802, 783, 773, 751, 815, 805, 736, 816, 785, 804, 730, 761, 746, 808, 779, 737, 800, 719, 727, 789, 772, 708, 735, 771, 747, 775, 749, 721, 806, 770, 784, 719, 776, 739, 841, 770, 744, 802, 776, 781, 759, 758, 770] payload_rows_per_dst=[9070, 9176, 9292, 9344, 9286, 9214, 9576, 9394] dup_rows_per_dst=[2920, 3062, 2948, 2978, 2922, 2878, 3156, 3088] combine_rows_per_src=[9070, 9176, 9292, 9344, 9286, 9214, 9576, 9394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74474 payload_rows_remote=65174 dup_rows=23830 dup_rows_remote=20808 dedup_ratio=0.242411 router_tokens_per_expert=[806, 779, 737, 765, 784, 761, 785, 773, 763, 782, 752, 805, 763, 723, 788, 785, 765, 722, 748, 775, 791, 733, 737, 753, 734, 759, 863, 763, 802, 813, 786, 746, 714, 767, 791, 784, 776, 736, 737, 721, 790, 739, 757, 797, 776, 786, 767, 772, 789, 736, 786, 753, 792, 782, 758, 792, 783, 716, 792, 775, 741, 789, 759, 758] payload_rows_per_dst=[9366, 9300, 9180, 9414, 9178, 9404, 9318, 9314] dup_rows_per_dst=[3014, 3022, 2868, 3118, 2874, 2964, 3058, 2912] combine_rows_per_src=[9366, 9300, 9180, 9414, 9178, 9404, 9318, 9314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266575872 dispatch_meta_bytes=342296 combine_payload_bytes=266575872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266918168 ingress_bytes=335781888 link_bytes=335781888 token_bytes=533494040 weight_movement_bytes=69206016 total_nvlink_bytes=602700056 weight_to_token_ratio=0.129722 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263520212e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74470 payload_rows_remote=65014 dup_rows=23834 dup_rows_remote=20738 dedup_ratio=0.242452 router_tokens_per_expert=[751, 729, 750, 777, 759, 732, 796, 758, 786, 787, 765, 743, 798, 784, 754, 749, 755, 750, 764, 764, 796, 808, 764, 757, 787, 789, 807, 769, 790, 770, 784, 780, 792, 773, 761, 792, 795, 798, 760, 743, 739, 760, 766, 783, 753, 750, 779, 783, 754, 790, 787, 719, 739, 725, 762, 768, 785, 756, 767, 728, 783, 762, 755, 793] payload_rows_per_dst=[9206, 9234, 9298, 9456, 9382, 9330, 9250, 9314] dup_rows_per_dst=[2898, 3098, 3018, 3096, 3046, 2896, 2838, 2944] combine_rows_per_src=[9206, 9234, 9298, 9456, 9382, 9330, 9250, 9314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74428 payload_rows_remote=65134 dup_rows=23876 dup_rows_remote=20894 dedup_ratio=0.242879 router_tokens_per_expert=[752, 746, 800, 785, 787, 831, 783, 792, 786, 793, 801, 727, 772, 789, 771, 737, 770, 774, 805, 751, 748, 766, 778, 744, 749, 776, 748, 754, 769, 790, 776, 748, 770, 731, 747, 741, 789, 778, 782, 804, 749, 758, 759, 790, 773, 755, 775, 779, 753, 761, 781, 703, 720, 741, 814, 760, 794, 755, 746, 760, 752, 772, 800, 762] payload_rows_per_dst=[9514, 9302, 9330, 9202, 9296, 9294, 9228, 9262] dup_rows_per_dst=[3038, 3050, 2942, 3018, 2988, 2982, 2838, 3020] combine_rows_per_src=[9514, 9302, 9330, 9202, 9296, 9294, 9228, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=64918 dup_rows=23930 dup_rows_remote=20874 dedup_ratio=0.243429 router_tokens_per_expert=[750, 773, 790, 810, 782, 760, 776, 789, 778, 817, 775, 738, 732, 756, 736, 716, 735, 837, 764, 781, 803, 783, 795, 758, 776, 747, 787, 761, 766, 739, 791, 793, 772, 730, 766, 735, 760, 695, 747, 742, 769, 798, 694, 724, 802, 761, 789, 770, 801, 783, 735, 724, 749, 786, 740, 770, 776, 757, 816, 835, 833, 769, 744, 786] payload_rows_per_dst=[9348, 9260, 9456, 9302, 9116, 9216, 9280, 9396] dup_rows_per_dst=[3112, 2836, 3056, 3018, 2778, 2998, 2896, 3236] combine_rows_per_src=[9348, 9260, 9456, 9302, 9116, 9216, 9280, 9396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74432 payload_rows_remote=65128 dup_rows=23872 dup_rows_remote=20900 dedup_ratio=0.242839 router_tokens_per_expert=[759, 754, 778, 793, 781, 749, 720, 722, 756, 758, 745, 770, 781, 738, 774, 789, 755, 766, 807, 748, 805, 714, 745, 753, 796, 756, 744, 757, 785, 799, 773, 785, 802, 771, 760, 759, 801, 788, 746, 770, 754, 763, 791, 801, 759, 772, 788, 746, 765, 790, 753, 816, 726, 761, 766, 761, 760, 729, 774, 820, 758, 801, 771, 775] payload_rows_per_dst=[9226, 9240, 9192, 9342, 9410, 9330, 9304, 9388] dup_rows_per_dst=[2886, 2982, 2994, 3048, 2984, 3018, 2972, 2988] combine_rows_per_src=[9226, 9240, 9192, 9342, 9410, 9330, 9304, 9388] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=264544256 dispatch_meta_bytes=341976 combine_payload_bytes=264544256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264886232 ingress_bytes=299147264 link_bytes=299147264 token_bytes=529430488 weight_movement_bytes=34603008 total_nvlink_bytes=564033496 weight_to_token_ratio=0.065359 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125667071e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74468 payload_rows_remote=65168 dup_rows=23836 dup_rows_remote=20794 dedup_ratio=0.242472 router_tokens_per_expert=[760, 776, 739, 795, 772, 752, 766, 794, 832, 751, 732, 780, 732, 725, 765, 807, 748, 779, 751, 749, 745, 796, 747, 797, 779, 824, 773, 766, 810, 775, 785, 749, 789, 791, 762, 754, 762, 735, 785, 749, 692, 772, 766, 717, 774, 751, 751, 772, 790, 743, 746, 742, 786, 781, 799, 829, 771, 742, 764, 770, 794, 768, 772, 782] payload_rows_per_dst=[9268, 9316, 9294, 9290, 9502, 9088, 9410, 9300] dup_rows_per_dst=[3024, 2932, 2930, 2862, 3122, 2902, 3022, 3042] combine_rows_per_src=[9268, 9316, 9294, 9290, 9502, 9088, 9410, 9300] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=343168 combine_payload_bytes=265904128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266247296 ingress_bytes=300507136 link_bytes=300507136 token_bytes=532151424 weight_movement_bytes=34603008 total_nvlink_bytes=566754432 weight_to_token_ratio=0.065025 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130784160e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=343288 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266411256 ingress_bytes=335273984 link_bytes=335273984 token_bytes=532479224 weight_movement_bytes=69206016 total_nvlink_bytes=601685240 weight_to_token_ratio=0.129969 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261609010e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266952704 dispatch_meta_bytes=343928 combine_payload_bytes=266952704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267296632 ingress_bytes=301555712 link_bytes=301555712 token_bytes=534249336 weight_movement_bytes=34603008 total_nvlink_bytes=568852344 weight_to_token_ratio=0.064769 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134729867e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=267395072 dispatch_meta_bytes=344912 combine_payload_bytes=267395072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267739984 ingress_bytes=301998080 link_bytes=301998080 token_bytes=535135056 weight_movement_bytes=34603008 total_nvlink_bytes=569738064 weight_to_token_ratio=0.064662 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136394462e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266297344 dispatch_meta_bytes=343008 combine_payload_bytes=266297344 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266640352 ingress_bytes=335503360 link_bytes=335503360 token_bytes=532937696 weight_movement_bytes=69206016 total_nvlink_bytes=602143712 weight_to_token_ratio=0.129858 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262472133e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=64918 dup_rows=24072 dup_rows_remote=21002 dedup_ratio=0.244873 router_tokens_per_expert=[770, 758, 761, 732, 770, 794, 746, 704, 799, 769, 696, 753, 738, 769, 775, 770, 748, 788, 793, 808, 734, 797, 788, 702, 789, 776, 781, 760, 770, 744, 758, 761, 764, 790, 813, 777, 740, 814, 745, 749, 697, 763, 779, 736, 788, 743, 781, 804, 772, 776, 818, 784, 780, 752, 762, 765, 769, 791, 823, 741, 762, 817, 764, 792] payload_rows_per_dst=[9100, 9254, 9318, 9268, 9314, 9264, 9340, 9374] dup_rows_per_dst=[2970, 2884, 2998, 3010, 3070, 2918, 3078, 3144] combine_rows_per_src=[9100, 9254, 9318, 9268, 9314, 9264, 9340, 9374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=344112 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267132976 ingress_bytes=301391872 link_bytes=301391872 token_bytes=533921840 weight_movement_bytes=34603008 total_nvlink_bytes=568524848 weight_to_token_ratio=0.064809 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134113350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=343168 combine_payload_bytes=265904128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266247296 ingress_bytes=300507136 link_bytes=300507136 token_bytes=532151424 weight_movement_bytes=34603008 total_nvlink_bytes=566754432 weight_to_token_ratio=0.065025 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130784160e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266764288 dispatch_meta_bytes=344112 combine_payload_bytes=266764288 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267108400 ingress_bytes=335970304 link_bytes=335970304 token_bytes=533872688 weight_movement_bytes=69206016 total_nvlink_bytes=603078704 weight_to_token_ratio=0.129630 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264229206e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74108 payload_rows_remote=64978 dup_rows=24196 dup_rows_remote=21148 dedup_ratio=0.246134 router_tokens_per_expert=[778, 787, 741, 784, 750, 766, 758, 725, 774, 797, 790, 751, 753, 738, 803, 810, 771, 776, 766, 773, 771, 727, 753, 737, 778, 788, 757, 740, 751, 749, 776, 783, 758, 769, 741, 804, 741, 759, 803, 812, 798, 746, 801, 773, 777, 790, 776, 768, 711, 773, 791, 720, 814, 747, 799, 807, 734, 781, 745, 754, 766, 755, 764, 774] payload_rows_per_dst=[9130, 9582, 9378, 9124, 9266, 9616, 9266, 8746] dup_rows_per_dst=[3048, 3364, 3078, 2882, 3108, 3126, 2878, 2712] combine_rows_per_src=[9130, 9582, 9378, 9124, 9266, 9616, 9266, 8746] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=343680 combine_payload_bytes=265904128 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266247808 ingress_bytes=335110144 link_bytes=335110144 token_bytes=532151936 weight_movement_bytes=69206016 total_nvlink_bytes=601357952 weight_to_token_ratio=0.130049 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260992493e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74468 payload_rows_remote=65168 dup_rows=23836 dup_rows_remote=20794 dedup_ratio=0.242472 router_tokens_per_expert=[760, 776, 739, 795, 772, 752, 766, 794, 832, 751, 732, 780, 732, 725, 765, 807, 748, 779, 751, 749, 745, 796, 747, 797, 779, 824, 773, 766, 810, 775, 785, 749, 789, 791, 762, 754, 762, 735, 785, 749, 692, 772, 766, 717, 774, 751, 751, 772, 790, 743, 746, 742, 786, 781, 799, 829, 771, 742, 764, 770, 794, 768, 772, 782] payload_rows_per_dst=[9268, 9316, 9294, 9290, 9502, 9088, 9410, 9300] dup_rows_per_dst=[3024, 2932, 2930, 2862, 3122, 2902, 3022, 3042] combine_rows_per_src=[9268, 9316, 9294, 9290, 9502, 9088, 9410, 9300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=344504 combine_payload_bytes=266149888 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266494392 ingress_bytes=300752896 link_bytes=300752896 token_bytes=532644280 weight_movement_bytes=34603008 total_nvlink_bytes=567247288 weight_to_token_ratio=0.064965 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131708935e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74260 payload_rows_remote=64956 dup_rows=24044 dup_rows_remote=20970 dedup_ratio=0.244588 router_tokens_per_expert=[752, 712, 747, 789, 763, 735, 804, 722, 738, 760, 812, 749, 799, 788, 784, 759, 731, 743, 759, 737, 749, 736, 749, 762, 757, 771, 775, 765, 780, 775, 820, 753, 778, 737, 775, 830, 775, 784, 793, 794, 750, 749, 749, 744, 781, 788, 829, 816, 782, 790, 776, 730, 760, 854, 760, 761, 752, 789, 767, 748, 701, 779, 794, 762] payload_rows_per_dst=[9242, 9304, 9090, 9274, 9504, 9316, 9334, 9196] dup_rows_per_dst=[2806, 3074, 2842, 3118, 3028, 3096, 3092, 2988] combine_rows_per_src=[9242, 9304, 9090, 9274, 9504, 9316, 9334, 9196] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74104 payload_rows_remote=64820 dup_rows=24200 dup_rows_remote=21192 dedup_ratio=0.246175 router_tokens_per_expert=[765, 794, 752, 795, 719, 795, 748, 733, 768, 772, 726, 751, 805, 769, 718, 765, 767, 804, 727, 760, 754, 763, 758, 787, 761, 799, 743, 740, 786, 730, 750, 837, 782, 770, 768, 775, 824, 781, 770, 745, 720, 741, 781, 804, 765, 733, 789, 741, 785, 762, 774, 757, 776, 746, 798, 761, 771, 775, 841, 746, 774, 805, 804, 747] payload_rows_per_dst=[9224, 9144, 9242, 9284, 9326, 9134, 9280, 9470] dup_rows_per_dst=[2978, 3004, 2998, 3008, 3104, 3014, 3038, 3056] combine_rows_per_src=[9224, 9144, 9242, 9284, 9326, 9134, 9280, 9470] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266928128 dispatch_meta_bytes=343848 combine_payload_bytes=266928128 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=267271976 ingress_bytes=405340160 link_bytes=405340160 token_bytes=534200104 weight_movement_bytes=138412032 total_nvlink_bytes=672612136 weight_to_token_ratio=0.259101 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.525262390e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74474 payload_rows_remote=65174 dup_rows=23830 dup_rows_remote=20808 dedup_ratio=0.242411 router_tokens_per_expert=[806, 779, 737, 765, 784, 761, 785, 773, 763, 782, 752, 805, 763, 723, 788, 785, 765, 722, 748, 775, 791, 733, 737, 753, 734, 759, 863, 763, 802, 813, 786, 746, 714, 767, 791, 784, 776, 736, 737, 721, 790, 739, 757, 797, 776, 786, 767, 772, 789, 736, 786, 753, 792, 782, 758, 792, 783, 716, 792, 775, 741, 789, 759, 758] payload_rows_per_dst=[9366, 9300, 9180, 9414, 9178, 9404, 9318, 9314] dup_rows_per_dst=[3014, 3022, 2868, 3118, 2874, 2964, 3058, 2912] combine_rows_per_src=[9366, 9300, 9180, 9414, 9178, 9404, 9318, 9314] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=64894 dup_rows=24050 dup_rows_remote=21040 dedup_ratio=0.244649 router_tokens_per_expert=[771, 758, 772, 757, 790, 745, 761, 768, 734, 814, 761, 776, 769, 793, 752, 763, 820, 771, 760, 780, 765, 788, 759, 756, 819, 729, 794, 825, 814, 730, 754, 731, 746, 746, 812, 755, 789, 804, 735, 798, 771, 746, 819, 760, 750, 774, 796, 814, 764, 738, 757, 769, 718, 749, 781, 805, 748, 765, 738, 753, 776, 743, 734, 720] payload_rows_per_dst=[9236, 9258, 9310, 9362, 9360, 9376, 9258, 9094] dup_rows_per_dst=[3008, 3066, 3088, 3030, 3010, 3084, 2904, 2860] combine_rows_per_src=[9236, 9258, 9310, 9362, 9360, 9376, 9258, 9094] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=344048 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265846768 ingress_bytes=369311744 link_bytes=369311744 token_bytes=531349488 weight_movement_bytes=103809024 total_nvlink_bytes=635158512 weight_to_token_ratio=0.195369 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389690361e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266952704 dispatch_meta_bytes=343928 combine_payload_bytes=266952704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267296632 ingress_bytes=301555712 link_bytes=301555712 token_bytes=534249336 weight_movement_bytes=34603008 total_nvlink_bytes=568852344 weight_to_token_ratio=0.064769 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134729867e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74518 payload_rows_remote=65082 dup_rows=23786 dup_rows_remote=20492 dedup_ratio=0.241964 router_tokens_per_expert=[759, 779, 760, 802, 808, 804, 769, 792, 766, 784, 811, 796, 734, 801, 806, 763, 743, 777, 749, 755, 764, 780, 802, 759, 738, 739, 771, 798, 752, 804, 746, 804, 697, 769, 772, 782, 772, 769, 682, 765, 774, 717, 782, 779, 758, 771, 763, 741, 731, 802, 796, 764, 806, 717, 745, 784, 765, 768, 785, 758, 760, 801, 725, 737] payload_rows_per_dst=[9436, 9570, 9234, 9426, 9120, 9556, 9282, 8894] dup_rows_per_dst=[3294, 3078, 2842, 2876, 2770, 3326, 3008, 2592] combine_rows_per_src=[9436, 9570, 9234, 9426, 9120, 9556, 9282, 8894] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74394 payload_rows_remote=65282 dup_rows=23910 dup_rows_remote=20946 dedup_ratio=0.243225 router_tokens_per_expert=[759, 759, 780, 782, 773, 766, 777, 783, 758, 767, 778, 784, 774, 724, 771, 725, 760, 752, 802, 769, 769, 787, 713, 808, 756, 791, 728, 801, 760, 780, 812, 766, 751, 725, 761, 763, 775, 746, 778, 784, 727, 772, 794, 759, 744, 777, 730, 735, 787, 756, 798, 761, 740, 779, 786, 745, 770, 850, 787, 723, 796, 777, 804, 758] payload_rows_per_dst=[9352, 9250, 9344, 9376, 9168, 9112, 9322, 9470] dup_rows_per_dst=[3006, 2912, 2976, 3012, 2998, 2964, 2982, 3060] combine_rows_per_src=[9352, 9250, 9344, 9376, 9168, 9112, 9322, 9470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74278 payload_rows_remote=65102 dup_rows=24026 dup_rows_remote=21116 dedup_ratio=0.244405 router_tokens_per_expert=[742, 773, 778, 782, 803, 771, 779, 783, 764, 786, 780, 733, 764, 818, 772, 787, 774, 757, 794, 745, 732, 753, 741, 747, 733, 812, 763, 769, 785, 748, 777, 735, 779, 746, 786, 799, 723, 828, 784, 752, 795, 743, 783, 779, 788, 746, 787, 787, 714, 723, 783, 772, 806, 706, 763, 813, 758, 769, 746, 763, 801, 736, 727, 787] payload_rows_per_dst=[9336, 9276, 9176, 9238, 9344, 9390, 9276, 9242] dup_rows_per_dst=[3086, 3132, 2910, 3006, 3050, 3026, 2884, 2932] combine_rows_per_src=[9336, 9276, 9176, 9238, 9344, 9390, 9276, 9242] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265805824 dispatch_meta_bytes=343736 combine_payload_bytes=265805824 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266149560 ingress_bytes=369614848 link_bytes=369614848 token_bytes=531955384 weight_movement_bytes=103809024 total_nvlink_bytes=635764408 weight_to_token_ratio=0.195146 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390830917e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74502 payload_rows_remote=65188 dup_rows=23802 dup_rows_remote=20842 dedup_ratio=0.242126 router_tokens_per_expert=[771, 759, 744, 791, 771, 783, 754, 784, 771, 772, 766, 778, 757, 783, 769, 800, 764, 730, 791, 741, 749, 780, 808, 778, 757, 765, 750, 791, 750, 746, 772, 742, 794, 764, 741, 730, 766, 750, 706, 760, 771, 753, 778, 806, 781, 778, 765, 705, 755, 750, 836, 783, 749, 751, 790, 765, 804, 782, 788, 788, 747, 775, 789, 785] payload_rows_per_dst=[9334, 9332, 9316, 9270, 9096, 9314, 9304, 9536] dup_rows_per_dst=[2980, 3060, 2966, 2876, 2926, 2960, 3054, 2980] combine_rows_per_src=[9334, 9332, 9316, 9270, 9096, 9314, 9304, 9536] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74136 payload_rows_remote=64742 dup_rows=24168 dup_rows_remote=21118 dedup_ratio=0.245850 router_tokens_per_expert=[745, 741, 756, 794, 767, 756, 773, 735, 794, 751, 760, 780, 769, 718, 759, 769, 779, 805, 754, 801, 722, 765, 744, 776, 733, 808, 784, 805, 807, 763, 814, 786, 722, 760, 738, 805, 789, 753, 750, 800, 734, 778, 795, 781, 748, 752, 762, 763, 805, 797, 752, 753, 828, 724, 780, 783, 774, 802, 725, 753, 799, 737, 742, 755] payload_rows_per_dst=[9188, 9236, 9252, 9384, 9228, 9226, 9394, 9228] dup_rows_per_dst=[2946, 2964, 3040, 3216, 3006, 3000, 3050, 2946] combine_rows_per_src=[9188, 9236, 9252, 9384, 9228, 9226, 9394, 9228] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266575872 dispatch_meta_bytes=342296 combine_payload_bytes=266575872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266918168 ingress_bytes=335781888 link_bytes=335781888 token_bytes=533494040 weight_movement_bytes=69206016 total_nvlink_bytes=602700056 weight_to_token_ratio=0.129722 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263520212e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74114 payload_rows_remote=64754 dup_rows=24190 dup_rows_remote=20928 dedup_ratio=0.246073 router_tokens_per_expert=[775, 780, 784, 806, 759, 750, 769, 783, 816, 837, 787, 786, 720, 769, 757, 769, 745, 777, 831, 744, 742, 791, 775, 747, 770, 821, 721, 757, 756, 749, 781, 784, 816, 803, 798, 791, 761, 712, 707, 812, 735, 771, 764, 789, 725, 799, 732, 795, 743, 718, 796, 776, 707, 755, 761, 776, 772, 733, 804, 724, 766, 780, 728, 765] payload_rows_per_dst=[9360, 9406, 9276, 8988, 9338, 9262, 9124, 9360] dup_rows_per_dst=[3052, 3030, 3028, 2858, 3062, 2958, 2940, 3262] combine_rows_per_src=[9360, 9406, 9276, 8988, 9338, 9262, 9124, 9360] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266657792 dispatch_meta_bytes=344872 combine_payload_bytes=266657792 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267002664 ingress_bytes=301260800 link_bytes=301260800 token_bytes=533660456 weight_movement_bytes=34603008 total_nvlink_bytes=568263464 weight_to_token_ratio=0.064841 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133620137e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=65060 dup_rows=24010 dup_rows_remote=21042 dedup_ratio=0.244242 router_tokens_per_expert=[750, 747, 834, 738, 758, 827, 788, 727, 787, 757, 778, 746, 732, 800, 779, 741, 812, 799, 767, 738, 760, 718, 782, 792, 783, 738, 741, 724, 841, 765, 762, 747, 745, 782, 753, 731, 772, 781, 771, 775, 778, 784, 789, 734, 808, 783, 728, 767, 777, 739, 775, 771, 724, 822, 777, 762, 744, 766, 794, 801, 812, 772, 751, 726] payload_rows_per_dst=[9450, 9214, 9264, 9234, 9266, 9244, 9300, 9322] dup_rows_per_dst=[2888, 3026, 3072, 2968, 2954, 3098, 2994, 3010] combine_rows_per_src=[9450, 9214, 9264, 9234, 9266, 9244, 9300, 9322] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=267010048 dispatch_meta_bytes=344120 combine_payload_bytes=267010048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267354168 ingress_bytes=301613056 link_bytes=301613056 token_bytes=534364216 weight_movement_bytes=34603008 total_nvlink_bytes=568967224 weight_to_token_ratio=0.064755 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134945648e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265183232 dispatch_meta_bytes=343440 combine_payload_bytes=265183232 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265526672 ingress_bytes=334389248 link_bytes=334389248 token_bytes=530709904 weight_movement_bytes=69206016 total_nvlink_bytes=599915920 weight_to_token_ratio=0.130403 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258279820e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74260 payload_rows_remote=64956 dup_rows=24044 dup_rows_remote=20970 dedup_ratio=0.244588 router_tokens_per_expert=[752, 712, 747, 789, 763, 735, 804, 722, 738, 760, 812, 749, 799, 788, 784, 759, 731, 743, 759, 737, 749, 736, 749, 762, 757, 771, 775, 765, 780, 775, 820, 753, 778, 737, 775, 830, 775, 784, 793, 794, 750, 749, 749, 744, 781, 788, 829, 816, 782, 790, 776, 730, 760, 854, 760, 761, 752, 789, 767, 748, 701, 779, 794, 762] payload_rows_per_dst=[9242, 9304, 9090, 9274, 9504, 9316, 9334, 9196] dup_rows_per_dst=[2806, 3074, 2842, 3118, 3028, 3096, 3092, 2988] combine_rows_per_src=[9242, 9304, 9090, 9274, 9504, 9316, 9334, 9196] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265232384 dispatch_meta_bytes=342728 combine_payload_bytes=265232384 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265575112 ingress_bytes=299835392 link_bytes=299835392 token_bytes=530807496 weight_movement_bytes=34603008 total_nvlink_bytes=565410504 weight_to_token_ratio=0.065189 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128256441e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74146 payload_rows_remote=64896 dup_rows=24158 dup_rows_remote=21074 dedup_ratio=0.245748 router_tokens_per_expert=[779, 778, 763, 783, 763, 757, 723, 756, 731, 720, 769, 730, 766, 742, 781, 734, 789, 800, 748, 779, 755, 711, 800, 777, 794, 793, 778, 742, 816, 751, 784, 730, 753, 738, 765, 768, 808, 788, 783, 764, 822, 780, 813, 751, 752, 808, 773, 755, 791, 791, 751, 760, 771, 763, 777, 808, 750, 757, 798, 760, 763, 789, 735, 745] payload_rows_per_dst=[9236, 9070, 9290, 9266, 9250, 9460, 9322, 9252] dup_rows_per_dst=[2968, 2876, 3028, 3110, 3084, 3048, 3102, 2942] combine_rows_per_src=[9236, 9070, 9290, 9266, 9250, 9460, 9322, 9252] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266485760 dispatch_meta_bytes=344408 combine_payload_bytes=266485760 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266830168 ingress_bytes=335691776 link_bytes=335691776 token_bytes=533315928 weight_movement_bytes=69206016 total_nvlink_bytes=602521944 weight_to_token_ratio=0.129766 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263181128e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266928128 dispatch_meta_bytes=343848 combine_payload_bytes=266928128 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=267271976 ingress_bytes=405340160 link_bytes=405340160 token_bytes=534200104 weight_movement_bytes=138412032 total_nvlink_bytes=672612136 weight_to_token_ratio=0.259101 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.525262390e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74518 payload_rows_remote=65146 dup_rows=23786 dup_rows_remote=20788 dedup_ratio=0.241964 router_tokens_per_expert=[792, 766, 796, 785, 753, 756, 794, 719, 796, 750, 692, 790, 747, 779, 780, 746, 758, 753, 790, 780, 780, 765, 770, 763, 750, 791, 753, 772, 745, 725, 790, 786, 772, 761, 820, 752, 769, 784, 764, 763, 751, 736, 826, 801, 791, 804, 761, 739, 762, 755, 741, 747, 742, 805, 779, 808, 785, 728, 804, 745, 767, 791, 741, 746] payload_rows_per_dst=[9374, 9344, 9328, 9214, 9372, 9342, 9262, 9282] dup_rows_per_dst=[2948, 2816, 2990, 3010, 2998, 3076, 3016, 2932] combine_rows_per_src=[9374, 9344, 9328, 9214, 9372, 9342, 9262, 9282] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=343704 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266403480 ingress_bytes=369868800 link_bytes=369868800 token_bytes=532463256 weight_movement_bytes=103809024 total_nvlink_bytes=636272280 weight_to_token_ratio=0.194960 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391786518e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74108 payload_rows_remote=64978 dup_rows=24196 dup_rows_remote=21148 dedup_ratio=0.246134 router_tokens_per_expert=[778, 787, 741, 784, 750, 766, 758, 725, 774, 797, 790, 751, 753, 738, 803, 810, 771, 776, 766, 773, 771, 727, 753, 737, 778, 788, 757, 740, 751, 749, 776, 783, 758, 769, 741, 804, 741, 759, 803, 812, 798, 746, 801, 773, 777, 790, 776, 768, 711, 773, 791, 720, 814, 747, 799, 807, 734, 781, 745, 754, 766, 755, 764, 774] payload_rows_per_dst=[9130, 9582, 9378, 9124, 9266, 9616, 9266, 8746] dup_rows_per_dst=[3048, 3364, 3078, 2882, 3108, 3126, 2878, 2712] combine_rows_per_src=[9130, 9582, 9378, 9124, 9266, 9616, 9266, 8746] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=344048 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265846768 ingress_bytes=369311744 link_bytes=369311744 token_bytes=531349488 weight_movement_bytes=103809024 total_nvlink_bytes=635158512 weight_to_token_ratio=0.195369 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389690361e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=267395072 dispatch_meta_bytes=344912 combine_payload_bytes=267395072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267739984 ingress_bytes=301998080 link_bytes=301998080 token_bytes=535135056 weight_movement_bytes=34603008 total_nvlink_bytes=569738064 weight_to_token_ratio=0.064662 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136394462e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266657792 dispatch_meta_bytes=344872 combine_payload_bytes=266657792 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267002664 ingress_bytes=301260800 link_bytes=301260800 token_bytes=533660456 weight_movement_bytes=34603008 total_nvlink_bytes=568263464 weight_to_token_ratio=0.064841 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133620137e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265183232 dispatch_meta_bytes=343440 combine_payload_bytes=265183232 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265526672 ingress_bytes=334389248 link_bytes=334389248 token_bytes=530709904 weight_movement_bytes=69206016 total_nvlink_bytes=599915920 weight_to_token_ratio=0.130403 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258279820e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73988 payload_rows_remote=64814 dup_rows=24316 dup_rows_remote=21098 dedup_ratio=0.247355 router_tokens_per_expert=[805, 764, 771, 732, 762, 750, 767, 770, 762, 756, 724, 754, 734, 772, 733, 795, 794, 776, 788, 786, 752, 782, 725, 769, 798, 797, 773, 763, 795, 754, 774, 787, 777, 851, 722, 768, 763, 778, 766, 812, 780, 755, 745, 713, 715, 748, 741, 739, 771, 798, 787, 749, 767, 776, 802, 791, 790, 754, 782, 761, 756, 764, 794, 773] payload_rows_per_dst=[9174, 9078, 9450, 9386, 9224, 9078, 9364, 9234] dup_rows_per_dst=[3218, 2848, 3224, 3096, 2908, 2876, 3032, 3114] combine_rows_per_src=[9174, 9078, 9450, 9386, 9224, 9078, 9364, 9234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266838016 dispatch_meta_bytes=343736 combine_payload_bytes=266838016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267181752 ingress_bytes=301441024 link_bytes=301441024 token_bytes=534019768 weight_movement_bytes=34603008 total_nvlink_bytes=568622776 weight_to_token_ratio=0.064797 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134298305e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265478144 dispatch_meta_bytes=343648 combine_payload_bytes=265478144 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265821792 ingress_bytes=300081152 link_bytes=300081152 token_bytes=531299936 weight_movement_bytes=34603008 total_nvlink_bytes=565902944 weight_to_token_ratio=0.065129 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129181216e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74114 payload_rows_remote=64754 dup_rows=24190 dup_rows_remote=20928 dedup_ratio=0.246073 router_tokens_per_expert=[775, 780, 784, 806, 759, 750, 769, 783, 816, 837, 787, 786, 720, 769, 757, 769, 745, 777, 831, 744, 742, 791, 775, 747, 770, 821, 721, 757, 756, 749, 781, 784, 816, 803, 798, 791, 761, 712, 707, 812, 735, 771, 764, 789, 725, 799, 732, 795, 743, 718, 796, 776, 707, 755, 761, 776, 772, 733, 804, 724, 766, 780, 728, 765] payload_rows_per_dst=[9360, 9406, 9276, 8988, 9338, 9262, 9124, 9360] dup_rows_per_dst=[3052, 3030, 3028, 2858, 3062, 2958, 2940, 3262] combine_rows_per_src=[9360, 9406, 9276, 8988, 9338, 9262, 9124, 9360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=65018 dup_rows=23998 dup_rows_remote=21022 dedup_ratio=0.244120 router_tokens_per_expert=[794, 735, 798, 778, 773, 776, 760, 771, 757, 746, 766, 811, 753, 793, 711, 795, 782, 794, 785, 747, 795, 823, 776, 757, 795, 727, 759, 727, 739, 775, 736, 741, 800, 747, 778, 738, 806, 802, 814, 711, 787, 772, 796, 683, 732, 718, 766, 796, 789, 796, 798, 685, 755, 823, 757, 747, 749, 776, 754, 750, 754, 781, 874, 743] payload_rows_per_dst=[9332, 9288, 9388, 9092, 9276, 9266, 9344, 9320] dup_rows_per_dst=[3038, 2976, 3130, 2906, 3116, 2834, 2956, 3042] combine_rows_per_src=[9332, 9288, 9388, 9092, 9276, 9266, 9344, 9320] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=65060 dup_rows=24010 dup_rows_remote=21042 dedup_ratio=0.244242 router_tokens_per_expert=[750, 747, 834, 738, 758, 827, 788, 727, 787, 757, 778, 746, 732, 800, 779, 741, 812, 799, 767, 738, 760, 718, 782, 792, 783, 738, 741, 724, 841, 765, 762, 747, 745, 782, 753, 731, 772, 781, 771, 775, 778, 784, 789, 734, 808, 783, 728, 767, 777, 739, 775, 771, 724, 822, 777, 762, 744, 766, 794, 801, 812, 772, 751, 726] payload_rows_per_dst=[9450, 9214, 9264, 9234, 9266, 9244, 9300, 9322] dup_rows_per_dst=[2888, 3026, 3072, 2968, 2954, 3098, 2994, 3010] combine_rows_per_src=[9450, 9214, 9264, 9234, 9266, 9244, 9300, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74502 payload_rows_remote=65188 dup_rows=23802 dup_rows_remote=20842 dedup_ratio=0.242126 router_tokens_per_expert=[771, 759, 744, 791, 771, 783, 754, 784, 771, 772, 766, 778, 757, 783, 769, 800, 764, 730, 791, 741, 749, 780, 808, 778, 757, 765, 750, 791, 750, 746, 772, 742, 794, 764, 741, 730, 766, 750, 706, 760, 771, 753, 778, 806, 781, 778, 765, 705, 755, 750, 836, 783, 749, 751, 790, 765, 804, 782, 788, 788, 747, 775, 789, 785] payload_rows_per_dst=[9334, 9332, 9316, 9270, 9096, 9314, 9304, 9536] dup_rows_per_dst=[2980, 3060, 2966, 2876, 2926, 2960, 3054, 2980] combine_rows_per_src=[9334, 9332, 9316, 9270, 9096, 9314, 9304, 9536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64868 dup_rows=24114 dup_rows_remote=21020 dedup_ratio=0.245300 router_tokens_per_expert=[754, 789, 790, 763, 764, 782, 782, 774, 768, 808, 788, 790, 812, 722, 802, 783, 740, 806, 823, 772, 718, 762, 738, 754, 719, 725, 717, 753, 773, 792, 816, 771, 758, 746, 769, 795, 757, 797, 772, 763, 748, 816, 777, 724, 727, 749, 792, 746, 758, 772, 807, 789, 783, 789, 790, 720, 805, 750, 776, 737, 754, 698, 744, 794] payload_rows_per_dst=[9344, 9438, 9322, 9132, 9228, 9232, 9322, 9172] dup_rows_per_dst=[3052, 3108, 2904, 3000, 3086, 2926, 3094, 2944] combine_rows_per_src=[9344, 9438, 9322, 9132, 9228, 9232, 9322, 9172] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74216 payload_rows_remote=64944 dup_rows=24088 dup_rows_remote=21124 dedup_ratio=0.245036 router_tokens_per_expert=[742, 742, 723, 786, 763, 830, 779, 787, 807, 740, 765, 773, 764, 817, 766, 758, 757, 786, 721, 799, 765, 774, 765, 751, 724, 770, 719, 734, 775, 789, 758, 773, 786, 796, 760, 770, 792, 775, 773, 779, 745, 765, 757, 783, 769, 790, 741, 767, 737, 832, 786, 754, 780, 710, 748, 771, 796, 741, 771, 797, 732, 785, 774, 788] payload_rows_per_dst=[9340, 9328, 9272, 9130, 9310, 9294, 9198, 9344] dup_rows_per_dst=[2964, 3052, 2964, 2954, 3152, 2940, 3038, 3024] combine_rows_per_src=[9340, 9328, 9272, 9130, 9310, 9294, 9198, 9344] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265814016 dispatch_meta_bytes=343880 combine_payload_bytes=265814016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266157896 ingress_bytes=300417024 link_bytes=300417024 token_bytes=531971912 weight_movement_bytes=34603008 total_nvlink_bytes=566574920 weight_to_token_ratio=0.065047 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130445076e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265232384 dispatch_meta_bytes=342728 combine_payload_bytes=265232384 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265575112 ingress_bytes=299835392 link_bytes=299835392 token_bytes=530807496 weight_movement_bytes=34603008 total_nvlink_bytes=565410504 weight_to_token_ratio=0.065189 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128256441e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=344160 combine_payload_bytes=266313728 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266657888 ingress_bytes=335519744 link_bytes=335519744 token_bytes=532971616 weight_movement_bytes=69206016 total_nvlink_bytes=602177632 weight_to_token_ratio=0.129849 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262533785e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=64894 dup_rows=24050 dup_rows_remote=21040 dedup_ratio=0.244649 router_tokens_per_expert=[771, 758, 772, 757, 790, 745, 761, 768, 734, 814, 761, 776, 769, 793, 752, 763, 820, 771, 760, 780, 765, 788, 759, 756, 819, 729, 794, 825, 814, 730, 754, 731, 746, 746, 812, 755, 789, 804, 735, 798, 771, 746, 819, 760, 750, 774, 796, 814, 764, 738, 757, 769, 718, 749, 781, 805, 748, 765, 738, 753, 776, 743, 734, 720] payload_rows_per_dst=[9236, 9258, 9310, 9362, 9360, 9376, 9258, 9094] dup_rows_per_dst=[3008, 3066, 3088, 3030, 3010, 3084, 2904, 2860] combine_rows_per_src=[9236, 9258, 9310, 9362, 9360, 9376, 9258, 9094] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64868 dup_rows=24114 dup_rows_remote=21020 dedup_ratio=0.245300 router_tokens_per_expert=[754, 789, 790, 763, 764, 782, 782, 774, 768, 808, 788, 790, 812, 722, 802, 783, 740, 806, 823, 772, 718, 762, 738, 754, 719, 725, 717, 753, 773, 792, 816, 771, 758, 746, 769, 795, 757, 797, 772, 763, 748, 816, 777, 724, 727, 749, 792, 746, 758, 772, 807, 789, 783, 789, 790, 720, 805, 750, 776, 737, 754, 698, 744, 794] payload_rows_per_dst=[9344, 9438, 9322, 9132, 9228, 9232, 9322, 9172] dup_rows_per_dst=[3052, 3108, 2904, 3000, 3086, 2926, 3094, 2944] combine_rows_per_src=[9344, 9438, 9322, 9132, 9228, 9232, 9322, 9172] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74604 payload_rows_remote=65296 dup_rows=23700 dup_rows_remote=20716 dedup_ratio=0.241089 router_tokens_per_expert=[733, 743, 759, 768, 781, 811, 848, 736, 818, 745, 807, 790, 765, 739, 779, 721, 779, 772, 766, 766, 743, 731, 799, 735, 733, 786, 726, 757, 793, 704, 762, 801, 789, 775, 748, 774, 750, 773, 754, 761, 804, 772, 782, 785, 762, 732, 761, 748, 802, 787, 778, 798, 777, 783, 775, 820, 769, 791, 706, 721, 764, 746, 795, 774] payload_rows_per_dst=[9370, 9338, 9264, 9198, 9320, 9308, 9558, 9248] dup_rows_per_dst=[2988, 2990, 2918, 2926, 2928, 2984, 3082, 2884] combine_rows_per_src=[9370, 9338, 9264, 9198, 9320, 9308, 9558, 9248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=344504 combine_payload_bytes=266149888 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266494392 ingress_bytes=300752896 link_bytes=300752896 token_bytes=532644280 weight_movement_bytes=34603008 total_nvlink_bytes=567247288 weight_to_token_ratio=0.064965 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131708935e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74228 payload_rows_remote=64718 dup_rows=24076 dup_rows_remote=20848 dedup_ratio=0.244914 router_tokens_per_expert=[817, 760, 763, 797, 751, 858, 787, 784, 784, 778, 784, 755, 803, 745, 771, 757, 805, 753, 807, 758, 772, 795, 810, 794, 750, 755, 797, 763, 778, 740, 786, 747, 767, 775, 776, 751, 713, 734, 769, 764, 726, 758, 758, 788, 793, 762, 803, 737, 772, 743, 729, 749, 756, 809, 702, 766, 753, 767, 805, 704, 757, 733, 746, 783] payload_rows_per_dst=[9510, 9306, 9490, 9320, 9092, 9240, 9048, 9222] dup_rows_per_dst=[3228, 3050, 3096, 2912, 3006, 3010, 2900, 2874] combine_rows_per_src=[9510, 9306, 9490, 9320, 9092, 9240, 9048, 9222] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266485760 dispatch_meta_bytes=344408 combine_payload_bytes=266485760 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266830168 ingress_bytes=335691776 link_bytes=335691776 token_bytes=533315928 weight_movement_bytes=69206016 total_nvlink_bytes=602521944 weight_to_token_ratio=0.129766 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263181128e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=267010048 dispatch_meta_bytes=344120 combine_payload_bytes=267010048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267354168 ingress_bytes=301613056 link_bytes=301613056 token_bytes=534364216 weight_movement_bytes=34603008 total_nvlink_bytes=568967224 weight_to_token_ratio=0.064755 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134945648e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265699328 dispatch_meta_bytes=343552 combine_payload_bytes=265699328 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266042880 ingress_bytes=369508352 link_bytes=369508352 token_bytes=531742208 weight_movement_bytes=103809024 total_nvlink_bytes=635551232 weight_to_token_ratio=0.195224 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390430181e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266010624 dispatch_meta_bytes=344272 combine_payload_bytes=266010624 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266354896 ingress_bytes=369819648 link_bytes=369819648 token_bytes=532365520 weight_movement_bytes=103809024 total_nvlink_bytes=636174544 weight_to_token_ratio=0.194996 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391601562e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265805824 dispatch_meta_bytes=343736 combine_payload_bytes=265805824 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266149560 ingress_bytes=369614848 link_bytes=369614848 token_bytes=531955384 weight_movement_bytes=103809024 total_nvlink_bytes=635764408 weight_to_token_ratio=0.195146 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390830917e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265084928 dispatch_meta_bytes=342264 combine_payload_bytes=265084928 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265427192 ingress_bytes=334290944 link_bytes=334290944 token_bytes=530512120 weight_movement_bytes=69206016 total_nvlink_bytes=599718136 weight_to_token_ratio=0.130451 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257909910e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74196 payload_rows_remote=64594 dup_rows=24108 dup_rows_remote=20782 dedup_ratio=0.245239 router_tokens_per_expert=[807, 817, 775, 796, 743, 766, 775, 771, 772, 771, 776, 749, 800, 740, 765, 716, 819, 774, 755, 749, 774, 740, 748, 763, 743, 783, 744, 807, 763, 781, 765, 758, 754, 744, 796, 771, 733, 766, 790, 734, 811, 768, 787, 725, 740, 753, 775, 703, 743, 784, 748, 773, 739, 776, 768, 787, 763, 807, 810, 781, 776, 768, 811, 763] payload_rows_per_dst=[8936, 9160, 9170, 9716, 9352, 9256, 9004, 9602] dup_rows_per_dst=[2666, 2780, 2864, 3470, 2990, 3106, 2906, 3326] combine_rows_per_src=[8936, 9160, 9170, 9716, 9352, 9256, 9004, 9602] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=65044 dup_rows=23982 dup_rows_remote=21088 dedup_ratio=0.243958 router_tokens_per_expert=[774, 745, 817, 743, 783, 720, 774, 810, 832, 769, 787, 788, 800, 791, 765, 740, 736, 800, 772, 750, 742, 777, 742, 783, 777, 756, 775, 801, 710, 766, 743, 758, 745, 735, 720, 777, 778, 746, 754, 802, 760, 771, 790, 825, 810, 715, 730, 799, 788, 732, 758, 751, 844, 762, 791, 789, 766, 715, 751, 747, 747, 772, 791, 765] payload_rows_per_dst=[9234, 9418, 9234, 9278, 9206, 9362, 9380, 9210] dup_rows_per_dst=[3098, 3126, 2970, 2894, 2908, 3038, 3050, 2898] combine_rows_per_src=[9234, 9418, 9234, 9278, 9206, 9362, 9380, 9210] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74066 payload_rows_remote=64788 dup_rows=24238 dup_rows_remote=21346 dedup_ratio=0.246562 router_tokens_per_expert=[787, 769, 797, 766, 747, 809, 819, 726, 771, 714, 774, 783, 733, 783, 742, 785, 749, 752, 782, 706, 743, 761, 774, 747, 772, 765, 779, 760, 770, 733, 760, 764, 739, 784, 846, 760, 792, 786, 814, 744, 739, 789, 789, 764, 773, 753, 764, 747, 781, 786, 758, 748, 772, 761, 728, 750, 758, 778, 793, 732, 805, 785, 825, 787] payload_rows_per_dst=[9400, 9278, 9072, 9168, 9338, 9204, 9192, 9414] dup_rows_per_dst=[3040, 2892, 2956, 3038, 3192, 3032, 2976, 3112] combine_rows_per_src=[9400, 9278, 9072, 9168, 9338, 9204, 9192, 9414] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265699328 dispatch_meta_bytes=343552 combine_payload_bytes=265699328 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266042880 ingress_bytes=369508352 link_bytes=369508352 token_bytes=531742208 weight_movement_bytes=103809024 total_nvlink_bytes=635551232 weight_to_token_ratio=0.195224 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390430181e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=65018 dup_rows=23998 dup_rows_remote=21022 dedup_ratio=0.244120 router_tokens_per_expert=[794, 735, 798, 778, 773, 776, 760, 771, 757, 746, 766, 811, 753, 793, 711, 795, 782, 794, 785, 747, 795, 823, 776, 757, 795, 727, 759, 727, 739, 775, 736, 741, 800, 747, 778, 738, 806, 802, 814, 711, 787, 772, 796, 683, 732, 718, 766, 796, 789, 796, 798, 685, 755, 823, 757, 747, 749, 776, 754, 750, 754, 781, 874, 743] payload_rows_per_dst=[9332, 9288, 9388, 9092, 9276, 9266, 9344, 9320] dup_rows_per_dst=[3038, 2976, 3130, 2906, 3116, 2834, 2956, 3042] combine_rows_per_src=[9332, 9288, 9388, 9092, 9276, 9266, 9344, 9320] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=267452416 dispatch_meta_bytes=344048 combine_payload_bytes=267452416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267796464 ingress_bytes=302055424 link_bytes=302055424 token_bytes=535248880 weight_movement_bytes=34603008 total_nvlink_bytes=569851888 weight_to_token_ratio=0.064648 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136610243e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=65044 dup_rows=23982 dup_rows_remote=21088 dedup_ratio=0.243958 router_tokens_per_expert=[774, 745, 817, 743, 783, 720, 774, 810, 832, 769, 787, 788, 800, 791, 765, 740, 736, 800, 772, 750, 742, 777, 742, 783, 777, 756, 775, 801, 710, 766, 743, 758, 745, 735, 720, 777, 778, 746, 754, 802, 760, 771, 790, 825, 810, 715, 730, 799, 788, 732, 758, 751, 844, 762, 791, 789, 766, 715, 751, 747, 747, 772, 791, 765] payload_rows_per_dst=[9234, 9418, 9234, 9278, 9206, 9362, 9380, 9210] dup_rows_per_dst=[3098, 3126, 2970, 2894, 2908, 3038, 3050, 2898] combine_rows_per_src=[9234, 9418, 9234, 9278, 9206, 9362, 9380, 9210] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74506 payload_rows_remote=65220 dup_rows=23798 dup_rows_remote=20822 dedup_ratio=0.242086 router_tokens_per_expert=[799, 735, 769, 807, 745, 707, 733, 783, 726, 741, 732, 755, 726, 771, 787, 738, 729, 754, 757, 820, 785, 755, 735, 771, 734, 819, 758, 751, 810, 777, 768, 832, 776, 750, 780, 783, 772, 756, 783, 797, 810, 761, 799, 733, 741, 822, 784, 749, 750, 731, 843, 778, 766, 756, 745, 762, 766, 739, 733, 767, 795, 818, 806, 792] payload_rows_per_dst=[9196, 9194, 9230, 9390, 9426, 9352, 9286, 9432] dup_rows_per_dst=[2960, 2758, 2982, 3108, 2968, 3046, 2976, 3000] combine_rows_per_src=[9196, 9194, 9230, 9390, 9426, 9352, 9286, 9432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74604 payload_rows_remote=65296 dup_rows=23700 dup_rows_remote=20716 dedup_ratio=0.241089 router_tokens_per_expert=[733, 743, 759, 768, 781, 811, 848, 736, 818, 745, 807, 790, 765, 739, 779, 721, 779, 772, 766, 766, 743, 731, 799, 735, 733, 786, 726, 757, 793, 704, 762, 801, 789, 775, 748, 774, 750, 773, 754, 761, 804, 772, 782, 785, 762, 732, 761, 748, 802, 787, 778, 798, 777, 783, 775, 820, 769, 791, 706, 721, 764, 746, 795, 774] payload_rows_per_dst=[9370, 9338, 9264, 9198, 9320, 9308, 9558, 9248] dup_rows_per_dst=[2988, 2990, 2918, 2926, 2928, 2984, 3082, 2884] combine_rows_per_src=[9370, 9338, 9264, 9198, 9320, 9308, 9558, 9248] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=264577024 dispatch_meta_bytes=341504 combine_payload_bytes=264577024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264918528 ingress_bytes=299180032 link_bytes=299180032 token_bytes=529495552 weight_movement_bytes=34603008 total_nvlink_bytes=564098560 weight_to_token_ratio=0.065351 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125790374e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74316 payload_rows_remote=64886 dup_rows=23988 dup_rows_remote=20960 dedup_ratio=0.244019 router_tokens_per_expert=[749, 754, 788, 741, 758, 799, 765, 803, 761, 750, 769, 722, 752, 793, 820, 816, 773, 769, 719, 772, 747, 733, 729, 737, 761, 779, 763, 788, 813, 750, 760, 753, 773, 809, 745, 810, 791, 750, 748, 803, 764, 736, 718, 729, 762, 771, 782, 795, 790, 796, 777, 790, 779, 767, 787, 737, 752, 763, 786, 774, 770, 785, 760, 767] payload_rows_per_dst=[9316, 9312, 9094, 9300, 9430, 9204, 9280, 9380] dup_rows_per_dst=[2998, 3054, 2864, 3034, 3028, 2910, 3166, 2934] combine_rows_per_src=[9316, 9312, 9094, 9300, 9430, 9204, 9280, 9380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=344528 combine_payload_bytes=266420224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266764752 ingress_bytes=301023232 link_bytes=301023232 token_bytes=533184976 weight_movement_bytes=34603008 total_nvlink_bytes=567787984 weight_to_token_ratio=0.064899 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132726188e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73988 payload_rows_remote=64814 dup_rows=24316 dup_rows_remote=21098 dedup_ratio=0.247355 router_tokens_per_expert=[805, 764, 771, 732, 762, 750, 767, 770, 762, 756, 724, 754, 734, 772, 733, 795, 794, 776, 788, 786, 752, 782, 725, 769, 798, 797, 773, 763, 795, 754, 774, 787, 777, 851, 722, 768, 763, 778, 766, 812, 780, 755, 745, 713, 715, 748, 741, 739, 771, 798, 787, 749, 767, 776, 802, 791, 790, 754, 782, 761, 756, 764, 794, 773] payload_rows_per_dst=[9174, 9078, 9450, 9386, 9224, 9078, 9364, 9234] dup_rows_per_dst=[3218, 2848, 3224, 3096, 2908, 2876, 3032, 3114] combine_rows_per_src=[9174, 9078, 9450, 9386, 9224, 9078, 9364, 9234] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=344160 combine_payload_bytes=266313728 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266657888 ingress_bytes=335519744 link_bytes=335519744 token_bytes=532971616 weight_movement_bytes=69206016 total_nvlink_bytes=602177632 weight_to_token_ratio=0.129849 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262533785e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265371648 dispatch_meta_bytes=344536 combine_payload_bytes=265371648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265716184 ingress_bytes=299974656 link_bytes=299974656 token_bytes=531087832 weight_movement_bytes=34603008 total_nvlink_bytes=565690840 weight_to_token_ratio=0.065155 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128780481e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74146 payload_rows_remote=64896 dup_rows=24158 dup_rows_remote=21074 dedup_ratio=0.245748 router_tokens_per_expert=[779, 778, 763, 783, 763, 757, 723, 756, 731, 720, 769, 730, 766, 742, 781, 734, 789, 800, 748, 779, 755, 711, 800, 777, 794, 793, 778, 742, 816, 751, 784, 730, 753, 738, 765, 768, 808, 788, 783, 764, 822, 780, 813, 751, 752, 808, 773, 755, 791, 791, 751, 760, 771, 763, 777, 808, 750, 757, 798, 760, 763, 789, 735, 745] payload_rows_per_dst=[9236, 9070, 9290, 9266, 9250, 9460, 9322, 9252] dup_rows_per_dst=[2968, 2876, 3028, 3110, 3084, 3048, 3102, 2942] combine_rows_per_src=[9236, 9070, 9290, 9266, 9250, 9460, 9322, 9252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74462 payload_rows_remote=65064 dup_rows=23842 dup_rows_remote=20880 dedup_ratio=0.242533 router_tokens_per_expert=[727, 741, 784, 765, 753, 782, 773, 728, 775, 783, 771, 807, 753, 753, 729, 806, 795, 779, 771, 774, 767, 783, 737, 774, 792, 728, 777, 731, 756, 762, 769, 793, 796, 774, 761, 769, 766, 802, 758, 771, 739, 761, 753, 798, 809, 754, 722, 788, 743, 799, 807, 721, 727, 776, 797, 792, 775, 760, 764, 778, 777, 751, 789, 757] payload_rows_per_dst=[9220, 9324, 9398, 9258, 9330, 9278, 9364, 9290] dup_rows_per_dst=[2886, 3030, 2962, 2958, 3064, 2970, 2960, 3012] combine_rows_per_src=[9220, 9324, 9398, 9258, 9330, 9278, 9364, 9290] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=343384 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266116440 ingress_bytes=369582080 link_bytes=369582080 token_bytes=531889496 weight_movement_bytes=103809024 total_nvlink_bytes=635698520 weight_to_token_ratio=0.195170 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390707613e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74492 payload_rows_remote=65022 dup_rows=23812 dup_rows_remote=20782 dedup_ratio=0.242228 router_tokens_per_expert=[779, 778, 787, 747, 789, 724, 795, 781, 774, 771, 772, 796, 772, 765, 766, 781, 779, 762, 768, 775, 806, 756, 778, 765, 760, 787, 772, 746, 742, 800, 715, 773, 804, 783, 756, 771, 779, 760, 740, 738, 780, 768, 769, 768, 799, 760, 761, 759, 756, 768, 747, 734, 750, 759, 763, 764, 782, 751, 753, 793, 790, 765, 754, 767] payload_rows_per_dst=[9470, 9490, 9502, 9254, 9188, 9380, 9142, 9066] dup_rows_per_dst=[3030, 2914, 3266, 3020, 2952, 2948, 2940, 2742] combine_rows_per_src=[9470, 9490, 9502, 9254, 9188, 9380, 9142, 9066] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=344528 combine_payload_bytes=266420224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266764752 ingress_bytes=301023232 link_bytes=301023232 token_bytes=533184976 weight_movement_bytes=34603008 total_nvlink_bytes=567787984 weight_to_token_ratio=0.064899 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132726188e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265478144 dispatch_meta_bytes=343648 combine_payload_bytes=265478144 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265821792 ingress_bytes=300081152 link_bytes=300081152 token_bytes=531299936 weight_movement_bytes=34603008 total_nvlink_bytes=565902944 weight_to_token_ratio=0.065129 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129181216e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74506 payload_rows_remote=65220 dup_rows=23798 dup_rows_remote=20822 dedup_ratio=0.242086 router_tokens_per_expert=[799, 735, 769, 807, 745, 707, 733, 783, 726, 741, 732, 755, 726, 771, 787, 738, 729, 754, 757, 820, 785, 755, 735, 771, 734, 819, 758, 751, 810, 777, 768, 832, 776, 750, 780, 783, 772, 756, 783, 797, 810, 761, 799, 733, 741, 822, 784, 749, 750, 731, 843, 778, 766, 756, 745, 762, 766, 739, 733, 767, 795, 818, 806, 792] payload_rows_per_dst=[9196, 9194, 9230, 9390, 9426, 9352, 9286, 9432] dup_rows_per_dst=[2960, 2758, 2982, 3108, 2968, 3046, 2976, 3000] combine_rows_per_src=[9196, 9194, 9230, 9390, 9426, 9352, 9286, 9432] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=64908 dup_rows=24010 dup_rows_remote=20990 dedup_ratio=0.244242 router_tokens_per_expert=[782, 765, 768, 741, 755, 764, 771, 768, 762, 797, 783, 810, 792, 742, 733, 763, 723, 746, 775, 709, 776, 789, 782, 722, 776, 775, 841, 764, 782, 763, 755, 792, 769, 784, 769, 758, 819, 749, 813, 736, 804, 763, 775, 828, 744, 754, 781, 754, 742, 777, 774, 726, 737, 835, 771, 758, 758, 725, 735, 784, 768, 768, 781, 747] payload_rows_per_dst=[9238, 9270, 9118, 9398, 9418, 9386, 9222, 9244] dup_rows_per_dst=[2990, 3094, 2926, 3098, 2976, 3020, 3018, 2888] combine_rows_per_src=[9238, 9270, 9118, 9398, 9418, 9386, 9222, 9244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=267141120 dispatch_meta_bytes=344168 combine_payload_bytes=267141120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267485288 ingress_bytes=301744128 link_bytes=301744128 token_bytes=534626408 weight_movement_bytes=34603008 total_nvlink_bytes=569229416 weight_to_token_ratio=0.064724 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135438861e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=267452416 dispatch_meta_bytes=344048 combine_payload_bytes=267452416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267796464 ingress_bytes=302055424 link_bytes=302055424 token_bytes=535248880 weight_movement_bytes=34603008 total_nvlink_bytes=569851888 weight_to_token_ratio=0.064648 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136610243e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265814016 dispatch_meta_bytes=343880 combine_payload_bytes=265814016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266157896 ingress_bytes=300417024 link_bytes=300417024 token_bytes=531971912 weight_movement_bytes=34603008 total_nvlink_bytes=566574920 weight_to_token_ratio=0.065047 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130445076e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266502144 dispatch_meta_bytes=343776 combine_payload_bytes=266502144 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266845920 ingress_bytes=301105152 link_bytes=301105152 token_bytes=533348064 weight_movement_bytes=34603008 total_nvlink_bytes=567951072 weight_to_token_ratio=0.064879 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133034446e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266330112 dispatch_meta_bytes=343216 combine_payload_bytes=266330112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266673328 ingress_bytes=300933120 link_bytes=300933120 token_bytes=533003440 weight_movement_bytes=34603008 total_nvlink_bytes=567606448 weight_to_token_ratio=0.064921 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132387103e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74418 payload_rows_remote=65174 dup_rows=23886 dup_rows_remote=20854 dedup_ratio=0.242981 router_tokens_per_expert=[785, 767, 800, 762, 789, 798, 791, 745, 778, 809, 790, 803, 734, 750, 749, 753, 771, 720, 759, 743, 814, 755, 802, 801, 788, 773, 761, 741, 776, 816, 723, 749, 746, 768, 755, 760, 750, 714, 799, 776, 800, 799, 720, 773, 767, 760, 797, 736, 764, 740, 795, 769, 755, 758, 752, 766, 783, 761, 739, 768, 818, 786, 708, 775] payload_rows_per_dst=[9468, 9274, 9440, 9128, 9028, 9464, 9372, 9244] dup_rows_per_dst=[3006, 3058, 2890, 2832, 2780, 3134, 3154, 3032] combine_rows_per_src=[9468, 9274, 9440, 9128, 9028, 9464, 9372, 9244] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74110 payload_rows_remote=64790 dup_rows=24194 dup_rows_remote=21194 dedup_ratio=0.246114 router_tokens_per_expert=[721, 755, 743, 804, 762, 759, 766, 733, 754, 805, 795, 730, 748, 743, 798, 808, 783, 746, 763, 834, 768, 804, 807, 797, 769, 762, 774, 792, 784, 792, 737, 750, 722, 750, 791, 778, 728, 771, 782, 771, 780, 742, 766, 742, 766, 718, 718, 756, 775, 717, 737, 799, 775, 756, 786, 783, 803, 763, 765, 762, 787, 767, 812, 798] payload_rows_per_dst=[9184, 9350, 9446, 9320, 9190, 9094, 9258, 9268] dup_rows_per_dst=[2902, 3012, 3158, 3000, 2996, 2882, 2998, 3246] combine_rows_per_src=[9184, 9350, 9446, 9320, 9190, 9094, 9258, 9268] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74494 payload_rows_remote=65236 dup_rows=23810 dup_rows_remote=20834 dedup_ratio=0.242208 router_tokens_per_expert=[744, 799, 786, 787, 811, 743, 752, 769, 853, 755, 749, 812, 732, 775, 709, 732, 760, 756, 796, 763, 778, 765, 756, 764, 779, 760, 733, 762, 790, 735, 742, 732, 755, 802, 745, 732, 825, 786, 735, 818, 769, 733, 753, 741, 778, 804, 758, 785, 743, 804, 780, 772, 811, 778, 756, 785, 793, 768, 692, 746, 749, 781, 764, 832] payload_rows_per_dst=[9298, 9258, 9310, 9216, 9372, 9270, 9448, 9322] dup_rows_per_dst=[3084, 2976, 2966, 2850, 3024, 2972, 3010, 2928] combine_rows_per_src=[9298, 9258, 9310, 9216, 9372, 9270, 9448, 9322] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=267141120 dispatch_meta_bytes=344168 combine_payload_bytes=267141120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267485288 ingress_bytes=301744128 link_bytes=301744128 token_bytes=534626408 weight_movement_bytes=34603008 total_nvlink_bytes=569229416 weight_to_token_ratio=0.064724 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135438861e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265863168 dispatch_meta_bytes=343592 combine_payload_bytes=265863168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266206760 ingress_bytes=300466176 link_bytes=300466176 token_bytes=532069928 weight_movement_bytes=34603008 total_nvlink_bytes=566672936 weight_to_token_ratio=0.065035 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130630031e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74066 payload_rows_remote=64788 dup_rows=24238 dup_rows_remote=21346 dedup_ratio=0.246562 router_tokens_per_expert=[787, 769, 797, 766, 747, 809, 819, 726, 771, 714, 774, 783, 733, 783, 742, 785, 749, 752, 782, 706, 743, 761, 774, 747, 772, 765, 779, 760, 770, 733, 760, 764, 739, 784, 846, 760, 792, 786, 814, 744, 739, 789, 789, 764, 773, 753, 764, 747, 781, 786, 758, 748, 772, 761, 728, 750, 758, 778, 793, 732, 805, 785, 825, 787] payload_rows_per_dst=[9400, 9278, 9072, 9168, 9338, 9204, 9192, 9414] dup_rows_per_dst=[3040, 2892, 2956, 3038, 3192, 3032, 2976, 3112] combine_rows_per_src=[9400, 9278, 9072, 9168, 9338, 9204, 9192, 9414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74110 payload_rows_remote=64790 dup_rows=24194 dup_rows_remote=21194 dedup_ratio=0.246114 router_tokens_per_expert=[721, 755, 743, 804, 762, 759, 766, 733, 754, 805, 795, 730, 748, 743, 798, 808, 783, 746, 763, 834, 768, 804, 807, 797, 769, 762, 774, 792, 784, 792, 737, 750, 722, 750, 791, 778, 728, 771, 782, 771, 780, 742, 766, 742, 766, 718, 718, 756, 775, 717, 737, 799, 775, 756, 786, 783, 803, 763, 765, 762, 787, 767, 812, 798] payload_rows_per_dst=[9184, 9350, 9446, 9320, 9190, 9094, 9258, 9268] dup_rows_per_dst=[2902, 3012, 3158, 3000, 2996, 2882, 2998, 3246] combine_rows_per_src=[9184, 9350, 9446, 9320, 9190, 9094, 9258, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74010 payload_rows_remote=64690 dup_rows=24294 dup_rows_remote=21154 dedup_ratio=0.247131 router_tokens_per_expert=[752, 743, 762, 755, 756, 778, 779, 779, 750, 786, 766, 773, 777, 765, 748, 795, 798, 762, 798, 782, 737, 766, 770, 733, 745, 788, 768, 732, 743, 782, 765, 749, 755, 765, 800, 709, 804, 744, 771, 752, 816, 773, 760, 768, 759, 745, 861, 736, 745, 783, 745, 808, 754, 782, 807, 806, 781, 777, 789, 778, 735, 745, 764, 753] payload_rows_per_dst=[9186, 9304, 9144, 9140, 9246, 9360, 9320, 9310] dup_rows_per_dst=[3022, 3016, 3148, 3004, 2954, 3076, 3140, 2934] combine_rows_per_src=[9186, 9304, 9144, 9140, 9246, 9360, 9320, 9310] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266952704 dispatch_meta_bytes=344112 combine_payload_bytes=266952704 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267296816 ingress_bytes=405364736 link_bytes=405364736 token_bytes=534249520 weight_movement_bytes=138412032 total_nvlink_bytes=672661552 weight_to_token_ratio=0.259078 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.525354867e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265379840 dispatch_meta_bytes=343936 combine_payload_bytes=265379840 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265723776 ingress_bytes=334585856 link_bytes=334585856 token_bytes=531103616 weight_movement_bytes=69206016 total_nvlink_bytes=600309632 weight_to_token_ratio=0.130306 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259019640e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=65116 dup_rows=24002 dup_rows_remote=21130 dedup_ratio=0.244161 router_tokens_per_expert=[789, 767, 770, 760, 747, 818, 777, 748, 788, 770, 775, 800, 767, 750, 799, 760, 762, 816, 769, 734, 781, 773, 727, 796, 766, 792, 794, 749, 799, 772, 745, 801, 737, 776, 776, 720, 732, 770, 748, 770, 795, 791, 749, 762, 780, 813, 794, 784, 816, 733, 798, 725, 746, 742, 769, 789, 744, 743, 730, 761, 731, 711, 758, 798] payload_rows_per_dst=[9306, 9308, 9304, 9332, 9186, 9364, 9356, 9146] dup_rows_per_dst=[3046, 3110, 3012, 3104, 2872, 3172, 2880, 2806] combine_rows_per_src=[9306, 9308, 9304, 9332, 9186, 9364, 9356, 9146] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74228 payload_rows_remote=64718 dup_rows=24076 dup_rows_remote=20848 dedup_ratio=0.244914 router_tokens_per_expert=[817, 760, 763, 797, 751, 858, 787, 784, 784, 778, 784, 755, 803, 745, 771, 757, 805, 753, 807, 758, 772, 795, 810, 794, 750, 755, 797, 763, 778, 740, 786, 747, 767, 775, 776, 751, 713, 734, 769, 764, 726, 758, 758, 788, 793, 762, 803, 737, 772, 743, 729, 749, 756, 809, 702, 766, 753, 767, 805, 704, 757, 733, 746, 783] payload_rows_per_dst=[9510, 9306, 9490, 9320, 9092, 9240, 9048, 9222] dup_rows_per_dst=[3228, 3050, 3096, 2912, 3006, 3010, 2900, 2874] combine_rows_per_src=[9510, 9306, 9490, 9320, 9092, 9240, 9048, 9222] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=267206656 dispatch_meta_bytes=344280 combine_payload_bytes=267206656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267550936 ingress_bytes=301809664 link_bytes=301809664 token_bytes=534757592 weight_movement_bytes=34603008 total_nvlink_bytes=569360600 weight_to_token_ratio=0.064708 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135685468e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74316 payload_rows_remote=64886 dup_rows=23988 dup_rows_remote=20960 dedup_ratio=0.244019 router_tokens_per_expert=[749, 754, 788, 741, 758, 799, 765, 803, 761, 750, 769, 722, 752, 793, 820, 816, 773, 769, 719, 772, 747, 733, 729, 737, 761, 779, 763, 788, 813, 750, 760, 753, 773, 809, 745, 810, 791, 750, 748, 803, 764, 736, 718, 729, 762, 771, 782, 795, 790, 796, 777, 790, 779, 767, 787, 737, 752, 763, 786, 774, 770, 785, 760, 767] payload_rows_per_dst=[9316, 9312, 9094, 9300, 9430, 9204, 9280, 9380] dup_rows_per_dst=[2998, 3054, 2864, 3034, 3028, 2910, 3166, 2934] combine_rows_per_src=[9316, 9312, 9094, 9300, 9430, 9204, 9280, 9380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=64908 dup_rows=24010 dup_rows_remote=20990 dedup_ratio=0.244242 router_tokens_per_expert=[782, 765, 768, 741, 755, 764, 771, 768, 762, 797, 783, 810, 792, 742, 733, 763, 723, 746, 775, 709, 776, 789, 782, 722, 776, 775, 841, 764, 782, 763, 755, 792, 769, 784, 769, 758, 819, 749, 813, 736, 804, 763, 775, 828, 744, 754, 781, 754, 742, 777, 774, 726, 737, 835, 771, 758, 758, 725, 735, 784, 768, 768, 781, 747] payload_rows_per_dst=[9238, 9270, 9118, 9398, 9418, 9386, 9222, 9244] dup_rows_per_dst=[2990, 3094, 2926, 3098, 2976, 3020, 3018, 2888] combine_rows_per_src=[9238, 9270, 9118, 9398, 9418, 9386, 9222, 9244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=64992 dup_rows=24028 dup_rows_remote=21012 dedup_ratio=0.244425 router_tokens_per_expert=[765, 724, 777, 809, 740, 780, 801, 714, 740, 774, 758, 735, 756, 766, 771, 797, 816, 749, 751, 788, 770, 746, 752, 778, 774, 810, 762, 793, 769, 803, 784, 785, 742, 742, 777, 774, 696, 766, 779, 796, 792, 757, 755, 742, 772, 804, 739, 736, 805, 752, 690, 742, 751, 758, 751, 789, 785, 793, 787, 780, 816, 821, 757, 769] payload_rows_per_dst=[9242, 9260, 9284, 9444, 9260, 9198, 9114, 9474] dup_rows_per_dst=[2978, 2934, 3016, 3116, 2884, 2996, 2962, 3142] combine_rows_per_src=[9242, 9260, 9284, 9444, 9260, 9198, 9114, 9474] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265371648 dispatch_meta_bytes=344536 combine_payload_bytes=265371648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265716184 ingress_bytes=299974656 link_bytes=299974656 token_bytes=531087832 weight_movement_bytes=34603008 total_nvlink_bytes=565690840 weight_to_token_ratio=0.065155 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128780481e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266715136 dispatch_meta_bytes=344984 combine_payload_bytes=266715136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267060120 ingress_bytes=301318144 link_bytes=301318144 token_bytes=533775256 weight_movement_bytes=34603008 total_nvlink_bytes=568378264 weight_to_token_ratio=0.064827 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133835918e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74080 payload_rows_remote=64816 dup_rows=24224 dup_rows_remote=21210 dedup_ratio=0.246419 router_tokens_per_expert=[749, 759, 753, 767, 808, 750, 713, 757, 744, 760, 801, 801, 756, 724, 755, 732, 839, 776, 756, 753, 745, 784, 800, 802, 765, 793, 751, 819, 824, 755, 708, 730, 733, 775, 755, 789, 737, 798, 761, 793, 759, 805, 771, 780, 784, 771, 781, 801, 745, 779, 781, 781, 697, 798, 764, 739, 815, 762, 788, 757, 741, 755, 762, 766] payload_rows_per_dst=[9264, 9228, 9564, 9118, 9158, 9216, 9300, 9232] dup_rows_per_dst=[3014, 3040, 3112, 2884, 3124, 3122, 2868, 3060] combine_rows_per_src=[9264, 9228, 9564, 9118, 9158, 9216, 9300, 9232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265379840 dispatch_meta_bytes=343936 combine_payload_bytes=265379840 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265723776 ingress_bytes=334585856 link_bytes=334585856 token_bytes=531103616 weight_movement_bytes=69206016 total_nvlink_bytes=600309632 weight_to_token_ratio=0.130306 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259019640e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=264970240 dispatch_meta_bytes=343376 combine_payload_bytes=264970240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265313616 ingress_bytes=299573248 link_bytes=299573248 token_bytes=530283856 weight_movement_bytes=34603008 total_nvlink_bytes=564886864 weight_to_token_ratio=0.065254 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127270015e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265084928 dispatch_meta_bytes=342264 combine_payload_bytes=265084928 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265427192 ingress_bytes=334290944 link_bytes=334290944 token_bytes=530512120 weight_movement_bytes=69206016 total_nvlink_bytes=599718136 weight_to_token_ratio=0.130451 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257909910e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74010 payload_rows_remote=64690 dup_rows=24294 dup_rows_remote=21154 dedup_ratio=0.247131 router_tokens_per_expert=[752, 743, 762, 755, 756, 778, 779, 779, 750, 786, 766, 773, 777, 765, 748, 795, 798, 762, 798, 782, 737, 766, 770, 733, 745, 788, 768, 732, 743, 782, 765, 749, 755, 765, 800, 709, 804, 744, 771, 752, 816, 773, 760, 768, 759, 745, 861, 736, 745, 783, 745, 808, 754, 782, 807, 806, 781, 777, 789, 778, 735, 745, 764, 753] payload_rows_per_dst=[9186, 9304, 9144, 9140, 9246, 9360, 9320, 9310] dup_rows_per_dst=[3022, 3016, 3148, 3004, 2954, 3076, 3140, 2934] combine_rows_per_src=[9186, 9304, 9144, 9140, 9246, 9360, 9320, 9310] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74020 payload_rows_remote=64738 dup_rows=24284 dup_rows_remote=21196 dedup_ratio=0.247030 router_tokens_per_expert=[797, 709, 811, 783, 761, 794, 789, 796, 785, 767, 820, 753, 776, 786, 733, 770, 738, 769, 759, 767, 781, 735, 777, 771, 766, 746, 786, 748, 822, 761, 748, 742, 736, 804, 796, 754, 747, 768, 781, 759, 775, 744, 739, 793, 781, 800, 787, 766, 697, 770, 721, 748, 752, 763, 785, 767, 804, 792, 703, 778, 772, 763, 780, 781] payload_rows_per_dst=[9372, 9284, 9168, 9210, 9276, 9282, 9120, 9308] dup_rows_per_dst=[3108, 3096, 3026, 3028, 3014, 3088, 2886, 3038] combine_rows_per_src=[9372, 9284, 9168, 9210, 9276, 9282, 9120, 9308] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=343384 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266116440 ingress_bytes=369582080 link_bytes=369582080 token_bytes=531889496 weight_movement_bytes=103809024 total_nvlink_bytes=635698520 weight_to_token_ratio=0.195170 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390707613e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265863168 dispatch_meta_bytes=343592 combine_payload_bytes=265863168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266206760 ingress_bytes=300466176 link_bytes=300466176 token_bytes=532069928 weight_movement_bytes=34603008 total_nvlink_bytes=566672936 weight_to_token_ratio=0.065035 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130630031e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266207232 dispatch_meta_bytes=344016 combine_payload_bytes=266207232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266551248 ingress_bytes=300810240 link_bytes=300810240 token_bytes=532758480 weight_movement_bytes=34603008 total_nvlink_bytes=567361488 weight_to_token_ratio=0.064951 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131924716e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265486336 dispatch_meta_bytes=344104 combine_payload_bytes=265486336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265830440 ingress_bytes=300089344 link_bytes=300089344 token_bytes=531316776 weight_movement_bytes=34603008 total_nvlink_bytes=565919784 weight_to_token_ratio=0.065127 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129212042e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74310 payload_rows_remote=64958 dup_rows=23994 dup_rows_remote=20976 dedup_ratio=0.244080 router_tokens_per_expert=[732, 752, 771, 752, 746, 767, 797, 747, 763, 725, 761, 732, 804, 749, 785, 776, 767, 769, 762, 714, 822, 754, 758, 746, 761, 773, 808, 799, 775, 798, 757, 803, 795, 770, 740, 761, 782, 720, 766, 772, 786, 724, 783, 784, 764, 789, 799, 795, 781, 776, 813, 786, 763, 779, 744, 747, 720, 797, 849, 765, 763, 743, 781, 690] payload_rows_per_dst=[9200, 9192, 9298, 9250, 9380, 9114, 9524, 9352] dup_rows_per_dst=[2928, 2998, 2886, 3024, 3102, 2880, 3158, 3018] combine_rows_per_src=[9200, 9192, 9298, 9250, 9380, 9114, 9524, 9352] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74334 payload_rows_remote=64874 dup_rows=23970 dup_rows_remote=20846 dedup_ratio=0.243835 router_tokens_per_expert=[765, 774, 829, 782, 722, 751, 763, 742, 778, 734, 756, 749, 785, 736, 795, 817, 756, 743, 744, 734, 774, 765, 773, 744, 794, 806, 752, 818, 738, 818, 764, 802, 766, 743, 741, 784, 809, 801, 805, 777, 771, 707, 729, 775, 791, 757, 788, 779, 764, 787, 761, 732, 765, 767, 788, 745, 749, 765, 738, 798, 773, 736, 802, 756] payload_rows_per_dst=[9338, 9236, 9198, 9460, 9340, 9234, 9296, 9232] dup_rows_per_dst=[2918, 3064, 2868, 3124, 3112, 2960, 2922, 3002] combine_rows_per_src=[9338, 9236, 9198, 9460, 9340, 9234, 9296, 9232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74496 payload_rows_remote=65276 dup_rows=23808 dup_rows_remote=20802 dedup_ratio=0.242188 router_tokens_per_expert=[732, 820, 806, 772, 775, 787, 787, 747, 771, 758, 769, 748, 786, 791, 748, 742, 759, 785, 736, 776, 774, 801, 710, 785, 799, 767, 738, 783, 754, 787, 801, 741, 758, 724, 771, 780, 764, 739, 782, 780, 762, 725, 776, 801, 730, 781, 749, 804, 776, 763, 785, 794, 721, 802, 725, 740, 766, 761, 772, 776, 759, 758, 787, 806] payload_rows_per_dst=[9506, 9220, 9296, 9342, 9176, 9362, 9254, 9340] dup_rows_per_dst=[2946, 3006, 2956, 2998, 3020, 2894, 2958, 3030] combine_rows_per_src=[9506, 9220, 9296, 9342, 9176, 9362, 9254, 9340] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=264970240 dispatch_meta_bytes=343376 combine_payload_bytes=264970240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265313616 ingress_bytes=299573248 link_bytes=299573248 token_bytes=530283856 weight_movement_bytes=34603008 total_nvlink_bytes=564886864 weight_to_token_ratio=0.065254 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127270015e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265166848 dispatch_meta_bytes=343736 combine_payload_bytes=265166848 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265510584 ingress_bytes=403578880 link_bytes=403578880 token_bytes=530677432 weight_movement_bytes=138412032 total_nvlink_bytes=669089464 weight_to_token_ratio=0.260821 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.518634835e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=65038 dup_rows=23962 dup_rows_remote=21016 dedup_ratio=0.243754 router_tokens_per_expert=[751, 763, 832, 785, 775, 759, 744, 789, 822, 764, 736, 777, 769, 742, 797, 788, 751, 754, 757, 760, 804, 746, 762, 813, 769, 749, 765, 788, 718, 789, 738, 764, 724, 797, 775, 749, 821, 750, 771, 784, 804, 765, 769, 731, 775, 761, 776, 771, 764, 726, 682, 798, 786, 819, 777, 773, 747, 727, 751, 829, 740, 745, 795, 750] payload_rows_per_dst=[9368, 9316, 9350, 9178, 9324, 9272, 9304, 9230] dup_rows_per_dst=[3028, 3074, 2944, 2982, 3018, 3032, 2946, 2938] combine_rows_per_src=[9368, 9316, 9350, 9178, 9324, 9272, 9304, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74334 payload_rows_remote=64874 dup_rows=23970 dup_rows_remote=20846 dedup_ratio=0.243835 router_tokens_per_expert=[765, 774, 829, 782, 722, 751, 763, 742, 778, 734, 756, 749, 785, 736, 795, 817, 756, 743, 744, 734, 774, 765, 773, 744, 794, 806, 752, 818, 738, 818, 764, 802, 766, 743, 741, 784, 809, 801, 805, 777, 771, 707, 729, 775, 791, 757, 788, 779, 764, 787, 761, 732, 765, 767, 788, 745, 749, 765, 738, 798, 773, 736, 802, 756] payload_rows_per_dst=[9338, 9236, 9198, 9460, 9340, 9234, 9296, 9232] dup_rows_per_dst=[2918, 3064, 2868, 3124, 3112, 2960, 2922, 3002] combine_rows_per_src=[9338, 9236, 9198, 9460, 9340, 9234, 9296, 9232] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=343736 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266411704 ingress_bytes=300670976 link_bytes=300670976 token_bytes=532479672 weight_movement_bytes=34603008 total_nvlink_bytes=567082680 weight_to_token_ratio=0.064985 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131400677e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74494 payload_rows_remote=65236 dup_rows=23810 dup_rows_remote=20834 dedup_ratio=0.242208 router_tokens_per_expert=[744, 799, 786, 787, 811, 743, 752, 769, 853, 755, 749, 812, 732, 775, 709, 732, 760, 756, 796, 763, 778, 765, 756, 764, 779, 760, 733, 762, 790, 735, 742, 732, 755, 802, 745, 732, 825, 786, 735, 818, 769, 733, 753, 741, 778, 804, 758, 785, 743, 804, 780, 772, 811, 778, 756, 785, 793, 768, 692, 746, 749, 781, 764, 832] payload_rows_per_dst=[9298, 9258, 9310, 9216, 9372, 9270, 9448, 9322] dup_rows_per_dst=[3084, 2976, 2966, 2850, 3024, 2972, 3010, 2928] combine_rows_per_src=[9298, 9258, 9310, 9216, 9372, 9270, 9448, 9322] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74352 payload_rows_remote=65032 dup_rows=23952 dup_rows_remote=20900 dedup_ratio=0.243652 router_tokens_per_expert=[781, 731, 793, 727, 713, 755, 770, 765, 781, 732, 778, 745, 749, 751, 766, 750, 805, 765, 794, 766, 713, 805, 758, 780, 796, 752, 760, 793, 755, 792, 753, 735, 800, 752, 766, 759, 783, 725, 821, 780, 780, 800, 754, 750, 745, 767, 737, 803, 750, 808, 737, 791, 778, 810, 724, 780, 764, 741, 829, 797, 825, 774, 747, 766] payload_rows_per_dst=[9182, 9200, 9410, 9358, 9320, 9300, 9294, 9288] dup_rows_per_dst=[2888, 2904, 2962, 2914, 3052, 2972, 3062, 3198] combine_rows_per_src=[9182, 9200, 9410, 9358, 9320, 9300, 9294, 9288] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265723904 dispatch_meta_bytes=342880 combine_payload_bytes=265723904 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266066784 ingress_bytes=404135936 link_bytes=404135936 token_bytes=531790688 weight_movement_bytes=138412032 total_nvlink_bytes=670202720 weight_to_token_ratio=0.260275 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520730992e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74492 payload_rows_remote=65022 dup_rows=23812 dup_rows_remote=20782 dedup_ratio=0.242228 router_tokens_per_expert=[779, 778, 787, 747, 789, 724, 795, 781, 774, 771, 772, 796, 772, 765, 766, 781, 779, 762, 768, 775, 806, 756, 778, 765, 760, 787, 772, 746, 742, 800, 715, 773, 804, 783, 756, 771, 779, 760, 740, 738, 780, 768, 769, 768, 799, 760, 761, 759, 756, 768, 747, 734, 750, 759, 763, 764, 782, 751, 753, 793, 790, 765, 754, 767] payload_rows_per_dst=[9470, 9490, 9502, 9254, 9188, 9380, 9142, 9066] dup_rows_per_dst=[3030, 2914, 3266, 3020, 2952, 2948, 2940, 2742] combine_rows_per_src=[9470, 9490, 9502, 9254, 9188, 9380, 9142, 9066] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267370496 dispatch_meta_bytes=344312 combine_payload_bytes=267370496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267714808 ingress_bytes=301973504 link_bytes=301973504 token_bytes=535085304 weight_movement_bytes=34603008 total_nvlink_bytes=569688312 weight_to_token_ratio=0.064668 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136301985e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=65116 dup_rows=24002 dup_rows_remote=21130 dedup_ratio=0.244161 router_tokens_per_expert=[789, 767, 770, 760, 747, 818, 777, 748, 788, 770, 775, 800, 767, 750, 799, 760, 762, 816, 769, 734, 781, 773, 727, 796, 766, 792, 794, 749, 799, 772, 745, 801, 737, 776, 776, 720, 732, 770, 748, 770, 795, 791, 749, 762, 780, 813, 794, 784, 816, 733, 798, 725, 746, 742, 769, 789, 744, 743, 730, 761, 731, 711, 758, 798] payload_rows_per_dst=[9306, 9308, 9304, 9332, 9186, 9364, 9356, 9146] dup_rows_per_dst=[3046, 3110, 3012, 3104, 2872, 3172, 2880, 2806] combine_rows_per_src=[9306, 9308, 9304, 9332, 9186, 9364, 9356, 9146] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74020 payload_rows_remote=64738 dup_rows=24284 dup_rows_remote=21196 dedup_ratio=0.247030 router_tokens_per_expert=[797, 709, 811, 783, 761, 794, 789, 796, 785, 767, 820, 753, 776, 786, 733, 770, 738, 769, 759, 767, 781, 735, 777, 771, 766, 746, 786, 748, 822, 761, 748, 742, 736, 804, 796, 754, 747, 768, 781, 759, 775, 744, 739, 793, 781, 800, 787, 766, 697, 770, 721, 748, 752, 763, 785, 767, 804, 792, 703, 778, 772, 763, 780, 781] payload_rows_per_dst=[9372, 9284, 9168, 9210, 9276, 9282, 9120, 9308] dup_rows_per_dst=[3108, 3096, 3026, 3028, 3014, 3088, 2886, 3038] combine_rows_per_src=[9372, 9284, 9168, 9210, 9276, 9282, 9120, 9308] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74274 payload_rows_remote=65022 dup_rows=24030 dup_rows_remote=21124 dedup_ratio=0.244446 router_tokens_per_expert=[737, 722, 793, 758, 787, 836, 745, 746, 743, 737, 798, 734, 749, 813, 791, 804, 765, 770, 754, 760, 783, 782, 765, 700, 754, 746, 807, 786, 811, 783, 770, 745, 790, 761, 731, 746, 764, 752, 739, 780, 802, 747, 815, 782, 789, 823, 764, 712, 791, 796, 783, 779, 760, 748, 799, 730, 746, 799, 796, 734, 761, 728, 785, 746] payload_rows_per_dst=[9262, 9314, 9252, 9358, 9140, 9356, 9274, 9318] dup_rows_per_dst=[2986, 3024, 2906, 3046, 2986, 3112, 3098, 2872] combine_rows_per_src=[9262, 9314, 9252, 9358, 9140, 9356, 9274, 9318] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=267206656 dispatch_meta_bytes=344280 combine_payload_bytes=267206656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267550936 ingress_bytes=301809664 link_bytes=301809664 token_bytes=534757592 weight_movement_bytes=34603008 total_nvlink_bytes=569360600 weight_to_token_ratio=0.064708 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135685468e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=343728 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266714800 ingress_bytes=300974080 link_bytes=300974080 token_bytes=533085872 weight_movement_bytes=34603008 total_nvlink_bytes=567688880 weight_to_token_ratio=0.064911 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132541233e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73954 payload_rows_remote=64634 dup_rows=24350 dup_rows_remote=21124 dedup_ratio=0.247701 router_tokens_per_expert=[679, 772, 766, 794, 777, 777, 745, 769, 792, 758, 769, 762, 769, 690, 786, 716, 736, 773, 774, 769, 765, 814, 782, 741, 774, 801, 798, 757, 820, 753, 761, 765, 803, 748, 791, 799, 795, 754, 704, 768, 806, 763, 758, 786, 752, 738, 768, 789, 824, 712, 797, 761, 770, 781, 749, 763, 790, 743, 783, 766, 780, 776, 763, 768] payload_rows_per_dst=[9320, 9312, 9254, 9244, 9470, 9230, 9234, 8890] dup_rows_per_dst=[3226, 3098, 3054, 2990, 3260, 3090, 2916, 2716] combine_rows_per_src=[9320, 9312, 9254, 9244, 9470, 9230, 9234, 8890] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=344216 combine_payload_bytes=266395648 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266739864 ingress_bytes=335601664 link_bytes=335601664 token_bytes=533135512 weight_movement_bytes=69206016 total_nvlink_bytes=602341528 weight_to_token_ratio=0.129809 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262842043e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265723904 dispatch_meta_bytes=342880 combine_payload_bytes=265723904 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266066784 ingress_bytes=404135936 link_bytes=404135936 token_bytes=531790688 weight_movement_bytes=138412032 total_nvlink_bytes=670202720 weight_to_token_ratio=0.260275 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520730992e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=65038 dup_rows=23962 dup_rows_remote=21016 dedup_ratio=0.243754 router_tokens_per_expert=[751, 763, 832, 785, 775, 759, 744, 789, 822, 764, 736, 777, 769, 742, 797, 788, 751, 754, 757, 760, 804, 746, 762, 813, 769, 749, 765, 788, 718, 789, 738, 764, 724, 797, 775, 749, 821, 750, 771, 784, 804, 765, 769, 731, 775, 761, 776, 771, 764, 726, 682, 798, 786, 819, 777, 773, 747, 727, 751, 829, 740, 745, 795, 750] payload_rows_per_dst=[9368, 9316, 9350, 9178, 9324, 9272, 9304, 9230] dup_rows_per_dst=[3028, 3074, 2944, 2982, 3018, 3032, 2946, 2938] combine_rows_per_src=[9368, 9316, 9350, 9178, 9324, 9272, 9304, 9230] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=65058 dup_rows=23946 dup_rows_remote=20948 dedup_ratio=0.243591 router_tokens_per_expert=[739, 773, 764, 763, 764, 730, 772, 757, 772, 733, 768, 749, 777, 760, 769, 776, 760, 754, 775, 774, 782, 803, 744, 802, 785, 732, 767, 769, 767, 786, 724, 763, 819, 764, 773, 734, 787, 756, 768, 725, 773, 798, 740, 730, 759, 781, 762, 806, 764, 809, 769, 808, 805, 787, 759, 775, 748, 754, 820, 788, 743, 764, 742, 789] payload_rows_per_dst=[9204, 9246, 9328, 9228, 9272, 9300, 9422, 9358] dup_rows_per_dst=[2920, 2962, 3060, 2958, 2980, 2998, 3130, 2938] combine_rows_per_src=[9204, 9246, 9328, 9228, 9272, 9300, 9422, 9358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266330112 dispatch_meta_bytes=343216 combine_payload_bytes=266330112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266673328 ingress_bytes=300933120 link_bytes=300933120 token_bytes=533003440 weight_movement_bytes=34603008 total_nvlink_bytes=567606448 weight_to_token_ratio=0.064921 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132387103e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266715136 dispatch_meta_bytes=344984 combine_payload_bytes=266715136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267060120 ingress_bytes=301318144 link_bytes=301318144 token_bytes=533775256 weight_movement_bytes=34603008 total_nvlink_bytes=568378264 weight_to_token_ratio=0.064827 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133835918e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265166848 dispatch_meta_bytes=343736 combine_payload_bytes=265166848 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265510584 ingress_bytes=403578880 link_bytes=403578880 token_bytes=530677432 weight_movement_bytes=138412032 total_nvlink_bytes=669089464 weight_to_token_ratio=0.260821 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.518634835e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266330112 dispatch_meta_bytes=344584 combine_payload_bytes=266330112 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266674696 ingress_bytes=335536128 link_bytes=335536128 token_bytes=533004808 weight_movement_bytes=69206016 total_nvlink_bytes=602210824 weight_to_token_ratio=0.129841 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262595437e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=264740864 dispatch_meta_bytes=343032 combine_payload_bytes=264740864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265083896 ingress_bytes=299343872 link_bytes=299343872 token_bytes=529824760 weight_movement_bytes=34603008 total_nvlink_bytes=564427768 weight_to_token_ratio=0.065310 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126406891e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=64968 dup_rows=23946 dup_rows_remote=20648 dedup_ratio=0.243591 router_tokens_per_expert=[766, 717, 771, 784, 779, 757, 750, 798, 821, 731, 791, 757, 751, 765, 718, 787, 729, 711, 769, 758, 743, 736, 797, 788, 787, 774, 776, 808, 778, 772, 714, 784, 835, 765, 734, 730, 767, 775, 772, 786, 740, 749, 760, 765, 756, 771, 783, 815, 751, 751, 758, 789, 729, 802, 731, 783, 796, 802, 788, 736, 811, 797, 758, 800] payload_rows_per_dst=[9306, 9350, 9088, 9338, 9298, 9304, 9284, 9390] dup_rows_per_dst=[2938, 2892, 2916, 3048, 3030, 2888, 2936, 3298] combine_rows_per_src=[9306, 9350, 9088, 9338, 9298, 9304, 9284, 9390] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74188 payload_rows_remote=64976 dup_rows=24116 dup_rows_remote=21006 dedup_ratio=0.245321 router_tokens_per_expert=[749, 730, 777, 754, 774, 758, 780, 790, 769, 760, 783, 778, 850, 765, 727, 761, 758, 740, 785, 850, 801, 809, 755, 766, 764, 742, 823, 806, 747, 742, 805, 732, 757, 745, 716, 784, 755, 757, 730, 735, 780, 780, 730, 769, 767, 722, 731, 818, 730, 799, 739, 808, 787, 735, 786, 712, 763, 821, 793, 768, 768, 777, 774, 786] payload_rows_per_dst=[9252, 9296, 9440, 9212, 9116, 9260, 9214, 9398] dup_rows_per_dst=[2972, 3090, 3088, 3110, 2842, 2934, 2978, 3102] combine_rows_per_src=[9252, 9296, 9440, 9212, 9116, 9260, 9214, 9398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74336 payload_rows_remote=64996 dup_rows=23968 dup_rows_remote=21058 dedup_ratio=0.243815 router_tokens_per_expert=[814, 741, 769, 791, 783, 742, 781, 722, 771, 755, 746, 779, 782, 739, 798, 755, 771, 779, 790, 778, 770, 794, 765, 723, 737, 728, 797, 796, 765, 749, 732, 747, 740, 756, 787, 763, 699, 747, 800, 788, 787, 804, 813, 781, 780, 794, 773, 740, 775, 763, 828, 742, 786, 752, 733, 778, 785, 726, 774, 756, 787, 764, 778, 784] payload_rows_per_dst=[9278, 9340, 9312, 9162, 9266, 9366, 9300, 9312] dup_rows_per_dst=[3008, 2910, 3028, 2940, 2894, 3178, 3014, 2996] combine_rows_per_src=[9278, 9340, 9312, 9162, 9266, 9366, 9300, 9312] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=344216 combine_payload_bytes=266395648 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266739864 ingress_bytes=335601664 link_bytes=335601664 token_bytes=533135512 weight_movement_bytes=69206016 total_nvlink_bytes=602341528 weight_to_token_ratio=0.129809 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262842043e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=344024 combine_payload_bytes=266477568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266821592 ingress_bytes=335683584 link_bytes=335683584 token_bytes=533299160 weight_movement_bytes=69206016 total_nvlink_bytes=602505176 weight_to_token_ratio=0.129770 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263150302e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74182 payload_rows_remote=64942 dup_rows=24122 dup_rows_remote=21042 dedup_ratio=0.245382 router_tokens_per_expert=[792, 786, 803, 760, 736, 744, 748, 759, 763, 766, 747, 754, 766, 787, 739, 818, 735, 753, 756, 790, 743, 737, 742, 775, 752, 769, 780, 801, 751, 711, 766, 740, 760, 781, 802, 797, 739, 777, 769, 769, 778, 766, 811, 762, 774, 745, 784, 742, 770, 767, 812, 754, 756, 761, 798, 742, 807, 805, 815, 762, 768, 754, 794, 762] payload_rows_per_dst=[9232, 9316, 9190, 9256, 9312, 9262, 9240, 9374] dup_rows_per_dst=[3024, 2964, 2872, 2884, 3076, 3062, 3080, 3160] combine_rows_per_src=[9232, 9316, 9190, 9256, 9312, 9262, 9240, 9374] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=342464 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266451392 ingress_bytes=369917952 link_bytes=369917952 token_bytes=532560320 weight_movement_bytes=103809024 total_nvlink_bytes=636369344 weight_to_token_ratio=0.194924 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391971473e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74496 payload_rows_remote=65276 dup_rows=23808 dup_rows_remote=20802 dedup_ratio=0.242188 router_tokens_per_expert=[732, 820, 806, 772, 775, 787, 787, 747, 771, 758, 769, 748, 786, 791, 748, 742, 759, 785, 736, 776, 774, 801, 710, 785, 799, 767, 738, 783, 754, 787, 801, 741, 758, 724, 771, 780, 764, 739, 782, 780, 762, 725, 776, 801, 730, 781, 749, 804, 776, 763, 785, 794, 721, 802, 725, 740, 766, 761, 772, 776, 759, 758, 787, 806] payload_rows_per_dst=[9506, 9220, 9296, 9342, 9176, 9362, 9254, 9340] dup_rows_per_dst=[2946, 3006, 2956, 2998, 3020, 2894, 2958, 3030] combine_rows_per_src=[9506, 9220, 9296, 9342, 9176, 9362, 9254, 9340] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74114 payload_rows_remote=64794 dup_rows=24190 dup_rows_remote=20966 dedup_ratio=0.246073 router_tokens_per_expert=[749, 768, 736, 744, 753, 775, 760, 803, 765, 773, 726, 715, 771, 802, 745, 740, 758, 746, 758, 756, 807, 760, 824, 752, 784, 772, 749, 780, 737, 735, 838, 765, 771, 781, 803, 810, 785, 752, 785, 785, 761, 760, 756, 757, 793, 742, 821, 813, 786, 809, 805, 751, 813, 706, 798, 722, 754, 746, 787, 736, 772, 765, 705, 776] payload_rows_per_dst=[9264, 9126, 9282, 9302, 9320, 9274, 9350, 9196] dup_rows_per_dst=[2912, 2948, 3040, 3018, 3224, 3132, 3030, 2886] combine_rows_per_src=[9264, 9126, 9282, 9302, 9320, 9274, 9350, 9196] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266141696 dispatch_meta_bytes=343928 combine_payload_bytes=266141696 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266485624 ingress_bytes=335347712 link_bytes=335347712 token_bytes=532627320 weight_movement_bytes=69206016 total_nvlink_bytes=601833336 weight_to_token_ratio=0.129933 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261886443e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74080 payload_rows_remote=64816 dup_rows=24224 dup_rows_remote=21210 dedup_ratio=0.246419 router_tokens_per_expert=[749, 759, 753, 767, 808, 750, 713, 757, 744, 760, 801, 801, 756, 724, 755, 732, 839, 776, 756, 753, 745, 784, 800, 802, 765, 793, 751, 819, 824, 755, 708, 730, 733, 775, 755, 789, 737, 798, 761, 793, 759, 805, 771, 780, 784, 771, 781, 801, 745, 779, 781, 781, 697, 798, 764, 739, 815, 762, 788, 757, 741, 755, 762, 766] payload_rows_per_dst=[9264, 9228, 9564, 9118, 9158, 9216, 9300, 9232] dup_rows_per_dst=[3014, 3040, 3112, 2884, 3124, 3122, 2868, 3060] combine_rows_per_src=[9264, 9228, 9564, 9118, 9158, 9216, 9300, 9232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=344216 combine_payload_bytes=266223616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266567832 ingress_bytes=300826624 link_bytes=300826624 token_bytes=532791448 weight_movement_bytes=34603008 total_nvlink_bytes=567394456 weight_to_token_ratio=0.064947 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131986368e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74188 payload_rows_remote=64976 dup_rows=24116 dup_rows_remote=21006 dedup_ratio=0.245321 router_tokens_per_expert=[749, 730, 777, 754, 774, 758, 780, 790, 769, 760, 783, 778, 850, 765, 727, 761, 758, 740, 785, 850, 801, 809, 755, 766, 764, 742, 823, 806, 747, 742, 805, 732, 757, 745, 716, 784, 755, 757, 730, 735, 780, 780, 730, 769, 767, 722, 731, 818, 730, 799, 739, 808, 787, 735, 786, 712, 763, 821, 793, 768, 768, 777, 774, 786] payload_rows_per_dst=[9252, 9296, 9440, 9212, 9116, 9260, 9214, 9398] dup_rows_per_dst=[2972, 3090, 3088, 3110, 2842, 2934, 2978, 3102] combine_rows_per_src=[9252, 9296, 9440, 9212, 9116, 9260, 9214, 9398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74352 payload_rows_remote=65032 dup_rows=23952 dup_rows_remote=20900 dedup_ratio=0.243652 router_tokens_per_expert=[781, 731, 793, 727, 713, 755, 770, 765, 781, 732, 778, 745, 749, 751, 766, 750, 805, 765, 794, 766, 713, 805, 758, 780, 796, 752, 760, 793, 755, 792, 753, 735, 800, 752, 766, 759, 783, 725, 821, 780, 780, 800, 754, 750, 745, 767, 737, 803, 750, 808, 737, 791, 778, 810, 724, 780, 764, 741, 829, 797, 825, 774, 747, 766] payload_rows_per_dst=[9182, 9200, 9410, 9358, 9320, 9300, 9294, 9288] dup_rows_per_dst=[2888, 2904, 2962, 2914, 3052, 2972, 3062, 3198] combine_rows_per_src=[9182, 9200, 9410, 9358, 9320, 9300, 9294, 9288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=65058 dup_rows=23946 dup_rows_remote=20948 dedup_ratio=0.243591 router_tokens_per_expert=[739, 773, 764, 763, 764, 730, 772, 757, 772, 733, 768, 749, 777, 760, 769, 776, 760, 754, 775, 774, 782, 803, 744, 802, 785, 732, 767, 769, 767, 786, 724, 763, 819, 764, 773, 734, 787, 756, 768, 725, 773, 798, 740, 730, 759, 781, 762, 806, 764, 809, 769, 808, 805, 787, 759, 775, 748, 754, 820, 788, 743, 764, 742, 789] payload_rows_per_dst=[9204, 9246, 9328, 9228, 9272, 9300, 9422, 9358] dup_rows_per_dst=[2920, 2962, 3060, 2958, 2980, 2998, 3130, 2938] combine_rows_per_src=[9204, 9246, 9328, 9228, 9272, 9300, 9422, 9358] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74134 payload_rows_remote=64856 dup_rows=24170 dup_rows_remote=21152 dedup_ratio=0.245870 router_tokens_per_expert=[794, 751, 755, 744, 715, 774, 741, 770, 758, 697, 788, 797, 823, 763, 744, 726, 794, 835, 725, 761, 799, 758, 756, 720, 780, 807, 774, 789, 786, 745, 764, 789, 792, 753, 813, 801, 769, 781, 758, 760, 732, 796, 748, 784, 792, 822, 815, 752, 779, 771, 751, 748, 782, 736, 761, 792, 725, 770, 746, 801, 756, 751, 752, 741] payload_rows_per_dst=[9268, 9178, 9278, 9386, 9306, 9368, 9198, 9152] dup_rows_per_dst=[2820, 3014, 3018, 3082, 3148, 3114, 3042, 2932] combine_rows_per_src=[9268, 9178, 9278, 9386, 9306, 9368, 9198, 9152] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267370496 dispatch_meta_bytes=344312 combine_payload_bytes=267370496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267714808 ingress_bytes=301973504 link_bytes=301973504 token_bytes=535085304 weight_movement_bytes=34603008 total_nvlink_bytes=569688312 weight_to_token_ratio=0.064668 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136301985e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265396224 dispatch_meta_bytes=343040 combine_payload_bytes=265396224 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265739264 ingress_bytes=334602240 link_bytes=334602240 token_bytes=531135488 weight_movement_bytes=69206016 total_nvlink_bytes=600341504 weight_to_token_ratio=0.130298 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259081291e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266002432 dispatch_meta_bytes=343936 combine_payload_bytes=266002432 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266346368 ingress_bytes=300605440 link_bytes=300605440 token_bytes=532348800 weight_movement_bytes=34603008 total_nvlink_bytes=566951808 weight_to_token_ratio=0.065001 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131154070e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74182 payload_rows_remote=64942 dup_rows=24122 dup_rows_remote=21042 dedup_ratio=0.245382 router_tokens_per_expert=[792, 786, 803, 760, 736, 744, 748, 759, 763, 766, 747, 754, 766, 787, 739, 818, 735, 753, 756, 790, 743, 737, 742, 775, 752, 769, 780, 801, 751, 711, 766, 740, 760, 781, 802, 797, 739, 777, 769, 769, 778, 766, 811, 762, 774, 745, 784, 742, 770, 767, 812, 754, 756, 761, 798, 742, 807, 805, 815, 762, 768, 754, 794, 762] payload_rows_per_dst=[9232, 9316, 9190, 9256, 9312, 9262, 9240, 9374] dup_rows_per_dst=[3024, 2964, 2872, 2884, 3076, 3062, 3080, 3160] combine_rows_per_src=[9232, 9316, 9190, 9256, 9312, 9262, 9240, 9374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=64908 dup_rows=24012 dup_rows_remote=21004 dedup_ratio=0.244263 router_tokens_per_expert=[716, 747, 742, 760, 752, 805, 743, 768, 738, 736, 785, 711, 803, 811, 758, 754, 787, 789, 805, 753, 764, 744, 757, 745, 741, 769, 757, 823, 782, 768, 814, 790, 781, 786, 750, 800, 765, 760, 778, 804, 771, 778, 810, 746, 777, 824, 762, 728, 763, 755, 752, 761, 759, 704, 789, 818, 798, 765, 733, 787, 755, 813, 732, 731] payload_rows_per_dst=[9130, 9320, 9294, 9406, 9310, 9384, 9246, 9202] dup_rows_per_dst=[2936, 2872, 2994, 3082, 3138, 3008, 2956, 3026] combine_rows_per_src=[9130, 9320, 9294, 9406, 9310, 9384, 9246, 9202] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265486336 dispatch_meta_bytes=344104 combine_payload_bytes=265486336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265830440 ingress_bytes=300089344 link_bytes=300089344 token_bytes=531316776 weight_movement_bytes=34603008 total_nvlink_bytes=565919784 weight_to_token_ratio=0.065127 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129212042e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74348 payload_rows_remote=64792 dup_rows=23956 dup_rows_remote=20750 dedup_ratio=0.243693 router_tokens_per_expert=[739, 749, 839, 772, 757, 780, 779, 776, 774, 752, 788, 733, 700, 775, 780, 697, 781, 730, 730, 771, 758, 776, 782, 774, 776, 774, 800, 753, 795, 776, 745, 779, 774, 784, 763, 799, 756, 763, 720, 783, 825, 779, 772, 730, 728, 789, 787, 770, 746, 761, 767, 725, 777, 782, 782, 754, 797, 770, 795, 795, 774, 790, 766, 759] payload_rows_per_dst=[9386, 9176, 9094, 9296, 9164, 9404, 9272, 9556] dup_rows_per_dst=[2996, 2822, 2888, 3100, 2774, 3032, 3138, 3206] combine_rows_per_src=[9386, 9176, 9094, 9296, 9164, 9404, 9272, 9556] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=64980 dup_rows=24028 dup_rows_remote=20794 dedup_ratio=0.244425 router_tokens_per_expert=[756, 793, 745, 731, 764, 788, 742, 762, 771, 772, 766, 777, 732, 722, 799, 812, 730, 795, 803, 828, 764, 735, 779, 770, 812, 798, 768, 746, 781, 775, 784, 801, 766, 778, 730, 800, 776, 800, 739, 792, 752, 753, 738, 836, 777, 746, 808, 756, 749, 752, 730, 766, 731, 761, 796, 780, 752, 749, 752, 772, 722, 752, 764, 776] payload_rows_per_dst=[9194, 9284, 9380, 9296, 9290, 9416, 9164, 9252] dup_rows_per_dst=[2968, 3018, 3028, 3234, 3072, 2916, 2966, 2826] combine_rows_per_src=[9194, 9284, 9380, 9296, 9290, 9416, 9164, 9252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74076 payload_rows_remote=64464 dup_rows=24228 dup_rows_remote=20952 dedup_ratio=0.246460 router_tokens_per_expert=[785, 776, 737, 788, 812, 794, 737, 791, 779, 745, 747, 774, 769, 777, 775, 766, 741, 776, 809, 851, 761, 768, 761, 811, 731, 788, 771, 781, 802, 741, 782, 783, 768, 763, 709, 770, 759, 724, 738, 774, 810, 763, 731, 794, 710, 772, 731, 759, 787, 809, 748, 754, 790, 749, 782, 735, 773, 752, 774, 755, 741, 767, 787, 765] payload_rows_per_dst=[9612, 9684, 9420, 9040, 9086, 8720, 9298, 9216] dup_rows_per_dst=[3276, 3332, 3186, 2820, 2924, 2668, 3010, 3012] combine_rows_per_src=[9612, 9684, 9420, 9040, 9086, 8720, 9298, 9216] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266141696 dispatch_meta_bytes=343928 combine_payload_bytes=266141696 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266485624 ingress_bytes=335347712 link_bytes=335347712 token_bytes=532627320 weight_movement_bytes=69206016 total_nvlink_bytes=601833336 weight_to_token_ratio=0.129933 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261886443e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=343728 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266714800 ingress_bytes=300974080 link_bytes=300974080 token_bytes=533085872 weight_movement_bytes=34603008 total_nvlink_bytes=567688880 weight_to_token_ratio=0.064911 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132541233e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=344024 combine_payload_bytes=266477568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266821592 ingress_bytes=335683584 link_bytes=335683584 token_bytes=533299160 weight_movement_bytes=69206016 total_nvlink_bytes=602505176 weight_to_token_ratio=0.129770 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263150302e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265650176 dispatch_meta_bytes=344032 combine_payload_bytes=265650176 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265994208 ingress_bytes=334856192 link_bytes=334856192 token_bytes=531644384 weight_movement_bytes=69206016 total_nvlink_bytes=600850400 weight_to_token_ratio=0.130174 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260036892e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74348 payload_rows_remote=64792 dup_rows=23956 dup_rows_remote=20750 dedup_ratio=0.243693 router_tokens_per_expert=[739, 749, 839, 772, 757, 780, 779, 776, 774, 752, 788, 733, 700, 775, 780, 697, 781, 730, 730, 771, 758, 776, 782, 774, 776, 774, 800, 753, 795, 776, 745, 779, 774, 784, 763, 799, 756, 763, 720, 783, 825, 779, 772, 730, 728, 789, 787, 770, 746, 761, 767, 725, 777, 782, 782, 754, 797, 770, 795, 795, 774, 790, 766, 759] payload_rows_per_dst=[9386, 9176, 9094, 9296, 9164, 9404, 9272, 9556] dup_rows_per_dst=[2996, 2822, 2888, 3100, 2774, 3032, 3138, 3206] combine_rows_per_src=[9386, 9176, 9094, 9296, 9164, 9404, 9272, 9556] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=65016 dup_rows=24040 dup_rows_remote=21012 dedup_ratio=0.244548 router_tokens_per_expert=[750, 780, 781, 803, 777, 761, 802, 810, 754, 724, 764, 790, 816, 778, 753, 759, 740, 786, 771, 764, 749, 768, 740, 702, 767, 762, 762, 767, 736, 723, 735, 773, 832, 733, 759, 795, 743, 763, 781, 763, 783, 788, 736, 762, 729, 802, 811, 762, 789, 787, 755, 805, 802, 750, 795, 783, 732, 776, 749, 779, 760, 755, 785, 761] payload_rows_per_dst=[9366, 9248, 9112, 9160, 9288, 9364, 9464, 9262] dup_rows_per_dst=[3162, 3028, 2928, 2890, 3050, 2982, 3068, 2932] combine_rows_per_src=[9366, 9248, 9112, 9160, 9288, 9364, 9464, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=264044544 dispatch_meta_bytes=341664 combine_payload_bytes=264044544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264386208 ingress_bytes=298647552 link_bytes=298647552 token_bytes=528430752 weight_movement_bytes=34603008 total_nvlink_bytes=563033760 weight_to_token_ratio=0.065483 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123786695e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265863168 dispatch_meta_bytes=343648 combine_payload_bytes=265863168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266206816 ingress_bytes=335069184 link_bytes=335069184 token_bytes=532069984 weight_movement_bytes=69206016 total_nvlink_bytes=601276000 weight_to_token_ratio=0.130069 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260838364e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266002432 dispatch_meta_bytes=343936 combine_payload_bytes=266002432 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266346368 ingress_bytes=300605440 link_bytes=300605440 token_bytes=532348800 weight_movement_bytes=34603008 total_nvlink_bytes=566951808 weight_to_token_ratio=0.065001 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131154070e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74214 payload_rows_remote=64970 dup_rows=24090 dup_rows_remote=20982 dedup_ratio=0.245056 router_tokens_per_expert=[755, 806, 790, 755, 760, 762, 756, 779, 745, 788, 788, 760, 779, 829, 738, 769, 755, 772, 777, 729, 792, 751, 793, 771, 764, 764, 795, 771, 756, 792, 742, 794, 734, 779, 713, 825, 754, 735, 732, 751, 674, 763, 741, 792, 760, 794, 792, 787, 776, 789, 731, 828, 780, 736, 744, 792, 827, 753, 746, 777, 745, 784, 742, 799] payload_rows_per_dst=[9306, 9412, 9280, 9336, 9106, 9234, 9244, 9296] dup_rows_per_dst=[3020, 2980, 3000, 3020, 2940, 2972, 3108, 3050] combine_rows_per_src=[9306, 9412, 9280, 9336, 9106, 9234, 9244, 9296] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265388032 dispatch_meta_bytes=342168 combine_payload_bytes=265388032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265730200 ingress_bytes=299991040 link_bytes=299991040 token_bytes=531118232 weight_movement_bytes=34603008 total_nvlink_bytes=565721240 weight_to_token_ratio=0.065151 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128842132e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266158080 dispatch_meta_bytes=343096 combine_payload_bytes=266158080 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266501176 ingress_bytes=335364096 link_bytes=335364096 token_bytes=532659256 weight_movement_bytes=69206016 total_nvlink_bytes=601865272 weight_to_token_ratio=0.129925 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261948094e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74336 payload_rows_remote=64996 dup_rows=23968 dup_rows_remote=21058 dedup_ratio=0.243815 router_tokens_per_expert=[814, 741, 769, 791, 783, 742, 781, 722, 771, 755, 746, 779, 782, 739, 798, 755, 771, 779, 790, 778, 770, 794, 765, 723, 737, 728, 797, 796, 765, 749, 732, 747, 740, 756, 787, 763, 699, 747, 800, 788, 787, 804, 813, 781, 780, 794, 773, 740, 775, 763, 828, 742, 786, 752, 733, 778, 785, 726, 774, 756, 787, 764, 778, 784] payload_rows_per_dst=[9278, 9340, 9312, 9162, 9266, 9366, 9300, 9312] dup_rows_per_dst=[3008, 2910, 3028, 2940, 2894, 3178, 3014, 2996] combine_rows_per_src=[9278, 9340, 9312, 9162, 9266, 9366, 9300, 9312] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=64976 dup_rows=23998 dup_rows_remote=20946 dedup_ratio=0.244120 router_tokens_per_expert=[795, 741, 787, 737, 753, 724, 768, 732, 789, 791, 813, 764, 781, 743, 793, 800, 759, 814, 728, 760, 802, 741, 751, 747, 763, 726, 785, 744, 749, 745, 749, 775, 742, 790, 745, 808, 777, 766, 786, 777, 775, 783, 768, 780, 765, 790, 754, 769, 798, 696, 795, 741, 745, 783, 822, 795, 714, 723, 751, 772, 788, 828, 807, 770] payload_rows_per_dst=[9184, 9402, 9230, 9208, 9330, 9354, 9282, 9316] dup_rows_per_dst=[2890, 3146, 2974, 2864, 3052, 3014, 3068, 2990] combine_rows_per_src=[9184, 9402, 9230, 9208, 9330, 9354, 9282, 9316] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73954 payload_rows_remote=64634 dup_rows=24350 dup_rows_remote=21124 dedup_ratio=0.247701 router_tokens_per_expert=[679, 772, 766, 794, 777, 777, 745, 769, 792, 758, 769, 762, 769, 690, 786, 716, 736, 773, 774, 769, 765, 814, 782, 741, 774, 801, 798, 757, 820, 753, 761, 765, 803, 748, 791, 799, 795, 754, 704, 768, 806, 763, 758, 786, 752, 738, 768, 789, 824, 712, 797, 761, 770, 781, 749, 763, 790, 743, 783, 766, 780, 776, 763, 768] payload_rows_per_dst=[9320, 9312, 9254, 9244, 9470, 9230, 9234, 8890] dup_rows_per_dst=[3226, 3098, 3054, 2990, 3260, 3090, 2916, 2716] combine_rows_per_src=[9320, 9312, 9254, 9244, 9470, 9230, 9234, 8890] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=344216 combine_payload_bytes=266223616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266567832 ingress_bytes=300826624 link_bytes=300826624 token_bytes=532791448 weight_movement_bytes=34603008 total_nvlink_bytes=567394456 weight_to_token_ratio=0.064947 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131986368e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=344112 combine_payload_bytes=266305536 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266649648 ingress_bytes=370114560 link_bytes=370114560 token_bytes=532955184 weight_movement_bytes=103809024 total_nvlink_bytes=636764208 weight_to_token_ratio=0.194780 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392711293e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74114 payload_rows_remote=64794 dup_rows=24190 dup_rows_remote=20966 dedup_ratio=0.246073 router_tokens_per_expert=[749, 768, 736, 744, 753, 775, 760, 803, 765, 773, 726, 715, 771, 802, 745, 740, 758, 746, 758, 756, 807, 760, 824, 752, 784, 772, 749, 780, 737, 735, 838, 765, 771, 781, 803, 810, 785, 752, 785, 785, 761, 760, 756, 757, 793, 742, 821, 813, 786, 809, 805, 751, 813, 706, 798, 722, 754, 746, 787, 736, 772, 765, 705, 776] payload_rows_per_dst=[9264, 9126, 9282, 9302, 9320, 9274, 9350, 9196] dup_rows_per_dst=[2912, 2948, 3040, 3018, 3224, 3132, 3030, 2886] combine_rows_per_src=[9264, 9126, 9282, 9302, 9320, 9274, 9350, 9196] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=64980 dup_rows=24028 dup_rows_remote=20794 dedup_ratio=0.244425 router_tokens_per_expert=[756, 793, 745, 731, 764, 788, 742, 762, 771, 772, 766, 777, 732, 722, 799, 812, 730, 795, 803, 828, 764, 735, 779, 770, 812, 798, 768, 746, 781, 775, 784, 801, 766, 778, 730, 800, 776, 800, 739, 792, 752, 753, 738, 836, 777, 746, 808, 756, 749, 752, 730, 766, 731, 761, 796, 780, 752, 749, 752, 772, 722, 752, 764, 776] payload_rows_per_dst=[9194, 9284, 9380, 9296, 9290, 9416, 9164, 9252] dup_rows_per_dst=[2968, 3018, 3028, 3234, 3072, 2916, 2966, 2826] combine_rows_per_src=[9194, 9284, 9380, 9296, 9290, 9416, 9164, 9252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=64908 dup_rows=24012 dup_rows_remote=21004 dedup_ratio=0.244263 router_tokens_per_expert=[716, 747, 742, 760, 752, 805, 743, 768, 738, 736, 785, 711, 803, 811, 758, 754, 787, 789, 805, 753, 764, 744, 757, 745, 741, 769, 757, 823, 782, 768, 814, 790, 781, 786, 750, 800, 765, 760, 778, 804, 771, 778, 810, 746, 777, 824, 762, 728, 763, 755, 752, 761, 759, 704, 789, 818, 798, 765, 733, 787, 755, 813, 732, 731] payload_rows_per_dst=[9130, 9320, 9294, 9406, 9310, 9384, 9246, 9202] dup_rows_per_dst=[2936, 2872, 2994, 3082, 3138, 3008, 2956, 3026] combine_rows_per_src=[9130, 9320, 9294, 9406, 9310, 9384, 9246, 9202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266117120 dispatch_meta_bytes=343808 combine_payload_bytes=266117120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266460928 ingress_bytes=300720128 link_bytes=300720128 token_bytes=532578048 weight_movement_bytes=34603008 total_nvlink_bytes=567181056 weight_to_token_ratio=0.064973 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131585632e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74090 payload_rows_remote=64708 dup_rows=24214 dup_rows_remote=21216 dedup_ratio=0.246318 router_tokens_per_expert=[738, 772, 728, 779, 803, 785, 740, 763, 768, 797, 769, 755, 786, 754, 812, 778, 763, 783, 765, 739, 816, 762, 770, 792, 774, 749, 744, 770, 746, 808, 743, 761, 752, 785, 763, 769, 776, 812, 731, 766, 761, 756, 799, 744, 724, 775, 764, 742, 777, 734, 714, 771, 753, 796, 764, 764, 739, 822, 794, 752, 795, 747, 824, 775] payload_rows_per_dst=[9246, 9364, 9382, 9138, 9312, 9138, 9118, 9392] dup_rows_per_dst=[2970, 3074, 2998, 3052, 2996, 2992, 3028, 3104] combine_rows_per_src=[9246, 9364, 9382, 9138, 9312, 9138, 9118, 9392] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266141696 dispatch_meta_bytes=343688 combine_payload_bytes=266141696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266485384 ingress_bytes=300744704 link_bytes=300744704 token_bytes=532627080 weight_movement_bytes=34603008 total_nvlink_bytes=567230088 weight_to_token_ratio=0.064967 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131678109e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12262175432 prefetch_bytes=761266176 reduce_bytes=519045120 weight_movement_bytes=1280311296 egress_bytes=6135042760 ingress_bytes=7407443968 link_bytes=7407443968 total_nvlink_bytes=13542486728 weight_to_token_ratio=0.104411 prefetch_share_of_ingress=0.102770 dedup_ratio=0.243764 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=15 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.211896318e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74318 payload_rows_remote=65082 dup_rows=23986 dup_rows_remote=21034 dedup_ratio=0.243998 router_tokens_per_expert=[715, 798, 784, 768, 778, 736, 729, 729, 771, 784, 768, 802, 774, 787, 823, 751, 769, 762, 771, 690, 752, 762, 779, 778, 787, 783, 742, 752, 738, 799, 731, 761, 821, 709, 755, 801, 765, 811, 729, 806, 765, 776, 754, 731, 794, 756, 774, 744, 723, 799, 733, 796, 752, 795, 808, 752, 747, 762, 802, 811, 820, 789, 756, 763] payload_rows_per_dst=[9186, 9368, 9218, 9210, 9368, 9236, 9354, 9378] dup_rows_per_dst=[2888, 3152, 2908, 2976, 3026, 2952, 2962, 3122] combine_rows_per_src=[9186, 9368, 9218, 9210, 9368, 9236, 9354, 9378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=264740864 dispatch_meta_bytes=343032 combine_payload_bytes=264740864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265083896 ingress_bytes=299343872 link_bytes=299343872 token_bytes=529824760 weight_movement_bytes=34603008 total_nvlink_bytes=564427768 weight_to_token_ratio=0.065310 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126406891e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74214 payload_rows_remote=64970 dup_rows=24090 dup_rows_remote=20982 dedup_ratio=0.245056 router_tokens_per_expert=[755, 806, 790, 755, 760, 762, 756, 779, 745, 788, 788, 760, 779, 829, 738, 769, 755, 772, 777, 729, 792, 751, 793, 771, 764, 764, 795, 771, 756, 792, 742, 794, 734, 779, 713, 825, 754, 735, 732, 751, 674, 763, 741, 792, 760, 794, 792, 787, 776, 789, 731, 828, 780, 736, 744, 792, 827, 753, 746, 777, 745, 784, 742, 799] payload_rows_per_dst=[9306, 9412, 9280, 9336, 9106, 9234, 9244, 9296] dup_rows_per_dst=[3020, 2980, 3000, 3020, 2940, 2972, 3108, 3050] combine_rows_per_src=[9306, 9412, 9280, 9336, 9106, 9234, 9244, 9296] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74136 payload_rows_remote=64822 dup_rows=24168 dup_rows_remote=21116 dedup_ratio=0.245850 router_tokens_per_expert=[756, 828, 801, 773, 743, 781, 749, 732, 750, 783, 754, 762, 775, 777, 747, 791, 782, 790, 782, 783, 771, 840, 781, 747, 750, 784, 731, 805, 798, 781, 746, 766, 724, 736, 750, 786, 745, 748, 831, 769, 720, 796, 708, 782, 768, 790, 776, 757, 713, 753, 752, 772, 757, 738, 787, 772, 770, 782, 810, 781, 752, 779, 746, 763] payload_rows_per_dst=[9156, 9004, 9376, 9576, 9232, 9218, 9260, 9314] dup_rows_per_dst=[3170, 2768, 3094, 3252, 2946, 2976, 2910, 3052] combine_rows_per_src=[9156, 9004, 9376, 9576, 9232, 9218, 9260, 9314] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74416 payload_rows_remote=64968 dup_rows=23888 dup_rows_remote=20904 dedup_ratio=0.243001 router_tokens_per_expert=[762, 767, 769, 763, 793, 767, 769, 743, 776, 761, 773, 808, 755, 828, 742, 745, 748, 776, 808, 751, 734, 774, 788, 767, 788, 796, 815, 745, 796, 821, 758, 697, 758, 721, 759, 770, 729, 727, 778, 790, 805, 827, 762, 783, 748, 754, 769, 778, 747, 755, 748, 752, 771, 756, 791, 786, 778, 741, 743, 771, 765, 764, 803, 740] payload_rows_per_dst=[9260, 9358, 9194, 9448, 9238, 9414, 9186, 9318] dup_rows_per_dst=[3006, 3018, 3098, 2984, 2826, 3038, 3026, 2892] combine_rows_per_src=[9260, 9358, 9194, 9448, 9238, 9414, 9186, 9318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265396224 dispatch_meta_bytes=343040 combine_payload_bytes=265396224 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265739264 ingress_bytes=334602240 link_bytes=334602240 token_bytes=531135488 weight_movement_bytes=69206016 total_nvlink_bytes=600341504 weight_to_token_ratio=0.130298 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259081291e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266158080 dispatch_meta_bytes=343096 combine_payload_bytes=266158080 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266501176 ingress_bytes=335364096 link_bytes=335364096 token_bytes=532659256 weight_movement_bytes=69206016 total_nvlink_bytes=601865272 weight_to_token_ratio=0.129925 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261948094e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74394 payload_rows_remote=64904 dup_rows=23910 dup_rows_remote=20906 dedup_ratio=0.243225 router_tokens_per_expert=[800, 767, 784, 752, 746, 743, 790, 807, 824, 782, 716, 775, 749, 739, 769, 692, 800, 785, 753, 797, 780, 792, 751, 800, 759, 809, 744, 777, 764, 746, 722, 774, 740, 744, 771, 731, 755, 797, 773, 766, 777, 790, 801, 748, 774, 770, 758, 755, 791, 778, 755, 767, 735, 748, 786, 753, 765, 811, 778, 767, 777, 760, 795, 748] payload_rows_per_dst=[9490, 9040, 9400, 9292, 9480, 9342, 9278, 9072] dup_rows_per_dst=[3004, 2942, 3112, 2898, 3114, 3078, 2948, 2814] combine_rows_per_src=[9490, 9040, 9400, 9292, 9480, 9342, 9278, 9072] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265863168 dispatch_meta_bytes=343648 combine_payload_bytes=265863168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266206816 ingress_bytes=335069184 link_bytes=335069184 token_bytes=532069984 weight_movement_bytes=69206016 total_nvlink_bytes=601276000 weight_to_token_ratio=0.130069 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260838364e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12253039416 prefetch_bytes=726663168 reduce_bytes=553648128 weight_movement_bytes=1280311296 egress_bytes=6130477880 ingress_bytes=7402872832 link_bytes=7402872832 total_nvlink_bytes=13533350712 weight_to_token_ratio=0.104489 prefetch_share_of_ingress=0.098160 dedup_ratio=0.244448 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=16 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.211148456e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265043968 dispatch_meta_bytes=343696 combine_payload_bytes=265043968 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265387664 ingress_bytes=368852992 link_bytes=368852992 token_bytes=530431632 weight_movement_bytes=103809024 total_nvlink_bytes=634240656 weight_to_token_ratio=0.195707 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.387964114e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12264226080 prefetch_bytes=692060160 reduce_bytes=519045120 weight_movement_bytes=1211105280 egress_bytes=6136069408 ingress_bytes=7339261952 link_bytes=7339261952 total_nvlink_bytes=13475331360 weight_to_token_ratio=0.098751 prefetch_share_of_ingress=0.094296 dedup_ratio=0.244002 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=15 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029348 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.200741385e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74136 payload_rows_remote=64822 dup_rows=24168 dup_rows_remote=21116 dedup_ratio=0.245850 router_tokens_per_expert=[756, 828, 801, 773, 743, 781, 749, 732, 750, 783, 754, 762, 775, 777, 747, 791, 782, 790, 782, 783, 771, 840, 781, 747, 750, 784, 731, 805, 798, 781, 746, 766, 724, 736, 750, 786, 745, 748, 831, 769, 720, 796, 708, 782, 768, 790, 776, 757, 713, 753, 752, 772, 757, 738, 787, 772, 770, 782, 810, 781, 752, 779, 746, 763] payload_rows_per_dst=[9156, 9004, 9376, 9576, 9232, 9218, 9260, 9314] dup_rows_per_dst=[3170, 2768, 3094, 3252, 2946, 2976, 2910, 3052] combine_rows_per_src=[9156, 9004, 9376, 9576, 9232, 9218, 9260, 9314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266575872 dispatch_meta_bytes=344464 combine_payload_bytes=266575872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266920336 ingress_bytes=301178880 link_bytes=301178880 token_bytes=533496208 weight_movement_bytes=34603008 total_nvlink_bytes=568099216 weight_to_token_ratio=0.064861 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133311879e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266117120 dispatch_meta_bytes=343808 combine_payload_bytes=266117120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266460928 ingress_bytes=300720128 link_bytes=300720128 token_bytes=532578048 weight_movement_bytes=34603008 total_nvlink_bytes=567181056 weight_to_token_ratio=0.064973 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131585632e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=65016 dup_rows=24040 dup_rows_remote=21012 dedup_ratio=0.244548 router_tokens_per_expert=[750, 780, 781, 803, 777, 761, 802, 810, 754, 724, 764, 790, 816, 778, 753, 759, 740, 786, 771, 764, 749, 768, 740, 702, 767, 762, 762, 767, 736, 723, 735, 773, 832, 733, 759, 795, 743, 763, 781, 763, 783, 788, 736, 762, 729, 802, 811, 762, 789, 787, 755, 805, 802, 750, 795, 783, 732, 776, 749, 779, 760, 755, 785, 761] payload_rows_per_dst=[9366, 9248, 9112, 9160, 9288, 9364, 9464, 9262] dup_rows_per_dst=[3162, 3028, 2928, 2890, 3050, 2982, 3068, 2932] combine_rows_per_src=[9366, 9248, 9112, 9160, 9288, 9364, 9464, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265846784 dispatch_meta_bytes=343240 combine_payload_bytes=265846784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266190024 ingress_bytes=300449792 link_bytes=300449792 token_bytes=532036808 weight_movement_bytes=34603008 total_nvlink_bytes=566639816 weight_to_token_ratio=0.065039 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130568379e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12258788984 prefetch_bytes=657457152 reduce_bytes=761266176 weight_movement_bytes=1418723328 egress_bytes=6133352056 ingress_bytes=7544160256 link_bytes=7544160256 total_nvlink_bytes=13677512312 weight_to_token_ratio=0.115731 prefetch_share_of_ingress=0.087148 dedup_ratio=0.244121 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=22 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029348 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.234263813e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265510912 dispatch_meta_bytes=343752 combine_payload_bytes=265510912 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265854664 ingress_bytes=369319936 link_bytes=369319936 token_bytes=531365576 weight_movement_bytes=103809024 total_nvlink_bytes=635174600 weight_to_token_ratio=0.195363 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389721186e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=343488 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266452416 ingress_bytes=300711936 link_bytes=300711936 token_bytes=532561344 weight_movement_bytes=34603008 total_nvlink_bytes=567164352 weight_to_token_ratio=0.064975 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131554806e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12272712288 prefetch_bytes=657457152 reduce_bytes=726663168 weight_movement_bytes=1384120320 egress_bytes=6140312160 ingress_bytes=7516520448 link_bytes=7516520448 total_nvlink_bytes=13656832608 weight_to_token_ratio=0.112780 prefetch_share_of_ingress=0.087468 dedup_ratio=0.243181 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=21 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.229741796e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74076 payload_rows_remote=64464 dup_rows=24228 dup_rows_remote=20952 dedup_ratio=0.246460 router_tokens_per_expert=[785, 776, 737, 788, 812, 794, 737, 791, 779, 745, 747, 774, 769, 777, 775, 766, 741, 776, 809, 851, 761, 768, 761, 811, 731, 788, 771, 781, 802, 741, 782, 783, 768, 763, 709, 770, 759, 724, 738, 774, 810, 763, 731, 794, 710, 772, 731, 759, 787, 809, 748, 754, 790, 749, 782, 735, 773, 752, 774, 755, 741, 767, 787, 765] payload_rows_per_dst=[9612, 9684, 9420, 9040, 9086, 8720, 9298, 9216] dup_rows_per_dst=[3276, 3332, 3186, 2820, 2924, 2668, 3010, 3012] combine_rows_per_src=[9612, 9684, 9420, 9040, 9086, 8720, 9298, 9216] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12253039416 prefetch_bytes=726663168 reduce_bytes=553648128 weight_movement_bytes=1280311296 egress_bytes=6130477880 ingress_bytes=7402872832 link_bytes=7402872832 total_nvlink_bytes=13533350712 weight_to_token_ratio=0.104489 prefetch_share_of_ingress=0.098160 dedup_ratio=0.244448 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=16 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.211148456e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=344112 combine_payload_bytes=266305536 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266649648 ingress_bytes=370114560 link_bytes=370114560 token_bytes=532955184 weight_movement_bytes=103809024 total_nvlink_bytes=636764208 weight_to_token_ratio=0.194780 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392711293e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=64976 dup_rows=23998 dup_rows_remote=20946 dedup_ratio=0.244120 router_tokens_per_expert=[795, 741, 787, 737, 753, 724, 768, 732, 789, 791, 813, 764, 781, 743, 793, 800, 759, 814, 728, 760, 802, 741, 751, 747, 763, 726, 785, 744, 749, 745, 749, 775, 742, 790, 745, 808, 777, 766, 786, 777, 775, 783, 768, 780, 765, 790, 754, 769, 798, 696, 795, 741, 745, 783, 822, 795, 714, 723, 751, 772, 788, 828, 807, 770] payload_rows_per_dst=[9184, 9402, 9230, 9208, 9330, 9354, 9282, 9316] dup_rows_per_dst=[2890, 3146, 2974, 2864, 3052, 3014, 3068, 2990] combine_rows_per_src=[9184, 9402, 9230, 9208, 9330, 9354, 9282, 9316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74416 payload_rows_remote=64968 dup_rows=23888 dup_rows_remote=20904 dedup_ratio=0.243001 router_tokens_per_expert=[762, 767, 769, 763, 793, 767, 769, 743, 776, 761, 773, 808, 755, 828, 742, 745, 748, 776, 808, 751, 734, 774, 788, 767, 788, 796, 815, 745, 796, 821, 758, 697, 758, 721, 759, 770, 729, 727, 778, 790, 805, 827, 762, 783, 748, 754, 769, 778, 747, 755, 748, 752, 771, 756, 791, 786, 778, 741, 743, 771, 765, 764, 803, 740] payload_rows_per_dst=[9260, 9358, 9194, 9448, 9238, 9414, 9186, 9318] dup_rows_per_dst=[3006, 3018, 3098, 2984, 2826, 3038, 3026, 2892] combine_rows_per_src=[9260, 9358, 9194, 9448, 9238, 9414, 9186, 9318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12233767928 prefetch_bytes=692060160 reduce_bytes=622854144 weight_movement_bytes=1314914304 egress_bytes=6120831992 ingress_bytes=7427850240 link_bytes=7427850240 total_nvlink_bytes=13548682232 weight_to_token_ratio=0.107482 prefetch_share_of_ingress=0.093171 dedup_ratio=0.244459 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=18 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028370 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.215234890e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74318 payload_rows_remote=65082 dup_rows=23986 dup_rows_remote=21034 dedup_ratio=0.243998 router_tokens_per_expert=[715, 798, 784, 768, 778, 736, 729, 729, 771, 784, 768, 802, 774, 787, 823, 751, 769, 762, 771, 690, 752, 762, 779, 778, 787, 783, 742, 752, 738, 799, 731, 761, 821, 709, 755, 801, 765, 811, 729, 806, 765, 776, 754, 731, 794, 756, 774, 744, 723, 799, 733, 796, 752, 795, 808, 752, 747, 762, 802, 811, 820, 789, 756, 763] payload_rows_per_dst=[9186, 9368, 9218, 9210, 9368, 9236, 9354, 9378] dup_rows_per_dst=[2888, 3152, 2908, 2976, 3026, 2952, 2962, 3122] combine_rows_per_src=[9186, 9368, 9218, 9210, 9368, 9236, 9354, 9378] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12238422760 prefetch_bytes=588251136 reduce_bytes=1038090240 weight_movement_bytes=1626341376 egress_bytes=6123160296 ingress_bytes=7741603840 link_bytes=7741603840 total_nvlink_bytes=13864764136 weight_to_token_ratio=0.132888 prefetch_share_of_ingress=0.075986 dedup_ratio=0.244481 prefetch_slots=34 unused_prefetch_slots=334 slot_utilization=0.092391 reduce_slots=30 remote_demand=17 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28448 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.266566609e-04 prefetch_bytes_per_gemm_flop=9.624094203e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266141696 dispatch_meta_bytes=343688 combine_payload_bytes=266141696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266485384 ingress_bytes=300744704 link_bytes=300744704 token_bytes=532627080 weight_movement_bytes=34603008 total_nvlink_bytes=567230088 weight_to_token_ratio=0.064967 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131678109e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=343488 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266452416 ingress_bytes=300711936 link_bytes=300711936 token_bytes=532561344 weight_movement_bytes=34603008 total_nvlink_bytes=567164352 weight_to_token_ratio=0.064975 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131554806e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266575872 dispatch_meta_bytes=344464 combine_payload_bytes=266575872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266920336 ingress_bytes=301178880 link_bytes=301178880 token_bytes=533496208 weight_movement_bytes=34603008 total_nvlink_bytes=568099216 weight_to_token_ratio=0.064861 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133311879e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12264226080 prefetch_bytes=692060160 reduce_bytes=519045120 weight_movement_bytes=1211105280 egress_bytes=6136069408 ingress_bytes=7339261952 link_bytes=7339261952 total_nvlink_bytes=13475331360 weight_to_token_ratio=0.098751 prefetch_share_of_ingress=0.094296 dedup_ratio=0.244002 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=15 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029348 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.200741385e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12258456552 prefetch_bytes=692060160 reduce_bytes=726663168 weight_movement_bytes=1418723328 egress_bytes=6133183464 ingress_bytes=7543996416 link_bytes=7543996416 total_nvlink_bytes=13677179880 weight_to_token_ratio=0.115734 prefetch_share_of_ingress=0.091737 dedup_ratio=0.244023 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=21 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029130 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.234237007e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=264044544 dispatch_meta_bytes=341664 combine_payload_bytes=264044544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264386208 ingress_bytes=298647552 link_bytes=298647552 token_bytes=528430752 weight_movement_bytes=34603008 total_nvlink_bytes=563033760 weight_to_token_ratio=0.065483 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123786695e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12262175432 prefetch_bytes=761266176 reduce_bytes=519045120 weight_movement_bytes=1280311296 egress_bytes=6135042760 ingress_bytes=7407443968 link_bytes=7407443968 total_nvlink_bytes=13542486728 weight_to_token_ratio=0.104411 prefetch_share_of_ingress=0.102770 dedup_ratio=0.243764 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=15 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.211896318e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74394 payload_rows_remote=64904 dup_rows=23910 dup_rows_remote=20906 dedup_ratio=0.243225 router_tokens_per_expert=[800, 767, 784, 752, 746, 743, 790, 807, 824, 782, 716, 775, 749, 739, 769, 692, 800, 785, 753, 797, 780, 792, 751, 800, 759, 809, 744, 777, 764, 746, 722, 774, 740, 744, 771, 731, 755, 797, 773, 766, 777, 790, 801, 748, 774, 770, 758, 755, 791, 778, 755, 767, 735, 748, 786, 753, 765, 811, 778, 767, 777, 760, 795, 748] payload_rows_per_dst=[9490, 9040, 9400, 9292, 9480, 9342, 9278, 9072] dup_rows_per_dst=[3004, 2942, 3112, 2898, 3114, 3078, 2948, 2814] combine_rows_per_src=[9490, 9040, 9400, 9292, 9480, 9342, 9278, 9072] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265846784 dispatch_meta_bytes=343240 combine_payload_bytes=265846784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266190024 ingress_bytes=300449792 link_bytes=300449792 token_bytes=532036808 weight_movement_bytes=34603008 total_nvlink_bytes=566639816 weight_to_token_ratio=0.065039 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130568379e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12233767928 prefetch_bytes=692060160 reduce_bytes=622854144 weight_movement_bytes=1314914304 egress_bytes=6120831992 ingress_bytes=7427850240 link_bytes=7427850240 total_nvlink_bytes=13548682232 weight_to_token_ratio=0.107482 prefetch_share_of_ingress=0.093171 dedup_ratio=0.244459 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=18 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028370 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.215234890e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none - [2026-08-26 15:10:26] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 792.8 | throughput per GPU (TFLOP/s/GPU): 160.9 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.156840E+00 | loss scale: 1.0 | grad norm: 1060.668 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74148 payload_rows_remote=64932 dup_rows=24156 dup_rows_remote=21136 dedup_ratio=0.245728 router_tokens_per_expert=[773, 701, 738, 745, 795, 748, 783, 820, 809, 729, 727, 772, 811, 782, 796, 766, 775, 774, 717, 724, 762, 772, 770, 734, 790, 777, 751, 832, 719, 792, 791, 766, 788, 759, 773, 736, 774, 771, 761, 756, 776, 805, 760, 756, 754, 801, 780, 766, 800, 809, 756, 758, 740, 771, 821, 743, 803, 776, 745, 787, 771, 741, 768, 706] payload_rows_per_dst=[9222, 9346, 9146, 9372, 9216, 9262, 9342, 9242] dup_rows_per_dst=[2984, 3038, 2910, 3064, 3020, 3134, 3054, 2952] combine_rows_per_src=[9222, 9346, 9146, 9372, 9216, 9262, 9342, 9242] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74470 payload_rows_remote=65238 dup_rows=23834 dup_rows_remote=20844 dedup_ratio=0.242452 router_tokens_per_expert=[784, 757, 753, 777, 768, 769, 790, 811, 773, 778, 764, 746, 761, 751, 769, 779, 734, 801, 803, 737, 797, 768, 734, 737, 805, 766, 744, 712, 800, 742, 781, 822, 771, 762, 766, 830, 755, 761, 734, 745, 744, 754, 749, 778, 840, 771, 786, 707, 780, 799, 747, 741, 756, 753, 740, 779, 754, 732, 758, 818, 787, 804, 785, 753] payload_rows_per_dst=[9374, 9306, 9232, 9392, 9320, 9342, 9182, 9322] dup_rows_per_dst=[3044, 2936, 2990, 2952, 2928, 2916, 3008, 3060] combine_rows_per_src=[9374, 9306, 9232, 9392, 9320, 9342, 9182, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12272712288 prefetch_bytes=657457152 reduce_bytes=726663168 weight_movement_bytes=1384120320 egress_bytes=6140312160 ingress_bytes=7516520448 link_bytes=7516520448 total_nvlink_bytes=13656832608 weight_to_token_ratio=0.112780 prefetch_share_of_ingress=0.087468 dedup_ratio=0.243181 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=21 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.229741796e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none + [2026-08-26 15:58:15] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 795.0 | throughput per GPU (TFLOP/s/GPU): 160.5 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.156840E+00 | loss scale: 1.0 | grad norm: 1060.668 | number of skipped iterations: 0 | number of nan iterations: 0 | WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74458 payload_rows_remote=65238 dup_rows=23846 dup_rows_remote=20814 dedup_ratio=0.242574 router_tokens_per_expert=[722, 745, 759, 742, 789, 724, 820, 787, 778, 800, 737, 778, 762, 794, 770, 748, 757, 799, 770, 753, 786, 754, 792, 759, 726, 762, 792, 803, 801, 768, 777, 779, 739, 747, 751, 774, 808, 733, 752, 753, 799, 809, 757, 778, 786, 780, 705, 712, 773, 772, 768, 749, 787, 785, 763, 776, 736, 769, 772, 745, 814, 751, 782, 794] payload_rows_per_dst=[9258, 9372, 9330, 9478, 9168, 9220, 9284, 9348] dup_rows_per_dst=[2918, 2962, 3010, 2938, 2946, 3032, 3062, 2978] combine_rows_per_src=[9258, 9372, 9330, 9478, 9168, 9220, 9284, 9348] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=65166 dup_rows=23998 dup_rows_remote=20930 dedup_ratio=0.244120 router_tokens_per_expert=[760, 740, 801, 766, 803, 777, 749, 724, 782, 793, 763, 736, 764, 776, 724, 766, 737, 790, 744, 775, 722, 800, 761, 746, 772, 812, 773, 698, 776, 819, 723, 781, 763, 766, 769, 787, 778, 725, 753, 753, 761, 800, 799, 820, 757, 746, 761, 738, 771, 752, 738, 814, 822, 800, 771, 771, 776, 753, 812, 752, 777, 802, 752, 760] payload_rows_per_dst=[9320, 9140, 9226, 9348, 9260, 9304, 9402, 9306] dup_rows_per_dst=[2920, 3068, 2924, 2960, 2928, 3060, 3076, 3062] combine_rows_per_src=[9320, 9140, 9226, 9348, 9260, 9304, 9402, 9306] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74488 payload_rows_remote=65304 dup_rows=23816 dup_rows_remote=20818 dedup_ratio=0.242269 router_tokens_per_expert=[787, 814, 771, 737, 763, 749, 755, 772, 706, 783, 793, 725, 780, 743, 786, 742, 779, 740, 811, 808, 731, 813, 770, 770, 743, 783, 794, 779, 741, 753, 762, 759, 808, 742, 749, 744, 807, 794, 729, 758, 808, 736, 797, 753, 739, 763, 755, 792, 766, 829, 815, 724, 746, 693, 789, 729, 768, 762, 761, 794, 813, 798, 744, 805] payload_rows_per_dst=[9242, 9222, 9458, 9300, 9330, 9324, 9184, 9428] dup_rows_per_dst=[3054, 2894, 2986, 2928, 2932, 2962, 2998, 3062] combine_rows_per_src=[9242, 9222, 9458, 9300, 9330, 9324, 9184, 9428] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74018 payload_rows_remote=64758 dup_rows=24286 dup_rows_remote=21208 dedup_ratio=0.247050 router_tokens_per_expert=[773, 774, 726, 797, 741, 784, 766, 796, 808, 802, 754, 769, 756, 784, 777, 768, 763, 751, 799, 793, 735, 762, 742, 786, 797, 761, 793, 781, 742, 803, 756, 736, 823, 784, 747, 764, 754, 709, 763, 734, 775, 775, 760, 775, 797, 804, 745, 721, 797, 749, 768, 764, 757, 758, 757, 762, 799, 711, 805, 770, 715, 766, 819, 750] payload_rows_per_dst=[9268, 9366, 9230, 9260, 9212, 9232, 9216, 9234] dup_rows_per_dst=[3046, 3070, 3032, 3078, 2944, 3072, 3008, 3036] combine_rows_per_src=[9268, 9366, 9230, 9260, 9212, 9232, 9216, 9234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74180 payload_rows_remote=64784 dup_rows=24124 dup_rows_remote=21072 dedup_ratio=0.245402 router_tokens_per_expert=[741, 782, 785, 733, 789, 704, 796, 749, 766, 759, 776, 745, 789, 764, 799, 800, 775, 801, 762, 704, 770, 765, 735, 734, 740, 778, 757, 762, 748, 802, 779, 803, 823, 749, 738, 734, 811, 769, 741, 803, 793, 747, 781, 807, 781, 723, 756, 750, 770, 791, 711, 810, 767, 806, 767, 746, 804, 801, 765, 744, 750, 761, 781, 780] payload_rows_per_dst=[9396, 9296, 9320, 9398, 9298, 9096, 9332, 9044] dup_rows_per_dst=[3052, 3100, 3152, 3030, 3038, 2890, 3004, 2858] combine_rows_per_src=[9396, 9296, 9320, 9398, 9298, 9096, 9332, 9044] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74470 payload_rows_remote=65238 dup_rows=23834 dup_rows_remote=20844 dedup_ratio=0.242452 router_tokens_per_expert=[784, 757, 753, 777, 768, 769, 790, 811, 773, 778, 764, 746, 761, 751, 769, 779, 734, 801, 803, 737, 797, 768, 734, 737, 805, 766, 744, 712, 800, 742, 781, 822, 771, 762, 766, 830, 755, 761, 734, 745, 744, 754, 749, 778, 840, 771, 786, 707, 780, 799, 747, 741, 756, 753, 740, 779, 754, 732, 758, 818, 787, 804, 785, 753] payload_rows_per_dst=[9374, 9306, 9232, 9392, 9320, 9342, 9182, 9322] dup_rows_per_dst=[3044, 2936, 2990, 2952, 2928, 2916, 3008, 3060] combine_rows_per_src=[9374, 9306, 9232, 9392, 9320, 9342, 9182, 9322] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74488 payload_rows_remote=65202 dup_rows=23816 dup_rows_remote=20730 dedup_ratio=0.242269 router_tokens_per_expert=[783, 740, 755, 838, 722, 731, 780, 754, 802, 720, 766, 737, 801, 769, 790, 709, 804, 768, 782, 797, 709, 791, 761, 767, 794, 771, 764, 792, 780, 751, 759, 832, 805, 765, 802, 727, 775, 758, 754, 741, 799, 791, 750, 782, 771, 746, 737, 768, 748, 763, 788, 739, 814, 760, 739, 741, 774, 776, 777, 755, 738, 788, 787, 775] payload_rows_per_dst=[9266, 9346, 9310, 9046, 9530, 9456, 9248, 9286] dup_rows_per_dst=[2940, 2842, 3048, 2760, 3234, 3002, 2904, 3086] combine_rows_per_src=[9266, 9346, 9310, 9046, 9530, 9456, 9248, 9286] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74018 payload_rows_remote=64758 dup_rows=24286 dup_rows_remote=21208 dedup_ratio=0.247050 router_tokens_per_expert=[773, 774, 726, 797, 741, 784, 766, 796, 808, 802, 754, 769, 756, 784, 777, 768, 763, 751, 799, 793, 735, 762, 742, 786, 797, 761, 793, 781, 742, 803, 756, 736, 823, 784, 747, 764, 754, 709, 763, 734, 775, 775, 760, 775, 797, 804, 745, 721, 797, 749, 768, 764, 757, 758, 757, 762, 799, 711, 805, 770, 715, 766, 819, 750] payload_rows_per_dst=[9268, 9366, 9230, 9260, 9212, 9232, 9216, 9234] dup_rows_per_dst=[3046, 3070, 3032, 3078, 2944, 3072, 3008, 3036] combine_rows_per_src=[9268, 9366, 9230, 9260, 9212, 9232, 9216, 9234] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=65166 dup_rows=23998 dup_rows_remote=20930 dedup_ratio=0.244120 router_tokens_per_expert=[760, 740, 801, 766, 803, 777, 749, 724, 782, 793, 763, 736, 764, 776, 724, 766, 737, 790, 744, 775, 722, 800, 761, 746, 772, 812, 773, 698, 776, 819, 723, 781, 763, 766, 769, 787, 778, 725, 753, 753, 761, 800, 799, 820, 757, 746, 761, 738, 771, 752, 738, 814, 822, 800, 771, 771, 776, 753, 812, 752, 777, 802, 752, 760] payload_rows_per_dst=[9320, 9140, 9226, 9348, 9260, 9304, 9402, 9306] dup_rows_per_dst=[2920, 3068, 2924, 2960, 2928, 3060, 3076, 3062] combine_rows_per_src=[9320, 9140, 9226, 9348, 9260, 9304, 9402, 9306] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267214848 dispatch_meta_bytes=344208 combine_payload_bytes=267214848 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267559056 ingress_bytes=336420864 link_bytes=336420864 token_bytes=534773904 weight_movement_bytes=69206016 total_nvlink_bytes=603979920 weight_to_token_ratio=0.129412 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265924627e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265961472 dispatch_meta_bytes=344272 combine_payload_bytes=265961472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266305744 ingress_bytes=300564480 link_bytes=300564480 token_bytes=532267216 weight_movement_bytes=34603008 total_nvlink_bytes=566870224 weight_to_token_ratio=0.065011 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130999941e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267067392 dispatch_meta_bytes=343728 combine_payload_bytes=267067392 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267411120 ingress_bytes=370876416 link_bytes=370876416 token_bytes=534478512 weight_movement_bytes=103809024 total_nvlink_bytes=638287536 weight_to_token_ratio=0.194225 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395578095e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265355264 dispatch_meta_bytes=343424 combine_payload_bytes=265355264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265698688 ingress_bytes=299958272 link_bytes=299958272 token_bytes=531053952 weight_movement_bytes=34603008 total_nvlink_bytes=565656960 weight_to_token_ratio=0.065159 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128718829e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267214848 dispatch_meta_bytes=344328 combine_payload_bytes=267214848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267559176 ingress_bytes=301817856 link_bytes=301817856 token_bytes=534774024 weight_movement_bytes=34603008 total_nvlink_bytes=569377032 weight_to_token_ratio=0.064706 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135716294e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267214848 dispatch_meta_bytes=344208 combine_payload_bytes=267214848 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267559056 ingress_bytes=336420864 link_bytes=336420864 token_bytes=534773904 weight_movement_bytes=69206016 total_nvlink_bytes=603979920 weight_to_token_ratio=0.129412 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265924627e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74148 payload_rows_remote=64932 dup_rows=24156 dup_rows_remote=21136 dedup_ratio=0.245728 router_tokens_per_expert=[773, 701, 738, 745, 795, 748, 783, 820, 809, 729, 727, 772, 811, 782, 796, 766, 775, 774, 717, 724, 762, 772, 770, 734, 790, 777, 751, 832, 719, 792, 791, 766, 788, 759, 773, 736, 774, 771, 761, 756, 776, 805, 760, 756, 754, 801, 780, 766, 800, 809, 756, 758, 740, 771, 821, 743, 803, 776, 745, 787, 771, 741, 768, 706] payload_rows_per_dst=[9222, 9346, 9146, 9372, 9216, 9262, 9342, 9242] dup_rows_per_dst=[2984, 3038, 2910, 3064, 3020, 3134, 3054, 2952] combine_rows_per_src=[9222, 9346, 9146, 9372, 9216, 9262, 9342, 9242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265248768 dispatch_meta_bytes=343864 combine_payload_bytes=265248768 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265592632 ingress_bytes=369057792 link_bytes=369057792 token_bytes=530841400 weight_movement_bytes=103809024 total_nvlink_bytes=634650424 weight_to_token_ratio=0.195556 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388734760e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74488 payload_rows_remote=65304 dup_rows=23816 dup_rows_remote=20818 dedup_ratio=0.242269 router_tokens_per_expert=[787, 814, 771, 737, 763, 749, 755, 772, 706, 783, 793, 725, 780, 743, 786, 742, 779, 740, 811, 808, 731, 813, 770, 770, 743, 783, 794, 779, 741, 753, 762, 759, 808, 742, 749, 744, 807, 794, 729, 758, 808, 736, 797, 753, 739, 763, 755, 792, 766, 829, 815, 724, 746, 693, 789, 729, 768, 762, 761, 794, 813, 798, 744, 805] payload_rows_per_dst=[9242, 9222, 9458, 9300, 9330, 9324, 9184, 9428] dup_rows_per_dst=[3054, 2894, 2986, 2928, 2932, 2962, 2998, 3062] combine_rows_per_src=[9242, 9222, 9458, 9300, 9330, 9324, 9184, 9428] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266919936 dispatch_meta_bytes=344384 combine_payload_bytes=266919936 prefetch_bytes=0 reduce_bytes=0 egress_bytes=267264320 ingress_bytes=266919936 link_bytes=267264320 token_bytes=534184256 weight_movement_bytes=0 total_nvlink_bytes=534184256 weight_to_token_ratio=0.000000 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.005694120e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265961472 dispatch_meta_bytes=344272 combine_payload_bytes=265961472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266305744 ingress_bytes=300564480 link_bytes=300564480 token_bytes=532267216 weight_movement_bytes=34603008 total_nvlink_bytes=566870224 weight_to_token_ratio=0.065011 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130999941e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74326 payload_rows_remote=65070 dup_rows=23978 dup_rows_remote=21018 dedup_ratio=0.243917 router_tokens_per_expert=[755, 780, 718, 767, 822, 795, 753, 692, 777, 722, 804, 801, 785, 741, 788, 773, 768, 801, 756, 770, 785, 733, 757, 731, 810, 765, 785, 783, 797, 777, 828, 802, 766, 757, 760, 756, 742, 754, 742, 753, 706, 758, 787, 781, 798, 738, 765, 775, 779, 772, 757, 797, 759, 730, 777, 754, 763, 747, 810, 762, 782, 766, 763, 775] payload_rows_per_dst=[9302, 9372, 9198, 9478, 9154, 9256, 9288, 9278] dup_rows_per_dst=[2862, 3010, 3004, 3216, 2906, 2960, 2962, 3058] combine_rows_per_src=[9302, 9372, 9198, 9478, 9154, 9256, 9288, 9278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73982 payload_rows_remote=64688 dup_rows=24322 dup_rows_remote=21280 dedup_ratio=0.247416 router_tokens_per_expert=[807, 716, 789, 817, 782, 752, 721, 751, 774, 788, 779, 769, 764, 756, 765, 759, 767, 761, 776, 738, 802, 725, 791, 748, 777, 775, 747, 754, 750, 756, 806, 757, 758, 819, 736, 783, 762, 748, 778, 808, 791, 830, 830, 782, 768, 763, 767, 714, 746, 735, 781, 750, 806, 767, 757, 800, 777, 791, 757, 735, 782, 740, 747, 725] payload_rows_per_dst=[9062, 9172, 9318, 9026, 9276, 9494, 9340, 9294] dup_rows_per_dst=[2948, 2856, 2898, 2854, 3140, 3276, 3308, 3042] combine_rows_per_src=[9062, 9172, 9318, 9026, 9276, 9494, 9340, 9294] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267485184 dispatch_meta_bytes=344488 combine_payload_bytes=267485184 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267829672 ingress_bytes=302088192 link_bytes=302088192 token_bytes=535314856 weight_movement_bytes=34603008 total_nvlink_bytes=569917864 weight_to_token_ratio=0.064640 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136733546e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265248768 dispatch_meta_bytes=343864 combine_payload_bytes=265248768 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265592632 ingress_bytes=369057792 link_bytes=369057792 token_bytes=530841400 weight_movement_bytes=103809024 total_nvlink_bytes=634650424 weight_to_token_ratio=0.195556 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388734760e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267067392 dispatch_meta_bytes=343728 combine_payload_bytes=267067392 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267411120 ingress_bytes=370876416 link_bytes=370876416 token_bytes=534478512 weight_movement_bytes=103809024 total_nvlink_bytes=638287536 weight_to_token_ratio=0.194225 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395578095e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74216 payload_rows_remote=64904 dup_rows=24088 dup_rows_remote=21146 dedup_ratio=0.245036 router_tokens_per_expert=[763, 824, 746, 810, 800, 767, 799, 772, 811, 802, 753, 736, 759, 772, 734, 743, 759, 764, 796, 807, 784, 739, 762, 778, 789, 762, 736, 746, 806, 760, 766, 781, 756, 743, 782, 773, 761, 758, 775, 779, 778, 754, 790, 792, 771, 712, 776, 755, 749, 724, 783, 763, 765, 748, 750, 784, 756, 745, 717, 775, 782, 770, 778, 782] payload_rows_per_dst=[9286, 9206, 9366, 9302, 9312, 9244, 9204, 9296] dup_rows_per_dst=[3276, 3014, 3012, 2990, 2942, 3012, 2928, 2914] combine_rows_per_src=[9286, 9206, 9366, 9302, 9312, 9244, 9204, 9296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74126 payload_rows_remote=65006 dup_rows=24178 dup_rows_remote=21130 dedup_ratio=0.245951 router_tokens_per_expert=[704, 773, 778, 781, 737, 780, 761, 763, 798, 752, 746, 784, 749, 785, 771, 753, 825, 792, 728, 750, 765, 793, 803, 747, 740, 754, 819, 752, 737, 767, 748, 772, 753, 759, 775, 741, 779, 756, 745, 798, 752, 770, 748, 741, 732, 784, 770, 756, 732, 768, 774, 770, 778, 780, 806, 810, 794, 772, 823, 779, 812, 733, 794, 761] payload_rows_per_dst=[9120, 9292, 9360, 9182, 9206, 9178, 9388, 9400] dup_rows_per_dst=[3048, 2984, 3032, 3070, 2932, 2928, 3048, 3136] combine_rows_per_src=[9120, 9292, 9360, 9182, 9206, 9178, 9388, 9400] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74336 payload_rows_remote=65182 dup_rows=23968 dup_rows_remote=20976 dedup_ratio=0.243815 router_tokens_per_expert=[769, 789, 719, 766, 771, 810, 808, 751, 793, 778, 698, 760, 766, 751, 770, 781, 741, 764, 746, 769, 770, 804, 709, 770, 772, 824, 766, 776, 788, 763, 709, 770, 763, 742, 752, 796, 773, 724, 788, 774, 765, 779, 776, 799, 766, 769, 759, 751, 777, 737, 761, 776, 773, 774, 805, 775, 830, 788, 783, 763, 737, 743, 748, 785] payload_rows_per_dst=[9364, 9270, 9154, 9346, 9316, 9250, 9352, 9284] dup_rows_per_dst=[3002, 2924, 2992, 2990, 2908, 3078, 3004, 3070] combine_rows_per_src=[9364, 9270, 9154, 9346, 9316, 9250, 9352, 9284] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74326 payload_rows_remote=65070 dup_rows=23978 dup_rows_remote=21018 dedup_ratio=0.243917 router_tokens_per_expert=[755, 780, 718, 767, 822, 795, 753, 692, 777, 722, 804, 801, 785, 741, 788, 773, 768, 801, 756, 770, 785, 733, 757, 731, 810, 765, 785, 783, 797, 777, 828, 802, 766, 757, 760, 756, 742, 754, 742, 753, 706, 758, 787, 781, 798, 738, 765, 775, 779, 772, 757, 797, 759, 730, 777, 754, 763, 747, 810, 762, 782, 766, 763, 775] payload_rows_per_dst=[9302, 9372, 9198, 9478, 9154, 9256, 9288, 9278] dup_rows_per_dst=[2862, 3010, 3004, 3216, 2906, 2960, 2962, 3058] combine_rows_per_src=[9302, 9372, 9198, 9478, 9154, 9256, 9288, 9278] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74180 payload_rows_remote=64784 dup_rows=24124 dup_rows_remote=21072 dedup_ratio=0.245402 router_tokens_per_expert=[741, 782, 785, 733, 789, 704, 796, 749, 766, 759, 776, 745, 789, 764, 799, 800, 775, 801, 762, 704, 770, 765, 735, 734, 740, 778, 757, 762, 748, 802, 779, 803, 823, 749, 738, 734, 811, 769, 741, 803, 793, 747, 781, 807, 781, 723, 756, 750, 770, 791, 711, 810, 767, 806, 767, 746, 804, 801, 765, 744, 750, 761, 781, 780] payload_rows_per_dst=[9396, 9296, 9320, 9398, 9298, 9096, 9332, 9044] dup_rows_per_dst=[3052, 3100, 3152, 3030, 3038, 2890, 3004, 2858] combine_rows_per_src=[9396, 9296, 9320, 9398, 9298, 9096, 9332, 9044] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74474 payload_rows_remote=65066 dup_rows=23830 dup_rows_remote=20782 dedup_ratio=0.242411 router_tokens_per_expert=[771, 743, 772, 765, 743, 828, 731, 797, 781, 781, 750, 738, 829, 785, 788, 776, 787, 772, 767, 744, 789, 726, 784, 822, 803, 800, 763, 731, 719, 742, 722, 793, 786, 769, 790, 828, 748, 772, 767, 781, 792, 768, 789, 753, 787, 749, 751, 763, 760, 740, 749, 767, 796, 782, 704, 764, 766, 761, 775, 724, 763, 747, 814, 705] payload_rows_per_dst=[9266, 9408, 9340, 9238, 9410, 9338, 9186, 9288] dup_rows_per_dst=[3034, 3048, 3042, 2908, 3072, 2966, 2938, 2822] combine_rows_per_src=[9266, 9408, 9340, 9238, 9410, 9338, 9186, 9288] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=65132 dup_rows=23966 dup_rows_remote=20930 dedup_ratio=0.243795 router_tokens_per_expert=[796, 758, 801, 762, 735, 754, 790, 759, 765, 784, 763, 759, 790, 771, 793, 758, 788, 820, 791, 764, 755, 763, 786, 820, 797, 752, 788, 766, 758, 736, 758, 766, 774, 812, 777, 750, 709, 781, 782, 808, 754, 705, 723, 751, 801, 767, 795, 745, 760, 802, 766, 732, 757, 773, 745, 759, 798, 767, 725, 751, 750, 752, 769, 766] payload_rows_per_dst=[9268, 9234, 9496, 9206, 9434, 9180, 9276, 9244] dup_rows_per_dst=[3042, 3132, 3078, 3036, 2952, 2902, 2912, 2912] combine_rows_per_src=[9268, 9234, 9496, 9206, 9434, 9180, 9276, 9244] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=65062 dup_rows=23926 dup_rows_remote=21072 dedup_ratio=0.243388 router_tokens_per_expert=[808, 745, 774, 785, 795, 755, 788, 762, 813, 722, 799, 775, 782, 749, 786, 765, 750, 752, 751, 755, 749, 741, 761, 748, 779, 810, 765, 777, 730, 812, 779, 779, 787, 800, 739, 740, 779, 756, 768, 803, 758, 807, 788, 756, 766, 777, 749, 720, 749, 774, 780, 719, 749, 796, 765, 753, 732, 798, 750, 796, 788, 736, 732, 801] payload_rows_per_dst=[9358, 9308, 9160, 9336, 9332, 9218, 9316, 9350] dup_rows_per_dst=[3066, 3074, 2854, 3126, 3012, 3024, 2854, 2916] combine_rows_per_src=[9358, 9308, 9160, 9336, 9332, 9218, 9316, 9350] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73982 payload_rows_remote=64688 dup_rows=24322 dup_rows_remote=21280 dedup_ratio=0.247416 router_tokens_per_expert=[807, 716, 789, 817, 782, 752, 721, 751, 774, 788, 779, 769, 764, 756, 765, 759, 767, 761, 776, 738, 802, 725, 791, 748, 777, 775, 747, 754, 750, 756, 806, 757, 758, 819, 736, 783, 762, 748, 778, 808, 791, 830, 830, 782, 768, 763, 767, 714, 746, 735, 781, 750, 806, 767, 757, 800, 777, 791, 757, 735, 782, 740, 747, 725] payload_rows_per_dst=[9062, 9172, 9318, 9026, 9276, 9494, 9340, 9294] dup_rows_per_dst=[2948, 2856, 2898, 2854, 3140, 3276, 3308, 3042] combine_rows_per_src=[9062, 9172, 9318, 9026, 9276, 9494, 9340, 9294] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265846784 dispatch_meta_bytes=344200 combine_payload_bytes=265846784 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266190984 ingress_bytes=335052800 link_bytes=335052800 token_bytes=532037768 weight_movement_bytes=69206016 total_nvlink_bytes=601243784 weight_to_token_ratio=0.130077 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260776712e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=65132 dup_rows=23966 dup_rows_remote=20930 dedup_ratio=0.243795 router_tokens_per_expert=[796, 758, 801, 762, 735, 754, 790, 759, 765, 784, 763, 759, 790, 771, 793, 758, 788, 820, 791, 764, 755, 763, 786, 820, 797, 752, 788, 766, 758, 736, 758, 766, 774, 812, 777, 750, 709, 781, 782, 808, 754, 705, 723, 751, 801, 767, 795, 745, 760, 802, 766, 732, 757, 773, 745, 759, 798, 767, 725, 751, 750, 752, 769, 766] payload_rows_per_dst=[9268, 9234, 9496, 9206, 9434, 9180, 9276, 9244] dup_rows_per_dst=[3042, 3132, 3078, 3036, 2952, 2902, 2912, 2912] combine_rows_per_src=[9268, 9234, 9496, 9206, 9434, 9180, 9276, 9244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266526720 dispatch_meta_bytes=344352 combine_payload_bytes=266526720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266871072 ingress_bytes=301129728 link_bytes=301129728 token_bytes=533397792 weight_movement_bytes=34603008 total_nvlink_bytes=568000800 weight_to_token_ratio=0.064873 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133126924e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=264962048 dispatch_meta_bytes=343872 combine_payload_bytes=264962048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265305920 ingress_bytes=299565056 link_bytes=299565056 token_bytes=530267968 weight_movement_bytes=34603008 total_nvlink_bytes=564870976 weight_to_token_ratio=0.065256 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127239189e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266264576 dispatch_meta_bytes=344544 combine_payload_bytes=266264576 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266609120 ingress_bytes=370073600 link_bytes=370073600 token_bytes=532873696 weight_movement_bytes=103809024 total_nvlink_bytes=636682720 weight_to_token_ratio=0.194810 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392557163e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266985472 dispatch_meta_bytes=344632 combine_payload_bytes=266985472 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267330104 ingress_bytes=301588480 link_bytes=301588480 token_bytes=534315576 weight_movement_bytes=34603008 total_nvlink_bytes=568918584 weight_to_token_ratio=0.064761 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134853170e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266526720 dispatch_meta_bytes=344352 combine_payload_bytes=266526720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266871072 ingress_bytes=301129728 link_bytes=301129728 token_bytes=533397792 weight_movement_bytes=34603008 total_nvlink_bytes=568000800 weight_to_token_ratio=0.064873 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133126924e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266510336 dispatch_meta_bytes=343392 combine_payload_bytes=266510336 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266853728 ingress_bytes=301113344 link_bytes=301113344 token_bytes=533364064 weight_movement_bytes=34603008 total_nvlink_bytes=567967072 weight_to_token_ratio=0.064877 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133065272e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266780672 dispatch_meta_bytes=344248 combine_payload_bytes=266780672 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267124920 ingress_bytes=335986688 link_bytes=335986688 token_bytes=533905592 weight_movement_bytes=69206016 total_nvlink_bytes=603111608 weight_to_token_ratio=0.129622 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264290858e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266493952 dispatch_meta_bytes=344536 combine_payload_bytes=266493952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266838488 ingress_bytes=301096960 link_bytes=301096960 token_bytes=533332440 weight_movement_bytes=34603008 total_nvlink_bytes=567935448 weight_to_token_ratio=0.064881 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133003620e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=264962048 dispatch_meta_bytes=343872 combine_payload_bytes=264962048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265305920 ingress_bytes=299565056 link_bytes=299565056 token_bytes=530267968 weight_movement_bytes=34603008 total_nvlink_bytes=564870976 weight_to_token_ratio=0.065256 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127239189e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265355264 dispatch_meta_bytes=343424 combine_payload_bytes=265355264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265698688 ingress_bytes=299958272 link_bytes=299958272 token_bytes=531053952 weight_movement_bytes=34603008 total_nvlink_bytes=565656960 weight_to_token_ratio=0.065159 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128718829e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74422 payload_rows_remote=65244 dup_rows=23882 dup_rows_remote=20996 dedup_ratio=0.242940 router_tokens_per_expert=[782, 744, 750, 831, 773, 754, 742, 753, 775, 739, 774, 788, 735, 797, 800, 773, 756, 799, 734, 791, 804, 745, 763, 780, 762, 746, 760, 710, 784, 795, 731, 755, 784, 783, 738, 744, 776, 715, 731, 761, 767, 802, 813, 787, 743, 765, 776, 730, 802, 774, 758, 745, 808, 770, 783, 778, 734, 786, 790, 775, 817, 726, 794, 772] payload_rows_per_dst=[9302, 9352, 9388, 9208, 9178, 9382, 9294, 9318] dup_rows_per_dst=[2956, 3010, 2956, 2878, 2886, 2984, 3142, 3070] combine_rows_per_src=[9302, 9352, 9388, 9208, 9178, 9382, 9294, 9318] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=65222 dup_rows=23920 dup_rows_remote=20980 dedup_ratio=0.243327 router_tokens_per_expert=[766, 750, 781, 760, 747, 768, 772, 739, 758, 782, 794, 744, 758, 750, 767, 799, 767, 754, 748, 775, 772, 747, 741, 747, 867, 772, 803, 751, 775, 759, 756, 803, 769, 753, 752, 824, 756, 783, 789, 757, 756, 759, 781, 756, 721, 833, 753, 716, 758, 774, 770, 759, 793, 808, 781, 726, 778, 781, 789, 751, 785, 749, 742, 778] payload_rows_per_dst=[9204, 9376, 9162, 9512, 9280, 9154, 9324, 9372] dup_rows_per_dst=[2962, 2928, 2940, 3060, 3086, 2996, 3014, 2934] combine_rows_per_src=[9204, 9376, 9162, 9512, 9280, 9154, 9324, 9372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74216 payload_rows_remote=64904 dup_rows=24088 dup_rows_remote=21146 dedup_ratio=0.245036 router_tokens_per_expert=[763, 824, 746, 810, 800, 767, 799, 772, 811, 802, 753, 736, 759, 772, 734, 743, 759, 764, 796, 807, 784, 739, 762, 778, 789, 762, 736, 746, 806, 760, 766, 781, 756, 743, 782, 773, 761, 758, 775, 779, 778, 754, 790, 792, 771, 712, 776, 755, 749, 724, 783, 763, 765, 748, 750, 784, 756, 745, 717, 775, 782, 770, 778, 782] payload_rows_per_dst=[9286, 9206, 9366, 9302, 9312, 9244, 9204, 9296] dup_rows_per_dst=[3276, 3014, 3012, 2990, 2942, 3012, 2928, 2914] combine_rows_per_src=[9286, 9206, 9366, 9302, 9312, 9244, 9204, 9296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=65062 dup_rows=23926 dup_rows_remote=21072 dedup_ratio=0.243388 router_tokens_per_expert=[808, 745, 774, 785, 795, 755, 788, 762, 813, 722, 799, 775, 782, 749, 786, 765, 750, 752, 751, 755, 749, 741, 761, 748, 779, 810, 765, 777, 730, 812, 779, 779, 787, 800, 739, 740, 779, 756, 768, 803, 758, 807, 788, 756, 766, 777, 749, 720, 749, 774, 780, 719, 749, 796, 765, 753, 732, 798, 750, 796, 788, 736, 732, 801] payload_rows_per_dst=[9358, 9308, 9160, 9336, 9332, 9218, 9316, 9350] dup_rows_per_dst=[3066, 3074, 2854, 3126, 3012, 3024, 2854, 2916] combine_rows_per_src=[9358, 9308, 9160, 9336, 9332, 9218, 9316, 9350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74246 payload_rows_remote=64920 dup_rows=24058 dup_rows_remote=21056 dedup_ratio=0.244731 router_tokens_per_expert=[791, 737, 770, 795, 761, 767, 771, 808, 789, 790, 766, 748, 783, 742, 769, 818, 797, 755, 769, 802, 766, 762, 806, 743, 735, 748, 724, 750, 750, 738, 748, 790, 744, 757, 731, 727, 756, 763, 743, 777, 751, 777, 768, 733, 816, 757, 754, 808, 824, 754, 769, 760, 760, 764, 747, 762, 778, 798, 783, 781, 769, 789, 777, 787] payload_rows_per_dst=[9312, 9360, 9292, 9104, 9106, 9326, 9340, 9406] dup_rows_per_dst=[3088, 3050, 3108, 2862, 2890, 3002, 2940, 3118] combine_rows_per_src=[9312, 9360, 9292, 9104, 9106, 9326, 9340, 9406] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74146 payload_rows_remote=64750 dup_rows=24158 dup_rows_remote=21154 dedup_ratio=0.245748 router_tokens_per_expert=[784, 761, 730, 767, 799, 693, 791, 745, 761, 778, 754, 759, 780, 742, 765, 756, 852, 733, 792, 801, 721, 732, 766, 791, 763, 786, 757, 746, 793, 787, 778, 790, 771, 775, 728, 786, 738, 759, 758, 768, 796, 758, 741, 755, 757, 714, 778, 759, 812, 754, 744, 762, 775, 784, 773, 788, 743, 815, 745, 781, 808, 837, 768, 769] payload_rows_per_dst=[9134, 9240, 9296, 9396, 9194, 9216, 9248, 9422] dup_rows_per_dst=[3006, 2950, 3080, 3004, 2972, 2900, 3136, 3110] combine_rows_per_src=[9134, 9240, 9296, 9396, 9194, 9216, 9248, 9422] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74014 payload_rows_remote=64658 dup_rows=24290 dup_rows_remote=21308 dedup_ratio=0.247091 router_tokens_per_expert=[795, 748, 770, 727, 787, 741, 825, 774, 782, 805, 718, 764, 816, 787, 775, 756, 721, 781, 808, 767, 753, 755, 777, 727, 745, 740, 769, 758, 726, 772, 798, 749, 738, 754, 741, 796, 756, 785, 748, 760, 774, 829, 821, 759, 783, 747, 773, 796, 785, 719, 759, 796, 743, 765, 791, 811, 740, 786, 742, 776, 769, 792, 751, 751] payload_rows_per_dst=[9280, 9322, 9262, 9046, 9180, 9418, 9356, 9150] dup_rows_per_dst=[3054, 3084, 2916, 3068, 2976, 3146, 2982, 3064] combine_rows_per_src=[9280, 9322, 9262, 9046, 9180, 9418, 9356, 9150] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=64966 dup_rows=23950 dup_rows_remote=20988 dedup_ratio=0.243632 router_tokens_per_expert=[786, 752, 760, 747, 830, 749, 728, 767, 746, 821, 811, 803, 776, 757, 732, 729, 804, 714, 779, 748, 804, 772, 798, 784, 782, 761, 775, 751, 786, 788, 777, 789, 791, 754, 776, 745, 731, 793, 797, 737, 809, 751, 755, 827, 784, 745, 817, 718, 743, 755, 775, 741, 736, 738, 703, 826, 768, 734, 780, 730, 780, 747, 791, 769] payload_rows_per_dst=[9260, 9388, 9320, 9360, 9306, 9356, 9182, 9182] dup_rows_per_dst=[2978, 2962, 3086, 3058, 2942, 3056, 2852, 3016] combine_rows_per_src=[9260, 9388, 9320, 9360, 9306, 9356, 9182, 9182] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74112 payload_rows_remote=64904 dup_rows=24192 dup_rows_remote=21128 dedup_ratio=0.246094 router_tokens_per_expert=[758, 774, 769, 765, 764, 757, 788, 792, 742, 768, 771, 778, 778, 785, 782, 755, 786, 753, 798, 766, 736, 857, 691, 775, 767, 817, 779, 753, 782, 786, 722, 785, 744, 772, 782, 778, 729, 769, 732, 778, 804, 783, 784, 762, 722, 757, 774, 729, 769, 767, 743, 779, 760, 748, 807, 771, 744, 794, 773, 784, 754, 779, 766, 736] payload_rows_per_dst=[9304, 9112, 9224, 9288, 9172, 9148, 9656, 9208] dup_rows_per_dst=[3018, 3006, 2944, 3094, 2996, 2780, 3290, 3064] combine_rows_per_src=[9304, 9112, 9224, 9288, 9172, 9148, 9656, 9208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265846784 dispatch_meta_bytes=344200 combine_payload_bytes=265846784 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266190984 ingress_bytes=335052800 link_bytes=335052800 token_bytes=532037768 weight_movement_bytes=69206016 total_nvlink_bytes=601243784 weight_to_token_ratio=0.130077 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260776712e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74034 payload_rows_remote=64614 dup_rows=24270 dup_rows_remote=21244 dedup_ratio=0.246887 router_tokens_per_expert=[753, 804, 757, 748, 775, 856, 730, 798, 776, 738, 704, 796, 703, 774, 733, 810, 739, 788, 785, 752, 778, 754, 794, 775, 765, 784, 798, 808, 742, 774, 761, 753, 725, 764, 774, 763, 754, 776, 733, 753, 806, 804, 735, 741, 767, 796, 774, 797, 784, 793, 749, 792, 775, 785, 817, 807, 752, 704, 814, 763, 746, 707, 760, 737] payload_rows_per_dst=[9420, 9136, 9244, 9306, 9694, 9320, 8888, 9026] dup_rows_per_dst=[3026, 3028, 3086, 3060, 3446, 3120, 2660, 2844] combine_rows_per_src=[9420, 9136, 9244, 9306, 9694, 9320, 8888, 9026] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=65222 dup_rows=23920 dup_rows_remote=20980 dedup_ratio=0.243327 router_tokens_per_expert=[766, 750, 781, 760, 747, 768, 772, 739, 758, 782, 794, 744, 758, 750, 767, 799, 767, 754, 748, 775, 772, 747, 741, 747, 867, 772, 803, 751, 775, 759, 756, 803, 769, 753, 752, 824, 756, 783, 789, 757, 756, 759, 781, 756, 721, 833, 753, 716, 758, 774, 770, 759, 793, 808, 781, 726, 778, 781, 789, 751, 785, 749, 742, 778] payload_rows_per_dst=[9204, 9376, 9162, 9512, 9280, 9154, 9324, 9372] dup_rows_per_dst=[2962, 2928, 2940, 3060, 3086, 2996, 3014, 2934] combine_rows_per_src=[9204, 9376, 9162, 9512, 9280, 9154, 9324, 9372] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267149312 dispatch_meta_bytes=344808 combine_payload_bytes=267149312 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267494120 ingress_bytes=336355328 link_bytes=336355328 token_bytes=534643432 weight_movement_bytes=69206016 total_nvlink_bytes=603849448 weight_to_token_ratio=0.129443 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265678020e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267239424 dispatch_meta_bytes=344960 combine_payload_bytes=267239424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267584384 ingress_bytes=301842432 link_bytes=301842432 token_bytes=534823808 weight_movement_bytes=34603008 total_nvlink_bytes=569426816 weight_to_token_ratio=0.064700 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135808771e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74126 payload_rows_remote=65006 dup_rows=24178 dup_rows_remote=21130 dedup_ratio=0.245951 router_tokens_per_expert=[704, 773, 778, 781, 737, 780, 761, 763, 798, 752, 746, 784, 749, 785, 771, 753, 825, 792, 728, 750, 765, 793, 803, 747, 740, 754, 819, 752, 737, 767, 748, 772, 753, 759, 775, 741, 779, 756, 745, 798, 752, 770, 748, 741, 732, 784, 770, 756, 732, 768, 774, 770, 778, 780, 806, 810, 794, 772, 823, 779, 812, 733, 794, 761] payload_rows_per_dst=[9120, 9292, 9360, 9182, 9206, 9178, 9388, 9400] dup_rows_per_dst=[3048, 2984, 3032, 3070, 2932, 2928, 3048, 3136] combine_rows_per_src=[9120, 9292, 9360, 9182, 9206, 9178, 9388, 9400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266493952 dispatch_meta_bytes=344536 combine_payload_bytes=266493952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266838488 ingress_bytes=301096960 link_bytes=301096960 token_bytes=533332440 weight_movement_bytes=34603008 total_nvlink_bytes=567935448 weight_to_token_ratio=0.064881 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133003620e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=64966 dup_rows=23950 dup_rows_remote=20988 dedup_ratio=0.243632 router_tokens_per_expert=[786, 752, 760, 747, 830, 749, 728, 767, 746, 821, 811, 803, 776, 757, 732, 729, 804, 714, 779, 748, 804, 772, 798, 784, 782, 761, 775, 751, 786, 788, 777, 789, 791, 754, 776, 745, 731, 793, 797, 737, 809, 751, 755, 827, 784, 745, 817, 718, 743, 755, 775, 741, 736, 738, 703, 826, 768, 734, 780, 730, 780, 747, 791, 769] payload_rows_per_dst=[9260, 9388, 9320, 9360, 9306, 9356, 9182, 9182] dup_rows_per_dst=[2978, 2962, 3086, 3058, 2942, 3056, 2852, 3016] combine_rows_per_src=[9260, 9388, 9320, 9360, 9306, 9356, 9182, 9182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74146 payload_rows_remote=64750 dup_rows=24158 dup_rows_remote=21154 dedup_ratio=0.245748 router_tokens_per_expert=[784, 761, 730, 767, 799, 693, 791, 745, 761, 778, 754, 759, 780, 742, 765, 756, 852, 733, 792, 801, 721, 732, 766, 791, 763, 786, 757, 746, 793, 787, 778, 790, 771, 775, 728, 786, 738, 759, 758, 768, 796, 758, 741, 755, 757, 714, 778, 759, 812, 754, 744, 762, 775, 784, 773, 788, 743, 815, 745, 781, 808, 837, 768, 769] payload_rows_per_dst=[9134, 9240, 9296, 9396, 9194, 9216, 9248, 9422] dup_rows_per_dst=[3006, 2950, 3080, 3004, 2972, 2900, 3136, 3110] combine_rows_per_src=[9134, 9240, 9296, 9396, 9194, 9216, 9248, 9422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265912320 dispatch_meta_bytes=343904 combine_payload_bytes=265912320 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266256224 ingress_bytes=335118336 link_bytes=335118336 token_bytes=532168544 weight_movement_bytes=69206016 total_nvlink_bytes=601374560 weight_to_token_ratio=0.130045 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261023319e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265216000 dispatch_meta_bytes=343616 combine_payload_bytes=265216000 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=265559616 ingress_bytes=299819008 link_bytes=299819008 token_bytes=530775616 weight_movement_bytes=34603008 total_nvlink_bytes=565378624 weight_to_token_ratio=0.065193 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128194790e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266100736 dispatch_meta_bytes=343816 combine_payload_bytes=266100736 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266444552 ingress_bytes=335306752 link_bytes=335306752 token_bytes=532545288 weight_movement_bytes=69206016 total_nvlink_bytes=601751304 weight_to_token_ratio=0.129953 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261732313e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=264839168 dispatch_meta_bytes=343864 combine_payload_bytes=264839168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265183032 ingress_bytes=334045184 link_bytes=334045184 token_bytes=530022200 weight_movement_bytes=69206016 total_nvlink_bytes=599228216 weight_to_token_ratio=0.130572 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256985135e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265846784 dispatch_meta_bytes=344128 combine_payload_bytes=265846784 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266190912 ingress_bytes=335052800 link_bytes=335052800 token_bytes=532037696 weight_movement_bytes=69206016 total_nvlink_bytes=601243712 weight_to_token_ratio=0.130077 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260776712e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266264576 dispatch_meta_bytes=344544 combine_payload_bytes=266264576 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266609120 ingress_bytes=370073600 link_bytes=370073600 token_bytes=532873696 weight_movement_bytes=103809024 total_nvlink_bytes=636682720 weight_to_token_ratio=0.194810 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392557163e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74386 payload_rows_remote=65074 dup_rows=23918 dup_rows_remote=20782 dedup_ratio=0.243306 router_tokens_per_expert=[743, 768, 821, 747, 762, 744, 734, 737, 798, 770, 769, 785, 770, 760, 788, 797, 745, 799, 786, 760, 787, 757, 757, 833, 744, 787, 774, 735, 820, 805, 811, 819, 738, 779, 739, 744, 735, 771, 761, 763, 773, 736, 683, 767, 822, 773, 827, 716, 709, 788, 794, 742, 804, 788, 774, 736, 769, 722, 753, 743, 738, 761, 762, 830] payload_rows_per_dst=[9206, 9434, 9312, 9474, 9138, 9256, 9324, 9242] dup_rows_per_dst=[2906, 3040, 3136, 3116, 2922, 2938, 2946, 2914] combine_rows_per_src=[9206, 9434, 9312, 9474, 9138, 9256, 9324, 9242] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=65108 dup_rows=23946 dup_rows_remote=20952 dedup_ratio=0.243591 router_tokens_per_expert=[769, 804, 751, 778, 780, 818, 752, 780, 746, 715, 760, 765, 774, 780, 740, 800, 772, 774, 759, 764, 748, 747, 767, 777, 805, 719, 787, 770, 764, 768, 783, 806, 778, 751, 783, 781, 774, 754, 782, 719, 768, 753, 738, 791, 782, 760, 766, 797, 769, 759, 763, 747, 791, 743, 765, 776, 791, 775, 763, 818, 737, 735, 759, 762] payload_rows_per_dst=[9334, 9152, 9300, 9378, 9250, 9416, 9248, 9280] dup_rows_per_dst=[3130, 3008, 2916, 3026, 2994, 2894, 2978, 3000] combine_rows_per_src=[9334, 9152, 9300, 9378, 9250, 9416, 9248, 9280] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=264658944 dispatch_meta_bytes=343432 combine_payload_bytes=264658944 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265002376 ingress_bytes=333864960 link_bytes=333864960 token_bytes=529661320 weight_movement_bytes=69206016 total_nvlink_bytes=598867336 weight_to_token_ratio=0.130661 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256306966e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267149312 dispatch_meta_bytes=344808 combine_payload_bytes=267149312 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267494120 ingress_bytes=336355328 link_bytes=336355328 token_bytes=534643432 weight_movement_bytes=69206016 total_nvlink_bytes=603849448 weight_to_token_ratio=0.129443 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265678020e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266100736 dispatch_meta_bytes=343816 combine_payload_bytes=266100736 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266444552 ingress_bytes=335306752 link_bytes=335306752 token_bytes=532545288 weight_movement_bytes=69206016 total_nvlink_bytes=601751304 weight_to_token_ratio=0.129953 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261732313e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265216000 dispatch_meta_bytes=343616 combine_payload_bytes=265216000 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=265559616 ingress_bytes=299819008 link_bytes=299819008 token_bytes=530775616 weight_movement_bytes=34603008 total_nvlink_bytes=565378624 weight_to_token_ratio=0.065193 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128194790e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74422 payload_rows_remote=65244 dup_rows=23882 dup_rows_remote=20996 dedup_ratio=0.242940 router_tokens_per_expert=[782, 744, 750, 831, 773, 754, 742, 753, 775, 739, 774, 788, 735, 797, 800, 773, 756, 799, 734, 791, 804, 745, 763, 780, 762, 746, 760, 710, 784, 795, 731, 755, 784, 783, 738, 744, 776, 715, 731, 761, 767, 802, 813, 787, 743, 765, 776, 730, 802, 774, 758, 745, 808, 770, 783, 778, 734, 786, 790, 775, 817, 726, 794, 772] payload_rows_per_dst=[9302, 9352, 9388, 9208, 9178, 9382, 9294, 9318] dup_rows_per_dst=[2956, 3010, 2956, 2878, 2886, 2984, 3142, 3070] combine_rows_per_src=[9302, 9352, 9388, 9208, 9178, 9382, 9294, 9318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74014 payload_rows_remote=64658 dup_rows=24290 dup_rows_remote=21308 dedup_ratio=0.247091 router_tokens_per_expert=[795, 748, 770, 727, 787, 741, 825, 774, 782, 805, 718, 764, 816, 787, 775, 756, 721, 781, 808, 767, 753, 755, 777, 727, 745, 740, 769, 758, 726, 772, 798, 749, 738, 754, 741, 796, 756, 785, 748, 760, 774, 829, 821, 759, 783, 747, 773, 796, 785, 719, 759, 796, 743, 765, 791, 811, 740, 786, 742, 776, 769, 792, 751, 751] payload_rows_per_dst=[9280, 9322, 9262, 9046, 9180, 9418, 9356, 9150] dup_rows_per_dst=[3054, 3084, 2916, 3068, 2976, 3146, 2982, 3064] combine_rows_per_src=[9280, 9322, 9262, 9046, 9180, 9418, 9356, 9150] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=65032 dup_rows=24010 dup_rows_remote=20968 dedup_ratio=0.244242 router_tokens_per_expert=[776, 718, 729, 761, 769, 760, 758, 733, 735, 792, 750, 763, 794, 752, 744, 739, 778, 799, 771, 773, 747, 797, 776, 796, 783, 719, 876, 791, 749, 752, 752, 765, 777, 782, 740, 775, 835, 787, 734, 769, 749, 775, 782, 754, 738, 754, 792, 808, 780, 793, 742, 768, 747, 763, 810, 736, 744, 757, 762, 751, 761, 820, 776, 794] payload_rows_per_dst=[9102, 9212, 9486, 9414, 9330, 9262, 9196, 9292] dup_rows_per_dst=[2906, 2926, 2988, 2960, 3068, 3042, 3082, 3038] combine_rows_per_src=[9102, 9212, 9486, 9414, 9330, 9262, 9196, 9292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74252 payload_rows_remote=64874 dup_rows=24052 dup_rows_remote=20898 dedup_ratio=0.244670 router_tokens_per_expert=[744, 796, 761, 793, 751, 754, 807, 766, 708, 744, 750, 736, 810, 755, 763, 769, 772, 757, 808, 775, 726, 746, 777, 773, 784, 763, 826, 788, 736, 768, 769, 774, 755, 736, 772, 798, 798, 714, 726, 828, 785, 748, 803, 739, 788, 749, 720, 760, 768, 803, 789, 768, 775, 776, 746, 788, 773, 754, 780, 746, 802, 790, 759, 767] payload_rows_per_dst=[9324, 9186, 9274, 9026, 9284, 9392, 9388, 9378] dup_rows_per_dst=[3020, 2884, 2994, 2794, 2970, 3150, 3086, 3154] combine_rows_per_src=[9324, 9186, 9274, 9026, 9284, 9392, 9388, 9378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74460 payload_rows_remote=65026 dup_rows=23844 dup_rows_remote=20834 dedup_ratio=0.242554 router_tokens_per_expert=[750, 784, 774, 733, 787, 746, 820, 800, 745, 746, 769, 750, 713, 820, 775, 764, 774, 799, 766, 693, 781, 780, 761, 804, 749, 802, 725, 771, 796, 780, 808, 734, 810, 733, 777, 794, 769, 811, 780, 769, 789, 751, 742, 741, 730, 745, 778, 785, 770, 772, 763, 717, 773, 738, 798, 783, 762, 750, 756, 749, 783, 783, 763, 789] payload_rows_per_dst=[9434, 9356, 9360, 9150, 9402, 9292, 9174, 9292] dup_rows_per_dst=[3010, 3212, 3026, 2788, 2946, 2830, 3054, 2978] combine_rows_per_src=[9434, 9356, 9360, 9150, 9402, 9292, 9174, 9292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74386 payload_rows_remote=65074 dup_rows=23918 dup_rows_remote=20782 dedup_ratio=0.243306 router_tokens_per_expert=[743, 768, 821, 747, 762, 744, 734, 737, 798, 770, 769, 785, 770, 760, 788, 797, 745, 799, 786, 760, 787, 757, 757, 833, 744, 787, 774, 735, 820, 805, 811, 819, 738, 779, 739, 744, 735, 771, 761, 763, 773, 736, 683, 767, 822, 773, 827, 716, 709, 788, 794, 742, 804, 788, 774, 736, 769, 722, 753, 743, 738, 761, 762, 830] payload_rows_per_dst=[9206, 9434, 9312, 9474, 9138, 9256, 9324, 9242] dup_rows_per_dst=[2906, 3040, 3136, 3116, 2922, 2938, 2946, 2914] combine_rows_per_src=[9206, 9434, 9312, 9474, 9138, 9256, 9324, 9242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74412 payload_rows_remote=65038 dup_rows=23892 dup_rows_remote=20926 dedup_ratio=0.243042 router_tokens_per_expert=[771, 817, 766, 788, 800, 779, 763, 745, 794, 825, 741, 737, 720, 798, 760, 795, 794, 729, 785, 714, 781, 760, 726, 753, 769, 712, 799, 758, 790, 727, 813, 772, 754, 802, 818, 776, 696, 760, 758, 786, 746, 779, 802, 798, 730, 760, 772, 745, 738, 755, 751, 788, 809, 770, 758, 753, 782, 723, 764, 769, 795, 761, 774, 799] payload_rows_per_dst=[9318, 9374, 9138, 9360, 9346, 9244, 9312, 9320] dup_rows_per_dst=[3140, 2966, 2946, 2920, 2954, 3020, 2932, 3014] combine_rows_per_src=[9318, 9374, 9138, 9360, 9346, 9244, 9312, 9320] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267239424 dispatch_meta_bytes=344960 combine_payload_bytes=267239424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267584384 ingress_bytes=301842432 link_bytes=301842432 token_bytes=534823808 weight_movement_bytes=34603008 total_nvlink_bytes=569426816 weight_to_token_ratio=0.064700 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135808771e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=264839168 dispatch_meta_bytes=343864 combine_payload_bytes=264839168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265183032 ingress_bytes=334045184 link_bytes=334045184 token_bytes=530022200 weight_movement_bytes=69206016 total_nvlink_bytes=599228216 weight_to_token_ratio=0.130572 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256985135e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344000 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266715072 ingress_bytes=300974080 link_bytes=300974080 token_bytes=533086144 weight_movement_bytes=34603008 total_nvlink_bytes=567689152 weight_to_token_ratio=0.064911 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132541233e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74072 payload_rows_remote=64610 dup_rows=24232 dup_rows_remote=21172 dedup_ratio=0.246501 router_tokens_per_expert=[790, 734, 789, 776, 783, 742, 729, 749, 765, 769, 747, 751, 765, 774, 805, 768, 757, 800, 748, 782, 805, 784, 728, 738, 829, 798, 794, 793, 748, 816, 739, 744, 768, 781, 789, 792, 748, 774, 795, 761, 757, 759, 728, 759, 729, 817, 750, 817, 778, 775, 744, 765, 819, 680, 764, 787, 760, 768, 781, 730, 746, 725, 765, 802] payload_rows_per_dst=[9178, 9224, 9320, 9462, 9284, 9246, 9220, 9138] dup_rows_per_dst=[3006, 3064, 2964, 3060, 3132, 2986, 3004, 3016] combine_rows_per_src=[9178, 9224, 9320, 9462, 9284, 9246, 9220, 9138] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265723904 dispatch_meta_bytes=343088 combine_payload_bytes=265723904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266066992 ingress_bytes=300326912 link_bytes=300326912 token_bytes=531790896 weight_movement_bytes=34603008 total_nvlink_bytes=566393904 weight_to_token_ratio=0.065069 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130105992e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266346496 dispatch_meta_bytes=343440 combine_payload_bytes=266346496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266689936 ingress_bytes=300949504 link_bytes=300949504 token_bytes=533036432 weight_movement_bytes=34603008 total_nvlink_bytes=567639440 weight_to_token_ratio=0.064917 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132448755e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=343424 combine_payload_bytes=266543104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266886528 ingress_bytes=301146112 link_bytes=301146112 token_bytes=533429632 weight_movement_bytes=34603008 total_nvlink_bytes=568032640 weight_to_token_ratio=0.064869 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133188575e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=343856 combine_payload_bytes=266395648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266739504 ingress_bytes=300998656 link_bytes=300998656 token_bytes=533135152 weight_movement_bytes=34603008 total_nvlink_bytes=567738160 weight_to_token_ratio=0.064905 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132633710e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=264642560 dispatch_meta_bytes=343128 combine_payload_bytes=264642560 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=264985688 ingress_bytes=437657600 link_bytes=437657600 token_bytes=529628248 weight_movement_bytes=173015040 total_nvlink_bytes=702643288 weight_to_token_ratio=0.326673 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.646870314e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=65108 dup_rows=23946 dup_rows_remote=20952 dedup_ratio=0.243591 router_tokens_per_expert=[769, 804, 751, 778, 780, 818, 752, 780, 746, 715, 760, 765, 774, 780, 740, 800, 772, 774, 759, 764, 748, 747, 767, 777, 805, 719, 787, 770, 764, 768, 783, 806, 778, 751, 783, 781, 774, 754, 782, 719, 768, 753, 738, 791, 782, 760, 766, 797, 769, 759, 763, 747, 791, 743, 765, 776, 791, 775, 763, 818, 737, 735, 759, 762] payload_rows_per_dst=[9334, 9152, 9300, 9378, 9250, 9416, 9248, 9280] dup_rows_per_dst=[3130, 3008, 2916, 3026, 2994, 2894, 2978, 3000] combine_rows_per_src=[9334, 9152, 9300, 9378, 9250, 9416, 9248, 9280] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74348 payload_rows_remote=65024 dup_rows=23956 dup_rows_remote=20920 dedup_ratio=0.243693 router_tokens_per_expert=[771, 771, 767, 729, 770, 797, 757, 783, 807, 720, 778, 761, 755, 739, 736, 837, 773, 799, 734, 743, 751, 735, 784, 753, 756, 777, 770, 742, 778, 790, 759, 816, 749, 768, 759, 746, 808, 789, 764, 759, 715, 752, 770, 792, 849, 797, 756, 737, 736, 726, 783, 795, 746, 799, 828, 767, 798, 783, 770, 768, 754, 794, 709, 748] payload_rows_per_dst=[9268, 9272, 9216, 9308, 9308, 9384, 9324, 9268] dup_rows_per_dst=[3022, 2994, 2928, 3068, 2976, 2952, 3036, 2980] combine_rows_per_src=[9268, 9272, 9216, 9308, 9308, 9384, 9324, 9268] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74412 payload_rows_remote=65038 dup_rows=23892 dup_rows_remote=20926 dedup_ratio=0.243042 router_tokens_per_expert=[771, 817, 766, 788, 800, 779, 763, 745, 794, 825, 741, 737, 720, 798, 760, 795, 794, 729, 785, 714, 781, 760, 726, 753, 769, 712, 799, 758, 790, 727, 813, 772, 754, 802, 818, 776, 696, 760, 758, 786, 746, 779, 802, 798, 730, 760, 772, 745, 738, 755, 751, 788, 809, 770, 758, 753, 782, 723, 764, 769, 795, 761, 774, 799] payload_rows_per_dst=[9318, 9374, 9138, 9360, 9346, 9244, 9312, 9320] dup_rows_per_dst=[3140, 2966, 2946, 2920, 2954, 3020, 2932, 3014] combine_rows_per_src=[9318, 9374, 9138, 9360, 9346, 9244, 9312, 9320] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74252 payload_rows_remote=64874 dup_rows=24052 dup_rows_remote=20898 dedup_ratio=0.244670 router_tokens_per_expert=[744, 796, 761, 793, 751, 754, 807, 766, 708, 744, 750, 736, 810, 755, 763, 769, 772, 757, 808, 775, 726, 746, 777, 773, 784, 763, 826, 788, 736, 768, 769, 774, 755, 736, 772, 798, 798, 714, 726, 828, 785, 748, 803, 739, 788, 749, 720, 760, 768, 803, 789, 768, 775, 776, 746, 788, 773, 754, 780, 746, 802, 790, 759, 767] payload_rows_per_dst=[9324, 9186, 9274, 9026, 9284, 9392, 9388, 9378] dup_rows_per_dst=[3020, 2884, 2994, 2794, 2970, 3150, 3086, 3154] combine_rows_per_src=[9324, 9186, 9274, 9026, 9284, 9392, 9388, 9378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266682368 dispatch_meta_bytes=344240 combine_payload_bytes=266682368 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267026608 ingress_bytes=370491392 link_bytes=370491392 token_bytes=533708976 weight_movement_bytes=103809024 total_nvlink_bytes=637518000 weight_to_token_ratio=0.194505 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394129281e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74320 payload_rows_remote=65030 dup_rows=23984 dup_rows_remote=21092 dedup_ratio=0.243978 router_tokens_per_expert=[725, 763, 750, 768, 794, 759, 791, 735, 778, 733, 840, 777, 749, 740, 762, 768, 716, 796, 798, 758, 783, 750, 753, 760, 789, 758, 798, 781, 799, 763, 801, 757, 762, 740, 781, 778, 752, 771, 800, 781, 746, 746, 751, 737, 774, 733, 788, 816, 800, 781, 748, 812, 761, 772, 748, 761, 730, 791, 781, 753, 762, 744, 808, 752] payload_rows_per_dst=[9164, 9320, 9192, 9360, 9258, 9290, 9408, 9328] dup_rows_per_dst=[3006, 2974, 3036, 3132, 3072, 2892, 2958, 2914] combine_rows_per_src=[9164, 9320, 9192, 9360, 9258, 9290, 9408, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64882 dup_rows=24112 dup_rows_remote=21192 dedup_ratio=0.245280 router_tokens_per_expert=[805, 766, 733, 789, 741, 757, 778, 790, 761, 750, 749, 763, 813, 779, 769, 756, 756, 812, 755, 783, 696, 785, 778, 824, 748, 815, 785, 764, 819, 766, 768, 703, 753, 761, 773, 731, 770, 805, 749, 778, 763, 757, 753, 749, 794, 814, 762, 760, 780, 770, 736, 785, 788, 716, 787, 747, 790, 720, 749, 765, 758, 774, 751, 808] payload_rows_per_dst=[9158, 9286, 9256, 9144, 9228, 9482, 9328, 9310] dup_rows_per_dst=[2930, 2994, 2942, 2858, 3012, 3336, 3120, 2920] combine_rows_per_src=[9158, 9286, 9256, 9144, 9228, 9482, 9328, 9310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74228 payload_rows_remote=64838 dup_rows=24076 dup_rows_remote=20970 dedup_ratio=0.244914 router_tokens_per_expert=[746, 755, 787, 787, 811, 826, 754, 771, 690, 760, 785, 802, 772, 785, 774, 772, 805, 800, 735, 795, 793, 759, 755, 765, 764, 799, 773, 740, 792, 777, 727, 750, 813, 759, 782, 747, 785, 742, 748, 734, 766, 752, 778, 780, 715, 745, 786, 761, 773, 742, 754, 742, 799, 775, 821, 774, 743, 771, 759, 790, 772, 725, 775, 738] payload_rows_per_dst=[9390, 9322, 9344, 9254, 9506, 9196, 9056, 9160] dup_rows_per_dst=[3106, 3048, 3054, 2990, 3166, 2970, 2852, 2890] combine_rows_per_src=[9390, 9322, 9344, 9254, 9506, 9196, 9056, 9160] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344000 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266715072 ingress_bytes=300974080 link_bytes=300974080 token_bytes=533086144 weight_movement_bytes=34603008 total_nvlink_bytes=567689152 weight_to_token_ratio=0.064911 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132541233e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74034 payload_rows_remote=64614 dup_rows=24270 dup_rows_remote=21244 dedup_ratio=0.246887 router_tokens_per_expert=[753, 804, 757, 748, 775, 856, 730, 798, 776, 738, 704, 796, 703, 774, 733, 810, 739, 788, 785, 752, 778, 754, 794, 775, 765, 784, 798, 808, 742, 774, 761, 753, 725, 764, 774, 763, 754, 776, 733, 753, 806, 804, 735, 741, 767, 796, 774, 797, 784, 793, 749, 792, 775, 785, 817, 807, 752, 704, 814, 763, 746, 707, 760, 737] payload_rows_per_dst=[9420, 9136, 9244, 9306, 9694, 9320, 8888, 9026] dup_rows_per_dst=[3026, 3028, 3086, 3060, 3446, 3120, 2660, 2844] combine_rows_per_src=[9420, 9136, 9244, 9306, 9694, 9320, 8888, 9026] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=264642560 dispatch_meta_bytes=343128 combine_payload_bytes=264642560 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=264985688 ingress_bytes=437657600 link_bytes=437657600 token_bytes=529628248 weight_movement_bytes=173015040 total_nvlink_bytes=702643288 weight_to_token_ratio=0.326673 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.646870314e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=343776 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266682080 ingress_bytes=300941312 link_bytes=300941312 token_bytes=533020384 weight_movement_bytes=34603008 total_nvlink_bytes=567623392 weight_to_token_ratio=0.064919 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132417929e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=343856 combine_payload_bytes=266395648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266739504 ingress_bytes=300998656 link_bytes=300998656 token_bytes=533135152 weight_movement_bytes=34603008 total_nvlink_bytes=567738160 weight_to_token_ratio=0.064905 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132633710e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265723904 dispatch_meta_bytes=343088 combine_payload_bytes=265723904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266066992 ingress_bytes=300326912 link_bytes=300326912 token_bytes=531790896 weight_movement_bytes=34603008 total_nvlink_bytes=566393904 weight_to_token_ratio=0.065069 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130105992e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=264658944 dispatch_meta_bytes=343432 combine_payload_bytes=264658944 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265002376 ingress_bytes=333864960 link_bytes=333864960 token_bytes=529661320 weight_movement_bytes=69206016 total_nvlink_bytes=598867336 weight_to_token_ratio=0.130661 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256306966e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74108 payload_rows_remote=64824 dup_rows=24196 dup_rows_remote=21104 dedup_ratio=0.246134 router_tokens_per_expert=[774, 737, 774, 753, 776, 794, 780, 802, 801, 773, 773, 783, 729, 807, 759, 763, 770, 781, 760, 744, 760, 758, 742, 730, 755, 716, 760, 750, 729, 756, 777, 807, 782, 678, 801, 763, 769, 766, 798, 753, 776, 741, 759, 788, 748, 739, 802, 808, 738, 776, 814, 772, 788, 775, 772, 790, 793, 754, 793, 767, 814, 768, 741, 753] payload_rows_per_dst=[9308, 9284, 9184, 9138, 9318, 9216, 9402, 9258] dup_rows_per_dst=[3072, 3092, 2906, 2962, 2902, 3106, 3048, 3108] combine_rows_per_src=[9308, 9284, 9184, 9138, 9318, 9216, 9402, 9258] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266682368 dispatch_meta_bytes=344240 combine_payload_bytes=266682368 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267026608 ingress_bytes=370491392 link_bytes=370491392 token_bytes=533708976 weight_movement_bytes=103809024 total_nvlink_bytes=637518000 weight_to_token_ratio=0.194505 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394129281e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=64850 dup_rows=24046 dup_rows_remote=20958 dedup_ratio=0.244609 router_tokens_per_expert=[805, 781, 754, 796, 768, 769, 754, 764, 767, 792, 768, 738, 752, 784, 773, 781, 787, 787, 828, 775, 788, 756, 737, 790, 817, 711, 794, 784, 785, 819, 766, 791, 734, 738, 750, 828, 709, 767, 710, 746, 797, 766, 773, 746, 718, 771, 735, 766, 791, 773, 765, 744, 779, 764, 742, 758, 802, 780, 772, 744, 740, 765, 753, 765] payload_rows_per_dst=[9278, 9368, 9408, 9424, 9092, 9188, 9258, 9242] dup_rows_per_dst=[3104, 2942, 3088, 3110, 2872, 2956, 2974, 3000] combine_rows_per_src=[9278, 9368, 9408, 9424, 9092, 9188, 9258, 9242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=343776 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266682080 ingress_bytes=300941312 link_bytes=300941312 token_bytes=533020384 weight_movement_bytes=34603008 total_nvlink_bytes=567623392 weight_to_token_ratio=0.064919 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132417929e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344488 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266707368 ingress_bytes=300965888 link_bytes=300965888 token_bytes=533070248 weight_movement_bytes=34603008 total_nvlink_bytes=567673256 weight_to_token_ratio=0.064913 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132510407e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=344296 combine_payload_bytes=265756672 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266100968 ingress_bytes=334962688 link_bytes=334962688 token_bytes=531857640 weight_movement_bytes=69206016 total_nvlink_bytes=601063656 weight_to_token_ratio=0.130121 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260437628e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74118 payload_rows_remote=64984 dup_rows=24186 dup_rows_remote=21310 dedup_ratio=0.246033 router_tokens_per_expert=[788, 776, 756, 805, 807, 770, 729, 739, 770, 745, 757, 804, 738, 797, 808, 752, 764, 760, 737, 785, 789, 756, 806, 744, 763, 704, 738, 804, 745, 747, 724, 780, 794, 759, 722, 788, 780, 738, 791, 820, 772, 757, 723, 758, 764, 737, 751, 724, 808, 770, 766, 802, 732, 767, 732, 783, 780, 838, 766, 766, 787, 824, 773, 793] payload_rows_per_dst=[9210, 9376, 9254, 9134, 9252, 9142, 9290, 9460] dup_rows_per_dst=[3130, 2966, 3028, 2876, 3132, 2830, 3030, 3194] combine_rows_per_src=[9210, 9376, 9254, 9134, 9252, 9142, 9290, 9460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265576448 dispatch_meta_bytes=343232 combine_payload_bytes=265576448 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265919680 ingress_bytes=334782464 link_bytes=334782464 token_bytes=531496128 weight_movement_bytes=69206016 total_nvlink_bytes=600702144 weight_to_token_ratio=0.130210 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259759460e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74402 payload_rows_remote=65090 dup_rows=23902 dup_rows_remote=20920 dedup_ratio=0.243144 router_tokens_per_expert=[786, 821, 739, 763, 787, 815, 724, 715, 741, 794, 712, 685, 768, 734, 795, 765, 741, 801, 741, 738, 758, 776, 784, 770, 779, 810, 797, 805, 810, 745, 782, 785, 753, 788, 740, 777, 761, 768, 785, 775, 768, 759, 773, 763, 760, 763, 787, 797, 769, 775, 764, 795, 780, 786, 752, 743, 791, 781, 770, 751, 746, 720, 806, 740] payload_rows_per_dst=[9308, 9174, 9246, 9430, 9312, 9330, 9336, 9266] dup_rows_per_dst=[2992, 2814, 2972, 3196, 2982, 3010, 2992, 2944] combine_rows_per_src=[9308, 9174, 9246, 9430, 9312, 9330, 9336, 9266] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74320 payload_rows_remote=65030 dup_rows=23984 dup_rows_remote=21092 dedup_ratio=0.243978 router_tokens_per_expert=[725, 763, 750, 768, 794, 759, 791, 735, 778, 733, 840, 777, 749, 740, 762, 768, 716, 796, 798, 758, 783, 750, 753, 760, 789, 758, 798, 781, 799, 763, 801, 757, 762, 740, 781, 778, 752, 771, 800, 781, 746, 746, 751, 737, 774, 733, 788, 816, 800, 781, 748, 812, 761, 772, 748, 761, 730, 791, 781, 753, 762, 744, 808, 752] payload_rows_per_dst=[9164, 9320, 9192, 9360, 9258, 9290, 9408, 9328] dup_rows_per_dst=[3006, 2974, 3036, 3132, 3072, 2892, 2958, 2914] combine_rows_per_src=[9164, 9320, 9192, 9360, 9258, 9290, 9408, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265519104 dispatch_meta_bytes=343712 combine_payload_bytes=265519104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265862816 ingress_bytes=300122112 link_bytes=300122112 token_bytes=531381920 weight_movement_bytes=34603008 total_nvlink_bytes=565984928 weight_to_token_ratio=0.065119 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129335346e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265625600 dispatch_meta_bytes=343232 combine_payload_bytes=265625600 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265968832 ingress_bytes=369434624 link_bytes=369434624 token_bytes=531594432 weight_movement_bytes=103809024 total_nvlink_bytes=635403456 weight_to_token_ratio=0.195279 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390152748e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266174464 dispatch_meta_bytes=345176 combine_payload_bytes=266174464 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266519640 ingress_bytes=335380480 link_bytes=335380480 token_bytes=532694104 weight_movement_bytes=69206016 total_nvlink_bytes=601900120 weight_to_token_ratio=0.129917 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262009746e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74402 payload_rows_remote=65090 dup_rows=23902 dup_rows_remote=20920 dedup_ratio=0.243144 router_tokens_per_expert=[786, 821, 739, 763, 787, 815, 724, 715, 741, 794, 712, 685, 768, 734, 795, 765, 741, 801, 741, 738, 758, 776, 784, 770, 779, 810, 797, 805, 810, 745, 782, 785, 753, 788, 740, 777, 761, 768, 785, 775, 768, 759, 773, 763, 760, 763, 787, 797, 769, 775, 764, 795, 780, 786, 752, 743, 791, 781, 770, 751, 746, 720, 806, 740] payload_rows_per_dst=[9308, 9174, 9246, 9430, 9312, 9330, 9336, 9266] dup_rows_per_dst=[2992, 2814, 2972, 3196, 2982, 3010, 2992, 2944] combine_rows_per_src=[9308, 9174, 9246, 9430, 9312, 9330, 9336, 9266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74234 payload_rows_remote=64950 dup_rows=24070 dup_rows_remote=21062 dedup_ratio=0.244853 router_tokens_per_expert=[812, 770, 814, 807, 766, 761, 745, 747, 765, 749, 725, 785, 736, 795, 708, 734, 757, 767, 722, 752, 758, 734, 772, 773, 777, 761, 737, 741, 773, 781, 756, 781, 807, 802, 746, 801, 749, 805, 766, 794, 757, 770, 735, 757, 754, 779, 798, 796, 753, 771, 829, 778, 751, 770, 809, 813, 737, 795, 772, 796, 766, 710, 769, 756] payload_rows_per_dst=[9338, 9170, 9190, 9338, 9336, 9284, 9386, 9192] dup_rows_per_dst=[3106, 2824, 2880, 2876, 3204, 3008, 3162, 3010] combine_rows_per_src=[9338, 9170, 9190, 9338, 9336, 9284, 9386, 9192] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74568 payload_rows_remote=65262 dup_rows=23736 dup_rows_remote=20790 dedup_ratio=0.241455 router_tokens_per_expert=[774, 809, 759, 751, 755, 773, 724, 729, 789, 708, 773, 773, 827, 742, 769, 791, 757, 791, 752, 778, 769, 771, 803, 776, 769, 734, 721, 786, 777, 791, 780, 790, 775, 770, 744, 741, 751, 779, 769, 771, 777, 792, 759, 745, 776, 811, 752, 768, 726, 787, 745, 739, 807, 761, 762, 799, 724, 773, 753, 778, 797, 768, 741, 821] payload_rows_per_dst=[9250, 9340, 9452, 9326, 9236, 9310, 9306, 9348] dup_rows_per_dst=[2898, 3004, 2942, 2970, 2964, 3050, 2946, 2962] combine_rows_per_src=[9250, 9340, 9452, 9326, 9236, 9310, 9306, 9348] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74118 payload_rows_remote=64984 dup_rows=24186 dup_rows_remote=21310 dedup_ratio=0.246033 router_tokens_per_expert=[788, 776, 756, 805, 807, 770, 729, 739, 770, 745, 757, 804, 738, 797, 808, 752, 764, 760, 737, 785, 789, 756, 806, 744, 763, 704, 738, 804, 745, 747, 724, 780, 794, 759, 722, 788, 780, 738, 791, 820, 772, 757, 723, 758, 764, 737, 751, 724, 808, 770, 766, 802, 732, 767, 732, 783, 780, 838, 766, 766, 787, 824, 773, 793] payload_rows_per_dst=[9210, 9376, 9254, 9134, 9252, 9142, 9290, 9460] dup_rows_per_dst=[3130, 2966, 3028, 2876, 3132, 2830, 3030, 3194] combine_rows_per_src=[9210, 9376, 9254, 9134, 9252, 9142, 9290, 9460] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74108 payload_rows_remote=64824 dup_rows=24196 dup_rows_remote=21104 dedup_ratio=0.246134 router_tokens_per_expert=[774, 737, 774, 753, 776, 794, 780, 802, 801, 773, 773, 783, 729, 807, 759, 763, 770, 781, 760, 744, 760, 758, 742, 730, 755, 716, 760, 750, 729, 756, 777, 807, 782, 678, 801, 763, 769, 766, 798, 753, 776, 741, 759, 788, 748, 739, 802, 808, 738, 776, 814, 772, 788, 775, 772, 790, 793, 754, 793, 767, 814, 768, 741, 753] payload_rows_per_dst=[9308, 9284, 9184, 9138, 9318, 9216, 9402, 9258] dup_rows_per_dst=[3072, 3092, 2906, 2962, 2902, 3106, 3048, 3108] combine_rows_per_src=[9308, 9284, 9184, 9138, 9318, 9216, 9402, 9258] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266608640 dispatch_meta_bytes=344040 combine_payload_bytes=266608640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266952680 ingress_bytes=301211648 link_bytes=301211648 token_bytes=533561320 weight_movement_bytes=34603008 total_nvlink_bytes=568164328 weight_to_token_ratio=0.064853 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133435182e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64882 dup_rows=24112 dup_rows_remote=21192 dedup_ratio=0.245280 router_tokens_per_expert=[805, 766, 733, 789, 741, 757, 778, 790, 761, 750, 749, 763, 813, 779, 769, 756, 756, 812, 755, 783, 696, 785, 778, 824, 748, 815, 785, 764, 819, 766, 768, 703, 753, 761, 773, 731, 770, 805, 749, 778, 763, 757, 753, 749, 794, 814, 762, 760, 780, 770, 736, 785, 788, 716, 787, 747, 790, 720, 749, 765, 758, 774, 751, 808] payload_rows_per_dst=[9158, 9286, 9256, 9144, 9228, 9482, 9328, 9310] dup_rows_per_dst=[2930, 2994, 2942, 2858, 3012, 3336, 3120, 2920] combine_rows_per_src=[9158, 9286, 9256, 9144, 9228, 9482, 9328, 9310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74214 payload_rows_remote=64748 dup_rows=24090 dup_rows_remote=21064 dedup_ratio=0.245056 router_tokens_per_expert=[768, 777, 773, 820, 741, 757, 752, 738, 780, 771, 769, 741, 739, 763, 752, 744, 757, 755, 783, 740, 762, 751, 740, 806, 699, 767, 760, 798, 811, 808, 799, 813, 748, 762, 761, 781, 724, 772, 768, 797, 796, 767, 744, 797, 779, 748, 749, 779, 754, 784, 759, 787, 778, 759, 782, 749, 801, 772, 759, 770, 779, 773, 748, 792] payload_rows_per_dst=[9162, 9182, 9234, 9314, 9268, 9262, 9326, 9466] dup_rows_per_dst=[2986, 2936, 2954, 3088, 2958, 3056, 3086, 3026] combine_rows_per_src=[9162, 9182, 9234, 9314, 9268, 9262, 9326, 9466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=65058 dup_rows=24112 dup_rows_remote=21064 dedup_ratio=0.245280 router_tokens_per_expert=[727, 759, 761, 738, 793, 736, 786, 784, 799, 774, 723, 797, 786, 774, 759, 795, 740, 824, 742, 745, 748, 718, 757, 762, 775, 782, 739, 752, 769, 747, 767, 802, 785, 802, 693, 771, 768, 736, 742, 752, 805, 730, 825, 803, 787, 776, 741, 783, 724, 754, 732, 793, 791, 783, 780, 788, 755, 795, 773, 795, 781, 821, 807, 721] payload_rows_per_dst=[9134, 9742, 9418, 9344, 9416, 8898, 9032, 9208] dup_rows_per_dst=[3048, 3364, 3048, 2908, 3210, 2758, 2730, 3046] combine_rows_per_src=[9134, 9742, 9418, 9344, 9416, 8898, 9032, 9208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344488 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266707368 ingress_bytes=300965888 link_bytes=300965888 token_bytes=533070248 weight_movement_bytes=34603008 total_nvlink_bytes=567673256 weight_to_token_ratio=0.064913 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132510407e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74460 payload_rows_remote=65026 dup_rows=23844 dup_rows_remote=20834 dedup_ratio=0.242554 router_tokens_per_expert=[750, 784, 774, 733, 787, 746, 820, 800, 745, 746, 769, 750, 713, 820, 775, 764, 774, 799, 766, 693, 781, 780, 761, 804, 749, 802, 725, 771, 796, 780, 808, 734, 810, 733, 777, 794, 769, 811, 780, 769, 789, 751, 742, 741, 730, 745, 778, 785, 770, 772, 763, 717, 773, 738, 798, 783, 762, 750, 756, 749, 783, 783, 763, 789] payload_rows_per_dst=[9434, 9356, 9360, 9150, 9402, 9292, 9174, 9292] dup_rows_per_dst=[3010, 3212, 3026, 2788, 2946, 2830, 3054, 2978] combine_rows_per_src=[9434, 9356, 9360, 9150, 9402, 9292, 9174, 9292] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=267313152 dispatch_meta_bytes=344208 combine_payload_bytes=267313152 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267657360 ingress_bytes=336519168 link_bytes=336519168 token_bytes=534970512 weight_movement_bytes=69206016 total_nvlink_bytes=604176528 weight_to_token_ratio=0.129364 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266294537e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266174464 dispatch_meta_bytes=345176 combine_payload_bytes=266174464 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266519640 ingress_bytes=335380480 link_bytes=335380480 token_bytes=532694104 weight_movement_bytes=69206016 total_nvlink_bytes=601900120 weight_to_token_ratio=0.129917 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262009746e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265519104 dispatch_meta_bytes=343712 combine_payload_bytes=265519104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265862816 ingress_bytes=300122112 link_bytes=300122112 token_bytes=531381920 weight_movement_bytes=34603008 total_nvlink_bytes=565984928 weight_to_token_ratio=0.065119 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129335346e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=344296 combine_payload_bytes=265756672 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266100968 ingress_bytes=334962688 link_bytes=334962688 token_bytes=531857640 weight_movement_bytes=69206016 total_nvlink_bytes=601063656 weight_to_token_ratio=0.130121 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260437628e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266608640 dispatch_meta_bytes=344040 combine_payload_bytes=266608640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266952680 ingress_bytes=301211648 link_bytes=301211648 token_bytes=533561320 weight_movement_bytes=34603008 total_nvlink_bytes=568164328 weight_to_token_ratio=0.064853 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133435182e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74644 payload_rows_remote=65252 dup_rows=23660 dup_rows_remote=20668 dedup_ratio=0.240682 router_tokens_per_expert=[768, 801, 789, 749, 749, 803, 706, 786, 783, 753, 817, 764, 788, 730, 794, 763, 778, 759, 809, 797, 761, 790, 748, 784, 818, 739, 758, 756, 757, 715, 770, 798, 731, 725, 793, 750, 751, 731, 822, 797, 825, 803, 744, 789, 780, 789, 724, 750, 761, 778, 744, 778, 750, 744, 765, 749, 793, 784, 785, 763, 736, 752, 739, 747] payload_rows_per_dst=[9372, 9392, 9382, 9270, 9264, 9428, 9274, 9262] dup_rows_per_dst=[2930, 2992, 3070, 2952, 2936, 2980, 2864, 2936] combine_rows_per_src=[9372, 9392, 9382, 9270, 9264, 9428, 9274, 9262] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74644 payload_rows_remote=65424 dup_rows=23660 dup_rows_remote=20672 dedup_ratio=0.240682 router_tokens_per_expert=[762, 754, 754, 770, 751, 787, 799, 759, 733, 765, 782, 754, 744, 785, 735, 735, 765, 743, 782, 750, 784, 746, 774, 760, 712, 806, 832, 751, 799, 763, 762, 776, 680, 757, 727, 758, 710, 720, 770, 758, 798, 770, 806, 828, 799, 797, 768, 747, 781, 742, 787, 769, 777, 807, 845, 768, 769, 795, 762, 767, 743, 770, 819, 784] payload_rows_per_dst=[9302, 9180, 9220, 9320, 9092, 9504, 9544, 9482] dup_rows_per_dst=[2970, 2886, 2988, 3082, 2668, 3122, 3008, 2936] combine_rows_per_src=[9302, 9180, 9220, 9320, 9092, 9504, 9544, 9482] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266346496 dispatch_meta_bytes=343440 combine_payload_bytes=266346496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266689936 ingress_bytes=300949504 link_bytes=300949504 token_bytes=533036432 weight_movement_bytes=34603008 total_nvlink_bytes=567639440 weight_to_token_ratio=0.064917 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132448755e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266035200 dispatch_meta_bytes=344048 combine_payload_bytes=266035200 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266379248 ingress_bytes=369844224 link_bytes=369844224 token_bytes=532414448 weight_movement_bytes=103809024 total_nvlink_bytes=636223472 weight_to_token_ratio=0.194978 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391694040e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=267313152 dispatch_meta_bytes=344208 combine_payload_bytes=267313152 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267657360 ingress_bytes=336519168 link_bytes=336519168 token_bytes=534970512 weight_movement_bytes=69206016 total_nvlink_bytes=604176528 weight_to_token_ratio=0.129364 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266294537e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74118 payload_rows_remote=64820 dup_rows=24186 dup_rows_remote=21132 dedup_ratio=0.246033 router_tokens_per_expert=[754, 752, 722, 778, 813, 853, 803, 768, 758, 757, 806, 796, 779, 739, 788, 778, 745, 772, 813, 733, 727, 732, 767, 783, 770, 769, 788, 750, 769, 783, 738, 809, 767, 757, 812, 766, 755, 731, 781, 761, 739, 742, 755, 732, 767, 838, 775, 751, 769, 750, 751, 748, 749, 786, 762, 791, 780, 761, 728, 759, 757, 768, 776, 796] payload_rows_per_dst=[9404, 9282, 9186, 9298, 9266, 9256, 9248, 9178] dup_rows_per_dst=[3082, 3120, 2958, 3054, 2994, 2942, 2964, 3072] combine_rows_per_src=[9404, 9282, 9186, 9298, 9266, 9256, 9248, 9178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265207808 dispatch_meta_bytes=343248 combine_payload_bytes=265207808 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265551056 ingress_bytes=334413824 link_bytes=334413824 token_bytes=530758864 weight_movement_bytes=69206016 total_nvlink_bytes=599964880 weight_to_token_ratio=0.130391 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258372297e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=344488 combine_payload_bytes=266477568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266822056 ingress_bytes=335683584 link_bytes=335683584 token_bytes=533299624 weight_movement_bytes=69206016 total_nvlink_bytes=602505640 weight_to_token_ratio=0.129769 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263150302e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74332 payload_rows_remote=65030 dup_rows=23972 dup_rows_remote=20928 dedup_ratio=0.243856 router_tokens_per_expert=[845, 784, 755, 778, 790, 764, 745, 747, 729, 824, 718, 730, 809, 803, 717, 795, 810, 760, 751, 764, 759, 818, 804, 682, 766, 788, 776, 728, 790, 746, 748, 714, 789, 843, 751, 736, 755, 764, 749, 786, 772, 766, 754, 776, 775, 763, 755, 789, 800, 769, 769, 744, 791, 755, 805, 773, 776, 754, 707, 735, 760, 809, 774, 771] payload_rows_per_dst=[9348, 9234, 9252, 9244, 9302, 9288, 9326, 9338] dup_rows_per_dst=[3068, 3016, 3044, 2868, 3044, 3012, 3086, 2834] combine_rows_per_src=[9348, 9234, 9252, 9244, 9302, 9288, 9326, 9338] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74234 payload_rows_remote=64950 dup_rows=24070 dup_rows_remote=21062 dedup_ratio=0.244853 router_tokens_per_expert=[812, 770, 814, 807, 766, 761, 745, 747, 765, 749, 725, 785, 736, 795, 708, 734, 757, 767, 722, 752, 758, 734, 772, 773, 777, 761, 737, 741, 773, 781, 756, 781, 807, 802, 746, 801, 749, 805, 766, 794, 757, 770, 735, 757, 754, 779, 798, 796, 753, 771, 829, 778, 751, 770, 809, 813, 737, 795, 772, 796, 766, 710, 769, 756] payload_rows_per_dst=[9338, 9170, 9190, 9338, 9336, 9284, 9386, 9192] dup_rows_per_dst=[3106, 2824, 2880, 2876, 3204, 3008, 3162, 3010] combine_rows_per_src=[9338, 9170, 9190, 9338, 9336, 9284, 9386, 9192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267272192 dispatch_meta_bytes=343680 combine_payload_bytes=267272192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267615872 ingress_bytes=301875200 link_bytes=301875200 token_bytes=534888064 weight_movement_bytes=34603008 total_nvlink_bytes=569491072 weight_to_token_ratio=0.064692 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135932075e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267976704 dispatch_meta_bytes=344384 combine_payload_bytes=267976704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268321088 ingress_bytes=302579712 link_bytes=302579712 token_bytes=536297792 weight_movement_bytes=34603008 total_nvlink_bytes=570900800 weight_to_token_ratio=0.064522 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138583097e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=343808 combine_payload_bytes=265502720 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265846528 ingress_bytes=334708736 link_bytes=334708736 token_bytes=531349248 weight_movement_bytes=69206016 total_nvlink_bytes=600555264 weight_to_token_ratio=0.130246 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259482027e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74332 payload_rows_remote=65030 dup_rows=23972 dup_rows_remote=20928 dedup_ratio=0.243856 router_tokens_per_expert=[845, 784, 755, 778, 790, 764, 745, 747, 729, 824, 718, 730, 809, 803, 717, 795, 810, 760, 751, 764, 759, 818, 804, 682, 766, 788, 776, 728, 790, 746, 748, 714, 789, 843, 751, 736, 755, 764, 749, 786, 772, 766, 754, 776, 775, 763, 755, 789, 800, 769, 769, 744, 791, 755, 805, 773, 776, 754, 707, 735, 760, 809, 774, 771] payload_rows_per_dst=[9348, 9234, 9252, 9244, 9302, 9288, 9326, 9338] dup_rows_per_dst=[3068, 3016, 3044, 2868, 3044, 3012, 3086, 2834] combine_rows_per_src=[9348, 9234, 9252, 9244, 9302, 9288, 9326, 9338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=64970 dup_rows=23982 dup_rows_remote=20926 dedup_ratio=0.243958 router_tokens_per_expert=[750, 788, 775, 775, 755, 780, 783, 773, 771, 751, 742, 763, 702, 752, 775, 770, 786, 821, 786, 787, 777, 733, 714, 753, 753, 766, 789, 765, 783, 772, 769, 777, 828, 798, 764, 812, 765, 719, 736, 800, 762, 791, 756, 793, 736, 796, 765, 805, 753, 707, 802, 749, 801, 778, 771, 739, 744, 741, 764, 740, 769, 816, 745, 771] payload_rows_per_dst=[9306, 9132, 9300, 9392, 9322, 9352, 9272, 9246] dup_rows_per_dst=[3052, 2920, 3014, 2956, 3122, 3056, 2928, 2934] combine_rows_per_src=[9306, 9132, 9300, 9392, 9322, 9352, 9272, 9246] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74266 payload_rows_remote=64952 dup_rows=24038 dup_rows_remote=21084 dedup_ratio=0.244527 router_tokens_per_expert=[737, 751, 758, 780, 723, 774, 738, 766, 736, 752, 741, 819, 759, 720, 742, 723, 822, 804, 771, 761, 802, 748, 756, 776, 757, 784, 826, 743, 747, 811, 784, 722, 771, 763, 773, 783, 766, 732, 782, 803, 775, 744, 781, 779, 834, 797, 785, 758, 769, 743, 792, 703, 781, 767, 816, 763, 805, 800, 743, 738, 731, 778, 755, 809] payload_rows_per_dst=[9086, 9152, 9378, 9292, 9338, 9424, 9314, 9282] dup_rows_per_dst=[2968, 2832, 3102, 3056, 3008, 3082, 2954, 3036] combine_rows_per_src=[9086, 9152, 9378, 9292, 9338, 9424, 9314, 9282] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74118 payload_rows_remote=64820 dup_rows=24186 dup_rows_remote=21132 dedup_ratio=0.246033 router_tokens_per_expert=[754, 752, 722, 778, 813, 853, 803, 768, 758, 757, 806, 796, 779, 739, 788, 778, 745, 772, 813, 733, 727, 732, 767, 783, 770, 769, 788, 750, 769, 783, 738, 809, 767, 757, 812, 766, 755, 731, 781, 761, 739, 742, 755, 732, 767, 838, 775, 751, 769, 750, 751, 748, 749, 786, 762, 791, 780, 761, 728, 759, 757, 768, 776, 796] payload_rows_per_dst=[9404, 9282, 9186, 9298, 9266, 9256, 9248, 9178] dup_rows_per_dst=[3082, 3120, 2958, 3054, 2994, 2942, 2964, 3072] combine_rows_per_src=[9404, 9282, 9186, 9298, 9266, 9256, 9248, 9178] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74644 payload_rows_remote=65252 dup_rows=23660 dup_rows_remote=20668 dedup_ratio=0.240682 router_tokens_per_expert=[768, 801, 789, 749, 749, 803, 706, 786, 783, 753, 817, 764, 788, 730, 794, 763, 778, 759, 809, 797, 761, 790, 748, 784, 818, 739, 758, 756, 757, 715, 770, 798, 731, 725, 793, 750, 751, 731, 822, 797, 825, 803, 744, 789, 780, 789, 724, 750, 761, 778, 744, 778, 750, 744, 765, 749, 793, 784, 785, 763, 736, 752, 739, 747] payload_rows_per_dst=[9372, 9392, 9382, 9270, 9264, 9428, 9274, 9262] dup_rows_per_dst=[2930, 2992, 3070, 2952, 2936, 2980, 2864, 2936] combine_rows_per_src=[9372, 9392, 9382, 9270, 9264, 9428, 9274, 9262] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=343832 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266706712 ingress_bytes=300965888 link_bytes=300965888 token_bytes=533069592 weight_movement_bytes=34603008 total_nvlink_bytes=567672600 weight_to_token_ratio=0.064913 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132510407e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74404 payload_rows_remote=64962 dup_rows=23900 dup_rows_remote=20740 dedup_ratio=0.243123 router_tokens_per_expert=[798, 774, 767, 768, 732, 784, 756, 773, 792, 744, 809, 758, 730, 714, 787, 749, 791, 753, 778, 805, 724, 792, 776, 764, 744, 808, 775, 761, 722, 766, 783, 796, 722, 761, 715, 802, 781, 748, 775, 734, 752, 805, 773, 753, 766, 795, 780, 780, 742, 787, 721, 738, 805, 761, 724, 774, 743, 786, 771, 788, 780, 830, 758, 829] payload_rows_per_dst=[9342, 9194, 9372, 9296, 9238, 9278, 9242, 9442] dup_rows_per_dst=[2962, 2972, 2994, 3014, 2806, 3006, 2986, 3160] combine_rows_per_src=[9342, 9194, 9372, 9296, 9238, 9278, 9242, 9442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266035200 dispatch_meta_bytes=344048 combine_payload_bytes=266035200 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266379248 ingress_bytes=369844224 link_bytes=369844224 token_bytes=532414448 weight_movement_bytes=103809024 total_nvlink_bytes=636223472 weight_to_token_ratio=0.194978 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391694040e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74214 payload_rows_remote=64748 dup_rows=24090 dup_rows_remote=21064 dedup_ratio=0.245056 router_tokens_per_expert=[768, 777, 773, 820, 741, 757, 752, 738, 780, 771, 769, 741, 739, 763, 752, 744, 757, 755, 783, 740, 762, 751, 740, 806, 699, 767, 760, 798, 811, 808, 799, 813, 748, 762, 761, 781, 724, 772, 768, 797, 796, 767, 744, 797, 779, 748, 749, 779, 754, 784, 759, 787, 778, 759, 782, 749, 801, 772, 759, 770, 779, 773, 748, 792] payload_rows_per_dst=[9162, 9182, 9234, 9314, 9268, 9262, 9326, 9466] dup_rows_per_dst=[2986, 2936, 2954, 3088, 2958, 3056, 3086, 3026] combine_rows_per_src=[9162, 9182, 9234, 9314, 9268, 9262, 9326, 9466] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74228 payload_rows_remote=64838 dup_rows=24076 dup_rows_remote=20970 dedup_ratio=0.244914 router_tokens_per_expert=[746, 755, 787, 787, 811, 826, 754, 771, 690, 760, 785, 802, 772, 785, 774, 772, 805, 800, 735, 795, 793, 759, 755, 765, 764, 799, 773, 740, 792, 777, 727, 750, 813, 759, 782, 747, 785, 742, 748, 734, 766, 752, 778, 780, 715, 745, 786, 761, 773, 742, 754, 742, 799, 775, 821, 774, 743, 771, 759, 790, 772, 725, 775, 738] payload_rows_per_dst=[9390, 9322, 9344, 9254, 9506, 9196, 9056, 9160] dup_rows_per_dst=[3106, 3048, 3054, 2990, 3166, 2970, 2852, 2890] combine_rows_per_src=[9390, 9322, 9344, 9254, 9506, 9196, 9056, 9160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=343832 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266706712 ingress_bytes=300965888 link_bytes=300965888 token_bytes=533069592 weight_movement_bytes=34603008 total_nvlink_bytes=567672600 weight_to_token_ratio=0.064913 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132510407e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74096 payload_rows_remote=64800 dup_rows=24208 dup_rows_remote=21146 dedup_ratio=0.246257 router_tokens_per_expert=[722, 779, 759, 794, 762, 748, 774, 759, 726, 747, 748, 821, 731, 782, 706, 794, 786, 768, 758, 762, 765, 710, 781, 766, 732, 789, 742, 727, 766, 792, 724, 752, 742, 836, 753, 825, 785, 772, 791, 771, 785, 816, 805, 801, 764, 733, 778, 777, 786, 769, 802, 781, 799, 716, 789, 774, 741, 780, 731, 796, 788, 790, 764, 740] payload_rows_per_dst=[9296, 9284, 9320, 9178, 9430, 9080, 9400, 9108] dup_rows_per_dst=[3062, 3078, 2984, 3148, 3126, 2902, 3032, 2876] combine_rows_per_src=[9296, 9284, 9320, 9178, 9430, 9080, 9400, 9108] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266117120 dispatch_meta_bytes=343584 combine_payload_bytes=266117120 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266460704 ingress_bytes=404529152 link_bytes=404529152 token_bytes=532577824 weight_movement_bytes=138412032 total_nvlink_bytes=670989856 weight_to_token_ratio=0.259891 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522210632e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266043392 dispatch_meta_bytes=344144 combine_payload_bytes=266043392 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266387536 ingress_bytes=335249408 link_bytes=335249408 token_bytes=532430928 weight_movement_bytes=69206016 total_nvlink_bytes=601636944 weight_to_token_ratio=0.129981 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261516533e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=343808 combine_payload_bytes=265502720 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265846528 ingress_bytes=334708736 link_bytes=334708736 token_bytes=531349248 weight_movement_bytes=69206016 total_nvlink_bytes=600555264 weight_to_token_ratio=0.130246 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259482027e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267272192 dispatch_meta_bytes=343680 combine_payload_bytes=267272192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267615872 ingress_bytes=301875200 link_bytes=301875200 token_bytes=534888064 weight_movement_bytes=34603008 total_nvlink_bytes=569491072 weight_to_token_ratio=0.064692 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135932075e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266084352 dispatch_meta_bytes=342808 combine_payload_bytes=266084352 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266427160 ingress_bytes=369893376 link_bytes=369893376 token_bytes=532511512 weight_movement_bytes=103809024 total_nvlink_bytes=636320536 weight_to_token_ratio=0.194942 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391878995e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74160 payload_rows_remote=64940 dup_rows=24144 dup_rows_remote=21142 dedup_ratio=0.245605 router_tokens_per_expert=[760, 766, 747, 779, 740, 759, 791, 786, 734, 781, 751, 709, 813, 762, 779, 782, 767, 772, 748, 761, 747, 751, 714, 755, 768, 771, 785, 822, 732, 816, 763, 759, 841, 776, 763, 784, 800, 711, 742, 744, 792, 749, 804, 761, 794, 754, 709, 809, 779, 793, 797, 749, 765, 814, 763, 778, 749, 773, 742, 745, 779, 760, 784, 779] payload_rows_per_dst=[9224, 9220, 9190, 9290, 9264, 9366, 9382, 9224] dup_rows_per_dst=[3032, 3002, 2840, 3142, 3058, 2978, 3094, 2998] combine_rows_per_src=[9224, 9220, 9190, 9290, 9264, 9366, 9382, 9224] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74568 payload_rows_remote=65226 dup_rows=23736 dup_rows_remote=20828 dedup_ratio=0.241455 router_tokens_per_expert=[757, 807, 783, 749, 772, 701, 766, 763, 758, 772, 714, 785, 811, 743, 796, 780, 787, 787, 741, 706, 745, 778, 792, 789, 827, 806, 719, 758, 726, 777, 794, 762, 783, 730, 768, 779, 782, 774, 741, 756, 796, 773, 764, 731, 739, 784, 724, 799, 796, 818, 781, 782, 732, 783, 771, 775, 765, 750, 814, 744, 760, 768, 765, 774] payload_rows_per_dst=[9320, 9292, 9342, 9316, 9302, 9246, 9412, 9338] dup_rows_per_dst=[2876, 3026, 2908, 3022, 2924, 2974, 3064, 2942] combine_rows_per_src=[9320, 9292, 9342, 9316, 9302, 9246, 9412, 9338] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265207808 dispatch_meta_bytes=343248 combine_payload_bytes=265207808 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265551056 ingress_bytes=334413824 link_bytes=334413824 token_bytes=530758864 weight_movement_bytes=69206016 total_nvlink_bytes=599964880 weight_to_token_ratio=0.130391 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258372297e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265576448 dispatch_meta_bytes=343232 combine_payload_bytes=265576448 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265919680 ingress_bytes=334782464 link_bytes=334782464 token_bytes=531496128 weight_movement_bytes=69206016 total_nvlink_bytes=600702144 weight_to_token_ratio=0.130210 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259759460e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74308 payload_rows_remote=65138 dup_rows=23996 dup_rows_remote=21020 dedup_ratio=0.244100 router_tokens_per_expert=[740, 772, 761, 720, 797, 742, 742, 811, 765, 757, 804, 802, 751, 810, 800, 753, 769, 753, 789, 741, 767, 737, 747, 817, 758, 742, 741, 794, 734, 753, 767, 784, 798, 747, 788, 743, 781, 803, 730, 788, 796, 724, 800, 801, 742, 753, 749, 738, 760, 792, 762, 790, 759, 759, 790, 726, 807, 795, 813, 727, 766, 735, 793, 777] payload_rows_per_dst=[9290, 9386, 9230, 9170, 9366, 9226, 9286, 9354] dup_rows_per_dst=[2880, 3098, 3010, 2976, 2990, 2980, 2990, 3072] combine_rows_per_src=[9290, 9386, 9230, 9170, 9366, 9226, 9286, 9354] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265420800 dispatch_meta_bytes=343784 combine_payload_bytes=265420800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265764584 ingress_bytes=300023808 link_bytes=300023808 token_bytes=531185384 weight_movement_bytes=34603008 total_nvlink_bytes=565788392 weight_to_token_ratio=0.065143 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128965436e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74086 payload_rows_remote=64836 dup_rows=24218 dup_rows_remote=21274 dedup_ratio=0.246358 router_tokens_per_expert=[779, 753, 767, 771, 820, 756, 719, 783, 758, 785, 743, 790, 768, 741, 785, 767, 804, 805, 807, 734, 767, 807, 796, 787, 713, 752, 769, 733, 736, 802, 745, 763, 738, 798, 780, 772, 757, 759, 725, 768, 773, 757, 784, 787, 761, 790, 750, 761, 757, 792, 792, 759, 727, 781, 707, 780, 816, 741, 758, 773, 782, 794, 763, 765] payload_rows_per_dst=[9242, 9306, 9428, 9146, 9250, 9220, 9174, 9320] dup_rows_per_dst=[3054, 2968, 3186, 2880, 2944, 3106, 3016, 3064] combine_rows_per_src=[9242, 9306, 9428, 9146, 9250, 9220, 9174, 9320] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=344328 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266338568 ingress_bytes=300597248 link_bytes=300597248 token_bytes=532332808 weight_movement_bytes=34603008 total_nvlink_bytes=566935816 weight_to_token_ratio=0.065003 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131123244e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266805248 dispatch_meta_bytes=344632 combine_payload_bytes=266805248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267149880 ingress_bytes=301408256 link_bytes=301408256 token_bytes=533955128 weight_movement_bytes=34603008 total_nvlink_bytes=568558136 weight_to_token_ratio=0.064805 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134175002e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=267165696 dispatch_meta_bytes=344216 combine_payload_bytes=267165696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267509912 ingress_bytes=301768704 link_bytes=301768704 token_bytes=534675608 weight_movement_bytes=34603008 total_nvlink_bytes=569278616 weight_to_token_ratio=0.064718 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135531339e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=64970 dup_rows=23982 dup_rows_remote=20926 dedup_ratio=0.243958 router_tokens_per_expert=[750, 788, 775, 775, 755, 780, 783, 773, 771, 751, 742, 763, 702, 752, 775, 770, 786, 821, 786, 787, 777, 733, 714, 753, 753, 766, 789, 765, 783, 772, 769, 777, 828, 798, 764, 812, 765, 719, 736, 800, 762, 791, 756, 793, 736, 796, 765, 805, 753, 707, 802, 749, 801, 778, 771, 739, 744, 741, 764, 740, 769, 816, 745, 771] payload_rows_per_dst=[9306, 9132, 9300, 9392, 9322, 9352, 9272, 9246] dup_rows_per_dst=[3052, 2920, 3014, 2956, 3122, 3056, 2928, 2934] combine_rows_per_src=[9306, 9132, 9300, 9392, 9322, 9352, 9272, 9246] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74086 payload_rows_remote=64836 dup_rows=24218 dup_rows_remote=21274 dedup_ratio=0.246358 router_tokens_per_expert=[779, 753, 767, 771, 820, 756, 719, 783, 758, 785, 743, 790, 768, 741, 785, 767, 804, 805, 807, 734, 767, 807, 796, 787, 713, 752, 769, 733, 736, 802, 745, 763, 738, 798, 780, 772, 757, 759, 725, 768, 773, 757, 784, 787, 761, 790, 750, 761, 757, 792, 792, 759, 727, 781, 707, 780, 816, 741, 758, 773, 782, 794, 763, 765] payload_rows_per_dst=[9242, 9306, 9428, 9146, 9250, 9220, 9174, 9320] dup_rows_per_dst=[3054, 2968, 3186, 2880, 2944, 3106, 3016, 3064] combine_rows_per_src=[9242, 9306, 9428, 9146, 9250, 9220, 9174, 9320] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74334 payload_rows_remote=65074 dup_rows=23970 dup_rows_remote=21054 dedup_ratio=0.243835 router_tokens_per_expert=[753, 710, 767, 801, 787, 781, 787, 758, 816, 734, 778, 760, 742, 756, 736, 784, 783, 726, 784, 821, 743, 796, 733, 750, 771, 757, 793, 777, 781, 785, 804, 763, 800, 772, 766, 767, 811, 758, 707, 741, 754, 779, 736, 739, 758, 750, 789, 783, 787, 785, 762, 781, 756, 751, 763, 764, 778, 775, 793, 783, 800, 753, 757, 737] payload_rows_per_dst=[9334, 9150, 9282, 9390, 9268, 9260, 9348, 9302] dup_rows_per_dst=[2954, 3062, 2990, 3072, 2976, 2916, 2950, 3050] combine_rows_per_src=[9334, 9150, 9282, 9390, 9268, 9260, 9348, 9302] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=65242 dup_rows=23950 dup_rows_remote=21036 dedup_ratio=0.243632 router_tokens_per_expert=[745, 830, 773, 781, 773, 770, 765, 746, 778, 757, 750, 746, 793, 799, 792, 798, 767, 757, 741, 769, 749, 759, 775, 776, 809, 752, 741, 769, 759, 784, 758, 817, 781, 759, 776, 752, 764, 742, 776, 731, 808, 757, 745, 779, 792, 817, 775, 794, 783, 781, 704, 747, 774, 746, 730, 748, 771, 739, 726, 771, 735, 824, 768, 779] payload_rows_per_dst=[9326, 9368, 9222, 9344, 9234, 9416, 9112, 9332] dup_rows_per_dst=[3040, 3058, 2964, 3034, 2928, 3118, 2914, 2894] combine_rows_per_src=[9326, 9368, 9222, 9344, 9234, 9416, 9112, 9332] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265568256 dispatch_meta_bytes=344440 combine_payload_bytes=265568256 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265912696 ingress_bytes=334774272 link_bytes=334774272 token_bytes=531480952 weight_movement_bytes=69206016 total_nvlink_bytes=600686968 weight_to_token_ratio=0.130214 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259728634e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74308 payload_rows_remote=65138 dup_rows=23996 dup_rows_remote=21020 dedup_ratio=0.244100 router_tokens_per_expert=[740, 772, 761, 720, 797, 742, 742, 811, 765, 757, 804, 802, 751, 810, 800, 753, 769, 753, 789, 741, 767, 737, 747, 817, 758, 742, 741, 794, 734, 753, 767, 784, 798, 747, 788, 743, 781, 803, 730, 788, 796, 724, 800, 801, 742, 753, 749, 738, 760, 792, 762, 790, 759, 759, 790, 726, 807, 795, 813, 727, 766, 735, 793, 777] payload_rows_per_dst=[9290, 9386, 9230, 9170, 9366, 9226, 9286, 9354] dup_rows_per_dst=[2880, 3098, 3010, 2976, 2990, 2980, 2990, 3072] combine_rows_per_src=[9290, 9386, 9230, 9170, 9366, 9226, 9286, 9354] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=64964 dup_rows=23950 dup_rows_remote=21014 dedup_ratio=0.243632 router_tokens_per_expert=[723, 759, 751, 788, 757, 792, 778, 737, 746, 763, 723, 758, 767, 800, 733, 782, 760, 734, 775, 758, 765, 739, 791, 773, 782, 735, 787, 765, 779, 761, 750, 745, 751, 796, 806, 781, 782, 808, 764, 748, 790, 760, 740, 762, 833, 749, 819, 776, 742, 766, 784, 795, 811, 721, 789, 786, 757, 786, 763, 779, 728, 808, 762, 754] payload_rows_per_dst=[9210, 9230, 9244, 9210, 9382, 9384, 9304, 9390] dup_rows_per_dst=[2960, 2914, 2946, 2998, 3038, 3074, 3084, 2936] combine_rows_per_src=[9210, 9230, 9244, 9210, 9382, 9384, 9304, 9390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74160 payload_rows_remote=64940 dup_rows=24144 dup_rows_remote=21142 dedup_ratio=0.245605 router_tokens_per_expert=[760, 766, 747, 779, 740, 759, 791, 786, 734, 781, 751, 709, 813, 762, 779, 782, 767, 772, 748, 761, 747, 751, 714, 755, 768, 771, 785, 822, 732, 816, 763, 759, 841, 776, 763, 784, 800, 711, 742, 744, 792, 749, 804, 761, 794, 754, 709, 809, 779, 793, 797, 749, 765, 814, 763, 778, 749, 773, 742, 745, 779, 760, 784, 779] payload_rows_per_dst=[9224, 9220, 9190, 9290, 9264, 9366, 9382, 9224] dup_rows_per_dst=[3032, 3002, 2840, 3142, 3058, 2978, 3094, 2998] combine_rows_per_src=[9224, 9220, 9190, 9290, 9264, 9366, 9382, 9224] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266117120 dispatch_meta_bytes=343584 combine_payload_bytes=266117120 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266460704 ingress_bytes=404529152 link_bytes=404529152 token_bytes=532577824 weight_movement_bytes=138412032 total_nvlink_bytes=670989856 weight_to_token_ratio=0.259891 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522210632e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=65058 dup_rows=24112 dup_rows_remote=21064 dedup_ratio=0.245280 router_tokens_per_expert=[727, 759, 761, 738, 793, 736, 786, 784, 799, 774, 723, 797, 786, 774, 759, 795, 740, 824, 742, 745, 748, 718, 757, 762, 775, 782, 739, 752, 769, 747, 767, 802, 785, 802, 693, 771, 768, 736, 742, 752, 805, 730, 825, 803, 787, 776, 741, 783, 724, 754, 732, 793, 791, 783, 780, 788, 755, 795, 773, 795, 781, 821, 807, 721] payload_rows_per_dst=[9134, 9742, 9418, 9344, 9416, 8898, 9032, 9208] dup_rows_per_dst=[3048, 3364, 3048, 2908, 3210, 2758, 2730, 3046] combine_rows_per_src=[9134, 9742, 9418, 9344, 9416, 8898, 9032, 9208] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74404 payload_rows_remote=64962 dup_rows=23900 dup_rows_remote=20740 dedup_ratio=0.243123 router_tokens_per_expert=[798, 774, 767, 768, 732, 784, 756, 773, 792, 744, 809, 758, 730, 714, 787, 749, 791, 753, 778, 805, 724, 792, 776, 764, 744, 808, 775, 761, 722, 766, 783, 796, 722, 761, 715, 802, 781, 748, 775, 734, 752, 805, 773, 753, 766, 795, 780, 780, 742, 787, 721, 738, 805, 761, 724, 774, 743, 786, 771, 788, 780, 830, 758, 829] payload_rows_per_dst=[9342, 9194, 9372, 9296, 9238, 9278, 9242, 9442] dup_rows_per_dst=[2962, 2972, 2994, 3014, 2806, 3006, 2986, 3160] combine_rows_per_src=[9342, 9194, 9372, 9296, 9238, 9278, 9242, 9442] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64816 dup_rows=24112 dup_rows_remote=20840 dedup_ratio=0.245280 router_tokens_per_expert=[760, 744, 735, 804, 763, 742, 787, 772, 828, 758, 761, 776, 728, 782, 779, 740, 836, 762, 815, 737, 746, 740, 747, 754, 741, 741, 764, 775, 756, 783, 766, 788, 727, 758, 788, 740, 793, 810, 744, 735, 765, 801, 767, 776, 802, 751, 794, 778, 773, 800, 753, 799, 791, 746, 742, 806, 756, 785, 737, 770, 735, 795, 797, 728] payload_rows_per_dst=[9376, 9480, 9398, 9518, 9212, 8906, 9360, 8942] dup_rows_per_dst=[3272, 3200, 2966, 3112, 2998, 2756, 3036, 2772] combine_rows_per_src=[9376, 9480, 9398, 9518, 9212, 8906, 9360, 8942] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265568256 dispatch_meta_bytes=344440 combine_payload_bytes=265568256 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265912696 ingress_bytes=334774272 link_bytes=334774272 token_bytes=531480952 weight_movement_bytes=69206016 total_nvlink_bytes=600686968 weight_to_token_ratio=0.130214 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259728634e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=344512 combine_payload_bytes=266543104 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266887616 ingress_bytes=370352128 link_bytes=370352128 token_bytes=533430720 weight_movement_bytes=103809024 total_nvlink_bytes=637239744 weight_to_token_ratio=0.194606 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393605242e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=267231232 dispatch_meta_bytes=345112 combine_payload_bytes=267231232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267576344 ingress_bytes=301834240 link_bytes=301834240 token_bytes=534807576 weight_movement_bytes=34603008 total_nvlink_bytes=569410584 weight_to_token_ratio=0.064702 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135777945e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=343912 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266436456 ingress_bytes=369901568 link_bytes=369901568 token_bytes=532529000 weight_movement_bytes=103809024 total_nvlink_bytes=636338024 weight_to_token_ratio=0.194936 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391909821e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74252 payload_rows_remote=64954 dup_rows=24052 dup_rows_remote=21078 dedup_ratio=0.244670 router_tokens_per_expert=[767, 790, 752, 743, 810, 775, 779, 749, 714, 794, 817, 769, 753, 797, 807, 685, 758, 812, 755, 776, 754, 794, 805, 703, 780, 776, 723, 761, 763, 768, 803, 757, 758, 776, 705, 786, 782, 771, 763, 767, 770, 767, 785, 806, 790, 828, 744, 816, 783, 723, 767, 794, 713, 786, 744, 748, 750, 764, 812, 776, 725, 786, 726, 752] payload_rows_per_dst=[9250, 9298, 9376, 9248, 9182, 9516, 9198, 9184] dup_rows_per_dst=[3080, 2974, 2938, 3014, 3034, 3096, 2918, 2998] combine_rows_per_src=[9250, 9298, 9376, 9248, 9182, 9516, 9198, 9184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74226 payload_rows_remote=64986 dup_rows=24078 dup_rows_remote=21136 dedup_ratio=0.244934 router_tokens_per_expert=[783, 759, 725, 741, 775, 775, 793, 735, 775, 739, 772, 749, 771, 781, 735, 788, 754, 768, 751, 744, 775, 808, 803, 797, 777, 749, 735, 778, 749, 786, 736, 781, 767, 824, 743, 770, 773, 773, 780, 739, 789, 707, 762, 778, 792, 805, 725, 807, 737, 735, 750, 781, 733, 773, 759, 771, 788, 782, 786, 807, 767, 794, 755, 813] payload_rows_per_dst=[9182, 9250, 9304, 9240, 9306, 9338, 9132, 9474] dup_rows_per_dst=[2990, 2970, 3096, 2942, 3032, 2992, 2946, 3110] combine_rows_per_src=[9182, 9250, 9304, 9240, 9306, 9338, 9132, 9474] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=64974 dup_rows=24098 dup_rows_remote=21116 dedup_ratio=0.245138 router_tokens_per_expert=[783, 764, 748, 753, 719, 804, 768, 793, 772, 831, 755, 751, 744, 717, 775, 769, 719, 748, 770, 773, 770, 748, 789, 790, 837, 761, 762, 838, 771, 702, 750, 740, 774, 749, 768, 762, 730, 772, 828, 742, 795, 750, 790, 787, 803, 748, 751, 722, 796, 779, 796, 780, 787, 771, 707, 778, 768, 742, 810, 787, 771, 793, 717, 785] payload_rows_per_dst=[9248, 9320, 9232, 9250, 9198, 9248, 9380, 9330] dup_rows_per_dst=[3016, 2908, 2982, 3072, 3052, 3044, 3008, 3016] combine_rows_per_src=[9248, 9320, 9232, 9250, 9198, 9248, 9380, 9330] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266805248 dispatch_meta_bytes=344632 combine_payload_bytes=266805248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267149880 ingress_bytes=301408256 link_bytes=301408256 token_bytes=533955128 weight_movement_bytes=34603008 total_nvlink_bytes=568558136 weight_to_token_ratio=0.064805 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134175002e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=344328 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266338568 ingress_bytes=300597248 link_bytes=300597248 token_bytes=532332808 weight_movement_bytes=34603008 total_nvlink_bytes=566935816 weight_to_token_ratio=0.065003 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131123244e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=344488 combine_payload_bytes=266477568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266822056 ingress_bytes=335683584 link_bytes=335683584 token_bytes=533299624 weight_movement_bytes=69206016 total_nvlink_bytes=602505640 weight_to_token_ratio=0.129769 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263150302e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266084352 dispatch_meta_bytes=342808 combine_payload_bytes=266084352 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266427160 ingress_bytes=369893376 link_bytes=369893376 token_bytes=532511512 weight_movement_bytes=103809024 total_nvlink_bytes=636320536 weight_to_token_ratio=0.194942 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391878995e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265486336 dispatch_meta_bytes=342624 combine_payload_bytes=265486336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265828960 ingress_bytes=300089344 link_bytes=300089344 token_bytes=531315296 weight_movement_bytes=34603008 total_nvlink_bytes=565918304 weight_to_token_ratio=0.065127 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129212042e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=64864 dup_rows=24056 dup_rows_remote=21036 dedup_ratio=0.244710 router_tokens_per_expert=[762, 805, 740, 789, 762, 724, 761, 780, 792, 730, 788, 803, 774, 771, 746, 708, 776, 754, 807, 752, 735, 851, 775, 752, 794, 790, 779, 715, 760, 725, 730, 758, 765, 775, 827, 767, 735, 764, 739, 830, 735, 797, 733, 780, 775, 820, 725, 811, 774, 748, 755, 768, 783, 712, 805, 787, 731, 770, 789, 759, 768, 766, 797, 774] payload_rows_per_dst=[9320, 9262, 9218, 9246, 9384, 9340, 9248, 9230] dup_rows_per_dst=[2926, 2962, 3186, 2856, 3020, 3012, 3016, 3078] combine_rows_per_src=[9320, 9262, 9218, 9246, 9384, 9340, 9248, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266051584 dispatch_meta_bytes=344128 combine_payload_bytes=266051584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266395712 ingress_bytes=300654592 link_bytes=300654592 token_bytes=532447296 weight_movement_bytes=34603008 total_nvlink_bytes=567050304 weight_to_token_ratio=0.064989 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131339025e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266182656 dispatch_meta_bytes=344488 combine_payload_bytes=266182656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266527144 ingress_bytes=300785664 link_bytes=300785664 token_bytes=532709800 weight_movement_bytes=34603008 total_nvlink_bytes=567312808 weight_to_token_ratio=0.064957 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131832238e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=344360 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266477864 ingress_bytes=300736512 link_bytes=300736512 token_bytes=532611368 weight_movement_bytes=34603008 total_nvlink_bytes=567214376 weight_to_token_ratio=0.064969 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131647283e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74334 payload_rows_remote=65074 dup_rows=23970 dup_rows_remote=21054 dedup_ratio=0.243835 router_tokens_per_expert=[753, 710, 767, 801, 787, 781, 787, 758, 816, 734, 778, 760, 742, 756, 736, 784, 783, 726, 784, 821, 743, 796, 733, 750, 771, 757, 793, 777, 781, 785, 804, 763, 800, 772, 766, 767, 811, 758, 707, 741, 754, 779, 736, 739, 758, 750, 789, 783, 787, 785, 762, 781, 756, 751, 763, 764, 778, 775, 793, 783, 800, 753, 757, 737] payload_rows_per_dst=[9334, 9150, 9282, 9390, 9268, 9260, 9348, 9302] dup_rows_per_dst=[2954, 3062, 2990, 3072, 2976, 2916, 2950, 3050] combine_rows_per_src=[9334, 9150, 9282, 9390, 9268, 9260, 9348, 9302] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74472 payload_rows_remote=65082 dup_rows=23832 dup_rows_remote=20870 dedup_ratio=0.242432 router_tokens_per_expert=[766, 783, 747, 768, 758, 747, 808, 791, 743, 755, 771, 778, 738, 778, 732, 828, 761, 768, 742, 759, 732, 788, 795, 741, 797, 764, 746, 745, 762, 775, 712, 775, 762, 787, 792, 810, 750, 771, 740, 776, 773, 832, 786, 735, 775, 760, 779, 736, 875, 773, 732, 764, 739, 745, 748, 814, 791, 826, 793, 710, 782, 723, 775, 745] payload_rows_per_dst=[9392, 9320, 9208, 9224, 9362, 9390, 9308, 9268] dup_rows_per_dst=[2944, 2926, 2964, 2928, 3014, 2962, 3072, 3022] combine_rows_per_src=[9392, 9320, 9208, 9224, 9362, 9390, 9308, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=64864 dup_rows=24056 dup_rows_remote=21036 dedup_ratio=0.244710 router_tokens_per_expert=[762, 805, 740, 789, 762, 724, 761, 780, 792, 730, 788, 803, 774, 771, 746, 708, 776, 754, 807, 752, 735, 851, 775, 752, 794, 790, 779, 715, 760, 725, 730, 758, 765, 775, 827, 767, 735, 764, 739, 830, 735, 797, 733, 780, 775, 820, 725, 811, 774, 748, 755, 768, 783, 712, 805, 787, 731, 770, 789, 759, 768, 766, 797, 774] payload_rows_per_dst=[9320, 9262, 9218, 9246, 9384, 9340, 9248, 9230] dup_rows_per_dst=[2926, 2962, 3186, 2856, 3020, 3012, 3016, 3078] combine_rows_per_src=[9320, 9262, 9218, 9246, 9384, 9340, 9248, 9230] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=65050 dup_rows=23928 dup_rows_remote=20878 dedup_ratio=0.243408 router_tokens_per_expert=[750, 780, 788, 755, 766, 745, 766, 743, 776, 806, 753, 775, 766, 730, 782, 774, 784, 775, 748, 771, 719, 763, 764, 749, 770, 761, 748, 776, 730, 763, 774, 746, 783, 777, 794, 777, 704, 722, 780, 774, 782, 774, 766, 750, 771, 774, 784, 810, 759, 745, 733, 829, 777, 753, 790, 802, 782, 798, 775, 777, 813, 770, 749, 782] payload_rows_per_dst=[9202, 9324, 9240, 9154, 9256, 9444, 9326, 9430] dup_rows_per_dst=[2984, 3000, 2906, 2982, 2966, 2978, 3050, 3062] combine_rows_per_src=[9202, 9324, 9240, 9154, 9256, 9444, 9326, 9430] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265682944 dispatch_meta_bytes=343600 combine_payload_bytes=265682944 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266026544 ingress_bytes=334888960 link_bytes=334888960 token_bytes=531709488 weight_movement_bytes=69206016 total_nvlink_bytes=600915504 weight_to_token_ratio=0.130158 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260160196e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74482 payload_rows_remote=65124 dup_rows=23822 dup_rows_remote=20726 dedup_ratio=0.242330 router_tokens_per_expert=[771, 776, 746, 792, 740, 805, 767, 787, 732, 813, 754, 742, 773, 732, 760, 732, 789, 739, 779, 787, 756, 733, 733, 795, 749, 768, 785, 764, 746, 779, 767, 829, 789, 780, 750, 813, 726, 758, 765, 790, 761, 737, 771, 749, 791, 732, 730, 776, 783, 758, 808, 784, 801, 777, 779, 817, 802, 744, 742, 769, 808, 753, 745, 744] payload_rows_per_dst=[9348, 9230, 9284, 9314, 9254, 9166, 9528, 9358] dup_rows_per_dst=[3020, 2846, 2908, 3060, 2876, 2928, 3088, 3096] combine_rows_per_src=[9348, 9230, 9284, 9314, 9254, 9166, 9528, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=344512 combine_payload_bytes=266543104 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266887616 ingress_bytes=370352128 link_bytes=370352128 token_bytes=533430720 weight_movement_bytes=103809024 total_nvlink_bytes=637239744 weight_to_token_ratio=0.194606 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393605242e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74226 payload_rows_remote=64986 dup_rows=24078 dup_rows_remote=21136 dedup_ratio=0.244934 router_tokens_per_expert=[783, 759, 725, 741, 775, 775, 793, 735, 775, 739, 772, 749, 771, 781, 735, 788, 754, 768, 751, 744, 775, 808, 803, 797, 777, 749, 735, 778, 749, 786, 736, 781, 767, 824, 743, 770, 773, 773, 780, 739, 789, 707, 762, 778, 792, 805, 725, 807, 737, 735, 750, 781, 733, 773, 759, 771, 788, 782, 786, 807, 767, 794, 755, 813] payload_rows_per_dst=[9182, 9250, 9304, 9240, 9306, 9338, 9132, 9474] dup_rows_per_dst=[2990, 2970, 3096, 2942, 3032, 2992, 2946, 3110] combine_rows_per_src=[9182, 9250, 9304, 9240, 9306, 9338, 9132, 9474] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74252 payload_rows_remote=64954 dup_rows=24052 dup_rows_remote=21078 dedup_ratio=0.244670 router_tokens_per_expert=[767, 790, 752, 743, 810, 775, 779, 749, 714, 794, 817, 769, 753, 797, 807, 685, 758, 812, 755, 776, 754, 794, 805, 703, 780, 776, 723, 761, 763, 768, 803, 757, 758, 776, 705, 786, 782, 771, 763, 767, 770, 767, 785, 806, 790, 828, 744, 816, 783, 723, 767, 794, 713, 786, 744, 748, 750, 764, 812, 776, 725, 786, 726, 752] payload_rows_per_dst=[9250, 9298, 9376, 9248, 9182, 9516, 9198, 9184] dup_rows_per_dst=[3080, 2974, 2938, 3014, 3034, 3096, 2918, 2998] combine_rows_per_src=[9250, 9298, 9376, 9248, 9182, 9516, 9198, 9184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74096 payload_rows_remote=64800 dup_rows=24208 dup_rows_remote=21146 dedup_ratio=0.246257 router_tokens_per_expert=[722, 779, 759, 794, 762, 748, 774, 759, 726, 747, 748, 821, 731, 782, 706, 794, 786, 768, 758, 762, 765, 710, 781, 766, 732, 789, 742, 727, 766, 792, 724, 752, 742, 836, 753, 825, 785, 772, 791, 771, 785, 816, 805, 801, 764, 733, 778, 777, 786, 769, 802, 781, 799, 716, 789, 774, 741, 780, 731, 796, 788, 790, 764, 740] payload_rows_per_dst=[9296, 9284, 9320, 9178, 9430, 9080, 9400, 9108] dup_rows_per_dst=[3062, 3078, 2984, 3148, 3126, 2902, 3032, 2876] combine_rows_per_src=[9296, 9284, 9320, 9178, 9430, 9080, 9400, 9108] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=64964 dup_rows=23950 dup_rows_remote=21014 dedup_ratio=0.243632 router_tokens_per_expert=[723, 759, 751, 788, 757, 792, 778, 737, 746, 763, 723, 758, 767, 800, 733, 782, 760, 734, 775, 758, 765, 739, 791, 773, 782, 735, 787, 765, 779, 761, 750, 745, 751, 796, 806, 781, 782, 808, 764, 748, 790, 760, 740, 762, 833, 749, 819, 776, 742, 766, 784, 795, 811, 721, 789, 786, 757, 786, 763, 779, 728, 808, 762, 754] payload_rows_per_dst=[9210, 9230, 9244, 9210, 9382, 9384, 9304, 9390] dup_rows_per_dst=[2960, 2914, 2946, 2998, 3038, 3074, 3084, 2936] combine_rows_per_src=[9210, 9230, 9244, 9210, 9382, 9384, 9304, 9390] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74164 payload_rows_remote=65136 dup_rows=24140 dup_rows_remote=21272 dedup_ratio=0.245565 router_tokens_per_expert=[789, 698, 710, 752, 746, 745, 736, 749, 776, 816, 761, 768, 759, 735, 811, 771, 751, 745, 788, 762, 755, 774, 773, 741, 780, 780, 762, 791, 770, 766, 786, 790, 804, 830, 723, 744, 794, 794, 741, 790, 750, 749, 760, 760, 766, 772, 745, 772, 834, 768, 787, 767, 802, 820, 771, 784, 756, 771, 769, 781, 727, 755, 754, 776] payload_rows_per_dst=[9028, 9652, 9242, 9464, 9412, 9562, 8916, 8888] dup_rows_per_dst=[2868, 3386, 2936, 3130, 3028, 3260, 2886, 2646] combine_rows_per_src=[9028, 9652, 9242, 9464, 9412, 9562, 8916, 8888] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266575872 dispatch_meta_bytes=343808 combine_payload_bytes=266575872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266919680 ingress_bytes=301178880 link_bytes=301178880 token_bytes=533495552 weight_movement_bytes=34603008 total_nvlink_bytes=568098560 weight_to_token_ratio=0.064861 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133311879e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265682944 dispatch_meta_bytes=343600 combine_payload_bytes=265682944 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266026544 ingress_bytes=334888960 link_bytes=334888960 token_bytes=531709488 weight_movement_bytes=69206016 total_nvlink_bytes=600915504 weight_to_token_ratio=0.130158 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260160196e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266444800 dispatch_meta_bytes=343712 combine_payload_bytes=266444800 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266788512 ingress_bytes=335650816 link_bytes=335650816 token_bytes=533233312 weight_movement_bytes=69206016 total_nvlink_bytes=602439328 weight_to_token_ratio=0.129786 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263026999e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74332 payload_rows_remote=65022 dup_rows=23972 dup_rows_remote=20966 dedup_ratio=0.243856 router_tokens_per_expert=[755, 789, 749, 770, 783, 780, 754, 778, 777, 749, 813, 750, 766, 795, 736, 772, 786, 794, 759, 765, 779, 810, 729, 756, 797, 719, 776, 761, 749, 768, 734, 795, 762, 788, 748, 747, 741, 771, 727, 755, 749, 770, 756, 743, 786, 760, 756, 778, 784, 794, 790, 745, 766, 809, 787, 773, 802, 760, 800, 717, 761, 748, 810, 776] payload_rows_per_dst=[9264, 9310, 9302, 9260, 9064, 9280, 9404, 9448] dup_rows_per_dst=[3052, 3006, 3054, 2938, 3014, 2916, 3092, 2900] combine_rows_per_src=[9264, 9310, 9302, 9260, 9064, 9280, 9404, 9448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266747904 dispatch_meta_bytes=343400 combine_payload_bytes=266747904 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267091304 ingress_bytes=335953920 link_bytes=335953920 token_bytes=533839208 weight_movement_bytes=69206016 total_nvlink_bytes=603045224 weight_to_token_ratio=0.129638 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264167554e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65006 dup_rows=23944 dup_rows_remote=20950 dedup_ratio=0.243571 router_tokens_per_expert=[801, 757, 797, 786, 754, 741, 769, 812, 727, 736, 752, 729, 750, 767, 755, 787, 774, 767, 768, 780, 772, 784, 748, 772, 771, 763, 796, 765, 732, 775, 770, 802, 749, 794, 709, 789, 745, 787, 753, 753, 811, 742, 779, 753, 771, 758, 789, 748, 747, 741, 790, 782, 765, 769, 772, 792, 784, 821, 748, 786, 751, 761, 757, 797] payload_rows_per_dst=[9282, 9138, 9236, 9354, 9244, 9368, 9360, 9378] dup_rows_per_dst=[3152, 2868, 3094, 2994, 2914, 2934, 2956, 3032] combine_rows_per_src=[9282, 9138, 9236, 9354, 9244, 9368, 9360, 9378] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74120 payload_rows_remote=64776 dup_rows=24184 dup_rows_remote=21108 dedup_ratio=0.246012 router_tokens_per_expert=[745, 765, 778, 753, 783, 747, 726, 773, 752, 797, 783, 769, 787, 753, 735, 759, 777, 755, 779, 799, 812, 763, 759, 766, 742, 805, 754, 741, 781, 786, 756, 806, 732, 741, 753, 744, 767, 781, 774, 822, 789, 780, 753, 764, 735, 795, 728, 751, 787, 701, 803, 789, 779, 791, 750, 743, 801, 820, 772, 758, 769, 776, 764, 754] payload_rows_per_dst=[9196, 9218, 9344, 9252, 9206, 9226, 9320, 9358] dup_rows_per_dst=[2944, 3052, 3076, 3090, 3022, 2964, 2966, 3070] combine_rows_per_src=[9196, 9218, 9344, 9252, 9206, 9226, 9320, 9358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266182656 dispatch_meta_bytes=344488 combine_payload_bytes=266182656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266527144 ingress_bytes=300785664 link_bytes=300785664 token_bytes=532709800 weight_movement_bytes=34603008 total_nvlink_bytes=567312808 weight_to_token_ratio=0.064957 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131832238e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266051584 dispatch_meta_bytes=344128 combine_payload_bytes=266051584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266395712 ingress_bytes=300654592 link_bytes=300654592 token_bytes=532447296 weight_movement_bytes=34603008 total_nvlink_bytes=567050304 weight_to_token_ratio=0.064989 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131339025e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266797056 dispatch_meta_bytes=345632 combine_payload_bytes=266797056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267142688 ingress_bytes=301400064 link_bytes=301400064 token_bytes=533939744 weight_movement_bytes=34603008 total_nvlink_bytes=568542752 weight_to_token_ratio=0.064807 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134144176e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265420800 dispatch_meta_bytes=343784 combine_payload_bytes=265420800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265764584 ingress_bytes=300023808 link_bytes=300023808 token_bytes=531185384 weight_movement_bytes=34603008 total_nvlink_bytes=565788392 weight_to_token_ratio=0.065143 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128965436e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=343912 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266436456 ingress_bytes=369901568 link_bytes=369901568 token_bytes=532529000 weight_movement_bytes=103809024 total_nvlink_bytes=636338024 weight_to_token_ratio=0.194936 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391909821e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74166 payload_rows_remote=64872 dup_rows=24138 dup_rows_remote=21054 dedup_ratio=0.245544 router_tokens_per_expert=[776, 760, 779, 804, 694, 785, 746, 820, 786, 745, 765, 764, 802, 736, 751, 791, 736, 795, 758, 795, 786, 810, 787, 827, 722, 734, 826, 736, 752, 766, 756, 733, 800, 795, 749, 770, 730, 801, 780, 764, 773, 749, 730, 784, 734, 763, 787, 724, 730, 789, 804, 722, 731, 795, 809, 754, 766, 788, 755, 775, 747, 768, 788, 775] payload_rows_per_dst=[9282, 9238, 9492, 9190, 9294, 9244, 9246, 9180] dup_rows_per_dst=[3046, 3042, 3096, 2860, 3084, 2844, 3022, 3144] combine_rows_per_src=[9282, 9238, 9492, 9190, 9294, 9244, 9246, 9180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266330112 dispatch_meta_bytes=343952 combine_payload_bytes=266330112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266674064 ingress_bytes=300933120 link_bytes=300933120 token_bytes=533004176 weight_movement_bytes=34603008 total_nvlink_bytes=567607184 weight_to_token_ratio=0.064921 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132387103e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266264576 dispatch_meta_bytes=343824 combine_payload_bytes=266264576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266608400 ingress_bytes=300867584 link_bytes=300867584 token_bytes=532872976 weight_movement_bytes=34603008 total_nvlink_bytes=567475984 weight_to_token_ratio=0.064937 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132140497e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265322496 dispatch_meta_bytes=343536 combine_payload_bytes=265322496 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265666032 ingress_bytes=334528512 link_bytes=334528512 token_bytes=530988528 weight_movement_bytes=69206016 total_nvlink_bytes=600194544 weight_to_token_ratio=0.130334 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258803859e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74472 payload_rows_remote=65082 dup_rows=23832 dup_rows_remote=20870 dedup_ratio=0.242432 router_tokens_per_expert=[766, 783, 747, 768, 758, 747, 808, 791, 743, 755, 771, 778, 738, 778, 732, 828, 761, 768, 742, 759, 732, 788, 795, 741, 797, 764, 746, 745, 762, 775, 712, 775, 762, 787, 792, 810, 750, 771, 740, 776, 773, 832, 786, 735, 775, 760, 779, 736, 875, 773, 732, 764, 739, 745, 748, 814, 791, 826, 793, 710, 782, 723, 775, 745] payload_rows_per_dst=[9392, 9320, 9208, 9224, 9362, 9390, 9308, 9268] dup_rows_per_dst=[2944, 2926, 2964, 2928, 3014, 2962, 3072, 3022] combine_rows_per_src=[9392, 9320, 9208, 9224, 9362, 9390, 9308, 9268] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74486 payload_rows_remote=65364 dup_rows=23818 dup_rows_remote=20956 dedup_ratio=0.242289 router_tokens_per_expert=[815, 777, 766, 765, 790, 766, 770, 784, 807, 783, 744, 747, 767, 805, 781, 761, 768, 764, 771, 763, 767, 761, 761, 731, 768, 791, 741, 761, 797, 780, 762, 792, 815, 731, 770, 761, 734, 805, 775, 805, 723, 734, 790, 764, 700, 775, 763, 743, 734, 780, 761, 743, 754, 761, 776, 777, 785, 786, 810, 777, 784, 717, 771, 742] payload_rows_per_dst=[9452, 9302, 9292, 9400, 9402, 9122, 9174, 9342] dup_rows_per_dst=[3014, 3088, 2880, 2984, 2990, 2862, 2998, 3002] combine_rows_per_src=[9452, 9302, 9292, 9400, 9402, 9122, 9174, 9342] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=65050 dup_rows=24060 dup_rows_remote=21114 dedup_ratio=0.244751 router_tokens_per_expert=[781, 748, 782, 763, 772, 783, 804, 810, 797, 716, 742, 713, 811, 740, 758, 759, 744, 774, 739, 715, 792, 773, 740, 757, 778, 764, 758, 719, 760, 767, 788, 789, 800, 790, 767, 789, 811, 799, 759, 743, 748, 769, 760, 794, 787, 739, 762, 771, 777, 748, 754, 769, 762, 744, 772, 744, 817, 766, 745, 766, 794, 819, 784, 767] payload_rows_per_dst=[9340, 9166, 9206, 9304, 9396, 9228, 9194, 9410] dup_rows_per_dst=[3146, 2906, 2862, 2942, 3120, 3032, 2946, 3106] combine_rows_per_src=[9340, 9166, 9206, 9304, 9396, 9228, 9194, 9410] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265715712 dispatch_meta_bytes=343704 combine_payload_bytes=265715712 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266059416 ingress_bytes=334921728 link_bytes=334921728 token_bytes=531775128 weight_movement_bytes=69206016 total_nvlink_bytes=600981144 weight_to_token_ratio=0.130142 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260283499e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266575872 dispatch_meta_bytes=343808 combine_payload_bytes=266575872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266919680 ingress_bytes=301178880 link_bytes=301178880 token_bytes=533495552 weight_movement_bytes=34603008 total_nvlink_bytes=568098560 weight_to_token_ratio=0.064861 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133311879e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65006 dup_rows=23944 dup_rows_remote=20950 dedup_ratio=0.243571 router_tokens_per_expert=[801, 757, 797, 786, 754, 741, 769, 812, 727, 736, 752, 729, 750, 767, 755, 787, 774, 767, 768, 780, 772, 784, 748, 772, 771, 763, 796, 765, 732, 775, 770, 802, 749, 794, 709, 789, 745, 787, 753, 753, 811, 742, 779, 753, 771, 758, 789, 748, 747, 741, 790, 782, 765, 769, 772, 792, 784, 821, 748, 786, 751, 761, 757, 797] payload_rows_per_dst=[9282, 9138, 9236, 9354, 9244, 9368, 9360, 9378] dup_rows_per_dst=[3152, 2868, 3094, 2994, 2914, 2934, 2956, 3032] combine_rows_per_src=[9282, 9138, 9236, 9354, 9244, 9368, 9360, 9378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74166 payload_rows_remote=64872 dup_rows=24138 dup_rows_remote=21054 dedup_ratio=0.245544 router_tokens_per_expert=[776, 760, 779, 804, 694, 785, 746, 820, 786, 745, 765, 764, 802, 736, 751, 791, 736, 795, 758, 795, 786, 810, 787, 827, 722, 734, 826, 736, 752, 766, 756, 733, 800, 795, 749, 770, 730, 801, 780, 764, 773, 749, 730, 784, 734, 763, 787, 724, 730, 789, 804, 722, 731, 795, 809, 754, 766, 788, 755, 775, 747, 768, 788, 775] payload_rows_per_dst=[9282, 9238, 9492, 9190, 9294, 9244, 9246, 9180] dup_rows_per_dst=[3046, 3042, 3096, 2860, 3084, 2844, 3022, 3144] combine_rows_per_src=[9282, 9238, 9492, 9190, 9294, 9244, 9246, 9180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=65086 dup_rows=23992 dup_rows_remote=20908 dedup_ratio=0.244059 router_tokens_per_expert=[782, 742, 758, 727, 734, 797, 766, 758, 787, 774, 746, 753, 728, 777, 783, 772, 762, 752, 802, 728, 792, 734, 761, 811, 768, 811, 815, 750, 775, 741, 738, 791, 759, 760, 790, 771, 785, 748, 750, 750, 718, 794, 762, 753, 760, 794, 791, 781, 728, 785, 808, 807, 780, 766, 752, 808, 763, 776, 754, 750, 750, 786, 824, 734] payload_rows_per_dst=[8870, 9320, 9642, 9250, 9144, 9366, 9494, 9226] dup_rows_per_dst=[2566, 2920, 3334, 3092, 2840, 2940, 3216, 3084] combine_rows_per_src=[8870, 9320, 9642, 9250, 9144, 9366, 9494, 9226] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74332 payload_rows_remote=65022 dup_rows=23972 dup_rows_remote=20966 dedup_ratio=0.243856 router_tokens_per_expert=[755, 789, 749, 770, 783, 780, 754, 778, 777, 749, 813, 750, 766, 795, 736, 772, 786, 794, 759, 765, 779, 810, 729, 756, 797, 719, 776, 761, 749, 768, 734, 795, 762, 788, 748, 747, 741, 771, 727, 755, 749, 770, 756, 743, 786, 760, 756, 778, 784, 794, 790, 745, 766, 809, 787, 773, 802, 760, 800, 717, 761, 748, 810, 776] payload_rows_per_dst=[9264, 9310, 9302, 9260, 9064, 9280, 9404, 9448] dup_rows_per_dst=[3052, 3006, 3054, 2938, 3014, 2916, 3092, 2900] combine_rows_per_src=[9264, 9310, 9302, 9260, 9064, 9280, 9404, 9448] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64816 dup_rows=24112 dup_rows_remote=20840 dedup_ratio=0.245280 router_tokens_per_expert=[760, 744, 735, 804, 763, 742, 787, 772, 828, 758, 761, 776, 728, 782, 779, 740, 836, 762, 815, 737, 746, 740, 747, 754, 741, 741, 764, 775, 756, 783, 766, 788, 727, 758, 788, 740, 793, 810, 744, 735, 765, 801, 767, 776, 802, 751, 794, 778, 773, 800, 753, 799, 791, 746, 742, 806, 756, 785, 737, 770, 735, 795, 797, 728] payload_rows_per_dst=[9376, 9480, 9398, 9518, 9212, 8906, 9360, 8942] dup_rows_per_dst=[3272, 3200, 2966, 3112, 2998, 2756, 3036, 2772] combine_rows_per_src=[9376, 9480, 9398, 9518, 9212, 8906, 9360, 8942] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74536 payload_rows_remote=65076 dup_rows=23768 dup_rows_remote=20656 dedup_ratio=0.241781 router_tokens_per_expert=[807, 811, 759, 743, 750, 793, 773, 753, 771, 763, 761, 774, 794, 787, 756, 782, 786, 769, 708, 750, 757, 709, 793, 788, 729, 747, 749, 800, 780, 714, 760, 808, 726, 762, 755, 774, 774, 743, 821, 773, 740, 797, 750, 811, 757, 758, 727, 782, 780, 772, 794, 747, 778, 761, 722, 809, 780, 759, 771, 782, 774, 803, 745, 801] payload_rows_per_dst=[9460, 9442, 9290, 9124, 9274, 9610, 9150, 9186] dup_rows_per_dst=[3112, 3236, 2830, 2860, 2978, 3146, 2664, 2942] combine_rows_per_src=[9460, 9442, 9290, 9124, 9274, 9610, 9150, 9186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267730944 dispatch_meta_bytes=345280 combine_payload_bytes=267730944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268076224 ingress_bytes=302333952 link_bytes=302333952 token_bytes=535807168 weight_movement_bytes=34603008 total_nvlink_bytes=570410176 weight_to_token_ratio=0.064581 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137658321e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266444800 dispatch_meta_bytes=344656 combine_payload_bytes=266444800 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266789456 ingress_bytes=404856832 link_bytes=404856832 token_bytes=533234256 weight_movement_bytes=138412032 total_nvlink_bytes=671646288 weight_to_token_ratio=0.259571 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523443665e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74482 payload_rows_remote=65124 dup_rows=23822 dup_rows_remote=20726 dedup_ratio=0.242330 router_tokens_per_expert=[771, 776, 746, 792, 740, 805, 767, 787, 732, 813, 754, 742, 773, 732, 760, 732, 789, 739, 779, 787, 756, 733, 733, 795, 749, 768, 785, 764, 746, 779, 767, 829, 789, 780, 750, 813, 726, 758, 765, 790, 761, 737, 771, 749, 791, 732, 730, 776, 783, 758, 808, 784, 801, 777, 779, 817, 802, 744, 742, 769, 808, 753, 745, 744] payload_rows_per_dst=[9348, 9230, 9284, 9314, 9254, 9166, 9528, 9358] dup_rows_per_dst=[3020, 2846, 2908, 3060, 2876, 2928, 3088, 3096] combine_rows_per_src=[9348, 9230, 9284, 9314, 9254, 9166, 9528, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265715712 dispatch_meta_bytes=343704 combine_payload_bytes=265715712 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266059416 ingress_bytes=334921728 link_bytes=334921728 token_bytes=531775128 weight_movement_bytes=69206016 total_nvlink_bytes=600981144 weight_to_token_ratio=0.130142 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260283499e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266592256 dispatch_meta_bytes=343976 combine_payload_bytes=266592256 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266936232 ingress_bytes=335798272 link_bytes=335798272 token_bytes=533528488 weight_movement_bytes=69206016 total_nvlink_bytes=602734504 weight_to_token_ratio=0.129714 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263581864e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74370 payload_rows_remote=65116 dup_rows=23934 dup_rows_remote=20916 dedup_ratio=0.243469 router_tokens_per_expert=[777, 758, 728, 736, 788, 774, 810, 796, 740, 746, 747, 808, 812, 773, 743, 767, 796, 762, 778, 759, 739, 734, 711, 752, 807, 785, 762, 777, 824, 796, 727, 766, 763, 793, 775, 752, 773, 780, 781, 808, 776, 774, 758, 769, 727, 786, 780, 774, 734, 748, 776, 801, 795, 745, 749, 751, 705, 776, 770, 806, 795, 750, 790, 714] payload_rows_per_dst=[9372, 9254, 9204, 9360, 9332, 9350, 9228, 9270] dup_rows_per_dst=[2962, 3018, 2858, 3128, 3118, 2938, 2970, 2942] combine_rows_per_src=[9372, 9254, 9204, 9360, 9332, 9350, 9228, 9270] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=64828 dup_rows=24062 dup_rows_remote=20938 dedup_ratio=0.244771 router_tokens_per_expert=[817, 760, 789, 813, 758, 773, 714, 784, 792, 743, 757, 781, 740, 792, 767, 747, 750, 767, 722, 712, 740, 757, 713, 786, 725, 795, 821, 790, 784, 747, 842, 765, 774, 736, 770, 802, 748, 757, 753, 794, 748, 768, 753, 789, 741, 723, 781, 794, 765, 763, 744, 783, 769, 766, 792, 756, 732, 801, 757, 800, 791, 778, 819, 762] payload_rows_per_dst=[9292, 9230, 9102, 9414, 9274, 9236, 9250, 9444] dup_rows_per_dst=[3124, 3008, 2792, 3124, 2994, 2958, 3026, 3036] combine_rows_per_src=[9292, 9230, 9102, 9414, 9274, 9236, 9250, 9444] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74410 payload_rows_remote=64940 dup_rows=23894 dup_rows_remote=20806 dedup_ratio=0.243062 router_tokens_per_expert=[760, 737, 739, 778, 775, 791, 785, 812, 793, 801, 746, 735, 780, 734, 739, 750, 812, 772, 767, 781, 762, 809, 801, 775, 781, 724, 752, 739, 790, 726, 798, 735, 760, 813, 766, 730, 819, 754, 764, 777, 774, 697, 763, 801, 784, 774, 806, 736, 738, 751, 803, 757, 751, 783, 739, 781, 724, 794, 803, 729, 826, 733, 778, 765] payload_rows_per_dst=[9260, 9162, 9470, 9220, 9324, 9304, 9364, 9306] dup_rows_per_dst=[3094, 2994, 3088, 2870, 3042, 2966, 2842, 2998] combine_rows_per_src=[9260, 9162, 9470, 9220, 9324, 9304, 9364, 9306] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266330112 dispatch_meta_bytes=343952 combine_payload_bytes=266330112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266674064 ingress_bytes=300933120 link_bytes=300933120 token_bytes=533004176 weight_movement_bytes=34603008 total_nvlink_bytes=567607184 weight_to_token_ratio=0.064921 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132387103e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266264576 dispatch_meta_bytes=343824 combine_payload_bytes=266264576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266608400 ingress_bytes=300867584 link_bytes=300867584 token_bytes=532872976 weight_movement_bytes=34603008 total_nvlink_bytes=567475984 weight_to_token_ratio=0.064937 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132140497e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266551296 dispatch_meta_bytes=342928 combine_payload_bytes=266551296 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266894224 ingress_bytes=335757312 link_bytes=335757312 token_bytes=533445520 weight_movement_bytes=69206016 total_nvlink_bytes=602651536 weight_to_token_ratio=0.129734 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263427734e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265486336 dispatch_meta_bytes=342624 combine_payload_bytes=265486336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265828960 ingress_bytes=300089344 link_bytes=300089344 token_bytes=531315296 weight_movement_bytes=34603008 total_nvlink_bytes=565918304 weight_to_token_ratio=0.065127 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129212042e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=64966 dup_rows=24098 dup_rows_remote=21056 dedup_ratio=0.245138 router_tokens_per_expert=[755, 785, 784, 784, 732, 811, 770, 799, 772, 782, 776, 766, 740, 774, 770, 775, 769, 794, 752, 780, 772, 786, 741, 778, 742, 763, 768, 693, 771, 792, 763, 749, 740, 786, 797, 767, 768, 755, 750, 778, 754, 718, 772, 770, 774, 764, 735, 779, 746, 791, 781, 766, 810, 810, 792, 750, 762, 783, 740, 793, 772, 721, 724, 816] payload_rows_per_dst=[9340, 9340, 9374, 9118, 9240, 9194, 9374, 9226] dup_rows_per_dst=[3100, 2970, 2970, 2964, 3042, 2938, 3118, 2996] combine_rows_per_src=[9340, 9340, 9374, 9118, 9240, 9194, 9374, 9226] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266747904 dispatch_meta_bytes=343400 combine_payload_bytes=266747904 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267091304 ingress_bytes=335953920 link_bytes=335953920 token_bytes=533839208 weight_movement_bytes=69206016 total_nvlink_bytes=603045224 weight_to_token_ratio=0.129638 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264167554e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74486 payload_rows_remote=65364 dup_rows=23818 dup_rows_remote=20956 dedup_ratio=0.242289 router_tokens_per_expert=[815, 777, 766, 765, 790, 766, 770, 784, 807, 783, 744, 747, 767, 805, 781, 761, 768, 764, 771, 763, 767, 761, 761, 731, 768, 791, 741, 761, 797, 780, 762, 792, 815, 731, 770, 761, 734, 805, 775, 805, 723, 734, 790, 764, 700, 775, 763, 743, 734, 780, 761, 743, 754, 761, 776, 777, 785, 786, 810, 777, 784, 717, 771, 742] payload_rows_per_dst=[9452, 9302, 9292, 9400, 9402, 9122, 9174, 9342] dup_rows_per_dst=[3014, 3088, 2880, 2984, 2990, 2862, 2998, 3002] combine_rows_per_src=[9452, 9302, 9292, 9400, 9402, 9122, 9174, 9342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266715136 dispatch_meta_bytes=344128 combine_payload_bytes=266715136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267059264 ingress_bytes=301318144 link_bytes=301318144 token_bytes=533774400 weight_movement_bytes=34603008 total_nvlink_bytes=568377408 weight_to_token_ratio=0.064827 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133835918e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265535488 dispatch_meta_bytes=343064 combine_payload_bytes=265535488 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265878552 ingress_bytes=369344512 link_bytes=369344512 token_bytes=531414040 weight_movement_bytes=103809024 total_nvlink_bytes=635223064 weight_to_token_ratio=0.195345 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389813664e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=342984 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266337224 ingress_bytes=300597248 link_bytes=300597248 token_bytes=532331464 weight_movement_bytes=34603008 total_nvlink_bytes=566934472 weight_to_token_ratio=0.065003 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131123244e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74656 payload_rows_remote=65260 dup_rows=23648 dup_rows_remote=20748 dedup_ratio=0.240560 router_tokens_per_expert=[781, 760, 746, 750, 762, 792, 761, 776, 765, 802, 733, 760, 764, 762, 707, 758, 749, 745, 740, 789, 779, 791, 756, 769, 800, 757, 795, 804, 786, 749, 758, 718, 800, 791, 761, 832, 817, 764, 778, 776, 759, 790, 766, 764, 736, 765, 772, 796, 785, 737, 747, 763, 771, 799, 795, 771, 767, 744, 770, 746, 791, 747, 769, 719] payload_rows_per_dst=[9294, 9194, 9230, 9388, 9514, 9396, 9314, 9326] dup_rows_per_dst=[2962, 2908, 3006, 2946, 3124, 2900, 3022, 2780] combine_rows_per_src=[9294, 9194, 9230, 9388, 9514, 9396, 9314, 9326] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64826 dup_rows=24068 dup_rows_remote=21042 dedup_ratio=0.244832 router_tokens_per_expert=[821, 712, 726, 800, 716, 789, 772, 807, 757, 739, 790, 719, 734, 724, 796, 751, 776, 755, 743, 769, 764, 767, 733, 782, 839, 764, 756, 773, 751, 761, 784, 770, 753, 777, 771, 786, 746, 765, 786, 771, 763, 728, 823, 774, 768, 715, 800, 816, 807, 759, 779, 766, 745, 747, 790, 825, 728, 766, 794, 795, 772, 770, 770, 757] payload_rows_per_dst=[9312, 9174, 9218, 9362, 9298, 9206, 9410, 9256] dup_rows_per_dst=[2974, 2846, 2960, 3034, 3012, 3168, 3026, 3048] combine_rows_per_src=[9312, 9174, 9218, 9362, 9298, 9206, 9410, 9256] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266100736 dispatch_meta_bytes=344088 combine_payload_bytes=266100736 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266444824 ingress_bytes=335306752 link_bytes=335306752 token_bytes=532545560 weight_movement_bytes=69206016 total_nvlink_bytes=601751576 weight_to_token_ratio=0.129953 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261732313e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267730944 dispatch_meta_bytes=345280 combine_payload_bytes=267730944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268076224 ingress_bytes=302333952 link_bytes=302333952 token_bytes=535807168 weight_movement_bytes=34603008 total_nvlink_bytes=570410176 weight_to_token_ratio=0.064581 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137658321e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=64966 dup_rows=24098 dup_rows_remote=21056 dedup_ratio=0.245138 router_tokens_per_expert=[755, 785, 784, 784, 732, 811, 770, 799, 772, 782, 776, 766, 740, 774, 770, 775, 769, 794, 752, 780, 772, 786, 741, 778, 742, 763, 768, 693, 771, 792, 763, 749, 740, 786, 797, 767, 768, 755, 750, 778, 754, 718, 772, 770, 774, 764, 735, 779, 746, 791, 781, 766, 810, 810, 792, 750, 762, 783, 740, 793, 772, 721, 724, 816] payload_rows_per_dst=[9340, 9340, 9374, 9118, 9240, 9194, 9374, 9226] dup_rows_per_dst=[3100, 2970, 2970, 2964, 3042, 2938, 3118, 2996] combine_rows_per_src=[9340, 9340, 9374, 9118, 9240, 9194, 9374, 9226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74182 payload_rows_remote=64700 dup_rows=24122 dup_rows_remote=20840 dedup_ratio=0.245382 router_tokens_per_expert=[787, 786, 777, 760, 741, 779, 801, 715, 763, 777, 798, 767, 757, 772, 794, 768, 761, 758, 727, 847, 786, 765, 761, 780, 772, 751, 774, 783, 763, 815, 754, 795, 750, 767, 746, 723, 789, 836, 767, 788, 753, 739, 731, 810, 744, 746, 735, 782, 730, 745, 775, 774, 730, 796, 785, 723, 758, 777, 764, 794, 773, 790, 742, 756] payload_rows_per_dst=[9288, 9186, 9252, 9338, 9146, 9216, 9274, 9482] dup_rows_per_dst=[3004, 3012, 3048, 3060, 2924, 2882, 2910, 3282] combine_rows_per_src=[9288, 9186, 9252, 9338, 9146, 9216, 9274, 9482] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=64828 dup_rows=24062 dup_rows_remote=20938 dedup_ratio=0.244771 router_tokens_per_expert=[817, 760, 789, 813, 758, 773, 714, 784, 792, 743, 757, 781, 740, 792, 767, 747, 750, 767, 722, 712, 740, 757, 713, 786, 725, 795, 821, 790, 784, 747, 842, 765, 774, 736, 770, 802, 748, 757, 753, 794, 748, 768, 753, 789, 741, 723, 781, 794, 765, 763, 744, 783, 769, 766, 792, 756, 732, 801, 757, 800, 791, 778, 819, 762] payload_rows_per_dst=[9292, 9230, 9102, 9414, 9274, 9236, 9250, 9444] dup_rows_per_dst=[3124, 3008, 2792, 3124, 2994, 2958, 3026, 3036] combine_rows_per_src=[9292, 9230, 9102, 9414, 9274, 9236, 9250, 9444] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74370 payload_rows_remote=65116 dup_rows=23934 dup_rows_remote=20916 dedup_ratio=0.243469 router_tokens_per_expert=[777, 758, 728, 736, 788, 774, 810, 796, 740, 746, 747, 808, 812, 773, 743, 767, 796, 762, 778, 759, 739, 734, 711, 752, 807, 785, 762, 777, 824, 796, 727, 766, 763, 793, 775, 752, 773, 780, 781, 808, 776, 774, 758, 769, 727, 786, 780, 774, 734, 748, 776, 801, 795, 745, 749, 751, 705, 776, 770, 806, 795, 750, 790, 714] payload_rows_per_dst=[9372, 9254, 9204, 9360, 9332, 9350, 9228, 9270] dup_rows_per_dst=[2962, 3018, 2858, 3128, 3118, 2938, 2970, 2942] combine_rows_per_src=[9372, 9254, 9204, 9360, 9332, 9350, 9228, 9270] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=64956 dup_rows=23940 dup_rows_remote=20828 dedup_ratio=0.243530 router_tokens_per_expert=[770, 768, 755, 735, 798, 783, 742, 747, 802, 805, 741, 788, 757, 795, 759, 750, 729, 767, 752, 764, 802, 724, 797, 759, 765, 774, 795, 800, 759, 781, 761, 816, 797, 777, 739, 809, 785, 759, 815, 742, 789, 769, 711, 808, 742, 791, 725, 801, 802, 772, 750, 737, 751, 779, 738, 789, 739, 687, 764, 793, 783, 753, 757, 759] payload_rows_per_dst=[9408, 9324, 9292, 9334, 9376, 9292, 9108, 9230] dup_rows_per_dst=[3112, 3114, 2896, 3124, 3070, 2980, 2804, 2840] combine_rows_per_src=[9408, 9324, 9292, 9334, 9376, 9292, 9108, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=267304960 dispatch_meta_bytes=344032 combine_payload_bytes=267304960 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267648992 ingress_bytes=301907968 link_bytes=301907968 token_bytes=534953952 weight_movement_bytes=34603008 total_nvlink_bytes=569556960 weight_to_token_ratio=0.064684 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136055378e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265527296 dispatch_meta_bytes=343472 combine_payload_bytes=265527296 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265870768 ingress_bytes=334733312 link_bytes=334733312 token_bytes=531398064 weight_movement_bytes=69206016 total_nvlink_bytes=600604080 weight_to_token_ratio=0.130234 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259574505e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74164 payload_rows_remote=65136 dup_rows=24140 dup_rows_remote=21272 dedup_ratio=0.245565 router_tokens_per_expert=[789, 698, 710, 752, 746, 745, 736, 749, 776, 816, 761, 768, 759, 735, 811, 771, 751, 745, 788, 762, 755, 774, 773, 741, 780, 780, 762, 791, 770, 766, 786, 790, 804, 830, 723, 744, 794, 794, 741, 790, 750, 749, 760, 760, 766, 772, 745, 772, 834, 768, 787, 767, 802, 820, 771, 784, 756, 771, 769, 781, 727, 755, 754, 776] payload_rows_per_dst=[9028, 9652, 9242, 9464, 9412, 9562, 8916, 8888] dup_rows_per_dst=[2868, 3386, 2936, 3130, 3028, 3260, 2886, 2646] combine_rows_per_src=[9028, 9652, 9242, 9464, 9412, 9562, 8916, 8888] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=65086 dup_rows=23992 dup_rows_remote=20908 dedup_ratio=0.244059 router_tokens_per_expert=[782, 742, 758, 727, 734, 797, 766, 758, 787, 774, 746, 753, 728, 777, 783, 772, 762, 752, 802, 728, 792, 734, 761, 811, 768, 811, 815, 750, 775, 741, 738, 791, 759, 760, 790, 771, 785, 748, 750, 750, 718, 794, 762, 753, 760, 794, 791, 781, 728, 785, 808, 807, 780, 766, 752, 808, 763, 776, 754, 750, 750, 786, 824, 734] payload_rows_per_dst=[8870, 9320, 9642, 9250, 9144, 9366, 9494, 9226] dup_rows_per_dst=[2566, 2920, 3334, 3092, 2840, 2940, 3216, 3084] combine_rows_per_src=[8870, 9320, 9642, 9250, 9144, 9366, 9494, 9226] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266100736 dispatch_meta_bytes=344088 combine_payload_bytes=266100736 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266444824 ingress_bytes=335306752 link_bytes=335306752 token_bytes=532545560 weight_movement_bytes=69206016 total_nvlink_bytes=601751576 weight_to_token_ratio=0.129953 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261732313e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74386 payload_rows_remote=65004 dup_rows=23918 dup_rows_remote=20774 dedup_ratio=0.243306 router_tokens_per_expert=[756, 747, 768, 786, 807, 754, 766, 787, 815, 783, 803, 741, 768, 740, 822, 791, 744, 765, 779, 774, 735, 772, 726, 734, 737, 743, 762, 744, 809, 754, 773, 757, 758, 753, 770, 741, 721, 737, 779, 801, 785, 785, 773, 792, 830, 796, 744, 740, 762, 764, 744, 766, 788, 736, 760, 772, 743, 809, 777, 756, 816, 773, 751, 788] payload_rows_per_dst=[9272, 9382, 9214, 9226, 9244, 9384, 9232, 9432] dup_rows_per_dst=[3070, 3144, 2844, 2932, 2876, 3106, 2952, 2994] combine_rows_per_src=[9272, 9382, 9214, 9226, 9244, 9384, 9232, 9432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265011200 dispatch_meta_bytes=342160 combine_payload_bytes=265011200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265353360 ingress_bytes=299614208 link_bytes=299614208 token_bytes=530364560 weight_movement_bytes=34603008 total_nvlink_bytes=564967568 weight_to_token_ratio=0.065244 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127424144e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74214 payload_rows_remote=65076 dup_rows=24090 dup_rows_remote=21166 dedup_ratio=0.245056 router_tokens_per_expert=[715, 811, 745, 783, 730, 758, 755, 800, 776, 765, 789, 773, 757, 762, 775, 739, 749, 769, 792, 797, 780, 770, 804, 767, 712, 798, 755, 780, 735, 744, 737, 770, 781, 775, 750, 725, 789, 745, 773, 756, 793, 759, 771, 786, 784, 753, 793, 766, 820, 719, 787, 769, 775, 716, 809, 787, 761, 776, 766, 794, 746, 771, 800, 765] payload_rows_per_dst=[9198, 9284, 9408, 9138, 9182, 9374, 9342, 9288] dup_rows_per_dst=[2996, 2988, 3048, 2924, 3006, 3036, 3022, 3070] combine_rows_per_src=[9198, 9284, 9408, 9138, 9182, 9374, 9342, 9288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=343136 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266402912 ingress_bytes=300662784 link_bytes=300662784 token_bytes=532462688 weight_movement_bytes=34603008 total_nvlink_bytes=567065696 weight_to_token_ratio=0.064987 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131369851e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74436 payload_rows_remote=65142 dup_rows=23868 dup_rows_remote=20966 dedup_ratio=0.242798 router_tokens_per_expert=[766, 791, 752, 796, 766, 771, 733, 768, 752, 733, 785, 772, 751, 787, 832, 760, 834, 716, 779, 741, 717, 797, 767, 747, 761, 782, 737, 762, 744, 797, 762, 750, 764, 756, 785, 754, 742, 799, 761, 818, 777, 801, 772, 769, 815, 740, 769, 734, 790, 756, 791, 790, 754, 762, 769, 761, 729, 787, 785, 753, 724, 771, 745, 821] payload_rows_per_dst=[9342, 9346, 9294, 9230, 9300, 9300, 9342, 9282] dup_rows_per_dst=[2944, 2998, 2902, 2960, 3058, 3054, 3004, 2948] combine_rows_per_src=[9342, 9346, 9294, 9230, 9300, 9300, 9342, 9282] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265535488 dispatch_meta_bytes=343064 combine_payload_bytes=265535488 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265878552 ingress_bytes=369344512 link_bytes=369344512 token_bytes=531414040 weight_movement_bytes=103809024 total_nvlink_bytes=635223064 weight_to_token_ratio=0.195345 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389813664e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266715136 dispatch_meta_bytes=344128 combine_payload_bytes=266715136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267059264 ingress_bytes=301318144 link_bytes=301318144 token_bytes=533774400 weight_movement_bytes=34603008 total_nvlink_bytes=568377408 weight_to_token_ratio=0.064827 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133835918e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266797056 dispatch_meta_bytes=345632 combine_payload_bytes=266797056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267142688 ingress_bytes=301400064 link_bytes=301400064 token_bytes=533939744 weight_movement_bytes=34603008 total_nvlink_bytes=568542752 weight_to_token_ratio=0.064807 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134144176e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266592256 dispatch_meta_bytes=343976 combine_payload_bytes=266592256 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266936232 ingress_bytes=335798272 link_bytes=335798272 token_bytes=533528488 weight_movement_bytes=69206016 total_nvlink_bytes=602734504 weight_to_token_ratio=0.129714 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263581864e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74656 payload_rows_remote=65260 dup_rows=23648 dup_rows_remote=20748 dedup_ratio=0.240560 router_tokens_per_expert=[781, 760, 746, 750, 762, 792, 761, 776, 765, 802, 733, 760, 764, 762, 707, 758, 749, 745, 740, 789, 779, 791, 756, 769, 800, 757, 795, 804, 786, 749, 758, 718, 800, 791, 761, 832, 817, 764, 778, 776, 759, 790, 766, 764, 736, 765, 772, 796, 785, 737, 747, 763, 771, 799, 795, 771, 767, 744, 770, 746, 791, 747, 769, 719] payload_rows_per_dst=[9294, 9194, 9230, 9388, 9514, 9396, 9314, 9326] dup_rows_per_dst=[2962, 2908, 3006, 2946, 3124, 2900, 3022, 2780] combine_rows_per_src=[9294, 9194, 9230, 9388, 9514, 9396, 9314, 9326] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74130 payload_rows_remote=64812 dup_rows=24174 dup_rows_remote=21036 dedup_ratio=0.245911 router_tokens_per_expert=[796, 785, 781, 764, 730, 720, 766, 732, 780, 753, 771, 764, 765, 789, 728, 780, 750, 831, 740, 772, 771, 809, 807, 763, 778, 780, 786, 759, 772, 756, 737, 731, 802, 791, 788, 728, 781, 789, 760, 789, 765, 787, 778, 751, 724, 809, 773, 783, 755, 793, 766, 750, 746, 774, 766, 805, 736, 747, 732, 770, 786, 784, 756, 742] payload_rows_per_dst=[9184, 9164, 9396, 9218, 9318, 9380, 9332, 9138] dup_rows_per_dst=[2964, 3096, 3090, 2980, 3138, 2960, 2978, 2968] combine_rows_per_src=[9184, 9164, 9396, 9218, 9318, 9380, 9332, 9138] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266256384 dispatch_meta_bytes=343112 combine_payload_bytes=266256384 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266599496 ingress_bytes=335462400 link_bytes=335462400 token_bytes=532855880 weight_movement_bytes=69206016 total_nvlink_bytes=602061896 weight_to_token_ratio=0.129878 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262318004e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266551296 dispatch_meta_bytes=344968 combine_payload_bytes=266551296 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266896264 ingress_bytes=370360320 link_bytes=370360320 token_bytes=533447560 weight_movement_bytes=103809024 total_nvlink_bytes=637256584 weight_to_token_ratio=0.194600 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393636068e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266821632 dispatch_meta_bytes=344432 combine_payload_bytes=266821632 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267166064 ingress_bytes=336027648 link_bytes=336027648 token_bytes=533987696 weight_movement_bytes=69206016 total_nvlink_bytes=603193712 weight_to_token_ratio=0.129602 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264444987e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74324 payload_rows_remote=64958 dup_rows=23980 dup_rows_remote=21078 dedup_ratio=0.243937 router_tokens_per_expert=[768, 775, 774, 759, 828, 746, 752, 804, 765, 789, 755, 780, 778, 782, 779, 762, 719, 740, 767, 830, 787, 748, 761, 747, 758, 770, 809, 766, 788, 745, 732, 781, 815, 769, 730, 778, 778, 766, 749, 739, 796, 758, 761, 761, 754, 748, 796, 760, 764, 776, 772, 799, 771, 746, 795, 727, 756, 808, 735, 782, 751, 765, 734, 769] payload_rows_per_dst=[9370, 9376, 9204, 9166, 9316, 9366, 9310, 9216] dup_rows_per_dst=[3042, 3004, 2994, 3132, 2932, 2902, 2990, 2984] combine_rows_per_src=[9370, 9376, 9204, 9166, 9316, 9366, 9310, 9216] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74512 payload_rows_remote=65036 dup_rows=23792 dup_rows_remote=20600 dedup_ratio=0.242025 router_tokens_per_expert=[769, 760, 764, 766, 803, 760, 768, 743, 773, 753, 788, 771, 763, 747, 740, 747, 813, 798, 770, 726, 767, 769, 763, 786, 815, 772, 745, 759, 798, 793, 764, 807, 720, 738, 762, 790, 743, 773, 736, 752, 767, 772, 734, 772, 761, 751, 743, 722, 791, 798, 790, 821, 807, 786, 792, 749, 784, 796, 787, 790, 735, 702, 760, 768] payload_rows_per_dst=[9290, 9334, 9412, 9418, 9126, 9208, 9476, 9248] dup_rows_per_dst=[2976, 2830, 2972, 3088, 2902, 2836, 3192, 2996] combine_rows_per_src=[9290, 9334, 9412, 9418, 9126, 9208, 9476, 9248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=267304960 dispatch_meta_bytes=344032 combine_payload_bytes=267304960 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267648992 ingress_bytes=301907968 link_bytes=301907968 token_bytes=534953952 weight_movement_bytes=34603008 total_nvlink_bytes=569556960 weight_to_token_ratio=0.064684 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136055378e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265469952 dispatch_meta_bytes=343392 combine_payload_bytes=265469952 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265813344 ingress_bytes=334675968 link_bytes=334675968 token_bytes=531283296 weight_movement_bytes=69206016 total_nvlink_bytes=600489312 weight_to_token_ratio=0.130262 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259358724e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74214 payload_rows_remote=65076 dup_rows=24090 dup_rows_remote=21166 dedup_ratio=0.245056 router_tokens_per_expert=[715, 811, 745, 783, 730, 758, 755, 800, 776, 765, 789, 773, 757, 762, 775, 739, 749, 769, 792, 797, 780, 770, 804, 767, 712, 798, 755, 780, 735, 744, 737, 770, 781, 775, 750, 725, 789, 745, 773, 756, 793, 759, 771, 786, 784, 753, 793, 766, 820, 719, 787, 769, 775, 716, 809, 787, 761, 776, 766, 794, 746, 771, 800, 765] payload_rows_per_dst=[9198, 9284, 9408, 9138, 9182, 9374, 9342, 9288] dup_rows_per_dst=[2996, 2988, 3048, 2924, 3006, 3036, 3022, 3070] combine_rows_per_src=[9198, 9284, 9408, 9138, 9182, 9374, 9342, 9288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74130 payload_rows_remote=64812 dup_rows=24174 dup_rows_remote=21036 dedup_ratio=0.245911 router_tokens_per_expert=[796, 785, 781, 764, 730, 720, 766, 732, 780, 753, 771, 764, 765, 789, 728, 780, 750, 831, 740, 772, 771, 809, 807, 763, 778, 780, 786, 759, 772, 756, 737, 731, 802, 791, 788, 728, 781, 789, 760, 789, 765, 787, 778, 751, 724, 809, 773, 783, 755, 793, 766, 750, 746, 774, 766, 805, 736, 747, 732, 770, 786, 784, 756, 742] payload_rows_per_dst=[9184, 9164, 9396, 9218, 9318, 9380, 9332, 9138] dup_rows_per_dst=[2964, 3096, 3090, 2980, 3138, 2960, 2978, 2968] combine_rows_per_src=[9184, 9164, 9396, 9218, 9318, 9380, 9332, 9138] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74386 payload_rows_remote=65034 dup_rows=23918 dup_rows_remote=21022 dedup_ratio=0.243306 router_tokens_per_expert=[783, 756, 776, 724, 779, 787, 813, 777, 754, 785, 730, 771, 798, 771, 812, 739, 775, 818, 754, 703, 769, 833, 769, 768, 755, 724, 747, 770, 774, 773, 785, 766, 774, 812, 776, 735, 786, 801, 756, 727, 795, 759, 743, 766, 743, 798, 733, 778, 763, 756, 772, 748, 780, 767, 771, 766, 773, 760, 743, 780, 753, 768, 788, 744] payload_rows_per_dst=[9150, 9496, 9366, 8730, 9424, 9174, 9694, 9352] dup_rows_per_dst=[2888, 3176, 3012, 2616, 3076, 3026, 3228, 2896] combine_rows_per_src=[9150, 9496, 9366, 8730, 9424, 9174, 9694, 9352] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74386 payload_rows_remote=65004 dup_rows=23918 dup_rows_remote=20774 dedup_ratio=0.243306 router_tokens_per_expert=[756, 747, 768, 786, 807, 754, 766, 787, 815, 783, 803, 741, 768, 740, 822, 791, 744, 765, 779, 774, 735, 772, 726, 734, 737, 743, 762, 744, 809, 754, 773, 757, 758, 753, 770, 741, 721, 737, 779, 801, 785, 785, 773, 792, 830, 796, 744, 740, 762, 764, 744, 766, 788, 736, 760, 772, 743, 809, 777, 756, 816, 773, 751, 788] payload_rows_per_dst=[9272, 9382, 9214, 9226, 9244, 9384, 9232, 9432] dup_rows_per_dst=[3070, 3144, 2844, 2932, 2876, 3106, 2952, 2994] combine_rows_per_src=[9272, 9382, 9214, 9226, 9244, 9384, 9232, 9432] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=64944 dup_rows=23974 dup_rows_remote=20822 dedup_ratio=0.243876 router_tokens_per_expert=[735, 799, 798, 776, 735, 752, 785, 765, 698, 718, 764, 805, 743, 787, 748, 788, 740, 778, 787, 762, 749, 772, 763, 711, 806, 803, 835, 790, 795, 764, 748, 788, 799, 774, 732, 763, 758, 795, 776, 735, 757, 806, 758, 739, 765, 779, 754, 800, 800, 735, 791, 761, 737, 809, 782, 745, 733, 793, 799, 753, 815, 728, 743, 751] payload_rows_per_dst=[9386, 9196, 9594, 9510, 9180, 8982, 9276, 9206] dup_rows_per_dst=[3152, 2906, 3182, 3242, 2836, 2682, 3044, 2930] combine_rows_per_src=[9386, 9196, 9594, 9510, 9180, 8982, 9276, 9206] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=344144 combine_payload_bytes=266067968 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266412112 ingress_bytes=335273984 link_bytes=335273984 token_bytes=532480080 weight_movement_bytes=69206016 total_nvlink_bytes=601686096 weight_to_token_ratio=0.129969 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261609010e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266387456 dispatch_meta_bytes=342544 combine_payload_bytes=266387456 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266730000 ingress_bytes=335593472 link_bytes=335593472 token_bytes=533117456 weight_movement_bytes=69206016 total_nvlink_bytes=602323472 weight_to_token_ratio=0.129814 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262811218e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74536 payload_rows_remote=65076 dup_rows=23768 dup_rows_remote=20656 dedup_ratio=0.241781 router_tokens_per_expert=[807, 811, 759, 743, 750, 793, 773, 753, 771, 763, 761, 774, 794, 787, 756, 782, 786, 769, 708, 750, 757, 709, 793, 788, 729, 747, 749, 800, 780, 714, 760, 808, 726, 762, 755, 774, 774, 743, 821, 773, 740, 797, 750, 811, 757, 758, 727, 782, 780, 772, 794, 747, 778, 761, 722, 809, 780, 759, 771, 782, 774, 803, 745, 801] payload_rows_per_dst=[9460, 9442, 9290, 9124, 9274, 9610, 9150, 9186] dup_rows_per_dst=[3112, 3236, 2830, 2860, 2978, 3146, 2664, 2942] combine_rows_per_src=[9460, 9442, 9290, 9124, 9274, 9610, 9150, 9186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74182 payload_rows_remote=64700 dup_rows=24122 dup_rows_remote=20840 dedup_ratio=0.245382 router_tokens_per_expert=[787, 786, 777, 760, 741, 779, 801, 715, 763, 777, 798, 767, 757, 772, 794, 768, 761, 758, 727, 847, 786, 765, 761, 780, 772, 751, 774, 783, 763, 815, 754, 795, 750, 767, 746, 723, 789, 836, 767, 788, 753, 739, 731, 810, 744, 746, 735, 782, 730, 745, 775, 774, 730, 796, 785, 723, 758, 777, 764, 794, 773, 790, 742, 756] payload_rows_per_dst=[9288, 9186, 9252, 9338, 9146, 9216, 9274, 9482] dup_rows_per_dst=[3004, 3012, 3048, 3060, 2924, 2882, 2910, 3282] combine_rows_per_src=[9288, 9186, 9252, 9338, 9146, 9216, 9274, 9482] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265469952 dispatch_meta_bytes=343392 combine_payload_bytes=265469952 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265813344 ingress_bytes=334675968 link_bytes=334675968 token_bytes=531283296 weight_movement_bytes=69206016 total_nvlink_bytes=600489312 weight_to_token_ratio=0.130262 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259358724e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74078 payload_rows_remote=64800 dup_rows=24226 dup_rows_remote=21174 dedup_ratio=0.246440 router_tokens_per_expert=[820, 790, 735, 807, 764, 753, 692, 766, 805, 776, 755, 767, 768, 791, 764, 739, 719, 785, 772, 783, 780, 743, 762, 745, 791, 824, 737, 739, 833, 760, 760, 770, 777, 738, 727, 770, 739, 815, 705, 757, 793, 809, 791, 731, 768, 806, 834, 769, 713, 728, 733, 766, 814, 769, 761, 734, 831, 748, 700, 764, 799, 771, 783, 814] payload_rows_per_dst=[9250, 9278, 9266, 9338, 9132, 9372, 9150, 9292] dup_rows_per_dst=[3004, 3052, 2912, 3090, 2924, 3230, 2886, 3128] combine_rows_per_src=[9250, 9278, 9266, 9338, 9132, 9372, 9150, 9292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266379264 dispatch_meta_bytes=344224 combine_payload_bytes=266379264 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266723488 ingress_bytes=335585280 link_bytes=335585280 token_bytes=533102752 weight_movement_bytes=69206016 total_nvlink_bytes=602308768 weight_to_token_ratio=0.129817 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262780392e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74106 payload_rows_remote=64820 dup_rows=24198 dup_rows_remote=21078 dedup_ratio=0.246155 router_tokens_per_expert=[788, 760, 758, 745, 779, 766, 721, 726, 803, 802, 770, 774, 805, 755, 735, 740, 726, 745, 740, 739, 748, 793, 799, 770, 787, 772, 735, 779, 775, 781, 788, 786, 755, 797, 820, 764, 790, 783, 767, 793, 796, 774, 793, 768, 808, 828, 752, 734, 749, 726, 781, 728, 764, 789, 765, 759, 734, 745, 739, 800, 757, 751, 772, 781] payload_rows_per_dst=[9154, 9336, 9150, 9286, 9296, 9394, 9292, 9198] dup_rows_per_dst=[2932, 3032, 2970, 3120, 3242, 3112, 2830, 2960] combine_rows_per_src=[9154, 9336, 9150, 9286, 9296, 9394, 9292, 9198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266010624 dispatch_meta_bytes=343064 combine_payload_bytes=266010624 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266353688 ingress_bytes=335216640 link_bytes=335216640 token_bytes=532364312 weight_movement_bytes=69206016 total_nvlink_bytes=601570328 weight_to_token_ratio=0.129997 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261393229e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74458 payload_rows_remote=65188 dup_rows=23846 dup_rows_remote=20850 dedup_ratio=0.242574 router_tokens_per_expert=[711, 767, 785, 753, 745, 777, 766, 816, 719, 733, 788, 806, 782, 764, 758, 761, 769, 773, 798, 774, 755, 735, 759, 770, 762, 797, 766, 800, 799, 746, 754, 781, 822, 801, 741, 768, 756, 727, 769, 761, 788, 792, 804, 732, 724, 804, 733, 755, 768, 715, 750, 766, 757, 787, 776, 758, 811, 803, 758, 724, 785, 758, 770, 820] payload_rows_per_dst=[9216, 9290, 9270, 9350, 9266, 9354, 9342, 9370] dup_rows_per_dst=[3024, 2932, 2996, 3060, 3024, 2910, 2812, 3088] combine_rows_per_src=[9216, 9290, 9270, 9350, 9266, 9354, 9342, 9370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266551296 dispatch_meta_bytes=344968 combine_payload_bytes=266551296 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266896264 ingress_bytes=370360320 link_bytes=370360320 token_bytes=533447560 weight_movement_bytes=103809024 total_nvlink_bytes=637256584 weight_to_token_ratio=0.194600 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393636068e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266256384 dispatch_meta_bytes=343112 combine_payload_bytes=266256384 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266599496 ingress_bytes=335462400 link_bytes=335462400 token_bytes=532855880 weight_movement_bytes=69206016 total_nvlink_bytes=602061896 weight_to_token_ratio=0.129878 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262318004e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266551296 dispatch_meta_bytes=342928 combine_payload_bytes=266551296 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266894224 ingress_bytes=335757312 link_bytes=335757312 token_bytes=533445520 weight_movement_bytes=69206016 total_nvlink_bytes=602651536 weight_to_token_ratio=0.129734 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263427734e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265011200 dispatch_meta_bytes=342160 combine_payload_bytes=265011200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265353360 ingress_bytes=299614208 link_bytes=299614208 token_bytes=530364560 weight_movement_bytes=34603008 total_nvlink_bytes=564967568 weight_to_token_ratio=0.065244 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127424144e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74324 payload_rows_remote=64958 dup_rows=23980 dup_rows_remote=21078 dedup_ratio=0.243937 router_tokens_per_expert=[768, 775, 774, 759, 828, 746, 752, 804, 765, 789, 755, 780, 778, 782, 779, 762, 719, 740, 767, 830, 787, 748, 761, 747, 758, 770, 809, 766, 788, 745, 732, 781, 815, 769, 730, 778, 778, 766, 749, 739, 796, 758, 761, 761, 754, 748, 796, 760, 764, 776, 772, 799, 771, 746, 795, 727, 756, 808, 735, 782, 751, 765, 734, 769] payload_rows_per_dst=[9370, 9376, 9204, 9166, 9316, 9366, 9310, 9216] dup_rows_per_dst=[3042, 3004, 2994, 3132, 2932, 2902, 2990, 2984] combine_rows_per_src=[9370, 9376, 9204, 9166, 9316, 9366, 9310, 9216] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=64968 dup_rows=23926 dup_rows_remote=20768 dedup_ratio=0.243388 router_tokens_per_expert=[723, 821, 749, 757, 756, 752, 779, 790, 744, 728, 754, 780, 784, 774, 757, 732, 750, 743, 738, 777, 752, 783, 757, 771, 780, 709, 766, 726, 785, 798, 790, 797, 754, 841, 775, 788, 814, 766, 771, 775, 798, 774, 765, 760, 734, 742, 786, 799, 726, 776, 779, 791, 764, 746, 779, 752, 809, 781, 802, 752, 784, 748, 795, 724] payload_rows_per_dst=[9238, 9180, 9244, 9282, 9410, 9430, 9228, 9366] dup_rows_per_dst=[3016, 2926, 2898, 3020, 3158, 2886, 2998, 3024] combine_rows_per_src=[9238, 9180, 9244, 9282, 9410, 9430, 9228, 9366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265420800 dispatch_meta_bytes=343896 combine_payload_bytes=265420800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265764696 ingress_bytes=300023808 link_bytes=300023808 token_bytes=531185496 weight_movement_bytes=34603008 total_nvlink_bytes=565788504 weight_to_token_ratio=0.065143 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128965436e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=343592 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265846312 ingress_bytes=369311744 link_bytes=369311744 token_bytes=531349032 weight_movement_bytes=103809024 total_nvlink_bytes=635158056 weight_to_token_ratio=0.195369 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389690361e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=267010048 dispatch_meta_bytes=344152 combine_payload_bytes=267010048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267354200 ingress_bytes=301613056 link_bytes=301613056 token_bytes=534364248 weight_movement_bytes=34603008 total_nvlink_bytes=568967256 weight_to_token_ratio=0.064755 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134945648e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=65078 dup_rows=24016 dup_rows_remote=20998 dedup_ratio=0.244303 router_tokens_per_expert=[811, 720, 756, 768, 743, 778, 779, 791, 736, 822, 795, 795, 743, 802, 787, 798, 798, 745, 810, 734, 765, 758, 783, 795, 747, 728, 764, 744, 751, 773, 732, 799, 771, 811, 767, 777, 746, 745, 799, 763, 760, 774, 760, 759, 775, 776, 773, 737, 717, 765, 816, 793, 764, 741, 790, 715, 729, 763, 744, 749, 799, 779, 778, 767] payload_rows_per_dst=[9264, 9404, 9356, 9232, 9228, 9210, 9302, 9292] dup_rows_per_dst=[3028, 3152, 3020, 2844, 3130, 3018, 2900, 2924] combine_rows_per_src=[9264, 9404, 9356, 9232, 9228, 9210, 9302, 9292] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74112 payload_rows_remote=64930 dup_rows=24192 dup_rows_remote=21150 dedup_ratio=0.246094 router_tokens_per_expert=[776, 805, 800, 782, 772, 828, 767, 711, 739, 741, 754, 786, 766, 779, 747, 750, 798, 777, 755, 720, 760, 769, 760, 784, 743, 774, 770, 764, 756, 784, 805, 785, 760, 762, 721, 779, 736, 767, 775, 744, 789, 758, 743, 779, 785, 837, 816, 735, 759, 765, 750, 765, 800, 736, 753, 784, 760, 771, 767, 760, 728, 785, 796, 780] payload_rows_per_dst=[9398, 9162, 9186, 9278, 9212, 9402, 9182, 9292] dup_rows_per_dst=[3084, 2962, 3060, 3084, 2876, 3082, 3042, 3002] combine_rows_per_src=[9398, 9162, 9186, 9278, 9212, 9402, 9182, 9292] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=344144 combine_payload_bytes=266067968 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266412112 ingress_bytes=335273984 link_bytes=335273984 token_bytes=532480080 weight_movement_bytes=69206016 total_nvlink_bytes=601686096 weight_to_token_ratio=0.129969 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261609010e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=342944 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266451872 ingress_bytes=335314944 link_bytes=335314944 token_bytes=532560800 weight_movement_bytes=69206016 total_nvlink_bytes=601766816 weight_to_token_ratio=0.129950 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261763139e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=64968 dup_rows=23926 dup_rows_remote=20768 dedup_ratio=0.243388 router_tokens_per_expert=[723, 821, 749, 757, 756, 752, 779, 790, 744, 728, 754, 780, 784, 774, 757, 732, 750, 743, 738, 777, 752, 783, 757, 771, 780, 709, 766, 726, 785, 798, 790, 797, 754, 841, 775, 788, 814, 766, 771, 775, 798, 774, 765, 760, 734, 742, 786, 799, 726, 776, 779, 791, 764, 746, 779, 752, 809, 781, 802, 752, 784, 748, 795, 724] payload_rows_per_dst=[9238, 9180, 9244, 9282, 9410, 9430, 9228, 9366] dup_rows_per_dst=[3016, 2926, 2898, 3020, 3158, 2886, 2998, 3024] combine_rows_per_src=[9238, 9180, 9244, 9282, 9410, 9430, 9228, 9366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74084 payload_rows_remote=64850 dup_rows=24220 dup_rows_remote=21064 dedup_ratio=0.246379 router_tokens_per_expert=[775, 800, 797, 767, 719, 782, 747, 781, 771, 804, 752, 754, 771, 693, 771, 741, 757, 788, 771, 747, 762, 828, 765, 752, 789, 772, 715, 781, 737, 771, 758, 777, 781, 756, 769, 790, 748, 780, 785, 759, 794, 801, 745, 796, 770, 736, 792, 770, 761, 789, 770, 788, 754, 789, 790, 755, 743, 750, 737, 728, 830, 799, 723, 779] payload_rows_per_dst=[9260, 9196, 9322, 9236, 9308, 9282, 9246, 9234] dup_rows_per_dst=[3076, 2918, 3018, 2964, 3028, 3096, 2964, 3156] combine_rows_per_src=[9260, 9196, 9322, 9236, 9308, 9282, 9246, 9234] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74106 payload_rows_remote=64820 dup_rows=24198 dup_rows_remote=21078 dedup_ratio=0.246155 router_tokens_per_expert=[788, 760, 758, 745, 779, 766, 721, 726, 803, 802, 770, 774, 805, 755, 735, 740, 726, 745, 740, 739, 748, 793, 799, 770, 787, 772, 735, 779, 775, 781, 788, 786, 755, 797, 820, 764, 790, 783, 767, 793, 796, 774, 793, 768, 808, 828, 752, 734, 749, 726, 781, 728, 764, 789, 765, 759, 734, 745, 739, 800, 757, 751, 772, 781] payload_rows_per_dst=[9154, 9336, 9150, 9286, 9296, 9394, 9292, 9198] dup_rows_per_dst=[2932, 3032, 2970, 3120, 3242, 3112, 2830, 2960] combine_rows_per_src=[9154, 9336, 9150, 9286, 9296, 9394, 9292, 9198] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74078 payload_rows_remote=64800 dup_rows=24226 dup_rows_remote=21174 dedup_ratio=0.246440 router_tokens_per_expert=[820, 790, 735, 807, 764, 753, 692, 766, 805, 776, 755, 767, 768, 791, 764, 739, 719, 785, 772, 783, 780, 743, 762, 745, 791, 824, 737, 739, 833, 760, 760, 770, 777, 738, 727, 770, 739, 815, 705, 757, 793, 809, 791, 731, 768, 806, 834, 769, 713, 728, 733, 766, 814, 769, 761, 734, 831, 748, 700, 764, 799, 771, 783, 814] payload_rows_per_dst=[9250, 9278, 9266, 9338, 9132, 9372, 9150, 9292] dup_rows_per_dst=[3004, 3052, 2912, 3090, 2924, 3230, 2886, 3128] combine_rows_per_src=[9250, 9278, 9266, 9338, 9132, 9372, 9150, 9292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74388 payload_rows_remote=65040 dup_rows=23916 dup_rows_remote=20928 dedup_ratio=0.243286 router_tokens_per_expert=[723, 777, 776, 765, 765, 763, 802, 782, 752, 736, 777, 752, 766, 789, 778, 806, 755, 747, 787, 747, 767, 741, 737, 809, 747, 752, 788, 823, 753, 788, 734, 775, 754, 808, 772, 750, 730, 772, 769, 773, 768, 780, 771, 818, 719, 790, 773, 772, 778, 767, 752, 738, 755, 790, 810, 746, 734, 738, 712, 758, 800, 825, 772, 799] payload_rows_per_dst=[9348, 9388, 9258, 9392, 9446, 9142, 9156, 9258] dup_rows_per_dst=[2988, 3008, 2846, 3084, 3132, 2880, 2960, 3018] combine_rows_per_src=[9348, 9388, 9258, 9392, 9446, 9142, 9156, 9258] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266559488 dispatch_meta_bytes=344304 combine_payload_bytes=266559488 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266903792 ingress_bytes=404971520 link_bytes=404971520 token_bytes=533463280 weight_movement_bytes=138412032 total_nvlink_bytes=671875312 weight_to_token_ratio=0.259459 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523875227e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265953280 dispatch_meta_bytes=344320 combine_payload_bytes=265953280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266297600 ingress_bytes=300556288 link_bytes=300556288 token_bytes=532250880 weight_movement_bytes=34603008 total_nvlink_bytes=566853888 weight_to_token_ratio=0.065013 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130969115e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=64956 dup_rows=23940 dup_rows_remote=20828 dedup_ratio=0.243530 router_tokens_per_expert=[770, 768, 755, 735, 798, 783, 742, 747, 802, 805, 741, 788, 757, 795, 759, 750, 729, 767, 752, 764, 802, 724, 797, 759, 765, 774, 795, 800, 759, 781, 761, 816, 797, 777, 739, 809, 785, 759, 815, 742, 789, 769, 711, 808, 742, 791, 725, 801, 802, 772, 750, 737, 751, 779, 738, 789, 739, 687, 764, 793, 783, 753, 757, 759] payload_rows_per_dst=[9408, 9324, 9292, 9334, 9376, 9292, 9108, 9230] dup_rows_per_dst=[3112, 3114, 2896, 3124, 3070, 2980, 2804, 2840] combine_rows_per_src=[9408, 9324, 9292, 9334, 9376, 9292, 9108, 9230] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74386 payload_rows_remote=65034 dup_rows=23918 dup_rows_remote=21022 dedup_ratio=0.243306 router_tokens_per_expert=[783, 756, 776, 724, 779, 787, 813, 777, 754, 785, 730, 771, 798, 771, 812, 739, 775, 818, 754, 703, 769, 833, 769, 768, 755, 724, 747, 770, 774, 773, 785, 766, 774, 812, 776, 735, 786, 801, 756, 727, 795, 759, 743, 766, 743, 798, 733, 778, 763, 756, 772, 748, 780, 767, 771, 766, 773, 760, 743, 780, 753, 768, 788, 744] payload_rows_per_dst=[9150, 9496, 9366, 8730, 9424, 9174, 9694, 9352] dup_rows_per_dst=[2888, 3176, 3012, 2616, 3076, 3026, 3228, 2896] combine_rows_per_src=[9150, 9496, 9366, 8730, 9424, 9174, 9694, 9352] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=343592 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265846312 ingress_bytes=369311744 link_bytes=369311744 token_bytes=531349032 weight_movement_bytes=103809024 total_nvlink_bytes=635158056 weight_to_token_ratio=0.195369 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389690361e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74336 payload_rows_remote=65146 dup_rows=23968 dup_rows_remote=20914 dedup_ratio=0.243815 router_tokens_per_expert=[767, 749, 744, 755, 772, 773, 789, 811, 727, 833, 762, 723, 799, 752, 757, 769, 779, 769, 794, 770, 767, 830, 763, 784, 769, 720, 752, 787, 752, 763, 768, 810, 768, 737, 785, 779, 713, 771, 764, 768, 744, 790, 809, 827, 782, 786, 774, 731, 766, 762, 755, 752, 738, 801, 738, 777, 770, 794, 725, 758, 717, 747, 782, 783] payload_rows_per_dst=[9334, 9190, 9416, 9294, 9202, 9398, 9234, 9268] dup_rows_per_dst=[2986, 3054, 3096, 2948, 2968, 3088, 2944, 2884] combine_rows_per_src=[9334, 9190, 9416, 9294, 9202, 9398, 9234, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=342944 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266451872 ingress_bytes=335314944 link_bytes=335314944 token_bytes=532560800 weight_movement_bytes=69206016 total_nvlink_bytes=601766816 weight_to_token_ratio=0.129950 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261763139e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265625600 dispatch_meta_bytes=343656 combine_payload_bytes=265625600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265969256 ingress_bytes=300228608 link_bytes=300228608 token_bytes=531594856 weight_movement_bytes=34603008 total_nvlink_bytes=566197864 weight_to_token_ratio=0.065093 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129736082e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74174 payload_rows_remote=64944 dup_rows=24130 dup_rows_remote=21050 dedup_ratio=0.245463 router_tokens_per_expert=[766, 756, 717, 761, 722, 762, 810, 792, 711, 729, 780, 773, 783, 752, 810, 770, 740, 807, 825, 764, 720, 746, 775, 796, 795, 730, 774, 775, 777, 765, 759, 780, 726, 775, 777, 754, 734, 769, 802, 770, 726, 777, 785, 776, 793, 798, 770, 753, 789, 764, 796, 764, 730, 767, 752, 770, 776, 785, 766, 820, 733, 776, 770, 787] payload_rows_per_dst=[9170, 9254, 9220, 9230, 9292, 9352, 9298, 9358] dup_rows_per_dst=[3002, 2962, 3126, 3080, 2922, 3004, 2966, 3068] combine_rows_per_src=[9170, 9254, 9220, 9230, 9292, 9352, 9298, 9358] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=65046 dup_rows=24098 dup_rows_remote=21112 dedup_ratio=0.245138 router_tokens_per_expert=[810, 735, 796, 777, 764, 763, 771, 808, 770, 756, 762, 766, 770, 791, 748, 779, 748, 755, 756, 769, 734, 772, 800, 739, 736, 745, 766, 735, 748, 805, 793, 773, 749, 775, 742, 786, 763, 776, 733, 780, 767, 749, 762, 781, 754, 768, 778, 773, 744, 761, 752, 887, 819, 760, 729, 785, 821, 753, 773, 729, 763, 766, 805, 729] payload_rows_per_dst=[9374, 9364, 9160, 9182, 9288, 9200, 9418, 9220] dup_rows_per_dst=[3074, 2920, 2986, 3020, 2920, 3064, 3056, 3058] combine_rows_per_src=[9374, 9364, 9160, 9182, 9288, 9200, 9418, 9220] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265420800 dispatch_meta_bytes=343896 combine_payload_bytes=265420800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265764696 ingress_bytes=300023808 link_bytes=300023808 token_bytes=531185496 weight_movement_bytes=34603008 total_nvlink_bytes=565788504 weight_to_token_ratio=0.065143 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128965436e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266403840 dispatch_meta_bytes=343872 combine_payload_bytes=266403840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266747712 ingress_bytes=301006848 link_bytes=301006848 token_bytes=533151552 weight_movement_bytes=34603008 total_nvlink_bytes=567754560 weight_to_token_ratio=0.064903 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132664536e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=65078 dup_rows=24016 dup_rows_remote=20998 dedup_ratio=0.244303 router_tokens_per_expert=[811, 720, 756, 768, 743, 778, 779, 791, 736, 822, 795, 795, 743, 802, 787, 798, 798, 745, 810, 734, 765, 758, 783, 795, 747, 728, 764, 744, 751, 773, 732, 799, 771, 811, 767, 777, 746, 745, 799, 763, 760, 774, 760, 759, 775, 776, 773, 737, 717, 765, 816, 793, 764, 741, 790, 715, 729, 763, 744, 749, 799, 779, 778, 767] payload_rows_per_dst=[9264, 9404, 9356, 9232, 9228, 9210, 9302, 9292] dup_rows_per_dst=[3028, 3152, 3020, 2844, 3130, 3018, 2900, 2924] combine_rows_per_src=[9264, 9404, 9356, 9232, 9228, 9210, 9302, 9292] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64982 dup_rows=24068 dup_rows_remote=20988 dedup_ratio=0.244832 router_tokens_per_expert=[766, 753, 803, 745, 760, 719, 796, 750, 781, 740, 809, 760, 779, 767, 743, 744, 774, 726, 776, 765, 727, 771, 791, 751, 779, 776, 789, 779, 817, 754, 809, 774, 733, 759, 816, 773, 775, 767, 786, 758, 735, 746, 788, 761, 781, 759, 788, 777, 772, 762, 774, 684, 765, 759, 781, 785, 800, 775, 818, 721, 770, 749, 777, 785] payload_rows_per_dst=[9236, 9320, 9230, 9466, 9254, 9286, 9122, 9322] dup_rows_per_dst=[2948, 2926, 2932, 3088, 3080, 2984, 3042, 3068] combine_rows_per_src=[9236, 9320, 9230, 9466, 9254, 9286, 9122, 9322] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266379264 dispatch_meta_bytes=344224 combine_payload_bytes=266379264 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266723488 ingress_bytes=335585280 link_bytes=335585280 token_bytes=533102752 weight_movement_bytes=69206016 total_nvlink_bytes=602308768 weight_to_token_ratio=0.129817 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262780392e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266838016 dispatch_meta_bytes=344240 combine_payload_bytes=266838016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267182256 ingress_bytes=301441024 link_bytes=301441024 token_bytes=534020272 weight_movement_bytes=34603008 total_nvlink_bytes=568623280 weight_to_token_ratio=0.064797 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134298305e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266010624 dispatch_meta_bytes=343976 combine_payload_bytes=266010624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266354600 ingress_bytes=300613632 link_bytes=300613632 token_bytes=532365224 weight_movement_bytes=34603008 total_nvlink_bytes=566968232 weight_to_token_ratio=0.064999 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131184896e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=344632 combine_payload_bytes=266428416 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266773048 ingress_bytes=335634432 link_bytes=335634432 token_bytes=533201464 weight_movement_bytes=69206016 total_nvlink_bytes=602407480 weight_to_token_ratio=0.129793 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262965347e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=343136 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266402912 ingress_bytes=300662784 link_bytes=300662784 token_bytes=532462688 weight_movement_bytes=34603008 total_nvlink_bytes=567065696 weight_to_token_ratio=0.064987 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131369851e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74304 payload_rows_remote=65042 dup_rows=24000 dup_rows_remote=20866 dedup_ratio=0.244141 router_tokens_per_expert=[795, 723, 748, 769, 792, 723, 803, 809, 812, 736, 797, 776, 799, 757, 754, 749, 791, 720, 764, 791, 758, 777, 736, 745, 756, 769, 792, 804, 797, 746, 749, 785, 751, 788, 785, 749, 781, 764, 728, 785, 753, 781, 794, 784, 753, 805, 731, 797, 781, 740, 728, 785, 748, 791, 792, 692, 743, 809, 734, 752, 788, 792, 752, 774] payload_rows_per_dst=[9280, 9396, 9178, 9382, 9258, 9262, 9218, 9330] dup_rows_per_dst=[3044, 2964, 2986, 3014, 3004, 3134, 2896, 2958] combine_rows_per_src=[9280, 9396, 9178, 9382, 9258, 9262, 9218, 9330] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74122 payload_rows_remote=64652 dup_rows=24182 dup_rows_remote=20956 dedup_ratio=0.245992 router_tokens_per_expert=[779, 784, 821, 765, 753, 746, 743, 717, 768, 775, 791, 779, 754, 750, 756, 751, 764, 783, 800, 742, 765, 749, 799, 757, 772, 747, 781, 743, 804, 806, 734, 735, 771, 730, 769, 719, 736, 756, 789, 783, 748, 747, 751, 788, 795, 768, 792, 761, 808, 754, 795, 809, 798, 793, 790, 801, 763, 785, 755, 779, 732, 772, 754, 748] payload_rows_per_dst=[9200, 9244, 9340, 9306, 9178, 9210, 9470, 9174] dup_rows_per_dst=[3016, 3004, 2978, 2938, 2928, 3090, 3226, 3002] combine_rows_per_src=[9200, 9244, 9340, 9306, 9178, 9210, 9470, 9174] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266559488 dispatch_meta_bytes=344304 combine_payload_bytes=266559488 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266903792 ingress_bytes=404971520 link_bytes=404971520 token_bytes=533463280 weight_movement_bytes=138412032 total_nvlink_bytes=671875312 weight_to_token_ratio=0.259459 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523875227e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266166272 dispatch_meta_bytes=343880 combine_payload_bytes=266166272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266510152 ingress_bytes=300769280 link_bytes=300769280 token_bytes=532676424 weight_movement_bytes=34603008 total_nvlink_bytes=567279432 weight_to_token_ratio=0.064961 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131770587e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=64928 dup_rows=23946 dup_rows_remote=20956 dedup_ratio=0.243591 router_tokens_per_expert=[759, 778, 756, 783, 747, 796, 758, 742, 765, 811, 756, 743, 771, 782, 788, 764, 753, 784, 742, 834, 700, 766, 774, 749, 744, 758, 743, 731, 788, 755, 757, 739, 765, 732, 776, 762, 808, 791, 751, 786, 770, 808, 786, 795, 789, 790, 768, 780, 757, 764, 751, 746, 825, 759, 749, 756, 761, 813, 746, 776, 792, 758, 736, 790] payload_rows_per_dst=[9042, 9334, 9420, 9062, 9338, 9416, 9316, 9430] dup_rows_per_dst=[2764, 3026, 3180, 2968, 2948, 2946, 3124, 2990] combine_rows_per_src=[9042, 9334, 9420, 9062, 9338, 9416, 9316, 9430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64982 dup_rows=24068 dup_rows_remote=20988 dedup_ratio=0.244832 router_tokens_per_expert=[766, 753, 803, 745, 760, 719, 796, 750, 781, 740, 809, 760, 779, 767, 743, 744, 774, 726, 776, 765, 727, 771, 791, 751, 779, 776, 789, 779, 817, 754, 809, 774, 733, 759, 816, 773, 775, 767, 786, 758, 735, 746, 788, 761, 781, 759, 788, 777, 772, 762, 774, 684, 765, 759, 781, 785, 800, 775, 818, 721, 770, 749, 777, 785] payload_rows_per_dst=[9236, 9320, 9230, 9466, 9254, 9286, 9122, 9322] dup_rows_per_dst=[2948, 2926, 2932, 3088, 3080, 2984, 3042, 3068] combine_rows_per_src=[9236, 9320, 9230, 9466, 9254, 9286, 9122, 9322] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74174 payload_rows_remote=64944 dup_rows=24130 dup_rows_remote=21050 dedup_ratio=0.245463 router_tokens_per_expert=[766, 756, 717, 761, 722, 762, 810, 792, 711, 729, 780, 773, 783, 752, 810, 770, 740, 807, 825, 764, 720, 746, 775, 796, 795, 730, 774, 775, 777, 765, 759, 780, 726, 775, 777, 754, 734, 769, 802, 770, 726, 777, 785, 776, 793, 798, 770, 753, 789, 764, 796, 764, 730, 767, 752, 770, 776, 785, 766, 820, 733, 776, 770, 787] payload_rows_per_dst=[9170, 9254, 9220, 9230, 9292, 9352, 9298, 9358] dup_rows_per_dst=[3002, 2962, 3126, 3080, 2922, 3004, 2966, 3068] combine_rows_per_src=[9170, 9254, 9220, 9230, 9292, 9352, 9298, 9358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74336 payload_rows_remote=65146 dup_rows=23968 dup_rows_remote=20914 dedup_ratio=0.243815 router_tokens_per_expert=[767, 749, 744, 755, 772, 773, 789, 811, 727, 833, 762, 723, 799, 752, 757, 769, 779, 769, 794, 770, 767, 830, 763, 784, 769, 720, 752, 787, 752, 763, 768, 810, 768, 737, 785, 779, 713, 771, 764, 768, 744, 790, 809, 827, 782, 786, 774, 731, 766, 762, 755, 752, 738, 801, 738, 777, 770, 794, 725, 758, 717, 747, 782, 783] payload_rows_per_dst=[9334, 9190, 9416, 9294, 9202, 9398, 9234, 9268] dup_rows_per_dst=[2986, 3054, 3096, 2948, 2968, 3088, 2944, 2884] combine_rows_per_src=[9334, 9190, 9416, 9294, 9202, 9398, 9234, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=343632 combine_payload_bytes=266412032 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266755664 ingress_bytes=370221056 link_bytes=370221056 token_bytes=533167696 weight_movement_bytes=103809024 total_nvlink_bytes=636976720 weight_to_token_ratio=0.194702 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393112028e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74346 payload_rows_remote=65024 dup_rows=23958 dup_rows_remote=20998 dedup_ratio=0.243713 router_tokens_per_expert=[726, 716, 786, 793, 808, 813, 758, 741, 779, 792, 756, 721, 739, 741, 764, 776, 768, 758, 766, 764, 779, 726, 791, 780, 757, 798, 819, 792, 741, 747, 779, 802, 786, 776, 749, 771, 778, 745, 797, 784, 735, 752, 764, 746, 760, 790, 784, 820, 785, 781, 788, 763, 730, 792, 754, 737, 778, 780, 752, 767, 747, 791, 743, 751] payload_rows_per_dst=[9322, 9422, 9312, 9652, 9352, 8946, 9232, 9108] dup_rows_per_dst=[2960, 3054, 2952, 3400, 3020, 2774, 3028, 2770] combine_rows_per_src=[9322, 9422, 9312, 9652, 9352, 8946, 9232, 9108] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=264814592 dispatch_meta_bytes=342432 combine_payload_bytes=264814592 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265157024 ingress_bytes=368623616 link_bytes=368623616 token_bytes=529971616 weight_movement_bytes=103809024 total_nvlink_bytes=633780640 weight_to_token_ratio=0.195877 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.387100990e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74084 payload_rows_remote=64850 dup_rows=24220 dup_rows_remote=21064 dedup_ratio=0.246379 router_tokens_per_expert=[775, 800, 797, 767, 719, 782, 747, 781, 771, 804, 752, 754, 771, 693, 771, 741, 757, 788, 771, 747, 762, 828, 765, 752, 789, 772, 715, 781, 737, 771, 758, 777, 781, 756, 769, 790, 748, 780, 785, 759, 794, 801, 745, 796, 770, 736, 792, 770, 761, 789, 770, 788, 754, 789, 790, 755, 743, 750, 737, 728, 830, 799, 723, 779] payload_rows_per_dst=[9260, 9196, 9322, 9236, 9308, 9282, 9246, 9234] dup_rows_per_dst=[3076, 2918, 3018, 2964, 3028, 3096, 2964, 3156] combine_rows_per_src=[9260, 9196, 9322, 9236, 9308, 9282, 9246, 9234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=65042 dup_rows=24060 dup_rows_remote=20974 dedup_ratio=0.244751 router_tokens_per_expert=[775, 740, 766, 786, 798, 748, 827, 797, 769, 800, 751, 770, 744, 819, 715, 776, 712, 795, 816, 755, 782, 809, 731, 765, 779, 774, 748, 750, 774, 775, 790, 785, 749, 766, 764, 771, 790, 753, 721, 742, 740, 782, 729, 741, 760, 760, 817, 747, 739, 720, 791, 774, 797, 749, 780, 793, 777, 768, 744, 794, 751, 735, 771, 816] payload_rows_per_dst=[9352, 9202, 9242, 9350, 9192, 9356, 9236, 9314] dup_rows_per_dst=[3122, 3086, 3088, 3000, 2920, 2796, 3050, 2998] combine_rows_per_src=[9352, 9202, 9242, 9350, 9192, 9356, 9236, 9314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74336 payload_rows_remote=65184 dup_rows=23968 dup_rows_remote=21210 dedup_ratio=0.243815 router_tokens_per_expert=[733, 802, 793, 752, 789, 779, 748, 790, 773, 770, 755, 804, 778, 757, 767, 801, 791, 732, 807, 745, 780, 731, 738, 738, 743, 764, 741, 737, 742, 754, 733, 741, 759, 731, 772, 781, 784, 775, 802, 718, 772, 822, 777, 743, 774, 779, 754, 757, 797, 757, 748, 764, 767, 752, 818, 718, 781, 784, 810, 768, 783, 813, 785, 799] payload_rows_per_dst=[9264, 9306, 9252, 9152, 9306, 9312, 9298, 9446] dup_rows_per_dst=[3108, 3104, 2872, 2758, 2938, 3044, 2944, 3200] combine_rows_per_src=[9264, 9306, 9252, 9152, 9306, 9312, 9298, 9446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265945088 dispatch_meta_bytes=343536 combine_payload_bytes=265945088 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266288624 ingress_bytes=335151104 link_bytes=335151104 token_bytes=532233712 weight_movement_bytes=69206016 total_nvlink_bytes=601439728 weight_to_token_ratio=0.130029 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261146622e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74596 payload_rows_remote=65206 dup_rows=23708 dup_rows_remote=20704 dedup_ratio=0.241170 router_tokens_per_expert=[749, 734, 806, 743, 812, 771, 795, 776, 776, 761, 738, 736, 765, 731, 847, 709, 754, 782, 761, 767, 796, 773, 807, 757, 766, 781, 774, 786, 773, 752, 749, 791, 777, 751, 789, 749, 765, 747, 761, 763, 784, 786, 783, 794, 748, 745, 801, 792, 774, 742, 743, 816, 784, 755, 773, 744, 694, 761, 738, 742, 806, 818, 780, 729] payload_rows_per_dst=[9320, 9270, 9390, 9338, 9336, 9404, 9308, 9230] dup_rows_per_dst=[3052, 2856, 3004, 3006, 2868, 3062, 2954, 2906] combine_rows_per_src=[9320, 9270, 9390, 9338, 9336, 9404, 9308, 9230] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266010624 dispatch_meta_bytes=343976 combine_payload_bytes=266010624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266354600 ingress_bytes=300613632 link_bytes=300613632 token_bytes=532365224 weight_movement_bytes=34603008 total_nvlink_bytes=566968232 weight_to_token_ratio=0.064999 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131184896e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=64944 dup_rows=23974 dup_rows_remote=20822 dedup_ratio=0.243876 router_tokens_per_expert=[735, 799, 798, 776, 735, 752, 785, 765, 698, 718, 764, 805, 743, 787, 748, 788, 740, 778, 787, 762, 749, 772, 763, 711, 806, 803, 835, 790, 795, 764, 748, 788, 799, 774, 732, 763, 758, 795, 776, 735, 757, 806, 758, 739, 765, 779, 754, 800, 800, 735, 791, 761, 737, 809, 782, 745, 733, 793, 799, 753, 815, 728, 743, 751] payload_rows_per_dst=[9386, 9196, 9594, 9510, 9180, 8982, 9276, 9206] dup_rows_per_dst=[3152, 2906, 3182, 3242, 2836, 2682, 3044, 2930] combine_rows_per_src=[9386, 9196, 9594, 9510, 9180, 8982, 9276, 9206] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266166272 dispatch_meta_bytes=343880 combine_payload_bytes=266166272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266510152 ingress_bytes=300769280 link_bytes=300769280 token_bytes=532676424 weight_movement_bytes=34603008 total_nvlink_bytes=567279432 weight_to_token_ratio=0.064961 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131770587e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=344088 combine_payload_bytes=266338304 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266682392 ingress_bytes=370147328 link_bytes=370147328 token_bytes=533020696 weight_movement_bytes=103809024 total_nvlink_bytes=636829720 weight_to_token_ratio=0.194756 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392834596e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266838016 dispatch_meta_bytes=344240 combine_payload_bytes=266838016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267182256 ingress_bytes=301441024 link_bytes=301441024 token_bytes=534020272 weight_movement_bytes=34603008 total_nvlink_bytes=568623280 weight_to_token_ratio=0.064797 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134298305e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74304 payload_rows_remote=65042 dup_rows=24000 dup_rows_remote=20866 dedup_ratio=0.244141 router_tokens_per_expert=[795, 723, 748, 769, 792, 723, 803, 809, 812, 736, 797, 776, 799, 757, 754, 749, 791, 720, 764, 791, 758, 777, 736, 745, 756, 769, 792, 804, 797, 746, 749, 785, 751, 788, 785, 749, 781, 764, 728, 785, 753, 781, 794, 784, 753, 805, 731, 797, 781, 740, 728, 785, 748, 791, 792, 692, 743, 809, 734, 752, 788, 792, 752, 774] payload_rows_per_dst=[9280, 9396, 9178, 9382, 9258, 9262, 9218, 9330] dup_rows_per_dst=[3044, 2964, 2986, 3014, 3004, 3134, 2896, 2958] combine_rows_per_src=[9280, 9396, 9178, 9382, 9258, 9262, 9218, 9330] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74496 payload_rows_remote=65134 dup_rows=23808 dup_rows_remote=20836 dedup_ratio=0.242188 router_tokens_per_expert=[727, 804, 796, 790, 761, 784, 762, 784, 797, 768, 773, 749, 749, 768, 738, 774, 749, 793, 762, 717, 701, 778, 798, 785, 742, 754, 795, 759, 831, 751, 705, 713, 743, 777, 757, 817, 739, 795, 747, 792, 806, 769, 783, 755, 779, 764, 772, 736, 751, 787, 794, 752, 755, 774, 752, 787, 768, 792, 794, 742, 760, 760, 793, 803] payload_rows_per_dst=[9380, 9342, 9278, 9208, 9362, 9316, 9252, 9358] dup_rows_per_dst=[3036, 2890, 2888, 2892, 2972, 3012, 3052, 3066] combine_rows_per_src=[9380, 9342, 9278, 9208, 9362, 9316, 9252, 9358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265625600 dispatch_meta_bytes=343656 combine_payload_bytes=265625600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265969256 ingress_bytes=300228608 link_bytes=300228608 token_bytes=531594856 weight_movement_bytes=34603008 total_nvlink_bytes=566197864 weight_to_token_ratio=0.065093 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129736082e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=344064 combine_payload_bytes=266412032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266756096 ingress_bytes=301015040 link_bytes=301015040 token_bytes=533168128 weight_movement_bytes=34603008 total_nvlink_bytes=567771136 weight_to_token_ratio=0.064901 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132695362e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266993664 dispatch_meta_bytes=345576 combine_payload_bytes=266993664 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267339240 ingress_bytes=301596672 link_bytes=301596672 token_bytes=534332904 weight_movement_bytes=34603008 total_nvlink_bytes=568935912 weight_to_token_ratio=0.064759 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134883996e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267083776 dispatch_meta_bytes=343640 combine_payload_bytes=267083776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267427416 ingress_bytes=301686784 link_bytes=301686784 token_bytes=534511192 weight_movement_bytes=34603008 total_nvlink_bytes=569114200 weight_to_token_ratio=0.064738 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135223080e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266010624 dispatch_meta_bytes=343064 combine_payload_bytes=266010624 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266353688 ingress_bytes=335216640 link_bytes=335216640 token_bytes=532364312 weight_movement_bytes=69206016 total_nvlink_bytes=601570328 weight_to_token_ratio=0.129997 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261393229e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=64996 dup_rows=24002 dup_rows_remote=21024 dedup_ratio=0.244161 router_tokens_per_expert=[766, 767, 689, 778, 766, 826, 774, 761, 773, 757, 753, 793, 761, 771, 769, 805, 781, 788, 741, 756, 783, 738, 776, 790, 796, 739, 746, 770, 753, 795, 780, 785, 782, 758, 787, 798, 754, 765, 744, 767, 713, 770, 763, 769, 760, 780, 807, 780, 780, 802, 767, 784, 761, 734, 800, 717, 791, 776, 760, 746, 754, 755, 754, 748] payload_rows_per_dst=[9212, 9240, 9352, 9310, 9368, 9306, 9354, 9160] dup_rows_per_dst=[3042, 3124, 2954, 3018, 2942, 2978, 2936, 3008] combine_rows_per_src=[9212, 9240, 9352, 9310, 9368, 9306, 9354, 9160] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74382 payload_rows_remote=64996 dup_rows=23922 dup_rows_remote=20912 dedup_ratio=0.243347 router_tokens_per_expert=[798, 759, 780, 723, 722, 755, 803, 789, 763, 760, 718, 796, 761, 756, 752, 720, 701, 785, 752, 762, 773, 802, 799, 757, 772, 739, 737, 775, 751, 705, 811, 771, 768, 778, 791, 780, 775, 770, 836, 747, 791, 812, 799, 765, 778, 792, 734, 791, 766, 787, 746, 748, 749, 789, 833, 780, 762, 750, 755, 774, 784, 773, 759, 743] payload_rows_per_dst=[9202, 9230, 9296, 9220, 9420, 9420, 9386, 9208] dup_rows_per_dst=[3056, 2822, 2966, 2902, 3070, 3104, 3010, 2992] combine_rows_per_src=[9202, 9230, 9296, 9220, 9420, 9420, 9386, 9208] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=343632 combine_payload_bytes=266412032 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266755664 ingress_bytes=370221056 link_bytes=370221056 token_bytes=533167696 weight_movement_bytes=103809024 total_nvlink_bytes=636976720 weight_to_token_ratio=0.194702 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393112028e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74416 payload_rows_remote=64956 dup_rows=23888 dup_rows_remote=20834 dedup_ratio=0.243001 router_tokens_per_expert=[777, 767, 764, 828, 733, 772, 789, 743, 762, 777, 757, 773, 748, 792, 793, 748, 739, 720, 733, 793, 734, 769, 774, 781, 722, 781, 802, 758, 735, 775, 763, 814, 774, 763, 800, 772, 755, 756, 768, 809, 738, 774, 775, 771, 760, 767, 798, 780, 801, 742, 757, 779, 816, 718, 733, 781, 764, 786, 771, 776, 747, 744, 761, 800] payload_rows_per_dst=[9378, 9296, 9268, 9346, 9164, 9274, 9230, 9460] dup_rows_per_dst=[2968, 3004, 2818, 2954, 3000, 3052, 3038, 3054] combine_rows_per_src=[9378, 9296, 9268, 9346, 9164, 9274, 9230, 9460] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=343880 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267132744 ingress_bytes=335994880 link_bytes=335994880 token_bytes=533921608 weight_movement_bytes=69206016 total_nvlink_bytes=603127624 weight_to_token_ratio=0.129618 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264321684e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74336 payload_rows_remote=65184 dup_rows=23968 dup_rows_remote=21210 dedup_ratio=0.243815 router_tokens_per_expert=[733, 802, 793, 752, 789, 779, 748, 790, 773, 770, 755, 804, 778, 757, 767, 801, 791, 732, 807, 745, 780, 731, 738, 738, 743, 764, 741, 737, 742, 754, 733, 741, 759, 731, 772, 781, 784, 775, 802, 718, 772, 822, 777, 743, 774, 779, 754, 757, 797, 757, 748, 764, 767, 752, 818, 718, 781, 784, 810, 768, 783, 813, 785, 799] payload_rows_per_dst=[9264, 9306, 9252, 9152, 9306, 9312, 9298, 9446] dup_rows_per_dst=[3108, 3104, 2872, 2758, 2938, 3044, 2944, 3200] combine_rows_per_src=[9264, 9306, 9252, 9152, 9306, 9312, 9298, 9446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=344080 combine_payload_bytes=266223616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266567696 ingress_bytes=300826624 link_bytes=300826624 token_bytes=532791312 weight_movement_bytes=34603008 total_nvlink_bytes=567394320 weight_to_token_ratio=0.064947 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131986368e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74496 payload_rows_remote=65134 dup_rows=23808 dup_rows_remote=20836 dedup_ratio=0.242188 router_tokens_per_expert=[727, 804, 796, 790, 761, 784, 762, 784, 797, 768, 773, 749, 749, 768, 738, 774, 749, 793, 762, 717, 701, 778, 798, 785, 742, 754, 795, 759, 831, 751, 705, 713, 743, 777, 757, 817, 739, 795, 747, 792, 806, 769, 783, 755, 779, 764, 772, 736, 751, 787, 794, 752, 755, 774, 752, 787, 768, 792, 794, 742, 760, 760, 793, 803] payload_rows_per_dst=[9380, 9342, 9278, 9208, 9362, 9316, 9252, 9358] dup_rows_per_dst=[3036, 2890, 2888, 2892, 2972, 3012, 3052, 3066] combine_rows_per_src=[9380, 9342, 9278, 9208, 9362, 9316, 9252, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74408 payload_rows_remote=65106 dup_rows=23896 dup_rows_remote=20786 dedup_ratio=0.243083 router_tokens_per_expert=[753, 774, 777, 760, 771, 772, 787, 755, 767, 758, 706, 758, 753, 768, 799, 798, 792, 775, 745, 779, 770, 759, 784, 789, 795, 746, 778, 785, 742, 778, 808, 740, 762, 782, 736, 830, 788, 779, 779, 757, 762, 735, 806, 710, 787, 780, 760, 792, 735, 776, 742, 829, 791, 727, 758, 737, 750, 760, 775, 766, 747, 771, 759, 763] payload_rows_per_dst=[9302, 9630, 9146, 9312, 9308, 9364, 9144, 9202] dup_rows_per_dst=[3110, 3212, 2774, 2876, 2998, 2948, 2998, 2980] combine_rows_per_src=[9302, 9630, 9146, 9312, 9308, 9364, 9144, 9202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=65012 dup_rows=24008 dup_rows_remote=20956 dedup_ratio=0.244222 router_tokens_per_expert=[765, 737, 752, 724, 753, 794, 762, 776, 762, 768, 735, 731, 770, 785, 816, 801, 780, 771, 793, 778, 754, 797, 744, 879, 775, 740, 761, 759, 749, 774, 731, 795, 797, 779, 730, 774, 794, 832, 771, 750, 783, 772, 793, 697, 728, 752, 774, 757, 761, 757, 776, 784, 792, 771, 813, 803, 743, 767, 765, 764, 732, 715, 715, 800] payload_rows_per_dst=[9254, 9284, 9426, 9262, 9426, 9142, 9426, 9076] dup_rows_per_dst=[2872, 3052, 3166, 2906, 3028, 2970, 3088, 2926] combine_rows_per_src=[9254, 9284, 9426, 9262, 9426, 9142, 9426, 9076] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=343632 combine_payload_bytes=266223616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266567248 ingress_bytes=300826624 link_bytes=300826624 token_bytes=532790864 weight_movement_bytes=34603008 total_nvlink_bytes=567393872 weight_to_token_ratio=0.064947 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131986368e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=65042 dup_rows=24060 dup_rows_remote=20974 dedup_ratio=0.244751 router_tokens_per_expert=[775, 740, 766, 786, 798, 748, 827, 797, 769, 800, 751, 770, 744, 819, 715, 776, 712, 795, 816, 755, 782, 809, 731, 765, 779, 774, 748, 750, 774, 775, 790, 785, 749, 766, 764, 771, 790, 753, 721, 742, 740, 782, 729, 741, 760, 760, 817, 747, 739, 720, 791, 774, 797, 749, 780, 793, 777, 768, 744, 794, 751, 735, 771, 816] payload_rows_per_dst=[9352, 9202, 9242, 9350, 9192, 9356, 9236, 9314] dup_rows_per_dst=[3122, 3086, 3088, 3000, 2920, 2796, 3050, 2998] combine_rows_per_src=[9352, 9202, 9242, 9350, 9192, 9356, 9236, 9314] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=64928 dup_rows=23946 dup_rows_remote=20956 dedup_ratio=0.243591 router_tokens_per_expert=[759, 778, 756, 783, 747, 796, 758, 742, 765, 811, 756, 743, 771, 782, 788, 764, 753, 784, 742, 834, 700, 766, 774, 749, 744, 758, 743, 731, 788, 755, 757, 739, 765, 732, 776, 762, 808, 791, 751, 786, 770, 808, 786, 795, 789, 790, 768, 780, 757, 764, 751, 746, 825, 759, 749, 756, 761, 813, 746, 776, 792, 758, 736, 790] payload_rows_per_dst=[9042, 9334, 9420, 9062, 9338, 9416, 9316, 9430] dup_rows_per_dst=[2764, 3026, 3180, 2968, 2948, 2946, 3124, 2990] combine_rows_per_src=[9042, 9334, 9420, 9062, 9338, 9416, 9316, 9430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74102 payload_rows_remote=64820 dup_rows=24202 dup_rows_remote=21104 dedup_ratio=0.246195 router_tokens_per_expert=[761, 785, 772, 773, 768, 762, 771, 727, 761, 712, 767, 757, 753, 765, 751, 731, 798, 777, 810, 746, 747, 772, 773, 778, 733, 813, 741, 774, 801, 755, 771, 802, 751, 745, 775, 779, 700, 756, 764, 768, 757, 754, 736, 790, 741, 739, 801, 833, 797, 792, 775, 823, 749, 764, 748, 814, 820, 737, 782, 746, 804, 775, 761, 769] payload_rows_per_dst=[9244, 9134, 9332, 9282, 9128, 9316, 9376, 9290] dup_rows_per_dst=[2994, 2860, 3070, 3098, 2948, 2986, 3148, 3098] combine_rows_per_src=[9244, 9134, 9332, 9282, 9128, 9316, 9376, 9290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=343160 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266402936 ingress_bytes=300662784 link_bytes=300662784 token_bytes=532462712 weight_movement_bytes=34603008 total_nvlink_bytes=567065720 weight_to_token_ratio=0.064987 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131369851e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74008 payload_rows_remote=64774 dup_rows=24296 dup_rows_remote=21188 dedup_ratio=0.247152 router_tokens_per_expert=[790, 800, 743, 811, 753, 815, 723, 775, 760, 753, 764, 774, 701, 683, 775, 731, 738, 762, 720, 724, 815, 833, 786, 793, 712, 809, 807, 779, 789, 762, 799, 778, 738, 796, 767, 721, 762, 775, 794, 801, 778, 821, 727, 753, 768, 807, 730, 761, 765, 762, 783, 806, 761, 739, 803, 779, 800, 714, 784, 770, 701, 762, 825, 742] payload_rows_per_dst=[9328, 9050, 9234, 9364, 9310, 9280, 9210, 9232] dup_rows_per_dst=[3092, 2832, 3108, 3106, 2998, 3010, 3186, 2964] combine_rows_per_src=[9328, 9050, 9234, 9364, 9310, 9280, 9210, 9232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266993664 dispatch_meta_bytes=345576 combine_payload_bytes=266993664 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267339240 ingress_bytes=301596672 link_bytes=301596672 token_bytes=534332904 weight_movement_bytes=34603008 total_nvlink_bytes=568935912 weight_to_token_ratio=0.064759 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134883996e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74388 payload_rows_remote=65040 dup_rows=23916 dup_rows_remote=20928 dedup_ratio=0.243286 router_tokens_per_expert=[723, 777, 776, 765, 765, 763, 802, 782, 752, 736, 777, 752, 766, 789, 778, 806, 755, 747, 787, 747, 767, 741, 737, 809, 747, 752, 788, 823, 753, 788, 734, 775, 754, 808, 772, 750, 730, 772, 769, 773, 768, 780, 771, 818, 719, 790, 773, 772, 778, 767, 752, 738, 755, 790, 810, 746, 734, 738, 712, 758, 800, 825, 772, 799] payload_rows_per_dst=[9348, 9388, 9258, 9392, 9446, 9142, 9156, 9258] dup_rows_per_dst=[2988, 3008, 2846, 3084, 3132, 2880, 2960, 3018] combine_rows_per_src=[9348, 9388, 9258, 9392, 9446, 9142, 9156, 9258] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=343880 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267132744 ingress_bytes=335994880 link_bytes=335994880 token_bytes=533921608 weight_movement_bytes=69206016 total_nvlink_bytes=603127624 weight_to_token_ratio=0.129618 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264321684e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266674176 dispatch_meta_bytes=343568 combine_payload_bytes=266674176 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267017744 ingress_bytes=301277184 link_bytes=301277184 token_bytes=533691920 weight_movement_bytes=34603008 total_nvlink_bytes=568294928 weight_to_token_ratio=0.064837 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133681789e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=344064 combine_payload_bytes=266412032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266756096 ingress_bytes=301015040 link_bytes=301015040 token_bytes=533168128 weight_movement_bytes=34603008 total_nvlink_bytes=567771136 weight_to_token_ratio=0.064901 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132695362e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=64996 dup_rows=24002 dup_rows_remote=21024 dedup_ratio=0.244161 router_tokens_per_expert=[766, 767, 689, 778, 766, 826, 774, 761, 773, 757, 753, 793, 761, 771, 769, 805, 781, 788, 741, 756, 783, 738, 776, 790, 796, 739, 746, 770, 753, 795, 780, 785, 782, 758, 787, 798, 754, 765, 744, 767, 713, 770, 763, 769, 760, 780, 807, 780, 780, 802, 767, 784, 761, 734, 800, 717, 791, 776, 760, 746, 754, 755, 754, 748] payload_rows_per_dst=[9212, 9240, 9352, 9310, 9368, 9306, 9354, 9160] dup_rows_per_dst=[3042, 3124, 2954, 3018, 2942, 2978, 2936, 3008] combine_rows_per_src=[9212, 9240, 9352, 9310, 9368, 9306, 9354, 9160] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=64900 dup_rows=24136 dup_rows_remote=21046 dedup_ratio=0.245524 router_tokens_per_expert=[810, 796, 718, 772, 733, 795, 748, 805, 706, 713, 765, 808, 777, 786, 778, 802, 799, 729, 777, 776, 729, 743, 784, 773, 790, 779, 763, 763, 757, 699, 837, 714, 772, 779, 805, 775, 740, 746, 776, 786, 757, 759, 780, 768, 743, 794, 743, 771, 756, 752, 748, 762, 837, 754, 726, 739, 791, 768, 755, 763, 805, 781, 758, 839] payload_rows_per_dst=[9260, 9196, 9246, 9240, 9268, 9192, 9270, 9496] dup_rows_per_dst=[3094, 3074, 2974, 2964, 3090, 3038, 2878, 3024] combine_rows_per_src=[9260, 9196, 9246, 9240, 9268, 9192, 9270, 9496] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265945088 dispatch_meta_bytes=343536 combine_payload_bytes=265945088 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266288624 ingress_bytes=335151104 link_bytes=335151104 token_bytes=532233712 weight_movement_bytes=69206016 total_nvlink_bytes=601439728 weight_to_token_ratio=0.130029 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261146622e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266289152 dispatch_meta_bytes=343872 combine_payload_bytes=266289152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266633024 ingress_bytes=300892160 link_bytes=300892160 token_bytes=532922176 weight_movement_bytes=34603008 total_nvlink_bytes=567525184 weight_to_token_ratio=0.064931 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132232974e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=343696 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265846416 ingress_bytes=334708736 link_bytes=334708736 token_bytes=531349136 weight_movement_bytes=69206016 total_nvlink_bytes=600555152 weight_to_token_ratio=0.130246 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259482027e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265314304 dispatch_meta_bytes=343848 combine_payload_bytes=265314304 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265658152 ingress_bytes=334520320 link_bytes=334520320 token_bytes=530972456 weight_movement_bytes=69206016 total_nvlink_bytes=600178472 weight_to_token_ratio=0.130338 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258773033e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266403840 dispatch_meta_bytes=343872 combine_payload_bytes=266403840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266747712 ingress_bytes=301006848 link_bytes=301006848 token_bytes=533151552 weight_movement_bytes=34603008 total_nvlink_bytes=567754560 weight_to_token_ratio=0.064903 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132664536e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=64972 dup_rows=24050 dup_rows_remote=20994 dedup_ratio=0.244649 router_tokens_per_expert=[735, 771, 789, 803, 734, 734, 773, 762, 758, 767, 733, 756, 776, 785, 761, 774, 765, 800, 765, 759, 775, 813, 763, 778, 769, 763, 780, 790, 749, 738, 780, 775, 764, 765, 786, 754, 716, 790, 726, 754, 793, 794, 756, 716, 766, 767, 785, 792, 750, 777, 764, 789, 780, 763, 764, 769, 776, 749, 680, 826, 771, 796, 813, 788] payload_rows_per_dst=[9250, 9222, 9408, 9304, 9214, 9282, 9266, 9308] dup_rows_per_dst=[2952, 2998, 3028, 2984, 2896, 3056, 3046, 3090] combine_rows_per_src=[9250, 9222, 9408, 9304, 9214, 9282, 9266, 9308] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74390 payload_rows_remote=65224 dup_rows=23914 dup_rows_remote=21058 dedup_ratio=0.243266 router_tokens_per_expert=[762, 759, 772, 698, 783, 741, 807, 755, 800, 742, 783, 771, 743, 787, 774, 758, 740, 808, 740, 810, 791, 768, 814, 747, 762, 733, 726, 743, 791, 794, 805, 779, 793, 790, 750, 780, 773, 793, 743, 742, 786, 792, 783, 761, 773, 765, 789, 775, 748, 804, 732, 764, 723, 749, 752, 739, 755, 826, 749, 780, 790, 719, 751, 797] payload_rows_per_dst=[9280, 9276, 9366, 9262, 9274, 9386, 9166, 9380] dup_rows_per_dst=[2874, 3040, 3070, 3004, 3054, 3062, 2856, 2954] combine_rows_per_src=[9280, 9276, 9366, 9262, 9274, 9386, 9166, 9380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74266 payload_rows_remote=64768 dup_rows=24038 dup_rows_remote=20898 dedup_ratio=0.244527 router_tokens_per_expert=[731, 755, 815, 774, 769, 715, 789, 806, 758, 748, 757, 762, 756, 822, 763, 732, 763, 795, 757, 782, 782, 745, 753, 794, 809, 764, 777, 810, 749, 799, 768, 756, 751, 800, 779, 749, 756, 772, 778, 760, 730, 779, 747, 743, 741, 767, 736, 747, 805, 790, 727, 822, 738, 730, 795, 730, 763, 779, 728, 815, 794, 764, 804, 778] payload_rows_per_dst=[9348, 9194, 9244, 9296, 9256, 9062, 9368, 9498] dup_rows_per_dst=[2960, 3002, 3058, 2980, 3074, 2918, 2906, 3140] combine_rows_per_src=[9348, 9194, 9244, 9296, 9256, 9062, 9368, 9498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266125312 dispatch_meta_bytes=343864 combine_payload_bytes=266125312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266469176 ingress_bytes=300728320 link_bytes=300728320 token_bytes=532594488 weight_movement_bytes=34603008 total_nvlink_bytes=567197496 weight_to_token_ratio=0.064971 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131616458e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=344080 combine_payload_bytes=266223616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266567696 ingress_bytes=300826624 link_bytes=300826624 token_bytes=532791312 weight_movement_bytes=34603008 total_nvlink_bytes=567394320 weight_to_token_ratio=0.064947 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131986368e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74102 payload_rows_remote=64820 dup_rows=24202 dup_rows_remote=21104 dedup_ratio=0.246195 router_tokens_per_expert=[761, 785, 772, 773, 768, 762, 771, 727, 761, 712, 767, 757, 753, 765, 751, 731, 798, 777, 810, 746, 747, 772, 773, 778, 733, 813, 741, 774, 801, 755, 771, 802, 751, 745, 775, 779, 700, 756, 764, 768, 757, 754, 736, 790, 741, 739, 801, 833, 797, 792, 775, 823, 749, 764, 748, 814, 820, 737, 782, 746, 804, 775, 761, 769] payload_rows_per_dst=[9244, 9134, 9332, 9282, 9128, 9316, 9376, 9290] dup_rows_per_dst=[2994, 2860, 3070, 3098, 2948, 2986, 3148, 3098] combine_rows_per_src=[9244, 9134, 9332, 9282, 9128, 9316, 9376, 9290] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265830400 dispatch_meta_bytes=343784 combine_payload_bytes=265830400 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266174184 ingress_bytes=404242432 link_bytes=404242432 token_bytes=532004584 weight_movement_bytes=138412032 total_nvlink_bytes=670416616 weight_to_token_ratio=0.260171 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521131727e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=64900 dup_rows=24136 dup_rows_remote=21046 dedup_ratio=0.245524 router_tokens_per_expert=[810, 796, 718, 772, 733, 795, 748, 805, 706, 713, 765, 808, 777, 786, 778, 802, 799, 729, 777, 776, 729, 743, 784, 773, 790, 779, 763, 763, 757, 699, 837, 714, 772, 779, 805, 775, 740, 746, 776, 786, 757, 759, 780, 768, 743, 794, 743, 771, 756, 752, 748, 762, 837, 754, 726, 739, 791, 768, 755, 763, 805, 781, 758, 839] payload_rows_per_dst=[9260, 9196, 9246, 9240, 9268, 9192, 9270, 9496] dup_rows_per_dst=[3094, 3074, 2974, 2964, 3090, 3038, 2878, 3024] combine_rows_per_src=[9260, 9196, 9246, 9240, 9268, 9192, 9270, 9496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73734 payload_rows_remote=64424 dup_rows=24570 dup_rows_remote=21476 dedup_ratio=0.249939 router_tokens_per_expert=[754, 748, 737, 754, 792, 743, 764, 799, 788, 762, 815, 801, 743, 752, 776, 744, 745, 767, 772, 749, 788, 813, 763, 789, 782, 773, 738, 748, 780, 794, 790, 768, 779, 805, 761, 732, 742, 744, 737, 732, 706, 746, 801, 768, 788, 720, 786, 793, 784, 736, 810, 758, 788, 768, 773, 833, 759, 749, 798, 761, 795, 731, 729, 809] payload_rows_per_dst=[9310, 9534, 9238, 9254, 9030, 9426, 9328, 8614] dup_rows_per_dst=[3094, 3208, 3136, 3140, 2812, 3370, 3122, 2688] combine_rows_per_src=[9310, 9534, 9238, 9254, 9030, 9426, 9328, 8614] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=267157504 dispatch_meta_bytes=345128 combine_payload_bytes=267157504 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267502632 ingress_bytes=336363520 link_bytes=336363520 token_bytes=534660136 weight_movement_bytes=69206016 total_nvlink_bytes=603866152 weight_to_token_ratio=0.129439 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265708846e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=65012 dup_rows=24008 dup_rows_remote=20956 dedup_ratio=0.244222 router_tokens_per_expert=[765, 737, 752, 724, 753, 794, 762, 776, 762, 768, 735, 731, 770, 785, 816, 801, 780, 771, 793, 778, 754, 797, 744, 879, 775, 740, 761, 759, 749, 774, 731, 795, 797, 779, 730, 774, 794, 832, 771, 750, 783, 772, 793, 697, 728, 752, 774, 757, 761, 757, 776, 784, 792, 771, 813, 803, 743, 767, 765, 764, 732, 715, 715, 800] payload_rows_per_dst=[9254, 9284, 9426, 9262, 9426, 9142, 9426, 9076] dup_rows_per_dst=[2872, 3052, 3166, 2906, 3028, 2970, 3088, 2926] combine_rows_per_src=[9254, 9284, 9426, 9262, 9426, 9142, 9426, 9076] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=343696 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265846416 ingress_bytes=334708736 link_bytes=334708736 token_bytes=531349136 weight_movement_bytes=69206016 total_nvlink_bytes=600555152 weight_to_token_ratio=0.130246 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259482027e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74416 payload_rows_remote=64956 dup_rows=23888 dup_rows_remote=20834 dedup_ratio=0.243001 router_tokens_per_expert=[777, 767, 764, 828, 733, 772, 789, 743, 762, 777, 757, 773, 748, 792, 793, 748, 739, 720, 733, 793, 734, 769, 774, 781, 722, 781, 802, 758, 735, 775, 763, 814, 774, 763, 800, 772, 755, 756, 768, 809, 738, 774, 775, 771, 760, 767, 798, 780, 801, 742, 757, 779, 816, 718, 733, 781, 764, 786, 771, 776, 747, 744, 761, 800] payload_rows_per_dst=[9378, 9296, 9268, 9346, 9164, 9274, 9230, 9460] dup_rows_per_dst=[2968, 3004, 2818, 2954, 3000, 3052, 3038, 3054] combine_rows_per_src=[9378, 9296, 9268, 9346, 9164, 9274, 9230, 9460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74026 payload_rows_remote=64844 dup_rows=24278 dup_rows_remote=21154 dedup_ratio=0.246969 router_tokens_per_expert=[810, 805, 781, 783, 769, 751, 758, 742, 788, 728, 746, 770, 789, 770, 791, 771, 788, 808, 767, 777, 751, 762, 771, 754, 778, 800, 730, 741, 737, 777, 739, 776, 761, 765, 688, 797, 759, 767, 818, 781, 784, 780, 769, 766, 793, 798, 769, 750, 807, 795, 769, 730, 725, 753, 787, 762, 756, 771, 765, 743, 782, 754, 744, 756] payload_rows_per_dst=[9298, 9182, 9254, 9174, 9304, 9354, 9266, 9194] dup_rows_per_dst=[3100, 3124, 3102, 2982, 2968, 3064, 2990, 2948] combine_rows_per_src=[9298, 9182, 9254, 9174, 9304, 9354, 9266, 9194] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=64902 dup_rows=24136 dup_rows_remote=21196 dedup_ratio=0.245524 router_tokens_per_expert=[765, 774, 768, 744, 756, 779, 771, 780, 756, 764, 764, 796, 780, 809, 762, 742, 789, 762, 730, 771, 729, 773, 778, 771, 718, 784, 762, 799, 784, 724, 801, 815, 786, 830, 791, 757, 748, 779, 742, 793, 748, 773, 729, 757, 797, 790, 735, 729, 752, 792, 741, 769, 770, 785, 735, 710, 754, 767, 776, 768, 769, 781, 802, 797] payload_rows_per_dst=[9236, 9322, 9266, 9344, 9418, 9146, 9088, 9348] dup_rows_per_dst=[3038, 3024, 2940, 3030, 3034, 2970, 3020, 3080] combine_rows_per_src=[9236, 9322, 9266, 9344, 9418, 9146, 9088, 9348] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74346 payload_rows_remote=65024 dup_rows=23958 dup_rows_remote=20998 dedup_ratio=0.243713 router_tokens_per_expert=[726, 716, 786, 793, 808, 813, 758, 741, 779, 792, 756, 721, 739, 741, 764, 776, 768, 758, 766, 764, 779, 726, 791, 780, 757, 798, 819, 792, 741, 747, 779, 802, 786, 776, 749, 771, 778, 745, 797, 784, 735, 752, 764, 746, 760, 790, 784, 820, 785, 781, 788, 763, 730, 792, 754, 737, 778, 780, 752, 767, 747, 791, 743, 751] payload_rows_per_dst=[9322, 9422, 9312, 9652, 9352, 8946, 9232, 9108] dup_rows_per_dst=[2960, 3054, 2952, 3400, 3020, 2774, 3028, 2770] combine_rows_per_src=[9322, 9422, 9312, 9652, 9352, 8946, 9232, 9108] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265289728 dispatch_meta_bytes=342664 combine_payload_bytes=265289728 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265632392 ingress_bytes=369098752 link_bytes=369098752 token_bytes=530922120 weight_movement_bytes=103809024 total_nvlink_bytes=634731144 weight_to_token_ratio=0.195526 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388888889e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74424 payload_rows_remote=65258 dup_rows=23880 dup_rows_remote=20918 dedup_ratio=0.242920 router_tokens_per_expert=[753, 729, 801, 821, 746, 722, 729, 714, 745, 792, 769, 785, 700, 817, 784, 792, 787, 740, 781, 779, 759, 786, 804, 783, 757, 767, 773, 770, 778, 777, 733, 709, 797, 805, 796, 807, 730, 732, 768, 790, 773, 722, 784, 793, 757, 743, 746, 772, 737, 743, 744, 757, 795, 774, 804, 824, 811, 770, 752, 760, 785, 758, 746, 795] payload_rows_per_dst=[9248, 9368, 9372, 9166, 9418, 9208, 9296, 9348] dup_rows_per_dst=[2782, 3000, 3066, 2962, 3032, 2972, 3060, 3006] combine_rows_per_src=[9248, 9368, 9372, 9166, 9418, 9208, 9296, 9348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265830400 dispatch_meta_bytes=343784 combine_payload_bytes=265830400 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266174184 ingress_bytes=404242432 link_bytes=404242432 token_bytes=532004584 weight_movement_bytes=138412032 total_nvlink_bytes=670416616 weight_to_token_ratio=0.260171 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521131727e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=64972 dup_rows=24050 dup_rows_remote=20994 dedup_ratio=0.244649 router_tokens_per_expert=[735, 771, 789, 803, 734, 734, 773, 762, 758, 767, 733, 756, 776, 785, 761, 774, 765, 800, 765, 759, 775, 813, 763, 778, 769, 763, 780, 790, 749, 738, 780, 775, 764, 765, 786, 754, 716, 790, 726, 754, 793, 794, 756, 716, 766, 767, 785, 792, 750, 777, 764, 789, 780, 763, 764, 769, 776, 749, 680, 826, 771, 796, 813, 788] payload_rows_per_dst=[9250, 9222, 9408, 9304, 9214, 9282, 9266, 9308] dup_rows_per_dst=[2952, 2998, 3028, 2984, 2896, 3056, 3046, 3090] combine_rows_per_src=[9250, 9222, 9408, 9304, 9214, 9282, 9266, 9308] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266289152 dispatch_meta_bytes=343872 combine_payload_bytes=266289152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266633024 ingress_bytes=300892160 link_bytes=300892160 token_bytes=532922176 weight_movement_bytes=34603008 total_nvlink_bytes=567525184 weight_to_token_ratio=0.064931 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132232974e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=343160 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266402936 ingress_bytes=300662784 link_bytes=300662784 token_bytes=532462712 weight_movement_bytes=34603008 total_nvlink_bytes=567065720 weight_to_token_ratio=0.064987 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131369851e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73998 payload_rows_remote=64608 dup_rows=24306 dup_rows_remote=21192 dedup_ratio=0.247253 router_tokens_per_expert=[796, 777, 781, 787, 740, 786, 742, 787, 752, 747, 737, 762, 746, 742, 694, 786, 718, 778, 756, 763, 766, 735, 790, 730, 768, 794, 757, 801, 717, 790, 760, 778, 772, 760, 848, 770, 796, 789, 735, 782, 724, 763, 729, 719, 770, 792, 844, 750, 800, 754, 757, 753, 814, 760, 756, 741, 796, 765, 775, 798, 807, 774, 801, 795] payload_rows_per_dst=[9344, 9044, 9104, 9276, 9390, 9198, 9248, 9394] dup_rows_per_dst=[3048, 2888, 2968, 3054, 3114, 2984, 3022, 3228] combine_rows_per_src=[9344, 9044, 9104, 9276, 9390, 9198, 9248, 9394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=263880704 dispatch_meta_bytes=343600 combine_payload_bytes=263880704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264224304 ingress_bytes=298483712 link_bytes=298483712 token_bytes=528105008 weight_movement_bytes=34603008 total_nvlink_bytes=562708016 weight_to_token_ratio=0.065523 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123170178e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265601024 dispatch_meta_bytes=343992 combine_payload_bytes=265601024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265945016 ingress_bytes=300204032 link_bytes=300204032 token_bytes=531546040 weight_movement_bytes=34603008 total_nvlink_bytes=566149048 weight_to_token_ratio=0.065099 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129643604e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265838592 dispatch_meta_bytes=344392 combine_payload_bytes=265838592 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266182984 ingress_bytes=335044608 link_bytes=335044608 token_bytes=532021576 weight_movement_bytes=69206016 total_nvlink_bytes=601227592 weight_to_token_ratio=0.130081 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260745887e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=65158 dup_rows=23966 dup_rows_remote=20902 dedup_ratio=0.243795 router_tokens_per_expert=[759, 766, 783, 750, 754, 760, 745, 805, 773, 813, 775, 775, 762, 763, 785, 759, 819, 736, 718, 737, 775, 783, 765, 818, 720, 761, 770, 777, 747, 743, 763, 768, 814, 776, 752, 769, 752, 773, 737, 763, 797, 744, 758, 758, 741, 746, 764, 814, 801, 769, 752, 799, 783, 764, 771, 747, 775, 787, 793, 786, 734, 795, 787, 724] payload_rows_per_dst=[9306, 9334, 9294, 9258, 9288, 9180, 9292, 9386] dup_rows_per_dst=[2938, 3076, 3008, 2840, 2984, 3064, 3080, 2976] combine_rows_per_src=[9306, 9334, 9294, 9258, 9288, 9180, 9292, 9386] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=267296768 dispatch_meta_bytes=344704 combine_payload_bytes=267296768 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267641472 ingress_bytes=336502784 link_bytes=336502784 token_bytes=534938240 weight_movement_bytes=69206016 total_nvlink_bytes=604144256 weight_to_token_ratio=0.129372 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266232885e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74226 payload_rows_remote=65000 dup_rows=24078 dup_rows_remote=21026 dedup_ratio=0.244934 router_tokens_per_expert=[758, 776, 734, 756, 778, 741, 778, 827, 764, 767, 771, 745, 832, 691, 777, 750, 750, 815, 810, 765, 722, 730, 736, 732, 785, 788, 761, 775, 743, 793, 744, 790, 776, 796, 795, 729, 770, 810, 768, 728, 787, 786, 768, 763, 740, 734, 774, 795, 767, 761, 798, 779, 750, 749, 732, 803, 801, 820, 770, 801, 741, 713, 792, 772] payload_rows_per_dst=[9338, 9182, 9200, 9378, 9292, 9264, 9226, 9346] dup_rows_per_dst=[2958, 3012, 2920, 2980, 3052, 3030, 3052, 3074] combine_rows_per_src=[9338, 9182, 9200, 9378, 9292, 9264, 9226, 9346] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=344088 combine_payload_bytes=266338304 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266682392 ingress_bytes=370147328 link_bytes=370147328 token_bytes=533020696 weight_movement_bytes=103809024 total_nvlink_bytes=636829720 weight_to_token_ratio=0.194756 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392834596e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266125312 dispatch_meta_bytes=343864 combine_payload_bytes=266125312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266469176 ingress_bytes=300728320 link_bytes=300728320 token_bytes=532594488 weight_movement_bytes=34603008 total_nvlink_bytes=567197496 weight_to_token_ratio=0.064971 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131616458e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74424 payload_rows_remote=65258 dup_rows=23880 dup_rows_remote=20918 dedup_ratio=0.242920 router_tokens_per_expert=[753, 729, 801, 821, 746, 722, 729, 714, 745, 792, 769, 785, 700, 817, 784, 792, 787, 740, 781, 779, 759, 786, 804, 783, 757, 767, 773, 770, 778, 777, 733, 709, 797, 805, 796, 807, 730, 732, 768, 790, 773, 722, 784, 793, 757, 743, 746, 772, 737, 743, 744, 757, 795, 774, 804, 824, 811, 770, 752, 760, 785, 758, 746, 795] payload_rows_per_dst=[9248, 9368, 9372, 9166, 9418, 9208, 9296, 9348] dup_rows_per_dst=[2782, 3000, 3066, 2962, 3032, 2972, 3060, 3006] combine_rows_per_src=[9248, 9368, 9372, 9166, 9418, 9208, 9296, 9348] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=264634368 dispatch_meta_bytes=343200 combine_payload_bytes=264634368 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=264977568 ingress_bytes=333840384 link_bytes=333840384 token_bytes=529611936 weight_movement_bytes=69206016 total_nvlink_bytes=598817952 weight_to_token_ratio=0.130673 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256214489e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=64912 dup_rows=24060 dup_rows_remote=21082 dedup_ratio=0.244751 router_tokens_per_expert=[760, 786, 755, 761, 776, 746, 783, 804, 824, 765, 788, 708, 800, 728, 842, 726, 709, 733, 776, 780, 801, 769, 775, 798, 779, 757, 737, 765, 782, 753, 776, 745, 770, 729, 784, 728, 739, 753, 762, 743, 716, 726, 798, 824, 777, 772, 781, 767, 821, 742, 800, 751, 777, 810, 781, 762, 768, 737, 784, 786, 776, 782, 755, 764] payload_rows_per_dst=[9252, 8836, 9152, 9648, 9252, 9372, 9400, 9332] dup_rows_per_dst=[3090, 2622, 3042, 3444, 2764, 3032, 3088, 2978] combine_rows_per_src=[9252, 8836, 9152, 9648, 9252, 9372, 9400, 9332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266887168 dispatch_meta_bytes=344240 combine_payload_bytes=266887168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267231408 ingress_bytes=336093184 link_bytes=336093184 token_bytes=534118576 weight_movement_bytes=69206016 total_nvlink_bytes=603324592 weight_to_token_ratio=0.129571 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264691594e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73998 payload_rows_remote=64608 dup_rows=24306 dup_rows_remote=21192 dedup_ratio=0.247253 router_tokens_per_expert=[796, 777, 781, 787, 740, 786, 742, 787, 752, 747, 737, 762, 746, 742, 694, 786, 718, 778, 756, 763, 766, 735, 790, 730, 768, 794, 757, 801, 717, 790, 760, 778, 772, 760, 848, 770, 796, 789, 735, 782, 724, 763, 729, 719, 770, 792, 844, 750, 800, 754, 757, 753, 814, 760, 756, 741, 796, 765, 775, 798, 807, 774, 801, 795] payload_rows_per_dst=[9344, 9044, 9104, 9276, 9390, 9198, 9248, 9394] dup_rows_per_dst=[3048, 2888, 2968, 3054, 3114, 2984, 3022, 3228] combine_rows_per_src=[9344, 9044, 9104, 9276, 9390, 9198, 9248, 9394] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266240000 dispatch_meta_bytes=344104 combine_payload_bytes=266240000 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266584104 ingress_bytes=335446016 link_bytes=335446016 token_bytes=532824104 weight_movement_bytes=69206016 total_nvlink_bytes=602030120 weight_to_token_ratio=0.129885 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262256353e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74026 payload_rows_remote=64844 dup_rows=24278 dup_rows_remote=21154 dedup_ratio=0.246969 router_tokens_per_expert=[810, 805, 781, 783, 769, 751, 758, 742, 788, 728, 746, 770, 789, 770, 791, 771, 788, 808, 767, 777, 751, 762, 771, 754, 778, 800, 730, 741, 737, 777, 739, 776, 761, 765, 688, 797, 759, 767, 818, 781, 784, 780, 769, 766, 793, 798, 769, 750, 807, 795, 769, 730, 725, 753, 787, 762, 756, 771, 765, 743, 782, 754, 744, 756] payload_rows_per_dst=[9298, 9182, 9254, 9174, 9304, 9354, 9266, 9194] dup_rows_per_dst=[3100, 3124, 3102, 2982, 2968, 3064, 2990, 2948] combine_rows_per_src=[9298, 9182, 9254, 9174, 9304, 9354, 9266, 9194] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=267296768 dispatch_meta_bytes=344704 combine_payload_bytes=267296768 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267641472 ingress_bytes=336502784 link_bytes=336502784 token_bytes=534938240 weight_movement_bytes=69206016 total_nvlink_bytes=604144256 weight_to_token_ratio=0.129372 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266232885e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74266 payload_rows_remote=64768 dup_rows=24038 dup_rows_remote=20898 dedup_ratio=0.244527 router_tokens_per_expert=[731, 755, 815, 774, 769, 715, 789, 806, 758, 748, 757, 762, 756, 822, 763, 732, 763, 795, 757, 782, 782, 745, 753, 794, 809, 764, 777, 810, 749, 799, 768, 756, 751, 800, 779, 749, 756, 772, 778, 760, 730, 779, 747, 743, 741, 767, 736, 747, 805, 790, 727, 822, 738, 730, 795, 730, 763, 779, 728, 815, 794, 764, 804, 778] payload_rows_per_dst=[9348, 9194, 9244, 9296, 9256, 9062, 9368, 9498] dup_rows_per_dst=[2960, 3002, 3058, 2980, 3074, 2918, 2906, 3140] combine_rows_per_src=[9348, 9194, 9244, 9296, 9256, 9062, 9368, 9498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74266 payload_rows_remote=64982 dup_rows=24038 dup_rows_remote=21112 dedup_ratio=0.244527 router_tokens_per_expert=[744, 771, 786, 701, 773, 771, 766, 777, 762, 773, 772, 820, 785, 780, 867, 743, 750, 771, 812, 822, 784, 771, 776, 749, 746, 755, 737, 753, 739, 732, 782, 714, 809, 769, 713, 777, 745, 717, 830, 799, 794, 731, 734, 742, 779, 779, 724, 787, 764, 769, 772, 774, 793, 768, 818, 706, 735, 753, 753, 833, 789, 795, 748, 769] payload_rows_per_dst=[9284, 9582, 9382, 9060, 9372, 9074, 9188, 9324] dup_rows_per_dst=[2926, 3356, 3056, 2856, 3176, 2836, 2836, 2996] combine_rows_per_src=[9284, 9582, 9382, 9060, 9372, 9074, 9188, 9324] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=65122 dup_rows=23860 dup_rows_remote=20898 dedup_ratio=0.242716 router_tokens_per_expert=[771, 793, 780, 753, 800, 769, 792, 710, 787, 729, 768, 795, 790, 782, 740, 751, 746, 765, 763, 784, 740, 706, 787, 820, 757, 747, 794, 746, 779, 721, 796, 791, 801, 781, 796, 727, 785, 781, 748, 739, 718, 767, 776, 758, 771, 775, 744, 784, 765, 772, 776, 738, 801, 793, 784, 752, 755, 774, 760, 734, 832, 761, 768, 784] payload_rows_per_dst=[9326, 9322, 9236, 9298, 9346, 9256, 9328, 9332] dup_rows_per_dst=[3010, 2962, 2986, 2964, 2970, 2930, 3034, 3004] combine_rows_per_src=[9326, 9322, 9236, 9298, 9346, 9256, 9328, 9332] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74438 payload_rows_remote=64984 dup_rows=23866 dup_rows_remote=20736 dedup_ratio=0.242778 router_tokens_per_expert=[776, 718, 763, 759, 744, 772, 814, 749, 781, 731, 808, 770, 768, 725, 790, 718, 856, 782, 748, 811, 738, 770, 805, 782, 789, 785, 724, 751, 755, 804, 772, 744, 752, 747, 808, 745, 763, 736, 816, 738, 760, 761, 756, 803, 766, 781, 724, 813, 798, 751, 779, 767, 705, 783, 797, 740, 772, 780, 740, 763, 754, 788, 800, 764] payload_rows_per_dst=[9202, 9216, 9454, 9288, 9254, 9466, 9276, 9282] dup_rows_per_dst=[2988, 2966, 3130, 2960, 2956, 2862, 2964, 3040] combine_rows_per_src=[9202, 9216, 9454, 9288, 9254, 9466, 9276, 9282] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74408 payload_rows_remote=65106 dup_rows=23896 dup_rows_remote=20786 dedup_ratio=0.243083 router_tokens_per_expert=[753, 774, 777, 760, 771, 772, 787, 755, 767, 758, 706, 758, 753, 768, 799, 798, 792, 775, 745, 779, 770, 759, 784, 789, 795, 746, 778, 785, 742, 778, 808, 740, 762, 782, 736, 830, 788, 779, 779, 757, 762, 735, 806, 710, 787, 780, 760, 792, 735, 776, 742, 829, 791, 727, 758, 737, 750, 760, 775, 766, 747, 771, 759, 763] payload_rows_per_dst=[9302, 9630, 9146, 9312, 9308, 9364, 9144, 9202] dup_rows_per_dst=[3110, 3212, 2774, 2876, 2998, 2948, 2998, 2980] combine_rows_per_src=[9302, 9630, 9146, 9312, 9308, 9364, 9144, 9202] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=65158 dup_rows=23966 dup_rows_remote=20902 dedup_ratio=0.243795 router_tokens_per_expert=[759, 766, 783, 750, 754, 760, 745, 805, 773, 813, 775, 775, 762, 763, 785, 759, 819, 736, 718, 737, 775, 783, 765, 818, 720, 761, 770, 777, 747, 743, 763, 768, 814, 776, 752, 769, 752, 773, 737, 763, 797, 744, 758, 758, 741, 746, 764, 814, 801, 769, 752, 799, 783, 764, 771, 747, 775, 787, 793, 786, 734, 795, 787, 724] payload_rows_per_dst=[9306, 9334, 9294, 9258, 9288, 9180, 9292, 9386] dup_rows_per_dst=[2938, 3076, 3008, 2840, 2984, 3064, 3080, 2976] combine_rows_per_src=[9306, 9334, 9294, 9258, 9288, 9180, 9292, 9386] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265879552 dispatch_meta_bytes=343976 combine_payload_bytes=265879552 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266223528 ingress_bytes=335085568 link_bytes=335085568 token_bytes=532103080 weight_movement_bytes=69206016 total_nvlink_bytes=601309096 weight_to_token_ratio=0.130061 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260900016e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74536 payload_rows_remote=65362 dup_rows=23768 dup_rows_remote=20900 dedup_ratio=0.241781 router_tokens_per_expert=[764, 820, 771, 727, 778, 733, 786, 834, 695, 759, 792, 790, 779, 803, 759, 769, 770, 763, 789, 748, 766, 764, 772, 776, 752, 752, 773, 747, 770, 755, 756, 716, 786, 723, 781, 720, 772, 777, 816, 786, 770, 805, 779, 800, 819, 739, 747, 749, 748, 770, 792, 757, 801, 759, 737, 796, 771, 748, 814, 736, 793, 756, 744, 733] payload_rows_per_dst=[9400, 9292, 9382, 9174, 9338, 9412, 9318, 9220] dup_rows_per_dst=[3026, 3000, 2914, 2868, 2984, 3004, 3002, 2970] combine_rows_per_src=[9400, 9292, 9382, 9174, 9338, 9412, 9318, 9220] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=264634368 dispatch_meta_bytes=343200 combine_payload_bytes=264634368 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=264977568 ingress_bytes=333840384 link_bytes=333840384 token_bytes=529611936 weight_movement_bytes=69206016 total_nvlink_bytes=598817952 weight_to_token_ratio=0.130673 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256214489e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265601024 dispatch_meta_bytes=343992 combine_payload_bytes=265601024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265945016 ingress_bytes=300204032 link_bytes=300204032 token_bytes=531546040 weight_movement_bytes=34603008 total_nvlink_bytes=566149048 weight_to_token_ratio=0.065099 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129643604e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74152 payload_rows_remote=64906 dup_rows=24152 dup_rows_remote=21116 dedup_ratio=0.245687 router_tokens_per_expert=[728, 808, 743, 785, 769, 785, 773, 742, 781, 751, 789, 774, 820, 758, 768, 734, 769, 736, 754, 763, 762, 757, 759, 760, 786, 773, 763, 793, 755, 751, 772, 759, 809, 782, 717, 746, 793, 703, 791, 800, 776, 799, 744, 776, 782, 734, 806, 778, 768, 807, 727, 773, 775, 779, 735, 767, 787, 798, 725, 734, 749, 745, 805, 822] payload_rows_per_dst=[9248, 9300, 9200, 9290, 9246, 9326, 9218, 9324] dup_rows_per_dst=[3018, 3050, 2920, 3014, 3036, 3064, 3044, 3006] combine_rows_per_src=[9248, 9300, 9200, 9290, 9246, 9326, 9218, 9324] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265289728 dispatch_meta_bytes=342664 combine_payload_bytes=265289728 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265632392 ingress_bytes=369098752 link_bytes=369098752 token_bytes=530922120 weight_movement_bytes=103809024 total_nvlink_bytes=634731144 weight_to_token_ratio=0.195526 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388888889e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266166272 dispatch_meta_bytes=344376 combine_payload_bytes=266166272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266510648 ingress_bytes=300769280 link_bytes=300769280 token_bytes=532676920 weight_movement_bytes=34603008 total_nvlink_bytes=567279928 weight_to_token_ratio=0.064961 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131770587e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266739712 dispatch_meta_bytes=344080 combine_payload_bytes=266739712 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267083792 ingress_bytes=335945728 link_bytes=335945728 token_bytes=533823504 weight_movement_bytes=69206016 total_nvlink_bytes=603029520 weight_to_token_ratio=0.129642 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264136729e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266174464 dispatch_meta_bytes=342880 combine_payload_bytes=266174464 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266517344 ingress_bytes=369983488 link_bytes=369983488 token_bytes=532691808 weight_movement_bytes=103809024 total_nvlink_bytes=636500832 weight_to_token_ratio=0.194876 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392218079e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74402 payload_rows_remote=65136 dup_rows=23902 dup_rows_remote=20932 dedup_ratio=0.243144 router_tokens_per_expert=[767, 768, 809, 753, 786, 721, 765, 776, 753, 806, 758, 806, 798, 764, 744, 766, 766, 745, 752, 730, 781, 704, 796, 800, 740, 731, 774, 817, 811, 791, 760, 747, 757, 739, 825, 731, 799, 772, 783, 763, 759, 808, 755, 782, 790, 783, 750, 786, 724, 753, 832, 778, 725, 732, 792, 782, 742, 792, 790, 700, 749, 726, 770, 798] payload_rows_per_dst=[9318, 9364, 9190, 9344, 9336, 9336, 9266, 9248] dup_rows_per_dst=[2972, 3026, 2958, 2998, 3002, 3090, 2970, 2886] combine_rows_per_src=[9318, 9364, 9190, 9344, 9336, 9336, 9266, 9248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266674176 dispatch_meta_bytes=343568 combine_payload_bytes=266674176 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267017744 ingress_bytes=301277184 link_bytes=301277184 token_bytes=533691920 weight_movement_bytes=34603008 total_nvlink_bytes=568294928 weight_to_token_ratio=0.064837 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133681789e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266887168 dispatch_meta_bytes=344240 combine_payload_bytes=266887168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267231408 ingress_bytes=336093184 link_bytes=336093184 token_bytes=534118576 weight_movement_bytes=69206016 total_nvlink_bytes=603324592 weight_to_token_ratio=0.129571 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264691594e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74536 payload_rows_remote=65362 dup_rows=23768 dup_rows_remote=20900 dedup_ratio=0.241781 router_tokens_per_expert=[764, 820, 771, 727, 778, 733, 786, 834, 695, 759, 792, 790, 779, 803, 759, 769, 770, 763, 789, 748, 766, 764, 772, 776, 752, 752, 773, 747, 770, 755, 756, 716, 786, 723, 781, 720, 772, 777, 816, 786, 770, 805, 779, 800, 819, 739, 747, 749, 748, 770, 792, 757, 801, 759, 737, 796, 771, 748, 814, 736, 793, 756, 744, 733] payload_rows_per_dst=[9400, 9292, 9382, 9174, 9338, 9412, 9318, 9220] dup_rows_per_dst=[3026, 3000, 2914, 2868, 2984, 3004, 3002, 2970] combine_rows_per_src=[9400, 9292, 9382, 9174, 9338, 9412, 9318, 9220] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65050 dup_rows=23908 dup_rows_remote=20884 dedup_ratio=0.243205 router_tokens_per_expert=[794, 746, 750, 728, 770, 760, 750, 748, 769, 768, 803, 759, 830, 730, 779, 708, 792, 780, 766, 747, 739, 807, 778, 808, 805, 749, 769, 764, 796, 741, 767, 758, 802, 722, 786, 786, 784, 734, 762, 756, 781, 725, 834, 793, 752, 771, 754, 775, 741, 768, 799, 751, 843, 771, 774, 781, 740, 776, 742, 735, 799, 781, 727, 749] payload_rows_per_dst=[9150, 9374, 9352, 9330, 9258, 9346, 9428, 9158] dup_rows_per_dst=[2942, 2918, 3082, 2968, 3006, 3024, 3028, 2940] combine_rows_per_src=[9150, 9374, 9352, 9330, 9258, 9346, 9428, 9158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=267722752 dispatch_meta_bytes=345048 combine_payload_bytes=267722752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268067800 ingress_bytes=302325760 link_bytes=302325760 token_bytes=535790552 weight_movement_bytes=34603008 total_nvlink_bytes=570393560 weight_to_token_ratio=0.064583 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137627496e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74402 payload_rows_remote=65136 dup_rows=23902 dup_rows_remote=20932 dedup_ratio=0.243144 router_tokens_per_expert=[767, 768, 809, 753, 786, 721, 765, 776, 753, 806, 758, 806, 798, 764, 744, 766, 766, 745, 752, 730, 781, 704, 796, 800, 740, 731, 774, 817, 811, 791, 760, 747, 757, 739, 825, 731, 799, 772, 783, 763, 759, 808, 755, 782, 790, 783, 750, 786, 724, 753, 832, 778, 725, 732, 792, 782, 742, 792, 790, 700, 749, 726, 770, 798] payload_rows_per_dst=[9318, 9364, 9190, 9344, 9336, 9336, 9266, 9248] dup_rows_per_dst=[2972, 3026, 2958, 2998, 3002, 3090, 2970, 2886] combine_rows_per_src=[9318, 9364, 9190, 9344, 9336, 9336, 9266, 9248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265854976 dispatch_meta_bytes=344088 combine_payload_bytes=265854976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266199064 ingress_bytes=300457984 link_bytes=300457984 token_bytes=532054040 weight_movement_bytes=34603008 total_nvlink_bytes=566657048 weight_to_token_ratio=0.065037 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130599205e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74212 payload_rows_remote=64830 dup_rows=24092 dup_rows_remote=20944 dedup_ratio=0.245076 router_tokens_per_expert=[756, 750, 752, 757, 810, 722, 756, 767, 772, 793, 775, 741, 778, 742, 767, 763, 781, 744, 747, 799, 744, 744, 788, 781, 760, 714, 799, 790, 746, 796, 759, 795, 750, 771, 787, 759, 764, 757, 765, 762, 817, 750, 814, 780, 736, 788, 786, 724, 789, 794, 767, 782, 797, 752, 786, 747, 785, 782, 778, 762, 811, 714, 768, 740] payload_rows_per_dst=[9216, 8964, 9240, 9336, 9336, 9308, 9430, 9382] dup_rows_per_dst=[2924, 2816, 3016, 3088, 3020, 3054, 3026, 3148] combine_rows_per_src=[9216, 8964, 9240, 9336, 9336, 9308, 9430, 9382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266444800 dispatch_meta_bytes=343736 combine_payload_bytes=266444800 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266788536 ingress_bytes=335650816 link_bytes=335650816 token_bytes=533233336 weight_movement_bytes=69206016 total_nvlink_bytes=602439352 weight_to_token_ratio=0.129786 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263026999e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74152 payload_rows_remote=64906 dup_rows=24152 dup_rows_remote=21116 dedup_ratio=0.245687 router_tokens_per_expert=[728, 808, 743, 785, 769, 785, 773, 742, 781, 751, 789, 774, 820, 758, 768, 734, 769, 736, 754, 763, 762, 757, 759, 760, 786, 773, 763, 793, 755, 751, 772, 759, 809, 782, 717, 746, 793, 703, 791, 800, 776, 799, 744, 776, 782, 734, 806, 778, 768, 807, 727, 773, 775, 779, 735, 767, 787, 798, 725, 734, 749, 745, 805, 822] payload_rows_per_dst=[9248, 9300, 9200, 9290, 9246, 9326, 9218, 9324] dup_rows_per_dst=[3018, 3050, 2920, 3014, 3036, 3064, 3044, 3006] combine_rows_per_src=[9248, 9300, 9200, 9290, 9246, 9326, 9218, 9324] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266797056 dispatch_meta_bytes=344272 combine_payload_bytes=266797056 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267141328 ingress_bytes=336003072 link_bytes=336003072 token_bytes=533938384 weight_movement_bytes=69206016 total_nvlink_bytes=603144400 weight_to_token_ratio=0.129614 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264352509e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=267722752 dispatch_meta_bytes=345048 combine_payload_bytes=267722752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268067800 ingress_bytes=302325760 link_bytes=302325760 token_bytes=535790552 weight_movement_bytes=34603008 total_nvlink_bytes=570393560 weight_to_token_ratio=0.064583 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137627496e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=65122 dup_rows=23860 dup_rows_remote=20898 dedup_ratio=0.242716 router_tokens_per_expert=[771, 793, 780, 753, 800, 769, 792, 710, 787, 729, 768, 795, 790, 782, 740, 751, 746, 765, 763, 784, 740, 706, 787, 820, 757, 747, 794, 746, 779, 721, 796, 791, 801, 781, 796, 727, 785, 781, 748, 739, 718, 767, 776, 758, 771, 775, 744, 784, 765, 772, 776, 738, 801, 793, 784, 752, 755, 774, 760, 734, 832, 761, 768, 784] payload_rows_per_dst=[9326, 9322, 9236, 9298, 9346, 9256, 9328, 9332] dup_rows_per_dst=[3010, 2962, 2986, 2964, 2970, 2930, 3034, 3004] combine_rows_per_src=[9326, 9322, 9236, 9298, 9346, 9256, 9328, 9332] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=64912 dup_rows=24060 dup_rows_remote=21082 dedup_ratio=0.244751 router_tokens_per_expert=[760, 786, 755, 761, 776, 746, 783, 804, 824, 765, 788, 708, 800, 728, 842, 726, 709, 733, 776, 780, 801, 769, 775, 798, 779, 757, 737, 765, 782, 753, 776, 745, 770, 729, 784, 728, 739, 753, 762, 743, 716, 726, 798, 824, 777, 772, 781, 767, 821, 742, 800, 751, 777, 810, 781, 762, 768, 737, 784, 786, 776, 782, 755, 764] payload_rows_per_dst=[9252, 8836, 9152, 9648, 9252, 9372, 9400, 9332] dup_rows_per_dst=[3090, 2622, 3042, 3444, 2764, 3032, 3088, 2978] combine_rows_per_src=[9252, 8836, 9152, 9648, 9252, 9372, 9400, 9332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265543680 dispatch_meta_bytes=343096 combine_payload_bytes=265543680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265886776 ingress_bytes=300146688 link_bytes=300146688 token_bytes=531430456 weight_movement_bytes=34603008 total_nvlink_bytes=566033464 weight_to_token_ratio=0.065113 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129427823e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74538 payload_rows_remote=65088 dup_rows=23766 dup_rows_remote=20596 dedup_ratio=0.241760 router_tokens_per_expert=[785, 781, 807, 756, 755, 726, 749, 785, 805, 788, 737, 812, 805, 768, 829, 766, 770, 793, 747, 767, 789, 760, 795, 800, 749, 821, 749, 746, 768, 779, 772, 748, 762, 771, 784, 777, 746, 780, 724, 733, 758, 755, 759, 747, 745, 733, 797, 759, 787, 720, 756, 720, 785, 723, 776, 783, 749, 754, 799, 773, 805, 762, 763, 760] payload_rows_per_dst=[9346, 9450, 9296, 9334, 9286, 9224, 9184, 9418] dup_rows_per_dst=[2942, 3170, 3146, 2930, 2868, 2882, 2916, 2912] combine_rows_per_src=[9346, 9450, 9296, 9334, 9286, 9224, 9184, 9418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=64668 dup_rows=24118 dup_rows_remote=20750 dedup_ratio=0.245341 router_tokens_per_expert=[732, 813, 814, 755, 792, 745, 783, 787, 792, 815, 753, 765, 787, 742, 795, 760, 786, 766, 748, 744, 736, 771, 739, 746, 752, 740, 782, 783, 754, 766, 782, 786, 812, 814, 739, 763, 744, 770, 765, 724, 789, 740, 768, 764, 745, 802, 769, 788, 799, 781, 787, 778, 765, 755, 757, 768, 750, 772, 699, 725, 777, 779, 767, 786] payload_rows_per_dst=[9518, 9326, 9182, 9304, 9244, 9296, 9088, 9228] dup_rows_per_dst=[3368, 3104, 2960, 2986, 3018, 3022, 2778, 2882] combine_rows_per_src=[9518, 9326, 9182, 9304, 9244, 9296, 9088, 9228] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74424 payload_rows_remote=65094 dup_rows=23880 dup_rows_remote=21010 dedup_ratio=0.242920 router_tokens_per_expert=[751, 817, 815, 752, 737, 747, 790, 738, 752, 764, 793, 798, 782, 824, 763, 791, 750, 785, 768, 793, 752, 757, 759, 736, 780, 727, 746, 774, 796, 783, 773, 742, 740, 791, 781, 775, 773, 800, 759, 769, 755, 777, 714, 773, 813, 732, 804, 765, 780, 776, 744, 777, 781, 781, 723, 762, 719, 743, 810, 746, 815, 734, 757, 748] payload_rows_per_dst=[9296, 9440, 9330, 9306, 9342, 9308, 9194, 9208] dup_rows_per_dst=[2998, 3094, 2870, 2936, 3034, 2958, 3054, 2936] combine_rows_per_src=[9296, 9440, 9330, 9306, 9342, 9308, 9194, 9208] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73734 payload_rows_remote=64424 dup_rows=24570 dup_rows_remote=21476 dedup_ratio=0.249939 router_tokens_per_expert=[754, 748, 737, 754, 792, 743, 764, 799, 788, 762, 815, 801, 743, 752, 776, 744, 745, 767, 772, 749, 788, 813, 763, 789, 782, 773, 738, 748, 780, 794, 790, 768, 779, 805, 761, 732, 742, 744, 737, 732, 706, 746, 801, 768, 788, 720, 786, 793, 784, 736, 810, 758, 788, 768, 773, 833, 759, 749, 798, 761, 795, 731, 729, 809] payload_rows_per_dst=[9310, 9534, 9238, 9254, 9030, 9426, 9328, 8614] dup_rows_per_dst=[3094, 3208, 3136, 3140, 2812, 3370, 3122, 2688] combine_rows_per_src=[9310, 9534, 9238, 9254, 9030, 9426, 9328, 8614] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65050 dup_rows=23908 dup_rows_remote=20884 dedup_ratio=0.243205 router_tokens_per_expert=[794, 746, 750, 728, 770, 760, 750, 748, 769, 768, 803, 759, 830, 730, 779, 708, 792, 780, 766, 747, 739, 807, 778, 808, 805, 749, 769, 764, 796, 741, 767, 758, 802, 722, 786, 786, 784, 734, 762, 756, 781, 725, 834, 793, 752, 771, 754, 775, 741, 768, 799, 751, 843, 771, 774, 781, 740, 776, 742, 735, 799, 781, 727, 749] payload_rows_per_dst=[9150, 9374, 9352, 9330, 9258, 9346, 9428, 9158] dup_rows_per_dst=[2942, 2918, 3082, 2968, 3006, 3024, 3028, 2940] combine_rows_per_src=[9150, 9374, 9352, 9330, 9258, 9346, 9428, 9158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265854976 dispatch_meta_bytes=344088 combine_payload_bytes=265854976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266199064 ingress_bytes=300457984 link_bytes=300457984 token_bytes=532054040 weight_movement_bytes=34603008 total_nvlink_bytes=566657048 weight_to_token_ratio=0.065037 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130599205e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65034 dup_rows=23924 dup_rows_remote=20928 dedup_ratio=0.243368 router_tokens_per_expert=[752, 796, 729, 775, 778, 767, 797, 767, 765, 751, 805, 789, 738, 776, 750, 761, 794, 764, 754, 714, 753, 737, 701, 759, 799, 753, 786, 793, 786, 745, 714, 795, 793, 755, 772, 797, 774, 801, 740, 811, 787, 765, 752, 744, 795, 777, 743, 740, 761, 765, 739, 804, 741, 807, 808, 801, 723, 778, 788, 717, 813, 775, 775, 768] payload_rows_per_dst=[9314, 9264, 9134, 9346, 9420, 9242, 9398, 9262] dup_rows_per_dst=[3008, 3006, 2818, 2996, 3066, 2964, 3054, 3012] combine_rows_per_src=[9314, 9264, 9134, 9346, 9420, 9242, 9398, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=64982 dup_rows=24008 dup_rows_remote=21136 dedup_ratio=0.244222 router_tokens_per_expert=[772, 768, 766, 765, 772, 729, 742, 766, 744, 738, 792, 759, 745, 767, 797, 778, 769, 733, 799, 771, 762, 776, 800, 769, 743, 742, 704, 756, 819, 734, 807, 805, 762, 723, 784, 750, 750, 786, 762, 776, 719, 756, 801, 745, 828, 791, 779, 772, 771, 740, 770, 769, 769, 809, 776, 797, 752, 768, 746, 775, 771, 829, 808, 729] payload_rows_per_dst=[9152, 9234, 9410, 9268, 9314, 9250, 9344, 9324] dup_rows_per_dst=[3008, 3006, 2948, 2952, 2872, 3132, 3058, 3032] combine_rows_per_src=[9152, 9234, 9410, 9268, 9314, 9250, 9344, 9324] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266739712 dispatch_meta_bytes=344080 combine_payload_bytes=266739712 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267083792 ingress_bytes=335945728 link_bytes=335945728 token_bytes=533823504 weight_movement_bytes=69206016 total_nvlink_bytes=603029520 weight_to_token_ratio=0.129642 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264136729e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265879552 dispatch_meta_bytes=343976 combine_payload_bytes=265879552 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266223528 ingress_bytes=335085568 link_bytes=335085568 token_bytes=532103080 weight_movement_bytes=69206016 total_nvlink_bytes=601309096 weight_to_token_ratio=0.130061 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260900016e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=342736 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266943184 ingress_bytes=405012480 link_bytes=405012480 token_bytes=533543632 weight_movement_bytes=138412032 total_nvlink_bytes=671955664 weight_to_token_ratio=0.259420 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524029356e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=264880128 dispatch_meta_bytes=341672 combine_payload_bytes=264880128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265221800 ingress_bytes=299483136 link_bytes=299483136 token_bytes=530101928 weight_movement_bytes=34603008 total_nvlink_bytes=564704936 weight_to_token_ratio=0.065276 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126930930e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74034 payload_rows_remote=64786 dup_rows=24270 dup_rows_remote=21362 dedup_ratio=0.246887 router_tokens_per_expert=[763, 748, 849, 764, 790, 754, 810, 766, 781, 737, 776, 781, 726, 814, 783, 760, 782, 723, 744, 779, 793, 766, 782, 734, 793, 724, 726, 798, 763, 766, 742, 773, 741, 779, 760, 764, 786, 798, 754, 784, 743, 740, 757, 737, 786, 779, 760, 776, 780, 811, 787, 809, 733, 767, 737, 764, 796, 745, 789, 734, 772, 765, 784, 745] payload_rows_per_dst=[9376, 9226, 9224, 9150, 9248, 9248, 9322, 9240] dup_rows_per_dst=[3112, 3090, 2982, 3020, 3084, 2908, 3054, 3020] combine_rows_per_src=[9376, 9226, 9224, 9150, 9248, 9248, 9322, 9240] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266625024 dispatch_meta_bytes=344416 combine_payload_bytes=266625024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266969440 ingress_bytes=301228032 link_bytes=301228032 token_bytes=533594464 weight_movement_bytes=34603008 total_nvlink_bytes=568197472 weight_to_token_ratio=0.064849 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133496834e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266379264 dispatch_meta_bytes=343848 combine_payload_bytes=266379264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266723112 ingress_bytes=300982272 link_bytes=300982272 token_bytes=533102376 weight_movement_bytes=34603008 total_nvlink_bytes=567705384 weight_to_token_ratio=0.064909 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132572058e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64890 dup_rows=24114 dup_rows_remote=21090 dedup_ratio=0.245300 router_tokens_per_expert=[787, 749, 771, 748, 765, 734, 730, 777, 780, 746, 736, 785, 728, 794, 748, 752, 802, 810, 765, 789, 771, 771, 779, 773, 756, 758, 732, 732, 783, 781, 771, 799, 806, 778, 738, 734, 767, 760, 752, 775, 798, 755, 762, 771, 764, 794, 751, 752, 772, 781, 746, 769, 775, 791, 733, 795, 807, 805, 804, 820, 739, 746, 766, 744] payload_rows_per_dst=[9142, 9142, 9468, 9254, 9214, 9340, 9300, 9330] dup_rows_per_dst=[2980, 2996, 3052, 2970, 3006, 2954, 3024, 3132] combine_rows_per_src=[9142, 9142, 9468, 9254, 9214, 9340, 9300, 9330] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65034 dup_rows=23924 dup_rows_remote=20928 dedup_ratio=0.243368 router_tokens_per_expert=[752, 796, 729, 775, 778, 767, 797, 767, 765, 751, 805, 789, 738, 776, 750, 761, 794, 764, 754, 714, 753, 737, 701, 759, 799, 753, 786, 793, 786, 745, 714, 795, 793, 755, 772, 797, 774, 801, 740, 811, 787, 765, 752, 744, 795, 777, 743, 740, 761, 765, 739, 804, 741, 807, 808, 801, 723, 778, 788, 717, 813, 775, 775, 768] payload_rows_per_dst=[9314, 9264, 9134, 9346, 9420, 9242, 9398, 9262] dup_rows_per_dst=[3008, 3006, 2818, 2996, 3066, 2964, 3054, 3012] combine_rows_per_src=[9314, 9264, 9134, 9346, 9420, 9242, 9398, 9262] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=263880704 dispatch_meta_bytes=343600 combine_payload_bytes=263880704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264224304 ingress_bytes=298483712 link_bytes=298483712 token_bytes=528105008 weight_movement_bytes=34603008 total_nvlink_bytes=562708016 weight_to_token_ratio=0.065523 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123170178e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74156 payload_rows_remote=64636 dup_rows=24148 dup_rows_remote=20926 dedup_ratio=0.245646 router_tokens_per_expert=[769, 765, 741, 814, 757, 740, 799, 750, 742, 808, 746, 796, 741, 806, 743, 763, 795, 759, 777, 773, 752, 759, 785, 775, 786, 737, 791, 784, 736, 795, 842, 777, 760, 736, 731, 776, 756, 739, 750, 740, 793, 822, 781, 744, 787, 783, 753, 796, 731, 746, 785, 762, 740, 755, 769, 794, 800, 739, 803, 722, 774, 756, 768, 758] payload_rows_per_dst=[9196, 9004, 9128, 9368, 8882, 9602, 9456, 9520] dup_rows_per_dst=[2956, 2808, 2832, 3138, 2700, 3306, 3186, 3222] combine_rows_per_src=[9196, 9004, 9128, 9368, 8882, 9602, 9456, 9520] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266444800 dispatch_meta_bytes=343736 combine_payload_bytes=266444800 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266788536 ingress_bytes=335650816 link_bytes=335650816 token_bytes=533233336 weight_movement_bytes=69206016 total_nvlink_bytes=602439352 weight_to_token_ratio=0.129786 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263026999e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266166272 dispatch_meta_bytes=344472 combine_payload_bytes=266166272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266510744 ingress_bytes=300769280 link_bytes=300769280 token_bytes=532677016 weight_movement_bytes=34603008 total_nvlink_bytes=567280024 weight_to_token_ratio=0.064961 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131770587e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=64982 dup_rows=24008 dup_rows_remote=21136 dedup_ratio=0.244222 router_tokens_per_expert=[772, 768, 766, 765, 772, 729, 742, 766, 744, 738, 792, 759, 745, 767, 797, 778, 769, 733, 799, 771, 762, 776, 800, 769, 743, 742, 704, 756, 819, 734, 807, 805, 762, 723, 784, 750, 750, 786, 762, 776, 719, 756, 801, 745, 828, 791, 779, 772, 771, 740, 770, 769, 769, 809, 776, 797, 752, 768, 746, 775, 771, 829, 808, 729] payload_rows_per_dst=[9152, 9234, 9410, 9268, 9314, 9250, 9344, 9324] dup_rows_per_dst=[3008, 3006, 2948, 2952, 2872, 3132, 3058, 3032] combine_rows_per_src=[9152, 9234, 9410, 9268, 9314, 9250, 9344, 9324] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=344592 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265708048 ingress_bytes=299966464 link_bytes=299966464 token_bytes=531071504 weight_movement_bytes=34603008 total_nvlink_bytes=565674512 weight_to_token_ratio=0.065157 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128749655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265789440 dispatch_meta_bytes=343920 combine_payload_bytes=265789440 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266133360 ingress_bytes=369598464 link_bytes=369598464 token_bytes=531922800 weight_movement_bytes=103809024 total_nvlink_bytes=635731824 weight_to_token_ratio=0.195158 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390769265e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266379264 dispatch_meta_bytes=343848 combine_payload_bytes=266379264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266723112 ingress_bytes=300982272 link_bytes=300982272 token_bytes=533102376 weight_movement_bytes=34603008 total_nvlink_bytes=567705384 weight_to_token_ratio=0.064909 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132572058e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74538 payload_rows_remote=65088 dup_rows=23766 dup_rows_remote=20596 dedup_ratio=0.241760 router_tokens_per_expert=[785, 781, 807, 756, 755, 726, 749, 785, 805, 788, 737, 812, 805, 768, 829, 766, 770, 793, 747, 767, 789, 760, 795, 800, 749, 821, 749, 746, 768, 779, 772, 748, 762, 771, 784, 777, 746, 780, 724, 733, 758, 755, 759, 747, 745, 733, 797, 759, 787, 720, 756, 720, 785, 723, 776, 783, 749, 754, 799, 773, 805, 762, 763, 760] payload_rows_per_dst=[9346, 9450, 9296, 9334, 9286, 9224, 9184, 9418] dup_rows_per_dst=[2942, 3170, 3146, 2930, 2868, 2882, 2916, 2912] combine_rows_per_src=[9346, 9450, 9296, 9334, 9286, 9224, 9184, 9418] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74212 payload_rows_remote=64830 dup_rows=24092 dup_rows_remote=20944 dedup_ratio=0.245076 router_tokens_per_expert=[756, 750, 752, 757, 810, 722, 756, 767, 772, 793, 775, 741, 778, 742, 767, 763, 781, 744, 747, 799, 744, 744, 788, 781, 760, 714, 799, 790, 746, 796, 759, 795, 750, 771, 787, 759, 764, 757, 765, 762, 817, 750, 814, 780, 736, 788, 786, 724, 789, 794, 767, 782, 797, 752, 786, 747, 785, 782, 778, 762, 811, 714, 768, 740] payload_rows_per_dst=[9216, 8964, 9240, 9336, 9336, 9308, 9430, 9382] dup_rows_per_dst=[2924, 2816, 3016, 3088, 3020, 3054, 3026, 3148] combine_rows_per_src=[9216, 8964, 9240, 9336, 9336, 9308, 9430, 9382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=264749056 dispatch_meta_bytes=342248 combine_payload_bytes=264749056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265091304 ingress_bytes=299352064 link_bytes=299352064 token_bytes=529840360 weight_movement_bytes=34603008 total_nvlink_bytes=564443368 weight_to_token_ratio=0.065308 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126437717e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65058 dup_rows=23924 dup_rows_remote=20882 dedup_ratio=0.243368 router_tokens_per_expert=[691, 778, 777, 726, 759, 748, 737, 761, 781, 806, 759, 833, 761, 773, 741, 728, 732, 747, 756, 813, 755, 744, 780, 795, 798, 779, 769, 799, 771, 780, 763, 814, 780, 732, 787, 799, 746, 729, 793, 774, 798, 777, 805, 801, 768, 772, 722, 784, 741, 732, 766, 749, 826, 770, 705, 774, 758, 761, 753, 800, 743, 802, 797, 754] payload_rows_per_dst=[9204, 9322, 9270, 9436, 9264, 9340, 9210, 9334] dup_rows_per_dst=[2750, 3042, 2974, 3110, 3016, 3114, 2916, 3002] combine_rows_per_src=[9204, 9322, 9270, 9436, 9264, 9340, 9210, 9334] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266166272 dispatch_meta_bytes=344472 combine_payload_bytes=266166272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266510744 ingress_bytes=300769280 link_bytes=300769280 token_bytes=532677016 weight_movement_bytes=34603008 total_nvlink_bytes=567280024 weight_to_token_ratio=0.064961 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131770587e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74126 payload_rows_remote=64776 dup_rows=24178 dup_rows_remote=21014 dedup_ratio=0.245951 router_tokens_per_expert=[789, 749, 781, 813, 801, 777, 770, 777, 816, 780, 776, 739, 780, 760, 773, 773, 762, 717, 721, 760, 749, 818, 770, 774, 766, 805, 717, 734, 739, 782, 722, 768, 754, 797, 804, 750, 791, 772, 774, 790, 766, 767, 747, 778, 764, 730, 749, 788, 778, 758, 804, 777, 748, 786, 726, 825, 768, 754, 749, 768, 793, 748, 777, 714] payload_rows_per_dst=[9350, 9420, 9194, 9040, 9382, 9266, 9332, 9142] dup_rows_per_dst=[3164, 3096, 3044, 2808, 3082, 2912, 3072, 3000] combine_rows_per_src=[9350, 9420, 9194, 9040, 9382, 9266, 9332, 9142] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74350 payload_rows_remote=64940 dup_rows=23954 dup_rows_remote=20938 dedup_ratio=0.243673 router_tokens_per_expert=[768, 765, 776, 764, 754, 787, 743, 760, 801, 765, 786, 751, 744, 739, 756, 768, 779, 829, 767, 821, 768, 760, 760, 729, 752, 767, 736, 764, 843, 770, 743, 739, 778, 787, 789, 767, 772, 742, 832, 799, 718, 784, 758, 737, 776, 796, 732, 740, 782, 737, 736, 788, 794, 762, 737, 789, 787, 788, 815, 726, 767, 763, 796, 724] payload_rows_per_dst=[9274, 9248, 9410, 9322, 9398, 9132, 9216, 9350] dup_rows_per_dst=[2960, 2972, 3016, 2906, 3134, 2950, 3034, 2982] combine_rows_per_src=[9274, 9248, 9410, 9322, 9398, 9132, 9216, 9350] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74266 payload_rows_remote=64982 dup_rows=24038 dup_rows_remote=21112 dedup_ratio=0.244527 router_tokens_per_expert=[744, 771, 786, 701, 773, 771, 766, 777, 762, 773, 772, 820, 785, 780, 867, 743, 750, 771, 812, 822, 784, 771, 776, 749, 746, 755, 737, 753, 739, 732, 782, 714, 809, 769, 713, 777, 745, 717, 830, 799, 794, 731, 734, 742, 779, 779, 724, 787, 764, 769, 772, 774, 793, 768, 818, 706, 735, 753, 753, 833, 789, 795, 748, 769] payload_rows_per_dst=[9284, 9582, 9382, 9060, 9372, 9074, 9188, 9324] dup_rows_per_dst=[2926, 3356, 3056, 2856, 3176, 2836, 2836, 2996] combine_rows_per_src=[9284, 9582, 9382, 9060, 9372, 9074, 9188, 9324] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=64856 dup_rows=24118 dup_rows_remote=21000 dedup_ratio=0.245341 router_tokens_per_expert=[748, 759, 761, 770, 749, 785, 777, 817, 767, 738, 763, 797, 714, 792, 783, 798, 753, 756, 774, 774, 791, 730, 783, 738, 783, 763, 721, 796, 792, 758, 804, 807, 777, 701, 769, 806, 747, 780, 763, 811, 732, 789, 764, 743, 824, 756, 783, 759, 715, 811, 718, 721, 791, 803, 787, 785, 761, 778, 754, 741, 774, 759, 765, 744] payload_rows_per_dst=[9322, 9248, 9276, 9330, 9236, 9288, 9224, 9262] dup_rows_per_dst=[3010, 3056, 2922, 3118, 3072, 3012, 3038, 2890] combine_rows_per_src=[9322, 9248, 9276, 9330, 9236, 9288, 9224, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74034 payload_rows_remote=64786 dup_rows=24270 dup_rows_remote=21362 dedup_ratio=0.246887 router_tokens_per_expert=[763, 748, 849, 764, 790, 754, 810, 766, 781, 737, 776, 781, 726, 814, 783, 760, 782, 723, 744, 779, 793, 766, 782, 734, 793, 724, 726, 798, 763, 766, 742, 773, 741, 779, 760, 764, 786, 798, 754, 784, 743, 740, 757, 737, 786, 779, 760, 776, 780, 811, 787, 809, 733, 767, 737, 764, 796, 745, 789, 734, 772, 765, 784, 745] payload_rows_per_dst=[9376, 9226, 9224, 9150, 9248, 9248, 9322, 9240] dup_rows_per_dst=[3112, 3090, 2982, 3020, 3084, 2908, 3054, 3020] combine_rows_per_src=[9376, 9226, 9224, 9150, 9248, 9248, 9322, 9240] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=342736 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266943184 ingress_bytes=405012480 link_bytes=405012480 token_bytes=533543632 weight_movement_bytes=138412032 total_nvlink_bytes=671955664 weight_to_token_ratio=0.259420 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524029356e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74270 payload_rows_remote=64964 dup_rows=24034 dup_rows_remote=21014 dedup_ratio=0.244486 router_tokens_per_expert=[765, 717, 793, 766, 784, 749, 755, 737, 785, 764, 786, 762, 779, 779, 751, 762, 787, 793, 783, 739, 789, 780, 813, 797, 780, 735, 769, 749, 763, 782, 753, 774, 781, 772, 757, 747, 740, 771, 770, 825, 750, 769, 764, 776, 730, 769, 754, 801, 825, 744, 739, 714, 777, 746, 757, 789, 736, 759, 780, 733, 827, 781, 777, 772] payload_rows_per_dst=[9192, 9380, 9352, 9284, 9306, 9270, 9208, 9278] dup_rows_per_dst=[2940, 2956, 3210, 2926, 3020, 2956, 2974, 3052] combine_rows_per_src=[9192, 9380, 9352, 9284, 9306, 9270, 9208, 9278] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265543680 dispatch_meta_bytes=343096 combine_payload_bytes=265543680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265886776 ingress_bytes=300146688 link_bytes=300146688 token_bytes=531430456 weight_movement_bytes=34603008 total_nvlink_bytes=566033464 weight_to_token_ratio=0.065113 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129427823e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=343760 combine_payload_bytes=266477568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266821328 ingress_bytes=335683584 link_bytes=335683584 token_bytes=533298896 weight_movement_bytes=69206016 total_nvlink_bytes=602504912 weight_to_token_ratio=0.129770 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263150302e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=343512 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266337752 ingress_bytes=335200256 link_bytes=335200256 token_bytes=532331992 weight_movement_bytes=69206016 total_nvlink_bytes=601538008 weight_to_token_ratio=0.130005 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261331577e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74298 payload_rows_remote=65086 dup_rows=24006 dup_rows_remote=21032 dedup_ratio=0.244202 router_tokens_per_expert=[775, 762, 782, 777, 807, 782, 788, 726, 782, 745, 797, 742, 791, 783, 747, 792, 762, 760, 754, 738, 780, 800, 751, 724, 766, 786, 751, 801, 764, 787, 735, 768, 779, 784, 742, 772, 788, 748, 767, 755, 781, 777, 730, 791, 747, 764, 771, 732, 819, 732, 721, 767, 753, 800, 768, 809, 784, 829, 724, 793, 794, 727, 745, 754] payload_rows_per_dst=[9394, 9302, 9116, 9300, 9288, 9212, 9328, 9358] dup_rows_per_dst=[3004, 3056, 3022, 3016, 2982, 2974, 3010, 2942] combine_rows_per_src=[9394, 9302, 9116, 9300, 9288, 9212, 9328, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265650176 dispatch_meta_bytes=343424 combine_payload_bytes=265650176 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265993600 ingress_bytes=369459200 link_bytes=369459200 token_bytes=531643776 weight_movement_bytes=103809024 total_nvlink_bytes=635452800 weight_to_token_ratio=0.195260 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390245226e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265322496 dispatch_meta_bytes=343160 combine_payload_bytes=265322496 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265665656 ingress_bytes=334528512 link_bytes=334528512 token_bytes=530988152 weight_movement_bytes=69206016 total_nvlink_bytes=600194168 weight_to_token_ratio=0.130334 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258803859e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74382 payload_rows_remote=65172 dup_rows=23922 dup_rows_remote=21070 dedup_ratio=0.243347 router_tokens_per_expert=[798, 750, 738, 797, 805, 805, 804, 752, 761, 783, 790, 784, 765, 770, 806, 759, 803, 755, 749, 778, 781, 785, 794, 790, 778, 746, 763, 769, 778, 801, 757, 782, 810, 741, 792, 744, 795, 790, 762, 806, 711, 715, 729, 781, 738, 762, 781, 783, 738, 734, 813, 772, 729, 759, 728, 758, 754, 702, 730, 770, 775, 733, 805, 736] payload_rows_per_dst=[9370, 9328, 9426, 9340, 9440, 9088, 9210, 9180] dup_rows_per_dst=[3128, 3108, 3044, 3008, 3040, 2912, 2852, 2830] combine_rows_per_src=[9370, 9328, 9426, 9340, 9440, 9088, 9210, 9180] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=64856 dup_rows=24118 dup_rows_remote=21000 dedup_ratio=0.245341 router_tokens_per_expert=[748, 759, 761, 770, 749, 785, 777, 817, 767, 738, 763, 797, 714, 792, 783, 798, 753, 756, 774, 774, 791, 730, 783, 738, 783, 763, 721, 796, 792, 758, 804, 807, 777, 701, 769, 806, 747, 780, 763, 811, 732, 789, 764, 743, 824, 756, 783, 759, 715, 811, 718, 721, 791, 803, 787, 785, 761, 778, 754, 741, 774, 759, 765, 744] payload_rows_per_dst=[9322, 9248, 9276, 9330, 9236, 9288, 9224, 9262] dup_rows_per_dst=[3010, 3056, 2922, 3118, 3072, 3012, 3038, 2890] combine_rows_per_src=[9322, 9248, 9276, 9330, 9236, 9288, 9224, 9262] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266166272 dispatch_meta_bytes=344376 combine_payload_bytes=266166272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266510648 ingress_bytes=300769280 link_bytes=300769280 token_bytes=532676920 weight_movement_bytes=34603008 total_nvlink_bytes=567279928 weight_to_token_ratio=0.064961 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131770587e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=344592 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265708048 ingress_bytes=299966464 link_bytes=299966464 token_bytes=531071504 weight_movement_bytes=34603008 total_nvlink_bytes=565674512 weight_to_token_ratio=0.065157 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128749655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=343912 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266436456 ingress_bytes=335298560 link_bytes=335298560 token_bytes=532529000 weight_movement_bytes=69206016 total_nvlink_bytes=601735016 weight_to_token_ratio=0.129957 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261701488e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74088 payload_rows_remote=64818 dup_rows=24216 dup_rows_remote=21052 dedup_ratio=0.246338 router_tokens_per_expert=[746, 731, 778, 826, 775, 752, 771, 766, 771, 755, 778, 772, 716, 766, 807, 805, 780, 738, 740, 721, 788, 792, 715, 791, 738, 764, 793, 772, 769, 734, 742, 775, 809, 792, 785, 741, 768, 788, 731, 771, 792, 754, 794, 748, 732, 737, 749, 791, 795, 769, 786, 752, 788, 787, 767, 765, 760, 734, 777, 805, 773, 798, 785, 762] payload_rows_per_dst=[9300, 9342, 9162, 9204, 9240, 9264, 9306, 9270] dup_rows_per_dst=[2990, 2998, 2968, 2970, 3084, 2930, 3112, 3164] combine_rows_per_src=[9300, 9342, 9162, 9204, 9240, 9264, 9306, 9270] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74270 payload_rows_remote=64964 dup_rows=24034 dup_rows_remote=21014 dedup_ratio=0.244486 router_tokens_per_expert=[765, 717, 793, 766, 784, 749, 755, 737, 785, 764, 786, 762, 779, 779, 751, 762, 787, 793, 783, 739, 789, 780, 813, 797, 780, 735, 769, 749, 763, 782, 753, 774, 781, 772, 757, 747, 740, 771, 770, 825, 750, 769, 764, 776, 730, 769, 754, 801, 825, 744, 739, 714, 777, 746, 757, 789, 736, 759, 780, 733, 827, 781, 777, 772] payload_rows_per_dst=[9192, 9380, 9352, 9284, 9306, 9270, 9208, 9278] dup_rows_per_dst=[2940, 2956, 3210, 2926, 3020, 2956, 2974, 3052] combine_rows_per_src=[9192, 9380, 9352, 9284, 9306, 9270, 9208, 9278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266592256 dispatch_meta_bytes=344472 combine_payload_bytes=266592256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266936728 ingress_bytes=301195264 link_bytes=301195264 token_bytes=533528984 weight_movement_bytes=34603008 total_nvlink_bytes=568131992 weight_to_token_ratio=0.064857 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133373530e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266944512 dispatch_meta_bytes=344968 combine_payload_bytes=266944512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267289480 ingress_bytes=301547520 link_bytes=301547520 token_bytes=534233992 weight_movement_bytes=34603008 total_nvlink_bytes=568837000 weight_to_token_ratio=0.064771 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134699041e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265650176 dispatch_meta_bytes=343424 combine_payload_bytes=265650176 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265993600 ingress_bytes=369459200 link_bytes=369459200 token_bytes=531643776 weight_movement_bytes=103809024 total_nvlink_bytes=635452800 weight_to_token_ratio=0.195260 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390245226e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65058 dup_rows=23924 dup_rows_remote=20882 dedup_ratio=0.243368 router_tokens_per_expert=[691, 778, 777, 726, 759, 748, 737, 761, 781, 806, 759, 833, 761, 773, 741, 728, 732, 747, 756, 813, 755, 744, 780, 795, 798, 779, 769, 799, 771, 780, 763, 814, 780, 732, 787, 799, 746, 729, 793, 774, 798, 777, 805, 801, 768, 772, 722, 784, 741, 732, 766, 749, 826, 770, 705, 774, 758, 761, 753, 800, 743, 802, 797, 754] payload_rows_per_dst=[9204, 9322, 9270, 9436, 9264, 9340, 9210, 9334] dup_rows_per_dst=[2750, 3042, 2974, 3110, 3016, 3114, 2916, 3002] combine_rows_per_src=[9204, 9322, 9270, 9436, 9264, 9340, 9210, 9334] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74156 payload_rows_remote=64636 dup_rows=24148 dup_rows_remote=20926 dedup_ratio=0.245646 router_tokens_per_expert=[769, 765, 741, 814, 757, 740, 799, 750, 742, 808, 746, 796, 741, 806, 743, 763, 795, 759, 777, 773, 752, 759, 785, 775, 786, 737, 791, 784, 736, 795, 842, 777, 760, 736, 731, 776, 756, 739, 750, 740, 793, 822, 781, 744, 787, 783, 753, 796, 731, 746, 785, 762, 740, 755, 769, 794, 800, 739, 803, 722, 774, 756, 768, 758] payload_rows_per_dst=[9196, 9004, 9128, 9368, 8882, 9602, 9456, 9520] dup_rows_per_dst=[2956, 2808, 2832, 3138, 2700, 3306, 3186, 3222] combine_rows_per_src=[9196, 9004, 9128, 9368, 8882, 9602, 9456, 9520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265494528 dispatch_meta_bytes=343480 combine_payload_bytes=265494528 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265838008 ingress_bytes=334700544 link_bytes=334700544 token_bytes=531332536 weight_movement_bytes=69206016 total_nvlink_bytes=600538552 weight_to_token_ratio=0.130250 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259451201e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74092 payload_rows_remote=64952 dup_rows=24212 dup_rows_remote=21254 dedup_ratio=0.246297 router_tokens_per_expert=[738, 742, 698, 743, 758, 753, 798, 792, 762, 723, 794, 778, 789, 729, 712, 762, 779, 798, 789, 797, 753, 780, 771, 770, 777, 775, 791, 759, 818, 788, 730, 779, 776, 754, 733, 804, 764, 774, 783, 777, 806, 739, 785, 826, 781, 770, 749, 761, 753, 753, 702, 797, 792, 771, 757, 833, 725, 747, 774, 791, 754, 717, 765, 814] payload_rows_per_dst=[9162, 9140, 9304, 9256, 9360, 9360, 9294, 9216] dup_rows_per_dst=[2882, 2958, 3170, 3178, 2970, 3074, 3022, 2958] combine_rows_per_src=[9162, 9140, 9304, 9256, 9360, 9360, 9294, 9216] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74470 payload_rows_remote=65194 dup_rows=23834 dup_rows_remote=20912 dedup_ratio=0.242452 router_tokens_per_expert=[762, 774, 770, 794, 749, 787, 791, 768, 813, 748, 787, 783, 725, 800, 776, 747, 719, 769, 790, 768, 733, 774, 769, 777, 769, 782, 749, 755, 744, 792, 774, 727, 704, 737, 764, 742, 760, 803, 774, 773, 799, 770, 762, 743, 715, 759, 780, 731, 760, 740, 839, 765, 771, 810, 770, 756, 781, 743, 784, 799, 749, 812, 827, 765] payload_rows_per_dst=[9392, 9426, 9276, 9250, 9190, 9134, 9386, 9416] dup_rows_per_dst=[2998, 2932, 2922, 2934, 2924, 2984, 3036, 3104] combine_rows_per_src=[9392, 9426, 9276, 9250, 9190, 9134, 9386, 9416] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=64668 dup_rows=24118 dup_rows_remote=20750 dedup_ratio=0.245341 router_tokens_per_expert=[732, 813, 814, 755, 792, 745, 783, 787, 792, 815, 753, 765, 787, 742, 795, 760, 786, 766, 748, 744, 736, 771, 739, 746, 752, 740, 782, 783, 754, 766, 782, 786, 812, 814, 739, 763, 744, 770, 765, 724, 789, 740, 768, 764, 745, 802, 769, 788, 799, 781, 787, 778, 765, 755, 757, 768, 750, 772, 699, 725, 777, 779, 767, 786] payload_rows_per_dst=[9518, 9326, 9182, 9304, 9244, 9296, 9088, 9228] dup_rows_per_dst=[3368, 3104, 2960, 2986, 3018, 3022, 2778, 2882] combine_rows_per_src=[9518, 9326, 9182, 9304, 9244, 9296, 9088, 9228] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=343912 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266436456 ingress_bytes=335298560 link_bytes=335298560 token_bytes=532529000 weight_movement_bytes=69206016 total_nvlink_bytes=601735016 weight_to_token_ratio=0.129957 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261701488e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74036 payload_rows_remote=64848 dup_rows=24268 dup_rows_remote=21246 dedup_ratio=0.246867 router_tokens_per_expert=[797, 779, 799, 784, 784, 774, 770, 764, 743, 751, 780, 794, 769, 749, 778, 752, 774, 789, 798, 792, 751, 766, 736, 775, 783, 798, 731, 749, 763, 770, 779, 732, 737, 778, 790, 762, 726, 781, 767, 800, 791, 752, 716, 785, 749, 803, 772, 785, 781, 749, 815, 729, 789, 768, 803, 715, 711, 791, 774, 725, 770, 739, 762, 784] payload_rows_per_dst=[9366, 9236, 9250, 9188, 9336, 9216, 9304, 9140] dup_rows_per_dst=[3136, 2996, 3112, 3022, 2946, 3090, 2994, 2972] combine_rows_per_src=[9366, 9236, 9250, 9188, 9336, 9216, 9304, 9140] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74406 payload_rows_remote=65064 dup_rows=23898 dup_rows_remote=20930 dedup_ratio=0.243103 router_tokens_per_expert=[776, 751, 752, 782, 741, 771, 774, 782, 769, 781, 815, 721, 783, 742, 795, 770, 732, 792, 708, 816, 742, 740, 728, 809, 773, 731, 776, 790, 727, 744, 765, 731, 779, 780, 765, 761, 752, 785, 844, 735, 754, 759, 759, 795, 753, 735, 785, 792, 803, 779, 770, 745, 764, 795, 774, 729, 762, 794, 757, 814, 781, 786, 785, 772] payload_rows_per_dst=[9342, 9520, 9364, 9196, 9364, 9500, 8850, 9270] dup_rows_per_dst=[2968, 3088, 3118, 2920, 2964, 3220, 2736, 2884] combine_rows_per_src=[9342, 9520, 9364, 9196, 9364, 9500, 8850, 9270] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=343760 combine_payload_bytes=266477568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266821328 ingress_bytes=335683584 link_bytes=335683584 token_bytes=533298896 weight_movement_bytes=69206016 total_nvlink_bytes=602504912 weight_to_token_ratio=0.129770 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263150302e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74298 payload_rows_remote=65086 dup_rows=24006 dup_rows_remote=21032 dedup_ratio=0.244202 router_tokens_per_expert=[775, 762, 782, 777, 807, 782, 788, 726, 782, 745, 797, 742, 791, 783, 747, 792, 762, 760, 754, 738, 780, 800, 751, 724, 766, 786, 751, 801, 764, 787, 735, 768, 779, 784, 742, 772, 788, 748, 767, 755, 781, 777, 730, 791, 747, 764, 771, 732, 819, 732, 721, 767, 753, 800, 768, 809, 784, 829, 724, 793, 794, 727, 745, 754] payload_rows_per_dst=[9394, 9302, 9116, 9300, 9288, 9212, 9328, 9358] dup_rows_per_dst=[3004, 3056, 3022, 3016, 2982, 2974, 3010, 2942] combine_rows_per_src=[9394, 9302, 9116, 9300, 9288, 9212, 9328, 9358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=264749056 dispatch_meta_bytes=342248 combine_payload_bytes=264749056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265091304 ingress_bytes=299352064 link_bytes=299352064 token_bytes=529840360 weight_movement_bytes=34603008 total_nvlink_bytes=564443368 weight_to_token_ratio=0.065308 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126437717e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=64970 dup_rows=24002 dup_rows_remote=21046 dedup_ratio=0.244161 router_tokens_per_expert=[755, 790, 751, 683, 759, 789, 787, 755, 769, 800, 769, 758, 749, 782, 790, 750, 774, 809, 736, 768, 761, 754, 758, 755, 778, 769, 800, 769, 804, 786, 806, 794, 802, 725, 766, 765, 763, 800, 737, 786, 805, 694, 793, 723, 729, 750, 750, 776, 735, 816, 786, 783, 755, 802, 708, 781, 791, 799, 766, 730, 774, 750, 784, 771] payload_rows_per_dst=[9176, 9268, 9306, 9414, 9332, 9180, 9264, 9362] dup_rows_per_dst=[2962, 3066, 2924, 3198, 2956, 2860, 3068, 2968] combine_rows_per_src=[9176, 9268, 9306, 9414, 9332, 9180, 9264, 9362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266043392 dispatch_meta_bytes=344824 combine_payload_bytes=266043392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266388216 ingress_bytes=300646400 link_bytes=300646400 token_bytes=532431608 weight_movement_bytes=34603008 total_nvlink_bytes=567034616 weight_to_token_ratio=0.064991 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131308199e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74110 payload_rows_remote=64806 dup_rows=24194 dup_rows_remote=21074 dedup_ratio=0.246114 router_tokens_per_expert=[776, 787, 812, 726, 733, 759, 739, 793, 743, 769, 817, 762, 773, 784, 752, 799, 734, 810, 765, 768, 781, 753, 768, 791, 778, 784, 776, 772, 826, 757, 754, 770, 783, 693, 769, 774, 759, 781, 753, 756, 762, 710, 793, 757, 791, 810, 805, 784, 748, 779, 775, 714, 756, 747, 756, 786, 738, 814, 752, 741, 787, 775, 727, 766] payload_rows_per_dst=[9204, 9328, 9326, 9320, 9160, 9304, 9178, 9290] dup_rows_per_dst=[3046, 3070, 3014, 3114, 2976, 3120, 2944, 2910] combine_rows_per_src=[9204, 9328, 9326, 9320, 9160, 9304, 9178, 9290] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=267034624 dispatch_meta_bytes=344424 combine_payload_bytes=267034624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267379048 ingress_bytes=301637632 link_bytes=301637632 token_bytes=534413672 weight_movement_bytes=34603008 total_nvlink_bytes=569016680 weight_to_token_ratio=0.064749 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135038125e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=264880128 dispatch_meta_bytes=341672 combine_payload_bytes=264880128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265221800 ingress_bytes=299483136 link_bytes=299483136 token_bytes=530101928 weight_movement_bytes=34603008 total_nvlink_bytes=564704936 weight_to_token_ratio=0.065276 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126930930e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74350 payload_rows_remote=64964 dup_rows=23954 dup_rows_remote=20746 dedup_ratio=0.243673 router_tokens_per_expert=[768, 733, 786, 741, 756, 773, 791, 768, 773, 712, 723, 774, 727, 765, 738, 776, 772, 762, 766, 742, 751, 793, 770, 789, 772, 802, 753, 787, 711, 809, 746, 749, 740, 838, 755, 767, 741, 768, 784, 791, 793, 732, 775, 792, 773, 771, 775, 742, 767, 786, 781, 782, 795, 809, 785, 792, 804, 720, 714, 777, 799, 778, 761, 787] payload_rows_per_dst=[9272, 9136, 9268, 9306, 9356, 9338, 9386, 9288] dup_rows_per_dst=[2960, 2840, 3022, 2952, 3012, 2968, 3208, 2992] combine_rows_per_src=[9272, 9136, 9268, 9306, 9356, 9338, 9386, 9288] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74036 payload_rows_remote=64848 dup_rows=24268 dup_rows_remote=21246 dedup_ratio=0.246867 router_tokens_per_expert=[797, 779, 799, 784, 784, 774, 770, 764, 743, 751, 780, 794, 769, 749, 778, 752, 774, 789, 798, 792, 751, 766, 736, 775, 783, 798, 731, 749, 763, 770, 779, 732, 737, 778, 790, 762, 726, 781, 767, 800, 791, 752, 716, 785, 749, 803, 772, 785, 781, 749, 815, 729, 789, 768, 803, 715, 711, 791, 774, 725, 770, 739, 762, 784] payload_rows_per_dst=[9366, 9236, 9250, 9188, 9336, 9216, 9304, 9140] dup_rows_per_dst=[3136, 2996, 3112, 3022, 2946, 3090, 2994, 2972] combine_rows_per_src=[9366, 9236, 9250, 9188, 9336, 9216, 9304, 9140] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266502144 dispatch_meta_bytes=343976 combine_payload_bytes=266502144 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266846120 ingress_bytes=301105152 link_bytes=301105152 token_bytes=533348264 weight_movement_bytes=34603008 total_nvlink_bytes=567951272 weight_to_token_ratio=0.064879 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133034446e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265617408 dispatch_meta_bytes=344376 combine_payload_bytes=265617408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265961784 ingress_bytes=300220416 link_bytes=300220416 token_bytes=531579192 weight_movement_bytes=34603008 total_nvlink_bytes=566182200 weight_to_token_ratio=0.065095 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129705256e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74300 payload_rows_remote=64882 dup_rows=24004 dup_rows_remote=20978 dedup_ratio=0.244181 router_tokens_per_expert=[757, 775, 773, 719, 745, 801, 766, 771, 768, 780, 696, 784, 792, 785, 756, 802, 736, 789, 741, 796, 769, 772, 792, 776, 771, 757, 760, 762, 778, 716, 767, 799, 764, 715, 732, 750, 794, 742, 804, 771, 768, 741, 802, 765, 770, 775, 788, 821, 735, 776, 762, 803, 768, 760, 781, 764, 781, 767, 775, 764, 756, 803, 772, 732] payload_rows_per_dst=[9242, 8808, 9340, 9338, 9398, 9456, 9300, 9418] dup_rows_per_dst=[2968, 2632, 3002, 2882, 3252, 3240, 3002, 3026] combine_rows_per_src=[9242, 8808, 9340, 9338, 9398, 9456, 9300, 9418] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266592256 dispatch_meta_bytes=344472 combine_payload_bytes=266592256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266936728 ingress_bytes=301195264 link_bytes=301195264 token_bytes=533528984 weight_movement_bytes=34603008 total_nvlink_bytes=568131992 weight_to_token_ratio=0.064857 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133373530e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266117120 dispatch_meta_bytes=344064 combine_payload_bytes=266117120 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266461184 ingress_bytes=404529152 link_bytes=404529152 token_bytes=532578304 weight_movement_bytes=138412032 total_nvlink_bytes=670990336 weight_to_token_ratio=0.259890 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522210632e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265445376 dispatch_meta_bytes=343520 combine_payload_bytes=265445376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265788896 ingress_bytes=300048384 link_bytes=300048384 token_bytes=531234272 weight_movement_bytes=34603008 total_nvlink_bytes=565837280 weight_to_token_ratio=0.065137 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129057913e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=64970 dup_rows=24002 dup_rows_remote=21046 dedup_ratio=0.244161 router_tokens_per_expert=[755, 790, 751, 683, 759, 789, 787, 755, 769, 800, 769, 758, 749, 782, 790, 750, 774, 809, 736, 768, 761, 754, 758, 755, 778, 769, 800, 769, 804, 786, 806, 794, 802, 725, 766, 765, 763, 800, 737, 786, 805, 694, 793, 723, 729, 750, 750, 776, 735, 816, 786, 783, 755, 802, 708, 781, 791, 799, 766, 730, 774, 750, 784, 771] payload_rows_per_dst=[9176, 9268, 9306, 9414, 9332, 9180, 9264, 9362] dup_rows_per_dst=[2962, 3066, 2924, 3198, 2956, 2860, 3068, 2968] combine_rows_per_src=[9176, 9268, 9306, 9414, 9332, 9180, 9264, 9362] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=342840 combine_payload_bytes=266092544 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266435384 ingress_bytes=369901568 link_bytes=369901568 token_bytes=532527928 weight_movement_bytes=103809024 total_nvlink_bytes=636336952 weight_to_token_ratio=0.194936 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391909821e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265617408 dispatch_meta_bytes=344376 combine_payload_bytes=265617408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265961784 ingress_bytes=300220416 link_bytes=300220416 token_bytes=531579192 weight_movement_bytes=34603008 total_nvlink_bytes=566182200 weight_to_token_ratio=0.065095 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129705256e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74092 payload_rows_remote=64952 dup_rows=24212 dup_rows_remote=21254 dedup_ratio=0.246297 router_tokens_per_expert=[738, 742, 698, 743, 758, 753, 798, 792, 762, 723, 794, 778, 789, 729, 712, 762, 779, 798, 789, 797, 753, 780, 771, 770, 777, 775, 791, 759, 818, 788, 730, 779, 776, 754, 733, 804, 764, 774, 783, 777, 806, 739, 785, 826, 781, 770, 749, 761, 753, 753, 702, 797, 792, 771, 757, 833, 725, 747, 774, 791, 754, 717, 765, 814] payload_rows_per_dst=[9162, 9140, 9304, 9256, 9360, 9360, 9294, 9216] dup_rows_per_dst=[2882, 2958, 3170, 3178, 2970, 3074, 3022, 2958] combine_rows_per_src=[9162, 9140, 9304, 9256, 9360, 9360, 9294, 9216] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74088 payload_rows_remote=64818 dup_rows=24216 dup_rows_remote=21052 dedup_ratio=0.246338 router_tokens_per_expert=[746, 731, 778, 826, 775, 752, 771, 766, 771, 755, 778, 772, 716, 766, 807, 805, 780, 738, 740, 721, 788, 792, 715, 791, 738, 764, 793, 772, 769, 734, 742, 775, 809, 792, 785, 741, 768, 788, 731, 771, 792, 754, 794, 748, 732, 737, 749, 791, 795, 769, 786, 752, 788, 787, 767, 765, 760, 734, 777, 805, 773, 798, 785, 762] payload_rows_per_dst=[9300, 9342, 9162, 9204, 9240, 9264, 9306, 9270] dup_rows_per_dst=[2990, 2998, 2968, 2970, 3084, 2930, 3112, 3164] combine_rows_per_src=[9300, 9342, 9162, 9204, 9240, 9264, 9306, 9270] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=343440 combine_payload_bytes=265756672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266100112 ingress_bytes=300359680 link_bytes=300359680 token_bytes=531856784 weight_movement_bytes=34603008 total_nvlink_bytes=566459792 weight_to_token_ratio=0.065061 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130229295e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=64978 dup_rows=24002 dup_rows_remote=20890 dedup_ratio=0.244161 router_tokens_per_expert=[817, 743, 822, 798, 780, 782, 769, 759, 780, 794, 795, 732, 785, 788, 776, 768, 753, 798, 737, 779, 706, 808, 773, 761, 767, 762, 807, 790, 710, 774, 729, 779, 733, 758, 752, 770, 778, 724, 759, 753, 742, 769, 749, 767, 741, 774, 752, 745, 807, 745, 771, 785, 765, 790, 810, 722, 824, 727, 796, 752, 725, 803, 785, 758] payload_rows_per_dst=[9512, 9324, 9336, 9194, 9140, 9182, 9328, 9286] dup_rows_per_dst=[3028, 3112, 2894, 3042, 2914, 2896, 3062, 3054] combine_rows_per_src=[9512, 9324, 9336, 9194, 9140, 9182, 9328, 9286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266117120 dispatch_meta_bytes=344064 combine_payload_bytes=266117120 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266461184 ingress_bytes=404529152 link_bytes=404529152 token_bytes=532578304 weight_movement_bytes=138412032 total_nvlink_bytes=670990336 weight_to_token_ratio=0.259890 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522210632e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74456 payload_rows_remote=65374 dup_rows=23848 dup_rows_remote=20880 dedup_ratio=0.242594 router_tokens_per_expert=[762, 783, 770, 804, 790, 807, 738, 788, 765, 804, 773, 744, 772, 817, 779, 814, 762, 762, 753, 756, 748, 743, 780, 721, 772, 731, 739, 780, 741, 768, 712, 764, 778, 824, 749, 791, 773, 765, 800, 735, 712, 786, 765, 763, 736, 757, 769, 767, 768, 798, 749, 765, 765, 779, 781, 780, 750, 770, 732, 740, 773, 806, 815, 769] payload_rows_per_dst=[9388, 9410, 9082, 9130, 9516, 9218, 9382, 9330] dup_rows_per_dst=[3096, 3126, 2968, 2884, 2914, 2892, 2988, 2980] combine_rows_per_src=[9388, 9410, 9082, 9130, 9516, 9218, 9382, 9330] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74126 payload_rows_remote=64776 dup_rows=24178 dup_rows_remote=21014 dedup_ratio=0.245951 router_tokens_per_expert=[789, 749, 781, 813, 801, 777, 770, 777, 816, 780, 776, 739, 780, 760, 773, 773, 762, 717, 721, 760, 749, 818, 770, 774, 766, 805, 717, 734, 739, 782, 722, 768, 754, 797, 804, 750, 791, 772, 774, 790, 766, 767, 747, 778, 764, 730, 749, 788, 778, 758, 804, 777, 748, 786, 726, 825, 768, 754, 749, 768, 793, 748, 777, 714] payload_rows_per_dst=[9350, 9420, 9194, 9040, 9382, 9266, 9332, 9142] dup_rows_per_dst=[3164, 3096, 3044, 2808, 3082, 2912, 3072, 3000] combine_rows_per_src=[9350, 9420, 9194, 9040, 9382, 9266, 9332, 9142] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=65014 dup_rows=23974 dup_rows_remote=20974 dedup_ratio=0.243876 router_tokens_per_expert=[761, 773, 736, 787, 754, 787, 779, 781, 768, 784, 760, 829, 762, 772, 804, 776, 785, 804, 807, 746, 787, 739, 763, 737, 734, 787, 776, 779, 786, 767, 757, 740, 774, 722, 746, 786, 744, 771, 743, 815, 781, 758, 792, 795, 733, 759, 712, 808, 759, 832, 790, 722, 768, 752, 755, 803, 757, 739, 739, 748, 754, 775, 747, 766] payload_rows_per_dst=[9316, 9480, 9470, 9380, 9218, 9318, 9010, 9138] dup_rows_per_dst=[3000, 3090, 3258, 3048, 2984, 2958, 2724, 2912] combine_rows_per_src=[9316, 9480, 9470, 9380, 9218, 9318, 9010, 9138] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74632 payload_rows_remote=65216 dup_rows=23672 dup_rows_remote=20636 dedup_ratio=0.240804 router_tokens_per_expert=[735, 786, 769, 776, 778, 739, 759, 768, 779, 773, 745, 754, 739, 749, 725, 727, 757, 829, 779, 765, 805, 730, 729, 833, 787, 793, 787, 808, 760, 741, 756, 794, 762, 768, 800, 766, 801, 718, 772, 790, 755, 816, 778, 784, 742, 700, 771, 773, 760, 800, 786, 772, 804, 751, 778, 753, 775, 745, 750, 759, 783, 756, 764, 766] payload_rows_per_dst=[9290, 9202, 9368, 9416, 9416, 9276, 9416, 9248] dup_rows_per_dst=[2930, 2780, 3086, 3036, 2938, 2962, 2992, 2948] combine_rows_per_src=[9290, 9202, 9368, 9416, 9416, 9276, 9416, 9248] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74110 payload_rows_remote=64806 dup_rows=24194 dup_rows_remote=21074 dedup_ratio=0.246114 router_tokens_per_expert=[776, 787, 812, 726, 733, 759, 739, 793, 743, 769, 817, 762, 773, 784, 752, 799, 734, 810, 765, 768, 781, 753, 768, 791, 778, 784, 776, 772, 826, 757, 754, 770, 783, 693, 769, 774, 759, 781, 753, 756, 762, 710, 793, 757, 791, 810, 805, 784, 748, 779, 775, 714, 756, 747, 756, 786, 738, 814, 752, 741, 787, 775, 727, 766] payload_rows_per_dst=[9204, 9328, 9326, 9320, 9160, 9304, 9178, 9290] dup_rows_per_dst=[3046, 3070, 3014, 3114, 2976, 3120, 2944, 2910] combine_rows_per_src=[9204, 9328, 9326, 9320, 9160, 9304, 9178, 9290] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265494528 dispatch_meta_bytes=343480 combine_payload_bytes=265494528 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265838008 ingress_bytes=334700544 link_bytes=334700544 token_bytes=531332536 weight_movement_bytes=69206016 total_nvlink_bytes=600538552 weight_to_token_ratio=0.130250 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259451201e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266043392 dispatch_meta_bytes=344824 combine_payload_bytes=266043392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266388216 ingress_bytes=300646400 link_bytes=300646400 token_bytes=532431608 weight_movement_bytes=34603008 total_nvlink_bytes=567034616 weight_to_token_ratio=0.064991 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131308199e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=343472 combine_payload_bytes=266149888 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266493360 ingress_bytes=404561920 link_bytes=404561920 token_bytes=532643248 weight_movement_bytes=138412032 total_nvlink_bytes=671055280 weight_to_token_ratio=0.259859 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522333935e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74334 payload_rows_remote=64936 dup_rows=23970 dup_rows_remote=20914 dedup_ratio=0.243835 router_tokens_per_expert=[750, 772, 746, 759, 842, 746, 763, 789, 785, 769, 775, 775, 725, 770, 804, 745, 772, 763, 762, 778, 720, 792, 761, 729, 744, 771, 731, 775, 776, 754, 744, 770, 780, 764, 743, 791, 730, 762, 809, 724, 774, 754, 811, 773, 793, 777, 774, 771, 784, 773, 774, 755, 772, 775, 753, 803, 783, 769, 783, 743, 746, 758, 786, 808] payload_rows_per_dst=[9222, 9346, 9200, 9156, 9246, 9398, 9414, 9352] dup_rows_per_dst=[3112, 2950, 2954, 2974, 2960, 3056, 2964, 3000] combine_rows_per_src=[9222, 9346, 9200, 9156, 9246, 9398, 9414, 9352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267771904 dispatch_meta_bytes=345016 combine_payload_bytes=267771904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268116920 ingress_bytes=302374912 link_bytes=302374912 token_bytes=535888824 weight_movement_bytes=34603008 total_nvlink_bytes=570491832 weight_to_token_ratio=0.064571 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137812451e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74220 payload_rows_remote=65032 dup_rows=24084 dup_rows_remote=21110 dedup_ratio=0.244995 router_tokens_per_expert=[760, 746, 787, 750, 748, 719, 719, 749, 749, 792, 739, 750, 733, 778, 791, 778, 791, 827, 738, 765, 784, 751, 760, 748, 806, 807, 729, 771, 751, 793, 764, 770, 725, 781, 765, 750, 758, 768, 732, 802, 734, 756, 780, 780, 795, 795, 801, 762, 805, 778, 768, 762, 788, 760, 764, 767, 769, 784, 808, 732, 782, 803, 765, 790] payload_rows_per_dst=[9158, 9176, 9306, 9300, 9188, 9374, 9330, 9388] dup_rows_per_dst=[2798, 3044, 3022, 3082, 2974, 3032, 3054, 3078] combine_rows_per_src=[9158, 9176, 9306, 9300, 9188, 9374, 9330, 9388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266297344 dispatch_meta_bytes=343952 combine_payload_bytes=266297344 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266641296 ingress_bytes=300900352 link_bytes=300900352 token_bytes=532938640 weight_movement_bytes=34603008 total_nvlink_bytes=567541648 weight_to_token_ratio=0.064929 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132263800e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74250 payload_rows_remote=64800 dup_rows=24054 dup_rows_remote=20910 dedup_ratio=0.244690 router_tokens_per_expert=[799, 754, 716, 788, 791, 785, 745, 786, 817, 786, 760, 827, 808, 754, 800, 809, 716, 752, 707, 754, 753, 771, 726, 737, 768, 743, 770, 717, 797, 715, 790, 769, 796, 761, 774, 730, 777, 793, 758, 769, 770, 734, 702, 740, 796, 735, 738, 823, 809, 749, 786, 831, 755, 801, 792, 774, 795, 790, 738, 760, 778, 742, 790, 756] payload_rows_per_dst=[9274, 9526, 9022, 9180, 9314, 9180, 9450, 9304] dup_rows_per_dst=[3054, 3196, 2810, 2958, 3002, 2896, 3144, 2994] combine_rows_per_src=[9274, 9526, 9022, 9180, 9314, 9180, 9450, 9304] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265322496 dispatch_meta_bytes=343160 combine_payload_bytes=265322496 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265665656 ingress_bytes=334528512 link_bytes=334528512 token_bytes=530988152 weight_movement_bytes=69206016 total_nvlink_bytes=600194168 weight_to_token_ratio=0.130334 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258803859e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74632 payload_rows_remote=65216 dup_rows=23672 dup_rows_remote=20636 dedup_ratio=0.240804 router_tokens_per_expert=[735, 786, 769, 776, 778, 739, 759, 768, 779, 773, 745, 754, 739, 749, 725, 727, 757, 829, 779, 765, 805, 730, 729, 833, 787, 793, 787, 808, 760, 741, 756, 794, 762, 768, 800, 766, 801, 718, 772, 790, 755, 816, 778, 784, 742, 700, 771, 773, 760, 800, 786, 772, 804, 751, 778, 753, 775, 745, 750, 759, 783, 756, 764, 766] payload_rows_per_dst=[9290, 9202, 9368, 9416, 9416, 9276, 9416, 9248] dup_rows_per_dst=[2930, 2780, 3086, 3036, 2938, 2962, 2992, 2948] combine_rows_per_src=[9290, 9202, 9368, 9416, 9416, 9276, 9416, 9248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74062 payload_rows_remote=64764 dup_rows=24242 dup_rows_remote=21128 dedup_ratio=0.246602 router_tokens_per_expert=[794, 758, 836, 757, 771, 742, 838, 771, 762, 745, 772, 782, 755, 814, 735, 779, 804, 790, 790, 770, 779, 744, 752, 760, 781, 785, 785, 784, 790, 729, 784, 733, 750, 739, 803, 775, 766, 734, 787, 742, 772, 718, 732, 806, 738, 787, 756, 773, 761, 764, 774, 734, 767, 742, 830, 741, 751, 754, 772, 752, 810, 726, 791, 734] payload_rows_per_dst=[8996, 9458, 9350, 9322, 9172, 9234, 9232, 9298] dup_rows_per_dst=[2856, 3280, 3028, 2974, 3024, 2930, 3036, 3114] combine_rows_per_src=[8996, 9458, 9350, 9322, 9172, 9234, 9232, 9298] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267124736 dispatch_meta_bytes=343408 combine_payload_bytes=267124736 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267468144 ingress_bytes=301727744 link_bytes=301727744 token_bytes=534592880 weight_movement_bytes=34603008 total_nvlink_bytes=569195888 weight_to_token_ratio=0.064728 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135377210e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265445376 dispatch_meta_bytes=343520 combine_payload_bytes=265445376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265788896 ingress_bytes=300048384 link_bytes=300048384 token_bytes=531234272 weight_movement_bytes=34603008 total_nvlink_bytes=565837280 weight_to_token_ratio=0.065137 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129057913e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344568 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266715640 ingress_bytes=300974080 link_bytes=300974080 token_bytes=533086712 weight_movement_bytes=34603008 total_nvlink_bytes=567689720 weight_to_token_ratio=0.064911 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132541233e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74220 payload_rows_remote=65032 dup_rows=24084 dup_rows_remote=21110 dedup_ratio=0.244995 router_tokens_per_expert=[760, 746, 787, 750, 748, 719, 719, 749, 749, 792, 739, 750, 733, 778, 791, 778, 791, 827, 738, 765, 784, 751, 760, 748, 806, 807, 729, 771, 751, 793, 764, 770, 725, 781, 765, 750, 758, 768, 732, 802, 734, 756, 780, 780, 795, 795, 801, 762, 805, 778, 768, 762, 788, 760, 764, 767, 769, 784, 808, 732, 782, 803, 765, 790] payload_rows_per_dst=[9158, 9176, 9306, 9300, 9188, 9374, 9330, 9388] dup_rows_per_dst=[2798, 3044, 3022, 3082, 2974, 3032, 3054, 3078] combine_rows_per_src=[9158, 9176, 9306, 9300, 9188, 9374, 9330, 9388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265977856 dispatch_meta_bytes=343400 combine_payload_bytes=265977856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266321256 ingress_bytes=300580864 link_bytes=300580864 token_bytes=532299112 weight_movement_bytes=34603008 total_nvlink_bytes=566902120 weight_to_token_ratio=0.065007 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131061592e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265420800 dispatch_meta_bytes=342840 combine_payload_bytes=265420800 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265763640 ingress_bytes=403832832 link_bytes=403832832 token_bytes=531184440 weight_movement_bytes=138412032 total_nvlink_bytes=669596472 weight_to_token_ratio=0.260572 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519590436e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267124736 dispatch_meta_bytes=343408 combine_payload_bytes=267124736 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267468144 ingress_bytes=301727744 link_bytes=301727744 token_bytes=534592880 weight_movement_bytes=34603008 total_nvlink_bytes=569195888 weight_to_token_ratio=0.064728 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135377210e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74300 payload_rows_remote=64882 dup_rows=24004 dup_rows_remote=20978 dedup_ratio=0.244181 router_tokens_per_expert=[757, 775, 773, 719, 745, 801, 766, 771, 768, 780, 696, 784, 792, 785, 756, 802, 736, 789, 741, 796, 769, 772, 792, 776, 771, 757, 760, 762, 778, 716, 767, 799, 764, 715, 732, 750, 794, 742, 804, 771, 768, 741, 802, 765, 770, 775, 788, 821, 735, 776, 762, 803, 768, 760, 781, 764, 781, 767, 775, 764, 756, 803, 772, 732] payload_rows_per_dst=[9242, 8808, 9340, 9338, 9398, 9456, 9300, 9418] dup_rows_per_dst=[2968, 2632, 3002, 2882, 3252, 3240, 3002, 3026] combine_rows_per_src=[9242, 8808, 9340, 9338, 9398, 9456, 9300, 9418] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=64978 dup_rows=24002 dup_rows_remote=20890 dedup_ratio=0.244161 router_tokens_per_expert=[817, 743, 822, 798, 780, 782, 769, 759, 780, 794, 795, 732, 785, 788, 776, 768, 753, 798, 737, 779, 706, 808, 773, 761, 767, 762, 807, 790, 710, 774, 729, 779, 733, 758, 752, 770, 778, 724, 759, 753, 742, 769, 749, 767, 741, 774, 752, 745, 807, 745, 771, 785, 765, 790, 810, 722, 824, 727, 796, 752, 725, 803, 785, 758] payload_rows_per_dst=[9512, 9324, 9336, 9194, 9140, 9182, 9328, 9286] dup_rows_per_dst=[3028, 3112, 2894, 3042, 2914, 2896, 3062, 3054] combine_rows_per_src=[9512, 9324, 9336, 9194, 9140, 9182, 9328, 9286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265273344 dispatch_meta_bytes=343568 combine_payload_bytes=265273344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265616912 ingress_bytes=299876352 link_bytes=299876352 token_bytes=530890256 weight_movement_bytes=34603008 total_nvlink_bytes=565493264 weight_to_token_ratio=0.065179 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128410571e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74478 payload_rows_remote=65164 dup_rows=23826 dup_rows_remote=20798 dedup_ratio=0.242371 router_tokens_per_expert=[763, 771, 745, 745, 762, 807, 752, 758, 748, 782, 790, 790, 827, 735, 723, 776, 781, 755, 748, 793, 755, 728, 761, 794, 756, 771, 785, 802, 793, 745, 763, 757, 784, 807, 747, 746, 698, 773, 750, 771, 738, 789, 792, 779, 772, 789, 774, 803, 769, 764, 743, 801, 762, 779, 755, 798, 773, 776, 784, 765, 763, 763, 741, 743] payload_rows_per_dst=[9306, 9314, 9202, 9406, 9182, 9422, 9368, 9278] dup_rows_per_dst=[2900, 3028, 3028, 2938, 2970, 3050, 2974, 2938] combine_rows_per_src=[9306, 9314, 9202, 9406, 9182, 9422, 9368, 9278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344568 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266715640 ingress_bytes=300974080 link_bytes=300974080 token_bytes=533086712 weight_movement_bytes=34603008 total_nvlink_bytes=567689720 weight_to_token_ratio=0.064911 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132541233e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73976 payload_rows_remote=64756 dup_rows=24328 dup_rows_remote=21280 dedup_ratio=0.247477 router_tokens_per_expert=[797, 841, 776, 780, 777, 799, 756, 808, 749, 734, 766, 786, 829, 725, 774, 786, 761, 728, 742, 747, 771, 792, 780, 813, 759, 789, 753, 788, 745, 755, 781, 798, 733, 770, 768, 788, 747, 772, 809, 741, 802, 807, 765, 733, 758, 810, 691, 756, 727, 734, 757, 761, 749, 786, 794, 764, 714, 778, 773, 772, 708, 744, 780, 776] payload_rows_per_dst=[9444, 9290, 9220, 9310, 9286, 9194, 9118, 9114] dup_rows_per_dst=[3224, 3008, 3048, 3026, 2970, 3050, 3026, 2976] combine_rows_per_src=[9444, 9290, 9220, 9310, 9286, 9194, 9118, 9114] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74406 payload_rows_remote=65064 dup_rows=23898 dup_rows_remote=20930 dedup_ratio=0.243103 router_tokens_per_expert=[776, 751, 752, 782, 741, 771, 774, 782, 769, 781, 815, 721, 783, 742, 795, 770, 732, 792, 708, 816, 742, 740, 728, 809, 773, 731, 776, 790, 727, 744, 765, 731, 779, 780, 765, 761, 752, 785, 844, 735, 754, 759, 759, 795, 753, 735, 785, 792, 803, 779, 770, 745, 764, 795, 774, 729, 762, 794, 757, 814, 781, 786, 785, 772] payload_rows_per_dst=[9342, 9520, 9364, 9196, 9364, 9500, 8850, 9270] dup_rows_per_dst=[2968, 3088, 3118, 2920, 2964, 3220, 2736, 2884] combine_rows_per_src=[9342, 9520, 9364, 9196, 9364, 9500, 8850, 9270] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74006 payload_rows_remote=64852 dup_rows=24298 dup_rows_remote=21220 dedup_ratio=0.247172 router_tokens_per_expert=[781, 791, 746, 753, 752, 761, 784, 741, 786, 732, 734, 770, 783, 706, 762, 770, 754, 779, 785, 750, 794, 770, 758, 754, 775, 748, 818, 745, 752, 786, 733, 745, 762, 735, 775, 777, 769, 783, 759, 787, 762, 770, 834, 693, 766, 790, 805, 809, 779, 771, 691, 806, 791, 761, 778, 752, 772, 793, 765, 788, 782, 761, 795, 793] payload_rows_per_dst=[9154, 9204, 9402, 9190, 9078, 9354, 9180, 9444] dup_rows_per_dst=[3078, 2882, 3086, 3014, 3002, 3250, 2932, 3054] combine_rows_per_src=[9154, 9204, 9402, 9190, 9078, 9354, 9180, 9444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266911744 dispatch_meta_bytes=343848 combine_payload_bytes=266911744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267255592 ingress_bytes=301514752 link_bytes=301514752 token_bytes=534167336 weight_movement_bytes=34603008 total_nvlink_bytes=568770344 weight_to_token_ratio=0.064779 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134575738e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=343440 combine_payload_bytes=265756672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266100112 ingress_bytes=300359680 link_bytes=300359680 token_bytes=531856784 weight_movement_bytes=34603008 total_nvlink_bytes=566459792 weight_to_token_ratio=0.065061 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130229295e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74334 payload_rows_remote=64936 dup_rows=23970 dup_rows_remote=20914 dedup_ratio=0.243835 router_tokens_per_expert=[750, 772, 746, 759, 842, 746, 763, 789, 785, 769, 775, 775, 725, 770, 804, 745, 772, 763, 762, 778, 720, 792, 761, 729, 744, 771, 731, 775, 776, 754, 744, 770, 780, 764, 743, 791, 730, 762, 809, 724, 774, 754, 811, 773, 793, 777, 774, 771, 784, 773, 774, 755, 772, 775, 753, 803, 783, 769, 783, 743, 746, 758, 786, 808] payload_rows_per_dst=[9222, 9346, 9200, 9156, 9246, 9398, 9414, 9352] dup_rows_per_dst=[3112, 2950, 2954, 2974, 2960, 3056, 2964, 3000] combine_rows_per_src=[9222, 9346, 9200, 9156, 9246, 9398, 9414, 9352] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=343472 combine_payload_bytes=266149888 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266493360 ingress_bytes=404561920 link_bytes=404561920 token_bytes=532643248 weight_movement_bytes=138412032 total_nvlink_bytes=671055280 weight_to_token_ratio=0.259859 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522333935e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74526 payload_rows_remote=65170 dup_rows=23778 dup_rows_remote=20776 dedup_ratio=0.241882 router_tokens_per_expert=[801, 800, 806, 750, 763, 761, 800, 740, 771, 761, 728, 754, 753, 813, 753, 766, 755, 756, 788, 799, 781, 770, 748, 761, 796, 788, 751, 741, 809, 798, 742, 748, 751, 816, 775, 788, 722, 779, 763, 785, 767, 710, 789, 718, 816, 765, 719, 785, 810, 790, 742, 773, 789, 791, 745, 746, 784, 734, 726, 808, 748, 817, 689, 761] payload_rows_per_dst=[9296, 9238, 9304, 9462, 9356, 9244, 9386, 9240] dup_rows_per_dst=[3146, 2960, 3012, 2884, 3002, 2894, 2986, 2894] combine_rows_per_src=[9296, 9238, 9304, 9462, 9356, 9244, 9386, 9240] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265240576 dispatch_meta_bytes=344144 combine_payload_bytes=265240576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265584720 ingress_bytes=299843584 link_bytes=299843584 token_bytes=530825296 weight_movement_bytes=34603008 total_nvlink_bytes=565428304 weight_to_token_ratio=0.065187 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128287267e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74318 payload_rows_remote=64892 dup_rows=23986 dup_rows_remote=20940 dedup_ratio=0.243998 router_tokens_per_expert=[726, 827, 760, 780, 766, 737, 812, 763, 774, 705, 794, 733, 815, 769, 739, 782, 750, 784, 730, 777, 753, 789, 764, 798, 772, 785, 742, 799, 749, 784, 743, 801, 759, 774, 772, 811, 805, 779, 739, 767, 739, 739, 742, 732, 754, 750, 791, 743, 794, 812, 766, 756, 796, 754, 792, 766, 754, 812, 767, 757, 767, 741, 752, 768] payload_rows_per_dst=[9374, 9258, 9256, 9326, 9280, 9124, 9426, 9274] dup_rows_per_dst=[2968, 2964, 3034, 3024, 3132, 2856, 3046, 2962] combine_rows_per_src=[9374, 9258, 9256, 9326, 9280, 9124, 9426, 9274] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65090 dup_rows=23908 dup_rows_remote=20916 dedup_ratio=0.243205 router_tokens_per_expert=[775, 766, 759, 762, 791, 751, 760, 750, 766, 739, 773, 784, 744, 734, 745, 777, 778, 766, 753, 758, 714, 743, 781, 797, 776, 737, 778, 773, 758, 795, 759, 773, 748, 774, 786, 789, 771, 795, 811, 848, 758, 800, 789, 742, 789, 769, 714, 763, 830, 754, 761, 781, 720, 710, 781, 771, 755, 788, 739, 757, 775, 816, 781, 772] payload_rows_per_dst=[9370, 9242, 9154, 9306, 9486, 9278, 9198, 9362] dup_rows_per_dst=[2858, 2882, 3026, 2992, 3158, 2970, 3018, 3004] combine_rows_per_src=[9370, 9242, 9154, 9306, 9486, 9278, 9198, 9362] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266502144 dispatch_meta_bytes=343976 combine_payload_bytes=266502144 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266846120 ingress_bytes=301105152 link_bytes=301105152 token_bytes=533348264 weight_movement_bytes=34603008 total_nvlink_bytes=567951272 weight_to_token_ratio=0.064879 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133034446e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265977856 dispatch_meta_bytes=343400 combine_payload_bytes=265977856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266321256 ingress_bytes=300580864 link_bytes=300580864 token_bytes=532299112 weight_movement_bytes=34603008 total_nvlink_bytes=566902120 weight_to_token_ratio=0.065007 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131061592e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266936320 dispatch_meta_bytes=343784 combine_payload_bytes=266936320 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267280104 ingress_bytes=370745344 link_bytes=370745344 token_bytes=534216424 weight_movement_bytes=103809024 total_nvlink_bytes=638025448 weight_to_token_ratio=0.194320 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395084882e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265797632 dispatch_meta_bytes=343328 combine_payload_bytes=265797632 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266140960 ingress_bytes=335003648 link_bytes=335003648 token_bytes=531938592 weight_movement_bytes=69206016 total_nvlink_bytes=601144608 weight_to_token_ratio=0.130102 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260591757e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266608640 dispatch_meta_bytes=344024 combine_payload_bytes=266608640 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266952664 ingress_bytes=335814656 link_bytes=335814656 token_bytes=533561304 weight_movement_bytes=69206016 total_nvlink_bytes=602767320 weight_to_token_ratio=0.129706 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263643515e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74062 payload_rows_remote=64764 dup_rows=24242 dup_rows_remote=21128 dedup_ratio=0.246602 router_tokens_per_expert=[794, 758, 836, 757, 771, 742, 838, 771, 762, 745, 772, 782, 755, 814, 735, 779, 804, 790, 790, 770, 779, 744, 752, 760, 781, 785, 785, 784, 790, 729, 784, 733, 750, 739, 803, 775, 766, 734, 787, 742, 772, 718, 732, 806, 738, 787, 756, 773, 761, 764, 774, 734, 767, 742, 830, 741, 751, 754, 772, 752, 810, 726, 791, 734] payload_rows_per_dst=[8996, 9458, 9350, 9322, 9172, 9234, 9232, 9298] dup_rows_per_dst=[2856, 3280, 3028, 2974, 3024, 2930, 3036, 3114] combine_rows_per_src=[8996, 9458, 9350, 9322, 9172, 9234, 9232, 9298] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74478 payload_rows_remote=65164 dup_rows=23826 dup_rows_remote=20798 dedup_ratio=0.242371 router_tokens_per_expert=[763, 771, 745, 745, 762, 807, 752, 758, 748, 782, 790, 790, 827, 735, 723, 776, 781, 755, 748, 793, 755, 728, 761, 794, 756, 771, 785, 802, 793, 745, 763, 757, 784, 807, 747, 746, 698, 773, 750, 771, 738, 789, 792, 779, 772, 789, 774, 803, 769, 764, 743, 801, 762, 779, 755, 798, 773, 776, 784, 765, 763, 763, 741, 743] payload_rows_per_dst=[9306, 9314, 9202, 9406, 9182, 9422, 9368, 9278] dup_rows_per_dst=[2900, 3028, 3028, 2938, 2970, 3050, 2974, 2938] combine_rows_per_src=[9306, 9314, 9202, 9406, 9182, 9422, 9368, 9278] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74068 payload_rows_remote=64940 dup_rows=24236 dup_rows_remote=21282 dedup_ratio=0.246541 router_tokens_per_expert=[740, 762, 771, 790, 798, 807, 786, 717, 707, 767, 740, 810, 761, 750, 797, 774, 713, 850, 740, 745, 719, 744, 752, 778, 777, 782, 784, 753, 747, 724, 741, 726, 776, 798, 754, 754, 836, 738, 771, 789, 768, 760, 814, 834, 747, 765, 785, 757, 796, 718, 773, 773, 806, 801, 790, 754, 778, 716, 746, 822, 784, 740, 757, 800] payload_rows_per_dst=[9314, 9192, 9128, 9116, 9370, 9386, 9400, 9162] dup_rows_per_dst=[3028, 3020, 2954, 2952, 3062, 3074, 3022, 3124] combine_rows_per_src=[9314, 9192, 9128, 9116, 9370, 9386, 9400, 9162] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265273344 dispatch_meta_bytes=343568 combine_payload_bytes=265273344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265616912 ingress_bytes=299876352 link_bytes=299876352 token_bytes=530890256 weight_movement_bytes=34603008 total_nvlink_bytes=565493264 weight_to_token_ratio=0.065179 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128410571e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=65014 dup_rows=23974 dup_rows_remote=20974 dedup_ratio=0.243876 router_tokens_per_expert=[761, 773, 736, 787, 754, 787, 779, 781, 768, 784, 760, 829, 762, 772, 804, 776, 785, 804, 807, 746, 787, 739, 763, 737, 734, 787, 776, 779, 786, 767, 757, 740, 774, 722, 746, 786, 744, 771, 743, 815, 781, 758, 792, 795, 733, 759, 712, 808, 759, 832, 790, 722, 768, 752, 755, 803, 757, 739, 739, 748, 754, 775, 747, 766] payload_rows_per_dst=[9316, 9480, 9470, 9380, 9218, 9318, 9010, 9138] dup_rows_per_dst=[3000, 3090, 3258, 3048, 2984, 2958, 2724, 2912] combine_rows_per_src=[9316, 9480, 9470, 9380, 9218, 9318, 9010, 9138] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74394 payload_rows_remote=65044 dup_rows=23910 dup_rows_remote=20944 dedup_ratio=0.243225 router_tokens_per_expert=[783, 748, 738, 786, 720, 748, 762, 793, 772, 704, 767, 754, 719, 723, 763, 742, 790, 800, 766, 805, 809, 744, 731, 800, 748, 758, 757, 807, 753, 771, 769, 734, 750, 747, 786, 826, 817, 785, 761, 772, 733, 734, 795, 776, 737, 823, 813, 747, 741, 761, 757, 736, 793, 797, 749, 753, 819, 795, 771, 766, 768, 762, 796, 822] payload_rows_per_dst=[9246, 9074, 9454, 9194, 9414, 9350, 9214, 9448] dup_rows_per_dst=[2910, 2814, 3036, 3000, 3074, 2966, 2960, 3150] combine_rows_per_src=[9246, 9074, 9454, 9194, 9414, 9350, 9214, 9448] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=65078 dup_rows=24018 dup_rows_remote=20834 dedup_ratio=0.244324 router_tokens_per_expert=[759, 752, 716, 789, 766, 763, 794, 742, 767, 739, 772, 818, 760, 721, 801, 715, 787, 774, 808, 778, 731, 765, 780, 747, 736, 752, 809, 768, 740, 779, 731, 783, 797, 702, 824, 743, 762, 775, 784, 809, 766, 782, 770, 788, 786, 777, 750, 808, 748, 763, 804, 767, 769, 794, 777, 738, 755, 788, 779, 784, 757, 742, 778, 744] payload_rows_per_dst=[9250, 9336, 9336, 9248, 9208, 9318, 9304, 9286] dup_rows_per_dst=[2912, 2850, 3004, 2948, 3184, 3136, 3016, 2968] combine_rows_per_src=[9250, 9336, 9336, 9248, 9208, 9318, 9304, 9286] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266911744 dispatch_meta_bytes=343848 combine_payload_bytes=266911744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267255592 ingress_bytes=301514752 link_bytes=301514752 token_bytes=534167336 weight_movement_bytes=34603008 total_nvlink_bytes=568770344 weight_to_token_ratio=0.064779 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134575738e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=344888 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266339128 ingress_bytes=335200256 link_bytes=335200256 token_bytes=532333368 weight_movement_bytes=69206016 total_nvlink_bytes=601539384 weight_to_token_ratio=0.130005 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261331577e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64976 dup_rows=24112 dup_rows_remote=21134 dedup_ratio=0.245280 router_tokens_per_expert=[755, 791, 805, 848, 804, 762, 811, 759, 776, 744, 771, 741, 757, 768, 747, 827, 812, 769, 784, 766, 698, 763, 744, 752, 750, 782, 774, 785, 740, 779, 816, 779, 770, 746, 717, 759, 760, 783, 777, 758, 796, 754, 759, 775, 806, 759, 720, 784, 739, 780, 757, 764, 787, 754, 761, 755, 775, 723, 800, 792, 779, 733, 721, 750] payload_rows_per_dst=[9474, 9248, 9260, 9410, 9148, 9244, 9216, 9192] dup_rows_per_dst=[3196, 3014, 2916, 3000, 2992, 3062, 2978, 2954] combine_rows_per_src=[9474, 9248, 9260, 9410, 9148, 9244, 9216, 9192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=64862 dup_rows=24062 dup_rows_remote=21132 dedup_ratio=0.244771 router_tokens_per_expert=[756, 805, 736, 788, 798, 745, 758, 745, 742, 754, 781, 742, 781, 781, 810, 759, 787, 782, 762, 733, 753, 791, 780, 827, 753, 756, 741, 836, 756, 826, 744, 743, 747, 778, 777, 765, 790, 758, 757, 773, 816, 736, 743, 745, 817, 745, 805, 726, 771, 755, 754, 740, 760, 796, 768, 766, 788, 757, 760, 809, 779, 729, 749, 742] payload_rows_per_dst=[9268, 9226, 9326, 9380, 9274, 9286, 9212, 9270] dup_rows_per_dst=[2994, 3074, 3104, 2930, 3016, 2980, 3008, 2956] combine_rows_per_src=[9268, 9226, 9326, 9380, 9274, 9286, 9212, 9270] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266297344 dispatch_meta_bytes=343952 combine_payload_bytes=266297344 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266641296 ingress_bytes=300900352 link_bytes=300900352 token_bytes=532938640 weight_movement_bytes=34603008 total_nvlink_bytes=567541648 weight_to_token_ratio=0.064929 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132263800e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=343952 combine_payload_bytes=266420224 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266764176 ingress_bytes=301023232 link_bytes=301023232 token_bytes=533184400 weight_movement_bytes=34603008 total_nvlink_bytes=567787408 weight_to_token_ratio=0.064899 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132726188e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266559488 dispatch_meta_bytes=343648 combine_payload_bytes=266559488 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266903136 ingress_bytes=370368512 link_bytes=370368512 token_bytes=533462624 weight_movement_bytes=103809024 total_nvlink_bytes=637271648 weight_to_token_ratio=0.194595 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393666894e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265240576 dispatch_meta_bytes=344144 combine_payload_bytes=265240576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265584720 ingress_bytes=299843584 link_bytes=299843584 token_bytes=530825296 weight_movement_bytes=34603008 total_nvlink_bytes=565428304 weight_to_token_ratio=0.065187 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128287267e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74318 payload_rows_remote=64892 dup_rows=23986 dup_rows_remote=20940 dedup_ratio=0.243998 router_tokens_per_expert=[726, 827, 760, 780, 766, 737, 812, 763, 774, 705, 794, 733, 815, 769, 739, 782, 750, 784, 730, 777, 753, 789, 764, 798, 772, 785, 742, 799, 749, 784, 743, 801, 759, 774, 772, 811, 805, 779, 739, 767, 739, 739, 742, 732, 754, 750, 791, 743, 794, 812, 766, 756, 796, 754, 792, 766, 754, 812, 767, 757, 767, 741, 752, 768] payload_rows_per_dst=[9374, 9258, 9256, 9326, 9280, 9124, 9426, 9274] dup_rows_per_dst=[2968, 2964, 3034, 3024, 3132, 2856, 3046, 2962] combine_rows_per_src=[9374, 9258, 9256, 9326, 9280, 9124, 9426, 9274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265633792 dispatch_meta_bytes=344288 combine_payload_bytes=265633792 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265978080 ingress_bytes=369442816 link_bytes=369442816 token_bytes=531611872 weight_movement_bytes=103809024 total_nvlink_bytes=635420896 weight_to_token_ratio=0.195272 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390183574e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=64956 dup_rows=23982 dup_rows_remote=20956 dedup_ratio=0.243958 router_tokens_per_expert=[828, 777, 754, 760, 757, 765, 775, 702, 807, 791, 773, 766, 757, 761, 777, 749, 782, 799, 752, 783, 755, 755, 774, 761, 757, 757, 755, 774, 744, 796, 770, 811, 799, 750, 789, 791, 781, 818, 741, 810, 727, 781, 739, 792, 757, 744, 708, 746, 736, 784, 761, 775, 772, 740, 784, 793, 783, 828, 738, 736, 766, 764, 786, 709] payload_rows_per_dst=[9158, 9328, 9294, 9346, 9442, 9180, 9208, 9366] dup_rows_per_dst=[3012, 3034, 3028, 2982, 3116, 2808, 2976, 3026] combine_rows_per_src=[9158, 9328, 9294, 9346, 9442, 9180, 9208, 9366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12266799912 prefetch_bytes=657457152 reduce_bytes=553648128 weight_movement_bytes=1211105280 egress_bytes=6137357096 ingress_bytes=7340548096 link_bytes=7340548096 total_nvlink_bytes=13477905192 weight_to_token_ratio=0.098730 prefetch_share_of_ingress=0.089565 dedup_ratio=0.243767 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=16 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029022 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.200951805e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266141696 dispatch_meta_bytes=344440 combine_payload_bytes=266141696 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266486136 ingress_bytes=369950720 link_bytes=369950720 token_bytes=532627832 weight_movement_bytes=103809024 total_nvlink_bytes=636436856 weight_to_token_ratio=0.194900 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392094776e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265674752 dispatch_meta_bytes=343976 combine_payload_bytes=265674752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266018728 ingress_bytes=300277760 link_bytes=300277760 token_bytes=531693480 weight_movement_bytes=34603008 total_nvlink_bytes=566296488 weight_to_token_ratio=0.065081 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129921037e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12262343336 prefetch_bytes=622854144 reduce_bytes=761266176 weight_movement_bytes=1384120320 egress_bytes=6135128744 ingress_bytes=7511334912 link_bytes=7511334912 total_nvlink_bytes=13646463656 weight_to_token_ratio=0.112876 prefetch_share_of_ingress=0.082922 dedup_ratio=0.243905 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=22 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028804 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.228893416e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266608640 dispatch_meta_bytes=344024 combine_payload_bytes=266608640 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266952664 ingress_bytes=335814656 link_bytes=335814656 token_bytes=533561304 weight_movement_bytes=69206016 total_nvlink_bytes=602767320 weight_to_token_ratio=0.129706 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263643515e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12249642576 prefetch_bytes=692060160 reduce_bytes=865075200 weight_movement_bytes=1557135360 egress_bytes=6128776784 ingress_bytes=7678001152 link_bytes=7678001152 total_nvlink_bytes=13806777936 weight_to_token_ratio=0.127117 prefetch_share_of_ingress=0.090135 dedup_ratio=0.244561 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=25 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027935 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.256160878e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74526 payload_rows_remote=65170 dup_rows=23778 dup_rows_remote=20776 dedup_ratio=0.241882 router_tokens_per_expert=[801, 800, 806, 750, 763, 761, 800, 740, 771, 761, 728, 754, 753, 813, 753, 766, 755, 756, 788, 799, 781, 770, 748, 761, 796, 788, 751, 741, 809, 798, 742, 748, 751, 816, 775, 788, 722, 779, 763, 785, 767, 710, 789, 718, 816, 765, 719, 785, 810, 790, 742, 773, 789, 791, 745, 746, 784, 734, 726, 808, 748, 817, 689, 761] payload_rows_per_dst=[9296, 9238, 9304, 9462, 9356, 9244, 9386, 9240] dup_rows_per_dst=[3146, 2960, 3012, 2884, 3002, 2894, 2986, 2894] combine_rows_per_src=[9296, 9238, 9304, 9462, 9356, 9244, 9386, 9240] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=343952 combine_payload_bytes=266420224 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266764176 ingress_bytes=301023232 link_bytes=301023232 token_bytes=533184400 weight_movement_bytes=34603008 total_nvlink_bytes=567787408 weight_to_token_ratio=0.064899 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132726188e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265797632 dispatch_meta_bytes=343328 combine_payload_bytes=265797632 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266140960 ingress_bytes=335003648 link_bytes=335003648 token_bytes=531938592 weight_movement_bytes=69206016 total_nvlink_bytes=601144608 weight_to_token_ratio=0.130102 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260591757e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74132 payload_rows_remote=64924 dup_rows=24172 dup_rows_remote=21192 dedup_ratio=0.245890 router_tokens_per_expert=[775, 791, 755, 766, 782, 783, 741, 804, 797, 744, 735, 750, 767, 741, 786, 774, 756, 711, 792, 791, 781, 768, 729, 790, 775, 790, 751, 744, 766, 734, 775, 794, 788, 750, 779, 749, 763, 830, 794, 714, 758, 790, 747, 728, 775, 779, 792, 768, 781, 711, 768, 782, 798, 768, 815, 785, 753, 757, 774, 776, 770, 765, 781, 726] payload_rows_per_dst=[9364, 9208, 9302, 9188, 9304, 9316, 9290, 9160] dup_rows_per_dst=[3030, 2980, 2934, 3070, 3030, 2958, 3126, 3044] combine_rows_per_src=[9364, 9208, 9302, 9188, 9304, 9316, 9290, 9160] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12255753552 prefetch_bytes=657457152 reduce_bytes=968884224 weight_movement_bytes=1626341376 egress_bytes=6131832144 ingress_bytes=7750262784 link_bytes=7750262784 total_nvlink_bytes=13882094928 weight_to_token_ratio=0.132700 prefetch_share_of_ingress=0.084830 dedup_ratio=0.244093 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=28 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.267983257e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=343648 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266403424 ingress_bytes=300662784 link_bytes=300662784 token_bytes=532463200 weight_movement_bytes=34603008 total_nvlink_bytes=567066208 weight_to_token_ratio=0.064987 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131369851e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12249891808 prefetch_bytes=692060160 reduce_bytes=795869184 weight_movement_bytes=1487929344 egress_bytes=6128903136 ingress_bytes=7608918016 link_bytes=7608918016 total_nvlink_bytes=13737821152 weight_to_token_ratio=0.121465 prefetch_share_of_ingress=0.090954 dedup_ratio=0.244708 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=23 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028043 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.244858519e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74006 payload_rows_remote=64852 dup_rows=24298 dup_rows_remote=21220 dedup_ratio=0.247172 router_tokens_per_expert=[781, 791, 746, 753, 752, 761, 784, 741, 786, 732, 734, 770, 783, 706, 762, 770, 754, 779, 785, 750, 794, 770, 758, 754, 775, 748, 818, 745, 752, 786, 733, 745, 762, 735, 775, 777, 769, 783, 759, 787, 762, 770, 834, 693, 766, 790, 805, 809, 779, 771, 691, 806, 791, 761, 778, 752, 772, 793, 765, 788, 782, 761, 795, 793] payload_rows_per_dst=[9154, 9204, 9402, 9190, 9078, 9354, 9180, 9444] dup_rows_per_dst=[3078, 2882, 3086, 3014, 3002, 3250, 2932, 3054] combine_rows_per_src=[9154, 9204, 9402, 9190, 9078, 9354, 9180, 9444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12262343336 prefetch_bytes=622854144 reduce_bytes=761266176 weight_movement_bytes=1384120320 egress_bytes=6135128744 ingress_bytes=7511334912 link_bytes=7511334912 total_nvlink_bytes=13646463656 weight_to_token_ratio=0.112876 prefetch_share_of_ingress=0.082922 dedup_ratio=0.243905 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=22 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028804 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.228893416e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74068 payload_rows_remote=64940 dup_rows=24236 dup_rows_remote=21282 dedup_ratio=0.246541 router_tokens_per_expert=[740, 762, 771, 790, 798, 807, 786, 717, 707, 767, 740, 810, 761, 750, 797, 774, 713, 850, 740, 745, 719, 744, 752, 778, 777, 782, 784, 753, 747, 724, 741, 726, 776, 798, 754, 754, 836, 738, 771, 789, 768, 760, 814, 834, 747, 765, 785, 757, 796, 718, 773, 773, 806, 801, 790, 754, 778, 716, 746, 822, 784, 740, 757, 800] payload_rows_per_dst=[9314, 9192, 9128, 9116, 9370, 9386, 9400, 9162] dup_rows_per_dst=[3028, 3020, 2954, 2952, 3062, 3074, 3022, 3124] combine_rows_per_src=[9314, 9192, 9128, 9116, 9370, 9386, 9400, 9162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266936320 dispatch_meta_bytes=343784 combine_payload_bytes=266936320 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267280104 ingress_bytes=370745344 link_bytes=370745344 token_bytes=534216424 weight_movement_bytes=103809024 total_nvlink_bytes=638025448 weight_to_token_ratio=0.194320 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395084882e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74110 payload_rows_remote=64910 dup_rows=24194 dup_rows_remote=21246 dedup_ratio=0.246114 router_tokens_per_expert=[764, 754, 750, 786, 718, 769, 773, 742, 827, 754, 765, 732, 767, 765, 789, 799, 773, 770, 830, 750, 729, 725, 771, 774, 813, 754, 799, 775, 748, 786, 789, 753, 773, 746, 718, 794, 742, 773, 780, 769, 765, 792, 776, 742, 778, 752, 716, 804, 799, 791, 784, 779, 767, 786, 746, 788, 798, 763, 759, 753, 772, 765, 754, 735] payload_rows_per_dst=[9200, 9610, 9222, 9418, 8870, 9314, 9274, 9202] dup_rows_per_dst=[2948, 3296, 3006, 3122, 2800, 2926, 3100, 2996] combine_rows_per_src=[9200, 9610, 9222, 9418, 8870, 9314, 9274, 9202] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265928704 dispatch_meta_bytes=344464 combine_payload_bytes=265928704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266273168 ingress_bytes=300531712 link_bytes=300531712 token_bytes=532201872 weight_movement_bytes=34603008 total_nvlink_bytes=566804880 weight_to_token_ratio=0.065019 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130876637e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12234197168 prefetch_bytes=761266176 reduce_bytes=622854144 weight_movement_bytes=1384120320 egress_bytes=6121048240 ingress_bytes=7497269248 link_bytes=7497269248 total_nvlink_bytes=13618317488 weight_to_token_ratio=0.113135 prefetch_share_of_ingress=0.101539 dedup_ratio=0.244638 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=18 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029022 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.226592200e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265633792 dispatch_meta_bytes=344288 combine_payload_bytes=265633792 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265978080 ingress_bytes=369442816 link_bytes=369442816 token_bytes=531611872 weight_movement_bytes=103809024 total_nvlink_bytes=635420896 weight_to_token_ratio=0.195272 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390183574e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12258865512 prefetch_bytes=726663168 reduce_bytes=380633088 weight_movement_bytes=1107296256 egress_bytes=6133387624 ingress_bytes=7232774144 link_bytes=7233118528 total_nvlink_bytes=13366161768 weight_to_token_ratio=0.090326 prefetch_share_of_ingress=0.100468 dedup_ratio=0.244073 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=11 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029674 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.183375770e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74110 payload_rows_remote=64910 dup_rows=24194 dup_rows_remote=21246 dedup_ratio=0.246114 router_tokens_per_expert=[764, 754, 750, 786, 718, 769, 773, 742, 827, 754, 765, 732, 767, 765, 789, 799, 773, 770, 830, 750, 729, 725, 771, 774, 813, 754, 799, 775, 748, 786, 789, 753, 773, 746, 718, 794, 742, 773, 780, 769, 765, 792, 776, 742, 778, 752, 716, 804, 799, 791, 784, 779, 767, 786, 746, 788, 798, 763, 759, 753, 772, 765, 754, 735] payload_rows_per_dst=[9200, 9610, 9222, 9418, 8870, 9314, 9274, 9202] dup_rows_per_dst=[2948, 3296, 3006, 3122, 2800, 2926, 3100, 2996] combine_rows_per_src=[9200, 9610, 9222, 9418, 8870, 9314, 9274, 9202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=64862 dup_rows=24062 dup_rows_remote=21132 dedup_ratio=0.244771 router_tokens_per_expert=[756, 805, 736, 788, 798, 745, 758, 745, 742, 754, 781, 742, 781, 781, 810, 759, 787, 782, 762, 733, 753, 791, 780, 827, 753, 756, 741, 836, 756, 826, 744, 743, 747, 778, 777, 765, 790, 758, 757, 773, 816, 736, 743, 745, 817, 745, 805, 726, 771, 755, 754, 740, 760, 796, 768, 766, 788, 757, 760, 809, 779, 729, 749, 742] payload_rows_per_dst=[9268, 9226, 9326, 9380, 9274, 9286, 9212, 9270] dup_rows_per_dst=[2994, 3074, 3104, 2930, 3016, 2980, 3008, 2956] combine_rows_per_src=[9268, 9226, 9326, 9380, 9274, 9286, 9212, 9270] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=344888 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266339128 ingress_bytes=335200256 link_bytes=335200256 token_bytes=532333368 weight_movement_bytes=69206016 total_nvlink_bytes=601539384 weight_to_token_ratio=0.130005 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261331577e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64976 dup_rows=24112 dup_rows_remote=21134 dedup_ratio=0.245280 router_tokens_per_expert=[755, 791, 805, 848, 804, 762, 811, 759, 776, 744, 771, 741, 757, 768, 747, 827, 812, 769, 784, 766, 698, 763, 744, 752, 750, 782, 774, 785, 740, 779, 816, 779, 770, 746, 717, 759, 760, 783, 777, 758, 796, 754, 759, 775, 806, 759, 720, 784, 739, 780, 757, 764, 787, 754, 761, 755, 775, 723, 800, 792, 779, 733, 721, 750] payload_rows_per_dst=[9474, 9248, 9260, 9410, 9148, 9244, 9216, 9192] dup_rows_per_dst=[3196, 3014, 2916, 3000, 2992, 3062, 2978, 2954] combine_rows_per_src=[9474, 9248, 9260, 9410, 9148, 9244, 9216, 9192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12258865512 prefetch_bytes=726663168 reduce_bytes=380633088 weight_movement_bytes=1107296256 egress_bytes=6133387624 ingress_bytes=7232774144 link_bytes=7233118528 total_nvlink_bytes=13366161768 weight_to_token_ratio=0.090326 prefetch_share_of_ingress=0.100468 dedup_ratio=0.244073 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=11 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029674 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.183375770e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265871360 dispatch_meta_bytes=344624 combine_payload_bytes=265871360 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266215984 ingress_bytes=335077376 link_bytes=335077376 token_bytes=532087344 weight_movement_bytes=69206016 total_nvlink_bytes=601293360 weight_to_token_ratio=0.130065 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260869190e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265674752 dispatch_meta_bytes=343976 combine_payload_bytes=265674752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266018728 ingress_bytes=300277760 link_bytes=300277760 token_bytes=531693480 weight_movement_bytes=34603008 total_nvlink_bytes=566296488 weight_to_token_ratio=0.065081 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129921037e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12266799912 prefetch_bytes=657457152 reduce_bytes=553648128 weight_movement_bytes=1211105280 egress_bytes=6137357096 ingress_bytes=7340548096 link_bytes=7340548096 total_nvlink_bytes=13477905192 weight_to_token_ratio=0.098730 prefetch_share_of_ingress=0.089565 dedup_ratio=0.243767 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=16 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029022 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.200951805e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266141696 dispatch_meta_bytes=344440 combine_payload_bytes=266141696 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266486136 ingress_bytes=369950720 link_bytes=369950720 token_bytes=532627832 weight_movement_bytes=103809024 total_nvlink_bytes=636436856 weight_to_token_ratio=0.194900 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392094776e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=65078 dup_rows=24018 dup_rows_remote=20834 dedup_ratio=0.244324 router_tokens_per_expert=[759, 752, 716, 789, 766, 763, 794, 742, 767, 739, 772, 818, 760, 721, 801, 715, 787, 774, 808, 778, 731, 765, 780, 747, 736, 752, 809, 768, 740, 779, 731, 783, 797, 702, 824, 743, 762, 775, 784, 809, 766, 782, 770, 788, 786, 777, 750, 808, 748, 763, 804, 767, 769, 794, 777, 738, 755, 788, 779, 784, 757, 742, 778, 744] payload_rows_per_dst=[9250, 9336, 9336, 9248, 9208, 9318, 9304, 9286] dup_rows_per_dst=[2912, 2850, 3004, 2948, 3184, 3136, 3016, 2968] combine_rows_per_src=[9250, 9336, 9336, 9248, 9208, 9318, 9304, 9286] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12237856392 prefetch_bytes=692060160 reduce_bytes=553648128 weight_movement_bytes=1245708288 egress_bytes=6122880648 ingress_bytes=7360684032 link_bytes=7360684032 total_nvlink_bytes=13483564680 weight_to_token_ratio=0.101791 prefetch_share_of_ingress=0.094021 dedup_ratio=0.244930 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=16 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028804 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.204246149e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none - [2026-08-26 15:10:27] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 791.4 | throughput per GPU (TFLOP/s/GPU): 161.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.020314E+01 | loss scale: 1.0 | grad norm: 158.229 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74154 payload_rows_remote=64876 dup_rows=24150 dup_rows_remote=20984 dedup_ratio=0.245667 router_tokens_per_expert=[747, 774, 758, 802, 774, 825, 727, 747, 706, 796, 791, 807, 742, 743, 761, 804, 765, 766, 797, 775, 754, 724, 799, 743, 774, 776, 706, 760, 800, 763, 740, 749, 774, 754, 738, 759, 790, 849, 770, 788, 734, 770, 773, 719, 790, 766, 735, 695, 729, 756, 779, 809, 774, 804, 785, 826, 766, 784, 762, 788, 759, 763, 780, 789] payload_rows_per_dst=[9244, 9318, 9198, 9238, 9278, 9222, 9380, 9276] dup_rows_per_dst=[3064, 2982, 3048, 2898, 3166, 2742, 3144, 3106] combine_rows_per_src=[9244, 9318, 9198, 9238, 9278, 9222, 9380, 9276] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=64826 dup_rows=24096 dup_rows_remote=21092 dedup_ratio=0.245117 router_tokens_per_expert=[783, 767, 739, 744, 740, 750, 797, 761, 778, 757, 759, 788, 762, 730, 766, 765, 811, 755, 736, 761, 788, 770, 806, 766, 765, 794, 745, 800, 751, 756, 813, 805, 745, 771, 778, 743, 739, 792, 761, 829, 759, 798, 810, 738, 745, 757, 744, 788, 750, 779, 769, 772, 792, 766, 748, 770, 778, 743, 744, 778, 771, 753, 737, 797] payload_rows_per_dst=[9188, 9228, 9382, 9372, 9318, 9272, 9294, 9154] dup_rows_per_dst=[2974, 2982, 3004, 3086, 2998, 3006, 2998, 3048] combine_rows_per_src=[9188, 9228, 9382, 9372, 9318, 9272, 9294, 9154] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=65084 dup_rows=24046 dup_rows_remote=21106 dedup_ratio=0.244609 router_tokens_per_expert=[771, 751, 771, 744, 781, 737, 727, 802, 737, 734, 737, 784, 788, 795, 751, 770, 738, 809, 740, 738, 769, 793, 763, 801, 785, 701, 784, 756, 777, 814, 787, 765, 796, 755, 734, 787, 749, 774, 772, 766, 754, 701, 750, 757, 747, 772, 780, 796, 768, 793, 822, 768, 732, 797, 838, 791, 800, 777, 815, 763, 758, 761, 775, 704] payload_rows_per_dst=[9202, 9260, 9244, 9334, 9240, 9174, 9512, 9292] dup_rows_per_dst=[2966, 2932, 3058, 3004, 3026, 2940, 3106, 3014] combine_rows_per_src=[9202, 9260, 9244, 9334, 9240, 9174, 9512, 9292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12249891808 prefetch_bytes=692060160 reduce_bytes=795869184 weight_movement_bytes=1487929344 egress_bytes=6128903136 ingress_bytes=7608918016 link_bytes=7608918016 total_nvlink_bytes=13737821152 weight_to_token_ratio=0.121465 prefetch_share_of_ingress=0.090954 dedup_ratio=0.244708 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=23 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028043 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.244858519e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12255753552 prefetch_bytes=657457152 reduce_bytes=968884224 weight_movement_bytes=1626341376 egress_bytes=6131832144 ingress_bytes=7750262784 link_bytes=7750262784 total_nvlink_bytes=13882094928 weight_to_token_ratio=0.132700 prefetch_share_of_ingress=0.084830 dedup_ratio=0.244093 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=28 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.267983257e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266559488 dispatch_meta_bytes=343648 combine_payload_bytes=266559488 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266903136 ingress_bytes=370368512 link_bytes=370368512 token_bytes=533462624 weight_movement_bytes=103809024 total_nvlink_bytes=637271648 weight_to_token_ratio=0.194595 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393666894e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12249642576 prefetch_bytes=692060160 reduce_bytes=865075200 weight_movement_bytes=1557135360 egress_bytes=6128776784 ingress_bytes=7678001152 link_bytes=7678001152 total_nvlink_bytes=13806777936 weight_to_token_ratio=0.127117 prefetch_share_of_ingress=0.090135 dedup_ratio=0.244561 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=25 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027935 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.256160878e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none + [2026-08-26 15:58:16] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 790.9 | throughput per GPU (TFLOP/s/GPU): 161.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.020314E+01 | loss scale: 1.0 | grad norm: 158.229 | number of skipped iterations: 0 | number of nan iterations: 0 | WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74156 payload_rows_remote=64832 dup_rows=24148 dup_rows_remote=21102 dedup_ratio=0.245646 router_tokens_per_expert=[773, 795, 777, 772, 737, 759, 738, 780, 736, 777, 829, 744, 744, 775, 777, 803, 758, 758, 784, 753, 804, 781, 768, 817, 776, 711, 756, 755, 786, 752, 782, 760, 784, 773, 747, 776, 807, 764, 747, 788, 756, 740, 754, 757, 764, 791, 737, 800, 763, 766, 720, 772, 762, 763, 726, 784, 769, 766, 809, 752, 783, 803, 747, 765] payload_rows_per_dst=[9324, 9324, 9398, 9086, 9386, 9220, 9098, 9320] dup_rows_per_dst=[2938, 3046, 3048, 3070, 2986, 2978, 3014, 3068] combine_rows_per_src=[9324, 9324, 9398, 9086, 9386, 9220, 9098, 9320] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74390 payload_rows_remote=65026 dup_rows=23914 dup_rows_remote=20866 dedup_ratio=0.243266 router_tokens_per_expert=[835, 772, 793, 789, 752, 752, 731, 781, 737, 809, 781, 775, 727, 792, 749, 786, 776, 789, 747, 788, 782, 726, 728, 766, 764, 757, 764, 769, 774, 783, 802, 793, 790, 735, 735, 793, 778, 755, 789, 725, 739, 784, 730, 762, 746, 775, 755, 723, 811, 794, 789, 730, 773, 771, 803, 766, 765, 802, 771, 730, 751, 759, 816, 738] payload_rows_per_dst=[9424, 9346, 9202, 9364, 9278, 9076, 9384, 9316] dup_rows_per_dst=[2986, 2966, 3002, 3048, 2922, 2952, 3090, 2948] combine_rows_per_src=[9424, 9346, 9202, 9364, 9278, 9076, 9384, 9316] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74238 payload_rows_remote=64702 dup_rows=24066 dup_rows_remote=20808 dedup_ratio=0.244812 router_tokens_per_expert=[751, 793, 791, 778, 782, 749, 780, 794, 726, 777, 808, 767, 774, 799, 752, 706, 750, 816, 747, 711, 746, 764, 747, 774, 758, 738, 753, 841, 724, 765, 774, 752, 769, 744, 768, 774, 753, 774, 748, 784, 791, 778, 806, 765, 758, 762, 733, 756, 767, 771, 770, 773, 802, 814, 753, 742, 743, 778, 742, 792, 789, 813, 759, 794] payload_rows_per_dst=[9374, 9250, 9160, 9264, 9236, 9284, 9134, 9536] dup_rows_per_dst=[3062, 2968, 2940, 2946, 2966, 3014, 2912, 3258] combine_rows_per_src=[9374, 9250, 9160, 9264, 9236, 9284, 9134, 9536] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74410 payload_rows_remote=65118 dup_rows=23894 dup_rows_remote=20838 dedup_ratio=0.243062 router_tokens_per_expert=[779, 781, 768, 780, 706, 745, 743, 790, 784, 732, 766, 741, 771, 769, 801, 792, 830, 770, 801, 762, 804, 728, 768, 784, 771, 739, 732, 751, 816, 745, 778, 764, 813, 770, 747, 777, 751, 767, 771, 730, 749, 742, 817, 751, 767, 790, 791, 749, 748, 746, 767, 791, 727, 790, 774, 743, 763, 742, 793, 779, 791, 797, 762, 766] payload_rows_per_dst=[9292, 9330, 9388, 9374, 9254, 9640, 9242, 8890] dup_rows_per_dst=[3056, 3064, 2942, 3004, 2812, 3470, 2848, 2698] combine_rows_per_src=[9292, 9330, 9388, 9374, 9254, 9640, 9242, 8890] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=64826 dup_rows=24096 dup_rows_remote=21092 dedup_ratio=0.245117 router_tokens_per_expert=[783, 767, 739, 744, 740, 750, 797, 761, 778, 757, 759, 788, 762, 730, 766, 765, 811, 755, 736, 761, 788, 770, 806, 766, 765, 794, 745, 800, 751, 756, 813, 805, 745, 771, 778, 743, 739, 792, 761, 829, 759, 798, 810, 738, 745, 757, 744, 788, 750, 779, 769, 772, 792, 766, 748, 770, 778, 743, 744, 778, 771, 753, 737, 797] payload_rows_per_dst=[9188, 9228, 9382, 9372, 9318, 9272, 9294, 9154] dup_rows_per_dst=[2974, 2982, 3004, 3086, 2998, 3006, 2998, 3048] combine_rows_per_src=[9188, 9228, 9382, 9372, 9318, 9272, 9294, 9154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74390 payload_rows_remote=65026 dup_rows=23914 dup_rows_remote=20866 dedup_ratio=0.243266 router_tokens_per_expert=[835, 772, 793, 789, 752, 752, 731, 781, 737, 809, 781, 775, 727, 792, 749, 786, 776, 789, 747, 788, 782, 726, 728, 766, 764, 757, 764, 769, 774, 783, 802, 793, 790, 735, 735, 793, 778, 755, 789, 725, 739, 784, 730, 762, 746, 775, 755, 723, 811, 794, 789, 730, 773, 771, 803, 766, 765, 802, 771, 730, 751, 759, 816, 738] payload_rows_per_dst=[9424, 9346, 9202, 9364, 9278, 9076, 9384, 9316] dup_rows_per_dst=[2986, 2966, 3002, 3048, 2922, 2952, 3090, 2948] combine_rows_per_src=[9424, 9346, 9202, 9364, 9278, 9076, 9384, 9316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=65084 dup_rows=24046 dup_rows_remote=21106 dedup_ratio=0.244609 router_tokens_per_expert=[771, 751, 771, 744, 781, 737, 727, 802, 737, 734, 737, 784, 788, 795, 751, 770, 738, 809, 740, 738, 769, 793, 763, 801, 785, 701, 784, 756, 777, 814, 787, 765, 796, 755, 734, 787, 749, 774, 772, 766, 754, 701, 750, 757, 747, 772, 780, 796, 768, 793, 822, 768, 732, 797, 838, 791, 800, 777, 815, 763, 758, 761, 775, 704] payload_rows_per_dst=[9202, 9260, 9244, 9334, 9240, 9174, 9512, 9292] dup_rows_per_dst=[2966, 2932, 3058, 3004, 3026, 2940, 3106, 3014] combine_rows_per_src=[9202, 9260, 9244, 9334, 9240, 9174, 9512, 9292] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265732096 dispatch_meta_bytes=343440 combine_payload_bytes=265732096 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266075536 ingress_bytes=369541120 link_bytes=369541120 token_bytes=531807632 weight_movement_bytes=103809024 total_nvlink_bytes=635616656 weight_to_token_ratio=0.195200 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390553484e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265527296 dispatch_meta_bytes=343672 combine_payload_bytes=265527296 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265870968 ingress_bytes=334733312 link_bytes=334733312 token_bytes=531398264 weight_movement_bytes=69206016 total_nvlink_bytes=600604280 weight_to_token_ratio=0.130234 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259574505e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266584064 dispatch_meta_bytes=344760 combine_payload_bytes=266584064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266928824 ingress_bytes=301187072 link_bytes=301187072 token_bytes=533512888 weight_movement_bytes=34603008 total_nvlink_bytes=568115896 weight_to_token_ratio=0.064859 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133342704e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266346496 dispatch_meta_bytes=343568 combine_payload_bytes=266346496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266690064 ingress_bytes=300949504 link_bytes=300949504 token_bytes=533036560 weight_movement_bytes=34603008 total_nvlink_bytes=567639568 weight_to_token_ratio=0.064917 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132448755e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265551872 dispatch_meta_bytes=343736 combine_payload_bytes=265551872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265895608 ingress_bytes=300154880 link_bytes=300154880 token_bytes=531447480 weight_movement_bytes=34603008 total_nvlink_bytes=566050488 weight_to_token_ratio=0.065111 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129458649e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265019392 dispatch_meta_bytes=342040 combine_payload_bytes=265019392 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265361432 ingress_bytes=334225408 link_bytes=334225408 token_bytes=530380824 weight_movement_bytes=69206016 total_nvlink_bytes=599586840 weight_to_token_ratio=0.130484 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257663303e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266723328 dispatch_meta_bytes=343824 combine_payload_bytes=266723328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267067152 ingress_bytes=301326336 link_bytes=301326336 token_bytes=533790480 weight_movement_bytes=34603008 total_nvlink_bytes=568393488 weight_to_token_ratio=0.064825 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133866744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265019392 dispatch_meta_bytes=342040 combine_payload_bytes=265019392 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265361432 ingress_bytes=334225408 link_bytes=334225408 token_bytes=530380824 weight_movement_bytes=69206016 total_nvlink_bytes=599586840 weight_to_token_ratio=0.130484 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257663303e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265527296 dispatch_meta_bytes=343672 combine_payload_bytes=265527296 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265870968 ingress_bytes=334733312 link_bytes=334733312 token_bytes=531398264 weight_movement_bytes=69206016 total_nvlink_bytes=600604280 weight_to_token_ratio=0.130234 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259574505e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266346496 dispatch_meta_bytes=343568 combine_payload_bytes=266346496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266690064 ingress_bytes=300949504 link_bytes=300949504 token_bytes=533036560 weight_movement_bytes=34603008 total_nvlink_bytes=567639568 weight_to_token_ratio=0.064917 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132448755e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266584064 dispatch_meta_bytes=344760 combine_payload_bytes=266584064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266928824 ingress_bytes=301187072 link_bytes=301187072 token_bytes=533512888 weight_movement_bytes=34603008 total_nvlink_bytes=568115896 weight_to_token_ratio=0.064859 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133342704e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74154 payload_rows_remote=64876 dup_rows=24150 dup_rows_remote=20984 dedup_ratio=0.245667 router_tokens_per_expert=[747, 774, 758, 802, 774, 825, 727, 747, 706, 796, 791, 807, 742, 743, 761, 804, 765, 766, 797, 775, 754, 724, 799, 743, 774, 776, 706, 760, 800, 763, 740, 749, 774, 754, 738, 759, 790, 849, 770, 788, 734, 770, 773, 719, 790, 766, 735, 695, 729, 756, 779, 809, 774, 804, 785, 826, 766, 784, 762, 788, 759, 763, 780, 789] payload_rows_per_dst=[9244, 9318, 9198, 9238, 9278, 9222, 9380, 9276] dup_rows_per_dst=[3064, 2982, 3048, 2898, 3166, 2742, 3144, 3106] combine_rows_per_src=[9244, 9318, 9198, 9238, 9278, 9222, 9380, 9276] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74182 payload_rows_remote=65002 dup_rows=24122 dup_rows_remote=21154 dedup_ratio=0.245382 router_tokens_per_expert=[781, 759, 725, 816, 751, 712, 798, 763, 774, 797, 761, 723, 788, 817, 768, 726, 782, 790, 791, 766, 746, 756, 765, 749, 794, 769, 756, 761, 765, 735, 758, 770, 828, 800, 790, 768, 742, 782, 770, 739, 723, 806, 811, 735, 776, 776, 729, 801, 744, 782, 784, 765, 739, 728, 774, 758, 768, 773, 819, 761, 770, 774, 784, 741] payload_rows_per_dst=[9192, 9316, 9272, 9234, 9326, 9322, 9180, 9340] dup_rows_per_dst=[3018, 2992, 3018, 2982, 3112, 2992, 2968, 3040] combine_rows_per_src=[9192, 9316, 9272, 9234, 9326, 9322, 9180, 9340] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=64880 dup_rows=24048 dup_rows_remote=21022 dedup_ratio=0.244629 router_tokens_per_expert=[806, 731, 790, 735, 723, 722, 767, 732, 780, 726, 768, 761, 738, 730, 845, 793, 753, 777, 798, 776, 748, 779, 751, 794, 771, 754, 769, 772, 750, 784, 741, 825, 781, 768, 745, 793, 788, 782, 756, 788, 774, 815, 776, 781, 771, 781, 768, 778, 763, 749, 750, 774, 759, 825, 770, 762, 753, 687, 716, 770, 797, 827, 764, 752] payload_rows_per_dst=[9154, 9294, 9362, 9240, 9376, 9426, 9262, 9142] dup_rows_per_dst=[2858, 2988, 2990, 3092, 3026, 3062, 3042, 2990] combine_rows_per_src=[9154, 9294, 9362, 9240, 9376, 9426, 9262, 9142] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74054 payload_rows_remote=64726 dup_rows=24250 dup_rows_remote=21066 dedup_ratio=0.246684 router_tokens_per_expert=[736, 750, 708, 773, 790, 771, 738, 783, 804, 749, 739, 764, 798, 764, 741, 784, 795, 808, 757, 814, 753, 768, 784, 777, 825, 785, 709, 798, 752, 773, 760, 785, 790, 773, 765, 754, 718, 771, 811, 756, 778, 802, 767, 813, 757, 737, 760, 721, 786, 775, 776, 766, 751, 733, 772, 751, 750, 758, 744, 760, 817, 801, 741, 763] payload_rows_per_dst=[9190, 9274, 9328, 9314, 9268, 9200, 9170, 9310] dup_rows_per_dst=[2908, 3012, 3184, 3060, 3008, 3070, 3050, 2958] combine_rows_per_src=[9190, 9274, 9328, 9314, 9268, 9200, 9170, 9310] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74250 payload_rows_remote=65108 dup_rows=24054 dup_rows_remote=21166 dedup_ratio=0.244690 router_tokens_per_expert=[738, 797, 823, 752, 782, 792, 810, 778, 761, 746, 760, 793, 750, 728, 785, 760, 804, 769, 766, 780, 801, 740, 829, 747, 738, 744, 773, 738, 733, 845, 772, 778, 791, 756, 757, 717, 767, 733, 817, 805, 786, 736, 723, 782, 789, 736, 767, 696, 792, 811, 773, 740, 769, 723, 766, 751, 744, 827, 810, 755, 769, 773, 746, 733] payload_rows_per_dst=[9462, 9184, 9326, 9266, 9302, 9142, 9264, 9304] dup_rows_per_dst=[3082, 2982, 3146, 2976, 2984, 2888, 2986, 3010] combine_rows_per_src=[9462, 9184, 9326, 9266, 9302, 9142, 9264, 9304] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=65008 dup_rows=23940 dup_rows_remote=20944 dedup_ratio=0.243530 router_tokens_per_expert=[757, 763, 777, 753, 758, 776, 714, 815, 774, 798, 735, 756, 700, 775, 811, 759, 760, 746, 774, 745, 757, 796, 735, 734, 761, 807, 787, 725, 802, 719, 821, 754, 726, 795, 793, 763, 795, 743, 722, 806, 794, 754, 786, 804, 781, 766, 763, 801, 748, 774, 825, 804, 734, 782, 745, 785, 774, 797, 754, 748, 742, 748, 776, 780] payload_rows_per_dst=[9258, 9218, 9198, 9356, 9330, 9458, 9340, 9206] dup_rows_per_dst=[2968, 2998, 2896, 2996, 2956, 3040, 3054, 3032] combine_rows_per_src=[9258, 9218, 9198, 9356, 9330, 9458, 9340, 9206] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74456 payload_rows_remote=65032 dup_rows=23848 dup_rows_remote=20778 dedup_ratio=0.242594 router_tokens_per_expert=[742, 768, 766, 732, 764, 749, 791, 767, 773, 803, 754, 804, 796, 746, 781, 790, 743, 832, 741, 736, 729, 812, 767, 745, 728, 783, 767, 797, 761, 785, 778, 773, 791, 761, 748, 781, 775, 810, 735, 754, 742, 793, 802, 766, 741, 772, 737, 757, 758, 728, 794, 787, 776, 768, 748, 775, 758, 752, 777, 811, 772, 756, 757, 767] payload_rows_per_dst=[9208, 9424, 9320, 9266, 9366, 9242, 9322, 9308] dup_rows_per_dst=[2950, 3070, 2890, 3078, 2944, 2978, 2946, 2992] combine_rows_per_src=[9208, 9424, 9320, 9266, 9366, 9242, 9322, 9308] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74224 payload_rows_remote=64886 dup_rows=24080 dup_rows_remote=20932 dedup_ratio=0.244954 router_tokens_per_expert=[807, 775, 720, 808, 791, 729, 750, 774, 807, 768, 741, 772, 737, 749, 727, 811, 764, 793, 759, 761, 749, 788, 760, 804, 767, 776, 779, 816, 747, 729, 763, 777, 777, 793, 746, 782, 787, 808, 752, 730, 760, 749, 779, 803, 789, 749, 754, 780, 747, 757, 761, 754, 820, 753, 781, 742, 719, 790, 748, 781, 766, 752, 780, 765] payload_rows_per_dst=[9314, 9182, 9328, 9098, 9138, 9354, 9472, 9338] dup_rows_per_dst=[2994, 2962, 3028, 2870, 2976, 2972, 3130, 3148] combine_rows_per_src=[9314, 9182, 9328, 9098, 9138, 9354, 9472, 9338] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74298 payload_rows_remote=64710 dup_rows=24006 dup_rows_remote=20718 dedup_ratio=0.244202 router_tokens_per_expert=[764, 785, 767, 759, 783, 814, 727, 736, 765, 816, 770, 757, 789, 797, 775, 775, 756, 754, 825, 782, 729, 762, 771, 787, 789, 740, 743, 771, 741, 786, 796, 777, 730, 803, 786, 787, 800, 734, 782, 739, 760, 751, 801, 756, 756, 791, 782, 711, 761, 746, 701, 779, 800, 756, 764, 765, 773, 743, 745, 752, 795, 769, 797, 749] payload_rows_per_dst=[9588, 9422, 8984, 9304, 9348, 9194, 9182, 9276] dup_rows_per_dst=[3288, 3070, 2832, 2982, 2974, 2928, 2962, 2970] combine_rows_per_src=[9588, 9422, 8984, 9304, 9348, 9194, 9182, 9276] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74224 payload_rows_remote=64886 dup_rows=24080 dup_rows_remote=20932 dedup_ratio=0.244954 router_tokens_per_expert=[807, 775, 720, 808, 791, 729, 750, 774, 807, 768, 741, 772, 737, 749, 727, 811, 764, 793, 759, 761, 749, 788, 760, 804, 767, 776, 779, 816, 747, 729, 763, 777, 777, 793, 746, 782, 787, 808, 752, 730, 760, 749, 779, 803, 789, 749, 754, 780, 747, 757, 761, 754, 820, 753, 781, 742, 719, 790, 748, 781, 766, 752, 780, 765] payload_rows_per_dst=[9314, 9182, 9328, 9098, 9138, 9354, 9472, 9338] dup_rows_per_dst=[2994, 2962, 3028, 2870, 2976, 2972, 3130, 3148] combine_rows_per_src=[9314, 9182, 9328, 9098, 9138, 9354, 9472, 9338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74054 payload_rows_remote=64726 dup_rows=24250 dup_rows_remote=21066 dedup_ratio=0.246684 router_tokens_per_expert=[736, 750, 708, 773, 790, 771, 738, 783, 804, 749, 739, 764, 798, 764, 741, 784, 795, 808, 757, 814, 753, 768, 784, 777, 825, 785, 709, 798, 752, 773, 760, 785, 790, 773, 765, 754, 718, 771, 811, 756, 778, 802, 767, 813, 757, 737, 760, 721, 786, 775, 776, 766, 751, 733, 772, 751, 750, 758, 744, 760, 817, 801, 741, 763] payload_rows_per_dst=[9190, 9274, 9328, 9314, 9268, 9200, 9170, 9310] dup_rows_per_dst=[2908, 3012, 3184, 3060, 3008, 3070, 3050, 2958] combine_rows_per_src=[9190, 9274, 9328, 9314, 9268, 9200, 9170, 9310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265732096 dispatch_meta_bytes=343440 combine_payload_bytes=265732096 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266075536 ingress_bytes=369541120 link_bytes=369541120 token_bytes=531807632 weight_movement_bytes=103809024 total_nvlink_bytes=635616656 weight_to_token_ratio=0.195200 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390553484e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=344624 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266592816 ingress_bytes=370057216 link_bytes=370057216 token_bytes=532841008 weight_movement_bytes=103809024 total_nvlink_bytes=636650032 weight_to_token_ratio=0.194822 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392495512e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265748480 dispatch_meta_bytes=343608 combine_payload_bytes=265748480 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266092088 ingress_bytes=334954496 link_bytes=334954496 token_bytes=531840568 weight_movement_bytes=69206016 total_nvlink_bytes=601046584 weight_to_token_ratio=0.130125 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260406802e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265117696 dispatch_meta_bytes=343168 combine_payload_bytes=265117696 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265460864 ingress_bytes=334323712 link_bytes=334323712 token_bytes=530578560 weight_movement_bytes=69206016 total_nvlink_bytes=599784576 weight_to_token_ratio=0.130435 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258033213e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=65008 dup_rows=23940 dup_rows_remote=20944 dedup_ratio=0.243530 router_tokens_per_expert=[757, 763, 777, 753, 758, 776, 714, 815, 774, 798, 735, 756, 700, 775, 811, 759, 760, 746, 774, 745, 757, 796, 735, 734, 761, 807, 787, 725, 802, 719, 821, 754, 726, 795, 793, 763, 795, 743, 722, 806, 794, 754, 786, 804, 781, 766, 763, 801, 748, 774, 825, 804, 734, 782, 745, 785, 774, 797, 754, 748, 742, 748, 776, 780] payload_rows_per_dst=[9258, 9218, 9198, 9356, 9330, 9458, 9340, 9206] dup_rows_per_dst=[2968, 2998, 2896, 2996, 2956, 3040, 3054, 3032] combine_rows_per_src=[9258, 9218, 9198, 9356, 9330, 9458, 9340, 9206] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266682368 dispatch_meta_bytes=345096 combine_payload_bytes=266682368 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267027464 ingress_bytes=335888384 link_bytes=335888384 token_bytes=533709832 weight_movement_bytes=69206016 total_nvlink_bytes=602915848 weight_to_token_ratio=0.129670 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263920948e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266272768 dispatch_meta_bytes=343808 combine_payload_bytes=266272768 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266616576 ingress_bytes=335478784 link_bytes=335478784 token_bytes=532889344 weight_movement_bytes=69206016 total_nvlink_bytes=602095360 weight_to_token_ratio=0.129869 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262379656e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74250 payload_rows_remote=65108 dup_rows=24054 dup_rows_remote=21166 dedup_ratio=0.244690 router_tokens_per_expert=[738, 797, 823, 752, 782, 792, 810, 778, 761, 746, 760, 793, 750, 728, 785, 760, 804, 769, 766, 780, 801, 740, 829, 747, 738, 744, 773, 738, 733, 845, 772, 778, 791, 756, 757, 717, 767, 733, 817, 805, 786, 736, 723, 782, 789, 736, 767, 696, 792, 811, 773, 740, 769, 723, 766, 751, 744, 827, 810, 755, 769, 773, 746, 733] payload_rows_per_dst=[9462, 9184, 9326, 9266, 9302, 9142, 9264, 9304] dup_rows_per_dst=[3082, 2982, 3146, 2976, 2984, 2888, 2986, 3010] combine_rows_per_src=[9462, 9184, 9326, 9266, 9302, 9142, 9264, 9304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=343240 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266714312 ingress_bytes=370180096 link_bytes=370180096 token_bytes=533085384 weight_movement_bytes=103809024 total_nvlink_bytes=636894408 weight_to_token_ratio=0.194732 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392957899e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=343272 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266116328 ingress_bytes=300376064 link_bytes=300376064 token_bytes=531889384 weight_movement_bytes=34603008 total_nvlink_bytes=566492392 weight_to_token_ratio=0.065057 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130290947e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=344624 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266592816 ingress_bytes=370057216 link_bytes=370057216 token_bytes=532841008 weight_movement_bytes=103809024 total_nvlink_bytes=636650032 weight_to_token_ratio=0.194822 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392495512e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265052160 dispatch_meta_bytes=341712 combine_payload_bytes=265052160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265393872 ingress_bytes=299655168 link_bytes=299655168 token_bytes=530446032 weight_movement_bytes=34603008 total_nvlink_bytes=565049040 weight_to_token_ratio=0.065234 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127578273e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74328 payload_rows_remote=65130 dup_rows=23976 dup_rows_remote=20898 dedup_ratio=0.243896 router_tokens_per_expert=[803, 732, 776, 778, 719, 776, 733, 769, 782, 774, 750, 785, 787, 761, 769, 749, 761, 751, 798, 741, 711, 749, 774, 783, 757, 727, 766, 749, 794, 795, 796, 767, 789, 761, 781, 728, 746, 764, 765, 804, 803, 781, 819, 775, 728, 760, 790, 749, 758, 783, 766, 780, 780, 817, 733, 695, 785, 766, 797, 782, 786, 789, 782, 748] payload_rows_per_dst=[9200, 9278, 9268, 9404, 9198, 9342, 9226, 9412] dup_rows_per_dst=[2972, 3036, 2868, 2898, 3078, 3068, 2998, 3058] combine_rows_per_src=[9200, 9278, 9268, 9404, 9198, 9342, 9226, 9412] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74454 payload_rows_remote=65156 dup_rows=23850 dup_rows_remote=20824 dedup_ratio=0.242615 router_tokens_per_expert=[757, 758, 770, 750, 782, 793, 796, 809, 764, 732, 687, 748, 738, 818, 772, 736, 749, 794, 755, 760, 765, 778, 789, 772, 834, 739, 816, 786, 778, 752, 744, 779, 770, 775, 750, 750, 760, 785, 725, 767, 759, 805, 790, 798, 768, 773, 815, 759, 781, 767, 782, 726, 813, 742, 759, 749, 731, 769, 777, 734, 775, 772, 731, 795] payload_rows_per_dst=[9478, 9114, 9298, 9432, 9242, 9464, 9180, 9246] dup_rows_per_dst=[2952, 2876, 3026, 3024, 2922, 3070, 3058, 2922] combine_rows_per_src=[9478, 9114, 9298, 9432, 9242, 9464, 9180, 9246] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74350 payload_rows_remote=65026 dup_rows=23954 dup_rows_remote=21014 dedup_ratio=0.243673 router_tokens_per_expert=[721, 764, 804, 802, 741, 757, 785, 779, 796, 814, 814, 766, 774, 776, 767, 701, 759, 739, 812, 775, 766, 737, 761, 778, 733, 770, 773, 825, 754, 786, 784, 738, 756, 779, 734, 780, 799, 746, 803, 794, 775, 770, 797, 780, 716, 765, 789, 740, 797, 810, 742, 722, 781, 767, 778, 746, 691, 766, 747, 790, 759, 757, 778, 747] payload_rows_per_dst=[9300, 9304, 9286, 9342, 9340, 9324, 9268, 9186] dup_rows_per_dst=[3006, 3112, 2968, 2984, 3042, 2940, 3018, 2884] combine_rows_per_src=[9300, 9304, 9286, 9342, 9340, 9324, 9268, 9186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74574 payload_rows_remote=65196 dup_rows=23730 dup_rows_remote=20730 dedup_ratio=0.241394 router_tokens_per_expert=[747, 768, 745, 798, 739, 748, 758, 790, 738, 771, 694, 764, 747, 797, 751, 756, 761, 775, 780, 728, 801, 766, 758, 788, 821, 731, 800, 788, 736, 757, 748, 808, 840, 782, 812, 819, 701, 717, 762, 760, 745, 773, 747, 746, 796, 779, 777, 749, 781, 765, 699, 777, 754, 781, 838, 765, 775, 794, 779, 754, 751, 761, 841, 775] payload_rows_per_dst=[9272, 9132, 9420, 9378, 9384, 9228, 9348, 9412] dup_rows_per_dst=[2914, 2904, 2894, 3000, 3002, 2996, 2972, 3048] combine_rows_per_src=[9272, 9132, 9420, 9378, 9384, 9228, 9348, 9412] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74508 payload_rows_remote=65022 dup_rows=23796 dup_rows_remote=20662 dedup_ratio=0.242065 router_tokens_per_expert=[734, 719, 775, 767, 729, 734, 803, 739, 730, 807, 763, 768, 767, 706, 792, 791, 764, 809, 801, 797, 765, 800, 776, 755, 766, 768, 785, 782, 771, 780, 767, 750, 772, 772, 755, 705, 816, 802, 798, 777, 744, 745, 706, 800, 712, 818, 778, 781, 752, 762, 744, 788, 748, 732, 754, 777, 734, 756, 777, 752, 798, 817, 789, 831] payload_rows_per_dst=[9156, 9292, 9418, 9260, 9374, 9240, 9282, 9486] dup_rows_per_dst=[2844, 2956, 3116, 3078, 3016, 2820, 2832, 3134] combine_rows_per_src=[9156, 9292, 9418, 9260, 9374, 9240, 9282, 9486] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=343272 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266116328 ingress_bytes=300376064 link_bytes=300376064 token_bytes=531889384 weight_movement_bytes=34603008 total_nvlink_bytes=566492392 weight_to_token_ratio=0.065057 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130290947e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265117696 dispatch_meta_bytes=343168 combine_payload_bytes=265117696 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265460864 ingress_bytes=334323712 link_bytes=334323712 token_bytes=530578560 weight_movement_bytes=69206016 total_nvlink_bytes=599784576 weight_to_token_ratio=0.130435 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258033213e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266272768 dispatch_meta_bytes=343808 combine_payload_bytes=266272768 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266616576 ingress_bytes=335478784 link_bytes=335478784 token_bytes=532889344 weight_movement_bytes=69206016 total_nvlink_bytes=602095360 weight_to_token_ratio=0.129869 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262379656e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266682368 dispatch_meta_bytes=345096 combine_payload_bytes=266682368 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267027464 ingress_bytes=335888384 link_bytes=335888384 token_bytes=533709832 weight_movement_bytes=69206016 total_nvlink_bytes=602915848 weight_to_token_ratio=0.129670 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263920948e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=64880 dup_rows=24048 dup_rows_remote=21022 dedup_ratio=0.244629 router_tokens_per_expert=[806, 731, 790, 735, 723, 722, 767, 732, 780, 726, 768, 761, 738, 730, 845, 793, 753, 777, 798, 776, 748, 779, 751, 794, 771, 754, 769, 772, 750, 784, 741, 825, 781, 768, 745, 793, 788, 782, 756, 788, 774, 815, 776, 781, 771, 781, 768, 778, 763, 749, 750, 774, 759, 825, 770, 762, 753, 687, 716, 770, 797, 827, 764, 752] payload_rows_per_dst=[9154, 9294, 9362, 9240, 9376, 9426, 9262, 9142] dup_rows_per_dst=[2858, 2988, 2990, 3092, 3026, 3062, 3042, 2990] combine_rows_per_src=[9154, 9294, 9362, 9240, 9376, 9426, 9262, 9142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74174 payload_rows_remote=64958 dup_rows=24130 dup_rows_remote=21224 dedup_ratio=0.245463 router_tokens_per_expert=[759, 728, 752, 768, 760, 734, 770, 770, 776, 768, 769, 762, 770, 767, 773, 727, 782, 805, 778, 747, 740, 807, 779, 808, 737, 743, 816, 789, 791, 760, 775, 792, 758, 753, 804, 784, 742, 799, 754, 760, 693, 791, 752, 791, 775, 798, 780, 795, 758, 774, 766, 765, 763, 726, 744, 765, 770, 768, 829, 751, 754, 735, 770, 783] payload_rows_per_dst=[9206, 9274, 9320, 9358, 9202, 9288, 9216, 9310] dup_rows_per_dst=[2876, 2950, 3172, 3048, 3106, 3062, 2906, 3010] combine_rows_per_src=[9206, 9274, 9320, 9358, 9202, 9288, 9216, 9310] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74606 payload_rows_remote=65210 dup_rows=23698 dup_rows_remote=20722 dedup_ratio=0.241069 router_tokens_per_expert=[802, 775, 763, 768, 719, 748, 783, 795, 782, 751, 763, 768, 818, 778, 779, 747, 783, 716, 785, 761, 739, 747, 763, 753, 742, 746, 793, 789, 741, 784, 836, 777, 765, 788, 811, 769, 749, 772, 787, 724, 749, 804, 746, 738, 759, 755, 753, 786, 768, 773, 808, 760, 754, 795, 733, 768, 751, 769, 760, 807, 753, 741, 794, 769] payload_rows_per_dst=[9370, 9396, 9166, 9376, 9282, 9252, 9344, 9420] dup_rows_per_dst=[2936, 2976, 2928, 3040, 3048, 2928, 2974, 2868] combine_rows_per_src=[9370, 9396, 9166, 9376, 9282, 9252, 9344, 9420] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=64868 dup_rows=24020 dup_rows_remote=20930 dedup_ratio=0.244344 router_tokens_per_expert=[769, 755, 778, 777, 741, 799, 781, 792, 781, 750, 771, 767, 789, 793, 732, 755, 779, 761, 739, 749, 764, 792, 751, 781, 709, 777, 775, 750, 753, 792, 766, 797, 722, 788, 793, 751, 774, 764, 711, 757, 792, 778, 812, 764, 783, 772, 758, 776, 755, 764, 743, 779, 774, 773, 751, 774, 762, 810, 764, 786, 749, 771, 765, 772] payload_rows_per_dst=[9416, 9424, 9366, 9136, 9030, 9368, 9518, 9026] dup_rows_per_dst=[3090, 3182, 2888, 2784, 2946, 3090, 3212, 2828] combine_rows_per_src=[9416, 9424, 9366, 9136, 9030, 9368, 9518, 9026] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74174 payload_rows_remote=64958 dup_rows=24130 dup_rows_remote=21224 dedup_ratio=0.245463 router_tokens_per_expert=[759, 728, 752, 768, 760, 734, 770, 770, 776, 768, 769, 762, 770, 767, 773, 727, 782, 805, 778, 747, 740, 807, 779, 808, 737, 743, 816, 789, 791, 760, 775, 792, 758, 753, 804, 784, 742, 799, 754, 760, 693, 791, 752, 791, 775, 798, 780, 795, 758, 774, 766, 765, 763, 726, 744, 765, 770, 768, 829, 751, 754, 735, 770, 783] payload_rows_per_dst=[9206, 9274, 9320, 9358, 9202, 9288, 9216, 9310] dup_rows_per_dst=[2876, 2950, 3172, 3048, 3106, 3062, 2906, 3010] combine_rows_per_src=[9206, 9274, 9320, 9358, 9202, 9288, 9216, 9310] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266878976 dispatch_meta_bytes=343920 combine_payload_bytes=266878976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267222896 ingress_bytes=301481984 link_bytes=301481984 token_bytes=534101872 weight_movement_bytes=34603008 total_nvlink_bytes=568704880 weight_to_token_ratio=0.064787 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134452435e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266346496 dispatch_meta_bytes=344160 combine_payload_bytes=266346496 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266690656 ingress_bytes=370155520 link_bytes=370155520 token_bytes=533037152 weight_movement_bytes=103809024 total_nvlink_bytes=636846176 weight_to_token_ratio=0.194750 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392865422e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266772480 dispatch_meta_bytes=344112 combine_payload_bytes=266772480 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267116592 ingress_bytes=370581504 link_bytes=370581504 token_bytes=533889072 weight_movement_bytes=103809024 total_nvlink_bytes=637698096 weight_to_token_ratio=0.194439 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394468365e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74508 payload_rows_remote=65022 dup_rows=23796 dup_rows_remote=20662 dedup_ratio=0.242065 router_tokens_per_expert=[734, 719, 775, 767, 729, 734, 803, 739, 730, 807, 763, 768, 767, 706, 792, 791, 764, 809, 801, 797, 765, 800, 776, 755, 766, 768, 785, 782, 771, 780, 767, 750, 772, 772, 755, 705, 816, 802, 798, 777, 744, 745, 706, 800, 712, 818, 778, 781, 752, 762, 744, 788, 748, 732, 754, 777, 734, 756, 777, 752, 798, 817, 789, 831] payload_rows_per_dst=[9156, 9292, 9418, 9260, 9374, 9240, 9282, 9486] dup_rows_per_dst=[2844, 2956, 3116, 3078, 3016, 2820, 2832, 3134] combine_rows_per_src=[9156, 9292, 9418, 9260, 9374, 9240, 9282, 9486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74454 payload_rows_remote=65156 dup_rows=23850 dup_rows_remote=20824 dedup_ratio=0.242615 router_tokens_per_expert=[757, 758, 770, 750, 782, 793, 796, 809, 764, 732, 687, 748, 738, 818, 772, 736, 749, 794, 755, 760, 765, 778, 789, 772, 834, 739, 816, 786, 778, 752, 744, 779, 770, 775, 750, 750, 760, 785, 725, 767, 759, 805, 790, 798, 768, 773, 815, 759, 781, 767, 782, 726, 813, 742, 759, 749, 731, 769, 777, 734, 775, 772, 731, 795] payload_rows_per_dst=[9478, 9114, 9298, 9432, 9242, 9464, 9180, 9246] dup_rows_per_dst=[2952, 2876, 3026, 3024, 2922, 3070, 3058, 2922] combine_rows_per_src=[9478, 9114, 9298, 9432, 9242, 9464, 9180, 9246] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74574 payload_rows_remote=65196 dup_rows=23730 dup_rows_remote=20730 dedup_ratio=0.241394 router_tokens_per_expert=[747, 768, 745, 798, 739, 748, 758, 790, 738, 771, 694, 764, 747, 797, 751, 756, 761, 775, 780, 728, 801, 766, 758, 788, 821, 731, 800, 788, 736, 757, 748, 808, 840, 782, 812, 819, 701, 717, 762, 760, 745, 773, 747, 746, 796, 779, 777, 749, 781, 765, 699, 777, 754, 781, 838, 765, 775, 794, 779, 754, 751, 761, 841, 775] payload_rows_per_dst=[9272, 9132, 9420, 9378, 9384, 9228, 9348, 9412] dup_rows_per_dst=[2914, 2904, 2894, 3000, 3002, 2996, 2972, 3048] combine_rows_per_src=[9272, 9132, 9420, 9378, 9384, 9228, 9348, 9412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74350 payload_rows_remote=65026 dup_rows=23954 dup_rows_remote=21014 dedup_ratio=0.243673 router_tokens_per_expert=[721, 764, 804, 802, 741, 757, 785, 779, 796, 814, 814, 766, 774, 776, 767, 701, 759, 739, 812, 775, 766, 737, 761, 778, 733, 770, 773, 825, 754, 786, 784, 738, 756, 779, 734, 780, 799, 746, 803, 794, 775, 770, 797, 780, 716, 765, 789, 740, 797, 810, 742, 722, 781, 767, 778, 746, 691, 766, 747, 790, 759, 757, 778, 747] payload_rows_per_dst=[9300, 9304, 9286, 9342, 9340, 9324, 9268, 9186] dup_rows_per_dst=[3006, 3112, 2968, 2984, 3042, 2940, 3018, 2884] combine_rows_per_src=[9300, 9304, 9286, 9342, 9340, 9324, 9268, 9186] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267042816 dispatch_meta_bytes=343704 combine_payload_bytes=267042816 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267386520 ingress_bytes=336248832 link_bytes=336248832 token_bytes=534429336 weight_movement_bytes=69206016 total_nvlink_bytes=603635352 weight_to_token_ratio=0.129495 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265277285e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266330112 dispatch_meta_bytes=342736 combine_payload_bytes=266330112 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266672848 ingress_bytes=335536128 link_bytes=335536128 token_bytes=533002960 weight_movement_bytes=69206016 total_nvlink_bytes=602208976 weight_to_token_ratio=0.129842 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262595437e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265748480 dispatch_meta_bytes=343608 combine_payload_bytes=265748480 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266092088 ingress_bytes=334954496 link_bytes=334954496 token_bytes=531840568 weight_movement_bytes=69206016 total_nvlink_bytes=601046584 weight_to_token_ratio=0.130125 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260406802e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=344728 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266412696 ingress_bytes=300670976 link_bytes=300670976 token_bytes=532480664 weight_movement_bytes=34603008 total_nvlink_bytes=567083672 weight_to_token_ratio=0.064985 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131400677e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267100160 dispatch_meta_bytes=343728 combine_payload_bytes=267100160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267443888 ingress_bytes=301703168 link_bytes=301703168 token_bytes=534544048 weight_movement_bytes=34603008 total_nvlink_bytes=569147056 weight_to_token_ratio=0.064734 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135284732e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265699328 dispatch_meta_bytes=343192 combine_payload_bytes=265699328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266042520 ingress_bytes=300302336 link_bytes=300302336 token_bytes=531741848 weight_movement_bytes=34603008 total_nvlink_bytes=566344856 weight_to_token_ratio=0.065075 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130013514e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=344728 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266412696 ingress_bytes=300670976 link_bytes=300670976 token_bytes=532480664 weight_movement_bytes=34603008 total_nvlink_bytes=567083672 weight_to_token_ratio=0.064985 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131400677e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=64618 dup_rows=24136 dup_rows_remote=21094 dedup_ratio=0.245524 router_tokens_per_expert=[779, 779, 800, 766, 773, 777, 854, 785, 769, 755, 771, 769, 744, 802, 763, 735, 766, 757, 807, 791, 820, 789, 780, 786, 761, 735, 813, 732, 769, 764, 772, 746, 735, 745, 729, 785, 786, 746, 748, 682, 798, 759, 788, 730, 742, 758, 706, 752, 749, 742, 801, 778, 728, 790, 763, 784, 799, 735, 793, 808, 757, 762, 817, 748] payload_rows_per_dst=[9414, 9166, 9550, 9254, 9152, 9094, 9202, 9336] dup_rows_per_dst=[3212, 3050, 3042, 2930, 2760, 2972, 3068, 3102] combine_rows_per_src=[9414, 9166, 9550, 9254, 9152, 9094, 9202, 9336] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74174 payload_rows_remote=64960 dup_rows=24130 dup_rows_remote=21090 dedup_ratio=0.245463 router_tokens_per_expert=[780, 754, 789, 777, 794, 759, 749, 759, 772, 757, 805, 750, 762, 739, 779, 759, 778, 741, 722, 784, 777, 787, 760, 743, 813, 746, 774, 813, 747, 817, 809, 765, 745, 772, 777, 766, 782, 785, 747, 744, 791, 787, 784, 761, 742, 778, 749, 735, 824, 742, 769, 749, 772, 831, 743, 798, 780, 708, 744, 792, 761, 754, 744, 736] payload_rows_per_dst=[9214, 9196, 9234, 9452, 9274, 9214, 9358, 9232] dup_rows_per_dst=[3108, 3050, 2950, 3116, 2962, 3040, 3098, 2806] combine_rows_per_src=[9214, 9196, 9234, 9452, 9274, 9214, 9358, 9232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74734 payload_rows_remote=65238 dup_rows=23570 dup_rows_remote=20552 dedup_ratio=0.239766 router_tokens_per_expert=[763, 749, 805, 712, 796, 728, 759, 795, 753, 771, 742, 733, 768, 727, 778, 793, 766, 716, 754, 775, 765, 761, 794, 742, 771, 769, 779, 769, 775, 749, 772, 773, 785, 811, 720, 794, 798, 773, 780, 796, 746, 786, 782, 800, 781, 761, 791, 807, 726, 787, 771, 790, 732, 776, 735, 743, 793, 791, 778, 803, 723, 777, 809, 705] payload_rows_per_dst=[9318, 9218, 9250, 9408, 9496, 9442, 9236, 9366] dup_rows_per_dst=[2896, 2912, 2896, 2906, 3018, 3066, 2884, 2992] combine_rows_per_src=[9318, 9218, 9250, 9408, 9496, 9442, 9236, 9366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=65138 dup_rows=23906 dup_rows_remote=20860 dedup_ratio=0.243184 router_tokens_per_expert=[746, 762, 781, 753, 743, 742, 774, 800, 769, 795, 785, 796, 762, 737, 747, 771, 813, 759, 764, 819, 765, 735, 755, 814, 768, 800, 757, 739, 791, 709, 823, 766, 687, 787, 764, 759, 793, 760, 784, 785, 788, 779, 789, 752, 756, 775, 748, 713, 730, 762, 773, 787, 737, 781, 788, 775, 739, 765, 767, 812, 788, 756, 762, 771] payload_rows_per_dst=[9252, 9350, 9420, 9260, 9222, 9288, 9262, 9344] dup_rows_per_dst=[2950, 2974, 3028, 3046, 3016, 2912, 3004, 2976] combine_rows_per_src=[9252, 9350, 9420, 9260, 9222, 9288, 9262, 9344] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74268 payload_rows_remote=64814 dup_rows=24036 dup_rows_remote=20888 dedup_ratio=0.244507 router_tokens_per_expert=[768, 735, 721, 732, 753, 772, 793, 767, 754, 788, 749, 773, 795, 748, 782, 817, 762, 763, 777, 755, 743, 817, 770, 768, 743, 789, 763, 790, 713, 769, 793, 716, 786, 784, 766, 735, 737, 762, 741, 778, 797, 723, 756, 778, 754, 786, 770, 772, 812, 737, 753, 762, 760, 771, 785, 768, 784, 798, 779, 752, 770, 764, 801, 853] payload_rows_per_dst=[8858, 9204, 8956, 9364, 9606, 9606, 9220, 9454] dup_rows_per_dst=[2642, 2950, 2636, 3046, 3444, 3094, 3076, 3148] combine_rows_per_src=[8858, 9204, 8956, 9364, 9606, 9606, 9220, 9454] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266330112 dispatch_meta_bytes=342736 combine_payload_bytes=266330112 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266672848 ingress_bytes=335536128 link_bytes=335536128 token_bytes=533002960 weight_movement_bytes=69206016 total_nvlink_bytes=602208976 weight_to_token_ratio=0.129842 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262595437e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267042816 dispatch_meta_bytes=343704 combine_payload_bytes=267042816 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267386520 ingress_bytes=336248832 link_bytes=336248832 token_bytes=534429336 weight_movement_bytes=69206016 total_nvlink_bytes=603635352 weight_to_token_ratio=0.129495 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265277285e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266878976 dispatch_meta_bytes=343920 combine_payload_bytes=266878976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267222896 ingress_bytes=301481984 link_bytes=301481984 token_bytes=534101872 weight_movement_bytes=34603008 total_nvlink_bytes=568704880 weight_to_token_ratio=0.064787 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134452435e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266346496 dispatch_meta_bytes=344160 combine_payload_bytes=266346496 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266690656 ingress_bytes=370155520 link_bytes=370155520 token_bytes=533037152 weight_movement_bytes=103809024 total_nvlink_bytes=636846176 weight_to_token_ratio=0.194750 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392865422e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74328 payload_rows_remote=65130 dup_rows=23976 dup_rows_remote=20898 dedup_ratio=0.243896 router_tokens_per_expert=[803, 732, 776, 778, 719, 776, 733, 769, 782, 774, 750, 785, 787, 761, 769, 749, 761, 751, 798, 741, 711, 749, 774, 783, 757, 727, 766, 749, 794, 795, 796, 767, 789, 761, 781, 728, 746, 764, 765, 804, 803, 781, 819, 775, 728, 760, 790, 749, 758, 783, 766, 780, 780, 817, 733, 695, 785, 766, 797, 782, 786, 789, 782, 748] payload_rows_per_dst=[9200, 9278, 9268, 9404, 9198, 9342, 9226, 9412] dup_rows_per_dst=[2972, 3036, 2868, 2898, 3078, 3068, 2998, 3058] combine_rows_per_src=[9200, 9278, 9268, 9404, 9198, 9342, 9226, 9412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74490 payload_rows_remote=65280 dup_rows=23814 dup_rows_remote=20950 dedup_ratio=0.242249 router_tokens_per_expert=[800, 757, 785, 764, 804, 766, 754, 769, 735, 769, 758, 762, 816, 763, 781, 777, 766, 727, 742, 720, 811, 777, 824, 830, 780, 741, 764, 738, 742, 788, 778, 753, 731, 754, 764, 772, 796, 786, 820, 777, 713, 743, 748, 816, 756, 801, 729, 781, 750, 742, 796, 776, 753, 778, 718, 724, 780, 769, 768, 775, 779, 764, 776, 776] payload_rows_per_dst=[9370, 9244, 9440, 9240, 9332, 9284, 9210, 9370] dup_rows_per_dst=[3028, 3078, 2954, 2928, 3068, 2890, 2864, 3004] combine_rows_per_src=[9370, 9244, 9440, 9240, 9332, 9284, 9210, 9370] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74570 payload_rows_remote=65208 dup_rows=23734 dup_rows_remote=20772 dedup_ratio=0.241435 router_tokens_per_expert=[777, 798, 812, 785, 735, 810, 778, 743, 746, 799, 804, 751, 722, 750, 816, 774, 745, 756, 743, 756, 740, 777, 786, 784, 742, 753, 768, 792, 773, 761, 788, 771, 778, 739, 729, 779, 806, 763, 727, 740, 760, 754, 777, 778, 737, 783, 708, 806, 809, 756, 779, 743, 759, 781, 807, 766, 812, 764, 794, 734, 796, 753, 743, 757] payload_rows_per_dst=[9440, 9362, 9228, 9388, 9296, 9158, 9388, 9310] dup_rows_per_dst=[3036, 2962, 2946, 2908, 2826, 3048, 3012, 2996] combine_rows_per_src=[9440, 9362, 9228, 9388, 9296, 9158, 9388, 9310] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74216 payload_rows_remote=64738 dup_rows=24088 dup_rows_remote=21032 dedup_ratio=0.245036 router_tokens_per_expert=[766, 788, 763, 781, 792, 815, 787, 766, 734, 786, 795, 793, 750, 788, 811, 757, 752, 782, 786, 788, 812, 778, 755, 751, 732, 742, 781, 782, 741, 775, 769, 768, 767, 786, 767, 796, 746, 806, 801, 699, 780, 771, 734, 791, 739, 768, 757, 762, 765, 693, 766, 773, 785, 833, 760, 730, 725, 771, 771, 744, 771, 764, 706, 759] payload_rows_per_dst=[9478, 9290, 9348, 9204, 9252, 9240, 9310, 9094] dup_rows_per_dst=[3056, 3126, 3060, 2976, 3084, 2964, 3116, 2706] combine_rows_per_src=[9478, 9290, 9348, 9204, 9252, 9240, 9310, 9094] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=264675328 dispatch_meta_bytes=342848 combine_payload_bytes=264675328 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265018176 ingress_bytes=333881344 link_bytes=333881344 token_bytes=529693504 weight_movement_bytes=69206016 total_nvlink_bytes=598899520 weight_to_token_ratio=0.130653 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256368618e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74490 payload_rows_remote=65280 dup_rows=23814 dup_rows_remote=20950 dedup_ratio=0.242249 router_tokens_per_expert=[800, 757, 785, 764, 804, 766, 754, 769, 735, 769, 758, 762, 816, 763, 781, 777, 766, 727, 742, 720, 811, 777, 824, 830, 780, 741, 764, 738, 742, 788, 778, 753, 731, 754, 764, 772, 796, 786, 820, 777, 713, 743, 748, 816, 756, 801, 729, 781, 750, 742, 796, 776, 753, 778, 718, 724, 780, 769, 768, 775, 779, 764, 776, 776] payload_rows_per_dst=[9370, 9244, 9440, 9240, 9332, 9284, 9210, 9370] dup_rows_per_dst=[3028, 3078, 2954, 2928, 3068, 2890, 2864, 3004] combine_rows_per_src=[9370, 9244, 9440, 9240, 9332, 9284, 9210, 9370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266076160 dispatch_meta_bytes=344200 combine_payload_bytes=266076160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266420360 ingress_bytes=300679168 link_bytes=300679168 token_bytes=532496520 weight_movement_bytes=34603008 total_nvlink_bytes=567099528 weight_to_token_ratio=0.064983 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131431503e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266805248 dispatch_meta_bytes=343992 combine_payload_bytes=266805248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267149240 ingress_bytes=301408256 link_bytes=301408256 token_bytes=533954488 weight_movement_bytes=34603008 total_nvlink_bytes=568557496 weight_to_token_ratio=0.064805 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134175002e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267214848 dispatch_meta_bytes=343160 combine_payload_bytes=267214848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267558008 ingress_bytes=301817856 link_bytes=301817856 token_bytes=534772856 weight_movement_bytes=34603008 total_nvlink_bytes=569375864 weight_to_token_ratio=0.064706 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135716294e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74268 payload_rows_remote=64814 dup_rows=24036 dup_rows_remote=20888 dedup_ratio=0.244507 router_tokens_per_expert=[768, 735, 721, 732, 753, 772, 793, 767, 754, 788, 749, 773, 795, 748, 782, 817, 762, 763, 777, 755, 743, 817, 770, 768, 743, 789, 763, 790, 713, 769, 793, 716, 786, 784, 766, 735, 737, 762, 741, 778, 797, 723, 756, 778, 754, 786, 770, 772, 812, 737, 753, 762, 760, 771, 785, 768, 784, 798, 779, 752, 770, 764, 801, 853] payload_rows_per_dst=[8858, 9204, 8956, 9364, 9606, 9606, 9220, 9454] dup_rows_per_dst=[2642, 2950, 2636, 3046, 3444, 3094, 3076, 3148] combine_rows_per_src=[8858, 9204, 8956, 9364, 9606, 9606, 9220, 9454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=65138 dup_rows=23906 dup_rows_remote=20860 dedup_ratio=0.243184 router_tokens_per_expert=[746, 762, 781, 753, 743, 742, 774, 800, 769, 795, 785, 796, 762, 737, 747, 771, 813, 759, 764, 819, 765, 735, 755, 814, 768, 800, 757, 739, 791, 709, 823, 766, 687, 787, 764, 759, 793, 760, 784, 785, 788, 779, 789, 752, 756, 775, 748, 713, 730, 762, 773, 787, 737, 781, 788, 775, 739, 765, 767, 812, 788, 756, 762, 771] payload_rows_per_dst=[9252, 9350, 9420, 9260, 9222, 9288, 9262, 9344] dup_rows_per_dst=[2950, 2974, 3028, 3046, 3016, 2912, 3004, 2976] combine_rows_per_src=[9252, 9350, 9420, 9260, 9222, 9288, 9262, 9344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74174 payload_rows_remote=64960 dup_rows=24130 dup_rows_remote=21090 dedup_ratio=0.245463 router_tokens_per_expert=[780, 754, 789, 777, 794, 759, 749, 759, 772, 757, 805, 750, 762, 739, 779, 759, 778, 741, 722, 784, 777, 787, 760, 743, 813, 746, 774, 813, 747, 817, 809, 765, 745, 772, 777, 766, 782, 785, 747, 744, 791, 787, 784, 761, 742, 778, 749, 735, 824, 742, 769, 749, 772, 831, 743, 798, 780, 708, 744, 792, 761, 754, 744, 736] payload_rows_per_dst=[9214, 9196, 9234, 9452, 9274, 9214, 9358, 9232] dup_rows_per_dst=[3108, 3050, 2950, 3116, 2962, 3040, 3098, 2806] combine_rows_per_src=[9214, 9196, 9234, 9452, 9274, 9214, 9358, 9232] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265478144 dispatch_meta_bytes=342808 combine_payload_bytes=265478144 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265820952 ingress_bytes=334684160 link_bytes=334684160 token_bytes=531299096 weight_movement_bytes=69206016 total_nvlink_bytes=600505112 weight_to_token_ratio=0.130258 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259389550e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267091968 dispatch_meta_bytes=343920 combine_payload_bytes=267091968 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267435888 ingress_bytes=370900992 link_bytes=370900992 token_bytes=534527856 weight_movement_bytes=103809024 total_nvlink_bytes=638336880 weight_to_token_ratio=0.194207 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395670573e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265166848 dispatch_meta_bytes=343080 combine_payload_bytes=265166848 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265509928 ingress_bytes=368975872 link_bytes=368975872 token_bytes=530676776 weight_movement_bytes=103809024 total_nvlink_bytes=634485800 weight_to_token_ratio=0.195616 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388426501e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=64618 dup_rows=24136 dup_rows_remote=21094 dedup_ratio=0.245524 router_tokens_per_expert=[779, 779, 800, 766, 773, 777, 854, 785, 769, 755, 771, 769, 744, 802, 763, 735, 766, 757, 807, 791, 820, 789, 780, 786, 761, 735, 813, 732, 769, 764, 772, 746, 735, 745, 729, 785, 786, 746, 748, 682, 798, 759, 788, 730, 742, 758, 706, 752, 749, 742, 801, 778, 728, 790, 763, 784, 799, 735, 793, 808, 757, 762, 817, 748] payload_rows_per_dst=[9414, 9166, 9550, 9254, 9152, 9094, 9202, 9336] dup_rows_per_dst=[3212, 3050, 3042, 2930, 2760, 2972, 3068, 3102] combine_rows_per_src=[9414, 9166, 9550, 9254, 9152, 9094, 9202, 9336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266772480 dispatch_meta_bytes=344112 combine_payload_bytes=266772480 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267116592 ingress_bytes=370581504 link_bytes=370581504 token_bytes=533889072 weight_movement_bytes=103809024 total_nvlink_bytes=637698096 weight_to_token_ratio=0.194439 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394468365e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267386880 dispatch_meta_bytes=344920 combine_payload_bytes=267386880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267731800 ingress_bytes=301989888 link_bytes=301989888 token_bytes=535118680 weight_movement_bytes=34603008 total_nvlink_bytes=569721688 weight_to_token_ratio=0.064664 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136363636e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74174 payload_rows_remote=64826 dup_rows=24130 dup_rows_remote=21138 dedup_ratio=0.245463 router_tokens_per_expert=[762, 817, 761, 718, 774, 781, 767, 769, 731, 769, 773, 748, 797, 740, 744, 776, 777, 750, 811, 780, 739, 768, 785, 760, 733, 775, 783, 743, 808, 781, 749, 735, 789, 681, 782, 747, 744, 768, 756, 796, 742, 736, 766, 797, 787, 749, 791, 736, 796, 773, 783, 748, 778, 834, 778, 773, 764, 831, 751, 755, 780, 792, 780, 765] payload_rows_per_dst=[9212, 9216, 9348, 9174, 9174, 9318, 9458, 9274] dup_rows_per_dst=[3086, 2940, 2992, 3040, 2952, 2890, 3068, 3162] combine_rows_per_src=[9212, 9216, 9348, 9174, 9174, 9318, 9458, 9274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265166848 dispatch_meta_bytes=343080 combine_payload_bytes=265166848 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265509928 ingress_bytes=368975872 link_bytes=368975872 token_bytes=530676776 weight_movement_bytes=103809024 total_nvlink_bytes=634485800 weight_to_token_ratio=0.195616 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388426501e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267091968 dispatch_meta_bytes=343920 combine_payload_bytes=267091968 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267435888 ingress_bytes=370900992 link_bytes=370900992 token_bytes=534527856 weight_movement_bytes=103809024 total_nvlink_bytes=638336880 weight_to_token_ratio=0.194207 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395670573e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265478144 dispatch_meta_bytes=342808 combine_payload_bytes=265478144 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265820952 ingress_bytes=334684160 link_bytes=334684160 token_bytes=531299096 weight_movement_bytes=69206016 total_nvlink_bytes=600505112 weight_to_token_ratio=0.130258 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259389550e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266805248 dispatch_meta_bytes=343992 combine_payload_bytes=266805248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267149240 ingress_bytes=301408256 link_bytes=301408256 token_bytes=533954488 weight_movement_bytes=34603008 total_nvlink_bytes=568557496 weight_to_token_ratio=0.064805 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134175002e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266076160 dispatch_meta_bytes=344200 combine_payload_bytes=266076160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266420360 ingress_bytes=300679168 link_bytes=300679168 token_bytes=532496520 weight_movement_bytes=34603008 total_nvlink_bytes=567099528 weight_to_token_ratio=0.064983 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131431503e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=264675328 dispatch_meta_bytes=342848 combine_payload_bytes=264675328 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265018176 ingress_bytes=333881344 link_bytes=333881344 token_bytes=529693504 weight_movement_bytes=69206016 total_nvlink_bytes=598899520 weight_to_token_ratio=0.130653 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256368618e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74734 payload_rows_remote=65238 dup_rows=23570 dup_rows_remote=20552 dedup_ratio=0.239766 router_tokens_per_expert=[763, 749, 805, 712, 796, 728, 759, 795, 753, 771, 742, 733, 768, 727, 778, 793, 766, 716, 754, 775, 765, 761, 794, 742, 771, 769, 779, 769, 775, 749, 772, 773, 785, 811, 720, 794, 798, 773, 780, 796, 746, 786, 782, 800, 781, 761, 791, 807, 726, 787, 771, 790, 732, 776, 735, 743, 793, 791, 778, 803, 723, 777, 809, 705] payload_rows_per_dst=[9318, 9218, 9250, 9408, 9496, 9442, 9236, 9366] dup_rows_per_dst=[2896, 2912, 2896, 2906, 3018, 3066, 2884, 2992] combine_rows_per_src=[9318, 9218, 9250, 9408, 9496, 9442, 9236, 9366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74084 payload_rows_remote=64784 dup_rows=24220 dup_rows_remote=21252 dedup_ratio=0.246379 router_tokens_per_expert=[796, 753, 779, 736, 777, 733, 795, 765, 748, 733, 762, 734, 772, 780, 738, 771, 767, 778, 782, 801, 807, 798, 786, 729, 731, 744, 779, 721, 820, 723, 749, 787, 768, 741, 775, 781, 744, 774, 820, 764, 772, 765, 756, 807, 778, 802, 764, 746, 756, 779, 748, 801, 810, 793, 767, 789, 755, 754, 776, 810, 724, 757, 737, 765] payload_rows_per_dst=[9300, 9130, 9370, 9454, 9274, 9306, 9124, 9126] dup_rows_per_dst=[2968, 2998, 3126, 3098, 3060, 3074, 2866, 3030] combine_rows_per_src=[9300, 9130, 9370, 9454, 9274, 9306, 9124, 9126] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74142 payload_rows_remote=64990 dup_rows=24162 dup_rows_remote=21244 dedup_ratio=0.245789 router_tokens_per_expert=[725, 783, 801, 808, 784, 772, 760, 782, 806, 772, 722, 789, 791, 797, 787, 783, 775, 814, 767, 792, 807, 825, 794, 806, 792, 738, 766, 714, 747, 751, 737, 778, 789, 732, 737, 767, 739, 711, 729, 744, 774, 775, 740, 738, 739, 774, 757, 786, 690, 791, 777, 785, 776, 763, 705, 748, 770, 772, 811, 758, 757, 783, 755, 815] payload_rows_per_dst=[9308, 9394, 9514, 9092, 9036, 9218, 9152, 9428] dup_rows_per_dst=[3122, 3100, 3246, 2954, 2860, 2948, 2918, 3014] combine_rows_per_src=[9308, 9394, 9514, 9092, 9036, 9218, 9152, 9428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74430 payload_rows_remote=65004 dup_rows=23874 dup_rows_remote=20938 dedup_ratio=0.242859 router_tokens_per_expert=[802, 783, 814, 757, 761, 790, 761, 749, 738, 827, 777, 797, 728, 788, 752, 774, 767, 726, 711, 747, 778, 770, 776, 767, 812, 772, 799, 734, 739, 809, 767, 748, 802, 737, 770, 750, 764, 780, 779, 759, 737, 732, 795, 748, 770, 777, 764, 773, 775, 719, 813, 798, 808, 806, 774, 775, 752, 760, 757, 803, 734, 736, 773, 712] payload_rows_per_dst=[9362, 9426, 9174, 9308, 9328, 9294, 9422, 9116] dup_rows_per_dst=[3072, 2936, 2910, 3052, 2954, 2898, 3114, 2938] combine_rows_per_src=[9362, 9426, 9174, 9308, 9328, 9294, 9422, 9116] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=64954 dup_rows=23966 dup_rows_remote=20810 dedup_ratio=0.243795 router_tokens_per_expert=[759, 794, 770, 765, 762, 780, 735, 811, 788, 749, 776, 740, 722, 778, 756, 857, 739, 774, 795, 778, 770, 747, 764, 750, 759, 769, 811, 776, 747, 724, 722, 805, 735, 761, 740, 808, 756, 820, 803, 815, 753, 713, 747, 733, 705, 789, 718, 777, 815, 780, 836, 772, 726, 769, 779, 773, 747, 806, 748, 794, 805, 736, 743, 778] payload_rows_per_dst=[9228, 9120, 9338, 9292, 9456, 9224, 9296, 9384] dup_rows_per_dst=[2894, 2818, 3056, 2934, 3230, 2880, 2998, 3156] combine_rows_per_src=[9228, 9120, 9338, 9292, 9456, 9224, 9296, 9384] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65136 dup_rows=23924 dup_rows_remote=20866 dedup_ratio=0.243368 router_tokens_per_expert=[784, 750, 753, 733, 803, 825, 760, 811, 782, 762, 768, 765, 810, 776, 736, 813, 803, 757, 763, 828, 778, 725, 751, 733, 769, 775, 775, 729, 776, 767, 746, 782, 743, 725, 712, 780, 729, 805, 719, 827, 804, 759, 729, 767, 836, 746, 763, 747, 758, 723, 716, 777, 799, 785, 736, 782, 795, 743, 744, 777, 780, 779, 792, 787] payload_rows_per_dst=[9414, 9382, 9298, 9200, 9164, 9244, 9236, 9442] dup_rows_per_dst=[3024, 3042, 2978, 3038, 2916, 3058, 2916, 2952] combine_rows_per_src=[9414, 9382, 9298, 9200, 9164, 9244, 9236, 9442] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74326 payload_rows_remote=64982 dup_rows=23978 dup_rows_remote=20986 dedup_ratio=0.243917 router_tokens_per_expert=[726, 728, 779, 795, 701, 774, 798, 776, 745, 826, 792, 828, 763, 792, 757, 762, 803, 735, 790, 747, 754, 749, 729, 724, 756, 730, 793, 799, 790, 756, 761, 783, 792, 770, 763, 773, 766, 752, 757, 762, 801, 740, 756, 833, 784, 760, 751, 780, 761, 764, 758, 787, 776, 799, 766, 754, 767, 763, 770, 769, 751, 749, 790, 747] payload_rows_per_dst=[9254, 9430, 9124, 9344, 9276, 9416, 9296, 9186] dup_rows_per_dst=[2900, 3100, 2938, 2992, 2994, 2994, 3034, 3026] combine_rows_per_src=[9254, 9430, 9124, 9344, 9276, 9416, 9296, 9186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267214848 dispatch_meta_bytes=343160 combine_payload_bytes=267214848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267558008 ingress_bytes=301817856 link_bytes=301817856 token_bytes=534772856 weight_movement_bytes=34603008 total_nvlink_bytes=569375864 weight_to_token_ratio=0.064706 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135716294e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74174 payload_rows_remote=64826 dup_rows=24130 dup_rows_remote=21138 dedup_ratio=0.245463 router_tokens_per_expert=[762, 817, 761, 718, 774, 781, 767, 769, 731, 769, 773, 748, 797, 740, 744, 776, 777, 750, 811, 780, 739, 768, 785, 760, 733, 775, 783, 743, 808, 781, 749, 735, 789, 681, 782, 747, 744, 768, 756, 796, 742, 736, 766, 797, 787, 749, 791, 736, 796, 773, 783, 748, 778, 834, 778, 773, 764, 831, 751, 755, 780, 792, 780, 765] payload_rows_per_dst=[9212, 9216, 9348, 9174, 9174, 9318, 9458, 9274] dup_rows_per_dst=[3086, 2940, 2992, 3040, 2952, 2890, 3068, 3162] combine_rows_per_src=[9212, 9216, 9348, 9174, 9174, 9318, 9458, 9274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265355264 dispatch_meta_bytes=344144 combine_payload_bytes=265355264 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265699408 ingress_bytes=334561280 link_bytes=334561280 token_bytes=531054672 weight_movement_bytes=69206016 total_nvlink_bytes=600260688 weight_to_token_ratio=0.130318 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258927162e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74240 payload_rows_remote=65018 dup_rows=24064 dup_rows_remote=21072 dedup_ratio=0.244792 router_tokens_per_expert=[749, 797, 725, 761, 748, 785, 770, 818, 772, 755, 796, 775, 767, 786, 736, 759, 783, 765, 753, 777, 732, 784, 822, 760, 792, 780, 732, 820, 749, 697, 794, 754, 781, 750, 782, 777, 752, 763, 726, 776, 785, 771, 756, 763, 753, 784, 780, 821, 756, 741, 753, 764, 757, 798, 742, 786, 754, 751, 753, 745, 800, 790, 797, 752] payload_rows_per_dst=[9354, 9264, 9234, 9330, 9222, 9368, 9168, 9300] dup_rows_per_dst=[2952, 3028, 3118, 2906, 2992, 3058, 3026, 2984] combine_rows_per_src=[9354, 9264, 9234, 9330, 9222, 9368, 9168, 9300] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=64954 dup_rows=23966 dup_rows_remote=20810 dedup_ratio=0.243795 router_tokens_per_expert=[759, 794, 770, 765, 762, 780, 735, 811, 788, 749, 776, 740, 722, 778, 756, 857, 739, 774, 795, 778, 770, 747, 764, 750, 759, 769, 811, 776, 747, 724, 722, 805, 735, 761, 740, 808, 756, 820, 803, 815, 753, 713, 747, 733, 705, 789, 718, 777, 815, 780, 836, 772, 726, 769, 779, 773, 747, 806, 748, 794, 805, 736, 743, 778] payload_rows_per_dst=[9228, 9120, 9338, 9292, 9456, 9224, 9296, 9384] dup_rows_per_dst=[2894, 2818, 3056, 2934, 3230, 2880, 2998, 3156] combine_rows_per_src=[9228, 9120, 9338, 9292, 9456, 9224, 9296, 9384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265527296 dispatch_meta_bytes=343856 combine_payload_bytes=265527296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265871152 ingress_bytes=300130304 link_bytes=300130304 token_bytes=531398448 weight_movement_bytes=34603008 total_nvlink_bytes=566001456 weight_to_token_ratio=0.065117 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129366171e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=344936 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266543976 ingress_bytes=300802048 link_bytes=300802048 token_bytes=532743016 weight_movement_bytes=34603008 total_nvlink_bytes=567346024 weight_to_token_ratio=0.064953 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131893890e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266256384 dispatch_meta_bytes=343768 combine_payload_bytes=266256384 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266600152 ingress_bytes=370065408 link_bytes=370065408 token_bytes=532856536 weight_movement_bytes=103809024 total_nvlink_bytes=636665560 weight_to_token_ratio=0.194816 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392526338e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266051584 dispatch_meta_bytes=343056 combine_payload_bytes=266051584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266394640 ingress_bytes=300654592 link_bytes=300654592 token_bytes=532446224 weight_movement_bytes=34603008 total_nvlink_bytes=567049232 weight_to_token_ratio=0.064989 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131339025e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266797056 dispatch_meta_bytes=344008 combine_payload_bytes=266797056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267141064 ingress_bytes=301400064 link_bytes=301400064 token_bytes=533938120 weight_movement_bytes=34603008 total_nvlink_bytes=568541128 weight_to_token_ratio=0.064807 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134144176e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74430 payload_rows_remote=65004 dup_rows=23874 dup_rows_remote=20938 dedup_ratio=0.242859 router_tokens_per_expert=[802, 783, 814, 757, 761, 790, 761, 749, 738, 827, 777, 797, 728, 788, 752, 774, 767, 726, 711, 747, 778, 770, 776, 767, 812, 772, 799, 734, 739, 809, 767, 748, 802, 737, 770, 750, 764, 780, 779, 759, 737, 732, 795, 748, 770, 777, 764, 773, 775, 719, 813, 798, 808, 806, 774, 775, 752, 760, 757, 803, 734, 736, 773, 712] payload_rows_per_dst=[9362, 9426, 9174, 9308, 9328, 9294, 9422, 9116] dup_rows_per_dst=[3072, 2936, 2910, 3052, 2954, 2898, 3114, 2938] combine_rows_per_src=[9362, 9426, 9174, 9308, 9328, 9294, 9422, 9116] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74084 payload_rows_remote=64784 dup_rows=24220 dup_rows_remote=21252 dedup_ratio=0.246379 router_tokens_per_expert=[796, 753, 779, 736, 777, 733, 795, 765, 748, 733, 762, 734, 772, 780, 738, 771, 767, 778, 782, 801, 807, 798, 786, 729, 731, 744, 779, 721, 820, 723, 749, 787, 768, 741, 775, 781, 744, 774, 820, 764, 772, 765, 756, 807, 778, 802, 764, 746, 756, 779, 748, 801, 810, 793, 767, 789, 755, 754, 776, 810, 724, 757, 737, 765] payload_rows_per_dst=[9300, 9130, 9370, 9454, 9274, 9306, 9124, 9126] dup_rows_per_dst=[2968, 2998, 3126, 3098, 3060, 3074, 2866, 3030] combine_rows_per_src=[9300, 9130, 9370, 9454, 9274, 9306, 9124, 9126] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266166272 dispatch_meta_bytes=343872 combine_payload_bytes=266166272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266510144 ingress_bytes=300769280 link_bytes=300769280 token_bytes=532676416 weight_movement_bytes=34603008 total_nvlink_bytes=567279424 weight_to_token_ratio=0.064961 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131770587e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74142 payload_rows_remote=64990 dup_rows=24162 dup_rows_remote=21244 dedup_ratio=0.245789 router_tokens_per_expert=[725, 783, 801, 808, 784, 772, 760, 782, 806, 772, 722, 789, 791, 797, 787, 783, 775, 814, 767, 792, 807, 825, 794, 806, 792, 738, 766, 714, 747, 751, 737, 778, 789, 732, 737, 767, 739, 711, 729, 744, 774, 775, 740, 738, 739, 774, 757, 786, 690, 791, 777, 785, 776, 763, 705, 748, 770, 772, 811, 758, 757, 783, 755, 815] payload_rows_per_dst=[9308, 9394, 9514, 9092, 9036, 9218, 9152, 9428] dup_rows_per_dst=[3122, 3100, 3246, 2954, 2860, 2948, 2918, 3014] combine_rows_per_src=[9308, 9394, 9514, 9092, 9036, 9218, 9152, 9428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265527296 dispatch_meta_bytes=343856 combine_payload_bytes=265527296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265871152 ingress_bytes=300130304 link_bytes=300130304 token_bytes=531398448 weight_movement_bytes=34603008 total_nvlink_bytes=566001456 weight_to_token_ratio=0.065117 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129366171e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74240 payload_rows_remote=65018 dup_rows=24064 dup_rows_remote=21072 dedup_ratio=0.244792 router_tokens_per_expert=[749, 797, 725, 761, 748, 785, 770, 818, 772, 755, 796, 775, 767, 786, 736, 759, 783, 765, 753, 777, 732, 784, 822, 760, 792, 780, 732, 820, 749, 697, 794, 754, 781, 750, 782, 777, 752, 763, 726, 776, 785, 771, 756, 763, 753, 784, 780, 821, 756, 741, 753, 764, 757, 798, 742, 786, 754, 751, 753, 745, 800, 790, 797, 752] payload_rows_per_dst=[9354, 9264, 9234, 9330, 9222, 9368, 9168, 9300] dup_rows_per_dst=[2952, 3028, 3118, 2906, 2992, 3058, 3026, 2984] combine_rows_per_src=[9354, 9264, 9234, 9330, 9222, 9368, 9168, 9300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74298 payload_rows_remote=64974 dup_rows=24006 dup_rows_remote=20812 dedup_ratio=0.244202 router_tokens_per_expert=[786, 727, 718, 771, 819, 784, 779, 766, 738, 785, 772, 777, 775, 807, 798, 782, 769, 778, 761, 742, 780, 717, 809, 785, 749, 788, 782, 745, 775, 772, 768, 760, 800, 845, 738, 812, 731, 799, 737, 768, 746, 767, 794, 773, 770, 814, 772, 740, 768, 755, 768, 754, 749, 742, 737, 731, 744, 743, 793, 706, 833, 703, 777, 779] payload_rows_per_dst=[9324, 9438, 9118, 9334, 9358, 9360, 9130, 9236] dup_rows_per_dst=[3194, 3034, 2946, 2944, 3098, 2992, 2878, 2920] combine_rows_per_src=[9324, 9438, 9118, 9334, 9358, 9360, 9130, 9236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74440 payload_rows_remote=65118 dup_rows=23864 dup_rows_remote=20904 dedup_ratio=0.242757 router_tokens_per_expert=[789, 770, 771, 756, 750, 756, 752, 798, 744, 784, 749, 830, 783, 790, 777, 780, 725, 811, 759, 805, 716, 810, 784, 740, 755, 802, 706, 723, 793, 765, 755, 772, 743, 746, 771, 795, 753, 798, 793, 757, 728, 779, 796, 759, 748, 787, 780, 764, 794, 763, 758, 773, 775, 782, 746, 750, 807, 757, 761, 738, 804, 819, 717, 711] payload_rows_per_dst=[9278, 9420, 9362, 9174, 9350, 9282, 9322, 9252] dup_rows_per_dst=[3006, 3054, 2938, 2968, 2962, 3000, 2960, 2976] combine_rows_per_src=[9278, 9420, 9362, 9174, 9350, 9282, 9322, 9252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=64864 dup_rows=24118 dup_rows_remote=21076 dedup_ratio=0.245341 router_tokens_per_expert=[790, 769, 827, 792, 777, 756, 762, 781, 822, 736, 765, 747, 807, 737, 772, 796, 734, 777, 787, 720, 765, 825, 760, 760, 760, 777, 774, 742, 745, 732, 723, 763, 763, 793, 798, 816, 755, 705, 752, 760, 793, 750, 749, 756, 802, 773, 748, 765, 782, 765, 763, 762, 776, 731, 749, 828, 773, 749, 789, 733, 806, 743, 813, 732] payload_rows_per_dst=[9480, 9322, 9278, 9120, 9288, 9252, 9236, 9210] dup_rows_per_dst=[3028, 3042, 2978, 2912, 2996, 3020, 3076, 3066] combine_rows_per_src=[9480, 9322, 9278, 9120, 9288, 9252, 9236, 9210] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74126 payload_rows_remote=64768 dup_rows=24178 dup_rows_remote=21008 dedup_ratio=0.245951 router_tokens_per_expert=[797, 720, 757, 708, 810, 713, 772, 771, 784, 794, 749, 761, 773, 793, 685, 799, 758, 766, 738, 734, 715, 777, 834, 787, 787, 746, 745, 777, 776, 805, 760, 749, 781, 710, 770, 822, 772, 736, 780, 752, 797, 784, 770, 829, 757, 798, 703, 815, 768, 761, 742, 779, 752, 789, 800, 804, 821, 731, 759, 759, 764, 822, 740, 745] payload_rows_per_dst=[8572, 8910, 9312, 9856, 9202, 9444, 9472, 9358] dup_rows_per_dst=[2438, 2608, 2968, 3520, 3044, 3118, 3312, 3170] combine_rows_per_src=[8572, 8910, 9312, 9856, 9202, 9444, 9472, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=344360 combine_payload_bytes=266313728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266658088 ingress_bytes=300916736 link_bytes=300916736 token_bytes=532971816 weight_movement_bytes=34603008 total_nvlink_bytes=567574824 weight_to_token_ratio=0.064925 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132325452e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266051584 dispatch_meta_bytes=343056 combine_payload_bytes=266051584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266394640 ingress_bytes=300654592 link_bytes=300654592 token_bytes=532446224 weight_movement_bytes=34603008 total_nvlink_bytes=567049232 weight_to_token_ratio=0.064989 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131339025e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266256384 dispatch_meta_bytes=343768 combine_payload_bytes=266256384 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266600152 ingress_bytes=370065408 link_bytes=370065408 token_bytes=532856536 weight_movement_bytes=103809024 total_nvlink_bytes=636665560 weight_to_token_ratio=0.194816 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392526338e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265355264 dispatch_meta_bytes=344144 combine_payload_bytes=265355264 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265699408 ingress_bytes=334561280 link_bytes=334561280 token_bytes=531054672 weight_movement_bytes=69206016 total_nvlink_bytes=600260688 weight_to_token_ratio=0.130318 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258927162e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=344936 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266543976 ingress_bytes=300802048 link_bytes=300802048 token_bytes=532743016 weight_movement_bytes=34603008 total_nvlink_bytes=567346024 weight_to_token_ratio=0.064953 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131893890e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=64968 dup_rows=23944 dup_rows_remote=20934 dedup_ratio=0.243571 router_tokens_per_expert=[773, 753, 756, 826, 800, 742, 801, 743, 741, 744, 770, 758, 769, 753, 826, 775, 757, 765, 767, 753, 860, 809, 741, 749, 732, 703, 731, 808, 761, 766, 765, 796, 763, 738, 768, 759, 762, 786, 787, 768, 776, 773, 763, 723, 774, 770, 762, 702, 788, 778, 763, 750, 786, 770, 734, 725, 776, 819, 738, 790, 785, 773, 793, 817] payload_rows_per_dst=[9430, 9240, 9392, 9204, 9252, 9124, 9216, 9502] dup_rows_per_dst=[2958, 3032, 3010, 2920, 3010, 2962, 2972, 3080] combine_rows_per_src=[9430, 9240, 9392, 9204, 9252, 9124, 9216, 9502] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=65146 dup_rows=23928 dup_rows_remote=21078 dedup_ratio=0.243408 router_tokens_per_expert=[783, 763, 765, 819, 752, 810, 808, 779, 750, 755, 815, 773, 784, 795, 763, 779, 723, 744, 798, 727, 747, 749, 762, 780, 770, 747, 775, 769, 761, 780, 770, 785, 741, 750, 773, 722, 762, 778, 775, 778, 755, 790, 714, 752, 745, 757, 781, 746, 833, 783, 785, 759, 780, 776, 781, 757, 790, 796, 743, 785, 747, 724, 737, 777] payload_rows_per_dst=[9402, 9378, 9180, 9392, 9188, 9230, 9340, 9266] dup_rows_per_dst=[3156, 3050, 2880, 2922, 2970, 2850, 3168, 2932] combine_rows_per_src=[9402, 9378, 9180, 9392, 9188, 9230, 9340, 9266] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74310 payload_rows_remote=65220 dup_rows=23994 dup_rows_remote=21126 dedup_ratio=0.244080 router_tokens_per_expert=[731, 749, 745, 746, 828, 793, 811, 783, 793, 815, 803, 723, 796, 799, 753, 780, 746, 796, 808, 802, 797, 773, 723, 806, 778, 741, 722, 745, 767, 692, 779, 755, 777, 807, 761, 773, 799, 772, 743, 724, 764, 791, 745, 780, 737, 794, 759, 755, 759, 749, 801, 770, 805, 750, 757, 782, 778, 752, 763, 737, 750, 755, 758, 727] payload_rows_per_dst=[9356, 9466, 9372, 9090, 9406, 9200, 9360, 9060] dup_rows_per_dst=[3016, 3058, 3130, 2868, 2906, 3050, 2986, 2980] combine_rows_per_src=[9356, 9466, 9372, 9090, 9406, 9200, 9360, 9060] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=343144 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266476648 ingress_bytes=335339520 link_bytes=335339520 token_bytes=532610152 weight_movement_bytes=69206016 total_nvlink_bytes=601816168 weight_to_token_ratio=0.129937 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261855617e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=64968 dup_rows=23944 dup_rows_remote=20934 dedup_ratio=0.243571 router_tokens_per_expert=[773, 753, 756, 826, 800, 742, 801, 743, 741, 744, 770, 758, 769, 753, 826, 775, 757, 765, 767, 753, 860, 809, 741, 749, 732, 703, 731, 808, 761, 766, 765, 796, 763, 738, 768, 759, 762, 786, 787, 768, 776, 773, 763, 723, 774, 770, 762, 702, 788, 778, 763, 750, 786, 770, 734, 725, 776, 819, 738, 790, 785, 773, 793, 817] payload_rows_per_dst=[9430, 9240, 9392, 9204, 9252, 9124, 9216, 9502] dup_rows_per_dst=[2958, 3032, 3010, 2920, 3010, 2962, 2972, 3080] combine_rows_per_src=[9430, 9240, 9392, 9204, 9252, 9124, 9216, 9502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266723328 dispatch_meta_bytes=344088 combine_payload_bytes=266723328 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=267067416 ingress_bytes=405135360 link_bytes=405135360 token_bytes=533790744 weight_movement_bytes=138412032 total_nvlink_bytes=672202776 weight_to_token_ratio=0.259300 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524491744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74280 payload_rows_remote=65020 dup_rows=24024 dup_rows_remote=21008 dedup_ratio=0.244385 router_tokens_per_expert=[791, 794, 761, 806, 803, 778, 759, 807, 749, 760, 756, 724, 757, 773, 781, 763, 742, 734, 772, 806, 785, 777, 812, 782, 776, 751, 751, 753, 789, 753, 784, 734, 780, 815, 721, 785, 750, 750, 764, 773, 777, 764, 738, 778, 779, 744, 780, 813, 755, 792, 723, 729, 792, 733, 773, 779, 777, 778, 779, 730, 740, 751, 777, 770] payload_rows_per_dst=[9436, 9282, 9440, 9180, 9260, 9300, 9202, 9180] dup_rows_per_dst=[3162, 2844, 2980, 3002, 3016, 3046, 2950, 3024] combine_rows_per_src=[9436, 9282, 9440, 9180, 9260, 9300, 9202, 9180] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74126 payload_rows_remote=64768 dup_rows=24178 dup_rows_remote=21008 dedup_ratio=0.245951 router_tokens_per_expert=[797, 720, 757, 708, 810, 713, 772, 771, 784, 794, 749, 761, 773, 793, 685, 799, 758, 766, 738, 734, 715, 777, 834, 787, 787, 746, 745, 777, 776, 805, 760, 749, 781, 710, 770, 822, 772, 736, 780, 752, 797, 784, 770, 829, 757, 798, 703, 815, 768, 761, 742, 779, 752, 789, 800, 804, 821, 731, 759, 759, 764, 822, 740, 745] payload_rows_per_dst=[8572, 8910, 9312, 9856, 9202, 9444, 9472, 9358] dup_rows_per_dst=[2438, 2608, 2968, 3520, 3044, 3118, 3312, 3170] combine_rows_per_src=[8572, 8910, 9312, 9856, 9202, 9444, 9472, 9358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=343608 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266452536 ingress_bytes=335314944 link_bytes=335314944 token_bytes=532561464 weight_movement_bytes=69206016 total_nvlink_bytes=601767480 weight_to_token_ratio=0.129949 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261763139e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265682944 dispatch_meta_bytes=343760 combine_payload_bytes=265682944 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=266026704 ingress_bytes=438697984 link_bytes=438697984 token_bytes=531709648 weight_movement_bytes=173015040 total_nvlink_bytes=704724688 weight_to_token_ratio=0.325394 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.650785196e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265289728 dispatch_meta_bytes=343104 combine_payload_bytes=265289728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265632832 ingress_bytes=299892736 link_bytes=299892736 token_bytes=530922560 weight_movement_bytes=34603008 total_nvlink_bytes=565525568 weight_to_token_ratio=0.065175 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128472222e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266838016 dispatch_meta_bytes=344896 combine_payload_bytes=266838016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267182912 ingress_bytes=301441024 link_bytes=301441024 token_bytes=534020928 weight_movement_bytes=34603008 total_nvlink_bytes=568623936 weight_to_token_ratio=0.064797 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134298305e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=64864 dup_rows=24118 dup_rows_remote=21076 dedup_ratio=0.245341 router_tokens_per_expert=[790, 769, 827, 792, 777, 756, 762, 781, 822, 736, 765, 747, 807, 737, 772, 796, 734, 777, 787, 720, 765, 825, 760, 760, 760, 777, 774, 742, 745, 732, 723, 763, 763, 793, 798, 816, 755, 705, 752, 760, 793, 750, 749, 756, 802, 773, 748, 765, 782, 765, 763, 762, 776, 731, 749, 828, 773, 749, 789, 733, 806, 743, 813, 732] payload_rows_per_dst=[9480, 9322, 9278, 9120, 9288, 9252, 9236, 9210] dup_rows_per_dst=[3028, 3042, 2978, 2912, 2996, 3020, 3076, 3066] combine_rows_per_src=[9480, 9322, 9278, 9120, 9288, 9252, 9236, 9210] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74298 payload_rows_remote=64974 dup_rows=24006 dup_rows_remote=20812 dedup_ratio=0.244202 router_tokens_per_expert=[786, 727, 718, 771, 819, 784, 779, 766, 738, 785, 772, 777, 775, 807, 798, 782, 769, 778, 761, 742, 780, 717, 809, 785, 749, 788, 782, 745, 775, 772, 768, 760, 800, 845, 738, 812, 731, 799, 737, 768, 746, 767, 794, 773, 770, 814, 772, 740, 768, 755, 768, 754, 749, 742, 737, 731, 744, 743, 793, 706, 833, 703, 777, 779] payload_rows_per_dst=[9324, 9438, 9118, 9334, 9358, 9360, 9130, 9236] dup_rows_per_dst=[3194, 3034, 2946, 2944, 3098, 2992, 2878, 2920] combine_rows_per_src=[9324, 9438, 9118, 9334, 9358, 9360, 9130, 9236] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74440 payload_rows_remote=65118 dup_rows=23864 dup_rows_remote=20904 dedup_ratio=0.242757 router_tokens_per_expert=[789, 770, 771, 756, 750, 756, 752, 798, 744, 784, 749, 830, 783, 790, 777, 780, 725, 811, 759, 805, 716, 810, 784, 740, 755, 802, 706, 723, 793, 765, 755, 772, 743, 746, 771, 795, 753, 798, 793, 757, 728, 779, 796, 759, 748, 787, 780, 764, 794, 763, 758, 773, 775, 782, 746, 750, 807, 757, 761, 738, 804, 819, 717, 711] payload_rows_per_dst=[9278, 9420, 9362, 9174, 9350, 9282, 9322, 9252] dup_rows_per_dst=[3006, 3054, 2938, 2968, 2962, 3000, 2960, 2976] combine_rows_per_src=[9278, 9420, 9362, 9174, 9350, 9282, 9322, 9252] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267141120 dispatch_meta_bytes=345384 combine_payload_bytes=267141120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267486504 ingress_bytes=301744128 link_bytes=301744128 token_bytes=534627624 weight_movement_bytes=34603008 total_nvlink_bytes=569230632 weight_to_token_ratio=0.064724 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135438861e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=343608 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266452536 ingress_bytes=335314944 link_bytes=335314944 token_bytes=532561464 weight_movement_bytes=69206016 total_nvlink_bytes=601767480 weight_to_token_ratio=0.129949 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261763139e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74280 payload_rows_remote=65020 dup_rows=24024 dup_rows_remote=21008 dedup_ratio=0.244385 router_tokens_per_expert=[791, 794, 761, 806, 803, 778, 759, 807, 749, 760, 756, 724, 757, 773, 781, 763, 742, 734, 772, 806, 785, 777, 812, 782, 776, 751, 751, 753, 789, 753, 784, 734, 780, 815, 721, 785, 750, 750, 764, 773, 777, 764, 738, 778, 779, 744, 780, 813, 755, 792, 723, 729, 792, 733, 773, 779, 777, 778, 779, 730, 740, 751, 777, 770] payload_rows_per_dst=[9436, 9282, 9440, 9180, 9260, 9300, 9202, 9180] dup_rows_per_dst=[3162, 2844, 2980, 3002, 3016, 3046, 2950, 3024] combine_rows_per_src=[9436, 9282, 9440, 9180, 9260, 9300, 9202, 9180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74222 payload_rows_remote=64784 dup_rows=24082 dup_rows_remote=20954 dedup_ratio=0.244975 router_tokens_per_expert=[796, 784, 761, 768, 798, 790, 742, 774, 780, 787, 764, 763, 771, 738, 796, 751, 775, 779, 777, 720, 791, 728, 747, 782, 734, 764, 763, 770, 744, 779, 736, 751, 757, 743, 770, 791, 739, 773, 793, 706, 772, 783, 753, 737, 770, 766, 766, 801, 725, 788, 800, 766, 752, 750, 763, 822, 813, 786, 788, 828, 786, 734, 746, 782] payload_rows_per_dst=[9438, 9502, 9148, 9394, 9544, 9328, 9074, 8794] dup_rows_per_dst=[3128, 3232, 2948, 3140, 3250, 2930, 2824, 2630] combine_rows_per_src=[9438, 9502, 9148, 9394, 9544, 9328, 9074, 8794] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=65176 dup_rows=23948 dup_rows_remote=21122 dedup_ratio=0.243612 router_tokens_per_expert=[796, 777, 747, 771, 765, 741, 781, 798, 792, 802, 807, 808, 808, 768, 734, 781, 792, 768, 755, 768, 791, 771, 730, 785, 741, 794, 773, 718, 753, 724, 784, 733, 793, 816, 756, 772, 740, 749, 719, 810, 762, 780, 790, 775, 798, 718, 760, 795, 721, 792, 735, 762, 768, 729, 768, 728, 757, 748, 787, 758, 750, 781, 789, 790] payload_rows_per_dst=[9342, 9414, 9382, 9086, 9316, 9354, 9180, 9282] dup_rows_per_dst=[3010, 3186, 2938, 2954, 2994, 3002, 2826, 3038] combine_rows_per_src=[9342, 9414, 9382, 9086, 9316, 9354, 9180, 9282] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266321920 dispatch_meta_bytes=344112 combine_payload_bytes=266321920 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266666032 ingress_bytes=300924928 link_bytes=300924928 token_bytes=532987952 weight_movement_bytes=34603008 total_nvlink_bytes=567590960 weight_to_token_ratio=0.064923 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132356278e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265289728 dispatch_meta_bytes=343104 combine_payload_bytes=265289728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265632832 ingress_bytes=299892736 link_bytes=299892736 token_bytes=530922560 weight_movement_bytes=34603008 total_nvlink_bytes=565525568 weight_to_token_ratio=0.065175 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128472222e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265682944 dispatch_meta_bytes=343760 combine_payload_bytes=265682944 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=266026704 ingress_bytes=438697984 link_bytes=438697984 token_bytes=531709648 weight_movement_bytes=173015040 total_nvlink_bytes=704724688 weight_to_token_ratio=0.325394 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.650785196e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=343144 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266476648 ingress_bytes=335339520 link_bytes=335339520 token_bytes=532610152 weight_movement_bytes=69206016 total_nvlink_bytes=601816168 weight_to_token_ratio=0.129937 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261855617e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266723328 dispatch_meta_bytes=344088 combine_payload_bytes=266723328 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=267067416 ingress_bytes=405135360 link_bytes=405135360 token_bytes=533790744 weight_movement_bytes=138412032 total_nvlink_bytes=672202776 weight_to_token_ratio=0.259300 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524491744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74092 payload_rows_remote=64734 dup_rows=24212 dup_rows_remote=21218 dedup_ratio=0.246297 router_tokens_per_expert=[737, 742, 802, 796, 796, 729, 789, 762, 738, 763, 761, 742, 812, 798, 753, 733, 834, 759, 794, 745, 771, 812, 730, 731, 788, 754, 818, 769, 759, 761, 702, 761, 755, 710, 755, 824, 787, 783, 718, 726, 729, 742, 752, 772, 743, 779, 786, 708, 776, 784, 735, 778, 823, 818, 789, 773, 788, 787, 780, 766, 777, 811, 827, 730] payload_rows_per_dst=[9296, 9248, 9358, 9184, 9180, 9108, 9396, 9322] dup_rows_per_dst=[3010, 2952, 2994, 3040, 2936, 2914, 3156, 3210] combine_rows_per_src=[9296, 9248, 9358, 9184, 9180, 9108, 9396, 9322] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=64850 dup_rows=24010 dup_rows_remote=21058 dedup_ratio=0.244242 router_tokens_per_expert=[796, 740, 761, 777, 754, 772, 776, 736, 786, 779, 775, 744, 767, 704, 801, 789, 776, 788, 761, 735, 721, 744, 739, 781, 789, 781, 779, 769, 721, 759, 753, 734, 775, 753, 806, 727, 793, 767, 763, 805, 778, 786, 785, 746, 782, 762, 718, 841, 767, 772, 771, 785, 782, 751, 770, 776, 767, 813, 780, 769, 782, 759, 746, 788] payload_rows_per_dst=[9252, 9240, 9182, 9220, 9266, 9444, 9324, 9366] dup_rows_per_dst=[2972, 3050, 2908, 2950, 3112, 2952, 3024, 3042] combine_rows_per_src=[9252, 9240, 9182, 9220, 9266, 9444, 9324, 9366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74262 payload_rows_remote=64882 dup_rows=24042 dup_rows_remote=21120 dedup_ratio=0.244568 router_tokens_per_expert=[736, 801, 746, 752, 757, 763, 783, 735, 765, 755, 736, 793, 797, 779, 779, 814, 782, 740, 715, 779, 762, 790, 751, 752, 803, 737, 762, 766, 754, 771, 775, 783, 753, 785, 752, 761, 742, 757, 771, 755, 790, 796, 739, 748, 817, 756, 760, 839, 729, 745, 764, 775, 798, 731, 773, 775, 756, 796, 762, 800, 779, 754, 771, 810] payload_rows_per_dst=[9142, 9322, 9278, 9380, 9222, 9364, 9176, 9378] dup_rows_per_dst=[3004, 3114, 2864, 2922, 2930, 3126, 3004, 3078] combine_rows_per_src=[9142, 9322, 9278, 9380, 9222, 9364, 9176, 9378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74346 payload_rows_remote=65090 dup_rows=23958 dup_rows_remote=20934 dedup_ratio=0.243713 router_tokens_per_expert=[771, 809, 783, 739, 711, 740, 760, 731, 756, 782, 758, 737, 781, 752, 748, 782, 817, 845, 784, 756, 747, 804, 766, 768, 779, 743, 768, 770, 745, 760, 762, 779, 763, 773, 769, 742, 774, 803, 764, 752, 778, 849, 774, 731, 736, 775, 805, 767, 714, 757, 832, 748, 740, 787, 740, 762, 796, 783, 763, 788, 788, 786, 707, 773] payload_rows_per_dst=[9242, 9222, 9494, 9266, 9256, 9318, 9254, 9294] dup_rows_per_dst=[2846, 2970, 3080, 2946, 3024, 3112, 2906, 3074] combine_rows_per_src=[9242, 9222, 9494, 9266, 9256, 9318, 9254, 9294] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265150464 dispatch_meta_bytes=343808 combine_payload_bytes=265150464 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265494272 ingress_bytes=334356480 link_bytes=334356480 token_bytes=530644736 weight_movement_bytes=69206016 total_nvlink_bytes=599850752 weight_to_token_ratio=0.130419 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258156516e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74480 payload_rows_remote=65264 dup_rows=23824 dup_rows_remote=20860 dedup_ratio=0.242350 router_tokens_per_expert=[762, 774, 805, 764, 821, 800, 762, 776, 765, 706, 770, 760, 747, 768, 795, 779, 782, 808, 789, 787, 754, 771, 726, 774, 792, 772, 719, 738, 751, 776, 789, 784, 771, 715, 784, 734, 793, 793, 758, 784, 756, 767, 765, 746, 806, 737, 731, 786, 776, 774, 762, 815, 768, 747, 774, 770, 758, 785, 806, 752, 701, 754, 747, 771] payload_rows_per_dst=[9192, 9308, 9322, 9282, 9304, 9356, 9500, 9216] dup_rows_per_dst=[3084, 2872, 3060, 2960, 2960, 2800, 3124, 2964] combine_rows_per_src=[9192, 9308, 9322, 9282, 9304, 9356, 9500, 9216] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265625600 dispatch_meta_bytes=343632 combine_payload_bytes=265625600 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265969232 ingress_bytes=334831616 link_bytes=334831616 token_bytes=531594832 weight_movement_bytes=69206016 total_nvlink_bytes=600800848 weight_to_token_ratio=0.130186 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259944415e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74154 payload_rows_remote=64954 dup_rows=24150 dup_rows_remote=21236 dedup_ratio=0.245667 router_tokens_per_expert=[780, 746, 805, 813, 750, 779, 722, 773, 722, 733, 815, 754, 733, 758, 790, 752, 737, 759, 751, 778, 738, 798, 807, 788, 756, 743, 760, 789, 808, 787, 762, 754, 762, 727, 723, 772, 733, 765, 779, 780, 747, 797, 808, 769, 769, 711, 796, 771, 777, 810, 794, 795, 741, 713, 750, 813, 767, 759, 750, 723, 828, 785, 814, 784] payload_rows_per_dst=[9256, 9200, 9244, 9302, 9160, 9280, 9324, 9388] dup_rows_per_dst=[3080, 2914, 3068, 3016, 2922, 3056, 3062, 3032] combine_rows_per_src=[9256, 9200, 9244, 9302, 9160, 9280, 9324, 9388] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74222 payload_rows_remote=64784 dup_rows=24082 dup_rows_remote=20954 dedup_ratio=0.244975 router_tokens_per_expert=[796, 784, 761, 768, 798, 790, 742, 774, 780, 787, 764, 763, 771, 738, 796, 751, 775, 779, 777, 720, 791, 728, 747, 782, 734, 764, 763, 770, 744, 779, 736, 751, 757, 743, 770, 791, 739, 773, 793, 706, 772, 783, 753, 737, 770, 766, 766, 801, 725, 788, 800, 766, 752, 750, 763, 822, 813, 786, 788, 828, 786, 734, 746, 782] payload_rows_per_dst=[9438, 9502, 9148, 9394, 9544, 9328, 9074, 8794] dup_rows_per_dst=[3128, 3232, 2948, 3140, 3250, 2930, 2824, 2630] combine_rows_per_src=[9438, 9502, 9148, 9394, 9544, 9328, 9074, 8794] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=64850 dup_rows=24010 dup_rows_remote=21058 dedup_ratio=0.244242 router_tokens_per_expert=[796, 740, 761, 777, 754, 772, 776, 736, 786, 779, 775, 744, 767, 704, 801, 789, 776, 788, 761, 735, 721, 744, 739, 781, 789, 781, 779, 769, 721, 759, 753, 734, 775, 753, 806, 727, 793, 767, 763, 805, 778, 786, 785, 746, 782, 762, 718, 841, 767, 772, 771, 785, 782, 751, 770, 776, 767, 813, 780, 769, 782, 759, 746, 788] payload_rows_per_dst=[9252, 9240, 9182, 9220, 9266, 9444, 9324, 9366] dup_rows_per_dst=[2972, 3050, 2908, 2950, 3112, 2952, 3024, 3042] combine_rows_per_src=[9252, 9240, 9182, 9220, 9266, 9444, 9324, 9366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265355264 dispatch_meta_bytes=342952 combine_payload_bytes=265355264 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265698216 ingress_bytes=334561280 link_bytes=334561280 token_bytes=531053480 weight_movement_bytes=69206016 total_nvlink_bytes=600259496 weight_to_token_ratio=0.130318 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258927162e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74262 payload_rows_remote=64882 dup_rows=24042 dup_rows_remote=21120 dedup_ratio=0.244568 router_tokens_per_expert=[736, 801, 746, 752, 757, 763, 783, 735, 765, 755, 736, 793, 797, 779, 779, 814, 782, 740, 715, 779, 762, 790, 751, 752, 803, 737, 762, 766, 754, 771, 775, 783, 753, 785, 752, 761, 742, 757, 771, 755, 790, 796, 739, 748, 817, 756, 760, 839, 729, 745, 764, 775, 798, 731, 773, 775, 756, 796, 762, 800, 779, 754, 771, 810] payload_rows_per_dst=[9142, 9322, 9278, 9380, 9222, 9364, 9176, 9378] dup_rows_per_dst=[3004, 3114, 2864, 2922, 2930, 3126, 3004, 3078] combine_rows_per_src=[9142, 9322, 9278, 9380, 9222, 9364, 9176, 9378] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=65176 dup_rows=23948 dup_rows_remote=21122 dedup_ratio=0.243612 router_tokens_per_expert=[796, 777, 747, 771, 765, 741, 781, 798, 792, 802, 807, 808, 808, 768, 734, 781, 792, 768, 755, 768, 791, 771, 730, 785, 741, 794, 773, 718, 753, 724, 784, 733, 793, 816, 756, 772, 740, 749, 719, 810, 762, 780, 790, 775, 798, 718, 760, 795, 721, 792, 735, 762, 768, 729, 768, 728, 757, 748, 787, 758, 750, 781, 789, 790] payload_rows_per_dst=[9342, 9414, 9382, 9086, 9316, 9354, 9180, 9282] dup_rows_per_dst=[3010, 3186, 2938, 2954, 2994, 3002, 2826, 3038] combine_rows_per_src=[9342, 9414, 9382, 9086, 9316, 9354, 9180, 9282] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=344008 combine_payload_bytes=265756672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266100680 ingress_bytes=300359680 link_bytes=300359680 token_bytes=531857352 weight_movement_bytes=34603008 total_nvlink_bytes=566460360 weight_to_token_ratio=0.065061 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130229295e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266608640 dispatch_meta_bytes=344096 combine_payload_bytes=266608640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266952736 ingress_bytes=301211648 link_bytes=301211648 token_bytes=533561376 weight_movement_bytes=34603008 total_nvlink_bytes=568164384 weight_to_token_ratio=0.064853 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133435182e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74092 payload_rows_remote=64734 dup_rows=24212 dup_rows_remote=21218 dedup_ratio=0.246297 router_tokens_per_expert=[737, 742, 802, 796, 796, 729, 789, 762, 738, 763, 761, 742, 812, 798, 753, 733, 834, 759, 794, 745, 771, 812, 730, 731, 788, 754, 818, 769, 759, 761, 702, 761, 755, 710, 755, 824, 787, 783, 718, 726, 729, 742, 752, 772, 743, 779, 786, 708, 776, 784, 735, 778, 823, 818, 789, 773, 788, 787, 780, 766, 777, 811, 827, 730] payload_rows_per_dst=[9296, 9248, 9358, 9184, 9180, 9108, 9396, 9322] dup_rows_per_dst=[3010, 2952, 2994, 3040, 2936, 2914, 3156, 3210] combine_rows_per_src=[9296, 9248, 9358, 9184, 9180, 9108, 9396, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266960896 dispatch_meta_bytes=345192 combine_payload_bytes=266960896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267306088 ingress_bytes=301563904 link_bytes=301563904 token_bytes=534266984 weight_movement_bytes=34603008 total_nvlink_bytes=568869992 weight_to_token_ratio=0.064767 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134760693e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=267321344 dispatch_meta_bytes=344496 combine_payload_bytes=267321344 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267665840 ingress_bytes=371130368 link_bytes=371130368 token_bytes=534987184 weight_movement_bytes=103809024 total_nvlink_bytes=638796208 weight_to_token_ratio=0.194040 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396533696e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266051584 dispatch_meta_bytes=344760 combine_payload_bytes=266051584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266396344 ingress_bytes=300654592 link_bytes=300654592 token_bytes=532447928 weight_movement_bytes=34603008 total_nvlink_bytes=567050936 weight_to_token_ratio=0.064989 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131339025e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265355264 dispatch_meta_bytes=342952 combine_payload_bytes=265355264 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265698216 ingress_bytes=334561280 link_bytes=334561280 token_bytes=531053480 weight_movement_bytes=69206016 total_nvlink_bytes=600259496 weight_to_token_ratio=0.130318 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258927162e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266960896 dispatch_meta_bytes=345192 combine_payload_bytes=266960896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267306088 ingress_bytes=301563904 link_bytes=301563904 token_bytes=534266984 weight_movement_bytes=34603008 total_nvlink_bytes=568869992 weight_to_token_ratio=0.064767 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134760693e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74504 payload_rows_remote=65174 dup_rows=23800 dup_rows_remote=20884 dedup_ratio=0.242106 router_tokens_per_expert=[788, 770, 748, 759, 783, 721, 769, 758, 740, 781, 776, 804, 779, 758, 774, 805, 798, 734, 760, 757, 738, 762, 756, 818, 741, 734, 757, 779, 796, 784, 780, 795, 770, 729, 706, 760, 794, 772, 782, 761, 783, 754, 732, 754, 759, 752, 808, 738, 825, 773, 787, 739, 777, 735, 780, 811, 777, 785, 707, 798, 796, 759, 768, 779] payload_rows_per_dst=[9200, 9412, 9330, 9422, 9206, 9200, 9396, 9338] dup_rows_per_dst=[2992, 3022, 2916, 2910, 2942, 2960, 3058, 3000] combine_rows_per_src=[9200, 9412, 9330, 9422, 9206, 9200, 9396, 9338] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=64832 dup_rows=24096 dup_rows_remote=20934 dedup_ratio=0.245117 router_tokens_per_expert=[759, 774, 790, 753, 720, 773, 730, 740, 743, 781, 768, 791, 750, 799, 774, 777, 759, 759, 770, 775, 746, 750, 792, 799, 740, 755, 722, 787, 800, 759, 820, 762, 718, 734, 763, 738, 796, 793, 768, 777, 771, 789, 764, 785, 810, 800, 763, 787, 798, 746, 741, 766, 760, 791, 786, 788, 770, 728, 770, 787, 752, 778, 737, 781] payload_rows_per_dst=[9182, 9334, 9384, 9270, 9226, 9376, 9280, 9156] dup_rows_per_dst=[2896, 3032, 2916, 3020, 2948, 3162, 3072, 3050] combine_rows_per_src=[9182, 9334, 9384, 9270, 9226, 9376, 9280, 9156] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266952704 dispatch_meta_bytes=344232 combine_payload_bytes=266952704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267296936 ingress_bytes=301555712 link_bytes=301555712 token_bytes=534249640 weight_movement_bytes=34603008 total_nvlink_bytes=568852648 weight_to_token_ratio=0.064769 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134729867e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74148 payload_rows_remote=64800 dup_rows=24156 dup_rows_remote=21254 dedup_ratio=0.245728 router_tokens_per_expert=[778, 778, 781, 773, 717, 753, 800, 777, 744, 733, 738, 745, 709, 785, 765, 773, 791, 796, 763, 752, 850, 761, 809, 774, 732, 777, 752, 748, 815, 751, 794, 756, 726, 777, 756, 832, 792, 795, 771, 747, 745, 753, 799, 823, 743, 768, 749, 735, 810, 776, 765, 779, 779, 831, 780, 717, 738, 785, 755, 747, 787, 747, 723, 752] payload_rows_per_dst=[9292, 9160, 9390, 9348, 9272, 9188, 9340, 9158] dup_rows_per_dst=[3022, 2824, 3202, 2902, 3120, 3042, 3134, 2910] combine_rows_per_src=[9292, 9160, 9390, 9348, 9272, 9188, 9340, 9158] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74334 payload_rows_remote=65094 dup_rows=23970 dup_rows_remote=20912 dedup_ratio=0.243835 router_tokens_per_expert=[734, 716, 793, 769, 737, 736, 776, 761, 784, 791, 785, 772, 766, 726, 712, 792, 726, 797, 789, 777, 720, 785, 793, 791, 730, 797, 767, 766, 794, 769, 779, 853, 789, 751, 735, 771, 756, 820, 762, 765, 708, 821, 797, 768, 744, 766, 771, 794, 810, 738, 772, 732, 821, 790, 776, 759, 741, 718, 795, 794, 767, 787, 739, 712] payload_rows_per_dst=[9198, 9328, 9298, 9430, 9240, 9264, 9350, 9226] dup_rows_per_dst=[2846, 2928, 3058, 3080, 3058, 3074, 3046, 2880] combine_rows_per_src=[9198, 9328, 9298, 9430, 9240, 9264, 9350, 9226] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265551872 dispatch_meta_bytes=343064 combine_payload_bytes=265551872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265894936 ingress_bytes=334757888 link_bytes=334757888 token_bytes=531446808 weight_movement_bytes=69206016 total_nvlink_bytes=600652824 weight_to_token_ratio=0.130222 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259666982e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265625600 dispatch_meta_bytes=343632 combine_payload_bytes=265625600 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265969232 ingress_bytes=334831616 link_bytes=334831616 token_bytes=531594832 weight_movement_bytes=69206016 total_nvlink_bytes=600800848 weight_to_token_ratio=0.130186 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259944415e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=344008 combine_payload_bytes=265756672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266100680 ingress_bytes=300359680 link_bytes=300359680 token_bytes=531857352 weight_movement_bytes=34603008 total_nvlink_bytes=566460360 weight_to_token_ratio=0.065061 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130229295e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265150464 dispatch_meta_bytes=343808 combine_payload_bytes=265150464 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265494272 ingress_bytes=334356480 link_bytes=334356480 token_bytes=530644736 weight_movement_bytes=69206016 total_nvlink_bytes=599850752 weight_to_token_ratio=0.130419 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258156516e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74346 payload_rows_remote=65090 dup_rows=23958 dup_rows_remote=20934 dedup_ratio=0.243713 router_tokens_per_expert=[771, 809, 783, 739, 711, 740, 760, 731, 756, 782, 758, 737, 781, 752, 748, 782, 817, 845, 784, 756, 747, 804, 766, 768, 779, 743, 768, 770, 745, 760, 762, 779, 763, 773, 769, 742, 774, 803, 764, 752, 778, 849, 774, 731, 736, 775, 805, 767, 714, 757, 832, 748, 740, 787, 740, 762, 796, 783, 763, 788, 788, 786, 707, 773] payload_rows_per_dst=[9242, 9222, 9494, 9266, 9256, 9318, 9254, 9294] dup_rows_per_dst=[2846, 2970, 3080, 2946, 3024, 3112, 2906, 3074] combine_rows_per_src=[9242, 9222, 9494, 9266, 9256, 9318, 9254, 9294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74234 payload_rows_remote=64790 dup_rows=24070 dup_rows_remote=20886 dedup_ratio=0.244853 router_tokens_per_expert=[788, 708, 782, 762, 735, 793, 826, 752, 772, 798, 759, 755, 784, 787, 766, 785, 780, 757, 744, 747, 727, 715, 740, 752, 733, 726, 728, 751, 810, 763, 723, 806, 772, 701, 782, 850, 741, 766, 755, 797, 774, 772, 743, 778, 748, 821, 806, 786, 765, 815, 774, 759, 820, 776, 810, 745, 741, 775, 761, 767, 741, 815, 744, 798] payload_rows_per_dst=[9444, 9168, 9266, 9384, 9140, 9336, 9312, 9184] dup_rows_per_dst=[3184, 2932, 3174, 2982, 2846, 2866, 2986, 3100] combine_rows_per_src=[9444, 9168, 9266, 9384, 9140, 9336, 9312, 9184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74528 payload_rows_remote=65142 dup_rows=23776 dup_rows_remote=20654 dedup_ratio=0.241862 router_tokens_per_expert=[760, 740, 775, 814, 753, 765, 778, 775, 749, 776, 793, 762, 801, 772, 789, 812, 766, 768, 745, 786, 734, 749, 780, 759, 773, 765, 810, 792, 777, 740, 727, 725, 775, 768, 780, 758, 760, 755, 768, 817, 763, 781, 775, 712, 794, 836, 727, 791, 770, 724, 750, 762, 804, 741, 773, 791, 737, 761, 790, 746, 743, 808, 733, 749] payload_rows_per_dst=[9320, 9386, 9220, 9262, 9416, 9444, 9220, 9260] dup_rows_per_dst=[3000, 3122, 2954, 2956, 2946, 2914, 3010, 2874] combine_rows_per_src=[9320, 9386, 9220, 9262, 9416, 9444, 9220, 9260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74176 payload_rows_remote=64918 dup_rows=24128 dup_rows_remote=21230 dedup_ratio=0.245443 router_tokens_per_expert=[781, 772, 766, 758, 798, 771, 766, 769, 795, 741, 759, 724, 764, 751, 801, 744, 815, 766, 761, 751, 781, 749, 761, 794, 751, 805, 753, 746, 796, 768, 769, 763, 760, 779, 755, 745, 755, 760, 782, 783, 759, 712, 797, 783, 803, 731, 720, 803, 732, 781, 765, 768, 755, 736, 774, 767, 769, 779, 790, 783, 812, 769, 781, 775] payload_rows_per_dst=[9350, 9132, 9366, 9332, 9216, 9152, 9258, 9370] dup_rows_per_dst=[3012, 3026, 2990, 2970, 3022, 3064, 2898, 3146] combine_rows_per_src=[9350, 9132, 9366, 9332, 9216, 9152, 9258, 9370] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74504 payload_rows_remote=64962 dup_rows=23800 dup_rows_remote=20442 dedup_ratio=0.242106 router_tokens_per_expert=[776, 785, 786, 743, 756, 778, 796, 795, 786, 793, 754, 739, 736, 810, 782, 783, 738, 717, 795, 766, 749, 708, 741, 747, 762, 788, 709, 764, 760, 733, 779, 801, 795, 783, 762, 777, 753, 759, 798, 802, 778, 728, 739, 795, 753, 783, 812, 749, 776, 764, 765, 763, 828, 766, 785, 739, 725, 770, 833, 776, 757, 796, 753, 735] payload_rows_per_dst=[9388, 9332, 9132, 9264, 9466, 9304, 9076, 9542] dup_rows_per_dst=[3042, 3034, 2790, 2928, 2992, 2970, 2686, 3358] combine_rows_per_src=[9388, 9332, 9132, 9264, 9466, 9304, 9076, 9542] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74528 payload_rows_remote=65142 dup_rows=23776 dup_rows_remote=20654 dedup_ratio=0.241862 router_tokens_per_expert=[760, 740, 775, 814, 753, 765, 778, 775, 749, 776, 793, 762, 801, 772, 789, 812, 766, 768, 745, 786, 734, 749, 780, 759, 773, 765, 810, 792, 777, 740, 727, 725, 775, 768, 780, 758, 760, 755, 768, 817, 763, 781, 775, 712, 794, 836, 727, 791, 770, 724, 750, 762, 804, 741, 773, 791, 737, 761, 790, 746, 743, 808, 733, 749] payload_rows_per_dst=[9320, 9386, 9220, 9262, 9416, 9444, 9220, 9260] dup_rows_per_dst=[3000, 3122, 2954, 2956, 2946, 2914, 3010, 2874] combine_rows_per_src=[9320, 9386, 9220, 9262, 9416, 9444, 9220, 9260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266608640 dispatch_meta_bytes=344096 combine_payload_bytes=266608640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266952736 ingress_bytes=301211648 link_bytes=301211648 token_bytes=533561376 weight_movement_bytes=34603008 total_nvlink_bytes=568164384 weight_to_token_ratio=0.064853 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133435182e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74176 payload_rows_remote=64918 dup_rows=24128 dup_rows_remote=21230 dedup_ratio=0.245443 router_tokens_per_expert=[781, 772, 766, 758, 798, 771, 766, 769, 795, 741, 759, 724, 764, 751, 801, 744, 815, 766, 761, 751, 781, 749, 761, 794, 751, 805, 753, 746, 796, 768, 769, 763, 760, 779, 755, 745, 755, 760, 782, 783, 759, 712, 797, 783, 803, 731, 720, 803, 732, 781, 765, 768, 755, 736, 774, 767, 769, 779, 790, 783, 812, 769, 781, 775] payload_rows_per_dst=[9350, 9132, 9366, 9332, 9216, 9152, 9258, 9370] dup_rows_per_dst=[3012, 3026, 2990, 2970, 3022, 3064, 2898, 3146] combine_rows_per_src=[9350, 9132, 9366, 9332, 9216, 9152, 9258, 9370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=64832 dup_rows=24096 dup_rows_remote=20934 dedup_ratio=0.245117 router_tokens_per_expert=[759, 774, 790, 753, 720, 773, 730, 740, 743, 781, 768, 791, 750, 799, 774, 777, 759, 759, 770, 775, 746, 750, 792, 799, 740, 755, 722, 787, 800, 759, 820, 762, 718, 734, 763, 738, 796, 793, 768, 777, 771, 789, 764, 785, 810, 800, 763, 787, 798, 746, 741, 766, 760, 791, 786, 788, 770, 728, 770, 787, 752, 778, 737, 781] payload_rows_per_dst=[9182, 9334, 9384, 9270, 9226, 9376, 9280, 9156] dup_rows_per_dst=[2896, 3032, 2916, 3020, 2948, 3162, 3072, 3050] combine_rows_per_src=[9182, 9334, 9384, 9270, 9226, 9376, 9280, 9156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74504 payload_rows_remote=65174 dup_rows=23800 dup_rows_remote=20884 dedup_ratio=0.242106 router_tokens_per_expert=[788, 770, 748, 759, 783, 721, 769, 758, 740, 781, 776, 804, 779, 758, 774, 805, 798, 734, 760, 757, 738, 762, 756, 818, 741, 734, 757, 779, 796, 784, 780, 795, 770, 729, 706, 760, 794, 772, 782, 761, 783, 754, 732, 754, 759, 752, 808, 738, 825, 773, 787, 739, 777, 735, 780, 811, 777, 785, 707, 798, 796, 759, 768, 779] payload_rows_per_dst=[9200, 9412, 9330, 9422, 9206, 9200, 9396, 9338] dup_rows_per_dst=[2992, 3022, 2916, 2910, 2942, 2960, 3058, 3000] combine_rows_per_src=[9200, 9412, 9330, 9422, 9206, 9200, 9396, 9338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74148 payload_rows_remote=64800 dup_rows=24156 dup_rows_remote=21254 dedup_ratio=0.245728 router_tokens_per_expert=[778, 778, 781, 773, 717, 753, 800, 777, 744, 733, 738, 745, 709, 785, 765, 773, 791, 796, 763, 752, 850, 761, 809, 774, 732, 777, 752, 748, 815, 751, 794, 756, 726, 777, 756, 832, 792, 795, 771, 747, 745, 753, 799, 823, 743, 768, 749, 735, 810, 776, 765, 779, 779, 831, 780, 717, 738, 785, 755, 747, 787, 747, 723, 752] payload_rows_per_dst=[9292, 9160, 9390, 9348, 9272, 9188, 9340, 9158] dup_rows_per_dst=[3022, 2824, 3202, 2902, 3120, 3042, 3134, 2910] combine_rows_per_src=[9292, 9160, 9390, 9348, 9272, 9188, 9340, 9158] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=344592 combine_payload_bytes=265904128 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266248720 ingress_bytes=335110144 link_bytes=335110144 token_bytes=532152848 weight_movement_bytes=69206016 total_nvlink_bytes=601358864 weight_to_token_ratio=0.130049 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260992493e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265379840 dispatch_meta_bytes=342704 combine_payload_bytes=265379840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265722544 ingress_bytes=299982848 link_bytes=299982848 token_bytes=531102384 weight_movement_bytes=34603008 total_nvlink_bytes=565705392 weight_to_token_ratio=0.065153 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128811306e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266084352 dispatch_meta_bytes=341616 combine_payload_bytes=266084352 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266425968 ingress_bytes=300687360 link_bytes=300687360 token_bytes=532510320 weight_movement_bytes=34603008 total_nvlink_bytes=567113328 weight_to_token_ratio=0.064981 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131462328e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266821632 dispatch_meta_bytes=343184 combine_payload_bytes=266821632 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267164816 ingress_bytes=336027648 link_bytes=336027648 token_bytes=533986448 weight_movement_bytes=69206016 total_nvlink_bytes=603192464 weight_to_token_ratio=0.129603 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264444987e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265551872 dispatch_meta_bytes=343064 combine_payload_bytes=265551872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265894936 ingress_bytes=334757888 link_bytes=334757888 token_bytes=531446808 weight_movement_bytes=69206016 total_nvlink_bytes=600652824 weight_to_token_ratio=0.130222 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259666982e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266952704 dispatch_meta_bytes=344232 combine_payload_bytes=266952704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267296936 ingress_bytes=301555712 link_bytes=301555712 token_bytes=534249640 weight_movement_bytes=34603008 total_nvlink_bytes=568852648 weight_to_token_ratio=0.064769 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134729867e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265420800 dispatch_meta_bytes=344216 combine_payload_bytes=265420800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265765016 ingress_bytes=300023808 link_bytes=300023808 token_bytes=531185816 weight_movement_bytes=34603008 total_nvlink_bytes=565788824 weight_to_token_ratio=0.065143 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128965436e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74334 payload_rows_remote=65094 dup_rows=23970 dup_rows_remote=20912 dedup_ratio=0.243835 router_tokens_per_expert=[734, 716, 793, 769, 737, 736, 776, 761, 784, 791, 785, 772, 766, 726, 712, 792, 726, 797, 789, 777, 720, 785, 793, 791, 730, 797, 767, 766, 794, 769, 779, 853, 789, 751, 735, 771, 756, 820, 762, 765, 708, 821, 797, 768, 744, 766, 771, 794, 810, 738, 772, 732, 821, 790, 776, 759, 741, 718, 795, 794, 767, 787, 739, 712] payload_rows_per_dst=[9198, 9328, 9298, 9430, 9240, 9264, 9350, 9226] dup_rows_per_dst=[2846, 2928, 3058, 3080, 3058, 3074, 3046, 2880] combine_rows_per_src=[9198, 9328, 9298, 9430, 9240, 9264, 9350, 9226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74464 payload_rows_remote=65120 dup_rows=23840 dup_rows_remote=20846 dedup_ratio=0.242513 router_tokens_per_expert=[797, 751, 775, 792, 782, 768, 729, 755, 803, 764, 747, 753, 756, 788, 765, 769, 732, 795, 796, 803, 773, 711, 731, 796, 712, 758, 703, 799, 767, 780, 786, 768, 774, 807, 784, 750, 775, 776, 804, 773, 755, 773, 753, 744, 753, 781, 803, 740, 768, 714, 792, 850, 781, 735, 748, 781, 766, 762, 752, 750, 776, 730, 810, 788] payload_rows_per_dst=[9308, 9368, 9280, 9206, 9436, 9270, 9344, 9252] dup_rows_per_dst=[2990, 2922, 2994, 2940, 3050, 2934, 2994, 3016] combine_rows_per_src=[9308, 9368, 9280, 9206, 9436, 9270, 9344, 9252] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266625024 dispatch_meta_bytes=344024 combine_payload_bytes=266625024 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266969048 ingress_bytes=335831040 link_bytes=335831040 token_bytes=533594072 weight_movement_bytes=69206016 total_nvlink_bytes=602800088 weight_to_token_ratio=0.129698 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263705167e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265379840 dispatch_meta_bytes=342704 combine_payload_bytes=265379840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265722544 ingress_bytes=299982848 link_bytes=299982848 token_bytes=531102384 weight_movement_bytes=34603008 total_nvlink_bytes=565705392 weight_to_token_ratio=0.065153 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128811306e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266821632 dispatch_meta_bytes=343184 combine_payload_bytes=266821632 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267164816 ingress_bytes=336027648 link_bytes=336027648 token_bytes=533986448 weight_movement_bytes=69206016 total_nvlink_bytes=603192464 weight_to_token_ratio=0.129603 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264444987e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266084352 dispatch_meta_bytes=341616 combine_payload_bytes=266084352 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266425968 ingress_bytes=300687360 link_bytes=300687360 token_bytes=532510320 weight_movement_bytes=34603008 total_nvlink_bytes=567113328 weight_to_token_ratio=0.064981 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131462328e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74092 payload_rows_remote=64898 dup_rows=24212 dup_rows_remote=21202 dedup_ratio=0.246297 router_tokens_per_expert=[787, 781, 774, 787, 776, 729, 781, 836, 813, 740, 724, 767, 746, 758, 787, 801, 804, 758, 771, 747, 765, 736, 776, 745, 752, 729, 747, 794, 791, 806, 746, 808, 773, 750, 737, 766, 770, 779, 766, 774, 759, 775, 737, 735, 787, 755, 783, 759, 788, 765, 769, 745, 741, 819, 765, 746, 776, 769, 753, 767, 759, 759, 780, 784] payload_rows_per_dst=[9456, 9200, 9194, 9298, 9228, 9214, 9244, 9258] dup_rows_per_dst=[3046, 3072, 3010, 3048, 3002, 2966, 3032, 3036] combine_rows_per_src=[9456, 9200, 9194, 9298, 9228, 9214, 9244, 9258] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=344592 combine_payload_bytes=265904128 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266248720 ingress_bytes=335110144 link_bytes=335110144 token_bytes=532152848 weight_movement_bytes=69206016 total_nvlink_bytes=601358864 weight_to_token_ratio=0.130049 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260992493e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74596 payload_rows_remote=65274 dup_rows=23708 dup_rows_remote=20742 dedup_ratio=0.241170 router_tokens_per_expert=[785, 726, 832, 780, 775, 755, 802, 762, 715, 763, 784, 748, 757, 753, 741, 798, 731, 793, 749, 751, 800, 790, 719, 775, 730, 780, 779, 822, 785, 774, 752, 757, 737, 783, 794, 794, 758, 798, 795, 773, 766, 805, 743, 788, 757, 744, 786, 755, 765, 775, 724, 784, 760, 724, 759, 794, 780, 748, 764, 802, 731, 803, 723, 777] payload_rows_per_dst=[9354, 9260, 9274, 9404, 9438, 9322, 9260, 9284] dup_rows_per_dst=[3080, 2858, 2942, 2954, 3026, 2966, 2910, 2972] combine_rows_per_src=[9354, 9260, 9274, 9404, 9438, 9322, 9260, 9284] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265822208 dispatch_meta_bytes=344400 combine_payload_bytes=265822208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266166608 ingress_bytes=300425216 link_bytes=300425216 token_bytes=531988816 weight_movement_bytes=34603008 total_nvlink_bytes=566591824 weight_to_token_ratio=0.065045 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130475902e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267362304 dispatch_meta_bytes=344064 combine_payload_bytes=267362304 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267706368 ingress_bytes=336568320 link_bytes=336568320 token_bytes=535068672 weight_movement_bytes=69206016 total_nvlink_bytes=604274688 weight_to_token_ratio=0.129340 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266479492e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74516 payload_rows_remote=65264 dup_rows=23788 dup_rows_remote=20796 dedup_ratio=0.241984 router_tokens_per_expert=[803, 686, 781, 770, 777, 763, 816, 770, 802, 747, 769, 744, 752, 768, 784, 709, 785, 807, 795, 712, 762, 761, 842, 731, 768, 771, 786, 717, 823, 779, 760, 718, 789, 785, 787, 848, 773, 774, 761, 752, 754, 740, 743, 748, 734, 813, 772, 737, 757, 764, 751, 764, 791, 756, 763, 777, 794, 773, 781, 847, 762, 691, 748, 765] payload_rows_per_dst=[9300, 9312, 9284, 9252, 9404, 9212, 9360, 9392] dup_rows_per_dst=[3032, 2838, 3106, 2992, 3134, 2870, 2886, 2930] combine_rows_per_src=[9300, 9312, 9284, 9252, 9404, 9212, 9360, 9392] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74326 payload_rows_remote=65020 dup_rows=23978 dup_rows_remote=20894 dedup_ratio=0.243917 router_tokens_per_expert=[706, 791, 777, 742, 755, 760, 736, 783, 782, 760, 736, 733, 728, 763, 783, 712, 772, 772, 774, 758, 744, 735, 768, 799, 763, 712, 778, 756, 758, 774, 798, 807, 762, 770, 765, 800, 786, 736, 784, 792, 763, 746, 774, 723, 807, 760, 809, 796, 794, 744, 759, 796, 778, 753, 800, 806, 793, 777, 813, 752, 765, 811, 769, 754] payload_rows_per_dst=[9258, 9154, 9298, 9286, 9306, 9306, 9372, 9346] dup_rows_per_dst=[2842, 2840, 2946, 3006, 3084, 3050, 3088, 3122] combine_rows_per_src=[9258, 9154, 9298, 9286, 9306, 9306, 9372, 9346] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74036 payload_rows_remote=64668 dup_rows=24268 dup_rows_remote=21120 dedup_ratio=0.246867 router_tokens_per_expert=[797, 793, 762, 770, 785, 802, 749, 800, 790, 724, 757, 782, 761, 781, 755, 760, 717, 747, 773, 777, 756, 763, 758, 768, 728, 814, 792, 759, 730, 776, 728, 750, 771, 738, 753, 769, 774, 755, 753, 801, 794, 744, 771, 833, 734, 761, 794, 772, 758, 796, 807, 805, 804, 728, 754, 789, 763, 797, 752, 756, 774, 749, 742, 757] payload_rows_per_dst=[9368, 9654, 9090, 9198, 8884, 9304, 9410, 9128] dup_rows_per_dst=[3148, 3290, 3086, 2956, 2620, 3044, 3072, 3052] combine_rows_per_src=[9368, 9654, 9090, 9198, 8884, 9304, 9410, 9128] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64872 dup_rows=24114 dup_rows_remote=21094 dedup_ratio=0.245300 router_tokens_per_expert=[813, 773, 733, 774, 747, 732, 736, 753, 712, 807, 745, 774, 731, 792, 801, 807, 764, 748, 745, 805, 811, 769, 724, 758, 714, 794, 756, 773, 795, 805, 771, 800, 764, 764, 767, 766, 784, 775, 782, 729, 790, 765, 790, 790, 795, 808, 772, 753, 739, 781, 785, 731, 828, 743, 746, 726, 735, 785, 773, 756, 800, 742, 745, 781] payload_rows_per_dst=[9102, 9318, 9252, 9374, 9260, 9474, 9212, 9198] dup_rows_per_dst=[3020, 3020, 2996, 3042, 3002, 3052, 2946, 3036] combine_rows_per_src=[9102, 9318, 9252, 9374, 9260, 9474, 9212, 9198] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74204 payload_rows_remote=64848 dup_rows=24100 dup_rows_remote=20990 dedup_ratio=0.245158 router_tokens_per_expert=[742, 771, 742, 755, 799, 763, 741, 808, 740, 748, 772, 767, 771, 756, 742, 759, 813, 775, 764, 792, 793, 796, 763, 726, 794, 750, 766, 767, 765, 788, 752, 797, 764, 746, 802, 771, 751, 772, 812, 801, 770, 813, 749, 776, 765, 784, 768, 759, 809, 790, 775, 730, 781, 759, 778, 732, 738, 731, 752, 834, 715, 738, 765, 745] payload_rows_per_dst=[9212, 9178, 9306, 9262, 9432, 9344, 9114, 9356] dup_rows_per_dst=[2896, 2932, 3138, 2972, 3116, 3040, 2896, 3110] combine_rows_per_src=[9212, 9178, 9306, 9262, 9432, 9344, 9114, 9356] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64872 dup_rows=24114 dup_rows_remote=21094 dedup_ratio=0.245300 router_tokens_per_expert=[813, 773, 733, 774, 747, 732, 736, 753, 712, 807, 745, 774, 731, 792, 801, 807, 764, 748, 745, 805, 811, 769, 724, 758, 714, 794, 756, 773, 795, 805, 771, 800, 764, 764, 767, 766, 784, 775, 782, 729, 790, 765, 790, 790, 795, 808, 772, 753, 739, 781, 785, 731, 828, 743, 746, 726, 735, 785, 773, 756, 800, 742, 745, 781] payload_rows_per_dst=[9102, 9318, 9252, 9374, 9260, 9474, 9212, 9198] dup_rows_per_dst=[3020, 3020, 2996, 3042, 3002, 3052, 2946, 3036] combine_rows_per_src=[9102, 9318, 9252, 9374, 9260, 9474, 9212, 9198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74596 payload_rows_remote=65274 dup_rows=23708 dup_rows_remote=20742 dedup_ratio=0.241170 router_tokens_per_expert=[785, 726, 832, 780, 775, 755, 802, 762, 715, 763, 784, 748, 757, 753, 741, 798, 731, 793, 749, 751, 800, 790, 719, 775, 730, 780, 779, 822, 785, 774, 752, 757, 737, 783, 794, 794, 758, 798, 795, 773, 766, 805, 743, 788, 757, 744, 786, 755, 765, 775, 724, 784, 760, 724, 759, 794, 780, 748, 764, 802, 731, 803, 723, 777] payload_rows_per_dst=[9354, 9260, 9274, 9404, 9438, 9322, 9260, 9284] dup_rows_per_dst=[3080, 2858, 2942, 2954, 3026, 2966, 2910, 2972] combine_rows_per_src=[9354, 9260, 9274, 9404, 9438, 9322, 9260, 9284] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74464 payload_rows_remote=65120 dup_rows=23840 dup_rows_remote=20846 dedup_ratio=0.242513 router_tokens_per_expert=[797, 751, 775, 792, 782, 768, 729, 755, 803, 764, 747, 753, 756, 788, 765, 769, 732, 795, 796, 803, 773, 711, 731, 796, 712, 758, 703, 799, 767, 780, 786, 768, 774, 807, 784, 750, 775, 776, 804, 773, 755, 773, 753, 744, 753, 781, 803, 740, 768, 714, 792, 850, 781, 735, 748, 781, 766, 762, 752, 750, 776, 730, 810, 788] payload_rows_per_dst=[9308, 9368, 9280, 9206, 9436, 9270, 9344, 9252] dup_rows_per_dst=[2990, 2922, 2994, 2940, 3050, 2934, 2994, 3016] combine_rows_per_src=[9308, 9368, 9280, 9206, 9436, 9270, 9344, 9252] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267321344 dispatch_meta_bytes=344240 combine_payload_bytes=267321344 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267665584 ingress_bytes=371130368 link_bytes=371130368 token_bytes=534986928 weight_movement_bytes=103809024 total_nvlink_bytes=638795952 weight_to_token_ratio=0.194040 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396533696e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266321920 dispatch_meta_bytes=343656 combine_payload_bytes=266321920 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266665576 ingress_bytes=335527936 link_bytes=335527936 token_bytes=532987496 weight_movement_bytes=69206016 total_nvlink_bytes=602193512 weight_to_token_ratio=0.129845 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262564611e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74092 payload_rows_remote=64898 dup_rows=24212 dup_rows_remote=21202 dedup_ratio=0.246297 router_tokens_per_expert=[787, 781, 774, 787, 776, 729, 781, 836, 813, 740, 724, 767, 746, 758, 787, 801, 804, 758, 771, 747, 765, 736, 776, 745, 752, 729, 747, 794, 791, 806, 746, 808, 773, 750, 737, 766, 770, 779, 766, 774, 759, 775, 737, 735, 787, 755, 783, 759, 788, 765, 769, 745, 741, 819, 765, 746, 776, 769, 753, 767, 759, 759, 780, 784] payload_rows_per_dst=[9456, 9200, 9194, 9298, 9228, 9214, 9244, 9258] dup_rows_per_dst=[3046, 3072, 3010, 3048, 3002, 2966, 3032, 3036] combine_rows_per_src=[9456, 9200, 9194, 9298, 9228, 9214, 9244, 9258] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=343864 combine_payload_bytes=266731520 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267075384 ingress_bytes=405143552 link_bytes=405143552 token_bytes=533806904 weight_movement_bytes=138412032 total_nvlink_bytes=672218936 weight_to_token_ratio=0.259292 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524522569e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74516 payload_rows_remote=65264 dup_rows=23788 dup_rows_remote=20796 dedup_ratio=0.241984 router_tokens_per_expert=[803, 686, 781, 770, 777, 763, 816, 770, 802, 747, 769, 744, 752, 768, 784, 709, 785, 807, 795, 712, 762, 761, 842, 731, 768, 771, 786, 717, 823, 779, 760, 718, 789, 785, 787, 848, 773, 774, 761, 752, 754, 740, 743, 748, 734, 813, 772, 737, 757, 764, 751, 764, 791, 756, 763, 777, 794, 773, 781, 847, 762, 691, 748, 765] payload_rows_per_dst=[9300, 9312, 9284, 9252, 9404, 9212, 9360, 9392] dup_rows_per_dst=[3032, 2838, 3106, 2992, 3134, 2870, 2886, 2930] combine_rows_per_src=[9300, 9312, 9284, 9252, 9404, 9212, 9360, 9392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=264880128 dispatch_meta_bytes=343152 combine_payload_bytes=264880128 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265223280 ingress_bytes=334086144 link_bytes=334086144 token_bytes=530103408 weight_movement_bytes=69206016 total_nvlink_bytes=599309424 weight_to_token_ratio=0.130552 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257139264e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265617408 dispatch_meta_bytes=343352 combine_payload_bytes=265617408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265960760 ingress_bytes=300220416 link_bytes=300220416 token_bytes=531578168 weight_movement_bytes=34603008 total_nvlink_bytes=566181176 weight_to_token_ratio=0.065095 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129705256e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265715712 dispatch_meta_bytes=343864 combine_payload_bytes=265715712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266059576 ingress_bytes=300318720 link_bytes=300318720 token_bytes=531775288 weight_movement_bytes=34603008 total_nvlink_bytes=566378296 weight_to_token_ratio=0.065071 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130075166e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73970 payload_rows_remote=64820 dup_rows=24334 dup_rows_remote=21424 dedup_ratio=0.247538 router_tokens_per_expert=[780, 760, 773, 783, 822, 729, 775, 765, 759, 788, 766, 721, 777, 737, 771, 816, 795, 779, 793, 734, 719, 731, 752, 727, 802, 779, 773, 761, 813, 759, 784, 726, 802, 769, 722, 761, 782, 722, 804, 749, 759, 745, 800, 737, 786, 781, 799, 796, 816, 767, 746, 750, 745, 774, 770, 805, 754, 742, 793, 758, 777, 772, 745, 775] payload_rows_per_dst=[9242, 9210, 9150, 9238, 9260, 9318, 9340, 9212] dup_rows_per_dst=[3132, 3060, 2910, 3156, 2962, 3088, 3006, 3020] combine_rows_per_src=[9242, 9210, 9150, 9238, 9260, 9318, 9340, 9212] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=65112 dup_rows=23930 dup_rows_remote=21062 dedup_ratio=0.243429 router_tokens_per_expert=[756, 788, 761, 771, 756, 732, 796, 782, 809, 734, 786, 785, 750, 778, 791, 771, 778, 800, 762, 769, 735, 764, 796, 855, 721, 752, 778, 742, 773, 775, 774, 790, 778, 701, 746, 785, 762, 803, 762, 737, 774, 748, 761, 752, 781, 718, 747, 784, 761, 740, 749, 774, 751, 762, 724, 763, 857, 800, 716, 775, 789, 772, 800, 770] payload_rows_per_dst=[9292, 9318, 9490, 9286, 9184, 9262, 9140, 9402] dup_rows_per_dst=[2992, 3090, 3028, 2924, 2964, 2868, 2908, 3156] combine_rows_per_src=[9292, 9318, 9490, 9286, 9184, 9262, 9140, 9402] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=343864 combine_payload_bytes=266731520 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267075384 ingress_bytes=405143552 link_bytes=405143552 token_bytes=533806904 weight_movement_bytes=138412032 total_nvlink_bytes=672218936 weight_to_token_ratio=0.259292 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524522569e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=344976 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265847696 ingress_bytes=300105728 link_bytes=300105728 token_bytes=531350416 weight_movement_bytes=34603008 total_nvlink_bytes=565953424 weight_to_token_ratio=0.065123 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129273694e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266698752 dispatch_meta_bytes=344696 combine_payload_bytes=266698752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267043448 ingress_bytes=301301760 link_bytes=301301760 token_bytes=533742200 weight_movement_bytes=34603008 total_nvlink_bytes=568345208 weight_to_token_ratio=0.064831 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133774266e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74350 payload_rows_remote=65182 dup_rows=23954 dup_rows_remote=21016 dedup_ratio=0.243673 router_tokens_per_expert=[756, 762, 775, 788, 762, 715, 792, 837, 760, 752, 806, 767, 789, 769, 807, 786, 778, 760, 784, 758, 808, 758, 750, 758, 772, 724, 766, 747, 735, 770, 745, 794, 704, 810, 730, 778, 793, 748, 759, 767, 806, 738, 710, 749, 804, 753, 770, 727, 811, 773, 821, 758, 786, 752, 817, 762, 794, 776, 779, 790, 725, 766, 711, 755] payload_rows_per_dst=[9290, 9484, 9296, 9168, 9272, 9152, 9480, 9208] dup_rows_per_dst=[3084, 2988, 3012, 2938, 2906, 2962, 3080, 2984] combine_rows_per_src=[9290, 9484, 9296, 9168, 9272, 9152, 9480, 9208] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74176 payload_rows_remote=64832 dup_rows=24128 dup_rows_remote=21080 dedup_ratio=0.245443 router_tokens_per_expert=[741, 769, 797, 818, 786, 766, 775, 762, 742, 731, 725, 780, 783, 779, 786, 754, 752, 788, 808, 794, 758, 757, 728, 763, 769, 749, 738, 744, 758, 744, 792, 750, 738, 767, 746, 824, 780, 795, 750, 796, 799, 732, 771, 819, 714, 793, 818, 780, 791, 717, 761, 805, 741, 802, 780, 846, 737, 717, 766, 761, 749, 726, 743, 802] payload_rows_per_dst=[9334, 9184, 9202, 9182, 9344, 9392, 9388, 9150] dup_rows_per_dst=[3094, 2976, 3094, 2906, 3048, 3060, 3098, 2852] combine_rows_per_src=[9334, 9184, 9202, 9182, 9344, 9392, 9388, 9150] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74250 payload_rows_remote=64990 dup_rows=24054 dup_rows_remote=21058 dedup_ratio=0.244690 router_tokens_per_expert=[746, 795, 745, 783, 775, 742, 740, 786, 823, 818, 753, 750, 783, 726, 777, 760, 711, 751, 786, 788, 741, 717, 730, 754, 739, 744, 751, 749, 775, 788, 776, 738, 764, 809, 770, 756, 771, 745, 737, 815, 772, 770, 781, 803, 777, 759, 740, 789, 710, 765, 736, 789, 813, 811, 746, 767, 774, 777, 750, 792, 788, 823, 841, 772] payload_rows_per_dst=[9260, 9304, 9214, 9230, 9382, 9278, 9138, 9444] dup_rows_per_dst=[2996, 3050, 2844, 2890, 3154, 3104, 2832, 3184] combine_rows_per_src=[9260, 9304, 9214, 9230, 9382, 9278, 9138, 9444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267362304 dispatch_meta_bytes=344064 combine_payload_bytes=267362304 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267706368 ingress_bytes=336568320 link_bytes=336568320 token_bytes=535068672 weight_movement_bytes=69206016 total_nvlink_bytes=604274688 weight_to_token_ratio=0.129340 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266479492e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265822208 dispatch_meta_bytes=344400 combine_payload_bytes=265822208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266166608 ingress_bytes=300425216 link_bytes=300425216 token_bytes=531988816 weight_movement_bytes=34603008 total_nvlink_bytes=566591824 weight_to_token_ratio=0.065045 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130475902e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74326 payload_rows_remote=65020 dup_rows=23978 dup_rows_remote=20894 dedup_ratio=0.243917 router_tokens_per_expert=[706, 791, 777, 742, 755, 760, 736, 783, 782, 760, 736, 733, 728, 763, 783, 712, 772, 772, 774, 758, 744, 735, 768, 799, 763, 712, 778, 756, 758, 774, 798, 807, 762, 770, 765, 800, 786, 736, 784, 792, 763, 746, 774, 723, 807, 760, 809, 796, 794, 744, 759, 796, 778, 753, 800, 806, 793, 777, 813, 752, 765, 811, 769, 754] payload_rows_per_dst=[9258, 9154, 9298, 9286, 9306, 9306, 9372, 9346] dup_rows_per_dst=[2842, 2840, 2946, 3006, 3084, 3050, 3088, 3122] combine_rows_per_src=[9258, 9154, 9298, 9286, 9306, 9306, 9372, 9346] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267321344 dispatch_meta_bytes=344240 combine_payload_bytes=267321344 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267665584 ingress_bytes=371130368 link_bytes=371130368 token_bytes=534986928 weight_movement_bytes=103809024 total_nvlink_bytes=638795952 weight_to_token_ratio=0.194040 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396533696e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266321920 dispatch_meta_bytes=343656 combine_payload_bytes=266321920 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266665576 ingress_bytes=335527936 link_bytes=335527936 token_bytes=532987496 weight_movement_bytes=69206016 total_nvlink_bytes=602193512 weight_to_token_ratio=0.129845 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262564611e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74314 payload_rows_remote=64890 dup_rows=23990 dup_rows_remote=20912 dedup_ratio=0.244039 router_tokens_per_expert=[748, 804, 759, 772, 783, 723, 757, 749, 815, 727, 817, 764, 805, 749, 744, 768, 777, 760, 802, 760, 766, 791, 748, 787, 737, 782, 773, 778, 757, 752, 773, 778, 747, 765, 756, 785, 803, 780, 777, 771, 769, 771, 767, 750, 761, 762, 763, 727, 779, 776, 795, 809, 761, 783, 748, 800, 713, 765, 806, 730, 749, 760, 771, 748] payload_rows_per_dst=[9154, 9300, 9336, 9344, 9276, 9300, 9424, 9180] dup_rows_per_dst=[3036, 3078, 3046, 2916, 3092, 2840, 3078, 2904] combine_rows_per_src=[9154, 9300, 9336, 9344, 9276, 9300, 9424, 9180] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74278 payload_rows_remote=65034 dup_rows=24026 dup_rows_remote=21112 dedup_ratio=0.244405 router_tokens_per_expert=[786, 748, 767, 747, 751, 740, 796, 783, 755, 807, 754, 765, 797, 808, 722, 806, 765, 748, 790, 776, 753, 809, 779, 786, 765, 770, 773, 801, 760, 816, 756, 732, 755, 781, 815, 793, 768, 750, 743, 710, 745, 774, 754, 777, 746, 817, 736, 745, 787, 772, 745, 789, 764, 807, 734, 755, 751, 770, 757, 772, 733, 767, 783, 746] payload_rows_per_dst=[8976, 9230, 9394, 9494, 9288, 9292, 9360, 9244] dup_rows_per_dst=[2860, 2962, 3018, 3252, 2942, 3096, 2982, 2914] combine_rows_per_src=[8976, 9230, 9394, 9494, 9288, 9292, 9360, 9244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74566 payload_rows_remote=65424 dup_rows=23738 dup_rows_remote=20782 dedup_ratio=0.241475 router_tokens_per_expert=[812, 769, 801, 749, 768, 792, 709, 815, 757, 742, 768, 741, 759, 767, 741, 774, 792, 749, 754, 780, 787, 798, 782, 782, 812, 775, 773, 749, 786, 805, 773, 767, 750, 785, 813, 712, 780, 786, 735, 763, 767, 770, 789, 778, 726, 766, 725, 764, 791, 749, 788, 754, 719, 811, 730, 774, 771, 761, 747, 776, 742, 760, 768, 774] payload_rows_per_dst=[9400, 9142, 9392, 9484, 9314, 9362, 9240, 9232] dup_rows_per_dst=[3030, 2956, 3056, 2996, 2934, 2808, 2992, 2966] combine_rows_per_src=[9400, 9142, 9392, 9484, 9314, 9362, 9240, 9232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74250 payload_rows_remote=64990 dup_rows=24054 dup_rows_remote=21058 dedup_ratio=0.244690 router_tokens_per_expert=[746, 795, 745, 783, 775, 742, 740, 786, 823, 818, 753, 750, 783, 726, 777, 760, 711, 751, 786, 788, 741, 717, 730, 754, 739, 744, 751, 749, 775, 788, 776, 738, 764, 809, 770, 756, 771, 745, 737, 815, 772, 770, 781, 803, 777, 759, 740, 789, 710, 765, 736, 789, 813, 811, 746, 767, 774, 777, 750, 792, 788, 823, 841, 772] payload_rows_per_dst=[9260, 9304, 9214, 9230, 9382, 9278, 9138, 9444] dup_rows_per_dst=[2996, 3050, 2844, 2890, 3154, 3104, 2832, 3184] combine_rows_per_src=[9260, 9304, 9214, 9230, 9382, 9278, 9138, 9444] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=65112 dup_rows=23930 dup_rows_remote=21062 dedup_ratio=0.243429 router_tokens_per_expert=[756, 788, 761, 771, 756, 732, 796, 782, 809, 734, 786, 785, 750, 778, 791, 771, 778, 800, 762, 769, 735, 764, 796, 855, 721, 752, 778, 742, 773, 775, 774, 790, 778, 701, 746, 785, 762, 803, 762, 737, 774, 748, 761, 752, 781, 718, 747, 784, 761, 740, 749, 774, 751, 762, 724, 763, 857, 800, 716, 775, 789, 772, 800, 770] payload_rows_per_dst=[9292, 9318, 9490, 9286, 9184, 9262, 9140, 9402] dup_rows_per_dst=[2992, 3090, 3028, 2924, 2964, 2868, 2908, 3156] combine_rows_per_src=[9292, 9318, 9490, 9286, 9184, 9262, 9140, 9402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265789440 dispatch_meta_bytes=343208 combine_payload_bytes=265789440 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266132648 ingress_bytes=334995456 link_bytes=334995456 token_bytes=531922088 weight_movement_bytes=69206016 total_nvlink_bytes=601128104 weight_to_token_ratio=0.130106 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260560932e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266985472 dispatch_meta_bytes=344792 combine_payload_bytes=266985472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267330264 ingress_bytes=301588480 link_bytes=301588480 token_bytes=534315736 weight_movement_bytes=34603008 total_nvlink_bytes=568918744 weight_to_token_ratio=0.064761 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134853170e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74314 payload_rows_remote=64890 dup_rows=23990 dup_rows_remote=20912 dedup_ratio=0.244039 router_tokens_per_expert=[748, 804, 759, 772, 783, 723, 757, 749, 815, 727, 817, 764, 805, 749, 744, 768, 777, 760, 802, 760, 766, 791, 748, 787, 737, 782, 773, 778, 757, 752, 773, 778, 747, 765, 756, 785, 803, 780, 777, 771, 769, 771, 767, 750, 761, 762, 763, 727, 779, 776, 795, 809, 761, 783, 748, 800, 713, 765, 806, 730, 749, 760, 771, 748] payload_rows_per_dst=[9154, 9300, 9336, 9344, 9276, 9300, 9424, 9180] dup_rows_per_dst=[3036, 3078, 3046, 2916, 3092, 2840, 3078, 2904] combine_rows_per_src=[9154, 9300, 9336, 9344, 9276, 9300, 9424, 9180] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265551872 dispatch_meta_bytes=343648 combine_payload_bytes=265551872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265895520 ingress_bytes=300154880 link_bytes=300154880 token_bytes=531447392 weight_movement_bytes=34603008 total_nvlink_bytes=566050400 weight_to_token_ratio=0.065111 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129458649e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=344192 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266543232 ingress_bytes=335405056 link_bytes=335405056 token_bytes=532742272 weight_movement_bytes=69206016 total_nvlink_bytes=601948288 weight_to_token_ratio=0.129905 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262102223e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73970 payload_rows_remote=64820 dup_rows=24334 dup_rows_remote=21424 dedup_ratio=0.247538 router_tokens_per_expert=[780, 760, 773, 783, 822, 729, 775, 765, 759, 788, 766, 721, 777, 737, 771, 816, 795, 779, 793, 734, 719, 731, 752, 727, 802, 779, 773, 761, 813, 759, 784, 726, 802, 769, 722, 761, 782, 722, 804, 749, 759, 745, 800, 737, 786, 781, 799, 796, 816, 767, 746, 750, 745, 774, 770, 805, 754, 742, 793, 758, 777, 772, 745, 775] payload_rows_per_dst=[9242, 9210, 9150, 9238, 9260, 9318, 9340, 9212] dup_rows_per_dst=[3132, 3060, 2910, 3156, 2962, 3088, 3006, 3020] combine_rows_per_src=[9242, 9210, 9150, 9238, 9260, 9318, 9340, 9212] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74350 payload_rows_remote=65182 dup_rows=23954 dup_rows_remote=21016 dedup_ratio=0.243673 router_tokens_per_expert=[756, 762, 775, 788, 762, 715, 792, 837, 760, 752, 806, 767, 789, 769, 807, 786, 778, 760, 784, 758, 808, 758, 750, 758, 772, 724, 766, 747, 735, 770, 745, 794, 704, 810, 730, 778, 793, 748, 759, 767, 806, 738, 710, 749, 804, 753, 770, 727, 811, 773, 821, 758, 786, 752, 817, 762, 794, 776, 779, 790, 725, 766, 711, 755] payload_rows_per_dst=[9290, 9484, 9296, 9168, 9272, 9152, 9480, 9208] dup_rows_per_dst=[3084, 2988, 3012, 2938, 2906, 2962, 3080, 2984] combine_rows_per_src=[9290, 9484, 9296, 9168, 9272, 9152, 9480, 9208] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266379264 dispatch_meta_bytes=344584 combine_payload_bytes=266379264 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266723848 ingress_bytes=300982272 link_bytes=300982272 token_bytes=533103112 weight_movement_bytes=34603008 total_nvlink_bytes=567706120 weight_to_token_ratio=0.064909 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132572058e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=64912 dup_rows=24046 dup_rows_remote=20954 dedup_ratio=0.244609 router_tokens_per_expert=[790, 781, 777, 806, 749, 761, 751, 758, 762, 755, 757, 763, 767, 782, 779, 715, 779, 760, 747, 790, 765, 774, 798, 806, 762, 760, 747, 756, 740, 778, 744, 777, 765, 765, 745, 750, 757, 766, 744, 828, 746, 799, 764, 772, 789, 776, 778, 749, 796, 753, 739, 714, 746, 803, 798, 770, 756, 760, 788, 783, 790, 772, 784, 771] payload_rows_per_dst=[9288, 9222, 9346, 9164, 9238, 9310, 9280, 9410] dup_rows_per_dst=[3058, 2938, 3092, 2964, 3002, 3036, 2958, 2998] combine_rows_per_src=[9288, 9222, 9346, 9164, 9238, 9310, 9280, 9410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267976704 dispatch_meta_bytes=344824 combine_payload_bytes=267976704 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=268321528 ingress_bytes=406388736 link_bytes=406388736 token_bytes=536298232 weight_movement_bytes=138412032 total_nvlink_bytes=674710264 weight_to_token_ratio=0.258088 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.529208097e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74418 payload_rows_remote=65138 dup_rows=23886 dup_rows_remote=20940 dedup_ratio=0.242981 router_tokens_per_expert=[769, 791, 744, 789, 798, 802, 751, 867, 783, 732, 754, 764, 771, 711, 765, 787, 762, 757, 775, 726, 781, 787, 802, 784, 785, 756, 793, 762, 762, 784, 768, 765, 753, 766, 757, 750, 736, 751, 777, 759, 720, 772, 757, 801, 775, 720, 820, 748, 796, 800, 774, 830, 791, 735, 732, 756, 777, 760, 799, 746, 759, 706, 791, 711] payload_rows_per_dst=[9524, 9182, 9374, 9292, 9270, 9280, 9338, 9158] dup_rows_per_dst=[3098, 2952, 2974, 3058, 2828, 2946, 3090, 2940] combine_rows_per_src=[9524, 9182, 9374, 9292, 9270, 9280, 9338, 9158] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265789440 dispatch_meta_bytes=343208 combine_payload_bytes=265789440 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266132648 ingress_bytes=334995456 link_bytes=334995456 token_bytes=531922088 weight_movement_bytes=69206016 total_nvlink_bytes=601128104 weight_to_token_ratio=0.130106 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260560932e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265879552 dispatch_meta_bytes=343464 combine_payload_bytes=265879552 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266223016 ingress_bytes=404291584 link_bytes=404291584 token_bytes=532102568 weight_movement_bytes=138412032 total_nvlink_bytes=670514600 weight_to_token_ratio=0.260123 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521316682e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266805248 dispatch_meta_bytes=344312 combine_payload_bytes=266805248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267149560 ingress_bytes=301408256 link_bytes=301408256 token_bytes=533954808 weight_movement_bytes=34603008 total_nvlink_bytes=568557816 weight_to_token_ratio=0.064805 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134175002e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=65220 dup_rows=24010 dup_rows_remote=21116 dedup_ratio=0.244242 router_tokens_per_expert=[801, 766, 786, 723, 793, 739, 763, 730, 768, 769, 785, 775, 781, 774, 827, 767, 785, 781, 749, 768, 718, 751, 798, 772, 756, 712, 734, 761, 747, 787, 716, 771, 755, 772, 778, 750, 773, 775, 804, 784, 808, 770, 779, 757, 782, 738, 767, 782, 803, 737, 731, 763, 767, 804, 752, 760, 728, 778, 747, 796, 829, 750, 802, 778] payload_rows_per_dst=[9268, 9454, 9250, 9074, 9328, 9290, 9260, 9370] dup_rows_per_dst=[2934, 3038, 2994, 2894, 3054, 3076, 2974, 3046] combine_rows_per_src=[9268, 9454, 9250, 9074, 9328, 9290, 9260, 9370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74270 payload_rows_remote=64810 dup_rows=24034 dup_rows_remote=20960 dedup_ratio=0.244486 router_tokens_per_expert=[786, 824, 746, 802, 761, 796, 811, 779, 738, 709, 750, 789, 760, 730, 772, 748, 786, 765, 747, 746, 808, 788, 789, 803, 773, 834, 764, 779, 740, 771, 722, 735, 827, 782, 799, 816, 784, 791, 733, 735, 771, 750, 698, 827, 760, 733, 782, 752, 712, 783, 718, 759, 786, 780, 783, 763, 757, 778, 789, 803, 751, 762, 693, 744] payload_rows_per_dst=[9482, 9138, 9404, 9224, 9460, 9212, 9188, 9162] dup_rows_per_dst=[3128, 2854, 3060, 3012, 3074, 2934, 2980, 2992] combine_rows_per_src=[9482, 9138, 9404, 9224, 9460, 9212, 9188, 9162] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74576 payload_rows_remote=65236 dup_rows=23728 dup_rows_remote=20804 dedup_ratio=0.241374 router_tokens_per_expert=[760, 753, 729, 764, 770, 771, 788, 777, 771, 757, 739, 748, 818, 775, 730, 792, 790, 770, 749, 716, 773, 832, 813, 778, 785, 740, 772, 766, 774, 751, 817, 753, 759, 761, 758, 768, 736, 790, 728, 778, 754, 768, 796, 744, 813, 767, 793, 772, 716, 735, 758, 757, 736, 731, 776, 834, 776, 799, 763, 807, 745, 783, 753, 777] payload_rows_per_dst=[9340, 9252, 9376, 9326, 9254, 9330, 9232, 9466] dup_rows_per_dst=[2924, 3008, 3072, 2990, 2856, 3084, 2854, 2940] combine_rows_per_src=[9340, 9252, 9376, 9326, 9254, 9330, 9232, 9466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=344192 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266543232 ingress_bytes=335405056 link_bytes=335405056 token_bytes=532742272 weight_movement_bytes=69206016 total_nvlink_bytes=601948288 weight_to_token_ratio=0.129905 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262102223e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266698752 dispatch_meta_bytes=344696 combine_payload_bytes=266698752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267043448 ingress_bytes=301301760 link_bytes=301301760 token_bytes=533742200 weight_movement_bytes=34603008 total_nvlink_bytes=568345208 weight_to_token_ratio=0.064831 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133774266e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74176 payload_rows_remote=64832 dup_rows=24128 dup_rows_remote=21080 dedup_ratio=0.245443 router_tokens_per_expert=[741, 769, 797, 818, 786, 766, 775, 762, 742, 731, 725, 780, 783, 779, 786, 754, 752, 788, 808, 794, 758, 757, 728, 763, 769, 749, 738, 744, 758, 744, 792, 750, 738, 767, 746, 824, 780, 795, 750, 796, 799, 732, 771, 819, 714, 793, 818, 780, 791, 717, 761, 805, 741, 802, 780, 846, 737, 717, 766, 761, 749, 726, 743, 802] payload_rows_per_dst=[9334, 9184, 9202, 9182, 9344, 9392, 9388, 9150] dup_rows_per_dst=[3094, 2976, 3094, 2906, 3048, 3060, 3098, 2852] combine_rows_per_src=[9334, 9184, 9202, 9182, 9344, 9392, 9388, 9150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=344976 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265847696 ingress_bytes=300105728 link_bytes=300105728 token_bytes=531350416 weight_movement_bytes=34603008 total_nvlink_bytes=565953424 weight_to_token_ratio=0.065123 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129273694e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266985472 dispatch_meta_bytes=344792 combine_payload_bytes=266985472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267330264 ingress_bytes=301588480 link_bytes=301588480 token_bytes=534315736 weight_movement_bytes=34603008 total_nvlink_bytes=568918744 weight_to_token_ratio=0.064761 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134853170e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265551872 dispatch_meta_bytes=343648 combine_payload_bytes=265551872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265895520 ingress_bytes=300154880 link_bytes=300154880 token_bytes=531447392 weight_movement_bytes=34603008 total_nvlink_bytes=566050400 weight_to_token_ratio=0.065111 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129458649e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74656 payload_rows_remote=65310 dup_rows=23648 dup_rows_remote=20674 dedup_ratio=0.240560 router_tokens_per_expert=[743, 726, 770, 779, 768, 802, 750, 823, 789, 777, 809, 766, 763, 784, 752, 735, 747, 774, 752, 741, 769, 749, 790, 780, 767, 784, 786, 758, 739, 720, 774, 752, 799, 720, 786, 785, 791, 747, 783, 764, 788, 783, 750, 765, 795, 770, 761, 719, 784, 747, 790, 761, 753, 798, 753, 774, 821, 773, 749, 749, 781, 740, 751, 804] payload_rows_per_dst=[9378, 9342, 9314, 9218, 9402, 9298, 9346, 9358] dup_rows_per_dst=[2944, 3008, 2890, 2942, 2948, 2964, 2974, 2978] combine_rows_per_src=[9378, 9342, 9314, 9218, 9402, 9298, 9346, 9358] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74170 payload_rows_remote=64936 dup_rows=24134 dup_rows_remote=21056 dedup_ratio=0.245504 router_tokens_per_expert=[802, 767, 798, 799, 751, 726, 801, 787, 775, 745, 735, 761, 690, 757, 741, 812, 822, 791, 702, 720, 763, 771, 783, 783, 801, 787, 775, 755, 770, 796, 747, 834, 778, 747, 769, 734, 769, 728, 708, 836, 754, 812, 789, 730, 739, 772, 733, 771, 790, 780, 793, 811, 789, 724, 757, 774, 746, 783, 779, 804, 747, 735, 764, 760] payload_rows_per_dst=[8842, 9454, 8992, 9558, 9194, 9366, 9530, 9234] dup_rows_per_dst=[2740, 3168, 2768, 3334, 2944, 2906, 3196, 3078] combine_rows_per_src=[8842, 9454, 8992, 9558, 9194, 9366, 9530, 9234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=65108 dup_rows=23948 dup_rows_remote=20858 dedup_ratio=0.243612 router_tokens_per_expert=[786, 748, 757, 768, 800, 761, 744, 773, 731, 829, 761, 767, 739, 767, 809, 766, 805, 795, 801, 739, 768, 754, 751, 750, 784, 769, 779, 777, 754, 766, 726, 777, 854, 749, 785, 811, 755, 754, 769, 777, 804, 767, 765, 735, 762, 743, 771, 766, 800, 756, 752, 766, 779, 808, 754, 704, 757, 756, 741, 732, 790, 772, 771, 746] payload_rows_per_dst=[9296, 9248, 9320, 9228, 9454, 9270, 9282, 9258] dup_rows_per_dst=[2978, 3090, 3006, 3036, 3054, 2956, 2956, 2872] combine_rows_per_src=[9296, 9248, 9320, 9228, 9454, 9270, 9282, 9258] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74576 payload_rows_remote=65236 dup_rows=23728 dup_rows_remote=20804 dedup_ratio=0.241374 router_tokens_per_expert=[760, 753, 729, 764, 770, 771, 788, 777, 771, 757, 739, 748, 818, 775, 730, 792, 790, 770, 749, 716, 773, 832, 813, 778, 785, 740, 772, 766, 774, 751, 817, 753, 759, 761, 758, 768, 736, 790, 728, 778, 754, 768, 796, 744, 813, 767, 793, 772, 716, 735, 758, 757, 736, 731, 776, 834, 776, 799, 763, 807, 745, 783, 753, 777] payload_rows_per_dst=[9340, 9252, 9376, 9326, 9254, 9330, 9232, 9466] dup_rows_per_dst=[2924, 3008, 3072, 2990, 2856, 3084, 2854, 2940] combine_rows_per_src=[9340, 9252, 9376, 9326, 9254, 9330, 9232, 9466] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74418 payload_rows_remote=65138 dup_rows=23886 dup_rows_remote=20940 dedup_ratio=0.242981 router_tokens_per_expert=[769, 791, 744, 789, 798, 802, 751, 867, 783, 732, 754, 764, 771, 711, 765, 787, 762, 757, 775, 726, 781, 787, 802, 784, 785, 756, 793, 762, 762, 784, 768, 765, 753, 766, 757, 750, 736, 751, 777, 759, 720, 772, 757, 801, 775, 720, 820, 748, 796, 800, 774, 830, 791, 735, 732, 756, 777, 760, 799, 746, 759, 706, 791, 711] payload_rows_per_dst=[9524, 9182, 9374, 9292, 9270, 9280, 9338, 9158] dup_rows_per_dst=[3098, 2952, 2974, 3058, 2828, 2946, 3090, 2940] combine_rows_per_src=[9524, 9182, 9374, 9292, 9270, 9280, 9338, 9158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267509760 dispatch_meta_bytes=343936 combine_payload_bytes=267509760 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267853696 ingress_bytes=371318784 link_bytes=371318784 token_bytes=535363456 weight_movement_bytes=103809024 total_nvlink_bytes=639172480 weight_to_token_ratio=0.193904 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397242691e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=267141120 dispatch_meta_bytes=345344 combine_payload_bytes=267141120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267486464 ingress_bytes=301744128 link_bytes=301744128 token_bytes=534627584 weight_movement_bytes=34603008 total_nvlink_bytes=569230592 weight_to_token_ratio=0.064724 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135438861e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74656 payload_rows_remote=65310 dup_rows=23648 dup_rows_remote=20674 dedup_ratio=0.240560 router_tokens_per_expert=[743, 726, 770, 779, 768, 802, 750, 823, 789, 777, 809, 766, 763, 784, 752, 735, 747, 774, 752, 741, 769, 749, 790, 780, 767, 784, 786, 758, 739, 720, 774, 752, 799, 720, 786, 785, 791, 747, 783, 764, 788, 783, 750, 765, 795, 770, 761, 719, 784, 747, 790, 761, 753, 798, 753, 774, 821, 773, 749, 749, 781, 740, 751, 804] payload_rows_per_dst=[9378, 9342, 9314, 9218, 9402, 9298, 9346, 9358] dup_rows_per_dst=[2944, 3008, 2890, 2942, 2948, 2964, 2974, 2978] combine_rows_per_src=[9378, 9342, 9314, 9218, 9402, 9298, 9346, 9358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265461760 dispatch_meta_bytes=343080 combine_payload_bytes=265461760 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265804840 ingress_bytes=334667776 link_bytes=334667776 token_bytes=531266600 weight_movement_bytes=69206016 total_nvlink_bytes=600472616 weight_to_token_ratio=0.130266 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259327898e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=267206656 dispatch_meta_bytes=344160 combine_payload_bytes=267206656 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267550816 ingress_bytes=371015680 link_bytes=371015680 token_bytes=534757472 weight_movement_bytes=103809024 total_nvlink_bytes=638566496 weight_to_token_ratio=0.194124 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396102135e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=64912 dup_rows=24046 dup_rows_remote=20954 dedup_ratio=0.244609 router_tokens_per_expert=[790, 781, 777, 806, 749, 761, 751, 758, 762, 755, 757, 763, 767, 782, 779, 715, 779, 760, 747, 790, 765, 774, 798, 806, 762, 760, 747, 756, 740, 778, 744, 777, 765, 765, 745, 750, 757, 766, 744, 828, 746, 799, 764, 772, 789, 776, 778, 749, 796, 753, 739, 714, 746, 803, 798, 770, 756, 760, 788, 783, 790, 772, 784, 771] payload_rows_per_dst=[9288, 9222, 9346, 9164, 9238, 9310, 9280, 9410] dup_rows_per_dst=[3058, 2938, 3092, 2964, 3002, 3036, 2958, 2998] combine_rows_per_src=[9288, 9222, 9346, 9164, 9238, 9310, 9280, 9410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=65220 dup_rows=24010 dup_rows_remote=21116 dedup_ratio=0.244242 router_tokens_per_expert=[801, 766, 786, 723, 793, 739, 763, 730, 768, 769, 785, 775, 781, 774, 827, 767, 785, 781, 749, 768, 718, 751, 798, 772, 756, 712, 734, 761, 747, 787, 716, 771, 755, 772, 778, 750, 773, 775, 804, 784, 808, 770, 779, 757, 782, 738, 767, 782, 803, 737, 731, 763, 767, 804, 752, 760, 728, 778, 747, 796, 829, 750, 802, 778] payload_rows_per_dst=[9268, 9454, 9250, 9074, 9328, 9290, 9260, 9370] dup_rows_per_dst=[2934, 3038, 2994, 2894, 3054, 3076, 2974, 3046] combine_rows_per_src=[9268, 9454, 9250, 9074, 9328, 9290, 9260, 9370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265977856 dispatch_meta_bytes=343968 combine_payload_bytes=265977856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266321824 ingress_bytes=300580864 link_bytes=300580864 token_bytes=532299680 weight_movement_bytes=34603008 total_nvlink_bytes=566902688 weight_to_token_ratio=0.065007 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131061592e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74382 payload_rows_remote=65178 dup_rows=23922 dup_rows_remote=20934 dedup_ratio=0.243347 router_tokens_per_expert=[794, 799, 779, 763, 734, 780, 795, 783, 732, 792, 760, 765, 771, 815, 756, 809, 772, 753, 757, 753, 773, 780, 768, 740, 798, 761, 794, 770, 763, 757, 739, 743, 747, 778, 730, 740, 779, 747, 802, 753, 744, 737, 762, 749, 784, 755, 803, 769, 715, 773, 837, 834, 807, 779, 730, 749, 769, 763, 773, 767, 724, 786, 769, 750] payload_rows_per_dst=[9372, 9350, 9204, 9252, 9238, 9242, 9430, 9294] dup_rows_per_dst=[3082, 3050, 2988, 2998, 2914, 2964, 3018, 2908] combine_rows_per_src=[9372, 9350, 9204, 9252, 9238, 9242, 9430, 9294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266682368 dispatch_meta_bytes=343864 combine_payload_bytes=266682368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267026232 ingress_bytes=301285376 link_bytes=301285376 token_bytes=533708600 weight_movement_bytes=34603008 total_nvlink_bytes=568311608 weight_to_token_ratio=0.064835 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133712614e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74270 payload_rows_remote=64810 dup_rows=24034 dup_rows_remote=20960 dedup_ratio=0.244486 router_tokens_per_expert=[786, 824, 746, 802, 761, 796, 811, 779, 738, 709, 750, 789, 760, 730, 772, 748, 786, 765, 747, 746, 808, 788, 789, 803, 773, 834, 764, 779, 740, 771, 722, 735, 827, 782, 799, 816, 784, 791, 733, 735, 771, 750, 698, 827, 760, 733, 782, 752, 712, 783, 718, 759, 786, 780, 783, 763, 757, 778, 789, 803, 751, 762, 693, 744] payload_rows_per_dst=[9482, 9138, 9404, 9224, 9460, 9212, 9188, 9162] dup_rows_per_dst=[3128, 2854, 3060, 3012, 3074, 2934, 2980, 2992] combine_rows_per_src=[9482, 9138, 9404, 9224, 9460, 9212, 9188, 9162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=267206656 dispatch_meta_bytes=344160 combine_payload_bytes=267206656 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267550816 ingress_bytes=371015680 link_bytes=371015680 token_bytes=534757472 weight_movement_bytes=103809024 total_nvlink_bytes=638566496 weight_to_token_ratio=0.194124 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396102135e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266805248 dispatch_meta_bytes=344312 combine_payload_bytes=266805248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267149560 ingress_bytes=301408256 link_bytes=301408256 token_bytes=533954808 weight_movement_bytes=34603008 total_nvlink_bytes=568557816 weight_to_token_ratio=0.064805 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134175002e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265879552 dispatch_meta_bytes=343464 combine_payload_bytes=265879552 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266223016 ingress_bytes=404291584 link_bytes=404291584 token_bytes=532102568 weight_movement_bytes=138412032 total_nvlink_bytes=670514600 weight_to_token_ratio=0.260123 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521316682e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=267141120 dispatch_meta_bytes=345344 combine_payload_bytes=267141120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267486464 ingress_bytes=301744128 link_bytes=301744128 token_bytes=534627584 weight_movement_bytes=34603008 total_nvlink_bytes=569230592 weight_to_token_ratio=0.064724 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135438861e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265461760 dispatch_meta_bytes=343080 combine_payload_bytes=265461760 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265804840 ingress_bytes=334667776 link_bytes=334667776 token_bytes=531266600 weight_movement_bytes=69206016 total_nvlink_bytes=600472616 weight_to_token_ratio=0.130266 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259327898e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74078 payload_rows_remote=64892 dup_rows=24226 dup_rows_remote=21226 dedup_ratio=0.246440 router_tokens_per_expert=[783, 757, 767, 789, 754, 754, 806, 792, 765, 815, 724, 723, 798, 768, 774, 786, 756, 801, 753, 742, 794, 748, 786, 758, 777, 741, 765, 771, 747, 709, 777, 788, 757, 785, 778, 739, 781, 773, 744, 790, 768, 776, 759, 715, 789, 784, 805, 758, 763, 804, 787, 752, 754, 733, 765, 735, 801, 838, 766, 782, 767, 764, 759, 713] payload_rows_per_dst=[9318, 9280, 9230, 9188, 9294, 9282, 9186, 9300] dup_rows_per_dst=[3086, 3026, 3046, 2962, 3000, 3026, 3000, 3080] combine_rows_per_src=[9318, 9280, 9230, 9188, 9294, 9282, 9186, 9300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74070 payload_rows_remote=64816 dup_rows=24234 dup_rows_remote=21246 dedup_ratio=0.246521 router_tokens_per_expert=[755, 759, 767, 852, 784, 817, 755, 751, 798, 722, 803, 733, 781, 757, 760, 761, 773, 756, 783, 802, 765, 758, 745, 751, 762, 763, 773, 728, 781, 779, 736, 751, 742, 771, 742, 811, 776, 788, 768, 773, 780, 809, 829, 720, 768, 762, 771, 728, 788, 707, 798, 753, 791, 810, 743, 742, 762, 810, 756, 783, 738, 790, 761, 721] payload_rows_per_dst=[8974, 9224, 9236, 9162, 9248, 9668, 9304, 9254] dup_rows_per_dst=[2790, 3006, 3030, 2984, 3094, 3382, 2960, 2988] combine_rows_per_src=[8974, 9224, 9236, 9162, 9248, 9668, 9304, 9254] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74082 payload_rows_remote=64698 dup_rows=24222 dup_rows_remote=21142 dedup_ratio=0.246399 router_tokens_per_expert=[748, 763, 726, 796, 754, 764, 743, 738, 756, 809, 720, 820, 787, 769, 782, 789, 765, 752, 783, 781, 726, 738, 748, 773, 782, 778, 762, 779, 795, 770, 816, 770, 768, 777, 809, 735, 790, 757, 781, 775, 772, 751, 816, 787, 770, 724, 740, 778, 785, 728, 764, 794, 713, 762, 749, 789, 745, 765, 795, 776, 789, 732, 777, 777] payload_rows_per_dst=[9188, 9384, 9206, 9392, 9274, 9286, 9122, 9230] dup_rows_per_dst=[2876, 3080, 2926, 3112, 3110, 2990, 3046, 3082] combine_rows_per_src=[9188, 9384, 9206, 9392, 9274, 9286, 9122, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265797632 dispatch_meta_bytes=344472 combine_payload_bytes=265797632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266142104 ingress_bytes=300400640 link_bytes=300400640 token_bytes=531939736 weight_movement_bytes=34603008 total_nvlink_bytes=566542744 weight_to_token_ratio=0.065051 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130383424e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=64992 dup_rows=23964 dup_rows_remote=20944 dedup_ratio=0.243774 router_tokens_per_expert=[730, 786, 769, 794, 769, 773, 789, 742, 758, 749, 762, 755, 747, 771, 773, 768, 800, 745, 783, 749, 802, 765, 789, 775, 787, 808, 821, 806, 767, 771, 807, 759, 779, 784, 786, 738, 771, 721, 781, 761, 771, 733, 770, 755, 737, 730, 763, 738, 811, 755, 734, 757, 749, 731, 767, 784, 801, 783, 801, 758, 802, 724, 757, 751] payload_rows_per_dst=[9348, 9318, 9440, 9600, 9394, 9188, 9102, 8950] dup_rows_per_dst=[3020, 2998, 3134, 3328, 3016, 2806, 2866, 2796] combine_rows_per_src=[9348, 9318, 9440, 9600, 9394, 9188, 9102, 8950] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74214 payload_rows_remote=64994 dup_rows=24090 dup_rows_remote=21262 dedup_ratio=0.245056 router_tokens_per_expert=[811, 786, 806, 821, 801, 791, 741, 768, 763, 772, 732, 781, 781, 714, 778, 772, 779, 764, 765, 752, 777, 772, 771, 814, 729, 755, 780, 770, 766, 745, 754, 775, 756, 795, 776, 803, 740, 762, 721, 771, 746, 731, 736, 771, 766, 717, 784, 773, 793, 756, 808, 779, 746, 724, 745, 799, 756, 809, 765, 772, 766, 773, 759, 768] payload_rows_per_dst=[9378, 9174, 9322, 9238, 9264, 9220, 9284, 9334] dup_rows_per_dst=[3272, 3012, 3066, 2910, 2984, 2828, 3016, 3002] combine_rows_per_src=[9378, 9174, 9322, 9238, 9264, 9220, 9284, 9334] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267509760 dispatch_meta_bytes=343936 combine_payload_bytes=267509760 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267853696 ingress_bytes=371318784 link_bytes=371318784 token_bytes=535363456 weight_movement_bytes=103809024 total_nvlink_bytes=639172480 weight_to_token_ratio=0.193904 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397242691e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266969088 dispatch_meta_bytes=344448 combine_payload_bytes=266969088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267313536 ingress_bytes=301572096 link_bytes=301572096 token_bytes=534282624 weight_movement_bytes=34603008 total_nvlink_bytes=568885632 weight_to_token_ratio=0.064765 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134791519e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266215424 dispatch_meta_bytes=345024 combine_payload_bytes=266215424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266560448 ingress_bytes=300818432 link_bytes=300818432 token_bytes=532775872 weight_movement_bytes=34603008 total_nvlink_bytes=567378880 weight_to_token_ratio=0.064949 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131955542e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74382 payload_rows_remote=65178 dup_rows=23922 dup_rows_remote=20934 dedup_ratio=0.243347 router_tokens_per_expert=[794, 799, 779, 763, 734, 780, 795, 783, 732, 792, 760, 765, 771, 815, 756, 809, 772, 753, 757, 753, 773, 780, 768, 740, 798, 761, 794, 770, 763, 757, 739, 743, 747, 778, 730, 740, 779, 747, 802, 753, 744, 737, 762, 749, 784, 755, 803, 769, 715, 773, 837, 834, 807, 779, 730, 749, 769, 763, 773, 767, 724, 786, 769, 750] payload_rows_per_dst=[9372, 9350, 9204, 9252, 9238, 9242, 9430, 9294] dup_rows_per_dst=[3082, 3050, 2988, 2998, 2914, 2964, 3018, 2908] combine_rows_per_src=[9372, 9350, 9204, 9252, 9238, 9242, 9430, 9294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265486336 dispatch_meta_bytes=344248 combine_payload_bytes=265486336 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=265830584 ingress_bytes=403898368 link_bytes=403898368 token_bytes=531316920 weight_movement_bytes=138412032 total_nvlink_bytes=669728952 weight_to_token_ratio=0.260507 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519837042e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74180 payload_rows_remote=64974 dup_rows=24124 dup_rows_remote=21242 dedup_ratio=0.245402 router_tokens_per_expert=[789, 733, 772, 774, 725, 758, 717, 760, 764, 763, 736, 735, 759, 721, 826, 808, 770, 769, 807, 741, 791, 795, 789, 739, 756, 716, 740, 765, 763, 792, 788, 724, 755, 792, 788, 734, 799, 810, 755, 803, 775, 782, 713, 740, 772, 773, 745, 778, 823, 772, 791, 772, 779, 736, 769, 751, 797, 792, 799, 786, 805, 744, 793, 744] payload_rows_per_dst=[9136, 9250, 9300, 9206, 9366, 9102, 9392, 9428] dup_rows_per_dst=[2920, 2974, 3102, 2882, 3106, 3054, 2994, 3092] combine_rows_per_src=[9136, 9250, 9300, 9206, 9366, 9102, 9392, 9428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265003008 dispatch_meta_bytes=343360 combine_payload_bytes=265003008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265346368 ingress_bytes=299606016 link_bytes=299606016 token_bytes=530349376 weight_movement_bytes=34603008 total_nvlink_bytes=564952384 weight_to_token_ratio=0.065246 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127393318e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74450 payload_rows_remote=65194 dup_rows=23854 dup_rows_remote=20980 dedup_ratio=0.242655 router_tokens_per_expert=[819, 802, 781, 793, 792, 806, 773, 743, 762, 772, 718, 763, 748, 783, 804, 787, 812, 747, 752, 749, 740, 736, 748, 754, 775, 758, 720, 757, 733, 750, 752, 750, 738, 739, 774, 773, 785, 744, 768, 744, 813, 752, 793, 754, 718, 748, 756, 785, 791, 801, 825, 754, 776, 803, 781, 778, 811, 780, 753, 757, 742, 803, 775, 759] payload_rows_per_dst=[9464, 9326, 9178, 9148, 9256, 9312, 9462, 9304] dup_rows_per_dst=[3154, 2948, 2898, 2842, 2874, 2926, 3156, 3056] combine_rows_per_src=[9464, 9326, 9178, 9148, 9256, 9312, 9462, 9304] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=64992 dup_rows=23964 dup_rows_remote=20944 dedup_ratio=0.243774 router_tokens_per_expert=[730, 786, 769, 794, 769, 773, 789, 742, 758, 749, 762, 755, 747, 771, 773, 768, 800, 745, 783, 749, 802, 765, 789, 775, 787, 808, 821, 806, 767, 771, 807, 759, 779, 784, 786, 738, 771, 721, 781, 761, 771, 733, 770, 755, 737, 730, 763, 738, 811, 755, 734, 757, 749, 731, 767, 784, 801, 783, 801, 758, 802, 724, 757, 751] payload_rows_per_dst=[9348, 9318, 9440, 9600, 9394, 9188, 9102, 8950] dup_rows_per_dst=[3020, 2998, 3134, 3328, 3016, 2806, 2866, 2796] combine_rows_per_src=[9348, 9318, 9440, 9600, 9394, 9188, 9102, 8950] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74070 payload_rows_remote=64816 dup_rows=24234 dup_rows_remote=21246 dedup_ratio=0.246521 router_tokens_per_expert=[755, 759, 767, 852, 784, 817, 755, 751, 798, 722, 803, 733, 781, 757, 760, 761, 773, 756, 783, 802, 765, 758, 745, 751, 762, 763, 773, 728, 781, 779, 736, 751, 742, 771, 742, 811, 776, 788, 768, 773, 780, 809, 829, 720, 768, 762, 771, 728, 788, 707, 798, 753, 791, 810, 743, 742, 762, 810, 756, 783, 738, 790, 761, 721] payload_rows_per_dst=[8974, 9224, 9236, 9162, 9248, 9668, 9304, 9254] dup_rows_per_dst=[2790, 3006, 3030, 2984, 3094, 3382, 2960, 2988] combine_rows_per_src=[8974, 9224, 9236, 9162, 9248, 9668, 9304, 9254] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74082 payload_rows_remote=64698 dup_rows=24222 dup_rows_remote=21142 dedup_ratio=0.246399 router_tokens_per_expert=[748, 763, 726, 796, 754, 764, 743, 738, 756, 809, 720, 820, 787, 769, 782, 789, 765, 752, 783, 781, 726, 738, 748, 773, 782, 778, 762, 779, 795, 770, 816, 770, 768, 777, 809, 735, 790, 757, 781, 775, 772, 751, 816, 787, 770, 724, 740, 778, 785, 728, 764, 794, 713, 762, 749, 789, 745, 765, 795, 776, 789, 732, 777, 777] payload_rows_per_dst=[9188, 9384, 9206, 9392, 9274, 9286, 9122, 9230] dup_rows_per_dst=[2876, 3080, 2926, 3112, 3110, 2990, 3046, 3082] combine_rows_per_src=[9188, 9384, 9206, 9392, 9274, 9286, 9122, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266215424 dispatch_meta_bytes=345024 combine_payload_bytes=266215424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266560448 ingress_bytes=300818432 link_bytes=300818432 token_bytes=532775872 weight_movement_bytes=34603008 total_nvlink_bytes=567378880 weight_to_token_ratio=0.064949 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131955542e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266207232 dispatch_meta_bytes=343744 combine_payload_bytes=266207232 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266550976 ingress_bytes=335413248 link_bytes=335413248 token_bytes=532758208 weight_movement_bytes=69206016 total_nvlink_bytes=601964224 weight_to_token_ratio=0.129901 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262133049e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266969088 dispatch_meta_bytes=344448 combine_payload_bytes=266969088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267313536 ingress_bytes=301572096 link_bytes=301572096 token_bytes=534282624 weight_movement_bytes=34603008 total_nvlink_bytes=568885632 weight_to_token_ratio=0.064765 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134791519e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=344864 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266478368 ingress_bytes=300736512 link_bytes=300736512 token_bytes=532611872 weight_movement_bytes=34603008 total_nvlink_bytes=567214880 weight_to_token_ratio=0.064969 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131647283e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74146 payload_rows_remote=65008 dup_rows=24158 dup_rows_remote=21144 dedup_ratio=0.245748 router_tokens_per_expert=[745, 754, 780, 812, 765, 770, 806, 733, 739, 775, 767, 768, 779, 709, 784, 784, 792, 778, 754, 792, 776, 744, 733, 768, 753, 727, 797, 766, 761, 794, 756, 771, 773, 750, 773, 753, 787, 730, 741, 776, 787, 776, 812, 787, 739, 763, 773, 731, 766, 778, 765, 783, 692, 786, 719, 787, 765, 816, 765, 788, 784, 798, 796, 781] payload_rows_per_dst=[9258, 9228, 9310, 9336, 9130, 9318, 9138, 9428] dup_rows_per_dst=[3072, 2982, 2964, 2914, 3036, 3018, 3014, 3158] combine_rows_per_src=[9258, 9228, 9310, 9336, 9130, 9318, 9138, 9428] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=267034624 dispatch_meta_bytes=344696 combine_payload_bytes=267034624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267379320 ingress_bytes=301637632 link_bytes=301637632 token_bytes=534413944 weight_movement_bytes=34603008 total_nvlink_bytes=569016952 weight_to_token_ratio=0.064749 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135038125e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74078 payload_rows_remote=64892 dup_rows=24226 dup_rows_remote=21226 dedup_ratio=0.246440 router_tokens_per_expert=[783, 757, 767, 789, 754, 754, 806, 792, 765, 815, 724, 723, 798, 768, 774, 786, 756, 801, 753, 742, 794, 748, 786, 758, 777, 741, 765, 771, 747, 709, 777, 788, 757, 785, 778, 739, 781, 773, 744, 790, 768, 776, 759, 715, 789, 784, 805, 758, 763, 804, 787, 752, 754, 733, 765, 735, 801, 838, 766, 782, 767, 764, 759, 713] payload_rows_per_dst=[9318, 9280, 9230, 9188, 9294, 9282, 9186, 9300] dup_rows_per_dst=[3086, 3026, 3046, 2962, 3000, 3026, 3000, 3080] combine_rows_per_src=[9318, 9280, 9230, 9188, 9294, 9282, 9186, 9300] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266207232 dispatch_meta_bytes=343744 combine_payload_bytes=266207232 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266550976 ingress_bytes=335413248 link_bytes=335413248 token_bytes=532758208 weight_movement_bytes=69206016 total_nvlink_bytes=601964224 weight_to_token_ratio=0.129901 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262133049e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265486336 dispatch_meta_bytes=344248 combine_payload_bytes=265486336 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=265830584 ingress_bytes=403898368 link_bytes=403898368 token_bytes=531316920 weight_movement_bytes=138412032 total_nvlink_bytes=669728952 weight_to_token_ratio=0.260507 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519837042e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265003008 dispatch_meta_bytes=343360 combine_payload_bytes=265003008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265346368 ingress_bytes=299606016 link_bytes=299606016 token_bytes=530349376 weight_movement_bytes=34603008 total_nvlink_bytes=564952384 weight_to_token_ratio=0.065246 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127393318e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=65026 dup_rows=23920 dup_rows_remote=21038 dedup_ratio=0.243327 router_tokens_per_expert=[807, 753, 764, 764, 751, 802, 751, 753, 735, 830, 778, 738, 764, 732, 742, 787, 756, 769, 754, 785, 747, 760, 753, 796, 769, 777, 761, 780, 769, 826, 737, 758, 780, 767, 760, 780, 757, 727, 787, 744, 794, 783, 759, 755, 760, 738, 802, 752, 755, 746, 758, 760, 804, 813, 770, 814, 745, 746, 770, 799, 742, 744, 792, 801] payload_rows_per_dst=[9316, 9296, 9358, 9364, 9178, 9216, 9396, 9260] dup_rows_per_dst=[2974, 2916, 2882, 2990, 3026, 3070, 3044, 3018] combine_rows_per_src=[9316, 9296, 9358, 9364, 9178, 9216, 9396, 9260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=64934 dup_rows=23908 dup_rows_remote=20842 dedup_ratio=0.243205 router_tokens_per_expert=[789, 752, 733, 764, 758, 769, 779, 779, 800, 819, 774, 759, 749, 755, 778, 742, 734, 774, 795, 755, 813, 773, 754, 822, 732, 797, 718, 780, 718, 740, 796, 695, 714, 737, 744, 786, 750, 731, 776, 772, 782, 778, 734, 808, 763, 744, 784, 784, 792, 772, 797, 793, 769, 775, 775, 787, 786, 766, 784, 771, 764, 788, 763, 788] payload_rows_per_dst=[9320, 9284, 9374, 8890, 9316, 9408, 9342, 9462] dup_rows_per_dst=[2926, 2910, 3062, 2640, 3082, 3040, 3182, 3066] combine_rows_per_src=[9320, 9284, 9374, 8890, 9316, 9408, 9342, 9462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=65038 dup_rows=24136 dup_rows_remote=21094 dedup_ratio=0.245524 router_tokens_per_expert=[761, 721, 805, 699, 809, 758, 764, 732, 722, 759, 762, 796, 718, 792, 787, 750, 733, 783, 741, 759, 763, 736, 754, 826, 781, 794, 780, 790, 750, 728, 779, 798, 752, 805, 777, 740, 789, 751, 783, 785, 746, 747, 760, 780, 715, 764, 774, 848, 761, 715, 768, 755, 807, 775, 821, 783, 790, 793, 785, 739, 806, 760, 793, 755] payload_rows_per_dst=[9170, 9130, 9184, 9406, 9316, 9260, 9328, 9374] dup_rows_per_dst=[2928, 3042, 3006, 2994, 3048, 3008, 3042, 3068] combine_rows_per_src=[9170, 9130, 9184, 9406, 9316, 9260, 9328, 9374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266272768 dispatch_meta_bytes=344608 combine_payload_bytes=266272768 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266617376 ingress_bytes=370081792 link_bytes=370081792 token_bytes=532890144 weight_movement_bytes=103809024 total_nvlink_bytes=636699168 weight_to_token_ratio=0.194804 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392587989e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74362 payload_rows_remote=65188 dup_rows=23942 dup_rows_remote=20968 dedup_ratio=0.243551 router_tokens_per_expert=[773, 766, 728, 773, 795, 759, 754, 796, 745, 765, 767, 781, 723, 765, 755, 782, 787, 792, 766, 732, 774, 763, 785, 794, 777, 792, 751, 790, 770, 771, 786, 761, 731, 730, 799, 791, 787, 749, 792, 804, 763, 736, 808, 714, 761, 770, 743, 779, 786, 762, 785, 785, 776, 746, 784, 758, 740, 787, 778, 767, 765, 773, 724, 761] payload_rows_per_dst=[9276, 9318, 9318, 9306, 9326, 9174, 9420, 9224] dup_rows_per_dst=[3012, 2848, 3068, 3090, 3040, 2974, 2944, 2966] combine_rows_per_src=[9276, 9318, 9318, 9306, 9326, 9174, 9420, 9224] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74446 payload_rows_remote=65036 dup_rows=23858 dup_rows_remote=20750 dedup_ratio=0.242696 router_tokens_per_expert=[767, 733, 769, 784, 759, 788, 758, 753, 735, 827, 777, 770, 762, 748, 801, 750, 779, 788, 784, 773, 768, 770, 713, 743, 779, 782, 757, 780, 793, 741, 783, 789, 755, 805, 778, 778, 758, 767, 778, 796, 746, 803, 747, 786, 770, 756, 768, 769, 817, 701, 754, 776, 767, 697, 778, 747, 732, 763, 740, 808, 786, 772, 756, 795] payload_rows_per_dst=[9410, 9330, 9284, 9384, 9424, 9378, 9178, 9058] dup_rows_per_dst=[3108, 3032, 2952, 3024, 3006, 2912, 2874, 2950] combine_rows_per_src=[9410, 9330, 9284, 9384, 9424, 9378, 9178, 9058] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=65026 dup_rows=23920 dup_rows_remote=21038 dedup_ratio=0.243327 router_tokens_per_expert=[807, 753, 764, 764, 751, 802, 751, 753, 735, 830, 778, 738, 764, 732, 742, 787, 756, 769, 754, 785, 747, 760, 753, 796, 769, 777, 761, 780, 769, 826, 737, 758, 780, 767, 760, 780, 757, 727, 787, 744, 794, 783, 759, 755, 760, 738, 802, 752, 755, 746, 758, 760, 804, 813, 770, 814, 745, 746, 770, 799, 742, 744, 792, 801] payload_rows_per_dst=[9316, 9296, 9358, 9364, 9178, 9216, 9396, 9260] dup_rows_per_dst=[2974, 2916, 2882, 2990, 3026, 3070, 3044, 3018] combine_rows_per_src=[9316, 9296, 9358, 9364, 9178, 9216, 9396, 9260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265969664 dispatch_meta_bytes=343104 combine_payload_bytes=265969664 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266312768 ingress_bytes=335175680 link_bytes=335175680 token_bytes=532282432 weight_movement_bytes=69206016 total_nvlink_bytes=601488448 weight_to_token_ratio=0.130017 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261239100e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=344528 combine_payload_bytes=266395648 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266740176 ingress_bytes=370204672 link_bytes=370204672 token_bytes=533135824 weight_movement_bytes=103809024 total_nvlink_bytes=636944848 weight_to_token_ratio=0.194714 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393050377e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265797632 dispatch_meta_bytes=344472 combine_payload_bytes=265797632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266142104 ingress_bytes=300400640 link_bytes=300400640 token_bytes=531939736 weight_movement_bytes=34603008 total_nvlink_bytes=566542744 weight_to_token_ratio=0.065051 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130383424e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266346496 dispatch_meta_bytes=344256 combine_payload_bytes=266346496 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266690752 ingress_bytes=300949504 link_bytes=300949504 token_bytes=533037248 weight_movement_bytes=34603008 total_nvlink_bytes=567640256 weight_to_token_ratio=0.064917 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132448755e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=267010048 dispatch_meta_bytes=344624 combine_payload_bytes=267010048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267354672 ingress_bytes=301613056 link_bytes=301613056 token_bytes=534364720 weight_movement_bytes=34603008 total_nvlink_bytes=568967728 weight_to_token_ratio=0.064755 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134945648e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73996 payload_rows_remote=64724 dup_rows=24308 dup_rows_remote=21244 dedup_ratio=0.247274 router_tokens_per_expert=[769, 788, 738, 715, 767, 798, 795, 766, 751, 778, 792, 777, 748, 794, 766, 723, 747, 773, 808, 777, 757, 719, 742, 733, 799, 792, 833, 755, 741, 715, 782, 751, 767, 823, 787, 746, 781, 754, 758, 771, 749, 792, 761, 820, 738, 792, 758, 766, 762, 786, 732, 751, 785, 789, 801, 777, 769, 793, 783, 750, 757, 792, 728, 745] payload_rows_per_dst=[9258, 9222, 9166, 9272, 9264, 9280, 9334, 9200] dup_rows_per_dst=[3014, 3036, 2946, 3064, 3110, 3072, 3032, 3034] combine_rows_per_src=[9258, 9222, 9166, 9272, 9264, 9280, 9334, 9200] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74446 payload_rows_remote=65036 dup_rows=23858 dup_rows_remote=20750 dedup_ratio=0.242696 router_tokens_per_expert=[767, 733, 769, 784, 759, 788, 758, 753, 735, 827, 777, 770, 762, 748, 801, 750, 779, 788, 784, 773, 768, 770, 713, 743, 779, 782, 757, 780, 793, 741, 783, 789, 755, 805, 778, 778, 758, 767, 778, 796, 746, 803, 747, 786, 770, 756, 768, 769, 817, 701, 754, 776, 767, 697, 778, 747, 732, 763, 740, 808, 786, 772, 756, 795] payload_rows_per_dst=[9410, 9330, 9284, 9384, 9424, 9378, 9178, 9058] dup_rows_per_dst=[3108, 3032, 2952, 3024, 3006, 2912, 2874, 2950] combine_rows_per_src=[9410, 9330, 9284, 9384, 9424, 9378, 9178, 9058] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=65112 dup_rows=23928 dup_rows_remote=21074 dedup_ratio=0.243408 router_tokens_per_expert=[779, 737, 754, 799, 790, 770, 705, 779, 778, 796, 763, 818, 783, 786, 750, 768, 731, 749, 790, 796, 744, 795, 802, 771, 789, 801, 821, 766, 732, 783, 783, 757, 759, 746, 794, 781, 766, 760, 716, 737, 765, 762, 786, 755, 752, 796, 766, 725, 761, 734, 744, 772, 785, 768, 755, 712, 769, 757, 806, 763, 778, 778, 764, 775] payload_rows_per_dst=[9232, 9424, 9346, 9378, 9264, 9252, 9142, 9338] dup_rows_per_dst=[2994, 3060, 3010, 3086, 2854, 2962, 2920, 3042] combine_rows_per_src=[9232, 9424, 9346, 9378, 9264, 9252, 9142, 9338] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=64934 dup_rows=23908 dup_rows_remote=20842 dedup_ratio=0.243205 router_tokens_per_expert=[789, 752, 733, 764, 758, 769, 779, 779, 800, 819, 774, 759, 749, 755, 778, 742, 734, 774, 795, 755, 813, 773, 754, 822, 732, 797, 718, 780, 718, 740, 796, 695, 714, 737, 744, 786, 750, 731, 776, 772, 782, 778, 734, 808, 763, 744, 784, 784, 792, 772, 797, 793, 769, 775, 775, 787, 786, 766, 784, 771, 764, 788, 763, 788] payload_rows_per_dst=[9320, 9284, 9374, 8890, 9316, 9408, 9342, 9462] dup_rows_per_dst=[2926, 2910, 3062, 2640, 3082, 3040, 3182, 3066] combine_rows_per_src=[9320, 9284, 9374, 8890, 9316, 9408, 9342, 9462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=267010048 dispatch_meta_bytes=344624 combine_payload_bytes=267010048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267354672 ingress_bytes=301613056 link_bytes=301613056 token_bytes=534364720 weight_movement_bytes=34603008 total_nvlink_bytes=568967728 weight_to_token_ratio=0.064755 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134945648e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=65038 dup_rows=24136 dup_rows_remote=21094 dedup_ratio=0.245524 router_tokens_per_expert=[761, 721, 805, 699, 809, 758, 764, 732, 722, 759, 762, 796, 718, 792, 787, 750, 733, 783, 741, 759, 763, 736, 754, 826, 781, 794, 780, 790, 750, 728, 779, 798, 752, 805, 777, 740, 789, 751, 783, 785, 746, 747, 760, 780, 715, 764, 774, 848, 761, 715, 768, 755, 807, 775, 821, 783, 790, 793, 785, 739, 806, 760, 793, 755] payload_rows_per_dst=[9170, 9130, 9184, 9406, 9316, 9260, 9328, 9374] dup_rows_per_dst=[2928, 3042, 3006, 2994, 3048, 3008, 3042, 3068] combine_rows_per_src=[9170, 9130, 9184, 9406, 9316, 9260, 9328, 9374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266387456 dispatch_meta_bytes=343144 combine_payload_bytes=266387456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266730600 ingress_bytes=300990464 link_bytes=300990464 token_bytes=533118056 weight_movement_bytes=34603008 total_nvlink_bytes=567721064 weight_to_token_ratio=0.064907 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132602884e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266346496 dispatch_meta_bytes=344256 combine_payload_bytes=266346496 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266690752 ingress_bytes=300949504 link_bytes=300949504 token_bytes=533037248 weight_movement_bytes=34603008 total_nvlink_bytes=567640256 weight_to_token_ratio=0.064917 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132448755e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=65144 dup_rows=23982 dup_rows_remote=21034 dedup_ratio=0.243958 router_tokens_per_expert=[738, 800, 752, 764, 794, 784, 761, 790, 790, 775, 805, 770, 781, 762, 813, 732, 760, 738, 773, 775, 765, 744, 788, 831, 791, 776, 820, 780, 768, 750, 768, 755, 742, 774, 759, 781, 764, 727, 771, 707, 746, 760, 775, 791, 770, 777, 771, 759, 753, 768, 744, 814, 727, 764, 749, 744, 719, 747, 795, 758, 779, 773, 808, 743] payload_rows_per_dst=[9290, 9412, 9378, 9388, 9188, 9302, 9178, 9186] dup_rows_per_dst=[3076, 3044, 2970, 3028, 2862, 2996, 2948, 3058] combine_rows_per_src=[9290, 9412, 9378, 9388, 9188, 9302, 9178, 9186] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265109504 dispatch_meta_bytes=343872 combine_payload_bytes=265109504 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265453376 ingress_bytes=368918528 link_bytes=368918528 token_bytes=530562880 weight_movement_bytes=103809024 total_nvlink_bytes=634371904 weight_to_token_ratio=0.195658 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388210720e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266387456 dispatch_meta_bytes=343144 combine_payload_bytes=266387456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266730600 ingress_bytes=300990464 link_bytes=300990464 token_bytes=533118056 weight_movement_bytes=34603008 total_nvlink_bytes=567721064 weight_to_token_ratio=0.064907 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132602884e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265969664 dispatch_meta_bytes=343104 combine_payload_bytes=265969664 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266312768 ingress_bytes=335175680 link_bytes=335175680 token_bytes=532282432 weight_movement_bytes=69206016 total_nvlink_bytes=601488448 weight_to_token_ratio=0.130017 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261239100e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74146 payload_rows_remote=65008 dup_rows=24158 dup_rows_remote=21144 dedup_ratio=0.245748 router_tokens_per_expert=[745, 754, 780, 812, 765, 770, 806, 733, 739, 775, 767, 768, 779, 709, 784, 784, 792, 778, 754, 792, 776, 744, 733, 768, 753, 727, 797, 766, 761, 794, 756, 771, 773, 750, 773, 753, 787, 730, 741, 776, 787, 776, 812, 787, 739, 763, 773, 731, 766, 778, 765, 783, 692, 786, 719, 787, 765, 816, 765, 788, 784, 798, 796, 781] payload_rows_per_dst=[9258, 9228, 9310, 9336, 9130, 9318, 9138, 9428] dup_rows_per_dst=[3072, 2982, 2964, 2914, 3036, 3018, 3014, 3158] combine_rows_per_src=[9258, 9228, 9310, 9336, 9130, 9318, 9138, 9428] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266698752 dispatch_meta_bytes=344744 combine_payload_bytes=266698752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267043496 ingress_bytes=301301760 link_bytes=301301760 token_bytes=533742248 weight_movement_bytes=34603008 total_nvlink_bytes=568345256 weight_to_token_ratio=0.064831 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133774266e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=344528 combine_payload_bytes=266395648 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266740176 ingress_bytes=370204672 link_bytes=370204672 token_bytes=533135824 weight_movement_bytes=103809024 total_nvlink_bytes=636944848 weight_to_token_ratio=0.194714 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393050377e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=64890 dup_rows=24048 dup_rows_remote=21036 dedup_ratio=0.244629 router_tokens_per_expert=[731, 736, 761, 761, 738, 768, 751, 757, 770, 757, 775, 789, 769, 800, 781, 754, 752, 763, 750, 759, 779, 812, 782, 792, 776, 716, 773, 724, 751, 786, 783, 735, 769, 773, 752, 776, 764, 775, 777, 793, 775, 771, 810, 765, 776, 757, 772, 782, 792, 758, 769, 713, 777, 749, 770, 784, 790, 783, 786, 822, 743, 747, 761, 790] payload_rows_per_dst=[9148, 9344, 9366, 9124, 9330, 9296, 9300, 9348] dup_rows_per_dst=[2858, 3046, 3012, 2964, 3028, 3120, 2924, 3096] combine_rows_per_src=[9148, 9344, 9366, 9124, 9330, 9296, 9300, 9348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=65102 dup_rows=23962 dup_rows_remote=20952 dedup_ratio=0.243754 router_tokens_per_expert=[745, 737, 750, 776, 790, 748, 774, 823, 756, 746, 805, 767, 784, 778, 755, 734, 819, 813, 727, 781, 722, 728, 757, 738, 770, 770, 771, 761, 808, 812, 832, 760, 775, 762, 789, 793, 754, 770, 760, 762, 751, 801, 767, 733, 735, 785, 746, 762, 806, 785, 812, 758, 795, 742, 758, 760, 733, 793, 745, 752, 804, 740, 735, 752] payload_rows_per_dst=[9278, 9240, 9242, 9410, 9284, 9262, 9358, 9268] dup_rows_per_dst=[3008, 3010, 2928, 3158, 3046, 2898, 3074, 2840] combine_rows_per_src=[9278, 9240, 9242, 9410, 9284, 9262, 9358, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73990 payload_rows_remote=64758 dup_rows=24314 dup_rows_remote=21276 dedup_ratio=0.247335 router_tokens_per_expert=[787, 748, 804, 767, 763, 798, 743, 793, 744, 756, 767, 747, 715, 767, 771, 766, 770, 794, 812, 743, 744, 765, 709, 736, 776, 775, 732, 747, 802, 852, 712, 775, 740, 809, 765, 789, 768, 733, 787, 743, 773, 739, 778, 776, 756, 793, 805, 801, 765, 783, 804, 757, 759, 783, 788, 743, 797, 742, 707, 814, 784, 737, 773, 781] payload_rows_per_dst=[9284, 8850, 9356, 9046, 9000, 9754, 9468, 9232] dup_rows_per_dst=[3122, 2786, 3220, 2924, 2568, 3388, 3268, 3038] combine_rows_per_src=[9284, 8850, 9356, 9046, 9000, 9754, 9468, 9232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266829824 dispatch_meta_bytes=344712 combine_payload_bytes=266829824 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267174536 ingress_bytes=336035840 link_bytes=336035840 token_bytes=534004360 weight_movement_bytes=69206016 total_nvlink_bytes=603210376 weight_to_token_ratio=0.129598 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264475813e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74368 payload_rows_remote=65260 dup_rows=23936 dup_rows_remote=21072 dedup_ratio=0.243490 router_tokens_per_expert=[787, 735, 785, 783, 759, 736, 772, 779, 777, 743, 742, 798, 766, 748, 749, 775, 772, 782, 802, 748, 756, 830, 760, 790, 738, 777, 792, 790, 755, 761, 775, 817, 798, 772, 778, 776, 693, 822, 793, 792, 816, 734, 736, 753, 774, 711, 728, 734, 750, 787, 725, 723, 743, 772, 761, 809, 777, 782, 753, 774, 764, 804, 770, 769] payload_rows_per_dst=[9308, 9284, 9398, 9310, 9354, 9108, 9238, 9368] dup_rows_per_dst=[2964, 2912, 3082, 3100, 3094, 2864, 2902, 3018] combine_rows_per_src=[9308, 9284, 9398, 9310, 9354, 9108, 9238, 9368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266272768 dispatch_meta_bytes=344608 combine_payload_bytes=266272768 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266617376 ingress_bytes=370081792 link_bytes=370081792 token_bytes=532890144 weight_movement_bytes=103809024 total_nvlink_bytes=636699168 weight_to_token_ratio=0.194804 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392587989e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265789440 dispatch_meta_bytes=343704 combine_payload_bytes=265789440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266133144 ingress_bytes=300392448 link_bytes=300392448 token_bytes=531922584 weight_movement_bytes=34603008 total_nvlink_bytes=566525592 weight_to_token_ratio=0.065053 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130352598e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267304960 dispatch_meta_bytes=345328 combine_payload_bytes=267304960 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267650288 ingress_bytes=301907968 link_bytes=301907968 token_bytes=534955248 weight_movement_bytes=34603008 total_nvlink_bytes=569558256 weight_to_token_ratio=0.064684 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136055378e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=64890 dup_rows=24048 dup_rows_remote=21036 dedup_ratio=0.244629 router_tokens_per_expert=[731, 736, 761, 761, 738, 768, 751, 757, 770, 757, 775, 789, 769, 800, 781, 754, 752, 763, 750, 759, 779, 812, 782, 792, 776, 716, 773, 724, 751, 786, 783, 735, 769, 773, 752, 776, 764, 775, 777, 793, 775, 771, 810, 765, 776, 757, 772, 782, 792, 758, 769, 713, 777, 749, 770, 784, 790, 783, 786, 822, 743, 747, 761, 790] payload_rows_per_dst=[9148, 9344, 9366, 9124, 9330, 9296, 9300, 9348] dup_rows_per_dst=[2858, 3046, 3012, 2964, 3028, 3120, 2924, 3096] combine_rows_per_src=[9148, 9344, 9366, 9124, 9330, 9296, 9300, 9348] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74260 payload_rows_remote=65010 dup_rows=24044 dup_rows_remote=21108 dedup_ratio=0.244588 router_tokens_per_expert=[704, 761, 803, 723, 771, 746, 802, 773, 794, 810, 773, 827, 778, 774, 798, 763, 785, 697, 794, 773, 741, 731, 808, 750, 733, 793, 759, 835, 817, 781, 770, 741, 855, 815, 749, 793, 758, 805, 775, 764, 770, 744, 766, 771, 751, 749, 751, 753, 767, 771, 771, 716, 718, 738, 771, 715, 766, 794, 751, 801, 733, 762, 722, 779] payload_rows_per_dst=[9250, 9430, 9188, 9350, 9562, 9216, 9184, 9080] dup_rows_per_dst=[2936, 3224, 2970, 3142, 3194, 2894, 2750, 2934] combine_rows_per_src=[9250, 9430, 9188, 9350, 9562, 9216, 9184, 9080] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266657792 dispatch_meta_bytes=344216 combine_payload_bytes=266657792 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267002008 ingress_bytes=335863808 link_bytes=335863808 token_bytes=533659800 weight_movement_bytes=69206016 total_nvlink_bytes=602865816 weight_to_token_ratio=0.129682 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263828470e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265248768 dispatch_meta_bytes=344136 combine_payload_bytes=265248768 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=265592904 ingress_bytes=403660800 link_bytes=403660800 token_bytes=530841672 weight_movement_bytes=138412032 total_nvlink_bytes=669253704 weight_to_token_ratio=0.260741 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.518943093e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74490 payload_rows_remote=65198 dup_rows=23814 dup_rows_remote=20924 dedup_ratio=0.242249 router_tokens_per_expert=[753, 760, 794, 768, 801, 779, 725, 748, 757, 787, 773, 751, 715, 837, 792, 763, 760, 782, 776, 752, 790, 805, 792, 734, 767, 801, 735, 750, 753, 767, 751, 767, 749, 802, 760, 812, 793, 753, 744, 718, 814, 766, 792, 721, 768, 771, 758, 703, 770, 722, 782, 784, 760, 781, 752, 763, 758, 752, 781, 797, 786, 780, 771, 804] payload_rows_per_dst=[9258, 9348, 9346, 9292, 9292, 9290, 9204, 9460] dup_rows_per_dst=[2998, 3002, 3036, 2890, 2970, 2896, 3024, 2998] combine_rows_per_src=[9258, 9348, 9346, 9292, 9292, 9290, 9204, 9460] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73990 payload_rows_remote=64758 dup_rows=24314 dup_rows_remote=21276 dedup_ratio=0.247335 router_tokens_per_expert=[787, 748, 804, 767, 763, 798, 743, 793, 744, 756, 767, 747, 715, 767, 771, 766, 770, 794, 812, 743, 744, 765, 709, 736, 776, 775, 732, 747, 802, 852, 712, 775, 740, 809, 765, 789, 768, 733, 787, 743, 773, 739, 778, 776, 756, 793, 805, 801, 765, 783, 804, 757, 759, 783, 788, 743, 797, 742, 707, 814, 784, 737, 773, 781] payload_rows_per_dst=[9284, 8850, 9356, 9046, 9000, 9754, 9468, 9232] dup_rows_per_dst=[3122, 2786, 3220, 2924, 2568, 3388, 3268, 3038] combine_rows_per_src=[9284, 8850, 9356, 9046, 9000, 9754, 9468, 9232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267304960 dispatch_meta_bytes=345328 combine_payload_bytes=267304960 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267650288 ingress_bytes=301907968 link_bytes=301907968 token_bytes=534955248 weight_movement_bytes=34603008 total_nvlink_bytes=569558256 weight_to_token_ratio=0.064684 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136055378e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74132 payload_rows_remote=64802 dup_rows=24172 dup_rows_remote=21092 dedup_ratio=0.245890 router_tokens_per_expert=[811, 754, 772, 756, 746, 774, 785, 774, 807, 810, 764, 743, 789, 788, 766, 803, 736, 740, 747, 749, 737, 756, 725, 778, 746, 796, 793, 763, 758, 768, 776, 718, 774, 777, 774, 788, 799, 767, 772, 754, 699, 787, 739, 712, 773, 750, 812, 781, 762, 815, 774, 781, 797, 759, 765, 786, 741, 777, 734, 774, 782, 811, 757, 751] payload_rows_per_dst=[9284, 9424, 9112, 9194, 9330, 9168, 9382, 9238] dup_rows_per_dst=[3060, 3116, 2824, 3042, 3080, 2938, 3096, 3016] combine_rows_per_src=[9284, 9424, 9112, 9194, 9330, 9168, 9382, 9238] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74132 payload_rows_remote=64802 dup_rows=24172 dup_rows_remote=21092 dedup_ratio=0.245890 router_tokens_per_expert=[811, 754, 772, 756, 746, 774, 785, 774, 807, 810, 764, 743, 789, 788, 766, 803, 736, 740, 747, 749, 737, 756, 725, 778, 746, 796, 793, 763, 758, 768, 776, 718, 774, 777, 774, 788, 799, 767, 772, 754, 699, 787, 739, 712, 773, 750, 812, 781, 762, 815, 774, 781, 797, 759, 765, 786, 741, 777, 734, 774, 782, 811, 757, 751] payload_rows_per_dst=[9284, 9424, 9112, 9194, 9330, 9168, 9382, 9238] dup_rows_per_dst=[3060, 3116, 2824, 3042, 3080, 2938, 3096, 3016] combine_rows_per_src=[9284, 9424, 9112, 9194, 9330, 9168, 9382, 9238] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=65102 dup_rows=23962 dup_rows_remote=20952 dedup_ratio=0.243754 router_tokens_per_expert=[745, 737, 750, 776, 790, 748, 774, 823, 756, 746, 805, 767, 784, 778, 755, 734, 819, 813, 727, 781, 722, 728, 757, 738, 770, 770, 771, 761, 808, 812, 832, 760, 775, 762, 789, 793, 754, 770, 760, 762, 751, 801, 767, 733, 735, 785, 746, 762, 806, 785, 812, 758, 795, 742, 758, 760, 733, 793, 745, 752, 804, 740, 735, 752] payload_rows_per_dst=[9278, 9240, 9242, 9410, 9284, 9262, 9358, 9268] dup_rows_per_dst=[3008, 3010, 2928, 3158, 3046, 2898, 3074, 2840] combine_rows_per_src=[9278, 9240, 9242, 9410, 9284, 9262, 9358, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265789440 dispatch_meta_bytes=343704 combine_payload_bytes=265789440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266133144 ingress_bytes=300392448 link_bytes=300392448 token_bytes=531922584 weight_movement_bytes=34603008 total_nvlink_bytes=566525592 weight_to_token_ratio=0.065053 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130352598e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266280960 dispatch_meta_bytes=344472 combine_payload_bytes=266280960 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266625432 ingress_bytes=300883968 link_bytes=300883968 token_bytes=532906392 weight_movement_bytes=34603008 total_nvlink_bytes=567509400 weight_to_token_ratio=0.064933 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132202148e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74678 payload_rows_remote=65408 dup_rows=23626 dup_rows_remote=20764 dedup_ratio=0.240336 router_tokens_per_expert=[848, 765, 767, 805, 779, 758, 765, 787, 766, 735, 815, 772, 748, 784, 725, 799, 769, 746, 755, 788, 763, 741, 797, 753, 783, 739, 771, 750, 772, 775, 777, 812, 743, 738, 754, 806, 757, 757, 783, 748, 743, 762, 758, 794, 777, 758, 742, 772, 723, 717, 776, 773, 786, 795, 755, 741, 740, 801, 731, 798, 785, 772, 759, 799] payload_rows_per_dst=[9474, 9376, 9282, 9452, 9338, 9184, 9270, 9302] dup_rows_per_dst=[3074, 2912, 2942, 2906, 2834, 3028, 2862, 3068] combine_rows_per_src=[9474, 9376, 9282, 9452, 9338, 9184, 9270, 9302] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267051008 dispatch_meta_bytes=344488 combine_payload_bytes=267051008 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267395496 ingress_bytes=336257024 link_bytes=336257024 token_bytes=534446504 weight_movement_bytes=69206016 total_nvlink_bytes=603652520 weight_to_token_ratio=0.129491 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265308110e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265248768 dispatch_meta_bytes=344136 combine_payload_bytes=265248768 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=265592904 ingress_bytes=403660800 link_bytes=403660800 token_bytes=530841672 weight_movement_bytes=138412032 total_nvlink_bytes=669253704 weight_to_token_ratio=0.260741 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.518943093e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=65144 dup_rows=23982 dup_rows_remote=21034 dedup_ratio=0.243958 router_tokens_per_expert=[738, 800, 752, 764, 794, 784, 761, 790, 790, 775, 805, 770, 781, 762, 813, 732, 760, 738, 773, 775, 765, 744, 788, 831, 791, 776, 820, 780, 768, 750, 768, 755, 742, 774, 759, 781, 764, 727, 771, 707, 746, 760, 775, 791, 770, 777, 771, 759, 753, 768, 744, 814, 727, 764, 749, 744, 719, 747, 795, 758, 779, 773, 808, 743] payload_rows_per_dst=[9290, 9412, 9378, 9388, 9188, 9302, 9178, 9186] dup_rows_per_dst=[3076, 3044, 2970, 3028, 2862, 2996, 2948, 3058] combine_rows_per_src=[9290, 9412, 9378, 9388, 9188, 9302, 9178, 9186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74328 payload_rows_remote=65224 dup_rows=23976 dup_rows_remote=20954 dedup_ratio=0.243896 router_tokens_per_expert=[825, 778, 725, 797, 734, 753, 760, 789, 718, 842, 767, 781, 740, 736, 755, 754, 755, 750, 757, 729, 795, 735, 765, 777, 776, 738, 797, 802, 749, 742, 768, 818, 753, 760, 772, 763, 768, 757, 758, 785, 805, 735, 790, 749, 749, 751, 747, 846, 785, 787, 780, 770, 758, 790, 792, 786, 746, 775, 757, 685, 771, 778, 789, 808] payload_rows_per_dst=[9268, 9290, 9104, 9366, 9278, 9350, 9438, 9234] dup_rows_per_dst=[3054, 2896, 3022, 3014, 2954, 2994, 3058, 2984] combine_rows_per_src=[9268, 9290, 9104, 9366, 9278, 9350, 9438, 9234] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74332 payload_rows_remote=64996 dup_rows=23972 dup_rows_remote=21066 dedup_ratio=0.243856 router_tokens_per_expert=[740, 760, 750, 756, 714, 768, 749, 770, 761, 753, 725, 764, 778, 814, 763, 716, 756, 768, 741, 798, 788, 826, 753, 748, 781, 777, 825, 758, 761, 719, 786, 796, 786, 765, 753, 788, 790, 816, 715, 758, 726, 803, 770, 733, 759, 791, 757, 782, 788, 809, 781, 813, 778, 746, 794, 728, 776, 746, 817, 786, 767, 747, 767, 755] payload_rows_per_dst=[9076, 9216, 9300, 9312, 9392, 9336, 9364, 9336] dup_rows_per_dst=[2938, 2932, 3056, 3094, 2950, 2906, 3110, 2986] combine_rows_per_src=[9076, 9216, 9300, 9312, 9392, 9336, 9364, 9336] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265428992 dispatch_meta_bytes=343576 combine_payload_bytes=265428992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265772568 ingress_bytes=334635008 link_bytes=334635008 token_bytes=531201560 weight_movement_bytes=69206016 total_nvlink_bytes=600407576 weight_to_token_ratio=0.130282 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259204595e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266657792 dispatch_meta_bytes=344216 combine_payload_bytes=266657792 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267002008 ingress_bytes=335863808 link_bytes=335863808 token_bytes=533659800 weight_movement_bytes=69206016 total_nvlink_bytes=602865816 weight_to_token_ratio=0.129682 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263828470e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74344 payload_rows_remote=65058 dup_rows=23960 dup_rows_remote=21062 dedup_ratio=0.243734 router_tokens_per_expert=[732, 767, 817, 761, 731, 812, 742, 823, 735, 746, 783, 812, 735, 760, 784, 737, 777, 732, 758, 716, 732, 768, 757, 787, 795, 740, 731, 793, 774, 749, 793, 764, 789, 769, 767, 764, 769, 795, 748, 725, 741, 790, 790, 798, 729, 781, 816, 764, 795, 770, 838, 773, 773, 727, 739, 816, 799, 739, 726, 791, 748, 763, 796, 781] payload_rows_per_dst=[9318, 9286, 9198, 9262, 9254, 9446, 9306, 9274] dup_rows_per_dst=[3052, 2898, 2856, 3016, 2998, 2972, 3156, 3012] combine_rows_per_src=[9318, 9286, 9198, 9262, 9254, 9446, 9306, 9274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267911168 dispatch_meta_bytes=344688 combine_payload_bytes=267911168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268255856 ingress_bytes=302514176 link_bytes=302514176 token_bytes=536167024 weight_movement_bytes=34603008 total_nvlink_bytes=570770032 weight_to_token_ratio=0.064538 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138336490e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74418 payload_rows_remote=64822 dup_rows=23886 dup_rows_remote=20678 dedup_ratio=0.242981 router_tokens_per_expert=[738, 768, 797, 757, 749, 805, 805, 740, 759, 778, 761, 754, 804, 779, 799, 749, 745, 811, 772, 781, 758, 733, 742, 781, 811, 789, 768, 756, 793, 797, 774, 792, 786, 720, 775, 717, 747, 751, 778, 764, 723, 763, 791, 807, 737, 784, 755, 759, 777, 759, 769, 789, 776, 786, 779, 751, 738, 768, 791, 728, 711, 798, 768, 762] payload_rows_per_dst=[9306, 9322, 9268, 9112, 9276, 9296, 9242, 9596] dup_rows_per_dst=[3012, 3044, 2978, 2772, 2800, 2942, 3130, 3208] combine_rows_per_src=[9306, 9322, 9268, 9112, 9276, 9296, 9242, 9596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74332 payload_rows_remote=64996 dup_rows=23972 dup_rows_remote=21066 dedup_ratio=0.243856 router_tokens_per_expert=[740, 760, 750, 756, 714, 768, 749, 770, 761, 753, 725, 764, 778, 814, 763, 716, 756, 768, 741, 798, 788, 826, 753, 748, 781, 777, 825, 758, 761, 719, 786, 796, 786, 765, 753, 788, 790, 816, 715, 758, 726, 803, 770, 733, 759, 791, 757, 782, 788, 809, 781, 813, 778, 746, 794, 728, 776, 746, 817, 786, 767, 747, 767, 755] payload_rows_per_dst=[9076, 9216, 9300, 9312, 9392, 9336, 9364, 9336] dup_rows_per_dst=[2938, 2932, 3056, 3094, 2950, 2906, 3110, 2986] combine_rows_per_src=[9076, 9216, 9300, 9312, 9392, 9336, 9364, 9336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74328 payload_rows_remote=65224 dup_rows=23976 dup_rows_remote=20954 dedup_ratio=0.243896 router_tokens_per_expert=[825, 778, 725, 797, 734, 753, 760, 789, 718, 842, 767, 781, 740, 736, 755, 754, 755, 750, 757, 729, 795, 735, 765, 777, 776, 738, 797, 802, 749, 742, 768, 818, 753, 760, 772, 763, 768, 757, 758, 785, 805, 735, 790, 749, 749, 751, 747, 846, 785, 787, 780, 770, 758, 790, 792, 786, 746, 775, 757, 685, 771, 778, 789, 808] payload_rows_per_dst=[9268, 9290, 9104, 9366, 9278, 9350, 9438, 9234] dup_rows_per_dst=[3054, 2896, 3022, 3014, 2954, 2994, 3058, 2984] combine_rows_per_src=[9268, 9290, 9104, 9366, 9278, 9350, 9438, 9234] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266829824 dispatch_meta_bytes=344712 combine_payload_bytes=266829824 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267174536 ingress_bytes=336035840 link_bytes=336035840 token_bytes=534004360 weight_movement_bytes=69206016 total_nvlink_bytes=603210376 weight_to_token_ratio=0.129598 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264475813e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=344480 combine_payload_bytes=266477568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266822048 ingress_bytes=301080576 link_bytes=301080576 token_bytes=533299616 weight_movement_bytes=34603008 total_nvlink_bytes=567902624 weight_to_token_ratio=0.064885 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132941969e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74026 payload_rows_remote=64458 dup_rows=24278 dup_rows_remote=20984 dedup_ratio=0.246969 router_tokens_per_expert=[784, 783, 796, 748, 771, 775, 743, 806, 729, 750, 760, 780, 765, 787, 760, 726, 715, 791, 830, 751, 787, 785, 745, 790, 781, 775, 743, 761, 788, 800, 740, 741, 762, 762, 763, 802, 752, 749, 812, 742, 791, 737, 766, 768, 758, 770, 805, 762, 733, 823, 820, 794, 733, 748, 786, 737, 758, 824, 731, 756, 775, 746, 758, 743] payload_rows_per_dst=[9568, 9496, 9552, 9216, 9240, 8914, 8910, 9130] dup_rows_per_dst=[3294, 3360, 3360, 2874, 3066, 2716, 2556, 3052] combine_rows_per_src=[9568, 9496, 9552, 9216, 9240, 8914, 8910, 9130] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=267157504 dispatch_meta_bytes=344712 combine_payload_bytes=267157504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267502216 ingress_bytes=301760512 link_bytes=301760512 token_bytes=534659720 weight_movement_bytes=34603008 total_nvlink_bytes=569262728 weight_to_token_ratio=0.064720 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135500513e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=344248 combine_payload_bytes=266223616 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266567864 ingress_bytes=370032640 link_bytes=370032640 token_bytes=532791480 weight_movement_bytes=103809024 total_nvlink_bytes=636600504 weight_to_token_ratio=0.194840 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392403034e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265510912 dispatch_meta_bytes=342000 combine_payload_bytes=265510912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265852912 ingress_bytes=300113920 link_bytes=300113920 token_bytes=531363824 weight_movement_bytes=34603008 total_nvlink_bytes=565966832 weight_to_token_ratio=0.065121 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129304520e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74580 payload_rows_remote=65232 dup_rows=23724 dup_rows_remote=20740 dedup_ratio=0.241333 router_tokens_per_expert=[769, 767, 762, 705, 764, 755, 758, 728, 752, 795, 747, 766, 795, 732, 763, 771, 813, 763, 735, 807, 783, 753, 718, 765, 785, 783, 736, 792, 751, 804, 771, 744, 723, 804, 762, 774, 768, 766, 743, 816, 804, 742, 786, 763, 745, 802, 765, 769, 778, 802, 742, 810, 822, 790, 785, 750, 800, 790, 751, 797, 745, 720, 762, 744] payload_rows_per_dst=[9206, 9370, 9346, 9348, 9276, 9400, 9430, 9204] dup_rows_per_dst=[2810, 2872, 2928, 2984, 3036, 2952, 3128, 3014] combine_rows_per_src=[9206, 9370, 9346, 9348, 9276, 9400, 9430, 9204] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74418 payload_rows_remote=64822 dup_rows=23886 dup_rows_remote=20678 dedup_ratio=0.242981 router_tokens_per_expert=[738, 768, 797, 757, 749, 805, 805, 740, 759, 778, 761, 754, 804, 779, 799, 749, 745, 811, 772, 781, 758, 733, 742, 781, 811, 789, 768, 756, 793, 797, 774, 792, 786, 720, 775, 717, 747, 751, 778, 764, 723, 763, 791, 807, 737, 784, 755, 759, 777, 759, 769, 789, 776, 786, 779, 751, 738, 768, 791, 728, 711, 798, 768, 762] payload_rows_per_dst=[9306, 9322, 9268, 9112, 9276, 9296, 9242, 9596] dup_rows_per_dst=[3012, 3044, 2978, 2772, 2800, 2942, 3130, 3208] combine_rows_per_src=[9306, 9322, 9268, 9112, 9276, 9296, 9242, 9596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=64866 dup_rows=23982 dup_rows_remote=20980 dedup_ratio=0.243958 router_tokens_per_expert=[757, 701, 789, 750, 776, 795, 736, 772, 741, 759, 777, 713, 759, 714, 820, 737, 762, 793, 765, 764, 747, 768, 790, 745, 777, 759, 792, 801, 771, 793, 802, 711, 774, 781, 810, 774, 787, 724, 787, 792, 804, 781, 785, 769, 810, 799, 778, 779, 776, 787, 739, 756, 752, 798, 721, 792, 769, 801, 735, 728, 749, 773, 726, 780] payload_rows_per_dst=[9270, 9178, 9188, 9356, 9456, 9426, 9264, 9184] dup_rows_per_dst=[2882, 2862, 3080, 3056, 3002, 3184, 2978, 2938] combine_rows_per_src=[9270, 9178, 9188, 9356, 9456, 9426, 9264, 9184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=344248 combine_payload_bytes=266223616 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266567864 ingress_bytes=370032640 link_bytes=370032640 token_bytes=532791480 weight_movement_bytes=103809024 total_nvlink_bytes=636600504 weight_to_token_ratio=0.194840 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392403034e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=267157504 dispatch_meta_bytes=344712 combine_payload_bytes=267157504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267502216 ingress_bytes=301760512 link_bytes=301760512 token_bytes=534659720 weight_movement_bytes=34603008 total_nvlink_bytes=569262728 weight_to_token_ratio=0.064720 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135500513e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=264019968 dispatch_meta_bytes=341768 combine_payload_bytes=264019968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264361736 ingress_bytes=298622976 link_bytes=298622976 token_bytes=528381704 weight_movement_bytes=34603008 total_nvlink_bytes=562984712 weight_to_token_ratio=0.065489 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123694218e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=64866 dup_rows=23982 dup_rows_remote=20980 dedup_ratio=0.243958 router_tokens_per_expert=[757, 701, 789, 750, 776, 795, 736, 772, 741, 759, 777, 713, 759, 714, 820, 737, 762, 793, 765, 764, 747, 768, 790, 745, 777, 759, 792, 801, 771, 793, 802, 711, 774, 781, 810, 774, 787, 724, 787, 792, 804, 781, 785, 769, 810, 799, 778, 779, 776, 787, 739, 756, 752, 798, 721, 792, 769, 801, 735, 728, 749, 773, 726, 780] payload_rows_per_dst=[9270, 9178, 9188, 9356, 9456, 9426, 9264, 9184] dup_rows_per_dst=[2882, 2862, 3080, 3056, 3002, 3184, 2978, 2938] combine_rows_per_src=[9270, 9178, 9188, 9356, 9456, 9426, 9264, 9184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74344 payload_rows_remote=65058 dup_rows=23960 dup_rows_remote=21062 dedup_ratio=0.243734 router_tokens_per_expert=[732, 767, 817, 761, 731, 812, 742, 823, 735, 746, 783, 812, 735, 760, 784, 737, 777, 732, 758, 716, 732, 768, 757, 787, 795, 740, 731, 793, 774, 749, 793, 764, 789, 769, 767, 764, 769, 795, 748, 725, 741, 790, 790, 798, 729, 781, 816, 764, 795, 770, 838, 773, 773, 727, 739, 816, 799, 739, 726, 791, 748, 763, 796, 781] payload_rows_per_dst=[9318, 9286, 9198, 9262, 9254, 9446, 9306, 9274] dup_rows_per_dst=[3052, 2898, 2856, 3016, 2998, 2972, 3156, 3012] combine_rows_per_src=[9318, 9286, 9198, 9262, 9254, 9446, 9306, 9274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=65000 dup_rows=23998 dup_rows_remote=21070 dedup_ratio=0.244120 router_tokens_per_expert=[746, 774, 732, 791, 755, 771, 805, 751, 804, 796, 762, 737, 764, 763, 762, 753, 745, 770, 741, 762, 714, 723, 778, 801, 793, 791, 758, 813, 774, 823, 765, 744, 794, 769, 789, 732, 759, 773, 764, 783, 787, 775, 760, 776, 811, 810, 746, 770, 843, 781, 756, 744, 730, 742, 746, 775, 761, 724, 765, 776, 787, 739, 737, 787] payload_rows_per_dst=[9198, 9216, 9142, 9404, 9356, 9474, 9306, 9210] dup_rows_per_dst=[3052, 3066, 2926, 3118, 2970, 2996, 2928, 2942] combine_rows_per_src=[9198, 9216, 9142, 9404, 9356, 9474, 9306, 9210] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=267190272 dispatch_meta_bytes=343888 combine_payload_bytes=267190272 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267534160 ingress_bytes=370999296 link_bytes=370999296 token_bytes=534724432 weight_movement_bytes=103809024 total_nvlink_bytes=638533456 weight_to_token_ratio=0.194136 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396040483e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265510912 dispatch_meta_bytes=342000 combine_payload_bytes=265510912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265852912 ingress_bytes=300113920 link_bytes=300113920 token_bytes=531363824 weight_movement_bytes=34603008 total_nvlink_bytes=565966832 weight_to_token_ratio=0.065121 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129304520e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74678 payload_rows_remote=65408 dup_rows=23626 dup_rows_remote=20764 dedup_ratio=0.240336 router_tokens_per_expert=[848, 765, 767, 805, 779, 758, 765, 787, 766, 735, 815, 772, 748, 784, 725, 799, 769, 746, 755, 788, 763, 741, 797, 753, 783, 739, 771, 750, 772, 775, 777, 812, 743, 738, 754, 806, 757, 757, 783, 748, 743, 762, 758, 794, 777, 758, 742, 772, 723, 717, 776, 773, 786, 795, 755, 741, 740, 801, 731, 798, 785, 772, 759, 799] payload_rows_per_dst=[9474, 9376, 9282, 9452, 9338, 9184, 9270, 9302] dup_rows_per_dst=[3074, 2912, 2942, 2906, 2834, 3028, 2862, 3068] combine_rows_per_src=[9474, 9376, 9282, 9452, 9338, 9184, 9270, 9302] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=65118 dup_rows=23964 dup_rows_remote=20928 dedup_ratio=0.243774 router_tokens_per_expert=[778, 778, 755, 778, 764, 706, 768, 733, 780, 749, 772, 753, 725, 771, 802, 753, 794, 797, 784, 757, 750, 727, 781, 739, 826, 758, 781, 735, 810, 743, 819, 766, 784, 769, 765, 788, 778, 755, 784, 811, 796, 804, 772, 750, 765, 770, 758, 731, 814, 759, 720, 758, 792, 781, 750, 760, 759, 781, 760, 780, 738, 796, 728, 764] payload_rows_per_dst=[9242, 9342, 9222, 9352, 9328, 9304, 9236, 9314] dup_rows_per_dst=[2878, 2868, 3036, 3124, 3140, 2988, 3032, 2898] combine_rows_per_src=[9242, 9342, 9222, 9352, 9328, 9304, 9236, 9314] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265691136 dispatch_meta_bytes=343384 combine_payload_bytes=265691136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266034520 ingress_bytes=300294144 link_bytes=300294144 token_bytes=531725656 weight_movement_bytes=34603008 total_nvlink_bytes=566328664 weight_to_token_ratio=0.065077 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129982688e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74484 payload_rows_remote=65166 dup_rows=23820 dup_rows_remote=20876 dedup_ratio=0.242310 router_tokens_per_expert=[762, 792, 766, 770, 749, 748, 768, 692, 776, 781, 791, 742, 763, 767, 783, 728, 829, 766, 755, 779, 848, 770, 726, 747, 708, 779, 785, 773, 774, 767, 789, 769, 806, 785, 805, 737, 761, 734, 754, 798, 783, 795, 765, 711, 748, 783, 754, 750, 786, 791, 782, 732, 816, 783, 740, 792, 785, 777, 745, 763, 765, 749, 776, 759] payload_rows_per_dst=[9276, 9318, 9450, 9218, 9380, 9262, 9300, 9280] dup_rows_per_dst=[2818, 2944, 2990, 3070, 2980, 2916, 3144, 2958] combine_rows_per_src=[9276, 9318, 9450, 9218, 9380, 9262, 9300, 9280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266240000 dispatch_meta_bytes=344280 combine_payload_bytes=266240000 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266584280 ingress_bytes=370049024 link_bytes=370049024 token_bytes=532824280 weight_movement_bytes=103809024 total_nvlink_bytes=636633304 weight_to_token_ratio=0.194828 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392464686e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74224 payload_rows_remote=64842 dup_rows=24080 dup_rows_remote=21034 dedup_ratio=0.244954 router_tokens_per_expert=[776, 754, 742, 829, 782, 774, 776, 794, 773, 798, 825, 781, 739, 758, 749, 720, 785, 781, 803, 780, 732, 768, 793, 775, 769, 812, 731, 771, 776, 727, 751, 773, 733, 726, 799, 747, 732, 773, 740, 798, 749, 747, 787, 727, 810, 778, 764, 751, 742, 790, 736, 778, 774, 763, 751, 763, 775, 768, 818, 777, 788, 720, 804, 747] payload_rows_per_dst=[9182, 9458, 9274, 9094, 8948, 9276, 9610, 9382] dup_rows_per_dst=[2986, 3114, 3160, 2756, 2720, 2950, 3348, 3046] combine_rows_per_src=[9182, 9458, 9274, 9094, 8948, 9276, 9610, 9382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74470 payload_rows_remote=65132 dup_rows=23834 dup_rows_remote=20880 dedup_ratio=0.242452 router_tokens_per_expert=[761, 776, 817, 806, 787, 719, 787, 760, 760, 760, 794, 763, 780, 726, 759, 775, 808, 766, 734, 794, 738, 754, 760, 778, 763, 734, 801, 750, 757, 777, 761, 802, 730, 787, 726, 772, 791, 777, 791, 803, 744, 775, 787, 736, 750, 775, 791, 788, 749, 768, 771, 724, 766, 780, 742, 743, 728, 738, 807, 811, 801, 767, 756, 771] payload_rows_per_dst=[9354, 9340, 9322, 9274, 9332, 9338, 9140, 9370] dup_rows_per_dst=[3072, 2894, 2942, 3016, 3022, 2954, 2946, 2988] combine_rows_per_src=[9354, 9340, 9322, 9274, 9332, 9338, 9140, 9370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=344480 combine_payload_bytes=266477568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266822048 ingress_bytes=301080576 link_bytes=301080576 token_bytes=533299616 weight_movement_bytes=34603008 total_nvlink_bytes=567902624 weight_to_token_ratio=0.064885 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132941969e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=65118 dup_rows=23964 dup_rows_remote=20928 dedup_ratio=0.243774 router_tokens_per_expert=[778, 778, 755, 778, 764, 706, 768, 733, 780, 749, 772, 753, 725, 771, 802, 753, 794, 797, 784, 757, 750, 727, 781, 739, 826, 758, 781, 735, 810, 743, 819, 766, 784, 769, 765, 788, 778, 755, 784, 811, 796, 804, 772, 750, 765, 770, 758, 731, 814, 759, 720, 758, 792, 781, 750, 760, 759, 781, 760, 780, 738, 796, 728, 764] payload_rows_per_dst=[9242, 9342, 9222, 9352, 9328, 9304, 9236, 9314] dup_rows_per_dst=[2878, 2868, 3036, 3124, 3140, 2988, 3032, 2898] combine_rows_per_src=[9242, 9342, 9222, 9352, 9328, 9304, 9236, 9314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266919936 dispatch_meta_bytes=344168 combine_payload_bytes=266919936 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267264104 ingress_bytes=301522944 link_bytes=301522944 token_bytes=534184040 weight_movement_bytes=34603008 total_nvlink_bytes=568787048 weight_to_token_ratio=0.064777 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134606564e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74492 payload_rows_remote=65164 dup_rows=23812 dup_rows_remote=20910 dedup_ratio=0.242228 router_tokens_per_expert=[751, 743, 805, 776, 746, 778, 787, 709, 782, 755, 812, 735, 740, 818, 753, 720, 785, 757, 807, 735, 752, 804, 766, 710, 787, 735, 743, 760, 759, 781, 782, 767, 722, 793, 768, 803, 795, 739, 774, 750, 764, 795, 802, 765, 780, 840, 795, 737, 785, 798, 737, 769, 772, 790, 776, 691, 760, 760, 770, 822, 760, 733, 785, 782] payload_rows_per_dst=[9328, 9280, 9310, 9268, 9296, 9476, 9234, 9300] dup_rows_per_dst=[2902, 2950, 3028, 2974, 2992, 3026, 3002, 2938] combine_rows_per_src=[9328, 9280, 9310, 9268, 9296, 9476, 9234, 9300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265592832 dispatch_meta_bytes=343504 combine_payload_bytes=265592832 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265936336 ingress_bytes=334798848 link_bytes=334798848 token_bytes=531529168 weight_movement_bytes=69206016 total_nvlink_bytes=600735184 weight_to_token_ratio=0.130202 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259821112e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=65052 dup_rows=23974 dup_rows_remote=20902 dedup_ratio=0.243876 router_tokens_per_expert=[766, 774, 781, 780, 755, 759, 805, 821, 750, 745, 747, 799, 819, 741, 762, 804, 739, 783, 759, 793, 763, 777, 758, 754, 769, 783, 771, 814, 774, 743, 782, 739, 737, 765, 766, 784, 768, 819, 763, 782, 738, 794, 752, 740, 762, 801, 745, 779, 749, 798, 782, 723, 736, 735, 752, 739, 777, 767, 796, 753, 719, 786, 762, 774] payload_rows_per_dst=[9410, 9334, 9272, 9278, 9308, 9300, 9154, 9274] dup_rows_per_dst=[3072, 3000, 2980, 3072, 3060, 2922, 2874, 2994] combine_rows_per_src=[9410, 9334, 9272, 9278, 9308, 9300, 9154, 9274] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74492 payload_rows_remote=65164 dup_rows=23812 dup_rows_remote=20910 dedup_ratio=0.242228 router_tokens_per_expert=[751, 743, 805, 776, 746, 778, 787, 709, 782, 755, 812, 735, 740, 818, 753, 720, 785, 757, 807, 735, 752, 804, 766, 710, 787, 735, 743, 760, 759, 781, 782, 767, 722, 793, 768, 803, 795, 739, 774, 750, 764, 795, 802, 765, 780, 840, 795, 737, 785, 798, 737, 769, 772, 790, 776, 691, 760, 760, 770, 822, 760, 733, 785, 782] payload_rows_per_dst=[9328, 9280, 9310, 9268, 9296, 9476, 9234, 9300] dup_rows_per_dst=[2902, 2950, 3028, 2974, 2992, 3026, 3002, 2938] combine_rows_per_src=[9328, 9280, 9310, 9268, 9296, 9476, 9234, 9300] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267911168 dispatch_meta_bytes=344688 combine_payload_bytes=267911168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268255856 ingress_bytes=302514176 link_bytes=302514176 token_bytes=536167024 weight_movement_bytes=34603008 total_nvlink_bytes=570770032 weight_to_token_ratio=0.064538 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138336490e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266723328 dispatch_meta_bytes=344184 combine_payload_bytes=266723328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267067512 ingress_bytes=301326336 link_bytes=301326336 token_bytes=533790840 weight_movement_bytes=34603008 total_nvlink_bytes=568393848 weight_to_token_ratio=0.064825 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133866744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266780672 dispatch_meta_bytes=344048 combine_payload_bytes=266780672 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267124720 ingress_bytes=335986688 link_bytes=335986688 token_bytes=533905392 weight_movement_bytes=69206016 total_nvlink_bytes=603111408 weight_to_token_ratio=0.129622 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264290858e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74154 payload_rows_remote=64908 dup_rows=24150 dup_rows_remote=21142 dedup_ratio=0.245667 router_tokens_per_expert=[829, 797, 762, 770, 754, 778, 752, 747, 791, 743, 758, 767, 811, 751, 759, 776, 766, 751, 721, 754, 785, 773, 748, 716, 783, 789, 807, 783, 765, 747, 790, 767, 777, 786, 773, 768, 763, 747, 733, 780, 778, 769, 729, 795, 768, 771, 802, 788, 782, 726, 739, 738, 796, 767, 762, 799, 744, 738, 765, 780, 787, 765, 767, 780] payload_rows_per_dst=[9316, 9228, 9152, 9374, 9246, 9310, 9292, 9236] dup_rows_per_dst=[3062, 3084, 2876, 3088, 3008, 3090, 2926, 3016] combine_rows_per_src=[9316, 9228, 9152, 9374, 9246, 9310, 9292, 9236] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=65052 dup_rows=23974 dup_rows_remote=20902 dedup_ratio=0.243876 router_tokens_per_expert=[766, 774, 781, 780, 755, 759, 805, 821, 750, 745, 747, 799, 819, 741, 762, 804, 739, 783, 759, 793, 763, 777, 758, 754, 769, 783, 771, 814, 774, 743, 782, 739, 737, 765, 766, 784, 768, 819, 763, 782, 738, 794, 752, 740, 762, 801, 745, 779, 749, 798, 782, 723, 736, 735, 752, 739, 777, 767, 796, 753, 719, 786, 762, 774] payload_rows_per_dst=[9410, 9334, 9272, 9278, 9308, 9300, 9154, 9274] dup_rows_per_dst=[3072, 3000, 2980, 3072, 3060, 2922, 2874, 2994] combine_rows_per_src=[9410, 9334, 9272, 9278, 9308, 9300, 9154, 9274] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74224 payload_rows_remote=64842 dup_rows=24080 dup_rows_remote=21034 dedup_ratio=0.244954 router_tokens_per_expert=[776, 754, 742, 829, 782, 774, 776, 794, 773, 798, 825, 781, 739, 758, 749, 720, 785, 781, 803, 780, 732, 768, 793, 775, 769, 812, 731, 771, 776, 727, 751, 773, 733, 726, 799, 747, 732, 773, 740, 798, 749, 747, 787, 727, 810, 778, 764, 751, 742, 790, 736, 778, 774, 763, 751, 763, 775, 768, 818, 777, 788, 720, 804, 747] payload_rows_per_dst=[9182, 9458, 9274, 9094, 8948, 9276, 9610, 9382] dup_rows_per_dst=[2986, 3114, 3160, 2756, 2720, 2950, 3348, 3046] combine_rows_per_src=[9182, 9458, 9274, 9094, 8948, 9276, 9610, 9382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266911744 dispatch_meta_bytes=344296 combine_payload_bytes=266911744 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267256040 ingress_bytes=336117760 link_bytes=336117760 token_bytes=534167784 weight_movement_bytes=69206016 total_nvlink_bytes=603373800 weight_to_token_ratio=0.129559 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264784071e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266780672 dispatch_meta_bytes=344048 combine_payload_bytes=266780672 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267124720 ingress_bytes=335986688 link_bytes=335986688 token_bytes=533905392 weight_movement_bytes=69206016 total_nvlink_bytes=603111408 weight_to_token_ratio=0.129622 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264290858e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=65074 dup_rows=24028 dup_rows_remote=21164 dedup_ratio=0.244425 router_tokens_per_expert=[758, 788, 804, 751, 787, 774, 782, 759, 785, 785, 761, 735, 781, 768, 733, 727, 768, 750, 790, 741, 775, 799, 758, 764, 770, 852, 801, 784, 701, 756, 728, 781, 757, 750, 780, 776, 776, 807, 807, 819, 780, 777, 744, 788, 749, 759, 757, 742, 716, 807, 783, 724, 813, 745, 709, 736, 763, 742, 762, 743, 770, 792, 773, 810] payload_rows_per_dst=[9244, 9236, 9258, 9306, 9452, 9284, 9202, 9294] dup_rows_per_dst=[3162, 2914, 3032, 3040, 3092, 2908, 2864, 3016] combine_rows_per_src=[9244, 9236, 9258, 9306, 9452, 9284, 9202, 9294] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74154 payload_rows_remote=64908 dup_rows=24150 dup_rows_remote=21142 dedup_ratio=0.245667 router_tokens_per_expert=[829, 797, 762, 770, 754, 778, 752, 747, 791, 743, 758, 767, 811, 751, 759, 776, 766, 751, 721, 754, 785, 773, 748, 716, 783, 789, 807, 783, 765, 747, 790, 767, 777, 786, 773, 768, 763, 747, 733, 780, 778, 769, 729, 795, 768, 771, 802, 788, 782, 726, 739, 738, 796, 767, 762, 799, 744, 738, 765, 780, 787, 765, 767, 780] payload_rows_per_dst=[9316, 9228, 9152, 9374, 9246, 9310, 9292, 9236] dup_rows_per_dst=[3062, 3084, 2876, 3088, 3008, 3090, 2926, 3016] combine_rows_per_src=[9316, 9228, 9152, 9374, 9246, 9310, 9292, 9236] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74484 payload_rows_remote=65166 dup_rows=23820 dup_rows_remote=20876 dedup_ratio=0.242310 router_tokens_per_expert=[762, 792, 766, 770, 749, 748, 768, 692, 776, 781, 791, 742, 763, 767, 783, 728, 829, 766, 755, 779, 848, 770, 726, 747, 708, 779, 785, 773, 774, 767, 789, 769, 806, 785, 805, 737, 761, 734, 754, 798, 783, 795, 765, 711, 748, 783, 754, 750, 786, 791, 782, 732, 816, 783, 740, 792, 785, 777, 745, 763, 765, 749, 776, 759] payload_rows_per_dst=[9276, 9318, 9450, 9218, 9380, 9262, 9300, 9280] dup_rows_per_dst=[2818, 2944, 2990, 3070, 2980, 2916, 3144, 2958] combine_rows_per_src=[9276, 9318, 9450, 9218, 9380, 9262, 9300, 9280] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266452992 dispatch_meta_bytes=343816 combine_payload_bytes=266452992 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266796808 ingress_bytes=370262016 link_bytes=370262016 token_bytes=533249800 weight_movement_bytes=103809024 total_nvlink_bytes=637058824 weight_to_token_ratio=0.194672 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393266158e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266723328 dispatch_meta_bytes=344184 combine_payload_bytes=266723328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267067512 ingress_bytes=301326336 link_bytes=301326336 token_bytes=533790840 weight_movement_bytes=34603008 total_nvlink_bytes=568393848 weight_to_token_ratio=0.064825 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133866744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265592832 dispatch_meta_bytes=343504 combine_payload_bytes=265592832 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265936336 ingress_bytes=334798848 link_bytes=334798848 token_bytes=531529168 weight_movement_bytes=69206016 total_nvlink_bytes=600735184 weight_to_token_ratio=0.130202 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259821112e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=65000 dup_rows=23998 dup_rows_remote=21070 dedup_ratio=0.244120 router_tokens_per_expert=[746, 774, 732, 791, 755, 771, 805, 751, 804, 796, 762, 737, 764, 763, 762, 753, 745, 770, 741, 762, 714, 723, 778, 801, 793, 791, 758, 813, 774, 823, 765, 744, 794, 769, 789, 732, 759, 773, 764, 783, 787, 775, 760, 776, 811, 810, 746, 770, 843, 781, 756, 744, 730, 742, 746, 775, 761, 724, 765, 776, 787, 739, 737, 787] payload_rows_per_dst=[9198, 9216, 9142, 9404, 9356, 9474, 9306, 9210] dup_rows_per_dst=[3052, 3066, 2926, 3118, 2970, 2996, 2928, 2942] combine_rows_per_src=[9198, 9216, 9142, 9404, 9356, 9474, 9306, 9210] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=65078 dup_rows=23946 dup_rows_remote=20898 dedup_ratio=0.243591 router_tokens_per_expert=[781, 719, 785, 704, 762, 746, 779, 786, 768, 738, 737, 790, 774, 743, 774, 734, 763, 770, 758, 779, 791, 785, 755, 763, 742, 788, 816, 757, 750, 787, 770, 740, 774, 796, 775, 754, 756, 742, 744, 792, 785, 750, 764, 814, 746, 757, 758, 772, 775, 781, 778, 787, 735, 787, 748, 763, 801, 789, 828, 765, 736, 821, 753, 792] payload_rows_per_dst=[9192, 9176, 9280, 9336, 9294, 9310, 9290, 9480] dup_rows_per_dst=[2932, 2940, 3048, 2964, 2972, 2982, 3018, 3090] combine_rows_per_src=[9192, 9176, 9280, 9336, 9294, 9310, 9290, 9480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266911744 dispatch_meta_bytes=344296 combine_payload_bytes=266911744 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267256040 ingress_bytes=336117760 link_bytes=336117760 token_bytes=534167784 weight_movement_bytes=69206016 total_nvlink_bytes=603373800 weight_to_token_ratio=0.129559 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264784071e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266452992 dispatch_meta_bytes=343816 combine_payload_bytes=266452992 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266796808 ingress_bytes=370262016 link_bytes=370262016 token_bytes=533249800 weight_movement_bytes=103809024 total_nvlink_bytes=637058824 weight_to_token_ratio=0.194672 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393266158e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265863168 dispatch_meta_bytes=344200 combine_payload_bytes=265863168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266207368 ingress_bytes=335069184 link_bytes=335069184 token_bytes=532070536 weight_movement_bytes=69206016 total_nvlink_bytes=601276552 weight_to_token_ratio=0.130069 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260838364e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74370 payload_rows_remote=65178 dup_rows=23934 dup_rows_remote=20990 dedup_ratio=0.243469 router_tokens_per_expert=[755, 805, 754, 771, 785, 779, 731, 776, 780, 714, 725, 768, 789, 760, 739, 793, 780, 776, 837, 768, 768, 727, 786, 749, 780, 766, 756, 746, 702, 777, 738, 797, 722, 775, 830, 778, 751, 760, 768, 786, 815, 760, 752, 782, 767, 736, 774, 738, 755, 784, 780, 755, 791, 763, 772, 806, 756, 804, 773, 788, 767, 769, 744, 774] payload_rows_per_dst=[9306, 9192, 9330, 9216, 9330, 9316, 9320, 9360] dup_rows_per_dst=[3006, 2944, 3052, 2908, 3010, 2932, 3092, 2990] combine_rows_per_src=[9306, 9192, 9330, 9216, 9330, 9316, 9320, 9360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74124 payload_rows_remote=64900 dup_rows=24180 dup_rows_remote=21008 dedup_ratio=0.245972 router_tokens_per_expert=[793, 735, 798, 768, 757, 737, 760, 811, 756, 790, 755, 826, 768, 731, 806, 759, 771, 716, 794, 793, 772, 690, 773, 775, 759, 743, 794, 748, 783, 790, 781, 784, 795, 752, 802, 763, 797, 785, 748, 789, 739, 754, 754, 746, 750, 797, 774, 774, 735, 778, 775, 727, 745, 807, 781, 731, 747, 767, 763, 768, 805, 719, 825, 744] payload_rows_per_dst=[9286, 9340, 9216, 9224, 9388, 9142, 9304, 9224] dup_rows_per_dst=[3006, 3042, 2918, 3140, 2978, 2858, 3066, 3172] combine_rows_per_src=[9286, 9340, 9216, 9224, 9388, 9142, 9304, 9224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=344952 combine_payload_bytes=266543104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266888056 ingress_bytes=301146112 link_bytes=301146112 token_bytes=533431160 weight_movement_bytes=34603008 total_nvlink_bytes=568034168 weight_to_token_ratio=0.064869 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133188575e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266969088 dispatch_meta_bytes=344672 combine_payload_bytes=266969088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267313760 ingress_bytes=301572096 link_bytes=301572096 token_bytes=534282848 weight_movement_bytes=34603008 total_nvlink_bytes=568885856 weight_to_token_ratio=0.064765 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134791519e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265830400 dispatch_meta_bytes=343632 combine_payload_bytes=265830400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266174032 ingress_bytes=300433408 link_bytes=300433408 token_bytes=532004432 weight_movement_bytes=34603008 total_nvlink_bytes=566607440 weight_to_token_ratio=0.065043 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130506727e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74290 payload_rows_remote=64880 dup_rows=24014 dup_rows_remote=20936 dedup_ratio=0.244283 router_tokens_per_expert=[772, 778, 745, 795, 744, 796, 755, 792, 756, 725, 799, 750, 750, 789, 764, 757, 825, 771, 757, 791, 828, 743, 764, 749, 747, 819, 755, 742, 756, 757, 739, 785, 773, 720, 783, 801, 758, 744, 764, 762, 738, 819, 779, 782, 785, 773, 807, 761, 761, 752, 765, 756, 731, 774, 754, 720, 777, 768, 782, 796, 751, 799, 772, 750] payload_rows_per_dst=[9350, 9220, 9314, 9284, 9192, 9410, 9150, 9370] dup_rows_per_dst=[3004, 2960, 3142, 2916, 3018, 3078, 2876, 3020] combine_rows_per_src=[9350, 9220, 9314, 9284, 9192, 9410, 9150, 9370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266919936 dispatch_meta_bytes=344168 combine_payload_bytes=266919936 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267264104 ingress_bytes=301522944 link_bytes=301522944 token_bytes=534184040 weight_movement_bytes=34603008 total_nvlink_bytes=568787048 weight_to_token_ratio=0.064777 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134606564e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=65078 dup_rows=23946 dup_rows_remote=20898 dedup_ratio=0.243591 router_tokens_per_expert=[781, 719, 785, 704, 762, 746, 779, 786, 768, 738, 737, 790, 774, 743, 774, 734, 763, 770, 758, 779, 791, 785, 755, 763, 742, 788, 816, 757, 750, 787, 770, 740, 774, 796, 775, 754, 756, 742, 744, 792, 785, 750, 764, 814, 746, 757, 758, 772, 775, 781, 778, 787, 735, 787, 748, 763, 801, 789, 828, 765, 736, 821, 753, 792] payload_rows_per_dst=[9192, 9176, 9280, 9336, 9294, 9310, 9290, 9480] dup_rows_per_dst=[2932, 2940, 3048, 2964, 2972, 2982, 3018, 3090] combine_rows_per_src=[9192, 9176, 9280, 9336, 9294, 9310, 9290, 9480] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74512 payload_rows_remote=65054 dup_rows=23792 dup_rows_remote=20808 dedup_ratio=0.242025 router_tokens_per_expert=[775, 756, 812, 791, 775, 763, 744, 805, 751, 776, 760, 795, 728, 805, 769, 755, 758, 782, 745, 770, 800, 751, 776, 799, 774, 771, 718, 810, 751, 750, 792, 764, 822, 762, 749, 773, 728, 776, 757, 750, 757, 777, 783, 763, 802, 731, 774, 753, 736, 787, 775, 767, 776, 764, 789, 810, 732, 799, 770, 737, 720, 739, 774, 749] payload_rows_per_dst=[9458, 9434, 9214, 9292, 9300, 9258, 9316, 9240] dup_rows_per_dst=[2984, 3172, 2820, 3046, 2934, 2944, 3092, 2800] combine_rows_per_src=[9458, 9434, 9214, 9292, 9300, 9258, 9316, 9240] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266240000 dispatch_meta_bytes=344280 combine_payload_bytes=266240000 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266584280 ingress_bytes=370049024 link_bytes=370049024 token_bytes=532824280 weight_movement_bytes=103809024 total_nvlink_bytes=636633304 weight_to_token_ratio=0.194828 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392464686e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266559488 dispatch_meta_bytes=343904 combine_payload_bytes=266559488 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266903392 ingress_bytes=370368512 link_bytes=370368512 token_bytes=533462880 weight_movement_bytes=103809024 total_nvlink_bytes=637271904 weight_to_token_ratio=0.194595 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393666894e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265748480 dispatch_meta_bytes=343264 combine_payload_bytes=265748480 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266091744 ingress_bytes=369557504 link_bytes=369557504 token_bytes=531840224 weight_movement_bytes=103809024 total_nvlink_bytes=635649248 weight_to_token_ratio=0.195188 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390615136e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74120 payload_rows_remote=64732 dup_rows=24184 dup_rows_remote=21148 dedup_ratio=0.246012 router_tokens_per_expert=[794, 759, 735, 770, 768, 776, 790, 791, 764, 724, 744, 751, 778, 810, 782, 766, 711, 803, 816, 764, 737, 776, 746, 781, 755, 765, 770, 807, 747, 767, 753, 767, 804, 771, 779, 760, 795, 753, 773, 777, 748, 732, 817, 737, 793, 778, 755, 784, 767, 783, 740, 800, 750, 799, 777, 787, 762, 775, 746, 721, 780, 743, 769, 730] payload_rows_per_dst=[9288, 9262, 9302, 9130, 9388, 9252, 9370, 9128] dup_rows_per_dst=[3078, 2976, 2966, 3132, 3036, 3036, 3036, 2924] combine_rows_per_src=[9288, 9262, 9302, 9130, 9388, 9252, 9370, 9128] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74522 payload_rows_remote=65164 dup_rows=23782 dup_rows_remote=20802 dedup_ratio=0.241923 router_tokens_per_expert=[808, 782, 785, 720, 731, 751, 754, 792, 809, 834, 753, 742, 783, 746, 739, 721, 775, 788, 806, 792, 776, 746, 760, 722, 801, 741, 774, 746, 782, 743, 782, 800, 809, 751, 749, 727, 781, 755, 770, 779, 774, 738, 816, 757, 749, 797, 806, 761, 730, 799, 768, 728, 748, 802, 718, 746, 832, 768, 797, 782, 762, 759, 751, 759] payload_rows_per_dst=[9316, 9248, 9346, 9358, 9340, 9396, 9148, 9370] dup_rows_per_dst=[2930, 3006, 2984, 2980, 2902, 3000, 2930, 3050] combine_rows_per_src=[9316, 9248, 9346, 9358, 9340, 9396, 9148, 9370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265748480 dispatch_meta_bytes=343264 combine_payload_bytes=265748480 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266091744 ingress_bytes=369557504 link_bytes=369557504 token_bytes=531840224 weight_movement_bytes=103809024 total_nvlink_bytes=635649248 weight_to_token_ratio=0.195188 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390615136e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266559488 dispatch_meta_bytes=343904 combine_payload_bytes=266559488 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266903392 ingress_bytes=370368512 link_bytes=370368512 token_bytes=533462880 weight_movement_bytes=103809024 total_nvlink_bytes=637271904 weight_to_token_ratio=0.194595 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393666894e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=343448 combine_payload_bytes=266461184 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266804632 ingress_bytes=335667200 link_bytes=335667200 token_bytes=533265816 weight_movement_bytes=69206016 total_nvlink_bytes=602471832 weight_to_token_ratio=0.129778 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263088650e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74124 payload_rows_remote=64900 dup_rows=24180 dup_rows_remote=21008 dedup_ratio=0.245972 router_tokens_per_expert=[793, 735, 798, 768, 757, 737, 760, 811, 756, 790, 755, 826, 768, 731, 806, 759, 771, 716, 794, 793, 772, 690, 773, 775, 759, 743, 794, 748, 783, 790, 781, 784, 795, 752, 802, 763, 797, 785, 748, 789, 739, 754, 754, 746, 750, 797, 774, 774, 735, 778, 775, 727, 745, 807, 781, 731, 747, 767, 763, 768, 805, 719, 825, 744] payload_rows_per_dst=[9286, 9340, 9216, 9224, 9388, 9142, 9304, 9224] dup_rows_per_dst=[3006, 3042, 2918, 3140, 2978, 2858, 3066, 3172] combine_rows_per_src=[9286, 9340, 9216, 9224, 9388, 9142, 9304, 9224] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74120 payload_rows_remote=64732 dup_rows=24184 dup_rows_remote=21148 dedup_ratio=0.246012 router_tokens_per_expert=[794, 759, 735, 770, 768, 776, 790, 791, 764, 724, 744, 751, 778, 810, 782, 766, 711, 803, 816, 764, 737, 776, 746, 781, 755, 765, 770, 807, 747, 767, 753, 767, 804, 771, 779, 760, 795, 753, 773, 777, 748, 732, 817, 737, 793, 778, 755, 784, 767, 783, 740, 800, 750, 799, 777, 787, 762, 775, 746, 721, 780, 743, 769, 730] payload_rows_per_dst=[9288, 9262, 9302, 9130, 9388, 9252, 9370, 9128] dup_rows_per_dst=[3078, 2976, 2966, 3132, 3036, 3036, 3036, 2924] combine_rows_per_src=[9288, 9262, 9302, 9130, 9388, 9252, 9370, 9128] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74370 payload_rows_remote=65178 dup_rows=23934 dup_rows_remote=20990 dedup_ratio=0.243469 router_tokens_per_expert=[755, 805, 754, 771, 785, 779, 731, 776, 780, 714, 725, 768, 789, 760, 739, 793, 780, 776, 837, 768, 768, 727, 786, 749, 780, 766, 756, 746, 702, 777, 738, 797, 722, 775, 830, 778, 751, 760, 768, 786, 815, 760, 752, 782, 767, 736, 774, 738, 755, 784, 780, 755, 791, 763, 772, 806, 756, 804, 773, 788, 767, 769, 744, 774] payload_rows_per_dst=[9306, 9192, 9330, 9216, 9330, 9316, 9320, 9360] dup_rows_per_dst=[3006, 2944, 3052, 2908, 3010, 2932, 3092, 2990] combine_rows_per_src=[9306, 9192, 9330, 9216, 9330, 9316, 9320, 9360] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266911744 dispatch_meta_bytes=343864 combine_payload_bytes=266911744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267255608 ingress_bytes=301514752 link_bytes=301514752 token_bytes=534167352 weight_movement_bytes=34603008 total_nvlink_bytes=568770360 weight_to_token_ratio=0.064779 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134575738e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=64886 dup_rows=24132 dup_rows_remote=21122 dedup_ratio=0.245483 router_tokens_per_expert=[802, 747, 782, 767, 740, 755, 741, 750, 763, 756, 746, 789, 824, 782, 764, 772, 777, 776, 781, 766, 801, 723, 795, 767, 788, 787, 722, 746, 785, 801, 785, 775, 776, 775, 730, 777, 715, 767, 773, 798, 778, 775, 772, 777, 774, 788, 785, 746, 775, 787, 726, 718, 787, 788, 826, 741, 776, 779, 726, 753, 767, 729, 781, 732] payload_rows_per_dst=[9178, 9298, 9280, 9294, 9248, 9406, 9286, 9182] dup_rows_per_dst=[2990, 3094, 3092, 3084, 2974, 2984, 3010, 2904] combine_rows_per_src=[9178, 9298, 9280, 9294, 9248, 9406, 9286, 9182] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265830400 dispatch_meta_bytes=343632 combine_payload_bytes=265830400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266174032 ingress_bytes=300433408 link_bytes=300433408 token_bytes=532004432 weight_movement_bytes=34603008 total_nvlink_bytes=566607440 weight_to_token_ratio=0.065043 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130506727e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=64840 dup_rows=24060 dup_rows_remote=21118 dedup_ratio=0.244751 router_tokens_per_expert=[768, 786, 772, 775, 796, 784, 784, 768, 762, 813, 818, 748, 734, 735, 811, 752, 821, 761, 750, 770, 800, 785, 777, 778, 791, 766, 743, 766, 790, 715, 753, 772, 794, 768, 751, 753, 785, 803, 777, 703, 725, 762, 795, 748, 761, 768, 755, 773, 732, 773, 705, 771, 751, 769, 776, 796, 743, 829, 726, 761, 713, 773, 790, 779] payload_rows_per_dst=[9292, 9404, 9366, 9164, 9324, 9224, 9248, 9222] dup_rows_per_dst=[3174, 2942, 3118, 3028, 2944, 2950, 2898, 3006] combine_rows_per_src=[9292, 9404, 9366, 9164, 9324, 9224, 9248, 9222] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265142272 dispatch_meta_bytes=343520 combine_payload_bytes=265142272 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265485792 ingress_bytes=334348288 link_bytes=334348288 token_bytes=530628064 weight_movement_bytes=69206016 total_nvlink_bytes=599834080 weight_to_token_ratio=0.130423 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258125690e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=65074 dup_rows=24028 dup_rows_remote=21164 dedup_ratio=0.244425 router_tokens_per_expert=[758, 788, 804, 751, 787, 774, 782, 759, 785, 785, 761, 735, 781, 768, 733, 727, 768, 750, 790, 741, 775, 799, 758, 764, 770, 852, 801, 784, 701, 756, 728, 781, 757, 750, 780, 776, 776, 807, 807, 819, 780, 777, 744, 788, 749, 759, 757, 742, 716, 807, 783, 724, 813, 745, 709, 736, 763, 742, 762, 743, 770, 792, 773, 810] payload_rows_per_dst=[9244, 9236, 9258, 9306, 9452, 9284, 9202, 9294] dup_rows_per_dst=[3162, 2914, 3032, 3040, 3092, 2908, 2864, 3016] combine_rows_per_src=[9244, 9236, 9258, 9306, 9452, 9284, 9202, 9294] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=65024 dup_rows=24012 dup_rows_remote=21050 dedup_ratio=0.244263 router_tokens_per_expert=[756, 745, 750, 816, 775, 777, 785, 787, 763, 759, 735, 783, 818, 796, 812, 750, 754, 759, 783, 746, 790, 761, 773, 749, 795, 783, 794, 748, 755, 767, 764, 783, 773, 753, 769, 755, 771, 769, 734, 730, 749, 759, 742, 781, 803, 736, 761, 763, 773, 763, 807, 749, 773, 761, 812, 716, 704, 744, 727, 753, 812, 794, 780, 825] payload_rows_per_dst=[9442, 9282, 9268, 9334, 9196, 9194, 9248, 9328] dup_rows_per_dst=[2940, 3150, 2962, 3044, 2912, 2994, 3060, 2950] combine_rows_per_src=[9442, 9282, 9268, 9334, 9196, 9194, 9248, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74394 payload_rows_remote=64572 dup_rows=23910 dup_rows_remote=20526 dedup_ratio=0.243225 router_tokens_per_expert=[757, 802, 728, 801, 777, 780, 738, 765, 759, 772, 783, 806, 752, 823, 773, 730, 771, 740, 794, 760, 766, 774, 787, 737, 770, 821, 713, 807, 780, 805, 795, 719, 788, 780, 791, 781, 748, 747, 764, 763, 732, 751, 782, 777, 772, 773, 788, 712, 785, 750, 784, 701, 788, 759, 829, 729, 789, 731, 768, 745, 754, 780, 759, 767] payload_rows_per_dst=[9148, 8812, 9280, 9342, 9336, 9276, 9378, 9822] dup_rows_per_dst=[2830, 2564, 3064, 3078, 3036, 2898, 3056, 3384] combine_rows_per_src=[9148, 8812, 9280, 9342, 9336, 9276, 9378, 9822] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266911744 dispatch_meta_bytes=343864 combine_payload_bytes=266911744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267255608 ingress_bytes=301514752 link_bytes=301514752 token_bytes=534167352 weight_movement_bytes=34603008 total_nvlink_bytes=568770360 weight_to_token_ratio=0.064779 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134575738e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=344032 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266117088 ingress_bytes=300376064 link_bytes=300376064 token_bytes=531890144 weight_movement_bytes=34603008 total_nvlink_bytes=566493152 weight_to_token_ratio=0.065057 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130290947e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265584640 dispatch_meta_bytes=343832 combine_payload_bytes=265584640 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265928472 ingress_bytes=334790656 link_bytes=334790656 token_bytes=531513112 weight_movement_bytes=69206016 total_nvlink_bytes=600719128 weight_to_token_ratio=0.130206 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259790286e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=264486912 dispatch_meta_bytes=340392 combine_payload_bytes=264486912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264827304 ingress_bytes=299089920 link_bytes=299089920 token_bytes=529314216 weight_movement_bytes=34603008 total_nvlink_bytes=563917224 weight_to_token_ratio=0.065373 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125451290e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64900 dup_rows=24112 dup_rows_remote=21270 dedup_ratio=0.245280 router_tokens_per_expert=[774, 785, 722, 784, 755, 771, 764, 771, 801, 758, 792, 831, 805, 812, 761, 804, 772, 791, 804, 787, 765, 800, 765, 772, 788, 762, 761, 768, 757, 755, 737, 766, 763, 799, 720, 774, 742, 716, 752, 761, 793, 730, 773, 778, 749, 735, 743, 766, 790, 771, 793, 739, 779, 792, 767, 755, 760, 732, 772, 758, 771, 767, 743, 729] payload_rows_per_dst=[9150, 9514, 9430, 9302, 9138, 9292, 9280, 9086] dup_rows_per_dst=[3102, 3214, 3082, 2886, 2916, 2842, 3092, 2978] combine_rows_per_src=[9150, 9514, 9430, 9302, 9138, 9292, 9280, 9086] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266969088 dispatch_meta_bytes=344672 combine_payload_bytes=266969088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267313760 ingress_bytes=301572096 link_bytes=301572096 token_bytes=534282848 weight_movement_bytes=34603008 total_nvlink_bytes=568885856 weight_to_token_ratio=0.064765 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134791519e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=65024 dup_rows=24012 dup_rows_remote=21050 dedup_ratio=0.244263 router_tokens_per_expert=[756, 745, 750, 816, 775, 777, 785, 787, 763, 759, 735, 783, 818, 796, 812, 750, 754, 759, 783, 746, 790, 761, 773, 749, 795, 783, 794, 748, 755, 767, 764, 783, 773, 753, 769, 755, 771, 769, 734, 730, 749, 759, 742, 781, 803, 736, 761, 763, 773, 763, 807, 749, 773, 761, 812, 716, 704, 744, 727, 753, 812, 794, 780, 825] payload_rows_per_dst=[9442, 9282, 9268, 9334, 9196, 9194, 9248, 9328] dup_rows_per_dst=[2940, 3150, 2962, 3044, 2912, 2994, 3060, 2950] combine_rows_per_src=[9442, 9282, 9268, 9334, 9196, 9194, 9248, 9328] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74560 payload_rows_remote=65160 dup_rows=23744 dup_rows_remote=20700 dedup_ratio=0.241536 router_tokens_per_expert=[794, 777, 756, 787, 748, 757, 796, 806, 775, 763, 722, 758, 738, 796, 764, 766, 763, 770, 741, 773, 800, 760, 767, 742, 781, 752, 781, 773, 764, 763, 823, 734, 790, 749, 802, 833, 761, 763, 772, 758, 805, 790, 765, 783, 775, 725, 778, 747, 735, 753, 782, 728, 832, 767, 760, 776, 764, 742, 773, 754, 762, 776, 727, 735] payload_rows_per_dst=[9400, 9498, 9346, 9088, 9364, 9652, 8928, 9284] dup_rows_per_dst=[3044, 3082, 2886, 2836, 3092, 3348, 2674, 2782] combine_rows_per_src=[9400, 9498, 9346, 9088, 9364, 9652, 8928, 9284] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=344952 combine_payload_bytes=266543104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266888056 ingress_bytes=301146112 link_bytes=301146112 token_bytes=533431160 weight_movement_bytes=34603008 total_nvlink_bytes=568034168 weight_to_token_ratio=0.064869 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133188575e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=344296 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266682600 ingress_bytes=300941312 link_bytes=300941312 token_bytes=533020904 weight_movement_bytes=34603008 total_nvlink_bytes=567623912 weight_to_token_ratio=0.064919 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132417929e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74260 payload_rows_remote=64742 dup_rows=24044 dup_rows_remote=20990 dedup_ratio=0.244588 router_tokens_per_expert=[726, 792, 757, 778, 735, 788, 761, 740, 728, 765, 815, 730, 725, 828, 799, 726, 787, 790, 730, 767, 707, 719, 814, 748, 771, 774, 766, 775, 794, 806, 746, 777, 788, 796, 742, 765, 779, 803, 803, 810, 792, 787, 780, 732, 757, 762, 780, 779, 784, 819, 778, 824, 753, 773, 761, 771, 745, 759, 785, 715, 751, 766, 674, 775] payload_rows_per_dst=[9218, 9216, 9130, 9328, 9518, 9380, 9418, 9052] dup_rows_per_dst=[2936, 3016, 2994, 3090, 3054, 2958, 3108, 2888] combine_rows_per_src=[9218, 9216, 9130, 9328, 9518, 9380, 9418, 9052] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265830400 dispatch_meta_bytes=344680 combine_payload_bytes=265830400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266175080 ingress_bytes=300433408 link_bytes=300433408 token_bytes=532005480 weight_movement_bytes=34603008 total_nvlink_bytes=566608488 weight_to_token_ratio=0.065043 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130506727e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74460 payload_rows_remote=65154 dup_rows=23844 dup_rows_remote=20824 dedup_ratio=0.242554 router_tokens_per_expert=[804, 740, 772, 793, 749, 757, 800, 720, 823, 754, 754, 737, 788, 758, 793, 795, 789, 752, 772, 728, 754, 781, 742, 715, 771, 788, 766, 744, 758, 802, 728, 806, 775, 740, 785, 778, 817, 729, 768, 720, 786, 744, 751, 749, 749, 734, 766, 788, 792, 730, 803, 806, 782, 790, 789, 750, 747, 756, 799, 762, 789, 740, 812, 793] payload_rows_per_dst=[9330, 9384, 9192, 9306, 9310, 9204, 9376, 9358] dup_rows_per_dst=[2940, 3020, 2874, 3020, 2914, 2930, 3108, 3038] combine_rows_per_src=[9330, 9384, 9192, 9306, 9310, 9204, 9376, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=344296 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266682600 ingress_bytes=300941312 link_bytes=300941312 token_bytes=533020904 weight_movement_bytes=34603008 total_nvlink_bytes=567623912 weight_to_token_ratio=0.064919 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132417929e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266895360 dispatch_meta_bytes=343440 combine_payload_bytes=266895360 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=267238800 ingress_bytes=405307392 link_bytes=405307392 token_bytes=534134160 weight_movement_bytes=138412032 total_nvlink_bytes=672546192 weight_to_token_ratio=0.259133 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.525139086e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74394 payload_rows_remote=64572 dup_rows=23910 dup_rows_remote=20526 dedup_ratio=0.243225 router_tokens_per_expert=[757, 802, 728, 801, 777, 780, 738, 765, 759, 772, 783, 806, 752, 823, 773, 730, 771, 740, 794, 760, 766, 774, 787, 737, 770, 821, 713, 807, 780, 805, 795, 719, 788, 780, 791, 781, 748, 747, 764, 763, 732, 751, 782, 777, 772, 773, 788, 712, 785, 750, 784, 701, 788, 759, 829, 729, 789, 731, 768, 745, 754, 780, 759, 767] payload_rows_per_dst=[9148, 8812, 9280, 9342, 9336, 9276, 9378, 9822] dup_rows_per_dst=[2830, 2564, 3064, 3078, 3036, 2898, 3056, 3384] combine_rows_per_src=[9148, 8812, 9280, 9342, 9336, 9276, 9378, 9822] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74260 payload_rows_remote=64742 dup_rows=24044 dup_rows_remote=20990 dedup_ratio=0.244588 router_tokens_per_expert=[726, 792, 757, 778, 735, 788, 761, 740, 728, 765, 815, 730, 725, 828, 799, 726, 787, 790, 730, 767, 707, 719, 814, 748, 771, 774, 766, 775, 794, 806, 746, 777, 788, 796, 742, 765, 779, 803, 803, 810, 792, 787, 780, 732, 757, 762, 780, 779, 784, 819, 778, 824, 753, 773, 761, 771, 745, 759, 785, 715, 751, 766, 674, 775] payload_rows_per_dst=[9218, 9216, 9130, 9328, 9518, 9380, 9418, 9052] dup_rows_per_dst=[2936, 3016, 2994, 3090, 3054, 2958, 3108, 2888] combine_rows_per_src=[9218, 9216, 9130, 9328, 9518, 9380, 9418, 9052] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=64840 dup_rows=24060 dup_rows_remote=21118 dedup_ratio=0.244751 router_tokens_per_expert=[768, 786, 772, 775, 796, 784, 784, 768, 762, 813, 818, 748, 734, 735, 811, 752, 821, 761, 750, 770, 800, 785, 777, 778, 791, 766, 743, 766, 790, 715, 753, 772, 794, 768, 751, 753, 785, 803, 777, 703, 725, 762, 795, 748, 761, 768, 755, 773, 732, 773, 705, 771, 751, 769, 776, 796, 743, 829, 726, 761, 713, 773, 790, 779] payload_rows_per_dst=[9292, 9404, 9366, 9164, 9324, 9224, 9248, 9222] dup_rows_per_dst=[3174, 2942, 3118, 3028, 2944, 2950, 2898, 3006] combine_rows_per_src=[9292, 9404, 9366, 9164, 9324, 9224, 9248, 9222] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266870784 dispatch_meta_bytes=343912 combine_payload_bytes=266870784 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267214696 ingress_bytes=336076800 link_bytes=336076800 token_bytes=534085480 weight_movement_bytes=69206016 total_nvlink_bytes=603291496 weight_to_token_ratio=0.129579 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264629942e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74452 payload_rows_remote=65098 dup_rows=23852 dup_rows_remote=20774 dedup_ratio=0.242635 router_tokens_per_expert=[815, 798, 759, 794, 776, 706, 765, 766, 735, 762, 795, 764, 762, 783, 758, 744, 757, 770, 751, 733, 760, 770, 811, 804, 817, 784, 784, 762, 758, 830, 722, 772, 763, 729, 747, 809, 765, 769, 786, 757, 770, 755, 775, 743, 713, 774, 720, 831, 785, 763, 752, 783, 766, 760, 778, 829, 744, 786, 790, 727, 763, 768, 751, 734] payload_rows_per_dst=[9362, 9254, 9380, 9364, 9254, 9258, 9354, 9226] dup_rows_per_dst=[2996, 2952, 2932, 3094, 2996, 2904, 3078, 2900] combine_rows_per_src=[9362, 9254, 9380, 9364, 9254, 9258, 9354, 9226] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=264486912 dispatch_meta_bytes=340392 combine_payload_bytes=264486912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264827304 ingress_bytes=299089920 link_bytes=299089920 token_bytes=529314216 weight_movement_bytes=34603008 total_nvlink_bytes=563917224 weight_to_token_ratio=0.065373 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125451290e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74680 payload_rows_remote=65424 dup_rows=23624 dup_rows_remote=20754 dedup_ratio=0.240316 router_tokens_per_expert=[784, 781, 785, 803, 766, 760, 678, 758, 792, 783, 816, 752, 806, 785, 718, 738, 751, 744, 758, 768, 747, 802, 763, 730, 747, 762, 730, 789, 763, 712, 816, 767, 792, 788, 752, 776, 819, 831, 794, 764, 729, 712, 763, 844, 787, 762, 777, 808, 757, 745, 789, 740, 754, 774, 770, 751, 801, 753, 766, 762, 777, 776, 753, 732] payload_rows_per_dst=[9244, 9370, 9256, 9336, 9526, 9310, 9274, 9364] dup_rows_per_dst=[2986, 3010, 2870, 2836, 3106, 3054, 2886, 2876] combine_rows_per_src=[9244, 9370, 9256, 9336, 9526, 9310, 9274, 9364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74426 payload_rows_remote=65162 dup_rows=23878 dup_rows_remote=20830 dedup_ratio=0.242900 router_tokens_per_expert=[763, 858, 788, 731, 755, 812, 793, 803, 762, 749, 803, 732, 774, 766, 797, 773, 777, 752, 773, 787, 789, 757, 769, 781, 753, 809, 768, 786, 790, 805, 744, 765, 797, 708, 778, 760, 750, 735, 805, 794, 713, 767, 748, 761, 720, 764, 778, 744, 860, 763, 771, 698, 743, 744, 761, 775, 782, 709, 774, 773, 754, 755, 748, 756] payload_rows_per_dst=[9518, 9264, 9348, 9404, 9302, 9140, 9290, 9160] dup_rows_per_dst=[3088, 3048, 3022, 3036, 2952, 2850, 2940, 2942] combine_rows_per_src=[9518, 9264, 9348, 9404, 9302, 9140, 9290, 9160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=64728 dup_rows=24132 dup_rows_remote=20932 dedup_ratio=0.245483 router_tokens_per_expert=[718, 788, 768, 768, 775, 790, 804, 772, 755, 713, 713, 775, 774, 782, 745, 766, 777, 811, 813, 763, 761, 773, 722, 738, 751, 751, 745, 759, 798, 798, 724, 739, 747, 783, 815, 749, 750, 783, 757, 773, 795, 767, 761, 769, 777, 808, 788, 746, 750, 716, 781, 758, 774, 762, 761, 797, 788, 782, 818, 778, 827, 736, 755, 772] payload_rows_per_dst=[8976, 9440, 9296, 9068, 9170, 9556, 9222, 9444] dup_rows_per_dst=[2702, 3294, 2888, 2986, 2896, 3172, 2994, 3200] combine_rows_per_src=[8976, 9440, 9296, 9068, 9170, 9556, 9222, 9444] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265183232 dispatch_meta_bytes=342928 combine_payload_bytes=265183232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265526160 ingress_bytes=299786240 link_bytes=299786240 token_bytes=530709392 weight_movement_bytes=34603008 total_nvlink_bytes=565312400 weight_to_token_ratio=0.065201 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128071486e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=64886 dup_rows=24132 dup_rows_remote=21122 dedup_ratio=0.245483 router_tokens_per_expert=[802, 747, 782, 767, 740, 755, 741, 750, 763, 756, 746, 789, 824, 782, 764, 772, 777, 776, 781, 766, 801, 723, 795, 767, 788, 787, 722, 746, 785, 801, 785, 775, 776, 775, 730, 777, 715, 767, 773, 798, 778, 775, 772, 777, 774, 788, 785, 746, 775, 787, 726, 718, 787, 788, 826, 741, 776, 779, 726, 753, 767, 729, 781, 732] payload_rows_per_dst=[9178, 9298, 9280, 9294, 9248, 9406, 9286, 9182] dup_rows_per_dst=[2990, 3094, 3092, 3084, 2974, 2984, 3010, 2904] combine_rows_per_src=[9178, 9298, 9280, 9294, 9248, 9406, 9286, 9182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266870784 dispatch_meta_bytes=343912 combine_payload_bytes=266870784 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267214696 ingress_bytes=336076800 link_bytes=336076800 token_bytes=534085480 weight_movement_bytes=69206016 total_nvlink_bytes=603291496 weight_to_token_ratio=0.129579 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264629942e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=343488 combine_payload_bytes=266641408 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266984896 ingress_bytes=335847424 link_bytes=335847424 token_bytes=533626304 weight_movement_bytes=69206016 total_nvlink_bytes=602832320 weight_to_token_ratio=0.129690 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263766819e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266903552 dispatch_meta_bytes=343968 combine_payload_bytes=266903552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267247520 ingress_bytes=301506560 link_bytes=301506560 token_bytes=534151072 weight_movement_bytes=34603008 total_nvlink_bytes=568754080 weight_to_token_ratio=0.064781 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134544912e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265125888 dispatch_meta_bytes=342640 combine_payload_bytes=265125888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265468528 ingress_bytes=299728896 link_bytes=299728896 token_bytes=530594416 weight_movement_bytes=34603008 total_nvlink_bytes=565197424 weight_to_token_ratio=0.065216 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127855705e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74536 payload_rows_remote=65122 dup_rows=23768 dup_rows_remote=20858 dedup_ratio=0.241781 router_tokens_per_expert=[739, 751, 755, 834, 738, 776, 760, 801, 729, 789, 796, 748, 753, 760, 767, 712, 767, 783, 796, 814, 791, 734, 748, 750, 789, 755, 775, 762, 793, 766, 773, 750, 774, 776, 765, 753, 722, 782, 772, 764, 747, 738, 824, 809, 788, 751, 745, 760, 727, 842, 761, 788, 783, 765, 776, 797, 778, 730, 756, 751, 752, 795, 766, 761] payload_rows_per_dst=[9278, 9230, 9348, 9380, 9154, 9414, 9408, 9324] dup_rows_per_dst=[3030, 2878, 3018, 2946, 3062, 2910, 3070, 2854] combine_rows_per_src=[9278, 9230, 9348, 9380, 9154, 9414, 9408, 9324] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265584640 dispatch_meta_bytes=343832 combine_payload_bytes=265584640 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265928472 ingress_bytes=334790656 link_bytes=334790656 token_bytes=531513112 weight_movement_bytes=69206016 total_nvlink_bytes=600719128 weight_to_token_ratio=0.130206 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259790286e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74680 payload_rows_remote=65424 dup_rows=23624 dup_rows_remote=20754 dedup_ratio=0.240316 router_tokens_per_expert=[784, 781, 785, 803, 766, 760, 678, 758, 792, 783, 816, 752, 806, 785, 718, 738, 751, 744, 758, 768, 747, 802, 763, 730, 747, 762, 730, 789, 763, 712, 816, 767, 792, 788, 752, 776, 819, 831, 794, 764, 729, 712, 763, 844, 787, 762, 777, 808, 757, 745, 789, 740, 754, 774, 770, 751, 801, 753, 766, 762, 777, 776, 753, 732] payload_rows_per_dst=[9244, 9370, 9256, 9336, 9526, 9310, 9274, 9364] dup_rows_per_dst=[2986, 3010, 2870, 2836, 3106, 3054, 2886, 2876] combine_rows_per_src=[9244, 9370, 9256, 9336, 9526, 9310, 9274, 9364] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74134 payload_rows_remote=64744 dup_rows=24170 dup_rows_remote=21004 dedup_ratio=0.245870 router_tokens_per_expert=[777, 772, 775, 790, 768, 750, 787, 751, 766, 760, 749, 755, 713, 755, 793, 761, 770, 823, 772, 778, 781, 778, 763, 751, 757, 818, 770, 754, 803, 785, 755, 785, 766, 760, 736, 773, 739, 745, 790, 800, 798, 750, 742, 774, 778, 728, 761, 765, 755, 765, 738, 729, 761, 789, 762, 731, 737, 799, 807, 801, 770, 728, 838, 772] payload_rows_per_dst=[9390, 9142, 9350, 9306, 9100, 9238, 9178, 9430] dup_rows_per_dst=[3166, 2962, 3102, 3128, 2902, 2954, 2882, 3074] combine_rows_per_src=[9390, 9142, 9350, 9306, 9100, 9238, 9178, 9430] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=267976704 dispatch_meta_bytes=344712 combine_payload_bytes=267976704 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=268321416 ingress_bytes=337182720 link_bytes=337182720 token_bytes=536298120 weight_movement_bytes=69206016 total_nvlink_bytes=605504136 weight_to_token_ratio=0.129044 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.268791430e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=344032 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266117088 ingress_bytes=300376064 link_bytes=300376064 token_bytes=531890144 weight_movement_bytes=34603008 total_nvlink_bytes=566493152 weight_to_token_ratio=0.065057 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130290947e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266739712 dispatch_meta_bytes=343920 combine_payload_bytes=266739712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267083632 ingress_bytes=301342720 link_bytes=301342720 token_bytes=533823344 weight_movement_bytes=34603008 total_nvlink_bytes=568426352 weight_to_token_ratio=0.064821 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133928395e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=267976704 dispatch_meta_bytes=344712 combine_payload_bytes=267976704 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=268321416 ingress_bytes=337182720 link_bytes=337182720 token_bytes=536298120 weight_movement_bytes=69206016 total_nvlink_bytes=605504136 weight_to_token_ratio=0.129044 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.268791430e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265191424 dispatch_meta_bytes=342992 combine_payload_bytes=265191424 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265534416 ingress_bytes=334397440 link_bytes=334397440 token_bytes=530725840 weight_movement_bytes=69206016 total_nvlink_bytes=599931856 weight_to_token_ratio=0.130399 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258310646e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74300 payload_rows_remote=64940 dup_rows=24004 dup_rows_remote=21006 dedup_ratio=0.244181 router_tokens_per_expert=[765, 725, 774, 776, 805, 793, 749, 769, 739, 766, 770, 801, 778, 730, 749, 739, 715, 778, 765, 748, 763, 777, 758, 749, 714, 768, 712, 779, 738, 803, 820, 845, 797, 776, 730, 792, 805, 746, 755, 771, 756, 783, 755, 787, 748, 748, 763, 792, 768, 793, 794, 807, 729, 811, 733, 775, 767, 783, 784, 750, 781, 764, 803, 746] payload_rows_per_dst=[9310, 9176, 9136, 9360, 9382, 9294, 9290, 9352] dup_rows_per_dst=[3002, 2968, 2970, 2998, 2962, 2970, 3130, 3004] combine_rows_per_src=[9310, 9176, 9136, 9360, 9382, 9294, 9290, 9352] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=64728 dup_rows=24132 dup_rows_remote=20932 dedup_ratio=0.245483 router_tokens_per_expert=[718, 788, 768, 768, 775, 790, 804, 772, 755, 713, 713, 775, 774, 782, 745, 766, 777, 811, 813, 763, 761, 773, 722, 738, 751, 751, 745, 759, 798, 798, 724, 739, 747, 783, 815, 749, 750, 783, 757, 773, 795, 767, 761, 769, 777, 808, 788, 746, 750, 716, 781, 758, 774, 762, 761, 797, 788, 782, 818, 778, 827, 736, 755, 772] payload_rows_per_dst=[8976, 9440, 9296, 9068, 9170, 9556, 9222, 9444] dup_rows_per_dst=[2702, 3294, 2888, 2986, 2896, 3172, 2994, 3200] combine_rows_per_src=[8976, 9440, 9296, 9068, 9170, 9556, 9222, 9444] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74202 payload_rows_remote=64846 dup_rows=24102 dup_rows_remote=21074 dedup_ratio=0.245178 router_tokens_per_expert=[787, 771, 758, 740, 776, 769, 733, 755, 791, 757, 768, 778, 704, 817, 771, 754, 796, 764, 785, 839, 766, 741, 738, 758, 776, 755, 735, 748, 806, 774, 784, 724, 793, 762, 827, 795, 735, 749, 738, 793, 747, 825, 700, 800, 777, 784, 712, 768, 811, 753, 805, 787, 762, 734, 761, 796, 797, 733, 796, 749, 779, 733, 756, 777] payload_rows_per_dst=[9086, 9320, 9330, 9218, 9356, 9228, 9414, 9250] dup_rows_per_dst=[3092, 2960, 3044, 2986, 3028, 2998, 3004, 2990] combine_rows_per_src=[9086, 9320, 9330, 9218, 9356, 9228, 9414, 9250] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74426 payload_rows_remote=65162 dup_rows=23878 dup_rows_remote=20830 dedup_ratio=0.242900 router_tokens_per_expert=[763, 858, 788, 731, 755, 812, 793, 803, 762, 749, 803, 732, 774, 766, 797, 773, 777, 752, 773, 787, 789, 757, 769, 781, 753, 809, 768, 786, 790, 805, 744, 765, 797, 708, 778, 760, 750, 735, 805, 794, 713, 767, 748, 761, 720, 764, 778, 744, 860, 763, 771, 698, 743, 744, 761, 775, 782, 709, 774, 773, 754, 755, 748, 756] payload_rows_per_dst=[9518, 9264, 9348, 9404, 9302, 9140, 9290, 9160] dup_rows_per_dst=[3088, 3048, 3022, 3036, 2952, 2850, 2940, 2942] combine_rows_per_src=[9518, 9264, 9348, 9404, 9302, 9140, 9290, 9160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74300 payload_rows_remote=64940 dup_rows=24004 dup_rows_remote=21006 dedup_ratio=0.244181 router_tokens_per_expert=[765, 725, 774, 776, 805, 793, 749, 769, 739, 766, 770, 801, 778, 730, 749, 739, 715, 778, 765, 748, 763, 777, 758, 749, 714, 768, 712, 779, 738, 803, 820, 845, 797, 776, 730, 792, 805, 746, 755, 771, 756, 783, 755, 787, 748, 748, 763, 792, 768, 793, 794, 807, 729, 811, 733, 775, 767, 783, 784, 750, 781, 764, 803, 746] payload_rows_per_dst=[9310, 9176, 9136, 9360, 9382, 9294, 9290, 9352] dup_rows_per_dst=[3002, 2968, 2970, 2998, 2962, 2970, 3130, 3004] combine_rows_per_src=[9310, 9176, 9136, 9360, 9382, 9294, 9290, 9352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74182 payload_rows_remote=64886 dup_rows=24122 dup_rows_remote=21096 dedup_ratio=0.245382 router_tokens_per_expert=[767, 775, 765, 754, 776, 809, 784, 788, 736, 745, 801, 800, 748, 764, 751, 776, 762, 787, 766, 752, 807, 770, 750, 769, 778, 747, 754, 800, 777, 787, 763, 752, 762, 795, 781, 750, 809, 759, 787, 760, 769, 783, 764, 757, 740, 766, 773, 756, 782, 831, 762, 735, 714, 773, 790, 774, 772, 738, 753, 737, 773, 750, 772, 725] payload_rows_per_dst=[9332, 9332, 9264, 9334, 9332, 9244, 9296, 9048] dup_rows_per_dst=[3104, 2910, 3062, 2982, 3074, 2972, 3026, 2992] combine_rows_per_src=[9332, 9332, 9264, 9334, 9332, 9244, 9296, 9048] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74474 payload_rows_remote=65158 dup_rows=23830 dup_rows_remote=20746 dedup_ratio=0.242411 router_tokens_per_expert=[752, 770, 790, 749, 782, 764, 720, 729, 741, 739, 785, 782, 804, 792, 770, 787, 748, 792, 780, 778, 801, 772, 767, 749, 772, 766, 780, 729, 775, 752, 719, 774, 735, 739, 756, 742, 724, 835, 776, 773, 742, 808, 758, 766, 749, 780, 761, 780, 810, 814, 755, 759, 812, 739, 788, 728, 757, 790, 757, 780, 794, 759, 749, 827] payload_rows_per_dst=[9108, 9316, 9368, 9250, 9306, 9402, 9394, 9330] dup_rows_per_dst=[3004, 3084, 3006, 2884, 2854, 2886, 3016, 3096] combine_rows_per_src=[9108, 9316, 9368, 9250, 9306, 9402, 9394, 9330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64588 dup_rows=24068 dup_rows_remote=20734 dedup_ratio=0.244832 router_tokens_per_expert=[780, 762, 772, 717, 732, 810, 777, 742, 751, 804, 769, 751, 791, 775, 768, 770, 802, 813, 757, 808, 755, 751, 771, 755, 798, 793, 741, 739, 763, 753, 781, 764, 764, 736, 805, 754, 756, 752, 764, 787, 747, 744, 789, 774, 743, 755, 769, 757, 785, 754, 709, 740, 760, 779, 777, 769, 853, 785, 753, 804, 792, 769, 782, 730] payload_rows_per_dst=[9126, 9062, 9600, 9270, 9170, 9238, 9122, 9648] dup_rows_per_dst=[3058, 2800, 3308, 2958, 2868, 2926, 2816, 3334] combine_rows_per_src=[9126, 9062, 9600, 9270, 9170, 9238, 9122, 9648] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265609216 dispatch_meta_bytes=343680 combine_payload_bytes=265609216 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265952896 ingress_bytes=369418240 link_bytes=369418240 token_bytes=531562112 weight_movement_bytes=103809024 total_nvlink_bytes=635371136 weight_to_token_ratio=0.195290 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390091097e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=343928 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266116984 ingress_bytes=300376064 link_bytes=300376064 token_bytes=531890040 weight_movement_bytes=34603008 total_nvlink_bytes=566493048 weight_to_token_ratio=0.065057 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130290947e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=343784 combine_payload_bytes=265994240 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266338024 ingress_bytes=369803264 link_bytes=369803264 token_bytes=532332264 weight_movement_bytes=103809024 total_nvlink_bytes=636141288 weight_to_token_ratio=0.195008 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391539911e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265125888 dispatch_meta_bytes=342640 combine_payload_bytes=265125888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265468528 ingress_bytes=299728896 link_bytes=299728896 token_bytes=530594416 weight_movement_bytes=34603008 total_nvlink_bytes=565197424 weight_to_token_ratio=0.065216 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127855705e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=264552448 dispatch_meta_bytes=341288 combine_payload_bytes=264552448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264893736 ingress_bytes=299155456 link_bytes=299155456 token_bytes=529446184 weight_movement_bytes=34603008 total_nvlink_bytes=564049192 weight_to_token_ratio=0.065357 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125697897e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266887168 dispatch_meta_bytes=343616 combine_payload_bytes=266887168 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267230784 ingress_bytes=370696192 link_bytes=370696192 token_bytes=534117952 weight_movement_bytes=103809024 total_nvlink_bytes=637926976 weight_to_token_ratio=0.194356 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394899927e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74516 payload_rows_remote=65268 dup_rows=23788 dup_rows_remote=20894 dedup_ratio=0.241984 router_tokens_per_expert=[706, 814, 802, 741, 775, 778, 765, 768, 753, 746, 785, 733, 747, 773, 764, 784, 759, 755, 763, 780, 747, 779, 742, 746, 786, 785, 770, 788, 756, 778, 746, 754, 772, 781, 801, 796, 719, 817, 766, 783, 768, 736, 790, 778, 789, 802, 759, 811, 751, 780, 773, 804, 767, 725, 808, 753, 716, 762, 727, 757, 771, 778, 774, 770] payload_rows_per_dst=[9236, 9254, 9248, 9402, 9394, 9388, 9320, 9274] dup_rows_per_dst=[3062, 2916, 2894, 2924, 3076, 3078, 3002, 2836] combine_rows_per_src=[9236, 9254, 9248, 9402, 9394, 9388, 9320, 9274] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74452 payload_rows_remote=65098 dup_rows=23852 dup_rows_remote=20774 dedup_ratio=0.242635 router_tokens_per_expert=[815, 798, 759, 794, 776, 706, 765, 766, 735, 762, 795, 764, 762, 783, 758, 744, 757, 770, 751, 733, 760, 770, 811, 804, 817, 784, 784, 762, 758, 830, 722, 772, 763, 729, 747, 809, 765, 769, 786, 757, 770, 755, 775, 743, 713, 774, 720, 831, 785, 763, 752, 783, 766, 760, 778, 829, 744, 786, 790, 727, 763, 768, 751, 734] payload_rows_per_dst=[9362, 9254, 9380, 9364, 9254, 9258, 9354, 9226] dup_rows_per_dst=[2996, 2952, 2932, 3094, 2996, 2904, 3078, 2900] combine_rows_per_src=[9362, 9254, 9380, 9364, 9254, 9258, 9354, 9226] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=64982 dup_rows=23946 dup_rows_remote=21008 dedup_ratio=0.243591 router_tokens_per_expert=[836, 793, 743, 786, 756, 790, 770, 796, 769, 767, 715, 793, 773, 812, 781, 795, 737, 756, 796, 769, 778, 749, 789, 729, 756, 737, 765, 785, 762, 747, 791, 769, 802, 850, 722, 795, 774, 774, 786, 767, 755, 748, 756, 792, 797, 792, 745, 772, 771, 702, 769, 783, 753, 778, 759, 723, 807, 742, 717, 764, 771, 713, 729, 754] payload_rows_per_dst=[9420, 9262, 9332, 9196, 9358, 9376, 9224, 9190] dup_rows_per_dst=[3120, 3148, 2874, 3028, 3182, 2938, 2852, 2804] combine_rows_per_src=[9420, 9262, 9332, 9196, 9358, 9376, 9224, 9190] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74368 payload_rows_remote=64956 dup_rows=23936 dup_rows_remote=20914 dedup_ratio=0.243490 router_tokens_per_expert=[775, 780, 762, 732, 773, 767, 774, 778, 718, 778, 761, 759, 781, 790, 802, 758, 749, 761, 779, 735, 767, 792, 780, 806, 780, 787, 767, 768, 781, 754, 763, 762, 814, 752, 749, 734, 801, 773, 752, 786, 740, 792, 751, 742, 772, 757, 750, 806, 823, 741, 763, 805, 755, 787, 764, 782, 729, 775, 766, 759, 737, 742, 745, 789] payload_rows_per_dst=[9412, 9320, 9324, 9288, 9348, 9182, 9378, 9116] dup_rows_per_dst=[3022, 2974, 3014, 3036, 2974, 2886, 3062, 2968] combine_rows_per_src=[9412, 9320, 9324, 9288, 9348, 9182, 9378, 9116] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265609216 dispatch_meta_bytes=343680 combine_payload_bytes=265609216 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265952896 ingress_bytes=369418240 link_bytes=369418240 token_bytes=531562112 weight_movement_bytes=103809024 total_nvlink_bytes=635371136 weight_to_token_ratio=0.195290 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390091097e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266903552 dispatch_meta_bytes=343968 combine_payload_bytes=266903552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267247520 ingress_bytes=301506560 link_bytes=301506560 token_bytes=534151072 weight_movement_bytes=34603008 total_nvlink_bytes=568754080 weight_to_token_ratio=0.064781 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134544912e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=267337728 dispatch_meta_bytes=344648 combine_payload_bytes=267337728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267682376 ingress_bytes=301940736 link_bytes=301940736 token_bytes=535020104 weight_movement_bytes=34603008 total_nvlink_bytes=569623112 weight_to_token_ratio=0.064676 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136178681e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266166272 dispatch_meta_bytes=343960 combine_payload_bytes=266166272 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266510232 ingress_bytes=335372288 link_bytes=335372288 token_bytes=532676504 weight_movement_bytes=69206016 total_nvlink_bytes=601882520 weight_to_token_ratio=0.129921 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261978920e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=343480 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266403256 ingress_bytes=300662784 link_bytes=300662784 token_bytes=532463032 weight_movement_bytes=34603008 total_nvlink_bytes=567066040 weight_to_token_ratio=0.064987 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131369851e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=343488 combine_payload_bytes=266641408 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266984896 ingress_bytes=335847424 link_bytes=335847424 token_bytes=533626304 weight_movement_bytes=69206016 total_nvlink_bytes=602832320 weight_to_token_ratio=0.129690 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263766819e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74394 payload_rows_remote=65000 dup_rows=23910 dup_rows_remote=20928 dedup_ratio=0.243225 router_tokens_per_expert=[782, 759, 797, 773, 777, 738, 747, 758, 780, 774, 728, 816, 800, 765, 783, 793, 763, 756, 794, 777, 742, 819, 741, 758, 771, 772, 788, 768, 755, 825, 743, 766, 755, 790, 730, 730, 732, 755, 797, 768, 713, 791, 789, 787, 742, 765, 789, 780, 773, 754, 734, 758, 810, 792, 770, 802, 757, 752, 776, 780, 734, 690, 705, 844] payload_rows_per_dst=[9268, 9424, 9238, 9394, 9202, 9368, 9276, 9224] dup_rows_per_dst=[2994, 3054, 3062, 2982, 2912, 2944, 3110, 2852] combine_rows_per_src=[9268, 9424, 9238, 9394, 9202, 9368, 9276, 9224] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64588 dup_rows=24068 dup_rows_remote=20734 dedup_ratio=0.244832 router_tokens_per_expert=[780, 762, 772, 717, 732, 810, 777, 742, 751, 804, 769, 751, 791, 775, 768, 770, 802, 813, 757, 808, 755, 751, 771, 755, 798, 793, 741, 739, 763, 753, 781, 764, 764, 736, 805, 754, 756, 752, 764, 787, 747, 744, 789, 774, 743, 755, 769, 757, 785, 754, 709, 740, 760, 779, 777, 769, 853, 785, 753, 804, 792, 769, 782, 730] payload_rows_per_dst=[9126, 9062, 9600, 9270, 9170, 9238, 9122, 9648] dup_rows_per_dst=[3058, 2800, 3308, 2958, 2868, 2926, 2816, 3334] combine_rows_per_src=[9126, 9062, 9600, 9270, 9170, 9238, 9122, 9648] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74162 payload_rows_remote=64934 dup_rows=24142 dup_rows_remote=21056 dedup_ratio=0.245585 router_tokens_per_expert=[769, 755, 747, 758, 774, 804, 788, 780, 781, 744, 782, 744, 764, 755, 792, 796, 710, 770, 740, 770, 757, 693, 795, 783, 781, 788, 754, 752, 760, 773, 740, 804, 775, 781, 750, 783, 732, 790, 740, 784, 776, 792, 759, 762, 797, 800, 752, 753, 739, 757, 797, 782, 756, 767, 796, 763, 788, 785, 792, 818, 763, 761, 707, 752] payload_rows_per_dst=[9336, 9242, 9180, 9290, 9306, 9362, 9228, 9218] dup_rows_per_dst=[3014, 3074, 2856, 3014, 2964, 3020, 3086, 3114] combine_rows_per_src=[9336, 9242, 9180, 9290, 9306, 9362, 9228, 9218] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74432 payload_rows_remote=65160 dup_rows=23872 dup_rows_remote=20754 dedup_ratio=0.242839 router_tokens_per_expert=[742, 814, 736, 751, 704, 724, 799, 792, 778, 773, 775, 753, 815, 754, 767, 772, 771, 744, 777, 753, 713, 740, 752, 750, 781, 778, 788, 757, 736, 791, 796, 767, 759, 734, 798, 762, 760, 821, 779, 782, 717, 803, 804, 784, 770, 754, 739, 784, 781, 793, 830, 761, 735, 794, 748, 767, 756, 784, 751, 759, 767, 752, 819, 762] payload_rows_per_dst=[9196, 9362, 9232, 9404, 9272, 9258, 9410, 9298] dup_rows_per_dst=[2928, 3012, 2768, 2984, 3118, 3052, 3008, 3002] combine_rows_per_src=[9196, 9362, 9232, 9404, 9272, 9258, 9410, 9298] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74474 payload_rows_remote=65158 dup_rows=23830 dup_rows_remote=20746 dedup_ratio=0.242411 router_tokens_per_expert=[752, 770, 790, 749, 782, 764, 720, 729, 741, 739, 785, 782, 804, 792, 770, 787, 748, 792, 780, 778, 801, 772, 767, 749, 772, 766, 780, 729, 775, 752, 719, 774, 735, 739, 756, 742, 724, 835, 776, 773, 742, 808, 758, 766, 749, 780, 761, 780, 810, 814, 755, 759, 812, 739, 788, 728, 757, 790, 757, 780, 794, 759, 749, 827] payload_rows_per_dst=[9108, 9316, 9368, 9250, 9306, 9402, 9394, 9330] dup_rows_per_dst=[3004, 3084, 3006, 2884, 2854, 2886, 3016, 3096] combine_rows_per_src=[9108, 9316, 9368, 9250, 9306, 9402, 9394, 9330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74394 payload_rows_remote=65000 dup_rows=23910 dup_rows_remote=20928 dedup_ratio=0.243225 router_tokens_per_expert=[782, 759, 797, 773, 777, 738, 747, 758, 780, 774, 728, 816, 800, 765, 783, 793, 763, 756, 794, 777, 742, 819, 741, 758, 771, 772, 788, 768, 755, 825, 743, 766, 755, 790, 730, 730, 732, 755, 797, 768, 713, 791, 789, 787, 742, 765, 789, 780, 773, 754, 734, 758, 810, 792, 770, 802, 757, 752, 776, 780, 734, 690, 705, 844] payload_rows_per_dst=[9268, 9424, 9238, 9394, 9202, 9368, 9276, 9224] dup_rows_per_dst=[2994, 3054, 3062, 2982, 2912, 2944, 3110, 2852] combine_rows_per_src=[9268, 9424, 9238, 9394, 9202, 9368, 9276, 9224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=64612 dup_rows=24028 dup_rows_remote=20504 dedup_ratio=0.244425 router_tokens_per_expert=[773, 788, 750, 746, 779, 801, 741, 761, 811, 786, 767, 765, 798, 754, 793, 783, 764, 784, 791, 781, 802, 794, 742, 810, 758, 742, 716, 761, 753, 707, 748, 772, 779, 760, 762, 729, 787, 804, 748, 731, 746, 777, 749, 786, 809, 759, 777, 804, 800, 749, 771, 765, 762, 744, 759, 773, 746, 790, 746, 797, 772, 732, 748, 770] payload_rows_per_dst=[9088, 9346, 8824, 9080, 9294, 9374, 9606, 9664] dup_rows_per_dst=[2698, 3058, 2726, 2834, 3004, 3052, 3132, 3524] combine_rows_per_src=[9088, 9346, 8824, 9080, 9294, 9374, 9606, 9664] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74268 payload_rows_remote=64996 dup_rows=24036 dup_rows_remote=20970 dedup_ratio=0.244507 router_tokens_per_expert=[735, 767, 754, 792, 802, 801, 787, 746, 761, 806, 793, 725, 773, 770, 760, 781, 748, 799, 816, 813, 812, 790, 765, 794, 771, 767, 805, 734, 752, 727, 768, 734, 728, 804, 763, 744, 725, 722, 783, 732, 786, 772, 744, 787, 733, 768, 771, 787, 795, 757, 742, 820, 720, 798, 776, 813, 761, 771, 707, 802, 737, 770, 758, 728] payload_rows_per_dst=[9364, 9272, 9468, 9166, 9128, 9336, 9372, 9162] dup_rows_per_dst=[3004, 3066, 3206, 2950, 2874, 2960, 3070, 2906] combine_rows_per_src=[9364, 9272, 9468, 9166, 9128, 9336, 9372, 9162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265969664 dispatch_meta_bytes=343960 combine_payload_bytes=265969664 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266313624 ingress_bytes=335175680 link_bytes=335175680 token_bytes=532283288 weight_movement_bytes=69206016 total_nvlink_bytes=601489304 weight_to_token_ratio=0.130017 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261239100e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266895360 dispatch_meta_bytes=343656 combine_payload_bytes=266895360 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267239016 ingress_bytes=336101376 link_bytes=336101376 token_bytes=534134376 weight_movement_bytes=69206016 total_nvlink_bytes=603340392 weight_to_token_ratio=0.129567 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264722420e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=264650752 dispatch_meta_bytes=340464 combine_payload_bytes=264650752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264991216 ingress_bytes=299253760 link_bytes=299253760 token_bytes=529641968 weight_movement_bytes=34603008 total_nvlink_bytes=564244976 weight_to_token_ratio=0.065333 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126067807e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266240000 dispatch_meta_bytes=343712 combine_payload_bytes=266240000 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266583712 ingress_bytes=335446016 link_bytes=335446016 token_bytes=532823712 weight_movement_bytes=69206016 total_nvlink_bytes=602029728 weight_to_token_ratio=0.129885 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262256353e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=343864 combine_payload_bytes=266223616 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266567480 ingress_bytes=370032640 link_bytes=370032640 token_bytes=532791096 weight_movement_bytes=103809024 total_nvlink_bytes=636600120 weight_to_token_ratio=0.194840 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392403034e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74480 payload_rows_remote=65210 dup_rows=23824 dup_rows_remote=20826 dedup_ratio=0.242350 router_tokens_per_expert=[752, 768, 778, 815, 795, 775, 772, 744, 732, 769, 738, 748, 770, 750, 723, 774, 753, 706, 779, 791, 795, 740, 790, 780, 790, 773, 795, 789, 757, 786, 794, 743, 782, 808, 741, 780, 769, 765, 757, 814, 766, 796, 796, 742, 754, 790, 738, 788, 729, 779, 736, 792, 763, 749, 752, 783, 723, 757, 761, 771, 759, 755, 762, 831] payload_rows_per_dst=[9452, 9128, 9270, 9442, 9364, 9288, 9248, 9288] dup_rows_per_dst=[2946, 2880, 2998, 3012, 3068, 3052, 2918, 2950] combine_rows_per_src=[9452, 9128, 9270, 9442, 9364, 9288, 9248, 9288] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=65012 dup_rows=23950 dup_rows_remote=20898 dedup_ratio=0.243632 router_tokens_per_expert=[764, 829, 747, 835, 747, 735, 809, 721, 736, 762, 759, 739, 749, 765, 791, 791, 777, 796, 781, 723, 739, 773, 769, 732, 781, 774, 762, 773, 792, 756, 766, 793, 778, 808, 793, 784, 757, 700, 791, 740, 795, 794, 829, 754, 786, 812, 734, 750, 774, 746, 779, 783, 746, 787, 759, 755, 774, 815, 684, 788, 755, 737, 762, 737] payload_rows_per_dst=[9342, 9252, 9476, 9434, 9076, 9238, 9324, 9212] dup_rows_per_dst=[3052, 2912, 3078, 3206, 2852, 3024, 2934, 2892] combine_rows_per_src=[9342, 9252, 9476, 9434, 9076, 9238, 9324, 9212] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74128 payload_rows_remote=64940 dup_rows=24176 dup_rows_remote=21174 dedup_ratio=0.245931 router_tokens_per_expert=[775, 745, 749, 771, 774, 757, 800, 746, 790, 755, 786, 783, 776, 795, 822, 750, 798, 785, 751, 745, 817, 775, 779, 752, 745, 777, 778, 813, 792, 794, 718, 792, 732, 749, 778, 786, 763, 769, 782, 771, 750, 763, 754, 745, 801, 789, 732, 761, 786, 752, 764, 781, 740, 796, 752, 720, 757, 716, 762, 803, 781, 731, 761, 740] payload_rows_per_dst=[9214, 9438, 9386, 9302, 9226, 9188, 9198, 9176] dup_rows_per_dst=[3020, 3076, 3018, 3116, 3034, 3002, 2984, 2926] combine_rows_per_src=[9214, 9438, 9386, 9302, 9226, 9188, 9198, 9176] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266240000 dispatch_meta_bytes=343712 combine_payload_bytes=266240000 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266583712 ingress_bytes=335446016 link_bytes=335446016 token_bytes=532823712 weight_movement_bytes=69206016 total_nvlink_bytes=602029728 weight_to_token_ratio=0.129885 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262256353e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=264552448 dispatch_meta_bytes=341288 combine_payload_bytes=264552448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264893736 ingress_bytes=299155456 link_bytes=299155456 token_bytes=529446184 weight_movement_bytes=34603008 total_nvlink_bytes=564049192 weight_to_token_ratio=0.065357 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125697897e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=65012 dup_rows=23950 dup_rows_remote=20898 dedup_ratio=0.243632 router_tokens_per_expert=[764, 829, 747, 835, 747, 735, 809, 721, 736, 762, 759, 739, 749, 765, 791, 791, 777, 796, 781, 723, 739, 773, 769, 732, 781, 774, 762, 773, 792, 756, 766, 793, 778, 808, 793, 784, 757, 700, 791, 740, 795, 794, 829, 754, 786, 812, 734, 750, 774, 746, 779, 783, 746, 787, 759, 755, 774, 815, 684, 788, 755, 737, 762, 737] payload_rows_per_dst=[9342, 9252, 9476, 9434, 9076, 9238, 9324, 9212] dup_rows_per_dst=[3052, 2912, 3078, 3206, 2852, 3024, 2934, 2892] combine_rows_per_src=[9342, 9252, 9476, 9434, 9076, 9238, 9324, 9212] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266895360 dispatch_meta_bytes=343656 combine_payload_bytes=266895360 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267239016 ingress_bytes=336101376 link_bytes=336101376 token_bytes=534134376 weight_movement_bytes=69206016 total_nvlink_bytes=603340392 weight_to_token_ratio=0.129567 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264722420e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74182 payload_rows_remote=64886 dup_rows=24122 dup_rows_remote=21096 dedup_ratio=0.245382 router_tokens_per_expert=[767, 775, 765, 754, 776, 809, 784, 788, 736, 745, 801, 800, 748, 764, 751, 776, 762, 787, 766, 752, 807, 770, 750, 769, 778, 747, 754, 800, 777, 787, 763, 752, 762, 795, 781, 750, 809, 759, 787, 760, 769, 783, 764, 757, 740, 766, 773, 756, 782, 831, 762, 735, 714, 773, 790, 774, 772, 738, 753, 737, 773, 750, 772, 725] payload_rows_per_dst=[9332, 9332, 9264, 9334, 9332, 9244, 9296, 9048] dup_rows_per_dst=[3104, 2910, 3062, 2982, 3074, 2972, 3026, 2992] combine_rows_per_src=[9332, 9332, 9264, 9334, 9332, 9244, 9296, 9048] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266887168 dispatch_meta_bytes=343616 combine_payload_bytes=266887168 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267230784 ingress_bytes=370696192 link_bytes=370696192 token_bytes=534117952 weight_movement_bytes=103809024 total_nvlink_bytes=637926976 weight_to_token_ratio=0.194356 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394899927e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267100160 dispatch_meta_bytes=344144 combine_payload_bytes=267100160 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267444304 ingress_bytes=336306176 link_bytes=336306176 token_bytes=534544464 weight_movement_bytes=69206016 total_nvlink_bytes=603750480 weight_to_token_ratio=0.129467 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265493065e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266289152 dispatch_meta_bytes=343640 combine_payload_bytes=266289152 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266632792 ingress_bytes=335495168 link_bytes=335495168 token_bytes=532921944 weight_movement_bytes=69206016 total_nvlink_bytes=602127960 weight_to_token_ratio=0.129861 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262441308e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=344456 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266338696 ingress_bytes=335200256 link_bytes=335200256 token_bytes=532332936 weight_movement_bytes=69206016 total_nvlink_bytes=601538952 weight_to_token_ratio=0.130005 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261331577e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=343928 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266116984 ingress_bytes=300376064 link_bytes=300376064 token_bytes=531890040 weight_movement_bytes=34603008 total_nvlink_bytes=566493048 weight_to_token_ratio=0.065057 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130290947e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=64612 dup_rows=24028 dup_rows_remote=20504 dedup_ratio=0.244425 router_tokens_per_expert=[773, 788, 750, 746, 779, 801, 741, 761, 811, 786, 767, 765, 798, 754, 793, 783, 764, 784, 791, 781, 802, 794, 742, 810, 758, 742, 716, 761, 753, 707, 748, 772, 779, 760, 762, 729, 787, 804, 748, 731, 746, 777, 749, 786, 809, 759, 777, 804, 800, 749, 771, 765, 762, 744, 759, 773, 746, 790, 746, 797, 772, 732, 748, 770] payload_rows_per_dst=[9088, 9346, 8824, 9080, 9294, 9374, 9606, 9664] dup_rows_per_dst=[2698, 3058, 2726, 2834, 3004, 3052, 3132, 3524] combine_rows_per_src=[9088, 9346, 8824, 9080, 9294, 9374, 9606, 9664] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74308 payload_rows_remote=65206 dup_rows=23996 dup_rows_remote=21090 dedup_ratio=0.244100 router_tokens_per_expert=[776, 741, 803, 745, 711, 762, 795, 743, 799, 751, 779, 784, 773, 768, 748, 766, 755, 770, 799, 809, 795, 767, 795, 804, 745, 766, 752, 777, 731, 715, 731, 787, 852, 801, 806, 799, 780, 800, 749, 749, 735, 750, 779, 757, 767, 752, 774, 797, 746, 767, 734, 771, 779, 750, 755, 795, 747, 754, 736, 752, 803, 754, 790, 730] payload_rows_per_dst=[9286, 9262, 9486, 9102, 9472, 9298, 9180, 9222] dup_rows_per_dst=[2866, 3074, 3102, 2906, 3200, 2924, 3014, 2910] combine_rows_per_src=[9286, 9262, 9486, 9102, 9472, 9298, 9180, 9222] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74442 payload_rows_remote=65180 dup_rows=23862 dup_rows_remote=20832 dedup_ratio=0.242737 router_tokens_per_expert=[796, 743, 790, 800, 768, 776, 801, 744, 760, 763, 795, 766, 752, 805, 800, 714, 731, 754, 719, 739, 740, 772, 796, 721, 814, 773, 825, 783, 780, 760, 753, 761, 748, 733, 730, 752, 790, 721, 760, 803, 728, 767, 808, 742, 781, 772, 786, 775, 784, 789, 781, 768, 728, 770, 756, 770, 784, 799, 772, 750, 790, 731, 828, 762] payload_rows_per_dst=[9404, 9310, 9082, 9448, 9154, 9344, 9262, 9438] dup_rows_per_dst=[3032, 3000, 2862, 3050, 2920, 2974, 3030, 2994] combine_rows_per_src=[9404, 9310, 9082, 9448, 9154, 9344, 9262, 9438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266289152 dispatch_meta_bytes=343640 combine_payload_bytes=266289152 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266632792 ingress_bytes=335495168 link_bytes=335495168 token_bytes=532921944 weight_movement_bytes=69206016 total_nvlink_bytes=602127960 weight_to_token_ratio=0.129861 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262441308e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74114 payload_rows_remote=64676 dup_rows=24190 dup_rows_remote=20990 dedup_ratio=0.246073 router_tokens_per_expert=[807, 797, 787, 764, 775, 775, 784, 779, 737, 835, 746, 776, 733, 768, 768, 752, 787, 755, 825, 763, 762, 747, 752, 770, 786, 795, 759, 766, 745, 785, 788, 704, 776, 795, 806, 772, 802, 820, 768, 780, 773, 777, 819, 737, 755, 772, 743, 749, 731, 752, 774, 739, 710, 763, 771, 776, 709, 755, 757, 775, 756, 737, 775, 756] payload_rows_per_dst=[9364, 9220, 9232, 9240, 9438, 9268, 9170, 9182] dup_rows_per_dst=[3172, 3010, 3090, 3016, 3200, 2982, 2862, 2858] combine_rows_per_src=[9364, 9220, 9232, 9240, 9438, 9268, 9170, 9182] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74268 payload_rows_remote=64996 dup_rows=24036 dup_rows_remote=20970 dedup_ratio=0.244507 router_tokens_per_expert=[735, 767, 754, 792, 802, 801, 787, 746, 761, 806, 793, 725, 773, 770, 760, 781, 748, 799, 816, 813, 812, 790, 765, 794, 771, 767, 805, 734, 752, 727, 768, 734, 728, 804, 763, 744, 725, 722, 783, 732, 786, 772, 744, 787, 733, 768, 771, 787, 795, 757, 742, 820, 720, 798, 776, 813, 761, 771, 707, 802, 737, 770, 758, 728] payload_rows_per_dst=[9364, 9272, 9468, 9166, 9128, 9336, 9372, 9162] dup_rows_per_dst=[3004, 3066, 3206, 2950, 2874, 2960, 3070, 2906] combine_rows_per_src=[9364, 9272, 9468, 9166, 9128, 9336, 9372, 9162] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74090 payload_rows_remote=64840 dup_rows=24214 dup_rows_remote=21156 dedup_ratio=0.246318 router_tokens_per_expert=[767, 730, 757, 758, 772, 796, 748, 774, 754, 774, 755, 716, 753, 797, 796, 747, 790, 766, 750, 802, 758, 783, 755, 750, 793, 747, 778, 777, 776, 769, 753, 734, 840, 802, 793, 762, 789, 786, 733, 773, 753, 782, 744, 782, 790, 728, 745, 759, 777, 807, 729, 791, 755, 746, 802, 731, 778, 736, 730, 786, 838, 759, 766, 785] payload_rows_per_dst=[9150, 9166, 9250, 9242, 9360, 9284, 9272, 9366] dup_rows_per_dst=[3054, 3018, 3058, 3012, 3196, 2882, 3004, 2990] combine_rows_per_src=[9150, 9166, 9250, 9242, 9360, 9284, 9272, 9366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74550 payload_rows_remote=65086 dup_rows=23754 dup_rows_remote=20644 dedup_ratio=0.241638 router_tokens_per_expert=[752, 801, 839, 772, 760, 723, 783, 817, 760, 722, 769, 750, 770, 769, 794, 731, 720, 763, 807, 789, 718, 805, 747, 780, 769, 807, 717, 765, 796, 764, 752, 727, 799, 781, 775, 729, 748, 830, 753, 742, 766, 766, 782, 750, 769, 791, 728, 775, 786, 729, 747, 787, 757, 768, 744, 725, 808, 776, 761, 790, 776, 807, 781, 788] payload_rows_per_dst=[9382, 9204, 9306, 9278, 9252, 9292, 9372, 9464] dup_rows_per_dst=[3020, 2868, 3010, 2916, 2918, 2962, 2950, 3110] combine_rows_per_src=[9382, 9204, 9306, 9278, 9252, 9292, 9372, 9464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74318 payload_rows_remote=64924 dup_rows=23986 dup_rows_remote=20998 dedup_ratio=0.243998 router_tokens_per_expert=[773, 767, 735, 741, 768, 791, 776, 830, 753, 760, 766, 771, 791, 769, 815, 766, 782, 757, 753, 821, 710, 789, 760, 790, 762, 772, 778, 791, 723, 779, 756, 768, 746, 777, 806, 765, 835, 750, 811, 814, 720, 758, 701, 796, 775, 712, 786, 766, 747, 760, 734, 757, 768, 826, 760, 771, 764, 781, 727, 743, 762, 758, 765, 748] payload_rows_per_dst=[9458, 9394, 9322, 9284, 9416, 9098, 9272, 9074] dup_rows_per_dst=[2904, 2988, 3002, 2974, 3192, 2930, 2974, 3022] combine_rows_per_src=[9458, 9394, 9322, 9284, 9416, 9098, 9272, 9074] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74308 payload_rows_remote=65206 dup_rows=23996 dup_rows_remote=21090 dedup_ratio=0.244100 router_tokens_per_expert=[776, 741, 803, 745, 711, 762, 795, 743, 799, 751, 779, 784, 773, 768, 748, 766, 755, 770, 799, 809, 795, 767, 795, 804, 745, 766, 752, 777, 731, 715, 731, 787, 852, 801, 806, 799, 780, 800, 749, 749, 735, 750, 779, 757, 767, 752, 774, 797, 746, 767, 734, 771, 779, 750, 755, 795, 747, 754, 736, 752, 803, 754, 790, 730] payload_rows_per_dst=[9286, 9262, 9486, 9102, 9472, 9298, 9180, 9222] dup_rows_per_dst=[2866, 3074, 3102, 2906, 3200, 2924, 3014, 2910] combine_rows_per_src=[9286, 9262, 9486, 9102, 9472, 9298, 9180, 9222] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=65166 dup_rows=23860 dup_rows_remote=20934 dedup_ratio=0.242716 router_tokens_per_expert=[709, 769, 758, 763, 760, 780, 755, 802, 780, 762, 755, 782, 767, 759, 765, 783, 773, 770, 705, 815, 766, 792, 797, 770, 714, 817, 745, 731, 776, 787, 762, 719, 762, 782, 808, 752, 735, 809, 774, 735, 747, 784, 817, 770, 802, 788, 793, 741, 794, 761, 775, 718, 762, 793, 758, 766, 810, 759, 741, 751, 777, 772, 779, 749] payload_rows_per_dst=[9278, 9262, 9340, 9402, 9232, 9574, 9324, 9032] dup_rows_per_dst=[2926, 3038, 3036, 2916, 2866, 3334, 2930, 2814] combine_rows_per_src=[9278, 9262, 9340, 9402, 9232, 9574, 9324, 9032] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266977280 dispatch_meta_bytes=344048 combine_payload_bytes=266977280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267321328 ingress_bytes=301580288 link_bytes=301580288 token_bytes=534298608 weight_movement_bytes=34603008 total_nvlink_bytes=568901616 weight_to_token_ratio=0.064763 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134822345e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=264912896 dispatch_meta_bytes=342664 combine_payload_bytes=264912896 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265255560 ingress_bytes=334118912 link_bytes=334118912 token_bytes=530168456 weight_movement_bytes=69206016 total_nvlink_bytes=599374472 weight_to_token_ratio=0.130536 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257262567e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266592256 dispatch_meta_bytes=342920 combine_payload_bytes=266592256 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266935176 ingress_bytes=335798272 link_bytes=335798272 token_bytes=533527432 weight_movement_bytes=69206016 total_nvlink_bytes=602733448 weight_to_token_ratio=0.129714 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263581864e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74090 payload_rows_remote=64840 dup_rows=24214 dup_rows_remote=21156 dedup_ratio=0.246318 router_tokens_per_expert=[767, 730, 757, 758, 772, 796, 748, 774, 754, 774, 755, 716, 753, 797, 796, 747, 790, 766, 750, 802, 758, 783, 755, 750, 793, 747, 778, 777, 776, 769, 753, 734, 840, 802, 793, 762, 789, 786, 733, 773, 753, 782, 744, 782, 790, 728, 745, 759, 777, 807, 729, 791, 755, 746, 802, 731, 778, 736, 730, 786, 838, 759, 766, 785] payload_rows_per_dst=[9150, 9166, 9250, 9242, 9360, 9284, 9272, 9366] dup_rows_per_dst=[3054, 3018, 3058, 3012, 3196, 2882, 3004, 2990] combine_rows_per_src=[9150, 9166, 9250, 9242, 9360, 9284, 9272, 9366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265928704 dispatch_meta_bytes=343688 combine_payload_bytes=265928704 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266272392 ingress_bytes=369737728 link_bytes=369737728 token_bytes=532201096 weight_movement_bytes=103809024 total_nvlink_bytes=636010120 weight_to_token_ratio=0.195056 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391293304e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267083776 dispatch_meta_bytes=345184 combine_payload_bytes=267083776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267428960 ingress_bytes=301686784 link_bytes=301686784 token_bytes=534512736 weight_movement_bytes=34603008 total_nvlink_bytes=569115744 weight_to_token_ratio=0.064737 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135223080e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74408 payload_rows_remote=65140 dup_rows=23896 dup_rows_remote=20924 dedup_ratio=0.243083 router_tokens_per_expert=[780, 787, 738, 774, 739, 728, 772, 812, 800, 792, 791, 722, 759, 775, 772, 738, 798, 761, 743, 749, 782, 760, 789, 728, 768, 779, 749, 788, 775, 776, 727, 774, 711, 782, 779, 806, 753, 735, 731, 762, 740, 765, 770, 830, 799, 750, 724, 742, 757, 765, 789, 757, 792, 802, 795, 772, 755, 749, 800, 785, 778, 766, 791, 795] payload_rows_per_dst=[9238, 9338, 9220, 9300, 9296, 9268, 9396, 9352] dup_rows_per_dst=[3022, 2960, 3000, 2972, 2822, 2972, 3062, 3086] combine_rows_per_src=[9238, 9338, 9220, 9300, 9296, 9268, 9396, 9352] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=264650752 dispatch_meta_bytes=340464 combine_payload_bytes=264650752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264991216 ingress_bytes=299253760 link_bytes=299253760 token_bytes=529641968 weight_movement_bytes=34603008 total_nvlink_bytes=564244976 weight_to_token_ratio=0.065333 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126067807e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=65166 dup_rows=23860 dup_rows_remote=20934 dedup_ratio=0.242716 router_tokens_per_expert=[709, 769, 758, 763, 760, 780, 755, 802, 780, 762, 755, 782, 767, 759, 765, 783, 773, 770, 705, 815, 766, 792, 797, 770, 714, 817, 745, 731, 776, 787, 762, 719, 762, 782, 808, 752, 735, 809, 774, 735, 747, 784, 817, 770, 802, 788, 793, 741, 794, 761, 775, 718, 762, 793, 758, 766, 810, 759, 741, 751, 777, 772, 779, 749] payload_rows_per_dst=[9278, 9262, 9340, 9402, 9232, 9574, 9324, 9032] dup_rows_per_dst=[2926, 3038, 3036, 2916, 2866, 3334, 2930, 2814] combine_rows_per_src=[9278, 9262, 9340, 9402, 9232, 9574, 9324, 9032] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74162 payload_rows_remote=64934 dup_rows=24142 dup_rows_remote=21056 dedup_ratio=0.245585 router_tokens_per_expert=[769, 755, 747, 758, 774, 804, 788, 780, 781, 744, 782, 744, 764, 755, 792, 796, 710, 770, 740, 770, 757, 693, 795, 783, 781, 788, 754, 752, 760, 773, 740, 804, 775, 781, 750, 783, 732, 790, 740, 784, 776, 792, 759, 762, 797, 800, 752, 753, 739, 757, 797, 782, 756, 767, 796, 763, 788, 785, 792, 818, 763, 761, 707, 752] payload_rows_per_dst=[9336, 9242, 9180, 9290, 9306, 9362, 9228, 9218] dup_rows_per_dst=[3014, 3074, 2856, 3014, 2964, 3020, 3086, 3114] combine_rows_per_src=[9336, 9242, 9180, 9290, 9306, 9362, 9228, 9218] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267083776 dispatch_meta_bytes=345184 combine_payload_bytes=267083776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267428960 ingress_bytes=301686784 link_bytes=301686784 token_bytes=534512736 weight_movement_bytes=34603008 total_nvlink_bytes=569115744 weight_to_token_ratio=0.064737 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135223080e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=264912896 dispatch_meta_bytes=342664 combine_payload_bytes=264912896 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265255560 ingress_bytes=334118912 link_bytes=334118912 token_bytes=530168456 weight_movement_bytes=69206016 total_nvlink_bytes=599374472 weight_to_token_ratio=0.130536 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257262567e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=343864 combine_payload_bytes=266223616 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266567480 ingress_bytes=370032640 link_bytes=370032640 token_bytes=532791096 weight_movement_bytes=103809024 total_nvlink_bytes=636600120 weight_to_token_ratio=0.194840 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392403034e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266919936 dispatch_meta_bytes=344400 combine_payload_bytes=266919936 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267264336 ingress_bytes=336125952 link_bytes=336125952 token_bytes=534184272 weight_movement_bytes=69206016 total_nvlink_bytes=603390288 weight_to_token_ratio=0.129555 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264814897e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265584640 dispatch_meta_bytes=343984 combine_payload_bytes=265584640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265928624 ingress_bytes=300187648 link_bytes=300187648 token_bytes=531513264 weight_movement_bytes=34603008 total_nvlink_bytes=566116272 weight_to_token_ratio=0.065103 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129581952e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266813440 dispatch_meta_bytes=344256 combine_payload_bytes=266813440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267157696 ingress_bytes=301416448 link_bytes=301416448 token_bytes=533971136 weight_movement_bytes=34603008 total_nvlink_bytes=568574144 weight_to_token_ratio=0.064803 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134205828e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265969664 dispatch_meta_bytes=343960 combine_payload_bytes=265969664 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266313624 ingress_bytes=335175680 link_bytes=335175680 token_bytes=532283288 weight_movement_bytes=69206016 total_nvlink_bytes=601489304 weight_to_token_ratio=0.130017 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261239100e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74550 payload_rows_remote=65086 dup_rows=23754 dup_rows_remote=20644 dedup_ratio=0.241638 router_tokens_per_expert=[752, 801, 839, 772, 760, 723, 783, 817, 760, 722, 769, 750, 770, 769, 794, 731, 720, 763, 807, 789, 718, 805, 747, 780, 769, 807, 717, 765, 796, 764, 752, 727, 799, 781, 775, 729, 748, 830, 753, 742, 766, 766, 782, 750, 769, 791, 728, 775, 786, 729, 747, 787, 757, 768, 744, 725, 808, 776, 761, 790, 776, 807, 781, 788] payload_rows_per_dst=[9382, 9204, 9306, 9278, 9252, 9292, 9372, 9464] dup_rows_per_dst=[3020, 2868, 3010, 2916, 2918, 2962, 2950, 3110] combine_rows_per_src=[9382, 9204, 9306, 9278, 9252, 9292, 9372, 9464] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74246 payload_rows_remote=65008 dup_rows=24058 dup_rows_remote=21102 dedup_ratio=0.244731 router_tokens_per_expert=[734, 780, 798, 753, 784, 757, 775, 750, 747, 835, 774, 769, 746, 784, 776, 791, 738, 785, 752, 796, 720, 758, 816, 774, 783, 743, 785, 752, 690, 790, 778, 776, 786, 736, 757, 748, 785, 833, 734, 751, 799, 801, 730, 774, 799, 745, 758, 750, 747, 758, 768, 825, 777, 756, 690, 790, 760, 761, 752, 758, 774, 792, 796, 773] payload_rows_per_dst=[9296, 9272, 9282, 9238, 9224, 9338, 9246, 9350] dup_rows_per_dst=[2966, 3172, 2996, 2956, 3036, 2974, 2976, 2982] combine_rows_per_src=[9296, 9272, 9282, 9238, 9224, 9338, 9246, 9350] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266919936 dispatch_meta_bytes=344400 combine_payload_bytes=266919936 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267264336 ingress_bytes=336125952 link_bytes=336125952 token_bytes=534184272 weight_movement_bytes=69206016 total_nvlink_bytes=603390288 weight_to_token_ratio=0.129555 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264814897e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74308 payload_rows_remote=64882 dup_rows=23996 dup_rows_remote=21070 dedup_ratio=0.244100 router_tokens_per_expert=[754, 795, 750, 738, 765, 740, 779, 787, 747, 724, 820, 747, 827, 776, 761, 755, 746, 762, 785, 786, 730, 765, 744, 760, 838, 763, 767, 784, 789, 800, 758, 776, 751, 747, 763, 801, 759, 759, 814, 759, 755, 754, 775, 778, 729, 755, 755, 738, 780, 775, 783, 772, 746, 763, 770, 787, 747, 786, 727, 765, 821, 781, 796, 743] payload_rows_per_dst=[9238, 9276, 9196, 9332, 9232, 9244, 9426, 9364] dup_rows_per_dst=[2978, 3038, 2960, 3218, 3074, 2834, 2926, 2968] combine_rows_per_src=[9238, 9276, 9196, 9332, 9232, 9244, 9426, 9364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74562 payload_rows_remote=65048 dup_rows=23742 dup_rows_remote=20616 dedup_ratio=0.241516 router_tokens_per_expert=[779, 735, 774, 769, 751, 727, 740, 773, 783, 783, 728, 743, 769, 762, 776, 760, 801, 781, 781, 767, 781, 743, 801, 716, 777, 749, 818, 763, 746, 793, 794, 780, 773, 774, 749, 789, 785, 740, 751, 742, 764, 750, 766, 810, 792, 752, 748, 817, 714, 727, 763, 787, 726, 766, 740, 764, 785, 782, 822, 828, 809, 770, 760, 764] payload_rows_per_dst=[9238, 9264, 9330, 9164, 9330, 9382, 9340, 9514] dup_rows_per_dst=[2858, 2944, 3012, 2880, 2966, 3016, 2940, 3126] combine_rows_per_src=[9238, 9264, 9330, 9164, 9330, 9382, 9340, 9514] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74260 payload_rows_remote=65134 dup_rows=24044 dup_rows_remote=20982 dedup_ratio=0.244588 router_tokens_per_expert=[773, 772, 758, 713, 845, 791, 782, 763, 763, 796, 761, 777, 759, 759, 749, 791, 744, 771, 800, 785, 728, 790, 747, 798, 810, 732, 775, 732, 802, 748, 797, 736, 776, 755, 779, 782, 752, 714, 763, 773, 713, 796, 764, 815, 821, 734, 756, 760, 785, 742, 755, 777, 774, 747, 739, 768, 799, 693, 781, 827, 780, 775, 721, 789] payload_rows_per_dst=[9290, 9278, 9310, 9306, 9126, 9328, 9328, 9294] dup_rows_per_dst=[3104, 3032, 3016, 2958, 3062, 2990, 2846, 3036] combine_rows_per_src=[9290, 9278, 9310, 9306, 9126, 9328, 9328, 9294] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74080 payload_rows_remote=64356 dup_rows=24224 dup_rows_remote=20922 dedup_ratio=0.246419 router_tokens_per_expert=[770, 781, 756, 765, 734, 767, 789, 788, 758, 763, 793, 782, 757, 761, 742, 743, 804, 775, 779, 760, 811, 749, 785, 795, 771, 738, 777, 746, 784, 803, 757, 781, 739, 784, 764, 745, 802, 750, 788, 783, 743, 775, 737, 784, 774, 740, 762, 745, 808, 720, 743, 757, 743, 758, 775, 760, 755, 754, 785, 794, 797, 729, 768, 827] payload_rows_per_dst=[9724, 9352, 9440, 9310, 9286, 9060, 9122, 8786] dup_rows_per_dst=[3302, 3110, 3266, 3012, 3024, 2910, 3006, 2594] combine_rows_per_src=[9724, 9352, 9440, 9310, 9286, 9060, 9122, 8786] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74318 payload_rows_remote=64924 dup_rows=23986 dup_rows_remote=20998 dedup_ratio=0.243998 router_tokens_per_expert=[773, 767, 735, 741, 768, 791, 776, 830, 753, 760, 766, 771, 791, 769, 815, 766, 782, 757, 753, 821, 710, 789, 760, 790, 762, 772, 778, 791, 723, 779, 756, 768, 746, 777, 806, 765, 835, 750, 811, 814, 720, 758, 701, 796, 775, 712, 786, 766, 747, 760, 734, 757, 768, 826, 760, 771, 764, 781, 727, 743, 762, 758, 765, 748] payload_rows_per_dst=[9458, 9394, 9322, 9284, 9416, 9098, 9272, 9074] dup_rows_per_dst=[2904, 2988, 3002, 2974, 3192, 2930, 2974, 3022] combine_rows_per_src=[9458, 9394, 9322, 9284, 9416, 9098, 9272, 9074] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74314 payload_rows_remote=65098 dup_rows=23990 dup_rows_remote=21080 dedup_ratio=0.244039 router_tokens_per_expert=[773, 770, 790, 802, 717, 755, 769, 744, 754, 780, 742, 805, 785, 753, 725, 760, 783, 777, 761, 764, 756, 726, 762, 734, 801, 796, 741, 768, 736, 777, 778, 783, 764, 778, 772, 758, 731, 768, 747, 773, 776, 756, 766, 795, 758, 774, 775, 741, 747, 797, 774, 782, 816, 769, 760, 790, 738, 787, 760, 796, 822, 729, 811, 775] payload_rows_per_dst=[9300, 9190, 9216, 9340, 9192, 9370, 9394, 9312] dup_rows_per_dst=[2940, 3018, 2910, 3020, 2990, 2912, 3076, 3124] combine_rows_per_src=[9300, 9190, 9216, 9340, 9192, 9370, 9394, 9312] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266592256 dispatch_meta_bytes=342920 combine_payload_bytes=266592256 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266935176 ingress_bytes=335798272 link_bytes=335798272 token_bytes=533527432 weight_movement_bytes=69206016 total_nvlink_bytes=602733448 weight_to_token_ratio=0.129714 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263581864e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74218 payload_rows_remote=65064 dup_rows=24086 dup_rows_remote=21080 dedup_ratio=0.245015 router_tokens_per_expert=[727, 734, 819, 751, 770, 730, 775, 778, 793, 769, 827, 723, 751, 735, 709, 773, 771, 791, 771, 752, 771, 800, 759, 827, 798, 760, 761, 709, 775, 794, 756, 774, 725, 798, 785, 780, 783, 752, 792, 771, 755, 794, 784, 737, 749, 719, 777, 783, 754, 771, 793, 737, 811, 781, 739, 777, 806, 725, 827, 774, 738, 792, 738, 772] payload_rows_per_dst=[9258, 9154, 9408, 9266, 9302, 9190, 9318, 9322] dup_rows_per_dst=[2910, 3006, 3076, 2988, 3070, 3006, 3008, 3022] combine_rows_per_src=[9258, 9154, 9408, 9266, 9302, 9190, 9318, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74246 payload_rows_remote=65008 dup_rows=24058 dup_rows_remote=21102 dedup_ratio=0.244731 router_tokens_per_expert=[734, 780, 798, 753, 784, 757, 775, 750, 747, 835, 774, 769, 746, 784, 776, 791, 738, 785, 752, 796, 720, 758, 816, 774, 783, 743, 785, 752, 690, 790, 778, 776, 786, 736, 757, 748, 785, 833, 734, 751, 799, 801, 730, 774, 799, 745, 758, 750, 747, 758, 768, 825, 777, 756, 690, 790, 760, 761, 752, 758, 774, 792, 796, 773] payload_rows_per_dst=[9296, 9272, 9282, 9238, 9224, 9338, 9246, 9350] dup_rows_per_dst=[2966, 3172, 2996, 2956, 3036, 2974, 2976, 2982] combine_rows_per_src=[9296, 9272, 9282, 9238, 9224, 9338, 9246, 9350] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=343808 combine_payload_bytes=265756672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266100480 ingress_bytes=300359680 link_bytes=300359680 token_bytes=531857152 weight_movement_bytes=34603008 total_nvlink_bytes=566460160 weight_to_token_ratio=0.065061 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130229295e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266436608 dispatch_meta_bytes=342656 combine_payload_bytes=266436608 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266779264 ingress_bytes=404848640 link_bytes=404848640 token_bytes=533215872 weight_movement_bytes=138412032 total_nvlink_bytes=671627904 weight_to_token_ratio=0.259580 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523412839e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74314 payload_rows_remote=65098 dup_rows=23990 dup_rows_remote=21080 dedup_ratio=0.244039 router_tokens_per_expert=[773, 770, 790, 802, 717, 755, 769, 744, 754, 780, 742, 805, 785, 753, 725, 760, 783, 777, 761, 764, 756, 726, 762, 734, 801, 796, 741, 768, 736, 777, 778, 783, 764, 778, 772, 758, 731, 768, 747, 773, 776, 756, 766, 795, 758, 774, 775, 741, 747, 797, 774, 782, 816, 769, 760, 790, 738, 787, 760, 796, 822, 729, 811, 775] payload_rows_per_dst=[9300, 9190, 9216, 9340, 9192, 9370, 9394, 9312] dup_rows_per_dst=[2940, 3018, 2910, 3020, 2990, 2912, 3076, 3124] combine_rows_per_src=[9300, 9190, 9216, 9340, 9192, 9370, 9394, 9312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=344464 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267133328 ingress_bytes=370597888 link_bytes=370597888 token_bytes=533922192 weight_movement_bytes=103809024 total_nvlink_bytes=637731216 weight_to_token_ratio=0.194427 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394530017e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266502144 dispatch_meta_bytes=344576 combine_payload_bytes=266502144 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266846720 ingress_bytes=301105152 link_bytes=301105152 token_bytes=533348864 weight_movement_bytes=34603008 total_nvlink_bytes=567951872 weight_to_token_ratio=0.064879 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133034446e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74212 payload_rows_remote=64782 dup_rows=24092 dup_rows_remote=21064 dedup_ratio=0.245076 router_tokens_per_expert=[767, 743, 729, 732, 811, 721, 790, 773, 737, 783, 781, 750, 812, 777, 816, 778, 744, 776, 756, 782, 760, 808, 768, 761, 747, 754, 778, 760, 812, 768, 833, 755, 766, 763, 758, 765, 756, 767, 765, 747, 768, 792, 770, 763, 794, 775, 798, 769, 746, 734, 708, 770, 772, 746, 736, 739, 807, 770, 782, 779, 801, 755, 773, 756] payload_rows_per_dst=[9206, 9330, 9332, 9336, 9184, 9430, 9112, 9282] dup_rows_per_dst=[2926, 3138, 2978, 3078, 2990, 3028, 2790, 3164] combine_rows_per_src=[9206, 9330, 9332, 9336, 9184, 9430, 9112, 9282] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266272768 dispatch_meta_bytes=344440 combine_payload_bytes=266272768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266617208 ingress_bytes=300875776 link_bytes=300875776 token_bytes=532889976 weight_movement_bytes=34603008 total_nvlink_bytes=567492984 weight_to_token_ratio=0.064935 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132171323e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74442 payload_rows_remote=65180 dup_rows=23862 dup_rows_remote=20832 dedup_ratio=0.242737 router_tokens_per_expert=[796, 743, 790, 800, 768, 776, 801, 744, 760, 763, 795, 766, 752, 805, 800, 714, 731, 754, 719, 739, 740, 772, 796, 721, 814, 773, 825, 783, 780, 760, 753, 761, 748, 733, 730, 752, 790, 721, 760, 803, 728, 767, 808, 742, 781, 772, 786, 775, 784, 789, 781, 768, 728, 770, 756, 770, 784, 799, 772, 750, 790, 731, 828, 762] payload_rows_per_dst=[9404, 9310, 9082, 9448, 9154, 9344, 9262, 9438] dup_rows_per_dst=[3032, 3000, 2862, 3050, 2920, 2974, 3030, 2994] combine_rows_per_src=[9404, 9310, 9082, 9448, 9154, 9344, 9262, 9438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74080 payload_rows_remote=64356 dup_rows=24224 dup_rows_remote=20922 dedup_ratio=0.246419 router_tokens_per_expert=[770, 781, 756, 765, 734, 767, 789, 788, 758, 763, 793, 782, 757, 761, 742, 743, 804, 775, 779, 760, 811, 749, 785, 795, 771, 738, 777, 746, 784, 803, 757, 781, 739, 784, 764, 745, 802, 750, 788, 783, 743, 775, 737, 784, 774, 740, 762, 745, 808, 720, 743, 757, 743, 758, 775, 760, 755, 754, 785, 794, 797, 729, 768, 827] payload_rows_per_dst=[9724, 9352, 9440, 9310, 9286, 9060, 9122, 8786] dup_rows_per_dst=[3302, 3110, 3266, 3012, 3024, 2910, 3006, 2594] combine_rows_per_src=[9724, 9352, 9440, 9310, 9286, 9060, 9122, 8786] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=344464 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267133328 ingress_bytes=370597888 link_bytes=370597888 token_bytes=533922192 weight_movement_bytes=103809024 total_nvlink_bytes=637731216 weight_to_token_ratio=0.194427 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394530017e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=263602176 dispatch_meta_bytes=341112 combine_payload_bytes=263602176 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263943288 ingress_bytes=298205184 link_bytes=298205184 token_bytes=527545464 weight_movement_bytes=34603008 total_nvlink_bytes=562148472 weight_to_token_ratio=0.065592 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.122122100e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266272768 dispatch_meta_bytes=344440 combine_payload_bytes=266272768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266617208 ingress_bytes=300875776 link_bytes=300875776 token_bytes=532889976 weight_movement_bytes=34603008 total_nvlink_bytes=567492984 weight_to_token_ratio=0.064935 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132171323e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12229498048 prefetch_bytes=588251136 reduce_bytes=795869184 weight_movement_bytes=1384120320 egress_bytes=6118692032 ingress_bytes=7494926336 link_bytes=7494926336 total_nvlink_bytes=13613618368 weight_to_token_ratio=0.113179 prefetch_share_of_ingress=0.078487 dedup_ratio=0.244248 prefetch_slots=34 unused_prefetch_slots=334 slot_utilization=0.092391 reduce_slots=23 remote_demand=17 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028804 gemm_inner_pad_upper_bound=28448 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.226208887e-04 prefetch_bytes_per_gemm_flop=9.624094203e-06 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265928704 dispatch_meta_bytes=343688 combine_payload_bytes=265928704 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266272392 ingress_bytes=369737728 link_bytes=369737728 token_bytes=532201096 weight_movement_bytes=103809024 total_nvlink_bytes=636010120 weight_to_token_ratio=0.195056 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391293304e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=344712 combine_payload_bytes=266641408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266986120 ingress_bytes=301244416 link_bytes=301244416 token_bytes=533627528 weight_movement_bytes=34603008 total_nvlink_bytes=568230536 weight_to_token_ratio=0.064845 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133558485e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12263422144 prefetch_bytes=726663168 reduce_bytes=899678208 weight_movement_bytes=1626341376 egress_bytes=6135666880 ingress_bytes=7754096640 link_bytes=7754096640 total_nvlink_bytes=13889763520 weight_to_token_ratio=0.132617 prefetch_share_of_ingress=0.093713 dedup_ratio=0.243723 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=26 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.268610496e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265347072 dispatch_meta_bytes=343384 combine_payload_bytes=265347072 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265690456 ingress_bytes=334553088 link_bytes=334553088 token_bytes=531037528 weight_movement_bytes=69206016 total_nvlink_bytes=600243544 weight_to_token_ratio=0.130322 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258896336e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266977280 dispatch_meta_bytes=344048 combine_payload_bytes=266977280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267321328 ingress_bytes=301580288 link_bytes=301580288 token_bytes=534298608 weight_movement_bytes=34603008 total_nvlink_bytes=568901616 weight_to_token_ratio=0.064763 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134822345e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12236651648 prefetch_bytes=692060160 reduce_bytes=726663168 weight_movement_bytes=1418723328 egress_bytes=6122273920 ingress_bytes=7533101056 link_bytes=7533101056 total_nvlink_bytes=13655374976 weight_to_token_ratio=0.115940 prefetch_share_of_ingress=0.091869 dedup_ratio=0.244171 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=21 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028370 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.232454470e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74562 payload_rows_remote=65048 dup_rows=23742 dup_rows_remote=20616 dedup_ratio=0.241516 router_tokens_per_expert=[779, 735, 774, 769, 751, 727, 740, 773, 783, 783, 728, 743, 769, 762, 776, 760, 801, 781, 781, 767, 781, 743, 801, 716, 777, 749, 818, 763, 746, 793, 794, 780, 773, 774, 749, 789, 785, 740, 751, 742, 764, 750, 766, 810, 792, 752, 748, 817, 714, 727, 763, 787, 726, 766, 740, 764, 785, 782, 822, 828, 809, 770, 760, 764] payload_rows_per_dst=[9238, 9264, 9330, 9164, 9330, 9382, 9340, 9514] dup_rows_per_dst=[2858, 2944, 3012, 2880, 2966, 3016, 2940, 3126] combine_rows_per_src=[9238, 9264, 9330, 9164, 9330, 9382, 9340, 9514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=263602176 dispatch_meta_bytes=341112 combine_payload_bytes=263602176 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263943288 ingress_bytes=298205184 link_bytes=298205184 token_bytes=527545464 weight_movement_bytes=34603008 total_nvlink_bytes=562148472 weight_to_token_ratio=0.065592 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.122122100e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74152 payload_rows_remote=64986 dup_rows=24152 dup_rows_remote=21038 dedup_ratio=0.245687 router_tokens_per_expert=[765, 775, 725, 756, 765, 800, 735, 741, 796, 762, 735, 804, 769, 803, 776, 781, 725, 740, 742, 753, 763, 767, 777, 785, 706, 743, 773, 743, 776, 793, 762, 789, 788, 768, 783, 769, 756, 774, 789, 732, 761, 802, 750, 774, 778, 800, 786, 788, 762, 783, 764, 795, 778, 752, 815, 691, 758, 736, 752, 821, 748, 751, 774, 849] payload_rows_per_dst=[9188, 9348, 9184, 9224, 9312, 9382, 9166, 9348] dup_rows_per_dst=[2936, 3104, 2920, 2946, 3006, 3096, 3114, 3030] combine_rows_per_src=[9188, 9348, 9184, 9224, 9312, 9382, 9166, 9348] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12255248568 prefetch_bytes=726663168 reduce_bytes=484442112 weight_movement_bytes=1211105280 egress_bytes=6131581112 ingress_bytes=7334772736 link_bytes=7334772736 total_nvlink_bytes=13466353848 weight_to_token_ratio=0.098823 prefetch_share_of_ingress=0.099071 dedup_ratio=0.244246 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=14 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028370 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.200006926e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12263002624 prefetch_bytes=692060160 reduce_bytes=588251136 weight_movement_bytes=1280311296 egress_bytes=6135460352 ingress_bytes=7407853568 link_bytes=7407853568 total_nvlink_bytes=13543313920 weight_to_token_ratio=0.104404 prefetch_share_of_ingress=0.093422 dedup_ratio=0.243895 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=17 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028804 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.211963330e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74226 payload_rows_remote=64886 dup_rows=24078 dup_rows_remote=21088 dedup_ratio=0.244934 router_tokens_per_expert=[760, 796, 778, 734, 780, 766, 750, 807, 788, 774, 738, 755, 758, 791, 737, 733, 803, 802, 733, 805, 763, 713, 794, 737, 731, 778, 787, 785, 792, 797, 765, 730, 764, 775, 726, 796, 789, 772, 772, 756, 732, 709, 779, 780, 780, 783, 788, 742, 727, 764, 799, 776, 797, 742, 829, 791, 765, 789, 753, 780, 768, 781, 756, 732] payload_rows_per_dst=[9342, 9214, 9246, 9340, 9228, 9232, 9358, 9266] dup_rows_per_dst=[3000, 2934, 3054, 2990, 3072, 2954, 3092, 2982] combine_rows_per_src=[9342, 9214, 9246, 9340, 9228, 9232, 9358, 9266] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74548 payload_rows_remote=65154 dup_rows=23756 dup_rows_remote=20798 dedup_ratio=0.241659 router_tokens_per_expert=[781, 750, 749, 816, 721, 711, 777, 783, 760, 751, 695, 789, 776, 743, 812, 773, 748, 707, 799, 756, 735, 707, 751, 763, 780, 795, 766, 731, 763, 722, 836, 812, 781, 801, 776, 760, 756, 796, 750, 756, 783, 802, 788, 782, 740, 792, 731, 750, 818, 736, 774, 769, 816, 766, 797, 758, 813, 780, 820, 793, 732, 791, 772, 715] payload_rows_per_dst=[9306, 9206, 9126, 9414, 9394, 9264, 9424, 9414] dup_rows_per_dst=[2870, 2992, 2806, 2996, 2958, 3072, 3044, 3018] combine_rows_per_src=[9306, 9206, 9126, 9414, 9394, 9264, 9424, 9414] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74218 payload_rows_remote=65064 dup_rows=24086 dup_rows_remote=21080 dedup_ratio=0.245015 router_tokens_per_expert=[727, 734, 819, 751, 770, 730, 775, 778, 793, 769, 827, 723, 751, 735, 709, 773, 771, 791, 771, 752, 771, 800, 759, 827, 798, 760, 761, 709, 775, 794, 756, 774, 725, 798, 785, 780, 783, 752, 792, 771, 755, 794, 784, 737, 749, 719, 777, 783, 754, 771, 793, 737, 811, 781, 739, 777, 806, 725, 827, 774, 738, 792, 738, 772] payload_rows_per_dst=[9258, 9154, 9408, 9266, 9302, 9190, 9318, 9322] dup_rows_per_dst=[2910, 3006, 3076, 2988, 3070, 3006, 3008, 3022] combine_rows_per_src=[9258, 9154, 9408, 9266, 9302, 9190, 9318, 9322] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266436608 dispatch_meta_bytes=342656 combine_payload_bytes=266436608 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266779264 ingress_bytes=404848640 link_bytes=404848640 token_bytes=533215872 weight_movement_bytes=138412032 total_nvlink_bytes=671627904 weight_to_token_ratio=0.259580 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523412839e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12236651648 prefetch_bytes=692060160 reduce_bytes=726663168 weight_movement_bytes=1418723328 egress_bytes=6122273920 ingress_bytes=7533101056 link_bytes=7533101056 total_nvlink_bytes=13655374976 weight_to_token_ratio=0.115940 prefetch_share_of_ingress=0.091869 dedup_ratio=0.244171 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=21 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028370 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.232454470e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266182656 dispatch_meta_bytes=344096 combine_payload_bytes=266182656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266526752 ingress_bytes=300785664 link_bytes=300785664 token_bytes=532709408 weight_movement_bytes=34603008 total_nvlink_bytes=567312416 weight_to_token_ratio=0.064957 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131832238e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74226 payload_rows_remote=64886 dup_rows=24078 dup_rows_remote=21088 dedup_ratio=0.244934 router_tokens_per_expert=[760, 796, 778, 734, 780, 766, 750, 807, 788, 774, 738, 755, 758, 791, 737, 733, 803, 802, 733, 805, 763, 713, 794, 737, 731, 778, 787, 785, 792, 797, 765, 730, 764, 775, 726, 796, 789, 772, 772, 756, 732, 709, 779, 780, 780, 783, 788, 742, 727, 764, 799, 776, 797, 742, 829, 791, 765, 789, 753, 780, 768, 781, 756, 732] payload_rows_per_dst=[9342, 9214, 9246, 9340, 9228, 9232, 9358, 9266] dup_rows_per_dst=[3000, 2934, 3054, 2990, 3072, 2954, 3092, 2982] combine_rows_per_src=[9342, 9214, 9246, 9340, 9228, 9232, 9358, 9266] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74308 payload_rows_remote=64882 dup_rows=23996 dup_rows_remote=21070 dedup_ratio=0.244100 router_tokens_per_expert=[754, 795, 750, 738, 765, 740, 779, 787, 747, 724, 820, 747, 827, 776, 761, 755, 746, 762, 785, 786, 730, 765, 744, 760, 838, 763, 767, 784, 789, 800, 758, 776, 751, 747, 763, 801, 759, 759, 814, 759, 755, 754, 775, 778, 729, 755, 755, 738, 780, 775, 783, 772, 746, 763, 770, 787, 747, 786, 727, 765, 821, 781, 796, 743] payload_rows_per_dst=[9238, 9276, 9196, 9332, 9232, 9244, 9426, 9364] dup_rows_per_dst=[2978, 3038, 2960, 3218, 3074, 2834, 2926, 2968] combine_rows_per_src=[9238, 9276, 9196, 9332, 9232, 9244, 9426, 9364] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=343896 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266116952 ingress_bytes=369582080 link_bytes=369582080 token_bytes=531890008 weight_movement_bytes=103809024 total_nvlink_bytes=635699032 weight_to_token_ratio=0.195170 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390707613e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266870784 dispatch_meta_bytes=343808 combine_payload_bytes=266870784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267214592 ingress_bytes=301473792 link_bytes=301473792 token_bytes=534085376 weight_movement_bytes=34603008 total_nvlink_bytes=568688384 weight_to_token_ratio=0.064789 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134421609e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266502144 dispatch_meta_bytes=344576 combine_payload_bytes=266502144 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266846720 ingress_bytes=301105152 link_bytes=301105152 token_bytes=533348864 weight_movement_bytes=34603008 total_nvlink_bytes=567951872 weight_to_token_ratio=0.064879 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133034446e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12229498048 prefetch_bytes=588251136 reduce_bytes=795869184 weight_movement_bytes=1384120320 egress_bytes=6118692032 ingress_bytes=7494926336 link_bytes=7494926336 total_nvlink_bytes=13613618368 weight_to_token_ratio=0.113179 prefetch_share_of_ingress=0.078487 dedup_ratio=0.244248 prefetch_slots=34 unused_prefetch_slots=334 slot_utilization=0.092391 reduce_slots=23 remote_demand=17 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028804 gemm_inner_pad_upper_bound=28448 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.226208887e-04 prefetch_bytes_per_gemm_flop=9.624094203e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=343808 combine_payload_bytes=265756672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266100480 ingress_bytes=300359680 link_bytes=300359680 token_bytes=531857152 weight_movement_bytes=34603008 total_nvlink_bytes=566460160 weight_to_token_ratio=0.065061 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130229295e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12266884200 prefetch_bytes=726663168 reduce_bytes=657457152 weight_movement_bytes=1384120320 egress_bytes=6137400424 ingress_bytes=7513604096 link_bytes=7513604096 total_nvlink_bytes=13651004520 weight_to_token_ratio=0.112834 prefetch_share_of_ingress=0.096713 dedup_ratio=0.243711 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=19 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.229264666e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12249490744 prefetch_bytes=692060160 reduce_bytes=692060160 weight_movement_bytes=1384120320 egress_bytes=6128698680 ingress_bytes=7504912384 link_bytes=7504912384 total_nvlink_bytes=13633611064 weight_to_token_ratio=0.112994 prefetch_share_of_ingress=0.092214 dedup_ratio=0.244219 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=20 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029130 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.227842657e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12263422144 prefetch_bytes=726663168 reduce_bytes=899678208 weight_movement_bytes=1626341376 egress_bytes=6135666880 ingress_bytes=7754096640 link_bytes=7754096640 total_nvlink_bytes=13889763520 weight_to_token_ratio=0.132617 prefetch_share_of_ingress=0.093713 dedup_ratio=0.243723 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=26 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.268610496e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74152 payload_rows_remote=64986 dup_rows=24152 dup_rows_remote=21038 dedup_ratio=0.245687 router_tokens_per_expert=[765, 775, 725, 756, 765, 800, 735, 741, 796, 762, 735, 804, 769, 803, 776, 781, 725, 740, 742, 753, 763, 767, 777, 785, 706, 743, 773, 743, 776, 793, 762, 789, 788, 768, 783, 769, 756, 774, 789, 732, 761, 802, 750, 774, 778, 800, 786, 788, 762, 783, 764, 795, 778, 752, 815, 691, 758, 736, 752, 821, 748, 751, 774, 849] payload_rows_per_dst=[9188, 9348, 9184, 9224, 9312, 9382, 9166, 9348] dup_rows_per_dst=[2936, 3104, 2920, 2946, 3006, 3096, 3114, 3030] combine_rows_per_src=[9188, 9348, 9184, 9224, 9312, 9382, 9166, 9348] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266182656 dispatch_meta_bytes=344096 combine_payload_bytes=266182656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266526752 ingress_bytes=300785664 link_bytes=300785664 token_bytes=532709408 weight_movement_bytes=34603008 total_nvlink_bytes=567312416 weight_to_token_ratio=0.064957 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131832238e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12264298040 prefetch_bytes=726663168 reduce_bytes=726663168 weight_movement_bytes=1453326336 egress_bytes=6136108600 ingress_bytes=7581515776 link_bytes=7581515776 total_nvlink_bytes=13717624376 weight_to_token_ratio=0.118501 prefetch_share_of_ingress=0.095847 dedup_ratio=0.244106 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=21 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028804 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.240375370e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none - [2026-08-26 15:10:28] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 793.6 | throughput per GPU (TFLOP/s/GPU): 160.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.618065E+00 | loss scale: 1.0 | grad norm: 47.305 | number of skipped iterations: 0 | number of nan iterations: 0 | -[after training is done] datetime: 2026-08-26 15:10:28 -[rank0]:[W826 15:10:29.205243568 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank3]:[W826 15:10:30.431376173 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank4]:[W826 15:10:30.435624248 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank7]:[W826 15:10:30.450058523 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank1]:[W826 15:10:30.450073647 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank5]:[W826 15:10:30.506067572 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank2]:[W826 15:10:30.900795660 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank6]:[W826 15:10:30.992040018 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -W0826 15:10:36.871000 71159 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers -W0826 15:10:36.871000 71159 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 71258 closing signal SIGTERM +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=343896 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266116952 ingress_bytes=369582080 link_bytes=369582080 token_bytes=531890008 weight_movement_bytes=103809024 total_nvlink_bytes=635699032 weight_to_token_ratio=0.195170 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390707613e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12249490744 prefetch_bytes=692060160 reduce_bytes=692060160 weight_movement_bytes=1384120320 egress_bytes=6128698680 ingress_bytes=7504912384 link_bytes=7504912384 total_nvlink_bytes=13633611064 weight_to_token_ratio=0.112994 prefetch_share_of_ingress=0.092214 dedup_ratio=0.244219 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=20 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029130 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.227842657e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12266884200 prefetch_bytes=726663168 reduce_bytes=657457152 weight_movement_bytes=1384120320 egress_bytes=6137400424 ingress_bytes=7513604096 link_bytes=7513604096 total_nvlink_bytes=13651004520 weight_to_token_ratio=0.112834 prefetch_share_of_ingress=0.096713 dedup_ratio=0.243711 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=19 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.229264666e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none + [2026-08-26 15:58:16] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 794.5 | throughput per GPU (TFLOP/s/GPU): 160.6 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.618065E+00 | loss scale: 1.0 | grad norm: 47.305 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-26 15:58:16 +[rank0]:[W826 15:58:18.304933901 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W826 15:58:19.129025812 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W826 15:58:19.466954006 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W826 15:58:19.472741025 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W826 15:58:19.493881674 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W826 15:58:19.943375036 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W826 15:58:19.943589188 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W826 15:58:20.129815327 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 15:58:25.799000 160561 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 15:58:25.800000 160561 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 160656 closing signal SIGTERM Traceback (most recent call last): File "/usr/local/bin/torchrun", line 8, in sys.exit(main()) @@ -7027,4 +7027,4 @@ Traceback (most recent call last): time.sleep(monitor_interval) File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 71159 got signal: 15 +torch.distributed.elastic.multiprocessing.api.SignalException: Process 160561 got signal: 15 diff --git a/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/mem_samples.csv b/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/mem_samples.csv index cb659dc5c826e8bcacdf5a5771f8f28651d31eb0..17baf11ca491d0d923a006a9b769e784b1611925 100644 --- a/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/mem_samples.csv +++ b/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/mem_samples.csv @@ -14,7 +14,7 @@ 5, 0 6, 0 7, 0 -0, 1 +0, 2 1, 1 2, 1 3, 1 @@ -38,63 +38,63 @@ 5, 4 6, 4 7, 4 -0, 4 -1, 629 -2, 629 -3, 629 -4, 629 -5, 629 -6, 629 -7, 629 -0, 975 -1, 971 -2, 981 -3, 987 -4, 973 -5, 965 -6, 957 -7, 953 -0, 1803 -1, 1803 -2, 1803 -3, 1803 -4, 1803 -5, 1803 -6, 1803 -7, 1803 -0, 6689 -1, 6699 -2, 6695 -3, 6701 -4, 6683 -5, 6687 -6, 6693 -7, 6695 -0, 58449 -1, 57903 -2, 57859 -3, 58599 -4, 58603 -5, 58577 -6, 58575 +0, 791 +1, 789 +2, 763 +3, 769 +4, 751 +5, 759 +6, 781 +7, 757 +0, 1081 +1, 1079 +2, 1079 +3, 1079 +4, 1079 +5, 1079 +6, 1079 +7, 1081 +0, 1863 +1, 1863 +2, 1863 +3, 1863 +4, 1863 +5, 1863 +6, 1863 +7, 1863 +0, 6795 +1, 6807 +2, 6825 +3, 6813 +4, 6785 +5, 6799 +6, 6801 +7, 6799 +0, 58445 +1, 57853 +2, 57881 +3, 58593 +4, 58557 +5, 58563 +6, 58561 7, 58567 -0, 59447 -1, 58893 -2, 58873 -3, 59577 -4, 59577 -5, 59575 -6, 59581 -7, 59587 +0, 59091 +1, 58523 +2, 58521 +3, 59221 +4, 59215 +5, 59201 +6, 59197 +7, 59223 0, 59719 -1, 59149 -2, 59149 -3, 59849 -4, 59849 -5, 59849 -6, 59849 -7, 59849 -0, 60419 +1, 59151 +2, 59151 +3, 59851 +4, 59851 +5, 59851 +6, 59851 +7, 59851 +0, 59723 1, 59851 2, 59851 3, 60551 @@ -102,91 +102,91 @@ 5, 60551 6, 60551 7, 60551 -0, 67691 -1, 67695 -2, 68433 -3, 69157 -4, 68407 -5, 76049 -6, 75091 -7, 69115 -0, 112523 -1, 113183 -2, 114755 -3, 113143 -4, 113013 -5, 117111 -6, 117111 -7, 113573 -0, 118643 -1, 118685 -2, 118655 -3, 118669 -4, 118675 -5, 118659 -6, 118667 -7, 118707 +0, 84015 +1, 84449 +2, 84133 +3, 85099 +4, 84261 +5, 83793 +6, 84371 +7, 84259 +0, 117963 +1, 118029 +2, 118031 +3, 118055 +4, 118041 +5, 118049 +6, 118049 +7, 118059 +0, 118939 +1, 118985 +2, 118967 +3, 119001 +4, 119001 +5, 119001 +6, 119001 +7, 119001 0, 118941 -1, 118987 +1, 118985 2, 118967 3, 119001 4, 119001 5, 119001 6, 119001 7, 119001 -0, 119103 -1, 119113 -2, 119139 -3, 119151 -4, 119127 -5, 119135 -6, 119151 -7, 119129 -0, 128615 -1, 128353 -2, 128475 -3, 128499 -4, 128445 -5, 128485 -6, 128443 -7, 128461 -0, 141323 -1, 141553 -2, 141527 -3, 141353 -4, 141353 -5, 141353 -6, 141353 -7, 141353 -0, 141323 -1, 141553 -2, 141527 -3, 141353 -4, 141353 -5, 141353 -6, 141353 -7, 141353 -0, 140677 -1, 140581 -2, 140299 -3, 140707 -4, 140707 -5, 140707 -6, 140707 -7, 140707 -0, 139463 -1, 139437 -2, 139239 -3, 139477 -4, 139819 -5, 139557 -6, 139819 -7, 139431 -0, 13561 +0, 128043 +1, 127729 +2, 127855 +3, 127853 +4, 127853 +5, 127853 +6, 127853 +7, 127853 +0, 141327 +1, 141555 +2, 141529 +3, 141355 +4, 141355 +5, 141355 +6, 141355 +7, 141355 +0, 141327 +1, 141555 +2, 141529 +3, 141355 +4, 141355 +5, 141355 +6, 141355 +7, 141355 +0, 141327 +1, 141555 +2, 141529 +3, 141355 +4, 141355 +5, 141355 +6, 141355 +7, 141355 +0, 140179 +1, 140909 +2, 140317 +3, 139823 +4, 139887 +5, 140709 +6, 140207 +7, 140709 +0, 137851 +1, 138989 +2, 138575 +3, 138401 +4, 138139 +5, 138789 +6, 138401 +7, 138789 +0, 12604 1, 12604 2, 12604 3, 12604 -4, 12604 -5, 12604 -6, 12796 -7, 139431 +4, 12668 +5, 137753 +6, 12924 +7, 12604 diff --git a/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/run.env b/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/run.env index 2e125eaee2a22bb3945198acdc1e06270cceb55c..0fc6665c865977c7ad35e847b78370c95e540fd2 100644 --- a/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/run.env +++ b/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/run.env @@ -28,7 +28,7 @@ router_balancing=aux_loss aux_loss_coeff=1e-2 router_pre_softmax=1 router_fusion=0 -router_dtype= +router_dtype=fp32 device_initiated_grouped_gemm=1 norm_epsilon=1e-6 seq_length=4096 diff --git a/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/train.log b/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/train.log index ec305b5d65943722e53184510ac2bf7de3c5004a..ea2e43e01310331a833a9e8aad9f31da92b88c27 100644 --- a/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/train.log +++ b/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/train.log @@ -1,5 +1,5 @@ -[15:10:39] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) -[15:10:39] WARN: moonep_backend_token_routing_analysis lacks --moe-debug-drop-logging; probe will have no pre/post rank loads +[15:58:27] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +[15:58:27] WARN: moonep_backend_token_routing_analysis lacks --moe-debug-drop-logging; probe will have no pre/post rank loads ============================================================== QWEN2_57B / moonep (slots/rank=8 policy=matched) E=64 K=8 H'=2560 m=8 @@ -8,23 +8,10 @@ memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/memory_snapshots/snapshot.pickle repo=/workspace/Megatron-MoE-Echo @ 35ae6e7f ============================================================== -W0826 15:10:40.400000 73803 torch/distributed/run.py:766] -W0826 15:10:40.400000 73803 torch/distributed/run.py:766] ***************************************** -W0826 15:10:40.400000 73803 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -W0826 15:10:40.400000 73803 torch/distributed/run.py:766] ***************************************** -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +W0826 15:58:29.303000 163203 torch/distributed/run.py:766] +W0826 15:58:29.303000 163203 torch/distributed/run.py:766] ***************************************** +W0826 15:58:29.303000 163203 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 15:58:29.303000 163203 torch/distributed/run.py:766] ***************************************** /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 @@ -345,7 +332,7 @@ using torch.bfloat16 for parameters ... moe_permute_fusion .............................. False moe_received_token_capacity ..................... None moe_router_bias_update_rate ..................... 0.001 - moe_router_dtype ................................ None + moe_router_dtype ................................ fp32 moe_router_enable_expert_bias ................... False moe_router_force_load_balancing ................. False moe_router_group_topk ........................... None @@ -580,68 +567,65 @@ INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS > initializing torch distributed ... -[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank 1 is connected to 72 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. Expected number of connected peer ranks is : +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +[Gloo] Rank [Gloo] Rank 3 is connected to 70 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7[Gloo] Rank -6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 4 is connected to 57 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7[Gloo] Rank +6 is connected to [Gloo] Rank 7 peer ranks. Expected number of connected peer ranks is : 27 is connected to +7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 1 is connected to 7[Gloo] Rank peer ranks. 3Expected number of connected peer ranks is : is connected to 77 - peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7[Gloo] Rank -7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank 1[Gloo] Rank 3[Gloo] Rank 2 is connected to 76 is connected to 4 is connected to 7 is connected to is connected to [Gloo] Rank 7 is connected to 7 peer ranks. 775 peer ranks. 7 peer ranks. Expected number of connected peer ranks is : peer ranks. peer ranks. is connected to Expected number of connected peer ranks is : peer ranks. Expected number of connected peer ranks is : 7Expected number of connected peer ranks is : +Expected number of connected peer ranks is : 77Expected number of connected peer ranks is : 777 peer ranks. +7 + + +Expected number of connected peer ranks is : +7 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 > initialized tensor model parallel with size 1 > initialized pipeline model parallel with size 1 > setting random seeds to 1234 ... -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank [Gloo] Rank 00 is connected to is connected to [Gloo] Rank 000 peer ranks. peer ranks. is connected to Expected number of connected peer ranks is : Expected number of connected peer ranks is : 000 peer ranks. - -Expected number of connected peer ranks is : 0 -[rank2]:[W826 15:10:58.769960677 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank5]:[W826 15:10:58.770302791 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank4]:[W826 15:10:58.770539930 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank6]:[W826 15:10:58.770543608 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank1]:[W826 15:10:58.770562066 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank3]:[W826 15:10:58.772324675 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank7]:[W826 15:10:58.772334463 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank2]:[W826 15:58:47.402455506 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 15:58:47.402455994 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 15:58:47.402464312 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 15:58:47.402524797 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 15:58:47.403726461 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 15:58:47.405768142 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 15:58:47.405793914 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. > compiling dataset index builder ... make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' make: Nothing to be done for 'default'. make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' ->>> done with dataset index builder. Compilation time: 0.033 seconds +>>> done with dataset index builder. Compilation time: 0.035 seconds WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. > compiling and loading fused kernels ... -[rank0]:[W826 15:10:58.933044025 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. ->>> done with compiling and loading fused kernels. Compilation time: 3.571 seconds -time to initialize megatron (seconds): 6.880 -[after megatron is initialized] datetime: 2026-08-26 15:11:04 +[rank0]:[W826 15:58:47.557701719 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.675 seconds +time to initialize megatron (seconds): 7.823 +[after megatron is initialized] datetime: 2026-08-26 15:58:53 building GPT model ... -/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1440: UserWarning: Using a large number of experts (e.g. >=32) without fp32 routing. Consider enabling moe_router_dtype for better numerical stability. - warnings.warn( -/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1440: UserWarning: Using a large number of experts (e.g. >=32) without fp32 routing. Consider enabling moe_router_dtype for better numerical stability. - warnings.warn( -/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1440: UserWarning: Using a large number of experts (e.g. >=32) without fp32 routing. Consider enabling moe_router_dtype for better numerical stability. - warnings.warn( -/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1440: UserWarning: Using a large number of experts (e.g. >=32) without fp32 routing. Consider enabling moe_router_dtype for better numerical stability. - warnings.warn( -/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1440: UserWarning: Using a large number of experts (e.g. >=32) without fp32 routing. Consider enabling moe_router_dtype for better numerical stability. - warnings.warn( -/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1440: UserWarning: Using a large number of experts (e.g. >=32) without fp32 routing. Consider enabling moe_router_dtype for better numerical stability. - warnings.warn( -/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1440: UserWarning: Using a large number of experts (e.g. >=32) without fp32 routing. Consider enabling moe_router_dtype for better numerical stability. - warnings.warn( -/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1440: UserWarning: Using a large number of experts (e.g. >=32) without fp32 routing. Consider enabling moe_router_dtype for better numerical stability. - warnings.warn( /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. warnings.warn( /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. @@ -670,77 +654,77 @@ Params for bucket 3 (146800640 elements, 146800640 padded size): module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight Params for bucket 4 (103001600 elements, 103001600 padded size): module.decoder.layers.9.mlp.router.weight - module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.9.self_attention.linear_qkv.bias - module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.9.self_attention.linear_proj.weight module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.pre_mlp_layernorm.weight Params for bucket 5 (146800640 elements, 146800640 padded size): module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight Params for bucket 6 (103001600 elements, 103001600 padded size): - module.decoder.layers.8.self_attention.linear_qkv.weight module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.weight module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_qkv.bias module.decoder.layers.8.self_attention.linear_proj.weight module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight module.decoder.layers.8.mlp.router.weight - module.decoder.layers.8.self_attention.linear_qkv.bias Params for bucket 7 (146800640 elements, 146800640 padded size): module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight Params for bucket 8 (103001600 elements, 103001600 padded size): module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_qkv.bias + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.7.self_attention.linear_qkv.weight module.decoder.layers.7.pre_mlp_layernorm.weight - module.decoder.layers.7.self_attention.linear_proj.weight module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.7.self_attention.linear_qkv.bias Params for bucket 9 (146800640 elements, 146800640 padded size): module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight Params for bucket 10 (103001600 elements, 103001600 padded size): - module.decoder.layers.6.self_attention.linear_qkv.bias - module.decoder.layers.6.self_attention.linear_proj.weight - module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight module.decoder.layers.6.mlp.router.weight module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.bias module.decoder.layers.6.self_attention.linear_qkv.weight module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight Params for bucket 11 (146800640 elements, 146800640 padded size): module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight Params for bucket 12 (103001600 elements, 103001600 padded size): module.decoder.layers.5.mlp.router.weight module.decoder.layers.5.self_attention.linear_qkv.bias module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.5.self_attention.linear_qkv.weight - module.decoder.layers.5.pre_mlp_layernorm.weight module.decoder.layers.5.self_attention.linear_proj.weight module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.weight Params for bucket 13 (146800640 elements, 146800640 padded size): module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight Params for bucket 14 (103001600 elements, 103001600 padded size): + module.decoder.layers.4.self_attention.linear_qkv.bias module.decoder.layers.4.self_attention.linear_qkv.weight - module.decoder.layers.4.pre_mlp_layernorm.weight module.decoder.layers.4.self_attention.linear_proj.weight module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight module.decoder.layers.4.mlp.router.weight - module.decoder.layers.4.self_attention.linear_qkv.bias module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.pre_mlp_layernorm.weight Params for bucket 15 (146800640 elements, 146800640 padded size): module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight Params for bucket 16 (103001600 elements, 103001600 padded size): - module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.3.self_attention.linear_qkv.weight - module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.3.pre_mlp_layernorm.weight module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.3.self_attention.linear_qkv.bias - module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.linear_proj.weight Params for bucket 17 (146800640 elements, 146800640 padded size): module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight Params for bucket 18 (103001600 elements, 103001600 padded size): - module.decoder.layers.2.self_attention.linear_qkv.bias module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_qkv.bias module.decoder.layers.2.mlp.router.weight module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.2.pre_mlp_layernorm.weight @@ -750,26 +734,26 @@ Params for bucket 19 (146800640 elements, 146800640 padded size): module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight Params for bucket 20 (103001600 elements, 103001600 padded size): module.decoder.layers.1.mlp.router.weight - module.decoder.layers.1.self_attention.linear_qkv.weight - module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.1.self_attention.linear_qkv.bias module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.bias module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.linear_qkv.weight module.decoder.layers.0.mlp.shared_experts.linear_fc2.weight Params for bucket 21 (146800640 elements, 146800640 padded size): module.decoder.layers.0.mlp.shared_experts.linear_fc1.weight Params for bucket 22 (209890816 elements, 209890816 padded size): - module.decoder.layers.0.self_attention.linear_qkv.weight module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight - module.embedding.word_embeddings.weight - module.decoder.layers.0.mlp.router.weight - module.decoder.layers.0.self_attention.linear_qkv.bias module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.bias + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_qkv.weight module.decoder.layers.0.self_attention.linear_proj.weight + module.embedding.word_embeddings.weight INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 44 Params for bucket 1 (45875200 elements, 45875200 padded size): - module.decoder.layers.9.mlp.experts.linear_fc2.weight6 module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 module.decoder.layers.9.mlp.experts.linear_fc2.weight7 module.decoder.layers.9.mlp.experts.linear_fc2.weight3 module.decoder.layers.9.mlp.experts.linear_fc2.weight5 @@ -784,61 +768,61 @@ Params for bucket 3 (55050240 elements, 55050240 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight4 Params for bucket 4 (55050240 elements, 55050240 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight3 - module.decoder.layers.9.mlp.experts.linear_fc1.weight1 module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 Params for bucket 5 (45875200 elements, 45875200 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight0 - module.decoder.layers.8.mlp.experts.linear_fc2.weight5 module.decoder.layers.8.mlp.experts.linear_fc2.weight6 module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 Params for bucket 6 (45875200 elements, 45875200 padded size): module.decoder.layers.8.mlp.experts.linear_fc2.weight4 - module.decoder.layers.8.mlp.experts.linear_fc2.weight1 - module.decoder.layers.8.mlp.experts.linear_fc2.weight0 module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 Params for bucket 7 (55050240 elements, 55050240 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight7 module.decoder.layers.8.mlp.experts.linear_fc1.weight5 module.decoder.layers.8.mlp.experts.linear_fc1.weight6 Params for bucket 8 (55050240 elements, 55050240 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight4 - module.decoder.layers.8.mlp.experts.linear_fc1.weight3 module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 Params for bucket 9 (45875200 elements, 45875200 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight1 - module.decoder.layers.8.mlp.experts.linear_fc1.weight0 module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 Params for bucket 10 (45875200 elements, 45875200 padded size): - module.decoder.layers.7.mlp.experts.linear_fc2.weight6 - module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 module.decoder.layers.7.mlp.experts.linear_fc2.weight5 module.decoder.layers.7.mlp.experts.linear_fc2.weight3 - module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 Params for bucket 11 (55050240 elements, 55050240 padded size): module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 module.decoder.layers.7.mlp.experts.linear_fc2.weight0 module.decoder.layers.7.mlp.experts.linear_fc1.weight6 - module.decoder.layers.7.mlp.experts.linear_fc1.weight7 Params for bucket 12 (55050240 elements, 55050240 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight5 - module.decoder.layers.7.mlp.experts.linear_fc1.weight3 module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 Params for bucket 13 (55050240 elements, 55050240 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight2 module.decoder.layers.7.mlp.experts.linear_fc1.weight0 module.decoder.layers.7.mlp.experts.linear_fc1.weight1 Params for bucket 14 (45875200 elements, 45875200 padded size): - module.decoder.layers.6.mlp.experts.linear_fc2.weight6 - module.decoder.layers.6.mlp.experts.linear_fc2.weight4 module.decoder.layers.6.mlp.experts.linear_fc2.weight7 - module.decoder.layers.6.mlp.experts.linear_fc2.weight5 module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 Params for bucket 15 (45875200 elements, 45875200 padded size): module.decoder.layers.6.mlp.experts.linear_fc2.weight2 - module.decoder.layers.6.mlp.experts.linear_fc1.weight7 module.decoder.layers.6.mlp.experts.linear_fc2.weight0 module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 Params for bucket 16 (55050240 elements, 55050240 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight6 module.decoder.layers.6.mlp.experts.linear_fc1.weight4 @@ -849,15 +833,15 @@ Params for bucket 17 (55050240 elements, 55050240 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight1 Params for bucket 18 (45875200 elements, 45875200 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight0 - module.decoder.layers.5.mlp.experts.linear_fc2.weight5 module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 module.decoder.layers.5.mlp.experts.linear_fc2.weight7 Params for bucket 19 (45875200 elements, 45875200 padded size): - module.decoder.layers.5.mlp.experts.linear_fc2.weight3 - module.decoder.layers.5.mlp.experts.linear_fc2.weight2 - module.decoder.layers.5.mlp.experts.linear_fc2.weight0 module.decoder.layers.5.mlp.experts.linear_fc2.weight4 module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 Params for bucket 20 (55050240 elements, 55050240 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight7 module.decoder.layers.5.mlp.experts.linear_fc1.weight5 @@ -868,65 +852,65 @@ Params for bucket 21 (55050240 elements, 55050240 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight2 Params for bucket 22 (45875200 elements, 45875200 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight1 - module.decoder.layers.5.mlp.experts.linear_fc1.weight0 module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 Params for bucket 23 (45875200 elements, 45875200 padded size): module.decoder.layers.4.mlp.experts.linear_fc2.weight6 module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 module.decoder.layers.4.mlp.experts.linear_fc2.weight5 module.decoder.layers.4.mlp.experts.linear_fc2.weight3 - module.decoder.layers.4.mlp.experts.linear_fc2.weight2 Params for bucket 24 (55050240 elements, 55050240 padded size): module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 module.decoder.layers.4.mlp.experts.linear_fc1.weight7 module.decoder.layers.4.mlp.experts.linear_fc1.weight6 - module.decoder.layers.4.mlp.experts.linear_fc2.weight0 Params for bucket 25 (55050240 elements, 55050240 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight5 module.decoder.layers.4.mlp.experts.linear_fc1.weight4 module.decoder.layers.4.mlp.experts.linear_fc1.weight3 Params for bucket 26 (55050240 elements, 55050240 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight2 - module.decoder.layers.4.mlp.experts.linear_fc1.weight0 module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 Params for bucket 27 (45875200 elements, 45875200 padded size): module.decoder.layers.3.mlp.experts.linear_fc2.weight7 module.decoder.layers.3.mlp.experts.linear_fc2.weight3 module.decoder.layers.3.mlp.experts.linear_fc2.weight5 - module.decoder.layers.3.mlp.experts.linear_fc2.weight4 module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 Params for bucket 28 (45875200 elements, 45875200 padded size): module.decoder.layers.3.mlp.experts.linear_fc2.weight2 - module.decoder.layers.3.mlp.experts.linear_fc2.weight1 - module.decoder.layers.3.mlp.experts.linear_fc2.weight0 module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 Params for bucket 29 (55050240 elements, 55050240 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight6 - module.decoder.layers.3.mlp.experts.linear_fc1.weight5 module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 Params for bucket 30 (55050240 elements, 55050240 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight3 - module.decoder.layers.3.mlp.experts.linear_fc1.weight1 module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 Params for bucket 31 (45875200 elements, 45875200 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight0 module.decoder.layers.2.mlp.experts.linear_fc2.weight6 module.decoder.layers.2.mlp.experts.linear_fc2.weight7 module.decoder.layers.2.mlp.experts.linear_fc2.weight5 Params for bucket 32 (45875200 elements, 45875200 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 module.decoder.layers.2.mlp.experts.linear_fc2.weight1 module.decoder.layers.2.mlp.experts.linear_fc2.weight2 - module.decoder.layers.2.mlp.experts.linear_fc2.weight3 - module.decoder.layers.2.mlp.experts.linear_fc2.weight0 Params for bucket 33 (55050240 elements, 55050240 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight7 module.decoder.layers.2.mlp.experts.linear_fc1.weight5 module.decoder.layers.2.mlp.experts.linear_fc1.weight6 Params for bucket 34 (55050240 elements, 55050240 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight4 - module.decoder.layers.2.mlp.experts.linear_fc1.weight3 module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 Params for bucket 35 (45875200 elements, 45875200 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight1 module.decoder.layers.1.mlp.experts.linear_fc2.weight7 @@ -935,13 +919,13 @@ Params for bucket 36 (45875200 elements, 45875200 padded size): module.decoder.layers.1.mlp.experts.linear_fc2.weight3 module.decoder.layers.1.mlp.experts.linear_fc2.weight5 module.decoder.layers.1.mlp.experts.linear_fc2.weight6 - module.decoder.layers.1.mlp.experts.linear_fc2.weight2 module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 Params for bucket 37 (55050240 elements, 55050240 padded size): module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 module.decoder.layers.1.mlp.experts.linear_fc2.weight0 module.decoder.layers.1.mlp.experts.linear_fc1.weight7 - module.decoder.layers.1.mlp.experts.linear_fc1.weight6 Params for bucket 38 (55050240 elements, 55050240 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight5 module.decoder.layers.1.mlp.experts.linear_fc1.weight4 @@ -951,29 +935,29 @@ Params for bucket 39 (55050240 elements, 55050240 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight1 module.decoder.layers.1.mlp.experts.linear_fc1.weight0 Params for bucket 40 (45875200 elements, 45875200 padded size): - module.decoder.layers.0.mlp.experts.linear_fc2.weight6 - module.decoder.layers.0.mlp.experts.linear_fc2.weight4 module.decoder.layers.0.mlp.experts.linear_fc2.weight7 module.decoder.layers.0.mlp.experts.linear_fc2.weight3 module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 Params for bucket 41 (45875200 elements, 45875200 padded size): module.decoder.layers.0.mlp.experts.linear_fc2.weight2 - module.decoder.layers.0.mlp.experts.linear_fc2.weight0 module.decoder.layers.0.mlp.experts.linear_fc1.weight7 module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 Params for bucket 42 (55050240 elements, 55050240 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight6 module.decoder.layers.0.mlp.experts.linear_fc1.weight4 module.decoder.layers.0.mlp.experts.linear_fc1.weight5 Params for bucket 43 (55050240 elements, 55050240 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight3 - module.decoder.layers.0.mlp.experts.linear_fc1.weight1 module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 Params for bucket 44 (18350080 elements, 18350080 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight0 -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine -[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 15:11:07 +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 15:58:56 > building train, validation, and test datasets ... > datasets target sizes (minimum size): train: 96 @@ -981,7 +965,7 @@ INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine test: 0 INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] > building train, validation, and test datasets for GPT ... -INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers @@ -994,14 +978,14 @@ INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 466b63c71b42 INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-shuffle_index.npy INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 > finished creating GPT datasets ... -[after dataloaders are built] datetime: 2026-08-26 15:11:07 +[after dataloaders are built] datetime: 2026-08-26 15:58:56 done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (2588.61, 2595.72) + train/valid/test-data-iterators-setup ..........: (59.48, 68.52) training ... Overwriting rerun_state_machine.current_iteration from -1 to 0... -(min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (2868.67, 2873.28) - train/valid/test-data-iterators-setup ..........: (62.17, 68.84) -[before the start of training step] datetime: 2026-08-26 15:11:07 +[before the start of training step] datetime: 2026-08-26 15:58:56 INFO:moonep.api:MoonEP comm buffer sizes: config : R=8, S=4096, H=3584, E=64, K=8, NvS=34800, NvS_padded=34816 sm_count : num_sms=32, num_sms_dedup=148 @@ -1010,2522 +994,2522 @@ INFO:moonep.api:MoonEP comm buffer sizes: local_temp_buffer_size : 1.26 MiB total_size : 241.26 MiB WARNING:megatron.core.rerun_state_machine:Result validation enabled - [2026-08-26 15:11:31] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 23768.3 | throughput per GPU (TFLOP/s/GPU): 5.2 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.097821E+01 | loss scale: 1.0 | grad norm: 29.824 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 15:59:17] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 21729.2 | throughput per GPU (TFLOP/s/GPU): 5.7 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.097850E+01 | loss scale: 1.0 | grad norm: 29.824 | number of skipped iterations: 0 | number of nan iterations: 0 | Number of parameters in transformer block in billions: 20.11 Number of parameters in embedding layers in billions: 0.36 Total number of parameters in billions: 20.47 Number of parameters in most loaded shard in billions: 20.4724 Theoretical memory footprints: weight and optimizer=146430.14 MB -[Rank 0] (after 1 iterations) memory (MB) | allocated: 104985.7431640625 | max allocated: 104985.74365234375 | reserved: 108634.0 | max reserved: 108634.0 - [2026-08-26 15:11:32] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 619.6 | throughput per GPU (TFLOP/s/GPU): 199.3 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.097819E+01 | loss scale: 1.0 | grad norm: 30.567 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46154 payload_rows_remote=39812 dup_rows=19382 dup_rows_remote=16926 dedup_ratio=0.295746 router_tokens_per_expert=[371, 119, 976, 25, 160, 38, 363, 114, 660, 1055, 399, 332, 860, 191, 193, 472, 64, 60, 534, 190, 521, 102, 260, 47, 341, 1261, 661, 948, 416, 403, 286, 91, 532, 194, 935, 559, 778, 48, 62, 695, 344, 1083, 1273, 212, 1003, 126, 289, 291, 1504, 457, 100, 14, 828, 1568, 2621, 1245, 334, 512, 278, 111, 562, 1361, 87, 249] payload_rows_per_dst=[5540, 5780, 6342, 5842, 5338, 6236, 6016, 5060] dup_rows_per_dst=[1800, 2544, 2456, 2972, 2268, 3006, 2408, 1928] combine_rows_per_src=[5540, 5780, 6342, 5842, 5338, 6236, 6016, 5060] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43976 payload_rows_remote=38426 dup_rows=21560 dup_rows_remote=19438 dedup_ratio=0.328979 router_tokens_per_expert=[377, 172, 999, 17, 136, 10, 1072, 184, 877, 1097, 201, 295, 906, 164, 354, 391, 91, 28, 499, 266, 238, 70, 181, 75, 261, 1380, 408, 862, 498, 876, 280, 32, 306, 134, 1413, 280, 1036, 26, 169, 565, 447, 684, 1484, 167, 671, 147, 132, 432, 1250, 689, 136, 27, 1074, 1888, 2478, 1270, 358, 520, 191, 67, 510, 631, 87, 202] payload_rows_per_dst=[5752, 5012, 2578, 4918, 6714, 5762, 7690, 5550] dup_rows_per_dst=[2224, 1746, 318, 1516, 3904, 2566, 7164, 2122] combine_rows_per_src=[5752, 5012, 2578, 4918, 6714, 5762, 7690, 5550] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46390 payload_rows_remote=40838 dup_rows=19146 dup_rows_remote=17358 dedup_ratio=0.292145 router_tokens_per_expert=[369, 133, 580, 36, 28, 17, 1061, 15, 1235, 930, 217, 95, 1219, 217, 271, 295, 48, 35, 666, 513, 455, 93, 263, 25, 208, 1115, 346, 1058, 847, 985, 213, 53, 526, 97, 1506, 593, 626, 14, 72, 236, 416, 629, 1210, 61, 663, 150, 304, 363, 1231, 749, 118, 50, 719, 867, 3142, 1776, 299, 427, 287, 261, 814, 708, 40, 173] payload_rows_per_dst=[5282, 6242, 7152, 6222, 5552, 5370, 4758, 5812] dup_rows_per_dst=[1658, 2716, 3328, 3428, 1788, 2222, 1268, 2738] combine_rows_per_src=[5282, 6242, 7152, 6222, 5552, 5370, 4758, 5812] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45556 payload_rows_remote=39644 dup_rows=19980 dup_rows_remote=17172 dedup_ratio=0.304871 router_tokens_per_expert=[512, 203, 364, 17, 31, 32, 1116, 23, 1047, 932, 186, 105, 1323, 150, 110, 421, 57, 52, 457, 566, 658, 21, 187, 57, 446, 1146, 371, 826, 1021, 431, 340, 52, 511, 271, 1275, 431, 680, 48, 165, 483, 525, 1236, 789, 150, 971, 260, 63, 366, 1510, 747, 208, 49, 606, 947, 2497, 1100, 404, 639, 276, 52, 549, 1204, 140, 356] payload_rows_per_dst=[5526, 5984, 6384, 5914, 5342, 5912, 4346, 6148] dup_rows_per_dst=[2090, 2564, 2720, 3352, 2386, 2808, 768, 3292] combine_rows_per_src=[5526, 5984, 6384, 5914, 5342, 5912, 4346, 6148] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45804 payload_rows_remote=40382 dup_rows=19732 dup_rows_remote=17654 dedup_ratio=0.301086 router_tokens_per_expert=[690, 164, 1162, 13, 165, 2, 734, 74, 928, 866, 193, 196, 1275, 229, 34, 365, 131, 19, 392, 368, 785, 30, 224, 47, 327, 648, 353, 778, 713, 532, 344, 55, 666, 389, 731, 320, 661, 48, 142, 507, 560, 1002, 1204, 210, 1251, 202, 74, 207, 1422, 374, 216, 70, 457, 828, 2448, 814, 643, 819, 451, 105, 651, 1625, 188, 647] payload_rows_per_dst=[6100, 5710, 6236, 5422, 4948, 6428, 4516, 6444] dup_rows_per_dst=[2752, 2462, 2652, 2078, 1980, 2992, 1002, 3814] combine_rows_per_src=[6100, 5710, 6236, 5422, 4948, 6428, 4516, 6444] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44882 payload_rows_remote=38608 dup_rows=20654 dup_rows_remote=17126 dedup_ratio=0.315155 router_tokens_per_expert=[403, 145, 600, 10, 168, 20, 905, 99, 1158, 632, 276, 277, 1371, 162, 53, 408, 160, 68, 294, 226, 925, 161, 105, 24, 247, 1185, 352, 713, 450, 544, 316, 30, 1088, 661, 1155, 484, 714, 36, 107, 751, 697, 620, 1209, 77, 805, 214, 188, 218, 1230, 398, 172, 37, 357, 1215, 3254, 1429, 229, 447, 222, 60, 817, 598, 80, 712] payload_rows_per_dst=[5404, 4958, 4946, 5134, 6382, 5582, 6274, 6202] dup_rows_per_dst=[1756, 1974, 1786, 2064, 4086, 2474, 3528, 2986] combine_rows_per_src=[5404, 4958, 4946, 5134, 6382, 5582, 6274, 6202] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45986 payload_rows_remote=40208 dup_rows=19550 dup_rows_remote=17094 dedup_ratio=0.298309 router_tokens_per_expert=[757, 166, 931, 8, 153, 6, 898, 108, 970, 1235, 215, 120, 922, 239, 86, 330, 79, 68, 362, 201, 353, 76, 581, 60, 660, 1505, 333, 775, 329, 701, 217, 133, 878, 242, 571, 264, 632, 195, 80, 751, 421, 1143, 631, 94, 800, 285, 59, 281, 1885, 280, 102, 20, 845, 1594, 2698, 1007, 378, 360, 174, 73, 429, 1532, 126, 361] payload_rows_per_dst=[6544, 5778, 6540, 6040, 5132, 5456, 5640, 4856] dup_rows_per_dst=[3280, 2456, 2416, 3266, 2094, 1972, 2056, 2010] combine_rows_per_src=[6544, 5778, 6540, 6040, 5132, 5456, 5640, 4856] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +[Rank 0] (after 1 iterations) memory (MB) | allocated: 104985.1162109375 | max allocated: 104985.11669921875 | reserved: 108632.0 | max reserved: 108632.0 + [2026-08-26 15:59:18] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 375.7 | throughput per GPU (TFLOP/s/GPU): 328.7 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.097848E+01 | loss scale: 1.0 | grad norm: 30.568 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46506 payload_rows_remote=40950 dup_rows=19030 dup_rows_remote=17244 dedup_ratio=0.290375 router_tokens_per_expert=[364, 133, 569, 36, 28, 17, 1065, 13, 1240, 928, 217, 94, 1204, 218, 270, 297, 48, 36, 670, 517, 452, 91, 262, 26, 204, 1093, 340, 1062, 844, 987, 212, 53, 527, 96, 1497, 587, 639, 15, 73, 237, 406, 630, 1217, 60, 667, 147, 298, 357, 1251, 729, 116, 50, 718, 879, 3161, 1761, 305, 438, 284, 264, 826, 721, 40, 182] payload_rows_per_dst=[5300, 6222, 7164, 6214, 5556, 5376, 5152, 5522] dup_rows_per_dst=[1652, 2714, 3362, 3376, 1786, 2188, 1556, 2396] combine_rows_per_src=[5300, 6222, 7164, 6214, 5556, 5376, 5152, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45540 payload_rows_remote=39622 dup_rows=19996 dup_rows_remote=17186 dedup_ratio=0.305115 router_tokens_per_expert=[508, 201, 354, 18, 32, 31, 1115, 24, 1045, 934, 188, 106, 1311, 143, 107, 426, 58, 57, 456, 566, 661, 22, 188, 57, 445, 1129, 370, 827, 1015, 426, 337, 52, 516, 275, 1265, 425, 689, 48, 165, 479, 521, 1231, 797, 153, 972, 263, 62, 365, 1521, 735, 208, 48, 608, 955, 2504, 1099, 406, 644, 279, 54, 552, 1217, 141, 362] payload_rows_per_dst=[5534, 5968, 6396, 5896, 5334, 5918, 4334, 6160] dup_rows_per_dst=[2074, 2552, 2742, 3306, 2390, 2810, 774, 3348] combine_rows_per_src=[5534, 5968, 6396, 5896, 5334, 5918, 4334, 6160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43514 payload_rows_remote=37952 dup_rows=22022 dup_rows_remote=19870 dedup_ratio=0.336029 router_tokens_per_expert=[376, 173, 989, 18, 131, 10, 1072, 187, 883, 1092, 195, 294, 895, 163, 355, 395, 92, 30, 498, 276, 240, 72, 183, 75, 256, 1349, 402, 862, 492, 884, 283, 32, 308, 137, 1398, 279, 1052, 26, 165, 555, 447, 682, 1490, 168, 675, 149, 127, 429, 1267, 683, 134, 28, 1071, 1902, 2485, 1274, 359, 529, 189, 67, 510, 640, 85, 204] payload_rows_per_dst=[5778, 5014, 2612, 4934, 6144, 5764, 7706, 5562] dup_rows_per_dst=[2246, 1740, 320, 1522, 3486, 2570, 7986, 2152] combine_rows_per_src=[5778, 5014, 2612, 4934, 6144, 5764, 7706, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45186 payload_rows_remote=39110 dup_rows=20350 dup_rows_remote=18216 dedup_ratio=0.310516 router_tokens_per_expert=[404, 144, 589, 10, 166, 20, 910, 96, 1152, 625, 268, 271, 1359, 158, 52, 405, 157, 67, 291, 226, 923, 162, 107, 24, 247, 1168, 346, 714, 452, 542, 317, 32, 1086, 674, 1143, 477, 731, 36, 109, 745, 686, 619, 1210, 79, 812, 215, 188, 215, 1243, 391, 175, 37, 355, 1228, 3265, 1432, 231, 459, 221, 64, 822, 614, 78, 724] payload_rows_per_dst=[5414, 5076, 4882, 5294, 6634, 5570, 6076, 6240] dup_rows_per_dst=[1750, 1988, 1724, 2342, 4884, 2478, 2134, 3050] combine_rows_per_src=[5414, 5076, 4882, 5294, 6634, 5570, 6076, 6240] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46502 payload_rows_remote=39790 dup_rows=19034 dup_rows_remote=15934 dedup_ratio=0.290436 router_tokens_per_expert=[682, 188, 506, 9, 119, 20, 993, 71, 705, 1266, 287, 71, 1364, 213, 160, 475, 89, 16, 547, 252, 385, 105, 199, 29, 187, 914, 388, 898, 514, 352, 194, 54, 526, 430, 1104, 399, 1037, 12, 69, 244, 679, 596, 1083, 103, 838, 210, 52, 203, 2318, 388, 170, 24, 423, 1251, 3225, 1794, 444, 374, 188, 142, 1009, 868, 87, 226] payload_rows_per_dst=[6712, 6382, 7184, 5080, 5314, 4916, 5962, 4952] dup_rows_per_dst=[3100, 3254, 2510, 2132, 2328, 1848, 2138, 1724] combine_rows_per_src=[6712, 6382, 7184, 5080, 5314, 4916, 5962, 4952] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46214 payload_rows_remote=39862 dup_rows=19322 dup_rows_remote=16898 dedup_ratio=0.294830 router_tokens_per_expert=[370, 117, 957, 25, 154, 40, 365, 116, 660, 1060, 396, 330, 854, 186, 190, 472, 63, 60, 524, 189, 518, 100, 263, 48, 340, 1235, 652, 955, 411, 404, 286, 92, 528, 198, 924, 551, 785, 47, 62, 691, 345, 1077, 1274, 218, 1003, 129, 284, 289, 1519, 455, 101, 14, 838, 1583, 2623, 1247, 336, 521, 279, 113, 575, 1385, 87, 255] payload_rows_per_dst=[5534, 5784, 6352, 5850, 5322, 6222, 6036, 5114] dup_rows_per_dst=[1792, 2512, 2424, 2900, 2250, 3016, 2440, 1988] combine_rows_per_src=[5534, 5784, 6352, 5850, 5322, 6222, 6036, 5114] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46488 payload_rows_remote=39776 dup_rows=19048 dup_rows_remote=15932 dedup_ratio=0.290649 router_tokens_per_expert=[687, 190, 523, 10, 123, 20, 990, 71, 708, 1274, 289, 75, 1383, 216, 164, 471, 91, 16, 554, 250, 377, 112, 199, 27, 187, 931, 399, 893, 515, 349, 195, 53, 528, 416, 1111, 405, 1020, 13, 69, 249, 686, 604, 1073, 105, 830, 211, 52, 201, 2300, 396, 175, 24, 415, 1240, 3215, 1801, 443, 367, 187, 138, 1000, 844, 86, 222] payload_rows_per_dst=[6712, 6398, 7170, 5146, 5292, 4900, 5966, 4904] dup_rows_per_dst=[3116, 3212, 2512, 2222, 2330, 1850, 2136, 1670] combine_rows_per_src=[6712, 6398, 7170, 5146, 5292, 4900, 5966, 4904] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=285372416 dispatch_meta_bytes=226952 combine_payload_bytes=285372416 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=285599368 ingress_bytes=395472896 link_bytes=395472896 token_bytes=570971784 weight_movement_bytes=110100480 total_nvlink_bytes=681072264 weight_to_token_ratio=0.192830 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=18048 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.877924103e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=275437568 dispatch_meta_bytes=231456 combine_payload_bytes=275437568 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=275669024 ingress_bytes=385538048 link_bytes=385538048 token_bytes=551106592 weight_movement_bytes=110100480 total_nvlink_bytes=661207072 weight_to_token_ratio=0.199781 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=8576 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.705140756e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=292726784 dispatch_meta_bytes=232784 combine_payload_bytes=292726784 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=292959568 ingress_bytes=402827264 link_bytes=402827264 token_bytes=585686352 weight_movement_bytes=110100480 total_nvlink_bytes=695786832 weight_to_token_ratio=0.187985 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=8704 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.005828658e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=284168192 dispatch_meta_bytes=227264 combine_payload_bytes=284168192 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=284395456 ingress_bytes=504369152 link_bytes=504369152 token_bytes=568563648 weight_movement_bytes=220200960 total_nvlink_bytes=788764608 weight_to_token_ratio=0.387293 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=8576 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.771809098e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45818 payload_rows_remote=40394 dup_rows=19718 dup_rows_remote=17666 dedup_ratio=0.300873 router_tokens_per_expert=[691, 165, 1150, 12, 162, 1, 741, 75, 929, 863, 191, 195, 1266, 227, 31, 366, 130, 18, 390, 368, 785, 31, 224, 47, 327, 634, 349, 782, 711, 532, 345, 58, 665, 395, 715, 320, 670, 47, 142, 492, 548, 1000, 1205, 212, 1246, 204, 73, 207, 1440, 365, 216, 71, 455, 840, 2446, 807, 649, 834, 450, 105, 665, 1640, 191, 657] payload_rows_per_dst=[6100, 5688, 6230, 5424, 4940, 6408, 4522, 6506] dup_rows_per_dst=[2774, 2448, 2648, 2052, 1952, 2982, 986, 3876] combine_rows_per_src=[6100, 5688, 6230, 5424, 4940, 6408, 4522, 6506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46050 payload_rows_remote=40302 dup_rows=19486 dup_rows_remote=17040 dedup_ratio=0.297333 router_tokens_per_expert=[748, 167, 903, 8, 154, 6, 900, 106, 966, 1229, 215, 118, 919, 234, 85, 331, 78, 67, 363, 198, 356, 75, 587, 60, 660, 1456, 326, 780, 327, 709, 218, 131, 878, 249, 563, 257, 641, 193, 82, 745, 417, 1147, 644, 96, 805, 289, 61, 276, 1896, 281, 98, 20, 853, 1596, 2709, 1019, 378, 361, 172, 73, 438, 1553, 130, 368] payload_rows_per_dst=[6522, 5748, 6546, 6040, 5132, 5490, 5666, 4906] dup_rows_per_dst=[3254, 2446, 2440, 3174, 2084, 1980, 2068, 2040] combine_rows_per_src=[6522, 5748, 6546, 6040, 5132, 5490, 5666, 4906] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=293529600 dispatch_meta_bytes=232776 combine_payload_bytes=293529600 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=293762376 ingress_bytes=403630080 link_bytes=403630080 token_bytes=587291976 weight_movement_bytes=110100480 total_nvlink_bytes=697392456 weight_to_token_ratio=0.187471 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=8704 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.019790949e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=289458176 dispatch_meta_bytes=232144 combine_payload_bytes=289458176 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=289690320 ingress_bytes=509659136 link_bytes=509659136 token_bytes=579148496 weight_movement_bytes=220200960 total_nvlink_bytes=799349456 weight_to_token_ratio=0.380215 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=7680 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.863810620e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=276742144 dispatch_meta_bytes=222936 combine_payload_bytes=276742144 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=276965080 ingress_bytes=717144064 link_bytes=717144064 token_bytes=553707224 weight_movement_bytes=440401920 total_nvlink_bytes=994109144 weight_to_token_ratio=0.795370 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=10240 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.247231477e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=288210944 dispatch_meta_bytes=229208 combine_payload_bytes=288210944 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=288440152 ingress_bytes=508411904 link_bytes=508411904 token_bytes=576651096 weight_movement_bytes=220200960 total_nvlink_bytes=796852056 weight_to_token_ratio=0.381862 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=8064 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.842119204e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=285114368 dispatch_meta_bytes=222832 combine_payload_bytes=285114368 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=285337200 ingress_bytes=395214848 link_bytes=395214848 token_bytes=570451568 weight_movement_bytes=110100480 total_nvlink_bytes=680552048 weight_to_token_ratio=0.193006 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=12160 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.873436223e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43946 payload_rows_remote=36484 dup_rows=21590 dup_rows_remote=14546 dedup_ratio=0.329437 router_tokens_per_expert=[15, 13, 1161, 328, 47, 28, 467, 26, 1028, 671, 217, 16, 1196, 1872, 34, 1763, 0, 367, 17, 491, 24, 480, 34, 868, 149, 486, 3, 1138, 1407, 210, 17, 31, 2372, 822, 4, 93, 125, 3, 103, 44, 9, 6, 1606, 187, 336, 0, 2, 167, 1382, 1843, 300, 394, 851, 1348, 59, 51, 47, 107, 778, 151, 1693, 2611, 121, 549] payload_rows_per_dst=[3654, 3542, 5960, 5092, 5148, 5874, 7214, 7462] dup_rows_per_dst=[516, 390, 2128, 1790, 1056, 2496, 6170, 7044] combine_rows_per_src=[3654, 3542, 5960, 5092, 5148, 5874, 7214, 7462] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47628 payload_rows_remote=41398 dup_rows=17908 dup_rows_remote=15966 dedup_ratio=0.273254 router_tokens_per_expert=[40, 13, 1611, 480, 29, 52, 443, 80, 994, 1377, 303, 15, 1999, 1734, 161, 2353, 1, 349, 64, 389, 4, 545, 12, 369, 235, 94, 12, 269, 1138, 330, 66, 101, 2773, 741, 2, 271, 292, 7, 182, 12, 16, 0, 1669, 117, 555, 2, 15, 254, 1737, 1883, 228, 117, 369, 758, 80, 8, 18, 37, 1520, 121, 469, 2496, 96, 261] payload_rows_per_dst=[6638, 6744, 6230, 6194, 6450, 5920, 5048, 4404] dup_rows_per_dst=[3850, 2954, 1942, 2062, 2110, 2804, 1546, 640] combine_rows_per_src=[6638, 6744, 6230, 6194, 6450, 5920, 5048, 4404] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47918 payload_rows_remote=40612 dup_rows=17618 dup_rows_remote=15860 dedup_ratio=0.268829 router_tokens_per_expert=[4, 38, 1686, 830, 17, 33, 554, 28, 999, 698, 317, 26, 2051, 3085, 20, 1788, 0, 85, 37, 955, 47, 229, 12, 775, 71, 308, 8, 829, 1213, 163, 76, 41, 3506, 736, 0, 33, 204, 5, 39, 9, 2, 19, 1291, 110, 149, 1, 0, 273, 1745, 1200, 92, 333, 266, 279, 27, 82, 30, 146, 500, 48, 2317, 2013, 121, 169] payload_rows_per_dst=[6586, 5224, 6168, 4266, 7306, 7164, 5704, 5500] dup_rows_per_dst=[3820, 1458, 3228, 1154, 1758, 2696, 2342, 1162] combine_rows_per_src=[6586, 5224, 6168, 4266, 7306, 7164, 5704, 5500] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47196 payload_rows_remote=39658 dup_rows=18340 dup_rows_remote=13482 dedup_ratio=0.279846 router_tokens_per_expert=[72, 59, 1036, 167, 30, 27, 616, 4, 1659, 1534, 341, 14, 1904, 2951, 26, 2125, 1, 100, 88, 1025, 1, 162, 0, 660, 147, 666, 14, 544, 1705, 49, 55, 19, 2455, 949, 0, 169, 232, 63, 6, 22, 16, 9, 762, 147, 319, 3, 14, 170, 1103, 1928, 747, 62, 531, 314, 126, 18, 29, 69, 1454, 22, 410, 1891, 370, 557] payload_rows_per_dst=[5688, 5540, 6344, 4906, 6020, 7538, 6324, 4836] dup_rows_per_dst=[2116, 1608, 2094, 1492, 1772, 4858, 3334, 1066] combine_rows_per_src=[5688, 5540, 6344, 4906, 6020, 7538, 6324, 4836] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46388 payload_rows_remote=40566 dup_rows=19148 dup_rows_remote=16798 dedup_ratio=0.292175 router_tokens_per_expert=[76, 7, 783, 376, 91, 42, 344, 5, 1550, 1469, 334, 0, 1890, 1729, 19, 1576, 0, 410, 33, 641, 2, 937, 17, 612, 109, 354, 3, 1136, 1342, 207, 3, 145, 3228, 752, 1, 198, 49, 0, 25, 110, 5, 6, 733, 72, 337, 0, 11, 277, 1293, 1512, 147, 615, 355, 78, 72, 14, 47, 47, 1798, 110, 1610, 2363, 197, 464] payload_rows_per_dst=[5164, 5278, 5888, 5028, 6958, 4928, 5822, 7322] dup_rows_per_dst=[1768, 1466, 2568, 1570, 1768, 1412, 2350, 6246] combine_rows_per_src=[5164, 5278, 5888, 5028, 6958, 4928, 5822, 7322] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48572 payload_rows_remote=41628 dup_rows=16964 dup_rows_remote=13536 dedup_ratio=0.258850 router_tokens_per_expert=[76, 64, 1298, 381, 6, 91, 403, 1, 969, 658, 323, 81, 2751, 1957, 9, 2641, 1, 181, 80, 436, 6, 79, 7, 863, 196, 273, 3, 1238, 1528, 82, 7, 17, 2984, 752, 0, 141, 188, 9, 193, 13, 5, 1, 566, 70, 315, 0, 0, 467, 1288, 1842, 946, 41, 391, 107, 20, 7, 27, 21, 2095, 69, 1001, 1731, 331, 441] payload_rows_per_dst=[5752, 6822, 6540, 6944, 6592, 5218, 4590, 6114] dup_rows_per_dst=[2350, 2760, 2048, 3428, 1968, 1544, 1010, 1856] combine_rows_per_src=[5752, 6822, 6540, 6944, 6592, 5218, 4590, 6114] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=284010496 dispatch_meta_bytes=227232 combine_payload_bytes=284010496 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=284237728 ingress_bytes=504211456 link_bytes=504211456 token_bytes=568248224 weight_movement_bytes=220200960 total_nvlink_bytes=788449184 weight_to_token_ratio=0.387508 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=8576 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.769066505e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=272039936 dispatch_meta_bytes=231288 combine_payload_bytes=272039936 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=272271224 ingress_bytes=382140416 link_bytes=382140416 token_bytes=544311160 weight_movement_bytes=110100480 total_nvlink_bytes=654411640 weight_to_token_ratio=0.202275 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=8704 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.646050347e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=285214720 dispatch_meta_bytes=222896 combine_payload_bytes=285214720 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=285437616 ingress_bytes=395315200 link_bytes=395315200 token_bytes=570652336 weight_movement_bytes=110100480 total_nvlink_bytes=680752816 weight_to_token_ratio=0.192938 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=12288 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.875181510e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280340480 dispatch_meta_bytes=229304 combine_payload_bytes=280340480 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=280569784 ingress_bytes=720742400 link_bytes=720742400 token_bytes=560910264 weight_movement_bytes=440401920 total_nvlink_bytes=1001312184 weight_to_token_ratio=0.785156 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=10240 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.253489575e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=285730816 dispatch_meta_bytes=227040 combine_payload_bytes=285730816 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=285957856 ingress_bytes=395831296 link_bytes=395831296 token_bytes=571688672 weight_movement_bytes=110100480 total_nvlink_bytes=681789152 weight_to_token_ratio=0.192588 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=18048 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.884157268e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=289544192 dispatch_meta_bytes=232240 combine_payload_bytes=289544192 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=289776432 ingress_bytes=509745152 link_bytes=509745152 token_bytes=579320624 weight_movement_bytes=220200960 total_nvlink_bytes=799521584 weight_to_token_ratio=0.380102 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=7680 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.865306580e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=288884736 dispatch_meta_bytes=229368 combine_payload_bytes=288884736 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=289114104 ingress_bytes=509085696 link_bytes=509085696 token_bytes=577998840 weight_movement_bytes=220200960 total_nvlink_bytes=798199800 weight_to_token_ratio=0.380971 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=8064 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.853837556e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48294 payload_rows_remote=40976 dup_rows=17242 dup_rows_remote=15486 dedup_ratio=0.263092 router_tokens_per_expert=[4, 31, 1730, 833, 18, 30, 532, 30, 972, 693, 305, 22, 2014, 3066, 18, 1810, 0, 83, 37, 933, 37, 223, 13, 800, 64, 317, 7, 838, 1217, 166, 76, 38, 3505, 735, 1, 32, 209, 7, 38, 10, 2, 17, 1312, 95, 145, 1, 0, 268, 1763, 1221, 88, 349, 270, 269, 30, 83, 30, 144, 506, 51, 2330, 2012, 127, 161] payload_rows_per_dst=[6574, 5942, 5740, 4296, 7318, 7166, 5762, 5496] dup_rows_per_dst=[3866, 2106, 2132, 1150, 1756, 2646, 2384, 1202] combine_rows_per_src=[6574, 5942, 5740, 4296, 7318, 7166, 5762, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47450 payload_rows_remote=40664 dup_rows=18086 dup_rows_remote=16048 dedup_ratio=0.275970 router_tokens_per_expert=[73, 61, 1061, 165, 33, 26, 578, 4, 1633, 1536, 340, 14, 1891, 2956, 25, 2129, 1, 107, 89, 999, 1, 158, 0, 679, 145, 666, 13, 553, 1713, 47, 57, 19, 2473, 931, 0, 168, 241, 62, 6, 21, 15, 10, 774, 148, 317, 2, 13, 177, 1120, 1926, 739, 63, 558, 305, 129, 21, 27, 69, 1471, 20, 414, 1849, 384, 543] payload_rows_per_dst=[5624, 6022, 6672, 4942, 6220, 6786, 6360, 4824] dup_rows_per_dst=[2076, 2438, 3262, 1484, 2394, 2038, 3362, 1032] combine_rows_per_src=[5624, 6022, 6672, 4942, 6220, 6786, 6360, 4824] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=41890 payload_rows_remote=34214 dup_rows=23646 dup_rows_remote=14600 dedup_ratio=0.360809 router_tokens_per_expert=[15, 16, 1170, 329, 43, 29, 445, 21, 1025, 663, 206, 17, 1198, 1900, 27, 1758, 0, 370, 17, 494, 20, 477, 33, 884, 150, 498, 4, 1152, 1433, 204, 14, 30, 2355, 804, 4, 97, 123, 3, 100, 40, 8, 6, 1626, 172, 341, 0, 2, 162, 1375, 1865, 299, 434, 862, 1328, 60, 54, 44, 117, 730, 152, 1711, 2595, 133, 524] payload_rows_per_dst=[3652, 3502, 5966, 5166, 2198, 6684, 7046, 7676] dup_rows_per_dst=[484, 374, 2140, 1804, 144, 4146, 5508, 9046] combine_rows_per_src=[3652, 3502, 5966, 5166, 2198, 6684, 7046, 7676] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48662 payload_rows_remote=41656 dup_rows=16874 dup_rows_remote=13306 dedup_ratio=0.257477 router_tokens_per_expert=[33, 11, 1756, 270, 43, 32, 707, 51, 1579, 1549, 342, 12, 1745, 1723, 60, 1974, 1, 85, 106, 410, 26, 630, 6, 353, 310, 75, 50, 485, 1903, 51, 13, 22, 3517, 679, 77, 24, 248, 79, 255, 4, 49, 2, 652, 307, 183, 20, 38, 248, 1908, 1410, 163, 54, 203, 970, 114, 49, 11, 21, 1982, 56, 341, 2433, 107, 151] payload_rows_per_dst=[6826, 7006, 5748, 6438, 7352, 4950, 5532, 4810] dup_rows_per_dst=[3846, 3568, 1434, 2200, 2414, 1494, 1390, 528] combine_rows_per_src=[6826, 7006, 5748, 6438, 7352, 4950, 5532, 4810] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=261517312 dispatch_meta_bytes=204120 combine_payload_bytes=261517312 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=261721432 ingress_bytes=481718272 link_bytes=481718272 token_bytes=523238744 weight_movement_bytes=220200960 total_nvlink_bytes=743439704 weight_to_token_ratio=0.420842 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=11520 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.377873040e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46798 payload_rows_remote=41314 dup_rows=18738 dup_rows_remote=17260 dedup_ratio=0.285919 router_tokens_per_expert=[2, 1, 669, 84, 10, 26, 856, 1, 930, 1495, 157, 12, 1771, 981, 5, 1403, 1, 237, 102, 398, 1, 453, 8, 1005, 134, 258, 2, 1128, 2426, 132, 29, 13, 3208, 828, 0, 224, 361, 14, 14, 1, 60, 0, 1453, 83, 606, 0, 6, 131, 1887, 2364, 369, 891, 213, 162, 58, 0, 12, 9, 1324, 47, 907, 1832, 474, 500] payload_rows_per_dst=[5484, 6958, 5508, 7518, 5618, 4852, 5738, 5122] dup_rows_per_dst=[1478, 4218, 1708, 5454, 1246, 1788, 1422, 1424] combine_rows_per_src=[5484, 6958, 5508, 7518, 5618, 4852, 5738, 5122] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=296740864 dispatch_meta_bytes=229456 combine_payload_bytes=296740864 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=296970320 ingress_bytes=516941824 link_bytes=516941824 token_bytes=593711184 weight_movement_bytes=220200960 total_nvlink_bytes=813912144 weight_to_token_ratio=0.370889 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.033088 gemm_active_groups=11 gemm_peak_group_rows=15744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.990468543e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=291106816 dispatch_meta_bytes=225888 combine_payload_bytes=291106816 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=291332704 ingress_bytes=511307776 link_bytes=511307776 token_bytes=582439520 weight_movement_bytes=220200960 total_nvlink_bytes=802640480 weight_to_token_ratio=0.378067 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=22400 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.892483181e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=284268544 dispatch_meta_bytes=212560 combine_payload_bytes=284268544 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=284481104 ingress_bytes=504469504 link_bytes=504469504 token_bytes=568749648 weight_movement_bytes=220200960 total_nvlink_bytes=788950608 weight_to_token_ratio=0.387167 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.036765 gemm_active_groups=11 gemm_peak_group_rows=16128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.773554384e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 2, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47512 payload_rows_remote=42060 dup_rows=18024 dup_rows_remote=16546 dedup_ratio=0.275024 router_tokens_per_expert=[3, 1, 683, 90, 12, 23, 839, 1, 888, 1466, 154, 10, 1746, 976, 4, 1370, 1, 238, 97, 376, 1, 450, 8, 1051, 127, 266, 2, 1139, 2453, 137, 31, 13, 3201, 834, 0, 221, 395, 14, 15, 0, 61, 0, 1471, 83, 613, 0, 5, 130, 1898, 2376, 349, 944, 217, 145, 54, 0, 11, 9, 1326, 48, 912, 1813, 489, 478] payload_rows_per_dst=[5452, 6336, 5520, 7540, 6982, 4872, 5670, 5140] dup_rows_per_dst=[1478, 2430, 1664, 5548, 2378, 1806, 1314, 1406] combine_rows_per_src=[5452, 6336, 5520, 7540, 6982, 4872, 5670, 5140] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45614 payload_rows_remote=39712 dup_rows=19922 dup_rows_remote=17516 dedup_ratio=0.303986 router_tokens_per_expert=[75, 7, 790, 395, 87, 39, 330, 4, 1547, 1474, 341, 0, 1857, 1721, 21, 1585, 0, 420, 33, 615, 2, 927, 17, 627, 111, 359, 2, 1150, 1346, 211, 4, 141, 3226, 752, 1, 196, 48, 0, 22, 115, 4, 8, 733, 69, 334, 0, 11, 269, 1305, 1519, 147, 664, 356, 79, 69, 15, 50, 47, 1789, 109, 1633, 2340, 198, 422] payload_rows_per_dst=[5424, 5282, 5882, 5066, 5926, 5484, 5902, 6648] dup_rows_per_dst=[1914, 1484, 2570, 1582, 3444, 1878, 2406, 4644] combine_rows_per_src=[5424, 5282, 5882, 5066, 5926, 5484, 5902, 6648] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=293715968 dispatch_meta_bytes=225848 combine_payload_bytes=293715968 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=293941816 ingress_bytes=513916928 link_bytes=513916928 token_bytes=587657784 weight_movement_bytes=220200960 total_nvlink_bytes=807858744 weight_to_token_ratio=0.374710 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=20736 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.937860626e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=291479552 dispatch_meta_bytes=226848 combine_payload_bytes=291479552 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=291706400 ingress_bytes=511680512 link_bytes=511680512 token_bytes=583185952 weight_movement_bytes=220200960 total_nvlink_bytes=803386912 weight_to_token_ratio=0.377583 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.033088 gemm_active_groups=11 gemm_peak_group_rows=16128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.898965674e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=290777088 dispatch_meta_bytes=229456 combine_payload_bytes=290777088 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=291006544 ingress_bytes=510978048 link_bytes=510978048 token_bytes=581783632 weight_movement_bytes=220200960 total_nvlink_bytes=801984592 weight_to_token_ratio=0.378493 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=12416 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.886748669e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=298389504 dispatch_meta_bytes=220656 combine_payload_bytes=298389504 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=298610160 ingress_bytes=408489984 link_bytes=408489984 token_bytes=596999664 weight_movement_bytes=110100480 total_nvlink_bytes=707100144 weight_to_token_ratio=0.184423 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=12672 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.104312672e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=298590208 dispatch_meta_bytes=219848 combine_payload_bytes=298590208 prefetch_bytes=110100480 reduce_bytes=550502400 egress_bytes=298810056 ingress_bytes=959193088 link_bytes=959193088 token_bytes=597400264 weight_movement_bytes=660602880 total_nvlink_bytes=1258003144 weight_to_token_ratio=1.105796 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=9728 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.668194540e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=296138752 dispatch_meta_bytes=234296 combine_payload_bytes=296138752 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=296373048 ingress_bytes=406239232 link_bytes=406239232 token_bytes=592511800 weight_movement_bytes=110100480 total_nvlink_bytes=702612280 weight_to_token_ratio=0.185820 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=14208 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.065168393e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47036 payload_rows_remote=40492 dup_rows=18500 dup_rows_remote=15586 dedup_ratio=0.282288 router_tokens_per_expert=[668, 1701, 44, 5, 11, 235, 336, 3, 18, 2660, 6, 22, 632, 1137, 3209, 3, 0, 562, 0, 84, 32, 2050, 5, 15, 1364, 0, 247, 1899, 1, 11, 25, 1446, 1251, 442, 11, 189, 424, 15, 1574, 3, 390, 446, 10, 63, 970, 2472, 0, 0, 4, 0, 1549, 0, 735, 0, 632, 2, 493, 8, 19, 19, 884, 0, 1532, 200] payload_rows_per_dst=[5234, 3370, 7298, 5808, 4030, 7122, 7630, 6544] dup_rows_per_dst=[772, 266, 3518, 1514, 752, 4132, 4632, 2914] combine_rows_per_src=[5234, 3370, 7298, 5808, 4030, 7122, 7630, 6544] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47322 payload_rows_remote=43156 dup_rows=18214 dup_rows_remote=17392 dedup_ratio=0.277924 router_tokens_per_expert=[443, 2121, 116, 24, 7, 166, 678, 2, 0, 2982, 9, 14, 320, 505, 2735, 0, 0, 262, 0, 1227, 61, 1510, 116, 42, 2120, 7, 1487, 1331, 1, 2, 19, 1169, 402, 367, 5, 137, 354, 1, 1219, 9, 637, 104, 0, 30, 1138, 2118, 0, 0, 33, 0, 1160, 1, 948, 0, 248, 0, 784, 0, 23, 4, 759, 1, 2420, 390] payload_rows_per_dst=[5862, 4904, 5894, 7414, 4166, 5932, 7146, 6004] dup_rows_per_dst=[1560, 856, 2442, 4858, 822, 2122, 3104, 2450] combine_rows_per_src=[5862, 4904, 5894, 7414, 4166, 5932, 7146, 6004] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46554 payload_rows_remote=42170 dup_rows=18982 dup_rows_remote=18090 dedup_ratio=0.289642 router_tokens_per_expert=[502, 1224, 122, 5, 21, 440, 856, 1, 0, 2533, 26, 1, 93, 960, 3018, 0, 5, 556, 7, 852, 13, 1077, 116, 12, 499, 0, 588, 2805, 16, 28, 36, 1158, 1072, 613, 6, 830, 436, 2, 2142, 31, 312, 158, 0, 3, 459, 2230, 0, 0, 0, 0, 2463, 45, 219, 9, 344, 2, 906, 0, 113, 7, 1242, 0, 792, 762] payload_rows_per_dst=[5800, 7742, 4384, 6188, 6824, 5368, 5384, 4864] dup_rows_per_dst=[2126, 7024, 892, 2568, 3440, 956, 780, 1196] combine_rows_per_src=[5800, 7742, 4384, 6188, 6824, 5368, 5384, 4864] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46950 payload_rows_remote=40792 dup_rows=18586 dup_rows_remote=17154 dedup_ratio=0.283600 router_tokens_per_expert=[1270, 983, 70, 19, 39, 194, 1212, 6, 4, 2037, 9, 2, 151, 203, 3054, 0, 0, 345, 0, 584, 212, 1272, 5, 1, 1397, 0, 191, 1976, 35, 14, 135, 719, 1318, 882, 16, 219, 1207, 2, 1432, 30, 388, 120, 0, 7, 569, 2705, 2, 4, 1, 0, 1782, 0, 495, 3, 208, 0, 812, 2, 15, 36, 840, 1, 2929, 604] payload_rows_per_dst=[6032, 722, 6566, 6640, 6560, 6158, 7398, 6874] dup_rows_per_dst=[1554, 16, 2346, 2294, 3652, 1432, 3688, 3604] combine_rows_per_src=[6032, 722, 6566, 6640, 6560, 6158, 7398, 6874] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47044 payload_rows_remote=41068 dup_rows=18492 dup_rows_remote=16606 dedup_ratio=0.282166 router_tokens_per_expert=[1830, 852, 119, 5, 0, 187, 397, 0, 76, 3326, 0, 26, 84, 674, 2196, 1, 0, 297, 14, 248, 16, 2519, 12, 10, 1541, 1, 436, 985, 0, 13, 41, 990, 261, 865, 59, 124, 130, 0, 3162, 24, 196, 252, 0, 83, 902, 2363, 53, 1, 3, 0, 1001, 0, 497, 4, 230, 0, 1521, 2, 3, 9, 1152, 1, 2870, 104] payload_rows_per_dst=[5612, 1666, 7768, 6016, 6944, 6008, 5976, 7054] dup_rows_per_dst=[1168, 56, 5116, 1998, 2086, 1692, 1886, 4490] combine_rows_per_src=[5612, 1666, 7768, 6016, 6944, 6008, 5976, 7054] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=290246656 dispatch_meta_bytes=224312 combine_payload_bytes=290246656 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=290470968 ingress_bytes=510447616 link_bytes=510447616 token_bytes=580717624 weight_movement_bytes=220200960 total_nvlink_bytes=800918584 weight_to_token_ratio=0.379188 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=9600 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.877523584e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47644 payload_rows_remote=41824 dup_rows=17892 dup_rows_remote=16110 dedup_ratio=0.273010 router_tokens_per_expert=[1644, 1384, 32, 19, 5, 92, 556, 0, 1, 1814, 9, 29, 104, 272, 1802, 2, 0, 902, 0, 746, 137, 1800, 1, 1, 1781, 0, 137, 1214, 0, 7, 71, 591, 268, 1432, 1, 285, 133, 4, 1916, 20, 339, 849, 0, 2, 1001, 1821, 1, 0, 0, 0, 1983, 0, 313, 0, 230, 0, 2245, 1, 5, 35, 1650, 0, 2845, 236] payload_rows_per_dst=[7690, 5892, 5480, 5820, 6098, 6004, 4554, 6106] dup_rows_per_dst=[5464, 2000, 1694, 1782, 2020, 2022, 672, 2238] combine_rows_per_src=[7690, 5892, 5480, 5820, 6098, 6004, 4554, 6106] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=302274560 dispatch_meta_bytes=241040 combine_payload_bytes=302274560 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=302515600 ingress_bytes=412375040 link_bytes=412375040 token_bytes=604790160 weight_movement_bytes=110100480 total_nvlink_bytes=714890640 weight_to_token_ratio=0.182047 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=13696 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.171880186e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=309342208 dispatch_meta_bytes=242192 combine_payload_bytes=309342208 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=309584400 ingress_bytes=529543168 link_bytes=529543168 token_bytes=618926608 weight_movement_bytes=220200960 total_nvlink_bytes=839127568 weight_to_token_ratio=0.355779 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=12672 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.209626640e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=292397056 dispatch_meta_bytes=231784 combine_payload_bytes=292397056 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=292628840 ingress_bytes=402497536 link_bytes=402497536 token_bytes=585025896 weight_movement_bytes=110100480 total_nvlink_bytes=695126376 weight_to_token_ratio=0.188198 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=19072 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.000094146e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47574 payload_rows_remote=41366 dup_rows=17962 dup_rows_remote=15998 dedup_ratio=0.274078 router_tokens_per_expert=[38, 15, 1604, 480, 27, 48, 425, 77, 985, 1381, 307, 15, 1984, 1723, 166, 2353, 1, 333, 66, 382, 4, 540, 12, 395, 237, 93, 12, 284, 1159, 330, 67, 101, 2767, 727, 1, 276, 289, 8, 177, 11, 15, 0, 1672, 114, 577, 2, 13, 257, 1755, 1911, 213, 135, 386, 736, 88, 8, 19, 38, 1516, 120, 463, 2467, 103, 260] payload_rows_per_dst=[6598, 6706, 6208, 6232, 6416, 5940, 5088, 4386] dup_rows_per_dst=[3764, 2970, 1964, 2156, 2096, 2806, 1554, 652] combine_rows_per_src=[6598, 6706, 6208, 6232, 6416, 5940, 5088, 4386] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48608 payload_rows_remote=41840 dup_rows=16928 dup_rows_remote=13842 dedup_ratio=0.258301 router_tokens_per_expert=[82, 62, 1306, 392, 4, 91, 390, 1, 960, 676, 311, 89, 2724, 1952, 9, 2639, 1, 179, 81, 414, 5, 76, 7, 876, 201, 276, 3, 1247, 1563, 81, 7, 17, 2969, 744, 0, 138, 187, 8, 193, 11, 5, 1, 578, 68, 324, 0, 0, 466, 1302, 1860, 931, 47, 401, 100, 20, 5, 26, 22, 2093, 72, 999, 1697, 353, 426] payload_rows_per_dst=[5722, 6796, 6534, 6768, 6556, 5226, 4858, 6148] dup_rows_per_dst=[2328, 2742, 2022, 3086, 1944, 1562, 1410, 1834] combine_rows_per_src=[5722, 6796, 6534, 6768, 6556, 5226, 4858, 6148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48780 payload_rows_remote=41808 dup_rows=16756 dup_rows_remote=13322 dedup_ratio=0.255676 router_tokens_per_expert=[31, 10, 1757, 278, 42, 35, 685, 52, 1549, 1534, 318, 15, 1728, 1726, 59, 2005, 1, 81, 109, 394, 25, 630, 6, 379, 306, 75, 52, 492, 1923, 48, 14, 21, 3514, 680, 77, 22, 258, 74, 251, 6, 48, 2, 678, 302, 186, 20, 35, 251, 1926, 1427, 148, 60, 215, 954, 109, 44, 11, 22, 1997, 60, 353, 2400, 105, 153] payload_rows_per_dst=[6796, 6972, 5808, 6454, 7348, 4980, 5554, 4868] dup_rows_per_dst=[3784, 3434, 1452, 2262, 2416, 1516, 1358, 534] combine_rows_per_src=[6796, 6972, 5808, 6454, 7348, 4980, 5554, 4868] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=245245952 dispatch_meta_bytes=195256 combine_payload_bytes=245245952 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=245441208 ingress_bytes=465446912 link_bytes=465446912 token_bytes=490687160 weight_movement_bytes=220200960 total_nvlink_bytes=710888120 weight_to_token_ratio=0.448760 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=11520 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.094887328e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=301486080 dispatch_meta_bytes=234424 combine_payload_bytes=301486080 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=301720504 ingress_bytes=411586560 link_bytes=411586560 token_bytes=603206584 weight_movement_bytes=110100480 total_nvlink_bytes=713307064 weight_to_token_ratio=0.182525 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=14208 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.158167222e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=284655616 dispatch_meta_bytes=228912 combine_payload_bytes=284655616 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=284884528 ingress_bytes=504856576 link_bytes=504856576 token_bytes=569540144 weight_movement_bytes=220200960 total_nvlink_bytes=789741104 weight_to_token_ratio=0.386629 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=12416 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.780286203e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 2, 0, 1, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=296511488 dispatch_meta_bytes=229456 combine_payload_bytes=296511488 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=296740944 ingress_bytes=516712448 link_bytes=516712448 token_bytes=593252432 weight_movement_bytes=220200960 total_nvlink_bytes=813453392 weight_to_token_ratio=0.371176 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.036765 gemm_active_groups=11 gemm_peak_group_rows=15744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.986479317e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=299909120 dispatch_meta_bytes=222728 combine_payload_bytes=299909120 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=300131848 ingress_bytes=410009600 link_bytes=410009600 token_bytes=600040968 weight_movement_bytes=110100480 total_nvlink_bytes=710141448 weight_to_token_ratio=0.183488 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=12928 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.130741294e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=299679744 dispatch_meta_bytes=220520 combine_payload_bytes=299679744 prefetch_bytes=110100480 reduce_bytes=550502400 egress_bytes=299900264 ingress_bytes=960282624 link_bytes=960282624 token_bytes=599580008 weight_movement_bytes=660602880 total_nvlink_bytes=1260182888 weight_to_token_ratio=1.101776 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.051471 gemm_active_groups=8 gemm_peak_group_rows=10880 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.670089422e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47352 payload_rows_remote=43210 dup_rows=18184 dup_rows_remote=17348 dedup_ratio=0.277466 router_tokens_per_expert=[438, 2107, 118, 24, 8, 166, 677, 3, 0, 3035, 12, 13, 336, 514, 2720, 0, 0, 242, 0, 1215, 64, 1489, 118, 34, 2144, 10, 1533, 1298, 1, 2, 19, 1170, 413, 368, 7, 130, 333, 0, 1229, 9, 664, 98, 0, 32, 1155, 2115, 0, 0, 34, 0, 1142, 1, 954, 0, 231, 0, 783, 0, 26, 5, 768, 1, 2375, 385] payload_rows_per_dst=[5928, 4766, 5984, 7312, 4142, 5934, 7118, 6168] dup_rows_per_dst=[1804, 874, 2520, 4392, 836, 2194, 3046, 2518] combine_rows_per_src=[5928, 4766, 5984, 7312, 4142, 5934, 7118, 6168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46928 payload_rows_remote=40726 dup_rows=18608 dup_rows_remote=17096 dedup_ratio=0.283936 router_tokens_per_expert=[1272, 978, 69, 17, 37, 198, 1177, 6, 4, 2069, 11, 2, 142, 194, 3028, 0, 0, 343, 0, 576, 228, 1253, 6, 1, 1434, 0, 192, 1962, 34, 11, 135, 729, 1365, 915, 14, 217, 1118, 3, 1414, 33, 387, 124, 0, 9, 614, 2717, 1, 5, 1, 0, 1792, 0, 477, 4, 204, 0, 801, 2, 16, 32, 877, 0, 2923, 595] payload_rows_per_dst=[5996, 686, 6592, 6648, 6558, 6202, 7366, 6880] dup_rows_per_dst=[1512, 20, 2360, 2346, 3600, 1512, 3646, 3612] combine_rows_per_src=[5996, 686, 6592, 6648, 6558, 6202, 7366, 6880] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=309729280 dispatch_meta_bytes=242232 combine_payload_bytes=309729280 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=309971512 ingress_bytes=529930240 link_bytes=529930240 token_bytes=619700792 weight_movement_bytes=220200960 total_nvlink_bytes=839901752 weight_to_token_ratio=0.355334 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=12416 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.216358459e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=291923968 dispatch_meta_bytes=231288 combine_payload_bytes=291923968 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=292155256 ingress_bytes=402024448 link_bytes=402024448 token_bytes=584079224 weight_movement_bytes=110100480 total_nvlink_bytes=694179704 weight_to_token_ratio=0.188503 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=19072 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.991866367e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47412 payload_rows_remote=42560 dup_rows=18124 dup_rows_remote=16734 dedup_ratio=0.276550 router_tokens_per_expert=[676, 1689, 47, 4, 11, 245, 335, 3, 18, 2703, 4, 22, 621, 1136, 3185, 3, 0, 549, 0, 85, 28, 2037, 5, 15, 1418, 0, 248, 1867, 1, 14, 27, 1451, 1290, 439, 11, 192, 370, 12, 1602, 2, 386, 439, 10, 71, 978, 2482, 0, 0, 5, 0, 1537, 0, 728, 0, 643, 3, 472, 7, 22, 22, 896, 0, 1497, 205] payload_rows_per_dst=[5254, 3356, 7336, 6650, 5814, 6558, 7592, 4852] dup_rows_per_dst=[766, 252, 3508, 3402, 2022, 2174, 4610, 1390] combine_rows_per_src=[5254, 3356, 7336, 6650, 5814, 6558, 7592, 4852] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45996 payload_rows_remote=40562 dup_rows=19540 dup_rows_remote=17730 dedup_ratio=0.298157 router_tokens_per_expert=[1342, 701, 19, 43, 34, 48, 533, 0, 0, 3393, 9, 29, 42, 244, 2991, 0, 0, 730, 0, 213, 175, 2292, 0, 3, 1634, 0, 152, 902, 1, 3, 112, 1151, 438, 1021, 3, 607, 578, 37, 1863, 8, 304, 1037, 5, 3, 794, 2884, 0, 1, 15, 0, 1717, 1, 547, 3, 187, 0, 656, 4, 96, 26, 1705, 0, 1052, 380] payload_rows_per_dst=[5434, 7848, 5582, 6520, 4448, 7112, 4248, 4804] dup_rows_per_dst=[1810, 7672, 1244, 3312, 936, 2944, 692, 930] combine_rows_per_src=[5434, 7848, 5582, 6520, 4448, 7112, 4248, 4804] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47038 payload_rows_remote=41130 dup_rows=18498 dup_rows_remote=16680 dedup_ratio=0.282257 router_tokens_per_expert=[1818, 860, 115, 3, 0, 183, 383, 0, 68, 3340, 0, 26, 94, 658, 2173, 1, 0, 287, 13, 231, 19, 2494, 14, 10, 1568, 1, 452, 987, 0, 14, 40, 977, 277, 905, 63, 124, 117, 0, 3160, 26, 211, 240, 0, 84, 960, 2368, 53, 1, 4, 0, 994, 0, 459, 4, 229, 0, 1491, 2, 3, 8, 1180, 1, 2864, 111] payload_rows_per_dst=[5562, 1640, 7754, 6044, 7110, 6028, 5908, 6992] dup_rows_per_dst=[1162, 54, 5062, 2034, 2234, 1806, 1818, 4328] combine_rows_per_src=[5562, 1640, 7754, 6044, 7110, 6028, 5908, 6992] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46926 payload_rows_remote=42304 dup_rows=18610 dup_rows_remote=17926 dedup_ratio=0.283966 router_tokens_per_expert=[1654, 1374, 27, 19, 5, 111, 536, 0, 1, 1828, 13, 25, 109, 274, 1771, 1, 0, 864, 0, 725, 150, 1766, 1, 0, 1835, 0, 140, 1188, 0, 7, 74, 597, 276, 1453, 1, 271, 114, 2, 1956, 27, 334, 851, 0, 2, 1044, 1837, 1, 0, 0, 0, 2000, 0, 338, 0, 212, 0, 2208, 1, 5, 34, 1662, 0, 2803, 241] payload_rows_per_dst=[6530, 5730, 5382, 4622, 6344, 6658, 4720, 6940] dup_rows_per_dst=[3298, 1796, 1630, 684, 2204, 3492, 898, 4608] combine_rows_per_src=[6530, 5730, 5382, 4622, 6344, 6658, 4720, 6940] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45704 payload_rows_remote=41320 dup_rows=19832 dup_rows_remote=18946 dedup_ratio=0.302612 router_tokens_per_expert=[510, 1208, 122, 6, 19, 449, 833, 1, 0, 2556, 24, 2, 90, 943, 2991, 0, 6, 551, 7, 846, 13, 1092, 109, 11, 512, 0, 631, 2791, 15, 27, 37, 1178, 1096, 630, 7, 831, 411, 2, 2142, 31, 311, 135, 0, 3, 465, 2227, 0, 0, 0, 0, 2450, 42, 223, 10, 355, 1, 871, 1, 116, 6, 1276, 1, 780, 765] payload_rows_per_dst=[7204, 7734, 4384, 3958, 6888, 5330, 5348, 4858] dup_rows_per_dst=[4674, 6850, 886, 842, 3600, 952, 814, 1214] combine_rows_per_src=[7204, 7734, 4384, 3958, 6888, 5330, 5348, 4858] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45904 payload_rows_remote=38072 dup_rows=19632 dup_rows_remote=11784 dedup_ratio=0.299561 router_tokens_per_expert=[1250, 544, 224, 0, 0, 197, 431, 15, 3, 1326, 7, 0, 133, 1575, 3153, 2, 65, 115, 32, 1198, 127, 1095, 3, 35, 1740, 8, 123, 1641, 46, 273, 11, 1077, 512, 1263, 2, 114, 1641, 4, 959, 27, 562, 36, 0, 6, 244, 2428, 8, 0, 10, 8, 1948, 5, 1011, 68, 820, 4, 587, 19, 40, 242, 1115, 0, 1897, 739] payload_rows_per_dst=[6486, 7832, 4464, 5296, 6152, 5436, 5704, 4534] dup_rows_per_dst=[2630, 7848, 876, 1260, 2892, 1132, 2044, 950] combine_rows_per_src=[6486, 7832, 4464, 5296, 6152, 5436, 5704, 4534] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46582 payload_rows_remote=40854 dup_rows=18954 dup_rows_remote=17136 dedup_ratio=0.289215 router_tokens_per_expert=[1331, 707, 20, 42, 30, 42, 544, 0, 0, 3390, 6, 33, 38, 238, 3049, 0, 0, 731, 0, 206, 172, 2288, 0, 2, 1620, 0, 152, 917, 1, 4, 130, 1133, 440, 981, 3, 625, 601, 35, 1855, 9, 295, 1033, 6, 3, 769, 2902, 0, 2, 13, 1, 1718, 2, 543, 3, 189, 0, 673, 1, 86, 27, 1686, 0, 1057, 384] payload_rows_per_dst=[5728, 7896, 5558, 4890, 6346, 7116, 4252, 4796] dup_rows_per_dst=[1818, 7446, 1240, 1190, 2752, 2904, 686, 918] combine_rows_per_src=[5728, 7896, 5558, 4890, 6346, 7116, 4252, 4796] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=294375424 dispatch_meta_bytes=230696 combine_payload_bytes=294375424 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=294606120 ingress_bytes=404475904 link_bytes=404475904 token_bytes=588981544 weight_movement_bytes=110100480 total_nvlink_bytes=699082024 weight_to_token_ratio=0.186934 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=13696 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.034501219e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=299794432 dispatch_meta_bytes=231736 combine_payload_bytes=299794432 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=300026168 ingress_bytes=630095872 link_bytes=630095872 token_bytes=599820600 weight_movement_bytes=330301440 total_nvlink_bytes=930122040 weight_to_token_ratio=0.550667 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=10752 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.095840354e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=272900096 dispatch_meta_bytes=199424 combine_payload_bytes=272900096 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=273099520 ingress_bytes=603201536 link_bytes=603201536 token_bytes=545999616 weight_movement_bytes=330301440 total_nvlink_bytes=876301056 weight_to_token_ratio=0.604948 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=11136 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.049066681e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=292841472 dispatch_meta_bytes=231960 combine_payload_bytes=292841472 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=293073432 ingress_bytes=402941952 link_bytes=402941952 token_bytes=585914904 weight_movement_bytes=110100480 total_nvlink_bytes=696015384 weight_to_token_ratio=0.187912 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=9600 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.007823271e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45628 payload_rows_remote=37618 dup_rows=19908 dup_rows_remote=17028 dedup_ratio=0.303772 router_tokens_per_expert=[4, 32, 89, 232, 2122, 9, 2750, 5, 7, 0, 4, 146, 3394, 3409, 121, 5, 630, 11, 2778, 23, 52, 0, 82, 6, 125, 335, 553, 23, 1, 52, 14, 0, 0, 1, 158, 1965, 3, 3974, 517, 81, 2, 3434, 4, 6, 206, 8, 1537, 0, 1619, 11, 149, 10, 376, 0, 159, 63, 206, 115, 2, 0, 2, 2, 127, 1017] payload_rows_per_dst=[2370, 560, 6004, 7200, 6068, 8114, 7302, 8010] dup_rows_per_dst=[178, 6, 1160, 1794, 1820, 9098, 2972, 2880] combine_rows_per_src=[2370, 560, 6004, 7200, 6068, 8114, 7302, 8010] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47174 payload_rows_remote=40160 dup_rows=18362 dup_rows_remote=14264 dedup_ratio=0.280182 router_tokens_per_expert=[8, 79, 254, 51, 3162, 4, 2691, 17, 15, 0, 65, 137, 3103, 2974, 145, 175, 840, 3, 2227, 22, 30, 0, 16, 9, 93, 335, 1240, 2, 0, 113, 19, 1, 0, 0, 115, 1672, 0, 3787, 46, 348, 20, 2409, 51, 1, 195, 0, 433, 3, 2825, 10, 76, 0, 53, 0, 247, 117, 434, 33, 0, 0, 43, 0, 5, 2015] payload_rows_per_dst=[7906, 7358, 7014, 4952, 8056, 1334, 6002, 4552] dup_rows_per_dst=[4626, 3178, 4098, 1346, 3880, 72, 654, 508] combine_rows_per_src=[7906, 7358, 7014, 4952, 8056, 1334, 6002, 4552] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52734 payload_rows_remote=46232 dup_rows=12802 dup_rows_remote=11474 dedup_ratio=0.195343 router_tokens_per_expert=[0, 17, 89, 191, 2945, 289, 3005, 68, 37, 0, 40, 130, 3236, 3388, 146, 23, 167, 40, 3432, 94, 164, 0, 39, 6, 7, 340, 680, 155, 0, 57, 384, 0, 21, 2, 20, 1059, 5, 3980, 618, 25, 1, 2768, 16, 61, 171, 0, 897, 1, 1898, 6, 278, 8, 178, 0, 196, 10, 314, 128, 14, 3, 4, 1, 82, 834] payload_rows_per_dst=[6344, 6960, 7060, 7120, 3228, 6502, 7416, 8104] dup_rows_per_dst=[854, 568, 824, 2598, 272, 1328, 3742, 2616] combine_rows_per_src=[6344, 6960, 7060, 7120, 3228, 6502, 7416, 8104] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51640 payload_rows_remote=45082 dup_rows=13896 dup_rows_remote=10696 dedup_ratio=0.212036 router_tokens_per_expert=[4, 224, 122, 51, 2116, 9, 1905, 5, 2, 0, 1, 191, 3636, 3854, 17, 3, 275, 3, 1739, 2, 13, 0, 144, 5, 281, 104, 974, 3, 1, 20, 210, 0, 0, 0, 0, 2300, 63, 3683, 559, 12, 0, 3921, 26, 1, 160, 0, 1155, 3, 2544, 1, 50, 0, 20, 0, 345, 6, 100, 163, 1, 0, 2, 0, 62, 1677] payload_rows_per_dst=[5154, 7734, 4066, 7632, 6558, 8000, 6690, 5806] dup_rows_per_dst=[610, 402, 296, 2826, 3200, 2532, 2350, 1680] combine_rows_per_src=[5154, 7734, 4066, 7632, 6558, 8000, 6690, 5806] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=305070080 dispatch_meta_bytes=237176 combine_payload_bytes=305070080 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=305307256 ingress_bytes=635371520 link_bytes=635371520 token_bytes=610377336 weight_movement_bytes=330301440 total_nvlink_bytes=940678776 weight_to_token_ratio=0.541143 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=11392 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.105015574e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45546 payload_rows_remote=37674 dup_rows=19990 dup_rows_remote=11604 dedup_ratio=0.305023 router_tokens_per_expert=[1263, 543, 223, 0, 0, 195, 436, 16, 5, 1392, 7, 0, 139, 1559, 3140, 2, 65, 106, 32, 1185, 124, 1106, 4, 34, 1797, 8, 124, 1625, 45, 269, 10, 1072, 519, 1300, 3, 115, 1548, 5, 943, 27, 580, 36, 1, 6, 251, 2439, 7, 0, 10, 8, 1924, 5, 1023, 67, 816, 3, 570, 18, 43, 241, 1116, 0, 1885, 733] payload_rows_per_dst=[6192, 7872, 4418, 5398, 6022, 5484, 5670, 4490] dup_rows_per_dst=[2410, 8386, 894, 1372, 2778, 1156, 2042, 952] combine_rows_per_src=[6192, 7872, 4418, 5398, 6022, 5484, 5670, 4490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=294819840 dispatch_meta_bytes=231240 combine_payload_bytes=294819840 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=295051080 ingress_bytes=404920320 link_bytes=404920320 token_bytes=589870920 weight_movement_bytes=110100480 total_nvlink_bytes=699971400 weight_to_token_ratio=0.186652 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=13568 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.042230344e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=290748416 dispatch_meta_bytes=233168 combine_payload_bytes=290748416 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=290981584 ingress_bytes=400848896 link_bytes=400848896 token_bytes=581730000 weight_movement_bytes=110100480 total_nvlink_bytes=691830480 weight_to_token_ratio=0.189264 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=9472 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.971421585e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=303235072 dispatch_meta_bytes=240920 combine_payload_bytes=303235072 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=303475992 ingress_bytes=523436032 link_bytes=523436032 token_bytes=606711064 weight_movement_bytes=220200960 total_nvlink_bytes=826912024 weight_to_token_ratio=0.362942 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=12416 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.103413501e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=296181760 dispatch_meta_bytes=241064 combine_payload_bytes=296181760 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=296422824 ingress_bytes=406282240 link_bytes=406282240 token_bytes=592604584 weight_movement_bytes=110100480 total_nvlink_bytes=702705064 weight_to_token_ratio=0.185791 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=13568 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.065916373e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51338 payload_rows_remote=44258 dup_rows=14198 dup_rows_remote=10272 dedup_ratio=0.216644 router_tokens_per_expert=[4, 224, 111, 45, 2051, 8, 1878, 5, 2, 0, 0, 181, 3652, 3849, 13, 4, 282, 3, 1718, 2, 16, 0, 142, 4, 282, 100, 991, 3, 0, 23, 203, 0, 0, 0, 1, 2361, 63, 3665, 562, 8, 0, 3931, 31, 0, 174, 0, 1159, 3, 2543, 2, 50, 0, 19, 0, 341, 3, 104, 159, 1, 0, 2, 0, 58, 1732] payload_rows_per_dst=[4816, 7722, 4040, 7658, 7080, 8006, 6794, 5222] dup_rows_per_dst=[508, 376, 294, 2850, 3926, 2590, 2450, 1204] combine_rows_per_src=[4816, 7722, 4040, 7658, 7080, 8006, 6794, 5222] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52792 payload_rows_remote=46220 dup_rows=12744 dup_rows_remote=11368 dedup_ratio=0.194458 router_tokens_per_expert=[0, 18, 87, 175, 2906, 264, 3013, 61, 35, 0, 40, 125, 3277, 3357, 140, 20, 176, 36, 3430, 94, 158, 0, 39, 6, 6, 340, 649, 149, 0, 57, 367, 0, 20, 1, 17, 1141, 4, 3977, 595, 22, 1, 2817, 17, 58, 165, 0, 914, 2, 1867, 5, 277, 8, 169, 0, 193, 6, 328, 139, 17, 3, 4, 1, 87, 888] payload_rows_per_dst=[6240, 6904, 7044, 7178, 3334, 6572, 7424, 8096] dup_rows_per_dst=[782, 530, 834, 2512, 266, 1376, 3652, 2792] combine_rows_per_src=[6240, 6904, 7044, 7178, 3334, 6572, 7424, 8096] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=270047232 dispatch_meta_bytes=197112 combine_payload_bytes=270047232 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=270244344 ingress_bytes=600348672 link_bytes=600348672 token_bytes=540291576 weight_movement_bytes=330301440 total_nvlink_bytes=870593016 weight_to_token_ratio=0.611339 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=11136 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.044105081e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46914 payload_rows_remote=39404 dup_rows=18622 dup_rows_remote=16708 dedup_ratio=0.284149 router_tokens_per_expert=[0, 11, 531, 22, 2219, 5, 3324, 8, 1, 1, 0, 537, 2284, 3537, 59, 1, 146, 12, 2581, 49, 62, 0, 21, 2, 35, 81, 641, 12, 5, 21, 359, 5, 15, 0, 101, 1341, 2, 4076, 1135, 3, 0, 3641, 1, 24, 1698, 0, 795, 0, 1238, 4, 77, 11, 49, 0, 3, 6, 48, 5, 0, 4, 0, 1, 191, 1727] payload_rows_per_dst=[5018, 2580, 5464, 5504, 4556, 8102, 7510, 8180] dup_rows_per_dst=[574, 446, 282, 1382, 638, 9462, 1914, 3924] combine_rows_per_src=[5018, 2580, 5464, 5504, 4556, 8102, 7510, 8180] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50278 payload_rows_remote=42506 dup_rows=15258 dup_rows_remote=10602 dedup_ratio=0.232819 router_tokens_per_expert=[0, 37, 91, 87, 1824, 144, 3557, 2, 5, 0, 0, 23, 3312, 2629, 297, 22, 98, 3, 2489, 52, 40, 0, 39, 9, 6, 177, 2529, 2, 0, 10, 178, 0, 1, 0, 4, 1006, 0, 3644, 563, 62, 0, 3682, 1, 3, 369, 0, 455, 0, 2144, 2, 82, 3, 188, 0, 47, 8, 102, 4, 0, 0, 10, 0, 36, 2690] payload_rows_per_dst=[7830, 5676, 6824, 7772, 7832, 4140, 4690, 5514] dup_rows_per_dst=[3654, 276, 2884, 4656, 2728, 632, 258, 170] combine_rows_per_src=[7830, 5676, 6824, 7772, 7832, 4140, 4690, 5514] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=269645824 dispatch_meta_bytes=218584 combine_payload_bytes=269645824 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=269864408 ingress_bytes=379746304 link_bytes=379746304 token_bytes=539510232 weight_movement_bytes=110100480 total_nvlink_bytes=649610712 weight_to_token_ratio=0.204075 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=14848 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.604412802e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=287866880 dispatch_meta_bytes=217696 combine_payload_bytes=287866880 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=288084576 ingress_bytes=397967360 link_bytes=397967360 token_bytes=575951456 weight_movement_bytes=110100480 total_nvlink_bytes=686051936 weight_to_token_ratio=0.191163 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=19200 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.921306934e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=331390976 dispatch_meta_bytes=230824 combine_payload_bytes=331390976 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=331621800 ingress_bytes=551591936 link_bytes=551591936 token_bytes=663012776 weight_movement_bytes=220200960 total_nvlink_bytes=883213736 weight_to_token_ratio=0.332122 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=15360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.593090980e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=331304960 dispatch_meta_bytes=230352 combine_payload_bytes=331304960 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=331535312 ingress_bytes=551505920 link_bytes=551505920 token_bytes=662840272 weight_movement_bytes=220200960 total_nvlink_bytes=883041232 weight_to_token_ratio=0.332208 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=15744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.591595020e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=317241344 dispatch_meta_bytes=218120 combine_payload_bytes=317241344 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=317459464 ingress_bytes=647542784 link_bytes=647542784 token_bytes=634700808 weight_movement_bytes=330301440 total_nvlink_bytes=965002248 weight_to_token_ratio=0.520405 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=14336 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.126183404e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47428 payload_rows_remote=39948 dup_rows=18108 dup_rows_remote=16176 dedup_ratio=0.276306 router_tokens_per_expert=[0, 9, 480, 22, 2176, 6, 3309, 9, 1, 1, 0, 540, 2320, 3531, 59, 1, 144, 8, 2557, 47, 61, 0, 18, 2, 32, 80, 615, 12, 7, 19, 340, 4, 20, 0, 89, 1370, 1, 4075, 1084, 3, 0, 3691, 1, 27, 1735, 0, 801, 0, 1248, 5, 77, 9, 50, 0, 5, 3, 56, 2, 0, 3, 0, 1, 188, 1814] payload_rows_per_dst=[4886, 3324, 5414, 5510, 4536, 8096, 7480, 8182] dup_rows_per_dst=[518, 680, 260, 1348, 598, 8676, 1932, 4096] combine_rows_per_src=[4886, 3324, 5414, 5510, 4536, 8096, 7480, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46332 payload_rows_remote=38340 dup_rows=19204 dup_rows_remote=16200 dedup_ratio=0.293030 router_tokens_per_expert=[5, 28, 76, 217, 2066, 10, 2737, 7, 8, 0, 4, 140, 3419, 3410, 121, 4, 649, 9, 2768, 27, 55, 0, 81, 5, 99, 338, 531, 23, 1, 53, 14, 0, 0, 1, 136, 2010, 2, 3962, 525, 76, 2, 3494, 5, 8, 211, 8, 1533, 1, 1614, 13, 143, 7, 356, 0, 159, 61, 232, 117, 2, 0, 2, 1, 115, 1067] payload_rows_per_dst=[3856, 546, 6018, 7218, 5316, 8122, 7264, 7992] dup_rows_per_dst=[216, 8, 1170, 1738, 930, 9222, 2916, 3004] combine_rows_per_src=[3856, 546, 6018, 7218, 5316, 8122, 7264, 7992] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46264 payload_rows_remote=39022 dup_rows=19272 dup_rows_remote=16594 dedup_ratio=0.294067 router_tokens_per_expert=[0, 2, 153, 340, 2976, 234, 1232, 23, 19, 0, 0, 69, 3219, 2404, 21, 12, 223, 0, 2550, 157, 58, 2, 6, 0, 23, 84, 2080, 10, 0, 14, 8, 0, 7, 0, 65, 1611, 8, 3810, 295, 325, 0, 2657, 193, 7, 273, 0, 973, 3, 2268, 1, 116, 3, 193, 0, 208, 20, 1500, 30, 18, 40, 0, 0, 61, 2164] payload_rows_per_dst=[7242, 8146, 7372, 4342, 5200, 2638, 5192, 6132] dup_rows_per_dst=[2678, 9010, 3934, 96, 1374, 260, 426, 1494] combine_rows_per_src=[7242, 8146, 7372, 4342, 5200, 2638, 5192, 6132] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=323147776 dispatch_meta_bytes=223112 combine_payload_bytes=323147776 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=323370888 ingress_bytes=653449216 link_bytes=653449216 token_bytes=646518664 weight_movement_bytes=330301440 total_nvlink_bytes=976820104 weight_to_token_ratio=0.510892 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=13568 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.136455661e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49396 payload_rows_remote=41988 dup_rows=16140 dup_rows_remote=11570 dedup_ratio=0.246277 router_tokens_per_expert=[18, 192, 25, 86, 2240, 94, 2527, 10, 87, 2, 34, 33, 2961, 2851, 0, 21, 752, 108, 1371, 25, 18, 1, 1, 10, 0, 41, 2106, 42, 0, 645, 229, 33, 77, 0, 0, 451, 47, 3386, 1216, 150, 2, 1899, 0, 10, 56, 4, 1608, 0, 2991, 20, 518, 112, 5, 1, 584, 1, 119, 130, 9, 90, 20, 13, 14, 2672] payload_rows_per_dst=[7174, 7408, 6236, 5122, 7504, 3336, 6794, 5822] dup_rows_per_dst=[3210, 4570, 2134, 1070, 3150, 24, 1670, 312] combine_rows_per_src=[7174, 7408, 6236, 5122, 7504, 3336, 6794, 5822] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50578 payload_rows_remote=42804 dup_rows=14958 dup_rows_remote=10302 dedup_ratio=0.228241 router_tokens_per_expert=[0, 33, 85, 90, 1758, 138, 3539, 4, 3, 0, 1, 27, 3336, 2632, 305, 20, 95, 2, 2501, 55, 37, 0, 35, 9, 6, 154, 2523, 1, 0, 9, 186, 0, 1, 0, 4, 1036, 0, 3603, 543, 58, 0, 3746, 1, 3, 377, 0, 442, 0, 2140, 3, 82, 2, 173, 0, 51, 9, 115, 3, 0, 0, 10, 0, 36, 2746] payload_rows_per_dst=[7776, 5696, 6710, 7774, 7816, 4524, 4670, 5612] dup_rows_per_dst=[3518, 280, 2702, 4656, 2674, 670, 250, 208] combine_rows_per_src=[7776, 5696, 6710, 7774, 7816, 4524, 4670, 5612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47406 payload_rows_remote=40362 dup_rows=18130 dup_rows_remote=13952 dedup_ratio=0.276642 router_tokens_per_expert=[10, 75, 224, 52, 3132, 4, 2681, 17, 16, 0, 64, 151, 3140, 2970, 132, 169, 865, 3, 2188, 25, 28, 0, 16, 11, 89, 331, 1203, 1, 0, 108, 17, 1, 0, 0, 118, 1697, 0, 3777, 33, 323, 20, 2475, 52, 1, 192, 0, 434, 2, 2802, 9, 87, 0, 50, 0, 234, 125, 470, 31, 0, 0, 41, 0, 7, 2065] payload_rows_per_dst=[7882, 7320, 7044, 5084, 8068, 1332, 5982, 4694] dup_rows_per_dst=[4508, 2984, 4178, 1396, 3828, 70, 632, 534] combine_rows_per_src=[7882, 7320, 7044, 5084, 8068, 1332, 5982, 4694] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=286347264 dispatch_meta_bytes=224496 combine_payload_bytes=286347264 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=286571760 ingress_bytes=396447744 link_bytes=396447744 token_bytes=572919024 weight_movement_bytes=110100480 total_nvlink_bytes=683019504 weight_to_token_ratio=0.192175 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=17536 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.894878313e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45948 payload_rows_remote=38680 dup_rows=19588 dup_rows_remote=16822 dedup_ratio=0.298889 router_tokens_per_expert=[0, 3, 175, 353, 3018, 241, 1204, 23, 13, 0, 0, 70, 3205, 2449, 26, 14, 210, 0, 2563, 169, 57, 2, 8, 1, 24, 84, 2141, 10, 0, 13, 9, 0, 7, 0, 67, 1576, 7, 3836, 300, 360, 0, 2585, 191, 7, 257, 0, 972, 3, 2239, 1, 110, 3, 204, 0, 209, 20, 1458, 31, 18, 41, 0, 0, 58, 2123] payload_rows_per_dst=[7268, 8154, 7334, 4460, 4898, 2610, 5160, 6064] dup_rows_per_dst=[2766, 9660, 3856, 102, 1148, 250, 412, 1394] combine_rows_per_src=[7268, 8154, 7334, 4460, 4898, 2610, 5160, 6064] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=282447872 dispatch_meta_bytes=224448 combine_payload_bytes=282447872 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=282672320 ingress_bytes=392548352 link_bytes=392548352 token_bytes=565120192 weight_movement_bytes=110100480 total_nvlink_bytes=675220672 weight_to_token_ratio=0.194827 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=17536 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.827061472e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=274821120 dispatch_meta_bytes=218160 combine_payload_bytes=274821120 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=275039280 ingress_bytes=384921600 link_bytes=384921600 token_bytes=549860400 weight_movement_bytes=110100480 total_nvlink_bytes=659960880 weight_to_token_ratio=0.200234 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=15232 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.694419711e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49510 payload_rows_remote=42124 dup_rows=16026 dup_rows_remote=11402 dedup_ratio=0.244537 router_tokens_per_expert=[21, 190, 22, 84, 2137, 88, 2501, 12, 87, 3, 34, 31, 2987, 2843, 1, 19, 732, 103, 1409, 22, 15, 1, 1, 11, 0, 35, 2087, 42, 0, 661, 230, 33, 81, 0, 0, 477, 48, 3376, 1172, 141, 2, 1967, 0, 11, 57, 5, 1616, 0, 2984, 19, 533, 113, 3, 1, 582, 1, 120, 126, 10, 84, 20, 14, 13, 2750] payload_rows_per_dst=[7110, 7386, 6310, 5112, 7518, 3356, 6776, 5942] dup_rows_per_dst=[3000, 4624, 2212, 1064, 3072, 26, 1696, 332] combine_rows_per_src=[7110, 7386, 6310, 5112, 7518, 3356, 6776, 5942] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279709696 dispatch_meta_bytes=222464 combine_payload_bytes=279709696 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279932160 ingress_bytes=499910656 link_bytes=499910656 token_bytes=559641856 weight_movement_bytes=220200960 total_nvlink_bytes=779842816 weight_to_token_ratio=0.393468 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=18944 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.694268520e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50072 payload_rows_remote=43260 dup_rows=15464 dup_rows_remote=13038 dedup_ratio=0.235962 router_tokens_per_expert=[1, 1400, 0, 0, 23, 9, 1117, 673, 1, 1, 0, 102, 1, 280, 0, 30, 618, 182, 655, 1928, 1162, 1299, 260, 2416, 129, 0, 7, 1, 496, 0, 41, 0, 260, 2, 230, 2806, 0, 0, 294, 1959, 1, 2108, 247, 20, 2099, 9, 0, 135, 31, 3328, 2, 272, 98, 1, 103, 60, 183, 3, 0, 2182, 20, 937, 73, 2473] payload_rows_per_dst=[5798, 6404, 5606, 5492, 5362, 6812, 7130, 7468] dup_rows_per_dst=[1958, 2300, 2250, 802, 1306, 2426, 660, 3762] combine_rows_per_src=[5798, 6404, 5606, 5492, 5362, 6812, 7130, 7468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47726 payload_rows_remote=40806 dup_rows=17810 dup_rows_remote=16282 dedup_ratio=0.271759 router_tokens_per_expert=[0, 1494, 0, 0, 2, 6, 1734, 417, 0, 2, 0, 516, 1, 1381, 0, 314, 356, 266, 2276, 451, 504, 1439, 455, 1991, 840, 0, 60, 36, 1001, 0, 440, 0, 25, 0, 417, 3139, 0, 0, 169, 474, 0, 444, 52, 68, 2782, 6, 0, 417, 2, 2887, 0, 764, 26, 3, 25, 468, 169, 2, 3, 696, 5, 699, 3, 3041] payload_rows_per_dst=[7092, 6142, 4838, 6970, 6920, 6224, 6738, 2802] dup_rows_per_dst=[4766, 3170, 1202, 3866, 1528, 1314, 1612, 352] combine_rows_per_src=[7092, 6142, 4838, 6970, 6920, 6224, 6738, 2802] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=306819072 dispatch_meta_bytes=212424 combine_payload_bytes=306819072 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=307031496 ingress_bytes=416919552 link_bytes=416919552 token_bytes=613850568 weight_movement_bytes=110100480 total_nvlink_bytes=723951048 weight_to_token_ratio=0.179360 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=17536 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.250916724e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=304683008 dispatch_meta_bytes=212432 combine_payload_bytes=304683008 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=304895440 ingress_bytes=414783488 link_bytes=414783488 token_bytes=609578448 weight_movement_bytes=110100480 total_nvlink_bytes=719678928 weight_to_token_ratio=0.180617 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=17280 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.213767058e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=300969984 dispatch_meta_bytes=214232 combine_payload_bytes=300969984 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=301184216 ingress_bytes=631271424 link_bytes=631271424 token_bytes=602154200 weight_movement_bytes=330301440 total_nvlink_bytes=932455640 weight_to_token_ratio=0.548533 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=19072 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.097884833e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277258240 dispatch_meta_bytes=222008 combine_payload_bytes=277258240 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=277480248 ingress_bytes=497459200 link_bytes=497459200 token_bytes=554738488 weight_movement_bytes=220200960 total_nvlink_bytes=774939448 weight_to_token_ratio=0.396946 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=18432 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.651633668e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45644 payload_rows_remote=38258 dup_rows=19892 dup_rows_remote=11836 dedup_ratio=0.303528 router_tokens_per_expert=[0, 1515, 270, 17, 355, 33, 158, 714, 2, 0, 5, 28, 0, 944, 1, 83, 732, 330, 2013, 470, 586, 318, 225, 2227, 60, 0, 10, 0, 729, 15, 329, 10, 44, 0, 784, 1822, 11, 1, 1075, 1341, 3, 415, 452, 62, 3661, 7, 1, 42, 10, 2764, 0, 677, 103, 10, 161, 192, 988, 0, 0, 1474, 6, 1557, 0, 2926] payload_rows_per_dst=[4994, 5496, 6370, 3602, 5174, 5786, 6836, 7386] dup_rows_per_dst=[1130, 1084, 2978, 808, 1338, 1228, 3270, 8056] combine_rows_per_src=[4994, 5496, 6370, 3602, 5174, 5786, 6836, 7386] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49780 payload_rows_remote=42908 dup_rows=15756 dup_rows_remote=13370 dedup_ratio=0.240417 router_tokens_per_expert=[1, 1458, 0, 0, 20, 9, 1128, 648, 1, 1, 0, 114, 1, 267, 0, 28, 626, 175, 685, 1943, 1162, 1271, 263, 2442, 112, 0, 7, 1, 495, 0, 42, 0, 266, 2, 208, 2788, 0, 0, 297, 1946, 1, 2146, 234, 22, 2081, 9, 0, 136, 29, 3347, 2, 228, 92, 1, 98, 60, 180, 3, 0, 2231, 22, 920, 60, 2459] payload_rows_per_dst=[5594, 6392, 5804, 5476, 4912, 6872, 7086, 7644] dup_rows_per_dst=[1592, 2206, 2898, 756, 778, 2386, 628, 4512] combine_rows_per_src=[5594, 6392, 5804, 5476, 4912, 6872, 7086, 7644] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49478 payload_rows_remote=42750 dup_rows=16058 dup_rows_remote=12748 dedup_ratio=0.245026 router_tokens_per_expert=[0, 1219, 0, 14, 15, 23, 599, 225, 3, 7, 4, 110, 4, 1105, 30, 27, 2219, 223, 1493, 1251, 1229, 1154, 194, 2173, 315, 6, 8, 3, 134, 0, 24, 13, 132, 11, 184, 1835, 0, 0, 1494, 1535, 5, 664, 542, 92, 2127, 27, 0, 2, 183, 2754, 6, 570, 380, 68, 2, 7, 428, 0, 0, 903, 24, 1396, 40, 3533] payload_rows_per_dst=[5446, 6460, 6728, 7158, 6978, 5390, 6544, 4774] dup_rows_per_dst=[1730, 2968, 3310, 914, 3404, 1528, 1396, 808] combine_rows_per_src=[5446, 6460, 6728, 7158, 6978, 5390, 6544, 4774] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47622 payload_rows_remote=40700 dup_rows=17914 dup_rows_remote=16372 dedup_ratio=0.273346 router_tokens_per_expert=[0, 1531, 0, 0, 2, 7, 1727, 416, 0, 1, 0, 528, 1, 1384, 0, 308, 375, 252, 2359, 431, 524, 1389, 463, 1986, 822, 0, 57, 34, 1029, 1, 419, 0, 28, 0, 408, 3148, 0, 0, 177, 471, 0, 445, 50, 69, 2762, 4, 0, 426, 1, 2887, 0, 760, 31, 3, 25, 454, 157, 2, 3, 709, 4, 679, 3, 3016] payload_rows_per_dst=[7168, 6122, 4770, 6968, 6922, 6182, 6696, 2794] dup_rows_per_dst=[4916, 3156, 1236, 3788, 1542, 1330, 1626, 320] combine_rows_per_src=[7168, 6122, 4770, 6968, 6922, 6182, 6696, 2794] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 2, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47898 payload_rows_remote=42958 dup_rows=17638 dup_rows_remote=17270 dedup_ratio=0.269135 router_tokens_per_expert=[3, 1379, 10, 0, 8, 24, 89, 1304, 0, 0, 0, 113, 0, 1126, 0, 169, 258, 306, 926, 1942, 569, 829, 834, 1869, 164, 4, 11, 1, 891, 1, 88, 0, 664, 0, 352, 2968, 2, 0, 436, 1494, 0, 872, 522, 2, 3291, 0, 0, 9, 9, 2073, 7, 131, 340, 0, 68, 26, 300, 1, 0, 1737, 0, 1771, 57, 2718] payload_rows_per_dst=[4748, 5100, 6914, 6384, 4874, 7248, 4940, 7690] dup_rows_per_dst=[886, 1454, 4414, 1372, 1022, 2144, 368, 5978] combine_rows_per_src=[4748, 5100, 6914, 6384, 4874, 7248, 4940, 7690] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=274233344 dispatch_meta_bytes=200376 combine_payload_bytes=274233344 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=274433720 ingress_bytes=494434304 link_bytes=494434304 token_bytes=548667064 weight_movement_bytes=220200960 total_nvlink_bytes=768868024 weight_to_token_ratio=0.401338 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=11136 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.599025751e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50612 payload_rows_remote=43662 dup_rows=14924 dup_rows_remote=13896 dedup_ratio=0.227722 router_tokens_per_expert=[0, 1679, 1, 0, 6, 0, 297, 1448, 0, 1, 0, 575, 1, 868, 0, 68, 338, 5, 1474, 670, 325, 1211, 538, 2819, 227, 0, 38, 1, 403, 1, 35, 0, 10, 0, 566, 2086, 0, 0, 384, 969, 0, 1154, 493, 15, 3526, 15, 0, 135, 3, 3639, 1, 163, 300, 3, 51, 555, 49, 0, 0, 1757, 20, 508, 53, 3284] payload_rows_per_dst=[6688, 7002, 4096, 6950, 6198, 7628, 7852, 4198] dup_rows_per_dst=[3122, 3002, 738, 1028, 1832, 3048, 1578, 576] combine_rows_per_src=[6688, 7002, 4096, 6950, 6198, 7628, 7852, 4198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=289314816 dispatch_meta_bytes=217256 combine_payload_bytes=289314816 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=289532072 ingress_bytes=399415296 link_bytes=399415296 token_bytes=578846888 weight_movement_bytes=110100480 total_nvlink_bytes=688947368 weight_to_token_ratio=0.190207 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=19200 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.946488923e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=307564544 dispatch_meta_bytes=225112 combine_payload_bytes=307564544 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=307789656 ingress_bytes=637865984 link_bytes=637865984 token_bytes=615354200 weight_movement_bytes=330301440 total_nvlink_bytes=945655640 weight_to_token_ratio=0.536766 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=19328 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.109353857e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=301944832 dispatch_meta_bytes=214104 combine_payload_bytes=301944832 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=302158936 ingress_bytes=632246272 link_bytes=632246272 token_bytes=604103768 weight_movement_bytes=330301440 total_nvlink_bytes=934405208 weight_to_token_ratio=0.546763 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=19456 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.099580254e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=310087680 dispatch_meta_bytes=225192 combine_payload_bytes=310087680 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=310312872 ingress_bytes=640389120 link_bytes=640389120 token_bytes=620400552 weight_movement_bytes=330301440 total_nvlink_bytes=950701992 weight_to_token_ratio=0.532400 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=19328 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.113742006e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=292497408 dispatch_meta_bytes=228352 combine_payload_bytes=292497408 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=292725760 ingress_bytes=512698368 link_bytes=512698368 token_bytes=585223168 weight_movement_bytes=220200960 total_nvlink_bytes=805424128 weight_to_token_ratio=0.376268 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=12416 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.916667863e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48000 payload_rows_remote=43024 dup_rows=17536 dup_rows_remote=17184 dedup_ratio=0.267578 router_tokens_per_expert=[3, 1364, 8, 0, 9, 22, 94, 1314, 0, 0, 1, 111, 1, 1130, 0, 163, 262, 304, 900, 1929, 570, 902, 784, 1843, 160, 2, 11, 1, 881, 1, 91, 0, 641, 0, 375, 3014, 2, 0, 428, 1478, 0, 864, 536, 3, 3269, 0, 0, 9, 9, 2083, 5, 136, 330, 1, 63, 37, 288, 1, 0, 1740, 0, 1806, 61, 2728] payload_rows_per_dst=[4946, 5052, 6808, 6400, 4860, 7214, 4976, 7744] dup_rows_per_dst=[1006, 1446, 4170, 1350, 988, 2148, 352, 6076] combine_rows_per_src=[4946, 5052, 6808, 6400, 4860, 7214, 4976, 7744] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45488 payload_rows_remote=38024 dup_rows=20048 dup_rows_remote=11596 dedup_ratio=0.305908 router_tokens_per_expert=[0, 1469, 259, 15, 359, 31, 167, 726, 2, 0, 4, 23, 0, 914, 1, 85, 705, 337, 1968, 492, 585, 325, 209, 2198, 64, 0, 10, 0, 743, 12, 355, 9, 41, 0, 789, 1848, 11, 1, 1020, 1352, 3, 406, 467, 67, 3666, 5, 1, 44, 10, 2768, 0, 710, 101, 16, 159, 229, 995, 0, 0, 1481, 5, 1535, 0, 2971] payload_rows_per_dst=[4938, 5402, 6298, 3418, 5148, 5962, 6858, 7464] dup_rows_per_dst=[1114, 1052, 2944, 722, 1280, 1306, 3178, 8452] combine_rows_per_src=[4938, 5402, 6298, 3418, 5148, 5962, 6858, 7464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50286 payload_rows_remote=44100 dup_rows=15250 dup_rows_remote=13238 dedup_ratio=0.232697 router_tokens_per_expert=[0, 2306, 3, 2, 19, 13, 17, 476, 0, 3, 0, 90, 6, 180, 0, 12, 1019, 55, 1263, 1418, 686, 1453, 44, 2018, 103, 0, 13, 0, 251, 0, 26, 0, 6, 0, 265, 2343, 0, 0, 695, 892, 2, 1881, 425, 79, 3529, 19, 14, 181, 2, 3789, 20, 546, 469, 1, 48, 47, 7, 6, 0, 1339, 6, 701, 21, 3959] payload_rows_per_dst=[6186, 5934, 6038, 7944, 6690, 5774, 7982, 3738] dup_rows_per_dst=[2012, 1520, 2496, 760, 4396, 1782, 1862, 422] combine_rows_per_src=[6186, 5934, 6038, 7944, 6690, 5774, 7982, 3738] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=306432000 dispatch_meta_bytes=221992 combine_payload_bytes=306432000 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=306653992 ingress_bytes=746833920 link_bytes=746833920 token_bytes=613085992 weight_movement_bytes=440401920 total_nvlink_bytes=1053487912 weight_to_token_ratio=0.718336 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=9088 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.298867020e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48900 payload_rows_remote=43312 dup_rows=16636 dup_rows_remote=14630 dedup_ratio=0.253845 router_tokens_per_expert=[3, 1621, 0, 13, 8, 13, 803, 607, 7, 0, 5, 118, 67, 241, 32, 251, 480, 136, 1525, 1526, 666, 1469, 1220, 1550, 491, 19, 59, 1, 2154, 5, 77, 15, 321, 23, 193, 2258, 70, 2, 266, 1203, 84, 515, 987, 0, 1556, 222, 4, 57, 17, 2586, 60, 413, 148, 0, 22, 191, 2032, 26, 5, 885, 102, 105, 211, 3022] payload_rows_per_dst=[6512, 5588, 5800, 7456, 6518, 5398, 6166, 5462] dup_rows_per_dst=[2674, 2006, 2142, 4230, 2154, 1452, 708, 1270] combine_rows_per_src=[6512, 5588, 5800, 7456, 6518, 5398, 6166, 5462] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=291737600 dispatch_meta_bytes=228288 combine_payload_bytes=291737600 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=291965888 ingress_bytes=511938560 link_bytes=511938560 token_bytes=583703488 weight_movement_bytes=220200960 total_nvlink_bytes=803904448 weight_to_token_ratio=0.377248 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=11776 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.903453553e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=307922944 dispatch_meta_bytes=240912 combine_payload_bytes=307922944 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=308163856 ingress_bytes=418023424 link_bytes=418023424 token_bytes=616086800 weight_movement_bytes=110100480 total_nvlink_bytes=726187280 weight_to_token_ratio=0.178709 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=23936 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.270114874e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50380 payload_rows_remote=44130 dup_rows=15156 dup_rows_remote=13070 dedup_ratio=0.231262 router_tokens_per_expert=[0, 2346, 2, 1, 17, 17, 16, 460, 0, 2, 0, 87, 7, 181, 0, 11, 1021, 62, 1309, 1418, 698, 1415, 43, 2044, 93, 0, 12, 0, 246, 0, 24, 0, 5, 0, 276, 2321, 0, 0, 722, 904, 2, 1867, 408, 87, 3527, 19, 15, 180, 2, 3799, 18, 538, 482, 1, 45, 36, 8, 6, 0, 1316, 11, 676, 17, 3948] payload_rows_per_dst=[6250, 5976, 5592, 7922, 7004, 5972, 7992, 3672] dup_rows_per_dst=[2086, 1524, 1746, 724, 4886, 1944, 1850, 396] combine_rows_per_src=[6250, 5976, 5592, 7922, 7004, 5972, 7992, 3672] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=312969216 dispatch_meta_bytes=230232 combine_payload_bytes=312969216 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=313199448 ingress_bytes=423069696 link_bytes=423069696 token_bytes=626168664 weight_movement_bytes=110100480 total_nvlink_bytes=736269144 weight_to_token_ratio=0.175832 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=23040 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.357877844e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50564 payload_rows_remote=43614 dup_rows=14972 dup_rows_remote=13966 dedup_ratio=0.228455 router_tokens_per_expert=[0, 1630, 3, 0, 6, 0, 297, 1412, 0, 1, 0, 558, 1, 893, 0, 64, 334, 3, 1445, 716, 318, 1237, 491, 2813, 226, 0, 37, 0, 381, 1, 39, 0, 6, 0, 581, 2118, 0, 0, 351, 985, 0, 1155, 508, 13, 3519, 17, 0, 150, 2, 3614, 1, 183, 290, 4, 50, 564, 46, 0, 0, 1800, 18, 535, 58, 3294] payload_rows_per_dst=[6594, 7022, 4046, 6950, 6202, 7636, 7826, 4288] dup_rows_per_dst=[2992, 3070, 720, 1006, 1880, 3088, 1590, 626] combine_rows_per_src=[6594, 7022, 4046, 6950, 6202, 7636, 7826, 4288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=310460416 dispatch_meta_bytes=231768 combine_payload_bytes=310460416 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=310692184 ingress_bytes=530661376 link_bytes=530661376 token_bytes=621152600 weight_movement_bytes=220200960 total_nvlink_bytes=841353560 weight_to_token_ratio=0.354504 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.036765 gemm_active_groups=11 gemm_peak_group_rows=17152 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.229074117e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=316323840 dispatch_meta_bytes=228800 combine_payload_bytes=316323840 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=316552640 ingress_bytes=426424320 link_bytes=426424320 token_bytes=632876480 weight_movement_bytes=110100480 total_nvlink_bytes=742976960 weight_to_token_ratio=0.173968 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=12800 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.416220272e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49794 payload_rows_remote=42206 dup_rows=15742 dup_rows_remote=12092 dedup_ratio=0.240204 router_tokens_per_expert=[4, 454, 20, 63, 179, 973, 7, 3443, 0, 1692, 1, 169, 202, 950, 3467, 4, 1, 7, 4, 874, 244, 3, 42, 3777, 246, 5, 1, 45, 0, 0, 10, 2571, 1465, 0, 96, 2158, 4, 700, 5, 296, 0, 22, 0, 701, 1245, 0, 0, 3, 3, 0, 2, 76, 1161, 3220, 1, 0, 2, 6, 1142, 25, 771, 39, 167, 0] payload_rows_per_dst=[3072, 4824, 7682, 5476, 6646, 7446, 7060, 7588] dup_rows_per_dst=[328, 848, 2222, 280, 2802, 3382, 2230, 3650] combine_rows_per_src=[3072, 4824, 7682, 5476, 6646, 7446, 7060, 7588] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50016 payload_rows_remote=42230 dup_rows=15520 dup_rows_remote=12242 dedup_ratio=0.236816 router_tokens_per_expert=[0, 1432, 100, 22, 140, 995, 2, 3684, 2, 1340, 1, 201, 1309, 919, 3128, 19, 6, 17, 99, 1302, 227, 0, 22, 2224, 190, 4, 1, 49, 0, 3, 64, 1396, 963, 0, 91, 2247, 28, 1109, 0, 1296, 7, 173, 0, 477, 1188, 0, 0, 3, 4, 43, 22, 71, 813, 3590, 5, 0, 4, 1, 834, 276, 34, 201, 379, 11] payload_rows_per_dst=[4348, 5956, 6160, 5008, 5936, 7786, 7618, 7204] dup_rows_per_dst=[1034, 1626, 1634, 894, 3044, 3278, 1478, 2532] combine_rows_per_src=[4348, 5956, 6160, 5008, 5936, 7786, 7618, 7204] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48622 payload_rows_remote=43182 dup_rows=16914 dup_rows_remote=15530 dedup_ratio=0.258087 router_tokens_per_expert=[19, 833, 251, 115, 1055, 322, 0, 3782, 0, 2369, 130, 130, 235, 466, 2664, 151, 0, 11, 118, 1025, 633, 2, 8, 1615, 211, 0, 1, 137, 0, 5, 288, 1850, 1576, 7, 550, 1330, 0, 368, 19, 1328, 0, 351, 20, 1079, 1567, 2, 0, 19, 12, 7, 200, 590, 737, 2814, 0, 0, 0, 100, 310, 195, 277, 76, 808, 0] payload_rows_per_dst=[7946, 5786, 5440, 5734, 5406, 4934, 6754, 6622] dup_rows_per_dst=[4808, 1176, 1384, 1910, 2290, 1142, 1966, 2238] combine_rows_per_src=[7946, 5786, 5440, 5734, 5406, 4934, 6754, 6622] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49600 payload_rows_remote=43498 dup_rows=15936 dup_rows_remote=15736 dedup_ratio=0.243164 router_tokens_per_expert=[38, 259, 31, 373, 337, 1127, 0, 3938, 0, 2408, 2, 1088, 151, 1080, 3095, 1, 0, 8, 0, 264, 107, 0, 114, 2971, 791, 0, 0, 36, 0, 2, 359, 1394, 1780, 0, 6, 2650, 0, 807, 1, 1031, 0, 10, 0, 145, 1642, 0, 0, 1, 18, 25, 12, 23, 182, 2884, 7, 0, 0, 9, 1047, 368, 25, 63, 58, 0] payload_rows_per_dst=[3700, 6678, 6268, 4468, 7330, 8036, 6102, 7018] dup_rows_per_dst=[630, 2782, 660, 696, 5220, 3436, 200, 2312] combine_rows_per_src=[3700, 6678, 6268, 4468, 7330, 8036, 6102, 7018] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=308396032 dispatch_meta_bytes=240832 combine_payload_bytes=308396032 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=308636864 ingress_bytes=418496512 link_bytes=418496512 token_bytes=617032896 weight_movement_bytes=110100480 total_nvlink_bytes=727133376 weight_to_token_ratio=0.178435 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=23808 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.278342652e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=272556032 dispatch_meta_bytes=198480 combine_payload_bytes=272556032 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=272754512 ingress_bytes=492756992 link_bytes=492756992 token_bytes=545310544 weight_movement_bytes=220200960 total_nvlink_bytes=765511504 weight_to_token_ratio=0.403808 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=11136 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.569854537e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49462 payload_rows_remote=42704 dup_rows=16074 dup_rows_remote=12604 dedup_ratio=0.245270 router_tokens_per_expert=[0, 1189, 0, 13, 13, 22, 637, 244, 4, 6, 3, 105, 3, 1137, 29, 31, 2212, 224, 1422, 1253, 1252, 1232, 194, 2171, 314, 7, 7, 2, 140, 0, 21, 14, 128, 11, 182, 1818, 0, 0, 1429, 1560, 4, 672, 538, 93, 2143, 22, 0, 2, 181, 2715, 2, 579, 378, 72, 1, 6, 428, 0, 0, 907, 21, 1382, 42, 3551] payload_rows_per_dst=[5428, 6474, 6758, 7200, 6952, 5398, 6472, 4780] dup_rows_per_dst=[1652, 3010, 3470, 912, 3304, 1550, 1396, 780] combine_rows_per_src=[5428, 6474, 6758, 7200, 6952, 5398, 6472, 4780] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49696 payload_rows_remote=41930 dup_rows=15840 dup_rows_remote=12660 dedup_ratio=0.241699 router_tokens_per_expert=[1, 1424, 95, 21, 133, 1014, 2, 3682, 1, 1361, 1, 206, 1302, 1004, 3075, 21, 8, 16, 106, 1324, 219, 0, 13, 2223, 196, 5, 2, 52, 0, 4, 69, 1448, 918, 0, 87, 2199, 28, 1106, 0, 1289, 6, 176, 0, 467, 1137, 0, 2, 3, 7, 41, 22, 79, 782, 3575, 2, 0, 5, 2, 863, 302, 36, 195, 394, 17] payload_rows_per_dst=[4358, 6050, 6160, 4238, 6332, 7766, 7614, 7178] dup_rows_per_dst=[1022, 1742, 1658, 566, 3670, 3180, 1402, 2600] combine_rows_per_src=[4358, 6050, 6160, 4238, 6332, 7766, 7614, 7178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=316108800 dispatch_meta_bytes=229352 combine_payload_bytes=316108800 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=316338152 ingress_bytes=426209280 link_bytes=426209280 token_bytes=632446952 weight_movement_bytes=110100480 total_nvlink_bytes=742547432 weight_to_token_ratio=0.174087 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=12544 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.412480373e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48886 payload_rows_remote=43296 dup_rows=16650 dup_rows_remote=14702 dedup_ratio=0.254059 router_tokens_per_expert=[2, 1583, 0, 10, 15, 12, 822, 600, 6, 0, 6, 124, 70, 254, 32, 236, 448, 139, 1483, 1539, 668, 1501, 1190, 1502, 480, 18, 54, 1, 2171, 6, 88, 15, 315, 23, 201, 2287, 72, 2, 259, 1201, 79, 523, 976, 1, 1591, 218, 5, 58, 18, 2591, 59, 429, 144, 0, 22, 189, 2025, 25, 4, 891, 104, 105, 237, 3039] payload_rows_per_dst=[6448, 5590, 5770, 7452, 6532, 5428, 6198, 5468] dup_rows_per_dst=[2606, 1948, 2122, 4292, 2188, 1474, 706, 1314] combine_rows_per_src=[6448, 5590, 5770, 7452, 6532, 5428, 6198, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=312625152 dispatch_meta_bytes=230320 combine_payload_bytes=312625152 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=312855472 ingress_bytes=422725632 link_bytes=422725632 token_bytes=625480624 weight_movement_bytes=110100480 total_nvlink_bytes=735581104 weight_to_token_ratio=0.176025 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=23040 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.351894005e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49224 payload_rows_remote=46638 dup_rows=16312 dup_rows_remote=16098 dedup_ratio=0.248901 router_tokens_per_expert=[0, 66, 11, 1, 701, 1644, 0, 2894, 0, 3229, 0, 280, 264, 1038, 3842, 66, 0, 76, 38, 93, 95, 0, 197, 2878, 297, 8, 0, 0, 1, 0, 136, 1590, 353, 0, 152, 2279, 0, 759, 0, 136, 0, 9, 0, 2346, 307, 0, 0, 5, 9, 6, 7, 328, 1206, 3309, 0, 0, 0, 0, 752, 255, 933, 34, 59, 79] payload_rows_per_dst=[4584, 7254, 5988, 6442, 2586, 7210, 7168, 7992] dup_rows_per_dst=[728, 2500, 766, 2180, 214, 3446, 2562, 3916] combine_rows_per_src=[4584, 7254, 5988, 6442, 2586, 7210, 7168, 7992] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 2, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49788 payload_rows_remote=46422 dup_rows=15748 dup_rows_remote=15286 dedup_ratio=0.240295 router_tokens_per_expert=[0, 69, 10, 1, 743, 1672, 0, 2888, 0, 3223, 0, 277, 242, 979, 3862, 65, 0, 77, 34, 102, 98, 0, 204, 2887, 296, 7, 0, 0, 1, 0, 121, 1492, 384, 0, 151, 2299, 0, 784, 0, 135, 0, 9, 0, 2357, 382, 1, 0, 6, 10, 6, 6, 310, 1193, 3348, 0, 0, 0, 0, 753, 225, 916, 39, 58, 46] payload_rows_per_dst=[4464, 7260, 6008, 6396, 3366, 7036, 7246, 8012] dup_rows_per_dst=[568, 2376, 796, 2036, 462, 3224, 2500, 3786] combine_rows_per_src=[4464, 7260, 6008, 6396, 3366, 7036, 7246, 8012] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=302532608 dispatch_meta_bytes=217192 combine_payload_bytes=302532608 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=302749800 ingress_bytes=412633088 link_bytes=412633088 token_bytes=605282408 weight_movement_bytes=110100480 total_nvlink_bytes=715382888 weight_to_token_ratio=0.181899 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=19456 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.176368065e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48532 payload_rows_remote=41722 dup_rows=17004 dup_rows_remote=14390 dedup_ratio=0.259460 router_tokens_per_expert=[68, 416, 38, 1, 514, 2034, 0, 3838, 0, 829, 0, 116, 687, 884, 2385, 4, 0, 4, 17, 568, 1, 0, 83, 3647, 27, 1, 4, 0, 0, 2, 551, 2840, 877, 0, 8, 1287, 0, 1422, 0, 1894, 0, 10, 0, 1628, 800, 3, 0, 0, 63, 10, 73, 211, 1913, 1695, 22, 0, 6, 10, 837, 141, 137, 34, 126, 2] payload_rows_per_dst=[8052, 4154, 7410, 6810, 6034, 4290, 6056, 5726] dup_rows_per_dst=[5766, 886, 1230, 2614, 2368, 592, 1918, 1630] combine_rows_per_src=[8052, 4154, 7410, 6810, 6034, 4290, 6056, 5726] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=306102272 dispatch_meta_bytes=221232 combine_payload_bytes=306102272 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=306323504 ingress_bytes=746504192 link_bytes=746504192 token_bytes=612425776 weight_movement_bytes=440401920 total_nvlink_bytes=1052827696 weight_to_token_ratio=0.719111 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=9472 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.298293569e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=300554240 dispatch_meta_bytes=218360 combine_payload_bytes=300554240 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=300772600 ingress_bytes=410654720 link_bytes=410654720 token_bytes=601326840 weight_movement_bytes=110100480 total_nvlink_bytes=711427320 weight_to_token_ratio=0.183096 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=13824 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.141960992e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=310345728 dispatch_meta_bytes=231992 combine_payload_bytes=310345728 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=310577720 ingress_bytes=530546688 link_bytes=530546688 token_bytes=620923448 weight_movement_bytes=220200960 total_nvlink_bytes=841124408 weight_to_token_ratio=0.354635 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.036765 gemm_active_groups=11 gemm_peak_group_rows=17024 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.227079504e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=334301184 dispatch_meta_bytes=250944 combine_payload_bytes=334301184 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=334552128 ingress_bytes=554502144 link_bytes=554502144 token_bytes=668853312 weight_movement_bytes=220200960 total_nvlink_bytes=889054272 weight_to_token_ratio=0.329222 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=10112 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.643704283e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49548 payload_rows_remote=43546 dup_rows=15988 dup_rows_remote=15798 dedup_ratio=0.243958 router_tokens_per_expert=[37, 256, 29, 327, 319, 1107, 0, 3944, 0, 2482, 2, 1119, 135, 1169, 3033, 1, 0, 7, 0, 282, 110, 0, 99, 3008, 807, 0, 0, 43, 0, 2, 365, 1413, 1698, 0, 7, 2636, 0, 795, 0, 1041, 0, 11, 0, 154, 1612, 0, 0, 1, 19, 29, 12, 19, 181, 2832, 4, 0, 0, 9, 1071, 380, 32, 65, 64, 0] payload_rows_per_dst=[3546, 6858, 6316, 4560, 7264, 8058, 6002, 6944] dup_rows_per_dst=[604, 2958, 696, 700, 5090, 3386, 190, 2364] combine_rows_per_src=[3546, 6858, 6316, 4560, 7264, 8058, 6002, 6944] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49138 payload_rows_remote=41604 dup_rows=16398 dup_rows_remote=12732 dedup_ratio=0.250214 router_tokens_per_expert=[6, 493, 18, 69, 179, 981, 6, 3444, 0, 1704, 1, 183, 209, 1022, 3403, 3, 1, 7, 6, 859, 230, 2, 39, 3788, 240, 7, 1, 45, 0, 0, 13, 2641, 1356, 0, 93, 2176, 4, 699, 6, 310, 0, 21, 0, 695, 1186, 1, 0, 2, 4, 0, 2, 76, 1164, 3176, 0, 0, 1, 7, 1177, 27, 788, 29, 168, 0] payload_rows_per_dst=[3126, 3894, 7704, 5602, 6680, 7398, 7200, 7534] dup_rows_per_dst=[378, 668, 2160, 292, 3268, 3300, 2666, 3666] combine_rows_per_src=[3126, 3894, 7704, 5602, 6680, 7398, 7200, 7534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45804 payload_rows_remote=39508 dup_rows=19732 dup_rows_remote=17926 dedup_ratio=0.301086 router_tokens_per_expert=[0, 686, 22, 40, 173, 575, 0, 2479, 0, 2540, 0, 53, 593, 412, 2596, 51, 0, 265, 33, 55, 24, 1, 77, 3604, 34, 0, 0, 149, 41, 88, 595, 3391, 1448, 0, 8, 1670, 180, 405, 7, 2119, 0, 139, 0, 2499, 376, 32, 0, 52, 192, 4, 33, 376, 946, 2493, 20, 0, 6, 0, 165, 535, 63, 88, 334, 1] payload_rows_per_dst=[6296, 7296, 7694, 7660, 5826, 5454, 3352, 2226] dup_rows_per_dst=[1806, 5042, 4782, 4276, 2508, 742, 418, 158] combine_rows_per_src=[6296, 7296, 7694, 7660, 5826, 5454, 3352, 2226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48646 payload_rows_remote=41784 dup_rows=16890 dup_rows_remote=14232 dedup_ratio=0.257721 router_tokens_per_expert=[63, 388, 39, 4, 491, 2004, 0, 3805, 0, 834, 0, 111, 699, 958, 2354, 5, 0, 4, 21, 596, 0, 0, 74, 3643, 32, 1, 3, 0, 0, 0, 560, 2906, 831, 0, 11, 1258, 0, 1460, 0, 1876, 0, 12, 0, 1623, 781, 4, 0, 0, 59, 10, 75, 225, 1906, 1695, 21, 0, 6, 15, 852, 150, 139, 27, 135, 2] payload_rows_per_dst=[8026, 4280, 7418, 6862, 6030, 4262, 6050, 5718] dup_rows_per_dst=[5562, 934, 1258, 2658, 2326, 578, 1932, 1642] combine_rows_per_src=[8026, 4280, 7418, 6862, 6030, 4262, 6050, 5718] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=302704640 dispatch_meta_bytes=217888 combine_payload_bytes=302704640 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=302922528 ingress_bytes=412805120 link_bytes=412805120 token_bytes=605627168 weight_movement_bytes=110100480 total_nvlink_bytes=715727648 weight_to_token_ratio=0.181796 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=13440 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.179359985e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=309528576 dispatch_meta_bytes=234848 combine_payload_bytes=309528576 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=309763424 ingress_bytes=419629056 link_bytes=419629056 token_bytes=619292000 weight_movement_bytes=110100480 total_nvlink_bytes=729392480 weight_to_token_ratio=0.177784 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=22912 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.298039455e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47690 payload_rows_remote=41292 dup_rows=17846 dup_rows_remote=15570 dedup_ratio=0.272308 router_tokens_per_expert=[2, 294, 6, 9, 1703, 318, 73, 3972, 1, 1043, 56, 313, 62, 371, 3170, 69, 40, 291, 167, 466, 56, 0, 77, 2070, 92, 119, 0, 294, 2, 19, 2359, 1928, 110, 11, 34, 913, 10, 939, 25, 1949, 0, 525, 0, 976, 861, 12, 1, 0, 549, 55, 1456, 115, 778, 2283, 2, 5, 0, 3, 285, 1080, 37, 50, 262, 0] payload_rows_per_dst=[8036, 6398, 5386, 7158, 5346, 3994, 7186, 4186] dup_rows_per_dst=[4718, 2276, 948, 4294, 810, 756, 3300, 744] combine_rows_per_src=[8036, 6398, 5386, 7158, 5346, 3994, 7186, 4186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=311793664 dispatch_meta_bytes=236936 combine_payload_bytes=311793664 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=312030600 ingress_bytes=531994624 link_bytes=531994624 token_bytes=623824264 weight_movement_bytes=220200960 total_nvlink_bytes=844025224 weight_to_token_ratio=0.352986 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=20352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.252261492e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=332752896 dispatch_meta_bytes=246832 combine_payload_bytes=332752896 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=332999728 ingress_bytes=552953856 link_bytes=552953856 token_bytes=665752624 weight_movement_bytes=220200960 total_nvlink_bytes=885953584 weight_to_token_ratio=0.330755 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=10112 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.616777009e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45874 payload_rows_remote=39602 dup_rows=19662 dup_rows_remote=17868 dedup_ratio=0.300018 router_tokens_per_expert=[0, 655, 22, 40, 182, 606, 0, 2528, 0, 2556, 0, 54, 576, 343, 2675, 51, 0, 267, 33, 57, 27, 1, 76, 3597, 32, 0, 0, 150, 44, 98, 577, 3337, 1532, 0, 4, 1643, 171, 405, 8, 2118, 0, 124, 0, 2508, 390, 33, 0, 47, 179, 5, 31, 355, 962, 2505, 20, 0, 6, 0, 152, 523, 57, 82, 323, 1] payload_rows_per_dst=[6272, 7410, 7664, 7644, 5872, 5448, 3426, 2138] dup_rows_per_dst=[1794, 5100, 4644, 4118, 2604, 756, 496, 150] combine_rows_per_src=[6272, 7410, 7664, 7644, 5872, 5448, 3426, 2138] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=299063296 dispatch_meta_bytes=224448 combine_payload_bytes=299063296 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=299287744 ingress_bytes=409163776 link_bytes=409163776 token_bytes=598351040 weight_movement_bytes=110100480 total_nvlink_bytes=708451520 weight_to_token_ratio=0.184006 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=16896 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.116031023e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=295981056 dispatch_meta_bytes=227448 combine_payload_bytes=295981056 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=296208504 ingress_bytes=626282496 link_bytes=626282496 token_bytes=592189560 weight_movement_bytes=330301440 total_nvlink_bytes=922491000 weight_to_token_ratio=0.557763 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=15360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.089208266e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=283867136 dispatch_meta_bytes=229880 combine_payload_bytes=283867136 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=284097016 ingress_bytes=614168576 link_bytes=614168576 token_bytes=567964152 weight_movement_bytes=330301440 total_nvlink_bytes=898265592 weight_to_token_ratio=0.581553 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=14208 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.068140167e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47880 payload_rows_remote=39842 dup_rows=17656 dup_rows_remote=11154 dedup_ratio=0.269409 router_tokens_per_expert=[568, 473, 1, 3, 992, 85, 6, 1, 22, 9, 0, 9, 6, 3656, 0, 0, 2, 5, 127, 575, 1, 3727, 270, 0, 1555, 3674, 1, 17, 0, 2, 0, 3611, 3, 2, 0, 257, 0, 78, 1, 1, 1, 923, 48, 3178, 6, 7, 1441, 0, 0, 3, 7, 6, 153, 29, 208, 1408, 6, 257, 0, 19, 1270, 3817, 241, 0] payload_rows_per_dst=[1134, 7320, 7788, 3322, 7890, 4874, 7514, 8038] dup_rows_per_dst=[4, 84, 3330, 108, 4044, 1114, 2470, 6502] combine_rows_per_src=[1134, 7320, 7788, 3322, 7890, 4874, 7514, 8038] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51346 payload_rows_remote=44302 dup_rows=14190 dup_rows_remote=10830 dedup_ratio=0.216522 router_tokens_per_expert=[2994, 531, 0, 0, 1450, 137, 2, 8, 31, 0, 0, 39, 14, 2108, 0, 1, 0, 1, 252, 672, 2, 3278, 62, 53, 1315, 2785, 171, 23, 0, 0, 2, 2734, 4, 1, 0, 211, 0, 1442, 0, 0, 4, 290, 385, 3657, 0, 0, 1529, 0, 1, 36, 7, 0, 42, 730, 230, 502, 0, 223, 3, 47, 802, 3817, 140, 0] payload_rows_per_dst=[3754, 4334, 7142, 6392, 6804, 7044, 7864, 8012] dup_rows_per_dst=[502, 52, 1498, 2200, 1980, 3360, 2546, 2052] combine_rows_per_src=[3754, 4334, 7142, 6392, 6804, 7044, 7864, 8012] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48358 payload_rows_remote=42688 dup_rows=17178 dup_rows_remote=16054 dedup_ratio=0.262115 router_tokens_per_expert=[2211, 2006, 0, 0, 1877, 1493, 3, 1, 2, 4, 2, 5, 76, 2318, 0, 2, 10, 4, 47, 1791, 1, 1087, 426, 31, 2653, 1576, 90, 187, 27, 0, 0, 2125, 15, 29, 3, 80, 0, 103, 0, 4, 45, 1184, 71, 2846, 1, 0, 923, 0, 0, 38, 2, 4, 8, 181, 4, 730, 7, 1150, 0, 54, 1445, 3632, 146, 8] payload_rows_per_dst=[7604, 6514, 5670, 6630, 4488, 4436, 6398, 6618] dup_rows_per_dst=[6870, 1906, 1124, 2436, 230, 874, 1074, 2664] combine_rows_per_src=[7604, 6514, 5670, 6630, 4488, 4436, 6398, 6618] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50752 payload_rows_remote=42716 dup_rows=14784 dup_rows_remote=11446 dedup_ratio=0.225586 router_tokens_per_expert=[1555, 473, 1, 0, 1131, 718, 7, 0, 0, 0, 0, 0, 0, 3761, 0, 1, 1, 0, 291, 889, 1, 1897, 5, 1, 2716, 3762, 170, 8, 0, 0, 0, 3354, 0, 0, 0, 117, 0, 758, 2, 0, 11, 65, 431, 3812, 0, 0, 583, 0, 4, 3, 12, 3, 406, 32, 93, 1322, 0, 64, 0, 69, 730, 3464, 44, 1] payload_rows_per_dst=[3628, 7522, 5502, 6712, 7258, 4696, 8036, 7398] dup_rows_per_dst=[450, 2, 1250, 3726, 4078, 594, 3338, 1346] combine_rows_per_src=[3628, 7522, 5502, 6712, 7258, 4696, 8036, 7398] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=285587456 dispatch_meta_bytes=203984 combine_payload_bytes=285587456 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=285791440 ingress_bytes=505788416 link_bytes=505788416 token_bytes=571378896 weight_movement_bytes=220200960 total_nvlink_bytes=791579856 weight_to_token_ratio=0.385385 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=20608 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.796492433e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53194 payload_rows_remote=46550 dup_rows=12342 dup_rows_remote=11372 dedup_ratio=0.188324 router_tokens_per_expert=[2213, 2468, 0, 0, 1471, 69, 0, 4, 0, 0, 0, 3, 25, 3245, 0, 65, 2, 0, 160, 1370, 0, 1861, 20, 58, 1424, 1957, 38, 0, 0, 1, 0, 3095, 11, 0, 0, 616, 0, 85, 0, 0, 3, 487, 1376, 3608, 1, 0, 149, 0, 0, 0, 1, 0, 118, 11, 71, 1697, 2, 713, 0, 248, 116, 3890, 16, 0] payload_rows_per_dst=[6494, 6572, 5598, 5534, 6644, 6586, 7902, 7864] dup_rows_per_dst=[1530, 104, 1344, 1306, 970, 1872, 3110, 2106] combine_rows_per_src=[6494, 6572, 5598, 5534, 6644, 6586, 7902, 7864] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51996 payload_rows_remote=44400 dup_rows=13540 dup_rows_remote=8578 dedup_ratio=0.206604 router_tokens_per_expert=[1645, 657, 0, 1, 1958, 192, 0, 1, 0, 0, 0, 4, 11, 2067, 0, 2, 0, 0, 300, 1815, 0, 1940, 34, 0, 2848, 3247, 167, 17, 0, 0, 0, 3716, 0, 0, 0, 114, 0, 326, 0, 0, 4, 361, 398, 3489, 0, 0, 1580, 0, 2, 0, 1, 0, 94, 174, 14, 907, 0, 181, 0, 218, 508, 3710, 65, 0] payload_rows_per_dst=[5006, 4164, 6506, 7596, 7498, 6106, 7296, 7824] dup_rows_per_dst=[612, 4, 1672, 4962, 814, 1870, 2066, 1540] combine_rows_per_src=[5006, 4164, 6506, 7596, 7498, 6106, 7296, 7824] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=317556736 dispatch_meta_bytes=220528 combine_payload_bytes=317556736 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=317777264 ingress_bytes=537757696 link_bytes=537757696 token_bytes=635334000 weight_movement_bytes=220200960 total_nvlink_bytes=855534960 weight_to_token_ratio=0.346591 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=9344 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.352490793e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=305987584 dispatch_meta_bytes=234968 combine_payload_bytes=305987584 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=306222552 ingress_bytes=416088064 link_bytes=416088064 token_bytes=612210136 weight_movement_bytes=110100480 total_nvlink_bytes=722310616 weight_to_token_ratio=0.179841 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=20864 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.236455780e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=312137728 dispatch_meta_bytes=237376 combine_payload_bytes=312137728 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=312375104 ingress_bytes=532338688 link_bytes=532338688 token_bytes=624512832 weight_movement_bytes=220200960 total_nvlink_bytes=844713792 weight_to_token_ratio=0.352596 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=19968 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.258245331e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51302 payload_rows_remote=47230 dup_rows=14234 dup_rows_remote=13728 dedup_ratio=0.217194 router_tokens_per_expert=[2930, 524, 0, 0, 1408, 128, 3, 9, 25, 0, 0, 39, 15, 2084, 0, 1, 0, 1, 251, 673, 1, 3287, 67, 52, 1289, 2744, 177, 28, 0, 0, 2, 2769, 3, 1, 0, 233, 0, 1394, 0, 0, 5, 301, 373, 3675, 0, 0, 1610, 0, 2, 36, 6, 0, 36, 654, 237, 599, 0, 268, 3, 45, 814, 3819, 147, 0] payload_rows_per_dst=[3650, 7344, 7158, 6348, 6816, 4072, 7890, 8024] dup_rows_per_dst=[494, 2844, 1506, 2132, 1984, 506, 2600, 2168] combine_rows_per_src=[3650, 7344, 7158, 6348, 6816, 4072, 7890, 8024] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=283193344 dispatch_meta_bytes=229736 combine_payload_bytes=283193344 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=283423080 ingress_bytes=503394304 link_bytes=503394304 token_bytes=566616424 weight_movement_bytes=220200960 total_nvlink_bytes=786817384 weight_to_token_ratio=0.388624 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=14336 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.754854888e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48590 payload_rows_remote=43166 dup_rows=16946 dup_rows_remote=15572 dedup_ratio=0.258575 router_tokens_per_expert=[16, 868, 226, 112, 1055, 327, 0, 3762, 0, 2376, 133, 144, 239, 541, 2608, 145, 0, 14, 125, 1018, 618, 2, 9, 1613, 213, 0, 1, 149, 0, 6, 298, 1893, 1517, 5, 564, 1325, 0, 351, 22, 1324, 0, 374, 20, 1053, 1546, 2, 0, 21, 13, 7, 219, 594, 726, 2766, 0, 0, 1, 105, 314, 193, 301, 62, 832, 0] payload_rows_per_dst=[7944, 5902, 5424, 5784, 5324, 4898, 6712, 6602] dup_rows_per_dst=[4788, 1254, 1374, 1986, 2242, 1134, 1938, 2230] combine_rows_per_src=[7944, 5902, 5424, 5784, 5324, 4898, 6712, 6602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=298217472 dispatch_meta_bytes=217344 combine_payload_bytes=298217472 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=298434816 ingress_bytes=408317952 link_bytes=408317952 token_bytes=596652288 weight_movement_bytes=110100480 total_nvlink_bytes=706752768 weight_to_token_ratio=0.184530 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=19072 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.101320753e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47640 payload_rows_remote=41110 dup_rows=17896 dup_rows_remote=15522 dedup_ratio=0.273071 router_tokens_per_expert=[2, 286, 5, 11, 1688, 338, 81, 3973, 1, 1049, 54, 326, 62, 400, 3138, 70, 39, 272, 174, 436, 50, 0, 70, 2028, 90, 118, 0, 308, 2, 21, 2377, 2025, 102, 10, 35, 924, 13, 958, 21, 1949, 0, 540, 0, 981, 829, 11, 1, 0, 542, 48, 1469, 103, 767, 2228, 2, 6, 0, 5, 294, 1063, 47, 49, 277, 0] payload_rows_per_dst=[8040, 6530, 5278, 7228, 5386, 3984, 7144, 4050] dup_rows_per_dst=[4728, 2374, 860, 4502, 790, 740, 3186, 716] combine_rows_per_src=[8040, 6530, 5278, 7228, 5386, 3984, 7144, 4050] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51184 payload_rows_remote=44504 dup_rows=14352 dup_rows_remote=13328 dedup_ratio=0.218994 router_tokens_per_expert=[2065, 2450, 0, 0, 1461, 59, 0, 6, 0, 0, 0, 1, 23, 3257, 0, 59, 4, 0, 160, 1401, 0, 1892, 25, 68, 1354, 1898, 40, 0, 0, 1, 0, 3110, 10, 0, 0, 647, 0, 85, 0, 0, 3, 468, 1344, 3601, 1, 0, 146, 0, 0, 0, 1, 0, 134, 8, 80, 1813, 2, 773, 0, 251, 121, 3923, 23, 0] payload_rows_per_dst=[6454, 7420, 5698, 5394, 6680, 3690, 7944, 7904] dup_rows_per_dst=[1498, 3390, 1402, 1192, 1024, 234, 3330, 2282] combine_rows_per_src=[6454, 7420, 5698, 5394, 6680, 3690, 7944, 7904] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=299507712 dispatch_meta_bytes=224064 combine_payload_bytes=299507712 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=299731776 ingress_bytes=409608192 link_bytes=409608192 token_bytes=599239488 weight_movement_bytes=110100480 total_nvlink_bytes=709339968 weight_to_token_ratio=0.183734 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=17408 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.123760149e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46696 payload_rows_remote=38612 dup_rows=18840 dup_rows_remote=13390 dedup_ratio=0.287476 router_tokens_per_expert=[1068, 2709, 0, 26, 1890, 214, 87, 14, 1, 0, 1, 0, 238, 2815, 0, 89, 10, 1, 32, 1157, 0, 3296, 7, 62, 1828, 1486, 10, 294, 9, 28, 1, 2452, 0, 103, 0, 155, 1, 237, 9, 8, 1257, 1192, 7, 2112, 0, 9, 1289, 81, 9, 59, 1, 2, 1, 72, 129, 1578, 1, 148, 28, 172, 655, 3623, 5, 0] payload_rows_per_dst=[7496, 8084, 7374, 5682, 5382, 7288, 3516, 1874] dup_rows_per_dst=[4520, 5450, 1756, 1630, 548, 4606, 186, 144] combine_rows_per_src=[7496, 8084, 7374, 5682, 5382, 7288, 3516, 1874] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=306188288 dispatch_meta_bytes=216648 combine_payload_bytes=306188288 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=306404936 ingress_bytes=416288768 link_bytes=416288768 token_bytes=612593224 weight_movement_bytes=110100480 total_nvlink_bytes=722693704 weight_to_token_ratio=0.179729 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=20608 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.239946353e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=333670400 dispatch_meta_bytes=231688 combine_payload_bytes=333670400 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=333902088 ingress_bytes=553871360 link_bytes=553871360 token_bytes=667572488 weight_movement_bytes=220200960 total_nvlink_bytes=887773448 weight_to_token_ratio=0.329853 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.033088 gemm_active_groups=11 gemm_peak_group_rows=24064 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.632733912e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47492 payload_rows_remote=39398 dup_rows=18044 dup_rows_remote=10206 dedup_ratio=0.275330 router_tokens_per_expert=[3332, 801, 0, 0, 3309, 491, 5, 28, 1, 14, 0, 20, 175, 2764, 0, 13, 3, 0, 190, 76, 1, 3688, 16, 17, 927, 2621, 39, 63, 0, 0, 0, 2968, 1, 0, 0, 288, 0, 197, 0, 0, 60, 627, 264, 3772, 8, 3, 374, 0, 71, 17, 1, 0, 265, 110, 29, 617, 26, 105, 0, 20, 513, 3725, 113, 0] payload_rows_per_dst=[8094, 7962, 7462, 6054, 6294, 8012, 2100, 1514] dup_rows_per_dst=[7838, 5462, 520, 1246, 614, 2204, 120, 40] combine_rows_per_src=[8094, 7962, 7462, 6054, 6294, 8012, 2100, 1514] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=318259200 dispatch_meta_bytes=211912 combine_payload_bytes=318259200 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=318471112 ingress_bytes=758661120 link_bytes=758661120 token_bytes=636730312 weight_movement_bytes=440401920 total_nvlink_bytes=1077132232 weight_to_token_ratio=0.691662 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=16000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.319436466e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=276770816 dispatch_meta_bytes=208008 combine_payload_bytes=276770816 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276978824 ingress_bytes=386871296 link_bytes=386871296 token_bytes=553749640 weight_movement_bytes=110100480 total_nvlink_bytes=663850120 weight_to_token_ratio=0.198827 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=22784 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.728328131e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48998 payload_rows_remote=42602 dup_rows=16538 dup_rows_remote=13884 dedup_ratio=0.252350 router_tokens_per_expert=[1, 456, 0, 1364, 1, 0, 3, 266, 0, 2092, 1, 0, 1039, 35, 1554, 0, 0, 1, 918, 1257, 1, 2089, 1, 0, 1597, 1, 3747, 70, 749, 442, 1592, 2173, 1447, 203, 4, 3, 2, 0, 88, 0, 5, 10, 0, 1322, 24, 1142, 605, 0, 104, 0, 8, 0, 218, 10, 6, 3765, 1969, 0, 0, 250, 97, 29, 3, 4] payload_rows_per_dst=[3798, 5392, 6942, 5764, 7874, 5042, 7790, 6396] dup_rows_per_dst=[384, 942, 4700, 2254, 3114, 1174, 1316, 2654] combine_rows_per_src=[3798, 5392, 6942, 5764, 7874, 5042, 7790, 6396] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=282404864 dispatch_meta_bytes=198416 combine_payload_bytes=282404864 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=282603280 ingress_bytes=612706304 link_bytes=612706304 token_bytes=565008144 weight_movement_bytes=330301440 total_nvlink_bytes=895309584 weight_to_token_ratio=0.584596 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=12800 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.065597036e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51446 payload_rows_remote=45638 dup_rows=14090 dup_rows_remote=12650 dedup_ratio=0.214996 router_tokens_per_expert=[143, 475, 3, 1542, 12, 0, 0, 928, 2, 2965, 12, 0, 418, 60, 2177, 0, 0, 1, 91, 1325, 6, 1343, 62, 2, 1121, 5, 3957, 197, 32, 804, 722, 2125, 930, 24, 38, 10, 7, 13, 136, 0, 0, 6, 0, 1935, 7, 1491, 184, 1, 458, 1, 9, 6, 264, 2, 1, 4018, 2324, 0, 33, 194, 96, 7, 2, 41] payload_rows_per_dst=[4944, 6366, 6750, 4822, 7992, 5808, 8094, 6670] dup_rows_per_dst=[1262, 548, 3264, 940, 2238, 1440, 1424, 2974] combine_rows_per_src=[4944, 6366, 6750, 4822, 7992, 5808, 8094, 6670] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48078 payload_rows_remote=43318 dup_rows=17458 dup_rows_remote=16310 dedup_ratio=0.266388 router_tokens_per_expert=[54, 650, 2, 794, 3, 14, 1, 264, 3, 2373, 0, 4, 163, 22, 3566, 13, 4, 0, 112, 1243, 0, 1556, 25, 14, 152, 4, 3378, 466, 26, 1226, 281, 2622, 1827, 55, 0, 7, 10, 0, 89, 1, 1, 15, 4, 1250, 12, 1815, 248, 19, 378, 0, 52, 0, 839, 22, 3, 4018, 1834, 2, 4, 690, 133, 278, 45, 82] payload_rows_per_dst=[3308, 7820, 4760, 7730, 3862, 5442, 8118, 7038] dup_rows_per_dst=[552, 4468, 1148, 3712, 116, 1286, 2506, 3670] combine_rows_per_src=[3308, 7820, 4760, 7730, 3862, 5442, 8118, 7038] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=305371136 dispatch_meta_bytes=225944 combine_payload_bytes=305371136 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=305597080 ingress_bytes=415471616 link_bytes=415471616 token_bytes=610968216 weight_movement_bytes=110100480 total_nvlink_bytes=721068696 weight_to_token_ratio=0.180207 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=16000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.225734736e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50218 payload_rows_remote=42062 dup_rows=15318 dup_rows_remote=12932 dedup_ratio=0.233734 router_tokens_per_expert=[229, 464, 0, 2341, 5, 0, 30, 1006, 1, 2190, 0, 6, 756, 84, 3078, 0, 27, 0, 79, 1874, 0, 845, 0, 0, 1485, 0, 3861, 466, 210, 1486, 341, 1652, 945, 1, 1, 34, 0, 0, 50, 0, 0, 2, 0, 310, 49, 1660, 476, 0, 9, 0, 15, 0, 11, 0, 17, 4076, 2350, 0, 8, 79, 93, 11, 54, 1] payload_rows_per_dst=[6084, 5374, 7506, 4576, 7840, 4478, 8156, 6204] dup_rows_per_dst=[2066, 700, 4300, 1114, 1944, 516, 2386, 2292] combine_rows_per_src=[6084, 5374, 7506, 4576, 7840, 4478, 8156, 6204] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52040 payload_rows_remote=43854 dup_rows=13496 dup_rows_remote=12524 dedup_ratio=0.205933 router_tokens_per_expert=[13, 56, 0, 2631, 0, 1, 0, 2890, 0, 1201, 0, 1, 567, 105, 2900, 0, 4, 0, 209, 1505, 0, 643, 11, 0, 699, 0, 4090, 11, 833, 535, 233, 1787, 233, 82, 106, 0, 0, 0, 68, 0, 0, 19, 0, 3565, 3, 938, 379, 0, 133, 0, 277, 0, 329, 1, 2, 3937, 1134, 0, 5, 468, 97, 46, 17, 4] payload_rows_per_dst=[7632, 6280, 4806, 3910, 8186, 7486, 8024, 5716] dup_rows_per_dst=[3550, 1690, 1516, 712, 972, 2322, 1334, 1400] combine_rows_per_src=[7632, 6280, 4806, 3910, 8186, 7486, 8024, 5716] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=327133184 dispatch_meta_bytes=233152 combine_payload_bytes=327133184 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=327366336 ingress_bytes=437233664 link_bytes=437233664 token_bytes=654499520 weight_movement_bytes=110100480 total_nvlink_bytes=764600000 weight_to_token_ratio=0.168221 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=12160 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.604212543e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=310503424 dispatch_meta_bytes=238512 combine_payload_bytes=310503424 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=310741936 ingress_bytes=420603904 link_bytes=420603904 token_bytes=621245360 weight_movement_bytes=110100480 total_nvlink_bytes=731345840 weight_to_token_ratio=0.177225 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=12672 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.314993665e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48954 payload_rows_remote=44278 dup_rows=16582 dup_rows_remote=14314 dedup_ratio=0.253021 router_tokens_per_expert=[238, 40, 1, 1087, 1, 0, 0, 1391, 1, 2061, 0, 6, 784, 15, 2543, 0, 59, 0, 343, 1237, 0, 1042, 1, 0, 422, 0, 4033, 133, 157, 1211, 1212, 1814, 1273, 55, 3, 0, 0, 0, 44, 0, 0, 36, 0, 1012, 7, 2337, 493, 16, 32, 0, 30, 0, 660, 2, 0, 4014, 2344, 0, 4, 147, 217, 82, 78, 50] payload_rows_per_dst=[4648, 7226, 4242, 4676, 7562, 6060, 8076, 6464] dup_rows_per_dst=[868, 3594, 1122, 2268, 2580, 1742, 1400, 3008] combine_rows_per_src=[4648, 7226, 4242, 4676, 7562, 6060, 8076, 6464] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=338544640 dispatch_meta_bytes=243832 combine_payload_bytes=338544640 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=338788472 ingress_bytes=558745600 link_bytes=558745600 token_bytes=677333112 weight_movement_bytes=220200960 total_nvlink_bytes=897534072 weight_to_token_ratio=0.325100 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=8704 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.717504963e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=309413888 dispatch_meta_bytes=234952 combine_payload_bytes=309413888 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=309648840 ingress_bytes=419514368 link_bytes=419514368 token_bytes=619062728 weight_movement_bytes=110100480 total_nvlink_bytes=729163208 weight_to_token_ratio=0.177850 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=22912 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.296044842e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=294676480 dispatch_meta_bytes=226528 combine_payload_bytes=294676480 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=294903008 ingress_bytes=735078400 link_bytes=735078400 token_bytes=589579488 weight_movement_bytes=440401920 total_nvlink_bytes=1029981408 weight_to_token_ratio=0.746976 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=14720 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.278422237e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=319004672 dispatch_meta_bytes=231328 combine_payload_bytes=319004672 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=319236000 ingress_bytes=539205632 link_bytes=539205632 token_bytes=638240672 weight_movement_bytes=220200960 total_nvlink_bytes=858441632 weight_to_token_ratio=0.345012 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.033088 gemm_active_groups=11 gemm_peak_group_rows=24192 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.377672781e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49654 payload_rows_remote=41602 dup_rows=15882 dup_rows_remote=12302 dedup_ratio=0.242340 router_tokens_per_expert=[1501, 477, 1, 0, 1123, 647, 8, 0, 0, 0, 0, 0, 0, 3738, 0, 0, 1, 0, 280, 925, 0, 1910, 3, 1, 2701, 3760, 170, 9, 0, 0, 0, 3328, 0, 0, 0, 125, 0, 761, 2, 0, 12, 67, 406, 3830, 0, 0, 603, 0, 4, 4, 11, 5, 383, 21, 99, 1459, 0, 76, 0, 69, 740, 3459, 48, 1] payload_rows_per_dst=[4046, 7826, 5280, 7876, 7044, 2144, 8052, 7386] dup_rows_per_dst=[466, 2652, 960, 5404, 1388, 32, 3580, 1400] combine_rows_per_src=[4046, 7826, 5280, 7876, 7044, 2144, 8052, 7386] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43984 payload_rows_remote=35870 dup_rows=21552 dup_rows_remote=13584 dedup_ratio=0.328857 router_tokens_per_expert=[3279, 803, 0, 0, 3319, 470, 5, 28, 2, 15, 0, 16, 175, 2759, 0, 16, 3, 0, 179, 81, 2, 3688, 16, 21, 905, 2577, 41, 55, 0, 0, 0, 2999, 1, 0, 0, 292, 0, 197, 0, 1, 63, 590, 250, 3762, 9, 1, 405, 0, 76, 19, 1, 0, 256, 101, 24, 714, 26, 125, 0, 24, 531, 3725, 121, 0] payload_rows_per_dst=[8114, 5848, 7500, 4120, 6348, 8186, 2268, 1600] dup_rows_per_dst=[7968, 118, 2422, 820, 632, 9424, 114, 54] combine_rows_per_src=[8114, 5848, 7500, 4120, 6348, 8186, 2268, 1600] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47854 payload_rows_remote=39810 dup_rows=17682 dup_rows_remote=13120 dedup_ratio=0.269806 router_tokens_per_expert=[506, 468, 1, 3, 944, 78, 6, 2, 13, 8, 0, 8, 5, 3648, 0, 0, 3, 5, 117, 595, 1, 3702, 272, 0, 1542, 3673, 1, 17, 0, 2, 0, 3628, 3, 2, 0, 257, 0, 80, 1, 1, 1, 851, 27, 3206, 6, 7, 1500, 0, 0, 5, 8, 8, 158, 25, 206, 1506, 4, 278, 0, 20, 1285, 3824, 251, 0] payload_rows_per_dst=[1660, 7514, 7688, 3514, 7848, 4042, 7544, 8044] dup_rows_per_dst=[62, 862, 1702, 296, 6756, 742, 2700, 4562] combine_rows_per_src=[1660, 7514, 7688, 3514, 7848, 4042, 7544, 8044] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51940 payload_rows_remote=44386 dup_rows=13596 dup_rows_remote=8696 dedup_ratio=0.207458 router_tokens_per_expert=[1581, 685, 0, 0, 1942, 187, 0, 0, 1, 0, 0, 4, 11, 2073, 0, 2, 0, 0, 285, 1827, 0, 1970, 35, 0, 2844, 3203, 161, 19, 0, 0, 0, 3688, 0, 0, 0, 123, 0, 331, 0, 0, 5, 333, 415, 3511, 0, 0, 1586, 0, 2, 0, 1, 1, 94, 161, 15, 940, 0, 198, 0, 218, 508, 3734, 74, 0] payload_rows_per_dst=[5000, 5800, 6534, 7554, 7460, 4378, 7348, 7866] dup_rows_per_dst=[628, 1544, 1700, 4900, 824, 300, 2102, 1598] combine_rows_per_src=[5000, 5800, 6534, 7554, 7460, 4378, 7348, 7866] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=298203136 dispatch_meta_bytes=215616 combine_payload_bytes=298203136 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=298418752 ingress_bytes=408303616 link_bytes=408303616 token_bytes=596621888 weight_movement_bytes=110100480 total_nvlink_bytes=706722368 weight_to_token_ratio=0.184540 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=19968 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.101071426e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=257116160 dispatch_meta_bytes=197816 combine_payload_bytes=257116160 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=257313976 ingress_bytes=587417600 link_bytes=587417600 token_bytes=514430136 weight_movement_bytes=330301440 total_nvlink_bytes=844731576 weight_to_token_ratio=0.642072 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=13312 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.021615820e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46952 payload_rows_remote=41300 dup_rows=18584 dup_rows_remote=17508 dedup_ratio=0.283569 router_tokens_per_expert=[2185, 2038, 0, 0, 1838, 1486, 2, 2, 3, 5, 2, 4, 76, 2350, 0, 3, 9, 6, 40, 1786, 0, 1089, 407, 27, 2613, 1542, 80, 182, 29, 0, 0, 2116, 17, 24, 4, 81, 0, 100, 0, 4, 46, 1137, 64, 2861, 1, 0, 957, 0, 0, 37, 1, 4, 8, 152, 4, 796, 7, 1225, 0, 61, 1461, 3639, 149, 8] payload_rows_per_dst=[7440, 5286, 5652, 6534, 4456, 5078, 5286, 7220] dup_rows_per_dst=[6380, 882, 1076, 2358, 236, 3372, 862, 3418] combine_rows_per_src=[7440, 5286, 5652, 6534, 4456, 5078, 5286, 7220] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50710 payload_rows_remote=43482 dup_rows=14826 dup_rows_remote=10590 dedup_ratio=0.226227 router_tokens_per_expert=[143, 401, 2, 1543, 12, 0, 0, 947, 2, 2910, 9, 1, 439, 64, 2204, 0, 0, 1, 66, 1342, 5, 1360, 60, 2, 1138, 4, 3958, 204, 29, 790, 749, 2171, 1023, 23, 37, 11, 7, 15, 135, 0, 0, 6, 0, 1908, 8, 1424, 181, 1, 468, 1, 9, 8, 262, 2, 2, 4027, 2298, 0, 35, 189, 85, 4, 2, 41] payload_rows_per_dst=[4870, 6290, 4682, 4866, 8008, 7228, 8110, 6656] dup_rows_per_dst=[1226, 560, 990, 962, 2410, 4236, 1448, 2994] combine_rows_per_src=[4870, 6290, 4682, 4866, 8008, 7228, 8110, 6656] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=285358080 dispatch_meta_bytes=211720 combine_payload_bytes=285358080 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=285569800 ingress_bytes=505559040 link_bytes=505559040 token_bytes=570927880 weight_movement_bytes=220200960 total_nvlink_bytes=791128840 weight_to_token_ratio=0.385690 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=21248 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.792503207e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45414 payload_rows_remote=39492 dup_rows=20122 dup_rows_remote=19838 dedup_ratio=0.307037 router_tokens_per_expert=[1007, 2687, 0, 25, 1814, 200, 86, 19, 1, 0, 1, 0, 231, 2782, 0, 88, 8, 3, 36, 1195, 1, 3311, 6, 69, 1795, 1433, 7, 269, 11, 31, 1, 2538, 0, 107, 0, 133, 1, 227, 8, 6, 1286, 1160, 8, 2175, 0, 10, 1320, 86, 10, 57, 0, 3, 1, 67, 127, 1693, 1, 162, 26, 178, 633, 3623, 4, 1] payload_rows_per_dst=[7414, 5922, 7394, 5594, 5510, 7960, 3726, 1894] dup_rows_per_dst=[4262, 284, 1864, 1500, 530, 11376, 190, 116] combine_rows_per_src=[7414, 5922, 7394, 5594, 5510, 7960, 3726, 1894] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51780 payload_rows_remote=43594 dup_rows=13756 dup_rows_remote=12798 dedup_ratio=0.209900 router_tokens_per_expert=[14, 49, 0, 2639, 0, 1, 0, 2869, 0, 1164, 0, 0, 544, 115, 2925, 0, 5, 0, 194, 1543, 0, 662, 9, 0, 727, 0, 4091, 9, 832, 521, 261, 1775, 221, 77, 107, 0, 0, 0, 76, 0, 0, 16, 0, 3571, 7, 918, 385, 0, 147, 0, 290, 0, 340, 1, 2, 3948, 1080, 0, 3, 475, 93, 45, 12, 5] payload_rows_per_dst=[7610, 6402, 4122, 3962, 8186, 7746, 8038, 5714] dup_rows_per_dst=[3534, 1790, 704, 738, 958, 3352, 1418, 1262] combine_rows_per_src=[7610, 6402, 4122, 3962, 8186, 7746, 8038, 5714] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=318158848 dispatch_meta_bytes=212328 combine_payload_bytes=318158848 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=318371176 ingress_bytes=758560768 link_bytes=758560768 token_bytes=636530024 weight_movement_bytes=440401920 total_nvlink_bytes=1076931944 weight_to_token_ratio=0.691879 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=16512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.319261937e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49388 payload_rows_remote=42842 dup_rows=16148 dup_rows_remote=14498 dedup_ratio=0.246399 router_tokens_per_expert=[51, 328, 1, 1806, 22, 3, 37, 919, 1, 2980, 2, 1, 82, 11, 1017, 4, 15, 2, 31, 2752, 5, 986, 46, 73, 13, 6, 3610, 1460, 516, 593, 463, 2178, 2361, 13, 7, 64, 25, 20, 18, 14, 0, 0, 4, 741, 27, 184, 368, 20, 954, 6, 390, 1, 287, 3, 189, 3640, 2345, 0, 11, 903, 13, 35, 40, 71] payload_rows_per_dst=[8050, 6546, 6460, 6888, 4956, 7174, 3534, 5780] dup_rows_per_dst=[5504, 1650, 1360, 3570, 88, 2232, 688, 1056] combine_rows_per_src=[8050, 6546, 6460, 6888, 4956, 7174, 3534, 5780] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=301500416 dispatch_meta_bytes=219976 combine_payload_bytes=301500416 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=301720392 ingress_bytes=521701376 link_bytes=521701376 token_bytes=603220808 weight_movement_bytes=220200960 total_nvlink_bytes=823421768 weight_to_token_ratio=0.365042 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=24192 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.073244980e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=314345472 dispatch_meta_bytes=225512 combine_payload_bytes=314345472 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=314570984 ingress_bytes=424445952 link_bytes=424445952 token_bytes=628916456 weight_movement_bytes=110100480 total_nvlink_bytes=739016936 weight_to_token_ratio=0.175064 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=19456 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.381813199e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46706 payload_rows_remote=38552 dup_rows=18830 dup_rows_remote=14098 dedup_ratio=0.287323 router_tokens_per_expert=[0, 367, 81, 1283, 1, 2, 0, 689, 0, 430, 5, 2, 569, 0, 1048, 0, 132, 0, 426, 1448, 0, 1252, 72, 0, 1798, 0, 4020, 763, 8, 1249, 1439, 2459, 2991, 16, 69, 0, 0, 1, 29, 0, 0, 8, 0, 199, 7, 2534, 99, 9, 729, 0, 4, 3, 608, 0, 1, 3811, 1595, 0, 2, 240, 233, 2, 33, 2] payload_rows_per_dst=[8154, 5550, 5160, 7294, 6114, 8102, 2500, 3832] dup_rows_per_dst=[4732, 1390, 1500, 5306, 98, 5232, 190, 382] combine_rows_per_src=[8154, 5550, 5160, 7294, 6114, 8102, 2500, 3832] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=317384704 dispatch_meta_bytes=234368 combine_payload_bytes=317384704 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=317619072 ingress_bytes=867887104 link_bytes=867887104 token_bytes=635003776 weight_movement_bytes=550502400 total_nvlink_bytes=1185506176 weight_to_token_ratio=0.866928 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.509398417e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=296038400 dispatch_meta_bytes=235232 combine_payload_bytes=296038400 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=296273632 ingress_bytes=406138880 link_bytes=406138880 token_bytes=592312032 weight_movement_bytes=110100480 total_nvlink_bytes=702412512 weight_to_token_ratio=0.185883 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=20864 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.063423107e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=311678976 dispatch_meta_bytes=216288 combine_payload_bytes=311678976 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=311895264 ingress_bytes=421779456 link_bytes=421779456 token_bytes=623574240 weight_movement_bytes=110100480 total_nvlink_bytes=733674720 weight_to_token_ratio=0.176564 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=11776 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.335438448e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=283078656 dispatch_meta_bytes=237320 combine_payload_bytes=283078656 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=283315976 ingress_bytes=393179136 link_bytes=393179136 token_bytes=566394632 weight_movement_bytes=110100480 total_nvlink_bytes=676495112 weight_to_token_ratio=0.194388 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=22784 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.838031844e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=312481792 dispatch_meta_bytes=225568 combine_payload_bytes=312481792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=312707360 ingress_bytes=422582272 link_bytes=422582272 token_bytes=625189152 weight_movement_bytes=110100480 total_nvlink_bytes=735289632 weight_to_token_ratio=0.176107 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=19072 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.349400738e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51754 payload_rows_remote=43596 dup_rows=13782 dup_rows_remote=13668 dedup_ratio=0.210297 router_tokens_per_expert=[222, 418, 0, 2339, 6, 1, 30, 988, 1, 2145, 0, 5, 788, 98, 3091, 0, 25, 0, 67, 1902, 0, 872, 0, 0, 1559, 0, 3836, 498, 194, 1446, 339, 1675, 958, 1, 1, 30, 0, 0, 50, 0, 0, 2, 0, 326, 54, 1585, 454, 0, 10, 0, 15, 0, 15, 0, 19, 4077, 2374, 0, 6, 93, 84, 13, 55, 1] payload_rows_per_dst=[6012, 5350, 4892, 5742, 7802, 7542, 8158, 6256] dup_rows_per_dst=[1996, 724, 840, 2330, 1950, 3482, 114, 2346] combine_rows_per_src=[6012, 5350, 4892, 5742, 7802, 7542, 8158, 6256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47058 payload_rows_remote=38914 dup_rows=18478 dup_rows_remote=13864 dedup_ratio=0.281952 router_tokens_per_expert=[0, 316, 83, 1282, 2, 2, 0, 674, 0, 428, 5, 1, 552, 0, 1067, 0, 129, 0, 390, 1465, 0, 1267, 71, 0, 1835, 0, 4020, 821, 12, 1211, 1476, 2422, 3031, 16, 60, 0, 1, 1, 24, 0, 0, 7, 0, 194, 4, 2468, 77, 7, 748, 0, 2, 3, 664, 0, 2, 3817, 1597, 0, 1, 251, 222, 3, 36, 1] payload_rows_per_dst=[8144, 5334, 8070, 7632, 6176, 5224, 2622, 3856] dup_rows_per_dst=[4614, 1194, 6208, 5500, 90, 290, 216, 366] combine_rows_per_src=[8144, 5334, 8070, 7632, 6176, 5224, 2622, 3856] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=307091456 dispatch_meta_bytes=229360 combine_payload_bytes=307091456 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=307320816 ingress_bytes=527292416 link_bytes=527292416 token_bytes=614412272 weight_movement_bytes=220200960 total_nvlink_bytes=834613232 weight_to_token_ratio=0.358393 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=16384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.170482361e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43944 payload_rows_remote=36942 dup_rows=21592 dup_rows_remote=18120 dedup_ratio=0.329468 router_tokens_per_expert=[2, 1167, 2428, 0, 0, 1513, 7, 1, 1697, 1122, 6, 1, 0, 4, 1, 2382, 0, 0, 0, 0, 100, 136, 2930, 4, 21, 3, 1542, 0, 0, 3150, 773, 111, 0, 0, 34, 15, 1, 1, 1, 1179, 55, 510, 0, 617, 481, 5, 26, 1, 204, 5, 29, 200, 3771, 2490, 4, 1183, 178, 18, 25, 98, 322, 2171, 42, 1] payload_rows_per_dst=[6242, 6148, 6110, 5056, 2444, 2878, 8064, 7002] dup_rows_per_dst=[968, 2540, 230, 1176, 18, 512, 12676, 3472] combine_rows_per_src=[6242, 6148, 6110, 5056, 2444, 2878, 8064, 7002] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=276340736 dispatch_meta_bytes=210600 combine_payload_bytes=276340736 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276551336 ingress_bytes=386441216 link_bytes=386441216 token_bytes=552892072 weight_movement_bytes=110100480 total_nvlink_bytes=662992552 weight_to_token_ratio=0.199136 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=12928 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.720848333e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46576 payload_rows_remote=38472 dup_rows=18960 dup_rows_remote=13820 dedup_ratio=0.289307 router_tokens_per_expert=[25, 478, 2353, 19, 0, 2215, 8, 10, 1386, 488, 75, 4, 1, 0, 1, 2157, 9, 42, 0, 0, 90, 1994, 3784, 703, 4, 1, 883, 8, 36, 2859, 1588, 13, 10, 0, 0, 42, 21, 0, 10, 1780, 11, 40, 2, 1544, 173, 12, 25, 17, 36, 1, 1, 357, 3174, 1576, 73, 421, 200, 3, 27, 774, 570, 435, 199, 0] payload_rows_per_dst=[6850, 6284, 8104, 7186, 7434, 5386, 1670, 3662] dup_rows_per_dst=[3366, 1940, 5140, 3598, 2640, 1414, 108, 754] combine_rows_per_src=[6850, 6284, 8104, 7186, 7434, 5386, 1670, 3662] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48538 payload_rows_remote=44284 dup_rows=16998 dup_rows_remote=16202 dedup_ratio=0.259369 router_tokens_per_expert=[2, 349, 2177, 64, 4, 2045, 12, 6, 2744, 1299, 0, 3, 0, 54, 0, 2156, 6, 15, 1, 0, 23, 243, 3569, 11, 1, 0, 75, 0, 3, 1974, 888, 0, 0, 1, 24, 0, 2, 15, 66, 1845, 3, 89, 0, 814, 1394, 0, 97, 2, 45, 47, 0, 131, 3643, 1463, 2, 2298, 659, 5, 125, 532, 499, 1226, 15, 2] payload_rows_per_dst=[6630, 6592, 7270, 5002, 5374, 4254, 6804, 6612] dup_rows_per_dst=[2688, 1608, 466, 880, 1666, 796, 5068, 3826] combine_rows_per_src=[6630, 6592, 7270, 5002, 5374, 4254, 6804, 6612] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=264800256 dispatch_meta_bytes=220248 combine_payload_bytes=264800256 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=265020504 ingress_bytes=374900736 link_bytes=374900736 token_bytes=529820760 weight_movement_bytes=110100480 total_nvlink_bytes=639921240 weight_to_token_ratio=0.207807 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=12032 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.520140405e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48270 payload_rows_remote=41936 dup_rows=17266 dup_rows_remote=14744 dedup_ratio=0.263458 router_tokens_per_expert=[1, 414, 0, 1378, 1, 0, 2, 281, 0, 2048, 1, 0, 1069, 42, 1607, 0, 0, 1, 886, 1253, 1, 2086, 2, 0, 1677, 0, 3741, 73, 753, 441, 1634, 2174, 1453, 200, 3, 3, 2, 0, 80, 0, 5, 8, 0, 1344, 23, 1079, 592, 0, 124, 0, 12, 0, 239, 8, 5, 3768, 1872, 0, 0, 261, 85, 28, 3, 5] payload_rows_per_dst=[3762, 5360, 5970, 4822, 7840, 6416, 7766, 6334] dup_rows_per_dst=[392, 960, 2488, 1094, 3124, 2900, 3786, 2522] combine_rows_per_src=[3762, 5360, 5970, 4822, 7840, 6416, 7766, 6334] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48656 payload_rows_remote=43848 dup_rows=16880 dup_rows_remote=14162 dedup_ratio=0.257568 router_tokens_per_expert=[238, 40, 0, 1074, 1, 0, 0, 1377, 1, 1979, 0, 5, 791, 16, 2568, 0, 66, 0, 329, 1266, 0, 1010, 2, 0, 451, 0, 4028, 149, 154, 1206, 1216, 1834, 1333, 49, 3, 0, 1, 0, 37, 0, 0, 30, 0, 1062, 7, 2263, 491, 17, 32, 0, 26, 0, 720, 2, 0, 4016, 2320, 0, 2, 148, 200, 86, 66, 56] payload_rows_per_dst=[4624, 7212, 4226, 4808, 7232, 6020, 8074, 6460] dup_rows_per_dst=[836, 3508, 1120, 2718, 2496, 1720, 1518, 2964] combine_rows_per_src=[4624, 7212, 4226, 4808, 7232, 6020, 8074, 6460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=312496128 dispatch_meta_bytes=229056 combine_payload_bytes=312496128 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=312725184 ingress_bytes=532697088 link_bytes=532697088 token_bytes=625221312 weight_movement_bytes=220200960 total_nvlink_bytes=845422272 weight_to_token_ratio=0.352197 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=23808 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.264478496e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278935552 dispatch_meta_bytes=211112 combine_payload_bytes=278935552 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279146664 ingress_bytes=389036032 link_bytes=389036032 token_bytes=558082216 weight_movement_bytes=110100480 total_nvlink_bytes=668182696 weight_to_token_ratio=0.197284 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=13056 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.765976451e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48230 payload_rows_remote=43350 dup_rows=17306 dup_rows_remote=16042 dedup_ratio=0.264069 router_tokens_per_expert=[54, 605, 1, 833, 3, 15, 0, 265, 3, 2320, 0, 3, 163, 25, 3549, 14, 3, 0, 100, 1226, 0, 1532, 27, 15, 153, 6, 3365, 473, 27, 1199, 301, 2609, 1896, 56, 0, 6, 9, 0, 100, 1, 1, 16, 5, 1294, 10, 1756, 250, 20, 426, 1, 51, 0, 891, 20, 3, 4015, 1785, 2, 6, 700, 130, 299, 42, 88] payload_rows_per_dst=[3618, 7776, 4880, 7446, 4002, 5458, 7924, 7126] dup_rows_per_dst=[744, 4378, 1264, 3150, 134, 1246, 2552, 3838] combine_rows_per_src=[3618, 7776, 4880, 7446, 4002, 5458, 7924, 7126] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48452 payload_rows_remote=44274 dup_rows=17084 dup_rows_remote=16312 dedup_ratio=0.260681 router_tokens_per_expert=[2, 352, 2229, 71, 4, 2077, 11, 7, 2744, 1265, 0, 3, 0, 52, 0, 2064, 6, 14, 0, 0, 28, 240, 3503, 9, 1, 0, 73, 0, 3, 2003, 886, 0, 0, 1, 26, 1, 2, 14, 75, 1813, 3, 88, 0, 850, 1430, 0, 102, 2, 42, 44, 0, 131, 3648, 1536, 2, 2269, 626, 6, 123, 574, 501, 1197, 14, 1] payload_rows_per_dst=[6640, 6560, 7146, 5064, 5480, 4178, 6832, 6552] dup_rows_per_dst=[2738, 1568, 454, 868, 1696, 772, 5328, 3660] combine_rows_per_src=[6640, 6560, 7146, 5064, 5480, 4178, 6832, 6552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46026 payload_rows_remote=39416 dup_rows=19510 dup_rows_remote=17422 dedup_ratio=0.297699 router_tokens_per_expert=[51, 325, 1, 1856, 22, 4, 37, 882, 1, 2849, 2, 0, 75, 12, 1033, 5, 8, 1, 34, 2795, 4, 946, 46, 76, 13, 6, 3599, 1508, 492, 587, 492, 2157, 2386, 15, 6, 75, 30, 16, 20, 16, 0, 0, 3, 768, 24, 174, 352, 17, 979, 6, 390, 0, 329, 3, 189, 3630, 2326, 0, 13, 918, 12, 30, 39, 83] payload_rows_per_dst=[8036, 6610, 8012, 6744, 5028, 2604, 3246, 5746] dup_rows_per_dst=[5518, 2088, 7068, 3022, 100, 72, 546, 1096] combine_rows_per_src=[8036, 6610, 8012, 6744, 5028, 2604, 3246, 5746] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50390 payload_rows_remote=42398 dup_rows=15146 dup_rows_remote=10700 dedup_ratio=0.231110 router_tokens_per_expert=[0, 1493, 797, 389, 0, 1041, 0, 0, 406, 1521, 2, 0, 0, 7, 0, 3433, 31, 3, 0, 3, 6, 3, 4043, 5, 0, 1, 604, 0, 8, 1472, 534, 10, 0, 0, 15, 7, 4, 0, 19, 2735, 1, 397, 0, 544, 1924, 17, 0, 0, 0, 43, 0, 304, 3439, 2086, 53, 2089, 3, 5, 0, 819, 431, 1968, 53, 0] payload_rows_per_dst=[5972, 5306, 8088, 4546, 7992, 6786, 4730, 6970] dup_rows_per_dst=[1468, 1108, 100, 712, 4446, 2894, 506, 3912] combine_rows_per_src=[5972, 5306, 8088, 4546, 7992, 6786, 4730, 6970] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=300597248 dispatch_meta_bytes=226720 combine_payload_bytes=300597248 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=300823968 ingress_bytes=410697728 link_bytes=410697728 token_bytes=601421216 weight_movement_bytes=110100480 total_nvlink_bytes=711521696 weight_to_token_ratio=0.183067 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=15744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.142708972e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46368 payload_rows_remote=38182 dup_rows=19168 dup_rows_remote=10772 dedup_ratio=0.292480 router_tokens_per_expert=[0, 1084, 79, 49, 2, 510, 1, 0, 1501, 1871, 0, 0, 0, 127, 0, 1843, 5, 0, 0, 10, 193, 73, 3662, 24, 2, 0, 1366, 22, 0, 1554, 241, 0, 0, 0, 310, 12, 29, 0, 13, 1071, 18, 2793, 2, 1971, 451, 1, 21, 0, 154, 91, 0, 33, 3956, 2478, 0, 1579, 75, 31, 25, 454, 743, 2238, 0, 0] payload_rows_per_dst=[2430, 5838, 7408, 5354, 2712, 7246, 8186, 7194] dup_rows_per_dst=[64, 2116, 526, 1016, 158, 3268, 8396, 3624] combine_rows_per_src=[2430, 5838, 7408, 5354, 2712, 7246, 8186, 7194] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50532 payload_rows_remote=42576 dup_rows=15004 dup_rows_remote=10602 dedup_ratio=0.228943 router_tokens_per_expert=[0, 1500, 779, 365, 0, 1047, 0, 0, 410, 1603, 1, 0, 0, 7, 0, 3490, 34, 2, 0, 3, 4, 4, 4046, 5, 0, 1, 577, 0, 8, 1441, 558, 12, 0, 0, 15, 7, 2, 0, 22, 2711, 0, 457, 0, 566, 1822, 16, 0, 0, 1, 38, 0, 300, 3422, 2063, 54, 2129, 3, 3, 0, 812, 422, 1956, 50, 0] payload_rows_per_dst=[5972, 5812, 8096, 4498, 7956, 6864, 4638, 6696] dup_rows_per_dst=[1410, 1430, 100, 696, 4402, 2984, 406, 3576] combine_rows_per_src=[5972, 5812, 8096, 4498, 7956, 6864, 4638, 6696] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=275767296 dispatch_meta_bytes=209168 combine_payload_bytes=275767296 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=275976464 ingress_bytes=495968256 link_bytes=495968256 token_bytes=551743760 weight_movement_bytes=220200960 total_nvlink_bytes=771944720 weight_to_token_ratio=0.399100 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=28032 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.625703699e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48806 payload_rows_remote=41602 dup_rows=16730 dup_rows_remote=13428 dedup_ratio=0.255280 router_tokens_per_expert=[0, 2173, 291, 79, 0, 711, 0, 0, 1759, 1002, 0, 0, 0, 447, 0, 1884, 0, 0, 0, 2, 84, 166, 4049, 148, 6, 0, 1568, 16, 0, 2711, 952, 0, 0, 0, 4, 49, 24, 0, 76, 1827, 0, 164, 8, 1078, 1362, 3, 219, 0, 33, 3, 0, 236, 3986, 2683, 0, 964, 16, 48, 0, 1259, 300, 374, 4, 0] payload_rows_per_dst=[5536, 7002, 8104, 7204, 8078, 7042, 2312, 3528] dup_rows_per_dst=[972, 3182, 794, 3302, 3854, 3992, 160, 474] combine_rows_per_src=[5536, 7002, 8104, 7204, 8078, 7042, 2312, 3528] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=317427712 dispatch_meta_bytes=241944 combine_payload_bytes=317427712 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=317669656 ingress_bytes=427528192 link_bytes=427528192 token_bytes=635097368 weight_movement_bytes=110100480 total_nvlink_bytes=745197848 weight_to_token_ratio=0.173360 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=10240 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.435418422e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=314302464 dispatch_meta_bytes=232040 combine_payload_bytes=314302464 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=314534504 ingress_bytes=864804864 link_bytes=864804864 token_bytes=628836968 weight_movement_bytes=550502400 total_nvlink_bytes=1179339368 weight_to_token_ratio=0.875429 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=6528 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.504037894e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=310732800 dispatch_meta_bytes=237568 combine_payload_bytes=310732800 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=310970368 ingress_bytes=420833280 link_bytes=420833280 token_bytes=621703168 weight_movement_bytes=110100480 total_nvlink_bytes=731803648 weight_to_token_ratio=0.177095 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=12800 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.318982891e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49656 payload_rows_remote=44666 dup_rows=15880 dup_rows_remote=15316 dedup_ratio=0.242310 router_tokens_per_expert=[0, 1314, 1736, 383, 0, 3054, 3, 0, 422, 1831, 0, 32, 16, 6, 2, 468, 1, 0, 12, 0, 39, 217, 3246, 51, 6, 61, 8, 2, 47, 3707, 1840, 2, 7, 1, 418, 14, 0, 171, 685, 303, 24, 73, 11, 1759, 659, 5, 34, 30, 4, 8, 277, 1707, 1879, 1303, 46, 569, 946, 41, 63, 18, 1968, 1181, 23, 35] payload_rows_per_dst=[7618, 4990, 6836, 7854, 5642, 5898, 4482, 6336] dup_rows_per_dst=[5362, 564, 296, 3492, 1314, 1898, 740, 2214] combine_rows_per_src=[7618, 4990, 6836, 7854, 5642, 5898, 4482, 6336] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=273688576 dispatch_meta_bytes=195816 combine_payload_bytes=273688576 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=273884392 ingress_bytes=603990016 link_bytes=603990016 token_bytes=547572968 weight_movement_bytes=330301440 total_nvlink_bytes=877874408 weight_to_token_ratio=0.603210 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=9984 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.050437977e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48478 payload_rows_remote=41950 dup_rows=17058 dup_rows_remote=14158 dedup_ratio=0.260284 router_tokens_per_expert=[12, 367, 1020, 118, 0, 2568, 0, 4, 1804, 1361, 1, 1, 0, 163, 0, 1415, 3, 106, 0, 1, 4, 50, 3351, 20, 2, 0, 499, 0, 5, 2526, 1694, 3, 0, 0, 0, 0, 53, 0, 77, 2238, 47, 1025, 0, 333, 1436, 1, 229, 4, 2, 3, 0, 18, 3554, 2477, 64, 826, 44, 29, 3, 299, 437, 2365, 105, 1] payload_rows_per_dst=[6528, 6652, 5892, 6896, 7956, 7200, 1810, 5544] dup_rows_per_dst=[2900, 2838, 164, 2326, 3888, 3904, 16, 1022] combine_rows_per_src=[6528, 6652, 5892, 6896, 7956, 7200, 1810, 5544] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=305184768 dispatch_meta_bytes=212712 combine_payload_bytes=305184768 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=305397480 ingress_bytes=415285248 link_bytes=415285248 token_bytes=610582248 weight_movement_bytes=110100480 total_nvlink_bytes=720682728 weight_to_token_ratio=0.180320 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=17664 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.222493490e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=298203136 dispatch_meta_bytes=220120 combine_payload_bytes=298203136 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=298423256 ingress_bytes=518404096 link_bytes=518404096 token_bytes=596626392 weight_movement_bytes=220200960 total_nvlink_bytes=816827352 weight_to_token_ratio=0.369077 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=17408 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.015899858e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45048 payload_rows_remote=37318 dup_rows=20488 dup_rows_remote=13082 dedup_ratio=0.312622 router_tokens_per_expert=[1, 117, 0, 2581, 34, 359, 0, 1, 0, 1, 168, 3, 0, 346, 118, 5, 12, 0, 22, 3, 44, 5, 0, 116, 372, 204, 2264, 3230, 0, 0, 0, 1177, 3518, 19, 47, 631, 831, 5, 0, 11, 8, 84, 7, 59, 4043, 2, 8, 911, 17, 52, 307, 2, 1481, 1, 83, 3, 1952, 1, 0, 1887, 0, 2604, 1664, 1347] payload_rows_per_dst=[5546, 1206, 3998, 7914, 2838, 8124, 7692, 7730] dup_rows_per_dst=[640, 76, 180, 6580, 250, 2120, 3236, 7406] combine_rows_per_src=[5546, 1206, 3998, 7914, 2838, 8124, 7692, 7730] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=320165888 dispatch_meta_bytes=239928 combine_payload_bytes=320165888 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=320405816 ingress_bytes=540366848 link_bytes=540366848 token_bytes=640571704 weight_movement_bytes=220200960 total_nvlink_bytes=860772664 weight_to_token_ratio=0.343757 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=11776 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.397868237e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=300697600 dispatch_meta_bytes=224432 combine_payload_bytes=300697600 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=300922032 ingress_bytes=520898560 link_bytes=520898560 token_bytes=601619632 weight_movement_bytes=220200960 total_nvlink_bytes=821820592 weight_to_token_ratio=0.366014 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=11776 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.059282689e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 1, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45074 payload_rows_remote=37444 dup_rows=20462 dup_rows_remote=16496 dedup_ratio=0.312225 router_tokens_per_expert=[94, 24, 2, 2098, 108, 265, 0, 14, 1, 507, 211, 1, 0, 1811, 48, 22, 0, 2, 26, 3, 191, 7, 3, 24, 286, 181, 21, 3581, 2, 1, 0, 291, 3182, 2, 13, 381, 1415, 59, 0, 65, 3, 26, 5, 251, 4064, 1, 40, 1287, 27, 87, 384, 0, 2233, 1, 438, 5, 2260, 0, 0, 3282, 3, 1402, 391, 1636] payload_rows_per_dst=[7876, 8168, 7630, 3180, 7714, 3110, 5484, 1912] dup_rows_per_dst=[4496, 5162, 3966, 348, 5324, 116, 866, 184] combine_rows_per_src=[7876, 8168, 7630, 3180, 7714, 3110, 5484, 1912] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48824 payload_rows_remote=40760 dup_rows=16712 dup_rows_remote=13146 dedup_ratio=0.255005 router_tokens_per_expert=[17, 66, 0, 1957, 42, 213, 2, 4, 1, 15, 690, 40, 5, 1302, 75, 10, 6, 0, 5, 32, 28, 2, 3, 399, 2068, 39, 445, 3698, 0, 2, 3, 341, 3498, 0, 33, 1505, 161, 71, 0, 4, 174, 7, 0, 562, 4015, 0, 76, 981, 31, 78, 622, 0, 1680, 2, 26, 1, 1902, 1, 0, 3116, 14, 347, 1169, 1182] payload_rows_per_dst=[7886, 3710, 6488, 5092, 3424, 8064, 7768, 6392] dup_rows_per_dst=[4112, 566, 694, 704, 124, 3566, 4108, 2838] combine_rows_per_src=[7886, 3710, 6488, 5092, 3424, 8064, 7768, 6392] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267495424 dispatch_meta_bytes=201600 combine_payload_bytes=267495424 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=267697024 ingress_bytes=817997824 link_bytes=817997824 token_bytes=535192448 weight_movement_bytes=550502400 total_nvlink_bytes=1085694848 weight_to_token_ratio=1.028606 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=8960 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.422632753e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=317356032 dispatch_meta_bytes=242344 combine_payload_bytes=317356032 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=317598376 ingress_bytes=427456512 link_bytes=427456512 token_bytes=634954408 weight_movement_bytes=110100480 total_nvlink_bytes=745054888 weight_to_token_ratio=0.173399 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=10368 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.434171789e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=282533888 dispatch_meta_bytes=227352 combine_payload_bytes=282533888 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=282761240 ingress_bytes=502734848 link_bytes=502734848 token_bytes=565295128 weight_movement_bytes=220200960 total_nvlink_bytes=785496088 weight_to_token_ratio=0.389533 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=15872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.743385863e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=303908864 dispatch_meta_bytes=212392 combine_payload_bytes=303908864 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=304121256 ingress_bytes=414009344 link_bytes=414009344 token_bytes=608030120 weight_movement_bytes=110100480 total_nvlink_bytes=718130600 weight_to_token_ratio=0.181077 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=17792 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.200303421e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45972 payload_rows_remote=37786 dup_rows=19564 dup_rows_remote=10000 dedup_ratio=0.298523 router_tokens_per_expert=[0, 1094, 91, 46, 1, 533, 0, 0, 1471, 1840, 0, 0, 0, 136, 0, 1805, 3, 0, 0, 11, 212, 71, 3630, 24, 2, 0, 1380, 21, 0, 1584, 261, 0, 0, 0, 308, 15, 28, 0, 14, 1047, 17, 2721, 0, 1921, 473, 1, 22, 0, 141, 89, 0, 35, 3952, 2527, 0, 1598, 79, 31, 29, 484, 793, 2227, 0, 0] payload_rows_per_dst=[2402, 5492, 7352, 5446, 2668, 7218, 8186, 7208] dup_rows_per_dst=[62, 1402, 550, 1050, 156, 3092, 9564, 3688] combine_rows_per_src=[2402, 5492, 7352, 5446, 2668, 7218, 8186, 7208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46558 payload_rows_remote=40028 dup_rows=18978 dup_rows_remote=16306 dedup_ratio=0.289581 router_tokens_per_expert=[8, 346, 1042, 123, 0, 2601, 0, 3, 1840, 1357, 1, 1, 0, 165, 0, 1374, 2, 112, 0, 1, 5, 43, 3244, 18, 1, 0, 518, 0, 6, 2579, 1694, 2, 0, 0, 0, 0, 59, 0, 66, 2202, 37, 989, 0, 344, 1514, 2, 241, 3, 1, 3, 0, 16, 3535, 2501, 73, 851, 52, 26, 3, 286, 422, 2350, 105, 1] payload_rows_per_dst=[6530, 7692, 5808, 4048, 7930, 7212, 1872, 5466] dup_rows_per_dst=[2672, 6942, 86, 394, 3794, 4050, 16, 1024] combine_rows_per_src=[6530, 7692, 5808, 4048, 7930, 7212, 1872, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45000 payload_rows_remote=38102 dup_rows=20536 dup_rows_remote=17254 dedup_ratio=0.313354 router_tokens_per_expert=[1, 1161, 2440, 2, 0, 1510, 9, 1, 1747, 1087, 5, 2, 1, 3, 1, 2295, 0, 0, 0, 0, 121, 143, 2892, 4, 22, 3, 1574, 0, 0, 3170, 794, 107, 0, 0, 32, 18, 2, 1, 0, 1150, 43, 493, 1, 578, 486, 6, 28, 1, 190, 5, 28, 198, 3767, 2591, 4, 1256, 174, 14, 28, 99, 305, 2133, 41, 1] payload_rows_per_dst=[6280, 4796, 6056, 7710, 2386, 2784, 8090, 6898] dup_rows_per_dst=[948, 896, 264, 3630, 20, 488, 11008, 3282] combine_rows_per_src=[6280, 4796, 6056, 7710, 2386, 2784, 8090, 6898] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=270850048 dispatch_meta_bytes=191144 combine_payload_bytes=270850048 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=271041192 ingress_bytes=601151488 link_bytes=601151488 token_bytes=541891240 weight_movement_bytes=330301440 total_nvlink_bytes=872192680 weight_to_token_ratio=0.609535 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=10112 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.045501310e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=286920704 dispatch_meta_bytes=225336 combine_payload_bytes=286920704 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=287146040 ingress_bytes=507121664 link_bytes=507121664 token_bytes=574066744 weight_movement_bytes=220200960 total_nvlink_bytes=794267704 weight_to_token_ratio=0.383581 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=11648 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.819679809e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48824 payload_rows_remote=41574 dup_rows=16712 dup_rows_remote=13322 dedup_ratio=0.255005 router_tokens_per_expert=[0, 2168, 292, 70, 0, 689, 0, 0, 1766, 945, 0, 0, 0, 444, 0, 1861, 0, 0, 0, 1, 103, 165, 4038, 141, 7, 0, 1627, 13, 0, 2700, 973, 0, 0, 0, 3, 53, 22, 0, 78, 1791, 1, 159, 8, 1019, 1410, 2, 225, 0, 28, 1, 0, 258, 3988, 2748, 0, 969, 17, 41, 1, 1284, 299, 355, 5, 0] payload_rows_per_dst=[5526, 6964, 8086, 7250, 8074, 7076, 2334, 3514] dup_rows_per_dst=[912, 3068, 810, 3390, 3796, 4068, 178, 490] combine_rows_per_src=[5526, 6964, 8086, 7250, 8074, 7076, 2334, 3514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46578 payload_rows_remote=38500 dup_rows=18958 dup_rows_remote=13882 dedup_ratio=0.289276 router_tokens_per_expert=[24, 476, 2385, 19, 0, 2201, 8, 9, 1347, 419, 79, 4, 1, 0, 0, 2135, 10, 38, 0, 1, 116, 1982, 3747, 683, 3, 2, 907, 10, 42, 2881, 1617, 15, 9, 0, 0, 40, 19, 0, 9, 1754, 14, 36, 1, 1551, 184, 11, 23, 18, 33, 1, 0, 325, 3178, 1690, 75, 421, 188, 3, 24, 758, 574, 451, 217, 0] payload_rows_per_dst=[6838, 6168, 8078, 7252, 7452, 5506, 1616, 3668] dup_rows_per_dst=[3406, 1802, 5076, 3702, 2566, 1550, 94, 762] combine_rows_per_src=[6838, 6168, 8078, 7252, 7452, 5506, 1616, 3668] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48972 payload_rows_remote=40910 dup_rows=16564 dup_rows_remote=13026 dedup_ratio=0.252747 router_tokens_per_expert=[21, 67, 0, 1869, 42, 196, 2, 3, 0, 17, 661, 30, 6, 1415, 72, 10, 7, 0, 6, 34, 26, 2, 1, 427, 2073, 47, 443, 3705, 0, 2, 3, 377, 3502, 0, 35, 1483, 168, 76, 0, 1, 158, 8, 0, 517, 4016, 0, 84, 1017, 32, 78, 591, 0, 1687, 0, 29, 1, 1891, 1, 1, 3118, 12, 311, 1169, 1218] payload_rows_per_dst=[7876, 3872, 6514, 5114, 3400, 8062, 7772, 6362] dup_rows_per_dst=[3934, 550, 728, 776, 126, 3538, 4068, 2844] combine_rows_per_src=[7876, 3872, 6514, 5114, 3400, 8062, 7772, 6362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49502 payload_rows_remote=44406 dup_rows=16034 dup_rows_remote=15358 dedup_ratio=0.244659 router_tokens_per_expert=[0, 1274, 1819, 407, 0, 3068, 3, 0, 439, 1736, 0, 30, 17, 4, 1, 465, 1, 0, 13, 0, 39, 219, 3228, 62, 5, 52, 8, 2, 46, 3708, 1837, 2, 7, 1, 378, 11, 0, 177, 699, 305, 25, 66, 9, 1734, 660, 4, 40, 29, 4, 7, 284, 1664, 1962, 1364, 43, 589, 921, 43, 62, 19, 1961, 1156, 29, 30] payload_rows_per_dst=[7630, 5096, 7104, 7340, 5722, 5908, 4444, 6258] dup_rows_per_dst=[5512, 676, 478, 3134, 1358, 1954, 738, 2184] combine_rows_per_src=[7630, 5096, 7104, 7340, 5722, 5908, 4444, 6258] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48132 payload_rows_remote=43676 dup_rows=17404 dup_rows_remote=16100 dedup_ratio=0.265564 router_tokens_per_expert=[0, 101, 0, 885, 1690, 54, 0, 0, 0, 124, 555, 6, 27, 95, 500, 30, 0, 0, 55, 2, 8, 1, 0, 48, 1445, 148, 962, 3354, 0, 0, 0, 620, 4076, 180, 22, 866, 831, 10, 0, 0, 4, 794, 15, 14, 4096, 3, 16, 883, 2, 2, 111, 0, 951, 1, 73, 0, 2746, 0, 0, 2751, 0, 832, 2166, 613] payload_rows_per_dst=[7568, 7254, 5618, 5108, 4456, 4514, 6690, 6924] dup_rows_per_dst=[4600, 2458, 112, 1242, 1304, 1306, 1800, 4582] combine_rows_per_src=[7568, 7254, 5618, 5108, 4456, 4514, 6690, 6924] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45656 payload_rows_remote=37630 dup_rows=19880 dup_rows_remote=15808 dedup_ratio=0.303345 router_tokens_per_expert=[0, 141, 0, 1069, 171, 83, 1, 0, 9, 2, 43, 0, 2, 759, 246, 37, 0, 0, 15, 0, 18, 0, 1, 160, 386, 3, 398, 3373, 0, 1, 3, 2016, 3891, 0, 1, 44, 580, 5, 0, 97, 0, 103, 3, 600, 4088, 0, 159, 563, 83, 28, 910, 0, 1093, 8, 31, 5, 2048, 0, 0, 3113, 0, 3618, 958, 1802] payload_rows_per_dst=[5200, 2106, 6306, 6320, 1418, 8184, 8026, 8096] dup_rows_per_dst=[1624, 90, 308, 2146, 36, 2848, 4072, 8756] combine_rows_per_src=[5200, 2106, 6306, 6320, 1418, 8184, 8026, 8096] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=268398592 dispatch_meta_bytes=215760 combine_payload_bytes=268398592 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=268614352 ingress_bytes=488599552 link_bytes=488599552 token_bytes=537012944 weight_movement_bytes=220200960 total_nvlink_bytes=757213904 weight_to_token_ratio=0.410048 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.040441 gemm_active_groups=11 gemm_peak_group_rows=23936 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.497549817e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48352 payload_rows_remote=44064 dup_rows=17184 dup_rows_remote=16020 dedup_ratio=0.262207 router_tokens_per_expert=[0, 90, 0, 928, 1680, 62, 0, 0, 0, 124, 566, 6, 31, 93, 517, 31, 0, 0, 63, 3, 13, 1, 0, 44, 1484, 148, 994, 3320, 0, 0, 0, 640, 4073, 195, 26, 841, 799, 12, 0, 0, 3, 797, 12, 15, 4095, 4, 14, 891, 5, 2, 112, 0, 944, 1, 62, 0, 2747, 0, 0, 2684, 0, 831, 2193, 572] payload_rows_per_dst=[7556, 7640, 5506, 5206, 4288, 3724, 6858, 7574] dup_rows_per_dst=[4604, 2604, 110, 1326, 1164, 430, 1834, 5112] combine_rows_per_src=[7556, 7640, 5506, 5206, 4288, 3724, 6858, 7574] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50932 payload_rows_remote=45512 dup_rows=14604 dup_rows_remote=13588 dedup_ratio=0.222839 router_tokens_per_expert=[0, 21, 0, 699, 527, 39, 0, 14, 0, 86, 41, 3, 0, 453, 369, 19, 0, 1, 13, 35, 66, 21, 0, 2, 1492, 5, 357, 3500, 1, 0, 4, 1359, 3247, 7, 39, 1913, 478, 0, 0, 82, 203, 53, 0, 1866, 4087, 0, 113, 1848, 76, 3, 668, 0, 1301, 0, 1, 0, 1251, 0, 0, 2891, 0, 1085, 552, 1877] payload_rows_per_dst=[7340, 8182, 6552, 5420, 7766, 6140, 3720, 5812] dup_rows_per_dst=[2260, 1934, 1390, 1016, 3988, 2026, 378, 1612] combine_rows_per_src=[7340, 8182, 6552, 5420, 7766, 6140, 3720, 5812] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=292167680 dispatch_meta_bytes=215624 combine_payload_bytes=292167680 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=292383304 ingress_bytes=512368640 link_bytes=512368640 token_bytes=584550984 weight_movement_bytes=220200960 total_nvlink_bytes=804751944 weight_to_token_ratio=0.376701 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=12544 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.910933351e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5595882984 prefetch_bytes=990904320 reduce_bytes=990904320 weight_movement_bytes=1981808640 egress_bytes=2799015400 ingress_bytes=4778676224 link_bytes=4778676224 total_nvlink_bytes=7577691624 weight_to_token_ratio=0.354155 prefetch_share_of_ingress=0.207360 dedup_ratio=0.295206 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=9 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040074 gemm_inner_pad_upper_bound=12319 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.310903911e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=273115136 dispatch_meta_bytes=221424 combine_payload_bytes=273115136 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=273336560 ingress_bytes=383215616 link_bytes=383215616 token_bytes=546451696 weight_movement_bytes=110100480 total_nvlink_bytes=656552176 weight_to_token_ratio=0.201483 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=11904 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.664749844e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=298002432 dispatch_meta_bytes=219584 combine_payload_bytes=298002432 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=298222016 ingress_bytes=518203392 link_bytes=518203392 token_bytes=596224448 weight_movement_bytes=220200960 total_nvlink_bytes=816425408 weight_to_token_ratio=0.369326 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=17152 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.012409285e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=293242880 dispatch_meta_bytes=215744 combine_payload_bytes=293242880 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=293458624 ingress_bytes=513443840 link_bytes=513443840 token_bytes=586701504 weight_movement_bytes=220200960 total_nvlink_bytes=806902464 weight_to_token_ratio=0.375320 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=12800 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.929632848e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=275968000 dispatch_meta_bytes=209528 combine_payload_bytes=275968000 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=276177528 ingress_bytes=496168960 link_bytes=496168960 token_bytes=552145528 weight_movement_bytes=220200960 total_nvlink_bytes=772346488 weight_to_token_ratio=0.398810 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=27904 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.629194272e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=318302208 dispatch_meta_bytes=239056 combine_payload_bytes=318302208 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=318541264 ingress_bytes=538503168 link_bytes=538503168 token_bytes=636843472 weight_movement_bytes=220200960 total_nvlink_bytes=857044432 weight_to_token_ratio=0.345769 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=11776 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.365455777e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=313069568 dispatch_meta_bytes=239104 combine_payload_bytes=313069568 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=313308672 ingress_bytes=533270528 link_bytes=533270528 token_bytes=626378240 weight_movement_bytes=220200960 total_nvlink_bytes=846579200 weight_to_token_ratio=0.351546 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=14592 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.274451561e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45886 payload_rows_remote=37866 dup_rows=19650 dup_rows_remote=15594 dedup_ratio=0.299835 router_tokens_per_expert=[0, 140, 0, 1018, 166, 75, 1, 0, 10, 2, 43, 0, 1, 780, 223, 26, 0, 0, 13, 0, 21, 0, 0, 183, 416, 4, 379, 3390, 0, 1, 4, 2069, 3876, 0, 1, 39, 587, 4, 0, 95, 0, 95, 2, 468, 4084, 1, 160, 567, 83, 26, 900, 0, 1099, 10, 38, 6, 2044, 0, 0, 3170, 0, 3632, 1004, 1812] payload_rows_per_dst=[5772, 2066, 6414, 5910, 1422, 8184, 8020, 8098] dup_rows_per_dst=[1808, 104, 360, 1836, 30, 2570, 4056, 8886] combine_rows_per_src=[5772, 2066, 6414, 5910, 1422, 8184, 8020, 8098] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44576 payload_rows_remote=41672 dup_rows=20960 dup_rows_remote=20724 dedup_ratio=0.319824 router_tokens_per_expert=[0, 22, 0, 205, 16, 409, 1, 3, 0, 42, 116, 5, 15, 1360, 22, 9, 0, 0, 252, 2, 150, 46, 41, 29, 1120, 125, 1116, 914, 0, 0, 3, 578, 3541, 13, 336, 1585, 810, 3, 0, 13, 538, 17, 0, 2460, 4049, 0, 10, 1124, 9, 56, 24, 0, 1087, 14, 0, 1, 2511, 0, 0, 3637, 0, 877, 801, 2651] payload_rows_per_dst=[2904, 8128, 7916, 7272, 7936, 6452, 2366, 1602] dup_rows_per_dst=[236, 3108, 5420, 3914, 6420, 1846, 16, 0] combine_rows_per_src=[2904, 8128, 7916, 7272, 7936, 6452, 2366, 1602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6142644328 prefetch_bytes=990904320 reduce_bytes=880803840 weight_movement_bytes=1871708160 egress_bytes=3072460904 ingress_bytes=4941891584 link_bytes=4941891584 total_nvlink_bytes=8014352488 weight_to_token_ratio=0.304707 prefetch_share_of_ingress=0.200511 dedup_ratio=0.249399 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=8 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.039706 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.594762266e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6181266024 prefetch_bytes=1211105280 reduce_bytes=770703360 weight_movement_bytes=1981808640 egress_bytes=3091786344 ingress_bytes=5071288320 link_bytes=5071288320 total_nvlink_bytes=8163074664 weight_to_token_ratio=0.320615 prefetch_share_of_ingress=0.238816 dedup_ratio=0.249380 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=7 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040074 gemm_inner_pad_upper_bound=12827 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.819804472e-05 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45038 payload_rows_remote=37294 dup_rows=20498 dup_rows_remote=12994 dedup_ratio=0.312775 router_tokens_per_expert=[1, 111, 0, 2516, 31, 350, 0, 0, 0, 1, 173, 4, 0, 376, 110, 5, 11, 0, 23, 3, 43, 6, 0, 137, 356, 185, 2213, 3258, 0, 0, 0, 1168, 3504, 20, 46, 690, 830, 7, 0, 10, 8, 83, 4, 48, 4044, 3, 9, 915, 21, 50, 303, 1, 1488, 1, 80, 1, 1988, 1, 0, 1897, 0, 2611, 1631, 1393] payload_rows_per_dst=[5396, 1258, 4010, 7894, 2948, 8128, 7660, 7744] dup_rows_per_dst=[622, 80, 230, 6466, 258, 2100, 3238, 7504] combine_rows_per_src=[5396, 1258, 4010, 7894, 2948, 8128, 7660, 7744] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=298704896 dispatch_meta_bytes=249584 combine_payload_bytes=298704896 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=298954480 ingress_bytes=408805376 link_bytes=408805376 token_bytes=597659376 weight_movement_bytes=110100480 total_nvlink_bytes=707759856 weight_to_token_ratio=0.184219 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=18432 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.109797858e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=271423488 dispatch_meta_bytes=213840 combine_payload_bytes=271423488 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=271637328 ingress_bytes=381523968 link_bytes=381523968 token_bytes=543060816 weight_movement_bytes=110100480 total_nvlink_bytes=653161296 weight_to_token_ratio=0.202741 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=14080 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.635329303e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51224 payload_rows_remote=45802 dup_rows=14312 dup_rows_remote=13304 dedup_ratio=0.218384 router_tokens_per_expert=[0, 21, 0, 660, 498, 36, 0, 13, 0, 79, 28, 1, 0, 494, 347, 17, 0, 0, 12, 31, 70, 18, 0, 2, 1450, 3, 345, 3509, 1, 0, 4, 1412, 3235, 5, 33, 1930, 446, 0, 0, 78, 196, 50, 0, 1839, 4088, 0, 119, 1873, 90, 3, 666, 0, 1314, 0, 1, 0, 1324, 0, 0, 2957, 0, 1102, 499, 1869] payload_rows_per_dst=[7358, 8184, 6480, 5422, 7716, 6708, 3766, 5590] dup_rows_per_dst=[2116, 1924, 952, 1008, 3738, 2842, 382, 1350] combine_rows_per_src=[7358, 8184, 6480, 5422, 7716, 6708, 3766, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45106 payload_rows_remote=37484 dup_rows=20430 dup_rows_remote=16422 dedup_ratio=0.311737 router_tokens_per_expert=[83, 22, 2, 2031, 105, 246, 0, 13, 1, 495, 219, 1, 0, 1861, 53, 20, 0, 2, 24, 3, 177, 11, 3, 26, 278, 188, 20, 3569, 2, 1, 0, 292, 3194, 2, 10, 376, 1447, 61, 0, 66, 2, 29, 4, 232, 4062, 1, 46, 1256, 25, 84, 371, 0, 2277, 1, 455, 3, 2284, 0, 0, 3285, 4, 1401, 366, 1676] payload_rows_per_dst=[7826, 8172, 7622, 3958, 7698, 3032, 5498, 1300] dup_rows_per_dst=[4316, 5252, 4008, 480, 5086, 108, 934, 246] combine_rows_per_src=[7826, 8172, 7622, 3958, 7698, 3032, 5498, 1300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49014 payload_rows_remote=40834 dup_rows=16522 dup_rows_remote=11576 dedup_ratio=0.252106 router_tokens_per_expert=[126, 3, 1, 170, 299, 39, 0, 2, 2, 147, 177, 6, 22, 2028, 6, 110, 2, 13, 554, 0, 66, 274, 150, 12, 323, 111, 254, 962, 34, 5, 6, 2919, 3564, 688, 38, 510, 1291, 34, 1, 77, 6, 15, 63, 1139, 4065, 8, 194, 1648, 203, 16, 113, 13, 3652, 0, 535, 103, 225, 13, 0, 3280, 28, 296, 1211, 916] payload_rows_per_dst=[3030, 8180, 7380, 6914, 8094, 5172, 7744, 2500] dup_rows_per_dst=[174, 4946, 1772, 2222, 4904, 974, 1526, 4] combine_rows_per_src=[3030, 8180, 7380, 6914, 8094, 5172, 7744, 2500] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48914 payload_rows_remote=40734 dup_rows=16622 dup_rows_remote=11840 dedup_ratio=0.253632 router_tokens_per_expert=[135, 3, 1, 165, 321, 40, 0, 1, 2, 138, 163, 6, 22, 1969, 7, 108, 2, 13, 573, 0, 76, 287, 154, 9, 343, 117, 279, 950, 36, 5, 6, 2849, 3568, 665, 38, 489, 1298, 28, 1, 85, 6, 14, 68, 1229, 4066, 7, 183, 1683, 203, 13, 118, 6, 3652, 0, 510, 89, 217, 10, 0, 3226, 29, 291, 1296, 900] payload_rows_per_dst=[3038, 8180, 7312, 6854, 8100, 5586, 7732, 2112] dup_rows_per_dst=[194, 4782, 1802, 2216, 4826, 1352, 1450, 0] combine_rows_per_src=[3038, 8180, 7312, 6854, 8100, 5586, 7732, 2112] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=269731840 dispatch_meta_bytes=213752 combine_payload_bytes=269731840 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=269945592 ingress_bytes=379832320 link_bytes=379832320 token_bytes=539677432 weight_movement_bytes=110100480 total_nvlink_bytes=649777912 weight_to_token_ratio=0.204012 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=14208 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.605908762e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5900014776 prefetch_bytes=1321205760 reduce_bytes=440401920 weight_movement_bytes=1761607680 egress_bytes=2951142584 ingress_bytes=4710479872 link_bytes=4710479872 total_nvlink_bytes=7661622456 weight_to_token_ratio=0.298577 prefetch_share_of_ingress=0.280482 dedup_ratio=0.277197 prefetch_slots=24 unused_prefetch_slots=136 slot_utilization=0.150000 reduce_slots=4 remote_demand=12 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040441 gemm_inner_pad_upper_bound=12827 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.192299238e-05 prefetch_bytes_per_gemm_flop=2.297794118e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=315850752 dispatch_meta_bytes=240336 combine_payload_bytes=315850752 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=316091088 ingress_bytes=536051712 link_bytes=536051712 token_bytes=631941840 weight_movement_bytes=220200960 total_nvlink_bytes=852142800 weight_to_token_ratio=0.348451 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=14464 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.322820925e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43464 payload_rows_remote=40548 dup_rows=22072 dup_rows_remote=21824 dedup_ratio=0.336792 router_tokens_per_expert=[0, 19, 0, 230, 16, 440, 1, 3, 0, 39, 119, 6, 17, 1306, 20, 12, 1, 0, 282, 3, 159, 50, 45, 27, 1179, 123, 1108, 873, 0, 0, 3, 567, 3542, 10, 363, 1515, 814, 3, 0, 14, 502, 14, 0, 2562, 4046, 0, 12, 1173, 8, 60, 25, 0, 1049, 12, 1, 1, 2469, 0, 0, 3611, 0, 877, 810, 2627] payload_rows_per_dst=[2916, 8124, 7904, 7312, 7962, 6954, 2292, 0] dup_rows_per_dst=[248, 3006, 5390, 3902, 6314, 3192, 20, 0] combine_rows_per_src=[2916, 8124, 7904, 7312, 7962, 6954, 2292, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6115815208 prefetch_bytes=990904320 reduce_bytes=880803840 weight_movement_bytes=1871708160 egress_bytes=3059035944 ingress_bytes=4928487424 link_bytes=4928487424 total_nvlink_bytes=7987523368 weight_to_token_ratio=0.306044 prefetch_share_of_ingress=0.201056 dedup_ratio=0.248679 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=8 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040074 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.571450227e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=326230016 dispatch_meta_bytes=236400 combine_payload_bytes=326230016 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=326466416 ingress_bytes=546430976 link_bytes=546430976 token_bytes=652696432 weight_movement_bytes=220200960 total_nvlink_bytes=872897392 weight_to_token_ratio=0.337371 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=9344 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.503333397e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5758532624 prefetch_bytes=990904320 reduce_bytes=660602880 weight_movement_bytes=1651507200 egress_bytes=2880394256 ingress_bytes=4529645568 link_bytes=4529645568 total_nvlink_bytes=7410039824 weight_to_token_ratio=0.286793 prefetch_share_of_ingress=0.218760 dedup_ratio=0.291168 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=6 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040441 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.877798641e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5841581336 prefetch_bytes=990904320 reduce_bytes=990904320 weight_movement_bytes=1981808640 egress_bytes=2921911576 ingress_bytes=4901478400 link_bytes=4901478400 total_nvlink_bytes=7823389976 weight_to_token_ratio=0.339259 prefetch_share_of_ingress=0.202164 dedup_ratio=0.273560 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=9 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.524477092e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267323392 dispatch_meta_bytes=201152 combine_payload_bytes=267323392 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=267524544 ingress_bytes=817825792 link_bytes=817825792 token_bytes=534847936 weight_movement_bytes=550502400 total_nvlink_bytes=1085350336 weight_to_token_ratio=1.029269 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=8960 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.422333562e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=291981312 dispatch_meta_bytes=210296 combine_payload_bytes=291981312 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=292191608 ingress_bytes=402081792 link_bytes=402081792 token_bytes=584172920 weight_movement_bytes=110100480 total_nvlink_bytes=694273400 weight_to_token_ratio=0.188472 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=20096 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.992863674e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5832568088 prefetch_bytes=1101004800 reduce_bytes=880803840 weight_movement_bytes=1981808640 egress_bytes=2917399832 ingress_bytes=4896976896 link_bytes=4896976896 total_nvlink_bytes=7814376728 weight_to_token_ratio=0.339783 prefetch_share_of_ingress=0.224834 dedup_ratio=0.274109 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=8 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.516648236e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6222040288 prefetch_bytes=1101004800 reduce_bytes=880803840 weight_movement_bytes=1981808640 egress_bytes=3112174816 ingress_bytes=5091674112 link_bytes=5091674112 total_nvlink_bytes=8203848928 weight_to_token_ratio=0.318514 prefetch_share_of_ingress=0.216236 dedup_ratio=0.244998 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=8 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.039706 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.855258717e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=290648064 dispatch_meta_bytes=249488 combine_payload_bytes=290648064 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=290897552 ingress_bytes=400748544 link_bytes=400748544 token_bytes=581545616 weight_movement_bytes=110100480 total_nvlink_bytes=691646096 weight_to_token_ratio=0.189324 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=17920 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.969676298e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6131123824 prefetch_bytes=770703360 reduce_bytes=1651507200 weight_movement_bytes=2422210560 egress_bytes=3066689136 ingress_bytes=5486645248 link_bytes=5486645248 total_nvlink_bytes=8553334384 weight_to_token_ratio=0.395068 prefetch_share_of_ingress=0.140469 dedup_ratio=0.249069 prefetch_slots=14 unused_prefetch_slots=146 slot_utilization=0.087500 reduce_slots=15 remote_demand=7 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=12192 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.542178484e-05 prefetch_bytes_per_gemm_flop=1.340379902e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5805522856 prefetch_bytes=770703360 reduce_bytes=990904320 weight_movement_bytes=1761607680 egress_bytes=2903887784 ingress_bytes=4663242752 link_bytes=4663242752 total_nvlink_bytes=7567130536 weight_to_token_ratio=0.303437 prefetch_share_of_ingress=0.165272 dedup_ratio=0.285568 prefetch_slots=14 unused_prefetch_slots=146 slot_utilization=0.087500 reduce_slots=9 remote_demand=7 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042647 gemm_inner_pad_upper_bound=12319 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.110146117e-05 prefetch_bytes_per_gemm_flop=1.340379902e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5928453872 prefetch_bytes=1211105280 reduce_bytes=1541406720 weight_movement_bytes=2752512000 egress_bytes=2965331696 ingress_bytes=5715634176 link_bytes=5715634176 total_nvlink_bytes=8680965872 weight_to_token_ratio=0.464288 prefetch_share_of_ingress=0.211893 dedup_ratio=0.265759 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=14 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.940427892e-05 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none - [2026-08-26 15:11:32] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 366.6 | throughput per GPU (TFLOP/s/GPU): 336.8 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.534207E+00 | loss scale: 1.0 | grad norm: 32.010 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45416 payload_rows_remote=39722 dup_rows=20120 dup_rows_remote=17578 dedup_ratio=0.307007 router_tokens_per_expert=[348, 149, 470, 13, 22, 39, 1128, 469, 767, 274, 194, 495, 246, 211, 4, 463, 148, 176, 1278, 126, 198, 200, 104, 160, 679, 801, 326, 1012, 84, 773, 427, 25, 449, 139, 406, 1054, 143, 21, 123, 791, 507, 1389, 1114, 66, 1728, 845, 291, 532, 484, 1415, 417, 53, 845, 1201, 1403, 1210, 1030, 748, 95, 88, 267, 1503, 393, 209] payload_rows_per_dst=[5314, 5644, 5694, 5342, 5618, 6284, 5668, 5852] dup_rows_per_dst=[2382, 2470, 2542, 2038, 2658, 3204, 2012, 2814] combine_rows_per_src=[5314, 5644, 5694, 5342, 5618, 6284, 5668, 5852] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45674 payload_rows_remote=39912 dup_rows=19862 dup_rows_remote=16858 dedup_ratio=0.303070 router_tokens_per_expert=[324, 108, 449, 22, 23, 39, 1456, 414, 770, 723, 191, 147, 404, 202, 6, 417, 215, 159, 1081, 126, 245, 125, 44, 67, 680, 844, 176, 1017, 253, 1034, 375, 17, 542, 61, 576, 1767, 92, 4, 65, 223, 875, 737, 1225, 69, 1649, 588, 397, 458, 1222, 945, 514, 31, 615, 1056, 2287, 1246, 883, 782, 267, 55, 363, 616, 315, 90] payload_rows_per_dst=[5604, 6780, 5428, 5070, 6196, 4660, 5762, 6174] dup_rows_per_dst=[2558, 3514, 1994, 1688, 2498, 1588, 3004, 3018] combine_rows_per_src=[5604, 6780, 5428, 5070, 6196, 4660, 5762, 6174] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45300 payload_rows_remote=39348 dup_rows=20236 dup_rows_remote=17320 dedup_ratio=0.308777 router_tokens_per_expert=[383, 66, 394, 43, 131, 18, 1168, 621, 555, 396, 245, 375, 245, 289, 31, 392, 121, 200, 1355, 93, 372, 87, 122, 28, 1133, 1002, 194, 1364, 64, 1265, 397, 68, 480, 84, 656, 1287, 280, 39, 44, 200, 737, 1366, 1092, 230, 1304, 740, 135, 359, 396, 1192, 727, 35, 790, 1293, 1582, 1397, 803, 532, 325, 44, 309, 744, 238, 111] payload_rows_per_dst=[5688, 5678, 5422, 5664, 5952, 6268, 6048, 4580] dup_rows_per_dst=[2754, 2542, 1942, 2582, 2916, 3050, 2818, 1632] combine_rows_per_src=[5688, 5678, 5422, 5664, 5952, 6268, 6048, 4580] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45212 payload_rows_remote=39756 dup_rows=20324 dup_rows_remote=17472 dedup_ratio=0.310120 router_tokens_per_expert=[328, 111, 457, 16, 105, 54, 1121, 525, 1027, 485, 179, 169, 191, 217, 22, 444, 148, 202, 1373, 106, 331, 65, 264, 88, 789, 568, 223, 682, 266, 1111, 591, 54, 377, 393, 257, 1480, 107, 22, 114, 627, 701, 907, 884, 112, 1876, 907, 724, 430, 560, 1394, 325, 89, 551, 689, 1756, 1823, 720, 508, 270, 51, 329, 1050, 285, 138] payload_rows_per_dst=[6012, 6112, 6112, 5284, 5516, 5456, 5386, 5334] dup_rows_per_dst=[3068, 2868, 2794, 1920, 2602, 2852, 1830, 2390] combine_rows_per_src=[6012, 6112, 6112, 5284, 5516, 5456, 5386, 5334] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45238 payload_rows_remote=39516 dup_rows=20298 dup_rows_remote=17632 dedup_ratio=0.309723 router_tokens_per_expert=[237, 89, 704, 78, 172, 33, 2032, 381, 330, 486, 219, 266, 169, 117, 14, 325, 190, 300, 1325, 117, 250, 102, 172, 25, 828, 1312, 260, 1056, 111, 1186, 478, 19, 364, 65, 721, 1512, 134, 10, 59, 604, 633, 597, 956, 47, 1791, 425, 370, 289, 734, 1248, 839, 20, 753, 1244, 1725, 1055, 1109, 563, 178, 24, 270, 561, 411, 74] payload_rows_per_dst=[6274, 5490, 5844, 5722, 6088, 4884, 6200, 4736] dup_rows_per_dst=[3288, 1812, 2700, 2666, 2962, 1750, 3476, 1644] combine_rows_per_src=[6274, 5490, 5844, 5722, 6088, 4884, 6200, 4736] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44462 payload_rows_remote=37858 dup_rows=21074 dup_rows_remote=17094 dedup_ratio=0.321564 router_tokens_per_expert=[541, 107, 585, 22, 15, 22, 1219, 259, 731, 400, 127, 280, 267, 196, 4, 259, 287, 40, 1330, 99, 362, 375, 118, 25, 590, 1125, 167, 1309, 188, 829, 319, 12, 493, 136, 763, 1429, 63, 0, 58, 412, 781, 728, 1332, 52, 1628, 411, 399, 591, 542, 1046, 286, 61, 524, 1204, 1328, 2332, 1016, 642, 166, 42, 521, 1019, 458, 96] payload_rows_per_dst=[4622, 4512, 4784, 4836, 6226, 5466, 7412, 6604] dup_rows_per_dst=[1002, 1528, 1534, 1624, 3100, 2668, 5638, 3980] combine_rows_per_src=[4622, 4512, 4784, 4836, 6226, 5466, 7412, 6604] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44560 payload_rows_remote=38916 dup_rows=20976 dup_rows_remote=18516 dedup_ratio=0.320068 router_tokens_per_expert=[359, 204, 1106, 16, 51, 10, 1326, 697, 946, 543, 232, 171, 172, 168, 14, 399, 111, 203, 1577, 282, 327, 238, 114, 94, 703, 1108, 284, 1126, 287, 829, 390, 40, 484, 52, 572, 1060, 111, 5, 37, 475, 346, 1392, 963, 162, 1350, 664, 280, 429, 440, 914, 503, 42, 510, 649, 1407, 1077, 1168, 1038, 213, 78, 804, 791, 529, 96] payload_rows_per_dst=[5982, 5644, 5822, 6000, 5334, 5862, 3720, 6196] dup_rows_per_dst=[3710, 2460, 2770, 3362, 2258, 2610, 568, 3238] combine_rows_per_src=[5982, 5644, 5822, 6000, 5334, 5862, 3720, 6196] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=328308736 dispatch_meta_bytes=236424 combine_payload_bytes=328308736 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=328545160 ingress_bytes=548509696 link_bytes=548509696 token_bytes=656853896 weight_movement_bytes=220200960 total_nvlink_bytes=877054856 weight_to_token_ratio=0.335236 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=9472 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.539485757e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=268685312 dispatch_meta_bytes=215624 combine_payload_bytes=268685312 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=268900936 ingress_bytes=488886272 link_bytes=488886272 token_bytes=537586248 weight_movement_bytes=220200960 total_nvlink_bytes=757787208 weight_to_token_ratio=0.409610 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.036765 gemm_active_groups=11 gemm_peak_group_rows=24192 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.502536350e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=292698112 dispatch_meta_bytes=209640 combine_payload_bytes=292698112 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=292907752 ingress_bytes=402798592 link_bytes=402798592 token_bytes=585605864 weight_movement_bytes=110100480 total_nvlink_bytes=695706344 weight_to_token_ratio=0.188011 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=19840 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.005330005e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5590847616 prefetch_bytes=880803840 reduce_bytes=1211105280 weight_movement_bytes=2091909120 egress_bytes=2796503168 ingress_bytes=4886253568 link_bytes=4886253568 total_nvlink_bytes=7682756736 weight_to_token_ratio=0.374167 prefetch_share_of_ingress=0.180262 dedup_ratio=0.298193 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=11 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040074 gemm_inner_pad_upper_bound=12319 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.497998605e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6143088672 prefetch_bytes=770703360 reduce_bytes=1541406720 weight_movement_bytes=2312110080 egress_bytes=3072675872 ingress_bytes=5382522880 link_bytes=5382522880 total_nvlink_bytes=8455198752 weight_to_token_ratio=0.376376 prefetch_share_of_ingress=0.143186 dedup_ratio=0.249597 prefetch_slots=14 unused_prefetch_slots=146 slot_utilization=0.087500 reduce_slots=14 remote_demand=7 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040441 gemm_inner_pad_upper_bound=12192 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.361092561e-05 prefetch_bytes_per_gemm_flop=1.340379902e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5871628056 prefetch_bytes=1321205760 reduce_bytes=440401920 weight_movement_bytes=1761607680 egress_bytes=2936948504 ingress_bytes=4696287232 link_bytes=4696287232 total_nvlink_bytes=7633235736 weight_to_token_ratio=0.300020 prefetch_share_of_ingress=0.281330 dedup_ratio=0.280066 prefetch_slots=24 unused_prefetch_slots=136 slot_utilization=0.150000 reduce_slots=4 remote_demand=12 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040441 gemm_inner_pad_upper_bound=12827 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.167615903e-05 prefetch_bytes_per_gemm_flop=2.297794118e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5886616224 prefetch_bytes=1101004800 reduce_bytes=1761607680 weight_movement_bytes=2862612480 egress_bytes=2944424608 ingress_bytes=5804804096 link_bytes=5804804096 total_nvlink_bytes=8749228704 weight_to_token_ratio=0.486292 prefetch_share_of_ingress=0.189671 dedup_ratio=0.273120 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=16 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=12319 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.009550905e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none + [2026-08-26 15:59:18] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 362.2 | throughput per GPU (TFLOP/s/GPU): 341.0 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.533396E+00 | loss scale: 1.0 | grad norm: 31.996 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45366 payload_rows_remote=39410 dup_rows=20170 dup_rows_remote=17242 dedup_ratio=0.307770 router_tokens_per_expert=[383, 67, 382, 45, 115, 18, 1163, 613, 542, 393, 243, 378, 242, 285, 31, 393, 120, 203, 1351, 96, 376, 87, 123, 27, 1148, 979, 195, 1373, 65, 1267, 393, 68, 476, 84, 664, 1282, 289, 39, 45, 190, 738, 1367, 1091, 227, 1284, 764, 132, 359, 407, 1145, 712, 35, 793, 1311, 1600, 1404, 804, 557, 327, 44, 316, 768, 241, 109] payload_rows_per_dst=[5690, 5696, 5394, 5656, 5956, 6282, 6048, 4644] dup_rows_per_dst=[2690, 2518, 1940, 2574, 2928, 3074, 2758, 1688] combine_rows_per_src=[5690, 5696, 5394, 5656, 5956, 6282, 6048, 4644] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44750 payload_rows_remote=38400 dup_rows=20786 dup_rows_remote=17020 dedup_ratio=0.317169 router_tokens_per_expert=[541, 107, 565, 24, 15, 23, 1216, 254, 712, 402, 123, 278, 256, 198, 5, 265, 281, 40, 1331, 103, 362, 369, 125, 22, 591, 1100, 165, 1317, 186, 840, 321, 11, 494, 141, 763, 1427, 66, 0, 59, 400, 776, 734, 1341, 54, 1597, 423, 377, 585, 549, 1021, 286, 62, 525, 1225, 1339, 2339, 1012, 675, 164, 43, 531, 1052, 464, 96] payload_rows_per_dst=[4768, 4538, 4928, 4818, 6232, 6456, 6660, 6350] dup_rows_per_dst=[1184, 1534, 1666, 1610, 3102, 3820, 4104, 3766] combine_rows_per_src=[4768, 4538, 4928, 4818, 6232, 6456, 6660, 6350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45784 payload_rows_remote=40840 dup_rows=19752 dup_rows_remote=17732 dedup_ratio=0.301392 router_tokens_per_expert=[321, 106, 433, 23, 23, 39, 1449, 414, 755, 730, 189, 150, 396, 197, 6, 420, 214, 161, 1085, 134, 249, 122, 47, 66, 681, 811, 176, 1021, 247, 1036, 372, 18, 541, 64, 576, 1765, 104, 4, 65, 213, 868, 732, 1226, 70, 1626, 615, 385, 459, 1248, 909, 501, 31, 625, 1077, 2307, 1247, 873, 822, 265, 54, 367, 629, 318, 91] payload_rows_per_dst=[5574, 6786, 5426, 5044, 6194, 5920, 4944, 5896] dup_rows_per_dst=[2536, 3514, 1982, 1638, 2512, 2790, 2020, 2760] combine_rows_per_src=[5574, 6786, 5426, 5044, 6194, 5920, 4944, 5896] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45128 payload_rows_remote=39192 dup_rows=20408 dup_rows_remote=17070 dedup_ratio=0.311401 router_tokens_per_expert=[322, 110, 448, 15, 106, 53, 1118, 525, 1020, 482, 177, 167, 183, 213, 22, 449, 150, 211, 1368, 107, 335, 65, 267, 83, 795, 554, 220, 692, 264, 1126, 588, 55, 370, 399, 257, 1466, 111, 22, 118, 610, 698, 904, 889, 116, 1843, 919, 684, 427, 579, 1362, 321, 89, 552, 699, 1773, 1834, 727, 551, 271, 49, 332, 1070, 293, 143] payload_rows_per_dst=[6016, 6106, 6088, 5282, 5522, 5936, 4774, 5404] dup_rows_per_dst=[3046, 2866, 2770, 1922, 2568, 3338, 1320, 2578] combine_rows_per_src=[6016, 6106, 6088, 5282, 5522, 5936, 4774, 5404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45382 payload_rows_remote=39708 dup_rows=20154 dup_rows_remote=17654 dedup_ratio=0.307526 router_tokens_per_expert=[345, 145, 455, 14, 22, 39, 1129, 473, 758, 273, 190, 489, 244, 211, 5, 467, 150, 180, 1267, 127, 199, 195, 106, 158, 684, 771, 322, 1019, 82, 768, 424, 25, 445, 138, 402, 1051, 161, 21, 124, 777, 518, 1385, 1108, 67, 1705, 862, 282, 533, 495, 1393, 414, 53, 843, 1215, 1416, 1219, 1029, 777, 95, 89, 270, 1527, 407, 211] payload_rows_per_dst=[5320, 5654, 5674, 5134, 5608, 6006, 6060, 5926] dup_rows_per_dst=[2362, 2452, 2500, 1790, 2668, 2732, 2766, 2884] combine_rows_per_src=[5320, 5654, 5674, 5134, 5608, 6006, 6060, 5926] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45310 payload_rows_remote=39598 dup_rows=20226 dup_rows_remote=17590 dedup_ratio=0.308624 router_tokens_per_expert=[233, 88, 674, 78, 162, 35, 2030, 381, 322, 491, 214, 263, 162, 115, 14, 330, 193, 306, 1325, 117, 253, 102, 167, 26, 829, 1284, 255, 1070, 110, 1199, 479, 18, 362, 70, 715, 1510, 143, 11, 60, 586, 643, 591, 953, 49, 1754, 438, 358, 284, 743, 1212, 831, 20, 762, 1258, 1749, 1079, 1114, 598, 179, 26, 279, 569, 423, 74] payload_rows_per_dst=[6260, 5520, 5832, 5712, 6086, 4882, 6184, 4834] dup_rows_per_dst=[3260, 1800, 2654, 2636, 2968, 1750, 3468, 1690] combine_rows_per_src=[6260, 5520, 5832, 5712, 6086, 4882, 6184, 4834] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44820 payload_rows_remote=39172 dup_rows=20716 dup_rows_remote=18258 dedup_ratio=0.316101 router_tokens_per_expert=[354, 201, 1083, 16, 49, 10, 1323, 696, 928, 545, 226, 173, 169, 166, 14, 404, 108, 207, 1572, 291, 337, 240, 114, 90, 701, 1077, 281, 1123, 284, 835, 391, 42, 476, 55, 577, 1054, 125, 7, 40, 466, 350, 1380, 968, 162, 1328, 687, 261, 423, 449, 882, 487, 45, 503, 665, 1428, 1081, 1169, 1065, 211, 77, 831, 827, 541, 98] payload_rows_per_dst=[5964, 5648, 5802, 6080, 5330, 4898, 4844, 6254] dup_rows_per_dst=[3662, 2458, 2772, 3338, 2256, 1628, 1218, 3384] combine_rows_per_src=[5964, 5648, 5802, 6080, 5330, 4898, 4844, 6254] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=284727296 dispatch_meta_bytes=229200 combine_payload_bytes=284727296 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=284956496 ingress_bytes=394827776 link_bytes=394827776 token_bytes=569683792 weight_movement_bytes=110100480 total_nvlink_bytes=679784272 weight_to_token_ratio=0.193266 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=12032 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.866704405e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=286089216 dispatch_meta_bytes=227080 combine_payload_bytes=286089216 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=286316296 ingress_bytes=726491136 link_bytes=726491136 token_bytes=572405512 weight_movement_bytes=440401920 total_nvlink_bytes=1012807432 weight_to_token_ratio=0.769388 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=8320 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.263487573e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43774 payload_rows_remote=38664 dup_rows=21762 dup_rows_remote=19668 dedup_ratio=0.332062 router_tokens_per_expert=[311, 146, 737, 24, 23, 34, 1167, 381, 308, 781, 357, 402, 174, 224, 9, 463, 112, 253, 1712, 117, 788, 85, 165, 100, 976, 525, 290, 1636, 212, 705, 613, 33, 120, 53, 369, 1053, 306, 38, 137, 727, 625, 925, 1161, 111, 1722, 425, 265, 661, 1236, 866, 142, 64, 605, 1271, 1505, 779, 995, 894, 391, 66, 602, 309, 298, 184] payload_rows_per_dst=[5110, 5636, 6310, 6926, 4186, 4532, 5846, 5228] dup_rows_per_dst=[2094, 2810, 3798, 5376, 1420, 1492, 2522, 2250] combine_rows_per_src=[5110, 5636, 6310, 6926, 4186, 4532, 5846, 5228] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=282490880 dispatch_meta_bytes=226608 combine_payload_bytes=282490880 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=282717488 ingress_bytes=392591360 link_bytes=392591360 token_bytes=565208368 weight_movement_bytes=110100480 total_nvlink_bytes=675308848 weight_to_token_ratio=0.194796 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=10624 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.827809452e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=275251200 dispatch_meta_bytes=221680 combine_payload_bytes=275251200 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=275472880 ingress_bytes=385351680 link_bytes=385351680 token_bytes=550724080 weight_movement_bytes=110100480 total_nvlink_bytes=660824560 weight_to_token_ratio=0.199919 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=7808 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.701899510e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=292741120 dispatch_meta_bytes=234288 combine_payload_bytes=292741120 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=292975408 ingress_bytes=733143040 link_bytes=733143040 token_bytes=585716528 weight_movement_bytes=440401920 total_nvlink_bytes=1026118448 weight_to_token_ratio=0.751903 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=8832 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.275056328e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=282046464 dispatch_meta_bytes=226672 combine_payload_bytes=282046464 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=282273136 ingress_bytes=392146944 link_bytes=392146944 token_bytes=564319600 weight_movement_bytes=110100480 total_nvlink_bytes=674420080 weight_to_token_ratio=0.195103 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=10752 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.820080327e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43778 payload_rows_remote=38644 dup_rows=21758 dup_rows_remote=19614 dedup_ratio=0.332001 router_tokens_per_expert=[309, 153, 767, 24, 23, 35, 1177, 380, 322, 787, 364, 405, 178, 226, 10, 459, 113, 243, 1718, 113, 781, 93, 169, 103, 965, 538, 291, 1634, 213, 700, 610, 28, 121, 50, 363, 1062, 297, 37, 137, 739, 622, 933, 1155, 114, 1744, 416, 278, 662, 1211, 890, 145, 60, 607, 1261, 1490, 771, 997, 867, 391, 64, 582, 298, 289, 184] payload_rows_per_dst=[5134, 5628, 6324, 6774, 4196, 5910, 4650, 5162] dup_rows_per_dst=[2144, 2854, 3830, 4964, 1416, 3024, 1344, 2182] combine_rows_per_src=[5134, 5628, 6324, 6774, 4196, 5910, 4650, 5162] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=284971008 dispatch_meta_bytes=228912 combine_payload_bytes=284971008 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=285199920 ingress_bytes=615272448 link_bytes=615272448 token_bytes=570170928 weight_movement_bytes=330301440 total_nvlink_bytes=900472368 weight_to_token_ratio=0.579302 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=8064 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.070059982e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=283250688 dispatch_meta_bytes=228592 combine_payload_bytes=283250688 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=283479280 ingress_bytes=503451648 link_bytes=503451648 token_bytes=566729968 weight_movement_bytes=220200960 total_nvlink_bytes=786930928 weight_to_token_ratio=0.388547 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=7808 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.755852194e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=271366144 dispatch_meta_bytes=219808 combine_payload_bytes=271366144 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=271585952 ingress_bytes=381466624 link_bytes=381466624 token_bytes=542952096 weight_movement_bytes=110100480 total_nvlink_bytes=653052576 weight_to_token_ratio=0.202781 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=7808 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.634331996e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278949888 dispatch_meta_bytes=229728 combine_payload_bytes=278949888 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279179616 ingress_bytes=389050368 link_bytes=389050368 token_bytes=558129504 weight_movement_bytes=110100480 total_nvlink_bytes=668229984 weight_to_token_ratio=0.197267 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=12416 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.766225777e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=284626944 dispatch_meta_bytes=229448 combine_payload_bytes=284626944 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=284856392 ingress_bytes=394727424 link_bytes=394727424 token_bytes=569483336 weight_movement_bytes=110100480 total_nvlink_bytes=679583816 weight_to_token_ratio=0.193334 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=12032 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.864959118e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280928256 dispatch_meta_bytes=225048 combine_payload_bytes=280928256 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=281153304 ingress_bytes=611229696 link_bytes=611229696 token_bytes=562081560 weight_movement_bytes=330301440 total_nvlink_bytes=892383000 weight_to_token_ratio=0.587640 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=7936 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.063028971e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=283838464 dispatch_meta_bytes=228752 combine_payload_bytes=283838464 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=284067216 ingress_bytes=504039424 link_bytes=504039424 token_bytes=567905680 weight_movement_bytes=220200960 total_nvlink_bytes=788106640 weight_to_token_ratio=0.387742 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=7808 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.766074586e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49652 payload_rows_remote=41584 dup_rows=15884 dup_rows_remote=10794 dedup_ratio=0.242371 router_tokens_per_expert=[0, 447, 3917, 1072, 9, 10, 2444, 47, 687, 1585, 941, 1, 278, 1, 21, 28, 4, 260, 218, 68, 0, 2016, 2, 94, 1071, 283, 1, 32, 2383, 0, 225, 3, 1774, 276, 0, 5, 1, 4, 1, 0, 104, 0, 3569, 1, 119, 0, 0, 0, 443, 740, 1, 559, 3, 734, 2, 31, 90, 0, 2620, 7, 35, 2019, 1328, 154] payload_rows_per_dst=[6164, 5766, 8068, 6012, 3942, 7228, 6800, 5672] dup_rows_per_dst=[1894, 1318, 5090, 1984, 180, 358, 3466, 1594] combine_rows_per_src=[6164, 5766, 8068, 6012, 3942, 7228, 6800, 5672] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=277000192 dispatch_meta_bytes=233032 combine_payload_bytes=277000192 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277233224 ingress_bytes=387100672 link_bytes=387100672 token_bytes=554233416 weight_movement_bytes=110100480 total_nvlink_bytes=664333896 weight_to_token_ratio=0.198654 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=10752 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.732317357e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48012 payload_rows_remote=44068 dup_rows=17524 dup_rows_remote=16812 dedup_ratio=0.267395 router_tokens_per_expert=[0, 332, 4004, 1290, 0, 3, 2630, 103, 265, 2002, 1078, 0, 441, 4, 0, 19, 0, 75, 304, 95, 0, 1809, 0, 36, 401, 484, 0, 10, 2744, 13, 189, 0, 2779, 251, 2, 0, 18, 2, 1, 0, 42, 1, 3796, 2, 90, 0, 0, 0, 359, 566, 6, 446, 0, 934, 0, 17, 4, 0, 2054, 0, 21, 1127, 1179, 740] payload_rows_per_dst=[3166, 7634, 4260, 6248, 8134, 7666, 3944, 6960] dup_rows_per_dst=[290, 5244, 378, 1434, 5980, 196, 712, 3290] combine_rows_per_src=[3166, 7634, 4260, 6248, 8134, 7666, 3944, 6960] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50450 payload_rows_remote=43946 dup_rows=15086 dup_rows_remote=13538 dedup_ratio=0.230194 router_tokens_per_expert=[0, 1338, 4041, 1089, 3, 18, 1497, 98, 910, 1332, 1240, 0, 673, 2, 0, 13, 0, 49, 646, 50, 18, 1573, 0, 40, 343, 363, 3, 10, 2813, 21, 96, 11, 2975, 344, 0, 0, 0, 5, 0, 0, 8, 0, 3723, 0, 115, 1, 0, 0, 1180, 635, 1, 184, 0, 1303, 1, 9, 12, 0, 1065, 0, 17, 1698, 753, 449] payload_rows_per_dst=[5854, 6160, 7598, 6208, 6504, 7488, 5936, 4702] dup_rows_per_dst=[2232, 2180, 3832, 1112, 1548, 206, 2820, 1156] combine_rows_per_src=[5854, 6160, 7598, 6208, 6504, 7488, 5936, 4702] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280784896 dispatch_meta_bytes=229720 combine_payload_bytes=280784896 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281014616 ingress_bytes=390885376 link_bytes=390885376 token_bytes=561799512 weight_movement_bytes=110100480 total_nvlink_bytes=671899992 weight_to_token_ratio=0.195978 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=12672 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.798139585e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=277143552 dispatch_meta_bytes=233328 combine_payload_bytes=277143552 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277376880 ingress_bytes=387244032 link_bytes=387244032 token_bytes=554520432 weight_movement_bytes=110100480 total_nvlink_bytes=664620912 weight_to_token_ratio=0.198551 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=10624 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.734810623e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50494 payload_rows_remote=43852 dup_rows=15042 dup_rows_remote=13270 dedup_ratio=0.229523 router_tokens_per_expert=[0, 1350, 4044, 1117, 3, 20, 1473, 95, 924, 1349, 1271, 0, 631, 2, 0, 13, 0, 47, 643, 58, 19, 1547, 0, 36, 346, 378, 4, 15, 2807, 21, 114, 10, 2974, 324, 0, 0, 0, 5, 0, 0, 9, 0, 3729, 0, 113, 0, 0, 0, 1217, 669, 1, 209, 0, 1189, 1, 10, 12, 0, 1036, 0, 15, 1669, 777, 472] payload_rows_per_dst=[5868, 6218, 7440, 6208, 6642, 7498, 5914, 4706] dup_rows_per_dst=[2248, 2162, 3540, 1182, 1772, 204, 2750, 1184] combine_rows_per_src=[5868, 6218, 7440, 6208, 6642, 7498, 5914, 4706] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47304 payload_rows_remote=39706 dup_rows=18232 dup_rows_remote=12548 dedup_ratio=0.278198 router_tokens_per_expert=[0, 313, 4064, 1418, 4, 17, 2253, 122, 276, 1222, 807, 0, 323, 8, 2, 12, 1, 153, 373, 200, 9, 1807, 0, 38, 475, 974, 1, 25, 2667, 0, 181, 1, 1986, 317, 0, 0, 0, 4, 0, 0, 7, 0, 3858, 5, 236, 1, 0, 2, 501, 990, 8, 293, 3, 622, 0, 2, 96, 0, 2390, 0, 55, 1522, 1379, 745] payload_rows_per_dst=[3416, 5328, 4430, 6458, 8162, 7804, 4108, 7598] dup_rows_per_dst=[332, 1462, 732, 2190, 4580, 2522, 730, 5684] combine_rows_per_src=[3416, 5328, 4430, 6458, 8162, 7804, 4108, 7598] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48134 payload_rows_remote=44108 dup_rows=17402 dup_rows_remote=16666 dedup_ratio=0.265533 router_tokens_per_expert=[0, 350, 4008, 1321, 0, 4, 2575, 98, 262, 1975, 1108, 0, 419, 5, 0, 19, 0, 71, 332, 85, 0, 1818, 0, 42, 402, 487, 0, 9, 2743, 14, 207, 0, 2758, 229, 2, 0, 19, 2, 0, 0, 43, 1, 3790, 2, 83, 0, 0, 0, 388, 584, 5, 498, 0, 890, 0, 16, 4, 0, 1989, 0, 23, 1093, 1251, 744] payload_rows_per_dst=[3236, 7596, 4294, 6248, 8140, 7654, 4026, 6940] dup_rows_per_dst=[310, 5130, 402, 1476, 5896, 184, 736, 3268] combine_rows_per_src=[3236, 7596, 4294, 6248, 8140, 7654, 4026, 6940] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49654 payload_rows_remote=41588 dup_rows=15882 dup_rows_remote=10796 dedup_ratio=0.242340 router_tokens_per_expert=[0, 436, 3921, 1131, 7, 10, 2419, 44, 705, 1573, 974, 1, 258, 1, 20, 27, 4, 238, 224, 72, 0, 2020, 2, 95, 1072, 286, 0, 32, 2380, 0, 247, 4, 1792, 266, 0, 6, 0, 5, 1, 0, 108, 0, 3578, 1, 116, 0, 0, 0, 452, 766, 1, 577, 3, 703, 2, 35, 81, 0, 2548, 7, 31, 1979, 1361, 146] payload_rows_per_dst=[6148, 5784, 8066, 6032, 3966, 7250, 6752, 5656] dup_rows_per_dst=[1946, 1334, 5086, 2010, 174, 356, 3422, 1554] combine_rows_per_src=[6148, 5784, 8066, 6032, 3966, 7250, 6752, 5656] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49050 payload_rows_remote=41316 dup_rows=16486 dup_rows_remote=16010 dedup_ratio=0.251556 router_tokens_per_expert=[0, 639, 3858, 731, 2, 44, 2845, 116, 234, 1292, 1064, 1, 445, 4, 0, 124, 0, 58, 226, 359, 6, 1212, 0, 28, 236, 563, 17, 7, 2585, 16, 87, 1, 1625, 517, 0, 0, 3, 0, 0, 4, 10, 0, 3857, 1, 233, 4, 0, 0, 245, 914, 8, 211, 21, 1860, 0, 1, 52, 0, 2604, 0, 15, 920, 2329, 534] payload_rows_per_dst=[4954, 5942, 3408, 5896, 7934, 7734, 6712, 6470] dup_rows_per_dst=[1346, 2840, 370, 1128, 4080, 476, 3466, 2780] combine_rows_per_src=[4954, 5942, 3408, 5896, 7934, 7734, 6712, 6470] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47734 payload_rows_remote=40110 dup_rows=17802 dup_rows_remote=12136 dedup_ratio=0.271637 router_tokens_per_expert=[1, 312, 4062, 1364, 5, 14, 2313, 118, 256, 1197, 798, 0, 338, 7, 2, 12, 1, 167, 359, 190, 9, 1814, 0, 42, 491, 966, 0, 27, 2717, 0, 143, 0, 2020, 327, 0, 0, 0, 4, 0, 0, 6, 0, 3855, 5, 231, 1, 0, 2, 500, 947, 8, 274, 2, 645, 0, 2, 104, 0, 2464, 0, 59, 1572, 1315, 700] payload_rows_per_dst=[3314, 5214, 4440, 7190, 8160, 7734, 4058, 7624] dup_rows_per_dst=[314, 1412, 724, 3856, 4666, 466, 698, 5666] combine_rows_per_src=[3314, 5214, 4440, 7190, 8160, 7734, 4058, 7624] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=298074112 dispatch_meta_bytes=209512 combine_payload_bytes=298074112 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=298283624 ingress_bytes=408174592 link_bytes=408174592 token_bytes=596357736 weight_movement_bytes=110100480 total_nvlink_bytes=706458216 weight_to_token_ratio=0.184622 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=12800 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.098827487e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48828 payload_rows_remote=43040 dup_rows=16708 dup_rows_remote=14948 dedup_ratio=0.254944 router_tokens_per_expert=[0, 253, 3946, 1661, 7, 22, 1507, 521, 833, 1531, 973, 5, 666, 0, 3, 291, 0, 101, 1149, 45, 24, 1528, 0, 77, 531, 734, 0, 0, 2283, 0, 225, 1, 1556, 454, 0, 0, 25, 3, 3, 3, 83, 0, 3848, 1, 127, 1, 0, 0, 639, 663, 12, 270, 7, 1016, 16, 41, 103, 0, 1532, 6, 46, 1803, 1190, 403] payload_rows_per_dst=[5720, 6152, 8046, 5788, 3882, 7716, 6106, 5418] dup_rows_per_dst=[2222, 2452, 5694, 1760, 206, 404, 2286, 1684] combine_rows_per_src=[5720, 6152, 8046, 5788, 3882, 7716, 6106, 5418] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49398 payload_rows_remote=44588 dup_rows=16138 dup_rows_remote=15300 dedup_ratio=0.246246 router_tokens_per_expert=[0, 260, 4036, 918, 1, 182, 2347, 63, 486, 1362, 605, 5, 325, 4, 0, 37, 0, 62, 1074, 98, 3, 1043, 5, 26, 390, 961, 7, 20, 2790, 2, 154, 2, 2388, 606, 0, 0, 12, 4, 0, 0, 24, 0, 3539, 4, 105, 0, 4, 0, 558, 460, 1, 179, 3, 1086, 0, 1, 7, 0, 2123, 1, 68, 2212, 1015, 1100] payload_rows_per_dst=[7366, 4810, 5588, 6538, 5326, 7118, 6310, 6342] dup_rows_per_dst=[5204, 838, 2078, 2114, 694, 234, 2512, 2464] combine_rows_per_src=[7366, 4810, 5588, 6538, 5326, 7118, 6310, 6342] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=315879424 dispatch_meta_bytes=243520 combine_payload_bytes=315879424 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=316122944 ingress_bytes=425979904 link_bytes=425979904 token_bytes=632002368 weight_movement_bytes=110100480 total_nvlink_bytes=742102848 weight_to_token_ratio=0.174209 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=11136 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.408491147e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=315004928 dispatch_meta_bytes=229936 combine_payload_bytes=315004928 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=315234864 ingress_bytes=425105408 link_bytes=425105408 token_bytes=630239792 weight_movement_bytes=110100480 total_nvlink_bytes=740340272 weight_to_token_ratio=0.174696 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=17024 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.393282223e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49206 payload_rows_remote=41458 dup_rows=16330 dup_rows_remote=15856 dedup_ratio=0.249176 router_tokens_per_expert=[0, 646, 3874, 786, 2, 40, 2809, 113, 256, 1288, 1086, 1, 438, 4, 0, 127, 0, 58, 218, 363, 6, 1202, 0, 25, 225, 570, 18, 10, 2596, 15, 106, 1, 1609, 491, 0, 0, 4, 0, 0, 4, 11, 0, 3862, 1, 235, 2, 0, 0, 259, 930, 6, 215, 44, 1761, 0, 1, 48, 0, 2549, 0, 16, 896, 2393, 548] payload_rows_per_dst=[5020, 5966, 3366, 5956, 7964, 7748, 6758, 6428] dup_rows_per_dst=[1344, 2862, 378, 1126, 4000, 474, 3616, 2530] combine_rows_per_src=[5020, 5966, 3366, 5956, 7964, 7748, 6758, 6428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48910 payload_rows_remote=43108 dup_rows=16626 dup_rows_remote=14890 dedup_ratio=0.253693 router_tokens_per_expert=[0, 251, 3945, 1712, 8, 21, 1469, 507, 864, 1515, 994, 5, 643, 0, 2, 285, 0, 94, 1156, 46, 21, 1546, 0, 78, 500, 730, 1, 0, 2284, 0, 253, 1, 1588, 430, 0, 0, 26, 3, 5, 3, 82, 0, 3853, 1, 120, 1, 0, 0, 655, 720, 11, 276, 10, 959, 14, 53, 91, 0, 1480, 5, 43, 1778, 1232, 398] payload_rows_per_dst=[5720, 6186, 8052, 5802, 3908, 7732, 6088, 5422] dup_rows_per_dst=[2216, 2430, 5720, 1736, 202, 382, 2268, 1672] combine_rows_per_src=[5720, 6186, 8052, 5802, 3908, 7732, 6088, 5422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49396 payload_rows_remote=44676 dup_rows=16140 dup_rows_remote=15322 dedup_ratio=0.246277 router_tokens_per_expert=[0, 259, 4037, 937, 1, 182, 2303, 58, 498, 1314, 599, 5, 307, 7, 0, 39, 0, 51, 1076, 97, 3, 1072, 4, 25, 394, 980, 7, 24, 2804, 2, 197, 1, 2363, 534, 0, 0, 12, 5, 0, 0, 26, 0, 3553, 3, 104, 0, 4, 0, 603, 483, 1, 177, 4, 1042, 0, 1, 7, 0, 2086, 1, 66, 2174, 1092, 1144] payload_rows_per_dst=[7270, 4720, 5764, 6600, 5218, 7148, 6272, 6404] dup_rows_per_dst=[4960, 818, 2216, 2218, 610, 232, 2522, 2564] combine_rows_per_src=[7270, 4720, 5764, 6600, 5218, 7148, 6272, 6404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=314331136 dispatch_meta_bytes=228488 combine_payload_bytes=314331136 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=314559624 ingress_bytes=424431616 link_bytes=424431616 token_bytes=628890760 weight_movement_bytes=110100480 total_nvlink_bytes=738991240 weight_to_token_ratio=0.175071 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=17024 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.381563872e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=284612608 dispatch_meta_bytes=209016 combine_payload_bytes=284612608 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=284821624 ingress_bytes=614914048 link_bytes=614914048 token_bytes=569434232 weight_movement_bytes=330301440 total_nvlink_bytes=899735672 weight_to_token_ratio=0.580052 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=11520 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.069436665e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47152 payload_rows_remote=39000 dup_rows=18384 dup_rows_remote=11034 dedup_ratio=0.280518 router_tokens_per_expert=[0, 663, 3998, 1254, 2, 21, 1583, 230, 486, 1631, 917, 0, 483, 11, 0, 242, 1, 209, 975, 24, 0, 1809, 0, 7, 516, 1124, 0, 7, 2418, 5, 117, 2, 1845, 304, 0, 0, 2, 0, 0, 0, 6, 0, 3780, 2, 385, 0, 1, 0, 575, 1233, 5, 177, 9, 942, 0, 0, 43, 0, 1846, 1, 116, 1278, 1238, 245] payload_rows_per_dst=[8152, 5846, 4894, 6736, 4122, 7652, 6350, 3400] dup_rows_per_dst=[7350, 1694, 1156, 3558, 180, 696, 3224, 526] combine_rows_per_src=[8152, 5846, 4894, 6736, 4122, 7652, 6350, 3400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=316166144 dispatch_meta_bytes=243096 combine_payload_bytes=316166144 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=316409240 ingress_bytes=426266624 link_bytes=426266624 token_bytes=632575384 weight_movement_bytes=110100480 total_nvlink_bytes=742675864 weight_to_token_ratio=0.174051 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=11008 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.413477679e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=298102784 dispatch_meta_bytes=209536 combine_payload_bytes=298102784 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=298312320 ingress_bytes=408203264 link_bytes=408203264 token_bytes=596415104 weight_movement_bytes=110100480 total_nvlink_bytes=706515584 weight_to_token_ratio=0.184604 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=12928 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.099326140e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=296153088 dispatch_meta_bytes=229304 combine_payload_bytes=296153088 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=296382392 ingress_bytes=406253568 link_bytes=406253568 token_bytes=592535480 weight_movement_bytes=110100480 total_nvlink_bytes=702635960 weight_to_token_ratio=0.185812 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=30080 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.065417720e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=287508480 dispatch_meta_bytes=208984 combine_payload_bytes=287508480 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=287717464 ingress_bytes=617809920 link_bytes=617809920 token_bytes=575225944 weight_movement_bytes=330301440 total_nvlink_bytes=905527384 weight_to_token_ratio=0.574212 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=11648 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.074473063e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=308510720 dispatch_meta_bytes=231952 combine_payload_bytes=308510720 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=308742672 ingress_bytes=418611200 link_bytes=418611200 token_bytes=617253392 weight_movement_bytes=110100480 total_nvlink_bytes=727353872 weight_to_token_ratio=0.178372 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=20736 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.280337265e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=319606784 dispatch_meta_bytes=239552 combine_payload_bytes=319606784 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=319846336 ingress_bytes=429707264 link_bytes=429707264 token_bytes=639453120 weight_movement_bytes=110100480 total_nvlink_bytes=749553600 weight_to_token_ratio=0.172179 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=12032 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.473316068e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46590 payload_rows_remote=38432 dup_rows=18946 dup_rows_remote=11608 dedup_ratio=0.289093 router_tokens_per_expert=[0, 666, 3995, 1214, 3, 20, 1607, 243, 487, 1673, 890, 0, 497, 9, 0, 231, 1, 216, 960, 19, 0, 1777, 0, 8, 525, 1092, 0, 6, 2399, 5, 105, 2, 1836, 311, 0, 0, 3, 0, 0, 0, 6, 0, 3786, 1, 383, 0, 1, 0, 556, 1209, 5, 174, 8, 998, 0, 0, 49, 0, 1905, 1, 125, 1303, 1213, 245] payload_rows_per_dst=[8158, 5868, 4836, 6142, 4138, 7766, 6416, 3266] dup_rows_per_dst=[7338, 1706, 1126, 2126, 162, 2746, 3294, 448] combine_rows_per_src=[8158, 5868, 4836, 6142, 4138, 7766, 6416, 3266] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56280 payload_rows_remote=48692 dup_rows=9256 dup_rows_remote=9254 dedup_ratio=0.141235 router_tokens_per_expert=[488, 3800, 721, 40, 287, 0, 20, 0, 0, 33, 1, 7, 93, 42, 3954, 1, 0, 1, 0, 3793, 1, 0, 0, 0, 0, 501, 1, 1009, 0, 54, 0, 334, 146, 3158, 0, 927, 0, 17, 1316, 1869, 198, 11, 0, 2, 3923, 34, 0, 0, 0, 0, 4074, 0, 577, 0, 20, 0, 0, 0, 53, 4, 1234, 5, 1, 18] payload_rows_per_dst=[7932, 7942, 7588, 3408, 6504, 7880, 8148, 6878] dup_rows_per_dst=[2780, 320, 2, 390, 2046, 456, 1194, 2068] combine_rows_per_src=[7932, 7942, 7588, 3408, 6504, 7880, 8148, 6878] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54822 payload_rows_remote=48380 dup_rows=10714 dup_rows_remote=8508 dedup_ratio=0.163483 router_tokens_per_expert=[210, 3743, 1131, 104, 266, 0, 107, 0, 0, 2, 0, 0, 23, 68, 3669, 0, 1, 1, 0, 3976, 0, 1, 0, 0, 1, 693, 0, 1045, 0, 20, 0, 226, 139, 3219, 0, 2148, 0, 0, 1018, 1019, 229, 0, 0, 0, 3392, 31, 1, 0, 0, 0, 3989, 0, 987, 0, 37, 0, 0, 0, 424, 1, 847, 0, 0, 0] payload_rows_per_dst=[3356, 7356, 7954, 7806, 6442, 6886, 7998, 7024] dup_rows_per_dst=[280, 168, 4, 3650, 2206, 420, 2028, 1958] combine_rows_per_src=[3356, 7356, 7954, 7806, 6442, 6886, 7998, 7024] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54230 payload_rows_remote=46078 dup_rows=11306 dup_rows_remote=8638 dedup_ratio=0.172516 router_tokens_per_expert=[590, 3819, 507, 20, 170, 0, 28, 0, 0, 16, 0, 1, 120, 57, 3989, 0, 0, 2, 1, 3802, 3, 12, 0, 0, 4, 628, 1, 844, 0, 8, 1, 506, 313, 3323, 0, 1105, 0, 0, 966, 724, 183, 2, 0, 0, 3955, 121, 0, 0, 0, 0, 4073, 0, 1267, 0, 70, 0, 0, 1, 130, 3, 1399, 0, 4, 0] payload_rows_per_dst=[2410, 8020, 7630, 7926, 4960, 7940, 8152, 7192] dup_rows_per_dst=[220, 346, 10, 3696, 1256, 582, 2668, 2528] combine_rows_per_src=[2410, 8020, 7630, 7926, 4960, 7940, 8152, 7192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=275480576 dispatch_meta_bytes=200160 combine_payload_bytes=275480576 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=275680736 ingress_bytes=825982976 link_bytes=825982976 token_bytes=551161312 weight_movement_bytes=550502400 total_nvlink_bytes=1101663712 weight_to_token_ratio=0.998805 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=11136 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.436520246e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54354 payload_rows_remote=46564 dup_rows=11182 dup_rows_remote=10136 dedup_ratio=0.170624 router_tokens_per_expert=[769, 3759, 232, 66, 168, 0, 19, 1, 2, 4, 1, 0, 102, 40, 4040, 1, 0, 5, 0, 3932, 0, 0, 2, 2, 21, 445, 2, 875, 0, 34, 0, 306, 280, 3110, 0, 1115, 0, 0, 409, 1207, 216, 200, 2, 0, 3797, 195, 8, 0, 0, 0, 4056, 0, 1080, 0, 1, 0, 0, 0, 29, 0, 2235, 0, 0, 0] payload_rows_per_dst=[2338, 8084, 7870, 7920, 5016, 7790, 8122, 7214] dup_rows_per_dst=[172, 296, 12, 2964, 1006, 1046, 2152, 3534] combine_rows_per_src=[2338, 8084, 7870, 7920, 5016, 7790, 8122, 7214] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55282 payload_rows_remote=48262 dup_rows=10254 dup_rows_remote=8986 dedup_ratio=0.156464 router_tokens_per_expert=[364, 3899, 713, 21, 43, 0, 13, 0, 12, 15, 5, 1, 71, 56, 3971, 0, 2, 12, 5, 3947, 3, 2, 1, 0, 0, 733, 1, 707, 0, 8, 2, 750, 87, 3391, 4, 1544, 1, 4, 801, 1763, 176, 7, 1, 3, 3998, 83, 1, 0, 3, 1, 4073, 3, 697, 0, 17, 0, 2, 8, 12, 1, 724, 1, 0, 5] payload_rows_per_dst=[2256, 7562, 7916, 7972, 6344, 8038, 8174, 7020] dup_rows_per_dst=[52, 226, 28, 4228, 2096, 580, 1776, 1268] combine_rows_per_src=[2256, 7562, 7916, 7972, 6344, 8038, 8174, 7020] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55150 payload_rows_remote=49214 dup_rows=10386 dup_rows_remote=7890 dedup_ratio=0.158478 router_tokens_per_expert=[660, 3842, 769, 9, 243, 1, 58, 0, 2, 7, 1, 0, 183, 241, 3742, 0, 0, 1, 0, 3807, 0, 5, 0, 1, 23, 418, 0, 1470, 0, 131, 2, 413, 423, 2657, 0, 2006, 2, 21, 402, 500, 95, 3, 1, 0, 3785, 97, 0, 0, 0, 0, 4030, 0, 673, 1, 9, 0, 0, 0, 353, 6, 1675, 0, 0, 0] payload_rows_per_dst=[5726, 7720, 7626, 5936, 5624, 7658, 8076, 6784] dup_rows_per_dst=[1920, 632, 2, 2496, 1084, 304, 1350, 2598] combine_rows_per_src=[5726, 7720, 7626, 5936, 5624, 7658, 8076, 6784] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=349024256 dispatch_meta_bytes=231784 combine_payload_bytes=349024256 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=349256040 ingress_bytes=459124736 link_bytes=459124736 token_bytes=698280296 weight_movement_bytes=110100480 total_nvlink_bytes=808380776 weight_to_token_ratio=0.157674 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=30848 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.984934289e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49846 payload_rows_remote=41868 dup_rows=15690 dup_rows_remote=15380 dedup_ratio=0.239410 router_tokens_per_expert=[265, 3845, 847, 77, 269, 1, 71, 1, 13, 17, 0, 4, 89, 57, 3964, 0, 17, 12, 0, 3665, 6, 1, 0, 0, 8, 498, 0, 511, 1, 38, 5, 480, 58, 3417, 0, 1619, 31, 4, 1077, 1214, 81, 0, 0, 1, 3577, 51, 38, 1, 1, 0, 4068, 4, 1519, 0, 11, 1, 0, 0, 27, 4, 1177, 0, 0, 25] payload_rows_per_dst=[8154, 7978, 7380, 2866, 6266, 7280, 3836, 6086] dup_rows_per_dst=[9548, 310, 22, 216, 3318, 218, 422, 1636] combine_rows_per_src=[8154, 7978, 7380, 2866, 6266, 7280, 3836, 6086] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=346787840 dispatch_meta_bytes=227552 combine_payload_bytes=346787840 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=347015392 ingress_bytes=677089280 link_bytes=677089280 token_bytes=693803232 weight_movement_bytes=330301440 total_nvlink_bytes=1024104672 weight_to_token_ratio=0.476074 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=10752 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.177569620e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=330287104 dispatch_meta_bytes=218864 combine_payload_bytes=330287104 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=330505968 ingress_bytes=550488064 link_bytes=550488064 token_bytes=660793072 weight_movement_bytes=220200960 total_nvlink_bytes=880994032 weight_to_token_ratio=0.333237 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=24960 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.573892830e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=297170944 dispatch_meta_bytes=229256 combine_payload_bytes=297170944 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=297400200 ingress_bytes=407271424 link_bytes=407271424 token_bytes=594571144 weight_movement_bytes=110100480 total_nvlink_bytes=704671624 weight_to_token_ratio=0.185176 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=30080 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.083119910e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=308998144 dispatch_meta_bytes=231992 combine_payload_bytes=308998144 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=309230136 ingress_bytes=419098624 link_bytes=419098624 token_bytes=618228280 weight_movement_bytes=110100480 total_nvlink_bytes=728328760 weight_to_token_ratio=0.178090 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=20736 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.288814370e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=320237568 dispatch_meta_bytes=239992 combine_payload_bytes=320237568 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=320477560 ingress_bytes=430338048 link_bytes=430338048 token_bytes=640715128 weight_movement_bytes=110100480 total_nvlink_bytes=750815608 weight_to_token_ratio=0.171840 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=11904 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.484286439e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279552000 dispatch_meta_bytes=200136 combine_payload_bytes=279552000 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=279752136 ingress_bytes=830054400 link_bytes=830054400 token_bytes=559304136 weight_movement_bytes=550502400 total_nvlink_bytes=1109806536 weight_to_token_ratio=0.984263 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=10880 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.443601122e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54734 payload_rows_remote=48184 dup_rows=10802 dup_rows_remote=8428 dedup_ratio=0.164825 router_tokens_per_expert=[210, 3749, 1123, 87, 220, 0, 99, 0, 0, 2, 0, 0, 25, 74, 3694, 0, 0, 1, 0, 3970, 0, 1, 0, 0, 1, 724, 0, 1065, 0, 20, 0, 232, 125, 3054, 0, 2200, 0, 0, 1076, 1061, 202, 0, 0, 0, 3418, 40, 1, 0, 0, 0, 3988, 0, 951, 0, 33, 0, 0, 0, 454, 1, 867, 0, 0, 0] payload_rows_per_dst=[3232, 7408, 7942, 7832, 6550, 6938, 7998, 6834] dup_rows_per_dst=[246, 182, 2, 3750, 2374, 384, 1946, 1918] combine_rows_per_src=[3232, 7408, 7942, 7832, 6550, 6938, 7998, 6834] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54420 payload_rows_remote=47572 dup_rows=11116 dup_rows_remote=9908 dedup_ratio=0.169617 router_tokens_per_expert=[330, 3885, 721, 21, 37, 0, 12, 0, 13, 12, 5, 2, 66, 57, 3981, 0, 2, 9, 5, 3942, 3, 1, 1, 0, 0, 781, 0, 698, 0, 7, 3, 730, 94, 3283, 4, 1611, 3, 4, 809, 1813, 197, 4, 1, 3, 4000, 102, 1, 0, 3, 1, 4075, 3, 671, 0, 17, 0, 3, 7, 14, 1, 713, 1, 0, 6] payload_rows_per_dst=[2194, 6804, 7906, 7964, 6484, 8046, 8174, 6848] dup_rows_per_dst=[48, 198, 20, 4244, 2192, 570, 2636, 1208] combine_rows_per_src=[2194, 6804, 7906, 7964, 6484, 8046, 8174, 6848] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54168 payload_rows_remote=46028 dup_rows=11368 dup_rows_remote=8772 dedup_ratio=0.173462 router_tokens_per_expert=[556, 3827, 498, 19, 142, 0, 28, 0, 1, 15, 0, 1, 127, 54, 3986, 0, 0, 2, 1, 3808, 3, 10, 0, 0, 3, 672, 1, 839, 0, 9, 0, 509, 329, 3283, 0, 1144, 0, 0, 986, 699, 196, 2, 0, 0, 3951, 159, 0, 0, 0, 0, 4067, 0, 1233, 0, 68, 0, 0, 1, 150, 4, 1381, 0, 4, 0] payload_rows_per_dst=[2290, 8012, 7640, 7932, 5066, 7932, 8140, 7156] dup_rows_per_dst=[196, 356, 8, 3788, 1250, 684, 2596, 2490] combine_rows_per_src=[2290, 8012, 7640, 7932, 5066, 7932, 8140, 7156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56310 payload_rows_remote=48718 dup_rows=9226 dup_rows_remote=9224 dedup_ratio=0.140778 router_tokens_per_expert=[502, 3801, 713, 35, 248, 0, 22, 0, 0, 33, 1, 6, 102, 41, 3969, 3, 0, 1, 0, 3795, 1, 0, 0, 0, 0, 537, 1, 1010, 0, 50, 0, 304, 148, 3069, 0, 970, 0, 15, 1318, 1868, 207, 5, 0, 2, 3923, 39, 0, 0, 0, 0, 4070, 0, 571, 0, 18, 0, 0, 0, 66, 4, 1273, 4, 1, 22] payload_rows_per_dst=[7936, 7972, 7592, 3396, 6572, 7880, 8140, 6822] dup_rows_per_dst=[2706, 338, 2, 408, 2066, 472, 1178, 2056] combine_rows_per_src=[7936, 7972, 7592, 3396, 6572, 7880, 8140, 6822] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54438 payload_rows_remote=46600 dup_rows=11098 dup_rows_remote=10050 dedup_ratio=0.169342 router_tokens_per_expert=[729, 3753, 225, 65, 143, 0, 26, 1, 2, 1, 1, 0, 110, 42, 4050, 1, 0, 4, 0, 3945, 0, 0, 2, 2, 19, 469, 2, 831, 0, 31, 0, 344, 279, 3036, 0, 1163, 0, 0, 428, 1266, 222, 156, 2, 0, 3829, 226, 8, 0, 0, 0, 4056, 0, 1046, 0, 1, 0, 0, 0, 29, 0, 2223, 0, 0, 0] payload_rows_per_dst=[2226, 8104, 7896, 7902, 5182, 7838, 8120, 7170] dup_rows_per_dst=[152, 310, 10, 2996, 1090, 1048, 2086, 3406] combine_rows_per_src=[2226, 8104, 7896, 7902, 5182, 7838, 8120, 7170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55288 payload_rows_remote=49528 dup_rows=10248 dup_rows_remote=7956 dedup_ratio=0.156372 router_tokens_per_expert=[628, 3839, 748, 10, 218, 1, 65, 0, 2, 7, 1, 0, 187, 231, 3768, 0, 0, 0, 0, 3794, 0, 4, 0, 1, 22, 445, 1, 1464, 0, 132, 0, 415, 421, 2551, 0, 2094, 0, 24, 422, 495, 94, 3, 1, 0, 3796, 105, 5, 0, 0, 0, 4029, 1, 667, 1, 10, 2, 0, 0, 396, 7, 1661, 0, 0, 0] payload_rows_per_dst=[5964, 7754, 7596, 5760, 5734, 7682, 8076, 6722] dup_rows_per_dst=[1960, 638, 2, 2292, 1178, 326, 1344, 2508] combine_rows_per_src=[5964, 7754, 7596, 5760, 5734, 7682, 8076, 6722] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50358 payload_rows_remote=42370 dup_rows=15178 dup_rows_remote=14860 dedup_ratio=0.231598 router_tokens_per_expert=[253, 3830, 816, 74, 213, 0, 83, 1, 12, 12, 0, 4, 94, 58, 3973, 0, 15, 12, 0, 3663, 7, 1, 0, 0, 8, 520, 0, 519, 1, 35, 2, 487, 65, 3346, 0, 1723, 29, 3, 1115, 1222, 88, 0, 0, 1, 3582, 64, 40, 1, 0, 0, 4064, 4, 1517, 0, 8, 1, 0, 0, 33, 5, 1134, 0, 0, 30] payload_rows_per_dst=[8156, 7988, 7468, 2920, 6242, 7292, 3442, 6850] dup_rows_per_dst=[9916, 318, 160, 224, 2308, 260, 214, 1778] combine_rows_per_src=[8156, 7988, 7468, 2920, 6242, 7292, 3442, 6850] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46330 payload_rows_remote=38138 dup_rows=19206 dup_rows_remote=9084 dedup_ratio=0.293060 router_tokens_per_expert=[355, 3898, 593, 17, 176, 0, 52, 0, 0, 14, 0, 0, 57, 182, 4032, 0, 1, 39, 0, 3819, 0, 4, 0, 0, 0, 912, 0, 1002, 0, 81, 0, 69, 221, 3094, 0, 2075, 0, 0, 1042, 1129, 138, 49, 0, 0, 3954, 88, 1, 0, 0, 3, 4066, 0, 697, 0, 23, 0, 0, 0, 191, 2, 692, 0, 0, 0] payload_rows_per_dst=[8192, 8094, 7934, 3692, 7514, 7732, 1440, 1732] dup_rows_per_dst=[10122, 1060, 3358, 436, 3734, 452, 6, 38] combine_rows_per_src=[8192, 8094, 7934, 3692, 7514, 7732, 1440, 1732] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=333770752 dispatch_meta_bytes=226800 combine_payload_bytes=333770752 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=333997552 ingress_bytes=443871232 link_bytes=443871232 token_bytes=667768304 weight_movement_bytes=110100480 total_nvlink_bytes=777868784 weight_to_token_ratio=0.164878 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=30464 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.719650767e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=345942016 dispatch_meta_bytes=228992 combine_payload_bytes=345942016 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=346171008 ingress_bytes=456042496 link_bytes=456042496 token_bytes=692113024 weight_movement_bytes=110100480 total_nvlink_bytes=802213504 weight_to_token_ratio=0.159079 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=21504 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.931329066e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=352765952 dispatch_meta_bytes=228416 combine_payload_bytes=352765952 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=352994368 ingress_bytes=462866432 link_bytes=462866432 token_bytes=705760320 weight_movement_bytes=110100480 total_nvlink_bytes=815860800 weight_to_token_ratio=0.156003 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=17024 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.050008537e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=300109824 dispatch_meta_bytes=228992 combine_payload_bytes=300109824 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=300338816 ingress_bytes=630411264 link_bytes=630411264 token_bytes=600448640 weight_movement_bytes=330301440 total_nvlink_bytes=930750080 weight_to_token_ratio=0.550091 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=31232 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.096388873e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46464 payload_rows_remote=38274 dup_rows=19072 dup_rows_remote=8870 dedup_ratio=0.291016 router_tokens_per_expert=[366, 3892, 607, 19, 198, 0, 49, 0, 0, 15, 0, 0, 53, 185, 4035, 0, 1, 39, 0, 3826, 0, 4, 0, 0, 0, 872, 0, 1013, 0, 86, 0, 60, 208, 3188, 0, 2037, 0, 0, 1026, 1101, 133, 56, 0, 0, 3962, 77, 1, 0, 0, 3, 4065, 0, 690, 0, 27, 0, 0, 0, 175, 2, 697, 0, 0, 0] payload_rows_per_dst=[8190, 8074, 7948, 3620, 7526, 7950, 1434, 1722] dup_rows_per_dst=[10202, 502, 3544, 442, 3842, 508, 6, 26] combine_rows_per_src=[8190, 8074, 7948, 3620, 7526, 7950, 1434, 1722] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53944 payload_rows_remote=45830 dup_rows=11592 dup_rows_remote=11318 dedup_ratio=0.176880 router_tokens_per_expert=[0, 1, 1, 1819, 1602, 0, 16, 1, 22, 0, 0, 0, 0, 4085, 0, 0, 0, 31, 0, 1, 5, 0, 59, 48, 2, 0, 76, 668, 0, 0, 0, 292, 0, 0, 19, 6, 2, 4087, 1758, 13, 4, 1, 2, 0, 1381, 0, 34, 0, 3385, 0, 0, 0, 0, 0, 1984, 8, 3959, 4050, 2358, 0, 2, 3, 983, 0] payload_rows_per_dst=[6058, 8172, 8114, 8160, 8184, 2818, 6716, 5722] dup_rows_per_dst=[822, 42, 274, 1834, 5348, 26, 2276, 970] combine_rows_per_src=[6058, 8172, 8114, 8160, 8184, 2818, 6716, 5722] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=345382912 dispatch_meta_bytes=226448 combine_payload_bytes=345382912 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=345609360 ingress_bytes=785784832 link_bytes=785784832 token_bytes=690992272 weight_movement_bytes=440401920 total_nvlink_bytes=1131394192 weight_to_token_ratio=0.637347 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=10880 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.366609062e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=340996096 dispatch_meta_bytes=229920 combine_payload_bytes=340996096 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=341226016 ingress_bytes=451096576 link_bytes=451096576 token_bytes=682222112 weight_movement_bytes=110100480 total_nvlink_bytes=792322592 weight_to_token_ratio=0.161385 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=21504 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.845311383e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=329928704 dispatch_meta_bytes=219200 combine_payload_bytes=329928704 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=330147904 ingress_bytes=550129664 link_bytes=550129664 token_bytes=660076608 weight_movement_bytes=220200960 total_nvlink_bytes=880277568 weight_to_token_ratio=0.333599 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=24704 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.567659665e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=349210624 dispatch_meta_bytes=231768 combine_payload_bytes=349210624 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=349442392 ingress_bytes=459311104 link_bytes=459311104 token_bytes=698653016 weight_movement_bytes=110100480 total_nvlink_bytes=808753496 weight_to_token_ratio=0.157590 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=30848 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.988175536e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=334028800 dispatch_meta_bytes=226600 combine_payload_bytes=334028800 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=334255400 ingress_bytes=444129280 link_bytes=444129280 token_bytes=668284200 weight_movement_bytes=110100480 total_nvlink_bytes=778384680 weight_to_token_ratio=0.164751 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=30336 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.724138646e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52642 payload_rows_remote=45110 dup_rows=12894 dup_rows_remote=9494 dedup_ratio=0.196747 router_tokens_per_expert=[0, 0, 5, 2162, 1459, 1, 3, 0, 0, 0, 0, 0, 2, 4096, 0, 1, 0, 68, 2, 0, 0, 1, 16, 67, 12, 12, 3, 547, 0, 0, 1, 47, 0, 0, 0, 0, 0, 4094, 1968, 13, 2, 5, 5, 0, 1838, 0, 0, 0, 3529, 0, 37, 0, 0, 0, 1899, 1, 4065, 4084, 1406, 0, 0, 2, 1315, 0] payload_rows_per_dst=[6198, 8192, 8172, 3956, 3950, 8190, 7532, 6452] dup_rows_per_dst=[1062, 6, 304, 628, 12, 3698, 3400, 3784] combine_rows_per_src=[6198, 8192, 8172, 3956, 3950, 8190, 7532, 6452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=355016704 dispatch_meta_bytes=229936 combine_payload_bytes=355016704 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=355246640 ingress_bytes=465117184 link_bytes=465117184 token_bytes=710263344 weight_movement_bytes=110100480 total_nvlink_bytes=820363824 weight_to_token_ratio=0.155014 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=16768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.089152816e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=303708160 dispatch_meta_bytes=228920 combine_payload_bytes=303708160 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=303937080 ingress_bytes=523909120 link_bytes=523909120 token_bytes=607645240 weight_movement_bytes=220200960 total_nvlink_bytes=827846200 weight_to_token_ratio=0.362384 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=30848 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.111641279e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=273373184 dispatch_meta_bytes=188888 combine_payload_bytes=273373184 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=273562072 ingress_bytes=493574144 link_bytes=493574144 token_bytes=546935256 weight_movement_bytes=220200960 total_nvlink_bytes=767136216 weight_to_token_ratio=0.402609 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=13824 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.584066154e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48200 payload_rows_remote=40044 dup_rows=17336 dup_rows_remote=7180 dedup_ratio=0.264526 router_tokens_per_expert=[1, 0, 20, 2429, 1376, 1, 8, 0, 1, 0, 1, 0, 10, 4041, 0, 6, 12, 103, 0, 1, 0, 0, 11, 65, 10, 2, 0, 580, 2, 0, 0, 124, 0, 2, 6, 4, 0, 4095, 1985, 0, 5, 8, 3, 1, 1906, 1, 15, 0, 3804, 7, 2, 12, 1, 0, 1424, 10, 4053, 4088, 951, 2, 2, 4, 1572, 1] payload_rows_per_dst=[6474, 8110, 6000, 1432, 3978, 8190, 5860, 8156] dup_rows_per_dst=[1196, 8, 350, 4, 16, 3878, 1728, 10156] combine_rows_per_src=[6474, 8110, 6000, 1432, 3978, 8190, 5860, 8156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55214 payload_rows_remote=52038 dup_rows=10322 dup_rows_remote=10316 dedup_ratio=0.157501 router_tokens_per_expert=[0, 4, 34, 1125, 2290, 0, 0, 0, 0, 0, 0, 0, 1, 4074, 0, 0, 2, 64, 0, 11, 0, 0, 13, 39, 3, 1, 11, 260, 0, 0, 2, 15, 0, 0, 1, 1, 0, 4096, 1584, 5, 6, 7, 1, 0, 1830, 0, 0, 0, 3732, 0, 3, 0, 0, 0, 2278, 4, 4095, 4074, 1569, 0, 0, 2, 1531, 0] payload_rows_per_dst=[6198, 8148, 8180, 8190, 3176, 8192, 7882, 5248] dup_rows_per_dst=[708, 2, 226, 584, 6, 3688, 4152, 956] combine_rows_per_src=[6198, 8148, 8180, 8190, 3176, 8192, 7882, 5248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54456 payload_rows_remote=46346 dup_rows=11080 dup_rows_remote=10816 dedup_ratio=0.169067 router_tokens_per_expert=[0, 2, 1, 1804, 1638, 0, 16, 2, 23, 0, 0, 0, 0, 4079, 0, 0, 0, 31, 0, 1, 4, 0, 60, 42, 0, 0, 83, 551, 0, 0, 0, 263, 0, 0, 17, 9, 3, 4087, 1828, 11, 3, 1, 2, 0, 1289, 0, 35, 0, 3457, 0, 0, 0, 0, 0, 1964, 7, 3956, 4049, 2463, 0, 2, 2, 983, 0] payload_rows_per_dst=[6062, 8162, 8110, 8148, 8184, 2636, 7262, 5892] dup_rows_per_dst=[864, 42, 264, 1558, 4482, 24, 2838, 1008] combine_rows_per_src=[6062, 8162, 8110, 8148, 8184, 2636, 7262, 5892] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54032 payload_rows_remote=46440 dup_rows=11504 dup_rows_remote=8032 dedup_ratio=0.175537 router_tokens_per_expert=[0, 0, 5, 2102, 1537, 1, 2, 0, 0, 0, 0, 0, 1, 4096, 0, 1, 0, 58, 2, 0, 0, 1, 18, 66, 13, 19, 5, 429, 0, 0, 1, 39, 0, 0, 0, 0, 0, 4094, 2018, 9, 3, 5, 5, 0, 1740, 0, 0, 0, 3584, 0, 36, 0, 0, 0, 1910, 2, 4061, 4082, 1462, 0, 0, 3, 1358, 0] payload_rows_per_dst=[6210, 8192, 8170, 5998, 4046, 8190, 7592, 5634] dup_rows_per_dst=[1084, 4, 284, 824, 8, 3504, 3472, 2324] combine_rows_per_src=[6210, 8192, 8170, 5998, 4046, 8190, 7592, 5634] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55982 payload_rows_remote=47836 dup_rows=9554 dup_rows_remote=8666 dedup_ratio=0.145782 router_tokens_per_expert=[0, 4, 28, 1639, 1633, 34, 92, 0, 0, 2, 1, 8, 0, 4085, 0, 0, 7, 21, 2, 11, 0, 0, 71, 30, 2, 13, 15, 339, 0, 0, 5, 92, 0, 0, 1, 3, 0, 4096, 1758, 3, 31, 5, 0, 0, 2717, 0, 9, 0, 3314, 0, 0, 3, 4, 0, 2285, 1, 4051, 4090, 1774, 0, 0, 6, 483, 0] payload_rows_per_dst=[6024, 8172, 8184, 8146, 7494, 5990, 7686, 4286] dup_rows_per_dst=[836, 20, 280, 888, 3094, 668, 3528, 240] combine_rows_per_src=[6024, 8172, 8184, 8146, 7494, 5990, 7686, 4286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55240 payload_rows_remote=47048 dup_rows=10296 dup_rows_remote=6394 dedup_ratio=0.157104 router_tokens_per_expert=[2, 9, 2, 2303, 1192, 1, 16, 0, 1, 0, 0, 0, 1, 4093, 1, 0, 1, 99, 0, 1, 0, 0, 43, 21, 0, 8, 0, 657, 0, 0, 14, 288, 0, 0, 0, 9, 0, 4093, 1645, 6, 2, 1, 0, 0, 1939, 0, 12, 0, 3062, 0, 0, 0, 0, 0, 1140, 2, 4087, 4071, 1615, 0, 0, 2, 2329, 0] payload_rows_per_dst=[6054, 8188, 8152, 8184, 3314, 8192, 6936, 6220] dup_rows_per_dst=[996, 4, 320, 1924, 6, 3902, 1472, 1672] combine_rows_per_src=[6054, 8188, 8152, 8184, 3314, 8192, 6936, 6220] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54872 payload_rows_remote=51846 dup_rows=10664 dup_rows_remote=10660 dedup_ratio=0.162720 router_tokens_per_expert=[0, 4, 32, 1145, 2221, 0, 0, 0, 0, 0, 0, 0, 1, 4074, 0, 0, 2, 72, 1, 11, 0, 0, 13, 43, 4, 0, 11, 358, 0, 0, 2, 19, 0, 0, 0, 1, 0, 4096, 1508, 6, 7, 7, 1, 0, 1901, 0, 0, 0, 3710, 0, 3, 0, 0, 0, 2342, 5, 4095, 4074, 1473, 0, 0, 2, 1524, 0] payload_rows_per_dst=[6130, 8148, 8178, 8192, 3026, 8192, 7900, 5106] dup_rows_per_dst=[674, 2, 254, 786, 4, 3832, 4220, 892] combine_rows_per_src=[6130, 8148, 8178, 8192, 3026, 8192, 7900, 5106] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=274348032 dispatch_meta_bytes=188576 combine_payload_bytes=274348032 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=274536608 ingress_bytes=494548992 link_bytes=494548992 token_bytes=548884640 weight_movement_bytes=220200960 total_nvlink_bytes=769085600 weight_to_token_ratio=0.401179 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=13696 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.601020364e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54780 payload_rows_remote=46588 dup_rows=10756 dup_rows_remote=6716 dedup_ratio=0.164124 router_tokens_per_expert=[2, 9, 3, 2355, 1167, 1, 17, 0, 1, 0, 0, 0, 1, 4092, 1, 0, 0, 117, 0, 0, 0, 0, 47, 25, 0, 6, 0, 833, 0, 0, 15, 299, 0, 0, 0, 11, 0, 4093, 1513, 6, 2, 1, 0, 0, 2007, 0, 13, 0, 2969, 0, 0, 0, 0, 0, 1139, 1, 4085, 4072, 1507, 0, 0, 0, 2358, 0] payload_rows_per_dst=[6074, 8186, 8154, 8184, 3054, 8192, 6774, 6162] dup_rows_per_dst=[1034, 4, 368, 2292, 6, 4040, 1444, 1568] combine_rows_per_src=[6074, 8186, 8154, 8184, 3054, 8192, 6774, 6162] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=328509440 dispatch_meta_bytes=228592 combine_payload_bytes=328509440 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=328738032 ingress_bytes=438609920 link_bytes=438609920 token_bytes=657247472 weight_movement_bytes=110100480 total_nvlink_bytes=767347952 weight_to_token_ratio=0.167518 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=31360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.628147898e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56252 payload_rows_remote=48112 dup_rows=9284 dup_rows_remote=8194 dedup_ratio=0.141663 router_tokens_per_expert=[0, 3, 22, 1691, 1651, 32, 98, 0, 0, 1, 1, 7, 0, 4086, 0, 0, 5, 22, 2, 11, 1, 0, 84, 30, 2, 11, 13, 415, 0, 0, 6, 109, 0, 0, 2, 1, 0, 4096, 1660, 5, 30, 9, 0, 0, 2756, 0, 9, 0, 3268, 0, 0, 3, 2, 0, 2270, 1, 4059, 4090, 1730, 0, 0, 7, 467, 0] payload_rows_per_dst=[6110, 8174, 8184, 8140, 8150, 5634, 7684, 4176] dup_rows_per_dst=[884, 16, 306, 1090, 3308, 44, 3404, 232] combine_rows_per_src=[6110, 8174, 8184, 8140, 8150, 5634, 7684, 4176] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48880 payload_rows_remote=40724 dup_rows=16656 dup_rows_remote=7862 dedup_ratio=0.254150 router_tokens_per_expert=[1, 0, 22, 2515, 1273, 1, 9, 0, 1, 0, 1, 0, 11, 4046, 0, 7, 13, 115, 0, 1, 0, 0, 20, 84, 9, 2, 0, 704, 2, 0, 0, 131, 0, 3, 6, 4, 0, 4095, 1846, 0, 4, 11, 2, 0, 1938, 2, 12, 0, 3772, 7, 2, 11, 1, 0, 1449, 10, 4058, 4086, 900, 2, 2, 5, 1571, 1] payload_rows_per_dst=[6448, 8124, 5740, 1682, 3700, 8190, 6840, 8156] dup_rows_per_dst=[1194, 8, 412, 14, 18, 3938, 2278, 8794] combine_rows_per_src=[6448, 8124, 5740, 1682, 3700, 8190, 6840, 8156] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51234 payload_rows_remote=43054 dup_rows=14302 dup_rows_remote=14206 dedup_ratio=0.218231 router_tokens_per_expert=[0, 1, 0, 2038, 1560, 0, 1, 0, 2, 0, 13, 1, 3, 4078, 0, 0, 5, 62, 14, 30, 0, 4, 97, 210, 52, 2, 0, 953, 1, 0, 8, 207, 2, 0, 0, 0, 1, 4091, 1104, 5, 2, 0, 0, 0, 1698, 0, 0, 2, 3439, 1, 0, 4, 0, 1, 1348, 0, 4044, 4022, 1485, 1, 0, 2, 2166, 8] payload_rows_per_dst=[6664, 8180, 8184, 8186, 8186, 3404, 2708, 5722] dup_rows_per_dst=[1788, 96, 704, 2348, 7764, 0, 0, 1602] combine_rows_per_src=[6664, 8180, 8184, 8186, 8186, 3404, 2708, 5722] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=323348480 dispatch_meta_bytes=218416 combine_payload_bytes=323348480 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=323566896 ingress_bytes=763750400 link_bytes=763750400 token_bytes=646915376 weight_movement_bytes=440401920 total_nvlink_bytes=1087317296 weight_to_token_ratio=0.680772 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=19072 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.328287561e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=371632128 dispatch_meta_bytes=250024 combine_payload_bytes=371632128 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=371882152 ingress_bytes=591833088 link_bytes=591833088 token_bytes=743514280 weight_movement_bytes=220200960 total_nvlink_bytes=963715240 weight_to_token_ratio=0.296162 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=16384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.029295080e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51102 payload_rows_remote=42920 dup_rows=14434 dup_rows_remote=14320 dedup_ratio=0.220245 router_tokens_per_expert=[0, 2, 0, 1952, 1678, 0, 1, 0, 2, 0, 13, 1, 3, 4076, 0, 0, 6, 53, 14, 30, 0, 5, 86, 205, 51, 4, 0, 756, 1, 0, 7, 193, 2, 0, 0, 0, 1, 4090, 1185, 3, 3, 0, 0, 0, 1626, 0, 0, 2, 3520, 1, 0, 4, 0, 1, 1350, 0, 4038, 4023, 1562, 1, 1, 0, 2208, 8] payload_rows_per_dst=[6530, 8182, 8188, 8182, 8184, 3262, 2712, 5862] dup_rows_per_dst=[1610, 114, 656, 1918, 8438, 0, 0, 1698] combine_rows_per_src=[6530, 8182, 8188, 8182, 8184, 3262, 2712, 5862] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=332208128 dispatch_meta_bytes=228648 combine_payload_bytes=332208128 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=332436776 ingress_bytes=442308608 link_bytes=442308608 token_bytes=664644904 weight_movement_bytes=110100480 total_nvlink_bytes=774745384 weight_to_token_ratio=0.165653 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=31488 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.692474166e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=287035392 dispatch_meta_bytes=188896 combine_payload_bytes=287035392 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=287224288 ingress_bytes=617336832 link_bytes=617336832 token_bytes=574259680 weight_movement_bytes=330301440 total_nvlink_bytes=904561120 weight_to_token_ratio=0.575178 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=13312 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.073650285e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=373008384 dispatch_meta_bytes=249416 combine_payload_bytes=373008384 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=373257800 ingress_bytes=593209344 link_bytes=593209344 token_bytes=746266184 weight_movement_bytes=220200960 total_nvlink_bytes=966467144 weight_to_token_ratio=0.295070 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=15872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.031688615e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=332881920 dispatch_meta_bytes=217888 combine_payload_bytes=332881920 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=333099808 ingress_bytes=773283840 link_bytes=773283840 token_bytes=665981728 weight_movement_bytes=440401920 total_nvlink_bytes=1106383648 weight_to_token_ratio=0.661282 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=19200 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.344867781e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53656 payload_rows_remote=45584 dup_rows=11880 dup_rows_remote=5262 dedup_ratio=0.181274 router_tokens_per_expert=[0, 0, 9, 2439, 1779, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 6, 0, 3, 1, 0, 3, 1, 0, 20, 0, 475, 0, 0, 2, 186, 0, 0, 0, 2, 1, 4096, 1453, 3, 0, 0, 41, 0, 2555, 0, 14, 0, 3118, 0, 0, 0, 0, 0, 1208, 0, 4095, 4072, 1789, 0, 0, 1, 1302, 0] payload_rows_per_dst=[8072, 8188, 8148, 8192, 8192, 5136, 2416, 5312] dup_rows_per_dst=[6618, 0, 24, 1364, 2918, 84, 0, 872] combine_rows_per_src=[8072, 8188, 8148, 8192, 8192, 5136, 2416, 5312] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=333942784 dispatch_meta_bytes=213216 combine_payload_bytes=333942784 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=334156000 ingress_bytes=444043264 link_bytes=444043264 token_bytes=668098784 weight_movement_bytes=110100480 total_nvlink_bytes=778199264 weight_to_token_ratio=0.164797 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=16512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.722642687e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=344866816 dispatch_meta_bytes=225224 combine_payload_bytes=344866816 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=345092040 ingress_bytes=454967296 link_bytes=454967296 token_bytes=689958856 weight_movement_bytes=110100480 total_nvlink_bytes=800059336 weight_to_token_ratio=0.159575 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=26112 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.912629570e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=291909632 dispatch_meta_bytes=194344 combine_payload_bytes=291909632 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=292103976 ingress_bytes=622211072 link_bytes=622211072 token_bytes=584013608 weight_movement_bytes=330301440 total_nvlink_bytes=914315048 weight_to_token_ratio=0.565571 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=12672 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.082127390e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=308611072 dispatch_meta_bytes=229040 combine_payload_bytes=308611072 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=308840112 ingress_bytes=418711552 link_bytes=418711552 token_bytes=617451184 weight_movement_bytes=110100480 total_nvlink_bytes=727551664 weight_to_token_ratio=0.178314 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.029412 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.282082551e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53688 payload_rows_remote=45654 dup_rows=11848 dup_rows_remote=5486 dedup_ratio=0.180786 router_tokens_per_expert=[0, 1, 12, 2472, 1733, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 10, 0, 1, 1, 0, 6, 4, 0, 21, 0, 692, 0, 0, 2, 212, 0, 0, 0, 1, 1, 4096, 1312, 3, 1, 0, 43, 0, 2609, 0, 16, 0, 2980, 0, 0, 0, 0, 0, 1180, 0, 4095, 4072, 1767, 0, 0, 4, 1328, 0] payload_rows_per_dst=[8034, 8186, 8144, 8192, 8192, 5244, 2360, 5336] dup_rows_per_dst=[6362, 0, 44, 1852, 2634, 94, 0, 862] combine_rows_per_src=[8034, 8186, 8144, 8192, 8192, 5244, 2360, 5336] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58036 payload_rows_remote=50000 dup_rows=7500 dup_rows_remote=7478 dedup_ratio=0.114441 router_tokens_per_expert=[0, 381, 398, 2, 0, 0, 4095, 0, 0, 0, 0, 16, 0, 825, 0, 2, 0, 0, 4086, 15, 0, 4091, 0, 4014, 2735, 0, 0, 0, 282, 1, 257, 0, 2, 3, 2757, 0, 0, 0, 2524, 24, 1, 0, 2169, 0, 0, 1, 5, 0, 0, 3, 0, 0, 0, 0, 0, 0, 5, 0, 8, 0, 0, 173, 5, 3888] payload_rows_per_dst=[8190, 8176, 8036, 7412, 5874, 4350, 8184, 7814] dup_rows_per_dst=[1562, 1682, 22, 3540, 344, 2, 4, 344] combine_rows_per_src=[8190, 8176, 8036, 7412, 5874, 4350, 8184, 7814] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=342888448 dispatch_meta_bytes=226008 combine_payload_bytes=342888448 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=343114456 ingress_bytes=452988928 link_bytes=452988928 token_bytes=686002904 weight_movement_bytes=110100480 total_nvlink_bytes=796103384 weight_to_token_ratio=0.160496 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=27264 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.878222497e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=337240064 dispatch_meta_bytes=213768 combine_payload_bytes=337240064 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=337453832 ingress_bytes=447340544 link_bytes=447340544 token_bytes=674693896 weight_movement_bytes=110100480 total_nvlink_bytes=784794376 weight_to_token_ratio=0.163186 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=17024 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.779987809e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=327247872 dispatch_meta_bytes=204560 combine_payload_bytes=327247872 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=327452432 ingress_bytes=437348352 link_bytes=437348352 token_bytes=654700304 weight_movement_bytes=110100480 total_nvlink_bytes=764800784 weight_to_token_ratio=0.168169 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=16256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.606207156e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51978 payload_rows_remote=43808 dup_rows=13558 dup_rows_remote=13556 dedup_ratio=0.206879 router_tokens_per_expert=[0, 452, 558, 0, 4, 0, 4088, 0, 0, 0, 0, 45, 5, 1044, 0, 5, 0, 0, 4075, 9, 0, 4083, 0, 4058, 2792, 0, 0, 0, 144, 67, 442, 0, 0, 0, 2917, 0, 0, 0, 2252, 11, 10, 2, 1424, 4, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 17, 0, 1, 0, 0, 216, 0, 4038] payload_rows_per_dst=[1874, 4300, 8132, 6008, 7208, 8188, 8170, 8098] dup_rows_per_dst=[154, 624, 5426, 882, 3152, 2872, 2, 446] combine_rows_per_src=[1874, 4300, 8132, 6008, 7208, 8188, 8170, 8098] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54362 payload_rows_remote=47262 dup_rows=11174 dup_rows_remote=8896 dedup_ratio=0.170502 router_tokens_per_expert=[0, 1024, 318, 0, 0, 0, 4095, 0, 0, 0, 0, 12, 0, 733, 0, 0, 0, 0, 4096, 18, 0, 4096, 0, 4078, 2214, 0, 0, 0, 182, 12, 130, 0, 0, 0, 2195, 0, 0, 0, 2492, 2, 2, 0, 3268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 134, 0, 3658] payload_rows_per_dst=[2482, 8192, 8156, 4694, 7100, 8190, 8192, 7356] dup_rows_per_dst=[202, 1490, 36, 382, 2278, 6540, 0, 246] combine_rows_per_src=[2482, 8192, 8156, 4694, 7100, 8190, 8192, 7356] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55366 payload_rows_remote=47176 dup_rows=10170 dup_rows_remote=6136 dedup_ratio=0.155182 router_tokens_per_expert=[0, 1238, 443, 0, 3, 0, 4093, 0, 0, 2, 0, 61, 0, 911, 0, 0, 0, 0, 4090, 114, 0, 4093, 0, 4064, 1954, 0, 2, 0, 28, 22, 138, 3, 0, 0, 3258, 6, 0, 0, 2229, 2, 14, 0, 2004, 0, 1, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 52, 1, 3936] payload_rows_per_dst=[3080, 8190, 8144, 4136, 7554, 8190, 8188, 7884] dup_rows_per_dst=[288, 1938, 212, 158, 3436, 4034, 8, 96] combine_rows_per_src=[3080, 8190, 8144, 4136, 7554, 8190, 8188, 7884] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=358400000 dispatch_meta_bytes=229912 combine_payload_bytes=358400000 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=358629912 ingress_bytes=908902400 link_bytes=908902400 token_bytes=717029912 weight_movement_bytes=550502400 total_nvlink_bytes=1267532312 weight_to_token_ratio=0.767754 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=25728 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.580730762e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=307650560 dispatch_meta_bytes=228960 combine_payload_bytes=307650560 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=307879520 ingress_bytes=417751040 link_bytes=417751040 token_bytes=615530080 weight_movement_bytes=110100480 total_nvlink_bytes=725630560 weight_to_token_ratio=0.178871 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.265377668e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=326746112 dispatch_meta_bytes=203384 combine_payload_bytes=326746112 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=326949496 ingress_bytes=436846592 link_bytes=436846592 token_bytes=653695608 weight_movement_bytes=110100480 total_nvlink_bytes=763796088 weight_to_token_ratio=0.168428 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=15872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.597480724e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57978 payload_rows_remote=49940 dup_rows=7558 dup_rows_remote=7548 dedup_ratio=0.115326 router_tokens_per_expert=[0, 366, 339, 2, 0, 0, 4095, 0, 0, 0, 0, 17, 0, 832, 0, 2, 0, 0, 4086, 9, 0, 4090, 0, 4015, 2688, 0, 0, 0, 296, 1, 287, 0, 2, 3, 2766, 0, 0, 0, 2576, 27, 1, 0, 2169, 0, 0, 2, 5, 0, 0, 3, 0, 0, 0, 0, 0, 0, 9, 0, 8, 0, 0, 176, 5, 3891] payload_rows_per_dst=[8190, 8176, 8038, 7530, 5696, 4352, 8182, 7814] dup_rows_per_dst=[1414, 1698, 10, 3880, 186, 2, 4, 364] combine_rows_per_src=[8190, 8176, 8038, 7530, 5696, 4352, 8182, 7814] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54276 payload_rows_remote=47158 dup_rows=11260 dup_rows_remote=8964 dedup_ratio=0.171814 router_tokens_per_expert=[0, 1040, 300, 0, 0, 0, 4095, 0, 0, 0, 0, 10, 0, 744, 0, 0, 0, 0, 4096, 14, 0, 4096, 0, 4078, 2135, 0, 0, 0, 198, 11, 152, 0, 0, 0, 2135, 0, 0, 0, 2570, 2, 2, 0, 3337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 119, 0, 3620] payload_rows_per_dst=[2826, 8192, 8156, 4624, 7118, 7894, 8192, 7274] dup_rows_per_dst=[466, 1508, 28, 368, 2296, 6362, 0, 232] combine_rows_per_src=[2826, 8192, 8156, 4624, 7118, 7894, 8192, 7274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51926 payload_rows_remote=43760 dup_rows=13610 dup_rows_remote=13606 dedup_ratio=0.207672 router_tokens_per_expert=[0, 465, 527, 0, 3, 0, 4088, 0, 0, 0, 0, 51, 5, 988, 0, 6, 0, 0, 4077, 10, 0, 4082, 0, 4059, 2741, 1, 0, 0, 158, 66, 506, 0, 0, 0, 2890, 0, 0, 0, 2306, 9, 10, 2, 1487, 4, 1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 21, 0, 1, 0, 0, 181, 0, 4020] payload_rows_per_dst=[1840, 4308, 8132, 5998, 7236, 8188, 8166, 8058] dup_rows_per_dst=[150, 694, 5258, 946, 3174, 2996, 4, 388] combine_rows_per_src=[1840, 4308, 8132, 5998, 7236, 8188, 8166, 8058] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50654 payload_rows_remote=43412 dup_rows=14882 dup_rows_remote=12556 dedup_ratio=0.227081 router_tokens_per_expert=[0, 338, 1098, 6, 0, 0, 4073, 0, 0, 0, 4, 31, 17, 1057, 0, 0, 0, 0, 4069, 95, 5, 4075, 0, 3915, 3020, 0, 9, 13, 115, 0, 148, 0, 0, 0, 2840, 0, 0, 0, 3324, 4, 35, 6, 957, 10, 1, 0, 3, 0, 0, 3, 0, 17, 0, 0, 0, 0, 40, 0, 0, 15, 0, 98, 0, 3327] payload_rows_per_dst=[2662, 2210, 8168, 6182, 7844, 8170, 8176, 7242] dup_rows_per_dst=[222, 8, 5298, 428, 4492, 2000, 108, 2326] combine_rows_per_src=[2662, 2210, 8168, 6182, 7844, 8170, 8176, 7242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 2, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55014 payload_rows_remote=46824 dup_rows=10522 dup_rows_remote=6146 dedup_ratio=0.160553 router_tokens_per_expert=[0, 1187, 421, 0, 2, 0, 4093, 0, 0, 2, 0, 56, 0, 919, 0, 0, 0, 0, 4090, 105, 0, 4093, 0, 4076, 1791, 0, 2, 0, 29, 23, 164, 4, 0, 0, 3258, 6, 0, 0, 2262, 3, 11, 0, 2178, 0, 1, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 53, 1, 3932] payload_rows_per_dst=[2952, 8190, 8164, 3872, 7584, 8190, 8188, 7874] dup_rows_per_dst=[268, 1944, 198, 154, 3474, 4376, 8, 100] combine_rows_per_src=[2952, 8190, 8164, 3872, 7584, 8190, 8188, 7874] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57714 payload_rows_remote=52038 dup_rows=7822 dup_rows_remote=7224 dedup_ratio=0.119354 router_tokens_per_expert=[0, 704, 1273, 0, 2, 0, 4071, 1, 0, 0, 0, 2, 0, 683, 0, 0, 0, 0, 4092, 18, 0, 4065, 2, 4093, 2467, 0, 0, 0, 364, 79, 227, 0, 0, 0, 1519, 1, 0, 0, 2654, 4, 32, 0, 2388, 1, 2, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 41, 0, 1, 0, 1, 27, 0, 3952] payload_rows_per_dst=[8174, 8190, 8190, 5676, 6632, 4812, 8130, 7910] dup_rows_per_dst=[3928, 1364, 36, 598, 1724, 36, 2, 134] combine_rows_per_src=[8174, 8190, 8190, 5676, 6632, 4812, 8130, 7910] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50438 payload_rows_remote=43174 dup_rows=15098 dup_rows_remote=12976 dedup_ratio=0.230377 router_tokens_per_expert=[0, 339, 1208, 3, 0, 0, 4073, 0, 0, 0, 5, 28, 17, 1014, 0, 0, 0, 1, 4070, 95, 8, 4077, 0, 3863, 3099, 0, 9, 10, 99, 0, 135, 0, 0, 0, 2903, 0, 0, 0, 3311, 4, 37, 5, 870, 10, 1, 0, 3, 0, 0, 2, 0, 17, 0, 0, 0, 0, 25, 0, 0, 15, 0, 94, 0, 3318] payload_rows_per_dst=[2870, 2118, 8172, 6322, 7344, 8170, 8178, 7264] dup_rows_per_dst=[230, 10, 6882, 382, 3538, 1828, 106, 2122] combine_rows_per_src=[2870, 2118, 8172, 6322, 7344, 8170, 8178, 7264] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57424 payload_rows_remote=51686 dup_rows=8112 dup_rows_remote=7502 dedup_ratio=0.123779 router_tokens_per_expert=[0, 669, 1371, 0, 2, 0, 4068, 1, 0, 0, 0, 4, 0, 685, 0, 0, 0, 0, 4092, 24, 0, 4061, 2, 4093, 2553, 0, 0, 0, 339, 80, 202, 0, 1, 0, 1540, 3, 0, 0, 2642, 3, 31, 0, 2260, 1, 2, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 20, 0, 1, 0, 2, 28, 0, 3985] payload_rows_per_dst=[7542, 8188, 8190, 5738, 6616, 5046, 8124, 7980] dup_rows_per_dst=[3728, 1374, 48, 610, 1762, 496, 2, 92] combine_rows_per_src=[7542, 8188, 8190, 5738, 6616, 5046, 8124, 7980] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52686 payload_rows_remote=44620 dup_rows=12850 dup_rows_remote=10378 dedup_ratio=0.196075 router_tokens_per_expert=[0, 667, 416, 6, 3, 9, 4090, 4, 0, 0, 0, 9, 0, 1233, 0, 0, 0, 0, 4027, 46, 1, 4080, 0, 4042, 3037, 0, 2, 0, 154, 5, 117, 0, 0, 0, 2084, 1, 0, 0, 2760, 10, 3, 0, 1937, 2, 0, 0, 0, 9, 3, 0, 5, 4, 0, 0, 0, 0, 23, 1, 0, 0, 14, 18, 4, 3942] payload_rows_per_dst=[8192, 8066, 4700, 7578, 4172, 3898, 8174, 7906] dup_rows_per_dst=[5624, 2472, 52, 4572, 18, 4, 10, 98] combine_rows_per_src=[8192, 8066, 4700, 7578, 4172, 3898, 8174, 7906] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=357969920 dispatch_meta_bytes=229952 combine_payload_bytes=357969920 prefetch_bytes=110100480 reduce_bytes=550502400 egress_bytes=358199872 ingress_bytes=1018572800 link_bytes=1018572800 token_bytes=716169792 weight_movement_bytes=660602880 total_nvlink_bytes=1376772672 weight_to_token_ratio=0.922411 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=25216 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.771465626e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=338028544 dispatch_meta_bytes=224488 combine_payload_bytes=338028544 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=338253032 ingress_bytes=558229504 link_bytes=558229504 token_bytes=676281576 weight_movement_bytes=220200960 total_nvlink_bytes=896482536 weight_to_token_ratio=0.325605 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=19456 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.708529204e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49422 payload_rows_remote=41230 dup_rows=16114 dup_rows_remote=5678 dedup_ratio=0.245880 router_tokens_per_expert=[0, 489, 651, 0, 3, 0, 4096, 0, 0, 0, 0, 4, 0, 158, 0, 0, 1, 0, 4096, 5, 0, 4094, 0, 4075, 3408, 0, 0, 0, 113, 0, 96, 0, 0, 0, 2493, 0, 0, 0, 2948, 0, 6, 0, 2122, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 35, 0, 0, 1, 1, 18, 0, 3853] payload_rows_per_dst=[8192, 8192, 1102, 7888, 3890, 4250, 8188, 7720] dup_rows_per_dst=[10436, 324, 6, 5242, 0, 6, 4, 96] combine_rows_per_src=[8192, 8192, 1102, 7888, 3890, 4250, 8188, 7720] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51824 payload_rows_remote=43762 dup_rows=13712 dup_rows_remote=11084 dedup_ratio=0.209229 router_tokens_per_expert=[0, 645, 440, 6, 3, 9, 4090, 5, 0, 0, 0, 9, 0, 1310, 0, 0, 0, 0, 4026, 63, 1, 4081, 0, 4038, 3087, 0, 2, 0, 145, 6, 104, 0, 0, 0, 2120, 1, 1, 0, 2719, 9, 4, 0, 1814, 2, 0, 0, 0, 8, 3, 0, 4, 6, 0, 0, 0, 0, 21, 1, 1, 0, 14, 21, 4, 3945] payload_rows_per_dst=[8192, 8062, 4000, 7582, 4246, 3652, 8178, 7912] dup_rows_per_dst=[6352, 2628, 56, 4544, 16, 4, 10, 102] combine_rows_per_src=[8192, 8062, 4000, 7582, 4246, 3652, 8178, 7912] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=314015744 dispatch_meta_bytes=229456 combine_payload_bytes=314015744 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=314245200 ingress_bytes=534216704 link_bytes=534216704 token_bytes=628260944 weight_movement_bytes=220200960 total_nvlink_bytes=848461904 weight_to_token_ratio=0.350493 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.290907118e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=338774016 dispatch_meta_bytes=224632 combine_payload_bytes=338774016 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=338998648 ingress_bytes=669075456 link_bytes=669075456 token_bytes=677772664 weight_movement_bytes=330301440 total_nvlink_bytes=1008074104 weight_to_token_ratio=0.487334 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=19072 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.163632262e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=311177216 dispatch_meta_bytes=223872 combine_payload_bytes=311177216 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=311401088 ingress_bytes=421277696 link_bytes=421277696 token_bytes=622578304 weight_movement_bytes=110100480 total_nvlink_bytes=732678784 weight_to_token_ratio=0.176846 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=30592 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.326712016e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=338157568 dispatch_meta_bytes=213248 combine_payload_bytes=338157568 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=338370816 ingress_bytes=448258048 link_bytes=448258048 token_bytes=676528384 weight_movement_bytes=110100480 total_nvlink_bytes=786628864 weight_to_token_ratio=0.162743 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=16896 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.795944713e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=313671680 dispatch_meta_bytes=229464 combine_payload_bytes=313671680 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=313901144 ingress_bytes=533872640 link_bytes=533872640 token_bytes=627572824 weight_movement_bytes=220200960 total_nvlink_bytes=847773784 weight_to_token_ratio=0.350877 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.284923279e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=309471232 dispatch_meta_bytes=224600 combine_payload_bytes=309471232 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=309695832 ingress_bytes=419571712 link_bytes=419571712 token_bytes=619167064 weight_movement_bytes=110100480 total_nvlink_bytes=729267544 weight_to_token_ratio=0.177820 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=30720 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.297042149e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=370485248 dispatch_meta_bytes=236752 combine_payload_bytes=370485248 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=370722000 ingress_bytes=480585728 link_bytes=480585728 token_bytes=741207248 weight_movement_bytes=110100480 total_nvlink_bytes=851307728 weight_to_token_ratio=0.148542 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=22016 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.358176238e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=313686016 dispatch_meta_bytes=219384 combine_payload_bytes=313686016 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=313905400 ingress_bytes=423786496 link_bytes=423786496 token_bytes=627591416 weight_movement_bytes=110100480 total_nvlink_bytes=737691896 weight_to_token_ratio=0.175433 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=25856 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.370344174e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49250 payload_rows_remote=41058 dup_rows=16286 dup_rows_remote=5688 dedup_ratio=0.248505 router_tokens_per_expert=[0, 532, 689, 0, 5, 0, 4096, 0, 0, 0, 0, 6, 0, 180, 0, 0, 1, 0, 4096, 11, 0, 4095, 0, 4073, 3504, 0, 0, 0, 103, 0, 90, 0, 0, 0, 2534, 0, 0, 0, 2876, 0, 6, 0, 1981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 1, 1, 25, 0, 3853] payload_rows_per_dst=[8192, 8192, 24, 7904, 5068, 3968, 8190, 7712] dup_rows_per_dst=[10598, 372, 0, 5242, 0, 6, 0, 68] combine_rows_per_src=[8192, 8192, 24, 7904, 5068, 3968, 8190, 7712] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56254 payload_rows_remote=48432 dup_rows=9282 dup_rows_remote=9226 dedup_ratio=0.141632 router_tokens_per_expert=[0, 4, 0, 0, 3, 5, 0, 4018, 0, 3396, 0, 0, 0, 3998, 298, 15, 0, 0, 26, 3909, 1, 0, 3, 0, 2709, 0, 2, 68, 0, 0, 5, 0, 0, 0, 1908, 0, 0, 4090, 0, 2418, 0, 4095, 0, 0, 2, 0, 0, 0, 0, 0, 10, 0, 1762, 0, 0, 0, 2, 0, 2, 0, 0, 0, 1, 18] payload_rows_per_dst=[8036, 8176, 7822, 5498, 6798, 8190, 3544, 8190] dup_rows_per_dst=[24, 7238, 56, 70, 1854, 4, 0, 36] combine_rows_per_src=[8036, 8176, 7822, 5498, 6798, 8190, 3544, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=294303744 dispatch_meta_bytes=186984 combine_payload_bytes=294303744 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=294490728 ingress_bytes=514504704 link_bytes=514504704 token_bytes=588794472 weight_movement_bytes=220200960 total_nvlink_bytes=808995432 weight_to_token_ratio=0.373986 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=16000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.948083017e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55036 payload_rows_remote=47356 dup_rows=10500 dup_rows_remote=8062 dedup_ratio=0.160217 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3662, 0, 3540, 0, 3, 0, 3647, 536, 105, 1, 0, 17, 3793, 0, 0, 0, 0, 3501, 0, 3, 2, 0, 1, 32, 0, 1, 0, 1793, 14, 1, 4084, 0, 2511, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 1392, 1, 0, 0, 0, 3, 6, 6, 2, 0, 1, 0] payload_rows_per_dst=[7324, 1276, 7598, 8080, 6728, 8182, 7680, 8168] dup_rows_per_dst=[0, 12, 24, 6078, 1912, 0, 2438, 36] combine_rows_per_src=[7324, 1276, 7598, 8080, 6728, 8182, 7680, 8168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=335634432 dispatch_meta_bytes=211880 combine_payload_bytes=335634432 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=335846312 ingress_bytes=445734912 link_bytes=445734912 token_bytes=671480744 weight_movement_bytes=110100480 total_nvlink_bytes=781581224 weight_to_token_ratio=0.163967 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=16640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.752063228e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=373008384 dispatch_meta_bytes=237048 combine_payload_bytes=373008384 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=373245432 ingress_bytes=483108864 link_bytes=483108864 token_bytes=746253816 weight_movement_bytes=110100480 total_nvlink_bytes=856354296 weight_to_token_ratio=0.147538 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=21376 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.402057723e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=319836160 dispatch_meta_bytes=219992 combine_payload_bytes=319836160 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=320056152 ingress_bytes=429936640 link_bytes=429936640 token_bytes=639892312 weight_movement_bytes=110100480 total_nvlink_bytes=749992792 weight_to_token_ratio=0.172061 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=25984 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.477305294e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=295536640 dispatch_meta_bytes=187632 combine_payload_bytes=295536640 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=295724272 ingress_bytes=515737600 link_bytes=515737600 token_bytes=591260912 weight_movement_bytes=220200960 total_nvlink_bytes=811461872 weight_to_token_ratio=0.372426 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=16768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.969525107e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56386 payload_rows_remote=48588 dup_rows=9150 dup_rows_remote=9096 dedup_ratio=0.139618 router_tokens_per_expert=[0, 4, 0, 0, 5, 4, 0, 4003, 0, 3263, 0, 0, 0, 3966, 339, 15, 0, 0, 25, 3897, 1, 0, 3, 0, 2777, 0, 2, 72, 0, 0, 6, 0, 0, 0, 1861, 0, 0, 4089, 0, 2554, 0, 4095, 0, 0, 2, 0, 0, 0, 0, 0, 7, 0, 1760, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 15] payload_rows_per_dst=[8006, 8176, 7798, 5636, 6856, 8190, 3534, 8190] dup_rows_per_dst=[26, 6990, 54, 78, 1974, 4, 0, 24] combine_rows_per_src=[8006, 8176, 7798, 5636, 6856, 8190, 3534, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57934 payload_rows_remote=50168 dup_rows=7602 dup_rows_remote=3920 dedup_ratio=0.115997 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4074, 0, 2310, 0, 0, 0, 3973, 178, 53, 0, 0, 0, 3740, 0, 0, 0, 0, 2831, 0, 0, 35, 0, 7, 1, 0, 0, 0, 2731, 0, 0, 3968, 0, 2993, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 6, 0, 1740, 0, 0, 0, 0, 0, 29, 0, 1, 0, 0, 0] payload_rows_per_dst=[8148, 4310, 7480, 5992, 7766, 8192, 8060, 7986] dup_rows_per_dst=[0, 82, 0, 446, 3682, 0, 3382, 10] combine_rows_per_src=[8148, 4310, 7480, 5992, 7766, 8192, 8060, 7986] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57432 payload_rows_remote=49640 dup_rows=8104 dup_rows_remote=4406 dedup_ratio=0.123657 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4073, 0, 2327, 0, 0, 0, 4003, 163, 56, 0, 0, 0, 3763, 0, 0, 0, 0, 2715, 0, 0, 36, 0, 7, 1, 0, 0, 0, 2847, 0, 0, 3968, 0, 2898, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 1783, 0, 0, 0, 0, 0, 27, 0, 1, 0, 0, 0] payload_rows_per_dst=[8146, 3590, 7526, 6120, 7792, 8192, 8080, 7986] dup_rows_per_dst=[0, 60, 0, 840, 3698, 0, 3500, 6] combine_rows_per_src=[8146, 3590, 7526, 6120, 7792, 8192, 8080, 7986] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56882 payload_rows_remote=48692 dup_rows=8654 dup_rows_remote=8654 dedup_ratio=0.132050 router_tokens_per_expert=[0, 0, 0, 1, 0, 0, 0, 3985, 0, 3013, 5, 2, 0, 4027, 1487, 6, 0, 0, 7, 3906, 0, 0, 0, 1, 2221, 0, 0, 9, 1, 3, 22, 0, 0, 0, 2680, 0, 0, 3949, 0, 1531, 0, 4094, 0, 0, 0, 0, 1, 0, 11, 0, 2, 0, 1796, 0, 0, 3, 0, 0, 4, 0, 1, 0, 0, 0] payload_rows_per_dst=[7972, 2988, 7824, 7434, 6410, 8190, 8158, 7906] dup_rows_per_dst=[0, 12, 4, 3104, 2012, 0, 3520, 2] combine_rows_per_src=[7972, 2988, 7824, 7434, 6410, 8190, 8158, 7906] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=347160576 dispatch_meta_bytes=230632 combine_payload_bytes=347160576 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=347391208 ingress_bytes=457261056 link_bytes=457261056 token_bytes=694551784 weight_movement_bytes=110100480 total_nvlink_bytes=804652264 weight_to_token_ratio=0.158520 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.029412 gemm_active_groups=10 gemm_peak_group_rows=30976 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.952521829e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56290 payload_rows_remote=48142 dup_rows=9246 dup_rows_remote=8966 dedup_ratio=0.141083 router_tokens_per_expert=[0, 0, 1, 0, 0, 1, 0, 3493, 0, 3602, 0, 2, 0, 3944, 1014, 31, 0, 0, 4, 3876, 0, 0, 0, 1, 1757, 0, 0, 22, 0, 20, 179, 0, 1, 0, 1547, 29, 0, 4063, 0, 3179, 0, 4092, 0, 0, 5, 0, 0, 2, 0, 0, 4, 0, 1861, 0, 5, 0, 0, 0, 0, 0, 25, 0, 8, 0] payload_rows_per_dst=[6988, 2080, 7754, 7848, 7220, 8192, 8060, 8148] dup_rows_per_dst=[2, 14, 8, 3312, 2026, 36, 3568, 280] combine_rows_per_src=[6988, 2080, 7754, 7848, 7220, 8192, 8060, 8148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56432 payload_rows_remote=48284 dup_rows=9104 dup_rows_remote=8816 dedup_ratio=0.138916 router_tokens_per_expert=[0, 0, 1, 0, 0, 1, 0, 3495, 0, 3512, 0, 1, 0, 3871, 1158, 28, 0, 0, 4, 3873, 0, 0, 0, 1, 1849, 0, 0, 22, 0, 19, 165, 0, 0, 0, 1401, 27, 0, 4062, 0, 3297, 0, 4092, 0, 0, 5, 0, 0, 1, 0, 0, 4, 0, 1841, 0, 5, 0, 0, 0, 1, 0, 25, 0, 7, 0] payload_rows_per_dst=[6990, 2362, 7748, 7782, 7222, 8192, 7988, 8148] dup_rows_per_dst=[4, 12, 8, 3352, 1952, 34, 3454, 288] combine_rows_per_src=[6990, 2362, 7748, 7782, 7222, 8192, 7988, 8148] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 1, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57226 payload_rows_remote=52506 dup_rows=8310 dup_rows_remote=8226 dedup_ratio=0.126801 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 4014, 0, 3496, 0, 0, 0, 3814, 554, 31, 0, 0, 0, 3809, 0, 0, 0, 5, 2337, 0, 0, 44, 0, 1, 18, 2, 0, 0, 2708, 1, 0, 4090, 0, 1810, 0, 4095, 0, 0, 0, 0, 0, 0, 2, 0, 20, 0, 1897, 3, 0, 0, 0, 0, 11, 1, 4, 0, 0, 0] payload_rows_per_dst=[8030, 7490, 7620, 4720, 6652, 8190, 6334, 8190] dup_rows_per_dst=[0, 3694, 8, 84, 2386, 0, 2116, 22] combine_rows_per_src=[8030, 7490, 7620, 4720, 6652, 8190, 6334, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55152 payload_rows_remote=47572 dup_rows=10384 dup_rows_remote=8116 dedup_ratio=0.158447 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3680, 0, 3493, 0, 3, 0, 3539, 580, 105, 1, 0, 15, 3773, 0, 0, 0, 0, 3541, 0, 3, 2, 0, 1, 35, 0, 1, 0, 1703, 10, 1, 4086, 0, 2703, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 1366, 1, 0, 0, 0, 2, 5, 6, 3, 0, 1, 0] payload_rows_per_dst=[7360, 1362, 7558, 8078, 6860, 8182, 7580, 8172] dup_rows_per_dst=[0, 14, 20, 6072, 1976, 0, 2268, 34] combine_rows_per_src=[7360, 1362, 7558, 8078, 6860, 8182, 7580, 8172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57048 payload_rows_remote=48858 dup_rows=8488 dup_rows_remote=8488 dedup_ratio=0.129517 router_tokens_per_expert=[0, 0, 0, 2, 0, 0, 0, 3987, 0, 2920, 4, 2, 0, 3985, 1628, 4, 0, 0, 7, 3905, 0, 0, 0, 1, 2297, 0, 0, 7, 1, 2, 16, 0, 0, 0, 2570, 0, 0, 3951, 0, 1640, 0, 4094, 0, 0, 0, 0, 1, 0, 10, 0, 2, 0, 1724, 0, 0, 3, 0, 0, 3, 0, 1, 0, 0, 1] payload_rows_per_dst=[7978, 3264, 7824, 7412, 6366, 8190, 8106, 7908] dup_rows_per_dst=[0, 12, 2, 3074, 2054, 0, 3342, 4] combine_rows_per_src=[7978, 3264, 7824, 7412, 6366, 8190, 8106, 7908] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=348278784 dispatch_meta_bytes=230736 combine_payload_bytes=348278784 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=348509520 ingress_bytes=458379264 link_bytes=458379264 token_bytes=696788304 weight_movement_bytes=110100480 total_nvlink_bytes=806888784 weight_to_token_ratio=0.158011 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=30848 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.971969305e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=339447808 dispatch_meta_bytes=221672 combine_payload_bytes=339447808 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=339669480 ingress_bytes=449548288 link_bytes=449548288 token_bytes=679117288 weight_movement_bytes=110100480 total_nvlink_bytes=789217768 weight_to_token_ratio=0.162123 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=18048 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.818384108e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57138 payload_rows_remote=49020 dup_rows=8398 dup_rows_remote=3044 dedup_ratio=0.128143 router_tokens_per_expert=[0, 5, 0, 0, 1, 0, 2, 3968, 0, 2860, 0, 12, 0, 3548, 306, 10, 0, 16, 6, 3846, 1, 2, 0, 1, 3148, 0, 0, 31, 0, 1, 0, 0, 0, 0, 3090, 0, 6, 4061, 0, 1494, 0, 4081, 0, 1, 0, 1, 0, 1, 16, 6, 33, 1, 2145, 4, 1, 0, 1, 10, 12, 1, 6, 0, 30, 3] payload_rows_per_dst=[7946, 8118, 7760, 6330, 6284, 8168, 4380, 8152] dup_rows_per_dst=[6, 5354, 1262, 30, 1618, 0, 32, 96] combine_rows_per_src=[7946, 8118, 7760, 6330, 6284, 8168, 4380, 8152] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=355819520 dispatch_meta_bytes=216184 combine_payload_bytes=355819520 prefetch_bytes=0 reduce_bytes=660602880 egress_bytes=356035704 ingress_bytes=1016422400 link_bytes=1016422400 token_bytes=711855224 weight_movement_bytes=660602880 total_nvlink_bytes=1372458104 weight_to_token_ratio=0.928002 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=16896 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.767725726e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=349024256 dispatch_meta_bytes=229384 combine_payload_bytes=349024256 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=349253640 ingress_bytes=459124736 link_bytes=459124736 token_bytes=698277896 weight_movement_bytes=110100480 total_nvlink_bytes=808378376 weight_to_token_ratio=0.157674 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.984934289e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=345081856 dispatch_meta_bytes=228432 combine_payload_bytes=345081856 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=345310288 ingress_bytes=565282816 link_bytes=565282816 token_bytes=690392144 weight_movement_bytes=220200960 total_nvlink_bytes=910593104 weight_to_token_ratio=0.318951 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.831197901e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58084 payload_rows_remote=49994 dup_rows=7452 dup_rows_remote=2532 dedup_ratio=0.113708 router_tokens_per_expert=[0, 5, 0, 0, 2, 0, 2, 3977, 0, 2703, 0, 12, 0, 3412, 368, 10, 0, 15, 5, 3828, 1, 1, 0, 1, 3210, 0, 0, 24, 0, 1, 0, 0, 0, 0, 3051, 0, 6, 4063, 0, 1716, 0, 4081, 1, 1, 0, 0, 0, 2, 18, 3, 36, 1, 2146, 4, 2, 0, 1, 10, 9, 1, 6, 0, 30, 3] payload_rows_per_dst=[7964, 8090, 7682, 6450, 7180, 8170, 4388, 8160] dup_rows_per_dst=[8, 4920, 306, 20, 2080, 0, 32, 86] combine_rows_per_src=[7964, 8090, 7682, 6450, 7180, 8170, 4388, 8160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=359604224 dispatch_meta_bytes=216352 combine_payload_bytes=359604224 prefetch_bytes=0 reduce_bytes=660602880 egress_bytes=359820576 ingress_bytes=1020207104 link_bytes=1020207104 token_bytes=719424800 weight_movement_bytes=660602880 total_nvlink_bytes=1380027680 weight_to_token_ratio=0.918238 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=16640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.774307949e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57680 payload_rows_remote=52916 dup_rows=7856 dup_rows_remote=7772 dedup_ratio=0.119873 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 4022, 0, 3430, 0, 0, 0, 3724, 586, 31, 0, 0, 0, 3807, 0, 0, 0, 7, 2359, 0, 0, 41, 0, 0, 22, 2, 0, 0, 2624, 2, 0, 4090, 0, 2015, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 15, 0, 1878, 0, 0, 0, 0, 0, 11, 1, 4, 0, 0, 0] payload_rows_per_dst=[8046, 7328, 7618, 4764, 6708, 8190, 6836, 8190] dup_rows_per_dst=[0, 2726, 10, 84, 2574, 0, 2440, 22] combine_rows_per_src=[8046, 7328, 7618, 4764, 6708, 8190, 6836, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51382 payload_rows_remote=43280 dup_rows=14154 dup_rows_remote=11250 dedup_ratio=0.215973 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4019, 0, 2698, 0, 0, 0, 3981, 437, 26, 0, 0, 8, 3977, 0, 0, 0, 0, 3042, 0, 0, 97, 0, 2, 7, 0, 0, 0, 2429, 0, 0, 4089, 0, 1729, 0, 4096, 0, 0, 0, 0, 0, 0, 31, 0, 10, 0, 2022, 0, 0, 0, 0, 0, 39, 0, 0, 0, 28, 1] payload_rows_per_dst=[8102, 8182, 8126, 6194, 286, 8192, 4116, 8184] dup_rows_per_dst=[2904, 7706, 3302, 102, 0, 0, 10, 130] combine_rows_per_src=[8102, 8182, 8126, 6194, 286, 8192, 4116, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=346099712 dispatch_meta_bytes=228400 combine_payload_bytes=346099712 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=346328112 ingress_bytes=566300672 link_bytes=566300672 token_bytes=692427824 weight_movement_bytes=220200960 total_nvlink_bytes=912628784 weight_to_token_ratio=0.318013 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.848900091e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=376363008 dispatch_meta_bytes=242928 combine_payload_bytes=376363008 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=376605936 ingress_bytes=486463488 link_bytes=486463488 token_bytes=752968944 weight_movement_bytes=110100480 total_nvlink_bytes=863069424 weight_to_token_ratio=0.146222 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=21376 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.460400151e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52938 payload_rows_remote=44820 dup_rows=12598 dup_rows_remote=9654 dedup_ratio=0.192230 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4023, 0, 2841, 0, 0, 0, 4025, 396, 29, 0, 0, 7, 3972, 0, 0, 0, 0, 2884, 0, 0, 101, 0, 2, 7, 0, 0, 0, 2664, 0, 0, 4088, 0, 1499, 0, 4096, 0, 0, 0, 0, 0, 0, 36, 0, 4, 0, 2023, 0, 0, 0, 0, 0, 44, 0, 0, 0, 27, 0] payload_rows_per_dst=[8118, 8190, 8118, 5874, 2146, 8192, 4118, 8182] dup_rows_per_dst=[2944, 7244, 2152, 114, 0, 0, 8, 136] combine_rows_per_src=[8118, 8190, 8118, 5874, 2146, 8192, 4118, 8182] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=351375360 dispatch_meta_bytes=208256 combine_payload_bytes=351375360 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=351583616 ingress_bytes=681676800 link_bytes=681676800 token_bytes=702958976 weight_movement_bytes=330301440 total_nvlink_bytes=1033260416 weight_to_token_ratio=0.469873 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=14208 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.185548072e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59414 payload_rows_remote=51226 dup_rows=6122 dup_rows_remote=6116 dedup_ratio=0.093414 router_tokens_per_expert=[4087, 0, 0, 0, 3012, 3176, 1, 0, 0, 0, 0, 2, 4093, 11, 0, 2049, 0, 1, 0, 4093, 0, 0, 3, 0, 6, 3793, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 6, 0, 0, 0, 0, 0, 3944, 3703, 0, 775, 0, 5, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7994, 4118, 8188, 7592, 7964, 8192, 7190, 8176] dup_rows_per_dst=[4384, 6, 6, 18, 604, 8, 1088, 8] combine_rows_per_src=[7994, 4118, 8188, 7592, 7964, 8192, 7190, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=350214144 dispatch_meta_bytes=229384 combine_payload_bytes=350214144 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=350443528 ingress_bytes=460314624 link_bytes=460314624 token_bytes=700657672 weight_movement_bytes=110100480 total_nvlink_bytes=810758152 weight_to_token_ratio=0.157139 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.005628399e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=340996096 dispatch_meta_bytes=222752 combine_payload_bytes=340996096 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=341218848 ingress_bytes=451096576 link_bytes=451096576 token_bytes=682214944 weight_movement_bytes=110100480 total_nvlink_bytes=792315424 weight_to_token_ratio=0.161387 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=18176 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.845311383e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=358356992 dispatch_meta_bytes=210104 combine_payload_bytes=358356992 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=358567096 ingress_bytes=688658432 link_bytes=688658432 token_bytes=716924088 weight_movement_bytes=330301440 total_nvlink_bytes=1047225528 weight_to_token_ratio=0.460720 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=14080 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.197690278e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=379301888 dispatch_meta_bytes=242752 combine_payload_bytes=379301888 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=379544640 ingress_bytes=489402368 link_bytes=489402368 token_bytes=758846528 weight_movement_bytes=110100480 total_nvlink_bytes=868947008 weight_to_token_ratio=0.145089 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=22016 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.511512108e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=310231040 dispatch_meta_bytes=218120 combine_payload_bytes=310231040 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=310449160 ingress_bytes=420331520 link_bytes=420331520 token_bytes=620680200 weight_movement_bytes=110100480 total_nvlink_bytes=730780680 weight_to_token_ratio=0.177387 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=31360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.310256459e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60450 payload_rows_remote=52264 dup_rows=5086 dup_rows_remote=5082 dedup_ratio=0.077606 router_tokens_per_expert=[4087, 0, 0, 0, 2887, 3148, 1, 0, 0, 0, 0, 2, 4093, 18, 0, 2250, 0, 1, 0, 4091, 0, 0, 3, 0, 6, 3841, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 6, 0, 0, 0, 0, 0, 3956, 3783, 0, 579, 0, 8, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7996, 4528, 8186, 7690, 7912, 8192, 7764, 8182] dup_rows_per_dst=[4076, 12, 4, 16, 2, 8, 960, 8] combine_rows_per_src=[7996, 4528, 8186, 7690, 7912, 8192, 7764, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59814 payload_rows_remote=51878 dup_rows=5722 dup_rows_remote=5722 dedup_ratio=0.087311 router_tokens_per_expert=[4096, 0, 0, 0, 1705, 3607, 0, 0, 0, 29, 0, 0, 4096, 44, 0, 2269, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 3926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3968, 3955, 0, 971, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[3410, 7972, 8190, 7852, 7936, 8192, 8070, 8192] dup_rows_per_dst=[0, 3926, 0, 0, 0, 14, 1782, 0] combine_rows_per_src=[3410, 7972, 8190, 7852, 7936, 8192, 8070, 8192] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59778 payload_rows_remote=52062 dup_rows=5758 dup_rows_remote=4002 dedup_ratio=0.087860 router_tokens_per_expert=[4092, 0, 0, 0, 2003, 3243, 1, 0, 1, 0, 0, 8, 4095, 15, 0, 2666, 0, 0, 0, 4087, 0, 1, 0, 6, 0, 3761, 7, 0, 2, 9, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 1, 0, 0, 0, 0, 3, 5, 0, 4004, 3383, 3, 1342, 0, 6, 0, 0, 3, 0, 0, 0] payload_rows_per_dst=[4008, 7948, 8182, 7536, 8008, 8192, 7716, 8188] dup_rows_per_dst=[0, 3918, 6, 22, 4, 38, 1756, 14] combine_rows_per_src=[4008, 7948, 8182, 7536, 8008, 8192, 7716, 8188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=321269760 dispatch_meta_bytes=217896 combine_payload_bytes=321269760 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=321487656 ingress_bytes=431370240 link_bytes=431370240 token_bytes=642757416 weight_movement_bytes=110100480 total_nvlink_bytes=752857896 weight_to_token_ratio=0.171294 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=31360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.502237956e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59454 payload_rows_remote=51262 dup_rows=6082 dup_rows_remote=6060 dedup_ratio=0.092804 router_tokens_per_expert=[4096, 0, 0, 0, 2242, 2488, 0, 0, 0, 3, 0, 0, 4094, 7, 0, 2468, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 3803, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 6, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 3790, 3359, 0, 2302, 0, 0, 0, 0, 0, 0, 0, 1] payload_rows_per_dst=[4484, 7414, 8190, 7608, 7582, 8192, 7792, 8192] dup_rows_per_dst=[0, 2518, 2, 4, 0, 22, 3534, 2] combine_rows_per_src=[4484, 7414, 8190, 7608, 7582, 8192, 7792, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59882 payload_rows_remote=51962 dup_rows=5654 dup_rows_remote=5654 dedup_ratio=0.086273 router_tokens_per_expert=[4096, 0, 0, 0, 1792, 3566, 0, 0, 0, 27, 0, 0, 4096, 26, 0, 2071, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 3908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3960, 3953, 0, 1174, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[3584, 7902, 8186, 7816, 7920, 8192, 8090, 8192] dup_rows_per_dst=[0, 3478, 0, 0, 0, 12, 2164, 0] combine_rows_per_src=[3584, 7902, 8186, 7816, 7920, 8192, 8090, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=367187968 dispatch_meta_bytes=229368 combine_payload_bytes=367187968 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=367417336 ingress_bytes=477288448 link_bytes=477288448 token_bytes=734605304 weight_movement_bytes=110100480 total_nvlink_bytes=844705784 weight_to_token_ratio=0.149877 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.300831115e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61160 payload_rows_remote=52996 dup_rows=4376 dup_rows_remote=4336 dedup_ratio=0.066772 router_tokens_per_expert=[4080, 0, 0, 0, 3192, 3421, 0, 9, 2, 0, 0, 3, 4090, 12, 0, 1825, 0, 0, 1, 4079, 0, 0, 0, 0, 0, 3553, 28, 0, 0, 69, 0, 1, 0, 0, 2, 11, 0, 0, 1, 0, 2, 1, 7, 5, 0, 0, 0, 0, 0, 7, 0, 1, 3794, 3718, 4, 848, 2, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6358, 7758, 8160, 7142, 7594, 8188, 7796, 8164] dup_rows_per_dst=[4, 2768, 0, 160, 22, 22, 1360, 40] combine_rows_per_src=[6358, 7758, 8160, 7142, 7594, 8188, 7796, 8164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61080 payload_rows_remote=52920 dup_rows=4456 dup_rows_remote=4416 dedup_ratio=0.067993 router_tokens_per_expert=[4079, 0, 0, 0, 3044, 3337, 0, 9, 2, 0, 0, 4, 4090, 16, 0, 2098, 0, 0, 1, 4078, 0, 0, 0, 0, 0, 3570, 25, 0, 0, 74, 0, 1, 0, 0, 1, 11, 0, 0, 1, 0, 2, 1, 6, 4, 0, 0, 0, 0, 0, 7, 0, 1, 3832, 3774, 5, 693, 1, 0, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[6066, 7830, 8158, 7178, 7670, 8188, 7830, 8160] dup_rows_per_dst=[2, 3084, 0, 162, 20, 18, 1130, 40] combine_rows_per_src=[6066, 7830, 8158, 7178, 7670, 8188, 7830, 8160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59618 payload_rows_remote=51428 dup_rows=5918 dup_rows_remote=5898 dedup_ratio=0.090302 router_tokens_per_expert=[4096, 0, 0, 0, 2102, 2499, 1, 0, 0, 2, 0, 0, 4094, 7, 0, 2774, 1, 0, 0, 4094, 0, 0, 0, 0, 0, 3845, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 5, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 3852, 3508, 0, 1875, 0, 0, 0, 0, 0, 0, 0, 1] payload_rows_per_dst=[4206, 7666, 8188, 7692, 7706, 8190, 7778, 8192] dup_rows_per_dst=[0, 2898, 2, 4, 0, 20, 2992, 2] combine_rows_per_src=[4206, 7666, 8188, 7692, 7706, 8190, 7778, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59914 payload_rows_remote=52198 dup_rows=5622 dup_rows_remote=4204 dedup_ratio=0.085785 router_tokens_per_expert=[4092, 0, 0, 0, 1986, 3233, 1, 0, 1, 0, 0, 7, 4095, 16, 0, 2813, 0, 0, 0, 4087, 0, 1, 0, 4, 0, 3790, 7, 0, 3, 11, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 1, 0, 0, 0, 0, 2, 4, 0, 4026, 3529, 3, 1029, 0, 4, 0, 0, 4, 0, 0, 0] payload_rows_per_dst=[3974, 8012, 8182, 7596, 8052, 8192, 7716, 8190] dup_rows_per_dst=[0, 4128, 2, 26, 4, 34, 1418, 10] combine_rows_per_src=[3974, 8012, 8182, 7596, 8052, 8192, 7716, 8190] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=373180416 dispatch_meta_bytes=224256 combine_payload_bytes=373180416 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=373404672 ingress_bytes=923682816 link_bytes=923682816 token_bytes=746585088 weight_movement_bytes=550502400 total_nvlink_bytes=1297087488 weight_to_token_ratio=0.737361 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=21632 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.606436337e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60738 payload_rows_remote=53118 dup_rows=4798 dup_rows_remote=4788 dedup_ratio=0.073212 router_tokens_per_expert=[4095, 0, 0, 0, 2313, 3664, 0, 0, 0, 6, 0, 0, 4094, 34, 0, 1618, 0, 6, 1, 4092, 0, 0, 0, 0, 0, 3810, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 3803, 2, 1163, 0, 4, 0, 0, 0, 0, 1, 2] payload_rows_per_dst=[5838, 6676, 8186, 7620, 8080, 8188, 7960, 8190] dup_rows_per_dst=[476, 2280, 12, 10, 4, 26, 1976, 14] combine_rows_per_src=[5838, 6676, 8186, 7620, 8080, 8188, 7960, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=367446016 dispatch_meta_bytes=229288 combine_payload_bytes=367446016 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=367675304 ingress_bytes=477546496 link_bytes=477546496 token_bytes=735121320 weight_movement_bytes=110100480 total_nvlink_bytes=845221800 weight_to_token_ratio=0.149772 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.305318994e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=374628352 dispatch_meta_bytes=229384 combine_payload_bytes=374628352 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=374857736 ingress_bytes=484728832 link_bytes=484728832 token_bytes=749486088 weight_movement_bytes=110100480 total_nvlink_bytes=859586568 weight_to_token_ratio=0.146901 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.430231630e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58314 payload_rows_remote=53160 dup_rows=7222 dup_rows_remote=7222 dedup_ratio=0.110199 router_tokens_per_expert=[4079, 0, 0, 0, 1467, 3345, 0, 0, 0, 9, 0, 0, 4091, 13, 4, 2551, 0, 7, 1, 4076, 0, 0, 4, 4, 0, 3392, 21, 0, 0, 1, 0, 1, 0, 0, 0, 17, 2, 1, 0, 0, 9, 1, 2, 0, 0, 0, 10, 1, 0, 8, 0, 0, 4053, 3949, 0, 1647, 0, 0, 0, 0, 0, 0, 0, 2] payload_rows_per_dst=[7638, 5154, 8160, 7708, 8138, 8190, 5164, 8162] dup_rows_per_dst=[1986, 0, 24, 2570, 1912, 38, 692, 0] combine_rows_per_src=[7638, 5154, 8160, 7708, 8138, 8190, 5164, 8162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56860 payload_rows_remote=52096 dup_rows=8676 dup_rows_remote=8676 dedup_ratio=0.132385 router_tokens_per_expert=[4079, 0, 0, 0, 1525, 3325, 0, 0, 0, 7, 0, 0, 4093, 9, 4, 2362, 0, 6, 1, 4081, 0, 0, 4, 4, 0, 3318, 22, 0, 0, 1, 0, 1, 0, 0, 0, 17, 2, 1, 0, 0, 8, 1, 2, 0, 0, 0, 11, 1, 0, 8, 0, 0, 4049, 3896, 0, 1928, 0, 0, 0, 1, 0, 0, 0, 1] payload_rows_per_dst=[7606, 4764, 8170, 7976, 8134, 8190, 3858, 8162] dup_rows_per_dst=[2094, 0, 22, 5002, 1502, 42, 14, 0] combine_rows_per_src=[7606, 4764, 8170, 7976, 8134, 8190, 3858, 8162] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=372463616 dispatch_meta_bytes=230464 combine_payload_bytes=372463616 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=372694080 ingress_bytes=592664576 link_bytes=592664576 token_bytes=745157696 weight_movement_bytes=220200960 total_nvlink_bytes=965358656 weight_to_token_ratio=0.295509 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.029412 gemm_active_groups=10 gemm_peak_group_rows=31744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.030741174e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=379875328 dispatch_meta_bytes=229328 combine_payload_bytes=379875328 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=380104656 ingress_bytes=600076288 link_bytes=600076288 token_bytes=759979984 weight_movement_bytes=220200960 total_nvlink_bytes=980180944 weight_to_token_ratio=0.289746 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.043631360e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=371861504 dispatch_meta_bytes=230400 combine_payload_bytes=371861504 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=372091904 ingress_bytes=592062464 link_bytes=592062464 token_bytes=743953408 weight_movement_bytes=220200960 total_nvlink_bytes=964154368 weight_to_token_ratio=0.295988 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.029412 gemm_active_groups=10 gemm_peak_group_rows=31872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.029694002e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60234 payload_rows_remote=52568 dup_rows=5302 dup_rows_remote=5290 dedup_ratio=0.080902 router_tokens_per_expert=[4096, 0, 0, 0, 2174, 3714, 0, 0, 0, 7, 0, 0, 4094, 47, 0, 1849, 0, 6, 2, 4092, 0, 0, 0, 0, 0, 3833, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 3846, 2, 946, 0, 4, 0, 0, 0, 0, 1, 1] payload_rows_per_dst=[7148, 4818, 8188, 7666, 8066, 8188, 7968, 8192] dup_rows_per_dst=[2418, 1198, 12, 12, 4, 26, 1620, 12] combine_rows_per_src=[7148, 4818, 8188, 7666, 8066, 8188, 7968, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54910 payload_rows_remote=46798 dup_rows=10626 dup_rows_remote=2122 dedup_ratio=0.162140 router_tokens_per_expert=[4096, 0, 0, 0, 1913, 3583, 0, 0, 0, 0, 0, 0, 4096, 65, 0, 2015, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 4001, 0, 1326, 0, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8112, 4250, 8192, 7352, 8114, 8046, 2652, 8192] dup_rows_per_dst=[8504, 60, 48, 2012, 0, 0, 0, 2] combine_rows_per_src=[8112, 4250, 8192, 7352, 8114, 8046, 2652, 8192] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=380749824 dispatch_meta_bytes=231624 combine_payload_bytes=380749824 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=380981448 ingress_bytes=490850304 link_bytes=490850304 token_bytes=761731272 weight_movement_bytes=110100480 total_nvlink_bytes=871831752 weight_to_token_ratio=0.144540 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=29440 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.536694097e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54810 payload_rows_remote=46630 dup_rows=10726 dup_rows_remote=200 dedup_ratio=0.163666 router_tokens_per_expert=[4096, 0, 0, 0, 1950, 3489, 0, 0, 0, 0, 0, 0, 4096, 56, 0, 1855, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4059, 3999, 0, 1621, 0, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8180, 3934, 8192, 6920, 8118, 8032, 3242, 8192] dup_rows_per_dst=[10526, 54, 34, 110, 0, 0, 0, 2] combine_rows_per_src=[8180, 3934, 8192, 6920, 8118, 8032, 3242, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=373424128 dispatch_meta_bytes=243088 combine_payload_bytes=373424128 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=373667216 ingress_bytes=593625088 link_bytes=593625088 token_bytes=747091344 weight_movement_bytes=220200960 total_nvlink_bytes=967292304 weight_to_token_ratio=0.294744 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=17024 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.032411663e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59900 payload_rows_remote=55744 dup_rows=5636 dup_rows_remote=5208 dedup_ratio=0.085999 router_tokens_per_expert=[0, 3063, 0, 0, 0, 0, 0, 0, 4077, 0, 2, 0, 1, 4092, 0, 0, 0, 0, 1125, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 314, 3, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3846, 2, 0, 3790, 0, 1, 0, 0, 0, 1, 1, 0, 0, 155, 0, 0, 4096, 0, 2, 0, 0] payload_rows_per_dst=[7674, 8184, 4156, 7648, 8192, 7694, 8160, 8192] dup_rows_per_dst=[3806, 510, 428, 572, 4, 2, 0, 314] combine_rows_per_src=[7674, 8184, 4156, 7648, 8192, 7694, 8160, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=368635904 dispatch_meta_bytes=229304 combine_payload_bytes=368635904 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=368865208 ingress_bytes=478736384 link_bytes=478736384 token_bytes=737501112 weight_movement_bytes=110100480 total_nvlink_bytes=847601592 weight_to_token_ratio=0.149289 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.326013104e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=374155264 dispatch_meta_bytes=225608 combine_payload_bytes=374155264 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=374380872 ingress_bytes=924657664 link_bytes=924657664 token_bytes=748536136 weight_movement_bytes=550502400 total_nvlink_bytes=1299038536 weight_to_token_ratio=0.735439 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=23680 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.608131758e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=379330560 dispatch_meta_bytes=229344 combine_payload_bytes=379330560 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=379559904 ingress_bytes=599531520 link_bytes=599531520 token_bytes=758890464 weight_movement_bytes=220200960 total_nvlink_bytes=979091424 weight_to_token_ratio=0.290162 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.042683919e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59898 payload_rows_remote=55862 dup_rows=5638 dup_rows_remote=5266 dedup_ratio=0.086029 router_tokens_per_expert=[0, 3003, 0, 0, 0, 0, 0, 0, 4076, 0, 1, 0, 1, 4093, 0, 0, 0, 0, 1142, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 328, 2, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3861, 3, 0, 3813, 0, 0, 0, 0, 0, 2, 1, 0, 0, 146, 0, 0, 4096, 0, 1, 0, 0] payload_rows_per_dst=[7706, 8186, 4036, 7702, 8192, 7726, 8158, 8192] dup_rows_per_dst=[3876, 496, 372, 596, 2, 2, 0, 294] combine_rows_per_src=[7706, 8186, 4036, 7702, 8192, 7726, 8158, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59846 payload_rows_remote=51654 dup_rows=5690 dup_rows_remote=5690 dedup_ratio=0.086823 router_tokens_per_expert=[0, 2860, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4035, 0, 0, 0, 0, 1457, 0, 0, 4096, 0, 0, 0, 0, 10, 0, 1384, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2689, 0, 0, 3812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[5720, 8070, 8192, 7910, 8192, 5378, 8192, 8192] dup_rows_per_dst=[0, 0, 2914, 2502, 0, 0, 0, 274] combine_rows_per_src=[5720, 8070, 8192, 7910, 8192, 5378, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=381050880 dispatch_meta_bytes=241528 combine_payload_bytes=381050880 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=381292408 ingress_bytes=601251840 link_bytes=601251840 token_bytes=762343288 weight_movement_bytes=220200960 total_nvlink_bytes=982544248 weight_to_token_ratio=0.288848 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=18688 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.045675839e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61840 payload_rows_remote=53652 dup_rows=3696 dup_rows_remote=3696 dedup_ratio=0.056396 router_tokens_per_expert=[0, 2686, 1, 0, 0, 0, 0, 0, 4089, 0, 1, 0, 0, 4069, 0, 0, 0, 0, 1196, 0, 0, 4094, 2, 0, 0, 0, 62, 0, 565, 0, 0, 0, 0, 4093, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3867, 5, 0, 3876, 0, 5, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 4096, 0, 2, 0, 0] payload_rows_per_dst=[5374, 8140, 8192, 7814, 8188, 7752, 8188, 8192] dup_rows_per_dst=[0, 0, 2392, 1136, 0, 48, 0, 120] combine_rows_per_src=[5374, 8140, 8192, 7814, 8188, 7752, 8188, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59958 payload_rows_remote=52266 dup_rows=5578 dup_rows_remote=5578 dedup_ratio=0.085114 router_tokens_per_expert=[0, 1853, 1, 0, 1, 0, 0, 0, 4093, 3, 0, 0, 0, 4046, 0, 0, 0, 0, 2054, 0, 0, 4093, 1, 1, 1, 0, 19, 0, 726, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3844, 2, 0, 3739, 0, 0, 2, 0, 3, 0, 0, 0, 0, 94, 0, 0, 4094, 0, 3, 0, 0] payload_rows_per_dst=[3706, 8098, 8188, 7702, 8190, 7692, 8190, 8192] dup_rows_per_dst=[4, 0, 4110, 1268, 0, 0, 6, 190] combine_rows_per_src=[3706, 8098, 8188, 7702, 8190, 7692, 8190, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59802 payload_rows_remote=51610 dup_rows=5734 dup_rows_remote=5734 dedup_ratio=0.087494 router_tokens_per_expert=[0, 2868, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4029, 0, 0, 0, 0, 1499, 0, 0, 4096, 0, 0, 0, 0, 7, 0, 1379, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2694, 0, 0, 3772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[5736, 8058, 8192, 7852, 8192, 5388, 8192, 8192] dup_rows_per_dst=[0, 0, 2998, 2464, 0, 0, 0, 272] combine_rows_per_src=[5736, 8058, 8192, 7852, 8192, 5388, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=334243840 dispatch_meta_bytes=187320 combine_payload_bytes=334243840 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=334431160 ingress_bytes=774645760 link_bytes=774645760 token_bytes=668675000 weight_movement_bytes=440401920 total_nvlink_bytes=1109076920 weight_to_token_ratio=0.658619 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=18048 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.347236384e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55926 payload_rows_remote=47742 dup_rows=9610 dup_rows_remote=9162 dedup_ratio=0.146637 router_tokens_per_expert=[0, 3190, 3, 15, 0, 0, 0, 0, 4074, 4, 0, 0, 3, 4088, 0, 6, 0, 0, 1361, 0, 0, 4096, 0, 0, 5, 0, 16, 0, 360, 0, 2, 0, 0, 4089, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3782, 0, 5, 3491, 0, 13, 0, 2, 0, 0, 1, 0, 0, 69, 0, 0, 4083, 0, 7, 0, 1] payload_rows_per_dst=[6398, 8096, 7948, 754, 8180, 8188, 8178, 8184] dup_rows_per_dst=[18, 18, 2644, 12, 12, 6368, 90, 448] combine_rows_per_src=[6398, 8096, 7948, 754, 8180, 8188, 8178, 8184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=399572992 dispatch_meta_bytes=243808 combine_payload_bytes=399572992 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=399816800 ingress_bytes=839974912 link_bytes=839974912 token_bytes=799389792 weight_movement_bytes=440401920 total_nvlink_bytes=1239791712 weight_to_token_ratio=0.550923 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=21504 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.460854524e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=376807424 dispatch_meta_bytes=231432 combine_payload_bytes=376807424 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=377038856 ingress_bytes=486907904 link_bytes=486907904 token_bytes=753846280 weight_movement_bytes=110100480 total_nvlink_bytes=863946760 weight_to_token_ratio=0.146052 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=29824 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.468129276e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=335448064 dispatch_meta_bytes=195680 combine_payload_bytes=335448064 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=335643744 ingress_bytes=775849984 link_bytes=775849984 token_bytes=671091808 weight_movement_bytes=440401920 total_nvlink_bytes=1111493728 weight_to_token_ratio=0.656247 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=17280 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.349330728e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=400418816 dispatch_meta_bytes=244512 combine_payload_bytes=400418816 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=400663328 ingress_bytes=730720256 link_bytes=730720256 token_bytes=801082144 weight_movement_bytes=330301440 total_nvlink_bytes=1131383584 weight_to_token_ratio=0.412319 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=21632 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.270842708e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=370255872 dispatch_meta_bytes=229376 combine_payload_bytes=370255872 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=370485248 ingress_bytes=480356352 link_bytes=480356352 token_bytes=740741120 weight_movement_bytes=110100480 total_nvlink_bytes=850841600 weight_to_token_ratio=0.148636 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.354187012e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60004 payload_rows_remote=52306 dup_rows=5532 dup_rows_remote=5532 dedup_ratio=0.084412 router_tokens_per_expert=[0, 1902, 1, 0, 1, 0, 0, 0, 4093, 3, 0, 0, 0, 4048, 0, 0, 0, 0, 2004, 0, 0, 4090, 1, 1, 1, 0, 35, 0, 759, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3847, 2, 0, 3689, 0, 0, 2, 0, 3, 0, 0, 0, 0, 94, 0, 0, 4094, 0, 3, 0, 0] payload_rows_per_dst=[3804, 8102, 8182, 7646, 8190, 7698, 8190, 8192] dup_rows_per_dst=[4, 0, 4010, 1322, 0, 0, 6, 190] combine_rows_per_src=[3804, 8102, 8182, 7646, 8190, 7698, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61136 payload_rows_remote=52946 dup_rows=4400 dup_rows_remote=4400 dedup_ratio=0.067139 router_tokens_per_expert=[0, 2624, 1, 0, 0, 0, 0, 0, 4089, 0, 1, 0, 0, 4072, 0, 0, 0, 0, 1238, 0, 0, 4095, 2, 0, 0, 0, 67, 0, 577, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3854, 3, 0, 3888, 0, 6, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 4096, 0, 2, 0, 0] payload_rows_per_dst=[5250, 8146, 8192, 7198, 8186, 7782, 8190, 8192] dup_rows_per_dst=[0, 0, 2478, 1068, 0, 730, 0, 124] combine_rows_per_src=[5250, 8146, 8192, 7198, 8186, 7782, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56256 payload_rows_remote=48076 dup_rows=9280 dup_rows_remote=9138 dedup_ratio=0.141602 router_tokens_per_expert=[0, 3208, 3, 17, 0, 0, 0, 0, 4074, 4, 0, 0, 3, 4087, 0, 6, 0, 0, 1309, 0, 0, 4096, 0, 0, 5, 0, 17, 0, 385, 0, 2, 0, 0, 4089, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3785, 0, 6, 3492, 0, 15, 0, 1, 0, 0, 1, 0, 0, 72, 0, 0, 4082, 0, 6, 0, 1] payload_rows_per_dst=[6434, 8094, 8192, 812, 8180, 8184, 8180, 8180] dup_rows_per_dst=[22, 16, 2618, 6, 2, 6382, 92, 142] combine_rows_per_src=[6434, 8094, 8192, 812, 8180, 8184, 8180, 8180] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61940 payload_rows_remote=53868 dup_rows=3596 dup_rows_remote=1932 dedup_ratio=0.054871 router_tokens_per_expert=[0, 2907, 0, 0, 0, 1, 0, 0, 4071, 4, 0, 0, 0, 4055, 0, 0, 0, 0, 886, 0, 0, 4096, 0, 0, 0, 0, 93, 0, 785, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3607, 1, 2, 3990, 0, 0, 0, 1, 0, 0, 0, 0, 0, 74, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[5816, 8112, 8192, 8072, 8192, 7220, 8144, 8192] dup_rows_per_dst=[0, 6, 1772, 1664, 6, 0, 0, 148] combine_rows_per_src=[5816, 8112, 8192, 8072, 8192, 7220, 8144, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58338 payload_rows_remote=50218 dup_rows=7198 dup_rows_remote=7196 dedup_ratio=0.109833 router_tokens_per_expert=[0, 2944, 0, 0, 1, 0, 0, 1, 4094, 0, 0, 2, 1, 4058, 0, 0, 4, 0, 1749, 0, 0, 4077, 9, 5, 0, 2, 0, 0, 615, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 3, 0, 0, 5, 3174, 1, 0, 3740, 0, 1, 0, 1, 0, 0, 0, 2, 0, 80, 0, 9, 4094, 4, 0, 1, 0] payload_rows_per_dst=[8174, 8120, 3522, 7592, 8188, 6360, 8190, 8192] dup_rows_per_dst=[5872, 2, 12, 1122, 0, 0, 6, 184] combine_rows_per_src=[8174, 8120, 3522, 7592, 8188, 6360, 8190, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=384577536 dispatch_meta_bytes=229392 combine_payload_bytes=384577536 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=384806928 ingress_bytes=604778496 link_bytes=604778496 token_bytes=769384464 weight_movement_bytes=220200960 total_nvlink_bytes=989585424 weight_to_token_ratio=0.286204 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.029412 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.051809274e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=374642688 dispatch_meta_bytes=231376 combine_payload_bytes=374642688 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=374874064 ingress_bytes=594843648 link_bytes=594843648 token_bytes=749516752 weight_movement_bytes=220200960 total_nvlink_bytes=969717712 weight_to_token_ratio=0.293791 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=28288 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.034530939e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=369940480 dispatch_meta_bytes=229376 combine_payload_bytes=369940480 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=370169856 ingress_bytes=480040960 link_bytes=480040960 token_bytes=740110336 weight_movement_bytes=110100480 total_nvlink_bytes=850210816 weight_to_token_ratio=0.148762 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.348701826e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58294 payload_rows_remote=50184 dup_rows=7242 dup_rows_remote=7240 dedup_ratio=0.110504 router_tokens_per_expert=[0, 2923, 0, 0, 1, 0, 0, 2, 4095, 0, 0, 2, 1, 4053, 0, 0, 4, 0, 1725, 0, 0, 4078, 11, 5, 0, 2, 2, 0, 661, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 1, 0, 0, 5, 3141, 1, 0, 3774, 0, 0, 0, 1, 0, 0, 0, 1, 0, 80, 0, 8, 4093, 5, 0, 2, 0] payload_rows_per_dst=[8176, 8110, 3478, 7670, 8184, 6294, 8190, 8192] dup_rows_per_dst=[5832, 2, 12, 1208, 0, 0, 4, 184] combine_rows_per_src=[8176, 8110, 3478, 7670, 8184, 6294, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61822 payload_rows_remote=53738 dup_rows=3714 dup_rows_remote=1896 dedup_ratio=0.056671 router_tokens_per_expert=[0, 2829, 0, 0, 0, 0, 0, 0, 4070, 4, 0, 0, 1, 4056, 0, 0, 0, 0, 862, 0, 0, 4096, 0, 0, 0, 0, 101, 0, 861, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3621, 1, 2, 3989, 0, 0, 0, 1, 0, 0, 0, 0, 0, 79, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[5658, 8114, 8192, 8084, 8192, 7248, 8142, 8192] dup_rows_per_dst=[0, 8, 1724, 1818, 6, 0, 0, 158] combine_rows_per_src=[5658, 8114, 8192, 8084, 8192, 7248, 8142, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=374929408 dispatch_meta_bytes=231352 combine_payload_bytes=374929408 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=375160760 ingress_bytes=595130368 link_bytes=595130368 token_bytes=750090168 weight_movement_bytes=220200960 total_nvlink_bytes=970291128 weight_to_token_ratio=0.293566 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=28288 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.035029592e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=342214656 dispatch_meta_bytes=227616 combine_payload_bytes=342214656 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=342442272 ingress_bytes=562415616 link_bytes=562415616 token_bytes=684656928 weight_movement_bytes=220200960 total_nvlink_bytes=904857888 weight_to_token_ratio=0.321622 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=31872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.781332577e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=386125824 dispatch_meta_bytes=223200 combine_payload_bytes=386125824 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=386349024 ingress_bytes=496226304 link_bytes=496226304 token_bytes=772474848 weight_movement_bytes=110100480 total_nvlink_bytes=882575328 weight_to_token_ratio=0.142530 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=26880 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.630191579e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=359962624 dispatch_meta_bytes=229656 combine_payload_bytes=359962624 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=360192280 ingress_bytes=690264064 link_bytes=690264064 token_bytes=720154904 weight_movement_bytes=330301440 total_nvlink_bytes=1050456344 weight_to_token_ratio=0.458653 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.200482736e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57108 payload_rows_remote=48916 dup_rows=8428 dup_rows_remote=3662 dedup_ratio=0.128601 router_tokens_per_expert=[0, 2383, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1395, 0, 0, 4096, 0, 0, 0, 0, 4, 0, 993, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3472, 0, 0, 3935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 0, 4095, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 2790, 7972, 8192, 7176, 8192, 6402] dup_rows_per_dst=[4766, 0, 0, 1892, 0, 1648, 0, 122] combine_rows_per_src=[8192, 8192, 2790, 7972, 8192, 7176, 8192, 6402] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57454 payload_rows_remote=49278 dup_rows=8082 dup_rows_remote=7406 dedup_ratio=0.123322 router_tokens_per_expert=[0, 20, 0, 3969, 0, 0, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 953, 0, 2, 0, 0, 336, 4088, 0, 0, 0, 0, 0, 0, 4, 0, 1095, 3468, 0, 1620, 286, 0, 1, 0, 0, 44, 0, 2, 0, 0, 11, 0, 3967, 0, 0, 0, 0, 1, 0, 0, 3643, 1, 4095, 1, 1032, 0, 30, 3, 0, 0] payload_rows_per_dst=[7976, 8188, 8176, 7570, 8190, 7940, 7286, 2128] dup_rows_per_dst=[6, 1906, 676, 1564, 3902, 20, 4, 4] combine_rows_per_src=[7976, 8188, 8176, 7570, 8190, 7940, 7286, 2128] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=379516928 dispatch_meta_bytes=229384 combine_payload_bytes=379516928 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=379746312 ingress_bytes=599717888 link_bytes=599717888 token_bytes=759263240 weight_movement_bytes=220200960 total_nvlink_bytes=979464200 weight_to_token_ratio=0.290019 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.029412 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.043008044e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57302 payload_rows_remote=49110 dup_rows=8234 dup_rows_remote=3372 dedup_ratio=0.125641 router_tokens_per_expert=[0, 2431, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1386, 0, 0, 4096, 0, 0, 0, 0, 8, 0, 1046, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3375, 0, 0, 3943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 4095, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 2942, 7980, 8192, 6906, 8192, 6706] dup_rows_per_dst=[4862, 0, 2, 2014, 18, 1206, 0, 132] combine_rows_per_src=[8192, 8192, 2942, 7980, 8192, 6906, 8192, 6706] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=344608768 dispatch_meta_bytes=228856 combine_payload_bytes=344608768 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=344837624 ingress_bytes=674910208 link_bytes=674910208 token_bytes=689446392 weight_movement_bytes=330301440 total_nvlink_bytes=1019747832 weight_to_token_ratio=0.479082 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.173779855e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57442 payload_rows_remote=49266 dup_rows=8094 dup_rows_remote=7508 dedup_ratio=0.123505 router_tokens_per_expert=[0, 19, 0, 3939, 0, 0, 1, 0, 0, 0, 4094, 0, 0, 0, 0, 907, 0, 2, 0, 0, 292, 4087, 0, 0, 0, 0, 0, 0, 6, 0, 1057, 3485, 0, 1740, 285, 0, 1, 0, 0, 49, 0, 2, 0, 0, 6, 0, 3979, 0, 0, 0, 0, 1, 0, 0, 3706, 1, 4095, 1, 978, 0, 32, 3, 0, 0] payload_rows_per_dst=[7912, 8190, 8176, 7582, 8190, 7964, 7412, 2016] dup_rows_per_dst=[6, 1812, 586, 1514, 4150, 10, 4, 12] combine_rows_per_src=[7912, 8190, 8176, 7582, 8190, 7964, 7412, 2016] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58194 payload_rows_remote=50168 dup_rows=7342 dup_rows_remote=3618 dedup_ratio=0.112030 router_tokens_per_expert=[0, 0, 0, 3955, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1195, 0, 0, 0, 0, 223, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 671, 2536, 0, 1919, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3999, 0, 0, 0, 0, 0, 0, 0, 3456, 0, 4096, 0, 2484, 0, 2, 0, 1, 0] payload_rows_per_dst=[7910, 8192, 8188, 5832, 8026, 7998, 6912, 5136] dup_rows_per_dst=[0, 2390, 446, 582, 3724, 8, 0, 192] combine_rows_per_src=[7910, 8192, 8188, 5832, 8026, 7998, 6912, 5136] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=359718912 dispatch_meta_bytes=229696 combine_payload_bytes=359718912 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=359948608 ingress_bytes=690020352 link_bytes=690020352 token_bytes=719667520 weight_movement_bytes=330301440 total_nvlink_bytes=1049968960 weight_to_token_ratio=0.458964 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.200058881e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=385193984 dispatch_meta_bytes=222536 combine_payload_bytes=385193984 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=385416520 ingress_bytes=495294464 link_bytes=495294464 token_bytes=770610504 weight_movement_bytes=110100480 total_nvlink_bytes=880710984 weight_to_token_ratio=0.142874 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=26624 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.613985348e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51712 payload_rows_remote=43830 dup_rows=13824 dup_rows_remote=12356 dedup_ratio=0.210938 router_tokens_per_expert=[0, 0, 0, 3852, 2, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 535, 0, 0, 0, 0, 147, 4093, 2, 0, 0, 0, 0, 0, 0, 0, 985, 3828, 0, 1710, 108, 3, 0, 0, 0, 2, 0, 0, 0, 0, 7, 0, 4071, 1, 0, 0, 0, 11, 0, 0, 3910, 0, 4079, 0, 1330, 0, 12, 5, 0, 1] payload_rows_per_dst=[7708, 1184, 8188, 6810, 3608, 8150, 7882, 8182] dup_rows_per_dst=[0, 82, 296, 2622, 38, 8, 1468, 9310] combine_rows_per_src=[7708, 1184, 8188, 6810, 3608, 8150, 7882, 8182] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56984 payload_rows_remote=48952 dup_rows=8552 dup_rows_remote=8458 dedup_ratio=0.130493 router_tokens_per_expert=[0, 4, 0, 3632, 0, 0, 1, 0, 2, 0, 4096, 0, 0, 0, 1, 928, 0, 3, 0, 0, 293, 4078, 1, 0, 0, 1, 0, 0, 0, 0, 1585, 3899, 0, 566, 467, 0, 0, 0, 0, 2, 0, 0, 0, 0, 48, 0, 4014, 1, 0, 0, 0, 0, 0, 0, 3482, 0, 4094, 0, 1569, 0, 1, 0, 0, 0] payload_rows_per_dst=[7264, 8192, 8162, 8020, 2412, 8032, 6964, 7938] dup_rows_per_dst=[10, 1862, 588, 2950, 254, 94, 0, 2794] combine_rows_per_src=[7264, 8192, 8162, 8020, 2412, 8032, 6964, 7938] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58170 payload_rows_remote=49978 dup_rows=7366 dup_rows_remote=3706 dedup_ratio=0.112396 router_tokens_per_expert=[0, 0, 0, 3964, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1273, 0, 0, 0, 0, 257, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 690, 2594, 0, 1786, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 3999, 0, 0, 0, 0, 0, 0, 0, 3407, 0, 4096, 0, 2452, 0, 1, 0, 3, 0] payload_rows_per_dst=[7928, 8192, 8190, 5950, 8192, 7998, 6814, 4906] dup_rows_per_dst=[0, 2546, 514, 618, 3660, 22, 0, 6] combine_rows_per_src=[7928, 8192, 8190, 5950, 8192, 7998, 6814, 4906] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=350629888 dispatch_meta_bytes=210312 combine_payload_bytes=350629888 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=350840200 ingress_bytes=460730368 link_bytes=460730368 token_bytes=701470088 weight_movement_bytes=110100480 total_nvlink_bytes=811570568 weight_to_token_ratio=0.156957 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=22016 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.012858871e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=353224704 dispatch_meta_bytes=226736 combine_payload_bytes=353224704 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=353451440 ingress_bytes=683526144 link_bytes=683526144 token_bytes=706676144 weight_movement_bytes=330301440 total_nvlink_bytes=1036977584 weight_to_token_ratio=0.467401 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=30336 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.188764385e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46278 payload_rows_remote=38092 dup_rows=19258 dup_rows_remote=9468 dedup_ratio=0.293854 router_tokens_per_expert=[0, 0, 20, 3969, 1, 1, 3, 0, 0, 0, 4078, 0, 0, 0, 0, 769, 0, 6, 0, 0, 134, 4088, 0, 1, 0, 0, 0, 1, 5, 1, 1530, 3948, 0, 1191, 278, 0, 0, 0, 0, 3, 1, 0, 0, 0, 10, 0, 3889, 0, 0, 0, 1, 0, 0, 0, 3930, 0, 4087, 0, 804, 0, 13, 5, 1, 0] payload_rows_per_dst=[7980, 1538, 6622, 3070, 2892, 7802, 8188, 8186] dup_rows_per_dst=[8, 0, 208, 4, 52, 1626, 7570, 9790] combine_rows_per_src=[7980, 1538, 6622, 3070, 2892, 7802, 8188, 8186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58650 payload_rows_remote=51436 dup_rows=6886 dup_rows_remote=5904 dedup_ratio=0.105072 router_tokens_per_expert=[0, 0, 1, 3974, 0, 0, 0, 0, 0, 2, 4095, 1, 0, 0, 0, 1081, 0, 0, 0, 0, 279, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 783, 3315, 0, 1403, 153, 1, 2, 0, 0, 0, 0, 0, 0, 0, 25, 0, 3927, 0, 0, 0, 0, 2, 0, 0, 2752, 1, 4095, 1, 2739, 0, 36, 0, 8, 1] payload_rows_per_dst=[7950, 8192, 8182, 7214, 8190, 7860, 5504, 5558] dup_rows_per_dst=[0, 2166, 558, 982, 3118, 44, 6, 12] combine_rows_per_src=[7950, 8192, 8182, 7214, 8190, 7860, 5504, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58120 payload_rows_remote=49934 dup_rows=7416 dup_rows_remote=6264 dedup_ratio=0.113159 router_tokens_per_expert=[0, 1, 3, 3900, 2, 7, 0, 5, 2, 0, 4086, 0, 0, 4, 0, 577, 0, 0, 0, 0, 425, 4089, 0, 15, 0, 0, 0, 1, 0, 4, 1109, 3543, 0, 1403, 455, 0, 0, 3, 1, 10, 3, 0, 0, 7, 1, 0, 3815, 0, 0, 3, 0, 1, 0, 0, 3049, 0, 4064, 1, 2171, 0, 0, 6, 2, 0] payload_rows_per_dst=[7826, 8186, 8190, 7642, 8176, 7648, 6104, 4348] dup_rows_per_dst=[10, 1152, 868, 1672, 3696, 4, 2, 12] combine_rows_per_src=[7826, 8186, 8190, 7642, 8176, 7648, 6104, 4348] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=314173440 dispatch_meta_bytes=224744 combine_payload_bytes=314173440 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=314398184 ingress_bytes=424273920 link_bytes=424273920 token_bytes=628571624 weight_movement_bytes=110100480 total_nvlink_bytes=738672104 weight_to_token_ratio=0.175160 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=28160 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.378821279e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=352020480 dispatch_meta_bytes=209928 combine_payload_bytes=352020480 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=352230408 ingress_bytes=462120960 link_bytes=462120960 token_bytes=704250888 weight_movement_bytes=110100480 total_nvlink_bytes=814351368 weight_to_token_ratio=0.156337 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=21888 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.037043553e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=353138688 dispatch_meta_bytes=227096 combine_payload_bytes=353138688 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=353365784 ingress_bytes=683440128 link_bytes=683440128 token_bytes=706504472 weight_movement_bytes=330301440 total_nvlink_bytes=1036805912 weight_to_token_ratio=0.467515 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=30848 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.188614789e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=359604224 dispatch_meta_bytes=215144 combine_payload_bytes=359604224 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=359819368 ingress_bytes=469704704 link_bytes=469704704 token_bytes=719423592 weight_movement_bytes=110100480 total_nvlink_bytes=829524072 weight_to_token_ratio=0.153040 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=20352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.168937334e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52950 payload_rows_remote=45046 dup_rows=12586 dup_rows_remote=12254 dedup_ratio=0.192047 router_tokens_per_expert=[0, 0, 0, 3843, 2, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 512, 0, 0, 0, 1, 87, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 944, 3831, 0, 1782, 114, 4, 0, 0, 0, 3, 0, 0, 0, 0, 5, 0, 4067, 1, 0, 0, 0, 10, 0, 0, 3949, 0, 4079, 0, 1346, 0, 12, 5, 0, 1] payload_rows_per_dst=[7690, 1350, 8188, 7720, 3770, 8146, 7904, 8182] dup_rows_per_dst=[0, 142, 176, 1512, 36, 1762, 332, 8626] combine_rows_per_src=[7690, 1350, 8188, 7720, 3770, 8146, 7904, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57230 payload_rows_remote=49212 dup_rows=8306 dup_rows_remote=8236 dedup_ratio=0.126740 router_tokens_per_expert=[0, 4, 0, 3611, 0, 0, 1, 0, 2, 0, 4096, 0, 0, 0, 1, 815, 0, 3, 0, 0, 246, 4079, 1, 0, 0, 1, 0, 0, 0, 0, 1574, 3889, 0, 630, 497, 0, 0, 0, 0, 2, 0, 0, 0, 0, 36, 0, 4007, 1, 0, 0, 0, 0, 0, 0, 3616, 0, 4094, 0, 1562, 0, 0, 0, 0, 0] payload_rows_per_dst=[7222, 8192, 8166, 8020, 2192, 8018, 7232, 8188] dup_rows_per_dst=[10, 1636, 492, 2908, 66, 70, 0, 3124] combine_rows_per_src=[7222, 8192, 8166, 8020, 2192, 8018, 7232, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47704 payload_rows_remote=39518 dup_rows=17832 dup_rows_remote=8006 dedup_ratio=0.272095 router_tokens_per_expert=[0, 0, 20, 3957, 1, 1, 3, 0, 0, 0, 4077, 0, 0, 0, 0, 689, 0, 5, 0, 0, 87, 4087, 0, 1, 0, 0, 0, 1, 6, 1, 1585, 3947, 0, 1170, 313, 0, 0, 0, 0, 3, 1, 0, 0, 0, 8, 0, 3901, 0, 0, 0, 1, 0, 0, 0, 3974, 0, 4087, 0, 824, 0, 12, 5, 1, 0] payload_rows_per_dst=[7956, 1378, 7930, 3334, 2922, 7808, 8190, 8186] dup_rows_per_dst=[8, 0, 178, 104, 50, 12, 7654, 9826] combine_rows_per_src=[7956, 1378, 7930, 3334, 2922, 7808, 8190, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58026 payload_rows_remote=49840 dup_rows=7510 dup_rows_remote=6378 dedup_ratio=0.114594 router_tokens_per_expert=[0, 2, 4, 3895, 2, 7, 0, 5, 2, 0, 4086, 0, 0, 4, 0, 567, 0, 0, 0, 0, 346, 4089, 0, 15, 0, 0, 0, 1, 0, 2, 1146, 3519, 0, 1476, 491, 0, 0, 3, 1, 11, 3, 0, 0, 8, 1, 0, 3801, 0, 0, 1, 0, 1, 0, 0, 3158, 0, 4066, 1, 2045, 0, 1, 6, 2, 0] payload_rows_per_dst=[7818, 8186, 8190, 7604, 8186, 7622, 6320, 4100] dup_rows_per_dst=[12, 1132, 710, 1732, 3910, 4, 0, 10] combine_rows_per_src=[7818, 8186, 8190, 7604, 8186, 7622, 6320, 4100] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58758 payload_rows_remote=51580 dup_rows=6778 dup_rows_remote=5744 dedup_ratio=0.103424 router_tokens_per_expert=[0, 0, 0, 3945, 0, 0, 0, 0, 0, 2, 4096, 1, 0, 0, 0, 1106, 0, 0, 0, 0, 248, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 793, 3313, 0, 1339, 151, 1, 2, 0, 0, 0, 0, 0, 0, 0, 16, 0, 3905, 0, 0, 0, 0, 0, 0, 0, 2880, 1, 4095, 1, 2735, 0, 36, 0, 8, 1] payload_rows_per_dst=[7890, 8192, 8186, 7178, 8190, 7814, 5760, 5548] dup_rows_per_dst=[0, 2218, 496, 1034, 2986, 28, 2, 14] combine_rows_per_src=[7890, 8192, 8186, 7178, 8190, 7814, 5760, 5548] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=350887936 dispatch_meta_bytes=229640 combine_payload_bytes=350887936 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=351117576 ingress_bytes=460988416 link_bytes=460988416 token_bytes=702005512 weight_movement_bytes=110100480 total_nvlink_bytes=812105992 weight_to_token_ratio=0.156837 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=31872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.017346750e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=358242304 dispatch_meta_bytes=214736 combine_payload_bytes=358242304 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=358457040 ingress_bytes=468342784 link_bytes=468342784 token_bytes=716699344 weight_movement_bytes=110100480 total_nvlink_bytes=826799824 weight_to_token_ratio=0.153622 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=20864 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.145251305e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=273043456 dispatch_meta_bytes=190240 combine_payload_bytes=273043456 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=273233696 ingress_bytes=493244416 link_bytes=493244416 token_bytes=546277152 weight_movement_bytes=220200960 total_nvlink_bytes=766478112 weight_to_token_ratio=0.403094 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=13312 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.578331642e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=368693248 dispatch_meta_bytes=229360 combine_payload_bytes=368693248 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=368922608 ingress_bytes=588894208 link_bytes=588894208 token_bytes=737615856 weight_movement_bytes=220200960 total_nvlink_bytes=957816816 weight_to_token_ratio=0.298531 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=23680 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.024183884e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=357926912 dispatch_meta_bytes=224792 combine_payload_bytes=357926912 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=358151704 ingress_bytes=688228352 link_bytes=688228352 token_bytes=716078616 weight_movement_bytes=330301440 total_nvlink_bytes=1046380056 weight_to_token_ratio=0.461264 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=24704 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.196942298e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55816 payload_rows_remote=47810 dup_rows=9720 dup_rows_remote=6856 dedup_ratio=0.148315 router_tokens_per_expert=[0, 2, 0, 3948, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1863, 0, 0, 0, 0, 565, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 761, 3662, 0, 458, 20, 0, 0, 0, 0, 4, 0, 0, 0, 0, 60, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 3872, 0, 4096, 0, 1173, 0, 8, 0, 0, 0] payload_rows_per_dst=[8006, 8192, 5436, 7716, 8192, 8168, 7744, 2362] dup_rows_per_dst=[2864, 3958, 682, 1132, 964, 120, 0, 0] combine_rows_per_src=[8006, 8192, 5436, 7716, 8192, 8168, 7744, 2362] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59484 payload_rows_remote=51294 dup_rows=6052 dup_rows_remote=3880 dedup_ratio=0.092346 router_tokens_per_expert=[236, 0, 0, 0, 0, 4064, 0, 0, 1, 4054, 0, 0, 0, 0, 2, 0, 0, 0, 1092, 0, 0, 1, 0, 0, 103, 0, 334, 0, 0, 0, 0, 31, 0, 1643, 2067, 4091, 0, 4095, 0, 294, 0, 0, 2590, 0, 1218, 0, 0, 4088, 0, 10, 0, 0, 0, 0, 114, 0, 0, 0, 0, 1, 0, 0, 0, 2639] payload_rows_per_dst=[8128, 8112, 8190, 8192, 6734, 6658, 8190, 5280] dup_rows_per_dst=[472, 2, 2172, 926, 1274, 958, 248, 0] combine_rows_per_src=[8128, 8112, 8190, 8192, 6734, 6658, 8190, 5280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=322889728 dispatch_meta_bytes=229200 combine_payload_bytes=322889728 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=323118928 ingress_bytes=432990208 link_bytes=432990208 token_bytes=646008656 weight_movement_bytes=110100480 total_nvlink_bytes=756109136 weight_to_token_ratio=0.170432 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=28672 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.530411864e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55448 payload_rows_remote=47426 dup_rows=10088 dup_rows_remote=7008 dedup_ratio=0.153931 router_tokens_per_expert=[0, 3, 0, 3952, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1917, 0, 0, 0, 0, 465, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 782, 3657, 0, 450, 19, 0, 0, 0, 0, 4, 0, 0, 0, 0, 51, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 3907, 0, 4096, 0, 1172, 0, 16, 0, 0, 0] payload_rows_per_dst=[8022, 8192, 4962, 7720, 8192, 8170, 7814, 2376] dup_rows_per_dst=[3080, 4334, 466, 1160, 946, 102, 0, 0] combine_rows_per_src=[8022, 8192, 4962, 7720, 8192, 8170, 7814, 2376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56568 payload_rows_remote=52730 dup_rows=8968 dup_rows_remote=8908 dedup_ratio=0.136841 router_tokens_per_expert=[903, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 805, 0, 0, 0, 0, 0, 74, 0, 215, 0, 0, 0, 0, 0, 0, 3487, 1798, 4088, 0, 4096, 0, 151, 0, 0, 1701, 0, 239, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 2817] payload_rows_per_dst=[8192, 8192, 8192, 8182, 3838, 3788, 8192, 7992] dup_rows_per_dst=[1806, 0, 1610, 572, 60, 92, 212, 4616] combine_rows_per_src=[8192, 8192, 8192, 8182, 3838, 3788, 8192, 7992] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59394 payload_rows_remote=51204 dup_rows=6142 dup_rows_remote=3770 dedup_ratio=0.093719 router_tokens_per_expert=[252, 0, 0, 0, 0, 4063, 0, 0, 1, 4052, 0, 0, 0, 0, 2, 0, 0, 0, 1193, 0, 0, 1, 0, 0, 138, 0, 310, 0, 0, 0, 0, 32, 0, 1752, 1788, 4090, 0, 4095, 0, 369, 0, 0, 2569, 0, 1172, 0, 0, 4087, 0, 11, 0, 0, 0, 0, 113, 0, 0, 0, 0, 1, 0, 0, 0, 2677] payload_rows_per_dst=[8126, 8108, 8190, 8192, 6640, 6592, 8190, 5356] dup_rows_per_dst=[504, 2, 2372, 948, 1178, 890, 248, 0] combine_rows_per_src=[8126, 8108, 8190, 8192, 6640, 6592, 8190, 5356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=352751616 dispatch_meta_bytes=229792 combine_payload_bytes=352751616 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=352981408 ingress_bytes=462852096 link_bytes=462852096 token_bytes=705733024 weight_movement_bytes=110100480 total_nvlink_bytes=815833504 weight_to_token_ratio=0.156009 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=31744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.049759210e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=283265024 dispatch_meta_bytes=190096 combine_payload_bytes=283265024 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=283455120 ingress_bytes=503465984 link_bytes=503465984 token_bytes=566720144 weight_movement_bytes=220200960 total_nvlink_bytes=786921104 weight_to_token_ratio=0.388553 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=13184 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.756101521e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=357253120 dispatch_meta_bytes=224872 combine_payload_bytes=357253120 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=357477992 ingress_bytes=687554560 link_bytes=687554560 token_bytes=714731112 weight_movement_bytes=330301440 total_nvlink_bytes=1045032552 weight_to_token_ratio=0.462134 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=24832 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.195770463e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=342702080 dispatch_meta_bytes=218664 combine_payload_bytes=342702080 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=342920744 ingress_bytes=452802560 link_bytes=452802560 token_bytes=685622824 weight_movement_bytes=110100480 total_nvlink_bytes=795723304 weight_to_token_ratio=0.160585 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=31232 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.874981251e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55930 payload_rows_remote=47738 dup_rows=9606 dup_rows_remote=9356 dedup_ratio=0.146576 router_tokens_per_expert=[231, 0, 0, 0, 0, 4094, 1, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 1752, 0, 0, 0, 4, 11, 46, 0, 137, 0, 0, 0, 0, 0, 0, 2994, 1768, 4077, 0, 4096, 0, 305, 0, 0, 2496, 1, 468, 1, 1, 4092, 0, 0, 0, 0, 0, 0, 125, 0, 0, 0, 0, 0, 0, 0, 0, 1974] payload_rows_per_dst=[8192, 8186, 3520, 8188, 4044, 8188, 8192, 7420] dup_rows_per_dst=[460, 2, 14, 332, 102, 5930, 250, 2516] combine_rows_per_src=[8192, 8186, 3520, 8188, 4044, 8188, 8192, 7420] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55934 payload_rows_remote=48960 dup_rows=9602 dup_rows_remote=5404 dedup_ratio=0.146515 router_tokens_per_expert=[323, 0, 0, 2, 3, 4095, 0, 0, 0, 4092, 0, 1, 0, 0, 0, 1, 0, 0, 1379, 0, 0, 0, 0, 0, 192, 0, 278, 0, 0, 0, 0, 5, 6, 3066, 1895, 4095, 0, 4096, 0, 98, 0, 1, 2499, 0, 490, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 106, 0, 2, 0, 0, 0, 0, 0, 0, 1947] payload_rows_per_dst=[8190, 8186, 4900, 8190, 3916, 6974, 8192, 7386] dup_rows_per_dst=[656, 2, 856, 950, 82, 4198, 214, 2644] combine_rows_per_src=[8190, 8186, 4900, 8190, 3916, 6974, 8192, 7386] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56882 payload_rows_remote=52496 dup_rows=8654 dup_rows_remote=8566 dedup_ratio=0.132050 router_tokens_per_expert=[919, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 662, 0, 0, 0, 0, 0, 48, 0, 206, 0, 0, 0, 0, 0, 0, 3446, 2117, 4088, 0, 4096, 0, 120, 0, 0, 1563, 0, 265, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 2844] payload_rows_per_dst=[8192, 8192, 8192, 8182, 4386, 3568, 8192, 7978] dup_rows_per_dst=[1838, 0, 1324, 502, 88, 88, 212, 4602] combine_rows_per_src=[8192, 8192, 8192, 8182, 4386, 3568, 8192, 7978] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=367675392 dispatch_meta_bytes=220696 combine_payload_bytes=367675392 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=367896088 ingress_bytes=477775872 link_bytes=477775872 token_bytes=735571480 weight_movement_bytes=110100480 total_nvlink_bytes=845671960 weight_to_token_ratio=0.149680 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=22016 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.309308220e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=369725440 dispatch_meta_bytes=229296 combine_payload_bytes=369725440 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=369954736 ingress_bytes=589926400 link_bytes=589926400 token_bytes=739680176 weight_movement_bytes=220200960 total_nvlink_bytes=959881136 weight_to_token_ratio=0.297698 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=24192 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.025979036e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=339949568 dispatch_meta_bytes=217736 combine_payload_bytes=339949568 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=340167304 ingress_bytes=450050048 link_bytes=450050048 token_bytes=680116872 weight_movement_bytes=110100480 total_nvlink_bytes=790217352 weight_to_token_ratio=0.161885 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=31104 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.827110540e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=377968640 dispatch_meta_bytes=246552 combine_payload_bytes=377968640 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=378215192 ingress_bytes=818370560 link_bytes=818370560 token_bytes=756183832 weight_movement_bytes=440401920 total_nvlink_bytes=1196585752 weight_to_token_ratio=0.582401 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=16384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.423281003e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=367030272 dispatch_meta_bytes=219896 combine_payload_bytes=367030272 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=367250168 ingress_bytes=477130752 link_bytes=477130752 token_bytes=734280440 weight_movement_bytes=110100480 total_nvlink_bytes=844380920 weight_to_token_ratio=0.149943 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=20992 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.298088523e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55982 payload_rows_remote=47790 dup_rows=9554 dup_rows_remote=9314 dedup_ratio=0.145782 router_tokens_per_expert=[261, 0, 0, 0, 0, 4093, 1, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 1874, 0, 0, 0, 4, 10, 73, 0, 124, 0, 0, 0, 0, 0, 0, 2985, 1611, 4078, 0, 4096, 0, 356, 0, 0, 2494, 0, 385, 1, 1, 4093, 0, 0, 0, 0, 0, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0, 2014] payload_rows_per_dst=[8192, 8186, 3762, 8188, 3838, 8188, 8192, 7436] dup_rows_per_dst=[518, 2, 14, 362, 96, 5760, 240, 2562] combine_rows_per_src=[8192, 8186, 3762, 8188, 3838, 8188, 8192, 7436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6987365912 prefetch_bytes=990904320 reduce_bytes=1651507200 weight_movement_bytes=2642411520 egress_bytes=3494829592 ingress_bytes=6134947840 link_bytes=6134947840 total_nvlink_bytes=9629777432 weight_to_token_ratio=0.378170 prefetch_share_of_ingress=0.161518 dedup_ratio=0.157043 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=15 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.039338 gemm_inner_pad_upper_bound=11049 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.066968332e-04 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6933507600 prefetch_bytes=990904320 reduce_bytes=1101004800 weight_movement_bytes=2091909120 egress_bytes=3467894288 ingress_bytes=5557522432 link_bytes=5557522432 total_nvlink_bytes=9025416720 weight_to_token_ratio=0.301710 prefetch_share_of_ingress=0.178300 dedup_ratio=0.149551 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=10 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.038235 gemm_inner_pad_upper_bound=11049 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.665445565e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55214 payload_rows_remote=47440 dup_rows=10322 dup_rows_remote=4958 dedup_ratio=0.157501 router_tokens_per_expert=[337, 0, 1, 2, 2, 4095, 0, 0, 0, 4091, 0, 1, 0, 0, 0, 1, 0, 0, 1548, 0, 0, 0, 0, 0, 270, 0, 257, 0, 0, 0, 0, 4, 6, 3124, 1711, 4095, 0, 4096, 0, 131, 0, 1, 2450, 0, 436, 0, 0, 4095, 0, 0, 1, 0, 1, 0, 104, 0, 3, 0, 0, 0, 0, 0, 0, 1905] payload_rows_per_dst=[8190, 8184, 3672, 8190, 3602, 7774, 8192, 7410] dup_rows_per_dst=[684, 2, 250, 1062, 94, 5364, 212, 2654] combine_rows_per_src=[8190, 8184, 3672, 8190, 3602, 7774, 8192, 7410] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46436 payload_rows_remote=39974 dup_rows=19100 dup_rows_remote=17794 dedup_ratio=0.291443 router_tokens_per_expert=[468, 0, 0, 0, 2, 4046, 0, 0, 0, 4076, 0, 0, 7, 0, 0, 0, 0, 0, 1089, 0, 0, 0, 13, 1, 113, 0, 303, 0, 0, 0, 0, 29, 4, 1637, 2549, 4090, 1, 4095, 0, 594, 1, 0, 2013, 0, 1174, 2, 1, 4095, 0, 7, 0, 0, 0, 0, 111, 0, 0, 0, 0, 0, 0, 0, 0, 2247] payload_rows_per_dst=[1860, 8166, 2186, 3468, 7912, 8192, 8190, 6462] dup_rows_per_dst=[124, 118, 20, 682, 7616, 8998, 236, 1306] combine_rows_per_src=[1860, 8166, 2186, 3468, 7912, 8192, 8190, 6462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57922 payload_rows_remote=49736 dup_rows=7614 dup_rows_remote=7600 dedup_ratio=0.116180 router_tokens_per_expert=[533, 2, 2, 0, 0, 4087, 0, 0, 2, 4091, 2, 0, 1, 1, 2, 1, 0, 0, 1245, 0, 0, 0, 0, 0, 450, 0, 134, 0, 0, 0, 0, 0, 0, 1924, 2637, 4085, 0, 4086, 0, 431, 0, 0, 1861, 1, 117, 5, 0, 4092, 0, 2, 4, 2, 0, 0, 77, 0, 14, 0, 0, 0, 0, 1, 5, 2871] payload_rows_per_dst=[8182, 8186, 8184, 8188, 7310, 3918, 8186, 5768] dup_rows_per_dst=[1066, 14, 2490, 1150, 2674, 50, 156, 14] combine_rows_per_src=[8182, 8186, 8184, 8188, 7310, 3918, 8186, 5768] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46044 payload_rows_remote=39590 dup_rows=19492 dup_rows_remote=18266 dedup_ratio=0.297424 router_tokens_per_expert=[488, 0, 0, 0, 0, 4049, 0, 0, 0, 4076, 0, 0, 8, 0, 0, 0, 0, 0, 1243, 0, 0, 1, 13, 1, 219, 0, 245, 0, 0, 0, 0, 29, 4, 1685, 2307, 4091, 1, 4095, 0, 722, 1, 0, 2012, 0, 1108, 2, 1, 4095, 0, 8, 0, 0, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 2155] payload_rows_per_dst=[1820, 8168, 2492, 2818, 7910, 8192, 8190, 6454] dup_rows_per_dst=[94, 130, 24, 744, 9620, 7420, 234, 1226] combine_rows_per_src=[1820, 8168, 2492, 2818, 7910, 8192, 8190, 6454] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57872 payload_rows_remote=49742 dup_rows=7664 dup_rows_remote=7348 dedup_ratio=0.116943 router_tokens_per_expert=[463, 0, 0, 2, 0, 4087, 0, 0, 0, 4089, 0, 0, 1, 0, 0, 0, 0, 0, 1845, 0, 0, 0, 1, 0, 26, 0, 140, 0, 0, 0, 0, 19, 0, 1925, 1828, 4038, 2, 4095, 0, 671, 2, 0, 1966, 0, 319, 3, 4, 4085, 0, 18, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 3043] payload_rows_per_dst=[8182, 8180, 8184, 8130, 5048, 4420, 8190, 7538] dup_rows_per_dst=[922, 0, 3678, 316, 522, 168, 228, 1830] combine_rows_per_src=[8182, 8180, 8184, 8130, 5048, 4420, 8190, 7538] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58044 payload_rows_remote=49854 dup_rows=7492 dup_rows_remote=7480 dedup_ratio=0.114319 router_tokens_per_expert=[552, 0, 1, 0, 0, 4087, 0, 0, 2, 4092, 3, 0, 1, 0, 2, 1, 0, 0, 1220, 0, 0, 0, 0, 0, 314, 0, 140, 0, 0, 0, 0, 1, 0, 1918, 2768, 4086, 2, 4086, 0, 405, 0, 0, 1825, 1, 162, 6, 0, 4092, 0, 1, 3, 1, 1, 0, 86, 0, 14, 0, 0, 0, 0, 1, 4, 2890] payload_rows_per_dst=[8176, 8190, 8184, 8188, 7408, 3906, 8184, 5808] dup_rows_per_dst=[1104, 12, 2440, 894, 2778, 82, 172, 10] combine_rows_per_src=[8176, 8190, 8184, 8188, 7408, 3906, 8184, 5808] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=342185984 dispatch_meta_bytes=228376 combine_payload_bytes=342185984 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=342414360 ingress_bytes=452286464 link_bytes=452286464 token_bytes=684600344 weight_movement_bytes=110100480 total_nvlink_bytes=794700824 weight_to_token_ratio=0.160824 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=31872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.866005492e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=342558720 dispatch_meta_bytes=228416 combine_payload_bytes=342558720 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=342787136 ingress_bytes=452659200 link_bytes=452659200 token_bytes=685345856 weight_movement_bytes=110100480 total_nvlink_bytes=795446336 weight_to_token_ratio=0.160650 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.872487984e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57698 payload_rows_remote=49560 dup_rows=7838 dup_rows_remote=7492 dedup_ratio=0.119598 router_tokens_per_expert=[460, 0, 0, 2, 0, 4088, 0, 0, 0, 4086, 0, 0, 1, 0, 0, 0, 0, 0, 2031, 0, 0, 0, 1, 0, 43, 0, 141, 0, 0, 0, 0, 16, 0, 1964, 1627, 4042, 1, 4096, 0, 712, 2, 0, 1981, 0, 240, 4, 1, 4085, 0, 21, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 3037] payload_rows_per_dst=[8182, 8174, 8184, 8138, 4966, 4338, 8192, 7524] dup_rows_per_dst=[918, 0, 4050, 346, 532, 118, 214, 1660] combine_rows_per_src=[8182, 8174, 8184, 8138, 4966, 4338, 8192, 7524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54802 payload_rows_remote=46610 dup_rows=10734 dup_rows_remote=9864 dedup_ratio=0.163788 router_tokens_per_expert=[435, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1929, 0, 0, 0, 0, 0, 85, 0, 78, 0, 0, 0, 0, 0, 0, 1924, 2859, 4065, 0, 4096, 0, 934, 0, 0, 1346, 0, 180, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 2432] payload_rows_per_dst=[8192, 8188, 8192, 8150, 7770, 2974, 6472, 4864] dup_rows_per_dst=[870, 0, 3858, 306, 5408, 78, 214, 0] combine_rows_per_src=[8192, 8188, 8192, 8150, 7770, 2974, 6472, 4864] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=350945280 dispatch_meta_bytes=217456 combine_payload_bytes=350945280 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=351162736 ingress_bytes=571146240 link_bytes=571146240 token_bytes=702108016 weight_movement_bytes=220200960 total_nvlink_bytes=922308976 weight_to_token_ratio=0.313628 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=16384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.933172488e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=376291328 dispatch_meta_bytes=244248 combine_payload_bytes=376291328 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=376535576 ingress_bytes=816693248 link_bytes=816693248 token_bytes=752826904 weight_movement_bytes=440401920 total_nvlink_bytes=1193228824 weight_to_token_ratio=0.584998 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=18048 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.420363881e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=340049920 dispatch_meta_bytes=209592 combine_payload_bytes=340049920 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=340259512 ingress_bytes=560250880 link_bytes=560250880 token_bytes=680309432 weight_movement_bytes=220200960 total_nvlink_bytes=900510392 weight_to_token_ratio=0.323678 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=16512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.743684258e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6909393712 prefetch_bytes=770703360 reduce_bytes=1321205760 weight_movement_bytes=2091909120 egress_bytes=3455836976 ingress_bytes=5545465856 link_bytes=5545465856 total_nvlink_bytes=9001302832 weight_to_token_ratio=0.302763 prefetch_share_of_ingress=0.138979 dedup_ratio=0.151865 prefetch_slots=14 unused_prefetch_slots=146 slot_utilization=0.087500 reduce_slots=12 remote_demand=7 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.038971 gemm_inner_pad_upper_bound=10795 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.644477196e-05 prefetch_bytes_per_gemm_flop=1.340379902e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=286533632 dispatch_meta_bytes=231072 combine_payload_bytes=286533632 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=286764704 ingress_bytes=396634112 link_bytes=396634112 token_bytes=573298336 weight_movement_bytes=110100480 total_nvlink_bytes=683398816 weight_to_token_ratio=0.192047 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=19968 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.898119559e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=356550656 dispatch_meta_bytes=228360 combine_payload_bytes=356550656 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=356779016 ingress_bytes=466651136 link_bytes=466651136 token_bytes=713329672 weight_movement_bytes=110100480 total_nvlink_bytes=823430152 weight_to_token_ratio=0.154347 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.055147 gemm_active_groups=5 gemm_peak_group_rows=30208 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.115830764e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=357353472 dispatch_meta_bytes=229336 combine_payload_bytes=357353472 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=357582808 ingress_bytes=467453952 link_bytes=467453952 token_bytes=714936280 weight_movement_bytes=110100480 total_nvlink_bytes=825036760 weight_to_token_ratio=0.154000 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.129793055e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6648636080 prefetch_bytes=990904320 reduce_bytes=1541406720 weight_movement_bytes=2532311040 egress_bytes=3325450928 ingress_bytes=5855496192 link_bytes=5855496192 total_nvlink_bytes=9180947120 weight_to_token_ratio=0.380877 prefetch_share_of_ingress=0.169226 dedup_ratio=0.180859 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=14 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.038235 gemm_inner_pad_upper_bound=11176 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.018367094e-04 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54866 payload_rows_remote=46674 dup_rows=10670 dup_rows_remote=9822 dedup_ratio=0.162811 router_tokens_per_expert=[424, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1742, 0, 0, 0, 0, 0, 47, 0, 87, 0, 0, 0, 0, 0, 0, 1998, 3018, 4064, 0, 4096, 0, 905, 0, 0, 1352, 0, 252, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 2378] payload_rows_per_dst=[8192, 8188, 8192, 8148, 7838, 3120, 6432, 4756] dup_rows_per_dst=[848, 0, 3484, 248, 5788, 88, 214, 0] combine_rows_per_src=[8192, 8188, 8192, 8148, 7838, 3120, 6432, 4756] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6762131376 prefetch_bytes=1101004800 reduce_bytes=440401920 weight_movement_bytes=1541406720 egress_bytes=3382190000 ingress_bytes=4921348096 link_bytes=4921348096 total_nvlink_bytes=8303538096 weight_to_token_ratio=0.227947 prefetch_share_of_ingress=0.223720 dedup_ratio=0.163858 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=4 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.036397 gemm_inner_pad_upper_bound=11557 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.559033762e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6976299072 prefetch_bytes=1101004800 reduce_bytes=1541406720 weight_movement_bytes=2642411520 egress_bytes=3489296448 ingress_bytes=6129414144 link_bytes=6129414144 total_nvlink_bytes=9618710592 weight_to_token_ratio=0.378770 prefetch_share_of_ingress=0.179626 dedup_ratio=0.157755 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=14 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.037868 gemm_inner_pad_upper_bound=11430 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.066005931e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6268076280 prefetch_bytes=1321205760 reduce_bytes=660602880 weight_movement_bytes=1981808640 egress_bytes=3135129848 ingress_bytes=5114755072 link_bytes=5114755072 total_nvlink_bytes=8249884920 weight_to_token_ratio=0.316175 prefetch_share_of_ingress=0.258313 dedup_ratio=0.217398 prefetch_slots=24 unused_prefetch_slots=136 slot_utilization=0.150000 reduce_slots=6 remote_demand=12 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.038235 gemm_inner_pad_upper_bound=11811 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.895400303e-05 prefetch_bytes_per_gemm_flop=2.297794118e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=356507648 dispatch_meta_bytes=229344 combine_payload_bytes=356507648 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=356736992 ingress_bytes=466608128 link_bytes=466608128 token_bytes=713244640 weight_movement_bytes=110100480 total_nvlink_bytes=823345120 weight_to_token_ratio=0.154366 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.029412 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.115082784e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7059030376 prefetch_bytes=1101004800 reduce_bytes=220200960 weight_movement_bytes=1321205760 egress_bytes=3530668392 ingress_bytes=4849567744 link_bytes=4849567744 total_nvlink_bytes=8380236136 weight_to_token_ratio=0.187165 prefetch_share_of_ingress=0.227032 dedup_ratio=0.146548 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=2 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040441 gemm_inner_pad_upper_bound=10922 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.434195924e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6644293984 prefetch_bytes=990904320 reduce_bytes=1101004800 weight_movement_bytes=2091909120 egress_bytes=3323287904 ingress_bytes=5412915200 link_bytes=5412915200 total_nvlink_bytes=8736203104 weight_to_token_ratio=0.314843 prefetch_share_of_ingress=0.183063 dedup_ratio=0.183102 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=10 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.038603 gemm_inner_pad_upper_bound=11430 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.413949804e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=334559232 dispatch_meta_bytes=225984 combine_payload_bytes=334559232 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=334785216 ingress_bytes=664860672 link_bytes=664860672 token_bytes=669344448 weight_movement_bytes=330301440 total_nvlink_bytes=999645888 weight_to_token_ratio=0.493470 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=29184 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.156302059e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6265643920 prefetch_bytes=880803840 reduce_bytes=1431306240 weight_movement_bytes=2312110080 egress_bytes=3133858704 ingress_bytes=5443895296 link_bytes=5443895296 total_nvlink_bytes=8577754000 weight_to_token_ratio=0.369014 prefetch_share_of_ingress=0.161797 dedup_ratio=0.213702 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=13 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.038971 gemm_inner_pad_upper_bound=11049 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.467829287e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none - [2026-08-26 15:11:32] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 369.9 | throughput per GPU (TFLOP/s/GPU): 333.8 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.110655E+01 | loss scale: 1.0 | grad norm: 107.468 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46850 payload_rows_remote=40428 dup_rows=18686 dup_rows_remote=15926 dedup_ratio=0.285126 router_tokens_per_expert=[273, 71, 110, 20, 45, 26, 985, 1018, 151, 141, 241, 528, 47, 160, 2, 139, 108, 557, 2431, 34, 507, 281, 14, 60, 1826, 277, 430, 881, 24, 1382, 731, 17, 26, 19, 117, 1749, 9, 6, 8, 360, 945, 948, 959, 26, 2358, 1318, 1344, 797, 244, 1344, 887, 106, 456, 594, 327, 710, 1200, 669, 191, 212, 299, 174, 849, 0] payload_rows_per_dst=[5512, 6272, 6422, 5704, 5952, 5518, 6178, 5292] dup_rows_per_dst=[2272, 2004, 2760, 1780, 2288, 2528, 3158, 1896] combine_rows_per_src=[5512, 6272, 6422, 5704, 5952, 5518, 6178, 5292] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46358 payload_rows_remote=41364 dup_rows=19178 dup_rows_remote=17848 dedup_ratio=0.292633 router_tokens_per_expert=[234, 56, 267, 58, 32, 16, 1222, 1422, 355, 57, 208, 443, 9, 109, 3, 252, 112, 714, 2028, 4, 308, 170, 35, 59, 1957, 369, 357, 1102, 8, 1922, 834, 29, 5, 25, 279, 1901, 22, 0, 17, 350, 484, 1090, 1004, 44, 2446, 1013, 702, 871, 68, 931, 684, 119, 438, 294, 213, 415, 1651, 638, 189, 153, 433, 585, 950, 3] payload_rows_per_dst=[4906, 6094, 5384, 6410, 6286, 5618, 4994, 6666] dup_rows_per_dst=[1708, 1670, 1476, 2832, 2826, 2790, 1330, 4546] combine_rows_per_src=[4906, 6094, 5384, 6410, 6286, 5618, 4994, 6666] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47534 payload_rows_remote=41114 dup_rows=18002 dup_rows_remote=15260 dedup_ratio=0.274689 router_tokens_per_expert=[196, 85, 91, 14, 6, 10, 1352, 1089, 229, 60, 167, 377, 3, 137, 0, 161, 218, 293, 2533, 11, 331, 209, 24, 168, 1931, 360, 172, 1053, 17, 1784, 619, 6, 21, 5, 314, 2153, 3, 0, 13, 141, 451, 857, 945, 20, 2584, 1260, 915, 734, 100, 1356, 782, 53, 628, 521, 299, 1027, 1526, 631, 111, 70, 298, 394, 849, 1] payload_rows_per_dst=[5366, 6180, 5986, 5940, 6420, 5772, 6360, 5510] dup_rows_per_dst=[2072, 1256, 1588, 2082, 2742, 2840, 3172, 2250] combine_rows_per_src=[5366, 6180, 5986, 5940, 6420, 5772, 6360, 5510] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47802 payload_rows_remote=41562 dup_rows=17734 dup_rows_remote=14522 dedup_ratio=0.270599 router_tokens_per_expert=[211, 18, 386, 41, 12, 14, 1839, 875, 310, 83, 168, 276, 9, 230, 0, 164, 360, 201, 3006, 0, 206, 192, 44, 102, 1348, 847, 150, 790, 49, 1580, 809, 14, 18, 22, 380, 1879, 7, 9, 16, 87, 398, 652, 949, 44, 2407, 1112, 1523, 791, 197, 1705, 501, 92, 506, 616, 223, 523, 1477, 342, 251, 146, 327, 91, 1141, 2] payload_rows_per_dst=[5292, 5954, 6650, 5862, 5758, 5830, 6216, 6240] dup_rows_per_dst=[1500, 1340, 1572, 2260, 1774, 3210, 2866, 3212] combine_rows_per_src=[5292, 5954, 6650, 5862, 5758, 5830, 6216, 6240] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46194 payload_rows_remote=39020 dup_rows=19342 dup_rows_remote=15968 dedup_ratio=0.295135 router_tokens_per_expert=[287, 33, 129, 44, 270, 17, 885, 1995, 232, 134, 101, 820, 11, 113, 0, 227, 98, 779, 1718, 19, 133, 230, 52, 168, 1743, 461, 186, 1248, 73, 1752, 1013, 11, 28, 16, 113, 1855, 12, 9, 15, 207, 558, 947, 657, 70, 2689, 1033, 998, 729, 364, 1359, 576, 110, 401, 413, 444, 833, 946, 706, 148, 122, 513, 258, 653, 4] payload_rows_per_dst=[6162, 7174, 6112, 6448, 5864, 3262, 6194, 4978] dup_rows_per_dst=[3154, 3374, 2348, 3040, 2132, 766, 2806, 1722] combine_rows_per_src=[6162, 7174, 6112, 6448, 5864, 3262, 6194, 4978] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47820 payload_rows_remote=40454 dup_rows=17716 dup_rows_remote=11762 dedup_ratio=0.270325 router_tokens_per_expert=[260, 42, 75, 18, 14, 48, 1338, 1263, 139, 36, 141, 395, 8, 46, 2, 121, 183, 318, 2727, 4, 125, 307, 34, 123, 1452, 227, 178, 829, 5, 1625, 667, 5, 24, 16, 68, 2585, 8, 0, 4, 144, 852, 699, 864, 63, 2725, 1422, 2585, 498, 210, 1964, 232, 90, 337, 361, 427, 572, 1214, 585, 211, 101, 423, 160, 568, 1] payload_rows_per_dst=[4768, 6222, 6234, 5446, 6332, 7366, 6148, 5304] dup_rows_per_dst=[1348, 1004, 1408, 1626, 2270, 5954, 2238, 1868] combine_rows_per_src=[4768, 6222, 6234, 5446, 6332, 7366, 6148, 5304] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47794 payload_rows_remote=42294 dup_rows=17742 dup_rows_remote=16128 dedup_ratio=0.270721 router_tokens_per_expert=[155, 76, 30, 10, 75, 18, 1430, 1774, 311, 45, 142, 607, 5, 31, 0, 249, 80, 471, 2605, 28, 146, 183, 21, 160, 1413, 187, 207, 658, 4, 1736, 746, 19, 39, 21, 93, 2331, 2, 0, 59, 396, 743, 663, 917, 14, 2396, 635, 2536, 1210, 191, 1149, 556, 27, 286, 451, 198, 598, 1438, 311, 112, 211, 256, 522, 782, 3] payload_rows_per_dst=[7322, 6148, 6126, 5500, 6212, 5830, 5486, 5170] dup_rows_per_dst=[4886, 1424, 1262, 1614, 2496, 2534, 1426, 2100] combine_rows_per_src=[7322, 6148, 6126, 5500, 6212, 5830, 5486, 5170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=283781120 dispatch_meta_bytes=231424 combine_payload_bytes=283781120 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=284012544 ingress_bytes=393881600 link_bytes=393881600 token_bytes=567793664 weight_movement_bytes=110100480 total_nvlink_bytes=677894144 weight_to_token_ratio=0.193909 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=19968 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.850248848e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6693291904 prefetch_bytes=1101004800 reduce_bytes=1431306240 weight_movement_bytes=2532311040 egress_bytes=3347785600 ingress_bytes=5877817344 link_bytes=5877817344 total_nvlink_bytes=9225602944 weight_to_token_ratio=0.378336 prefetch_share_of_ingress=0.187315 dedup_ratio=0.177280 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=13 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.037500 gemm_inner_pad_upper_bound=11303 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.022249110e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=355246080 dispatch_meta_bytes=228208 combine_payload_bytes=355246080 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=355474288 ingress_bytes=465346560 link_bytes=465346560 token_bytes=710720368 weight_movement_bytes=110100480 total_nvlink_bytes=820820848 weight_to_token_ratio=0.154914 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=29952 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.093142042e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6745402952 prefetch_bytes=880803840 reduce_bytes=660602880 weight_movement_bytes=1541406720 egress_bytes=3373819464 ingress_bytes=4912990208 link_bytes=4912990208 total_nvlink_bytes=8286809672 weight_to_token_ratio=0.228512 prefetch_share_of_ingress=0.179281 dedup_ratio=0.163605 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=6 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.037500 gemm_inner_pad_upper_bound=11430 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.544498020e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=334100480 dispatch_meta_bytes=225896 combine_payload_bytes=334100480 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=334326376 ingress_bytes=664401920 link_bytes=664401920 token_bytes=668426856 weight_movement_bytes=330301440 total_nvlink_bytes=998728296 weight_to_token_ratio=0.494147 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=29184 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.155504214e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6692492920 prefetch_bytes=1101004800 reduce_bytes=880803840 weight_movement_bytes=1981808640 egress_bytes=3347388024 ingress_bytes=5326913536 link_bytes=5326913536 total_nvlink_bytes=8674301560 weight_to_token_ratio=0.296124 prefetch_share_of_ingress=0.206687 dedup_ratio=0.177548 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=8 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.038603 gemm_inner_pad_upper_bound=11684 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.264378766e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6274496896 prefetch_bytes=1211105280 reduce_bytes=880803840 weight_movement_bytes=2091909120 egress_bytes=3138339200 ingress_bytes=5228066816 link_bytes=5228066816 total_nvlink_bytes=8366406016 weight_to_token_ratio=0.333399 prefetch_share_of_ingress=0.231655 dedup_ratio=0.217462 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=8 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.038971 gemm_inner_pad_upper_bound=11557 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.092468062e-05 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7062357880 prefetch_bytes=1101004800 reduce_bytes=220200960 weight_movement_bytes=1321205760 egress_bytes=3532332920 ingress_bytes=4851230720 link_bytes=4851230720 total_nvlink_bytes=8383563640 weight_to_token_ratio=0.187077 prefetch_share_of_ingress=0.226954 dedup_ratio=0.146429 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=2 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=11049 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.437088112e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6250282968 prefetch_bytes=880803840 reduce_bytes=1431306240 weight_movement_bytes=2312110080 egress_bytes=3126181848 ingress_bytes=5436211200 link_bytes=5436211200 total_nvlink_bytes=8562393048 weight_to_token_ratio=0.369921 prefetch_share_of_ingress=0.162025 dedup_ratio=0.215427 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=13 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.039706 gemm_inner_pad_upper_bound=10922 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.454465380e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none + [2026-08-26 15:59:19] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 359.2 | throughput per GPU (TFLOP/s/GPU): 343.8 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.111541E+01 | loss scale: 1.0 | grad norm: 107.640 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47622 payload_rows_remote=41182 dup_rows=17914 dup_rows_remote=15146 dedup_ratio=0.273346 router_tokens_per_expert=[191, 85, 91, 16, 6, 10, 1356, 1100, 225, 66, 162, 376, 3, 140, 0, 171, 214, 320, 2528, 11, 346, 201, 24, 167, 1957, 333, 173, 1065, 18, 1794, 622, 6, 21, 6, 311, 2156, 3, 0, 11, 139, 467, 862, 953, 23, 2534, 1321, 812, 724, 108, 1270, 725, 58, 650, 568, 304, 1001, 1524, 662, 108, 73, 303, 430, 863, 1] payload_rows_per_dst=[5324, 6132, 6022, 5940, 6440, 5820, 6330, 5614] dup_rows_per_dst=[2010, 1222, 1600, 2082, 2768, 2880, 3038, 2314] combine_rows_per_src=[5324, 6132, 6022, 5940, 6440, 5820, 6330, 5614] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46820 payload_rows_remote=40584 dup_rows=18716 dup_rows_remote=16310 dedup_ratio=0.285583 router_tokens_per_expert=[279, 71, 106, 20, 41, 27, 991, 1031, 146, 138, 232, 518, 46, 162, 1, 143, 105, 578, 2439, 35, 520, 271, 14, 59, 1841, 267, 430, 888, 24, 1397, 724, 17, 26, 23, 115, 1759, 9, 7, 8, 346, 957, 950, 965, 25, 2318, 1355, 1261, 789, 249, 1306, 849, 115, 460, 633, 335, 701, 1195, 692, 196, 214, 301, 189, 859, 0] payload_rows_per_dst=[5432, 6602, 6236, 5710, 5968, 5386, 6162, 5324] dup_rows_per_dst=[2222, 2736, 2406, 1784, 2300, 2166, 3134, 1968] combine_rows_per_src=[5432, 6602, 6236, 5710, 5968, 5386, 6162, 5324] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47580 payload_rows_remote=41370 dup_rows=17956 dup_rows_remote=14184 dedup_ratio=0.273987 router_tokens_per_expert=[207, 19, 375, 42, 11, 14, 1835, 910, 310, 87, 157, 270, 8, 237, 0, 174, 343, 205, 3002, 0, 230, 189, 45, 101, 1353, 828, 146, 803, 51, 1580, 803, 15, 20, 24, 383, 1893, 9, 9, 17, 84, 418, 656, 950, 54, 2354, 1157, 1427, 783, 211, 1603, 487, 105, 525, 670, 235, 510, 1473, 352, 260, 152, 334, 103, 1158, 2] payload_rows_per_dst=[5312, 5874, 6636, 5656, 5786, 5982, 6124, 6210] dup_rows_per_dst=[1514, 1320, 1594, 1978, 1798, 3278, 2702, 3772] combine_rows_per_src=[5312, 5874, 6636, 5656, 5786, 5982, 6124, 6210] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47626 payload_rows_remote=40380 dup_rows=17910 dup_rows_remote=12008 dedup_ratio=0.273285 router_tokens_per_expert=[260, 40, 68, 20, 14, 48, 1341, 1283, 141, 37, 135, 385, 7, 48, 3, 121, 184, 339, 2737, 5, 129, 297, 32, 124, 1472, 220, 179, 841, 9, 1635, 666, 6, 26, 17, 68, 2583, 14, 0, 4, 143, 881, 695, 860, 64, 2676, 1479, 2465, 496, 217, 1878, 217, 101, 338, 393, 450, 568, 1207, 608, 218, 95, 429, 176, 575, 1] payload_rows_per_dst=[5124, 6140, 6252, 5468, 6354, 7246, 6060, 4982] dup_rows_per_dst=[1756, 966, 1442, 1644, 2300, 5902, 2264, 1636] combine_rows_per_src=[5124, 6140, 6252, 5468, 6354, 7246, 6060, 4982] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46602 payload_rows_remote=39668 dup_rows=18934 dup_rows_remote=15996 dedup_ratio=0.288910 router_tokens_per_expert=[283, 32, 121, 50, 252, 16, 892, 2023, 226, 146, 97, 798, 13, 119, 0, 239, 97, 818, 1711, 20, 145, 214, 59, 164, 1761, 435, 191, 1254, 75, 1754, 1015, 11, 27, 16, 110, 1862, 16, 9, 14, 201, 582, 944, 653, 75, 2645, 1076, 888, 715, 371, 1284, 550, 116, 416, 445, 462, 820, 946, 739, 148, 118, 525, 297, 663, 4] payload_rows_per_dst=[6110, 6934, 6136, 6432, 5872, 3866, 6136, 5116] dup_rows_per_dst=[3004, 2938, 2472, 3038, 2160, 766, 2792, 1764] combine_rows_per_src=[6110, 6934, 6136, 6432, 5872, 3866, 6136, 5116] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46540 payload_rows_remote=41598 dup_rows=18996 dup_rows_remote=17674 dedup_ratio=0.289856 router_tokens_per_expert=[227, 53, 251, 58, 30, 17, 1218, 1445, 347, 62, 202, 438, 8, 114, 1, 258, 106, 763, 2024, 3, 318, 165, 36, 60, 1976, 351, 360, 1110, 9, 1938, 852, 32, 5, 27, 276, 1906, 24, 0, 17, 334, 493, 1077, 998, 44, 2387, 1032, 642, 854, 71, 885, 658, 135, 457, 307, 217, 402, 1648, 655, 191, 151, 441, 626, 973, 3] payload_rows_per_dst=[4882, 6006, 5426, 6442, 6304, 5982, 4942, 6556] dup_rows_per_dst=[1716, 1628, 1524, 2862, 2826, 3118, 1322, 4000] combine_rows_per_src=[4882, 6006, 5426, 6442, 6304, 5982, 4942, 6556] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47832 payload_rows_remote=42278 dup_rows=17704 dup_rows_remote=16084 dedup_ratio=0.270142 router_tokens_per_expert=[155, 73, 28, 12, 71, 17, 1433, 1769, 310, 43, 142, 598, 5, 32, 0, 259, 81, 499, 2616, 28, 152, 177, 21, 161, 1425, 182, 211, 670, 5, 1755, 742, 22, 37, 22, 93, 2333, 2, 0, 58, 389, 774, 672, 925, 13, 2325, 666, 2449, 1199, 196, 1089, 532, 31, 296, 469, 209, 590, 1433, 316, 110, 216, 260, 568, 799, 3] payload_rows_per_dst=[7270, 6152, 6160, 5554, 6208, 5808, 5446, 5234] dup_rows_per_dst=[4744, 1538, 1310, 1620, 2510, 2428, 1378, 2176] combine_rows_per_src=[7270, 6152, 6160, 5554, 6208, 5808, 5446, 5234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 2, 1, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 2, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=296497152 dispatch_meta_bytes=236848 combine_payload_bytes=296497152 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=296734000 ingress_bytes=516698112 link_bytes=516698112 token_bytes=593231152 weight_movement_bytes=220200960 total_nvlink_bytes=813432112 weight_to_token_ratio=0.371189 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=10880 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.986229990e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=289787904 dispatch_meta_bytes=225416 combine_payload_bytes=289787904 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=290013320 ingress_bytes=399888384 link_bytes=399888384 token_bytes=579801224 weight_movement_bytes=110100480 total_nvlink_bytes=689901704 weight_to_token_ratio=0.189893 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=19456 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.954716701e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=297916416 dispatch_meta_bytes=224336 combine_payload_bytes=297916416 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=298140752 ingress_bytes=408016896 link_bytes=408016896 token_bytes=596057168 weight_movement_bytes=110100480 total_nvlink_bytes=706157648 weight_to_token_ratio=0.184715 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=11136 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.096084894e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46924 payload_rows_remote=40846 dup_rows=18612 dup_rows_remote=15422 dedup_ratio=0.283997 router_tokens_per_expert=[200, 88, 109, 25, 72, 14, 1382, 1531, 332, 15, 96, 475, 25, 38, 3, 216, 256, 572, 2351, 8, 232, 258, 13, 141, 1581, 304, 249, 976, 25, 1243, 843, 33, 42, 13, 112, 1995, 8, 1, 8, 155, 713, 828, 831, 99, 2682, 1273, 1213, 802, 211, 1373, 331, 226, 573, 289, 223, 670, 1679, 997, 237, 91, 302, 508, 568, 9] payload_rows_per_dst=[6078, 6900, 6914, 6314, 5012, 4268, 5366, 6072] dup_rows_per_dst=[3190, 2520, 3294, 2678, 1172, 1450, 1598, 2710] combine_rows_per_src=[6078, 6900, 6914, 6314, 5012, 4268, 5366, 6072] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=294705152 dispatch_meta_bytes=225496 combine_payload_bytes=294705152 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=294930648 ingress_bytes=404805632 link_bytes=404805632 token_bytes=589635800 weight_movement_bytes=110100480 total_nvlink_bytes=699736280 weight_to_token_ratio=0.186726 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=16512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.040235731e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279695360 dispatch_meta_bytes=219952 combine_payload_bytes=279695360 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=279915312 ingress_bytes=499896320 link_bytes=499896320 token_bytes=559610672 weight_movement_bytes=220200960 total_nvlink_bytes=779811632 weight_to_token_ratio=0.393490 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.044118 gemm_active_groups=11 gemm_peak_group_rows=20352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.694019193e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=289974272 dispatch_meta_bytes=208864 combine_payload_bytes=289974272 prefetch_bytes=110100480 reduce_bytes=550502400 egress_bytes=290183136 ingress_bytes=950577152 link_bytes=950577152 token_bytes=580157408 weight_movement_bytes=660602880 total_nvlink_bytes=1240760288 weight_to_token_ratio=1.138661 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=6528 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.653210010e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=303163392 dispatch_meta_bytes=233688 combine_payload_bytes=303163392 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=303397080 ingress_bytes=523364352 link_bytes=523364352 token_bytes=606560472 weight_movement_bytes=220200960 total_nvlink_bytes=826761432 weight_to_token_ratio=0.363032 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=12928 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.102166868e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47020 payload_rows_remote=40974 dup_rows=18516 dup_rows_remote=15446 dedup_ratio=0.282532 router_tokens_per_expert=[198, 83, 106, 28, 68, 13, 1388, 1557, 331, 15, 95, 471, 22, 43, 4, 219, 250, 597, 2344, 9, 242, 247, 13, 142, 1594, 291, 252, 989, 24, 1267, 848, 34, 42, 14, 112, 2004, 10, 1, 8, 144, 725, 827, 834, 100, 2640, 1316, 1117, 802, 217, 1304, 304, 249, 581, 307, 228, 663, 1681, 1001, 234, 92, 305, 531, 582, 9] payload_rows_per_dst=[6046, 6816, 6946, 6414, 4928, 4022, 5750, 6098] dup_rows_per_dst=[3070, 2532, 3374, 2912, 1014, 886, 1956, 2772] combine_rows_per_src=[6046, 6816, 6946, 6414, 4928, 4022, 5750, 6098] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=295192576 dispatch_meta_bytes=225312 combine_payload_bytes=295192576 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=295417888 ingress_bytes=405293056 link_bytes=405293056 token_bytes=590610464 weight_movement_bytes=110100480 total_nvlink_bytes=700710944 weight_to_token_ratio=0.186418 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=16512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.048712836e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=296540160 dispatch_meta_bytes=222216 combine_payload_bytes=296540160 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=296762376 ingress_bytes=406640640 link_bytes=406640640 token_bytes=593302536 weight_movement_bytes=110100480 total_nvlink_bytes=703403016 weight_to_token_ratio=0.185572 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=11136 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.072149539e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=290906112 dispatch_meta_bytes=227576 combine_payload_bytes=290906112 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=291133688 ingress_bytes=401006592 link_bytes=401006592 token_bytes=582039800 weight_movement_bytes=110100480 total_nvlink_bytes=692140280 weight_to_token_ratio=0.189163 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=19456 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.974164177e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=289443840 dispatch_meta_bytes=209552 combine_payload_bytes=289443840 prefetch_bytes=110100480 reduce_bytes=550502400 egress_bytes=289653392 ingress_bytes=950046720 link_bytes=950046720 token_bytes=579097232 weight_movement_bytes=660602880 total_nvlink_bytes=1239700112 weight_to_token_ratio=1.140746 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=6784 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.652287502e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=284340224 dispatch_meta_bytes=222656 combine_payload_bytes=284340224 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=284562880 ingress_bytes=504541184 link_bytes=504541184 token_bytes=568903104 weight_movement_bytes=220200960 total_nvlink_bytes=789104064 weight_to_token_ratio=0.387062 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.040441 gemm_active_groups=11 gemm_peak_group_rows=19968 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.774801017e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=298174464 dispatch_meta_bytes=237088 combine_payload_bytes=298174464 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=298411552 ingress_bytes=408274944 link_bytes=408274944 token_bytes=596586016 weight_movement_bytes=110100480 total_nvlink_bytes=706686496 weight_to_token_ratio=0.184551 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=10624 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.100572773e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=292784128 dispatch_meta_bytes=225072 combine_payload_bytes=292784128 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=293009200 ingress_bytes=402884608 link_bytes=402884608 token_bytes=585793328 weight_movement_bytes=110100480 total_nvlink_bytes=695893808 weight_to_token_ratio=0.187951 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=11008 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.006825964e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53036 payload_rows_remote=45190 dup_rows=12500 dup_rows_remote=11500 dedup_ratio=0.190735 router_tokens_per_expert=[0, 4094, 2753, 372, 0, 201, 1420, 1516, 0, 957, 19, 0, 0, 0, 0, 0, 0, 1635, 1057, 4, 0, 58, 0, 5, 62, 4032, 18, 0, 0, 0, 2614, 0, 0, 121, 0, 0, 0, 0, 0, 0, 597, 0, 3463, 0, 2, 0, 0, 0, 0, 0, 0, 121, 0, 569, 0, 0, 977, 0, 111, 0, 0, 0, 3733, 2257] payload_rows_per_dst=[3546, 7038, 4616, 8068, 8188, 7940, 7846, 5794] dup_rows_per_dst=[440, 3452, 902, 156, 242, 5412, 1000, 896] combine_rows_per_src=[3546, 7038, 4616, 8068, 8188, 7940, 7846, 5794] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52938 payload_rows_remote=45640 dup_rows=12598 dup_rows_remote=10022 dedup_ratio=0.192230 router_tokens_per_expert=[0, 4093, 3365, 415, 0, 187, 1654, 1726, 0, 810, 1, 0, 3, 0, 0, 0, 0, 666, 1010, 6, 1, 59, 0, 7, 29, 4014, 2, 0, 10, 0, 1926, 1, 1, 52, 0, 0, 0, 0, 0, 0, 581, 0, 3417, 0, 1, 0, 0, 0, 1, 0, 0, 84, 2, 475, 0, 0, 1588, 0, 141, 0, 0, 0, 3357, 3083] payload_rows_per_dst=[4048, 7552, 3138, 8038, 8188, 7778, 6898, 7298] dup_rows_per_dst=[464, 4258, 360, 74, 104, 4072, 690, 2576] combine_rows_per_src=[4048, 7552, 3138, 8038, 8188, 7778, 6898, 7298] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52574 payload_rows_remote=44394 dup_rows=12962 dup_rows_remote=5900 dedup_ratio=0.197784 router_tokens_per_expert=[0, 4084, 2749, 375, 0, 135, 1171, 2127, 0, 689, 8, 0, 0, 0, 0, 0, 0, 1335, 2093, 24, 0, 102, 0, 0, 56, 4067, 117, 0, 0, 0, 1747, 0, 0, 105, 0, 0, 0, 0, 0, 0, 322, 0, 3146, 0, 1, 0, 0, 0, 0, 0, 0, 409, 0, 344, 0, 0, 1132, 0, 107, 0, 0, 0, 3297, 3026] payload_rows_per_dst=[5938, 5980, 8180, 3742, 8168, 6494, 7160, 6912] dup_rows_per_dst=[1678, 912, 7062, 98, 210, 444, 940, 1618] combine_rows_per_src=[5938, 5980, 8180, 3742, 8168, 6494, 7160, 6912] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53248 payload_rows_remote=45068 dup_rows=12288 dup_rows_remote=12174 dedup_ratio=0.187500 router_tokens_per_expert=[0, 4085, 2692, 402, 4, 366, 1505, 2323, 0, 668, 23, 0, 0, 0, 3, 0, 0, 1375, 795, 19, 0, 32, 0, 2, 26, 4048, 9, 0, 2, 0, 2702, 0, 0, 56, 0, 2, 0, 0, 0, 4, 200, 0, 2642, 3, 15, 0, 0, 0, 0, 0, 1, 78, 0, 207, 0, 0, 1428, 3, 105, 0, 0, 0, 3761, 3182] payload_rows_per_dst=[6658, 5848, 3876, 8150, 8180, 5472, 7766, 7298] dup_rows_per_dst=[2542, 924, 570, 5424, 114, 248, 328, 2138] combine_rows_per_src=[6658, 5848, 3876, 8150, 8180, 5472, 7766, 7298] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52962 payload_rows_remote=46138 dup_rows=12574 dup_rows_remote=11280 dedup_ratio=0.191864 router_tokens_per_expert=[3, 4080, 2244, 548, 0, 128, 1832, 2246, 1, 1606, 19, 0, 0, 0, 7, 1, 0, 789, 559, 21, 0, 45, 1, 2, 56, 4058, 3, 3, 0, 1, 2174, 0, 2, 105, 1, 0, 0, 0, 0, 0, 602, 0, 3158, 2, 2, 0, 2, 0, 0, 0, 0, 135, 10, 117, 4, 6, 1105, 0, 55, 1, 3, 8, 3589, 3434] payload_rows_per_dst=[5852, 6648, 2550, 8148, 8170, 6824, 7328, 7442] dup_rows_per_dst=[2114, 2656, 284, 3856, 206, 1294, 394, 1770] combine_rows_per_src=[5852, 6648, 2550, 8148, 8170, 6824, 7328, 7442] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53222 payload_rows_remote=47210 dup_rows=12314 dup_rows_remote=10874 dedup_ratio=0.187897 router_tokens_per_expert=[0, 4081, 2705, 477, 2, 276, 1648, 2393, 0, 1002, 14, 1, 1, 1, 0, 2, 0, 611, 1769, 8, 0, 61, 9, 24, 121, 3800, 31, 0, 0, 4, 1840, 0, 0, 12, 0, 1, 1, 0, 0, 0, 460, 1, 2748, 4, 3, 0, 0, 0, 6, 0, 2, 70, 19, 184, 0, 0, 1639, 0, 74, 4, 0, 0, 3788, 2871] payload_rows_per_dst=[6710, 6012, 7954, 3896, 8168, 5750, 7730, 7002] dup_rows_per_dst=[2882, 1440, 4610, 96, 22, 682, 408, 2174] combine_rows_per_src=[6710, 6012, 7954, 3896, 8168, 5750, 7730, 7002] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=323921920 dispatch_meta_bytes=226760 combine_payload_bytes=323921920 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=324148680 ingress_bytes=434022400 link_bytes=434022400 token_bytes=648070600 weight_movement_bytes=110100480 total_nvlink_bytes=758171080 weight_to_token_ratio=0.169890 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=29056 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.548363380e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=303048704 dispatch_meta_bytes=233448 combine_payload_bytes=303048704 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=303282152 ingress_bytes=633350144 link_bytes=633350144 token_bytes=606330856 weight_movement_bytes=330301440 total_nvlink_bytes=936632296 weight_to_token_ratio=0.544754 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=12800 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.101500069e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=293701632 dispatch_meta_bytes=225680 combine_payload_bytes=293701632 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=293927312 ingress_bytes=403802112 link_bytes=403802112 token_bytes=587628944 weight_movement_bytes=110100480 total_nvlink_bytes=697729424 weight_to_token_ratio=0.187364 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=11136 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.022782868e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53138 payload_rows_remote=44960 dup_rows=12398 dup_rows_remote=12336 dedup_ratio=0.189178 router_tokens_per_expert=[0, 4084, 2795, 376, 5, 382, 1469, 2172, 0, 737, 24, 0, 0, 0, 2, 0, 0, 1274, 880, 21, 0, 34, 0, 1, 22, 4049, 9, 0, 2, 0, 2856, 0, 0, 30, 0, 2, 0, 0, 0, 4, 238, 0, 2611, 3, 13, 0, 0, 0, 0, 0, 1, 86, 0, 180, 0, 0, 1334, 3, 101, 0, 0, 0, 3777, 3191] payload_rows_per_dst=[6496, 6036, 3826, 8154, 8178, 5430, 7780, 7238] dup_rows_per_dst=[2312, 1080, 594, 5722, 62, 300, 308, 2020] combine_rows_per_src=[6496, 6036, 3826, 8154, 8178, 5430, 7780, 7238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52730 payload_rows_remote=44552 dup_rows=12806 dup_rows_remote=5708 dedup_ratio=0.195404 router_tokens_per_expert=[0, 4084, 2898, 362, 0, 134, 1156, 1948, 0, 667, 10, 0, 0, 0, 0, 0, 0, 1254, 2198, 20, 0, 101, 0, 0, 47, 4065, 103, 0, 0, 0, 1909, 0, 0, 91, 0, 0, 0, 0, 0, 0, 356, 0, 3115, 0, 0, 0, 0, 0, 0, 0, 1, 434, 0, 303, 0, 0, 1038, 0, 94, 0, 0, 0, 3379, 3001] payload_rows_per_dst=[5688, 6176, 8178, 3998, 8168, 6480, 7250, 6792] dup_rows_per_dst=[1512, 974, 7098, 120, 182, 462, 984, 1474] combine_rows_per_src=[5688, 6176, 8178, 3998, 8168, 6480, 7250, 6792] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52280 payload_rows_remote=45014 dup_rows=13256 dup_rows_remote=10024 dedup_ratio=0.202271 router_tokens_per_expert=[0, 4093, 3436, 373, 0, 204, 1680, 1531, 0, 828, 1, 0, 3, 0, 0, 0, 0, 600, 1120, 10, 1, 52, 0, 5, 28, 4015, 3, 0, 11, 0, 2130, 1, 1, 42, 0, 0, 0, 0, 0, 0, 600, 0, 3382, 0, 1, 0, 0, 0, 1, 0, 0, 75, 2, 451, 0, 0, 1427, 0, 137, 0, 0, 0, 3444, 3080] payload_rows_per_dst=[4072, 7582, 3236, 7382, 8188, 7798, 6756, 7266] dup_rows_per_dst=[442, 4016, 340, 78, 84, 4428, 636, 3232] combine_rows_per_src=[4072, 7582, 3236, 7382, 8188, 7798, 6756, 7266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53302 payload_rows_remote=46494 dup_rows=12234 dup_rows_remote=11324 dedup_ratio=0.186676 router_tokens_per_expert=[2, 4080, 2374, 488, 0, 132, 1875, 2038, 1, 1653, 21, 0, 0, 0, 7, 0, 0, 733, 564, 20, 0, 49, 1, 1, 53, 4058, 3, 2, 0, 2, 2288, 0, 2, 68, 1, 0, 0, 0, 0, 0, 679, 0, 3175, 2, 1, 0, 2, 0, 0, 1, 1, 130, 10, 108, 4, 5, 1026, 0, 53, 2, 3, 7, 3617, 3426] payload_rows_per_dst=[6472, 6432, 2482, 8162, 8172, 6808, 7382, 7392] dup_rows_per_dst=[2532, 1746, 254, 4650, 130, 910, 370, 1642] combine_rows_per_src=[6472, 6432, 2482, 8162, 8172, 6808, 7382, 7392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52908 payload_rows_remote=45052 dup_rows=12628 dup_rows_remote=11620 dedup_ratio=0.192688 router_tokens_per_expert=[0, 4094, 2838, 365, 0, 224, 1418, 1363, 0, 994, 25, 0, 0, 0, 0, 0, 0, 1581, 1108, 2, 0, 56, 0, 5, 61, 4029, 19, 0, 0, 0, 2718, 0, 0, 107, 0, 0, 0, 0, 0, 0, 648, 0, 3423, 0, 2, 0, 0, 0, 0, 0, 0, 139, 0, 543, 0, 0, 916, 0, 104, 0, 0, 0, 3750, 2236] payload_rows_per_dst=[3552, 7068, 4590, 8090, 8188, 7858, 7856, 5706] dup_rows_per_dst=[462, 3372, 914, 558, 214, 5294, 1008, 806] combine_rows_per_src=[3552, 7068, 4590, 8090, 8188, 7858, 7856, 5706] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53900 payload_rows_remote=47006 dup_rows=11636 dup_rows_remote=9646 dedup_ratio=0.177551 router_tokens_per_expert=[0, 4094, 2188, 304, 0, 465, 1500, 2331, 0, 728, 0, 1, 0, 0, 0, 43, 0, 1118, 851, 8, 0, 33, 3, 5, 30, 3941, 29, 0, 0, 0, 2647, 0, 13, 68, 0, 3, 0, 0, 0, 0, 345, 0, 2756, 0, 0, 3, 3, 0, 0, 22, 1, 210, 5, 354, 1, 0, 1780, 0, 123, 0, 0, 18, 3817, 2927] payload_rows_per_dst=[6656, 5124, 6164, 6894, 8190, 5792, 7890, 7190] dup_rows_per_dst=[2544, 796, 2282, 1990, 166, 422, 930, 2506] combine_rows_per_src=[6656, 5124, 6164, 6894, 8190, 5792, 7890, 7190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53542 payload_rows_remote=47282 dup_rows=11994 dup_rows_remote=10492 dedup_ratio=0.183014 router_tokens_per_expert=[0, 4082, 2852, 435, 2, 276, 1594, 2253, 0, 1008, 17, 2, 0, 1, 0, 1, 0, 576, 1859, 10, 0, 62, 10, 17, 114, 3780, 31, 0, 0, 4, 2008, 0, 0, 1, 0, 1, 1, 0, 0, 0, 540, 0, 2711, 4, 2, 0, 0, 0, 6, 0, 2, 83, 17, 149, 0, 0, 1522, 0, 67, 4, 0, 0, 3801, 2863] payload_rows_per_dst=[6544, 6260, 7934, 4200, 8168, 5752, 7772, 6912] dup_rows_per_dst=[2576, 1502, 4694, 114, 2, 762, 344, 2000] combine_rows_per_src=[6544, 6260, 7934, 4200, 8168, 5752, 7772, 6912] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=327147520 dispatch_meta_bytes=222648 combine_payload_bytes=327147520 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=327370168 ingress_bytes=657448960 link_bytes=657448960 token_bytes=654517688 weight_movement_bytes=330301440 total_nvlink_bytes=984819128 weight_to_token_ratio=0.504649 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=21248 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.143411873e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54008 payload_rows_remote=46872 dup_rows=11528 dup_rows_remote=9298 dedup_ratio=0.175903 router_tokens_per_expert=[0, 4094, 2294, 264, 0, 482, 1495, 2164, 0, 745, 1, 1, 0, 0, 0, 44, 0, 1098, 890, 8, 0, 27, 3, 5, 31, 3951, 27, 0, 0, 0, 2819, 0, 13, 42, 0, 3, 0, 0, 0, 0, 389, 0, 2728, 0, 0, 3, 4, 0, 0, 20, 1, 214, 5, 347, 1, 0, 1674, 0, 125, 0, 0, 16, 3826, 2914] payload_rows_per_dst=[6470, 5288, 6130, 7136, 8188, 5764, 7904, 7128] dup_rows_per_dst=[2340, 882, 2222, 2230, 116, 484, 924, 2330] combine_rows_per_src=[6470, 5288, 6130, 7136, 8188, 5764, 7904, 7128] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=322273280 dispatch_meta_bytes=229184 combine_payload_bytes=322273280 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=322502464 ingress_bytes=432373760 link_bytes=432373760 token_bytes=644775744 weight_movement_bytes=110100480 total_nvlink_bytes=754876224 weight_to_token_ratio=0.170758 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.519690819e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=318216192 dispatch_meta_bytes=201176 combine_payload_bytes=318216192 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=318417368 ingress_bytes=428316672 link_bytes=428316672 token_bytes=636633560 weight_movement_bytes=110100480 total_nvlink_bytes=746734040 weight_to_token_ratio=0.172942 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=14080 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.449131386e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=323047424 dispatch_meta_bytes=228968 combine_payload_bytes=323047424 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=323276392 ingress_bytes=433147904 link_bytes=433147904 token_bytes=646323816 weight_movement_bytes=110100480 total_nvlink_bytes=756424296 weight_to_token_ratio=0.170349 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.533154456e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50242 payload_rows_remote=43024 dup_rows=15294 dup_rows_remote=9048 dedup_ratio=0.233368 router_tokens_per_expert=[0, 4095, 2619, 316, 5, 312, 2009, 2028, 0, 815, 14, 0, 0, 0, 0, 0, 0, 1169, 1274, 55, 0, 68, 0, 15, 20, 3913, 0, 0, 0, 0, 2278, 0, 0, 111, 0, 0, 0, 0, 0, 0, 616, 0, 2839, 0, 0, 0, 0, 1, 0, 0, 0, 91, 1, 245, 0, 0, 1020, 1, 94, 0, 0, 0, 3777, 2967] payload_rows_per_dst=[7218, 5854, 8042, 5492, 7032, 6160, 7752, 2692] dup_rows_per_dst=[6246, 1042, 4946, 1414, 188, 752, 476, 230] combine_rows_per_src=[7218, 5854, 8042, 5492, 7032, 6160, 7752, 2692] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=330717184 dispatch_meta_bytes=229672 combine_payload_bytes=330717184 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=330946856 ingress_bytes=440817664 link_bytes=440817664 token_bytes=661664040 weight_movement_bytes=110100480 total_nvlink_bytes=771764520 weight_to_token_ratio=0.166399 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=24192 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.666544197e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=338401280 dispatch_meta_bytes=232336 combine_payload_bytes=338401280 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=338633616 ingress_bytes=558602240 link_bytes=558602240 token_bytes=677034896 weight_movement_bytes=220200960 total_nvlink_bytes=897235856 weight_to_token_ratio=0.325243 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.033088 gemm_active_groups=11 gemm_peak_group_rows=21376 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.715011696e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=319348736 dispatch_meta_bytes=201040 combine_payload_bytes=319348736 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=319549776 ingress_bytes=429449216 link_bytes=429449216 token_bytes=638898512 weight_movement_bytes=110100480 total_nvlink_bytes=748998992 weight_to_token_ratio=0.172329 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=13952 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.468828189e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=322660352 dispatch_meta_bytes=220152 combine_payload_bytes=322660352 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=322880504 ingress_bytes=652961792 link_bytes=652961792 token_bytes=645540856 weight_movement_bytes=330301440 total_nvlink_bytes=975842296 weight_to_token_ratio=0.511666 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=21504 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.135607950e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=333268992 dispatch_meta_bytes=231272 combine_payload_bytes=333268992 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=333500264 ingress_bytes=443369472 link_bytes=443369472 token_bytes=666769256 weight_movement_bytes=110100480 total_nvlink_bytes=776869736 weight_to_token_ratio=0.165125 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=24064 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.710924336e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50894 payload_rows_remote=43420 dup_rows=14642 dup_rows_remote=8050 dedup_ratio=0.223419 router_tokens_per_expert=[0, 4095, 2727, 273, 3, 345, 1986, 1777, 0, 816, 18, 0, 0, 0, 0, 0, 0, 1102, 1366, 60, 0, 67, 0, 12, 24, 3915, 0, 0, 0, 0, 2447, 0, 0, 92, 0, 0, 0, 0, 0, 0, 703, 0, 2799, 0, 0, 0, 0, 1, 0, 0, 0, 94, 1, 248, 0, 0, 935, 1, 90, 0, 0, 0, 3788, 2983] payload_rows_per_dst=[7474, 6028, 8052, 4914, 7324, 6182, 7778, 3142] dup_rows_per_dst=[6592, 1094, 4992, 28, 172, 824, 484, 456] combine_rows_per_src=[7474, 6028, 8052, 4914, 7324, 6182, 7778, 3142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=322932736 dispatch_meta_bytes=226688 combine_payload_bytes=322932736 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=323159424 ingress_bytes=433033216 link_bytes=433033216 token_bytes=646092160 weight_movement_bytes=110100480 total_nvlink_bytes=756192640 weight_to_token_ratio=0.170410 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=29184 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.531159843e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=338917376 dispatch_meta_bytes=231096 combine_payload_bytes=338917376 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=339148472 ingress_bytes=559118336 link_bytes=559118336 token_bytes=678065848 weight_movement_bytes=220200960 total_nvlink_bytes=898266808 weight_to_token_ratio=0.324749 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.033088 gemm_active_groups=11 gemm_peak_group_rows=22272 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.723987455e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 2, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=336939008 dispatch_meta_bytes=226608 combine_payload_bytes=336939008 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=337165616 ingress_bytes=667240448 link_bytes=667240448 token_bytes=674104624 weight_movement_bytes=330301440 total_nvlink_bytes=1004406064 weight_to_token_ratio=0.489985 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=17920 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.160440881e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=308396032 dispatch_meta_bytes=208288 combine_payload_bytes=308396032 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=308604320 ingress_bytes=748797952 link_bytes=748797952 token_bytes=617000352 weight_movement_bytes=440401920 total_nvlink_bytes=1057402272 weight_to_token_ratio=0.713779 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=12800 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.302282795e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45704 payload_rows_remote=37940 dup_rows=19832 dup_rows_remote=16336 dedup_ratio=0.302612 router_tokens_per_expert=[1906, 2608, 3003, 0, 2991, 0, 0, 0, 0, 46, 0, 346, 9, 4, 1048, 0, 0, 0, 0, 828, 1471, 0, 0, 0, 0, 4096, 0, 2280, 0, 0, 0, 0, 2436, 1176, 0, 503, 0, 7, 18, 2198, 0, 0, 0, 0, 4096, 0, 830, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 842, 0, 0, 0, 0, 7] payload_rows_per_dst=[0, 2642, 4036, 8192, 7348, 8192, 7764, 7530] dup_rows_per_dst=[0, 264, 562, 4560, 5328, 1660, 3496, 3962] combine_rows_per_src=[0, 2642, 4036, 8192, 7348, 8192, 7764, 7530] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46938 payload_rows_remote=39620 dup_rows=18598 dup_rows_remote=14526 dedup_ratio=0.283783 router_tokens_per_expert=[2712, 2536, 3189, 0, 2064, 0, 0, 0, 0, 54, 1, 404, 111, 7, 1404, 0, 0, 0, 0, 1151, 1731, 0, 1, 0, 0, 4096, 1, 1219, 0, 19, 0, 1, 2733, 1464, 0, 427, 0, 2, 24, 1549, 0, 1, 0, 0, 4095, 0, 849, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 890, 0, 0, 0, 0, 29] payload_rows_per_dst=[0, 3610, 4952, 8192, 6816, 8190, 7860, 7318] dup_rows_per_dst=[0, 352, 814, 2480, 4066, 3216, 3598, 4072] combine_rows_per_src=[0, 3610, 4952, 8192, 6816, 8190, 7860, 7318] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51078 payload_rows_remote=44606 dup_rows=14458 dup_rows_remote=11582 dedup_ratio=0.220612 router_tokens_per_expert=[2230, 3265, 1998, 8, 2539, 0, 0, 0, 0, 156, 1, 406, 43, 2, 1513, 0, 0, 0, 0, 1377, 1518, 0, 0, 0, 0, 4095, 0, 1371, 1, 2, 0, 0, 3314, 1068, 1, 173, 0, 0, 3, 1236, 0, 0, 2, 0, 4095, 0, 805, 1, 0, 0, 3, 0, 5, 0, 0, 0, 0, 0, 1471, 2, 0, 0, 1, 63] payload_rows_per_dst=[6702, 4588, 5712, 6748, 6472, 8190, 6538, 6128] dup_rows_per_dst=[2388, 1446, 2320, 2398, 2876, 1616, 8, 1406] combine_rows_per_src=[6702, 4588, 5712, 6748, 6472, 8190, 6538, 6128] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51610 payload_rows_remote=45420 dup_rows=13926 dup_rows_remote=12216 dedup_ratio=0.212494 router_tokens_per_expert=[2851, 3456, 2744, 0, 2535, 0, 0, 0, 0, 77, 0, 262, 97, 2, 1026, 0, 0, 0, 0, 840, 864, 0, 0, 0, 0, 4096, 0, 1894, 0, 24, 0, 0, 2246, 950, 0, 251, 0, 3, 46, 2146, 0, 0, 0, 0, 4096, 0, 1225, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 973, 0, 0, 0, 0, 62] payload_rows_per_dst=[7232, 8192, 6190, 5460, 5496, 5652, 6912, 6476] dup_rows_per_dst=[3326, 2928, 1710, 1868, 1296, 1498, 4, 1296] combine_rows_per_src=[7232, 8192, 6190, 5460, 5496, 5652, 6912, 6476] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49408 payload_rows_remote=41216 dup_rows=16128 dup_rows_remote=12452 dedup_ratio=0.246094 router_tokens_per_expert=[3049, 3602, 2989, 0, 1571, 2, 0, 0, 6, 114, 0, 694, 45, 3, 737, 0, 0, 2, 0, 1600, 1309, 1, 1, 3, 9, 4080, 3, 2208, 0, 5, 10, 0, 2101, 477, 0, 349, 0, 2, 45, 1152, 2, 3, 7, 0, 4080, 11, 1831, 0, 0, 0, 0, 1, 3, 0, 1, 0, 3, 0, 541, 0, 0, 7, 0, 109] payload_rows_per_dst=[5090, 2932, 4826, 8186, 5994, 8192, 7460, 6728] dup_rows_per_dst=[1286, 266, 1006, 4444, 2258, 3676, 1868, 1324] combine_rows_per_src=[5090, 2932, 4826, 8186, 5994, 8192, 7460, 6728] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48330 payload_rows_remote=40146 dup_rows=17206 dup_rows_remote=13438 dedup_ratio=0.262543 router_tokens_per_expert=[2706, 3230, 2237, 3, 1864, 3, 0, 0, 0, 108, 0, 461, 329, 4, 985, 0, 5, 0, 0, 1734, 947, 2, 1, 3, 0, 4089, 0, 2406, 0, 13, 0, 0, 2794, 1054, 1, 586, 1, 0, 4, 1437, 0, 0, 1, 0, 4087, 0, 1037, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5, 591, 0, 0, 6, 4, 29] payload_rows_per_dst=[7980, 8184, 7408, 5336, 5000, 2076, 6460, 5886] dup_rows_per_dst=[7394, 3768, 3564, 516, 1166, 0, 2, 796] combine_rows_per_src=[7980, 8184, 7408, 5336, 5000, 2076, 6460, 5886] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=271953920 dispatch_meta_bytes=217104 combine_payload_bytes=271953920 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=272171024 ingress_bytes=492154880 link_bytes=492154880 token_bytes=544124944 weight_movement_bytes=220200960 total_nvlink_bytes=764325904 weight_to_token_ratio=0.404688 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=25472 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.559382819e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=335978496 dispatch_meta_bytes=224680 combine_payload_bytes=335978496 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=336203176 ingress_bytes=666279936 link_bytes=666279936 token_bytes=672181672 weight_movement_bytes=330301440 total_nvlink_bytes=1002483112 weight_to_token_ratio=0.491387 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=17664 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.158770393e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=311234560 dispatch_meta_bytes=205880 combine_payload_bytes=311234560 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=311440440 ingress_bytes=751636480 link_bytes=751636480 token_bytes=622675000 weight_movement_bytes=440401920 total_nvlink_bytes=1063076920 weight_to_token_ratio=0.707274 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=12800 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.307219462e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50864 payload_rows_remote=45540 dup_rows=14672 dup_rows_remote=12878 dedup_ratio=0.223877 router_tokens_per_expert=[2225, 3301, 2077, 8, 2477, 0, 0, 0, 0, 136, 0, 349, 43, 3, 1595, 0, 0, 0, 0, 1304, 1500, 0, 0, 0, 0, 4095, 0, 1391, 1, 1, 0, 0, 3253, 946, 1, 209, 0, 0, 9, 1255, 0, 0, 2, 0, 4095, 0, 886, 0, 0, 0, 2, 0, 7, 0, 0, 0, 0, 0, 1548, 2, 0, 0, 1, 46] payload_rows_per_dst=[6230, 4326, 6516, 7204, 5324, 8190, 6876, 6198] dup_rows_per_dst=[2322, 1126, 3320, 2572, 1794, 1776, 612, 1150] combine_rows_per_src=[6230, 4326, 6516, 7204, 5324, 8190, 6876, 6198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51232 payload_rows_remote=45190 dup_rows=14304 dup_rows_remote=12662 dedup_ratio=0.218262 router_tokens_per_expert=[2762, 3309, 2853, 0, 2445, 0, 0, 0, 0, 55, 0, 208, 89, 2, 1153, 0, 0, 0, 0, 814, 824, 0, 0, 0, 0, 4096, 0, 1927, 0, 22, 0, 0, 2204, 848, 0, 284, 0, 3, 45, 2233, 0, 0, 0, 0, 4095, 0, 1405, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1028, 0, 0, 0, 0, 60] payload_rows_per_dst=[7112, 8192, 6042, 5176, 5964, 5466, 6618, 6662] dup_rows_per_dst=[3302, 3014, 1642, 1514, 2048, 1556, 8, 1220] combine_rows_per_src=[7112, 8192, 6042, 5176, 5964, 5466, 6618, 6662] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47110 payload_rows_remote=39400 dup_rows=18426 dup_rows_remote=13406 dedup_ratio=0.281158 router_tokens_per_expert=[2541, 2497, 3344, 0, 2050, 0, 0, 0, 0, 62, 1, 400, 118, 7, 1546, 0, 0, 0, 0, 1111, 1660, 0, 1, 0, 0, 4096, 1, 1249, 0, 18, 0, 1, 2600, 1336, 0, 490, 0, 2, 24, 1552, 0, 0, 0, 0, 4095, 0, 991, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 949, 0, 0, 0, 0, 22] payload_rows_per_dst=[0, 3898, 4812, 8192, 7120, 8190, 7188, 7710] dup_rows_per_dst=[0, 370, 732, 2538, 4888, 1982, 2896, 5020] combine_rows_per_src=[0, 3898, 4812, 8192, 7120, 8190, 7188, 7710] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47516 payload_rows_remote=39324 dup_rows=18020 dup_rows_remote=14004 dedup_ratio=0.274963 router_tokens_per_expert=[2906, 3543, 3059, 0, 1556, 2, 0, 0, 6, 70, 0, 585, 37, 4, 829, 0, 0, 2, 0, 1603, 1232, 1, 0, 2, 8, 4080, 3, 2219, 0, 5, 10, 0, 2232, 425, 0, 386, 0, 2, 50, 1142, 2, 4, 9, 0, 4080, 11, 1998, 0, 0, 0, 0, 1, 2, 0, 3, 0, 3, 0, 588, 0, 0, 7, 0, 61] payload_rows_per_dst=[3114, 2804, 4750, 8184, 6092, 8192, 7860, 6520] dup_rows_per_dst=[2, 258, 930, 4466, 2382, 4016, 5050, 916] combine_rows_per_src=[3114, 2804, 4750, 8184, 6092, 8192, 7860, 6520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45080 payload_rows_remote=38180 dup_rows=20456 dup_rows_remote=18450 dedup_ratio=0.312134 router_tokens_per_expert=[1842, 2584, 3104, 0, 2933, 0, 0, 0, 0, 32, 0, 320, 11, 5, 1157, 0, 0, 0, 0, 769, 1390, 0, 0, 0, 0, 4096, 0, 2361, 0, 0, 0, 0, 2335, 1098, 0, 579, 0, 8, 25, 2198, 0, 0, 0, 0, 4096, 0, 926, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 865, 0, 0, 0, 0, 7] payload_rows_per_dst=[30, 2826, 3852, 8192, 7276, 8192, 6900, 7812] dup_rows_per_dst=[0, 224, 466, 4722, 5210, 1852, 2006, 5976] combine_rows_per_src=[30, 2826, 3852, 8192, 7276, 8192, 6900, 7812] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48534 payload_rows_remote=46004 dup_rows=17002 dup_rows_remote=17000 dedup_ratio=0.259430 router_tokens_per_expert=[2456, 3722, 1617, 15, 1883, 32, 0, 0, 0, 136, 0, 1695, 28, 6, 1505, 0, 4, 0, 0, 1143, 1829, 0, 0, 0, 0, 4091, 0, 1265, 0, 1, 0, 0, 2606, 198, 0, 490, 0, 0, 14, 1319, 0, 8, 1, 0, 4078, 3, 1957, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 625, 0, 0, 2, 0, 21] payload_rows_per_dst=[5804, 8182, 7232, 2530, 3704, 8182, 7448, 5452] dup_rows_per_dst=[1290, 6740, 3932, 2, 338, 3912, 32, 756] combine_rows_per_src=[5804, 8182, 7232, 2530, 3704, 8182, 7448, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47650 payload_rows_remote=39464 dup_rows=17886 dup_rows_remote=14118 dedup_ratio=0.272919 router_tokens_per_expert=[2701, 3132, 2376, 3, 1841, 3, 0, 0, 0, 76, 0, 435, 321, 4, 1051, 0, 5, 0, 0, 1679, 895, 2, 0, 3, 1, 4090, 0, 2462, 0, 14, 0, 0, 2734, 997, 1, 664, 1, 0, 5, 1413, 0, 0, 1, 0, 4087, 0, 1094, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4, 637, 0, 0, 6, 4, 24] payload_rows_per_dst=[8156, 8186, 7300, 5076, 4980, 2190, 6264, 5498] dup_rows_per_dst=[8866, 3768, 3336, 126, 1182, 0, 4, 604] combine_rows_per_src=[8156, 8186, 7300, 5076, 4980, 2190, 6264, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=326430720 dispatch_meta_bytes=233672 combine_payload_bytes=326430720 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=326664392 ingress_bytes=546631680 link_bytes=546631680 token_bytes=653095112 weight_movement_bytes=220200960 total_nvlink_bytes=873296072 weight_to_token_ratio=0.337165 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=13696 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.506823970e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=323921920 dispatch_meta_bytes=231408 combine_payload_bytes=323921920 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=324153328 ingress_bytes=434022400 link_bytes=434022400 token_bytes=648075248 weight_movement_bytes=110100480 total_nvlink_bytes=758175728 weight_to_token_ratio=0.169888 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=12928 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.548363380e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=282419200 dispatch_meta_bytes=211224 combine_payload_bytes=282419200 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=282630424 ingress_bytes=502620160 link_bytes=502620160 token_bytes=565049624 weight_movement_bytes=220200960 total_nvlink_bytes=785250584 weight_to_token_ratio=0.389702 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.029412 gemm_active_groups=11 gemm_peak_group_rows=20864 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.741391251e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47820 payload_rows_remote=39630 dup_rows=17716 dup_rows_remote=8862 dedup_ratio=0.270325 router_tokens_per_expert=[2737, 3023, 2108, 0, 2319, 0, 0, 0, 0, 85, 0, 221, 54, 0, 1266, 0, 0, 0, 0, 969, 1477, 0, 0, 0, 1, 4095, 0, 2295, 0, 16, 0, 0, 3190, 938, 0, 140, 0, 0, 2, 2609, 2, 0, 7, 0, 4095, 0, 633, 0, 0, 0, 0, 0, 18, 1, 0, 1, 0, 0, 357, 0, 1, 0, 0, 108] payload_rows_per_dst=[8190, 8190, 7650, 4614, 6010, 1282, 6062, 5822] dup_rows_per_dst=[8854, 3252, 3622, 10, 1368, 2, 24, 584] combine_rows_per_src=[8190, 8190, 7650, 4614, 6010, 1282, 6062, 5822] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=283996160 dispatch_meta_bytes=216584 combine_payload_bytes=283996160 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=284212744 ingress_bytes=504197120 link_bytes=504197120 token_bytes=568208904 weight_movement_bytes=220200960 total_nvlink_bytes=788409864 weight_to_token_ratio=0.387535 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.029412 gemm_active_groups=11 gemm_peak_group_rows=20736 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.768817179e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=319735808 dispatch_meta_bytes=224752 combine_payload_bytes=319735808 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=319960560 ingress_bytes=539936768 link_bytes=539936768 token_bytes=639696368 weight_movement_bytes=220200960 total_nvlink_bytes=859897328 weight_to_token_ratio=0.344227 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=12928 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.390388439e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=295436288 dispatch_meta_bytes=214672 combine_payload_bytes=295436288 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=295650960 ingress_bytes=625737728 link_bytes=625737728 token_bytes=591087248 weight_movement_bytes=330301440 total_nvlink_bytes=921388688 weight_to_token_ratio=0.558803 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=22912 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.088260825e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=325570560 dispatch_meta_bytes=230544 combine_payload_bytes=325570560 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=325801104 ingress_bytes=435671040 link_bytes=435671040 token_bytes=651371664 weight_movement_bytes=110100480 total_nvlink_bytes=761472144 weight_to_token_ratio=0.169029 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=12032 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.577035941e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47736 payload_rows_remote=45048 dup_rows=17800 dup_rows_remote=17798 dedup_ratio=0.271606 router_tokens_per_expert=[2384, 3661, 1678, 15, 1796, 30, 0, 0, 0, 127, 0, 1647, 20, 8, 1628, 0, 4, 0, 0, 1061, 1676, 0, 0, 0, 0, 4091, 0, 1344, 0, 1, 0, 0, 2576, 150, 0, 553, 0, 0, 19, 1320, 0, 8, 1, 0, 4078, 3, 2163, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 692, 1, 0, 3, 0, 12] payload_rows_per_dst=[6152, 8184, 7068, 2688, 3706, 8182, 7324, 4432] dup_rows_per_dst=[2298, 6858, 3566, 2, 378, 4324, 34, 340] combine_rows_per_src=[6152, 8184, 7068, 2688, 3706, 8182, 7324, 4432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47142 payload_rows_remote=38952 dup_rows=18394 dup_rows_remote=8632 dedup_ratio=0.280670 router_tokens_per_expert=[2668, 2927, 2283, 0, 2213, 0, 0, 0, 0, 77, 0, 182, 58, 0, 1339, 0, 0, 0, 0, 990, 1435, 0, 0, 0, 1, 4095, 0, 2425, 0, 16, 0, 0, 3177, 853, 0, 160, 0, 0, 1, 2582, 2, 0, 8, 0, 4095, 0, 704, 0, 0, 0, 0, 0, 19, 1, 0, 1, 0, 0, 356, 0, 1, 0, 0, 99] payload_rows_per_dst=[8190, 8190, 7608, 4870, 5892, 1426, 5874, 5092] dup_rows_per_dst=[9762, 3312, 3596, 14, 1300, 2, 22, 386] combine_rows_per_src=[8190, 8190, 7608, 4870, 5892, 1426, 5874, 5092] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=273674240 dispatch_meta_bytes=226520 combine_payload_bytes=273674240 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=273900760 ingress_bytes=493875200 link_bytes=493875200 token_bytes=547575000 weight_movement_bytes=220200960 total_nvlink_bytes=767775960 weight_to_token_ratio=0.402138 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=24960 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.589302013e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=281874432 dispatch_meta_bytes=213312 combine_payload_bytes=281874432 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=282087744 ingress_bytes=612175872 link_bytes=612175872 token_bytes=563962176 weight_movement_bytes=330301440 total_nvlink_bytes=894263616 weight_to_token_ratio=0.585680 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=22656 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.064674527e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=282877952 dispatch_meta_bytes=214328 combine_payload_bytes=282877952 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=283092280 ingress_bytes=392978432 link_bytes=392978432 token_bytes=565970232 weight_movement_bytes=110100480 total_nvlink_bytes=676070712 weight_to_token_ratio=0.194534 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=17024 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.834541271e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=287766528 dispatch_meta_bytes=214336 combine_payload_bytes=287766528 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=287980864 ingress_bytes=397867008 link_bytes=397867008 token_bytes=575747392 weight_movement_bytes=110100480 total_nvlink_bytes=685847872 weight_to_token_ratio=0.191231 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=17280 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.919561648e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=329756672 dispatch_meta_bytes=252016 combine_payload_bytes=329756672 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=330008688 ingress_bytes=549957632 link_bytes=549957632 token_bytes=659765360 weight_movement_bytes=220200960 total_nvlink_bytes=879966320 weight_to_token_ratio=0.333756 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=15488 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.564667745e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=284067840 dispatch_meta_bytes=193968 combine_payload_bytes=284067840 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=284261808 ingress_bytes=834570240 link_bytes=834570240 token_bytes=568329648 weight_movement_bytes=550502400 total_nvlink_bytes=1118832048 weight_to_token_ratio=0.968632 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=12672 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.451454911e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52234 payload_rows_remote=44046 dup_rows=13302 dup_rows_remote=8230 dedup_ratio=0.202972 router_tokens_per_expert=[0, 0, 0, 367, 3696, 0, 0, 0, 0, 0, 0, 0, 0, 2438, 0, 0, 0, 1370, 0, 0, 0, 0, 5, 945, 4094, 0, 0, 0, 0, 0, 1, 1433, 0, 196, 0, 0, 0, 4096, 3229, 0, 11, 0, 0, 0, 4096, 159, 0, 0, 568, 0, 0, 1958, 0, 0, 0, 10, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7462, 4876, 4264, 2868, 8192, 8192, 8188, 8192] dup_rows_per_dst=[664, 0, 376, 0, 6850, 340, 5072, 0] combine_rows_per_src=[7462, 4876, 4264, 2868, 8192, 8192, 8188, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54664 payload_rows_remote=46476 dup_rows=10872 dup_rows_remote=3694 dedup_ratio=0.165894 router_tokens_per_expert=[0, 0, 0, 298, 3651, 0, 0, 0, 32, 0, 2, 0, 1, 2848, 0, 0, 0, 1599, 0, 0, 0, 2, 5, 690, 4090, 0, 1, 5, 0, 0, 0, 1098, 0, 125, 0, 0, 0, 4089, 3469, 0, 0, 0, 0, 0, 4094, 163, 0, 0, 462, 0, 0, 1832, 0, 0, 2, 120, 0, 4088, 2, 0, 0, 0, 0, 0] payload_rows_per_dst=[7346, 5746, 4310, 8186, 8188, 8190, 4520, 8178] dup_rows_per_dst=[552, 20, 282, 2202, 7178, 324, 312, 2] combine_rows_per_src=[7346, 5746, 4310, 8186, 8188, 8190, 4520, 8178] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52426 payload_rows_remote=44248 dup_rows=13110 dup_rows_remote=11720 dedup_ratio=0.200043 router_tokens_per_expert=[0, 0, 1, 285, 3765, 0, 0, 2, 7, 0, 1, 2, 0, 2833, 0, 0, 6, 725, 0, 0, 0, 2, 2, 1014, 4066, 0, 0, 0, 0, 1, 22, 1404, 2, 139, 1, 0, 5, 4096, 3425, 0, 0, 0, 0, 0, 4089, 128, 0, 0, 601, 0, 0, 2005, 0, 0, 0, 29, 10, 4094, 2, 0, 1, 0, 3, 0] payload_rows_per_dst=[7564, 5678, 3412, 6006, 8192, 8178, 5204, 8192] dup_rows_per_dst=[542, 8, 86, 1914, 8228, 1390, 914, 28] combine_rows_per_src=[7564, 5678, 3412, 6006, 8192, 8178, 5204, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50874 payload_rows_remote=42682 dup_rows=14662 dup_rows_remote=14658 dedup_ratio=0.223724 router_tokens_per_expert=[0, 1, 0, 181, 3146, 1, 0, 0, 5, 0, 0, 0, 0, 2711, 0, 0, 0, 1443, 0, 0, 0, 1, 0, 967, 4086, 1, 4, 0, 0, 0, 0, 967, 1, 67, 0, 1, 0, 4094, 3561, 1, 0, 0, 0, 0, 4095, 108, 0, 0, 1056, 0, 0, 2170, 0, 0, 0, 2, 1, 4095, 0, 0, 0, 0, 0, 2] payload_rows_per_dst=[6336, 5432, 4408, 1944, 8192, 8190, 8180, 8192] dup_rows_per_dst=[322, 0, 414, 0, 7258, 216, 6448, 4] combine_rows_per_src=[6336, 5432, 4408, 1944, 8192, 8190, 8180, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=315721728 dispatch_meta_bytes=209104 combine_payload_bytes=315721728 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=315930832 ingress_bytes=425822208 link_bytes=425822208 token_bytes=631652560 weight_movement_bytes=110100480 total_nvlink_bytes=741753040 weight_to_token_ratio=0.174305 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=17024 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.405748554e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53304 payload_rows_remote=45112 dup_rows=12232 dup_rows_remote=7256 dedup_ratio=0.186646 router_tokens_per_expert=[0, 0, 0, 251, 3499, 0, 0, 0, 2, 0, 0, 0, 0, 2719, 0, 0, 0, 1505, 0, 0, 0, 0, 0, 983, 4076, 0, 0, 1, 0, 0, 12, 1293, 0, 215, 0, 0, 0, 4096, 2755, 0, 0, 0, 0, 0, 4094, 125, 0, 0, 606, 0, 0, 2438, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7090, 7338, 8192, 8170, 430, 8188, 5704, 8192] dup_rows_per_dst=[410, 3614, 4976, 2594, 0, 250, 388, 0] combine_rows_per_src=[7090, 7338, 8192, 8170, 430, 8188, 5704, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54506 payload_rows_remote=46676 dup_rows=11030 dup_rows_remote=7440 dedup_ratio=0.168304 router_tokens_per_expert=[0, 0, 0, 252, 3715, 0, 0, 0, 6, 0, 0, 0, 0, 1914, 1, 0, 0, 1470, 15, 6, 0, 4, 15, 637, 4085, 0, 6, 1, 0, 0, 4, 665, 0, 118, 9, 0, 1, 4087, 3789, 0, 1, 0, 1, 0, 4082, 171, 0, 0, 1410, 0, 0, 2196, 0, 0, 3, 12, 8, 4082, 1, 0, 0, 1, 0, 0] payload_rows_per_dst=[7478, 7830, 8190, 8180, 254, 8168, 6234, 8172] dup_rows_per_dst=[456, 3590, 4278, 1342, 2, 342, 1008, 12] combine_rows_per_src=[7478, 7830, 8190, 8180, 254, 8168, 6234, 8172] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=322904064 dispatch_meta_bytes=251384 combine_payload_bytes=322904064 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=323155448 ingress_bytes=543105024 link_bytes=543105024 token_bytes=646059512 weight_movement_bytes=220200960 total_nvlink_bytes=866260472 weight_to_token_ratio=0.340837 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=15872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.445489622e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279207936 dispatch_meta_bytes=190336 combine_payload_bytes=279207936 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=279398272 ingress_bytes=829710336 link_bytes=829710336 token_bytes=558606208 weight_movement_bytes=550502400 total_nvlink_bytes=1109108608 weight_to_token_ratio=0.985493 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=12416 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.443002738e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56320 payload_rows_remote=48128 dup_rows=9216 dup_rows_remote=9210 dedup_ratio=0.140625 router_tokens_per_expert=[0, 1, 0, 160, 3355, 1, 0, 0, 5, 0, 0, 0, 1, 2431, 0, 0, 0, 1145, 0, 0, 0, 0, 0, 916, 4087, 1, 5, 0, 0, 0, 0, 945, 1, 67, 0, 1, 0, 4094, 3507, 1, 0, 0, 0, 0, 4095, 125, 0, 0, 1133, 0, 0, 2589, 0, 0, 0, 3, 2, 4095, 0, 0, 0, 0, 0, 2] payload_rows_per_dst=[6734, 4874, 3844, 8180, 8192, 8190, 8114, 8192] dup_rows_per_dst=[300, 0, 278, 1896, 136, 250, 6350, 6] combine_rows_per_src=[6734, 4874, 3844, 8180, 8192, 8190, 8114, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54850 payload_rows_remote=46658 dup_rows=10686 dup_rows_remote=6384 dedup_ratio=0.163055 router_tokens_per_expert=[0, 0, 0, 217, 3659, 0, 0, 0, 1, 0, 0, 0, 0, 2478, 0, 0, 0, 1245, 0, 0, 0, 0, 0, 906, 4080, 0, 0, 1, 0, 0, 15, 1224, 0, 184, 0, 0, 0, 4096, 2893, 0, 0, 0, 0, 0, 4094, 122, 0, 0, 668, 0, 0, 2788, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7382, 8188, 8192, 2468, 5856, 8188, 6384, 8192] dup_rows_per_dst=[370, 4930, 4302, 12, 298, 244, 530, 0] combine_rows_per_src=[7382, 8188, 8192, 2468, 5856, 8188, 6384, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57060 payload_rows_remote=49614 dup_rows=8476 dup_rows_remote=5812 dedup_ratio=0.129333 router_tokens_per_expert=[0, 0, 0, 253, 3790, 0, 0, 0, 34, 0, 2, 0, 1, 2547, 0, 0, 0, 1217, 0, 0, 0, 3, 5, 538, 4088, 0, 1, 5, 0, 0, 0, 1005, 0, 108, 0, 0, 0, 4089, 3527, 0, 1, 0, 0, 0, 4094, 177, 0, 0, 572, 0, 0, 2485, 0, 0, 2, 135, 0, 4087, 2, 0, 0, 0, 0, 0] payload_rows_per_dst=[7610, 5146, 6400, 8182, 7446, 8190, 5910, 8176] dup_rows_per_dst=[476, 22, 2464, 2016, 2664, 354, 478, 2] combine_rows_per_src=[7610, 5146, 6400, 8182, 7446, 8190, 5910, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54482 payload_rows_remote=48324 dup_rows=11054 dup_rows_remote=9404 dedup_ratio=0.168671 router_tokens_per_expert=[0, 0, 0, 348, 3811, 0, 0, 0, 0, 0, 0, 0, 0, 2181, 0, 0, 0, 1066, 0, 0, 0, 0, 4, 816, 4094, 0, 1, 0, 0, 0, 1, 1351, 0, 171, 0, 0, 0, 4096, 3368, 0, 12, 0, 0, 0, 4096, 174, 0, 0, 679, 0, 0, 2392, 0, 0, 0, 11, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7662, 4362, 3534, 8190, 8192, 8192, 6158, 8192] dup_rows_per_dst=[656, 0, 238, 2704, 5434, 372, 1650, 0] combine_rows_per_src=[7662, 4362, 3534, 8190, 8192, 8192, 6158, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51410 payload_rows_remote=43744 dup_rows=14126 dup_rows_remote=11060 dedup_ratio=0.215546 router_tokens_per_expert=[0, 0, 1, 129, 3522, 0, 1, 0, 0, 1, 0, 0, 0, 2281, 0, 0, 0, 1604, 0, 0, 0, 0, 4, 565, 4088, 0, 0, 0, 0, 0, 0, 846, 0, 162, 0, 0, 0, 4096, 3252, 0, 0, 0, 0, 0, 4092, 156, 0, 0, 1324, 0, 0, 2542, 0, 0, 1, 6, 0, 4095, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7666, 7488, 8192, 8180, 324, 4916, 6476, 8168] dup_rows_per_dst=[3066, 3580, 4346, 1710, 0, 154, 1270, 0] combine_rows_per_src=[7666, 7488, 8192, 8180, 324, 4916, 6476, 8168] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54108 payload_rows_remote=45918 dup_rows=11428 dup_rows_remote=7378 dedup_ratio=0.174377 router_tokens_per_expert=[0, 2, 0, 726, 3563, 0, 0, 0, 48, 0, 0, 0, 0, 3221, 0, 0, 0, 657, 0, 0, 0, 0, 1, 431, 4089, 0, 9, 0, 0, 0, 22, 2000, 0, 167, 0, 0, 0, 4096, 2576, 0, 7, 0, 2, 0, 4051, 189, 0, 0, 882, 0, 0, 1833, 0, 0, 0, 94, 10, 4086, 6, 0, 0, 0, 0, 0] payload_rows_per_dst=[7422, 7382, 7892, 8190, 1798, 8124, 5118, 8182] dup_rows_per_dst=[1160, 2954, 1918, 4050, 450, 374, 500, 22] combine_rows_per_src=[7422, 7382, 7892, 8190, 1798, 8124, 5118, 8182] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=333139968 dispatch_meta_bytes=200680 combine_payload_bytes=333139968 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=333340648 ingress_bytes=663441408 link_bytes=663441408 token_bytes=666480616 weight_movement_bytes=330301440 total_nvlink_bytes=996782056 weight_to_token_ratio=0.495590 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.029412 gemm_active_groups=10 gemm_peak_group_rows=16768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.153833726e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=317169664 dispatch_meta_bytes=223872 combine_payload_bytes=317169664 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=317393536 ingress_bytes=537370624 link_bytes=537370624 token_bytes=634563200 weight_movement_bytes=220200960 total_nvlink_bytes=854764160 weight_to_token_ratio=0.347012 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=31104 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.345758974e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=323362816 dispatch_meta_bytes=209472 combine_payload_bytes=323362816 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=323572288 ingress_bytes=433463296 link_bytes=433463296 token_bytes=646935104 weight_movement_bytes=110100480 total_nvlink_bytes=757035584 weight_to_token_ratio=0.170188 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=16128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.538639642e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=305944576 dispatch_meta_bytes=229360 combine_payload_bytes=305944576 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=306173936 ingress_bytes=416045056 link_bytes=416045056 token_bytes=612118512 weight_movement_bytes=110100480 total_nvlink_bytes=722218992 weight_to_token_ratio=0.179868 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.235707800e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54450 payload_rows_remote=46270 dup_rows=11086 dup_rows_remote=10780 dedup_ratio=0.169159 router_tokens_per_expert=[0, 0, 1, 252, 3882, 0, 0, 1, 8, 0, 1, 2, 0, 2587, 0, 0, 6, 513, 0, 0, 1, 2, 2, 899, 4063, 0, 1, 0, 0, 1, 21, 1264, 2, 139, 1, 0, 5, 4096, 3453, 0, 0, 0, 0, 0, 4090, 153, 0, 0, 719, 0, 0, 2467, 0, 0, 0, 27, 10, 4094, 1, 0, 1, 0, 3, 0] payload_rows_per_dst=[7782, 5190, 2780, 8172, 8192, 8180, 5962, 8192] dup_rows_per_dst=[490, 6, 66, 2528, 7200, 306, 464, 26] combine_rows_per_src=[7782, 5190, 2780, 8172, 8192, 8180, 5962, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55922 payload_rows_remote=47744 dup_rows=9614 dup_rows_remote=6246 dedup_ratio=0.146698 router_tokens_per_expert=[0, 0, 0, 245, 3800, 0, 0, 0, 7, 0, 0, 0, 0, 1684, 0, 0, 0, 1249, 15, 6, 0, 4, 16, 526, 4082, 0, 7, 1, 0, 0, 4, 652, 0, 106, 9, 0, 1, 4087, 3807, 0, 1, 0, 1, 0, 4081, 188, 0, 0, 1486, 0, 0, 2601, 0, 0, 2, 10, 6, 4081, 1, 0, 0, 2, 0, 0] payload_rows_per_dst=[7648, 8178, 8192, 3588, 5124, 8166, 6854, 8172] dup_rows_per_dst=[442, 3368, 3614, 330, 132, 376, 1344, 8] combine_rows_per_src=[7648, 8178, 8192, 3588, 5124, 8166, 6854, 8172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=344981504 dispatch_meta_bytes=229352 combine_payload_bytes=344981504 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=345210856 ingress_bytes=455081984 link_bytes=455081984 token_bytes=690192360 weight_movement_bytes=110100480 total_nvlink_bytes=800292840 weight_to_token_ratio=0.159521 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.914624183e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=334444544 dispatch_meta_bytes=212168 combine_payload_bytes=334444544 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=334656712 ingress_bytes=444545024 link_bytes=444545024 token_bytes=669101256 weight_movement_bytes=110100480 total_nvlink_bytes=779201736 weight_to_token_ratio=0.164550 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=19072 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.731369118e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53400 payload_rows_remote=45692 dup_rows=12136 dup_rows_remote=10420 dedup_ratio=0.185181 router_tokens_per_expert=[0, 0, 1, 50, 3736, 0, 1, 0, 0, 1, 0, 0, 0, 2024, 0, 0, 0, 1396, 0, 0, 0, 0, 4, 429, 4088, 0, 0, 0, 0, 0, 0, 697, 0, 140, 0, 0, 0, 4096, 3315, 0, 0, 0, 0, 0, 4092, 185, 0, 0, 1517, 0, 0, 2892, 0, 0, 1, 8, 0, 4095, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7708, 8192, 8192, 7290, 988, 5766, 7088, 8176] dup_rows_per_dst=[1716, 4034, 3658, 734, 20, 226, 1748, 0] combine_rows_per_src=[7708, 8192, 8192, 7290, 988, 5766, 7088, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=355633152 dispatch_meta_bytes=221704 combine_payload_bytes=355633152 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=355854856 ingress_bytes=796035072 link_bytes=796035072 token_bytes=711488008 weight_movement_bytes=440401920 total_nvlink_bytes=1151889928 weight_to_token_ratio=0.618987 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=17664 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.384435915e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=346386432 dispatch_meta_bytes=230912 combine_payload_bytes=346386432 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=346617344 ingress_bytes=456486912 link_bytes=456486912 token_bytes=693003776 weight_movement_bytes=110100480 total_nvlink_bytes=803104256 weight_to_token_ratio=0.158874 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=20480 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.939058192e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57594 payload_rows_remote=50450 dup_rows=7942 dup_rows_remote=5840 dedup_ratio=0.121185 router_tokens_per_expert=[0, 2, 0, 620, 3658, 0, 0, 0, 47, 0, 0, 0, 0, 2998, 0, 0, 0, 451, 0, 0, 0, 0, 7, 337, 4087, 0, 8, 0, 0, 0, 20, 1986, 0, 147, 0, 0, 0, 4096, 2684, 0, 10, 0, 2, 0, 4053, 203, 0, 0, 937, 0, 0, 2224, 0, 0, 0, 90, 8, 4086, 7, 0, 0, 0, 0, 0] payload_rows_per_dst=[7544, 7134, 8192, 7144, 5410, 8132, 5856, 8182] dup_rows_per_dst=[1016, 1912, 1590, 2102, 252, 404, 646, 20] combine_rows_per_src=[7544, 7134, 8192, 7144, 5410, 8132, 5856, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=331663360 dispatch_meta_bytes=228200 combine_payload_bytes=331663360 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=331891560 ingress_bytes=661964800 link_bytes=661964800 token_bytes=663554920 weight_movement_bytes=330301440 total_nvlink_bytes=993856360 weight_to_token_ratio=0.497776 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=31488 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.151265662e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=334573568 dispatch_meta_bytes=216464 combine_payload_bytes=334573568 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=334790032 ingress_bytes=444674048 link_bytes=444674048 token_bytes=669363600 weight_movement_bytes=110100480 total_nvlink_bytes=779464080 weight_to_token_ratio=0.164485 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=20992 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.733613058e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=313556992 dispatch_meta_bytes=219216 combine_payload_bytes=313556992 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=313776208 ingress_bytes=423657472 link_bytes=423657472 token_bytes=627333200 weight_movement_bytes=110100480 total_nvlink_bytes=737433680 weight_to_token_ratio=0.175506 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=28672 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.368100235e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65270 payload_rows_remote=57094 dup_rows=266 dup_rows_remote=266 dedup_ratio=0.004059 router_tokens_per_expert=[0, 0, 4096, 72, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 898, 0, 0, 4096, 4088, 3180, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 4048, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8042, 8188, 8096, 8192, 8176, 8192] dup_rows_per_dst=[144, 0, 114, 0, 0, 8, 0, 0] combine_rows_per_src=[8192, 8192, 8042, 8188, 8096, 8192, 8176, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=329140224 dispatch_meta_bytes=213184 combine_payload_bytes=329140224 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=329353408 ingress_bytes=769542144 link_bytes=769542144 token_bytes=658493632 weight_movement_bytes=440401920 total_nvlink_bytes=1098895552 weight_to_token_ratio=0.668802 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=23040 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.338360356e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=342228992 dispatch_meta_bytes=215960 combine_payload_bytes=342228992 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=342444952 ingress_bytes=452329472 link_bytes=452329472 token_bytes=684673944 weight_movement_bytes=110100480 total_nvlink_bytes=794774424 weight_to_token_ratio=0.160807 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=18944 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.866753472e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=327520256 dispatch_meta_bytes=224448 combine_payload_bytes=327520256 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=327744704 ingress_bytes=437620736 link_bytes=437620736 token_bytes=655264960 weight_movement_bytes=110100480 total_nvlink_bytes=765365440 weight_to_token_ratio=0.168024 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=29696 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.610944361e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=361625600 dispatch_meta_bytes=225160 combine_payload_bytes=361625600 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=361850760 ingress_bytes=581826560 link_bytes=581826560 token_bytes=723476360 weight_movement_bytes=220200960 total_nvlink_bytes=943677320 weight_to_token_ratio=0.304365 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=18944 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.011892082e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63702 payload_rows_remote=55512 dup_rows=1834 dup_rows_remote=1834 dedup_ratio=0.027985 router_tokens_per_expert=[0, 0, 4095, 53, 0, 0, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1523, 1, 0, 4092, 4008, 2650, 1, 0, 0, 0, 4094, 0, 4037, 0, 0, 0, 4091, 1, 1, 1, 0, 0, 16, 0, 0, 4, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7270, 8164, 7588, 8080, 8184, 8192, 8034, 8190] dup_rows_per_dst=[82, 0, 346, 310, 384, 40, 672, 0] combine_rows_per_src=[7270, 8164, 7588, 8080, 8184, 8192, 8034, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65264 payload_rows_remote=57222 dup_rows=272 dup_rows_remote=162 dedup_ratio=0.004150 router_tokens_per_expert=[0, 0, 4095, 73, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 746, 0, 0, 4096, 4069, 3330, 0, 0, 0, 0, 4096, 0, 4078, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8042, 8156, 8162, 8192, 8138, 8192] dup_rows_per_dst=[146, 0, 110, 0, 0, 16, 0, 0] combine_rows_per_src=[8190, 8192, 8042, 8156, 8162, 8192, 8138, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65280 payload_rows_remote=57106 dup_rows=256 dup_rows_remote=256 dedup_ratio=0.003906 router_tokens_per_expert=[0, 0, 4095, 65, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 778, 0, 0, 4096, 4087, 3307, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 4, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8054, 8188, 8098, 8192, 8174, 8192] dup_rows_per_dst=[130, 0, 116, 0, 0, 10, 0, 0] combine_rows_per_src=[8190, 8192, 8054, 8188, 8098, 8192, 8174, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65282 payload_rows_remote=57132 dup_rows=254 dup_rows_remote=248 dedup_ratio=0.003876 router_tokens_per_expert=[0, 4, 4093, 13, 0, 0, 4094, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1145, 0, 0, 4092, 4057, 3031, 2, 0, 0, 0, 4093, 0, 4045, 0, 0, 0, 4074, 1, 0, 0, 3, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 2, 0, 0, 2, 0, 0, 3, 5] payload_rows_per_dst=[8186, 8188, 8168, 8094, 8150, 8188, 8116, 8192] dup_rows_per_dst=[34, 2, 184, 0, 6, 6, 2, 20] combine_rows_per_src=[8186, 8188, 8168, 8094, 8150, 8188, 8116, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 2, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65258 payload_rows_remote=57120 dup_rows=278 dup_rows_remote=274 dedup_ratio=0.004242 router_tokens_per_expert=[0, 4, 4094, 36, 0, 0, 4094, 0, 2, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1279, 0, 0, 4093, 4068, 2869, 2, 0, 0, 0, 4093, 0, 4045, 0, 0, 0, 4068, 1, 0, 0, 2, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 2, 0, 0, 3, 4] payload_rows_per_dst=[8188, 8192, 8128, 8094, 8138, 8188, 8138, 8192] dup_rows_per_dst=[80, 4, 168, 0, 4, 6, 2, 14] combine_rows_per_src=[8188, 8192, 8128, 8094, 8138, 8188, 8138, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=409249792 dispatch_meta_bytes=229440 combine_payload_bytes=409249792 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=409479232 ingress_bytes=629450752 link_bytes=629450752 token_bytes=818729024 weight_movement_bytes=220200960 total_nvlink_bytes=1038929984 weight_to_token_ratio=0.268955 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.094718385e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64950 payload_rows_remote=56762 dup_rows=586 dup_rows_remote=526 dedup_ratio=0.008942 router_tokens_per_expert=[0, 1, 4083, 145, 0, 0, 4084, 0, 0, 0, 0, 7, 1, 0, 0, 5, 0, 0, 965, 0, 0, 4095, 4067, 3038, 0, 2, 1, 0, 4087, 0, 4074, 0, 0, 0, 4046, 3, 0, 0, 4, 0, 27, 0, 1, 9, 0, 0, 0, 0, 6, 0, 1, 11, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8168, 8182, 7838, 8150, 8098, 8188, 8136, 8190] dup_rows_per_dst=[290, 12, 168, 4, 8, 60, 34, 10] combine_rows_per_src=[8168, 8182, 7838, 8150, 8098, 8188, 8136, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65220 payload_rows_remote=57190 dup_rows=316 dup_rows_remote=174 dedup_ratio=0.004822 router_tokens_per_expert=[0, 0, 4095, 79, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 886, 0, 0, 4096, 4073, 3200, 0, 0, 0, 0, 4096, 0, 4059, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8030, 8118, 8160, 8192, 8146, 8192] dup_rows_per_dst=[158, 0, 142, 0, 0, 16, 0, 0] combine_rows_per_src=[8190, 8192, 8030, 8118, 8160, 8192, 8146, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63434 payload_rows_remote=55250 dup_rows=2102 dup_rows_remote=2102 dedup_ratio=0.032074 router_tokens_per_expert=[0, 0, 4095, 75, 1, 0, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1521, 1, 0, 4092, 4034, 2627, 1, 0, 0, 0, 4094, 0, 4014, 0, 0, 0, 4091, 2, 1, 1, 0, 0, 16, 0, 0, 4, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6980, 8192, 7592, 8034, 8184, 8192, 8076, 8184] dup_rows_per_dst=[90, 0, 560, 488, 442, 40, 482, 0] combine_rows_per_src=[6980, 8192, 7592, 8034, 8184, 8192, 8076, 8184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65086 payload_rows_remote=56898 dup_rows=450 dup_rows_remote=442 dedup_ratio=0.006866 router_tokens_per_expert=[0, 0, 4093, 20, 0, 0, 4089, 0, 6, 0, 1, 0, 1, 1, 0, 0, 0, 0, 635, 2, 0, 4092, 4048, 3585, 0, 0, 0, 0, 4094, 0, 4035, 0, 0, 3, 4007, 0, 0, 0, 13, 0, 8, 1, 0, 14, 1, 0, 0, 1, 0, 3, 0, 11, 0, 0, 0, 0, 1, 0, 1, 0, 2, 0, 0, 0] payload_rows_per_dst=[8186, 8188, 8124, 8070, 8020, 8190, 8120, 8188] dup_rows_per_dst=[40, 8, 320, 0, 26, 48, 4, 4] combine_rows_per_src=[8186, 8188, 8124, 8070, 8020, 8190, 8120, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64982 payload_rows_remote=56794 dup_rows=554 dup_rows_remote=498 dedup_ratio=0.008453 router_tokens_per_expert=[0, 1, 4084, 122, 0, 0, 4084, 0, 0, 0, 0, 7, 1, 0, 0, 5, 0, 0, 1002, 0, 0, 4094, 4050, 3024, 0, 2, 1, 0, 4088, 0, 4079, 0, 0, 0, 4056, 3, 0, 0, 5, 0, 25, 0, 1, 8, 0, 0, 0, 0, 6, 0, 1, 12, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8170, 8192, 7876, 8160, 8118, 8188, 8102, 8176] dup_rows_per_dst=[244, 22, 176, 4, 10, 56, 36, 6] combine_rows_per_src=[8170, 8192, 7876, 8160, 8118, 8188, 8102, 8176] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64992 payload_rows_remote=56812 dup_rows=544 dup_rows_remote=414 dedup_ratio=0.008301 router_tokens_per_expert=[0, 0, 4089, 62, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1081, 1, 0, 4095, 4067, 3061, 0, 0, 0, 0, 4095, 0, 4014, 1, 0, 0, 4094, 0, 0, 0, 0, 0, 9, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8180, 8182, 8078, 8078, 8188, 7974, 8136, 8176] dup_rows_per_dst=[130, 0, 224, 186, 0, 4, 0, 0] combine_rows_per_src=[8180, 8182, 8078, 8078, 8188, 7974, 8136, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65138 payload_rows_remote=56948 dup_rows=398 dup_rows_remote=386 dedup_ratio=0.006073 router_tokens_per_expert=[0, 0, 4091, 13, 0, 0, 4088, 0, 5, 0, 1, 0, 1, 1, 0, 0, 0, 0, 562, 2, 0, 4093, 4038, 3646, 0, 0, 0, 0, 4094, 0, 4054, 0, 0, 3, 4020, 0, 0, 0, 11, 0, 7, 2, 0, 16, 0, 0, 0, 1, 0, 2, 0, 12, 0, 0, 0, 0, 1, 0, 1, 0, 2, 0, 0, 1] payload_rows_per_dst=[8182, 8190, 8136, 8108, 8046, 8192, 8102, 8182] dup_rows_per_dst=[26, 12, 284, 0, 22, 48, 2, 4] combine_rows_per_src=[8182, 8190, 8136, 8108, 8046, 8192, 8102, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=410167296 dispatch_meta_bytes=229536 combine_payload_bytes=410167296 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=410396832 ingress_bytes=850569216 link_bytes=850569216 token_bytes=820564128 weight_movement_bytes=440401920 total_nvlink_bytes=1260966048 weight_to_token_ratio=0.536706 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=24320 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.479279761e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=397910016 dispatch_meta_bytes=229384 combine_payload_bytes=397910016 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=398139400 ingress_bytes=508010496 link_bytes=508010496 token_bytes=796049416 weight_movement_bytes=110100480 total_nvlink_bytes=906149896 weight_to_token_ratio=0.138309 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.835138059e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=409335808 dispatch_meta_bytes=229448 combine_payload_bytes=409335808 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=409565256 ingress_bytes=629536768 link_bytes=629536768 token_bytes=818901064 weight_movement_bytes=220200960 total_nvlink_bytes=1039102024 weight_to_token_ratio=0.268898 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.029412 gemm_active_groups=10 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.094867981e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65018 payload_rows_remote=56892 dup_rows=518 dup_rows_remote=518 dedup_ratio=0.007904 router_tokens_per_expert=[0, 0, 4095, 165, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1654, 0, 0, 4096, 4043, 2350, 0, 0, 0, 0, 4096, 0, 4063, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 13, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1] payload_rows_per_dst=[8190, 8192, 7868, 8126, 8172, 8192, 8086, 8192] dup_rows_per_dst=[330, 0, 140, 0, 0, 46, 0, 2] combine_rows_per_src=[8190, 8192, 7868, 8126, 8172, 8192, 8086, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=406870016 dispatch_meta_bytes=229152 combine_payload_bytes=406870016 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=407099168 ingress_bytes=516970496 link_bytes=516970496 token_bytes=813969184 weight_movement_bytes=110100480 total_nvlink_bytes=924069664 weight_to_token_ratio=0.135264 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.990967196e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=409436160 dispatch_meta_bytes=229576 combine_payload_bytes=409436160 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=409665736 ingress_bytes=519536640 link_bytes=519536640 token_bytes=819101896 weight_movement_bytes=110100480 total_nvlink_bytes=929202376 weight_to_token_ratio=0.134416 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.035596661e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=396032000 dispatch_meta_bytes=229408 combine_payload_bytes=396032000 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=396261408 ingress_bytes=506132480 link_bytes=506132480 token_bytes=792293408 weight_movement_bytes=110100480 total_nvlink_bytes=902393888 weight_to_token_ratio=0.138964 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.802476272e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=409522176 dispatch_meta_bytes=229520 combine_payload_bytes=409522176 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=409751696 ingress_bytes=519622656 link_bytes=519622656 token_bytes=819273872 weight_movement_bytes=110100480 total_nvlink_bytes=929374352 weight_to_token_ratio=0.134388 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.037092620e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64986 payload_rows_remote=56816 dup_rows=550 dup_rows_remote=434 dedup_ratio=0.008392 router_tokens_per_expert=[0, 0, 4084, 57, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 972, 0, 0, 4095, 4047, 3187, 0, 0, 0, 0, 4095, 0, 4027, 1, 0, 0, 4094, 0, 0, 0, 0, 0, 9, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8170, 8190, 8088, 8098, 8188, 7968, 8096, 8188] dup_rows_per_dst=[116, 0, 230, 198, 0, 4, 2, 0] combine_rows_per_src=[8170, 8190, 8088, 8098, 8188, 7968, 8096, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65098 payload_rows_remote=56958 dup_rows=438 dup_rows_remote=438 dedup_ratio=0.006683 router_tokens_per_expert=[0, 0, 4094, 106, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1697, 0, 0, 4096, 4030, 2372, 0, 0, 0, 0, 4096, 0, 4070, 0, 0, 0, 4083, 0, 0, 0, 1, 0, 16, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1] payload_rows_per_dst=[8188, 8192, 7968, 8140, 8166, 8192, 8060, 8192] dup_rows_per_dst=[212, 0, 170, 0, 2, 52, 0, 2] combine_rows_per_src=[8188, 8192, 7968, 8140, 8166, 8192, 8060, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=407099392 dispatch_meta_bytes=229168 combine_payload_bytes=407099392 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=407328560 ingress_bytes=517199872 link_bytes=517199872 token_bytes=814427952 weight_movement_bytes=110100480 total_nvlink_bytes=924528432 weight_to_token_ratio=0.135188 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.994956422e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=408203264 dispatch_meta_bytes=229336 combine_payload_bytes=408203264 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=408432600 ingress_bytes=518303744 link_bytes=518303744 token_bytes=816635864 weight_movement_bytes=110100480 total_nvlink_bytes=926736344 weight_to_token_ratio=0.134822 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.014154571e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=409937920 dispatch_meta_bytes=229456 combine_payload_bytes=409937920 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=410167376 ingress_bytes=850339840 link_bytes=850339840 token_bytes=820105296 weight_movement_bytes=440401920 total_nvlink_bytes=1260507216 weight_to_token_ratio=0.537007 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=23680 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.478880839e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=407844864 dispatch_meta_bytes=229360 combine_payload_bytes=407844864 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=408074224 ingress_bytes=517945344 link_bytes=517945344 token_bytes=815919088 weight_movement_bytes=110100480 total_nvlink_bytes=926019568 weight_to_token_ratio=0.134940 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.007921406e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=407228416 dispatch_meta_bytes=228904 combine_payload_bytes=407228416 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=407457320 ingress_bytes=847630336 link_bytes=847630336 token_bytes=814685736 weight_movement_bytes=440401920 total_nvlink_bytes=1255087656 weight_to_token_ratio=0.540579 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.474168566e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54724 payload_rows_remote=46546 dup_rows=10812 dup_rows_remote=6576 dedup_ratio=0.164978 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 644, 0, 275, 0, 0, 0, 0, 3881, 553, 0, 0, 0, 0, 1224, 0, 0, 0, 0, 4084, 4036, 0, 0, 0, 4096, 0, 0, 208, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2171, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3404, 0, 0, 0] payload_rows_per_dst=[8168, 3788, 7444, 2852, 8192, 8192, 8178, 7910] dup_rows_per_dst=[0, 470, 860, 160, 416, 0, 4236, 4670] combine_rows_per_src=[8168, 3788, 7444, 2852, 8192, 8192, 8178, 7910] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=407801856 dispatch_meta_bytes=229640 combine_payload_bytes=407801856 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=408031496 ingress_bytes=628002816 link_bytes=628002816 token_bytes=815833352 weight_movement_bytes=220200960 total_nvlink_bytes=1036034312 weight_to_token_ratio=0.269909 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.092200186e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=407257088 dispatch_meta_bytes=229000 combine_payload_bytes=407257088 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=407486088 ingress_bytes=847659008 link_bytes=847659008 token_bytes=814743176 weight_movement_bytes=440401920 total_nvlink_bytes=1255145096 weight_to_token_ratio=0.540541 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.474218431e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=408274944 dispatch_meta_bytes=229584 combine_payload_bytes=408274944 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=408504528 ingress_bytes=628475904 link_bytes=628475904 token_bytes=816779472 weight_movement_bytes=220200960 total_nvlink_bytes=1036980432 weight_to_token_ratio=0.269597 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.093022964e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61718 payload_rows_remote=53702 dup_rows=3818 dup_rows_remote=3106 dedup_ratio=0.058258 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 206, 0, 395, 0, 0, 0, 0, 3852, 512, 0, 0, 0, 0, 716, 0, 0, 0, 0, 4096, 4069, 0, 0, 0, 4096, 0, 0, 243, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3388, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3003, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8016, 8152, 8192, 8192, 6776, 6006] dup_rows_per_dst=[0, 1202, 712, 1418, 486, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8016, 8152, 8192, 8192, 6776, 6006] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46668 payload_rows_remote=38476 dup_rows=18868 dup_rows_remote=8254 dedup_ratio=0.287903 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 572, 0, 179, 0, 0, 0, 0, 3812, 29, 0, 0, 0, 0, 814, 0, 0, 0, 0, 4091, 4035, 0, 0, 0, 4095, 0, 0, 224, 0, 2, 0, 4094, 0, 0, 1, 3, 0, 1, 0, 1, 3526, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 3190, 0, 0, 3] payload_rows_per_dst=[8186, 1490, 2600, 1628, 8192, 8192, 8188, 8192] dup_rows_per_dst=[0, 12, 10, 0, 1290, 6, 6936, 10614] combine_rows_per_src=[8186, 1490, 2600, 1628, 8192, 8192, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55778 payload_rows_remote=48850 dup_rows=9758 dup_rows_remote=5988 dedup_ratio=0.148895 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 629, 0, 130, 0, 0, 0, 0, 3727, 548, 0, 0, 0, 0, 1432, 0, 0, 0, 0, 4079, 4039, 0, 0, 0, 4096, 0, 0, 234, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2341, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3321, 0, 0, 0] payload_rows_per_dst=[8158, 4728, 7714, 4278, 8192, 8192, 6928, 7588] dup_rows_per_dst=[0, 804, 836, 648, 468, 0, 3770, 3232] combine_rows_per_src=[8158, 4728, 7714, 4278, 8192, 8192, 6928, 7588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60746 payload_rows_remote=52554 dup_rows=4790 dup_rows_remote=4568 dedup_ratio=0.073090 router_tokens_per_expert=[0, 2, 6, 1, 0, 1, 0, 0, 0, 0, 0, 1037, 0, 224, 0, 0, 0, 0, 3507, 498, 0, 0, 0, 2, 760, 0, 0, 0, 0, 4085, 4026, 0, 0, 1, 4095, 0, 0, 111, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 1, 3086, 1, 0, 0, 0, 0, 0, 4087, 1, 0, 3134, 1, 1, 3] payload_rows_per_dst=[8176, 8186, 7468, 8078, 8192, 8192, 6176, 6278] dup_rows_per_dst=[14, 2510, 546, 1494, 222, 0, 2, 2] combine_rows_per_src=[8176, 8186, 7468, 8078, 8192, 8192, 6176, 6278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=333641728 dispatch_meta_bytes=212488 combine_payload_bytes=333641728 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=333854216 ingress_bytes=443742208 link_bytes=443742208 token_bytes=667495944 weight_movement_bytes=110100480 total_nvlink_bytes=777596424 weight_to_token_ratio=0.164946 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=24704 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.717406828e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62944 payload_rows_remote=54752 dup_rows=2592 dup_rows_remote=2592 dedup_ratio=0.039551 router_tokens_per_expert=[0, 0, 0, 0, 0, 9, 0, 0, 0, 2, 0, 113, 0, 558, 0, 0, 0, 0, 3875, 435, 0, 3, 0, 0, 266, 0, 0, 0, 0, 4088, 4075, 0, 1, 0, 4090, 0, 1, 58, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 3452, 3, 2, 2, 0, 0, 0, 4090, 13, 0, 3532, 0, 1, 0] payload_rows_per_dst=[8186, 8190, 8092, 8076, 8190, 8192, 6926, 7092] dup_rows_per_dst=[8, 1336, 534, 522, 112, 0, 80, 0] combine_rows_per_src=[8186, 8190, 8092, 8076, 8190, 8192, 6926, 7092] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50564 payload_rows_remote=42372 dup_rows=14972 dup_rows_remote=8080 dedup_ratio=0.228455 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 478, 0, 309, 0, 0, 0, 0, 3899, 23, 0, 0, 0, 0, 642, 0, 0, 0, 0, 4090, 4024, 0, 0, 0, 4095, 0, 0, 202, 0, 2, 0, 4094, 0, 0, 1, 3, 0, 1, 0, 1, 3362, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 3443, 0, 0, 3] payload_rows_per_dst=[8184, 1550, 46, 8024, 8192, 8188, 8188, 8192] dup_rows_per_dst=[0, 24, 0, 1058, 406, 6, 6586, 6892] combine_rows_per_src=[8184, 1550, 46, 8024, 8192, 8188, 8188, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61038 payload_rows_remote=52846 dup_rows=4498 dup_rows_remote=4282 dedup_ratio=0.068634 router_tokens_per_expert=[0, 1, 5, 1, 0, 1, 0, 0, 0, 0, 0, 866, 0, 427, 1, 0, 0, 0, 3764, 473, 0, 0, 0, 2, 545, 0, 0, 0, 0, 4084, 4029, 0, 0, 1, 4095, 0, 0, 108, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 1, 2922, 1, 0, 0, 0, 0, 0, 4087, 1, 0, 3251, 1, 1, 3] payload_rows_per_dst=[8178, 8190, 7858, 8068, 8192, 8192, 5848, 6512] dup_rows_per_dst=[6, 2572, 620, 1080, 216, 0, 2, 2] combine_rows_per_src=[8178, 8190, 7858, 8068, 8192, 8192, 5848, 6512] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62178 payload_rows_remote=54114 dup_rows=3358 dup_rows_remote=2878 dedup_ratio=0.051239 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 202, 0, 547, 0, 0, 0, 0, 3937, 335, 0, 0, 0, 0, 499, 0, 0, 0, 0, 4094, 4072, 0, 0, 0, 4096, 0, 0, 197, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3284, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3313, 0, 0, 0] payload_rows_per_dst=[8188, 8192, 8064, 8156, 8192, 8192, 6568, 6626] dup_rows_per_dst=[0, 1498, 480, 986, 394, 0, 0, 0] combine_rows_per_src=[8188, 8192, 8064, 8156, 8192, 8192, 6568, 6626] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60528 payload_rows_remote=52336 dup_rows=5008 dup_rows_remote=3640 dedup_ratio=0.076416 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4, 1, 0, 3, 0, 281, 0, 405, 0, 0, 0, 1, 3975, 1198, 0, 0, 0, 0, 561, 0, 0, 0, 0, 4095, 4026, 0, 0, 0, 4092, 0, 0, 168, 0, 3, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 3079, 0, 0, 0, 1, 0, 1, 4091, 5, 4, 2668, 0, 2, 9] payload_rows_per_dst=[8192, 8192, 8164, 8084, 8190, 8190, 6160, 5356] dup_rows_per_dst=[8, 1368, 2184, 1090, 336, 0, 0, 22] combine_rows_per_src=[8192, 8192, 8164, 8084, 8190, 8190, 6160, 5356] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=392462336 dispatch_meta_bytes=229376 combine_payload_bytes=392462336 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=392691712 ingress_bytes=502562816 link_bytes=502562816 token_bytes=785154048 weight_movement_bytes=110100480 total_nvlink_bytes=895254528 weight_to_token_ratio=0.140228 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.740393944e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58716 payload_rows_remote=50550 dup_rows=6820 dup_rows_remote=6752 dedup_ratio=0.104065 router_tokens_per_expert=[1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 486, 0, 526, 0, 0, 0, 0, 3955, 540, 0, 0, 0, 0, 494, 1, 0, 0, 0, 4079, 4056, 0, 0, 0, 4095, 0, 0, 215, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 2902, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 3225, 0, 0, 0] payload_rows_per_dst=[8166, 8190, 8120, 8060, 5878, 8192, 5804, 6306] dup_rows_per_dst=[68, 2024, 3484, 970, 274, 0, 0, 0] combine_rows_per_src=[8166, 8190, 8120, 8060, 5878, 8192, 5804, 6306] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61746 payload_rows_remote=53822 dup_rows=3790 dup_rows_remote=3364 dedup_ratio=0.057831 router_tokens_per_expert=[0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 0, 608, 0, 273, 0, 0, 0, 0, 4031, 762, 0, 0, 0, 0, 224, 0, 0, 0, 0, 4088, 3951, 0, 0, 0, 4096, 0, 0, 82, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 3479, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 2978, 0, 0, 0] payload_rows_per_dst=[8176, 8192, 8154, 7924, 8192, 8192, 6960, 5956] dup_rows_per_dst=[8, 1760, 1432, 426, 164, 0, 0, 0] combine_rows_per_src=[8176, 8192, 8154, 7924, 8192, 8192, 6960, 5956] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=303722496 dispatch_meta_bytes=201808 combine_payload_bytes=303722496 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=303924304 ingress_bytes=634023936 link_bytes=634023936 token_bytes=607646800 weight_movement_bytes=330301440 total_nvlink_bytes=937948240 weight_to_token_ratio=0.543575 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.029412 gemm_active_groups=10 gemm_peak_group_rows=25856 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.102671904e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63066 payload_rows_remote=54874 dup_rows=2470 dup_rows_remote=2470 dedup_ratio=0.037689 router_tokens_per_expert=[0, 0, 0, 0, 0, 9, 0, 0, 0, 2, 0, 171, 0, 314, 0, 1, 0, 0, 3821, 536, 0, 3, 0, 0, 358, 0, 0, 0, 0, 4088, 4074, 0, 1, 0, 4090, 0, 0, 67, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 3578, 3, 2, 2, 0, 0, 0, 4091, 13, 0, 3444, 0, 1, 0] payload_rows_per_dst=[8186, 8192, 8064, 8160, 8188, 8192, 7168, 6916] dup_rows_per_dst=[8, 966, 656, 704, 130, 0, 6, 0] combine_rows_per_src=[8186, 8192, 8064, 8160, 8188, 8192, 7168, 6916] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60402 payload_rows_remote=52210 dup_rows=5134 dup_rows_remote=4044 dedup_ratio=0.078339 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4, 1, 0, 4, 0, 306, 0, 238, 0, 0, 0, 1, 3925, 1213, 0, 0, 0, 0, 706, 0, 0, 0, 0, 4095, 4043, 0, 0, 0, 4093, 0, 0, 249, 0, 3, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 3248, 0, 0, 0, 1, 0, 0, 4093, 5, 3, 2432, 0, 1, 9] payload_rows_per_dst=[8192, 8192, 8144, 8112, 8190, 8190, 6498, 4884] dup_rows_per_dst=[8, 1090, 2134, 1386, 500, 0, 0, 16] combine_rows_per_src=[8192, 8192, 8144, 8112, 8190, 8190, 6498, 4884] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=384935936 dispatch_meta_bytes=227232 combine_payload_bytes=384935936 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=385163168 ingress_bytes=715237376 link_bytes=715237376 token_bytes=770099104 weight_movement_bytes=330301440 total_nvlink_bytes=1100400544 weight_to_token_ratio=0.428908 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=28032 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.243915433e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=275795968 dispatch_meta_bytes=186920 combine_payload_bytes=275795968 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=275982888 ingress_bytes=495996928 link_bytes=495996928 token_bytes=551778856 weight_movement_bytes=220200960 total_nvlink_bytes=771979816 weight_to_token_ratio=0.399075 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=24448 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.626202353e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=376707072 dispatch_meta_bytes=228488 combine_payload_bytes=376707072 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=376935560 ingress_bytes=596908032 link_bytes=596908032 token_bytes=753642632 weight_movement_bytes=220200960 total_nvlink_bytes=973843592 weight_to_token_ratio=0.292182 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=30976 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.038121242e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=350156800 dispatch_meta_bytes=219352 combine_payload_bytes=350156800 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=350376152 ingress_bytes=460257280 link_bytes=460257280 token_bytes=700532952 weight_movement_bytes=110100480 total_nvlink_bytes=810633432 weight_to_token_ratio=0.157167 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=25728 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.004631092e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=378800128 dispatch_meta_bytes=228512 combine_payload_bytes=378800128 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=379028640 ingress_bytes=599001088 link_bytes=599001088 token_bytes=757828768 weight_movement_bytes=220200960 total_nvlink_bytes=978029728 weight_to_token_ratio=0.290568 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=31488 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.041761411e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=387889152 dispatch_meta_bytes=227968 combine_payload_bytes=387889152 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=388117120 ingress_bytes=608090112 link_bytes=608090112 token_bytes=776006272 weight_movement_bytes=220200960 total_nvlink_bytes=996207232 weight_to_token_ratio=0.283762 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=27136 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057568718e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57688 payload_rows_remote=49522 dup_rows=7848 dup_rows_remote=7784 dedup_ratio=0.119751 router_tokens_per_expert=[1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 614, 0, 298, 0, 0, 0, 0, 3921, 558, 0, 0, 0, 0, 609, 0, 1, 0, 0, 4080, 4056, 0, 0, 0, 4095, 0, 0, 229, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 3022, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 3091, 0, 0, 0] payload_rows_per_dst=[8166, 8190, 8084, 8136, 4698, 8188, 6044, 6182] dup_rows_per_dst=[64, 1824, 878, 4874, 206, 0, 0, 2] combine_rows_per_src=[8166, 8190, 8084, 8136, 4698, 8188, 6044, 6182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61506 payload_rows_remote=53562 dup_rows=4030 dup_rows_remote=3506 dedup_ratio=0.061493 router_tokens_per_expert=[0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 716, 0, 168, 0, 0, 0, 0, 3983, 854, 0, 0, 0, 0, 270, 0, 0, 0, 0, 4090, 3964, 0, 0, 0, 4096, 0, 0, 99, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 3521, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 2812, 0, 0, 0] payload_rows_per_dst=[8180, 8192, 8138, 7944, 8192, 8192, 7044, 5624] dup_rows_per_dst=[6, 1766, 1536, 524, 198, 0, 0, 0] combine_rows_per_src=[8180, 8192, 8138, 7944, 8192, 8192, 7044, 5624] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=393336832 dispatch_meta_bytes=229376 combine_payload_bytes=393336832 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=393566208 ingress_bytes=503437312 link_bytes=503437312 token_bytes=786903040 weight_movement_bytes=110100480 total_nvlink_bytes=897003520 weight_to_token_ratio=0.139916 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.755602868e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=374241280 dispatch_meta_bytes=225016 combine_payload_bytes=374241280 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=374466296 ingress_bytes=484341760 link_bytes=484341760 token_bytes=748707576 weight_movement_bytes=110100480 total_nvlink_bytes=858808056 weight_to_token_ratio=0.147054 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=26624 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.423499812e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58374 payload_rows_remote=50182 dup_rows=7162 dup_rows_remote=6804 dedup_ratio=0.109283 router_tokens_per_expert=[4096, 0, 534, 0, 3796, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 317, 0, 0, 8, 0, 0, 0, 0, 158, 0, 0, 0, 4096, 4096, 0, 21, 0, 0, 0, 0, 0, 0, 4085, 0] payload_rows_per_dst=[1068, 8192, 8192, 8176, 8192, 8192, 8192, 8170] dup_rows_per_dst=[0, 0, 0, 6154, 0, 650, 358, 0] combine_rows_per_src=[1068, 8192, 8192, 8176, 8192, 8192, 8192, 8170] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=375144448 dispatch_meta_bytes=223904 combine_payload_bytes=375144448 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=375368352 ingress_bytes=485244928 link_bytes=485244928 token_bytes=750512800 weight_movement_bytes=110100480 total_nvlink_bytes=860613280 weight_to_token_ratio=0.146700 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=25856 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.439207389e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=362342400 dispatch_meta_bytes=229208 combine_payload_bytes=362342400 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=362571608 ingress_bytes=582543360 link_bytes=582543360 token_bytes=724914008 weight_movement_bytes=220200960 total_nvlink_bytes=945114968 weight_to_token_ratio=0.303761 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.029412 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.013138715e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=385796096 dispatch_meta_bytes=228744 combine_payload_bytes=385796096 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=386024840 ingress_bytes=826198016 link_bytes=826198016 token_bytes=771820936 weight_movement_bytes=440401920 total_nvlink_bytes=1212222856 weight_to_token_ratio=0.570601 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=24192 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.436894236e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=359704576 dispatch_meta_bytes=227944 combine_payload_bytes=359704576 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=359932520 ingress_bytes=910206976 link_bytes=910206976 token_bytes=719637096 weight_movement_bytes=550502400 total_nvlink_bytes=1270139496 weight_to_token_ratio=0.764972 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=29056 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.582999635e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60038 payload_rows_remote=51846 dup_rows=5498 dup_rows_remote=5308 dedup_ratio=0.083893 router_tokens_per_expert=[4095, 0, 1508, 1, 4035, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 3, 0, 0, 0, 0, 4075, 0, 0, 0, 1, 8, 0, 0, 0, 0, 1829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 1, 1, 0, 621, 0, 25, 0, 4094, 4087, 4, 114, 0, 1, 0, 13, 0, 0, 4060, 0] payload_rows_per_dst=[6720, 8192, 8152, 4268, 8190, 8192, 8192, 8132] dup_rows_per_dst=[2524, 8, 0, 1250, 0, 190, 1510, 16] combine_rows_per_src=[6720, 8192, 8152, 4268, 8190, 8192, 8192, 8132] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63036 payload_rows_remote=54846 dup_rows=2500 dup_rows_remote=2498 dedup_ratio=0.038147 router_tokens_per_expert=[4093, 0, 923, 0, 3798, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 1, 3, 0, 0, 0, 0, 3062, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 367, 0, 0, 0, 0, 0, 0, 0, 118, 1, 0, 2, 4095, 4096, 0, 7, 0, 0, 0, 10, 0, 1, 3997, 0] payload_rows_per_dst=[7954, 8188, 8192, 6130, 8190, 8190, 8192, 8000] dup_rows_per_dst=[1488, 0, 4, 0, 2, 734, 256, 16] combine_rows_per_src=[7954, 8188, 8192, 6130, 8190, 8190, 8192, 8000] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63830 payload_rows_remote=55638 dup_rows=1706 dup_rows_remote=1706 dedup_ratio=0.026031 router_tokens_per_expert=[4096, 0, 631, 0, 3807, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 0, 0, 0, 0, 0, 0, 0, 178, 0, 6, 0, 4096, 4096, 0, 42, 0, 0, 0, 0, 0, 0, 4086, 0] payload_rows_per_dst=[8012, 8192, 8192, 6686, 8192, 8192, 8192, 8172] dup_rows_per_dst=[864, 0, 0, 0, 0, 390, 452, 0] combine_rows_per_src=[8012, 8192, 8192, 6686, 8192, 8192, 8192, 8172] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56954 payload_rows_remote=48792 dup_rows=8582 dup_rows_remote=7504 dedup_ratio=0.130951 router_tokens_per_expert=[4095, 0, 1264, 0, 3944, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 1, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2671, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 133, 0, 1, 2, 1, 0, 0, 1, 162, 0, 0, 0, 4094, 4096, 0, 30, 0, 0, 0, 0, 0, 0, 4079, 1] payload_rows_per_dst=[2528, 8190, 8190, 8172, 8192, 5328, 8192, 8162] dup_rows_per_dst=[0, 0, 2, 5060, 2, 156, 2284, 1078] combine_rows_per_src=[2528, 8190, 8190, 8172, 8192, 5328, 8192, 8162] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=354973696 dispatch_meta_bytes=229224 combine_payload_bytes=354973696 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=355202920 ingress_bytes=575174656 link_bytes=575174656 token_bytes=710176616 weight_movement_bytes=220200960 total_nvlink_bytes=930377576 weight_to_token_ratio=0.310065 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.029412 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.000323327e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=383932416 dispatch_meta_bytes=228272 combine_payload_bytes=383932416 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=384160688 ingress_bytes=824334336 link_bytes=824334336 token_bytes=768093104 weight_movement_bytes=440401920 total_nvlink_bytes=1208495024 weight_to_token_ratio=0.573370 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=25344 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.433652990e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64130 payload_rows_remote=55940 dup_rows=1406 dup_rows_remote=1406 dedup_ratio=0.021454 router_tokens_per_expert=[4096, 0, 542, 0, 3812, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 0, 0, 0, 0, 0, 0, 187, 0, 4, 0, 4096, 4096, 0, 47, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8000, 8192, 8190, 6980, 8192, 8192, 8192, 8192] dup_rows_per_dst=[708, 0, 0, 0, 0, 222, 476, 0] combine_rows_per_src=[8000, 8192, 8190, 6980, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63684 payload_rows_remote=55498 dup_rows=1852 dup_rows_remote=1850 dedup_ratio=0.028259 router_tokens_per_expert=[4091, 0, 695, 0, 3849, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 1, 3, 0, 0, 0, 0, 3338, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 260, 0, 0, 0, 0, 0, 0, 0, 100, 1, 0, 2, 4095, 4096, 0, 8, 0, 0, 0, 9, 0, 1, 4025, 0] payload_rows_per_dst=[8000, 8188, 8192, 6682, 8186, 8190, 8192, 8054] dup_rows_per_dst=[1088, 0, 4, 0, 2, 520, 222, 16] combine_rows_per_src=[8000, 8188, 8192, 6682, 8186, 8190, 8192, 8054] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57262 payload_rows_remote=49086 dup_rows=8274 dup_rows_remote=7448 dedup_ratio=0.126251 router_tokens_per_expert=[4095, 0, 1046, 0, 3949, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 1, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2929, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 97, 0, 1, 1, 1, 0, 0, 1, 144, 0, 0, 0, 4094, 4095, 0, 33, 0, 0, 0, 0, 0, 0, 4086, 1] payload_rows_per_dst=[2092, 8190, 8190, 8182, 8192, 6050, 8190, 8176] dup_rows_per_dst=[0, 0, 2, 5574, 6, 128, 1738, 826] combine_rows_per_src=[2092, 8190, 8190, 8182, 8192, 6050, 8190, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59574 payload_rows_remote=51382 dup_rows=5962 dup_rows_remote=5582 dedup_ratio=0.090973 router_tokens_per_expert=[4096, 0, 447, 0, 3821, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 0, 0, 7, 0, 0, 0, 0, 162, 0, 0, 0, 4096, 4096, 0, 28, 0, 0, 0, 0, 0, 0, 4089, 0] payload_rows_per_dst=[2408, 8192, 8192, 8028, 8192, 8192, 8192, 8178] dup_rows_per_dst=[40, 0, 0, 5066, 0, 476, 380, 0] combine_rows_per_src=[2408, 8192, 8192, 8028, 8192, 8192, 8192, 8178] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=371632128 dispatch_meta_bytes=228616 combine_payload_bytes=371632128 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=371860744 ingress_bytes=481732608 link_bytes=481732608 token_bytes=743492872 weight_movement_bytes=110100480 total_nvlink_bytes=853593352 weight_to_token_ratio=0.148085 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=31360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.378122367e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63548 payload_rows_remote=55362 dup_rows=1988 dup_rows_remote=1986 dedup_ratio=0.030334 router_tokens_per_expert=[4096, 0, 899, 0, 3686, 4, 0, 3, 0, 0, 0, 2, 4091, 1, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 1, 1, 0, 0, 3365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 0, 4, 0, 0, 0, 0, 196, 0, 0, 0, 4094, 4087, 0, 53, 0, 0, 0, 7, 0, 0, 3953, 0] payload_rows_per_dst=[7978, 8186, 8162, 6730, 8192, 8190, 8190, 7920] dup_rows_per_dst=[1206, 2, 0, 4, 0, 294, 482, 0] combine_rows_per_src=[7978, 8186, 8162, 6730, 8192, 8190, 8190, 7920] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57938 payload_rows_remote=49750 dup_rows=7598 dup_rows_remote=388 dedup_ratio=0.115936 router_tokens_per_expert=[4096, 0, 608, 0, 3933, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 0, 1, 0, 0, 0, 0, 112, 0, 0, 0, 4096, 4095, 0, 12, 0, 1, 0, 0, 0, 0, 4069, 0] payload_rows_per_dst=[8188, 8190, 8190, 6838, 8192, 8192, 2010, 8138] dup_rows_per_dst=[7210, 4, 68, 0, 0, 274, 40, 2] combine_rows_per_src=[8188, 8190, 8190, 6838, 8192, 8192, 2010, 8138] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62754 payload_rows_remote=56686 dup_rows=2782 dup_rows_remote=2782 dedup_ratio=0.042450 router_tokens_per_expert=[4096, 0, 1122, 0, 3776, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 251, 0, 0, 0, 4092, 4096, 0, 150, 0, 0, 0, 0, 0, 0, 3865, 0] payload_rows_per_dst=[8002, 8192, 8192, 6068, 8192, 8186, 8192, 7730] dup_rows_per_dst=[1794, 0, 0, 0, 0, 186, 802, 0] combine_rows_per_src=[8002, 8192, 8192, 6068, 8192, 8186, 8192, 7730] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=393136128 dispatch_meta_bytes=229376 combine_payload_bytes=393136128 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=393365504 ingress_bytes=503236608 link_bytes=503236608 token_bytes=786501632 weight_movement_bytes=110100480 total_nvlink_bytes=896602112 weight_to_token_ratio=0.139988 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.752112295e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=398813184 dispatch_meta_bytes=229376 combine_payload_bytes=398813184 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=399042560 ingress_bytes=508913664 link_bytes=508913664 token_bytes=797855744 weight_movement_bytes=110100480 total_nvlink_bytes=907956224 weight_to_token_ratio=0.137995 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.850845636e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63898 payload_rows_remote=55712 dup_rows=1638 dup_rows_remote=1638 dedup_ratio=0.024994 router_tokens_per_expert=[4096, 0, 703, 0, 3688, 4, 0, 3, 0, 0, 0, 2, 4091, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 2, 1, 0, 0, 3533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 1, 0, 2, 0, 0, 0, 0, 199, 0, 0, 0, 4094, 4088, 0, 58, 0, 0, 0, 7, 0, 0, 3990, 0] payload_rows_per_dst=[7916, 8186, 8162, 7066, 8192, 8192, 8190, 7994] dup_rows_per_dst=[880, 0, 0, 6, 0, 252, 500, 0] combine_rows_per_src=[7916, 8186, 8162, 7066, 8192, 8192, 8190, 7994] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61550 payload_rows_remote=53358 dup_rows=3986 dup_rows_remote=3814 dedup_ratio=0.060822 router_tokens_per_expert=[4095, 0, 1236, 1, 4019, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 3, 0, 0, 0, 0, 4075, 0, 0, 0, 1, 8, 0, 0, 0, 0, 2184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 1, 0, 1, 1, 0, 558, 0, 25, 0, 4094, 4089, 4, 118, 0, 1, 0, 12, 0, 0, 4060, 0] payload_rows_per_dst=[8116, 8192, 8152, 4384, 8190, 8192, 8192, 8132] dup_rows_per_dst=[2396, 8, 0, 0, 0, 172, 1396, 14] combine_rows_per_src=[8116, 8192, 8152, 4384, 8190, 8192, 8192, 8132] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=400977920 dispatch_meta_bytes=229384 combine_payload_bytes=400977920 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=401207304 ingress_bytes=511078400 link_bytes=511078400 token_bytes=802185224 weight_movement_bytes=110100480 total_nvlink_bytes=912285704 weight_to_token_ratio=0.137251 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.888493956e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=397809664 dispatch_meta_bytes=229392 combine_payload_bytes=397809664 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=398039056 ingress_bytes=618010624 link_bytes=618010624 token_bytes=795848720 weight_movement_bytes=220200960 total_nvlink_bytes=1016049680 weight_to_token_ratio=0.276687 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.074822120e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=351848448 dispatch_meta_bytes=226136 combine_payload_bytes=351848448 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=352074584 ingress_bytes=461948928 link_bytes=461948928 token_bytes=703923032 weight_movement_bytes=110100480 total_nvlink_bytes=814023512 weight_to_token_ratio=0.156410 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.034051633e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=368306176 dispatch_meta_bytes=227856 combine_payload_bytes=368306176 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=368534032 ingress_bytes=918808576 link_bytes=918808576 token_bytes=736840208 weight_movement_bytes=550502400 total_nvlink_bytes=1287342608 weight_to_token_ratio=0.747112 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=29568 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.597959232e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=349741056 dispatch_meta_bytes=225184 combine_payload_bytes=349741056 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=349966240 ingress_bytes=459841536 link_bytes=459841536 token_bytes=699707296 weight_movement_bytes=110100480 total_nvlink_bytes=809807776 weight_to_token_ratio=0.157352 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.997400620e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58902 payload_rows_remote=50722 dup_rows=6634 dup_rows_remote=342 dedup_ratio=0.101227 router_tokens_per_expert=[4096, 0, 465, 0, 3931, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 0, 1, 0, 0, 0, 0, 100, 0, 0, 0, 4096, 4095, 0, 13, 0, 1, 0, 0, 0, 0, 4077, 0] payload_rows_per_dst=[8180, 8190, 8190, 7188, 8192, 8192, 2616, 8154] dup_rows_per_dst=[6292, 6, 70, 0, 0, 220, 44, 2] combine_rows_per_src=[8180, 8190, 8190, 7188, 8192, 8192, 2616, 8154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63084 payload_rows_remote=56704 dup_rows=2452 dup_rows_remote=2452 dedup_ratio=0.037415 router_tokens_per_expert=[4096, 0, 944, 0, 3766, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 248, 0, 0, 0, 4087, 4096, 0, 164, 0, 0, 0, 0, 0, 0, 3899, 0] payload_rows_per_dst=[7962, 8192, 8192, 6380, 8192, 8176, 8192, 7798] dup_rows_per_dst=[1458, 0, 0, 0, 0, 170, 824, 0] combine_rows_per_src=[7962, 8192, 8192, 6380, 8192, 8176, 8192, 7798] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=399343616 dispatch_meta_bytes=229400 combine_payload_bytes=399343616 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=399573016 ingress_bytes=509444096 link_bytes=509444096 token_bytes=798916632 weight_movement_bytes=110100480 total_nvlink_bytes=909017112 weight_to_token_ratio=0.137812 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.860070721e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=382470144 dispatch_meta_bytes=228688 combine_payload_bytes=382470144 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=382698832 ingress_bytes=492570624 link_bytes=492570624 token_bytes=765168976 weight_movement_bytes=110100480 total_nvlink_bytes=875269456 weight_to_token_ratio=0.143890 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.029412 gemm_active_groups=10 gemm_peak_group_rows=31744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.566613291e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62958 payload_rows_remote=54766 dup_rows=2578 dup_rows_remote=1796 dedup_ratio=0.039337 router_tokens_per_expert=[0, 4096, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3697, 0, 0, 0, 0, 4096, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 444, 0, 2484, 371, 0, 391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 780] payload_rows_per_dst=[8184, 7394, 8192, 8192, 8192, 6420, 8192, 8192] dup_rows_per_dst=[0, 0, 58, 0, 0, 178, 782, 1560] combine_rows_per_src=[8184, 7394, 8192, 8192, 8192, 6420, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=396834816 dispatch_meta_bytes=229392 combine_payload_bytes=396834816 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=397064208 ingress_bytes=506935296 link_bytes=506935296 token_bytes=793899024 weight_movement_bytes=110100480 total_nvlink_bytes=903999504 weight_to_token_ratio=0.138683 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.816438563e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=356608000 dispatch_meta_bytes=200552 combine_payload_bytes=356608000 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=356808552 ingress_bytes=686909440 link_bytes=686909440 token_bytes=713416552 weight_movement_bytes=330301440 total_nvlink_bytes=1043717992 weight_to_token_ratio=0.462985 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=22144 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.194648493e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=406325248 dispatch_meta_bytes=237872 combine_payload_bytes=406325248 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=406563120 ingress_bytes=516425728 link_bytes=516425728 token_bytes=812888368 weight_movement_bytes=110100480 total_nvlink_bytes=922988848 weight_to_token_ratio=0.135444 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=24192 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.981492784e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=392562688 dispatch_meta_bytes=226248 combine_payload_bytes=392562688 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=392788936 ingress_bytes=502663168 link_bytes=502663168 token_bytes=785351624 weight_movement_bytes=110100480 total_nvlink_bytes=895452104 weight_to_token_ratio=0.140193 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=29568 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.742139230e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62808 payload_rows_remote=55850 dup_rows=2728 dup_rows_remote=2362 dedup_ratio=0.041626 router_tokens_per_expert=[0, 4096, 4094, 0, 0, 0, 0, 3, 4093, 0, 8, 0, 0, 3342, 4, 2, 0, 0, 4096, 0, 0, 1, 13, 0, 0, 1, 0, 0, 5, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 418, 0, 2975, 269, 0, 1016, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 1, 139] payload_rows_per_dst=[8192, 6704, 8192, 8192, 8190, 6958, 8188, 8192] dup_rows_per_dst=[2, 8, 28, 12, 0, 366, 2032, 280] combine_rows_per_src=[8192, 6704, 8192, 8192, 8190, 6958, 8188, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63114 payload_rows_remote=54922 dup_rows=2422 dup_rows_remote=2420 dedup_ratio=0.036957 router_tokens_per_expert=[1, 4095, 4092, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 3418, 2, 0, 0, 0, 4093, 0, 0, 1, 73, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 1, 0, 0, 0, 0, 336, 0, 3053, 324, 0, 426, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 1, 0, 562] payload_rows_per_dst=[8186, 6836, 8186, 8190, 8192, 7142, 8190, 8192] dup_rows_per_dst=[2, 4, 148, 4, 2, 284, 852, 1126] combine_rows_per_src=[8186, 6836, 8186, 8190, 8192, 7142, 8190, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63482 payload_rows_remote=55290 dup_rows=2054 dup_rows_remote=1960 dedup_ratio=0.031342 router_tokens_per_expert=[0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3423, 0, 0, 0, 0, 4096, 0, 0, 14, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 358, 0, 3337, 288, 0, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 334] payload_rows_per_dst=[8192, 6846, 8192, 8192, 8192, 7484, 8192, 8192] dup_rows_per_dst=[0, 0, 94, 0, 0, 482, 810, 668] combine_rows_per_src=[8192, 6846, 8192, 8192, 8192, 7484, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62500 payload_rows_remote=54310 dup_rows=3036 dup_rows_remote=870 dedup_ratio=0.046326 router_tokens_per_expert=[0, 4095, 4095, 0, 0, 0, 0, 0, 4094, 0, 2, 0, 0, 3316, 0, 0, 0, 1, 4096, 0, 1, 0, 8, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4, 0, 0, 0, 1, 290, 0, 3108, 271, 0, 304, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 889] payload_rows_per_dst=[8190, 6636, 7804, 8190, 8192, 7108, 8190, 8190] dup_rows_per_dst=[0, 0, 20, 2, 8, 232, 608, 2166] combine_rows_per_src=[8190, 6636, 7804, 8190, 8192, 7108, 8190, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=363575296 dispatch_meta_bytes=204256 combine_payload_bytes=363575296 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=363779552 ingress_bytes=803977216 link_bytes=803977216 token_bytes=727354848 weight_movement_bytes=440401920 total_nvlink_bytes=1167756768 weight_to_token_ratio=0.605484 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=23936 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.398248610e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=406454272 dispatch_meta_bytes=236624 combine_payload_bytes=406454272 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=406690896 ingress_bytes=516554752 link_bytes=516554752 token_bytes=813145168 weight_movement_bytes=110100480 total_nvlink_bytes=923245648 weight_to_token_ratio=0.135401 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=25856 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.983736724e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63688 payload_rows_remote=55496 dup_rows=1848 dup_rows_remote=1766 dedup_ratio=0.028198 router_tokens_per_expert=[0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3383, 0, 0, 0, 0, 4096, 0, 0, 14, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 324, 0, 3599, 192, 0, 347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 306] payload_rows_per_dst=[8192, 6766, 8192, 8192, 8192, 7770, 8192, 8192] dup_rows_per_dst=[0, 0, 82, 0, 0, 460, 694, 612] combine_rows_per_src=[8192, 6766, 8192, 8192, 8192, 7770, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63378 payload_rows_remote=55186 dup_rows=2158 dup_rows_remote=2156 dedup_ratio=0.032928 router_tokens_per_expert=[1, 4095, 4092, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 3338, 2, 1, 0, 0, 4092, 0, 0, 1, 58, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 4095, 1, 0, 0, 0, 1, 0, 0, 0, 0, 299, 0, 3434, 218, 0, 272, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 0, 1, 0, 573] payload_rows_per_dst=[8186, 6676, 8184, 8190, 8192, 7570, 8190, 8190] dup_rows_per_dst=[2, 6, 118, 6, 2, 332, 544, 1148] combine_rows_per_src=[8186, 6676, 8184, 8190, 8192, 7570, 8190, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63048 payload_rows_remote=54858 dup_rows=2488 dup_rows_remote=546 dedup_ratio=0.037964 router_tokens_per_expert=[0, 4094, 4095, 0, 0, 0, 0, 0, 4094, 0, 2, 0, 0, 3299, 0, 0, 0, 1, 4096, 0, 1, 0, 4, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4, 0, 0, 0, 1, 252, 0, 3471, 156, 0, 173, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 832] payload_rows_per_dst=[8190, 6602, 7912, 8190, 8192, 7582, 8190, 8190] dup_rows_per_dst=[0, 0, 14, 0, 8, 178, 346, 1942] combine_rows_per_src=[8190, 6602, 7912, 8190, 8192, 7582, 8190, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63532 payload_rows_remote=55340 dup_rows=2004 dup_rows_remote=1562 dedup_ratio=0.030579 router_tokens_per_expert=[0, 4096, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3627, 0, 0, 0, 0, 4096, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 350, 0, 3143, 165, 0, 221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 678] payload_rows_per_dst=[8186, 7254, 8192, 8192, 8192, 7132, 8192, 8192] dup_rows_per_dst=[0, 0, 22, 0, 0, 184, 442, 1356] combine_rows_per_src=[8186, 7254, 8192, 8192, 8192, 7132, 8192, 8192] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=400332800 dispatch_meta_bytes=232848 combine_payload_bytes=400332800 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=400565648 ingress_bytes=510433280 link_bytes=510433280 token_bytes=800898448 weight_movement_bytes=110100480 total_nvlink_bytes=910998928 weight_to_token_ratio=0.137471 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=25344 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.877274258e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=393680896 dispatch_meta_bytes=229368 combine_payload_bytes=393680896 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=393910264 ingress_bytes=613881856 link_bytes=613881856 token_bytes=787591160 weight_movement_bytes=220200960 total_nvlink_bytes=1007792120 weight_to_token_ratio=0.279588 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.067641514e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64156 payload_rows_remote=56654 dup_rows=1380 dup_rows_remote=1380 dedup_ratio=0.021057 router_tokens_per_expert=[0, 4096, 4085, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 3746, 5, 0, 0, 0, 4094, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 0, 0, 0, 332, 0, 3339, 157, 0, 219, 0, 0, 0, 0, 1, 2, 0, 0, 0, 2, 4094, 0, 0, 0, 394] payload_rows_per_dst=[8172, 7502, 8188, 8192, 8192, 7528, 8192, 8190] dup_rows_per_dst=[0, 0, 6, 2, 12, 128, 442, 790] combine_rows_per_src=[8172, 7502, 8188, 8192, 8192, 7528, 8192, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64370 payload_rows_remote=56178 dup_rows=1166 dup_rows_remote=1166 dedup_ratio=0.017792 router_tokens_per_expert=[0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 3778, 0, 0, 0, 0, 4096, 0, 0, 11, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 261, 0, 3400, 271, 0, 386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 74] payload_rows_per_dst=[8192, 7576, 8192, 8192, 8192, 7642, 8192, 8192] dup_rows_per_dst=[0, 0, 24, 0, 0, 222, 772, 148] combine_rows_per_src=[8192, 7576, 8192, 8192, 8192, 7642, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52072 payload_rows_remote=43880 dup_rows=13464 dup_rows_remote=13432 dedup_ratio=0.205444 router_tokens_per_expert=[0, 4095, 4094, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 3667, 1, 0, 0, 0, 4096, 1, 0, 2, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 367, 0, 3645, 160, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 287] payload_rows_per_dst=[8192, 7964, 8192, 8176, 8172, 8144, 1866, 1366] dup_rows_per_dst=[32, 8066, 20, 0, 0, 5218, 36, 92] combine_rows_per_src=[8192, 7964, 8192, 8176, 8172, 8144, 1866, 1366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=396318720 dispatch_meta_bytes=229000 combine_payload_bytes=396318720 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=396547720 ingress_bytes=616519680 link_bytes=616519680 token_bytes=792866440 weight_movement_bytes=220200960 total_nvlink_bytes=1013067400 weight_to_token_ratio=0.277728 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.072229124e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=389294080 dispatch_meta_bytes=220720 combine_payload_bytes=389294080 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=389514800 ingress_bytes=719595520 link_bytes=719595520 token_bytes=778808880 weight_movement_bytes=330301440 total_nvlink_bytes=1109110320 weight_to_token_ratio=0.424111 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=29184 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.251494962e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64396 payload_rows_remote=56968 dup_rows=1140 dup_rows_remote=1140 dedup_ratio=0.017395 router_tokens_per_expert=[0, 4096, 4085, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 3710, 4, 0, 0, 0, 4094, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 0, 0, 0, 247, 0, 3613, 131, 0, 143, 0, 0, 1, 0, 1, 2, 0, 0, 0, 2, 4094, 0, 0, 0, 344] payload_rows_per_dst=[8172, 7428, 8188, 8192, 8192, 7842, 8192, 8190] dup_rows_per_dst=[0, 0, 2, 4, 12, 140, 292, 690] combine_rows_per_src=[8172, 7428, 8188, 8192, 8192, 7842, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=397795328 dispatch_meta_bytes=229048 combine_payload_bytes=397795328 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=398024376 ingress_bytes=617996288 link_bytes=617996288 token_bytes=795819704 weight_movement_bytes=220200960 total_nvlink_bytes=1016020664 weight_to_token_ratio=0.276697 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.074797188e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=395573248 dispatch_meta_bytes=229368 combine_payload_bytes=395573248 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=395802616 ingress_bytes=615774208 link_bytes=615774208 token_bytes=791375864 weight_movement_bytes=220200960 total_nvlink_bytes=1011576824 weight_to_token_ratio=0.278251 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.070932625e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63408 payload_rows_remote=55730 dup_rows=2128 dup_rows_remote=1662 dedup_ratio=0.032471 router_tokens_per_expert=[0, 4095, 4094, 0, 0, 0, 0, 2, 4092, 0, 7, 0, 0, 3282, 3, 2, 0, 0, 4096, 0, 0, 1, 9, 0, 0, 1, 0, 0, 4, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 0, 371, 0, 3511, 190, 0, 705, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 1, 109] payload_rows_per_dst=[8192, 6584, 8192, 8192, 8192, 7678, 8186, 8192] dup_rows_per_dst=[0, 4, 20, 8, 0, 466, 1410, 220] combine_rows_per_src=[8192, 6584, 8192, 8192, 8192, 7678, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54388 payload_rows_remote=46196 dup_rows=11148 dup_rows_remote=11116 dedup_ratio=0.170105 router_tokens_per_expert=[0, 4095, 4094, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 3581, 1, 0, 0, 0, 4096, 1, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 338, 0, 3818, 134, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 277] payload_rows_per_dst=[8192, 7682, 8192, 8178, 8170, 8114, 3998, 1862] dup_rows_per_dst=[32, 8956, 14, 0, 0, 1908, 58, 180] combine_rows_per_src=[8192, 7682, 8192, 8178, 8170, 8114, 3998, 1862] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64678 payload_rows_remote=56486 dup_rows=858 dup_rows_remote=858 dedup_ratio=0.013092 router_tokens_per_expert=[0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 7, 0, 0, 3782, 0, 0, 0, 0, 4096, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 229, 0, 3666, 195, 0, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 94] payload_rows_per_dst=[8192, 7578, 8192, 8192, 8192, 7948, 8192, 8192] dup_rows_per_dst=[0, 0, 20, 0, 0, 232, 418, 188] combine_rows_per_src=[8192, 7578, 8192, 8192, 8192, 7948, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=396677120 dispatch_meta_bytes=227608 combine_payload_bytes=396677120 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=396904728 ingress_bytes=506777600 link_bytes=506777600 token_bytes=793581848 weight_movement_bytes=110100480 total_nvlink_bytes=903682328 weight_to_token_ratio=0.138739 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=30720 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.813695970e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=393222144 dispatch_meta_bytes=221616 combine_payload_bytes=393222144 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=393443760 ingress_bytes=723523584 link_bytes=723523584 token_bytes=786665904 weight_movement_bytes=330301440 total_nvlink_bytes=1116967344 weight_to_token_ratio=0.419875 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=29440 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.258326512e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57986 payload_rows_remote=50502 dup_rows=7550 dup_rows_remote=3984 dedup_ratio=0.115204 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1263, 0, 0, 0, 0, 3918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 259, 2674, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 3553, 0] payload_rows_per_dst=[8192, 6056, 8192, 4266, 8192, 8192, 7484, 7412] dup_rows_per_dst=[0, 706, 0, 912, 0, 4, 3566, 2362] combine_rows_per_src=[8192, 6056, 8192, 4266, 8192, 8192, 7484, 7412] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=406095872 dispatch_meta_bytes=232136 combine_payload_bytes=406095872 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=406328008 ingress_bytes=626296832 link_bytes=626296832 token_bytes=812423880 weight_movement_bytes=220200960 total_nvlink_bytes=1032624840 weight_to_token_ratio=0.271042 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=28416 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.089233199e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=402683904 dispatch_meta_bytes=229376 combine_payload_bytes=402683904 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=402913280 ingress_bytes=512784384 link_bytes=512784384 token_bytes=805597184 weight_movement_bytes=110100480 total_nvlink_bytes=915697664 weight_to_token_ratio=0.136669 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.918163823e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=314531840 dispatch_meta_bytes=229248 combine_payload_bytes=314531840 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=314761088 ingress_bytes=534732800 link_bytes=534732800 token_bytes=629292928 weight_movement_bytes=220200960 total_nvlink_bytes=849493888 weight_to_token_ratio=0.349918 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.299882876e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60542 payload_rows_remote=52350 dup_rows=4994 dup_rows_remote=4920 dedup_ratio=0.076202 router_tokens_per_expert=[1, 4095, 1, 0, 0, 0, 0, 0, 1296, 0, 0, 0, 0, 0, 1, 0, 0, 2, 1, 0, 4093, 0, 0, 0, 0, 1135, 0, 0, 0, 0, 3824, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 4, 9, 0, 13, 0, 13, 0, 0, 0, 3, 505, 1914, 0, 12, 0, 4094, 4083, 4092, 0, 1, 0, 3574, 0] payload_rows_per_dst=[8192, 8178, 8192, 7798, 8186, 8192, 4654, 7150] dup_rows_per_dst=[2, 2582, 0, 2120, 2, 74, 214, 0] combine_rows_per_src=[8192, 8178, 8192, 7798, 8186, 8192, 4654, 7150] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62410 payload_rows_remote=54218 dup_rows=3126 dup_rows_remote=3124 dedup_ratio=0.047699 router_tokens_per_expert=[0, 4093, 0, 0, 0, 0, 0, 0, 706, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 812, 0, 0, 0, 0, 3988, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 2, 0, 2, 0, 14, 0, 0, 0, 13, 317, 2610, 0, 1, 0, 4088, 4094, 4096, 0, 1, 0, 3830, 0] payload_rows_per_dst=[8186, 8190, 8192, 8024, 8192, 8192, 5772, 7662] dup_rows_per_dst=[0, 1414, 0, 1576, 2, 24, 110, 0] combine_rows_per_src=[8186, 8190, 8192, 8024, 8192, 8192, 5772, 7662] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=361998336 dispatch_meta_bytes=217944 combine_payload_bytes=361998336 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=362216280 ingress_bytes=472098816 link_bytes=472098816 token_bytes=724214616 weight_movement_bytes=110100480 total_nvlink_bytes=834315096 weight_to_token_ratio=0.152027 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=19456 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.210574879e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60364 payload_rows_remote=52172 dup_rows=5172 dup_rows_remote=5172 dedup_ratio=0.078918 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1669, 0, 0, 0, 0, 4014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 519, 2993, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 2400, 0] payload_rows_per_dst=[8192, 8192, 8192, 8054, 8192, 8192, 6550, 4800] dup_rows_per_dst=[0, 1360, 0, 3312, 0, 26, 474, 0] combine_rows_per_src=[8192, 8192, 8192, 8054, 8192, 8192, 6550, 4800] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52358 payload_rows_remote=44166 dup_rows=13178 dup_rows_remote=5206 dedup_ratio=0.201080 router_tokens_per_expert=[0, 4096, 0, 0, 0, 1, 0, 0, 667, 0, 0, 0, 1, 1, 0, 2, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1135, 0, 0, 0, 0, 3953, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5, 1, 0, 0, 1, 0, 2, 680, 2185, 0, 0, 0, 4094, 4095, 4095, 0, 0, 0, 3658, 0] payload_rows_per_dst=[8188, 1338, 8190, 2272, 8180, 7974, 8024, 8192] dup_rows_per_dst=[2, 4, 2, 0, 0, 8, 5190, 7972] combine_rows_per_src=[8188, 1338, 8190, 2272, 8180, 7974, 8024, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=408346624 dispatch_meta_bytes=232432 combine_payload_bytes=408346624 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=408579056 ingress_bytes=628547584 link_bytes=628547584 token_bytes=816925680 weight_movement_bytes=220200960 total_nvlink_bytes=1037126640 weight_to_token_ratio=0.269548 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=28032 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.093147627e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=399472640 dispatch_meta_bytes=229568 combine_payload_bytes=399472640 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=399702208 ingress_bytes=509573120 link_bytes=509573120 token_bytes=799174848 weight_movement_bytes=110100480 total_nvlink_bytes=909275328 weight_to_token_ratio=0.137768 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=28288 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.862314661e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=331132928 dispatch_meta_bytes=229248 combine_payload_bytes=331132928 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=331362176 ingress_bytes=551333888 link_bytes=551333888 token_bytes=662495104 weight_movement_bytes=220200960 total_nvlink_bytes=882696064 weight_to_token_ratio=0.332381 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.588603101e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=404891648 dispatch_meta_bytes=229376 combine_payload_bytes=404891648 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=405121024 ingress_bytes=514992128 link_bytes=514992128 token_bytes=810012672 weight_movement_bytes=110100480 total_nvlink_bytes=920113152 weight_to_token_ratio=0.135924 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.956560122e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62834 payload_rows_remote=54642 dup_rows=2702 dup_rows_remote=2700 dedup_ratio=0.041229 router_tokens_per_expert=[0, 4092, 0, 0, 0, 0, 0, 0, 411, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 1, 0, 0, 1, 877, 0, 0, 0, 0, 4015, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 2, 2, 0, 2, 0, 12, 0, 0, 0, 12, 306, 2875, 0, 2, 0, 4088, 4094, 4095, 0, 1, 0, 3782, 0] payload_rows_per_dst=[8184, 8188, 8192, 8068, 8192, 8192, 6252, 7566] dup_rows_per_dst=[0, 824, 0, 1718, 2, 20, 138, 0] combine_rows_per_src=[8184, 8188, 8192, 8068, 8192, 8192, 6252, 7566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60196 payload_rows_remote=52004 dup_rows=5340 dup_rows_remote=5340 dedup_ratio=0.081482 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2021, 0, 0, 0, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 548, 3138, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 2175, 0] payload_rows_per_dst=[8192, 8192, 8192, 8066, 8192, 8192, 6820, 4350] dup_rows_per_dst=[0, 754, 0, 4020, 0, 14, 552, 0] combine_rows_per_src=[8192, 8192, 8192, 8066, 8192, 8192, 6820, 4350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58998 payload_rows_remote=50810 dup_rows=6538 dup_rows_remote=658 dedup_ratio=0.099762 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1468, 0, 0, 0, 0, 3951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 227, 2856, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 3457, 0] payload_rows_per_dst=[8192, 8192, 8192, 2936, 8192, 8192, 8188, 6914] dup_rows_per_dst=[0, 656, 0, 0, 0, 2, 5880, 0] combine_rows_per_src=[8192, 8192, 8192, 2936, 8192, 8192, 8188, 6914] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=375244800 dispatch_meta_bytes=229080 combine_payload_bytes=375244800 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=375473880 ingress_bytes=485345280 link_bytes=485345280 token_bytes=750718680 weight_movement_bytes=110100480 total_nvlink_bytes=860819160 weight_to_token_ratio=0.146660 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.440952675e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52302 payload_rows_remote=44242 dup_rows=13234 dup_rows_remote=1244 dedup_ratio=0.201935 router_tokens_per_expert=[0, 4096, 0, 0, 0, 1, 0, 0, 316, 0, 0, 0, 1, 1, 0, 2, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1465, 0, 0, 0, 0, 3975, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5, 1, 0, 0, 1, 0, 0, 615, 2279, 0, 0, 0, 4094, 4094, 4095, 0, 0, 0, 3631, 0] payload_rows_per_dst=[8184, 2690, 8190, 2932, 8178, 8046, 6022, 8060] dup_rows_per_dst=[2, 140, 4, 2, 0, 8, 1088, 11990] combine_rows_per_src=[8184, 2690, 8190, 2932, 8178, 8046, 6022, 8060] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=391673856 dispatch_meta_bytes=229368 combine_payload_bytes=391673856 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=391903224 ingress_bytes=501774336 link_bytes=501774336 token_bytes=783577080 weight_movement_bytes=110100480 total_nvlink_bytes=893677560 weight_to_token_ratio=0.140510 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.726680980e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=372764672 dispatch_meta_bytes=229376 combine_payload_bytes=372764672 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=372994048 ingress_bytes=482865152 link_bytes=482865152 token_bytes=745758720 weight_movement_bytes=110100480 total_nvlink_bytes=855859200 weight_to_token_ratio=0.147636 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.397819170e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61564 payload_rows_remote=53374 dup_rows=3972 dup_rows_remote=3552 dedup_ratio=0.060608 router_tokens_per_expert=[0, 4095, 0, 0, 0, 0, 6, 0, 207, 0, 0, 0, 0, 0, 13, 0, 0, 1, 0, 0, 4093, 0, 0, 0, 0, 1425, 1, 0, 0, 0, 3820, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 2, 0, 6, 0, 2, 0, 0, 0, 0, 1591, 1917, 1, 0, 0, 4089, 4085, 4096, 0, 0, 0, 3311, 0] payload_rows_per_dst=[8192, 8190, 8186, 7742, 8192, 8190, 6250, 6622] dup_rows_per_dst=[10, 420, 2, 2750, 14, 8, 768, 0] combine_rows_per_src=[8192, 8190, 8186, 7742, 8192, 8190, 6250, 6622] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=388634624 dispatch_meta_bytes=229368 combine_payload_bytes=388634624 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=388863992 ingress_bytes=498735104 link_bytes=498735104 token_bytes=777498616 weight_movement_bytes=110100480 total_nvlink_bytes=887599096 weight_to_token_ratio=0.141609 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.673823737e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=373968896 dispatch_meta_bytes=229376 combine_payload_bytes=373968896 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=374198272 ingress_bytes=484069376 link_bytes=484069376 token_bytes=748167168 weight_movement_bytes=110100480 total_nvlink_bytes=858267648 weight_to_token_ratio=0.147160 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.418762606e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=316581888 dispatch_meta_bytes=197488 combine_payload_bytes=316581888 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=316779376 ingress_bytes=756983808 link_bytes=756983808 token_bytes=633361264 weight_movement_bytes=440401920 total_nvlink_bytes=1073763184 weight_to_token_ratio=0.695341 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=27008 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.316519345e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61678 payload_rows_remote=53488 dup_rows=3858 dup_rows_remote=3156 dedup_ratio=0.058868 router_tokens_per_expert=[0, 4095, 0, 0, 0, 0, 6, 0, 348, 0, 0, 0, 0, 0, 14, 0, 0, 1, 0, 0, 4093, 0, 0, 0, 0, 1208, 1, 0, 0, 0, 3821, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 15, 0, 2, 0, 0, 0, 0, 1733, 1824, 1, 0, 0, 4089, 4084, 4096, 0, 0, 0, 3330, 0] payload_rows_per_dst=[8192, 8190, 8186, 7734, 8192, 8190, 6334, 6660] dup_rows_per_dst=[10, 702, 2, 2326, 14, 22, 782, 0] combine_rows_per_src=[8192, 8190, 8186, 7734, 8192, 8190, 6334, 6660] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62396 payload_rows_remote=55072 dup_rows=3140 dup_rows_remote=1960 dedup_ratio=0.047913 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 436, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 753, 0, 0, 0, 0, 3499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 14, 0, 0, 0, 0, 0, 1, 1330, 2751, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 3496, 0] payload_rows_per_dst=[8192, 8192, 8192, 7324, 8192, 8192, 7120, 6992] dup_rows_per_dst=[0, 874, 0, 1180, 0, 42, 1044, 0] combine_rows_per_src=[8192, 8192, 8192, 7324, 8192, 8192, 7120, 6992] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62034 payload_rows_remote=53846 dup_rows=3502 dup_rows_remote=3502 dedup_ratio=0.053436 router_tokens_per_expert=[0, 4094, 0, 0, 0, 0, 0, 0, 682, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 4095, 0, 0, 1, 0, 447, 0, 0, 0, 0, 3872, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 11, 0, 0, 0, 0, 1, 0, 1696, 2412, 0, 0, 0, 4095, 4095, 4095, 0, 0, 0, 3167, 0] payload_rows_per_dst=[8188, 8192, 8192, 7756, 8192, 8190, 6990, 6334] dup_rows_per_dst=[0, 1366, 4, 882, 0, 22, 1228, 0] combine_rows_per_src=[8188, 8192, 8192, 7756, 8192, 8190, 6990, 6334] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61494 payload_rows_remote=53302 dup_rows=4042 dup_rows_remote=6 dedup_ratio=0.061676 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 0, 2694, 0, 45, 0, 0, 0, 0, 0, 0, 3559, 4096, 0, 0, 1983, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3962] payload_rows_per_dst=[8192, 8192, 8192, 8192, 5492, 7118, 8192, 7924] dup_rows_per_dst=[0, 0, 0, 0, 6, 0, 4036, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 5492, 7118, 8192, 7924] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60376 payload_rows_remote=52186 dup_rows=5160 dup_rows_remote=5096 dedup_ratio=0.078735 router_tokens_per_expert=[1, 4095, 3, 0, 0, 0, 0, 0, 650, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 4093, 0, 0, 0, 0, 1604, 0, 0, 0, 0, 3895, 2, 0, 0, 2, 0, 1, 1, 0, 0, 0, 5, 9, 0, 9, 0, 10, 0, 0, 0, 3, 435, 2139, 0, 11, 0, 4094, 4083, 4091, 0, 1, 0, 3528, 0] payload_rows_per_dst=[8192, 8178, 8190, 6966, 8186, 8190, 5416, 7058] dup_rows_per_dst=[6, 1290, 0, 2652, 4, 64, 1144, 0] combine_rows_per_src=[8192, 8178, 8190, 6966, 8186, 8190, 5416, 7058] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=364206080 dispatch_meta_bytes=205872 combine_payload_bytes=364206080 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=364411952 ingress_bytes=474306560 link_bytes=474306560 token_bytes=728618032 weight_movement_bytes=110100480 total_nvlink_bytes=838718512 weight_to_token_ratio=0.151109 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=20736 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.248971179e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62102 payload_rows_remote=53912 dup_rows=3434 dup_rows_remote=3434 dedup_ratio=0.052399 router_tokens_per_expert=[0, 4095, 0, 0, 0, 0, 0, 0, 468, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 4095, 0, 0, 1, 0, 629, 0, 0, 0, 0, 3920, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 1549, 2593, 0, 0, 0, 4095, 4095, 4095, 0, 0, 0, 3121, 0] payload_rows_per_dst=[8190, 8192, 8192, 7850, 8192, 8190, 7054, 6242] dup_rows_per_dst=[0, 938, 4, 1248, 0, 14, 1230, 0] combine_rows_per_src=[8190, 8192, 8192, 7850, 8192, 8190, 7054, 6242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62446 payload_rows_remote=55098 dup_rows=3090 dup_rows_remote=1852 dedup_ratio=0.047150 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 249, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 783, 0, 0, 0, 0, 3510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 6, 0, 0, 0, 0, 0, 2, 1455, 2910, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 3367, 0] payload_rows_per_dst=[8192, 8192, 8192, 7348, 8192, 8192, 7404, 6734] dup_rows_per_dst=[0, 500, 0, 1238, 0, 22, 1330, 0] combine_rows_per_src=[8192, 8192, 8192, 7348, 8192, 8192, 7404, 6734] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=317126656 dispatch_meta_bytes=181944 combine_payload_bytes=317126656 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=317308600 ingress_bytes=757528576 link_bytes=757528576 token_bytes=634435256 weight_movement_bytes=440401920 total_nvlink_bytes=1074837176 weight_to_token_ratio=0.694164 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=26496 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.317466786e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=383401984 dispatch_meta_bytes=226576 combine_payload_bytes=383401984 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=383628560 ingress_bytes=493502464 link_bytes=493502464 token_bytes=767030544 weight_movement_bytes=110100480 total_nvlink_bytes=877131024 weight_to_token_ratio=0.143541 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=27392 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.582819521e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=394756096 dispatch_meta_bytes=228128 combine_payload_bytes=394756096 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=394984224 ingress_bytes=725057536 link_bytes=725057536 token_bytes=789740320 weight_movement_bytes=330301440 total_nvlink_bytes=1120041760 weight_to_token_ratio=0.418241 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=24448 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.260994307e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=385968128 dispatch_meta_bytes=229392 combine_payload_bytes=385968128 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=386197520 ingress_bytes=606169088 link_bytes=606169088 token_bytes=772165648 weight_movement_bytes=220200960 total_nvlink_bytes=992366608 weight_to_token_ratio=0.285173 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.054227742e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61836 payload_rows_remote=54342 dup_rows=3700 dup_rows_remote=3700 dedup_ratio=0.056458 router_tokens_per_expert=[3, 0, 32, 0, 4080, 0, 3, 4095, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 0, 0, 0, 4095, 1, 13, 0, 0, 7, 2768, 0, 39, 0, 0, 0, 0, 1, 0, 3746, 4087, 0, 4, 1799, 0, 16, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3878] payload_rows_per_dst=[8184, 8192, 8188, 8190, 5640, 7494, 8192, 7756] dup_rows_per_dst=[52, 6, 0, 2, 16, 0, 3624, 0] combine_rows_per_src=[8184, 8192, 8188, 8190, 5640, 7494, 8192, 7756] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63566 payload_rows_remote=57046 dup_rows=1970 dup_rows_remote=1970 dedup_ratio=0.030060 router_tokens_per_expert=[0, 0, 10, 0, 4093, 1, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 24, 0, 0, 0, 3199, 1, 36, 0, 0, 0, 0, 1, 0, 4046, 4094, 0, 7, 928, 0, 40, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 3999] payload_rows_per_dst=[8192, 8192, 8192, 8184, 6520, 8094, 8192, 8000] dup_rows_per_dst=[18, 0, 4, 0, 0, 0, 1946, 2] combine_rows_per_src=[8192, 8192, 8192, 8184, 6520, 8094, 8192, 8000] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=382068736 dispatch_meta_bytes=213232 combine_payload_bytes=382068736 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=382281968 ingress_bytes=492169216 link_bytes=492169216 token_bytes=764350704 weight_movement_bytes=110100480 total_nvlink_bytes=874451184 weight_to_token_ratio=0.144044 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=19072 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.559632146e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58990 payload_rows_remote=51174 dup_rows=6546 dup_rows_remote=3502 dedup_ratio=0.099884 router_tokens_per_expert=[0, 0, 8, 0, 4094, 0, 0, 4095, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 32, 0, 0, 0, 2872, 1, 31, 8, 0, 0, 0, 0, 0, 3787, 4092, 0, 0, 1717, 0, 3, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 3831] payload_rows_per_dst=[7886, 8192, 8190, 8190, 5846, 4680, 8190, 7816] dup_rows_per_dst=[14, 2, 4, 12, 28, 8, 3434, 3044] combine_rows_per_src=[7886, 8192, 8190, 8190, 5846, 4680, 8190, 7816] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60110 payload_rows_remote=51918 dup_rows=5426 dup_rows_remote=5426 dedup_ratio=0.082794 router_tokens_per_expert=[0, 0, 2, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 23, 0, 0, 0, 2319, 0, 35, 1, 0, 0, 0, 0, 0, 3800, 4095, 0, 0, 2193, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3849] payload_rows_per_dst=[8192, 8192, 8192, 8192, 6932, 8192, 4520, 7698] dup_rows_per_dst=[4, 0, 0, 0, 2616, 2806, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 6932, 8192, 4520, 7698] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=374069248 dispatch_meta_bytes=229128 combine_payload_bytes=374069248 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=374298376 ingress_bytes=484169728 link_bytes=484169728 token_bytes=748367624 weight_movement_bytes=110100480 total_nvlink_bytes=858468104 weight_to_token_ratio=0.147121 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.420507892e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=382584832 dispatch_meta_bytes=227704 combine_payload_bytes=382584832 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=382812536 ingress_bytes=492685312 link_bytes=492685312 token_bytes=765397368 weight_movement_bytes=110100480 total_nvlink_bytes=875497848 weight_to_token_ratio=0.143847 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=29824 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.568607904e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=386441216 dispatch_meta_bytes=229384 combine_payload_bytes=386441216 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=386670600 ingress_bytes=606642176 link_bytes=606642176 token_bytes=773111816 weight_movement_bytes=220200960 total_nvlink_bytes=993312776 weight_to_token_ratio=0.284824 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.055050520e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=394942464 dispatch_meta_bytes=227800 combine_payload_bytes=394942464 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=395170264 ingress_bytes=725243904 link_bytes=725243904 token_bytes=790112728 weight_movement_bytes=330301440 total_nvlink_bytes=1120414168 weight_to_token_ratio=0.418043 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=22528 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.261318431e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63810 payload_rows_remote=57140 dup_rows=1726 dup_rows_remote=1726 dedup_ratio=0.026337 router_tokens_per_expert=[0, 0, 11, 0, 4093, 1, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 55, 0, 0, 0, 3238, 1, 41, 0, 0, 0, 0, 1, 0, 4041, 4095, 0, 5, 730, 0, 114, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 4052] payload_rows_per_dst=[8190, 8192, 8192, 8184, 6670, 8084, 8192, 8106] dup_rows_per_dst=[22, 0, 6, 0, 0, 0, 1696, 2] combine_rows_per_src=[8190, 8192, 8192, 8184, 6670, 8084, 8192, 8106] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59008 payload_rows_remote=50816 dup_rows=6528 dup_rows_remote=6528 dedup_ratio=0.099609 router_tokens_per_expert=[0, 0, 1, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 60, 0, 0, 0, 2434, 0, 49, 1, 0, 0, 0, 0, 0, 3834, 4096, 0, 0, 1889, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3913] payload_rows_per_dst=[8192, 8192, 8192, 8192, 6234, 8192, 3988, 7826] dup_rows_per_dst=[2, 0, 0, 0, 2152, 4370, 4, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 6234, 8192, 3988, 7826] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=389523456 dispatch_meta_bytes=232168 combine_payload_bytes=389523456 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=389755624 ingress_bytes=829925376 link_bytes=829925376 token_bytes=779279080 weight_movement_bytes=440401920 total_nvlink_bytes=1219681000 weight_to_token_ratio=0.565140 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=18944 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.443376728e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61482 payload_rows_remote=53290 dup_rows=4054 dup_rows_remote=10 dedup_ratio=0.061859 router_tokens_per_expert=[0, 0, 1, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 23, 0, 0, 0, 2622, 0, 47, 0, 0, 0, 0, 0, 0, 3551, 4096, 0, 0, 1939, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022] payload_rows_per_dst=[8192, 8192, 8192, 8192, 5376, 7102, 8192, 8044] dup_rows_per_dst=[2, 0, 0, 0, 8, 0, 4044, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 5376, 7102, 8192, 8044] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=408905728 dispatch_meta_bytes=236064 combine_payload_bytes=408905728 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=409141792 ingress_bytes=519006208 link_bytes=519006208 token_bytes=818047520 weight_movement_bytes=110100480 total_nvlink_bytes=928148000 weight_to_token_ratio=0.134589 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=22144 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.026371576e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6896858264 prefetch_bytes=1211105280 reduce_bytes=660602880 weight_movement_bytes=1871708160 egress_bytes=3449537688 ingress_bytes=5319028736 link_bytes=5319028736 total_nvlink_bytes=8768566424 weight_to_token_ratio=0.271386 prefetch_share_of_ingress=0.227693 dedup_ratio=0.148349 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=6 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.038971 gemm_inner_pad_upper_bound=10541 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.250665802e-05 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=366815232 dispatch_meta_bytes=218704 combine_payload_bytes=366815232 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=367033936 ingress_bytes=476915712 link_bytes=476915712 token_bytes=733849168 weight_movement_bytes=110100480 total_nvlink_bytes=843949648 weight_to_token_ratio=0.150031 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=31232 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.294348623e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=372148224 dispatch_meta_bytes=229376 combine_payload_bytes=372148224 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=372377600 ingress_bytes=482248704 link_bytes=482248704 token_bytes=744525824 weight_movement_bytes=110100480 total_nvlink_bytes=854626304 weight_to_token_ratio=0.147880 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.387098125e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7140984208 prefetch_bytes=990904320 reduce_bytes=1321205760 weight_movement_bytes=2312110080 egress_bytes=3571621264 ingress_bytes=5881473024 link_bytes=5881473024 total_nvlink_bytes=9453094288 weight_to_token_ratio=0.323780 prefetch_share_of_ingress=0.168479 dedup_ratio=0.121500 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=12 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=9652 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.022884892e-04 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60362 payload_rows_remote=52170 dup_rows=5174 dup_rows_remote=5168 dedup_ratio=0.078949 router_tokens_per_expert=[0, 0, 40, 0, 4087, 0, 0, 4095, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2, 1, 1, 10, 2590, 0, 56, 3, 0, 0, 0, 0, 0, 3976, 4095, 0, 9, 1708, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3841] payload_rows_per_dst=[8180, 8192, 8192, 8190, 8184, 8190, 3552, 7682] dup_rows_per_dst=[74, 6, 0, 0, 5088, 6, 0, 0] combine_rows_per_src=[8180, 8192, 8192, 8190, 8184, 8190, 3552, 7682] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60516 payload_rows_remote=52324 dup_rows=5020 dup_rows_remote=5020 dedup_ratio=0.076599 router_tokens_per_expert=[0, 0, 84, 0, 4085, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 19, 0, 0, 1, 2620, 0, 41, 9, 0, 0, 0, 0, 0, 3892, 4096, 0, 27, 1576, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4020] payload_rows_per_dst=[8182, 8192, 8192, 8192, 5362, 8020, 6336, 8040] dup_rows_per_dst=[156, 0, 0, 0, 0, 2736, 2128, 0] combine_rows_per_src=[8182, 8192, 8192, 8192, 5362, 8020, 6336, 8040] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59904 payload_rows_remote=51712 dup_rows=5632 dup_rows_remote=5614 dedup_ratio=0.085938 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 8, 0, 0, 1, 3032, 0, 42, 0, 0, 0, 0, 0, 0, 3820, 4096, 0, 0, 1537, 0, 60, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3786] payload_rows_per_dst=[8192, 8178, 8190, 8192, 8192, 8192, 3194, 7574] dup_rows_per_dst=[18, 0, 0, 0, 5614, 0, 0, 0] combine_rows_per_src=[8192, 8178, 8190, 8192, 8192, 8192, 3194, 7574] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7288706192 prefetch_bytes=990904320 reduce_bytes=660602880 weight_movement_bytes=1651507200 egress_bytes=3645484176 ingress_bytes=5294729216 link_bytes=5294729216 total_nvlink_bytes=8940213392 weight_to_token_ratio=0.226584 prefetch_share_of_ingress=0.187149 dedup_ratio=0.107443 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=6 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.038971 gemm_inner_pad_upper_bound=10287 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.208404940e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=409579520 dispatch_meta_bytes=235464 combine_payload_bytes=409579520 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=409814984 ingress_bytes=519680000 link_bytes=519680000 token_bytes=819394504 weight_movement_bytes=110100480 total_nvlink_bytes=929494984 weight_to_token_ratio=0.134368 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=22656 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.038089927e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60624 payload_rows_remote=52706 dup_rows=4912 dup_rows_remote=2932 dedup_ratio=0.074951 router_tokens_per_expert=[0, 0, 7, 0, 4095, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 39, 0, 0, 0, 3018, 1, 31, 9, 0, 0, 0, 0, 0, 3819, 4092, 0, 0, 1429, 0, 9, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 3928] payload_rows_per_dst=[7882, 8192, 8190, 8190, 6156, 5906, 8190, 7918] dup_rows_per_dst=[14, 0, 4, 12, 24, 8, 2870, 1980] combine_rows_per_src=[7882, 8192, 8190, 8190, 6156, 5906, 8190, 7918] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=364249088 dispatch_meta_bytes=229376 combine_payload_bytes=364249088 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=364478464 ingress_bytes=474349568 link_bytes=474349568 token_bytes=728727552 weight_movement_bytes=110100480 total_nvlink_bytes=838828032 weight_to_token_ratio=0.151086 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.249719158e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62132 payload_rows_remote=54558 dup_rows=3404 dup_rows_remote=3402 dedup_ratio=0.051941 router_tokens_per_expert=[3, 0, 32, 0, 4081, 0, 3, 4095, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 23, 0, 0, 8, 2779, 0, 55, 0, 1, 0, 0, 1, 0, 3786, 4089, 0, 6, 1613, 0, 46, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3952] payload_rows_per_dst=[8184, 8190, 8188, 8190, 5710, 7574, 8192, 7904] dup_rows_per_dst=[54, 6, 0, 2, 20, 2, 3320, 0] combine_rows_per_src=[8184, 8190, 8188, 8190, 5710, 7574, 8192, 7904] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=381982720 dispatch_meta_bytes=213200 combine_payload_bytes=381982720 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=382195920 ingress_bytes=492083200 link_bytes=492083200 token_bytes=764178640 weight_movement_bytes=110100480 total_nvlink_bytes=874279120 weight_to_token_ratio=0.144077 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=20480 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.558136186e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60128 payload_rows_remote=51936 dup_rows=5408 dup_rows_remote=5402 dedup_ratio=0.082520 router_tokens_per_expert=[0, 0, 38, 0, 4089, 0, 0, 4095, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 6, 1, 1, 8, 2716, 0, 58, 3, 0, 0, 0, 0, 0, 3965, 4095, 0, 9, 1482, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3940] payload_rows_per_dst=[8184, 8192, 8192, 8190, 8184, 8190, 3116, 7880] dup_rows_per_dst=[70, 6, 0, 0, 5326, 6, 0, 0] combine_rows_per_src=[8184, 8192, 8192, 8190, 8184, 8190, 3116, 7880] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59628 payload_rows_remote=51436 dup_rows=5908 dup_rows_remote=5892 dedup_ratio=0.090149 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 30, 0, 0, 1, 3075, 0, 43, 0, 0, 0, 0, 0, 0, 3893, 4096, 0, 0, 1307, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3859] payload_rows_per_dst=[8192, 8182, 8188, 8192, 8192, 8192, 2772, 7718] dup_rows_per_dst=[16, 0, 0, 0, 5892, 0, 0, 0] combine_rows_per_src=[8192, 8182, 8188, 8192, 8192, 8192, 2772, 7718] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7363082000 prefetch_bytes=1211105280 reduce_bytes=660602880 weight_movement_bytes=1871708160 egress_bytes=3682686736 ingress_bytes=5552103424 link_bytes=5552103424 total_nvlink_bytes=9234790160 weight_to_token_ratio=0.254202 prefetch_share_of_ingress=0.218134 dedup_ratio=0.102145 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=6 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.038235 gemm_inner_pad_upper_bound=10795 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.656021018e-05 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62564 payload_rows_remote=54372 dup_rows=2972 dup_rows_remote=2972 dedup_ratio=0.045349 router_tokens_per_expert=[0, 0, 82, 0, 4088, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 31, 0, 0, 1, 2706, 0, 58, 7, 0, 0, 0, 0, 0, 3901, 4096, 0, 27, 1367, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079] payload_rows_per_dst=[8184, 8192, 8192, 8192, 5588, 7890, 8168, 8158] dup_rows_per_dst=[156, 0, 0, 0, 4, 20, 2792, 0] combine_rows_per_src=[8184, 8192, 8192, 8192, 5588, 7890, 8168, 8158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7201269248 prefetch_bytes=880803840 reduce_bytes=880803840 weight_movement_bytes=1761607680 egress_bytes=3601757696 ingress_bytes=5361119232 link_bytes=5361119232 total_nvlink_bytes=8962876928 weight_to_token_ratio=0.244625 prefetch_share_of_ingress=0.164295 dedup_ratio=0.115546 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=8 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=10033 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.323868097e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6676569088 prefetch_bytes=990904320 reduce_bytes=1211105280 weight_movement_bytes=2202009600 egress_bytes=3339377664 ingress_bytes=5539201024 link_bytes=5539201024 total_nvlink_bytes=8878578688 weight_to_token_ratio=0.329812 prefetch_share_of_ingress=0.178889 dedup_ratio=0.170911 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=11 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.037500 gemm_inner_pad_upper_bound=10795 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.633581623e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=377796608 dispatch_meta_bytes=222552 combine_payload_bytes=377796608 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=378019160 ingress_bytes=487897088 link_bytes=487897088 token_bytes=755815768 weight_movement_bytes=110100480 total_nvlink_bytes=865916248 weight_to_token_ratio=0.145671 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=31872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.485332813e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7194268296 prefetch_bytes=880803840 reduce_bytes=770703360 weight_movement_bytes=1651507200 egress_bytes=3598254728 ingress_bytes=5247520768 link_bytes=5247520768 total_nvlink_bytes=8845775496 weight_to_token_ratio=0.229559 prefetch_share_of_ingress=0.167851 dedup_ratio=0.115720 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=7 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040441 gemm_inner_pad_upper_bound=10033 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.126301684e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=373954560 dispatch_meta_bytes=229352 combine_payload_bytes=373954560 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=374183912 ingress_bytes=484055040 link_bytes=484055040 token_bytes=748138472 weight_movement_bytes=110100480 total_nvlink_bytes=858238952 weight_to_token_ratio=0.147166 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.418513279e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=375058432 dispatch_meta_bytes=229376 combine_payload_bytes=375058432 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=375287808 ingress_bytes=485158912 link_bytes=485158912 token_bytes=750346240 weight_movement_bytes=110100480 total_nvlink_bytes=860446720 weight_to_token_ratio=0.146733 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.437711429e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=370671616 dispatch_meta_bytes=229304 combine_payload_bytes=370671616 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=370900920 ingress_bytes=811073536 link_bytes=811073536 token_bytes=741572536 weight_movement_bytes=440401920 total_nvlink_bytes=1181974456 weight_to_token_ratio=0.593876 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.410590278e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7169680368 prefetch_bytes=1321205760 reduce_bytes=110100480 weight_movement_bytes=1431306240 egress_bytes=3585967088 ingress_bytes=5015019520 link_bytes=5015019520 total_nvlink_bytes=8600986608 weight_to_token_ratio=0.199633 prefetch_share_of_ingress=0.263450 dedup_ratio=0.118637 prefetch_slots=24 unused_prefetch_slots=136 slot_utilization=0.150000 reduce_slots=1 remote_demand=12 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040441 gemm_inner_pad_upper_bound=10287 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.721943774e-05 prefetch_bytes_per_gemm_flop=2.297794118e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6794503936 prefetch_bytes=990904320 reduce_bytes=2091909120 weight_movement_bytes=3082813440 egress_bytes=3398348544 ingress_bytes=6478968832 link_bytes=6478968832 total_nvlink_bytes=9877317376 weight_to_token_ratio=0.453722 prefetch_share_of_ingress=0.152942 dedup_ratio=0.157635 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=19 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.039706 gemm_inner_pad_upper_bound=10033 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.126799241e-04 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7345150488 prefetch_bytes=990904320 reduce_bytes=1541406720 weight_movement_bytes=2532311040 egress_bytes=3673729560 ingress_bytes=6203731968 link_bytes=6203731968 total_nvlink_bytes=9877461528 weight_to_token_ratio=0.344760 prefetch_share_of_ingress=0.159727 dedup_ratio=0.113257 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=14 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043382 gemm_inner_pad_upper_bound=9017 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.078931023e-04 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none - [2026-08-26 15:11:33] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 368.6 | throughput per GPU (TFLOP/s/GPU): 335.0 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 9.942503E+00 | loss scale: 1.0 | grad norm: 22.926 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44604 payload_rows_remote=39144 dup_rows=20932 dup_rows_remote=18856 dedup_ratio=0.319397 router_tokens_per_expert=[134, 18, 33, 69, 35, 11, 843, 2454, 196, 24, 144, 1066, 2, 107, 1, 210, 137, 790, 1249, 0, 162, 390, 10, 240, 1413, 215, 277, 896, 1, 1890, 1337, 20, 30, 3, 51, 1732, 7, 3, 5, 402, 943, 664, 399, 7, 2426, 723, 740, 1073, 41, 825, 1242, 422, 677, 200, 70, 291, 941, 1620, 404, 422, 454, 435, 1140, 2] payload_rows_per_dst=[5652, 3260, 4692, 7126, 4774, 6968, 5460, 6672] dup_rows_per_dst=[1542, 414, 1264, 4972, 1172, 5328, 2076, 4164] combine_rows_per_src=[5652, 3260, 4692, 7126, 4774, 6968, 5460, 6672] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46820 payload_rows_remote=40976 dup_rows=18716 dup_rows_remote=16646 dedup_ratio=0.285583 router_tokens_per_expert=[100, 77, 52, 34, 0, 11, 1121, 1702, 372, 94, 193, 753, 0, 128, 0, 391, 653, 527, 1819, 0, 149, 484, 6, 144, 903, 111, 501, 516, 4, 1873, 1033, 4, 11, 12, 87, 1667, 3, 0, 3, 349, 918, 290, 1336, 8, 2684, 633, 1825, 617, 140, 875, 1350, 157, 496, 301, 42, 145, 1266, 1097, 293, 864, 308, 192, 1044, 0] payload_rows_per_dst=[6306, 6840, 5564, 4902, 5844, 5588, 5256, 6520] dup_rows_per_dst=[3580, 2390, 2000, 1296, 2070, 2016, 1756, 3608] combine_rows_per_src=[6306, 6840, 5564, 4902, 5844, 5588, 5256, 6520] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46662 payload_rows_remote=41540 dup_rows=18874 dup_rows_remote=17190 dedup_ratio=0.287994 router_tokens_per_expert=[124, 126, 50, 137, 3, 18, 1220, 1833, 310, 17, 155, 866, 0, 57, 1, 167, 288, 683, 1535, 0, 257, 268, 2, 370, 1092, 86, 356, 780, 5, 1623, 899, 0, 10, 3, 112, 2029, 9, 0, 6, 255, 1207, 420, 572, 6, 2852, 748, 1470, 938, 33, 1046, 1456, 392, 524, 152, 39, 172, 1322, 1206, 398, 518, 394, 122, 1029, 0] payload_rows_per_dst=[6368, 6884, 5122, 5044, 6340, 5590, 5508, 5806] dup_rows_per_dst=[3900, 1966, 1684, 1392, 3324, 2344, 2120, 2144] combine_rows_per_src=[6368, 6884, 5122, 5044, 6340, 5590, 5508, 5806] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46328 payload_rows_remote=41168 dup_rows=19208 dup_rows_remote=17210 dedup_ratio=0.293091 router_tokens_per_expert=[107, 55, 84, 70, 6, 34, 829, 1193, 281, 89, 323, 1819, 1, 83, 0, 298, 336, 479, 2003, 12, 569, 346, 3, 218, 736, 219, 503, 750, 1, 1247, 886, 16, 27, 10, 103, 1910, 5, 14, 3, 708, 993, 656, 663, 24, 2491, 493, 1132, 750, 29, 1423, 1109, 177, 666, 189, 69, 329, 1210, 1248, 395, 191, 324, 229, 1599, 3] payload_rows_per_dst=[3892, 7142, 5770, 6170, 5718, 5160, 5760, 6716] dup_rows_per_dst=[864, 3628, 2162, 2546, 2106, 1998, 2222, 3682] combine_rows_per_src=[3892, 7142, 5770, 6170, 5718, 5160, 5760, 6716] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45452 payload_rows_remote=39342 dup_rows=20084 dup_rows_remote=17444 dedup_ratio=0.306458 router_tokens_per_expert=[189, 42, 180, 58, 91, 5, 983, 1302, 447, 69, 112, 1193, 1, 176, 0, 145, 213, 826, 1854, 4, 252, 596, 2, 522, 1473, 579, 353, 735, 11, 2122, 1215, 9, 6, 1, 113, 1484, 4, 5, 34, 453, 374, 671, 509, 52, 2050, 628, 1267, 1282, 10, 1779, 1417, 155, 1294, 152, 43, 264, 558, 413, 219, 269, 470, 195, 838, 0] payload_rows_per_dst=[6252, 6174, 5982, 6110, 5944, 3908, 6430, 4652] dup_rows_per_dst=[3692, 2212, 2556, 2640, 2614, 1300, 3798, 1272] combine_rows_per_src=[6252, 6174, 5982, 6110, 5944, 3908, 6430, 4652] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45456 payload_rows_remote=38498 dup_rows=20080 dup_rows_remote=15976 dedup_ratio=0.306396 router_tokens_per_expert=[190, 60, 169, 99, 44, 10, 838, 1453, 1314, 212, 194, 1266, 0, 84, 1, 279, 178, 1461, 1641, 2, 306, 358, 8, 690, 1641, 255, 450, 652, 34, 1306, 786, 3, 42, 22, 169, 1493, 8, 4, 0, 450, 569, 736, 746, 29, 2181, 549, 929, 1319, 13, 969, 1103, 318, 1052, 182, 118, 34, 1111, 459, 164, 378, 516, 188, 932, 1] payload_rows_per_dst=[5712, 6958, 6722, 5182, 6004, 5248, 5598, 4032] dup_rows_per_dst=[2626, 4104, 4092, 1692, 2868, 1874, 1980, 844] combine_rows_per_src=[5712, 6958, 6722, 5182, 6004, 5248, 5598, 4032] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45234 payload_rows_remote=38964 dup_rows=20302 dup_rows_remote=16962 dedup_ratio=0.309784 router_tokens_per_expert=[123, 34, 61, 70, 14, 36, 1345, 1418, 387, 14, 154, 1326, 1, 85, 0, 198, 319, 519, 1640, 2, 499, 436, 8, 264, 988, 367, 277, 962, 1, 1087, 892, 0, 5, 8, 198, 2256, 0, 0, 23, 548, 1150, 376, 407, 4, 2419, 598, 1106, 1114, 70, 1483, 1046, 129, 1283, 419, 46, 295, 1130, 997, 170, 184, 649, 130, 996, 2] payload_rows_per_dst=[4762, 3788, 5380, 6222, 5894, 7076, 5842, 6270] dup_rows_per_dst=[1440, 542, 1994, 2926, 2394, 5060, 2606, 3340] combine_rows_per_src=[4762, 3788, 5380, 6222, 5894, 7076, 5842, 6270] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45866 payload_rows_remote=40120 dup_rows=19670 dup_rows_remote=16468 dedup_ratio=0.300140 router_tokens_per_expert=[111, 44, 113, 70, 2, 35, 683, 1536, 152, 117, 196, 654, 1, 43, 39, 380, 248, 538, 1465, 61, 415, 757, 11, 370, 1411, 196, 353, 1160, 7, 1880, 1141, 13, 60, 15, 65, 1188, 24, 0, 10, 992, 606, 524, 559, 1, 3035, 525, 933, 948, 312, 1517, 1288, 271, 452, 281, 89, 352, 788, 1127, 417, 739, 414, 154, 824, 56] payload_rows_per_dst=[5746, 6932, 6636, 5444, 5794, 4510, 6168, 4636] dup_rows_per_dst=[3202, 2302, 3916, 1872, 2676, 1568, 2956, 1178] combine_rows_per_src=[5746, 6932, 6636, 5444, 5794, 4510, 6168, 4636] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7025909696 prefetch_bytes=1211105280 reduce_bytes=550502400 weight_movement_bytes=1761607680 egress_bytes=3514077120 ingress_bytes=5273440256 link_bytes=5273440256 total_nvlink_bytes=8787517376 weight_to_token_ratio=0.250730 prefetch_share_of_ingress=0.229661 dedup_ratio=0.139932 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=5 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.038971 gemm_inner_pad_upper_bound=10541 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.171379937e-05 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=391071744 dispatch_meta_bytes=231840 combine_payload_bytes=391071744 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=391303584 ingress_bytes=831473664 link_bytes=831473664 token_bytes=782375328 weight_movement_bytes=440401920 total_nvlink_bytes=1222777248 weight_to_token_ratio=0.562904 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=20352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.446069456e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=372277248 dispatch_meta_bytes=229352 combine_payload_bytes=372277248 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=372506600 ingress_bytes=482377728 link_bytes=482377728 token_bytes=744783848 weight_movement_bytes=110100480 total_nvlink_bytes=854884328 weight_to_token_ratio=0.147829 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.389342065e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=368693248 dispatch_meta_bytes=229312 combine_payload_bytes=368693248 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=368922560 ingress_bytes=809095168 link_bytes=809095168 token_bytes=737615808 weight_movement_bytes=440401920 total_nvlink_bytes=1178017728 weight_to_token_ratio=0.597061 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.407149570e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=389738496 dispatch_meta_bytes=229376 combine_payload_bytes=389738496 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=389967872 ingress_bytes=499838976 link_bytes=499838976 token_bytes=779706368 weight_movement_bytes=110100480 total_nvlink_bytes=889806848 weight_to_token_ratio=0.141208 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.693021886e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6722753608 prefetch_bytes=1101004800 reduce_bytes=990904320 weight_movement_bytes=2091909120 egress_bytes=3362452552 ingress_bytes=5452210176 link_bytes=5452210176 total_nvlink_bytes=8814662728 weight_to_token_ratio=0.311168 prefetch_share_of_ingress=0.201937 dedup_ratio=0.165503 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=9 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.037868 gemm_inner_pad_upper_bound=10795 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.482290231e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7169801352 prefetch_bytes=880803840 reduce_bytes=1541406720 weight_movement_bytes=2422210560 egress_bytes=3586030728 ingress_bytes=6005981184 link_bytes=6005981184 total_nvlink_bytes=9592011912 weight_to_token_ratio=0.337835 prefetch_share_of_ingress=0.146654 dedup_ratio=0.117419 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=14 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=9525 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.044538909e-04 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7188980096 prefetch_bytes=1321205760 reduce_bytes=110100480 weight_movement_bytes=1431306240 egress_bytes=3595618688 ingress_bytes=5024667648 link_bytes=5024667648 total_nvlink_bytes=8620286336 weight_to_token_ratio=0.199097 prefetch_share_of_ingress=0.262944 dedup_ratio=0.116147 prefetch_slots=24 unused_prefetch_slots=136 slot_utilization=0.150000 reduce_slots=1 remote_demand=12 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=10033 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.738723456e-05 prefetch_bytes_per_gemm_flop=2.297794118e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6849672480 prefetch_bytes=990904320 reduce_bytes=2202009600 weight_movement_bytes=3192913920 egress_bytes=3425934624 ingress_bytes=6616651776 link_bytes=6616651776 total_nvlink_bytes=10042586400 weight_to_token_ratio=0.466141 prefetch_share_of_ingress=0.149759 dedup_ratio=0.151382 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=20 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040441 gemm_inner_pad_upper_bound=9906 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.150744569e-04 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7425897912 prefetch_bytes=880803840 reduce_bytes=1541406720 weight_movement_bytes=2422210560 egress_bytes=3714106808 ingress_bytes=6134001664 link_bytes=6134001664 total_nvlink_bytes=9848108472 weight_to_token_ratio=0.326184 prefetch_share_of_ingress=0.143594 dedup_ratio=0.105002 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=14 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=9271 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.066803776e-04 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none + [2026-08-26 15:59:19] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 366.7 | throughput per GPU (TFLOP/s/GPU): 336.7 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 9.942634E+00 | loss scale: 1.0 | grad norm: 22.920 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45528 payload_rows_remote=40064 dup_rows=20008 dup_rows_remote=17888 dedup_ratio=0.305298 router_tokens_per_expert=[137, 18, 32, 75, 33, 10, 857, 2454, 191, 26, 144, 999, 2, 112, 1, 222, 138, 811, 1254, 0, 173, 374, 10, 238, 1397, 199, 283, 905, 2, 1906, 1334, 22, 31, 3, 53, 1718, 8, 3, 4, 375, 911, 715, 361, 8, 2423, 734, 702, 1071, 46, 811, 1253, 458, 676, 200, 70, 278, 919, 1653, 408, 442, 459, 482, 1132, 2] payload_rows_per_dst=[6226, 3884, 4684, 5948, 6000, 6610, 5464, 6712] dup_rows_per_dst=[2410, 634, 1312, 2336, 2202, 4712, 2120, 4282] combine_rows_per_src=[6226, 3884, 4684, 5948, 6000, 6610, 5464, 6712] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45442 payload_rows_remote=38534 dup_rows=20094 dup_rows_remote=16106 dedup_ratio=0.306610 router_tokens_per_expert=[189, 59, 164, 105, 40, 10, 838, 1457, 1301, 220, 189, 1197, 0, 84, 0, 283, 192, 1502, 1648, 2, 318, 345, 8, 677, 1619, 247, 460, 659, 36, 1278, 794, 4, 45, 24, 176, 1491, 9, 4, 0, 415, 556, 805, 690, 31, 2174, 560, 868, 1314, 14, 962, 1120, 368, 1056, 187, 119, 34, 1110, 485, 168, 392, 523, 209, 933, 1] payload_rows_per_dst=[5336, 6908, 6976, 6018, 5120, 4478, 5624, 4982] dup_rows_per_dst=[2124, 3988, 4436, 2590, 1764, 1406, 2096, 1690] combine_rows_per_src=[5336, 6908, 6976, 6018, 5120, 4478, 5624, 4982] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47000 payload_rows_remote=41156 dup_rows=18536 dup_rows_remote=16468 dedup_ratio=0.282837 router_tokens_per_expert=[95, 76, 44, 36, 0, 11, 1136, 1683, 351, 94, 178, 701, 0, 138, 0, 400, 653, 539, 1847, 0, 153, 460, 6, 137, 888, 105, 509, 534, 5, 1878, 1036, 5, 12, 15, 91, 1642, 3, 0, 3, 312, 892, 366, 1257, 8, 2682, 643, 1773, 616, 152, 854, 1355, 182, 498, 316, 50, 128, 1274, 1172, 300, 885, 329, 209, 1051, 0] payload_rows_per_dst=[6418, 6804, 5566, 5466, 5844, 5580, 5280, 6042] dup_rows_per_dst=[3290, 2284, 2024, 2096, 2068, 1984, 1790, 3000] combine_rows_per_src=[6418, 6804, 5566, 5466, 5844, 5580, 5280, 6042] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46924 payload_rows_remote=41814 dup_rows=18612 dup_rows_remote=16892 dedup_ratio=0.283997 router_tokens_per_expert=[123, 123, 47, 151, 3, 19, 1223, 1815, 301, 18, 157, 812, 0, 64, 1, 172, 294, 704, 1532, 0, 266, 255, 2, 362, 1079, 86, 354, 803, 5, 1618, 897, 0, 10, 4, 108, 2021, 10, 0, 6, 238, 1181, 479, 516, 6, 2840, 757, 1396, 940, 33, 1026, 1484, 432, 530, 160, 44, 167, 1325, 1250, 413, 545, 407, 129, 1025, 0] payload_rows_per_dst=[6270, 6858, 5110, 6104, 5970, 5548, 5544, 5520] dup_rows_per_dst=[3530, 1872, 1720, 2844, 2060, 2210, 2208, 2168] combine_rows_per_src=[6270, 6858, 5110, 6104, 5970, 5548, 5544, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46424 payload_rows_remote=41298 dup_rows=19112 dup_rows_remote=17086 dedup_ratio=0.291626 router_tokens_per_expert=[108, 54, 81, 79, 5, 34, 828, 1184, 286, 91, 309, 1752, 1, 85, 0, 306, 348, 497, 2041, 12, 580, 337, 3, 213, 732, 214, 507, 768, 2, 1255, 891, 17, 27, 13, 105, 1901, 5, 13, 3, 665, 977, 721, 610, 24, 2489, 504, 1048, 740, 31, 1403, 1128, 185, 673, 199, 71, 321, 1205, 1299, 395, 203, 338, 243, 1604, 5] payload_rows_per_dst=[5088, 7098, 5852, 4996, 5688, 5126, 5792, 6784] dup_rows_per_dst=[1754, 3540, 2210, 1266, 2286, 2026, 2230, 3800] combine_rows_per_src=[5088, 7098, 5852, 4996, 5688, 5126, 5792, 6784] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45000 payload_rows_remote=38950 dup_rows=20536 dup_rows_remote=17816 dedup_ratio=0.313354 router_tokens_per_expert=[122, 33, 55, 76, 11, 35, 1362, 1398, 383, 14, 152, 1263, 1, 89, 0, 207, 333, 538, 1655, 2, 503, 406, 8, 248, 982, 351, 278, 975, 3, 1086, 905, 1, 5, 10, 194, 2248, 0, 0, 24, 503, 1143, 437, 346, 4, 2414, 608, 1039, 1117, 75, 1458, 1060, 146, 1301, 439, 48, 289, 1136, 1051, 166, 202, 686, 147, 995, 2] payload_rows_per_dst=[5560, 3712, 5396, 4518, 6300, 7108, 6356, 6050] dup_rows_per_dst=[2450, 506, 1990, 822, 3490, 5282, 3276, 2720] combine_rows_per_src=[5560, 3712, 5396, 4518, 6300, 7108, 6356, 6050] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45754 payload_rows_remote=39326 dup_rows=19782 dup_rows_remote=16596 dedup_ratio=0.301849 router_tokens_per_expert=[188, 40, 170, 64, 76, 5, 1001, 1316, 439, 74, 106, 1132, 1, 186, 0, 162, 219, 847, 1869, 4, 260, 571, 2, 524, 1467, 564, 364, 740, 9, 2121, 1219, 10, 5, 1, 114, 1480, 4, 5, 37, 423, 370, 723, 452, 52, 2053, 631, 1217, 1271, 13, 1770, 1448, 165, 1295, 167, 43, 257, 555, 434, 218, 283, 481, 213, 838, 0] payload_rows_per_dst=[5536, 6150, 6014, 6428, 6042, 4926, 6442, 4216] dup_rows_per_dst=[2618, 2156, 2578, 3186, 2338, 2072, 3874, 960] combine_rows_per_src=[5536, 6150, 6014, 6428, 6042, 4926, 6442, 4216] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280584192 dispatch_meta_bytes=232000 combine_payload_bytes=280584192 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=280816192 ingress_bytes=390684672 link_bytes=390684672 token_bytes=561400384 weight_movement_bytes=110100480 total_nvlink_bytes=671500864 weight_to_token_ratio=0.196118 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=9984 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.794649012e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=293715968 dispatch_meta_bytes=230488 combine_payload_bytes=293715968 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=293946456 ingress_bytes=513916928 link_bytes=513916928 token_bytes=587662424 weight_movement_bytes=220200960 total_nvlink_bytes=807863384 weight_to_token_ratio=0.374707 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.036765 gemm_active_groups=11 gemm_peak_group_rows=13824 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.937860626e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=295092224 dispatch_meta_bytes=233512 combine_payload_bytes=295092224 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=295325736 ingress_bytes=735494144 link_bytes=735494144 token_bytes=590417960 weight_movement_bytes=440401920 total_nvlink_bytes=1030819880 weight_to_token_ratio=0.745916 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=6784 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.279145284e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=297758720 dispatch_meta_bytes=234920 combine_payload_bytes=297758720 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=297993640 ingress_bytes=407859200 link_bytes=407859200 token_bytes=595752360 weight_movement_bytes=110100480 total_nvlink_bytes=705852840 weight_to_token_ratio=0.184809 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=13312 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.093342301e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=282003456 dispatch_meta_bytes=227144 combine_payload_bytes=282003456 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=282230600 ingress_bytes=612304896 link_bytes=612304896 token_bytes=564234056 weight_movement_bytes=330301440 total_nvlink_bytes=894535496 weight_to_token_ratio=0.585398 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=8192 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.064898921e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=275953664 dispatch_meta_bytes=217896 combine_payload_bytes=275953664 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276171560 ingress_bytes=386054144 link_bytes=386054144 token_bytes=552125224 weight_movement_bytes=110100480 total_nvlink_bytes=662225704 weight_to_token_ratio=0.199412 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=15488 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.714116514e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279293952 dispatch_meta_bytes=223704 combine_payload_bytes=279293952 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279517656 ingress_bytes=609595392 link_bytes=609595392 token_bytes=558811608 weight_movement_bytes=330301440 total_nvlink_bytes=889113048 weight_to_token_ratio=0.591078 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=8192 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060186648e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=287580160 dispatch_meta_bytes=226352 combine_payload_bytes=287580160 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=287806512 ingress_bytes=397680640 link_bytes=397680640 token_bytes=575386672 weight_movement_bytes=110100480 total_nvlink_bytes=685487152 weight_to_token_ratio=0.191350 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=12928 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.916320402e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57298 payload_rows_remote=51710 dup_rows=8238 dup_rows_remote=7670 dedup_ratio=0.125702 router_tokens_per_expert=[0, 3509, 0, 892, 1, 4087, 59, 26, 0, 30, 15, 0, 0, 0, 2, 1, 0, 1136, 3406, 77, 0, 2843, 0, 3309, 36, 3891, 159, 0, 3, 0, 0, 0, 0, 4, 0, 0, 0, 2, 0, 223, 905, 2, 11, 0, 229, 2, 1, 0, 0, 8, 1, 3853, 501, 2560, 8, 0, 105, 7, 13, 0, 0, 1, 847, 3] payload_rows_per_dst=[6964, 6856, 6800, 7812, 8180, 8020, 5588, 7078] dup_rows_per_dst=[1522, 52, 1800, 366, 452, 1986, 568, 1492] combine_rows_per_src=[6964, 6856, 6800, 7812, 8180, 8020, 5588, 7078] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58782 payload_rows_remote=50594 dup_rows=6754 dup_rows_remote=6040 dedup_ratio=0.103058 router_tokens_per_expert=[1, 3552, 0, 636, 0, 4092, 45, 40, 0, 24, 7, 0, 0, 0, 0, 1, 0, 1521, 3641, 138, 0, 1690, 0, 3217, 0, 3937, 188, 2, 0, 1, 0, 0, 2, 0, 0, 0, 1, 0, 0, 356, 430, 0, 4, 0, 146, 0, 0, 0, 0, 0, 0, 3937, 683, 3291, 0, 0, 90, 167, 9, 0, 0, 0, 913, 6] payload_rows_per_dst=[7400, 7298, 5648, 7900, 8188, 7986, 7098, 7264] dup_rows_per_dst=[1148, 48, 1050, 356, 714, 1048, 850, 1540] combine_rows_per_src=[7400, 7298, 5648, 7900, 8188, 7986, 7098, 7264] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58234 payload_rows_remote=50488 dup_rows=7302 dup_rows_remote=6400 dedup_ratio=0.111420 router_tokens_per_expert=[0, 3301, 3, 714, 0, 4029, 40, 15, 0, 6, 2, 0, 25, 0, 0, 0, 0, 949, 3811, 65, 0, 2586, 0, 3066, 69, 3868, 444, 0, 0, 0, 8, 0, 0, 4, 0, 0, 0, 0, 0, 659, 324, 0, 6, 0, 202, 0, 0, 0, 0, 0, 0, 3792, 576, 3004, 0, 14, 71, 211, 49, 2, 12, 0, 837, 4] payload_rows_per_dst=[7022, 7636, 6148, 7866, 8104, 7746, 6572, 7140] dup_rows_per_dst=[1124, 52, 1052, 912, 1280, 902, 616, 1364] combine_rows_per_src=[7022, 7636, 6148, 7866, 8104, 7746, 6572, 7140] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58842 payload_rows_remote=52242 dup_rows=6694 dup_rows_remote=5486 dedup_ratio=0.102142 router_tokens_per_expert=[0, 3633, 5, 737, 0, 4092, 32, 40, 1, 2, 3, 0, 0, 0, 0, 0, 0, 841, 3797, 145, 0, 2913, 5, 3138, 9, 3578, 234, 0, 2, 1, 0, 0, 2, 3, 0, 0, 1, 1, 0, 237, 228, 2, 0, 0, 220, 0, 0, 0, 0, 0, 4, 3929, 544, 3117, 0, 0, 119, 45, 31, 0, 1, 0, 1064, 12] payload_rows_per_dst=[7522, 7596, 6600, 7218, 8192, 7958, 6670, 7086] dup_rows_per_dst=[1372, 10, 1208, 430, 480, 800, 660, 1734] combine_rows_per_src=[7522, 7596, 6600, 7218, 8192, 7958, 6670, 7086] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=287178752 dispatch_meta_bytes=231808 combine_payload_bytes=287178752 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=287410560 ingress_bytes=507379712 link_bytes=507379712 token_bytes=574589312 weight_movement_bytes=220200960 total_nvlink_bytes=794790272 weight_to_token_ratio=0.383232 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=9856 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.824167688e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46146 payload_rows_remote=41334 dup_rows=19390 dup_rows_remote=17276 dedup_ratio=0.295868 router_tokens_per_expert=[107, 42, 105, 76, 2, 37, 689, 1522, 148, 116, 188, 622, 1, 44, 38, 386, 262, 555, 1477, 61, 427, 726, 11, 367, 1410, 186, 350, 1166, 7, 1887, 1155, 12, 65, 21, 65, 1178, 23, 0, 11, 932, 595, 572, 507, 1, 3028, 533, 883, 948, 319, 1482, 1299, 311, 450, 290, 93, 344, 785, 1173, 424, 758, 427, 189, 824, 56] payload_rows_per_dst=[4812, 6898, 6306, 6746, 5878, 5006, 5634, 4866] dup_rows_per_dst=[2114, 2244, 3362, 4172, 2486, 1462, 1806, 1744] combine_rows_per_src=[4812, 6898, 6306, 6746, 5878, 5006, 5634, 4866] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=296024064 dispatch_meta_bytes=233536 combine_payload_bytes=296024064 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=296257600 ingress_bytes=736425984 link_bytes=736425984 token_bytes=592281664 weight_movement_bytes=440401920 total_nvlink_bytes=1032683584 weight_to_token_ratio=0.743568 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=6656 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.280765907e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=276211712 dispatch_meta_bytes=218560 combine_payload_bytes=276211712 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276430272 ingress_bytes=386312192 link_bytes=386312192 token_bytes=552641984 weight_movement_bytes=110100480 total_nvlink_bytes=662742464 weight_to_token_ratio=0.199226 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=15872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.718604393e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=295006208 dispatch_meta_bytes=230496 combine_payload_bytes=295006208 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=295236704 ingress_bytes=515207168 link_bytes=515207168 token_bytes=590242912 weight_movement_bytes=220200960 total_nvlink_bytes=810443872 weight_to_token_ratio=0.373068 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.040441 gemm_active_groups=11 gemm_peak_group_rows=13696 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.960300022e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=299722752 dispatch_meta_bytes=234824 combine_payload_bytes=299722752 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=299957576 ingress_bytes=409823232 link_bytes=409823232 token_bytes=599680328 weight_movement_bytes=110100480 total_nvlink_bytes=709780808 weight_to_token_ratio=0.183599 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=13440 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.127500048e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279193600 dispatch_meta_bytes=227064 combine_payload_bytes=279193600 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279420664 ingress_bytes=499394560 link_bytes=499394560 token_bytes=558614264 weight_movement_bytes=220200960 total_nvlink_bytes=778815224 weight_to_token_ratio=0.394191 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=8320 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.685292761e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=281888768 dispatch_meta_bytes=223688 combine_payload_bytes=281888768 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=282112456 ingress_bytes=612190208 link_bytes=612190208 token_bytes=564001224 weight_movement_bytes=330301440 total_nvlink_bytes=894302664 weight_to_token_ratio=0.585640 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=8832 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.064699460e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=296282112 dispatch_meta_bytes=234440 combine_payload_bytes=296282112 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=296516552 ingress_bytes=406382592 link_bytes=406382592 token_bytes=592798664 weight_movement_bytes=110100480 total_nvlink_bytes=702899144 weight_to_token_ratio=0.185730 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=12928 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.067661659e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57256 payload_rows_remote=51614 dup_rows=8280 dup_rows_remote=7708 dedup_ratio=0.126343 router_tokens_per_expert=[0, 3399, 0, 760, 1, 4086, 46, 33, 0, 30, 14, 0, 0, 0, 3, 2, 0, 1581, 3238, 70, 0, 2910, 0, 3157, 30, 3879, 181, 0, 3, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 153, 1069, 2, 11, 0, 161, 2, 0, 0, 0, 9, 1, 3828, 504, 2587, 6, 0, 203, 11, 13, 0, 0, 1, 778, 4] payload_rows_per_dst=[7154, 6524, 7100, 7784, 8176, 8008, 5642, 6868] dup_rows_per_dst=[1324, 50, 2022, 402, 306, 2138, 572, 1466] combine_rows_per_src=[7154, 6524, 7100, 7784, 8176, 8008, 5642, 6868] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58228 payload_rows_remote=50596 dup_rows=7308 dup_rows_remote=6430 dedup_ratio=0.111511 router_tokens_per_expert=[0, 3285, 3, 627, 0, 4034, 26, 22, 0, 5, 1, 0, 27, 0, 0, 0, 0, 1318, 3712, 55, 0, 2676, 0, 2942, 68, 3844, 467, 0, 0, 0, 8, 0, 0, 4, 0, 0, 0, 0, 0, 522, 369, 0, 4, 0, 164, 0, 0, 0, 0, 0, 0, 3718, 583, 3064, 0, 14, 129, 199, 53, 2, 11, 0, 808, 4] payload_rows_per_dst=[6964, 7440, 6644, 7842, 8106, 7632, 6678, 6922] dup_rows_per_dst=[962, 50, 1454, 932, 1014, 878, 644, 1374] combine_rows_per_src=[6964, 7440, 6644, 7842, 8106, 7632, 6678, 6922] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55532 payload_rows_remote=47694 dup_rows=10004 dup_rows_remote=5954 dedup_ratio=0.152649 router_tokens_per_expert=[0, 3480, 0, 413, 0, 4095, 58, 20, 0, 13, 2, 0, 0, 0, 0, 0, 0, 1452, 3496, 67, 0, 2824, 0, 2640, 4, 3974, 200, 0, 0, 0, 1, 0, 0, 9, 0, 0, 0, 0, 0, 486, 428, 0, 12, 1, 285, 0, 1, 0, 0, 0, 1, 3831, 894, 2957, 0, 0, 467, 160, 59, 0, 0, 0, 432, 6] payload_rows_per_dst=[7130, 7838, 3572, 7960, 8190, 7766, 6748, 6328] dup_rows_per_dst=[812, 4050, 248, 398, 990, 1350, 956, 1200] combine_rows_per_src=[7130, 7838, 3572, 7960, 8190, 7766, 6748, 6328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58692 payload_rows_remote=50504 dup_rows=6844 dup_rows_remote=6320 dedup_ratio=0.104431 router_tokens_per_expert=[1, 3503, 0, 572, 0, 4092, 51, 62, 0, 12, 5, 0, 0, 0, 0, 1, 0, 2013, 3477, 117, 0, 1714, 0, 3060, 0, 3913, 191, 2, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 262, 449, 0, 4, 0, 105, 0, 0, 0, 0, 0, 0, 3901, 758, 3308, 0, 0, 129, 156, 10, 0, 0, 0, 888, 8] payload_rows_per_dst=[7296, 6960, 6290, 7856, 8188, 7916, 7166, 7020] dup_rows_per_dst=[1082, 30, 1398, 360, 524, 1002, 966, 1482] combine_rows_per_src=[7296, 6960, 6290, 7856, 8188, 7916, 7166, 7020] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=369969152 dispatch_meta_bytes=237288 combine_payload_bytes=369969152 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=370206440 ingress_bytes=700270592 link_bytes=700270592 token_bytes=740175592 weight_movement_bytes=330301440 total_nvlink_bytes=1070477032 weight_to_token_ratio=0.446247 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=24320 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.217885734e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58654 payload_rows_remote=51666 dup_rows=6882 dup_rows_remote=5098 dedup_ratio=0.105011 router_tokens_per_expert=[0, 3610, 5, 671, 0, 4092, 28, 63, 1, 0, 2, 0, 0, 0, 0, 0, 0, 1272, 3647, 112, 0, 2997, 5, 2920, 8, 3592, 244, 0, 2, 1, 0, 0, 2, 3, 0, 0, 1, 1, 0, 153, 257, 2, 0, 0, 156, 0, 0, 0, 0, 0, 4, 3925, 601, 3119, 0, 0, 217, 48, 26, 0, 1, 0, 962, 18] payload_rows_per_dst=[7492, 7296, 6988, 7252, 8192, 7942, 6702, 6790] dup_rows_per_dst=[1262, 4, 1784, 442, 312, 738, 746, 1594] combine_rows_per_src=[7492, 7296, 6988, 7252, 8192, 7942, 6702, 6790] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51390 payload_rows_remote=44354 dup_rows=14146 dup_rows_remote=12266 dedup_ratio=0.215851 router_tokens_per_expert=[0, 3006, 0, 664, 0, 4077, 10, 59, 6, 0, 2, 0, 0, 2, 0, 0, 0, 1778, 3503, 62, 1, 3100, 0, 2927, 7, 3985, 259, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 0, 123, 175, 3, 2, 0, 159, 0, 0, 0, 0, 0, 0, 3908, 417, 2998, 0, 0, 859, 46, 7, 0, 0, 0, 610, 9] payload_rows_per_dst=[3862, 7020, 7928, 7754, 8178, 1948, 7664, 7036] dup_rows_per_dst=[420, 6, 5150, 508, 230, 164, 5788, 1880] combine_rows_per_src=[3862, 7020, 7928, 7754, 8178, 1948, 7664, 7036] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57244 payload_rows_remote=49396 dup_rows=8292 dup_rows_remote=7662 dedup_ratio=0.126526 router_tokens_per_expert=[0, 3189, 0, 579, 0, 4082, 46, 7, 0, 0, 31, 0, 0, 0, 0, 0, 0, 2115, 3311, 115, 0, 2575, 0, 2985, 26, 3879, 333, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 341, 710, 0, 1, 0, 579, 0, 0, 0, 0, 2, 0, 3403, 709, 3128, 0, 0, 98, 153, 11, 0, 0, 0, 359, 0] payload_rows_per_dst=[6688, 6642, 7114, 7848, 8166, 7398, 6864, 6524] dup_rows_per_dst=[954, 42, 2496, 630, 680, 1988, 814, 688] combine_rows_per_src=[6688, 6642, 7114, 7848, 8166, 7398, 6864, 6524] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57674 payload_rows_remote=49752 dup_rows=7862 dup_rows_remote=7244 dedup_ratio=0.119965 router_tokens_per_expert=[0, 3205, 0, 616, 0, 4079, 33, 4, 0, 1, 30, 0, 0, 0, 0, 0, 0, 1693, 3499, 133, 0, 2460, 0, 3146, 29, 3932, 308, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 405, 654, 0, 1, 0, 682, 0, 0, 0, 0, 1, 0, 3501, 655, 3115, 0, 0, 54, 148, 11, 0, 0, 0, 372, 0] payload_rows_per_dst=[6752, 7018, 6664, 7922, 8164, 7552, 6804, 6798] dup_rows_per_dst=[964, 42, 1908, 618, 804, 2124, 738, 664] combine_rows_per_src=[6752, 7018, 6664, 7922, 8164, 7552, 6804, 6798] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56696 payload_rows_remote=49136 dup_rows=8840 dup_rows_remote=6618 dedup_ratio=0.134888 router_tokens_per_expert=[0, 3524, 0, 475, 0, 4095, 63, 12, 0, 16, 3, 0, 0, 0, 0, 0, 0, 1109, 3615, 77, 0, 2694, 0, 2961, 6, 3980, 190, 0, 0, 0, 1, 0, 0, 14, 0, 0, 0, 0, 0, 581, 386, 0, 14, 1, 347, 0, 1, 0, 0, 0, 1, 3843, 843, 2932, 0, 0, 314, 160, 59, 0, 0, 0, 450, 1] payload_rows_per_dst=[7244, 7560, 4580, 7966, 8190, 7774, 6674, 6708] dup_rows_per_dst=[904, 2222, 666, 388, 1190, 1410, 878, 1182] combine_rows_per_src=[7244, 7560, 4580, 7966, 8190, 7774, 6674, 6708] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49060 payload_rows_remote=41926 dup_rows=16476 dup_rows_remote=14776 dedup_ratio=0.251404 router_tokens_per_expert=[0, 3151, 0, 735, 0, 4077, 18, 42, 6, 1, 2, 0, 0, 2, 0, 0, 0, 1342, 3658, 71, 1, 3084, 0, 3112, 7, 3988, 257, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 166, 152, 3, 2, 0, 219, 0, 0, 0, 1, 0, 0, 3959, 423, 2982, 0, 0, 601, 56, 7, 0, 0, 0, 636, 5] payload_rows_per_dst=[1580, 7330, 8068, 7630, 8174, 1192, 7952, 7134] dup_rows_per_dst=[10, 8, 7230, 490, 316, 142, 6580, 1700] combine_rows_per_src=[1580, 7330, 8068, 7630, 8174, 1192, 7952, 7134] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=370657280 dispatch_meta_bytes=237520 combine_payload_bytes=370657280 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=370894800 ingress_bytes=700958720 link_bytes=700958720 token_bytes=741552080 weight_movement_bytes=330301440 total_nvlink_bytes=1071853520 weight_to_token_ratio=0.445419 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=24064 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.219082502e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=362657792 dispatch_meta_bytes=226536 combine_payload_bytes=362657792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=362884328 ingress_bytes=472758272 link_bytes=472758272 token_bytes=725542120 weight_movement_bytes=110100480 total_nvlink_bytes=835642600 weight_to_token_ratio=0.151749 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=29568 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.222043904e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=361897984 dispatch_meta_bytes=227552 combine_payload_bytes=361897984 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=362125536 ingress_bytes=471998464 link_bytes=471998464 token_bytes=724023520 weight_movement_bytes=110100480 total_nvlink_bytes=834124000 weight_to_token_ratio=0.152068 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=26624 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.208829593e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49954 payload_rows_remote=42212 dup_rows=15582 dup_rows_remote=9316 dedup_ratio=0.237762 router_tokens_per_expert=[0, 2723, 0, 618, 20, 3796, 102, 131, 1, 4, 12, 0, 47, 0, 1, 2, 1, 1686, 3404, 145, 41, 2433, 0, 2849, 26, 3505, 279, 4, 4, 0, 13, 0, 0, 68, 0, 1, 0, 3, 0, 507, 340, 5, 8, 5, 698, 1, 0, 14, 2, 4, 2, 3377, 684, 3536, 0, 5, 618, 189, 6, 1, 1, 11, 696, 139] payload_rows_per_dst=[7742, 7848, 4944, 7210, 2864, 7376, 6474, 5496] dup_rows_per_dst=[6266, 3960, 1170, 452, 244, 1520, 814, 1156] combine_rows_per_src=[7742, 7848, 4944, 7210, 2864, 7376, 6474, 5496] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=374470656 dispatch_meta_bytes=230912 combine_payload_bytes=374470656 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=374701568 ingress_bytes=814872576 link_bytes=814872576 token_bytes=749172224 weight_movement_bytes=440401920 total_nvlink_bytes=1189574144 weight_to_token_ratio=0.587851 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=17024 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.417197433e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=356622336 dispatch_meta_bytes=227984 combine_payload_bytes=356622336 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=356850320 ingress_bytes=466722816 link_bytes=466722816 token_bytes=713472656 weight_movement_bytes=110100480 total_nvlink_bytes=823573136 weight_to_token_ratio=0.154316 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=30592 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.117077397e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=352206848 dispatch_meta_bytes=223016 combine_payload_bytes=352206848 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=352429864 ingress_bytes=572407808 link_bytes=572407808 token_bytes=704636712 weight_movement_bytes=220200960 total_nvlink_bytes=924837672 weight_to_token_ratio=0.312503 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=28928 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.955113230e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=300525568 dispatch_meta_bytes=226808 combine_payload_bytes=300525568 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=300752376 ingress_bytes=410626048 link_bytes=410626048 token_bytes=601277944 weight_movement_bytes=110100480 total_nvlink_bytes=711378424 weight_to_token_ratio=0.183111 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=23680 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.141462339e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51426 payload_rows_remote=43928 dup_rows=14110 dup_rows_remote=9474 dedup_ratio=0.215302 router_tokens_per_expert=[0, 2661, 0, 524, 18, 3790, 91, 177, 1, 3, 13, 0, 46, 0, 1, 2, 1, 2052, 3267, 117, 39, 2459, 0, 2742, 23, 3459, 280, 4, 4, 1, 13, 0, 0, 64, 0, 1, 0, 3, 0, 406, 396, 5, 7, 5, 598, 1, 0, 15, 2, 6, 3, 3307, 730, 3521, 0, 5, 884, 185, 5, 1, 0, 13, 654, 163] payload_rows_per_dst=[7498, 7760, 4692, 7118, 3976, 7244, 6668, 6470] dup_rows_per_dst=[4636, 3824, 656, 450, 308, 1424, 918, 1894] combine_rows_per_src=[7498, 7760, 4692, 7118, 3976, 7244, 6668, 6470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=362672128 dispatch_meta_bytes=228104 combine_payload_bytes=362672128 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=362900232 ingress_bytes=472772608 link_bytes=472772608 token_bytes=725572360 weight_movement_bytes=110100480 total_nvlink_bytes=835672840 weight_to_token_ratio=0.151743 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.029412 gemm_active_groups=10 gemm_peak_group_rows=26752 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.222293230e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=370341888 dispatch_meta_bytes=227056 combine_payload_bytes=370341888 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=370568944 ingress_bytes=810743808 link_bytes=810743808 token_bytes=740910832 weight_movement_bytes=440401920 total_nvlink_bytes=1181312752 weight_to_token_ratio=0.594406 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=16384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.410016827e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=341870592 dispatch_meta_bytes=214592 combine_payload_bytes=341870592 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=342085184 ingress_bytes=562071552 link_bytes=562071552 token_bytes=683955776 weight_movement_bytes=220200960 total_nvlink_bytes=904156736 weight_to_token_ratio=0.321952 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=27776 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.775348738e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=362012672 dispatch_meta_bytes=227296 combine_payload_bytes=362012672 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=362239968 ingress_bytes=472113152 link_bytes=472113152 token_bytes=724252640 weight_movement_bytes=110100480 total_nvlink_bytes=834353120 weight_to_token_ratio=0.152019 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=30336 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.210824206e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=317929472 dispatch_meta_bytes=226480 combine_payload_bytes=317929472 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=318155952 ingress_bytes=428029952 link_bytes=428029952 token_bytes=636085424 weight_movement_bytes=110100480 total_nvlink_bytes=746185904 weight_to_token_ratio=0.173091 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=22912 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.444144854e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=302575616 dispatch_meta_bytes=206112 combine_payload_bytes=302575616 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=302781728 ingress_bytes=632877056 link_bytes=632877056 token_bytes=605357344 weight_movement_bytes=330301440 total_nvlink_bytes=935658784 weight_to_token_ratio=0.545631 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=23296 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.100677291e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54098 payload_rows_remote=46636 dup_rows=11438 dup_rows_remote=10456 dedup_ratio=0.174530 router_tokens_per_expert=[1, 0, 0, 1, 171, 0, 0, 0, 13, 4, 0, 4091, 3865, 3990, 456, 0, 0, 1, 0, 0, 4053, 0, 1, 0, 0, 1493, 0, 0, 1, 0, 1, 0, 1, 410, 1, 3838, 0, 0, 18, 0, 0, 15, 7, 0, 3709, 0, 2040, 1, 0, 0, 0, 0, 3678, 0, 544, 0, 2, 0, 0, 0, 328, 3, 0, 31] payload_rows_per_dst=[8186, 946, 8110, 5408, 7798, 8174, 7462, 8014] dup_rows_per_dst=[342, 0, 0, 1276, 738, 7406, 982, 694] combine_rows_per_src=[8186, 946, 8110, 5408, 7798, 8174, 7462, 8014] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56716 payload_rows_remote=48944 dup_rows=8820 dup_rows_remote=6648 dedup_ratio=0.134583 router_tokens_per_expert=[0, 0, 0, 13, 545, 0, 0, 0, 1, 1, 0, 4096, 3849, 3868, 662, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 1275, 0, 1, 0, 0, 0, 0, 0, 728, 0, 3753, 0, 6, 95, 15, 0, 52, 0, 0, 3638, 0, 1343, 0, 0, 0, 0, 0, 3817, 0, 570, 0, 0, 0, 28, 0, 337, 0, 0, 0] payload_rows_per_dst=[8192, 1794, 8150, 7536, 7766, 7772, 7706, 7800] dup_rows_per_dst=[1116, 78, 0, 2292, 1428, 2172, 1068, 666] combine_rows_per_src=[8192, 1794, 8150, 7536, 7766, 7772, 7706, 7800] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59962 payload_rows_remote=52484 dup_rows=5574 dup_rows_remote=5146 dedup_ratio=0.085052 router_tokens_per_expert=[0, 2, 0, 0, 97, 0, 0, 0, 3, 2, 0, 4095, 3834, 4025, 391, 0, 0, 0, 0, 0, 4044, 0, 0, 0, 0, 984, 1, 0, 0, 0, 0, 0, 0, 272, 0, 3669, 0, 2, 8, 2, 0, 2, 0, 1, 3869, 0, 2063, 0, 0, 0, 0, 1, 3930, 0, 521, 0, 0, 0, 2, 0, 947, 0, 0, 1] payload_rows_per_dst=[8192, 8094, 8088, 7962, 7478, 4132, 7878, 8138] dup_rows_per_dst=[196, 366, 0, 1746, 428, 0, 1026, 1812] combine_rows_per_src=[8192, 8094, 8088, 7962, 7478, 4132, 7878, 8138] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=354070528 dispatch_meta_bytes=228232 combine_payload_bytes=354070528 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=354298760 ingress_bytes=464171008 link_bytes=464171008 token_bytes=708369288 weight_movement_bytes=110100480 total_nvlink_bytes=818469768 weight_to_token_ratio=0.155428 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=30464 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.072697259e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52746 payload_rows_remote=45188 dup_rows=12790 dup_rows_remote=11706 dedup_ratio=0.195160 router_tokens_per_expert=[1, 0, 0, 1, 199, 0, 0, 0, 11, 3, 0, 4091, 3870, 3988, 355, 0, 0, 1, 0, 0, 4057, 0, 1, 0, 0, 1659, 0, 0, 1, 0, 0, 0, 1, 785, 1, 3770, 0, 0, 22, 0, 0, 15, 7, 0, 3747, 0, 1359, 1, 0, 0, 1, 0, 3731, 0, 589, 0, 2, 0, 0, 0, 467, 3, 0, 29] payload_rows_per_dst=[8186, 738, 8118, 4160, 7794, 8178, 7558, 8014] dup_rows_per_dst=[398, 0, 0, 582, 1364, 8398, 1084, 964] combine_rows_per_src=[8186, 738, 8118, 4160, 7794, 8178, 7558, 8014] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=314875904 dispatch_meta_bytes=213608 combine_payload_bytes=314875904 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=315089512 ingress_bytes=645177344 link_bytes=645177344 token_bytes=629965416 weight_movement_bytes=330301440 total_nvlink_bytes=960266856 weight_to_token_ratio=0.524317 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=24576 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.122069515e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=323907584 dispatch_meta_bytes=227576 combine_payload_bytes=323907584 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=324135160 ingress_bytes=544108544 link_bytes=544108544 token_bytes=648042744 weight_movement_bytes=220200960 total_nvlink_bytes=868243704 weight_to_token_ratio=0.339794 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=27520 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.462942485e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56188 payload_rows_remote=48090 dup_rows=9348 dup_rows_remote=7812 dedup_ratio=0.142639 router_tokens_per_expert=[0, 0, 0, 12, 537, 0, 0, 0, 0, 0, 0, 4096, 3850, 3882, 415, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 1342, 0, 0, 0, 0, 0, 0, 0, 1214, 0, 3633, 0, 6, 132, 14, 0, 53, 0, 0, 3659, 0, 914, 0, 0, 0, 0, 0, 3816, 0, 593, 0, 0, 0, 26, 0, 495, 0, 0, 0] payload_rows_per_dst=[8192, 830, 8158, 7554, 7820, 8098, 7704, 7832] dup_rows_per_dst=[1098, 0, 0, 2448, 2178, 1536, 1114, 974] combine_rows_per_src=[8192, 830, 8158, 7554, 7820, 8098, 7704, 7832] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52426 payload_rows_remote=44234 dup_rows=13110 dup_rows_remote=5386 dedup_ratio=0.200043 router_tokens_per_expert=[0, 0, 0, 1, 315, 0, 0, 0, 0, 0, 0, 4095, 3911, 3881, 348, 0, 0, 0, 0, 0, 3993, 0, 0, 0, 0, 1058, 0, 3, 0, 0, 17, 0, 1, 993, 0, 3699, 0, 0, 13, 0, 0, 0, 0, 0, 3623, 0, 1351, 0, 0, 0, 0, 0, 3928, 0, 700, 0, 0, 0, 0, 0, 834, 0, 0, 4] payload_rows_per_dst=[8192, 8192, 7986, 7532, 2010, 2702, 7876, 7936] dup_rows_per_dst=[630, 7724, 0, 1870, 4, 0, 1380, 1502] combine_rows_per_src=[8192, 8192, 7986, 7532, 2010, 2702, 7876, 7936] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57362 payload_rows_remote=49186 dup_rows=8174 dup_rows_remote=8172 dedup_ratio=0.124725 router_tokens_per_expert=[1, 1, 1, 0, 157, 0, 0, 1, 0, 0, 0, 4093, 3855, 4058, 317, 3, 0, 4, 0, 0, 4084, 1, 0, 0, 4, 1057, 0, 0, 1, 0, 0, 1, 0, 787, 0, 3770, 0, 0, 19, 0, 1, 9, 0, 1, 3952, 0, 1010, 0, 3, 1, 0, 0, 3959, 0, 837, 0, 1, 2, 0, 0, 758, 0, 0, 19] payload_rows_per_dst=[8190, 8106, 8176, 7962, 6790, 2036, 7954, 8148] dup_rows_per_dst=[318, 1328, 2, 2068, 1278, 6, 1646, 1528] combine_rows_per_src=[8190, 8106, 8176, 7962, 6790, 2036, 7954, 8148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56674 payload_rows_remote=49312 dup_rows=8862 dup_rows_remote=8014 dedup_ratio=0.135223 router_tokens_per_expert=[0, 2, 0, 0, 115, 0, 0, 0, 0, 1, 0, 4094, 3845, 4044, 323, 0, 0, 0, 0, 1, 4047, 0, 0, 0, 0, 1191, 1, 0, 0, 0, 1, 0, 0, 549, 0, 3543, 0, 2, 9, 2, 0, 2, 0, 1, 3899, 0, 1419, 0, 0, 0, 0, 1, 3920, 0, 546, 0, 0, 0, 2, 0, 1206, 0, 0, 2] payload_rows_per_dst=[8192, 1758, 8096, 8002, 7362, 7254, 7862, 8148] dup_rows_per_dst=[230, 28, 0, 2182, 848, 2142, 1072, 2360] combine_rows_per_src=[8192, 1758, 8096, 8002, 7362, 7254, 7862, 8148] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=334286848 dispatch_meta_bytes=228368 combine_payload_bytes=334286848 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=334515216 ingress_bytes=554487808 link_bytes=554487808 token_bytes=668802064 weight_movement_bytes=220200960 total_nvlink_bytes=889003024 weight_to_token_ratio=0.329247 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=26880 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.643454957e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57346 payload_rows_remote=50246 dup_rows=8190 dup_rows_remote=6192 dedup_ratio=0.124969 router_tokens_per_expert=[0, 0, 0, 1, 323, 0, 0, 0, 0, 0, 0, 4096, 3902, 3868, 1097, 0, 0, 0, 0, 0, 4031, 0, 0, 0, 0, 1302, 0, 0, 0, 0, 2, 0, 0, 1284, 0, 3802, 0, 0, 6, 0, 0, 0, 0, 0, 3245, 0, 1142, 0, 0, 0, 0, 0, 3963, 0, 441, 0, 0, 0, 0, 0, 262, 0, 0, 1] payload_rows_per_dst=[8192, 8138, 8062, 7100, 7874, 2284, 7936, 7760] dup_rows_per_dst=[648, 1860, 0, 1998, 2310, 0, 872, 502] combine_rows_per_src=[8192, 8138, 8062, 7100, 7874, 2284, 7936, 7760] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59430 payload_rows_remote=51256 dup_rows=6106 dup_rows_remote=6104 dedup_ratio=0.093170 router_tokens_per_expert=[0, 1, 1, 0, 154, 0, 0, 0, 1, 0, 0, 4093, 3853, 4050, 402, 3, 0, 4, 0, 0, 4083, 1, 0, 0, 4, 941, 0, 0, 1, 0, 0, 1, 0, 410, 0, 3831, 0, 1, 14, 0, 1, 10, 0, 1, 3934, 0, 1580, 0, 2, 1, 0, 0, 3984, 0, 725, 0, 1, 1, 0, 2, 660, 0, 0, 17] payload_rows_per_dst=[8190, 8058, 8174, 7976, 7714, 3174, 8002, 8142] dup_rows_per_dst=[308, 460, 2, 1786, 798, 10, 1422, 1320] combine_rows_per_src=[8190, 8058, 8174, 7976, 7714, 3174, 8002, 8142] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=350830592 dispatch_meta_bytes=222368 combine_payload_bytes=350830592 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=351052960 ingress_bytes=571031552 link_bytes=571031552 token_bytes=701883552 weight_movement_bytes=220200960 total_nvlink_bytes=922084512 weight_to_token_ratio=0.313729 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=15616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.931177875e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48910 payload_rows_remote=40934 dup_rows=16626 dup_rows_remote=16104 dedup_ratio=0.253693 router_tokens_per_expert=[0, 1, 0, 0, 82, 3, 0, 1, 0, 2, 1, 4094, 3864, 3981, 347, 0, 0, 0, 0, 0, 4070, 2, 2, 0, 0, 1018, 0, 25, 1, 2, 0, 0, 0, 146, 0, 3988, 0, 0, 24, 6, 1, 0, 1, 0, 3973, 0, 2365, 0, 0, 0, 0, 0, 4037, 3, 460, 0, 0, 0, 0, 0, 268, 0, 0, 0] payload_rows_per_dst=[4570, 4022, 8144, 2058, 5834, 8190, 8116, 7976] dup_rows_per_dst=[104, 366, 4, 34, 120, 12218, 3258, 522] combine_rows_per_src=[4570, 4022, 8144, 2058, 5834, 8190, 8116, 7976] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58368 payload_rows_remote=50232 dup_rows=7168 dup_rows_remote=5356 dedup_ratio=0.109375 router_tokens_per_expert=[0, 0, 0, 1, 294, 0, 0, 0, 0, 0, 0, 4095, 3901, 3858, 460, 0, 0, 0, 0, 0, 3967, 0, 1, 0, 0, 956, 0, 3, 0, 0, 21, 0, 1, 486, 0, 3781, 0, 0, 8, 0, 0, 2, 0, 0, 3580, 0, 2099, 0, 0, 0, 0, 0, 3967, 0, 647, 0, 0, 0, 0, 0, 635, 0, 0, 5] payload_rows_per_dst=[8192, 8136, 7936, 7480, 7686, 4198, 6858, 7882] dup_rows_per_dst=[588, 1812, 0, 1640, 866, 4, 1144, 1114] combine_rows_per_src=[8192, 8136, 7936, 7480, 7686, 4198, 6858, 7882] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=376205312 dispatch_meta_bytes=230520 combine_payload_bytes=376205312 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=376435832 ingress_bytes=596406272 link_bytes=596406272 token_bytes=752641144 weight_movement_bytes=220200960 total_nvlink_bytes=972842104 weight_to_token_ratio=0.292571 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=28416 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.037248599e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51446 payload_rows_remote=43426 dup_rows=14090 dup_rows_remote=13470 dedup_ratio=0.214996 router_tokens_per_expert=[0, 1, 1, 0, 92, 3, 1, 1, 0, 2, 1, 4094, 3874, 4002, 298, 0, 0, 0, 0, 0, 4073, 1, 1, 0, 0, 1254, 0, 22, 1, 2, 0, 0, 0, 393, 0, 3957, 0, 0, 31, 6, 1, 0, 1, 0, 3983, 0, 1690, 0, 0, 0, 0, 0, 4033, 3, 628, 0, 0, 0, 0, 0, 318, 0, 0, 0] payload_rows_per_dst=[4344, 4204, 8148, 2522, 7942, 8190, 8076, 8020] dup_rows_per_dst=[120, 320, 2, 36, 832, 10908, 1252, 620] combine_rows_per_src=[4344, 4204, 8148, 2522, 7942, 8190, 8076, 8020] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56888 payload_rows_remote=49582 dup_rows=8648 dup_rows_remote=6872 dedup_ratio=0.131958 router_tokens_per_expert=[0, 0, 0, 1, 351, 0, 0, 0, 0, 0, 0, 4096, 3903, 3894, 667, 0, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 1154, 0, 0, 0, 0, 1, 0, 0, 2038, 0, 3730, 0, 0, 7, 0, 0, 0, 0, 0, 3386, 0, 684, 0, 0, 0, 0, 0, 3997, 0, 459, 0, 0, 0, 0, 0, 347, 0, 0, 3] payload_rows_per_dst=[8192, 8132, 8100, 7306, 7966, 1368, 8012, 7812] dup_rows_per_dst=[704, 1008, 0, 1776, 3584, 0, 900, 676] combine_rows_per_src=[8192, 8132, 8100, 7306, 7966, 1368, 8012, 7812] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=344709120 dispatch_meta_bytes=223608 combine_payload_bytes=344709120 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=344932728 ingress_bytes=564910080 link_bytes=564910080 token_bytes=689641848 weight_movement_bytes=220200960 total_nvlink_bytes=909842808 weight_to_token_ratio=0.319298 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=14976 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.824715409e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46214 payload_rows_remote=38052 dup_rows=19322 dup_rows_remote=18934 dedup_ratio=0.294830 router_tokens_per_expert=[0, 0, 0, 48, 138, 47, 0, 0, 166, 7, 0, 4042, 3715, 4018, 285, 0, 0, 0, 0, 0, 3756, 2, 2, 2, 0, 1107, 3, 4, 0, 0, 64, 0, 0, 686, 2, 3522, 0, 0, 13, 58, 3, 14, 0, 0, 3590, 0, 2343, 0, 0, 0, 17, 0, 3797, 0, 945, 0, 0, 0, 3, 19, 322, 27, 1, 0] payload_rows_per_dst=[8162, 8190, 7900, 7736, 6570, 4702, 2008, 946] dup_rows_per_dst=[388, 14978, 850, 1800, 1194, 18, 38, 56] combine_rows_per_src=[8162, 8190, 7900, 7736, 6570, 4702, 2008, 946] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48834 payload_rows_remote=40676 dup_rows=16702 dup_rows_remote=16340 dedup_ratio=0.254852 router_tokens_per_expert=[0, 0, 0, 50, 125, 44, 1, 0, 178, 6, 0, 4040, 3713, 4018, 334, 0, 0, 0, 0, 0, 3742, 1, 2, 2, 0, 993, 4, 4, 0, 0, 64, 0, 0, 345, 2, 3620, 0, 0, 10, 58, 3, 14, 0, 0, 3557, 0, 2884, 0, 0, 0, 17, 0, 3788, 0, 886, 0, 0, 0, 3, 15, 215, 29, 1, 0] payload_rows_per_dst=[8158, 8192, 7952, 7734, 6224, 5784, 1790, 3000] dup_rows_per_dst=[362, 13214, 914, 1510, 474, 18, 16, 194] combine_rows_per_src=[8158, 8192, 7952, 7734, 6224, 5784, 1790, 3000] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=360163328 dispatch_meta_bytes=225752 combine_payload_bytes=360163328 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=360389080 ingress_bytes=470263808 link_bytes=470263808 token_bytes=720552408 weight_movement_bytes=110100480 total_nvlink_bytes=830652888 weight_to_token_ratio=0.152800 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=23680 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.178661072e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=367403008 dispatch_meta_bytes=229440 combine_payload_bytes=367403008 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=367632448 ingress_bytes=477503488 link_bytes=477503488 token_bytes=735035456 weight_movement_bytes=110100480 total_nvlink_bytes=845135936 weight_to_token_ratio=0.149789 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.304571015e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=293414912 dispatch_meta_bytes=228152 combine_payload_bytes=293414912 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=293643064 ingress_bytes=403515392 link_bytes=403515392 token_bytes=587057976 weight_movement_bytes=110100480 total_nvlink_bytes=697158456 weight_to_token_ratio=0.187546 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=29056 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.017796336e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=360062976 dispatch_meta_bytes=222352 combine_payload_bytes=360062976 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=360285328 ingress_bytes=800464896 link_bytes=800464896 token_bytes=720348304 weight_movement_bytes=440401920 total_nvlink_bytes=1160750224 weight_to_token_ratio=0.611374 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=19584 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.392140108e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=317069312 dispatch_meta_bytes=198480 combine_payload_bytes=317069312 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=317267792 ingress_bytes=757471232 link_bytes=757471232 token_bytes=634337104 weight_movement_bytes=440401920 total_nvlink_bytes=1074739024 weight_to_token_ratio=0.694271 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=16000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.317367055e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=352565248 dispatch_meta_bytes=229432 combine_payload_bytes=352565248 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=352794680 ingress_bytes=462665728 link_bytes=462665728 token_bytes=705359928 weight_movement_bytes=110100480 total_nvlink_bytes=815460408 weight_to_token_ratio=0.156091 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.046517964e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=353468416 dispatch_meta_bytes=229304 combine_payload_bytes=353468416 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=353697720 ingress_bytes=573669376 link_bytes=573669376 token_bytes=707166136 weight_movement_bytes=220200960 total_nvlink_bytes=927367096 weight_to_token_ratio=0.311385 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=21888 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.977053973e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=291565568 dispatch_meta_bytes=228064 combine_payload_bytes=291565568 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=291793632 ingress_bytes=401666048 link_bytes=401666048 token_bytes=583359200 weight_movement_bytes=110100480 total_nvlink_bytes=693459680 weight_to_token_ratio=0.188735 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=30976 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.985633202e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63720 payload_rows_remote=55528 dup_rows=1816 dup_rows_remote=1810 dedup_ratio=0.027710 router_tokens_per_expert=[1, 3, 0, 1, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4083, 3207, 0, 4094, 1, 0, 2, 7, 4095, 0, 4095, 42, 0, 0, 0, 2, 2, 0, 0, 0, 0, 882, 4054, 1, 0, 4095, 0, 0, 0, 2, 1, 0, 1, 0, 4093, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8168, 6416, 8192, 8186, 8192, 8186] dup_rows_per_dst=[10, 4, 0, 18, 90, 1688, 6, 0] combine_rows_per_src=[8192, 8188, 8168, 6416, 8192, 8186, 8192, 8186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63496 payload_rows_remote=55310 dup_rows=2040 dup_rows_remote=132 dedup_ratio=0.031128 router_tokens_per_expert=[0, 5, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 3132, 0, 4096, 0, 0, 0, 6, 4096, 0, 4079, 14, 0, 0, 7, 0, 0, 0, 0, 2, 0, 1022, 4023, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8138, 6264, 8170, 8186, 8192, 8162] dup_rows_per_dst=[10, 80, 0, 12, 30, 1908, 0, 0] combine_rows_per_src=[8192, 8192, 8138, 6264, 8170, 8186, 8192, 8162] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64764 payload_rows_remote=56572 dup_rows=772 dup_rows_remote=670 dedup_ratio=0.011780 router_tokens_per_expert=[0, 11, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3713, 0, 4094, 1, 0, 0, 0, 4095, 0, 4095, 51, 0, 0, 1, 0, 0, 2, 0, 0, 0, 412, 4005, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8188, 7426, 8192, 8192, 8192, 8192] dup_rows_per_dst=[22, 0, 0, 2, 102, 646, 0, 0] combine_rows_per_src=[8192, 8190, 8188, 7426, 8192, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=311277568 dispatch_meta_bytes=227584 combine_payload_bytes=311277568 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=311505152 ingress_bytes=421378048 link_bytes=421378048 token_bytes=622782720 weight_movement_bytes=110100480 total_nvlink_bytes=732883200 weight_to_token_ratio=0.176788 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=27904 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.328457303e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63278 payload_rows_remote=55086 dup_rows=2258 dup_rows_remote=2252 dedup_ratio=0.034454 router_tokens_per_expert=[1, 2, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 152, 0, 0, 0, 0, 1, 4083, 2984, 0, 4094, 1, 0, 1, 9, 4095, 0, 4095, 40, 0, 0, 0, 2, 2, 0, 0, 0, 0, 927, 4083, 1, 0, 4095, 0, 0, 0, 2, 1, 0, 1, 0, 4093, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8168, 5972, 8192, 8188, 8192, 8186] dup_rows_per_dst=[6, 4, 304, 18, 86, 1834, 6, 0] combine_rows_per_src=[8192, 8188, 8168, 5972, 8192, 8188, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=355403776 dispatch_meta_bytes=225816 combine_payload_bytes=355403776 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=355629592 ingress_bytes=465504256 link_bytes=465504256 token_bytes=711033368 weight_movement_bytes=110100480 total_nvlink_bytes=821133848 weight_to_token_ratio=0.154846 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=22912 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.095884635e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=272756736 dispatch_meta_bytes=227944 combine_payload_bytes=272756736 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=272984680 ingress_bytes=382857216 link_bytes=382857216 token_bytes=545741416 weight_movement_bytes=110100480 total_nvlink_bytes=655841896 weight_to_token_ratio=0.201745 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=30848 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.658516678e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=396462080 dispatch_meta_bytes=221768 combine_payload_bytes=396462080 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=396683848 ingress_bytes=616663040 link_bytes=616663040 token_bytes=793145928 weight_movement_bytes=220200960 total_nvlink_bytes=1013346888 weight_to_token_ratio=0.277630 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=27008 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.072478450e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=398024704 dispatch_meta_bytes=229352 combine_payload_bytes=398024704 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=398254056 ingress_bytes=508125184 link_bytes=508125184 token_bytes=796278760 weight_movement_bytes=110100480 total_nvlink_bytes=906379240 weight_to_token_ratio=0.138269 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.837132672e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64560 payload_rows_remote=57324 dup_rows=976 dup_rows_remote=976 dedup_ratio=0.014893 router_tokens_per_expert=[0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 3618, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 3999, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8172, 7236, 8192, 8192, 8192, 8192] dup_rows_per_dst=[18, 0, 0, 0, 48, 910, 0, 0] combine_rows_per_src=[8192, 8192, 8172, 7236, 8192, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62690 payload_rows_remote=54534 dup_rows=2846 dup_rows_remote=2536 dedup_ratio=0.043427 router_tokens_per_expert=[0, 62, 0, 0, 0, 0, 0, 1, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4080, 3411, 0, 4096, 1, 1, 0, 0, 4094, 0, 4094, 48, 0, 0, 0, 2, 0, 1, 0, 0, 0, 647, 4010, 1, 0, 4093, 0, 0, 0, 0, 0, 0, 1, 0, 4074, 4, 0, 1, 0, 0, 1] payload_rows_per_dst=[7664, 7644, 8164, 6910, 7820, 8146, 8186, 8156] dup_rows_per_dst=[118, 86, 0, 996, 94, 1240, 2, 310] combine_rows_per_src=[7664, 7644, 8164, 6910, 7820, 8146, 8186, 8156] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64510 payload_rows_remote=56326 dup_rows=1026 dup_rows_remote=1026 dedup_ratio=0.015656 router_tokens_per_expert=[0, 35, 0, 2, 3, 0, 0, 0, 19, 1, 0, 3, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4091, 3615, 0, 4092, 0, 0, 1, 1, 4094, 0, 4091, 43, 1, 3, 0, 0, 1, 4, 0, 1, 0, 524, 3960, 0, 0, 4091, 0, 0, 0, 0, 1, 2, 0, 3, 4083, 0, 0, 1, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8184, 7234, 8192, 8158, 8186, 8172] dup_rows_per_dst=[76, 40, 0, 0, 86, 820, 2, 2] combine_rows_per_src=[8192, 8192, 8184, 7234, 8192, 8158, 8186, 8172] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62322 payload_rows_remote=54130 dup_rows=3214 dup_rows_remote=3214 dedup_ratio=0.049042 router_tokens_per_expert=[0, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 3222, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 3896, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 6752, 8190, 6422, 8192, 8190] dup_rows_per_dst=[26, 0, 10, 1754, 0, 1420, 4, 0] combine_rows_per_src=[8192, 8192, 8192, 6752, 8190, 6422, 8192, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=405508096 dispatch_meta_bytes=228968 combine_payload_bytes=405508096 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=405737064 ingress_bytes=845910016 link_bytes=845910016 token_bytes=811245160 weight_movement_bytes=440401920 total_nvlink_bytes=1251647080 weight_to_token_ratio=0.542872 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.471176646e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58424 payload_rows_remote=50232 dup_rows=7112 dup_rows_remote=6864 dedup_ratio=0.108521 router_tokens_per_expert=[0, 121, 1, 2, 0, 0, 0, 0, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4094, 3371, 0, 4096, 13, 0, 0, 90, 4096, 0, 4096, 27, 0, 0, 0, 0, 0, 1, 0, 20, 0, 690, 3784, 0, 0, 4094, 0, 0, 4, 0, 0, 0, 3, 2, 3985, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8188, 8084, 1422, 8180, 7974] dup_rows_per_dst=[248, 344, 128, 6328, 54, 0, 10, 0] combine_rows_per_src=[8192, 8192, 8192, 8188, 8084, 1422, 8180, 7974] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=394856448 dispatch_meta_bytes=229352 combine_payload_bytes=394856448 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=395085800 ingress_bytes=504956928 link_bytes=504956928 token_bytes=789942248 weight_movement_bytes=110100480 total_nvlink_bytes=900042728 weight_to_token_ratio=0.139378 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.782031489e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62842 payload_rows_remote=54652 dup_rows=2694 dup_rows_remote=282 dedup_ratio=0.041107 router_tokens_per_expert=[0, 2, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 4070, 2795, 0, 4096, 0, 0, 0, 7, 4096, 0, 4077, 20, 0, 0, 8, 0, 0, 0, 0, 1, 0, 1212, 4088, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8140, 5592, 8168, 8190, 8192, 8176] dup_rows_per_dst=[4, 86, 138, 12, 42, 2412, 0, 0] combine_rows_per_src=[8192, 8192, 8140, 5592, 8168, 8190, 8192, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63928 payload_rows_remote=55742 dup_rows=1608 dup_rows_remote=1264 dedup_ratio=0.024536 router_tokens_per_expert=[0, 22, 0, 2, 5, 0, 0, 0, 25, 1, 0, 3, 0, 1, 0, 0, 0, 172, 0, 1, 0, 0, 1, 4091, 3321, 0, 4092, 0, 0, 1, 1, 4094, 0, 4090, 42, 1, 3, 0, 0, 1, 4, 0, 0, 0, 620, 3993, 0, 0, 4091, 0, 0, 0, 0, 2, 2, 0, 2, 4083, 0, 0, 1, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8186, 6646, 8190, 8162, 8188, 8172] dup_rows_per_dst=[54, 52, 344, 0, 84, 1072, 2, 0] combine_rows_per_src=[8192, 8192, 8186, 6646, 8190, 8162, 8188, 8172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60654 payload_rows_remote=52462 dup_rows=4882 dup_rows_remote=4882 dedup_ratio=0.074493 router_tokens_per_expert=[0, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 6, 4096, 2968, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1177, 3985, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7526, 8190, 3982, 8192, 8188] dup_rows_per_dst=[16, 0, 112, 4120, 0, 632, 2, 0] combine_rows_per_src=[8192, 8192, 8192, 7526, 8190, 3982, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64340 payload_rows_remote=56148 dup_rows=1196 dup_rows_remote=1102 dedup_ratio=0.018250 router_tokens_per_expert=[0, 4, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 0, 4092, 3503, 0, 4095, 1, 0, 0, 0, 4095, 0, 4095, 47, 0, 0, 1, 0, 0, 2, 0, 0, 0, 516, 4031, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8184, 7006, 8192, 8192, 8192, 8192] dup_rows_per_dst=[8, 4, 182, 2, 94, 906, 0, 0] combine_rows_per_src=[8192, 8190, 8184, 7006, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=391745536 dispatch_meta_bytes=219736 combine_payload_bytes=391745536 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=391965272 ingress_bytes=611946496 link_bytes=611946496 token_bytes=783710808 weight_movement_bytes=220200960 total_nvlink_bytes=1003911768 weight_to_token_ratio=0.280972 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=25472 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.064275604e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61322 payload_rows_remote=53150 dup_rows=4214 dup_rows_remote=3942 dedup_ratio=0.064301 router_tokens_per_expert=[0, 30, 0, 0, 0, 0, 0, 1, 47, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 2, 0, 0, 4079, 3342, 0, 4096, 1, 1, 0, 0, 4094, 0, 4094, 49, 0, 0, 0, 2, 0, 1, 0, 0, 0, 668, 4041, 1, 0, 4093, 0, 0, 0, 0, 0, 0, 2, 0, 4082, 4, 0, 0, 0, 0, 1] payload_rows_per_dst=[7842, 7594, 6638, 6810, 7924, 8154, 8188, 8172] dup_rows_per_dst=[56, 94, 58, 828, 96, 2808, 2, 272] combine_rows_per_src=[7842, 7594, 6638, 6810, 7924, 8154, 8188, 8172] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64142 payload_rows_remote=57326 dup_rows=1394 dup_rows_remote=1394 dedup_ratio=0.021271 router_tokens_per_expert=[0, 9, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 4087, 3408, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 690, 4013, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8174, 6816, 8192, 8192, 8192, 8192] dup_rows_per_dst=[18, 2, 122, 0, 38, 1214, 0, 0] combine_rows_per_src=[8192, 8192, 8174, 6816, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58526 payload_rows_remote=50334 dup_rows=7010 dup_rows_remote=6822 dedup_ratio=0.106964 router_tokens_per_expert=[0, 90, 1, 3, 0, 0, 0, 0, 181, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 0, 4, 4093, 3152, 0, 4096, 14, 0, 0, 92, 4096, 0, 4096, 24, 0, 0, 0, 0, 0, 3, 0, 15, 1, 780, 3836, 0, 0, 4094, 0, 0, 3, 0, 0, 0, 3, 2, 3991, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 8190, 8000, 1596, 8180, 7986] dup_rows_per_dst=[188, 362, 404, 5998, 48, 2, 8, 0] combine_rows_per_src=[8192, 8192, 8190, 8190, 8000, 1596, 8180, 7986] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=399558656 dispatch_meta_bytes=228024 combine_payload_bytes=399558656 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=399786680 ingress_bytes=619759616 link_bytes=619759616 token_bytes=799345336 weight_movement_bytes=220200960 total_nvlink_bytes=1019546296 weight_to_token_ratio=0.275477 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.077863905e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=376047616 dispatch_meta_bytes=229376 combine_payload_bytes=376047616 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=376276992 ingress_bytes=486148096 link_bytes=486148096 token_bytes=752324608 weight_movement_bytes=110100480 total_nvlink_bytes=862425088 weight_to_token_ratio=0.146347 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.454914966e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64594 payload_rows_remote=56404 dup_rows=942 dup_rows_remote=942 dedup_ratio=0.014374 router_tokens_per_expert=[1, 1, 0, 4095, 0, 0, 4095, 0, 0, 0, 0, 4089, 0, 0, 1, 0, 0, 8, 6, 1, 0, 0, 4, 0, 0, 4071, 0, 5, 4095, 0, 3650, 0, 0, 0, 405, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 1, 6, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8180, 8192, 7304, 8188, 8192, 8190, 8156] dup_rows_per_dst=[2, 0, 36, 6, 810, 0, 0, 88] combine_rows_per_src=[8192, 8180, 8192, 7304, 8188, 8192, 8190, 8156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=402468864 dispatch_meta_bytes=229000 combine_payload_bytes=402468864 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=402697864 ingress_bytes=732770304 link_bytes=732770304 token_bytes=805166728 weight_movement_bytes=330301440 total_nvlink_bytes=1135468168 weight_to_token_ratio=0.410227 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.274408079e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=410898432 dispatch_meta_bytes=233200 combine_payload_bytes=410898432 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=411131632 ingress_bytes=741199872 link_bytes=741199872 token_bytes=822030064 weight_movement_bytes=330301440 total_nvlink_bytes=1152331504 weight_to_token_ratio=0.401812 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=27392 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.289068484e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=403744768 dispatch_meta_bytes=229408 combine_payload_bytes=403744768 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=403974176 ingress_bytes=513845248 link_bytes=513845248 token_bytes=807718944 weight_movement_bytes=110100480 total_nvlink_bytes=917819424 weight_to_token_ratio=0.136310 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.936613993e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=390899712 dispatch_meta_bytes=228280 combine_payload_bytes=390899712 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=391127992 ingress_bytes=501000192 link_bytes=501000192 token_bytes=782027704 weight_movement_bytes=110100480 total_nvlink_bytes=892128184 weight_to_token_ratio=0.140788 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.713217343e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=388003840 dispatch_meta_bytes=229376 combine_payload_bytes=388003840 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=388233216 ingress_bytes=498104320 link_bytes=498104320 token_bytes=776237056 weight_movement_bytes=110100480 total_nvlink_bytes=886337536 weight_to_token_ratio=0.141839 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.662853366e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=360062976 dispatch_meta_bytes=228384 combine_payload_bytes=360062976 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=360291360 ingress_bytes=470163456 link_bytes=470163456 token_bytes=720354336 weight_movement_bytes=110100480 total_nvlink_bytes=830454816 weight_to_token_ratio=0.152842 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.176915786e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65068 payload_rows_remote=56876 dup_rows=468 dup_rows_remote=468 dedup_ratio=0.007141 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 2, 0, 0, 14, 0, 0, 0, 4038, 0, 0, 4096, 0, 3927, 0, 0, 0, 187, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8170, 8192, 7854, 8192, 8192, 8192, 8084] dup_rows_per_dst=[0, 0, 32, 0, 374, 0, 0, 62] combine_rows_per_src=[8192, 8170, 8192, 7854, 8192, 8192, 8192, 8084] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64812 payload_rows_remote=56622 dup_rows=724 dup_rows_remote=724 dedup_ratio=0.011047 router_tokens_per_expert=[2, 2, 0, 4095, 0, 0, 4095, 0, 0, 0, 0, 4089, 0, 0, 1, 0, 0, 11, 3, 1, 0, 0, 4, 0, 0, 4083, 0, 3, 4095, 0, 3748, 0, 0, 0, 290, 0, 0, 0, 4094, 0, 3, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 1, 6, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8180, 8192, 7500, 8188, 8192, 8190, 8178] dup_rows_per_dst=[6, 0, 36, 2, 580, 8, 0, 92] combine_rows_per_src=[8192, 8180, 8192, 7500, 8188, 8192, 8190, 8178] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=380979200 dispatch_meta_bytes=228368 combine_payload_bytes=380979200 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=381207568 ingress_bytes=491079680 link_bytes=491079680 token_bytes=762186768 weight_movement_bytes=110100480 total_nvlink_bytes=872287248 weight_to_token_ratio=0.144453 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.540683322e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=410912768 dispatch_meta_bytes=234880 combine_payload_bytes=410912768 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=411147648 ingress_bytes=741214208 link_bytes=741214208 token_bytes=822060416 weight_movement_bytes=330301440 total_nvlink_bytes=1152361856 weight_to_token_ratio=0.401797 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=25600 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.289093417e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64316 payload_rows_remote=56124 dup_rows=1220 dup_rows_remote=40 dedup_ratio=0.018616 router_tokens_per_expert=[0, 1, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4, 1, 0, 0, 4058, 0, 0, 4093, 0, 3537, 0, 0, 0, 590, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8174, 8192, 7074, 8192, 8192, 8186, 8116] dup_rows_per_dst=[0, 0, 14, 0, 1180, 2, 0, 24] combine_rows_per_src=[8190, 8174, 8192, 7074, 8192, 8192, 8186, 8116] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=407687168 dispatch_meta_bytes=229376 combine_payload_bytes=407687168 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=407916544 ingress_bytes=627888128 link_bytes=627888128 token_bytes=815603712 weight_movement_bytes=220200960 total_nvlink_bytes=1035804672 weight_to_token_ratio=0.269985 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.092000724e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=405866496 dispatch_meta_bytes=229384 combine_payload_bytes=405866496 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=406095880 ingress_bytes=626067456 link_bytes=626067456 token_bytes=811962376 weight_movement_bytes=220200960 total_nvlink_bytes=1032163336 weight_to_token_ratio=0.271196 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.088834276e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64192 payload_rows_remote=56002 dup_rows=1344 dup_rows_remote=1324 dedup_ratio=0.020508 router_tokens_per_expert=[0, 0, 2, 4093, 5, 0, 4094, 0, 0, 0, 0, 4082, 0, 4, 0, 3, 0, 1, 8, 3, 0, 0, 0, 0, 0, 4002, 0, 2, 4091, 0, 3530, 0, 0, 1, 632, 0, 0, 0, 4091, 0, 15, 0, 2, 4092, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 8, 0, 1, 1, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8174, 8190, 7062, 8184, 8192, 8188, 8010] dup_rows_per_dst=[10, 4, 20, 2, 1264, 28, 2, 14] combine_rows_per_src=[8192, 8174, 8190, 7062, 8184, 8192, 8188, 8010] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64276 payload_rows_remote=57192 dup_rows=1260 dup_rows_remote=1260 dedup_ratio=0.019226 router_tokens_per_expert=[0, 0, 6, 4096, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4022, 0, 0, 4096, 0, 3542, 0, 0, 0, 604, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 7084, 8192, 8188, 8192, 8046] dup_rows_per_dst=[12, 0, 2, 0, 1208, 0, 0, 38] combine_rows_per_src=[8192, 8190, 8192, 7084, 8192, 8188, 8192, 8046] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64140 payload_rows_remote=55972 dup_rows=1396 dup_rows_remote=1302 dedup_ratio=0.021301 router_tokens_per_expert=[0, 0, 0, 4094, 1, 0, 4094, 0, 0, 0, 2, 4092, 0, 0, 1, 0, 0, 7, 1, 2, 0, 1, 0, 1, 0, 4078, 0, 5, 4091, 0, 3790, 0, 0, 0, 265, 0, 0, 0, 4094, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 49, 0, 0, 0, 0, 1, 2, 1] payload_rows_per_dst=[7472, 8186, 8192, 7560, 8188, 8188, 8186, 8168] dup_rows_per_dst=[0, 582, 20, 4, 530, 140, 26, 94] combine_rows_per_src=[7472, 8186, 8192, 7560, 8188, 8188, 8186, 8168] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64612 payload_rows_remote=56428 dup_rows=924 dup_rows_remote=924 dedup_ratio=0.014099 router_tokens_per_expert=[0, 1, 34, 4096, 0, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 3, 1, 0, 1, 0, 0, 0, 4055, 0, 0, 4096, 0, 3668, 0, 0, 0, 329, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8184, 8192, 7336, 8190, 8192, 8192, 8134] dup_rows_per_dst=[70, 0, 10, 0, 658, 0, 0, 186] combine_rows_per_src=[8192, 8184, 8192, 7336, 8190, 8192, 8192, 8134] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58388 payload_rows_remote=50196 dup_rows=7148 dup_rows_remote=6524 dedup_ratio=0.109070 router_tokens_per_expert=[0, 60, 33, 4096, 0, 0, 4096, 0, 2, 0, 0, 3844, 0, 0, 0, 0, 0, 66, 0, 0, 0, 16, 71, 0, 0, 3901, 0, 10, 4091, 0, 3680, 0, 0, 0, 299, 0, 0, 0, 4096, 0, 91, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7692, 7994, 8072, 2196, 8020, 8182, 8040] dup_rows_per_dst=[624, 0, 66, 5986, 220, 2, 0, 250] combine_rows_per_src=[8192, 7692, 7994, 8072, 2196, 8020, 8182, 8040] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=402296832 dispatch_meta_bytes=224656 combine_payload_bytes=402296832 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=402521488 ingress_bytes=622497792 link_bytes=622497792 token_bytes=804818320 weight_movement_bytes=220200960 total_nvlink_bytes=1025019280 weight_to_token_ratio=0.273603 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=29440 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.082626044e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=360794112 dispatch_meta_bytes=228624 combine_payload_bytes=360794112 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=361022736 ingress_bytes=470894592 link_bytes=470894592 token_bytes=721816848 weight_movement_bytes=110100480 total_nvlink_bytes=831917328 weight_to_token_ratio=0.152532 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.189631443e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=404303872 dispatch_meta_bytes=229384 combine_payload_bytes=404303872 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=404533256 ingress_bytes=624504832 link_bytes=624504832 token_bytes=808837128 weight_movement_bytes=220200960 total_nvlink_bytes=1029038088 weight_to_token_ratio=0.272244 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.086116616e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65010 payload_rows_remote=56818 dup_rows=526 dup_rows_remote=526 dedup_ratio=0.008026 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 4, 0, 0, 9, 0, 0, 0, 4011, 0, 0, 4095, 0, 3926, 0, 0, 0, 223, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8166, 8192, 7852, 8192, 8192, 8190, 8034] dup_rows_per_dst=[0, 0, 26, 0, 446, 0, 0, 54] combine_rows_per_src=[8192, 8166, 8192, 7852, 8192, 8192, 8190, 8034] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63726 payload_rows_remote=55538 dup_rows=1810 dup_rows_remote=1784 dedup_ratio=0.027618 router_tokens_per_expert=[0, 0, 1, 4092, 5, 0, 4094, 0, 0, 0, 0, 4061, 0, 4, 1, 2, 0, 1, 9, 3, 0, 0, 0, 0, 0, 3909, 0, 2, 4091, 0, 3410, 0, 0, 1, 867, 0, 0, 0, 4091, 0, 15, 0, 2, 4092, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4, 6, 1, 1, 1, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8132, 8188, 6822, 8184, 8192, 8188, 7828] dup_rows_per_dst=[4, 4, 26, 2, 1734, 30, 2, 8] combine_rows_per_src=[8192, 8132, 8188, 6822, 8184, 8192, 8188, 7828] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64428 payload_rows_remote=56242 dup_rows=1108 dup_rows_remote=1108 dedup_ratio=0.016907 router_tokens_per_expert=[0, 1, 33, 4096, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 3, 1, 0, 2, 0, 0, 0, 3983, 0, 0, 4095, 0, 3677, 0, 0, 0, 379, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8186, 8192, 7384, 8078, 8192, 8190, 8014] dup_rows_per_dst=[68, 0, 12, 96, 744, 0, 0, 188] combine_rows_per_src=[8192, 8186, 8192, 7384, 8078, 8192, 8190, 8014] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=407271424 dispatch_meta_bytes=229376 combine_payload_bytes=407271424 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=407500800 ingress_bytes=627472384 link_bytes=627472384 token_bytes=814772224 weight_movement_bytes=220200960 total_nvlink_bytes=1034973184 weight_to_token_ratio=0.270261 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.091277677e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=401422336 dispatch_meta_bytes=229304 combine_payload_bytes=401422336 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=401651640 ingress_bytes=511522816 link_bytes=511522816 token_bytes=803073976 weight_movement_bytes=110100480 total_nvlink_bytes=913174456 weight_to_token_ratio=0.137099 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.896223081e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=409952256 dispatch_meta_bytes=233808 combine_payload_bytes=409952256 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=410186064 ingress_bytes=850354176 link_bytes=850354176 token_bytes=820138320 weight_movement_bytes=440401920 total_nvlink_bytes=1260540240 weight_to_token_ratio=0.536985 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=29440 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.478905771e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=401207296 dispatch_meta_bytes=229096 combine_payload_bytes=401207296 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=401436392 ingress_bytes=511307776 link_bytes=511307776 token_bytes=802643688 weight_movement_bytes=110100480 total_nvlink_bytes=912744168 weight_to_token_ratio=0.137172 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.892483181e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=404475904 dispatch_meta_bytes=229408 combine_payload_bytes=404475904 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=404705312 ingress_bytes=514576384 link_bytes=514576384 token_bytes=809181216 weight_movement_bytes=110100480 total_nvlink_bytes=919281696 weight_to_token_ratio=0.136064 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.949329650e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=359804928 dispatch_meta_bytes=226880 combine_payload_bytes=359804928 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=360031808 ingress_bytes=690106368 link_bytes=690106368 token_bytes=719836736 weight_movement_bytes=330301440 total_nvlink_bytes=1050138176 weight_to_token_ratio=0.458856 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.200208477e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61204 payload_rows_remote=53012 dup_rows=4332 dup_rows_remote=4332 dedup_ratio=0.066101 router_tokens_per_expert=[0, 0, 4095, 0, 0, 0, 0, 0, 0, 4018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2042, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 1605, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8190, 8036, 4084, 8136, 8192, 8192, 8192, 8182] dup_rows_per_dst=[0, 0, 0, 0, 1142, 0, 38, 3152] combine_rows_per_src=[8190, 8036, 4084, 8136, 8192, 8192, 8192, 8182] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61208 payload_rows_remote=53018 dup_rows=4328 dup_rows_remote=4272 dedup_ratio=0.066040 router_tokens_per_expert=[0, 0, 4095, 1, 0, 0, 0, 0, 1, 4053, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2013, 1, 0, 0, 0, 4059, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 4095, 0, 0, 366, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 26, 0, 0, 0, 0, 0, 2, 0, 1, 4050, 1811, 0, 0, 1, 0, 4094] payload_rows_per_dst=[8192, 8110, 4028, 8120, 8190, 8190, 8190, 8188] dup_rows_per_dst=[0, 0, 0, 0, 732, 2, 56, 3538] combine_rows_per_src=[8192, 8110, 4028, 8120, 8190, 8190, 8190, 8188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64148 payload_rows_remote=55956 dup_rows=1388 dup_rows_remote=28 dedup_ratio=0.021179 router_tokens_per_expert=[0, 1, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4, 1, 0, 0, 3995, 0, 0, 4093, 0, 3534, 0, 0, 0, 680, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8134, 8192, 7068, 8192, 8192, 8186, 7992] dup_rows_per_dst=[2, 0, 10, 0, 1360, 2, 0, 14] combine_rows_per_src=[8192, 8134, 8192, 7068, 8192, 8192, 8186, 7992] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63962 payload_rows_remote=57102 dup_rows=1574 dup_rows_remote=1574 dedup_ratio=0.024017 router_tokens_per_expert=[0, 0, 9, 4096, 0, 0, 4096, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3979, 0, 0, 4096, 0, 3430, 0, 0, 0, 762, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8180, 8192, 6860, 8192, 8190, 8192, 7964] dup_rows_per_dst=[18, 0, 2, 0, 1524, 0, 0, 30] combine_rows_per_src=[8192, 8180, 8192, 6860, 8192, 8190, 8192, 7964] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63230 payload_rows_remote=55088 dup_rows=2306 dup_rows_remote=1472 dedup_ratio=0.035187 router_tokens_per_expert=[0, 0, 0, 4094, 1, 0, 4095, 0, 0, 0, 2, 4090, 0, 0, 1, 0, 0, 6, 1, 2, 0, 2, 0, 1, 0, 4060, 0, 4, 4091, 0, 3719, 0, 0, 0, 359, 0, 0, 0, 4094, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 46, 0, 0, 0, 0, 1, 2, 1] payload_rows_per_dst=[7496, 8182, 8192, 6656, 8188, 8188, 8186, 8142] dup_rows_per_dst=[0, 698, 22, 2, 718, 0, 32, 834] combine_rows_per_src=[7496, 8182, 8192, 6656, 8188, 8188, 8186, 8142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=398096384 dispatch_meta_bytes=229288 combine_payload_bytes=398096384 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=398325672 ingress_bytes=508196864 link_bytes=508196864 token_bytes=796422056 weight_movement_bytes=110100480 total_nvlink_bytes=906522536 weight_to_token_ratio=0.138244 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.838379305e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=403142656 dispatch_meta_bytes=229400 combine_payload_bytes=403142656 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=403372056 ingress_bytes=513243136 link_bytes=513243136 token_bytes=806514712 weight_movement_bytes=110100480 total_nvlink_bytes=916615192 weight_to_token_ratio=0.136514 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.926142275e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57232 payload_rows_remote=49040 dup_rows=8304 dup_rows_remote=7702 dedup_ratio=0.126709 router_tokens_per_expert=[0, 52, 35, 4096, 0, 0, 4096, 0, 2, 0, 0, 3835, 0, 0, 0, 0, 0, 49, 0, 0, 0, 15, 68, 0, 0, 3824, 0, 12, 4090, 0, 3634, 0, 0, 0, 415, 0, 0, 0, 4096, 0, 103, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 271, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7674, 7990, 8192, 936, 8134, 8180, 7934] dup_rows_per_dst=[602, 0, 72, 7292, 50, 32, 0, 256] combine_rows_per_src=[8192, 7674, 7990, 8192, 936, 8134, 8180, 7934] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60550 payload_rows_remote=52360 dup_rows=4986 dup_rows_remote=4928 dedup_ratio=0.076080 router_tokens_per_expert=[0, 0, 4095, 1, 0, 0, 0, 1, 1, 4058, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1723, 1, 0, 0, 0, 4014, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 4095, 0, 0, 380, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 27, 0, 0, 0, 0, 0, 2, 0, 1, 4068, 2108, 0, 0, 0, 0, 4094] payload_rows_per_dst=[8192, 8120, 3448, 8030, 8190, 8190, 8190, 8190] dup_rows_per_dst=[2, 0, 0, 0, 760, 2, 58, 4164] combine_rows_per_src=[8192, 8120, 3448, 8030, 8190, 8190, 8190, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=401092608 dispatch_meta_bytes=223936 combine_payload_bytes=401092608 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=401316544 ingress_bytes=621293568 link_bytes=621293568 token_bytes=802409152 weight_movement_bytes=220200960 total_nvlink_bytes=1022610112 weight_to_token_ratio=0.274425 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=28672 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.080531700e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=379990016 dispatch_meta_bytes=229376 combine_payload_bytes=379990016 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=380219392 ingress_bytes=490090496 link_bytes=490090496 token_bytes=760209408 weight_movement_bytes=110100480 total_nvlink_bytes=870309888 weight_to_token_ratio=0.144829 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.523479786e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61074 payload_rows_remote=52882 dup_rows=4462 dup_rows_remote=4100 dedup_ratio=0.068085 router_tokens_per_expert=[0, 0, 4095, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1945, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 2009, 0, 0, 0, 0, 4095] payload_rows_per_dst=[8190, 8118, 3890, 8132, 8192, 8190, 8192, 8170] dup_rows_per_dst=[0, 0, 2, 0, 362, 0, 104, 3994] combine_rows_per_src=[8190, 8118, 3890, 8132, 8192, 8190, 8192, 8170] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=380033024 dispatch_meta_bytes=229160 combine_payload_bytes=380033024 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=380262184 ingress_bytes=490133504 link_bytes=490133504 token_bytes=760295208 weight_movement_bytes=110100480 total_nvlink_bytes=870395688 weight_to_token_ratio=0.144813 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.524227766e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=409307136 dispatch_meta_bytes=234704 combine_payload_bytes=409307136 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=409541840 ingress_bytes=959809536 link_bytes=959809536 token_bytes=818848976 weight_movement_bytes=550502400 total_nvlink_bytes=1369351376 weight_to_token_ratio=0.672288 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=28672 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.669266645e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=394870784 dispatch_meta_bytes=226240 combine_payload_bytes=394870784 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=395097024 ingress_bytes=615071744 link_bytes=615071744 token_bytes=789967808 weight_movement_bytes=220200960 total_nvlink_bytes=1010168768 weight_to_token_ratio=0.278747 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=31872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.069710925e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=351518720 dispatch_meta_bytes=226968 combine_payload_bytes=351518720 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=351745688 ingress_bytes=681820160 link_bytes=681820160 token_bytes=703264408 weight_movement_bytes=330301440 total_nvlink_bytes=1033565848 weight_to_token_ratio=0.469669 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.185797398e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60396 payload_rows_remote=52204 dup_rows=5140 dup_rows_remote=5140 dedup_ratio=0.078430 router_tokens_per_expert=[0, 0, 4094, 0, 0, 0, 0, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1646, 0, 0, 0, 0, 4051, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 1997, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8188, 8046, 3292, 8102, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 1138, 0, 38, 3964] combine_rows_per_src=[8188, 8046, 3292, 8102, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=375316480 dispatch_meta_bytes=229152 combine_payload_bytes=375316480 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=375545632 ingress_bytes=485416960 link_bytes=485416960 token_bytes=750862112 weight_movement_bytes=110100480 total_nvlink_bytes=860962592 weight_to_token_ratio=0.146632 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.442199308e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59718 payload_rows_remote=51750 dup_rows=5818 dup_rows_remote=5818 dedup_ratio=0.088776 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 3984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2279, 0, 0, 0, 0, 3738, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 1658, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 7968, 8190, 7476, 8192, 8192, 8192, 3316] dup_rows_per_dst=[0, 0, 4556, 0, 1186, 0, 76, 0] combine_rows_per_src=[8192, 7968, 8190, 7476, 8192, 8192, 8192, 3316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61438 payload_rows_remote=53252 dup_rows=4098 dup_rows_remote=916 dedup_ratio=0.062531 router_tokens_per_expert=[0, 0, 4091, 0, 0, 0, 0, 3, 0, 4022, 0, 1, 0, 1, 0, 0, 0, 0, 2, 1596, 0, 0, 0, 0, 4035, 0, 0, 0, 0, 5, 4093, 0, 0, 0, 4093, 0, 3, 422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 1, 0, 0, 0, 2, 0, 4086, 2187, 1, 0, 0, 0, 4096] payload_rows_per_dst=[8188, 8046, 8186, 8070, 8190, 8192, 8190, 4376] dup_rows_per_dst=[0, 2, 3182, 10, 846, 0, 58, 0] combine_rows_per_src=[8188, 8046, 8186, 8070, 8190, 8192, 8190, 4376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=374198272 dispatch_meta_bytes=229376 combine_payload_bytes=374198272 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=374427648 ingress_bytes=484298752 link_bytes=484298752 token_bytes=748625920 weight_movement_bytes=110100480 total_nvlink_bytes=858726400 weight_to_token_ratio=0.147070 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.422751832e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60814 payload_rows_remote=52622 dup_rows=4722 dup_rows_remote=4350 dedup_ratio=0.072052 router_tokens_per_expert=[0, 0, 4095, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1629, 0, 0, 0, 0, 4041, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 2323, 0, 0, 0, 0, 4095] payload_rows_per_dst=[8190, 8136, 4822, 8082, 8192, 8190, 8192, 7010] dup_rows_per_dst=[0, 0, 1096, 0, 372, 0, 114, 3140] combine_rows_per_src=[8190, 8136, 4822, 8082, 8192, 8190, 8192, 7010] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=370944000 dispatch_meta_bytes=230272 combine_payload_bytes=370944000 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=371174272 ingress_bytes=481044480 link_bytes=481044480 token_bytes=742118272 weight_movement_bytes=110100480 total_nvlink_bytes=852218752 weight_to_token_ratio=0.148360 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=31872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.366154689e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=381710336 dispatch_meta_bytes=216672 combine_payload_bytes=381710336 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=381927008 ingress_bytes=491810816 link_bytes=491810816 token_bytes=763637344 weight_movement_bytes=110100480 total_nvlink_bytes=873737824 weight_to_token_ratio=0.144179 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=17664 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.553398980e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60320 payload_rows_remote=52260 dup_rows=5216 dup_rows_remote=5216 dedup_ratio=0.079590 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1297, 0, 0, 0, 0, 4030, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 1644, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8012, 8192, 8060, 8192, 8192, 8192, 3288] dup_rows_per_dst=[0, 0, 2594, 0, 2584, 0, 38, 0] combine_rows_per_src=[8192, 8012, 8192, 8060, 8192, 8192, 8192, 3288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56902 payload_rows_remote=48720 dup_rows=8634 dup_rows_remote=374 dedup_ratio=0.131744 router_tokens_per_expert=[0, 0, 4094, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 1, 0, 1, 1, 0, 1, 2582, 0, 0, 0, 1, 4047, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 4094, 0, 0, 252, 0, 1, 0, 0, 0, 0, 1, 0, 2, 0, 67, 0, 0, 2, 0, 0, 5, 0, 0, 4045, 1305, 0, 0, 0, 0, 4094] payload_rows_per_dst=[8188, 8154, 5168, 8094, 2732, 8192, 8192, 8182] dup_rows_per_dst=[0, 2, 2, 0, 222, 2, 146, 8260] combine_rows_per_src=[8188, 8154, 5168, 8094, 2732, 8192, 8192, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57784 payload_rows_remote=49592 dup_rows=7752 dup_rows_remote=7740 dedup_ratio=0.118286 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 3559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2454, 0, 0, 0, 0, 3768, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 238, 0, 0, 0, 0, 0, 9, 0, 0, 3950, 1727, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8096, 8192, 8058, 6090, 8192, 7510, 3454] dup_rows_per_dst=[12, 954, 4616, 1554, 458, 0, 158, 0] combine_rows_per_src=[8192, 8096, 8192, 8058, 6090, 8192, 7510, 3454] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59798 payload_rows_remote=51700 dup_rows=5738 dup_rows_remote=5738 dedup_ratio=0.087555 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1582, 0, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 1, 0, 0, 0, 0, 0, 0, 4094, 1296, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8038, 7462, 8098, 8192, 8192, 8192, 3432] dup_rows_per_dst=[0, 0, 2600, 0, 2646, 0, 42, 450] combine_rows_per_src=[8192, 8038, 7462, 8098, 8192, 8192, 8192, 3432] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56890 payload_rows_remote=48708 dup_rows=8646 dup_rows_remote=252 dedup_ratio=0.131927 router_tokens_per_expert=[0, 0, 4094, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 1, 0, 1, 1, 0, 1, 2788, 0, 0, 0, 1, 4084, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 4094, 0, 0, 226, 0, 1, 0, 0, 0, 0, 1, 0, 2, 0, 36, 0, 0, 2, 0, 0, 4, 0, 0, 4020, 1149, 0, 0, 0, 0, 4094] payload_rows_per_dst=[8188, 8146, 5580, 8168, 2242, 8192, 8192, 8182] dup_rows_per_dst=[0, 2, 2, 0, 164, 2, 82, 8394] combine_rows_per_src=[8188, 8146, 5580, 8168, 2242, 8192, 8192, 8182] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59756 payload_rows_remote=51790 dup_rows=5780 dup_rows_remote=5780 dedup_ratio=0.088196 router_tokens_per_expert=[0, 0, 4095, 0, 0, 0, 0, 0, 0, 3983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2380, 0, 0, 0, 0, 3913, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 1, 0, 0, 4087, 1507, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8190, 7966, 8184, 7826, 8192, 8192, 8192, 3014] dup_rows_per_dst=[0, 0, 4750, 0, 970, 0, 60, 0] combine_rows_per_src=[8190, 7966, 8184, 7826, 8192, 8192, 8192, 3014] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58148 payload_rows_remote=49956 dup_rows=7388 dup_rows_remote=7374 dedup_ratio=0.112732 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 3558, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2581, 0, 0, 0, 0, 3885, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 228, 0, 0, 0, 0, 0, 11, 0, 0, 3891, 1565, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8108, 8182, 8110, 6698, 8192, 7536, 3130] dup_rows_per_dst=[14, 1006, 4762, 842, 586, 0, 178, 0] combine_rows_per_src=[8192, 8108, 8182, 8110, 6698, 8192, 7536, 3130] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60756 payload_rows_remote=52568 dup_rows=4780 dup_rows_remote=958 dedup_ratio=0.072937 router_tokens_per_expert=[0, 0, 4091, 0, 0, 0, 0, 3, 0, 4001, 0, 1, 0, 1, 0, 0, 0, 0, 3, 1922, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 5, 4093, 0, 0, 0, 4093, 0, 3, 422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 1, 0, 0, 0, 2, 0, 4080, 1854, 1, 0, 0, 0, 4096] payload_rows_per_dst=[8188, 8004, 8188, 8094, 8190, 8192, 8190, 3710] dup_rows_per_dst=[0, 2, 3822, 10, 846, 0, 100, 0] combine_rows_per_src=[8188, 8004, 8188, 8094, 8190, 8192, 8190, 3710] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=379058176 dispatch_meta_bytes=227928 combine_payload_bytes=379058176 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=379286104 ingress_bytes=819460096 link_bytes=819460096 token_bytes=758344280 weight_movement_bytes=440401920 total_nvlink_bytes=1198746200 weight_to_token_ratio=0.580741 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=28544 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.425175885e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61302 payload_rows_remote=53112 dup_rows=4234 dup_rows_remote=2954 dedup_ratio=0.064606 router_tokens_per_expert=[4016, 2086, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 1, 0, 0, 4067, 1, 4095, 0, 231, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 537, 0, 0, 4095, 0, 0, 703, 0, 2, 0, 1, 4, 0, 0, 4095, 0, 636, 0, 0, 0, 0, 1, 4095, 1, 1, 0, 0] payload_rows_per_dst=[4172, 8190, 8136, 8190, 8152, 8080, 8190, 8192] dup_rows_per_dst=[0, 0, 4, 470, 1070, 1406, 1280, 4] combine_rows_per_src=[4172, 8190, 8136, 8190, 8152, 8080, 8190, 8192] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=370585600 dispatch_meta_bytes=229752 combine_payload_bytes=370585600 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=370815352 ingress_bytes=590786560 link_bytes=590786560 token_bytes=741400952 weight_movement_bytes=220200960 total_nvlink_bytes=961601912 weight_to_token_ratio=0.297007 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.027474996e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=349138944 dispatch_meta_bytes=195840 combine_payload_bytes=349138944 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=349334784 ingress_bytes=679440384 link_bytes=679440384 token_bytes=698473728 weight_movement_bytes=330301440 total_nvlink_bytes=1028775168 weight_to_token_ratio=0.472890 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=16896 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.181658577e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=371230720 dispatch_meta_bytes=230280 combine_payload_bytes=371230720 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=371461000 ingress_bytes=481331200 link_bytes=481331200 token_bytes=742691720 weight_movement_bytes=110100480 total_nvlink_bytes=852792200 weight_to_token_ratio=0.148245 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=31872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.371141222e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61604 payload_rows_remote=53412 dup_rows=3932 dup_rows_remote=2018 dedup_ratio=0.059998 router_tokens_per_expert=[3848, 2228, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 4096, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 700, 0, 0, 4096, 0, 0, 951, 0, 6, 0, 0, 13, 0, 0, 4096, 0, 474, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0] payload_rows_per_dst=[4456, 8192, 8088, 8192, 8100, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 40, 996, 1914, 974, 8] combine_rows_per_src=[4456, 8192, 8088, 8192, 8100, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=358084608 dispatch_meta_bytes=229320 combine_payload_bytes=358084608 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=358313928 ingress_bytes=468185088 link_bytes=468185088 token_bytes=716398536 weight_movement_bytes=110100480 total_nvlink_bytes=826499016 weight_to_token_ratio=0.153686 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.142508712e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=376807424 dispatch_meta_bytes=214104 combine_payload_bytes=376807424 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=377021528 ingress_bytes=486907904 link_bytes=486907904 token_bytes=753828952 weight_movement_bytes=110100480 total_nvlink_bytes=863929432 weight_to_token_ratio=0.146055 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=17280 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.468129276e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62482 payload_rows_remote=54292 dup_rows=3054 dup_rows_remote=2578 dedup_ratio=0.046600 router_tokens_per_expert=[3863, 2637, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 1, 0, 0, 4069, 1, 4095, 0, 78, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 561, 0, 0, 4095, 0, 0, 834, 0, 3, 0, 1, 3, 0, 0, 4095, 0, 235, 0, 0, 0, 0, 1, 4095, 1, 1, 0, 0] payload_rows_per_dst=[5274, 8190, 8140, 8190, 8114, 8192, 8190, 8192] dup_rows_per_dst=[0, 0, 4, 162, 734, 1674, 476, 4] combine_rows_per_src=[5274, 8190, 8140, 8190, 8114, 8192, 8190, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=382857216 dispatch_meta_bytes=221720 combine_payload_bytes=382857216 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=383078936 ingress_bytes=603058176 link_bytes=603058176 token_bytes=765936152 weight_movement_bytes=220200960 total_nvlink_bytes=986137112 weight_to_token_ratio=0.287493 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=17408 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.048817354e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60554 payload_rows_remote=52470 dup_rows=4982 dup_rows_remote=4136 dedup_ratio=0.076019 router_tokens_per_expert=[3824, 1675, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 4096, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 641, 0, 0, 4096, 0, 0, 1861, 0, 1, 0, 0, 2, 0, 0, 4096, 0, 131, 0, 2, 0, 0, 0, 4096, 0, 0, 4, 0] payload_rows_per_dst=[3350, 8192, 8160, 8192, 8084, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 134, 846, 3724, 270, 8] combine_rows_per_src=[3350, 8192, 8160, 8192, 8084, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=389165056 dispatch_meta_bytes=227480 combine_payload_bytes=389165056 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=389392536 ingress_bytes=609366016 link_bytes=609366016 token_bytes=778557592 weight_movement_bytes=220200960 total_nvlink_bytes=998758552 weight_to_token_ratio=0.282832 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=29824 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059787725e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61030 payload_rows_remote=52838 dup_rows=4506 dup_rows_remote=4344 dedup_ratio=0.068756 router_tokens_per_expert=[3834, 1494, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 4096, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 639, 0, 0, 4096, 0, 0, 1817, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 364, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[7146, 8192, 8116, 8192, 7982, 5018, 8192, 8192] dup_rows_per_dst=[1998, 0, 0, 162, 964, 654, 728, 0] combine_rows_per_src=[7146, 8192, 8116, 8192, 7982, 5018, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61480 payload_rows_remote=53288 dup_rows=4056 dup_rows_remote=4054 dedup_ratio=0.061890 router_tokens_per_expert=[3911, 2337, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 4071, 0, 4096, 0, 48, 0, 0, 2, 8, 0, 0, 0, 0, 0, 0, 560, 0, 0, 4094, 0, 0, 1025, 2, 13, 1, 0, 10, 0, 0, 4094, 0, 303, 1, 0, 1, 0, 0, 4094, 0, 0, 1, 1] payload_rows_per_dst=[4672, 8186, 8144, 7854, 8048, 8192, 8192, 8192] dup_rows_per_dst=[0, 4, 0, 114, 894, 2078, 964, 2] combine_rows_per_src=[4672, 8186, 8144, 7854, 8048, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=376104960 dispatch_meta_bytes=226424 combine_payload_bytes=376104960 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=376331384 ingress_bytes=486205440 link_bytes=486205440 token_bytes=752436344 weight_movement_bytes=110100480 total_nvlink_bytes=862536824 weight_to_token_ratio=0.146325 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=28288 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.455912272e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61622 payload_rows_remote=53430 dup_rows=3914 dup_rows_remote=3914 dedup_ratio=0.059723 router_tokens_per_expert=[3597, 865, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 0, 4096, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 0, 0, 4096, 0, 0, 2488, 0, 6, 0, 0, 1, 0, 0, 4096, 0, 741, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8052, 8192, 7634, 4976, 8192, 8192] dup_rows_per_dst=[1730, 0, 0, 184, 504, 12, 1484, 0] combine_rows_per_src=[8192, 8192, 8052, 8192, 7634, 4976, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59670 payload_rows_remote=51512 dup_rows=5866 dup_rows_remote=5866 dedup_ratio=0.089508 router_tokens_per_expert=[3881, 2114, 4094, 0, 0, 4, 2, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4078, 0, 4094, 0, 160, 0, 0, 2, 2, 2, 0, 0, 0, 0, 0, 477, 0, 0, 4096, 0, 0, 1230, 0, 40, 0, 0, 7, 0, 0, 4093, 0, 284, 1, 0, 0, 1, 0, 4094, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8158, 8192, 8028, 2526, 8192, 8190] dup_rows_per_dst=[4240, 18, 0, 328, 688, 14, 578, 0] combine_rows_per_src=[8192, 8192, 8158, 8192, 8028, 2526, 8192, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52950 payload_rows_remote=44758 dup_rows=12586 dup_rows_remote=4492 dedup_ratio=0.192047 router_tokens_per_expert=[3490, 2083, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3885, 0, 4096, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 434, 0, 0, 4096, 0, 0, 2006, 0, 95, 0, 0, 53, 0, 0, 4096, 0, 193, 0, 6, 0, 0, 0, 4096, 0, 0, 19, 0] payload_rows_per_dst=[8192, 8172, 8188, 7346, 3706, 6600, 2582, 8164] dup_rows_per_dst=[8094, 6, 486, 62, 214, 3578, 138, 8] combine_rows_per_src=[8192, 8172, 8188, 7346, 3706, 6600, 2582, 8164] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=377194496 dispatch_meta_bytes=227888 combine_payload_bytes=377194496 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=377422384 ingress_bytes=817596416 link_bytes=817596416 token_bytes=754616880 weight_movement_bytes=440401920 total_nvlink_bytes=1195018800 weight_to_token_ratio=0.583610 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=28416 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.421934639e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=374599680 dispatch_meta_bytes=229904 combine_payload_bytes=374599680 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=374829584 ingress_bytes=594800640 link_bytes=594800640 token_bytes=749429264 weight_movement_bytes=220200960 total_nvlink_bytes=969630224 weight_to_token_ratio=0.293825 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=31744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.034456141e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=349224960 dispatch_meta_bytes=196376 combine_payload_bytes=349224960 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=349421336 ingress_bytes=679526400 link_bytes=679526400 token_bytes=698646296 weight_movement_bytes=330301440 total_nvlink_bytes=1028947736 weight_to_token_ratio=0.472773 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=14976 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.181808172e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=355475456 dispatch_meta_bytes=229328 combine_payload_bytes=355475456 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=355704784 ingress_bytes=465575936 link_bytes=465575936 token_bytes=711180240 weight_movement_bytes=110100480 total_nvlink_bytes=821280720 weight_to_token_ratio=0.154814 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.097131268e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=380706816 dispatch_meta_bytes=224264 combine_payload_bytes=380706816 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=380931080 ingress_bytes=600907776 link_bytes=600907776 token_bytes=761637896 weight_movement_bytes=220200960 total_nvlink_bytes=981838856 weight_to_token_ratio=0.289115 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=24832 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.045077455e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60138 payload_rows_remote=52010 dup_rows=5398 dup_rows_remote=3872 dedup_ratio=0.082367 router_tokens_per_expert=[4042, 1479, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 0, 4096, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 691, 0, 0, 4096, 0, 0, 778, 0, 7, 0, 0, 12, 0, 0, 4096, 0, 1115, 0, 0, 0, 0, 0, 4096, 0, 0, 19, 0] payload_rows_per_dst=[2958, 8192, 8092, 8192, 8192, 8128, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 198, 1382, 1526, 2254, 38] combine_rows_per_src=[2958, 8192, 8092, 8192, 8192, 8128, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55674 payload_rows_remote=47482 dup_rows=9862 dup_rows_remote=9862 dedup_ratio=0.150482 router_tokens_per_expert=[3839, 680, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 0, 4096, 0, 196, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 329, 0, 0, 4096, 0, 0, 1939, 0, 6, 0, 0, 1, 0, 0, 4096, 0, 1277, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8184, 8192, 8040, 8192, 8192, 3878, 2804, 8192] dup_rows_per_dst=[8644, 0, 0, 394, 658, 12, 154, 0] combine_rows_per_src=[8184, 8192, 8040, 8192, 8192, 3878, 2804, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59116 payload_rows_remote=50956 dup_rows=6420 dup_rows_remote=6420 dedup_ratio=0.097961 router_tokens_per_expert=[4048, 1469, 4093, 0, 0, 4, 2, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4078, 0, 4094, 0, 510, 0, 0, 1, 2, 3, 0, 0, 0, 0, 0, 375, 0, 0, 4096, 0, 0, 887, 0, 41, 1, 0, 6, 0, 0, 4093, 0, 857, 1, 0, 0, 0, 0, 4094, 0, 0, 0, 0] payload_rows_per_dst=[8150, 8192, 8160, 8192, 8192, 1850, 8192, 8188] dup_rows_per_dst=[2896, 16, 0, 1028, 750, 8, 1722, 0] combine_rows_per_src=[8150, 8192, 8160, 8192, 8192, 1850, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=372807680 dispatch_meta_bytes=223528 combine_payload_bytes=372807680 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=373031208 ingress_bytes=593008640 link_bytes=593008640 token_bytes=745838888 weight_movement_bytes=220200960 total_nvlink_bytes=966039848 weight_to_token_ratio=0.295239 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=17920 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.031339558e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=381968384 dispatch_meta_bytes=229368 combine_payload_bytes=381968384 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=382197752 ingress_bytes=492068864 link_bytes=492068864 token_bytes=764166136 weight_movement_bytes=110100480 total_nvlink_bytes=874266616 weight_to_token_ratio=0.144079 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.557886859e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=378742784 dispatch_meta_bytes=228728 combine_payload_bytes=378742784 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=378971512 ingress_bytes=709044224 link_bytes=709044224 token_bytes=757714296 weight_movement_bytes=330301440 total_nvlink_bytes=1088015736 weight_to_token_ratio=0.435918 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.233144523e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63910 payload_rows_remote=55722 dup_rows=1626 dup_rows_remote=1626 dedup_ratio=0.024811 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 3768, 4096, 0, 3613, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7536, 7226, 8192, 8192, 8192, 8188, 8192, 8192] dup_rows_per_dst=[0, 0, 706, 0, 0, 0, 920, 0] combine_rows_per_src=[7536, 7226, 8192, 8192, 8192, 8188, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=382986240 dispatch_meta_bytes=229376 combine_payload_bytes=382986240 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=383215616 ingress_bytes=603187200 link_bytes=603187200 token_bytes=766201856 weight_movement_bytes=220200960 total_nvlink_bytes=986402816 weight_to_token_ratio=0.287393 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.049041748e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64752 payload_rows_remote=56560 dup_rows=784 dup_rows_remote=320 dedup_ratio=0.011963 router_tokens_per_expert=[0, 0, 4095, 0, 1, 0, 0, 3948, 4095, 0, 3861, 1, 0, 0, 0, 0, 0, 0, 4049, 3, 1, 0, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4089, 0, 0, 4095, 0, 0, 0, 230, 0, 1, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7898, 7722, 8190, 8190, 8190, 8178, 8192, 8192] dup_rows_per_dst=[0, 2, 308, 0, 10, 0, 464, 0] combine_rows_per_src=[7898, 7722, 8190, 8190, 8190, 8178, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=369238016 dispatch_meta_bytes=229512 combine_payload_bytes=369238016 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=369467528 ingress_bytes=479338496 link_bytes=479338496 token_bytes=738705544 weight_movement_bytes=110100480 total_nvlink_bytes=848806024 weight_to_token_ratio=0.149045 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.336484822e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=320825344 dispatch_meta_bytes=197000 combine_payload_bytes=320825344 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=321022344 ingress_bytes=761227264 link_bytes=761227264 token_bytes=641847688 weight_movement_bytes=440401920 total_nvlink_bytes=1082249608 weight_to_token_ratio=0.686147 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=15488 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.323899412e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59592 payload_rows_remote=51400 dup_rows=5944 dup_rows_remote=4816 dedup_ratio=0.090698 router_tokens_per_expert=[4048, 1156, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 4096, 0, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 564, 0, 0, 4096, 0, 0, 1828, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 435, 0, 1, 0, 0, 0, 4096, 0, 0, 5, 0] payload_rows_per_dst=[2312, 8192, 8166, 8192, 8192, 8154, 8192, 8192] dup_rows_per_dst=[0, 2, 0, 330, 1128, 3598, 876, 10] combine_rows_per_src=[2312, 8192, 8166, 8192, 8192, 8154, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=340350976 dispatch_meta_bytes=229376 combine_payload_bytes=340350976 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=340580352 ingress_bytes=450451456 link_bytes=450451456 token_bytes=680931328 weight_movement_bytes=110100480 total_nvlink_bytes=791031808 weight_to_token_ratio=0.161691 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.834091685e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=365252608 dispatch_meta_bytes=229504 combine_payload_bytes=365252608 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=365482112 ingress_bytes=475353088 link_bytes=475353088 token_bytes=730734720 weight_movement_bytes=110100480 total_nvlink_bytes=840835200 weight_to_token_ratio=0.150671 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.267172022e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60108 payload_rows_remote=51916 dup_rows=5428 dup_rows_remote=4942 dedup_ratio=0.082825 router_tokens_per_expert=[3983, 858, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 4096, 0, 242, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 618, 0, 0, 4096, 0, 0, 1684, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 839, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0] payload_rows_per_dst=[5496, 8192, 8114, 8192, 8192, 5538, 8192, 8192] dup_rows_per_dst=[812, 0, 0, 486, 1236, 1206, 1678, 10] combine_rows_per_src=[5496, 8192, 8114, 8192, 8192, 5538, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50772 payload_rows_remote=42580 dup_rows=14764 dup_rows_remote=14758 dedup_ratio=0.225281 router_tokens_per_expert=[4057, 1473, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 4069, 0, 4096, 0, 155, 0, 0, 2, 10, 0, 0, 0, 0, 0, 0, 650, 0, 0, 4094, 0, 0, 911, 2, 10, 1, 0, 12, 0, 0, 4094, 0, 937, 1, 0, 1, 0, 0, 4094, 0, 0, 3, 1] payload_rows_per_dst=[2946, 8188, 8140, 5622, 1300, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 220, 0, 9958, 4580, 6] combine_rows_per_src=[2946, 8188, 8140, 5622, 1300, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53038 payload_rows_remote=44846 dup_rows=12498 dup_rows_remote=1724 dedup_ratio=0.190704 router_tokens_per_expert=[3758, 1628, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3878, 0, 4096, 0, 132, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 354, 0, 0, 4096, 0, 0, 1915, 0, 86, 0, 0, 60, 0, 0, 4096, 0, 452, 0, 7, 0, 0, 0, 4096, 0, 0, 20, 0] payload_rows_per_dst=[8192, 8176, 8188, 7374, 8192, 3832, 958, 8126] dup_rows_per_dst=[10774, 4, 496, 264, 708, 170, 80, 2] combine_rows_per_src=[8192, 8176, 8188, 7374, 8192, 3832, 958, 8126] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64574 payload_rows_remote=56382 dup_rows=962 dup_rows_remote=372 dedup_ratio=0.014679 router_tokens_per_expert=[0, 0, 4095, 0, 1, 0, 0, 3957, 4095, 0, 3756, 1, 0, 0, 0, 0, 0, 0, 4057, 3, 1, 0, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4095, 0, 0, 4095, 0, 0, 0, 293, 0, 1, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7916, 7512, 8192, 8190, 8190, 8190, 8192, 8192] dup_rows_per_dst=[0, 2, 360, 0, 10, 0, 590, 0] combine_rows_per_src=[7916, 7512, 8192, 8190, 8190, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=368435200 dispatch_meta_bytes=224864 combine_payload_bytes=368435200 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=368660064 ingress_bytes=478535680 link_bytes=478535680 token_bytes=737095264 weight_movement_bytes=110100480 total_nvlink_bytes=847195744 weight_to_token_ratio=0.149371 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=28672 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.322522531e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=372133888 dispatch_meta_bytes=227432 combine_payload_bytes=372133888 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=372361320 ingress_bytes=702435328 link_bytes=702435328 token_bytes=744495208 weight_movement_bytes=330301440 total_nvlink_bytes=1074796648 weight_to_token_ratio=0.443658 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=30976 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.221650566e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=321456128 dispatch_meta_bytes=186280 combine_payload_bytes=321456128 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=321642408 ingress_bytes=651757568 link_bytes=651757568 token_bytes=643098536 weight_movement_bytes=330301440 total_nvlink_bytes=973399976 weight_to_token_ratio=0.513609 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=13952 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.133513606e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=404146176 dispatch_meta_bytes=227016 combine_payload_bytes=404146176 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=404373192 ingress_bytes=514246656 link_bytes=514246656 token_bytes=808519368 weight_movement_bytes=110100480 total_nvlink_bytes=918619848 weight_to_token_ratio=0.136175 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=29824 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.943595138e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63664 payload_rows_remote=55472 dup_rows=1872 dup_rows_remote=1872 dedup_ratio=0.028564 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 3800, 4096, 0, 3456, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7600, 6912, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 854, 0, 0, 0, 1018, 0] combine_rows_per_src=[7600, 6912, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63898 payload_rows_remote=55716 dup_rows=1638 dup_rows_remote=842 dedup_ratio=0.024994 router_tokens_per_expert=[0, 2, 4096, 0, 0, 1, 0, 3946, 4096, 0, 3434, 0, 0, 0, 2, 0, 2, 0, 4054, 0, 0, 0, 433, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4092, 0, 0, 4094, 0, 0, 0, 412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7894, 6872, 8182, 8190, 8192, 8184, 8192, 8192] dup_rows_per_dst=[4, 0, 796, 0, 14, 0, 824, 0] combine_rows_per_src=[7894, 6872, 8182, 8190, 8192, 8184, 8192, 8192] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62244 payload_rows_remote=54052 dup_rows=3292 dup_rows_remote=3288 dedup_ratio=0.050232 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 3461, 4096, 0, 3085, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 1096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 2, 4096, 0, 0, 0, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6922, 6170, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 2160, 0, 4, 0, 1128, 0] combine_rows_per_src=[6922, 6170, 8192, 8192, 8192, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=399415296 dispatch_meta_bytes=229392 combine_payload_bytes=399415296 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=399644688 ingress_bytes=619616256 link_bytes=619616256 token_bytes=799059984 weight_movement_bytes=220200960 total_nvlink_bytes=1019260944 weight_to_token_ratio=0.275575 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.077614579e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=405422080 dispatch_meta_bytes=227520 combine_payload_bytes=405422080 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=405649600 ingress_bytes=515522560 link_bytes=515522560 token_bytes=811071680 weight_movement_bytes=110100480 total_nvlink_bytes=921172160 weight_to_token_ratio=0.135747 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=30336 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.965785207e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62482 payload_rows_remote=54550 dup_rows=3054 dup_rows_remote=1028 dedup_ratio=0.046600 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 3422, 4096, 0, 3957, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 1, 4096, 0, 0, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6844, 7932, 6748, 8192, 8192, 8190, 8192, 8192] dup_rows_per_dst=[0, 2026, 760, 0, 2, 2, 264, 0] combine_rows_per_src=[6844, 7932, 6748, 8192, 8192, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=397623296 dispatch_meta_bytes=229376 combine_payload_bytes=397623296 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=397852672 ingress_bytes=617824256 link_bytes=617824256 token_bytes=795475968 weight_movement_bytes=220200960 total_nvlink_bytes=1015676928 weight_to_token_ratio=0.276817 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.074497996e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=399372288 dispatch_meta_bytes=226232 combine_payload_bytes=399372288 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=399598520 ingress_bytes=729673728 link_bytes=729673728 token_bytes=798970808 weight_movement_bytes=330301440 total_nvlink_bytes=1129272248 weight_to_token_ratio=0.413409 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=27648 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.269022624e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62138 payload_rows_remote=53946 dup_rows=3398 dup_rows_remote=3394 dedup_ratio=0.051849 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 3492, 4096, 0, 3001, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 2, 4096, 0, 0, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6984, 6002, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 2114, 0, 4, 0, 1280, 0] combine_rows_per_src=[6984, 6002, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=391014400 dispatch_meta_bytes=222312 combine_payload_bytes=391014400 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=391236712 ingress_bytes=611215360 link_bytes=611215360 token_bytes=782251112 weight_movement_bytes=220200960 total_nvlink_bytes=1002452072 weight_to_token_ratio=0.281497 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=28416 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.063004039e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63638 payload_rows_remote=55446 dup_rows=1898 dup_rows_remote=1898 dedup_ratio=0.028961 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 3519, 4096, 0, 3727, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7038, 7454, 8192, 8192, 8192, 8186, 8192, 8192] dup_rows_per_dst=[0, 0, 1130, 0, 0, 0, 768, 0] combine_rows_per_src=[7038, 7454, 8192, 8192, 8192, 8186, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53542 payload_rows_remote=45438 dup_rows=11994 dup_rows_remote=6606 dedup_ratio=0.183014 router_tokens_per_expert=[0, 12, 4096, 0, 0, 0, 0, 3011, 4096, 0, 3807, 0, 0, 0, 0, 0, 0, 0, 4080, 30, 0, 0, 1137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4042, 0, 4, 4096, 0, 0, 0, 166, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0] payload_rows_per_dst=[8104, 8034, 2368, 8192, 8186, 8092, 2384, 8182] dup_rows_per_dst=[5388, 6274, 48, 0, 102, 0, 172, 10] combine_rows_per_src=[8104, 8034, 2368, 8192, 8186, 8092, 2384, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61970 payload_rows_remote=53788 dup_rows=3566 dup_rows_remote=3566 dedup_ratio=0.054413 router_tokens_per_expert=[1, 0, 0, 0, 0, 1, 0, 1, 4096, 4095, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 546, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 4090, 0, 1138, 2291, 4094, 4095, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 130, 0] payload_rows_per_dst=[8192, 8192, 5522, 8192, 7310, 8190, 8182, 8190] dup_rows_per_dst=[4, 2, 718, 0, 2582, 0, 0, 260] combine_rows_per_src=[8192, 8192, 5522, 8192, 7310, 8190, 8182, 8190] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=387444736 dispatch_meta_bytes=229360 combine_payload_bytes=387444736 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=387674096 ingress_bytes=607645696 link_bytes=607645696 token_bytes=775118832 weight_movement_bytes=220200960 total_nvlink_bytes=995319792 weight_to_token_ratio=0.284087 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.056795806e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63392 payload_rows_remote=55200 dup_rows=2144 dup_rows_remote=2142 dedup_ratio=0.032715 router_tokens_per_expert=[0, 2, 4094, 0, 1, 0, 0, 3828, 4095, 0, 3398, 0, 0, 1, 0, 0, 0, 0, 4088, 2, 1, 0, 554, 0, 0, 2, 0, 0, 1, 0, 2, 0, 0, 2, 0, 1, 0, 0, 4094, 0, 0, 0, 0, 4088, 1, 0, 4094, 0, 0, 0, 416, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2] payload_rows_per_dst=[7662, 6796, 8192, 8190, 7992, 8178, 8190, 8192] dup_rows_per_dst=[0, 2, 1098, 6, 4, 200, 832, 2] combine_rows_per_src=[7662, 6796, 8192, 8190, 7992, 8178, 8190, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63758 payload_rows_remote=55566 dup_rows=1778 dup_rows_remote=1778 dedup_ratio=0.027130 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 3414, 4096, 0, 3901, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4084, 0, 0, 4096, 0, 0, 0, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6828, 7802, 8192, 8192, 8192, 8168, 8192, 8192] dup_rows_per_dst=[0, 0, 1310, 0, 0, 0, 468, 0] combine_rows_per_src=[6828, 7802, 8192, 8192, 8192, 8168, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64064 payload_rows_remote=55890 dup_rows=1472 dup_rows_remote=764 dedup_ratio=0.022461 router_tokens_per_expert=[0, 2, 4096, 0, 0, 0, 0, 3923, 4096, 0, 3549, 0, 0, 0, 2, 0, 1, 0, 4039, 0, 0, 0, 401, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4089, 0, 0, 4094, 0, 0, 0, 370, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[7846, 7100, 8174, 8190, 8192, 8178, 8192, 8192] dup_rows_per_dst=[4, 2, 708, 0, 16, 0, 740, 2] combine_rows_per_src=[7846, 7100, 8174, 8190, 8192, 8178, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62200 payload_rows_remote=54130 dup_rows=3336 dup_rows_remote=1104 dedup_ratio=0.050903 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 3258, 4096, 0, 4030, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 823, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4087, 0, 0, 4096, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6516, 8070, 6672, 8192, 8192, 8174, 8192, 8192] dup_rows_per_dst=[0, 2232, 894, 0, 2, 0, 208, 0] combine_rows_per_src=[6516, 8070, 6672, 8192, 8192, 8174, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54116 payload_rows_remote=46496 dup_rows=11420 dup_rows_remote=4716 dedup_ratio=0.174255 router_tokens_per_expert=[0, 14, 4096, 0, 0, 0, 0, 2808, 4096, 0, 3923, 0, 0, 0, 0, 0, 0, 0, 4075, 33, 0, 0, 1230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4036, 0, 4, 4096, 0, 0, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0] payload_rows_per_dst=[7620, 8152, 2516, 8192, 8188, 8080, 3186, 8182] dup_rows_per_dst=[6704, 4260, 32, 0, 170, 0, 248, 6] combine_rows_per_src=[7620, 8152, 2516, 8192, 8188, 8080, 3186, 8182] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=395673600 dispatch_meta_bytes=229368 combine_payload_bytes=395673600 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=395902968 ingress_bytes=505774080 link_bytes=505774080 token_bytes=791576568 weight_movement_bytes=110100480 total_nvlink_bytes=901677048 weight_to_token_ratio=0.139090 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.796243107e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=398297088 dispatch_meta_bytes=229376 combine_payload_bytes=398297088 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=398526464 ingress_bytes=508397568 link_bytes=508397568 token_bytes=796823552 weight_movement_bytes=110100480 total_nvlink_bytes=906924032 weight_to_token_ratio=0.138174 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.841869878e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62382 payload_rows_remote=54190 dup_rows=3154 dup_rows_remote=3154 dedup_ratio=0.048126 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1173, 0, 0, 4095, 0, 370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 831, 1692, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 33, 0] payload_rows_per_dst=[8192, 8192, 8190, 8188, 5044, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 2346, 740, 2, 0, 0, 66] combine_rows_per_src=[8192, 8192, 8190, 8188, 5044, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59894 payload_rows_remote=51704 dup_rows=5642 dup_rows_remote=5640 dedup_ratio=0.086090 router_tokens_per_expert=[1, 0, 0, 0, 0, 1, 0, 1, 4096, 4095, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 751, 0, 0, 4084, 1, 203, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 4092, 0, 1125, 1938, 4094, 4095, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 88, 0] payload_rows_per_dst=[8192, 8192, 2954, 8186, 7800, 8190, 8190, 8190] dup_rows_per_dst=[4, 0, 270, 408, 4782, 0, 2, 176] combine_rows_per_src=[8192, 8192, 2954, 8186, 7800, 8190, 8190, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=400619520 dispatch_meta_bytes=226616 combine_payload_bytes=400619520 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=400846136 ingress_bytes=730920960 link_bytes=730920960 token_bytes=801465656 weight_movement_bytes=330301440 total_nvlink_bytes=1131767096 weight_to_token_ratio=0.412122 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=27520 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.271191765e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=388003840 dispatch_meta_bytes=220936 combine_payload_bytes=388003840 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=388224776 ingress_bytes=608204800 link_bytes=608204800 token_bytes=776228616 weight_movement_bytes=220200960 total_nvlink_bytes=996429576 weight_to_token_ratio=0.283681 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=29440 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057768180e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=333283328 dispatch_meta_bytes=204848 combine_payload_bytes=333283328 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=333488176 ingress_bytes=553484288 link_bytes=553484288 token_bytes=666771504 weight_movement_bytes=220200960 total_nvlink_bytes=886972464 weight_to_token_ratio=0.330250 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=28416 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.626002094e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63502 payload_rows_remote=57334 dup_rows=2034 dup_rows_remote=2030 dedup_ratio=0.031036 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 879, 0, 0, 4083, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1258, 1828, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 68, 0] payload_rows_per_dst=[8192, 8192, 8184, 8192, 6168, 8190, 8192, 8192] dup_rows_per_dst=[0, 0, 1742, 148, 4, 0, 4, 136] combine_rows_per_src=[8192, 8192, 8184, 8192, 6168, 8190, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=370614272 dispatch_meta_bytes=229376 combine_payload_bytes=370614272 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=370843648 ingress_bytes=480714752 link_bytes=480714752 token_bytes=741457920 weight_movement_bytes=110100480 total_nvlink_bytes=851558400 weight_to_token_ratio=0.148492 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.360420177e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=388433920 dispatch_meta_bytes=229376 combine_payload_bytes=388433920 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=388663296 ingress_bytes=608634880 link_bytes=608634880 token_bytes=777097216 weight_movement_bytes=220200960 total_nvlink_bytes=997298176 weight_to_token_ratio=0.283363 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058516160e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=410970112 dispatch_meta_bytes=237456 combine_payload_bytes=410970112 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=411207568 ingress_bytes=631171072 link_bytes=631171072 token_bytes=822177680 weight_movement_bytes=220200960 total_nvlink_bytes=1042378640 weight_to_token_ratio=0.267826 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=14976 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.097710304e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55226 payload_rows_remote=47034 dup_rows=10310 dup_rows_remote=10220 dedup_ratio=0.157318 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4087, 4094, 0, 0, 0, 1, 0, 1, 3, 2, 0, 0, 512, 0, 1, 4077, 0, 109, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 4094, 1, 923, 2525, 4094, 4094, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 2, 0, 2, 2, 0, 0, 4094, 0, 0, 45, 0] payload_rows_per_dst=[8188, 8176, 1028, 5070, 8190, 8190, 8192, 8192] dup_rows_per_dst=[0, 2, 8, 118, 10086, 0, 6, 90] combine_rows_per_src=[8188, 8176, 1028, 5070, 8190, 8190, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63846 payload_rows_remote=55654 dup_rows=1690 dup_rows_remote=1346 dedup_ratio=0.025787 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 0, 0, 4093, 0, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1692, 1561, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 154, 0] payload_rows_per_dst=[8192, 8192, 8188, 8192, 6506, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 1038, 344, 0, 0, 0, 308] combine_rows_per_src=[8192, 8192, 8188, 8192, 6506, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63662 payload_rows_remote=55470 dup_rows=1874 dup_rows_remote=232 dedup_ratio=0.028595 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 821, 0, 0, 4092, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 927, 2239, 4091, 4095, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 33, 0] payload_rows_per_dst=[8192, 8190, 8192, 8192, 6334, 8186, 8192, 8184] dup_rows_per_dst=[0, 2, 1642, 152, 2, 0, 10, 66] combine_rows_per_src=[8192, 8190, 8192, 8192, 6334, 8186, 8192, 8184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63308 payload_rows_remote=55116 dup_rows=2228 dup_rows_remote=2228 dedup_ratio=0.033997 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 588, 0, 0, 4096, 0, 383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1002, 1980, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 142, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 5964, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 1178, 766, 0, 0, 0, 284] combine_rows_per_src=[8192, 8192, 8192, 8192, 5964, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60574 payload_rows_remote=52382 dup_rows=4962 dup_rows_remote=4962 dedup_ratio=0.075714 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 1253, 0, 0, 4094, 0, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1140, 1187, 4088, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 290, 0] payload_rows_per_dst=[8192, 8192, 8190, 8192, 4652, 8180, 8172, 6804] dup_rows_per_dst=[0, 14, 4194, 456, 2, 24, 0, 272] combine_rows_per_src=[8192, 8192, 8190, 8192, 4652, 8180, 8172, 6804] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=386684928 dispatch_meta_bytes=229360 combine_payload_bytes=386684928 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=386914288 ingress_bytes=606885888 link_bytes=606885888 token_bytes=773599216 weight_movement_bytes=220200960 total_nvlink_bytes=993800176 weight_to_token_ratio=0.284645 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.055474375e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=397436928 dispatch_meta_bytes=229376 combine_payload_bytes=397436928 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=397666304 ingress_bytes=507537408 link_bytes=507537408 token_bytes=795103232 weight_movement_bytes=110100480 total_nvlink_bytes=905203712 weight_to_token_ratio=0.138473 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.826910281e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=325699584 dispatch_meta_bytes=208176 combine_payload_bytes=325699584 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=325907760 ingress_bytes=545900544 link_bytes=545900544 token_bytes=651607344 weight_movement_bytes=220200960 total_nvlink_bytes=871808304 weight_to_token_ratio=0.337935 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=29056 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.494108312e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=385552384 dispatch_meta_bytes=229416 combine_payload_bytes=385552384 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=385781800 ingress_bytes=495652864 link_bytes=495652864 token_bytes=771334184 weight_movement_bytes=110100480 total_nvlink_bytes=881434664 weight_to_token_ratio=0.142740 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.620218514e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63852 payload_rows_remote=55660 dup_rows=1684 dup_rows_remote=1684 dedup_ratio=0.025696 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 786, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 940, 2317, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 56, 0] payload_rows_per_dst=[8192, 8192, 8190, 8192, 6512, 8192, 8190, 8192] dup_rows_per_dst=[0, 0, 1570, 0, 2, 0, 0, 112] combine_rows_per_src=[8192, 8192, 8190, 8192, 6512, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64310 payload_rows_remote=56118 dup_rows=1226 dup_rows_remote=192 dedup_ratio=0.018707 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 2, 0, 4, 1, 0, 0, 516, 0, 0, 4092, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 890, 2600, 4091, 4095, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 90, 0] payload_rows_per_dst=[8192, 8190, 8192, 8192, 6982, 8186, 8192, 8184] dup_rows_per_dst=[0, 4, 1034, 0, 2, 0, 6, 180] combine_rows_per_src=[8192, 8190, 8192, 8192, 6982, 8186, 8192, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64228 payload_rows_remote=56036 dup_rows=1308 dup_rows_remote=1308 dedup_ratio=0.019958 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 363, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1028, 2416, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 289, 0] payload_rows_per_dst=[8192, 8192, 8190, 8192, 6886, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 728, 0, 2, 0, 0, 578] combine_rows_per_src=[8192, 8192, 8190, 8192, 6886, 8192, 8192, 8192] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=398970880 dispatch_meta_bytes=229376 combine_payload_bytes=398970880 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=399200256 ingress_bytes=729272320 link_bytes=729272320 token_bytes=798171136 weight_movement_bytes=330301440 total_nvlink_bytes=1128472576 weight_to_token_ratio=0.413823 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.268324509e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=337139712 dispatch_meta_bytes=229016 combine_payload_bytes=337139712 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=337368728 ingress_bytes=667441152 link_bytes=667441152 token_bytes=674508440 weight_movement_bytes=330301440 total_nvlink_bytes=1004809880 weight_to_token_ratio=0.489692 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.160789939e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=398927872 dispatch_meta_bytes=228000 combine_payload_bytes=398927872 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=399155872 ingress_bytes=509028352 link_bytes=509028352 token_bytes=798083744 weight_movement_bytes=110100480 total_nvlink_bytes=908184224 weight_to_token_ratio=0.137956 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=31232 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.852840249e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=402253824 dispatch_meta_bytes=225240 combine_payload_bytes=402253824 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=402479064 ingress_bytes=622454784 link_bytes=622454784 token_bytes=804732888 weight_movement_bytes=220200960 total_nvlink_bytes=1024933848 weight_to_token_ratio=0.273632 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=27136 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.082551246e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64236 payload_rows_remote=57326 dup_rows=1300 dup_rows_remote=1284 dedup_ratio=0.019836 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1132, 2331, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 85, 0] payload_rows_per_dst=[8192, 8192, 8176, 8192, 6910, 8190, 8192, 8192] dup_rows_per_dst=[0, 0, 1110, 0, 16, 0, 4, 170] combine_rows_per_src=[8192, 8192, 8176, 8192, 6910, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=401666048 dispatch_meta_bytes=229376 combine_payload_bytes=401666048 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=401895424 ingress_bytes=621867008 link_bytes=621867008 token_bytes=803561472 weight_movement_bytes=220200960 total_nvlink_bytes=1023762432 weight_to_token_ratio=0.274031 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.081529006e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64410 payload_rows_remote=56218 dup_rows=1126 dup_rows_remote=1126 dedup_ratio=0.017181 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 280, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1590, 1946, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 283, 0] payload_rows_per_dst=[8192, 8192, 8190, 8192, 7070, 8192, 8190, 8192] dup_rows_per_dst=[0, 0, 558, 0, 2, 0, 0, 566] combine_rows_per_src=[8192, 8192, 8190, 8192, 7070, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60746 payload_rows_remote=52554 dup_rows=4790 dup_rows_remote=4790 dedup_ratio=0.073090 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 773, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1213, 1679, 4088, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 434, 0] payload_rows_per_dst=[8192, 8192, 8190, 8192, 5784, 8174, 8190, 5832] dup_rows_per_dst=[0, 12, 4434, 2, 2, 30, 0, 310] combine_rows_per_src=[8192, 8192, 8190, 8192, 5784, 8174, 8190, 5832] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60140 payload_rows_remote=52074 dup_rows=5396 dup_rows_remote=200 dedup_ratio=0.082336 router_tokens_per_expert=[0, 0, 4095, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 4095, 0, 0, 1206, 3785, 1, 296, 0, 0, 48, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4095, 1, 2845, 0, 0, 11, 1, 0, 0, 1, 0, 0] payload_rows_per_dst=[8190, 8190, 8192, 8192, 2928, 8190, 8066, 8192] dup_rows_per_dst=[0, 0, 0, 2, 78, 96, 5196, 24] combine_rows_per_src=[8190, 8190, 8192, 8192, 2928, 8190, 8066, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=397608960 dispatch_meta_bytes=222808 combine_payload_bytes=397608960 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=397831768 ingress_bytes=617809920 link_bytes=617809920 token_bytes=795440728 weight_movement_bytes=220200960 total_nvlink_bytes=1015641688 weight_to_token_ratio=0.276829 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=25472 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.074473063e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59974 payload_rows_remote=51894 dup_rows=5562 dup_rows_remote=190 dedup_ratio=0.084869 router_tokens_per_expert=[0, 0, 4095, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 1217, 3926, 1, 194, 0, 0, 43, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4095, 1, 2799, 0, 0, 11, 1, 0, 0, 1, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 8192, 2746, 8190, 8080, 8192] dup_rows_per_dst=[0, 0, 2, 0, 78, 86, 5372, 24] combine_rows_per_src=[8192, 8190, 8192, 8192, 2746, 8190, 8080, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62526 payload_rows_remote=54334 dup_rows=3010 dup_rows_remote=2868 dedup_ratio=0.045929 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1029, 3855, 0, 189, 0, 0, 71, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4081, 0, 3063, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 6914, 8192, 6490, 8162] dup_rows_per_dst=[0, 0, 0, 0, 1638, 142, 1230, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 6914, 8192, 6490, 8162] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=395071488 dispatch_meta_bytes=229376 combine_payload_bytes=395071488 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=395300864 ingress_bytes=615272448 link_bytes=615272448 token_bytes=790372352 weight_movement_bytes=220200960 total_nvlink_bytes=1010573312 weight_to_token_ratio=0.278604 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.070059982e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=375474176 dispatch_meta_bytes=229376 combine_payload_bytes=375474176 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=375703552 ingress_bytes=485574656 link_bytes=485574656 token_bytes=751177728 weight_movement_bytes=110100480 total_nvlink_bytes=861278208 weight_to_token_ratio=0.146570 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.444941901e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63506 payload_rows_remote=55316 dup_rows=2030 dup_rows_remote=168 dedup_ratio=0.030975 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 822, 3817, 0, 387, 0, 0, 84, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4089, 0, 3087, 0, 0, 2, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8190, 8192, 6174, 8182] dup_rows_per_dst=[0, 0, 0, 0, 1862, 168, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8190, 8192, 6174, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=410912768 dispatch_meta_bytes=234440 combine_payload_bytes=410912768 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=411147208 ingress_bytes=631113728 link_bytes=631113728 token_bytes=822059976 weight_movement_bytes=220200960 total_nvlink_bytes=1042260936 weight_to_token_ratio=0.267865 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=18432 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.097610573e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=376707072 dispatch_meta_bytes=229376 combine_payload_bytes=376707072 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=376936448 ingress_bytes=486807552 link_bytes=486807552 token_bytes=753643520 weight_movement_bytes=110100480 total_nvlink_bytes=863744000 weight_to_token_ratio=0.146091 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.466383990e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=402970624 dispatch_meta_bytes=229376 combine_payload_bytes=402970624 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=403200000 ingress_bytes=623171584 link_bytes=623171584 token_bytes=806170624 weight_movement_bytes=220200960 total_nvlink_bytes=1026371584 weight_to_token_ratio=0.273144 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.083797879e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62978 payload_rows_remote=54786 dup_rows=2558 dup_rows_remote=2380 dedup_ratio=0.039032 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1005, 3726, 0, 202, 0, 0, 89, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4085, 0, 3181, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7240, 8192, 6608, 8170] dup_rows_per_dst=[0, 0, 0, 0, 1500, 178, 880, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 7240, 8192, 6608, 8170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=373266432 dispatch_meta_bytes=209096 combine_payload_bytes=373266432 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=373475528 ingress_bytes=703567872 link_bytes=703567872 token_bytes=746741960 weight_movement_bytes=330301440 total_nvlink_bytes=1077043400 weight_to_token_ratio=0.442323 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=24576 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.223620246e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=371976192 dispatch_meta_bytes=208336 combine_payload_bytes=371976192 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=372184528 ingress_bytes=702277632 link_bytes=702277632 token_bytes=744160720 weight_movement_bytes=330301440 total_nvlink_bytes=1074462160 weight_to_token_ratio=0.443858 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=24064 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.221376307e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=389466112 dispatch_meta_bytes=228808 combine_payload_bytes=389466112 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=389694920 ingress_bytes=719767552 link_bytes=719767552 token_bytes=779161032 weight_movement_bytes=330301440 total_nvlink_bytes=1109462472 weight_to_token_ratio=0.423919 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=31616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.251794154e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57986 payload_rows_remote=49794 dup_rows=7550 dup_rows_remote=7424 dedup_ratio=0.115204 router_tokens_per_expert=[0, 0, 4096, 0, 1, 0, 1, 4095, 0, 2, 0, 1, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 188, 3986, 0, 189, 0, 0, 358, 0, 0, 1, 0, 4093, 0, 0, 0, 0, 4095, 0, 3463, 0, 0, 5, 0, 2, 0, 2, 1, 0] payload_rows_per_dst=[8192, 8192, 8178, 8188, 754, 8188, 8102, 8192] dup_rows_per_dst=[2, 6, 0, 0, 2, 726, 6688, 126] combine_rows_per_src=[8192, 8192, 8178, 8188, 754, 8188, 8102, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63142 payload_rows_remote=54950 dup_rows=2394 dup_rows_remote=2394 dedup_ratio=0.036530 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1583, 3553, 0, 105, 0, 0, 39, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 2910, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8170, 8192, 5820, 8192] dup_rows_per_dst=[0, 0, 4, 0, 2312, 78, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8170, 8192, 5820, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=396505088 dispatch_meta_bytes=221936 combine_payload_bytes=396505088 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=396727024 ingress_bytes=836907008 link_bytes=836907008 token_bytes=793232112 weight_movement_bytes=440401920 total_nvlink_bytes=1233634032 weight_to_token_ratio=0.555199 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=21376 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.455518934e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7415538784 prefetch_bytes=990904320 reduce_bytes=880803840 weight_movement_bytes=1871708160 egress_bytes=3708908640 ingress_bytes=5578338304 link_bytes=5578338304 total_nvlink_bytes=9287246944 weight_to_token_ratio=0.252404 prefetch_share_of_ingress=0.177634 dedup_ratio=0.095395 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=8 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=9017 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.701647790e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7506538464 prefetch_bytes=1101004800 reduce_bytes=1211105280 weight_movement_bytes=2312110080 egress_bytes=3754405856 ingress_bytes=6064242688 link_bytes=6064242688 total_nvlink_bytes=9818648544 weight_to_token_ratio=0.308013 prefetch_share_of_ingress=0.181557 dedup_ratio=0.082233 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=11 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=9398 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.054671543e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63946 payload_rows_remote=55754 dup_rows=1590 dup_rows_remote=1584 dedup_ratio=0.024261 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 1, 0, 0, 0, 4094, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1068, 3608, 0, 176, 0, 0, 26, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4094, 0, 3309, 0, 0, 3, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8190, 8184, 8192, 6618, 8188] dup_rows_per_dst=[0, 4, 6, 0, 1520, 54, 0, 6] combine_rows_per_src=[8190, 8192, 8192, 8190, 8184, 8192, 6618, 8188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62462 payload_rows_remote=54270 dup_rows=3074 dup_rows_remote=3074 dedup_ratio=0.046906 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1621, 3603, 0, 153, 0, 0, 254, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4088, 0, 2568, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8190, 8192, 5136, 8176] dup_rows_per_dst=[0, 0, 2, 0, 2564, 508, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8190, 8192, 5136, 8176] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60858 payload_rows_remote=52704 dup_rows=4678 dup_rows_remote=4640 dedup_ratio=0.071381 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1199, 3712, 0, 151, 0, 0, 284, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3998, 0, 2913, 0, 0, 50, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8154, 8184, 8192, 8192, 8156, 6076, 5826, 8078] dup_rows_per_dst=[38, 0, 0, 6, 4256, 360, 0, 18] combine_rows_per_src=[8154, 8184, 8192, 8192, 8156, 6076, 5826, 8078] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=356923392 dispatch_meta_bytes=228872 combine_payload_bytes=356923392 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=357152264 ingress_bytes=577124352 link_bytes=577124352 token_bytes=714075656 weight_movement_bytes=220200960 total_nvlink_bytes=934276616 weight_to_token_ratio=0.308372 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.003714169e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7583218416 prefetch_bytes=880803840 reduce_bytes=1761607680 weight_movement_bytes=2642411520 egress_bytes=3792751344 ingress_bytes=6432878592 link_bytes=6432878592 total_nvlink_bytes=10225629936 weight_to_token_ratio=0.348455 prefetch_share_of_ingress=0.136922 dedup_ratio=0.076044 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=16 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043382 gemm_inner_pad_upper_bound=9144 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.118783390e-04 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64224 payload_rows_remote=56032 dup_rows=1312 dup_rows_remote=1308 dedup_ratio=0.020020 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 1, 0, 0, 0, 4093, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1052, 3458, 0, 192, 0, 0, 33, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 3451, 0, 0, 3, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8190, 8174, 8192, 6902, 8192] dup_rows_per_dst=[0, 4, 4, 0, 1230, 68, 0, 6] combine_rows_per_src=[8190, 8192, 8192, 8190, 8174, 8192, 6902, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63052 payload_rows_remote=54860 dup_rows=2484 dup_rows_remote=2484 dedup_ratio=0.037903 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1545, 3335, 0, 204, 0, 0, 226, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4091, 0, 2887, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8136, 8192, 5774, 8182] dup_rows_per_dst=[0, 0, 0, 0, 2032, 452, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8136, 8192, 5774, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7400682544 prefetch_bytes=990904320 reduce_bytes=990904320 weight_movement_bytes=1981808640 egress_bytes=3701478448 ingress_bytes=5681012736 link_bytes=5681012736 total_nvlink_bytes=9382491184 weight_to_token_ratio=0.267787 prefetch_share_of_ingress=0.174424 dedup_ratio=0.096774 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=9 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=9017 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.880215514e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=392706048 dispatch_meta_bytes=228664 combine_payload_bytes=392706048 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=392934712 ingress_bytes=723007488 link_bytes=723007488 token_bytes=785640760 weight_movement_bytes=330301440 total_nvlink_bytes=1115942200 weight_to_token_ratio=0.420423 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=31488 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.257428936e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=393881600 dispatch_meta_bytes=229376 combine_payload_bytes=393881600 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=394110976 ingress_bytes=614082560 link_bytes=614082560 token_bytes=787992576 weight_movement_bytes=220200960 total_nvlink_bytes=1008193536 weight_to_token_ratio=0.279445 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.067990571e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63160 payload_rows_remote=55044 dup_rows=2376 dup_rows_remote=224 dedup_ratio=0.036255 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 747, 3678, 0, 709, 0, 0, 112, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4092, 0, 2948, 0, 0, 2, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8116, 8192, 5896, 8188] dup_rows_per_dst=[0, 0, 0, 0, 2152, 224, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8116, 8192, 5896, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60508 payload_rows_remote=52360 dup_rows=5028 dup_rows_remote=4980 dedup_ratio=0.076721 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1211, 3567, 0, 200, 0, 0, 362, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4017, 0, 2907, 0, 0, 46, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8148, 8184, 8192, 8192, 8066, 5808, 5814, 8104] dup_rows_per_dst=[48, 0, 0, 4, 4484, 470, 0, 22] combine_rows_per_src=[8148, 8184, 8192, 8192, 8066, 5808, 5814, 8104] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=401637376 dispatch_meta_bytes=229360 combine_payload_bytes=401637376 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=401866736 ingress_bytes=511737856 link_bytes=511737856 token_bytes=803504112 weight_movement_bytes=110100480 total_nvlink_bytes=913604592 weight_to_token_ratio=0.137025 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.899962980e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=393236480 dispatch_meta_bytes=229376 combine_payload_bytes=393236480 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=393465856 ingress_bytes=503336960 link_bytes=503336960 token_bytes=786702336 weight_movement_bytes=110100480 total_nvlink_bytes=896802816 weight_to_token_ratio=0.139952 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.753857581e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63318 payload_rows_remote=55126 dup_rows=2218 dup_rows_remote=2218 dedup_ratio=0.033844 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1669, 3341, 0, 149, 0, 0, 37, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 2994, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8178, 8192, 5988, 8192] dup_rows_per_dst=[0, 0, 4, 0, 2140, 74, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8178, 8192, 5988, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7479731576 prefetch_bytes=990904320 reduce_bytes=1431306240 weight_movement_bytes=2422210560 egress_bytes=3741003128 ingress_bytes=6160939008 link_bytes=6160939008 total_nvlink_bytes=9901942136 weight_to_token_ratio=0.323837 prefetch_share_of_ingress=0.160837 dedup_ratio=0.084900 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=13 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=9144 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.071488624e-04 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=399644672 dispatch_meta_bytes=229352 combine_payload_bytes=399644672 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=399874024 ingress_bytes=509745152 link_bytes=509745152 token_bytes=799518696 weight_movement_bytes=110100480 total_nvlink_bytes=909619176 weight_to_token_ratio=0.137708 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.865306580e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6974619448 prefetch_bytes=990904320 reduce_bytes=880803840 weight_movement_bytes=1871708160 egress_bytes=3488433976 ingress_bytes=5357893632 link_bytes=5357893632 total_nvlink_bytes=8846327608 weight_to_token_ratio=0.268360 prefetch_share_of_ingress=0.184943 dedup_ratio=0.137866 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=8 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=10033 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.318258248e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=389007360 dispatch_meta_bytes=229376 combine_payload_bytes=389007360 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=389236736 ingress_bytes=499107840 link_bytes=499107840 token_bytes=778244096 weight_movement_bytes=110100480 total_nvlink_bytes=888344576 weight_to_token_ratio=0.141473 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.680306229e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=377782272 dispatch_meta_bytes=229376 combine_payload_bytes=377782272 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=378011648 ingress_bytes=597983232 link_bytes=597983232 token_bytes=755793920 weight_movement_bytes=220200960 total_nvlink_bytes=975994880 weight_to_token_ratio=0.291351 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.039991192e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7522442624 prefetch_bytes=1101004800 reduce_bytes=1211105280 weight_movement_bytes=2312110080 egress_bytes=3762367872 ingress_bytes=6072184832 link_bytes=6072184832 total_nvlink_bytes=9834552704 weight_to_token_ratio=0.307362 prefetch_share_of_ingress=0.181319 dedup_ratio=0.083127 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=11 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043015 gemm_inner_pad_upper_bound=8890 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.056052813e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7579712536 prefetch_bytes=990904320 reduce_bytes=770703360 weight_movement_bytes=1761607680 egress_bytes=3790994456 ingress_bytes=5550325760 link_bytes=5550325760 total_nvlink_bytes=9341320216 weight_to_token_ratio=0.232411 prefetch_share_of_ingress=0.178531 dedup_ratio=0.075723 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=7 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=9525 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.652929368e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7416267152 prefetch_bytes=1321205760 reduce_bytes=550502400 weight_movement_bytes=1871708160 egress_bytes=3709264272 ingress_bytes=5578711040 link_bytes=5578711040 total_nvlink_bytes=9287975312 weight_to_token_ratio=0.252379 prefetch_share_of_ingress=0.236830 dedup_ratio=0.089352 prefetch_slots=24 unused_prefetch_slots=136 slot_utilization=0.150000 reduce_slots=5 remote_demand=12 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=9525 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.702296039e-05 prefetch_bytes_per_gemm_flop=2.297794118e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6736283664 prefetch_bytes=990904320 reduce_bytes=1101004800 weight_movement_bytes=2091909120 egress_bytes=3369244688 ingress_bytes=5458948096 link_bytes=5458948096 total_nvlink_bytes=8828192784 weight_to_token_ratio=0.310544 prefetch_share_of_ingress=0.181519 dedup_ratio=0.163647 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=10 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=9525 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.494008582e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none - [2026-08-26 15:11:33] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 362.1 | throughput per GPU (TFLOP/s/GPU): 341.1 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.046564E+01 | loss scale: 1.0 | grad norm: 8.627 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45328 payload_rows_remote=40524 dup_rows=20208 dup_rows_remote=18494 dedup_ratio=0.308350 router_tokens_per_expert=[351, 261, 924, 598, 21, 93, 117, 333, 1046, 682, 465, 1410, 11, 137, 6, 922, 677, 539, 112, 3, 1031, 1495, 9, 453, 259, 581, 1035, 666, 13, 348, 347, 1, 366, 102, 232, 207, 12, 0, 19, 731, 1129, 581, 876, 81, 182, 341, 247, 698, 48, 86, 449, 1279, 1193, 87, 42, 600, 253, 551, 1425, 2278, 2365, 941, 421, 0] payload_rows_per_dst=[5882, 5344, 4608, 5798, 5890, 4804, 5450, 7552] dup_rows_per_dst=[2504, 2336, 1040, 2380, 2178, 1714, 2118, 5938] combine_rows_per_src=[5882, 5344, 4608, 5798, 5890, 4804, 5450, 7552] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46056 payload_rows_remote=40474 dup_rows=19480 dup_rows_remote=16966 dedup_ratio=0.297241 router_tokens_per_expert=[582, 90, 342, 658, 20, 25, 194, 139, 623, 754, 901, 1741, 0, 196, 2, 680, 975, 502, 228, 2, 923, 1623, 17, 715, 318, 631, 1036, 542, 8, 430, 403, 0, 42, 4, 259, 140, 13, 0, 49, 797, 751, 950, 1208, 103, 65, 442, 293, 1098, 2, 234, 275, 771, 1533, 419, 5, 809, 272, 393, 1023, 1689, 2373, 1240, 216, 0] payload_rows_per_dst=[5652, 4774, 5130, 6716, 5784, 5226, 5582, 7192] dup_rows_per_dst=[1694, 1538, 1594, 3502, 1570, 2178, 2514, 4890] combine_rows_per_src=[5652, 4774, 5130, 6716, 5784, 5226, 5582, 7192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44848 payload_rows_remote=37670 dup_rows=20688 dup_rows_remote=15446 dedup_ratio=0.315674 router_tokens_per_expert=[400, 164, 546, 661, 8, 30, 353, 338, 765, 549, 340, 1284, 0, 357, 4, 806, 596, 420, 196, 1, 908, 1907, 8, 494, 290, 821, 746, 518, 46, 489, 610, 0, 35, 14, 200, 337, 28, 0, 99, 1477, 864, 590, 1389, 120, 516, 293, 109, 858, 15, 244, 580, 1357, 1955, 229, 3, 417, 207, 423, 747, 1746, 1563, 1406, 291, 1] payload_rows_per_dst=[6286, 4352, 4202, 6238, 5402, 4792, 6398, 7178] dup_rows_per_dst=[2528, 1290, 1044, 3370, 2104, 1908, 3202, 5242] combine_rows_per_src=[6286, 4352, 4202, 6238, 5402, 4792, 6398, 7178] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45464 payload_rows_remote=40084 dup_rows=20072 dup_rows_remote=18362 dedup_ratio=0.306274 router_tokens_per_expert=[566, 155, 967, 840, 15, 19, 170, 283, 958, 435, 476, 990, 0, 356, 0, 763, 1144, 263, 328, 1, 786, 1682, 19, 254, 391, 587, 836, 609, 3, 674, 440, 5, 68, 21, 349, 144, 22, 1, 82, 1110, 947, 944, 865, 145, 155, 760, 145, 689, 5, 212, 258, 1118, 2357, 70, 18, 688, 288, 276, 1038, 1778, 1719, 1336, 145, 0] payload_rows_per_dst=[6192, 5372, 6314, 5380, 5130, 6032, 6464, 4580] dup_rows_per_dst=[3202, 2584, 2832, 1710, 1902, 3268, 2988, 1586] combine_rows_per_src=[6192, 5372, 6314, 5380, 5130, 6032, 6464, 4580] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46018 payload_rows_remote=40130 dup_rows=19518 dup_rows_remote=17712 dedup_ratio=0.297821 router_tokens_per_expert=[387, 105, 330, 575, 54, 39, 436, 155, 591, 444, 1004, 1544, 0, 363, 0, 672, 1346, 231, 358, 0, 915, 2064, 8, 529, 263, 1163, 910, 646, 17, 528, 267, 1, 35, 14, 367, 200, 13, 0, 97, 717, 711, 609, 1568, 45, 83, 520, 111, 910, 14, 115, 145, 655, 2053, 535, 26, 502, 282, 380, 926, 2307, 2404, 325, 154, 0] payload_rows_per_dst=[6088, 6038, 6470, 5460, 5888, 4988, 5760, 5326] dup_rows_per_dst=[2202, 3198, 3440, 2130, 1806, 1882, 2330, 2530] combine_rows_per_src=[6088, 6038, 6470, 5460, 5888, 4988, 5760, 5326] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45124 payload_rows_remote=39268 dup_rows=20412 dup_rows_remote=17286 dedup_ratio=0.311462 router_tokens_per_expert=[578, 108, 309, 431, 47, 46, 235, 199, 720, 787, 214, 1284, 0, 344, 0, 870, 672, 371, 412, 0, 567, 1868, 14, 693, 639, 237, 982, 352, 18, 1266, 649, 0, 19, 30, 143, 181, 16, 2, 98, 1554, 1118, 544, 874, 107, 355, 426, 123, 920, 18, 313, 313, 1276, 2021, 408, 6, 452, 225, 334, 1009, 1916, 1388, 1392, 258, 17] payload_rows_per_dst=[5748, 5856, 6272, 5856, 6358, 5800, 6070, 3164] dup_rows_per_dst=[1894, 3126, 3018, 2430, 3288, 3322, 2812, 522] combine_rows_per_src=[5748, 5856, 6272, 5856, 6358, 5800, 6070, 3164] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=290476032 dispatch_meta_bytes=236072 combine_payload_bytes=290476032 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=290712104 ingress_bytes=510676992 link_bytes=510676992 token_bytes=581188136 weight_movement_bytes=220200960 total_nvlink_bytes=801389096 weight_to_token_ratio=0.378881 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=7296 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.881512810e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=394555392 dispatch_meta_bytes=221072 combine_payload_bytes=394555392 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=394776464 ingress_bytes=834957312 link_bytes=834957312 token_bytes=789331856 weight_movement_bytes=440401920 total_nvlink_bytes=1229733776 weight_to_token_ratio=0.557943 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=20608 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.452128092e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=375316480 dispatch_meta_bytes=229360 combine_payload_bytes=375316480 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=375545840 ingress_bytes=595517440 link_bytes=595517440 token_bytes=750862320 weight_movement_bytes=220200960 total_nvlink_bytes=971063280 weight_to_token_ratio=0.293264 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.035702774e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7543298352 prefetch_bytes=990904320 reduce_bytes=880803840 weight_movement_bytes=1871708160 egress_bytes=3772786992 ingress_bytes=5642219520 link_bytes=5642219520 total_nvlink_bytes=9415006512 weight_to_token_ratio=0.248129 prefetch_share_of_ingress=0.175623 dedup_ratio=0.079010 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=8 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040441 gemm_inner_pad_upper_bound=9906 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.812747731e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7225338704 prefetch_bytes=1211105280 reduce_bytes=550502400 weight_movement_bytes=1761607680 egress_bytes=3613784912 ingress_bytes=5373161472 link_bytes=5373161472 total_nvlink_bytes=8986946384 weight_to_token_ratio=0.243810 prefetch_share_of_ingress=0.225399 dedup_ratio=0.109442 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=5 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=9144 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.344811533e-05 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=395143168 dispatch_meta_bytes=229376 combine_payload_bytes=395143168 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=395372544 ingress_bytes=615344128 link_bytes=615344128 token_bytes=790515712 weight_movement_bytes=220200960 total_nvlink_bytes=1010716672 weight_to_token_ratio=0.278554 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.070184645e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7505940760 prefetch_bytes=990904320 reduce_bytes=1541406720 weight_movement_bytes=2532311040 egress_bytes=3754109208 ingress_bytes=6284142592 link_bytes=6284142592 total_nvlink_bytes=10038251800 weight_to_token_ratio=0.337374 prefetch_share_of_ingress=0.157683 dedup_ratio=0.083261 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=14 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043015 gemm_inner_pad_upper_bound=9271 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.092915753e-04 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6703978712 prefetch_bytes=1101004800 reduce_bytes=880803840 weight_movement_bytes=1981808640 egress_bytes=3353096408 ingress_bytes=5332690944 link_bytes=5332690944 total_nvlink_bytes=8685787352 weight_to_token_ratio=0.295617 prefetch_share_of_ingress=0.206463 dedup_ratio=0.168149 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=8 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=9652 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.274426629e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7510027312 prefetch_bytes=1211105280 reduce_bytes=1321205760 weight_movement_bytes=2532311040 egress_bytes=3756160048 ingress_bytes=6286178304 link_bytes=6286178304 total_nvlink_bytes=10042338352 weight_to_token_ratio=0.337191 prefetch_share_of_ingress=0.192662 dedup_ratio=0.084802 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=12 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043382 gemm_inner_pad_upper_bound=8763 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.093269797e-04 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=305213440 dispatch_meta_bytes=229352 combine_payload_bytes=305213440 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=305442792 ingress_bytes=415313920 link_bytes=415313920 token_bytes=610656232 weight_movement_bytes=110100480 total_nvlink_bytes=720756712 weight_to_token_ratio=0.180299 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.222992143e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63296 payload_rows_remote=55104 dup_rows=2240 dup_rows_remote=2240 dedup_ratio=0.034180 router_tokens_per_expert=[1, 2, 4094, 0, 1, 0, 0, 3878, 4095, 0, 3258, 0, 0, 0, 0, 0, 0, 0, 4089, 2, 0, 1, 612, 0, 0, 2, 0, 0, 1, 0, 2, 0, 0, 0, 0, 1, 0, 0, 4094, 0, 0, 0, 0, 4091, 0, 1, 4094, 0, 0, 0, 447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2] payload_rows_per_dst=[7764, 6516, 8192, 8190, 8068, 8184, 8190, 8192] dup_rows_per_dst=[0, 0, 1216, 6, 0, 124, 894, 0] combine_rows_per_src=[7764, 6516, 8192, 8190, 8068, 8184, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=394985472 dispatch_meta_bytes=229376 combine_payload_bytes=394985472 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=395214848 ingress_bytes=505085952 link_bytes=505085952 token_bytes=790200320 weight_movement_bytes=110100480 total_nvlink_bytes=900300800 weight_to_token_ratio=0.139332 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.784275429e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58042 payload_rows_remote=49850 dup_rows=7494 dup_rows_remote=7300 dedup_ratio=0.114349 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4087, 4094, 0, 0, 0, 0, 0, 1, 3, 2, 0, 0, 361, 0, 1, 4076, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 4094, 2, 821, 2836, 4094, 4094, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 2, 0, 2, 2, 0, 0, 4094, 0, 0, 97, 0] payload_rows_per_dst=[8188, 8176, 724, 8190, 8190, 8190, 8192, 8192] dup_rows_per_dst=[0, 0, 10, 0, 7284, 0, 6, 194] combine_rows_per_src=[8188, 8176, 724, 8190, 8190, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=357324800 dispatch_meta_bytes=228600 combine_payload_bytes=357324800 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=357553400 ingress_bytes=687626240 link_bytes=687626240 token_bytes=714878200 weight_movement_bytes=330301440 total_nvlink_bytes=1045179640 weight_to_token_ratio=0.462039 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=31360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.195895126e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58276 payload_rows_remote=50084 dup_rows=7260 dup_rows_remote=7188 dedup_ratio=0.110779 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 1, 4095, 0, 1, 0, 1, 1, 0, 0, 0, 4094, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 250, 3941, 0, 290, 0, 0, 410, 0, 0, 1, 0, 4094, 0, 0, 0, 0, 4096, 0, 3293, 0, 0, 4, 0, 2, 0, 1, 1, 0] payload_rows_per_dst=[8192, 8192, 8184, 8188, 1068, 8190, 8070, 8192] dup_rows_per_dst=[0, 6, 0, 0, 14, 826, 6342, 72] combine_rows_per_src=[8192, 8192, 8184, 8188, 1068, 8190, 8070, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=359002112 dispatch_meta_bytes=229088 combine_payload_bytes=359002112 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=359231200 ingress_bytes=579203072 link_bytes=579203072 token_bytes=718233312 weight_movement_bytes=220200960 total_nvlink_bytes=938434272 weight_to_token_ratio=0.306587 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.007329405e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6902251344 prefetch_bytes=990904320 reduce_bytes=880803840 weight_movement_bytes=1871708160 egress_bytes=3452249936 ingress_bytes=5321709568 link_bytes=5321709568 total_nvlink_bytes=8773959504 weight_to_token_ratio=0.271174 prefetch_share_of_ingress=0.186200 dedup_ratio=0.146002 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=8 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=9906 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.255328210e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none + [2026-08-26 15:59:19] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 363.6 | throughput per GPU (TFLOP/s/GPU): 339.6 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.046476E+01 | loss scale: 1.0 | grad norm: 8.604 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44502 payload_rows_remote=37392 dup_rows=21034 dup_rows_remote=15916 dedup_ratio=0.320953 router_tokens_per_expert=[398, 160, 513, 696, 7, 29, 348, 309, 728, 583, 328, 1332, 0, 377, 4, 810, 623, 390, 196, 1, 914, 1874, 7, 485, 274, 793, 766, 483, 46, 449, 614, 0, 36, 15, 204, 329, 30, 0, 109, 1492, 904, 649, 1374, 120, 511, 267, 114, 855, 16, 243, 576, 1420, 1951, 224, 3, 426, 207, 375, 735, 1750, 1581, 1437, 277, 1] payload_rows_per_dst=[5584, 4358, 4198, 6486, 5432, 4890, 6444, 7110] dup_rows_per_dst=[2084, 1302, 1034, 4112, 2160, 1950, 3274, 5118] combine_rows_per_src=[5584, 4358, 4198, 6486, 5432, 4890, 6444, 7110] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45344 payload_rows_remote=40074 dup_rows=20192 dup_rows_remote=18554 dedup_ratio=0.308105 router_tokens_per_expert=[564, 153, 923, 916, 13, 17, 164, 254, 922, 457, 455, 1046, 0, 371, 0, 758, 1171, 221, 300, 1, 784, 1629, 18, 251, 383, 565, 852, 563, 4, 644, 438, 5, 71, 22, 349, 137, 29, 1, 80, 1080, 986, 1109, 843, 141, 156, 727, 156, 685, 5, 206, 229, 1166, 2366, 67, 18, 684, 282, 238, 1040, 1785, 1744, 1385, 139, 0] payload_rows_per_dst=[6198, 5384, 6138, 5270, 5158, 6064, 6514, 4618] dup_rows_per_dst=[3468, 2634, 2612, 1638, 1868, 3454, 2968, 1550] combine_rows_per_src=[6198, 5384, 6138, 5270, 5158, 6064, 6514, 4618] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45032 payload_rows_remote=40054 dup_rows=20504 dup_rows_remote=18704 dedup_ratio=0.312866 router_tokens_per_expert=[347, 257, 851, 648, 21, 87, 115, 299, 1024, 708, 457, 1495, 11, 140, 6, 924, 695, 489, 106, 3, 1052, 1429, 9, 444, 256, 580, 1043, 628, 16, 321, 333, 1, 377, 106, 227, 204, 14, 0, 19, 721, 1179, 692, 843, 80, 179, 307, 260, 692, 49, 86, 443, 1363, 1183, 85, 43, 594, 254, 493, 1445, 2261, 2386, 975, 413, 0] payload_rows_per_dst=[5126, 5228, 4870, 5786, 5936, 4978, 5528, 7580] dup_rows_per_dst=[1810, 2286, 1402, 2752, 2172, 1800, 2164, 6118] combine_rows_per_src=[5126, 5228, 4870, 5786, 5936, 4978, 5528, 7580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45900 payload_rows_remote=39424 dup_rows=19636 dup_rows_remote=16786 dedup_ratio=0.299622 router_tokens_per_expert=[355, 148, 428, 551, 11, 45, 422, 250, 1132, 753, 367, 731, 1, 343, 9, 746, 829, 339, 375, 13, 1000, 1749, 18, 340, 393, 719, 870, 406, 35, 466, 417, 0, 76, 25, 526, 175, 19, 0, 174, 875, 1028, 773, 1550, 92, 94, 369, 106, 969, 23, 225, 249, 1212, 1239, 431, 81, 704, 479, 290, 857, 2076, 2188, 1242, 354, 6] payload_rows_per_dst=[6358, 5454, 6476, 5116, 5804, 6228, 5804, 4660] dup_rows_per_dst=[3218, 2710, 2850, 1496, 2312, 2730, 2524, 1796] combine_rows_per_src=[6358, 5454, 6476, 5116, 5804, 6228, 5804, 4660] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44916 payload_rows_remote=39066 dup_rows=20620 dup_rows_remote=17704 dedup_ratio=0.314636 router_tokens_per_expert=[585, 104, 299, 462, 41, 45, 233, 181, 667, 813, 209, 1364, 0, 361, 0, 869, 694, 340, 395, 0, 570, 1785, 13, 657, 620, 238, 1005, 313, 18, 1210, 637, 0, 19, 30, 145, 181, 17, 2, 100, 1551, 1155, 632, 863, 107, 356, 394, 135, 914, 21, 306, 304, 1346, 2035, 410, 6, 463, 228, 286, 1016, 1907, 1402, 1447, 245, 17] payload_rows_per_dst=[5578, 5850, 6592, 5758, 6348, 5804, 5832, 3154] dup_rows_per_dst=[2358, 2916, 3580, 2324, 3332, 3086, 2486, 538] combine_rows_per_src=[5578, 5850, 6592, 5758, 6348, 5804, 5832, 3154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45174 payload_rows_remote=39582 dup_rows=20362 dup_rows_remote=17796 dedup_ratio=0.310699 router_tokens_per_expert=[584, 92, 325, 712, 13, 23, 169, 120, 581, 806, 878, 1815, 0, 204, 3, 687, 998, 451, 228, 2, 931, 1555, 19, 688, 309, 620, 1063, 476, 9, 408, 411, 0, 40, 10, 262, 138, 17, 0, 55, 774, 785, 1078, 1186, 101, 64, 400, 307, 1098, 2, 230, 277, 820, 1523, 409, 4, 814, 271, 342, 1019, 1689, 2383, 1291, 199, 0] payload_rows_per_dst=[4938, 4770, 5092, 6220, 5798, 5352, 5592, 7412] dup_rows_per_dst=[1510, 1548, 1542, 3482, 1560, 2314, 2566, 5840] combine_rows_per_src=[4938, 4770, 5092, 6220, 5798, 5352, 5592, 7412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45868 payload_rows_remote=39954 dup_rows=19668 dup_rows_remote=17852 dedup_ratio=0.300110 router_tokens_per_expert=[388, 98, 311, 632, 34, 37, 424, 143, 554, 470, 981, 1603, 0, 378, 0, 667, 1376, 202, 351, 0, 917, 1971, 7, 510, 257, 1147, 928, 605, 18, 502, 270, 2, 38, 14, 367, 202, 19, 0, 102, 693, 783, 702, 1541, 45, 79, 484, 113, 912, 14, 111, 138, 716, 2041, 538, 27, 518, 276, 337, 929, 2317, 2430, 359, 140, 0] payload_rows_per_dst=[6040, 6046, 6754, 5412, 5914, 5048, 5812, 4842] dup_rows_per_dst=[2694, 3260, 3914, 2046, 1816, 1970, 2394, 1574] combine_rows_per_src=[6040, 6046, 6754, 5412, 5914, 5048, 5812, 4842] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43842 payload_rows_remote=38096 dup_rows=21694 dup_rows_remote=18868 dedup_ratio=0.331024 router_tokens_per_expert=[565, 104, 473, 733, 11, 39, 274, 263, 796, 640, 450, 1268, 0, 255, 0, 920, 1127, 344, 368, 13, 1029, 1074, 25, 261, 391, 683, 1023, 704, 48, 467, 496, 7, 158, 12, 262, 171, 56, 1, 90, 1119, 775, 913, 1049, 140, 312, 665, 73, 1041, 28, 140, 525, 782, 1566, 522, 10, 810, 185, 462, 1045, 1790, 1789, 1305, 121, 0] payload_rows_per_dst=[5746, 6522, 6030, 5590, 6412, 6202, 4102, 3238] dup_rows_per_dst=[2826, 5268, 2452, 2048, 3514, 3666, 1532, 388] combine_rows_per_src=[5746, 6522, 6030, 5590, 6412, 6202, 4102, 3238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=268025856 dispatch_meta_bytes=213232 combine_payload_bytes=268025856 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=268239088 ingress_bytes=708427776 link_bytes=708427776 token_bytes=536264944 weight_movement_bytes=440401920 total_nvlink_bytes=976666864 weight_to_token_ratio=0.821239 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=8192 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.232072419e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=287250432 dispatch_meta_bytes=234512 combine_payload_bytes=287250432 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=287484944 ingress_bytes=397350912 link_bytes=397350912 token_bytes=574735376 weight_movement_bytes=110100480 total_nvlink_bytes=684835856 weight_to_token_ratio=0.191567 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=7552 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.910585890e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=290117632 dispatch_meta_bytes=229760 combine_payload_bytes=290117632 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=290347392 ingress_bytes=510318592 link_bytes=510318592 token_bytes=580465024 weight_movement_bytes=220200960 total_nvlink_bytes=800665984 weight_to_token_ratio=0.379353 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=12032 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.875279645e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=270018560 dispatch_meta_bytes=212464 combine_payload_bytes=270018560 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=270231024 ingress_bytes=710420480 link_bytes=710420480 token_bytes=540249584 weight_movement_bytes=440401920 total_nvlink_bytes=980651504 weight_to_token_ratio=0.815182 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=8192 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.235538059e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45320 payload_rows_remote=38842 dup_rows=20216 dup_rows_remote=16620 dedup_ratio=0.308472 router_tokens_per_expert=[356, 155, 449, 494, 13, 49, 428, 267, 1174, 698, 376, 706, 1, 321, 8, 739, 800, 383, 392, 9, 999, 1819, 18, 352, 395, 732, 846, 443, 35, 488, 411, 0, 70, 22, 528, 184, 17, 0, 170, 887, 991, 697, 1578, 98, 101, 402, 98, 975, 24, 232, 248, 1149, 1258, 440, 84, 682, 479, 318, 863, 2084, 2172, 1196, 359, 6] payload_rows_per_dst=[5814, 5402, 6478, 5162, 5924, 6372, 5710, 4458] dup_rows_per_dst=[2246, 2644, 3596, 1538, 2486, 3508, 2524, 1674] combine_rows_per_src=[5814, 5402, 6478, 5162, 5924, 6372, 5710, 4458] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=287322112 dispatch_meta_bytes=233784 combine_payload_bytes=287322112 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=287555896 ingress_bytes=397422592 link_bytes=397422592 token_bytes=574878008 weight_movement_bytes=110100480 total_nvlink_bytes=684978488 weight_to_token_ratio=0.191520 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=7424 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.911832523e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43994 payload_rows_remote=38464 dup_rows=21542 dup_rows_remote=19468 dedup_ratio=0.328705 router_tokens_per_expert=[564, 101, 511, 643, 12, 40, 280, 292, 833, 600, 473, 1203, 0, 250, 0, 910, 1098, 371, 390, 12, 1024, 1136, 24, 268, 406, 712, 1012, 757, 46, 495, 498, 7, 152, 12, 261, 175, 47, 1, 85, 1129, 733, 810, 1060, 147, 315, 709, 66, 1031, 25, 142, 532, 756, 1572, 526, 10, 816, 184, 502, 1048, 1785, 1765, 1286, 118, 0] payload_rows_per_dst=[5530, 6496, 6292, 5690, 6366, 6224, 4104, 3292] dup_rows_per_dst=[2074, 5186, 3206, 2176, 3460, 3518, 1510, 412] combine_rows_per_src=[5530, 6496, 6292, 5690, 6366, 6224, 4104, 3292] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=287651840 dispatch_meta_bytes=231368 combine_payload_bytes=287651840 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=287883208 ingress_bytes=507852800 link_bytes=507852800 token_bytes=575535048 weight_movement_bytes=220200960 total_nvlink_bytes=795736008 weight_to_token_ratio=0.382602 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.033088 gemm_active_groups=11 gemm_peak_group_rows=15872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.832395466e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=281473024 dispatch_meta_bytes=226216 combine_payload_bytes=281473024 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=281699240 ingress_bytes=501673984 link_bytes=501673984 token_bytes=563172264 weight_movement_bytes=220200960 total_nvlink_bytes=783373224 weight_to_token_ratio=0.391001 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=6784 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.724935694e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278419456 dispatch_meta_bytes=221848 combine_payload_bytes=278419456 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278641304 ingress_bytes=608720896 link_bytes=608720896 token_bytes=557060760 weight_movement_bytes=330301440 total_nvlink_bytes=887362200 weight_to_token_ratio=0.592936 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=9216 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058665756e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58812 payload_rows_remote=50794 dup_rows=6724 dup_rows_remote=6718 dedup_ratio=0.102600 router_tokens_per_expert=[0, 0, 0, 715, 2, 0, 0, 16, 0, 0, 30, 0, 3200, 0, 0, 0, 0, 1, 1, 4094, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 20, 2, 0, 3, 3894, 1, 0, 0, 0, 0, 57, 4008, 0, 4, 0, 4093, 0, 0, 0, 0, 1, 0, 2, 1232, 0, 0, 0, 0, 1489, 4092, 0, 1724, 0, 2, 0] payload_rows_per_dst=[8192, 6420, 8190, 8184, 7812, 8018, 4362, 7634] dup_rows_per_dst=[1460, 40, 2, 30, 98, 6, 466, 4622] combine_rows_per_src=[8192, 6420, 8190, 8184, 7812, 8018, 4362, 7634] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=275709952 dispatch_meta_bytes=231728 combine_payload_bytes=275709952 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=275941680 ingress_bytes=495910912 link_bytes=495910912 token_bytes=551651632 weight_movement_bytes=220200960 total_nvlink_bytes=771852592 weight_to_token_ratio=0.399167 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.040441 gemm_active_groups=11 gemm_peak_group_rows=13696 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.624706393e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55708 payload_rows_remote=53022 dup_rows=9828 dup_rows_remote=9814 dedup_ratio=0.149963 router_tokens_per_expert=[0, 0, 0, 511, 2, 0, 0, 0, 2, 0, 1, 0, 3097, 0, 0, 0, 0, 0, 0, 4093, 3, 3, 0, 0, 4087, 0, 0, 0, 0, 1, 0, 0, 0, 3376, 0, 0, 0, 3, 0, 1, 4066, 0, 0, 1, 4096, 0, 0, 0, 2, 1, 0, 0, 1339, 6, 0, 2, 0, 1970, 4095, 0, 2010, 0, 0, 0] payload_rows_per_dst=[7364, 6200, 8190, 8174, 6760, 8144, 2686, 8190] dup_rows_per_dst=[882, 0, 8, 2, 0, 962, 14, 7960] combine_rows_per_src=[7364, 6200, 8190, 8174, 6760, 8144, 2686, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49648 payload_rows_remote=41460 dup_rows=15888 dup_rows_remote=8216 dedup_ratio=0.242432 router_tokens_per_expert=[0, 1, 0, 559, 0, 0, 0, 11, 0, 0, 2, 1, 2768, 0, 0, 0, 0, 0, 0, 4091, 6, 0, 1, 6, 4069, 0, 0, 0, 0, 6, 2, 1, 0, 3625, 1, 0, 0, 4, 0, 34, 4064, 0, 4, 0, 4094, 1, 0, 1, 0, 0, 0, 0, 1507, 3, 0, 2, 0, 1740, 4091, 0, 2069, 0, 4, 0] payload_rows_per_dst=[1140, 5540, 8140, 8146, 7280, 8190, 3024, 8188] dup_rows_per_dst=[2, 2, 16, 10, 48, 8138, 0, 7672] combine_rows_per_src=[1140, 5540, 8140, 8146, 7280, 8190, 3024, 8188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58902 payload_rows_remote=50746 dup_rows=6634 dup_rows_remote=6630 dedup_ratio=0.101227 router_tokens_per_expert=[0, 0, 0, 326, 0, 1, 2, 50, 1, 0, 3, 0, 2957, 1, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 1, 4078, 0, 0, 0, 0, 2, 0, 0, 0, 2929, 0, 0, 0, 0, 0, 39, 4066, 2, 0, 0, 4095, 0, 0, 0, 0, 2, 1, 0, 1505, 0, 0, 0, 0, 2771, 4093, 0, 1746, 0, 0, 1] payload_rows_per_dst=[8192, 6934, 8192, 8156, 5902, 8136, 8192, 5198] dup_rows_per_dst=[756, 2128, 2, 4, 34, 0, 3010, 700] combine_rows_per_src=[8192, 6934, 8192, 8156, 5902, 8136, 8192, 5198] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60608 payload_rows_remote=54172 dup_rows=4928 dup_rows_remote=4926 dedup_ratio=0.075195 router_tokens_per_expert=[0, 0, 0, 630, 0, 0, 0, 19, 0, 0, 0, 0, 3172, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 2, 0, 0, 0, 3214, 0, 0, 0, 0, 0, 5, 4090, 0, 0, 0, 4095, 0, 3, 0, 0, 0, 0, 0, 1039, 0, 0, 0, 0, 2339, 4096, 0, 1877, 0, 0, 0] payload_rows_per_dst=[8192, 6344, 8190, 8184, 6436, 8180, 8192, 6890] dup_rows_per_dst=[1296, 0, 0, 4, 2, 6, 2078, 1542] combine_rows_per_src=[8192, 6344, 8190, 8184, 6436, 8180, 8192, 6890] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=364091392 dispatch_meta_bytes=230048 combine_payload_bytes=364091392 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=364321440 ingress_bytes=694392832 link_bytes=694392832 token_bytes=728412832 weight_movement_bytes=330301440 total_nvlink_bytes=1058714272 weight_to_token_ratio=0.453454 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=26624 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.207663343e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=380061696 dispatch_meta_bytes=251344 combine_payload_bytes=380061696 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=380313040 ingress_bytes=490162176 link_bytes=490162176 token_bytes=760374736 weight_movement_bytes=110100480 total_nvlink_bytes=870475216 weight_to_token_ratio=0.144798 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=21632 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.524726419e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=297185280 dispatch_meta_bytes=198704 combine_payload_bytes=297185280 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=297383984 ingress_bytes=737587200 link_bytes=737587200 token_bytes=594569264 weight_movement_bytes=440401920 total_nvlink_bytes=1034971184 weight_to_token_ratio=0.740708 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=12928 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.282785453e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=363747328 dispatch_meta_bytes=229504 combine_payload_bytes=363747328 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=363976832 ingress_bytes=473847808 link_bytes=473847808 token_bytes=727724160 weight_movement_bytes=110100480 total_nvlink_bytes=837824640 weight_to_token_ratio=0.151294 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.240992727e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55864 payload_rows_remote=48226 dup_rows=9672 dup_rows_remote=6742 dedup_ratio=0.147583 router_tokens_per_expert=[0, 0, 0, 416, 0, 0, 0, 11, 0, 0, 5, 0, 3187, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4088, 0, 3, 0, 0, 17, 0, 0, 0, 3182, 0, 0, 0, 0, 0, 21, 4052, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1611, 0, 0, 0, 0, 2092, 4081, 0, 1809, 0, 0, 0] payload_rows_per_dst=[8192, 7638, 8192, 8178, 7470, 4918, 8166, 3110] dup_rows_per_dst=[854, 2930, 0, 38, 2630, 2, 3218, 0] combine_rows_per_src=[8192, 7638, 8192, 8178, 7470, 4918, 8166, 3110] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=388304896 dispatch_meta_bytes=236392 combine_payload_bytes=388304896 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=388541288 ingress_bytes=498405376 link_bytes=498405376 token_bytes=776846184 weight_movement_bytes=110100480 total_nvlink_bytes=886946664 weight_to_token_ratio=0.141728 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=26752 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.668089225e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58842 payload_rows_remote=50650 dup_rows=6694 dup_rows_remote=6688 dedup_ratio=0.102142 router_tokens_per_expert=[0, 1, 2, 481, 0, 0, 3, 12, 0, 0, 36, 0, 2693, 0, 0, 0, 3, 0, 0, 4095, 0, 1, 0, 0, 4074, 0, 0, 0, 0, 41, 0, 0, 0, 3269, 0, 0, 0, 0, 0, 8, 4056, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 1, 0, 1205, 7, 0, 0, 1, 2685, 4091, 0, 1905, 1, 3, 0] payload_rows_per_dst=[8192, 7616, 8192, 8176, 6544, 8114, 8190, 3818] dup_rows_per_dst=[992, 3212, 6, 54, 10, 0, 2418, 2] combine_rows_per_src=[8192, 7616, 8192, 8176, 6544, 8114, 8190, 3818] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51250 payload_rows_remote=43058 dup_rows=14286 dup_rows_remote=12648 dedup_ratio=0.217987 router_tokens_per_expert=[0, 0, 0, 702, 0, 0, 0, 117, 0, 0, 1, 0, 2841, 0, 0, 6, 0, 3, 1, 4095, 0, 0, 1, 0, 4083, 4, 0, 0, 1, 16, 0, 2, 0, 3177, 0, 0, 1, 0, 0, 29, 4048, 0, 2, 1, 4096, 2, 3, 0, 0, 0, 0, 2, 1666, 3, 0, 1, 5, 2418, 4088, 0, 1353, 0, 0, 0] payload_rows_per_dst=[8192, 7624, 8192, 8162, 8168, 2718, 8184, 10] dup_rows_per_dst=[1638, 2908, 24, 34, 6342, 4, 3336, 0] combine_rows_per_src=[8192, 7624, 8192, 8162, 8168, 2718, 8184, 10] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=345683968 dispatch_meta_bytes=219872 combine_payload_bytes=345683968 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=345903840 ingress_bytes=455784448 link_bytes=455784448 token_bytes=691587808 weight_movement_bytes=110100480 total_nvlink_bytes=801688288 weight_to_token_ratio=0.159200 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=23936 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.926841187e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64510 payload_rows_remote=56320 dup_rows=1026 dup_rows_remote=1026 dedup_ratio=0.015656 router_tokens_per_expert=[0, 0, 0, 130, 4096, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 349, 0, 2, 4091, 0, 0, 0, 0, 0, 2, 3, 0, 0, 21, 0, 3, 4095, 0, 0, 4093, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 6, 0, 3604, 0, 0, 0] payload_rows_per_dst=[8192, 8170, 8182, 8192, 8188, 8190, 8186, 7210] dup_rows_per_dst=[260, 696, 8, 52, 0, 0, 0, 10] combine_rows_per_src=[8192, 8170, 8182, 8192, 8188, 8190, 8186, 7210] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63864 payload_rows_remote=55676 dup_rows=1672 dup_rows_remote=1672 dedup_ratio=0.025513 router_tokens_per_expert=[0, 0, 0, 58, 4092, 2, 0, 0, 4093, 0, 0, 2, 0, 0, 415, 0, 0, 4090, 1, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 5, 4094, 2, 0, 4094, 0, 0, 0, 4, 2, 2, 1, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 3623, 0, 0, 0] payload_rows_per_dst=[8188, 7370, 8180, 8192, 8192, 8182, 8188, 7372] dup_rows_per_dst=[116, 718, 4, 16, 12, 0, 0, 806] combine_rows_per_src=[8188, 7370, 8180, 8192, 8192, 8182, 8188, 7372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57962 payload_rows_remote=49786 dup_rows=7574 dup_rows_remote=588 dedup_ratio=0.115570 router_tokens_per_expert=[0, 0, 0, 109, 4093, 6, 0, 0, 4080, 0, 0, 0, 0, 0, 477, 0, 0, 4089, 0, 0, 0, 2, 0, 0, 0, 0, 6, 28, 0, 0, 4090, 0, 2, 4090, 0, 0, 0, 2, 0, 1, 0, 4090, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 3, 2, 3495, 0, 0, 1] payload_rows_per_dst=[6432, 2460, 8180, 8172, 8160, 8190, 8192, 8176] dup_rows_per_dst=[192, 286, 2, 56, 2, 22, 28, 6986] combine_rows_per_src=[6432, 2460, 8180, 8172, 8160, 8190, 8192, 8176] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=308639744 dispatch_meta_bytes=222824 combine_payload_bytes=308639744 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=308862568 ingress_bytes=418740224 link_bytes=418740224 token_bytes=617502312 weight_movement_bytes=110100480 total_nvlink_bytes=727602792 weight_to_token_ratio=0.178300 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=28288 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.282581205e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=363059200 dispatch_meta_bytes=229352 combine_payload_bytes=363059200 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=363288552 ingress_bytes=583260160 link_bytes=583260160 token_bytes=726347752 weight_movement_bytes=220200960 total_nvlink_bytes=946548712 weight_to_token_ratio=0.303162 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.029412 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.014385348e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64692 payload_rows_remote=56500 dup_rows=844 dup_rows_remote=812 dedup_ratio=0.012878 router_tokens_per_expert=[0, 0, 0, 59, 4095, 2, 0, 1, 4095, 0, 0, 0, 0, 0, 342, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 1, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 3681, 0, 0, 2] payload_rows_per_dst=[8192, 8190, 8178, 8192, 8192, 8192, 8190, 7366] dup_rows_per_dst=[122, 684, 0, 32, 4, 2, 0, 0] combine_rows_per_src=[8192, 8190, 8178, 8192, 8192, 8192, 8190, 7366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=403701760 dispatch_meta_bytes=229384 combine_payload_bytes=403701760 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=403931144 ingress_bytes=513802240 link_bytes=513802240 token_bytes=807632904 weight_movement_bytes=110100480 total_nvlink_bytes=917733384 weight_to_token_ratio=0.136325 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.935866013e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64172 payload_rows_remote=55980 dup_rows=1364 dup_rows_remote=1362 dedup_ratio=0.020813 router_tokens_per_expert=[0, 0, 0, 98, 4096, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 581, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 3418, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8188, 8192, 8192, 8186, 8192, 6842] dup_rows_per_dst=[196, 1158, 0, 8, 2, 0, 0, 0] combine_rows_per_src=[8192, 8188, 8188, 8192, 8192, 8186, 8192, 6842] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=399085568 dispatch_meta_bytes=229392 combine_payload_bytes=399085568 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=399314960 ingress_bytes=509186048 link_bytes=509186048 token_bytes=798400528 weight_movement_bytes=110100480 total_nvlink_bytes=908501008 weight_to_token_ratio=0.137901 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.855582842e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=356866048 dispatch_meta_bytes=201496 combine_payload_bytes=356866048 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=357067544 ingress_bytes=466966528 link_bytes=466966528 token_bytes=713933592 weight_movement_bytes=110100480 total_nvlink_bytes=824034072 weight_to_token_ratio=0.154217 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=28288 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.121315950e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64418 payload_rows_remote=56238 dup_rows=1118 dup_rows_remote=160 dedup_ratio=0.017059 router_tokens_per_expert=[0, 0, 0, 72, 4096, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 487, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 3544, 0, 0, 0] payload_rows_per_dst=[8192, 8180, 8190, 8192, 8192, 8192, 8192, 7088] dup_rows_per_dst=[144, 958, 0, 14, 0, 0, 0, 2] combine_rows_per_src=[8192, 8180, 8190, 8192, 8192, 8192, 8192, 7088] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=404992000 dispatch_meta_bytes=229248 combine_payload_bytes=404992000 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=405221248 ingress_bytes=845393920 link_bytes=845393920 token_bytes=810213248 weight_movement_bytes=440401920 total_nvlink_bytes=1250615168 weight_to_token_ratio=0.543563 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.470279070e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=401264640 dispatch_meta_bytes=229368 combine_payload_bytes=401264640 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=401494008 ingress_bytes=621465600 link_bytes=621465600 token_bytes=802758648 weight_movement_bytes=220200960 total_nvlink_bytes=1022959608 weight_to_token_ratio=0.274305 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.080830892e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63926 payload_rows_remote=55734 dup_rows=1610 dup_rows_remote=1208 dedup_ratio=0.024567 router_tokens_per_expert=[0, 0, 0, 166, 4095, 0, 0, 0, 4074, 1, 0, 0, 5, 0, 551, 0, 3, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 4095, 0, 0, 4095, 0, 0, 0, 0, 0, 4, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 4089, 0, 0, 0, 3, 4, 3413, 0, 0, 0] payload_rows_per_dst=[8192, 8172, 8148, 8022, 8186, 8190, 8180, 6836] dup_rows_per_dst=[402, 1090, 102, 0, 8, 2, 2, 4] combine_rows_per_src=[8192, 8172, 8148, 8022, 8186, 8190, 8180, 6836] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64028 payload_rows_remote=55854 dup_rows=1508 dup_rows_remote=1508 dedup_ratio=0.023010 router_tokens_per_expert=[0, 0, 0, 168, 4094, 0, 0, 0, 4033, 0, 0, 0, 0, 0, 575, 0, 0, 4086, 0, 1, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4093, 0, 1, 4095, 0, 1, 0, 1, 2, 3, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 14, 3, 3402, 0, 0, 0] payload_rows_per_dst=[8192, 8078, 8174, 8192, 8192, 8188, 8186, 6826] dup_rows_per_dst=[332, 1138, 0, 12, 14, 0, 0, 12] combine_rows_per_src=[8192, 8078, 8174, 8192, 8192, 8188, 8186, 6826] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=403113984 dispatch_meta_bytes=225592 combine_payload_bytes=403113984 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=403339576 ingress_bytes=623314944 link_bytes=623314944 token_bytes=806453560 weight_movement_bytes=220200960 total_nvlink_bytes=1026654520 weight_to_token_ratio=0.273049 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=28160 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.084047205e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=287107072 dispatch_meta_bytes=235032 combine_payload_bytes=287107072 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=287342104 ingress_bytes=617408512 link_bytes=617408512 token_bytes=574449176 weight_movement_bytes=330301440 total_nvlink_bytes=904750616 weight_to_token_ratio=0.574988 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=8832 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.073774949e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=282591232 dispatch_meta_bytes=224840 combine_payload_bytes=282591232 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=282816072 ingress_bytes=502792192 link_bytes=502792192 token_bytes=565407304 weight_movement_bytes=220200960 total_nvlink_bytes=785608264 weight_to_token_ratio=0.389455 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=8576 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.744383170e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280025088 dispatch_meta_bytes=227080 combine_payload_bytes=280025088 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280252168 ingress_bytes=500226048 link_bytes=500226048 token_bytes=560277256 weight_movement_bytes=220200960 total_nvlink_bytes=780478216 weight_to_token_ratio=0.393021 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=6528 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.699753705e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=283723776 dispatch_meta_bytes=229512 combine_payload_bytes=283723776 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=283953288 ingress_bytes=503924736 link_bytes=503924736 token_bytes=567677064 weight_movement_bytes=220200960 total_nvlink_bytes=787878024 weight_to_token_ratio=0.387898 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=11648 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.764079973e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=286390272 dispatch_meta_bytes=231224 combine_payload_bytes=286390272 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=286621496 ingress_bytes=506591232 link_bytes=506591232 token_bytes=573011768 weight_movement_bytes=220200960 total_nvlink_bytes=793212728 weight_to_token_ratio=0.384287 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.036765 gemm_active_groups=11 gemm_peak_group_rows=16000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.810454724e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=273072128 dispatch_meta_bytes=227856 combine_payload_bytes=273072128 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=273299984 ingress_bytes=493273088 link_bytes=493273088 token_bytes=546372112 weight_movement_bytes=220200960 total_nvlink_bytes=766573072 weight_to_token_ratio=0.403024 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.040441 gemm_active_groups=11 gemm_peak_group_rows=9344 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.578830295e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52368 payload_rows_remote=44180 dup_rows=13168 dup_rows_remote=3234 dedup_ratio=0.200928 router_tokens_per_expert=[0, 2, 0, 657, 0, 0, 0, 41, 0, 0, 2, 1, 2421, 0, 0, 0, 0, 0, 0, 4091, 7, 0, 1, 6, 4082, 0, 0, 0, 0, 8, 2, 2, 0, 3395, 1, 0, 0, 4, 0, 38, 4069, 0, 5, 0, 4094, 1, 0, 1, 0, 0, 0, 0, 1568, 2, 0, 2, 0, 2027, 4090, 0, 2144, 0, 4, 0] payload_rows_per_dst=[1396, 4846, 8152, 8122, 6826, 6648, 8190, 8188] dup_rows_per_dst=[4, 2, 18, 14, 50, 4, 3142, 9934] combine_rows_per_src=[1396, 4846, 8152, 8122, 6826, 6648, 8190, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58798 payload_rows_remote=50638 dup_rows=6738 dup_rows_remote=6736 dedup_ratio=0.102814 router_tokens_per_expert=[0, 0, 0, 412, 0, 1, 3, 129, 1, 0, 4, 0, 2733, 1, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 1, 4080, 0, 0, 0, 0, 1, 0, 0, 0, 2422, 0, 0, 0, 0, 0, 37, 4068, 2, 0, 0, 4095, 0, 0, 0, 0, 2, 1, 0, 1644, 0, 0, 0, 0, 3090, 4093, 0, 1851, 0, 0, 1] payload_rows_per_dst=[8190, 5470, 8192, 8160, 5944, 8140, 8192, 6510] dup_rows_per_dst=[1086, 8, 2, 2, 758, 0, 3292, 1590] combine_rows_per_src=[8190, 5470, 8192, 8160, 5944, 8140, 8192, 6510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60326 payload_rows_remote=52320 dup_rows=5210 dup_rows_remote=5204 dedup_ratio=0.079498 router_tokens_per_expert=[0, 0, 0, 797, 2, 0, 0, 79, 0, 0, 29, 0, 2966, 0, 0, 0, 0, 1, 1, 4094, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 20, 2, 0, 3, 3679, 1, 0, 0, 0, 0, 55, 4002, 0, 4, 0, 4093, 0, 0, 0, 0, 1, 0, 2, 1309, 0, 0, 0, 0, 1706, 4092, 0, 1741, 0, 2, 0] payload_rows_per_dst=[8192, 5956, 8190, 8188, 7392, 8006, 8192, 6210] dup_rows_per_dst=[1748, 34, 2, 30, 84, 6, 2618, 688] combine_rows_per_src=[8192, 5956, 8190, 8188, 7392, 8006, 8192, 6210] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54962 payload_rows_remote=46770 dup_rows=10574 dup_rows_remote=10568 dedup_ratio=0.161346 router_tokens_per_expert=[0, 1, 2, 621, 0, 0, 3, 53, 0, 0, 40, 0, 2455, 0, 0, 0, 3, 0, 0, 4095, 0, 1, 0, 0, 4074, 0, 0, 0, 0, 37, 0, 0, 0, 3019, 0, 0, 0, 0, 0, 8, 4065, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 1, 0, 1271, 6, 0, 0, 2, 2851, 4091, 0, 1972, 0, 3, 0] payload_rows_per_dst=[8190, 7314, 8192, 8174, 7816, 8188, 3134, 3954] dup_rows_per_dst=[1352, 3138, 6, 48, 3940, 1820, 270, 0] combine_rows_per_src=[8190, 7314, 8192, 8174, 7816, 8188, 3134, 3954] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55636 payload_rows_remote=47460 dup_rows=9900 dup_rows_remote=4112 dedup_ratio=0.151062 router_tokens_per_expert=[0, 0, 0, 578, 0, 0, 0, 32, 0, 0, 5, 0, 2919, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4088, 0, 7, 0, 0, 14, 0, 0, 0, 2805, 0, 0, 0, 0, 0, 22, 4058, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1747, 0, 0, 0, 0, 2348, 4082, 0, 1870, 0, 0, 0] payload_rows_per_dst=[8184, 8176, 8192, 8176, 7482, 8192, 3494, 3740] dup_rows_per_dst=[1200, 5788, 0, 42, 2868, 2, 0, 0] combine_rows_per_src=[8184, 8176, 8192, 8176, 7482, 8192, 3494, 3740] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56844 payload_rows_remote=48652 dup_rows=8692 dup_rows_remote=5884 dedup_ratio=0.132629 router_tokens_per_expert=[0, 0, 0, 630, 2, 0, 0, 15, 2, 0, 1, 0, 2842, 0, 0, 0, 0, 0, 0, 4093, 4, 3, 0, 0, 4091, 0, 0, 0, 0, 1, 0, 0, 0, 3021, 0, 0, 0, 4, 0, 2, 4067, 0, 0, 0, 4096, 0, 0, 0, 1, 1, 0, 0, 1394, 6, 0, 2, 0, 2433, 4095, 0, 1962, 0, 0, 0] payload_rows_per_dst=[4870, 5690, 8192, 8182, 6054, 8134, 8192, 7530] dup_rows_per_dst=[626, 0, 8, 2, 0, 0, 2808, 5248] combine_rows_per_src=[4870, 5690, 8192, 8182, 6054, 8134, 8192, 7530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59412 payload_rows_remote=53774 dup_rows=6124 dup_rows_remote=6124 dedup_ratio=0.093445 router_tokens_per_expert=[0, 0, 0, 740, 0, 0, 0, 169, 0, 0, 0, 0, 2883, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 1, 0, 0, 0, 2813, 0, 0, 0, 0, 0, 6, 4092, 0, 0, 0, 4095, 0, 3, 0, 0, 0, 0, 0, 1110, 0, 0, 0, 0, 2681, 4096, 0, 1891, 0, 0, 0] payload_rows_per_dst=[8192, 5766, 8190, 8186, 5638, 8184, 8190, 7066] dup_rows_per_dst=[1818, 0, 0, 2, 0, 6, 2220, 2078] combine_rows_per_src=[8192, 5766, 8190, 8186, 5638, 8184, 8190, 7066] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=316682240 dispatch_meta_bytes=189656 combine_payload_bytes=316682240 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=316871896 ingress_bytes=646983680 link_bytes=646983680 token_bytes=633554136 weight_movement_bytes=330301440 total_nvlink_bytes=963855576 weight_to_token_ratio=0.521347 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=14848 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.125211030e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=362973184 dispatch_meta_bytes=229496 combine_payload_bytes=362973184 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=363202680 ingress_bytes=583174144 link_bytes=583174144 token_bytes=726175864 weight_movement_bytes=220200960 total_nvlink_bytes=946376824 weight_to_token_ratio=0.303234 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.014235752e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54752 payload_rows_remote=46566 dup_rows=10784 dup_rows_remote=8482 dedup_ratio=0.164551 router_tokens_per_expert=[0, 0, 0, 880, 0, 0, 0, 276, 1, 0, 0, 0, 2619, 0, 0, 7, 0, 3, 1, 4095, 0, 0, 1, 0, 4082, 4, 0, 0, 1, 15, 0, 2, 0, 2767, 0, 0, 1, 0, 0, 32, 4047, 0, 2, 0, 4096, 2, 3, 0, 0, 0, 0, 2, 1796, 3, 0, 0, 6, 2542, 4088, 0, 1393, 1, 0, 0] payload_rows_per_dst=[8186, 8166, 8192, 8176, 7442, 8192, 3600, 2798] dup_rows_per_dst=[2302, 5182, 8, 32, 3242, 14, 2, 2] combine_rows_per_src=[8186, 8166, 8192, 8176, 7442, 8192, 3600, 2798] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=335247360 dispatch_meta_bytes=229352 combine_payload_bytes=335247360 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=335476712 ingress_bytes=445347840 link_bytes=445347840 token_bytes=670724072 weight_movement_bytes=110100480 total_nvlink_bytes=780824552 weight_to_token_ratio=0.164152 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.745331409e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=375029760 dispatch_meta_bytes=230096 combine_payload_bytes=375029760 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=375259856 ingress_bytes=815431680 link_bytes=815431680 token_bytes=750289616 weight_movement_bytes=440401920 total_nvlink_bytes=1190691536 weight_to_token_ratio=0.586976 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=20864 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.418169807e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=340193280 dispatch_meta_bytes=206288 combine_payload_bytes=340193280 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=340399568 ingress_bytes=450293760 link_bytes=450293760 token_bytes=680592848 weight_movement_bytes=110100480 total_nvlink_bytes=790693328 weight_to_token_ratio=0.161771 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=21888 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.831349092e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=348737536 dispatch_meta_bytes=218144 combine_payload_bytes=348737536 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=348955680 ingress_bytes=458838016 link_bytes=458838016 token_bytes=697693216 weight_movement_bytes=110100480 total_nvlink_bytes=807793696 weight_to_token_ratio=0.157806 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=20992 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.979947757e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=385452032 dispatch_meta_bytes=239592 combine_payload_bytes=385452032 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=385691624 ingress_bytes=495552512 link_bytes=495552512 token_bytes=771143656 weight_movement_bytes=110100480 total_nvlink_bytes=881244136 weight_to_token_ratio=0.142776 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=23936 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.618473228e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=333785088 dispatch_meta_bytes=220192 combine_payload_bytes=333785088 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=334005280 ingress_bytes=443885568 link_bytes=443885568 token_bytes=667790368 weight_movement_bytes=110100480 total_nvlink_bytes=777890848 weight_to_token_ratio=0.164873 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=26752 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.719900094e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58844 payload_rows_remote=51128 dup_rows=6692 dup_rows_remote=3414 dedup_ratio=0.102112 router_tokens_per_expert=[0, 0, 0, 172, 4092, 7, 0, 0, 4073, 3, 0, 0, 0, 0, 2730, 0, 0, 4090, 0, 0, 0, 2, 0, 0, 0, 0, 6, 58, 0, 0, 4090, 0, 3, 1510, 0, 0, 0, 2, 0, 1, 1, 4090, 4, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 4093, 0, 0, 0, 52, 2, 3680, 0, 0, 3] payload_rows_per_dst=[4764, 5466, 8182, 8156, 8182, 8190, 8188, 7716] dup_rows_per_dst=[258, 0, 2, 116, 2996, 10, 32, 3278] combine_rows_per_src=[4764, 5466, 8182, 8156, 8182, 8190, 8188, 7716] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63358 payload_rows_remote=55166 dup_rows=2178 dup_rows_remote=2126 dedup_ratio=0.033234 router_tokens_per_expert=[0, 0, 0, 202, 4095, 2, 0, 1, 4095, 1, 0, 0, 0, 0, 3108, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 4096, 0, 0, 857, 0, 0, 0, 1, 0, 1, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 10, 0, 3985, 0, 0, 8] payload_rows_per_dst=[8192, 6218, 8178, 8192, 8192, 8192, 8190, 8004] dup_rows_per_dst=[408, 0, 0, 52, 1716, 0, 0, 2] combine_rows_per_src=[8192, 6218, 8178, 8192, 8192, 8192, 8190, 8004] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63066 payload_rows_remote=54880 dup_rows=2470 dup_rows_remote=2470 dedup_ratio=0.037689 router_tokens_per_expert=[0, 0, 0, 255, 4095, 0, 0, 0, 4027, 1, 0, 0, 0, 0, 3285, 0, 0, 4092, 0, 1, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 4093, 0, 1, 1020, 0, 1, 0, 1, 1, 3, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 132, 2, 3561, 0, 0, 0] payload_rows_per_dst=[8192, 6572, 8186, 8192, 8174, 8188, 8186, 7376] dup_rows_per_dst=[508, 0, 0, 14, 1934, 0, 0, 14] combine_rows_per_src=[8192, 6572, 8186, 8192, 8174, 8188, 8186, 7376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62380 payload_rows_remote=54190 dup_rows=3156 dup_rows_remote=3154 dedup_ratio=0.048157 router_tokens_per_expert=[0, 0, 0, 281, 4096, 1, 0, 0, 4084, 0, 0, 0, 0, 0, 2741, 0, 2, 4095, 0, 0, 0, 0, 0, 2, 2, 0, 0, 51, 0, 3, 4095, 0, 0, 1230, 0, 0, 0, 1, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 33, 0, 3862, 0, 0, 0] payload_rows_per_dst=[8192, 5482, 8190, 8192, 8172, 8190, 8186, 7776] dup_rows_per_dst=[564, 0, 8, 110, 2458, 2, 0, 14] combine_rows_per_src=[8192, 5482, 8190, 8192, 8172, 8190, 8186, 7776] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58438 payload_rows_remote=50344 dup_rows=7098 dup_rows_remote=422 dedup_ratio=0.108307 router_tokens_per_expert=[0, 0, 0, 163, 4096, 0, 0, 0, 4083, 1, 0, 0, 0, 0, 3466, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4096, 0, 0, 598, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 6, 0, 3959, 0, 0, 0] payload_rows_per_dst=[8192, 8094, 8192, 8192, 1454, 8192, 8192, 7930] dup_rows_per_dst=[326, 6676, 0, 22, 74, 0, 0, 0] combine_rows_per_src=[8192, 8094, 8192, 8192, 1454, 8192, 8192, 7930] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 1, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62896 payload_rows_remote=54704 dup_rows=2640 dup_rows_remote=2598 dedup_ratio=0.040283 router_tokens_per_expert=[0, 2983, 0, 3912, 0, 0, 0, 0, 4096, 0, 0, 1, 1, 0, 0, 0, 0, 569, 0, 0, 0, 0, 16, 0, 2, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 20, 1, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 728, 1, 0, 4010, 0, 0, 0, 70, 0] payload_rows_per_dst=[5966, 8192, 7984, 8192, 8186, 8192, 8160, 8024] dup_rows_per_dst=[0, 4, 1010, 8, 0, 42, 1438, 138] combine_rows_per_src=[5966, 8192, 7984, 8192, 8186, 8192, 8160, 8024] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=399501312 dispatch_meta_bytes=227768 combine_payload_bytes=399501312 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=399729080 ingress_bytes=619702272 link_bytes=619702272 token_bytes=799230392 weight_movement_bytes=220200960 total_nvlink_bytes=1019431352 weight_to_token_ratio=0.275516 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=31872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.077764175e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62886 payload_rows_remote=54760 dup_rows=2650 dup_rows_remote=1352 dedup_ratio=0.040436 router_tokens_per_expert=[0, 3196, 0, 3846, 0, 3, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 748, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4096, 0, 6, 1, 0, 0, 1, 0, 4047, 0, 0, 0, 0, 0, 1, 664, 0, 0, 3854, 0, 0, 2, 6, 0] payload_rows_per_dst=[6398, 8188, 7898, 8190, 8182, 8192, 8126, 7712] dup_rows_per_dst=[0, 0, 1324, 0, 0, 16, 1298, 12] combine_rows_per_src=[6398, 8188, 7898, 8190, 8182, 8192, 8126, 7712] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=400361472 dispatch_meta_bytes=229448 combine_payload_bytes=400361472 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=400590920 ingress_bytes=510461952 link_bytes=510461952 token_bytes=800952392 weight_movement_bytes=110100480 total_nvlink_bytes=911052872 weight_to_token_ratio=0.137462 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.877772911e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62196 payload_rows_remote=54212 dup_rows=3340 dup_rows_remote=1452 dedup_ratio=0.050964 router_tokens_per_expert=[0, 3536, 0, 3796, 0, 8, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 194, 0, 0, 0, 0, 13, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 4096, 0, 115, 1, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 558, 4, 0, 3973, 0, 0, 0, 112, 0] payload_rows_per_dst=[7076, 8192, 7702, 8180, 8170, 7668, 7224, 7984] dup_rows_per_dst=[12, 0, 304, 8, 0, 232, 896, 1888] combine_rows_per_src=[7076, 8192, 7702, 8180, 8170, 7668, 7224, 7984] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64068 payload_rows_remote=55876 dup_rows=1468 dup_rows_remote=1466 dedup_ratio=0.022400 router_tokens_per_expert=[0, 3582, 0, 3819, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 19, 1, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 385, 0, 0, 4070, 0, 0, 0, 2, 0] payload_rows_per_dst=[7164, 8192, 7820, 8192, 8192, 8192, 8176, 8140] dup_rows_per_dst=[0, 0, 658, 2, 0, 40, 764, 4] combine_rows_per_src=[7164, 8192, 7820, 8192, 8192, 8192, 8176, 8140] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=392118272 dispatch_meta_bytes=229208 combine_payload_bytes=392118272 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=392347480 ingress_bytes=722419712 link_bytes=722419712 token_bytes=784465752 weight_movement_bytes=330301440 total_nvlink_bytes=1114767192 weight_to_token_ratio=0.421053 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.256406697e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63278 payload_rows_remote=55086 dup_rows=2258 dup_rows_remote=2258 dedup_ratio=0.034454 router_tokens_per_expert=[0, 3452, 0, 3681, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 458, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 9, 0, 0, 0, 0, 0, 4036, 0, 0, 0, 0, 0, 0, 771, 0, 0, 3957, 0, 0, 0, 9, 0] payload_rows_per_dst=[6904, 8192, 7554, 8192, 8192, 8192, 8138, 7914] dup_rows_per_dst=[0, 0, 746, 0, 0, 18, 1476, 18] combine_rows_per_src=[6904, 8192, 7554, 8192, 8192, 8192, 8138, 7914] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=366485504 dispatch_meta_bytes=218168 combine_payload_bytes=366485504 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=366703672 ingress_bytes=476585984 link_bytes=476585984 token_bytes=733189176 weight_movement_bytes=110100480 total_nvlink_bytes=843289656 weight_to_token_ratio=0.150167 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=30464 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.288614111e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63248 payload_rows_remote=55060 dup_rows=2288 dup_rows_remote=340 dedup_ratio=0.034912 router_tokens_per_expert=[0, 0, 0, 160, 4096, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 3169, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 0, 973, 0, 0, 0, 1, 2, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 161, 0, 3726, 0, 0, 1] payload_rows_per_dst=[8192, 6338, 8192, 8192, 8188, 8190, 8192, 7764] dup_rows_per_dst=[320, 0, 0, 8, 1948, 0, 0, 12] combine_rows_per_src=[8192, 6338, 8192, 8192, 8188, 8190, 8192, 7764] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63958 payload_rows_remote=55770 dup_rows=1578 dup_rows_remote=1578 dedup_ratio=0.024078 router_tokens_per_expert=[0, 0, 0, 147, 4092, 2, 0, 0, 4093, 3, 0, 2, 0, 0, 3406, 0, 0, 4095, 1, 0, 0, 0, 1, 0, 0, 0, 0, 4, 0, 5, 4095, 1, 0, 621, 0, 0, 0, 4, 3, 2, 1, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 9, 0, 3994, 0, 0, 0] payload_rows_per_dst=[8188, 6820, 8190, 8192, 8192, 8188, 8188, 8000] dup_rows_per_dst=[294, 2, 4, 18, 1254, 0, 0, 6] combine_rows_per_src=[8188, 6820, 8190, 8192, 8192, 8188, 8188, 8000] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=395429888 dispatch_meta_bytes=229168 combine_payload_bytes=395429888 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=395659056 ingress_bytes=945932288 link_bytes=945932288 token_bytes=791088944 weight_movement_bytes=550502400 total_nvlink_bytes=1341591344 weight_to_token_ratio=0.695879 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.645131828e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59026 payload_rows_remote=50834 dup_rows=6510 dup_rows_remote=5756 dedup_ratio=0.099335 router_tokens_per_expert=[0, 0, 0, 242, 4096, 0, 0, 0, 4057, 2, 0, 0, 5, 0, 2869, 0, 2, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 4095, 0, 0, 1298, 0, 0, 0, 0, 0, 3, 0, 4095, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 4089, 0, 0, 0, 126, 4, 3690, 0, 0, 2] payload_rows_per_dst=[8192, 8176, 8174, 7886, 2596, 8192, 8180, 7630] dup_rows_per_dst=[754, 5692, 38, 2, 6, 2, 2, 14] combine_rows_per_src=[8192, 8176, 8174, 7886, 2596, 8192, 8180, 7630] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=393379840 dispatch_meta_bytes=229400 combine_payload_bytes=393379840 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=393609240 ingress_bytes=503480320 link_bytes=503480320 token_bytes=786989080 weight_movement_bytes=110100480 total_nvlink_bytes=897089560 weight_to_token_ratio=0.139901 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.756350848e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=388433920 dispatch_meta_bytes=229376 combine_payload_bytes=388433920 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=388663296 ingress_bytes=498534400 link_bytes=498534400 token_bytes=777097216 weight_movement_bytes=110100480 total_nvlink_bytes=887197696 weight_to_token_ratio=0.141682 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.670333164e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=360865792 dispatch_meta_bytes=203064 combine_payload_bytes=360865792 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=361068856 ingress_bytes=581066752 link_bytes=581066752 token_bytes=721934648 weight_movement_bytes=220200960 total_nvlink_bytes=942135608 weight_to_token_ratio=0.305015 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=24832 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.010570651e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=394670080 dispatch_meta_bytes=221600 combine_payload_bytes=394670080 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=394891680 ingress_bytes=504770560 link_bytes=504770560 token_bytes=789561760 weight_movement_bytes=110100480 total_nvlink_bytes=899662240 weight_to_token_ratio=0.139445 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=24704 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.778790243e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=399759360 dispatch_meta_bytes=229392 combine_payload_bytes=399759360 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=399988752 ingress_bytes=509859840 link_bytes=509859840 token_bytes=799748112 weight_movement_bytes=110100480 total_nvlink_bytes=909848592 weight_to_token_ratio=0.137669 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.867301193e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=364378112 dispatch_meta_bytes=226360 combine_payload_bytes=364378112 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=364604472 ingress_bytes=584579072 link_bytes=584579072 token_bytes=728982584 weight_movement_bytes=220200960 total_nvlink_bytes=949183544 weight_to_token_ratio=0.302066 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=31104 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.016679153e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58696 payload_rows_remote=50924 dup_rows=6840 dup_rows_remote=2340 dedup_ratio=0.104370 router_tokens_per_expert=[0, 1316, 0, 3626, 0, 10, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 8, 1, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4083, 0, 0, 0, 0, 1, 4096, 0, 36, 0, 0, 0, 0, 0, 3510, 0, 0, 0, 0, 0, 0, 116, 4, 0, 3688, 0, 0, 0, 0, 0] payload_rows_per_dst=[2636, 8192, 8164, 8180, 8168, 8192, 7392, 7772] dup_rows_per_dst=[16, 0, 8, 8, 0, 84, 2224, 4500] combine_rows_per_src=[2636, 8192, 8164, 8180, 8168, 8192, 7392, 7772] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63762 payload_rows_remote=55570 dup_rows=1774 dup_rows_remote=1770 dedup_ratio=0.027069 router_tokens_per_expert=[0, 1264, 0, 3618, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 8, 1, 0, 0, 0, 0, 3476, 0, 0, 0, 0, 0, 0, 95, 0, 0, 3826, 0, 0, 0, 1, 0] payload_rows_per_dst=[8106, 8192, 8186, 8192, 8192, 8192, 7048, 7654] dup_rows_per_dst=[1658, 0, 0, 4, 0, 18, 94, 0] combine_rows_per_src=[8106, 8192, 8186, 8192, 8192, 8192, 7048, 7654] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62616 payload_rows_remote=54466 dup_rows=2920 dup_rows_remote=2914 dedup_ratio=0.044556 router_tokens_per_expert=[0, 1558, 0, 3715, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 0, 0, 15, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4094, 0, 0, 0, 0, 1, 4094, 0, 36, 0, 0, 0, 0, 0, 3091, 0, 0, 0, 0, 0, 0, 338, 1, 1, 3570, 0, 0, 0, 0, 0] payload_rows_per_dst=[8026, 8190, 8150, 8190, 8188, 8190, 6542, 7140] dup_rows_per_dst=[2520, 0, 6, 2, 2, 70, 316, 4] combine_rows_per_src=[8026, 8190, 8150, 8190, 8188, 8190, 6542, 7140] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62974 payload_rows_remote=54782 dup_rows=2562 dup_rows_remote=2562 dedup_ratio=0.039093 router_tokens_per_expert=[0, 1483, 0, 3789, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 3200, 0, 0, 0, 0, 0, 0, 140, 0, 0, 3671, 0, 0, 0, 0, 0] payload_rows_per_dst=[8126, 8192, 8188, 8192, 8190, 8192, 6552, 7342] dup_rows_per_dst=[2418, 0, 8, 0, 0, 8, 128, 0] combine_rows_per_src=[8126, 8192, 8188, 8192, 8190, 8192, 6552, 7342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64264 payload_rows_remote=56072 dup_rows=1272 dup_rows_remote=1272 dedup_ratio=0.019409 router_tokens_per_expert=[0, 725, 0, 3879, 0, 0, 0, 0, 4096, 0, 0, 1, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 13, 0, 2, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3572, 0, 0, 0, 0, 0, 0, 155, 0, 0, 3940, 0, 0, 0, 3, 0] payload_rows_per_dst=[8090, 8192, 8192, 8192, 8184, 8192, 7336, 7886] dup_rows_per_dst=[1118, 4, 26, 6, 0, 0, 118, 0] combine_rows_per_src=[8090, 8192, 8192, 8192, 8184, 8192, 7336, 7886] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=392519680 dispatch_meta_bytes=224448 combine_payload_bytes=392519680 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=392744128 ingress_bytes=722821120 link_bytes=722821120 token_bytes=785263808 weight_movement_bytes=330301440 total_nvlink_bytes=1115565248 weight_to_token_ratio=0.420625 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=27776 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.257104811e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=388591616 dispatch_meta_bytes=222656 combine_payload_bytes=388591616 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=388814272 ingress_bytes=498692096 link_bytes=498692096 token_bytes=777405888 weight_movement_bytes=110100480 total_nvlink_bytes=887506368 weight_to_token_ratio=0.141625 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=31744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.673075757e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=400519168 dispatch_meta_bytes=229368 combine_payload_bytes=400519168 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=400748536 ingress_bytes=620720128 link_bytes=620720128 token_bytes=801267704 weight_movement_bytes=220200960 total_nvlink_bytes=1021468664 weight_to_token_ratio=0.274816 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.079534394e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63582 payload_rows_remote=55390 dup_rows=1954 dup_rows_remote=1954 dedup_ratio=0.029816 router_tokens_per_expert=[0, 3403, 0, 3868, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 630, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 11, 0, 0, 0, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 468, 0, 0, 3945, 0, 0, 0, 3, 0] payload_rows_per_dst=[6806, 8192, 7966, 8192, 8190, 8192, 8154, 7890] dup_rows_per_dst=[0, 0, 1038, 0, 0, 22, 888, 6] combine_rows_per_src=[6806, 8192, 7966, 8192, 8190, 8192, 8154, 7890] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=394856448 dispatch_meta_bytes=229376 combine_payload_bytes=394856448 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=395085824 ingress_bytes=504956928 link_bytes=504956928 token_bytes=789942272 weight_movement_bytes=110100480 total_nvlink_bytes=900042752 weight_to_token_ratio=0.139378 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.782031489e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54788 payload_rows_remote=46640 dup_rows=10748 dup_rows_remote=498 dedup_ratio=0.164001 router_tokens_per_expert=[0, 3642, 0, 3929, 0, 7, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 344, 0, 0, 0, 0, 2, 0, 1, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 4096, 0, 20, 0, 0, 0, 0, 0, 4051, 0, 0, 0, 0, 0, 0, 323, 3, 0, 4060, 0, 0, 1, 10, 0] payload_rows_per_dst=[8148, 8192, 4864, 8188, 8192, 8148, 930, 8126] dup_rows_per_dst=[10250, 4, 382, 0, 46, 22, 22, 22] combine_rows_per_src=[8148, 8192, 4864, 8188, 8192, 8148, 930, 8126] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62900 payload_rows_remote=55286 dup_rows=2636 dup_rows_remote=1964 dedup_ratio=0.040222 router_tokens_per_expert=[0, 3405, 1, 3720, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 410, 0, 0, 0, 0, 13, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4094, 0, 90, 0, 0, 0, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 875, 1, 2, 3778, 0, 0, 0, 48, 0] payload_rows_per_dst=[6812, 8188, 7614, 8190, 8186, 8190, 8164, 7556] dup_rows_per_dst=[0, 0, 672, 2, 0, 178, 1682, 102] combine_rows_per_src=[6812, 8188, 7614, 8190, 8186, 8190, 8164, 7556] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63032 payload_rows_remote=54840 dup_rows=2504 dup_rows_remote=2504 dedup_ratio=0.038208 router_tokens_per_expert=[0, 1589, 0, 3613, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 3051, 0, 0, 0, 0, 0, 0, 208, 0, 0, 3822, 0, 0, 0, 0, 0] payload_rows_per_dst=[8084, 8192, 8184, 8192, 8192, 8192, 6352, 7644] dup_rows_per_dst=[2320, 0, 12, 0, 0, 6, 166, 0] combine_rows_per_src=[8084, 8192, 8184, 8192, 8192, 8192, 6352, 7644] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59254 payload_rows_remote=51092 dup_rows=6282 dup_rows_remote=72 dedup_ratio=0.095856 router_tokens_per_expert=[0, 1102, 0, 3843, 0, 3, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 1, 4093, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4096, 0, 2, 1, 0, 0, 1, 0, 3361, 0, 0, 0, 0, 0, 1, 136, 0, 0, 3840, 0, 0, 2, 0, 0] payload_rows_per_dst=[2468, 8188, 8190, 8188, 8182, 8192, 8162, 7684] dup_rows_per_dst=[52, 0, 12, 0, 0, 8, 6210, 0] combine_rows_per_src=[2468, 8188, 8190, 8188, 8182, 8192, 8162, 7684] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=365023232 dispatch_meta_bytes=213056 combine_payload_bytes=365023232 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=365236288 ingress_bytes=475123712 link_bytes=475123712 token_bytes=730259520 weight_movement_bytes=110100480 total_nvlink_bytes=840360000 weight_to_token_ratio=0.150769 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=30336 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.263182796e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59256 payload_rows_remote=51064 dup_rows=6280 dup_rows_remote=6036 dedup_ratio=0.095825 router_tokens_per_expert=[0, 4, 0, 3148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2368, 0, 4018, 642, 3788, 0, 0, 0, 0, 443, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3423, 1, 0, 0, 4096, 2241, 118, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4035, 0, 3, 0, 309, 4096, 0, 30, 0, 0, 0, 0, 0] payload_rows_per_dst=[6298, 4736, 7800, 8042, 7918, 8192, 8078, 8192] dup_rows_per_dst=[6, 0, 1060, 882, 3412, 244, 616, 60] combine_rows_per_src=[6298, 4736, 7800, 8042, 7918, 8192, 8078, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=397035520 dispatch_meta_bytes=229376 combine_payload_bytes=397035520 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=397264896 ingress_bytes=507136000 link_bytes=507136000 token_bytes=794300416 weight_movement_bytes=110100480 total_nvlink_bytes=904400896 weight_to_token_ratio=0.138613 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.819929135e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=334315520 dispatch_meta_bytes=188552 combine_payload_bytes=334315520 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=334504072 ingress_bytes=554516480 link_bytes=554516480 token_bytes=668819592 weight_movement_bytes=220200960 total_nvlink_bytes=889020552 weight_to_token_ratio=0.329238 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=27264 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.643953610e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=396290048 dispatch_meta_bytes=229000 combine_payload_bytes=396290048 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=396519048 ingress_bytes=506390528 link_bytes=506390528 token_bytes=792809096 weight_movement_bytes=110100480 total_nvlink_bytes=902909576 weight_to_token_ratio=0.138874 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=28928 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.806964151e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58948 payload_rows_remote=50800 dup_rows=6588 dup_rows_remote=5000 dedup_ratio=0.100525 router_tokens_per_expert=[0, 5, 0, 3036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2574, 0, 4029, 888, 3500, 0, 0, 0, 0, 279, 0, 0, 4, 0, 0, 0, 0, 0, 0, 3217, 0, 0, 0, 4095, 2437, 32, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4073, 0, 0, 0, 493, 4096, 0, 8, 0, 0, 0, 1, 0] payload_rows_per_dst=[6080, 5148, 7432, 8072, 7686, 8190, 8148, 8192] dup_rows_per_dst=[2, 0, 1344, 552, 3020, 64, 1588, 18] combine_rows_per_src=[6080, 5148, 7432, 8072, 7686, 8190, 8148, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57930 payload_rows_remote=49738 dup_rows=7606 dup_rows_remote=7566 dedup_ratio=0.116058 router_tokens_per_expert=[0, 16, 0, 2253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3036, 0, 3836, 984, 3697, 0, 0, 0, 0, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3758, 0, 0, 0, 4096, 2264, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3841, 0, 0, 0, 476, 4096, 0, 17, 0, 0, 0, 3, 0] payload_rows_per_dst=[4508, 6072, 7728, 7690, 7516, 8192, 8032, 8192] dup_rows_per_dst=[30, 0, 1634, 440, 0, 332, 5130, 40] combine_rows_per_src=[4508, 6072, 7728, 7690, 7516, 8192, 8032, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58476 payload_rows_remote=50498 dup_rows=7060 dup_rows_remote=6256 dedup_ratio=0.107727 router_tokens_per_expert=[0, 2, 0, 2678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2574, 0, 3987, 871, 3674, 0, 0, 0, 0, 404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3430, 0, 0, 0, 4096, 2294, 61, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 1, 0, 540, 4096, 0, 2, 0, 0, 0, 0, 0] payload_rows_per_dst=[5358, 5536, 7706, 7978, 7378, 8192, 8136, 8192] dup_rows_per_dst=[2, 1158, 1384, 804, 2524, 124, 1060, 4] combine_rows_per_src=[5358, 5536, 7706, 7978, 7378, 8192, 8136, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=366026752 dispatch_meta_bytes=228400 combine_payload_bytes=366026752 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=366255152 ingress_bytes=476127232 link_bytes=476127232 token_bytes=732281904 weight_movement_bytes=110100480 total_nvlink_bytes=842382384 weight_to_token_ratio=0.150353 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.280635659e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60122 payload_rows_remote=52348 dup_rows=5414 dup_rows_remote=2634 dedup_ratio=0.082611 router_tokens_per_expert=[0, 3, 0, 3022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2977, 0, 4042, 693, 3698, 0, 0, 0, 0, 424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3341, 1, 0, 0, 4096, 1935, 24, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 322, 4096, 0, 5, 0, 0, 0, 0, 0] payload_rows_per_dst=[6050, 5954, 7696, 8090, 7774, 8192, 8174, 8192] dup_rows_per_dst=[0, 0, 1086, 842, 2780, 52, 644, 10] combine_rows_per_src=[6050, 5954, 7696, 8090, 7774, 8192, 8174, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63480 payload_rows_remote=55358 dup_rows=2056 dup_rows_remote=280 dedup_ratio=0.031372 router_tokens_per_expert=[0, 1148, 0, 3765, 0, 8, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 4096, 0, 8, 0, 0, 0, 0, 0, 3316, 0, 0, 0, 0, 0, 0, 132, 4, 0, 3950, 0, 0, 1, 0, 0] payload_rows_per_dst=[8122, 8192, 8134, 8190, 8176, 8000, 6758, 7908] dup_rows_per_dst=[1776, 2, 76, 0, 62, 0, 138, 2] combine_rows_per_src=[8122, 8192, 8134, 8190, 8176, 8000, 6758, 7908] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=398325760 dispatch_meta_bytes=229360 combine_payload_bytes=398325760 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=398555120 ingress_bytes=618526720 link_bytes=618526720 token_bytes=796880880 weight_movement_bytes=220200960 total_nvlink_bytes=1017081840 weight_to_token_ratio=0.276329 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.075719696e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=390412288 dispatch_meta_bytes=229520 combine_payload_bytes=390412288 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=390641808 ingress_bytes=500512768 link_bytes=500512768 token_bytes=781054096 weight_movement_bytes=110100480 total_nvlink_bytes=891154576 weight_to_token_ratio=0.140964 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.704740238e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=392677376 dispatch_meta_bytes=229376 combine_payload_bytes=392677376 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=392906752 ingress_bytes=502777856 link_bytes=502777856 token_bytes=785584128 weight_movement_bytes=110100480 total_nvlink_bytes=895684608 weight_to_token_ratio=0.140151 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.744133843e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=401924096 dispatch_meta_bytes=229376 combine_payload_bytes=401924096 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=402153472 ingress_bytes=842326016 link_bytes=842326016 token_bytes=804077568 weight_movement_bytes=440401920 total_nvlink_bytes=1244479488 weight_to_token_ratio=0.547711 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.464943481e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=393093120 dispatch_meta_bytes=229376 combine_payload_bytes=393093120 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=393322496 ingress_bytes=503193600 link_bytes=503193600 token_bytes=786415616 weight_movement_bytes=110100480 total_nvlink_bytes=896516096 weight_to_token_ratio=0.140003 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.751364315e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=366227456 dispatch_meta_bytes=204656 combine_payload_bytes=366227456 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=366432112 ingress_bytes=476327936 link_bytes=476327936 token_bytes=732659568 weight_movement_bytes=110100480 total_nvlink_bytes=842760048 weight_to_token_ratio=0.150275 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=26624 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.284126232e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=396806144 dispatch_meta_bytes=222552 combine_payload_bytes=396806144 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=397028696 ingress_bytes=727107584 link_bytes=727107584 token_bytes=793834840 weight_movement_bytes=330301440 total_nvlink_bytes=1124136280 weight_to_token_ratio=0.416083 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=22656 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.264559677e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55930 payload_rows_remote=47738 dup_rows=9606 dup_rows_remote=9546 dedup_ratio=0.146576 router_tokens_per_expert=[0, 15, 0, 1879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2703, 0, 3465, 2849, 2627, 0, 0, 0, 0, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2464, 0, 0, 0, 4096, 3346, 221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3926, 0, 1, 0, 784, 4096, 0, 27, 0, 0, 0, 3, 0] payload_rows_per_dst=[3760, 5406, 7490, 7006, 7952, 8192, 7932, 8192] dup_rows_per_dst=[28, 0, 3462, 456, 3668, 442, 1490, 60] combine_rows_per_src=[3760, 5406, 7490, 7006, 7952, 8192, 7932, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56326 payload_rows_remote=48634 dup_rows=9210 dup_rows_remote=8250 dedup_ratio=0.140533 router_tokens_per_expert=[0, 2, 0, 2337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2622, 0, 3810, 2662, 2613, 0, 0, 0, 0, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1841, 0, 0, 0, 4096, 3263, 54, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 1, 0, 777, 4096, 0, 3, 0, 0, 0, 0, 0] payload_rows_per_dst=[7566, 7336, 5226, 7692, 3958, 8192, 8164, 8192] dup_rows_per_dst=[3258, 3232, 0, 960, 104, 110, 1540, 6] combine_rows_per_src=[7566, 7336, 5226, 7692, 3958, 8192, 8164, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56742 payload_rows_remote=51202 dup_rows=8794 dup_rows_remote=8794 dedup_ratio=0.134186 router_tokens_per_expert=[0, 7, 0, 2383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2444, 0, 3529, 2514, 2770, 0, 0, 0, 0, 467, 0, 0, 3, 0, 0, 0, 0, 0, 0, 2178, 2, 0, 0, 4096, 3317, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 3, 0, 805, 4096, 0, 36, 0, 0, 0, 0, 0] payload_rows_per_dst=[7816, 7268, 5540, 7250, 4356, 8192, 8128, 8192] dup_rows_per_dst=[3598, 2648, 0, 748, 4, 118, 1606, 72] combine_rows_per_src=[7816, 7268, 5540, 7250, 4356, 8192, 8128, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=342185984 dispatch_meta_bytes=229136 combine_payload_bytes=342185984 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=342415120 ingress_bytes=452286464 link_bytes=452286464 token_bytes=684601104 weight_movement_bytes=110100480 total_nvlink_bytes=794701584 weight_to_token_ratio=0.160824 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.866005492e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55776 payload_rows_remote=48098 dup_rows=9760 dup_rows_remote=5984 dedup_ratio=0.148926 router_tokens_per_expert=[0, 6, 0, 2262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2791, 0, 3866, 2936, 2728, 0, 0, 0, 0, 276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1524, 0, 0, 0, 4096, 3453, 60, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4026, 0, 0, 0, 621, 4096, 0, 21, 0, 0, 0, 0, 0] payload_rows_per_dst=[7882, 7678, 5456, 7762, 5154, 8192, 5460, 8192] dup_rows_per_dst=[3560, 3776, 0, 522, 1092, 132, 636, 42] combine_rows_per_src=[7882, 7678, 5456, 7762, 5154, 8192, 5460, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57416 payload_rows_remote=49224 dup_rows=8120 dup_rows_remote=7758 dedup_ratio=0.123901 router_tokens_per_expert=[0, 4, 0, 2845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2473, 0, 3689, 2257, 2779, 0, 0, 0, 0, 554, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1945, 1, 0, 0, 4096, 3115, 177, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 3, 0, 639, 4096, 0, 38, 0, 0, 0, 0, 0] payload_rows_per_dst=[5692, 4946, 7274, 7524, 7482, 8192, 8114, 8192] dup_rows_per_dst=[6, 0, 2798, 962, 2642, 362, 1274, 76] combine_rows_per_src=[5692, 4946, 7274, 7524, 7482, 8192, 8114, 8192] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=364134400 dispatch_meta_bytes=223200 combine_payload_bytes=364134400 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=364357600 ingress_bytes=584335360 link_bytes=584335360 token_bytes=728492000 weight_movement_bytes=220200960 total_nvlink_bytes=948692960 weight_to_token_ratio=0.302270 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=26752 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.016255298e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=356521984 dispatch_meta_bytes=229216 combine_payload_bytes=356521984 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=356751200 ingress_bytes=466622464 link_bytes=466622464 token_bytes=713273184 weight_movement_bytes=110100480 total_nvlink_bytes=823373664 weight_to_token_ratio=0.154359 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.115332111e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=361969664 dispatch_meta_bytes=227016 combine_payload_bytes=361969664 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=362196680 ingress_bytes=472070144 link_bytes=472070144 token_bytes=724166344 weight_movement_bytes=110100480 total_nvlink_bytes=834266824 weight_to_token_ratio=0.152038 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=30080 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.210076226e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=375230464 dispatch_meta_bytes=219928 combine_payload_bytes=375230464 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=375450392 ingress_bytes=815632384 link_bytes=815632384 token_bytes=750680856 weight_movement_bytes=440401920 total_nvlink_bytes=1191082776 weight_to_token_ratio=0.586670 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=16512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.418518864e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55360 payload_rows_remote=47358 dup_rows=10176 dup_rows_remote=5822 dedup_ratio=0.155273 router_tokens_per_expert=[0, 5, 0, 2408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2745, 0, 4063, 995, 3756, 0, 0, 0, 0, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3433, 1, 0, 0, 4096, 2467, 46, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4000, 0, 0, 0, 423, 4096, 0, 20, 0, 0, 0, 0, 0] payload_rows_per_dst=[8114, 8002, 4430, 8126, 4934, 8192, 5370, 8192] dup_rows_per_dst=[4224, 4354, 518, 418, 2, 102, 518, 40] combine_rows_per_src=[8114, 8002, 4430, 8126, 4934, 8192, 5370, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49798 payload_rows_remote=41718 dup_rows=15738 dup_rows_remote=10140 dedup_ratio=0.240143 router_tokens_per_expert=[0, 23, 0, 3076, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2571, 0, 3897, 615, 3740, 0, 0, 0, 0, 329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3679, 0, 0, 0, 4096, 2005, 177, 0, 0, 7, 0, 1, 0, 0, 0, 0, 0, 3998, 0, 0, 0, 430, 4096, 0, 27, 0, 0, 0, 0, 0] payload_rows_per_dst=[8080, 8080, 8174, 4592, 4952, 7094, 860, 7966] dup_rows_per_dst=[5598, 4422, 4572, 340, 726, 80, 0, 0] combine_rows_per_src=[8080, 8080, 8174, 4592, 4952, 7094, 860, 7966] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55570 payload_rows_remote=52790 dup_rows=9966 dup_rows_remote=9774 dedup_ratio=0.152069 router_tokens_per_expert=[0, 7, 0, 2743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2452, 0, 3953, 664, 3665, 0, 0, 0, 0, 378, 0, 0, 3, 0, 0, 0, 0, 0, 0, 3585, 1, 0, 0, 4096, 2456, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 0, 3, 0, 559, 4096, 0, 27, 0, 0, 0, 0, 0] payload_rows_per_dst=[7524, 7950, 2780, 7950, 4912, 8192, 8070, 8192] dup_rows_per_dst=[3662, 4124, 192, 718, 2, 102, 1112, 54] combine_rows_per_src=[7524, 7950, 2780, 7950, 4912, 8192, 8070, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60118 payload_rows_remote=51926 dup_rows=5418 dup_rows_remote=5416 dedup_ratio=0.082672 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2572, 0, 4096, 0, 4095, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 2597, 0, 6, 4096, 0, 4092, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3032, 0, 0, 0, 4077, 0, 0, 0, 0, 3] payload_rows_per_dst=[8192, 5144, 8190, 8192, 7424, 8192, 6630, 8154] dup_rows_per_dst=[0, 0, 6, 4, 3890, 2, 1510, 6] combine_rows_per_src=[8192, 5144, 8190, 8192, 7424, 8192, 6630, 8154] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=339462144 dispatch_meta_bytes=212720 combine_payload_bytes=339462144 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=339674864 ingress_bytes=449562624 link_bytes=449562624 token_bytes=679137008 weight_movement_bytes=110100480 total_nvlink_bytes=789237488 weight_to_token_ratio=0.162118 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=21376 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.818633435e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=299034624 dispatch_meta_bytes=207432 combine_payload_bytes=299034624 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=299242056 ingress_bytes=409135104 link_bytes=409135104 token_bytes=598276680 weight_movement_bytes=110100480 total_nvlink_bytes=708377160 weight_to_token_ratio=0.184029 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=22400 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.115532370e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=378398720 dispatch_meta_bytes=250256 combine_payload_bytes=378398720 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=378648976 ingress_bytes=708700160 link_bytes=708700160 token_bytes=757047696 weight_movement_bytes=330301440 total_nvlink_bytes=1087349136 weight_to_token_ratio=0.436302 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.055147 gemm_active_groups=5 gemm_peak_group_rows=19200 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.232546139e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59126 payload_rows_remote=50934 dup_rows=6410 dup_rows_remote=12 dedup_ratio=0.097809 router_tokens_per_expert=[0, 0, 0, 0, 0, 2, 0, 0, 0, 2748, 0, 4096, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2306, 0, 4, 4096, 0, 4088, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3207, 0, 0, 0, 4025, 0, 0, 0, 0, 2] payload_rows_per_dst=[8192, 5498, 8190, 8192, 4620, 8192, 8192, 8050] dup_rows_per_dst=[4, 0, 0, 0, 0, 4, 6398, 4] combine_rows_per_src=[8192, 5498, 8190, 8192, 4620, 8192, 8192, 8050] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59806 payload_rows_remote=51628 dup_rows=5730 dup_rows_remote=5508 dedup_ratio=0.087433 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 0, 0, 2886, 0, 4096, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2547, 0, 16, 4096, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 2753, 0, 0, 0, 4088, 0, 0, 0, 0, 7] payload_rows_per_dst=[8176, 5772, 8192, 8192, 4914, 8192, 8190, 8178] dup_rows_per_dst=[2, 0, 0, 2, 2, 0, 5502, 222] combine_rows_per_src=[8176, 5772, 8192, 8192, 4914, 8192, 8190, 8178] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60018 payload_rows_remote=51828 dup_rows=5518 dup_rows_remote=5518 dedup_ratio=0.084198 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 3026, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2749, 0, 9, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2415, 0, 0, 0, 4089, 0, 0, 0, 0, 2] payload_rows_per_dst=[8192, 6052, 8192, 8190, 8192, 8192, 4830, 8178] dup_rows_per_dst=[0, 0, 0, 0, 5514, 0, 0, 4] combine_rows_per_src=[8192, 6052, 8192, 8190, 8192, 8192, 4830, 8178] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59408 payload_rows_remote=51216 dup_rows=6128 dup_rows_remote=6 dedup_ratio=0.093506 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2317, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3061, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2812, 0, 0, 0, 4095, 0, 0, 0, 0, 3] payload_rows_per_dst=[8192, 4634, 8192, 8192, 8192, 8192, 5624, 8190] dup_rows_per_dst=[0, 0, 0, 0, 6122, 0, 0, 6] combine_rows_per_src=[8192, 4634, 8192, 8192, 8192, 8192, 5624, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=372205568 dispatch_meta_bytes=229368 combine_payload_bytes=372205568 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=372434936 ingress_bytes=482306048 link_bytes=482306048 token_bytes=744640504 weight_movement_bytes=110100480 total_nvlink_bytes=854740984 weight_to_token_ratio=0.147857 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.388095432e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=365094912 dispatch_meta_bytes=203784 combine_payload_bytes=365094912 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=365298696 ingress_bytes=585295872 link_bytes=585295872 token_bytes=730393608 weight_movement_bytes=220200960 total_nvlink_bytes=950594568 weight_to_token_ratio=0.301483 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=23552 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.017925786e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=370069504 dispatch_meta_bytes=228544 combine_payload_bytes=370069504 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=370298048 ingress_bytes=480169984 link_bytes=480169984 token_bytes=740367552 weight_movement_bytes=110100480 total_nvlink_bytes=850468032 weight_to_token_ratio=0.148711 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.350945766e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=371503104 dispatch_meta_bytes=229384 combine_payload_bytes=371503104 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=371732488 ingress_bytes=481603584 link_bytes=481603584 token_bytes=743235592 weight_movement_bytes=110100480 total_nvlink_bytes=853336072 weight_to_token_ratio=0.148137 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.375878428e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=367116288 dispatch_meta_bytes=204888 combine_payload_bytes=367116288 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=367321176 ingress_bytes=807518208 link_bytes=807518208 token_bytes=734437464 weight_movement_bytes=440401920 total_nvlink_bytes=1174839384 weight_to_token_ratio=0.599645 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=20864 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.404406978e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60088 payload_rows_remote=51896 dup_rows=5448 dup_rows_remote=5444 dedup_ratio=0.083130 router_tokens_per_expert=[0, 0, 0, 0, 0, 2, 0, 0, 0, 2704, 0, 4095, 0, 4095, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2283, 0, 4, 4096, 0, 4091, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 6, 0, 3189, 0, 0, 0, 4093, 0, 0, 0, 0, 7] payload_rows_per_dst=[8192, 8190, 8184, 8188, 4586, 8184, 6378, 8186] dup_rows_per_dst=[4, 5416, 0, 0, 2, 0, 12, 14] combine_rows_per_src=[8192, 8190, 8184, 8188, 4586, 8184, 6378, 8186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60116 payload_rows_remote=51930 dup_rows=5420 dup_rows_remote=40 dedup_ratio=0.082703 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2691, 0, 4096, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2747, 0, 1, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2772, 0, 0, 0, 4062, 0, 0, 0, 0, 15] payload_rows_per_dst=[8192, 8186, 8192, 8192, 5494, 8192, 5544, 8124] dup_rows_per_dst=[0, 5380, 8, 0, 2, 0, 0, 30] combine_rows_per_src=[8192, 8186, 8192, 8192, 5494, 8192, 5544, 8124] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61114 payload_rows_remote=52926 dup_rows=4422 dup_rows_remote=4420 dedup_ratio=0.067474 router_tokens_per_expert=[0, 0, 0, 0, 0, 3, 0, 0, 0, 2158, 0, 4096, 0, 4094, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2674, 0, 1, 4096, 0, 4090, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3363, 0, 0, 0, 4093, 0, 0, 0, 0, 2] payload_rows_per_dst=[8192, 7670, 8188, 8192, 5768, 8192, 6726, 8186] dup_rows_per_dst=[6, 3594, 2, 0, 814, 2, 0, 4] combine_rows_per_src=[8192, 7670, 8188, 8192, 5768, 8192, 6726, 8186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57798 payload_rows_remote=50050 dup_rows=7738 dup_rows_remote=5154 dedup_ratio=0.118073 router_tokens_per_expert=[0, 3, 0, 2709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2676, 0, 3756, 2599, 2283, 0, 0, 0, 0, 642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1858, 2, 0, 0, 4096, 3306, 23, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 619, 4096, 0, 7, 0, 0, 0, 0, 0] payload_rows_per_dst=[5424, 6570, 5860, 7630, 7748, 8192, 8182, 8192] dup_rows_per_dst=[0, 1648, 1038, 1166, 2584, 50, 1238, 14] combine_rows_per_src=[5424, 6570, 5860, 7630, 7748, 8192, 8182, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57650 payload_rows_remote=49484 dup_rows=7886 dup_rows_remote=6116 dedup_ratio=0.120331 router_tokens_per_expert=[0, 5, 0, 2720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2490, 0, 3845, 2943, 2069, 0, 0, 0, 0, 454, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1551, 0, 0, 0, 4095, 3475, 43, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4082, 0, 0, 0, 885, 4096, 0, 10, 0, 0, 0, 1, 0] payload_rows_per_dst=[5450, 4980, 7236, 7754, 7682, 8190, 8166, 8192] dup_rows_per_dst=[0, 0, 2788, 850, 2370, 86, 1770, 22] combine_rows_per_src=[5450, 4980, 7236, 7754, 7682, 8190, 8166, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51778 payload_rows_remote=43978 dup_rows=13758 dup_rows_remote=10062 dedup_ratio=0.209930 router_tokens_per_expert=[0, 20, 0, 2698, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2620, 0, 3386, 2403, 2774, 0, 0, 0, 0, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2211, 0, 0, 0, 4096, 3030, 180, 0, 0, 8, 0, 1, 0, 0, 0, 0, 0, 4018, 0, 1, 0, 737, 4096, 0, 34, 0, 0, 0, 0, 0] payload_rows_per_dst=[7800, 7294, 7206, 6554, 6418, 6814, 1796, 7896] dup_rows_per_dst=[3696, 2754, 4162, 700, 2322, 80, 40, 4] combine_rows_per_src=[7800, 7294, 7206, 6554, 6418, 6814, 1796, 7896] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=348608512 dispatch_meta_bytes=227536 combine_payload_bytes=348608512 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=348836048 ingress_bytes=458708992 link_bytes=458708992 token_bytes=697444560 weight_movement_bytes=110100480 total_nvlink_bytes=807545040 weight_to_token_ratio=0.157863 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=29184 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.977703818e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=367015936 dispatch_meta_bytes=239984 combine_payload_bytes=367015936 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=367255920 ingress_bytes=697317376 link_bytes=697317376 token_bytes=734271856 weight_movement_bytes=330301440 total_nvlink_bytes=1064573296 weight_to_token_ratio=0.449835 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=20736 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.212749606e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=344766464 dispatch_meta_bytes=216328 combine_payload_bytes=344766464 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=344982792 ingress_bytes=454866944 link_bytes=454866944 token_bytes=689749256 weight_movement_bytes=110100480 total_nvlink_bytes=799849736 weight_to_token_ratio=0.159624 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=20864 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.910884284e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=352837632 dispatch_meta_bytes=227928 combine_payload_bytes=352837632 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=353065560 ingress_bytes=462938112 link_bytes=462938112 token_bytes=705903192 weight_movement_bytes=110100480 total_nvlink_bytes=816003672 weight_to_token_ratio=0.155971 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.051255170e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=358758400 dispatch_meta_bytes=220816 combine_payload_bytes=358758400 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=358979216 ingress_bytes=689059840 link_bytes=689059840 token_bytes=717737616 weight_movement_bytes=330301440 total_nvlink_bytes=1048039056 weight_to_token_ratio=0.460198 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=15616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.198388393e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=354701312 dispatch_meta_bytes=222400 combine_payload_bytes=354701312 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=354923712 ingress_bytes=685002752 link_bytes=685002752 token_bytes=709625024 weight_movement_bytes=330301440 total_nvlink_bytes=1039926464 weight_to_token_ratio=0.465459 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=26752 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.191332449e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=315234304 dispatch_meta_bytes=216160 combine_payload_bytes=315234304 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=315450464 ingress_bytes=425334784 link_bytes=425334784 token_bytes=630684768 weight_movement_bytes=110100480 total_nvlink_bytes=740785248 weight_to_token_ratio=0.174573 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=19840 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.397271449e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60356 payload_rows_remote=52164 dup_rows=5180 dup_rows_remote=234 dedup_ratio=0.079041 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 0, 0, 3661, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3606, 0, 139, 4096, 0, 4083, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 134, 0, 2286, 0, 0, 0, 2466, 0, 0, 0, 0, 7] payload_rows_per_dst=[8182, 7322, 8190, 8188, 7284, 8168, 4830, 8192] dup_rows_per_dst=[2, 10, 0, 2, 164, 46, 10, 4946] combine_rows_per_src=[8182, 7322, 8190, 8188, 7284, 8168, 4830, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61666 payload_rows_remote=53474 dup_rows=3870 dup_rows_remote=3870 dedup_ratio=0.059052 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 3754, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3750, 0, 41, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 20, 0, 1884, 0, 0, 0, 2839, 0, 0, 0, 0, 2] payload_rows_per_dst=[8190, 7508, 8192, 8192, 7524, 8190, 8192, 5678] dup_rows_per_dst=[0, 0, 0, 0, 58, 0, 3808, 4] combine_rows_per_src=[8190, 7508, 8192, 8192, 7524, 8190, 8192, 5678] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=373911552 dispatch_meta_bytes=209592 combine_payload_bytes=373911552 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=374121144 ingress_bytes=484012032 link_bytes=484012032 token_bytes=748032696 weight_movement_bytes=110100480 total_nvlink_bytes=858133176 weight_to_token_ratio=0.147187 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=19840 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.417765299e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60430 payload_rows_remote=52240 dup_rows=5106 dup_rows_remote=5106 dedup_ratio=0.077911 router_tokens_per_expert=[0, 0, 0, 0, 0, 3, 0, 0, 0, 3446, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3776, 0, 13, 4096, 0, 4093, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 65, 0, 2473, 0, 0, 0, 2514, 0, 0, 0, 0, 1] payload_rows_per_dst=[8192, 6892, 8190, 8192, 7556, 8188, 8192, 5028] dup_rows_per_dst=[6, 0, 0, 0, 22, 0, 5076, 2] combine_rows_per_src=[8192, 6892, 8190, 8192, 7556, 8188, 8192, 5028] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60516 payload_rows_remote=53062 dup_rows=5020 dup_rows_remote=4808 dedup_ratio=0.076599 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 3640, 0, 4096, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3753, 0, 32, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 69, 0, 2476, 0, 0, 0, 2303, 0, 0, 0, 0, 16] payload_rows_per_dst=[8192, 7454, 8192, 8192, 7518, 8184, 8178, 4606] dup_rows_per_dst=[0, 212, 6, 0, 52, 0, 4718, 32] combine_rows_per_src=[8192, 7454, 8192, 8192, 7518, 8184, 8178, 4606] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60046 payload_rows_remote=51858 dup_rows=5490 dup_rows_remote=5488 dedup_ratio=0.083771 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 3392, 0, 4096, 0, 4095, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3690, 0, 33, 4096, 0, 4094, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 117, 0, 2341, 0, 0, 0, 2710, 0, 0, 0, 0, 3] payload_rows_per_dst=[8192, 6784, 8190, 8192, 7410, 8188, 4898, 8192] dup_rows_per_dst=[0, 0, 6, 2, 36, 2, 18, 5426] combine_rows_per_src=[8192, 6784, 8190, 8192, 7410, 8188, 4898, 8192] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61458 payload_rows_remote=53316 dup_rows=4078 dup_rows_remote=4000 dedup_ratio=0.062225 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 1775, 4096, 0, 4096, 0, 483, 0, 0, 0, 623, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4071, 39, 0, 0, 0, 0, 0, 0, 1354, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 2, 0] payload_rows_per_dst=[8192, 4474, 8192, 7882, 8192, 8142, 8192, 8192] dup_rows_per_dst=[0, 42, 1246, 0, 0, 78, 2712, 0] combine_rows_per_src=[8192, 4474, 8192, 7882, 8192, 8142, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=371990528 dispatch_meta_bytes=229360 combine_payload_bytes=371990528 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=372219888 ingress_bytes=482091008 link_bytes=482091008 token_bytes=744210416 weight_movement_bytes=110100480 total_nvlink_bytes=854310896 weight_to_token_ratio=0.147943 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.384355533e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=372234240 dispatch_meta_bytes=207880 combine_payload_bytes=372234240 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=372442120 ingress_bytes=702535680 link_bytes=702535680 token_bytes=744676360 weight_movement_bytes=330301440 total_nvlink_bytes=1074977800 weight_to_token_ratio=0.443550 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=21120 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.221825095e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=379373568 dispatch_meta_bytes=229384 combine_payload_bytes=379373568 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=379602952 ingress_bytes=489474048 link_bytes=489474048 token_bytes=758976520 weight_movement_bytes=110100480 total_nvlink_bytes=869077000 weight_to_token_ratio=0.145064 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.512758741e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61754 payload_rows_remote=53562 dup_rows=3782 dup_rows_remote=1382 dedup_ratio=0.057709 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 1919, 4096, 0, 4096, 0, 449, 0, 0, 0, 642, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3967, 0, 0, 1, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 4074, 37, 0, 0, 0, 0, 0, 0, 1198, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 4718, 8192, 7934, 8192, 8148, 8192, 8186] dup_rows_per_dst=[0, 18, 1284, 2, 4, 74, 2400, 0] combine_rows_per_src=[8192, 4718, 8192, 7934, 8192, 8148, 8192, 8186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61262 payload_rows_remote=53072 dup_rows=4274 dup_rows_remote=4258 dedup_ratio=0.065216 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 0, 1870, 4096, 0, 4096, 0, 585, 0, 0, 0, 832, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 57, 0, 0, 0, 0, 0, 0, 913, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[8188, 4846, 8192, 7850, 8174, 7630, 8192, 8190] dup_rows_per_dst=[2, 68, 1664, 0, 0, 114, 2410, 16] combine_rows_per_src=[8188, 4846, 8192, 7850, 8174, 7630, 8192, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61646 payload_rows_remote=53764 dup_rows=3890 dup_rows_remote=3890 dedup_ratio=0.059357 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 0, 1935, 4096, 0, 4096, 0, 409, 0, 0, 0, 710, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4076, 56, 0, 0, 0, 0, 0, 0, 1160, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 4654, 8192, 7882, 8192, 8152, 8192, 8190] dup_rows_per_dst=[2, 34, 1420, 0, 0, 112, 2322, 0] combine_rows_per_src=[8192, 4654, 8192, 7882, 8192, 8152, 8192, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63294 payload_rows_remote=55102 dup_rows=2242 dup_rows_remote=2238 dedup_ratio=0.034210 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 2069, 4096, 0, 4096, 0, 1233, 0, 0, 0, 343, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3873, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 4081, 15, 0, 0, 0, 0, 0, 0, 672, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 6426, 8192, 7746, 8192, 8162, 8192, 8192] dup_rows_per_dst=[0, 178, 686, 0, 4, 30, 1344, 0] combine_rows_per_src=[8192, 6426, 8192, 7746, 8192, 8162, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54660 payload_rows_remote=46468 dup_rows=10876 dup_rows_remote=10874 dedup_ratio=0.165955 router_tokens_per_expert=[0, 0, 0, 0, 0, 2, 0, 0, 0, 3311, 0, 4095, 0, 4095, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3531, 0, 30, 4096, 0, 4091, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 131, 0, 2796, 0, 0, 0, 2479, 0, 0, 0, 0, 7] payload_rows_per_dst=[8192, 8192, 8184, 8192, 8156, 8190, 596, 4958] dup_rows_per_dst=[2, 6632, 4, 0, 4224, 0, 0, 14] combine_rows_per_src=[8192, 8192, 8184, 8192, 8156, 8190, 596, 4958] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61036 payload_rows_remote=53118 dup_rows=4500 dup_rows_remote=4486 dedup_ratio=0.068665 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 3435, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3959, 0, 7, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 54, 0, 2140, 0, 0, 0, 2690, 0, 0, 0, 0, 3] payload_rows_per_dst=[8192, 6870, 8192, 8192, 7918, 8192, 7602, 5878] dup_rows_per_dst=[0, 0, 0, 0, 14, 0, 3732, 754] combine_rows_per_src=[8192, 6870, 8192, 8192, 7918, 8192, 7602, 5878] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61870 payload_rows_remote=56186 dup_rows=3666 dup_rows_remote=3664 dedup_ratio=0.055939 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 0, 0, 3829, 0, 4096, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3782, 0, 18, 4096, 0, 4092, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 99, 0, 2744, 0, 0, 0, 1814, 0, 0, 0, 0, 3] payload_rows_per_dst=[8192, 7660, 8190, 8192, 7572, 8188, 5684, 8192] dup_rows_per_dst=[2, 0, 0, 0, 28, 0, 2, 3634] combine_rows_per_src=[8192, 7660, 8190, 8192, 7572, 8188, 5684, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=383301632 dispatch_meta_bytes=229376 combine_payload_bytes=383301632 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=383531008 ingress_bytes=493402112 link_bytes=493402112 token_bytes=766832640 weight_movement_bytes=110100480 total_nvlink_bytes=876933120 weight_to_token_ratio=0.143578 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.581074235e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=374456320 dispatch_meta_bytes=229384 combine_payload_bytes=374456320 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=374685704 ingress_bytes=484556800 link_bytes=484556800 token_bytes=749142024 weight_movement_bytes=110100480 total_nvlink_bytes=859242504 weight_to_token_ratio=0.146969 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.427239711e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=380348416 dispatch_meta_bytes=231480 combine_payload_bytes=380348416 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=380579896 ingress_bytes=710649856 link_bytes=710649856 token_bytes=760928312 weight_movement_bytes=330301440 total_nvlink_bytes=1091229752 weight_to_token_ratio=0.434077 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=28544 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.235936981e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=371718144 dispatch_meta_bytes=229384 combine_payload_bytes=371718144 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=371947528 ingress_bytes=591919104 link_bytes=591919104 token_bytes=743665672 weight_movement_bytes=220200960 total_nvlink_bytes=963866632 weight_to_token_ratio=0.296102 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.029444676e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=333082624 dispatch_meta_bytes=229368 combine_payload_bytes=333082624 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=333311992 ingress_bytes=443183104 link_bytes=443183104 token_bytes=666394616 weight_movement_bytes=110100480 total_nvlink_bytes=776495096 weight_to_token_ratio=0.165218 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.707683090e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=380749824 dispatch_meta_bytes=230416 combine_payload_bytes=380749824 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=380980240 ingress_bytes=821151744 link_bytes=821151744 token_bytes=761730064 weight_movement_bytes=440401920 total_nvlink_bytes=1202131984 weight_to_token_ratio=0.578160 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=29824 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.428117939e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=402741248 dispatch_meta_bytes=239400 combine_payload_bytes=402741248 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=402980648 ingress_bytes=733042688 link_bytes=733042688 token_bytes=805721896 weight_movement_bytes=330301440 total_nvlink_bytes=1136023336 weight_to_token_ratio=0.409945 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=16512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.274881799e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62656 payload_rows_remote=54468 dup_rows=2880 dup_rows_remote=2866 dedup_ratio=0.043945 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 0, 969, 4096, 0, 4096, 0, 2040, 0, 0, 0, 198, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4090, 383, 0, 0, 0, 0, 0, 0, 783, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[8190, 5886, 8192, 7634, 8192, 8182, 8192, 8188] dup_rows_per_dst=[2, 134, 396, 0, 0, 764, 1570, 14] combine_rows_per_src=[8190, 5886, 8192, 7634, 8192, 8182, 8192, 8188] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=382169088 dispatch_meta_bytes=229264 combine_payload_bytes=382169088 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=382398352 ingress_bytes=602370048 link_bytes=602370048 token_bytes=764567440 weight_movement_bytes=220200960 total_nvlink_bytes=984768400 weight_to_token_ratio=0.288007 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.047620586e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=383932416 dispatch_meta_bytes=219776 combine_payload_bytes=383932416 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=384152192 ingress_bytes=714233856 link_bytes=714233856 token_bytes=768084608 weight_movement_bytes=330301440 total_nvlink_bytes=1098386048 weight_to_token_ratio=0.430033 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=23808 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.242170147e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=380420096 dispatch_meta_bytes=229320 combine_payload_bytes=380420096 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=380649416 ingress_bytes=600621056 link_bytes=600621056 token_bytes=761069512 weight_movement_bytes=220200960 total_nvlink_bytes=981270472 weight_to_token_ratio=0.289331 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.044578802e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=385380352 dispatch_meta_bytes=230616 combine_payload_bytes=385380352 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=385610968 ingress_bytes=715681792 link_bytes=715681792 token_bytes=770991320 weight_movement_bytes=330301440 total_nvlink_bytes=1101292760 weight_to_token_ratio=0.428411 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.244688346e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=394971136 dispatch_meta_bytes=229360 combine_payload_bytes=394971136 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=395200496 ingress_bytes=505071616 link_bytes=505071616 token_bytes=790171632 weight_movement_bytes=110100480 total_nvlink_bytes=900272112 weight_to_token_ratio=0.139337 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.784026102e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52570 payload_rows_remote=44378 dup_rows=12966 dup_rows_remote=12966 dedup_ratio=0.197845 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 1776, 4096, 0, 4096, 0, 506, 0, 0, 0, 695, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3911, 0, 0, 2, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 4096, 100, 0, 0, 0, 0, 0, 0, 1202, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 1390, 7822, 8192, 8192, 2404, 8186] dup_rows_per_dst=[0, 12756, 0, 4, 4, 200, 2, 0] combine_rows_per_src=[8192, 8192, 1390, 7822, 8192, 8192, 2404, 8186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57368 payload_rows_remote=49502 dup_rows=8168 dup_rows_remote=1830 dedup_ratio=0.124634 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 2286, 4096, 0, 4096, 0, 372, 0, 0, 0, 532, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4009, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 4091, 50, 0, 0, 0, 0, 0, 0, 945, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7866, 6824, 8030, 8192, 8182, 1890, 8192] dup_rows_per_dst=[0, 6338, 898, 826, 6, 100, 0, 0] combine_rows_per_src=[8192, 7866, 6824, 8030, 8192, 8182, 1890, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60840 payload_rows_remote=52648 dup_rows=4696 dup_rows_remote=3724 dedup_ratio=0.071655 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 1697, 4096, 0, 4096, 0, 766, 0, 0, 0, 486, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4084, 26, 0, 0, 0, 0, 0, 0, 1297, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 4868, 8192, 7930, 8192, 8168, 7106, 8192] dup_rows_per_dst=[0, 58, 972, 1628, 0, 52, 1986, 0] combine_rows_per_src=[8192, 4868, 8192, 7930, 8192, 8168, 7106, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=390426624 dispatch_meta_bytes=229336 combine_payload_bytes=390426624 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=390655960 ingress_bytes=610627584 link_bytes=610627584 token_bytes=781082584 weight_movement_bytes=220200960 total_nvlink_bytes=1001283544 weight_to_token_ratio=0.281918 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061981800e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62386 payload_rows_remote=55138 dup_rows=3150 dup_rows_remote=3150 dedup_ratio=0.048065 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 1246, 4096, 0, 4096, 0, 2015, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4053, 383, 0, 0, 0, 0, 0, 0, 844, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 6070, 8192, 7248, 8192, 8108, 8192, 8192] dup_rows_per_dst=[0, 452, 244, 0, 0, 764, 1690, 0] combine_rows_per_src=[8192, 6070, 8192, 7248, 8192, 8108, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62674 payload_rows_remote=54482 dup_rows=2862 dup_rows_remote=2712 dedup_ratio=0.043671 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 1112, 4096, 0, 4096, 0, 2357, 0, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4032, 209, 0, 0, 0, 0, 0, 0, 897, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 6432, 8192, 7212, 8192, 8070, 8192, 8192] dup_rows_per_dst=[0, 506, 150, 0, 0, 412, 1794, 0] combine_rows_per_src=[8192, 6432, 8192, 7212, 8192, 8070, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58686 payload_rows_remote=51976 dup_rows=6850 dup_rows_remote=4162 dedup_ratio=0.104523 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 1386, 4096, 0, 4096, 0, 1709, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4061, 326, 0, 0, 0, 0, 0, 0, 854, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 6710, 8192, 7808, 8192, 8156, 3244, 8192] dup_rows_per_dst=[0, 2688, 96, 0, 0, 3642, 424, 0] combine_rows_per_src=[8192, 6710, 8192, 7808, 8192, 8156, 3244, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62030 payload_rows_remote=53922 dup_rows=3506 dup_rows_remote=2928 dedup_ratio=0.053497 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 1368, 4096, 0, 4096, 0, 1523, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4054, 289, 0, 0, 0, 0, 0, 0, 1290, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0] payload_rows_per_dst=[8192, 5556, 8192, 7406, 8192, 8108, 8192, 8192] dup_rows_per_dst=[0, 226, 120, 0, 0, 578, 2580, 2] combine_rows_per_src=[8192, 5556, 8192, 7406, 8192, 8108, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51470 payload_rows_remote=43278 dup_rows=14066 dup_rows_remote=14066 dedup_ratio=0.214630 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 1079, 4096, 0, 4096, 0, 1969, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3634, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4085, 433, 0, 0, 0, 0, 0, 0, 1002, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 6812, 8072, 8192, 1820, 2004, 8186] dup_rows_per_dst=[0, 7522, 126, 6268, 0, 148, 2, 0] combine_rows_per_src=[8192, 8192, 6812, 8072, 8192, 1820, 2004, 8186] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=318101504 dispatch_meta_bytes=229376 combine_payload_bytes=318101504 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=318330880 ingress_bytes=428201984 link_bytes=428201984 token_bytes=636432384 weight_movement_bytes=110100480 total_nvlink_bytes=746532864 weight_to_token_ratio=0.172996 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.447136773e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61854 payload_rows_remote=53662 dup_rows=3682 dup_rows_remote=4 dedup_ratio=0.056183 router_tokens_per_expert=[0, 4083, 3, 0, 4068, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 34, 2274, 0, 4092, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1837, 3, 3, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4082] payload_rows_per_dst=[8142, 8188, 4616, 8168, 8192, 8192, 8192, 8164] dup_rows_per_dst=[0, 0, 0, 0, 0, 3678, 4, 0] combine_rows_per_src=[8142, 8188, 4616, 8168, 8192, 8192, 8192, 8164] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=354830336 dispatch_meta_bytes=205328 combine_payload_bytes=354830336 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=355035664 ingress_bytes=685131776 link_bytes=685131776 token_bytes=709866000 weight_movement_bytes=330301440 total_nvlink_bytes=1040167440 weight_to_token_ratio=0.465301 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=15360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.191556843e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62696 payload_rows_remote=54504 dup_rows=2840 dup_rows_remote=2838 dedup_ratio=0.043335 router_tokens_per_expert=[0, 4092, 0, 0, 4067, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 38, 2681, 0, 4095, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1407, 3, 1, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4086] payload_rows_per_dst=[8134, 8190, 5438, 8186, 8192, 8192, 8192, 8172] dup_rows_per_dst=[0, 2, 0, 16, 0, 2820, 2, 0] combine_rows_per_src=[8134, 8190, 5438, 8186, 8192, 8192, 8192, 8172] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=377380864 dispatch_meta_bytes=225488 combine_payload_bytes=377380864 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=377606352 ingress_bytes=487481344 link_bytes=487481344 token_bytes=754987216 weight_movement_bytes=110100480 total_nvlink_bytes=865087696 weight_to_token_ratio=0.145831 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=28032 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.478102341e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62626 payload_rows_remote=54456 dup_rows=2910 dup_rows_remote=2782 dedup_ratio=0.044403 router_tokens_per_expert=[0, 3988, 0, 0, 4007, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 42, 3079, 0, 4094, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1160, 3, 6, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4085] payload_rows_per_dst=[7590, 8158, 6146, 7986, 8192, 8192, 8192, 8170] dup_rows_per_dst=[0, 22, 2, 424, 0, 2334, 0, 128] combine_rows_per_src=[7590, 8158, 6146, 7986, 8192, 8192, 8192, 8170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=395229184 dispatch_meta_bytes=233152 combine_payload_bytes=395229184 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=395462336 ingress_bytes=725530624 link_bytes=725530624 token_bytes=790691520 weight_movement_bytes=330301440 total_nvlink_bytes=1120992960 weight_to_token_ratio=0.417737 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=29312 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.261817084e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62394 payload_rows_remote=54202 dup_rows=3142 dup_rows_remote=952 dedup_ratio=0.047943 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 1261, 4096, 0, 4096, 0, 1992, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3622, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4040, 197, 0, 0, 0, 0, 0, 0, 1093, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 6106, 8192, 7244, 8192, 8090, 8192, 8186] dup_rows_per_dst=[0, 400, 166, 2, 0, 384, 2190, 0] combine_rows_per_src=[8192, 6106, 8192, 7244, 8192, 8090, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63272 payload_rows_remote=55080 dup_rows=2264 dup_rows_remote=2264 dedup_ratio=0.034546 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 1270, 4096, 0, 4096, 0, 3058, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4028, 127, 0, 0, 0, 0, 0, 0, 443, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7614, 8192, 6642, 8192, 8056, 8192, 8192] dup_rows_per_dst=[0, 1042, 82, 0, 0, 254, 886, 0] combine_rows_per_src=[8192, 7614, 8192, 6642, 8192, 8056, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=390526976 dispatch_meta_bytes=228776 combine_payload_bytes=390526976 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=390755752 ingress_bytes=500627456 link_bytes=500627456 token_bytes=781282728 weight_movement_bytes=110100480 total_nvlink_bytes=891383208 weight_to_token_ratio=0.140923 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.706734851e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=372563968 dispatch_meta_bytes=224552 combine_payload_bytes=372563968 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=372788520 ingress_bytes=702865408 link_bytes=702865408 token_bytes=745352488 weight_movement_bytes=330301440 total_nvlink_bytes=1075653928 weight_to_token_ratio=0.443148 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=16768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.222398546e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58934 payload_rows_remote=50762 dup_rows=6602 dup_rows_remote=6484 dedup_ratio=0.100739 router_tokens_per_expert=[0, 4031, 0, 0, 3850, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 42, 857, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3502, 3, 6, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4086] payload_rows_per_dst=[7496, 8080, 2724, 8186, 8192, 7894, 8190, 8172] dup_rows_per_dst=[0, 206, 648, 0, 0, 5630, 0, 118] combine_rows_per_src=[7496, 8080, 2724, 8186, 8192, 7894, 8190, 8172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=386512896 dispatch_meta_bytes=227400 combine_payload_bytes=386512896 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=386740296 ingress_bytes=606713856 link_bytes=606713856 token_bytes=773253192 weight_movement_bytes=220200960 total_nvlink_bytes=993454152 weight_to_token_ratio=0.284772 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=28928 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.055175183e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=310216704 dispatch_meta_bytes=229376 combine_payload_bytes=310216704 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=310446080 ingress_bytes=420317184 link_bytes=420317184 token_bytes=620662784 weight_movement_bytes=110100480 total_nvlink_bytes=730763264 weight_to_token_ratio=0.177392 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.310007133e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=388519936 dispatch_meta_bytes=220616 combine_payload_bytes=388519936 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=388740552 ingress_bytes=608720896 link_bytes=608720896 token_bytes=777260488 weight_movement_bytes=220200960 total_nvlink_bytes=997461448 weight_to_token_ratio=0.283304 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=25600 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058665756e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=394813440 dispatch_meta_bytes=229376 combine_payload_bytes=394813440 prefetch_bytes=0 reduce_bytes=0 egress_bytes=395042816 ingress_bytes=394813440 link_bytes=395042816 token_bytes=789856256 weight_movement_bytes=0 total_nvlink_bytes=789856256 weight_to_token_ratio=0.000000 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.870444304e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=363862016 dispatch_meta_bytes=228984 combine_payload_bytes=363862016 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=364091000 ingress_bytes=584062976 link_bytes=584062976 token_bytes=727953016 weight_movement_bytes=220200960 total_nvlink_bytes=948153976 weight_to_token_ratio=0.302493 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.015781577e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64082 payload_rows_remote=55892 dup_rows=1454 dup_rows_remote=1452 dedup_ratio=0.022186 router_tokens_per_expert=[0, 4084, 1, 0, 3766, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 39, 686, 0, 4095, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3715, 1, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4090] payload_rows_per_dst=[7534, 8170, 8192, 8190, 8192, 7432, 8192, 8180] dup_rows_per_dst=[0, 2, 1448, 2, 0, 2, 0, 0] combine_rows_per_src=[7534, 8170, 8192, 8190, 8192, 7432, 8192, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64554 payload_rows_remote=56364 dup_rows=982 dup_rows_remote=16 dedup_ratio=0.014984 router_tokens_per_expert=[0, 4094, 1, 0, 4012, 0, 0, 0, 0, 0, 2, 0, 0, 3, 0, 0, 0, 0, 24, 461, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3698, 3, 1, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4087] payload_rows_per_dst=[8026, 8192, 8190, 8192, 8192, 7396, 8192, 8174] dup_rows_per_dst=[0, 6, 966, 0, 0, 8, 2, 0] combine_rows_per_src=[8026, 8192, 8190, 8192, 8192, 7396, 8192, 8174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64188 payload_rows_remote=56002 dup_rows=1348 dup_rows_remote=1348 dedup_ratio=0.020569 router_tokens_per_expert=[0, 4093, 0, 0, 3861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 635, 0, 4096, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3663, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093] payload_rows_per_dst=[7722, 8186, 8192, 8192, 8192, 7326, 8192, 8186] dup_rows_per_dst=[0, 0, 1342, 6, 0, 0, 0, 0] combine_rows_per_src=[7722, 8186, 8192, 8192, 8192, 7326, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64448 payload_rows_remote=57082 dup_rows=1088 dup_rows_remote=1076 dedup_ratio=0.016602 router_tokens_per_expert=[0, 4084, 3, 0, 3988, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 5, 535, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3683, 3, 3, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4083] payload_rows_per_dst=[7982, 8170, 8192, 8188, 8192, 7366, 8192, 8166] dup_rows_per_dst=[0, 0, 1074, 0, 0, 12, 2, 0] combine_rows_per_src=[7982, 8170, 8192, 8188, 8192, 7366, 8192, 8166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61834 payload_rows_remote=53912 dup_rows=3702 dup_rows_remote=804 dedup_ratio=0.056488 router_tokens_per_expert=[0, 4066, 0, 0, 3847, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 42, 820, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3508, 2, 2, 0, 6, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 4087] payload_rows_per_dst=[7922, 8140, 6012, 8188, 8192, 7018, 8186, 8176] dup_rows_per_dst=[2898, 4, 794, 0, 0, 6, 0, 0] combine_rows_per_src=[7922, 8140, 6012, 8188, 8192, 7018, 8186, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=400633856 dispatch_meta_bytes=229376 combine_payload_bytes=400633856 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=400863232 ingress_bytes=510734336 link_bytes=510734336 token_bytes=801497088 weight_movement_bytes=110100480 total_nvlink_bytes=911597568 weight_to_token_ratio=0.137369 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.882510117e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64620 payload_rows_remote=56428 dup_rows=916 dup_rows_remote=912 dedup_ratio=0.013977 router_tokens_per_expert=[0, 4093, 0, 0, 3931, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 19, 429, 0, 4095, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3815, 3, 1, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4087] payload_rows_per_dst=[7862, 8186, 8190, 8192, 8192, 7632, 8192, 8174] dup_rows_per_dst=[0, 4, 896, 6, 0, 6, 4, 0] combine_rows_per_src=[7862, 8186, 8190, 8192, 8192, 7632, 8192, 8174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65050 payload_rows_remote=56858 dup_rows=486 dup_rows_remote=486 dedup_ratio=0.007416 router_tokens_per_expert=[0, 4095, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 223, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3904, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095] payload_rows_per_dst=[8094, 8190, 8192, 8192, 8192, 7808, 8192, 8190] dup_rows_per_dst=[0, 0, 484, 2, 0, 0, 0, 0] combine_rows_per_src=[8094, 8190, 8192, 8192, 8192, 7808, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=404017152 dispatch_meta_bytes=225520 combine_payload_bytes=404017152 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=404242672 ingress_bytes=954519552 link_bytes=954519552 token_bytes=808259824 weight_movement_bytes=550502400 total_nvlink_bytes=1358762224 weight_to_token_ratio=0.681096 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=27904 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.660066492e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61706 payload_rows_remote=53608 dup_rows=3830 dup_rows_remote=3828 dedup_ratio=0.058441 router_tokens_per_expert=[0, 4049, 1, 0, 4037, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 37, 2498, 0, 4093, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1668, 1, 1, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4092] payload_rows_per_dst=[8076, 8192, 5168, 8098, 8192, 7604, 8192, 8184] dup_rows_per_dst=[0, 2, 678, 2, 0, 3146, 2, 0] combine_rows_per_src=[8076, 8192, 5168, 8098, 8192, 7604, 8192, 8184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61964 payload_rows_remote=53772 dup_rows=3572 dup_rows_remote=3572 dedup_ratio=0.054504 router_tokens_per_expert=[0, 4094, 0, 0, 4089, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 44, 2276, 0, 4095, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1785, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094] payload_rows_per_dst=[8178, 8192, 4640, 8190, 8192, 8192, 8192, 8188] dup_rows_per_dst=[0, 2, 0, 2, 0, 3568, 0, 0] combine_rows_per_src=[8178, 8192, 4640, 8190, 8192, 8192, 8192, 8188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=384649216 dispatch_meta_bytes=214664 combine_payload_bytes=384649216 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=384863880 ingress_bytes=494749696 link_bytes=494749696 token_bytes=769513096 weight_movement_bytes=110100480 total_nvlink_bytes=879613576 weight_to_token_ratio=0.143078 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=20096 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.604510937e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=390684672 dispatch_meta_bytes=229368 combine_payload_bytes=390684672 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=390914040 ingress_bytes=500785152 link_bytes=500785152 token_bytes=781598712 weight_movement_bytes=110100480 total_nvlink_bytes=891699192 weight_to_token_ratio=0.140866 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.709477443e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=390340608 dispatch_meta_bytes=228952 combine_payload_bytes=390340608 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=390569560 ingress_bytes=610541568 link_bytes=610541568 token_bytes=780910168 weight_movement_bytes=220200960 total_nvlink_bytes=1001111128 weight_to_token_ratio=0.281980 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061832204e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65416 payload_rows_remote=57224 dup_rows=120 dup_rows_remote=42 dedup_ratio=0.001831 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4092, 5, 6, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4092, 0, 0, 0, 10, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8170, 8160, 8134, 8192, 8184, 8192] dup_rows_per_dst=[4, 0, 2, 0, 14, 0, 22, 78] combine_rows_per_src=[8192, 8192, 8170, 8160, 8134, 8192, 8184, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=401422336 dispatch_meta_bytes=229400 combine_payload_bytes=401422336 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=401651736 ingress_bytes=621623296 link_bytes=621623296 token_bytes=803074072 weight_movement_bytes=220200960 total_nvlink_bytes=1023275032 weight_to_token_ratio=0.274198 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.081105151e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=409163776 dispatch_meta_bytes=232632 combine_payload_bytes=409163776 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=409396408 ingress_bytes=519264256 link_bytes=519264256 token_bytes=818560184 weight_movement_bytes=110100480 total_nvlink_bytes=928660664 weight_to_token_ratio=0.134505 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=29568 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.030859455e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=386441216 dispatch_meta_bytes=218864 combine_payload_bytes=386441216 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=386660080 ingress_bytes=716742656 link_bytes=716742656 token_bytes=773101296 weight_movement_bytes=330301440 total_nvlink_bytes=1103402736 weight_to_token_ratio=0.427242 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31232 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.246533363e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=384262144 dispatch_meta_bytes=229744 combine_payload_bytes=384262144 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=384491888 ingress_bytes=604463104 link_bytes=604463104 token_bytes=768754032 weight_movement_bytes=220200960 total_nvlink_bytes=988954992 weight_to_token_ratio=0.286439 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.051260755e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=385437696 dispatch_meta_bytes=229376 combine_payload_bytes=385437696 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=385667072 ingress_bytes=495538176 link_bytes=495538176 token_bytes=771104768 weight_movement_bytes=110100480 total_nvlink_bytes=881205248 weight_to_token_ratio=0.142783 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.618223901e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59984 payload_rows_remote=51826 dup_rows=5552 dup_rows_remote=4410 dedup_ratio=0.084717 router_tokens_per_expert=[0, 4056, 1, 0, 4047, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 44, 2746, 0, 4095, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1390, 3, 3, 0, 5, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 4085] payload_rows_per_dst=[8158, 8192, 8192, 8112, 8192, 2784, 8182, 8172] dup_rows_per_dst=[1142, 6, 4394, 2, 0, 8, 0, 0] combine_rows_per_src=[8158, 8192, 8192, 8112, 8192, 2784, 8182, 8172] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60544 payload_rows_remote=52352 dup_rows=4992 dup_rows_remote=4992 dedup_ratio=0.076172 router_tokens_per_expert=[0, 4086, 0, 0, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 2451, 0, 4096, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1695, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4092] payload_rows_per_dst=[8030, 8192, 8192, 8172, 8192, 3390, 8192, 8184] dup_rows_per_dst=[0, 0, 4980, 12, 0, 0, 0, 0] combine_rows_per_src=[8030, 8192, 8192, 8172, 8192, 3390, 8192, 8184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60622 payload_rows_remote=52432 dup_rows=4914 dup_rows_remote=14 dedup_ratio=0.074982 router_tokens_per_expert=[0, 4083, 2, 0, 4041, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 38, 2414, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1715, 3, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4087] payload_rows_per_dst=[8086, 8192, 8190, 8166, 8192, 3430, 8192, 8174] dup_rows_per_dst=[0, 6, 4900, 0, 0, 6, 2, 0] combine_rows_per_src=[8086, 8192, 8190, 8166, 8192, 3430, 8192, 8174] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65524 payload_rows_remote=57336 dup_rows=12 dup_rows_remote=12 dedup_ratio=0.000183 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 3, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4095, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8186, 8192, 8192, 8188, 8190] dup_rows_per_dst=[2, 0, 0, 0, 10, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8186, 8192, 8192, 8188, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65516 payload_rows_remote=57324 dup_rows=20 dup_rows_remote=20 dedup_ratio=0.000305 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 2, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4092, 0, 0, 0, 2, 0, 4094, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8184, 8192, 8192, 8184, 8188] dup_rows_per_dst=[4, 0, 2, 0, 10, 0, 0, 4] combine_rows_per_src=[8192, 8192, 8192, 8184, 8192, 8192, 8184, 8188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=371488768 dispatch_meta_bytes=224944 combine_payload_bytes=371488768 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=371713712 ingress_bytes=701790208 link_bytes=701790208 token_bytes=743202480 weight_movement_bytes=330301440 total_nvlink_bytes=1073503920 weight_to_token_ratio=0.444430 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.220528596e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=375259136 dispatch_meta_bytes=229376 combine_payload_bytes=375259136 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=375488512 ingress_bytes=485359616 link_bytes=485359616 token_bytes=750747648 weight_movement_bytes=110100480 total_nvlink_bytes=860848128 weight_to_token_ratio=0.146654 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.441202002e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65472 payload_rows_remote=57282 dup_rows=64 dup_rows_remote=52 dedup_ratio=0.000977 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4093, 3, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4091, 0, 0, 0, 1, 0, 4095, 0, 0] payload_rows_per_dst=[8178, 8192, 8184, 8188, 8166, 8192, 8182, 8190] dup_rows_per_dst=[2, 0, 2, 14, 10, 0, 24, 12] combine_rows_per_src=[8178, 8192, 8184, 8188, 8166, 8192, 8182, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=375832576 dispatch_meta_bytes=209784 combine_payload_bytes=375832576 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=376042360 ingress_bytes=926334976 link_bytes=926334976 token_bytes=751874936 weight_movement_bytes=550502400 total_nvlink_bytes=1302377336 weight_to_token_ratio=0.732173 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=19840 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.611048879e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65532 payload_rows_remote=57342 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4095, 1, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8190] dup_rows_per_dst=[2, 0, 0, 0, 2, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=410984448 dispatch_meta_bytes=229392 combine_payload_bytes=410984448 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=411213840 ingress_bytes=741285888 link_bytes=741285888 token_bytes=822198288 weight_movement_bytes=330301440 total_nvlink_bytes=1152499728 weight_to_token_ratio=0.401730 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.289218080e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=410898432 dispatch_meta_bytes=229376 combine_payload_bytes=410898432 prefetch_bytes=0 reduce_bytes=0 egress_bytes=411127808 ingress_bytes=410898432 link_bytes=411127808 token_bytes=822026240 weight_movement_bytes=0 total_nvlink_bytes=822026240 weight_to_token_ratio=0.000000 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.150188770e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=410597376 dispatch_meta_bytes=229336 combine_payload_bytes=410597376 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=410826712 ingress_bytes=520697856 link_bytes=520697856 token_bytes=821424088 weight_movement_bytes=110100480 total_nvlink_bytes=931524568 weight_to_token_ratio=0.134036 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.055792117e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=411027456 dispatch_meta_bytes=229384 combine_payload_bytes=411027456 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=411256840 ingress_bytes=631228416 link_bytes=631228416 token_bytes=822284296 weight_movement_bytes=220200960 total_nvlink_bytes=1042485256 weight_to_token_ratio=0.267792 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.097810035e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=411041792 dispatch_meta_bytes=229376 combine_payload_bytes=411041792 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=411271168 ingress_bytes=741343232 link_bytes=741343232 token_bytes=822312960 weight_movement_bytes=330301440 total_nvlink_bytes=1152614400 weight_to_token_ratio=0.401674 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.289317810e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65518 payload_rows_remote=57326 dup_rows=18 dup_rows_remote=14 dedup_ratio=0.000275 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8178, 8192, 8192, 8190] dup_rows_per_dst=[4, 0, 0, 14, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8190, 8178, 8192, 8192, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65522 payload_rows_remote=57330 dup_rows=14 dup_rows_remote=10 dedup_ratio=0.000214 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 3, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 4093, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8186, 8192, 8192, 8190, 8186] dup_rows_per_dst=[0, 0, 4, 0, 8, 0, 0, 2] combine_rows_per_src=[8192, 8192, 8192, 8186, 8192, 8192, 8190, 8186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65364 payload_rows_remote=57172 dup_rows=172 dup_rows_remote=170 dedup_ratio=0.002625 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4094, 0, 67, 0, 0, 0, 0, 0, 0, 4012, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 15, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8188, 8024, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 6, 134, 0, 30, 2, 0] combine_rows_per_src=[8192, 8192, 8192, 8188, 8024, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=404475904 dispatch_meta_bytes=229360 combine_payload_bytes=404475904 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=404705264 ingress_bytes=514576384 link_bytes=514576384 token_bytes=809181168 weight_movement_bytes=110100480 total_nvlink_bytes=919281648 weight_to_token_ratio=0.136064 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.949329650e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=407558144 dispatch_meta_bytes=229376 combine_payload_bytes=407558144 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=407787520 ingress_bytes=517658624 link_bytes=517658624 token_bytes=815345664 weight_movement_bytes=110100480 total_nvlink_bytes=925446144 weight_to_token_ratio=0.135035 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.002934873e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=410181632 dispatch_meta_bytes=229064 combine_payload_bytes=410181632 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=410410696 ingress_bytes=630382592 link_bytes=630382592 token_bytes=820592328 weight_movement_bytes=220200960 total_nvlink_bytes=1040793288 weight_to_token_ratio=0.268344 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.096339008e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65528 payload_rows_remote=57340 dup_rows=8 dup_rows_remote=8 dedup_ratio=0.000122 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 1, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4095, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8188, 8192, 8192, 8192, 8190] dup_rows_per_dst=[0, 0, 2, 0, 4, 0, 0, 2] combine_rows_per_src=[8190, 8192, 8192, 8188, 8192, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65518 payload_rows_remote=57326 dup_rows=18 dup_rows_remote=14 dedup_ratio=0.000275 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 3, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 4093, 0, 0] payload_rows_per_dst=[8188, 8192, 8192, 8186, 8192, 8192, 8190, 8186] dup_rows_per_dst=[0, 0, 4, 0, 12, 0, 0, 2] combine_rows_per_src=[8188, 8192, 8192, 8186, 8192, 8192, 8190, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=411013120 dispatch_meta_bytes=229392 combine_payload_bytes=411013120 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=411242512 ingress_bytes=741314560 link_bytes=741314560 token_bytes=822255632 weight_movement_bytes=330301440 total_nvlink_bytes=1152557072 weight_to_token_ratio=0.401702 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.289267945e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65512 payload_rows_remote=57320 dup_rows=24 dup_rows_remote=24 dedup_ratio=0.000366 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 10, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8168, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 4, 0, 0, 0, 0, 20] combine_rows_per_src=[8192, 8192, 8192, 8168, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65468 payload_rows_remote=57276 dup_rows=68 dup_rows_remote=68 dedup_ratio=0.001038 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 3, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 27, 0, 4094, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8134, 8192, 8192, 8188, 8188] dup_rows_per_dst=[0, 0, 2, 0, 12, 0, 0, 54] combine_rows_per_src=[8190, 8192, 8192, 8134, 8192, 8192, 8188, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65386 payload_rows_remote=57194 dup_rows=150 dup_rows_remote=146 dedup_ratio=0.002289 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8188, 8046] dup_rows_per_dst=[4, 0, 2, 138, 6, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8188, 8046] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65426 payload_rows_remote=57234 dup_rows=110 dup_rows_remote=76 dedup_ratio=0.001678 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4093, 0, 35, 0, 0, 0, 0, 0, 0, 4044, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8186, 8088, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 6, 70, 0, 34, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8186, 8088, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64326 payload_rows_remote=56134 dup_rows=1210 dup_rows_remote=1210 dedup_ratio=0.018463 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4090, 0, 78, 0, 0, 0, 0, 0, 0, 4013, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8158, 7148, 8064, 8192, 8192, 8192] dup_rows_per_dst=[0, 2, 10, 120, 1064, 14, 0, 0] combine_rows_per_src=[8192, 8188, 8158, 7148, 8064, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=411041792 dispatch_meta_bytes=229376 combine_payload_bytes=411041792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=411271168 ingress_bytes=521142272 link_bytes=521142272 token_bytes=822312960 weight_movement_bytes=110100480 total_nvlink_bytes=932413440 weight_to_token_ratio=0.133891 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.063521242e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=410912768 dispatch_meta_bytes=229360 combine_payload_bytes=410912768 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=411142128 ingress_bytes=521013248 link_bytes=521013248 token_bytes=822054896 weight_movement_bytes=110100480 total_nvlink_bytes=932155376 weight_to_token_ratio=0.133933 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.061277302e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=410941440 dispatch_meta_bytes=229360 combine_payload_bytes=410941440 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=411170800 ingress_bytes=521041920 link_bytes=521041920 token_bytes=822112240 weight_movement_bytes=110100480 total_nvlink_bytes=932212720 weight_to_token_ratio=0.133924 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.061775955e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=409808896 dispatch_meta_bytes=229368 combine_payload_bytes=409808896 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=410038264 ingress_bytes=519909376 link_bytes=519909376 token_bytes=819847160 weight_movement_bytes=110100480 total_nvlink_bytes=929947640 weight_to_token_ratio=0.134294 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.042079153e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65520 payload_rows_remote=57332 dup_rows=16 dup_rows_remote=16 dedup_ratio=0.000244 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 3, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4095, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8182, 8192, 8192, 8188, 8190] dup_rows_per_dst=[0, 0, 6, 0, 10, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8182, 8192, 8192, 8188, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65532 payload_rows_remote=57340 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8188, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 4] combine_rows_per_src=[8192, 8192, 8192, 8188, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=410912768 dispatch_meta_bytes=229360 combine_payload_bytes=410912768 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=411142128 ingress_bytes=521013248 link_bytes=521013248 token_bytes=822054896 weight_movement_bytes=110100480 total_nvlink_bytes=932155376 weight_to_token_ratio=0.133933 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.061277302e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64848 payload_rows_remote=56656 dup_rows=688 dup_rows_remote=688 dedup_ratio=0.010498 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4090, 0, 6, 0, 0, 0, 0, 0, 0, 3892, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 197, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8158, 7946, 7784, 8192, 8192, 8192] dup_rows_per_dst=[0, 2, 10, 12, 34, 628, 2, 0] combine_rows_per_src=[8192, 8192, 8158, 7946, 7784, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=410869760 dispatch_meta_bytes=229376 combine_payload_bytes=410869760 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=411099136 ingress_bytes=520970240 link_bytes=520970240 token_bytes=821968896 weight_movement_bytes=110100480 total_nvlink_bytes=932069376 weight_to_token_ratio=0.133947 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.060529322e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=410554368 dispatch_meta_bytes=229376 combine_payload_bytes=410554368 prefetch_bytes=0 reduce_bytes=0 egress_bytes=410783744 ingress_bytes=410554368 link_bytes=410783744 token_bytes=821338112 weight_movement_bytes=0 total_nvlink_bytes=821338112 weight_to_token_ratio=0.000000 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.144204931e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=409966592 dispatch_meta_bytes=229360 combine_payload_bytes=409966592 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=410195952 ingress_bytes=630167552 link_bytes=630167552 token_bytes=820162544 weight_movement_bytes=220200960 total_nvlink_bytes=1040363504 weight_to_token_ratio=0.268485 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.095965018e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65454 payload_rows_remote=57264 dup_rows=82 dup_rows_remote=12 dedup_ratio=0.001251 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 35, 0, 0, 0, 0, 0, 0, 4056, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8112, 8192, 8192, 8192] dup_rows_per_dst=[0, 2, 2, 70, 0, 8, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8190, 8112, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=410253312 dispatch_meta_bytes=229240 combine_payload_bytes=410253312 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=410482552 ingress_bytes=740554752 link_bytes=740554752 token_bytes=820735864 weight_movement_bytes=330301440 total_nvlink_bytes=1151037304 weight_to_token_ratio=0.402445 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.287946514e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65460 payload_rows_remote=57344 dup_rows=76 dup_rows_remote=76 dedup_ratio=0.001160 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 34, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8116, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 68, 0, 8, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8116, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=402368512 dispatch_meta_bytes=229376 combine_payload_bytes=402368512 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=402597888 ingress_bytes=512468992 link_bytes=512468992 token_bytes=804966400 weight_movement_bytes=110100480 total_nvlink_bytes=915066880 weight_to_token_ratio=0.136776 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.912678638e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7575118472 prefetch_bytes=880803840 reduce_bytes=1211105280 weight_movement_bytes=2091909120 egress_bytes=3788694152 ingress_bytes=5878333440 link_bytes=5878333440 total_nvlink_bytes=9667027592 weight_to_token_ratio=0.276155 prefetch_share_of_ingress=0.149839 dedup_ratio=0.081534 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=11 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048162 gemm_inner_pad_upper_bound=6858 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.022338867e-04 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=410468352 dispatch_meta_bytes=229104 combine_payload_bytes=410468352 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=410697456 ingress_bytes=630669312 link_bytes=630669312 token_bytes=821165808 weight_movement_bytes=220200960 total_nvlink_bytes=1041366768 weight_to_token_ratio=0.268157 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.096837661e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7555464672 prefetch_bytes=990904320 reduce_bytes=880803840 weight_movement_bytes=1871708160 egress_bytes=3778874848 ingress_bytes=5648297984 link_bytes=5648527360 total_nvlink_bytes=9427172832 weight_to_token_ratio=0.247729 prefetch_share_of_ingress=0.175434 dedup_ratio=0.076578 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=8 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048162 gemm_inner_pad_upper_bound=7112 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.823718102e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=411041792 dispatch_meta_bytes=229680 combine_payload_bytes=411041792 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=411271472 ingress_bytes=741343232 link_bytes=741343232 token_bytes=822313264 weight_movement_bytes=330301440 total_nvlink_bytes=1152614704 weight_to_token_ratio=0.401673 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.289317810e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7248169232 prefetch_bytes=1101004800 reduce_bytes=880803840 weight_movement_bytes=1981808640 egress_bytes=3625189648 ingress_bytes=5604788224 link_bytes=5604788224 total_nvlink_bytes=9229977872 weight_to_token_ratio=0.273422 prefetch_share_of_ingress=0.196440 dedup_ratio=0.105719 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=8 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.047794 gemm_inner_pad_upper_bound=6858 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.747648551e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65416 payload_rows_remote=57224 dup_rows=120 dup_rows_remote=120 dedup_ratio=0.001831 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 45, 0, 0, 0, 0, 0, 0, 4036, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8072, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 90, 0, 30, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8072, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65328 payload_rows_remote=57136 dup_rows=208 dup_rows_remote=202 dedup_ratio=0.003174 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4093, 0, 46, 0, 0, 0, 0, 0, 0, 3995, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8186, 7990, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 6, 92, 0, 110, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8186, 7990, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65104 payload_rows_remote=56912 dup_rows=432 dup_rows_remote=432 dedup_ratio=0.006592 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 88, 0, 0, 0, 0, 0, 0, 4003, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8006, 7948, 8190, 8192] dup_rows_per_dst=[0, 0, 4, 418, 0, 10, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8006, 7948, 8190, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7564680512 prefetch_bytes=990904320 reduce_bytes=1101004800 weight_movement_bytes=2091909120 egress_bytes=3783488832 ingress_bytes=5873100800 link_bytes=5873100800 total_nvlink_bytes=9656589632 weight_to_token_ratio=0.276536 prefetch_share_of_ingress=0.168719 dedup_ratio=0.075381 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=10 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.046691 gemm_inner_pad_upper_bound=7239 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.021428825e-04 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=410955776 dispatch_meta_bytes=229392 combine_payload_bytes=410955776 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=411185168 ingress_bytes=741257216 link_bytes=741257216 token_bytes=822140944 weight_movement_bytes=330301440 total_nvlink_bytes=1152442384 weight_to_token_ratio=0.401758 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.289168214e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64718 payload_rows_remote=56526 dup_rows=818 dup_rows_remote=768 dedup_ratio=0.012482 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 25, 0, 0, 0, 0, 0, 0, 3687, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7374, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 2, 50, 0, 766, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 7374, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=411013120 dispatch_meta_bytes=229376 combine_payload_bytes=411013120 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=411242496 ingress_bytes=631214080 link_bytes=631214080 token_bytes=822255616 weight_movement_bytes=220200960 total_nvlink_bytes=1042456576 weight_to_token_ratio=0.267801 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.097785102e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=406110208 dispatch_meta_bytes=229376 combine_payload_bytes=406110208 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=406339584 ingress_bytes=516210688 link_bytes=516210688 token_bytes=812449792 weight_movement_bytes=110100480 total_nvlink_bytes=922550272 weight_to_token_ratio=0.135517 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.977752885e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=405178368 dispatch_meta_bytes=229176 combine_payload_bytes=405178368 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=405407544 ingress_bytes=515278848 link_bytes=515278848 token_bytes=810585912 weight_movement_bytes=110100480 total_nvlink_bytes=920686392 weight_to_token_ratio=0.135828 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.961546655e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7207979296 prefetch_bytes=1101004800 reduce_bytes=880803840 weight_movement_bytes=1981808640 egress_bytes=3605084448 ingress_bytes=5584703488 link_bytes=5584703488 total_nvlink_bytes=9189787936 weight_to_token_ratio=0.274946 prefetch_share_of_ingress=0.197147 dedup_ratio=0.111099 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=8 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048162 gemm_inner_pad_upper_bound=6858 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.712717892e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64472 payload_rows_remote=56280 dup_rows=1064 dup_rows_remote=1058 dedup_ratio=0.016235 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4093, 0, 23, 0, 0, 0, 0, 0, 0, 3567, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8186, 7134, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 6, 46, 0, 1012, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8186, 7134, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64850 payload_rows_remote=56658 dup_rows=686 dup_rows_remote=686 dedup_ratio=0.010468 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 3753, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7506, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 4, 0, 682, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 7506, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59928 payload_rows_remote=51736 dup_rows=5608 dup_rows_remote=5608 dedup_ratio=0.085571 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 27, 0, 0, 0, 0, 0, 0, 3900, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8188, 8004, 2782, 8186, 8192] dup_rows_per_dst=[0, 0, 4, 54, 5386, 164, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8188, 8004, 2782, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64644 payload_rows_remote=56452 dup_rows=892 dup_rows_remote=34 dedup_ratio=0.013611 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4093, 0, 14, 0, 0, 0, 0, 0, 0, 3653, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8186, 7306, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 6, 28, 0, 858, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8186, 7306, 8192, 8192, 8192] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7636103096 prefetch_bytes=770703360 reduce_bytes=1651507200 weight_movement_bytes=2422210560 egress_bytes=3819186104 ingress_bytes=6239127552 link_bytes=6239127552 total_nvlink_bytes=10058313656 weight_to_token_ratio=0.317205 prefetch_share_of_ingress=0.123527 dedup_ratio=0.069427 prefetch_slots=14 unused_prefetch_slots=146 slot_utilization=0.087500 reduce_slots=15 remote_demand=7 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.047426 gemm_inner_pad_upper_bound=6858 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.085086897e-04 prefetch_bytes_per_gemm_flop=1.340379902e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=410181632 dispatch_meta_bytes=229376 combine_payload_bytes=410181632 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=410411008 ingress_bytes=520282112 link_bytes=520282112 token_bytes=820592640 weight_movement_bytes=110100480 total_nvlink_bytes=930693120 weight_to_token_ratio=0.134172 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.048561645e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=409550848 dispatch_meta_bytes=229352 combine_payload_bytes=409550848 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=409780200 ingress_bytes=519651328 link_bytes=519651328 token_bytes=819331048 weight_movement_bytes=110100480 total_nvlink_bytes=929431528 weight_to_token_ratio=0.134379 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.037591273e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=407945216 dispatch_meta_bytes=229376 combine_payload_bytes=407945216 prefetch_bytes=0 reduce_bytes=0 egress_bytes=408174592 ingress_bytes=407945216 link_bytes=408174592 token_bytes=816119808 weight_movement_bytes=0 total_nvlink_bytes=816119808 weight_to_token_ratio=0.000000 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.098827487e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7382846664 prefetch_bytes=990904320 reduce_bytes=770703360 weight_movement_bytes=1761607680 egress_bytes=3692530888 ingress_bytes=5451923456 link_bytes=5451923456 total_nvlink_bytes=9144454344 weight_to_token_ratio=0.238608 prefetch_share_of_ingress=0.181753 dedup_ratio=0.094653 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=7 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048529 gemm_inner_pad_upper_bound=6858 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.481791577e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7541499656 prefetch_bytes=1101004800 reduce_bytes=990904320 weight_movement_bytes=2091909120 egress_bytes=3771891464 ingress_bytes=5861517312 link_bytes=5861517312 total_nvlink_bytes=9633408776 weight_to_token_ratio=0.277386 prefetch_share_of_ingress=0.187836 dedup_ratio=0.084341 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=9 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048529 gemm_inner_pad_upper_bound=6985 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.019414266e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6997500592 prefetch_bytes=1101004800 reduce_bytes=440401920 weight_movement_bytes=1541406720 egress_bytes=3499860656 ingress_bytes=5039046656 link_bytes=5039276032 total_nvlink_bytes=8538907312 weight_to_token_ratio=0.220280 prefetch_share_of_ingress=0.218495 dedup_ratio=0.134796 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=4 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050368 gemm_inner_pad_upper_bound=6477 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.764129838e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none - [2026-08-26 15:11:33] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 363.8 | throughput per GPU (TFLOP/s/GPU): 339.5 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 9.389417E+00 | loss scale: 1.0 | grad norm: 4.976 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44766 payload_rows_remote=38966 dup_rows=20770 dup_rows_remote=18796 dedup_ratio=0.316925 router_tokens_per_expert=[1947, 677, 1331, 1366, 14, 197, 346, 76, 844, 2085, 1356, 522, 5, 614, 0, 1242, 1638, 1441, 21, 0, 846, 590, 56, 1106, 124, 888, 608, 543, 76, 196, 511, 1, 157, 151, 856, 15, 98, 0, 442, 83, 500, 675, 215, 380, 2, 1059, 231, 396, 78, 346, 79, 1032, 415, 512, 24, 930, 221, 460, 289, 458, 425, 900, 72, 0] payload_rows_per_dst=[4318, 5054, 5570, 6450, 5800, 6398, 5886, 5290] dup_rows_per_dst=[964, 1628, 2550, 3338, 1974, 3794, 3678, 2844] combine_rows_per_src=[4318, 5054, 5570, 6450, 5800, 6398, 5886, 5290] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44968 payload_rows_remote=38872 dup_rows=20568 dup_rows_remote=17384 dedup_ratio=0.313843 router_tokens_per_expert=[2227, 770, 1537, 1759, 26, 191, 190, 49, 337, 2197, 1540, 482, 1, 701, 3, 1538, 1150, 917, 49, 2, 611, 766, 15, 1377, 93, 961, 522, 561, 60, 102, 195, 1, 548, 87, 636, 7, 157, 0, 590, 109, 150, 676, 603, 579, 2, 910, 95, 362, 23, 104, 150, 857, 220, 732, 69, 1053, 339, 601, 440, 359, 653, 572, 155, 0] payload_rows_per_dst=[6328, 6266, 6096, 5294, 6186, 5128, 4908, 4762] dup_rows_per_dst=[4076, 2938, 3184, 2138, 2476, 2120, 2160, 1476] combine_rows_per_src=[6328, 6266, 6096, 5294, 6186, 5128, 4908, 4762] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44774 payload_rows_remote=39426 dup_rows=20762 dup_rows_remote=18012 dedup_ratio=0.316803 router_tokens_per_expert=[1781, 808, 1799, 1284, 161, 85, 415, 132, 357, 1739, 1170, 458, 0, 992, 1, 1706, 865, 865, 9, 63, 906, 615, 185, 666, 309, 1052, 529, 435, 32, 200, 448, 0, 580, 369, 714, 9, 101, 0, 356, 167, 113, 514, 311, 640, 1, 1056, 40, 509, 23, 61, 183, 691, 511, 815, 34, 1047, 288, 271, 587, 1015, 781, 865, 49, 0] payload_rows_per_dst=[5104, 4828, 4942, 6074, 5804, 5348, 6078, 6596] dup_rows_per_dst=[1696, 1128, 1676, 3498, 2266, 2750, 3220, 4528] combine_rows_per_src=[5104, 4828, 4942, 6074, 5804, 5348, 6078, 6596] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44352 payload_rows_remote=37894 dup_rows=21184 dup_rows_remote=17664 dedup_ratio=0.323242 router_tokens_per_expert=[2111, 900, 1520, 1940, 25, 287, 267, 19, 487, 2273, 1616, 851, 8, 697, 20, 1310, 926, 685, 46, 3, 1216, 492, 61, 1175, 98, 1176, 511, 454, 98, 91, 367, 0, 165, 108, 622, 9, 311, 0, 873, 239, 114, 647, 810, 311, 0, 1099, 110, 422, 23, 89, 76, 338, 312, 826, 28, 924, 210, 554, 355, 628, 581, 146, 108, 0] payload_rows_per_dst=[7324, 6458, 6074, 4398, 6356, 5290, 4306, 4146] dup_rows_per_dst=[6814, 3520, 3134, 1192, 2844, 1736, 926, 1018] combine_rows_per_src=[7324, 6458, 6074, 4398, 6356, 5290, 4306, 4146] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45178 payload_rows_remote=39238 dup_rows=20358 dup_rows_remote=17522 dedup_ratio=0.310638 router_tokens_per_expert=[2008, 1524, 1380, 1372, 20, 98, 128, 39, 1035, 1839, 1306, 389, 0, 598, 0, 2099, 806, 662, 30, 1, 1389, 552, 34, 909, 205, 575, 443, 702, 21, 156, 277, 1, 370, 158, 893, 20, 84, 0, 839, 245, 227, 651, 467, 879, 11, 540, 40, 569, 63, 68, 234, 653, 188, 516, 34, 1001, 256, 798, 541, 550, 520, 499, 256, 0] payload_rows_per_dst=[5226, 5708, 5144, 5940, 6136, 5602, 5414, 6008] dup_rows_per_dst=[1152, 2718, 2010, 2836, 2760, 2778, 2844, 3260] combine_rows_per_src=[5226, 5708, 5144, 5940, 6136, 5602, 5414, 6008] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64636 payload_rows_remote=56444 dup_rows=900 dup_rows_remote=896 dedup_ratio=0.013733 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4094, 0, 23, 0, 0, 0, 0, 0, 0, 3649, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 422, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 8188, 7298, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 6, 46, 0, 844, 4, 0] combine_rows_per_src=[8192, 8190, 8192, 8188, 7298, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7578188416 prefetch_bytes=770703360 reduce_bytes=1431306240 weight_movement_bytes=2202009600 egress_bytes=3790244480 ingress_bytes=5989953536 link_bytes=5989953536 total_nvlink_bytes=9780198016 weight_to_token_ratio=0.290572 prefetch_share_of_ingress=0.128666 dedup_ratio=0.075366 prefetch_slots=14 unused_prefetch_slots=146 slot_utilization=0.087500 reduce_slots=13 remote_demand=7 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048897 gemm_inner_pad_upper_bound=6604 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.041751438e-04 prefetch_bytes_per_gemm_flop=1.340379902e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65054 payload_rows_remote=57344 dup_rows=482 dup_rows_remote=482 dedup_ratio=0.007355 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 13, 0, 0, 0, 0, 0, 0, 3855, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7710, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 26, 0, 456, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 7710, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=403415040 dispatch_meta_bytes=229352 combine_payload_bytes=403415040 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=403644392 ingress_bytes=513515520 link_bytes=513515520 token_bytes=807059432 weight_movement_bytes=110100480 total_nvlink_bytes=917159912 weight_to_token_ratio=0.136422 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.930879481e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=406124544 dispatch_meta_bytes=229376 combine_payload_bytes=406124544 prefetch_bytes=0 reduce_bytes=0 egress_bytes=406353920 ingress_bytes=406124544 link_bytes=406353920 token_bytes=812478464 weight_movement_bytes=0 total_nvlink_bytes=812478464 weight_to_token_ratio=0.000000 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.067163006e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=370843648 dispatch_meta_bytes=229376 combine_payload_bytes=370843648 prefetch_bytes=0 reduce_bytes=0 egress_bytes=371073024 ingress_bytes=370843648 link_bytes=371073024 token_bytes=741916672 weight_movement_bytes=0 total_nvlink_bytes=741916672 weight_to_token_ratio=0.000000 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.453570198e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=404647936 dispatch_meta_bytes=225944 combine_payload_bytes=404647936 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=404873880 ingress_bytes=734949376 link_bytes=734949376 token_bytes=809521816 weight_movement_bytes=330301440 total_nvlink_bytes=1139823256 weight_to_token_ratio=0.408020 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=30080 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.278197843e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=404590592 dispatch_meta_bytes=229360 combine_payload_bytes=404590592 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=404819952 ingress_bytes=514691072 link_bytes=514691072 token_bytes=809410544 weight_movement_bytes=110100480 total_nvlink_bytes=919511024 weight_to_token_ratio=0.136026 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.951324263e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=411041792 dispatch_meta_bytes=231304 combine_payload_bytes=411041792 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=411273096 ingress_bytes=741343232 link_bytes=741343232 token_bytes=822314888 weight_movement_bytes=330301440 total_nvlink_bytes=1152616328 weight_to_token_ratio=0.401673 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=30080 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.289317810e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7506245312 prefetch_bytes=990904320 reduce_bytes=880803840 weight_movement_bytes=1871708160 egress_bytes=3754270400 ingress_bytes=5623683072 link_bytes=5623683072 total_nvlink_bytes=9377953472 weight_to_token_ratio=0.249353 prefetch_share_of_ingress=0.176202 dedup_ratio=0.083047 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=8 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.047426 gemm_inner_pad_upper_bound=7239 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.780509799e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7381940368 prefetch_bytes=1101004800 reduce_bytes=660602880 weight_movement_bytes=1761607680 egress_bytes=3692083344 ingress_bytes=5451464704 link_bytes=5451694080 total_nvlink_bytes=9143548048 weight_to_token_ratio=0.238637 prefetch_share_of_ingress=0.201965 dedup_ratio=0.097455 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=6 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.047794 gemm_inner_pad_upper_bound=7112 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.481392655e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6989902584 prefetch_bytes=1211105280 reduce_bytes=550502400 weight_movement_bytes=1761607680 egress_bytes=3496076024 ingress_bytes=5255434240 link_bytes=5255663616 total_nvlink_bytes=8751510264 weight_to_token_ratio=0.252022 prefetch_share_of_ingress=0.230448 dedup_ratio=0.136115 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=5 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.049265 gemm_inner_pad_upper_bound=6477 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.140463436e-05 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7578155744 prefetch_bytes=1101004800 reduce_bytes=1211105280 weight_movement_bytes=2312110080 egress_bytes=3790226144 ingress_bytes=6100039680 link_bytes=6100269056 total_nvlink_bytes=9890265824 weight_to_token_ratio=0.305102 prefetch_share_of_ingress=0.180491 dedup_ratio=0.075235 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=11 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.047794 gemm_inner_pad_upper_bound=7112 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.060937121e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7531118024 prefetch_bytes=550502400 reduce_bytes=1431306240 weight_movement_bytes=1981808640 egress_bytes=3766685128 ingress_bytes=5746241536 link_bytes=5746241536 total_nvlink_bytes=9512926664 weight_to_token_ratio=0.263149 prefetch_share_of_ingress=0.095802 dedup_ratio=0.081543 prefetch_slots=10 unused_prefetch_slots=150 slot_utilization=0.062500 reduce_slots=13 remote_demand=5 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.049265 gemm_inner_pad_upper_bound=6477 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.993659125e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7649372904 prefetch_bytes=1101004800 reduce_bytes=880803840 weight_movement_bytes=1981808640 egress_bytes=3825832680 ingress_bytes=5805348864 link_bytes=5805578240 total_nvlink_bytes=9631181544 weight_to_token_ratio=0.259081 prefetch_share_of_ingress=0.189654 dedup_ratio=0.070279 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=8 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.047794 gemm_inner_pad_upper_bound=7239 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.009685541e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none + [2026-08-26 15:59:20] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 360.6 | throughput per GPU (TFLOP/s/GPU): 342.5 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 9.388139E+00 | loss scale: 1.0 | grad norm: 4.978 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43870 payload_rows_remote=37138 dup_rows=21666 dup_rows_remote=16848 dedup_ratio=0.330597 router_tokens_per_expert=[1660, 564, 1630, 1409, 42, 237, 192, 41, 438, 1633, 1744, 547, 13, 395, 10, 1488, 1526, 605, 31, 4, 1017, 703, 36, 943, 64, 1201, 612, 432, 171, 243, 451, 1, 575, 330, 1096, 14, 372, 1, 415, 88, 341, 736, 689, 482, 1, 952, 68, 213, 97, 72, 279, 297, 330, 1006, 54, 836, 202, 413, 743, 796, 652, 400, 131, 4] payload_rows_per_dst=[6732, 7032, 6252, 4734, 4344, 5116, 4634, 5026] dup_rows_per_dst=[4818, 5504, 3478, 1616, 1438, 1848, 1308, 1656] combine_rows_per_src=[6732, 7032, 6252, 4734, 4344, 5116, 4634, 5026] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45066 payload_rows_remote=38250 dup_rows=20470 dup_rows_remote=15848 dedup_ratio=0.312347 router_tokens_per_expert=[1483, 631, 1269, 1568, 78, 315, 199, 34, 910, 2224, 1638, 980, 1, 703, 0, 1704, 792, 492, 6, 1, 1166, 527, 97, 744, 226, 987, 573, 329, 94, 85, 282, 1, 410, 151, 848, 7, 117, 8, 725, 236, 568, 451, 308, 350, 3, 672, 60, 647, 165, 212, 44, 1236, 350, 1032, 21, 955, 284, 531, 346, 486, 433, 722, 249, 2] payload_rows_per_dst=[4622, 4342, 6154, 5528, 6518, 5262, 6816, 5824] dup_rows_per_dst=[1028, 846, 3188, 2592, 2934, 2440, 4622, 2820] combine_rows_per_src=[4622, 4342, 6154, 5528, 6518, 5262, 6816, 5824] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45606 payload_rows_remote=39748 dup_rows=19930 dup_rows_remote=17876 dedup_ratio=0.304108 router_tokens_per_expert=[1914, 664, 1413, 1280, 11, 181, 355, 67, 931, 1995, 1337, 571, 4, 663, 0, 1261, 1622, 1303, 14, 0, 878, 618, 53, 1092, 110, 955, 622, 539, 82, 195, 513, 1, 177, 168, 884, 17, 142, 0, 482, 91, 502, 508, 210, 382, 1, 1009, 269, 396, 83, 355, 83, 1007, 435, 524, 25, 945, 234, 364, 303, 497, 434, 922, 75, 0] payload_rows_per_dst=[4206, 5196, 5464, 6414, 5858, 6206, 6278, 5984] dup_rows_per_dst=[910, 1816, 2452, 3448, 2054, 3592, 3158, 2500] combine_rows_per_src=[4206, 5196, 5464, 6414, 5858, 6206, 6278, 5984] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44858 payload_rows_remote=39722 dup_rows=20678 dup_rows_remote=18122 dedup_ratio=0.315521 router_tokens_per_expert=[1762, 797, 1881, 1162, 104, 78, 416, 125, 390, 1614, 1168, 507, 0, 1044, 2, 1724, 841, 783, 9, 66, 951, 648, 177, 660, 307, 1107, 538, 429, 39, 197, 469, 1, 580, 398, 755, 11, 123, 0, 390, 180, 111, 400, 283, 646, 1, 1014, 50, 500, 29, 61, 198, 655, 521, 832, 33, 1071, 305, 209, 603, 1053, 813, 891, 56, 0] payload_rows_per_dst=[4484, 4736, 5060, 6134, 5820, 5136, 6506, 6982] dup_rows_per_dst=[880, 1220, 1794, 3564, 2282, 2556, 3742, 4640] combine_rows_per_src=[4484, 4736, 5060, 6134, 5820, 5136, 6506, 6982] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44742 payload_rows_remote=39334 dup_rows=20794 dup_rows_remote=18716 dedup_ratio=0.317291 router_tokens_per_expert=[2182, 764, 1631, 1601, 21, 166, 202, 43, 374, 2101, 1523, 517, 1, 755, 3, 1555, 1114, 858, 46, 2, 636, 807, 16, 1378, 90, 1048, 536, 567, 77, 95, 196, 1, 553, 96, 657, 7, 242, 0, 603, 119, 147, 549, 577, 583, 3, 872, 126, 367, 25, 106, 153, 836, 243, 754, 70, 1070, 365, 452, 464, 386, 669, 600, 168, 0] payload_rows_per_dst=[6902, 6312, 5408, 4778, 6150, 5714, 4748, 4730] dup_rows_per_dst=[5184, 3144, 2078, 1576, 2606, 2962, 1766, 1478] combine_rows_per_src=[6902, 6312, 5408, 4778, 6150, 5714, 4748, 4730] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45164 payload_rows_remote=39246 dup_rows=20372 dup_rows_remote=17448 dedup_ratio=0.310852 router_tokens_per_expert=[1963, 1509, 1450, 1224, 10, 88, 138, 35, 1121, 1705, 1287, 425, 0, 672, 0, 2097, 797, 606, 26, 5, 1433, 581, 33, 905, 202, 635, 455, 705, 29, 149, 282, 1, 386, 199, 923, 18, 115, 0, 844, 275, 234, 467, 441, 904, 13, 521, 57, 565, 67, 74, 260, 605, 207, 525, 32, 1031, 264, 662, 557, 601, 556, 534, 263, 0] payload_rows_per_dst=[5968, 5838, 5188, 5918, 6090, 5380, 5566, 5216] dup_rows_per_dst=[2324, 2932, 1990, 2924, 2840, 2618, 2470, 2274] combine_rows_per_src=[5968, 5838, 5188, 5918, 6090, 5380, 5566, 5216] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45170 payload_rows_remote=38070 dup_rows=20366 dup_rows_remote=16156 dedup_ratio=0.310760 router_tokens_per_expert=[2404, 926, 2233, 1457, 182, 289, 166, 34, 222, 1876, 1492, 760, 1, 900, 9, 1355, 1248, 623, 41, 2, 745, 518, 95, 1031, 142, 1131, 446, 520, 65, 224, 290, 0, 403, 74, 415, 21, 113, 0, 561, 244, 315, 576, 375, 473, 0, 871, 147, 401, 34, 137, 476, 495, 369, 703, 33, 683, 372, 378, 507, 449, 455, 1149, 112, 0] payload_rows_per_dst=[3212, 4890, 6158, 6588, 5648, 5734, 5840, 7100] dup_rows_per_dst=[500, 1290, 2936, 3856, 1766, 3078, 2730, 4210] combine_rows_per_src=[3212, 4890, 6158, 6588, 5648, 5734, 5840, 7100] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44706 payload_rows_remote=38938 dup_rows=20830 dup_rows_remote=17234 dedup_ratio=0.317841 router_tokens_per_expert=[2425, 935, 2136, 1595, 234, 312, 141, 40, 202, 1980, 1508, 690, 1, 850, 9, 1312, 1278, 681, 50, 2, 714, 491, 97, 1028, 146, 1065, 428, 528, 52, 244, 282, 0, 389, 67, 400, 19, 77, 0, 540, 228, 313, 728, 396, 468, 0, 914, 110, 398, 32, 136, 454, 529, 357, 685, 32, 670, 340, 489, 469, 410, 443, 1119, 100, 0] payload_rows_per_dst=[3662, 5120, 6222, 6562, 5678, 5914, 5780, 5768] dup_rows_per_dst=[842, 1400, 2996, 3778, 1722, 3296, 3200, 3596] combine_rows_per_src=[3662, 5120, 6222, 6562, 5678, 5914, 5780, 5768] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44322 payload_rows_remote=37906 dup_rows=21214 dup_rows_remote=16340 dedup_ratio=0.323700 router_tokens_per_expert=[1504, 622, 1169, 1719, 89, 329, 199, 40, 824, 2320, 1660, 920, 1, 661, 0, 1700, 823, 582, 7, 1, 1135, 504, 103, 739, 229, 917, 567, 325, 62, 85, 276, 0, 403, 139, 841, 6, 83, 8, 723, 215, 557, 583, 326, 350, 4, 705, 48, 641, 157, 186, 46, 1266, 335, 985, 22, 929, 260, 691, 343, 462, 408, 691, 241, 2] payload_rows_per_dst=[4002, 5828, 4662, 5466, 6564, 5428, 6416, 5956] dup_rows_per_dst=[894, 2304, 1480, 2464, 2912, 2646, 4874, 3640] combine_rows_per_src=[4002, 5828, 4662, 5466, 6564, 5428, 6416, 5956] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279308288 dispatch_meta_bytes=231048 combine_payload_bytes=279308288 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279539336 ingress_bytes=389408768 link_bytes=389408768 token_bytes=558847624 weight_movement_bytes=110100480 total_nvlink_bytes=668948104 weight_to_token_ratio=0.197013 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=14848 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.772458943e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278634496 dispatch_meta_bytes=225024 combine_payload_bytes=278634496 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278859520 ingress_bytes=498835456 link_bytes=498835456 token_bytes=557494016 weight_movement_bytes=220200960 total_nvlink_bytes=777694976 weight_to_token_ratio=0.394984 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=8064 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.675569023e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=282605568 dispatch_meta_bytes=229752 combine_payload_bytes=282605568 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=282835320 ingress_bytes=392706048 link_bytes=392706048 token_bytes=565440888 weight_movement_bytes=110100480 total_nvlink_bytes=675541368 weight_to_token_ratio=0.194716 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=7296 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.829804065e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=271624192 dispatch_meta_bytes=222232 combine_payload_bytes=271624192 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=271846424 ingress_bytes=601925632 link_bytes=601925632 token_bytes=543470616 weight_movement_bytes=330301440 total_nvlink_bytes=873772056 weight_to_token_ratio=0.607763 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=11264 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.046847674e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=281257984 dispatch_meta_bytes=227040 combine_payload_bytes=281257984 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281485024 ingress_bytes=391358464 link_bytes=391358464 token_bytes=562743008 weight_movement_bytes=110100480 total_nvlink_bytes=672843488 weight_to_token_ratio=0.195650 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=11008 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.806367363e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279107584 dispatch_meta_bytes=224688 combine_payload_bytes=279107584 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279332272 ingress_bytes=389208064 link_bytes=389208064 token_bytes=558439856 weight_movement_bytes=110100480 total_nvlink_bytes=668540336 weight_to_token_ratio=0.197157 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=7296 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.768968370e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=271710208 dispatch_meta_bytes=216984 combine_payload_bytes=271710208 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=271927192 ingress_bytes=381810688 link_bytes=381810688 token_bytes=543637400 weight_movement_bytes=110100480 total_nvlink_bytes=653737880 weight_to_token_ratio=0.202526 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=7680 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.640315835e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44092 payload_rows_remote=36964 dup_rows=21444 dup_rows_remote=15438 dedup_ratio=0.327209 router_tokens_per_expert=[1678, 564, 1574, 1506, 64, 268, 184, 43, 403, 1751, 1763, 512, 13, 368, 9, 1505, 1540, 666, 35, 1, 985, 672, 38, 961, 60, 1112, 591, 440, 136, 267, 447, 1, 562, 264, 1073, 14, 318, 1, 398, 80, 332, 887, 717, 470, 1, 981, 52, 214, 87, 70, 267, 317, 333, 979, 51, 819, 189, 508, 709, 763, 639, 390, 122, 4] payload_rows_per_dst=[7128, 6540, 6278, 4602, 4592, 5350, 4576, 5026] dup_rows_per_dst=[6006, 4098, 3518, 1506, 1466, 1958, 1270, 1622] combine_rows_per_src=[7128, 6540, 6278, 4602, 4592, 5350, 4576, 5026] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55210 payload_rows_remote=48566 dup_rows=10326 dup_rows_remote=9000 dedup_ratio=0.157562 router_tokens_per_expert=[0, 0, 0, 4096, 567, 0, 2, 4095, 2397, 1, 3186, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 111, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3152, 0, 0, 164, 0, 3905, 0, 0, 0, 0, 1, 940, 0, 8, 0, 0, 0, 1, 0, 0, 0, 1749, 0, 0, 204] payload_rows_per_dst=[1138, 6830, 8186, 8190, 6644, 7840, 8192, 8190] dup_rows_per_dst=[0, 2672, 224, 0, 1326, 298, 1898, 3908] combine_rows_per_src=[1138, 6830, 8186, 8190, 6644, 7840, 8192, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53120 payload_rows_remote=44934 dup_rows=12416 dup_rows_remote=12186 dedup_ratio=0.189453 router_tokens_per_expert=[0, 1, 0, 4095, 998, 0, 0, 4094, 2668, 0, 2549, 0, 1, 0, 1, 0, 0, 0, 0, 4093, 115, 0, 0, 0, 0, 1, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2857, 1, 0, 356, 0, 3894, 0, 0, 0, 0, 0, 1723, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1104, 0, 2, 119] payload_rows_per_dst=[8192, 7070, 8186, 8188, 5716, 7818, 5500, 2450] dup_rows_per_dst=[6854, 3368, 230, 0, 0, 684, 1276, 4] combine_rows_per_src=[8192, 7070, 8186, 8188, 5716, 7818, 5500, 2450] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54534 payload_rows_remote=46642 dup_rows=11002 dup_rows_remote=10558 dedup_ratio=0.167877 router_tokens_per_expert=[0, 0, 0, 4094, 492, 0, 0, 4085, 2653, 0, 3114, 0, 0, 0, 0, 0, 0, 1, 0, 4027, 92, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2606, 0, 0, 232, 0, 3936, 0, 0, 0, 0, 0, 1261, 0, 12, 0, 0, 0, 0, 0, 0, 0, 1881, 0, 0, 182] payload_rows_per_dst=[984, 5306, 8060, 8192, 7740, 7892, 8190, 8170] dup_rows_per_dst=[0, 0, 180, 8, 3700, 444, 2544, 4126] combine_rows_per_src=[984, 5306, 8060, 8192, 7740, 7892, 8190, 8170] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=284913664 dispatch_meta_bytes=230496 combine_payload_bytes=284913664 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=285144160 ingress_bytes=395014144 link_bytes=395014144 token_bytes=570057824 weight_movement_bytes=110100480 total_nvlink_bytes=680158304 weight_to_token_ratio=0.193139 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=13696 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.869945651e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=266205184 dispatch_meta_bytes=215944 combine_payload_bytes=266205184 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=266421128 ingress_bytes=596506624 link_bytes=596506624 token_bytes=532626312 weight_movement_bytes=330301440 total_nvlink_bytes=862927752 weight_to_token_ratio=0.620137 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=10368 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.037423128e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=274176000 dispatch_meta_bytes=216392 combine_payload_bytes=274176000 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=274392392 ingress_bytes=384276480 link_bytes=384276480 token_bytes=548568392 weight_movement_bytes=110100480 total_nvlink_bytes=658668872 weight_to_token_ratio=0.200705 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=7552 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.683200013e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=284727296 dispatch_meta_bytes=231376 combine_payload_bytes=284727296 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=284958672 ingress_bytes=394827776 link_bytes=394827776 token_bytes=569685968 weight_movement_bytes=110100480 total_nvlink_bytes=679786448 weight_to_token_ratio=0.193265 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=7040 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.866704405e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44016 payload_rows_remote=37528 dup_rows=21520 dup_rows_remote=17570 dedup_ratio=0.328369 router_tokens_per_expert=[2085, 885, 1575, 1778, 19, 267, 259, 15, 556, 2191, 1598, 906, 7, 737, 20, 1317, 905, 616, 40, 3, 1270, 514, 62, 1177, 100, 1270, 531, 456, 115, 89, 372, 0, 174, 125, 661, 11, 379, 0, 878, 253, 114, 529, 746, 316, 0, 1062, 138, 423, 30, 94, 78, 311, 326, 862, 27, 948, 218, 425, 367, 662, 599, 156, 121, 0] payload_rows_per_dst=[7250, 6488, 5424, 4544, 6266, 5556, 4408, 4080] dup_rows_per_dst=[6516, 3950, 2512, 1322, 2922, 2338, 944, 1016] combine_rows_per_src=[7250, 6488, 5424, 4544, 6266, 5556, 4408, 4080] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=281946112 dispatch_meta_bytes=232200 combine_payload_bytes=281946112 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=282178312 ingress_bytes=502147072 link_bytes=502147072 token_bytes=564124424 weight_movement_bytes=220200960 total_nvlink_bytes=784325384 weight_to_token_ratio=0.390341 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=8192 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.733163472e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=281315328 dispatch_meta_bytes=226776 combine_payload_bytes=281315328 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281542104 ingress_bytes=391415808 link_bytes=391415808 token_bytes=562857432 weight_movement_bytes=110100480 total_nvlink_bytes=672957912 weight_to_token_ratio=0.195610 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=10112 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.807364670e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=272885760 dispatch_meta_bytes=216904 combine_payload_bytes=272885760 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=273102664 ingress_bytes=382986240 link_bytes=382986240 token_bytes=545988424 weight_movement_bytes=110100480 total_nvlink_bytes=656088904 weight_to_token_ratio=0.201654 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=7168 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.660760618e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=269000704 dispatch_meta_bytes=220392 combine_payload_bytes=269000704 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=269221096 ingress_bytes=709402624 link_bytes=709402624 token_bytes=538221800 weight_movement_bytes=440401920 total_nvlink_bytes=978623720 weight_to_token_ratio=0.818254 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=8576 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.233767840e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46758 payload_rows_remote=38566 dup_rows=18778 dup_rows_remote=3354 dedup_ratio=0.286530 router_tokens_per_expert=[0, 0, 0, 4088, 667, 1, 1, 4094, 3164, 0, 2957, 0, 1, 0, 0, 0, 0, 1, 0, 4027, 31, 0, 0, 0, 0, 0, 23, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 1, 1, 0, 3174, 3, 0, 599, 0, 3889, 2, 4, 1, 0, 0, 1007, 0, 0, 0, 0, 0, 3, 1, 10, 0, 868, 0, 0, 57] payload_rows_per_dst=[8192, 7120, 8074, 8022, 6352, 5134, 2014, 1850] dup_rows_per_dst=[15424, 2728, 208, 46, 0, 344, 0, 28] combine_rows_per_src=[8192, 7120, 8074, 8022, 6352, 5134, 2014, 1850] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56376 payload_rows_remote=49746 dup_rows=9160 dup_rows_remote=9074 dedup_ratio=0.139771 router_tokens_per_expert=[0, 0, 0, 4096, 812, 0, 3, 4096, 2775, 1, 3058, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 18, 1, 0, 0, 0, 0, 3, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3308, 0, 0, 180, 0, 3945, 0, 0, 0, 0, 1, 851, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1251, 0, 0, 180] payload_rows_per_dst=[1630, 7466, 8180, 8190, 6630, 7896, 8192, 8192] dup_rows_per_dst=[0, 4102, 38, 6, 86, 354, 1704, 2870] combine_rows_per_src=[1630, 7466, 8180, 8190, 6630, 7896, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56164 payload_rows_remote=48154 dup_rows=9372 dup_rows_remote=8788 dedup_ratio=0.143005 router_tokens_per_expert=[0, 0, 0, 4092, 851, 0, 0, 4096, 3106, 0, 2982, 0, 0, 0, 0, 0, 0, 0, 0, 3971, 21, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2771, 0, 0, 299, 0, 3998, 0, 0, 0, 0, 0, 998, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1354, 0, 0, 124] payload_rows_per_dst=[1702, 5964, 7948, 8192, 7964, 8010, 8192, 8192] dup_rows_per_dst=[0, 0, 36, 8, 3790, 584, 1996, 2958] combine_rows_per_src=[1702, 5964, 7948, 8192, 7964, 8010, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56192 payload_rows_remote=48000 dup_rows=9344 dup_rows_remote=7076 dedup_ratio=0.142578 router_tokens_per_expert=[0, 0, 0, 4096, 1484, 0, 2, 4096, 2466, 1, 2506, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 96, 1, 0, 0, 0, 0, 64, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2688, 0, 0, 449, 0, 3994, 0, 0, 0, 0, 0, 1134, 0, 0, 0, 0, 0, 37, 0, 0, 0, 1234, 0, 0, 231] payload_rows_per_dst=[2972, 5014, 8188, 8190, 7430, 8014, 8192, 8192] dup_rows_per_dst=[0, 0, 194, 128, 2878, 872, 2268, 3004] combine_rows_per_src=[2972, 5014, 8188, 8190, 7430, 8014, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51438 payload_rows_remote=44336 dup_rows=14098 dup_rows_remote=10756 dedup_ratio=0.215118 router_tokens_per_expert=[0, 0, 0, 4095, 803, 0, 0, 4084, 2538, 0, 2683, 0, 0, 0, 0, 1, 0, 6, 1, 4093, 208, 0, 0, 0, 0, 0, 26, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2671, 0, 0, 240, 0, 3930, 0, 0, 0, 0, 1, 1222, 0, 3, 0, 0, 0, 0, 0, 1, 0, 2005, 0, 0, 61] payload_rows_per_dst=[8190, 7102, 8186, 8192, 5342, 7878, 2452, 4096] dup_rows_per_dst=[9774, 3342, 430, 52, 0, 462, 0, 38] combine_rows_per_src=[8190, 7102, 8186, 8192, 5342, 7878, 2452, 4096] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56038 payload_rows_remote=47850 dup_rows=9498 dup_rows_remote=9498 dedup_ratio=0.144928 router_tokens_per_expert=[0, 0, 0, 4096, 763, 0, 6, 4096, 2336, 0, 2517, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 58, 1, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3148, 0, 0, 298, 0, 3902, 0, 0, 0, 0, 0, 1859, 0, 6, 0, 0, 0, 0, 0, 1, 0, 1287, 0, 1, 234] payload_rows_per_dst=[4344, 7032, 8140, 8188, 6296, 7822, 8192, 6024] dup_rows_per_dst=[654, 2674, 108, 0, 0, 578, 3730, 1754] combine_rows_per_src=[4344, 7032, 8140, 8188, 6296, 7822, 8192, 6024] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52168 payload_rows_remote=43976 dup_rows=13368 dup_rows_remote=9472 dedup_ratio=0.203979 router_tokens_per_expert=[0, 0, 2, 4096, 531, 0, 0, 4096, 2554, 2, 3278, 0, 1, 0, 0, 1, 0, 0, 0, 4085, 58, 0, 0, 0, 0, 0, 22, 0, 1, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2838, 0, 0, 209, 1, 3956, 0, 0, 0, 0, 0, 986, 0, 6, 0, 0, 0, 1, 0, 0, 0, 1917, 0, 0, 30] payload_rows_per_dst=[1062, 4134, 6494, 8190, 7878, 8026, 8192, 8192] dup_rows_per_dst=[4, 0, 88, 50, 4354, 2992, 1984, 3896] combine_rows_per_src=[1062, 4134, 6494, 8190, 7878, 8026, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54216 payload_rows_remote=46024 dup_rows=11320 dup_rows_remote=8584 dedup_ratio=0.172729 router_tokens_per_expert=[0, 0, 0, 4095, 1083, 0, 2, 4091, 2022, 3, 2652, 0, 0, 0, 0, 0, 0, 3, 0, 4092, 208, 1, 0, 0, 0, 0, 54, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2635, 0, 0, 452, 0, 3935, 0, 0, 0, 0, 0, 1367, 0, 2, 0, 0, 0, 8, 0, 0, 0, 1752, 0, 0, 216] payload_rows_per_dst=[2170, 4046, 8188, 8190, 7336, 7910, 8192, 8184] dup_rows_per_dst=[0, 4, 420, 108, 3238, 864, 2736, 3950] combine_rows_per_src=[2170, 4046, 8188, 8190, 7336, 7910, 8192, 8184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=322086912 dispatch_meta_bytes=228480 combine_payload_bytes=322086912 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=322315392 ingress_bytes=542287872 link_bytes=542287872 token_bytes=644402304 weight_movement_bytes=220200960 total_nvlink_bytes=864603264 weight_to_token_ratio=0.341713 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=31360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.431278004e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=348121088 dispatch_meta_bytes=230264 combine_payload_bytes=348121088 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=348351352 ingress_bytes=458221568 link_bytes=458221568 token_bytes=696472440 weight_movement_bytes=110100480 total_nvlink_bytes=806572920 weight_to_token_ratio=0.158083 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=22912 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.969226712e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=334329856 dispatch_meta_bytes=228800 combine_payload_bytes=334329856 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=334558656 ingress_bytes=554530816 link_bytes=554530816 token_bytes=668888512 weight_movement_bytes=220200960 total_nvlink_bytes=889089472 weight_to_token_ratio=0.329204 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=28928 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.644202937e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56618 payload_rows_remote=48430 dup_rows=8918 dup_rows_remote=8916 dedup_ratio=0.136078 router_tokens_per_expert=[0, 0, 0, 4096, 1099, 0, 9, 4096, 2705, 0, 2440, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 16, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3348, 0, 0, 320, 0, 3932, 0, 0, 0, 0, 0, 1397, 0, 4, 0, 0, 0, 1, 0, 2, 0, 899, 0, 1, 266] payload_rows_per_dst=[2214, 7164, 8096, 8188, 6696, 7876, 8192, 8192] dup_rows_per_dst=[2, 3126, 20, 2, 0, 628, 2802, 2338] combine_rows_per_src=[2214, 7164, 8096, 8188, 6696, 7876, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55526 payload_rows_remote=47400 dup_rows=10010 dup_rows_remote=9964 dedup_ratio=0.152740 router_tokens_per_expert=[0, 0, 0, 4095, 1338, 0, 0, 4094, 3084, 0, 2398, 0, 1, 0, 1, 0, 0, 0, 0, 4063, 23, 0, 0, 0, 0, 1, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3053, 1, 0, 427, 0, 3949, 0, 0, 0, 0, 0, 1274, 0, 0, 0, 0, 0, 2, 0, 1, 0, 761, 0, 1, 104] payload_rows_per_dst=[5490, 7152, 8126, 8190, 6108, 7916, 7752, 4792] dup_rows_per_dst=[2496, 3538, 46, 4, 0, 838, 2166, 922] combine_rows_per_src=[5490, 7152, 8126, 8190, 6108, 7916, 7752, 4792] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=264957952 dispatch_meta_bytes=209608 combine_payload_bytes=264957952 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=265167560 ingress_bytes=705359872 link_bytes=705359872 token_bytes=530125512 weight_movement_bytes=440401920 total_nvlink_bytes=970527432 weight_to_token_ratio=0.830750 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=11008 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.226736829e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=317800448 dispatch_meta_bytes=220368 combine_payload_bytes=317800448 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=318020816 ingress_bytes=648101888 link_bytes=648101888 token_bytes=635821264 weight_movement_bytes=330301440 total_nvlink_bytes=966122704 weight_to_token_ratio=0.519488 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=19584 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.127155778e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=342988800 dispatch_meta_bytes=229392 combine_payload_bytes=342988800 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=343218192 ingress_bytes=453089280 link_bytes=453089280 token_bytes=686206992 weight_movement_bytes=110100480 total_nvlink_bytes=796307472 weight_to_token_ratio=0.160448 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.879967783e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=315219968 dispatch_meta_bytes=213792 combine_payload_bytes=315219968 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=315433760 ingress_bytes=425320448 link_bytes=425320448 token_bytes=630653728 weight_movement_bytes=110100480 total_nvlink_bytes=740754208 weight_to_token_ratio=0.174582 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=18816 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.397022123e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=329900032 dispatch_meta_bytes=218432 combine_payload_bytes=329900032 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=330118464 ingress_bytes=440000512 link_bytes=440000512 token_bytes=660018496 weight_movement_bytes=110100480 total_nvlink_bytes=770118976 weight_to_token_ratio=0.166814 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=22144 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.652332580e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64216 payload_rows_remote=56024 dup_rows=1320 dup_rows_remote=1320 dedup_ratio=0.020142 router_tokens_per_expert=[0, 0, 0, 4096, 3, 647, 4096, 0, 0, 4095, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 4093, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 3440] payload_rows_per_dst=[8192, 8190, 8192, 8190, 8188, 8192, 8192, 6880] dup_rows_per_dst=[1300, 0, 0, 16, 2, 2, 0, 0] combine_rows_per_src=[8192, 8190, 8192, 8190, 8188, 8192, 8192, 6880] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64824 payload_rows_remote=56632 dup_rows=712 dup_rows_remote=712 dedup_ratio=0.010864 router_tokens_per_expert=[0, 0, 0, 4096, 0, 275, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3740] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 7480] dup_rows_per_dst=[550, 0, 0, 162, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 7480] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64432 payload_rows_remote=56240 dup_rows=1104 dup_rows_remote=1104 dedup_ratio=0.016846 router_tokens_per_expert=[0, 0, 0, 4096, 0, 543, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3547] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8186, 8192, 8192, 7094] dup_rows_per_dst=[1086, 0, 4, 14, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8186, 8192, 8192, 7094] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56328 payload_rows_remote=48136 dup_rows=9208 dup_rows_remote=6404 dedup_ratio=0.140503 router_tokens_per_expert=[0, 0, 2, 4096, 776, 0, 1, 4096, 2928, 1, 3165, 0, 1, 0, 0, 1, 0, 0, 0, 4057, 15, 0, 0, 0, 0, 0, 29, 0, 1, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2960, 0, 0, 233, 1, 3983, 0, 0, 0, 0, 0, 921, 0, 2, 0, 0, 0, 6, 0, 0, 0, 1353, 0, 1, 42] payload_rows_per_dst=[1554, 6336, 8114, 8068, 7898, 7974, 8192, 8192] dup_rows_per_dst=[4, 0, 30, 64, 3878, 582, 1846, 2804] combine_rows_per_src=[1554, 6336, 8114, 8068, 7898, 7974, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=276441088 dispatch_meta_bytes=167680 combine_payload_bytes=276441088 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=276608768 ingress_bytes=606742528 link_bytes=606742528 token_bytes=553049856 weight_movement_bytes=330301440 total_nvlink_bytes=883351296 weight_to_token_ratio=0.597236 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=10368 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.055225048e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=356579328 dispatch_meta_bytes=235280 combine_payload_bytes=356579328 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=356814608 ingress_bytes=466679808 link_bytes=466679808 token_bytes=713393936 weight_movement_bytes=110100480 total_nvlink_bytes=823494416 weight_to_token_ratio=0.154333 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=24320 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.116329418e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=345167872 dispatch_meta_bytes=227768 combine_payload_bytes=345167872 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=345395640 ingress_bytes=565368832 link_bytes=565368832 token_bytes=690563512 weight_movement_bytes=220200960 total_nvlink_bytes=910764472 weight_to_token_ratio=0.318871 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=29952 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.832693860e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=344064000 dispatch_meta_bytes=220304 combine_payload_bytes=344064000 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=344284304 ingress_bytes=454164480 link_bytes=454164480 token_bytes=688348304 weight_movement_bytes=110100480 total_nvlink_bytes=798448784 weight_to_token_ratio=0.159949 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=24192 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.898667279e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49126 payload_rows_remote=43302 dup_rows=16410 dup_rows_remote=16408 dedup_ratio=0.250397 router_tokens_per_expert=[0, 0, 0, 4095, 1057, 0, 0, 4092, 2911, 0, 2684, 0, 0, 0, 0, 2, 0, 0, 1, 4066, 52, 0, 0, 0, 0, 0, 23, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2913, 0, 0, 294, 0, 3963, 0, 0, 0, 0, 1, 1013, 0, 0, 0, 0, 0, 5, 0, 1, 0, 1426, 0, 0, 73] payload_rows_per_dst=[8192, 5824, 8132, 8192, 5826, 7944, 2028, 2988] dup_rows_per_dst=[15664, 2, 106, 46, 0, 570, 0, 22] combine_rows_per_src=[8192, 5824, 8132, 8192, 5826, 7944, 2028, 2988] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=347146240 dispatch_meta_bytes=229384 combine_payload_bytes=347146240 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=347375624 ingress_bytes=567347200 link_bytes=567347200 token_bytes=694521864 weight_movement_bytes=220200960 total_nvlink_bytes=914722824 weight_to_token_ratio=0.317054 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.867100934e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=339763200 dispatch_meta_bytes=229456 combine_payload_bytes=339763200 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=339992656 ingress_bytes=449863680 link_bytes=449863680 token_bytes=679755856 weight_movement_bytes=110100480 total_nvlink_bytes=789856336 weight_to_token_ratio=0.161971 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.823869294e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=345038848 dispatch_meta_bytes=218160 combine_payload_bytes=345038848 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=345257008 ingress_bytes=455139328 link_bytes=455139328 token_bytes=690295856 weight_movement_bytes=110100480 total_nvlink_bytes=800396336 weight_to_token_ratio=0.159498 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=22528 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.915621489e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=310388736 dispatch_meta_bytes=238840 combine_payload_bytes=310388736 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=310627576 ingress_bytes=640690176 link_bytes=640690176 token_bytes=621016312 weight_movement_bytes=330301440 total_nvlink_bytes=951317752 weight_to_token_ratio=0.531872 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=16512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.114265591e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61490 payload_rows_remote=53448 dup_rows=4046 dup_rows_remote=222 dedup_ratio=0.061737 router_tokens_per_expert=[0, 0, 0, 4095, 12, 3963, 4096, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 0, 0, 0, 29, 4095, 0, 2, 4, 0, 1, 1, 0, 0, 0, 0, 4092, 2, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 1, 0, 4065] payload_rows_per_dst=[8042, 8184, 5408, 7348, 8034, 8188, 8190, 8096] dup_rows_per_dst=[3824, 20, 140, 46, 2, 12, 0, 2] combine_rows_per_src=[8042, 8184, 5408, 7348, 8034, 8188, 8190, 8096] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64766 payload_rows_remote=56574 dup_rows=770 dup_rows_remote=768 dedup_ratio=0.011749 router_tokens_per_expert=[0, 0, 0, 4096, 0, 3711, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 194, 0, 0, 0, 0, 126, 4096, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096] payload_rows_per_dst=[7422, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 388, 256, 2, 124, 0, 0] combine_rows_per_src=[7422, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64930 payload_rows_remote=56738 dup_rows=606 dup_rows_remote=606 dedup_ratio=0.009247 router_tokens_per_expert=[0, 0, 0, 4096, 1, 3799, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 289, 0, 0, 0, 0, 13, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4090] payload_rows_per_dst=[7598, 8192, 8192, 8192, 8192, 8192, 8192, 8180] dup_rows_per_dst=[2, 0, 578, 26, 0, 0, 0, 0] combine_rows_per_src=[7598, 8192, 8192, 8192, 8192, 8192, 8192, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64730 payload_rows_remote=56538 dup_rows=806 dup_rows_remote=806 dedup_ratio=0.012299 router_tokens_per_expert=[0, 0, 0, 4096, 0, 3693, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 267, 0, 0, 0, 0, 54, 4096, 0, 2, 0, 0, 0, 55, 0, 0, 0, 0, 4096, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096] payload_rows_per_dst=[7386, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 534, 112, 110, 50, 0, 0] combine_rows_per_src=[7386, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=401580032 dispatch_meta_bytes=229376 combine_payload_bytes=401580032 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=401809408 ingress_bytes=511680512 link_bytes=511680512 token_bytes=803389440 weight_movement_bytes=110100480 total_nvlink_bytes=913489920 weight_to_token_ratio=0.137045 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.898965674e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64146 payload_rows_remote=55954 dup_rows=1390 dup_rows_remote=1388 dedup_ratio=0.021210 router_tokens_per_expert=[0, 0, 0, 4096, 0, 381, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3706] payload_rows_per_dst=[7548, 8192, 8192, 8192, 8180, 8192, 8192, 7458] dup_rows_per_dst=[716, 0, 0, 30, 0, 0, 2, 642] combine_rows_per_src=[7548, 8192, 8192, 8192, 8180, 8192, 8192, 7458] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64356 payload_rows_remote=56164 dup_rows=1180 dup_rows_remote=1178 dedup_ratio=0.018005 router_tokens_per_expert=[0, 0, 0, 4096, 0, 588, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3509] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8186, 8192, 8192, 7018] dup_rows_per_dst=[1176, 0, 2, 2, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8186, 8192, 8192, 7018] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63926 payload_rows_remote=55742 dup_rows=1610 dup_rows_remote=1610 dedup_ratio=0.024567 router_tokens_per_expert=[0, 0, 0, 4096, 0, 732, 4095, 0, 0, 4092, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 4096, 0, 8, 1, 0, 1, 0, 0, 0, 0, 0, 4082, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 3310] payload_rows_per_dst=[8190, 8184, 8192, 8192, 8164, 8192, 8192, 6620] dup_rows_per_dst=[1464, 0, 0, 142, 4, 0, 0, 0] combine_rows_per_src=[8190, 8184, 8192, 8192, 8164, 8192, 8192, 6620] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=403128320 dispatch_meta_bytes=229376 combine_payload_bytes=403128320 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=403357696 ingress_bytes=513228800 link_bytes=513228800 token_bytes=806486016 weight_movement_bytes=110100480 total_nvlink_bytes=916586496 weight_to_token_ratio=0.136519 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.925892948e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58080 payload_rows_remote=49890 dup_rows=7456 dup_rows_remote=196 dedup_ratio=0.113770 router_tokens_per_expert=[0, 0, 0, 4096, 2, 435, 4095, 0, 0, 4094, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 32, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 3627] payload_rows_per_dst=[874, 8190, 8190, 8068, 8188, 8188, 8192, 8190] dup_rows_per_dst=[0, 0, 2, 68, 124, 2, 0, 7260] combine_rows_per_src=[874, 8190, 8190, 8068, 8188, 8188, 8192, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46514 payload_rows_remote=38324 dup_rows=19022 dup_rows_remote=5042 dedup_ratio=0.290253 router_tokens_per_expert=[0, 0, 0, 4088, 541, 1, 1, 4085, 2867, 0, 2993, 0, 1, 0, 0, 0, 0, 7, 0, 4053, 148, 0, 0, 0, 0, 0, 18, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 1, 1, 0, 2981, 3, 0, 563, 0, 3810, 2, 4, 1, 0, 0, 1229, 0, 0, 0, 0, 0, 0, 2, 9, 0, 1219, 0, 0, 47] payload_rows_per_dst=[8190, 7498, 8126, 7954, 5966, 3800, 2458, 2522] dup_rows_per_dst=[13980, 4458, 290, 34, 0, 228, 0, 32] combine_rows_per_src=[8190, 7498, 8126, 7954, 5966, 3800, 2458, 2522] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=405938176 dispatch_meta_bytes=229376 combine_payload_bytes=405938176 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=406167552 ingress_bytes=516038656 link_bytes=516038656 token_bytes=812105728 weight_movement_bytes=110100480 total_nvlink_bytes=922206208 weight_to_token_ratio=0.135574 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.974760966e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=401078272 dispatch_meta_bytes=229368 combine_payload_bytes=401078272 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=401307640 ingress_bytes=621279232 link_bytes=621279232 token_bytes=802385912 weight_movement_bytes=220200960 total_nvlink_bytes=1022586872 weight_to_token_ratio=0.274433 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.080506767e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=402583552 dispatch_meta_bytes=229368 combine_payload_bytes=402583552 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=402812920 ingress_bytes=732884992 link_bytes=732884992 token_bytes=805396472 weight_movement_bytes=330301440 total_nvlink_bytes=1135697912 weight_to_token_ratio=0.410110 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.274607540e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=399558656 dispatch_meta_bytes=229408 combine_payload_bytes=399558656 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=399788064 ingress_bytes=619759616 link_bytes=619759616 token_bytes=799346720 weight_movement_bytes=220200960 total_nvlink_bytes=1019547680 weight_to_token_ratio=0.275476 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.077863905e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=357611520 dispatch_meta_bytes=200344 combine_payload_bytes=357611520 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=357811864 ingress_bytes=577812480 link_bytes=577812480 token_bytes=715423384 weight_movement_bytes=220200960 total_nvlink_bytes=935624344 weight_to_token_ratio=0.307791 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=28416 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.004910937e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=274706432 dispatch_meta_bytes=173464 combine_payload_bytes=274706432 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=274879896 ingress_bytes=605007872 link_bytes=605007872 token_bytes=549586328 weight_movement_bytes=330301440 total_nvlink_bytes=879887768 weight_to_token_ratio=0.601000 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=14080 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.052208196e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63918 payload_rows_remote=55738 dup_rows=1618 dup_rows_remote=1590 dedup_ratio=0.024689 router_tokens_per_expert=[0, 0, 3, 0, 0, 4096, 0, 0, 6, 0, 0, 760, 0, 0, 0, 0, 0, 4090, 14, 0, 0, 0, 4096, 0, 0, 248, 0, 0, 0, 1, 3352, 0, 0, 0, 0, 0, 0, 3, 0, 4095, 0, 0, 3978, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3930, 0, 0, 0, 0, 0, 0, 4095, 0] payload_rows_per_dst=[8192, 8192, 8180, 7156, 8190, 7956, 7862, 8190] dup_rows_per_dst=[6, 1532, 28, 46, 6, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8180, 7156, 8190, 7956, 7862, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63944 payload_rows_remote=56330 dup_rows=1592 dup_rows_remote=1592 dedup_ratio=0.024292 router_tokens_per_expert=[0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 601, 0, 0, 0, 0, 0, 4096, 9, 0, 0, 0, 4096, 0, 0, 463, 0, 1, 0, 0, 3620, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 3807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3798, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8186, 8192, 8192, 7796, 8176, 7614, 7596, 8192] dup_rows_per_dst=[0, 1202, 18, 372, 0, 0, 0, 0] combine_rows_per_src=[8186, 8192, 8192, 7796, 8176, 7614, 7596, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63932 payload_rows_remote=55740 dup_rows=1604 dup_rows_remote=1602 dedup_ratio=0.024475 router_tokens_per_expert=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 747, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 362, 0, 0, 0, 0, 3247, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 3997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3933, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8192, 8192, 7112, 8192, 7994, 7866, 8192] dup_rows_per_dst=[0, 1494, 2, 106, 2, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 7112, 8192, 7994, 7866, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=399529984 dispatch_meta_bytes=229312 combine_payload_bytes=399529984 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=399759296 ingress_bytes=950032384 link_bytes=950032384 token_bytes=799289280 weight_movement_bytes=550502400 total_nvlink_bytes=1349791680 weight_to_token_ratio=0.688740 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=27392 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.652262569e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62342 payload_rows_remote=54458 dup_rows=3194 dup_rows_remote=3194 dedup_ratio=0.048737 router_tokens_per_expert=[0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 459, 0, 0, 0, 0, 0, 4096, 15, 0, 0, 0, 4096, 0, 0, 264, 0, 0, 0, 0, 3618, 0, 0, 0, 0, 0, 0, 4, 0, 4095, 0, 0, 3985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3942, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 6196, 7894, 7824, 8190, 7970, 7884, 8192] dup_rows_per_dst=[4, 602, 30, 2550, 8, 0, 0, 0] combine_rows_per_src=[8192, 6196, 7894, 7824, 8190, 7970, 7884, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65050 payload_rows_remote=56858 dup_rows=486 dup_rows_remote=470 dedup_ratio=0.007416 router_tokens_per_expert=[0, 0, 0, 4096, 0, 3853, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 223, 0, 0, 0, 0, 6, 4096, 0, 0, 2, 0, 0, 14, 0, 0, 0, 0, 4093, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096] payload_rows_per_dst=[7706, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 446, 16, 22, 2, 0, 0] combine_rows_per_src=[7706, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=405522432 dispatch_meta_bytes=229368 combine_payload_bytes=405522432 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=405751800 ingress_bytes=625723392 link_bytes=625723392 token_bytes=811274232 weight_movement_bytes=220200960 total_nvlink_bytes=1031475192 weight_to_token_ratio=0.271426 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.088235893e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=383115264 dispatch_meta_bytes=214680 combine_payload_bytes=383115264 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=383329944 ingress_bytes=823517184 link_bytes=823517184 token_bytes=766445208 weight_movement_bytes=440401920 total_nvlink_bytes=1206847128 weight_to_token_ratio=0.574603 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=30848 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.432231828e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65324 payload_rows_remote=57132 dup_rows=212 dup_rows_remote=48 dedup_ratio=0.003235 router_tokens_per_expert=[0, 0, 0, 4096, 3, 3991, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 81, 0, 0, 0, 0, 10, 4095, 0, 0, 4, 0, 0, 6, 0, 0, 0, 1, 4095, 1, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4094] payload_rows_per_dst=[7988, 8190, 8192, 8190, 8192, 8192, 8192, 8188] dup_rows_per_dst=[0, 0, 164, 28, 14, 6, 0, 0] combine_rows_per_src=[7988, 8190, 8192, 8190, 8192, 8192, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64972 payload_rows_remote=56788 dup_rows=564 dup_rows_remote=358 dedup_ratio=0.008606 router_tokens_per_expert=[0, 0, 0, 4096, 3, 3921, 4095, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 117, 0, 0, 0, 0, 52, 4096, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 4094, 2, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 4090] payload_rows_per_dst=[7846, 8190, 8190, 8192, 8192, 7986, 8192, 8184] dup_rows_per_dst=[0, 0, 234, 108, 4, 10, 2, 206] combine_rows_per_src=[7846, 8190, 8190, 8192, 8192, 7986, 8192, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=405264384 dispatch_meta_bytes=229376 combine_payload_bytes=405264384 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=405493760 ingress_bytes=625465344 link_bytes=625465344 token_bytes=810758144 weight_movement_bytes=220200960 total_nvlink_bytes=1030959104 weight_to_token_ratio=0.271599 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.087787105e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=406697984 dispatch_meta_bytes=229376 combine_payload_bytes=406697984 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=406927360 ingress_bytes=516798464 link_bytes=516798464 token_bytes=813625344 weight_movement_bytes=110100480 total_nvlink_bytes=923725824 weight_to_token_ratio=0.135321 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.987975276e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64968 payload_rows_remote=56784 dup_rows=568 dup_rows_remote=568 dedup_ratio=0.008667 router_tokens_per_expert=[0, 0, 0, 4096, 0, 3861, 4095, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 1, 0, 0, 0, 115, 4096, 0, 8, 4, 0, 1, 1, 0, 0, 0, 0, 4095, 1, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4052] payload_rows_per_dst=[7722, 8184, 8192, 8192, 8190, 8192, 8192, 8104] dup_rows_per_dst=[0, 0, 302, 254, 6, 6, 0, 0] combine_rows_per_src=[7722, 8184, 8192, 8192, 8190, 8192, 8192, 8104] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=407558144 dispatch_meta_bytes=229312 combine_payload_bytes=407558144 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=407787456 ingress_bytes=627759104 link_bytes=627759104 token_bytes=815345600 weight_movement_bytes=220200960 total_nvlink_bytes=1035546560 weight_to_token_ratio=0.270071 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.091776330e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=409522176 dispatch_meta_bytes=228720 combine_payload_bytes=409522176 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=409750896 ingress_bytes=519622656 link_bytes=519622656 token_bytes=819273072 weight_movement_bytes=110100480 total_nvlink_bytes=929373552 weight_to_token_ratio=0.134388 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=31360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.037092620e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=407056384 dispatch_meta_bytes=228584 combine_payload_bytes=407056384 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=407284968 ingress_bytes=737357824 link_bytes=737357824 token_bytes=814341352 weight_movement_bytes=330301440 total_nvlink_bytes=1144642792 weight_to_token_ratio=0.405606 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.282386530e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=407027712 dispatch_meta_bytes=229408 combine_payload_bytes=407027712 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=407257120 ingress_bytes=517128192 link_bytes=517128192 token_bytes=814284832 weight_movement_bytes=110100480 total_nvlink_bytes=924385312 weight_to_token_ratio=0.135211 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.993709789e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62836 payload_rows_remote=54644 dup_rows=2700 dup_rows_remote=2690 dedup_ratio=0.041199 router_tokens_per_expert=[0, 0, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 2747, 0, 0, 0, 0, 0, 0, 162, 0, 0, 0, 4096, 0, 0, 1192, 0, 4, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 5508, 8192, 8192, 8192, 8180, 8188, 8192] dup_rows_per_dst=[0, 0, 324, 2366, 10, 0, 0, 0] combine_rows_per_src=[8192, 5508, 8192, 8192, 8192, 8180, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59938 payload_rows_remote=51746 dup_rows=5598 dup_rows_remote=5584 dedup_ratio=0.085419 router_tokens_per_expert=[0, 0, 7, 0, 0, 4096, 0, 0, 6, 0, 0, 2281, 0, 0, 0, 0, 0, 0, 502, 0, 0, 0, 4095, 0, 0, 1285, 0, 40, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 4091, 1, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 2, 0, 0, 0, 0, 0, 4094, 0] payload_rows_per_dst=[8192, 8192, 8190, 2646, 8182, 8168, 8176, 8192] dup_rows_per_dst=[14, 4574, 1004, 4, 2, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8190, 2646, 8182, 8168, 8176, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61544 payload_rows_remote=53438 dup_rows=3992 dup_rows_remote=3992 dedup_ratio=0.060913 router_tokens_per_expert=[0, 0, 0, 0, 0, 4094, 0, 0, 3, 0, 0, 2179, 0, 0, 0, 0, 0, 0, 194, 0, 0, 0, 4096, 0, 0, 1775, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 24, 0, 4096, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8188, 4364, 8192, 8192, 8192, 8106, 8118, 8192] dup_rows_per_dst=[0, 0, 388, 3556, 48, 0, 0, 0] combine_rows_per_src=[8188, 4364, 8192, 8192, 8192, 8106, 8118, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62460 payload_rows_remote=54276 dup_rows=3076 dup_rows_remote=3076 dedup_ratio=0.046936 router_tokens_per_expert=[0, 0, 3, 0, 0, 4096, 0, 0, 15, 0, 0, 2566, 0, 0, 0, 0, 0, 0, 576, 0, 0, 0, 4096, 0, 0, 951, 0, 3, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 5162, 8192, 8192, 8192, 8154, 8184, 8192] dup_rows_per_dst=[6, 0, 1152, 1902, 16, 0, 0, 0] combine_rows_per_src=[8192, 5162, 8192, 8192, 8192, 8154, 8184, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64148 payload_rows_remote=56630 dup_rows=1388 dup_rows_remote=1260 dedup_ratio=0.021179 router_tokens_per_expert=[0, 0, 7, 0, 0, 4096, 0, 0, 0, 0, 0, 599, 0, 0, 0, 0, 0, 4096, 24, 0, 0, 0, 4096, 0, 0, 332, 0, 0, 0, 0, 3491, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 3892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3945, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8192, 8192, 7518, 8188, 7784, 7890, 8192] dup_rows_per_dst=[14, 1198, 48, 128, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 7518, 8188, 7784, 7890, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57574 payload_rows_remote=49384 dup_rows=7962 dup_rows_remote=7952 dedup_ratio=0.121490 router_tokens_per_expert=[0, 0, 2, 0, 0, 4096, 0, 0, 4, 0, 0, 656, 0, 0, 0, 0, 0, 4094, 12, 0, 0, 0, 4096, 0, 0, 351, 0, 2, 0, 0, 3556, 0, 0, 0, 0, 0, 0, 2, 0, 4095, 0, 0, 3826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3881, 1, 0, 0, 0, 0, 0, 4094, 0] payload_rows_per_dst=[8130, 1320, 8192, 7758, 8190, 7854, 7940, 8190] dup_rows_per_dst=[4, 0, 58, 2538, 22, 2318, 3012, 10] combine_rows_per_src=[8130, 1320, 8192, 7758, 8190, 7854, 7940, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63508 payload_rows_remote=55316 dup_rows=2028 dup_rows_remote=208 dedup_ratio=0.030945 router_tokens_per_expert=[0, 0, 11, 0, 0, 4096, 0, 0, 8, 0, 0, 902, 0, 0, 0, 0, 0, 4094, 49, 0, 0, 0, 4096, 0, 0, 364, 0, 11, 0, 0, 3050, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 2, 0, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3882, 1, 0, 0, 0, 0, 0, 4092, 0] payload_rows_per_dst=[8192, 8192, 8188, 6764, 8192, 8030, 7764, 8186] dup_rows_per_dst=[22, 1820, 98, 86, 2, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8188, 6764, 8192, 8030, 7764, 8186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=399544320 dispatch_meta_bytes=229368 combine_payload_bytes=399544320 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=399773688 ingress_bytes=509644800 link_bytes=509644800 token_bytes=799318008 weight_movement_bytes=110100480 total_nvlink_bytes=909418488 weight_to_token_ratio=0.137743 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.863561294e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=403773440 dispatch_meta_bytes=231688 combine_payload_bytes=403773440 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=404005128 ingress_bytes=513873920 link_bytes=513873920 token_bytes=807778568 weight_movement_bytes=110100480 total_nvlink_bytes=917879048 weight_to_token_ratio=0.136300 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.937112646e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63826 payload_rows_remote=55634 dup_rows=1710 dup_rows_remote=38 dedup_ratio=0.026093 router_tokens_per_expert=[0, 0, 0, 4095, 11, 668, 4096, 0, 0, 4090, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 9, 4096, 0, 1, 2, 0, 1, 0, 0, 0, 0, 0, 4058, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 3446] payload_rows_per_dst=[8192, 8186, 8186, 7868, 8120, 8190, 8190, 6894] dup_rows_per_dst=[1672, 18, 0, 16, 2, 2, 0, 0] combine_rows_per_src=[8192, 8186, 8186, 7868, 8120, 8190, 8190, 6894] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=391688192 dispatch_meta_bytes=229336 combine_payload_bytes=391688192 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=391917528 ingress_bytes=611889152 link_bytes=611889152 token_bytes=783605720 weight_movement_bytes=220200960 total_nvlink_bytes=1003806680 weight_to_token_ratio=0.281010 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.064175874e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61688 payload_rows_remote=54180 dup_rows=3848 dup_rows_remote=1266 dedup_ratio=0.058716 router_tokens_per_expert=[0, 0, 7, 0, 0, 4096, 0, 0, 7, 0, 0, 2263, 0, 0, 0, 0, 0, 0, 367, 0, 0, 0, 4096, 0, 0, 1463, 0, 15, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 5076, 8192, 7508, 8192, 8148, 8188, 8192] dup_rows_per_dst=[14, 518, 734, 2582, 0, 0, 0, 0] combine_rows_per_src=[8192, 5076, 8192, 7508, 8192, 8148, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=370915328 dispatch_meta_bytes=229320 combine_payload_bytes=370915328 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=371144648 ingress_bytes=701216768 link_bytes=701216768 token_bytes=742059976 weight_movement_bytes=330301440 total_nvlink_bytes=1072361416 weight_to_token_ratio=0.445114 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.219531290e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59320 payload_rows_remote=51128 dup_rows=6216 dup_rows_remote=5712 dedup_ratio=0.094849 router_tokens_per_expert=[0, 0, 3, 0, 0, 4096, 0, 0, 20, 0, 0, 2812, 0, 0, 0, 0, 0, 0, 252, 0, 0, 0, 4096, 0, 0, 1000, 0, 1, 0, 1, 4092, 0, 0, 0, 0, 0, 0, 24, 0, 4095, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8190, 8192, 2004, 8190, 8170, 8190, 8192] dup_rows_per_dst=[6, 5658, 504, 0, 48, 0, 0, 0] combine_rows_per_src=[8192, 8190, 8192, 2004, 8190, 8170, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56950 payload_rows_remote=48760 dup_rows=8586 dup_rows_remote=8580 dedup_ratio=0.131012 router_tokens_per_expert=[0, 0, 2, 0, 0, 4096, 0, 0, 6, 0, 0, 2482, 0, 0, 0, 0, 0, 0, 319, 0, 0, 0, 4096, 0, 0, 1328, 0, 2, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 1, 0, 0, 0, 0, 0, 4094, 0] payload_rows_per_dst=[8120, 4976, 3116, 8192, 8060, 8114, 8182, 8190] dup_rows_per_dst=[4, 0, 274, 7828, 12, 326, 136, 6] combine_rows_per_src=[8120, 4976, 3116, 8192, 8060, 8114, 8182, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=383043584 dispatch_meta_bytes=229720 combine_payload_bytes=383043584 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=383273304 ingress_bytes=493144064 link_bytes=493144064 token_bytes=766316888 weight_movement_bytes=110100480 total_nvlink_bytes=876417368 weight_to_token_ratio=0.143675 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.576586356e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=389050368 dispatch_meta_bytes=229408 combine_payload_bytes=389050368 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=389279776 ingress_bytes=499150848 link_bytes=499150848 token_bytes=778330144 weight_movement_bytes=110100480 total_nvlink_bytes=888430624 weight_to_token_ratio=0.141457 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.681054209e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59672 payload_rows_remote=51480 dup_rows=5864 dup_rows_remote=738 dedup_ratio=0.089478 router_tokens_per_expert=[0, 0, 16, 0, 0, 4096, 0, 0, 18, 0, 0, 2572, 0, 0, 0, 0, 0, 0, 349, 0, 0, 0, 4096, 0, 0, 1168, 0, 18, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 3, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 1, 0, 0, 0, 0, 0, 4093, 0] payload_rows_per_dst=[8192, 8192, 8192, 2372, 8192, 8170, 8174, 8188] dup_rows_per_dst=[32, 5126, 698, 0, 8, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 2372, 8192, 8170, 8174, 8188] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=390354944 dispatch_meta_bytes=230608 combine_payload_bytes=390354944 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=390585552 ingress_bytes=500455424 link_bytes=500455424 token_bytes=780940496 weight_movement_bytes=110100480 total_nvlink_bytes=891040976 weight_to_token_ratio=0.140984 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31232 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.703742931e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57138 payload_rows_remote=49022 dup_rows=8398 dup_rows_remote=8398 dedup_ratio=0.128143 router_tokens_per_expert=[0, 2253, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3219, 0, 7, 1, 0, 0, 0, 0, 0, 843, 0, 4095, 0, 0, 0, 0, 4096, 0, 18, 0, 0, 6, 0, 0, 0, 3788, 0, 0, 0, 3, 0, 4094, 577, 0, 4096, 0, 0, 0, 1614, 0] payload_rows_per_dst=[8192, 8188, 8116, 6876, 5812, 8192, 7576, 4186] dup_rows_per_dst=[4506, 0, 0, 2376, 1266, 48, 6, 196] combine_rows_per_src=[8192, 8188, 8116, 6876, 5812, 8192, 7576, 4186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=405923840 dispatch_meta_bytes=231560 combine_payload_bytes=405923840 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=406155400 ingress_bytes=516024320 link_bytes=516024320 token_bytes=812079240 weight_movement_bytes=110100480 total_nvlink_bytes=922179720 weight_to_token_ratio=0.135578 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=27520 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.974511639e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=353984512 dispatch_meta_bytes=229344 combine_payload_bytes=353984512 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=354213856 ingress_bytes=464084992 link_bytes=464084992 token_bytes=708198368 weight_movement_bytes=110100480 total_nvlink_bytes=818298848 weight_to_token_ratio=0.155466 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.071201300e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=396505088 dispatch_meta_bytes=222096 combine_payload_bytes=396505088 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=396727184 ingress_bytes=506605568 link_bytes=506605568 token_bytes=793232272 weight_movement_bytes=110100480 total_nvlink_bytes=903332752 weight_to_token_ratio=0.138800 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=27520 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.810704051e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=398784512 dispatch_meta_bytes=222688 combine_payload_bytes=398784512 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=399007200 ingress_bytes=839186432 link_bytes=839186432 token_bytes=797791712 weight_movement_bytes=440401920 total_nvlink_bytes=1238193632 weight_to_token_ratio=0.552026 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=28416 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.459483228e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=351389696 dispatch_meta_bytes=229680 combine_payload_bytes=351389696 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=351619376 ingress_bytes=571590656 link_bytes=571590656 token_bytes=703009072 weight_movement_bytes=220200960 total_nvlink_bytes=923210032 weight_to_token_ratio=0.313226 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.940901613e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60398 payload_rows_remote=52206 dup_rows=5138 dup_rows_remote=2652 dedup_ratio=0.078400 router_tokens_per_expert=[0, 2804, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3028, 0, 20, 0, 0, 0, 0, 0, 0, 1243, 0, 4096, 0, 0, 0, 0, 4096, 0, 37, 0, 0, 0, 0, 0, 0, 3924, 0, 0, 0, 0, 0, 4096, 381, 0, 4096, 0, 0, 0, 870, 0] payload_rows_per_dst=[5608, 8192, 8154, 6084, 8192, 8192, 7852, 8124] dup_rows_per_dst=[0, 0, 0, 12, 2486, 74, 88, 2478] combine_rows_per_src=[5608, 8192, 8154, 6084, 8192, 8192, 7852, 8124] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58388 payload_rows_remote=50196 dup_rows=7148 dup_rows_remote=6652 dedup_ratio=0.109070 router_tokens_per_expert=[0, 2190, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2922, 0, 13, 0, 0, 0, 0, 0, 0, 941, 0, 4096, 0, 0, 0, 0, 4096, 0, 248, 0, 0, 0, 0, 0, 0, 3611, 0, 0, 0, 0, 0, 4096, 819, 0, 4096, 0, 0, 0, 1557, 0] payload_rows_per_dst=[4380, 8192, 8166, 5852, 8192, 8192, 7222, 8192] dup_rows_per_dst=[0, 0, 0, 18, 1882, 496, 0, 4752] combine_rows_per_src=[4380, 8192, 8166, 5852, 8192, 8192, 7222, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57546 payload_rows_remote=49740 dup_rows=7990 dup_rows_remote=7990 dedup_ratio=0.121918 router_tokens_per_expert=[0, 1774, 0, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2683, 0, 14, 0, 0, 0, 0, 0, 0, 1797, 0, 4096, 0, 0, 0, 0, 4096, 0, 118, 0, 0, 0, 0, 0, 0, 3903, 0, 0, 0, 0, 0, 4096, 883, 0, 4096, 0, 0, 0, 1190, 0] payload_rows_per_dst=[3548, 8192, 8044, 5380, 8192, 8192, 7806, 8192] dup_rows_per_dst=[0, 0, 0, 14, 3594, 236, 0, 4146] combine_rows_per_src=[3548, 8192, 8044, 5380, 8192, 8192, 7806, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=359804928 dispatch_meta_bytes=227392 combine_payload_bytes=359804928 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=360032320 ingress_bytes=580005888 link_bytes=580005888 token_bytes=719837248 weight_movement_bytes=220200960 total_nvlink_bytes=940038208 weight_to_token_ratio=0.305904 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=31744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.008725634e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=374212608 dispatch_meta_bytes=219432 combine_payload_bytes=374212608 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=374432040 ingress_bytes=594413568 link_bytes=594413568 token_bytes=748644648 weight_movement_bytes=220200960 total_nvlink_bytes=968845608 weight_to_token_ratio=0.294133 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=22784 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.033782959e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56146 payload_rows_remote=47954 dup_rows=9390 dup_rows_remote=3718 dedup_ratio=0.143280 router_tokens_per_expert=[0, 2082, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2835, 0, 18, 0, 0, 0, 0, 0, 0, 1771, 0, 4094, 0, 0, 0, 0, 4096, 0, 25, 0, 0, 3, 0, 0, 0, 3812, 0, 0, 0, 3, 0, 4094, 535, 0, 4096, 0, 0, 0, 1230, 0] payload_rows_per_dst=[3724, 8188, 8148, 5692, 7626, 6952, 7624, 8192] dup_rows_per_dst=[0, 0, 440, 14, 3204, 54, 6, 5672] combine_rows_per_src=[3724, 8188, 8148, 5692, 7626, 6952, 7624, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56680 payload_rows_remote=50372 dup_rows=8856 dup_rows_remote=8830 dedup_ratio=0.135132 router_tokens_per_expert=[0, 2188, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3152, 0, 15, 0, 0, 0, 0, 0, 0, 1241, 0, 4096, 0, 0, 0, 0, 4096, 0, 23, 0, 0, 0, 0, 0, 0, 3861, 0, 0, 0, 2, 0, 4095, 735, 0, 4096, 0, 0, 0, 1082, 0] payload_rows_per_dst=[6076, 8190, 8172, 6308, 8192, 8192, 8018, 3532] dup_rows_per_dst=[2228, 0, 0, 26, 2482, 46, 3972, 102] combine_rows_per_src=[6076, 8190, 8172, 6308, 8192, 8192, 8018, 3532] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54596 payload_rows_remote=46418 dup_rows=10940 dup_rows_remote=10940 dedup_ratio=0.166931 router_tokens_per_expert=[0, 2539, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2788, 0, 7, 1, 0, 1, 0, 0, 0, 1100, 0, 4095, 0, 0, 0, 0, 4096, 0, 76, 0, 0, 1, 0, 0, 0, 3927, 0, 0, 0, 4, 0, 4089, 682, 0, 4096, 0, 0, 0, 1186, 0] payload_rows_per_dst=[8192, 8178, 8160, 8192, 2200, 8192, 7854, 3628] dup_rows_per_dst=[5078, 0, 0, 5592, 0, 154, 8, 108] combine_rows_per_src=[8192, 8178, 8160, 8192, 2200, 8192, 7854, 3628] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=356536320 dispatch_meta_bytes=230920 combine_payload_bytes=356536320 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=356767240 ingress_bytes=686837760 link_bytes=686837760 token_bytes=713303560 weight_movement_bytes=330301440 total_nvlink_bytes=1043605000 weight_to_token_ratio=0.463059 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=30592 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.194523830e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64326 payload_rows_remote=56134 dup_rows=1210 dup_rows_remote=1192 dedup_ratio=0.018463 router_tokens_per_expert=[0, 0, 9, 0, 0, 4096, 0, 0, 5, 0, 0, 506, 0, 0, 0, 0, 0, 4091, 21, 0, 0, 0, 4095, 0, 0, 314, 0, 16, 0, 0, 3561, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3861, 2, 0, 0, 0, 0, 0, 4094, 0] payload_rows_per_dst=[8192, 8192, 8182, 7654, 8178, 8016, 7722, 8190] dup_rows_per_dst=[18, 1020, 42, 128, 0, 0, 0, 2] combine_rows_per_src=[8192, 8192, 8182, 7654, 8178, 8016, 7722, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56050 payload_rows_remote=49924 dup_rows=9486 dup_rows_remote=7666 dedup_ratio=0.144745 router_tokens_per_expert=[0, 2908, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2227, 0, 40, 0, 0, 0, 0, 0, 0, 1662, 0, 4096, 0, 0, 0, 0, 4096, 0, 19, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3978, 1045, 0, 4096, 0, 0, 0, 423, 0] payload_rows_per_dst=[5816, 8192, 8192, 8180, 6126, 8192, 8076, 3276] dup_rows_per_dst=[0, 0, 0, 4518, 1820, 38, 2682, 428] combine_rows_per_src=[5816, 8192, 8192, 8180, 6126, 8192, 8076, 3276] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56452 payload_rows_remote=48260 dup_rows=9084 dup_rows_remote=3742 dedup_ratio=0.138611 router_tokens_per_expert=[0, 2689, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 1761, 0, 5, 0, 0, 0, 0, 0, 0, 1368, 0, 4094, 0, 0, 0, 0, 4096, 0, 17, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3883, 1834, 0, 4096, 0, 0, 0, 746, 0] payload_rows_per_dst=[8192, 8192, 8192, 3530, 8192, 7588, 7764, 4802] dup_rows_per_dst=[5342, 2, 22, 2, 3336, 22, 0, 358] combine_rows_per_src=[8192, 8192, 8192, 3530, 8192, 7588, 7764, 4802] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55050 payload_rows_remote=46858 dup_rows=10486 dup_rows_remote=10302 dedup_ratio=0.160004 router_tokens_per_expert=[0, 1887, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1949, 0, 18, 0, 0, 0, 0, 0, 0, 1207, 0, 4096, 0, 0, 0, 0, 4096, 0, 92, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 3964, 2384, 0, 4096, 0, 0, 0, 802, 0] payload_rows_per_dst=[3774, 8190, 8192, 8166, 2414, 8192, 7930, 8192] dup_rows_per_dst=[0, 0, 0, 3930, 0, 184, 0, 6372] combine_rows_per_src=[3774, 8190, 8192, 8166, 2414, 8192, 7930, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=388362240 dispatch_meta_bytes=221784 combine_payload_bytes=388362240 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=388584024 ingress_bytes=608563200 link_bytes=608563200 token_bytes=776946264 weight_movement_bytes=220200960 total_nvlink_bytes=997147224 weight_to_token_ratio=0.283419 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=19968 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058391496e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=366485504 dispatch_meta_bytes=227360 combine_payload_bytes=366485504 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=366712864 ingress_bytes=696786944 link_bytes=696786944 token_bytes=733198368 weight_movement_bytes=330301440 total_nvlink_bytes=1063499808 weight_to_token_ratio=0.450494 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=30080 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.211827097e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=349511680 dispatch_meta_bytes=229360 combine_payload_bytes=349511680 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=349741040 ingress_bytes=459612160 link_bytes=459612160 token_bytes=699252720 weight_movement_bytes=110100480 total_nvlink_bytes=809353200 weight_to_token_ratio=0.157454 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.993411395e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=369008640 dispatch_meta_bytes=208872 combine_payload_bytes=369008640 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=369217512 ingress_bytes=479109120 link_bytes=479109120 token_bytes=738226152 weight_movement_bytes=110100480 total_nvlink_bytes=848326632 weight_to_token_ratio=0.149142 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=19968 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.332495596e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=357855232 dispatch_meta_bytes=230360 combine_payload_bytes=357855232 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=358085592 ingress_bytes=688156672 link_bytes=688156672 token_bytes=715940824 weight_movement_bytes=330301440 total_nvlink_bytes=1046242264 weight_to_token_ratio=0.461353 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=18816 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.196817635e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=345927680 dispatch_meta_bytes=208008 combine_payload_bytes=345927680 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=346135688 ingress_bytes=566128640 link_bytes=566128640 token_bytes=692063368 weight_movement_bytes=220200960 total_nvlink_bytes=912264328 weight_to_token_ratio=0.318180 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=17024 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.845908171e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=335878144 dispatch_meta_bytes=228640 combine_payload_bytes=335878144 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=336106784 ingress_bytes=556079104 link_bytes=556079104 token_bytes=671984928 weight_movement_bytes=220200960 total_nvlink_bytes=892185888 weight_to_token_ratio=0.327687 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.671130212e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56498 payload_rows_remote=48642 dup_rows=9038 dup_rows_remote=9038 dedup_ratio=0.137909 router_tokens_per_expert=[0, 1520, 0, 0, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1435, 0, 19, 0, 0, 0, 0, 0, 0, 2370, 0, 4096, 0, 0, 0, 0, 4096, 0, 76, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3928, 2312, 0, 4096, 0, 0, 0, 700, 0] payload_rows_per_dst=[3040, 8192, 8192, 8094, 4740, 8192, 7856, 8192] dup_rows_per_dst=[0, 0, 0, 2862, 0, 152, 0, 6024] combine_rows_per_src=[3040, 8192, 8192, 8094, 4740, 8192, 7856, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57636 payload_rows_remote=53068 dup_rows=7900 dup_rows_remote=7290 dedup_ratio=0.120544 router_tokens_per_expert=[0, 1956, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2003, 0, 17, 0, 0, 0, 0, 0, 0, 1763, 0, 4096, 0, 0, 0, 0, 4096, 0, 14, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 3837, 2171, 0, 4096, 0, 0, 0, 531, 0] payload_rows_per_dst=[7536, 8192, 8192, 4568, 8192, 8192, 7680, 5084] dup_rows_per_dst=[3416, 0, 0, 610, 3526, 28, 0, 320] combine_rows_per_src=[7536, 8192, 8192, 4568, 8192, 8192, 7680, 5084] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58490 payload_rows_remote=50298 dup_rows=7046 dup_rows_remote=7046 dedup_ratio=0.107513 router_tokens_per_expert=[0, 1904, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2497, 0, 26, 1, 0, 0, 0, 0, 0, 1461, 0, 4095, 0, 0, 0, 0, 4096, 0, 8, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 3983, 1458, 0, 4096, 0, 0, 0, 980, 0] payload_rows_per_dst=[8188, 8192, 8192, 5034, 8192, 8192, 7974, 4526] dup_rows_per_dst=[3736, 0, 0, 14, 2920, 26, 0, 350] combine_rows_per_src=[8188, 8192, 8192, 5034, 8192, 8192, 7974, 4526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56558 payload_rows_remote=48370 dup_rows=8978 dup_rows_remote=4304 dedup_ratio=0.136993 router_tokens_per_expert=[0, 1612, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2096, 0, 44, 0, 0, 0, 0, 0, 0, 2245, 0, 4094, 0, 0, 0, 0, 4096, 0, 13, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 3978, 1722, 0, 4096, 0, 0, 0, 601, 0] payload_rows_per_dst=[3224, 8192, 8164, 8146, 4490, 8192, 7962, 8188] dup_rows_per_dst=[0, 0, 4, 4266, 0, 34, 0, 4674] combine_rows_per_src=[3224, 8192, 8164, 8146, 4490, 8192, 7962, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=348665856 dispatch_meta_bytes=230720 combine_payload_bytes=348665856 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=348896576 ingress_bytes=678967296 link_bytes=678967296 token_bytes=697562432 weight_movement_bytes=330301440 total_nvlink_bytes=1027863872 weight_to_token_ratio=0.473508 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=31360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.180835799e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57108 payload_rows_remote=48916 dup_rows=8428 dup_rows_remote=8100 dedup_ratio=0.128601 router_tokens_per_expert=[0, 0, 0, 4096, 0, 146, 0, 0, 0, 0, 0, 0, 0, 2658, 0, 15, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 1332, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 1, 261, 0, 0, 4096, 0, 3772, 0, 4094, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4092, 0] payload_rows_per_dst=[8192, 7976, 8186, 8192, 8184, 8192, 2, 8184] dup_rows_per_dst=[328, 4914, 0, 2660, 2, 524, 0, 0] combine_rows_per_src=[8192, 7976, 8186, 8192, 8184, 8192, 2, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61726 payload_rows_remote=53534 dup_rows=3810 dup_rows_remote=3810 dedup_ratio=0.058136 router_tokens_per_expert=[0, 0, 0, 4096, 0, 51, 0, 0, 0, 0, 0, 0, 0, 2314, 0, 15, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 789, 0, 0, 4096, 0, 3958, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 4658, 8192, 8192, 8192, 8192, 7916, 8192] dup_rows_per_dst=[102, 0, 0, 2130, 0, 1578, 0, 0] combine_rows_per_src=[8192, 4658, 8192, 8192, 8192, 8192, 7916, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56572 payload_rows_remote=48380 dup_rows=8964 dup_rows_remote=8964 dedup_ratio=0.136780 router_tokens_per_expert=[0, 2548, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1795, 0, 12, 1, 0, 0, 0, 0, 0, 1699, 0, 4095, 0, 0, 0, 0, 4096, 0, 59, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 10, 0, 3796, 1717, 0, 4096, 0, 0, 0, 658, 0] payload_rows_per_dst=[8188, 8192, 8192, 3614, 8190, 8192, 7612, 4392] dup_rows_per_dst=[5084, 0, 2, 2, 3398, 120, 0, 358] combine_rows_per_src=[8188, 8192, 8192, 3614, 8190, 8192, 7612, 4392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=380391424 dispatch_meta_bytes=241432 combine_payload_bytes=380391424 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=380632856 ingress_bytes=490491904 link_bytes=490491904 token_bytes=761024280 weight_movement_bytes=110100480 total_nvlink_bytes=871124760 weight_to_token_ratio=0.144674 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=16896 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.530460931e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60250 payload_rows_remote=52058 dup_rows=5286 dup_rows_remote=5088 dedup_ratio=0.080658 router_tokens_per_expert=[0, 0, 0, 4096, 0, 158, 0, 0, 0, 0, 0, 0, 0, 2438, 0, 43, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 1627, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 0, 4096, 0, 3848, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 4962, 8148, 6598, 8192, 8192, 7774, 8192] dup_rows_per_dst=[316, 0, 0, 2628, 0, 198, 2144, 0] combine_rows_per_src=[8192, 4962, 8148, 6598, 8192, 8192, 7774, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=360536064 dispatch_meta_bytes=229376 combine_payload_bytes=360536064 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=360765440 ingress_bytes=470636544 link_bytes=470636544 token_bytes=721301504 weight_movement_bytes=110100480 total_nvlink_bytes=831401984 weight_to_token_ratio=0.152641 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.185143564e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=346716160 dispatch_meta_bytes=210696 combine_payload_bytes=346716160 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=346926856 ingress_bytes=566917120 link_bytes=566917120 token_bytes=693643016 weight_movement_bytes=220200960 total_nvlink_bytes=913843976 weight_to_token_ratio=0.317456 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=14720 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.859621135e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57354 payload_rows_remote=49162 dup_rows=8182 dup_rows_remote=8182 dedup_ratio=0.124847 router_tokens_per_expert=[0, 0, 0, 4096, 0, 243, 0, 0, 0, 0, 0, 0, 0, 991, 0, 160, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2087, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 1, 620, 0, 0, 4096, 0, 4079, 0, 4095, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8190, 8192, 8186, 8192, 8192, 18, 8192] dup_rows_per_dst=[486, 2302, 0, 4146, 6, 1242, 0, 0] combine_rows_per_src=[8192, 8190, 8192, 8186, 8192, 8192, 18, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=343734272 dispatch_meta_bytes=206688 combine_payload_bytes=343734272 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=343940960 ingress_bytes=674035712 link_bytes=674035712 token_bytes=687675232 weight_movement_bytes=330301440 total_nvlink_bytes=1017976672 weight_to_token_ratio=0.480316 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=16384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.172258963e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=361066496 dispatch_meta_bytes=236808 combine_payload_bytes=361066496 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=361303304 ingress_bytes=471166976 link_bytes=471166976 token_bytes=722369800 weight_movement_bytes=110100480 total_nvlink_bytes=832470280 weight_to_token_ratio=0.152416 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=23168 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.194368649e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=332724224 dispatch_meta_bytes=229432 combine_payload_bytes=332724224 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=332953656 ingress_bytes=552925184 link_bytes=552925184 token_bytes=665677880 weight_movement_bytes=220200960 total_nvlink_bytes=885878840 weight_to_token_ratio=0.330792 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.616278355e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=402368512 dispatch_meta_bytes=229304 combine_payload_bytes=402368512 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=402597816 ingress_bytes=732669952 link_bytes=732669952 token_bytes=804966328 weight_movement_bytes=330301440 total_nvlink_bytes=1135267768 weight_to_token_ratio=0.410330 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.274233550e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=352393216 dispatch_meta_bytes=229376 combine_payload_bytes=352393216 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=352622592 ingress_bytes=462493696 link_bytes=462493696 token_bytes=705015808 weight_movement_bytes=110100480 total_nvlink_bytes=815116288 weight_to_token_ratio=0.156167 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.043526045e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62790 payload_rows_remote=54598 dup_rows=2746 dup_rows_remote=2670 dedup_ratio=0.041901 router_tokens_per_expert=[0, 0, 0, 4096, 0, 434, 0, 0, 0, 0, 0, 0, 0, 668, 0, 233, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 2794, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 4096, 0, 4049, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8192, 8144, 5588, 8192, 8192, 8098, 8192] dup_rows_per_dst=[868, 1802, 0, 0, 0, 76, 0, 0] combine_rows_per_src=[8192, 8192, 8144, 5588, 8192, 8192, 8098, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62252 payload_rows_remote=54060 dup_rows=3284 dup_rows_remote=3284 dedup_ratio=0.050110 router_tokens_per_expert=[0, 0, 0, 4096, 0, 293, 0, 0, 0, 0, 0, 0, 0, 503, 0, 132, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 2460, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 709, 0, 0, 4096, 0, 4091, 0, 4096, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8192, 8190, 4920, 8192, 8192, 8182, 8192] dup_rows_per_dst=[586, 1270, 0, 0, 0, 1418, 10, 0] combine_rows_per_src=[8192, 8192, 8190, 4920, 8192, 8192, 8182, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57306 payload_rows_remote=49146 dup_rows=8230 dup_rows_remote=1394 dedup_ratio=0.125580 router_tokens_per_expert=[0, 0, 0, 4096, 0, 233, 0, 0, 0, 0, 0, 0, 0, 1449, 0, 229, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 1939, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 266, 0, 0, 4096, 0, 4080, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 4318, 8182, 3878, 8192, 8192, 8160, 8192] dup_rows_per_dst=[466, 396, 0, 0, 0, 532, 6836, 0] combine_rows_per_src=[8192, 4318, 8182, 3878, 8192, 8192, 8160, 8192] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=350629888 dispatch_meta_bytes=228064 combine_payload_bytes=350629888 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=350857952 ingress_bytes=680931328 link_bytes=680931328 token_bytes=701487840 weight_movement_bytes=330301440 total_nvlink_bytes=1031789280 weight_to_token_ratio=0.470858 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.184251573e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=383731712 dispatch_meta_bytes=229376 combine_payload_bytes=383731712 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=383961088 ingress_bytes=603932672 link_bytes=603932672 token_bytes=767692800 weight_movement_bytes=220200960 total_nvlink_bytes=987893760 weight_to_token_ratio=0.286835 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.050338246e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=346787840 dispatch_meta_bytes=229376 combine_payload_bytes=346787840 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=347017216 ingress_bytes=566988800 link_bytes=566988800 token_bytes=693805056 weight_movement_bytes=220200960 total_nvlink_bytes=914006016 weight_to_token_ratio=0.317382 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.860867768e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=373151744 dispatch_meta_bytes=228584 combine_payload_bytes=373151744 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=373380328 ingress_bytes=483252224 link_bytes=483252224 token_bytes=746532072 weight_movement_bytes=110100480 total_nvlink_bytes=856632552 weight_to_token_ratio=0.147483 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=29824 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.404550989e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56942 payload_rows_remote=48750 dup_rows=8594 dup_rows_remote=560 dedup_ratio=0.131134 router_tokens_per_expert=[0, 0, 0, 4096, 0, 104, 0, 0, 0, 0, 0, 0, 0, 3143, 0, 27, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 739, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 4096, 0, 4016, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 6340, 8164, 1478, 8192, 8192, 8192, 8192] dup_rows_per_dst=[208, 0, 0, 0, 0, 352, 8034, 0] combine_rows_per_src=[8192, 6340, 8164, 1478, 8192, 8192, 8192, 8192] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=391358464 dispatch_meta_bytes=229072 combine_payload_bytes=391358464 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=391587536 ingress_bytes=501458944 link_bytes=501458944 token_bytes=782946000 weight_movement_bytes=110100480 total_nvlink_bytes=893046480 weight_to_token_ratio=0.140623 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=29568 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.721195794e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=387502080 dispatch_meta_bytes=229376 combine_payload_bytes=387502080 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=387731456 ingress_bytes=607703040 link_bytes=607703040 token_bytes=775233536 weight_movement_bytes=220200960 total_nvlink_bytes=995434496 weight_to_token_ratio=0.284045 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.056895537e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=352278528 dispatch_meta_bytes=202160 combine_payload_bytes=352278528 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=352480688 ingress_bytes=792680448 link_bytes=792680448 token_bytes=704759216 weight_movement_bytes=440401920 total_nvlink_bytes=1145161136 weight_to_token_ratio=0.624897 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=19328 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.378601673e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45004 payload_rows_remote=36814 dup_rows=20532 dup_rows_remote=20518 dedup_ratio=0.313293 router_tokens_per_expert=[0, 0, 0, 4096, 0, 264, 0, 0, 0, 1, 0, 0, 0, 696, 0, 54, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 2889, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 2, 198, 0, 0, 4096, 0, 4085, 0, 4095, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4095, 0] payload_rows_per_dst=[2896, 1502, 8180, 5778, 8192, 2076, 8190, 8190] dup_rows_per_dst=[344, 0, 0, 0, 32, 92, 20050, 14] combine_rows_per_src=[2896, 1502, 8180, 5778, 8192, 2076, 8190, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61226 payload_rows_remote=55068 dup_rows=4310 dup_rows_remote=2950 dedup_ratio=0.065765 router_tokens_per_expert=[0, 0, 0, 4096, 0, 322, 0, 0, 0, 2, 0, 0, 0, 494, 0, 90, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 2633, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 565, 0, 0, 4096, 0, 4085, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8192, 8186, 6158, 8192, 8192, 5922, 8192] dup_rows_per_dst=[644, 1172, 0, 1360, 0, 1130, 4, 0] combine_rows_per_src=[8192, 8192, 8186, 6158, 8192, 8192, 5922, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57356 payload_rows_remote=49166 dup_rows=8180 dup_rows_remote=6298 dedup_ratio=0.124817 router_tokens_per_expert=[0, 0, 0, 4096, 0, 512, 0, 0, 0, 0, 0, 0, 0, 893, 0, 48, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 2240, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 1, 431, 0, 0, 4096, 0, 4032, 0, 4095, 0, 0, 31, 0, 0, 0, 0, 0, 0, 4095, 0] payload_rows_per_dst=[8192, 8190, 8190, 8148, 8192, 8192, 62, 8190] dup_rows_per_dst=[1024, 1882, 0, 4396, 14, 864, 0, 0] combine_rows_per_src=[8192, 8190, 8190, 8148, 8192, 8192, 62, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54518 payload_rows_remote=46326 dup_rows=11018 dup_rows_remote=5206 dedup_ratio=0.168121 router_tokens_per_expert=[0, 2906, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 2449, 0, 7, 0, 0, 0, 0, 0, 0, 1108, 0, 4094, 0, 0, 0, 0, 4096, 0, 63, 0, 0, 3, 0, 0, 0, 3767, 0, 0, 0, 0, 0, 4093, 684, 0, 4096, 0, 0, 0, 1334, 0] payload_rows_per_dst=[8192, 8192, 8136, 8188, 2216, 8192, 7504, 3898] dup_rows_per_dst=[5812, 24, 0, 4912, 0, 132, 0, 138] combine_rows_per_src=[8192, 8192, 8136, 8188, 2216, 8192, 7504, 3898] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65488 payload_rows_remote=57298 dup_rows=48 dup_rows_remote=48 dedup_ratio=0.000732 router_tokens_per_expert=[0, 0, 0, 0, 0, 3, 0, 0, 0, 4079, 0, 3, 0, 4091, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 107, 0, 0, 0, 0, 0, 4096, 4010, 0, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8160, 8190, 8192, 8188, 8186, 8192, 8192] dup_rows_per_dst=[0, 4, 0, 0, 0, 0, 42, 2] combine_rows_per_src=[8188, 8160, 8190, 8192, 8188, 8186, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61584 payload_rows_remote=53392 dup_rows=3952 dup_rows_remote=1202 dedup_ratio=0.060303 router_tokens_per_expert=[0, 0, 0, 4096, 0, 42, 0, 0, 0, 0, 0, 0, 0, 2220, 0, 5, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 1375, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 557, 0, 0, 4096, 0, 4000, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0] payload_rows_per_dst=[8192, 4510, 8174, 8192, 8192, 8192, 7940, 8192] dup_rows_per_dst=[84, 0, 0, 2750, 0, 1114, 0, 4] combine_rows_per_src=[8192, 4510, 8174, 8192, 8192, 8192, 7940, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57256 payload_rows_remote=49072 dup_rows=8280 dup_rows_remote=8280 dedup_ratio=0.126343 router_tokens_per_expert=[0, 0, 0, 4096, 0, 22, 0, 0, 0, 0, 0, 0, 0, 2750, 0, 13, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 922, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4, 545, 0, 0, 4096, 0, 3941, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8112, 8184, 8192, 8192, 8192, 0, 8192] dup_rows_per_dst=[44, 5296, 0, 1842, 0, 1098, 0, 0] combine_rows_per_src=[8192, 8112, 8184, 8192, 8192, 8192, 0, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=349440000 dispatch_meta_bytes=197240 combine_payload_bytes=349440000 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=349637240 ingress_bytes=789841920 link_bytes=789841920 token_bytes=699077240 weight_movement_bytes=440401920 total_nvlink_bytes=1139479160 weight_to_token_ratio=0.629976 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=19840 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.373665006e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49914 payload_rows_remote=41724 dup_rows=15622 dup_rows_remote=15614 dedup_ratio=0.238373 router_tokens_per_expert=[0, 0, 0, 4096, 0, 66, 0, 0, 0, 0, 0, 0, 0, 2854, 0, 1, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 1200, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 2, 133, 0, 0, 4096, 0, 3941, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0] payload_rows_per_dst=[6704, 5710, 8182, 2400, 8184, 2354, 8190, 8190] dup_rows_per_dst=[108, 0, 124, 0, 4, 50, 15328, 8] combine_rows_per_src=[6704, 5710, 8182, 2400, 8184, 2354, 8190, 8190] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65508 payload_rows_remote=57316 dup_rows=28 dup_rows_remote=18 dedup_ratio=0.000427 router_tokens_per_expert=[0, 0, 0, 0, 0, 5, 0, 0, 0, 4092, 0, 8, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4092, 0, 1, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8186, 8178, 8186, 8190] dup_rows_per_dst=[10, 8, 4, 6, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8186, 8178, 8186, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=382713856 dispatch_meta_bytes=218376 combine_payload_bytes=382713856 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=382932232 ingress_bytes=492814336 link_bytes=492814336 token_bytes=765646088 weight_movement_bytes=110100480 total_nvlink_bytes=875746568 weight_to_token_ratio=0.143801 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=23552 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.570851843e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57190 payload_rows_remote=49156 dup_rows=8346 dup_rows_remote=3322 dedup_ratio=0.127350 router_tokens_per_expert=[0, 0, 0, 4096, 0, 149, 0, 0, 0, 0, 0, 0, 0, 2663, 0, 2, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 2, 460, 0, 0, 4096, 0, 3864, 0, 4095, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4095, 0] payload_rows_per_dst=[8192, 8034, 8182, 8192, 8192, 8192, 16, 8190] dup_rows_per_dst=[298, 5024, 0, 2090, 10, 924, 0, 0] combine_rows_per_src=[8192, 8034, 8182, 8192, 8192, 8192, 16, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=351748096 dispatch_meta_bytes=229408 combine_payload_bytes=351748096 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=351977504 ingress_bytes=461848576 link_bytes=461848576 token_bytes=703725600 weight_movement_bytes=110100480 total_nvlink_bytes=813826080 weight_to_token_ratio=0.156454 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.032306347e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=299077632 dispatch_meta_bytes=229352 combine_payload_bytes=299077632 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=299306984 ingress_bytes=409178112 link_bytes=409178112 token_bytes=598384616 weight_movement_bytes=110100480 total_nvlink_bytes=708485096 weight_to_token_ratio=0.183996 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.116280350e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=410841088 dispatch_meta_bytes=229336 combine_payload_bytes=410841088 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=411070424 ingress_bytes=520941568 link_bytes=520941568 token_bytes=821911512 weight_movement_bytes=110100480 total_nvlink_bytes=932011992 weight_to_token_ratio=0.133957 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.060030669e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=411041792 dispatch_meta_bytes=229376 combine_payload_bytes=411041792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=411271168 ingress_bytes=521142272 link_bytes=521142272 token_bytes=822312960 weight_movement_bytes=110100480 total_nvlink_bytes=932413440 weight_to_token_ratio=0.133891 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.063521242e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=352350208 dispatch_meta_bytes=209912 combine_payload_bytes=352350208 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=352560120 ingress_bytes=462450688 link_bytes=462450688 token_bytes=704910328 weight_movement_bytes=110100480 total_nvlink_bytes=815010808 weight_to_token_ratio=0.156191 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=21120 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.042778065e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=411041792 dispatch_meta_bytes=229376 combine_payload_bytes=411041792 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=411271168 ingress_bytes=631242752 link_bytes=631242752 token_bytes=822312960 weight_movement_bytes=220200960 total_nvlink_bytes=1042513920 weight_to_token_ratio=0.267782 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.097834967e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=411041792 dispatch_meta_bytes=229376 combine_payload_bytes=411041792 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=411271168 ingress_bytes=741343232 link_bytes=741343232 token_bytes=822312960 weight_movement_bytes=330301440 total_nvlink_bytes=1152614400 weight_to_token_ratio=0.401674 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.289317810e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65522 payload_rows_remote=57332 dup_rows=14 dup_rows_remote=14 dedup_ratio=0.000214 router_tokens_per_expert=[0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 8192, 8186, 8186, 8192, 8192] dup_rows_per_dst=[6, 6, 0, 0, 0, 0, 0, 2] combine_rows_per_src=[8192, 8192, 8190, 8192, 8186, 8186, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65486 payload_rows_remote=57294 dup_rows=50 dup_rows_remote=50 dedup_ratio=0.000763 router_tokens_per_expert=[0, 0, 0, 0, 0, 2, 0, 0, 0, 4092, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8170, 8174, 8192, 8186, 8192, 8188, 8192, 8192] dup_rows_per_dst=[4, 0, 0, 0, 12, 28, 6, 0] combine_rows_per_src=[8170, 8174, 8192, 8186, 8192, 8188, 8192, 8192] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=263882752 dispatch_meta_bytes=229328 combine_payload_bytes=263882752 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=264112080 ingress_bytes=373983232 link_bytes=373983232 token_bytes=527994832 weight_movement_bytes=110100480 total_nvlink_bytes=638095312 weight_to_token_ratio=0.208526 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.504183501e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=394727424 dispatch_meta_bytes=232072 combine_payload_bytes=394727424 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=394959496 ingress_bytes=504827904 link_bytes=504827904 token_bytes=789686920 weight_movement_bytes=110100480 total_nvlink_bytes=899787400 weight_to_token_ratio=0.139423 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=19584 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.779787550e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=352421888 dispatch_meta_bytes=221856 combine_payload_bytes=352421888 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=352643744 ingress_bytes=462522368 link_bytes=462522368 token_bytes=705065632 weight_movement_bytes=110100480 total_nvlink_bytes=815166112 weight_to_token_ratio=0.156156 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=25344 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.044024698e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=410712064 dispatch_meta_bytes=229384 combine_payload_bytes=410712064 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=410941448 ingress_bytes=520812544 link_bytes=520812544 token_bytes=821653512 weight_movement_bytes=110100480 total_nvlink_bytes=931753992 weight_to_token_ratio=0.133999 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.057786730e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=332064768 dispatch_meta_bytes=206128 combine_payload_bytes=332064768 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=332270896 ingress_bytes=662366208 link_bytes=662366208 token_bytes=664335664 weight_movement_bytes=330301440 total_nvlink_bytes=994637104 weight_to_token_ratio=0.497191 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=18560 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.151963776e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65512 payload_rows_remote=57320 dup_rows=24 dup_rows_remote=24 dedup_ratio=0.000366 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 39, 0, 0, 0, 0, 0, 4096, 4069, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8172, 8192, 8192, 8188, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 24, 0] combine_rows_per_src=[8192, 8172, 8192, 8192, 8188, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65492 payload_rows_remote=57300 dup_rows=44 dup_rows_remote=44 dedup_ratio=0.000671 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 170, 0, 0, 0, 0, 0, 4096, 3948, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8148, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 44, 0] combine_rows_per_src=[8192, 8148, 8192, 8192, 8192, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65466 payload_rows_remote=57274 dup_rows=70 dup_rows_remote=0 dedup_ratio=0.001068 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 153, 0, 0, 0, 0, 0, 4096, 3978, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8122, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 70, 0] combine_rows_per_src=[8192, 8122, 8192, 8192, 8192, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=410869760 dispatch_meta_bytes=229376 combine_payload_bytes=410869760 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=411099136 ingress_bytes=631070720 link_bytes=631070720 token_bytes=821968896 weight_movement_bytes=220200960 total_nvlink_bytes=1042169856 weight_to_token_ratio=0.267895 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.097535775e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=410726400 dispatch_meta_bytes=229376 combine_payload_bytes=410726400 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=410955776 ingress_bytes=520826880 link_bytes=520826880 token_bytes=821682176 weight_movement_bytes=110100480 total_nvlink_bytes=931782656 weight_to_token_ratio=0.133994 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.058036056e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=410540032 dispatch_meta_bytes=229096 combine_payload_bytes=410540032 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=410769128 ingress_bytes=961042432 link_bytes=961042432 token_bytes=821309160 weight_movement_bytes=550502400 total_nvlink_bytes=1371811560 weight_to_token_ratio=0.670274 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.671410854e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65468 payload_rows_remote=57276 dup_rows=68 dup_rows_remote=68 dedup_ratio=0.001038 router_tokens_per_expert=[0, 0, 0, 0, 0, 2, 0, 0, 0, 4066, 0, 3, 0, 4093, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 98, 0, 0, 0, 0, 0, 4096, 4028, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8138, 8190, 8186, 8192, 8188, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 2, 6, 60, 0] combine_rows_per_src=[8190, 8138, 8190, 8186, 8192, 8188, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65468 payload_rows_remote=57276 dup_rows=68 dup_rows_remote=68 dedup_ratio=0.001038 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 2, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 56, 0, 0, 0, 0, 0, 4096, 4073, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8128, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 2, 0, 0, 0, 0, 66, 0] combine_rows_per_src=[8188, 8128, 8192, 8192, 8192, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65434 payload_rows_remote=57320 dup_rows=102 dup_rows_remote=100 dedup_ratio=0.001556 router_tokens_per_expert=[0, 0, 0, 0, 0, 2, 0, 0, 0, 4055, 0, 3, 0, 4087, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 200, 0, 0, 0, 0, 0, 4096, 3945, 0, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8178, 8114, 8190, 8192, 8190, 8186, 8192, 8192] dup_rows_per_dst=[0, 2, 0, 0, 0, 0, 98, 2] combine_rows_per_src=[8178, 8114, 8190, 8192, 8190, 8186, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64004 payload_rows_remote=55812 dup_rows=1532 dup_rows_remote=1520 dedup_ratio=0.023376 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1168, 0, 0, 2, 0, 0, 48, 4096, 0, 0, 0, 7, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3367, 0, 0, 3600, 0, 0] payload_rows_per_dst=[8118, 8192, 8192, 8192, 8192, 8192, 8192, 6734] dup_rows_per_dst=[1418, 100, 12, 0, 0, 2, 0, 0] combine_rows_per_src=[8118, 8192, 8192, 8192, 8192, 8192, 8192, 6734] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64958 payload_rows_remote=56848 dup_rows=578 dup_rows_remote=394 dedup_ratio=0.008820 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 1, 0, 0, 98, 4096, 0, 0, 0, 5, 4096, 0, 0, 4095, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3941, 0, 0, 4055, 0, 0] payload_rows_per_dst=[8110, 8192, 8098, 8102, 8192, 8190, 8192, 7882] dup_rows_per_dst=[184, 382, 10, 2, 0, 0, 0, 0] combine_rows_per_src=[8110, 8192, 8098, 8102, 8192, 8190, 8192, 7882] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=411041792 dispatch_meta_bytes=229376 combine_payload_bytes=411041792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=411271168 ingress_bytes=521142272 link_bytes=521142272 token_bytes=822312960 weight_movement_bytes=110100480 total_nvlink_bytes=932413440 weight_to_token_ratio=0.133891 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.063521242e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57262 payload_rows_remote=49070 dup_rows=8274 dup_rows_remote=7396 dedup_ratio=0.126251 router_tokens_per_expert=[0, 0, 0, 4096, 0, 439, 0, 0, 0, 0, 0, 0, 0, 734, 0, 140, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2480, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 16, 0, 0, 0, 0, 0, 0, 1, 380, 0, 0, 4096, 0, 4009, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4091, 0] payload_rows_per_dst=[8192, 8188, 8188, 8176, 8140, 8192, 4, 8182] dup_rows_per_dst=[878, 1748, 84, 4802, 0, 762, 0, 0] combine_rows_per_src=[8192, 8188, 8188, 8176, 8140, 8192, 4, 8182] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=410554368 dispatch_meta_bytes=229376 combine_payload_bytes=410554368 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=410783744 ingress_bytes=520654848 link_bytes=520654848 token_bytes=821338112 weight_movement_bytes=110100480 total_nvlink_bytes=931438592 weight_to_token_ratio=0.134050 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.055044137e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=410554368 dispatch_meta_bytes=229376 combine_payload_bytes=410554368 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=410783744 ingress_bytes=520654848 link_bytes=520654848 token_bytes=821338112 weight_movement_bytes=110100480 total_nvlink_bytes=931438592 weight_to_token_ratio=0.134050 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.055044137e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=410869760 dispatch_meta_bytes=229680 combine_payload_bytes=410869760 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=411099440 ingress_bytes=741171200 link_bytes=741171200 token_bytes=821969200 weight_movement_bytes=330301440 total_nvlink_bytes=1152270640 weight_to_token_ratio=0.401842 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.289018619e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=400060416 dispatch_meta_bytes=229328 combine_payload_bytes=400060416 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=400289744 ingress_bytes=620261376 link_bytes=620261376 token_bytes=800350160 weight_movement_bytes=220200960 total_nvlink_bytes=1020551120 weight_to_token_ratio=0.275131 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.078736548e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=351733760 dispatch_meta_bytes=225864 combine_payload_bytes=351733760 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=351959624 ingress_bytes=682035200 link_bytes=682035200 token_bytes=703693384 weight_movement_bytes=330301440 total_nvlink_bytes=1033994824 weight_to_token_ratio=0.469383 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=30080 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.186171388e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64980 payload_rows_remote=56788 dup_rows=556 dup_rows_remote=556 dedup_ratio=0.008484 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 627, 0, 0, 0, 0, 0, 21, 4096, 0, 0, 0, 39, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3867, 0, 0, 3639, 0, 0] payload_rows_per_dst=[8096, 8192, 8190, 8192, 8192, 8192, 8192, 7734] dup_rows_per_dst=[436, 42, 78, 0, 0, 0, 0, 0] combine_rows_per_src=[8096, 8192, 8190, 8192, 8192, 8192, 8192, 7734] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64572 payload_rows_remote=56380 dup_rows=964 dup_rows_remote=964 dedup_ratio=0.014709 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 627, 0, 0, 0, 0, 0, 87, 4096, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3661, 0, 0, 3810, 0, 0] payload_rows_per_dst=[8098, 8192, 8192, 8192, 8192, 8192, 8192, 7322] dup_rows_per_dst=[776, 174, 14, 0, 0, 0, 0, 0] combine_rows_per_src=[8098, 8192, 8192, 8192, 8192, 8192, 8192, 7322] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64174 payload_rows_remote=55982 dup_rows=1362 dup_rows_remote=1362 dedup_ratio=0.020782 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 969, 0, 0, 0, 0, 0, 31, 4096, 0, 0, 0, 20, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3443, 0, 0, 3722, 0, 0] payload_rows_per_dst=[8136, 8192, 8192, 8192, 8192, 8192, 8192, 6886] dup_rows_per_dst=[1246, 62, 40, 14, 0, 0, 0, 0] combine_rows_per_src=[8136, 8192, 8192, 8192, 8192, 8192, 8192, 6886] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65068 payload_rows_remote=56876 dup_rows=468 dup_rows_remote=468 dedup_ratio=0.007141 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 154, 4096, 0, 0, 0, 4, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3889, 0, 0, 4068, 0, 0] payload_rows_per_dst=[8138, 8192, 8192, 8192, 8192, 8192, 8192, 7778] dup_rows_per_dst=[144, 308, 8, 8, 0, 0, 0, 0] combine_rows_per_src=[8138, 8192, 8192, 8192, 8192, 8192, 8192, 7778] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65530 payload_rows_remote=57338 dup_rows=6 dup_rows_remote=2 dedup_ratio=0.000092 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 8192, 8192, 8188, 8192, 8192] dup_rows_per_dst=[2, 4, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8190, 8192, 8192, 8188, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65260 payload_rows_remote=57068 dup_rows=276 dup_rows_remote=276 dedup_ratio=0.004211 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 35, 4096, 0, 0, 0, 25, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3991, 0, 0, 4057, 0, 0] payload_rows_per_dst=[8126, 8192, 8192, 8192, 8192, 8192, 8192, 7982] dup_rows_per_dst=[156, 70, 50, 0, 0, 0, 0, 0] combine_rows_per_src=[8126, 8192, 8192, 8192, 8192, 8192, 8192, 7982] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=410955776 dispatch_meta_bytes=229384 combine_payload_bytes=410955776 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=411185160 ingress_bytes=521056256 link_bytes=521056256 token_bytes=822140936 weight_movement_bytes=110100480 total_nvlink_bytes=932241416 weight_to_token_ratio=0.133919 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.062025282e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=410683392 dispatch_meta_bytes=229376 combine_payload_bytes=410683392 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=410912768 ingress_bytes=520783872 link_bytes=520783872 token_bytes=821596160 weight_movement_bytes=110100480 total_nvlink_bytes=931696640 weight_to_token_ratio=0.134008 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.057288076e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=407486464 dispatch_meta_bytes=228968 combine_payload_bytes=407486464 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=407715432 ingress_bytes=517586944 link_bytes=517586944 token_bytes=815201896 weight_movement_bytes=110100480 total_nvlink_bytes=925302376 weight_to_token_ratio=0.135059 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=31744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.001688240e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64858 payload_rows_remote=56666 dup_rows=678 dup_rows_remote=678 dedup_ratio=0.010345 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 229, 0, 0, 0, 0, 0, 74, 4096, 0, 0, 0, 9, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3794, 0, 0, 4058, 0, 0] payload_rows_per_dst=[8118, 8192, 8192, 8192, 8192, 8192, 8192, 7588] dup_rows_per_dst=[456, 148, 18, 56, 0, 0, 0, 0] combine_rows_per_src=[8118, 8192, 8192, 8192, 8192, 8192, 8192, 7588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=407687168 dispatch_meta_bytes=229376 combine_payload_bytes=407687168 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=407916544 ingress_bytes=517787648 link_bytes=517787648 token_bytes=815603712 weight_movement_bytes=110100480 total_nvlink_bytes=925704192 weight_to_token_ratio=0.134993 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.005178813e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=410998784 dispatch_meta_bytes=229360 combine_payload_bytes=410998784 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=411228144 ingress_bytes=631199744 link_bytes=631199744 token_bytes=822226928 weight_movement_bytes=220200960 total_nvlink_bytes=1042427888 weight_to_token_ratio=0.267810 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.097760169e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=409063424 dispatch_meta_bytes=229376 combine_payload_bytes=409063424 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=409292800 ingress_bytes=519163904 link_bytes=519163904 token_bytes=818356224 weight_movement_bytes=110100480 total_nvlink_bytes=928456704 weight_to_token_ratio=0.134539 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.029114168e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=407056384 dispatch_meta_bytes=229376 combine_payload_bytes=407056384 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=407285760 ingress_bytes=517156864 link_bytes=517156864 token_bytes=814342144 weight_movement_bytes=110100480 total_nvlink_bytes=924442624 weight_to_token_ratio=0.135202 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.994208442e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58176 payload_rows_remote=50050 dup_rows=7360 dup_rows_remote=178 dedup_ratio=0.112305 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 539, 0, 0, 1, 0, 0, 55, 4096, 0, 0, 0, 8, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3652, 0, 0, 3937, 0, 0] payload_rows_per_dst=[1998, 7092, 8192, 8192, 8192, 8192, 8192, 8126] dup_rows_per_dst=[58, 104, 16, 0, 0, 0, 0, 7182] combine_rows_per_src=[1998, 7092, 8192, 8192, 8192, 8192, 8192, 8126] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=404131840 dispatch_meta_bytes=229376 combine_payload_bytes=404131840 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=404361216 ingress_bytes=514232320 link_bytes=514232320 token_bytes=808493056 weight_movement_bytes=110100480 total_nvlink_bytes=918593536 weight_to_token_ratio=0.136180 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.943345812e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=401278976 dispatch_meta_bytes=229376 combine_payload_bytes=401278976 prefetch_bytes=0 reduce_bytes=0 egress_bytes=401508352 ingress_bytes=401278976 link_bytes=401508352 token_bytes=802787328 weight_movement_bytes=0 total_nvlink_bytes=802787328 weight_to_token_ratio=0.000000 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.982890609e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64674 payload_rows_remote=56482 dup_rows=862 dup_rows_remote=852 dedup_ratio=0.013153 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 844, 0, 0, 0, 0, 1, 25, 4096, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3689, 0, 0, 3622, 0, 0] payload_rows_per_dst=[8144, 8192, 8192, 8192, 8192, 8192, 8192, 7378] dup_rows_per_dst=[788, 52, 12, 10, 0, 0, 0, 0] combine_rows_per_src=[8144, 8192, 8192, 8192, 8192, 8192, 8192, 7378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64114 payload_rows_remote=55922 dup_rows=1422 dup_rows_remote=1214 dedup_ratio=0.021698 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 742, 0, 0, 1, 0, 0, 103, 4096, 0, 0, 0, 8, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3448, 0, 0, 3888, 0, 0] payload_rows_per_dst=[8066, 8192, 8192, 8192, 8192, 8192, 8192, 6896] dup_rows_per_dst=[1194, 208, 16, 2, 0, 2, 0, 0] combine_rows_per_src=[8066, 8192, 8192, 8192, 8192, 8192, 8192, 6896] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63228 payload_rows_remote=55036 dup_rows=2308 dup_rows_remote=2306 dedup_ratio=0.035217 router_tokens_per_expert=[0, 0, 922, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3405, 0, 2, 11, 0, 0, 0, 2, 0, 3662, 0, 0, 0, 0, 4, 4096, 5, 0, 0, 0, 220, 4093, 0, 0, 0, 0, 4095, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8114, 6836, 7324, 8192, 8186] dup_rows_per_dst=[1844, 0, 2, 0, 0, 4, 458, 0] combine_rows_per_src=[8192, 8192, 8192, 8114, 6836, 7324, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65306 payload_rows_remote=57114 dup_rows=230 dup_rows_remote=216 dedup_ratio=0.003510 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 40, 4096, 0, 0, 0, 2, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4000, 0, 0, 4075, 0, 0] payload_rows_per_dst=[8154, 8192, 8192, 8192, 8192, 8192, 8192, 8000] dup_rows_per_dst=[132, 80, 4, 14, 0, 0, 0, 0] combine_rows_per_src=[8154, 8192, 8192, 8192, 8192, 8192, 8192, 8000] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=406181888 dispatch_meta_bytes=229376 combine_payload_bytes=406181888 prefetch_bytes=0 reduce_bytes=0 egress_bytes=406411264 ingress_bytes=406181888 link_bytes=406411264 token_bytes=812593152 weight_movement_bytes=0 total_nvlink_bytes=812593152 weight_to_token_ratio=0.000000 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.068160313e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64722 payload_rows_remote=56530 dup_rows=814 dup_rows_remote=814 dedup_ratio=0.012421 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 296, 0, 0, 2, 0, 0, 111, 4096, 0, 0, 0, 1, 4096, 0, 0, 4095, 0, 4096, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3722, 0, 0, 4059, 0, 0] payload_rows_per_dst=[8126, 8192, 8192, 8192, 8192, 8192, 8192, 7444] dup_rows_per_dst=[584, 226, 0, 4, 0, 0, 0, 0] combine_rows_per_src=[8126, 8192, 8192, 8192, 8192, 8192, 8192, 7444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61896 payload_rows_remote=53844 dup_rows=3640 dup_rows_remote=190 dedup_ratio=0.055542 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 1, 0, 0, 95, 4096, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 0, 0, 4069, 0, 0] payload_rows_per_dst=[6592, 6292, 8192, 8192, 8192, 8192, 8192, 8052] dup_rows_per_dst=[44, 144, 2, 0, 0, 0, 0, 3450] combine_rows_per_src=[6592, 6292, 8192, 8192, 8192, 8192, 8192, 8052] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=409393152 dispatch_meta_bytes=229320 combine_payload_bytes=409393152 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=409622472 ingress_bytes=629594112 link_bytes=629594112 token_bytes=819015624 weight_movement_bytes=220200960 total_nvlink_bytes=1039216584 weight_to_token_ratio=0.268861 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.094967711e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64178 payload_rows_remote=55986 dup_rows=1358 dup_rows_remote=632 dedup_ratio=0.020721 router_tokens_per_expert=[0, 0, 361, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4070, 0, 10, 137, 0, 0, 0, 5, 0, 3843, 0, 0, 0, 0, 123, 4096, 880, 0, 0, 0, 2858, 4094, 0, 0, 0, 0, 4094, 0, 0] payload_rows_per_dst=[8192, 8192, 8188, 8192, 8170, 7686, 7372, 8186] dup_rows_per_dst=[726, 2, 0, 6, 264, 10, 350, 0] combine_rows_per_src=[8192, 8192, 8188, 8192, 8170, 7686, 7372, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63812 payload_rows_remote=55624 dup_rows=1724 dup_rows_remote=1318 dedup_ratio=0.026306 router_tokens_per_expert=[0, 0, 501, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 4081, 0, 0, 216, 0, 0, 0, 0, 0, 3843, 0, 0, 0, 0, 334, 4096, 756, 0, 0, 0, 2538, 4096, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8188, 7686, 6978, 8192] dup_rows_per_dst=[1002, 0, 0, 38, 406, 0, 278, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8188, 7686, 6978, 8192] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64344 payload_rows_remote=56700 dup_rows=1192 dup_rows_remote=1192 dedup_ratio=0.018188 router_tokens_per_expert=[0, 0, 270, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 4081, 0, 0, 105, 0, 0, 0, 0, 0, 3822, 0, 0, 0, 0, 170, 4096, 761, 0, 0, 0, 3067, 4096, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8184, 7644, 7556, 8192] dup_rows_per_dst=[540, 0, 0, 24, 188, 0, 440, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8184, 7644, 7556, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64626 payload_rows_remote=56434 dup_rows=910 dup_rows_remote=368 dedup_ratio=0.013885 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 179, 0, 0, 2, 0, 0, 269, 4096, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3680, 0, 0, 4057, 0, 0] payload_rows_per_dst=[8114, 8192, 8192, 8192, 8192, 8192, 8192, 7360] dup_rows_per_dst=[358, 542, 2, 6, 0, 2, 0, 0] combine_rows_per_src=[8114, 8192, 8192, 8192, 8192, 8192, 8192, 7360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=405207040 dispatch_meta_bytes=229376 combine_payload_bytes=405207040 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=405436416 ingress_bytes=735508480 link_bytes=735508480 token_bytes=810643456 weight_movement_bytes=330301440 total_nvlink_bytes=1140944896 weight_to_token_ratio=0.407456 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.279170217e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=385953792 dispatch_meta_bytes=216136 combine_payload_bytes=385953792 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=386169928 ingress_bytes=606154752 link_bytes=606154752 token_bytes=772123720 weight_movement_bytes=220200960 total_nvlink_bytes=992324680 weight_to_token_ratio=0.285189 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31104 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.054202809e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64050 payload_rows_remote=56858 dup_rows=1486 dup_rows_remote=1174 dedup_ratio=0.022675 router_tokens_per_expert=[0, 0, 471, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 4081, 0, 0, 115, 0, 0, 0, 0, 0, 3859, 0, 0, 0, 0, 136, 4096, 818, 0, 0, 0, 2798, 4096, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8180, 7718, 7192, 8192] dup_rows_per_dst=[942, 0, 0, 20, 212, 0, 312, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8180, 7718, 7192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=401307648 dispatch_meta_bytes=226472 combine_payload_bytes=401307648 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=401534120 ingress_bytes=841709568 link_bytes=841709568 token_bytes=802841768 weight_movement_bytes=440401920 total_nvlink_bytes=1243243688 weight_to_token_ratio=0.548554 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=29952 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.463871376e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=398712832 dispatch_meta_bytes=227768 combine_payload_bytes=398712832 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=398940600 ingress_bytes=618913792 link_bytes=618913792 token_bytes=797653432 weight_movement_bytes=220200960 total_nvlink_bytes=1017854392 weight_to_token_ratio=0.276061 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=30848 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.076392878e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=406425600 dispatch_meta_bytes=231568 combine_payload_bytes=406425600 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=406657168 ingress_bytes=516526080 link_bytes=516526080 token_bytes=813082768 weight_movement_bytes=110100480 total_nvlink_bytes=923183248 weight_to_token_ratio=0.135411 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=30720 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.983238071e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=404518912 dispatch_meta_bytes=227208 combine_payload_bytes=404518912 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=404746120 ingress_bytes=624719872 link_bytes=624719872 token_bytes=809265032 weight_movement_bytes=220200960 total_nvlink_bytes=1029465992 weight_to_token_ratio=0.272100 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=31872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.086490606e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63798 payload_rows_remote=55606 dup_rows=1738 dup_rows_remote=1684 dedup_ratio=0.026520 router_tokens_per_expert=[0, 0, 474, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 4084, 0, 4, 128, 0, 0, 0, 0, 0, 3730, 0, 0, 0, 0, 263, 4096, 1167, 0, 0, 0, 2411, 4096, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8178, 7460, 7200, 8192] dup_rows_per_dst=[948, 0, 0, 54, 254, 0, 482, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8178, 7460, 7200, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=407558144 dispatch_meta_bytes=232128 combine_payload_bytes=407558144 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=407790272 ingress_bytes=627759104 link_bytes=627759104 token_bytes=815348416 weight_movement_bytes=220200960 total_nvlink_bytes=1035549376 weight_to_token_ratio=0.270070 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=22528 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.091776330e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64068 payload_rows_remote=55876 dup_rows=1468 dup_rows_remote=1466 dedup_ratio=0.022400 router_tokens_per_expert=[0, 0, 349, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 4080, 0, 3, 174, 0, 0, 0, 2, 0, 3789, 0, 0, 0, 0, 232, 4096, 580, 0, 0, 0, 3060, 4094, 0, 0, 0, 0, 4095, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 8192, 8186, 7578, 7348, 8190] dup_rows_per_dst=[698, 0, 2, 40, 328, 4, 396, 0] combine_rows_per_src=[8192, 8190, 8192, 8192, 8186, 7578, 7348, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55438 payload_rows_remote=47250 dup_rows=10098 dup_rows_remote=10084 dedup_ratio=0.154083 router_tokens_per_expert=[0, 0, 174, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 4092, 0, 1, 148, 0, 0, 0, 2, 0, 3903, 0, 0, 0, 0, 127, 4096, 1161, 0, 0, 0, 2664, 4094, 0, 0, 0, 0, 4094, 0, 0] payload_rows_per_dst=[2558, 8192, 8192, 7896, 4510, 8008, 7894, 8188] dup_rows_per_dst=[112, 0, 0, 32, 1892, 3916, 4132, 14] combine_rows_per_src=[2558, 8192, 8192, 7896, 4510, 8008, 7894, 8188] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65082 payload_rows_remote=56898 dup_rows=454 dup_rows_remote=422 dedup_ratio=0.006927 router_tokens_per_expert=[0, 0, 0, 0, 0, 5, 0, 0, 0, 4072, 0, 9, 0, 4082, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4094, 0, 2, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 0, 97, 0, 0, 0, 0, 0, 4096, 4029, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8184, 8184, 8188, 8192, 8192, 8178, 7776, 8188] dup_rows_per_dst=[32, 362, 10, 6, 8, 24, 12, 0] combine_rows_per_src=[8184, 8184, 8188, 8192, 8192, 8178, 7776, 8188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=358758400 dispatch_meta_bytes=200912 combine_payload_bytes=358758400 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=358959312 ingress_bytes=578959360 link_bytes=578959360 token_bytes=717717712 weight_movement_bytes=220200960 total_nvlink_bytes=937918672 weight_to_token_ratio=0.306807 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=28800 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.006905549e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=400848896 dispatch_meta_bytes=228544 combine_payload_bytes=400848896 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=401077440 ingress_bytes=621049856 link_bytes=621049856 token_bytes=801926336 weight_movement_bytes=220200960 total_nvlink_bytes=1022127296 weight_to_token_ratio=0.274590 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.080107845e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=404862976 dispatch_meta_bytes=229336 combine_payload_bytes=404862976 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=405092312 ingress_bytes=735164416 link_bytes=735164416 token_bytes=809955288 weight_movement_bytes=330301440 total_nvlink_bytes=1140256728 weight_to_token_ratio=0.407802 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.278571833e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=394498048 dispatch_meta_bytes=229368 combine_payload_bytes=394498048 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=394727416 ingress_bytes=504598528 link_bytes=504598528 token_bytes=789225464 weight_movement_bytes=110100480 total_nvlink_bytes=899325944 weight_to_token_ratio=0.139504 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.775798324e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63324 payload_rows_remote=55742 dup_rows=2212 dup_rows_remote=2212 dedup_ratio=0.033752 router_tokens_per_expert=[0, 0, 736, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3376, 0, 0, 14, 0, 0, 0, 0, 0, 3791, 0, 0, 0, 0, 6, 4096, 2, 0, 0, 0, 362, 4096, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8002, 6780, 7582, 8192, 8192] dup_rows_per_dst=[1472, 0, 0, 0, 0, 0, 740, 0] combine_rows_per_src=[8192, 8192, 8192, 8002, 6780, 7582, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63088 payload_rows_remote=56546 dup_rows=2448 dup_rows_remote=2448 dedup_ratio=0.037354 router_tokens_per_expert=[0, 0, 971, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3266, 0, 0, 5, 0, 0, 0, 0, 0, 3712, 0, 0, 0, 0, 20, 4096, 1, 0, 0, 0, 232, 4096, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8162, 6542, 7424, 8192, 8192] dup_rows_per_dst=[1942, 0, 0, 0, 0, 0, 506, 0] combine_rows_per_src=[8192, 8192, 8192, 8162, 6542, 7424, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60282 payload_rows_remote=52090 dup_rows=5254 dup_rows_remote=4908 dedup_ratio=0.080170 router_tokens_per_expert=[0, 0, 853, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3535, 0, 0, 7, 0, 0, 0, 0, 0, 3700, 0, 0, 0, 0, 4, 4096, 2, 0, 0, 0, 167, 4096, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[4416, 8192, 8192, 8040, 7658, 7400, 8192, 8192] dup_rows_per_dst=[606, 0, 0, 830, 3472, 0, 346, 0] combine_rows_per_src=[4416, 8192, 8192, 8040, 7658, 7400, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=407844864 dispatch_meta_bytes=229280 combine_payload_bytes=407844864 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=408074144 ingress_bytes=628045824 link_bytes=628045824 token_bytes=815919008 weight_movement_bytes=220200960 total_nvlink_bytes=1036119968 weight_to_token_ratio=0.269881 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.092274984e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=399558656 dispatch_meta_bytes=231816 combine_payload_bytes=399558656 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=399790472 ingress_bytes=509659136 link_bytes=509659136 token_bytes=799349128 weight_movement_bytes=110100480 total_nvlink_bytes=909449608 weight_to_token_ratio=0.137738 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=29952 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.863810620e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=405321728 dispatch_meta_bytes=235976 combine_payload_bytes=405321728 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=405557704 ingress_bytes=515422208 link_bytes=515422208 token_bytes=810879432 weight_movement_bytes=110100480 total_nvlink_bytes=920979912 weight_to_token_ratio=0.135779 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=27520 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.964039921e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57650 payload_rows_remote=49462 dup_rows=7886 dup_rows_remote=7862 dedup_ratio=0.120331 router_tokens_per_expert=[0, 0, 481, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3624, 0, 1, 0, 0, 0, 0, 2, 0, 3881, 0, 0, 0, 0, 5, 4096, 19, 0, 0, 0, 205, 4094, 0, 0, 0, 0, 4094, 0, 0] payload_rows_per_dst=[962, 8190, 8192, 8146, 8018, 7762, 8192, 8188] dup_rows_per_dst=[0, 0, 0, 0, 5600, 4, 2258, 24] combine_rows_per_src=[962, 8190, 8192, 8146, 8018, 7762, 8192, 8188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=373381120 dispatch_meta_bytes=227992 combine_payload_bytes=373381120 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=373609112 ingress_bytes=593582080 link_bytes=593582080 token_bytes=746990232 weight_movement_bytes=220200960 total_nvlink_bytes=967191192 weight_to_token_ratio=0.294784 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=29952 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.032336865e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63788 payload_rows_remote=55598 dup_rows=1748 dup_rows_remote=1748 dedup_ratio=0.026672 router_tokens_per_expert=[0, 0, 665, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3627, 0, 2, 9, 0, 0, 0, 3, 0, 3744, 0, 0, 0, 0, 9, 4096, 1, 0, 0, 0, 195, 4095, 0, 0, 0, 0, 4095, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 8068, 7276, 7488, 8192, 8190] dup_rows_per_dst=[1330, 0, 2, 0, 0, 6, 410, 0] combine_rows_per_src=[8192, 8190, 8192, 8068, 7276, 7488, 8192, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60676 payload_rows_remote=52484 dup_rows=4860 dup_rows_remote=4860 dedup_ratio=0.074158 router_tokens_per_expert=[54, 0, 0, 0, 0, 4094, 0, 0, 0, 2373, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 5, 0, 0, 1667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8190, 8192, 3334, 8192] dup_rows_per_dst=[104, 4746, 0, 0, 0, 10, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8190, 8192, 3334, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63350 payload_rows_remote=55228 dup_rows=2186 dup_rows_remote=2186 dedup_ratio=0.033356 router_tokens_per_expert=[0, 0, 950, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3447, 0, 3, 0, 0, 0, 0, 0, 0, 3686, 0, 0, 0, 0, 18, 4096, 0, 0, 0, 0, 125, 4094, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8122, 6900, 7372, 8192, 8188] dup_rows_per_dst=[1900, 0, 0, 0, 0, 0, 286, 0] combine_rows_per_src=[8192, 8192, 8192, 8122, 6900, 7372, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=398583808 dispatch_meta_bytes=229160 combine_payload_bytes=398583808 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=398812968 ingress_bytes=508684288 link_bytes=508684288 token_bytes=797396776 weight_movement_bytes=110100480 total_nvlink_bytes=907497256 weight_to_token_ratio=0.138075 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.846856410e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62146 payload_rows_remote=53954 dup_rows=3390 dup_rows_remote=196 dedup_ratio=0.051727 router_tokens_per_expert=[331, 0, 0, 0, 0, 4082, 0, 0, 0, 100, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 5642, 8192, 8190, 8192, 8188, 7358, 8192] dup_rows_per_dst=[3194, 192, 0, 0, 4, 0, 0, 0] combine_rows_per_src=[8192, 5642, 8192, 8190, 8192, 8188, 7358, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64936 payload_rows_remote=56744 dup_rows=600 dup_rows_remote=600 dedup_ratio=0.009155 router_tokens_per_expert=[185, 0, 0, 0, 0, 4096, 0, 0, 0, 115, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 7592, 8192] dup_rows_per_dst=[370, 230, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 7592, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64258 payload_rows_remote=56066 dup_rows=1278 dup_rows_remote=1278 dedup_ratio=0.019501 router_tokens_per_expert=[387, 0, 0, 0, 0, 4096, 0, 0, 0, 252, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 6914, 8192] dup_rows_per_dst=[774, 504, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 6914, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=400519168 dispatch_meta_bytes=229368 combine_payload_bytes=400519168 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=400748536 ingress_bytes=510619648 link_bytes=510619648 token_bytes=801267704 weight_movement_bytes=110100480 total_nvlink_bytes=911368184 weight_to_token_ratio=0.137408 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.880515504e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64364 payload_rows_remote=56172 dup_rows=1172 dup_rows_remote=1172 dedup_ratio=0.017883 router_tokens_per_expert=[0, 0, 333, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 4081, 0, 6, 111, 0, 0, 0, 4, 0, 3909, 0, 0, 0, 0, 142, 4096, 647, 0, 0, 0, 3014, 4096, 0, 0, 0, 0, 4095, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8184, 7818, 7404, 8190] dup_rows_per_dst=[666, 0, 2, 82, 212, 8, 202, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8184, 7818, 7404, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=338688000 dispatch_meta_bytes=229336 combine_payload_bytes=338688000 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=338917336 ingress_bytes=668989440 link_bytes=668989440 token_bytes=677605336 weight_movement_bytes=330301440 total_nvlink_bytes=1007906776 weight_to_token_ratio=0.487454 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.163482666e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64972 payload_rows_remote=57344 dup_rows=564 dup_rows_remote=564 dedup_ratio=0.008606 router_tokens_per_expert=[85, 0, 0, 0, 0, 4096, 0, 0, 0, 197, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 7628, 8192] dup_rows_per_dst=[170, 394, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 7628, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=386742272 dispatch_meta_bytes=216600 combine_payload_bytes=386742272 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=386958872 ingress_bytes=606943232 link_bytes=606943232 token_bytes=773701144 weight_movement_bytes=220200960 total_nvlink_bytes=993902104 weight_to_token_ratio=0.284607 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=30208 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.055574105e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=406740992 dispatch_meta_bytes=229376 combine_payload_bytes=406740992 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=406970368 ingress_bytes=626941952 link_bytes=626941952 token_bytes=813711360 weight_movement_bytes=220200960 total_nvlink_bytes=1033912320 weight_to_token_ratio=0.270613 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.090355169e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=401881088 dispatch_meta_bytes=229376 combine_payload_bytes=401881088 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=402110464 ingress_bytes=511981568 link_bytes=511981568 token_bytes=803991552 weight_movement_bytes=110100480 total_nvlink_bytes=914092032 weight_to_token_ratio=0.136942 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.904201533e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=402640896 dispatch_meta_bytes=229376 combine_payload_bytes=402640896 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=402870272 ingress_bytes=622841856 link_bytes=622841856 token_bytes=805511168 weight_movement_bytes=220200960 total_nvlink_bytes=1025712128 weight_to_token_ratio=0.273368 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.083224427e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=411041792 dispatch_meta_bytes=231632 combine_payload_bytes=411041792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=411273424 ingress_bytes=521142272 link_bytes=521142272 token_bytes=822315216 weight_movement_bytes=110100480 total_nvlink_bytes=932415696 weight_to_token_ratio=0.133891 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=30208 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.063521242e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64908 payload_rows_remote=56716 dup_rows=628 dup_rows_remote=628 dedup_ratio=0.009583 router_tokens_per_expert=[94, 0, 0, 0, 0, 4095, 0, 0, 0, 221, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 7564, 8192] dup_rows_per_dst=[186, 442, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 7564, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7201388880 prefetch_bytes=880803840 reduce_bytes=1981808640 weight_movement_bytes=2862612480 egress_bytes=3601776976 ingress_bytes=6462224384 link_bytes=6462224384 total_nvlink_bytes=10064001360 weight_to_token_ratio=0.397508 prefetch_share_of_ingress=0.136300 dedup_ratio=0.111319 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=18 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050735 gemm_inner_pad_upper_bound=5969 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.123887106e-04 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7611246800 prefetch_bytes=990904320 reduce_bytes=880803840 weight_movement_bytes=1871708160 egress_bytes=3806773456 ingress_bytes=5676181504 link_bytes=5676181504 total_nvlink_bytes=9482954960 weight_to_token_ratio=0.245913 prefetch_share_of_ingress=0.174572 dedup_ratio=0.074231 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=8 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050368 gemm_inner_pad_upper_bound=5969 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.871813207e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7516452216 prefetch_bytes=880803840 reduce_bytes=550502400 weight_movement_bytes=1431306240 egress_bytes=3759373688 ingress_bytes=5188384768 link_bytes=5188384768 total_nvlink_bytes=8947758456 weight_to_token_ratio=0.190423 prefetch_share_of_ingress=0.169765 dedup_ratio=0.081125 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=5 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.049265 gemm_inner_pad_upper_bound=6350 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.023454454e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65212 payload_rows_remote=57020 dup_rows=324 dup_rows_remote=324 dedup_ratio=0.004944 router_tokens_per_expert=[43, 0, 0, 0, 0, 4093, 0, 0, 0, 122, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8190, 8192, 7870, 8192] dup_rows_per_dst=[80, 244, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8190, 8192, 7870, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60182 payload_rows_remote=51990 dup_rows=5354 dup_rows_remote=5354 dedup_ratio=0.081696 router_tokens_per_expert=[106, 0, 0, 0, 0, 4094, 0, 0, 0, 214, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 3778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[3060, 8188, 8190, 8188, 8192, 8192, 7980, 8192] dup_rows_per_dst=[122, 428, 0, 6, 0, 4, 4794, 0] combine_rows_per_src=[3060, 8188, 8190, 8188, 8192, 8192, 7980, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7495214400 prefetch_bytes=990904320 reduce_bytes=990904320 weight_movement_bytes=1981808640 egress_bytes=3748730176 ingress_bytes=5728292864 link_bytes=5728522240 total_nvlink_bytes=9477023040 weight_to_token_ratio=0.264410 prefetch_share_of_ingress=0.172984 dedup_ratio=0.082483 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=9 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.052206 gemm_inner_pad_upper_bound=5842 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.962842355e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64288 payload_rows_remote=56200 dup_rows=1248 dup_rows_remote=372 dedup_ratio=0.019043 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 785, 0, 0, 1, 0, 0, 59, 4096, 0, 0, 0, 11, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3639, 0, 0, 3697, 0, 0] payload_rows_per_dst=[8088, 8192, 7992, 8166, 8192, 8188, 8192, 7278] dup_rows_per_dst=[876, 358, 14, 0, 0, 0, 0, 0] combine_rows_per_src=[8088, 8192, 7992, 8166, 8192, 8188, 8192, 7278] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=354543616 dispatch_meta_bytes=229296 combine_payload_bytes=354543616 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=354772912 ingress_bytes=684845056 link_bytes=684845056 token_bytes=709316528 weight_movement_bytes=330301440 total_nvlink_bytes=1039617968 weight_to_token_ratio=0.465662 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.191058190e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61512 payload_rows_remote=53320 dup_rows=4024 dup_rows_remote=4024 dedup_ratio=0.061401 router_tokens_per_expert=[116, 0, 0, 0, 0, 4096, 0, 0, 0, 2084, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 6, 0, 0, 1890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 4168, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[232, 0, 0, 0, 0, 12, 3780, 0] combine_rows_per_src=[8192, 4168, 8192, 8192, 8192, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61270 payload_rows_remote=53078 dup_rows=4266 dup_rows_remote=4208 dedup_ratio=0.065094 router_tokens_per_expert=[214, 0, 0, 0, 0, 4096, 0, 0, 0, 1963, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 29, 0, 0, 1890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 3926, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[428, 0, 0, 0, 0, 58, 3780, 0] combine_rows_per_src=[8192, 3926, 8192, 8192, 8192, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=398526464 dispatch_meta_bytes=229384 combine_payload_bytes=398526464 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=398755848 ingress_bytes=618727424 link_bytes=618727424 token_bytes=797282312 weight_movement_bytes=220200960 total_nvlink_bytes=1017483272 weight_to_token_ratio=0.276189 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.076068754e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=376205312 dispatch_meta_bytes=229376 combine_payload_bytes=376205312 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=376434688 ingress_bytes=486305792 link_bytes=486305792 token_bytes=752640000 weight_movement_bytes=110100480 total_nvlink_bytes=862740480 weight_to_token_ratio=0.146286 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.457657558e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=395874304 dispatch_meta_bytes=229656 combine_payload_bytes=395874304 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=396103960 ingress_bytes=505974784 link_bytes=505974784 token_bytes=791978264 weight_movement_bytes=110100480 total_nvlink_bytes=902078744 weight_to_token_ratio=0.139020 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.799733679e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=402841600 dispatch_meta_bytes=226288 combine_payload_bytes=402841600 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=403067888 ingress_bytes=512942080 link_bytes=512942080 token_bytes=805909488 weight_movement_bytes=110100480 total_nvlink_bytes=916009968 weight_to_token_ratio=0.136616 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=26496 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.920906416e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61976 payload_rows_remote=53784 dup_rows=3560 dup_rows_remote=244 dedup_ratio=0.054321 router_tokens_per_expert=[115, 0, 0, 0, 0, 4096, 0, 0, 0, 2316, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 6, 0, 0, 1658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 4632, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[230, 0, 0, 0, 0, 14, 3316, 0] combine_rows_per_src=[8192, 4632, 8192, 8192, 8192, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=382197760 dispatch_meta_bytes=229376 combine_payload_bytes=382197760 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=382427136 ingress_bytes=602398720 link_bytes=602398720 token_bytes=764624896 weight_movement_bytes=220200960 total_nvlink_bytes=984825856 weight_to_token_ratio=0.287986 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.047670452e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=380463104 dispatch_meta_bytes=229144 combine_payload_bytes=380463104 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=380692248 ingress_bytes=710764544 link_bytes=710764544 token_bytes=761155352 weight_movement_bytes=330301440 total_nvlink_bytes=1091456792 weight_to_token_ratio=0.433947 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.236136443e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7376469056 prefetch_bytes=1101004800 reduce_bytes=880803840 weight_movement_bytes=1981808640 egress_bytes=3689378880 ingress_bytes=5668898816 link_bytes=5668898816 total_nvlink_bytes=9358277696 weight_to_token_ratio=0.268666 prefetch_share_of_ingress=0.194218 dedup_ratio=0.093460 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=8 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.051103 gemm_inner_pad_upper_bound=5842 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.859147415e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59746 payload_rows_remote=51554 dup_rows=5790 dup_rows_remote=5790 dedup_ratio=0.088348 router_tokens_per_expert=[93, 0, 0, 0, 0, 4094, 0, 0, 0, 1855, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2, 0, 0, 4096, 0, 0, 5, 0, 0, 2145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5876, 4724, 8190, 8188, 8192, 8192, 8192, 8192] dup_rows_per_dst=[160, 1328, 0, 2, 0, 14, 4286, 0] combine_rows_per_src=[5876, 4724, 8190, 8188, 8192, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7548192752 prefetch_bytes=880803840 reduce_bytes=770703360 weight_movement_bytes=1651507200 egress_bytes=3775244272 ingress_bytes=5424455680 link_bytes=5424455680 total_nvlink_bytes=9199699952 weight_to_token_ratio=0.218795 prefetch_share_of_ingress=0.162376 dedup_ratio=0.078448 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=7 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048897 gemm_inner_pad_upper_bound=6604 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.434020597e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7596301544 prefetch_bytes=990904320 reduce_bytes=440401920 weight_movement_bytes=1431306240 egress_bytes=3799297256 ingress_bytes=5228310528 link_bytes=5228310528 total_nvlink_bytes=9027607784 weight_to_token_ratio=0.188421 prefetch_share_of_ingress=0.189527 dedup_ratio=0.075247 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=4 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050368 gemm_inner_pad_upper_bound=5969 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.092891918e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=385523712 dispatch_meta_bytes=216112 combine_payload_bytes=385523712 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=385739824 ingress_bytes=495624192 link_bytes=495624192 token_bytes=771263536 weight_movement_bytes=110100480 total_nvlink_bytes=881364016 weight_to_token_ratio=0.142753 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=17536 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.619719861e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62176 payload_rows_remote=53984 dup_rows=3360 dup_rows_remote=250 dedup_ratio=0.051270 router_tokens_per_expert=[118, 0, 0, 0, 0, 4091, 0, 0, 0, 1555, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 3, 0, 0, 4095, 0, 0, 9, 0, 0, 2417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8190, 8192, 4834, 8192] dup_rows_per_dst=[226, 3110, 2, 0, 0, 22, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8190, 8192, 4834, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61010 payload_rows_remote=52818 dup_rows=4526 dup_rows_remote=4526 dedup_ratio=0.069061 router_tokens_per_expert=[49, 0, 0, 0, 0, 4095, 0, 0, 0, 2702, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 25, 0, 0, 1321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7632, 8192, 8192, 8192, 8192, 4226, 8192] dup_rows_per_dst=[96, 3828, 0, 0, 0, 50, 552, 0] combine_rows_per_src=[8192, 7632, 8192, 8192, 8192, 8192, 4226, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=369539072 dispatch_meta_bytes=229376 combine_payload_bytes=369539072 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=369768448 ingress_bytes=479639552 link_bytes=479639552 token_bytes=739307520 weight_movement_bytes=110100480 total_nvlink_bytes=849408000 weight_to_token_ratio=0.148924 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.341720681e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7347395336 prefetch_bytes=990904320 reduce_bytes=1211105280 weight_movement_bytes=2202009600 egress_bytes=3674813192 ingress_bytes=5874591744 link_bytes=5874821120 total_nvlink_bytes=9549404936 weight_to_token_ratio=0.299699 prefetch_share_of_ingress=0.168676 dedup_ratio=0.097021 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=11 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.051471 gemm_inner_pad_upper_bound=5969 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.021728017e-04 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58226 payload_rows_remote=50034 dup_rows=7310 dup_rows_remote=5398 dedup_ratio=0.111542 router_tokens_per_expert=[0, 0, 954, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3217, 0, 8, 2, 0, 0, 0, 5, 0, 3711, 0, 0, 0, 0, 2, 4096, 13, 0, 0, 0, 352, 4094, 0, 0, 0, 0, 4094, 0, 0] payload_rows_per_dst=[8192, 8192, 8188, 8052, 6454, 7908, 3056, 8184] dup_rows_per_dst=[1912, 4, 0, 0, 0, 5262, 132, 0] combine_rows_per_src=[8192, 8192, 8188, 8052, 6454, 7908, 3056, 8184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=386957312 dispatch_meta_bytes=216936 combine_payload_bytes=386957312 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=387174248 ingress_bytes=607158272 link_bytes=607158272 token_bytes=774131560 weight_movement_bytes=220200960 total_nvlink_bytes=994332520 weight_to_token_ratio=0.284449 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=16384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.055948095e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=378599424 dispatch_meta_bytes=229376 combine_payload_bytes=378599424 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=378828800 ingress_bytes=598800384 link_bytes=598800384 token_bytes=757428224 weight_movement_bytes=220200960 total_nvlink_bytes=977629184 weight_to_token_ratio=0.290722 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.041412354e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6816065272 prefetch_bytes=1101004800 reduce_bytes=660602880 weight_movement_bytes=1761607680 egress_bytes=3409129208 ingress_bytes=5168543744 link_bytes=5168543744 total_nvlink_bytes=8577672952 weight_to_token_ratio=0.258449 prefetch_share_of_ingress=0.213020 dedup_ratio=0.153568 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=6 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050368 gemm_inner_pad_upper_bound=6096 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.988947650e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=358643712 dispatch_meta_bytes=221728 combine_payload_bytes=358643712 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=358865440 ingress_bytes=909146112 link_bytes=909146112 token_bytes=717509152 weight_movement_bytes=550502400 total_nvlink_bytes=1268011552 weight_to_token_ratio=0.767241 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=26240 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.581154618e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7337923792 prefetch_bytes=1211105280 reduce_bytes=1101004800 weight_movement_bytes=2312110080 egress_bytes=3670086864 ingress_bytes=5979947008 link_bytes=5979947008 total_nvlink_bytes=9650033872 weight_to_token_ratio=0.315091 prefetch_share_of_ingress=0.202528 dedup_ratio=0.098682 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=10 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048897 gemm_inner_pad_upper_bound=6731 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.040011138e-04 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7559182320 prefetch_bytes=880803840 reduce_bytes=770703360 weight_movement_bytes=1651507200 egress_bytes=3780743152 ingress_bytes=5429946368 link_bytes=5429946368 total_nvlink_bytes=9210689520 weight_to_token_ratio=0.218477 prefetch_share_of_ingress=0.162212 dedup_ratio=0.081226 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=7 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.049632 gemm_inner_pad_upper_bound=6477 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.443569807e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61810 payload_rows_remote=53618 dup_rows=3726 dup_rows_remote=2886 dedup_ratio=0.056854 router_tokens_per_expert=[341, 0, 0, 0, 0, 4083, 0, 0, 0, 1432, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 3, 0, 0, 4095, 0, 0, 5, 0, 0, 2330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8188, 8192, 8002, 4660, 8192] dup_rows_per_dst=[840, 2866, 0, 6, 4, 10, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8188, 8192, 8002, 4660, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=384333824 dispatch_meta_bytes=226016 combine_payload_bytes=384333824 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=384559840 ingress_bytes=604534784 link_bytes=604534784 token_bytes=768893664 weight_movement_bytes=220200960 total_nvlink_bytes=989094624 weight_to_token_ratio=0.286387 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=31616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.051385418e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7158730240 prefetch_bytes=990904320 reduce_bytes=2312110080 weight_movement_bytes=3303014400 egress_bytes=3580450304 ingress_bytes=6881294336 link_bytes=6881294336 total_nvlink_bytes=10461744640 weight_to_token_ratio=0.461397 prefetch_share_of_ingress=0.144000 dedup_ratio=0.115076 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=21 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050368 gemm_inner_pad_upper_bound=6096 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.196770263e-04 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none - [2026-08-26 15:11:34] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 367.0 | throughput per GPU (TFLOP/s/GPU): 336.5 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.895617E+00 | loss scale: 1.0 | grad norm: 4.633 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44482 payload_rows_remote=39270 dup_rows=21054 dup_rows_remote=18682 dedup_ratio=0.321259 router_tokens_per_expert=[1054, 1317, 1367, 984, 118, 1101, 381, 48, 387, 1293, 1271, 327, 3, 1292, 35, 1048, 992, 531, 33, 57, 1127, 343, 84, 545, 213, 652, 506, 846, 464, 137, 377, 1, 565, 349, 877, 44, 447, 0, 1193, 140, 577, 479, 427, 1170, 5, 404, 76, 423, 209, 181, 171, 771, 255, 711, 40, 1182, 454, 632, 679, 265, 162, 622, 303, 21] payload_rows_per_dst=[5080, 5422, 6048, 5586, 5114, 5212, 6320, 5700] dup_rows_per_dst=[2456, 1548, 3060, 2914, 2188, 2372, 3306, 3210] combine_rows_per_src=[5080, 5422, 6048, 5586, 5114, 5212, 6320, 5700] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44650 payload_rows_remote=39742 dup_rows=20886 dup_rows_remote=18606 dedup_ratio=0.318695 router_tokens_per_expert=[1340, 1322, 842, 965, 142, 1094, 463, 91, 281, 1272, 1269, 339, 1, 1268, 30, 1345, 796, 690, 33, 23, 875, 243, 564, 703, 334, 761, 382, 923, 304, 179, 393, 1, 534, 508, 778, 5, 477, 0, 1127, 165, 483, 520, 362, 817, 0, 716, 57, 526, 138, 93, 266, 297, 174, 980, 124, 775, 414, 872, 563, 447, 583, 464, 234, 1] payload_rows_per_dst=[5122, 6252, 5684, 5894, 4908, 4968, 5818, 6004] dup_rows_per_dst=[2072, 2814, 2170, 3340, 2280, 1994, 2420, 3796] combine_rows_per_src=[5122, 6252, 5684, 5894, 4908, 4968, 5818, 6004] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43670 payload_rows_remote=36604 dup_rows=21866 dup_rows_remote=15842 dedup_ratio=0.333649 router_tokens_per_expert=[1431, 1234, 953, 713, 67, 1013, 231, 85, 720, 1556, 1006, 327, 7, 1043, 34, 1852, 1096, 824, 29, 39, 817, 252, 186, 824, 332, 598, 390, 568, 241, 113, 672, 1, 530, 420, 1425, 6, 396, 8, 806, 147, 333, 784, 677, 1106, 10, 783, 26, 491, 293, 209, 201, 379, 133, 848, 92, 674, 329, 735, 546, 299, 241, 452, 135, 0] payload_rows_per_dst=[6634, 7066, 5836, 4536, 5200, 5734, 4424, 4240] dup_rows_per_dst=[4820, 6024, 2298, 1294, 2276, 2686, 1234, 1234] combine_rows_per_src=[6634, 7066, 5836, 4536, 5200, 5734, 4424, 4240] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43228 payload_rows_remote=37106 dup_rows=22308 dup_rows_remote=18842 dedup_ratio=0.340393 router_tokens_per_expert=[1355, 1326, 1267, 613, 37, 1115, 344, 92, 466, 1816, 962, 355, 2, 720, 51, 1431, 868, 828, 11, 36, 1190, 231, 648, 655, 295, 588, 588, 688, 370, 242, 318, 7, 675, 489, 728, 10, 634, 0, 820, 99, 223, 1009, 578, 622, 2, 555, 48, 299, 139, 205, 214, 383, 143, 1106, 148, 656, 389, 886, 1116, 356, 263, 180, 277, 1] payload_rows_per_dst=[3440, 3252, 6414, 5664, 5930, 6044, 6362, 6122] dup_rows_per_dst=[962, 420, 3960, 3238, 3842, 3162, 3258, 3466] combine_rows_per_src=[3440, 3252, 6414, 5664, 5930, 6044, 6362, 6122] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44588 payload_rows_remote=38684 dup_rows=20948 dup_rows_remote=17584 dedup_ratio=0.319641 router_tokens_per_expert=[1580, 1010, 1286, 1007, 353, 344, 470, 30, 265, 1564, 966, 181, 3, 1349, 10, 1038, 1041, 721, 14, 28, 620, 340, 242, 861, 131, 1067, 339, 624, 261, 119, 505, 8, 633, 253, 1135, 9, 695, 0, 937, 183, 303, 732, 506, 1081, 1, 761, 56, 625, 113, 198, 161, 425, 324, 947, 69, 600, 671, 1213, 413, 202, 323, 591, 231, 0] payload_rows_per_dst=[4926, 5846, 5622, 5904, 5210, 5658, 5662, 5760] dup_rows_per_dst=[2054, 2426, 2112, 3364, 2480, 2472, 2492, 3548] combine_rows_per_src=[4926, 5846, 5622, 5904, 5210, 5658, 5662, 5760] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43982 payload_rows_remote=38848 dup_rows=21554 dup_rows_remote=20042 dedup_ratio=0.328888 router_tokens_per_expert=[1535, 882, 1106, 835, 251, 986, 500, 156, 242, 1430, 810, 357, 4, 1205, 32, 965, 662, 951, 8, 36, 635, 257, 331, 443, 427, 967, 283, 582, 141, 246, 649, 4, 713, 493, 664, 17, 668, 3, 965, 192, 280, 861, 307, 1312, 2, 924, 31, 509, 82, 113, 517, 468, 402, 987, 103, 561, 348, 707, 895, 172, 306, 1024, 224, 0] payload_rows_per_dst=[6408, 6302, 5134, 5382, 4906, 5744, 4816, 5290] dup_rows_per_dst=[4768, 3788, 1512, 2168, 2524, 2708, 1650, 2436] combine_rows_per_src=[6408, 6302, 5134, 5382, 4906, 5744, 4816, 5290] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43090 payload_rows_remote=36578 dup_rows=22446 dup_rows_remote=18290 dedup_ratio=0.342499 router_tokens_per_expert=[1579, 1408, 945, 682, 336, 828, 439, 55, 230, 1396, 1171, 388, 3, 929, 11, 1544, 825, 353, 12, 40, 731, 420, 499, 480, 169, 1086, 356, 623, 276, 248, 568, 33, 841, 300, 1013, 2, 537, 2, 1048, 74, 185, 653, 413, 581, 0, 957, 80, 370, 195, 186, 273, 760, 268, 961, 79, 1216, 310, 1105, 420, 156, 467, 388, 262, 3] payload_rows_per_dst=[3544, 3256, 5830, 6214, 6244, 5600, 6512, 5890] dup_rows_per_dst=[1136, 350, 2748, 3662, 4478, 2768, 4156, 3148] combine_rows_per_src=[3544, 3256, 5830, 6214, 6244, 5600, 6512, 5890] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44184 payload_rows_remote=37390 dup_rows=21352 dup_rows_remote=16032 dedup_ratio=0.325806 router_tokens_per_expert=[1391, 1145, 1126, 563, 156, 641, 264, 99, 315, 1074, 1095, 376, 16, 1694, 21, 1466, 1140, 289, 16, 58, 875, 268, 451, 457, 395, 807, 283, 750, 197, 252, 496, 8, 752, 361, 1034, 20, 800, 0, 713, 298, 599, 999, 388, 1208, 9, 718, 11, 275, 57, 186, 263, 376, 351, 1349, 74, 897, 293, 685, 665, 394, 230, 464, 114, 1] payload_rows_per_dst=[6794, 6696, 5364, 4780, 5164, 5732, 5408, 4246] dup_rows_per_dst=[5320, 4074, 1744, 1596, 2792, 2682, 1698, 1446] combine_rows_per_src=[6794, 6696, 5364, 4780, 5164, 5732, 5408, 4246] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=406540288 dispatch_meta_bytes=229376 combine_payload_bytes=406540288 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=406769664 ingress_bytes=626741248 link_bytes=626741248 token_bytes=813309952 weight_movement_bytes=220200960 total_nvlink_bytes=1033510912 weight_to_token_ratio=0.270747 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.090006111e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=408719360 dispatch_meta_bytes=229376 combine_payload_bytes=408719360 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=408948736 ingress_bytes=518819840 link_bytes=518819840 token_bytes=817668096 weight_movement_bytes=110100480 total_nvlink_bytes=927768576 weight_to_token_ratio=0.134652 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.023130330e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=372664320 dispatch_meta_bytes=229376 combine_payload_bytes=372664320 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=372893696 ingress_bytes=482764800 link_bytes=482764800 token_bytes=745558016 weight_movement_bytes=110100480 total_nvlink_bytes=855658496 weight_to_token_ratio=0.147675 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.396073884e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65178 payload_rows_remote=56986 dup_rows=358 dup_rows_remote=238 dedup_ratio=0.005463 router_tokens_per_expert=[123, 0, 0, 0, 0, 4091, 0, 0, 0, 60, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8190, 8192, 7836, 8192] dup_rows_per_dst=[236, 120, 2, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8190, 8192, 7836, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7628376840 prefetch_bytes=880803840 reduce_bytes=770703360 weight_movement_bytes=1651507200 egress_bytes=3815330568 ingress_bytes=5464553472 link_bytes=5464553472 total_nvlink_bytes=9279884040 weight_to_token_ratio=0.216495 prefetch_share_of_ingress=0.161185 dedup_ratio=0.073352 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=7 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.049265 gemm_inner_pad_upper_bound=6477 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.503757252e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7473099016 prefetch_bytes=990904320 reduce_bytes=660602880 weight_movement_bytes=1651507200 egress_bytes=3737696520 ingress_bytes=5386909696 link_bytes=5386909696 total_nvlink_bytes=9124606216 weight_to_token_ratio=0.220994 prefetch_share_of_ingress=0.183947 dedup_ratio=0.084195 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=6 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.051103 gemm_inner_pad_upper_bound=5715 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.368721956e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7058789216 prefetch_bytes=990904320 reduce_bytes=770703360 weight_movement_bytes=1761607680 egress_bytes=3530513248 ingress_bytes=5289883648 link_bytes=5289883648 total_nvlink_bytes=8820396896 weight_to_token_ratio=0.249562 prefetch_share_of_ingress=0.187321 dedup_ratio=0.125833 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=7 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050000 gemm_inner_pad_upper_bound=6223 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.199977700e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=408475648 dispatch_meta_bytes=228896 combine_payload_bytes=408475648 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=408704544 ingress_bytes=628676608 link_bytes=628676608 token_bytes=817180192 weight_movement_bytes=220200960 total_nvlink_bytes=1037381152 weight_to_token_ratio=0.269464 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31488 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.093372021e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7364647800 prefetch_bytes=1101004800 reduce_bytes=1101004800 weight_movement_bytes=2202009600 egress_bytes=3683449720 ingress_bytes=5883207680 link_bytes=5883207680 total_nvlink_bytes=9566657400 weight_to_token_ratio=0.298997 prefetch_share_of_ingress=0.187144 dedup_ratio=0.097836 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=10 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050735 gemm_inner_pad_upper_bound=6350 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.023186578e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none + [2026-08-26 15:59:20] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 364.7 | throughput per GPU (TFLOP/s/GPU): 338.6 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.893831E+00 | loss scale: 1.0 | grad norm: 4.642 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44714 payload_rows_remote=39398 dup_rows=20822 dup_rows_remote=18238 dedup_ratio=0.317719 router_tokens_per_expert=[1087, 1342, 1384, 911, 99, 1056, 391, 42, 408, 1227, 1239, 333, 4, 1262, 34, 1044, 984, 553, 30, 66, 1155, 393, 89, 536, 227, 710, 505, 858, 527, 123, 376, 1, 558, 410, 880, 42, 510, 1, 1227, 158, 519, 386, 381, 1214, 5, 402, 91, 421, 233, 193, 183, 686, 260, 729, 37, 1180, 495, 500, 688, 274, 171, 608, 311, 19] payload_rows_per_dst=[5194, 5422, 5690, 6088, 5172, 5316, 6266, 5566] dup_rows_per_dst=[2572, 1880, 2206, 3250, 2400, 2584, 3190, 2740] combine_rows_per_src=[5194, 5422, 5690, 6088, 5172, 5316, 6266, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44144 payload_rows_remote=38922 dup_rows=21392 dup_rows_remote=19830 dedup_ratio=0.326416 router_tokens_per_expert=[1565, 892, 1136, 758, 203, 953, 525, 148, 246, 1406, 746, 354, 4, 1174, 32, 945, 664, 954, 6, 40, 654, 291, 345, 438, 435, 1034, 282, 578, 160, 222, 644, 7, 700, 542, 680, 18, 723, 2, 988, 221, 254, 740, 287, 1378, 2, 918, 42, 505, 92, 118, 565, 416, 414, 1009, 102, 564, 391, 613, 902, 183, 316, 1004, 238, 0] payload_rows_per_dst=[6646, 6178, 5222, 5088, 5036, 5672, 4870, 5432] dup_rows_per_dst=[5156, 3636, 1562, 1636, 2712, 2580, 1690, 2420] combine_rows_per_src=[6646, 6178, 5222, 5088, 5036, 5672, 4870, 5432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43764 payload_rows_remote=37222 dup_rows=21772 dup_rows_remote=17560 dedup_ratio=0.332214 router_tokens_per_expert=[1582, 1459, 971, 592, 285, 727, 440, 52, 242, 1359, 1131, 402, 4, 895, 15, 1523, 819, 360, 12, 43, 772, 491, 504, 482, 180, 1148, 355, 636, 340, 226, 569, 34, 827, 355, 1000, 3, 611, 2, 1064, 88, 169, 546, 378, 627, 0, 950, 102, 382, 220, 194, 335, 685, 279, 1000, 78, 1227, 348, 892, 423, 179, 514, 374, 264, 2] payload_rows_per_dst=[3948, 3232, 5870, 6384, 5688, 6234, 6542, 5866] dup_rows_per_dst=[1390, 356, 2886, 3510, 3008, 3120, 4212, 3290] combine_rows_per_src=[3948, 3232, 5870, 6384, 5688, 6234, 6542, 5866] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45076 payload_rows_remote=39196 dup_rows=20460 dup_rows_remote=17946 dedup_ratio=0.312195 router_tokens_per_expert=[1604, 1031, 1303, 937, 315, 314, 496, 27, 278, 1513, 916, 185, 3, 1297, 11, 1039, 1035, 740, 14, 30, 630, 396, 244, 846, 137, 1119, 345, 635, 312, 107, 516, 8, 627, 321, 1147, 8, 762, 0, 953, 228, 275, 629, 455, 1122, 1, 761, 74, 629, 118, 210, 162, 367, 339, 986, 75, 582, 757, 1020, 406, 213, 335, 570, 253, 0] payload_rows_per_dst=[4862, 5892, 5692, 5880, 5350, 5510, 5552, 6338] dup_rows_per_dst=[1948, 2452, 2178, 2514, 2742, 2382, 2266, 3978] combine_rows_per_src=[4862, 5892, 5692, 5880, 5350, 5510, 5552, 6338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44066 payload_rows_remote=37870 dup_rows=21470 dup_rows_remote=18214 dedup_ratio=0.327606 router_tokens_per_expert=[1370, 1351, 1295, 513, 30, 1085, 349, 82, 495, 1767, 932, 357, 2, 700, 54, 1415, 855, 847, 10, 47, 1209, 275, 672, 653, 304, 627, 592, 707, 409, 219, 321, 9, 683, 539, 709, 9, 705, 0, 841, 115, 203, 889, 529, 659, 1, 564, 60, 297, 157, 232, 246, 340, 147, 1120, 151, 663, 412, 734, 1103, 380, 271, 178, 276, 2] payload_rows_per_dst=[3264, 3254, 6474, 6174, 6258, 6070, 6376, 6196] dup_rows_per_dst=[854, 426, 4062, 2904, 3534, 3164, 3270, 3256] combine_rows_per_src=[3264, 3254, 6474, 6174, 6258, 6070, 6376, 6196] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43730 payload_rows_remote=36700 dup_rows=21806 dup_rows_remote=15860 dedup_ratio=0.332733 router_tokens_per_expert=[1447, 1259, 976, 606, 50, 963, 263, 82, 757, 1509, 960, 329, 7, 1030, 32, 1864, 1082, 847, 26, 43, 835, 296, 199, 832, 342, 644, 389, 575, 283, 105, 674, 3, 519, 489, 1417, 7, 454, 8, 832, 176, 298, 656, 627, 1130, 10, 772, 39, 506, 322, 223, 228, 301, 136, 863, 95, 686, 369, 604, 553, 315, 257, 430, 137, 0] payload_rows_per_dst=[6628, 7030, 5900, 4618, 5310, 5564, 4486, 4194] dup_rows_per_dst=[4664, 5946, 2420, 1412, 2494, 2512, 1222, 1136] combine_rows_per_src=[6628, 7030, 5900, 4618, 5310, 5564, 4486, 4194] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44882 payload_rows_remote=39900 dup_rows=20654 dup_rows_remote=18190 dedup_ratio=0.315155 router_tokens_per_expert=[1345, 1348, 855, 835, 119, 1053, 461, 87, 296, 1221, 1241, 335, 2, 1240, 31, 1336, 791, 708, 31, 26, 894, 298, 589, 709, 345, 810, 374, 939, 363, 172, 405, 1, 532, 550, 758, 6, 562, 0, 1131, 184, 441, 419, 321, 864, 0, 722, 80, 534, 163, 102, 299, 260, 200, 1014, 125, 766, 448, 682, 562, 480, 609, 448, 245, 1] payload_rows_per_dst=[4928, 6226, 5782, 6182, 4982, 4878, 5838, 6066] dup_rows_per_dst=[1892, 2736, 2310, 3332, 2464, 1884, 2462, 3574] combine_rows_per_src=[4928, 6226, 5782, 6182, 4982, 4878, 5838, 6066] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=281487360 dispatch_meta_bytes=231808 combine_payload_bytes=281487360 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281719168 ingress_bytes=391587840 link_bytes=391587840 token_bytes=563206528 weight_movement_bytes=110100480 total_nvlink_bytes=673307008 weight_to_token_ratio=0.195489 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=7936 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.810356589e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=284870656 dispatch_meta_bytes=233392 combine_payload_bytes=284870656 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=285104048 ingress_bytes=394971136 link_bytes=394971136 token_bytes=569974704 weight_movement_bytes=110100480 total_nvlink_bytes=680075184 weight_to_token_ratio=0.193167 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=7680 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.869197671e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=262377472 dispatch_meta_bytes=209784 combine_payload_bytes=262377472 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=262587256 ingress_bytes=702779392 link_bytes=702779392 token_bytes=524964728 weight_movement_bytes=440401920 total_nvlink_bytes=965366648 weight_to_token_ratio=0.838917 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=7936 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.222248950e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=265975808 dispatch_meta_bytes=223792 combine_payload_bytes=265975808 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=266199600 ingress_bytes=376076288 link_bytes=376076288 token_bytes=532175408 weight_movement_bytes=110100480 total_nvlink_bytes=642275888 weight_to_token_ratio=0.206888 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=6912 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.540585188e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=277286912 dispatch_meta_bytes=225072 combine_payload_bytes=277286912 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277511984 ingress_bytes=387387392 link_bytes=387387392 token_bytes=554798896 weight_movement_bytes=110100480 total_nvlink_bytes=664899376 weight_to_token_ratio=0.198451 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=7424 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.737303890e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278462464 dispatch_meta_bytes=235560 combine_payload_bytes=278462464 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278698024 ingress_bytes=388562944 link_bytes=388562944 token_bytes=557160488 weight_movement_bytes=110100480 total_nvlink_bytes=667260968 weight_to_token_ratio=0.197610 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=7424 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.757748672e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=262191104 dispatch_meta_bytes=219472 combine_payload_bytes=262191104 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=262410576 ingress_bytes=372291584 link_bytes=372291584 token_bytes=524601680 weight_movement_bytes=110100480 total_nvlink_bytes=634702160 weight_to_token_ratio=0.209874 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=7936 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.474762960e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=268011520 dispatch_meta_bytes=213688 combine_payload_bytes=268011520 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=268225208 ingress_bytes=708413440 link_bytes=708413440 token_bytes=536236728 weight_movement_bytes=440401920 total_nvlink_bytes=976638648 weight_to_token_ratio=0.821283 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=7168 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.232047486e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54058 payload_rows_remote=46260 dup_rows=11478 dup_rows_remote=11478 dedup_ratio=0.175140 router_tokens_per_expert=[0, 0, 0, 1418, 4060, 0, 32, 3178, 1959, 443, 44, 0, 0, 1, 0, 179, 0, 825, 0, 34, 3856, 1036, 0, 1, 0, 0, 2891, 1, 14, 3255, 0, 2, 0, 0, 0, 0, 3, 0, 0, 23, 0, 2, 3897, 0, 0, 0, 0, 0, 0, 72, 1278, 0, 259, 0, 0, 1, 880, 2, 14, 0, 3, 2, 5, 3098] payload_rows_per_dst=[7050, 4878, 3388, 7988, 8126, 7798, 7950, 6880] dup_rows_per_dst=[2206, 374, 404, 4338, 46, 0, 2982, 1128] combine_rows_per_src=[7050, 4878, 3388, 7988, 8126, 7798, 7950, 6880] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54434 payload_rows_remote=46264 dup_rows=11102 dup_rows_remote=11102 dedup_ratio=0.169403 router_tokens_per_expert=[0, 0, 0, 1349, 4084, 0, 49, 3335, 2228, 282, 36, 0, 0, 0, 0, 4, 0, 800, 0, 39, 3932, 893, 1, 0, 0, 0, 2667, 0, 0, 3310, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 4021, 0, 0, 0, 12, 0, 0, 104, 1393, 0, 272, 0, 0, 0, 801, 0, 0, 0, 0, 1, 0, 3153] payload_rows_per_dst=[7294, 4872, 3164, 7974, 8170, 8062, 8022, 6876] dup_rows_per_dst=[2172, 228, 302, 3980, 0, 6, 3380, 1034] combine_rows_per_src=[7294, 4872, 3164, 7974, 8170, 8062, 8022, 6876] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44280 payload_rows_remote=37586 dup_rows=21256 dup_rows_remote=16360 dedup_ratio=0.324341 router_tokens_per_expert=[1426, 1175, 1140, 481, 130, 579, 282, 82, 328, 1023, 1045, 385, 16, 1674, 18, 1446, 1126, 304, 11, 67, 902, 319, 461, 451, 405, 871, 283, 748, 235, 229, 502, 9, 739, 409, 1035, 23, 891, 0, 728, 368, 549, 853, 355, 1266, 10, 715, 11, 274, 65, 198, 320, 352, 365, 1370, 76, 901, 311, 544, 648, 415, 242, 451, 130, 1] payload_rows_per_dst=[6694, 6686, 5418, 4890, 5312, 5590, 5512, 4178] dup_rows_per_dst=[4896, 4184, 1864, 1674, 3074, 2476, 1782, 1306] combine_rows_per_src=[6694, 6686, 5418, 4890, 5312, 5590, 5512, 4178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=282404864 dispatch_meta_bytes=230544 combine_payload_bytes=282404864 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=282635408 ingress_bytes=392505344 link_bytes=392505344 token_bytes=565040272 weight_movement_bytes=110100480 total_nvlink_bytes=675140752 weight_to_token_ratio=0.194854 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=8320 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.826313493e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278992896 dispatch_meta_bytes=235008 combine_payload_bytes=278992896 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279227904 ingress_bytes=389093376 link_bytes=389093376 token_bytes=558220800 weight_movement_bytes=110100480 total_nvlink_bytes=668321280 weight_to_token_ratio=0.197235 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=7424 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.766973757e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=266807296 dispatch_meta_bytes=219128 combine_payload_bytes=266807296 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=267026424 ingress_bytes=376907776 link_bytes=376907776 token_bytes=533833720 weight_movement_bytes=110100480 total_nvlink_bytes=643934200 weight_to_token_ratio=0.206245 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=8192 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.555046131e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280956928 dispatch_meta_bytes=228568 combine_payload_bytes=280956928 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281185496 ingress_bytes=391057408 link_bytes=391057408 token_bytes=562142424 weight_movement_bytes=110100480 total_nvlink_bytes=672242904 weight_to_token_ratio=0.195859 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=7040 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.801131504e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=271452160 dispatch_meta_bytes=224336 combine_payload_bytes=271452160 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=271676496 ingress_bytes=381552640 link_bytes=381552640 token_bytes=543128656 weight_movement_bytes=110100480 total_nvlink_bytes=653229136 weight_to_token_ratio=0.202715 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=7296 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.635827956e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=263065600 dispatch_meta_bytes=210240 combine_payload_bytes=263065600 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=263275840 ingress_bytes=703467520 link_bytes=703467520 token_bytes=526341440 weight_movement_bytes=440401920 total_nvlink_bytes=966743360 weight_to_token_ratio=0.836723 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=7808 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.223445718e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=286003200 dispatch_meta_bytes=232360 combine_payload_bytes=286003200 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=286235560 ingress_bytes=396103680 link_bytes=396103680 token_bytes=572238760 weight_movement_bytes=110100480 total_nvlink_bytes=682339240 weight_to_token_ratio=0.192403 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=7808 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.888894474e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=269416448 dispatch_meta_bytes=215784 combine_payload_bytes=269416448 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=269632232 ingress_bytes=709818368 link_bytes=709818368 token_bytes=539048680 weight_movement_bytes=440401920 total_nvlink_bytes=979450600 weight_to_token_ratio=0.816998 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=7424 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.234490887e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53456 payload_rows_remote=45746 dup_rows=12080 dup_rows_remote=12080 dedup_ratio=0.184326 router_tokens_per_expert=[0, 0, 0, 1235, 3910, 0, 33, 3945, 1202, 103, 56, 0, 0, 3, 0, 333, 0, 7, 0, 0, 3542, 182, 0, 2, 0, 0, 2896, 1, 16, 3372, 0, 2, 0, 0, 0, 0, 3, 1, 0, 26, 0, 2, 3853, 0, 0, 0, 0, 0, 0, 284, 2277, 0, 35, 0, 0, 0, 2252, 0, 15, 1, 6, 2, 7, 3164] payload_rows_per_dst=[6516, 3236, 7498, 8046, 7830, 7710, 4882, 7738] dup_rows_per_dst=[1776, 158, 2102, 4528, 50, 0, 310, 3156] combine_rows_per_src=[6516, 3236, 7498, 8046, 7830, 7710, 4882, 7738] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52786 payload_rows_remote=45850 dup_rows=12750 dup_rows_remote=12252 dedup_ratio=0.194550 router_tokens_per_expert=[0, 0, 0, 983, 3877, 0, 43, 3808, 2571, 49, 215, 0, 0, 0, 0, 205, 3, 2, 0, 33, 3388, 291, 0, 0, 0, 0, 2282, 0, 2, 3489, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 4008, 0, 0, 1, 1, 0, 0, 484, 2718, 0, 190, 0, 1, 0, 1047, 0, 0, 4, 42, 1, 15, 3013] payload_rows_per_dst=[7858, 7642, 6936, 4564, 7756, 8018, 7596, 2416] dup_rows_per_dst=[1810, 5416, 498, 4, 2, 2, 4862, 156] combine_rows_per_src=[7858, 7642, 6936, 4564, 7756, 8018, 7596, 2416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50658 payload_rows_remote=45006 dup_rows=14878 dup_rows_remote=14398 dedup_ratio=0.227020 router_tokens_per_expert=[0, 0, 0, 895, 4022, 0, 44, 3940, 1695, 57, 63, 0, 0, 0, 0, 70, 0, 0, 0, 23, 3543, 335, 0, 0, 0, 0, 2040, 1, 3, 2964, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3998, 0, 0, 0, 0, 0, 0, 305, 2656, 0, 105, 0, 0, 0, 2311, 0, 2, 0, 7, 0, 0, 3688] payload_rows_per_dst=[1878, 3636, 7382, 8022, 8056, 7996, 5652, 8036] dup_rows_per_dst=[0, 134, 2024, 7346, 914, 0, 480, 3980] combine_rows_per_src=[1878, 3636, 7382, 8022, 8056, 7996, 5652, 8036] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55770 payload_rows_remote=51188 dup_rows=9766 dup_rows_remote=9766 dedup_ratio=0.149017 router_tokens_per_expert=[0, 0, 0, 1072, 3878, 0, 51, 3876, 1859, 122, 36, 0, 1, 0, 0, 91, 0, 0, 0, 12, 3583, 294, 0, 0, 0, 0, 2291, 0, 0, 3252, 0, 0, 0, 3, 0, 0, 0, 7, 0, 6, 0, 0, 3892, 0, 0, 0, 6, 0, 0, 318, 2544, 2, 162, 0, 0, 0, 2199, 0, 0, 0, 11, 0, 2, 3198] payload_rows_per_dst=[7922, 7276, 7230, 4582, 7774, 7790, 5572, 7624] dup_rows_per_dst=[2076, 3446, 548, 0, 14, 6, 480, 3196] combine_rows_per_src=[7922, 7276, 7230, 4582, 7774, 7790, 5572, 7624] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50712 payload_rows_remote=43902 dup_rows=14824 dup_rows_remote=9712 dedup_ratio=0.226196 router_tokens_per_expert=[0, 0, 0, 1820, 3708, 0, 47, 3800, 1522, 109, 443, 0, 0, 10, 0, 394, 0, 0, 0, 39, 3521, 534, 0, 0, 0, 0, 2382, 0, 1, 3577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4063, 0, 1, 0, 8, 0, 0, 332, 2096, 0, 101, 0, 0, 0, 1624, 0, 0, 3, 68, 0, 2, 2555] payload_rows_per_dst=[3696, 4130, 7358, 8008, 7938, 8134, 4638, 6810] dup_rows_per_dst=[38, 826, 830, 3912, 1632, 2054, 420, 5112] combine_rows_per_src=[3696, 4130, 7358, 8008, 7938, 8134, 4638, 6810] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49284 payload_rows_remote=41400 dup_rows=16252 dup_rows_remote=11638 dedup_ratio=0.247986 router_tokens_per_expert=[0, 0, 0, 1749, 4057, 0, 62, 3189, 2144, 482, 159, 0, 0, 0, 0, 100, 0, 985, 0, 32, 3965, 1064, 0, 0, 0, 0, 2101, 0, 0, 3364, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4027, 0, 0, 0, 1, 0, 0, 60, 1034, 0, 313, 0, 0, 0, 536, 0, 0, 0, 5, 0, 9, 3326] payload_rows_per_dst=[7282, 7884, 8172, 0, 8122, 8054, 2728, 7042] dup_rows_per_dst=[2718, 4614, 8122, 0, 0, 2, 86, 710] combine_rows_per_src=[7282, 7884, 8172, 0, 8122, 8054, 2728, 7042] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=331591680 dispatch_meta_bytes=230952 combine_payload_bytes=331591680 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=331822632 ingress_bytes=441692160 link_bytes=441692160 token_bytes=663414312 weight_movement_bytes=110100480 total_nvlink_bytes=773514792 weight_to_token_ratio=0.165960 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.681753121e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52408 payload_rows_remote=46222 dup_rows=13128 dup_rows_remote=10106 dedup_ratio=0.200317 router_tokens_per_expert=[0, 0, 0, 2117, 3932, 0, 34, 2646, 2170, 426, 428, 0, 0, 2, 0, 222, 0, 1094, 0, 45, 3771, 1429, 0, 0, 0, 0, 2127, 0, 0, 3487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4059, 0, 0, 0, 42, 0, 0, 183, 1047, 0, 292, 0, 0, 0, 575, 0, 0, 2, 34, 0, 1, 2598] payload_rows_per_dst=[4286, 5534, 4454, 7906, 8046, 8126, 7870, 6186] dup_rows_per_dst=[16, 962, 682, 3480, 602, 1648, 2716, 3022] combine_rows_per_src=[4286, 5534, 4454, 7906, 8046, 8126, 7870, 6186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=331620352 dispatch_meta_bytes=229464 combine_payload_bytes=331620352 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=331849816 ingress_bytes=551821312 link_bytes=551821312 token_bytes=663470168 weight_movement_bytes=220200960 total_nvlink_bytes=883671128 weight_to_token_ratio=0.331893 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.597080206e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53318 payload_rows_remote=45390 dup_rows=12218 dup_rows_remote=7620 dedup_ratio=0.186432 router_tokens_per_expert=[0, 0, 0, 1140, 4032, 0, 34, 2951, 3016, 339, 267, 0, 0, 0, 0, 69, 3, 877, 0, 42, 3775, 1047, 0, 0, 0, 0, 2109, 0, 2, 3419, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 4020, 0, 0, 1, 1, 0, 0, 205, 1636, 0, 379, 0, 1, 0, 297, 0, 0, 3, 25, 1, 7, 3068] payload_rows_per_dst=[6714, 7276, 7928, 4836, 8066, 8042, 4064, 6392] dup_rows_per_dst=[1536, 3962, 4598, 1330, 2, 2, 378, 410] combine_rows_per_src=[6714, 7276, 7928, 4836, 8066, 8042, 4064, 6392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54928 payload_rows_remote=47552 dup_rows=10608 dup_rows_remote=7436 dedup_ratio=0.161865 router_tokens_per_expert=[0, 0, 0, 1425, 3992, 0, 68, 3915, 1499, 99, 129, 0, 0, 0, 0, 192, 0, 0, 0, 22, 3711, 270, 0, 0, 0, 0, 2362, 0, 0, 3355, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 3984, 0, 0, 0, 0, 0, 0, 289, 2013, 0, 120, 0, 0, 0, 1839, 0, 1, 0, 19, 0, 12, 3448] payload_rows_per_dst=[8032, 7376, 7484, 4724, 7992, 7968, 7624, 3728] dup_rows_per_dst=[2784, 3172, 522, 0, 0, 0, 4116, 14] combine_rows_per_src=[8032, 7376, 7484, 4724, 7992, 7968, 7624, 3728] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56772 payload_rows_remote=48758 dup_rows=8764 dup_rows_remote=8764 dedup_ratio=0.133728 router_tokens_per_expert=[0, 0, 0, 1182, 4006, 0, 55, 3959, 1510, 40, 33, 0, 0, 0, 0, 45, 0, 1, 0, 19, 3708, 110, 1, 0, 0, 0, 2678, 0, 2, 3314, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 3992, 0, 0, 0, 7, 0, 0, 348, 2337, 0, 23, 0, 0, 0, 2081, 0, 0, 0, 0, 1, 0, 3314] payload_rows_per_dst=[8054, 6468, 7454, 5962, 8014, 7998, 5068, 7754] dup_rows_per_dst=[2338, 2098, 224, 716, 0, 2, 348, 3038] combine_rows_per_src=[8054, 6468, 7454, 5962, 8014, 7998, 5068, 7754] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=327907328 dispatch_meta_bytes=231304 combine_payload_bytes=327907328 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=328138632 ingress_bytes=438007808 link_bytes=438007808 token_bytes=656045960 weight_movement_bytes=110100480 total_nvlink_bytes=766146440 weight_to_token_ratio=0.167824 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=31744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.617676180e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=328652800 dispatch_meta_bytes=232408 combine_payload_bytes=328652800 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=328885208 ingress_bytes=438753280 link_bytes=438753280 token_bytes=657538008 weight_movement_bytes=110100480 total_nvlink_bytes=767638488 weight_to_token_ratio=0.167444 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=28672 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.630641164e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53852 payload_rows_remote=45780 dup_rows=11684 dup_rows_remote=8998 dedup_ratio=0.178284 router_tokens_per_expert=[0, 0, 0, 1402, 3782, 0, 48, 3929, 1969, 60, 54, 0, 0, 0, 0, 125, 0, 0, 0, 11, 3649, 63, 0, 0, 0, 0, 2560, 0, 1, 3053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3926, 0, 2, 0, 0, 0, 1, 322, 2795, 0, 78, 0, 0, 0, 1375, 0, 6, 2, 1, 0, 5, 3548] payload_rows_per_dst=[8072, 7168, 7326, 5122, 7564, 7856, 7974, 2770] dup_rows_per_dst=[2686, 3354, 120, 0, 0, 2, 5514, 8] combine_rows_per_src=[8072, 7168, 7326, 5122, 7564, 7856, 7974, 2770] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54540 payload_rows_remote=46692 dup_rows=10996 dup_rows_remote=7638 dedup_ratio=0.167786 router_tokens_per_expert=[0, 0, 0, 1282, 4065, 0, 47, 2630, 2610, 539, 32, 0, 0, 0, 0, 23, 0, 915, 0, 36, 3861, 1326, 0, 0, 0, 0, 2310, 0, 0, 3293, 0, 0, 0, 2, 0, 0, 0, 5, 0, 6, 0, 0, 3934, 0, 0, 0, 7, 0, 0, 80, 1369, 2, 370, 0, 0, 0, 861, 0, 0, 0, 9, 0, 2, 3152] payload_rows_per_dst=[6244, 5948, 4460, 7848, 8136, 7874, 7106, 6924] dup_rows_per_dst=[1674, 460, 1432, 3358, 20, 8, 2920, 1124] combine_rows_per_src=[6244, 5948, 4460, 7848, 8136, 7874, 7106, 6924] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=296755200 dispatch_meta_bytes=212152 combine_payload_bytes=296755200 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=296967352 ingress_bytes=406855680 link_bytes=406855680 token_bytes=593722552 weight_movement_bytes=110100480 total_nvlink_bytes=703823032 weight_to_token_ratio=0.185441 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=19328 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.075889438e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54108 payload_rows_remote=46168 dup_rows=11428 dup_rows_remote=8214 dedup_ratio=0.174377 router_tokens_per_expert=[0, 0, 0, 1044, 4077, 0, 41, 3317, 2457, 407, 77, 0, 0, 0, 0, 51, 0, 418, 0, 25, 3904, 1427, 0, 0, 0, 0, 2006, 1, 1, 2950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 0, 0, 0, 0, 0, 0, 44, 1331, 0, 298, 0, 0, 0, 1206, 0, 2, 0, 5, 0, 0, 3673] payload_rows_per_dst=[5198, 5638, 3490, 7872, 8154, 8012, 7940, 7804] dup_rows_per_dst=[1026, 346, 250, 4624, 0, 0, 3214, 1968] combine_rows_per_src=[5198, 5638, 3490, 7872, 8154, 8012, 7940, 7804] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50628 payload_rows_remote=43224 dup_rows=14908 dup_rows_remote=12192 dedup_ratio=0.227478 router_tokens_per_expert=[0, 0, 0, 1714, 4021, 0, 43, 3303, 2640, 418, 56, 0, 0, 0, 0, 50, 0, 403, 0, 34, 3898, 525, 0, 0, 0, 0, 2485, 0, 1, 3116, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 3947, 0, 2, 0, 1, 0, 1, 70, 1747, 0, 272, 0, 0, 0, 518, 0, 5, 1, 1, 0, 2, 3491] payload_rows_per_dst=[7404, 7762, 8140, 2, 8044, 7898, 4080, 7298] dup_rows_per_dst=[2716, 4798, 6550, 0, 2, 4, 100, 738] combine_rows_per_src=[7404, 7762, 8140, 2, 8044, 7898, 4080, 7298] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=331319296 dispatch_meta_bytes=225312 combine_payload_bytes=331319296 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=331544608 ingress_bytes=441419776 link_bytes=441419776 token_bytes=662863904 weight_movement_bytes=110100480 total_nvlink_bytes=772964384 weight_to_token_ratio=0.166098 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=25600 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.677015915e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=325355520 dispatch_meta_bytes=212040 combine_payload_bytes=325355520 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=325567560 ingress_bytes=545556480 link_bytes=545556480 token_bytes=650923080 weight_movement_bytes=220200960 total_nvlink_bytes=871124040 weight_to_token_ratio=0.338290 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=12416 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.488124473e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 2, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=334688256 dispatch_meta_bytes=217320 combine_payload_bytes=334688256 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=334905576 ingress_bytes=664989696 link_bytes=664989696 token_bytes=669593832 weight_movement_bytes=330301440 total_nvlink_bytes=999895272 weight_to_token_ratio=0.493286 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=17792 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.156526453e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=330932224 dispatch_meta_bytes=217528 combine_payload_bytes=330932224 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=331149752 ingress_bytes=441032704 link_bytes=441032704 token_bytes=662081976 weight_movement_bytes=110100480 total_nvlink_bytes=772182456 weight_to_token_ratio=0.166294 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=18688 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.670284097e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60444 payload_rows_remote=53984 dup_rows=5092 dup_rows_remote=5092 dedup_ratio=0.077698 router_tokens_per_expert=[0, 0, 0, 3516, 12, 4082, 4060, 0, 0, 3418, 0, 0, 0, 0, 1647, 0, 2, 0, 0, 0, 0, 4021, 26, 0, 0, 0, 278, 4096, 0, 82, 0, 0, 0, 0, 0, 0, 0, 2, 276, 8, 3220, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4012, 0, 0, 0, 0, 0] payload_rows_per_dst=[7032, 6346, 8090, 8192, 8136, 6460, 8164, 8024] dup_rows_per_dst=[24, 1474, 2318, 720, 556, 0, 0, 0] combine_rows_per_src=[7032, 6346, 8090, 8192, 8136, 6460, 8164, 8024] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=309829632 dispatch_meta_bytes=221664 combine_payload_bytes=309829632 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=310051296 ingress_bytes=860332032 link_bytes=860332032 token_bytes=619880928 weight_movement_bytes=550502400 total_nvlink_bytes=1170383328 weight_to_token_ratio=0.888078 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=20736 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.496258904e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62052 payload_rows_remote=53968 dup_rows=3484 dup_rows_remote=3066 dedup_ratio=0.053162 router_tokens_per_expert=[0, 0, 0, 3458, 0, 4093, 4035, 0, 0, 3576, 0, 0, 0, 0, 1686, 0, 0, 0, 0, 0, 0, 3890, 17, 0, 0, 0, 276, 4096, 0, 43, 1, 0, 0, 1, 0, 0, 0, 0, 214, 1, 3313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0] payload_rows_per_dst=[6916, 8132, 7780, 8192, 8084, 6626, 8186, 8136] dup_rows_per_dst=[0, 2392, 34, 640, 418, 0, 0, 0] combine_rows_per_src=[6916, 8132, 7780, 8192, 8084, 6626, 8186, 8136] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62046 payload_rows_remote=53990 dup_rows=3490 dup_rows_remote=1070 dedup_ratio=0.053253 router_tokens_per_expert=[0, 0, 0, 3585, 0, 4085, 3825, 0, 0, 3289, 0, 0, 0, 0, 1949, 0, 0, 0, 0, 0, 0, 3951, 20, 0, 0, 0, 256, 4096, 0, 52, 0, 0, 0, 0, 0, 0, 0, 1, 225, 0, 3399, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 0, 0, 0, 0] payload_rows_per_dst=[7170, 8056, 7902, 8192, 7688, 6800, 8170, 8068] dup_rows_per_dst=[0, 2420, 40, 616, 414, 0, 0, 0] combine_rows_per_src=[7170, 8056, 7902, 8192, 7688, 6800, 8170, 8068] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=386957312 dispatch_meta_bytes=236304 combine_payload_bytes=386957312 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=387193616 ingress_bytes=497057792 link_bytes=497057792 token_bytes=774150928 weight_movement_bytes=110100480 total_nvlink_bytes=884251408 weight_to_token_ratio=0.142221 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=27392 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.644652522e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=386842624 dispatch_meta_bytes=228136 combine_payload_bytes=386842624 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=387070760 ingress_bytes=496943104 link_bytes=496943104 token_bytes=773913384 weight_movement_bytes=110100480 total_nvlink_bytes=884013864 weight_to_token_ratio=0.142265 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=31104 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.642657910e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58608 payload_rows_remote=50450 dup_rows=6928 dup_rows_remote=6276 dedup_ratio=0.105713 router_tokens_per_expert=[0, 0, 0, 3510, 0, 3921, 3924, 0, 0, 3038, 0, 0, 0, 0, 2086, 0, 0, 0, 0, 0, 0, 4001, 59, 0, 0, 0, 337, 4096, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 3516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0] payload_rows_per_dst=[6392, 4172, 8018, 8192, 7850, 7984, 7842, 8158] dup_rows_per_dst=[0, 0, 102, 834, 256, 4456, 628, 652] combine_rows_per_src=[6392, 4172, 8018, 8192, 7850, 7984, 7842, 8158] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62378 payload_rows_remote=54522 dup_rows=3158 dup_rows_remote=3118 dedup_ratio=0.048187 router_tokens_per_expert=[0, 0, 0, 3542, 3, 4047, 4054, 0, 0, 3585, 0, 0, 0, 0, 1463, 0, 0, 0, 0, 0, 0, 3925, 23, 0, 0, 0, 364, 4096, 0, 76, 1, 0, 0, 2, 0, 0, 0, 0, 111, 0, 3385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 3] payload_rows_per_dst=[7086, 8092, 7856, 8192, 8112, 6770, 8096, 8174] dup_rows_per_dst=[4, 2004, 40, 882, 222, 0, 0, 6] combine_rows_per_src=[7086, 8092, 7856, 8192, 8112, 6770, 8096, 8174] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63026 payload_rows_remote=54834 dup_rows=2510 dup_rows_remote=2060 dedup_ratio=0.038300 router_tokens_per_expert=[0, 0, 0, 3789, 2, 4086, 4077, 0, 0, 3379, 0, 0, 0, 0, 1203, 0, 0, 0, 0, 0, 0, 3809, 10, 0, 0, 0, 177, 4096, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 401, 2, 3604, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0] payload_rows_per_dst=[7578, 7926, 7618, 8192, 8162, 7212, 8172, 8166] dup_rows_per_dst=[4, 1238, 20, 450, 798, 0, 0, 0] combine_rows_per_src=[7578, 7926, 7618, 8192, 8162, 7212, 8172, 8166] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58138 payload_rows_remote=49976 dup_rows=7398 dup_rows_remote=7398 dedup_ratio=0.112885 router_tokens_per_expert=[0, 0, 0, 3596, 1, 4081, 4030, 0, 0, 3543, 0, 0, 0, 0, 1258, 0, 0, 0, 0, 0, 0, 4044, 5, 0, 0, 0, 249, 4096, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 3652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 0, 0, 0, 0, 1] payload_rows_per_dst=[7192, 2516, 8088, 8192, 8062, 7908, 8162, 8018] dup_rows_per_dst=[2, 0, 166, 2056, 310, 4862, 0, 2] combine_rows_per_src=[7192, 2516, 8088, 8192, 8062, 7908, 8162, 8018] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=387000320 dispatch_meta_bytes=220240 combine_payload_bytes=387000320 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=387220560 ingress_bytes=827402240 link_bytes=827402240 token_bytes=774220880 weight_movement_bytes=440401920 total_nvlink_bytes=1214622800 weight_to_token_ratio=0.568832 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=19840 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.438988580e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=361625600 dispatch_meta_bytes=226904 combine_payload_bytes=361625600 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=361852504 ingress_bytes=471726080 link_bytes=471726080 token_bytes=723478104 weight_movement_bytes=110100480 total_nvlink_bytes=833578584 weight_to_token_ratio=0.152182 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.204092387e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54408 payload_rows_remote=46346 dup_rows=11128 dup_rows_remote=3054 dedup_ratio=0.169800 router_tokens_per_expert=[0, 0, 0, 3793, 1, 4058, 3955, 0, 0, 3701, 0, 0, 0, 0, 1683, 0, 0, 0, 0, 0, 0, 3731, 1, 0, 0, 0, 186, 4096, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 193, 2, 3228, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4065, 0, 0, 0, 0, 0] payload_rows_per_dst=[8062, 8152, 7462, 1870, 6156, 6458, 8116, 8132] dup_rows_per_dst=[8074, 2616, 2, 124, 312, 0, 0, 0] combine_rows_per_src=[8062, 8152, 7462, 1870, 6156, 6458, 8116, 8132] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=390813696 dispatch_meta_bytes=230560 combine_payload_bytes=390813696 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=391044256 ingress_bytes=500914176 link_bytes=500914176 token_bytes=781857952 weight_movement_bytes=110100480 total_nvlink_bytes=891958432 weight_to_token_ratio=0.140819 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=31488 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.711721383e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=322603008 dispatch_meta_bytes=237616 combine_payload_bytes=322603008 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=322840624 ingress_bytes=432703488 link_bytes=432703488 token_bytes=645443632 weight_movement_bytes=110100480 total_nvlink_bytes=755544112 weight_to_token_ratio=0.170581 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=19456 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.525425331e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=366915584 dispatch_meta_bytes=243816 combine_payload_bytes=366915584 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=367159400 ingress_bytes=587116544 link_bytes=587116544 token_bytes=734074984 weight_movement_bytes=220200960 total_nvlink_bytes=954275944 weight_to_token_ratio=0.299971 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=19584 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.021092234e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=314689536 dispatch_meta_bytes=214456 combine_payload_bytes=314689536 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=314903992 ingress_bytes=534890496 link_bytes=534890496 token_bytes=629593528 weight_movement_bytes=220200960 total_nvlink_bytes=849794488 weight_to_token_ratio=0.349751 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=16128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.302625469e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 2, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=340852736 dispatch_meta_bytes=219952 combine_payload_bytes=340852736 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=341072688 ingress_bytes=450953216 link_bytes=450953216 token_bytes=681925424 weight_movement_bytes=110100480 total_nvlink_bytes=792025904 weight_to_token_ratio=0.161455 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=15872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.842818117e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=349497344 dispatch_meta_bytes=230088 combine_payload_bytes=349497344 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=349727432 ingress_bytes=569698304 link_bytes=569698304 token_bytes=699224776 weight_movement_bytes=220200960 total_nvlink_bytes=919425736 weight_to_token_ratio=0.314922 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=31232 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.907990499e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=328151040 dispatch_meta_bytes=219112 combine_payload_bytes=328151040 prefetch_bytes=0 reduce_bytes=660602880 egress_bytes=328370152 ingress_bytes=988753920 link_bytes=988753920 token_bytes=656521192 weight_movement_bytes=660602880 total_nvlink_bytes=1317124072 weight_to_token_ratio=1.006217 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=22400 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.719605689e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63984 payload_rows_remote=55944 dup_rows=1552 dup_rows_remote=1552 dedup_ratio=0.023682 router_tokens_per_expert=[0, 0, 0, 4026, 10, 0, 4091, 0, 0, 3770, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 4083, 67, 0, 0, 0, 482, 4096, 0, 123, 0, 0, 0, 288, 0, 0, 0, 52, 3542, 8, 3998, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4080, 0, 0, 0, 0, 4] payload_rows_per_dst=[8052, 7540, 8176, 8192, 7640, 8040, 8182, 8162] dup_rows_per_dst=[20, 0, 174, 1210, 140, 0, 0, 8] combine_rows_per_src=[8052, 7540, 8176, 8192, 7640, 8040, 8182, 8162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63660 payload_rows_remote=55472 dup_rows=1876 dup_rows_remote=1568 dedup_ratio=0.028625 router_tokens_per_expert=[0, 0, 0, 4040, 3, 0, 4094, 0, 0, 3792, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 4093, 132, 0, 0, 0, 462, 4096, 0, 161, 1, 0, 0, 509, 0, 0, 0, 130, 3123, 1, 3993, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 17] payload_rows_per_dst=[8082, 7584, 8188, 8192, 7246, 7986, 8190, 8192] dup_rows_per_dst=[4, 0, 308, 1248, 280, 2, 0, 34] combine_rows_per_src=[8082, 7584, 8188, 8192, 7246, 7986, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64328 payload_rows_remote=56138 dup_rows=1208 dup_rows_remote=1208 dedup_ratio=0.018433 router_tokens_per_expert=[0, 0, 0, 4052, 2, 0, 4095, 0, 0, 3820, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 4096, 66, 0, 0, 0, 362, 4096, 0, 71, 0, 0, 0, 255, 0, 0, 0, 16, 3603, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 22] payload_rows_per_dst=[8104, 7640, 8192, 8192, 7640, 8178, 8190, 8192] dup_rows_per_dst=[4, 0, 184, 866, 108, 0, 0, 46] combine_rows_per_src=[8104, 7640, 8192, 8192, 7640, 8178, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59584 payload_rows_remote=51392 dup_rows=5952 dup_rows_remote=5936 dedup_ratio=0.090820 router_tokens_per_expert=[0, 0, 0, 4001, 0, 0, 4096, 0, 0, 3648, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4095, 161, 0, 0, 0, 580, 4096, 0, 200, 0, 0, 0, 556, 0, 0, 0, 134, 3030, 0, 4058, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 4] payload_rows_per_dst=[7982, 7296, 7476, 5010, 7318, 8118, 8192, 8192] dup_rows_per_dst=[0, 0, 304, 1200, 3664, 748, 20, 16] combine_rows_per_src=[7982, 7296, 7476, 5010, 7318, 8118, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64440 payload_rows_remote=56248 dup_rows=1096 dup_rows_remote=330 dedup_ratio=0.016724 router_tokens_per_expert=[0, 0, 0, 4059, 2, 0, 4095, 0, 0, 3781, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4094, 53, 0, 0, 0, 333, 4096, 0, 50, 0, 0, 0, 264, 0, 0, 0, 25, 3720, 2, 4082, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4095, 0, 0, 0, 0, 4] payload_rows_per_dst=[8118, 7562, 8188, 8192, 7832, 8168, 8190, 8190] dup_rows_per_dst=[4, 0, 120, 766, 190, 2, 0, 14] combine_rows_per_src=[8118, 7562, 8188, 8192, 7832, 8168, 8190, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=393050112 dispatch_meta_bytes=227576 combine_payload_bytes=393050112 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=393277688 ingress_bytes=613251072 link_bytes=613251072 token_bytes=786327800 weight_movement_bytes=220200960 total_nvlink_bytes=1006528760 weight_to_token_ratio=0.280037 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=29440 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.066544477e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63752 payload_rows_remote=55560 dup_rows=1784 dup_rows_remote=1784 dedup_ratio=0.027222 router_tokens_per_expert=[0, 0, 407, 0, 0, 4096, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 3648, 3919, 0, 0, 0, 4091, 0, 0, 4096, 0, 3716, 0, 1, 10, 0, 0, 0, 0, 0, 0, 25, 0, 559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 4087, 0] payload_rows_per_dst=[8192, 8162, 7296, 7432, 8106, 8192, 8182, 8190] dup_rows_per_dst=[814, 0, 0, 22, 900, 0, 0, 48] combine_rows_per_src=[8192, 8162, 7296, 7432, 8106, 8192, 8182, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=358227968 dispatch_meta_bytes=229496 combine_payload_bytes=358227968 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=358457464 ingress_bytes=578428928 link_bytes=578428928 token_bytes=716685432 weight_movement_bytes=220200960 total_nvlink_bytes=936886392 weight_to_token_ratio=0.307249 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.005983041e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63978 payload_rows_remote=55856 dup_rows=1558 dup_rows_remote=578 dedup_ratio=0.023773 router_tokens_per_expert=[0, 0, 287, 0, 0, 4085, 0, 0, 1, 0, 0, 4095, 1, 0, 0, 0, 0, 3608, 3995, 0, 0, 0, 4090, 0, 0, 4096, 0, 3849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 506, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 4095, 0] payload_rows_per_dst=[8186, 8192, 7216, 7698, 8122, 8192, 8180, 8192] dup_rows_per_dst=[558, 2, 0, 0, 980, 2, 0, 16] combine_rows_per_src=[8186, 8192, 7216, 7698, 8122, 8192, 8180, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=332208128 dispatch_meta_bytes=197600 combine_payload_bytes=332208128 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=332405728 ingress_bytes=772610048 link_bytes=772610048 token_bytes=664613856 weight_movement_bytes=440401920 total_nvlink_bytes=1105015776 weight_to_token_ratio=0.662643 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=28544 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.343695946e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=398254080 dispatch_meta_bytes=229376 combine_payload_bytes=398254080 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=398483456 ingress_bytes=508354560 link_bytes=508354560 token_bytes=796737536 weight_movement_bytes=110100480 total_nvlink_bytes=906838016 weight_to_token_ratio=0.138189 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.841121898e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63152 payload_rows_remote=54968 dup_rows=2384 dup_rows_remote=2384 dedup_ratio=0.036377 router_tokens_per_expert=[0, 0, 522, 0, 0, 4014, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 3597, 3873, 0, 0, 0, 4008, 0, 0, 4096, 0, 3578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8162, 8184, 7194, 7156, 8056, 8192, 8016, 8192] dup_rows_per_dst=[910, 0, 0, 0, 1432, 0, 0, 42] combine_rows_per_src=[8162, 8184, 7194, 7156, 8056, 8192, 8016, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=400375808 dispatch_meta_bytes=225736 combine_payload_bytes=400375808 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=400601544 ingress_bytes=510476288 link_bytes=510476288 token_bytes=800977352 weight_movement_bytes=110100480 total_nvlink_bytes=911077832 weight_to_token_ratio=0.137458 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=27776 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.878022238e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57322 payload_rows_remote=49130 dup_rows=8214 dup_rows_remote=8130 dedup_ratio=0.125336 router_tokens_per_expert=[0, 0, 1294, 0, 0, 3997, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 3154, 3741, 0, 0, 0, 4058, 0, 0, 4096, 0, 3726, 0, 0, 0, 0, 0, 0, 0, 0, 2, 75, 0, 392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[2726, 8190, 6338, 7870, 7698, 8192, 8116, 8192] dup_rows_per_dst=[522, 0, 1500, 4882, 722, 0, 504, 84] combine_rows_per_src=[2726, 8190, 6338, 7870, 7698, 8192, 8116, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63186 payload_rows_remote=56386 dup_rows=2350 dup_rows_remote=2350 dedup_ratio=0.035858 router_tokens_per_expert=[0, 0, 524, 0, 0, 4094, 0, 0, 0, 0, 0, 4090, 2, 0, 0, 0, 0, 3400, 3711, 0, 0, 0, 4005, 0, 0, 4094, 0, 3866, 0, 0, 2, 0, 0, 0, 0, 0, 0, 50, 0, 786, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 48, 0, 0, 0, 0, 1, 4093, 0] payload_rows_per_dst=[8190, 8184, 6800, 7732, 7886, 8190, 8012, 8192] dup_rows_per_dst=[1046, 0, 0, 4, 1208, 0, 0, 92] combine_rows_per_src=[8190, 8184, 6800, 7732, 7886, 8190, 8012, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=394010624 dispatch_meta_bytes=229408 combine_payload_bytes=394010624 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=394240032 ingress_bytes=504111104 link_bytes=504111104 token_bytes=788250656 weight_movement_bytes=110100480 total_nvlink_bytes=898351136 weight_to_token_ratio=0.139677 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.767321219e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64328 payload_rows_remote=56162 dup_rows=1208 dup_rows_remote=1208 dedup_ratio=0.018433 router_tokens_per_expert=[0, 0, 212, 0, 0, 4084, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 3751, 4039, 0, 0, 0, 4083, 0, 0, 4096, 0, 3887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8168, 7502, 7774, 8142, 8192, 8166, 8192] dup_rows_per_dst=[400, 0, 0, 0, 804, 0, 0, 4] combine_rows_per_src=[8192, 8168, 7502, 7774, 8142, 8192, 8166, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63920 payload_rows_remote=56362 dup_rows=1616 dup_rows_remote=1614 dedup_ratio=0.024658 router_tokens_per_expert=[0, 0, 207, 0, 0, 4096, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 3640, 3977, 0, 0, 0, 4088, 0, 0, 4096, 0, 3779, 0, 0, 1, 0, 0, 0, 0, 0, 0, 36, 0, 635, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 4094, 0] payload_rows_per_dst=[8192, 8192, 7280, 7558, 8138, 8192, 8176, 8192] dup_rows_per_dst=[414, 0, 0, 2, 1158, 6, 0, 36] combine_rows_per_src=[8192, 8192, 7280, 7558, 8138, 8192, 8176, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=352163840 dispatch_meta_bytes=229040 combine_payload_bytes=352163840 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=352392880 ingress_bytes=572364800 link_bytes=572364800 token_bytes=704556720 weight_movement_bytes=220200960 total_nvlink_bytes=924757680 weight_to_token_ratio=0.312538 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.954365250e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58124 payload_rows_remote=49934 dup_rows=7412 dup_rows_remote=6570 dedup_ratio=0.113098 router_tokens_per_expert=[0, 0, 296, 0, 0, 4085, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 3643, 3987, 0, 0, 0, 4092, 0, 0, 4096, 0, 3695, 0, 0, 3, 0, 0, 0, 0, 0, 0, 57, 0, 610, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 1, 4094, 0] payload_rows_per_dst=[8190, 8174, 8098, 7390, 2066, 8186, 8184, 7836] dup_rows_per_dst=[842, 88, 6312, 16, 118, 0, 0, 36] combine_rows_per_src=[8190, 8174, 8098, 7390, 2066, 8186, 8184, 7836] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64232 payload_rows_remote=56642 dup_rows=1304 dup_rows_remote=1304 dedup_ratio=0.019897 router_tokens_per_expert=[0, 0, 0, 4040, 0, 0, 4094, 0, 0, 3795, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4096, 65, 0, 0, 0, 423, 4096, 0, 68, 0, 0, 0, 395, 0, 0, 0, 110, 3437, 0, 4046, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0] payload_rows_per_dst=[8080, 7590, 8192, 8192, 7704, 8094, 8188, 8192] dup_rows_per_dst=[0, 0, 140, 982, 180, 0, 0, 2] combine_rows_per_src=[8080, 7590, 8192, 8192, 7704, 8094, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=401006592 dispatch_meta_bytes=229984 combine_payload_bytes=401006592 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=401236576 ingress_bytes=511107072 link_bytes=511107072 token_bytes=802243168 weight_movement_bytes=110100480 total_nvlink_bytes=912343648 weight_to_token_ratio=0.137241 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.888992609e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64462 payload_rows_remote=56632 dup_rows=1074 dup_rows_remote=998 dedup_ratio=0.016388 router_tokens_per_expert=[0, 0, 0, 4036, 1, 0, 4096, 0, 0, 3857, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 4090, 50, 0, 0, 0, 373, 4096, 0, 58, 1, 0, 0, 222, 0, 0, 0, 44, 3686, 1, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 7] payload_rows_per_dst=[8074, 7714, 8190, 8192, 7830, 8084, 8192, 8186] dup_rows_per_dst=[0, 0, 120, 864, 76, 0, 0, 14] combine_rows_per_src=[8074, 7714, 8190, 8192, 7830, 8084, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=402397184 dispatch_meta_bytes=229384 combine_payload_bytes=402397184 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=402626568 ingress_bytes=622598144 link_bytes=622598144 token_bytes=805023752 weight_movement_bytes=220200960 total_nvlink_bytes=1025224712 weight_to_token_ratio=0.273533 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.082800572e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59338 payload_rows_remote=51224 dup_rows=6198 dup_rows_remote=5300 dedup_ratio=0.094574 router_tokens_per_expert=[0, 0, 0, 4057, 0, 0, 4096, 0, 0, 3864, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 4093, 75, 0, 0, 0, 359, 4096, 0, 94, 0, 0, 0, 385, 0, 0, 0, 14, 3443, 2, 4056, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4090, 0, 0, 0, 0, 24] payload_rows_per_dst=[8114, 7986, 7412, 3868, 7604, 8116, 8192, 8046] dup_rows_per_dst=[898, 4624, 172, 370, 84, 0, 0, 50] combine_rows_per_src=[8114, 7986, 7412, 3868, 7604, 8116, 8192, 8046] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=397623296 dispatch_meta_bytes=228160 combine_payload_bytes=397623296 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=397851456 ingress_bytes=727924736 link_bytes=727924736 token_bytes=795474752 weight_movement_bytes=330301440 total_nvlink_bytes=1125776192 weight_to_token_ratio=0.415226 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.265980839e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=368377856 dispatch_meta_bytes=229312 combine_payload_bytes=368377856 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=368607168 ingress_bytes=588578816 link_bytes=588578816 token_bytes=736985024 weight_movement_bytes=220200960 total_nvlink_bytes=957185984 weight_to_token_ratio=0.298786 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.023635366e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=403185664 dispatch_meta_bytes=226312 combine_payload_bytes=403185664 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=403411976 ingress_bytes=733487104 link_bytes=733487104 token_bytes=806597640 weight_movement_bytes=330301440 total_nvlink_bytes=1136899080 weight_to_token_ratio=0.409500 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=28672 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.275654712e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=406009856 dispatch_meta_bytes=231784 combine_payload_bytes=406009856 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=406241640 ingress_bytes=516110336 link_bytes=516110336 token_bytes=812251496 weight_movement_bytes=110100480 total_nvlink_bytes=922351976 weight_to_token_ratio=0.135550 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=30336 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.976007599e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=405938176 dispatch_meta_bytes=230520 combine_payload_bytes=405938176 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=406168696 ingress_bytes=516038656 link_bytes=516038656 token_bytes=812106872 weight_movement_bytes=110100480 total_nvlink_bytes=922207352 weight_to_token_ratio=0.135574 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=27648 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.974760966e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=367173632 dispatch_meta_bytes=226096 combine_payload_bytes=367173632 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=367399728 ingress_bytes=697475072 link_bytes=697475072 token_bytes=734573360 weight_movement_bytes=330301440 total_nvlink_bytes=1064874800 weight_to_token_ratio=0.449651 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.213023865e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63318 payload_rows_remote=55314 dup_rows=2218 dup_rows_remote=2208 dedup_ratio=0.033844 router_tokens_per_expert=[0, 0, 3230, 0, 0, 4096, 0, 0, 3, 0, 0, 406, 0, 0, 0, 0, 0, 0, 4018, 0, 0, 0, 4096, 0, 0, 4035, 0, 4002, 0, 0, 9, 0, 0, 0, 0, 0, 19, 287, 0, 4076, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 319, 4092, 0] payload_rows_per_dst=[6460, 8192, 8036, 8080, 8164, 8004, 8192, 8190] dup_rows_per_dst=[0, 818, 0, 8, 600, 10, 0, 782] combine_rows_per_src=[6460, 8192, 8036, 8080, 8164, 8004, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62022 payload_rows_remote=53836 dup_rows=3514 dup_rows_remote=3514 dedup_ratio=0.053619 router_tokens_per_expert=[0, 0, 2463, 0, 0, 4095, 0, 0, 0, 0, 0, 576, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 4093, 0, 0, 4030, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 12, 458, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 701, 4096, 0] payload_rows_per_dst=[4926, 8190, 8176, 8060, 8154, 8138, 8186, 8192] dup_rows_per_dst=[0, 1152, 0, 0, 900, 0, 0, 1462] combine_rows_per_src=[4926, 8190, 8176, 8060, 8154, 8138, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59812 payload_rows_remote=51620 dup_rows=5724 dup_rows_remote=2218 dedup_ratio=0.087341 router_tokens_per_expert=[0, 0, 3496, 0, 0, 4084, 0, 0, 0, 0, 0, 489, 0, 0, 0, 0, 0, 0, 4040, 0, 0, 0, 4096, 0, 0, 4010, 0, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 18, 278, 0, 3956, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 140, 4096, 0] payload_rows_per_dst=[6992, 8168, 8068, 7054, 5090, 8056, 8192, 8192] dup_rows_per_dst=[0, 978, 0, 0, 276, 952, 12, 3506] combine_rows_per_src=[6992, 8168, 8068, 7054, 5090, 8056, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62648 payload_rows_remote=54672 dup_rows=2888 dup_rows_remote=2888 dedup_ratio=0.044067 router_tokens_per_expert=[0, 0, 2942, 0, 0, 4096, 0, 0, 3, 0, 0, 688, 2, 0, 0, 0, 0, 0, 3988, 0, 0, 0, 4095, 0, 0, 4002, 0, 4037, 0, 0, 2, 0, 0, 0, 0, 0, 10, 431, 0, 4054, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 272, 4093, 0] payload_rows_per_dst=[5884, 8192, 7976, 8008, 8132, 8074, 8190, 8192] dup_rows_per_dst=[0, 1386, 0, 0, 858, 4, 0, 640] combine_rows_per_src=[5884, 8192, 7976, 8008, 8132, 8074, 8190, 8192] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=404174848 dispatch_meta_bytes=234944 combine_payload_bytes=404174848 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=404409792 ingress_bytes=844576768 link_bytes=844576768 token_bytes=808584640 weight_movement_bytes=440401920 total_nvlink_bytes=1248986560 weight_to_token_ratio=0.544658 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=28288 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.468857909e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64120 payload_rows_remote=56026 dup_rows=1416 dup_rows_remote=1208 dedup_ratio=0.021606 router_tokens_per_expert=[0, 44, 0, 0, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3982, 4, 0, 0, 0, 0, 4077, 4096, 0, 4079, 0, 0, 0, 0, 105, 0, 4094, 0, 0, 4046, 0, 0, 0, 85, 0, 0, 0, 3731, 47, 0, 0, 0, 2114, 0, 0, 0, 2044, 0] payload_rows_per_dst=[8192, 8192, 8188, 7972, 8154, 8094, 7486, 7842] dup_rows_per_dst=[494, 0, 0, 0, 0, 208, 240, 474] combine_rows_per_src=[8192, 8192, 8188, 7972, 8154, 8094, 7486, 7842] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64220 payload_rows_remote=56030 dup_rows=1316 dup_rows_remote=1316 dedup_ratio=0.020081 router_tokens_per_expert=[0, 1, 0, 0, 324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 2, 0, 0, 0, 0, 4095, 4096, 0, 4095, 0, 0, 0, 0, 119, 0, 4095, 0, 0, 4056, 0, 0, 0, 47, 0, 0, 0, 3789, 38, 2, 0, 0, 2268, 0, 0, 0, 1653, 0] payload_rows_per_dst=[8192, 8192, 8190, 8180, 8190, 8112, 7580, 7584] dup_rows_per_dst=[648, 0, 0, 0, 0, 238, 172, 258] combine_rows_per_src=[8192, 8192, 8190, 8180, 8190, 8112, 7580, 7584] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=402569216 dispatch_meta_bytes=229480 combine_payload_bytes=402569216 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=402798696 ingress_bytes=622770176 link_bytes=622770176 token_bytes=805367912 weight_movement_bytes=220200960 total_nvlink_bytes=1025568872 weight_to_token_ratio=0.273417 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.083099764e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=404002816 dispatch_meta_bytes=231904 combine_payload_bytes=404002816 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=404234720 ingress_bytes=624203776 link_bytes=624203776 token_bytes=808237536 weight_movement_bytes=220200960 total_nvlink_bytes=1028438496 weight_to_token_ratio=0.272446 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=30208 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.085593030e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=357926912 dispatch_meta_bytes=226016 combine_payload_bytes=357926912 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=358152928 ingress_bytes=688228352 link_bytes=688228352 token_bytes=716079840 weight_movement_bytes=330301440 total_nvlink_bytes=1046381280 weight_to_token_ratio=0.461263 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=28928 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.196942298e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 2, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=401594368 dispatch_meta_bytes=228936 combine_payload_bytes=401594368 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=401823304 ingress_bytes=731895808 link_bytes=731895808 token_bytes=803417672 weight_movement_bytes=330301440 total_nvlink_bytes=1133719112 weight_to_token_ratio=0.411120 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.272887186e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=401623040 dispatch_meta_bytes=229384 combine_payload_bytes=401623040 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=401852424 ingress_bytes=952125440 link_bytes=952125440 token_bytes=803475464 weight_movement_bytes=550502400 total_nvlink_bytes=1353977864 weight_to_token_ratio=0.685151 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=30208 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.655902738e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64426 payload_rows_remote=56238 dup_rows=1110 dup_rows_remote=1110 dedup_ratio=0.016937 router_tokens_per_expert=[0, 3, 0, 0, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 4091, 4094, 0, 4096, 0, 0, 0, 0, 92, 3, 4096, 0, 0, 4052, 0, 0, 0, 47, 0, 0, 0, 3836, 39, 1, 0, 0, 1538, 0, 0, 0, 2489, 0] payload_rows_per_dst=[8192, 8188, 8192, 8132, 8182, 8104, 7674, 7762] dup_rows_per_dst=[456, 0, 0, 0, 0, 190, 172, 292] combine_rows_per_src=[8192, 8188, 8192, 8132, 8182, 8104, 7674, 7762] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59792 payload_rows_remote=51892 dup_rows=5744 dup_rows_remote=5036 dedup_ratio=0.087646 router_tokens_per_expert=[0, 4, 0, 0, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 0, 0, 0, 0, 0, 4095, 4096, 0, 4096, 0, 0, 0, 0, 84, 0, 4096, 0, 0, 4052, 0, 0, 0, 99, 0, 0, 0, 3496, 24, 17, 0, 0, 2942, 0, 0, 0, 1362, 0] payload_rows_per_dst=[4034, 8192, 8192, 8086, 7574, 8104, 7710, 7900] dup_rows_per_dst=[252, 0, 0, 0, 0, 168, 4616, 708] combine_rows_per_src=[4034, 8192, 8192, 8086, 7574, 8104, 7710, 7900] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=396490752 dispatch_meta_bytes=230088 combine_payload_bytes=396490752 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=396720840 ingress_bytes=616691712 link_bytes=616691712 token_bytes=793211592 weight_movement_bytes=220200960 total_nvlink_bytes=1013412552 weight_to_token_ratio=0.277607 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.072528316e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61782 payload_rows_remote=53938 dup_rows=3754 dup_rows_remote=3220 dedup_ratio=0.057281 router_tokens_per_expert=[0, 0, 3191, 0, 0, 4089, 0, 0, 0, 0, 0, 267, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 0, 4096, 0, 0, 4032, 0, 4000, 0, 0, 0, 0, 0, 0, 0, 0, 2, 601, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 241, 4096, 0] payload_rows_per_dst=[6438, 7844, 8104, 8064, 8182, 8000, 8192, 6958] dup_rows_per_dst=[1536, 534, 0, 0, 1146, 0, 0, 538] combine_rows_per_src=[6438, 7844, 8104, 8064, 8182, 8000, 8192, 6958] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63132 payload_rows_remote=54984 dup_rows=2404 dup_rows_remote=1514 dedup_ratio=0.036682 router_tokens_per_expert=[0, 0, 3066, 0, 0, 4095, 0, 0, 1, 0, 0, 446, 1, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 4095, 0, 0, 4035, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 28, 430, 0, 4061, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 266, 4095, 0] payload_rows_per_dst=[6132, 8190, 8124, 8070, 8148, 8086, 8190, 8192] dup_rows_per_dst=[0, 896, 0, 0, 890, 4, 0, 614] combine_rows_per_src=[6132, 8190, 8124, 8070, 8148, 8086, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62468 payload_rows_remote=54448 dup_rows=3068 dup_rows_remote=3066 dedup_ratio=0.046814 router_tokens_per_expert=[0, 0, 2756, 0, 0, 4096, 0, 0, 3, 0, 0, 639, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 0, 4095, 0, 0, 4010, 0, 4057, 0, 0, 1, 0, 0, 0, 0, 0, 9, 358, 0, 4057, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 489, 4094, 0] payload_rows_per_dst=[5512, 8192, 8114, 8020, 8134, 8114, 8190, 8192] dup_rows_per_dst=[0, 1284, 0, 2, 714, 8, 0, 1060] combine_rows_per_src=[5512, 8192, 8114, 8020, 8134, 8114, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=385896448 dispatch_meta_bytes=229400 combine_payload_bytes=385896448 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=386125848 ingress_bytes=606097408 link_bytes=606097408 token_bytes=772022296 weight_movement_bytes=220200960 total_nvlink_bytes=992223256 weight_to_token_ratio=0.285226 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.054103078e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=370012160 dispatch_meta_bytes=215352 combine_payload_bytes=370012160 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=370227512 ingress_bytes=590213120 link_bytes=590213120 token_bytes=740239672 weight_movement_bytes=220200960 total_nvlink_bytes=960440632 weight_to_token_ratio=0.297473 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=28160 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.026477689e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48538 payload_rows_remote=40346 dup_rows=16998 dup_rows_remote=3534 dedup_ratio=0.259369 router_tokens_per_expert=[0, 0, 2642, 0, 0, 4092, 0, 0, 0, 0, 0, 760, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 0, 4096, 0, 0, 3992, 0, 4041, 0, 0, 3, 0, 0, 0, 0, 0, 11, 501, 0, 4067, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 359, 4094, 0] payload_rows_per_dst=[8192, 1520, 8132, 8084, 5510, 8082, 8192, 826] dup_rows_per_dst=[13464, 0, 806, 2086, 638, 4, 0, 0] combine_rows_per_src=[8192, 1520, 8132, 8084, 5510, 8082, 8192, 826] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=391888896 dispatch_meta_bytes=230240 combine_payload_bytes=391888896 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=392119136 ingress_bytes=722190336 link_bytes=722190336 token_bytes=784008032 weight_movement_bytes=330301440 total_nvlink_bytes=1114309472 weight_to_token_ratio=0.421299 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.256007774e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=386627584 dispatch_meta_bytes=228632 combine_payload_bytes=386627584 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=386856216 ingress_bytes=496728064 link_bytes=496728064 token_bytes=773483800 weight_movement_bytes=110100480 total_nvlink_bytes=883584280 weight_to_token_ratio=0.142344 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=28544 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.638918010e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=394125312 dispatch_meta_bytes=225992 combine_payload_bytes=394125312 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=394351304 ingress_bytes=724426752 link_bytes=724426752 token_bytes=788476616 weight_movement_bytes=330301440 total_nvlink_bytes=1118778056 weight_to_token_ratio=0.418911 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=29440 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.259897269e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=390283264 dispatch_meta_bytes=230056 combine_payload_bytes=390283264 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=390513320 ingress_bytes=720584704 link_bytes=720584704 token_bytes=780796584 weight_movement_bytes=330301440 total_nvlink_bytes=1111098024 weight_to_token_ratio=0.423031 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.253215316e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61676 payload_rows_remote=54236 dup_rows=3860 dup_rows_remote=3854 dedup_ratio=0.058899 router_tokens_per_expert=[0, 40, 0, 0, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 3728, 10, 0, 0, 0, 0, 4061, 3639, 0, 4082, 0, 0, 0, 0, 3, 0, 3720, 0, 0, 4030, 0, 0, 0, 118, 0, 0, 0, 3421, 41, 0, 0, 0, 1610, 0, 0, 0, 4053, 0] payload_rows_per_dst=[8172, 8164, 8064, 7476, 7278, 7440, 6890, 8192] dup_rows_per_dst=[348, 0, 102, 0, 0, 6, 270, 3134] combine_rows_per_src=[8172, 8164, 8064, 7476, 7278, 7440, 6890, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=289200128 dispatch_meta_bytes=175520 combine_payload_bytes=289200128 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=289375648 ingress_bytes=509401088 link_bytes=509401088 token_bytes=578575776 weight_movement_bytes=220200960 total_nvlink_bytes=798776736 weight_to_token_ratio=0.380591 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=23808 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.859322741e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62930 payload_rows_remote=56448 dup_rows=2606 dup_rows_remote=2410 dedup_ratio=0.039764 router_tokens_per_expert=[0, 4, 0, 0, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 4009, 0, 0, 0, 0, 0, 4093, 3883, 0, 4096, 0, 0, 0, 0, 15, 0, 3996, 0, 0, 4049, 0, 0, 0, 86, 0, 0, 0, 3214, 39, 0, 0, 0, 1007, 0, 0, 0, 4090, 0] payload_rows_per_dst=[8190, 8192, 8098, 8018, 7766, 7992, 6482, 8192] dup_rows_per_dst=[350, 0, 28, 0, 0, 30, 196, 2002] combine_rows_per_src=[8190, 8192, 8098, 8018, 7766, 7992, 6482, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58434 payload_rows_remote=50242 dup_rows=7102 dup_rows_remote=2604 dedup_ratio=0.108368 router_tokens_per_expert=[0, 2, 0, 0, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 0, 0, 0, 3466, 1, 0, 0, 0, 0, 4082, 3520, 0, 4096, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 3975, 0, 0, 0, 117, 0, 0, 0, 2820, 30, 0, 0, 0, 2271, 0, 0, 0, 4074, 0] payload_rows_per_dst=[5884, 8192, 7950, 6934, 7348, 8124, 5810, 8192] dup_rows_per_dst=[234, 0, 188, 0, 2058, 0, 124, 4498] combine_rows_per_src=[5884, 8192, 7950, 6934, 7348, 8124, 5810, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60278 payload_rows_remote=52488 dup_rows=5258 dup_rows_remote=5052 dedup_ratio=0.080231 router_tokens_per_expert=[0, 6, 0, 0, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 0, 0, 0, 0, 0, 0, 3969, 3, 0, 0, 0, 0, 4029, 3515, 0, 4094, 0, 0, 0, 0, 1, 0, 4073, 0, 0, 3895, 0, 0, 0, 96, 0, 0, 0, 2578, 39, 0, 0, 0, 2126, 0, 0, 0, 4088, 0] payload_rows_per_dst=[8068, 8188, 7790, 7944, 7030, 8146, 7542, 5570] dup_rows_per_dst=[308, 0, 206, 0, 0, 2, 3954, 788] combine_rows_per_src=[8068, 8188, 7790, 7944, 7030, 8146, 7542, 5570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=388763648 dispatch_meta_bytes=232360 combine_payload_bytes=388763648 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=388996008 ingress_bytes=719065088 link_bytes=719065088 token_bytes=777759656 weight_movement_bytes=330301440 total_nvlink_bytes=1108061096 weight_to_token_ratio=0.424683 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.250572454e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57876 payload_rows_remote=49684 dup_rows=7660 dup_rows_remote=7660 dedup_ratio=0.116882 router_tokens_per_expert=[0, 3, 0, 0, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 3919, 1, 0, 0, 0, 0, 4077, 3843, 0, 4096, 0, 0, 0, 0, 2, 0, 4000, 0, 0, 4038, 0, 0, 0, 56, 0, 0, 0, 3465, 37, 0, 0, 0, 929, 0, 0, 0, 4092, 0] payload_rows_per_dst=[8160, 8192, 8076, 7840, 7686, 8006, 8058, 1858] dup_rows_per_dst=[302, 0, 118, 0, 0, 928, 6312, 0] combine_rows_per_src=[8160, 8192, 8076, 7840, 7686, 8006, 8058, 1858] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=404619264 dispatch_meta_bytes=235432 combine_payload_bytes=404619264 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=404854696 ingress_bytes=514719744 link_bytes=514719744 token_bytes=809473960 weight_movement_bytes=110100480 total_nvlink_bytes=919574440 weight_to_token_ratio=0.136015 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=25344 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.951822917e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=360134656 dispatch_meta_bytes=211384 combine_payload_bytes=360134656 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=360346040 ingress_bytes=800536576 link_bytes=800536576 token_bytes=720480696 weight_movement_bytes=440401920 total_nvlink_bytes=1160882616 weight_to_token_ratio=0.611261 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=21504 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.392264771e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62842 payload_rows_remote=55264 dup_rows=2694 dup_rows_remote=2694 dedup_ratio=0.041107 router_tokens_per_expert=[0, 1, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 0, 0, 4023, 3, 0, 0, 0, 0, 4094, 3789, 0, 4095, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 4045, 0, 0, 0, 25, 0, 0, 0, 3194, 37, 0, 0, 0, 1043, 0, 0, 0, 4095, 0] payload_rows_per_dst=[8190, 8190, 8092, 8052, 7578, 8126, 6422, 8192] dup_rows_per_dst=[372, 0, 148, 0, 0, 0, 90, 2084] combine_rows_per_src=[8190, 8190, 8092, 8052, 7578, 8126, 6422, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62784 payload_rows_remote=54730 dup_rows=2752 dup_rows_remote=2752 dedup_ratio=0.041992 router_tokens_per_expert=[0, 21, 0, 0, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 4025, 2, 0, 0, 0, 0, 4084, 3901, 0, 4094, 0, 0, 0, 0, 2, 0, 3881, 0, 0, 4039, 0, 0, 0, 9, 0, 0, 0, 3260, 37, 0, 0, 0, 1129, 0, 0, 0, 4091, 0] payload_rows_per_dst=[8182, 8188, 8078, 8054, 7802, 7762, 6526, 8192] dup_rows_per_dst=[344, 0, 70, 0, 0, 4, 86, 2248] combine_rows_per_src=[8182, 8188, 8078, 8054, 7802, 7762, 6526, 8192] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63722 payload_rows_remote=55530 dup_rows=1814 dup_rows_remote=1810 dedup_ratio=0.027679 router_tokens_per_expert=[0, 11, 0, 0, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4063, 2, 0, 0, 0, 0, 4074, 4094, 0, 4094, 0, 0, 0, 0, 52, 0, 4096, 0, 0, 4023, 0, 0, 0, 64, 0, 0, 0, 3434, 26, 7, 0, 0, 2946, 0, 0, 0, 1494, 0] payload_rows_per_dst=[8192, 8188, 8192, 8130, 8148, 8046, 6930, 7896] dup_rows_per_dst=[590, 0, 4, 0, 0, 104, 132, 984] combine_rows_per_src=[8192, 8188, 8192, 8130, 8148, 8046, 6930, 7896] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64044 payload_rows_remote=56644 dup_rows=1492 dup_rows_remote=1312 dedup_ratio=0.022766 router_tokens_per_expert=[0, 5, 0, 0, 330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 0, 0, 0, 0, 4094, 4094, 0, 4096, 0, 0, 0, 0, 114, 0, 4096, 0, 0, 4051, 0, 0, 0, 59, 0, 0, 0, 3687, 44, 0, 0, 0, 1455, 0, 0, 0, 2599, 0] payload_rows_per_dst=[8192, 8188, 8192, 8088, 8188, 8104, 7400, 7692] dup_rows_per_dst=[670, 0, 0, 0, 0, 226, 180, 416] combine_rows_per_src=[8192, 8188, 8192, 8088, 8188, 8104, 7400, 7692] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=403113984 dispatch_meta_bytes=229392 combine_payload_bytes=403113984 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=403343376 ingress_bytes=623314944 link_bytes=623314944 token_bytes=806457360 weight_movement_bytes=220200960 total_nvlink_bytes=1026658320 weight_to_token_ratio=0.273047 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.084047205e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=371961856 dispatch_meta_bytes=227712 combine_payload_bytes=371961856 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=372189568 ingress_bytes=592162816 link_bytes=592162816 token_bytes=744151424 weight_movement_bytes=220200960 total_nvlink_bytes=964352384 weight_to_token_ratio=0.295909 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=16512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.029868531e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63026 payload_rows_remote=54834 dup_rows=2510 dup_rows_remote=2002 dedup_ratio=0.038300 router_tokens_per_expert=[0, 18, 0, 0, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4059, 2, 0, 0, 0, 0, 4085, 4096, 0, 4094, 0, 0, 0, 0, 45, 0, 4096, 0, 0, 4044, 0, 0, 0, 44, 0, 0, 0, 3818, 44, 2, 0, 0, 1926, 0, 0, 0, 2156, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8192, 7428, 7648, 6992] dup_rows_per_dst=[508, 12, 2, 616, 722, 220, 168, 262] combine_rows_per_src=[8192, 8192, 8192, 8190, 8192, 7428, 7648, 6992] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64292 payload_rows_remote=56150 dup_rows=1244 dup_rows_remote=1244 dedup_ratio=0.018982 router_tokens_per_expert=[0, 21, 0, 0, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 2, 0, 0, 0, 0, 4093, 4096, 0, 4094, 0, 0, 0, 0, 46, 0, 4096, 0, 0, 4049, 0, 0, 0, 6, 0, 0, 0, 3671, 37, 0, 0, 0, 1695, 0, 0, 0, 2542, 0] payload_rows_per_dst=[8192, 8192, 8192, 8142, 8186, 8098, 7350, 7940] dup_rows_per_dst=[540, 0, 0, 0, 0, 92, 78, 534] combine_rows_per_src=[8192, 8192, 8192, 8142, 8186, 8098, 7350, 7940] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59986 payload_rows_remote=53318 dup_rows=5550 dup_rows_remote=3868 dedup_ratio=0.084686 router_tokens_per_expert=[0, 2132, 0, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 25, 0, 0, 0, 1, 0, 0, 0, 0, 148, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1721, 0, 46, 2408, 0, 0, 4096, 0, 0, 0, 1930, 0, 0, 3886, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4264, 8184, 8192, 8182, 8192, 6668, 8112, 8192] dup_rows_per_dst=[0, 0, 52, 296, 0, 1682, 3520, 0] combine_rows_per_src=[4264, 8184, 8192, 8182, 8192, 6668, 8112, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=376233984 dispatch_meta_bytes=230160 combine_payload_bytes=376233984 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=376464144 ingress_bytes=596434944 link_bytes=596434944 token_bytes=752698128 weight_movement_bytes=220200960 total_nvlink_bytes=972899088 weight_to_token_ratio=0.292549 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=32128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.037298464e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56804 payload_rows_remote=48650 dup_rows=8732 dup_rows_remote=8454 dedup_ratio=0.133240 router_tokens_per_expert=[0, 16, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 3938, 2, 0, 0, 0, 0, 4064, 3852, 0, 4094, 0, 0, 0, 0, 3, 0, 3994, 0, 0, 3989, 0, 0, 0, 71, 0, 0, 0, 3266, 50, 0, 0, 0, 1197, 0, 0, 0, 4061, 0] payload_rows_per_dst=[8154, 8192, 7980, 8014, 7930, 7564, 6576, 2394] dup_rows_per_dst=[278, 38, 548, 3204, 3060, 1406, 198, 0] combine_rows_per_src=[8154, 8192, 7980, 8014, 7930, 7564, 6576, 2394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=356134912 dispatch_meta_bytes=229376 combine_payload_bytes=356134912 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=356364288 ingress_bytes=576335872 link_bytes=576335872 token_bytes=712499200 weight_movement_bytes=220200960 total_nvlink_bytes=932700160 weight_to_token_ratio=0.309054 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.002342872e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=396132352 dispatch_meta_bytes=231832 combine_payload_bytes=396132352 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=396364184 ingress_bytes=836534272 link_bytes=836534272 token_bytes=792496536 weight_movement_bytes=440401920 total_nvlink_bytes=1232898456 weight_to_token_ratio=0.555715 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=29952 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.454870685e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=398039040 dispatch_meta_bytes=229360 combine_payload_bytes=398039040 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=398268400 ingress_bytes=508139520 link_bytes=508139520 token_bytes=796307440 weight_movement_bytes=110100480 total_nvlink_bytes=906407920 weight_to_token_ratio=0.138264 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.837381999e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=406024192 dispatch_meta_bytes=231824 combine_payload_bytes=406024192 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=406256016 ingress_bytes=516124672 link_bytes=516124672 token_bytes=812280208 weight_movement_bytes=110100480 total_nvlink_bytes=922380688 weight_to_token_ratio=0.135545 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=29568 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.976256925e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=392304640 dispatch_meta_bytes=229928 combine_payload_bytes=392304640 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=392534568 ingress_bytes=502405120 link_bytes=502405120 token_bytes=784839208 weight_movement_bytes=110100480 total_nvlink_bytes=894939688 weight_to_token_ratio=0.140284 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31104 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.737651351e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62054 payload_rows_remote=55578 dup_rows=3482 dup_rows_remote=1910 dedup_ratio=0.053131 router_tokens_per_expert=[0, 127, 0, 4095, 0, 4012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 133, 0, 41, 1401, 0, 0, 4096, 0, 0, 0, 2931, 0, 0, 3621, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8072, 8184, 8192, 8192, 8192, 6476, 6554, 8192] dup_rows_per_dst=[206, 0, 52, 0, 0, 1572, 1652, 0] combine_rows_per_src=[8072, 8184, 8192, 8192, 8192, 6476, 6554, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=348723200 dispatch_meta_bytes=228416 combine_payload_bytes=348723200 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=348951616 ingress_bytes=458823680 link_bytes=458823680 token_bytes=697674816 weight_movement_bytes=110100480 total_nvlink_bytes=807775296 weight_to_token_ratio=0.157811 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=31488 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.979698430e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58942 payload_rows_remote=50758 dup_rows=6594 dup_rows_remote=472 dedup_ratio=0.100616 router_tokens_per_expert=[0, 264, 0, 4096, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 220, 0, 0, 587, 0, 0, 4096, 0, 0, 0, 3142, 0, 0, 4011, 0, 0, 0, 0, 5, 0, 0, 0] payload_rows_per_dst=[8184, 8192, 8192, 8192, 8192, 1614, 8184, 8192] dup_rows_per_dst=[458, 0, 4, 0, 0, 0, 6122, 10] combine_rows_per_src=[8184, 8192, 8192, 8192, 8192, 1614, 8184, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57300 payload_rows_remote=49108 dup_rows=8236 dup_rows_remote=8236 dedup_ratio=0.125671 router_tokens_per_expert=[0, 340, 0, 4096, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 335, 0, 0, 1139, 0, 0, 4096, 0, 0, 0, 2635, 0, 0, 3794, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5284, 8192, 8192, 8188, 8192, 2918, 8142, 8192] dup_rows_per_dst=[570, 12, 0, 10, 0, 30, 7614, 0] combine_rows_per_src=[5284, 8192, 8192, 8188, 8192, 2918, 8142, 8192] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60666 payload_rows_remote=52474 dup_rows=4870 dup_rows_remote=4870 dedup_ratio=0.074310 router_tokens_per_expert=[0, 2514, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1916, 0, 1, 2088, 0, 0, 4096, 0, 0, 0, 1625, 0, 0, 3987, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[5028, 8192, 8192, 8190, 8192, 6518, 8162, 8192] dup_rows_per_dst=[0, 0, 0, 314, 0, 1492, 3062, 2] combine_rows_per_src=[5028, 8192, 8192, 8190, 8192, 6518, 8162, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=393050112 dispatch_meta_bytes=227344 combine_payload_bytes=393050112 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=393277456 ingress_bytes=503150592 link_bytes=503150592 token_bytes=786327568 weight_movement_bytes=110100480 total_nvlink_bytes=896428048 weight_to_token_ratio=0.140019 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=30592 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.750616335e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=402483200 dispatch_meta_bytes=229576 combine_payload_bytes=402483200 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=402712776 ingress_bytes=512583680 link_bytes=512583680 token_bytes=805195976 weight_movement_bytes=110100480 total_nvlink_bytes=915296456 weight_to_token_ratio=0.136737 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.914673250e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=382183424 dispatch_meta_bytes=228744 combine_payload_bytes=382183424 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=382412168 ingress_bytes=492283904 link_bytes=492283904 token_bytes=764595592 weight_movement_bytes=110100480 total_nvlink_bytes=874696072 weight_to_token_ratio=0.143998 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=16000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.561626758e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=398383104 dispatch_meta_bytes=229952 combine_payload_bytes=398383104 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=398613056 ingress_bytes=508483584 link_bytes=508483584 token_bytes=796996160 weight_movement_bytes=110100480 total_nvlink_bytes=907096640 weight_to_token_ratio=0.138144 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=20992 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.843365838e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62628 payload_rows_remote=54436 dup_rows=2908 dup_rows_remote=2908 dedup_ratio=0.044373 router_tokens_per_expert=[0, 385, 0, 4093, 0, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 779, 0, 4, 1028, 0, 0, 4096, 0, 0, 0, 2763, 0, 0, 3316, 0, 0, 0, 2, 0, 0, 0, 0] payload_rows_per_dst=[8120, 8192, 8192, 8188, 8192, 8026, 5526, 8192] dup_rows_per_dst=[676, 0, 0, 0, 0, 2228, 0, 4] combine_rows_per_src=[8120, 8192, 8192, 8188, 8192, 8026, 5526, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62236 payload_rows_remote=54044 dup_rows=3300 dup_rows_remote=3300 dedup_ratio=0.050354 router_tokens_per_expert=[0, 121, 0, 4096, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 81, 0, 0, 1597, 0, 0, 4096, 0, 0, 0, 2468, 0, 0, 3962, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8160, 8192, 8192, 8190, 8192, 8182, 4936, 8192] dup_rows_per_dst=[202, 0, 0, 0, 0, 3098, 0, 0] combine_rows_per_src=[8160, 8192, 8192, 8190, 8192, 8182, 4936, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=363833344 dispatch_meta_bytes=204920 combine_payload_bytes=363833344 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=364038264 ingress_bytes=694134784 link_bytes=694134784 token_bytes=727871608 weight_movement_bytes=330301440 total_nvlink_bytes=1058173048 weight_to_token_ratio=0.453791 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=22400 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.207214555e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=352006144 dispatch_meta_bytes=229376 combine_payload_bytes=352006144 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=352235520 ingress_bytes=462106624 link_bytes=462106624 token_bytes=704241664 weight_movement_bytes=110100480 total_nvlink_bytes=814342144 weight_to_token_ratio=0.156339 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.036794226e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62550 payload_rows_remote=54358 dup_rows=2986 dup_rows_remote=2986 dedup_ratio=0.045563 router_tokens_per_expert=[0, 170, 0, 4096, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 121, 0, 1, 1418, 0, 0, 4096, 0, 0, 0, 2621, 0, 0, 3885, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[8168, 8192, 8192, 8192, 8192, 8180, 5242, 8192] dup_rows_per_dst=[314, 0, 0, 0, 0, 2670, 0, 2] combine_rows_per_src=[8168, 8192, 8192, 8192, 8192, 8180, 5242, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63582 payload_rows_remote=55390 dup_rows=1954 dup_rows_remote=1954 dedup_ratio=0.029816 router_tokens_per_expert=[0, 179, 0, 4095, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 158, 0, 10, 1153, 0, 0, 4096, 0, 0, 0, 3181, 0, 0, 3544, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[8152, 8190, 8192, 8192, 8192, 8110, 6362, 8192] dup_rows_per_dst=[304, 0, 28, 0, 0, 1620, 0, 2] combine_rows_per_src=[8152, 8190, 8192, 8192, 8192, 8110, 6362, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=390197248 dispatch_meta_bytes=229376 combine_payload_bytes=390197248 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=390426624 ingress_bytes=500297728 link_bytes=500297728 token_bytes=780623872 weight_movement_bytes=110100480 total_nvlink_bytes=890724352 weight_to_token_ratio=0.141042 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.701000338e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59264 payload_rows_remote=51072 dup_rows=6272 dup_rows_remote=6272 dedup_ratio=0.095703 router_tokens_per_expert=[0, 2247, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 147, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1684, 0, 0, 2387, 0, 0, 4096, 0, 0, 0, 1758, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8148, 8192, 8192, 8192, 8192, 6640, 3516, 8192] dup_rows_per_dst=[4466, 0, 10, 294, 0, 1502, 0, 0] combine_rows_per_src=[8148, 8192, 8192, 8192, 8192, 6640, 3516, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60564 payload_rows_remote=52372 dup_rows=4972 dup_rows_remote=4972 dedup_ratio=0.075867 router_tokens_per_expert=[0, 2646, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2037, 0, 0, 1802, 0, 0, 4096, 0, 0, 0, 1745, 0, 0, 3919, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5292, 8192, 8052, 8192, 8192, 6320, 8132, 8192] dup_rows_per_dst=[0, 0, 4, 274, 0, 1358, 3336, 0] combine_rows_per_src=[5292, 8192, 8052, 8192, 8192, 6320, 8132, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=376133632 dispatch_meta_bytes=229376 combine_payload_bytes=376133632 prefetch_bytes=0 reduce_bytes=0 egress_bytes=376363008 ingress_bytes=376133632 link_bytes=376363008 token_bytes=752496640 weight_movement_bytes=0 total_nvlink_bytes=752496640 weight_to_token_ratio=0.000000 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.545571720e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58988 payload_rows_remote=50796 dup_rows=6548 dup_rows_remote=6540 dedup_ratio=0.099915 router_tokens_per_expert=[0, 2744, 0, 4093, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 160, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2769, 0, 5, 1489, 0, 0, 4096, 0, 0, 0, 1594, 0, 0, 3525, 0, 0, 0, 2, 0, 0, 0, 0] payload_rows_per_dst=[7756, 8192, 8192, 8190, 8192, 7088, 3188, 8190] dup_rows_per_dst=[4608, 10, 8, 320, 0, 1602, 0, 0] combine_rows_per_src=[7756, 8192, 8192, 8190, 8192, 7088, 3188, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60688 payload_rows_remote=52498 dup_rows=4848 dup_rows_remote=1146 dedup_ratio=0.073975 router_tokens_per_expert=[0, 2877, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1778, 0, 0, 1539, 0, 0, 4096, 0, 0, 0, 1903, 0, 0, 4043, 0, 0, 0, 0, 1, 0, 0, 0] payload_rows_per_dst=[5754, 8192, 8192, 8192, 8192, 5784, 8190, 8192] dup_rows_per_dst=[0, 0, 8, 286, 0, 850, 3702, 2] combine_rows_per_src=[5754, 8192, 8192, 8192, 8192, 5784, 8190, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=366084096 dispatch_meta_bytes=229376 combine_payload_bytes=366084096 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=366313472 ingress_bytes=476184576 link_bytes=476184576 token_bytes=732397568 weight_movement_bytes=110100480 total_nvlink_bytes=842498048 weight_to_token_ratio=0.150329 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.281632966e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57038 payload_rows_remote=48846 dup_rows=8498 dup_rows_remote=402 dedup_ratio=0.129669 router_tokens_per_expert=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2267, 0, 155, 0, 0, 0, 0, 4034, 0, 0, 0, 0, 1745, 0, 0, 0, 0, 0, 4048, 0, 0, 0, 0, 4096, 0, 4096, 4061, 0, 0, 0, 8, 4096, 33, 0, 20, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 4534, 8068, 3490, 8192, 8178, 8192] dup_rows_per_dst=[0, 26, 0, 310, 0, 8096, 26, 40] combine_rows_per_src=[8192, 8192, 4534, 8068, 3490, 8192, 8178, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56794 payload_rows_remote=48834 dup_rows=8742 dup_rows_remote=1664 dedup_ratio=0.133392 router_tokens_per_expert=[0, 2495, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 142, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2461, 0, 5, 1549, 0, 0, 4096, 0, 0, 0, 1806, 0, 0, 3819, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[7960, 8190, 8192, 5864, 8192, 6596, 3612, 8188] dup_rows_per_dst=[7078, 0, 24, 204, 0, 1434, 0, 2] combine_rows_per_src=[7960, 8190, 8192, 5864, 8192, 6596, 3612, 8188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=375402496 dispatch_meta_bytes=229376 combine_payload_bytes=375402496 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=375631872 ingress_bytes=485502976 link_bytes=485502976 token_bytes=751034368 weight_movement_bytes=110100480 total_nvlink_bytes=861134848 weight_to_token_ratio=0.146598 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.443695268e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60466 payload_rows_remote=52276 dup_rows=5070 dup_rows_remote=4714 dedup_ratio=0.077362 router_tokens_per_expert=[0, 2591, 0, 4095, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 178, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1575, 0, 10, 2179, 0, 0, 4096, 0, 0, 0, 1954, 0, 0, 3785, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[5182, 8192, 8192, 8190, 8192, 7122, 7204, 8192] dup_rows_per_dst=[0, 0, 34, 356, 0, 2880, 1800, 0] combine_rows_per_src=[5182, 8192, 8192, 8190, 8192, 7122, 7204, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=364105728 dispatch_meta_bytes=229344 combine_payload_bytes=364105728 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=364335072 ingress_bytes=474206208 link_bytes=474206208 token_bytes=728440800 weight_movement_bytes=110100480 total_nvlink_bytes=838541280 weight_to_token_ratio=0.151145 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.247225892e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=376305664 dispatch_meta_bytes=214576 combine_payload_bytes=376305664 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=376520240 ingress_bytes=816707584 link_bytes=816707584 token_bytes=752825904 weight_movement_bytes=440401920 total_nvlink_bytes=1193227824 weight_to_token_ratio=0.584998 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=18432 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.420388814e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58556 payload_rows_remote=53786 dup_rows=6980 dup_rows_remote=6980 dedup_ratio=0.106506 router_tokens_per_expert=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1728, 0, 102, 0, 0, 0, 0, 3973, 0, 0, 0, 0, 2385, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 4096, 4094, 0, 0, 0, 3, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 4884, 7946, 4770, 8190, 8190, 8192] dup_rows_per_dst=[0, 2, 752, 204, 0, 6016, 4, 2] combine_rows_per_src=[8192, 8192, 4884, 7946, 4770, 8190, 8190, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=350128128 dispatch_meta_bytes=196992 combine_payload_bytes=350128128 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=350325120 ingress_bytes=680429568 link_bytes=680429568 token_bytes=700453248 weight_movement_bytes=330301440 total_nvlink_bytes=1030754688 weight_to_token_ratio=0.471554 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=16768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.183378930e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=350042112 dispatch_meta_bytes=201992 combine_payload_bytes=350042112 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=350244104 ingress_bytes=680343552 link_bytes=680343552 token_bytes=700286216 weight_movement_bytes=330301440 total_nvlink_bytes=1030587656 weight_to_token_ratio=0.471666 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=20352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.183229334e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=374714368 dispatch_meta_bytes=227960 combine_payload_bytes=374714368 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=374942328 ingress_bytes=484814848 link_bytes=484814848 token_bytes=749656696 weight_movement_bytes=110100480 total_nvlink_bytes=859757176 weight_to_token_ratio=0.146868 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=31616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.431727590e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57184 payload_rows_remote=48992 dup_rows=8352 dup_rows_remote=8352 dedup_ratio=0.127441 router_tokens_per_expert=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2186, 0, 123, 0, 0, 0, 0, 4025, 0, 0, 0, 0, 1861, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 0, 4096, 4093, 0, 0, 0, 1, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8050, 8180, 0, 8186, 8192] dup_rows_per_dst=[0, 0, 4372, 246, 3720, 0, 2, 12] combine_rows_per_src=[8192, 8192, 8192, 8050, 8180, 0, 8186, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=385538048 dispatch_meta_bytes=243064 combine_payload_bytes=385538048 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=385781112 ingress_bytes=495638528 link_bytes=495638528 token_bytes=771319160 weight_movement_bytes=110100480 total_nvlink_bytes=881419640 weight_to_token_ratio=0.142743 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=17024 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.619969187e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55574 payload_rows_remote=47382 dup_rows=9962 dup_rows_remote=9962 dedup_ratio=0.152008 router_tokens_per_expert=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 2142, 0, 160, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 1856, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 4096, 0, 4096, 4073, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8150, 4300, 6690, 3712, 8192, 8146, 8192] dup_rows_per_dst=[0, 0, 0, 242, 0, 9676, 44, 0] combine_rows_per_src=[8192, 8150, 4300, 6690, 3712, 8192, 8146, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57316 payload_rows_remote=49124 dup_rows=8220 dup_rows_remote=5558 dedup_ratio=0.125427 router_tokens_per_expert=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 1306, 0, 236, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 2556, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 4096, 0, 4096, 4090, 0, 0, 0, 9, 4096, 3, 0, 3, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8176, 8186, 0, 8186, 8192] dup_rows_per_dst=[0, 6, 2662, 472, 5056, 0, 18, 6] combine_rows_per_src=[8192, 8192, 8192, 8176, 8186, 0, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62366 payload_rows_remote=54200 dup_rows=3170 dup_rows_remote=2670 dedup_ratio=0.048370 router_tokens_per_expert=[0, 242, 0, 4096, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 535, 0, 4, 1083, 0, 0, 4096, 0, 0, 0, 2591, 0, 0, 3784, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[8166, 8190, 8108, 8192, 8192, 8156, 5188, 8174] dup_rows_per_dst=[500, 0, 0, 0, 0, 2656, 14, 0] combine_rows_per_src=[8166, 8190, 8108, 8192, 8192, 8156, 5188, 8174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=387387392 dispatch_meta_bytes=229376 combine_payload_bytes=387387392 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=387616768 ingress_bytes=497487872 link_bytes=497487872 token_bytes=775004160 weight_movement_bytes=110100480 total_nvlink_bytes=885104640 weight_to_token_ratio=0.142064 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.652132321e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=389638144 dispatch_meta_bytes=229376 combine_payload_bytes=389638144 prefetch_bytes=0 reduce_bytes=0 egress_bytes=389867520 ingress_bytes=389638144 link_bytes=389867520 token_bytes=779505664 weight_movement_bytes=0 total_nvlink_bytes=779505664 weight_to_token_ratio=0.000000 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.780437395e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=397035520 dispatch_meta_bytes=229376 combine_payload_bytes=397035520 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=397264896 ingress_bytes=507136000 link_bytes=507136000 token_bytes=794300416 weight_movement_bytes=110100480 total_nvlink_bytes=904400896 weight_to_token_ratio=0.138613 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.819929135e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64408 payload_rows_remote=56314 dup_rows=1128 dup_rows_remote=1128 dedup_ratio=0.017212 router_tokens_per_expert=[0, 0, 0, 0, 0, 4053, 0, 0, 4003, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 377, 0, 142, 0, 0, 0, 0, 3901, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 4096, 0, 3912, 4048, 0, 0, 0, 0, 4096, 36, 0, 11, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8106, 8028, 8170, 7802, 8192, 8094, 7824, 8192] dup_rows_per_dst=[0, 0, 680, 284, 70, 0, 72, 22] combine_rows_per_src=[8106, 8028, 8170, 7802, 8192, 8094, 7824, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60578 payload_rows_remote=52386 dup_rows=4958 dup_rows_remote=4958 dedup_ratio=0.075653 router_tokens_per_expert=[0, 0, 0, 0, 0, 4052, 0, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 200, 0, 149, 0, 0, 0, 0, 4026, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 4063, 0, 0, 0, 0, 4096, 0, 3925, 4068, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8104, 8124, 4330, 8052, 7566, 8126, 8084, 8192] dup_rows_per_dst=[0, 0, 194, 298, 38, 516, 3912, 0] combine_rows_per_src=[8104, 8124, 4330, 8052, 7566, 8126, 8084, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64726 payload_rows_remote=56736 dup_rows=810 dup_rows_remote=808 dedup_ratio=0.012360 router_tokens_per_expert=[0, 0, 0, 0, 0, 4061, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 265, 0, 102, 0, 0, 0, 0, 3861, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 4063, 0, 0, 0, 0, 4096, 0, 3995, 4090, 0, 0, 0, 0, 4096, 1, 0, 2, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8122, 8190, 8190, 7724, 8192, 8126, 7990, 8192] dup_rows_per_dst=[0, 0, 524, 202, 78, 0, 2, 4] combine_rows_per_src=[8122, 8190, 8190, 7724, 8192, 8126, 7990, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=388505600 dispatch_meta_bytes=227480 combine_payload_bytes=388505600 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=388733080 ingress_bytes=828907520 link_bytes=828907520 token_bytes=777238680 weight_movement_bytes=440401920 total_nvlink_bytes=1217640600 weight_to_token_ratio=0.566624 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=30976 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.441606509e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=351174656 dispatch_meta_bytes=229376 combine_payload_bytes=351174656 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=351404032 ingress_bytes=461275136 link_bytes=461275136 token_bytes=702578688 weight_movement_bytes=110100480 total_nvlink_bytes=812679168 weight_to_token_ratio=0.156709 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.022333282e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57090 payload_rows_remote=48900 dup_rows=8446 dup_rows_remote=8440 dedup_ratio=0.128876 router_tokens_per_expert=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1737, 0, 145, 0, 0, 0, 0, 3976, 0, 0, 0, 0, 2350, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 4096, 0, 4096, 4093, 0, 0, 0, 5, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 3476, 7956, 4700, 8192, 8190, 8192] dup_rows_per_dst=[0, 0, 0, 286, 0, 8152, 6, 2] combine_rows_per_src=[8192, 8192, 3476, 7956, 4700, 8192, 8190, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=339634176 dispatch_meta_bytes=229376 combine_payload_bytes=339634176 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=339863552 ingress_bytes=449734656 link_bytes=449734656 token_bytes=679497728 weight_movement_bytes=110100480 total_nvlink_bytes=789598208 weight_to_token_ratio=0.162032 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.821625354e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57262 payload_rows_remote=49086 dup_rows=8274 dup_rows_remote=5606 dedup_ratio=0.126251 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4039, 0, 0, 24, 0, 4096, 4074, 4085, 0, 0, 0, 41, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1337, 0, 0, 0, 2254, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8078, 48, 8192, 8192, 8192, 8176, 8192, 8192] dup_rows_per_dst=[0, 0, 82, 1042, 0, 2668, 4482, 0] combine_rows_per_src=[8078, 48, 8192, 8192, 8192, 8176, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57096 payload_rows_remote=48904 dup_rows=8440 dup_rows_remote=8440 dedup_ratio=0.128784 router_tokens_per_expert=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1892, 0, 179, 0, 0, 0, 0, 4036, 0, 0, 0, 0, 2126, 0, 0, 0, 0, 0, 4055, 0, 0, 0, 0, 4096, 0, 4096, 4087, 1, 0, 0, 3, 4096, 2, 0, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8072, 8186, 0, 8144, 8118] dup_rows_per_dst=[0, 4, 3784, 464, 4176, 0, 12, 0] combine_rows_per_src=[8192, 8192, 8192, 8072, 8186, 0, 8144, 8118] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57510 payload_rows_remote=49534 dup_rows=8026 dup_rows_remote=7648 dedup_ratio=0.122467 router_tokens_per_expert=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2000, 0, 193, 0, 0, 0, 0, 3984, 0, 0, 0, 0, 2018, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 4096, 0, 4096, 4093, 1, 0, 0, 7, 4096, 1, 0, 6, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7976, 7748, 830, 8188, 8192] dup_rows_per_dst=[0, 4, 4004, 378, 3612, 0, 16, 12] combine_rows_per_src=[8192, 8192, 8192, 7976, 7748, 830, 8188, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=352120832 dispatch_meta_bytes=218728 combine_payload_bytes=352120832 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=352339560 ingress_bytes=462221312 link_bytes=462221312 token_bytes=704460392 weight_movement_bytes=110100480 total_nvlink_bytes=814560872 weight_to_token_ratio=0.156291 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=17536 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.038788839e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=350515200 dispatch_meta_bytes=229360 combine_payload_bytes=350515200 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=350744560 ingress_bytes=680816640 link_bytes=680816640 token_bytes=701259760 weight_movement_bytes=330301440 total_nvlink_bytes=1031561200 weight_to_token_ratio=0.471012 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.184052112e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57326 payload_rows_remote=49134 dup_rows=8210 dup_rows_remote=8210 dedup_ratio=0.125275 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 1, 0, 4096, 4078, 4095, 0, 0, 0, 37, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1574, 1, 0, 0, 2126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8172, 2, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 74, 772, 0, 3148, 4216, 0] combine_rows_per_src=[8172, 2, 8192, 8192, 8192, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=351848448 dispatch_meta_bytes=218768 combine_payload_bytes=351848448 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=352067216 ingress_bytes=461948928 link_bytes=461948928 token_bytes=703915664 weight_movement_bytes=110100480 total_nvlink_bytes=814016144 weight_to_token_ratio=0.156411 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=19328 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.034051633e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=350543872 dispatch_meta_bytes=229376 combine_payload_bytes=350543872 prefetch_bytes=0 reduce_bytes=0 egress_bytes=350773248 ingress_bytes=350543872 link_bytes=350773248 token_bytes=701317120 weight_movement_bytes=0 total_nvlink_bytes=701317120 weight_to_token_ratio=0.000000 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.100523706e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=355059712 dispatch_meta_bytes=228728 combine_payload_bytes=355059712 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=355288440 ingress_bytes=575260672 link_bytes=575260672 token_bytes=710348152 weight_movement_bytes=220200960 total_nvlink_bytes=930549112 weight_to_token_ratio=0.309990 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31488 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.000472923e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56396 payload_rows_remote=48204 dup_rows=9140 dup_rows_remote=974 dedup_ratio=0.139465 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4042, 0, 0, 0, 0, 4096, 4040, 4096, 0, 0, 0, 37, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1654, 0, 0, 0, 2065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8084, 8192, 8192, 8106, 8192, 3308, 4130, 8192] dup_rows_per_dst=[0, 8166, 74, 900, 0, 0, 0, 0] combine_rows_per_src=[8084, 8192, 8192, 8106, 8192, 3308, 4130, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=352192512 dispatch_meta_bytes=229376 combine_payload_bytes=352192512 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=352421888 ingress_bytes=462292992 link_bytes=462292992 token_bytes=704614400 weight_movement_bytes=110100480 total_nvlink_bytes=814714880 weight_to_token_ratio=0.156256 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.040035472e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56494 payload_rows_remote=48314 dup_rows=9042 dup_rows_remote=9032 dedup_ratio=0.137970 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3990, 0, 0, 0, 0, 4090, 4038, 4092, 0, 0, 0, 41, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1766, 0, 0, 0, 1960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7980, 0, 7578, 8192, 8192, 8186, 8186, 8180] dup_rows_per_dst=[0, 0, 76, 1626, 0, 3530, 3800, 10] combine_rows_per_src=[7980, 0, 7578, 8192, 8192, 8186, 8186, 8180] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56238 payload_rows_remote=48046 dup_rows=9298 dup_rows_remote=9204 dedup_ratio=0.141876 router_tokens_per_expert=[0, 0, 0, 0, 0, 2, 0, 4005, 0, 0, 2, 0, 4096, 4054, 4094, 0, 0, 0, 47, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 4096, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1770, 0, 0, 0, 1863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[8014, 8192, 8192, 8190, 8192, 3540, 3726, 8192] dup_rows_per_dst=[0, 8108, 94, 1094, 0, 0, 0, 2] combine_rows_per_src=[8014, 8192, 8192, 8190, 8192, 3540, 3726, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=345526272 dispatch_meta_bytes=196712 combine_payload_bytes=345526272 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=345722984 ingress_bytes=896028672 link_bytes=896028672 token_bytes=691249256 weight_movement_bytes=550502400 total_nvlink_bytes=1241751656 weight_to_token_ratio=0.796388 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=15488 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.558341232e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57288 payload_rows_remote=49098 dup_rows=8248 dup_rows_remote=5756 dedup_ratio=0.125854 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 4096, 4089, 4096, 0, 0, 0, 30, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2306, 0, 0, 0, 1252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8138, 0, 8192, 8192, 8192, 8192, 8190, 8192] dup_rows_per_dst=[0, 0, 60, 1084, 0, 4612, 2492, 0] combine_rows_per_src=[8138, 0, 8192, 8192, 8192, 8192, 8190, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=346314752 dispatch_meta_bytes=229384 combine_payload_bytes=346314752 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=346544136 ingress_bytes=566515712 link_bytes=566515712 token_bytes=692858888 weight_movement_bytes=220200960 total_nvlink_bytes=913059848 weight_to_token_ratio=0.317815 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.852639990e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=403658752 dispatch_meta_bytes=229768 combine_payload_bytes=403658752 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=403888520 ingress_bytes=733960192 link_bytes=733960192 token_bytes=807547272 weight_movement_bytes=330301440 total_nvlink_bytes=1137848712 weight_to_token_ratio=0.409018 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.276477490e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64962 payload_rows_remote=56778 dup_rows=574 dup_rows_remote=444 dedup_ratio=0.008759 router_tokens_per_expert=[0, 0, 0, 0, 0, 4052, 0, 0, 4070, 0, 0, 3, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 71, 0, 181, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 4096, 0, 3932, 4084, 0, 0, 0, 0, 4096, 3, 0, 2, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8104, 8144, 8184, 8146, 8192, 8136, 7864, 8192] dup_rows_per_dst=[0, 2, 130, 362, 70, 0, 6, 4] combine_rows_per_src=[8104, 8144, 8184, 8146, 8192, 8136, 7864, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64564 payload_rows_remote=56516 dup_rows=972 dup_rows_remote=972 dedup_ratio=0.014832 router_tokens_per_expert=[0, 0, 0, 0, 0, 4059, 0, 0, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 333, 0, 119, 0, 0, 0, 0, 3927, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 4096, 0, 3920, 4082, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8118, 8048, 8192, 7854, 8192, 8128, 7840, 8192] dup_rows_per_dst=[0, 0, 640, 238, 88, 0, 0, 6] combine_rows_per_src=[8118, 8048, 8192, 7854, 8192, 8128, 7840, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=375502848 dispatch_meta_bytes=229376 combine_payload_bytes=375502848 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=375732224 ingress_bytes=485603328 link_bytes=485603328 token_bytes=751235072 weight_movement_bytes=110100480 total_nvlink_bytes=861335552 weight_to_token_ratio=0.146559 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.445440554e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=406683648 dispatch_meta_bytes=230176 combine_payload_bytes=406683648 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=406913824 ingress_bytes=957186048 link_bytes=957186048 token_bytes=813597472 weight_movement_bytes=550502400 total_nvlink_bytes=1364099872 weight_to_token_ratio=0.676627 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=30720 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.664703968e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64614 payload_rows_remote=56422 dup_rows=922 dup_rows_remote=774 dedup_ratio=0.014069 router_tokens_per_expert=[0, 0, 0, 0, 0, 4059, 0, 0, 4058, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 248, 0, 140, 0, 0, 0, 0, 3887, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 3920, 4092, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8118, 8118, 8186, 7778, 8192, 8190, 7840, 8192] dup_rows_per_dst=[0, 0, 496, 276, 148, 0, 0, 2] combine_rows_per_src=[8118, 8118, 8186, 7778, 8192, 8190, 7840, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64348 payload_rows_remote=56596 dup_rows=1188 dup_rows_remote=878 dedup_ratio=0.018127 router_tokens_per_expert=[0, 0, 0, 0, 0, 4058, 0, 0, 4088, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 403, 0, 155, 0, 0, 0, 0, 3876, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 4096, 0, 3793, 4092, 0, 0, 0, 0, 4096, 3, 0, 2, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8116, 8180, 8186, 7752, 8192, 8144, 7586, 8192] dup_rows_per_dst=[0, 0, 808, 310, 60, 0, 6, 4] combine_rows_per_src=[8116, 8180, 8186, 7752, 8192, 8144, 7586, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=406984704 dispatch_meta_bytes=228888 combine_payload_bytes=406984704 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=407213592 ingress_bytes=517085184 link_bytes=517085184 token_bytes=814198296 weight_movement_bytes=110100480 total_nvlink_bytes=924298776 weight_to_token_ratio=0.135226 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=30720 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.992961809e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=405106688 dispatch_meta_bytes=229952 combine_payload_bytes=405106688 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=405336640 ingress_bytes=515207168 link_bytes=515207168 token_bytes=810443328 weight_movement_bytes=110100480 total_nvlink_bytes=920543808 weight_to_token_ratio=0.135852 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.960300022e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=404432896 dispatch_meta_bytes=228784 combine_payload_bytes=404432896 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=404661680 ingress_bytes=514533376 link_bytes=514533376 token_bytes=809094576 weight_movement_bytes=110100480 total_nvlink_bytes=919195056 weight_to_token_ratio=0.136079 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.948581671e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=405680128 dispatch_meta_bytes=229896 combine_payload_bytes=405680128 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=405910024 ingress_bytes=515780608 link_bytes=515780608 token_bytes=811590152 weight_movement_bytes=110100480 total_nvlink_bytes=921690632 weight_to_token_ratio=0.135660 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.970273086e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63398 payload_rows_remote=55306 dup_rows=2138 dup_rows_remote=1482 dedup_ratio=0.032623 router_tokens_per_expert=[0, 0, 0, 0, 0, 4046, 0, 0, 4065, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 233, 0, 186, 0, 0, 0, 0, 4029, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 4052, 0, 0, 0, 0, 4096, 0, 3840, 4078, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8092, 8136, 8188, 8060, 8192, 8076, 6504, 8150] dup_rows_per_dst=[656, 566, 438, 398, 74, 0, 6, 0] combine_rows_per_src=[8092, 8136, 8188, 8060, 8192, 8076, 6504, 8150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64734 payload_rows_remote=56542 dup_rows=802 dup_rows_remote=802 dedup_ratio=0.012238 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3914, 0, 0, 23, 0, 4096, 3877, 4073, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7828, 7754, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 46, 0, 0, 0, 0, 756, 0] combine_rows_per_src=[7828, 7754, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65178 payload_rows_remote=56986 dup_rows=358 dup_rows_remote=0 dedup_ratio=0.005463 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4046, 0, 0, 0, 0, 4096, 3967, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8092, 7934, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 358, 0] combine_rows_per_src=[8092, 7934, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63718 payload_rows_remote=55526 dup_rows=1818 dup_rows_remote=1818 dedup_ratio=0.027740 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3535, 0, 0, 0, 0, 4089, 3765, 4092, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7070, 7510, 8192, 8192, 8192, 8178, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 2, 0, 18, 1798, 0] combine_rows_per_src=[7070, 7510, 8192, 8192, 8192, 8178, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=396433408 dispatch_meta_bytes=227152 combine_payload_bytes=396433408 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=396660560 ingress_bytes=506533888 link_bytes=506533888 token_bytes=793093968 weight_movement_bytes=110100480 total_nvlink_bytes=903194448 weight_to_token_ratio=0.138824 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.809457417e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=405293056 dispatch_meta_bytes=229376 combine_payload_bytes=405293056 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=405522432 ingress_bytes=625494016 link_bytes=625494016 token_bytes=810815488 weight_movement_bytes=220200960 total_nvlink_bytes=1031016448 weight_to_token_ratio=0.271580 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.087836970e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62034 payload_rows_remote=53854 dup_rows=3502 dup_rows_remote=3480 dedup_ratio=0.053436 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 3329, 0, 0, 0, 0, 0, 0, 4044, 24, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 19, 3111, 211, 0, 0, 4090, 0, 11, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1097, 0, 0, 0, 0, 3843, 658, 0] payload_rows_per_dst=[6658, 8088, 8192, 8192, 6424, 8180, 8192, 8108] dup_rows_per_dst=[0, 48, 0, 86, 258, 22, 0, 3088] combine_rows_per_src=[6658, 8088, 8192, 8192, 6424, 8180, 8192, 8108] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49312 payload_rows_remote=41120 dup_rows=16224 dup_rows_remote=15480 dedup_ratio=0.247559 router_tokens_per_expert=[0, 0, 0, 0, 0, 2, 0, 4057, 0, 0, 2, 0, 4096, 4016, 4094, 0, 0, 0, 37, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1662, 0, 0, 0, 2081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 866, 8192, 3324, 4162, 8192] dup_rows_per_dst=[744, 15406, 74, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 866, 8192, 3324, 4162, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59448 payload_rows_remote=51256 dup_rows=6088 dup_rows_remote=5192 dedup_ratio=0.092896 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 5, 0, 4096, 4085, 4095, 0, 0, 0, 40, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1493, 0, 0, 0, 2150, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8134, 6716, 8192, 8192, 8192, 6936, 4894, 8192] dup_rows_per_dst=[0, 2514, 80, 896, 0, 1724, 874, 0] combine_rows_per_src=[8134, 6716, 8192, 8192, 8192, 6936, 4894, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=344393728 dispatch_meta_bytes=229000 combine_payload_bytes=344393728 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=344622728 ingress_bytes=454494208 link_bytes=454494208 token_bytes=689016456 weight_movement_bytes=110100480 total_nvlink_bytes=799116936 weight_to_token_ratio=0.159794 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.904401792e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=351934464 dispatch_meta_bytes=219416 combine_payload_bytes=351934464 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=352153880 ingress_bytes=462034944 link_bytes=462034944 token_bytes=704088344 weight_movement_bytes=110100480 total_nvlink_bytes=814188824 weight_to_token_ratio=0.156373 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=17024 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.035547593e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62584 payload_rows_remote=55488 dup_rows=2952 dup_rows_remote=2512 dedup_ratio=0.045044 router_tokens_per_expert=[0, 0, 4095, 0, 0, 0, 3358, 0, 0, 0, 0, 0, 0, 4008, 1, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 1, 3447, 320, 0, 0, 4031, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 961, 0, 0, 0, 0, 3909, 404, 0] payload_rows_per_dst=[6716, 8016, 8192, 8190, 7096, 8062, 8192, 8120] dup_rows_per_dst=[0, 2, 2, 80, 440, 0, 0, 2428] combine_rows_per_src=[6716, 8016, 8192, 8190, 7096, 8062, 8192, 8120] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=386025472 dispatch_meta_bytes=229336 combine_payload_bytes=386025472 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=386254808 ingress_bytes=496125952 link_bytes=496125952 token_bytes=772280280 weight_movement_bytes=110100480 total_nvlink_bytes=882380760 weight_to_token_ratio=0.142565 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.628446292e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=294748160 dispatch_meta_bytes=226400 combine_payload_bytes=294748160 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=294974560 ingress_bytes=404848640 link_bytes=404848640 token_bytes=589722720 weight_movement_bytes=110100480 total_nvlink_bytes=699823200 weight_to_token_ratio=0.186699 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.040983711e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=367403008 dispatch_meta_bytes=225792 combine_payload_bytes=367403008 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=367628800 ingress_bytes=807804928 link_bytes=807804928 token_bytes=735031808 weight_movement_bytes=440401920 total_nvlink_bytes=1175433728 weight_to_token_ratio=0.599160 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=28672 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.404905631e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59752 payload_rows_remote=51656 dup_rows=5784 dup_rows_remote=5014 dedup_ratio=0.088257 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 3065, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 2780, 433, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1056, 0, 0, 0, 0, 3730, 1270, 0] payload_rows_per_dst=[7544, 8096, 8192, 8192, 6970, 8120, 8192, 4446] dup_rows_per_dst=[2702, 770, 0, 78, 2028, 0, 0, 206] combine_rows_per_src=[7544, 8096, 8192, 8192, 6970, 8120, 8192, 4446] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=397737984 dispatch_meta_bytes=232000 combine_payload_bytes=397737984 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=397969984 ingress_bytes=507838464 link_bytes=507838464 token_bytes=795707968 weight_movement_bytes=110100480 total_nvlink_bytes=905808448 weight_to_token_ratio=0.138368 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=25472 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.832146140e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62868 payload_rows_remote=54762 dup_rows=2668 dup_rows_remote=388 dedup_ratio=0.040710 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 3504, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 1, 3375, 215, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1354, 0, 0, 0, 0, 3221, 613, 0] payload_rows_per_dst=[7008, 8146, 8182, 8192, 6880, 8162, 8192, 8106] dup_rows_per_dst=[0, 0, 0, 86, 302, 0, 0, 2280] combine_rows_per_src=[7008, 8146, 8182, 8192, 6880, 8162, 8192, 8106] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=370270208 dispatch_meta_bytes=226680 combine_payload_bytes=370270208 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=370496888 ingress_bytes=480370688 link_bytes=480370688 token_bytes=740767096 weight_movement_bytes=110100480 total_nvlink_bytes=850867576 weight_to_token_ratio=0.148630 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.354436338e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60770 payload_rows_remote=52578 dup_rows=4766 dup_rows_remote=4766 dedup_ratio=0.072723 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 3016, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 2699, 380, 0, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 925, 0, 0, 0, 0, 3995, 1306, 0] payload_rows_per_dst=[6032, 8150, 8192, 8192, 5768, 8084, 8192, 8160] dup_rows_per_dst=[0, 0, 0, 84, 390, 0, 0, 4292] combine_rows_per_src=[6032, 8150, 8192, 8192, 5768, 8084, 8192, 8160] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58716 payload_rows_remote=50524 dup_rows=6820 dup_rows_remote=6816 dedup_ratio=0.104065 router_tokens_per_expert=[0, 0, 4094, 0, 0, 0, 3196, 0, 0, 0, 0, 0, 0, 4014, 3, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 3, 3457, 270, 0, 0, 4049, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1034, 0, 0, 0, 0, 3647, 758, 0] payload_rows_per_dst=[6392, 8030, 8192, 8188, 8118, 8102, 8192, 3502] dup_rows_per_dst=[0, 4, 4, 94, 6338, 298, 0, 82] combine_rows_per_src=[6392, 8030, 8192, 8188, 8118, 8102, 8192, 3502] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=392534016 dispatch_meta_bytes=220600 combine_payload_bytes=392534016 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=392754616 ingress_bytes=943036416 link_bytes=943036416 token_bytes=785288632 weight_movement_bytes=550502400 total_nvlink_bytes=1335791032 weight_to_token_ratio=0.701019 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=18816 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.640095430e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63648 payload_rows_remote=55470 dup_rows=1888 dup_rows_remote=394 dedup_ratio=0.028809 router_tokens_per_expert=[59, 0, 0, 0, 0, 4055, 0, 0, 0, 0, 0, 0, 8, 0, 4069, 0, 0, 0, 0, 0, 0, 1, 4062, 0, 0, 74, 4088, 0, 0, 0, 3497, 0, 0, 0, 0, 17, 1, 0, 0, 4092, 2, 0, 0, 4089, 0, 0, 745, 0, 0, 14, 0, 0, 0, 0, 99, 0, 0, 3796, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8154, 8126, 7022, 8192, 8178, 8192, 7592] dup_rows_per_dst=[36, 0, 0, 120, 28, 1494, 210, 0] combine_rows_per_src=[8192, 8154, 8126, 7022, 8192, 8178, 8192, 7592] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57776 payload_rows_remote=49586 dup_rows=7760 dup_rows_remote=768 dedup_ratio=0.118408 router_tokens_per_expert=[0, 0, 4095, 0, 0, 0, 3315, 0, 0, 0, 0, 0, 0, 4044, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 2, 3348, 391, 0, 0, 3965, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 722, 0, 0, 0, 0, 3929, 708, 0] payload_rows_per_dst=[8190, 8088, 8192, 7500, 6926, 7930, 8190, 2760] dup_rows_per_dst=[6992, 4, 4, 102, 556, 0, 2, 100] combine_rows_per_src=[8190, 8088, 8192, 7500, 6926, 7930, 8190, 2760] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62590 payload_rows_remote=54400 dup_rows=2946 dup_rows_remote=2856 dedup_ratio=0.044952 router_tokens_per_expert=[0, 0, 4095, 0, 0, 0, 3651, 0, 0, 0, 0, 0, 0, 4041, 6, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 2, 3218, 301, 0, 0, 4074, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 321, 0, 0, 0, 0, 3941, 880, 0] payload_rows_per_dst=[7302, 8082, 8192, 8190, 6736, 8148, 8192, 7748] dup_rows_per_dst=[0, 12, 0, 90, 306, 428, 0, 2110] combine_rows_per_src=[7302, 8082, 8192, 8190, 6736, 8148, 8192, 7748] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64202 payload_rows_remote=56010 dup_rows=1334 dup_rows_remote=1330 dedup_ratio=0.020355 router_tokens_per_expert=[2, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 9, 4095, 0, 0, 0, 3653, 0, 0, 0, 0, 2, 1, 0, 0, 4095, 0, 0, 0, 4095, 0, 0, 560, 0, 0, 1, 0, 0, 0, 0, 96, 0, 0, 3874, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8188, 7308, 8192, 8190, 8192, 7748] dup_rows_per_dst=[2, 0, 0, 16, 4, 1120, 192, 0] combine_rows_per_src=[8192, 8192, 8188, 7308, 8192, 8190, 8192, 7748] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=376879104 dispatch_meta_bytes=229376 combine_payload_bytes=376879104 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=377108480 ingress_bytes=486979584 link_bytes=486979584 token_bytes=753987584 weight_movement_bytes=110100480 total_nvlink_bytes=864088064 weight_to_token_ratio=0.146024 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.469375910e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=362156032 dispatch_meta_bytes=229360 combine_payload_bytes=362156032 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=362385392 ingress_bytes=582356992 link_bytes=582356992 token_bytes=724541424 weight_movement_bytes=220200960 total_nvlink_bytes=944742384 weight_to_token_ratio=0.303918 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.012814590e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=397608960 dispatch_meta_bytes=223456 combine_payload_bytes=397608960 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=397832416 ingress_bytes=838010880 link_bytes=838010880 token_bytes=795441376 weight_movement_bytes=440401920 total_nvlink_bytes=1235843296 weight_to_token_ratio=0.553657 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=26112 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.457438749e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=355432448 dispatch_meta_bytes=201416 combine_payload_bytes=355432448 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=355633864 ingress_bytes=575633408 link_bytes=575633408 token_bytes=711066312 weight_movement_bytes=220200960 total_nvlink_bytes=931267272 weight_to_token_ratio=0.309677 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=26496 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.001121172e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63392 payload_rows_remote=55200 dup_rows=2144 dup_rows_remote=2144 dedup_ratio=0.032715 router_tokens_per_expert=[17, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 9, 4096, 0, 0, 0, 3450, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 964, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 3697, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8172, 6914, 8192, 8144, 8192, 7394] dup_rows_per_dst=[18, 0, 0, 42, 14, 1928, 142, 0] combine_rows_per_src=[8192, 8192, 8172, 6914, 8192, 8144, 8192, 7394] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=389939200 dispatch_meta_bytes=229024 combine_payload_bytes=389939200 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=390168224 ingress_bytes=500039680 link_bytes=500039680 token_bytes=780107424 weight_movement_bytes=110100480 total_nvlink_bytes=890207904 weight_to_token_ratio=0.141135 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.696512459e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=401479680 dispatch_meta_bytes=229360 combine_payload_bytes=401479680 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=401709040 ingress_bytes=621680640 link_bytes=621680640 token_bytes=803188720 weight_movement_bytes=220200960 total_nvlink_bytes=1023389680 weight_to_token_ratio=0.274158 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.081204882e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60330 payload_rows_remote=52148 dup_rows=5206 dup_rows_remote=550 dedup_ratio=0.079437 router_tokens_per_expert=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 47, 4096, 0, 0, 0, 3648, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 496, 0, 0, 0, 0, 0, 0, 0, 106, 0, 0, 3892, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7984, 8192, 8192, 7382, 8192, 4014, 8192, 8182] dup_rows_per_dst=[0, 0, 208, 8, 6, 116, 212, 4656] combine_rows_per_src=[7984, 8192, 8192, 7382, 8192, 4014, 8192, 8182] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7337613136 prefetch_bytes=990904320 reduce_bytes=880803840 weight_movement_bytes=1871708160 egress_bytes=3669933904 ingress_bytes=5539387392 link_bytes=5539387392 total_nvlink_bytes=9209321296 weight_to_token_ratio=0.255084 prefetch_share_of_ingress=0.178883 dedup_ratio=0.104578 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=8 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.049632 gemm_inner_pad_upper_bound=6350 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.633905747e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=395673600 dispatch_meta_bytes=229376 combine_payload_bytes=395673600 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=395902976 ingress_bytes=505774080 link_bytes=505774080 token_bytes=791576576 weight_movement_bytes=110100480 total_nvlink_bytes=901677056 weight_to_token_ratio=0.139090 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.796243107e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7439137960 prefetch_bytes=1101004800 reduce_bytes=550502400 weight_movement_bytes=1651507200 egress_bytes=3720723624 ingress_bytes=5369921536 link_bytes=5370150912 total_nvlink_bytes=9090645160 weight_to_token_ratio=0.222002 prefetch_share_of_ingress=0.205032 dedup_ratio=0.095660 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=5 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050368 gemm_inner_pad_upper_bound=6350 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.339575674e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63284 payload_rows_remote=55092 dup_rows=2252 dup_rows_remote=2032 dedup_ratio=0.034363 router_tokens_per_expert=[7, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 15, 4096, 0, 0, 0, 3374, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1001, 0, 0, 0, 0, 0, 0, 0, 110, 0, 0, 3704, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8184, 8192, 8170, 6754, 8192, 8192, 8192, 7408] dup_rows_per_dst=[6, 0, 0, 24, 0, 2002, 220, 0] combine_rows_per_src=[8184, 8192, 8170, 6754, 8192, 8192, 8192, 7408] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63618 payload_rows_remote=55454 dup_rows=1918 dup_rows_remote=1918 dedup_ratio=0.029266 router_tokens_per_expert=[27, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 3, 0, 4093, 0, 0, 0, 0, 0, 0, 1, 4081, 0, 0, 20, 4094, 0, 0, 0, 3371, 0, 0, 0, 0, 9, 2, 0, 0, 4094, 0, 0, 0, 4094, 0, 0, 807, 0, 0, 2, 0, 0, 0, 0, 113, 0, 0, 3867, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8164, 6766, 8192, 8188, 8192, 7734] dup_rows_per_dst=[44, 0, 0, 16, 18, 1614, 226, 0] combine_rows_per_src=[8190, 8192, 8164, 6766, 8192, 8188, 8192, 7734] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=373796864 dispatch_meta_bytes=210792 combine_payload_bytes=373796864 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=374007656 ingress_bytes=483897344 link_bytes=483897344 token_bytes=747804520 weight_movement_bytes=110100480 total_nvlink_bytes=857905000 weight_to_token_ratio=0.147232 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=30336 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.415770687e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7146185360 prefetch_bytes=990904320 reduce_bytes=1321205760 weight_movement_bytes=2312110080 egress_bytes=3574198928 ingress_bytes=5884096512 link_bytes=5884096512 total_nvlink_bytes=9458295440 weight_to_token_ratio=0.323545 prefetch_share_of_ingress=0.168404 dedup_ratio=0.117178 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=12 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.051103 gemm_inner_pad_upper_bound=5969 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.023341160e-04 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64472 payload_rows_remote=56280 dup_rows=1064 dup_rows_remote=1064 dedup_ratio=0.016235 router_tokens_per_expert=[0, 0, 0, 0, 0, 2, 0, 3819, 0, 0, 2, 0, 4096, 3841, 4090, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[7642, 7682, 8192, 8188, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 4, 0, 0, 0, 0, 1058, 2] combine_rows_per_src=[7642, 7682, 8192, 8188, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64158 payload_rows_remote=56810 dup_rows=1378 dup_rows_remote=1378 dedup_ratio=0.021027 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3831, 0, 0, 0, 0, 4094, 3674, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7662, 7348, 8192, 8192, 8192, 8188, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 1378, 0] combine_rows_per_src=[7662, 7348, 8192, 8192, 8192, 8188, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64972 payload_rows_remote=56780 dup_rows=564 dup_rows_remote=564 dedup_ratio=0.008606 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3968, 0, 0, 1, 0, 4096, 3942, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7936, 7884, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 2, 0, 0, 0, 0, 562, 0] combine_rows_per_src=[7936, 7884, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64868 payload_rows_remote=56676 dup_rows=668 dup_rows_remote=668 dedup_ratio=0.010193 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3976, 0, 0, 2, 0, 4096, 3882, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7952, 7764, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 4, 0, 0, 0, 0, 664, 0] combine_rows_per_src=[7952, 7764, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=398010368 dispatch_meta_bytes=229376 combine_payload_bytes=398010368 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=398239744 ingress_bytes=508110848 link_bytes=508110848 token_bytes=796250112 weight_movement_bytes=110100480 total_nvlink_bytes=906350592 weight_to_token_ratio=0.138274 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.836883345e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=408475648 dispatch_meta_bytes=227944 combine_payload_bytes=408475648 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=408703592 ingress_bytes=518576128 link_bytes=518576128 token_bytes=817179240 weight_movement_bytes=110100480 total_nvlink_bytes=927279720 weight_to_token_ratio=0.134732 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=28800 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.018891777e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 2, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=403415040 dispatch_meta_bytes=229376 combine_payload_bytes=403415040 prefetch_bytes=0 reduce_bytes=0 egress_bytes=403644416 ingress_bytes=403415040 link_bytes=403644416 token_bytes=807059456 weight_movement_bytes=0 total_nvlink_bytes=807059456 weight_to_token_ratio=0.000000 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.020040275e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=406999040 dispatch_meta_bytes=229376 combine_payload_bytes=406999040 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=407228416 ingress_bytes=517099520 link_bytes=517099520 token_bytes=814227456 weight_movement_bytes=110100480 total_nvlink_bytes=924327936 weight_to_token_ratio=0.135221 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.993211135e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=407214080 dispatch_meta_bytes=232752 combine_payload_bytes=407214080 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=407446832 ingress_bytes=957716480 link_bytes=957716480 token_bytes=814660912 weight_movement_bytes=550502400 total_nvlink_bytes=1365163312 weight_to_token_ratio=0.675744 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=28800 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.665626476e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=406253568 dispatch_meta_bytes=229376 combine_payload_bytes=406253568 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=406482944 ingress_bytes=736555008 link_bytes=736555008 token_bytes=812736512 weight_movement_bytes=330301440 total_nvlink_bytes=1143037952 weight_to_token_ratio=0.406407 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.280990301e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58904 payload_rows_remote=50740 dup_rows=6632 dup_rows_remote=3442 dedup_ratio=0.101196 router_tokens_per_expert=[0, 0, 0, 0, 0, 2, 0, 3990, 0, 0, 2, 0, 4096, 3511, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8164, 8178, 8192, 8192, 8192, 8192, 1604, 8190] dup_rows_per_dst=[3190, 3426, 0, 2, 0, 0, 14, 0] combine_rows_per_src=[8164, 8178, 8192, 8192, 8192, 8192, 1604, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58930 payload_rows_remote=50752 dup_rows=6606 dup_rows_remote=6276 dedup_ratio=0.100800 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 2027, 0, 0, 0, 0, 0, 0, 4053, 21, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 26, 3186, 0, 0, 0, 4086, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2996, 0, 0, 0, 0, 4031, 8, 0] payload_rows_per_dst=[4054, 8106, 8192, 8068, 8118, 8178, 8192, 6022] dup_rows_per_dst=[0, 42, 0, 80, 6072, 330, 0, 82] combine_rows_per_src=[4054, 8106, 8192, 8068, 8118, 8178, 8192, 6022] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58662 payload_rows_remote=50470 dup_rows=6874 dup_rows_remote=34 dedup_ratio=0.104889 router_tokens_per_expert=[0, 0, 4095, 0, 0, 0, 2136, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 3, 2950, 0, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3399, 0, 0, 0, 0, 3782, 22, 0] payload_rows_per_dst=[4272, 8166, 8192, 7622, 5900, 8126, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 28, 6, 0, 0, 6840] combine_rows_per_src=[4272, 8166, 8192, 7622, 5900, 8126, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59470 payload_rows_remote=51278 dup_rows=6066 dup_rows_remote=6066 dedup_ratio=0.092560 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 2015, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 3175, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2973, 0, 0, 0, 0, 4077, 18, 0] payload_rows_per_dst=[4030, 8182, 8192, 8154, 6350, 8178, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 84, 0, 0, 0, 5982] combine_rows_per_src=[4030, 8182, 8192, 8154, 6350, 8178, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=363704320 dispatch_meta_bytes=216728 combine_payload_bytes=363704320 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=363921048 ingress_bytes=473804800 link_bytes=473804800 token_bytes=727625368 weight_movement_bytes=110100480 total_nvlink_bytes=837725848 weight_to_token_ratio=0.151315 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31104 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.240244747e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59368 payload_rows_remote=51560 dup_rows=6168 dup_rows_remote=1440 dedup_ratio=0.094116 router_tokens_per_expert=[0, 0, 4095, 0, 0, 0, 1984, 0, 0, 0, 0, 0, 0, 4032, 1, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 1, 3390, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2870, 0, 0, 0, 0, 4086, 9, 0] payload_rows_per_dst=[5046, 8066, 8192, 8182, 7808, 8134, 8192, 5748] dup_rows_per_dst=[964, 394, 2, 70, 4728, 0, 0, 10] combine_rows_per_src=[5046, 8066, 8192, 8182, 7808, 8134, 8192, 5748] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61884 payload_rows_remote=53692 dup_rows=3652 dup_rows_remote=3648 dedup_ratio=0.055725 router_tokens_per_expert=[0, 0, 4094, 0, 0, 0, 1790, 0, 0, 0, 0, 0, 0, 4072, 3, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 3, 3220, 0, 0, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3294, 0, 0, 0, 0, 3994, 15, 0] payload_rows_per_dst=[8192, 8144, 8192, 8044, 6446, 8084, 8192, 6590] dup_rows_per_dst=[3576, 6, 4, 38, 0, 0, 0, 28] combine_rows_per_src=[8192, 8144, 8192, 8044, 6446, 8084, 8192, 6590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=363790336 dispatch_meta_bytes=228112 combine_payload_bytes=363790336 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=364018448 ingress_bytes=473890816 link_bytes=473890816 token_bytes=727808784 weight_movement_bytes=110100480 total_nvlink_bytes=837909264 weight_to_token_ratio=0.151277 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.241740707e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62916 payload_rows_remote=54740 dup_rows=2620 dup_rows_remote=2620 dedup_ratio=0.039978 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 1256, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 3438, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3486, 0, 0, 0, 0, 4054, 26, 0] payload_rows_per_dst=[8192, 8176, 8192, 8114, 6876, 8186, 8192, 6988] dup_rows_per_dst=[2512, 0, 0, 72, 0, 0, 0, 36] combine_rows_per_src=[8192, 8176, 8192, 8114, 6876, 8186, 8192, 6988] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60338 payload_rows_remote=52176 dup_rows=5198 dup_rows_remote=5108 dedup_ratio=0.079315 router_tokens_per_expert=[0, 0, 4095, 0, 0, 0, 2543, 0, 0, 0, 0, 0, 0, 4036, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 2, 3221, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2443, 0, 0, 0, 0, 4081, 15, 0] payload_rows_per_dst=[8192, 8072, 8192, 8162, 6446, 8184, 8192, 4898] dup_rows_per_dst=[5084, 6, 0, 90, 0, 0, 0, 18] combine_rows_per_src=[8192, 8072, 8192, 8162, 6446, 8184, 8192, 4898] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=361768960 dispatch_meta_bytes=202016 combine_payload_bytes=361768960 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=361970976 ingress_bytes=692070400 link_bytes=692070400 token_bytes=723739936 weight_movement_bytes=330301440 total_nvlink_bytes=1054041376 weight_to_token_ratio=0.456381 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=24448 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.203624252e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=367560704 dispatch_meta_bytes=229376 combine_payload_bytes=367560704 prefetch_bytes=0 reduce_bytes=0 egress_bytes=367790080 ingress_bytes=367560704 link_bytes=367790080 token_bytes=735350784 weight_movement_bytes=0 total_nvlink_bytes=735350784 weight_to_token_ratio=0.000000 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.396474402e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=369582080 dispatch_meta_bytes=212000 combine_payload_bytes=369582080 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=369794080 ingress_bytes=479682560 link_bytes=479682560 token_bytes=739376160 weight_movement_bytes=110100480 total_nvlink_bytes=849476640 weight_to_token_ratio=0.148910 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=25984 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.342468661e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=384864256 dispatch_meta_bytes=229360 combine_payload_bytes=384864256 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=385093616 ingress_bytes=494964736 link_bytes=494964736 token_bytes=769957872 weight_movement_bytes=110100480 total_nvlink_bytes=880058352 weight_to_token_ratio=0.142995 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.608250836e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=392376320 dispatch_meta_bytes=229440 combine_payload_bytes=392376320 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=392605760 ingress_bytes=502476800 link_bytes=502476800 token_bytes=784982080 weight_movement_bytes=110100480 total_nvlink_bytes=895082560 weight_to_token_ratio=0.140259 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.738897984e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=373997568 dispatch_meta_bytes=229136 combine_payload_bytes=373997568 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=374226704 ingress_bytes=594198528 link_bytes=594198528 token_bytes=748224272 weight_movement_bytes=220200960 total_nvlink_bytes=968425232 weight_to_token_ratio=0.294298 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.033408969e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61760 payload_rows_remote=53568 dup_rows=3776 dup_rows_remote=546 dedup_ratio=0.057617 router_tokens_per_expert=[0, 0, 4095, 0, 0, 0, 1612, 0, 0, 0, 0, 0, 0, 4086, 2, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 2, 3391, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3195, 0, 0, 0, 0, 4070, 1, 0] payload_rows_per_dst=[8192, 8172, 8186, 8188, 6786, 8136, 8192, 5908] dup_rows_per_dst=[3230, 4, 0, 540, 0, 0, 0, 2] combine_rows_per_src=[8192, 8172, 8186, 8188, 6786, 8136, 8192, 5908] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64194 payload_rows_remote=56002 dup_rows=1342 dup_rows_remote=1230 dedup_ratio=0.020477 router_tokens_per_expert=[70, 0, 0, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 9, 0, 4055, 0, 0, 0, 0, 0, 0, 5, 4070, 0, 0, 203, 4088, 0, 0, 0, 3958, 0, 0, 0, 0, 216, 0, 0, 0, 4093, 1, 0, 0, 4096, 0, 0, 55, 0, 0, 1, 0, 0, 0, 0, 178, 0, 0, 3607, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8128, 8150, 7948, 8192, 8192, 8178, 7214] dup_rows_per_dst=[74, 0, 0, 374, 426, 112, 356, 0] combine_rows_per_src=[8192, 8128, 8150, 7948, 8192, 8192, 8178, 7214] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59044 payload_rows_remote=50912 dup_rows=6492 dup_rows_remote=1790 dedup_ratio=0.099060 router_tokens_per_expert=[5, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 78, 4096, 0, 0, 0, 3935, 0, 0, 0, 0, 57, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 126, 0, 0, 3971, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7922, 8192, 8180, 6830, 3404, 8192, 8192, 8132] dup_rows_per_dst=[10, 68, 196, 88, 10, 1166, 252, 4702] combine_rows_per_src=[7922, 8192, 8180, 6830, 3404, 8192, 8192, 8132] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64576 payload_rows_remote=56384 dup_rows=960 dup_rows_remote=616 dedup_ratio=0.014648 router_tokens_per_expert=[8, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 23, 4096, 0, 0, 0, 3933, 0, 0, 0, 0, 108, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 178, 0, 0, 0, 0, 0, 0, 0, 172, 0, 0, 3790, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8190, 8170, 7872, 8192, 8192, 8192, 7580] dup_rows_per_dst=[4, 0, 0, 40, 216, 356, 344, 0] combine_rows_per_src=[8188, 8190, 8170, 7872, 8192, 8192, 8192, 7580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=383975424 dispatch_meta_bytes=216456 combine_payload_bytes=383975424 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=384191880 ingress_bytes=934477824 link_bytes=934477824 token_bytes=768167304 weight_movement_bytes=550502400 total_nvlink_bytes=1318669704 weight_to_token_ratio=0.716644 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=17408 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.625210631e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56898 payload_rows_remote=48706 dup_rows=8638 dup_rows_remote=8530 dedup_ratio=0.131805 router_tokens_per_expert=[25, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 0, 0, 1, 4078, 0, 0, 11, 4094, 0, 0, 0, 3415, 0, 0, 0, 0, 3, 0, 0, 0, 4094, 0, 0, 0, 4095, 0, 0, 964, 0, 0, 2, 0, 0, 0, 0, 84, 0, 0, 3717, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8158, 8192, 8120, 1928, 6682, 7434] dup_rows_per_dst=[108, 38, 0, 8340, 2, 0, 150, 0] combine_rows_per_src=[8192, 8192, 8158, 8192, 8120, 1928, 6682, 7434] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63108 payload_rows_remote=56506 dup_rows=2428 dup_rows_remote=2398 dedup_ratio=0.037048 router_tokens_per_expert=[18, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 16, 4096, 0, 0, 0, 3300, 0, 0, 0, 0, 2, 0, 0, 0, 4094, 2, 0, 0, 4094, 0, 0, 1118, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 3694, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8158, 6602, 8192, 8192, 8192, 7388] dup_rows_per_dst=[4, 0, 0, 30, 0, 2236, 158, 0] combine_rows_per_src=[8192, 8192, 8158, 6602, 8192, 8192, 8192, 7388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64844 payload_rows_remote=56652 dup_rows=692 dup_rows_remote=448 dedup_ratio=0.010559 router_tokens_per_expert=[2, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 42, 4095, 0, 0, 0, 3961, 0, 0, 0, 0, 123, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 22, 0, 0, 1, 0, 0, 0, 0, 159, 0, 0, 3887, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8188, 7922, 8192, 8192, 8192, 7774] dup_rows_per_dst=[2, 0, 0, 84, 244, 44, 318, 0] combine_rows_per_src=[8192, 8192, 8188, 7922, 8192, 8192, 8192, 7774] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64628 payload_rows_remote=56470 dup_rows=908 dup_rows_remote=908 dedup_ratio=0.013855 router_tokens_per_expert=[39, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 3, 0, 4093, 0, 0, 0, 0, 0, 0, 1, 4078, 0, 0, 76, 4094, 0, 0, 0, 3878, 0, 0, 0, 0, 155, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 53, 0, 0, 2, 0, 1, 0, 0, 156, 0, 0, 3859, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8158, 7794, 8190, 8192, 8192, 7718] dup_rows_per_dst=[66, 0, 0, 114, 308, 106, 314, 0] combine_rows_per_src=[8192, 8192, 8158, 7794, 8190, 8192, 8192, 7718] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=401422336 dispatch_meta_bytes=228928 combine_payload_bytes=401422336 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=401651264 ingress_bytes=621623296 link_bytes=621623296 token_bytes=803073600 weight_movement_bytes=220200960 total_nvlink_bytes=1023274560 weight_to_token_ratio=0.274198 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=31616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.081105151e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64484 payload_rows_remote=56292 dup_rows=1052 dup_rows_remote=1052 dedup_ratio=0.016052 router_tokens_per_expert=[23, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 60, 4096, 0, 0, 0, 3910, 0, 0, 0, 0, 170, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 179, 0, 0, 3763, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8170, 7828, 8192, 8192, 8192, 7526] dup_rows_per_dst=[36, 0, 0, 112, 340, 206, 358, 0] combine_rows_per_src=[8192, 8192, 8170, 7828, 8192, 8192, 8192, 7526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=364937216 dispatch_meta_bytes=210808 combine_payload_bytes=364937216 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=365148024 ingress_bytes=475037696 link_bytes=475037696 token_bytes=730085240 weight_movement_bytes=110100480 total_nvlink_bytes=840185720 weight_to_token_ratio=0.150805 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=30336 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.261686836e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64538 payload_rows_remote=56682 dup_rows=998 dup_rows_remote=946 dedup_ratio=0.015228 router_tokens_per_expert=[22, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 32, 4096, 0, 0, 0, 3922, 0, 0, 0, 0, 221, 0, 0, 0, 4094, 1, 0, 0, 4095, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 199, 0, 0, 3782, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8158, 7856, 8192, 8192, 8192, 7564] dup_rows_per_dst=[20, 0, 0, 52, 438, 90, 398, 0] combine_rows_per_src=[8192, 8192, 8158, 7856, 8192, 8192, 8192, 7564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=404160512 dispatch_meta_bytes=228000 combine_payload_bytes=404160512 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=404388512 ingress_bytes=514260992 link_bytes=514260992 token_bytes=808549024 weight_movement_bytes=110100480 total_nvlink_bytes=918649504 weight_to_token_ratio=0.136170 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=31488 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.943844465e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=394899456 dispatch_meta_bytes=228496 combine_payload_bytes=394899456 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=395127952 ingress_bytes=504999936 link_bytes=504999936 token_bytes=790027408 weight_movement_bytes=110100480 total_nvlink_bytes=900127888 weight_to_token_ratio=0.139363 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.782779469e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7023709696 prefetch_bytes=990904320 reduce_bytes=880803840 weight_movement_bytes=1871708160 egress_bytes=3512980992 ingress_bytes=5382436864 link_bytes=5382436864 total_nvlink_bytes=8895417856 weight_to_token_ratio=0.266484 prefetch_share_of_ingress=0.184100 dedup_ratio=0.134540 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=8 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.049265 gemm_inner_pad_upper_bound=6477 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.360942965e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=397494272 dispatch_meta_bytes=229488 combine_payload_bytes=397494272 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=397723760 ingress_bytes=507594752 link_bytes=507594752 token_bytes=795218032 weight_movement_bytes=110100480 total_nvlink_bytes=905318512 weight_to_token_ratio=0.138453 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.827907587e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=349124608 dispatch_meta_bytes=228944 combine_payload_bytes=349124608 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=349353552 ingress_bytes=569325568 link_bytes=569325568 token_bytes=698478160 weight_movement_bytes=220200960 total_nvlink_bytes=918679120 weight_to_token_ratio=0.315258 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.901508007e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=405035008 dispatch_meta_bytes=235616 combine_payload_bytes=405035008 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=405270624 ingress_bytes=625235968 link_bytes=625235968 token_bytes=810305632 weight_movement_bytes=220200960 total_nvlink_bytes=1030506592 weight_to_token_ratio=0.271750 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.055147 gemm_active_groups=5 gemm_peak_group_rows=27776 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.087388182e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7223206208 prefetch_bytes=990904320 reduce_bytes=880803840 weight_movement_bytes=1871708160 egress_bytes=3612727616 ingress_bytes=5482186752 link_bytes=5482186752 total_nvlink_bytes=9094914368 weight_to_token_ratio=0.259124 prefetch_share_of_ingress=0.180750 dedup_ratio=0.110675 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=8 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048897 gemm_inner_pad_upper_bound=6985 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.534424427e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7236510704 prefetch_bytes=1101004800 reduce_bytes=550502400 weight_movement_bytes=1651507200 egress_bytes=3619394544 ingress_bytes=5268623360 link_bytes=5268623360 total_nvlink_bytes=8888017904 weight_to_token_ratio=0.228219 prefetch_share_of_ingress=0.208974 dedup_ratio=0.112759 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=5 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050000 gemm_inner_pad_upper_bound=6350 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.163002563e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6724009448 prefetch_bytes=880803840 reduce_bytes=1871708160 weight_movement_bytes=2752512000 egress_bytes=3363091944 ingress_bytes=6113429504 link_bytes=6113658880 total_nvlink_bytes=9476521448 weight_to_token_ratio=0.409356 prefetch_share_of_ingress=0.144077 dedup_ratio=0.163443 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=17 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050735 gemm_inner_pad_upper_bound=5842 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.063265832e-04 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7409603752 prefetch_bytes=990904320 reduce_bytes=1101004800 weight_movement_bytes=2091909120 egress_bytes=3705941160 ingress_bytes=5795571712 link_bytes=5795571712 total_nvlink_bytes=9501512872 weight_to_token_ratio=0.282324 prefetch_share_of_ingress=0.170976 dedup_ratio=0.094409 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=10 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.051103 gemm_inner_pad_upper_bound=6604 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.007945241e-04 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none - [2026-08-26 15:11:34] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 368.4 | throughput per GPU (TFLOP/s/GPU): 335.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.670588E+00 | loss scale: 1.0 | grad norm: 4.299 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44412 payload_rows_remote=39380 dup_rows=21124 dup_rows_remote=19806 dedup_ratio=0.322327 router_tokens_per_expert=[1198, 931, 475, 1005, 163, 1151, 420, 141, 248, 1887, 1325, 332, 5, 920, 35, 1250, 613, 718, 6, 37, 718, 436, 253, 394, 241, 795, 475, 1179, 153, 87, 451, 1, 1129, 684, 618, 18, 689, 0, 1233, 168, 482, 317, 510, 616, 20, 661, 131, 724, 307, 137, 284, 635, 168, 765, 74, 999, 342, 895, 527, 424, 397, 470, 299, 2] payload_rows_per_dst=[6246, 6970, 5032, 5240, 5920, 4872, 5352, 4780] dup_rows_per_dst=[3784, 5034, 1318, 1662, 3158, 2050, 2186, 1932] combine_rows_per_src=[6246, 6970, 5032, 5240, 5920, 4872, 5352, 4780] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44850 payload_rows_remote=39044 dup_rows=20686 dup_rows_remote=17696 dedup_ratio=0.315643 router_tokens_per_expert=[1525, 423, 489, 1028, 387, 474, 571, 183, 370, 1114, 991, 203, 7, 1352, 12, 991, 485, 314, 30, 58, 704, 349, 440, 640, 227, 1009, 343, 1086, 216, 268, 278, 4, 595, 256, 1017, 3, 822, 24, 1405, 627, 277, 681, 384, 1068, 5, 791, 78, 506, 173, 208, 379, 554, 642, 971, 185, 1262, 471, 916, 530, 216, 254, 533, 364, 0] payload_rows_per_dst=[4388, 4390, 5188, 5456, 6010, 6560, 7052, 5806] dup_rows_per_dst=[928, 758, 1698, 2354, 3488, 3724, 4746, 2990] combine_rows_per_src=[4388, 4390, 5188, 5456, 6010, 6560, 7052, 5806] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44274 payload_rows_remote=38794 dup_rows=21262 dup_rows_remote=19012 dedup_ratio=0.324432 router_tokens_per_expert=[1100, 1335, 694, 849, 109, 891, 396, 74, 499, 1252, 1018, 493, 0, 1100, 33, 1212, 560, 1001, 25, 35, 1126, 270, 415, 699, 467, 459, 346, 626, 274, 377, 416, 9, 636, 624, 754, 37, 861, 0, 1046, 209, 583, 684, 352, 955, 12, 478, 18, 318, 286, 185, 201, 473, 335, 933, 127, 755, 363, 1153, 575, 325, 280, 788, 260, 2] payload_rows_per_dst=[5076, 6644, 5878, 5480, 5338, 4930, 5902, 5026] dup_rows_per_dst=[1744, 4570, 2478, 2250, 2996, 1870, 2888, 2466] combine_rows_per_src=[5076, 6644, 5878, 5480, 5338, 4930, 5902, 5026] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43948 payload_rows_remote=37848 dup_rows=21588 dup_rows_remote=18018 dedup_ratio=0.329407 router_tokens_per_expert=[544, 1157, 988, 651, 101, 1104, 487, 41, 802, 1458, 1078, 310, 8, 1287, 77, 1163, 976, 836, 20, 33, 464, 172, 521, 585, 244, 465, 604, 822, 308, 239, 353, 9, 842, 716, 832, 15, 835, 0, 942, 353, 528, 310, 645, 790, 4, 767, 115, 389, 202, 135, 77, 512, 106, 1135, 146, 639, 1037, 840, 755, 200, 324, 411, 257, 2] payload_rows_per_dst=[3640, 5238, 6136, 5526, 5690, 6100, 5106, 6512] dup_rows_per_dst=[896, 1638, 3392, 2770, 3380, 3570, 1886, 4056] combine_rows_per_src=[3640, 5238, 6136, 5526, 5690, 6100, 5106, 6512] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44024 payload_rows_remote=39082 dup_rows=21512 dup_rows_remote=19128 dedup_ratio=0.328247 router_tokens_per_expert=[1189, 962, 1084, 949, 689, 1463, 740, 30, 184, 1399, 896, 467, 5, 901, 96, 475, 816, 608, 6, 27, 801, 342, 894, 493, 260, 779, 533, 637, 211, 183, 809, 2, 600, 211, 589, 8, 883, 0, 1003, 369, 252, 747, 478, 922, 0, 623, 127, 393, 197, 146, 274, 493, 257, 1229, 53, 497, 571, 985, 419, 92, 397, 826, 193, 4] payload_rows_per_dst=[5008, 4436, 6174, 6292, 4942, 5304, 5952, 5916] dup_rows_per_dst=[1976, 1110, 3724, 3462, 2384, 2382, 2718, 3756] combine_rows_per_src=[5008, 4436, 6174, 6292, 4942, 5304, 5952, 5916] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7540541952 prefetch_bytes=1211105280 reduce_bytes=660602880 weight_movement_bytes=1871708160 egress_bytes=3771421184 ingress_bytes=5640828928 link_bytes=5640828928 total_nvlink_bytes=9412250112 weight_to_token_ratio=0.248219 prefetch_share_of_ingress=0.214703 dedup_ratio=0.082233 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=6 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050368 gemm_inner_pad_upper_bound=6350 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.810329263e-05 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=406081536 dispatch_meta_bytes=228400 combine_payload_bytes=406081536 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=406309936 ingress_bytes=626282496 link_bytes=626282496 token_bytes=812391472 weight_movement_bytes=220200960 total_nvlink_bytes=1032592432 weight_to_token_ratio=0.271053 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=30336 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.089208266e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=404776960 dispatch_meta_bytes=229512 combine_payload_bytes=404776960 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=405006472 ingress_bytes=514877440 link_bytes=514877440 token_bytes=809783432 weight_movement_bytes=110100480 total_nvlink_bytes=919883912 weight_to_token_ratio=0.135963 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.954565509e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=403501056 dispatch_meta_bytes=229376 combine_payload_bytes=403501056 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=403730432 ingress_bytes=513601536 link_bytes=513601536 token_bytes=807231488 weight_movement_bytes=110100480 total_nvlink_bytes=917331968 weight_to_token_ratio=0.136393 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.932375440e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7075979600 prefetch_bytes=880803840 reduce_bytes=1101004800 weight_movement_bytes=1981808640 egress_bytes=3539087696 ingress_bytes=5518700544 link_bytes=5518700544 total_nvlink_bytes=9057788240 weight_to_token_ratio=0.280076 prefetch_share_of_ingress=0.159603 dedup_ratio=0.127335 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=10 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050368 gemm_inner_pad_upper_bound=6223 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.597927916e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7468345488 prefetch_bytes=1211105280 reduce_bytes=660602880 weight_movement_bytes=1871708160 egress_bytes=3735308432 ingress_bytes=5604745216 link_bytes=5604974592 total_nvlink_bytes=9340053648 weight_to_token_ratio=0.250619 prefetch_share_of_ingress=0.216086 dedup_ratio=0.089670 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=6 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050000 gemm_inner_pad_upper_bound=6477 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.747972675e-05 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59692 payload_rows_remote=51500 dup_rows=5844 dup_rows_remote=5782 dedup_ratio=0.089172 router_tokens_per_expert=[36, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 2, 0, 4093, 0, 0, 0, 0, 0, 0, 1, 4082, 0, 0, 61, 4094, 0, 0, 0, 3956, 0, 0, 0, 0, 237, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 129, 0, 0, 1, 0, 0, 0, 0, 132, 0, 0, 3663, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8166, 7994, 8192, 5946, 5686, 7326] dup_rows_per_dst=[62, 0, 0, 2628, 2808, 166, 180, 0] combine_rows_per_src=[8192, 8190, 8166, 7994, 8192, 5946, 5686, 7326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=406296576 dispatch_meta_bytes=230512 combine_payload_bytes=406296576 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=406527088 ingress_bytes=736598016 link_bytes=736598016 token_bytes=812823664 weight_movement_bytes=330301440 total_nvlink_bytes=1143125104 weight_to_token_ratio=0.406363 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=30976 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.281065099e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7619138888 prefetch_bytes=990904320 reduce_bytes=770703360 weight_movement_bytes=1761607680 egress_bytes=3810708808 ingress_bytes=5570037760 link_bytes=5570267136 total_nvlink_bytes=9380746568 weight_to_token_ratio=0.231208 prefetch_share_of_ingress=0.177899 dedup_ratio=0.071597 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=7 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.051471 gemm_inner_pad_upper_bound=5969 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.687610701e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7529562648 prefetch_bytes=990904320 reduce_bytes=550502400 weight_movement_bytes=1541406720 egress_bytes=3765932568 ingress_bytes=5305036800 link_bytes=5305266176 total_nvlink_bytes=9070969368 weight_to_token_ratio=0.204714 prefetch_share_of_ingress=0.186786 dedup_ratio=0.081284 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=5 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.051103 gemm_inner_pad_upper_bound=5969 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.226730446e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7498823328 prefetch_bytes=990904320 reduce_bytes=990904320 weight_movement_bytes=1981808640 egress_bytes=3750547104 ingress_bytes=5730084864 link_bytes=5730084864 total_nvlink_bytes=9480631968 weight_to_token_ratio=0.264283 prefetch_share_of_ingress=0.172930 dedup_ratio=0.083090 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=9 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050735 gemm_inner_pad_upper_bound=5969 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.965560016e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7648125856 prefetch_bytes=1211105280 reduce_bytes=990904320 weight_movement_bytes=2202009600 egress_bytes=3825216416 ingress_bytes=6024919040 link_bytes=6024919040 total_nvlink_bytes=9850135456 weight_to_token_ratio=0.287915 prefetch_share_of_ingress=0.201016 dedup_ratio=0.071942 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=9 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.049632 gemm_inner_pad_upper_bound=6477 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.047832514e-04 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=369152000 dispatch_meta_bytes=229128 combine_payload_bytes=369152000 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=369381128 ingress_bytes=699453440 link_bytes=699453440 token_bytes=738533128 weight_movement_bytes=330301440 total_nvlink_bytes=1068834568 weight_to_token_ratio=0.447240 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.216464572e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7011052272 prefetch_bytes=770703360 reduce_bytes=2532311040 weight_movement_bytes=3303014400 egress_bytes=3506617072 ingress_bytes=6807449600 link_bytes=6807449600 total_nvlink_bytes=10314066672 weight_to_token_ratio=0.471115 prefetch_share_of_ingress=0.113215 dedup_ratio=0.131879 prefetch_slots=14 unused_prefetch_slots=146 slot_utilization=0.087500 reduce_slots=23 remote_demand=7 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.051838 gemm_inner_pad_upper_bound=5588 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.183927449e-04 prefetch_bytes_per_gemm_flop=1.340379902e-05 anomalies=none + [2026-08-26 15:59:20] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 364.3 | throughput per GPU (TFLOP/s/GPU): 339.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.669404E+00 | loss scale: 1.0 | grad norm: 4.302 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44214 payload_rows_remote=38250 dup_rows=21322 dup_rows_remote=18124 dedup_ratio=0.325348 router_tokens_per_expert=[554, 1144, 999, 584, 76, 1094, 498, 36, 805, 1415, 1061, 302, 8, 1293, 70, 1133, 974, 850, 16, 40, 462, 206, 534, 580, 260, 479, 599, 818, 350, 229, 347, 10, 830, 757, 825, 15, 937, 0, 939, 390, 479, 266, 595, 816, 4, 788, 148, 391, 209, 151, 89, 476, 111, 1151, 140, 641, 1120, 725, 763, 214, 330, 380, 260, 2] payload_rows_per_dst=[4334, 6024, 5762, 4624, 5930, 5964, 5138, 6438] dup_rows_per_dst=[1312, 2946, 2370, 1560, 4050, 3198, 1906, 3980] combine_rows_per_src=[4334, 6024, 5762, 4624, 5930, 5964, 5138, 6438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44904 payload_rows_remote=38922 dup_rows=20632 dup_rows_remote=18484 dedup_ratio=0.314819 router_tokens_per_expert=[1136, 1331, 706, 769, 94, 871, 402, 63, 478, 1212, 1011, 472, 0, 1087, 29, 1174, 555, 1015, 22, 39, 1131, 306, 439, 714, 489, 481, 343, 619, 321, 351, 411, 9, 623, 664, 774, 40, 941, 0, 1049, 228, 543, 630, 314, 994, 14, 491, 34, 326, 305, 201, 231, 436, 345, 939, 122, 752, 408, 1034, 588, 351, 286, 753, 271, 1] payload_rows_per_dst=[5144, 6336, 5934, 5982, 4702, 5450, 5972, 5384] dup_rows_per_dst=[2032, 3712, 2508, 2148, 1854, 2538, 2962, 2878] combine_rows_per_src=[5144, 6336, 5934, 5982, 4702, 5450, 5972, 5384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44696 payload_rows_remote=39586 dup_rows=20840 dup_rows_remote=18318 dedup_ratio=0.317993 router_tokens_per_expert=[1209, 973, 1089, 884, 651, 1433, 750, 28, 178, 1369, 873, 464, 5, 890, 93, 453, 821, 610, 6, 29, 802, 391, 922, 496, 265, 847, 529, 621, 237, 154, 798, 2, 590, 232, 591, 9, 989, 2, 991, 412, 230, 695, 430, 937, 0, 631, 157, 389, 209, 163, 312, 451, 271, 1247, 51, 498, 613, 873, 409, 104, 418, 782, 205, 5] payload_rows_per_dst=[5862, 4672, 5710, 5166, 5110, 6234, 6000, 5942] dup_rows_per_dst=[2888, 1240, 2444, 1740, 2522, 3570, 2822, 3614] combine_rows_per_src=[5862, 4672, 5710, 5166, 5110, 6234, 6000, 5942] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44596 payload_rows_remote=38812 dup_rows=20940 dup_rows_remote=18040 dedup_ratio=0.319519 router_tokens_per_expert=[1547, 424, 488, 957, 336, 453, 595, 174, 370, 1085, 970, 190, 7, 1343, 12, 951, 482, 328, 27, 65, 709, 392, 467, 617, 242, 1070, 346, 1073, 253, 240, 269, 6, 594, 317, 1046, 2, 910, 25, 1379, 667, 251, 616, 325, 1091, 5, 816, 115, 499, 178, 224, 406, 501, 664, 977, 174, 1241, 523, 825, 533, 230, 259, 510, 377, 0] payload_rows_per_dst=[4290, 4280, 6238, 5046, 6228, 5646, 7084, 5784] dup_rows_per_dst=[810, 720, 2622, 1952, 4500, 2696, 4740, 2900] combine_rows_per_src=[4290, 4280, 6238, 5046, 6228, 5646, 7084, 5784] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43698 payload_rows_remote=37642 dup_rows=21838 dup_rows_remote=18320 dedup_ratio=0.333221 router_tokens_per_expert=[1084, 1116, 483, 575, 111, 1072, 456, 129, 383, 1222, 893, 688, 1, 1239, 3, 1036, 625, 935, 2, 36, 592, 426, 550, 375, 373, 546, 660, 700, 224, 391, 1118, 2, 1047, 468, 497, 1, 562, 6, 893, 167, 482, 438, 266, 775, 0, 593, 155, 630, 573, 196, 96, 993, 170, 747, 30, 898, 414, 801, 379, 550, 298, 1343, 254, 0] payload_rows_per_dst=[3042, 4758, 6310, 5786, 5750, 5904, 6056, 6092] dup_rows_per_dst=[466, 1250, 3250, 2242, 3764, 2918, 3518, 4430] combine_rows_per_src=[3042, 4758, 6310, 5786, 5750, 5904, 6056, 6092] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44636 payload_rows_remote=39452 dup_rows=20900 dup_rows_remote=19492 dedup_ratio=0.318909 router_tokens_per_expert=[1216, 941, 481, 941, 100, 1134, 426, 125, 249, 1837, 1305, 316, 5, 931, 35, 1211, 606, 731, 7, 44, 722, 481, 312, 393, 255, 835, 470, 1164, 192, 80, 445, 1, 1111, 741, 613, 17, 785, 0, 1227, 184, 417, 275, 459, 661, 20, 676, 179, 722, 328, 152, 323, 562, 183, 780, 72, 1004, 381, 761, 526, 432, 418, 448, 318, 2] payload_rows_per_dst=[6358, 6952, 5184, 6476, 4942, 4784, 5184, 4756] dup_rows_per_dst=[3984, 4826, 1408, 2862, 1960, 2034, 2010, 1816] combine_rows_per_src=[6358, 6952, 5184, 6476, 4942, 4784, 5184, 4756] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44058 payload_rows_remote=38492 dup_rows=21478 dup_rows_remote=18596 dedup_ratio=0.327728 router_tokens_per_expert=[1292, 1436, 786, 708, 447, 1454, 385, 78, 451, 1372, 303, 372, 30, 875, 112, 709, 659, 502, 22, 53, 866, 301, 360, 1081, 435, 630, 549, 868, 621, 141, 557, 6, 719, 605, 1071, 6, 621, 3, 1168, 217, 284, 689, 637, 811, 15, 607, 123, 505, 113, 563, 323, 308, 328, 518, 92, 574, 552, 546, 632, 424, 288, 670, 289, 6] payload_rows_per_dst=[7042, 5566, 5546, 6512, 4768, 5258, 4356, 5010] dup_rows_per_dst=[6130, 2882, 2142, 3438, 1716, 2084, 1282, 1804] combine_rows_per_src=[7042, 5566, 5546, 6512, 4768, 5258, 4356, 5010] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43488 payload_rows_remote=37424 dup_rows=22048 dup_rows_remote=18498 dedup_ratio=0.336426 router_tokens_per_expert=[1075, 1110, 465, 651, 133, 1112, 442, 154, 392, 1262, 887, 704, 1, 1216, 4, 1074, 633, 912, 3, 14, 605, 390, 528, 380, 352, 522, 684, 713, 176, 409, 1110, 1, 1052, 399, 483, 0, 473, 6, 881, 148, 551, 467, 287, 750, 0, 599, 106, 619, 541, 178, 92, 1081, 176, 738, 31, 895, 371, 912, 374, 530, 287, 1385, 242, 0] payload_rows_per_dst=[3204, 4826, 5998, 6438, 4850, 5988, 6064, 6120] dup_rows_per_dst=[486, 1298, 3152, 3720, 2034, 3202, 3550, 4606] combine_rows_per_src=[3204, 4826, 5998, 6438, 4850, 5988, 6064, 6120] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=282275840 dispatch_meta_bytes=236744 combine_payload_bytes=282275840 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=282512584 ingress_bytes=392376320 link_bytes=392376320 token_bytes=564788424 weight_movement_bytes=110100480 total_nvlink_bytes=674888904 weight_to_token_ratio=0.194941 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.824069553e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279867392 dispatch_meta_bytes=226960 combine_payload_bytes=279867392 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280094352 ingress_bytes=389967872 link_bytes=389967872 token_bytes=559961744 weight_movement_bytes=110100480 total_nvlink_bytes=670062224 weight_to_token_ratio=0.196621 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=7040 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.782182681e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278075392 dispatch_meta_bytes=231224 combine_payload_bytes=278075392 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278306616 ingress_bytes=388175872 link_bytes=388175872 token_bytes=556382008 weight_movement_bytes=110100480 total_nvlink_bytes=666482488 weight_to_token_ratio=0.197886 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=6784 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.751016854e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43542 payload_rows_remote=36296 dup_rows=21994 dup_rows_remote=14558 dedup_ratio=0.335602 router_tokens_per_expert=[1285, 997, 701, 1129, 236, 1233, 439, 73, 160, 709, 1599, 298, 3, 1160, 79, 755, 1125, 337, 12, 32, 893, 408, 651, 497, 325, 1415, 314, 784, 384, 119, 330, 5, 841, 361, 1212, 7, 1137, 0, 1248, 106, 395, 870, 792, 518, 0, 744, 41, 359, 128, 182, 410, 240, 217, 1086, 127, 870, 391, 715, 354, 232, 342, 149, 205, 2] payload_rows_per_dst=[7246, 6160, 5782, 5262, 5178, 5228, 4892, 3794] dup_rows_per_dst=[7436, 3366, 2128, 2090, 2150, 2210, 1628, 986] combine_rows_per_src=[7246, 6160, 5782, 5262, 5178, 5228, 4892, 3794] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280139776 dispatch_meta_bytes=232840 combine_payload_bytes=280139776 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=280372616 ingress_bytes=500340736 link_bytes=500340736 token_bytes=560512392 weight_movement_bytes=220200960 total_nvlink_bytes=780713352 weight_to_token_ratio=0.392857 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=7424 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.701748318e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=271294464 dispatch_meta_bytes=223464 combine_payload_bytes=271294464 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=271517928 ingress_bytes=381394944 link_bytes=381394944 token_bytes=542812392 weight_movement_bytes=110100480 total_nvlink_bytes=652912872 weight_to_token_ratio=0.202833 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=6528 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.633085363e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=260169728 dispatch_meta_bytes=203416 combine_payload_bytes=260169728 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=260373144 ingress_bytes=700571648 link_bytes=700571648 token_bytes=520542872 weight_movement_bytes=440401920 total_nvlink_bytes=960944792 weight_to_token_ratio=0.846044 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=7168 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.218409320e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=268255232 dispatch_meta_bytes=223688 combine_payload_bytes=268255232 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=268478920 ingress_bytes=378355712 link_bytes=378355712 token_bytes=536734152 weight_movement_bytes=110100480 total_nvlink_bytes=646834632 weight_to_token_ratio=0.205130 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=7424 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.580228120e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47806 payload_rows_remote=40884 dup_rows=17730 dup_rows_remote=15238 dedup_ratio=0.270538 router_tokens_per_expert=[0, 0, 1, 378, 3754, 1, 98, 731, 2254, 1601, 119, 0, 0, 26, 0, 914, 0, 2304, 0, 2, 3298, 2738, 1, 0, 0, 0, 2682, 1, 24, 806, 0, 0, 0, 0, 0, 0, 1, 5, 0, 1, 0, 0, 3712, 0, 0, 0, 210, 0, 0, 1018, 959, 0, 560, 1, 0, 2, 2424, 0, 1, 7, 20, 3, 34, 2077] payload_rows_per_dst=[2294, 4698, 4332, 6092, 7786, 7764, 7918, 6922] dup_rows_per_dst=[124, 622, 0, 934, 4300, 4588, 4670, 2492] combine_rows_per_src=[2294, 4698, 4332, 6092, 7786, 7764, 7918, 6922] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52326 payload_rows_remote=44704 dup_rows=13210 dup_rows_remote=10440 dedup_ratio=0.201569 router_tokens_per_expert=[0, 0, 0, 318, 4037, 0, 166, 859, 1482, 2036, 47, 0, 0, 125, 0, 718, 4, 2188, 0, 0, 3042, 3004, 0, 0, 0, 0, 2232, 1, 3, 816, 1, 3, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 3331, 1, 0, 0, 162, 0, 0, 1029, 968, 0, 692, 1, 0, 0, 2830, 1, 0, 3, 15, 1, 4, 2644] payload_rows_per_dst=[8130, 6604, 7622, 7238, 6090, 6798, 4206, 5638] dup_rows_per_dst=[2630, 2212, 2770, 3806, 2, 190, 1174, 426] combine_rows_per_src=[8130, 6604, 7622, 7238, 6090, 6798, 4206, 5638] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52544 payload_rows_remote=46456 dup_rows=12992 dup_rows_remote=11924 dedup_ratio=0.198242 router_tokens_per_expert=[0, 0, 0, 355, 3981, 0, 209, 735, 1992, 1589, 145, 0, 0, 155, 0, 934, 1, 2218, 0, 1, 2815, 2959, 0, 0, 0, 0, 2348, 0, 11, 1219, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3677, 1, 0, 0, 192, 0, 0, 1000, 1213, 0, 790, 0, 0, 0, 2119, 0, 0, 2, 5, 4, 21, 2076] payload_rows_per_dst=[8050, 6862, 7522, 6088, 5630, 7414, 4586, 6392] dup_rows_per_dst=[2510, 2768, 2836, 1068, 2, 326, 1420, 2062] combine_rows_per_src=[8050, 6862, 7522, 6088, 5630, 7414, 4586, 6392] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43760 payload_rows_remote=37900 dup_rows=21776 dup_rows_remote=18538 dedup_ratio=0.332275 router_tokens_per_expert=[1276, 1435, 791, 787, 477, 1475, 374, 83, 445, 1397, 316, 375, 29, 881, 115, 735, 661, 476, 23, 47, 864, 256, 316, 1086, 440, 599, 564, 868, 577, 152, 562, 5, 729, 564, 1084, 6, 554, 3, 1171, 195, 318, 749, 695, 782, 16, 623, 87, 503, 109, 529, 316, 340, 324, 516, 92, 581, 491, 614, 624, 402, 267, 704, 287, 6] payload_rows_per_dst=[7062, 5860, 5436, 5360, 5406, 5356, 4330, 4950] dup_rows_per_dst=[6416, 3238, 2022, 2174, 2612, 2190, 1284, 1840] combine_rows_per_src=[7062, 5860, 5436, 5360, 5406, 5356, 4330, 4950] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 2, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44534 payload_rows_remote=37810 dup_rows=21002 dup_rows_remote=15884 dedup_ratio=0.320465 router_tokens_per_expert=[1290, 990, 714, 1047, 148, 1211, 452, 69, 162, 683, 1588, 290, 3, 1147, 76, 737, 1130, 354, 8, 33, 889, 446, 694, 500, 334, 1466, 301, 782, 452, 113, 310, 6, 845, 441, 1230, 7, 1231, 0, 1237, 116, 367, 798, 744, 542, 0, 755, 54, 369, 134, 204, 424, 198, 220, 1103, 119, 851, 427, 614, 358, 244, 359, 139, 210, 3] payload_rows_per_dst=[6724, 6100, 5870, 6666, 5322, 5132, 4916, 3804] dup_rows_per_dst=[5118, 3272, 2238, 3336, 2418, 2126, 1590, 904] combine_rows_per_src=[6724, 6100, 5870, 6666, 5322, 5132, 4916, 3804] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278992896 dispatch_meta_bytes=229624 combine_payload_bytes=278992896 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279222520 ingress_bytes=389093376 link_bytes=389093376 token_bytes=558215416 weight_movement_bytes=110100480 total_nvlink_bytes=668315896 weight_to_token_ratio=0.197237 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=6656 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.766973757e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=283752448 dispatch_meta_bytes=231616 combine_payload_bytes=283752448 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=283984064 ingress_bytes=503953408 link_bytes=503953408 token_bytes=567736512 weight_movement_bytes=220200960 total_nvlink_bytes=787937472 weight_to_token_ratio=0.387858 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=7040 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.764578626e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=274176000 dispatch_meta_bytes=225496 combine_payload_bytes=274176000 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=274401496 ingress_bytes=384276480 link_bytes=384276480 token_bytes=548577496 weight_movement_bytes=110100480 total_nvlink_bytes=658677976 weight_to_token_ratio=0.200702 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=6656 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.683200013e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278204416 dispatch_meta_bytes=227408 combine_payload_bytes=278204416 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278431824 ingress_bytes=388304896 link_bytes=388304896 token_bytes=556636240 weight_movement_bytes=110100480 total_nvlink_bytes=666736720 weight_to_token_ratio=0.197796 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.753260793e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=269817856 dispatch_meta_bytes=223848 combine_payload_bytes=269817856 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=270041704 ingress_bytes=379918336 link_bytes=379918336 token_bytes=539859560 weight_movement_bytes=110100480 total_nvlink_bytes=649960040 weight_to_token_ratio=0.203943 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=7552 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.607404721e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=282791936 dispatch_meta_bytes=235776 combine_payload_bytes=282791936 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=283027712 ingress_bytes=392892416 link_bytes=392892416 token_bytes=565819648 weight_movement_bytes=110100480 total_nvlink_bytes=675920128 weight_to_token_ratio=0.194586 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.833045311e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=275910656 dispatch_meta_bytes=228352 combine_payload_bytes=275910656 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=276139008 ingress_bytes=606212096 link_bytes=606212096 token_bytes=552049664 weight_movement_bytes=330301440 total_nvlink_bytes=882351104 weight_to_token_ratio=0.598318 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=8064 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.054302540e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=271022080 dispatch_meta_bytes=214776 combine_payload_bytes=271022080 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=271236856 ingress_bytes=711424000 link_bytes=711424000 token_bytes=542258936 weight_movement_bytes=440401920 total_nvlink_bytes=982660856 weight_to_token_ratio=0.812162 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=6528 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.237283345e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53526 payload_rows_remote=46882 dup_rows=12010 dup_rows_remote=10656 dedup_ratio=0.183258 router_tokens_per_expert=[0, 0, 0, 313, 3547, 0, 344, 1822, 1538, 1534, 162, 0, 0, 131, 0, 1411, 1, 34, 0, 1, 2696, 2786, 0, 0, 0, 0, 2652, 0, 35, 1312, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3791, 2, 0, 0, 115, 0, 0, 1369, 2032, 0, 411, 0, 0, 0, 1872, 0, 0, 19, 19, 3, 33, 2782] payload_rows_per_dst=[7138, 6610, 5612, 6644, 7096, 7656, 5808, 6962] dup_rows_per_dst=[3212, 2942, 32, 1354, 0, 160, 1816, 2494] combine_rows_per_src=[7138, 6610, 5612, 6644, 7096, 7656, 5808, 6962] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53326 payload_rows_remote=46060 dup_rows=12210 dup_rows_remote=12210 dedup_ratio=0.186310 router_tokens_per_expert=[0, 0, 0, 345, 3633, 0, 235, 1387, 1829, 1253, 28, 0, 0, 33, 0, 1320, 0, 86, 0, 0, 2875, 2985, 0, 0, 0, 0, 2571, 0, 21, 699, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3511, 0, 0, 0, 285, 0, 0, 1836, 1835, 0, 306, 0, 0, 0, 2755, 0, 0, 11, 13, 0, 14, 2901] payload_rows_per_dst=[7004, 6448, 6040, 5820, 7266, 7140, 5956, 7652] dup_rows_per_dst=[2680, 2478, 102, 764, 0, 452, 1998, 3736] combine_rows_per_src=[7004, 6448, 6040, 5820, 7266, 7140, 5956, 7652] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48948 payload_rows_remote=41492 dup_rows=16588 dup_rows_remote=13426 dedup_ratio=0.253113 router_tokens_per_expert=[0, 0, 1, 328, 3084, 1, 179, 1704, 1735, 1418, 121, 0, 0, 34, 0, 1349, 0, 24, 0, 1, 3197, 2605, 1, 0, 0, 0, 3095, 2, 68, 878, 0, 0, 0, 0, 0, 0, 1, 4, 0, 1, 0, 0, 3825, 0, 0, 0, 101, 0, 0, 1569, 1844, 0, 287, 0, 0, 2, 2295, 0, 1, 10, 108, 3, 50, 2842] payload_rows_per_dst=[1008, 5168, 7722, 6820, 7222, 7834, 5718, 7456] dup_rows_per_dst=[10, 1310, 3934, 1266, 2366, 2854, 1686, 3162] combine_rows_per_src=[1008, 5168, 7722, 6820, 7222, 7834, 5718, 7456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52944 payload_rows_remote=45724 dup_rows=12592 dup_rows_remote=12410 dedup_ratio=0.192139 router_tokens_per_expert=[0, 0, 0, 323, 3778, 0, 306, 1675, 1065, 1772, 12, 0, 0, 51, 0, 1623, 2, 85, 0, 0, 2886, 2938, 0, 1, 0, 0, 2529, 2, 20, 1340, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 4, 3524, 0, 0, 0, 173, 0, 0, 1279, 1619, 2, 363, 1, 2, 2, 2543, 0, 3, 1, 5, 0, 4, 2830] payload_rows_per_dst=[6792, 6366, 5966, 6358, 7698, 7220, 5082, 7462] dup_rows_per_dst=[2426, 2680, 86, 1424, 1030, 182, 1454, 3310] combine_rows_per_src=[6792, 6366, 5966, 6358, 7698, 7220, 5082, 7462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47736 payload_rows_remote=42060 dup_rows=17800 dup_rows_remote=15928 dedup_ratio=0.271606 router_tokens_per_expert=[0, 0, 0, 167, 3869, 0, 428, 1710, 1237, 1689, 45, 1, 0, 69, 0, 703, 2, 0, 1, 0, 2909, 3121, 0, 0, 0, 0, 2449, 0, 23, 561, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 3635, 0, 0, 1, 10, 0, 1, 1824, 1644, 0, 305, 0, 0, 0, 2531, 0, 0, 1, 25, 0, 9, 3795] payload_rows_per_dst=[1216, 3896, 7810, 5546, 7972, 7586, 5676, 8034] dup_rows_per_dst=[12, 690, 4218, 520, 3190, 2610, 1872, 4688] combine_rows_per_src=[1216, 3896, 7810, 5546, 7972, 7586, 5676, 8034] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51846 payload_rows_remote=46068 dup_rows=13690 dup_rows_remote=13690 dedup_ratio=0.208893 router_tokens_per_expert=[0, 0, 0, 365, 4010, 0, 109, 533, 2274, 1460, 29, 0, 0, 30, 0, 812, 0, 2973, 0, 1, 2889, 3171, 0, 0, 0, 0, 1953, 0, 4, 542, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3412, 0, 0, 0, 374, 0, 0, 1206, 1010, 0, 662, 0, 0, 0, 2763, 0, 0, 2, 1, 0, 8, 2174] payload_rows_per_dst=[8094, 6878, 7982, 4572, 5778, 6938, 4412, 7192] dup_rows_per_dst=[1940, 2332, 4308, 428, 0, 634, 1344, 2704] combine_rows_per_src=[8094, 6878, 7982, 4572, 5778, 6938, 4412, 7192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=293056512 dispatch_meta_bytes=224488 combine_payload_bytes=293056512 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=293281000 ingress_bytes=513257472 link_bytes=513257472 token_bytes=586337512 weight_movement_bytes=220200960 total_nvlink_bytes=806538472 weight_to_token_ratio=0.375553 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=19328 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.926391602e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=320438272 dispatch_meta_bytes=220576 combine_payload_bytes=320438272 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=320658848 ingress_bytes=870940672 link_bytes=870940672 token_bytes=641097120 weight_movement_bytes=550502400 total_nvlink_bytes=1191599520 weight_to_token_ratio=0.858688 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=16640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.514709074e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50388 payload_rows_remote=43676 dup_rows=15148 dup_rows_remote=14872 dedup_ratio=0.231140 router_tokens_per_expert=[0, 0, 0, 383, 4040, 0, 141, 711, 1400, 1848, 5, 0, 0, 46, 0, 1066, 2, 2760, 0, 0, 3025, 3098, 0, 1, 0, 0, 2131, 2, 3, 1174, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 4, 3269, 0, 0, 0, 221, 0, 0, 840, 847, 2, 867, 1, 1, 2, 2564, 1, 3, 0, 0, 0, 1, 2304] payload_rows_per_dst=[4188, 6386, 6504, 5640, 7136, 6712, 6768, 7054] dup_rows_per_dst=[1016, 2344, 1446, 980, 2696, 276, 3698, 2692] combine_rows_per_src=[4188, 6386, 6504, 5640, 7136, 6712, 6768, 7054] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=332996608 dispatch_meta_bytes=233520 combine_payload_bytes=332996608 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=333230128 ingress_bytes=443097088 link_bytes=443097088 token_bytes=666226736 weight_movement_bytes=110100480 total_nvlink_bytes=776327216 weight_to_token_ratio=0.165260 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=17920 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.706187130e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49680 payload_rows_remote=41592 dup_rows=15856 dup_rows_remote=10768 dedup_ratio=0.241943 router_tokens_per_expert=[0, 0, 0, 576, 4005, 0, 75, 887, 1516, 1099, 3, 0, 0, 59, 0, 843, 0, 1908, 0, 1, 3487, 3078, 0, 0, 0, 0, 2800, 0, 7, 1168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2793, 0, 0, 0, 212, 0, 0, 992, 1095, 0, 769, 0, 0, 0, 3314, 0, 0, 0, 15, 0, 2, 2064] payload_rows_per_dst=[8088, 5834, 6484, 8038, 6974, 5746, 4354, 4162] dup_rows_per_dst=[5088, 2034, 572, 6540, 0, 264, 1358, 0] combine_rows_per_src=[8088, 5834, 6484, 8038, 6974, 5746, 4354, 4162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46140 payload_rows_remote=37976 dup_rows=19396 dup_rows_remote=13290 dedup_ratio=0.295959 router_tokens_per_expert=[0, 0, 0, 219, 4047, 0, 288, 718, 1648, 1805, 40, 1, 0, 63, 0, 492, 2, 1578, 1, 0, 3081, 3330, 0, 0, 0, 0, 1965, 0, 14, 581, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 3462, 0, 0, 1, 36, 0, 1, 1415, 922, 0, 750, 0, 0, 0, 2769, 0, 0, 1, 8, 2, 4, 3521] payload_rows_per_dst=[2358, 4472, 3162, 4638, 7968, 7418, 8164, 7960] dup_rows_per_dst=[92, 476, 0, 482, 4856, 2734, 6106, 4650] combine_rows_per_src=[2358, 4472, 3162, 4638, 7968, 7418, 8164, 7960] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=330215424 dispatch_meta_bytes=239032 combine_payload_bytes=330215424 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=330454456 ingress_bytes=550416384 link_bytes=550416384 token_bytes=660669880 weight_movement_bytes=220200960 total_nvlink_bytes=880870840 weight_to_token_ratio=0.333300 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=24832 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.572646197e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=313069568 dispatch_meta_bytes=234192 combine_payload_bytes=313069568 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=313303760 ingress_bytes=423170048 link_bytes=423170048 token_bytes=626373328 weight_movement_bytes=110100480 total_nvlink_bytes=736473808 weight_to_token_ratio=0.175775 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=27264 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.359623130e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50928 payload_rows_remote=43360 dup_rows=14608 dup_rows_remote=11852 dedup_ratio=0.222900 router_tokens_per_expert=[0, 0, 0, 648, 0, 4043, 3803, 0, 0, 1751, 0, 0, 0, 0, 668, 0, 982, 0, 0, 0, 0, 3908, 827, 0, 0, 0, 1746, 4096, 0, 2183, 1, 0, 0, 0, 0, 0, 0, 67, 1137, 2, 3480, 0, 9, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 11, 1, 3402, 1, 0, 0, 0, 1] payload_rows_per_dst=[4200, 4296, 5368, 5842, 8192, 7374, 8088, 7568] dup_rows_per_dst=[544, 542, 1626, 1748, 2412, 4060, 920, 2756] combine_rows_per_src=[4200, 4296, 5368, 5842, 8192, 7374, 8088, 7568] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=271667200 dispatch_meta_bytes=225752 combine_payload_bytes=271667200 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=271892952 ingress_bytes=601968640 link_bytes=601968640 token_bytes=543560152 weight_movement_bytes=330301440 total_nvlink_bytes=873861592 weight_to_token_ratio=0.607663 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=8192 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.046922472e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52744 payload_rows_remote=48536 dup_rows=12792 dup_rows_remote=12198 dedup_ratio=0.195190 router_tokens_per_expert=[0, 0, 0, 729, 4, 4075, 3550, 0, 0, 1667, 0, 0, 0, 0, 338, 0, 1215, 0, 0, 0, 0, 4067, 1186, 0, 0, 0, 2857, 4094, 1, 1550, 1, 0, 0, 0, 0, 2, 1, 44, 1152, 5, 3818, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 2391, 0, 0, 0, 0, 0] payload_rows_per_dst=[7222, 8176, 4208, 7218, 5310, 7640, 8150, 4820] dup_rows_per_dst=[1344, 3968, 594, 5486, 1400, 0, 0, 0] combine_rows_per_src=[7222, 8176, 4208, 7218, 5310, 7640, 8150, 4820] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=298131456 dispatch_meta_bytes=209440 combine_payload_bytes=298131456 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=298340896 ingress_bytes=518332416 link_bytes=518332416 token_bytes=596472352 weight_movement_bytes=220200960 total_nvlink_bytes=816673312 weight_to_token_ratio=0.369172 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=21504 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.014653225e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53548 payload_rows_remote=49792 dup_rows=11988 dup_rows_remote=11512 dedup_ratio=0.182922 router_tokens_per_expert=[0, 0, 0, 981, 0, 3983, 3362, 0, 0, 1754, 0, 0, 0, 0, 515, 0, 1644, 0, 0, 2, 0, 4037, 945, 0, 0, 0, 2015, 4096, 0, 1123, 0, 0, 0, 0, 0, 0, 0, 98, 1299, 2, 3851, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 3043, 0, 0, 0, 0, 0] payload_rows_per_dst=[6998, 5480, 6764, 3756, 8192, 7712, 7966, 6680] dup_rows_per_dst=[1688, 1866, 3684, 476, 2798, 4, 0, 1472] combine_rows_per_src=[6998, 5480, 6764, 3756, 8192, 7712, 7966, 6680] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=272211968 dispatch_meta_bytes=205064 combine_payload_bytes=272211968 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=272417032 ingress_bytes=382312448 link_bytes=382312448 token_bytes=544629000 weight_movement_bytes=110100480 total_nvlink_bytes=654729480 weight_to_token_ratio=0.202157 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=10496 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.649042267e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54414 payload_rows_remote=46222 dup_rows=11122 dup_rows_remote=8872 dedup_ratio=0.169708 router_tokens_per_expert=[0, 0, 0, 642, 2, 4064, 3657, 0, 0, 1557, 0, 0, 0, 0, 828, 0, 1273, 0, 0, 1, 0, 3826, 906, 0, 0, 0, 2048, 4096, 0, 1584, 0, 0, 0, 0, 0, 0, 0, 49, 1076, 0, 3850, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3300, 0, 0, 0, 0, 0] payload_rows_per_dst=[7384, 4190, 8034, 3168, 8192, 7714, 8128, 7604] dup_rows_per_dst=[1218, 580, 3978, 0, 2250, 2, 0, 3094] combine_rows_per_src=[7384, 4190, 8034, 3168, 8192, 7714, 8128, 7604] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=310804480 dispatch_meta_bytes=220848 combine_payload_bytes=310804480 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=311025328 ingress_bytes=420904960 link_bytes=420904960 token_bytes=621829808 weight_movement_bytes=110100480 total_nvlink_bytes=731930288 weight_to_token_ratio=0.177059 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=23168 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.320229524e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=347906048 dispatch_meta_bytes=242936 combine_payload_bytes=347906048 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=348148984 ingress_bytes=678207488 link_bytes=678207488 token_bytes=696055032 weight_movement_bytes=330301440 total_nvlink_bytes=1026356472 weight_to_token_ratio=0.474534 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=16256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.179514368e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=356909056 dispatch_meta_bytes=245216 combine_payload_bytes=356909056 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=357154272 ingress_bytes=687210496 link_bytes=687210496 token_bytes=714063328 weight_movement_bytes=330301440 total_nvlink_bytes=1044364768 weight_to_token_ratio=0.462566 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=11904 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.195172079e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=331319296 dispatch_meta_bytes=220376 combine_payload_bytes=331319296 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=331539672 ingress_bytes=441419776 link_bytes=441419776 token_bytes=662858968 weight_movement_bytes=110100480 total_nvlink_bytes=772959448 weight_to_token_ratio=0.166099 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=22656 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.677015915e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52476 payload_rows_remote=44674 dup_rows=13060 dup_rows_remote=13052 dedup_ratio=0.199280 router_tokens_per_expert=[0, 1, 0, 820, 0, 4063, 3409, 0, 0, 1593, 0, 0, 0, 0, 624, 0, 968, 0, 0, 0, 0, 4046, 1234, 1, 0, 0, 2557, 4094, 0, 880, 0, 0, 0, 0, 0, 0, 0, 119, 1367, 0, 3893, 2, 8, 0, 0, 1, 0, 1, 0, 0, 2, 1, 1, 0, 0, 0, 6, 0, 3076, 0, 0, 0, 1, 0] payload_rows_per_dst=[7002, 3886, 8160, 1760, 8188, 7802, 8130, 7548] dup_rows_per_dst=[1458, 548, 4338, 0, 2972, 8, 4, 3732] combine_rows_per_src=[7002, 3886, 8160, 1760, 8188, 7802, 8130, 7548] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49924 payload_rows_remote=42858 dup_rows=15612 dup_rows_remote=14034 dedup_ratio=0.238220 router_tokens_per_expert=[0, 0, 0, 934, 0, 4078, 3388, 0, 0, 2320, 0, 0, 0, 0, 283, 0, 572, 0, 0, 0, 0, 4062, 721, 0, 0, 0, 2461, 4096, 0, 1902, 0, 0, 0, 0, 0, 0, 0, 28, 1138, 0, 3936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2849, 0, 0, 0, 0, 0] payload_rows_per_dst=[7066, 8174, 2434, 8192, 2332, 7872, 8156, 5698] dup_rows_per_dst=[1578, 5156, 152, 8726, 0, 0, 0, 0] combine_rows_per_src=[7066, 8174, 2434, 8192, 2332, 7872, 8156, 5698] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53436 payload_rows_remote=45284 dup_rows=12100 dup_rows_remote=12100 dedup_ratio=0.184631 router_tokens_per_expert=[0, 0, 0, 787, 0, 4075, 3611, 0, 0, 2331, 0, 0, 0, 0, 199, 0, 1465, 0, 0, 1, 0, 4022, 833, 0, 0, 0, 2136, 4095, 0, 989, 3, 0, 0, 0, 0, 0, 0, 36, 1519, 1, 3919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 2743, 0, 0, 1, 0, 0] payload_rows_per_dst=[7358, 4806, 8136, 1978, 8192, 7838, 8152, 6976] dup_rows_per_dst=[1438, 254, 4506, 6, 3110, 0, 0, 2786] combine_rows_per_src=[7358, 4806, 8136, 1978, 8192, 7838, 8152, 6976] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51676 payload_rows_remote=43486 dup_rows=13860 dup_rows_remote=7418 dedup_ratio=0.211487 router_tokens_per_expert=[0, 0, 3840, 0, 0, 2942, 0, 0, 0, 0, 0, 4091, 2, 0, 0, 0, 0, 42, 2872, 0, 0, 0, 2727, 0, 0, 4096, 0, 3259, 0, 0, 2, 0, 0, 0, 0, 0, 881, 961, 0, 973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 1940, 4093, 0] payload_rows_per_dst=[1718, 8186, 3350, 6480, 7502, 8192, 8058, 8190] dup_rows_per_dst=[0, 0, 6, 4, 3582, 38, 3788, 6442] combine_rows_per_src=[1718, 8186, 3350, 6480, 7502, 8192, 8058, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51728 payload_rows_remote=44616 dup_rows=13808 dup_rows_remote=9906 dedup_ratio=0.210693 router_tokens_per_expert=[1, 0, 0, 405, 3985, 0, 258, 818, 2007, 1872, 174, 0, 0, 73, 2, 1379, 2, 2205, 0, 6, 3151, 2785, 0, 0, 6, 0, 1682, 3, 14, 1392, 0, 6, 0, 0, 0, 0, 4, 1, 0, 3, 0, 0, 3552, 2, 0, 2, 310, 1, 0, 842, 892, 0, 945, 0, 0, 0, 1393, 11, 0, 48, 51, 20, 3, 2462] payload_rows_per_dst=[8100, 7112, 7290, 6394, 6310, 7242, 4190, 5090] dup_rows_per_dst=[2834, 3902, 2706, 2598, 8, 492, 1168, 100] combine_rows_per_src=[8100, 7112, 7290, 6394, 6310, 7242, 4190, 5090] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=320223232 dispatch_meta_bytes=230904 combine_payload_bytes=320223232 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=320454136 ingress_bytes=430323712 link_bytes=430323712 token_bytes=640677368 weight_movement_bytes=110100480 total_nvlink_bytes=750777848 weight_to_token_ratio=0.171850 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=30592 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.484037113e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54868 payload_rows_remote=47080 dup_rows=10668 dup_rows_remote=6774 dedup_ratio=0.162781 router_tokens_per_expert=[0, 0, 3743, 0, 0, 3165, 0, 0, 1, 0, 0, 4092, 5, 0, 2, 0, 0, 33, 2656, 0, 0, 0, 3152, 0, 0, 4096, 0, 3093, 0, 0, 0, 0, 0, 0, 0, 0, 496, 511, 0, 1173, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 63, 0, 0, 0, 0, 2390, 4090, 0] payload_rows_per_dst=[7452, 8192, 7788, 6928, 3962, 8192, 7490, 4864] dup_rows_per_dst=[4158, 8, 3894, 2158, 398, 8, 2, 42] combine_rows_per_src=[7452, 8192, 7788, 6928, 3962, 8192, 7490, 4864] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54706 payload_rows_remote=46514 dup_rows=10830 dup_rows_remote=3784 dedup_ratio=0.165253 router_tokens_per_expert=[0, 0, 3943, 0, 0, 2876, 0, 0, 2, 0, 0, 4094, 0, 0, 0, 0, 0, 78, 2208, 0, 0, 0, 2638, 0, 0, 4090, 0, 3523, 0, 0, 1, 0, 0, 0, 0, 0, 1157, 1212, 0, 905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 1882, 4095, 0] payload_rows_per_dst=[5752, 8188, 7294, 8192, 5324, 8180, 7886, 3890] dup_rows_per_dst=[0, 4, 2554, 7046, 1224, 0, 2, 0] combine_rows_per_src=[5752, 8188, 7294, 8192, 5324, 8180, 7886, 3890] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56456 payload_rows_remote=49628 dup_rows=9080 dup_rows_remote=6640 dedup_ratio=0.138550 router_tokens_per_expert=[0, 0, 3763, 0, 0, 3101, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 43, 2990, 0, 0, 0, 2839, 0, 0, 4092, 0, 2894, 0, 0, 0, 0, 0, 0, 0, 0, 615, 623, 0, 1645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 1928, 4096, 0] payload_rows_per_dst=[6202, 8180, 6722, 6520, 6828, 8184, 7526, 6294] dup_rows_per_dst=[0, 0, 1520, 2630, 2440, 0, 6, 2484] combine_rows_per_src=[6202, 8180, 6722, 6520, 6828, 8184, 7526, 6294] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=324595712 dispatch_meta_bytes=229536 combine_payload_bytes=324595712 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=324825248 ingress_bytes=544796672 link_bytes=544796672 token_bytes=649420960 weight_movement_bytes=220200960 total_nvlink_bytes=869621920 weight_to_token_ratio=0.339073 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.474910163e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=307206144 dispatch_meta_bytes=227568 combine_payload_bytes=307206144 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=307433712 ingress_bytes=747608064 link_bytes=747608064 token_bytes=614639856 weight_movement_bytes=440401920 total_nvlink_bytes=1055041776 weight_to_token_ratio=0.716520 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=26112 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.300213384e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=311707648 dispatch_meta_bytes=203616 combine_payload_bytes=311707648 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=311911264 ingress_bytes=642009088 link_bytes=642009088 token_bytes=623618912 weight_movement_bytes=330301440 total_nvlink_bytes=953920352 weight_to_token_ratio=0.529653 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.055147 gemm_active_groups=5 gemm_peak_group_rows=16384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.116559396e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=319807488 dispatch_meta_bytes=218088 combine_payload_bytes=319807488 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=320025576 ingress_bytes=540008448 link_bytes=540008448 token_bytes=639833064 weight_movement_bytes=220200960 total_nvlink_bytes=860034024 weight_to_token_ratio=0.344154 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=13312 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.391635072e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=337469440 dispatch_meta_bytes=215416 combine_payload_bytes=337469440 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=337684856 ingress_bytes=667770880 link_bytes=667770880 token_bytes=675154296 weight_movement_bytes=330301440 total_nvlink_bytes=1005455736 weight_to_token_ratio=0.489224 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=20224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.161363390e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=333412352 dispatch_meta_bytes=201192 combine_payload_bytes=333412352 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=333613544 ingress_bytes=663713792 link_bytes=663713792 token_bytes=667025896 weight_movement_bytes=330301440 total_nvlink_bytes=997327336 weight_to_token_ratio=0.495185 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=25600 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.154307446e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=355733504 dispatch_meta_bytes=225072 combine_payload_bytes=355733504 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=355958576 ingress_bytes=465833984 link_bytes=465833984 token_bytes=711692080 weight_movement_bytes=110100480 total_nvlink_bytes=821792560 weight_to_token_ratio=0.154702 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.055147 gemm_active_groups=5 gemm_peak_group_rows=10368 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.101619147e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57422 payload_rows_remote=49238 dup_rows=8114 dup_rows_remote=8112 dedup_ratio=0.123810 router_tokens_per_expert=[0, 0, 3917, 0, 0, 3133, 0, 0, 0, 0, 0, 4092, 2, 0, 1, 0, 0, 29, 2670, 0, 0, 0, 2762, 0, 0, 4092, 0, 2947, 0, 1, 4, 0, 0, 0, 0, 0, 593, 778, 0, 1160, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 40, 0, 0, 0, 0, 2453, 4091, 0] payload_rows_per_dst=[6266, 8188, 5394, 5894, 7474, 8184, 7834, 8188] dup_rows_per_dst=[0, 2, 4, 10, 3112, 2, 4, 4980] combine_rows_per_src=[6266, 8188, 5394, 5894, 7474, 8184, 7834, 8188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57426 payload_rows_remote=50046 dup_rows=8110 dup_rows_remote=8110 dedup_ratio=0.123749 router_tokens_per_expert=[0, 0, 3690, 0, 0, 3203, 0, 0, 0, 0, 0, 4091, 1, 0, 0, 0, 0, 40, 2840, 0, 0, 0, 2861, 0, 0, 4095, 0, 2859, 0, 0, 1, 0, 0, 0, 0, 0, 329, 1018, 0, 1392, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 2196, 4095, 0] payload_rows_per_dst=[6406, 8184, 5744, 5720, 7608, 8192, 7380, 8192] dup_rows_per_dst=[0, 0, 16, 0, 3592, 4, 0, 4498] combine_rows_per_src=[6406, 8184, 5744, 5720, 7608, 8192, 7380, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53590 payload_rows_remote=45694 dup_rows=11946 dup_rows_remote=9554 dedup_ratio=0.182281 router_tokens_per_expert=[0, 5, 0, 0, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3548, 36, 0, 0, 0, 0, 4094, 4096, 0, 4094, 0, 0, 0, 0, 2, 1193, 4082, 0, 0, 3852, 0, 0, 0, 93, 0, 0, 0, 2172, 3714, 0, 0, 0, 277, 0, 0, 0, 1153, 0] payload_rows_per_dst=[2880, 8192, 8164, 7112, 8188, 3328, 7830, 7896] dup_rows_per_dst=[204, 0, 2, 56, 4400, 488, 4404, 2392] combine_rows_per_src=[2880, 8192, 8164, 7112, 8188, 3328, 7830, 7896] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=352937984 dispatch_meta_bytes=229400 combine_payload_bytes=352937984 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=353167384 ingress_bytes=573138944 link_bytes=573138944 token_bytes=706105368 weight_movement_bytes=220200960 total_nvlink_bytes=926306328 weight_to_token_ratio=0.311853 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.967828888e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59832 payload_rows_remote=51664 dup_rows=5704 dup_rows_remote=5700 dedup_ratio=0.087036 router_tokens_per_expert=[0, 10, 0, 0, 374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3727, 46, 0, 0, 0, 0, 4087, 4096, 0, 4091, 0, 0, 0, 0, 0, 1662, 4084, 0, 0, 3872, 0, 2, 0, 43, 0, 0, 0, 1758, 3848, 0, 0, 0, 212, 0, 0, 0, 854, 0] payload_rows_per_dst=[8192, 8192, 8168, 7526, 8174, 8106, 3568, 7906] dup_rows_per_dst=[758, 0, 4, 20, 0, 2966, 34, 1922] combine_rows_per_src=[8192, 8192, 8168, 7526, 8174, 8106, 3568, 7906] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48740 payload_rows_remote=40648 dup_rows=16796 dup_rows_remote=11274 dedup_ratio=0.256287 router_tokens_per_expert=[0, 0, 0, 1127, 18, 3921, 3374, 0, 0, 1955, 0, 0, 0, 0, 956, 0, 943, 0, 0, 3, 0, 3896, 830, 1, 0, 0, 2827, 4095, 0, 1666, 2, 0, 1, 0, 0, 0, 0, 98, 878, 21, 3723, 3, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 2363, 0, 0, 0, 0, 0] payload_rows_per_dst=[7062, 8092, 3282, 8192, 1980, 7494, 7842, 4796] dup_rows_per_dst=[1976, 5522, 272, 8988, 16, 16, 0, 6] combine_rows_per_src=[7062, 8092, 3282, 8192, 1980, 7494, 7842, 4796] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55996 payload_rows_remote=47804 dup_rows=9540 dup_rows_remote=3966 dedup_ratio=0.145569 router_tokens_per_expert=[0, 0, 3911, 0, 0, 2787, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 35, 3167, 0, 0, 0, 2342, 0, 0, 4096, 0, 3539, 0, 0, 0, 0, 0, 0, 0, 0, 554, 608, 0, 1475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 2023, 4096, 0] payload_rows_per_dst=[8192, 8192, 7634, 7078, 4772, 8192, 7822, 4114] dup_rows_per_dst=[5574, 120, 3334, 0, 502, 0, 4, 6] combine_rows_per_src=[8192, 8192, 7634, 7078, 4772, 8192, 7822, 4114] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=327534592 dispatch_meta_bytes=220992 combine_payload_bytes=327534592 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=327755584 ingress_bytes=437635072 link_bytes=437635072 token_bytes=655290176 weight_movement_bytes=110100480 total_nvlink_bytes=765390656 weight_to_token_ratio=0.168018 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=24448 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.611193688e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60644 payload_rows_remote=53240 dup_rows=4892 dup_rows_remote=4846 dedup_ratio=0.074646 router_tokens_per_expert=[0, 6, 0, 0, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3684, 41, 0, 0, 0, 0, 4088, 4096, 0, 4096, 0, 0, 0, 0, 0, 1384, 4077, 0, 0, 3880, 0, 0, 0, 137, 0, 0, 0, 2405, 3512, 0, 0, 0, 282, 0, 0, 0, 775, 0] payload_rows_per_dst=[8192, 8192, 8154, 7404, 8176, 8062, 4944, 7520] dup_rows_per_dst=[622, 0, 0, 46, 0, 2466, 140, 1618] combine_rows_per_src=[8192, 8192, 8154, 7404, 8176, 8062, 4944, 7520] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=358729728 dispatch_meta_bytes=232624 combine_payload_bytes=358729728 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=358962352 ingress_bytes=578930688 link_bytes=578930688 token_bytes=717692080 weight_movement_bytes=220200960 total_nvlink_bytes=937893040 weight_to_token_ratio=0.306818 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=30720 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.006855684e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59708 payload_rows_remote=51518 dup_rows=5828 dup_rows_remote=5828 dedup_ratio=0.088928 router_tokens_per_expert=[0, 3, 0, 0, 434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3997, 145, 0, 0, 0, 0, 4095, 4096, 0, 4095, 0, 0, 0, 0, 4, 1639, 4005, 0, 0, 3789, 0, 0, 0, 33, 0, 0, 0, 1553, 3891, 0, 0, 0, 227, 0, 0, 0, 762, 0] payload_rows_per_dst=[8192, 8192, 8010, 8022, 8190, 8034, 3160, 7908] dup_rows_per_dst=[872, 0, 0, 262, 0, 2830, 12, 1852] combine_rows_per_src=[8192, 8192, 8010, 8022, 8190, 8034, 3160, 7908] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 2, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=370327552 dispatch_meta_bytes=229456 combine_payload_bytes=370327552 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=370557008 ingress_bytes=590528512 link_bytes=590528512 token_bytes=740884560 weight_movement_bytes=220200960 total_nvlink_bytes=961085520 weight_to_token_ratio=0.297214 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.027026208e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=291364864 dispatch_meta_bytes=207688 combine_payload_bytes=291364864 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=291572552 ingress_bytes=401465344 link_bytes=401465344 token_bytes=582937416 weight_movement_bytes=110100480 total_nvlink_bytes=693037896 weight_to_token_ratio=0.188872 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=14976 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.982142629e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=381624320 dispatch_meta_bytes=232344 combine_payload_bytes=381624320 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=381856664 ingress_bytes=491724800 link_bytes=491724800 token_bytes=763480984 weight_movement_bytes=110100480 total_nvlink_bytes=873581464 weight_to_token_ratio=0.144209 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=30080 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.551903020e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=342659072 dispatch_meta_bytes=207080 combine_payload_bytes=342659072 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=342866152 ingress_bytes=672960512 link_bytes=672960512 token_bytes=685525224 weight_movement_bytes=330301440 total_nvlink_bytes=1015826664 weight_to_token_ratio=0.481822 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=22016 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.170389013e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59362 payload_rows_remote=53966 dup_rows=6174 dup_rows_remote=4646 dedup_ratio=0.094208 router_tokens_per_expert=[0, 10, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3717, 75, 0, 0, 0, 0, 4087, 4096, 0, 4091, 0, 0, 0, 0, 5, 1398, 4057, 0, 0, 3892, 0, 1, 0, 82, 0, 0, 0, 2022, 3890, 0, 0, 0, 172, 0, 0, 0, 833, 0] payload_rows_per_dst=[8192, 8192, 8114, 7508, 8174, 5396, 5848, 7938] dup_rows_per_dst=[690, 0, 0, 76, 0, 1528, 2028, 1852] combine_rows_per_src=[8192, 8192, 8114, 7508, 8174, 5396, 5848, 7938] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=369281024 dispatch_meta_bytes=229384 combine_payload_bytes=369281024 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=369510408 ingress_bytes=919783424 link_bytes=919783424 token_bytes=738791432 weight_movement_bytes=550502400 total_nvlink_bytes=1289293832 weight_to_token_ratio=0.745139 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=29952 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.599654653e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53292 payload_rows_remote=45388 dup_rows=12244 dup_rows_remote=7872 dedup_ratio=0.186829 router_tokens_per_expert=[0, 2526, 0, 3610, 0, 2405, 2, 2427, 0, 1, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 0, 4078, 0, 0, 0, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0, 3892, 0, 0, 0, 0, 0, 2821, 0, 6, 43, 0, 0, 4096, 0, 0, 0, 166, 0, 0, 2477, 0, 0, 0, 2, 0, 0, 0, 0] payload_rows_per_dst=[4, 8178, 8156, 5948, 7784, 7578, 7740, 7904] dup_rows_per_dst=[0, 2, 14, 176, 0, 3016, 4664, 4372] combine_rows_per_src=[4, 8178, 8156, 5948, 7784, 7578, 7740, 7904] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=386828288 dispatch_meta_bytes=234448 combine_payload_bytes=386828288 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=387062736 ingress_bytes=827230208 link_bytes=827230208 token_bytes=773891024 weight_movement_bytes=440401920 total_nvlink_bytes=1214292944 weight_to_token_ratio=0.569075 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=21120 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.438689388e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59548 payload_rows_remote=51696 dup_rows=5988 dup_rows_remote=2142 dedup_ratio=0.091370 router_tokens_per_expert=[0, 5, 0, 0, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3928, 38, 0, 0, 0, 0, 4093, 4096, 0, 4095, 0, 0, 0, 0, 1, 1613, 4075, 0, 0, 3859, 0, 0, 0, 65, 0, 0, 0, 1925, 3879, 0, 0, 0, 126, 0, 0, 0, 709, 0] payload_rows_per_dst=[8192, 8192, 8150, 7866, 8186, 3228, 7852, 7882] dup_rows_per_dst=[530, 0, 0, 66, 0, 0, 3846, 1546] combine_rows_per_src=[8192, 8192, 8150, 7866, 8186, 3228, 7852, 7882] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=325341184 dispatch_meta_bytes=213040 combine_payload_bytes=325341184 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=325554224 ingress_bytes=545542144 link_bytes=545542144 token_bytes=650895408 weight_movement_bytes=220200960 total_nvlink_bytes=871096368 weight_to_token_ratio=0.338305 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=28032 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.487875147e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58534 payload_rows_remote=50348 dup_rows=7002 dup_rows_remote=6990 dedup_ratio=0.106842 router_tokens_per_expert=[0, 2196, 0, 3447, 0, 2494, 13, 2635, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 8, 0, 0, 0, 4090, 0, 0, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 3967, 0, 0, 0, 0, 0, 2792, 0, 30, 68, 0, 0, 4096, 0, 0, 0, 178, 0, 0, 2460, 0, 0, 0, 22, 0, 0, 0, 0] payload_rows_per_dst=[8144, 8192, 8186, 8192, 7934, 5694, 5256, 6936] dup_rows_per_dst=[6532, 0, 12, 350, 0, 86, 20, 2] combine_rows_per_src=[8144, 8192, 8186, 8192, 7934, 5694, 5256, 6936] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58688 payload_rows_remote=50496 dup_rows=6848 dup_rows_remote=6546 dedup_ratio=0.104492 router_tokens_per_expert=[0, 2367, 0, 3015, 0, 2494, 6, 2452, 0, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 151, 0, 0, 0, 0, 0, 0, 0, 0, 3874, 0, 0, 0, 0, 0, 3438, 0, 11, 61, 0, 0, 4096, 0, 0, 0, 123, 0, 0, 2477, 0, 0, 0, 8, 1, 0, 0, 0] payload_rows_per_dst=[8004, 8192, 8184, 8192, 7748, 7144, 5188, 6036] dup_rows_per_dst=[5692, 12, 0, 302, 0, 818, 12, 12] combine_rows_per_src=[8004, 8192, 8184, 8192, 7748, 7144, 5188, 6036] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59716 payload_rows_remote=51778 dup_rows=5820 dup_rows_remote=5820 dedup_ratio=0.088806 router_tokens_per_expert=[0, 2210, 0, 3758, 0, 2753, 10, 2218, 0, 8, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 3969, 0, 0, 0, 0, 0, 2928, 0, 10, 93, 0, 0, 4096, 0, 0, 0, 129, 0, 0, 2258, 0, 0, 0, 0, 1, 0, 0, 0] payload_rows_per_dst=[4878, 8192, 8192, 8192, 7938, 7616, 7190, 7518] dup_rows_per_dst=[408, 16, 6, 264, 0, 2882, 2244, 0] combine_rows_per_src=[4878, 8192, 8192, 8192, 7938, 7616, 7190, 7518] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55544 payload_rows_remote=47452 dup_rows=9992 dup_rows_remote=9992 dedup_ratio=0.152466 router_tokens_per_expert=[0, 0, 3864, 0, 0, 2827, 0, 0, 3, 0, 0, 4034, 9, 0, 0, 0, 0, 56, 2051, 0, 0, 0, 3034, 0, 0, 4082, 0, 3416, 0, 1, 21, 0, 0, 0, 0, 0, 998, 1051, 0, 1559, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 124, 0, 0, 0, 0, 1552, 4081, 0] payload_rows_per_dst=[8184, 8092, 7500, 6832, 5698, 8166, 7728, 3344] dup_rows_per_dst=[5632, 0, 2782, 44, 1518, 2, 6, 8] combine_rows_per_src=[8184, 8092, 7500, 6832, 5698, 8166, 7728, 3344] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53810 payload_rows_remote=45618 dup_rows=11726 dup_rows_remote=10760 dedup_ratio=0.178925 router_tokens_per_expert=[0, 0, 0, 0, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3859, 89, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 1102, 4049, 0, 0, 3868, 0, 0, 0, 101, 0, 0, 0, 2151, 3865, 0, 0, 0, 128, 0, 0, 0, 785, 0] payload_rows_per_dst=[8192, 8192, 8098, 7946, 4032, 7446, 8080, 1824] dup_rows_per_dst=[966, 2, 660, 4108, 0, 1834, 4154, 2] combine_rows_per_src=[8192, 8192, 8098, 7946, 4032, 7446, 8080, 1824] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=370556928 dispatch_meta_bytes=215352 combine_payload_bytes=370556928 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=370772280 ingress_bytes=590757888 link_bytes=590757888 token_bytes=741329208 weight_movement_bytes=220200960 total_nvlink_bytes=961530168 weight_to_token_ratio=0.297035 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=16640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.027425130e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53326 payload_rows_remote=47692 dup_rows=12210 dup_rows_remote=12196 dedup_ratio=0.186310 router_tokens_per_expert=[0, 0, 0, 308, 3766, 0, 290, 2043, 1113, 1817, 49, 0, 1, 116, 0, 1159, 3, 34, 0, 0, 2791, 2787, 0, 0, 0, 0, 2541, 1, 17, 777, 1, 2, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 3599, 1, 0, 0, 120, 0, 0, 1412, 1751, 0, 342, 2, 0, 0, 2739, 0, 0, 10, 37, 1, 12, 3122] payload_rows_per_dst=[7454, 6216, 5634, 5794, 7540, 7320, 7020, 6348] dup_rows_per_dst=[3410, 2294, 14, 884, 0, 120, 4210, 1278] combine_rows_per_src=[7454, 6216, 5634, 5794, 7540, 7320, 7020, 6348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=336050176 dispatch_meta_bytes=230152 combine_payload_bytes=336050176 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=336280328 ingress_bytes=446150656 link_bytes=446150656 token_bytes=672330504 weight_movement_bytes=110100480 total_nvlink_bytes=782430984 weight_to_token_ratio=0.163759 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=20992 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.759293700e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50458 payload_rows_remote=43488 dup_rows=15078 dup_rows_remote=11002 dedup_ratio=0.230072 router_tokens_per_expert=[1, 0, 0, 361, 3577, 0, 357, 1658, 1635, 1730, 188, 0, 0, 69, 0, 1901, 6, 49, 0, 3, 3083, 2563, 0, 0, 6, 0, 2081, 13, 24, 1584, 0, 7, 0, 1, 0, 0, 4, 1, 0, 3, 0, 0, 3616, 1, 0, 1, 208, 1, 0, 1199, 1671, 0, 370, 0, 0, 0, 1453, 8, 0, 74, 234, 29, 10, 2988] payload_rows_per_dst=[7276, 6970, 6278, 6498, 7166, 7328, 6176, 2766] dup_rows_per_dst=[3644, 4076, 1772, 1722, 6, 326, 3472, 60] combine_rows_per_src=[7276, 6970, 6278, 6498, 7166, 7328, 6176, 2766] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=330158080 dispatch_meta_bytes=233080 combine_payload_bytes=330158080 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=330391160 ingress_bytes=550359040 link_bytes=550359040 token_bytes=660549240 weight_movement_bytes=220200960 total_nvlink_bytes=880750200 weight_to_token_ratio=0.333360 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=28800 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.571648891e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47946 payload_rows_remote=40236 dup_rows=17590 dup_rows_remote=13046 dedup_ratio=0.268402 router_tokens_per_expert=[0, 0, 0, 471, 3492, 0, 140, 2136, 1102, 899, 3, 0, 0, 70, 0, 1311, 0, 4, 0, 1, 3380, 2889, 0, 0, 0, 0, 2981, 0, 20, 1139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3277, 0, 0, 0, 97, 0, 0, 1428, 2020, 0, 348, 0, 0, 0, 2921, 0, 0, 1, 54, 0, 9, 2575] payload_rows_per_dst=[7710, 5564, 7310, 7808, 6984, 6658, 5788, 124] dup_rows_per_dst=[4544, 1828, 3006, 6314, 0, 90, 1804, 4] combine_rows_per_src=[7710, 5564, 7310, 7808, 6984, 6658, 5788, 124] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=297414656 dispatch_meta_bytes=219672 combine_payload_bytes=297414656 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=297634328 ingress_bytes=737816576 link_bytes=737816576 token_bytes=595048984 weight_movement_bytes=440401920 total_nvlink_bytes=1035450904 weight_to_token_ratio=0.740110 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=16256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.283184376e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=327749632 dispatch_meta_bytes=232536 combine_payload_bytes=327749632 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=327982168 ingress_bytes=437850112 link_bytes=437850112 token_bytes=655731800 weight_movement_bytes=110100480 total_nvlink_bytes=765832280 weight_to_token_ratio=0.167905 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=28800 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.614933587e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=301486080 dispatch_meta_bytes=231952 combine_payload_bytes=301486080 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=301718032 ingress_bytes=411586560 link_bytes=411586560 token_bytes=603204112 weight_movement_bytes=110100480 total_nvlink_bytes=713304592 weight_to_token_ratio=0.182526 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=14464 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.158167222e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=341856256 dispatch_meta_bytes=239552 combine_payload_bytes=341856256 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=342095808 ingress_bytes=562057216 link_bytes=562057216 token_bytes=683952064 weight_movement_bytes=220200960 total_nvlink_bytes=904153024 weight_to_token_ratio=0.321954 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=22784 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.775099412e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=288411648 dispatch_meta_bytes=213128 combine_payload_bytes=288411648 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=288624776 ingress_bytes=618713088 link_bytes=618713088 token_bytes=577036424 weight_movement_bytes=330301440 total_nvlink_bytes=907337864 weight_to_token_ratio=0.572410 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=17792 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.076043821e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=311721984 dispatch_meta_bytes=217960 combine_payload_bytes=311721984 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=311939944 ingress_bytes=531922944 link_bytes=531922944 token_bytes=623661928 weight_movement_bytes=220200960 total_nvlink_bytes=843862888 weight_to_token_ratio=0.353077 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=11264 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.251014859e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57228 payload_rows_remote=52366 dup_rows=8308 dup_rows_remote=8008 dedup_ratio=0.126770 router_tokens_per_expert=[0, 0, 0, 1306, 0, 0, 3903, 0, 0, 779, 0, 0, 0, 0, 0, 0, 3007, 0, 0, 149, 0, 4089, 1540, 0, 0, 0, 1980, 4096, 0, 601, 0, 0, 0, 2771, 0, 0, 0, 904, 301, 1, 3895, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 342, 0, 3007, 0, 0, 0, 0, 0] payload_rows_per_dst=[7892, 8192, 7244, 4862, 6610, 7802, 8178, 6448] dup_rows_per_dst=[2526, 1558, 2148, 300, 1344, 182, 0, 250] combine_rows_per_src=[7892, 8192, 7244, 4862, 6610, 7802, 8178, 6448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58010 payload_rows_remote=51728 dup_rows=7526 dup_rows_remote=6308 dedup_ratio=0.114838 router_tokens_per_expert=[0, 0, 0, 999, 1, 0, 4079, 0, 0, 495, 0, 0, 0, 0, 0, 0, 2667, 0, 0, 89, 0, 4050, 1869, 0, 0, 0, 2214, 4096, 0, 862, 0, 0, 0, 2563, 0, 0, 0, 992, 195, 0, 3927, 0, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 0, 3391, 0, 0, 0, 0, 0] payload_rows_per_dst=[8164, 8192, 6980, 5508, 6282, 7884, 8100, 6900] dup_rows_per_dst=[1994, 990, 2270, 644, 1218, 192, 0, 218] combine_rows_per_src=[8164, 8192, 6980, 5508, 6282, 7884, 8100, 6900] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46746 payload_rows_remote=38554 dup_rows=18790 dup_rows_remote=11130 dedup_ratio=0.286713 router_tokens_per_expert=[0, 0, 0, 1047, 0, 0, 4069, 0, 0, 666, 0, 0, 0, 0, 0, 0, 2561, 0, 0, 91, 0, 4055, 1503, 0, 0, 0, 1837, 4096, 0, 1106, 0, 0, 0, 2662, 0, 0, 0, 1283, 186, 2, 3659, 0, 87, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 276, 1, 3578, 2, 0, 0, 0, 0] payload_rows_per_dst=[2094, 1332, 3926, 8192, 7564, 7332, 8114, 8192] dup_rows_per_dst=[0, 0, 570, 5886, 4034, 160, 480, 7660] combine_rows_per_src=[2094, 1332, 3926, 8192, 7564, 7332, 8114, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56942 payload_rows_remote=49062 dup_rows=8594 dup_rows_remote=8532 dedup_ratio=0.131134 router_tokens_per_expert=[0, 1, 0, 1060, 0, 0, 4039, 0, 0, 500, 0, 0, 0, 0, 0, 0, 2192, 0, 0, 101, 0, 4065, 2365, 1, 0, 0, 2563, 4094, 0, 367, 0, 0, 0, 2665, 0, 0, 0, 1114, 286, 0, 3928, 2, 39, 0, 0, 1, 0, 1, 0, 0, 2, 1, 0, 0, 0, 0, 256, 1, 3123, 0, 0, 0, 1, 0] payload_rows_per_dst=[5956, 8190, 6904, 6774, 6570, 7880, 8134, 6534] dup_rows_per_dst=[1182, 998, 2414, 2148, 1560, 62, 2, 228] combine_rows_per_src=[5956, 8190, 6904, 6774, 6570, 7880, 8134, 6534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52276 payload_rows_remote=44144 dup_rows=13260 dup_rows_remote=13258 dedup_ratio=0.202332 router_tokens_per_expert=[0, 0, 0, 1388, 0, 0, 4082, 0, 0, 868, 0, 0, 1, 0, 0, 0, 3184, 0, 0, 37, 0, 4066, 1518, 0, 0, 0, 2319, 4095, 0, 521, 2, 0, 0, 2545, 0, 0, 0, 628, 437, 1, 3979, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 31, 0, 3059, 0, 0, 1, 0, 0] payload_rows_per_dst=[2776, 5750, 7378, 6548, 6152, 7992, 8132, 7548] dup_rows_per_dst=[0, 1206, 2100, 2562, 1070, 2338, 2, 3982] combine_rows_per_src=[2776, 5750, 7378, 6548, 6152, 7992, 8132, 7548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58006 payload_rows_remote=50904 dup_rows=7530 dup_rows_remote=5118 dedup_ratio=0.114899 router_tokens_per_expert=[0, 0, 0, 1052, 4, 0, 4033, 0, 0, 394, 0, 0, 0, 0, 0, 0, 2683, 0, 0, 70, 0, 4085, 2004, 0, 0, 0, 2849, 4094, 1, 848, 1, 0, 0, 3192, 0, 2, 1, 624, 141, 5, 3931, 0, 28, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 0, 2563, 0, 0, 0, 0, 0] payload_rows_per_dst=[8090, 8192, 7102, 6366, 6842, 7870, 8170, 5374] dup_rows_per_dst=[2088, 784, 2412, 1032, 1088, 50, 0, 76] combine_rows_per_src=[8090, 8192, 7102, 6366, 6842, 7870, 8170, 5374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=375359488 dispatch_meta_bytes=241496 combine_payload_bytes=375359488 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=375600984 ingress_bytes=595560448 link_bytes=595560448 token_bytes=750960472 weight_movement_bytes=220200960 total_nvlink_bytes=971161432 weight_to_token_ratio=0.293226 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=19200 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.035777572e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=370786304 dispatch_meta_bytes=232144 combine_payload_bytes=370786304 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=371018448 ingress_bytes=480886784 link_bytes=480886784 token_bytes=741804752 weight_movement_bytes=110100480 total_nvlink_bytes=851905232 weight_to_token_ratio=0.148422 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=22144 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.363412097e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56294 payload_rows_remote=48134 dup_rows=9242 dup_rows_remote=5350 dedup_ratio=0.141022 router_tokens_per_expert=[0, 0, 0, 1609, 0, 0, 4059, 0, 0, 1114, 0, 0, 0, 0, 0, 0, 2046, 0, 0, 116, 0, 4084, 1586, 0, 0, 0, 2498, 4096, 0, 1020, 0, 0, 0, 2540, 0, 0, 0, 586, 158, 0, 3973, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 3173, 0, 0, 0, 0, 0] payload_rows_per_dst=[8160, 8192, 5488, 6086, 5854, 7946, 8168, 6400] dup_rows_per_dst=[3892, 2228, 1292, 950, 714, 50, 0, 116] combine_rows_per_src=[8160, 8192, 5488, 6086, 5854, 7946, 8168, 6400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=276355072 dispatch_meta_bytes=198736 combine_payload_bytes=276355072 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276553808 ingress_bytes=386455552 link_bytes=386455552 token_bytes=552908880 weight_movement_bytes=110100480 total_nvlink_bytes=663009360 weight_to_token_ratio=0.199130 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=24448 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.721097659e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57002 payload_rows_remote=48814 dup_rows=8534 dup_rows_remote=6990 dedup_ratio=0.130219 router_tokens_per_expert=[0, 0, 0, 1386, 14, 0, 3961, 0, 0, 772, 0, 0, 0, 0, 0, 0, 2055, 0, 0, 105, 0, 4028, 1493, 1, 0, 0, 2883, 4094, 0, 955, 1, 0, 1, 3138, 0, 0, 0, 976, 181, 21, 3771, 2, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 281, 0, 2524, 0, 0, 0, 0, 0] payload_rows_per_dst=[8010, 8188, 5916, 6702, 7020, 7600, 8056, 5510] dup_rows_per_dst=[2712, 1544, 1392, 976, 1614, 196, 0, 100] combine_rows_per_src=[8010, 8188, 5916, 6702, 7020, 7600, 8056, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=351676416 dispatch_meta_bytes=230376 combine_payload_bytes=351676416 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=351906792 ingress_bytes=461776896 link_bytes=461776896 token_bytes=703583208 weight_movement_bytes=110100480 total_nvlink_bytes=813683688 weight_to_token_ratio=0.156485 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=31104 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.031059714e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=364879872 dispatch_meta_bytes=224088 combine_payload_bytes=364879872 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=365103960 ingress_bytes=805281792 link_bytes=805281792 token_bytes=729983832 weight_movement_bytes=440401920 total_nvlink_bytes=1170385752 weight_to_token_ratio=0.603304 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=18176 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.400517482e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=316424192 dispatch_meta_bytes=229608 combine_payload_bytes=316424192 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=316653800 ingress_bytes=536625152 link_bytes=536625152 token_bytes=633077992 weight_movement_bytes=220200960 total_nvlink_bytes=853278952 weight_to_token_ratio=0.347826 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.332793990e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=345024512 dispatch_meta_bytes=213936 combine_payload_bytes=345024512 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=345238448 ingress_bytes=785426432 link_bytes=785426432 token_bytes=690262960 weight_movement_bytes=440401920 total_nvlink_bytes=1130664880 weight_to_token_ratio=0.638021 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=22656 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.365985746e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=349898752 dispatch_meta_bytes=223216 combine_payload_bytes=349898752 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=350121968 ingress_bytes=459999232 link_bytes=459999232 token_bytes=700020720 weight_movement_bytes=110100480 total_nvlink_bytes=810121200 weight_to_token_ratio=0.157282 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=27264 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.000143213e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58380 payload_rows_remote=51668 dup_rows=7156 dup_rows_remote=7152 dedup_ratio=0.109192 router_tokens_per_expert=[0, 0, 1942, 0, 0, 3853, 0, 0, 3, 0, 0, 779, 0, 0, 0, 0, 0, 0, 2506, 0, 0, 0, 3779, 0, 0, 3356, 0, 3926, 0, 0, 2, 0, 0, 0, 0, 0, 2595, 1437, 0, 821, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 3560, 4095, 0] payload_rows_per_dst=[8096, 7964, 5128, 6712, 7282, 8192, 7854, 7152] dup_rows_per_dst=[3494, 1158, 74, 4, 2234, 0, 80, 112] combine_rows_per_src=[8096, 7964, 5128, 6712, 7282, 8192, 7854, 7152] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60280 payload_rows_remote=53012 dup_rows=5256 dup_rows_remote=3294 dedup_ratio=0.080200 router_tokens_per_expert=[0, 0, 1165, 0, 0, 3985, 0, 0, 0, 0, 0, 496, 0, 0, 0, 0, 0, 0, 3180, 0, 0, 0, 4013, 0, 0, 3541, 0, 3861, 0, 0, 0, 0, 0, 0, 0, 0, 2197, 1000, 0, 1418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 3731, 4096, 0] payload_rows_per_dst=[8116, 8072, 6360, 7082, 7268, 8192, 7722, 7468] dup_rows_per_dst=[2184, 946, 0, 0, 1962, 0, 80, 84] combine_rows_per_src=[8116, 8072, 6360, 7082, 7268, 8192, 7722, 7468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48846 payload_rows_remote=41284 dup_rows=16690 dup_rows_remote=11286 dedup_ratio=0.254669 router_tokens_per_expert=[0, 0, 1237, 0, 0, 3916, 0, 0, 0, 0, 0, 565, 2, 0, 0, 0, 0, 0, 3026, 0, 0, 0, 3853, 0, 0, 3511, 0, 3872, 0, 0, 1, 0, 0, 0, 0, 0, 2634, 1381, 0, 1089, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 3500, 4094, 0] payload_rows_per_dst=[2474, 3042, 7294, 4670, 7670, 8192, 7942, 7562] dup_rows_per_dst=[0, 448, 4108, 1060, 3276, 0, 2394, 5404] combine_rows_per_src=[2474, 3042, 7294, 4670, 7670, 8192, 7942, 7562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56880 payload_rows_remote=48696 dup_rows=8656 dup_rows_remote=8654 dedup_ratio=0.132080 router_tokens_per_expert=[0, 0, 1760, 0, 0, 3996, 0, 0, 1, 0, 0, 623, 2, 0, 1, 0, 0, 0, 2738, 0, 0, 0, 3823, 0, 0, 3313, 0, 3798, 0, 1, 5, 0, 0, 0, 0, 0, 2133, 1181, 0, 1365, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 29, 0, 1, 0, 0, 0, 44, 0, 0, 0, 0, 3861, 4091, 0] payload_rows_per_dst=[5026, 7890, 5476, 7794, 7186, 8184, 7596, 7728] dup_rows_per_dst=[1234, 1010, 0, 4096, 2172, 2, 60, 82] combine_rows_per_src=[5026, 7890, 5476, 7794, 7186, 8184, 7596, 7728] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=379990016 dispatch_meta_bytes=225224 combine_payload_bytes=379990016 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=380215240 ingress_bytes=600190976 link_bytes=600190976 token_bytes=760205256 weight_movement_bytes=220200960 total_nvlink_bytes=980406216 weight_to_token_ratio=0.289660 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=11136 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.043830822e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=370356224 dispatch_meta_bytes=235280 combine_payload_bytes=370356224 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=370591504 ingress_bytes=700657664 link_bytes=700657664 token_bytes=740947728 weight_movement_bytes=330301440 total_nvlink_bytes=1071249168 weight_to_token_ratio=0.445782 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=25600 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.218558916e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59142 payload_rows_remote=51588 dup_rows=6394 dup_rows_remote=3460 dedup_ratio=0.097565 router_tokens_per_expert=[0, 0, 1119, 0, 0, 4002, 0, 0, 3, 0, 0, 363, 5, 0, 2, 0, 0, 0, 3005, 0, 0, 0, 4067, 0, 0, 3299, 0, 3956, 0, 0, 0, 0, 0, 0, 0, 0, 2239, 1119, 0, 1484, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 35, 2, 0, 0, 0, 0, 74, 0, 0, 0, 0, 3898, 4091, 0] payload_rows_per_dst=[8120, 8142, 7554, 6598, 4800, 8190, 7912, 7826] dup_rows_per_dst=[2122, 738, 2934, 0, 406, 2, 74, 118] combine_rows_per_src=[8120, 8142, 7554, 6598, 4800, 8190, 7912, 7826] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55782 payload_rows_remote=47864 dup_rows=9754 dup_rows_remote=9664 dedup_ratio=0.148834 router_tokens_per_expert=[0, 0, 929, 0, 0, 3988, 0, 0, 0, 0, 0, 503, 1, 0, 0, 0, 0, 0, 3132, 0, 0, 0, 3996, 0, 0, 3121, 0, 3957, 0, 0, 1, 0, 0, 0, 0, 0, 1606, 1725, 0, 1781, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 3827, 4095, 0] payload_rows_per_dst=[1858, 8078, 6264, 8188, 7604, 8192, 7918, 7680] dup_rows_per_dst=[0, 922, 0, 6032, 2620, 6, 90, 84] combine_rows_per_src=[1858, 8078, 6264, 8188, 7604, 8192, 7918, 7680] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=295923712 dispatch_meta_bytes=210280 combine_payload_bytes=295923712 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=296133992 ingress_bytes=626225152 link_bytes=626225152 token_bytes=592057704 weight_movement_bytes=330301440 total_nvlink_bytes=922359144 weight_to_token_ratio=0.557887 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=29312 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.089108536e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=349052928 dispatch_meta_bytes=229400 combine_payload_bytes=349052928 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=349282328 ingress_bytes=569253888 link_bytes=569253888 token_bytes=698335256 weight_movement_bytes=220200960 total_nvlink_bytes=918536216 weight_to_token_ratio=0.315323 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.900261374e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57886 payload_rows_remote=49936 dup_rows=7650 dup_rows_remote=7042 dedup_ratio=0.116730 router_tokens_per_expert=[0, 0, 1966, 0, 0, 3882, 0, 0, 9, 0, 0, 337, 7, 0, 0, 0, 0, 0, 2255, 0, 0, 0, 3926, 0, 0, 3297, 0, 3867, 0, 1, 23, 0, 0, 0, 0, 0, 2549, 1448, 0, 1669, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 159, 0, 0, 0, 0, 3220, 4083, 0] payload_rows_per_dst=[8158, 7950, 7246, 6612, 5350, 8168, 7752, 6650] dup_rows_per_dst=[3538, 608, 2362, 30, 884, 8, 112, 108] combine_rows_per_src=[8158, 7950, 7246, 6612, 5350, 8168, 7752, 6650] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58232 payload_rows_remote=50126 dup_rows=7304 dup_rows_remote=4294 dedup_ratio=0.111450 router_tokens_per_expert=[0, 0, 1646, 0, 0, 3912, 0, 0, 0, 0, 0, 566, 0, 0, 0, 0, 0, 0, 3321, 0, 0, 0, 3718, 0, 0, 3329, 0, 3934, 0, 0, 0, 0, 0, 0, 0, 0, 2047, 981, 0, 1430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 3710, 4096, 0] payload_rows_per_dst=[8106, 7772, 7736, 6658, 4496, 8192, 7868, 7404] dup_rows_per_dst=[3010, 796, 3000, 0, 326, 16, 82, 74] combine_rows_per_src=[8106, 7772, 7736, 6658, 4496, 8192, 7868, 7404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=369782784 dispatch_meta_bytes=220192 combine_payload_bytes=369782784 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=370002976 ingress_bytes=589983744 link_bytes=589983744 token_bytes=739785760 weight_movement_bytes=220200960 total_nvlink_bytes=959986720 weight_to_token_ratio=0.297655 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=23168 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.026078766e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57254 payload_rows_remote=50174 dup_rows=8282 dup_rows_remote=6396 dedup_ratio=0.126373 router_tokens_per_expert=[0, 2621, 0, 3555, 0, 2454, 2, 1674, 0, 5, 0, 0, 0, 0, 0, 4092, 0, 2, 1, 0, 0, 0, 4087, 0, 0, 0, 0, 155, 0, 2, 0, 0, 0, 0, 0, 0, 3922, 0, 0, 0, 0, 0, 2685, 0, 14, 110, 0, 0, 4094, 0, 0, 0, 109, 0, 0, 3179, 0, 0, 0, 4, 1, 0, 0, 0] payload_rows_per_dst=[2562, 8184, 8178, 8192, 8006, 7080, 7930, 7122] dup_rows_per_dst=[0, 10, 2, 310, 2498, 1886, 3554, 22] combine_rows_per_src=[2562, 8184, 8178, 8192, 8006, 7080, 7930, 7122] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=360894464 dispatch_meta_bytes=229352 combine_payload_bytes=360894464 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=361123816 ingress_bytes=470994944 link_bytes=470994944 token_bytes=722018280 weight_movement_bytes=110100480 total_nvlink_bytes=832118760 weight_to_token_ratio=0.152490 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.191376730e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=361955328 dispatch_meta_bytes=228168 combine_payload_bytes=361955328 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=362183496 ingress_bytes=472055808 link_bytes=472055808 token_bytes=724138824 weight_movement_bytes=110100480 total_nvlink_bytes=834239304 weight_to_token_ratio=0.152043 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.209826900e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=371144704 dispatch_meta_bytes=230392 combine_payload_bytes=371144704 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=371375096 ingress_bytes=481245184 link_bytes=481245184 token_bytes=742519800 weight_movement_bytes=110100480 total_nvlink_bytes=852620280 weight_to_token_ratio=0.148280 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31488 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.369645262e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=326989824 dispatch_meta_bytes=225512 combine_payload_bytes=326989824 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=327215336 ingress_bytes=437090304 link_bytes=437090304 token_bytes=654205160 weight_movement_bytes=110100480 total_nvlink_bytes=764305640 weight_to_token_ratio=0.168297 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=29824 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.601719277e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=340135936 dispatch_meta_bytes=229776 combine_payload_bytes=340135936 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=340365712 ingress_bytes=450236416 link_bytes=450236416 token_bytes=680501648 weight_movement_bytes=110100480 total_nvlink_bytes=790602128 weight_to_token_ratio=0.161793 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.830351786e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54092 payload_rows_remote=45900 dup_rows=11444 dup_rows_remote=11436 dedup_ratio=0.174622 router_tokens_per_expert=[0, 0, 0, 0, 0, 4056, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1209, 0, 4096, 0, 0, 0, 0, 2110, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 1405, 0, 0, 0, 0, 4096, 0, 4096, 4016, 2082, 0, 0, 1434, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8112, 8132, 2418, 8192, 8044, 2810, 8192, 8192] dup_rows_per_dst=[0, 4, 0, 4220, 120, 0, 7092, 8] combine_rows_per_src=[8112, 8132, 2418, 8192, 8044, 2810, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=359647232 dispatch_meta_bytes=226280 combine_payload_bytes=359647232 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=359873512 ingress_bytes=469747712 link_bytes=469747712 token_bytes=719520744 weight_movement_bytes=110100480 total_nvlink_bytes=829621224 weight_to_token_ratio=0.153019 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=23040 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.169685314e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=343089152 dispatch_meta_bytes=230112 combine_payload_bytes=343089152 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=343319264 ingress_bytes=563290112 link_bytes=563290112 token_bytes=686408416 weight_movement_bytes=220200960 total_nvlink_bytes=906609376 weight_to_token_ratio=0.320802 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=31232 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.796541501e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=357941248 dispatch_meta_bytes=227912 combine_payload_bytes=357941248 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=358169160 ingress_bytes=468041728 link_bytes=468041728 token_bytes=716110408 weight_movement_bytes=110100480 total_nvlink_bytes=826210888 weight_to_token_ratio=0.153748 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=28544 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.140015446e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60176 payload_rows_remote=52306 dup_rows=5360 dup_rows_remote=5360 dedup_ratio=0.081787 router_tokens_per_expert=[0, 3, 0, 0, 275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 0, 0, 0, 0, 0, 3392, 211, 0, 0, 0, 0, 4093, 3935, 0, 4095, 0, 0, 0, 0, 4, 476, 2082, 0, 0, 3745, 0, 0, 0, 56, 0, 0, 0, 2060, 3861, 0, 0, 0, 306, 0, 0, 0, 4042, 0] payload_rows_per_dst=[7904, 8190, 8186, 6872, 7870, 4942, 8090, 8122] dup_rows_per_dst=[142, 0, 264, 334, 0, 182, 3864, 574] combine_rows_per_src=[7904, 8190, 8186, 6872, 7870, 4942, 8090, 8122] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=359303168 dispatch_meta_bytes=217680 combine_payload_bytes=359303168 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=359520848 ingress_bytes=689604608 link_bytes=689604608 token_bytes=718824016 weight_movement_bytes=330301440 total_nvlink_bytes=1049125456 weight_to_token_ratio=0.459503 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=21120 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.199335834e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59434 payload_rows_remote=54826 dup_rows=6102 dup_rows_remote=6062 dedup_ratio=0.093109 router_tokens_per_expert=[0, 3, 0, 0, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 0, 0, 0, 0, 0, 0, 2155, 169, 0, 0, 0, 0, 4081, 3722, 0, 4096, 0, 0, 0, 0, 0, 443, 2741, 0, 0, 3917, 0, 0, 0, 175, 0, 0, 0, 2860, 3577, 0, 0, 0, 414, 0, 0, 0, 4056, 0] payload_rows_per_dst=[8036, 8192, 8164, 4608, 7444, 7350, 7508, 8132] dup_rows_per_dst=[260, 0, 260, 40, 0, 2928, 1806, 808] combine_rows_per_src=[8036, 8192, 8164, 4608, 7444, 7350, 7508, 8132] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 2, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=374929408 dispatch_meta_bytes=230664 combine_payload_bytes=374929408 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=375160072 ingress_bytes=815331328 link_bytes=815331328 token_bytes=750089480 weight_movement_bytes=440401920 total_nvlink_bytes=1190491400 weight_to_token_ratio=0.587133 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=30464 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.417995278e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59030 payload_rows_remote=53020 dup_rows=6506 dup_rows_remote=6262 dedup_ratio=0.099274 router_tokens_per_expert=[0, 10, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 0, 0, 0, 0, 0, 0, 2114, 148, 0, 0, 0, 0, 4087, 3872, 0, 4091, 0, 0, 0, 0, 5, 469, 2652, 0, 0, 3939, 0, 1, 0, 185, 0, 0, 0, 2623, 3919, 0, 0, 0, 247, 0, 0, 0, 4058, 0] payload_rows_per_dst=[8100, 8182, 8174, 4522, 7744, 6010, 8152, 8146] dup_rows_per_dst=[260, 0, 234, 2, 0, 244, 5302, 464] combine_rows_per_src=[8100, 8182, 8174, 4522, 7744, 6010, 8152, 8146] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56840 payload_rows_remote=48662 dup_rows=8696 dup_rows_remote=7970 dedup_ratio=0.132690 router_tokens_per_expert=[0, 3, 0, 0, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 0, 0, 0, 0, 0, 0, 1931, 206, 0, 0, 0, 0, 4088, 3694, 0, 4094, 0, 0, 0, 0, 1, 395, 2950, 0, 0, 3855, 0, 0, 0, 131, 0, 0, 0, 2865, 3739, 0, 0, 0, 377, 0, 0, 0, 4075, 0] payload_rows_per_dst=[7254, 8188, 6650, 4200, 7560, 6656, 8154, 8178] dup_rows_per_dst=[100, 0, 224, 74, 1412, 844, 5316, 726] combine_rows_per_src=[7254, 8188, 6650, 4200, 7560, 6656, 8154, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=392992768 dispatch_meta_bytes=243552 combine_payload_bytes=392992768 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=393236320 ingress_bytes=503093248 link_bytes=503093248 token_bytes=786229088 weight_movement_bytes=110100480 total_nvlink_bytes=896329568 weight_to_token_ratio=0.140036 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=20352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.749619029e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56606 payload_rows_remote=48450 dup_rows=8930 dup_rows_remote=8674 dedup_ratio=0.136261 router_tokens_per_expert=[0, 9, 0, 0, 237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 130, 0, 0, 0, 0, 0, 0, 2740, 140, 0, 0, 0, 0, 4075, 3833, 0, 4090, 0, 0, 0, 0, 1, 598, 2583, 0, 0, 3965, 0, 2, 0, 148, 0, 0, 0, 1932, 3894, 0, 0, 0, 319, 0, 0, 0, 4071, 0] payload_rows_per_dst=[8090, 8180, 8156, 7908, 7666, 7800, 618, 8188] dup_rows_per_dst=[332, 0, 256, 5308, 0, 2432, 10, 592] combine_rows_per_src=[8090, 8180, 8156, 7908, 7666, 7800, 618, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59216 payload_rows_remote=51072 dup_rows=6320 dup_rows_remote=992 dedup_ratio=0.096436 router_tokens_per_expert=[0, 4, 0, 0, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 0, 0, 0, 0, 0, 0, 2705, 118, 0, 0, 0, 0, 4092, 3794, 0, 4095, 0, 0, 0, 0, 1, 582, 2152, 0, 0, 3931, 0, 0, 0, 145, 0, 0, 0, 2698, 3893, 0, 0, 0, 151, 0, 0, 0, 4084, 0] payload_rows_per_dst=[8086, 8190, 8186, 5588, 7588, 5260, 8144, 8174] dup_rows_per_dst=[178, 0, 250, 58, 0, 210, 5328, 296] combine_rows_per_src=[8086, 8190, 8186, 5588, 7588, 5260, 8144, 8174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55682 payload_rows_remote=47552 dup_rows=9854 dup_rows_remote=9854 dedup_ratio=0.150360 router_tokens_per_expert=[0, 70, 0, 0, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 0, 0, 0, 0, 0, 0, 2438, 312, 0, 0, 0, 0, 3886, 3774, 0, 4065, 0, 0, 0, 0, 2, 204, 2571, 0, 0, 3866, 0, 0, 0, 447, 0, 0, 0, 2606, 3431, 0, 0, 0, 807, 0, 0, 0, 3909, 0] payload_rows_per_dst=[8046, 8130, 7830, 7558, 7548, 7640, 894, 8036] dup_rows_per_dst=[278, 0, 250, 4804, 0, 3126, 0, 1396] combine_rows_per_src=[8046, 8130, 7830, 7558, 7548, 7640, 894, 8036] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=380047360 dispatch_meta_bytes=237128 combine_payload_bytes=380047360 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=380284488 ingress_bytes=600248320 link_bytes=600248320 token_bytes=760331848 weight_movement_bytes=220200960 total_nvlink_bytes=980532808 weight_to_token_ratio=0.289612 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=19840 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.043930552e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=348809216 dispatch_meta_bytes=226528 combine_payload_bytes=348809216 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=349035744 ingress_bytes=569010176 link_bytes=569010176 token_bytes=697844960 weight_movement_bytes=220200960 total_nvlink_bytes=918045920 weight_to_token_ratio=0.315544 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=30080 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.896022822e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=347289600 dispatch_meta_bytes=228496 combine_payload_bytes=347289600 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=347518096 ingress_bytes=457390080 link_bytes=457390080 token_bytes=694807696 weight_movement_bytes=110100480 total_nvlink_bytes=804908176 weight_to_token_ratio=0.158462 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.954765769e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53986 payload_rows_remote=45890 dup_rows=11550 dup_rows_remote=11176 dedup_ratio=0.176239 router_tokens_per_expert=[0, 0, 0, 0, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 0, 0, 0, 0, 2765, 158, 0, 0, 0, 0, 4096, 3874, 0, 4096, 0, 0, 0, 0, 0, 379, 2046, 0, 0, 3936, 0, 0, 0, 281, 0, 0, 0, 2531, 3907, 0, 0, 0, 179, 0, 0, 0, 4091, 0] payload_rows_per_dst=[8096, 8192, 8192, 8062, 7884, 7700, 2274, 3586] dup_rows_per_dst=[374, 92, 260, 5598, 2820, 2212, 76, 118] combine_rows_per_src=[8096, 8192, 8192, 8062, 7884, 7700, 2274, 3586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=366084096 dispatch_meta_bytes=208256 combine_payload_bytes=366084096 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=366292352 ingress_bytes=696385536 link_bytes=696385536 token_bytes=732376448 weight_movement_bytes=330301440 total_nvlink_bytes=1062677888 weight_to_token_ratio=0.451000 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=19200 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.211128983e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=340852736 dispatch_meta_bytes=229624 combine_payload_bytes=340852736 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=341082360 ingress_bytes=561053696 link_bytes=561053696 token_bytes=681935096 weight_movement_bytes=220200960 total_nvlink_bytes=902136056 weight_to_token_ratio=0.322906 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.757646548e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59880 payload_rows_remote=55094 dup_rows=5656 dup_rows_remote=5096 dedup_ratio=0.086304 router_tokens_per_expert=[0, 3993, 0, 3358, 0, 571, 0, 258, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 3966, 0, 0, 0, 0, 0, 0, 0, 0, 2022, 0, 0, 0, 0, 0, 4035, 0, 9, 1506, 0, 0, 4096, 0, 0, 0, 116, 0, 0, 676, 0, 0, 0, 0, 8, 0, 0, 0] payload_rows_per_dst=[7294, 8190, 8118, 7932, 4786, 7380, 8192, 7988] dup_rows_per_dst=[1080, 0, 0, 0, 560, 2418, 1584, 14] combine_rows_per_src=[7294, 8190, 8118, 7932, 4786, 7380, 8192, 7988] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58462 payload_rows_remote=50696 dup_rows=7074 dup_rows_remote=7074 dedup_ratio=0.107941 router_tokens_per_expert=[0, 3963, 0, 2500, 0, 921, 0, 281, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 0, 0, 3883, 0, 0, 0, 0, 0, 0, 0, 0, 2098, 0, 0, 0, 0, 0, 4048, 0, 9, 1814, 0, 0, 4096, 0, 0, 0, 116, 0, 0, 875, 0, 0, 0, 1, 15, 0, 0, 0] payload_rows_per_dst=[6454, 8192, 8104, 7766, 8180, 3642, 8192, 7932] dup_rows_per_dst=[950, 0, 0, 0, 4112, 4, 1982, 26] combine_rows_per_src=[6454, 8192, 8104, 7766, 8180, 3642, 8192, 7932] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=328939520 dispatch_meta_bytes=228264 combine_payload_bytes=328939520 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=329167784 ingress_bytes=439040000 link_bytes=439040000 token_bytes=658107304 weight_movement_bytes=110100480 total_nvlink_bytes=768207784 weight_to_token_ratio=0.167299 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=30848 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.635627697e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=394913792 dispatch_meta_bytes=240760 combine_payload_bytes=394913792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=395154552 ingress_bytes=505014272 link_bytes=505014272 token_bytes=790068344 weight_movement_bytes=110100480 total_nvlink_bytes=900168824 weight_to_token_ratio=0.139356 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=16640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.783028796e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59146 payload_rows_remote=51106 dup_rows=6390 dup_rows_remote=2760 dedup_ratio=0.097504 router_tokens_per_expert=[0, 3955, 0, 3290, 0, 518, 0, 101, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 2, 2, 0, 0, 0, 4046, 0, 0, 0, 0, 3984, 0, 2, 0, 0, 0, 0, 0, 0, 1865, 0, 0, 0, 0, 0, 3924, 0, 16, 1895, 0, 0, 4094, 0, 0, 0, 110, 0, 0, 866, 0, 0, 0, 3, 4, 0, 0, 0] payload_rows_per_dst=[7026, 8182, 8096, 7968, 3730, 8040, 8190, 7914] dup_rows_per_dst=[792, 0, 4, 4, 0, 3630, 1950, 10] combine_rows_per_src=[7026, 8182, 8096, 7968, 3730, 8040, 8190, 7914] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58294 payload_rows_remote=50190 dup_rows=7242 dup_rows_remote=5882 dedup_ratio=0.110504 router_tokens_per_expert=[0, 4042, 0, 3486, 0, 776, 0, 478, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 0, 4037, 0, 0, 0, 0, 3896, 0, 0, 0, 0, 0, 0, 0, 0, 1666, 0, 0, 0, 0, 0, 3997, 0, 8, 1323, 0, 0, 4096, 0, 0, 0, 125, 0, 0, 737, 0, 0, 0, 2, 3, 0, 0, 0] payload_rows_per_dst=[6554, 8188, 8074, 7792, 3332, 8058, 8192, 8104] dup_rows_per_dst=[1556, 0, 4, 0, 0, 2598, 1724, 1360] combine_rows_per_src=[6554, 8188, 8074, 7792, 3332, 8058, 8192, 8104] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58756 payload_rows_remote=50606 dup_rows=6780 dup_rows_remote=6768 dedup_ratio=0.103455 router_tokens_per_expert=[0, 3997, 0, 2805, 0, 624, 0, 372, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 8, 0, 0, 0, 4072, 0, 0, 0, 0, 4039, 0, 1, 0, 0, 0, 0, 0, 0, 2147, 0, 0, 0, 0, 0, 4009, 0, 27, 1645, 0, 0, 4096, 0, 0, 0, 134, 0, 0, 660, 0, 0, 0, 11, 25, 0, 0, 0] payload_rows_per_dst=[6678, 8190, 8150, 8078, 8106, 3330, 8192, 8032] dup_rows_per_dst=[924, 0, 12, 2, 4206, 14, 1588, 34] combine_rows_per_src=[6678, 8190, 8150, 8078, 8106, 3330, 8192, 8032] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=363388928 dispatch_meta_bytes=231080 combine_payload_bytes=363388928 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=363620008 ingress_bytes=473489408 link_bytes=473489408 token_bytes=727008936 weight_movement_bytes=110100480 total_nvlink_bytes=837109416 weight_to_token_ratio=0.151443 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.234759561e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60176 payload_rows_remote=51984 dup_rows=5360 dup_rows_remote=4062 dedup_ratio=0.081787 router_tokens_per_expert=[0, 3966, 0, 3220, 0, 502, 0, 276, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 4012, 0, 0, 0, 0, 0, 0, 0, 0, 2285, 0, 0, 0, 0, 0, 4043, 0, 4, 1545, 0, 0, 4096, 0, 0, 0, 32, 0, 0, 617, 0, 0, 0, 2, 4, 0, 0, 0] payload_rows_per_dst=[6966, 8192, 8136, 8024, 4570, 8164, 8192, 7932] dup_rows_per_dst=[1030, 0, 0, 0, 0, 3020, 1298, 12] combine_rows_per_src=[6966, 8192, 8136, 8024, 4570, 8164, 8192, 7932] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57206 payload_rows_remote=49042 dup_rows=8330 dup_rows_remote=8330 dedup_ratio=0.127106 router_tokens_per_expert=[0, 3906, 0, 2814, 0, 825, 0, 455, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 27, 0, 0, 0, 4050, 0, 0, 0, 0, 3869, 0, 0, 0, 0, 0, 0, 0, 0, 2139, 0, 0, 0, 0, 0, 3818, 0, 94, 1357, 0, 0, 4096, 0, 0, 0, 110, 0, 0, 918, 0, 0, 0, 26, 182, 0, 0, 0] payload_rows_per_dst=[6898, 8164, 8106, 7738, 8042, 7064, 3322, 7872] dup_rows_per_dst=[1290, 0, 48, 0, 3872, 2334, 430, 356] combine_rows_per_src=[6898, 8164, 8106, 7738, 8042, 7064, 3322, 7872] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=366327808 dispatch_meta_bytes=215464 combine_payload_bytes=366327808 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=366543272 ingress_bytes=586528768 link_bytes=586528768 token_bytes=732871080 weight_movement_bytes=220200960 total_nvlink_bytes=953072040 weight_to_token_ratio=0.300463 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=15360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.020069995e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=359761920 dispatch_meta_bytes=224288 combine_payload_bytes=359761920 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=359986208 ingress_bytes=579962880 link_bytes=579962880 token_bytes=719748128 weight_movement_bytes=220200960 total_nvlink_bytes=939949088 weight_to_token_ratio=0.305942 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=31872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.008650836e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=362743808 dispatch_meta_bytes=229496 combine_payload_bytes=362743808 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=362973304 ingress_bytes=472844288 link_bytes=472844288 token_bytes=725717112 weight_movement_bytes=110100480 total_nvlink_bytes=835817592 weight_to_token_ratio=0.151713 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.223539864e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=372621312 dispatch_meta_bytes=224184 combine_payload_bytes=372621312 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=372845496 ingress_bytes=923123712 link_bytes=923123712 token_bytes=745466808 weight_movement_bytes=550502400 total_nvlink_bytes=1295969208 weight_to_token_ratio=0.738467 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=25472 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.605463963e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55176 payload_rows_remote=47598 dup_rows=10360 dup_rows_remote=7894 dedup_ratio=0.158081 router_tokens_per_expert=[0, 3971, 0, 3460, 0, 527, 0, 313, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 3945, 0, 0, 0, 0, 0, 0, 0, 0, 1994, 0, 0, 0, 0, 0, 4026, 0, 1, 1012, 0, 0, 4096, 0, 0, 0, 86, 0, 0, 1168, 0, 0, 0, 0, 4, 0, 0, 0] payload_rows_per_dst=[7578, 8192, 8142, 7956, 8116, 4752, 2498, 7942] dup_rows_per_dst=[2466, 46, 542, 2276, 3924, 1088, 10, 8] combine_rows_per_src=[7578, 8192, 8142, 7956, 8116, 4752, 2498, 7942] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60860 payload_rows_remote=52876 dup_rows=4676 dup_rows_remote=4318 dedup_ratio=0.071350 router_tokens_per_expert=[0, 82, 0, 0, 0, 4054, 0, 0, 3696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 4091, 0, 0, 0, 0, 1015, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 1052, 0, 0, 0, 0, 4096, 0, 65, 3833, 2318, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8108, 7392, 8050, 8192, 7984, 8192, 4750, 8192] dup_rows_per_dst=[164, 0, 0, 2020, 358, 2104, 16, 14] combine_rows_per_src=[8108, 7392, 8050, 8192, 7984, 8192, 4750, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=351533056 dispatch_meta_bytes=229488 combine_payload_bytes=351533056 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=351762544 ingress_bytes=461633536 link_bytes=461633536 token_bytes=703295600 weight_movement_bytes=110100480 total_nvlink_bytes=813396080 weight_to_token_ratio=0.156549 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.028566448e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60426 payload_rows_remote=52234 dup_rows=5110 dup_rows_remote=2168 dedup_ratio=0.077972 router_tokens_per_expert=[0, 75, 0, 0, 0, 4044, 0, 0, 3454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3862, 0, 4085, 0, 0, 0, 0, 1482, 0, 0, 0, 0, 222, 0, 0, 0, 0, 0, 921, 0, 0, 0, 0, 4096, 0, 90, 3699, 2632, 0, 0, 0, 4096, 1, 0, 9, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8088, 6908, 7724, 8192, 7720, 8192, 5410, 8192] dup_rows_per_dst=[150, 0, 0, 2942, 122, 1842, 36, 18] combine_rows_per_src=[8088, 6908, 7724, 8192, 7720, 8192, 5410, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=341182464 dispatch_meta_bytes=221968 combine_payload_bytes=341182464 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=341404432 ingress_bytes=671483904 link_bytes=671483904 token_bytes=682586896 weight_movement_bytes=330301440 total_nvlink_bytes=1012888336 weight_to_token_ratio=0.483897 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=24320 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.167820949e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=379015168 dispatch_meta_bytes=228776 combine_payload_bytes=379015168 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=379243944 ingress_bytes=599216128 link_bytes=599216128 token_bytes=758259112 weight_movement_bytes=220200960 total_nvlink_bytes=978460072 weight_to_token_ratio=0.290403 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=30464 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.042135401e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58538 payload_rows_remote=54756 dup_rows=6998 dup_rows_remote=6250 dedup_ratio=0.106781 router_tokens_per_expert=[0, 7, 0, 0, 0, 4065, 0, 0, 3455, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3928, 0, 4096, 0, 0, 0, 0, 791, 0, 0, 0, 0, 203, 0, 0, 0, 0, 0, 1414, 0, 0, 0, 0, 4094, 0, 77, 3805, 2720, 0, 0, 0, 4096, 2, 0, 10, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8130, 6920, 7856, 8192, 7788, 3782, 7678, 8192] dup_rows_per_dst=[14, 0, 0, 1582, 228, 748, 4406, 20] combine_rows_per_src=[8130, 6920, 7856, 8192, 7788, 3782, 7678, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57260 payload_rows_remote=49068 dup_rows=8276 dup_rows_remote=8272 dedup_ratio=0.126282 router_tokens_per_expert=[0, 14, 0, 0, 0, 4051, 0, 0, 3818, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3935, 0, 4093, 0, 0, 0, 0, 874, 0, 0, 0, 0, 260, 0, 0, 0, 0, 0, 940, 0, 0, 0, 0, 4096, 0, 77, 3686, 2824, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8102, 7640, 7870, 4792, 7726, 4746, 8192, 8192] dup_rows_per_dst=[28, 0, 0, 1132, 166, 1144, 5802, 4] combine_rows_per_src=[8102, 7640, 7870, 4792, 7726, 4746, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60334 payload_rows_remote=52438 dup_rows=5202 dup_rows_remote=5202 dedup_ratio=0.079376 router_tokens_per_expert=[0, 5, 0, 0, 0, 4057, 0, 1, 3311, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3948, 0, 4078, 0, 0, 0, 0, 512, 0, 0, 0, 0, 233, 0, 0, 0, 0, 0, 1961, 0, 0, 0, 0, 4095, 0, 50, 3827, 2580, 0, 0, 0, 4096, 1, 0, 9, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8114, 6630, 7896, 8192, 7922, 8190, 5198, 8192] dup_rows_per_dst=[12, 0, 0, 988, 198, 3922, 64, 18] combine_rows_per_src=[8114, 6630, 7896, 8192, 7922, 8190, 5198, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=374413312 dispatch_meta_bytes=217608 combine_payload_bytes=374413312 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=374630920 ingress_bytes=814815232 link_bytes=814815232 token_bytes=749044232 weight_movement_bytes=440401920 total_nvlink_bytes=1189446152 weight_to_token_ratio=0.587952 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=24576 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.417097703e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55716 payload_rows_remote=48040 dup_rows=9820 dup_rows_remote=7102 dedup_ratio=0.149841 router_tokens_per_expert=[0, 2155, 0, 3831, 0, 2675, 1, 2338, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 3988, 0, 0, 0, 0, 0, 2808, 0, 7, 48, 0, 0, 4096, 0, 0, 0, 68, 0, 0, 2454, 0, 0, 0, 2, 0, 0, 0, 0] payload_rows_per_dst=[2, 8192, 8176, 8192, 7976, 7458, 7676, 8044] dup_rows_per_dst=[0, 0, 0, 226, 0, 2944, 2718, 3932] combine_rows_per_src=[2, 8192, 8176, 8192, 7976, 7458, 7676, 8044] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=329011200 dispatch_meta_bytes=229344 combine_payload_bytes=329011200 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=329240544 ingress_bytes=439111680 link_bytes=439111680 token_bytes=658251744 weight_movement_bytes=110100480 total_nvlink_bytes=768352224 weight_to_token_ratio=0.167262 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.636874330e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57422 payload_rows_remote=49236 dup_rows=8114 dup_rows_remote=6002 dedup_ratio=0.123810 router_tokens_per_expert=[0, 0, 0, 0, 0, 4060, 0, 1, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 0, 4090, 0, 0, 0, 0, 1757, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 2500, 0, 0, 0, 0, 4096, 0, 4096, 4043, 1810, 0, 0, 1090, 4096, 1, 0, 14, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8120, 8192, 8186, 3514, 8090, 8192, 4936, 8192] dup_rows_per_dst=[2, 10, 2112, 0, 96, 5000, 866, 28] combine_rows_per_src=[8120, 8192, 8186, 3514, 8090, 8192, 4936, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59686 payload_rows_remote=54002 dup_rows=5850 dup_rows_remote=5850 dedup_ratio=0.089264 router_tokens_per_expert=[0, 0, 0, 0, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1339, 0, 4092, 0, 0, 0, 0, 2842, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 1124, 0, 0, 0, 0, 4096, 0, 4096, 3972, 2058, 0, 0, 939, 4096, 1, 0, 12, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8092, 8192, 8192, 5684, 7956, 8192, 5186, 8192] dup_rows_per_dst=[0, 0, 2670, 0, 98, 2248, 810, 24] combine_rows_per_src=[8092, 8192, 8192, 5684, 7956, 8192, 5186, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58284 payload_rows_remote=50264 dup_rows=7252 dup_rows_remote=7110 dedup_ratio=0.110657 router_tokens_per_expert=[0, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1418, 0, 4095, 0, 0, 0, 0, 2358, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 1661, 0, 0, 0, 0, 4096, 0, 4096, 4000, 1629, 0, 0, 1173, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8118, 8192, 8192, 4716, 8020, 8192, 4662, 8192] dup_rows_per_dst=[0, 0, 2834, 0, 142, 3322, 942, 12] combine_rows_per_src=[8118, 8192, 8192, 4716, 8020, 8192, 4662, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59202 payload_rows_remote=51026 dup_rows=6334 dup_rows_remote=304 dedup_ratio=0.096649 router_tokens_per_expert=[0, 2237, 0, 3582, 0, 2340, 6, 2511, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 102, 0, 0, 0, 0, 0, 0, 0, 0, 3880, 0, 0, 0, 0, 0, 2817, 0, 1, 22, 0, 0, 4096, 0, 0, 0, 107, 0, 0, 2876, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8176, 8174, 8190, 8192, 7760, 5638, 5908, 7164] dup_rows_per_dst=[6030, 0, 0, 204, 0, 42, 58, 0] combine_rows_per_src=[8176, 8174, 8190, 8192, 7760, 5638, 5908, 7164] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59464 payload_rows_remote=51274 dup_rows=6072 dup_rows_remote=6072 dedup_ratio=0.092651 router_tokens_per_expert=[0, 83, 0, 0, 283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3512, 247, 0, 0, 0, 0, 4008, 4095, 0, 4065, 0, 0, 0, 0, 9, 1027, 4006, 0, 0, 3876, 0, 0, 0, 281, 0, 0, 0, 2574, 3565, 0, 0, 0, 666, 0, 0, 0, 471, 0] payload_rows_per_dst=[8192, 8190, 8012, 7294, 8016, 8098, 6804, 4858] dup_rows_per_dst=[670, 0, 0, 224, 0, 1726, 2832, 620] combine_rows_per_src=[8192, 8190, 8012, 7294, 8016, 8098, 6804, 4858] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=344350720 dispatch_meta_bytes=220568 combine_payload_bytes=344350720 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=344571288 ingress_bytes=564551680 link_bytes=564551680 token_bytes=688922008 weight_movement_bytes=220200960 total_nvlink_bytes=909122968 weight_to_token_ratio=0.319631 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=20864 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.818482243e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56588 payload_rows_remote=50132 dup_rows=8948 dup_rows_remote=5748 dedup_ratio=0.136536 router_tokens_per_expert=[0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1258, 0, 4096, 0, 0, 0, 0, 1914, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 2185, 0, 0, 0, 0, 4094, 0, 4096, 3988, 2070, 0, 0, 821, 4096, 1, 0, 11, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8136, 8192, 5172, 6236, 8014, 6456, 6190, 8192] dup_rows_per_dst=[0, 10, 712, 2416, 92, 3200, 2496, 22] combine_rows_per_src=[8136, 8192, 5172, 6236, 8014, 6456, 6190, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=352923648 dispatch_meta_bytes=220952 combine_payload_bytes=352923648 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=353144600 ingress_bytes=463024128 link_bytes=463024128 token_bytes=706068248 weight_movement_bytes=110100480 total_nvlink_bytes=816168728 weight_to_token_ratio=0.155935 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=22272 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.052751130e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=387086336 dispatch_meta_bytes=239408 combine_payload_bytes=387086336 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=387325744 ingress_bytes=607287296 link_bytes=607287296 token_bytes=774412080 weight_movement_bytes=220200960 total_nvlink_bytes=994613040 weight_to_token_ratio=0.284346 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=18304 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.056172489e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=360292352 dispatch_meta_bytes=229496 combine_payload_bytes=360292352 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=360521848 ingress_bytes=580493312 link_bytes=580493312 token_bytes=720814200 weight_movement_bytes=220200960 total_nvlink_bytes=941015160 weight_to_token_ratio=0.305489 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.009573344e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=365754368 dispatch_meta_bytes=205320 combine_payload_bytes=365754368 prefetch_bytes=110100480 reduce_bytes=550502400 egress_bytes=365959688 ingress_bytes=1026357248 link_bytes=1026357248 token_bytes=731714056 weight_movement_bytes=660602880 total_nvlink_bytes=1392316936 weight_to_token_ratio=0.902816 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=12416 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.785004061e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=367532032 dispatch_meta_bytes=229384 combine_payload_bytes=367532032 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=367761416 ingress_bytes=587732992 link_bytes=587732992 token_bytes=735293448 weight_movement_bytes=220200960 total_nvlink_bytes=955494408 weight_to_token_ratio=0.299474 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.022164339e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63146 payload_rows_remote=54954 dup_rows=2390 dup_rows_remote=2390 dedup_ratio=0.036469 router_tokens_per_expert=[0, 0, 0, 0, 0, 2, 0, 4046, 0, 0, 5, 0, 4094, 3307, 4052, 0, 0, 0, 47, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4089, 0, 0, 0, 0, 0, 0, 610, 0, 0, 0, 0, 0, 0, 182, 0, 0, 0, 36, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8096, 6614, 7582, 8178, 8104, 8188, 8192, 8192] dup_rows_per_dst=[0, 10, 88, 616, 1220, 364, 92, 0] combine_rows_per_src=[8096, 6614, 7582, 8178, 8104, 8188, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=359346176 dispatch_meta_bytes=223520 combine_payload_bytes=359346176 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=359569696 ingress_bytes=689647616 link_bytes=689647616 token_bytes=718915872 weight_movement_bytes=330301440 total_nvlink_bytes=1049217312 weight_to_token_ratio=0.459444 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=18944 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.199410632e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=393910272 dispatch_meta_bytes=229376 combine_payload_bytes=393910272 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=394139648 ingress_bytes=504010752 link_bytes=504010752 token_bytes=788049920 weight_movement_bytes=110100480 total_nvlink_bytes=898150400 weight_to_token_ratio=0.139713 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.765575933e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56376 payload_rows_remote=48184 dup_rows=9160 dup_rows_remote=3438 dedup_ratio=0.139771 router_tokens_per_expert=[0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 952, 0, 4095, 0, 0, 0, 0, 1592, 0, 0, 0, 0, 125, 0, 0, 0, 0, 0, 2703, 0, 0, 0, 0, 4096, 0, 4096, 4086, 1851, 0, 0, 1010, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8124, 8192, 1904, 8192, 8174, 5406, 8192, 8192] dup_rows_per_dst=[0, 2, 0, 3182, 248, 0, 5722, 6] combine_rows_per_src=[8124, 8192, 1904, 8192, 8174, 5406, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64170 payload_rows_remote=55978 dup_rows=1366 dup_rows_remote=1310 dedup_ratio=0.020844 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 13, 0, 4095, 3519, 4060, 0, 0, 0, 28, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 4096, 4071, 0, 1, 0, 0, 0, 0, 567, 0, 0, 0, 0, 0, 0, 14, 3, 0, 0, 38, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8100, 7038, 8192, 8142, 8122, 8192, 8192, 8192] dup_rows_per_dst=[0, 26, 56, 0, 1134, 32, 118, 0] combine_rows_per_src=[8100, 7038, 8192, 8142, 8122, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 2, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63722 payload_rows_remote=55552 dup_rows=1814 dup_rows_remote=1814 dedup_ratio=0.027679 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3965, 0, 0, 8, 0, 4096, 3394, 4033, 0, 0, 0, 64, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4085, 0, 1, 0, 0, 0, 0, 611, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 109, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7930, 6788, 8192, 8170, 8066, 8192, 8192, 8192] dup_rows_per_dst=[0, 16, 128, 0, 1224, 178, 268, 0] combine_rows_per_src=[7930, 6788, 8192, 8170, 8066, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54844 payload_rows_remote=46716 dup_rows=10692 dup_rows_remote=10096 dedup_ratio=0.163147 router_tokens_per_expert=[0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2126, 0, 4096, 0, 0, 0, 0, 2459, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 1121, 0, 0, 0, 0, 4096, 0, 4096, 3954, 1537, 0, 0, 1054, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8128, 8192, 8192, 8034, 7950, 2242, 4120, 7986] dup_rows_per_dst=[596, 0, 4252, 4480, 492, 0, 872, 0] combine_rows_per_src=[8128, 8192, 8192, 8034, 7950, 2242, 4120, 7986] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62820 payload_rows_remote=54702 dup_rows=2716 dup_rows_remote=250 dedup_ratio=0.041443 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 2, 0, 4096, 2809, 4059, 0, 0, 0, 36, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 1233, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 23, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8126, 5618, 8192, 8190, 8118, 8192, 8192, 8192] dup_rows_per_dst=[0, 4, 72, 0, 2466, 122, 52, 0] combine_rows_per_src=[8126, 5618, 8192, 8190, 8118, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57984 payload_rows_remote=49798 dup_rows=7552 dup_rows_remote=7546 dedup_ratio=0.115234 router_tokens_per_expert=[0, 2092, 0, 3146, 0, 2844, 8, 2608, 0, 3, 0, 0, 0, 0, 0, 4093, 0, 0, 34, 0, 0, 0, 4086, 0, 0, 0, 0, 206, 0, 0, 0, 0, 0, 0, 0, 0, 3945, 1, 0, 0, 0, 0, 2648, 0, 124, 55, 0, 0, 4096, 0, 0, 0, 120, 0, 0, 2569, 0, 0, 0, 43, 47, 0, 0, 0] payload_rows_per_dst=[8180, 8186, 8178, 8192, 7890, 5570, 5372, 6416] dup_rows_per_dst=[6924, 6, 62, 412, 2, 84, 6, 56] combine_rows_per_src=[8180, 8186, 8178, 8192, 7890, 5570, 5372, 6416] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=345382912 dispatch_meta_bytes=206488 combine_payload_bytes=345382912 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=345589400 ingress_bytes=895885312 link_bytes=895885312 token_bytes=690972312 weight_movement_bytes=550502400 total_nvlink_bytes=1241474712 weight_to_token_ratio=0.796707 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=14720 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.558091905e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59044 payload_rows_remote=50852 dup_rows=6492 dup_rows_remote=1236 dedup_ratio=0.099060 router_tokens_per_expert=[0, 4, 0, 0, 0, 4054, 0, 0, 3527, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3907, 0, 4095, 0, 0, 0, 0, 421, 0, 0, 0, 0, 251, 0, 0, 0, 0, 0, 1730, 0, 0, 0, 0, 4096, 0, 20, 3955, 2608, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8108, 7056, 7814, 8192, 8030, 3460, 8192, 8192] dup_rows_per_dst=[8, 0, 0, 840, 382, 0, 5256, 6] combine_rows_per_src=[8108, 7056, 7814, 8192, 8030, 3460, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=392491008 dispatch_meta_bytes=244024 combine_payload_bytes=392491008 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=392735032 ingress_bytes=612691968 link_bytes=612691968 token_bytes=785226040 weight_movement_bytes=220200960 total_nvlink_bytes=1005427000 weight_to_token_ratio=0.280430 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=21376 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.065572103e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=351719424 dispatch_meta_bytes=229360 combine_payload_bytes=351719424 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=351948784 ingress_bytes=461819904 link_bytes=461819904 token_bytes=703668208 weight_movement_bytes=110100480 total_nvlink_bytes=813768688 weight_to_token_ratio=0.156466 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.031807694e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59068 payload_rows_remote=52072 dup_rows=6468 dup_rows_remote=5584 dedup_ratio=0.098694 router_tokens_per_expert=[0, 51, 0, 0, 0, 4052, 0, 0, 2995, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3888, 0, 4091, 0, 0, 0, 0, 2213, 0, 0, 0, 0, 259, 0, 0, 0, 0, 0, 606, 0, 0, 0, 0, 4096, 0, 214, 3689, 2447, 0, 0, 0, 4096, 4, 0, 49, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8104, 6996, 7904, 6972, 7604, 8192, 5104, 8192] dup_rows_per_dst=[102, 884, 2326, 1328, 292, 1212, 226, 98] combine_rows_per_src=[8104, 6996, 7904, 6972, 7604, 8192, 5104, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=375875584 dispatch_meta_bytes=230560 combine_payload_bytes=375875584 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=376106144 ingress_bytes=485976064 link_bytes=485976064 token_bytes=751981728 weight_movement_bytes=110100480 total_nvlink_bytes=862082208 weight_to_token_ratio=0.146414 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=31616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.451923046e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55866 payload_rows_remote=47748 dup_rows=9670 dup_rows_remote=9484 dedup_ratio=0.147552 router_tokens_per_expert=[0, 82, 0, 0, 0, 4059, 0, 0, 3470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 4096, 0, 0, 0, 0, 990, 0, 0, 0, 0, 272, 0, 0, 0, 0, 0, 809, 0, 0, 0, 0, 4096, 0, 30, 3854, 2866, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8118, 8192, 8096, 1980, 8092, 8192, 5182, 8014] dup_rows_per_dst=[186, 6918, 0, 0, 924, 1618, 24, 0] combine_rows_per_src=[8118, 8192, 8096, 1980, 8092, 8192, 5182, 8014] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=364507136 dispatch_meta_bytes=208352 combine_payload_bytes=364507136 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=364715488 ingress_bytes=804909056 link_bytes=804909056 token_bytes=729222624 weight_movement_bytes=440401920 total_nvlink_bytes=1169624544 weight_to_token_ratio=0.603933 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=20736 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.399869233e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64670 payload_rows_remote=56478 dup_rows=866 dup_rows_remote=758 dedup_ratio=0.013214 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3980, 0, 0, 2, 0, 4096, 367, 3981, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3811, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7960, 8128, 8192, 7622, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 572, 0, 0, 108, 0, 186, 0] combine_rows_per_src=[7960, 8128, 8192, 7622, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=373252096 dispatch_meta_bytes=230624 combine_payload_bytes=373252096 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=373482720 ingress_bytes=483352576 link_bytes=483352576 token_bytes=746734816 weight_movement_bytes=110100480 total_nvlink_bytes=856835296 weight_to_token_ratio=0.147443 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=27776 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.406296275e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63552 payload_rows_remote=56604 dup_rows=1984 dup_rows_remote=1984 dedup_ratio=0.030273 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3773, 0, 0, 11, 0, 4095, 810, 3849, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3474, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 321, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7546, 8100, 8192, 6948, 8190, 8192, 8192, 8192] dup_rows_per_dst=[0, 1240, 10, 0, 88, 0, 646, 0] combine_rows_per_src=[7546, 8100, 8192, 6948, 8190, 8192, 8192, 8192] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64248 payload_rows_remote=56056 dup_rows=1288 dup_rows_remote=1196 dedup_ratio=0.019653 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4043, 0, 0, 9, 0, 4095, 3543, 4061, 0, 0, 0, 28, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 4096, 4087, 0, 8, 0, 0, 0, 0, 504, 0, 0, 0, 0, 0, 0, 45, 2, 0, 0, 39, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8086, 7086, 8192, 8174, 8134, 8192, 8192, 8192] dup_rows_per_dst=[0, 18, 56, 0, 1012, 92, 110, 0] combine_rows_per_src=[8086, 7086, 8192, 8174, 8134, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=401250304 dispatch_meta_bytes=229152 combine_payload_bytes=401250304 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=401479456 ingress_bytes=511350784 link_bytes=511350784 token_bytes=802729760 weight_movement_bytes=110100480 total_nvlink_bytes=912830240 weight_to_token_ratio=0.137158 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.893231161e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51046 payload_rows_remote=42912 dup_rows=14490 dup_rows_remote=6842 dedup_ratio=0.221100 router_tokens_per_expert=[0, 0, 3118, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 3988, 9, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 0, 0, 0, 0, 0, 620, 1, 89, 3859, 0, 0, 3761, 0, 1, 0, 1396, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3093, 0, 0, 0, 0, 13, 4092, 0] payload_rows_per_dst=[6272, 7976, 8180, 2206, 2002, 8084, 8192, 8134] dup_rows_per_dst=[6, 18, 4, 226, 56, 6532, 0, 7648] combine_rows_per_src=[6272, 7976, 8180, 2206, 2002, 8084, 8192, 8134] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=398196736 dispatch_meta_bytes=229464 combine_payload_bytes=398196736 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=398426200 ingress_bytes=948699136 link_bytes=948699136 token_bytes=796622936 weight_movement_bytes=550502400 total_nvlink_bytes=1347125336 weight_to_token_ratio=0.691045 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=31872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.649943832e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=392103936 dispatch_meta_bytes=219808 combine_payload_bytes=392103936 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=392323744 ingress_bytes=502204416 link_bytes=502204416 token_bytes=784427680 weight_movement_bytes=110100480 total_nvlink_bytes=894528160 weight_to_token_ratio=0.140358 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=27264 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.734160779e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=334860288 dispatch_meta_bytes=227248 combine_payload_bytes=334860288 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=335087536 ingress_bytes=444960768 link_bytes=444960768 token_bytes=669947824 weight_movement_bytes=110100480 total_nvlink_bytes=780048304 weight_to_token_ratio=0.164342 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.738599590e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=356952064 dispatch_meta_bytes=229376 combine_payload_bytes=356952064 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=357181440 ingress_bytes=577153024 link_bytes=577153024 token_bytes=714133504 weight_movement_bytes=220200960 total_nvlink_bytes=934334464 weight_to_token_ratio=0.308347 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.003764034e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=401809408 dispatch_meta_bytes=229008 combine_payload_bytes=401809408 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=402038416 ingress_bytes=511909888 link_bytes=511909888 token_bytes=803847824 weight_movement_bytes=110100480 total_nvlink_bytes=913948304 weight_to_token_ratio=0.136967 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.902954899e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=307593216 dispatch_meta_bytes=199016 combine_payload_bytes=307593216 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=307792232 ingress_bytes=527794176 link_bytes=527794176 token_bytes=615385448 weight_movement_bytes=220200960 total_nvlink_bytes=835586408 weight_to_token_ratio=0.357826 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=20736 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.179208793e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63854 payload_rows_remote=55664 dup_rows=1682 dup_rows_remote=1662 dedup_ratio=0.025665 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 4065, 0, 0, 3, 0, 4096, 3315, 4065, 0, 0, 0, 25, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 4096, 4093, 0, 6, 0, 0, 0, 0, 763, 0, 0, 0, 0, 0, 0, 38, 1, 0, 0, 8, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8132, 6630, 8192, 8186, 8140, 8192, 8190, 8192] dup_rows_per_dst=[0, 6, 50, 0, 1528, 78, 20, 0] combine_rows_per_src=[8132, 6630, 8192, 8186, 8140, 8192, 8190, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58022 payload_rows_remote=49830 dup_rows=7514 dup_rows_remote=7510 dedup_ratio=0.114655 router_tokens_per_expert=[0, 0, 2942, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 3915, 15, 0, 4, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 0, 0, 0, 0, 0, 682, 4, 16, 3889, 0, 0, 3715, 0, 8, 0, 2682, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2106, 0, 0, 0, 0, 16, 4086, 0] payload_rows_per_dst=[7996, 7832, 8192, 8176, 7864, 5540, 8192, 4230] dup_rows_per_dst=[4940, 28, 4, 942, 1318, 268, 0, 14] combine_rows_per_src=[7996, 7832, 8192, 8176, 7864, 5540, 8192, 4230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=342257664 dispatch_meta_bytes=228928 combine_payload_bytes=342257664 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=342486592 ingress_bytes=452358144 link_bytes=452358144 token_bytes=684744256 weight_movement_bytes=110100480 total_nvlink_bytes=794844736 weight_to_token_ratio=0.160791 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.867252125e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=404834304 dispatch_meta_bytes=228944 combine_payload_bytes=404834304 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=405063248 ingress_bytes=514934784 link_bytes=514934784 token_bytes=809897552 weight_movement_bytes=110100480 total_nvlink_bytes=919998032 weight_to_token_ratio=0.135944 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.955562816e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64212 payload_rows_remote=56020 dup_rows=1324 dup_rows_remote=1322 dedup_ratio=0.020203 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3869, 0, 0, 10, 0, 4095, 633, 3894, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 4096, 3710, 0, 2, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 123, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7738, 8096, 8192, 7420, 8190, 8192, 8192, 8192] dup_rows_per_dst=[0, 978, 2, 0, 102, 2, 240, 0] combine_rows_per_src=[7738, 8096, 8192, 7420, 8190, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61562 payload_rows_remote=53370 dup_rows=3974 dup_rows_remote=3974 dedup_ratio=0.060638 router_tokens_per_expert=[0, 0, 0, 0, 0, 2, 0, 3942, 0, 0, 6, 0, 4087, 424, 3906, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3773, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 198, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7888, 5022, 8192, 7710, 8174, 8192, 8192, 8192] dup_rows_per_dst=[0, 396, 10, 3090, 80, 0, 398, 0] combine_rows_per_src=[7888, 5022, 8192, 7710, 8174, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64450 payload_rows_remote=56258 dup_rows=1086 dup_rows_remote=1084 dedup_ratio=0.016571 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3931, 0, 0, 13, 0, 4095, 484, 3988, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 4096, 3743, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 74, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7862, 8144, 8192, 7486, 8190, 8192, 8192, 8192] dup_rows_per_dst=[0, 826, 2, 0, 108, 4, 146, 0] combine_rows_per_src=[7862, 8144, 8192, 7486, 8190, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=405737472 dispatch_meta_bytes=234352 combine_payload_bytes=405737472 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=405971824 ingress_bytes=846139392 link_bytes=846139392 token_bytes=811709296 weight_movement_bytes=440401920 total_nvlink_bytes=1252111216 weight_to_token_ratio=0.542561 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=29696 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.471575569e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64398 payload_rows_remote=56206 dup_rows=1138 dup_rows_remote=1024 dedup_ratio=0.017365 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 3965, 0, 0, 3, 0, 4095, 643, 3856, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 4096, 3701, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[7932, 8106, 8192, 7402, 8190, 8192, 8192, 8192] dup_rows_per_dst=[0, 898, 2, 0, 122, 0, 114, 2] combine_rows_per_src=[7932, 8106, 8192, 7402, 8190, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=401551360 dispatch_meta_bytes=229368 combine_payload_bytes=401551360 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=401780728 ingress_bytes=511651840 link_bytes=511651840 token_bytes=803332088 weight_movement_bytes=110100480 total_nvlink_bytes=913432568 weight_to_token_ratio=0.137055 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.898467020e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=382556160 dispatch_meta_bytes=229376 combine_payload_bytes=382556160 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=382785536 ingress_bytes=492656640 link_bytes=492656640 token_bytes=765341696 weight_movement_bytes=110100480 total_nvlink_bytes=875442176 weight_to_token_ratio=0.143858 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.568109251e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63656 payload_rows_remote=55824 dup_rows=1880 dup_rows_remote=1222 dedup_ratio=0.028687 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3787, 0, 0, 67, 0, 4093, 735, 3443, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4086, 4096, 3645, 0, 6, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7574, 7832, 8192, 7290, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 658, 12, 0, 108, 0, 1102, 0] combine_rows_per_src=[7574, 7832, 8192, 7290, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=403257344 dispatch_meta_bytes=229368 combine_payload_bytes=403257344 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=403486712 ingress_bytes=513357824 link_bytes=513357824 token_bytes=806744056 weight_movement_bytes=110100480 total_nvlink_bytes=916844536 weight_to_token_ratio=0.136475 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.928136888e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58334 payload_rows_remote=50160 dup_rows=7202 dup_rows_remote=6514 dedup_ratio=0.109894 router_tokens_per_expert=[0, 0, 2608, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 3908, 18, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 349, 0, 0, 0, 0, 0, 1250, 6, 27, 3623, 0, 0, 3515, 0, 1, 0, 2514, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2604, 0, 0, 0, 0, 33, 4082, 0] payload_rows_per_dst=[5252, 7816, 8192, 8174, 7602, 7896, 8192, 5210] dup_rows_per_dst=[40, 36, 0, 688, 2210, 4164, 0, 64] combine_rows_per_src=[5252, 7816, 8192, 8174, 7602, 7896, 8192, 5210] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59478 payload_rows_remote=51536 dup_rows=6058 dup_rows_remote=4644 dedup_ratio=0.092438 router_tokens_per_expert=[0, 0, 2365, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 3954, 6, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 435, 0, 0, 0, 0, 0, 722, 0, 38, 3918, 0, 0, 3746, 0, 0, 0, 2694, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2575, 0, 0, 0, 0, 20, 4087, 0] payload_rows_per_dst=[4758, 7908, 8192, 8178, 7942, 7668, 8192, 6640] dup_rows_per_dst=[4, 12, 0, 866, 1414, 2220, 0, 1542] combine_rows_per_src=[4758, 7908, 8192, 8178, 7942, 7668, 8192, 6640] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55408 payload_rows_remote=47228 dup_rows=10128 dup_rows_remote=9000 dedup_ratio=0.154541 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 4096, 3539, 4061, 0, 0, 0, 35, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 481, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 9, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8180, 8192, 8192, 7812, 962, 6950, 6928, 8192] dup_rows_per_dst=[1128, 8762, 70, 0, 0, 142, 26, 0] combine_rows_per_src=[8180, 8192, 8192, 7812, 962, 6950, 6928, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59540 payload_rows_remote=51348 dup_rows=5996 dup_rows_remote=5962 dedup_ratio=0.091492 router_tokens_per_expert=[0, 0, 0, 0, 0, 4054, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1261, 0, 4089, 0, 0, 0, 0, 3214, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 1163, 0, 0, 0, 0, 4096, 0, 4096, 3912, 1856, 0, 0, 773, 4096, 0, 0, 53, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8108, 8192, 8178, 6430, 7846, 7950, 4644, 8192] dup_rows_per_dst=[0, 34, 2522, 240, 154, 2326, 614, 106] combine_rows_per_src=[8108, 8192, 8178, 6430, 7846, 7950, 4644, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=398999552 dispatch_meta_bytes=229304 combine_payload_bytes=398999552 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=399228856 ingress_bytes=509100032 link_bytes=509100032 token_bytes=798228408 weight_movement_bytes=110100480 total_nvlink_bytes=908328888 weight_to_token_ratio=0.137931 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=32128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.854086882e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60290 payload_rows_remote=52850 dup_rows=5246 dup_rows_remote=5240 dedup_ratio=0.080048 router_tokens_per_expert=[0, 0, 2436, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 3955, 15, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 304, 0, 0, 2, 0, 0, 755, 7, 4, 3921, 0, 0, 3720, 0, 3, 0, 3115, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2231, 0, 0, 0, 0, 8, 4070, 0] payload_rows_per_dst=[4908, 7912, 8190, 8148, 7912, 7440, 8192, 7588] dup_rows_per_dst=[24, 28, 2, 600, 1466, 6, 0, 3120] combine_rows_per_src=[4908, 7912, 8190, 8148, 7912, 7440, 8192, 7588] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=357181440 dispatch_meta_bytes=229360 combine_payload_bytes=357181440 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=357410800 ingress_bytes=467281920 link_bytes=467281920 token_bytes=714592240 weight_movement_bytes=110100480 total_nvlink_bytes=824692720 weight_to_token_ratio=0.154075 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.126801135e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61596 payload_rows_remote=53404 dup_rows=3940 dup_rows_remote=3926 dedup_ratio=0.060120 router_tokens_per_expert=[18, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 3, 0, 2354, 0, 0, 0, 0, 0, 0, 2, 4037, 0, 0, 3659, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 1, 0, 0, 4094, 0, 0, 67, 0, 0, 4, 0, 2, 0, 0, 2151, 0, 0, 7, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7860, 4714, 8078, 8188, 8188, 8190, 8186, 8192] dup_rows_per_dst=[14, 0, 0, 0, 0, 466, 3446, 14] combine_rows_per_src=[7860, 4714, 8078, 8188, 8188, 8190, 8186, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=359546880 dispatch_meta_bytes=226696 combine_payload_bytes=359546880 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=359773576 ingress_bytes=469647360 link_bytes=469647360 token_bytes=719320456 weight_movement_bytes=110100480 total_nvlink_bytes=829420936 weight_to_token_ratio=0.153062 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=29312 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.167940028e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=368062464 dispatch_meta_bytes=229240 combine_payload_bytes=368062464 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=368291704 ingress_bytes=478162944 link_bytes=478162944 token_bytes=736354168 weight_movement_bytes=110100480 total_nvlink_bytes=846454648 weight_to_token_ratio=0.149521 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.316040039e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=369410048 dispatch_meta_bytes=224720 combine_payload_bytes=369410048 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=369634768 ingress_bytes=699711488 link_bytes=699711488 token_bytes=739044816 weight_movement_bytes=330301440 total_nvlink_bytes=1069346256 weight_to_token_ratio=0.446930 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=25984 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.216913360e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=338530304 dispatch_meta_bytes=224912 combine_payload_bytes=338530304 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=338755216 ingress_bytes=448630784 link_bytes=448630784 token_bytes=677285520 weight_movement_bytes=110100480 total_nvlink_bytes=787386000 weight_to_token_ratio=0.162561 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.802427205e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=378828800 dispatch_meta_bytes=232360 combine_payload_bytes=378828800 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=379061160 ingress_bytes=709130240 link_bytes=709130240 token_bytes=757889960 weight_movement_bytes=330301440 total_nvlink_bytes=1088191400 weight_to_token_ratio=0.435817 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.055147 gemm_active_groups=5 gemm_peak_group_rows=18304 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.233294119e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=382799872 dispatch_meta_bytes=229320 combine_payload_bytes=382799872 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=383029192 ingress_bytes=492900352 link_bytes=492900352 token_bytes=765829064 weight_movement_bytes=110100480 total_nvlink_bytes=875929544 weight_to_token_ratio=0.143766 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.572347803e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60096 payload_rows_remote=51904 dup_rows=5440 dup_rows_remote=5440 dedup_ratio=0.083008 router_tokens_per_expert=[0, 0, 2780, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 3933, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 590, 0, 0, 1, 0, 0, 876, 3, 17, 3999, 0, 0, 3345, 0, 0, 0, 2591, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2329, 0, 0, 0, 0, 14, 4086, 0] payload_rows_per_dst=[5576, 7866, 8192, 8182, 8078, 6690, 8192, 7320] dup_rows_per_dst=[0, 8, 0, 1170, 1714, 0, 0, 2548] combine_rows_per_src=[5576, 7866, 8192, 8182, 8078, 6690, 8192, 7320] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61120 payload_rows_remote=52958 dup_rows=4416 dup_rows_remote=4416 dedup_ratio=0.067383 router_tokens_per_expert=[37, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 9, 0, 2133, 0, 0, 0, 0, 0, 0, 4, 4077, 0, 0, 3647, 4092, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 4089, 0, 0, 0, 4092, 0, 0, 34, 0, 0, 5, 0, 3, 0, 0, 2359, 0, 0, 27, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8162, 7294, 8178, 8184, 4728, 8192] dup_rows_per_dst=[4, 4276, 0, 8, 0, 68, 6, 54] combine_rows_per_src=[8190, 8192, 8162, 7294, 8178, 8184, 4728, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6525449736 prefetch_bytes=1211105280 reduce_bytes=440401920 weight_movement_bytes=1651507200 egress_bytes=3263823368 ingress_bytes=4913133568 link_bytes=4913133568 total_nvlink_bytes=8176956936 weight_to_token_ratio=0.253087 prefetch_share_of_ingress=0.246504 dedup_ratio=0.188199 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=4 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.049265 gemm_inner_pad_upper_bound=7239 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.544747346e-05 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59154 payload_rows_remote=51040 dup_rows=6382 dup_rows_remote=3460 dedup_ratio=0.097382 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3958, 0, 0, 0, 0, 4096, 423, 3989, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3789, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8114, 5962, 8154, 7862, 7398, 8186, 5288, 8190] dup_rows_per_dst=[2922, 578, 0, 2706, 104, 0, 72, 0] combine_rows_per_src=[8114, 5962, 8154, 7862, 7398, 8186, 5288, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=402884608 dispatch_meta_bytes=228920 combine_payload_bytes=402884608 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=403113528 ingress_bytes=512985088 link_bytes=512985088 token_bytes=805998136 weight_movement_bytes=110100480 total_nvlink_bytes=916098616 weight_to_token_ratio=0.136601 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.921654396e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59016 payload_rows_remote=50826 dup_rows=6520 dup_rows_remote=5838 dedup_ratio=0.099487 router_tokens_per_expert=[0, 0, 3681, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 4085, 8, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2067, 0, 0, 0, 0, 0, 336, 0, 10, 0, 0, 0, 4064, 0, 0, 0, 2422, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2846, 0, 0, 0, 0, 937, 4090, 0] payload_rows_per_dst=[7392, 8170, 8192, 4134, 8190, 8182, 8192, 6564] dup_rows_per_dst=[30, 16, 0, 0, 682, 4790, 0, 1002] combine_rows_per_src=[7392, 8170, 8192, 4134, 8190, 8182, 8192, 6564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=400146432 dispatch_meta_bytes=228184 combine_payload_bytes=400146432 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=400374616 ingress_bytes=840548352 link_bytes=840548352 token_bytes=800521048 weight_movement_bytes=440401920 total_nvlink_bytes=1240922968 weight_to_token_ratio=0.550144 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=27776 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.461851831e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59492 payload_rows_remote=51792 dup_rows=6044 dup_rows_remote=2178 dedup_ratio=0.092224 router_tokens_per_expert=[0, 0, 3685, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 4065, 16, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2320, 0, 0, 0, 0, 0, 342, 8, 10, 0, 0, 0, 4057, 0, 1, 0, 2254, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2906, 0, 0, 0, 0, 758, 4095, 0] payload_rows_per_dst=[7410, 8130, 8192, 7700, 8192, 5144, 8192, 6532] dup_rows_per_dst=[78, 32, 0, 3866, 718, 554, 0, 796] combine_rows_per_src=[7410, 8130, 8192, 7700, 8192, 5144, 8192, 6532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=365854720 dispatch_meta_bytes=218000 combine_payload_bytes=365854720 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=366072720 ingress_bytes=475955200 link_bytes=475955200 token_bytes=731927440 weight_movement_bytes=110100480 total_nvlink_bytes=842027920 weight_to_token_ratio=0.150425 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31232 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.277643740e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=364320768 dispatch_meta_bytes=226656 combine_payload_bytes=364320768 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=364547424 ingress_bytes=474421248 link_bytes=474421248 token_bytes=728868192 weight_movement_bytes=110100480 total_nvlink_bytes=838968672 weight_to_token_ratio=0.151057 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=30336 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.250965792e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58660 payload_rows_remote=50488 dup_rows=6876 dup_rows_remote=1378 dedup_ratio=0.104919 router_tokens_per_expert=[0, 0, 3689, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 4088, 15, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2259, 0, 0, 1, 0, 0, 336, 7, 0, 0, 0, 0, 4005, 0, 3, 0, 2827, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2496, 0, 0, 0, 0, 704, 4090, 0] payload_rows_per_dst=[7432, 8178, 8192, 4518, 8192, 8172, 8192, 5784] dup_rows_per_dst=[56, 28, 2, 0, 676, 5498, 0, 616] combine_rows_per_src=[7432, 8178, 8192, 4518, 8192, 8172, 8192, 5784] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58198 payload_rows_remote=50096 dup_rows=7338 dup_rows_remote=410 dedup_ratio=0.111969 router_tokens_per_expert=[0, 0, 3970, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 4090, 7, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2765, 0, 0, 0, 0, 0, 375, 13, 43, 0, 0, 0, 4068, 0, 0, 0, 1235, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3196, 0, 0, 0, 0, 672, 4094, 0] payload_rows_per_dst=[7952, 8162, 8192, 5530, 3932, 8136, 8192, 8102] dup_rows_per_dst=[80, 14, 4, 0, 294, 18, 0, 6928] combine_rows_per_src=[7952, 8162, 8192, 5530, 3932, 8136, 8192, 8102] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59186 payload_rows_remote=50994 dup_rows=6350 dup_rows_remote=6348 dedup_ratio=0.096893 router_tokens_per_expert=[0, 0, 3873, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 4083, 13, 0, 3, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2624, 0, 0, 0, 0, 0, 143, 4, 0, 0, 0, 0, 4038, 0, 6, 0, 2180, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2359, 0, 0, 0, 0, 1116, 4091, 0] payload_rows_per_dst=[7778, 8168, 8192, 8176, 8190, 4372, 8192, 6118] dup_rows_per_dst=[58, 24, 2, 5148, 286, 0, 0, 832] combine_rows_per_src=[7778, 8168, 8192, 8176, 8190, 4372, 8192, 6118] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=371245056 dispatch_meta_bytes=215880 combine_payload_bytes=371245056 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=371460936 ingress_bytes=481345536 link_bytes=481345536 token_bytes=742705992 weight_movement_bytes=110100480 total_nvlink_bytes=852806472 weight_to_token_ratio=0.148242 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=19328 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.371390548e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59706 payload_rows_remote=51514 dup_rows=5830 dup_rows_remote=5830 dedup_ratio=0.088959 router_tokens_per_expert=[0, 0, 3888, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 4090, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2777, 0, 0, 1, 0, 0, 160, 3, 10, 0, 0, 0, 4054, 0, 0, 0, 2719, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2109, 0, 0, 0, 0, 622, 4095, 0] payload_rows_per_dst=[7798, 8180, 8192, 5554, 8192, 8170, 8192, 5428] dup_rows_per_dst=[66, 8, 0, 0, 346, 4356, 0, 1054] combine_rows_per_src=[7798, 8180, 8192, 5554, 8192, 8170, 8192, 5428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=361897984 dispatch_meta_bytes=207464 combine_payload_bytes=361897984 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=362105448 ingress_bytes=802299904 link_bytes=802299904 token_bytes=724003432 weight_movement_bytes=440401920 total_nvlink_bytes=1164405352 weight_to_token_ratio=0.608287 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=16896 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.395331489e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59208 payload_rows_remote=51098 dup_rows=6328 dup_rows_remote=6190 dedup_ratio=0.096558 router_tokens_per_expert=[0, 0, 3546, 0, 0, 0, 115, 0, 0, 0, 0, 0, 0, 4054, 70, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2303, 0, 0, 0, 0, 0, 272, 68, 60, 0, 0, 0, 4075, 0, 16, 0, 2103, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3236, 0, 0, 0, 0, 606, 4052, 0] payload_rows_per_dst=[7210, 8110, 8192, 8182, 8190, 4238, 8192, 6894] dup_rows_per_dst=[112, 138, 0, 4574, 714, 0, 0, 790] combine_rows_per_src=[7210, 8110, 8192, 8182, 8190, 4238, 8192, 6894] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=359088128 dispatch_meta_bytes=202024 combine_payload_bytes=359088128 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=359290152 ingress_bytes=579289088 link_bytes=579289088 token_bytes=718378280 weight_movement_bytes=220200960 total_nvlink_bytes=938579240 weight_to_token_ratio=0.306525 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=22528 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.007479001e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=365524992 dispatch_meta_bytes=229368 combine_payload_bytes=365524992 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=365754360 ingress_bytes=475625472 link_bytes=475625472 token_bytes=731279352 weight_movement_bytes=110100480 total_nvlink_bytes=841379832 weight_to_token_ratio=0.150559 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.271909228e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60746 payload_rows_remote=53106 dup_rows=4790 dup_rows_remote=2846 dedup_ratio=0.073090 router_tokens_per_expert=[14, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 1809, 0, 0, 0, 0, 0, 0, 1, 4046, 0, 0, 3669, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 63, 0, 0, 2, 0, 0, 0, 0, 2693, 0, 0, 6, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8186, 6604, 8094, 7640, 8192, 8192, 5648, 8190] dup_rows_per_dst=[4, 2326, 0, 1944, 0, 126, 378, 12] combine_rows_per_src=[8186, 6604, 8094, 7640, 8192, 8192, 5648, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61378 payload_rows_remote=54604 dup_rows=4158 dup_rows_remote=2204 dedup_ratio=0.063446 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3879, 0, 0, 65, 0, 4093, 3242, 3928, 0, 0, 0, 45, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4090, 4096, 4034, 0, 22, 0, 0, 0, 0, 760, 0, 0, 0, 0, 0, 0, 117, 5, 0, 0, 224, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7758, 6774, 8192, 8068, 6010, 8192, 8192, 8192] dup_rows_per_dst=[0, 1954, 90, 0, 1296, 238, 580, 0] combine_rows_per_src=[7758, 6774, 8192, 8068, 6010, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61878 payload_rows_remote=53686 dup_rows=3658 dup_rows_remote=3658 dedup_ratio=0.055817 router_tokens_per_expert=[18, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 2309, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 3646, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 2213, 0, 0, 8, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 4618, 8108, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[16, 0, 0, 2, 0, 98, 3526, 16] combine_rows_per_src=[8192, 4618, 8108, 8192, 8192, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=372047872 dispatch_meta_bytes=229376 combine_payload_bytes=372047872 prefetch_bytes=0 reduce_bytes=0 egress_bytes=372277248 ingress_bytes=372047872 link_bytes=372277248 token_bytes=744325120 weight_movement_bytes=0 total_nvlink_bytes=744325120 weight_to_token_ratio=0.000000 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.474513634e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=379602944 dispatch_meta_bytes=229496 combine_payload_bytes=379602944 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=379832440 ingress_bytes=489703424 link_bytes=489703424 token_bytes=759435384 weight_movement_bytes=110100480 total_nvlink_bytes=869535864 weight_to_token_ratio=0.144977 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.516747967e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56548 payload_rows_remote=48456 dup_rows=8988 dup_rows_remote=4056 dedup_ratio=0.137146 router_tokens_per_expert=[0, 0, 3023, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 3968, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 334, 0, 0, 0, 0, 0, 871, 0, 25, 3892, 0, 0, 3461, 0, 0, 0, 2309, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2561, 0, 0, 0, 0, 17, 4087, 0] payload_rows_per_dst=[8092, 8018, 8192, 8178, 6128, 4618, 8192, 5130] dup_rows_per_dst=[4932, 1974, 0, 664, 1392, 0, 0, 26] combine_rows_per_src=[8092, 8018, 8192, 8178, 6128, 4618, 8192, 5130] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61918 payload_rows_remote=53730 dup_rows=3618 dup_rows_remote=3520 dedup_ratio=0.055206 router_tokens_per_expert=[30, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 9, 0, 2312, 0, 0, 0, 0, 0, 0, 6, 4068, 0, 0, 3668, 4091, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 0, 0, 0, 4094, 0, 0, 49, 0, 0, 8, 0, 5, 0, 0, 2153, 0, 0, 0, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[8192, 4642, 8148, 8184, 8182, 8188, 8190, 8192] dup_rows_per_dst=[40, 0, 0, 0, 0, 98, 3478, 2] combine_rows_per_src=[8192, 4642, 8148, 8184, 8182, 8188, 8190, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=380663808 dispatch_meta_bytes=223808 combine_payload_bytes=380663808 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=380887616 ingress_bytes=710965248 link_bytes=710965248 token_bytes=761551424 weight_movement_bytes=330301440 total_nvlink_bytes=1091852864 weight_to_token_ratio=0.433722 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=17920 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.236485500e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57928 payload_rows_remote=49922 dup_rows=7608 dup_rows_remote=3530 dedup_ratio=0.116089 router_tokens_per_expert=[0, 0, 3883, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2097, 0, 0, 0, 0, 0, 267, 0, 8, 0, 0, 0, 4075, 0, 0, 0, 1953, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3295, 0, 0, 0, 0, 753, 4096, 0] payload_rows_per_dst=[8006, 8170, 8184, 6220, 8192, 3906, 8192, 7058] dup_rows_per_dst=[4078, 8, 0, 1934, 550, 0, 0, 1038] combine_rows_per_src=[8006, 8170, 8184, 6220, 8192, 3906, 8192, 7058] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64158 payload_rows_remote=55966 dup_rows=1378 dup_rows_remote=1378 dedup_ratio=0.021027 router_tokens_per_expert=[21, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 652, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 4030, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 3493, 0, 0, 3, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8152, 8060, 8192, 8192, 6986, 8192] dup_rows_per_dst=[34, 1304, 0, 2, 0, 32, 0, 6] combine_rows_per_src=[8192, 8192, 8152, 8060, 8192, 8192, 6986, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=369252352 dispatch_meta_bytes=229376 combine_payload_bytes=369252352 prefetch_bytes=0 reduce_bytes=0 egress_bytes=369481728 ingress_bytes=369252352 link_bytes=369481728 token_bytes=738734080 weight_movement_bytes=0 total_nvlink_bytes=738734080 weight_to_token_ratio=0.000000 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.425894943e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=391401472 dispatch_meta_bytes=227232 combine_payload_bytes=391401472 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=391628704 ingress_bytes=721702912 link_bytes=721702912 token_bytes=783030176 weight_movement_bytes=330301440 total_nvlink_bytes=1113331616 weight_to_token_ratio=0.421825 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.055147 gemm_active_groups=6 gemm_peak_group_rows=26752 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.255160064e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=384821248 dispatch_meta_bytes=229376 combine_payload_bytes=384821248 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=385050624 ingress_bytes=605022208 link_bytes=605022208 token_bytes=769871872 weight_movement_bytes=220200960 total_nvlink_bytes=990072832 weight_to_token_ratio=0.286023 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.052233129e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7022823344 prefetch_bytes=880803840 reduce_bytes=1211105280 weight_movement_bytes=2091909120 egress_bytes=3512553392 ingress_bytes=5602179072 link_bytes=5602179072 total_nvlink_bytes=9114732464 weight_to_token_ratio=0.297873 prefetch_share_of_ingress=0.157225 dedup_ratio=0.140244 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=11 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.049265 gemm_inner_pad_upper_bound=6604 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.743110806e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=347332608 dispatch_meta_bytes=210048 combine_payload_bytes=347332608 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=347542656 ingress_bytes=457433088 link_bytes=457433088 token_bytes=694875264 weight_movement_bytes=110100480 total_nvlink_bytes=804975744 weight_to_token_ratio=0.158446 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=21760 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.955513748e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=385136640 dispatch_meta_bytes=229000 combine_payload_bytes=385136640 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=385365640 ingress_bytes=715438080 link_bytes=715438080 token_bytes=770502280 weight_movement_bytes=330301440 total_nvlink_bytes=1100803720 weight_to_token_ratio=0.428683 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.244264490e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7143224672 prefetch_bytes=990904320 reduce_bytes=1321205760 weight_movement_bytes=2312110080 egress_bytes=3572757856 ingress_bytes=5882576896 link_bytes=5882576896 total_nvlink_bytes=9455334752 weight_to_token_ratio=0.323679 prefetch_share_of_ingress=0.168447 dedup_ratio=0.134991 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=12 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048529 gemm_inner_pad_upper_bound=6985 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.023076874e-04 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7091203120 prefetch_bytes=990904320 reduce_bytes=1211105280 weight_movement_bytes=2202009600 egress_bytes=3546741808 ingress_bytes=5746470912 link_bytes=5746470912 total_nvlink_bytes=9293212720 weight_to_token_ratio=0.310527 prefetch_share_of_ingress=0.172437 dedup_ratio=0.132349 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=11 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.049265 gemm_inner_pad_upper_bound=6858 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.994058048e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61424 payload_rows_remote=53234 dup_rows=4112 dup_rows_remote=92 dedup_ratio=0.062744 router_tokens_per_expert=[8, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 2, 0, 2066, 0, 0, 0, 0, 0, 0, 1, 4071, 0, 0, 3632, 4095, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 4094, 0, 0, 39, 0, 0, 2, 0, 1, 0, 0, 2470, 0, 0, 2, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 4136, 8144, 8192, 8190, 8188, 8190, 8192] dup_rows_per_dst=[8, 0, 0, 2, 0, 78, 4020, 4] combine_rows_per_src=[8192, 4136, 8144, 8192, 8190, 8188, 8190, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7060536160 prefetch_bytes=1101004800 reduce_bytes=1101004800 weight_movement_bytes=2202009600 egress_bytes=3531414368 ingress_bytes=5731131392 link_bytes=5731131392 total_nvlink_bytes=9262545760 weight_to_token_ratio=0.311876 prefetch_share_of_ingress=0.192110 dedup_ratio=0.139566 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=10 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.046691 gemm_inner_pad_upper_bound=7620 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.967380100e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=366270464 dispatch_meta_bytes=229152 combine_payload_bytes=366270464 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=366499616 ingress_bytes=586471424 link_bytes=586471424 token_bytes=732770080 weight_movement_bytes=220200960 total_nvlink_bytes=952971040 weight_to_token_ratio=0.300505 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.019970264e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64714 payload_rows_remote=56662 dup_rows=822 dup_rows_remote=822 dedup_ratio=0.012543 router_tokens_per_expert=[14, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 1, 0, 400, 0, 0, 0, 0, 0, 0, 2, 4084, 0, 0, 4026, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3766, 0, 0, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8170, 8052, 8192, 8192, 7532, 8192] dup_rows_per_dst=[10, 802, 2, 0, 0, 6, 0, 2] combine_rows_per_src=[8192, 8192, 8170, 8052, 8192, 8192, 7532, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=357840896 dispatch_meta_bytes=213808 combine_payload_bytes=357840896 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=358054704 ingress_bytes=467941376 link_bytes=467941376 token_bytes=715895600 weight_movement_bytes=110100480 total_nvlink_bytes=825996080 weight_to_token_ratio=0.153794 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=30336 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.138270160e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=401164288 dispatch_meta_bytes=229376 combine_payload_bytes=401164288 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=401393664 ingress_bytes=621365248 link_bytes=621365248 token_bytes=802557952 weight_movement_bytes=220200960 total_nvlink_bytes=1022758912 weight_to_token_ratio=0.274374 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.080656363e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64310 payload_rows_remote=56122 dup_rows=1226 dup_rows_remote=1214 dedup_ratio=0.018707 router_tokens_per_expert=[35, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 9, 0, 567, 0, 0, 0, 0, 0, 0, 5, 4081, 0, 0, 4056, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 0, 0, 0, 4094, 0, 0, 6, 0, 0, 2, 0, 5, 0, 0, 3535, 0, 0, 0, 0, 0, 0, 0, 7, 0] payload_rows_per_dst=[8192, 8192, 8172, 8112, 8182, 8188, 7080, 8192] dup_rows_per_dst=[52, 1144, 0, 0, 0, 12, 4, 14] combine_rows_per_src=[8192, 8192, 8172, 8112, 8182, 8188, 7080, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58050 payload_rows_remote=49858 dup_rows=7486 dup_rows_remote=7480 dedup_ratio=0.114227 router_tokens_per_expert=[23, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4, 0, 591, 0, 0, 0, 0, 0, 0, 2, 4059, 0, 0, 4050, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 1, 0, 0, 4094, 0, 0, 12, 0, 0, 2, 0, 3, 0, 0, 3550, 0, 0, 3, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7788, 1190, 8122, 8188, 8188, 8190, 8192, 8192] dup_rows_per_dst=[34, 0, 0, 288, 0, 140, 7018, 6] combine_rows_per_src=[7788, 1190, 8122, 8188, 8188, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64104 payload_rows_remote=55930 dup_rows=1432 dup_rows_remote=1432 dedup_ratio=0.021851 router_tokens_per_expert=[41, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 9, 0, 662, 0, 0, 0, 0, 0, 0, 4, 4083, 0, 0, 4054, 4092, 0, 0, 0, 6, 0, 0, 0, 0, 4096, 0, 0, 0, 4087, 0, 0, 0, 4094, 0, 0, 6, 0, 0, 3, 0, 2, 0, 0, 3439, 0, 0, 15, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[8192, 8192, 8174, 8108, 8174, 8188, 6884, 8192] dup_rows_per_dst=[38, 1334, 0, 12, 0, 12, 4, 32] combine_rows_per_src=[8192, 8192, 8174, 8108, 8174, 8188, 6884, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=406153216 dispatch_meta_bytes=229936 combine_payload_bytes=406153216 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=406383152 ingress_bytes=736454656 link_bytes=736454656 token_bytes=812536368 weight_movement_bytes=330301440 total_nvlink_bytes=1142837808 weight_to_token_ratio=0.406507 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=27776 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.280815773e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7370036392 prefetch_bytes=1211105280 reduce_bytes=770703360 weight_movement_bytes=1981808640 egress_bytes=3686171816 ingress_bytes=5665673216 link_bytes=5665673216 total_nvlink_bytes=9351845032 weight_to_token_ratio=0.268901 prefetch_share_of_ingress=0.213762 dedup_ratio=0.107251 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=7 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048897 gemm_inner_pad_upper_bound=7112 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.853537566e-05 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63166 payload_rows_remote=55816 dup_rows=2370 dup_rows_remote=1336 dedup_ratio=0.036163 router_tokens_per_expert=[7, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 2, 0, 497, 0, 0, 0, 0, 0, 0, 1, 4091, 0, 0, 4048, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 4095, 0, 0, 13, 0, 0, 1, 0, 1, 0, 0, 3629, 0, 0, 2, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[8192, 6764, 8184, 8104, 8190, 8190, 7350, 8192] dup_rows_per_dst=[8, 850, 0, 446, 0, 26, 1034, 6] combine_rows_per_src=[8192, 6764, 8184, 8104, 8190, 8190, 7350, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63316 payload_rows_remote=55124 dup_rows=2220 dup_rows_remote=308 dedup_ratio=0.033875 router_tokens_per_expert=[141, 0, 0, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 39, 0, 932, 0, 0, 0, 0, 0, 0, 15, 4037, 0, 0, 4019, 4081, 0, 0, 0, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 4066, 19, 0, 0, 4094, 0, 0, 4, 0, 0, 0, 0, 36, 0, 0, 3102, 0, 0, 30, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8104, 8038, 8132, 8190, 6276, 8192] dup_rows_per_dst=[194, 1912, 0, 10, 0, 44, 0, 60] combine_rows_per_src=[8192, 8192, 8104, 8038, 8132, 8190, 6276, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=402282496 dispatch_meta_bytes=229344 combine_payload_bytes=402282496 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=402511840 ingress_bytes=622483456 link_bytes=622483456 token_bytes=804794336 weight_movement_bytes=220200960 total_nvlink_bytes=1024995296 weight_to_token_ratio=0.273611 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.082601111e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=357382144 dispatch_meta_bytes=229352 combine_payload_bytes=357382144 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=357611496 ingress_bytes=467482624 link_bytes=467482624 token_bytes=714993640 weight_movement_bytes=110100480 total_nvlink_bytes=825094120 weight_to_token_ratio=0.153988 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.130291708e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=400906240 dispatch_meta_bytes=229448 combine_payload_bytes=400906240 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=401135688 ingress_bytes=511006720 link_bytes=511006720 token_bytes=802041928 weight_movement_bytes=110100480 total_nvlink_bytes=912142408 weight_to_token_ratio=0.137275 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.887247322e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7351688032 prefetch_bytes=990904320 reduce_bytes=1321205760 weight_movement_bytes=2312110080 egress_bytes=3676998496 ingress_bytes=5986799616 link_bytes=5986799616 total_nvlink_bytes=9663798112 weight_to_token_ratio=0.314501 prefetch_share_of_ingress=0.165515 dedup_ratio=0.114810 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=12 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048529 gemm_inner_pad_upper_bound=7112 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.041202919e-04 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58428 payload_rows_remote=50478 dup_rows=7108 dup_rows_remote=6954 dedup_ratio=0.108459 router_tokens_per_expert=[0, 0, 2475, 0, 0, 0, 75, 0, 0, 0, 0, 0, 0, 3975, 77, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 0, 0, 0, 0, 0, 753, 54, 81, 3331, 0, 0, 3818, 0, 34, 0, 2171, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3120, 0, 0, 0, 0, 41, 4023, 0] payload_rows_per_dst=[8148, 7950, 8192, 8078, 7210, 4410, 8192, 6248] dup_rows_per_dst=[4588, 154, 4, 1060, 1228, 0, 0, 74] combine_rows_per_src=[8148, 7950, 8192, 8078, 7210, 4410, 8192, 6248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=381581312 dispatch_meta_bytes=213304 combine_payload_bytes=381581312 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=381794616 ingress_bytes=491681792 link_bytes=491681792 token_bytes=763375928 weight_movement_bytes=110100480 total_nvlink_bytes=873476408 weight_to_token_ratio=0.144228 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=17792 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.551155040e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59524 payload_rows_remote=51332 dup_rows=6012 dup_rows_remote=5850 dedup_ratio=0.091736 router_tokens_per_expert=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2273, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 3817, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 2060, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8166, 7674, 8192, 7044, 4120, 7944] dup_rows_per_dst=[162, 4546, 598, 508, 86, 112, 0, 0] combine_rows_per_src=[8192, 8192, 8166, 7674, 8192, 7044, 4120, 7944] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=361826304 dispatch_meta_bytes=229728 combine_payload_bytes=361826304 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=362056032 ingress_bytes=471926784 link_bytes=471926784 token_bytes=723882336 weight_movement_bytes=110100480 total_nvlink_bytes=833982816 weight_to_token_ratio=0.152097 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.207582960e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6875629176 prefetch_bytes=1211105280 reduce_bytes=660602880 weight_movement_bytes=1871708160 egress_bytes=3438917240 ingress_bytes=5308420096 link_bytes=5308649472 total_nvlink_bytes=8747337336 weight_to_token_ratio=0.272224 prefetch_share_of_ingress=0.228148 dedup_ratio=0.149316 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=6 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.047426 gemm_inner_pad_upper_bound=7747 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.232614555e-05 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=367947776 dispatch_meta_bytes=228728 combine_payload_bytes=367947776 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=368176504 ingress_bytes=588148736 link_bytes=588148736 token_bytes=736124280 weight_movement_bytes=220200960 total_nvlink_bytes=956325240 weight_to_token_ratio=0.299136 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.022887386e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6581332408 prefetch_bytes=990904320 reduce_bytes=1761607680 weight_movement_bytes=2752512000 egress_bytes=3291750840 ingress_bytes=6042093568 link_bytes=6042093568 total_nvlink_bytes=9333844408 weight_to_token_ratio=0.418230 prefetch_share_of_ingress=0.164000 dedup_ratio=0.178348 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=16 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.049265 gemm_inner_pad_upper_bound=6731 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.050819447e-04 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60036 payload_rows_remote=51844 dup_rows=5500 dup_rows_remote=402 dedup_ratio=0.083923 router_tokens_per_expert=[113, 0, 0, 0, 0, 4029, 0, 0, 0, 0, 0, 0, 42, 0, 2522, 0, 0, 0, 0, 0, 0, 7, 4012, 0, 0, 3852, 4081, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 4068, 26, 0, 0, 4078, 0, 0, 52, 0, 0, 21, 0, 27, 0, 0, 1666, 0, 0, 74, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8038, 7704, 8136, 8190, 3392, 8192] dup_rows_per_dst=[92, 5098, 0, 4, 0, 122, 36, 148] combine_rows_per_src=[8192, 8192, 8038, 7704, 8136, 8190, 3392, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64038 payload_rows_remote=55846 dup_rows=1498 dup_rows_remote=1464 dedup_ratio=0.022858 router_tokens_per_expert=[1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 542, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4058, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 3588, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8184, 8116, 8192, 7920, 7176, 8066] dup_rows_per_dst=[34, 1084, 272, 0, 94, 14, 0, 0] combine_rows_per_src=[8192, 8192, 8184, 8116, 8192, 7920, 7176, 8066] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=371617792 dispatch_meta_bytes=208984 combine_payload_bytes=371617792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=371826776 ingress_bytes=481718272 link_bytes=481718272 token_bytes=743444568 weight_movement_bytes=110100480 total_nvlink_bytes=853545048 weight_to_token_ratio=0.148095 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=17792 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.377873040e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6882970480 prefetch_bytes=1101004800 reduce_bytes=770703360 weight_movement_bytes=1871708160 egress_bytes=3442602864 ingress_bytes=5312075776 link_bytes=5312075776 total_nvlink_bytes=8754678640 weight_to_token_ratio=0.271933 prefetch_share_of_ingress=0.207265 dedup_ratio=0.150693 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=7 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.051471 gemm_inner_pad_upper_bound=6604 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.238573461e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none - [2026-08-26 15:11:34] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 369.2 | throughput per GPU (TFLOP/s/GPU): 334.5 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.601799E+00 | loss scale: 1.0 | grad norm: 4.396 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43890 payload_rows_remote=38298 dup_rows=21646 dup_rows_remote=19340 dedup_ratio=0.330292 router_tokens_per_expert=[827, 808, 796, 651, 536, 796, 594, 166, 709, 669, 400, 433, 13, 1061, 259, 763, 578, 829, 196, 88, 831, 338, 203, 886, 383, 1440, 636, 723, 1058, 414, 867, 10, 220, 619, 416, 13, 919, 5, 1237, 357, 228, 633, 774, 1166, 3, 468, 144, 754, 230, 237, 121, 774, 212, 546, 205, 667, 547, 527, 265, 323, 288, 447, 445, 17] payload_rows_per_dst=[6204, 5584, 5592, 6614, 5218, 5766, 4530, 4382] dup_rows_per_dst=[4144, 3030, 2306, 4448, 2354, 2574, 1454, 1336] combine_rows_per_src=[6204, 5584, 5592, 6614, 5218, 5766, 4530, 4382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44142 payload_rows_remote=37896 dup_rows=21394 dup_rows_remote=17338 dedup_ratio=0.326447 router_tokens_per_expert=[1081, 1356, 884, 319, 366, 962, 536, 129, 617, 1007, 284, 540, 18, 924, 167, 682, 341, 792, 4, 69, 631, 420, 720, 454, 439, 292, 597, 546, 367, 463, 787, 10, 1005, 1054, 546, 37, 369, 12, 678, 793, 643, 525, 316, 774, 0, 540, 105, 630, 597, 160, 451, 1018, 250, 768, 120, 706, 519, 694, 594, 678, 314, 847, 209, 12] payload_rows_per_dst=[3552, 5070, 6186, 5298, 5844, 5456, 6246, 6490] dup_rows_per_dst=[916, 2068, 3388, 1704, 3144, 2950, 4056, 3168] combine_rows_per_src=[3552, 5070, 6186, 5298, 5844, 5456, 6246, 6490] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44892 payload_rows_remote=39024 dup_rows=20644 dup_rows_remote=17424 dedup_ratio=0.315002 router_tokens_per_expert=[1217, 1035, 395, 903, 332, 628, 616, 23, 251, 835, 641, 326, 15, 1011, 37, 979, 531, 758, 3, 25, 864, 192, 847, 427, 284, 583, 377, 776, 718, 294, 808, 5, 567, 551, 453, 3, 994, 0, 1034, 527, 681, 678, 354, 1531, 0, 764, 109, 427, 446, 331, 510, 456, 311, 770, 90, 401, 405, 1373, 653, 254, 440, 609, 303, 7] payload_rows_per_dst=[4688, 5566, 5818, 5440, 5332, 5868, 5954, 6226] dup_rows_per_dst=[1060, 2624, 2336, 2250, 2926, 3220, 3110, 3118] combine_rows_per_src=[4688, 5566, 5818, 5440, 5332, 5868, 5954, 6226] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44564 payload_rows_remote=38468 dup_rows=20972 dup_rows_remote=17976 dedup_ratio=0.320007 router_tokens_per_expert=[785, 1539, 1162, 555, 558, 1123, 861, 62, 748, 944, 609, 328, 34, 1018, 191, 794, 470, 493, 9, 71, 771, 209, 582, 316, 367, 704, 463, 750, 594, 356, 524, 16, 861, 891, 545, 4, 712, 4, 1148, 424, 264, 709, 333, 1248, 0, 368, 118, 441, 201, 339, 538, 523, 218, 668, 74, 708, 835, 550, 447, 257, 461, 487, 380, 6] payload_rows_per_dst=[4824, 5434, 5934, 5338, 5756, 5760, 5422, 6096] dup_rows_per_dst=[1572, 2010, 2986, 2210, 3422, 3090, 2686, 2996] combine_rows_per_src=[4824, 5434, 5934, 5338, 5756, 5760, 5422, 6096] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44734 payload_rows_remote=38638 dup_rows=20802 dup_rows_remote=17648 dedup_ratio=0.317413 router_tokens_per_expert=[1299, 786, 608, 306, 512, 605, 616, 233, 263, 1225, 905, 358, 9, 876, 218, 771, 714, 701, 47, 127, 784, 229, 402, 550, 516, 618, 409, 684, 578, 272, 462, 31, 785, 684, 1158, 16, 792, 12, 941, 257, 545, 919, 402, 776, 0, 748, 91, 500, 336, 193, 374, 622, 208, 1066, 167, 905, 364, 1270, 585, 243, 450, 291, 342, 12] payload_rows_per_dst=[6234, 6096, 5348, 5412, 5644, 5336, 5504, 5160] dup_rows_per_dst=[3696, 3154, 1760, 2246, 3128, 2626, 2238, 1954] combine_rows_per_src=[6234, 6096, 5348, 5412, 5644, 5336, 5504, 5160] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44002 payload_rows_remote=38842 dup_rows=21534 dup_rows_remote=19996 dedup_ratio=0.328583 router_tokens_per_expert=[1275, 897, 499, 712, 505, 790, 449, 240, 366, 829, 511, 541, 17, 815, 99, 957, 470, 1177, 10, 89, 1026, 357, 689, 427, 582, 418, 365, 653, 287, 393, 641, 10, 788, 515, 572, 13, 1074, 2, 931, 517, 326, 586, 526, 1227, 5, 477, 34, 397, 228, 170, 672, 646, 451, 808, 250, 528, 327, 1022, 596, 311, 339, 907, 390, 37] payload_rows_per_dst=[6450, 5396, 5932, 5160, 5586, 5184, 5162, 5132] dup_rows_per_dst=[4284, 2874, 2558, 1538, 3238, 1972, 2344, 2726] combine_rows_per_src=[6450, 5396, 5932, 5160, 5586, 5184, 5162, 5132] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44418 payload_rows_remote=38874 dup_rows=21118 dup_rows_remote=18244 dedup_ratio=0.322235 router_tokens_per_expert=[1503, 530, 622, 271, 531, 542, 569, 198, 285, 1013, 414, 738, 7, 490, 76, 829, 548, 1101, 42, 38, 653, 178, 817, 550, 676, 563, 363, 845, 413, 315, 741, 24, 971, 599, 369, 19, 730, 12, 1186, 275, 490, 776, 281, 1021, 4, 636, 214, 725, 209, 401, 453, 730, 319, 907, 137, 694, 492, 940, 439, 362, 413, 1098, 367, 14] payload_rows_per_dst=[5512, 5238, 5662, 5546, 5544, 5478, 5476, 5962] dup_rows_per_dst=[2372, 2514, 2192, 2334, 2874, 2816, 2804, 3212] combine_rows_per_src=[5512, 5238, 5662, 5546, 5544, 5478, 5476, 5962] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=400089088 dispatch_meta_bytes=228608 combine_payload_bytes=400089088 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=400317696 ingress_bytes=510189568 link_bytes=510189568 token_bytes=800406784 weight_movement_bytes=110100480 total_nvlink_bytes=910507264 weight_to_token_ratio=0.137556 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=28160 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.873035705e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6616626720 prefetch_bytes=990904320 reduce_bytes=990904320 weight_movement_bytes=1981808640 egress_bytes=3309411872 ingress_bytes=5289023488 link_bytes=5289023488 total_nvlink_bytes=8598435360 weight_to_token_ratio=0.299519 prefetch_share_of_ingress=0.187351 dedup_ratio=0.177032 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=9 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.049632 gemm_inner_pad_upper_bound=6858 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.198481741e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7216786584 prefetch_bytes=1211105280 reduce_bytes=770703360 weight_movement_bytes=1981808640 egress_bytes=3609533592 ingress_bytes=5589061632 link_bytes=5589061632 total_nvlink_bytes=9198595224 weight_to_token_ratio=0.274611 prefetch_share_of_ingress=0.216692 dedup_ratio=0.122809 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=7 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.047426 gemm_inner_pad_upper_bound=7747 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.720297421e-05 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7232113176 prefetch_bytes=770703360 reduce_bytes=880803840 weight_movement_bytes=1651507200 egress_bytes=3617204760 ingress_bytes=5266415616 link_bytes=5266415616 total_nvlink_bytes=8883620376 weight_to_token_ratio=0.228357 prefetch_share_of_ingress=0.146343 dedup_ratio=0.117209 prefetch_slots=14 unused_prefetch_slots=146 slot_utilization=0.087500 reduce_slots=8 remote_demand=7 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.049632 gemm_inner_pad_upper_bound=6477 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.159162933e-05 prefetch_bytes_per_gemm_flop=1.340379902e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=395128832 dispatch_meta_bytes=221728 combine_payload_bytes=395128832 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=395350560 ingress_bytes=505229312 link_bytes=505229312 token_bytes=790479392 weight_movement_bytes=110100480 total_nvlink_bytes=900579872 weight_to_token_ratio=0.139283 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=27904 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.786768695e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=400304128 dispatch_meta_bytes=229240 combine_payload_bytes=400304128 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=400533368 ingress_bytes=730605568 link_bytes=730605568 token_bytes=800837496 weight_movement_bytes=330301440 total_nvlink_bytes=1131138936 weight_to_token_ratio=0.412445 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.270643247e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7014754960 prefetch_bytes=1101004800 reduce_bytes=1101004800 weight_movement_bytes=2202009600 egress_bytes=3508499088 ingress_bytes=5708265472 link_bytes=5708494848 total_nvlink_bytes=9216764560 weight_to_token_ratio=0.313911 prefetch_share_of_ingress=0.192879 dedup_ratio=0.137576 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=10 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.047794 gemm_inner_pad_upper_bound=7366 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.928011427e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6802481328 prefetch_bytes=880803840 reduce_bytes=1761607680 weight_movement_bytes=2642411520 egress_bytes=3402340528 ingress_bytes=6042552320 link_bytes=6042552320 total_nvlink_bytes=9444892848 weight_to_token_ratio=0.388448 prefetch_share_of_ingress=0.145767 dedup_ratio=0.155954 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=16 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.049632 gemm_inner_pad_upper_bound=6604 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.050899231e-04 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7047578752 prefetch_bytes=1211105280 reduce_bytes=770703360 weight_movement_bytes=1981808640 egress_bytes=3524922496 ingress_bytes=5504464896 link_bytes=5504464896 total_nvlink_bytes=9029387392 weight_to_token_ratio=0.281204 prefetch_share_of_ingress=0.220022 dedup_ratio=0.134003 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=7 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.049265 gemm_inner_pad_upper_bound=6731 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.573169783e-05 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none + [2026-08-26 15:59:21] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 361.8 | throughput per GPU (TFLOP/s/GPU): 341.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.600108E+00 | loss scale: 1.0 | grad norm: 4.394 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45058 payload_rows_remote=39180 dup_rows=20478 dup_rows_remote=17154 dedup_ratio=0.312469 router_tokens_per_expert=[1238, 1020, 374, 888, 306, 605, 608, 23, 254, 853, 668, 313, 13, 966, 31, 941, 512, 824, 3, 31, 846, 226, 858, 429, 290, 602, 373, 774, 757, 245, 799, 6, 571, 581, 439, 3, 1021, 0, 1040, 563, 616, 652, 333, 1539, 0, 780, 148, 434, 457, 369, 530, 422, 310, 742, 82, 400, 435, 1341, 644, 271, 470, 587, 305, 7] payload_rows_per_dst=[5110, 5554, 5650, 5386, 5338, 5878, 5956, 6186] dup_rows_per_dst=[1320, 2524, 1816, 2306, 2900, 3324, 3144, 3144] combine_rows_per_src=[5110, 5554, 5650, 5386, 5338, 5878, 5956, 6186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44016 payload_rows_remote=37798 dup_rows=21520 dup_rows_remote=17514 dedup_ratio=0.328369 router_tokens_per_expert=[1087, 1346, 878, 299, 339, 936, 530, 127, 643, 1009, 292, 525, 16, 897, 157, 666, 329, 821, 2, 72, 610, 473, 716, 455, 455, 298, 609, 543, 402, 429, 784, 13, 996, 1072, 534, 37, 397, 12, 713, 845, 572, 507, 291, 752, 0, 550, 148, 632, 623, 173, 470, 960, 247, 745, 108, 699, 563, 665, 602, 702, 320, 847, 216, 12] payload_rows_per_dst=[3470, 5328, 6190, 5458, 5490, 5338, 6218, 6524] dup_rows_per_dst=[876, 2720, 3458, 1970, 2296, 2992, 4006, 3202] combine_rows_per_src=[3470, 5328, 6190, 5458, 5490, 5338, 6218, 6524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43850 payload_rows_remote=38208 dup_rows=21686 dup_rows_remote=19314 dedup_ratio=0.330902 router_tokens_per_expert=[836, 799, 784, 615, 489, 766, 608, 148, 707, 671, 402, 417, 11, 1029, 253, 726, 568, 878, 171, 101, 812, 362, 232, 883, 386, 1449, 630, 723, 1120, 388, 892, 13, 210, 638, 410, 14, 957, 7, 1244, 395, 189, 617, 737, 1150, 2, 473, 198, 761, 256, 248, 117, 739, 214, 534, 196, 676, 596, 517, 257, 340, 300, 429, 462, 16] payload_rows_per_dst=[6110, 5482, 5642, 6640, 5290, 5694, 4526, 4466] dup_rows_per_dst=[3980, 2950, 2372, 4562, 2460, 2560, 1434, 1368] combine_rows_per_src=[6110, 5482, 5642, 6640, 5290, 5694, 4526, 4466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44350 payload_rows_remote=38774 dup_rows=21186 dup_rows_remote=18260 dedup_ratio=0.323273 router_tokens_per_expert=[1523, 526, 606, 253, 495, 532, 571, 179, 281, 1024, 433, 710, 7, 460, 68, 808, 523, 1146, 41, 46, 637, 202, 822, 548, 686, 564, 362, 834, 446, 279, 750, 24, 958, 619, 359, 18, 767, 12, 1194, 324, 447, 757, 267, 1000, 4, 637, 277, 730, 229, 438, 469, 684, 307, 885, 134, 695, 546, 907, 434, 379, 429, 1081, 381, 14] payload_rows_per_dst=[5608, 5158, 5708, 5558, 5576, 5474, 5538, 5730] dup_rows_per_dst=[2610, 2424, 2222, 2332, 2926, 2764, 2836, 3072] combine_rows_per_src=[5608, 5158, 5708, 5558, 5576, 5474, 5538, 5730] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44016 payload_rows_remote=38864 dup_rows=21520 dup_rows_remote=19940 dedup_ratio=0.328369 router_tokens_per_expert=[1305, 896, 482, 668, 469, 791, 449, 222, 378, 837, 535, 528, 16, 787, 96, 933, 462, 1238, 8, 94, 1023, 386, 701, 417, 597, 428, 366, 643, 326, 350, 645, 11, 780, 533, 570, 11, 1107, 2, 949, 560, 290, 569, 496, 1205, 4, 482, 55, 393, 249, 183, 701, 609, 459, 793, 236, 523, 350, 980, 587, 334, 351, 873, 407, 40] payload_rows_per_dst=[6402, 5418, 6018, 5152, 5648, 5076, 5162, 5140] dup_rows_per_dst=[4162, 2802, 2640, 1580, 3376, 1912, 2344, 2704] combine_rows_per_src=[6402, 5418, 6018, 5152, 5648, 5076, 5162, 5140] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44386 payload_rows_remote=38268 dup_rows=21150 dup_rows_remote=18104 dedup_ratio=0.322723 router_tokens_per_expert=[796, 1522, 1141, 520, 501, 1087, 861, 53, 764, 945, 631, 314, 31, 997, 185, 779, 448, 530, 9, 75, 757, 233, 590, 312, 371, 695, 461, 752, 634, 330, 517, 17, 865, 921, 535, 4, 747, 5, 1148, 451, 238, 693, 304, 1236, 0, 378, 156, 442, 220, 364, 564, 489, 219, 659, 69, 708, 892, 527, 447, 274, 485, 465, 399, 6] payload_rows_per_dst=[4694, 5368, 5948, 6104, 4990, 5688, 5476, 6118] dup_rows_per_dst=[1458, 1934, 3004, 3340, 2166, 3502, 2700, 3046] combine_rows_per_src=[4694, 5368, 5948, 6104, 4990, 5688, 5476, 6118] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44840 payload_rows_remote=38750 dup_rows=20696 dup_rows_remote=17622 dedup_ratio=0.315796 router_tokens_per_expert=[1306, 785, 584, 301, 470, 583, 613, 211, 277, 1233, 924, 335, 8, 835, 218, 752, 697, 767, 36, 134, 769, 248, 413, 546, 526, 643, 402, 689, 608, 246, 451, 40, 778, 704, 1134, 17, 863, 13, 942, 273, 478, 902, 377, 780, 0, 761, 126, 509, 355, 202, 397, 580, 209, 1047, 161, 901, 413, 1259, 571, 257, 475, 268, 354, 12] payload_rows_per_dst=[6208, 6090, 5442, 5188, 5870, 5300, 5482, 5260] dup_rows_per_dst=[3498, 3074, 1778, 2022, 3578, 2566, 2222, 1958] combine_rows_per_src=[6208, 6090, 5442, 5188, 5870, 5300, 5482, 5260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43836 payload_rows_remote=37838 dup_rows=21700 dup_rows_remote=17668 dedup_ratio=0.331116 router_tokens_per_expert=[1158, 317, 586, 347, 493, 1094, 825, 70, 212, 959, 726, 537, 13, 1160, 120, 628, 917, 497, 13, 21, 328, 421, 664, 963, 280, 445, 523, 1129, 409, 564, 599, 20, 864, 328, 450, 14, 836, 0, 1063, 386, 518, 569, 292, 646, 2, 722, 521, 464, 452, 356, 354, 393, 380, 1577, 81, 653, 519, 891, 564, 180, 232, 1057, 364, 2] payload_rows_per_dst=[5998, 5532, 5314, 5758, 5120, 5080, 5766, 5268] dup_rows_per_dst=[4032, 3178, 2334, 2180, 2512, 2388, 2726, 2350] combine_rows_per_src=[5998, 5532, 5314, 5758, 5120, 5080, 5766, 5268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=273874944 dispatch_meta_bytes=230088 combine_payload_bytes=273874944 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=274105032 ingress_bytes=383975424 link_bytes=383975424 token_bytes=547979976 weight_movement_bytes=110100480 total_nvlink_bytes=658080456 weight_to_token_ratio=0.200921 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=6784 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.677964154e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=274520064 dispatch_meta_bytes=230552 combine_payload_bytes=274520064 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=274750616 ingress_bytes=384620544 link_bytes=384620544 token_bytes=549270680 weight_movement_bytes=110100480 total_nvlink_bytes=659371160 weight_to_token_ratio=0.200448 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.689183852e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=271638528 dispatch_meta_bytes=220936 combine_payload_bytes=271638528 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=271859464 ingress_bytes=381739008 link_bytes=381739008 token_bytes=543497992 weight_movement_bytes=110100480 total_nvlink_bytes=653598472 weight_to_token_ratio=0.202578 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=7168 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.639069202e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279724032 dispatch_meta_bytes=225792 combine_payload_bytes=279724032 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279949824 ingress_bytes=389824512 link_bytes=389824512 token_bytes=559673856 weight_movement_bytes=110100480 total_nvlink_bytes=669774336 weight_to_token_ratio=0.196723 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=8448 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.779689415e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=275738624 dispatch_meta_bytes=225776 combine_payload_bytes=275738624 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=275964400 ingress_bytes=385839104 link_bytes=385839104 token_bytes=551703024 weight_movement_bytes=110100480 total_nvlink_bytes=661803504 weight_to_token_ratio=0.199565 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=7424 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.710376615e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43912 payload_rows_remote=37948 dup_rows=21624 dup_rows_remote=17560 dedup_ratio=0.329956 router_tokens_per_expert=[1139, 331, 636, 379, 537, 1108, 810, 74, 210, 956, 702, 575, 13, 1215, 122, 646, 939, 453, 15, 15, 328, 387, 669, 963, 266, 441, 520, 1149, 353, 609, 586, 19, 858, 298, 457, 12, 782, 0, 1056, 356, 583, 577, 316, 670, 2, 723, 383, 466, 433, 331, 328, 439, 382, 1620, 86, 656, 467, 929, 553, 176, 216, 1091, 355, 2] payload_rows_per_dst=[5964, 5590, 5288, 6722, 4262, 5060, 5810, 5216] dup_rows_per_dst=[4064, 3288, 2250, 3276, 1264, 2380, 2740, 2362] combine_rows_per_src=[5964, 5590, 5288, 6722, 4262, 5060, 5810, 5216] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=276957184 dispatch_meta_bytes=225144 combine_payload_bytes=276957184 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277182328 ingress_bytes=497158144 link_bytes=497158144 token_bytes=554139512 weight_movement_bytes=220200960 total_nvlink_bytes=774340472 weight_to_token_ratio=0.397375 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=7424 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.646397809e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278419456 dispatch_meta_bytes=235352 combine_payload_bytes=278419456 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278654808 ingress_bytes=388519936 link_bytes=388519936 token_bytes=557074264 weight_movement_bytes=110100480 total_nvlink_bytes=667174744 weight_to_token_ratio=0.197641 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=6144 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.757000693e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278648832 dispatch_meta_bytes=228472 combine_payload_bytes=278648832 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278877304 ingress_bytes=498849792 link_bytes=498849792 token_bytes=557526136 weight_movement_bytes=220200960 total_nvlink_bytes=777727096 weight_to_token_ratio=0.394961 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=6912 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.675818350e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=272011264 dispatch_meta_bytes=222032 combine_payload_bytes=272011264 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=272233296 ingress_bytes=822513664 link_bytes=822513664 token_bytes=544244560 weight_movement_bytes=550502400 total_nvlink_bytes=1094746960 weight_to_token_ratio=1.011498 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=5760 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.430486542e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49294 payload_rows_remote=41478 dup_rows=16242 dup_rows_remote=10670 dedup_ratio=0.247833 router_tokens_per_expert=[1, 0, 0, 51, 1947, 3, 1926, 105, 1424, 1444, 79, 0, 0, 1480, 0, 3526, 2, 2929, 0, 0, 785, 3678, 0, 0, 0, 1, 1154, 18, 514, 109, 0, 2, 1, 7, 0, 0, 0, 1, 0, 0, 0, 2, 1193, 0, 3, 0, 1758, 1, 0, 2151, 977, 0, 2661, 0, 0, 0, 1718, 5, 0, 36, 34, 234, 232, 576] payload_rows_per_dst=[6596, 6180, 7816, 3814, 7066, 6762, 6092, 4968] dup_rows_per_dst=[1470, 2674, 5572, 1182, 4, 3454, 1184, 702] combine_rows_per_src=[6596, 6180, 7816, 3814, 7066, 6762, 6092, 4968] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50382 payload_rows_remote=43586 dup_rows=15154 dup_rows_remote=11460 dedup_ratio=0.231232 router_tokens_per_expert=[0, 0, 0, 25, 2916, 0, 2386, 123, 1416, 2385, 89, 0, 0, 1911, 0, 2924, 0, 2443, 0, 0, 308, 3516, 0, 0, 0, 0, 443, 13, 603, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 696, 0, 0, 0, 2137, 0, 0, 2426, 816, 0, 2002, 0, 0, 1, 1957, 0, 0, 39, 100, 95, 185, 749] payload_rows_per_dst=[4944, 6330, 5286, 7532, 6952, 5144, 6796, 7398] dup_rows_per_dst=[124, 1450, 216, 1746, 2718, 522, 3694, 4684] combine_rows_per_src=[4944, 6330, 5286, 7532, 6952, 5144, 6796, 7398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280842240 dispatch_meta_bytes=225336 combine_payload_bytes=280842240 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281067576 ingress_bytes=390942720 link_bytes=390942720 token_bytes=561909816 weight_movement_bytes=110100480 total_nvlink_bytes=672010296 weight_to_token_ratio=0.195940 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=8320 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.799136891e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=270936064 dispatch_meta_bytes=221248 combine_payload_bytes=270936064 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=271157312 ingress_bytes=381036544 link_bytes=381036544 token_bytes=542093376 weight_movement_bytes=110100480 total_nvlink_bytes=652193856 weight_to_token_ratio=0.203102 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=7040 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.626852198e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=277932032 dispatch_meta_bytes=228136 combine_payload_bytes=277932032 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278160168 ingress_bytes=608233472 link_bytes=608233472 token_bytes=556092200 weight_movement_bytes=330301440 total_nvlink_bytes=886393640 weight_to_token_ratio=0.593969 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=6656 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057818045e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278577152 dispatch_meta_bytes=235216 combine_payload_bytes=278577152 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278812368 ingress_bytes=388677632 link_bytes=388677632 token_bytes=557389520 weight_movement_bytes=110100480 total_nvlink_bytes=667490000 weight_to_token_ratio=0.197529 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=6144 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.759743285e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=274305024 dispatch_meta_bytes=225488 combine_payload_bytes=274305024 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=274530512 ingress_bytes=384405504 link_bytes=384405504 token_bytes=548835536 weight_movement_bytes=110100480 total_nvlink_bytes=658936016 weight_to_token_ratio=0.200607 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=7168 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.685443953e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=277760000 dispatch_meta_bytes=225488 combine_payload_bytes=277760000 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=277985488 ingress_bytes=497960960 link_bytes=497960960 token_bytes=555745488 weight_movement_bytes=220200960 total_nvlink_bytes=775946448 weight_to_token_ratio=0.396226 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=7168 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.660360099e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=271222784 dispatch_meta_bytes=222024 combine_payload_bytes=271222784 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=271444808 ingress_bytes=711624704 link_bytes=711624704 token_bytes=542667592 weight_movement_bytes=440401920 total_nvlink_bytes=983069512 weight_to_token_ratio=0.811550 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=5888 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.237632402e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49300 payload_rows_remote=41206 dup_rows=16236 dup_rows_remote=11562 dedup_ratio=0.247742 router_tokens_per_expert=[1, 0, 0, 52, 882, 3, 2283, 113, 1421, 1776, 114, 0, 1, 1884, 0, 3534, 2, 1654, 0, 0, 856, 3872, 0, 0, 0, 1, 1238, 388, 908, 148, 0, 22, 1, 9, 0, 0, 0, 20, 0, 0, 0, 1, 1652, 0, 4, 0, 1686, 1, 0, 2238, 811, 0, 1893, 0, 0, 0, 1103, 0, 0, 39, 442, 352, 536, 827] payload_rows_per_dst=[5966, 6620, 8094, 4170, 7086, 5566, 6518, 5280] dup_rows_per_dst=[702, 3772, 4674, 1240, 42, 1122, 3366, 1318] combine_rows_per_src=[5966, 6620, 8094, 4170, 7086, 5566, 6518, 5280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48984 payload_rows_remote=43270 dup_rows=16552 dup_rows_remote=15582 dedup_ratio=0.252563 router_tokens_per_expert=[0, 0, 0, 107, 1116, 0, 1882, 180, 1003, 1947, 31, 0, 1, 3093, 0, 3442, 0, 1449, 0, 0, 494, 3993, 0, 0, 0, 0, 744, 215, 1131, 14, 0, 20, 0, 0, 0, 0, 0, 6, 0, 1, 0, 0, 712, 0, 0, 0, 2488, 0, 0, 2382, 729, 0, 1563, 0, 0, 0, 1824, 2, 1, 61, 503, 276, 352, 1006] payload_rows_per_dst=[5626, 6018, 5118, 6442, 7562, 5714, 6378, 6126] dup_rows_per_dst=[944, 1776, 1454, 3106, 3408, 970, 2970, 1924] combine_rows_per_src=[5626, 6018, 5118, 6442, 7562, 5714, 6378, 6126] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46262 payload_rows_remote=39936 dup_rows=19274 dup_rows_remote=16856 dedup_ratio=0.294098 router_tokens_per_expert=[0, 0, 0, 19, 1812, 0, 2630, 146, 1675, 2869, 101, 0, 1, 2141, 0, 3016, 1, 1127, 0, 0, 332, 3767, 0, 0, 0, 0, 409, 339, 1243, 79, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 912, 0, 0, 0, 2005, 0, 0, 2404, 713, 0, 1255, 0, 0, 0, 1544, 0, 0, 40, 565, 110, 532, 979] payload_rows_per_dst=[7340, 212, 2840, 7826, 7204, 7428, 6326, 7086] dup_rows_per_dst=[1874, 26, 80, 3852, 3110, 4144, 2418, 3770] combine_rows_per_src=[7340, 212, 2840, 7826, 7204, 7428, 6326, 7086] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50550 payload_rows_remote=45016 dup_rows=14986 dup_rows_remote=14524 dedup_ratio=0.228668 router_tokens_per_expert=[0, 0, 0, 102, 2235, 0, 1430, 159, 851, 1473, 15, 0, 0, 2643, 0, 3291, 0, 2951, 0, 0, 467, 3764, 0, 0, 0, 0, 725, 9, 543, 4, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 440, 0, 0, 0, 2558, 0, 0, 2469, 824, 0, 2335, 0, 0, 0, 2447, 6, 1, 55, 80, 183, 62, 643] payload_rows_per_dst=[5588, 4266, 6216, 7854, 7662, 5534, 7000, 6430] dup_rows_per_dst=[1092, 412, 620, 2236, 4212, 462, 4256, 1696] combine_rows_per_src=[5588, 4266, 6216, 7854, 7662, 5534, 7000, 6430] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50000 payload_rows_remote=42646 dup_rows=15536 dup_rows_remote=11306 dedup_ratio=0.237061 router_tokens_per_expert=[0, 0, 0, 59, 2649, 0, 1945, 162, 1019, 2389, 20, 0, 0, 1720, 0, 3538, 4, 2812, 0, 0, 413, 3658, 0, 0, 0, 0, 966, 25, 847, 48, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 432, 0, 0, 4, 2312, 0, 0, 2107, 758, 0, 1734, 0, 0, 2, 2201, 0, 0, 29, 17, 85, 126, 685] payload_rows_per_dst=[4256, 5286, 6048, 7778, 7644, 5110, 6524, 7354] dup_rows_per_dst=[76, 796, 410, 3312, 2874, 386, 3452, 4230] combine_rows_per_src=[4256, 5286, 6048, 7778, 7644, 5110, 6524, 7354] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48790 payload_rows_remote=41658 dup_rows=16746 dup_rows_remote=12330 dedup_ratio=0.255524 router_tokens_per_expert=[0, 0, 0, 26, 2679, 0, 1300, 112, 1787, 1608, 149, 0, 0, 2230, 0, 3063, 1, 2141, 0, 0, 597, 3152, 0, 0, 0, 0, 1288, 57, 689, 90, 0, 0, 0, 0, 1, 0, 6, 0, 0, 2, 0, 0, 764, 0, 1, 0, 2481, 0, 0, 2203, 941, 0, 1874, 0, 0, 2, 1858, 4, 0, 31, 35, 117, 97, 1382] payload_rows_per_dst=[7230, 7132, 7098, 3512, 6134, 7014, 4824, 5846] dup_rows_per_dst=[3046, 4416, 2642, 736, 10, 3884, 810, 1202] combine_rows_per_src=[7230, 7132, 7098, 3512, 6134, 7014, 4824, 5846] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=297314304 dispatch_meta_bytes=208592 combine_payload_bytes=297314304 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=297522896 ingress_bytes=627615744 link_bytes=627615744 token_bytes=594837200 weight_movement_bytes=330301440 total_nvlink_bytes=925138640 weight_to_token_ratio=0.555280 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=17536 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.091527004e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51368 payload_rows_remote=43874 dup_rows=14168 dup_rows_remote=11502 dedup_ratio=0.216187 router_tokens_per_expert=[0, 0, 0, 130, 2761, 0, 1265, 197, 1523, 1282, 75, 0, 0, 1601, 0, 3142, 0, 2875, 0, 0, 632, 3401, 0, 0, 0, 0, 683, 68, 888, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 851, 1, 0, 0, 2244, 0, 0, 2156, 939, 0, 2502, 0, 0, 0, 1939, 1, 1, 41, 33, 247, 106, 1140] payload_rows_per_dst=[6856, 6164, 6106, 7494, 6284, 6458, 6096, 5910] dup_rows_per_dst=[1850, 2798, 908, 2666, 4, 3112, 1724, 1106] combine_rows_per_src=[6856, 6164, 6106, 7494, 6284, 6458, 6096, 5910] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50882 payload_rows_remote=44178 dup_rows=14654 dup_rows_remote=12958 dedup_ratio=0.223602 router_tokens_per_expert=[0, 0, 0, 123, 2550, 1, 1753, 224, 2144, 1104, 70, 0, 0, 1409, 0, 3043, 0, 2306, 0, 3, 605, 3574, 0, 0, 1, 0, 1234, 14, 761, 42, 1, 0, 0, 0, 0, 1, 0, 5, 0, 2, 0, 3, 722, 2, 0, 0, 2325, 0, 0, 2335, 749, 0, 2252, 0, 0, 0, 1985, 1, 0, 49, 25, 186, 138, 1026] payload_rows_per_dst=[7148, 5764, 5360, 7760, 6704, 5410, 6872, 5864] dup_rows_per_dst=[2154, 1392, 468, 3494, 1696, 694, 3800, 956] combine_rows_per_src=[7148, 5764, 5360, 7760, 6704, 5410, 6872, 5864] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=312424448 dispatch_meta_bytes=220184 combine_payload_bytes=312424448 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=312644632 ingress_bytes=642725888 link_bytes=642725888 token_bytes=625069080 weight_movement_bytes=330301440 total_nvlink_bytes=955370520 weight_to_token_ratio=0.528424 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=15872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.117806029e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=322674688 dispatch_meta_bytes=238160 combine_payload_bytes=322674688 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=322912848 ingress_bytes=432775168 link_bytes=432775168 token_bytes=645587536 weight_movement_bytes=110100480 total_nvlink_bytes=755688016 weight_to_token_ratio=0.170543 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=18304 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.526671964e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=305686528 dispatch_meta_bytes=215808 combine_payload_bytes=305686528 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=305902336 ingress_bytes=415787008 link_bytes=415787008 token_bytes=611588864 weight_movement_bytes=110100480 total_nvlink_bytes=721689344 weight_to_token_ratio=0.180024 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=16512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.231219921e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50584 payload_rows_remote=44220 dup_rows=14952 dup_rows_remote=14934 dedup_ratio=0.228149 router_tokens_per_expert=[0, 0, 0, 112, 1541, 1, 2190, 225, 2171, 1406, 99, 0, 1, 1799, 0, 3177, 0, 946, 0, 2, 659, 3867, 0, 0, 1, 0, 1442, 183, 1346, 54, 1, 1, 0, 0, 0, 1, 0, 12, 0, 1, 0, 3, 1107, 2, 0, 0, 2373, 0, 0, 2226, 649, 0, 1373, 0, 0, 0, 1621, 0, 0, 58, 299, 188, 420, 1211] payload_rows_per_dst=[6654, 6894, 8004, 4638, 6364, 5854, 6102, 6074] dup_rows_per_dst=[1484, 4058, 2944, 1418, 18, 1116, 2394, 1520] combine_rows_per_src=[6654, 6894, 8004, 4638, 6364, 5854, 6102, 6074] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49642 payload_rows_remote=45178 dup_rows=15894 dup_rows_remote=14402 dedup_ratio=0.242523 router_tokens_per_expert=[0, 0, 0, 102, 1711, 0, 1722, 188, 1721, 1610, 96, 0, 8, 1686, 0, 3338, 0, 1501, 0, 0, 726, 3499, 0, 0, 0, 0, 810, 581, 1518, 67, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1140, 1, 0, 0, 2141, 0, 0, 2142, 827, 0, 1634, 0, 0, 0, 1448, 0, 1, 43, 396, 455, 299, 1354] payload_rows_per_dst=[6238, 6578, 7750, 4464, 6676, 5592, 6238, 6106] dup_rows_per_dst=[1208, 3664, 3702, 1492, 2, 972, 2968, 1886] combine_rows_per_src=[6238, 6578, 7750, 4464, 6676, 5592, 6238, 6106] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45858 payload_rows_remote=39188 dup_rows=19678 dup_rows_remote=16988 dedup_ratio=0.300262 router_tokens_per_expert=[0, 0, 0, 61, 1560, 0, 2311, 155, 1167, 2896, 33, 0, 1, 2339, 0, 3605, 4, 1320, 0, 0, 444, 3831, 0, 0, 0, 0, 969, 417, 1544, 90, 1, 4, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 643, 0, 0, 4, 2232, 0, 0, 2069, 524, 0, 1025, 0, 0, 2, 1653, 0, 0, 29, 300, 111, 416, 1004] payload_rows_per_dst=[6828, 68, 3410, 8034, 7818, 7414, 5616, 6670] dup_rows_per_dst=[1346, 0, 126, 5678, 4078, 4136, 1624, 2690] combine_rows_per_src=[6828, 68, 3410, 8034, 7818, 7414, 5616, 6670] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=295364608 dispatch_meta_bytes=211072 combine_payload_bytes=295364608 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=295575680 ingress_bytes=625666048 link_bytes=625666048 token_bytes=590940288 weight_movement_bytes=330301440 total_nvlink_bytes=921241728 weight_to_token_ratio=0.558942 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=17152 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.088136162e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46682 payload_rows_remote=38978 dup_rows=18854 dup_rows_remote=13452 dedup_ratio=0.287689 router_tokens_per_expert=[0, 0, 0, 43, 2563, 0, 1094, 176, 1681, 1585, 77, 0, 0, 1895, 0, 2780, 3, 2677, 0, 0, 442, 3668, 0, 0, 0, 0, 1313, 19, 840, 38, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 1304, 0, 0, 0, 2430, 0, 0, 2061, 719, 0, 1759, 0, 0, 0, 2326, 4, 0, 70, 29, 158, 120, 888] payload_rows_per_dst=[7704, 6794, 7644, 3680, 5566, 7022, 2194, 6078] dup_rows_per_dst=[5402, 3682, 3126, 740, 6, 4568, 218, 1112] combine_rows_per_src=[7704, 6794, 7644, 3680, 5566, 7022, 2194, 6078] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=298604544 dispatch_meta_bytes=215952 combine_payload_bytes=298604544 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=298820496 ingress_bytes=628905984 link_bytes=628905984 token_bytes=597425040 weight_movement_bytes=330301440 total_nvlink_bytes=927726480 weight_to_token_ratio=0.552875 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=12928 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.093770943e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=314488832 dispatch_meta_bytes=221504 combine_payload_bytes=314488832 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=314710336 ingress_bytes=424589312 link_bytes=424589312 token_bytes=629199168 weight_movement_bytes=110100480 total_nvlink_bytes=739299648 weight_to_token_ratio=0.174985 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=18688 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.384306465e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=316667904 dispatch_meta_bytes=228544 combine_payload_bytes=316667904 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=316896448 ingress_bytes=536868864 link_bytes=536868864 token_bytes=633564352 weight_movement_bytes=220200960 total_nvlink_bytes=853765312 weight_to_token_ratio=0.347559 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=25344 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.337032543e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279394304 dispatch_meta_bytes=209720 combine_payload_bytes=279394304 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279604024 ingress_bytes=499595264 link_bytes=499595264 token_bytes=558998328 weight_movement_bytes=220200960 total_nvlink_bytes=779199288 weight_to_token_ratio=0.393921 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=14208 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.688783334e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53986 payload_rows_remote=46728 dup_rows=11550 dup_rows_remote=7330 dedup_ratio=0.176239 router_tokens_per_expert=[0, 0, 0, 9, 1, 1310, 397, 0, 0, 188, 0, 0, 0, 1, 26, 0, 3871, 1, 0, 1496, 1, 1406, 4081, 0, 0, 0, 3435, 4041, 0, 3238, 1, 0, 2, 1, 0, 0, 0, 2728, 1736, 4, 1947, 0, 1225, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 1455, 0, 3, 133, 0, 0, 0, 19] payload_rows_per_dst=[7526, 8106, 7258, 6476, 6744, 5092, 8162, 4622] dup_rows_per_dst=[2778, 406, 4220, 2, 2198, 1252, 24, 670] combine_rows_per_src=[7526, 8106, 7258, 6476, 6744, 5092, 8162, 4622] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55272 payload_rows_remote=47120 dup_rows=10264 dup_rows_remote=10174 dedup_ratio=0.156616 router_tokens_per_expert=[0, 0, 0, 43, 0, 2878, 608, 0, 0, 170, 0, 0, 0, 0, 13, 0, 4033, 0, 0, 1639, 0, 1748, 4076, 0, 0, 0, 3029, 4040, 0, 2221, 0, 0, 0, 0, 0, 0, 0, 2259, 1702, 0, 2576, 0, 409, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 1240, 0, 16, 23, 0, 0, 0, 0] payload_rows_per_dst=[6142, 8100, 4318, 6822, 6212, 7364, 8152, 8162] dup_rows_per_dst=[916, 346, 378, 1314, 1710, 3048, 90, 2462] combine_rows_per_src=[6142, 8100, 4318, 6822, 6212, 7364, 8152, 8162] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55482 payload_rows_remote=47986 dup_rows=10054 dup_rows_remote=5954 dedup_ratio=0.153412 router_tokens_per_expert=[0, 0, 0, 33, 0, 2291, 648, 0, 0, 191, 0, 0, 0, 0, 26, 0, 4030, 0, 0, 1836, 0, 1297, 4095, 0, 0, 0, 3447, 4069, 0, 1942, 1, 0, 0, 0, 0, 0, 0, 2077, 1814, 0, 2781, 0, 1075, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 1041, 0, 1, 47, 0, 0, 0, 2] payload_rows_per_dst=[5244, 8176, 5398, 6894, 5930, 7496, 8190, 8154] dup_rows_per_dst=[700, 396, 868, 2, 1852, 4100, 48, 2088] combine_rows_per_src=[5244, 8176, 5398, 6894, 5930, 7496, 8190, 8154] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50840 payload_rows_remote=42722 dup_rows=14696 dup_rows_remote=11868 dedup_ratio=0.224243 router_tokens_per_expert=[0, 0, 0, 28, 0, 2007, 386, 0, 0, 149, 0, 0, 0, 0, 2, 0, 3984, 0, 0, 1721, 0, 2124, 4088, 0, 0, 0, 3362, 4003, 0, 2317, 0, 0, 0, 0, 0, 0, 0, 2155, 1714, 4, 2410, 0, 797, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 1412, 0, 0, 75, 0, 0, 0, 2] payload_rows_per_dst=[4396, 5370, 3108, 7812, 6412, 7448, 8176, 8118] dup_rows_per_dst=[446, 206, 0, 5086, 2398, 3600, 132, 2828] combine_rows_per_src=[4396, 5370, 3108, 7812, 6412, 7448, 8176, 8118] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=310159360 dispatch_meta_bytes=235408 combine_payload_bytes=310159360 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=310394768 ingress_bytes=420259840 link_bytes=420259840 token_bytes=620554128 weight_movement_bytes=110100480 total_nvlink_bytes=730654608 weight_to_token_ratio=0.177423 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=17792 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.309009826e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=286261248 dispatch_meta_bytes=227168 combine_payload_bytes=286261248 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=286488416 ingress_bytes=506462208 link_bytes=506462208 token_bytes=572749664 weight_movement_bytes=220200960 total_nvlink_bytes=792950624 weight_to_token_ratio=0.384463 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=16512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.808210784e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47658 payload_rows_remote=40070 dup_rows=17878 dup_rows_remote=13274 dedup_ratio=0.272797 router_tokens_per_expert=[0, 0, 0, 4, 1465, 0, 1759, 167, 1617, 1863, 89, 0, 1, 2260, 0, 2825, 3, 1272, 0, 0, 508, 3885, 0, 0, 0, 0, 1457, 328, 1341, 60, 0, 8, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 1837, 0, 0, 0, 2479, 0, 0, 2084, 598, 0, 877, 0, 0, 0, 1803, 0, 0, 73, 358, 251, 284, 1204] payload_rows_per_dst=[7588, 7274, 6120, 4862, 5658, 6432, 3732, 5992] dup_rows_per_dst=[4604, 5082, 1644, 1526, 8, 2200, 860, 1954] combine_rows_per_src=[7588, 7274, 6120, 4862, 5658, 6432, 3732, 5992] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50354 payload_rows_remote=42790 dup_rows=15182 dup_rows_remote=9252 dedup_ratio=0.231659 router_tokens_per_expert=[0, 0, 0, 14, 1702, 0, 1736, 115, 2006, 1962, 160, 0, 0, 2619, 0, 3239, 1, 782, 0, 0, 631, 3374, 0, 0, 0, 0, 1485, 420, 1376, 107, 0, 6, 0, 1, 1, 0, 7, 1, 0, 2, 0, 0, 1069, 0, 1, 0, 2368, 0, 0, 2157, 785, 0, 1069, 0, 0, 2, 1381, 2, 0, 31, 106, 199, 295, 1556] payload_rows_per_dst=[6224, 7564, 7364, 4924, 6488, 5984, 5972, 5834] dup_rows_per_dst=[910, 5930, 2212, 1864, 14, 892, 2054, 1306] combine_rows_per_src=[6224, 7564, 7364, 4924, 6488, 5984, 5972, 5834] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=316968960 dispatch_meta_bytes=236616 combine_payload_bytes=316968960 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=317205576 ingress_bytes=537169920 link_bytes=537169920 token_bytes=634174536 weight_movement_bytes=220200960 total_nvlink_bytes=854375496 weight_to_token_ratio=0.347225 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=26240 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.342268402e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=323835904 dispatch_meta_bytes=238320 combine_payload_bytes=323835904 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=324074224 ingress_bytes=433936384 link_bytes=433936384 token_bytes=647910128 weight_movement_bytes=110100480 total_nvlink_bytes=758010608 weight_to_token_ratio=0.169932 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=10496 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.546867420e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=280899584 dispatch_meta_bytes=224704 combine_payload_bytes=280899584 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281124288 ingress_bytes=391000064 link_bytes=391000064 token_bytes=562023872 weight_movement_bytes=110100480 total_nvlink_bytes=672124352 weight_to_token_ratio=0.195900 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=12416 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.800134198e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=287221760 dispatch_meta_bytes=213376 combine_payload_bytes=287221760 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=287435136 ingress_bytes=397322240 link_bytes=397322240 token_bytes=574656896 weight_movement_bytes=110100480 total_nvlink_bytes=684757376 weight_to_token_ratio=0.191593 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=16640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.910087236e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=306718720 dispatch_meta_bytes=208168 combine_payload_bytes=306718720 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=306926888 ingress_bytes=857221120 link_bytes=857221120 token_bytes=613645608 weight_movement_bytes=550502400 total_nvlink_bytes=1164148008 weight_to_token_ratio=0.897102 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=11392 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.490848516e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53706 payload_rows_remote=47470 dup_rows=11830 dup_rows_remote=10750 dedup_ratio=0.180511 router_tokens_per_expert=[0, 0, 0, 1, 1, 0, 411, 0, 0, 142, 0, 0, 0, 1, 0, 0, 2441, 1, 0, 2691, 1, 965, 3727, 0, 0, 0, 2369, 4007, 0, 2797, 1, 0, 1, 2757, 0, 0, 0, 3206, 2, 4, 1111, 0, 2366, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 3158, 0, 19, 581, 0, 0, 0, 0] payload_rows_per_dst=[5116, 8014, 6236, 7212, 7402, 5618, 7454, 6654] dup_rows_per_dst=[592, 286, 1080, 3122, 4538, 1336, 14, 862] combine_rows_per_src=[5116, 8014, 6236, 7212, 7402, 5618, 7454, 6654] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55130 payload_rows_remote=49136 dup_rows=10406 dup_rows_remote=8406 dedup_ratio=0.158783 router_tokens_per_expert=[0, 0, 0, 2, 0, 0, 837, 0, 0, 128, 0, 0, 0, 0, 0, 0, 3383, 0, 0, 3214, 0, 617, 3921, 0, 0, 0, 2386, 4058, 0, 1339, 0, 0, 0, 2735, 0, 0, 0, 2962, 1, 0, 1684, 0, 2313, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 2804, 0, 37, 285, 0, 0, 0, 0] payload_rows_per_dst=[6996, 8116, 6740, 6202, 7312, 5994, 7842, 5928] dup_rows_per_dst=[1448, 256, 922, 1248, 4084, 2000, 124, 324] combine_rows_per_src=[6996, 8116, 6740, 6202, 7312, 5994, 7842, 5928] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54282 payload_rows_remote=46966 dup_rows=11254 dup_rows_remote=10980 dedup_ratio=0.171722 router_tokens_per_expert=[0, 0, 0, 7, 0, 0, 747, 0, 0, 125, 0, 0, 0, 0, 0, 0, 3303, 0, 0, 3180, 0, 1425, 3645, 0, 0, 0, 1912, 4071, 0, 1683, 0, 0, 0, 3122, 0, 0, 0, 3172, 5, 0, 1551, 0, 1395, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 3106, 0, 11, 263, 0, 0, 0, 0] payload_rows_per_dst=[6804, 8142, 7004, 6034, 4792, 7608, 7316, 6582] dup_rows_per_dst=[1310, 250, 1996, 1156, 380, 4130, 274, 1758] combine_rows_per_src=[6804, 8142, 7004, 6034, 4792, 7608, 7316, 6582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=340264960 dispatch_meta_bytes=232880 combine_payload_bytes=340264960 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=340497840 ingress_bytes=890767360 link_bytes=890767360 token_bytes=680762800 weight_movement_bytes=550502400 total_nvlink_bytes=1231265200 weight_to_token_ratio=0.808655 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=19200 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.549190945e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=334946304 dispatch_meta_bytes=216232 combine_payload_bytes=334946304 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=335162536 ingress_bytes=885448704 link_bytes=885448704 token_bytes=670108840 weight_movement_bytes=550502400 total_nvlink_bytes=1220611240 weight_to_token_ratio=0.821512 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=11776 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.539940928e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50986 payload_rows_remote=42852 dup_rows=14550 dup_rows_remote=13960 dedup_ratio=0.222015 router_tokens_per_expert=[0, 0, 0, 102, 5, 2772, 677, 0, 0, 218, 0, 0, 0, 0, 82, 0, 4006, 0, 0, 1524, 0, 1394, 4062, 0, 0, 0, 3051, 4062, 0, 2117, 1, 0, 0, 0, 0, 0, 0, 2203, 1351, 5, 2455, 1, 1176, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 1380, 0, 87, 31, 0, 1, 0, 1] payload_rows_per_dst=[7892, 8134, 8146, 4236, 5768, 5748, 8130, 2932] dup_rows_per_dst=[5322, 590, 5702, 0, 1350, 1516, 2, 68] combine_rows_per_src=[7892, 8134, 8146, 4236, 5768, 5748, 8130, 2932] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=337756160 dispatch_meta_bytes=229176 combine_payload_bytes=337756160 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=337985336 ingress_bytes=557957120 link_bytes=557957120 token_bytes=675741496 weight_movement_bytes=220200960 total_nvlink_bytes=895942456 weight_to_token_ratio=0.325866 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.703791999e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53956 payload_rows_remote=46660 dup_rows=11580 dup_rows_remote=8316 dedup_ratio=0.176697 router_tokens_per_expert=[0, 0, 0, 75, 0, 2173, 565, 0, 0, 214, 0, 0, 0, 0, 67, 0, 3963, 0, 0, 1622, 0, 1529, 4080, 0, 0, 0, 3073, 3998, 0, 2825, 0, 0, 0, 0, 0, 0, 0, 2777, 1268, 0, 1957, 0, 986, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 1434, 0, 48, 69, 0, 0, 0, 23] payload_rows_per_dst=[5536, 8098, 5520, 7296, 6354, 4872, 8160, 8120] dup_rows_per_dst=[1326, 460, 782, 3264, 1736, 1014, 44, 2954] combine_rows_per_src=[5536, 8098, 5520, 7296, 6354, 4872, 8160, 8120] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=343963648 dispatch_meta_bytes=215760 combine_payload_bytes=343963648 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=344179408 ingress_bytes=454064128 link_bytes=454064128 token_bytes=688143056 weight_movement_bytes=110100480 total_nvlink_bytes=798243536 weight_to_token_ratio=0.159996 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=18816 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.896921993e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54322 payload_rows_remote=48704 dup_rows=11214 dup_rows_remote=10026 dedup_ratio=0.171112 router_tokens_per_expert=[0, 0, 0, 4, 1, 2061, 969, 0, 0, 170, 0, 0, 0, 0, 62, 0, 3878, 0, 0, 1664, 0, 1432, 4042, 0, 0, 0, 3536, 4070, 0, 2552, 0, 0, 0, 1, 0, 0, 0, 2198, 1203, 1, 2598, 0, 853, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 1391, 1, 1, 58, 0, 0, 0, 4] payload_rows_per_dst=[5260, 8174, 5470, 7808, 5618, 5818, 8084, 8090] dup_rows_per_dst=[810, 430, 722, 4062, 1188, 1390, 36, 2576] combine_rows_per_src=[5260, 8174, 5470, 7808, 5618, 5818, 8084, 8090] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54114 payload_rows_remote=46912 dup_rows=11422 dup_rows_remote=7796 dedup_ratio=0.174286 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1198, 0, 0, 123, 0, 0, 0, 0, 0, 0, 2845, 0, 0, 3142, 0, 752, 3332, 0, 0, 0, 2425, 4071, 0, 2233, 0, 0, 0, 2334, 0, 0, 0, 3075, 4, 1, 1891, 0, 1935, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 3063, 1, 15, 306, 0, 0, 0, 0] payload_rows_per_dst=[6378, 8148, 6784, 6858, 7202, 5770, 6666, 6308] dup_rows_per_dst=[1708, 240, 1004, 2458, 3626, 1882, 42, 462] combine_rows_per_src=[6378, 8148, 6784, 6858, 7202, 5770, 6666, 6308] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46936 payload_rows_remote=39474 dup_rows=18600 dup_rows_remote=12294 dedup_ratio=0.283813 router_tokens_per_expert=[0, 0, 0, 2, 0, 0, 532, 0, 0, 111, 0, 0, 0, 0, 0, 0, 2909, 0, 0, 3256, 0, 1574, 3853, 0, 0, 0, 2294, 3967, 0, 2025, 0, 0, 0, 2066, 0, 0, 0, 3059, 4, 4, 1517, 0, 1739, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 3256, 0, 2, 567, 0, 0, 0, 0] payload_rows_per_dst=[6080, 7084, 3148, 5708, 2206, 7164, 8084, 7462] dup_rows_per_dst=[806, 190, 0, 1622, 0, 3480, 6196, 6306] combine_rows_per_src=[6080, 7084, 3148, 5708, 2206, 7164, 8084, 7462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53652 payload_rows_remote=47182 dup_rows=11884 dup_rows_remote=10124 dedup_ratio=0.181335 router_tokens_per_expert=[0, 0, 0, 3, 0, 0, 565, 0, 0, 131, 0, 0, 0, 0, 0, 0, 2991, 0, 0, 2969, 0, 989, 3568, 0, 0, 0, 2001, 3982, 0, 2114, 0, 0, 0, 3317, 0, 0, 0, 3330, 0, 2, 1125, 0, 2023, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 3131, 0, 79, 392, 0, 0, 0, 5] payload_rows_per_dst=[6200, 7986, 6642, 6470, 7566, 4984, 7142, 6662] dup_rows_per_dst=[918, 240, 1274, 1760, 5732, 1312, 96, 552] combine_rows_per_src=[6200, 7986, 6642, 6470, 7566, 4984, 7142, 6662] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=352206848 dispatch_meta_bytes=230168 combine_payload_bytes=352206848 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=352437016 ingress_bytes=462307328 link_bytes=462307328 token_bytes=704643864 weight_movement_bytes=110100480 total_nvlink_bytes=814744344 weight_to_token_ratio=0.156250 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=16000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.040284799e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=336652288 dispatch_meta_bytes=231784 combine_payload_bytes=336652288 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=336884072 ingress_bytes=556853248 link_bytes=556853248 token_bytes=673536360 weight_movement_bytes=220200960 total_nvlink_bytes=893737320 weight_to_token_ratio=0.326933 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=29184 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.684593849e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53050 payload_rows_remote=45328 dup_rows=12486 dup_rows_remote=7614 dedup_ratio=0.190521 router_tokens_per_expert=[0, 0, 0, 1, 0, 0, 824, 0, 0, 106, 0, 0, 0, 0, 0, 0, 3315, 0, 0, 3064, 0, 697, 3354, 0, 0, 0, 2772, 4006, 0, 1659, 0, 0, 0, 2212, 0, 0, 0, 3141, 61, 1, 1218, 0, 1978, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 3446, 0, 4, 837, 0, 0, 0, 3] payload_rows_per_dst=[7722, 8014, 7094, 6878, 4520, 4992, 6708, 7122] dup_rows_per_dst=[4872, 210, 2396, 1984, 28, 1400, 138, 1458] combine_rows_per_src=[7722, 8014, 7094, 6878, 4520, 4992, 6708, 7122] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=336265216 dispatch_meta_bytes=218832 combine_payload_bytes=336265216 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=336484048 ingress_bytes=776667136 link_bytes=776667136 token_bytes=672749264 weight_movement_bytes=440401920 total_nvlink_bytes=1113151184 weight_to_token_ratio=0.654630 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=16256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.350751889e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54624 payload_rows_remote=46544 dup_rows=10912 dup_rows_remote=10632 dedup_ratio=0.166504 router_tokens_per_expert=[0, 0, 0, 62, 5, 0, 794, 0, 0, 140, 0, 0, 0, 0, 0, 0, 3432, 0, 0, 3099, 0, 765, 3717, 0, 0, 0, 1955, 4040, 0, 1456, 1, 0, 0, 2699, 0, 0, 0, 3113, 1, 5, 1691, 0, 2242, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 3138, 0, 114, 294, 0, 1, 0, 1] payload_rows_per_dst=[7276, 8080, 7636, 5886, 5622, 6066, 7440, 6618] dup_rows_per_dst=[1310, 280, 5580, 938, 526, 1800, 0, 478] combine_rows_per_src=[7276, 8080, 7636, 5886, 5622, 6066, 7440, 6618] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=282949632 dispatch_meta_bytes=207072 combine_payload_bytes=282949632 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=283156704 ingress_bytes=393050112 link_bytes=393050112 token_bytes=566106336 weight_movement_bytes=110100480 total_nvlink_bytes=676206816 weight_to_token_ratio=0.194487 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=25216 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.835787904e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=338200576 dispatch_meta_bytes=229224 combine_payload_bytes=338200576 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=338429800 ingress_bytes=558401536 link_bytes=558401536 token_bytes=676630376 weight_movement_bytes=220200960 total_nvlink_bytes=896831336 weight_to_token_ratio=0.325438 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=17024 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.711521124e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=306231296 dispatch_meta_bytes=218360 combine_payload_bytes=306231296 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=306449656 ingress_bytes=416331776 link_bytes=416331776 token_bytes=612680952 weight_movement_bytes=110100480 total_nvlink_bytes=722781432 weight_to_token_ratio=0.179703 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=20992 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.240694333e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51972 payload_rows_remote=44140 dup_rows=13564 dup_rows_remote=10154 dedup_ratio=0.206970 router_tokens_per_expert=[0, 0, 0, 4, 0, 1373, 607, 0, 0, 182, 0, 0, 0, 0, 47, 0, 4046, 0, 0, 1440, 0, 1337, 4039, 0, 0, 0, 3637, 4043, 0, 2231, 0, 0, 0, 0, 0, 0, 0, 2279, 2754, 0, 2000, 0, 889, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 1714, 0, 2, 122, 0, 0, 0, 8] payload_rows_per_dst=[7832, 8130, 8138, 4462, 7056, 4698, 8078, 3578] dup_rows_per_dst=[3410, 414, 5508, 0, 3010, 1080, 28, 114] combine_rows_per_src=[7832, 8130, 8138, 4462, 7056, 4698, 8078, 3578] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=307163136 dispatch_meta_bytes=227248 combine_payload_bytes=307163136 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=307390384 ingress_bytes=417263616 link_bytes=417263616 token_bytes=614553520 weight_movement_bytes=110100480 total_nvlink_bytes=724654000 weight_to_token_ratio=0.179155 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=30976 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.256900563e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=334458880 dispatch_meta_bytes=219904 combine_payload_bytes=334458880 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=334678784 ingress_bytes=774860800 link_bytes=774860800 token_bytes=669137664 weight_movement_bytes=440401920 total_nvlink_bytes=1109539584 weight_to_token_ratio=0.658163 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=15872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.347610374e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=349110272 dispatch_meta_bytes=234920 combine_payload_bytes=349110272 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=349345192 ingress_bytes=459210752 link_bytes=459210752 token_bytes=698455464 weight_movement_bytes=110100480 total_nvlink_bytes=808555944 weight_to_token_ratio=0.157634 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=18688 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.986430249e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59172 payload_rows_remote=51510 dup_rows=6364 dup_rows_remote=5698 dedup_ratio=0.097107 router_tokens_per_expert=[0, 0, 3892, 0, 0, 2, 0, 0, 0, 0, 0, 3945, 5, 0, 0, 0, 0, 0, 131, 0, 0, 0, 315, 0, 0, 3842, 0, 1430, 0, 0, 4, 0, 0, 0, 0, 0, 4073, 3082, 0, 1577, 3, 0, 0, 0, 0, 0, 160, 0, 0, 0, 261, 1, 0, 0, 0, 0, 2242, 0, 0, 0, 0, 3718, 4085, 0] payload_rows_per_dst=[7784, 7900, 7662, 7862, 6664, 8152, 8178, 4970] dup_rows_per_dst=[4, 0, 666, 2690, 1844, 320, 516, 324] combine_rows_per_src=[7784, 7900, 7662, 7862, 6664, 8152, 8178, 4970] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=316395520 dispatch_meta_bytes=217176 combine_payload_bytes=316395520 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=316612696 ingress_bytes=426496000 link_bytes=426496000 token_bytes=633008216 weight_movement_bytes=110100480 total_nvlink_bytes=743108696 weight_to_token_ratio=0.173932 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=16896 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.417466905e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59562 payload_rows_remote=51390 dup_rows=5974 dup_rows_remote=5834 dedup_ratio=0.091156 router_tokens_per_expert=[0, 0, 3809, 0, 0, 3, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 308, 0, 0, 0, 591, 0, 0, 3978, 0, 843, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 2967, 0, 2670, 0, 0, 0, 0, 0, 0, 172, 0, 0, 0, 73, 0, 0, 0, 0, 0, 1375, 0, 0, 0, 0, 3743, 4083, 0] payload_rows_per_dst=[7618, 8188, 7778, 6752, 5902, 8118, 8172, 7034] dup_rows_per_dst=[6, 0, 1506, 1280, 1048, 344, 140, 1650] combine_rows_per_src=[7618, 8188, 7778, 6752, 5902, 8118, 8172, 7034] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59938 payload_rows_remote=51868 dup_rows=5598 dup_rows_remote=5302 dedup_ratio=0.085419 router_tokens_per_expert=[0, 0, 3759, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 444, 0, 0, 0, 417, 0, 0, 3878, 0, 894, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 2680, 0, 2566, 0, 0, 0, 0, 0, 0, 148, 0, 0, 0, 72, 0, 0, 0, 0, 0, 2118, 0, 0, 0, 0, 3593, 4095, 0] payload_rows_per_dst=[7518, 8138, 7658, 7922, 5132, 8070, 8192, 7308] dup_rows_per_dst=[0, 0, 1250, 1622, 0, 296, 142, 2288] combine_rows_per_src=[7518, 8138, 7658, 7922, 5132, 8070, 8192, 7308] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53438 payload_rows_remote=45622 dup_rows=12098 dup_rows_remote=5138 dedup_ratio=0.184601 router_tokens_per_expert=[0, 0, 3911, 0, 0, 2, 0, 0, 4, 0, 0, 4008, 4, 0, 0, 0, 0, 0, 335, 0, 0, 0, 387, 0, 0, 3848, 0, 1128, 0, 1, 3, 0, 0, 0, 0, 0, 4059, 2808, 0, 2298, 0, 0, 0, 0, 0, 0, 158, 0, 0, 0, 83, 0, 0, 0, 0, 0, 1839, 0, 0, 0, 0, 3804, 4088, 0] payload_rows_per_dst=[7822, 8024, 4952, 2256, 8130, 8118, 6320, 7816] dup_rows_per_dst=[4, 8, 490, 8, 4162, 316, 150, 6960] combine_rows_per_src=[7822, 8024, 4952, 2256, 8130, 8118, 6320, 7816] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=369223680 dispatch_meta_bytes=228832 combine_payload_bytes=369223680 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=369452512 ingress_bytes=479324160 link_bytes=479324160 token_bytes=738676192 weight_movement_bytes=110100480 total_nvlink_bytes=848776672 weight_to_token_ratio=0.149051 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=27648 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.336235495e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=368363520 dispatch_meta_bytes=228896 combine_payload_bytes=368363520 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=368592416 ingress_bytes=478464000 link_bytes=478464000 token_bytes=736955936 weight_movement_bytes=110100480 total_nvlink_bytes=847056416 weight_to_token_ratio=0.149399 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.321275898e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=371789824 dispatch_meta_bytes=228680 combine_payload_bytes=371789824 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=372018504 ingress_bytes=481890304 link_bytes=481890304 token_bytes=743808328 weight_movement_bytes=110100480 total_nvlink_bytes=853908808 weight_to_token_ratio=0.148023 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.380864960e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59196 payload_rows_remote=51162 dup_rows=6340 dup_rows_remote=4260 dedup_ratio=0.096741 router_tokens_per_expert=[0, 0, 3822, 0, 0, 1, 0, 0, 1, 0, 0, 4056, 5, 0, 0, 0, 0, 0, 341, 0, 0, 0, 352, 0, 0, 3938, 0, 1117, 0, 0, 2, 0, 0, 0, 0, 0, 4067, 3538, 0, 1806, 2, 0, 0, 0, 0, 0, 134, 0, 0, 0, 119, 0, 0, 0, 0, 0, 1866, 0, 0, 0, 0, 3518, 4083, 0] payload_rows_per_dst=[7644, 8118, 7594, 8034, 3612, 8134, 8182, 7878] dup_rows_per_dst=[2, 6, 828, 2080, 0, 272, 222, 2930] combine_rows_per_src=[7644, 8118, 7594, 8034, 3612, 8134, 8182, 7878] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58854 payload_rows_remote=50796 dup_rows=6682 dup_rows_remote=6682 dedup_ratio=0.101959 router_tokens_per_expert=[0, 0, 3893, 0, 0, 5, 0, 0, 0, 0, 0, 4024, 5, 0, 0, 0, 0, 0, 202, 0, 0, 0, 226, 0, 0, 3926, 0, 1268, 0, 2, 2, 0, 0, 0, 0, 0, 4055, 3233, 0, 2094, 5, 0, 0, 0, 0, 0, 133, 0, 0, 0, 212, 0, 0, 0, 0, 0, 1770, 0, 0, 0, 0, 3707, 4006, 0] payload_rows_per_dst=[7786, 8058, 7620, 7944, 7636, 8120, 8150, 3540] dup_rows_per_dst=[10, 0, 650, 2452, 3018, 266, 286, 0] combine_rows_per_src=[7786, 8058, 7620, 7944, 7636, 8120, 8150, 3540] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60292 payload_rows_remote=55636 dup_rows=5244 dup_rows_remote=5244 dedup_ratio=0.080017 router_tokens_per_expert=[0, 0, 3888, 0, 0, 5, 0, 0, 4, 0, 0, 3959, 0, 0, 0, 0, 0, 0, 180, 0, 0, 0, 399, 0, 0, 3894, 0, 1309, 0, 0, 1, 0, 0, 0, 0, 0, 4077, 3534, 0, 2328, 1, 0, 0, 0, 0, 0, 165, 0, 0, 0, 178, 1, 0, 0, 0, 0, 1017, 0, 0, 0, 0, 3735, 4093, 0] payload_rows_per_dst=[7778, 7924, 7910, 8028, 4656, 8154, 8190, 7652] dup_rows_per_dst=[8, 2, 718, 2380, 0, 332, 354, 1450] combine_rows_per_src=[7778, 7924, 7910, 8028, 4656, 8154, 8190, 7652] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=327018496 dispatch_meta_bytes=203040 combine_payload_bytes=327018496 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=327221536 ingress_bytes=657319936 link_bytes=657319936 token_bytes=654240032 weight_movement_bytes=330301440 total_nvlink_bytes=984541472 weight_to_token_ratio=0.504863 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=16000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.143187479e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53290 payload_rows_remote=45396 dup_rows=12246 dup_rows_remote=8588 dedup_ratio=0.186859 router_tokens_per_expert=[0, 0, 3786, 0, 0, 0, 0, 0, 2, 0, 0, 4006, 2, 0, 0, 0, 0, 0, 223, 0, 0, 0, 349, 0, 0, 3886, 0, 792, 0, 0, 0, 0, 0, 0, 0, 0, 3999, 3049, 0, 2926, 4, 0, 0, 0, 0, 0, 150, 0, 0, 0, 102, 0, 0, 0, 0, 0, 1868, 0, 0, 0, 0, 3555, 4069, 0] payload_rows_per_dst=[7894, 8030, 7514, 3694, 7960, 6276, 8186, 3736] dup_rows_per_dst=[3658, 1140, 740, 532, 5772, 248, 156, 0] combine_rows_per_src=[7894, 8030, 7514, 3694, 7960, 6276, 8186, 3736] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=366729216 dispatch_meta_bytes=221688 combine_payload_bytes=366729216 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=366950904 ingress_bytes=697030656 link_bytes=697030656 token_bytes=733680120 weight_movement_bytes=330301440 total_nvlink_bytes=1063981560 weight_to_token_ratio=0.450198 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=24064 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.212250953e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=364105728 dispatch_meta_bytes=229912 combine_payload_bytes=364105728 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=364335640 ingress_bytes=474206208 link_bytes=474206208 token_bytes=728441368 weight_movement_bytes=110100480 total_nvlink_bytes=838541848 weight_to_token_ratio=0.151145 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.247225892e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=398798848 dispatch_meta_bytes=243520 combine_payload_bytes=398798848 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=399042368 ingress_bytes=729100288 link_bytes=729100288 token_bytes=797841216 weight_movement_bytes=330301440 total_nvlink_bytes=1128142656 weight_to_token_ratio=0.413994 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.055147 gemm_active_groups=5 gemm_peak_group_rows=18304 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.268025317e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55280 payload_rows_remote=47104 dup_rows=10256 dup_rows_remote=10254 dedup_ratio=0.156494 router_tokens_per_expert=[0, 11, 0, 0, 1671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 44, 3035, 0, 17, 0, 0, 4088, 4075, 0, 4085, 0, 0, 0, 201, 0, 4049, 1735, 0, 0, 1482, 0, 0, 0, 2281, 0, 0, 0, 418, 3978, 0, 0, 0, 377, 0, 0, 362, 858, 0] payload_rows_per_dst=[8124, 8150, 8176, 6136, 8170, 5370, 8170, 2984] dup_rows_per_dst=[3338, 0, 2, 56, 402, 1064, 5184, 210] combine_rows_per_src=[8124, 8150, 8176, 6136, 8170, 5370, 8170, 2984] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=325398528 dispatch_meta_bytes=215936 combine_payload_bytes=325398528 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=325614464 ingress_bytes=435499008 link_bytes=435499008 token_bytes=651012992 weight_movement_bytes=110100480 total_nvlink_bytes=761113472 weight_to_token_ratio=0.169122 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=30848 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.574044022e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53912 payload_rows_remote=49546 dup_rows=11624 dup_rows_remote=11252 dedup_ratio=0.177368 router_tokens_per_expert=[0, 0, 0, 0, 1782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 21, 2936, 0, 0, 0, 0, 4095, 4095, 0, 4096, 0, 0, 0, 199, 0, 4064, 995, 0, 0, 1048, 0, 0, 0, 1927, 0, 0, 0, 442, 4026, 0, 0, 0, 167, 0, 0, 1443, 1429, 0] payload_rows_per_dst=[3564, 8190, 8190, 7090, 8192, 6162, 4366, 8158] dup_rows_per_dst=[0, 0, 6, 2820, 398, 2056, 372, 5972] combine_rows_per_src=[3564, 8190, 8190, 7090, 8192, 6162, 4366, 8158] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55376 payload_rows_remote=50338 dup_rows=10160 dup_rows_remote=9610 dedup_ratio=0.155029 router_tokens_per_expert=[0, 1, 0, 0, 2671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 2493, 0, 3, 0, 0, 4095, 4094, 0, 4095, 0, 0, 0, 127, 0, 4087, 852, 0, 0, 1942, 0, 0, 0, 1955, 0, 0, 0, 150, 4041, 0, 0, 0, 101, 0, 0, 1299, 718, 0] payload_rows_per_dst=[5344, 8188, 8190, 8192, 8190, 5038, 4116, 8118] dup_rows_per_dst=[0, 0, 2, 5060, 254, 550, 94, 4200] combine_rows_per_src=[5344, 8188, 8190, 8192, 8190, 5038, 4116, 8118] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53132 payload_rows_remote=44974 dup_rows=12404 dup_rows_remote=7198 dedup_ratio=0.189270 router_tokens_per_expert=[0, 18, 0, 0, 1887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4, 0, 0, 0, 0, 0, 0, 20, 2525, 0, 5, 0, 0, 4090, 4095, 0, 4088, 0, 0, 0, 157, 0, 4081, 807, 0, 0, 1864, 0, 0, 0, 2191, 0, 0, 0, 252, 4054, 0, 0, 0, 182, 0, 0, 1112, 1334, 0] payload_rows_per_dst=[3810, 8190, 8190, 5078, 5702, 8190, 5814, 8158] dup_rows_per_dst=[0, 0, 2, 22, 246, 5314, 1614, 5206] combine_rows_per_src=[3810, 8190, 8190, 5078, 5702, 8190, 5814, 8158] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=337641472 dispatch_meta_bytes=229432 combine_payload_bytes=337641472 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=337870904 ingress_bytes=557842432 link_bytes=557842432 token_bytes=675512376 weight_movement_bytes=220200960 total_nvlink_bytes=895713336 weight_to_token_ratio=0.325976 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.701797386e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=355145728 dispatch_meta_bytes=243192 combine_payload_bytes=355145728 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=355388920 ingress_bytes=685447168 link_bytes=685447168 token_bytes=710534648 weight_movement_bytes=330301440 total_nvlink_bytes=1040836088 weight_to_token_ratio=0.464863 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=15872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.192105362e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=360822784 dispatch_meta_bytes=239792 combine_payload_bytes=360822784 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=361062576 ingress_bytes=691124224 link_bytes=691124224 token_bytes=721885360 weight_movement_bytes=330301440 total_nvlink_bytes=1052186800 weight_to_token_ratio=0.457554 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=12672 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.201978696e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55512 payload_rows_remote=48780 dup_rows=10024 dup_rows_remote=9870 dedup_ratio=0.152954 router_tokens_per_expert=[0, 9, 0, 0, 1842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 3351, 0, 11, 0, 0, 4090, 4096, 0, 4088, 0, 0, 0, 164, 2, 4028, 1232, 0, 0, 1266, 0, 0, 0, 2055, 0, 0, 0, 541, 4030, 0, 0, 0, 155, 0, 0, 549, 1174, 0] payload_rows_per_dst=[8108, 8192, 8184, 6732, 8176, 4436, 8188, 3496] dup_rows_per_dst=[3650, 0, 4, 154, 328, 564, 5064, 260] combine_rows_per_src=[8108, 8192, 8184, 6732, 8176, 4436, 8188, 3496] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55468 payload_rows_remote=47278 dup_rows=10068 dup_rows_remote=10068 dedup_ratio=0.153625 router_tokens_per_expert=[0, 19, 0, 0, 2082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4, 0, 0, 0, 0, 0, 0, 54, 3188, 0, 9, 0, 0, 4056, 4095, 0, 4080, 0, 0, 0, 112, 4, 3982, 908, 0, 0, 1546, 0, 0, 0, 1747, 0, 0, 0, 651, 3953, 0, 0, 0, 204, 0, 0, 881, 1185, 0] payload_rows_per_dst=[8030, 8190, 8120, 6408, 8160, 5394, 7096, 4070] dup_rows_per_dst=[4136, 0, 16, 94, 224, 1892, 3236, 470] combine_rows_per_src=[8030, 8190, 8120, 6408, 8160, 5394, 7096, 4070] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=322373632 dispatch_meta_bytes=208688 combine_payload_bytes=322373632 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=322582320 ingress_bytes=432474112 link_bytes=432474112 token_bytes=644955952 weight_movement_bytes=110100480 total_nvlink_bytes=755056432 weight_to_token_ratio=0.170710 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=14592 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.521436105e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54540 payload_rows_remote=46374 dup_rows=10996 dup_rows_remote=10476 dedup_ratio=0.167786 router_tokens_per_expert=[0, 7, 0, 0, 1988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 210, 2397, 0, 15, 0, 0, 4092, 4094, 0, 4083, 0, 0, 0, 260, 0, 4058, 1345, 0, 0, 1659, 0, 0, 0, 2192, 0, 0, 0, 345, 3994, 0, 0, 0, 335, 0, 0, 797, 897, 0] payload_rows_per_dst=[4710, 8188, 8184, 7838, 8166, 5056, 6128, 6270] dup_rows_per_dst=[174, 0, 0, 4628, 520, 952, 2068, 2654] combine_rows_per_src=[4710, 8188, 8184, 7838, 8166, 5056, 6128, 6270] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54078 payload_rows_remote=45984 dup_rows=11458 dup_rows_remote=7004 dedup_ratio=0.174835 router_tokens_per_expert=[0, 14, 0, 0, 2227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 44, 2567, 0, 3, 0, 0, 4095, 4094, 0, 4072, 0, 0, 0, 201, 0, 4033, 959, 0, 0, 1762, 0, 0, 0, 1482, 0, 0, 0, 531, 4028, 0, 0, 0, 120, 0, 0, 1272, 1263, 0] payload_rows_per_dst=[8094, 8188, 8190, 5174, 8012, 8190, 3634, 4596] dup_rows_per_dst=[4454, 60, 80, 54, 396, 5308, 392, 714] combine_rows_per_src=[8094, 8188, 8190, 5174, 8012, 8190, 3634, 4596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=324911104 dispatch_meta_bytes=211768 combine_payload_bytes=324911104 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=325122872 ingress_bytes=545112064 link_bytes=545112064 token_bytes=650033976 weight_movement_bytes=220200960 total_nvlink_bytes=870234936 weight_to_token_ratio=0.338753 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=24704 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.480395348e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49978 payload_rows_remote=41814 dup_rows=15558 dup_rows_remote=12996 dedup_ratio=0.237396 router_tokens_per_expert=[0, 0, 3407, 0, 0, 59, 0, 0, 3, 0, 0, 1965, 7, 0, 0, 0, 0, 0, 339, 0, 0, 0, 987, 0, 0, 422, 0, 2498, 0, 0, 4, 0, 0, 0, 0, 0, 4037, 3317, 0, 160, 13, 0, 0, 0, 0, 0, 179, 0, 0, 0, 3660, 1, 0, 0, 0, 0, 3676, 0, 0, 0, 0, 3947, 4087, 0] payload_rows_per_dst=[7552, 8054, 8164, 7276, 3422, 8190, 7320, 0] dup_rows_per_dst=[3290, 3790, 2562, 3818, 1728, 368, 2, 0] combine_rows_per_src=[7552, 8054, 8164, 7276, 3422, 8190, 7320, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=333627392 dispatch_meta_bytes=228704 combine_payload_bytes=333627392 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=333856096 ingress_bytes=443727872 link_bytes=443727872 token_bytes=667483488 weight_movement_bytes=110100480 total_nvlink_bytes=777583968 weight_to_token_ratio=0.164949 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.717157501e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52686 payload_rows_remote=44922 dup_rows=12850 dup_rows_remote=8846 dedup_ratio=0.196075 router_tokens_per_expert=[0, 0, 2249, 0, 0, 349, 0, 0, 0, 0, 0, 2749, 0, 0, 0, 0, 0, 0, 828, 0, 0, 0, 2186, 0, 0, 580, 0, 2005, 0, 0, 0, 0, 0, 0, 0, 0, 3966, 3092, 0, 392, 0, 0, 0, 0, 0, 0, 191, 0, 0, 3, 2789, 0, 0, 0, 0, 0, 3398, 0, 0, 0, 0, 3895, 4096, 0] payload_rows_per_dst=[5030, 5498, 5284, 4876, 7958, 8192, 7764, 8084] dup_rows_per_dst=[166, 0, 744, 294, 758, 382, 4004, 6502] combine_rows_per_src=[5030, 5498, 5284, 4876, 7958, 8192, 7764, 8084] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51696 payload_rows_remote=43504 dup_rows=13840 dup_rows_remote=13446 dedup_ratio=0.211182 router_tokens_per_expert=[0, 0, 2109, 0, 0, 311, 0, 0, 0, 0, 0, 2503, 0, 0, 0, 0, 0, 0, 800, 0, 0, 0, 1930, 0, 0, 493, 0, 2550, 0, 0, 0, 0, 0, 0, 0, 0, 3973, 3025, 0, 566, 1, 0, 0, 0, 0, 0, 196, 0, 0, 0, 2605, 0, 0, 0, 0, 0, 3670, 0, 0, 0, 0, 3940, 4096, 0] payload_rows_per_dst=[4714, 5006, 4972, 5746, 8010, 8192, 6994, 8062] dup_rows_per_dst=[126, 0, 488, 340, 2280, 394, 3054, 7158] combine_rows_per_src=[4714, 5006, 4972, 5746, 8010, 8192, 6994, 8062] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=299722752 dispatch_meta_bytes=219240 combine_payload_bytes=299722752 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=299941992 ingress_bytes=409823232 link_bytes=409823232 token_bytes=599664744 weight_movement_bytes=110100480 total_nvlink_bytes=709765224 weight_to_token_ratio=0.183603 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=14976 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.127500048e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50128 payload_rows_remote=41972 dup_rows=15408 dup_rows_remote=8058 dedup_ratio=0.235107 router_tokens_per_expert=[0, 0, 2957, 0, 0, 534, 0, 0, 6, 0, 0, 1920, 0, 0, 0, 0, 0, 0, 415, 0, 0, 0, 1572, 0, 0, 564, 0, 2487, 0, 0, 0, 0, 0, 0, 0, 0, 4028, 3586, 0, 139, 2, 0, 0, 1, 0, 0, 189, 0, 0, 1, 3025, 1, 0, 0, 0, 0, 3303, 0, 0, 0, 0, 3943, 4095, 0] payload_rows_per_dst=[6454, 3852, 3762, 5626, 8156, 8192, 6052, 8034] dup_rows_per_dst=[528, 0, 212, 476, 7350, 382, 2, 6458] combine_rows_per_src=[6454, 3852, 3762, 5626, 8156, 8192, 6052, 8034] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49250 payload_rows_remote=41058 dup_rows=16286 dup_rows_remote=6164 dedup_ratio=0.248505 router_tokens_per_expert=[0, 0, 2788, 0, 0, 319, 0, 0, 4, 0, 0, 2032, 4, 0, 0, 0, 0, 0, 805, 0, 0, 0, 1868, 0, 0, 415, 0, 2231, 0, 1, 4, 0, 0, 0, 0, 0, 4016, 3049, 0, 272, 0, 0, 0, 0, 0, 0, 219, 0, 0, 1, 3011, 0, 0, 0, 0, 0, 3617, 0, 0, 0, 0, 4021, 4091, 0] payload_rows_per_dst=[5968, 4072, 4812, 5094, 8044, 5314, 7754, 8192] dup_rows_per_dst=[246, 8, 534, 208, 532, 268, 4368, 10122] combine_rows_per_src=[5968, 4072, 4812, 5094, 8044, 5314, 7754, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49680 payload_rows_remote=43866 dup_rows=15856 dup_rows_remote=15414 dedup_ratio=0.241943 router_tokens_per_expert=[0, 0, 2625, 0, 0, 243, 0, 0, 6, 0, 0, 2312, 6, 0, 0, 0, 0, 0, 698, 0, 0, 0, 1266, 0, 0, 587, 0, 2539, 0, 0, 2, 0, 0, 0, 0, 0, 4040, 3610, 0, 117, 4, 0, 0, 0, 0, 0, 152, 0, 0, 0, 3010, 0, 0, 0, 0, 0, 3565, 0, 0, 0, 0, 3895, 4091, 0] payload_rows_per_dst=[5672, 7264, 6938, 5814, 8172, 8188, 6020, 1612] dup_rows_per_dst=[64, 3580, 2744, 442, 7408, 306, 0, 1312] combine_rows_per_src=[5672, 7264, 6938, 5814, 8172, 8188, 6020, 1612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=322000896 dispatch_meta_bytes=215072 combine_payload_bytes=322000896 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=322215968 ingress_bytes=432101376 link_bytes=432101376 token_bytes=644216864 weight_movement_bytes=110100480 total_nvlink_bytes=754317344 weight_to_token_ratio=0.170906 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=24320 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.514953613e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=311836672 dispatch_meta_bytes=227800 combine_payload_bytes=311836672 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=312064472 ingress_bytes=421937152 link_bytes=421937152 token_bytes=623901144 weight_movement_bytes=110100480 total_nvlink_bytes=734001624 weight_to_token_ratio=0.176471 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=31232 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.338181041e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47138 payload_rows_remote=39224 dup_rows=18398 dup_rows_remote=14132 dedup_ratio=0.280731 router_tokens_per_expert=[0, 0, 2441, 0, 0, 69, 0, 0, 4, 0, 0, 2286, 2, 0, 0, 0, 0, 0, 795, 0, 0, 0, 1710, 0, 0, 486, 0, 2007, 0, 0, 0, 0, 0, 0, 0, 0, 3943, 3580, 0, 717, 11, 0, 0, 0, 0, 0, 209, 0, 0, 1, 2766, 0, 0, 0, 0, 0, 3694, 0, 0, 0, 0, 3955, 4092, 0] payload_rows_per_dst=[7914, 7994, 8114, 7958, 1434, 8192, 5532, 0] dup_rows_per_dst=[4266, 4500, 4782, 4416, 0, 432, 2, 0] combine_rows_per_src=[7914, 7994, 8114, 7958, 1434, 8192, 5532, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=294303744 dispatch_meta_bytes=188888 combine_payload_bytes=294303744 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=294492632 ingress_bytes=734705664 link_bytes=734705664 token_bytes=588796376 weight_movement_bytes=440401920 total_nvlink_bytes=1029198296 weight_to_token_ratio=0.747970 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=16640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.277773988e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=300855296 dispatch_meta_bytes=200120 combine_payload_bytes=300855296 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=301055416 ingress_bytes=741257216 link_bytes=741257216 token_bytes=601910712 weight_movement_bytes=440401920 total_nvlink_bytes=1042312632 weight_to_token_ratio=0.731673 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.055147 gemm_active_groups=5 gemm_peak_group_rows=17152 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.289168214e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47270 payload_rows_remote=39260 dup_rows=18266 dup_rows_remote=14358 dedup_ratio=0.278717 router_tokens_per_expert=[0, 0, 3072, 0, 0, 297, 0, 0, 0, 0, 0, 2059, 5, 0, 0, 0, 0, 0, 636, 0, 0, 0, 1060, 0, 0, 495, 0, 2535, 0, 2, 3, 0, 0, 0, 0, 0, 4016, 3404, 0, 142, 23, 0, 0, 0, 0, 0, 175, 0, 0, 0, 3334, 0, 0, 0, 0, 0, 3541, 0, 0, 0, 0, 3895, 4074, 0] payload_rows_per_dst=[8016, 8010, 8150, 7970, 284, 8172, 6668, 0] dup_rows_per_dst=[5530, 3908, 3274, 5182, 0, 372, 0, 0] combine_rows_per_src=[8016, 8010, 8150, 7970, 284, 8172, 6668, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=314431488 dispatch_meta_bytes=237120 combine_payload_bytes=314431488 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=314668608 ingress_bytes=424531968 link_bytes=424531968 token_bytes=629100096 weight_movement_bytes=110100480 total_nvlink_bytes=739200576 weight_to_token_ratio=0.175013 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=18944 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.383309159e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=281157632 dispatch_meta_bytes=213424 combine_payload_bytes=281157632 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281371056 ingress_bytes=391258112 link_bytes=391258112 token_bytes=562528688 weight_movement_bytes=110100480 total_nvlink_bytes=672629168 weight_to_token_ratio=0.195724 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=21760 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.804622077e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56706 payload_rows_remote=49172 dup_rows=8830 dup_rows_remote=8830 dedup_ratio=0.134735 router_tokens_per_expert=[0, 11, 0, 0, 617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3767, 0, 0, 0, 0, 0, 0, 7, 2574, 0, 9, 0, 0, 4079, 2567, 0, 4085, 0, 0, 0, 0, 0, 4067, 223, 0, 0, 768, 0, 0, 0, 2655, 0, 0, 0, 207, 3925, 0, 0, 0, 432, 0, 0, 0, 2775, 0] payload_rows_per_dst=[8180, 8170, 7534, 8072, 5134, 8190, 5458, 5968] dup_rows_per_dst=[1234, 0, 0, 4958, 0, 1926, 266, 446] combine_rows_per_src=[8180, 8170, 7534, 8072, 5134, 8190, 5458, 5968] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=281415680 dispatch_meta_bytes=214472 combine_payload_bytes=281415680 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281630152 ingress_bytes=391516160 link_bytes=391516160 token_bytes=563045832 weight_movement_bytes=110100480 total_nvlink_bytes=673146312 weight_to_token_ratio=0.195544 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=17920 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.809109956e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53546 payload_rows_remote=45402 dup_rows=11990 dup_rows_remote=7882 dedup_ratio=0.182953 router_tokens_per_expert=[0, 0, 0, 0, 680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3913, 0, 0, 0, 0, 0, 0, 0, 2393, 0, 0, 0, 0, 4078, 2844, 0, 4096, 0, 0, 0, 0, 0, 4067, 164, 0, 0, 459, 0, 0, 0, 1963, 0, 0, 0, 165, 3998, 0, 0, 0, 179, 0, 0, 42, 3727, 0] payload_rows_per_dst=[4728, 8192, 7826, 4786, 6886, 5186, 8144, 7798] dup_rows_per_dst=[582, 0, 0, 0, 3008, 674, 4108, 3618] combine_rows_per_src=[4728, 8192, 7826, 4786, 6886, 5186, 8144, 7798] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56076 payload_rows_remote=47884 dup_rows=9460 dup_rows_remote=7344 dedup_ratio=0.144348 router_tokens_per_expert=[0, 1, 0, 0, 1387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3817, 0, 0, 0, 0, 0, 0, 6, 1949, 0, 2, 0, 0, 4094, 3050, 0, 4093, 0, 0, 0, 0, 0, 4089, 104, 0, 0, 961, 0, 0, 0, 2177, 0, 0, 0, 90, 4024, 0, 0, 0, 104, 0, 0, 10, 2810, 0] payload_rows_per_dst=[8190, 8186, 7634, 3910, 6100, 8192, 8174, 5690] dup_rows_per_dst=[2774, 0, 0, 4, 0, 2116, 4408, 158] combine_rows_per_src=[8190, 8186, 7634, 3910, 6100, 8192, 8174, 5690] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47802 payload_rows_remote=39610 dup_rows=17734 dup_rows_remote=10494 dedup_ratio=0.270599 router_tokens_per_expert=[0, 18, 0, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3888, 0, 0, 0, 0, 0, 0, 3, 1923, 0, 3, 0, 0, 4078, 3069, 0, 4087, 0, 0, 0, 0, 1, 4085, 78, 0, 0, 994, 0, 0, 0, 2109, 0, 0, 0, 113, 4048, 0, 0, 0, 177, 0, 0, 8, 3446, 0] payload_rows_per_dst=[1316, 8174, 7776, 3852, 8182, 2126, 8184, 8192] dup_rows_per_dst=[0, 0, 0, 6, 6112, 20, 4356, 7240] combine_rows_per_src=[1316, 8174, 7776, 3852, 8182, 2126, 8184, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=352464896 dispatch_meta_bytes=232008 combine_payload_bytes=352464896 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=352696904 ingress_bytes=462565376 link_bytes=462565376 token_bytes=705161800 weight_movement_bytes=110100480 total_nvlink_bytes=815262280 weight_to_token_ratio=0.156135 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=30464 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.044772678e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56374 payload_rows_remote=48960 dup_rows=9162 dup_rows_remote=7642 dedup_ratio=0.139801 router_tokens_per_expert=[0, 4, 0, 0, 697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3828, 0, 0, 0, 0, 0, 0, 32, 1829, 0, 3, 0, 0, 4089, 3615, 0, 4078, 0, 0, 0, 0, 0, 4070, 214, 0, 0, 958, 0, 0, 0, 2282, 0, 0, 0, 141, 3988, 0, 0, 0, 322, 0, 0, 1, 2617, 0] payload_rows_per_dst=[8186, 8156, 7656, 3714, 7414, 8192, 7470, 5586] dup_rows_per_dst=[1394, 0, 0, 14, 1520, 2292, 3648, 294] combine_rows_per_src=[8186, 8156, 7656, 3714, 7414, 8192, 7470, 5586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=325441536 dispatch_meta_bytes=213136 combine_payload_bytes=325441536 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=325654672 ingress_bytes=655742976 link_bytes=655742976 token_bytes=651096208 weight_movement_bytes=330301440 total_nvlink_bytes=981397648 weight_to_token_ratio=0.507301 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=16128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.140444886e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57342 payload_rows_remote=49998 dup_rows=8194 dup_rows_remote=5052 dedup_ratio=0.125031 router_tokens_per_expert=[0, 9, 0, 0, 579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3656, 0, 0, 0, 0, 0, 0, 5, 3108, 0, 7, 0, 0, 4076, 2791, 0, 4089, 0, 0, 0, 0, 0, 4039, 143, 0, 0, 460, 0, 0, 0, 2178, 0, 0, 0, 167, 4016, 0, 0, 0, 156, 0, 0, 1, 3288, 0] payload_rows_per_dst=[8174, 8178, 7312, 7344, 6954, 8164, 4574, 6642] dup_rows_per_dst=[1154, 0, 0, 3142, 2414, 1120, 116, 248] combine_rows_per_src=[8174, 8178, 7312, 7344, 6954, 8164, 4574, 6642] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=343232512 dispatch_meta_bytes=220912 combine_payload_bytes=343232512 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=343453424 ingress_bytes=673533952 link_bytes=673533952 token_bytes=686685936 weight_movement_bytes=330301440 total_nvlink_bytes=1016987376 weight_to_token_ratio=0.481008 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=24320 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.171386320e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53666 payload_rows_remote=45480 dup_rows=11870 dup_rows_remote=10210 dedup_ratio=0.181122 router_tokens_per_expert=[0, 14, 0, 0, 820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3936, 0, 0, 0, 0, 0, 0, 4, 2132, 0, 0, 0, 0, 4089, 3130, 0, 4070, 0, 0, 0, 0, 0, 4045, 123, 0, 0, 975, 0, 0, 0, 1699, 0, 0, 0, 277, 4031, 0, 0, 0, 122, 0, 0, 16, 3285, 0] payload_rows_per_dst=[8186, 8168, 8066, 8156, 6082, 6278, 2048, 6682] dup_rows_per_dst=[1660, 150, 4474, 4178, 0, 1228, 16, 164] combine_rows_per_src=[8186, 8168, 8066, 8156, 6082, 6278, 2048, 6682] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=350945280 dispatch_meta_bytes=226408 combine_payload_bytes=350945280 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=351171688 ingress_bytes=791347200 link_bytes=791347200 token_bytes=702116968 weight_movement_bytes=440401920 total_nvlink_bytes=1142518888 weight_to_token_ratio=0.627249 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=23936 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.376282935e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=283924480 dispatch_meta_bytes=200416 combine_payload_bytes=283924480 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=284124896 ingress_bytes=394024960 link_bytes=394024960 token_bytes=568049376 weight_movement_bytes=110100480 total_nvlink_bytes=678149856 weight_to_token_ratio=0.193822 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=25216 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.852742114e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56292 payload_rows_remote=48130 dup_rows=9244 dup_rows_remote=9244 dedup_ratio=0.141052 router_tokens_per_expert=[0, 11, 0, 0, 868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3628, 0, 0, 0, 0, 0, 0, 1, 2750, 0, 3, 0, 0, 4010, 2865, 0, 4081, 0, 0, 0, 0, 4, 3992, 175, 0, 0, 745, 0, 0, 0, 1926, 0, 0, 0, 357, 3946, 0, 0, 0, 225, 0, 0, 10, 3170, 0] payload_rows_per_dst=[8036, 8162, 7256, 8162, 5730, 8182, 4264, 6500] dup_rows_per_dst=[1742, 0, 2, 5238, 0, 1650, 302, 310] combine_rows_per_src=[8036, 8162, 7256, 8162, 5730, 8182, 4264, 6500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=358385664 dispatch_meta_bytes=220200 combine_payload_bytes=358385664 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=358605864 ingress_bytes=468486144 link_bytes=468486144 token_bytes=716991528 weight_movement_bytes=110100480 total_nvlink_bytes=827092008 weight_to_token_ratio=0.153559 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=18688 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.147744571e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 2, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=349655040 dispatch_meta_bytes=234600 combine_payload_bytes=349655040 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=349889640 ingress_bytes=459755520 link_bytes=459755520 token_bytes=699544680 weight_movement_bytes=110100480 total_nvlink_bytes=809645160 weight_to_token_ratio=0.157389 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=22528 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.995904661e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=338888704 dispatch_meta_bytes=229384 combine_payload_bytes=338888704 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=339118088 ingress_bytes=559089664 link_bytes=559089664 token_bytes=678006792 weight_movement_bytes=220200960 total_nvlink_bytes=898207752 weight_to_token_ratio=0.324777 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.723488801e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56926 payload_rows_remote=48736 dup_rows=8610 dup_rows_remote=8604 dedup_ratio=0.131378 router_tokens_per_expert=[0, 1794, 0, 1, 0, 167, 4081, 4091, 0, 2691, 0, 0, 0, 0, 0, 4093, 0, 0, 2, 1, 0, 0, 4095, 0, 0, 0, 0, 1753, 0, 1, 0, 0, 0, 1, 0, 0, 2, 1, 0, 0, 0, 0, 425, 0, 822, 16, 0, 0, 4096, 0, 0, 132, 31, 0, 0, 143, 0, 0, 0, 3553, 776, 0, 0, 0] payload_rows_per_dst=[7122, 6024, 8190, 3506, 8182, 8192, 8192, 7518] dup_rows_per_dst=[3144, 1178, 6, 2, 8, 2520, 612, 1140] combine_rows_per_src=[7122, 6024, 8190, 3506, 8182, 8192, 8192, 7518] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=332408832 dispatch_meta_bytes=227400 combine_payload_bytes=332408832 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=332636232 ingress_bytes=882911232 link_bytes=882911232 token_bytes=665045064 weight_movement_bytes=550502400 total_nvlink_bytes=1215547464 weight_to_token_ratio=0.827767 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=31360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.535527846e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55374 payload_rows_remote=47182 dup_rows=10162 dup_rows_remote=8980 dedup_ratio=0.155060 router_tokens_per_expert=[0, 2052, 0, 0, 0, 75, 4082, 4096, 0, 1418, 0, 0, 0, 1, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 401, 0, 175, 5, 0, 0, 4096, 0, 0, 433, 67, 0, 0, 91, 0, 0, 0, 3328, 1011, 0, 0, 0] payload_rows_per_dst=[4242, 2838, 8192, 8182, 8192, 8190, 8192, 7346] dup_rows_per_dst=[12, 0, 0, 6478, 0, 1158, 1182, 1332] combine_rows_per_src=[4242, 2838, 8192, 8182, 8192, 8190, 8192, 7346] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57260 payload_rows_remote=49068 dup_rows=8276 dup_rows_remote=7170 dedup_ratio=0.126282 router_tokens_per_expert=[0, 2280, 0, 0, 0, 65, 4094, 4094, 0, 2042, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 379, 0, 174, 7, 0, 0, 4096, 0, 0, 92, 122, 0, 0, 89, 0, 0, 0, 3518, 1016, 0, 0, 0] payload_rows_per_dst=[4684, 4084, 8192, 8192, 8188, 8192, 8192, 7536] dup_rows_per_dst=[6, 0, 0, 5026, 0, 1106, 606, 1532] combine_rows_per_src=[4684, 4084, 8192, 8192, 8188, 8192, 8192, 7536] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=329613312 dispatch_meta_bytes=211952 combine_payload_bytes=329613312 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=329825264 ingress_bytes=439713792 link_bytes=439713792 token_bytes=659438576 weight_movement_bytes=110100480 total_nvlink_bytes=769539056 weight_to_token_ratio=0.166961 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=16640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.647346048e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56754 payload_rows_remote=49424 dup_rows=8782 dup_rows_remote=7284 dedup_ratio=0.134003 router_tokens_per_expert=[0, 2317, 0, 0, 0, 71, 4088, 4096, 0, 1873, 0, 0, 0, 0, 0, 4090, 0, 0, 4, 1, 0, 0, 4096, 0, 0, 0, 0, 2647, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 396, 0, 309, 3, 0, 0, 4096, 0, 0, 105, 84, 0, 0, 90, 0, 0, 0, 3288, 1112, 0, 0, 0] payload_rows_per_dst=[4750, 3746, 8164, 8188, 8192, 8192, 8192, 7330] dup_rows_per_dst=[26, 0, 10, 5282, 4, 1404, 558, 1498] combine_rows_per_src=[4750, 3746, 8164, 8188, 8192, 8192, 8192, 7330] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=338200576 dispatch_meta_bytes=224648 combine_payload_bytes=338200576 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=338425224 ingress_bytes=558401536 link_bytes=558401536 token_bytes=676625800 weight_movement_bytes=220200960 total_nvlink_bytes=896826760 weight_to_token_ratio=0.325440 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=27648 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.711521124e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=349339648 dispatch_meta_bytes=229360 combine_payload_bytes=349339648 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=349569008 ingress_bytes=459440128 link_bytes=459440128 token_bytes=698908656 weight_movement_bytes=110100480 total_nvlink_bytes=809009136 weight_to_token_ratio=0.157532 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.990419475e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=351719424 dispatch_meta_bytes=224952 combine_payload_bytes=351719424 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=351944376 ingress_bytes=461819904 link_bytes=461819904 token_bytes=703663800 weight_movement_bytes=110100480 total_nvlink_bytes=813764280 weight_to_token_ratio=0.156467 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=25472 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.031807694e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57360 payload_rows_remote=52322 dup_rows=8176 dup_rows_remote=8176 dedup_ratio=0.124756 router_tokens_per_expert=[0, 1801, 0, 0, 0, 124, 4087, 4096, 0, 1851, 0, 0, 0, 0, 0, 4092, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 2519, 0, 0, 0, 0, 0, 4, 0, 0, 1, 0, 0, 0, 0, 0, 555, 0, 482, 10, 0, 0, 4096, 0, 0, 109, 118, 0, 0, 107, 0, 0, 0, 3492, 1126, 0, 0, 0] payload_rows_per_dst=[3846, 8174, 8192, 5038, 8192, 8192, 8192, 7534] dup_rows_per_dst=[4, 3702, 4, 0, 10, 2086, 668, 1702] combine_rows_per_src=[3846, 8174, 8192, 5038, 8192, 8192, 8192, 7534] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=354271232 dispatch_meta_bytes=226832 combine_payload_bytes=354271232 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=354498064 ingress_bytes=574472192 link_bytes=574472192 token_bytes=708769296 weight_movement_bytes=220200960 total_nvlink_bytes=928970256 weight_to_token_ratio=0.310681 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=25600 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.991016263e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54222 payload_rows_remote=46732 dup_rows=11314 dup_rows_remote=8518 dedup_ratio=0.172638 router_tokens_per_expert=[0, 1878, 0, 0, 0, 127, 4087, 4096, 0, 1861, 0, 0, 0, 0, 0, 4094, 0, 0, 4, 1, 0, 0, 4090, 0, 0, 0, 0, 2452, 0, 0, 0, 0, 0, 2, 0, 0, 12, 1, 0, 0, 0, 0, 649, 0, 675, 29, 0, 0, 4096, 0, 0, 47, 117, 0, 0, 263, 0, 0, 0, 3298, 889, 0, 0, 0] payload_rows_per_dst=[8192, 7490, 8188, 4904, 8192, 3716, 6320, 7220] dup_rows_per_dst=[6062, 2796, 2, 0, 30, 614, 656, 1154] combine_rows_per_src=[8192, 7490, 8188, 4904, 8192, 3716, 6320, 7220] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56902 payload_rows_remote=48720 dup_rows=8634 dup_rows_remote=8626 dedup_ratio=0.131744 router_tokens_per_expert=[0, 1522, 0, 0, 0, 58, 4074, 4091, 0, 2396, 0, 0, 0, 0, 0, 4090, 0, 1, 4, 0, 0, 0, 4095, 0, 0, 0, 0, 2694, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 0, 0, 588, 0, 568, 4, 0, 0, 4096, 0, 0, 69, 110, 0, 0, 72, 0, 0, 0, 3587, 645, 0, 0, 0] payload_rows_per_dst=[3154, 7804, 8192, 5588, 8182, 8190, 8192, 7600] dup_rows_per_dst=[6, 4444, 8, 492, 8, 2310, 502, 864] combine_rows_per_src=[3154, 7804, 8192, 5588, 8182, 8190, 8192, 7600] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=375044096 dispatch_meta_bytes=241992 combine_payload_bytes=375044096 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=375286088 ingress_bytes=485144576 link_bytes=485144576 token_bytes=750330184 weight_movement_bytes=110100480 total_nvlink_bytes=860430664 weight_to_token_ratio=0.146736 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=20224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.437462102e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47684 payload_rows_remote=39492 dup_rows=17852 dup_rows_remote=5758 dedup_ratio=0.272400 router_tokens_per_expert=[0, 1830, 0, 0, 0, 115, 4078, 4093, 0, 2535, 0, 0, 0, 0, 0, 4093, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 2329, 0, 0, 0, 0, 0, 8, 0, 0, 1, 0, 0, 0, 0, 0, 408, 0, 323, 6, 0, 0, 4096, 0, 0, 204, 80, 0, 0, 92, 0, 0, 0, 3715, 664, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 4658, 8154, 1376, 5068, 3854] dup_rows_per_dst=[12094, 5066, 4, 0, 2, 98, 212, 376] combine_rows_per_src=[8192, 8190, 8192, 4658, 8154, 1376, 5068, 3854] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=326000640 dispatch_meta_bytes=222760 combine_payload_bytes=326000640 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=326223400 ingress_bytes=436101120 link_bytes=436101120 token_bytes=652224040 weight_movement_bytes=110100480 total_nvlink_bytes=762324520 weight_to_token_ratio=0.168808 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=26496 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.584515740e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61528 payload_rows_remote=53338 dup_rows=4008 dup_rows_remote=3642 dedup_ratio=0.061157 router_tokens_per_expert=[0, 3924, 0, 0, 0, 184, 3226, 4096, 0, 0, 0, 0, 0, 0, 0, 4017, 0, 0, 0, 183, 0, 0, 4095, 0, 0, 0, 0, 330, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3327, 0, 453, 38, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 2083, 2643, 0, 0, 0] payload_rows_per_dst=[6494, 8034, 8190, 7908, 8192, 6974, 8192, 7544] dup_rows_per_dst=[326, 0, 366, 602, 2, 662, 142, 1908] combine_rows_per_src=[6494, 8034, 8190, 7908, 8192, 6974, 8192, 7544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60614 payload_rows_remote=52422 dup_rows=4922 dup_rows_remote=4734 dedup_ratio=0.075104 router_tokens_per_expert=[0, 4015, 0, 1, 0, 127, 2742, 4096, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 210, 0, 0, 4096, 0, 0, 0, 0, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3404, 0, 91, 6, 0, 0, 4096, 0, 0, 8, 11, 0, 0, 75, 0, 0, 0, 1453, 3196, 0, 0, 0] payload_rows_per_dst=[5554, 8170, 8192, 8102, 8192, 6862, 8192, 7350] dup_rows_per_dst=[186, 0, 420, 2040, 0, 140, 188, 1948] combine_rows_per_src=[5554, 8170, 8192, 8102, 8192, 6862, 8192, 7350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=344995840 dispatch_meta_bytes=229496 combine_payload_bytes=344995840 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=345225336 ingress_bytes=565196800 link_bytes=565196800 token_bytes=690221176 weight_movement_bytes=220200960 total_nvlink_bytes=910422136 weight_to_token_ratio=0.319030 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.829701941e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61660 payload_rows_remote=55046 dup_rows=3876 dup_rows_remote=3784 dedup_ratio=0.059143 router_tokens_per_expert=[0, 3999, 0, 0, 0, 74, 3324, 4096, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 0, 0, 258, 0, 0, 4096, 0, 0, 0, 0, 370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3277, 0, 69, 7, 0, 0, 4096, 0, 0, 0, 76, 0, 0, 88, 0, 0, 0, 1641, 3263, 0, 0, 0] payload_rows_per_dst=[6702, 8068, 8192, 8018, 8192, 6614, 8192, 7682] dup_rows_per_dst=[94, 0, 516, 720, 0, 92, 328, 2126] combine_rows_per_src=[6702, 8068, 8192, 8018, 8192, 6614, 8192, 7682] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=382326784 dispatch_meta_bytes=227920 combine_payload_bytes=382326784 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=382554704 ingress_bytes=602527744 link_bytes=602527744 token_bytes=764881488 weight_movement_bytes=220200960 total_nvlink_bytes=985082448 weight_to_token_ratio=0.287889 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=29568 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.047894846e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61142 payload_rows_remote=52950 dup_rows=4394 dup_rows_remote=4394 dedup_ratio=0.067047 router_tokens_per_expert=[0, 3804, 0, 0, 0, 119, 2977, 4096, 0, 0, 0, 0, 0, 0, 0, 3973, 0, 1, 3, 138, 0, 0, 4094, 0, 0, 0, 0, 590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3551, 0, 404, 8, 0, 0, 4096, 0, 0, 0, 31, 0, 0, 60, 0, 0, 0, 2338, 2485, 0, 0, 0] payload_rows_per_dst=[6034, 7946, 8192, 7748, 8192, 7322, 8192, 7516] dup_rows_per_dst=[158, 0, 280, 1040, 0, 604, 182, 2130] combine_rows_per_src=[6034, 7946, 8192, 7748, 8192, 7322, 8192, 7516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=375760896 dispatch_meta_bytes=228624 combine_payload_bytes=375760896 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=375989520 ingress_bytes=595961856 link_bytes=595961856 token_bytes=751750416 weight_movement_bytes=220200960 total_nvlink_bytes=971951376 weight_to_token_ratio=0.292918 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.036475686e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55756 payload_rows_remote=48118 dup_rows=9780 dup_rows_remote=7422 dedup_ratio=0.149231 router_tokens_per_expert=[0, 3988, 0, 0, 0, 142, 3305, 4096, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 0, 4, 67, 0, 0, 4096, 0, 0, 0, 0, 683, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3352, 0, 196, 3, 0, 0, 4096, 0, 0, 0, 9, 0, 0, 90, 0, 0, 0, 1303, 3292, 0, 0, 0] payload_rows_per_dst=[6648, 8088, 1860, 8064, 8192, 7872, 7394, 7638] dup_rows_per_dst=[246, 0, 50, 1278, 4, 5654, 190, 2358] combine_rows_per_src=[6648, 8088, 1860, 8064, 8192, 7872, 7394, 7638] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61654 payload_rows_remote=53802 dup_rows=3882 dup_rows_remote=3066 dedup_ratio=0.059235 router_tokens_per_expert=[0, 3894, 0, 0, 0, 124, 3211, 4096, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 0, 243, 0, 0, 4096, 0, 0, 0, 0, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3447, 0, 310, 17, 0, 0, 4096, 0, 0, 0, 15, 0, 0, 65, 0, 0, 0, 1857, 2803, 0, 0, 0] payload_rows_per_dst=[6486, 8108, 8192, 7852, 8192, 7112, 8192, 7520] dup_rows_per_dst=[184, 0, 486, 816, 0, 436, 160, 1800] combine_rows_per_src=[6486, 8108, 8192, 7852, 8192, 7112, 8192, 7520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=394569728 dispatch_meta_bytes=235320 combine_payload_bytes=394569728 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=394805048 ingress_bytes=504670208 link_bytes=504670208 token_bytes=789374776 weight_movement_bytes=110100480 total_nvlink_bytes=899475256 weight_to_token_ratio=0.139478 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=27520 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.777044957e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60306 payload_rows_remote=52230 dup_rows=5230 dup_rows_remote=1982 dedup_ratio=0.079803 router_tokens_per_expert=[0, 0, 0, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 452, 0, 0, 0, 0, 0, 0, 0, 1798, 0, 4096, 0, 0, 0, 0, 358, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4059, 1571, 4089, 0, 0, 3902, 4096, 0, 0, 132, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8106, 8184, 3596, 7838, 8192, 8192, 8076, 8122] dup_rows_per_dst=[0, 898, 0, 692, 132, 0, 3248, 260] combine_rows_per_src=[8106, 8184, 3596, 7838, 8192, 8192, 8076, 8122] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58110 payload_rows_remote=49964 dup_rows=7426 dup_rows_remote=3904 dedup_ratio=0.113312 router_tokens_per_expert=[0, 0, 0, 0, 0, 4037, 0, 1, 0, 0, 0, 2, 0, 0, 0, 143, 0, 0, 0, 0, 0, 0, 0, 1832, 0, 4096, 0, 0, 0, 0, 962, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4059, 515, 4046, 0, 0, 4002, 4096, 2, 0, 803, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8074, 8104, 8146, 8192, 8192, 8192, 1034, 8176] dup_rows_per_dst=[2, 278, 3522, 1924, 152, 0, 0, 1548] combine_rows_per_src=[8074, 8104, 8146, 8192, 8192, 8192, 1034, 8176] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=334974976 dispatch_meta_bytes=221000 combine_payload_bytes=334974976 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=335195976 ingress_bytes=555175936 link_bytes=555175936 token_bytes=670170952 weight_movement_bytes=220200960 total_nvlink_bytes=890371912 weight_to_token_ratio=0.328574 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=16768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.655422635e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=349224960 dispatch_meta_bytes=229384 combine_payload_bytes=349224960 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=349454344 ingress_bytes=459325440 link_bytes=459325440 token_bytes=698679304 weight_movement_bytes=110100480 total_nvlink_bytes=808779784 weight_to_token_ratio=0.157584 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.988424862e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61768 payload_rows_remote=53576 dup_rows=3768 dup_rows_remote=3768 dedup_ratio=0.057495 router_tokens_per_expert=[0, 0, 0, 0, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 0, 0, 0, 0, 0, 0, 0, 2364, 0, 4096, 0, 0, 0, 0, 569, 0, 0, 0, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4052, 542, 4090, 0, 0, 3960, 4096, 0, 0, 437, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8086, 8180, 4728, 8192, 8192, 8192, 8014, 8184] dup_rows_per_dst=[0, 600, 0, 1138, 246, 0, 990, 794] combine_rows_per_src=[8086, 8180, 4728, 8192, 8192, 8192, 8014, 8184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=283078656 dispatch_meta_bytes=181000 combine_payload_bytes=283078656 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=283259656 ingress_bytes=723480576 link_bytes=723480576 token_bytes=566338312 weight_movement_bytes=440401920 total_nvlink_bytes=1006740232 weight_to_token_ratio=0.777630 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=15488 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.258251714e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49314 payload_rows_remote=41162 dup_rows=16222 dup_rows_remote=15730 dedup_ratio=0.247528 router_tokens_per_expert=[0, 0, 0, 0, 0, 4063, 0, 0, 0, 0, 0, 2, 0, 0, 0, 286, 0, 0, 0, 0, 0, 0, 0, 1953, 0, 4096, 0, 0, 0, 0, 780, 0, 0, 0, 0, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4068, 781, 4086, 0, 0, 4037, 4096, 0, 0, 254, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8126, 4614, 3906, 1560, 6572, 8192, 8192, 8152] dup_rows_per_dst=[0, 278, 0, 0, 292, 0, 15160, 492] combine_rows_per_src=[8126, 4614, 3906, 1560, 6572, 8192, 8192, 8152] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=374384640 dispatch_meta_bytes=216848 combine_payload_bytes=374384640 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=374601488 ingress_bytes=924887040 link_bytes=924887040 token_bytes=748986128 weight_movement_bytes=550502400 total_nvlink_bytes=1299488528 weight_to_token_ratio=0.734997 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=16512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.608530680e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=358141952 dispatch_meta_bytes=215472 combine_payload_bytes=358141952 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=358357424 ingress_bytes=468242432 link_bytes=468242432 token_bytes=716499376 weight_movement_bytes=110100480 total_nvlink_bytes=826599856 weight_to_token_ratio=0.153664 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=17664 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.143506019e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=384032768 dispatch_meta_bytes=229376 combine_payload_bytes=384032768 prefetch_bytes=0 reduce_bytes=0 egress_bytes=384262144 ingress_bytes=384032768 link_bytes=384262144 token_bytes=768294912 weight_movement_bytes=0 total_nvlink_bytes=768294912 weight_to_token_ratio=0.000000 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.682950687e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53822 payload_rows_remote=45682 dup_rows=11714 dup_rows_remote=1652 dedup_ratio=0.178741 router_tokens_per_expert=[0, 3934, 0, 0, 0, 120, 3229, 4096, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 3, 159, 0, 0, 4096, 0, 0, 0, 0, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3396, 0, 157, 6, 0, 0, 4095, 0, 0, 1, 23, 0, 0, 71, 0, 0, 0, 2077, 2903, 0, 0, 0] payload_rows_per_dst=[8140, 8166, 8192, 2714, 8182, 6926, 7348, 4154] dup_rows_per_dst=[10062, 834, 324, 118, 0, 192, 184, 0] combine_rows_per_src=[8140, 8166, 8192, 2714, 8182, 6926, 7348, 4154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=379545600 dispatch_meta_bytes=229376 combine_payload_bytes=379545600 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=379774976 ingress_bytes=489646080 link_bytes=489646080 token_bytes=759320576 weight_movement_bytes=110100480 total_nvlink_bytes=869421056 weight_to_token_ratio=0.144999 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.515750661e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58820 payload_rows_remote=50698 dup_rows=6716 dup_rows_remote=6716 dedup_ratio=0.102478 router_tokens_per_expert=[0, 3928, 0, 0, 0, 108, 3291, 4096, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 3, 318, 0, 0, 4083, 0, 0, 0, 0, 330, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 3649, 0, 361, 27, 0, 0, 4096, 0, 0, 0, 10, 0, 0, 124, 0, 0, 0, 1434, 2847, 0, 0, 0] payload_rows_per_dst=[7078, 8122, 8180, 7914, 8192, 7502, 8192, 3640] dup_rows_per_dst=[1144, 0, 3904, 602, 4, 572, 268, 222] combine_rows_per_src=[7078, 8122, 8180, 7914, 8192, 7502, 8192, 3640] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=344909824 dispatch_meta_bytes=222160 combine_payload_bytes=344909824 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=345131984 ingress_bytes=675211264 link_bytes=675211264 token_bytes=690041808 weight_movement_bytes=330301440 total_nvlink_bytes=1020343248 weight_to_token_ratio=0.478669 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=18304 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.174303441e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=385652736 dispatch_meta_bytes=227472 combine_payload_bytes=385652736 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=385880208 ingress_bytes=495753216 link_bytes=495753216 token_bytes=771532944 weight_movement_bytes=110100480 total_nvlink_bytes=881633424 weight_to_token_ratio=0.142704 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=28672 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.621963800e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54926 payload_rows_remote=46744 dup_rows=10610 dup_rows_remote=9698 dedup_ratio=0.161896 router_tokens_per_expert=[0, 4078, 0, 0, 0, 3877, 0, 0, 1213, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 3458, 0, 4096, 0, 0, 0, 0, 571, 0, 0, 0, 0, 2665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 456, 4090, 0, 0, 0, 4096, 0, 0, 66, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5692, 8168, 7166, 1142, 8192, 8192, 8182, 8192] dup_rows_per_dst=[0, 2424, 1812, 0, 5330, 0, 912, 132] combine_rows_per_src=[5692, 8168, 7166, 1142, 8192, 8192, 8182, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=327448576 dispatch_meta_bytes=189336 combine_payload_bytes=327448576 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=327637912 ingress_bytes=657750016 link_bytes=657750016 token_bytes=655086488 weight_movement_bytes=330301440 total_nvlink_bytes=985387928 weight_to_token_ratio=0.504210 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=25344 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.143935459e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60360 payload_rows_remote=53370 dup_rows=5176 dup_rows_remote=5176 dedup_ratio=0.078979 router_tokens_per_expert=[0, 4059, 0, 0, 0, 3830, 0, 1, 407, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3495, 0, 4096, 0, 0, 0, 0, 1739, 0, 0, 0, 0, 2465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 12, 213, 4000, 0, 0, 0, 4096, 4, 0, 253, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[7660, 8138, 6990, 8192, 4930, 8190, 8068, 8192] dup_rows_per_dst=[2, 798, 0, 3478, 0, 0, 390, 508] combine_rows_per_src=[7660, 8138, 6990, 8192, 4930, 8190, 8068, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=363403264 dispatch_meta_bytes=229656 combine_payload_bytes=363403264 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=363632920 ingress_bytes=473503744 link_bytes=473503744 token_bytes=727036184 weight_movement_bytes=110100480 total_nvlink_bytes=837136664 weight_to_token_ratio=0.151437 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.235008888e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61428 payload_rows_remote=53236 dup_rows=4108 dup_rows_remote=4108 dedup_ratio=0.062683 router_tokens_per_expert=[0, 4087, 0, 0, 0, 3785, 0, 0, 1000, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3557, 0, 4096, 0, 0, 0, 0, 422, 0, 0, 0, 0, 3208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 180, 4090, 0, 0, 0, 4096, 2, 0, 145, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7570, 8174, 7114, 7278, 6724, 8192, 8184, 8192] dup_rows_per_dst=[0, 2004, 0, 746, 704, 0, 364, 290] combine_rows_per_src=[7570, 8174, 7114, 7278, 6724, 8192, 8184, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=335060992 dispatch_meta_bytes=225768 combine_payload_bytes=335060992 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=335286760 ingress_bytes=775462912 link_bytes=775462912 token_bytes=670347752 weight_movement_bytes=440401920 total_nvlink_bytes=1110749672 weight_to_token_ratio=0.656975 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=29440 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.348657546e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61044 payload_rows_remote=55282 dup_rows=4492 dup_rows_remote=4492 dedup_ratio=0.068542 router_tokens_per_expert=[0, 4067, 0, 0, 0, 3776, 0, 0, 1046, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3428, 0, 4092, 0, 0, 0, 0, 724, 0, 0, 0, 0, 2881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 328, 4066, 0, 0, 0, 4096, 3, 0, 158, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7552, 8142, 6856, 8192, 5762, 8192, 8156, 8192] dup_rows_per_dst=[0, 2088, 0, 1440, 0, 0, 648, 316] combine_rows_per_src=[7552, 8142, 6856, 8192, 5762, 8192, 8156, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=382556160 dispatch_meta_bytes=234184 combine_payload_bytes=382556160 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=382790344 ingress_bytes=492656640 link_bytes=492656640 token_bytes=765346504 weight_movement_bytes=110100480 total_nvlink_bytes=875446984 weight_to_token_ratio=0.143857 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=28288 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.568109251e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51136 payload_rows_remote=42944 dup_rows=14400 dup_rows_remote=14194 dedup_ratio=0.219727 router_tokens_per_expert=[0, 4087, 0, 0, 0, 3537, 0, 0, 496, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3574, 0, 4096, 0, 0, 0, 0, 1038, 0, 0, 0, 0, 3342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 205, 4089, 0, 0, 0, 4096, 0, 0, 103, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 8182, 8110, 2076, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 996, 6112, 0, 6684, 0, 402, 206] combine_rows_per_src=[0, 8182, 8110, 2076, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=381595648 dispatch_meta_bytes=229376 combine_payload_bytes=381595648 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=381825024 ingress_bytes=491696128 link_bytes=491696128 token_bytes=763420672 weight_movement_bytes=110100480 total_nvlink_bytes=873521152 weight_to_token_ratio=0.144220 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.551404367e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61580 payload_rows_remote=53388 dup_rows=3956 dup_rows_remote=2878 dedup_ratio=0.060364 router_tokens_per_expert=[0, 4068, 0, 0, 0, 3780, 0, 0, 992, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3613, 0, 4095, 0, 0, 0, 0, 540, 0, 0, 0, 0, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 289, 4087, 0, 0, 0, 4096, 3, 0, 157, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7560, 8140, 7226, 8192, 5888, 8192, 8190, 8192] dup_rows_per_dst=[0, 1982, 0, 1078, 0, 0, 582, 314] combine_rows_per_src=[7560, 8140, 7226, 8192, 5888, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59518 payload_rows_remote=51326 dup_rows=6018 dup_rows_remote=5236 dedup_ratio=0.091827 router_tokens_per_expert=[0, 0, 0, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 2285, 0, 4096, 0, 0, 0, 0, 391, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4043, 1001, 4083, 0, 0, 3966, 4096, 1, 0, 407, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8106, 8166, 7560, 8192, 8192, 8192, 2986, 8124] dup_rows_per_dst=[0, 372, 3550, 782, 128, 0, 410, 776] combine_rows_per_src=[8106, 8166, 7560, 8192, 8192, 8192, 2986, 8124] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=295049216 dispatch_meta_bytes=227568 combine_payload_bytes=295049216 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=295276784 ingress_bytes=405149696 link_bytes=405149696 token_bytes=590326000 weight_movement_bytes=110100480 total_nvlink_bytes=700426480 weight_to_token_ratio=0.186508 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=29696 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.046219570e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58348 payload_rows_remote=50210 dup_rows=7188 dup_rows_remote=6810 dedup_ratio=0.109680 router_tokens_per_expert=[0, 0, 0, 0, 0, 4057, 0, 0, 0, 0, 0, 5, 0, 0, 0, 192, 0, 0, 0, 0, 0, 0, 0, 2313, 0, 4094, 0, 0, 0, 0, 660, 0, 0, 0, 0, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4067, 659, 4061, 0, 0, 3973, 4096, 0, 0, 387, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8114, 8138, 8028, 8192, 8192, 8192, 1318, 8174] dup_rows_per_dst=[0, 378, 4544, 1316, 216, 0, 0, 734] combine_rows_per_src=[8114, 8138, 8028, 8192, 8192, 8192, 1318, 8174] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=367904768 dispatch_meta_bytes=226248 combine_payload_bytes=367904768 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=368131016 ingress_bytes=588105728 link_bytes=588105728 token_bytes=736035784 weight_movement_bytes=220200960 total_nvlink_bytes=956236744 weight_to_token_ratio=0.299172 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=28160 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.022812588e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61650 payload_rows_remote=53458 dup_rows=3886 dup_rows_remote=3746 dedup_ratio=0.059296 router_tokens_per_expert=[0, 0, 0, 0, 0, 4053, 0, 0, 0, 0, 0, 2, 0, 0, 0, 204, 0, 0, 0, 0, 0, 0, 0, 2326, 0, 4084, 0, 0, 0, 0, 602, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4054, 834, 4073, 0, 0, 3915, 4096, 2, 0, 357, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8106, 8152, 4652, 8192, 8192, 8192, 8024, 8140] dup_rows_per_dst=[0, 406, 0, 1180, 140, 0, 1478, 682] combine_rows_per_src=[8106, 8152, 4652, 8192, 8192, 8192, 8024, 8140] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57434 payload_rows_remote=49312 dup_rows=8102 dup_rows_remote=7408 dedup_ratio=0.123627 router_tokens_per_expert=[0, 0, 0, 0, 0, 4055, 0, 0, 0, 0, 0, 1, 0, 0, 0, 350, 0, 0, 0, 0, 0, 0, 0, 2673, 0, 4096, 0, 0, 0, 0, 330, 0, 0, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4060, 470, 4043, 0, 0, 3979, 4096, 0, 0, 362, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8122, 8106, 8128, 7698, 8192, 8192, 5932, 3064] dup_rows_per_dst=[694, 682, 5176, 448, 314, 0, 674, 114] combine_rows_per_src=[8122, 8106, 8128, 7698, 8192, 8192, 5932, 3064] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64970 payload_rows_remote=57256 dup_rows=566 dup_rows_remote=564 dedup_ratio=0.008636 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 223, 0, 0, 1, 0, 4086, 1, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4079, 0, 4078, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 20, 0, 0, 17, 1, 0, 0, 0, 0, 0, 3857, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8174, 8192, 8158, 8156, 8192, 7714, 8192] dup_rows_per_dst=[446, 2, 42, 0, 0, 74, 2, 0] combine_rows_per_src=[8192, 8174, 8192, 8158, 8156, 8192, 7714, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64986 payload_rows_remote=56814 dup_rows=550 dup_rows_remote=494 dedup_ratio=0.008392 router_tokens_per_expert=[0, 0, 0, 0, 0, 13, 0, 193, 0, 0, 12, 0, 4075, 6, 0, 0, 0, 0, 41, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 4089, 4096, 4061, 0, 4091, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 3, 3, 0, 25, 11, 0, 0, 0, 0, 0, 3879, 0, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8172, 8172, 8122, 8182, 8192, 7762, 8192] dup_rows_per_dst=[398, 14, 56, 0, 0, 62, 18, 2] combine_rows_per_src=[8192, 8172, 8172, 8122, 8182, 8192, 7762, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65244 payload_rows_remote=57052 dup_rows=292 dup_rows_remote=288 dedup_ratio=0.004456 router_tokens_per_expert=[0, 0, 0, 0, 0, 2, 0, 116, 0, 0, 4, 0, 4058, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 4096, 4046, 0, 4091, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5, 0, 0, 0, 0, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8116, 8188, 8092, 8182, 8192, 8090, 8192] dup_rows_per_dst=[234, 8, 36, 0, 0, 4, 10, 0] combine_rows_per_src=[8192, 8116, 8188, 8092, 8182, 8192, 8090, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=359905280 dispatch_meta_bytes=228080 combine_payload_bytes=359905280 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=360133360 ingress_bytes=470005760 link_bytes=470005760 token_bytes=720038640 weight_movement_bytes=110100480 total_nvlink_bytes=830139120 weight_to_token_ratio=0.152909 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=30720 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.174173193e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=383186944 dispatch_meta_bytes=228816 combine_payload_bytes=383186944 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=383415760 ingress_bytes=493287424 link_bytes=493287424 token_bytes=766602704 weight_movement_bytes=110100480 total_nvlink_bytes=876703184 weight_to_token_ratio=0.143621 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.579079622e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=396261376 dispatch_meta_bytes=239096 combine_payload_bytes=396261376 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=396500472 ingress_bytes=506361856 link_bytes=506361856 token_bytes=792761848 weight_movement_bytes=110100480 total_nvlink_bytes=902862328 weight_to_token_ratio=0.138882 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=24192 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.806465498e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=307822592 dispatch_meta_bytes=228552 combine_payload_bytes=307822592 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=308051144 ingress_bytes=417923072 link_bytes=417923072 token_bytes=615873736 weight_movement_bytes=110100480 total_nvlink_bytes=725974216 weight_to_token_ratio=0.178771 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.268369587e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50668 payload_rows_remote=42476 dup_rows=14868 dup_rows_remote=6162 dedup_ratio=0.226868 router_tokens_per_expert=[0, 4046, 0, 0, 0, 3824, 0, 0, 753, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3634, 0, 4096, 0, 0, 0, 0, 382, 0, 0, 0, 0, 3466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 142, 4061, 0, 0, 0, 4096, 0, 0, 161, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 1508, 7268, 8192, 6932, 8192, 4444, 5940] dup_rows_per_dst=[8706, 0, 0, 5902, 0, 2, 174, 84] combine_rows_per_src=[8192, 1508, 7268, 8192, 6932, 8192, 4444, 5940] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57920 payload_rows_remote=53674 dup_rows=7616 dup_rows_remote=7070 dedup_ratio=0.116211 router_tokens_per_expert=[0, 4003, 0, 0, 0, 3672, 0, 0, 1017, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3441, 0, 4095, 0, 0, 0, 0, 719, 0, 0, 0, 0, 3099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 248, 4066, 0, 0, 0, 4096, 0, 0, 201, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7868, 4246, 6882, 8192, 6198, 8192, 8150, 8192] dup_rows_per_dst=[4734, 546, 0, 1436, 0, 0, 498, 402] combine_rows_per_src=[7868, 4246, 6882, 8192, 6198, 8192, 8150, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=382685184 dispatch_meta_bytes=225064 combine_payload_bytes=382685184 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=382910248 ingress_bytes=602886144 link_bytes=602886144 token_bytes=765595432 weight_movement_bytes=220200960 total_nvlink_bytes=985796392 weight_to_token_ratio=0.287621 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=24064 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.048518162e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60386 payload_rows_remote=53958 dup_rows=5150 dup_rows_remote=4454 dedup_ratio=0.078583 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1078, 0, 0, 0, 0, 4071, 7, 0, 0, 0, 0, 4, 3995, 0, 0, 0, 0, 4096, 0, 0, 0, 3966, 4096, 3938, 0, 211, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 328, 0, 0, 0, 0, 0, 2879, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5244, 8150, 7990, 7998, 8192, 8192, 6428, 8192] dup_rows_per_dst=[970, 6, 8, 3042, 422, 6, 696, 0] combine_rows_per_src=[5244, 8150, 7990, 7998, 8192, 8192, 6428, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62764 payload_rows_remote=54642 dup_rows=2772 dup_rows_remote=2712 dedup_ratio=0.042297 router_tokens_per_expert=[0, 0, 0, 0, 0, 11, 0, 918, 0, 0, 14, 0, 4062, 41, 0, 0, 0, 0, 30, 4061, 0, 0, 0, 0, 4096, 0, 0, 0, 3708, 4096, 3541, 0, 226, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4, 690, 0, 0, 0, 0, 0, 3182, 0, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[7836, 8152, 8122, 7082, 8172, 8192, 7016, 8192] dup_rows_per_dst=[1438, 82, 60, 0, 450, 12, 728, 2] combine_rows_per_src=[7836, 8152, 8122, 7082, 8172, 8192, 7016, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63694 payload_rows_remote=55502 dup_rows=1842 dup_rows_remote=1842 dedup_ratio=0.028107 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 635, 0, 0, 4, 0, 4085, 8, 0, 0, 0, 0, 0, 4022, 0, 0, 0, 0, 4096, 0, 0, 0, 3925, 4096, 3846, 0, 272, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 0, 0, 0, 0, 0, 3541, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8054, 8176, 8044, 7692, 8190, 8192, 7154, 8192] dup_rows_per_dst=[1068, 18, 0, 0, 544, 0, 212, 0] combine_rows_per_src=[8054, 8176, 8044, 7692, 8190, 8192, 7154, 8192] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62136 payload_rows_remote=53944 dup_rows=3400 dup_rows_remote=3400 dedup_ratio=0.051880 router_tokens_per_expert=[0, 0, 0, 0, 0, 4, 0, 304, 0, 0, 12, 0, 4077, 0, 1, 0, 0, 0, 11, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 4096, 4006, 0, 4092, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8, 0, 0, 0, 0, 0, 3876, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5564, 8154, 7904, 8096, 8184, 8192, 7850, 8192] dup_rows_per_dst=[340, 26, 18, 712, 282, 6, 2016, 0] combine_rows_per_src=[5564, 8154, 7904, 8096, 8184, 8192, 7850, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=304467968 dispatch_meta_bytes=194552 combine_payload_bytes=304467968 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=304662520 ingress_bytes=634769408 link_bytes=634769408 token_bytes=609130488 weight_movement_bytes=330301440 total_nvlink_bytes=939431928 weight_to_token_ratio=0.542251 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=25600 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.103968402e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=384735232 dispatch_meta_bytes=242976 combine_payload_bytes=384735232 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=384978208 ingress_bytes=494835712 link_bytes=494835712 token_bytes=769713440 weight_movement_bytes=110100480 total_nvlink_bytes=879813920 weight_to_token_ratio=0.143041 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=25856 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.606006897e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=353468416 dispatch_meta_bytes=226880 combine_payload_bytes=353468416 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=353695296 ingress_bytes=463568896 link_bytes=463568896 token_bytes=707163712 weight_movement_bytes=110100480 total_nvlink_bytes=817264192 weight_to_token_ratio=0.155693 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.062225541e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=410411008 dispatch_meta_bytes=231280 combine_payload_bytes=410411008 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=410642288 ingress_bytes=520511488 link_bytes=520511488 token_bytes=821053296 weight_movement_bytes=110100480 total_nvlink_bytes=931153776 weight_to_token_ratio=0.134097 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.052550871e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=386770944 dispatch_meta_bytes=233648 combine_payload_bytes=386770944 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=387004592 ingress_bytes=496871424 link_bytes=496871424 token_bytes=773775536 weight_movement_bytes=110100480 total_nvlink_bytes=883876016 weight_to_token_ratio=0.142290 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=26496 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.641411276e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62980 payload_rows_remote=54798 dup_rows=2556 dup_rows_remote=1954 dedup_ratio=0.039001 router_tokens_per_expert=[0, 0, 0, 0, 0, 4, 0, 875, 0, 0, 19, 0, 4077, 6, 0, 0, 0, 0, 18, 4080, 0, 0, 0, 0, 4096, 0, 0, 0, 4043, 4096, 3739, 0, 301, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 163, 0, 0, 0, 0, 0, 3160, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8174, 8166, 8160, 7478, 8182, 8192, 6436, 8192] dup_rows_per_dst=[1670, 38, 36, 0, 602, 0, 210, 0] combine_rows_per_src=[8174, 8166, 8160, 7478, 8182, 8192, 6436, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58248 payload_rows_remote=50056 dup_rows=7288 dup_rows_remote=7288 dedup_ratio=0.111206 router_tokens_per_expert=[0, 0, 0, 0, 0, 4, 0, 1176, 0, 0, 13, 0, 4069, 11, 0, 0, 0, 0, 7, 3954, 0, 0, 0, 0, 4096, 0, 0, 0, 3545, 4096, 3794, 0, 207, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 355, 0, 0, 0, 0, 0, 3347, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[2360, 8142, 7908, 7090, 8186, 8192, 8178, 8192] dup_rows_per_dst=[0, 44, 14, 0, 412, 4, 6814, 0] combine_rows_per_src=[2360, 8142, 7908, 7090, 8186, 8192, 8178, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=391673856 dispatch_meta_bytes=229416 combine_payload_bytes=391673856 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=391903272 ingress_bytes=501774336 link_bytes=501774336 token_bytes=783577128 weight_movement_bytes=110100480 total_nvlink_bytes=893677608 weight_to_token_ratio=0.140510 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.726680980e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=397838336 dispatch_meta_bytes=229376 combine_payload_bytes=397838336 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=398067712 ingress_bytes=507938816 link_bytes=507938816 token_bytes=795906048 weight_movement_bytes=110100480 total_nvlink_bytes=906006528 weight_to_token_ratio=0.138334 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.833891426e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63366 payload_rows_remote=55596 dup_rows=2170 dup_rows_remote=2170 dedup_ratio=0.033112 router_tokens_per_expert=[0, 0, 0, 0, 0, 7, 0, 925, 0, 0, 9, 0, 4068, 15, 0, 0, 0, 0, 14, 4081, 0, 0, 0, 0, 4096, 0, 0, 0, 3928, 4096, 3885, 0, 210, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 0, 0, 0, 0, 0, 3230, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8134, 8156, 8162, 7770, 8174, 8192, 6586, 8192] dup_rows_per_dst=[1586, 28, 28, 0, 420, 0, 108, 0] combine_rows_per_src=[8134, 8156, 8162, 7770, 8174, 8192, 6586, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=407242752 dispatch_meta_bytes=229232 combine_payload_bytes=407242752 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=407471984 ingress_bytes=517343232 link_bytes=517343232 token_bytes=814714736 weight_movement_bytes=110100480 total_nvlink_bytes=924815216 weight_to_token_ratio=0.135140 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.997449688e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=408948736 dispatch_meta_bytes=229360 combine_payload_bytes=408948736 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=409178096 ingress_bytes=519049216 link_bytes=519049216 token_bytes=818126832 weight_movement_bytes=110100480 total_nvlink_bytes=928227312 weight_to_token_ratio=0.134576 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.027119556e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64956 payload_rows_remote=56876 dup_rows=580 dup_rows_remote=580 dedup_ratio=0.008850 router_tokens_per_expert=[0, 0, 0, 0, 0, 8, 0, 213, 0, 0, 9, 0, 4046, 6, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 4096, 4040, 0, 4088, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 10, 0, 0, 9, 11, 0, 0, 0, 0, 0, 3919, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8110, 8174, 8080, 8176, 8192, 7840, 8192] dup_rows_per_dst=[436, 12, 74, 0, 0, 38, 20, 0] combine_rows_per_src=[8192, 8110, 8174, 8080, 8176, 8192, 7840, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=386670592 dispatch_meta_bytes=229376 combine_payload_bytes=386670592 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=386899968 ingress_bytes=496771072 link_bytes=496771072 token_bytes=773570560 weight_movement_bytes=110100480 total_nvlink_bytes=883671040 weight_to_token_ratio=0.142328 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.639665990e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=407687168 dispatch_meta_bytes=229824 combine_payload_bytes=407687168 prefetch_bytes=110100480 reduce_bytes=550502400 egress_bytes=407916992 ingress_bytes=1068290048 link_bytes=1068290048 token_bytes=815604160 weight_movement_bytes=660602880 total_nvlink_bytes=1476207040 weight_to_token_ratio=0.809955 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=31232 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.857932097e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64998 payload_rows_remote=56844 dup_rows=538 dup_rows_remote=484 dedup_ratio=0.008209 router_tokens_per_expert=[0, 0, 0, 0, 0, 8, 0, 112, 0, 0, 66, 0, 4035, 3, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 4096, 3953, 0, 4036, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 69, 1, 0, 0, 31, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8154, 8168, 7906, 8072, 8192, 8124, 8192] dup_rows_per_dst=[232, 54, 50, 0, 0, 140, 62, 0] combine_rows_per_src=[8190, 8154, 8168, 7906, 8072, 8192, 8124, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65288 payload_rows_remote=57100 dup_rows=248 dup_rows_remote=248 dedup_ratio=0.003784 router_tokens_per_expert=[0, 0, 0, 0, 0, 4, 0, 74, 0, 0, 18, 0, 4081, 1, 1, 0, 0, 0, 26, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 4096, 4048, 0, 4094, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8166, 8182, 8096, 8188, 8192, 8080, 8192] dup_rows_per_dst=[154, 36, 50, 0, 0, 4, 4, 0] combine_rows_per_src=[8192, 8166, 8182, 8096, 8188, 8192, 8080, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64326 payload_rows_remote=56134 dup_rows=1210 dup_rows_remote=1210 dedup_ratio=0.018463 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 3904, 0, 0, 11, 4096, 0, 0, 0, 0, 0, 4074, 0, 0, 3712, 0, 0, 14, 0, 0, 0, 4084, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 608, 0, 0, 0, 0, 0, 4059, 0] payload_rows_per_dst=[8192, 8158, 8190, 7808, 7424, 8172, 8192, 8190] dup_rows_per_dst=[0, 20, 0, 22, 0, 24, 0, 1144] combine_rows_per_src=[8192, 8158, 8190, 7808, 7424, 8172, 8192, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64610 payload_rows_remote=56418 dup_rows=926 dup_rows_remote=536 dedup_ratio=0.014130 router_tokens_per_expert=[0, 0, 0, 0, 0, 8, 0, 192, 0, 0, 31, 0, 4075, 0, 8, 0, 0, 0, 32, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 4096, 3984, 0, 4083, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 8, 0, 0, 9, 28, 0, 0, 0, 0, 0, 3944, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8168, 8170, 8044, 8114, 7844, 7888, 8190] dup_rows_per_dst=[390, 170, 60, 230, 0, 20, 56, 0] combine_rows_per_src=[8192, 8168, 8170, 8044, 8114, 7844, 7888, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63468 payload_rows_remote=55302 dup_rows=2068 dup_rows_remote=2068 dedup_ratio=0.031555 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 16, 35, 0, 1, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 4001, 0, 0, 38, 4091, 0, 0, 4, 0, 0, 4077, 2, 0, 3205, 0, 0, 87, 0, 4, 0, 4081, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 960, 0, 0, 0, 0, 0, 3985, 0] payload_rows_per_dst=[8186, 8172, 8166, 8002, 6418, 8186, 8192, 8146] dup_rows_per_dst=[0, 84, 0, 76, 4, 158, 2, 1744] combine_rows_per_src=[8186, 8172, 8166, 8002, 6418, 8186, 8192, 8146] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=358801408 dispatch_meta_bytes=229376 combine_payload_bytes=358801408 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=359030784 ingress_bytes=468901888 link_bytes=468901888 token_bytes=717832192 weight_movement_bytes=110100480 total_nvlink_bytes=827932672 weight_to_token_ratio=0.153379 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.154975043e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=392792064 dispatch_meta_bytes=227008 combine_payload_bytes=392792064 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=393019072 ingress_bytes=723093504 link_bytes=723093504 token_bytes=785811136 weight_movement_bytes=330301440 total_nvlink_bytes=1116112576 weight_to_token_ratio=0.420332 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=31104 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.257578532e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=398512128 dispatch_meta_bytes=231064 combine_payload_bytes=398512128 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=398743192 ingress_bytes=949014528 link_bytes=949014528 token_bytes=797255320 weight_movement_bytes=550502400 total_nvlink_bytes=1347757720 weight_to_token_ratio=0.690497 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=25984 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.650492350e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60248 payload_rows_remote=52106 dup_rows=5288 dup_rows_remote=3512 dedup_ratio=0.080688 router_tokens_per_expert=[0, 0, 0, 0, 0, 5, 0, 921, 0, 0, 33, 0, 4068, 2, 0, 0, 0, 0, 0, 4042, 0, 0, 0, 0, 4096, 0, 0, 0, 4033, 4096, 3677, 0, 159, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 142, 0, 0, 0, 0, 0, 3404, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8142, 8170, 8094, 7648, 4962, 8160, 6882, 8190] dup_rows_per_dst=[1776, 42, 730, 2358, 172, 0, 210, 0] combine_rows_per_src=[8142, 8170, 8094, 7648, 4962, 8160, 6882, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63778 payload_rows_remote=55654 dup_rows=1758 dup_rows_remote=1716 dedup_ratio=0.026825 router_tokens_per_expert=[0, 0, 0, 0, 0, 7, 0, 756, 0, 0, 75, 0, 3998, 10, 0, 0, 0, 0, 8, 4082, 0, 0, 0, 0, 4096, 0, 0, 0, 3779, 4096, 3689, 0, 146, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 189, 0, 0, 0, 0, 0, 3750, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8018, 8124, 8164, 7378, 8176, 8192, 7534, 8192] dup_rows_per_dst=[1066, 42, 16, 0, 280, 10, 344, 0] combine_rows_per_src=[8018, 8124, 8164, 7378, 8176, 8192, 7534, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59146 payload_rows_remote=50954 dup_rows=6390 dup_rows_remote=6390 dedup_ratio=0.097504 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 472, 20, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 0, 0, 10, 4064, 0, 0, 0, 0, 0, 4089, 1, 0, 244, 0, 0, 1182, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2252, 0, 0, 0, 0, 8, 4096, 0] payload_rows_per_dst=[8128, 7844, 8192, 8054, 8180, 2364, 8192, 8192] dup_rows_per_dst=[32, 928, 0, 20, 488, 0, 0, 4922] combine_rows_per_src=[8128, 7844, 8192, 8054, 8180, 2364, 8192, 8192] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64706 payload_rows_remote=56516 dup_rows=830 dup_rows_remote=820 dedup_ratio=0.012665 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 23, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 18, 4096, 0, 0, 0, 0, 0, 4083, 1, 0, 3742, 0, 0, 13, 0, 0, 0, 4087, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 396, 0, 0, 0, 0, 0, 4068, 0] payload_rows_per_dst=[8192, 8170, 8190, 8094, 7486, 8190, 8192, 8192] dup_rows_per_dst=[0, 48, 0, 36, 0, 10, 0, 736] combine_rows_per_src=[8192, 8170, 8190, 8094, 7486, 8190, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=407457792 dispatch_meta_bytes=229312 combine_payload_bytes=407457792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=407687104 ingress_bytes=517558272 link_bytes=517558272 token_bytes=815144896 weight_movement_bytes=110100480 total_nvlink_bytes=925245376 weight_to_token_ratio=0.135069 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.001189587e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63952 payload_rows_remote=55792 dup_rows=1584 dup_rows_remote=724 dedup_ratio=0.024170 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 42, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 10, 4094, 0, 0, 4, 0, 0, 4077, 4, 0, 3772, 0, 0, 34, 0, 1, 0, 4085, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 380, 0, 0, 0, 0, 0, 3992, 0] payload_rows_per_dst=[8188, 7596, 8192, 8140, 7568, 7916, 8192, 8160] dup_rows_per_dst=[0, 82, 0, 20, 572, 48, 2, 860] combine_rows_per_src=[8188, 7596, 8192, 8140, 7568, 7916, 8192, 8160] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=409292800 dispatch_meta_bytes=229392 combine_payload_bytes=409292800 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=409522192 ingress_bytes=629493760 link_bytes=629493760 token_bytes=818814992 weight_movement_bytes=220200960 total_nvlink_bytes=1039015952 weight_to_token_ratio=0.268926 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.094793183e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=402368512 dispatch_meta_bytes=229376 combine_payload_bytes=402368512 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=402597888 ingress_bytes=512468992 link_bytes=512468992 token_bytes=804966400 weight_movement_bytes=110100480 total_nvlink_bytes=915066880 weight_to_token_ratio=0.136776 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.912678638e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=404404224 dispatch_meta_bytes=227816 combine_payload_bytes=404404224 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=404632040 ingress_bytes=514504704 link_bytes=514504704 token_bytes=809036264 weight_movement_bytes=110100480 total_nvlink_bytes=919136744 weight_to_token_ratio=0.136088 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.948083017e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64270 payload_rows_remote=56190 dup_rows=1266 dup_rows_remote=1254 dedup_ratio=0.019318 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 17, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 0, 0, 6, 4096, 0, 0, 3, 0, 0, 4086, 3, 0, 3532, 0, 0, 48, 0, 0, 0, 4057, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 650, 0, 0, 0, 0, 0, 4021, 0] payload_rows_per_dst=[8192, 8188, 8192, 8080, 7076, 8168, 8192, 8182] dup_rows_per_dst=[0, 50, 0, 12, 0, 42, 2, 1160] combine_rows_per_src=[8192, 8188, 8192, 8080, 7076, 8168, 8192, 8182] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=396404736 dispatch_meta_bytes=229480 combine_payload_bytes=396404736 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=396634216 ingress_bytes=506505216 link_bytes=506505216 token_bytes=793038952 weight_movement_bytes=110100480 total_nvlink_bytes=903139432 weight_to_token_ratio=0.138834 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.808958764e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=405106688 dispatch_meta_bytes=229344 combine_payload_bytes=405106688 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=405336032 ingress_bytes=735408128 link_bytes=735408128 token_bytes=810442720 weight_movement_bytes=330301440 total_nvlink_bytes=1140744160 weight_to_token_ratio=0.407557 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.278995688e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=399917056 dispatch_meta_bytes=226064 combine_payload_bytes=399917056 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=400143120 ingress_bytes=730218496 link_bytes=730218496 token_bytes=800060176 weight_movement_bytes=330301440 total_nvlink_bytes=1130361616 weight_to_token_ratio=0.412846 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=27648 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.269970065e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=402769920 dispatch_meta_bytes=229776 combine_payload_bytes=402769920 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=402999696 ingress_bytes=733071360 link_bytes=733071360 token_bytes=805769616 weight_movement_bytes=330301440 total_nvlink_bytes=1136071056 weight_to_token_ratio=0.409920 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.274931665e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63396 payload_rows_remote=55210 dup_rows=2140 dup_rows_remote=1872 dedup_ratio=0.032654 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 76, 0, 3, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 0, 0, 24, 4092, 0, 0, 5, 0, 0, 4063, 7, 0, 3277, 0, 0, 86, 0, 1, 0, 4027, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 772, 0, 0, 0, 0, 0, 4054, 0] payload_rows_per_dst=[8184, 8186, 8188, 8006, 7074, 8154, 8192, 7412] dup_rows_per_dst=[0, 268, 6, 46, 700, 74, 2, 1044] combine_rows_per_src=[8184, 8186, 8188, 8006, 7074, 8154, 8192, 7412] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64250 payload_rows_remote=56060 dup_rows=1286 dup_rows_remote=724 dedup_ratio=0.019623 router_tokens_per_expert=[96, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 228, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3869, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 2304, 0, 0, 1426, 0, 0, 6, 0, 275, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7738, 8192, 8192, 7362, 8190, 8192] dup_rows_per_dst=[152, 474, 0, 0, 0, 98, 562, 0] combine_rows_per_src=[8192, 8192, 7738, 8192, 8192, 7362, 8190, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64082 payload_rows_remote=57194 dup_rows=1454 dup_rows_remote=1452 dedup_ratio=0.022186 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 47, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 20, 4095, 0, 0, 1, 0, 0, 4072, 5, 0, 3439, 0, 0, 41, 0, 0, 0, 4081, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 709, 0, 0, 0, 0, 0, 4003, 0] payload_rows_per_dst=[8190, 8166, 8168, 8120, 6888, 8190, 8192, 8168] dup_rows_per_dst=[0, 92, 0, 40, 2, 54, 10, 1256] combine_rows_per_src=[8190, 8166, 8168, 8120, 6888, 8190, 8192, 8168] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57838 payload_rows_remote=49646 dup_rows=7698 dup_rows_remote=7678 dedup_ratio=0.117462 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 36, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 3962, 0, 0, 13, 4095, 0, 0, 1, 0, 0, 4065, 3, 0, 3731, 0, 0, 35, 0, 2, 0, 4070, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 571, 0, 0, 0, 0, 0, 3980, 0] payload_rows_per_dst=[8192, 8172, 8158, 7940, 7942, 8118, 8174, 1142] dup_rows_per_dst=[20, 84, 86, 1094, 6382, 32, 0, 0] combine_rows_per_src=[8192, 8172, 8158, 7940, 7942, 8118, 8174, 1142] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63454 payload_rows_remote=56132 dup_rows=2082 dup_rows_remote=2082 dedup_ratio=0.031769 router_tokens_per_expert=[46, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 266, 32, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3651, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4087, 1, 0, 0, 2807, 0, 0, 759, 0, 0, 42, 0, 628, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 12, 0] payload_rows_per_dst=[8188, 8192, 7322, 8180, 8174, 7022, 8184, 8192] dup_rows_per_dst=[44, 596, 0, 0, 0, 112, 1306, 24] combine_rows_per_src=[8188, 8192, 7322, 8180, 8174, 7022, 8184, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58208 payload_rows_remote=50016 dup_rows=7328 dup_rows_remote=7328 dedup_ratio=0.111816 router_tokens_per_expert=[40, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 471, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3810, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4090, 9, 0, 0, 2063, 0, 0, 1268, 0, 0, 24, 0, 526, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7704, 3538, 7628, 8184, 8180, 6590, 8192, 8192] dup_rows_per_dst=[54, 434, 0, 28, 4394, 90, 2328, 0] combine_rows_per_src=[7704, 3538, 7628, 8184, 8180, 6590, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=395745280 dispatch_meta_bytes=228328 combine_payload_bytes=395745280 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=395973608 ingress_bytes=505845760 link_bytes=505845760 token_bytes=791718888 weight_movement_bytes=110100480 total_nvlink_bytes=901819368 weight_to_token_ratio=0.139065 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.797489740e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64108 payload_rows_remote=57040 dup_rows=1428 dup_rows_remote=1404 dedup_ratio=0.021790 router_tokens_per_expert=[20, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 356, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3949, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 2334, 0, 0, 1212, 0, 0, 6, 0, 333, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8192, 7900, 8192, 8192, 7068, 8192, 8184] dup_rows_per_dst=[18, 716, 0, 0, 0, 24, 670, 0] combine_rows_per_src=[8188, 8192, 7900, 8192, 8192, 7068, 8192, 8184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=401838080 dispatch_meta_bytes=227136 combine_payload_bytes=401838080 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=402065216 ingress_bytes=622039040 link_bytes=622039040 token_bytes=803903296 weight_movement_bytes=220200960 total_nvlink_bytes=1024104256 weight_to_token_ratio=0.273915 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=28928 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.081828198e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60810 payload_rows_remote=52618 dup_rows=4726 dup_rows_remote=4726 dedup_ratio=0.072113 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 339, 71, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 23, 4044, 0, 0, 4, 0, 0, 4085, 6, 0, 18, 0, 0, 1883, 0, 1, 0, 4086, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 1801, 0, 0, 0, 0, 0, 4094, 0] payload_rows_per_dst=[8096, 8192, 8192, 8164, 8180, 8192, 8192, 3602] dup_rows_per_dst=[68, 816, 0, 46, 46, 3748, 2, 0] combine_rows_per_src=[8096, 8192, 8192, 8164, 8180, 8192, 8192, 3602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60678 payload_rows_remote=57300 dup_rows=4858 dup_rows_remote=4858 dedup_ratio=0.074127 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 457, 76, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 19, 4085, 0, 0, 0, 0, 0, 4094, 2, 0, 228, 0, 0, 1689, 0, 0, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1626, 0, 0, 0, 0, 3, 4096, 0] payload_rows_per_dst=[8170, 8192, 8192, 8170, 8192, 3378, 8192, 8192] dup_rows_per_dst=[50, 1066, 0, 38, 456, 0, 0, 3248] combine_rows_per_src=[8170, 8192, 8192, 8170, 8192, 3378, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=373495808 dispatch_meta_bytes=222472 combine_payload_bytes=373495808 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=373718280 ingress_bytes=483596288 link_bytes=483596288 token_bytes=747214088 weight_movement_bytes=110100480 total_nvlink_bytes=857314568 weight_to_token_ratio=0.147348 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=25472 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.410534828e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=398927872 dispatch_meta_bytes=229480 combine_payload_bytes=398927872 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=399157352 ingress_bytes=509028352 link_bytes=509028352 token_bytes=798085224 weight_movement_bytes=110100480 total_nvlink_bytes=908185704 weight_to_token_ratio=0.137956 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.852840249e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=365238272 dispatch_meta_bytes=229376 combine_payload_bytes=365238272 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=365467648 ingress_bytes=585439232 link_bytes=585439232 token_bytes=730705920 weight_movement_bytes=220200960 total_nvlink_bytes=950906880 weight_to_token_ratio=0.301354 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.018175113e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52044 payload_rows_remote=43860 dup_rows=13492 dup_rows_remote=1190 dedup_ratio=0.205872 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 301, 74, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 11, 4064, 0, 0, 4, 0, 0, 4079, 4, 0, 177, 0, 0, 1491, 0, 1, 0, 4088, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 2063, 0, 0, 0, 0, 0, 4092, 0] payload_rows_per_dst=[8088, 730, 8184, 8172, 6190, 4304, 8192, 8184] dup_rows_per_dst=[74, 20, 0, 62, 168, 850, 16, 12302] combine_rows_per_src=[8088, 730, 8184, 8172, 6190, 4304, 8192, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60900 payload_rows_remote=52722 dup_rows=4636 dup_rows_remote=4382 dedup_ratio=0.070740 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 554, 89, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 26, 4081, 0, 0, 1, 0, 0, 4085, 5, 0, 125, 0, 0, 1502, 0, 0, 0, 4082, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 1806, 0, 0, 0, 0, 13, 4092, 0] payload_rows_per_dst=[8164, 8192, 8192, 8172, 8178, 7916, 8192, 3894] dup_rows_per_dst=[50, 1278, 2, 52, 254, 2796, 4, 200] combine_rows_per_src=[8164, 8192, 8192, 8172, 8178, 7916, 8192, 3894] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=377165824 dispatch_meta_bytes=229376 combine_payload_bytes=377165824 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=377395200 ingress_bytes=487266304 link_bytes=487266304 token_bytes=754561024 weight_movement_bytes=110100480 total_nvlink_bytes=864661504 weight_to_token_ratio=0.145913 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.474362442e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61592 payload_rows_remote=53434 dup_rows=3944 dup_rows_remote=3918 dedup_ratio=0.060181 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 392, 90, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 13, 4091, 0, 0, 0, 0, 0, 4087, 7, 0, 205, 0, 0, 1274, 0, 0, 0, 4072, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 2148, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8184, 8192, 8192, 8158, 8188, 8190, 8192, 4296] dup_rows_per_dst=[36, 964, 0, 26, 410, 2502, 6, 0] combine_rows_per_src=[8184, 8192, 8192, 8158, 8188, 8190, 8192, 4296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=410726400 dispatch_meta_bytes=248632 combine_payload_bytes=410726400 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=410975032 ingress_bytes=630927360 link_bytes=630927360 token_bytes=821701432 weight_movement_bytes=220200960 total_nvlink_bytes=1041902392 weight_to_token_ratio=0.267982 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=20224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.097286449e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=314388480 dispatch_meta_bytes=180200 combine_payload_bytes=314388480 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=314568680 ingress_bytes=534589440 link_bytes=534589440 token_bytes=628957160 weight_movement_bytes=220200960 total_nvlink_bytes=849158120 weight_to_token_ratio=0.350105 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=16000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.297389610e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=377911296 dispatch_meta_bytes=228416 combine_payload_bytes=377911296 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=378139712 ingress_bytes=708212736 link_bytes=708212736 token_bytes=756051008 weight_movement_bytes=330301440 total_nvlink_bytes=1086352448 weight_to_token_ratio=0.436877 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=31488 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.231698429e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=383014912 dispatch_meta_bytes=229408 combine_payload_bytes=383014912 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=383244320 ingress_bytes=603215872 link_bytes=603215872 token_bytes=766259232 weight_movement_bytes=220200960 total_nvlink_bytes=986460192 weight_to_token_ratio=0.287371 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.049091613e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62130 payload_rows_remote=53996 dup_rows=3406 dup_rows_remote=3374 dedup_ratio=0.051971 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 326, 56, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 26, 4067, 0, 0, 1, 0, 0, 4072, 7, 0, 174, 0, 0, 1047, 0, 2, 0, 4074, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 2562, 0, 0, 0, 0, 1, 4091, 0] payload_rows_per_dst=[8134, 8192, 8192, 8146, 7990, 8188, 8162, 5126] dup_rows_per_dst=[32, 754, 0, 200, 362, 2058, 0, 0] combine_rows_per_src=[8134, 8192, 8192, 8146, 7990, 8188, 8162, 5126] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60410 payload_rows_remote=52218 dup_rows=5126 dup_rows_remote=3142 dedup_ratio=0.078217 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 815, 183, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 0, 0, 48, 4088, 0, 0, 4, 0, 0, 4060, 13, 0, 50, 0, 0, 1490, 0, 0, 0, 4035, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 1673, 0, 0, 0, 0, 16, 4090, 0] payload_rows_per_dst=[8178, 8192, 8192, 7960, 8136, 8182, 8192, 3378] dup_rows_per_dst=[52, 1984, 6, 96, 118, 2868, 2, 0] combine_rows_per_src=[8178, 8192, 8192, 7960, 8136, 8182, 8192, 3378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63630 payload_rows_remote=55456 dup_rows=1906 dup_rows_remote=1508 dedup_ratio=0.029083 router_tokens_per_expert=[82, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3254, 0, 0, 4083, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3261, 0, 0, 1300, 0, 0, 0, 0, 199, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 126, 0] payload_rows_per_dst=[8190, 8192, 6508, 8166, 8192, 8016, 8174, 8192] dup_rows_per_dst=[134, 16, 0, 0, 0, 1106, 398, 252] combine_rows_per_src=[8190, 8192, 6508, 8166, 8192, 8016, 8174, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62210 payload_rows_remote=55890 dup_rows=3326 dup_rows_remote=2462 dedup_ratio=0.050751 router_tokens_per_expert=[70, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 10, 13, 0, 0, 0, 0, 0, 0, 0, 0, 10, 2947, 0, 0, 4095, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4086, 2, 0, 0, 3697, 0, 0, 730, 0, 0, 1, 0, 652, 0, 0, 4017, 0, 0, 0, 0, 0, 0, 0, 175, 0] payload_rows_per_dst=[8190, 8192, 6320, 8190, 8172, 6822, 8132, 8192] dup_rows_per_dst=[100, 46, 864, 0, 0, 766, 1208, 342] combine_rows_per_src=[8190, 8192, 6320, 8190, 8172, 6822, 8132, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63352 payload_rows_remote=55404 dup_rows=2184 dup_rows_remote=918 dedup_ratio=0.033325 router_tokens_per_expert=[23, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3169, 0, 0, 4061, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 3162, 0, 0, 1445, 0, 0, 0, 0, 295, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 142, 0] payload_rows_per_dst=[8190, 8192, 6340, 8122, 8190, 7948, 8178, 8192] dup_rows_per_dst=[32, 34, 0, 0, 0, 1266, 568, 284] combine_rows_per_src=[8190, 8192, 6340, 8122, 8190, 7948, 8178, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=387043328 dispatch_meta_bytes=229480 combine_payload_bytes=387043328 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=387272808 ingress_bytes=497143808 link_bytes=497143808 token_bytes=774316136 weight_movement_bytes=110100480 total_nvlink_bytes=884416616 weight_to_token_ratio=0.142191 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.646148482e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=374298624 dispatch_meta_bytes=221440 combine_payload_bytes=374298624 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=374520064 ingress_bytes=484399104 link_bytes=484399104 token_bytes=748818688 weight_movement_bytes=110100480 total_nvlink_bytes=858919168 weight_to_token_ratio=0.147032 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=28544 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.424497118e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63076 payload_rows_remote=54892 dup_rows=2460 dup_rows_remote=2460 dedup_ratio=0.037537 router_tokens_per_expert=[102, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 19, 12, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2962, 0, 0, 4073, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 3, 0, 0, 3617, 0, 0, 1057, 0, 0, 1, 0, 408, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 74, 0] payload_rows_per_dst=[8190, 8192, 5934, 8146, 8184, 8070, 8170, 8190] dup_rows_per_dst=[168, 62, 0, 0, 0, 1284, 802, 144] combine_rows_per_src=[8190, 8192, 5934, 8146, 8184, 8070, 8170, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=397508608 dispatch_meta_bytes=227856 combine_payload_bytes=397508608 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=397736464 ingress_bytes=617709568 link_bytes=617709568 token_bytes=795245072 weight_movement_bytes=220200960 total_nvlink_bytes=1015446032 weight_to_token_ratio=0.276897 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=28544 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.074298535e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60304 payload_rows_remote=52112 dup_rows=5232 dup_rows_remote=5014 dedup_ratio=0.079834 router_tokens_per_expert=[50, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 10, 9, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3230, 0, 0, 4046, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4088, 5, 0, 0, 3301, 0, 0, 1156, 0, 0, 1, 0, 428, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 113, 0] payload_rows_per_dst=[7110, 7728, 6468, 6550, 8176, 7912, 8168, 8192] dup_rows_per_dst=[80, 38, 0, 0, 464, 1012, 3420, 218] combine_rows_per_src=[7110, 7728, 6468, 6550, 8176, 7912, 8168, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=400619520 dispatch_meta_bytes=233408 combine_payload_bytes=400619520 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=400852928 ingress_bytes=510720000 link_bytes=510720000 token_bytes=801472448 weight_movement_bytes=110100480 total_nvlink_bytes=911572928 weight_to_token_ratio=0.137373 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=25344 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.882260790e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63742 payload_rows_remote=55566 dup_rows=1794 dup_rows_remote=1794 dedup_ratio=0.027374 router_tokens_per_expert=[92, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 10, 14, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3328, 0, 0, 4088, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4090, 3, 0, 0, 3429, 0, 0, 1014, 0, 0, 0, 0, 334, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 40, 0] payload_rows_per_dst=[8188, 8192, 6670, 8176, 8180, 7982, 8162, 8192] dup_rows_per_dst=[146, 48, 0, 0, 0, 910, 620, 70] combine_rows_per_src=[8188, 8192, 6670, 8176, 8180, 7982, 8162, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=397135872 dispatch_meta_bytes=225288 combine_payload_bytes=397135872 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=397361160 ingress_bytes=617336832 link_bytes=617336832 token_bytes=794497032 weight_movement_bytes=220200960 total_nvlink_bytes=1014697992 weight_to_token_ratio=0.277158 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=19840 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.073650285e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=409966592 dispatch_meta_bytes=234584 combine_payload_bytes=409966592 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=410201176 ingress_bytes=630167552 link_bytes=630167552 token_bytes=820167768 weight_movement_bytes=220200960 total_nvlink_bytes=1040368728 weight_to_token_ratio=0.268483 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=28416 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.095965018e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=355862528 dispatch_meta_bytes=229296 combine_payload_bytes=355862528 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=356091824 ingress_bytes=576063488 link_bytes=576063488 token_bytes=711954352 weight_movement_bytes=220200960 total_nvlink_bytes=932155312 weight_to_token_ratio=0.309291 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.001869152e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64334 payload_rows_remote=56152 dup_rows=1202 dup_rows_remote=1202 dedup_ratio=0.018341 router_tokens_per_expert=[87, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 143, 23, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3831, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 2, 0, 0, 2836, 0, 0, 975, 0, 0, 27, 0, 314, 0, 0, 4084, 0, 0, 1, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[8190, 8192, 7674, 8182, 8182, 7532, 8190, 8192] dup_rows_per_dst=[112, 332, 0, 0, 0, 94, 660, 4] combine_rows_per_src=[8190, 8192, 7674, 8182, 8182, 7532, 8190, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=402354176 dispatch_meta_bytes=232856 combine_payload_bytes=402354176 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=402587032 ingress_bytes=512454656 link_bytes=512454656 token_bytes=804941208 weight_movement_bytes=110100480 total_nvlink_bytes=915041688 weight_to_token_ratio=0.136781 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=30592 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.912429311e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7147334072 prefetch_bytes=1101004800 reduce_bytes=1211105280 weight_movement_bytes=2312110080 egress_bytes=3574802872 ingress_bytes=5884641280 link_bytes=5884641280 total_nvlink_bytes=9459444152 weight_to_token_ratio=0.323493 prefetch_share_of_ingress=0.187098 dedup_ratio=0.126440 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=11 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050000 gemm_inner_pad_upper_bound=7112 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.023435904e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=358514688 dispatch_meta_bytes=229376 combine_payload_bytes=358514688 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=358744064 ingress_bytes=468615168 link_bytes=468615168 token_bytes=717258752 weight_movement_bytes=110100480 total_nvlink_bytes=827359232 weight_to_token_ratio=0.153502 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.149988511e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=408862720 dispatch_meta_bytes=233776 combine_payload_bytes=408862720 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=409096496 ingress_bytes=518963200 link_bytes=518963200 token_bytes=817959216 weight_movement_bytes=110100480 total_nvlink_bytes=928059696 weight_to_token_ratio=0.134604 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=20224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.025623596e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=402497536 dispatch_meta_bytes=229416 combine_payload_bytes=402497536 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=402726952 ingress_bytes=622698496 link_bytes=622698496 token_bytes=805224488 weight_movement_bytes=220200960 total_nvlink_bytes=1025425448 weight_to_token_ratio=0.273465 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.082975101e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7056508216 prefetch_bytes=990904320 reduce_bytes=880803840 weight_movement_bytes=1871708160 egress_bytes=3529379128 ingress_bytes=5398837248 link_bytes=5398837248 total_nvlink_bytes=8928216376 weight_to_token_ratio=0.265246 prefetch_share_of_ingress=0.183540 dedup_ratio=0.132437 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=8 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.049632 gemm_inner_pad_upper_bound=6604 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.389465930e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6665153608 prefetch_bytes=1101004800 reduce_bytes=550502400 weight_movement_bytes=1651507200 egress_bytes=3333682248 ingress_bytes=4982978560 link_bytes=4982978560 total_nvlink_bytes=8316660808 weight_to_token_ratio=0.247782 prefetch_share_of_ingress=0.220953 dedup_ratio=0.172458 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=5 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048529 gemm_inner_pad_upper_bound=7112 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.666219275e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63976 payload_rows_remote=55784 dup_rows=1560 dup_rows_remote=992 dedup_ratio=0.023804 router_tokens_per_expert=[147, 0, 0, 0, 0, 3966, 0, 0, 0, 0, 0, 213, 71, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3783, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4088, 13, 0, 0, 2303, 0, 0, 1362, 0, 0, 48, 0, 405, 0, 0, 4074, 0, 0, 2, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[8180, 8192, 7584, 8182, 8176, 7284, 8186, 8192] dup_rows_per_dst=[46, 568, 0, 0, 0, 72, 868, 6] combine_rows_per_src=[8180, 8192, 7584, 8182, 8176, 7284, 8186, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7277585616 prefetch_bytes=990904320 reduce_bytes=440401920 weight_movement_bytes=1431306240 egress_bytes=3639940304 ingress_bytes=5068951552 link_bytes=5069180928 total_nvlink_bytes=8708891856 weight_to_token_ratio=0.196673 prefetch_share_of_ingress=0.195485 dedup_ratio=0.116022 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=4 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048529 gemm_inner_pad_upper_bound=7112 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.816139371e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6805517168 prefetch_bytes=770703360 reduce_bytes=1431306240 weight_movement_bytes=2202009600 egress_bytes=3403885424 ingress_bytes=5603641344 link_bytes=5603641344 total_nvlink_bytes=9007526768 weight_to_token_ratio=0.323562 prefetch_share_of_ingress=0.137536 dedup_ratio=0.161539 prefetch_slots=14 unused_prefetch_slots=146 slot_utilization=0.087500 reduce_slots=13 remote_demand=7 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048162 gemm_inner_pad_upper_bound=6731 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.745653938e-05 prefetch_bytes_per_gemm_flop=1.340379902e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=393465856 dispatch_meta_bytes=229408 combine_payload_bytes=393465856 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=393695264 ingress_bytes=613666816 link_bytes=613666816 token_bytes=787161120 weight_movement_bytes=220200960 total_nvlink_bytes=1007362080 weight_to_token_ratio=0.279741 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.067267524e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=373538816 dispatch_meta_bytes=228504 combine_payload_bytes=373538816 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=373767320 ingress_bytes=483639296 link_bytes=483639296 token_bytes=747306136 weight_movement_bytes=110100480 total_nvlink_bytes=857406616 weight_to_token_ratio=0.147330 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=31872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.411282807e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6994348200 prefetch_bytes=990904320 reduce_bytes=880803840 weight_movement_bytes=1871708160 egress_bytes=3498313896 ingress_bytes=5367742464 link_bytes=5367742464 total_nvlink_bytes=8866056360 weight_to_token_ratio=0.267603 prefetch_share_of_ingress=0.184604 dedup_ratio=0.143658 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=8 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050000 gemm_inner_pad_upper_bound=6477 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.335386987e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=398297088 dispatch_meta_bytes=229440 combine_payload_bytes=398297088 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=398526528 ingress_bytes=728598528 link_bytes=728598528 token_bytes=796823616 weight_movement_bytes=330301440 total_nvlink_bytes=1127125056 weight_to_token_ratio=0.414523 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.267152674e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7162594848 prefetch_bytes=1101004800 reduce_bytes=440401920 weight_movement_bytes=1541406720 egress_bytes=3582451232 ingress_bytes=5121550336 link_bytes=5121550336 total_nvlink_bytes=8704001568 weight_to_token_ratio=0.215202 prefetch_share_of_ingress=0.214975 dedup_ratio=0.133673 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=4 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048162 gemm_inner_pad_upper_bound=7239 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.907218385e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55500 payload_rows_remote=47308 dup_rows=10036 dup_rows_remote=9650 dedup_ratio=0.153137 router_tokens_per_expert=[79, 0, 0, 0, 0, 4041, 0, 0, 0, 0, 0, 8, 33, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3398, 0, 0, 4067, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4075, 6, 0, 0, 3624, 0, 0, 821, 0, 0, 0, 0, 261, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 86, 0] payload_rows_per_dst=[8192, 8192, 8094, 8138, 7812, 8046, 678, 6348] dup_rows_per_dst=[386, 82, 5958, 1938, 0, 1598, 6, 68] combine_rows_per_src=[8192, 8192, 8094, 8138, 7812, 8046, 678, 6348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59568 payload_rows_remote=51376 dup_rows=5968 dup_rows_remote=5830 dedup_ratio=0.091064 router_tokens_per_expert=[145, 0, 0, 0, 0, 3975, 0, 0, 0, 0, 0, 23, 46, 0, 0, 0, 0, 0, 0, 0, 0, 11, 2979, 0, 0, 4055, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4083, 3, 0, 0, 3200, 0, 0, 1396, 0, 0, 3, 0, 391, 0, 0, 4032, 0, 0, 0, 0, 0, 0, 0, 239, 0] payload_rows_per_dst=[8182, 8192, 7812, 8110, 8166, 2798, 8118, 8190] dup_rows_per_dst=[58, 138, 4568, 0, 0, 0, 734, 470] combine_rows_per_src=[8182, 8192, 7812, 8110, 8166, 2798, 8118, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7048149368 prefetch_bytes=990904320 reduce_bytes=1981808640 weight_movement_bytes=2972712960 egress_bytes=3525206392 ingress_bytes=6495655936 link_bytes=6495655936 total_nvlink_bytes=10020862328 weight_to_token_ratio=0.421772 prefetch_share_of_ingress=0.152549 dedup_ratio=0.138348 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=18 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048897 gemm_inner_pad_upper_bound=7112 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.129701402e-04 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6233822528 prefetch_bytes=990904320 reduce_bytes=770703360 weight_movement_bytes=1761607680 egress_bytes=3117978944 ingress_bytes=4877451264 link_bytes=4877451264 total_nvlink_bytes=7995430208 weight_to_token_ratio=0.282589 prefetch_share_of_ingress=0.203160 dedup_ratio=0.219177 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=7 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048897 gemm_inner_pad_upper_bound=7112 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.482689951e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64048 payload_rows_remote=55864 dup_rows=1488 dup_rows_remote=1488 dedup_ratio=0.022705 router_tokens_per_expert=[51, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 291, 26, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3756, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 4, 0, 0, 2939, 0, 0, 805, 0, 0, 9, 0, 343, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7522, 8184, 8184, 7390, 8192, 8192] dup_rows_per_dst=[56, 634, 0, 0, 0, 106, 692, 0] combine_rows_per_src=[8192, 8192, 7522, 8184, 8184, 7390, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53896 payload_rows_remote=45708 dup_rows=11640 dup_rows_remote=11608 dedup_ratio=0.177612 router_tokens_per_expert=[72, 0, 0, 0, 0, 4038, 0, 0, 0, 0, 0, 407, 43, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3927, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4084, 14, 0, 0, 2549, 0, 0, 943, 0, 0, 14, 0, 304, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8192, 8096, 8192, 3862, 8138, 1062, 8166] dup_rows_per_dst=[32, 900, 4072, 20, 0, 6554, 62, 0] combine_rows_per_src=[8188, 8192, 8096, 8192, 3862, 8138, 1062, 8166] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7201600128 prefetch_bytes=880803840 reduce_bytes=1761607680 weight_movement_bytes=2642411520 egress_bytes=3601945216 ingress_bytes=6242066432 link_bytes=6242066432 total_nvlink_bytes=9844011648 weight_to_token_ratio=0.366920 prefetch_share_of_ingress=0.141108 dedup_ratio=0.123425 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=16 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.047794 gemm_inner_pad_upper_bound=7239 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.085598017e-04 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=399859712 dispatch_meta_bytes=227104 combine_payload_bytes=399859712 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=400086816 ingress_bytes=509960192 link_bytes=509960192 token_bytes=799946528 weight_movement_bytes=110100480 total_nvlink_bytes=910047008 weight_to_token_ratio=0.137635 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=30208 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.869046479e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=400433152 dispatch_meta_bytes=229408 combine_payload_bytes=400433152 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=400662560 ingress_bytes=840835072 link_bytes=840835072 token_bytes=801095712 weight_movement_bytes=440401920 total_nvlink_bytes=1241497632 weight_to_token_ratio=0.549749 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=29952 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.462350484e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=327634944 dispatch_meta_bytes=229264 combine_payload_bytes=327634944 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=327864208 ingress_bytes=547835904 link_bytes=547835904 token_bytes=655499152 weight_movement_bytes=220200960 total_nvlink_bytes=875700112 weight_to_token_ratio=0.335929 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.527767406e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6969586136 prefetch_bytes=1101004800 reduce_bytes=550502400 weight_movement_bytes=1651507200 egress_bytes=3485923800 ingress_bytes=5135169536 link_bytes=5135169536 total_nvlink_bytes=8621093336 weight_to_token_ratio=0.236959 prefetch_share_of_ingress=0.214405 dedup_ratio=0.139752 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=5 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050368 gemm_inner_pad_upper_bound=6477 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.930904413e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7257792712 prefetch_bytes=990904320 reduce_bytes=1541406720 weight_movement_bytes=2532311040 egress_bytes=3630053576 ingress_bytes=6160050176 link_bytes=6160050176 total_nvlink_bytes=9790103752 weight_to_token_ratio=0.348909 prefetch_share_of_ingress=0.160860 dedup_ratio=0.120447 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=14 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.049265 gemm_inner_pad_upper_bound=7112 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.071334041e-04 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6496694464 prefetch_bytes=990904320 reduce_bytes=1211105280 weight_movement_bytes=2202009600 egress_bytes=3249432768 ingress_bytes=5449271296 link_bytes=5449271296 total_nvlink_bytes=8698704064 weight_to_token_ratio=0.338943 prefetch_share_of_ingress=0.181842 dedup_ratio=0.189154 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=11 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050000 gemm_inner_pad_upper_bound=6731 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.477179035e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none - [2026-08-26 15:11:35] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 369.5 | throughput per GPU (TFLOP/s/GPU): 334.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.542182E+00 | loss scale: 1.0 | grad norm: 4.303 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43828 payload_rows_remote=38640 dup_rows=21708 dup_rows_remote=19950 dedup_ratio=0.331238 router_tokens_per_expert=[1046, 967, 1244, 353, 448, 800, 644, 72, 643, 526, 350, 539, 21, 796, 383, 814, 514, 515, 44, 70, 581, 496, 689, 215, 544, 697, 705, 449, 669, 635, 730, 22, 630, 699, 773, 29, 1231, 34, 274, 323, 497, 797, 758, 828, 6, 914, 96, 544, 382, 289, 684, 415, 423, 1057, 238, 475, 627, 530, 427, 187, 293, 659, 409, 19] payload_rows_per_dst=[6076, 5402, 5188, 6018, 5340, 5862, 5396, 4546] dup_rows_per_dst=[4374, 2742, 1758, 2884, 2646, 3018, 2530, 1756] combine_rows_per_src=[6076, 5402, 5188, 6018, 5340, 5862, 5396, 4546] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43904 payload_rows_remote=38396 dup_rows=21632 dup_rows_remote=18478 dedup_ratio=0.330078 router_tokens_per_expert=[618, 1230, 404, 545, 306, 990, 1139, 210, 387, 509, 445, 287, 13, 909, 207, 507, 449, 528, 28, 63, 884, 145, 540, 319, 633, 506, 572, 828, 771, 308, 622, 19, 676, 618, 768, 25, 814, 0, 1163, 267, 564, 622, 510, 638, 3, 518, 429, 457, 411, 441, 715, 791, 393, 791, 665, 437, 897, 1150, 649, 192, 329, 462, 444, 8] payload_rows_per_dst=[5348, 4972, 5694, 5832, 5508, 5214, 5870, 5466] dup_rows_per_dst=[2364, 2268, 2678, 2686, 3154, 2268, 3418, 2796] combine_rows_per_src=[5348, 4972, 5694, 5832, 5508, 5214, 5870, 5466] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44070 payload_rows_remote=38568 dup_rows=21466 dup_rows_remote=19436 dedup_ratio=0.327545 router_tokens_per_expert=[998, 1184, 482, 201, 403, 674, 555, 75, 420, 390, 209, 253, 47, 892, 103, 793, 662, 731, 30, 78, 554, 616, 556, 185, 339, 400, 535, 567, 814, 405, 680, 26, 813, 967, 478, 17, 841, 71, 808, 680, 483, 602, 529, 784, 6, 962, 110, 592, 564, 493, 356, 1150, 397, 654, 192, 690, 1035, 780, 439, 411, 508, 1215, 267, 17] payload_rows_per_dst=[4736, 4488, 6120, 5502, 5948, 5416, 5668, 6192] dup_rows_per_dst=[2040, 1726, 3072, 2030, 3402, 2720, 3324, 3152] combine_rows_per_src=[4736, 4488, 6120, 5502, 5948, 5416, 5668, 6192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44376 payload_rows_remote=38800 dup_rows=21160 dup_rows_remote=18590 dedup_ratio=0.322876 router_tokens_per_expert=[1107, 740, 478, 782, 656, 710, 397, 81, 254, 412, 701, 470, 68, 708, 236, 677, 1009, 539, 78, 61, 618, 499, 600, 386, 457, 458, 518, 499, 820, 412, 741, 29, 433, 709, 400, 19, 865, 7, 1129, 431, 547, 579, 536, 700, 9, 719, 139, 844, 628, 439, 659, 611, 417, 959, 262, 576, 725, 666, 608, 340, 351, 826, 425, 9] payload_rows_per_dst=[4964, 5650, 5988, 5614, 5312, 5576, 5802, 5470] dup_rows_per_dst=[2038, 2822, 3072, 2254, 2674, 2570, 3300, 2430] combine_rows_per_src=[4964, 5650, 5988, 5614, 5312, 5576, 5802, 5470] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43870 payload_rows_remote=38944 dup_rows=21666 dup_rows_remote=19558 dedup_ratio=0.330597 router_tokens_per_expert=[361, 923, 682, 987, 136, 1146, 850, 80, 849, 690, 840, 615, 21, 1020, 288, 1035, 798, 412, 16, 40, 775, 196, 678, 319, 306, 389, 589, 549, 649, 404, 303, 6, 935, 1368, 584, 51, 1157, 0, 874, 392, 1064, 458, 426, 436, 23, 472, 296, 693, 533, 296, 142, 800, 173, 561, 175, 837, 498, 522, 468, 182, 238, 570, 577, 15] payload_rows_per_dst=[4630, 7120, 5698, 4810, 5878, 5380, 4926, 5428] dup_rows_per_dst=[1562, 5888, 2616, 1580, 3188, 2356, 2108, 2368] combine_rows_per_src=[4630, 7120, 5698, 4810, 5878, 5380, 4926, 5428] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44474 payload_rows_remote=38640 dup_rows=21062 dup_rows_remote=18664 dedup_ratio=0.321381 router_tokens_per_expert=[762, 1048, 269, 551, 1029, 647, 888, 226, 360, 458, 686, 234, 3, 1029, 481, 512, 905, 523, 117, 85, 766, 772, 682, 381, 452, 942, 434, 626, 843, 374, 570, 18, 672, 663, 1088, 12, 798, 25, 997, 185, 394, 757, 384, 764, 1, 761, 146, 565, 331, 205, 701, 339, 384, 841, 253, 511, 933, 678, 487, 213, 453, 304, 215, 35] payload_rows_per_dst=[5192, 5840, 6482, 5832, 4628, 5270, 5396, 5834] dup_rows_per_dst=[2044, 2980, 4076, 2900, 1454, 2664, 2546, 2398] combine_rows_per_src=[5192, 5840, 6482, 5832, 4628, 5270, 5396, 5834] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7125488192 prefetch_bytes=990904320 reduce_bytes=1101004800 weight_movement_bytes=2091909120 egress_bytes=3563895360 ingress_bytes=5653501952 link_bytes=5653501952 total_nvlink_bytes=9217397312 weight_to_token_ratio=0.293581 prefetch_share_of_ingress=0.175273 dedup_ratio=0.135947 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=10 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.047794 gemm_inner_pad_upper_bound=7493 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.832369736e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=339103744 dispatch_meta_bytes=227832 combine_payload_bytes=339103744 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=339331576 ingress_bytes=559304704 link_bytes=559304704 token_bytes=678435320 weight_movement_bytes=220200960 total_nvlink_bytes=898636280 weight_to_token_ratio=0.324572 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.727228701e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=368263168 dispatch_meta_bytes=228824 combine_payload_bytes=368263168 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=368491992 ingress_bytes=478363648 link_bytes=478363648 token_bytes=736755160 weight_movement_bytes=110100480 total_nvlink_bytes=846855640 weight_to_token_ratio=0.149440 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.319530612e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6446293712 prefetch_bytes=1211105280 reduce_bytes=880803840 weight_movement_bytes=2091909120 egress_bytes=3224220368 ingress_bytes=5313982464 link_bytes=5313982464 total_nvlink_bytes=8538202832 weight_to_token_ratio=0.324513 prefetch_share_of_ingress=0.227909 dedup_ratio=0.194769 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=8 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048529 gemm_inner_pad_upper_bound=6985 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.241889505e-05 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6870550288 prefetch_bytes=1101004800 reduce_bytes=660602880 weight_movement_bytes=1761607680 egress_bytes=3436404496 ingress_bytes=5195753472 link_bytes=5195753472 total_nvlink_bytes=8632157968 weight_to_token_ratio=0.256400 prefetch_share_of_ingress=0.211905 dedup_ratio=0.154852 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=6 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050368 gemm_inner_pad_upper_bound=6223 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.036269842e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none + [2026-08-26 15:59:21] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 630.6 | throughput per GPU (TFLOP/s/GPU): 195.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.540568E+00 | loss scale: 1.0 | grad norm: 4.298 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44070 payload_rows_remote=38598 dup_rows=21466 dup_rows_remote=19450 dedup_ratio=0.327545 router_tokens_per_expert=[1030, 1175, 480, 197, 373, 681, 530, 64, 418, 396, 207, 234, 43, 850, 106, 785, 619, 772, 25, 98, 549, 653, 529, 186, 336, 396, 538, 565, 844, 371, 664, 30, 817, 947, 450, 16, 825, 76, 822, 728, 429, 626, 471, 740, 6, 980, 153, 620, 572, 537, 361, 1125, 385, 637, 181, 702, 1127, 791, 438, 441, 518, 1205, 283, 15] payload_rows_per_dst=[4732, 4426, 6124, 5472, 5930, 5388, 5682, 6316] dup_rows_per_dst=[1978, 1652, 3088, 2016, 3432, 2662, 3318, 3320] combine_rows_per_src=[4732, 4426, 6124, 5472, 5930, 5388, 5682, 6316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44310 payload_rows_remote=38826 dup_rows=21226 dup_rows_remote=18686 dedup_ratio=0.323883 router_tokens_per_expert=[1147, 736, 462, 777, 622, 712, 385, 73, 254, 419, 701, 443, 54, 698, 253, 650, 969, 580, 58, 67, 606, 535, 571, 402, 458, 453, 514, 506, 845, 383, 746, 31, 429, 708, 395, 21, 858, 10, 1129, 480, 449, 604, 479, 678, 9, 728, 191, 874, 640, 465, 686, 599, 413, 954, 246, 565, 775, 690, 601, 341, 359, 833, 439, 10] payload_rows_per_dst=[4914, 5586, 5968, 5610, 5338, 5484, 5834, 5576] dup_rows_per_dst=[2018, 2782, 3080, 2262, 2722, 2540, 3302, 2520] combine_rows_per_src=[4914, 5586, 5968, 5610, 5338, 5484, 5834, 5576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43986 payload_rows_remote=38520 dup_rows=21550 dup_rows_remote=18406 dedup_ratio=0.328827 router_tokens_per_expert=[651, 1223, 393, 524, 288, 1015, 1139, 171, 388, 510, 439, 265, 12, 884, 223, 504, 409, 563, 27, 74, 870, 166, 522, 326, 636, 506, 570, 839, 790, 280, 618, 21, 683, 615, 745, 29, 791, 0, 1152, 290, 480, 655, 458, 604, 3, 535, 546, 476, 426, 468, 732, 769, 384, 775, 636, 445, 961, 1174, 644, 193, 337, 453, 455, 8] payload_rows_per_dst=[5064, 5268, 5676, 5820, 5466, 5260, 5874, 5558] dup_rows_per_dst=[1928, 2552, 2684, 2700, 3144, 2254, 3396, 2892] combine_rows_per_src=[5064, 5268, 5676, 5820, 5466, 5260, 5874, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44568 payload_rows_remote=38678 dup_rows=20968 dup_rows_remote=18506 dedup_ratio=0.319946 router_tokens_per_expert=[794, 1027, 266, 518, 981, 649, 877, 218, 364, 458, 693, 226, 2, 1006, 507, 504, 852, 561, 94, 98, 757, 808, 669, 392, 445, 948, 430, 626, 881, 339, 565, 24, 680, 655, 1092, 14, 766, 24, 982, 196, 338, 789, 350, 735, 1, 790, 198, 584, 339, 211, 733, 328, 378, 824, 241, 507, 997, 705, 484, 224, 461, 308, 214, 41] payload_rows_per_dst=[5206, 5830, 6436, 5744, 4880, 5424, 5158, 5890] dup_rows_per_dst=[2102, 2988, 4080, 2772, 1752, 2800, 2012, 2462] combine_rows_per_src=[5206, 5830, 6436, 5744, 4880, 5424, 5158, 5890] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43870 payload_rows_remote=38618 dup_rows=21666 dup_rows_remote=19838 dedup_ratio=0.330597 router_tokens_per_expert=[1083, 950, 1239, 341, 429, 830, 632, 59, 651, 540, 363, 510, 17, 774, 400, 798, 496, 572, 36, 75, 561, 528, 667, 224, 537, 698, 707, 449, 678, 601, 731, 26, 633, 699, 752, 30, 1216, 36, 281, 353, 412, 838, 706, 805, 6, 948, 118, 556, 402, 301, 710, 391, 423, 1041, 228, 475, 671, 537, 422, 176, 296, 662, 422, 20] payload_rows_per_dst=[6040, 5420, 5252, 6006, 5338, 5798, 5396, 4620] dup_rows_per_dst=[4324, 2686, 1828, 2848, 2662, 2980, 2546, 1792] combine_rows_per_src=[6040, 5420, 5252, 6006, 5338, 5798, 5396, 4620] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43428 payload_rows_remote=38340 dup_rows=22108 dup_rows_remote=19432 dedup_ratio=0.337341 router_tokens_per_expert=[375, 900, 677, 959, 115, 1155, 842, 72, 852, 698, 839, 585, 14, 1006, 300, 1012, 744, 457, 13, 48, 755, 214, 671, 325, 307, 383, 590, 553, 668, 403, 297, 6, 931, 1361, 569, 48, 1130, 0, 885, 429, 960, 531, 372, 419, 23, 483, 417, 723, 549, 325, 148, 777, 159, 550, 166, 849, 542, 534, 467, 185, 245, 563, 578, 15] payload_rows_per_dst=[4546, 7140, 5684, 4826, 6112, 5490, 5088, 4542] dup_rows_per_dst=[1534, 5782, 2570, 1588, 3876, 2366, 2676, 1716] combine_rows_per_src=[4546, 7140, 5684, 4826, 6112, 5490, 5088, 4542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44744 payload_rows_remote=39346 dup_rows=20792 dup_rows_remote=18384 dedup_ratio=0.317261 router_tokens_per_expert=[620, 993, 332, 201, 548, 1131, 642, 61, 897, 485, 621, 348, 16, 569, 202, 765, 749, 372, 69, 103, 580, 373, 602, 309, 422, 648, 418, 753, 826, 391, 741, 21, 821, 563, 801, 28, 648, 1, 716, 418, 655, 712, 576, 764, 1, 1035, 185, 613, 549, 433, 437, 490, 334, 669, 202, 723, 834, 1320, 323, 581, 405, 501, 594, 28] payload_rows_per_dst=[5972, 5398, 4992, 5944, 5318, 5888, 5174, 6058] dup_rows_per_dst=[3084, 2408, 1322, 2496, 2674, 3194, 2500, 3114] combine_rows_per_src=[5972, 5398, 4992, 5944, 5318, 5888, 5174, 6058] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43306 payload_rows_remote=36750 dup_rows=22230 dup_rows_remote=16692 dedup_ratio=0.339203 router_tokens_per_expert=[821, 815, 543, 380, 1003, 1057, 683, 81, 325, 496, 317, 287, 18, 635, 188, 543, 509, 391, 15, 92, 742, 533, 690, 404, 403, 956, 525, 892, 691, 282, 734, 50, 1096, 615, 673, 7, 743, 9, 729, 421, 360, 920, 448, 684, 0, 793, 161, 638, 627, 192, 650, 659, 211, 681, 214, 556, 986, 1152, 590, 219, 420, 848, 356, 9] payload_rows_per_dst=[6556, 4210, 5148, 6064, 4736, 5490, 5186, 5916] dup_rows_per_dst=[5538, 1408, 1604, 3368, 2156, 2518, 2394, 3244] combine_rows_per_src=[6556, 4210, 5148, 6064, 4736, 5490, 5186, 5916] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=276971520 dispatch_meta_bytes=234360 combine_payload_bytes=276971520 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277205880 ingress_bytes=387072000 link_bytes=387072000 token_bytes=554177400 weight_movement_bytes=110100480 total_nvlink_bytes=664277880 weight_to_token_ratio=0.198674 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=5760 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.731818704e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44686 payload_rows_remote=39220 dup_rows=20850 dup_rows_remote=18384 dedup_ratio=0.318146 router_tokens_per_expert=[592, 1000, 358, 209, 588, 1118, 659, 72, 883, 485, 636, 387, 18, 591, 192, 774, 786, 350, 84, 87, 599, 354, 627, 298, 424, 663, 424, 751, 804, 410, 746, 17, 817, 559, 815, 26, 658, 1, 708, 380, 755, 678, 612, 801, 1, 996, 133, 596, 526, 399, 432, 526, 343, 679, 210, 736, 737, 1274, 327, 567, 387, 498, 578, 27] payload_rows_per_dst=[5998, 5466, 5038, 5946, 5314, 5926, 5146, 5852] dup_rows_per_dst=[3194, 2466, 1332, 2532, 2614, 3218, 2556, 2938] combine_rows_per_src=[5998, 5466, 5038, 5946, 5314, 5926, 5146, 5852] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=275222528 dispatch_meta_bytes=227496 combine_payload_bytes=275222528 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=275450024 ingress_bytes=715624448 link_bytes=715624448 token_bytes=550672552 weight_movement_bytes=440401920 total_nvlink_bytes=991074472 weight_to_token_ratio=0.799753 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.244588615e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=276455424 dispatch_meta_bytes=232016 combine_payload_bytes=276455424 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276687440 ingress_bytes=386555904 link_bytes=386555904 token_bytes=553142864 weight_movement_bytes=110100480 total_nvlink_bytes=663243344 weight_to_token_ratio=0.199045 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=6144 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.722842946e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43624 payload_rows_remote=37212 dup_rows=21912 dup_rows_remote=17348 dedup_ratio=0.334351 router_tokens_per_expert=[774, 824, 556, 393, 1080, 1044, 696, 88, 325, 491, 307, 313, 20, 656, 185, 556, 539, 360, 19, 79, 753, 497, 715, 400, 406, 963, 523, 901, 669, 310, 740, 46, 1088, 618, 675, 7, 760, 8, 722, 393, 431, 870, 506, 715, 0, 775, 129, 613, 618, 179, 636, 675, 210, 687, 225, 545, 903, 1125, 601, 208, 406, 853, 350, 9] payload_rows_per_dst=[6412, 4258, 5120, 5986, 5332, 5574, 5158, 5784] dup_rows_per_dst=[4564, 1448, 1604, 3130, 3144, 2504, 2392, 3126] combine_rows_per_src=[6412, 4258, 5120, 5986, 5332, 5574, 5158, 5784] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278118400 dispatch_meta_bytes=229560 combine_payload_bytes=278118400 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278347960 ingress_bytes=388218880 link_bytes=388218880 token_bytes=556466360 weight_movement_bytes=110100480 total_nvlink_bytes=666566840 weight_to_token_ratio=0.197856 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=6144 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.751764834e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279150592 dispatch_meta_bytes=234008 combine_payload_bytes=279150592 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279384600 ingress_bytes=389251072 link_bytes=389251072 token_bytes=558535192 weight_movement_bytes=110100480 total_nvlink_bytes=668635672 weight_to_token_ratio=0.197124 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.769716350e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=276971520 dispatch_meta_bytes=229216 combine_payload_bytes=276971520 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277200736 ingress_bytes=387072000 link_bytes=387072000 token_bytes=554172256 weight_movement_bytes=110100480 total_nvlink_bytes=664272736 weight_to_token_ratio=0.198676 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=6784 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.731818704e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=281128960 dispatch_meta_bytes=230416 combine_payload_bytes=281128960 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281359376 ingress_bytes=391229440 link_bytes=391229440 token_bytes=562488336 weight_movement_bytes=110100480 total_nvlink_bytes=672588816 weight_to_token_ratio=0.195738 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=6656 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.804123423e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=266735616 dispatch_meta_bytes=218240 combine_payload_bytes=266735616 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=266953856 ingress_bytes=707137536 link_bytes=707137536 token_bytes=533689472 weight_movement_bytes=440401920 total_nvlink_bytes=974091392 weight_to_token_ratio=0.825203 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.229828479e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51126 payload_rows_remote=44236 dup_rows=14410 dup_rows_remote=13194 dedup_ratio=0.219879 router_tokens_per_expert=[0, 0, 0, 10, 269, 0, 1914, 1, 1523, 753, 183, 0, 1, 3325, 0, 3540, 52, 944, 0, 0, 106, 2951, 0, 0, 0, 0, 143, 3039, 1172, 14, 0, 144, 0, 0, 0, 0, 22, 115, 0, 4, 0, 59, 170, 0, 0, 0, 2512, 0, 0, 1209, 1134, 0, 2714, 0, 0, 172, 1167, 152, 0, 96, 99, 1610, 1408, 41] payload_rows_per_dst=[5922, 7876, 6890, 6732, 6802, 5302, 6920, 4682] dup_rows_per_dst=[1686, 4124, 1216, 2292, 130, 180, 3538, 1244] combine_rows_per_src=[5922, 7876, 6890, 6732, 6802, 5302, 6920, 4682] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51224 payload_rows_remote=44422 dup_rows=14312 dup_rows_remote=14300 dedup_ratio=0.218384 router_tokens_per_expert=[0, 0, 0, 1, 216, 0, 1744, 35, 832, 657, 39, 0, 1, 3355, 0, 3284, 1, 1362, 0, 0, 52, 3375, 0, 0, 0, 0, 385, 2103, 1962, 11, 0, 129, 0, 0, 0, 0, 7, 45, 0, 0, 0, 173, 17, 0, 0, 0, 3136, 0, 0, 1522, 525, 0, 2732, 0, 0, 76, 1333, 274, 0, 105, 109, 1886, 1241, 43] payload_rows_per_dst=[5952, 7370, 7568, 4682, 6802, 7366, 5766, 5718] dup_rows_per_dst=[1812, 2256, 2012, 1258, 12, 2526, 1818, 2618] combine_rows_per_src=[5952, 7370, 7568, 4682, 6802, 7366, 5766, 5718] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52102 payload_rows_remote=46030 dup_rows=13434 dup_rows_remote=11492 dedup_ratio=0.204987 router_tokens_per_expert=[0, 0, 0, 1, 194, 0, 1929, 26, 1190, 548, 116, 0, 2, 3547, 0, 3372, 8, 1143, 0, 2, 20, 3602, 0, 0, 3, 0, 82, 2601, 1243, 7, 0, 71, 0, 0, 0, 0, 56, 36, 0, 10, 0, 62, 32, 7, 0, 0, 2701, 0, 0, 1470, 885, 0, 2510, 0, 0, 22, 1273, 202, 0, 221, 75, 1495, 1925, 79] payload_rows_per_dst=[5838, 7542, 7778, 6072, 7232, 5490, 6692, 5458] dup_rows_per_dst=[1452, 2914, 1772, 1942, 66, 114, 3082, 2092] combine_rows_per_src=[5838, 7542, 7778, 6072, 7232, 5490, 6692, 5458] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=276670464 dispatch_meta_bytes=232192 combine_payload_bytes=276670464 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276902656 ingress_bytes=386770944 link_bytes=386770944 token_bytes=553573120 weight_movement_bytes=110100480 total_nvlink_bytes=663673600 weight_to_token_ratio=0.198891 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.726582845e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278304768 dispatch_meta_bytes=230048 combine_payload_bytes=278304768 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278534816 ingress_bytes=388405248 link_bytes=388405248 token_bytes=556839584 weight_movement_bytes=110100480 total_nvlink_bytes=666940064 weight_to_token_ratio=0.197724 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.755006080e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=276111360 dispatch_meta_bytes=227704 combine_payload_bytes=276111360 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=276339064 ingress_bytes=826613760 link_bytes=826613760 token_bytes=552450424 weight_movement_bytes=550502400 total_nvlink_bytes=1102952824 weight_to_token_ratio=0.996474 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=6144 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.437617283e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=277243904 dispatch_meta_bytes=228736 combine_payload_bytes=277243904 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277472640 ingress_bytes=387344384 link_bytes=387344384 token_bytes=554716544 weight_movement_bytes=110100480 total_nvlink_bytes=664817024 weight_to_token_ratio=0.198481 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=6912 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.736555910e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=276813824 dispatch_meta_bytes=233824 combine_payload_bytes=276813824 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277047648 ingress_bytes=386914304 link_bytes=386914304 token_bytes=553861472 weight_movement_bytes=110100480 total_nvlink_bytes=663961952 weight_to_token_ratio=0.198787 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=5504 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.729076111e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=274821120 dispatch_meta_bytes=231088 combine_payload_bytes=274821120 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=275052208 ingress_bytes=384921600 link_bytes=384921600 token_bytes=549873328 weight_movement_bytes=110100480 total_nvlink_bytes=659973808 weight_to_token_ratio=0.200229 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=6144 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.694419711e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=282032128 dispatch_meta_bytes=230920 combine_payload_bytes=282032128 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=282263048 ingress_bytes=392132608 link_bytes=392132608 token_bytes=564295176 weight_movement_bytes=110100480 total_nvlink_bytes=674395656 weight_to_token_ratio=0.195112 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=6528 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.819831000e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=263424000 dispatch_meta_bytes=213768 combine_payload_bytes=263424000 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=263637768 ingress_bytes=593725440 link_bytes=593725440 token_bytes=527061768 weight_movement_bytes=330301440 total_nvlink_bytes=857363208 weight_to_token_ratio=0.626684 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=6528 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.032586191e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52274 payload_rows_remote=45324 dup_rows=13262 dup_rows_remote=11212 dedup_ratio=0.202362 router_tokens_per_expert=[0, 0, 0, 1, 56, 0, 1555, 1, 1156, 516, 130, 0, 96, 3369, 0, 2905, 10, 2449, 0, 2, 15, 3910, 0, 0, 3, 0, 37, 3313, 998, 5, 0, 144, 0, 0, 0, 0, 63, 84, 0, 10, 0, 22, 22, 6, 0, 0, 2559, 0, 0, 1038, 122, 0, 2466, 0, 0, 0, 298, 0, 0, 343, 625, 1959, 2452, 28] payload_rows_per_dst=[8032, 7224, 4936, 6950, 6892, 7312, 5994, 4934] dup_rows_per_dst=[3014, 2382, 16, 2050, 160, 2810, 1258, 1572] combine_rows_per_src=[8032, 7224, 4936, 6950, 6892, 7312, 5994, 4934] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46536 payload_rows_remote=39374 dup_rows=19000 dup_rows_remote=14922 dedup_ratio=0.289917 router_tokens_per_expert=[0, 0, 0, 9, 61, 0, 1327, 84, 1008, 424, 100, 0, 24, 3188, 0, 2806, 87, 2581, 0, 0, 127, 3123, 0, 0, 0, 0, 258, 3122, 2248, 26, 0, 318, 0, 0, 0, 0, 15, 92, 0, 0, 0, 39, 71, 8, 2, 0, 2549, 0, 0, 1007, 55, 1, 2358, 0, 0, 30, 273, 0, 0, 155, 695, 2545, 1922, 30] payload_rows_per_dst=[2944, 2916, 7834, 5006, 7772, 5230, 7672, 7162] dup_rows_per_dst=[18, 196, 4002, 694, 4430, 108, 5474, 4078] combine_rows_per_src=[2944, 2916, 7834, 5006, 7772, 5230, 7672, 7162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50268 payload_rows_remote=43754 dup_rows=15268 dup_rows_remote=13446 dedup_ratio=0.232971 router_tokens_per_expert=[0, 0, 0, 48, 90, 0, 1376, 3, 838, 485, 256, 0, 81, 2935, 0, 2699, 47, 2609, 0, 0, 29, 3356, 0, 0, 0, 0, 245, 3275, 1758, 3, 0, 535, 0, 0, 0, 0, 1, 253, 0, 0, 0, 55, 27, 0, 0, 0, 2665, 0, 0, 1077, 75, 0, 2183, 0, 0, 2, 377, 0, 0, 233, 618, 2089, 2435, 10] payload_rows_per_dst=[7154, 5908, 5388, 7270, 6482, 6514, 5538, 6014] dup_rows_per_dst=[2426, 1504, 148, 4362, 1202, 1822, 1136, 2668] combine_rows_per_src=[7154, 5908, 5388, 7270, 6482, 6514, 5538, 6014] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52038 payload_rows_remote=45812 dup_rows=13498 dup_rows_remote=13404 dedup_ratio=0.205963 router_tokens_per_expert=[0, 0, 0, 5, 57, 0, 1341, 1, 854, 639, 44, 0, 29, 3058, 0, 2719, 2, 2793, 0, 0, 56, 3590, 0, 0, 0, 0, 176, 2868, 1851, 8, 0, 319, 0, 0, 0, 0, 10, 92, 0, 0, 0, 83, 13, 0, 0, 0, 2908, 0, 0, 997, 84, 0, 2681, 0, 0, 2, 327, 0, 0, 120, 951, 2293, 1775, 22] payload_rows_per_dst=[7478, 6784, 5604, 6892, 6226, 7148, 6268, 5638] dup_rows_per_dst=[2510, 1786, 98, 3552, 94, 2410, 1260, 1788] combine_rows_per_src=[7478, 6784, 5604, 6892, 6226, 7148, 6268, 5638] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51046 payload_rows_remote=45418 dup_rows=14490 dup_rows_remote=13414 dedup_ratio=0.221100 router_tokens_per_expert=[0, 0, 0, 50, 100, 0, 1567, 0, 1363, 747, 199, 0, 55, 3006, 0, 3038, 38, 2418, 0, 0, 110, 3141, 0, 0, 0, 0, 49, 3524, 1065, 7, 0, 417, 0, 0, 0, 0, 20, 241, 0, 1, 0, 4, 165, 0, 0, 0, 2291, 0, 0, 856, 433, 0, 2783, 0, 0, 45, 268, 0, 1, 125, 664, 2100, 1855, 22] payload_rows_per_dst=[7066, 7566, 5628, 6296, 6208, 6666, 6482, 5134] dup_rows_per_dst=[2650, 3238, 1076, 2256, 328, 1964, 1752, 1226] combine_rows_per_src=[7066, 7566, 5628, 6296, 6208, 6666, 6482, 5134] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47060 payload_rows_remote=41096 dup_rows=18476 dup_rows_remote=16634 dedup_ratio=0.281921 router_tokens_per_expert=[0, 0, 0, 7, 117, 0, 1522, 16, 684, 665, 94, 0, 114, 2815, 0, 3070, 15, 2238, 0, 0, 26, 3483, 0, 0, 0, 0, 176, 3310, 1510, 1, 0, 353, 0, 0, 0, 0, 1, 76, 0, 1, 0, 29, 16, 0, 0, 0, 2516, 0, 0, 1002, 101, 0, 2278, 0, 0, 8, 709, 0, 3, 299, 657, 1953, 2864, 39] payload_rows_per_dst=[3294, 2876, 7996, 6588, 7750, 5102, 5964, 7490] dup_rows_per_dst=[30, 238, 3528, 3084, 4176, 20, 1842, 5558] combine_rows_per_src=[3294, 2876, 7996, 6588, 7750, 5102, 5964, 7490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49904 payload_rows_remote=42294 dup_rows=15632 dup_rows_remote=13382 dedup_ratio=0.238525 router_tokens_per_expert=[0, 0, 0, 15, 121, 0, 1189, 0, 1095, 577, 40, 0, 24, 3257, 0, 3135, 5, 2401, 0, 0, 43, 3605, 0, 0, 0, 0, 51, 3270, 1924, 6, 0, 417, 0, 0, 0, 0, 9, 95, 0, 0, 0, 59, 27, 0, 0, 0, 2819, 0, 0, 1166, 141, 0, 2210, 0, 0, 5, 321, 0, 0, 183, 568, 2239, 1716, 35] payload_rows_per_dst=[7610, 7690, 5772, 5060, 6640, 7046, 5670, 4416] dup_rows_per_dst=[2250, 5710, 2008, 828, 82, 2196, 1374, 1184] combine_rows_per_src=[7610, 7690, 5772, 5060, 6640, 7046, 5670, 4416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50938 payload_rows_remote=43832 dup_rows=14598 dup_rows_remote=12186 dedup_ratio=0.222748 router_tokens_per_expert=[0, 0, 0, 6, 40, 0, 1168, 8, 1363, 502, 154, 0, 32, 3422, 0, 2708, 21, 2182, 0, 5, 268, 3308, 0, 0, 1, 0, 280, 3275, 1552, 39, 0, 452, 1, 9, 0, 2, 8, 103, 2, 0, 0, 66, 24, 3, 0, 0, 2402, 0, 1, 1336, 74, 0, 2476, 0, 0, 39, 457, 6, 0, 188, 690, 1914, 2099, 82] payload_rows_per_dst=[7082, 7106, 7442, 3926, 7012, 7034, 6282, 5054] dup_rows_per_dst=[1978, 2412, 4060, 722, 82, 2154, 1570, 1620] combine_rows_per_src=[7082, 7106, 7442, 3926, 7012, 7034, 6282, 5054] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48882 payload_rows_remote=41494 dup_rows=16654 dup_rows_remote=12986 dedup_ratio=0.254120 router_tokens_per_expert=[0, 0, 0, 30, 198, 0, 1675, 84, 881, 492, 240, 0, 4, 3278, 0, 3285, 28, 1395, 0, 0, 45, 3136, 0, 0, 0, 0, 360, 2546, 2098, 3, 0, 186, 0, 0, 0, 0, 0, 161, 0, 0, 0, 111, 30, 0, 0, 0, 2841, 0, 0, 1462, 542, 0, 2153, 0, 0, 48, 1283, 163, 0, 147, 94, 1771, 1972, 26] payload_rows_per_dst=[5902, 4692, 7366, 6274, 6750, 7388, 3480, 7030] dup_rows_per_dst=[1614, 1168, 1842, 2964, 128, 3668, 624, 4646] combine_rows_per_src=[5902, 4692, 7366, 6274, 6750, 7388, 3480, 7030] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47094 payload_rows_remote=43350 dup_rows=18442 dup_rows_remote=17902 dedup_ratio=0.281403 router_tokens_per_expert=[0, 0, 0, 2, 287, 0, 1759, 51, 815, 668, 83, 0, 0, 3179, 0, 3353, 13, 1030, 0, 0, 29, 3298, 0, 0, 0, 0, 259, 2702, 1642, 3, 0, 144, 0, 0, 0, 0, 0, 43, 0, 2, 0, 112, 27, 0, 0, 0, 2591, 0, 0, 1524, 591, 0, 2211, 0, 0, 27, 1598, 197, 3, 217, 116, 1592, 2510, 90] payload_rows_per_dst=[5796, 2908, 7256, 5446, 7156, 7346, 3744, 7442] dup_rows_per_dst=[1586, 224, 1484, 1850, 2798, 3518, 540, 6442] combine_rows_per_src=[5796, 2908, 7256, 5446, 7156, 7346, 3744, 7442] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46626 payload_rows_remote=39842 dup_rows=18910 dup_rows_remote=15242 dedup_ratio=0.288544 router_tokens_per_expert=[0, 0, 0, 20, 175, 0, 1749, 157, 980, 345, 92, 0, 0, 3387, 0, 3201, 59, 1382, 0, 0, 123, 2898, 0, 0, 0, 0, 460, 2432, 2400, 34, 0, 128, 0, 0, 0, 0, 7, 50, 0, 0, 0, 115, 67, 8, 2, 0, 2758, 0, 0, 1533, 549, 1, 2315, 0, 0, 145, 1134, 209, 0, 136, 88, 2285, 1288, 56] payload_rows_per_dst=[6436, 2682, 3258, 5154, 7976, 7424, 6912, 6784] dup_rows_per_dst=[2336, 152, 216, 890, 5314, 3340, 2994, 3668] combine_rows_per_src=[6436, 2682, 3258, 5154, 7976, 7424, 6912, 6784] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=317083648 dispatch_meta_bytes=229720 combine_payload_bytes=317083648 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=317313368 ingress_bytes=537284608 link_bytes=537284608 token_bytes=634397016 weight_movement_bytes=220200960 total_nvlink_bytes=854597976 weight_to_token_ratio=0.347103 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=22656 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.344263014e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=318416896 dispatch_meta_bytes=234888 combine_payload_bytes=318416896 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=318651784 ingress_bytes=538617856 link_bytes=538617856 token_bytes=637068680 weight_movement_bytes=220200960 total_nvlink_bytes=857269640 weight_to_token_ratio=0.345647 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=27136 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.367450390e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=329943040 dispatch_meta_bytes=230088 combine_payload_bytes=329943040 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=330173128 ingress_bytes=550144000 link_bytes=550144000 token_bytes=660116168 weight_movement_bytes=220200960 total_nvlink_bytes=880317128 weight_to_token_ratio=0.333579 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.051471 gemm_active_groups=8 gemm_peak_group_rows=14464 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.567908991e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50832 payload_rows_remote=43372 dup_rows=14704 dup_rows_remote=11798 dedup_ratio=0.224365 router_tokens_per_expert=[0, 0, 0, 4, 170, 0, 1556, 69, 1348, 480, 136, 0, 0, 3530, 0, 3219, 20, 1009, 0, 6, 256, 2998, 1, 0, 1, 0, 415, 2571, 1716, 46, 0, 193, 1, 6, 0, 2, 7, 77, 2, 1, 0, 181, 23, 2, 0, 0, 2668, 0, 1, 1939, 567, 0, 2303, 0, 0, 123, 1488, 199, 0, 148, 40, 1610, 1494, 142] payload_rows_per_dst=[6360, 7460, 7058, 6506, 7204, 5552, 6782, 3910] dup_rows_per_dst=[2904, 2906, 1522, 3378, 48, 196, 3084, 666] combine_rows_per_src=[6360, 7460, 7058, 6506, 7204, 5552, 6782, 3910] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=297428992 dispatch_meta_bytes=217920 combine_payload_bytes=297428992 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=297646912 ingress_bytes=407529472 link_bytes=407529472 token_bytes=595075904 weight_movement_bytes=110100480 total_nvlink_bytes=705176384 weight_to_token_ratio=0.185019 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=22144 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.087607789e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=310732800 dispatch_meta_bytes=245008 combine_payload_bytes=310732800 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=310977808 ingress_bytes=530933760 link_bytes=530933760 token_bytes=621710608 weight_movement_bytes=220200960 total_nvlink_bytes=841911568 weight_to_token_ratio=0.354186 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=12544 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.233811322e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=285587456 dispatch_meta_bytes=220336 combine_payload_bytes=285587456 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=285807792 ingress_bytes=615888896 link_bytes=615888896 token_bytes=571395248 weight_movement_bytes=330301440 total_nvlink_bytes=901696688 weight_to_token_ratio=0.578061 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=10624 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.071132086e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50620 payload_rows_remote=43784 dup_rows=14916 dup_rows_remote=11812 dedup_ratio=0.227600 router_tokens_per_expert=[0, 0, 0, 6, 287, 0, 1585, 87, 1148, 659, 33, 0, 0, 3489, 0, 3458, 2, 966, 0, 0, 67, 3447, 0, 0, 0, 0, 183, 2560, 2167, 9, 0, 129, 0, 0, 0, 0, 12, 58, 0, 0, 0, 120, 52, 0, 0, 0, 2891, 0, 0, 1801, 725, 0, 2180, 0, 0, 74, 1159, 179, 0, 107, 51, 1776, 1229, 72] payload_rows_per_dst=[6836, 7740, 6952, 6768, 7106, 5988, 6554, 2676] dup_rows_per_dst=[3104, 3522, 1346, 3328, 12, 138, 3006, 460] combine_rows_per_src=[6836, 7740, 6952, 6768, 7106, 5988, 6554, 2676] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=310890496 dispatch_meta_bytes=220680 combine_payload_bytes=310890496 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=311111176 ingress_bytes=751292416 link_bytes=751292416 token_bytes=622001672 weight_movement_bytes=440401920 total_nvlink_bytes=1062403592 weight_to_token_ratio=0.708040 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=18304 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.306621078e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=313843712 dispatch_meta_bytes=222384 combine_payload_bytes=313843712 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=314066096 ingress_bytes=534044672 link_bytes=534044672 token_bytes=627909808 weight_movement_bytes=220200960 total_nvlink_bytes=848110768 weight_to_token_ratio=0.350689 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=14080 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.287915199e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57836 payload_rows_remote=50108 dup_rows=7700 dup_rows_remote=7638 dedup_ratio=0.117493 router_tokens_per_expert=[0, 0, 0, 8, 0, 11, 195, 0, 0, 9, 0, 0, 0, 0, 2, 0, 3847, 0, 0, 3861, 0, 34, 3985, 0, 0, 0, 1226, 728, 0, 926, 9, 0, 0, 0, 0, 0, 326, 3419, 1453, 0, 206, 0, 3691, 0, 0, 0, 0, 0, 0, 0, 1055, 0, 0, 0, 0, 0, 3736, 0, 0, 4007, 0, 0, 0, 34] payload_rows_per_dst=[7702, 7972, 7728, 7728, 3786, 7388, 8060, 7472] dup_rows_per_dst=[420, 20, 62, 4342, 318, 406, 2064, 68] combine_rows_per_src=[7702, 7972, 7728, 7728, 3786, 7388, 8060, 7472] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57972 payload_rows_remote=50066 dup_rows=7564 dup_rows_remote=4286 dedup_ratio=0.115417 router_tokens_per_expert=[0, 0, 0, 4, 6, 0, 167, 0, 0, 42, 0, 0, 0, 0, 5, 0, 3877, 0, 0, 3857, 0, 24, 3951, 0, 0, 0, 1547, 379, 0, 326, 0, 0, 0, 0, 0, 0, 523, 3396, 1672, 1, 226, 0, 3685, 0, 0, 0, 0, 0, 0, 0, 1522, 0, 0, 0, 0, 0, 3593, 0, 0, 3962, 0, 1, 0, 2] payload_rows_per_dst=[7764, 7902, 7718, 4042, 7906, 7390, 8062, 7188] dup_rows_per_dst=[344, 94, 44, 462, 3278, 432, 2906, 4] combine_rows_per_src=[7764, 7902, 7718, 4042, 7906, 7390, 8062, 7188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 2, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58056 payload_rows_remote=52858 dup_rows=7480 dup_rows_remote=6430 dedup_ratio=0.114136 router_tokens_per_expert=[0, 0, 0, 0, 2, 1, 138, 0, 0, 90, 0, 0, 0, 0, 1, 0, 4039, 0, 0, 3762, 0, 28, 3935, 0, 0, 0, 1721, 744, 0, 656, 3, 0, 0, 0, 0, 0, 495, 3330, 910, 5, 237, 0, 3565, 0, 0, 0, 0, 0, 0, 0, 1780, 0, 0, 0, 0, 0, 3366, 0, 0, 3946, 0, 0, 0, 14] payload_rows_per_dst=[8078, 7870, 7526, 5198, 7468, 7144, 8026, 6746] dup_rows_per_dst=[282, 182, 54, 1050, 2012, 460, 3426, 14] combine_rows_per_src=[8078, 7870, 7526, 5198, 7468, 7144, 8026, 6746] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56184 payload_rows_remote=48914 dup_rows=9352 dup_rows_remote=8898 dedup_ratio=0.142700 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 165, 0, 0, 113, 0, 0, 0, 0, 1, 0, 3881, 0, 0, 3778, 0, 22, 3949, 0, 0, 0, 2664, 488, 0, 492, 0, 0, 0, 0, 0, 0, 589, 3067, 994, 0, 235, 0, 3627, 0, 0, 0, 0, 0, 0, 0, 1174, 0, 0, 0, 0, 0, 3549, 0, 0, 3975, 0, 0, 0, 5] payload_rows_per_dst=[7762, 7902, 7556, 6146, 7308, 7270, 4316, 7924] dup_rows_per_dst=[330, 224, 44, 1142, 1992, 454, 404, 4762] combine_rows_per_src=[7762, 7902, 7556, 6146, 7308, 7270, 4316, 7924] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=359174144 dispatch_meta_bytes=230984 combine_payload_bytes=359174144 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=359405128 ingress_bytes=799576064 link_bytes=799576064 token_bytes=718579272 weight_movement_bytes=440401920 total_nvlink_bytes=1158981192 weight_to_token_ratio=0.612879 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=30336 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.390594283e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54538 payload_rows_remote=52128 dup_rows=10998 dup_rows_remote=10998 dedup_ratio=0.167816 router_tokens_per_expert=[0, 0, 0, 2, 0, 1, 80, 0, 0, 118, 0, 0, 0, 0, 1, 0, 3891, 0, 0, 3898, 0, 79, 4068, 0, 0, 0, 2444, 619, 0, 491, 0, 0, 0, 0, 0, 0, 983, 2482, 1228, 0, 271, 2, 3476, 0, 0, 0, 0, 0, 0, 0, 1205, 0, 0, 0, 0, 0, 3441, 0, 0, 3987, 0, 0, 0, 1] payload_rows_per_dst=[7786, 8136, 7804, 6102, 7158, 6972, 2410, 8170] dup_rows_per_dst=[162, 238, 150, 1006, 2228, 526, 0, 6688] combine_rows_per_src=[7786, 8136, 7804, 6102, 7158, 6972, 2410, 8170] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54232 payload_rows_remote=46052 dup_rows=11304 dup_rows_remote=4124 dedup_ratio=0.172485 router_tokens_per_expert=[0, 0, 0, 3, 3, 6, 136, 0, 0, 36, 0, 0, 0, 0, 4, 0, 4000, 0, 0, 3803, 0, 13, 4017, 0, 0, 0, 1927, 438, 0, 651, 0, 0, 3, 0, 0, 0, 392, 3003, 1539, 7, 248, 0, 3533, 0, 0, 0, 0, 0, 0, 0, 1326, 0, 0, 0, 0, 0, 3678, 1, 0, 3977, 0, 1, 0, 23] payload_rows_per_dst=[7962, 8036, 7610, 5226, 7492, 7074, 2652, 8180] dup_rows_per_dst=[292, 78, 64, 806, 2396, 488, 0, 7180] combine_rows_per_src=[7962, 8036, 7610, 5226, 7492, 7074, 2652, 8180] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=358873088 dispatch_meta_bytes=217408 combine_payload_bytes=358873088 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=359090496 ingress_bytes=579074048 link_bytes=579074048 token_bytes=717963584 weight_movement_bytes=220200960 total_nvlink_bytes=938164544 weight_to_token_ratio=0.306702 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=15616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.007105011e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52884 payload_rows_remote=44964 dup_rows=12652 dup_rows_remote=12514 dedup_ratio=0.193054 router_tokens_per_expert=[3, 0, 0, 9, 3, 4, 165, 0, 0, 71, 0, 2, 0, 0, 2, 2, 3997, 0, 0, 3833, 5, 11, 3952, 3, 1, 0, 1450, 457, 5, 814, 0, 0, 0, 0, 4, 0, 627, 3022, 1352, 1, 244, 0, 3397, 2, 0, 4, 1, 0, 1, 0, 1925, 1, 0, 0, 3, 0, 3583, 0, 1, 3796, 0, 0, 0, 15] payload_rows_per_dst=[8006, 7920, 7674, 7782, 4740, 7730, 7926, 1106] dup_rows_per_dst=[356, 138, 30, 3716, 554, 4332, 3526, 0] combine_rows_per_src=[8006, 7920, 7674, 7782, 4740, 7730, 7926, 1106] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=378886144 dispatch_meta_bytes=237152 combine_payload_bytes=378886144 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=379123296 ingress_bytes=488986624 link_bytes=488986624 token_bytes=758009440 weight_movement_bytes=110100480 total_nvlink_bytes=868109920 weight_to_token_ratio=0.145249 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=14080 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.504281636e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=373653504 dispatch_meta_bytes=252504 combine_payload_bytes=373653504 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=373906008 ingress_bytes=483753984 link_bytes=483753984 token_bytes=747559512 weight_movement_bytes=110100480 total_nvlink_bytes=857659992 weight_to_token_ratio=0.147280 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=20992 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.413277420e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=350615552 dispatch_meta_bytes=231248 combine_payload_bytes=350615552 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=350846800 ingress_bytes=460716032 link_bytes=460716032 token_bytes=701462352 weight_movement_bytes=110100480 total_nvlink_bytes=811562832 weight_to_token_ratio=0.156959 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=28416 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.012609544e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52046 payload_rows_remote=44076 dup_rows=13490 dup_rows_remote=13170 dedup_ratio=0.205841 router_tokens_per_expert=[0, 0, 0, 2, 1, 9, 149, 0, 0, 21, 0, 0, 0, 0, 4, 0, 3984, 0, 0, 3840, 0, 27, 4025, 0, 0, 0, 1092, 729, 0, 525, 1, 0, 0, 0, 0, 0, 434, 3047, 1672, 0, 364, 0, 3371, 0, 0, 0, 0, 0, 0, 0, 1899, 0, 0, 0, 0, 0, 3575, 0, 0, 3980, 0, 0, 0, 17] payload_rows_per_dst=[7970, 8064, 7768, 7784, 5540, 6806, 8080, 34] dup_rows_per_dst=[320, 760, 3704, 3004, 1360, 664, 3678, 0] combine_rows_per_src=[7970, 8064, 7768, 7784, 5540, 6806, 8080, 34] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=330100736 dispatch_meta_bytes=200704 combine_payload_bytes=330100736 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=330301440 ingress_bytes=770502656 link_bytes=770502656 token_bytes=660402176 weight_movement_bytes=440401920 total_nvlink_bytes=1100804096 weight_to_token_ratio=0.666869 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=22016 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.340030845e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=324882432 dispatch_meta_bytes=226144 combine_payload_bytes=324882432 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=325108576 ingress_bytes=765284352 link_bytes=765284352 token_bytes=649991008 weight_movement_bytes=440401920 total_nvlink_bytes=1090392928 weight_to_token_ratio=0.677551 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=15104 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.330955356e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=282232832 dispatch_meta_bytes=217184 combine_payload_bytes=282232832 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=282450016 ingress_bytes=502433792 link_bytes=502433792 token_bytes=564682848 weight_movement_bytes=220200960 total_nvlink_bytes=784883808 weight_to_token_ratio=0.389955 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=16640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.738150004e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=313628672 dispatch_meta_bytes=228800 combine_payload_bytes=313628672 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=313857472 ingress_bytes=423729152 link_bytes=423729152 token_bytes=627486144 weight_movement_bytes=110100480 total_nvlink_bytes=737586624 weight_to_token_ratio=0.175463 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=20736 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.369346868e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=328380416 dispatch_meta_bytes=236864 combine_payload_bytes=328380416 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=328617280 ingress_bytes=548581376 link_bytes=548581376 token_bytes=656997696 weight_movement_bytes=220200960 total_nvlink_bytes=877198656 weight_to_token_ratio=0.335162 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=25088 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.540732390e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=325556224 dispatch_meta_bytes=235328 combine_payload_bytes=325556224 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=325791552 ingress_bytes=545757184 link_bytes=545757184 token_bytes=651347776 weight_movement_bytes=220200960 total_nvlink_bytes=871548736 weight_to_token_ratio=0.338070 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=19712 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.491615046e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=294576128 dispatch_meta_bytes=230920 combine_payload_bytes=294576128 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=294807048 ingress_bytes=404676608 link_bytes=404676608 token_bytes=589383176 weight_movement_bytes=110100480 total_nvlink_bytes=699483656 weight_to_token_ratio=0.186806 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=19456 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.037991792e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=314187776 dispatch_meta_bytes=224072 combine_payload_bytes=314187776 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=314411848 ingress_bytes=644489216 link_bytes=644489216 token_bytes=628599624 weight_movement_bytes=330301440 total_nvlink_bytes=958901064 weight_to_token_ratio=0.525456 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=16640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.120872747e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=303163392 dispatch_meta_bytes=222704 combine_payload_bytes=303163392 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=303386096 ingress_bytes=413263872 link_bytes=413263872 token_bytes=606549488 weight_movement_bytes=110100480 total_nvlink_bytes=716649968 weight_to_token_ratio=0.181519 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=20864 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.187338436e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54538 payload_rows_remote=48698 dup_rows=10998 dup_rows_remote=9218 dedup_ratio=0.167816 router_tokens_per_expert=[0, 0, 0, 0, 1, 0, 134, 0, 0, 4, 0, 0, 0, 0, 0, 0, 3124, 0, 0, 3651, 0, 23, 2870, 0, 0, 0, 1383, 718, 0, 1709, 0, 0, 0, 2348, 0, 0, 1917, 2923, 0, 5, 265, 1, 3069, 0, 0, 0, 0, 0, 0, 0, 1211, 0, 0, 0, 0, 0, 3328, 0, 0, 4079, 0, 0, 0, 5] payload_rows_per_dst=[8164, 7310, 7690, 5840, 6348, 6158, 6364, 6664] dup_rows_per_dst=[264, 0, 4344, 1780, 2192, 512, 1904, 2] combine_rows_per_src=[8164, 7310, 7690, 5840, 6348, 6158, 6364, 6664] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55710 payload_rows_remote=48906 dup_rows=9826 dup_rows_remote=7072 dedup_ratio=0.149933 router_tokens_per_expert=[0, 0, 0, 0, 5, 0, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2432, 0, 0, 3836, 0, 14, 2950, 0, 0, 0, 1409, 431, 0, 1364, 0, 0, 0, 2629, 0, 0, 2107, 2971, 40, 3, 258, 0, 3453, 0, 0, 0, 0, 0, 0, 0, 967, 0, 0, 0, 0, 0, 3655, 0, 0, 4087, 0, 1, 0, 0] payload_rows_per_dst=[8184, 7672, 7370, 5192, 6804, 6910, 6266, 7312] dup_rows_per_dst=[312, 0, 3422, 1216, 2754, 512, 1610, 0] combine_rows_per_src=[8184, 7672, 7370, 5192, 6804, 6910, 6266, 7312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55456 payload_rows_remote=48626 dup_rows=10080 dup_rows_remote=9598 dedup_ratio=0.153809 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 167, 0, 0, 34, 0, 0, 0, 0, 0, 0, 2439, 0, 0, 3739, 0, 13, 2925, 0, 0, 0, 2396, 509, 0, 1439, 0, 0, 0, 2035, 0, 0, 2334, 2614, 4, 0, 248, 0, 3408, 0, 0, 0, 0, 0, 0, 0, 775, 0, 0, 0, 0, 0, 3599, 0, 0, 4090, 0, 0, 0, 0] payload_rows_per_dst=[8188, 7532, 5870, 7604, 6578, 6830, 5656, 7198] dup_rows_per_dst=[326, 14, 1202, 4766, 2168, 482, 1122, 0] combine_rows_per_src=[8188, 7532, 5870, 7604, 6578, 6830, 5656, 7198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46244 payload_rows_remote=38112 dup_rows=19292 dup_rows_remote=11622 dedup_ratio=0.294373 router_tokens_per_expert=[0, 0, 0, 0, 1, 0, 132, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3164, 0, 0, 3812, 0, 7, 3305, 0, 0, 0, 1525, 444, 0, 1658, 0, 0, 3, 2517, 0, 0, 1561, 2431, 6, 7, 270, 0, 3195, 0, 0, 0, 0, 0, 0, 0, 859, 0, 0, 0, 0, 0, 3786, 1, 0, 4082, 0, 0, 0, 1] payload_rows_per_dst=[6132, 7626, 230, 5702, 3142, 8048, 7232, 8132] dup_rows_per_dst=[184, 0, 0, 1552, 12, 5492, 4382, 7670] combine_rows_per_src=[6132, 7626, 230, 5702, 3142, 8048, 7232, 8132] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54184 payload_rows_remote=47938 dup_rows=11352 dup_rows_remote=8782 dedup_ratio=0.173218 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 78, 0, 0, 4, 0, 0, 0, 0, 0, 0, 2311, 0, 0, 3822, 0, 53, 3749, 0, 0, 0, 2105, 630, 0, 1582, 0, 0, 0, 1880, 0, 0, 2859, 2060, 2, 0, 260, 2, 3231, 0, 0, 0, 0, 0, 0, 0, 742, 0, 0, 0, 0, 0, 3305, 0, 0, 4093, 0, 0, 0, 0] payload_rows_per_dst=[8190, 7652, 4708, 6730, 6962, 7086, 6246, 6610] dup_rows_per_dst=[152, 0, 20, 3164, 2604, 2842, 2570, 0] combine_rows_per_src=[8190, 7652, 4708, 6730, 6962, 7086, 6246, 6610] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56344 payload_rows_remote=48910 dup_rows=9192 dup_rows_remote=5986 dedup_ratio=0.140259 router_tokens_per_expert=[0, 0, 0, 7, 0, 0, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2331, 0, 0, 3786, 0, 42, 2947, 0, 0, 0, 1091, 779, 0, 2203, 2, 0, 0, 2596, 0, 0, 1539, 3017, 7, 0, 299, 0, 3386, 0, 0, 0, 0, 0, 0, 0, 661, 0, 0, 0, 0, 0, 3783, 0, 0, 4079, 0, 0, 0, 17] payload_rows_per_dst=[8190, 7572, 7434, 6240, 6294, 6792, 6254, 7568] dup_rows_per_dst=[374, 0, 3206, 1910, 1990, 578, 1102, 32] combine_rows_per_src=[8190, 7572, 7434, 6240, 6294, 6792, 6254, 7568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54540 payload_rows_remote=46952 dup_rows=10996 dup_rows_remote=10986 dedup_ratio=0.167786 router_tokens_per_expert=[2, 0, 0, 4, 3, 1, 163, 0, 0, 15, 0, 2, 0, 0, 0, 3, 3010, 0, 0, 3779, 4, 14, 3134, 3, 1, 0, 1184, 426, 5, 1929, 0, 0, 0, 2726, 4, 0, 1899, 2301, 7, 4, 274, 0, 2969, 2, 0, 4, 1, 0, 1, 0, 1232, 2, 0, 0, 2, 0, 3615, 0, 1, 4042, 0, 0, 0, 0] payload_rows_per_dst=[8130, 7588, 7768, 5394, 6910, 5976, 5542, 7232] dup_rows_per_dst=[300, 10, 4562, 1696, 2370, 524, 1534, 0] combine_rows_per_src=[8130, 7588, 7768, 5394, 6910, 5976, 5542, 7232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53760 payload_rows_remote=45572 dup_rows=11776 dup_rows_remote=11518 dedup_ratio=0.179688 router_tokens_per_expert=[0, 0, 0, 0, 2, 0, 130, 0, 0, 4, 0, 0, 0, 0, 0, 0, 3079, 0, 0, 3838, 0, 22, 3192, 0, 0, 0, 921, 714, 0, 1740, 0, 0, 0, 2370, 0, 0, 1830, 2505, 4, 0, 376, 0, 3021, 0, 0, 0, 0, 0, 0, 0, 1371, 0, 0, 0, 0, 0, 3558, 0, 0, 4091, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8052, 6698, 5312, 6378, 6070, 5946, 7116] dup_rows_per_dst=[258, 4508, 1012, 1438, 2030, 724, 1806, 0] combine_rows_per_src=[8188, 8052, 6698, 5312, 6378, 6070, 5946, 7116] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 1, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=349067264 dispatch_meta_bytes=231664 combine_payload_bytes=349067264 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=349298928 ingress_bytes=459167744 link_bytes=459167744 token_bytes=698366192 weight_movement_bytes=110100480 total_nvlink_bytes=808466672 weight_to_token_ratio=0.157654 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=13696 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.985682269e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=322301952 dispatch_meta_bytes=229912 combine_payload_bytes=322301952 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=322531864 ingress_bytes=542502912 link_bytes=542502912 token_bytes=644833816 weight_movement_bytes=220200960 total_nvlink_bytes=865034776 weight_to_token_ratio=0.341485 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.435017904e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57816 payload_rows_remote=49906 dup_rows=7720 dup_rows_remote=7518 dedup_ratio=0.117798 router_tokens_per_expert=[123, 0, 2564, 0, 314, 0, 0, 0, 0, 0, 0, 2207, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 85, 0, 0, 273, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 2507, 0, 1694, 0, 0, 0, 0, 0, 0, 3319, 0, 0, 476, 4076, 0, 0, 0, 0, 0, 3952, 0, 0, 0, 0, 3446, 3248, 0] payload_rows_per_dst=[5836, 7536, 7910, 8190, 6636, 6638, 8178, 6892] dup_rows_per_dst=[166, 3374, 202, 1286, 1766, 0, 926, 0] combine_rows_per_src=[5836, 7536, 7910, 8190, 6636, 6638, 8178, 6892] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=315936768 dispatch_meta_bytes=228984 combine_payload_bytes=315936768 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=316165752 ingress_bytes=426037248 link_bytes=426037248 token_bytes=632102520 weight_movement_bytes=110100480 total_nvlink_bytes=742203000 weight_to_token_ratio=0.174181 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=31616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.409488454e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59398 payload_rows_remote=53634 dup_rows=6138 dup_rows_remote=5348 dedup_ratio=0.093658 router_tokens_per_expert=[211, 0, 3125, 0, 239, 0, 0, 0, 4, 0, 0, 1374, 11, 0, 0, 0, 0, 0, 28, 0, 0, 0, 61, 0, 0, 210, 0, 604, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 1662, 0, 1615, 3, 0, 0, 0, 0, 0, 3921, 0, 0, 400, 4092, 0, 0, 0, 0, 0, 3810, 0, 0, 0, 0, 3517, 3803, 0] payload_rows_per_dst=[6810, 7948, 7620, 8188, 5764, 7848, 8186, 7034] dup_rows_per_dst=[340, 2436, 178, 1596, 790, 0, 798, 0] combine_rows_per_src=[6810, 7948, 7620, 8188, 5764, 7848, 8186, 7034] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58164 payload_rows_remote=49976 dup_rows=7372 dup_rows_remote=5796 dedup_ratio=0.112488 router_tokens_per_expert=[187, 0, 2373, 0, 134, 2, 0, 0, 3, 0, 0, 1467, 8, 0, 1, 0, 0, 0, 2, 0, 0, 0, 64, 0, 0, 209, 0, 609, 0, 1, 1, 0, 0, 0, 0, 0, 4062, 2049, 0, 2249, 12, 0, 0, 0, 0, 0, 3782, 0, 0, 568, 4087, 0, 0, 0, 0, 0, 3855, 0, 0, 0, 0, 3203, 3840, 0] payload_rows_per_dst=[5246, 8002, 7710, 8188, 6846, 7584, 8182, 6406] dup_rows_per_dst=[146, 2636, 132, 1576, 1750, 4, 1128, 0] combine_rows_per_src=[5246, 8002, 7710, 8188, 6846, 7584, 8182, 6406] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=357726208 dispatch_meta_bytes=229696 combine_payload_bytes=357726208 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=357955904 ingress_bytes=577927168 link_bytes=577927168 token_bytes=715682112 weight_movement_bytes=220200960 total_nvlink_bytes=935883072 weight_to_token_ratio=0.307680 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=31360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.005110398e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51850 payload_rows_remote=43662 dup_rows=13686 dup_rows_remote=9726 dedup_ratio=0.208832 router_tokens_per_expert=[143, 0, 2868, 0, 177, 0, 0, 0, 1, 0, 0, 868, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 114, 0, 0, 163, 0, 562, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 1278, 0, 2396, 16, 0, 0, 0, 0, 0, 3937, 0, 0, 685, 4088, 0, 0, 0, 0, 0, 3925, 0, 0, 0, 0, 3714, 3747, 0] payload_rows_per_dst=[6240, 1738, 7858, 4768, 7020, 7902, 8188, 8136] dup_rows_per_dst=[136, 0, 264, 892, 1644, 4, 3960, 6786] combine_rows_per_src=[6240, 1738, 7858, 4768, 7020, 7902, 8188, 8136] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=384448512 dispatch_meta_bytes=235928 combine_payload_bytes=384448512 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=384684440 ingress_bytes=714749952 link_bytes=714749952 token_bytes=769132952 weight_movement_bytes=330301440 total_nvlink_bytes=1099434392 weight_to_token_ratio=0.429446 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=14976 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.243067723e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58758 payload_rows_remote=51130 dup_rows=6778 dup_rows_remote=6778 dedup_ratio=0.103424 router_tokens_per_expert=[145, 0, 2936, 0, 197, 0, 0, 0, 0, 0, 0, 1008, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 88, 0, 0, 201, 0, 491, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 2172, 0, 1988, 1, 0, 0, 0, 0, 0, 3813, 0, 0, 386, 4090, 0, 0, 0, 0, 0, 3914, 0, 0, 0, 0, 3394, 3884, 0] payload_rows_per_dst=[6344, 6796, 7828, 8166, 6810, 7628, 8188, 6998] dup_rows_per_dst=[212, 1412, 214, 1300, 1510, 0, 764, 1366] combine_rows_per_src=[6344, 6796, 7828, 8166, 6810, 7628, 8188, 6998] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50264 payload_rows_remote=42182 dup_rows=15272 dup_rows_remote=9326 dedup_ratio=0.233032 router_tokens_per_expert=[183, 0, 2957, 0, 172, 0, 0, 0, 0, 0, 0, 1191, 9, 0, 0, 0, 0, 0, 54, 0, 0, 0, 64, 0, 0, 199, 0, 640, 0, 4, 0, 0, 0, 0, 0, 0, 4049, 1691, 0, 2496, 50, 0, 0, 0, 0, 0, 3737, 0, 0, 492, 4086, 0, 0, 0, 0, 0, 3952, 0, 0, 0, 0, 2957, 3785, 0] payload_rows_per_dst=[6330, 2400, 7904, 1676, 8130, 7550, 8192, 8082] dup_rows_per_dst=[294, 0, 236, 10, 2806, 24, 5956, 5946] combine_rows_per_src=[6330, 2400, 7904, 1676, 8130, 7550, 8192, 8082] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=358227968 dispatch_meta_bytes=223088 combine_payload_bytes=358227968 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=358451056 ingress_bytes=468328448 link_bytes=468328448 token_bytes=716679024 weight_movement_bytes=110100480 total_nvlink_bytes=826779504 weight_to_token_ratio=0.153626 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=26496 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.145001979e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 2, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53890 payload_rows_remote=45870 dup_rows=11646 dup_rows_remote=9278 dedup_ratio=0.177704 router_tokens_per_expert=[241, 0, 2826, 0, 155, 0, 4, 4, 0, 0, 0, 1313, 4, 0, 0, 1, 0, 0, 53, 0, 0, 0, 61, 0, 0, 191, 4, 774, 0, 1, 0, 2, 0, 0, 0, 1, 4053, 1975, 0, 1621, 14, 2, 4, 0, 0, 0, 3844, 0, 0, 412, 4071, 1, 0, 5, 0, 0, 3920, 0, 0, 4, 3, 3328, 3876, 0] payload_rows_per_dst=[8040, 8020, 7848, 8186, 6110, 7724, 1298, 6664] dup_rows_per_dst=[6050, 2368, 220, 1864, 1084, 4, 50, 6] combine_rows_per_src=[8040, 8020, 7848, 8186, 6110, 7724, 1298, 6664] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=312969216 dispatch_meta_bytes=213552 combine_payload_bytes=312969216 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=313182768 ingress_bytes=533170176 link_bytes=533170176 token_bytes=626151984 weight_movement_bytes=220200960 total_nvlink_bytes=846352944 weight_to_token_ratio=0.351673 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=24832 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.272706275e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=366499840 dispatch_meta_bytes=231632 combine_payload_bytes=366499840 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=366731472 ingress_bytes=476600320 link_bytes=476600320 token_bytes=733231312 weight_movement_bytes=110100480 total_nvlink_bytes=843331792 weight_to_token_ratio=0.150158 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=30208 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.288863438e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48796 payload_rows_remote=40624 dup_rows=16740 dup_rows_remote=11566 dedup_ratio=0.255432 router_tokens_per_expert=[158, 0, 2181, 0, 270, 0, 0, 0, 4, 0, 0, 1596, 2, 0, 0, 0, 0, 0, 27, 0, 0, 0, 109, 0, 0, 206, 0, 559, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 1835, 0, 1600, 3, 0, 0, 0, 0, 0, 3736, 0, 0, 771, 4064, 0, 0, 0, 0, 0, 3986, 0, 0, 0, 0, 3768, 3833, 0] payload_rows_per_dst=[8172, 7970, 7976, 8188, 5840, 7888, 1542, 1220] dup_rows_per_dst=[5174, 2900, 1436, 1462, 1030, 4738, 0, 0] combine_rows_per_src=[8172, 7970, 7976, 8188, 5840, 7888, 1542, 1220] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=302360576 dispatch_meta_bytes=206032 combine_payload_bytes=302360576 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=302566608 ingress_bytes=852862976 link_bytes=852862976 token_bytes=604927184 weight_movement_bytes=550502400 total_nvlink_bytes=1155429584 weight_to_token_ratio=0.910031 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=24192 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.483268987e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=350558208 dispatch_meta_bytes=223912 combine_payload_bytes=350558208 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=350782120 ingress_bytes=680859648 link_bytes=680859648 token_bytes=701340328 weight_movement_bytes=330301440 total_nvlink_bytes=1031641768 weight_to_token_ratio=0.470957 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=16128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.184126910e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=348551168 dispatch_meta_bytes=232896 combine_payload_bytes=348551168 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=348784064 ingress_bytes=458651648 link_bytes=458651648 token_bytes=697335232 weight_movement_bytes=110100480 total_nvlink_bytes=807435712 weight_to_token_ratio=0.157887 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=25856 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.976706511e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=343619584 dispatch_meta_bytes=226880 combine_payload_bytes=343619584 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=343846464 ingress_bytes=563820544 link_bytes=563820544 token_bytes=687466048 weight_movement_bytes=220200960 total_nvlink_bytes=907667008 weight_to_token_ratio=0.320308 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=20864 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.805766586e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=273186816 dispatch_meta_bytes=198936 combine_payload_bytes=273186816 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=273385752 ingress_bytes=493387776 link_bytes=493387776 token_bytes=546572568 weight_movement_bytes=220200960 total_nvlink_bytes=766773528 weight_to_token_ratio=0.402876 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=28672 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.580824908e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=350586880 dispatch_meta_bytes=219584 combine_payload_bytes=350586880 prefetch_bytes=0 reduce_bytes=660602880 egress_bytes=350806464 ingress_bytes=1011189760 link_bytes=1011189760 token_bytes=701393344 weight_movement_bytes=660602880 total_nvlink_bytes=1361996224 weight_to_token_ratio=0.941844 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=16512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.758625305e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=336551936 dispatch_meta_bytes=231752 combine_payload_bytes=336551936 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=336783688 ingress_bytes=556752896 link_bytes=556752896 token_bytes=673335624 weight_movement_bytes=220200960 total_nvlink_bytes=893536584 weight_to_token_ratio=0.327030 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=30336 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.682848563e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=326660096 dispatch_meta_bytes=228360 combine_payload_bytes=326660096 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=326888456 ingress_bytes=436760576 link_bytes=436760576 token_bytes=653548552 weight_movement_bytes=110100480 total_nvlink_bytes=763649032 weight_to_token_ratio=0.168466 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=31616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.595984764e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54432 payload_rows_remote=46326 dup_rows=11104 dup_rows_remote=10218 dedup_ratio=0.169434 router_tokens_per_expert=[870, 0, 624, 0, 115, 2, 0, 0, 8, 0, 0, 2867, 11, 0, 1, 0, 0, 0, 4, 0, 0, 0, 63, 0, 0, 56, 0, 391, 0, 2, 2, 0, 0, 0, 0, 0, 3456, 1143, 0, 124, 913, 0, 0, 0, 0, 0, 4021, 0, 0, 3425, 4045, 0, 0, 0, 0, 0, 3949, 0, 0, 0, 0, 2593, 4083, 0] payload_rows_per_dst=[8118, 7516, 8170, 8106, 7566, 8106, 6850, 0] dup_rows_per_dst=[3002, 3444, 130, 886, 1880, 1762, 0, 0] combine_rows_per_src=[8118, 7516, 8170, 8106, 7566, 8106, 6850, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55670 payload_rows_remote=48186 dup_rows=9866 dup_rows_remote=8660 dedup_ratio=0.150543 router_tokens_per_expert=[935, 0, 1117, 0, 130, 0, 0, 0, 15, 0, 0, 2662, 12, 0, 0, 0, 0, 0, 26, 0, 0, 0, 49, 0, 0, 75, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 3568, 721, 0, 56, 759, 0, 0, 0, 0, 0, 4001, 0, 0, 3281, 4070, 0, 0, 0, 0, 0, 3912, 0, 0, 0, 0, 2910, 4085, 0] payload_rows_per_dst=[4278, 5390, 8170, 8142, 7484, 8034, 6562, 7610] dup_rows_per_dst=[850, 76, 150, 916, 1206, 1486, 0, 5182] combine_rows_per_src=[4278, 5390, 8170, 8142, 7484, 8034, 6562, 7610] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53504 payload_rows_remote=46806 dup_rows=12032 dup_rows_remote=12032 dedup_ratio=0.183594 router_tokens_per_expert=[1020, 0, 1205, 0, 143, 0, 0, 0, 6, 0, 0, 2199, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 73, 0, 0, 28, 0, 401, 0, 0, 0, 0, 0, 0, 0, 0, 3358, 421, 0, 140, 955, 0, 0, 0, 0, 0, 4023, 0, 0, 3349, 3993, 0, 0, 0, 0, 0, 3980, 0, 0, 0, 0, 3382, 4087, 0] payload_rows_per_dst=[4356, 4408, 8174, 7986, 7152, 6708, 6698, 8022] dup_rows_per_dst=[380, 2, 156, 858, 686, 1520, 0, 8430] combine_rows_per_src=[4356, 4408, 8174, 7986, 7152, 6708, 6698, 8022] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55074 payload_rows_remote=47030 dup_rows=10462 dup_rows_remote=9082 dedup_ratio=0.159637 router_tokens_per_expert=[736, 0, 1528, 0, 149, 0, 0, 0, 0, 0, 0, 2282, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 74, 0, 0, 68, 0, 420, 0, 0, 0, 0, 0, 0, 0, 0, 2974, 1209, 0, 111, 711, 0, 0, 0, 0, 0, 4001, 0, 0, 3365, 4043, 0, 0, 0, 0, 0, 3973, 0, 0, 0, 0, 3007, 4091, 0] payload_rows_per_dst=[4450, 4564, 8184, 8090, 7046, 8044, 6730, 7966] dup_rows_per_dst=[376, 0, 198, 972, 1542, 1380, 0, 5994] combine_rows_per_src=[4450, 4564, 8184, 8090, 7046, 8044, 6730, 7966] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53820 payload_rows_remote=45726 dup_rows=11716 dup_rows_remote=11548 dedup_ratio=0.178772 router_tokens_per_expert=[438, 0, 885, 0, 275, 0, 0, 0, 1, 0, 0, 3054, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 80, 0, 0, 125, 0, 194, 0, 0, 0, 0, 0, 0, 0, 0, 3710, 1454, 0, 56, 754, 0, 1, 0, 0, 0, 3867, 0, 0, 3084, 4046, 0, 0, 0, 0, 0, 3963, 0, 0, 0, 0, 2730, 4046, 0] payload_rows_per_dst=[8154, 7620, 8094, 8094, 7816, 7874, 6168, 0] dup_rows_per_dst=[2968, 3950, 168, 636, 2624, 1370, 0, 0] combine_rows_per_src=[8154, 7620, 8094, 8094, 7816, 7874, 6168, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=38854 payload_rows_remote=30662 dup_rows=26682 dup_rows_remote=12558 dedup_ratio=0.407135 router_tokens_per_expert=[885, 0, 1352, 0, 161, 0, 0, 0, 14, 0, 0, 2633, 9, 0, 0, 0, 0, 0, 32, 0, 0, 0, 52, 0, 0, 37, 0, 538, 0, 5, 0, 0, 0, 0, 0, 0, 3252, 763, 0, 172, 854, 0, 0, 0, 0, 0, 3976, 0, 0, 3310, 4046, 0, 0, 0, 0, 0, 4029, 0, 0, 0, 0, 2570, 4078, 0] payload_rows_per_dst=[4358, 5296, 7022, 2350, 2352, 1708, 7576, 8192] dup_rows_per_dst=[438, 16, 136, 274, 348, 0, 11346, 14124] combine_rows_per_src=[4358, 5296, 7022, 2350, 2352, 1708, 7576, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54262 payload_rows_remote=46094 dup_rows=11274 dup_rows_remote=8186 dedup_ratio=0.172028 router_tokens_per_expert=[910, 0, 617, 0, 110, 0, 0, 0, 7, 0, 0, 2838, 2, 0, 0, 0, 0, 0, 14, 0, 0, 0, 86, 0, 0, 51, 0, 334, 0, 0, 1, 0, 0, 0, 0, 0, 3502, 864, 0, 49, 557, 0, 0, 0, 0, 0, 3961, 0, 0, 3569, 3966, 0, 0, 0, 0, 0, 3991, 0, 0, 0, 0, 3254, 4085, 0] payload_rows_per_dst=[8168, 7780, 8176, 7946, 7522, 7532, 7138, 0] dup_rows_per_dst=[3088, 4422, 194, 758, 1856, 956, 0, 0] combine_rows_per_src=[8168, 7780, 8176, 7946, 7522, 7532, 7138, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53608 payload_rows_remote=46152 dup_rows=11928 dup_rows_remote=8332 dedup_ratio=0.182007 router_tokens_per_expert=[900, 0, 1287, 0, 124, 0, 3, 4, 2, 0, 0, 2681, 4, 0, 0, 1, 0, 0, 66, 0, 0, 0, 57, 0, 0, 55, 4, 651, 0, 1, 0, 2, 0, 0, 0, 1, 3270, 873, 0, 84, 519, 2, 4, 0, 0, 0, 4003, 0, 0, 3228, 4048, 1, 0, 5, 0, 0, 3958, 0, 1, 4, 3, 2838, 4084, 0] payload_rows_per_dst=[8146, 7456, 8170, 8122, 7180, 8062, 6462, 10] dup_rows_per_dst=[4406, 3596, 244, 1400, 1276, 994, 6, 6] combine_rows_per_src=[8146, 7456, 8170, 8122, 7180, 8062, 6462, 10] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=332064768 dispatch_meta_bytes=226176 combine_payload_bytes=332064768 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=332290944 ingress_bytes=442165248 link_bytes=442165248 token_bytes=664355712 weight_movement_bytes=110100480 total_nvlink_bytes=774456192 weight_to_token_ratio=0.165725 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=29056 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.689980900e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=345397248 dispatch_meta_bytes=227384 combine_payload_bytes=345397248 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=345624632 ingress_bytes=565598208 link_bytes=565598208 token_bytes=691021880 weight_movement_bytes=220200960 total_nvlink_bytes=911222840 weight_to_token_ratio=0.318660 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=24320 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.836683086e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=328796160 dispatch_meta_bytes=220592 combine_payload_bytes=328796160 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=329016752 ingress_bytes=438896640 link_bytes=438896640 token_bytes=657812912 weight_movement_bytes=110100480 total_nvlink_bytes=767913392 weight_to_token_ratio=0.167374 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=21760 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.633134431e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55664 payload_rows_remote=48366 dup_rows=9872 dup_rows_remote=9596 dedup_ratio=0.150635 router_tokens_per_expert=[0, 4, 177, 0, 1829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3648, 0, 0, 139, 0, 0, 0, 0, 0, 0, 0, 194, 0, 1132, 0, 0, 4013, 3148, 0, 237, 0, 0, 0, 1471, 0, 3976, 43, 0, 0, 40, 1724, 0, 0, 2700, 0, 0, 0, 25, 3708, 0, 0, 0, 88, 0, 284, 3927, 261, 0] payload_rows_per_dst=[8070, 8026, 7298, 7850, 7550, 3556, 5410, 7904] dup_rows_per_dst=[3902, 0, 276, 2218, 2162, 58, 40, 1216] combine_rows_per_src=[8070, 8026, 7298, 7850, 7550, 3556, 5410, 7904] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56216 payload_rows_remote=48324 dup_rows=9320 dup_rows_remote=5612 dedup_ratio=0.142212 router_tokens_per_expert=[0, 11, 169, 0, 1913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3706, 0, 0, 151, 0, 0, 0, 0, 0, 0, 0, 186, 0, 782, 0, 0, 4020, 3247, 0, 310, 0, 0, 0, 2243, 0, 4010, 4, 0, 0, 23, 1283, 0, 0, 2405, 0, 0, 0, 9, 3838, 0, 0, 0, 65, 0, 204, 3919, 270, 0] payload_rows_per_dst=[4124, 8040, 7422, 7966, 7892, 8072, 4816, 7884] dup_rows_per_dst=[62, 0, 292, 1646, 3708, 2568, 12, 1032] combine_rows_per_src=[4124, 8040, 7422, 7966, 7892, 8072, 4816, 7884] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=291192832 dispatch_meta_bytes=208760 combine_payload_bytes=291192832 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=291401592 ingress_bytes=401293312 link_bytes=401293312 token_bytes=582594424 weight_movement_bytes=110100480 total_nvlink_bytes=692694904 weight_to_token_ratio=0.188983 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=21888 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.979150710e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 2, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54588 payload_rows_remote=46672 dup_rows=10948 dup_rows_remote=9462 dedup_ratio=0.167053 router_tokens_per_expert=[0, 7, 175, 0, 2275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3534, 0, 0, 133, 0, 0, 0, 0, 0, 0, 0, 109, 0, 792, 0, 0, 4061, 3265, 0, 217, 0, 0, 0, 2133, 2, 4039, 27, 0, 0, 0, 1521, 0, 0, 2204, 0, 0, 0, 15, 3800, 0, 0, 0, 101, 0, 175, 3927, 256, 0] payload_rows_per_dst=[7680, 8122, 7074, 7916, 7856, 3602, 4408, 7930] dup_rows_per_dst=[4300, 0, 260, 1486, 3374, 510, 30, 988] combine_rows_per_src=[7680, 8122, 7074, 7916, 7856, 3602, 4408, 7930] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=346687488 dispatch_meta_bytes=231848 combine_payload_bytes=346687488 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=346919336 ingress_bytes=456787968 link_bytes=456787968 token_bytes=693606824 weight_movement_bytes=110100480 total_nvlink_bytes=803707304 weight_to_token_ratio=0.158736 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=29312 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.944294051e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50890 payload_rows_remote=43032 dup_rows=14646 dup_rows_remote=6496 dedup_ratio=0.223480 router_tokens_per_expert=[0, 9, 94, 0, 1715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3705, 0, 0, 139, 0, 0, 0, 0, 0, 0, 0, 64, 0, 1231, 0, 0, 4021, 3171, 0, 52, 0, 0, 0, 1360, 0, 4036, 11, 0, 0, 0, 2136, 0, 0, 2937, 0, 0, 0, 1, 3785, 0, 0, 0, 65, 0, 184, 3923, 129, 0] payload_rows_per_dst=[3618, 8042, 7410, 5226, 2736, 8102, 7858, 7898] dup_rows_per_dst=[18, 0, 278, 1144, 88, 4264, 8150, 704] combine_rows_per_src=[3618, 8042, 7410, 5226, 2736, 8102, 7858, 7898] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=346386432 dispatch_meta_bytes=215744 combine_payload_bytes=346386432 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=346602176 ingress_bytes=896888832 link_bytes=896888832 token_bytes=692988608 weight_movement_bytes=550502400 total_nvlink_bytes=1243491008 weight_to_token_ratio=0.794389 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=17152 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.559837192e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=335505408 dispatch_meta_bytes=235352 combine_payload_bytes=335505408 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=335740760 ingress_bytes=555706368 link_bytes=555706368 token_bytes=671246168 weight_movement_bytes=220200960 total_nvlink_bytes=891447128 weight_to_token_ratio=0.328048 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=26624 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.664647719e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=327763968 dispatch_meta_bytes=229096 combine_payload_bytes=327763968 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=327993064 ingress_bytes=437864448 link_bytes=437864448 token_bytes=655757032 weight_movement_bytes=110100480 total_nvlink_bytes=765857512 weight_to_token_ratio=0.167898 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.615182914e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=337111040 dispatch_meta_bytes=224448 combine_payload_bytes=337111040 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=337335488 ingress_bytes=557312000 link_bytes=557312000 token_bytes=674446528 weight_movement_bytes=220200960 total_nvlink_bytes=894647488 weight_to_token_ratio=0.326491 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=26752 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.692572301e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=219785216 dispatch_meta_bytes=172880 combine_payload_bytes=219785216 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=219958096 ingress_bytes=660187136 link_bytes=660187136 token_bytes=439743312 weight_movement_bytes=440401920 total_nvlink_bytes=880145232 weight_to_token_ratio=1.001498 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=15616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.148174012e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=330401792 dispatch_meta_bytes=217120 combine_payload_bytes=330401792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=330618912 ingress_bytes=440502272 link_bytes=440502272 token_bytes=661020704 weight_movement_bytes=110100480 total_nvlink_bytes=771121184 weight_to_token_ratio=0.166561 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=16256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.661059012e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=330817536 dispatch_meta_bytes=217936 combine_payload_bytes=330817536 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=331035472 ingress_bytes=440918016 link_bytes=440918016 token_bytes=661853008 weight_movement_bytes=110100480 total_nvlink_bytes=771953488 weight_to_token_ratio=0.166352 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=21248 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.668289484e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58736 payload_rows_remote=50772 dup_rows=6800 dup_rows_remote=6020 dedup_ratio=0.103760 router_tokens_per_expert=[0, 7, 175, 0, 3403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3244, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 391, 0, 61, 0, 0, 3803, 810, 0, 255, 0, 0, 0, 0, 3, 4077, 28, 0, 0, 0, 1552, 0, 0, 2734, 0, 0, 0, 22, 3920, 0, 0, 0, 188, 0, 37, 3937, 35, 0] payload_rows_per_dst=[6910, 8172, 6490, 7964, 7782, 7988, 5476, 7954] dup_rows_per_dst=[260, 0, 222, 780, 1954, 3108, 36, 440] combine_rows_per_src=[6910, 8172, 6490, 7964, 7782, 7988, 5476, 7954] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59378 payload_rows_remote=52020 dup_rows=6158 dup_rows_remote=4634 dedup_ratio=0.093964 router_tokens_per_expert=[0, 8, 137, 0, 3151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3517, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 649, 0, 140, 0, 0, 3553, 700, 0, 188, 0, 0, 0, 0, 0, 4073, 4, 0, 0, 26, 1263, 0, 0, 3111, 0, 0, 0, 12, 3955, 0, 0, 0, 111, 0, 62, 3925, 109, 0] payload_rows_per_dst=[6450, 8148, 7034, 8064, 7358, 8186, 6230, 7908] dup_rows_per_dst=[142, 0, 0, 1424, 1524, 2546, 16, 506] combine_rows_per_src=[6450, 8148, 7034, 8064, 7358, 8186, 6230, 7908] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52992 payload_rows_remote=44848 dup_rows=12544 dup_rows_remote=5010 dedup_ratio=0.191406 router_tokens_per_expert=[0, 0, 222, 0, 1797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3571, 0, 0, 139, 0, 0, 0, 0, 0, 0, 0, 189, 0, 1071, 0, 0, 3945, 3423, 0, 31, 0, 0, 0, 1382, 0, 4046, 24, 0, 0, 0, 1764, 0, 0, 2790, 0, 0, 0, 24, 3588, 0, 0, 0, 60, 0, 504, 3928, 270, 0] payload_rows_per_dst=[3930, 7890, 7146, 7762, 4530, 8144, 5674, 7916] dup_rows_per_dst=[108, 0, 274, 1934, 798, 7534, 288, 1608] combine_rows_per_src=[3930, 7890, 7146, 7762, 4530, 8144, 5674, 7916] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49306 payload_rows_remote=41456 dup_rows=16230 dup_rows_remote=15232 dedup_ratio=0.247650 router_tokens_per_expert=[0, 16, 216, 0, 1770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3887, 0, 0, 199, 0, 0, 0, 0, 0, 0, 0, 322, 0, 751, 0, 0, 4024, 3465, 0, 122, 0, 0, 0, 1700, 0, 3983, 42, 0, 0, 1, 1688, 0, 0, 2453, 0, 0, 0, 17, 3688, 0, 0, 0, 99, 0, 171, 3869, 285, 0] payload_rows_per_dst=[3918, 8048, 7786, 1984, 3496, 8056, 8168, 7850] dup_rows_per_dst=[86, 0, 386, 162, 148, 3372, 11078, 998] combine_rows_per_src=[3918, 8048, 7786, 1984, 3496, 8056, 8168, 7850] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54726 payload_rows_remote=46534 dup_rows=10810 dup_rows_remote=4094 dedup_ratio=0.164948 router_tokens_per_expert=[0, 2, 174, 0, 2899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3373, 0, 0, 3980, 0, 0, 0, 0, 0, 0, 0, 671, 0, 136, 0, 0, 3453, 700, 0, 241, 0, 0, 0, 0, 0, 4081, 46, 0, 0, 72, 1560, 0, 0, 3165, 0, 0, 0, 62, 3825, 0, 0, 0, 243, 0, 96, 3944, 45, 0] payload_rows_per_dst=[5958, 7960, 8192, 7882, 7196, 3276, 6340, 7922] dup_rows_per_dst=[192, 0, 6716, 1382, 1592, 80, 114, 734] combine_rows_per_src=[5958, 7960, 8192, 7882, 7196, 3276, 6340, 7922] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58522 payload_rows_remote=51398 dup_rows=7014 dup_rows_remote=5824 dedup_ratio=0.107025 router_tokens_per_expert=[0, 7, 97, 0, 2971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3623, 0, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 385, 0, 103, 0, 0, 3460, 627, 0, 55, 0, 0, 0, 0, 0, 4081, 9, 0, 0, 0, 1872, 0, 0, 3415, 0, 0, 0, 6, 3898, 0, 0, 0, 103, 0, 70, 3934, 7, 0] payload_rows_per_dst=[6060, 8090, 7246, 7928, 5978, 8182, 7124, 7914] dup_rows_per_dst=[90, 0, 0, 844, 834, 3742, 1190, 314] combine_rows_per_src=[6060, 8090, 7246, 7928, 5978, 8182, 7124, 7914] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48772 payload_rows_remote=40846 dup_rows=16764 dup_rows_remote=15062 dedup_ratio=0.255798 router_tokens_per_expert=[0, 20, 237, 0, 2780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3709, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 871, 0, 85, 0, 0, 3306, 983, 0, 102, 0, 0, 0, 0, 0, 4054, 41, 0, 0, 1, 1474, 0, 0, 3069, 0, 0, 0, 30, 3794, 0, 0, 0, 143, 0, 59, 3884, 77, 0] payload_rows_per_dst=[5858, 8098, 7418, 1840, 2088, 7356, 8188, 7926] dup_rows_per_dst=[216, 0, 0, 106, 82, 2482, 12176, 1702] combine_rows_per_src=[5858, 8098, 7418, 1840, 2088, 7356, 8188, 7926] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58290 payload_rows_remote=50106 dup_rows=7246 dup_rows_remote=4208 dedup_ratio=0.110565 router_tokens_per_expert=[0, 0, 308, 0, 2992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3432, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 766, 0, 153, 0, 0, 3138, 997, 0, 59, 0, 0, 0, 0, 0, 4088, 26, 0, 0, 5, 1492, 0, 0, 3115, 0, 0, 0, 25, 3754, 0, 0, 0, 109, 0, 261, 3936, 60, 0] payload_rows_per_dst=[6252, 8104, 6864, 7906, 6826, 8184, 6234, 7920] dup_rows_per_dst=[348, 0, 0, 1440, 1562, 3038, 46, 812] combine_rows_per_src=[6252, 8104, 6864, 7906, 6826, 8184, 6234, 7920] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52034 payload_rows_remote=43842 dup_rows=13502 dup_rows_remote=6494 dedup_ratio=0.206024 router_tokens_per_expert=[0, 9, 208, 0, 3297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3140, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 320, 0, 43, 0, 0, 3690, 857, 0, 136, 0, 0, 0, 0, 0, 4086, 11, 0, 0, 0, 1850, 0, 0, 2838, 0, 0, 0, 44, 4008, 0, 0, 0, 103, 0, 54, 3968, 37, 0] payload_rows_per_dst=[8192, 8140, 5514, 8042, 7708, 3722, 5678, 5038] dup_rows_per_dst=[7008, 764, 0, 700, 4746, 0, 86, 198] combine_rows_per_src=[8192, 8140, 5514, 8042, 7708, 3722, 5678, 5038] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=363933696 dispatch_meta_bytes=227168 combine_payload_bytes=363933696 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=364160864 ingress_bytes=474034176 link_bytes=474034176 token_bytes=728094560 weight_movement_bytes=110100480 total_nvlink_bytes=838195040 weight_to_token_ratio=0.151217 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=27264 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.244233973e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=334544896 dispatch_meta_bytes=224536 combine_payload_bytes=334544896 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=334769432 ingress_bytes=444645376 link_bytes=444645376 token_bytes=669314328 weight_movement_bytes=110100480 total_nvlink_bytes=779414808 weight_to_token_ratio=0.164497 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=24448 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.733114405e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55956 payload_rows_remote=48158 dup_rows=9580 dup_rows_remote=9570 dedup_ratio=0.146179 router_tokens_per_expert=[0, 0, 388, 3, 1849, 0, 0, 0, 1, 3, 1, 0, 0, 0, 0, 0, 3609, 2, 1, 128, 3, 0, 0, 2, 0, 0, 0, 256, 4, 401, 0, 0, 3899, 3492, 0, 97, 0, 1, 1, 1871, 0, 3992, 10, 1, 0, 15, 1813, 0, 0, 2794, 0, 0, 1, 36, 3656, 0, 0, 0, 117, 0, 183, 3854, 284, 0] payload_rows_per_dst=[8126, 7798, 7228, 7688, 7984, 3672, 5598, 7862] dup_rows_per_dst=[4338, 10, 262, 946, 2940, 6, 64, 1014] combine_rows_per_src=[8126, 7798, 7228, 7688, 7984, 3672, 5598, 7862] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=308453376 dispatch_meta_bytes=198112 combine_payload_bytes=308453376 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=308651488 ingress_bytes=528654336 link_bytes=528654336 token_bytes=617104864 weight_movement_bytes=220200960 total_nvlink_bytes=837305824 weight_to_token_ratio=0.356829 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=20352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.194168390e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=321470464 dispatch_meta_bytes=199432 combine_payload_bytes=321470464 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=321669896 ingress_bytes=541671424 link_bytes=541671424 token_bytes=643140360 weight_movement_bytes=220200960 total_nvlink_bytes=863341320 weight_to_token_ratio=0.342384 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=16640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.420556960e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=297156608 dispatch_meta_bytes=226752 combine_payload_bytes=297156608 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=297383360 ingress_bytes=517357568 link_bytes=517357568 token_bytes=594539968 weight_movement_bytes=220200960 total_nvlink_bytes=814740928 weight_to_token_ratio=0.370372 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=28288 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.997699015e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49556 payload_rows_remote=41378 dup_rows=15980 dup_rows_remote=10902 dedup_ratio=0.243835 router_tokens_per_expert=[0, 9, 235, 0, 2323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3587, 0, 0, 121, 0, 0, 0, 0, 0, 0, 0, 69, 0, 807, 0, 0, 4062, 3375, 0, 120, 0, 0, 0, 1567, 0, 4061, 10, 0, 0, 0, 2000, 0, 0, 1955, 0, 0, 0, 16, 3928, 0, 0, 0, 71, 0, 192, 3990, 270, 0] payload_rows_per_dst=[8178, 8140, 7858, 8012, 6858, 4020, 3934, 2556] dup_rows_per_dst=[5078, 1420, 4286, 1596, 3416, 0, 8, 176] combine_rows_per_src=[8178, 8140, 7858, 8012, 6858, 4020, 3934, 2556] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=345196544 dispatch_meta_bytes=230912 combine_payload_bytes=345196544 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=345427456 ingress_bytes=565397504 link_bytes=565397504 token_bytes=690624000 weight_movement_bytes=220200960 total_nvlink_bytes=910824960 weight_to_token_ratio=0.318843 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.833192514e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55658 payload_rows_remote=47598 dup_rows=9878 dup_rows_remote=9878 dedup_ratio=0.150726 router_tokens_per_expert=[0, 0, 323, 3, 3028, 0, 0, 0, 1, 2, 1, 0, 0, 0, 0, 0, 3287, 2, 1, 4026, 3, 0, 0, 3, 0, 0, 0, 768, 3, 22, 0, 0, 3094, 927, 0, 130, 0, 2, 0, 0, 0, 4070, 10, 0, 0, 24, 1789, 0, 0, 3241, 0, 0, 1, 59, 3824, 0, 0, 0, 160, 0, 47, 3856, 61, 0] payload_rows_per_dst=[7708, 8060, 7190, 7900, 6770, 3638, 6496, 7896] dup_rows_per_dst=[3892, 0, 2650, 1334, 1536, 8, 106, 352] combine_rows_per_src=[7708, 8060, 7190, 7900, 6770, 3638, 6496, 7896] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=372879360 dispatch_meta_bytes=226616 combine_payload_bytes=372879360 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=373105976 ingress_bytes=593080320 link_bytes=593080320 token_bytes=745985336 weight_movement_bytes=220200960 total_nvlink_bytes=966186296 weight_to_token_ratio=0.295181 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=23552 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.031464221e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55318 payload_rows_remote=49458 dup_rows=10218 dup_rows_remote=8918 dedup_ratio=0.155914 router_tokens_per_expert=[0, 7, 0, 0, 0, 0, 4080, 4095, 0, 3811, 0, 0, 0, 0, 0, 2274, 0, 93, 1, 739, 0, 0, 4095, 1, 0, 0, 0, 589, 0, 0, 0, 0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3001, 0, 0, 0, 231, 0, 0, 3714, 2, 0, 0, 0, 0, 0, 0, 4087, 1861, 0, 0, 0] payload_rows_per_dst=[8162, 5728, 5860, 7836, 8192, 8188, 7630, 3722] dup_rows_per_dst=[12, 1518, 1300, 964, 172, 5988, 264, 0] combine_rows_per_src=[8162, 5728, 5860, 7836, 8192, 8188, 7630, 3722] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58692 payload_rows_remote=50500 dup_rows=6844 dup_rows_remote=6712 dedup_ratio=0.104431 router_tokens_per_expert=[0, 38, 0, 0, 0, 0, 4091, 4096, 0, 3993, 0, 0, 0, 11, 0, 2207, 0, 62, 3, 830, 0, 0, 4095, 0, 0, 0, 0, 584, 0, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2776, 0, 0, 0, 448, 0, 0, 3818, 0, 0, 0, 0, 0, 0, 0, 4094, 1556, 0, 0, 0] payload_rows_per_dst=[8182, 4436, 8192, 8072, 8192, 5552, 7878, 8188] dup_rows_per_dst=[76, 0, 1788, 1082, 132, 0, 654, 3112] combine_rows_per_src=[8182, 4436, 8192, 8072, 8192, 5552, 7878, 8188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=296597504 dispatch_meta_bytes=209120 combine_payload_bytes=296597504 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=296806624 ingress_bytes=406697984 link_bytes=406697984 token_bytes=593404128 weight_movement_bytes=110100480 total_nvlink_bytes=703504608 weight_to_token_ratio=0.185540 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=15616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.073146845e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=333555712 dispatch_meta_bytes=202512 combine_payload_bytes=333555712 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=333758224 ingress_bytes=663857152 link_bytes=663857152 token_bytes=667313936 weight_movement_bytes=330301440 total_nvlink_bytes=997615376 weight_to_token_ratio=0.494972 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=27008 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.154556773e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=368420864 dispatch_meta_bytes=228888 combine_payload_bytes=368420864 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=368649752 ingress_bytes=588621824 link_bytes=588621824 token_bytes=737070616 weight_movement_bytes=220200960 total_nvlink_bytes=957271576 weight_to_token_ratio=0.298752 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=24704 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.023710164e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=292784128 dispatch_meta_bytes=223632 combine_payload_bytes=292784128 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=293007760 ingress_bytes=512985088 link_bytes=512985088 token_bytes=585791888 weight_movement_bytes=220200960 total_nvlink_bytes=805992848 weight_to_token_ratio=0.375903 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=29952 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.921654396e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=359159808 dispatch_meta_bytes=217256 combine_payload_bytes=359159808 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=359377064 ingress_bytes=689461248 link_bytes=689461248 token_bytes=718536872 weight_movement_bytes=330301440 total_nvlink_bytes=1048838312 weight_to_token_ratio=0.459686 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=19712 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.199086507e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=314259456 dispatch_meta_bytes=201344 combine_payload_bytes=314259456 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=314460800 ingress_bytes=424359936 link_bytes=424359936 token_bytes=628720256 weight_movement_bytes=110100480 total_nvlink_bytes=738820736 weight_to_token_ratio=0.175118 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=24576 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.380317239e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=341182464 dispatch_meta_bytes=229904 combine_payload_bytes=341182464 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=341412368 ingress_bytes=561383424 link_bytes=561383424 token_bytes=682594832 weight_movement_bytes=220200960 total_nvlink_bytes=902795792 weight_to_token_ratio=0.322594 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.763381060e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57938 payload_rows_remote=49746 dup_rows=7598 dup_rows_remote=7360 dedup_ratio=0.115936 router_tokens_per_expert=[0, 1256, 0, 0, 0, 9, 4045, 4096, 0, 0, 0, 0, 0, 0, 0, 2210, 0, 3, 5, 3962, 0, 0, 4095, 0, 0, 0, 0, 120, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 3852, 0, 0, 0, 192, 0, 0, 3556, 41, 0, 0, 4, 0, 0, 0, 4096, 1150, 0, 0, 0] payload_rows_per_dst=[8152, 8192, 7926, 8192, 8192, 7704, 7280, 2300] dup_rows_per_dst=[2468, 4420, 14, 238, 120, 32, 306, 0] combine_rows_per_src=[8152, 8192, 7926, 8192, 8192, 7704, 7280, 2300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59382 payload_rows_remote=51190 dup_rows=6154 dup_rows_remote=6042 dedup_ratio=0.093903 router_tokens_per_expert=[0, 1424, 0, 0, 0, 6, 4056, 4096, 0, 0, 0, 0, 0, 0, 0, 1652, 0, 16, 3, 4015, 0, 0, 4095, 0, 0, 0, 0, 133, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 3875, 0, 0, 0, 321, 0, 0, 3637, 7, 0, 0, 155, 0, 0, 0, 4096, 1091, 0, 0, 0] payload_rows_per_dst=[8178, 4086, 8032, 8190, 8192, 7750, 7510, 7444] dup_rows_per_dst=[2794, 256, 36, 266, 112, 68, 730, 1892] combine_rows_per_src=[8178, 4086, 8032, 8190, 8192, 7750, 7510, 7444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58548 payload_rows_remote=50658 dup_rows=6988 dup_rows_remote=6890 dedup_ratio=0.106628 router_tokens_per_expert=[0, 867, 0, 0, 0, 12, 3981, 4096, 0, 0, 0, 0, 0, 0, 0, 2080, 0, 81, 1, 3912, 0, 0, 4096, 0, 0, 0, 0, 135, 0, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 3900, 5, 0, 0, 339, 0, 0, 3472, 46, 0, 0, 53, 0, 0, 0, 4096, 1481, 0, 0, 0] payload_rows_per_dst=[8044, 8192, 7890, 8192, 8192, 7802, 7274, 2962] dup_rows_per_dst=[1676, 4160, 98, 270, 132, 106, 546, 0] combine_rows_per_src=[8044, 8192, 7890, 8192, 8192, 7802, 7274, 2962] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54130 payload_rows_remote=46236 dup_rows=11406 dup_rows_remote=11008 dedup_ratio=0.174042 router_tokens_per_expert=[0, 724, 0, 0, 0, 11, 4042, 4096, 0, 0, 0, 0, 0, 0, 0, 2301, 0, 6, 1, 3961, 0, 0, 4096, 4, 0, 0, 0, 136, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 3739, 22, 0, 0, 67, 0, 0, 3912, 2, 0, 0, 165, 0, 0, 0, 4096, 1329, 0, 0, 0] payload_rows_per_dst=[1470, 4602, 7924, 8192, 8192, 7664, 7894, 8192] dup_rows_per_dst=[0, 0, 20, 272, 96, 2818, 398, 7802] combine_rows_per_src=[1470, 4602, 7924, 8192, 8192, 7664, 7894, 8192] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58582 payload_rows_remote=50512 dup_rows=6954 dup_rows_remote=5728 dedup_ratio=0.106110 router_tokens_per_expert=[0, 23, 0, 0, 0, 5, 4082, 4096, 0, 3976, 0, 0, 0, 3, 0, 2724, 0, 33, 5, 659, 0, 0, 4096, 0, 0, 0, 0, 672, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2773, 0, 0, 0, 219, 0, 0, 3802, 0, 0, 0, 0, 0, 0, 0, 4094, 1445, 0, 0, 0] payload_rows_per_dst=[8174, 5450, 8192, 8070, 8192, 6160, 7758, 6586] dup_rows_per_dst=[46, 4, 1394, 1226, 122, 1788, 284, 2090] combine_rows_per_src=[8174, 5450, 8192, 8070, 8192, 6160, 7758, 6586] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59920 payload_rows_remote=51888 dup_rows=5616 dup_rows_remote=5456 dedup_ratio=0.085693 router_tokens_per_expert=[0, 31, 0, 0, 0, 0, 4087, 4096, 0, 4048, 0, 0, 0, 4, 0, 2679, 0, 70, 0, 543, 0, 0, 4096, 0, 0, 0, 0, 500, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2825, 1, 0, 0, 118, 0, 0, 3978, 0, 0, 0, 0, 0, 0, 0, 4094, 1549, 0, 0, 0] payload_rows_per_dst=[8174, 5366, 8192, 8120, 8192, 5652, 8032, 8192] dup_rows_per_dst=[62, 0, 1226, 976, 98, 0, 160, 3094] combine_rows_per_src=[8174, 5366, 8192, 8120, 8192, 5652, 8032, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=354514944 dispatch_meta_bytes=233504 combine_payload_bytes=354514944 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=354748448 ingress_bytes=574715904 link_bytes=574715904 token_bytes=709263392 weight_movement_bytes=220200960 total_nvlink_bytes=929464352 weight_to_token_ratio=0.310464 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=23296 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.995254816e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=361984000 dispatch_meta_bytes=228848 combine_payload_bytes=361984000 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=362212848 ingress_bytes=472084480 link_bytes=472084480 token_bytes=724196848 weight_movement_bytes=110100480 total_nvlink_bytes=834297328 weight_to_token_ratio=0.152031 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.210325553e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57080 payload_rows_remote=48888 dup_rows=8456 dup_rows_remote=2904 dedup_ratio=0.129028 router_tokens_per_expert=[0, 31, 0, 0, 0, 2, 4052, 4096, 0, 3976, 0, 0, 0, 9, 0, 2265, 0, 133, 3, 589, 0, 0, 4096, 3, 0, 0, 0, 389, 0, 0, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2852, 0, 0, 0, 212, 0, 0, 3759, 0, 0, 0, 0, 0, 0, 0, 4094, 2101, 0, 0, 0] payload_rows_per_dst=[6570, 4540, 8192, 8016, 8192, 5704, 7674, 8192] dup_rows_per_dst=[38, 8, 1456, 714, 212, 0, 476, 5552] combine_rows_per_src=[6570, 4540, 8192, 8016, 8192, 5704, 7674, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59142 payload_rows_remote=53236 dup_rows=6394 dup_rows_remote=6394 dedup_ratio=0.097565 router_tokens_per_expert=[0, 6, 0, 0, 0, 0, 4090, 4096, 0, 4044, 0, 0, 0, 6, 0, 2220, 0, 88, 0, 634, 0, 0, 4096, 1, 0, 0, 0, 424, 0, 0, 0, 0, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2953, 0, 0, 0, 172, 0, 0, 3905, 0, 0, 0, 0, 0, 0, 0, 4091, 1826, 0, 0, 0] payload_rows_per_dst=[8180, 4448, 8192, 8106, 8192, 5906, 7934, 8184] dup_rows_per_dst=[12, 4, 1446, 830, 232, 0, 220, 3650] combine_rows_per_src=[8180, 4448, 8192, 8106, 8192, 5906, 7934, 8184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=362070016 dispatch_meta_bytes=224960 combine_payload_bytes=362070016 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=362294976 ingress_bytes=472170496 link_bytes=472170496 token_bytes=724364992 weight_movement_bytes=110100480 total_nvlink_bytes=834465472 weight_to_token_ratio=0.151996 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=28544 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.211821512e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52694 payload_rows_remote=44504 dup_rows=12842 dup_rows_remote=8044 dedup_ratio=0.195953 router_tokens_per_expert=[3, 23, 0, 1, 1, 0, 4087, 4091, 0, 3984, 0, 0, 0, 8, 1, 2394, 1, 64, 1, 735, 3, 1, 4091, 0, 0, 2, 0, 473, 0, 0, 0, 1, 5, 143, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 3226, 0, 1, 0, 231, 3, 2, 3874, 1, 4, 0, 0, 2, 0, 0, 4092, 1216, 0, 1, 0] payload_rows_per_dst=[8182, 8190, 1592, 8040, 8192, 8188, 7872, 2438] dup_rows_per_dst=[48, 4798, 18, 880, 288, 6452, 358, 0] combine_rows_per_src=[8182, 8190, 1592, 8040, 8192, 8188, 7872, 2438] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=371933184 dispatch_meta_bytes=229376 combine_payload_bytes=371933184 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=372162560 ingress_bytes=482033664 link_bytes=482033664 token_bytes=744095744 weight_movement_bytes=110100480 total_nvlink_bytes=854196224 weight_to_token_ratio=0.147965 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=30720 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.383358226e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=350429184 dispatch_meta_bytes=207168 combine_payload_bytes=350429184 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=350636352 ingress_bytes=680730624 link_bytes=680730624 token_bytes=701065536 weight_movement_bytes=330301440 total_nvlink_bytes=1031366976 weight_to_token_ratio=0.471142 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=19328 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.183902516e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=381595648 dispatch_meta_bytes=238520 combine_payload_bytes=381595648 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=381834168 ingress_bytes=491696128 link_bytes=491696128 token_bytes=763429816 weight_movement_bytes=110100480 total_nvlink_bytes=873530296 weight_to_token_ratio=0.144218 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=22784 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.551404367e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48532 payload_rows_remote=40340 dup_rows=17004 dup_rows_remote=15578 dedup_ratio=0.259460 router_tokens_per_expert=[0, 15, 0, 0, 0, 0, 4091, 4096, 0, 4038, 0, 0, 0, 16, 0, 3046, 0, 150, 0, 660, 0, 0, 4095, 0, 0, 0, 0, 630, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2324, 0, 0, 0, 352, 0, 0, 3819, 0, 0, 0, 0, 0, 0, 0, 4093, 1268, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7172, 2414, 6814, 5304, 7908, 2536] dup_rows_per_dst=[1426, 12854, 1324, 190, 122, 654, 434, 0] combine_rows_per_src=[8192, 8192, 7172, 2414, 6814, 5304, 7908, 2536] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60328 payload_rows_remote=52136 dup_rows=5208 dup_rows_remote=5046 dedup_ratio=0.079468 router_tokens_per_expert=[0, 0, 0, 0, 0, 1882, 0, 0, 1997, 0, 0, 0, 0, 0, 0, 4025, 0, 0, 0, 0, 0, 0, 0, 81, 0, 4096, 0, 0, 0, 0, 184, 0, 0, 0, 0, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 56, 0, 4090, 0, 0, 3551, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8184, 3994, 8192, 8192, 8184, 8192, 7204, 8186] dup_rows_per_dst=[3630, 0, 162, 368, 1038, 0, 10, 0] combine_rows_per_src=[8184, 3994, 8192, 8192, 8184, 8192, 7204, 8186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60074 payload_rows_remote=51906 dup_rows=5462 dup_rows_remote=3704 dedup_ratio=0.083344 router_tokens_per_expert=[0, 0, 0, 0, 0, 1671, 0, 0, 1823, 0, 0, 0, 0, 0, 0, 4024, 0, 0, 0, 0, 0, 0, 0, 96, 0, 4096, 0, 0, 0, 0, 111, 0, 0, 0, 0, 881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 87, 0, 4082, 0, 0, 3612, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8168, 3646, 8192, 8192, 8168, 8192, 7330, 8186] dup_rows_per_dst=[3222, 0, 192, 222, 1758, 0, 68, 0] combine_rows_per_src=[8168, 3646, 8192, 8192, 8168, 8192, 7330, 8186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=319004672 dispatch_meta_bytes=210192 combine_payload_bytes=319004672 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=319214864 ingress_bytes=539205632 link_bytes=539205632 token_bytes=638219536 weight_movement_bytes=220200960 total_nvlink_bytes=858420496 weight_to_token_ratio=0.345024 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=19840 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.377672781e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52028 payload_rows_remote=43836 dup_rows=13508 dup_rows_remote=2008 dedup_ratio=0.206116 router_tokens_per_expert=[0, 1169, 0, 0, 0, 16, 3998, 4096, 0, 0, 0, 0, 0, 0, 0, 1683, 0, 60, 2, 3966, 0, 0, 4096, 2, 0, 0, 0, 125, 0, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 3777, 1, 0, 0, 199, 0, 0, 3484, 17, 0, 0, 85, 0, 0, 0, 4096, 1752, 0, 0, 0] payload_rows_per_dst=[2360, 3366, 6928, 8192, 8192, 7570, 7228, 8192] dup_rows_per_dst=[10, 0, 88, 250, 186, 88, 1386, 11500] combine_rows_per_src=[2360, 3366, 6928, 8192, 8192, 7570, 7228, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=356579328 dispatch_meta_bytes=228424 combine_payload_bytes=356579328 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=356807752 ingress_bytes=466679808 link_bytes=466679808 token_bytes=713387080 weight_movement_bytes=110100480 total_nvlink_bytes=823487560 weight_to_token_ratio=0.154335 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.116329418e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54566 payload_rows_remote=46398 dup_rows=10970 dup_rows_remote=7560 dedup_ratio=0.167389 router_tokens_per_expert=[0, 1106, 0, 0, 0, 14, 4038, 4096, 0, 0, 0, 0, 0, 0, 0, 2331, 0, 52, 0, 3960, 0, 0, 4096, 0, 0, 0, 0, 119, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 3729, 1, 0, 0, 299, 0, 0, 3721, 14, 0, 0, 23, 0, 0, 0, 4096, 939, 0, 0, 0] payload_rows_per_dst=[8168, 8192, 7978, 6230, 6962, 7482, 7676, 1878] dup_rows_per_dst=[3410, 4662, 2072, 174, 88, 126, 438, 0] combine_rows_per_src=[8168, 8192, 7978, 6230, 6962, 7482, 7676, 1878] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58606 payload_rows_remote=50414 dup_rows=6930 dup_rows_remote=3114 dedup_ratio=0.105743 router_tokens_per_expert=[1, 812, 0, 1, 1, 7, 4060, 4091, 0, 0, 0, 0, 0, 4, 0, 1908, 0, 13, 1, 4019, 3, 1, 4091, 0, 0, 2, 0, 119, 0, 0, 0, 0, 5, 144, 0, 3, 0, 0, 0, 0, 0, 0, 55, 0, 3955, 3, 1, 0, 292, 3, 2, 3818, 10, 4, 0, 196, 2, 0, 0, 4092, 1047, 0, 2, 0] payload_rows_per_dst=[8166, 8192, 8046, 8186, 8192, 7916, 7806, 2102] dup_rows_per_dst=[1598, 3816, 28, 238, 294, 112, 844, 0] combine_rows_per_src=[8166, 8192, 8046, 8186, 8192, 7916, 7806, 2102] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58806 payload_rows_remote=51060 dup_rows=6730 dup_rows_remote=6178 dedup_ratio=0.102692 router_tokens_per_expert=[0, 942, 0, 0, 0, 39, 4004, 4096, 0, 0, 0, 0, 0, 0, 0, 1700, 0, 54, 0, 3981, 0, 0, 4096, 1, 0, 0, 0, 131, 0, 0, 0, 0, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 3854, 7, 0, 0, 142, 0, 0, 3773, 4, 0, 0, 191, 0, 0, 0, 4096, 1540, 0, 0, 0] payload_rows_per_dst=[7388, 3400, 7998, 8192, 8192, 7746, 7698, 8192] dup_rows_per_dst=[1654, 0, 458, 262, 202, 552, 522, 3080] combine_rows_per_src=[7388, 3400, 7998, 8192, 8192, 7746, 7698, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=366929920 dispatch_meta_bytes=228928 combine_payload_bytes=366929920 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=367158848 ingress_bytes=477030400 link_bytes=477030400 token_bytes=734088768 weight_movement_bytes=110100480 total_nvlink_bytes=844189248 weight_to_token_ratio=0.149983 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.296343236e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=363116544 dispatch_meta_bytes=230192 combine_payload_bytes=363116544 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=363346736 ingress_bytes=693417984 link_bytes=693417984 token_bytes=726463280 weight_movement_bytes=330301440 total_nvlink_bytes=1056764720 weight_to_token_ratio=0.454671 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=31360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.205967922e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=331419648 dispatch_meta_bytes=228976 combine_payload_bytes=331419648 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=331648624 ingress_bytes=441520128 link_bytes=441520128 token_bytes=663068272 weight_movement_bytes=110100480 total_nvlink_bytes=773168752 weight_to_token_ratio=0.166047 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=29440 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.678761202e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=314216448 dispatch_meta_bytes=183376 combine_payload_bytes=314216448 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=314399824 ingress_bytes=534417408 link_bytes=534417408 token_bytes=628616272 weight_movement_bytes=220200960 total_nvlink_bytes=848817232 weight_to_token_ratio=0.350295 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=15872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.294397691e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=332580864 dispatch_meta_bytes=215832 combine_payload_bytes=332580864 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=332796696 ingress_bytes=772982784 link_bytes=772982784 token_bytes=665377560 weight_movement_bytes=440401920 total_nvlink_bytes=1105779480 weight_to_token_ratio=0.661883 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=23808 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.344344195e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=361367552 dispatch_meta_bytes=214112 combine_payload_bytes=361367552 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=361581664 ingress_bytes=471468032 link_bytes=471468032 token_bytes=722949216 weight_movement_bytes=110100480 total_nvlink_bytes=833049696 weight_to_token_ratio=0.152294 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=17024 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.199604508e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=365998080 dispatch_meta_bytes=228952 combine_payload_bytes=365998080 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=366227032 ingress_bytes=476098560 link_bytes=476098560 token_bytes=732225112 weight_movement_bytes=110100480 total_nvlink_bytes=842325592 weight_to_token_ratio=0.150364 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=30720 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.280137006e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64316 payload_rows_remote=56124 dup_rows=1220 dup_rows_remote=860 dedup_ratio=0.018616 router_tokens_per_expert=[0, 4083, 0, 0, 0, 233, 0, 3, 203, 0, 0, 5, 0, 0, 0, 3952, 0, 0, 0, 0, 0, 0, 0, 33, 0, 4096, 0, 0, 0, 0, 180, 0, 0, 0, 0, 3596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 10, 0, 4086, 0, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 7982, 8192, 8192, 7192, 8190, 8188, 8192] dup_rows_per_dst=[450, 338, 66, 360, 0, 0, 6, 0] combine_rows_per_src=[8188, 7982, 8192, 8192, 7192, 8190, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64426 payload_rows_remote=57056 dup_rows=1110 dup_rows_remote=1110 dedup_ratio=0.016937 router_tokens_per_expert=[0, 4083, 0, 0, 0, 245, 0, 0, 33, 0, 0, 0, 0, 0, 0, 3945, 0, 0, 0, 0, 0, 0, 0, 33, 0, 4096, 0, 0, 0, 0, 256, 0, 0, 0, 0, 3685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 11, 0, 4093, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 7910, 8192, 8192, 7370, 8192, 8188, 8192] dup_rows_per_dst=[466, 46, 66, 512, 0, 0, 20, 0] combine_rows_per_src=[8190, 7910, 8192, 8192, 7370, 8192, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63210 payload_rows_remote=55018 dup_rows=2326 dup_rows_remote=2136 dedup_ratio=0.035492 router_tokens_per_expert=[0, 4085, 0, 0, 0, 352, 0, 0, 103, 0, 0, 0, 0, 0, 0, 3800, 0, 0, 0, 0, 0, 0, 0, 95, 0, 4096, 0, 0, 0, 0, 627, 0, 0, 0, 0, 3223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 4094, 0, 0, 0, 4096, 3, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8178, 7628, 8192, 8192, 6446, 8192, 8190, 8192] dup_rows_per_dst=[696, 178, 190, 1254, 0, 0, 8, 0] combine_rows_per_src=[8178, 7628, 8192, 8192, 6446, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64116 payload_rows_remote=55924 dup_rows=1420 dup_rows_remote=1406 dedup_ratio=0.021667 router_tokens_per_expert=[0, 4091, 0, 0, 0, 296, 0, 0, 4, 0, 0, 0, 0, 0, 0, 3842, 0, 0, 0, 0, 0, 0, 0, 31, 0, 4093, 0, 0, 0, 0, 382, 0, 0, 0, 0, 3638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 7, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7688, 8192, 8192, 7276, 8192, 8192, 8192] dup_rows_per_dst=[582, 4, 62, 758, 0, 0, 14, 0] combine_rows_per_src=[8192, 7688, 8192, 8192, 7276, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=402296832 dispatch_meta_bytes=227936 combine_payload_bytes=402296832 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=402524768 ingress_bytes=622497792 link_bytes=622497792 token_bytes=804821600 weight_movement_bytes=220200960 total_nvlink_bytes=1025022560 weight_to_token_ratio=0.273602 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=28672 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.082626044e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61532 payload_rows_remote=53340 dup_rows=4004 dup_rows_remote=3854 dedup_ratio=0.061096 router_tokens_per_expert=[0, 0, 0, 0, 0, 1074, 0, 3, 2449, 0, 0, 5, 0, 0, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 61, 0, 4096, 0, 0, 0, 0, 75, 0, 0, 0, 0, 828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 68, 0, 4085, 0, 0, 3722, 4096, 0, 0, 4088, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8172, 4908, 8192, 8192, 8170, 8192, 7530, 8176] dup_rows_per_dst=[2026, 0, 122, 150, 1656, 0, 50, 0] combine_rows_per_src=[8172, 4908, 8192, 8192, 8170, 8192, 7530, 8176] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=289157120 dispatch_meta_bytes=223672 combine_payload_bytes=289157120 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=289380792 ingress_bytes=619458560 link_bytes=619458560 token_bytes=578537912 weight_movement_bytes=330301440 total_nvlink_bytes=908839352 weight_to_token_ratio=0.570924 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.077340319e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=373710848 dispatch_meta_bytes=228728 combine_payload_bytes=373710848 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=373939576 ingress_bytes=483811328 link_bytes=483811328 token_bytes=747650424 weight_movement_bytes=110100480 total_nvlink_bytes=857750904 weight_to_token_ratio=0.147262 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.414274727e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55984 payload_rows_remote=47792 dup_rows=9552 dup_rows_remote=2200 dedup_ratio=0.145752 router_tokens_per_expert=[0, 0, 0, 0, 0, 1990, 0, 0, 941, 0, 0, 0, 0, 0, 0, 4032, 0, 0, 0, 0, 0, 0, 0, 53, 0, 4094, 0, 0, 0, 0, 119, 0, 0, 0, 0, 1483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 39, 0, 4096, 0, 0, 3635, 4096, 2, 0, 4092, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[3980, 8086, 8192, 8192, 2966, 8192, 8192, 8184] dup_rows_per_dst=[0, 1860, 106, 234, 0, 0, 7352, 0] combine_rows_per_src=[3980, 8086, 8192, 8192, 2966, 8192, 8192, 8184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=372062208 dispatch_meta_bytes=222440 combine_payload_bytes=372062208 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=372284648 ingress_bytes=482162688 link_bytes=482162688 token_bytes=744346856 weight_movement_bytes=110100480 total_nvlink_bytes=854447336 weight_to_token_ratio=0.147916 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=24320 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.385602166e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52348 payload_rows_remote=44156 dup_rows=13188 dup_rows_remote=13144 dedup_ratio=0.201233 router_tokens_per_expert=[0, 0, 0, 0, 0, 2197, 0, 0, 909, 0, 0, 7, 0, 0, 0, 4025, 0, 0, 0, 0, 0, 0, 0, 118, 0, 4096, 0, 0, 0, 0, 131, 0, 0, 0, 0, 1269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 62, 0, 4077, 0, 0, 3588, 4096, 1, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4394, 8066, 6650, 6124, 2538, 8192, 8192, 8192] dup_rows_per_dst=[0, 1816, 206, 222, 0, 0, 10900, 44] combine_rows_per_src=[4394, 8066, 6650, 6124, 2538, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58898 payload_rows_remote=50706 dup_rows=6638 dup_rows_remote=6638 dedup_ratio=0.101288 router_tokens_per_expert=[0, 0, 0, 0, 0, 1438, 0, 0, 1372, 0, 0, 0, 0, 0, 0, 4026, 0, 0, 0, 0, 0, 0, 0, 207, 0, 4096, 0, 0, 0, 0, 193, 0, 0, 0, 0, 1411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 38, 0, 4089, 0, 0, 3614, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[2876, 8092, 8192, 8192, 7874, 8192, 7296, 8184] dup_rows_per_dst=[0, 2704, 414, 386, 2660, 0, 474, 0] combine_rows_per_src=[2876, 8092, 8192, 8192, 7874, 8192, 7296, 8184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=382341120 dispatch_meta_bytes=228776 combine_payload_bytes=382341120 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=382569896 ingress_bytes=712642560 link_bytes=712642560 token_bytes=764911016 weight_movement_bytes=330301440 total_nvlink_bytes=1095212456 weight_to_token_ratio=0.431817 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=31744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.239402621e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=342573056 dispatch_meta_bytes=199968 combine_payload_bytes=342573056 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=342773024 ingress_bytes=782974976 link_bytes=782974976 token_bytes=685346080 weight_movement_bytes=440401920 total_nvlink_bytes=1125748000 weight_to_token_ratio=0.642598 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=22144 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.361722261e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=316510208 dispatch_meta_bytes=229200 combine_payload_bytes=316510208 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=316739408 ingress_bytes=426610688 link_bytes=426610688 token_bytes=633249616 weight_movement_bytes=110100480 total_nvlink_bytes=743350096 weight_to_token_ratio=0.173866 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.419461518e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57296 payload_rows_remote=49888 dup_rows=8240 dup_rows_remote=5518 dedup_ratio=0.125732 router_tokens_per_expert=[0, 6, 0, 0, 0, 1177, 0, 0, 1713, 0, 3, 0, 0, 0, 0, 3963, 0, 0, 0, 0, 0, 3, 0, 138, 0, 4092, 0, 0, 0, 0, 172, 2, 0, 1, 5, 1349, 0, 0, 3, 0, 0, 0, 0, 3, 2, 0, 4089, 4, 57, 2, 4063, 0, 0, 3729, 4090, 0, 0, 4091, 0, 0, 6, 5, 0, 0] payload_rows_per_dst=[8100, 7408, 8186, 8186, 8160, 8186, 878, 8192] dup_rows_per_dst=[2192, 2722, 276, 346, 2682, 10, 0, 12] combine_rows_per_src=[8100, 7408, 8186, 8186, 8160, 8186, 878, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=363460608 dispatch_meta_bytes=229376 combine_payload_bytes=363460608 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=363689984 ingress_bytes=473561088 link_bytes=473561088 token_bytes=727150592 weight_movement_bytes=110100480 total_nvlink_bytes=837251072 weight_to_token_ratio=0.151414 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.236006194e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62782 payload_rows_remote=54590 dup_rows=2754 dup_rows_remote=2510 dedup_ratio=0.042023 router_tokens_per_expert=[0, 0, 0, 0, 0, 36, 0, 74, 0, 0, 1188, 0, 1944, 0, 0, 0, 1, 0, 120, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 3642, 4096, 1215, 0, 4092, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 57, 1, 0, 4042, 3, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 5982, 8192, 7820, 8192, 8086, 8128, 8192] dup_rows_per_dst=[214, 282, 244, 1894, 0, 114, 6, 0] combine_rows_per_src=[8190, 5982, 8192, 7820, 8192, 8086, 8128, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57288 payload_rows_remote=49098 dup_rows=8248 dup_rows_remote=5846 dedup_ratio=0.125854 router_tokens_per_expert=[0, 0, 0, 0, 0, 1264, 0, 0, 2381, 0, 0, 0, 0, 0, 0, 4032, 0, 0, 0, 0, 0, 0, 0, 35, 0, 4096, 0, 0, 0, 0, 71, 0, 0, 0, 0, 761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 28, 0, 4080, 0, 0, 3736, 4096, 2, 0, 4090, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8126, 8192, 8192, 8164, 8188, 56, 8180] dup_rows_per_dst=[2402, 4108, 70, 146, 1518, 0, 4, 0] combine_rows_per_src=[8190, 8126, 8192, 8192, 8164, 8188, 56, 8180] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63142 payload_rows_remote=54976 dup_rows=2394 dup_rows_remote=2394 dedup_ratio=0.036530 router_tokens_per_expert=[0, 0, 0, 0, 0, 48, 0, 39, 0, 0, 731, 0, 2462, 0, 0, 0, 9, 0, 51, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 3704, 4096, 1183, 0, 4095, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 26, 0, 0, 4042, 13, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 6200, 8192, 7936, 8166, 8088, 8178, 8192] dup_rows_per_dst=[174, 186, 122, 1838, 0, 48, 26, 0] combine_rows_per_src=[8190, 6200, 8192, 7936, 8166, 8088, 8178, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=357597184 dispatch_meta_bytes=221624 combine_payload_bytes=357597184 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=357818808 ingress_bytes=577798144 link_bytes=577798144 token_bytes=715415992 weight_movement_bytes=220200960 total_nvlink_bytes=935616952 weight_to_token_ratio=0.307794 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=13696 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.004886004e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=391301120 dispatch_meta_bytes=228400 combine_payload_bytes=391301120 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=391529520 ingress_bytes=501401600 link_bytes=501401600 token_bytes=782830640 weight_movement_bytes=110100480 total_nvlink_bytes=892931120 weight_to_token_ratio=0.140644 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.720198488e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63988 payload_rows_remote=56110 dup_rows=1548 dup_rows_remote=608 dedup_ratio=0.023621 router_tokens_per_expert=[0, 0, 0, 0, 0, 66, 0, 52, 0, 0, 704, 0, 2943, 1, 0, 0, 10, 0, 49, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 3826, 4096, 583, 0, 4093, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 10, 0, 0, 4060, 1, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 2, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7064, 8192, 7878, 8178, 8120, 8172, 8192] dup_rows_per_dst=[230, 232, 120, 940, 0, 20, 2, 4] combine_rows_per_src=[8192, 7064, 8192, 7878, 8178, 8120, 8172, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63400 payload_rows_remote=55226 dup_rows=2136 dup_rows_remote=2134 dedup_ratio=0.032593 router_tokens_per_expert=[0, 0, 0, 0, 0, 62, 0, 62, 0, 0, 1196, 0, 2453, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3493, 4096, 919, 0, 4095, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4049, 3, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 6802, 8192, 7568, 8184, 8098, 8174, 8192] dup_rows_per_dst=[248, 496, 124, 1256, 0, 10, 2, 0] combine_rows_per_src=[8190, 6802, 8192, 7568, 8184, 8098, 8174, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=351934464 dispatch_meta_bytes=219776 combine_payload_bytes=351934464 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=352154240 ingress_bytes=462034944 link_bytes=462034944 token_bytes=704088704 weight_movement_bytes=110100480 total_nvlink_bytes=814189184 weight_to_token_ratio=0.156373 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=20096 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.035547593e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=394067968 dispatch_meta_bytes=229480 combine_payload_bytes=394067968 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=394297448 ingress_bytes=834469888 link_bytes=834469888 token_bytes=788365416 weight_movement_bytes=440401920 total_nvlink_bytes=1228767336 weight_to_token_ratio=0.558627 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=31744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.451280382e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59590 payload_rows_remote=51398 dup_rows=5946 dup_rows_remote=5944 dedup_ratio=0.090729 router_tokens_per_expert=[0, 0, 0, 0, 0, 82, 0, 55, 0, 0, 685, 0, 2706, 0, 0, 0, 22, 0, 136, 11, 0, 0, 0, 0, 4096, 0, 0, 0, 3538, 4096, 997, 0, 4090, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 35, 0, 0, 4044, 12, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[6152, 6478, 6598, 7756, 8150, 8088, 8176, 8192] dup_rows_per_dst=[236, 304, 314, 2932, 1640, 390, 128, 2] combine_rows_per_src=[6152, 6478, 6598, 7756, 8150, 8088, 8176, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63242 payload_rows_remote=55056 dup_rows=2294 dup_rows_remote=2276 dedup_ratio=0.035004 router_tokens_per_expert=[0, 0, 0, 0, 0, 120, 0, 75, 0, 0, 909, 0, 2499, 0, 0, 0, 3, 0, 73, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 3573, 4096, 939, 0, 4089, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4090, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8178, 6462, 8192, 7648, 8192, 8186, 8192, 8192] dup_rows_per_dst=[390, 354, 154, 1376, 0, 18, 2, 0] combine_rows_per_src=[8178, 6462, 8192, 7648, 8192, 8186, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=402196480 dispatch_meta_bytes=226872 combine_payload_bytes=402196480 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=402423352 ingress_bytes=842598400 link_bytes=842598400 token_bytes=804619832 weight_movement_bytes=440401920 total_nvlink_bytes=1245021752 weight_to_token_ratio=0.547342 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=23424 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.465417201e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=395859968 dispatch_meta_bytes=229440 combine_payload_bytes=395859968 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=396089408 ingress_bytes=505960448 link_bytes=505960448 token_bytes=791949376 weight_movement_bytes=110100480 total_nvlink_bytes=902049856 weight_to_token_ratio=0.139025 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.799484353e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=368420864 dispatch_meta_bytes=229368 combine_payload_bytes=368420864 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=368650232 ingress_bytes=478521344 link_bytes=478521344 token_bytes=737071096 weight_movement_bytes=110100480 total_nvlink_bytes=847171576 weight_to_token_ratio=0.149376 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.322273205e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=394641408 dispatch_meta_bytes=229328 combine_payload_bytes=394641408 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=394870736 ingress_bytes=504741888 link_bytes=504741888 token_bytes=789512144 weight_movement_bytes=110100480 total_nvlink_bytes=899612624 weight_to_token_ratio=0.139454 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.778291590e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59088 payload_rows_remote=50896 dup_rows=6448 dup_rows_remote=6448 dedup_ratio=0.098389 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 750, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1878, 4051, 0, 0, 4, 0, 0, 4073, 0, 0, 24, 0, 0, 3, 0, 1, 0, 4071, 0, 0, 0, 4096, 0, 0, 2961, 4096, 0, 1475, 0, 0, 0, 0, 0, 152, 0] payload_rows_per_dst=[8192, 8192, 8192, 7056, 8154, 8150, 5922, 5230] dup_rows_per_dst=[0, 3574, 0, 1788, 48, 0, 0, 1038] combine_rows_per_src=[8192, 8192, 8192, 7056, 8154, 8150, 5922, 5230] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61054 payload_rows_remote=53488 dup_rows=4482 dup_rows_remote=1194 dedup_ratio=0.068390 router_tokens_per_expert=[13, 0, 0, 0, 0, 83, 0, 97, 0, 0, 752, 4, 2784, 2, 4, 1, 53, 0, 101, 5, 5, 0, 1, 0, 4095, 0, 0, 0, 3467, 4091, 898, 4, 4093, 0, 1, 0, 0, 4081, 0, 1, 0, 0, 0, 0, 24, 0, 0, 4016, 4, 0, 0, 0, 0, 0, 4080, 0, 0, 3, 0, 0, 1, 0, 4, 0] payload_rows_per_dst=[8190, 7566, 8192, 4554, 8166, 8034, 8160, 8192] dup_rows_per_dst=[382, 3288, 320, 424, 0, 46, 8, 14] combine_rows_per_src=[8190, 7566, 8192, 4554, 8166, 8034, 8160, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60518 payload_rows_remote=52326 dup_rows=5018 dup_rows_remote=4942 dedup_ratio=0.076569 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 617, 265, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 2514, 4086, 0, 0, 9, 0, 0, 4091, 5, 0, 29, 0, 0, 12, 0, 3, 0, 4082, 0, 0, 0, 4096, 0, 0, 3180, 4096, 0, 1298, 0, 0, 0, 0, 0, 289, 0] payload_rows_per_dst=[8192, 8192, 8192, 5028, 8192, 8182, 6360, 8180] dup_rows_per_dst=[0, 1764, 0, 0, 76, 12, 0, 3166] combine_rows_per_src=[8192, 8192, 8192, 5028, 8192, 8182, 6360, 8180] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58776 payload_rows_remote=50584 dup_rows=6760 dup_rows_remote=6572 dedup_ratio=0.103149 router_tokens_per_expert=[0, 0, 0, 0, 0, 31, 0, 63, 0, 0, 457, 0, 3003, 0, 0, 0, 7, 0, 91, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3460, 4096, 1093, 0, 4096, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 28, 0, 0, 4067, 7, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8036, 8192, 3214, 6668, 8134, 8168, 8172] dup_rows_per_dst=[188, 5804, 196, 502, 0, 56, 14, 0] combine_rows_per_src=[8192, 8036, 8192, 3214, 6668, 8134, 8168, 8172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63484 payload_rows_remote=55292 dup_rows=2052 dup_rows_remote=700 dedup_ratio=0.031311 router_tokens_per_expert=[0, 4087, 0, 0, 0, 213, 0, 0, 85, 0, 0, 0, 0, 0, 0, 4001, 0, 0, 0, 0, 0, 0, 0, 38, 0, 4096, 0, 0, 0, 0, 221, 0, 0, 0, 0, 3635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 9, 0, 4089, 0, 0, 0, 4096, 6, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8014, 8192, 8192, 7270, 8192, 7252, 8180] dup_rows_per_dst=[1352, 158, 76, 442, 0, 2, 22, 0] combine_rows_per_src=[8192, 8014, 8192, 8192, 7270, 8192, 7252, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64296 payload_rows_remote=56616 dup_rows=1240 dup_rows_remote=1212 dedup_ratio=0.018921 router_tokens_per_expert=[0, 4092, 0, 0, 0, 202, 0, 0, 22, 0, 3, 0, 2, 0, 0, 3827, 0, 0, 0, 0, 0, 2, 0, 44, 0, 4092, 0, 0, 0, 0, 350, 2, 0, 1, 5, 3752, 0, 0, 4, 0, 0, 0, 0, 1, 2, 0, 4088, 5, 7, 2, 4068, 0, 0, 1, 4091, 1, 0, 4091, 0, 0, 6, 5, 0, 0] payload_rows_per_dst=[8190, 7680, 8186, 8186, 7518, 8186, 8158, 8192] dup_rows_per_dst=[398, 28, 88, 702, 6, 6, 0, 12] combine_rows_per_src=[8190, 7680, 8186, 8186, 7518, 8186, 8158, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54576 payload_rows_remote=46384 dup_rows=10960 dup_rows_remote=10960 dedup_ratio=0.167236 router_tokens_per_expert=[0, 4067, 0, 0, 0, 318, 0, 0, 5, 0, 0, 8, 0, 0, 0, 3993, 0, 0, 0, 0, 0, 0, 0, 43, 0, 4096, 0, 0, 0, 0, 242, 0, 0, 0, 0, 3602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 4084, 0, 0, 0, 4096, 17, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[3466, 8034, 7284, 3038, 8186, 8192, 8184, 8192] dup_rows_per_dst=[318, 2332, 86, 84, 7204, 0, 936, 0] combine_rows_per_src=[3466, 8034, 7284, 3038, 8186, 8192, 8184, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=408977408 dispatch_meta_bytes=232664 combine_payload_bytes=408977408 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=409210072 ingress_bytes=519077888 link_bytes=519077888 token_bytes=818187480 weight_movement_bytes=110100480 total_nvlink_bytes=928287960 weight_to_token_ratio=0.134566 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=28672 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.027618209e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63684 payload_rows_remote=55492 dup_rows=1852 dup_rows_remote=1852 dedup_ratio=0.028259 router_tokens_per_expert=[0, 4086, 0, 0, 0, 253, 0, 0, 14, 0, 0, 0, 0, 0, 0, 3744, 0, 0, 0, 0, 0, 0, 0, 140, 0, 4096, 0, 0, 0, 0, 524, 0, 0, 0, 0, 3534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4089, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8174, 7496, 8192, 8192, 7068, 8192, 8178, 8192] dup_rows_per_dst=[504, 20, 280, 1048, 0, 0, 0, 0] combine_rows_per_src=[8174, 7496, 8192, 8192, 7068, 8192, 8178, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=394369024 dispatch_meta_bytes=228616 combine_payload_bytes=394369024 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=394597640 ingress_bytes=504469504 link_bytes=504469504 token_bytes=788966664 weight_movement_bytes=110100480 total_nvlink_bytes=899067144 weight_to_token_ratio=0.139550 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.773554384e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=400863232 dispatch_meta_bytes=229320 combine_payload_bytes=400863232 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=401092552 ingress_bytes=731164672 link_bytes=731164672 token_bytes=801955784 weight_movement_bytes=330301440 total_nvlink_bytes=1132257224 weight_to_token_ratio=0.411870 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.271615621e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=396333056 dispatch_meta_bytes=223968 combine_payload_bytes=396333056 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=396557024 ingress_bytes=726634496 link_bytes=726634496 token_bytes=792890080 weight_movement_bytes=330301440 total_nvlink_bytes=1123191520 weight_to_token_ratio=0.416579 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=30208 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.263736899e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=405823488 dispatch_meta_bytes=231312 combine_payload_bytes=405823488 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=406054800 ingress_bytes=515923968 link_bytes=515923968 token_bytes=811878288 weight_movement_bytes=110100480 total_nvlink_bytes=921978768 weight_to_token_ratio=0.135612 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=31104 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.972766353e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=332480512 dispatch_meta_bytes=229376 combine_payload_bytes=332480512 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=332709888 ingress_bytes=552681472 link_bytes=552681472 token_bytes=665190400 weight_movement_bytes=220200960 total_nvlink_bytes=885391360 weight_to_token_ratio=0.331034 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.612039803e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=397766656 dispatch_meta_bytes=229376 combine_payload_bytes=397766656 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=397996032 ingress_bytes=507867136 link_bytes=507867136 token_bytes=795762688 weight_movement_bytes=110100480 total_nvlink_bytes=905863168 weight_to_token_ratio=0.138358 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.832644793e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60110 payload_rows_remote=51918 dup_rows=5426 dup_rows_remote=2576 dedup_ratio=0.082794 router_tokens_per_expert=[0, 0, 0, 0, 0, 32, 0, 256, 0, 0, 2701, 0, 1717, 0, 0, 0, 2, 0, 33, 6, 0, 0, 0, 0, 4096, 0, 0, 0, 401, 4096, 1024, 0, 4095, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 128, 0, 0, 3963, 2126, 0, 0, 0, 0, 0, 4000, 0, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[8046, 7110, 8192, 8192, 8182, 7944, 4252, 8192] dup_rows_per_dst=[530, 1726, 80, 2850, 0, 238, 0, 2] combine_rows_per_src=[8046, 7110, 8192, 8192, 8182, 7944, 4252, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61142 payload_rows_remote=52968 dup_rows=4394 dup_rows_remote=4394 dedup_ratio=0.067047 router_tokens_per_expert=[0, 0, 0, 0, 0, 24, 0, 120, 0, 0, 2315, 0, 1551, 1, 0, 0, 0, 0, 39, 9, 0, 0, 0, 0, 4096, 0, 0, 0, 273, 4096, 1148, 0, 4096, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 101, 0, 0, 3915, 2936, 0, 0, 0, 0, 0, 3961, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7968, 6694, 8192, 8192, 8174, 7858, 5872, 8192] dup_rows_per_dst=[242, 1040, 96, 2842, 0, 174, 0, 0] combine_rows_per_src=[7968, 6694, 8192, 8192, 8174, 7858, 5872, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=372148224 dispatch_meta_bytes=217976 combine_payload_bytes=372148224 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=372366200 ingress_bytes=812550144 link_bytes=812550144 token_bytes=744514424 weight_movement_bytes=440401920 total_nvlink_bytes=1184916344 weight_to_token_ratio=0.591529 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=20736 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.413158342e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60148 payload_rows_remote=51956 dup_rows=5388 dup_rows_remote=5250 dedup_ratio=0.082214 router_tokens_per_expert=[0, 0, 0, 0, 0, 5, 0, 304, 0, 0, 2803, 0, 1012, 0, 0, 0, 0, 0, 55, 14, 0, 0, 0, 0, 4096, 0, 0, 0, 271, 4096, 1413, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 249, 0, 0, 3502, 2869, 0, 0, 0, 0, 0, 3887, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7890, 6560, 8192, 8192, 8192, 7192, 5738, 8192] dup_rows_per_dst=[502, 1070, 138, 3368, 0, 310, 0, 0] combine_rows_per_src=[7890, 6560, 8192, 8192, 8192, 7192, 5738, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57926 payload_rows_remote=49734 dup_rows=7610 dup_rows_remote=1736 dedup_ratio=0.116119 router_tokens_per_expert=[0, 0, 0, 0, 0, 15, 0, 151, 0, 0, 2538, 0, 1457, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 138, 4096, 1168, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 102, 0, 0, 3840, 2937, 0, 0, 0, 0, 0, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8030, 6836, 8192, 2568, 8188, 7728, 8192, 8192] dup_rows_per_dst=[306, 1154, 76, 44, 0, 156, 5874, 0] combine_rows_per_src=[8030, 6836, 8192, 2568, 8188, 7728, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58844 payload_rows_remote=50918 dup_rows=6692 dup_rows_remote=6148 dedup_ratio=0.102112 router_tokens_per_expert=[0, 0, 0, 0, 0, 9, 0, 328, 0, 0, 1698, 0, 1819, 0, 0, 0, 1, 0, 47, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 316, 4096, 1751, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 169, 0, 0, 3918, 2433, 0, 0, 0, 0, 0, 3898, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7926, 7324, 8192, 6332, 8182, 7844, 4866, 8178] dup_rows_per_dst=[544, 4118, 100, 1600, 0, 330, 0, 0] combine_rows_per_src=[7926, 7324, 8192, 6332, 8182, 7844, 4866, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=379674624 dispatch_meta_bytes=229448 combine_payload_bytes=379674624 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=379904072 ingress_bytes=709976064 link_bytes=709976064 token_bytes=759578696 weight_movement_bytes=330301440 total_nvlink_bytes=1089880136 weight_to_token_ratio=0.434848 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.234765146e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60450 payload_rows_remote=53742 dup_rows=5086 dup_rows_remote=3988 dedup_ratio=0.077606 router_tokens_per_expert=[11, 0, 0, 0, 0, 27, 0, 283, 0, 0, 2265, 4, 1621, 9, 4, 0, 10, 0, 41, 28, 5, 0, 1, 0, 4095, 0, 0, 0, 266, 4091, 1094, 4, 4094, 0, 1, 0, 0, 4083, 0, 2, 0, 0, 0, 0, 293, 0, 0, 3784, 2621, 0, 0, 0, 0, 0, 4024, 0, 0, 3, 0, 0, 0, 0, 4, 0] payload_rows_per_dst=[8088, 6708, 8192, 8190, 8172, 7666, 5242, 8192] dup_rows_per_dst=[602, 1098, 166, 2720, 0, 488, 0, 12] combine_rows_per_src=[8088, 6708, 8192, 8190, 8172, 7666, 5242, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60296 payload_rows_remote=52264 dup_rows=5240 dup_rows_remote=5144 dedup_ratio=0.079956 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 312, 0, 0, 2303, 0, 1255, 1, 0, 0, 0, 0, 42, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 237, 4096, 1295, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 66, 0, 0, 3998, 2815, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8138, 6354, 8190, 6400, 8192, 8032, 6798, 8192] dup_rows_per_dst=[602, 764, 90, 1588, 88, 96, 2012, 0] combine_rows_per_src=[8138, 6354, 8190, 6400, 8192, 8032, 6798, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54304 payload_rows_remote=46112 dup_rows=11232 dup_rows_remote=11232 dedup_ratio=0.171387 router_tokens_per_expert=[0, 0, 0, 0, 0, 45, 0, 192, 0, 0, 1997, 0, 1491, 2, 0, 0, 2, 0, 42, 23, 0, 0, 0, 0, 4096, 0, 0, 0, 137, 4096, 1092, 0, 4096, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 392, 0, 0, 3915, 3084, 0, 0, 0, 0, 0, 3989, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5866, 6262, 7324, 2418, 8154, 7896, 8192, 8192] dup_rows_per_dst=[256, 718, 124, 40, 12, 1584, 8498, 0] combine_rows_per_src=[5866, 6262, 7324, 2418, 8154, 7896, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=372420608 dispatch_meta_bytes=228824 combine_payload_bytes=372420608 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=372649432 ingress_bytes=482521088 link_bytes=482521088 token_bytes=745070040 weight_movement_bytes=110100480 total_nvlink_bytes=855170520 weight_to_token_ratio=0.147772 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.391835331e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=356493312 dispatch_meta_bytes=205880 combine_payload_bytes=356493312 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=356699192 ingress_bytes=576694272 link_bytes=576694272 token_bytes=713192504 weight_movement_bytes=220200960 total_nvlink_bytes=933393464 weight_to_token_ratio=0.308754 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=21888 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.002966189e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=364822528 dispatch_meta_bytes=229376 combine_payload_bytes=364822528 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=365051904 ingress_bytes=474923008 link_bytes=474923008 token_bytes=729874432 weight_movement_bytes=110100480 total_nvlink_bytes=839974912 weight_to_token_ratio=0.150849 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.259692223e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55994 payload_rows_remote=47802 dup_rows=9542 dup_rows_remote=3040 dedup_ratio=0.145599 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 148, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2302, 4091, 0, 0, 0, 0, 0, 4079, 0, 0, 54, 0, 0, 0, 0, 3, 0, 4073, 0, 0, 0, 4096, 0, 0, 3251, 4094, 0, 1307, 0, 0, 0, 0, 0, 132, 0] payload_rows_per_dst=[8188, 2318, 8192, 4604, 8158, 8152, 8192, 8190] dup_rows_per_dst=[0, 50, 12, 0, 108, 0, 6502, 2870] combine_rows_per_src=[8188, 2318, 8192, 4604, 8158, 8152, 8192, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59156 payload_rows_remote=56448 dup_rows=6380 dup_rows_remote=6380 dedup_ratio=0.097351 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 688, 435, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1354, 4089, 0, 0, 10, 0, 0, 4085, 1, 0, 73, 0, 0, 10, 0, 4, 0, 4081, 0, 0, 0, 4096, 0, 0, 3665, 4096, 0, 1557, 0, 0, 0, 0, 0, 425, 0] payload_rows_per_dst=[8192, 8192, 8192, 2708, 8188, 8176, 7330, 8178] dup_rows_per_dst=[0, 2246, 6, 0, 150, 14, 0, 3964] combine_rows_per_src=[8192, 8192, 8192, 2708, 8188, 8176, 7330, 8178] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=383401984 dispatch_meta_bytes=218728 combine_payload_bytes=383401984 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=383620712 ingress_bytes=493502464 link_bytes=493502464 token_bytes=767022696 weight_movement_bytes=110100480 total_nvlink_bytes=877123176 weight_to_token_ratio=0.143543 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=20864 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.582819521e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57252 payload_rows_remote=49076 dup_rows=8284 dup_rows_remote=6224 dedup_ratio=0.126404 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 942, 503, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 3019, 4086, 0, 0, 19, 0, 0, 4065, 10, 0, 45, 0, 0, 10, 0, 1, 0, 4064, 0, 0, 0, 4096, 0, 0, 2688, 4096, 0, 873, 0, 0, 0, 0, 0, 159, 0] payload_rows_per_dst=[8192, 2776, 8162, 6040, 7582, 8132, 8192, 8176] dup_rows_per_dst=[0, 114, 2, 18, 112, 206, 5772, 2060] combine_rows_per_src=[8192, 2776, 8162, 6040, 7582, 8132, 8192, 8176] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58008 payload_rows_remote=49850 dup_rows=7528 dup_rows_remote=7528 dedup_ratio=0.114868 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1461, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2674, 4088, 0, 0, 0, 0, 0, 4086, 0, 0, 48, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 4096, 0, 0, 2678, 4096, 0, 878, 0, 0, 0, 0, 0, 115, 0] payload_rows_per_dst=[8192, 5404, 8188, 5348, 8172, 8158, 6368, 8178] dup_rows_per_dst=[0, 2338, 0, 0, 96, 0, 3110, 1984] combine_rows_per_src=[8192, 5404, 8188, 5348, 8172, 8158, 6368, 8178] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=375072768 dispatch_meta_bytes=229072 combine_payload_bytes=375072768 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=375301840 ingress_bytes=595273728 link_bytes=595273728 token_bytes=750374608 weight_movement_bytes=220200960 total_nvlink_bytes=970575568 weight_to_token_ratio=0.293455 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.035278919e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=362586112 dispatch_meta_bytes=228624 combine_payload_bytes=362586112 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=362814736 ingress_bytes=472686592 link_bytes=472686592 token_bytes=725400848 weight_movement_bytes=110100480 total_nvlink_bytes=835501328 weight_to_token_ratio=0.151779 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=31744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.220797271e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[2, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=342644736 dispatch_meta_bytes=203368 combine_payload_bytes=342644736 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=342848104 ingress_bytes=893147136 link_bytes=893147136 token_bytes=685492840 weight_movement_bytes=550502400 total_nvlink_bytes=1235995240 weight_to_token_ratio=0.803075 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=22400 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.553329767e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=404619264 dispatch_meta_bytes=251312 combine_payload_bytes=404619264 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=404870576 ingress_bytes=734920704 link_bytes=734920704 token_bytes=809489840 weight_movement_bytes=330301440 total_nvlink_bytes=1139791280 weight_to_token_ratio=0.408037 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=19712 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.278147978e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=351776768 dispatch_meta_bytes=221200 combine_payload_bytes=351776768 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=351997968 ingress_bytes=461877248 link_bytes=461877248 token_bytes=703774736 weight_movement_bytes=110100480 total_nvlink_bytes=813875216 weight_to_token_ratio=0.156443 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=22016 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.032805000e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=357324800 dispatch_meta_bytes=229512 combine_payload_bytes=357324800 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=357554312 ingress_bytes=467425280 link_bytes=467425280 token_bytes=714879112 weight_movement_bytes=110100480 total_nvlink_bytes=824979592 weight_to_token_ratio=0.154013 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.129294402e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60406 payload_rows_remote=52214 dup_rows=5130 dup_rows_remote=5080 dedup_ratio=0.078278 router_tokens_per_expert=[1541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 3971, 0, 0, 0, 0, 0, 0, 0, 0, 19, 6, 0, 0, 213, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3489, 78, 0, 0, 0, 0, 0, 0, 0, 0, 800, 0, 4096, 0, 0, 4026, 0, 0, 0, 0, 0, 0, 0, 2302, 0] payload_rows_per_dst=[8192, 7942, 8192, 8180, 6978, 8188, 8130, 4604] dup_rows_per_dst=[3082, 0, 50, 382, 0, 94, 1522, 0] combine_rows_per_src=[8192, 7942, 8192, 8180, 6978, 8188, 8130, 4604] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57382 payload_rows_remote=49194 dup_rows=8154 dup_rows_remote=5936 dedup_ratio=0.124420 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 1, 876, 238, 0, 4, 3, 0, 0, 2, 0, 0, 0, 0, 4090, 0, 2, 2824, 4090, 1, 0, 9, 1, 3, 3998, 0, 0, 91, 1, 0, 2, 5, 0, 0, 4074, 4, 0, 0, 4084, 0, 0, 3067, 4091, 2, 981, 0, 0, 0, 2, 0, 217, 0] payload_rows_per_dst=[8184, 8188, 8188, 8188, 8026, 8162, 6138, 2308] dup_rows_per_dst=[0, 2218, 10, 5646, 178, 10, 0, 92] combine_rows_per_src=[8184, 8188, 8188, 8188, 8026, 8162, 6138, 2308] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61216 payload_rows_remote=53476 dup_rows=4320 dup_rows_remote=4320 dedup_ratio=0.065918 router_tokens_per_expert=[988, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4070, 4006, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 225, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3870, 99, 0, 0, 0, 0, 0, 0, 0, 0, 904, 0, 4096, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 2277, 0] payload_rows_per_dst=[8192, 8012, 8192, 8192, 7740, 8186, 8148, 4554] dup_rows_per_dst=[1980, 0, 4, 426, 0, 152, 1758, 0] combine_rows_per_src=[8192, 8012, 8192, 8192, 7740, 8186, 8148, 4554] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52626 payload_rows_remote=44434 dup_rows=12910 dup_rows_remote=12898 dedup_ratio=0.196991 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 432, 457, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 3020, 4076, 0, 0, 5, 0, 0, 4084, 0, 0, 59, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4096, 0, 0, 3291, 4096, 0, 640, 0, 0, 0, 0, 0, 321, 0] payload_rows_per_dst=[8192, 8192, 8190, 8192, 8178, 8182, 1650, 1850] dup_rows_per_dst=[12, 1778, 10, 10920, 118, 0, 0, 72] combine_rows_per_src=[8192, 8192, 8190, 8192, 8178, 8182, 1650, 1850] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=374269952 dispatch_meta_bytes=229176 combine_payload_bytes=374269952 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=374499128 ingress_bytes=484370432 link_bytes=484370432 token_bytes=748769080 weight_movement_bytes=110100480 total_nvlink_bytes=858869560 weight_to_token_ratio=0.147042 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.423998465e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56226 payload_rows_remote=50052 dup_rows=9310 dup_rows_remote=8272 dedup_ratio=0.142059 router_tokens_per_expert=[585, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4080, 4004, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 318, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3905, 105, 0, 0, 0, 0, 0, 0, 0, 0, 691, 0, 4096, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 2734, 0] payload_rows_per_dst=[1174, 8008, 8192, 8192, 8148, 8188, 6174, 8150] dup_rows_per_dst=[44, 0, 14, 608, 1980, 182, 1038, 5444] combine_rows_per_src=[1174, 8008, 8192, 8192, 8148, 8188, 6174, 8150] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62770 payload_rows_remote=54578 dup_rows=2766 dup_rows_remote=2126 dedup_ratio=0.042206 router_tokens_per_expert=[596, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4076, 4025, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 339, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3679, 68, 0, 0, 0, 0, 0, 0, 0, 0, 423, 0, 4096, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 3230, 0] payload_rows_per_dst=[8192, 8050, 8192, 8192, 7358, 8180, 8146, 6460] dup_rows_per_dst=[1196, 0, 8, 640, 0, 108, 814, 0] combine_rows_per_src=[8192, 8050, 8192, 8192, 7358, 8180, 8146, 6460] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55926 payload_rows_remote=47820 dup_rows=9610 dup_rows_remote=4674 dedup_ratio=0.146637 router_tokens_per_expert=[1332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 223, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3750, 88, 0, 0, 0, 0, 0, 0, 0, 0, 527, 0, 4096, 0, 0, 4005, 0, 0, 0, 0, 0, 0, 0, 2516, 0] payload_rows_per_dst=[2664, 8058, 8034, 5072, 8192, 8126, 7674, 8106] dup_rows_per_dst=[0, 0, 48, 256, 3218, 114, 1038, 4936] combine_rows_per_src=[2664, 8058, 8034, 5072, 8192, 8126, 7674, 8106] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=364980224 dispatch_meta_bytes=228264 combine_payload_bytes=364980224 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=365208488 ingress_bytes=475080704 link_bytes=475080704 token_bytes=730188712 weight_movement_bytes=110100480 total_nvlink_bytes=840289192 weight_to_token_ratio=0.150784 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=30720 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.262434816e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=385222656 dispatch_meta_bytes=230920 combine_payload_bytes=385222656 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=385453576 ingress_bytes=495323136 link_bytes=495323136 token_bytes=770676232 weight_movement_bytes=110100480 total_nvlink_bytes=880776712 weight_to_token_ratio=0.142862 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=18688 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.614484002e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=374628352 dispatch_meta_bytes=229632 combine_payload_bytes=374628352 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=374857984 ingress_bytes=484728832 link_bytes=484728832 token_bytes=749486336 weight_movement_bytes=110100480 total_nvlink_bytes=859586816 weight_to_token_ratio=0.146901 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=30848 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.430231630e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63166 payload_rows_remote=54976 dup_rows=2370 dup_rows_remote=1556 dedup_ratio=0.036163 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, 3740, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 31, 376, 0, 0, 6, 0, 0, 4085, 4, 0, 3400, 0, 0, 11, 0, 2, 0, 4091, 0, 0, 0, 4096, 0, 0, 608, 4087, 0, 2, 0, 0, 0, 0, 0, 3833, 0] payload_rows_per_dst=[8192, 7754, 8176, 8190, 6812, 8190, 8186, 7666] dup_rows_per_dst=[0, 322, 0, 814, 8, 18, 1204, 4] combine_rows_per_src=[8192, 7754, 8176, 8190, 6812, 8190, 8186, 7666] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=330530816 dispatch_meta_bytes=229376 combine_payload_bytes=330530816 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=330760192 ingress_bytes=440631296 link_bytes=440631296 token_bytes=661291008 weight_movement_bytes=110100480 total_nvlink_bytes=771391488 weight_to_token_ratio=0.166493 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.663302951e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62882 payload_rows_remote=56324 dup_rows=2654 dup_rows_remote=2644 dedup_ratio=0.040497 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 371, 3755, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 157, 407, 0, 0, 8, 0, 0, 4091, 5, 0, 3271, 0, 0, 13, 0, 0, 0, 4091, 0, 0, 0, 4096, 0, 0, 481, 4086, 0, 5, 0, 0, 0, 0, 0, 3831, 0] payload_rows_per_dst=[8192, 7722, 8192, 8190, 6558, 8192, 8174, 7662] dup_rows_per_dst=[0, 530, 0, 1128, 10, 16, 960, 10] combine_rows_per_src=[8192, 7722, 8192, 8190, 6558, 8192, 8174, 7662] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=394067968 dispatch_meta_bytes=226128 combine_payload_bytes=394067968 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=394294096 ingress_bytes=614268928 link_bytes=614268928 token_bytes=788362064 weight_movement_bytes=220200960 total_nvlink_bytes=1008563024 weight_to_token_ratio=0.279315 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=26624 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.068314696e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62886 payload_rows_remote=54728 dup_rows=2650 dup_rows_remote=2650 dedup_ratio=0.040436 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 4008, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 54, 396, 0, 0, 2, 0, 0, 4078, 0, 0, 3182, 0, 0, 1, 0, 0, 0, 4085, 0, 0, 0, 4096, 0, 0, 414, 4081, 0, 20, 0, 0, 0, 0, 0, 3772, 0] payload_rows_per_dst=[8192, 8090, 8158, 8192, 6368, 8172, 8166, 7548] dup_rows_per_dst=[0, 890, 0, 900, 0, 0, 824, 36] combine_rows_per_src=[8192, 8090, 8158, 8192, 6368, 8172, 8166, 7548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59828 payload_rows_remote=51664 dup_rows=5708 dup_rows_remote=4504 dedup_ratio=0.087097 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 666, 3623, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 130, 301, 0, 0, 0, 0, 0, 4086, 3, 0, 3227, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 4096, 0, 0, 605, 4079, 0, 25, 0, 0, 0, 0, 0, 3752, 0] payload_rows_per_dst=[8192, 7712, 8180, 4470, 7454, 8152, 8164, 7504] dup_rows_per_dst=[0, 866, 0, 418, 3170, 0, 1204, 50] combine_rows_per_src=[8192, 7712, 8180, 4470, 7454, 8152, 8164, 7504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62102 payload_rows_remote=53910 dup_rows=3434 dup_rows_remote=3412 dedup_ratio=0.052399 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 3940, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 73, 201, 0, 0, 5, 0, 0, 4087, 3, 0, 3502, 0, 0, 2, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 0, 428, 4091, 0, 11, 0, 0, 0, 0, 0, 3914, 0] payload_rows_per_dst=[8192, 8062, 8188, 8186, 6746, 8186, 6700, 7842] dup_rows_per_dst=[22, 1806, 278, 546, 4, 100, 670, 8] combine_rows_per_src=[8192, 8062, 8188, 8186, 6746, 8186, 6700, 7842] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=403730432 dispatch_meta_bytes=235872 combine_payload_bytes=403730432 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=403966304 ingress_bytes=734031872 link_bytes=734031872 token_bytes=807696736 weight_movement_bytes=330301440 total_nvlink_bytes=1137998176 weight_to_token_ratio=0.408942 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=26624 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.276602153e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 1, 0, 0, 0, 1, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=352622592 dispatch_meta_bytes=220520 combine_payload_bytes=352622592 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=352843112 ingress_bytes=462723072 link_bytes=462723072 token_bytes=705465704 weight_movement_bytes=110100480 total_nvlink_bytes=815566184 weight_to_token_ratio=0.156068 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=22528 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.047515271e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58328 payload_rows_remote=50142 dup_rows=7208 dup_rows_remote=7110 dedup_ratio=0.109985 router_tokens_per_expert=[1098, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4062, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 309, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3566, 80, 0, 0, 0, 0, 0, 0, 0, 0, 1101, 0, 4096, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 2154, 0] payload_rows_per_dst=[8192, 8048, 8192, 8186, 7132, 8186, 2490, 7902] dup_rows_per_dst=[2200, 0, 8, 610, 0, 98, 114, 4178] combine_rows_per_src=[8192, 8048, 8192, 8186, 7132, 8186, 2490, 7902] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=383315968 dispatch_meta_bytes=231184 combine_payload_bytes=383315968 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=383547152 ingress_bytes=713617408 link_bytes=713617408 token_bytes=766863120 weight_movement_bytes=330301440 total_nvlink_bytes=1097164560 weight_to_token_ratio=0.430718 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=23040 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.241098042e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7034830592 prefetch_bytes=1101004800 reduce_bytes=660602880 weight_movement_bytes=1761607680 egress_bytes=3518568192 ingress_bytes=5277870080 link_bytes=5277870080 total_nvlink_bytes=8796438272 weight_to_token_ratio=0.250412 prefetch_share_of_ingress=0.208608 dedup_ratio=0.139111 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=6 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.044118 gemm_inner_pad_upper_bound=8890 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.179084130e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=318502912 dispatch_meta_bytes=229328 combine_payload_bytes=318502912 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=318732240 ingress_bytes=538703872 link_bytes=538703872 token_bytes=637235152 weight_movement_bytes=220200960 total_nvlink_bytes=857436112 weight_to_token_ratio=0.345557 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.368946350e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=358772736 dispatch_meta_bytes=233296 combine_payload_bytes=358772736 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=359006032 ingress_bytes=689074176 link_bytes=689074176 token_bytes=717778768 weight_movement_bytes=330301440 total_nvlink_bytes=1048080208 weight_to_token_ratio=0.460172 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=20480 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.198413325e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=391215104 dispatch_meta_bytes=226816 combine_payload_bytes=391215104 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=391441920 ingress_bytes=611416064 link_bytes=611416064 token_bytes=782657024 weight_movement_bytes=220200960 total_nvlink_bytes=1002857984 weight_to_token_ratio=0.281351 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=30848 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.063353096e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=342773760 dispatch_meta_bytes=209976 combine_payload_bytes=342773760 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=342983736 ingress_bytes=452874240 link_bytes=452874240 token_bytes=685757496 weight_movement_bytes=110100480 total_nvlink_bytes=795857976 weight_to_token_ratio=0.160553 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=20864 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.876227884e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7141973224 prefetch_bytes=770703360 reduce_bytes=2202009600 weight_movement_bytes=2972712960 egress_bytes=3572122856 ingress_bytes=6542563328 link_bytes=6542563328 total_nvlink_bytes=10114686184 weight_to_token_ratio=0.416231 prefetch_share_of_ingress=0.117798 dedup_ratio=0.126654 prefetch_slots=14 unused_prefetch_slots=146 slot_utilization=0.087500 reduce_slots=20 remote_demand=7 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.044853 gemm_inner_pad_upper_bound=8382 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.137859369e-04 prefetch_bytes_per_gemm_flop=1.340379902e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=359417856 dispatch_meta_bytes=229008 combine_payload_bytes=359417856 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=359646864 ingress_bytes=469518336 link_bytes=469518336 token_bytes=719064720 weight_movement_bytes=110100480 total_nvlink_bytes=829165200 weight_to_token_ratio=0.153116 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.165696088e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6795724968 prefetch_bytes=1101004800 reduce_bytes=1321205760 weight_movement_bytes=2422210560 egress_bytes=3398981800 ingress_bytes=5818953728 link_bytes=5818953728 total_nvlink_bytes=9217935528 weight_to_token_ratio=0.356432 prefetch_share_of_ingress=0.189210 dedup_ratio=0.176382 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=12 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043750 gemm_inner_pad_upper_bound=8636 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.012011759e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7243304528 prefetch_bytes=990904320 reduce_bytes=1101004800 weight_movement_bytes=2091909120 egress_bytes=3622805072 ingress_bytes=5712408576 link_bytes=5712408576 total_nvlink_bytes=9335213648 weight_to_token_ratio=0.288806 prefetch_share_of_ingress=0.173465 dedup_ratio=0.125659 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=10 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.045221 gemm_inner_pad_upper_bound=8763 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.934818043e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6446355312 prefetch_bytes=880803840 reduce_bytes=1541406720 weight_movement_bytes=2422210560 egress_bytes=3224267632 ingress_bytes=5644298240 link_bytes=5644298240 total_nvlink_bytes=8868565872 weight_to_token_ratio=0.375749 prefetch_share_of_ingress=0.156052 dedup_ratio=0.195712 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=14 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.045221 gemm_inner_pad_upper_bound=8128 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.816362967e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61314 payload_rows_remote=53350 dup_rows=4222 dup_rows_remote=4222 dedup_ratio=0.064423 router_tokens_per_expert=[1189, 0, 0, 3, 0, 9, 0, 0, 0, 3, 0, 4087, 3979, 0, 0, 0, 0, 0, 1, 0, 5, 1, 2, 0, 4, 267, 4042, 1, 5, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 3500, 143, 1, 4, 0, 0, 0, 0, 0, 0, 577, 0, 4091, 0, 2, 3982, 5, 0, 0, 1, 0, 0, 5, 2768, 0] payload_rows_per_dst=[8188, 7964, 8192, 8190, 7000, 8190, 8044, 5546] dup_rows_per_dst=[2396, 0, 8, 448, 0, 280, 1088, 2] combine_rows_per_src=[8188, 7964, 8192, 8190, 7000, 8190, 8044, 5546] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6943412672 prefetch_bytes=990904320 reduce_bytes=220200960 weight_movement_bytes=1211105280 egress_bytes=3472839104 ingress_bytes=4681678848 link_bytes=4681678848 total_nvlink_bytes=8154517952 weight_to_token_ratio=0.174425 prefetch_share_of_ingress=0.211656 dedup_ratio=0.147324 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=2 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.044485 gemm_inner_pad_upper_bound=8636 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.142209521e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60946 payload_rows_remote=52754 dup_rows=4590 dup_rows_remote=2938 dedup_ratio=0.070038 router_tokens_per_expert=[822, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4079, 3992, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 0, 0, 94, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3895, 89, 0, 0, 0, 0, 0, 0, 0, 0, 613, 0, 4096, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 2827, 0] payload_rows_per_dst=[8192, 7988, 8192, 7822, 7790, 7134, 8174, 5654] dup_rows_per_dst=[1652, 1466, 18, 140, 0, 130, 1184, 0] combine_rows_per_src=[8192, 7988, 8192, 7822, 7790, 7134, 8174, 5654] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=382412800 dispatch_meta_bytes=230288 combine_payload_bytes=382412800 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=382643088 ingress_bytes=602613760 link_bytes=602613760 token_bytes=765055888 weight_movement_bytes=220200960 total_nvlink_bytes=985256848 weight_to_token_ratio=0.287823 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.048044442e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62882 payload_rows_remote=54710 dup_rows=2654 dup_rows_remote=2654 dedup_ratio=0.040497 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 724, 3798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 125, 289, 0, 0, 0, 0, 0, 4086, 0, 0, 3378, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 4096, 0, 0, 358, 4089, 0, 8, 0, 0, 0, 0, 0, 3641, 0] payload_rows_per_dst=[8192, 7946, 8172, 8184, 6756, 8172, 8178, 7282] dup_rows_per_dst=[0, 1098, 0, 824, 0, 0, 716, 16] combine_rows_per_src=[8192, 7946, 8172, 8184, 6756, 8172, 8178, 7282] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62390 payload_rows_remote=54752 dup_rows=3146 dup_rows_remote=2520 dedup_ratio=0.048004 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 1, 506, 3621, 0, 9, 3, 0, 0, 2, 0, 0, 0, 0, 4087, 0, 2, 241, 662, 1, 0, 6, 1, 3, 3998, 0, 0, 3397, 1, 0, 2, 5, 0, 0, 4059, 4, 0, 0, 4084, 0, 0, 342, 4069, 2, 26, 0, 0, 0, 2, 0, 3627, 0] payload_rows_per_dst=[8170, 7638, 8022, 8192, 6812, 8132, 8144, 7280] dup_rows_per_dst=[0, 626, 2, 1794, 2, 10, 682, 30] combine_rows_per_src=[8170, 7638, 8022, 8192, 6812, 8132, 8144, 7280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=392290304 dispatch_meta_bytes=229512 combine_payload_bytes=392290304 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=392519816 ingress_bytes=612491264 link_bytes=612491264 token_bytes=784810120 weight_movement_bytes=220200960 total_nvlink_bytes=1005011080 weight_to_token_ratio=0.280579 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.065223046e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53108 payload_rows_remote=45216 dup_rows=12428 dup_rows_remote=7722 dedup_ratio=0.189636 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 3777, 2, 11, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 212, 507, 0, 0, 16, 0, 0, 4071, 24, 0, 3293, 0, 0, 12, 0, 0, 0, 4086, 0, 0, 0, 4096, 0, 0, 251, 4088, 0, 7, 0, 0, 0, 0, 0, 3735, 0] payload_rows_per_dst=[8192, 4010, 8164, 5038, 8192, 8174, 3446, 7892] dup_rows_per_dst=[0, 464, 4, 470, 6644, 22, 118, 4706] combine_rows_per_src=[8192, 4010, 8164, 5038, 8192, 8174, 3446, 7892] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=370327552 dispatch_meta_bytes=224672 combine_payload_bytes=370327552 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=370552224 ingress_bytes=920829952 link_bytes=920829952 token_bytes=740879776 weight_movement_bytes=550502400 total_nvlink_bytes=1291382176 weight_to_token_ratio=0.743039 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=29312 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.601474737e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=386426880 dispatch_meta_bytes=229288 combine_payload_bytes=386426880 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=386656168 ingress_bytes=606627840 link_bytes=606627840 token_bytes=773083048 weight_movement_bytes=220200960 total_nvlink_bytes=993284008 weight_to_token_ratio=0.284835 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.055025587e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=392161280 dispatch_meta_bytes=229456 combine_payload_bytes=392161280 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=392390736 ingress_bytes=502261760 link_bytes=502261760 token_bytes=784552016 weight_movement_bytes=110100480 total_nvlink_bytes=894652496 weight_to_token_ratio=0.140335 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.735158085e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61546 payload_rows_remote=53354 dup_rows=3990 dup_rows_remote=2986 dedup_ratio=0.060883 router_tokens_per_expert=[2500, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 4030, 3865, 0, 0, 0, 0, 0, 0, 0, 0, 20, 8, 0, 0, 514, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1342, 62, 0, 0, 0, 0, 0, 88, 0, 0, 1, 0, 4096, 0, 0, 3897, 0, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[5136, 7730, 8192, 8192, 8192, 8116, 7796, 8192] dup_rows_per_dst=[2, 0, 56, 1004, 2684, 244, 0, 0] combine_rows_per_src=[5136, 7730, 8192, 8192, 8192, 8116, 7796, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=392462336 dispatch_meta_bytes=229088 combine_payload_bytes=392462336 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=392691424 ingress_bytes=612663296 link_bytes=612663296 token_bytes=785153760 weight_movement_bytes=220200960 total_nvlink_bytes=1005354720 weight_to_token_ratio=0.280456 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=28288 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.065522238e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=324108288 dispatch_meta_bytes=211752 combine_payload_bytes=324108288 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=324320040 ingress_bytes=434208768 link_bytes=434208768 token_bytes=648428328 weight_movement_bytes=110100480 total_nvlink_bytes=758528808 weight_to_token_ratio=0.169796 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=30208 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.551604626e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62214 payload_rows_remote=54022 dup_rows=3322 dup_rows_remote=822 dedup_ratio=0.050690 router_tokens_per_expert=[2855, 0, 0, 0, 0, 118, 0, 0, 0, 0, 0, 4037, 3857, 0, 0, 0, 0, 0, 0, 0, 0, 37, 2, 0, 0, 292, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1250, 85, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 4096, 0, 0, 3853, 0, 0, 0, 0, 0, 0, 0, 4095, 0] payload_rows_per_dst=[5882, 7714, 8192, 8192, 8192, 8146, 7706, 8190] dup_rows_per_dst=[64, 0, 78, 560, 2500, 120, 0, 0] combine_rows_per_src=[5882, 7714, 8192, 8192, 8192, 8146, 7706, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=382441472 dispatch_meta_bytes=225360 combine_payload_bytes=382441472 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=382666832 ingress_bytes=712742912 link_bytes=712742912 token_bytes=765108304 weight_movement_bytes=330301440 total_nvlink_bytes=1095409744 weight_to_token_ratio=0.431705 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=29440 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.239577150e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60480 payload_rows_remote=52288 dup_rows=5056 dup_rows_remote=4840 dedup_ratio=0.077148 router_tokens_per_expert=[3305, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 4036, 3818, 0, 0, 0, 0, 0, 0, 0, 0, 92, 16, 0, 0, 319, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 934, 59, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 4096, 0, 0, 3779, 0, 0, 0, 0, 0, 0, 0, 4095, 0] payload_rows_per_dst=[6836, 7636, 8192, 8190, 5746, 8132, 7558, 8190] dup_rows_per_dst=[1456, 0, 216, 1912, 1382, 90, 0, 0] combine_rows_per_src=[6836, 7636, 8192, 8190, 5746, 8132, 7558, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61264 payload_rows_remote=53434 dup_rows=4272 dup_rows_remote=4272 dedup_ratio=0.065186 router_tokens_per_expert=[2315, 0, 0, 0, 0, 113, 0, 0, 0, 0, 0, 4057, 3825, 0, 0, 0, 0, 0, 0, 0, 0, 22, 3, 0, 0, 572, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1472, 81, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 4096, 0, 0, 3915, 0, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[4846, 7650, 8192, 8188, 8192, 8174, 7830, 8192] dup_rows_per_dst=[10, 0, 50, 1124, 2944, 144, 0, 0] combine_rows_per_src=[4846, 7650, 8192, 8188, 8192, 8174, 7830, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55734 payload_rows_remote=47542 dup_rows=9802 dup_rows_remote=4282 dedup_ratio=0.149567 router_tokens_per_expert=[2684, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 4049, 3827, 0, 0, 0, 0, 0, 0, 0, 0, 17, 15, 0, 0, 138, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1529, 65, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 4096, 0, 0, 3936, 0, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 7862, 8192, 4704, 3058, 7662, 7872, 8192] dup_rows_per_dst=[5520, 3940, 64, 126, 0, 152, 0, 0] combine_rows_per_src=[8192, 7862, 8192, 4704, 3058, 7662, 7872, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=387229696 dispatch_meta_bytes=219376 combine_payload_bytes=387229696 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=387449072 ingress_bytes=717531136 link_bytes=717531136 token_bytes=774678768 weight_movement_bytes=330301440 total_nvlink_bytes=1104980208 weight_to_token_ratio=0.426372 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=23168 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.247904659e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58154 payload_rows_remote=50528 dup_rows=7382 dup_rows_remote=7382 dedup_ratio=0.112640 router_tokens_per_expert=[3044, 0, 0, 2, 0, 156, 0, 0, 0, 2, 0, 4047, 3811, 0, 0, 0, 0, 0, 1, 0, 5, 29, 12, 0, 5, 376, 4067, 0, 5, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 1010, 77, 1, 4, 0, 1, 1, 27, 0, 0, 0, 0, 4091, 0, 2, 3798, 5, 0, 0, 2, 0, 0, 5, 4091, 0] payload_rows_per_dst=[8186, 7626, 8192, 8192, 2020, 8140, 7606, 8192] dup_rows_per_dst=[6400, 0, 84, 714, 0, 176, 4, 4] combine_rows_per_src=[8186, 7626, 8192, 8192, 2020, 8140, 7606, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7110574064 prefetch_bytes=990904320 reduce_bytes=1211105280 weight_movement_bytes=2202009600 egress_bytes=3556421616 ingress_bytes=5756162048 link_bytes=5756162048 total_nvlink_bytes=9312583664 weight_to_token_ratio=0.309681 prefetch_share_of_ingress=0.172147 dedup_ratio=0.128531 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=11 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.046691 gemm_inner_pad_upper_bound=8509 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.001091253e-04 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=378140672 dispatch_meta_bytes=222768 combine_payload_bytes=378140672 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=378363440 ingress_bytes=488241152 link_bytes=488241152 token_bytes=756504112 weight_movement_bytes=110100480 total_nvlink_bytes=866604592 weight_to_token_ratio=0.145539 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=24704 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.491316652e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6768940552 prefetch_bytes=1321205760 reduce_bytes=660602880 weight_movement_bytes=1981808640 egress_bytes=3385587208 ingress_bytes=5365161984 link_bytes=5365161984 total_nvlink_bytes=8750749192 weight_to_token_ratio=0.292780 prefetch_share_of_ingress=0.246256 dedup_ratio=0.163837 prefetch_slots=24 unused_prefetch_slots=136 slot_utilization=0.150000 reduce_slots=6 remote_demand=12 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043382 gemm_inner_pad_upper_bound=8763 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.330899108e-05 prefetch_bytes_per_gemm_flop=2.297794118e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6371467080 prefetch_bytes=1211105280 reduce_bytes=660602880 weight_movement_bytes=1871708160 egress_bytes=3186839368 ingress_bytes=5056335872 link_bytes=5056335872 total_nvlink_bytes=8243175240 weight_to_token_ratio=0.293764 prefetch_share_of_ingress=0.239522 dedup_ratio=0.202255 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=6 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.044853 gemm_inner_pad_upper_bound=8255 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.793799706e-05 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none - [2026-08-26 15:11:35] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 368.5 | throughput per GPU (TFLOP/s/GPU): 335.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.468465E+00 | loss scale: 1.0 | grad norm: 4.146 | number of skipped iterations: 0 | number of nan iterations: 0 | -[after training is done] datetime: 2026-08-26 15:11:35 -[rank0]:[W826 15:11:37.144435875 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank2]:[W826 15:11:37.801431803 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank1]:[W826 15:11:37.801669478 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank3]:[W826 15:11:37.995492411 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank7]:[W826 15:11:38.262636435 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank4]:[W826 15:11:38.474730581 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank6]:[W826 15:11:38.494563640 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank5]:[W826 15:11:38.539917551 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -W0826 15:11:39.070000 73803 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers -W0826 15:11:39.070000 73803 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 73896 closing signal SIGTERM -W0826 15:11:39.071000 73803 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 73897 closing signal SIGTERM -W0826 15:11:39.072000 73803 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 73898 closing signal SIGTERM -W0826 15:11:39.072000 73803 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 73899 closing signal SIGTERM -W0826 15:11:39.074000 73803 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 73900 closing signal SIGTERM -W0826 15:11:39.074000 73803 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 73901 closing signal SIGTERM -W0826 15:11:39.075000 73803 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 73902 closing signal SIGTERM -W0826 15:11:39.076000 73803 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 73903 closing signal SIGTERM +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62570 payload_rows_remote=54446 dup_rows=2966 dup_rows_remote=2884 dedup_ratio=0.045258 router_tokens_per_expert=[2954, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 4023, 3879, 0, 0, 0, 0, 0, 0, 0, 0, 23, 7, 0, 0, 283, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1107, 61, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 4096, 0, 0, 3952, 0, 0, 0, 0, 0, 0, 0, 4095, 0] payload_rows_per_dst=[6022, 7758, 8192, 8188, 8192, 8124, 7904, 8190] dup_rows_per_dst=[46, 0, 60, 564, 2214, 82, 0, 0] combine_rows_per_src=[6022, 7758, 8192, 8188, 8192, 8124, 7904, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=374800384 dispatch_meta_bytes=228512 combine_payload_bytes=374800384 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=375028896 ingress_bytes=484900864 link_bytes=484900864 token_bytes=749829280 weight_movement_bytes=110100480 total_nvlink_bytes=859929760 weight_to_token_ratio=0.146834 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.433223550e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59242 payload_rows_remote=51050 dup_rows=6294 dup_rows_remote=6292 dedup_ratio=0.096039 router_tokens_per_expert=[3438, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 3934, 3810, 0, 0, 0, 0, 0, 0, 0, 0, 66, 5, 0, 0, 293, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1015, 78, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4096, 0, 0, 3747, 0, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[6920, 7620, 7446, 5340, 8190, 7944, 7590, 8192] dup_rows_per_dst=[18, 0, 138, 346, 2030, 90, 3670, 2] combine_rows_per_src=[6920, 7620, 7446, 5340, 8190, 7944, 7590, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7242026112 prefetch_bytes=990904320 reduce_bytes=1761607680 weight_movement_bytes=2752512000 egress_bytes=3622157440 ingress_bytes=6372380672 link_bytes=6372380672 total_nvlink_bytes=9994538112 weight_to_token_ratio=0.380075 prefetch_share_of_ingress=0.155500 dedup_ratio=0.119830 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=16 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.044118 gemm_inner_pad_upper_bound=8382 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.108261807e-04 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=383014912 dispatch_meta_bytes=230824 combine_payload_bytes=383014912 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=383245736 ingress_bytes=603215872 link_bytes=603215872 token_bytes=766260648 weight_movement_bytes=220200960 total_nvlink_bytes=986461608 weight_to_token_ratio=0.287371 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=30976 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.049091613e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=340781056 dispatch_meta_bytes=207296 combine_payload_bytes=340781056 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=340988352 ingress_bytes=450881536 link_bytes=450881536 token_bytes=681769408 weight_movement_bytes=110100480 total_nvlink_bytes=791869888 weight_to_token_ratio=0.161492 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=23168 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.841571484e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=362184704 dispatch_meta_bytes=231640 combine_payload_bytes=362184704 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=362416344 ingress_bytes=582385664 link_bytes=582385664 token_bytes=724601048 weight_movement_bytes=220200960 total_nvlink_bytes=944802008 weight_to_token_ratio=0.303893 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=30720 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.012864456e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7024812944 prefetch_bytes=1101004800 reduce_bytes=1211105280 weight_movement_bytes=2312110080 egress_bytes=3513539472 ingress_bytes=5823383552 link_bytes=5823383552 total_nvlink_bytes=9336923024 weight_to_token_ratio=0.329135 prefetch_share_of_ingress=0.189066 dedup_ratio=0.137759 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=11 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.044118 gemm_inner_pad_upper_bound=8509 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.012782178e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=390268928 dispatch_meta_bytes=229320 combine_payload_bytes=390268928 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=390498248 ingress_bytes=500369408 link_bytes=500369408 token_bytes=780767176 weight_movement_bytes=110100480 total_nvlink_bytes=890867656 weight_to_token_ratio=0.141016 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.702246971e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=365926400 dispatch_meta_bytes=229368 combine_payload_bytes=365926400 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=366155768 ingress_bytes=476026880 link_bytes=476026880 token_bytes=732082168 weight_movement_bytes=110100480 total_nvlink_bytes=842182648 weight_to_token_ratio=0.150394 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.278890373e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6920396320 prefetch_bytes=1101004800 reduce_bytes=1211105280 weight_movement_bytes=2312110080 egress_bytes=3461334560 ingress_bytes=5771171840 link_bytes=5771171840 total_nvlink_bytes=9232506400 weight_to_token_ratio=0.334101 prefetch_share_of_ingress=0.190777 dedup_ratio=0.154721 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=11 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.044853 gemm_inner_pad_upper_bound=8763 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.003701702e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6720209576 prefetch_bytes=1101004800 reduce_bytes=880803840 weight_movement_bytes=1981808640 egress_bytes=3361198760 ingress_bytes=5340819456 link_bytes=5340819456 total_nvlink_bytes=8702018216 weight_to_token_ratio=0.294903 prefetch_share_of_ingress=0.206149 dedup_ratio=0.163824 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=8 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.045956 gemm_inner_pad_upper_bound=8128 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.288563448e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7025936808 prefetch_bytes=1321205760 reduce_bytes=440401920 weight_movement_bytes=1761607680 egress_bytes=3514104232 ingress_bytes=5273440256 link_bytes=5273440256 total_nvlink_bytes=8787544488 weight_to_token_ratio=0.250729 prefetch_share_of_ingress=0.250540 dedup_ratio=0.140344 prefetch_slots=24 unused_prefetch_slots=136 slot_utilization=0.150000 reduce_slots=4 remote_demand=12 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.044485 gemm_inner_pad_upper_bound=8382 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.171379937e-05 prefetch_bytes_per_gemm_flop=2.297794118e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7117437688 prefetch_bytes=880803840 reduce_bytes=550502400 weight_movement_bytes=1431306240 egress_bytes=3559858936 ingress_bytes=4988884992 link_bytes=4988884992 total_nvlink_bytes=8548743928 weight_to_token_ratio=0.201099 prefetch_share_of_ingress=0.176553 dedup_ratio=0.127753 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=5 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.044485 gemm_inner_pad_upper_bound=8636 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.676491532e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6027115336 prefetch_bytes=880803840 reduce_bytes=1101004800 weight_movement_bytes=1981808640 egress_bytes=3014619976 ingress_bytes=4994304000 link_bytes=4994304000 total_nvlink_bytes=8008923976 weight_to_token_ratio=0.328815 prefetch_share_of_ingress=0.176362 dedup_ratio=0.239758 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=10 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.045588 gemm_inner_pad_upper_bound=8382 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.685916078e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none + [2026-08-26 15:59:22] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 362.0 | throughput per GPU (TFLOP/s/GPU): 341.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.467156E+00 | loss scale: 1.0 | grad norm: 4.143 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-26 15:59:22 +[rank0]:[W826 15:59:23.408941377 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W826 15:59:24.774845987 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W826 15:59:24.775767535 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W826 15:59:24.776909486 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W826 15:59:24.777610146 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W826 15:59:24.994825562 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W826 15:59:24.037332017 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W826 15:59:24.041894535 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 15:59:27.986000 163203 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 15:59:27.987000 163203 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 163296 closing signal SIGTERM +W0826 15:59:27.988000 163203 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 163297 closing signal SIGTERM +W0826 15:59:27.988000 163203 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 163298 closing signal SIGTERM +W0826 15:59:27.989000 163203 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 163299 closing signal SIGTERM +W0826 15:59:27.990000 163203 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 163300 closing signal SIGTERM +W0826 15:59:27.991000 163203 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 163301 closing signal SIGTERM +W0826 15:59:27.992000 163203 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 163302 closing signal SIGTERM +W0826 15:59:27.993000 163203 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 163303 closing signal SIGTERM Traceback (most recent call last): File "/usr/local/bin/torchrun", line 8, in sys.exit(main()) @@ -3553,4 +3537,4 @@ Traceback (most recent call last): time.sleep(monitor_interval) File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 73803 got signal: 15 +torch.distributed.elastic.multiprocessing.api.SignalException: Process 163203 got signal: 15 diff --git a/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1/mem_samples.csv b/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1/mem_samples.csv index 598e5f6798adb62647a4f1056d20f258074e47d0..90a60ede32ce90438033e9bd55ef9d8a40e39df8 100644 --- a/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1/mem_samples.csv +++ b/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1/mem_samples.csv @@ -14,7 +14,7 @@ 5, 0 6, 0 7, 0 -0, 1 +0, 4 1, 1 2, 1 3, 1 @@ -37,55 +37,63 @@ 4, 4 5, 4 6, 4 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 7, 4 -0, 725 -1, 725 -2, 725 -3, 725 -4, 725 -5, 725 -6, 725 -7, 725 -0, 953 -1, 945 -2, 959 -3, 943 -4, 953 -5, 955 -6, 923 -7, 945 -0, 1803 -1, 1803 -2, 1803 -3, 1803 -4, 1803 -5, 1803 -6, 1803 -7, 1803 -0, 6793 -1, 6799 -2, 6801 -3, 6819 -4, 6785 -5, 6795 -6, 6803 -7, 6801 -0, 58463 -1, 57881 -2, 57909 -3, 58565 -4, 58555 -5, 58609 -6, 58607 -7, 58573 -0, 59479 -1, 58911 -2, 58911 -3, 59611 -4, 59611 -5, 59611 -6, 59611 -7, 59611 +0, 821 +1, 821 +2, 819 +3, 819 +4, 819 +5, 819 +6, 819 +7, 819 +0, 1537 +1, 1537 +2, 1537 +3, 1537 +4, 1537 +5, 1537 +6, 1537 +7, 1537 +0, 1863 +1, 1863 +2, 1863 +3, 1863 +4, 1863 +5, 1863 +6, 1863 +7, 1863 +0, 6853 +1, 6845 +2, 6825 +3, 6863 +4, 6851 +5, 6833 +6, 6851 +7, 6861 +0, 58437 +1, 57883 +2, 57887 +3, 58601 +4, 58617 +5, 58581 +6, 58561 +7, 58609 +0, 59091 +1, 58523 +2, 58523 +3, 59223 +4, 59223 +5, 59223 +6, 59223 +7, 59223 0, 59719 1, 59151 2, 59151 @@ -94,99 +102,99 @@ 5, 59851 6, 59851 7, 59851 -0, 60419 +0, 60421 1, 59851 2, 59851 3, 60551 4, 60551 -5, 60551 -6, 60551 -7, 60551 -0, 86691 -1, 86427 -2, 86883 -3, 85141 -4, 86917 -5, 86117 -6, 86711 -7, 89399 -0, 117915 -1, 117957 -2, 117943 -3, 117989 -4, 117977 -5, 117985 -6, 117987 -7, 117993 -0, 118939 -1, 118983 -2, 118967 -3, 119001 -4, 119001 -5, 119001 -6, 119001 -7, 119001 +5, 59853 +6, 59853 +7, 59853 +0, 90701 +1, 84935 +2, 84469 +3, 86709 +4, 89677 +5, 83943 +6, 83721 +7, 83533 +0, 118049 +1, 118105 +2, 118067 +3, 118129 +4, 118115 +5, 118097 +6, 118085 +7, 118081 +0, 118885 +1, 118985 +2, 118943 +3, 118925 +4, 118969 +5, 118995 +6, 118985 +7, 118965 0, 118939 -1, 118983 +1, 118985 2, 118967 3, 119001 4, 119001 5, 119001 6, 119001 7, 119001 -0, 119537 -1, 119599 -2, 119569 -3, 119615 -4, 119607 -5, 119617 -6, 119609 -7, 119607 -0, 128707 -1, 128371 -2, 128547 -3, 128579 -4, 128525 -5, 128523 -6, 128521 -7, 128469 -0, 141321 -1, 141551 -2, 141525 -3, 141351 -4, 141351 -5, 141351 -6, 141351 -7, 141351 -0, 141321 -1, 141551 -2, 141525 -3, 141351 -4, 141351 -5, 141351 -6, 141351 -7, 141351 -0, 140675 -1, 140905 -2, 140879 -3, 140705 -4, 140705 -5, 140705 -6, 140705 -7, 140705 -0, 139401 -1, 139305 -2, 138959 -3, 138785 -4, 138785 -5, 138785 -6, 138785 -7, 138849 +0, 119601 +1, 119645 +2, 119629 +3, 119663 +4, 119663 +5, 119663 +6, 119663 +7, 119663 +0, 128615 +1, 128305 +2, 128451 +3, 128485 +4, 128481 +5, 128495 +6, 128473 +7, 128447 +0, 141327 +1, 141555 +2, 141531 +3, 141357 +4, 141357 +5, 141357 +6, 141357 +7, 141357 +0, 141327 +1, 141555 +2, 141531 +3, 141357 +4, 141357 +5, 141357 +6, 141357 +7, 141357 +0, 140681 +1, 140909 +2, 140885 +3, 140711 +4, 140711 +5, 140711 +6, 140711 +7, 140711 +0, 13756 +1, 139633 +2, 139735 +3, 139823 +4, 139435 +5, 139561 +6, 139435 +7, 139371 0, 12601 1, 12604 -2, 12732 -3, 138783 -4, 13244 -5, 12604 -6, 12604 -7, 12604 +2, 12604 +3, 12604 +4, 12604 +5, 12796 +6, 139435 +7, 13564 diff --git a/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1/run.env b/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1/run.env index 66b08e2f77f3f5247f015262451f1caecf9e0532..5b83a6d3938f0dd55ac5db725790b703ca846d9c 100644 --- a/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1/run.env +++ b/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1/run.env @@ -28,7 +28,7 @@ router_balancing=aux_loss aux_loss_coeff=1e-2 router_pre_softmax=1 router_fusion=0 -router_dtype= +router_dtype=fp32 device_initiated_grouped_gemm=1 norm_epsilon=1e-6 seq_length=4096 diff --git a/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1/train.log b/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1/train.log index eb7e759406ef035130b5405ca22542f4bc79defb..956368458a96c32ac006c62c460cdeb1d11576ba 100644 --- a/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1/train.log +++ b/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1/train.log @@ -1,5 +1,5 @@ -[15:11:45] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) -[15:11:45] WARN: moonep_backend_token_routing_analysis lacks --moe-debug-drop-logging; probe will have no pre/post rank loads +[15:59:33] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +[15:59:33] WARN: moonep_backend_token_routing_analysis lacks --moe-debug-drop-logging; probe will have no pre/post rank loads ============================================================== QWEN2_57B / moonep (slots/rank=8 policy=matched) E=64 K=8 H'=2560 m=8 @@ -8,10 +8,10 @@ memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1/memory_snapshots/snapshot.pickle repo=/workspace/Megatron-MoE-Echo @ 35ae6e7f ============================================================== -W0826 15:11:46.591000 76686 torch/distributed/run.py:766] -W0826 15:11:46.591000 76686 torch/distributed/run.py:766] ***************************************** -W0826 15:11:46.591000 76686 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -W0826 15:11:46.591000 76686 torch/distributed/run.py:766] ***************************************** +W0826 15:59:34.519000 165720 torch/distributed/run.py:766] +W0826 15:59:34.519000 165720 torch/distributed/run.py:766] ***************************************** +W0826 15:59:34.519000 165720 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 15:59:34.519000 165720 torch/distributed/run.py:766] ***************************************** /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it @@ -19,6 +19,8 @@ WARNING: one_logger package is required to enable e2e metrics tracking. please g warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer Number of virtual stages per pipeline stage: None @@ -337,7 +339,7 @@ using torch.bfloat16 for parameters ... moe_permute_fusion .............................. False moe_received_token_capacity ..................... None moe_router_bias_update_rate ..................... 0.001 - moe_router_dtype ................................ None + moe_router_dtype ................................ fp32 moe_router_enable_expert_bias ................... False moe_router_force_load_balancing ................. True moe_router_group_topk ........................... None @@ -569,6 +571,8 @@ using torch.bfloat16 for parameters ... -------------------- end of arguments --------------------- INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 > building GPT2BPETokenizer tokenizer ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) @@ -576,26 +580,22 @@ WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode > initializing torch distributed ... /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 67 is connected to +7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank [Gloo] Rank 013 is connected to is connected to is connected to 777 peer ranks. peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : [Gloo] Rank 7772 - - +[Gloo] Rank 5 is connected to 7 peer ranks. [Gloo] Rank Expected number of connected peer ranks is : 74 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank 67 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 - +[Gloo] Rank [Gloo] Rank 7 is connected to 67 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 > initialized tensor model parallel with size 1 > initialized pipeline model parallel with size 1 @@ -603,45 +603,29 @@ WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 - +[Gloo] Rank 0 is connected to 0[Gloo] Rank peer ranks. Expected number of connected peer ranks is : 00 is connected to +0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[rank2]:[W826 15:12:05.316303597 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank1]:[W826 15:12:05.316469674 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank7]:[W826 15:12:05.318137300 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank6]:[W826 15:12:05.318207730 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank4]:[W826 15:12:05.318214020 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank5]:[W826 15:12:05.318236344 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank3]:[W826 15:12:05.318502940 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 15:59:53.281753240 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 15:59:53.281817196 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 15:59:53.281849996 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 15:59:53.281861957 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 15:59:53.281918431 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 15:59:53.283886635 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 15:59:53.283895769 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. > compiling dataset index builder ... make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' make: Nothing to be done for 'default'. make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' ->>> done with dataset index builder. Compilation time: 0.034 seconds +>>> done with dataset index builder. Compilation time: 0.037 seconds WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. > compiling and loading fused kernels ... -[rank0]:[W826 15:12:05.495476701 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. ->>> done with compiling and loading fused kernels. Compilation time: 3.843 seconds -time to initialize megatron (seconds): 7.808 -[after megatron is initialized] datetime: 2026-08-26 15:12:11 +[rank0]:[W826 15:59:53.455727868 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.952 seconds +time to initialize megatron (seconds): 7.893 +[after megatron is initialized] datetime: 2026-08-26 15:59:59 building GPT model ... -/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1440: UserWarning: Using a large number of experts (e.g. >=32) without fp32 routing. Consider enabling moe_router_dtype for better numerical stability. - warnings.warn( -/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1440: UserWarning: Using a large number of experts (e.g. >=32) without fp32 routing. Consider enabling moe_router_dtype for better numerical stability. - warnings.warn( -/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1440: UserWarning: Using a large number of experts (e.g. >=32) without fp32 routing. Consider enabling moe_router_dtype for better numerical stability. - warnings.warn( -/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1440: UserWarning: Using a large number of experts (e.g. >=32) without fp32 routing. Consider enabling moe_router_dtype for better numerical stability. - warnings.warn( -/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1440: UserWarning: Using a large number of experts (e.g. >=32) without fp32 routing. Consider enabling moe_router_dtype for better numerical stability. - warnings.warn( -/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1440: UserWarning: Using a large number of experts (e.g. >=32) without fp32 routing. Consider enabling moe_router_dtype for better numerical stability. - warnings.warn( -/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1440: UserWarning: Using a large number of experts (e.g. >=32) without fp32 routing. Consider enabling moe_router_dtype for better numerical stability. - warnings.warn( -/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1440: UserWarning: Using a large number of experts (e.g. >=32) without fp32 routing. Consider enabling moe_router_dtype for better numerical stability. - warnings.warn( /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. warnings.warn( /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. @@ -670,80 +654,80 @@ Params for bucket 3 (146800640 elements, 146800640 padded size): module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight Params for bucket 4 (103001600 elements, 103001600 padded size): module.decoder.layers.9.mlp.router.weight - module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.9.self_attention.linear_qkv.bias + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_qkv.weight module.decoder.layers.9.pre_mlp_layernorm.weight module.decoder.layers.9.self_attention.linear_proj.weight module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.9.self_attention.linear_qkv.weight Params for bucket 5 (146800640 elements, 146800640 padded size): module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight Params for bucket 6 (103001600 elements, 103001600 padded size): + module.decoder.layers.8.mlp.router.weight module.decoder.layers.8.pre_mlp_layernorm.weight module.decoder.layers.8.self_attention.linear_qkv.bias + module.decoder.layers.8.self_attention.linear_qkv.weight module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.8.self_attention.linear_proj.weight module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.8.mlp.router.weight - module.decoder.layers.8.self_attention.linear_qkv.weight Params for bucket 7 (146800640 elements, 146800640 padded size): module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight Params for bucket 8 (103001600 elements, 103001600 padded size): - module.decoder.layers.7.self_attention.linear_qkv.bias + module.decoder.layers.7.pre_mlp_layernorm.weight module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.7.mlp.router.weight module.decoder.layers.7.self_attention.linear_qkv.weight - module.decoder.layers.7.pre_mlp_layernorm.weight module.decoder.layers.7.self_attention.linear_proj.weight module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_qkv.bias Params for bucket 9 (146800640 elements, 146800640 padded size): module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight Params for bucket 10 (103001600 elements, 103001600 padded size): module.decoder.layers.6.mlp.router.weight module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.bias module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.6.self_attention.linear_qkv.weight - module.decoder.layers.6.self_attention.linear_qkv.bias module.decoder.layers.6.self_attention.linear_proj.weight module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight Params for bucket 11 (146800640 elements, 146800640 padded size): module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight Params for bucket 12 (103001600 elements, 103001600 padded size): module.decoder.layers.5.mlp.router.weight - module.decoder.layers.5.pre_mlp_layernorm.weight - module.decoder.layers.5.self_attention.linear_qkv.weight - module.decoder.layers.5.self_attention.linear_proj.weight - module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight module.decoder.layers.5.self_attention.linear_qkv.bias module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.weight Params for bucket 13 (146800640 elements, 146800640 padded size): module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight Params for bucket 14 (103001600 elements, 103001600 padded size): - module.decoder.layers.4.pre_mlp_layernorm.weight module.decoder.layers.4.self_attention.linear_qkv.weight module.decoder.layers.4.self_attention.linear_proj.weight module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight module.decoder.layers.4.mlp.router.weight module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.pre_mlp_layernorm.weight module.decoder.layers.4.self_attention.linear_qkv.bias Params for bucket 15 (146800640 elements, 146800640 padded size): module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight Params for bucket 16 (103001600 elements, 103001600 padded size): - module.decoder.layers.3.pre_mlp_layernorm.weight - module.decoder.layers.3.self_attention.linear_qkv.weight - module.decoder.layers.3.mlp.router.weight module.decoder.layers.3.self_attention.linear_qkv.bias + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.pre_mlp_layernorm.weight module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.3.self_attention.linear_proj.weight module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight Params for bucket 17 (146800640 elements, 146800640 padded size): module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight Params for bucket 18 (103001600 elements, 103001600 padded size): - module.decoder.layers.2.self_attention.linear_qkv.weight - module.decoder.layers.2.pre_mlp_layernorm.weight module.decoder.layers.2.mlp.router.weight module.decoder.layers.2.self_attention.linear_qkv.bias module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.weight module.decoder.layers.2.self_attention.linear_proj.weight module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight Params for bucket 19 (146800640 elements, 146800640 padded size): @@ -760,28 +744,28 @@ Params for bucket 21 (146800640 elements, 146800640 padded size): module.decoder.layers.0.mlp.shared_experts.linear_fc1.weight Params for bucket 22 (209890816 elements, 209890816 padded size): module.decoder.layers.0.mlp.router.weight - module.decoder.layers.0.self_attention.linear_qkv.bias - module.decoder.layers.0.self_attention.linear_proj.weight - module.decoder.layers.0.self_attention.linear_qkv.weight module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.self_attention.linear_qkv.bias module.embedding.word_embeddings.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.linear_proj.weight INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 44 Params for bucket 1 (45875200 elements, 45875200 padded size): - module.decoder.layers.9.mlp.experts.linear_fc2.weight6 module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 module.decoder.layers.9.mlp.experts.linear_fc2.weight7 module.decoder.layers.9.mlp.experts.linear_fc2.weight3 module.decoder.layers.9.mlp.experts.linear_fc2.weight5 Params for bucket 2 (45875200 elements, 45875200 padded size): module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 module.decoder.layers.9.mlp.experts.linear_fc2.weight0 module.decoder.layers.9.mlp.experts.linear_fc2.weight1 - module.decoder.layers.9.mlp.experts.linear_fc1.weight7 Params for bucket 3 (55050240 elements, 55050240 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight6 - module.decoder.layers.9.mlp.experts.linear_fc1.weight4 module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 Params for bucket 4 (55050240 elements, 55050240 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight3 module.decoder.layers.9.mlp.experts.linear_fc1.weight1 @@ -792,133 +776,133 @@ Params for bucket 5 (45875200 elements, 45875200 padded size): module.decoder.layers.8.mlp.experts.linear_fc2.weight5 module.decoder.layers.8.mlp.experts.linear_fc2.weight6 Params for bucket 6 (45875200 elements, 45875200 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 module.decoder.layers.8.mlp.experts.linear_fc2.weight4 module.decoder.layers.8.mlp.experts.linear_fc2.weight1 - module.decoder.layers.8.mlp.experts.linear_fc2.weight0 module.decoder.layers.8.mlp.experts.linear_fc2.weight2 - module.decoder.layers.8.mlp.experts.linear_fc2.weight3 Params for bucket 7 (55050240 elements, 55050240 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight7 module.decoder.layers.8.mlp.experts.linear_fc1.weight5 module.decoder.layers.8.mlp.experts.linear_fc1.weight6 Params for bucket 8 (55050240 elements, 55050240 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight4 - module.decoder.layers.8.mlp.experts.linear_fc1.weight3 module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 Params for bucket 9 (45875200 elements, 45875200 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight1 module.decoder.layers.7.mlp.experts.linear_fc2.weight7 module.decoder.layers.8.mlp.experts.linear_fc1.weight0 Params for bucket 10 (45875200 elements, 45875200 padded size): - module.decoder.layers.7.mlp.experts.linear_fc2.weight6 - module.decoder.layers.7.mlp.experts.linear_fc2.weight4 - module.decoder.layers.7.mlp.experts.linear_fc2.weight2 module.decoder.layers.7.mlp.experts.linear_fc2.weight5 module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 Params for bucket 11 (55050240 elements, 55050240 padded size): module.decoder.layers.7.mlp.experts.linear_fc2.weight1 - module.decoder.layers.7.mlp.experts.linear_fc2.weight0 module.decoder.layers.7.mlp.experts.linear_fc1.weight6 module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 Params for bucket 12 (55050240 elements, 55050240 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight5 module.decoder.layers.7.mlp.experts.linear_fc1.weight3 module.decoder.layers.7.mlp.experts.linear_fc1.weight4 Params for bucket 13 (55050240 elements, 55050240 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight2 - module.decoder.layers.7.mlp.experts.linear_fc1.weight0 module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 Params for bucket 14 (45875200 elements, 45875200 padded size): - module.decoder.layers.6.mlp.experts.linear_fc2.weight4 module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 module.decoder.layers.6.mlp.experts.linear_fc2.weight7 module.decoder.layers.6.mlp.experts.linear_fc2.weight3 module.decoder.layers.6.mlp.experts.linear_fc2.weight5 Params for bucket 15 (45875200 elements, 45875200 padded size): module.decoder.layers.6.mlp.experts.linear_fc2.weight2 - module.decoder.layers.6.mlp.experts.linear_fc2.weight0 module.decoder.layers.6.mlp.experts.linear_fc2.weight1 module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 Params for bucket 16 (55050240 elements, 55050240 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight6 - module.decoder.layers.6.mlp.experts.linear_fc1.weight5 module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 Params for bucket 17 (55050240 elements, 55050240 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight3 - module.decoder.layers.6.mlp.experts.linear_fc1.weight2 module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 Params for bucket 18 (45875200 elements, 45875200 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 module.decoder.layers.5.mlp.experts.linear_fc2.weight6 module.decoder.layers.5.mlp.experts.linear_fc2.weight7 - module.decoder.layers.5.mlp.experts.linear_fc2.weight5 Params for bucket 19 (45875200 elements, 45875200 padded size): - module.decoder.layers.5.mlp.experts.linear_fc2.weight4 - module.decoder.layers.5.mlp.experts.linear_fc2.weight1 - module.decoder.layers.5.mlp.experts.linear_fc2.weight2 module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 Params for bucket 20 (55050240 elements, 55050240 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight7 - module.decoder.layers.5.mlp.experts.linear_fc1.weight5 module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 Params for bucket 21 (55050240 elements, 55050240 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight4 - module.decoder.layers.5.mlp.experts.linear_fc1.weight2 module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 Params for bucket 22 (45875200 elements, 45875200 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight1 - module.decoder.layers.5.mlp.experts.linear_fc1.weight0 module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 Params for bucket 23 (45875200 elements, 45875200 padded size): module.decoder.layers.4.mlp.experts.linear_fc2.weight6 - module.decoder.layers.4.mlp.experts.linear_fc2.weight4 module.decoder.layers.4.mlp.experts.linear_fc2.weight2 - module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 Params for bucket 24 (55050240 elements, 55050240 padded size): module.decoder.layers.4.mlp.experts.linear_fc2.weight1 module.decoder.layers.4.mlp.experts.linear_fc2.weight0 - module.decoder.layers.4.mlp.experts.linear_fc1.weight7 module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 Params for bucket 25 (55050240 elements, 55050240 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight5 module.decoder.layers.4.mlp.experts.linear_fc1.weight4 module.decoder.layers.4.mlp.experts.linear_fc1.weight3 Params for bucket 26 (55050240 elements, 55050240 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight2 - module.decoder.layers.4.mlp.experts.linear_fc1.weight1 module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 Params for bucket 27 (45875200 elements, 45875200 padded size): - module.decoder.layers.3.mlp.experts.linear_fc2.weight6 module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 module.decoder.layers.3.mlp.experts.linear_fc2.weight7 module.decoder.layers.3.mlp.experts.linear_fc2.weight5 module.decoder.layers.3.mlp.experts.linear_fc2.weight3 Params for bucket 28 (45875200 elements, 45875200 padded size): module.decoder.layers.3.mlp.experts.linear_fc2.weight2 - module.decoder.layers.3.mlp.experts.linear_fc1.weight7 module.decoder.layers.3.mlp.experts.linear_fc2.weight0 module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 Params for bucket 29 (55050240 elements, 55050240 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight6 - module.decoder.layers.3.mlp.experts.linear_fc1.weight4 module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 Params for bucket 30 (55050240 elements, 55050240 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight3 module.decoder.layers.3.mlp.experts.linear_fc1.weight2 module.decoder.layers.3.mlp.experts.linear_fc1.weight1 Params for bucket 31 (45875200 elements, 45875200 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight0 - module.decoder.layers.2.mlp.experts.linear_fc2.weight5 module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 module.decoder.layers.2.mlp.experts.linear_fc2.weight6 Params for bucket 32 (45875200 elements, 45875200 padded size): module.decoder.layers.2.mlp.experts.linear_fc2.weight3 module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 module.decoder.layers.2.mlp.experts.linear_fc2.weight1 module.decoder.layers.2.mlp.experts.linear_fc2.weight2 - module.decoder.layers.2.mlp.experts.linear_fc2.weight0 Params for bucket 33 (55050240 elements, 55050240 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight7 module.decoder.layers.2.mlp.experts.linear_fc1.weight5 @@ -929,8 +913,8 @@ Params for bucket 34 (55050240 elements, 55050240 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight3 Params for bucket 35 (45875200 elements, 45875200 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight1 - module.decoder.layers.1.mlp.experts.linear_fc2.weight7 module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 Params for bucket 36 (45875200 elements, 45875200 padded size): module.decoder.layers.1.mlp.experts.linear_fc2.weight3 module.decoder.layers.1.mlp.experts.linear_fc2.weight5 @@ -939,13 +923,13 @@ Params for bucket 36 (45875200 elements, 45875200 padded size): module.decoder.layers.1.mlp.experts.linear_fc2.weight2 Params for bucket 37 (55050240 elements, 55050240 padded size): module.decoder.layers.1.mlp.experts.linear_fc2.weight1 - module.decoder.layers.1.mlp.experts.linear_fc2.weight0 - module.decoder.layers.1.mlp.experts.linear_fc1.weight6 module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 Params for bucket 38 (55050240 elements, 55050240 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight5 - module.decoder.layers.1.mlp.experts.linear_fc1.weight4 module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 Params for bucket 39 (55050240 elements, 55050240 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight2 module.decoder.layers.1.mlp.experts.linear_fc1.weight1 @@ -954,8 +938,8 @@ Params for bucket 40 (45875200 elements, 45875200 padded size): module.decoder.layers.0.mlp.experts.linear_fc2.weight7 module.decoder.layers.0.mlp.experts.linear_fc2.weight5 module.decoder.layers.0.mlp.experts.linear_fc2.weight3 - module.decoder.layers.0.mlp.experts.linear_fc2.weight6 module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 Params for bucket 41 (45875200 elements, 45875200 padded size): module.decoder.layers.0.mlp.experts.linear_fc2.weight2 module.decoder.layers.0.mlp.experts.linear_fc1.weight7 @@ -963,17 +947,17 @@ Params for bucket 41 (45875200 elements, 45875200 padded size): module.decoder.layers.0.mlp.experts.linear_fc2.weight0 Params for bucket 42 (55050240 elements, 55050240 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight6 - module.decoder.layers.0.mlp.experts.linear_fc1.weight4 module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 Params for bucket 43 (55050240 elements, 55050240 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight3 - module.decoder.layers.0.mlp.experts.linear_fc1.weight2 module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 Params for bucket 44 (18350080 elements, 18350080 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight0 -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine -[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 15:12:14 +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 16:00:02 > building train, validation, and test datasets ... > datasets target sizes (minimum size): train: 96 @@ -981,7 +965,7 @@ INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine test: 0 INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] > building train, validation, and test datasets for GPT ... -INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers @@ -994,14 +978,14 @@ INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 466b63c71b42 INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-shuffle_index.npy INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 > finished creating GPT datasets ... -[after dataloaders are built] datetime: 2026-08-26 15:12:14 +[after dataloaders are built] datetime: 2026-08-26 16:00:02 done with setup ... -(min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (3147.67, 3154.64) - train/valid/test-data-iterators-setup ..........: (61.65, 70.58) training ... Overwriting rerun_state_machine.current_iteration from -1 to 0... -[before the start of training step] datetime: 2026-08-26 15:12:14 +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (2585.52, 2595.86) + train/valid/test-data-iterators-setup ..........: (57.04, 65.64) +[before the start of training step] datetime: 2026-08-26 16:00:02 INFO:moonep.api:MoonEP comm buffer sizes: config : R=8, S=4096, H=3584, E=64, K=8, NvS=34800, NvS_padded=34816 sm_count : num_sms=32, num_sms_dedup=148 @@ -1010,2522 +994,2522 @@ INFO:moonep.api:MoonEP comm buffer sizes: local_temp_buffer_size : 1.26 MiB total_size : 241.26 MiB WARNING:megatron.core.rerun_state_machine:Result validation enabled - [2026-08-26 15:12:36] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 22148.9 | throughput per GPU (TFLOP/s/GPU): 5.6 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.097710E+01 | loss scale: 1.0 | grad norm: 29.747 | number of skipped iterations: 0 | number of nan iterations: 0 | -Number of parameters in transformer block in billions: 20.11 + [2026-08-26 16:00:25] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 23045.2 | throughput per GPU (TFLOP/s/GPU): 5.4 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.097734E+01 | loss scale: 1.0 | grad norm: 29.718 | number of skipped iterations: 0 | number of nan iterations: 0 |Number of parameters in transformer block in billions: 20.11 + Number of parameters in embedding layers in billions: 0.36 Total number of parameters in billions: 20.47 Number of parameters in most loaded shard in billions: 20.4724 Theoretical memory footprints: weight and optimizer=146430.14 MB [Rank 0] (after 1 iterations) memory (MB) | allocated: 104985.1162109375 | max allocated: 104985.11669921875 | reserved: 108632.0 | max reserved: 108632.0 - [2026-08-26 15:12:37] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 407.5 | throughput per GPU (TFLOP/s/GPU): 303.1 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.097741E+01 | loss scale: 1.0 | grad norm: 30.467 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44568 payload_rows_remote=39040 dup_rows=20968 dup_rows_remote=18332 dedup_ratio=0.319946 router_tokens_per_expert=[496, 482, 509, 505, 507, 492, 492, 527, 494, 483, 516, 528, 518, 519, 519, 519, 504, 567, 529, 522, 529, 484, 532, 507, 477, 520, 517, 521, 492, 518, 523, 539, 511, 512, 526, 497, 506, 522, 514, 490, 508, 523, 530, 522, 545, 512, 526, 510, 530, 485, 526, 512, 500, 505, 503, 485, 508, 480, 494, 521, 510, 526, 540, 502] payload_rows_per_dst=[5516, 5556, 5556, 5640, 5560, 5348, 5864, 5528] dup_rows_per_dst=[2504, 2534, 2602, 2676, 2596, 2406, 3014, 2636] combine_rows_per_src=[5516, 5556, 5556, 5640, 5560, 5348, 5864, 5528] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44446 payload_rows_remote=38838 dup_rows=21090 dup_rows_remote=18486 dedup_ratio=0.321808 router_tokens_per_expert=[507, 543, 509, 540, 514, 554, 515, 514, 489, 497, 518, 510, 529, 510, 506, 497, 517, 481, 506, 484, 497, 513, 522, 524, 485, 487, 504, 477, 510, 514, 530, 524, 509, 530, 551, 518, 540, 524, 548, 538, 519, 495, 490, 486, 501, 536, 545, 534, 512, 519, 493, 534, 495, 474, 479, 495, 523, 544, 497, 502, 492, 524, 475, 519] payload_rows_per_dst=[5614, 5468, 5494, 5570, 5704, 5608, 5504, 5484] dup_rows_per_dst=[2778, 2644, 2594, 2492, 2812, 2604, 2498, 2668] combine_rows_per_src=[5614, 5468, 5494, 5570, 5704, 5608, 5504, 5484] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44600 payload_rows_remote=38966 dup_rows=20936 dup_rows_remote=18234 dedup_ratio=0.319458 router_tokens_per_expert=[498, 481, 495, 496, 545, 514, 512, 467, 511, 514, 518, 543, 456, 553, 527, 514, 524, 511, 509, 549, 494, 523, 546, 512, 497, 546, 476, 518, 528, 540, 509, 530, 516, 518, 522, 476, 490, 521, 523, 496, 535, 489, 502, 522, 534, 501, 514, 549, 504, 505, 545, 490, 504, 488, 514, 479, 490, 510, 507, 478, 544, 525, 514, 507] payload_rows_per_dst=[5468, 5548, 5634, 5602, 5658, 5648, 5480, 5562] dup_rows_per_dst=[2548, 2724, 2702, 2686, 2466, 2644, 2578, 2588] combine_rows_per_src=[5468, 5548, 5634, 5602, 5658, 5648, 5480, 5562] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44572 payload_rows_remote=38988 dup_rows=20964 dup_rows_remote=18382 dedup_ratio=0.319885 router_tokens_per_expert=[462, 525, 499, 556, 531, 479, 509, 489, 496, 522, 511, 493, 554, 499, 489, 519, 516, 500, 528, 505, 539, 468, 480, 520, 510, 513, 485, 535, 508, 508, 522, 593, 494, 505, 490, 486, 549, 510, 532, 500, 540, 514, 470, 559, 517, 540, 487, 517, 461, 494, 475, 526, 517, 518, 558, 525, 571, 490, 480, 495, 522, 515, 523, 525] payload_rows_per_dst=[5512, 5584, 5510, 5618, 5542, 5642, 5570, 5594] dup_rows_per_dst=[2588, 2582, 2602, 2730, 2590, 2646, 2578, 2648] combine_rows_per_src=[5512, 5584, 5510, 5618, 5542, 5642, 5570, 5594] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44540 payload_rows_remote=39116 dup_rows=20996 dup_rows_remote=18418 dedup_ratio=0.320374 router_tokens_per_expert=[490, 498, 511, 518, 502, 507, 512, 518, 525, 525, 464, 509, 500, 493, 512, 501, 503, 506, 486, 518, 493, 527, 563, 508, 486, 480, 469, 574, 502, 468, 521, 501, 479, 509, 502, 493, 554, 508, 522, 524, 498, 508, 489, 502, 547, 541, 542, 503, 532, 527, 515, 499, 554, 532, 495, 507, 463, 548, 497, 525, 561, 529, 545, 528] payload_rows_per_dst=[5530, 5544, 5592, 5424, 5554, 5568, 5670, 5658] dup_rows_per_dst=[2582, 2514, 2616, 2578, 2628, 2692, 2652, 2734] combine_rows_per_src=[5530, 5544, 5592, 5424, 5554, 5568, 5670, 5658] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44302 payload_rows_remote=38810 dup_rows=21234 dup_rows_remote=18582 dedup_ratio=0.324005 router_tokens_per_expert=[486, 514, 499, 517, 539, 500, 560, 526, 507, 522, 504, 533, 510, 524, 541, 507, 512, 501, 496, 545, 529, 531, 499, 503, 513, 536, 500, 517, 519, 523, 519, 470, 475, 564, 500, 494, 492, 498, 497, 552, 490, 529, 495, 519, 529, 511, 526, 489, 499, 483, 529, 505, 475, 495, 519, 498, 517, 510, 552, 506, 480, 521, 497, 520] payload_rows_per_dst=[5620, 5502, 5540, 5562, 5492, 5532, 5446, 5608] dup_rows_per_dst=[2662, 2794, 2692, 2632, 2652, 2644, 2560, 2598] combine_rows_per_src=[5620, 5502, 5540, 5562, 5492, 5532, 5446, 5608] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44512 payload_rows_remote=38942 dup_rows=21024 dup_rows_remote=18478 dedup_ratio=0.320801 router_tokens_per_expert=[511, 520, 507, 546, 510, 491, 528, 515, 550, 515, 508, 518, 504, 489, 491, 524, 505, 514, 493, 494, 544, 518, 526, 492, 519, 520, 516, 494, 486, 512, 512, 493, 515, 530, 559, 500, 485, 510, 499, 493, 478, 545, 524, 506, 526, 519, 532, 504, 513, 525, 518, 511, 487, 521, 505, 478, 538, 511, 533, 535, 512, 477, 515, 499] payload_rows_per_dst=[5588, 5580, 5522, 5498, 5538, 5584, 5570, 5632] dup_rows_per_dst=[2668, 2618, 2650, 2606, 2644, 2684, 2546, 2608] combine_rows_per_src=[5588, 5580, 5522, 5498, 5538, 5584, 5570, 5632] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279838720 dispatch_meta_bytes=229488 combine_payload_bytes=279838720 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280068208 ingress_bytes=610140160 link_bytes=610140160 token_bytes=559906928 weight_movement_bytes=330301440 total_nvlink_bytes=890208368 weight_to_token_ratio=0.589922 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061134089e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44488 payload_rows_remote=38822 dup_rows=21048 dup_rows_remote=18356 dedup_ratio=0.321167 router_tokens_per_expert=[525, 456, 542, 532, 534, 556, 487, 544, 542, 493, 539, 512, 522, 508, 550, 500, 494, 527, 532, 506, 508, 498, 540, 513, 521, 505, 503, 522, 511, 550, 512, 487, 485, 477, 515, 512, 518, 500, 480, 501, 536, 490, 505, 503, 521, 531, 500, 557, 487, 483, 537, 485, 491, 509, 460, 551, 471, 500, 483, 507, 513, 519, 534, 536] payload_rows_per_dst=[5666, 5608, 5540, 5570, 5540, 5588, 5496, 5480] dup_rows_per_dst=[2692, 2736, 2690, 2652, 2560, 2698, 2510, 2510] combine_rows_per_src=[5666, 5608, 5540, 5570, 5540, 5588, 5496, 5480] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278390784 dispatch_meta_bytes=229296 combine_payload_bytes=278390784 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278620080 ingress_bytes=498591744 link_bytes=498591744 token_bytes=557010864 weight_movement_bytes=220200960 total_nvlink_bytes=777211824 weight_to_token_ratio=0.395326 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.671330471e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279308288 dispatch_meta_bytes=228800 combine_payload_bytes=279308288 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279537088 ingress_bytes=609609728 link_bytes=609609728 token_bytes=558845376 weight_movement_bytes=330301440 total_nvlink_bytes=889146816 weight_to_token_ratio=0.591043 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060211581e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 + [2026-08-26 16:00:25] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 380.7 | throughput per GPU (TFLOP/s/GPU): 324.4 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.097746E+01 | loss scale: 1.0 | grad norm: 30.473 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44560 payload_rows_remote=38988 dup_rows=20976 dup_rows_remote=18408 dedup_ratio=0.320068 router_tokens_per_expert=[460, 523, 495, 549, 527, 473, 502, 488, 493, 518, 510, 491, 551, 499, 489, 519, 510, 501, 526, 505, 539, 470, 479, 521, 509, 514, 484, 535, 511, 507, 522, 591, 496, 502, 493, 487, 550, 510, 531, 501, 540, 514, 472, 562, 520, 543, 487, 517, 461, 494, 475, 526, 519, 519, 558, 528, 572, 496, 481, 500, 523, 519, 530, 531] payload_rows_per_dst=[5480, 5572, 5498, 5620, 5546, 5652, 5574, 5618] dup_rows_per_dst=[2554, 2568, 2604, 2726, 2594, 2658, 2586, 2686] combine_rows_per_src=[5480, 5572, 5498, 5620, 5546, 5652, 5574, 5618] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44588 payload_rows_remote=38964 dup_rows=20948 dup_rows_remote=18254 dedup_ratio=0.319641 router_tokens_per_expert=[498, 478, 496, 493, 542, 512, 510, 464, 509, 513, 517, 542, 454, 553, 527, 512, 522, 507, 508, 549, 493, 522, 545, 513, 498, 544, 473, 517, 527, 539, 509, 528, 517, 519, 520, 477, 488, 523, 522, 496, 536, 494, 501, 523, 535, 501, 517, 553, 506, 507, 544, 492, 504, 490, 514, 478, 491, 513, 513, 479, 549, 528, 517, 507] payload_rows_per_dst=[5448, 5540, 5624, 5598, 5656, 5662, 5480, 5580] dup_rows_per_dst=[2538, 2714, 2694, 2672, 2468, 2658, 2590, 2614] combine_rows_per_src=[5448, 5540, 5624, 5598, 5656, 5662, 5480, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44450 payload_rows_remote=38842 dup_rows=21086 dup_rows_remote=18480 dedup_ratio=0.321747 router_tokens_per_expert=[503, 544, 506, 539, 511, 552, 514, 511, 487, 496, 517, 508, 525, 510, 505, 496, 516, 482, 506, 482, 498, 509, 523, 523, 485, 490, 505, 476, 509, 511, 531, 523, 508, 531, 550, 518, 541, 523, 549, 539, 518, 496, 490, 484, 502, 536, 546, 535, 513, 523, 494, 537, 498, 477, 481, 497, 525, 544, 498, 504, 494, 525, 480, 519] payload_rows_per_dst=[5598, 5454, 5492, 5574, 5706, 5608, 5522, 5496] dup_rows_per_dst=[2762, 2634, 2586, 2486, 2812, 2606, 2518, 2682] combine_rows_per_src=[5598, 5454, 5492, 5574, 5706, 5608, 5522, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44298 payload_rows_remote=38800 dup_rows=21238 dup_rows_remote=18596 dedup_ratio=0.324066 router_tokens_per_expert=[483, 512, 499, 513, 538, 496, 559, 520, 502, 521, 501, 531, 508, 522, 537, 508, 511, 502, 498, 541, 529, 528, 499, 502, 512, 536, 501, 515, 524, 521, 517, 469, 476, 562, 498, 496, 494, 497, 497, 550, 492, 527, 493, 522, 528, 514, 526, 491, 503, 483, 531, 507, 476, 496, 524, 501, 523, 515, 557, 511, 484, 521, 498, 520] payload_rows_per_dst=[5602, 5486, 5532, 5558, 5498, 5538, 5458, 5626] dup_rows_per_dst=[2638, 2774, 2688, 2632, 2642, 2648, 2584, 2632] combine_rows_per_src=[5602, 5486, 5532, 5558, 5498, 5538, 5458, 5626] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44444 payload_rows_remote=38700 dup_rows=21092 dup_rows_remote=18192 dedup_ratio=0.321838 router_tokens_per_expert=[524, 456, 538, 530, 533, 553, 486, 544, 540, 490, 540, 509, 519, 506, 548, 498, 492, 528, 528, 503, 508, 496, 539, 511, 517, 504, 505, 522, 513, 552, 511, 488, 486, 477, 516, 512, 518, 501, 482, 502, 535, 488, 505, 503, 521, 533, 502, 557, 491, 481, 540, 488, 494, 509, 463, 553, 471, 502, 486, 509, 515, 523, 538, 536] payload_rows_per_dst=[5744, 5634, 5538, 5626, 5490, 5538, 5506, 5368] dup_rows_per_dst=[2900, 2800, 2674, 2756, 2498, 2630, 2532, 2302] combine_rows_per_src=[5744, 5634, 5538, 5626, 5490, 5538, 5506, 5368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44496 payload_rows_remote=38914 dup_rows=21040 dup_rows_remote=18484 dedup_ratio=0.321045 router_tokens_per_expert=[510, 516, 504, 545, 510, 487, 524, 513, 546, 510, 506, 512, 503, 490, 491, 521, 501, 513, 487, 493, 543, 519, 524, 491, 519, 520, 516, 493, 484, 510, 513, 494, 517, 531, 560, 504, 486, 510, 503, 493, 476, 545, 523, 509, 528, 519, 531, 505, 515, 527, 520, 513, 485, 523, 506, 480, 543, 513, 539, 539, 514, 484, 517, 502] payload_rows_per_dst=[5574, 5558, 5508, 5498, 5538, 5584, 5582, 5654] dup_rows_per_dst=[2644, 2600, 2634, 2600, 2670, 2688, 2556, 2648] combine_rows_per_src=[5574, 5558, 5508, 5498, 5538, 5584, 5582, 5654] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44520 payload_rows_remote=39116 dup_rows=21016 dup_rows_remote=18448 dedup_ratio=0.320679 router_tokens_per_expert=[487, 493, 509, 515, 497, 507, 511, 517, 522, 524, 462, 509, 498, 492, 512, 500, 503, 504, 485, 518, 493, 525, 561, 506, 481, 477, 467, 574, 502, 465, 519, 501, 481, 508, 504, 493, 557, 509, 525, 526, 498, 512, 488, 503, 549, 542, 544, 506, 533, 531, 515, 499, 553, 534, 496, 514, 463, 553, 498, 527, 563, 531, 547, 530] payload_rows_per_dst=[5506, 5538, 5586, 5404, 5568, 5570, 5678, 5670] dup_rows_per_dst=[2566, 2500, 2604, 2568, 2638, 2714, 2672, 2754] combine_rows_per_src=[5506, 5538, 5586, 5404, 5568, 5570, 5678, 5670] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44592 payload_rows_remote=39054 dup_rows=20944 dup_rows_remote=18276 dedup_ratio=0.319580 router_tokens_per_expert=[493, 476, 508, 502, 505, 493, 489, 525, 491, 482, 513, 527, 514, 519, 517, 519, 503, 564, 525, 520, 529, 483, 530, 505, 476, 518, 517, 521, 493, 519, 525, 540, 514, 510, 526, 499, 505, 523, 515, 492, 510, 525, 530, 521, 547, 513, 525, 513, 531, 482, 527, 513, 502, 509, 505, 487, 511, 484, 497, 524, 513, 527, 541, 506] payload_rows_per_dst=[5496, 5592, 5616, 5608, 5580, 5376, 5786, 5538] dup_rows_per_dst=[2486, 2572, 2658, 2610, 2632, 2406, 2912, 2668] combine_rows_per_src=[5496, 5592, 5616, 5608, 5580, 5376, 5786, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279465984 dispatch_meta_bytes=229584 combine_payload_bytes=279465984 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279695568 ingress_bytes=389566464 link_bytes=389566464 token_bytes=559161552 weight_movement_bytes=110100480 total_nvlink_bytes=669262032 weight_to_token_ratio=0.196903 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.775201536e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279465984 dispatch_meta_bytes=229480 combine_payload_bytes=279465984 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279695464 ingress_bytes=499666944 link_bytes=499666944 token_bytes=559161448 weight_movement_bytes=220200960 total_nvlink_bytes=779362408 weight_to_token_ratio=0.393806 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.690029967e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280383488 dispatch_meta_bytes=230136 combine_payload_bytes=280383488 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280613624 ingress_bytes=390483968 link_bytes=390483968 token_bytes=560997112 weight_movement_bytes=110100480 total_nvlink_bytes=671097592 weight_to_token_ratio=0.196259 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.791158439e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278190080 dispatch_meta_bytes=229568 combine_payload_bytes=278190080 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278419648 ingress_bytes=388290560 link_bytes=388290560 token_bytes=556609728 weight_movement_bytes=110100480 total_nvlink_bytes=666710208 weight_to_token_ratio=0.197806 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.753011467e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279136256 dispatch_meta_bytes=229680 combine_payload_bytes=279136256 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279365936 ingress_bytes=389236736 link_bytes=389236736 token_bytes=558502192 weight_movement_bytes=110100480 total_nvlink_bytes=668602672 weight_to_token_ratio=0.197135 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.769467023e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278276096 dispatch_meta_bytes=228712 combine_payload_bytes=278276096 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278504808 ingress_bytes=388376576 link_bytes=388376576 token_bytes=556780904 weight_movement_bytes=110100480 total_nvlink_bytes=666881384 weight_to_token_ratio=0.197745 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.754507426e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44510 payload_rows_remote=38796 dup_rows=21026 dup_rows_remote=18242 dedup_ratio=0.320831 router_tokens_per_expert=[517, 541, 521, 499, 514, 528, 523, 500, 520, 485, 517, 519, 525, 521, 520, 518, 509, 475, 538, 522, 484, 512, 495, 464, 516, 479, 481, 514, 495, 533, 502, 506, 513, 496, 561, 538, 556, 527, 504, 512, 521, 513, 508, 508, 548, 486, 499, 482, 515, 544, 511, 513, 536, 489, 492, 483, 480, 507, 567, 532, 519, 522, 480, 513] payload_rows_per_dst=[5354, 5630, 5628, 5362, 5788, 5410, 5624, 5714] dup_rows_per_dst=[2396, 2620, 2746, 2312, 2972, 2494, 2702, 2784] combine_rows_per_src=[5354, 5630, 5628, 5362, 5788, 5410, 5624, 5714] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44442 payload_rows_remote=38848 dup_rows=21094 dup_rows_remote=18512 dedup_ratio=0.321869 router_tokens_per_expert=[526, 538, 565, 514, 502, 476, 479, 485, 564, 508, 512, 510, 535, 520, 532, 516, 524, 554, 517, 507, 522, 502, 501, 469, 529, 480, 503, 519, 511, 542, 540, 530, 525, 500, 442, 543, 513, 535, 499, 500, 503, 512, 555, 528, 494, 532, 467, 497, 483, 509, 519, 500, 551, 528, 473, 494, 505, 548, 470, 513, 513, 496, 486, 503] payload_rows_per_dst=[5646, 5630, 5520, 5558, 5586, 5594, 5468, 5440] dup_rows_per_dst=[2524, 2764, 2672, 2750, 2528, 2582, 2646, 2628] combine_rows_per_src=[5646, 5630, 5520, 5558, 5586, 5594, 5468, 5440] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44578 payload_rows_remote=38982 dup_rows=20958 dup_rows_remote=18402 dedup_ratio=0.319794 router_tokens_per_expert=[519, 518, 490, 484, 521, 541, 541, 494, 516, 521, 507, 515, 543, 512, 494, 501, 509, 514, 514, 505, 528, 502, 515, 489, 506, 528, 516, 511, 497, 492, 528, 495, 505, 497, 517, 495, 517, 521, 520, 516, 524, 499, 552, 533, 498, 494, 526, 556, 539, 514, 517, 490, 485, 487, 483, 500, 508, 516, 539, 521, 501, 505, 526, 501] payload_rows_per_dst=[5596, 5518, 5596, 5556, 5524, 5638, 5506, 5644] dup_rows_per_dst=[2620, 2700, 2556, 2590, 2652, 2726, 2524, 2590] combine_rows_per_src=[5596, 5518, 5596, 5556, 5524, 5638, 5506, 5644] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44482 payload_rows_remote=38982 dup_rows=21054 dup_rows_remote=18422 dedup_ratio=0.321259 router_tokens_per_expert=[515, 546, 532, 527, 498, 501, 531, 493, 518, 471, 502, 512, 528, 528, 509, 498, 508, 508, 526, 471, 523, 547, 558, 484, 526, 534, 511, 523, 493, 543, 512, 521, 530, 491, 493, 547, 543, 491, 521, 511, 535, 485, 521, 519, 501, 526, 545, 514, 496, 498, 521, 475, 499, 523, 520, 508, 501, 502, 490, 461, 502, 488, 492, 522] payload_rows_per_dst=[5600, 5500, 5554, 5578, 5562, 5678, 5550, 5460] dup_rows_per_dst=[2686, 2632, 2696, 2748, 2692, 2614, 2530, 2456] combine_rows_per_src=[5600, 5500, 5554, 5578, 5562, 5678, 5550, 5460] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44274 payload_rows_remote=38716 dup_rows=21262 dup_rows_remote=18596 dedup_ratio=0.324432 router_tokens_per_expert=[487, 503, 514, 554, 535, 522, 503, 484, 523, 517, 508, 515, 482, 486, 505, 507, 541, 515, 506, 540, 511, 498, 511, 454, 504, 545, 534, 503, 493, 502, 518, 513, 517, 502, 527, 499, 488, 523, 513, 494, 519, 548, 500, 512, 541, 498, 513, 502, 500, 501, 516, 473, 548, 501, 534, 492, 514, 511, 536, 508, 535, 519, 533, 518] payload_rows_per_dst=[5560, 5458, 5558, 5558, 5444, 5580, 5506, 5610] dup_rows_per_dst=[2644, 2628, 2594, 2666, 2682, 2686, 2624, 2738] combine_rows_per_src=[5560, 5458, 5558, 5558, 5444, 5580, 5506, 5610] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44522 payload_rows_remote=39148 dup_rows=21014 dup_rows_remote=18514 dedup_ratio=0.320648 router_tokens_per_expert=[530, 534, 530, 569, 516, 490, 483, 515, 553, 508, 497, 482, 543, 490, 497, 496, 540, 477, 492, 512, 504, 473, 522, 492, 481, 531, 515, 540, 547, 518, 539, 516, 487, 438, 496, 473, 510, 518, 492, 523, 510, 517, 547, 542, 540, 490, 508, 497, 513, 526, 485, 480, 528, 527, 530, 520, 552, 505, 508, 488, 484, 519, 525, 558] payload_rows_per_dst=[5632, 5516, 5520, 5630, 5374, 5622, 5608, 5620] dup_rows_per_dst=[2702, 2616, 2504, 2744, 2500, 2680, 2610, 2658] combine_rows_per_src=[5632, 5516, 5520, 5630, 5374, 5622, 5608, 5620] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278089728 dispatch_meta_bytes=228152 combine_payload_bytes=278089728 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278317880 ingress_bytes=388190208 link_bytes=388190208 token_bytes=556407608 weight_movement_bytes=110100480 total_nvlink_bytes=666508088 weight_to_token_ratio=0.197877 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.751266180e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44408 payload_rows_remote=38866 dup_rows=21128 dup_rows_remote=18420 dedup_ratio=0.322388 router_tokens_per_expert=[471, 498, 549, 505, 506, 512, 534, 536, 501, 509, 529, 488, 525, 527, 547, 486, 514, 514, 525, 518, 493, 528, 499, 511, 561, 528, 523, 523, 515, 503, 509, 526, 505, 479, 544, 507, 521, 518, 451, 517, 504, 498, 495, 480, 515, 477, 510, 528, 538, 517, 557, 527, 524, 498, 469, 495, 494, 537, 485, 542, 507, 471, 508, 537] payload_rows_per_dst=[5544, 5600, 5498, 5658, 5534, 5532, 5542, 5500] dup_rows_per_dst=[2678, 2624, 2706, 2718, 2550, 2482, 2708, 2662] combine_rows_per_src=[5544, 5600, 5498, 5658, 5534, 5532, 5542, 5500] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44560 payload_rows_remote=38830 dup_rows=20976 dup_rows_remote=18352 dedup_ratio=0.320068 router_tokens_per_expert=[519, 500, 496, 516, 555, 479, 515, 573, 514, 509, 506, 480, 478, 535, 528, 490, 498, 490, 511, 525, 538, 510, 500, 522, 500, 516, 516, 506, 505, 494, 525, 526, 514, 546, 498, 492, 520, 531, 490, 483, 525, 505, 522, 507, 507, 517, 537, 489, 512, 510, 537, 555, 456, 548, 559, 517, 513, 493, 538, 482, 462, 524, 477, 527] payload_rows_per_dst=[5730, 5544, 5592, 5484, 5570, 5594, 5604, 5442] dup_rows_per_dst=[2624, 2556, 2596, 2624, 2578, 2624, 2784, 2590] combine_rows_per_src=[5730, 5544, 5592, 5484, 5570, 5594, 5604, 5442] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279422976 dispatch_meta_bytes=229536 combine_payload_bytes=279422976 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279652512 ingress_bytes=389523456 link_bytes=389523456 token_bytes=559075488 weight_movement_bytes=110100480 total_nvlink_bytes=669175968 weight_to_token_ratio=0.196933 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.774453556e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278462464 dispatch_meta_bytes=229440 combine_payload_bytes=278462464 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278691904 ingress_bytes=498663424 link_bytes=498663424 token_bytes=557154368 weight_movement_bytes=220200960 total_nvlink_bytes=777355328 weight_to_token_ratio=0.395224 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.672577104e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279422976 dispatch_meta_bytes=229616 combine_payload_bytes=279422976 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279652592 ingress_bytes=389523456 link_bytes=389523456 token_bytes=559075568 weight_movement_bytes=110100480 total_nvlink_bytes=669176048 weight_to_token_ratio=0.196933 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.774453556e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277516288 dispatch_meta_bytes=229248 combine_payload_bytes=277516288 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=277745536 ingress_bytes=717918208 link_bytes=717918208 token_bytes=555261824 weight_movement_bytes=440401920 total_nvlink_bytes=995663744 weight_to_token_ratio=0.793143 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.248577841e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=280612864 dispatch_meta_bytes=230648 combine_payload_bytes=280612864 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280843512 ingress_bytes=390713344 link_bytes=390713344 token_bytes=561456376 weight_movement_bytes=110100480 total_nvlink_bytes=671556856 weight_to_token_ratio=0.196098 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.795147665e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278591488 dispatch_meta_bytes=229144 combine_payload_bytes=278591488 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278820632 ingress_bytes=388691968 link_bytes=388691968 token_bytes=557412120 weight_movement_bytes=110100480 total_nvlink_bytes=667512600 weight_to_token_ratio=0.197521 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.759992612e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278333440 dispatch_meta_bytes=228728 combine_payload_bytes=278333440 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278562168 ingress_bytes=608634880 link_bytes=608634880 token_bytes=556895608 weight_movement_bytes=330301440 total_nvlink_bytes=887197048 weight_to_token_ratio=0.593112 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058516160e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44360 payload_rows_remote=38572 dup_rows=21176 dup_rows_remote=18200 dedup_ratio=0.323120 router_tokens_per_expert=[479, 517, 470, 537, 495, 527, 512, 484, 497, 569, 545, 529, 457, 525, 560, 495, 551, 520, 524, 541, 520, 493, 542, 508, 488, 555, 529, 498, 481, 558, 494, 491, 522, 493, 489, 512, 485, 495, 513, 523, 531, 462, 505, 503, 490, 479, 558, 500, 508, 504, 518, 529, 490, 515, 518, 531, 502, 536, 501, 540, 489, 508, 529, 499] payload_rows_per_dst=[5420, 5504, 5296, 5446, 5572, 5600, 5734, 5788] dup_rows_per_dst=[2482, 2614, 2398, 2496, 2686, 2650, 2874, 2976] combine_rows_per_src=[5420, 5504, 5296, 5446, 5572, 5600, 5734, 5788] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44536 payload_rows_remote=38920 dup_rows=21000 dup_rows_remote=18248 dedup_ratio=0.320435 router_tokens_per_expert=[513, 518, 526, 526, 510, 499, 508, 528, 540, 519, 542, 504, 520, 502, 537, 474, 489, 512, 539, 522, 511, 521, 550, 540, 514, 503, 471, 507, 534, 550, 526, 501, 530, 484, 497, 534, 532, 502, 521, 474, 476, 507, 518, 487, 535, 503, 514, 541, 518, 507, 489, 482, 481, 512, 521, 504, 509, 536, 522, 524, 511, 491, 479, 471] payload_rows_per_dst=[5630, 5538, 5616, 5612, 5552, 5642, 5474, 5472] dup_rows_per_dst=[2626, 2738, 2752, 2600, 2596, 2520, 2554, 2614] combine_rows_per_src=[5630, 5538, 5616, 5612, 5552, 5642, 5474, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279293952 dispatch_meta_bytes=228872 combine_payload_bytes=279293952 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279522824 ingress_bytes=499494912 link_bytes=499494912 token_bytes=558816776 weight_movement_bytes=220200960 total_nvlink_bytes=779017736 weight_to_token_ratio=0.394049 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.687038048e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=277401600 dispatch_meta_bytes=227568 combine_payload_bytes=277401600 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277629168 ingress_bytes=387502080 link_bytes=387502080 token_bytes=555030768 weight_movement_bytes=110100480 total_nvlink_bytes=665131248 weight_to_token_ratio=0.198368 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.739298503e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278419456 dispatch_meta_bytes=229288 combine_payload_bytes=278419456 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278648744 ingress_bytes=608720896 link_bytes=608720896 token_bytes=557068200 weight_movement_bytes=330301440 total_nvlink_bytes=887369640 weight_to_token_ratio=0.592928 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058665756e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278118400 dispatch_meta_bytes=229584 combine_payload_bytes=278118400 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278347984 ingress_bytes=388218880 link_bytes=388218880 token_bytes=556466384 weight_movement_bytes=110100480 total_nvlink_bytes=666566864 weight_to_token_ratio=0.197856 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.751764834e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278935552 dispatch_meta_bytes=229592 combine_payload_bytes=278935552 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279165144 ingress_bytes=389036032 link_bytes=389036032 token_bytes=558100696 weight_movement_bytes=110100480 total_nvlink_bytes=668201176 weight_to_token_ratio=0.197277 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.765976451e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280383488 dispatch_meta_bytes=230256 combine_payload_bytes=280383488 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280613744 ingress_bytes=390483968 link_bytes=390483968 token_bytes=560997232 weight_movement_bytes=110100480 total_nvlink_bytes=671097712 weight_to_token_ratio=0.196259 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.791158439e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279939072 dispatch_meta_bytes=229320 combine_payload_bytes=279939072 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=280168392 ingress_bytes=720340992 link_bytes=720340992 token_bytes=560107464 weight_movement_bytes=440401920 total_nvlink_bytes=1000509384 weight_to_token_ratio=0.786281 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.252791461e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44488 payload_rows_remote=38994 dup_rows=21048 dup_rows_remote=18450 dedup_ratio=0.321167 router_tokens_per_expert=[508, 542, 531, 526, 493, 496, 530, 491, 516, 468, 493, 512, 530, 526, 506, 495, 506, 506, 524, 474, 524, 547, 559, 485, 525, 536, 513, 522, 495, 543, 514, 521, 530, 489, 495, 545, 544, 493, 521, 509, 538, 487, 522, 521, 501, 529, 547, 514, 498, 500, 523, 476, 502, 524, 522, 509, 504, 501, 492, 464, 504, 488, 493, 526] payload_rows_per_dst=[5584, 5494, 5554, 5588, 5562, 5682, 5558, 5466] dup_rows_per_dst=[2650, 2598, 2696, 2750, 2690, 2636, 2550, 2478] combine_rows_per_src=[5584, 5494, 5554, 5588, 5562, 5682, 5558, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44610 payload_rows_remote=38858 dup_rows=20926 dup_rows_remote=18282 dedup_ratio=0.319305 router_tokens_per_expert=[516, 496, 494, 515, 550, 476, 512, 571, 512, 508, 504, 479, 479, 529, 526, 490, 498, 486, 509, 526, 537, 504, 499, 522, 499, 516, 515, 507, 504, 497, 525, 525, 516, 549, 496, 494, 519, 532, 492, 481, 525, 506, 522, 512, 508, 520, 536, 491, 513, 509, 538, 559, 458, 551, 560, 518, 516, 492, 541, 486, 463, 525, 481, 533] payload_rows_per_dst=[5752, 5528, 5794, 5366, 5696, 5468, 5590, 5416] dup_rows_per_dst=[2644, 2526, 2912, 2388, 2758, 2372, 2790, 2536] combine_rows_per_src=[5752, 5528, 5794, 5366, 5696, 5468, 5590, 5416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44584 payload_rows_remote=38988 dup_rows=20952 dup_rows_remote=18408 dedup_ratio=0.319702 router_tokens_per_expert=[517, 516, 486, 481, 519, 541, 539, 490, 515, 521, 501, 511, 542, 510, 491, 499, 509, 514, 512, 507, 523, 501, 516, 488, 506, 525, 514, 511, 500, 490, 527, 494, 505, 495, 521, 494, 518, 527, 518, 519, 523, 499, 554, 535, 498, 495, 529, 553, 541, 513, 519, 493, 488, 489, 483, 504, 511, 518, 539, 529, 503, 510, 528, 501] payload_rows_per_dst=[5586, 5506, 5596, 5550, 5534, 5642, 5510, 5660] dup_rows_per_dst=[2592, 2674, 2544, 2584, 2660, 2730, 2550, 2618] combine_rows_per_src=[5586, 5506, 5596, 5550, 5534, 5642, 5510, 5660] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44452 payload_rows_remote=38848 dup_rows=21084 dup_rows_remote=18490 dedup_ratio=0.321716 router_tokens_per_expert=[523, 536, 562, 509, 499, 474, 478, 479, 563, 507, 510, 508, 534, 519, 530, 516, 521, 552, 517, 505, 518, 501, 499, 469, 526, 479, 497, 518, 508, 543, 539, 530, 525, 502, 442, 545, 514, 535, 499, 502, 505, 512, 555, 531, 497, 531, 469, 499, 484, 510, 523, 500, 554, 531, 475, 495, 504, 552, 476, 517, 515, 502, 491, 507] payload_rows_per_dst=[5626, 5624, 5510, 5550, 5592, 5604, 5476, 5470] dup_rows_per_dst=[2494, 2750, 2654, 2730, 2536, 2594, 2668, 2658] combine_rows_per_src=[5626, 5624, 5510, 5550, 5592, 5604, 5476, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44528 payload_rows_remote=39142 dup_rows=21008 dup_rows_remote=18506 dedup_ratio=0.320557 router_tokens_per_expert=[527, 529, 527, 564, 515, 484, 480, 511, 552, 506, 496, 484, 537, 491, 496, 497, 538, 477, 491, 507, 502, 472, 519, 492, 480, 532, 515, 539, 547, 519, 539, 515, 488, 440, 497, 474, 509, 519, 494, 523, 511, 515, 548, 543, 539, 493, 509, 500, 514, 526, 485, 482, 530, 531, 534, 522, 553, 507, 512, 491, 486, 523, 527, 563] payload_rows_per_dst=[5598, 5502, 5516, 5636, 5386, 5632, 5622, 5636] dup_rows_per_dst=[2676, 2616, 2480, 2736, 2502, 2684, 2626, 2688] combine_rows_per_src=[5598, 5502, 5516, 5636, 5386, 5632, 5622, 5636] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279508992 dispatch_meta_bytes=229776 combine_payload_bytes=279508992 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279738768 ingress_bytes=389609472 link_bytes=389609472 token_bytes=559247760 weight_movement_bytes=110100480 total_nvlink_bytes=669348240 weight_to_token_ratio=0.196872 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.775949516e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44432 payload_rows_remote=38872 dup_rows=21104 dup_rows_remote=18386 dedup_ratio=0.322021 router_tokens_per_expert=[469, 495, 548, 503, 503, 510, 528, 535, 500, 508, 526, 488, 524, 523, 544, 484, 515, 514, 524, 518, 491, 528, 492, 508, 561, 524, 521, 520, 516, 503, 510, 526, 504, 482, 544, 511, 520, 516, 451, 519, 503, 500, 498, 483, 515, 480, 514, 528, 539, 518, 558, 527, 527, 500, 473, 497, 496, 543, 487, 544, 509, 471, 511, 541] payload_rows_per_dst=[5532, 5586, 5496, 5654, 5540, 5548, 5560, 5516] dup_rows_per_dst=[2650, 2608, 2684, 2708, 2554, 2494, 2718, 2688] combine_rows_per_src=[5532, 5586, 5496, 5654, 5540, 5548, 5560, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44272 payload_rows_remote=38724 dup_rows=21264 dup_rows_remote=18594 dedup_ratio=0.324463 router_tokens_per_expert=[482, 499, 512, 552, 532, 519, 501, 483, 524, 515, 506, 514, 481, 485, 499, 509, 540, 514, 504, 541, 510, 495, 509, 452, 501, 545, 531, 503, 496, 500, 520, 513, 515, 504, 526, 495, 490, 522, 514, 493, 523, 550, 503, 512, 542, 500, 517, 502, 503, 503, 518, 478, 550, 502, 534, 494, 516, 514, 537, 510, 537, 524, 534, 519] payload_rows_per_dst=[5534, 5456, 5550, 5548, 5444, 5592, 5522, 5626] dup_rows_per_dst=[2626, 2610, 2580, 2670, 2674, 2706, 2642, 2756] combine_rows_per_src=[5534, 5456, 5550, 5548, 5444, 5592, 5522, 5626] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44610 payload_rows_remote=38962 dup_rows=20926 dup_rows_remote=18250 dedup_ratio=0.319305 router_tokens_per_expert=[515, 538, 517, 498, 511, 525, 523, 495, 519, 482, 516, 515, 523, 517, 517, 514, 508, 473, 535, 520, 481, 513, 494, 461, 519, 481, 482, 515, 496, 533, 504, 509, 510, 496, 562, 540, 555, 527, 503, 511, 522, 515, 509, 508, 548, 486, 503, 485, 519, 544, 513, 513, 536, 492, 492, 487, 483, 511, 570, 533, 521, 527, 483, 515] payload_rows_per_dst=[5478, 5716, 5474, 5548, 5660, 5524, 5562, 5648] dup_rows_per_dst=[2514, 2742, 2496, 2530, 2748, 2628, 2592, 2676] combine_rows_per_src=[5478, 5716, 5474, 5548, 5660, 5524, 5562, 5648] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279465984 dispatch_meta_bytes=229584 combine_payload_bytes=279465984 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279695568 ingress_bytes=389566464 link_bytes=389566464 token_bytes=559161552 weight_movement_bytes=110100480 total_nvlink_bytes=669262032 weight_to_token_ratio=0.196903 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.775201536e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278462464 dispatch_meta_bytes=229352 combine_payload_bytes=278462464 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278691816 ingress_bytes=608763904 link_bytes=608763904 token_bytes=557154280 weight_movement_bytes=330301440 total_nvlink_bytes=887455720 weight_to_token_ratio=0.592837 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058740554e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278534144 dispatch_meta_bytes=228560 combine_payload_bytes=278534144 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278762704 ingress_bytes=498735104 link_bytes=498735104 token_bytes=557296848 weight_movement_bytes=220200960 total_nvlink_bytes=777497808 weight_to_token_ratio=0.395123 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.673823737e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=280569856 dispatch_meta_bytes=230592 combine_payload_bytes=280569856 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280800448 ingress_bytes=390670336 link_bytes=390670336 token_bytes=561370304 weight_movement_bytes=110100480 total_nvlink_bytes=671470784 weight_to_token_ratio=0.196128 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.794399685e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278634496 dispatch_meta_bytes=229032 combine_payload_bytes=278634496 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=278863528 ingress_bytes=498835456 link_bytes=498835456 token_bytes=557498024 weight_movement_bytes=220200960 total_nvlink_bytes=777698984 weight_to_token_ratio=0.394981 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.675569023e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277573632 dispatch_meta_bytes=229272 combine_payload_bytes=277573632 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277802904 ingress_bytes=497774592 link_bytes=497774592 token_bytes=555376536 weight_movement_bytes=220200960 total_nvlink_bytes=775577496 weight_to_token_ratio=0.396489 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.657118853e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279279616 dispatch_meta_bytes=228848 combine_payload_bytes=279279616 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279508464 ingress_bytes=499480576 link_bytes=499480576 token_bytes=558788080 weight_movement_bytes=220200960 total_nvlink_bytes=778989040 weight_to_token_ratio=0.394069 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.686788721e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44514 payload_rows_remote=38970 dup_rows=21022 dup_rows_remote=18452 dedup_ratio=0.320770 router_tokens_per_expert=[513, 466, 566, 529, 519, 502, 516, 513, 522, 512, 510, 497, 510, 510, 484, 512, 484, 570, 499, 511, 500, 512, 529, 507, 513, 548, 512, 509, 520, 535, 490, 495, 493, 527, 527, 511, 532, 536, 506, 524, 510, 509, 550, 492, 464, 515, 504, 495, 489, 485, 548, 513, 521, 493, 567, 537, 457, 503, 494, 530, 496, 515, 508, 502] payload_rows_per_dst=[5630, 5544, 5558, 5510, 5578, 5562, 5598, 5534] dup_rows_per_dst=[2618, 2570, 2666, 2734, 2734, 2516, 2708, 2476] combine_rows_per_src=[5630, 5544, 5558, 5510, 5578, 5562, 5598, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279336960 dispatch_meta_bytes=229688 combine_payload_bytes=279336960 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=279566648 ingress_bytes=389437440 link_bytes=389437440 token_bytes=558903608 weight_movement_bytes=110100480 total_nvlink_bytes=669004088 weight_to_token_ratio=0.196994 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.772957596e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44544 payload_rows_remote=38958 dup_rows=20992 dup_rows_remote=18418 dedup_ratio=0.320312 router_tokens_per_expert=[532, 527, 524, 503, 533, 489, 508, 518, 548, 510, 474, 511, 487, 509, 495, 546, 497, 499, 533, 514, 525, 493, 503, 505, 511, 502, 503, 528, 524, 482, 499, 499, 491, 536, 515, 471, 512, 544, 515, 496, 516, 518, 550, 515, 519, 501, 509, 525, 508, 520, 485, 517, 518, 520, 512, 505, 532, 491, 521, 541, 487, 530, 504, 513] payload_rows_per_dst=[5612, 5616, 5522, 5508, 5586, 5540, 5544, 5616] dup_rows_per_dst=[2656, 2544, 2616, 2588, 2574, 2766, 2626, 2622] combine_rows_per_src=[5612, 5616, 5522, 5508, 5586, 5540, 5544, 5616] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44546 payload_rows_remote=38932 dup_rows=20990 dup_rows_remote=18254 dedup_ratio=0.320282 router_tokens_per_expert=[509, 515, 524, 523, 508, 498, 506, 527, 540, 518, 542, 505, 518, 499, 536, 473, 490, 512, 539, 516, 507, 521, 550, 540, 512, 503, 472, 507, 533, 552, 527, 499, 528, 482, 496, 535, 529, 504, 523, 473, 479, 508, 516, 490, 536, 505, 514, 540, 517, 508, 491, 483, 485, 514, 522, 507, 511, 538, 524, 527, 511, 495, 481, 475] payload_rows_per_dst=[5618, 5534, 5614, 5606, 5546, 5654, 5486, 5488] dup_rows_per_dst=[2602, 2728, 2736, 2604, 2594, 2522, 2568, 2636] combine_rows_per_src=[5618, 5534, 5614, 5606, 5546, 5654, 5486, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44540 payload_rows_remote=38884 dup_rows=20996 dup_rows_remote=18314 dedup_ratio=0.320374 router_tokens_per_expert=[529, 510, 521, 502, 508, 522, 504, 492, 506, 507, 496, 476, 535, 534, 515, 515, 560, 548, 483, 532, 513, 547, 512, 517, 510, 525, 512, 475, 504, 472, 527, 532, 539, 478, 526, 483, 495, 523, 519, 514, 514, 517, 518, 527, 545, 484, 501, 518, 497, 502, 538, 509, 492, 500, 515, 517, 505, 560, 526, 484, 504, 508, 498, 471] payload_rows_per_dst=[5656, 5624, 5556, 5514, 5528, 5620, 5516, 5526] dup_rows_per_dst=[2682, 2544, 2706, 2600, 2626, 2628, 2624, 2586] combine_rows_per_src=[5656, 5624, 5556, 5514, 5528, 5620, 5516, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44494 payload_rows_remote=38964 dup_rows=21042 dup_rows_remote=18406 dedup_ratio=0.321075 router_tokens_per_expert=[463, 493, 473, 500, 489, 524, 537, 530, 537, 485, 486, 470, 527, 550, 519, 516, 499, 467, 479, 529, 540, 492, 512, 526, 500, 525, 482, 542, 548, 533, 539, 518, 525, 470, 517, 476, 524, 531, 502, 521, 535, 515, 518, 482, 546, 502, 500, 485, 532, 498, 533, 505, 500, 499, 542, 544, 526, 513, 497, 511, 520, 537, 521, 511] payload_rows_per_dst=[5466, 5548, 5574, 5664, 5530, 5530, 5578, 5604] dup_rows_per_dst=[2552, 2632, 2514, 2710, 2602, 2636, 2728, 2668] combine_rows_per_src=[5466, 5548, 5574, 5664, 5530, 5530, 5578, 5604] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44616 payload_rows_remote=39044 dup_rows=20920 dup_rows_remote=18406 dedup_ratio=0.319214 router_tokens_per_expert=[494, 531, 544, 524, 527, 473, 540, 568, 484, 497, 499, 545, 511, 545, 490, 527, 511, 519, 489, 534, 518, 505, 521, 516, 494, 496, 516, 535, 496, 506, 536, 544, 476, 497, 527, 501, 516, 514, 491, 538, 490, 519, 518, 498, 512, 534, 533, 517, 502, 534, 482, 528, 453, 514, 517, 513, 481, 485, 493, 534, 505, 510, 498, 503] payload_rows_per_dst=[5610, 5596, 5598, 5604, 5486, 5636, 5572, 5514] dup_rows_per_dst=[2792, 2600, 2628, 2642, 2634, 2606, 2514, 2504] combine_rows_per_src=[5610, 5596, 5598, 5604, 5486, 5636, 5572, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44558 payload_rows_remote=38964 dup_rows=20978 dup_rows_remote=18318 dedup_ratio=0.320099 router_tokens_per_expert=[494, 514, 502, 526, 500, 476, 475, 534, 504, 503, 523, 502, 480, 499, 544, 518, 511, 506, 534, 496, 503, 540, 505, 520, 496, 581, 517, 521, 510, 494, 496, 512, 520, 518, 496, 512, 549, 511, 525, 522, 502, 496, 512, 514, 502, 545, 505, 511, 513, 507, 521, 511, 493, 493, 515, 493, 500, 542, 519, 541, 519, 538, 508, 479] payload_rows_per_dst=[5518, 5584, 5570, 5594, 5618, 5568, 5478, 5628] dup_rows_per_dst=[2524, 2562, 2660, 2660, 2688, 2606, 2614, 2664] combine_rows_per_src=[5518, 5584, 5570, 5594, 5618, 5568, 5478, 5628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44386 payload_rows_remote=38724 dup_rows=21150 dup_rows_remote=18342 dedup_ratio=0.322723 router_tokens_per_expert=[478, 515, 466, 536, 493, 523, 510, 479, 497, 564, 541, 527, 455, 521, 559, 493, 551, 519, 525, 535, 519, 492, 542, 508, 486, 554, 529, 494, 482, 559, 497, 493, 523, 494, 491, 512, 485, 492, 512, 525, 531, 464, 504, 504, 490, 479, 562, 500, 511, 505, 519, 532, 489, 519, 519, 535, 505, 538, 504, 543, 494, 511, 531, 503] payload_rows_per_dst=[5470, 5590, 5432, 5466, 5554, 5570, 5642, 5662] dup_rows_per_dst=[2530, 2720, 2586, 2532, 2658, 2576, 2740, 2808] combine_rows_per_src=[5470, 5590, 5432, 5466, 5554, 5570, 5642, 5662] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279250944 dispatch_meta_bytes=229504 combine_payload_bytes=279250944 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279480448 ingress_bytes=389351424 link_bytes=389351424 token_bytes=558731392 weight_movement_bytes=110100480 total_nvlink_bytes=668831872 weight_to_token_ratio=0.197054 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.771461636e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278720512 dispatch_meta_bytes=228792 combine_payload_bytes=278720512 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278949304 ingress_bytes=388820992 link_bytes=388820992 token_bytes=557669816 weight_movement_bytes=110100480 total_nvlink_bytes=667770296 weight_to_token_ratio=0.197430 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.762236552e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279867392 dispatch_meta_bytes=229800 combine_payload_bytes=279867392 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280097192 ingress_bytes=389967872 link_bytes=389967872 token_bytes=559964584 weight_movement_bytes=110100480 total_nvlink_bytes=670065064 weight_to_token_ratio=0.196620 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.782182681e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279064576 dispatch_meta_bytes=228744 combine_payload_bytes=279064576 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=279293320 ingress_bytes=829566976 link_bytes=829566976 token_bytes=558357896 weight_movement_bytes=550502400 total_nvlink_bytes=1108860296 weight_to_token_ratio=0.985931 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.442753412e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279293952 dispatch_meta_bytes=229480 combine_payload_bytes=279293952 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279523432 ingress_bytes=389394432 link_bytes=389394432 token_bytes=558817384 weight_movement_bytes=110100480 total_nvlink_bytes=668917864 weight_to_token_ratio=0.197024 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.772209616e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44502 payload_rows_remote=38976 dup_rows=21034 dup_rows_remote=18396 dedup_ratio=0.320953 router_tokens_per_expert=[465, 496, 476, 503, 492, 524, 538, 531, 540, 487, 487, 472, 530, 553, 522, 518, 500, 468, 481, 529, 542, 491, 518, 528, 499, 526, 481, 544, 548, 536, 540, 520, 527, 467, 517, 475, 521, 530, 501, 518, 537, 514, 520, 481, 546, 498, 501, 485, 530, 494, 535, 504, 496, 497, 536, 539, 520, 511, 498, 506, 519, 533, 520, 507] payload_rows_per_dst=[5488, 5566, 5584, 5674, 5514, 5526, 5568, 5582] dup_rows_per_dst=[2562, 2652, 2530, 2714, 2598, 2638, 2694, 2646] combine_rows_per_src=[5488, 5566, 5584, 5674, 5514, 5526, 5568, 5582] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44522 payload_rows_remote=38970 dup_rows=21014 dup_rows_remote=18428 dedup_ratio=0.320648 router_tokens_per_expert=[515, 468, 568, 530, 523, 506, 517, 514, 525, 516, 510, 496, 512, 512, 485, 513, 487, 571, 499, 513, 500, 513, 529, 507, 511, 551, 513, 509, 521, 535, 491, 497, 494, 525, 529, 511, 532, 532, 505, 525, 509, 510, 550, 492, 463, 515, 502, 495, 488, 484, 544, 511, 517, 492, 565, 534, 457, 500, 491, 529, 496, 511, 503, 500] payload_rows_per_dst=[5644, 5552, 5572, 5508, 5576, 5568, 5584, 5518] dup_rows_per_dst=[2638, 2586, 2666, 2748, 2730, 2504, 2686, 2456] combine_rows_per_src=[5644, 5552, 5572, 5508, 5576, 5568, 5584, 5518] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=276484096 dispatch_meta_bytes=227088 combine_payload_bytes=276484096 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276711184 ingress_bytes=386584576 link_bytes=386584576 token_bytes=553195280 weight_movement_bytes=110100480 total_nvlink_bytes=663295760 weight_to_token_ratio=0.199026 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.723341599e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44568 payload_rows_remote=38976 dup_rows=20968 dup_rows_remote=18306 dedup_ratio=0.319946 router_tokens_per_expert=[496, 513, 505, 528, 504, 478, 479, 535, 507, 504, 526, 505, 481, 503, 549, 518, 512, 508, 537, 499, 505, 540, 508, 521, 497, 582, 517, 520, 511, 493, 495, 512, 520, 518, 496, 510, 549, 513, 528, 519, 502, 497, 511, 513, 498, 541, 504, 511, 511, 506, 518, 506, 493, 492, 509, 492, 500, 539, 517, 537, 516, 532, 507, 475] payload_rows_per_dst=[5530, 5606, 5586, 5592, 5622, 5558, 5466, 5608] dup_rows_per_dst=[2546, 2580, 2674, 2662, 2684, 2596, 2588, 2638] combine_rows_per_src=[5530, 5606, 5586, 5592, 5622, 5558, 5466, 5608] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44558 payload_rows_remote=38974 dup_rows=20978 dup_rows_remote=18418 dedup_ratio=0.320099 router_tokens_per_expert=[532, 529, 528, 505, 537, 491, 509, 522, 550, 510, 476, 512, 490, 510, 498, 548, 499, 499, 537, 514, 527, 493, 506, 506, 512, 501, 504, 527, 522, 482, 501, 500, 490, 535, 515, 470, 510, 542, 515, 495, 513, 516, 550, 514, 517, 501, 506, 526, 509, 519, 485, 517, 515, 519, 509, 503, 530, 487, 519, 540, 484, 528, 501, 511] payload_rows_per_dst=[5626, 5632, 5534, 5512, 5584, 5534, 5536, 5600] dup_rows_per_dst=[2680, 2556, 2628, 2586, 2560, 2752, 2616, 2600] combine_rows_per_src=[5626, 5632, 5534, 5512, 5584, 5534, 5536, 5600] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278978560 dispatch_meta_bytes=228672 combine_payload_bytes=278978560 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279207232 ingress_bytes=609280000 link_bytes=609280000 token_bytes=558185792 weight_movement_bytes=330301440 total_nvlink_bytes=888487232 weight_to_token_ratio=0.591741 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059638129e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279379968 dispatch_meta_bytes=229488 combine_payload_bytes=279379968 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279609456 ingress_bytes=389480448 link_bytes=389480448 token_bytes=558989424 weight_movement_bytes=110100480 total_nvlink_bytes=669089904 weight_to_token_ratio=0.196963 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773705576e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44602 payload_rows_remote=39052 dup_rows=20934 dup_rows_remote=18440 dedup_ratio=0.319427 router_tokens_per_expert=[498, 534, 547, 526, 531, 475, 540, 571, 485, 497, 499, 548, 511, 547, 488, 530, 512, 519, 492, 535, 520, 508, 522, 516, 498, 497, 518, 533, 498, 507, 539, 544, 477, 498, 527, 499, 514, 512, 493, 539, 489, 521, 518, 497, 510, 533, 533, 515, 501, 534, 478, 524, 454, 507, 513, 511, 478, 483, 491, 530, 503, 505, 495, 501] payload_rows_per_dst=[5624, 5600, 5606, 5616, 5480, 5630, 5550, 5496] dup_rows_per_dst=[2820, 2610, 2642, 2652, 2638, 2602, 2494, 2476] combine_rows_per_src=[5624, 5600, 5606, 5616, 5480, 5630, 5550, 5496] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279336960 dispatch_meta_bytes=229592 combine_payload_bytes=279336960 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279566552 ingress_bytes=609638400 link_bytes=609638400 token_bytes=558903512 weight_movement_bytes=330301440 total_nvlink_bytes=889204952 weight_to_token_ratio=0.590981 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060261446e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44546 payload_rows_remote=38928 dup_rows=20990 dup_rows_remote=18388 dedup_ratio=0.320282 router_tokens_per_expert=[531, 517, 522, 506, 510, 523, 507, 494, 505, 513, 498, 479, 538, 533, 516, 519, 560, 553, 484, 532, 515, 551, 511, 520, 510, 526, 515, 475, 503, 472, 528, 530, 536, 478, 527, 481, 494, 522, 520, 514, 510, 518, 516, 526, 543, 482, 495, 518, 499, 497, 537, 508, 490, 499, 515, 514, 502, 557, 521, 482, 499, 508, 497, 467] payload_rows_per_dst=[5618, 5640, 5638, 5514, 5520, 5612, 5504, 5500] dup_rows_per_dst=[2602, 2562, 2814, 2604, 2624, 2604, 2614, 2566] combine_rows_per_src=[5618, 5640, 5638, 5514, 5520, 5612, 5504, 5500] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279379968 dispatch_meta_bytes=229128 combine_payload_bytes=279379968 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279609096 ingress_bytes=499580928 link_bytes=499580928 token_bytes=558989064 weight_movement_bytes=220200960 total_nvlink_bytes=779190024 weight_to_token_ratio=0.393927 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.688534007e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279365632 dispatch_meta_bytes=229568 combine_payload_bytes=279365632 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279595200 ingress_bytes=389466112 link_bytes=389466112 token_bytes=558960832 weight_movement_bytes=110100480 total_nvlink_bytes=669061312 weight_to_token_ratio=0.196974 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773456249e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279924736 dispatch_meta_bytes=229968 combine_payload_bytes=279924736 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280154704 ingress_bytes=390025216 link_bytes=390025216 token_bytes=560079440 weight_movement_bytes=110100480 total_nvlink_bytes=670179920 weight_to_token_ratio=0.196580 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.783179987e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279035904 dispatch_meta_bytes=229264 combine_payload_bytes=279035904 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279265168 ingress_bytes=499236864 link_bytes=499236864 token_bytes=558301072 weight_movement_bytes=220200960 total_nvlink_bytes=778502032 weight_to_token_ratio=0.394413 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.682550169e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44432 payload_rows_remote=38622 dup_rows=21104 dup_rows_remote=17984 dedup_ratio=0.322021 router_tokens_per_expert=[516, 526, 541, 521, 474, 531, 561, 541, 529, 478, 541, 501, 513, 488, 554, 512, 513, 509, 510, 517, 508, 471, 516, 530, 543, 490, 489, 505, 508, 522, 491, 507, 500, 544, 520, 506, 525, 527, 483, 536, 500, 480, 499, 515, 527, 479, 502, 500, 489, 531, 505, 505, 492, 531, 521, 495, 514, 475, 515, 517, 516, 523, 527, 513] payload_rows_per_dst=[5380, 5550, 5566, 5590, 5556, 5370, 5610, 5810] dup_rows_per_dst=[2312, 2518, 2548, 2684, 2760, 2442, 2720, 3120] combine_rows_per_src=[5380, 5550, 5566, 5590, 5556, 5370, 5610, 5810] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44596 payload_rows_remote=38942 dup_rows=20940 dup_rows_remote=18310 dedup_ratio=0.319519 router_tokens_per_expert=[518, 522, 544, 501, 490, 444, 526, 545, 527, 508, 536, 523, 506, 492, 497, 499, 515, 514, 513, 515, 538, 465, 520, 562, 468, 523, 476, 554, 541, 500, 472, 520, 486, 525, 530, 504, 537, 507, 520, 482, 491, 520, 512, 536, 564, 542, 493, 527, 505, 527, 503, 517, 536, 484, 513, 504, 455, 505, 517, 495, 526, 509, 518, 504] payload_rows_per_dst=[5586, 5578, 5654, 5518, 5494, 5640, 5562, 5564] dup_rows_per_dst=[2594, 2598, 2630, 2590, 2688, 2730, 2616, 2494] combine_rows_per_src=[5586, 5578, 5654, 5518, 5494, 5640, 5562, 5564] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44510 payload_rows_remote=38970 dup_rows=21026 dup_rows_remote=18280 dedup_ratio=0.320831 router_tokens_per_expert=[504, 531, 519, 505, 522, 477, 495, 501, 466, 553, 497, 522, 501, 518, 516, 502, 549, 515, 514, 491, 490, 503, 499, 539, 542, 526, 527, 503, 490, 538, 522, 502, 536, 508, 500, 497, 536, 490, 507, 473, 526, 515, 548, 526, 516, 479, 488, 545, 521, 504, 518, 558, 530, 531, 508, 538, 478, 493, 505, 524, 505, 496, 507, 483] payload_rows_per_dst=[5528, 5590, 5590, 5594, 5520, 5540, 5720, 5428] dup_rows_per_dst=[2580, 2560, 2610, 2706, 2574, 2746, 2696, 2554] combine_rows_per_src=[5528, 5590, 5590, 5594, 5520, 5540, 5720, 5428] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=276842496 dispatch_meta_bytes=226424 combine_payload_bytes=276842496 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277068920 ingress_bytes=386942976 link_bytes=386942976 token_bytes=553911416 weight_movement_bytes=110100480 total_nvlink_bytes=664011896 weight_to_token_ratio=0.198769 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.729574764e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44344 payload_rows_remote=38726 dup_rows=21192 dup_rows_remote=18588 dedup_ratio=0.323364 router_tokens_per_expert=[502, 501, 534, 508, 504, 504, 505, 527, 558, 524, 486, 478, 521, 487, 555, 502, 509, 525, 521, 525, 515, 499, 505, 498, 507, 526, 511, 514, 546, 508, 494, 538, 487, 531, 504, 508, 546, 524, 448, 553, 545, 502, 528, 533, 522, 480, 504, 538, 500, 538, 497, 523, 493, 496, 552, 498, 484, 516, 515, 483, 527, 493, 481, 482] payload_rows_per_dst=[5546, 5618, 5556, 5592, 5492, 5564, 5568, 5408] dup_rows_per_dst=[2624, 2604, 2638, 2696, 2710, 2740, 2626, 2554] combine_rows_per_src=[5546, 5618, 5556, 5592, 5492, 5564, 5568, 5408] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44550 payload_rows_remote=39020 dup_rows=20986 dup_rows_remote=18374 dedup_ratio=0.320221 router_tokens_per_expert=[527, 470, 544, 546, 529, 528, 523, 499, 482, 521, 512, 505, 522, 553, 531, 525, 516, 551, 514, 518, 489, 494, 536, 514, 521, 538, 528, 534, 527, 469, 532, 481, 519, 506, 491, 540, 534, 477, 521, 483, 523, 522, 529, 540, 476, 495, 504, 500, 482, 529, 510, 494, 491, 524, 486, 484, 513, 490, 481, 498, 479, 558, 496, 514] payload_rows_per_dst=[5570, 5632, 5648, 5634, 5530, 5566, 5452, 5518] dup_rows_per_dst=[2762, 2670, 2616, 2626, 2612, 2612, 2548, 2540] combine_rows_per_src=[5570, 5632, 5648, 5634, 5530, 5566, 5452, 5518] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44696 payload_rows_remote=39146 dup_rows=20840 dup_rows_remote=18258 dedup_ratio=0.317993 router_tokens_per_expert=[525, 494, 495, 519, 551, 494, 510, 476, 517, 519, 523, 521, 484, 523, 500, 519, 526, 511, 516, 501, 567, 516, 507, 504, 491, 524, 517, 520, 487, 524, 499, 504, 521, 500, 527, 523, 515, 512, 475, 501, 478, 535, 508, 501, 512, 508, 516, 517, 518, 478, 543, 522, 500, 503, 538, 497, 506, 530, 525, 530, 548, 537, 480, 480] payload_rows_per_dst=[5588, 5588, 5624, 5550, 5558, 5620, 5572, 5596] dup_rows_per_dst=[2540, 2624, 2672, 2582, 2590, 2530, 2626, 2676] combine_rows_per_src=[5588, 5588, 5624, 5550, 5558, 5620, 5572, 5596] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279136256 dispatch_meta_bytes=229008 combine_payload_bytes=279136256 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279365264 ingress_bytes=499337216 link_bytes=499337216 token_bytes=558501520 weight_movement_bytes=220200960 total_nvlink_bytes=778702480 weight_to_token_ratio=0.394271 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.684295455e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44616 payload_rows_remote=39102 dup_rows=20920 dup_rows_remote=18298 dedup_ratio=0.319214 router_tokens_per_expert=[533, 526, 534, 529, 520, 496, 536, 514, 490, 538, 551, 474, 497, 509, 536, 503, 488, 503, 526, 508, 485, 490, 520, 495, 522, 495, 521, 475, 498, 512, 531, 536, 544, 504, 512, 493, 536, 509, 497, 554, 519, 517, 515, 520, 498, 516, 485, 511, 510, 502, 532, 492, 452, 515, 550, 515, 495, 520, 526, 539, 516, 489, 521, 473] payload_rows_per_dst=[5630, 5632, 5476, 5594, 5604, 5598, 5514, 5568] dup_rows_per_dst=[2746, 2564, 2554, 2586, 2694, 2564, 2622, 2590] combine_rows_per_src=[5630, 5632, 5476, 5594, 5604, 5598, 5514, 5568] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44434 payload_rows_remote=38742 dup_rows=21102 dup_rows_remote=18244 dedup_ratio=0.321991 router_tokens_per_expert=[516, 528, 516, 513, 528, 544, 529, 526, 523, 490, 542, 517, 502, 540, 504, 527, 472, 508, 488, 545, 526, 506, 532, 498, 481, 477, 463, 516, 502, 538, 520, 500, 535, 521, 496, 503, 486, 519, 497, 520, 494, 505, 519, 513, 550, 532, 537, 524, 538, 489, 482, 522, 506, 467, 544, 494, 474, 546, 483, 529, 480, 479, 558, 509] payload_rows_per_dst=[5692, 5572, 5554, 5464, 5522, 5522, 5512, 5596] dup_rows_per_dst=[2858, 2638, 2600, 2530, 2632, 2752, 2572, 2520] combine_rows_per_src=[5692, 5572, 5554, 5464, 5522, 5522, 5512, 5596] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279336960 dispatch_meta_bytes=229000 combine_payload_bytes=279336960 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279565960 ingress_bytes=609638400 link_bytes=609638400 token_bytes=558902920 weight_movement_bytes=330301440 total_nvlink_bytes=889204360 weight_to_token_ratio=0.590982 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060261446e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277587968 dispatch_meta_bytes=229256 combine_payload_bytes=277587968 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=277817224 ingress_bytes=607889408 link_bytes=607889408 token_bytes=555405192 weight_movement_bytes=330301440 total_nvlink_bytes=885706632 weight_to_token_ratio=0.594704 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057219661e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279695360 dispatch_meta_bytes=229576 combine_payload_bytes=279695360 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279924936 ingress_bytes=389795840 link_bytes=389795840 token_bytes=559620296 weight_movement_bytes=110100480 total_nvlink_bytes=669720776 weight_to_token_ratio=0.196741 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.779190762e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=280598528 dispatch_meta_bytes=229616 combine_payload_bytes=280598528 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280828144 ingress_bytes=390699008 link_bytes=390699008 token_bytes=561426672 weight_movement_bytes=110100480 total_nvlink_bytes=671527152 weight_to_token_ratio=0.196108 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.794898339e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=280283136 dispatch_meta_bytes=229600 combine_payload_bytes=280283136 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280512736 ingress_bytes=390383616 link_bytes=390383616 token_bytes=560795872 weight_movement_bytes=110100480 total_nvlink_bytes=670896352 weight_to_token_ratio=0.196329 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.789413153e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44442 payload_rows_remote=38716 dup_rows=21094 dup_rows_remote=18356 dedup_ratio=0.321869 router_tokens_per_expert=[514, 492, 560, 542, 528, 492, 499, 484, 518, 517, 490, 486, 519, 521, 494, 515, 539, 498, 518, 492, 502, 528, 543, 509, 518, 493, 546, 519, 540, 524, 511, 505, 527, 508, 485, 527, 509, 496, 515, 504, 515, 471, 559, 524, 496, 507, 508, 488, 477, 531, 488, 500, 465, 504, 543, 521, 548, 462, 539, 496, 519, 532, 484, 564] payload_rows_per_dst=[5472, 5560, 5592, 5570, 5432, 5606, 5484, 5726] dup_rows_per_dst=[2462, 2560, 2666, 2742, 2534, 2818, 2574, 2738] combine_rows_per_src=[5472, 5560, 5592, 5570, 5432, 5606, 5484, 5726] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277702656 dispatch_meta_bytes=227944 combine_payload_bytes=277702656 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277930600 ingress_bytes=497903616 link_bytes=497903616 token_bytes=555633256 weight_movement_bytes=220200960 total_nvlink_bytes=775834216 weight_to_token_ratio=0.396306 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.659362793e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44330 payload_rows_remote=38724 dup_rows=21206 dup_rows_remote=18610 dedup_ratio=0.323578 router_tokens_per_expert=[535, 538, 536, 518, 520, 539, 473, 479, 535, 495, 511, 482, 474, 534, 514, 521, 523, 511, 511, 490, 488, 507, 524, 547, 512, 500, 525, 540, 484, 504, 523, 511, 497, 522, 502, 488, 511, 486, 496, 522, 497, 507, 506, 513, 492, 556, 504, 479, 530, 505, 538, 499, 525, 514, 520, 516, 533, 552, 520, 515, 470, 512, 528, 509] payload_rows_per_dst=[5554, 5558, 5606, 5518, 5494, 5454, 5586, 5560] dup_rows_per_dst=[2722, 2574, 2596, 2680, 2554, 2654, 2708, 2718] combine_rows_per_src=[5554, 5558, 5606, 5518, 5494, 5454, 5586, 5560] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277516288 dispatch_meta_bytes=228288 combine_payload_bytes=277516288 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277744576 ingress_bytes=387616768 link_bytes=387616768 token_bytes=555260864 weight_movement_bytes=110100480 total_nvlink_bytes=665361344 weight_to_token_ratio=0.198286 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.741293116e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44494 payload_rows_remote=38954 dup_rows=21042 dup_rows_remote=18454 dedup_ratio=0.321075 router_tokens_per_expert=[517, 498, 498, 538, 534, 511, 506, 476, 487, 528, 510, 524, 493, 522, 447, 519, 557, 535, 531, 522, 536, 544, 520, 506, 536, 490, 517, 526, 520, 515, 538, 514, 529, 563, 549, 495, 522, 498, 507, 492, 506, 505, 521, 505, 538, 482, 519, 488, 485, 532, 496, 517, 514, 490, 484, 505, 516, 540, 491, 516, 511, 506, 451, 480] payload_rows_per_dst=[5546, 5522, 5664, 5626, 5570, 5540, 5486, 5540] dup_rows_per_dst=[2610, 2538, 2838, 2686, 2740, 2588, 2560, 2482] combine_rows_per_src=[5546, 5522, 5664, 5626, 5570, 5540, 5486, 5540] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277573632 dispatch_meta_bytes=229336 combine_payload_bytes=277573632 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277802968 ingress_bytes=497774592 link_bytes=497774592 token_bytes=555376600 weight_movement_bytes=220200960 total_nvlink_bytes=775577560 weight_to_token_ratio=0.396489 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.657118853e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44460 payload_rows_remote=38928 dup_rows=21076 dup_rows_remote=18514 dedup_ratio=0.321594 router_tokens_per_expert=[493, 480, 538, 496, 513, 522, 533, 494, 541, 545, 483, 509, 490, 476, 497, 507, 499, 493, 506, 518, 542, 496, 500, 553, 505, 484, 524, 539, 524, 513, 535, 553, 500, 508, 476, 499, 502, 475, 533, 554, 512, 505, 500, 532, 496, 453, 491, 514, 500, 531, 506, 491, 541, 529, 524, 534, 463, 530, 500, 540, 564, 548, 506, 510] payload_rows_per_dst=[5520, 5502, 5546, 5634, 5532, 5444, 5626, 5656] dup_rows_per_dst=[2618, 2594, 2668, 2720, 2562, 2562, 2686, 2666] combine_rows_per_src=[5520, 5502, 5546, 5634, 5532, 5444, 5626, 5656] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44376 payload_rows_remote=38898 dup_rows=21160 dup_rows_remote=18654 dedup_ratio=0.322876 router_tokens_per_expert=[511, 505, 499, 493, 517, 522, 505, 545, 479, 518, 496, 496, 498, 516, 488, 501, 490, 505, 502, 543, 527, 536, 534, 523, 512, 499, 534, 501, 511, 515, 544, 496, 523, 523, 517, 530, 550, 520, 498, 518, 502, 503, 494, 521, 552, 519, 483, 506, 543, 457, 519, 526, 490, 517, 478, 490, 524, 510, 491, 525, 510, 519, 507, 542] payload_rows_per_dst=[5596, 5478, 5580, 5620, 5578, 5476, 5462, 5586] dup_rows_per_dst=[2598, 2506, 2740, 2604, 2780, 2684, 2578, 2670] combine_rows_per_src=[5596, 5478, 5580, 5620, 5578, 5476, 5462, 5586] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44566 payload_rows_remote=39028 dup_rows=20970 dup_rows_remote=18292 dedup_ratio=0.319977 router_tokens_per_expert=[513, 534, 527, 520, 533, 504, 541, 496, 524, 523, 484, 517, 522, 528, 494, 495, 529, 499, 500, 493, 523, 508, 505, 504, 514, 521, 486, 497, 514, 544, 513, 519, 501, 524, 499, 526, 532, 486, 514, 504, 512, 506, 485, 517, 507, 498, 549, 533, 492, 518, 536, 509, 511, 499, 547, 517, 510, 477, 522, 531, 475, 482, 487, 538] payload_rows_per_dst=[5626, 5540, 5582, 5538, 5582, 5602, 5612, 5484] dup_rows_per_dst=[2710, 2634, 2540, 2678, 2590, 2612, 2646, 2560] combine_rows_per_src=[5626, 5540, 5582, 5538, 5582, 5602, 5612, 5484] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44460 payload_rows_remote=38896 dup_rows=21076 dup_rows_remote=18386 dedup_ratio=0.321594 router_tokens_per_expert=[503, 543, 481, 490, 508, 528, 484, 482, 511, 517, 513, 499, 532, 530, 514, 524, 456, 502, 511, 560, 532, 474, 541, 484, 521, 527, 478, 531, 521, 529, 452, 496, 543, 488, 503, 535, 518, 489, 524, 501, 513, 561, 540, 548, 492, 488, 522, 505, 534, 522, 522, 523, 477, 527, 511, 511, 529, 502, 457, 496, 527, 542, 520, 524] payload_rows_per_dst=[5466, 5536, 5546, 5594, 5542, 5632, 5564, 5580] dup_rows_per_dst=[2572, 2744, 2574, 2516, 2660, 2706, 2690, 2614] combine_rows_per_src=[5466, 5536, 5546, 5594, 5542, 5632, 5564, 5580] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44330 payload_rows_remote=38670 dup_rows=21206 dup_rows_remote=18402 dedup_ratio=0.323578 router_tokens_per_expert=[514, 518, 495, 501, 563, 528, 516, 569, 498, 526, 506, 487, 502, 505, 462, 509, 556, 519, 499, 484, 492, 486, 518, 540, 479, 471, 538, 534, 542, 502, 511, 504, 494, 503, 505, 554, 513, 534, 546, 505, 522, 486, 529, 505, 487, 518, 533, 499, 507, 544, 513, 516, 514, 511, 520, 558, 517, 498, 501, 476, 472, 507, 483, 524] payload_rows_per_dst=[5660, 5650, 5428, 5330, 5672, 5506, 5590, 5494] dup_rows_per_dst=[2804, 3040, 2370, 2480, 2714, 2652, 2684, 2462] combine_rows_per_src=[5660, 5650, 5428, 5330, 5672, 5506, 5590, 5494] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279222272 dispatch_meta_bytes=229632 combine_payload_bytes=279222272 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279451904 ingress_bytes=389322752 link_bytes=389322752 token_bytes=558674176 weight_movement_bytes=110100480 total_nvlink_bytes=668774656 weight_to_token_ratio=0.197075 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.770962983e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279035904 dispatch_meta_bytes=229768 combine_payload_bytes=279035904 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279265672 ingress_bytes=499236864 link_bytes=499236864 token_bytes=558301576 weight_movement_bytes=220200960 total_nvlink_bytes=778502536 weight_to_token_ratio=0.394412 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.682550169e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278820864 dispatch_meta_bytes=230208 combine_payload_bytes=278820864 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279051072 ingress_bytes=388921344 link_bytes=388921344 token_bytes=557871936 weight_movement_bytes=110100480 total_nvlink_bytes=667972416 weight_to_token_ratio=0.197358 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.763981838e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279752704 dispatch_meta_bytes=229280 combine_payload_bytes=279752704 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279981984 ingress_bytes=610054144 link_bytes=610054144 token_bytes=559734688 weight_movement_bytes=330301440 total_nvlink_bytes=890036128 weight_to_token_ratio=0.590104 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060984493e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44538 payload_rows_remote=38932 dup_rows=20998 dup_rows_remote=18316 dedup_ratio=0.320404 router_tokens_per_expert=[457, 527, 522, 517, 524, 548, 512, 494, 507, 533, 516, 527, 521, 516, 551, 510, 534, 515, 488, 493, 513, 497, 480, 512, 490, 545, 519, 517, 529, 530, 522, 508, 521, 552, 529, 533, 516, 493, 522, 487, 491, 502, 488, 490, 535, 512, 498, 453, 545, 507, 484, 481, 540, 477, 530, 508, 497, 550, 486, 542, 535, 461, 520, 509] payload_rows_per_dst=[5300, 5702, 5480, 5600, 5600, 5614, 5636, 5606] dup_rows_per_dst=[2226, 2732, 2456, 2768, 2798, 2612, 2724, 2682] combine_rows_per_src=[5300, 5702, 5480, 5600, 5600, 5614, 5636, 5606] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278806528 dispatch_meta_bytes=229128 combine_payload_bytes=278806528 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279035656 ingress_bytes=499007488 link_bytes=499007488 token_bytes=557842184 weight_movement_bytes=220200960 total_nvlink_bytes=778043144 weight_to_token_ratio=0.394737 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.678560943e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277186560 dispatch_meta_bytes=228288 combine_payload_bytes=277186560 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277414848 ingress_bytes=497387520 link_bytes=497387520 token_bytes=554601408 weight_movement_bytes=220200960 total_nvlink_bytes=774802368 weight_to_token_ratio=0.397044 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.650387035e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44548 payload_rows_remote=38892 dup_rows=20988 dup_rows_remote=18300 dedup_ratio=0.320251 router_tokens_per_expert=[522, 534, 522, 535, 557, 515, 505, 505, 523, 493, 524, 496, 511, 518, 500, 520, 538, 534, 544, 512, 495, 514, 534, 501, 494, 503, 521, 516, 528, 474, 522, 534, 509, 423, 501, 500, 511, 492, 553, 516, 493, 560, 488, 530, 534, 496, 531, 529, 489, 468, 521, 541, 535, 495, 497, 488, 505, 494, 521, 481, 491, 521, 507, 504] payload_rows_per_dst=[5620, 5558, 5656, 5570, 5500, 5638, 5476, 5530] dup_rows_per_dst=[2770, 2612, 2688, 2614, 2510, 2684, 2592, 2518] combine_rows_per_src=[5620, 5558, 5656, 5570, 5500, 5638, 5476, 5530] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279064576 dispatch_meta_bytes=228992 combine_payload_bytes=279064576 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279293568 ingress_bytes=389165056 link_bytes=389165056 token_bytes=558358144 weight_movement_bytes=110100480 total_nvlink_bytes=668458624 weight_to_token_ratio=0.197186 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.768220390e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278777856 dispatch_meta_bytes=228768 combine_payload_bytes=278777856 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279006624 ingress_bytes=609079296 link_bytes=609079296 token_bytes=557784480 weight_movement_bytes=330301440 total_nvlink_bytes=888085920 weight_to_token_ratio=0.592167 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059289072e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44418 payload_rows_remote=38954 dup_rows=21118 dup_rows_remote=18622 dedup_ratio=0.322235 router_tokens_per_expert=[495, 529, 520, 463, 516, 552, 522, 493, 508, 546, 508, 509, 500, 500, 489, 533, 553, 498, 493, 517, 551, 511, 524, 528, 486, 526, 495, 485, 509, 500, 493, 532, 505, 499, 518, 503, 543, 481, 519, 524, 510, 510, 500, 490, 475, 513, 494, 488, 490, 505, 527, 528, 497, 480, 497, 536, 489, 532, 523, 553, 550, 570, 520, 515] payload_rows_per_dst=[5512, 5494, 5578, 5556, 5576, 5464, 5572, 5666] dup_rows_per_dst=[2668, 2692, 2772, 2496, 2608, 2496, 2548, 2838] combine_rows_per_src=[5512, 5494, 5578, 5556, 5576, 5464, 5572, 5666] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44476 payload_rows_remote=38854 dup_rows=21060 dup_rows_remote=18326 dedup_ratio=0.321350 router_tokens_per_expert=[542, 518, 509, 513, 504, 533, 497, 508, 516, 519, 502, 494, 508, 502, 499, 496, 513, 529, 478, 522, 554, 486, 515, 502, 491, 543, 483, 496, 544, 484, 540, 530, 531, 513, 521, 499, 504, 535, 539, 536, 523, 511, 536, 482, 506, 522, 500, 463, 518, 530, 496, 517, 527, 504, 532, 521, 481, 507, 512, 502, 496, 515, 509, 510] payload_rows_per_dst=[5640, 5574, 5574, 5524, 5622, 5478, 5550, 5514] dup_rows_per_dst=[2608, 2498, 2624, 2698, 2734, 2608, 2740, 2550] combine_rows_per_src=[5640, 5574, 5574, 5524, 5622, 5478, 5550, 5514] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44450 payload_rows_remote=38866 dup_rows=21086 dup_rows_remote=18448 dedup_ratio=0.321747 router_tokens_per_expert=[528, 502, 507, 479, 519, 491, 515, 555, 515, 513, 501, 532, 495, 536, 502, 517, 487, 492, 536, 525, 454, 527, 516, 510, 473, 504, 532, 507, 524, 548, 523, 507, 509, 529, 569, 517, 519, 500, 482, 522, 467, 552, 518, 511, 478, 554, 538, 535, 493, 477, 533, 471, 491, 467, 514, 510, 489, 502, 492, 522, 535, 529, 537, 534] payload_rows_per_dst=[5590, 5584, 5488, 5530, 5624, 5612, 5492, 5530] dup_rows_per_dst=[2602, 2638, 2606, 2706, 2670, 2694, 2420, 2750] combine_rows_per_src=[5590, 5584, 5488, 5530, 5624, 5612, 5492, 5530] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44596 payload_rows_remote=39066 dup_rows=20940 dup_rows_remote=18330 dedup_ratio=0.319519 router_tokens_per_expert=[465, 494, 506, 493, 545, 522, 554, 522, 541, 510, 488, 504, 528, 540, 486, 543, 538, 513, 497, 532, 480, 517, 489, 527, 535, 517, 505, 514, 512, 476, 521, 490, 510, 491, 524, 497, 537, 524, 500, 512, 510, 495, 540, 502, 480, 521, 486, 528, 486, 527, 532, 489, 513, 534, 486, 511, 508, 503, 528, 525, 498, 506, 518, 543] payload_rows_per_dst=[5618, 5576, 5588, 5530, 5588, 5546, 5514, 5636] dup_rows_per_dst=[2584, 2704, 2598, 2610, 2602, 2578, 2642, 2622] combine_rows_per_src=[5618, 5576, 5588, 5530, 5588, 5546, 5514, 5636] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44462 payload_rows_remote=38858 dup_rows=21074 dup_rows_remote=18406 dedup_ratio=0.321564 router_tokens_per_expert=[547, 465, 528, 516, 534, 521, 545, 525, 499, 478, 511, 485, 499, 499, 512, 502, 536, 517, 519, 500, 464, 515, 550, 505, 498, 511, 528, 497, 509, 537, 543, 494, 522, 512, 501, 498, 516, 503, 516, 500, 516, 485, 548, 505, 531, 542, 549, 542, 509, 512, 557, 532, 504, 513, 503, 506, 492, 481, 517, 474, 493, 479, 524, 497] payload_rows_per_dst=[5646, 5428, 5562, 5592, 5484, 5688, 5604, 5458] dup_rows_per_dst=[2716, 2542, 2650, 2642, 2652, 2748, 2668, 2456] combine_rows_per_src=[5646, 5428, 5562, 5592, 5484, 5688, 5604, 5458] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44486 payload_rows_remote=38768 dup_rows=21050 dup_rows_remote=18332 dedup_ratio=0.321198 router_tokens_per_expert=[525, 530, 518, 510, 512, 508, 537, 578, 540, 495, 528, 473, 521, 507, 513, 524, 539, 500, 507, 539, 525, 487, 497, 514, 506, 507, 544, 496, 513, 501, 479, 504, 501, 496, 515, 497, 516, 486, 478, 495, 506, 450, 497, 524, 513, 517, 532, 499, 513, 525, 504, 550, 515, 493, 524, 555, 517, 534, 501, 498, 501, 511, 534, 494] payload_rows_per_dst=[5718, 5536, 5588, 5476, 5424, 5540, 5624, 5580] dup_rows_per_dst=[2718, 2666, 2628, 2624, 2544, 2536, 2734, 2600] combine_rows_per_src=[5718, 5536, 5588, 5476, 5424, 5540, 5624, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279293952 dispatch_meta_bytes=229128 combine_payload_bytes=279293952 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279523080 ingress_bytes=609595392 link_bytes=609595392 token_bytes=558817032 weight_movement_bytes=330301440 total_nvlink_bytes=889118472 weight_to_token_ratio=0.591073 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060186648e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277573632 dispatch_meta_bytes=228264 combine_payload_bytes=277573632 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277801896 ingress_bytes=387674112 link_bytes=387674112 token_bytes=555375528 weight_movement_bytes=110100480 total_nvlink_bytes=665476008 weight_to_token_ratio=0.198245 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.742290422e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44356 payload_rows_remote=38748 dup_rows=21180 dup_rows_remote=18592 dedup_ratio=0.323181 router_tokens_per_expert=[496, 497, 532, 508, 501, 503, 502, 527, 555, 523, 485, 475, 518, 487, 556, 499, 503, 527, 519, 522, 512, 499, 504, 496, 506, 527, 512, 515, 547, 511, 495, 537, 491, 529, 502, 505, 546, 523, 451, 554, 547, 504, 529, 534, 524, 480, 504, 540, 501, 537, 498, 524, 494, 497, 554, 500, 489, 517, 517, 485, 529, 494, 487, 486] payload_rows_per_dst=[5538, 5608, 5542, 5598, 5494, 5574, 5576, 5426] dup_rows_per_dst=[2594, 2588, 2622, 2702, 2708, 2750, 2634, 2582] combine_rows_per_src=[5538, 5608, 5542, 5598, 5494, 5574, 5576, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277745664 dispatch_meta_bytes=229360 combine_payload_bytes=277745664 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277975024 ingress_bytes=497946624 link_bytes=497946624 token_bytes=555720688 weight_movement_bytes=220200960 total_nvlink_bytes=775921648 weight_to_token_ratio=0.396244 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.660110773e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44548 payload_rows_remote=39014 dup_rows=20988 dup_rows_remote=18360 dedup_ratio=0.320251 router_tokens_per_expert=[524, 468, 540, 543, 526, 525, 519, 499, 483, 518, 509, 505, 523, 544, 530, 526, 514, 547, 511, 517, 489, 494, 533, 514, 521, 536, 527, 534, 526, 470, 532, 481, 519, 505, 496, 538, 535, 478, 522, 488, 523, 522, 530, 540, 478, 495, 507, 502, 486, 530, 512, 493, 492, 525, 488, 487, 513, 492, 483, 499, 484, 563, 499, 516] payload_rows_per_dst=[5552, 5626, 5636, 5636, 5534, 5562, 5466, 5536] dup_rows_per_dst=[2736, 2650, 2602, 2618, 2628, 2632, 2560, 2562] combine_rows_per_src=[5552, 5626, 5636, 5636, 5534, 5562, 5466, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44386 payload_rows_remote=38758 dup_rows=21150 dup_rows_remote=18390 dedup_ratio=0.322723 router_tokens_per_expert=[513, 525, 514, 509, 528, 542, 526, 526, 517, 488, 540, 515, 503, 537, 505, 526, 472, 507, 487, 543, 524, 505, 533, 495, 478, 477, 464, 515, 504, 535, 519, 500, 536, 520, 497, 505, 486, 517, 497, 523, 492, 505, 521, 514, 551, 532, 539, 523, 542, 491, 485, 523, 507, 468, 542, 496, 478, 548, 484, 533, 483, 483, 562, 513] payload_rows_per_dst=[5628, 5570, 5578, 5460, 5524, 5492, 5518, 5616] dup_rows_per_dst=[2760, 2676, 2676, 2524, 2638, 2740, 2584, 2552] combine_rows_per_src=[5628, 5570, 5578, 5460, 5524, 5492, 5518, 5616] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44596 payload_rows_remote=39080 dup_rows=20940 dup_rows_remote=18304 dedup_ratio=0.319519 router_tokens_per_expert=[527, 520, 533, 524, 518, 494, 534, 514, 486, 538, 551, 473, 496, 508, 533, 498, 485, 501, 524, 508, 485, 489, 519, 497, 518, 496, 521, 474, 501, 509, 529, 538, 545, 506, 511, 494, 536, 510, 499, 555, 524, 518, 520, 523, 498, 516, 485, 512, 509, 502, 534, 493, 455, 515, 553, 515, 499, 521, 527, 542, 522, 490, 523, 475] payload_rows_per_dst=[5604, 5612, 5472, 5594, 5614, 5612, 5516, 5572] dup_rows_per_dst=[2724, 2554, 2544, 2578, 2698, 2580, 2636, 2626] combine_rows_per_src=[5604, 5612, 5472, 5594, 5614, 5612, 5516, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44508 payload_rows_remote=38952 dup_rows=21028 dup_rows_remote=18278 dedup_ratio=0.320862 router_tokens_per_expert=[501, 530, 517, 500, 521, 474, 495, 497, 462, 554, 495, 520, 500, 517, 516, 502, 547, 513, 513, 492, 487, 502, 500, 539, 541, 523, 524, 504, 491, 536, 521, 502, 533, 506, 501, 499, 534, 489, 506, 474, 527, 517, 548, 528, 515, 480, 490, 548, 523, 508, 517, 559, 531, 531, 509, 540, 482, 496, 506, 527, 508, 501, 511, 488] payload_rows_per_dst=[5514, 5582, 5578, 5582, 5520, 5556, 5728, 5448] dup_rows_per_dst=[2556, 2550, 2608, 2702, 2564, 2750, 2708, 2590] combine_rows_per_src=[5514, 5582, 5578, 5582, 5520, 5556, 5728, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44586 payload_rows_remote=38938 dup_rows=20950 dup_rows_remote=18322 dedup_ratio=0.319672 router_tokens_per_expert=[517, 520, 540, 501, 487, 444, 526, 545, 525, 504, 535, 518, 504, 490, 496, 495, 510, 515, 513, 515, 540, 464, 522, 559, 469, 518, 473, 549, 539, 500, 473, 520, 487, 528, 530, 503, 538, 509, 521, 482, 490, 521, 514, 534, 568, 546, 493, 530, 510, 526, 505, 519, 534, 484, 513, 504, 459, 507, 518, 498, 528, 510, 525, 508] payload_rows_per_dst=[5574, 5558, 5648, 5510, 5500, 5644, 5574, 5578] dup_rows_per_dst=[2586, 2576, 2628, 2572, 2696, 2748, 2616, 2528] combine_rows_per_src=[5574, 5558, 5648, 5510, 5500, 5644, 5574, 5578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44706 payload_rows_remote=39156 dup_rows=20830 dup_rows_remote=18262 dedup_ratio=0.317841 router_tokens_per_expert=[522, 493, 494, 519, 548, 493, 511, 471, 513, 517, 520, 519, 483, 523, 500, 517, 523, 510, 515, 500, 565, 513, 504, 504, 490, 524, 513, 519, 486, 523, 498, 506, 520, 502, 529, 524, 510, 511, 478, 500, 481, 537, 509, 504, 514, 509, 516, 517, 522, 481, 543, 525, 503, 504, 542, 497, 510, 533, 527, 530, 552, 537, 482, 483] payload_rows_per_dst=[5576, 5576, 5616, 5550, 5558, 5630, 5588, 5612] dup_rows_per_dst=[2526, 2608, 2652, 2568, 2590, 2544, 2646, 2696] combine_rows_per_src=[5576, 5576, 5616, 5550, 5558, 5630, 5588, 5612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44394 payload_rows_remote=38710 dup_rows=21142 dup_rows_remote=18214 dedup_ratio=0.322601 router_tokens_per_expert=[511, 523, 539, 520, 468, 529, 557, 539, 525, 478, 539, 500, 511, 487, 550, 511, 512, 508, 508, 519, 506, 469, 517, 528, 542, 486, 487, 505, 509, 522, 490, 506, 499, 542, 518, 505, 527, 529, 482, 533, 504, 480, 499, 517, 526, 483, 505, 500, 496, 536, 507, 504, 498, 531, 525, 496, 516, 478, 519, 519, 518, 525, 531, 519] payload_rows_per_dst=[5516, 5592, 5580, 5594, 5538, 5330, 5560, 5684] dup_rows_per_dst=[2552, 2610, 2554, 2748, 2732, 2392, 2626, 2928] combine_rows_per_src=[5516, 5592, 5580, 5594, 5538, 5330, 5560, 5684] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277817344 dispatch_meta_bytes=228592 combine_payload_bytes=277817344 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278045936 ingress_bytes=608118784 link_bytes=608118784 token_bytes=555863280 weight_movement_bytes=330301440 total_nvlink_bytes=886164720 weight_to_token_ratio=0.594213 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057618584e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279652352 dispatch_meta_bytes=229496 combine_payload_bytes=279652352 prefetch_bytes=0 reduce_bytes=0 egress_bytes=279881848 ingress_bytes=279652352 link_bytes=279881848 token_bytes=559534200 weight_movement_bytes=0 total_nvlink_bytes=559534200 weight_to_token_ratio=0.000000 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=4.867605663e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=280125440 dispatch_meta_bytes=229536 combine_payload_bytes=280125440 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=280354976 ingress_bytes=390225920 link_bytes=390225920 token_bytes=560480416 weight_movement_bytes=110100480 total_nvlink_bytes=670580896 weight_to_token_ratio=0.196439 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.786670560e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279207936 dispatch_meta_bytes=228920 combine_payload_bytes=279207936 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279436856 ingress_bytes=609509376 link_bytes=609509376 token_bytes=558644792 weight_movement_bytes=330301440 total_nvlink_bytes=888946232 weight_to_token_ratio=0.591255 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060037052e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279107584 dispatch_meta_bytes=229040 combine_payload_bytes=279107584 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279336624 ingress_bytes=389208064 link_bytes=389208064 token_bytes=558444208 weight_movement_bytes=110100480 total_nvlink_bytes=668544688 weight_to_token_ratio=0.197156 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.768968370e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=280670208 dispatch_meta_bytes=229672 combine_payload_bytes=280670208 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280899880 ingress_bytes=390770688 link_bytes=390770688 token_bytes=561570088 weight_movement_bytes=110100480 total_nvlink_bytes=671670568 weight_to_token_ratio=0.196058 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.796144972e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277473280 dispatch_meta_bytes=227696 combine_payload_bytes=277473280 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277700976 ingress_bytes=387573760 link_bytes=387573760 token_bytes=555174256 weight_movement_bytes=110100480 total_nvlink_bytes=665274736 weight_to_token_ratio=0.198317 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.740545136e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44404 payload_rows_remote=38934 dup_rows=21132 dup_rows_remote=18636 dedup_ratio=0.322449 router_tokens_per_expert=[507, 501, 495, 488, 514, 519, 503, 546, 479, 514, 495, 495, 497, 516, 487, 500, 486, 503, 501, 540, 528, 535, 533, 521, 512, 498, 533, 500, 509, 515, 538, 495, 525, 525, 517, 532, 551, 520, 496, 518, 503, 503, 493, 522, 550, 518, 484, 506, 544, 458, 524, 527, 493, 520, 481, 491, 530, 513, 495, 532, 514, 520, 510, 550] payload_rows_per_dst=[5580, 5470, 5570, 5618, 5590, 5474, 5480, 5622] dup_rows_per_dst=[2566, 2496, 2724, 2582, 2778, 2684, 2596, 2706] combine_rows_per_src=[5580, 5470, 5570, 5618, 5590, 5474, 5480, 5622] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279078912 dispatch_meta_bytes=230280 combine_payload_bytes=279078912 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279309192 ingress_bytes=389179392 link_bytes=389179392 token_bytes=558388104 weight_movement_bytes=110100480 total_nvlink_bytes=668488584 weight_to_token_ratio=0.197176 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.768469717e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44258 payload_rows_remote=38576 dup_rows=21278 dup_rows_remote=18434 dedup_ratio=0.324677 router_tokens_per_expert=[513, 513, 495, 496, 562, 527, 513, 565, 496, 524, 504, 483, 503, 504, 462, 507, 555, 518, 500, 485, 492, 485, 516, 539, 479, 471, 538, 532, 542, 504, 511, 502, 493, 502, 505, 555, 513, 532, 550, 509, 522, 485, 529, 506, 486, 520, 533, 500, 509, 546, 515, 516, 518, 513, 522, 560, 519, 500, 502, 477, 473, 510, 485, 527] payload_rows_per_dst=[5682, 5738, 5592, 5496, 5582, 5570, 5212, 5386] dup_rows_per_dst=[2844, 3166, 2660, 2662, 2578, 2740, 2248, 2380] combine_rows_per_src=[5682, 5738, 5592, 5496, 5582, 5570, 5212, 5386] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44478 payload_rows_remote=38932 dup_rows=21058 dup_rows_remote=18486 dedup_ratio=0.321320 router_tokens_per_expert=[492, 475, 534, 494, 503, 521, 530, 494, 538, 541, 481, 507, 490, 475, 493, 503, 499, 492, 506, 517, 540, 494, 499, 551, 504, 483, 523, 539, 524, 510, 537, 555, 499, 510, 479, 501, 505, 478, 532, 555, 514, 506, 500, 535, 499, 453, 494, 514, 501, 534, 507, 490, 543, 532, 525, 536, 464, 536, 501, 544, 566, 548, 511, 512] payload_rows_per_dst=[5498, 5490, 5544, 5632, 5546, 5458, 5638, 5672] dup_rows_per_dst=[2588, 2566, 2652, 2718, 2572, 2572, 2698, 2692] combine_rows_per_src=[5498, 5490, 5544, 5632, 5546, 5458, 5638, 5672] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44462 payload_rows_remote=38882 dup_rows=21074 dup_rows_remote=18374 dedup_ratio=0.321564 router_tokens_per_expert=[499, 541, 478, 485, 505, 527, 482, 477, 509, 511, 509, 497, 530, 528, 513, 523, 453, 500, 512, 560, 532, 474, 538, 483, 522, 528, 482, 528, 523, 526, 451, 500, 545, 485, 504, 535, 517, 489, 524, 500, 514, 563, 540, 550, 495, 491, 524, 507, 536, 523, 523, 525, 477, 528, 514, 514, 532, 504, 461, 499, 530, 545, 523, 525] payload_rows_per_dst=[5440, 5524, 5546, 5600, 5540, 5638, 5580, 5594] dup_rows_per_dst=[2548, 2716, 2558, 2520, 2658, 2730, 2700, 2644] combine_rows_per_src=[5440, 5524, 5546, 5600, 5540, 5638, 5580, 5594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44492 payload_rows_remote=38936 dup_rows=21044 dup_rows_remote=18430 dedup_ratio=0.321106 router_tokens_per_expert=[516, 497, 494, 535, 532, 507, 505, 474, 484, 527, 509, 522, 491, 518, 445, 517, 557, 535, 530, 520, 535, 542, 517, 504, 535, 489, 514, 526, 518, 511, 538, 515, 529, 567, 548, 493, 520, 496, 507, 495, 509, 508, 523, 508, 541, 483, 522, 491, 487, 533, 496, 517, 513, 494, 486, 510, 518, 540, 492, 519, 514, 510, 454, 486] payload_rows_per_dst=[5536, 5506, 5658, 5616, 5568, 5556, 5492, 5560] dup_rows_per_dst=[2584, 2520, 2822, 2676, 2742, 2614, 2580, 2506] combine_rows_per_src=[5536, 5506, 5658, 5616, 5568, 5556, 5492, 5560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44352 payload_rows_remote=38752 dup_rows=21184 dup_rows_remote=18602 dedup_ratio=0.323242 router_tokens_per_expert=[530, 532, 533, 517, 514, 534, 470, 476, 533, 490, 510, 481, 474, 532, 514, 522, 522, 509, 510, 486, 487, 506, 526, 545, 512, 501, 523, 540, 485, 503, 526, 511, 497, 523, 501, 490, 514, 491, 496, 524, 496, 509, 506, 513, 493, 558, 506, 478, 532, 506, 544, 501, 525, 514, 524, 517, 535, 553, 523, 516, 470, 514, 531, 514] payload_rows_per_dst=[5546, 5552, 5600, 5526, 5500, 5456, 5596, 5576] dup_rows_per_dst=[2666, 2560, 2582, 2676, 2572, 2662, 2730, 2736] combine_rows_per_src=[5546, 5552, 5600, 5526, 5500, 5456, 5596, 5576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44568 payload_rows_remote=39034 dup_rows=20968 dup_rows_remote=18300 dedup_ratio=0.319946 router_tokens_per_expert=[509, 533, 525, 515, 530, 501, 538, 495, 524, 520, 485, 515, 523, 527, 490, 494, 524, 496, 500, 494, 522, 508, 506, 503, 510, 521, 485, 496, 514, 544, 513, 518, 501, 525, 499, 525, 528, 489, 515, 503, 513, 506, 487, 516, 510, 500, 551, 534, 492, 517, 536, 512, 512, 503, 552, 518, 515, 478, 524, 536, 478, 483, 489, 543] payload_rows_per_dst=[5608, 5536, 5582, 5534, 5580, 5604, 5622, 5502] dup_rows_per_dst=[2684, 2620, 2524, 2668, 2590, 2630, 2662, 2590] combine_rows_per_src=[5608, 5536, 5582, 5534, 5580, 5604, 5622, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44418 payload_rows_remote=38730 dup_rows=21118 dup_rows_remote=18436 dedup_ratio=0.322235 router_tokens_per_expert=[510, 490, 557, 540, 526, 488, 493, 479, 517, 513, 483, 483, 515, 520, 491, 516, 538, 497, 517, 490, 503, 529, 546, 510, 519, 491, 546, 518, 539, 524, 511, 502, 530, 509, 486, 530, 511, 497, 516, 507, 515, 472, 559, 527, 495, 508, 509, 489, 476, 531, 488, 502, 467, 505, 543, 526, 552, 463, 541, 498, 523, 535, 489, 568] payload_rows_per_dst=[5568, 5538, 5450, 5432, 5486, 5502, 5754, 5688] dup_rows_per_dst=[2598, 2538, 2472, 2580, 2654, 2646, 2948, 2682] combine_rows_per_src=[5568, 5538, 5450, 5432, 5486, 5502, 5754, 5688] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=276512768 dispatch_meta_bytes=228040 combine_payload_bytes=276512768 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276740808 ingress_bytes=386613248 link_bytes=386613248 token_bytes=553253576 weight_movement_bytes=110100480 total_nvlink_bytes=663354056 weight_to_token_ratio=0.199005 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.723840252e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279064576 dispatch_meta_bytes=229672 combine_payload_bytes=279064576 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279294248 ingress_bytes=499265536 link_bytes=499265536 token_bytes=558358824 weight_movement_bytes=220200960 total_nvlink_bytes=778559784 weight_to_token_ratio=0.394372 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.683048822e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278706176 dispatch_meta_bytes=229024 combine_payload_bytes=278706176 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278935200 ingress_bytes=498907136 link_bytes=498907136 token_bytes=557641376 weight_movement_bytes=220200960 total_nvlink_bytes=777842336 weight_to_token_ratio=0.394879 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.676815656e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279093248 dispatch_meta_bytes=229464 combine_payload_bytes=279093248 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279322712 ingress_bytes=389193728 link_bytes=389193728 token_bytes=558415960 weight_movement_bytes=110100480 total_nvlink_bytes=668516440 weight_to_token_ratio=0.197166 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.768719044e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277774336 dispatch_meta_bytes=229416 combine_payload_bytes=277774336 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278003752 ingress_bytes=497975296 link_bytes=497975296 token_bytes=555778088 weight_movement_bytes=220200960 total_nvlink_bytes=775979048 weight_to_token_ratio=0.396203 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.660609426e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44452 payload_rows_remote=38884 dup_rows=21084 dup_rows_remote=18448 dedup_ratio=0.321716 router_tokens_per_expert=[522, 502, 507, 476, 515, 487, 515, 550, 514, 511, 499, 530, 494, 536, 502, 516, 487, 491, 534, 526, 455, 527, 515, 511, 472, 503, 532, 508, 522, 548, 521, 506, 510, 531, 569, 517, 518, 500, 482, 524, 467, 553, 520, 512, 479, 557, 540, 536, 493, 481, 535, 472, 495, 468, 515, 510, 491, 502, 492, 526, 536, 529, 539, 535] payload_rows_per_dst=[5576, 5568, 5488, 5528, 5630, 5622, 5500, 5540] dup_rows_per_dst=[2572, 2636, 2604, 2696, 2672, 2706, 2438, 2760] combine_rows_per_src=[5576, 5568, 5488, 5528, 5630, 5622, 5500, 5540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279795712 dispatch_meta_bytes=229336 combine_payload_bytes=279795712 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280025048 ingress_bytes=499996672 link_bytes=499996672 token_bytes=559820760 weight_movement_bytes=220200960 total_nvlink_bytes=780021720 weight_to_token_ratio=0.393342 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.695764479e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277616640 dispatch_meta_bytes=228664 combine_payload_bytes=277616640 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=277845304 ingress_bytes=607918080 link_bytes=607918080 token_bytes=555461944 weight_movement_bytes=330301440 total_nvlink_bytes=885763384 weight_to_token_ratio=0.594643 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057269526e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279222272 dispatch_meta_bytes=230304 combine_payload_bytes=279222272 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279452576 ingress_bytes=389322752 link_bytes=389322752 token_bytes=558674848 weight_movement_bytes=110100480 total_nvlink_bytes=668775328 weight_to_token_ratio=0.197074 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.770962983e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278505472 dispatch_meta_bytes=228720 combine_payload_bytes=278505472 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278734192 ingress_bytes=388605952 link_bytes=388605952 token_bytes=557239664 weight_movement_bytes=110100480 total_nvlink_bytes=667340144 weight_to_token_ratio=0.197582 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.758496652e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278591488 dispatch_meta_bytes=229256 combine_payload_bytes=278591488 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278820744 ingress_bytes=388691968 link_bytes=388691968 token_bytes=557412232 weight_movement_bytes=110100480 total_nvlink_bytes=667512712 weight_to_token_ratio=0.197521 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.759992612e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=280025088 dispatch_meta_bytes=229584 combine_payload_bytes=280025088 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280254672 ingress_bytes=390125568 link_bytes=390125568 token_bytes=560279760 weight_movement_bytes=110100480 total_nvlink_bytes=670380240 weight_to_token_ratio=0.196510 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.784925274e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278534144 dispatch_meta_bytes=229056 combine_payload_bytes=278534144 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278763200 ingress_bytes=388634624 link_bytes=388634624 token_bytes=557297344 weight_movement_bytes=110100480 total_nvlink_bytes=667397824 weight_to_token_ratio=0.197561 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.758995305e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44202 payload_rows_remote=38712 dup_rows=21334 dup_rows_remote=18618 dedup_ratio=0.325531 router_tokens_per_expert=[517, 527, 503, 570, 470, 509, 529, 540, 517, 524, 507, 514, 473, 508, 501, 520, 489, 504, 544, 537, 503, 475, 494, 501, 503, 514, 480, 514, 489, 560, 528, 542, 522, 533, 500, 523, 536, 478, 522, 533, 533, 512, 528, 512, 486, 492, 505, 504, 485, 508, 515, 512, 513, 468, 519, 524, 507, 463, 510, 524, 520, 521, 524, 530] payload_rows_per_dst=[5220, 5462, 5478, 5568, 5576, 5628, 5780, 5490] dup_rows_per_dst=[2410, 2380, 2616, 2692, 2710, 2802, 3008, 2716] combine_rows_per_src=[5220, 5462, 5478, 5568, 5576, 5628, 5780, 5490] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44486 payload_rows_remote=38966 dup_rows=21050 dup_rows_remote=18368 dedup_ratio=0.321198 router_tokens_per_expert=[513, 518, 516, 536, 533, 537, 498, 537, 507, 496, 519, 482, 474, 498, 527, 510, 521, 506, 513, 515, 534, 488, 503, 521, 545, 532, 483, 493, 513, 495, 533, 497, 465, 486, 515, 505, 527, 529, 504, 483, 519, 472, 490, 497, 516, 543, 523, 497, 486, 514, 477, 515, 530, 533, 516, 534, 529, 558, 490, 526, 543, 496, 514, 543] payload_rows_per_dst=[5730, 5446, 5520, 5532, 5492, 5526, 5572, 5668] dup_rows_per_dst=[2646, 2580, 2682, 2650, 2536, 2588, 2638, 2730] combine_rows_per_src=[5730, 5446, 5520, 5532, 5492, 5526, 5572, 5668] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277889024 dispatch_meta_bytes=228400 combine_payload_bytes=277889024 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=278117424 ingress_bytes=828391424 link_bytes=828391424 token_bytes=556006448 weight_movement_bytes=550502400 total_nvlink_bytes=1106508848 weight_to_token_ratio=0.990101 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.440708933e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277487616 dispatch_meta_bytes=229320 combine_payload_bytes=277487616 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=277716936 ingress_bytes=717889536 link_bytes=717889536 token_bytes=555204552 weight_movement_bytes=440401920 total_nvlink_bytes=995606472 weight_to_token_ratio=0.793225 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.248527976e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279308288 dispatch_meta_bytes=229336 combine_payload_bytes=279308288 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279537624 ingress_bytes=389408768 link_bytes=389408768 token_bytes=558845912 weight_movement_bytes=110100480 total_nvlink_bytes=668946392 weight_to_token_ratio=0.197014 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.772458943e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44264 payload_rows_remote=38746 dup_rows=21272 dup_rows_remote=18578 dedup_ratio=0.324585 router_tokens_per_expert=[501, 517, 517, 532, 478, 487, 525, 485, 490, 509, 479, 541, 504, 516, 473, 525, 517, 516, 516, 527, 574, 487, 533, 519, 509, 513, 505, 539, 511, 499, 488, 496, 524, 498, 507, 520, 541, 523, 511, 504, 501, 463, 498, 519, 547, 507, 540, 531, 499, 524, 528, 511, 500, 517, 552, 491, 518, 488, 516, 530, 524, 480, 518, 510] payload_rows_per_dst=[5494, 5524, 5528, 5504, 5562, 5518, 5606, 5528] dup_rows_per_dst=[2590, 2550, 2850, 2616, 2694, 2694, 2638, 2640] combine_rows_per_src=[5494, 5524, 5528, 5504, 5562, 5518, 5606, 5528] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44364 payload_rows_remote=38830 dup_rows=21172 dup_rows_remote=18486 dedup_ratio=0.323059 router_tokens_per_expert=[515, 537, 489, 493, 533, 494, 511, 525, 547, 533, 521, 517, 504, 523, 492, 547, 547, 487, 536, 496, 495, 528, 514, 459, 527, 511, 468, 497, 536, 514, 487, 557, 498, 516, 513, 517, 545, 531, 503, 487, 515, 501, 522, 471, 514, 500, 487, 514, 503, 513, 543, 546, 515, 496, 493, 530, 543, 498, 517, 506, 486, 513, 486, 506] payload_rows_per_dst=[5500, 5582, 5556, 5598, 5534, 5470, 5610, 5514] dup_rows_per_dst=[2694, 2786, 2568, 2596, 2686, 2578, 2668, 2596] combine_rows_per_src=[5500, 5582, 5556, 5598, 5534, 5470, 5610, 5514] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44632 payload_rows_remote=39042 dup_rows=20904 dup_rows_remote=18320 dedup_ratio=0.318970 router_tokens_per_expert=[517, 522, 533, 504, 499, 546, 491, 511, 536, 512, 495, 522, 503, 514, 494, 511, 485, 543, 533, 488, 521, 513, 531, 479, 516, 537, 505, 495, 516, 504, 558, 514, 538, 485, 487, 483, 479, 515, 548, 500, 521, 543, 528, 511, 513, 506, 505, 505, 447, 525, 501, 520, 511, 505, 504, 555, 496, 525, 514, 528, 507, 524, 480, 511] payload_rows_per_dst=[5646, 5590, 5626, 5614, 5574, 5588, 5518, 5476] dup_rows_per_dst=[2600, 2584, 2560, 2676, 2496, 2676, 2618, 2694] combine_rows_per_src=[5646, 5590, 5626, 5614, 5574, 5588, 5518, 5476] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44432 payload_rows_remote=38850 dup_rows=21104 dup_rows_remote=18332 dedup_ratio=0.322021 router_tokens_per_expert=[531, 533, 514, 493, 522, 558, 505, 536, 519, 513, 533, 510, 518, 495, 515, 482, 496, 542, 530, 530, 479, 486, 517, 536, 533, 486, 506, 526, 526, 557, 529, 514, 518, 497, 523, 527, 515, 518, 501, 493, 490, 514, 548, 463, 513, 522, 462, 518, 507, 532, 511, 487, 471, 495, 455, 502, 518, 494, 500, 524, 504, 525, 508, 543] payload_rows_per_dst=[5678, 5514, 5566, 5582, 5552, 5510, 5444, 5586] dup_rows_per_dst=[2706, 2656, 2666, 2772, 2632, 2550, 2476, 2646] combine_rows_per_src=[5678, 5514, 5566, 5582, 5552, 5510, 5444, 5586] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44690 payload_rows_remote=39134 dup_rows=20846 dup_rows_remote=18402 dedup_ratio=0.318085 router_tokens_per_expert=[482, 532, 547, 483, 524, 529, 529, 536, 535, 508, 495, 534, 549, 537, 525, 517, 496, 484, 502, 514, 474, 513, 511, 517, 488, 538, 514, 511, 510, 500, 491, 542, 520, 508, 517, 514, 516, 541, 498, 539, 530, 516, 467, 488, 531, 491, 499, 517, 538, 510, 474, 525, 479, 512, 479, 483, 518, 499, 543, 500, 548, 522, 495, 484] payload_rows_per_dst=[5652, 5646, 5550, 5594, 5652, 5506, 5556, 5534] dup_rows_per_dst=[2672, 2754, 2472, 2594, 2654, 2572, 2444, 2684] combine_rows_per_src=[5652, 5646, 5550, 5594, 5652, 5506, 5556, 5534] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44350 payload_rows_remote=38756 dup_rows=21186 dup_rows_remote=18482 dedup_ratio=0.323273 router_tokens_per_expert=[515, 525, 497, 542, 523, 528, 466, 481, 520, 511, 519, 507, 539, 549, 529, 552, 501, 513, 487, 504, 499, 520, 504, 499, 482, 544, 461, 509, 539, 503, 520, 522, 503, 493, 468, 503, 533, 527, 529, 537, 506, 535, 523, 512, 536, 526, 483, 544, 483, 510, 519, 530, 505, 448, 532, 453, 519, 494, 500, 529, 497, 515, 545, 521] payload_rows_per_dst=[5594, 5672, 5620, 5392, 5540, 5632, 5476, 5424] dup_rows_per_dst=[2704, 2810, 2678, 2556, 2646, 2698, 2484, 2610] combine_rows_per_src=[5594, 5672, 5620, 5392, 5540, 5632, 5476, 5424] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277731328 dispatch_meta_bytes=229296 combine_payload_bytes=277731328 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277960624 ingress_bytes=387831808 link_bytes=387831808 token_bytes=555691952 weight_movement_bytes=110100480 total_nvlink_bytes=665792432 weight_to_token_ratio=0.198132 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.745033015e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278333440 dispatch_meta_bytes=229264 combine_payload_bytes=278333440 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=278562704 ingress_bytes=388433920 link_bytes=388433920 token_bytes=556896144 weight_movement_bytes=110100480 total_nvlink_bytes=666996624 weight_to_token_ratio=0.197704 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.755504733e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279853056 dispatch_meta_bytes=229448 combine_payload_bytes=279853056 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280082504 ingress_bytes=500054016 link_bytes=500054016 token_bytes=559935560 weight_movement_bytes=220200960 total_nvlink_bytes=780136520 weight_to_token_ratio=0.393261 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.696761786e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278476800 dispatch_meta_bytes=228728 combine_payload_bytes=278476800 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278705528 ingress_bytes=498677760 link_bytes=498677760 token_bytes=557182328 weight_movement_bytes=220200960 total_nvlink_bytes=777383288 weight_to_token_ratio=0.395204 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.672826430e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44442 payload_rows_remote=38904 dup_rows=21094 dup_rows_remote=18428 dedup_ratio=0.321869 router_tokens_per_expert=[508, 508, 545, 477, 543, 535, 495, 500, 497, 512, 519, 520, 502, 503, 482, 515, 499, 553, 532, 503, 473, 471, 534, 537, 479, 514, 494, 521, 457, 494, 529, 493, 557, 513, 497, 562, 542, 480, 469, 538, 553, 531, 567, 553, 517, 502, 496, 523, 545, 499, 498, 492, 501, 496, 490, 547, 528, 511, 481, 508, 519, 497, 472, 540] payload_rows_per_dst=[5548, 5440, 5538, 5478, 5662, 5654, 5560, 5562] dup_rows_per_dst=[2674, 2660, 2666, 2484, 2654, 2830, 2576, 2550] combine_rows_per_src=[5548, 5440, 5538, 5478, 5662, 5654, 5560, 5562] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280512512 dispatch_meta_bytes=230144 combine_payload_bytes=280512512 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280742656 ingress_bytes=390612992 link_bytes=390612992 token_bytes=561255168 weight_movement_bytes=110100480 total_nvlink_bytes=671355648 weight_to_token_ratio=0.196168 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.793402379e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44462 payload_rows_remote=38770 dup_rows=21074 dup_rows_remote=18064 dedup_ratio=0.321564 router_tokens_per_expert=[487, 527, 522, 488, 509, 508, 497, 508, 513, 509, 525, 543, 484, 557, 527, 500, 498, 490, 494, 506, 516, 506, 502, 454, 527, 534, 503, 531, 508, 541, 528, 529, 498, 515, 541, 518, 521, 489, 516, 507, 508, 489, 528, 502, 502, 508, 499, 522, 519, 492, 516, 503, 493, 502, 525, 522, 480, 531, 520, 525, 516, 545, 532, 513] payload_rows_per_dst=[5440, 5428, 5602, 5602, 5460, 5562, 5676, 5692] dup_rows_per_dst=[2274, 2488, 2730, 2732, 2396, 2622, 2822, 3010] combine_rows_per_src=[5440, 5428, 5602, 5602, 5460, 5562, 5676, 5692] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277803008 dispatch_meta_bytes=228952 combine_payload_bytes=277803008 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278031960 ingress_bytes=498003968 link_bytes=498003968 token_bytes=555834968 weight_movement_bytes=220200960 total_nvlink_bytes=776035928 weight_to_token_ratio=0.396162 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.661108079e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278863872 dispatch_meta_bytes=229328 combine_payload_bytes=278863872 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279093200 ingress_bytes=388964352 link_bytes=388964352 token_bytes=557957072 weight_movement_bytes=110100480 total_nvlink_bytes=668057552 weight_to_token_ratio=0.197328 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.764729818e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277903360 dispatch_meta_bytes=227336 combine_payload_bytes=277903360 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278130696 ingress_bytes=388003840 link_bytes=388003840 token_bytes=556034056 weight_movement_bytes=110100480 total_nvlink_bytes=666134536 weight_to_token_ratio=0.198010 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.748024934e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44856 payload_rows_remote=39234 dup_rows=20680 dup_rows_remote=17996 dedup_ratio=0.315552 router_tokens_per_expert=[517, 541, 505, 561, 513, 535, 500, 542, 545, 509, 503, 528, 543, 522, 506, 486, 516, 494, 514, 487, 467, 513, 511, 512, 500, 492, 519, 539, 537, 538, 492, 482, 511, 490, 548, 524, 484, 519, 539, 479, 514, 502, 535, 518, 504, 530, 488, 562, 543, 471, 519, 486, 506, 494, 516, 499, 492, 479, 519, 480, 488, 514, 524, 522] payload_rows_per_dst=[5634, 5612, 5628, 5656, 5668, 5622, 5516, 5520] dup_rows_per_dst=[2794, 2672, 2400, 2542, 2520, 2684, 2552, 2516] combine_rows_per_src=[5634, 5612, 5628, 5656, 5668, 5622, 5516, 5520] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44378 payload_rows_remote=38840 dup_rows=21158 dup_rows_remote=18552 dedup_ratio=0.322845 router_tokens_per_expert=[474, 504, 530, 502, 493, 510, 502, 520, 521, 481, 504, 562, 548, 544, 509, 507, 514, 486, 513, 538, 510, 542, 515, 515, 500, 526, 507, 543, 487, 502, 523, 509, 515, 489, 471, 497, 522, 529, 534, 515, 516, 558, 511, 496, 492, 539, 490, 491, 532, 534, 503, 536, 513, 487, 529, 535, 491, 505, 507, 490, 485, 534, 507, 474] payload_rows_per_dst=[5450, 5642, 5560, 5586, 5538, 5540, 5588, 5474] dup_rows_per_dst=[2620, 2710, 2706, 2608, 2606, 2646, 2750, 2512] combine_rows_per_src=[5450, 5642, 5560, 5586, 5538, 5540, 5588, 5474] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44542 payload_rows_remote=38964 dup_rows=20994 dup_rows_remote=18302 dedup_ratio=0.320343 router_tokens_per_expert=[497, 503, 507, 511, 531, 511, 515, 496, 517, 510, 516, 548, 519, 521, 507, 497, 520, 525, 516, 528, 526, 515, 527, 516, 573, 549, 534, 533, 527, 512, 496, 511, 498, 514, 493, 504, 539, 483, 487, 501, 482, 530, 538, 498, 519, 483, 466, 515, 504, 492, 512, 491, 514, 495, 519, 543, 506, 513, 465, 555, 536, 496, 476, 487] payload_rows_per_dst=[5580, 5578, 5628, 5698, 5476, 5602, 5508, 5472] dup_rows_per_dst=[2562, 2692, 2718, 2772, 2562, 2460, 2632, 2596] combine_rows_per_src=[5580, 5578, 5628, 5698, 5476, 5602, 5508, 5472] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44598 payload_rows_remote=39048 dup_rows=20938 dup_rows_remote=18364 dedup_ratio=0.319489 router_tokens_per_expert=[521, 559, 528, 513, 530, 514, 480, 491, 548, 515, 529, 556, 530, 544, 495, 526, 488, 490, 551, 515, 509, 501, 492, 494, 503, 539, 509, 517, 478, 523, 493, 500, 497, 562, 507, 517, 505, 553, 488, 473, 517, 525, 496, 507, 531, 485, 514, 466, 494, 492, 499, 490, 533, 486, 522, 530, 508, 564, 517, 495, 521, 479, 506, 508] payload_rows_per_dst=[5558, 5684, 5510, 5550, 5610, 5600, 5542, 5544] dup_rows_per_dst=[2714, 2802, 2570, 2574, 2594, 2482, 2550, 2652] combine_rows_per_src=[5558, 5684, 5510, 5550, 5610, 5600, 5542, 5544] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44518 payload_rows_remote=38938 dup_rows=21018 dup_rows_remote=18406 dedup_ratio=0.320709 router_tokens_per_expert=[499, 543, 500, 520, 514, 510, 490, 502, 518, 488, 505, 511, 481, 539, 487, 494, 500, 534, 527, 546, 528, 531, 485, 473, 517, 517, 506, 479, 493, 513, 486, 542, 520, 525, 517, 514, 543, 519, 521, 529, 502, 488, 535, 516, 495, 501, 499, 533, 513, 534, 532, 513, 472, 496, 514, 522, 537, 476, 539, 506, 530, 548, 507, 494] payload_rows_per_dst=[5610, 5546, 5540, 5508, 5642, 5544, 5580, 5548] dup_rows_per_dst=[2546, 2500, 2708, 2598, 2734, 2594, 2612, 2726] combine_rows_per_src=[5610, 5546, 5540, 5508, 5642, 5544, 5580, 5548] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44532 payload_rows_remote=38908 dup_rows=21004 dup_rows_remote=18096 dedup_ratio=0.320496 router_tokens_per_expert=[523, 529, 488, 545, 532, 512, 528, 517, 503, 537, 454, 548, 537, 508, 524, 510, 473, 526, 515, 485, 496, 513, 499, 509, 492, 519, 513, 508, 516, 505, 506, 527, 521, 538, 517, 503, 462, 521, 539, 523, 490, 529, 513, 539, 494, 522, 502, 541, 506, 474, 501, 581, 508, 513, 475, 478, 492, 497, 526, 492, 531, 536, 485, 522] payload_rows_per_dst=[5624, 5516, 5610, 5606, 5648, 5528, 5462, 5538] dup_rows_per_dst=[2908, 2566, 2422, 2542, 2766, 2566, 2610, 2624] combine_rows_per_src=[5624, 5516, 5610, 5606, 5648, 5528, 5462, 5538] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=281229312 dispatch_meta_bytes=228920 combine_payload_bytes=281229312 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=281458232 ingress_bytes=501430272 link_bytes=501430272 token_bytes=562687544 weight_movement_bytes=220200960 total_nvlink_bytes=782888504 weight_to_token_ratio=0.391338 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.720697141e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278405120 dispatch_meta_bytes=229568 combine_payload_bytes=278405120 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278634688 ingress_bytes=498606080 link_bytes=498606080 token_bytes=557039808 weight_movement_bytes=220200960 total_nvlink_bytes=777240768 weight_to_token_ratio=0.395306 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.671579797e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278720512 dispatch_meta_bytes=229328 combine_payload_bytes=278720512 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278949840 ingress_bytes=388820992 link_bytes=388820992 token_bytes=557670352 weight_movement_bytes=110100480 total_nvlink_bytes=667770832 weight_to_token_ratio=0.197429 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.762236552e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44450 payload_rows_remote=38746 dup_rows=21086 dup_rows_remote=18398 dedup_ratio=0.321747 router_tokens_per_expert=[525, 525, 516, 505, 509, 506, 532, 577, 538, 495, 526, 473, 522, 503, 511, 520, 535, 499, 505, 538, 525, 488, 495, 515, 503, 503, 545, 495, 512, 500, 477, 504, 501, 496, 515, 500, 516, 491, 477, 497, 504, 452, 500, 526, 515, 516, 534, 500, 513, 529, 507, 556, 517, 495, 524, 556, 521, 534, 504, 503, 500, 513, 537, 497] payload_rows_per_dst=[5704, 5530, 5588, 5512, 5434, 5594, 5634, 5454] dup_rows_per_dst=[2688, 2646, 2642, 2684, 2552, 2626, 2758, 2490] combine_rows_per_src=[5704, 5530, 5588, 5512, 5434, 5594, 5634, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44492 payload_rows_remote=38874 dup_rows=21044 dup_rows_remote=18316 dedup_ratio=0.321106 router_tokens_per_expert=[539, 513, 507, 509, 502, 531, 496, 506, 512, 518, 501, 492, 504, 501, 500, 494, 512, 529, 478, 521, 553, 484, 515, 502, 491, 539, 482, 494, 544, 483, 540, 531, 530, 512, 520, 498, 502, 535, 542, 534, 524, 512, 536, 484, 507, 523, 502, 463, 520, 531, 498, 518, 529, 511, 533, 526, 483, 512, 515, 505, 497, 519, 512, 512] payload_rows_per_dst=[5626, 5556, 5578, 5524, 5618, 5486, 5570, 5534] dup_rows_per_dst=[2580, 2488, 2610, 2684, 2728, 2616, 2762, 2576] combine_rows_per_src=[5626, 5556, 5578, 5524, 5618, 5486, 5570, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44434 payload_rows_remote=38828 dup_rows=21102 dup_rows_remote=18408 dedup_ratio=0.321991 router_tokens_per_expert=[546, 463, 525, 513, 528, 516, 542, 522, 495, 477, 510, 485, 498, 497, 511, 500, 532, 515, 518, 499, 462, 514, 550, 503, 497, 510, 529, 497, 511, 536, 537, 494, 523, 510, 502, 499, 518, 503, 518, 501, 516, 487, 550, 506, 532, 544, 550, 542, 510, 514, 560, 531, 508, 514, 505, 508, 497, 483, 519, 480, 498, 482, 528, 498] payload_rows_per_dst=[5620, 5414, 5552, 5582, 5490, 5692, 5606, 5478] dup_rows_per_dst=[2690, 2532, 2634, 2640, 2658, 2762, 2694, 2492] combine_rows_per_src=[5620, 5414, 5552, 5582, 5490, 5692, 5606, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44420 payload_rows_remote=38946 dup_rows=21116 dup_rows_remote=18604 dedup_ratio=0.322205 router_tokens_per_expert=[492, 528, 517, 462, 512, 549, 520, 488, 508, 542, 504, 508, 500, 499, 487, 531, 550, 497, 493, 516, 549, 511, 524, 528, 484, 527, 494, 486, 511, 500, 493, 536, 505, 497, 518, 503, 539, 480, 523, 522, 512, 514, 503, 493, 477, 512, 493, 489, 489, 505, 529, 530, 499, 482, 498, 535, 493, 536, 524, 556, 556, 571, 520, 519] payload_rows_per_dst=[5498, 5488, 5572, 5564, 5564, 5474, 5574, 5686] dup_rows_per_dst=[2638, 2670, 2764, 2498, 2610, 2512, 2560, 2864] combine_rows_per_src=[5498, 5488, 5572, 5564, 5564, 5474, 5574, 5686] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44550 payload_rows_remote=38896 dup_rows=20986 dup_rows_remote=18304 dedup_ratio=0.320221 router_tokens_per_expert=[513, 532, 521, 527, 555, 513, 502, 503, 522, 490, 521, 497, 508, 517, 494, 519, 537, 533, 541, 511, 495, 513, 534, 504, 493, 504, 518, 515, 528, 475, 523, 533, 509, 422, 500, 502, 511, 491, 555, 516, 497, 562, 489, 529, 534, 499, 531, 530, 491, 469, 522, 545, 535, 499, 499, 492, 508, 497, 522, 482, 493, 526, 513, 507] payload_rows_per_dst=[5598, 5534, 5654, 5572, 5504, 5652, 5488, 5548] dup_rows_per_dst=[2734, 2602, 2682, 2606, 2508, 2690, 2616, 2548] combine_rows_per_src=[5598, 5534, 5654, 5572, 5504, 5652, 5488, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277731328 dispatch_meta_bytes=228576 combine_payload_bytes=277731328 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277959904 ingress_bytes=497932288 link_bytes=497932288 token_bytes=555691232 weight_movement_bytes=220200960 total_nvlink_bytes=775892192 weight_to_token_ratio=0.396265 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.659861446e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44588 payload_rows_remote=39066 dup_rows=20948 dup_rows_remote=18338 dedup_ratio=0.319641 router_tokens_per_expert=[463, 490, 504, 490, 544, 518, 552, 521, 537, 507, 487, 501, 528, 538, 482, 541, 537, 514, 496, 531, 481, 517, 485, 525, 531, 519, 506, 514, 512, 475, 520, 489, 510, 491, 523, 496, 541, 522, 501, 513, 510, 498, 538, 506, 480, 522, 486, 531, 487, 529, 532, 490, 516, 539, 488, 513, 510, 506, 530, 528, 500, 512, 519, 546] payload_rows_per_dst=[5600, 5564, 5582, 5522, 5594, 5544, 5530, 5652] dup_rows_per_dst=[2564, 2678, 2590, 2610, 2600, 2598, 2658, 2650] combine_rows_per_src=[5600, 5564, 5582, 5522, 5594, 5544, 5530, 5652] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44596 payload_rows_remote=39006 dup_rows=20940 dup_rows_remote=18266 dedup_ratio=0.319519 router_tokens_per_expert=[456, 523, 519, 515, 520, 544, 513, 490, 504, 531, 513, 521, 518, 513, 549, 510, 533, 511, 489, 492, 512, 494, 482, 514, 488, 544, 518, 519, 529, 532, 522, 505, 520, 553, 529, 534, 515, 492, 527, 488, 494, 504, 487, 490, 536, 517, 498, 455, 545, 507, 483, 482, 539, 480, 536, 508, 497, 551, 492, 548, 540, 464, 525, 509] payload_rows_per_dst=[5476, 5764, 5512, 5582, 5620, 5502, 5550, 5590] dup_rows_per_dst=[2372, 2866, 2484, 2732, 2754, 2460, 2598, 2674] combine_rows_per_src=[5476, 5764, 5512, 5582, 5620, 5502, 5550, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278648832 dispatch_meta_bytes=228760 combine_payload_bytes=278648832 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278877592 ingress_bytes=388749312 link_bytes=388749312 token_bytes=557526424 weight_movement_bytes=110100480 total_nvlink_bytes=667626904 weight_to_token_ratio=0.197480 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.760989918e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278319104 dispatch_meta_bytes=228944 combine_payload_bytes=278319104 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=278548048 ingress_bytes=498520064 link_bytes=498520064 token_bytes=556867152 weight_movement_bytes=220200960 total_nvlink_bytes=777068112 weight_to_token_ratio=0.395428 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.670083838e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279164928 dispatch_meta_bytes=230200 combine_payload_bytes=279164928 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279395128 ingress_bytes=389265408 link_bytes=389265408 token_bytes=558560056 weight_movement_bytes=110100480 total_nvlink_bytes=668660536 weight_to_token_ratio=0.197115 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.769965677e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44642 payload_rows_remote=39060 dup_rows=20894 dup_rows_remote=18328 dedup_ratio=0.318817 router_tokens_per_expert=[514, 522, 529, 501, 494, 541, 488, 510, 534, 507, 494, 521, 501, 514, 493, 510, 483, 543, 530, 489, 517, 509, 529, 476, 517, 536, 504, 497, 517, 502, 560, 514, 538, 483, 488, 481, 477, 515, 549, 499, 522, 545, 528, 513, 516, 510, 508, 506, 449, 527, 503, 524, 513, 508, 507, 557, 498, 531, 519, 530, 508, 525, 484, 511] payload_rows_per_dst=[5622, 5582, 5612, 5610, 5570, 5608, 5540, 5498] dup_rows_per_dst=[2576, 2566, 2540, 2684, 2490, 2688, 2636, 2714] combine_rows_per_src=[5622, 5582, 5612, 5610, 5570, 5608, 5540, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278806528 dispatch_meta_bytes=228800 combine_payload_bytes=278806528 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279035328 ingress_bytes=499007488 link_bytes=499007488 token_bytes=557841856 weight_movement_bytes=220200960 total_nvlink_bytes=778042816 weight_to_token_ratio=0.394737 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.678560943e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=280025088 dispatch_meta_bytes=229616 combine_payload_bytes=280025088 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280254704 ingress_bytes=390125568 link_bytes=390125568 token_bytes=560279792 weight_movement_bytes=110100480 total_nvlink_bytes=670380272 weight_to_token_ratio=0.196510 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.784925274e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279595008 dispatch_meta_bytes=229088 combine_payload_bytes=279595008 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=279824096 ingress_bytes=719996928 link_bytes=719996928 token_bytes=559419104 weight_movement_bytes=440401920 total_nvlink_bytes=999821024 weight_to_token_ratio=0.787249 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.252193077e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279982080 dispatch_meta_bytes=229552 combine_payload_bytes=279982080 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280211632 ingress_bytes=390082560 link_bytes=390082560 token_bytes=560193712 weight_movement_bytes=110100480 total_nvlink_bytes=670294192 weight_to_token_ratio=0.196540 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.784177294e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44354 payload_rows_remote=38764 dup_rows=21182 dup_rows_remote=18486 dedup_ratio=0.323212 router_tokens_per_expert=[512, 525, 492, 540, 520, 524, 464, 478, 519, 508, 517, 506, 536, 549, 525, 548, 502, 512, 486, 501, 496, 516, 502, 500, 483, 539, 463, 510, 538, 502, 523, 520, 503, 495, 467, 507, 532, 529, 531, 539, 503, 536, 525, 514, 538, 528, 483, 544, 485, 513, 518, 532, 509, 450, 534, 457, 520, 497, 502, 533, 498, 519, 548, 523] payload_rows_per_dst=[5590, 5646, 5518, 5402, 5540, 5686, 5654, 5318] dup_rows_per_dst=[2696, 2778, 2512, 2594, 2642, 2796, 2772, 2392] combine_rows_per_src=[5590, 5646, 5518, 5402, 5540, 5686, 5654, 5318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44378 payload_rows_remote=38840 dup_rows=21158 dup_rows_remote=18462 dedup_ratio=0.322845 router_tokens_per_expert=[509, 533, 488, 493, 528, 492, 510, 524, 544, 529, 515, 513, 501, 520, 487, 546, 547, 483, 537, 494, 502, 529, 510, 457, 520, 510, 467, 493, 537, 515, 489, 558, 497, 516, 513, 522, 542, 531, 505, 491, 516, 499, 528, 471, 515, 507, 488, 514, 503, 516, 548, 549, 516, 500, 497, 531, 544, 498, 519, 509, 490, 514, 488, 511] payload_rows_per_dst=[5486, 5558, 5552, 5588, 5538, 5486, 5626, 5544] dup_rows_per_dst=[2668, 2752, 2566, 2590, 2696, 2590, 2694, 2602] combine_rows_per_src=[5486, 5558, 5552, 5588, 5538, 5486, 5626, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44262 payload_rows_remote=38734 dup_rows=21274 dup_rows_remote=18564 dedup_ratio=0.324615 router_tokens_per_expert=[496, 512, 516, 531, 476, 486, 523, 483, 485, 503, 478, 538, 505, 515, 475, 524, 515, 514, 513, 529, 573, 484, 529, 520, 510, 513, 504, 540, 511, 500, 488, 496, 521, 500, 506, 524, 539, 525, 509, 502, 500, 464, 503, 521, 546, 510, 543, 532, 502, 526, 529, 513, 501, 520, 555, 495, 520, 488, 520, 531, 523, 482, 520, 513] payload_rows_per_dst=[5476, 5512, 5518, 5508, 5566, 5528, 5618, 5536] dup_rows_per_dst=[2570, 2534, 2836, 2616, 2686, 2710, 2664, 2658] combine_rows_per_src=[5476, 5512, 5518, 5508, 5566, 5528, 5618, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44676 payload_rows_remote=39106 dup_rows=20860 dup_rows_remote=18394 dedup_ratio=0.318298 router_tokens_per_expert=[481, 529, 546, 481, 521, 527, 526, 534, 533, 506, 494, 530, 548, 538, 525, 516, 493, 485, 501, 511, 471, 510, 511, 518, 487, 540, 513, 513, 509, 499, 490, 540, 523, 508, 517, 514, 516, 542, 497, 541, 529, 516, 469, 488, 531, 491, 500, 517, 539, 512, 474, 528, 481, 516, 484, 484, 520, 500, 546, 503, 551, 523, 496, 486] payload_rows_per_dst=[5642, 5630, 5538, 5590, 5648, 5508, 5570, 5550] dup_rows_per_dst=[2648, 2750, 2462, 2592, 2668, 2574, 2466, 2700] combine_rows_per_src=[5642, 5630, 5538, 5590, 5648, 5508, 5570, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44466 payload_rows_remote=38952 dup_rows=21070 dup_rows_remote=18406 dedup_ratio=0.321503 router_tokens_per_expert=[511, 516, 510, 536, 530, 535, 497, 534, 504, 492, 514, 481, 472, 496, 523, 509, 521, 504, 511, 511, 532, 488, 502, 520, 539, 531, 483, 492, 513, 493, 534, 500, 463, 486, 514, 503, 533, 527, 505, 485, 521, 472, 494, 496, 515, 544, 526, 498, 489, 517, 479, 516, 530, 536, 521, 536, 534, 560, 493, 529, 548, 499, 518, 547] payload_rows_per_dst=[5718, 5424, 5514, 5524, 5484, 5530, 5578, 5694] dup_rows_per_dst=[2620, 2558, 2664, 2646, 2548, 2602, 2670, 2762] combine_rows_per_src=[5718, 5424, 5514, 5524, 5484, 5530, 5578, 5694] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277860352 dispatch_meta_bytes=229000 combine_payload_bytes=277860352 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278089352 ingress_bytes=498061312 link_bytes=498061312 token_bytes=555949704 weight_movement_bytes=220200960 total_nvlink_bytes=776150664 weight_to_token_ratio=0.396081 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.662105386e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44412 payload_rows_remote=38842 dup_rows=21124 dup_rows_remote=18340 dedup_ratio=0.322327 router_tokens_per_expert=[526, 529, 511, 491, 520, 557, 503, 533, 516, 513, 529, 507, 518, 493, 514, 480, 496, 541, 529, 528, 475, 484, 516, 535, 533, 486, 505, 524, 528, 557, 529, 515, 517, 498, 527, 527, 515, 519, 501, 491, 490, 513, 548, 467, 515, 523, 460, 523, 508, 533, 514, 487, 474, 496, 455, 502, 524, 498, 503, 528, 509, 527, 510, 545] payload_rows_per_dst=[5656, 5508, 5554, 5570, 5556, 5514, 5452, 5602] dup_rows_per_dst=[2684, 2632, 2654, 2784, 2634, 2564, 2486, 2686] combine_rows_per_src=[5656, 5508, 5554, 5570, 5556, 5514, 5452, 5602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44212 payload_rows_remote=38698 dup_rows=21324 dup_rows_remote=18558 dedup_ratio=0.325378 router_tokens_per_expert=[511, 525, 500, 568, 464, 507, 530, 538, 517, 522, 505, 513, 474, 504, 499, 519, 485, 505, 545, 536, 502, 478, 494, 502, 502, 514, 478, 510, 490, 560, 525, 540, 524, 533, 500, 524, 537, 477, 523, 534, 535, 512, 527, 513, 487, 493, 504, 505, 485, 506, 516, 511, 517, 472, 520, 528, 509, 465, 515, 529, 520, 524, 528, 533] payload_rows_per_dst=[5342, 5586, 5630, 5552, 5572, 5518, 5498, 5514] dup_rows_per_dst=[2542, 2520, 2866, 2686, 2698, 2634, 2612, 2766] combine_rows_per_src=[5342, 5586, 5630, 5552, 5572, 5518, 5498, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278405120 dispatch_meta_bytes=229208 combine_payload_bytes=278405120 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278634328 ingress_bytes=608706560 link_bytes=608706560 token_bytes=557039448 weight_movement_bytes=330301440 total_nvlink_bytes=887340888 weight_to_token_ratio=0.592959 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058640823e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44538 payload_rows_remote=38976 dup_rows=20998 dup_rows_remote=18316 dedup_ratio=0.320404 router_tokens_per_expert=[493, 499, 504, 508, 526, 508, 515, 492, 516, 509, 512, 547, 517, 519, 505, 497, 520, 523, 512, 527, 527, 513, 524, 514, 573, 548, 532, 534, 529, 513, 497, 510, 496, 515, 496, 504, 539, 486, 486, 501, 481, 531, 543, 498, 519, 483, 468, 519, 507, 492, 515, 492, 516, 497, 518, 544, 511, 517, 467, 557, 538, 501, 478, 490] payload_rows_per_dst=[5556, 5562, 5618, 5696, 5484, 5614, 5520, 5488] dup_rows_per_dst=[2534, 2682, 2702, 2776, 2562, 2470, 2642, 2630] combine_rows_per_src=[5556, 5562, 5618, 5696, 5484, 5614, 5520, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277645312 dispatch_meta_bytes=229192 combine_payload_bytes=277645312 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277874504 ingress_bytes=387745792 link_bytes=387745792 token_bytes=555519816 weight_movement_bytes=110100480 total_nvlink_bytes=665620296 weight_to_token_ratio=0.198194 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.743537055e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280311808 dispatch_meta_bytes=230000 combine_payload_bytes=280311808 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280541808 ingress_bytes=390412288 link_bytes=390412288 token_bytes=560853616 weight_movement_bytes=110100480 total_nvlink_bytes=670954096 weight_to_token_ratio=0.196309 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.789911806e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279207936 dispatch_meta_bytes=229432 combine_payload_bytes=279207936 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279437368 ingress_bytes=389308416 link_bytes=389308416 token_bytes=558645304 weight_movement_bytes=110100480 total_nvlink_bytes=668745784 weight_to_token_ratio=0.197085 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.770713657e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278419456 dispatch_meta_bytes=228728 combine_payload_bytes=278419456 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278648184 ingress_bytes=498620416 link_bytes=498620416 token_bytes=557067640 weight_movement_bytes=220200960 total_nvlink_bytes=777268600 weight_to_token_ratio=0.395286 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.671829124e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277387264 dispatch_meta_bytes=229024 combine_payload_bytes=277387264 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=277616288 ingress_bytes=607688704 link_bytes=607688704 token_bytes=555003552 weight_movement_bytes=330301440 total_nvlink_bytes=885304992 weight_to_token_ratio=0.595134 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.056870604e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279379968 dispatch_meta_bytes=229168 combine_payload_bytes=279379968 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279609136 ingress_bytes=499580928 link_bytes=499580928 token_bytes=558989104 weight_movement_bytes=220200960 total_nvlink_bytes=779190064 weight_to_token_ratio=0.393927 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.688534007e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44544 payload_rows_remote=38958 dup_rows=20992 dup_rows_remote=18138 dedup_ratio=0.320312 router_tokens_per_expert=[521, 527, 485, 539, 530, 511, 525, 513, 501, 535, 450, 545, 537, 503, 523, 511, 471, 526, 515, 488, 497, 510, 499, 507, 496, 520, 513, 507, 513, 506, 504, 525, 521, 537, 517, 505, 463, 523, 541, 522, 491, 530, 516, 539, 495, 524, 501, 544, 507, 474, 502, 583, 510, 514, 478, 481, 493, 500, 526, 492, 534, 542, 487, 523] payload_rows_per_dst=[5586, 5662, 5612, 5618, 5526, 5524, 5470, 5546] dup_rows_per_dst=[2854, 2770, 2414, 2558, 2594, 2526, 2628, 2648] combine_rows_per_src=[5586, 5662, 5612, 5618, 5526, 5524, 5470, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44392 payload_rows_remote=38852 dup_rows=21144 dup_rows_remote=18542 dedup_ratio=0.322632 router_tokens_per_expert=[473, 501, 529, 499, 492, 510, 500, 517, 519, 481, 502, 558, 547, 543, 510, 506, 511, 485, 513, 536, 507, 541, 514, 512, 503, 527, 506, 542, 489, 503, 523, 509, 516, 487, 471, 494, 523, 530, 532, 518, 515, 559, 510, 498, 494, 539, 493, 490, 531, 534, 503, 537, 514, 487, 532, 538, 494, 509, 507, 495, 485, 538, 509, 478] payload_rows_per_dst=[5436, 5626, 5546, 5596, 5540, 5552, 5594, 5502] dup_rows_per_dst=[2606, 2706, 2692, 2608, 2602, 2644, 2758, 2528] combine_rows_per_src=[5436, 5626, 5546, 5596, 5540, 5552, 5594, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44828 payload_rows_remote=39192 dup_rows=20708 dup_rows_remote=18022 dedup_ratio=0.315979 router_tokens_per_expert=[516, 538, 503, 560, 510, 532, 496, 536, 545, 505, 500, 523, 542, 519, 505, 486, 517, 495, 515, 483, 466, 510, 506, 512, 500, 488, 519, 536, 536, 540, 494, 484, 511, 490, 551, 523, 487, 519, 542, 482, 516, 502, 536, 519, 503, 534, 489, 562, 547, 474, 521, 489, 507, 498, 517, 499, 493, 480, 521, 485, 490, 516, 525, 523] payload_rows_per_dst=[5610, 5584, 5610, 5660, 5672, 5636, 5530, 5526] dup_rows_per_dst=[2772, 2666, 2398, 2534, 2538, 2686, 2574, 2540] combine_rows_per_src=[5610, 5584, 5610, 5660, 5672, 5636, 5530, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44528 payload_rows_remote=38940 dup_rows=21008 dup_rows_remote=18374 dedup_ratio=0.320557 router_tokens_per_expert=[498, 542, 496, 519, 514, 509, 486, 499, 518, 484, 504, 510, 483, 534, 488, 494, 497, 532, 528, 543, 528, 529, 482, 471, 515, 517, 505, 477, 492, 511, 486, 546, 516, 527, 518, 511, 541, 519, 524, 530, 505, 493, 537, 519, 495, 502, 499, 532, 513, 539, 534, 514, 473, 499, 515, 524, 539, 476, 541, 507, 534, 549, 511, 495] payload_rows_per_dst=[5598, 5538, 5530, 5514, 5642, 5556, 5588, 5562] dup_rows_per_dst=[2528, 2492, 2690, 2584, 2730, 2608, 2634, 2742] combine_rows_per_src=[5598, 5538, 5530, 5514, 5642, 5556, 5588, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44418 payload_rows_remote=38892 dup_rows=21118 dup_rows_remote=18468 dedup_ratio=0.322235 router_tokens_per_expert=[503, 505, 540, 475, 542, 531, 494, 498, 493, 509, 517, 518, 500, 502, 479, 514, 500, 548, 531, 500, 473, 470, 532, 534, 479, 515, 492, 521, 456, 494, 529, 496, 557, 513, 497, 562, 541, 481, 468, 537, 555, 533, 567, 552, 517, 504, 497, 522, 545, 501, 500, 500, 504, 500, 495, 547, 530, 515, 484, 512, 520, 500, 478, 544] payload_rows_per_dst=[5522, 5424, 5526, 5474, 5654, 5654, 5584, 5580] dup_rows_per_dst=[2654, 2640, 2650, 2490, 2658, 2840, 2600, 2586] combine_rows_per_src=[5522, 5424, 5526, 5474, 5654, 5654, 5584, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44590 payload_rows_remote=39034 dup_rows=20946 dup_rows_remote=18374 dedup_ratio=0.319611 router_tokens_per_expert=[515, 556, 526, 512, 528, 511, 477, 487, 544, 511, 529, 555, 527, 544, 491, 523, 486, 489, 550, 514, 510, 500, 491, 493, 503, 536, 509, 516, 481, 523, 495, 501, 502, 559, 509, 516, 502, 553, 488, 474, 515, 525, 500, 508, 531, 488, 518, 468, 496, 493, 504, 490, 533, 486, 525, 530, 513, 564, 520, 499, 524, 480, 510, 512] payload_rows_per_dst=[5522, 5668, 5506, 5556, 5612, 5614, 5556, 5556] dup_rows_per_dst=[2702, 2780, 2560, 2572, 2594, 2492, 2558, 2688] combine_rows_per_src=[5522, 5668, 5506, 5556, 5612, 5614, 5556, 5556] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279250944 dispatch_meta_bytes=228384 combine_payload_bytes=279250944 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279479328 ingress_bytes=389351424 link_bytes=389351424 token_bytes=558730272 weight_movement_bytes=110100480 total_nvlink_bytes=668830752 weight_to_token_ratio=0.197055 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.771461636e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278491136 dispatch_meta_bytes=229576 combine_payload_bytes=278491136 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278720712 ingress_bytes=608792576 link_bytes=608792576 token_bytes=557211848 weight_movement_bytes=330301440 total_nvlink_bytes=887513288 weight_to_token_ratio=0.592775 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058790419e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44652 payload_rows_remote=39064 dup_rows=20884 dup_rows_remote=18128 dedup_ratio=0.318665 router_tokens_per_expert=[488, 516, 462, 521, 494, 536, 530, 511, 514, 488, 539, 504, 465, 490, 547, 505, 520, 494, 561, 529, 514, 532, 520, 502, 508, 530, 538, 505, 535, 511, 524, 485, 546, 483, 526, 529, 493, 507, 538, 521, 489, 500, 507, 496, 515, 488, 481, 517, 529, 501, 502, 509, 498, 522, 480, 527, 531, 528, 489, 530, 521, 520, 523, 504] payload_rows_per_dst=[5526, 5608, 5588, 5690, 5616, 5466, 5552, 5606] dup_rows_per_dst=[2590, 2496, 2756, 2582, 2670, 2520, 2584, 2686] combine_rows_per_src=[5526, 5608, 5588, 5690, 5616, 5466, 5552, 5606] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279293952 dispatch_meta_bytes=229064 combine_payload_bytes=279293952 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279523016 ingress_bytes=609595392 link_bytes=609595392 token_bytes=558816968 weight_movement_bytes=330301440 total_nvlink_bytes=889118408 weight_to_token_ratio=0.591073 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060186648e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44436 payload_rows_remote=38954 dup_rows=21100 dup_rows_remote=18516 dedup_ratio=0.321960 router_tokens_per_expert=[567, 494, 506, 493, 540, 505, 502, 511, 517, 487, 538, 525, 499, 563, 469, 488, 502, 481, 540, 530, 494, 464, 517, 493, 504, 518, 545, 534, 497, 482, 496, 503, 558, 565, 526, 514, 508, 513, 524, 517, 515, 533, 492, 552, 531, 503, 505, 510, 548, 494, 502, 500, 473, 539, 498, 511, 520, 517, 536, 516, 504, 482, 484, 474] payload_rows_per_dst=[5578, 5560, 5460, 5508, 5676, 5584, 5588, 5482] dup_rows_per_dst=[2658, 2612, 2582, 2522, 2672, 2698, 2772, 2584] combine_rows_per_src=[5578, 5560, 5460, 5508, 5676, 5584, 5588, 5482] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279896064 dispatch_meta_bytes=229648 combine_payload_bytes=279896064 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=280125712 ingress_bytes=500097024 link_bytes=500097024 token_bytes=560021776 weight_movement_bytes=220200960 total_nvlink_bytes=780222736 weight_to_token_ratio=0.393201 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.697509766e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279107584 dispatch_meta_bytes=229376 combine_payload_bytes=279107584 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279336960 ingress_bytes=389208064 link_bytes=389208064 token_bytes=558444544 weight_movement_bytes=110100480 total_nvlink_bytes=668545024 weight_to_token_ratio=0.197156 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.768968370e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278892544 dispatch_meta_bytes=228016 combine_payload_bytes=278892544 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279120560 ingress_bytes=499093504 link_bytes=499093504 token_bytes=558013104 weight_movement_bytes=220200960 total_nvlink_bytes=778214064 weight_to_token_ratio=0.394616 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.680056902e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=280010752 dispatch_meta_bytes=228768 combine_payload_bytes=280010752 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280239520 ingress_bytes=390111232 link_bytes=390111232 token_bytes=560250272 weight_movement_bytes=110100480 total_nvlink_bytes=670350752 weight_to_token_ratio=0.196520 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.784675947e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279222272 dispatch_meta_bytes=229880 combine_payload_bytes=279222272 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279452152 ingress_bytes=499423232 link_bytes=499423232 token_bytes=558674424 weight_movement_bytes=220200960 total_nvlink_bytes=778875384 weight_to_token_ratio=0.394149 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.685791415e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44480 payload_rows_remote=38994 dup_rows=21056 dup_rows_remote=18390 dedup_ratio=0.321289 router_tokens_per_expert=[543, 496, 526, 526, 491, 525, 511, 440, 528, 483, 506, 535, 494, 539, 533, 545, 497, 502, 481, 522, 525, 499, 497, 475, 523, 527, 440, 525, 480, 488, 549, 489, 521, 519, 522, 496, 498, 499, 530, 491, 533, 544, 526, 544, 545, 532, 460, 495, 535, 524, 521, 514, 506, 502, 493, 562, 493, 526, 534, 503, 526, 485, 525, 524] payload_rows_per_dst=[5530, 5628, 5542, 5468, 5486, 5716, 5600, 5510] dup_rows_per_dst=[2586, 2698, 2454, 2574, 2666, 2642, 2714, 2722] combine_rows_per_src=[5530, 5628, 5542, 5468, 5486, 5716, 5600, 5510] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44496 payload_rows_remote=38932 dup_rows=21040 dup_rows_remote=18494 dedup_ratio=0.321045 router_tokens_per_expert=[534, 507, 542, 533, 444, 527, 511, 490, 517, 494, 503, 495, 495, 481, 515, 514, 522, 528, 452, 536, 501, 517, 518, 482, 536, 548, 513, 490, 520, 524, 497, 531, 528, 563, 493, 533, 498, 511, 547, 506, 509, 502, 525, 541, 486, 497, 497, 498, 532, 507, 507, 518, 541, 435, 529, 497, 532, 531, 518, 536, 497, 484, 526, 527] payload_rows_per_dst=[5560, 5460, 5552, 5674, 5606, 5564, 5518, 5562] dup_rows_per_dst=[2616, 2568, 2560, 2644, 2752, 2546, 2614, 2740] combine_rows_per_src=[5560, 5460, 5552, 5674, 5606, 5564, 5518, 5562] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44414 payload_rows_remote=38922 dup_rows=21122 dup_rows_remote=18436 dedup_ratio=0.322296 router_tokens_per_expert=[480, 524, 491, 486, 552, 517, 508, 515, 509, 490, 513, 517, 477, 521, 552, 510, 494, 553, 521, 479, 565, 523, 505, 516, 544, 518, 479, 490, 482, 495, 503, 534, 495, 500, 496, 513, 556, 517, 551, 525, 483, 503, 519, 544, 516, 519, 501, 517, 542, 553, 502, 504, 513, 482, 533, 482, 494, 543, 528, 512, 474, 494, 499, 495] payload_rows_per_dst=[5508, 5492, 5630, 5548, 5602, 5618, 5548, 5468] dup_rows_per_dst=[2638, 2686, 2682, 2542, 2704, 2586, 2674, 2610] combine_rows_per_src=[5508, 5492, 5630, 5548, 5602, 5618, 5548, 5468] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44442 payload_rows_remote=38900 dup_rows=21094 dup_rows_remote=18582 dedup_ratio=0.321869 router_tokens_per_expert=[522, 535, 545, 493, 522, 487, 512, 511, 487, 548, 519, 509, 513, 512, 518, 520, 501, 496, 497, 481, 542, 504, 535, 518, 565, 530, 553, 455, 521, 544, 519, 539, 477, 534, 496, 539, 522, 517, 513, 473, 514, 492, 487, 499, 502, 509, 485, 526, 512, 541, 494, 472, 510, 531, 471, 496, 484, 517, 506, 505, 551, 509, 532, 499] payload_rows_per_dst=[5610, 5564, 5526, 5658, 5554, 5470, 5542, 5518] dup_rows_per_dst=[2644, 2688, 2622, 2794, 2588, 2558, 2512, 2688] combine_rows_per_src=[5610, 5564, 5526, 5658, 5554, 5470, 5542, 5518] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44464 payload_rows_remote=38876 dup_rows=21072 dup_rows_remote=18246 dedup_ratio=0.321533 router_tokens_per_expert=[483, 524, 518, 487, 508, 506, 495, 507, 509, 508, 525, 540, 482, 556, 527, 498, 498, 492, 492, 503, 516, 505, 503, 455, 527, 529, 503, 527, 510, 542, 526, 527, 498, 516, 540, 517, 521, 489, 517, 507, 509, 491, 529, 503, 503, 505, 498, 522, 522, 489, 518, 505, 496, 507, 529, 522, 482, 533, 524, 528, 516, 548, 537, 519] payload_rows_per_dst=[5572, 5356, 5656, 5578, 5570, 5550, 5594, 5588] dup_rows_per_dst=[2484, 2380, 2826, 2722, 2558, 2570, 2706, 2826] combine_rows_per_src=[5572, 5356, 5656, 5578, 5570, 5550, 5594, 5588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44408 payload_rows_remote=38926 dup_rows=21128 dup_rows_remote=18442 dedup_ratio=0.322388 router_tokens_per_expert=[477, 523, 487, 482, 549, 513, 508, 509, 506, 489, 513, 518, 475, 521, 552, 510, 493, 548, 520, 479, 568, 520, 505, 517, 543, 518, 479, 487, 480, 494, 502, 536, 496, 499, 496, 513, 556, 515, 553, 524, 481, 504, 520, 547, 516, 519, 502, 519, 543, 553, 504, 504, 517, 485, 534, 486, 495, 548, 530, 512, 474, 500, 502, 500] payload_rows_per_dst=[5480, 5482, 5628, 5544, 5602, 5620, 5566, 5486] dup_rows_per_dst=[2616, 2686, 2672, 2534, 2702, 2596, 2686, 2636] combine_rows_per_src=[5480, 5482, 5628, 5544, 5602, 5620, 5566, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280928256 dispatch_meta_bytes=228856 combine_payload_bytes=280928256 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=281157112 ingress_bytes=501129216 link_bytes=501129216 token_bytes=562085368 weight_movement_bytes=220200960 total_nvlink_bytes=782286328 weight_to_token_ratio=0.391757 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.715461282e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279121920 dispatch_meta_bytes=229256 combine_payload_bytes=279121920 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279351176 ingress_bytes=389222400 link_bytes=389222400 token_bytes=558473096 weight_movement_bytes=110100480 total_nvlink_bytes=668573576 weight_to_token_ratio=0.197146 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.769217697e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278777856 dispatch_meta_bytes=229440 combine_payload_bytes=278777856 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279007296 ingress_bytes=388878336 link_bytes=388878336 token_bytes=557785152 weight_movement_bytes=110100480 total_nvlink_bytes=667885632 weight_to_token_ratio=0.197389 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.763233858e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44374 payload_rows_remote=38734 dup_rows=21162 dup_rows_remote=18360 dedup_ratio=0.322906 router_tokens_per_expert=[527, 500, 508, 507, 502, 521, 521, 514, 509, 479, 522, 486, 512, 467, 531, 502, 492, 522, 514, 504, 527, 506, 531, 494, 519, 510, 540, 505, 535, 517, 512, 523, 524, 502, 523, 522, 522, 507, 500, 496, 564, 528, 507, 513, 491, 507, 486, 527, 560, 513, 481, 505, 505, 529, 529, 543, 490, 503, 521, 494, 518, 492, 511, 496] payload_rows_per_dst=[5640, 5616, 5708, 5488, 5318, 5630, 5604, 5370] dup_rows_per_dst=[2802, 2668, 2866, 2598, 2238, 2662, 2726, 2602] combine_rows_per_src=[5640, 5616, 5708, 5488, 5318, 5630, 5604, 5370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44474 payload_rows_remote=38922 dup_rows=21062 dup_rows_remote=18368 dedup_ratio=0.321381 router_tokens_per_expert=[515, 513, 481, 505, 488, 490, 503, 530, 496, 527, 490, 517, 500, 515, 549, 502, 506, 487, 470, 516, 481, 519, 505, 520, 552, 560, 505, 474, 522, 505, 500, 505, 528, 531, 527, 541, 519, 487, 498, 563, 518, 502, 515, 533, 525, 538, 498, 509, 514, 520, 537, 519, 468, 459, 463, 514, 527, 500, 523, 552, 522, 535, 502, 533] payload_rows_per_dst=[5470, 5578, 5524, 5552, 5650, 5634, 5504, 5562] dup_rows_per_dst=[2580, 2614, 2484, 2694, 2738, 2642, 2484, 2826] combine_rows_per_src=[5470, 5578, 5524, 5552, 5650, 5634, 5504, 5562] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279508992 dispatch_meta_bytes=229536 combine_payload_bytes=279508992 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=279738528 ingress_bytes=719910912 link_bytes=719910912 token_bytes=559247520 weight_movement_bytes=440401920 total_nvlink_bytes=999649440 weight_to_token_ratio=0.787490 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.252043481e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44476 payload_rows_remote=38872 dup_rows=21060 dup_rows_remote=18408 dedup_ratio=0.321350 router_tokens_per_expert=[522, 508, 554, 493, 560, 551, 462, 501, 547, 492, 503, 549, 484, 504, 482, 534, 518, 493, 513, 511, 541, 511, 510, 531, 529, 490, 518, 502, 510, 481, 530, 529, 494, 518, 468, 567, 484, 542, 531, 497, 499, 521, 542, 521, 495, 502, 499, 534, 515, 511, 511, 496, 483, 523, 486, 525, 517, 492, 481, 498, 543, 502, 488, 520] payload_rows_per_dst=[5596, 5510, 5604, 5524, 5558, 5572, 5566, 5546] dup_rows_per_dst=[2706, 2680, 2652, 2654, 2644, 2654, 2534, 2536] combine_rows_per_src=[5596, 5510, 5604, 5524, 5558, 5572, 5566, 5546] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279064576 dispatch_meta_bytes=229704 combine_payload_bytes=279064576 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279294280 ingress_bytes=389165056 link_bytes=389165056 token_bytes=558358856 weight_movement_bytes=110100480 total_nvlink_bytes=668459336 weight_to_token_ratio=0.197186 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.768220390e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278992896 dispatch_meta_bytes=229432 combine_payload_bytes=278992896 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279222328 ingress_bytes=389093376 link_bytes=389093376 token_bytes=558215224 weight_movement_bytes=110100480 total_nvlink_bytes=668315704 weight_to_token_ratio=0.197237 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.766973757e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44372 payload_rows_remote=38696 dup_rows=21164 dup_rows_remote=18516 dedup_ratio=0.322937 router_tokens_per_expert=[540, 544, 472, 477, 513, 558, 546, 465, 526, 484, 535, 517, 493, 524, 512, 534, 512, 477, 483, 503, 481, 533, 498, 502, 494, 511, 518, 498, 470, 525, 576, 535, 511, 489, 479, 500, 512, 520, 500, 484, 518, 496, 501, 514, 530, 552, 515, 524, 537, 507, 526, 512, 475, 528, 541, 497, 505, 553, 515, 528, 502, 496, 528, 517] payload_rows_per_dst=[5558, 5502, 5438, 5588, 5418, 5570, 5622, 5676] dup_rows_per_dst=[2616, 2748, 2540, 2664, 2572, 2732, 2644, 2648] combine_rows_per_src=[5558, 5502, 5438, 5588, 5418, 5570, 5622, 5676] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279795712 dispatch_meta_bytes=229632 combine_payload_bytes=279795712 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280025344 ingress_bytes=610097152 link_bytes=610097152 token_bytes=559821056 weight_movement_bytes=330301440 total_nvlink_bytes=890122496 weight_to_token_ratio=0.590013 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061059291e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278663168 dispatch_meta_bytes=228488 combine_payload_bytes=278663168 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278891656 ingress_bytes=388763648 link_bytes=388763648 token_bytes=557554824 weight_movement_bytes=110100480 total_nvlink_bytes=667655304 weight_to_token_ratio=0.197470 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.761239245e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279021568 dispatch_meta_bytes=229472 combine_payload_bytes=279021568 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279251040 ingress_bytes=389122048 link_bytes=389122048 token_bytes=558272608 weight_movement_bytes=110100480 total_nvlink_bytes=668373088 weight_to_token_ratio=0.197216 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.767472411e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44414 payload_rows_remote=38652 dup_rows=21122 dup_rows_remote=18126 dedup_ratio=0.322296 router_tokens_per_expert=[523, 500, 505, 498, 499, 515, 518, 510, 502, 480, 521, 483, 511, 467, 528, 500, 492, 523, 513, 504, 526, 506, 530, 491, 518, 509, 540, 504, 534, 517, 511, 523, 528, 502, 524, 519, 522, 505, 502, 498, 566, 530, 510, 512, 491, 509, 487, 530, 561, 518, 484, 509, 506, 531, 529, 547, 495, 508, 522, 496, 522, 494, 512, 498] payload_rows_per_dst=[5762, 5710, 5766, 5462, 5266, 5634, 5608, 5206] dup_rows_per_dst=[2996, 2772, 2936, 2570, 2130, 2658, 2704, 2356] combine_rows_per_src=[5762, 5710, 5766, 5462, 5266, 5634, 5608, 5206] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44482 payload_rows_remote=38994 dup_rows=21054 dup_rows_remote=18384 dedup_ratio=0.321259 router_tokens_per_expert=[536, 492, 522, 522, 489, 523, 509, 438, 526, 481, 506, 531, 492, 536, 533, 543, 495, 502, 479, 521, 529, 501, 498, 474, 522, 528, 439, 524, 478, 489, 549, 489, 520, 517, 522, 499, 499, 502, 531, 489, 531, 545, 525, 540, 546, 538, 462, 497, 536, 527, 522, 517, 507, 503, 494, 564, 498, 527, 539, 506, 530, 488, 526, 525] payload_rows_per_dst=[5510, 5612, 5544, 5466, 5488, 5716, 5610, 5536] dup_rows_per_dst=[2552, 2684, 2454, 2570, 2670, 2652, 2730, 2742] combine_rows_per_src=[5510, 5612, 5544, 5466, 5488, 5716, 5610, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44494 payload_rows_remote=38924 dup_rows=21042 dup_rows_remote=18478 dedup_ratio=0.321075 router_tokens_per_expert=[532, 503, 540, 532, 444, 520, 510, 487, 513, 494, 499, 495, 491, 480, 513, 515, 521, 525, 448, 533, 499, 515, 519, 479, 531, 549, 509, 489, 520, 526, 497, 532, 527, 562, 497, 532, 500, 511, 546, 511, 509, 502, 528, 541, 485, 501, 499, 502, 534, 508, 508, 520, 543, 439, 530, 498, 535, 536, 520, 540, 501, 487, 529, 527] payload_rows_per_dst=[5548, 5446, 5532, 5672, 5620, 5570, 5530, 5576] dup_rows_per_dst=[2588, 2554, 2546, 2634, 2752, 2564, 2630, 2774] combine_rows_per_src=[5548, 5446, 5532, 5672, 5620, 5570, 5530, 5576] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278835200 dispatch_meta_bytes=229928 combine_payload_bytes=278835200 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279065128 ingress_bytes=388935680 link_bytes=388935680 token_bytes=557900328 weight_movement_bytes=110100480 total_nvlink_bytes=668000808 weight_to_token_ratio=0.197348 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.764231164e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=277645312 dispatch_meta_bytes=228376 combine_payload_bytes=277645312 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277873688 ingress_bytes=387745792 link_bytes=387745792 token_bytes=555519000 weight_movement_bytes=110100480 total_nvlink_bytes=665619480 weight_to_token_ratio=0.198194 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.743537055e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278992896 dispatch_meta_bytes=229160 combine_payload_bytes=278992896 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279222056 ingress_bytes=609294336 link_bytes=609294336 token_bytes=558214952 weight_movement_bytes=330301440 total_nvlink_bytes=888516392 weight_to_token_ratio=0.591710 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059663062e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278634496 dispatch_meta_bytes=229120 combine_payload_bytes=278634496 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278863616 ingress_bytes=388734976 link_bytes=388734976 token_bytes=557498112 weight_movement_bytes=110100480 total_nvlink_bytes=667598592 weight_to_token_ratio=0.197490 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.760740592e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=277372928 dispatch_meta_bytes=228848 combine_payload_bytes=277372928 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277601776 ingress_bytes=497573888 link_bytes=497573888 token_bytes=554974704 weight_movement_bytes=220200960 total_nvlink_bytes=775175664 weight_to_token_ratio=0.396777 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.653628281e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44492 payload_rows_remote=38862 dup_rows=21044 dup_rows_remote=18318 dedup_ratio=0.321106 router_tokens_per_expert=[534, 506, 474, 516, 520, 493, 477, 504, 532, 497, 494, 529, 509, 514, 506, 525, 518, 525, 488, 541, 521, 507, 482, 527, 508, 514, 522, 532, 480, 542, 477, 538, 521, 505, 486, 513, 531, 521, 526, 575, 515, 511, 535, 514, 538, 511, 498, 510, 521, 489, 498, 520, 496, 519, 518, 517, 507, 506, 486, 478, 509, 522, 509, 511] payload_rows_per_dst=[5434, 5590, 5548, 5654, 5630, 5650, 5524, 5462] dup_rows_per_dst=[2614, 2622, 2670, 2572, 2726, 2614, 2632, 2594] combine_rows_per_src=[5434, 5590, 5548, 5654, 5630, 5650, 5524, 5462] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5582320624 prefetch_bytes=990904320 reduce_bytes=990904320 weight_movement_bytes=1981808640 egress_bytes=2792305648 ingress_bytes=4771823616 link_bytes=4771823616 total_nvlink_bytes=7564129264 weight_to_token_ratio=0.355015 prefetch_share_of_ingress=0.207657 dedup_ratio=0.320612 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=9 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.298986098e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44530 payload_rows_remote=38942 dup_rows=21006 dup_rows_remote=18472 dedup_ratio=0.320526 router_tokens_per_expert=[488, 519, 484, 501, 512, 495, 545, 514, 511, 522, 497, 507, 558, 513, 515, 528, 526, 492, 549, 522, 488, 463, 497, 508, 498, 488, 522, 526, 524, 513, 539, 500, 545, 519, 520, 513, 509, 508, 493, 518, 469, 504, 550, 505, 514, 525, 513, 481, 519, 525, 523, 524, 499, 495, 532, 526, 486, 519, 511, 488, 526, 510, 511, 524] payload_rows_per_dst=[5524, 5590, 5468, 5614, 5580, 5588, 5638, 5528] dup_rows_per_dst=[2592, 2712, 2622, 2606, 2670, 2534, 2648, 2622] combine_rows_per_src=[5524, 5590, 5468, 5614, 5580, 5588, 5638, 5528] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44672 payload_rows_remote=39174 dup_rows=20864 dup_rows_remote=18274 dedup_ratio=0.318359 router_tokens_per_expert=[552, 522, 520, 537, 533, 521, 520, 532, 497, 530, 522, 496, 516, 514, 476, 493, 506, 492, 537, 527, 532, 514, 540, 491, 498, 535, 545, 530, 497, 518, 459, 505, 535, 502, 513, 494, 483, 482, 520, 460, 482, 513, 521, 502, 530, 507, 486, 506, 514, 490, 515, 514, 531, 487, 499, 505, 558, 511, 531, 521, 507, 479, 543, 520] payload_rows_per_dst=[5768, 5498, 5646, 5548, 5518, 5510, 5524, 5660] dup_rows_per_dst=[2706, 2590, 2632, 2626, 2460, 2584, 2586, 2680] combine_rows_per_src=[5768, 5498, 5646, 5548, 5518, 5510, 5524, 5660] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5561927976 prefetch_bytes=990904320 reduce_bytes=880803840 weight_movement_bytes=1871708160 egress_bytes=2782105896 ingress_bytes=4651530240 link_bytes=4651530240 total_nvlink_bytes=7433636136 weight_to_token_ratio=0.336521 prefetch_share_of_ingress=0.213028 dedup_ratio=0.322018 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=8 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042647 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.089776133e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44318 payload_rows_remote=38742 dup_rows=21218 dup_rows_remote=18506 dedup_ratio=0.323761 router_tokens_per_expert=[510, 496, 515, 515, 509, 516, 503, 499, 482, 505, 492, 523, 509, 528, 529, 539, 527, 502, 529, 507, 526, 493, 512, 504, 535, 525, 496, 558, 483, 517, 464, 490, 496, 547, 513, 528, 487, 527, 513, 501, 469, 505, 533, 501, 499, 485, 497, 512, 540, 529, 516, 549, 505, 500, 467, 538, 521, 503, 547, 526, 498, 530, 517, 531] payload_rows_per_dst=[5582, 5542, 5564, 5578, 5500, 5402, 5576, 5574] dup_rows_per_dst=[2544, 2672, 2636, 2558, 2724, 2600, 2712, 2772] combine_rows_per_src=[5582, 5542, 5564, 5578, 5500, 5402, 5576, 5574] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44590 payload_rows_remote=39058 dup_rows=20946 dup_rows_remote=18336 dedup_ratio=0.319611 router_tokens_per_expert=[475, 510, 501, 493, 511, 509, 556, 511, 464, 489, 529, 510, 524, 489, 503, 508, 494, 510, 523, 524, 519, 546, 512, 475, 532, 488, 493, 512, 528, 517, 517, 503, 536, 513, 517, 536, 530, 514, 471, 520, 499, 479, 504, 517, 517, 511, 545, 547, 524, 511, 538, 539, 490, 492, 536, 493, 521, 527, 509, 549, 517, 511, 484, 496] payload_rows_per_dst=[5532, 5474, 5654, 5576, 5634, 5600, 5592, 5528] dup_rows_per_dst=[2610, 2642, 2678, 2604, 2552, 2632, 2654, 2574] combine_rows_per_src=[5532, 5474, 5654, 5576, 5634, 5600, 5592, 5528] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278562816 dispatch_meta_bytes=228720 combine_payload_bytes=278562816 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278791536 ingress_bytes=608864256 link_bytes=608864256 token_bytes=557354352 weight_movement_bytes=330301440 total_nvlink_bytes=887655792 weight_to_token_ratio=0.592624 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058915082e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44450 payload_rows_remote=38936 dup_rows=21086 dup_rows_remote=18430 dedup_ratio=0.321747 router_tokens_per_expert=[519, 481, 511, 477, 544, 540, 486, 519, 519, 533, 505, 508, 497, 497, 525, 498, 491, 537, 522, 502, 509, 504, 496, 531, 522, 529, 515, 515, 489, 473, 510, 532, 539, 506, 502, 514, 500, 538, 525, 551, 531, 508, 544, 505, 512, 484, 538, 525, 498, 529, 469, 500, 482, 527, 507, 530, 517, 487, 489, 479, 525, 516, 522, 533] payload_rows_per_dst=[5516, 5548, 5590, 5514, 5598, 5622, 5510, 5552] dup_rows_per_dst=[2638, 2616, 2594, 2656, 2752, 2672, 2574, 2584] combine_rows_per_src=[5516, 5548, 5590, 5514, 5598, 5622, 5510, 5552] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279136256 dispatch_meta_bytes=229656 combine_payload_bytes=279136256 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279365912 ingress_bytes=499337216 link_bytes=499337216 token_bytes=558502168 weight_movement_bytes=220200960 total_nvlink_bytes=778703128 weight_to_token_ratio=0.394271 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.684295455e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=280799232 dispatch_meta_bytes=229792 combine_payload_bytes=280799232 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281029024 ingress_bytes=390899712 link_bytes=390899712 token_bytes=561828256 weight_movement_bytes=110100480 total_nvlink_bytes=671928736 weight_to_token_ratio=0.195968 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.798388911e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=277702656 dispatch_meta_bytes=228992 combine_payload_bytes=277702656 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277931648 ingress_bytes=387803136 link_bytes=387803136 token_bytes=555634304 weight_movement_bytes=110100480 total_nvlink_bytes=665734784 weight_to_token_ratio=0.198153 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.744534362e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279967744 dispatch_meta_bytes=229576 combine_payload_bytes=279967744 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280197320 ingress_bytes=610269184 link_bytes=610269184 token_bytes=560165064 weight_movement_bytes=330301440 total_nvlink_bytes=890466504 weight_to_token_ratio=0.589650 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061358483e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279093248 dispatch_meta_bytes=229464 combine_payload_bytes=279093248 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=279322712 ingress_bytes=389193728 link_bytes=389193728 token_bytes=558415960 weight_movement_bytes=110100480 total_nvlink_bytes=668516440 weight_to_token_ratio=0.197166 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.768719044e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5582726296 prefetch_bytes=990904320 reduce_bytes=880803840 weight_movement_bytes=1871708160 egress_bytes=2792510616 ingress_bytes=4661923840 link_bytes=4661923840 total_nvlink_bytes=7454434456 weight_to_token_ratio=0.335268 prefetch_share_of_ingress=0.212553 dedup_ratio=0.321622 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=8 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.107852312e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5584647120 prefetch_bytes=990904320 reduce_bytes=770703360 weight_movement_bytes=1761607680 egress_bytes=2793470928 ingress_bytes=4552783872 link_bytes=4552783872 total_nvlink_bytes=7346254800 weight_to_token_ratio=0.315438 prefetch_share_of_ingress=0.217648 dedup_ratio=0.321048 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=7 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.918039958e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5586625896 prefetch_bytes=1101004800 reduce_bytes=880803840 weight_movement_bytes=1981808640 egress_bytes=2794460520 ingress_bytes=4773974016 link_bytes=4773974016 total_nvlink_bytes=7568434536 weight_to_token_ratio=0.354742 prefetch_share_of_ingress=0.230626 dedup_ratio=0.320975 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=8 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043382 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.302725998e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5585163496 prefetch_bytes=990904320 reduce_bytes=220200960 weight_movement_bytes=1211105280 egress_bytes=2793729256 ingress_bytes=4002539520 link_bytes=4002539520 total_nvlink_bytes=6796268776 weight_to_token_ratio=0.216843 prefetch_share_of_ingress=0.247569 dedup_ratio=0.320941 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=2 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043382 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=6.961074530e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5567750832 prefetch_bytes=880803840 reduce_bytes=1651507200 weight_movement_bytes=2532311040 egress_bytes=2785018544 ingress_bytes=5315043328 link_bytes=5315043328 total_nvlink_bytes=8100061872 weight_to_token_ratio=0.454818 prefetch_share_of_ingress=0.165719 dedup_ratio=0.321457 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=15 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.243734522e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5590524136 prefetch_bytes=770703360 reduce_bytes=1431306240 weight_movement_bytes=2202009600 egress_bytes=2796409064 ingress_bytes=4996124672 link_bytes=4996124672 total_nvlink_bytes=7792533736 weight_to_token_ratio=0.393882 prefetch_share_of_ingress=0.154260 dedup_ratio=0.320688 prefetch_slots=14 unused_prefetch_slots=146 slot_utilization=0.087500 reduce_slots=13 remote_demand=7 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.039338 gemm_inner_pad_upper_bound=12319 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.689082526e-05 prefetch_bytes_per_gemm_flop=1.340379902e-05 anomalies=none - [2026-08-26 15:12:37] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 361.6 | throughput per GPU (TFLOP/s/GPU): 341.5 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.516228E+00 | loss scale: 1.0 | grad norm: 30.542 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44526 payload_rows_remote=38900 dup_rows=21010 dup_rows_remote=18376 dedup_ratio=0.320587 router_tokens_per_expert=[524, 512, 482, 524, 529, 515, 512, 506, 483, 494, 541, 560, 508, 519, 514, 509, 517, 502, 555, 569, 501, 523, 496, 480, 548, 509, 524, 499, 530, 480, 539, 491, 510, 507, 499, 505, 526, 517, 512, 554, 516, 546, 487, 547, 529, 547, 503, 494, 467, 485, 494, 527, 484, 493, 485, 496, 506, 502, 521, 489, 533, 456, 543, 493] payload_rows_per_dst=[5598, 5572, 5594, 5540, 5626, 5632, 5458, 5506] dup_rows_per_dst=[2610, 2684, 2692, 2700, 2634, 2706, 2404, 2580] combine_rows_per_src=[5598, 5572, 5594, 5540, 5626, 5632, 5458, 5506] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44464 payload_rows_remote=38876 dup_rows=21072 dup_rows_remote=18312 dedup_ratio=0.321533 router_tokens_per_expert=[495, 536, 525, 508, 494, 495, 490, 504, 510, 523, 484, 468, 511, 507, 514, 512, 523, 505, 499, 503, 507, 520, 494, 522, 513, 518, 510, 504, 507, 553, 515, 554, 520, 493, 530, 540, 483, 518, 491, 535, 506, 550, 489, 539, 547, 485, 503, 550, 527, 537, 509, 484, 504, 500, 547, 508, 483, 487, 491, 506, 537, 513, 506, 527] payload_rows_per_dst=[5550, 5450, 5580, 5588, 5602, 5618, 5534, 5542] dup_rows_per_dst=[2544, 2608, 2566, 2760, 2618, 2720, 2698, 2558] combine_rows_per_src=[5550, 5450, 5580, 5588, 5602, 5618, 5534, 5542] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44300 payload_rows_remote=38746 dup_rows=21236 dup_rows_remote=18504 dedup_ratio=0.324036 router_tokens_per_expert=[519, 508, 509, 517, 455, 519, 516, 515, 508, 517, 511, 480, 541, 568, 500, 474, 488, 467, 545, 522, 517, 497, 513, 499, 528, 526, 541, 531, 515, 519, 504, 547, 501, 483, 507, 522, 505, 560, 509, 492, 525, 529, 534, 531, 495, 496, 540, 493, 482, 505, 521, 513, 507, 494, 505, 518, 518, 493, 483, 497, 528, 541, 528, 497] payload_rows_per_dst=[5482, 5520, 5536, 5636, 5556, 5554, 5494, 5522] dup_rows_per_dst=[2634, 2678, 2560, 2786, 2602, 2732, 2596, 2648] combine_rows_per_src=[5482, 5520, 5536, 5636, 5556, 5554, 5494, 5522] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44486 payload_rows_remote=38970 dup_rows=21050 dup_rows_remote=18494 dedup_ratio=0.321198 router_tokens_per_expert=[475, 547, 498, 505, 535, 461, 522, 494, 492, 500, 532, 482, 551, 514, 528, 533, 499, 488, 506, 503, 526, 516, 521, 481, 523, 540, 517, 543, 528, 509, 517, 524, 534, 546, 483, 498, 511, 495, 548, 517, 532, 540, 498, 452, 564, 529, 538, 523, 493, 513, 510, 491, 495, 528, 542, 464, 475, 515, 518, 513, 494, 516, 508, 475] payload_rows_per_dst=[5538, 5564, 5520, 5638, 5576, 5648, 5516, 5486] dup_rows_per_dst=[2536, 2700, 2560, 2764, 2688, 2704, 2556, 2542] combine_rows_per_src=[5538, 5564, 5520, 5638, 5576, 5648, 5516, 5486] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44556 payload_rows_remote=38878 dup_rows=20980 dup_rows_remote=18484 dedup_ratio=0.320129 router_tokens_per_expert=[512, 489, 500, 488, 495, 521, 479, 534, 525, 516, 542, 538, 524, 484, 568, 512, 504, 546, 512, 498, 510, 464, 517, 536, 528, 532, 525, 539, 510, 494, 494, 503, 492, 514, 501, 515, 518, 491, 507, 550, 503, 556, 512, 495, 498, 466, 516, 535, 483, 480, 501, 558, 509, 539, 516, 554, 490, 504, 501, 500, 463, 489, 528, 545] payload_rows_per_dst=[5464, 5624, 5678, 5568, 5536, 5544, 5626, 5516] dup_rows_per_dst=[2572, 2794, 2496, 2682, 2640, 2618, 2654, 2524] combine_rows_per_src=[5464, 5624, 5678, 5568, 5536, 5544, 5626, 5516] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44390 payload_rows_remote=38580 dup_rows=21146 dup_rows_remote=17946 dedup_ratio=0.322662 router_tokens_per_expert=[527, 526, 537, 496, 500, 516, 521, 519, 507, 535, 519, 507, 553, 510, 523, 523, 483, 485, 543, 524, 537, 512, 483, 454, 553, 543, 510, 540, 510, 486, 494, 517, 518, 499, 530, 474, 518, 530, 513, 488, 502, 505, 505, 540, 477, 495, 523, 492, 522, 520, 525, 502, 496, 491, 517, 481, 501, 525, 511, 487, 515, 519, 543, 511] payload_rows_per_dst=[5646, 5558, 5542, 5282, 5318, 5540, 5694, 5810] dup_rows_per_dst=[2638, 2700, 2500, 2238, 2452, 2538, 2880, 3200] combine_rows_per_src=[5646, 5558, 5542, 5282, 5318, 5540, 5694, 5810] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44624 payload_rows_remote=39104 dup_rows=20912 dup_rows_remote=18352 dedup_ratio=0.319092 router_tokens_per_expert=[575, 513, 504, 502, 509, 517, 516, 510, 525, 501, 499, 533, 456, 485, 505, 536, 497, 522, 512, 541, 498, 505, 516, 559, 513, 492, 473, 497, 480, 498, 555, 534, 518, 504, 505, 500, 533, 529, 512, 518, 503, 507, 532, 513, 510, 509, 522, 498, 525, 549, 491, 523, 561, 477, 490, 522, 520, 486, 499, 509, 476, 529, 527, 493] payload_rows_per_dst=[5618, 5520, 5660, 5508, 5574, 5594, 5614, 5536] dup_rows_per_dst=[2674, 2560, 2640, 2576, 2664, 2594, 2662, 2542] combine_rows_per_src=[5618, 5520, 5660, 5508, 5574, 5594, 5614, 5536] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278663168 dispatch_meta_bytes=228752 combine_payload_bytes=278663168 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278891920 ingress_bytes=498864128 link_bytes=498864128 token_bytes=557555088 weight_movement_bytes=220200960 total_nvlink_bytes=777756048 weight_to_token_ratio=0.394940 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.676067676e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44428 payload_rows_remote=38870 dup_rows=21108 dup_rows_remote=18580 dedup_ratio=0.322083 router_tokens_per_expert=[520, 529, 543, 492, 520, 484, 511, 505, 485, 544, 517, 507, 512, 512, 517, 518, 498, 497, 492, 479, 542, 502, 530, 518, 562, 530, 554, 457, 518, 544, 519, 538, 477, 535, 497, 537, 525, 517, 511, 474, 516, 495, 487, 500, 502, 510, 488, 528, 512, 544, 494, 475, 511, 535, 473, 499, 488, 520, 513, 509, 553, 510, 536, 501] payload_rows_per_dst=[5588, 5554, 5510, 5648, 5554, 5478, 5558, 5538] dup_rows_per_dst=[2620, 2670, 2606, 2796, 2592, 2574, 2528, 2722] combine_rows_per_src=[5588, 5554, 5510, 5648, 5554, 5478, 5558, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44658 payload_rows_remote=39080 dup_rows=20878 dup_rows_remote=18134 dedup_ratio=0.318573 router_tokens_per_expert=[486, 514, 460, 518, 488, 532, 527, 510, 509, 488, 538, 501, 463, 486, 544, 504, 520, 492, 561, 528, 510, 531, 519, 500, 510, 529, 535, 508, 535, 511, 523, 486, 546, 484, 529, 530, 495, 504, 541, 523, 491, 501, 507, 498, 517, 489, 482, 518, 533, 504, 502, 512, 499, 522, 484, 530, 535, 531, 492, 531, 521, 522, 525, 504] payload_rows_per_dst=[5508, 5590, 5578, 5688, 5628, 5478, 5568, 5620] dup_rows_per_dst=[2562, 2476, 2744, 2586, 2676, 2528, 2604, 2702] combine_rows_per_src=[5508, 5590, 5578, 5688, 5628, 5478, 5568, 5620] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=277057536 dispatch_meta_bytes=227112 combine_payload_bytes=277057536 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277284648 ingress_bytes=387158016 link_bytes=387158016 token_bytes=554342184 weight_movement_bytes=110100480 total_nvlink_bytes=664442664 weight_to_token_ratio=0.198615 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.733314664e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279508992 dispatch_meta_bytes=229512 combine_payload_bytes=279508992 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279738504 ingress_bytes=609810432 link_bytes=609810432 token_bytes=559247496 weight_movement_bytes=330301440 total_nvlink_bytes=889548936 weight_to_token_ratio=0.590618 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060560638e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44458 payload_rows_remote=38918 dup_rows=21078 dup_rows_remote=18398 dedup_ratio=0.321625 router_tokens_per_expert=[514, 509, 481, 502, 485, 486, 501, 528, 492, 525, 490, 516, 500, 515, 543, 500, 505, 484, 468, 516, 481, 519, 504, 519, 550, 559, 505, 472, 520, 504, 498, 502, 529, 531, 529, 540, 519, 488, 500, 566, 517, 500, 515, 534, 526, 541, 501, 513, 515, 522, 539, 521, 468, 467, 464, 521, 527, 503, 528, 553, 525, 535, 504, 534] payload_rows_per_dst=[5448, 5564, 5516, 5540, 5652, 5634, 5528, 5576] dup_rows_per_dst=[2564, 2598, 2476, 2680, 2752, 2660, 2506, 2842] combine_rows_per_src=[5448, 5564, 5516, 5540, 5652, 5634, 5528, 5576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44466 payload_rows_remote=38916 dup_rows=21070 dup_rows_remote=18396 dedup_ratio=0.321503 router_tokens_per_expert=[566, 488, 504, 490, 537, 502, 499, 511, 515, 483, 535, 524, 498, 561, 467, 486, 500, 481, 542, 529, 493, 462, 515, 492, 506, 517, 544, 532, 498, 482, 493, 501, 558, 566, 528, 517, 505, 514, 523, 515, 515, 531, 490, 554, 533, 502, 506, 511, 551, 497, 504, 505, 475, 540, 504, 514, 524, 519, 539, 522, 507, 483, 486, 477] payload_rows_per_dst=[5566, 5548, 5458, 5564, 5710, 5542, 5528, 5550] dup_rows_per_dst=[2628, 2590, 2570, 2582, 2742, 2632, 2652, 2674] combine_rows_per_src=[5566, 5548, 5458, 5564, 5710, 5542, 5528, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279007232 dispatch_meta_bytes=229608 combine_payload_bytes=279007232 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279236840 ingress_bytes=499208192 link_bytes=499208192 token_bytes=558244072 weight_movement_bytes=220200960 total_nvlink_bytes=778445032 weight_to_token_ratio=0.394453 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.682051515e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44678 payload_rows_remote=39196 dup_rows=20858 dup_rows_remote=18292 dedup_ratio=0.318268 router_tokens_per_expert=[550, 519, 515, 536, 530, 520, 513, 528, 495, 524, 520, 494, 515, 510, 474, 492, 503, 493, 535, 526, 533, 510, 536, 489, 495, 531, 546, 531, 494, 519, 461, 504, 536, 503, 514, 497, 483, 484, 520, 463, 487, 515, 521, 505, 532, 510, 488, 506, 515, 491, 520, 515, 536, 492, 500, 503, 560, 511, 537, 524, 511, 479, 547, 522] payload_rows_per_dst=[5744, 5482, 5632, 5542, 5526, 5532, 5542, 5678] dup_rows_per_dst=[2678, 2566, 2618, 2620, 2474, 2596, 2602, 2704] combine_rows_per_src=[5744, 5482, 5632, 5542, 5526, 5532, 5542, 5678] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278620160 dispatch_meta_bytes=229800 combine_payload_bytes=278620160 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=278849960 ingress_bytes=498821120 link_bytes=498821120 token_bytes=557470120 weight_movement_bytes=220200960 total_nvlink_bytes=777671080 weight_to_token_ratio=0.395000 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.675319697e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=280125440 dispatch_meta_bytes=228856 combine_payload_bytes=280125440 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280354296 ingress_bytes=390225920 link_bytes=390225920 token_bytes=560479736 weight_movement_bytes=110100480 total_nvlink_bytes=670580216 weight_to_token_ratio=0.196440 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.786670560e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278964224 dispatch_meta_bytes=229264 combine_payload_bytes=278964224 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279193488 ingress_bytes=499165184 link_bytes=499165184 token_bytes=558157712 weight_movement_bytes=220200960 total_nvlink_bytes=778358672 weight_to_token_ratio=0.394514 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.681303535e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278949888 dispatch_meta_bytes=229248 combine_payload_bytes=278949888 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279179136 ingress_bytes=499150848 link_bytes=499150848 token_bytes=558129024 weight_movement_bytes=220200960 total_nvlink_bytes=778329984 weight_to_token_ratio=0.394534 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.681054209e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=280956928 dispatch_meta_bytes=229952 combine_payload_bytes=280956928 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281186880 ingress_bytes=391057408 link_bytes=391057408 token_bytes=562143808 weight_movement_bytes=110100480 total_nvlink_bytes=672244288 weight_to_token_ratio=0.195858 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.801131504e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44670 payload_rows_remote=39046 dup_rows=20866 dup_rows_remote=18106 dedup_ratio=0.318390 router_tokens_per_expert=[472, 505, 498, 490, 508, 506, 553, 507, 465, 488, 527, 508, 518, 488, 499, 508, 494, 508, 524, 524, 518, 544, 510, 476, 533, 488, 491, 512, 530, 519, 516, 505, 535, 512, 517, 539, 530, 514, 471, 522, 500, 481, 505, 519, 517, 513, 545, 548, 525, 513, 544, 539, 492, 493, 537, 498, 522, 530, 510, 551, 520, 513, 485, 496] payload_rows_per_dst=[5624, 5588, 5718, 5588, 5674, 5566, 5504, 5408] dup_rows_per_dst=[2760, 2734, 2760, 2622, 2584, 2542, 2472, 2392] combine_rows_per_src=[5624, 5588, 5718, 5588, 5674, 5566, 5504, 5408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44510 payload_rows_remote=38880 dup_rows=21026 dup_rows_remote=18308 dedup_ratio=0.320831 router_tokens_per_expert=[532, 503, 471, 515, 519, 490, 476, 499, 531, 495, 492, 526, 506, 515, 503, 524, 517, 525, 485, 540, 520, 507, 480, 523, 509, 513, 522, 534, 477, 542, 476, 538, 520, 503, 481, 513, 534, 522, 526, 575, 517, 513, 534, 515, 541, 513, 501, 511, 526, 490, 500, 526, 498, 519, 519, 522, 508, 507, 489, 480, 511, 526, 509, 514] payload_rows_per_dst=[5422, 5576, 5536, 5656, 5630, 5660, 5552, 5478] dup_rows_per_dst=[2588, 2608, 2658, 2566, 2718, 2630, 2648, 2610] combine_rows_per_src=[5422, 5576, 5536, 5656, 5630, 5660, 5552, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44532 payload_rows_remote=38942 dup_rows=21004 dup_rows_remote=18462 dedup_ratio=0.320496 router_tokens_per_expert=[485, 517, 482, 501, 511, 493, 542, 510, 509, 522, 496, 503, 556, 512, 513, 527, 525, 492, 549, 520, 487, 463, 495, 507, 496, 485, 524, 525, 520, 511, 542, 499, 546, 519, 519, 513, 507, 506, 492, 519, 469, 504, 549, 506, 517, 526, 513, 482, 523, 526, 525, 525, 500, 496, 536, 530, 490, 522, 513, 490, 531, 512, 515, 528] payload_rows_per_dst=[5508, 5580, 5460, 5612, 5578, 5590, 5650, 5554] dup_rows_per_dst=[2574, 2696, 2616, 2592, 2664, 2542, 2672, 2648] combine_rows_per_src=[5508, 5580, 5460, 5612, 5578, 5590, 5650, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5588691296 prefetch_bytes=990904320 reduce_bytes=330301440 weight_movement_bytes=1321205760 egress_bytes=2795493728 ingress_bytes=4114403328 link_bytes=4114403328 total_nvlink_bytes=6909897056 weight_to_token_ratio=0.236407 prefetch_share_of_ingress=0.240838 dedup_ratio=0.320923 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=3 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043382 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.155624090e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44314 payload_rows_remote=38724 dup_rows=21222 dup_rows_remote=18490 dedup_ratio=0.323822 router_tokens_per_expert=[503, 494, 515, 514, 506, 509, 499, 499, 480, 504, 492, 522, 504, 528, 527, 538, 522, 499, 528, 501, 524, 491, 511, 505, 533, 527, 497, 558, 484, 516, 467, 491, 495, 549, 516, 526, 487, 528, 513, 502, 470, 505, 533, 501, 499, 487, 498, 512, 543, 528, 517, 548, 509, 504, 468, 544, 523, 506, 549, 531, 502, 532, 520, 535] payload_rows_per_dst=[5566, 5526, 5556, 5578, 5502, 5406, 5590, 5590] dup_rows_per_dst=[2512, 2664, 2606, 2568, 2730, 2604, 2732, 2806] combine_rows_per_src=[5566, 5526, 5556, 5578, 5502, 5406, 5590, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279881728 dispatch_meta_bytes=228608 combine_payload_bytes=279881728 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280110336 ingress_bytes=389982208 link_bytes=389982208 token_bytes=559992064 weight_movement_bytes=110100480 total_nvlink_bytes=670092544 weight_to_token_ratio=0.196611 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.782432008e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44470 payload_rows_remote=38874 dup_rows=21066 dup_rows_remote=18418 dedup_ratio=0.321442 router_tokens_per_expert=[518, 505, 551, 491, 555, 549, 460, 500, 546, 490, 503, 548, 482, 503, 482, 531, 518, 493, 512, 510, 540, 508, 511, 530, 530, 491, 519, 501, 509, 482, 530, 528, 494, 518, 468, 568, 484, 537, 527, 499, 500, 519, 544, 521, 496, 502, 504, 536, 517, 510, 513, 498, 482, 527, 488, 526, 521, 496, 485, 500, 545, 503, 491, 523] payload_rows_per_dst=[5574, 5506, 5596, 5522, 5554, 5578, 5570, 5570] dup_rows_per_dst=[2684, 2664, 2648, 2658, 2636, 2666, 2552, 2558] combine_rows_per_src=[5574, 5506, 5596, 5522, 5554, 5578, 5570, 5570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278691840 dispatch_meta_bytes=228752 combine_payload_bytes=278691840 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278920592 ingress_bytes=608993280 link_bytes=608993280 token_bytes=557612432 weight_movement_bytes=330301440 total_nvlink_bytes=887913872 weight_to_token_ratio=0.592349 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059139476e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44468 payload_rows_remote=38958 dup_rows=21068 dup_rows_remote=18424 dedup_ratio=0.321472 router_tokens_per_expert=[518, 479, 507, 474, 545, 537, 483, 518, 515, 528, 505, 507, 494, 498, 522, 496, 487, 538, 523, 498, 506, 502, 495, 530, 519, 530, 514, 514, 488, 472, 508, 532, 539, 506, 499, 515, 499, 536, 526, 552, 530, 511, 547, 506, 515, 482, 543, 526, 499, 531, 472, 501, 484, 527, 508, 532, 518, 489, 495, 481, 532, 522, 525, 538] payload_rows_per_dst=[5506, 5536, 5580, 5510, 5596, 5640, 5518, 5582] dup_rows_per_dst=[2616, 2594, 2578, 2644, 2748, 2680, 2590, 2618] combine_rows_per_src=[5506, 5536, 5580, 5510, 5596, 5640, 5518, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44388 payload_rows_remote=38700 dup_rows=21148 dup_rows_remote=18496 dedup_ratio=0.322693 router_tokens_per_expert=[536, 540, 469, 476, 509, 556, 540, 464, 525, 481, 531, 517, 489, 522, 509, 534, 511, 478, 484, 500, 481, 533, 498, 502, 494, 512, 518, 499, 469, 525, 576, 536, 511, 490, 479, 502, 514, 519, 502, 482, 518, 497, 503, 514, 528, 552, 516, 524, 536, 509, 527, 515, 477, 530, 545, 496, 508, 555, 520, 532, 505, 497, 532, 519] payload_rows_per_dst=[5562, 5484, 5436, 5594, 5398, 5574, 5652, 5688] dup_rows_per_dst=[2618, 2732, 2538, 2664, 2524, 2730, 2690, 2652] combine_rows_per_src=[5562, 5484, 5436, 5594, 5398, 5574, 5652, 5688] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279136256 dispatch_meta_bytes=229616 combine_payload_bytes=279136256 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279365872 ingress_bytes=499337216 link_bytes=499337216 token_bytes=558502128 weight_movement_bytes=220200960 total_nvlink_bytes=778703088 weight_to_token_ratio=0.394271 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.684295455e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=277573632 dispatch_meta_bytes=228856 combine_payload_bytes=277573632 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277802488 ingress_bytes=497774592 link_bytes=497774592 token_bytes=555376120 weight_movement_bytes=220200960 total_nvlink_bytes=775577080 weight_to_token_ratio=0.396490 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.657118853e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5563819744 prefetch_bytes=1101004800 reduce_bytes=550502400 weight_movement_bytes=1651507200 egress_bytes=2783051488 ingress_bytes=4432275456 link_bytes=4432275456 total_nvlink_bytes=7215326944 weight_to_token_ratio=0.296830 prefetch_share_of_ingress=0.248406 dedup_ratio=0.321487 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=5 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.708456002e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278648832 dispatch_meta_bytes=229168 combine_payload_bytes=278648832 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278878000 ingress_bytes=388749312 link_bytes=388749312 token_bytes=557526832 weight_movement_bytes=110100480 total_nvlink_bytes=667627312 weight_to_token_ratio=0.197480 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.760989918e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5583098752 prefetch_bytes=660602880 reduce_bytes=1321205760 weight_movement_bytes=1981808640 egress_bytes=2792696704 ingress_bytes=4772210688 link_bytes=4772440184 total_nvlink_bytes=7564907392 weight_to_token_ratio=0.354966 prefetch_share_of_ingress=0.138427 dedup_ratio=0.321518 prefetch_slots=12 unused_prefetch_slots=148 slot_utilization=0.075000 reduce_slots=12 remote_demand=6 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12192 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.300058412e-05 prefetch_bytes_per_gemm_flop=1.148897059e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279250944 dispatch_meta_bytes=229528 combine_payload_bytes=279250944 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279480472 ingress_bytes=389351424 link_bytes=389351424 token_bytes=558731416 weight_movement_bytes=110100480 total_nvlink_bytes=668831896 weight_to_token_ratio=0.197054 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.771461636e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=277401600 dispatch_meta_bytes=228784 combine_payload_bytes=277401600 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=277630384 ingress_bytes=607703040 link_bytes=607703040 token_bytes=555031984 weight_movement_bytes=330301440 total_nvlink_bytes=885333424 weight_to_token_ratio=0.595103 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.056895537e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5583012056 prefetch_bytes=990904320 reduce_bytes=1101004800 weight_movement_bytes=2091909120 egress_bytes=2792653016 ingress_bytes=4882268160 link_bytes=4882268160 total_nvlink_bytes=7674921176 weight_to_token_ratio=0.374692 prefetch_share_of_ingress=0.202960 dedup_ratio=0.321088 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=10 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.491067325e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5582725200 prefetch_bytes=660602880 reduce_bytes=990904320 weight_movement_bytes=1651507200 egress_bytes=2792509520 ingress_bytes=4441722880 link_bytes=4441722880 total_nvlink_bytes=7234232400 weight_to_token_ratio=0.295825 prefetch_share_of_ingress=0.148727 dedup_ratio=0.321011 prefetch_slots=12 unused_prefetch_slots=148 slot_utilization=0.075000 reduce_slots=9 remote_demand=6 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=12192 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.724886626e-05 prefetch_bytes_per_gemm_flop=1.148897059e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5582837400 prefetch_bytes=1101004800 reduce_bytes=770703360 weight_movement_bytes=1871708160 egress_bytes=2792564376 ingress_bytes=4661981184 link_bytes=4661981184 total_nvlink_bytes=7454545560 weight_to_token_ratio=0.335261 prefetch_share_of_ingress=0.236167 dedup_ratio=0.320651 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=7 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.107952043e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5590639264 prefetch_bytes=1101004800 reduce_bytes=880803840 weight_movement_bytes=1981808640 egress_bytes=2796466848 ingress_bytes=4775981056 link_bytes=4775981056 total_nvlink_bytes=7572447904 weight_to_token_ratio=0.354487 prefetch_share_of_ingress=0.230530 dedup_ratio=0.320770 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=8 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040441 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.306216570e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5570045760 prefetch_bytes=990904320 reduce_bytes=1651507200 weight_movement_bytes=2642411520 egress_bytes=2786166592 ingress_bytes=5426290688 link_bytes=5426290688 total_nvlink_bytes=8212457280 weight_to_token_ratio=0.474397 prefetch_share_of_ingress=0.182612 dedup_ratio=0.321707 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=15 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.437211978e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none + [2026-08-26 16:00:26] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 362.8 | throughput per GPU (TFLOP/s/GPU): 340.4 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.515411E+00 | loss scale: 1.0 | grad norm: 30.540 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44636 payload_rows_remote=39126 dup_rows=20900 dup_rows_remote=18358 dedup_ratio=0.318909 router_tokens_per_expert=[573, 511, 503, 499, 508, 515, 513, 508, 524, 496, 497, 531, 453, 485, 505, 535, 494, 522, 513, 539, 496, 505, 512, 558, 509, 489, 471, 495, 479, 500, 558, 534, 521, 501, 504, 502, 536, 529, 512, 517, 503, 507, 533, 514, 512, 509, 525, 500, 530, 552, 491, 525, 565, 480, 490, 522, 522, 489, 502, 511, 475, 535, 529, 495] payload_rows_per_dst=[5610, 5510, 5656, 5500, 5578, 5602, 5626, 5554] dup_rows_per_dst=[2650, 2542, 2622, 2570, 2666, 2604, 2684, 2562] combine_rows_per_src=[5610, 5510, 5656, 5500, 5578, 5602, 5626, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44310 payload_rows_remote=38742 dup_rows=21226 dup_rows_remote=18492 dedup_ratio=0.323883 router_tokens_per_expert=[518, 507, 504, 511, 454, 514, 511, 512, 504, 514, 508, 483, 536, 569, 499, 472, 487, 468, 545, 519, 516, 495, 508, 498, 527, 525, 541, 531, 514, 522, 504, 547, 500, 480, 508, 522, 504, 561, 510, 494, 526, 529, 535, 532, 495, 497, 544, 493, 484, 505, 522, 515, 509, 496, 511, 521, 519, 497, 488, 497, 531, 546, 534, 500] payload_rows_per_dst=[5456, 5506, 5520, 5634, 5562, 5568, 5516, 5548] dup_rows_per_dst=[2606, 2664, 2552, 2788, 2596, 2734, 2610, 2676] combine_rows_per_src=[5456, 5506, 5520, 5634, 5562, 5568, 5516, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44394 payload_rows_remote=38668 dup_rows=21142 dup_rows_remote=18128 dedup_ratio=0.322601 router_tokens_per_expert=[526, 522, 534, 495, 499, 513, 517, 517, 505, 530, 516, 506, 549, 509, 522, 522, 483, 483, 544, 521, 535, 514, 485, 452, 551, 540, 511, 539, 509, 488, 493, 516, 515, 499, 531, 474, 515, 530, 515, 492, 504, 510, 507, 539, 479, 497, 525, 489, 522, 523, 525, 503, 498, 495, 520, 486, 502, 527, 514, 493, 518, 520, 544, 511] payload_rows_per_dst=[5630, 5592, 5542, 5284, 5464, 5560, 5596, 5726] dup_rows_per_dst=[2616, 2726, 2492, 2298, 2688, 2576, 2732, 3014] combine_rows_per_src=[5630, 5592, 5542, 5284, 5464, 5560, 5596, 5726] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44510 payload_rows_remote=38888 dup_rows=21026 dup_rows_remote=18380 dedup_ratio=0.320831 router_tokens_per_expert=[521, 510, 480, 520, 528, 513, 510, 506, 481, 493, 540, 556, 504, 516, 512, 510, 516, 500, 551, 566, 502, 522, 495, 480, 547, 507, 523, 496, 530, 481, 538, 490, 511, 508, 498, 506, 528, 518, 514, 551, 517, 546, 489, 548, 529, 550, 505, 494, 467, 484, 500, 530, 490, 495, 485, 497, 506, 504, 524, 493, 535, 461, 547, 494] payload_rows_per_dst=[5582, 5558, 5586, 5524, 5622, 5646, 5470, 5522] dup_rows_per_dst=[2594, 2666, 2678, 2700, 2646, 2710, 2426, 2606] combine_rows_per_src=[5582, 5558, 5586, 5524, 5622, 5646, 5470, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44468 payload_rows_remote=38886 dup_rows=21068 dup_rows_remote=18304 dedup_ratio=0.321472 router_tokens_per_expert=[494, 534, 523, 505, 494, 491, 487, 501, 506, 522, 483, 466, 511, 504, 514, 511, 521, 504, 499, 502, 506, 519, 494, 524, 512, 516, 509, 503, 507, 556, 516, 554, 519, 493, 531, 540, 484, 518, 489, 532, 509, 551, 491, 542, 548, 486, 504, 550, 527, 536, 509, 485, 506, 504, 549, 514, 482, 488, 493, 507, 539, 514, 509, 531] payload_rows_per_dst=[5536, 5444, 5578, 5582, 5596, 5632, 5546, 5554] dup_rows_per_dst=[2522, 2590, 2560, 2764, 2616, 2730, 2714, 2572] combine_rows_per_src=[5536, 5444, 5578, 5582, 5596, 5632, 5546, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44494 payload_rows_remote=38960 dup_rows=21042 dup_rows_remote=18460 dedup_ratio=0.321075 router_tokens_per_expert=[475, 541, 493, 504, 534, 458, 518, 490, 489, 497, 529, 483, 545, 512, 529, 527, 495, 486, 503, 506, 526, 515, 524, 481, 525, 537, 520, 542, 527, 510, 518, 526, 531, 545, 484, 497, 511, 493, 551, 516, 531, 540, 500, 453, 564, 530, 539, 525, 494, 518, 515, 492, 502, 529, 544, 464, 477, 516, 524, 516, 496, 520, 510, 476] payload_rows_per_dst=[5522, 5556, 5512, 5648, 5572, 5652, 5534, 5498] dup_rows_per_dst=[2504, 2666, 2560, 2762, 2684, 2712, 2582, 2572] combine_rows_per_src=[5522, 5556, 5512, 5648, 5572, 5652, 5534, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44536 payload_rows_remote=38872 dup_rows=21000 dup_rows_remote=18512 dedup_ratio=0.320435 router_tokens_per_expert=[509, 486, 497, 485, 489, 515, 474, 531, 526, 514, 540, 538, 523, 484, 565, 508, 499, 545, 510, 495, 508, 464, 519, 536, 528, 531, 526, 536, 507, 497, 490, 505, 494, 514, 502, 518, 519, 494, 508, 550, 503, 559, 509, 493, 503, 466, 518, 537, 484, 480, 507, 558, 513, 541, 520, 558, 491, 511, 504, 501, 466, 493, 529, 545] payload_rows_per_dst=[5438, 5612, 5664, 5564, 5540, 5546, 5642, 5530] dup_rows_per_dst=[2534, 2784, 2488, 2676, 2658, 2630, 2680, 2550] combine_rows_per_src=[5438, 5612, 5664, 5564, 5540, 5546, 5642, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278835200 dispatch_meta_bytes=229104 combine_payload_bytes=278835200 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279064304 ingress_bytes=499036160 link_bytes=499036160 token_bytes=557899504 weight_movement_bytes=220200960 total_nvlink_bytes=778100464 weight_to_token_ratio=0.394696 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.679059596e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=277731328 dispatch_meta_bytes=229000 combine_payload_bytes=277731328 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=277960328 ingress_bytes=718133248 link_bytes=718133248 token_bytes=555691656 weight_movement_bytes=440401920 total_nvlink_bytes=996093576 weight_to_token_ratio=0.792529 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.248951831e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44556 payload_rows_remote=38900 dup_rows=20980 dup_rows_remote=18204 dedup_ratio=0.320129 router_tokens_per_expert=[498, 544, 545, 487, 517, 510, 529, 525, 477, 465, 489, 512, 508, 498, 533, 514, 563, 529, 536, 512, 507, 531, 486, 481, 504, 496, 445, 526, 543, 545, 541, 528, 512, 519, 518, 502, 542, 555, 563, 514, 482, 485, 488, 493, 539, 519, 483, 543, 506, 518, 520, 489, 502, 517, 509, 514, 505, 447, 515, 486, 495, 516, 513, 535] payload_rows_per_dst=[5656, 5470, 5632, 5590, 5706, 5356, 5622, 5524] dup_rows_per_dst=[2776, 2534, 2780, 2680, 2744, 2438, 2528, 2500] combine_rows_per_src=[5656, 5470, 5632, 5590, 5706, 5356, 5622, 5524] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279336960 dispatch_meta_bytes=229856 combine_payload_bytes=279336960 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279566816 ingress_bytes=389437440 link_bytes=389437440 token_bytes=558903776 weight_movement_bytes=110100480 total_nvlink_bytes=669004256 weight_to_token_ratio=0.196994 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.772957596e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=276541440 dispatch_meta_bytes=226104 combine_payload_bytes=276541440 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276767544 ingress_bytes=386641920 link_bytes=386641920 token_bytes=553308984 weight_movement_bytes=110100480 total_nvlink_bytes=663409464 weight_to_token_ratio=0.198986 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.724338905e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278677504 dispatch_meta_bytes=229448 combine_payload_bytes=278677504 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278906952 ingress_bytes=388777984 link_bytes=388777984 token_bytes=557584456 weight_movement_bytes=110100480 total_nvlink_bytes=667684936 weight_to_token_ratio=0.197460 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.761488572e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44476 payload_rows_remote=38716 dup_rows=21060 dup_rows_remote=18084 dedup_ratio=0.321350 router_tokens_per_expert=[496, 543, 543, 485, 516, 506, 528, 521, 477, 463, 486, 511, 506, 497, 531, 511, 561, 528, 533, 512, 509, 532, 486, 477, 505, 494, 445, 525, 541, 545, 540, 526, 512, 520, 518, 502, 542, 554, 565, 514, 485, 486, 488, 491, 543, 523, 486, 545, 511, 522, 519, 491, 502, 518, 513, 513, 506, 449, 517, 488, 496, 518, 514, 538] payload_rows_per_dst=[5760, 5510, 5654, 5572, 5534, 5280, 5638, 5528] dup_rows_per_dst=[2976, 2610, 2872, 2670, 2514, 2354, 2540, 2524] combine_rows_per_src=[5760, 5510, 5654, 5572, 5534, 5280, 5638, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280455168 dispatch_meta_bytes=229936 combine_payload_bytes=280455168 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280685104 ingress_bytes=390555648 link_bytes=390555648 token_bytes=561140272 weight_movement_bytes=110100480 total_nvlink_bytes=671240752 weight_to_token_ratio=0.196208 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.792405072e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=277702656 dispatch_meta_bytes=228936 combine_payload_bytes=277702656 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277931592 ingress_bytes=497903616 link_bytes=497903616 token_bytes=555634248 weight_movement_bytes=220200960 total_nvlink_bytes=775835208 weight_to_token_ratio=0.396306 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.659362793e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278749184 dispatch_meta_bytes=229072 combine_payload_bytes=278749184 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278978256 ingress_bytes=609050624 link_bytes=609050624 token_bytes=557727440 weight_movement_bytes=330301440 total_nvlink_bytes=888028880 weight_to_token_ratio=0.592227 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059239207e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=277172224 dispatch_meta_bytes=227184 combine_payload_bytes=277172224 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277399408 ingress_bytes=387272704 link_bytes=387272704 token_bytes=554571632 weight_movement_bytes=110100480 total_nvlink_bytes=664672112 weight_to_token_ratio=0.198532 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.735309277e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278734848 dispatch_meta_bytes=228760 combine_payload_bytes=278734848 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=278963608 ingress_bytes=719136768 link_bytes=719136768 token_bytes=557698456 weight_movement_bytes=440401920 total_nvlink_bytes=998100376 weight_to_token_ratio=0.789677 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.250697117e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279265280 dispatch_meta_bytes=229680 combine_payload_bytes=279265280 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279494960 ingress_bytes=389365760 link_bytes=389365760 token_bytes=558760240 weight_movement_bytes=110100480 total_nvlink_bytes=668860720 weight_to_token_ratio=0.197044 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.771710963e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278634496 dispatch_meta_bytes=229536 combine_payload_bytes=278634496 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278864032 ingress_bytes=388734976 link_bytes=388734976 token_bytes=557498528 weight_movement_bytes=110100480 total_nvlink_bytes=667599008 weight_to_token_ratio=0.197490 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.760740592e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280297472 dispatch_meta_bytes=229824 combine_payload_bytes=280297472 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280527296 ingress_bytes=500498432 link_bytes=500498432 token_bytes=560824768 weight_movement_bytes=220200960 total_nvlink_bytes=781025728 weight_to_token_ratio=0.392638 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.704490911e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278835200 dispatch_meta_bytes=228416 combine_payload_bytes=278835200 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279063616 ingress_bytes=388935680 link_bytes=388935680 token_bytes=557898816 weight_movement_bytes=110100480 total_nvlink_bytes=667999296 weight_to_token_ratio=0.197348 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.764231164e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44360 payload_rows_remote=38726 dup_rows=21176 dup_rows_remote=18408 dedup_ratio=0.323120 router_tokens_per_expert=[540, 506, 487, 496, 503, 495, 482, 518, 507, 523, 490, 527, 476, 489, 497, 535, 517, 530, 526, 523, 528, 511, 504, 511, 533, 515, 512, 531, 521, 534, 520, 535, 509, 506, 517, 495, 524, 510, 531, 512, 507, 546, 502, 471, 529, 481, 519, 529, 495, 506, 514, 519, 520, 523, 496, 488, 525, 507, 526, 525, 508, 486, 492, 528] payload_rows_per_dst=[5442, 5508, 5604, 5634, 5542, 5546, 5552, 5532] dup_rows_per_dst=[2612, 2580, 2696, 2768, 2666, 2622, 2570, 2662] combine_rows_per_src=[5442, 5508, 5604, 5634, 5542, 5546, 5552, 5532] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44388 payload_rows_remote=38780 dup_rows=21148 dup_rows_remote=18380 dedup_ratio=0.322693 router_tokens_per_expert=[508, 532, 493, 517, 522, 473, 523, 514, 515, 557, 521, 530, 505, 509, 481, 545, 528, 514, 500, 520, 529, 545, 551, 491, 481, 484, 508, 510, 492, 523, 478, 479, 495, 507, 493, 535, 510, 502, 519, 522, 514, 546, 468, 556, 533, 530, 518, 523, 530, 547, 447, 513, 512, 505, 525, 482, 466, 508, 509, 535, 500, 480, 526, 534] payload_rows_per_dst=[5568, 5552, 5630, 5360, 5622, 5608, 5584, 5464] dup_rows_per_dst=[2596, 2774, 2726, 2550, 2544, 2768, 2538, 2652] combine_rows_per_src=[5568, 5552, 5630, 5360, 5622, 5608, 5584, 5464] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44380 payload_rows_remote=38870 dup_rows=21156 dup_rows_remote=18432 dedup_ratio=0.322815 router_tokens_per_expert=[514, 501, 526, 535, 506, 539, 479, 559, 484, 508, 535, 507, 496, 539, 515, 484, 482, 515, 496, 547, 523, 525, 494, 502, 539, 544, 476, 506, 506, 510, 547, 521, 503, 532, 528, 515, 490, 537, 515, 497, 530, 535, 552, 484, 487, 525, 500, 478, 515, 510, 527, 511, 503, 501, 510, 485, 514, 543, 508, 521, 480, 477, 475, 520] payload_rows_per_dst=[5652, 5490, 5544, 5644, 5510, 5518, 5546, 5476] dup_rows_per_dst=[2666, 2646, 2624, 2654, 2724, 2664, 2578, 2600] combine_rows_per_src=[5652, 5490, 5544, 5644, 5510, 5518, 5546, 5476] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44666 payload_rows_remote=39018 dup_rows=20870 dup_rows_remote=18174 dedup_ratio=0.318451 router_tokens_per_expert=[527, 533, 509, 548, 514, 529, 507, 503, 475, 490, 524, 522, 504, 517, 511, 512, 527, 504, 483, 515, 507, 541, 530, 513, 520, 542, 491, 538, 484, 469, 526, 512, 491, 535, 492, 524, 477, 516, 499, 482, 515, 522, 505, 495, 522, 490, 524, 498, 521, 498, 539, 506, 501, 516, 554, 537, 495, 503, 493, 515, 567, 524, 499, 486] payload_rows_per_dst=[5676, 5524, 5616, 5530, 5564, 5590, 5648, 5518] dup_rows_per_dst=[2664, 2586, 2624, 2634, 2468, 2552, 2696, 2646] combine_rows_per_src=[5676, 5524, 5616, 5530, 5564, 5590, 5648, 5518] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44116 payload_rows_remote=38376 dup_rows=21420 dup_rows_remote=18304 dedup_ratio=0.326843 router_tokens_per_expert=[547, 495, 516, 492, 503, 528, 502, 507, 500, 560, 533, 539, 487, 477, 526, 529, 527, 493, 506, 506, 513, 538, 502, 525, 465, 496, 552, 492, 509, 494, 492, 502, 509, 487, 513, 548, 535, 490, 523, 508, 534, 479, 478, 489, 496, 510, 543, 519, 480, 531, 488, 534, 517, 483, 530, 530, 507, 486, 533, 500, 540, 534, 523, 538] payload_rows_per_dst=[5280, 5540, 5442, 5516, 5576, 5494, 5528, 5740] dup_rows_per_dst=[2286, 2762, 2432, 2578, 2886, 2622, 2738, 3116] combine_rows_per_src=[5280, 5540, 5442, 5516, 5576, 5494, 5528, 5740] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277587968 dispatch_meta_bytes=228536 combine_payload_bytes=277587968 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277816504 ingress_bytes=387688448 link_bytes=387688448 token_bytes=555404472 weight_movement_bytes=110100480 total_nvlink_bytes=665504952 weight_to_token_ratio=0.198235 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.742539749e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44532 payload_rows_remote=38956 dup_rows=21004 dup_rows_remote=18374 dedup_ratio=0.320496 router_tokens_per_expert=[516, 528, 537, 530, 515, 535, 537, 504, 528, 517, 470, 533, 509, 531, 502, 503, 507, 488, 532, 509, 529, 500, 499, 539, 489, 529, 526, 539, 527, 517, 498, 498, 528, 489, 537, 512, 482, 486, 541, 512, 509, 514, 506, 529, 469, 445, 504, 535, 509, 508, 502, 521, 512, 486, 517, 502, 511, 514, 476, 539, 502, 514, 525, 511] payload_rows_per_dst=[5596, 5510, 5576, 5540, 5582, 5522, 5560, 5646] dup_rows_per_dst=[2808, 2676, 2630, 2706, 2592, 2500, 2554, 2538] combine_rows_per_src=[5596, 5510, 5576, 5540, 5582, 5522, 5560, 5646] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277975040 dispatch_meta_bytes=228640 combine_payload_bytes=277975040 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278203680 ingress_bytes=388075520 link_bytes=388075520 token_bytes=556178720 weight_movement_bytes=110100480 total_nvlink_bytes=666279200 weight_to_token_ratio=0.197959 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.749271567e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44250 payload_rows_remote=38734 dup_rows=21286 dup_rows_remote=18652 dedup_ratio=0.324799 router_tokens_per_expert=[553, 505, 554, 498, 493, 539, 530, 506, 512, 536, 513, 488, 491, 499, 528, 508, 482, 546, 545, 497, 544, 483, 549, 516, 515, 488, 490, 544, 514, 507, 495, 530, 459, 479, 524, 519, 505, 524, 515, 518, 545, 486, 537, 531, 518, 505, 509, 507, 545, 521, 503, 489, 501, 533, 507, 528, 485, 473, 480, 514, 531, 474, 493, 512] payload_rows_per_dst=[5616, 5516, 5606, 5476, 5456, 5608, 5548, 5424] dup_rows_per_dst=[2740, 2634, 2718, 2690, 2630, 2668, 2706, 2500] combine_rows_per_src=[5616, 5516, 5606, 5476, 5456, 5608, 5548, 5424] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278620160 dispatch_meta_bytes=229208 combine_payload_bytes=278620160 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278849368 ingress_bytes=388720640 link_bytes=388720640 token_bytes=557469528 weight_movement_bytes=110100480 total_nvlink_bytes=667570008 weight_to_token_ratio=0.197500 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.760491265e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279681024 dispatch_meta_bytes=228768 combine_payload_bytes=279681024 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279909792 ingress_bytes=389781504 link_bytes=389781504 token_bytes=559590816 weight_movement_bytes=110100480 total_nvlink_bytes=669691296 weight_to_token_ratio=0.196752 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778941435e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44504 payload_rows_remote=38872 dup_rows=21032 dup_rows_remote=18330 dedup_ratio=0.320923 router_tokens_per_expert=[528, 548, 520, 505, 518, 493, 558, 497, 510, 496, 497, 511, 507, 472, 521, 520, 517, 520, 509, 501, 506, 529, 506, 531, 524, 523, 537, 519, 482, 539, 509, 495, 542, 522, 512, 552, 498, 481, 507, 473, 523, 533, 526, 536, 465, 526, 510, 508, 521, 481, 518, 496, 527, 486, 531, 535, 490, 484, 533, 534, 496, 455, 535, 484] payload_rows_per_dst=[5632, 5602, 5548, 5580, 5548, 5550, 5580, 5464] dup_rows_per_dst=[2702, 2636, 2520, 2676, 2626, 2704, 2610, 2558] combine_rows_per_src=[5632, 5602, 5548, 5580, 5548, 5550, 5580, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=277516288 dispatch_meta_bytes=227200 combine_payload_bytes=277516288 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277743488 ingress_bytes=387616768 link_bytes=387616768 token_bytes=555259776 weight_movement_bytes=110100480 total_nvlink_bytes=665360256 weight_to_token_ratio=0.198286 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.741293116e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44368 payload_rows_remote=38760 dup_rows=21168 dup_rows_remote=18396 dedup_ratio=0.322998 router_tokens_per_expert=[506, 526, 491, 514, 517, 469, 521, 510, 512, 557, 519, 526, 503, 507, 477, 544, 527, 511, 500, 520, 530, 543, 551, 489, 480, 484, 509, 510, 490, 522, 479, 478, 495, 506, 496, 537, 510, 502, 520, 524, 513, 548, 470, 555, 530, 531, 522, 521, 534, 552, 451, 514, 512, 505, 527, 486, 470, 510, 511, 537, 503, 484, 528, 542] payload_rows_per_dst=[5548, 5540, 5624, 5352, 5622, 5608, 5598, 5476] dup_rows_per_dst=[2560, 2750, 2718, 2552, 2558, 2772, 2564, 2694] combine_rows_per_src=[5548, 5540, 5624, 5352, 5622, 5608, 5598, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44270 payload_rows_remote=38760 dup_rows=21266 dup_rows_remote=18656 dedup_ratio=0.324493 router_tokens_per_expert=[546, 501, 553, 497, 490, 536, 527, 501, 509, 533, 510, 487, 490, 498, 526, 507, 482, 546, 546, 496, 543, 483, 546, 515, 515, 485, 486, 543, 512, 507, 495, 530, 459, 481, 525, 523, 506, 525, 515, 523, 547, 488, 538, 530, 517, 506, 510, 508, 547, 520, 505, 491, 503, 537, 510, 531, 487, 474, 481, 517, 533, 477, 497, 516] payload_rows_per_dst=[5604, 5510, 5602, 5466, 5466, 5618, 5560, 5444] dup_rows_per_dst=[2698, 2610, 2712, 2680, 2648, 2670, 2728, 2520] combine_rows_per_src=[5604, 5510, 5602, 5466, 5466, 5618, 5560, 5444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44388 payload_rows_remote=38872 dup_rows=21148 dup_rows_remote=18412 dedup_ratio=0.322693 router_tokens_per_expert=[511, 498, 521, 529, 505, 538, 478, 558, 479, 508, 535, 503, 496, 531, 512, 486, 483, 510, 494, 545, 520, 523, 493, 499, 536, 541, 476, 509, 509, 509, 546, 521, 504, 533, 531, 513, 492, 540, 515, 498, 530, 539, 555, 485, 489, 525, 503, 480, 520, 512, 530, 508, 506, 501, 510, 487, 515, 547, 510, 526, 481, 479, 477, 525] payload_rows_per_dst=[5644, 5482, 5532, 5636, 5516, 5530, 5558, 5490] dup_rows_per_dst=[2632, 2618, 2602, 2658, 2736, 2682, 2590, 2630] combine_rows_per_src=[5644, 5482, 5532, 5636, 5516, 5530, 5558, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44180 payload_rows_remote=38548 dup_rows=21356 dup_rows_remote=18444 dedup_ratio=0.325867 router_tokens_per_expert=[546, 490, 516, 489, 503, 524, 501, 506, 499, 557, 531, 536, 484, 473, 527, 526, 527, 490, 505, 504, 513, 539, 502, 526, 466, 498, 549, 492, 508, 494, 492, 500, 510, 489, 514, 547, 536, 488, 522, 506, 536, 478, 477, 491, 500, 508, 543, 523, 479, 533, 492, 536, 518, 487, 529, 532, 511, 489, 534, 500, 541, 537, 526, 543] payload_rows_per_dst=[5436, 5524, 5602, 5480, 5486, 5508, 5512, 5632] dup_rows_per_dst=[2460, 2742, 2620, 2518, 2738, 2658, 2708, 2912] combine_rows_per_src=[5436, 5524, 5602, 5480, 5486, 5508, 5512, 5632] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44386 payload_rows_remote=38746 dup_rows=21150 dup_rows_remote=18390 dedup_ratio=0.322723 router_tokens_per_expert=[538, 503, 484, 494, 496, 491, 479, 514, 507, 519, 488, 526, 475, 487, 496, 535, 516, 528, 526, 522, 527, 513, 502, 509, 532, 513, 511, 533, 521, 533, 523, 534, 508, 507, 520, 496, 526, 511, 533, 510, 507, 551, 502, 469, 533, 482, 522, 529, 496, 507, 513, 521, 524, 521, 497, 492, 529, 507, 530, 529, 509, 487, 494, 531] payload_rows_per_dst=[5420, 5504, 5602, 5640, 5550, 5562, 5556, 5552] dup_rows_per_dst=[2578, 2562, 2684, 2760, 2672, 2628, 2586, 2680] combine_rows_per_src=[5420, 5504, 5602, 5640, 5550, 5562, 5556, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44518 payload_rows_remote=38946 dup_rows=21018 dup_rows_remote=18398 dedup_ratio=0.320709 router_tokens_per_expert=[513, 525, 536, 528, 511, 535, 534, 498, 522, 516, 469, 532, 509, 529, 500, 500, 508, 487, 531, 506, 527, 499, 499, 539, 488, 528, 526, 535, 527, 519, 498, 497, 528, 488, 538, 513, 483, 485, 538, 513, 510, 513, 507, 529, 470, 443, 505, 538, 511, 510, 503, 520, 514, 494, 520, 506, 515, 520, 477, 542, 506, 515, 526, 517] payload_rows_per_dst=[5580, 5504, 5572, 5532, 5578, 5524, 5572, 5656] dup_rows_per_dst=[2780, 2650, 2620, 2704, 2594, 2506, 2584, 2580] combine_rows_per_src=[5580, 5504, 5572, 5532, 5578, 5524, 5572, 5656] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44628 payload_rows_remote=38978 dup_rows=20908 dup_rows_remote=18198 dedup_ratio=0.319031 router_tokens_per_expert=[524, 532, 507, 546, 511, 526, 502, 504, 472, 487, 524, 520, 500, 518, 510, 513, 528, 500, 482, 513, 510, 542, 530, 511, 518, 540, 491, 536, 483, 467, 522, 514, 491, 538, 491, 526, 478, 516, 499, 482, 515, 521, 507, 495, 523, 491, 527, 500, 521, 499, 541, 505, 500, 519, 555, 540, 500, 504, 496, 517, 572, 527, 500, 489] payload_rows_per_dst=[5658, 5516, 5608, 5512, 5556, 5596, 5650, 5532] dup_rows_per_dst=[2646, 2572, 2624, 2630, 2486, 2562, 2710, 2678] combine_rows_per_src=[5658, 5516, 5608, 5512, 5556, 5596, 5650, 5532] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=275079168 dispatch_meta_bytes=226720 combine_payload_bytes=275079168 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=275305888 ingress_bytes=385179648 link_bytes=385179648 token_bytes=550385056 weight_movement_bytes=110100480 total_nvlink_bytes=660485536 weight_to_token_ratio=0.200043 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.698907590e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279236608 dispatch_meta_bytes=229320 combine_payload_bytes=279236608 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=279465928 ingress_bytes=719638528 link_bytes=719638528 token_bytes=558702536 weight_movement_bytes=440401920 total_nvlink_bytes=999104456 weight_to_token_ratio=0.788258 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.251569760e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277645312 dispatch_meta_bytes=229544 combine_payload_bytes=277645312 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277874856 ingress_bytes=387745792 link_bytes=387745792 token_bytes=555520168 weight_movement_bytes=110100480 total_nvlink_bytes=665620648 weight_to_token_ratio=0.198193 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.743537055e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44466 payload_rows_remote=38888 dup_rows=21070 dup_rows_remote=18362 dedup_ratio=0.321503 router_tokens_per_expert=[509, 519, 527, 497, 496, 532, 506, 532, 494, 492, 543, 539, 525, 486, 481, 516, 529, 507, 512, 522, 497, 546, 504, 533, 517, 543, 500, 517, 514, 513, 508, 531, 518, 504, 490, 535, 517, 497, 506, 513, 532, 527, 525, 535, 510, 533, 509, 528, 502, 483, 519, 495, 482, 495, 499, 480, 536, 500, 529, 535, 480, 506, 475, 486] payload_rows_per_dst=[5596, 5506, 5598, 5578, 5506, 5684, 5440, 5558] dup_rows_per_dst=[2640, 2646, 2702, 2708, 2654, 2714, 2470, 2536] combine_rows_per_src=[5596, 5506, 5598, 5578, 5506, 5684, 5440, 5558] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278634496 dispatch_meta_bytes=228808 combine_payload_bytes=278634496 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=278863304 ingress_bytes=498835456 link_bytes=498835456 token_bytes=557497800 weight_movement_bytes=220200960 total_nvlink_bytes=777698760 weight_to_token_ratio=0.394981 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.675569023e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44504 payload_rows_remote=38898 dup_rows=21032 dup_rows_remote=18432 dedup_ratio=0.320923 router_tokens_per_expert=[509, 495, 491, 498, 487, 512, 533, 489, 518, 509, 539, 503, 493, 482, 516, 538, 528, 568, 531, 529, 492, 548, 471, 488, 501, 514, 481, 519, 562, 531, 486, 508, 551, 551, 475, 492, 525, 512, 484, 481, 502, 489, 512, 514, 528, 502, 521, 535, 515, 509, 544, 519, 497, 525, 518, 537, 501, 524, 516, 523, 492, 517, 471, 517] payload_rows_per_dst=[5394, 5594, 5656, 5596, 5530, 5606, 5654, 5474] dup_rows_per_dst=[2634, 2602, 2654, 2608, 2612, 2600, 2674, 2648] combine_rows_per_src=[5394, 5594, 5656, 5596, 5530, 5606, 5654, 5474] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44484 payload_rows_remote=38882 dup_rows=21052 dup_rows_remote=18368 dedup_ratio=0.321228 router_tokens_per_expert=[495, 507, 537, 525, 540, 485, 487, 490, 523, 513, 530, 484, 498, 465, 522, 524, 531, 500, 537, 479, 516, 511, 491, 491, 543, 507, 501, 489, 513, 554, 508, 498, 527, 537, 500, 495, 500, 539, 477, 568, 497, 496, 532, 540, 484, 480, 501, 495, 532, 547, 506, 506, 529, 500, 529, 522, 540, 527, 519, 513, 476, 531, 523, 506] payload_rows_per_dst=[5544, 5550, 5534, 5598, 5602, 5476, 5576, 5604] dup_rows_per_dst=[2588, 2568, 2578, 2628, 2684, 2574, 2766, 2666] combine_rows_per_src=[5544, 5550, 5534, 5598, 5602, 5476, 5576, 5604] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278749184 dispatch_meta_bytes=229000 combine_payload_bytes=278749184 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278978184 ingress_bytes=388849664 link_bytes=388849664 token_bytes=557727368 weight_movement_bytes=110100480 total_nvlink_bytes=667827848 weight_to_token_ratio=0.197409 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.762735205e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44588 payload_rows_remote=39070 dup_rows=20948 dup_rows_remote=18454 dedup_ratio=0.319641 router_tokens_per_expert=[539, 524, 482, 521, 540, 522, 476, 502, 514, 506, 507, 520, 577, 520, 505, 504, 480, 523, 535, 495, 520, 500, 538, 497, 523, 534, 476, 537, 487, 493, 499, 486, 536, 501, 546, 466, 531, 525, 550, 503, 484, 483, 510, 525, 556, 513, 491, 495, 512, 464, 509, 511, 511, 468, 513, 518, 557, 500, 518, 519, 524, 508, 547, 492] payload_rows_per_dst=[5564, 5592, 5552, 5556, 5682, 5536, 5518, 5588] dup_rows_per_dst=[2648, 2714, 2624, 2514, 2634, 2578, 2494, 2742] combine_rows_per_src=[5564, 5592, 5552, 5556, 5682, 5536, 5518, 5588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277831680 dispatch_meta_bytes=228624 combine_payload_bytes=277831680 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278060304 ingress_bytes=498032640 link_bytes=498032640 token_bytes=555891984 weight_movement_bytes=220200960 total_nvlink_bytes=776092944 weight_to_token_ratio=0.396122 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.661606733e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277831680 dispatch_meta_bytes=229664 combine_payload_bytes=277831680 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278061344 ingress_bytes=387932160 link_bytes=387932160 token_bytes=555893024 weight_movement_bytes=110100480 total_nvlink_bytes=665993504 weight_to_token_ratio=0.198061 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.746778301e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278634496 dispatch_meta_bytes=229136 combine_payload_bytes=278634496 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278863632 ingress_bytes=388734976 link_bytes=388734976 token_bytes=557498128 weight_movement_bytes=110100480 total_nvlink_bytes=667598608 weight_to_token_ratio=0.197490 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.760740592e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=276312064 dispatch_meta_bytes=227968 combine_payload_bytes=276312064 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276540032 ingress_bytes=386412544 link_bytes=386412544 token_bytes=552852096 weight_movement_bytes=110100480 total_nvlink_bytes=662952576 weight_to_token_ratio=0.199150 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.720349680e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277731328 dispatch_meta_bytes=228544 combine_payload_bytes=277731328 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277959872 ingress_bytes=387831808 link_bytes=387831808 token_bytes=555691200 weight_movement_bytes=110100480 total_nvlink_bytes=665791680 weight_to_token_ratio=0.198132 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.745033015e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44484 payload_rows_remote=38870 dup_rows=21052 dup_rows_remote=18374 dedup_ratio=0.321228 router_tokens_per_expert=[523, 545, 518, 505, 517, 490, 555, 493, 506, 495, 498, 509, 505, 472, 522, 517, 517, 518, 510, 497, 504, 527, 504, 532, 525, 523, 534, 518, 481, 542, 507, 495, 543, 522, 510, 551, 499, 477, 505, 474, 523, 533, 526, 539, 470, 529, 509, 510, 522, 484, 518, 500, 531, 487, 533, 539, 493, 485, 534, 535, 497, 459, 540, 487] payload_rows_per_dst=[5614, 5702, 5452, 5636, 5610, 5482, 5520, 5468] dup_rows_per_dst=[2678, 2750, 2362, 2780, 2720, 2630, 2540, 2592] combine_rows_per_src=[5614, 5702, 5452, 5636, 5610, 5482, 5520, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279164928 dispatch_meta_bytes=229376 combine_payload_bytes=279164928 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279394304 ingress_bytes=499365888 link_bytes=499365888 token_bytes=558559232 weight_movement_bytes=220200960 total_nvlink_bytes=778760192 weight_to_token_ratio=0.394230 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.684794108e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279394304 dispatch_meta_bytes=228704 combine_payload_bytes=279394304 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279623008 ingress_bytes=499595264 link_bytes=499595264 token_bytes=559017312 weight_movement_bytes=220200960 total_nvlink_bytes=779218272 weight_to_token_ratio=0.393907 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.688783334e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278620160 dispatch_meta_bytes=228976 combine_payload_bytes=278620160 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=278849136 ingress_bytes=719022080 link_bytes=719022080 token_bytes=557469296 weight_movement_bytes=440401920 total_nvlink_bytes=997871216 weight_to_token_ratio=0.790002 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.250497656e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44520 payload_rows_remote=38910 dup_rows=21016 dup_rows_remote=18400 dedup_ratio=0.320679 router_tokens_per_expert=[505, 492, 488, 494, 482, 509, 531, 486, 517, 507, 537, 501, 492, 480, 514, 538, 526, 566, 530, 527, 491, 548, 470, 488, 499, 517, 481, 518, 563, 531, 487, 507, 552, 552, 477, 490, 524, 511, 486, 484, 503, 491, 511, 515, 531, 502, 523, 537, 516, 509, 545, 522, 499, 527, 522, 537, 503, 525, 517, 529, 492, 519, 474, 521] payload_rows_per_dst=[5380, 5588, 5648, 5596, 5544, 5610, 5666, 5488] dup_rows_per_dst=[2594, 2584, 2644, 2610, 2608, 2616, 2688, 2672] combine_rows_per_src=[5380, 5588, 5648, 5596, 5544, 5610, 5666, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44506 payload_rows_remote=38900 dup_rows=21030 dup_rows_remote=18346 dedup_ratio=0.320892 router_tokens_per_expert=[494, 504, 535, 521, 540, 482, 486, 489, 521, 511, 527, 483, 496, 463, 520, 523, 526, 498, 534, 479, 515, 510, 488, 489, 545, 506, 501, 490, 515, 556, 507, 496, 527, 537, 500, 495, 500, 537, 478, 571, 498, 498, 532, 542, 485, 481, 500, 497, 532, 550, 506, 509, 531, 501, 532, 523, 543, 530, 522, 514, 478, 533, 526, 510] payload_rows_per_dst=[5532, 5538, 5520, 5610, 5606, 5486, 5590, 5624] dup_rows_per_dst=[2570, 2550, 2558, 2622, 2684, 2580, 2778, 2688] combine_rows_per_src=[5532, 5538, 5520, 5610, 5606, 5486, 5590, 5624] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44394 payload_rows_remote=38914 dup_rows=21142 dup_rows_remote=18644 dedup_ratio=0.322601 router_tokens_per_expert=[518, 541, 520, 558, 512, 532, 536, 531, 441, 525, 529, 543, 528, 495, 539, 543, 515, 539, 543, 487, 519, 504, 517, 507, 522, 528, 503, 504, 511, 494, 512, 509, 500, 512, 537, 494, 517, 536, 531, 517, 495, 503, 473, 523, 508, 521, 502, 431, 515, 488, 485, 506, 495, 522, 538, 525, 517, 515, 481, 519, 541, 488, 477, 451] payload_rows_per_dst=[5564, 5518, 5494, 5562, 5636, 5496, 5644, 5480] dup_rows_per_dst=[2708, 2658, 2560, 2656, 2808, 2564, 2690, 2498] combine_rows_per_src=[5564, 5518, 5494, 5562, 5636, 5496, 5644, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44582 payload_rows_remote=38930 dup_rows=20954 dup_rows_remote=18168 dedup_ratio=0.319733 router_tokens_per_expert=[522, 528, 529, 533, 525, 515, 490, 528, 544, 551, 529, 525, 536, 516, 499, 519, 502, 512, 510, 550, 519, 513, 532, 553, 532, 500, 522, 477, 478, 493, 507, 542, 531, 496, 511, 505, 513, 486, 488, 536, 490, 488, 515, 515, 545, 478, 495, 532, 536, 525, 496, 488, 479, 507, 505, 530, 476, 479, 475, 493, 468, 505, 527, 524] payload_rows_per_dst=[5688, 5652, 5684, 5534, 5524, 5528, 5518, 5454] dup_rows_per_dst=[2652, 2786, 2698, 2568, 2608, 2588, 2614, 2440] combine_rows_per_src=[5688, 5652, 5684, 5534, 5524, 5528, 5518, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44454 payload_rows_remote=38870 dup_rows=21082 dup_rows_remote=18378 dedup_ratio=0.321686 router_tokens_per_expert=[506, 510, 520, 493, 496, 531, 502, 532, 493, 490, 541, 536, 525, 486, 476, 514, 526, 507, 509, 525, 497, 543, 501, 529, 515, 546, 498, 515, 516, 513, 509, 532, 518, 503, 493, 536, 515, 498, 507, 514, 535, 527, 526, 534, 514, 538, 506, 530, 504, 486, 523, 497, 482, 497, 501, 486, 538, 502, 530, 536, 482, 508, 481, 489] payload_rows_per_dst=[5570, 5490, 5582, 5584, 5516, 5688, 5452, 5572] dup_rows_per_dst=[2610, 2632, 2692, 2704, 2652, 2732, 2500, 2560] combine_rows_per_src=[5570, 5490, 5582, 5584, 5516, 5688, 5452, 5572] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44232 payload_rows_remote=38696 dup_rows=21304 dup_rows_remote=18668 dedup_ratio=0.325073 router_tokens_per_expert=[538, 530, 525, 532, 489, 536, 478, 537, 543, 494, 509, 511, 485, 496, 495, 507, 518, 479, 527, 481, 520, 534, 531, 496, 505, 517, 519, 494, 496, 521, 500, 556, 497, 499, 507, 524, 494, 501, 526, 508, 495, 544, 496, 501, 544, 507, 463, 541, 495, 524, 482, 531, 514, 516, 524, 509, 505, 538, 513, 509, 508, 537, 509, 508] payload_rows_per_dst=[5508, 5502, 5536, 5574, 5522, 5498, 5498, 5594] dup_rows_per_dst=[2822, 2578, 2636, 2642, 2590, 2684, 2692, 2660] combine_rows_per_src=[5508, 5502, 5536, 5574, 5522, 5498, 5498, 5594] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44406 payload_rows_remote=38820 dup_rows=21130 dup_rows_remote=18472 dedup_ratio=0.322418 router_tokens_per_expert=[518, 542, 522, 565, 515, 534, 539, 532, 442, 530, 532, 545, 529, 495, 542, 545, 515, 542, 545, 488, 522, 505, 519, 508, 520, 529, 504, 501, 510, 492, 512, 510, 501, 512, 537, 497, 516, 535, 531, 518, 494, 505, 472, 521, 509, 521, 498, 427, 512, 487, 482, 502, 491, 519, 536, 522, 514, 514, 480, 515, 540, 487, 475, 449] payload_rows_per_dst=[5464, 5418, 5440, 5570, 5666, 5542, 5720, 5586] dup_rows_per_dst=[2530, 2564, 2462, 2636, 2814, 2640, 2826, 2658] combine_rows_per_src=[5464, 5418, 5440, 5570, 5666, 5542, 5720, 5586] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278820864 dispatch_meta_bytes=229320 combine_payload_bytes=278820864 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279050184 ingress_bytes=388921344 link_bytes=388921344 token_bytes=557871048 weight_movement_bytes=110100480 total_nvlink_bytes=667971528 weight_to_token_ratio=0.197358 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.763981838e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44242 payload_rows_remote=38706 dup_rows=21294 dup_rows_remote=18672 dedup_ratio=0.324921 router_tokens_per_expert=[533, 527, 524, 532, 487, 532, 477, 534, 543, 489, 504, 510, 483, 493, 495, 502, 516, 478, 524, 481, 521, 533, 531, 495, 504, 516, 517, 493, 495, 523, 498, 557, 499, 499, 506, 524, 495, 502, 528, 509, 492, 549, 496, 502, 545, 508, 462, 544, 495, 525, 485, 529, 520, 517, 527, 512, 506, 541, 518, 515, 510, 541, 511, 509] payload_rows_per_dst=[5496, 5482, 5536, 5564, 5538, 5508, 5508, 5610] dup_rows_per_dst=[2796, 2556, 2622, 2642, 2586, 2688, 2712, 2692] combine_rows_per_src=[5496, 5482, 5536, 5564, 5538, 5508, 5508, 5610] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44590 payload_rows_remote=39064 dup_rows=20946 dup_rows_remote=18444 dedup_ratio=0.319611 router_tokens_per_expert=[538, 520, 480, 521, 536, 519, 473, 501, 512, 504, 505, 520, 576, 521, 501, 502, 480, 523, 533, 492, 517, 499, 540, 499, 522, 531, 474, 538, 487, 494, 499, 487, 536, 501, 544, 467, 527, 523, 555, 502, 484, 483, 515, 525, 559, 514, 492, 494, 512, 468, 510, 512, 511, 468, 515, 518, 558, 504, 524, 522, 529, 509, 548, 495] payload_rows_per_dst=[5540, 5588, 5554, 5554, 5672, 5544, 5526, 5612] dup_rows_per_dst=[2636, 2694, 2612, 2510, 2638, 2588, 2502, 2766] combine_rows_per_src=[5540, 5588, 5554, 5554, 5672, 5544, 5526, 5612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278835200 dispatch_meta_bytes=228984 combine_payload_bytes=278835200 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279064184 ingress_bytes=388935680 link_bytes=388935680 token_bytes=557899384 weight_movement_bytes=110100480 total_nvlink_bytes=667999864 weight_to_token_ratio=0.197348 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.764231164e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278906880 dispatch_meta_bytes=229240 combine_payload_bytes=278906880 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279136120 ingress_bytes=389007360 link_bytes=389007360 token_bytes=558043000 weight_movement_bytes=110100480 total_nvlink_bytes=668143480 weight_to_token_ratio=0.197297 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.765477798e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278706176 dispatch_meta_bytes=229000 combine_payload_bytes=278706176 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278935176 ingress_bytes=388806656 link_bytes=388806656 token_bytes=557641352 weight_movement_bytes=110100480 total_nvlink_bytes=667741832 weight_to_token_ratio=0.197440 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.761987225e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44578 payload_rows_remote=38912 dup_rows=20958 dup_rows_remote=18144 dedup_ratio=0.319794 router_tokens_per_expert=[523, 529, 531, 533, 527, 518, 494, 528, 548, 552, 535, 527, 539, 517, 501, 521, 505, 515, 510, 550, 521, 513, 532, 557, 531, 498, 519, 479, 478, 493, 505, 541, 533, 498, 509, 504, 513, 485, 487, 535, 492, 488, 514, 514, 544, 476, 492, 531, 536, 524, 494, 483, 478, 505, 502, 529, 476, 475, 473, 491, 468, 504, 522, 523] payload_rows_per_dst=[5702, 5666, 5698, 5530, 5518, 5518, 5506, 5440] dup_rows_per_dst=[2664, 2814, 2708, 2558, 2610, 2584, 2596, 2424] combine_rows_per_src=[5702, 5666, 5698, 5530, 5518, 5518, 5506, 5440] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280053760 dispatch_meta_bytes=230096 combine_payload_bytes=280053760 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280283856 ingress_bytes=390154240 link_bytes=390154240 token_bytes=560337616 weight_movement_bytes=110100480 total_nvlink_bytes=670438096 weight_to_token_ratio=0.196490 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.785423927e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44400 payload_rows_remote=38790 dup_rows=21136 dup_rows_remote=18508 dedup_ratio=0.322510 router_tokens_per_expert=[535, 517, 507, 527, 490, 512, 506, 517, 506, 505, 527, 501, 523, 514, 508, 535, 496, 505, 484, 540, 507, 506, 543, 498, 512, 497, 515, 529, 521, 509, 493, 544, 511, 498, 466, 494, 477, 512, 511, 487, 502, 542, 534, 500, 494, 519, 526, 534, 508, 501, 517, 503, 538, 497, 512, 477, 527, 502, 540, 546, 509, 521, 528, 506] payload_rows_per_dst=[5610, 5526, 5496, 5568, 5458, 5568, 5564, 5610] dup_rows_per_dst=[2628, 2696, 2662, 2672, 2454, 2734, 2542, 2748] combine_rows_per_src=[5610, 5526, 5496, 5568, 5458, 5568, 5564, 5610] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277372928 dispatch_meta_bytes=229456 combine_payload_bytes=277372928 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=277602384 ingress_bytes=607674368 link_bytes=607674368 token_bytes=554975312 weight_movement_bytes=330301440 total_nvlink_bytes=885276752 weight_to_token_ratio=0.595164 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.056845671e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44490 payload_rows_remote=38892 dup_rows=21046 dup_rows_remote=18520 dedup_ratio=0.321136 router_tokens_per_expert=[508, 552, 561, 512, 517, 504, 496, 491, 550, 526, 514, 513, 472, 496, 519, 552, 541, 476, 469, 524, 555, 476, 510, 490, 530, 482, 511, 551, 490, 471, 499, 528, 477, 492, 519, 502, 532, 510, 503, 537, 507, 532, 516, 519, 523, 531, 458, 502, 533, 524, 509, 517, 533, 488, 529, 491, 529, 488, 509, 491, 508, 517, 548, 508] payload_rows_per_dst=[5612, 5582, 5528, 5598, 5478, 5540, 5590, 5562] dup_rows_per_dst=[2670, 2702, 2554, 2526, 2666, 2636, 2658, 2634] combine_rows_per_src=[5612, 5582, 5528, 5598, 5478, 5540, 5590, 5562] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278261760 dispatch_meta_bytes=229168 combine_payload_bytes=278261760 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278490928 ingress_bytes=388362240 link_bytes=388362240 token_bytes=556752688 weight_movement_bytes=110100480 total_nvlink_bytes=666853168 weight_to_token_ratio=0.197755 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.754258100e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278921216 dispatch_meta_bytes=228224 combine_payload_bytes=278921216 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279149440 ingress_bytes=609222656 link_bytes=609222656 token_bytes=558070656 weight_movement_bytes=330301440 total_nvlink_bytes=888372096 weight_to_token_ratio=0.591863 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059538399e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278046720 dispatch_meta_bytes=229192 combine_payload_bytes=278046720 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278275912 ingress_bytes=608348160 link_bytes=608348160 token_bytes=556322632 weight_movement_bytes=330301440 total_nvlink_bytes=886624072 weight_to_token_ratio=0.593723 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058017506e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278777856 dispatch_meta_bytes=229648 combine_payload_bytes=278777856 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279007504 ingress_bytes=388878336 link_bytes=388878336 token_bytes=557785360 weight_movement_bytes=110100480 total_nvlink_bytes=667885840 weight_to_token_ratio=0.197389 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.763233858e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44550 payload_rows_remote=39074 dup_rows=20986 dup_rows_remote=18402 dedup_ratio=0.320221 router_tokens_per_expert=[537, 530, 529, 536, 539, 525, 520, 501, 506, 496, 527, 509, 502, 528, 513, 519, 505, 544, 527, 532, 498, 489, 533, 502, 485, 499, 521, 484, 469, 533, 528, 556, 531, 502, 551, 512, 542, 520, 480, 532, 515, 505, 509, 497, 469, 512, 505, 518, 520, 487, 481, 521, 483, 461, 506, 561, 515, 522, 502, 477, 497, 470, 538, 505] payload_rows_per_dst=[5670, 5594, 5574, 5532, 5624, 5476, 5526, 5554] dup_rows_per_dst=[2764, 2606, 2686, 2618, 2716, 2584, 2514, 2498] combine_rows_per_src=[5670, 5594, 5574, 5532, 5624, 5476, 5526, 5554] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44576 payload_rows_remote=38882 dup_rows=20960 dup_rows_remote=18258 dedup_ratio=0.319824 router_tokens_per_expert=[521, 525, 480, 499, 500, 488, 519, 489, 542, 485, 498, 514, 544, 511, 502, 544, 526, 504, 486, 502, 496, 505, 501, 550, 500, 521, 546, 519, 520, 495, 515, 473, 517, 503, 544, 533, 515, 548, 518, 520, 529, 474, 531, 473, 482, 507, 509, 516, 505, 501, 530, 494, 512, 494, 533, 506, 524, 505, 484, 545, 525, 533, 523, 515] payload_rows_per_dst=[5492, 5640, 5586, 5558, 5694, 5492, 5532, 5582] dup_rows_per_dst=[2550, 2640, 2554, 2620, 2702, 2550, 2618, 2726] combine_rows_per_src=[5492, 5640, 5586, 5558, 5694, 5492, 5532, 5582] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44216 payload_rows_remote=38744 dup_rows=21320 dup_rows_remote=18810 dedup_ratio=0.325317 router_tokens_per_expert=[519, 533, 542, 515, 526, 539, 506, 539, 488, 534, 509, 486, 553, 568, 538, 511, 495, 478, 500, 486, 471, 534, 466, 535, 497, 516, 498, 493, 550, 524, 505, 507, 505, 479, 497, 499, 528, 503, 522, 525, 544, 483, 508, 537, 519, 497, 480, 569, 475, 531, 529, 504, 487, 458, 518, 489, 523, 496, 476, 529, 480, 550, 526, 541] payload_rows_per_dst=[5674, 5594, 5458, 5500, 5486, 5530, 5472, 5502] dup_rows_per_dst=[2764, 2780, 2472, 2680, 2630, 2744, 2510, 2740] combine_rows_per_src=[5674, 5594, 5458, 5500, 5486, 5530, 5472, 5502] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44400 payload_rows_remote=38992 dup_rows=21136 dup_rows_remote=18450 dedup_ratio=0.322510 router_tokens_per_expert=[487, 504, 502, 522, 514, 519, 507, 515, 524, 494, 501, 503, 511, 511, 486, 489, 546, 473, 464, 516, 515, 525, 497, 511, 511, 501, 518, 513, 504, 545, 528, 499, 524, 519, 506, 542, 506, 514, 525, 523, 526, 514, 549, 562, 529, 494, 464, 548, 515, 501, 527, 494, 533, 535, 488, 531, 521, 495, 523, 473, 510, 516, 512, 494] payload_rows_per_dst=[5536, 5508, 5408, 5574, 5602, 5602, 5658, 5512] dup_rows_per_dst=[2604, 2530, 2686, 2664, 2716, 2770, 2590, 2576] combine_rows_per_src=[5536, 5508, 5408, 5574, 5602, 5602, 5658, 5512] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=280082432 dispatch_meta_bytes=229904 combine_payload_bytes=280082432 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280312336 ingress_bytes=390182912 link_bytes=390182912 token_bytes=560394768 weight_movement_bytes=110100480 total_nvlink_bytes=670495248 weight_to_token_ratio=0.196470 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.785922580e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278706176 dispatch_meta_bytes=228560 combine_payload_bytes=278706176 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278934736 ingress_bytes=498907136 link_bytes=498907136 token_bytes=557640912 weight_movement_bytes=220200960 total_nvlink_bytes=777841872 weight_to_token_ratio=0.394879 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.676815656e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44398 payload_rows_remote=38822 dup_rows=21138 dup_rows_remote=18434 dedup_ratio=0.322540 router_tokens_per_expert=[557, 506, 497, 519, 519, 514, 549, 490, 511, 482, 539, 557, 544, 489, 510, 508, 503, 504, 496, 543, 509, 525, 487, 486, 512, 537, 498, 526, 486, 488, 482, 492, 504, 521, 515, 552, 461, 488, 495, 486, 516, 512, 510, 510, 472, 494, 523, 553, 521, 541, 512, 506, 508, 529, 545, 495, 549, 511, 493, 492, 526, 511, 539, 513] payload_rows_per_dst=[5556, 5576, 5504, 5462, 5524, 5566, 5616, 5594] dup_rows_per_dst=[2746, 2704, 2602, 2580, 2520, 2614, 2698, 2674] combine_rows_per_src=[5556, 5576, 5504, 5462, 5524, 5566, 5616, 5594] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44360 payload_rows_remote=38776 dup_rows=21176 dup_rows_remote=18350 dedup_ratio=0.323120 router_tokens_per_expert=[513, 534, 485, 541, 503, 473, 536, 518, 497, 517, 524, 508, 513, 503, 510, 501, 503, 492, 524, 502, 529, 544, 527, 524, 479, 524, 488, 502, 458, 514, 481, 483, 525, 511, 520, 523, 502, 546, 499, 509, 557, 483, 529, 509, 558, 475, 530, 466, 517, 542, 524, 530, 467, 536, 530, 483, 510, 527, 518, 535, 509, 510, 502, 536] payload_rows_per_dst=[5382, 5374, 5662, 5628, 5540, 5558, 5632, 5584] dup_rows_per_dst=[2252, 2458, 2628, 2750, 2760, 2670, 2832, 2826] combine_rows_per_src=[5382, 5374, 5662, 5628, 5540, 5558, 5632, 5584] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277716992 dispatch_meta_bytes=230216 combine_payload_bytes=277716992 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277947208 ingress_bytes=387817472 link_bytes=387817472 token_bytes=555664200 weight_movement_bytes=110100480 total_nvlink_bytes=665764680 weight_to_token_ratio=0.198142 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.744783688e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44546 payload_rows_remote=38976 dup_rows=20990 dup_rows_remote=18394 dedup_ratio=0.320282 router_tokens_per_expert=[529, 511, 519, 567, 522, 525, 497, 508, 509, 538, 469, 518, 533, 519, 521, 527, 473, 533, 521, 459, 501, 534, 517, 449, 501, 516, 533, 473, 528, 535, 494, 503, 533, 463, 550, 524, 506, 494, 490, 509, 489, 541, 502, 507, 501, 522, 493, 493, 497, 537, 555, 517, 519, 495, 521, 486, 492, 484, 543, 519, 520, 521, 510, 553] payload_rows_per_dst=[5608, 5620, 5464, 5570, 5604, 5502, 5590, 5588] dup_rows_per_dst=[2748, 2648, 2510, 2596, 2534, 2594, 2664, 2696] combine_rows_per_src=[5608, 5620, 5464, 5570, 5604, 5502, 5590, 5588] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279494656 dispatch_meta_bytes=229768 combine_payload_bytes=279494656 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279724424 ingress_bytes=389595136 link_bytes=389595136 token_bytes=559219080 weight_movement_bytes=110100480 total_nvlink_bytes=669319560 weight_to_token_ratio=0.196883 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.775700189e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44528 payload_rows_remote=38994 dup_rows=21008 dup_rows_remote=18278 dedup_ratio=0.320557 router_tokens_per_expert=[499, 528, 539, 503, 495, 524, 514, 530, 503, 526, 519, 513, 539, 511, 499, 514, 459, 485, 552, 505, 563, 525, 508, 517, 502, 522, 564, 486, 502, 514, 548, 485, 504, 518, 522, 496, 520, 516, 518, 548, 515, 460, 542, 537, 531, 494, 527, 496, 497, 501, 534, 510, 498, 513, 488, 504, 489, 501, 499, 490, 496, 505, 519, 487] payload_rows_per_dst=[5534, 5612, 5706, 5660, 5470, 5596, 5480, 5470] dup_rows_per_dst=[2730, 2636, 2928, 2586, 2408, 2608, 2610, 2502] combine_rows_per_src=[5534, 5612, 5706, 5660, 5470, 5596, 5480, 5470] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278620160 dispatch_meta_bytes=228992 combine_payload_bytes=278620160 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278849152 ingress_bytes=388720640 link_bytes=388720640 token_bytes=557469312 weight_movement_bytes=110100480 total_nvlink_bytes=667569792 weight_to_token_ratio=0.197501 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.760491265e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278935552 dispatch_meta_bytes=230232 combine_payload_bytes=278935552 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279165784 ingress_bytes=499136512 link_bytes=499136512 token_bytes=558101336 weight_movement_bytes=220200960 total_nvlink_bytes=778302296 weight_to_token_ratio=0.394554 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.680804882e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279050240 dispatch_meta_bytes=228392 combine_payload_bytes=279050240 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279278632 ingress_bytes=499251200 link_bytes=499251200 token_bytes=558328872 weight_movement_bytes=220200960 total_nvlink_bytes=778529832 weight_to_token_ratio=0.394393 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.682799495e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44360 payload_rows_remote=38772 dup_rows=21176 dup_rows_remote=18564 dedup_ratio=0.323120 router_tokens_per_expert=[534, 513, 505, 523, 489, 506, 507, 516, 504, 504, 526, 500, 523, 516, 506, 531, 492, 504, 480, 541, 505, 506, 541, 494, 509, 495, 515, 527, 523, 507, 492, 542, 508, 502, 469, 492, 478, 512, 509, 488, 506, 542, 533, 501, 493, 520, 530, 535, 509, 501, 520, 506, 543, 499, 516, 480, 531, 505, 541, 550, 513, 522, 532, 506] payload_rows_per_dst=[5588, 5530, 5478, 5560, 5454, 5562, 5580, 5608] dup_rows_per_dst=[2612, 2690, 2654, 2660, 2462, 2758, 2568, 2772] combine_rows_per_src=[5588, 5530, 5478, 5560, 5454, 5562, 5580, 5608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277444608 dispatch_meta_bytes=229512 combine_payload_bytes=277444608 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=277674120 ingress_bytes=607746048 link_bytes=607746048 token_bytes=555118728 weight_movement_bytes=330301440 total_nvlink_bytes=885420168 weight_to_token_ratio=0.595010 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.056970335e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280010752 dispatch_meta_bytes=230032 combine_payload_bytes=280010752 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280240784 ingress_bytes=390111232 link_bytes=390111232 token_bytes=560251536 weight_movement_bytes=110100480 total_nvlink_bytes=670352016 weight_to_token_ratio=0.196520 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.784675947e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277917696 dispatch_meta_bytes=229344 combine_payload_bytes=277917696 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278147040 ingress_bytes=608219136 link_bytes=608219136 token_bytes=556064736 weight_movement_bytes=330301440 total_nvlink_bytes=886366176 weight_to_token_ratio=0.593998 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057793112e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44582 payload_rows_remote=38888 dup_rows=20954 dup_rows_remote=18246 dedup_ratio=0.319733 router_tokens_per_expert=[520, 522, 478, 496, 499, 480, 517, 488, 539, 485, 495, 512, 544, 508, 498, 541, 526, 505, 486, 502, 497, 501, 499, 551, 499, 522, 543, 518, 520, 494, 517, 471, 517, 500, 545, 534, 513, 551, 521, 520, 530, 476, 531, 474, 486, 508, 511, 518, 505, 500, 531, 494, 513, 495, 539, 508, 528, 506, 488, 548, 527, 537, 523, 518] payload_rows_per_dst=[5482, 5634, 5584, 5556, 5694, 5504, 5534, 5594] dup_rows_per_dst=[2518, 2610, 2550, 2612, 2708, 2564, 2636, 2756] combine_rows_per_src=[5482, 5634, 5584, 5556, 5694, 5504, 5534, 5594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44484 payload_rows_remote=38884 dup_rows=21052 dup_rows_remote=18540 dedup_ratio=0.321228 router_tokens_per_expert=[504, 551, 559, 513, 515, 504, 495, 488, 547, 524, 514, 510, 471, 494, 517, 551, 538, 472, 468, 526, 556, 476, 506, 489, 530, 479, 511, 550, 489, 470, 500, 527, 475, 491, 519, 504, 532, 506, 503, 537, 508, 534, 524, 520, 522, 532, 460, 503, 534, 522, 512, 517, 534, 490, 532, 492, 532, 494, 512, 492, 512, 518, 550, 511] payload_rows_per_dst=[5598, 5578, 5516, 5600, 5468, 5552, 5596, 5576] dup_rows_per_dst=[2660, 2678, 2546, 2512, 2666, 2654, 2670, 2666] combine_rows_per_src=[5598, 5578, 5516, 5600, 5468, 5552, 5596, 5576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44528 payload_rows_remote=39050 dup_rows=21008 dup_rows_remote=18416 dedup_ratio=0.320557 router_tokens_per_expert=[534, 529, 526, 535, 538, 522, 519, 498, 499, 492, 525, 505, 499, 527, 511, 521, 504, 544, 528, 532, 496, 487, 533, 500, 483, 499, 522, 485, 469, 531, 526, 554, 532, 505, 552, 513, 545, 521, 482, 533, 516, 507, 508, 497, 473, 513, 504, 517, 522, 490, 483, 522, 486, 464, 506, 563, 519, 521, 504, 478, 498, 475, 541, 505] payload_rows_per_dst=[5652, 5574, 5566, 5528, 5634, 5478, 5532, 5564] dup_rows_per_dst=[2750, 2584, 2682, 2610, 2732, 2592, 2540, 2518] combine_rows_per_src=[5652, 5574, 5566, 5528, 5634, 5478, 5532, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44350 payload_rows_remote=38760 dup_rows=21186 dup_rows_remote=18374 dedup_ratio=0.323273 router_tokens_per_expert=[509, 530, 482, 538, 499, 474, 533, 518, 496, 513, 523, 505, 510, 503, 509, 501, 494, 491, 520, 501, 529, 544, 526, 522, 480, 524, 486, 502, 459, 517, 481, 484, 527, 511, 519, 523, 504, 546, 501, 510, 556, 483, 532, 508, 556, 475, 533, 466, 519, 543, 527, 532, 470, 538, 531, 485, 514, 530, 521, 536, 513, 512, 506, 538] payload_rows_per_dst=[5382, 5452, 5830, 5458, 5532, 5550, 5556, 5590] dup_rows_per_dst=[2250, 2606, 2958, 2408, 2750, 2668, 2734, 2812] combine_rows_per_src=[5382, 5452, 5830, 5458, 5532, 5550, 5556, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44390 payload_rows_remote=38826 dup_rows=21146 dup_rows_remote=18470 dedup_ratio=0.322662 router_tokens_per_expert=[554, 501, 495, 517, 516, 511, 547, 489, 509, 480, 536, 554, 541, 484, 509, 507, 501, 507, 498, 542, 509, 523, 487, 489, 509, 536, 501, 525, 485, 487, 483, 490, 504, 524, 516, 553, 462, 491, 492, 488, 514, 512, 509, 513, 472, 493, 521, 557, 522, 542, 515, 512, 509, 529, 548, 497, 551, 511, 496, 493, 529, 513, 543, 515] payload_rows_per_dst=[5538, 5564, 5510, 5454, 5520, 5568, 5624, 5612] dup_rows_per_dst=[2722, 2676, 2602, 2578, 2540, 2614, 2724, 2690] combine_rows_per_src=[5538, 5564, 5510, 5454, 5520, 5568, 5624, 5612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44400 payload_rows_remote=39004 dup_rows=21136 dup_rows_remote=18464 dedup_ratio=0.322510 router_tokens_per_expert=[486, 498, 502, 517, 511, 518, 500, 512, 520, 494, 501, 502, 510, 509, 487, 486, 543, 471, 460, 516, 514, 523, 498, 509, 511, 499, 516, 517, 502, 543, 524, 498, 526, 519, 504, 543, 503, 516, 526, 521, 527, 516, 551, 567, 529, 495, 467, 549, 515, 503, 529, 496, 534, 541, 488, 535, 522, 499, 526, 476, 515, 522, 515, 496] payload_rows_per_dst=[5506, 5506, 5396, 5558, 5608, 5612, 5672, 5542] dup_rows_per_dst=[2582, 2512, 2672, 2662, 2708, 2790, 2610, 2600] combine_rows_per_src=[5506, 5506, 5396, 5558, 5608, 5612, 5672, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44222 payload_rows_remote=38738 dup_rows=21314 dup_rows_remote=18788 dedup_ratio=0.325226 router_tokens_per_expert=[519, 531, 538, 512, 524, 535, 503, 535, 486, 530, 506, 485, 549, 566, 538, 511, 493, 476, 502, 484, 471, 533, 468, 534, 497, 517, 496, 493, 549, 522, 509, 506, 504, 477, 497, 499, 524, 506, 523, 530, 544, 483, 508, 538, 522, 498, 484, 572, 476, 531, 532, 505, 487, 460, 521, 493, 526, 500, 477, 533, 480, 553, 525, 542] payload_rows_per_dst=[5656, 5584, 5456, 5500, 5480, 5542, 5484, 5520] dup_rows_per_dst=[2738, 2758, 2466, 2678, 2640, 2756, 2526, 2752] combine_rows_per_src=[5656, 5584, 5456, 5500, 5480, 5542, 5484, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278749184 dispatch_meta_bytes=228536 combine_payload_bytes=278749184 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278977720 ingress_bytes=498950144 link_bytes=498950144 token_bytes=557726904 weight_movement_bytes=220200960 total_nvlink_bytes=777927864 weight_to_token_ratio=0.394819 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.677563636e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278720512 dispatch_meta_bytes=229696 combine_payload_bytes=278720512 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278950208 ingress_bytes=388820992 link_bytes=388820992 token_bytes=557670720 weight_movement_bytes=110100480 total_nvlink_bytes=667771200 weight_to_token_ratio=0.197429 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.762236552e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279910400 dispatch_meta_bytes=229864 combine_payload_bytes=279910400 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280140264 ingress_bytes=500111360 link_bytes=500111360 token_bytes=560050664 weight_movement_bytes=220200960 total_nvlink_bytes=780251624 weight_to_token_ratio=0.393180 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.697759092e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277831680 dispatch_meta_bytes=228536 combine_payload_bytes=277831680 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278060216 ingress_bytes=498032640 link_bytes=498032640 token_bytes=555891896 weight_movement_bytes=220200960 total_nvlink_bytes=776092856 weight_to_token_ratio=0.396122 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.661606733e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44476 payload_rows_remote=38856 dup_rows=21060 dup_rows_remote=18178 dedup_ratio=0.321350 router_tokens_per_expert=[496, 525, 536, 498, 495, 523, 513, 530, 502, 519, 517, 510, 537, 510, 499, 514, 457, 482, 550, 505, 560, 523, 507, 517, 501, 522, 559, 486, 505, 512, 548, 486, 503, 518, 521, 498, 519, 516, 519, 548, 517, 461, 544, 541, 532, 493, 528, 500, 498, 503, 536, 512, 500, 512, 490, 508, 492, 505, 503, 493, 497, 508, 519, 490] payload_rows_per_dst=[5620, 5594, 5546, 5836, 5484, 5538, 5480, 5378] dup_rows_per_dst=[2882, 2626, 2656, 2986, 2372, 2554, 2618, 2366] combine_rows_per_src=[5620, 5594, 5546, 5836, 5484, 5538, 5480, 5378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278304768 dispatch_meta_bytes=229184 combine_payload_bytes=278304768 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278533952 ingress_bytes=498505728 link_bytes=498505728 token_bytes=556838720 weight_movement_bytes=220200960 total_nvlink_bytes=777039680 weight_to_token_ratio=0.395448 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.669834511e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279580672 dispatch_meta_bytes=229872 combine_payload_bytes=279580672 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279810544 ingress_bytes=389681152 link_bytes=389681152 token_bytes=559391216 weight_movement_bytes=110100480 total_nvlink_bytes=669491696 weight_to_token_ratio=0.196822 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.777196149e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277673984 dispatch_meta_bytes=230104 combine_payload_bytes=277673984 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=277904088 ingress_bytes=497874944 link_bytes=497874944 token_bytes=555578072 weight_movement_bytes=220200960 total_nvlink_bytes=775779032 weight_to_token_ratio=0.396346 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.658864140e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278519808 dispatch_meta_bytes=228136 combine_payload_bytes=278519808 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278747944 ingress_bytes=498720768 link_bytes=498720768 token_bytes=557267752 weight_movement_bytes=220200960 total_nvlink_bytes=777468712 weight_to_token_ratio=0.395144 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.673574410e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44418 payload_rows_remote=38884 dup_rows=21118 dup_rows_remote=18446 dedup_ratio=0.322235 router_tokens_per_expert=[512, 509, 539, 542, 519, 512, 543, 519, 510, 481, 527, 529, 479, 510, 541, 505, 477, 508, 510, 509, 534, 488, 513, 479, 497, 501, 516, 489, 537, 500, 493, 491, 524, 514, 521, 504, 531, 534, 472, 503, 511, 507, 506, 504, 507, 533, 550, 550, 516, 511, 503, 503, 493, 544, 514, 475, 498, 502, 544, 498, 532, 523, 531, 491] payload_rows_per_dst=[5648, 5524, 5594, 5448, 5534, 5592, 5556, 5522] dup_rows_per_dst=[2742, 2640, 2442, 2600, 2672, 2744, 2562, 2716] combine_rows_per_src=[5648, 5524, 5594, 5448, 5534, 5592, 5556, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44548 payload_rows_remote=38972 dup_rows=20988 dup_rows_remote=18390 dedup_ratio=0.320251 router_tokens_per_expert=[526, 509, 516, 567, 521, 521, 493, 507, 506, 535, 465, 515, 532, 517, 520, 525, 474, 532, 520, 457, 501, 533, 514, 448, 500, 515, 532, 474, 529, 536, 497, 504, 533, 463, 552, 523, 505, 494, 492, 508, 491, 541, 503, 507, 501, 522, 495, 496, 498, 538, 558, 519, 520, 494, 521, 490, 495, 487, 545, 522, 524, 521, 514, 555] payload_rows_per_dst=[5592, 5612, 5456, 5576, 5604, 5516, 5598, 5594] dup_rows_per_dst=[2728, 2618, 2502, 2598, 2536, 2596, 2678, 2732] combine_rows_per_src=[5592, 5612, 5456, 5576, 5604, 5516, 5598, 5594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44416 payload_rows_remote=38998 dup_rows=21120 dup_rows_remote=18586 dedup_ratio=0.322266 router_tokens_per_expert=[474, 507, 536, 554, 520, 513, 477, 538, 517, 495, 535, 500, 511, 484, 516, 503, 488, 478, 505, 551, 507, 525, 510, 517, 480, 548, 515, 512, 544, 488, 562, 510, 555, 512, 523, 492, 524, 504, 542, 524, 495, 530, 521, 513, 491, 520, 513, 495, 508, 527, 529, 508, 520, 517, 491, 518, 524, 505, 458, 492, 498, 481, 526, 492] payload_rows_per_dst=[5372, 5482, 5572, 5640, 5600, 5584, 5748, 5418] dup_rows_per_dst=[2422, 2510, 2590, 2746, 2814, 2572, 2932, 2534] combine_rows_per_src=[5372, 5482, 5572, 5640, 5600, 5584, 5748, 5418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44668 payload_rows_remote=39040 dup_rows=20868 dup_rows_remote=18236 dedup_ratio=0.318420 router_tokens_per_expert=[492, 477, 518, 494, 505, 535, 497, 499, 495, 553, 459, 469, 511, 536, 532, 490, 530, 544, 498, 522, 520, 524, 501, 488, 521, 531, 521, 524, 545, 504, 508, 559, 516, 518, 510, 516, 512, 518, 510, 524, 509, 519, 476, 529, 497, 497, 570, 533, 495, 515, 461, 515, 514, 500, 477, 512, 505, 567, 540, 518, 505, 480, 471, 537] payload_rows_per_dst=[5482, 5564, 5614, 5686, 5594, 5628, 5468, 5632] dup_rows_per_dst=[2552, 2526, 2640, 2740, 2654, 2632, 2510, 2614] combine_rows_per_src=[5482, 5564, 5614, 5686, 5594, 5628, 5468, 5632] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278276096 dispatch_meta_bytes=229024 combine_payload_bytes=278276096 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278505120 ingress_bytes=608577536 link_bytes=608577536 token_bytes=556781216 weight_movement_bytes=330301440 total_nvlink_bytes=887082656 weight_to_token_ratio=0.593234 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058416429e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279379968 dispatch_meta_bytes=229480 combine_payload_bytes=279379968 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279609448 ingress_bytes=389480448 link_bytes=389480448 token_bytes=558989416 weight_movement_bytes=110100480 total_nvlink_bytes=669089896 weight_to_token_ratio=0.196963 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773705576e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277946368 dispatch_meta_bytes=228504 combine_payload_bytes=277946368 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278174872 ingress_bytes=388046848 link_bytes=388046848 token_bytes=556121240 weight_movement_bytes=110100480 total_nvlink_bytes=666221720 weight_to_token_ratio=0.197979 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.748772914e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279508992 dispatch_meta_bytes=229088 combine_payload_bytes=279508992 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=279738080 ingress_bytes=719910912 link_bytes=719910912 token_bytes=559247072 weight_movement_bytes=440401920 total_nvlink_bytes=999648992 weight_to_token_ratio=0.787491 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.252043481e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44672 payload_rows_remote=39052 dup_rows=20864 dup_rows_remote=18244 dedup_ratio=0.318359 router_tokens_per_expert=[492, 481, 527, 500, 508, 537, 501, 498, 497, 553, 461, 470, 511, 536, 533, 493, 530, 543, 501, 521, 518, 525, 503, 489, 521, 534, 517, 526, 546, 503, 507, 556, 513, 519, 511, 515, 514, 519, 514, 523, 507, 518, 476, 527, 495, 497, 568, 532, 493, 512, 461, 514, 514, 495, 478, 510, 501, 563, 539, 516, 504, 478, 469, 535] payload_rows_per_dst=[5508, 5570, 5612, 5688, 5602, 5620, 5458, 5614] dup_rows_per_dst=[2580, 2538, 2648, 2732, 2654, 2620, 2496, 2596] combine_rows_per_src=[5508, 5570, 5612, 5688, 5602, 5620, 5458, 5614] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44422 payload_rows_remote=38890 dup_rows=21114 dup_rows_remote=18444 dedup_ratio=0.322174 router_tokens_per_expert=[512, 515, 542, 547, 522, 515, 544, 525, 512, 484, 529, 531, 478, 514, 542, 504, 479, 509, 514, 510, 537, 489, 515, 479, 497, 501, 517, 486, 538, 499, 492, 488, 524, 512, 521, 502, 534, 531, 474, 503, 513, 504, 506, 505, 505, 533, 550, 550, 514, 509, 503, 500, 488, 537, 512, 475, 495, 500, 543, 495, 530, 520, 529, 485] payload_rows_per_dst=[5670, 5538, 5600, 5448, 5532, 5592, 5538, 5504] dup_rows_per_dst=[2774, 2650, 2464, 2588, 2670, 2740, 2538, 2690] combine_rows_per_src=[5670, 5538, 5600, 5448, 5532, 5592, 5538, 5504] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44472 payload_rows_remote=38858 dup_rows=21064 dup_rows_remote=18380 dedup_ratio=0.321411 router_tokens_per_expert=[548, 531, 549, 554, 498, 529, 507, 545, 497, 556, 511, 515, 515, 525, 487, 513, 503, 536, 525, 545, 496, 490, 506, 510, 541, 519, 510, 475, 495, 491, 507, 509, 555, 545, 513, 526, 480, 500, 469, 507, 509, 481, 524, 480, 479, 488, 523, 472, 522, 541, 521, 512, 521, 510, 504, 518, 481, 486, 506, 500, 491, 537, 530, 499] payload_rows_per_dst=[5662, 5582, 5590, 5520, 5546, 5466, 5614, 5492] dup_rows_per_dst=[2860, 2656, 2632, 2574, 2644, 2446, 2684, 2568] combine_rows_per_src=[5662, 5582, 5590, 5520, 5546, 5466, 5614, 5492] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44604 payload_rows_remote=39024 dup_rows=20932 dup_rows_remote=18274 dedup_ratio=0.319397 router_tokens_per_expert=[485, 486, 530, 540, 514, 477, 496, 488, 504, 546, 520, 514, 519, 493, 505, 511, 535, 514, 565, 496, 496, 502, 506, 505, 521, 491, 523, 493, 525, 541, 489, 517, 508, 546, 523, 508, 489, 538, 534, 515, 517, 528, 519, 511, 539, 514, 554, 516, 498, 516, 491, 508, 491, 496, 482, 537, 516, 498, 513, 488, 519, 506, 517, 486] payload_rows_per_dst=[5484, 5588, 5580, 5612, 5622, 5656, 5524, 5538] dup_rows_per_dst=[2548, 2636, 2658, 2588, 2700, 2740, 2514, 2548] combine_rows_per_src=[5484, 5588, 5580, 5612, 5622, 5656, 5524, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44626 payload_rows_remote=38960 dup_rows=20910 dup_rows_remote=18172 dedup_ratio=0.319061 router_tokens_per_expert=[476, 503, 562, 488, 519, 493, 519, 549, 516, 557, 515, 556, 503, 498, 517, 540, 503, 517, 505, 503, 536, 447, 483, 506, 546, 485, 500, 508, 509, 514, 484, 468, 501, 527, 518, 520, 551, 495, 544, 559, 508, 525, 504, 529, 531, 474, 488, 510, 479, 504, 527, 553, 518, 497, 506, 478, 510, 508, 490, 517, 516, 508, 506, 542] payload_rows_per_dst=[5576, 5666, 5538, 5518, 5650, 5586, 5556, 5536] dup_rows_per_dst=[2642, 2738, 2462, 2510, 2780, 2552, 2568, 2658] combine_rows_per_src=[5576, 5666, 5538, 5518, 5650, 5586, 5556, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44610 payload_rows_remote=39028 dup_rows=20926 dup_rows_remote=18284 dedup_ratio=0.319305 router_tokens_per_expert=[483, 485, 530, 535, 513, 477, 495, 486, 501, 544, 516, 513, 518, 493, 502, 511, 534, 514, 563, 496, 494, 499, 509, 503, 521, 490, 523, 492, 523, 540, 492, 519, 507, 545, 524, 506, 489, 538, 532, 516, 517, 528, 521, 511, 542, 515, 555, 518, 498, 516, 491, 509, 494, 496, 486, 538, 515, 499, 515, 491, 524, 509, 521, 488] payload_rows_per_dst=[5470, 5574, 5582, 5620, 5614, 5666, 5530, 5554] dup_rows_per_dst=[2538, 2622, 2642, 2580, 2700, 2748, 2526, 2570] combine_rows_per_src=[5470, 5574, 5582, 5620, 5614, 5666, 5530, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278720512 dispatch_meta_bytes=229320 combine_payload_bytes=278720512 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278949832 ingress_bytes=498921472 link_bytes=498921472 token_bytes=557670344 weight_movement_bytes=220200960 total_nvlink_bytes=777871304 weight_to_token_ratio=0.394859 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.677064983e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279351296 dispatch_meta_bytes=229448 combine_payload_bytes=279351296 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279580744 ingress_bytes=389451776 link_bytes=389451776 token_bytes=558932040 weight_movement_bytes=110100480 total_nvlink_bytes=669032520 weight_to_token_ratio=0.196984 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773206923e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44478 payload_rows_remote=38858 dup_rows=21058 dup_rows_remote=18346 dedup_ratio=0.321320 router_tokens_per_expert=[545, 529, 549, 552, 496, 527, 505, 541, 494, 557, 510, 514, 514, 526, 486, 511, 504, 535, 527, 539, 496, 484, 506, 508, 541, 519, 508, 475, 494, 491, 507, 510, 555, 543, 514, 527, 478, 502, 468, 509, 508, 482, 524, 481, 480, 489, 523, 473, 523, 543, 524, 516, 522, 510, 506, 522, 482, 486, 508, 502, 494, 537, 534, 503] payload_rows_per_dst=[5648, 5586, 5576, 5518, 5554, 5468, 5620, 5508] dup_rows_per_dst=[2840, 2638, 2622, 2572, 2638, 2452, 2712, 2584] combine_rows_per_src=[5648, 5586, 5576, 5518, 5554, 5468, 5620, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279537664 dispatch_meta_bytes=230336 combine_payload_bytes=279537664 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279768000 ingress_bytes=499738624 link_bytes=499738624 token_bytes=559305664 weight_movement_bytes=220200960 total_nvlink_bytes=779506624 weight_to_token_ratio=0.393704 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.691276600e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279838720 dispatch_meta_bytes=229104 combine_payload_bytes=279838720 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280067824 ingress_bytes=389939200 link_bytes=389939200 token_bytes=559906544 weight_movement_bytes=110100480 total_nvlink_bytes=670007024 weight_to_token_ratio=0.196641 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.781684028e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279265280 dispatch_meta_bytes=228528 combine_payload_bytes=279265280 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=279493808 ingress_bytes=719667200 link_bytes=719667200 token_bytes=558759088 weight_movement_bytes=440401920 total_nvlink_bytes=999161008 weight_to_token_ratio=0.788179 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.251619626e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44280 payload_rows_remote=38692 dup_rows=21256 dup_rows_remote=18560 dedup_ratio=0.324341 router_tokens_per_expert=[536, 524, 488, 530, 505, 531, 495, 473, 513, 523, 527, 506, 471, 557, 550, 529, 498, 490, 509, 513, 525, 540, 488, 520, 499, 498, 529, 514, 494, 501, 525, 532, 505, 535, 518, 479, 532, 495, 530, 471, 525, 504, 503, 524, 527, 489, 510, 511, 500, 485, 503, 494, 509, 471, 521, 503, 509, 551, 535, 483, 511, 522, 556, 524] payload_rows_per_dst=[5588, 5612, 5714, 5578, 5306, 5544, 5442, 5496] dup_rows_per_dst=[2696, 2624, 2990, 2606, 2286, 2656, 2530, 2868] combine_rows_per_src=[5588, 5612, 5714, 5578, 5306, 5544, 5442, 5496] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278763520 dispatch_meta_bytes=229336 combine_payload_bytes=278763520 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=278992856 ingress_bytes=719165440 link_bytes=719165440 token_bytes=557756376 weight_movement_bytes=440401920 total_nvlink_bytes=998158296 weight_to_token_ratio=0.789595 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.250746983e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279924736 dispatch_meta_bytes=229184 combine_payload_bytes=279924736 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280153920 ingress_bytes=390025216 link_bytes=390025216 token_bytes=560078656 weight_movement_bytes=110100480 total_nvlink_bytes=670179136 weight_to_token_ratio=0.196580 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.783179987e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44634 payload_rows_remote=38948 dup_rows=20902 dup_rows_remote=18130 dedup_ratio=0.318939 router_tokens_per_expert=[476, 510, 570, 492, 524, 495, 520, 551, 518, 562, 515, 558, 507, 502, 524, 543, 503, 518, 501, 507, 539, 449, 484, 505, 545, 487, 502, 508, 506, 511, 482, 467, 501, 526, 516, 515, 550, 493, 544, 559, 508, 524, 506, 529, 528, 473, 487, 511, 476, 504, 522, 552, 517, 493, 507, 474, 509, 504, 484, 515, 510, 506, 504, 540] payload_rows_per_dst=[5604, 5686, 5540, 5518, 5642, 5588, 5542, 5514] dup_rows_per_dst=[2672, 2772, 2472, 2498, 2766, 2544, 2548, 2630] combine_rows_per_src=[5604, 5686, 5540, 5518, 5642, 5588, 5542, 5514] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278534144 dispatch_meta_bytes=228952 combine_payload_bytes=278534144 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278763096 ingress_bytes=388634624 link_bytes=388634624 token_bytes=557297240 weight_movement_bytes=110100480 total_nvlink_bytes=667397720 weight_to_token_ratio=0.197562 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.758995305e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44236 payload_rows_remote=38734 dup_rows=21300 dup_rows_remote=18646 dedup_ratio=0.325012 router_tokens_per_expert=[557, 512, 506, 520, 510, 504, 556, 496, 561, 514, 512, 502, 511, 515, 509, 501, 491, 475, 451, 506, 514, 514, 494, 566, 491, 511, 516, 510, 496, 566, 478, 510, 482, 501, 512, 475, 537, 524, 514, 541, 527, 493, 510, 512, 515, 522, 533, 495, 485, 491, 507, 540, 513, 533, 498, 529, 504, 516, 539, 524, 543, 469, 502, 507] payload_rows_per_dst=[5594, 5532, 5430, 5502, 5522, 5550, 5542, 5564] dup_rows_per_dst=[2728, 2718, 2592, 2654, 2650, 2664, 2650, 2644] combine_rows_per_src=[5594, 5532, 5430, 5502, 5522, 5550, 5542, 5564] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44344 payload_rows_remote=38844 dup_rows=21192 dup_rows_remote=18484 dedup_ratio=0.323364 router_tokens_per_expert=[478, 509, 539, 557, 522, 514, 482, 538, 519, 500, 537, 502, 512, 487, 515, 505, 488, 479, 508, 553, 508, 525, 509, 516, 480, 550, 516, 512, 544, 489, 563, 510, 554, 512, 525, 490, 525, 503, 543, 525, 495, 526, 521, 512, 490, 520, 510, 492, 506, 526, 528, 507, 519, 514, 491, 514, 523, 502, 454, 488, 497, 478, 522, 490] payload_rows_per_dst=[5292, 5352, 5720, 5640, 5384, 5626, 5830, 5500] dup_rows_per_dst=[2316, 2300, 2872, 2770, 2536, 2640, 3050, 2708] combine_rows_per_src=[5292, 5352, 5720, 5640, 5384, 5626, 5830, 5500] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279724032 dispatch_meta_bytes=229192 combine_payload_bytes=279724032 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279953224 ingress_bytes=389824512 link_bytes=389824512 token_bytes=559677256 weight_movement_bytes=110100480 total_nvlink_bytes=669777736 weight_to_token_ratio=0.196721 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.779689415e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44274 payload_rows_remote=38710 dup_rows=21262 dup_rows_remote=18616 dedup_ratio=0.324432 router_tokens_per_expert=[541, 524, 490, 532, 509, 533, 496, 475, 513, 524, 530, 508, 473, 556, 553, 532, 503, 489, 509, 511, 525, 541, 487, 522, 500, 499, 529, 515, 494, 501, 528, 531, 504, 535, 517, 480, 532, 493, 532, 473, 524, 503, 501, 525, 524, 488, 506, 510, 497, 485, 504, 494, 508, 468, 518, 499, 507, 551, 533, 482, 509, 518, 554, 521] payload_rows_per_dst=[5564, 5660, 5512, 5584, 5506, 5528, 5428, 5492] dup_rows_per_dst=[2646, 2740, 2662, 2610, 2594, 2634, 2518, 2858] combine_rows_per_src=[5564, 5660, 5512, 5584, 5506, 5528, 5428, 5492] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279179264 dispatch_meta_bytes=228312 combine_payload_bytes=279179264 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279407576 ingress_bytes=609480704 link_bytes=609480704 token_bytes=558586840 weight_movement_bytes=330301440 total_nvlink_bytes=888888280 weight_to_token_ratio=0.591316 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059987187e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277645312 dispatch_meta_bytes=229520 combine_payload_bytes=277645312 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=277874832 ingress_bytes=607946752 link_bytes=607946752 token_bytes=555520144 weight_movement_bytes=330301440 total_nvlink_bytes=885821584 weight_to_token_ratio=0.594580 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057319392e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278433792 dispatch_meta_bytes=229312 combine_payload_bytes=278433792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278663104 ingress_bytes=388534272 link_bytes=388534272 token_bytes=557096896 weight_movement_bytes=110100480 total_nvlink_bytes=667197376 weight_to_token_ratio=0.197633 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.757250019e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44430 payload_rows_remote=38804 dup_rows=21106 dup_rows_remote=18346 dedup_ratio=0.322052 router_tokens_per_expert=[508, 485, 535, 501, 546, 522, 488, 507, 489, 503, 550, 492, 506, 532, 517, 513, 519, 521, 508, 506, 530, 490, 544, 533, 513, 500, 511, 517, 513, 493, 480, 506, 531, 506, 548, 537, 523, 538, 519, 491, 503, 486, 511, 530, 485, 515, 519, 487, 501, 533, 517, 518, 525, 547, 511, 512, 490, 556, 494, 516, 472, 462, 524, 483] payload_rows_per_dst=[5552, 5506, 5596, 5486, 5626, 5590, 5634, 5440] dup_rows_per_dst=[2632, 2698, 2706, 2580, 2760, 2482, 2694, 2554] combine_rows_per_src=[5552, 5506, 5596, 5486, 5626, 5590, 5634, 5440] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277473280 dispatch_meta_bytes=229304 combine_payload_bytes=277473280 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277702584 ingress_bytes=497674240 link_bytes=497674240 token_bytes=555175864 weight_movement_bytes=220200960 total_nvlink_bytes=775376824 weight_to_token_ratio=0.396633 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.655373567e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44280 payload_rows_remote=38790 dup_rows=21256 dup_rows_remote=18550 dedup_ratio=0.324341 router_tokens_per_expert=[533, 502, 536, 477, 515, 535, 493, 526, 534, 526, 522, 548, 480, 478, 495, 531, 501, 461, 493, 511, 557, 510, 471, 510, 500, 508, 493, 514, 508, 508, 516, 511, 510, 536, 553, 513, 479, 510, 511, 529, 513, 533, 485, 543, 538, 500, 492, 494, 486, 519, 493, 536, 513, 545, 526, 506, 520, 524, 464, 542, 497, 519, 514, 522] payload_rows_per_dst=[5570, 5544, 5458, 5514, 5564, 5490, 5574, 5566] dup_rows_per_dst=[2664, 2684, 2570, 2602, 2718, 2706, 2674, 2638] combine_rows_per_src=[5570, 5544, 5458, 5514, 5564, 5490, 5574, 5566] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44412 payload_rows_remote=38774 dup_rows=21124 dup_rows_remote=18514 dedup_ratio=0.322327 router_tokens_per_expert=[540, 538, 493, 499, 533, 511, 515, 537, 486, 475, 520, 511, 523, 550, 519, 525, 509, 472, 503, 518, 505, 484, 525, 540, 506, 543, 488, 533, 479, 562, 526, 491, 501, 504, 495, 470, 511, 505, 537, 494, 492, 474, 528, 490, 545, 549, 515, 533, 485, 553, 520, 519, 503, 521, 515, 508, 486, 497, 483, 529, 524, 538, 488, 497] payload_rows_per_dst=[5656, 5568, 5518, 5528, 5430, 5544, 5638, 5530] dup_rows_per_dst=[2676, 2650, 2594, 2728, 2604, 2708, 2610, 2554] combine_rows_per_src=[5656, 5568, 5518, 5528, 5430, 5544, 5638, 5530] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44496 payload_rows_remote=38950 dup_rows=21040 dup_rows_remote=18342 dedup_ratio=0.321045 router_tokens_per_expert=[559, 492, 511, 480, 529, 502, 486, 533, 486, 507, 494, 520, 506, 494, 512, 492, 472, 521, 544, 523, 545, 514, 493, 510, 533, 548, 514, 530, 515, 488, 471, 509, 531, 510, 537, 492, 481, 524, 535, 495, 511, 487, 503, 541, 499, 574, 532, 526, 516, 490, 491, 513, 546, 515, 508, 552, 537, 475, 468, 482, 515, 518, 497, 534] payload_rows_per_dst=[5622, 5456, 5546, 5582, 5596, 5642, 5560, 5492] dup_rows_per_dst=[2562, 2566, 2698, 2634, 2614, 2704, 2702, 2560] combine_rows_per_src=[5622, 5456, 5546, 5582, 5596, 5642, 5560, 5492] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44670 payload_rows_remote=39050 dup_rows=20866 dup_rows_remote=18318 dedup_ratio=0.318390 router_tokens_per_expert=[489, 517, 519, 565, 541, 520, 508, 525, 542, 517, 487, 492, 492, 518, 508, 528, 548, 492, 504, 501, 525, 511, 530, 548, 536, 527, 553, 535, 494, 565, 532, 476, 528, 499, 526, 468, 528, 509, 493, 507, 536, 486, 506, 482, 474, 500, 511, 528, 476, 486, 489, 495, 524, 528, 516, 515, 476, 502, 488, 484, 513, 499, 504, 547] payload_rows_per_dst=[5658, 5620, 5648, 5648, 5498, 5542, 5512, 5544] dup_rows_per_dst=[2710, 2548, 2670, 2788, 2618, 2504, 2546, 2482] combine_rows_per_src=[5658, 5620, 5648, 5648, 5498, 5542, 5512, 5544] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44594 payload_rows_remote=39014 dup_rows=20942 dup_rows_remote=18438 dedup_ratio=0.319550 router_tokens_per_expert=[540, 526, 532, 498, 522, 546, 521, 535, 486, 535, 499, 511, 530, 521, 521, 498, 504, 506, 531, 527, 555, 526, 484, 510, 534, 489, 528, 527, 489, 464, 485, 526, 532, 530, 476, 495, 510, 470, 532, 508, 531, 471, 485, 522, 511, 469, 514, 528, 548, 486, 481, 517, 499, 515, 516, 493, 506, 524, 532, 528, 495, 512, 511, 515] payload_rows_per_dst=[5640, 5558, 5600, 5580, 5558, 5530, 5554, 5574] dup_rows_per_dst=[2800, 2644, 2686, 2504, 2548, 2532, 2556, 2672] combine_rows_per_src=[5640, 5558, 5600, 5580, 5558, 5530, 5554, 5574] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278147072 dispatch_meta_bytes=228600 combine_payload_bytes=278147072 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278375672 ingress_bytes=388247552 link_bytes=388247552 token_bytes=556522744 weight_movement_bytes=110100480 total_nvlink_bytes=666623224 weight_to_token_ratio=0.197836 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.752263487e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277932032 dispatch_meta_bytes=229152 combine_payload_bytes=277932032 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=278161184 ingress_bytes=388032512 link_bytes=388032512 token_bytes=556093216 weight_movement_bytes=110100480 total_nvlink_bytes=666193696 weight_to_token_ratio=0.197989 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.748523587e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278046720 dispatch_meta_bytes=229360 combine_payload_bytes=278046720 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278276080 ingress_bytes=388147200 link_bytes=388147200 token_bytes=556322800 weight_movement_bytes=110100480 total_nvlink_bytes=666423280 weight_to_token_ratio=0.197908 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.750518200e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279193600 dispatch_meta_bytes=229168 combine_payload_bytes=279193600 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279422768 ingress_bytes=389294080 link_bytes=389294080 token_bytes=558616368 weight_movement_bytes=110100480 total_nvlink_bytes=668716848 weight_to_token_ratio=0.197095 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.770464330e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279910400 dispatch_meta_bytes=229472 combine_payload_bytes=279910400 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=280139872 ingress_bytes=500111360 link_bytes=500111360 token_bytes=560050272 weight_movement_bytes=220200960 total_nvlink_bytes=780251232 weight_to_token_ratio=0.393181 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.697759092e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279652352 dispatch_meta_bytes=229808 combine_payload_bytes=279652352 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279882160 ingress_bytes=389752832 link_bytes=389752832 token_bytes=559534512 weight_movement_bytes=110100480 total_nvlink_bytes=669634992 weight_to_token_ratio=0.196772 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778442782e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44486 payload_rows_remote=38990 dup_rows=21050 dup_rows_remote=18364 dedup_ratio=0.321198 router_tokens_per_expert=[534, 520, 487, 474, 529, 506, 525, 504, 550, 516, 535, 495, 529, 522, 518, 522, 487, 509, 466, 518, 514, 544, 497, 558, 522, 493, 487, 529, 510, 515, 521, 508, 541, 496, 489, 551, 474, 518, 517, 513, 517, 481, 507, 513, 519, 505, 493, 522, 509, 496, 488, 484, 511, 546, 473, 524, 503, 544, 537, 479, 496, 532, 529, 517] payload_rows_per_dst=[5496, 5670, 5644, 5516, 5570, 5498, 5498, 5594] dup_rows_per_dst=[2686, 2704, 2698, 2498, 2628, 2616, 2540, 2680] combine_rows_per_src=[5496, 5670, 5644, 5516, 5570, 5498, 5498, 5594] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44420 payload_rows_remote=38692 dup_rows=21116 dup_rows_remote=18150 dedup_ratio=0.322205 router_tokens_per_expert=[510, 524, 506, 513, 489, 494, 518, 544, 499, 537, 509, 512, 562, 490, 506, 494, 527, 472, 536, 509, 537, 490, 525, 488, 487, 513, 515, 545, 522, 508, 526, 539, 484, 485, 532, 484, 517, 488, 531, 517, 493, 487, 502, 476, 540, 542, 528, 521, 525, 499, 522, 495, 484, 524, 526, 506, 524, 529, 517, 540, 521, 480, 486, 517] payload_rows_per_dst=[5396, 5466, 5546, 5612, 5518, 5566, 5588, 5728] dup_rows_per_dst=[2334, 2606, 2622, 2664, 2620, 2730, 2574, 2966] combine_rows_per_src=[5396, 5466, 5546, 5612, 5518, 5566, 5588, 5728] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279480320 dispatch_meta_bytes=229416 combine_payload_bytes=279480320 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279709736 ingress_bytes=499681280 link_bytes=499681280 token_bytes=559190056 weight_movement_bytes=220200960 total_nvlink_bytes=779391016 weight_to_token_ratio=0.393786 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.690279294e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277344256 dispatch_meta_bytes=227368 combine_payload_bytes=277344256 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277571624 ingress_bytes=387444736 link_bytes=387444736 token_bytes=554915880 weight_movement_bytes=110100480 total_nvlink_bytes=665016360 weight_to_token_ratio=0.198409 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.738301196e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44574 payload_rows_remote=39104 dup_rows=20962 dup_rows_remote=18418 dedup_ratio=0.319855 router_tokens_per_expert=[522, 502, 496, 536, 551, 490, 516, 512, 519, 552, 509, 526, 527, 520, 537, 533, 488, 519, 557, 480, 521, 506, 498, 525, 497, 480, 494, 517, 562, 470, 504, 505, 497, 547, 508, 502, 512, 482, 476, 483, 516, 533, 505, 508, 535, 497, 517, 530, 523, 513, 511, 466, 504, 488, 513, 519, 490, 535, 487, 528, 514, 518, 516, 524] payload_rows_per_dst=[5582, 5710, 5594, 5516, 5470, 5662, 5502, 5538] dup_rows_per_dst=[2668, 2736, 2594, 2542, 2544, 2620, 2572, 2686] combine_rows_per_src=[5582, 5710, 5594, 5516, 5470, 5662, 5502, 5538] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44654 payload_rows_remote=39154 dup_rows=20882 dup_rows_remote=18328 dedup_ratio=0.318634 router_tokens_per_expert=[482, 505, 500, 500, 513, 523, 492, 502, 511, 513, 533, 490, 555, 476, 510, 498, 524, 512, 536, 548, 505, 518, 502, 511, 511, 518, 529, 572, 545, 464, 512, 544, 533, 482, 527, 502, 494, 508, 512, 543, 496, 503, 514, 516, 484, 492, 513, 519, 522, 511, 485, 488, 530, 465, 531, 495, 555, 499, 497, 549, 509, 507, 499, 534] payload_rows_per_dst=[5516, 5548, 5642, 5652, 5608, 5560, 5500, 5628] dup_rows_per_dst=[2518, 2624, 2670, 2738, 2594, 2514, 2554, 2670] combine_rows_per_src=[5516, 5548, 5642, 5652, 5608, 5560, 5500, 5628] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44566 payload_rows_remote=38956 dup_rows=20970 dup_rows_remote=18366 dedup_ratio=0.319977 router_tokens_per_expert=[522, 506, 487, 531, 487, 521, 537, 511, 483, 525, 478, 520, 535, 484, 524, 477, 481, 547, 510, 506, 500, 508, 541, 514, 491, 472, 510, 549, 512, 518, 512, 548, 501, 523, 500, 485, 509, 513, 536, 491, 506, 510, 530, 475, 537, 528, 510, 518, 528, 521, 517, 531, 530, 501, 538, 498, 517, 489, 511, 524, 533, 492, 521, 498] payload_rows_per_dst=[5570, 5508, 5610, 5598, 5548, 5618, 5564, 5550] dup_rows_per_dst=[2634, 2544, 2604, 2626, 2568, 2610, 2764, 2620] combine_rows_per_src=[5570, 5508, 5610, 5598, 5548, 5618, 5564, 5550] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44472 payload_rows_remote=38992 dup_rows=21064 dup_rows_remote=18418 dedup_ratio=0.321411 router_tokens_per_expert=[480, 545, 493, 530, 525, 534, 510, 554, 553, 506, 525, 497, 498, 489, 502, 526, 496, 516, 538, 497, 557, 526, 467, 517, 503, 485, 521, 520, 512, 518, 511, 500, 505, 523, 541, 498, 507, 548, 528, 524, 525, 485, 528, 543, 533, 491, 462, 496, 480, 532, 513, 498, 498, 525, 539, 547, 491, 493, 476, 505, 457, 508, 487, 531] payload_rows_per_dst=[5546, 5568, 5602, 5544, 5672, 5480, 5632, 5428] dup_rows_per_dst=[2796, 2624, 2626, 2596, 2676, 2646, 2632, 2468] combine_rows_per_src=[5546, 5568, 5602, 5544, 5672, 5480, 5632, 5428] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44448 payload_rows_remote=38974 dup_rows=21088 dup_rows_remote=18484 dedup_ratio=0.321777 router_tokens_per_expert=[514, 537, 524, 517, 500, 507, 513, 504, 503, 501, 486, 500, 521, 499, 550, 479, 489, 504, 511, 544, 507, 497, 513, 475, 458, 517, 507, 492, 489, 527, 554, 483, 529, 513, 510, 524, 486, 488, 533, 515, 539, 535, 505, 524, 516, 506, 561, 505, 541, 514, 498, 506, 531, 534, 514, 535, 488, 549, 508, 500, 519, 494, 538, 488] payload_rows_per_dst=[5584, 5474, 5560, 5480, 5590, 5648, 5546, 5566] dup_rows_per_dst=[2648, 2604, 2520, 2574, 2606, 2734, 2800, 2602] combine_rows_per_src=[5584, 5474, 5560, 5480, 5590, 5648, 5546, 5566] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44666 payload_rows_remote=39068 dup_rows=20870 dup_rows_remote=18168 dedup_ratio=0.318451 router_tokens_per_expert=[528, 515, 517, 532, 508, 488, 504, 506, 530, 516, 506, 577, 507, 531, 504, 517, 525, 515, 524, 490, 521, 502, 525, 495, 471, 497, 512, 544, 532, 527, 517, 550, 500, 518, 542, 515, 524, 466, 493, 505, 525, 492, 478, 532, 471, 510, 472, 503, 511, 506, 499, 491, 524, 512, 514, 524, 550, 505, 501, 502, 514, 529, 506, 501] payload_rows_per_dst=[5538, 5572, 5650, 5598, 5566, 5474, 5612, 5656] dup_rows_per_dst=[2658, 2804, 2544, 2702, 2560, 2492, 2550, 2560] combine_rows_per_src=[5538, 5572, 5650, 5598, 5566, 5474, 5612, 5656] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280297472 dispatch_meta_bytes=230088 combine_payload_bytes=280297472 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280527560 ingress_bytes=390397952 link_bytes=390397952 token_bytes=560825032 weight_movement_bytes=110100480 total_nvlink_bytes=670925512 weight_to_token_ratio=0.196319 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.789662480e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280655872 dispatch_meta_bytes=229928 combine_payload_bytes=280655872 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280885800 ingress_bytes=390756352 link_bytes=390756352 token_bytes=561541672 weight_movement_bytes=110100480 total_nvlink_bytes=671642152 weight_to_token_ratio=0.196068 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.795895645e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279236608 dispatch_meta_bytes=229288 combine_payload_bytes=279236608 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279465896 ingress_bytes=609538048 link_bytes=609538048 token_bytes=558702504 weight_movement_bytes=330301440 total_nvlink_bytes=889003944 weight_to_token_ratio=0.591194 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060086917e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279494656 dispatch_meta_bytes=229640 combine_payload_bytes=279494656 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279724296 ingress_bytes=499695616 link_bytes=499695616 token_bytes=559218952 weight_movement_bytes=220200960 total_nvlink_bytes=779419912 weight_to_token_ratio=0.393765 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.690528620e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279365632 dispatch_meta_bytes=229832 combine_payload_bytes=279365632 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279595464 ingress_bytes=499566592 link_bytes=499566592 token_bytes=558961096 weight_movement_bytes=220200960 total_nvlink_bytes=779162056 weight_to_token_ratio=0.393947 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.688284681e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280039424 dispatch_meta_bytes=228944 combine_payload_bytes=280039424 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=280268368 ingress_bytes=500240384 link_bytes=500240384 token_bytes=560307792 weight_movement_bytes=220200960 total_nvlink_bytes=780508752 weight_to_token_ratio=0.393000 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.700003032e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44502 payload_rows_remote=38850 dup_rows=21034 dup_rows_remote=18342 dedup_ratio=0.320953 router_tokens_per_expert=[523, 525, 525, 516, 461, 560, 521, 522, 503, 502, 518, 503, 538, 508, 501, 513, 511, 523, 515, 538, 511, 492, 552, 513, 539, 528, 537, 471, 521, 487, 524, 485, 503, 513, 524, 468, 517, 513, 494, 488, 536, 470, 530, 520, 492, 507, 499, 561, 496, 543, 492, 532, 504, 532, 503, 528, 470, 504, 508, 492, 532, 491, 511, 509] payload_rows_per_dst=[5652, 5562, 5540, 5532, 5524, 5602, 5560, 5530] dup_rows_per_dst=[2692, 2610, 2732, 2652, 2516, 2628, 2700, 2504] combine_rows_per_src=[5652, 5562, 5540, 5532, 5524, 5602, 5560, 5530] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44572 payload_rows_remote=38844 dup_rows=20964 dup_rows_remote=18192 dedup_ratio=0.319885 router_tokens_per_expert=[510, 500, 514, 497, 522, 510, 520, 489, 486, 505, 504, 480, 521, 552, 526, 545, 537, 503, 536, 467, 489, 528, 484, 508, 518, 520, 515, 515, 540, 487, 517, 514, 525, 522, 533, 483, 506, 490, 497, 524, 515, 538, 528, 550, 511, 506, 509, 521, 462, 553, 488, 509, 471, 531, 494, 502, 519, 509, 532, 494, 548, 504, 525, 510] payload_rows_per_dst=[5342, 5624, 5494, 5664, 5640, 5542, 5538, 5728] dup_rows_per_dst=[2348, 2598, 2462, 2738, 2874, 2610, 2562, 2772] combine_rows_per_src=[5342, 5624, 5494, 5664, 5640, 5542, 5538, 5728] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278433792 dispatch_meta_bytes=228144 combine_payload_bytes=278433792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278661936 ingress_bytes=388534272 link_bytes=388534272 token_bytes=557095728 weight_movement_bytes=110100480 total_nvlink_bytes=667196208 weight_to_token_ratio=0.197633 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.757250019e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44466 payload_rows_remote=38790 dup_rows=21070 dup_rows_remote=18368 dedup_ratio=0.321503 router_tokens_per_expert=[542, 509, 519, 514, 493, 494, 511, 495, 513, 511, 492, 494, 544, 504, 517, 547, 495, 519, 494, 506, 450, 518, 513, 526, 555, 551, 481, 485, 509, 487, 488, 516, 493, 480, 531, 545, 551, 528, 510, 551, 509, 571, 525, 508, 492, 534, 520, 512, 503, 504, 473, 466, 535, 501, 535, 486, 518, 522, 520, 500, 558, 510, 489, 496] payload_rows_per_dst=[5544, 5562, 5516, 5536, 5676, 5596, 5482, 5554] dup_rows_per_dst=[2610, 2682, 2526, 2608, 2702, 2746, 2524, 2672] combine_rows_per_src=[5544, 5562, 5516, 5536, 5676, 5596, 5482, 5554] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44392 payload_rows_remote=38774 dup_rows=21144 dup_rows_remote=18460 dedup_ratio=0.322632 router_tokens_per_expert=[534, 486, 510, 517, 501, 515, 483, 519, 513, 516, 514, 523, 500, 494, 541, 513, 504, 528, 468, 469, 510, 503, 539, 528, 524, 533, 518, 515, 501, 538, 548, 483, 482, 507, 519, 466, 536, 508, 539, 480, 511, 502, 488, 512, 506, 499, 513, 548, 509, 556, 473, 531, 530, 497, 545, 510, 523, 501, 486, 520, 505, 524, 531, 523] payload_rows_per_dst=[5466, 5552, 5542, 5580, 5540, 5544, 5618, 5550] dup_rows_per_dst=[2664, 2676, 2556, 2740, 2534, 2614, 2684, 2676] combine_rows_per_src=[5466, 5552, 5542, 5580, 5540, 5544, 5618, 5550] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278476800 dispatch_meta_bytes=228768 combine_payload_bytes=278476800 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278705568 ingress_bytes=608778240 link_bytes=608778240 token_bytes=557182368 weight_movement_bytes=330301440 total_nvlink_bytes=887483808 weight_to_token_ratio=0.592807 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058765486e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44494 payload_rows_remote=38956 dup_rows=21042 dup_rows_remote=18380 dedup_ratio=0.321075 router_tokens_per_expert=[501, 508, 507, 508, 491, 512, 513, 483, 480, 552, 512, 573, 525, 530, 514, 513, 526, 523, 531, 513, 500, 477, 526, 504, 517, 530, 506, 479, 529, 515, 510, 477, 545, 519, 476, 506, 524, 498, 520, 523, 509, 520, 547, 505, 515, 506, 523, 507, 528, 507, 524, 475, 510, 578, 499, 462, 530, 471, 473, 485, 526, 542, 533, 497] payload_rows_per_dst=[5532, 5662, 5538, 5566, 5610, 5546, 5540, 5500] dup_rows_per_dst=[2514, 2736, 2662, 2560, 2612, 2718, 2626, 2614] combine_rows_per_src=[5532, 5662, 5538, 5566, 5610, 5546, 5540, 5500] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44576 payload_rows_remote=39014 dup_rows=20960 dup_rows_remote=18246 dedup_ratio=0.319824 router_tokens_per_expert=[518, 530, 517, 513, 540, 502, 517, 535, 523, 542, 506, 511, 530, 487, 517, 522, 543, 459, 492, 473, 497, 550, 489, 533, 490, 533, 520, 500, 518, 533, 511, 484, 532, 485, 515, 520, 528, 486, 527, 489, 545, 549, 520, 489, 517, 481, 517, 504, 503, 533, 475, 544, 521, 483, 513, 498, 511, 511, 508, 482, 490, 528, 524, 505] payload_rows_per_dst=[5676, 5562, 5530, 5600, 5526, 5598, 5482, 5602] dup_rows_per_dst=[2668, 2714, 2542, 2578, 2638, 2646, 2658, 2516] combine_rows_per_src=[5676, 5562, 5530, 5600, 5526, 5598, 5482, 5602] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44306 payload_rows_remote=38762 dup_rows=21230 dup_rows_remote=18686 dedup_ratio=0.323944 router_tokens_per_expert=[512, 484, 501, 516, 490, 530, 562, 509, 479, 499, 474, 527, 520, 517, 502, 504, 504, 545, 514, 491, 531, 517, 526, 498, 491, 526, 526, 509, 513, 539, 542, 519, 519, 499, 518, 518, 489, 515, 574, 560, 468, 530, 505, 500, 509, 520, 519, 493, 505, 538, 497, 503, 510, 502, 511, 492, 532, 509, 492, 497, 494, 511, 490, 532] payload_rows_per_dst=[5554, 5432, 5624, 5526, 5568, 5544, 5498, 5560] dup_rows_per_dst=[2654, 2612, 2628, 2804, 2816, 2544, 2618, 2554] combine_rows_per_src=[5554, 5432, 5624, 5526, 5568, 5544, 5498, 5560] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44650 payload_rows_remote=39084 dup_rows=20886 dup_rows_remote=18412 dedup_ratio=0.318695 router_tokens_per_expert=[515, 524, 514, 504, 550, 491, 539, 494, 516, 522, 539, 553, 502, 521, 507, 474, 514, 480, 505, 501, 526, 530, 503, 509, 551, 528, 507, 495, 473, 464, 522, 480, 516, 531, 514, 510, 528, 490, 527, 504, 518, 549, 484, 523, 511, 517, 520, 515, 507, 500, 512, 515, 510, 514, 510, 484, 506, 459, 557, 516, 512, 521, 517, 518] payload_rows_per_dst=[5562, 5644, 5512, 5566, 5556, 5600, 5614, 5596] dup_rows_per_dst=[2700, 2624, 2624, 2474, 2684, 2674, 2490, 2616] combine_rows_per_src=[5562, 5644, 5512, 5566, 5556, 5600, 5614, 5596] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278046720 dispatch_meta_bytes=228632 combine_payload_bytes=278046720 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278275352 ingress_bytes=498247680 link_bytes=498247680 token_bytes=556322072 weight_movement_bytes=220200960 total_nvlink_bytes=776523032 weight_to_token_ratio=0.395816 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.665346632e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277932032 dispatch_meta_bytes=228936 combine_payload_bytes=277932032 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278160968 ingress_bytes=388032512 link_bytes=388032512 token_bytes=556093000 weight_movement_bytes=110100480 total_nvlink_bytes=666193480 weight_to_token_ratio=0.197989 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.748523587e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279752704 dispatch_meta_bytes=229248 combine_payload_bytes=279752704 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279981952 ingress_bytes=389853184 link_bytes=389853184 token_bytes=559734656 weight_movement_bytes=110100480 total_nvlink_bytes=669835136 weight_to_token_ratio=0.196701 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.780188068e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278534144 dispatch_meta_bytes=228816 combine_payload_bytes=278534144 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278762960 ingress_bytes=388634624 link_bytes=388634624 token_bytes=557297104 weight_movement_bytes=110100480 total_nvlink_bytes=667397584 weight_to_token_ratio=0.197562 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.758995305e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277344256 dispatch_meta_bytes=229008 combine_payload_bytes=277344256 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277573264 ingress_bytes=497545216 link_bytes=497545216 token_bytes=554917520 weight_movement_bytes=220200960 total_nvlink_bytes=775118480 weight_to_token_ratio=0.396817 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.653129628e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44426 payload_rows_remote=38798 dup_rows=21110 dup_rows_remote=18344 dedup_ratio=0.322113 router_tokens_per_expert=[506, 482, 531, 498, 545, 519, 487, 504, 489, 500, 545, 491, 500, 533, 513, 510, 521, 521, 509, 505, 531, 491, 543, 531, 511, 497, 511, 518, 512, 493, 481, 505, 531, 507, 547, 537, 523, 539, 520, 493, 502, 485, 511, 532, 486, 518, 520, 489, 504, 536, 516, 524, 533, 548, 510, 514, 491, 557, 495, 516, 474, 464, 527, 486] payload_rows_per_dst=[5532, 5484, 5592, 5478, 5628, 5602, 5658, 5452] dup_rows_per_dst=[2612, 2678, 2712, 2578, 2766, 2484, 2712, 2568] combine_rows_per_src=[5532, 5484, 5592, 5478, 5628, 5602, 5658, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44238 payload_rows_remote=38740 dup_rows=21298 dup_rows_remote=18656 dedup_ratio=0.324982 router_tokens_per_expert=[554, 512, 505, 517, 507, 504, 553, 493, 557, 513, 512, 500, 509, 512, 509, 500, 488, 475, 450, 507, 511, 514, 492, 564, 491, 513, 514, 507, 495, 565, 478, 507, 481, 500, 516, 475, 535, 525, 514, 544, 527, 496, 511, 513, 516, 526, 534, 496, 486, 493, 510, 540, 514, 536, 501, 531, 505, 516, 541, 526, 545, 472, 504, 511] payload_rows_per_dst=[5572, 5528, 5426, 5498, 5524, 5560, 5554, 5576] dup_rows_per_dst=[2718, 2696, 2576, 2642, 2656, 2678, 2668, 2664] combine_rows_per_src=[5572, 5528, 5426, 5498, 5524, 5560, 5554, 5576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44442 payload_rows_remote=38822 dup_rows=21094 dup_rows_remote=18298 dedup_ratio=0.321869 router_tokens_per_expert=[504, 520, 504, 512, 485, 493, 514, 542, 497, 536, 509, 511, 561, 489, 503, 493, 526, 473, 534, 506, 536, 489, 522, 487, 486, 514, 514, 545, 522, 508, 527, 538, 485, 486, 534, 483, 513, 486, 531, 519, 494, 488, 502, 476, 544, 544, 530, 522, 524, 499, 522, 497, 490, 526, 527, 509, 528, 534, 517, 542, 523, 482, 492, 519] payload_rows_per_dst=[5532, 5568, 5542, 5554, 5502, 5528, 5596, 5620] dup_rows_per_dst=[2502, 2736, 2604, 2612, 2572, 2678, 2594, 2796] combine_rows_per_src=[5532, 5568, 5542, 5554, 5502, 5528, 5596, 5620] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44288 payload_rows_remote=38782 dup_rows=21248 dup_rows_remote=18546 dedup_ratio=0.324219 router_tokens_per_expert=[532, 498, 535, 473, 511, 535, 492, 525, 532, 526, 518, 545, 476, 476, 493, 527, 501, 459, 491, 509, 555, 507, 473, 509, 496, 509, 496, 513, 508, 508, 514, 507, 510, 536, 556, 513, 480, 509, 514, 527, 513, 535, 487, 542, 539, 500, 492, 496, 487, 522, 495, 539, 515, 546, 527, 508, 523, 526, 468, 548, 500, 521, 515, 530] payload_rows_per_dst=[5554, 5526, 5452, 5504, 5564, 5506, 5592, 5590] dup_rows_per_dst=[2648, 2660, 2556, 2598, 2726, 2702, 2686, 2672] combine_rows_per_src=[5554, 5526, 5452, 5504, 5564, 5506, 5592, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44714 payload_rows_remote=39108 dup_rows=20822 dup_rows_remote=18296 dedup_ratio=0.317719 router_tokens_per_expert=[488, 512, 518, 562, 540, 518, 504, 523, 539, 515, 488, 490, 492, 512, 503, 527, 546, 491, 503, 500, 526, 509, 533, 546, 535, 526, 552, 531, 494, 565, 527, 475, 529, 499, 528, 466, 527, 511, 493, 508, 539, 485, 504, 484, 478, 499, 512, 532, 476, 488, 493, 499, 527, 527, 519, 519, 479, 503, 492, 488, 513, 502, 509, 550] payload_rows_per_dst=[5654, 5606, 5652, 5642, 5498, 5558, 5538, 5566] dup_rows_per_dst=[2676, 2526, 2656, 2768, 2624, 2508, 2558, 2506] combine_rows_per_src=[5654, 5606, 5652, 5642, 5498, 5558, 5538, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44480 payload_rows_remote=38942 dup_rows=21056 dup_rows_remote=18374 dedup_ratio=0.321289 router_tokens_per_expert=[557, 489, 506, 475, 525, 502, 482, 531, 484, 506, 493, 518, 507, 493, 513, 491, 471, 521, 542, 520, 543, 512, 492, 509, 533, 548, 512, 530, 515, 487, 471, 507, 534, 512, 533, 489, 482, 523, 534, 496, 510, 486, 503, 543, 501, 574, 538, 526, 519, 490, 492, 518, 546, 517, 511, 555, 537, 475, 473, 489, 520, 524, 498, 535] payload_rows_per_dst=[5598, 5452, 5538, 5578, 5594, 5644, 5568, 5508] dup_rows_per_dst=[2536, 2558, 2682, 2628, 2612, 2718, 2728, 2594] combine_rows_per_src=[5598, 5452, 5538, 5578, 5594, 5644, 5568, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278104064 dispatch_meta_bytes=228568 combine_payload_bytes=278104064 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278332632 ingress_bytes=388204544 link_bytes=388204544 token_bytes=556436696 weight_movement_bytes=110100480 total_nvlink_bytes=666537176 weight_to_token_ratio=0.197867 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.751515507e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277688320 dispatch_meta_bytes=229584 combine_payload_bytes=277688320 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277917904 ingress_bytes=497889280 link_bytes=497889280 token_bytes=555606224 weight_movement_bytes=220200960 total_nvlink_bytes=775807184 weight_to_token_ratio=0.396326 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.659113466e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278276096 dispatch_meta_bytes=228480 combine_payload_bytes=278276096 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278504576 ingress_bytes=388376576 link_bytes=388376576 token_bytes=556780672 weight_movement_bytes=110100480 total_nvlink_bytes=666881152 weight_to_token_ratio=0.197745 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.754507426e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44410 payload_rows_remote=38754 dup_rows=21126 dup_rows_remote=18490 dedup_ratio=0.322357 router_tokens_per_expert=[539, 536, 490, 496, 531, 509, 513, 531, 482, 474, 515, 509, 520, 549, 516, 523, 507, 470, 504, 517, 504, 479, 522, 542, 505, 543, 485, 533, 478, 565, 527, 493, 500, 503, 495, 472, 513, 502, 539, 495, 492, 471, 531, 492, 548, 550, 517, 533, 487, 557, 519, 523, 503, 525, 519, 513, 490, 500, 484, 532, 526, 541, 491, 498] payload_rows_per_dst=[5638, 5556, 5510, 5530, 5432, 5548, 5656, 5540] dup_rows_per_dst=[2652, 2620, 2580, 2728, 2606, 2720, 2636, 2584] combine_rows_per_src=[5638, 5556, 5510, 5530, 5432, 5548, 5656, 5540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44476 payload_rows_remote=39010 dup_rows=21060 dup_rows_remote=18414 dedup_ratio=0.321350 router_tokens_per_expert=[529, 518, 483, 470, 528, 505, 523, 500, 546, 514, 536, 495, 525, 523, 513, 519, 485, 507, 466, 516, 515, 545, 498, 557, 523, 492, 488, 526, 512, 513, 523, 509, 543, 497, 490, 553, 471, 518, 516, 513, 517, 483, 508, 514, 522, 506, 494, 522, 510, 501, 488, 489, 514, 547, 475, 525, 503, 547, 538, 480, 499, 533, 532, 518] payload_rows_per_dst=[5466, 5658, 5690, 5596, 5582, 5494, 5384, 5606] dup_rows_per_dst=[2646, 2684, 2768, 2602, 2650, 2608, 2408, 2694] combine_rows_per_src=[5466, 5658, 5690, 5596, 5582, 5494, 5384, 5606] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277989376 dispatch_meta_bytes=229312 combine_payload_bytes=277989376 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278218688 ingress_bytes=498190336 link_bytes=498190336 token_bytes=556208064 weight_movement_bytes=220200960 total_nvlink_bytes=776409024 weight_to_token_ratio=0.395897 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.664349325e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=280326144 dispatch_meta_bytes=229616 combine_payload_bytes=280326144 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280555760 ingress_bytes=390426624 link_bytes=390426624 token_bytes=560881904 weight_movement_bytes=110100480 total_nvlink_bytes=670982384 weight_to_token_ratio=0.196299 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.790161133e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279136256 dispatch_meta_bytes=229264 combine_payload_bytes=279136256 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279365520 ingress_bytes=389236736 link_bytes=389236736 token_bytes=558501776 weight_movement_bytes=110100480 total_nvlink_bytes=668602256 weight_to_token_ratio=0.197135 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.769467023e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277788672 dispatch_meta_bytes=228976 combine_payload_bytes=277788672 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278017648 ingress_bytes=608090112 link_bytes=608090112 token_bytes=555806320 weight_movement_bytes=330301440 total_nvlink_bytes=886107760 weight_to_token_ratio=0.594274 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057568718e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279623680 dispatch_meta_bytes=229696 combine_payload_bytes=279623680 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279853376 ingress_bytes=609925120 link_bytes=609925120 token_bytes=559477056 weight_movement_bytes=330301440 total_nvlink_bytes=889778496 weight_to_token_ratio=0.590375 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060760099e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44552 payload_rows_remote=39082 dup_rows=20984 dup_rows_remote=18440 dedup_ratio=0.320190 router_tokens_per_expert=[518, 501, 492, 530, 546, 489, 512, 511, 517, 548, 508, 525, 526, 518, 535, 534, 488, 517, 558, 478, 521, 506, 497, 524, 492, 477, 491, 516, 561, 470, 504, 505, 495, 545, 510, 499, 512, 484, 476, 486, 514, 534, 506, 509, 534, 502, 520, 532, 526, 513, 510, 469, 507, 493, 515, 521, 492, 536, 491, 531, 518, 521, 519, 533] payload_rows_per_dst=[5560, 5694, 5590, 5494, 5470, 5674, 5510, 5560] dup_rows_per_dst=[2638, 2728, 2588, 2538, 2544, 2628, 2598, 2722] combine_rows_per_src=[5560, 5694, 5590, 5494, 5470, 5674, 5510, 5560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44610 payload_rows_remote=39034 dup_rows=20926 dup_rows_remote=18442 dedup_ratio=0.319305 router_tokens_per_expert=[536, 522, 527, 495, 522, 543, 516, 529, 484, 531, 499, 508, 529, 519, 522, 498, 503, 505, 530, 527, 555, 527, 482, 507, 535, 486, 530, 525, 489, 462, 480, 523, 535, 530, 476, 497, 511, 474, 534, 508, 531, 474, 488, 521, 511, 469, 513, 528, 548, 486, 484, 521, 501, 518, 519, 494, 509, 527, 541, 528, 497, 517, 513, 519] payload_rows_per_dst=[5620, 5544, 5606, 5576, 5570, 5530, 5564, 5600] dup_rows_per_dst=[2760, 2636, 2666, 2484, 2560, 2540, 2578, 2702] combine_rows_per_src=[5620, 5544, 5606, 5576, 5570, 5530, 5564, 5600] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44604 payload_rows_remote=38918 dup_rows=20932 dup_rows_remote=18248 dedup_ratio=0.319397 router_tokens_per_expert=[509, 499, 509, 496, 520, 507, 519, 489, 483, 503, 502, 477, 521, 550, 523, 543, 535, 503, 535, 465, 490, 528, 483, 505, 518, 521, 514, 513, 536, 487, 518, 515, 523, 526, 532, 482, 506, 492, 499, 524, 516, 537, 527, 551, 513, 506, 514, 522, 464, 550, 491, 510, 470, 533, 497, 504, 524, 513, 533, 497, 549, 506, 529, 512] payload_rows_per_dst=[5486, 5606, 5520, 5620, 5518, 5616, 5552, 5686] dup_rows_per_dst=[2558, 2598, 2480, 2624, 2650, 2764, 2574, 2684] combine_rows_per_src=[5486, 5606, 5520, 5620, 5518, 5616, 5552, 5686] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280139776 dispatch_meta_bytes=230088 combine_payload_bytes=280139776 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280369864 ingress_bytes=390240256 link_bytes=390240256 token_bytes=560509640 weight_movement_bytes=110100480 total_nvlink_bytes=670610120 weight_to_token_ratio=0.196429 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.786919887e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44476 payload_rows_remote=38988 dup_rows=21060 dup_rows_remote=18412 dedup_ratio=0.321350 router_tokens_per_expert=[476, 541, 490, 530, 523, 529, 506, 554, 552, 503, 524, 496, 497, 487, 497, 530, 496, 515, 536, 496, 556, 527, 467, 518, 500, 483, 519, 519, 514, 517, 511, 501, 505, 524, 540, 500, 504, 551, 528, 526, 523, 485, 529, 544, 532, 492, 466, 497, 479, 536, 513, 499, 500, 528, 542, 549, 492, 494, 479, 507, 459, 510, 492, 533] payload_rows_per_dst=[5526, 5568, 5598, 5534, 5670, 5488, 5648, 5444] dup_rows_per_dst=[2772, 2604, 2624, 2594, 2686, 2648, 2644, 2488] combine_rows_per_src=[5526, 5568, 5598, 5534, 5670, 5488, 5648, 5444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44450 payload_rows_remote=38986 dup_rows=21086 dup_rows_remote=18502 dedup_ratio=0.321747 router_tokens_per_expert=[514, 535, 522, 515, 497, 506, 508, 500, 503, 497, 483, 500, 518, 495, 551, 477, 487, 501, 507, 543, 504, 497, 512, 475, 459, 519, 507, 493, 490, 528, 551, 484, 529, 509, 510, 522, 487, 490, 536, 515, 544, 535, 509, 526, 515, 508, 561, 506, 539, 511, 500, 507, 532, 533, 515, 538, 491, 550, 514, 503, 524, 499, 541, 491] payload_rows_per_dst=[5564, 5464, 5546, 5478, 5598, 5660, 5548, 5592] dup_rows_per_dst=[2630, 2584, 2506, 2584, 2598, 2748, 2802, 2634] combine_rows_per_src=[5564, 5464, 5546, 5478, 5598, 5660, 5548, 5592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44558 payload_rows_remote=38954 dup_rows=20978 dup_rows_remote=18384 dedup_ratio=0.320099 router_tokens_per_expert=[520, 504, 485, 528, 485, 515, 533, 509, 481, 523, 478, 518, 535, 482, 521, 478, 480, 547, 509, 505, 500, 508, 540, 510, 489, 471, 511, 549, 513, 517, 513, 549, 503, 521, 499, 483, 509, 516, 537, 491, 508, 512, 532, 478, 537, 525, 511, 520, 531, 523, 521, 535, 532, 501, 540, 497, 518, 490, 512, 527, 535, 494, 523, 501] payload_rows_per_dst=[5538, 5504, 5604, 5594, 5552, 5618, 5584, 5564] dup_rows_per_dst=[2620, 2528, 2594, 2630, 2566, 2628, 2776, 2636] combine_rows_per_src=[5538, 5504, 5604, 5594, 5552, 5618, 5584, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279795712 dispatch_meta_bytes=229904 combine_payload_bytes=279795712 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280025616 ingress_bytes=389896192 link_bytes=389896192 token_bytes=559821328 weight_movement_bytes=110100480 total_nvlink_bytes=669921808 weight_to_token_ratio=0.196671 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.780936048e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44656 payload_rows_remote=39146 dup_rows=20880 dup_rows_remote=18304 dedup_ratio=0.318604 router_tokens_per_expert=[476, 504, 500, 500, 512, 518, 490, 501, 509, 512, 531, 486, 553, 476, 509, 497, 519, 512, 535, 546, 505, 518, 500, 512, 513, 518, 528, 573, 545, 463, 508, 547, 531, 481, 528, 506, 496, 508, 514, 545, 498, 503, 514, 515, 485, 494, 514, 520, 525, 513, 487, 490, 533, 466, 533, 496, 555, 500, 497, 550, 510, 510, 500, 535] payload_rows_per_dst=[5500, 5546, 5636, 5644, 5620, 5566, 5510, 5634] dup_rows_per_dst=[2502, 2600, 2658, 2746, 2598, 2520, 2576, 2680] combine_rows_per_src=[5500, 5546, 5636, 5644, 5620, 5566, 5510, 5634] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278964224 dispatch_meta_bytes=228664 combine_payload_bytes=278964224 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279192888 ingress_bytes=499165184 link_bytes=499165184 token_bytes=558157112 weight_movement_bytes=220200960 total_nvlink_bytes=778358072 weight_to_token_ratio=0.394514 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.681303535e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279465984 dispatch_meta_bytes=229600 combine_payload_bytes=279465984 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279695584 ingress_bytes=609767424 link_bytes=609767424 token_bytes=559161568 weight_movement_bytes=330301440 total_nvlink_bytes=889463008 weight_to_token_ratio=0.590708 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060485840e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279451648 dispatch_meta_bytes=229952 combine_payload_bytes=279451648 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279681600 ingress_bytes=389552128 link_bytes=389552128 token_bytes=559133248 weight_movement_bytes=110100480 total_nvlink_bytes=669233728 weight_to_token_ratio=0.196913 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.774952209e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44446 payload_rows_remote=38828 dup_rows=21090 dup_rows_remote=18446 dedup_ratio=0.321808 router_tokens_per_expert=[520, 524, 522, 512, 458, 556, 522, 517, 502, 498, 518, 503, 537, 508, 500, 513, 512, 518, 512, 537, 511, 491, 551, 512, 538, 526, 536, 471, 520, 485, 522, 484, 503, 514, 525, 466, 519, 515, 494, 487, 536, 469, 531, 518, 493, 511, 496, 561, 500, 545, 493, 534, 508, 533, 507, 532, 473, 505, 509, 495, 534, 496, 515, 515] payload_rows_per_dst=[5618, 5622, 5556, 5622, 5626, 5448, 5470, 5484] dup_rows_per_dst=[2644, 2710, 2764, 2774, 2710, 2460, 2602, 2426] combine_rows_per_src=[5618, 5622, 5556, 5622, 5626, 5448, 5470, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279222272 dispatch_meta_bytes=229352 combine_payload_bytes=279222272 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279451624 ingress_bytes=499423232 link_bytes=499423232 token_bytes=558673896 weight_movement_bytes=220200960 total_nvlink_bytes=778874856 weight_to_token_ratio=0.394149 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.685791415e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280598528 dispatch_meta_bytes=229800 combine_payload_bytes=280598528 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280828328 ingress_bytes=500799488 link_bytes=500799488 token_bytes=561426856 weight_movement_bytes=220200960 total_nvlink_bytes=781627816 weight_to_token_ratio=0.392217 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.709726770e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44468 payload_rows_remote=38788 dup_rows=21068 dup_rows_remote=18368 dedup_ratio=0.321472 router_tokens_per_expert=[539, 507, 518, 511, 490, 491, 506, 493, 511, 507, 490, 490, 543, 504, 514, 545, 494, 518, 493, 503, 449, 515, 514, 527, 554, 554, 478, 486, 509, 486, 486, 518, 491, 482, 532, 544, 552, 531, 509, 549, 511, 575, 525, 511, 493, 534, 521, 513, 503, 506, 475, 468, 538, 503, 537, 488, 522, 523, 520, 502, 562, 514, 491, 500] payload_rows_per_dst=[5530, 5544, 5510, 5534, 5680, 5608, 5492, 5570] dup_rows_per_dst=[2580, 2664, 2516, 2608, 2700, 2758, 2544, 2698] combine_rows_per_src=[5530, 5544, 5510, 5534, 5680, 5608, 5492, 5570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278319104 dispatch_meta_bytes=229096 combine_payload_bytes=278319104 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=278548200 ingress_bytes=498520064 link_bytes=498520064 token_bytes=556867304 weight_movement_bytes=220200960 total_nvlink_bytes=777068264 weight_to_token_ratio=0.395428 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.670083838e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44660 payload_rows_remote=39062 dup_rows=20876 dup_rows_remote=18186 dedup_ratio=0.318542 router_tokens_per_expert=[526, 513, 516, 526, 503, 485, 501, 501, 531, 512, 503, 577, 502, 527, 502, 514, 524, 514, 523, 491, 521, 502, 527, 493, 470, 497, 510, 543, 529, 528, 519, 548, 499, 517, 544, 514, 525, 467, 494, 505, 526, 491, 478, 534, 471, 513, 473, 502, 516, 512, 502, 494, 525, 514, 517, 525, 555, 505, 505, 507, 516, 533, 510, 501] payload_rows_per_dst=[5510, 5550, 5648, 5598, 5568, 5476, 5636, 5674] dup_rows_per_dst=[2632, 2786, 2542, 2690, 2562, 2500, 2574, 2590] combine_rows_per_src=[5510, 5550, 5648, 5598, 5568, 5476, 5636, 5674] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44586 payload_rows_remote=38882 dup_rows=20950 dup_rows_remote=18160 dedup_ratio=0.319672 router_tokens_per_expert=[468, 476, 509, 504, 490, 480, 555, 493, 502, 501, 522, 518, 478, 534, 499, 523, 502, 520, 480, 508, 501, 489, 497, 549, 536, 513, 494, 479, 513, 511, 532, 504, 512, 528, 545, 449, 533, 502, 483, 530, 501, 536, 519, 481, 507, 505, 537, 486, 543, 500, 546, 540, 530, 554, 507, 528, 529, 537, 537, 505, 537, 511, 536, 494] payload_rows_per_dst=[5488, 5422, 5554, 5624, 5428, 5660, 5706, 5704] dup_rows_per_dst=[2462, 2452, 2538, 2540, 2638, 2740, 2790, 2790] combine_rows_per_src=[5488, 5422, 5554, 5624, 5428, 5660, 5706, 5704] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44318 payload_rows_remote=38758 dup_rows=21218 dup_rows_remote=18664 dedup_ratio=0.323761 router_tokens_per_expert=[509, 479, 500, 513, 487, 526, 558, 507, 480, 496, 469, 525, 520, 514, 501, 505, 503, 545, 513, 491, 526, 517, 527, 498, 491, 522, 528, 509, 514, 540, 540, 520, 519, 495, 516, 522, 490, 518, 575, 562, 471, 531, 506, 500, 510, 521, 523, 495, 505, 539, 500, 506, 511, 504, 514, 495, 533, 509, 492, 499, 495, 514, 491, 534] payload_rows_per_dst=[5534, 5430, 5614, 5526, 5570, 5560, 5514, 5570] dup_rows_per_dst=[2624, 2590, 2626, 2802, 2824, 2554, 2634, 2564] combine_rows_per_src=[5534, 5430, 5614, 5526, 5570, 5560, 5514, 5570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278032384 dispatch_meta_bytes=228624 combine_payload_bytes=278032384 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278261008 ingress_bytes=498233344 link_bytes=498233344 token_bytes=556293392 weight_movement_bytes=220200960 total_nvlink_bytes=776494352 weight_to_token_ratio=0.395836 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.665097305e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44576 payload_rows_remote=39030 dup_rows=20960 dup_rows_remote=18262 dedup_ratio=0.319824 router_tokens_per_expert=[514, 529, 516, 513, 538, 498, 514, 532, 518, 539, 505, 511, 527, 485, 515, 522, 541, 456, 489, 474, 499, 545, 487, 529, 489, 533, 523, 498, 517, 530, 512, 486, 531, 484, 517, 522, 528, 489, 526, 490, 545, 552, 521, 492, 516, 483, 519, 504, 503, 534, 475, 545, 521, 482, 514, 503, 511, 511, 512, 487, 494, 535, 528, 510] payload_rows_per_dst=[5660, 5546, 5518, 5602, 5532, 5608, 5484, 5626] dup_rows_per_dst=[2648, 2698, 2522, 2574, 2642, 2656, 2670, 2550] combine_rows_per_src=[5660, 5546, 5518, 5602, 5532, 5608, 5484, 5626] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44488 payload_rows_remote=38952 dup_rows=21048 dup_rows_remote=18394 dedup_ratio=0.321167 router_tokens_per_expert=[497, 506, 504, 504, 490, 510, 512, 475, 476, 552, 507, 572, 525, 529, 511, 510, 526, 521, 532, 510, 500, 478, 526, 502, 516, 530, 507, 477, 529, 515, 509, 477, 546, 520, 476, 505, 525, 497, 523, 524, 511, 522, 547, 508, 513, 506, 522, 511, 532, 507, 526, 481, 511, 579, 498, 463, 531, 474, 475, 487, 530, 544, 537, 502] payload_rows_per_dst=[5512, 5650, 5536, 5562, 5612, 5546, 5548, 5522] dup_rows_per_dst=[2484, 2714, 2654, 2558, 2620, 2734, 2646, 2638] combine_rows_per_src=[5512, 5650, 5536, 5562, 5612, 5546, 5548, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279996416 dispatch_meta_bytes=228992 combine_payload_bytes=279996416 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280225408 ingress_bytes=610297856 link_bytes=610297856 token_bytes=560221824 weight_movement_bytes=330301440 total_nvlink_bytes=890523264 weight_to_token_ratio=0.589590 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061408348e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279236608 dispatch_meta_bytes=229344 combine_payload_bytes=279236608 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279465952 ingress_bytes=389337088 link_bytes=389337088 token_bytes=558702560 weight_movement_bytes=110100480 total_nvlink_bytes=668803040 weight_to_token_ratio=0.197065 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.771212310e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279652352 dispatch_meta_bytes=229040 combine_payload_bytes=279652352 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279881392 ingress_bytes=609953792 link_bytes=609953792 token_bytes=559533744 weight_movement_bytes=330301440 total_nvlink_bytes=889835184 weight_to_token_ratio=0.590315 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060809964e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277846016 dispatch_meta_bytes=229792 combine_payload_bytes=277846016 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278075808 ingress_bytes=387946496 link_bytes=387946496 token_bytes=555921824 weight_movement_bytes=110100480 total_nvlink_bytes=666022304 weight_to_token_ratio=0.198050 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.747027628e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44586 payload_rows_remote=38932 dup_rows=20950 dup_rows_remote=18276 dedup_ratio=0.319672 router_tokens_per_expert=[471, 477, 510, 506, 491, 481, 561, 494, 502, 502, 522, 518, 483, 537, 502, 525, 504, 522, 479, 509, 499, 493, 496, 549, 536, 514, 495, 480, 516, 512, 533, 502, 510, 531, 541, 449, 535, 501, 483, 532, 499, 536, 518, 481, 505, 505, 536, 486, 540, 497, 546, 538, 528, 553, 507, 526, 527, 535, 532, 501, 533, 509, 534, 493] payload_rows_per_dst=[5504, 5302, 5632, 5518, 5490, 5696, 5790, 5654] dup_rows_per_dst=[2478, 2312, 2666, 2418, 2674, 2808, 2920, 2674] combine_rows_per_src=[5504, 5302, 5632, 5518, 5490, 5696, 5790, 5654] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44578 payload_rows_remote=38910 dup_rows=20958 dup_rows_remote=18150 dedup_ratio=0.319794 router_tokens_per_expert=[495, 521, 543, 531, 531, 526, 522, 503, 508, 502, 529, 538, 496, 562, 530, 489, 530, 557, 537, 472, 528, 494, 531, 541, 503, 538, 525, 516, 528, 538, 530, 489, 502, 444, 493, 527, 524, 490, 544, 503, 534, 469, 504, 485, 538, 475, 467, 513, 471, 531, 505, 472, 545, 481, 493, 469, 505, 536, 472, 490, 511, 523, 538, 531] payload_rows_per_dst=[5668, 5636, 5664, 5608, 5446, 5512, 5458, 5586] dup_rows_per_dst=[2808, 2720, 2716, 2594, 2578, 2458, 2476, 2608] combine_rows_per_src=[5668, 5636, 5664, 5608, 5446, 5512, 5458, 5586] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=280154112 dispatch_meta_bytes=229984 combine_payload_bytes=280154112 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280384096 ingress_bytes=610455552 link_bytes=610455552 token_bytes=560538208 weight_movement_bytes=330301440 total_nvlink_bytes=890839648 weight_to_token_ratio=0.589258 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061682608e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278906880 dispatch_meta_bytes=228240 combine_payload_bytes=278906880 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279135120 ingress_bytes=389007360 link_bytes=389007360 token_bytes=558042000 weight_movement_bytes=110100480 total_nvlink_bytes=668142480 weight_to_token_ratio=0.197298 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.765477798e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279064576 dispatch_meta_bytes=228832 combine_payload_bytes=279064576 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=279293408 ingress_bytes=389165056 link_bytes=389165056 token_bytes=558357984 weight_movement_bytes=110100480 total_nvlink_bytes=668458464 weight_to_token_ratio=0.197186 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.768220390e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44346 payload_rows_remote=38974 dup_rows=21190 dup_rows_remote=18620 dedup_ratio=0.323334 router_tokens_per_expert=[505, 540, 506, 494, 519, 479, 534, 527, 509, 500, 514, 500, 515, 490, 509, 496, 547, 514, 518, 542, 517, 528, 506, 481, 510, 514, 516, 502, 510, 519, 517, 500, 513, 524, 494, 459, 482, 517, 525, 457, 531, 520, 512, 555, 563, 474, 522, 529, 498, 525, 518, 532, 530, 522, 461, 497, 477, 509, 534, 520, 532, 543, 502, 513] payload_rows_per_dst=[5518, 5560, 5580, 5508, 5372, 5672, 5504, 5632] dup_rows_per_dst=[2690, 2506, 2726, 2668, 2570, 2740, 2662, 2628] combine_rows_per_src=[5518, 5560, 5580, 5508, 5372, 5672, 5504, 5632] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44518 payload_rows_remote=38830 dup_rows=21018 dup_rows_remote=18226 dedup_ratio=0.320709 router_tokens_per_expert=[496, 537, 511, 492, 465, 495, 487, 514, 535, 509, 551, 534, 531, 514, 480, 527, 509, 519, 563, 520, 560, 442, 530, 516, 525, 521, 488, 494, 491, 531, 515, 522, 518, 510, 507, 486, 515, 540, 496, 505, 535, 498, 498, 504, 485, 486, 480, 514, 511, 497, 556, 532, 524, 569, 525, 526, 492, 514, 491, 501, 516, 501, 518, 494] payload_rows_per_dst=[5508, 5614, 5558, 5608, 5492, 5508, 5688, 5542] dup_rows_per_dst=[2486, 2748, 2760, 2566, 2662, 2492, 2792, 2512] combine_rows_per_src=[5508, 5614, 5558, 5608, 5492, 5508, 5688, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44384 payload_rows_remote=38748 dup_rows=21152 dup_rows_remote=18454 dedup_ratio=0.322754 router_tokens_per_expert=[531, 484, 506, 514, 499, 514, 481, 517, 512, 514, 512, 522, 500, 492, 538, 511, 500, 522, 469, 468, 509, 504, 536, 526, 525, 532, 520, 513, 498, 540, 546, 483, 484, 509, 518, 468, 540, 510, 538, 480, 513, 500, 490, 511, 507, 498, 511, 548, 511, 557, 474, 534, 532, 498, 549, 512, 526, 502, 486, 523, 513, 526, 535, 527] payload_rows_per_dst=[5450, 5536, 5528, 5582, 5544, 5538, 5636, 5570] dup_rows_per_dst=[2642, 2666, 2540, 2732, 2550, 2618, 2698, 2706] combine_rows_per_src=[5450, 5536, 5528, 5582, 5544, 5538, 5636, 5570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278706176 dispatch_meta_bytes=228168 combine_payload_bytes=278706176 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278934344 ingress_bytes=498907136 link_bytes=498907136 token_bytes=557640520 weight_movement_bytes=220200960 total_nvlink_bytes=777841480 weight_to_token_ratio=0.394880 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.676815656e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277817344 dispatch_meta_bytes=229688 combine_payload_bytes=277817344 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278047032 ingress_bytes=387917824 link_bytes=387917824 token_bytes=555864376 weight_movement_bytes=110100480 total_nvlink_bytes=665964856 weight_to_token_ratio=0.198071 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.746528975e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44600 payload_rows_remote=38808 dup_rows=20936 dup_rows_remote=17948 dedup_ratio=0.319458 router_tokens_per_expert=[491, 518, 540, 526, 530, 524, 521, 500, 507, 500, 528, 536, 494, 558, 528, 489, 531, 556, 535, 472, 530, 492, 529, 538, 504, 539, 524, 515, 530, 538, 529, 488, 503, 444, 493, 527, 525, 489, 547, 505, 535, 470, 503, 485, 540, 476, 466, 515, 474, 535, 507, 472, 547, 481, 496, 471, 505, 537, 477, 490, 517, 525, 538, 533] payload_rows_per_dst=[5792, 5618, 5802, 5526, 5458, 5518, 5472, 5414] dup_rows_per_dst=[2988, 2670, 2888, 2478, 2608, 2462, 2492, 2350] combine_rows_per_src=[5792, 5618, 5802, 5526, 5458, 5518, 5472, 5414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279767040 dispatch_meta_bytes=229168 combine_payload_bytes=279767040 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279996208 ingress_bytes=610068480 link_bytes=610068480 token_bytes=559763248 weight_movement_bytes=330301440 total_nvlink_bytes=890064688 weight_to_token_ratio=0.590073 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061009426e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279207936 dispatch_meta_bytes=229384 combine_payload_bytes=279207936 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279437320 ingress_bytes=389308416 link_bytes=389308416 token_bytes=558645256 weight_movement_bytes=110100480 total_nvlink_bytes=668745736 weight_to_token_ratio=0.197085 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.770713657e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44346 payload_rows_remote=38974 dup_rows=21190 dup_rows_remote=18620 dedup_ratio=0.323334 router_tokens_per_expert=[498, 539, 504, 492, 516, 479, 530, 526, 508, 500, 512, 499, 512, 486, 506, 494, 546, 511, 518, 542, 516, 525, 504, 480, 510, 513, 515, 502, 507, 519, 519, 500, 513, 526, 494, 461, 482, 514, 525, 456, 531, 521, 512, 558, 564, 477, 524, 532, 498, 527, 517, 534, 533, 530, 462, 498, 481, 511, 536, 523, 532, 547, 505, 516] payload_rows_per_dst=[5500, 5550, 5568, 5508, 5372, 5676, 5524, 5648] dup_rows_per_dst=[2668, 2484, 2716, 2662, 2570, 2762, 2674, 2654] combine_rows_per_src=[5500, 5550, 5568, 5508, 5372, 5676, 5524, 5648] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277745664 dispatch_meta_bytes=228808 combine_payload_bytes=277745664 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=277974472 ingress_bytes=387846144 link_bytes=387846144 token_bytes=555720136 weight_movement_bytes=110100480 total_nvlink_bytes=665820616 weight_to_token_ratio=0.198122 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.745282341e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278175744 dispatch_meta_bytes=227024 combine_payload_bytes=278175744 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278402768 ingress_bytes=388276224 link_bytes=388276224 token_bytes=556578512 weight_movement_bytes=110100480 total_nvlink_bytes=666678992 weight_to_token_ratio=0.197817 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.752762140e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44244 payload_rows_remote=38684 dup_rows=21292 dup_rows_remote=18612 dedup_ratio=0.324890 router_tokens_per_expert=[499, 531, 502, 528, 511, 528, 530, 507, 477, 488, 530, 504, 553, 530, 514, 524, 523, 510, 512, 542, 494, 515, 564, 512, 510, 526, 531, 511, 517, 514, 532, 495, 495, 480, 520, 510, 542, 499, 518, 518, 496, 475, 522, 503, 476, 482, 520, 506, 545, 529, 474, 511, 547, 493, 481, 528, 477, 521, 495, 512, 511, 487, 536, 495] payload_rows_per_dst=[5530, 5560, 5624, 5542, 5500, 5498, 5546, 5444] dup_rows_per_dst=[2742, 2680, 2720, 2730, 2664, 2462, 2670, 2624] combine_rows_per_src=[5530, 5560, 5624, 5542, 5500, 5498, 5546, 5444] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44516 payload_rows_remote=38894 dup_rows=21020 dup_rows_remote=18270 dedup_ratio=0.320740 router_tokens_per_expert=[527, 488, 537, 469, 545, 498, 495, 519, 506, 516, 521, 512, 515, 518, 491, 482, 505, 521, 518, 512, 536, 545, 521, 528, 507, 489, 500, 518, 561, 537, 532, 495, 496, 529, 496, 516, 487, 513, 501, 506, 543, 486, 516, 527, 518, 503, 446, 535, 524, 539, 501, 529, 523, 544, 475, 526, 511, 494, 509, 512, 516, 473, 509, 501] payload_rows_per_dst=[5560, 5566, 5622, 5594, 5504, 5520, 5586, 5564] dup_rows_per_dst=[2596, 2556, 2750, 2684, 2584, 2628, 2736, 2486] combine_rows_per_src=[5560, 5566, 5622, 5594, 5504, 5520, 5586, 5564] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44616 payload_rows_remote=39094 dup_rows=20920 dup_rows_remote=18284 dedup_ratio=0.319214 router_tokens_per_expert=[486, 494, 556, 512, 520, 508, 468, 522, 521, 559, 540, 492, 495, 512, 483, 548, 535, 528, 522, 507, 512, 523, 520, 500, 500, 485, 538, 537, 497, 513, 502, 533, 512, 477, 510, 515, 509, 510, 523, 517, 545, 475, 522, 512, 492, 519, 516, 498, 478, 517, 516, 487, 508, 558, 497, 490, 479, 545, 473, 515, 516, 518, 538, 513] payload_rows_per_dst=[5554, 5606, 5636, 5588, 5562, 5522, 5572, 5576] dup_rows_per_dst=[2578, 2694, 2658, 2622, 2584, 2636, 2530, 2618] combine_rows_per_src=[5554, 5606, 5636, 5588, 5562, 5522, 5572, 5576] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44640 payload_rows_remote=39066 dup_rows=20896 dup_rows_remote=18424 dedup_ratio=0.318848 router_tokens_per_expert=[512, 519, 511, 501, 546, 490, 537, 494, 514, 520, 536, 552, 503, 517, 508, 474, 511, 478, 504, 501, 524, 528, 500, 512, 554, 528, 506, 494, 473, 470, 518, 480, 514, 531, 513, 511, 525, 488, 527, 509, 516, 552, 483, 522, 512, 519, 520, 517, 511, 500, 512, 516, 510, 518, 513, 490, 507, 462, 559, 518, 516, 522, 519, 521] payload_rows_per_dst=[5542, 5632, 5498, 5574, 5546, 5598, 5634, 5616] dup_rows_per_dst=[2678, 2616, 2618, 2472, 2690, 2684, 2506, 2632] combine_rows_per_src=[5542, 5632, 5498, 5574, 5546, 5598, 5634, 5616] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44552 payload_rows_remote=38924 dup_rows=20984 dup_rows_remote=18316 dedup_ratio=0.320190 router_tokens_per_expert=[454, 495, 550, 509, 531, 530, 488, 523, 502, 480, 511, 498, 508, 548, 515, 505, 490, 511, 514, 527, 524, 498, 544, 558, 500, 515, 523, 496, 527, 515, 503, 497, 503, 497, 517, 520, 505, 494, 489, 494, 505, 492, 514, 510, 518, 499, 541, 484, 490, 534, 510, 531, 521, 558, 519, 538, 528, 511, 494, 526, 530, 502, 490, 515] payload_rows_per_dst=[5550, 5510, 5390, 5402, 5674, 5682, 5716, 5628] dup_rows_per_dst=[2610, 2560, 2334, 2398, 2906, 2796, 2712, 2668] combine_rows_per_src=[5550, 5510, 5390, 5402, 5674, 5682, 5716, 5628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44612 payload_rows_remote=39088 dup_rows=20924 dup_rows_remote=18264 dedup_ratio=0.319275 router_tokens_per_expert=[485, 492, 555, 509, 518, 505, 465, 520, 519, 557, 537, 492, 493, 510, 481, 550, 533, 527, 523, 503, 510, 522, 519, 496, 498, 483, 537, 534, 497, 514, 501, 534, 509, 477, 507, 516, 509, 511, 522, 519, 546, 475, 524, 512, 495, 519, 520, 501, 480, 517, 516, 489, 508, 563, 498, 493, 480, 547, 481, 517, 517, 522, 542, 517] payload_rows_per_dst=[5544, 5594, 5624, 5582, 5558, 5524, 5582, 5604] dup_rows_per_dst=[2554, 2684, 2642, 2614, 2582, 2660, 2546, 2642] combine_rows_per_src=[5544, 5594, 5624, 5582, 5558, 5524, 5582, 5604] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279365632 dispatch_meta_bytes=230376 combine_payload_bytes=279365632 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279596008 ingress_bytes=389466112 link_bytes=389466112 token_bytes=558961640 weight_movement_bytes=110100480 total_nvlink_bytes=669062120 weight_to_token_ratio=0.196973 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773456249e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44440 payload_rows_remote=38922 dup_rows=21096 dup_rows_remote=18390 dedup_ratio=0.321899 router_tokens_per_expert=[514, 581, 501, 520, 519, 497, 512, 531, 478, 519, 544, 503, 548, 473, 550, 527, 521, 529, 478, 528, 493, 513, 479, 507, 520, 490, 522, 552, 496, 522, 514, 496, 487, 487, 527, 504, 538, 478, 524, 502, 500, 526, 487, 536, 516, 507, 500, 493, 513, 469, 537, 508, 526, 539, 510, 531, 508, 509, 518, 509, 470, 489, 507, 536] payload_rows_per_dst=[5636, 5630, 5532, 5518, 5548, 5552, 5576, 5448] dup_rows_per_dst=[2714, 2654, 2564, 2706, 2546, 2578, 2690, 2644] combine_rows_per_src=[5636, 5630, 5532, 5518, 5548, 5552, 5576, 5448] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278333440 dispatch_meta_bytes=228224 combine_payload_bytes=278333440 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278561664 ingress_bytes=498534400 link_bytes=498534400 token_bytes=556895104 weight_movement_bytes=220200960 total_nvlink_bytes=777096064 weight_to_token_ratio=0.395408 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.670333164e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44386 payload_rows_remote=38752 dup_rows=21150 dup_rows_remote=18468 dedup_ratio=0.322723 router_tokens_per_expert=[534, 518, 518, 523, 478, 511, 522, 502, 565, 517, 485, 547, 519, 515, 494, 516, 512, 475, 506, 502, 534, 502, 525, 546, 529, 528, 526, 515, 500, 501, 560, 566, 480, 528, 514, 520, 524, 526, 520, 479, 459, 487, 506, 506, 508, 508, 498, 514, 501, 506, 524, 483, 536, 537, 493, 465, 508, 512, 521, 506, 519, 486, 508, 495] payload_rows_per_dst=[5634, 5678, 5502, 5650, 5576, 5582, 5254, 5510] dup_rows_per_dst=[2682, 2780, 2560, 2696, 2606, 2888, 2338, 2600] combine_rows_per_src=[5634, 5678, 5502, 5650, 5576, 5582, 5254, 5510] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=277286912 dispatch_meta_bytes=229184 combine_payload_bytes=277286912 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277516096 ingress_bytes=497487872 link_bytes=497487872 token_bytes=554803008 weight_movement_bytes=220200960 total_nvlink_bytes=775003968 weight_to_token_ratio=0.396899 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.652132321e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44568 payload_rows_remote=38824 dup_rows=20968 dup_rows_remote=18124 dedup_ratio=0.319946 router_tokens_per_expert=[458, 496, 553, 513, 532, 531, 491, 524, 503, 482, 514, 501, 510, 548, 517, 503, 492, 513, 516, 525, 527, 499, 544, 559, 502, 516, 527, 496, 527, 513, 502, 495, 503, 497, 517, 518, 505, 490, 489, 491, 504, 490, 515, 509, 519, 496, 541, 483, 489, 534, 507, 530, 523, 556, 518, 539, 527, 506, 493, 523, 526, 499, 491, 511] payload_rows_per_dst=[5566, 5372, 5370, 5506, 5684, 5566, 5760, 5744] dup_rows_per_dst=[2630, 2348, 2334, 2534, 2936, 2548, 2794, 2844] combine_rows_per_src=[5566, 5372, 5370, 5506, 5684, 5566, 5760, 5744] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278792192 dispatch_meta_bytes=228656 combine_payload_bytes=278792192 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279020848 ingress_bytes=609093632 link_bytes=609093632 token_bytes=557813040 weight_movement_bytes=330301440 total_nvlink_bytes=888114480 weight_to_token_ratio=0.592136 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059314005e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=280225792 dispatch_meta_bytes=229512 combine_payload_bytes=280225792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280455304 ingress_bytes=390326272 link_bytes=390326272 token_bytes=560681096 weight_movement_bytes=110100480 total_nvlink_bytes=670781576 weight_to_token_ratio=0.196369 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.788415846e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44432 payload_rows_remote=38958 dup_rows=21104 dup_rows_remote=18496 dedup_ratio=0.322021 router_tokens_per_expert=[527, 548, 512, 505, 490, 501, 496, 507, 501, 536, 539, 503, 524, 509, 525, 482, 526, 499, 506, 509, 502, 528, 497, 533, 527, 487, 484, 535, 525, 556, 486, 494, 512, 530, 491, 496, 523, 521, 456, 512, 525, 475, 511, 500, 487, 495, 554, 522, 533, 503, 528, 504, 490, 541, 537, 519, 487, 499, 556, 510, 526, 494, 511, 521] payload_rows_per_dst=[5540, 5538, 5550, 5604, 5474, 5516, 5636, 5574] dup_rows_per_dst=[2632, 2700, 2650, 2584, 2608, 2622, 2674, 2634] combine_rows_per_src=[5540, 5538, 5550, 5604, 5474, 5516, 5636, 5574] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44478 payload_rows_remote=39016 dup_rows=21058 dup_rows_remote=18554 dedup_ratio=0.321320 router_tokens_per_expert=[516, 518, 532, 481, 522, 522, 495, 515, 553, 497, 526, 507, 558, 518, 478, 476, 531, 500, 539, 498, 496, 521, 491, 479, 489, 493, 546, 489, 540, 528, 520, 513, 509, 519, 475, 503, 529, 477, 575, 558, 521, 498, 512, 494, 546, 522, 535, 472, 496, 537, 483, 490, 493, 498, 517, 469, 538, 496, 524, 534, 530, 499, 514, 518] payload_rows_per_dst=[5610, 5554, 5482, 5598, 5614, 5548, 5462, 5610] dup_rows_per_dst=[2592, 2672, 2628, 2638, 2676, 2652, 2504, 2696] combine_rows_per_src=[5610, 5554, 5482, 5598, 5614, 5548, 5462, 5610] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278992896 dispatch_meta_bytes=229248 combine_payload_bytes=278992896 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279222144 ingress_bytes=609294336 link_bytes=609294336 token_bytes=558215040 weight_movement_bytes=330301440 total_nvlink_bytes=888516480 weight_to_token_ratio=0.591710 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059663062e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=277774336 dispatch_meta_bytes=228880 combine_payload_bytes=277774336 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278003216 ingress_bytes=387874816 link_bytes=387874816 token_bytes=555777552 weight_movement_bytes=110100480 total_nvlink_bytes=665878032 weight_to_token_ratio=0.198102 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.745780995e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278290432 dispatch_meta_bytes=227792 combine_payload_bytes=278290432 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278518224 ingress_bytes=388390912 link_bytes=388390912 token_bytes=556808656 weight_movement_bytes=110100480 total_nvlink_bytes=666909136 weight_to_token_ratio=0.197735 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.754756753e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279250944 dispatch_meta_bytes=229816 combine_payload_bytes=279250944 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279480760 ingress_bytes=389351424 link_bytes=389351424 token_bytes=558731704 weight_movement_bytes=110100480 total_nvlink_bytes=668832184 weight_to_token_ratio=0.197054 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.771461636e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5581577400 prefetch_bytes=990904320 reduce_bytes=990904320 weight_movement_bytes=1981808640 egress_bytes=2791935160 ingress_bytes=4771450880 link_bytes=4771450880 total_nvlink_bytes=7563386040 weight_to_token_ratio=0.355062 prefetch_share_of_ingress=0.207674 dedup_ratio=0.321118 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=9 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043015 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.298337849e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279666688 dispatch_meta_bytes=230280 combine_payload_bytes=279666688 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279896968 ingress_bytes=389767168 link_bytes=389767168 token_bytes=559563656 weight_movement_bytes=110100480 total_nvlink_bytes=669664136 weight_to_token_ratio=0.196761 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778692108e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44600 payload_rows_remote=39032 dup_rows=20936 dup_rows_remote=18410 dedup_ratio=0.319458 router_tokens_per_expert=[516, 532, 511, 498, 500, 503, 524, 480, 500, 518, 500, 501, 508, 496, 515, 509, 533, 525, 506, 499, 554, 500, 480, 492, 511, 538, 549, 543, 513, 503, 538, 513, 535, 506, 484, 531, 547, 497, 541, 525, 495, 500, 507, 530, 490, 544, 539, 508, 527, 507, 509, 503, 503, 489, 493, 494, 494, 493, 504, 535, 511, 504, 516, 499] payload_rows_per_dst=[5582, 5568, 5584, 5700, 5596, 5568, 5540, 5462] dup_rows_per_dst=[2546, 2526, 2594, 2716, 2736, 2658, 2510, 2650] combine_rows_per_src=[5582, 5568, 5584, 5700, 5596, 5568, 5540, 5462] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5579770864 prefetch_bytes=990904320 reduce_bytes=770703360 weight_movement_bytes=1761607680 egress_bytes=2791031792 ingress_bytes=4550346752 link_bytes=4550346752 total_nvlink_bytes=7341378544 weight_to_token_ratio=0.315713 prefetch_share_of_ingress=0.217765 dedup_ratio=0.321539 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=7 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.913801405e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44364 payload_rows_remote=38872 dup_rows=21172 dup_rows_remote=18588 dedup_ratio=0.323059 router_tokens_per_expert=[511, 554, 545, 534, 531, 548, 499, 520, 515, 501, 519, 526, 513, 482, 540, 516, 495, 494, 505, 529, 492, 494, 532, 497, 509, 520, 513, 500, 506, 518, 544, 517, 519, 474, 525, 522, 467, 512, 533, 510, 476, 504, 502, 514, 538, 537, 525, 517, 505, 499, 510, 471, 497, 466, 519, 505, 459, 489, 532, 535, 508, 510, 570, 499] payload_rows_per_dst=[5622, 5614, 5492, 5556, 5516, 5534, 5440, 5590] dup_rows_per_dst=[2862, 2610, 2584, 2698, 2608, 2692, 2504, 2614] combine_rows_per_src=[5622, 5614, 5492, 5556, 5516, 5534, 5440, 5590] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44540 payload_rows_remote=38922 dup_rows=20996 dup_rows_remote=18354 dedup_ratio=0.320374 router_tokens_per_expert=[522, 499, 498, 515, 503, 508, 527, 537, 502, 497, 510, 532, 525, 503, 487, 513, 508, 529, 502, 510, 566, 522, 524, 500, 487, 507, 495, 533, 504, 495, 525, 525, 526, 526, 497, 517, 503, 517, 521, 513, 510, 525, 550, 542, 489, 507, 488, 519, 486, 515, 502, 526, 497, 485, 538, 486, 520, 518, 479, 539, 517, 519, 478, 503] payload_rows_per_dst=[5542, 5586, 5608, 5518, 5590, 5618, 5512, 5566] dup_rows_per_dst=[2676, 2552, 2714, 2624, 2650, 2642, 2558, 2580] combine_rows_per_src=[5542, 5586, 5608, 5518, 5590, 5618, 5512, 5566] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5581980296 prefetch_bytes=990904320 reduce_bytes=220200960 weight_movement_bytes=1211105280 egress_bytes=2792137352 ingress_bytes=4000948224 link_bytes=4000948224 total_nvlink_bytes=6793085576 weight_to_token_ratio=0.216967 prefetch_share_of_ingress=0.247667 dedup_ratio=0.321164 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=2 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042647 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=6.958307004e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44208 payload_rows_remote=38662 dup_rows=21328 dup_rows_remote=18672 dedup_ratio=0.325439 router_tokens_per_expert=[495, 528, 498, 527, 510, 525, 529, 508, 476, 487, 525, 500, 551, 528, 511, 523, 521, 509, 512, 542, 492, 511, 561, 511, 509, 525, 532, 510, 517, 514, 535, 494, 495, 482, 517, 509, 541, 498, 524, 523, 498, 475, 524, 503, 478, 488, 519, 506, 546, 533, 474, 512, 550, 495, 481, 531, 478, 523, 497, 512, 516, 488, 540, 496] payload_rows_per_dst=[5512, 5546, 5614, 5538, 5504, 5498, 5546, 5450] dup_rows_per_dst=[2728, 2656, 2704, 2734, 2674, 2484, 2698, 2650] combine_rows_per_src=[5512, 5546, 5614, 5538, 5504, 5498, 5546, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44494 payload_rows_remote=38892 dup_rows=21042 dup_rows_remote=18304 dedup_ratio=0.321075 router_tokens_per_expert=[524, 485, 533, 466, 540, 499, 493, 519, 504, 515, 520, 512, 513, 518, 491, 478, 502, 519, 516, 510, 535, 541, 519, 528, 507, 487, 496, 517, 559, 540, 530, 497, 499, 525, 496, 517, 489, 515, 502, 509, 544, 486, 521, 528, 520, 504, 447, 537, 527, 538, 504, 530, 526, 545, 479, 528, 513, 496, 511, 512, 516, 474, 511, 506] payload_rows_per_dst=[5536, 5562, 5602, 5588, 5510, 5524, 5598, 5574] dup_rows_per_dst=[2582, 2540, 2738, 2678, 2594, 2650, 2756, 2504] combine_rows_per_src=[5536, 5562, 5602, 5588, 5510, 5524, 5598, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44432 payload_rows_remote=38696 dup_rows=21104 dup_rows_remote=18276 dedup_ratio=0.322021 router_tokens_per_expert=[527, 515, 512, 518, 474, 506, 518, 502, 562, 516, 483, 544, 518, 514, 493, 515, 511, 476, 505, 502, 534, 499, 522, 549, 530, 530, 524, 513, 498, 498, 559, 564, 480, 526, 515, 522, 525, 527, 521, 484, 465, 484, 505, 506, 510, 507, 500, 517, 503, 507, 524, 482, 539, 542, 498, 470, 511, 514, 523, 508, 522, 488, 514, 498] payload_rows_per_dst=[5736, 5626, 5568, 5746, 5584, 5434, 5214, 5524] dup_rows_per_dst=[2828, 2664, 2628, 2872, 2616, 2554, 2310, 2632] combine_rows_per_src=[5736, 5626, 5568, 5746, 5584, 5434, 5214, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44530 payload_rows_remote=38824 dup_rows=21006 dup_rows_remote=18192 dedup_ratio=0.320526 router_tokens_per_expert=[494, 537, 510, 491, 465, 490, 487, 510, 531, 503, 550, 530, 527, 514, 475, 526, 507, 517, 563, 521, 559, 438, 528, 513, 523, 518, 486, 494, 494, 530, 517, 522, 515, 511, 507, 485, 517, 540, 499, 504, 535, 497, 502, 505, 486, 489, 479, 512, 515, 498, 560, 535, 526, 571, 527, 528, 498, 517, 490, 503, 519, 504, 521, 503] payload_rows_per_dst=[5502, 5596, 5546, 5604, 5494, 5510, 5706, 5572] dup_rows_per_dst=[2466, 2716, 2746, 2564, 2662, 2500, 2814, 2538] combine_rows_per_src=[5502, 5596, 5546, 5604, 5494, 5510, 5706, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=280025088 dispatch_meta_bytes=229960 combine_payload_bytes=280025088 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280255048 ingress_bytes=500226048 link_bytes=500226048 token_bytes=560280136 weight_movement_bytes=220200960 total_nvlink_bytes=780481096 weight_to_token_ratio=0.393019 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.699753705e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279007232 dispatch_meta_bytes=228960 combine_payload_bytes=279007232 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279236192 ingress_bytes=389107712 link_bytes=389107712 token_bytes=558243424 weight_movement_bytes=110100480 total_nvlink_bytes=668343904 weight_to_token_ratio=0.197227 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.767223084e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=280182784 dispatch_meta_bytes=229408 combine_payload_bytes=280182784 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280412192 ingress_bytes=390283264 link_bytes=390283264 token_bytes=560594976 weight_movement_bytes=110100480 total_nvlink_bytes=670695456 weight_to_token_ratio=0.196399 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.787667867e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=277129216 dispatch_meta_bytes=229336 combine_payload_bytes=277129216 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=277358552 ingress_bytes=497330176 link_bytes=497330176 token_bytes=554487768 weight_movement_bytes=220200960 total_nvlink_bytes=774688728 weight_to_token_ratio=0.397125 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.649389728e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278777856 dispatch_meta_bytes=228784 combine_payload_bytes=278777856 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279006640 ingress_bytes=609079296 link_bytes=609079296 token_bytes=557784496 weight_movement_bytes=330301440 total_nvlink_bytes=888085936 weight_to_token_ratio=0.592167 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059289072e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44442 payload_rows_remote=38968 dup_rows=21094 dup_rows_remote=18488 dedup_ratio=0.321869 router_tokens_per_expert=[525, 546, 510, 504, 487, 498, 494, 504, 500, 535, 537, 502, 526, 509, 523, 477, 525, 500, 504, 509, 502, 527, 495, 533, 523, 485, 483, 539, 522, 556, 488, 492, 513, 529, 490, 497, 524, 521, 457, 509, 527, 475, 513, 503, 488, 496, 556, 521, 534, 504, 530, 507, 490, 545, 538, 520, 488, 503, 558, 512, 531, 494, 513, 522] payload_rows_per_dst=[5526, 5530, 5550, 5604, 5474, 5526, 5648, 5584] dup_rows_per_dst=[2610, 2688, 2640, 2572, 2606, 2632, 2688, 2658] combine_rows_per_src=[5526, 5530, 5550, 5604, 5474, 5526, 5648, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=277372928 dispatch_meta_bytes=227888 combine_payload_bytes=277372928 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277600816 ingress_bytes=387473408 link_bytes=387473408 token_bytes=554973744 weight_movement_bytes=110100480 total_nvlink_bytes=665074224 weight_to_token_ratio=0.198389 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.738799849e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278290432 dispatch_meta_bytes=228064 combine_payload_bytes=278290432 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278518496 ingress_bytes=608591872 link_bytes=608591872 token_bytes=556808928 weight_movement_bytes=330301440 total_nvlink_bytes=887110368 weight_to_token_ratio=0.593204 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058441362e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279322624 dispatch_meta_bytes=229824 combine_payload_bytes=279322624 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279552448 ingress_bytes=389423104 link_bytes=389423104 token_bytes=558875072 weight_movement_bytes=110100480 total_nvlink_bytes=668975552 weight_to_token_ratio=0.197004 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.772708270e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44448 payload_rows_remote=38938 dup_rows=21088 dup_rows_remote=18398 dedup_ratio=0.321777 router_tokens_per_expert=[511, 578, 501, 517, 516, 491, 510, 527, 477, 513, 540, 504, 547, 475, 549, 524, 519, 527, 477, 526, 494, 512, 480, 508, 516, 488, 522, 549, 494, 521, 513, 497, 485, 490, 527, 505, 538, 476, 526, 501, 500, 525, 489, 537, 516, 509, 504, 493, 517, 472, 539, 509, 528, 538, 516, 536, 510, 510, 519, 512, 472, 491, 510, 545] payload_rows_per_dst=[5628, 5616, 5524, 5510, 5548, 5558, 5600, 5464] dup_rows_per_dst=[2674, 2642, 2562, 2690, 2548, 2588, 2710, 2674] combine_rows_per_src=[5628, 5616, 5524, 5510, 5548, 5558, 5600, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44558 payload_rows_remote=38968 dup_rows=20978 dup_rows_remote=18408 dedup_ratio=0.320099 router_tokens_per_expert=[511, 527, 530, 517, 493, 457, 517, 526, 475, 488, 521, 561, 523, 495, 515, 542, 521, 504, 484, 524, 506, 512, 524, 488, 525, 525, 572, 501, 520, 513, 542, 455, 511, 487, 473, 509, 510, 529, 502, 512, 505, 523, 523, 503, 519, 496, 530, 499, 504, 484, 530, 516, 517, 533, 524, 535, 505, 486, 500, 492, 511, 556, 515, 515] payload_rows_per_dst=[5496, 5572, 5530, 5564, 5534, 5592, 5680, 5590] dup_rows_per_dst=[2424, 2668, 2596, 2742, 2532, 2604, 2842, 2570] combine_rows_per_src=[5496, 5572, 5530, 5564, 5534, 5592, 5680, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44536 payload_rows_remote=38906 dup_rows=21000 dup_rows_remote=18344 dedup_ratio=0.320435 router_tokens_per_expert=[520, 494, 495, 514, 501, 505, 522, 537, 498, 495, 506, 528, 525, 501, 485, 512, 508, 528, 503, 507, 564, 520, 523, 498, 485, 504, 495, 533, 501, 493, 524, 527, 525, 527, 500, 517, 503, 519, 521, 514, 512, 527, 550, 545, 492, 506, 489, 522, 488, 517, 504, 528, 496, 490, 541, 488, 524, 520, 480, 541, 524, 520, 480, 507] payload_rows_per_dst=[5518, 5562, 5606, 5514, 5598, 5630, 5528, 5580] dup_rows_per_dst=[2658, 2538, 2696, 2610, 2654, 2656, 2576, 2612] combine_rows_per_src=[5518, 5562, 5606, 5514, 5598, 5630, 5528, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5579598640 prefetch_bytes=1101004800 reduce_bytes=550502400 weight_movement_bytes=1651507200 egress_bytes=2790945584 ingress_bytes=4440160256 link_bytes=4440160256 total_nvlink_bytes=7231105840 weight_to_token_ratio=0.295990 prefetch_share_of_ingress=0.247965 dedup_ratio=0.321536 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=5 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.722168966e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44602 payload_rows_remote=39042 dup_rows=20934 dup_rows_remote=18432 dedup_ratio=0.319427 router_tokens_per_expert=[511, 530, 507, 495, 497, 500, 522, 477, 500, 515, 496, 501, 506, 493, 511, 509, 533, 525, 506, 499, 553, 502, 480, 490, 509, 538, 548, 543, 517, 503, 534, 514, 536, 506, 487, 533, 548, 499, 542, 525, 496, 501, 508, 532, 489, 544, 538, 508, 529, 508, 511, 508, 504, 491, 496, 495, 496, 496, 504, 534, 513, 508, 518, 501] payload_rows_per_dst=[5562, 5560, 5586, 5692, 5600, 5568, 5562, 5472] dup_rows_per_dst=[2516, 2502, 2590, 2720, 2752, 2664, 2522, 2668] combine_rows_per_src=[5562, 5560, 5586, 5692, 5600, 5568, 5562, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44370 payload_rows_remote=38880 dup_rows=21166 dup_rows_remote=18590 dedup_ratio=0.322968 router_tokens_per_expert=[509, 552, 541, 534, 525, 545, 492, 517, 511, 499, 516, 525, 512, 480, 537, 515, 496, 492, 504, 527, 492, 495, 529, 498, 508, 520, 513, 499, 507, 520, 545, 514, 516, 475, 524, 523, 464, 515, 531, 512, 477, 505, 502, 513, 543, 537, 525, 519, 508, 500, 512, 473, 501, 471, 522, 509, 459, 493, 532, 539, 511, 513, 573, 502] payload_rows_per_dst=[5596, 5598, 5490, 5558, 5516, 5536, 5464, 5612] dup_rows_per_dst=[2834, 2592, 2576, 2694, 2604, 2706, 2528, 2632] combine_rows_per_src=[5596, 5598, 5490, 5558, 5516, 5536, 5464, 5612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44634 payload_rows_remote=38976 dup_rows=20902 dup_rows_remote=18220 dedup_ratio=0.318939 router_tokens_per_expert=[548, 511, 536, 497, 509, 521, 542, 490, 495, 546, 510, 530, 502, 545, 495, 516, 498, 494, 502, 500, 513, 489, 521, 498, 533, 524, 508, 506, 486, 521, 500, 529, 508, 497, 527, 555, 517, 477, 470, 500, 493, 506, 513, 521, 546, 501, 482, 538, 529, 487, 535, 507, 524, 515, 522, 475, 546, 516, 478, 528, 504, 521, 496, 519] payload_rows_per_dst=[5658, 5602, 5706, 5398, 5608, 5496, 5592, 5574] dup_rows_per_dst=[2682, 2676, 2796, 2420, 2658, 2554, 2596, 2520] combine_rows_per_src=[5658, 5602, 5706, 5398, 5608, 5496, 5592, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44478 payload_rows_remote=39010 dup_rows=21058 dup_rows_remote=18526 dedup_ratio=0.321320 router_tokens_per_expert=[512, 515, 526, 477, 521, 521, 492, 513, 550, 494, 528, 506, 553, 517, 474, 473, 532, 497, 537, 496, 493, 515, 493, 481, 491, 491, 548, 488, 540, 531, 521, 514, 511, 519, 477, 503, 531, 478, 575, 558, 522, 499, 514, 495, 546, 525, 538, 474, 496, 539, 487, 493, 494, 499, 518, 474, 539, 498, 525, 534, 533, 499, 517, 518] payload_rows_per_dst=[5594, 5534, 5482, 5608, 5612, 5564, 5468, 5616] dup_rows_per_dst=[2560, 2656, 2606, 2640, 2692, 2662, 2532, 2710] combine_rows_per_src=[5594, 5534, 5482, 5608, 5612, 5564, 5468, 5616] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44600 payload_rows_remote=38924 dup_rows=20936 dup_rows_remote=18228 dedup_ratio=0.319458 router_tokens_per_expert=[549, 512, 540, 501, 510, 527, 548, 493, 496, 545, 512, 532, 505, 548, 498, 519, 499, 495, 508, 501, 516, 492, 522, 497, 532, 527, 509, 509, 485, 522, 502, 530, 506, 496, 528, 552, 518, 477, 472, 500, 496, 503, 513, 518, 542, 500, 477, 535, 528, 486, 533, 506, 518, 513, 519, 466, 543, 514, 476, 525, 502, 518, 492, 515] payload_rows_per_dst=[5676, 5604, 5670, 5412, 5544, 5532, 5568, 5594] dup_rows_per_dst=[2708, 2682, 2746, 2464, 2554, 2636, 2570, 2576] combine_rows_per_src=[5676, 5604, 5670, 5412, 5544, 5532, 5568, 5594] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279781376 dispatch_meta_bytes=229768 combine_payload_bytes=279781376 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=280011144 ingress_bytes=610082816 link_bytes=610082816 token_bytes=559792520 weight_movement_bytes=330301440 total_nvlink_bytes=890093960 weight_to_token_ratio=0.590043 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061034358e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44470 payload_rows_remote=38834 dup_rows=21066 dup_rows_remote=18404 dedup_ratio=0.321442 router_tokens_per_expert=[514, 532, 531, 520, 495, 462, 518, 532, 478, 490, 521, 565, 523, 496, 519, 544, 520, 506, 485, 528, 507, 511, 523, 489, 524, 525, 574, 499, 520, 511, 540, 455, 512, 489, 472, 509, 510, 527, 499, 511, 503, 521, 523, 504, 519, 497, 526, 497, 505, 483, 528, 514, 516, 532, 522, 533, 501, 483, 496, 490, 511, 553, 513, 512] payload_rows_per_dst=[5340, 5482, 5536, 5516, 5612, 5588, 5760, 5636] dup_rows_per_dst=[2258, 2564, 2602, 2684, 2664, 2600, 3032, 2662] combine_rows_per_src=[5340, 5482, 5536, 5516, 5612, 5588, 5760, 5636] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278634496 dispatch_meta_bytes=229840 combine_payload_bytes=278634496 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278864336 ingress_bytes=388734976 link_bytes=388734976 token_bytes=557498832 weight_movement_bytes=110100480 total_nvlink_bytes=667599312 weight_to_token_ratio=0.197490 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.760740592e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278992896 dispatch_meta_bytes=229104 combine_payload_bytes=278992896 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279222000 ingress_bytes=389093376 link_bytes=389093376 token_bytes=558214896 weight_movement_bytes=110100480 total_nvlink_bytes=668315376 weight_to_token_ratio=0.197237 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.766973757e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279007232 dispatch_meta_bytes=228608 combine_payload_bytes=279007232 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279235840 ingress_bytes=609308672 link_bytes=609308672 token_bytes=558243072 weight_movement_bytes=330301440 total_nvlink_bytes=888544512 weight_to_token_ratio=0.591680 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059687995e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5582924288 prefetch_bytes=660602880 reduce_bytes=1981808640 weight_movement_bytes=2642411520 egress_bytes=2792608256 ingress_bytes=5432727552 link_bytes=5432727552 total_nvlink_bytes=8225335808 weight_to_token_ratio=0.473302 prefetch_share_of_ingress=0.121597 dedup_ratio=0.320950 prefetch_slots=12 unused_prefetch_slots=148 slot_utilization=0.075000 reduce_slots=18 remote_demand=6 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042647 gemm_inner_pad_upper_bound=12192 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.448406743e-05 prefetch_bytes_per_gemm_flop=1.148897059e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278362112 dispatch_meta_bytes=228952 combine_payload_bytes=278362112 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278591064 ingress_bytes=388462592 link_bytes=388462592 token_bytes=556953176 weight_movement_bytes=110100480 total_nvlink_bytes=667053656 weight_to_token_ratio=0.197684 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.756003386e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5580574416 prefetch_bytes=990904320 reduce_bytes=550502400 weight_movement_bytes=1541406720 egress_bytes=2791433936 ingress_bytes=4330547200 link_bytes=4330547200 total_nvlink_bytes=7121981136 weight_to_token_ratio=0.276209 prefetch_share_of_ingress=0.228817 dedup_ratio=0.321552 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=5 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042647 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.531533833e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5581491944 prefetch_bytes=880803840 reduce_bytes=1211105280 weight_movement_bytes=2091909120 egress_bytes=2791892712 ingress_bytes=4881508352 link_bytes=4881508352 total_nvlink_bytes=7673401064 weight_to_token_ratio=0.374794 prefetch_share_of_ingress=0.180437 dedup_ratio=0.321350 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=11 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.489745894e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5574577232 prefetch_bytes=880803840 reduce_bytes=1541406720 weight_movement_bytes=2422210560 egress_bytes=2788432976 ingress_bytes=5208354816 link_bytes=5208354816 total_nvlink_bytes=7996787792 weight_to_token_ratio=0.434510 prefetch_share_of_ingress=0.169114 dedup_ratio=0.321268 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=14 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.058185652e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5557796288 prefetch_bytes=990904320 reduce_bytes=110100480 weight_movement_bytes=1101004800 egress_bytes=2780038592 ingress_bytes=3878762496 link_bytes=3878762496 total_nvlink_bytes=6658801088 weight_to_token_ratio=0.198101 prefetch_share_of_ingress=0.255469 dedup_ratio=0.322156 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=1 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=6.745805927e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none - [2026-08-26 15:12:38] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 367.5 | throughput per GPU (TFLOP/s/GPU): 336.0 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.117351E+01 | loss scale: 1.0 | grad norm: 107.413 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44448 payload_rows_remote=38940 dup_rows=21088 dup_rows_remote=18422 dedup_ratio=0.321777 router_tokens_per_expert=[510, 510, 525, 496, 515, 507, 520, 504, 506, 504, 519, 496, 529, 519, 504, 508, 498, 547, 509, 483, 499, 505, 526, 529, 529, 532, 523, 512, 487, 501, 550, 508, 525, 495, 527, 513, 515, 480, 544, 503, 541, 558, 525, 463, 496, 522, 518, 464, 504, 500, 542, 526, 536, 530, 493, 479, 504, 553, 466, 515, 491, 468, 537, 525] payload_rows_per_dst=[5616, 5538, 5590, 5572, 5496, 5508, 5612, 5516] dup_rows_per_dst=[2558, 2632, 2602, 2712, 2708, 2666, 2608, 2602] combine_rows_per_src=[5616, 5538, 5590, 5572, 5496, 5508, 5612, 5516] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44490 payload_rows_remote=38824 dup_rows=21046 dup_rows_remote=18432 dedup_ratio=0.321136 router_tokens_per_expert=[516, 572, 524, 532, 526, 505, 517, 488, 530, 522, 495, 498, 497, 541, 476, 500, 493, 502, 552, 533, 503, 529, 499, 529, 506, 519, 505, 482, 517, 512, 521, 498, 500, 528, 513, 498, 542, 505, 491, 495, 530, 486, 523, 517, 519, 525, 519, 498, 555, 490, 510, 512, 519, 494, 544, 489, 515, 474, 495, 509, 517, 497, 518, 502] payload_rows_per_dst=[5658, 5534, 5666, 5550, 5566, 5542, 5516, 5458] dup_rows_per_dst=[2702, 2584, 2614, 2570, 2578, 2692, 2710, 2596] combine_rows_per_src=[5658, 5534, 5666, 5550, 5566, 5542, 5516, 5458] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44506 payload_rows_remote=38972 dup_rows=21030 dup_rows_remote=18394 dedup_ratio=0.320892 router_tokens_per_expert=[511, 529, 529, 501, 507, 497, 506, 554, 545, 522, 507, 476, 513, 507, 523, 541, 555, 510, 515, 533, 496, 507, 512, 511, 542, 516, 515, 467, 495, 507, 508, 505, 515, 505, 522, 452, 486, 547, 554, 504, 526, 486, 522, 496, 495, 497, 546, 508, 526, 545, 519, 509, 529, 491, 480, 504, 504, 491, 500, 485, 536, 522, 490, 514] payload_rows_per_dst=[5568, 5644, 5564, 5602, 5534, 5512, 5600, 5482] dup_rows_per_dst=[2700, 2624, 2714, 2508, 2636, 2640, 2606, 2602] combine_rows_per_src=[5568, 5644, 5564, 5602, 5534, 5512, 5600, 5482] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44452 payload_rows_remote=38822 dup_rows=21084 dup_rows_remote=18444 dedup_ratio=0.321716 router_tokens_per_expert=[522, 491, 485, 520, 546, 513, 471, 494, 507, 540, 513, 515, 497, 558, 557, 525, 495, 509, 534, 546, 496, 528, 497, 524, 483, 540, 471, 547, 511, 539, 507, 537, 518, 521, 529, 517, 504, 476, 527, 499, 511, 523, 527, 509, 505, 474, 526, 493, 490, 499, 480, 537, 510, 512, 462, 522, 541, 490, 528, 524, 533, 518, 476, 469] payload_rows_per_dst=[5520, 5652, 5534, 5630, 5538, 5518, 5528, 5532] dup_rows_per_dst=[2564, 2772, 2724, 2640, 2644, 2618, 2496, 2626] combine_rows_per_src=[5520, 5652, 5534, 5630, 5538, 5518, 5528, 5532] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44550 payload_rows_remote=38808 dup_rows=20986 dup_rows_remote=17952 dedup_ratio=0.320221 router_tokens_per_expert=[535, 463, 504, 504, 519, 500, 507, 535, 478, 497, 544, 549, 515, 519, 486, 518, 525, 508, 537, 505, 516, 515, 473, 515, 499, 529, 496, 522, 520, 509, 517, 548, 522, 544, 515, 520, 507, 509, 526, 500, 511, 509, 490, 509, 522, 507, 485, 503, 535, 493, 532, 522, 501, 504, 485, 504, 509, 510, 531, 508, 511, 533, 499, 505] payload_rows_per_dst=[5518, 5472, 5336, 5588, 5616, 5524, 5754, 5742] dup_rows_per_dst=[2468, 2310, 2306, 2698, 2670, 2556, 2944, 3034] combine_rows_per_src=[5518, 5472, 5336, 5588, 5616, 5524, 5754, 5742] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44594 payload_rows_remote=39152 dup_rows=20942 dup_rows_remote=18352 dedup_ratio=0.319550 router_tokens_per_expert=[491, 529, 467, 488, 528, 522, 525, 486, 507, 511, 553, 470, 538, 531, 528, 535, 500, 499, 490, 535, 542, 496, 509, 513, 504, 507, 502, 512, 517, 531, 540, 492, 542, 512, 531, 507, 512, 529, 502, 541, 555, 549, 498, 491, 530, 513, 551, 511, 473, 528, 471, 523, 517, 509, 493, 502, 471, 521, 480, 496, 479, 496, 526, 511] payload_rows_per_dst=[5516, 5582, 5578, 5616, 5636, 5720, 5442, 5504] dup_rows_per_dst=[2556, 2764, 2590, 2594, 2716, 2676, 2590, 2456] combine_rows_per_src=[5516, 5582, 5578, 5616, 5636, 5720, 5442, 5504] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44392 payload_rows_remote=38796 dup_rows=21144 dup_rows_remote=18500 dedup_ratio=0.322632 router_tokens_per_expert=[523, 528, 519, 495, 553, 526, 506, 526, 512, 512, 511, 505, 537, 523, 483, 537, 526, 515, 526, 499, 546, 522, 549, 472, 472, 528, 494, 502, 525, 513, 507, 544, 513, 530, 491, 436, 540, 523, 526, 549, 523, 522, 466, 523, 500, 497, 536, 495, 505, 510, 478, 508, 496, 512, 454, 527, 509, 496, 494, 511, 500, 537, 523, 502] payload_rows_per_dst=[5588, 5596, 5610, 5568, 5518, 5494, 5484, 5534] dup_rows_per_dst=[2764, 2644, 2700, 2602, 2698, 2630, 2496, 2610] combine_rows_per_src=[5588, 5596, 5610, 5568, 5518, 5494, 5484, 5534] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44506 payload_rows_remote=38962 dup_rows=21030 dup_rows_remote=18334 dedup_ratio=0.320892 router_tokens_per_expert=[549, 480, 537, 489, 527, 523, 487, 528, 510, 557, 495, 519, 516, 493, 491, 513, 512, 536, 511, 495, 493, 491, 531, 509, 491, 480, 502, 506, 516, 529, 472, 547, 513, 529, 466, 492, 498, 522, 557, 515, 501, 512, 508, 534, 524, 520, 535, 511, 517, 533, 509, 537, 497, 491, 496, 495, 539, 520, 507, 533, 541, 496, 487, 498] payload_rows_per_dst=[5544, 5500, 5676, 5536, 5428, 5636, 5576, 5610] dup_rows_per_dst=[2696, 2688, 2732, 2550, 2554, 2604, 2574, 2632] combine_rows_per_src=[5544, 5500, 5676, 5536, 5428, 5636, 5576, 5610] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279121920 dispatch_meta_bytes=229448 combine_payload_bytes=279121920 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279351368 ingress_bytes=499322880 link_bytes=499322880 token_bytes=558473288 weight_movement_bytes=220200960 total_nvlink_bytes=778674248 weight_to_token_ratio=0.394291 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.684046128e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278290432 dispatch_meta_bytes=229024 combine_payload_bytes=278290432 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278519456 ingress_bytes=498491392 link_bytes=498491392 token_bytes=556809888 weight_movement_bytes=220200960 total_nvlink_bytes=777010848 weight_to_token_ratio=0.395469 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.669585184e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279351296 dispatch_meta_bytes=229464 combine_payload_bytes=279351296 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279580760 ingress_bytes=499552256 link_bytes=499552256 token_bytes=558932056 weight_movement_bytes=220200960 total_nvlink_bytes=779133016 weight_to_token_ratio=0.393967 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.688035354e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280641536 dispatch_meta_bytes=230016 combine_payload_bytes=280641536 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280871552 ingress_bytes=390742016 link_bytes=390742016 token_bytes=561513088 weight_movement_bytes=110100480 total_nvlink_bytes=671613568 weight_to_token_ratio=0.196078 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.795646318e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278276096 dispatch_meta_bytes=229064 combine_payload_bytes=278276096 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278505160 ingress_bytes=388376576 link_bytes=388376576 token_bytes=556781256 weight_movement_bytes=110100480 total_nvlink_bytes=666881736 weight_to_token_ratio=0.197745 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.754507426e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278175744 dispatch_meta_bytes=227040 combine_payload_bytes=278175744 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278402784 ingress_bytes=388276224 link_bytes=388276224 token_bytes=556578528 weight_movement_bytes=110100480 total_nvlink_bytes=666679008 weight_to_token_ratio=0.197817 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.752762140e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278089728 dispatch_meta_bytes=229184 combine_payload_bytes=278089728 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278318912 ingress_bytes=498290688 link_bytes=498290688 token_bytes=556408640 weight_movement_bytes=220200960 total_nvlink_bytes=776609600 weight_to_token_ratio=0.395754 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.666094612e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279279616 dispatch_meta_bytes=229184 combine_payload_bytes=279279616 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279508800 ingress_bytes=609581056 link_bytes=609581056 token_bytes=558788416 weight_movement_bytes=330301440 total_nvlink_bytes=889089856 weight_to_token_ratio=0.591103 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060161715e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44482 payload_rows_remote=38804 dup_rows=21054 dup_rows_remote=18450 dedup_ratio=0.321259 router_tokens_per_expert=[520, 546, 509, 532, 550, 505, 504, 521, 479, 503, 550, 547, 498, 524, 505, 494, 522, 477, 474, 474, 503, 474, 520, 537, 512, 511, 521, 486, 531, 514, 482, 502, 505, 459, 448, 521, 499, 514, 521, 517, 514, 562, 516, 500, 521, 484, 547, 497, 538, 503, 499, 547, 476, 539, 506, 497, 545, 551, 496, 532, 553, 493, 511, 530] payload_rows_per_dst=[5618, 5524, 5484, 5508, 5446, 5678, 5550, 5674] dup_rows_per_dst=[2756, 2676, 2478, 2610, 2522, 2604, 2660, 2748] combine_rows_per_src=[5618, 5524, 5484, 5508, 5446, 5678, 5550, 5674] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44416 payload_rows_remote=38806 dup_rows=21120 dup_rows_remote=18402 dedup_ratio=0.322266 router_tokens_per_expert=[489, 501, 537, 500, 507, 520, 532, 513, 534, 501, 515, 520, 534, 559, 485, 537, 534, 516, 499, 541, 507, 551, 511, 505, 513, 502, 554, 519, 497, 532, 508, 517, 477, 518, 531, 499, 549, 530, 511, 505, 507, 488, 508, 497, 486, 476, 526, 497, 535, 504, 501, 509, 498, 495, 524, 496, 493, 517, 499, 511, 492, 526, 501, 472] payload_rows_per_dst=[5552, 5612, 5610, 5570, 5554, 5472, 5576, 5470] dup_rows_per_dst=[2646, 2758, 2718, 2714, 2686, 2498, 2548, 2552] combine_rows_per_src=[5552, 5612, 5610, 5570, 5554, 5472, 5576, 5470] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44430 payload_rows_remote=38940 dup_rows=21106 dup_rows_remote=18490 dedup_ratio=0.322052 router_tokens_per_expert=[536, 499, 528, 518, 544, 487, 508, 522, 509, 564, 473, 487, 544, 496, 467, 496, 529, 484, 496, 495, 531, 549, 517, 461, 522, 497, 548, 525, 476, 559, 505, 546, 501, 528, 517, 508, 507, 500, 515, 517, 510, 503, 511, 520, 513, 500, 521, 522, 512, 514, 529, 494, 493, 500, 493, 518, 491, 497, 517, 485, 509, 548, 510, 547] payload_rows_per_dst=[5568, 5540, 5560, 5600, 5498, 5586, 5490, 5588] dup_rows_per_dst=[2716, 2532, 2564, 2756, 2688, 2614, 2616, 2620] combine_rows_per_src=[5568, 5540, 5560, 5600, 5498, 5586, 5490, 5588] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44550 payload_rows_remote=38942 dup_rows=20986 dup_rows_remote=18402 dedup_ratio=0.320221 router_tokens_per_expert=[490, 558, 508, 481, 513, 494, 508, 489, 553, 530, 522, 533, 499, 483, 493, 515, 496, 500, 527, 469, 509, 538, 518, 508, 524, 528, 532, 533, 519, 508, 530, 524, 542, 519, 544, 481, 501, 504, 501, 504, 504, 545, 471, 510, 477, 492, 516, 515, 491, 530, 516, 536, 506, 489, 501, 491, 526, 546, 538, 513, 493, 495, 533, 506] payload_rows_per_dst=[5548, 5672, 5484, 5612, 5608, 5500, 5538, 5588] dup_rows_per_dst=[2534, 2584, 2646, 2784, 2584, 2560, 2582, 2712] combine_rows_per_src=[5548, 5672, 5484, 5612, 5608, 5500, 5538, 5588] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44338 payload_rows_remote=38740 dup_rows=21198 dup_rows_remote=18530 dedup_ratio=0.323456 router_tokens_per_expert=[505, 519, 533, 522, 493, 509, 496, 502, 510, 521, 496, 543, 521, 496, 521, 503, 502, 501, 515, 469, 521, 519, 546, 532, 531, 526, 516, 510, 494, 511, 547, 498, 507, 561, 528, 494, 531, 507, 502, 487, 525, 506, 502, 534, 551, 508, 534, 505, 530, 487, 495, 476, 471, 511, 519, 479, 504, 505, 467, 523, 549, 522, 511, 509] payload_rows_per_dst=[5534, 5550, 5536, 5598, 5588, 5584, 5486, 5462] dup_rows_per_dst=[2624, 2672, 2674, 2668, 2646, 2746, 2450, 2718] combine_rows_per_src=[5534, 5550, 5536, 5598, 5588, 5584, 5486, 5462] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44622 payload_rows_remote=39026 dup_rows=20914 dup_rows_remote=18254 dedup_ratio=0.319122 router_tokens_per_expert=[510, 516, 513, 506, 530, 544, 532, 504, 503, 523, 486, 487, 512, 499, 510, 489, 498, 491, 508, 510, 564, 510, 490, 488, 534, 521, 521, 505, 519, 528, 532, 461, 510, 521, 496, 523, 479, 517, 518, 515, 477, 517, 550, 519, 528, 485, 532, 532, 527, 475, 511, 518, 524, 517, 505, 500, 515, 535, 512, 529, 487, 507, 510, 533] payload_rows_per_dst=[5532, 5344, 5658, 5384, 5722, 5740, 5646, 5596] dup_rows_per_dst=[2536, 2388, 2702, 2288, 2676, 2826, 2838, 2660] combine_rows_per_src=[5532, 5344, 5658, 5384, 5722, 5740, 5646, 5596] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278147072 dispatch_meta_bytes=229016 combine_payload_bytes=278147072 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278376088 ingress_bytes=388247552 link_bytes=388247552 token_bytes=556523160 weight_movement_bytes=110100480 total_nvlink_bytes=666623640 weight_to_token_ratio=0.197836 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.752263487e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279121920 dispatch_meta_bytes=229720 combine_payload_bytes=279121920 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279351640 ingress_bytes=389222400 link_bytes=389222400 token_bytes=558473560 weight_movement_bytes=110100480 total_nvlink_bytes=668574040 weight_to_token_ratio=0.197145 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.769217697e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44616 payload_rows_remote=39018 dup_rows=20920 dup_rows_remote=18178 dedup_ratio=0.319214 router_tokens_per_expert=[500, 537, 530, 538, 515, 522, 476, 513, 498, 494, 526, 515, 528, 522, 532, 555, 485, 517, 517, 542, 503, 514, 524, 489, 541, 511, 498, 533, 526, 514, 517, 477, 488, 487, 488, 522, 530, 524, 528, 534, 502, 566, 467, 482, 485, 499, 478, 496, 542, 505, 491, 508, 522, 545, 516, 531, 503, 540, 502, 518, 487, 490, 504, 479] payload_rows_per_dst=[5630, 5598, 5636, 5626, 5582, 5398, 5624, 5522] dup_rows_per_dst=[2632, 2742, 2546, 2608, 2620, 2552, 2696, 2524] combine_rows_per_src=[5630, 5598, 5636, 5626, 5582, 5398, 5624, 5522] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278161408 dispatch_meta_bytes=228832 combine_payload_bytes=278161408 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278390240 ingress_bytes=388261888 link_bytes=388261888 token_bytes=556551648 weight_movement_bytes=110100480 total_nvlink_bytes=666652128 weight_to_token_ratio=0.197826 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.752512813e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44236 payload_rows_remote=38792 dup_rows=21300 dup_rows_remote=18634 dedup_ratio=0.325012 router_tokens_per_expert=[486, 516, 507, 531, 499, 510, 488, 515, 562, 521, 533, 532, 518, 512, 501, 492, 507, 511, 549, 540, 515, 508, 501, 489, 502, 497, 534, 528, 523, 479, 472, 543, 530, 502, 479, 506, 500, 506, 528, 507, 510, 483, 536, 539, 540, 524, 496, 461, 497, 528, 510, 504, 533, 553, 504, 489, 549, 464, 520, 524, 505, 505, 512, 503] payload_rows_per_dst=[5444, 5580, 5534, 5636, 5496, 5514, 5568, 5464] dup_rows_per_dst=[2666, 2764, 2706, 2574, 2620, 2664, 2668, 2638] combine_rows_per_src=[5444, 5580, 5534, 5636, 5496, 5514, 5568, 5464] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279136256 dispatch_meta_bytes=229376 combine_payload_bytes=279136256 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279365632 ingress_bytes=389236736 link_bytes=389236736 token_bytes=558501888 weight_movement_bytes=110100480 total_nvlink_bytes=668602368 weight_to_token_ratio=0.197135 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.769467023e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277688320 dispatch_meta_bytes=229080 combine_payload_bytes=277688320 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=277917400 ingress_bytes=607989760 link_bytes=607989760 token_bytes=555605720 weight_movement_bytes=330301440 total_nvlink_bytes=885907160 weight_to_token_ratio=0.594489 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057394190e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279738368 dispatch_meta_bytes=229120 combine_payload_bytes=279738368 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279967488 ingress_bytes=610039808 link_bytes=610039808 token_bytes=559705856 weight_movement_bytes=330301440 total_nvlink_bytes=890007296 weight_to_token_ratio=0.590134 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060959560e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279681024 dispatch_meta_bytes=228784 combine_payload_bytes=279681024 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279909808 ingress_bytes=499881984 link_bytes=499881984 token_bytes=559590832 weight_movement_bytes=220200960 total_nvlink_bytes=779791792 weight_to_token_ratio=0.393504 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.693769866e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278061056 dispatch_meta_bytes=229704 combine_payload_bytes=278061056 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278290760 ingress_bytes=498262016 link_bytes=498262016 token_bytes=556351816 weight_movement_bytes=220200960 total_nvlink_bytes=776552776 weight_to_token_ratio=0.395794 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.665595958e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44396 payload_rows_remote=38846 dup_rows=21140 dup_rows_remote=18616 dedup_ratio=0.322571 router_tokens_per_expert=[534, 556, 506, 517, 493, 501, 507, 526, 532, 542, 497, 551, 535, 507, 513, 498, 495, 510, 529, 527, 506, 547, 521, 498, 508, 479, 571, 492, 503, 541, 492, 488, 512, 493, 518, 532, 546, 482, 537, 558, 486, 504, 493, 549, 466, 503, 490, 515, 494, 543, 502, 503, 528, 478, 538, 451, 530, 489, 518, 467, 483, 534, 493, 511] payload_rows_per_dst=[5582, 5582, 5574, 5544, 5646, 5480, 5550, 5438] dup_rows_per_dst=[2698, 2768, 2692, 2604, 2710, 2532, 2524, 2612] combine_rows_per_src=[5582, 5582, 5574, 5544, 5646, 5480, 5550, 5438] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44446 payload_rows_remote=38836 dup_rows=21090 dup_rows_remote=18460 dedup_ratio=0.321808 router_tokens_per_expert=[538, 493, 517, 492, 536, 529, 493, 475, 505, 551, 506, 518, 519, 483, 517, 512, 483, 523, 512, 528, 492, 538, 516, 496, 512, 483, 533, 541, 552, 531, 538, 462, 521, 526, 494, 468, 532, 521, 533, 528, 530, 545, 507, 538, 508, 482, 487, 523, 496, 512, 506, 480, 555, 528, 515, 465, 517, 513, 500, 507, 515, 520, 487, 485] payload_rows_per_dst=[5510, 5622, 5530, 5600, 5548, 5610, 5556, 5470] dup_rows_per_dst=[2636, 2600, 2646, 2704, 2698, 2630, 2558, 2618] combine_rows_per_src=[5510, 5622, 5530, 5600, 5548, 5610, 5556, 5470] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44542 payload_rows_remote=38970 dup_rows=20994 dup_rows_remote=18358 dedup_ratio=0.320343 router_tokens_per_expert=[492, 498, 527, 439, 508, 508, 484, 517, 516, 508, 495, 489, 516, 507, 542, 534, 516, 522, 516, 525, 503, 501, 486, 535, 529, 541, 523, 528, 527, 487, 530, 449, 543, 507, 525, 508, 531, 480, 524, 504, 491, 493, 536, 516, 478, 520, 515, 503, 528, 539, 537, 511, 531, 496, 503, 548, 538, 484, 493, 504, 519, 520, 515, 530] payload_rows_per_dst=[5492, 5580, 5572, 5592, 5604, 5428, 5712, 5562] dup_rows_per_dst=[2454, 2634, 2636, 2636, 2640, 2676, 2674, 2644] combine_rows_per_src=[5492, 5580, 5572, 5592, 5604, 5428, 5712, 5562] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279107584 dispatch_meta_bytes=229344 combine_payload_bytes=279107584 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279336928 ingress_bytes=499308544 link_bytes=499308544 token_bytes=558444512 weight_movement_bytes=220200960 total_nvlink_bytes=778645472 weight_to_token_ratio=0.394311 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.683796802e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279322624 dispatch_meta_bytes=229504 combine_payload_bytes=279322624 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279552128 ingress_bytes=609624064 link_bytes=609624064 token_bytes=558874752 weight_movement_bytes=330301440 total_nvlink_bytes=889176192 weight_to_token_ratio=0.591012 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060236513e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278878208 dispatch_meta_bytes=229000 combine_payload_bytes=278878208 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279107208 ingress_bytes=499079168 link_bytes=499079168 token_bytes=557985416 weight_movement_bytes=220200960 total_nvlink_bytes=778186376 weight_to_token_ratio=0.394636 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.679807576e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279853056 dispatch_meta_bytes=229896 combine_payload_bytes=279853056 prefetch_bytes=0 reduce_bytes=0 egress_bytes=280082952 ingress_bytes=279853056 link_bytes=280082952 token_bytes=559936008 weight_movement_bytes=0 total_nvlink_bytes=559936008 weight_to_token_ratio=0.000000 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=4.871103193e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278691840 dispatch_meta_bytes=229880 combine_payload_bytes=278691840 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278921720 ingress_bytes=388792320 link_bytes=388792320 token_bytes=557613560 weight_movement_bytes=110100480 total_nvlink_bytes=667714040 weight_to_token_ratio=0.197449 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.761737898e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279379968 dispatch_meta_bytes=228784 combine_payload_bytes=279379968 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279608752 ingress_bytes=499580928 link_bytes=499580928 token_bytes=558988720 weight_movement_bytes=220200960 total_nvlink_bytes=779189680 weight_to_token_ratio=0.393927 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.688534007e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279623680 dispatch_meta_bytes=230144 combine_payload_bytes=279623680 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279853824 ingress_bytes=389724160 link_bytes=389724160 token_bytes=559477504 weight_movement_bytes=110100480 total_nvlink_bytes=669577984 weight_to_token_ratio=0.196792 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.777944128e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5581835752 prefetch_bytes=990904320 reduce_bytes=990904320 weight_movement_bytes=1981808640 egress_bytes=2792064488 ingress_bytes=4771579904 link_bytes=4771579904 total_nvlink_bytes=7563644392 weight_to_token_ratio=0.355046 prefetch_share_of_ingress=0.207668 dedup_ratio=0.321082 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=9 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043015 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.298562243e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5570419104 prefetch_bytes=770703360 reduce_bytes=1101004800 weight_movement_bytes=1871708160 egress_bytes=2786353568 ingress_bytes=4655773696 link_bytes=4655773696 total_nvlink_bytes=7442127264 weight_to_token_ratio=0.336008 prefetch_share_of_ingress=0.165537 dedup_ratio=0.321783 prefetch_slots=14 unused_prefetch_slots=146 slot_utilization=0.087500 reduce_slots=10 remote_demand=7 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=12319 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.097156201e-05 prefetch_bytes_per_gemm_flop=1.340379902e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5579340840 prefetch_bytes=1101004800 reduce_bytes=770703360 weight_movement_bytes=1871708160 egress_bytes=2790816808 ingress_bytes=4660232192 link_bytes=4660232192 total_nvlink_bytes=7451049000 weight_to_token_ratio=0.335471 prefetch_share_of_ingress=0.236255 dedup_ratio=0.321558 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=7 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.104910258e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5585162152 prefetch_bytes=880803840 reduce_bytes=990904320 weight_movement_bytes=1871708160 egress_bytes=2793727912 ingress_bytes=4663142400 link_bytes=4663372296 total_nvlink_bytes=7456870312 weight_to_token_ratio=0.335122 prefetch_share_of_ingress=0.188886 dedup_ratio=0.320901 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=9 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.110371416e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5581521344 prefetch_bytes=1101004800 reduce_bytes=660602880 weight_movement_bytes=1761607680 egress_bytes=2791907776 ingress_bytes=4551221248 link_bytes=4551221248 total_nvlink_bytes=7343129024 weight_to_token_ratio=0.315614 prefetch_share_of_ingress=0.241914 dedup_ratio=0.321448 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=6 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.915322298e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5567864416 prefetch_bytes=770703360 reduce_bytes=1541406720 weight_movement_bytes=2312110080 egress_bytes=2785074784 ingress_bytes=5094899712 link_bytes=5094899712 total_nvlink_bytes=7879974496 weight_to_token_ratio=0.415260 prefetch_share_of_ingress=0.151270 dedup_ratio=0.321497 prefetch_slots=14 unused_prefetch_slots=146 slot_utilization=0.087500 reduce_slots=14 remote_demand=7 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040074 gemm_inner_pad_upper_bound=12319 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.860868566e-05 prefetch_bytes_per_gemm_flop=1.340379902e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5580144008 prefetch_bytes=880803840 reduce_bytes=990904320 weight_movement_bytes=1871708160 egress_bytes=2791218568 ingress_bytes=4660633600 link_bytes=4660633600 total_nvlink_bytes=7451852168 weight_to_token_ratio=0.335423 prefetch_share_of_ingress=0.188988 dedup_ratio=0.321182 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=9 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.105608373e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none + [2026-08-26 16:00:26] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 366.9 | throughput per GPU (TFLOP/s/GPU): 336.5 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.115327E+01 | loss scale: 1.0 | grad norm: 107.089 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44384 payload_rows_remote=38810 dup_rows=21152 dup_rows_remote=18516 dedup_ratio=0.322754 router_tokens_per_expert=[523, 526, 517, 493, 551, 525, 505, 523, 509, 508, 508, 505, 536, 521, 482, 536, 525, 513, 525, 499, 543, 520, 548, 472, 471, 527, 496, 501, 524, 512, 506, 546, 513, 530, 492, 438, 538, 523, 526, 545, 523, 523, 466, 525, 499, 499, 539, 493, 504, 512, 478, 510, 499, 511, 457, 530, 510, 499, 498, 515, 504, 541, 527, 505] payload_rows_per_dst=[5576, 5574, 5606, 5570, 5514, 5498, 5484, 5562] dup_rows_per_dst=[2750, 2636, 2684, 2596, 2696, 2636, 2518, 2636] combine_rows_per_src=[5576, 5574, 5606, 5570, 5514, 5498, 5484, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44500 payload_rows_remote=38964 dup_rows=21036 dup_rows_remote=18400 dedup_ratio=0.320984 router_tokens_per_expert=[510, 527, 526, 501, 504, 493, 505, 550, 542, 519, 505, 475, 512, 506, 520, 539, 555, 509, 514, 531, 496, 508, 512, 510, 540, 514, 513, 466, 491, 505, 509, 503, 515, 505, 520, 454, 486, 547, 553, 506, 524, 489, 523, 499, 498, 498, 548, 510, 527, 547, 520, 510, 529, 493, 483, 503, 506, 498, 504, 488, 539, 525, 492, 519] payload_rows_per_dst=[5554, 5634, 5558, 5588, 5536, 5524, 5604, 5502] dup_rows_per_dst=[2678, 2602, 2712, 2494, 2636, 2654, 2620, 2640] combine_rows_per_src=[5554, 5634, 5558, 5588, 5536, 5524, 5604, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44496 payload_rows_remote=38836 dup_rows=21040 dup_rows_remote=18434 dedup_ratio=0.321045 router_tokens_per_expert=[515, 570, 520, 531, 522, 503, 514, 488, 526, 520, 493, 496, 492, 542, 474, 500, 493, 498, 550, 530, 502, 530, 500, 530, 507, 517, 503, 483, 515, 512, 520, 498, 500, 528, 514, 499, 541, 508, 489, 494, 531, 487, 525, 518, 520, 529, 521, 499, 560, 491, 509, 513, 521, 496, 547, 490, 516, 478, 496, 511, 519, 500, 522, 502] payload_rows_per_dst=[5646, 5516, 5660, 5552, 5558, 5560, 5534, 5470] dup_rows_per_dst=[2680, 2570, 2606, 2558, 2588, 2700, 2720, 2618] combine_rows_per_src=[5646, 5516, 5660, 5552, 5558, 5560, 5534, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44580 payload_rows_remote=38946 dup_rows=20956 dup_rows_remote=18134 dedup_ratio=0.319763 router_tokens_per_expert=[531, 461, 498, 498, 518, 496, 507, 534, 477, 494, 540, 548, 512, 516, 485, 516, 524, 510, 537, 504, 513, 516, 474, 518, 495, 530, 496, 524, 518, 509, 520, 547, 520, 544, 514, 519, 506, 509, 526, 499, 511, 509, 489, 509, 524, 508, 486, 507, 538, 495, 537, 527, 501, 505, 484, 507, 512, 510, 535, 510, 514, 537, 502, 508] payload_rows_per_dst=[5564, 5520, 5506, 5618, 5612, 5442, 5684, 5634] dup_rows_per_dst=[2522, 2458, 2486, 2756, 2662, 2440, 2810, 2822] combine_rows_per_src=[5564, 5520, 5506, 5618, 5612, 5442, 5684, 5634] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44422 payload_rows_remote=38802 dup_rows=21114 dup_rows_remote=18474 dedup_ratio=0.322174 router_tokens_per_expert=[518, 490, 482, 520, 542, 511, 469, 491, 505, 538, 510, 512, 499, 555, 557, 527, 492, 510, 534, 546, 495, 524, 497, 525, 482, 538, 471, 549, 510, 535, 507, 538, 519, 518, 529, 518, 504, 478, 529, 498, 510, 522, 525, 507, 506, 476, 528, 496, 491, 500, 481, 539, 512, 514, 463, 524, 546, 494, 530, 527, 538, 521, 477, 469] payload_rows_per_dst=[5494, 5640, 5528, 5620, 5534, 5516, 5536, 5554] dup_rows_per_dst=[2552, 2766, 2718, 2640, 2652, 2624, 2512, 2650] combine_rows_per_src=[5494, 5640, 5528, 5620, 5534, 5516, 5536, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44454 payload_rows_remote=38934 dup_rows=21082 dup_rows_remote=18408 dedup_ratio=0.321686 router_tokens_per_expert=[509, 503, 521, 494, 512, 507, 518, 503, 502, 499, 517, 494, 528, 519, 505, 505, 497, 544, 509, 483, 499, 504, 524, 530, 529, 534, 521, 509, 487, 499, 548, 510, 526, 493, 528, 511, 516, 479, 543, 504, 545, 559, 526, 463, 497, 523, 519, 465, 508, 500, 547, 531, 537, 532, 494, 479, 508, 554, 469, 516, 491, 473, 544, 525] payload_rows_per_dst=[5598, 5528, 5586, 5562, 5496, 5520, 5622, 5542] dup_rows_per_dst=[2536, 2610, 2594, 2712, 2704, 2674, 2634, 2618] combine_rows_per_src=[5598, 5528, 5586, 5562, 5496, 5520, 5622, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44600 payload_rows_remote=39142 dup_rows=20936 dup_rows_remote=18318 dedup_ratio=0.319458 router_tokens_per_expert=[489, 528, 464, 486, 523, 521, 521, 484, 506, 508, 548, 469, 537, 526, 525, 535, 497, 500, 489, 534, 541, 492, 507, 511, 505, 506, 503, 511, 515, 531, 538, 495, 544, 510, 530, 503, 511, 529, 504, 540, 553, 549, 497, 491, 532, 515, 557, 512, 476, 528, 473, 527, 522, 512, 496, 504, 472, 523, 482, 498, 481, 502, 533, 517] payload_rows_per_dst=[5500, 5570, 5572, 5616, 5634, 5724, 5458, 5526] dup_rows_per_dst=[2532, 2738, 2570, 2592, 2708, 2688, 2618, 2490] combine_rows_per_src=[5500, 5570, 5572, 5616, 5634, 5724, 5458, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44510 payload_rows_remote=38918 dup_rows=21026 dup_rows_remote=18264 dedup_ratio=0.320831 router_tokens_per_expert=[544, 477, 537, 485, 526, 521, 483, 524, 508, 554, 494, 517, 514, 491, 488, 513, 515, 530, 511, 491, 493, 490, 532, 509, 488, 479, 502, 505, 513, 526, 471, 547, 513, 529, 468, 493, 497, 523, 557, 515, 505, 514, 510, 536, 526, 524, 538, 515, 519, 532, 512, 537, 499, 495, 500, 497, 542, 521, 509, 536, 543, 498, 489, 498] payload_rows_per_dst=[5592, 5494, 5584, 5524, 5428, 5688, 5588, 5612] dup_rows_per_dst=[2762, 2680, 2566, 2538, 2578, 2648, 2594, 2660] combine_rows_per_src=[5592, 5494, 5584, 5524, 5428, 5688, 5588, 5612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278190080 dispatch_meta_bytes=229304 combine_payload_bytes=278190080 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278419384 ingress_bytes=608491520 link_bytes=608491520 token_bytes=556609464 weight_movement_bytes=330301440 total_nvlink_bytes=886910904 weight_to_token_ratio=0.593417 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058266833e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279293952 dispatch_meta_bytes=229456 combine_payload_bytes=279293952 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279523408 ingress_bytes=609595392 link_bytes=609595392 token_bytes=558817360 weight_movement_bytes=330301440 total_nvlink_bytes=889118800 weight_to_token_ratio=0.591072 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060186648e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279164928 dispatch_meta_bytes=228320 combine_payload_bytes=279164928 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279393248 ingress_bytes=389265408 link_bytes=389265408 token_bytes=558558176 weight_movement_bytes=110100480 total_nvlink_bytes=668658656 weight_to_token_ratio=0.197116 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.769965677e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278376448 dispatch_meta_bytes=229080 combine_payload_bytes=278376448 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278605528 ingress_bytes=498577408 link_bytes=498577408 token_bytes=556981976 weight_movement_bytes=220200960 total_nvlink_bytes=777182936 weight_to_token_ratio=0.395347 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.671081144e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278132736 dispatch_meta_bytes=229104 combine_payload_bytes=278132736 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278361840 ingress_bytes=388233216 link_bytes=388233216 token_bytes=556494576 weight_movement_bytes=110100480 total_nvlink_bytes=666595056 weight_to_token_ratio=0.197846 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.752014160e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279078912 dispatch_meta_bytes=229368 combine_payload_bytes=279078912 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279308280 ingress_bytes=499279872 link_bytes=499279872 token_bytes=558387192 weight_movement_bytes=220200960 total_nvlink_bytes=778588152 weight_to_token_ratio=0.394352 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.683298148e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280569856 dispatch_meta_bytes=229840 combine_payload_bytes=280569856 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280799696 ingress_bytes=390670336 link_bytes=390670336 token_bytes=561369552 weight_movement_bytes=110100480 total_nvlink_bytes=671470032 weight_to_token_ratio=0.196128 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.794399685e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278964224 dispatch_meta_bytes=228728 combine_payload_bytes=278964224 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279192952 ingress_bytes=389064704 link_bytes=389064704 token_bytes=558157176 weight_movement_bytes=110100480 total_nvlink_bytes=668257656 weight_to_token_ratio=0.197257 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.766475104e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44526 payload_rows_remote=38912 dup_rows=21010 dup_rows_remote=18420 dedup_ratio=0.320587 router_tokens_per_expert=[488, 553, 507, 476, 510, 488, 504, 487, 553, 528, 520, 530, 495, 480, 492, 515, 493, 499, 526, 469, 507, 537, 515, 507, 522, 528, 535, 531, 517, 507, 529, 524, 545, 519, 542, 482, 502, 504, 503, 505, 509, 546, 472, 516, 477, 492, 520, 515, 491, 533, 520, 540, 509, 491, 501, 491, 527, 550, 539, 517, 495, 495, 535, 513] payload_rows_per_dst=[5522, 5652, 5466, 5602, 5614, 5522, 5552, 5596] dup_rows_per_dst=[2504, 2574, 2640, 2784, 2590, 2572, 2600, 2746] combine_rows_per_src=[5522, 5652, 5466, 5602, 5614, 5522, 5552, 5596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44730 payload_rows_remote=39114 dup_rows=20806 dup_rows_remote=18124 dedup_ratio=0.317474 router_tokens_per_expert=[509, 512, 508, 502, 530, 542, 527, 504, 503, 523, 484, 488, 512, 497, 512, 488, 495, 490, 508, 509, 564, 508, 484, 487, 534, 523, 520, 505, 517, 526, 532, 462, 511, 521, 497, 523, 479, 520, 520, 515, 477, 517, 549, 520, 528, 488, 533, 529, 527, 477, 512, 519, 527, 519, 508, 502, 516, 537, 517, 530, 489, 510, 511, 535] payload_rows_per_dst=[5614, 5456, 5552, 5654, 5662, 5630, 5546, 5616] dup_rows_per_dst=[2654, 2502, 2538, 2584, 2510, 2652, 2684, 2682] combine_rows_per_src=[5614, 5456, 5552, 5654, 5662, 5630, 5546, 5616] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44632 payload_rows_remote=39046 dup_rows=20904 dup_rows_remote=18186 dedup_ratio=0.318970 router_tokens_per_expert=[497, 534, 526, 536, 513, 521, 474, 510, 494, 493, 525, 510, 525, 519, 531, 555, 485, 514, 515, 540, 502, 515, 524, 489, 540, 510, 495, 534, 525, 514, 516, 476, 491, 487, 487, 521, 531, 522, 532, 534, 503, 567, 467, 483, 485, 499, 479, 494, 543, 505, 493, 510, 523, 545, 516, 538, 506, 546, 506, 519, 489, 498, 512, 480] payload_rows_per_dst=[5618, 5586, 5632, 5622, 5588, 5394, 5638, 5554] dup_rows_per_dst=[2604, 2718, 2536, 2598, 2622, 2560, 2708, 2558] combine_rows_per_src=[5618, 5586, 5632, 5622, 5588, 5394, 5638, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44420 payload_rows_remote=38824 dup_rows=21116 dup_rows_remote=18414 dedup_ratio=0.322205 router_tokens_per_expert=[486, 499, 534, 496, 505, 516, 529, 512, 531, 497, 511, 519, 535, 557, 484, 533, 530, 514, 499, 540, 504, 550, 509, 503, 514, 502, 553, 518, 496, 530, 509, 516, 475, 519, 531, 500, 548, 531, 511, 505, 507, 489, 509, 499, 489, 477, 530, 497, 536, 508, 503, 509, 498, 496, 530, 500, 496, 522, 504, 512, 493, 532, 508, 473] payload_rows_per_dst=[5532, 5598, 5596, 5558, 5558, 5482, 5598, 5498] dup_rows_per_dst=[2622, 2736, 2702, 2718, 2682, 2512, 2562, 2582] combine_rows_per_src=[5532, 5598, 5596, 5558, 5558, 5482, 5598, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44472 payload_rows_remote=38790 dup_rows=21064 dup_rows_remote=18460 dedup_ratio=0.321411 router_tokens_per_expert=[517, 541, 507, 528, 547, 505, 504, 520, 478, 500, 550, 544, 498, 523, 501, 491, 520, 477, 471, 474, 496, 474, 518, 537, 513, 512, 524, 485, 530, 513, 480, 503, 510, 459, 450, 522, 499, 514, 523, 516, 514, 558, 518, 501, 522, 483, 549, 498, 542, 504, 504, 546, 480, 542, 506, 499, 548, 553, 498, 537, 553, 495, 511, 533] payload_rows_per_dst=[5598, 5502, 5478, 5506, 5454, 5682, 5562, 5690] dup_rows_per_dst=[2740, 2668, 2456, 2614, 2532, 2604, 2684, 2766] combine_rows_per_src=[5598, 5502, 5478, 5506, 5454, 5682, 5562, 5690] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44334 payload_rows_remote=38738 dup_rows=21202 dup_rows_remote=18542 dedup_ratio=0.323517 router_tokens_per_expert=[502, 516, 530, 518, 488, 505, 494, 502, 508, 519, 492, 544, 519, 495, 518, 503, 504, 499, 514, 470, 518, 521, 547, 530, 531, 525, 514, 509, 492, 512, 548, 497, 508, 562, 529, 494, 531, 510, 504, 485, 525, 507, 502, 533, 551, 508, 534, 503, 532, 489, 499, 477, 475, 517, 520, 479, 505, 506, 470, 527, 552, 524, 515, 511] payload_rows_per_dst=[5516, 5540, 5536, 5596, 5594, 5578, 5502, 5472] dup_rows_per_dst=[2594, 2656, 2670, 2660, 2652, 2748, 2474, 2748] combine_rows_per_src=[5516, 5540, 5536, 5596, 5594, 5578, 5502, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44420 payload_rows_remote=38916 dup_rows=21116 dup_rows_remote=18480 dedup_ratio=0.322205 router_tokens_per_expert=[533, 497, 527, 515, 539, 484, 506, 519, 505, 562, 471, 486, 543, 490, 465, 497, 529, 485, 494, 497, 530, 546, 518, 460, 521, 498, 548, 527, 476, 560, 505, 546, 501, 526, 517, 512, 505, 500, 515, 514, 510, 504, 512, 520, 516, 503, 523, 522, 512, 513, 534, 494, 498, 503, 497, 519, 492, 502, 519, 486, 511, 548, 513, 548] payload_rows_per_dst=[5548, 5524, 5558, 5602, 5496, 5592, 5504, 5596] dup_rows_per_dst=[2692, 2514, 2560, 2760, 2684, 2628, 2636, 2642] combine_rows_per_src=[5548, 5524, 5558, 5602, 5496, 5592, 5504, 5596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278921216 dispatch_meta_bytes=229328 combine_payload_bytes=278921216 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279150544 ingress_bytes=389021696 link_bytes=389021696 token_bytes=558071760 weight_movement_bytes=110100480 total_nvlink_bytes=668172240 weight_to_token_ratio=0.197287 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.765727124e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=280369152 dispatch_meta_bytes=228952 combine_payload_bytes=280369152 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=280598104 ingress_bytes=720771072 link_bytes=720771072 token_bytes=560967256 weight_movement_bytes=440401920 total_nvlink_bytes=1001369176 weight_to_token_ratio=0.785076 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.253539441e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279881728 dispatch_meta_bytes=228928 combine_payload_bytes=279881728 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=280110656 ingress_bytes=500082688 link_bytes=500082688 token_bytes=559992384 weight_movement_bytes=220200960 total_nvlink_bytes=780193344 weight_to_token_ratio=0.393221 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.697260439e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278046720 dispatch_meta_bytes=229000 combine_payload_bytes=278046720 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278275720 ingress_bytes=388147200 link_bytes=388147200 token_bytes=556322440 weight_movement_bytes=110100480 total_nvlink_bytes=666422920 weight_to_token_ratio=0.197908 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.750518200e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278290432 dispatch_meta_bytes=228952 combine_payload_bytes=278290432 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278519384 ingress_bytes=388390912 link_bytes=388390912 token_bytes=556809816 weight_movement_bytes=110100480 total_nvlink_bytes=666910296 weight_to_token_ratio=0.197734 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.754756753e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44154 payload_rows_remote=38686 dup_rows=21382 dup_rows_remote=18672 dedup_ratio=0.326263 router_tokens_per_expert=[484, 514, 505, 527, 498, 509, 486, 515, 560, 518, 532, 531, 515, 509, 499, 491, 506, 508, 544, 539, 512, 508, 502, 490, 500, 493, 530, 530, 523, 478, 470, 544, 530, 499, 475, 510, 501, 508, 528, 509, 509, 483, 535, 542, 541, 524, 498, 460, 501, 532, 511, 506, 536, 555, 510, 492, 553, 466, 520, 527, 509, 506, 514, 508] payload_rows_per_dst=[5468, 5566, 5652, 5414, 5556, 5594, 5588, 5316] dup_rows_per_dst=[2710, 2744, 2964, 2264, 2742, 2786, 2698, 2474] combine_rows_per_src=[5468, 5566, 5652, 5414, 5556, 5594, 5588, 5316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277673984 dispatch_meta_bytes=229120 combine_payload_bytes=277673984 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=277903104 ingress_bytes=607975424 link_bytes=607975424 token_bytes=555577088 weight_movement_bytes=330301440 total_nvlink_bytes=885878528 weight_to_token_ratio=0.594520 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057369257e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278949888 dispatch_meta_bytes=229584 combine_payload_bytes=278949888 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279179472 ingress_bytes=389050368 link_bytes=389050368 token_bytes=558129360 weight_movement_bytes=110100480 total_nvlink_bytes=668229840 weight_to_token_ratio=0.197267 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.766225777e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277301248 dispatch_meta_bytes=229432 combine_payload_bytes=277301248 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277530680 ingress_bytes=387401728 link_bytes=387401728 token_bytes=554831928 weight_movement_bytes=110100480 total_nvlink_bytes=664932408 weight_to_token_ratio=0.198439 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.737553216e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44424 payload_rows_remote=38846 dup_rows=21112 dup_rows_remote=18556 dedup_ratio=0.322144 router_tokens_per_expert=[495, 521, 527, 504, 518, 483, 498, 537, 479, 539, 569, 520, 521, 547, 511, 496, 491, 485, 534, 534, 521, 523, 508, 535, 533, 511, 501, 495, 491, 509, 529, 510, 473, 511, 531, 498, 510, 495, 540, 509, 509, 486, 514, 481, 532, 512, 481, 531, 505, 522, 522, 522, 505, 491, 526, 507, 512, 529, 517, 480, 506, 521, 489, 526] payload_rows_per_dst=[5558, 5606, 5636, 5502, 5578, 5500, 5518, 5526] dup_rows_per_dst=[2608, 2758, 2626, 2656, 2556, 2592, 2682, 2634] combine_rows_per_src=[5558, 5606, 5636, 5502, 5578, 5500, 5518, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44436 payload_rows_remote=38742 dup_rows=21100 dup_rows_remote=18234 dedup_ratio=0.321960 router_tokens_per_expert=[472, 505, 518, 549, 542, 512, 502, 509, 495, 552, 540, 526, 510, 510, 535, 521, 477, 480, 506, 504, 515, 509, 557, 523, 493, 507, 486, 521, 513, 502, 527, 508, 513, 458, 509, 527, 494, 558, 504, 510, 509, 531, 531, 523, 472, 522, 532, 519, 486, 512, 534, 525, 500, 510, 499, 507, 535, 503, 490, 528, 521, 495, 490, 495] payload_rows_per_dst=[5628, 5420, 5512, 5542, 5466, 5644, 5530, 5694] dup_rows_per_dst=[2590, 2512, 2630, 2512, 2548, 2820, 2622, 2866] combine_rows_per_src=[5628, 5420, 5512, 5542, 5466, 5644, 5530, 5694] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44442 payload_rows_remote=38830 dup_rows=21094 dup_rows_remote=18446 dedup_ratio=0.321869 router_tokens_per_expert=[535, 490, 511, 490, 535, 523, 489, 471, 505, 552, 505, 517, 517, 480, 517, 511, 480, 523, 507, 525, 493, 534, 516, 493, 513, 483, 533, 539, 551, 533, 538, 462, 520, 526, 495, 467, 536, 519, 534, 531, 532, 546, 509, 540, 506, 483, 488, 526, 492, 516, 508, 483, 560, 528, 520, 465, 520, 514, 504, 508, 519, 525, 489, 488] payload_rows_per_dst=[5482, 5610, 5522, 5602, 5554, 5612, 5570, 5490] dup_rows_per_dst=[2606, 2598, 2620, 2702, 2702, 2648, 2574, 2644] combine_rows_per_src=[5482, 5610, 5522, 5602, 5554, 5612, 5570, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44512 payload_rows_remote=38988 dup_rows=21024 dup_rows_remote=18404 dedup_ratio=0.320801 router_tokens_per_expert=[526, 495, 539, 541, 510, 513, 537, 478, 475, 530, 465, 510, 511, 542, 525, 514, 500, 495, 503, 515, 493, 519, 504, 517, 540, 524, 466, 511, 524, 505, 533, 506, 536, 526, 498, 519, 510, 519, 542, 509, 509, 497, 476, 520, 512, 546, 495, 480, 475, 484, 552, 500, 489, 511, 513, 517, 489, 531, 487, 544, 499, 575, 523, 519] payload_rows_per_dst=[5546, 5524, 5532, 5574, 5702, 5530, 5492, 5612] dup_rows_per_dst=[2732, 2620, 2560, 2644, 2616, 2540, 2590, 2722] combine_rows_per_src=[5546, 5524, 5532, 5574, 5702, 5530, 5492, 5612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44536 payload_rows_remote=38976 dup_rows=21000 dup_rows_remote=18368 dedup_ratio=0.320435 router_tokens_per_expert=[492, 495, 524, 435, 506, 507, 482, 517, 515, 505, 494, 486, 514, 506, 539, 533, 513, 522, 515, 523, 504, 501, 485, 533, 526, 541, 523, 529, 529, 487, 530, 450, 541, 505, 525, 509, 532, 480, 524, 502, 493, 493, 539, 521, 481, 520, 516, 502, 528, 539, 537, 512, 535, 497, 502, 550, 537, 486, 497, 508, 523, 523, 517, 533] payload_rows_per_dst=[5478, 5568, 5560, 5592, 5602, 5438, 5726, 5572] dup_rows_per_dst=[2438, 2616, 2632, 2638, 2634, 2692, 2674, 2676] combine_rows_per_src=[5478, 5568, 5560, 5592, 5602, 5438, 5726, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44420 payload_rows_remote=38852 dup_rows=21116 dup_rows_remote=18560 dedup_ratio=0.322205 router_tokens_per_expert=[497, 524, 530, 507, 521, 485, 501, 544, 480, 544, 571, 520, 520, 547, 512, 499, 489, 487, 537, 535, 523, 523, 509, 535, 530, 513, 503, 494, 486, 509, 530, 510, 473, 509, 532, 496, 510, 495, 540, 507, 508, 487, 511, 480, 530, 514, 481, 530, 504, 518, 521, 518, 504, 490, 522, 505, 510, 527, 517, 478, 506, 518, 487, 525] payload_rows_per_dst=[5574, 5616, 5634, 5500, 5568, 5504, 5506, 5518] dup_rows_per_dst=[2644, 2770, 2642, 2650, 2556, 2578, 2658, 2618] combine_rows_per_src=[5574, 5616, 5634, 5500, 5568, 5504, 5506, 5518] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278448128 dispatch_meta_bytes=229848 combine_payload_bytes=278448128 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278677976 ingress_bytes=388548608 link_bytes=388548608 token_bytes=557126104 weight_movement_bytes=110100480 total_nvlink_bytes=667226584 weight_to_token_ratio=0.197622 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.757499346e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278376448 dispatch_meta_bytes=229184 combine_payload_bytes=278376448 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278605632 ingress_bytes=388476928 link_bytes=388476928 token_bytes=556982080 weight_movement_bytes=110100480 total_nvlink_bytes=667082560 weight_to_token_ratio=0.197673 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.756252713e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44542 payload_rows_remote=38934 dup_rows=20994 dup_rows_remote=18282 dedup_ratio=0.320343 router_tokens_per_expert=[488, 522, 507, 498, 537, 473, 486, 524, 471, 556, 487, 528, 570, 522, 500, 525, 551, 491, 495, 520, 512, 485, 486, 476, 474, 549, 509, 510, 549, 539, 507, 523, 536, 514, 504, 526, 520, 534, 538, 505, 498, 514, 493, 494, 499, 523, 525, 512, 497, 488, 524, 545, 545, 507, 535, 515, 510, 493, 503, 494, 535, 487, 501, 484] payload_rows_per_dst=[5520, 5618, 5512, 5608, 5662, 5546, 5618, 5458] dup_rows_per_dst=[2550, 2700, 2520, 2712, 2692, 2570, 2694, 2556] combine_rows_per_src=[5520, 5618, 5512, 5608, 5662, 5546, 5618, 5458] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44442 payload_rows_remote=38654 dup_rows=21094 dup_rows_remote=18066 dedup_ratio=0.321869 router_tokens_per_expert=[477, 508, 519, 549, 544, 513, 507, 512, 499, 554, 540, 526, 510, 510, 536, 525, 481, 480, 508, 506, 517, 510, 557, 524, 496, 506, 485, 521, 512, 502, 529, 507, 514, 459, 508, 527, 492, 557, 503, 510, 509, 528, 532, 523, 470, 520, 532, 518, 483, 510, 536, 523, 497, 506, 497, 505, 529, 502, 490, 527, 517, 492, 489, 493] payload_rows_per_dst=[5568, 5298, 5522, 5560, 5402, 5710, 5594, 5788] dup_rows_per_dst=[2476, 2364, 2644, 2556, 2414, 2878, 2734, 3028] combine_rows_per_src=[5568, 5298, 5522, 5560, 5402, 5710, 5594, 5788] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44504 payload_rows_remote=38958 dup_rows=21032 dup_rows_remote=18390 dedup_ratio=0.320923 router_tokens_per_expert=[528, 498, 541, 547, 511, 516, 538, 479, 478, 536, 468, 511, 515, 543, 527, 516, 500, 494, 504, 517, 492, 520, 504, 517, 541, 527, 465, 512, 523, 506, 535, 503, 535, 526, 498, 520, 507, 519, 541, 509, 507, 496, 475, 520, 509, 547, 495, 478, 470, 483, 549, 499, 489, 507, 514, 514, 487, 529, 487, 541, 496, 572, 520, 517] payload_rows_per_dst=[5564, 5546, 5530, 5582, 5698, 5518, 5478, 5588] dup_rows_per_dst=[2752, 2642, 2566, 2642, 2612, 2536, 2572, 2710] combine_rows_per_src=[5564, 5546, 5530, 5582, 5698, 5518, 5478, 5588] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44744 payload_rows_remote=38974 dup_rows=20792 dup_rows_remote=18046 dedup_ratio=0.317261 router_tokens_per_expert=[517, 526, 555, 549, 546, 536, 492, 497, 509, 462, 501, 525, 529, 510, 558, 531, 507, 558, 508, 499, 489, 522, 477, 520, 507, 482, 520, 544, 498, 502, 525, 510, 501, 523, 497, 496, 543, 494, 501, 514, 572, 513, 542, 528, 502, 497, 504, 512, 484, 539, 513, 509, 529, 490, 521, 469, 505, 500, 505, 520, 470, 475, 503, 486] payload_rows_per_dst=[5770, 5608, 5664, 5532, 5532, 5634, 5510, 5494] dup_rows_per_dst=[2746, 2588, 2652, 2514, 2554, 2706, 2598, 2434] combine_rows_per_src=[5770, 5608, 5664, 5532, 5532, 5634, 5510, 5494] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279336960 dispatch_meta_bytes=229312 combine_payload_bytes=279336960 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279566272 ingress_bytes=389437440 link_bytes=389437440 token_bytes=558903232 weight_movement_bytes=110100480 total_nvlink_bytes=669003712 weight_to_token_ratio=0.196994 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.772957596e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278491136 dispatch_meta_bytes=229648 combine_payload_bytes=278491136 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278720784 ingress_bytes=608792576 link_bytes=608792576 token_bytes=557211920 weight_movement_bytes=330301440 total_nvlink_bytes=887513360 weight_to_token_ratio=0.592775 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058790419e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279078912 dispatch_meta_bytes=228864 combine_payload_bytes=279078912 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279307776 ingress_bytes=499279872 link_bytes=499279872 token_bytes=558386688 weight_movement_bytes=220200960 total_nvlink_bytes=778587648 weight_to_token_ratio=0.394352 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.683298148e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277071872 dispatch_meta_bytes=226880 combine_payload_bytes=277071872 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277298752 ingress_bytes=387172352 link_bytes=387172352 token_bytes=554370624 weight_movement_bytes=110100480 total_nvlink_bytes=664471104 weight_to_token_ratio=0.198604 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.733563990e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279365632 dispatch_meta_bytes=228080 combine_payload_bytes=279365632 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279593712 ingress_bytes=499566592 link_bytes=499566592 token_bytes=558959344 weight_movement_bytes=220200960 total_nvlink_bytes=779160304 weight_to_token_ratio=0.393948 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.688284681e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279250944 dispatch_meta_bytes=229392 combine_payload_bytes=279250944 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279480336 ingress_bytes=609552384 link_bytes=609552384 token_bytes=558731280 weight_movement_bytes=330301440 total_nvlink_bytes=889032720 weight_to_token_ratio=0.591163 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060111850e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44564 payload_rows_remote=39040 dup_rows=20972 dup_rows_remote=18390 dedup_ratio=0.320007 router_tokens_per_expert=[475, 520, 519, 514, 514, 549, 514, 485, 530, 504, 517, 528, 519, 472, 517, 492, 516, 507, 531, 520, 540, 499, 529, 563, 555, 496, 519, 508, 529, 485, 512, 467, 490, 502, 509, 487, 479, 517, 503, 540, 519, 516, 533, 516, 488, 538, 482, 503, 496, 517, 510, 509, 499, 520, 491, 511, 503, 515, 486, 525, 530, 530, 501, 558] payload_rows_per_dst=[5512, 5550, 5636, 5562, 5532, 5622, 5524, 5626] dup_rows_per_dst=[2668, 2608, 2774, 2580, 2522, 2568, 2582, 2670] combine_rows_per_src=[5512, 5550, 5636, 5562, 5532, 5622, 5524, 5626] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44540 payload_rows_remote=38932 dup_rows=20996 dup_rows_remote=18286 dedup_ratio=0.320374 router_tokens_per_expert=[486, 521, 504, 498, 535, 471, 483, 523, 469, 553, 485, 525, 568, 521, 499, 523, 548, 489, 493, 518, 509, 485, 484, 475, 474, 548, 509, 510, 549, 539, 507, 523, 536, 516, 507, 528, 523, 535, 537, 504, 500, 516, 494, 496, 498, 524, 525, 512, 500, 488, 526, 546, 546, 509, 536, 517, 511, 496, 506, 496, 536, 489, 505, 486] payload_rows_per_dst=[5512, 5602, 5510, 5608, 5666, 5542, 5626, 5474] dup_rows_per_dst=[2530, 2684, 2492, 2710, 2706, 2588, 2710, 2576] combine_rows_per_src=[5512, 5602, 5510, 5608, 5666, 5542, 5626, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278448128 dispatch_meta_bytes=229608 combine_payload_bytes=278448128 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278677736 ingress_bytes=608749568 link_bytes=608749568 token_bytes=557125864 weight_movement_bytes=330301440 total_nvlink_bytes=887427304 weight_to_token_ratio=0.592867 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058715621e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44402 payload_rows_remote=38850 dup_rows=21134 dup_rows_remote=18598 dedup_ratio=0.322479 router_tokens_per_expert=[533, 553, 504, 513, 491, 500, 504, 521, 532, 541, 497, 544, 534, 504, 511, 498, 493, 508, 531, 524, 503, 548, 520, 495, 509, 479, 569, 493, 504, 539, 492, 487, 510, 493, 519, 534, 548, 486, 539, 557, 488, 506, 495, 553, 467, 505, 490, 518, 495, 545, 502, 504, 532, 478, 537, 451, 533, 490, 519, 468, 484, 536, 498, 514] payload_rows_per_dst=[5568, 5570, 5566, 5544, 5654, 5496, 5552, 5452] dup_rows_per_dst=[2670, 2752, 2678, 2600, 2718, 2548, 2536, 2632] combine_rows_per_src=[5568, 5570, 5566, 5544, 5654, 5496, 5552, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278333440 dispatch_meta_bytes=229104 combine_payload_bytes=278333440 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278562544 ingress_bytes=388433920 link_bytes=388433920 token_bytes=556895984 weight_movement_bytes=110100480 total_nvlink_bytes=666996464 weight_to_token_ratio=0.197704 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.755504733e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277702656 dispatch_meta_bytes=227904 combine_payload_bytes=277702656 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277930560 ingress_bytes=387803136 link_bytes=387803136 token_bytes=555633216 weight_movement_bytes=110100480 total_nvlink_bytes=665733696 weight_to_token_ratio=0.198153 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.744534362e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279465984 dispatch_meta_bytes=229568 combine_payload_bytes=279465984 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279695552 ingress_bytes=609767424 link_bytes=609767424 token_bytes=559161536 weight_movement_bytes=330301440 total_nvlink_bytes=889462976 weight_to_token_ratio=0.590708 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060485840e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279379968 dispatch_meta_bytes=229376 combine_payload_bytes=279379968 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279609344 ingress_bytes=389480448 link_bytes=389480448 token_bytes=558989312 weight_movement_bytes=110100480 total_nvlink_bytes=669089792 weight_to_token_ratio=0.196963 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773705576e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44716 payload_rows_remote=38928 dup_rows=20820 dup_rows_remote=18018 dedup_ratio=0.317688 router_tokens_per_expert=[515, 523, 555, 548, 545, 535, 490, 491, 508, 462, 499, 524, 528, 509, 558, 529, 506, 559, 508, 498, 487, 522, 477, 521, 506, 482, 520, 541, 495, 501, 526, 508, 501, 521, 497, 496, 545, 496, 500, 513, 572, 514, 544, 529, 501, 498, 503, 512, 486, 542, 513, 512, 531, 489, 523, 471, 505, 502, 507, 524, 472, 479, 504, 490] payload_rows_per_dst=[5788, 5576, 5710, 5568, 5404, 5638, 5522, 5510] dup_rows_per_dst=[2802, 2520, 2750, 2542, 2430, 2708, 2612, 2456] combine_rows_per_src=[5788, 5576, 5710, 5568, 5404, 5638, 5522, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279064576 dispatch_meta_bytes=228872 combine_payload_bytes=279064576 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279293448 ingress_bytes=609366016 link_bytes=609366016 token_bytes=558358024 weight_movement_bytes=330301440 total_nvlink_bytes=888659464 weight_to_token_ratio=0.591559 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059787725e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278476800 dispatch_meta_bytes=229792 combine_payload_bytes=278476800 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278706592 ingress_bytes=388577280 link_bytes=388577280 token_bytes=557183392 weight_movement_bytes=110100480 total_nvlink_bytes=667283872 weight_to_token_ratio=0.197602 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.757997999e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279035904 dispatch_meta_bytes=227784 combine_payload_bytes=279035904 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279263688 ingress_bytes=389136384 link_bytes=389136384 token_bytes=558299592 weight_movement_bytes=110100480 total_nvlink_bytes=668400072 weight_to_token_ratio=0.197207 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.767721737e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44638 payload_rows_remote=38972 dup_rows=20898 dup_rows_remote=18212 dedup_ratio=0.318878 router_tokens_per_expert=[496, 505, 521, 525, 518, 505, 482, 552, 509, 539, 509, 543, 515, 508, 512, 523, 541, 527, 543, 516, 495, 496, 469, 480, 512, 485, 540, 513, 512, 501, 516, 487, 544, 527, 478, 508, 525, 528, 547, 519, 513, 519, 454, 530, 517, 501, 475, 505, 504, 501, 509, 510, 513, 512, 511, 478, 501, 525, 531, 497, 471, 550, 540, 530] payload_rows_per_dst=[5624, 5602, 5560, 5538, 5666, 5458, 5508, 5682] dup_rows_per_dst=[2584, 2714, 2574, 2594, 2686, 2570, 2568, 2608] combine_rows_per_src=[5624, 5602, 5560, 5538, 5666, 5458, 5508, 5682] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44532 payload_rows_remote=39032 dup_rows=21004 dup_rows_remote=18448 dedup_ratio=0.320496 router_tokens_per_expert=[476, 536, 512, 492, 564, 532, 545, 488, 531, 527, 501, 497, 515, 516, 554, 481, 538, 534, 566, 493, 521, 508, 523, 535, 525, 483, 525, 520, 517, 500, 445, 469, 483, 498, 472, 501, 525, 537, 494, 516, 509, 527, 472, 497, 495, 519, 512, 497, 500, 533, 518, 541, 482, 497, 493, 486, 520, 522, 537, 547, 534, 493, 501, 541] payload_rows_per_dst=[5594, 5630, 5678, 5470, 5510, 5500, 5520, 5630] dup_rows_per_dst=[2696, 2614, 2758, 2498, 2542, 2556, 2580, 2760] combine_rows_per_src=[5594, 5630, 5678, 5470, 5510, 5500, 5520, 5630] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44238 payload_rows_remote=38732 dup_rows=21298 dup_rows_remote=18676 dedup_ratio=0.324982 router_tokens_per_expert=[550, 526, 502, 479, 508, 492, 517, 505, 491, 500, 498, 485, 529, 510, 498, 480, 512, 479, 540, 509, 495, 526, 522, 552, 511, 531, 541, 490, 504, 518, 527, 507, 515, 529, 513, 523, 517, 522, 527, 544, 525, 524, 504, 488, 534, 543, 518, 506, 498, 509, 527, 491, 492, 523, 511, 492, 546, 510, 481, 487, 547, 525, 483, 480] payload_rows_per_dst=[5270, 5304, 5278, 5720, 5638, 5766, 5756, 5506] dup_rows_per_dst=[2370, 2262, 2342, 3056, 2742, 2934, 2970, 2622] combine_rows_per_src=[5270, 5304, 5278, 5720, 5638, 5766, 5756, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44448 payload_rows_remote=38858 dup_rows=21088 dup_rows_remote=18430 dedup_ratio=0.321777 router_tokens_per_expert=[531, 500, 542, 530, 547, 522, 513, 488, 506, 532, 523, 477, 549, 543, 497, 497, 519, 545, 532, 536, 494, 464, 525, 469, 529, 487, 488, 533, 510, 522, 525, 538, 462, 522, 520, 481, 519, 539, 506, 495, 510, 507, 505, 502, 528, 546, 528, 488, 471, 528, 537, 478, 529, 491, 508, 491, 493, 475, 541, 514, 541, 531, 497, 472] payload_rows_per_dst=[5588, 5590, 5546, 5542, 5588, 5544, 5534, 5516] dup_rows_per_dst=[2758, 2658, 2622, 2722, 2500, 2684, 2532, 2612] combine_rows_per_src=[5588, 5590, 5546, 5542, 5588, 5544, 5534, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44270 payload_rows_remote=38750 dup_rows=21266 dup_rows_remote=18558 dedup_ratio=0.324493 router_tokens_per_expert=[544, 525, 486, 539, 506, 549, 496, 527, 503, 533, 519, 527, 515, 505, 511, 515, 537, 514, 493, 517, 522, 500, 556, 475, 486, 492, 526, 477, 509, 492, 521, 545, 521, 527, 523, 520, 495, 533, 516, 505, 523, 496, 470, 516, 519, 517, 503, 491, 493, 497, 516, 487, 476, 558, 528, 506, 501, 481, 519, 529, 491, 503, 526, 520] payload_rows_per_dst=[5584, 5566, 5520, 5480, 5562, 5504, 5504, 5550] dup_rows_per_dst=[2760, 2690, 2708, 2616, 2718, 2566, 2618, 2590] combine_rows_per_src=[5584, 5566, 5520, 5480, 5562, 5504, 5504, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44584 payload_rows_remote=38996 dup_rows=20952 dup_rows_remote=18452 dedup_ratio=0.319702 router_tokens_per_expert=[505, 493, 558, 500, 528, 521, 493, 518, 525, 483, 522, 545, 520, 515, 503, 526, 531, 499, 568, 502, 518, 508, 551, 541, 484, 507, 498, 497, 535, 479, 531, 513, 480, 533, 503, 497, 511, 530, 504, 506, 485, 485, 538, 514, 520, 534, 493, 484, 504, 546, 532, 497, 516, 531, 512, 499, 523, 488, 499, 469, 524, 492, 474, 528] payload_rows_per_dst=[5632, 5634, 5684, 5588, 5502, 5466, 5618, 5460] dup_rows_per_dst=[2600, 2644, 2752, 2500, 2626, 2640, 2656, 2534] combine_rows_per_src=[5632, 5634, 5684, 5588, 5502, 5466, 5618, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279351296 dispatch_meta_bytes=228736 combine_payload_bytes=279351296 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279580032 ingress_bytes=389451776 link_bytes=389451776 token_bytes=558931328 weight_movement_bytes=110100480 total_nvlink_bytes=669031808 weight_to_token_ratio=0.196984 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773206923e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279781376 dispatch_meta_bytes=229920 combine_payload_bytes=279781376 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280011296 ingress_bytes=389881856 link_bytes=389881856 token_bytes=559792672 weight_movement_bytes=110100480 total_nvlink_bytes=669893152 weight_to_token_ratio=0.196681 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.780686721e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44556 payload_rows_remote=39018 dup_rows=20980 dup_rows_remote=18386 dedup_ratio=0.320129 router_tokens_per_expert=[471, 518, 519, 510, 512, 548, 512, 482, 529, 502, 512, 525, 518, 471, 515, 491, 517, 505, 528, 521, 539, 499, 527, 563, 553, 499, 519, 509, 527, 484, 509, 468, 487, 503, 509, 487, 479, 516, 503, 541, 519, 517, 533, 515, 488, 537, 485, 504, 496, 517, 513, 513, 500, 521, 493, 513, 506, 520, 489, 526, 535, 533, 505, 563] payload_rows_per_dst=[5496, 5530, 5630, 5564, 5532, 5626, 5538, 5640] dup_rows_per_dst=[2648, 2596, 2768, 2572, 2518, 2570, 2594, 2714] combine_rows_per_src=[5496, 5530, 5630, 5564, 5532, 5626, 5538, 5640] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277630976 dispatch_meta_bytes=229632 combine_payload_bytes=277630976 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=277860608 ingress_bytes=607932416 link_bytes=607932416 token_bytes=555491584 weight_movement_bytes=330301440 total_nvlink_bytes=885793024 weight_to_token_ratio=0.594611 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057294459e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278534144 dispatch_meta_bytes=229152 combine_payload_bytes=278534144 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278763296 ingress_bytes=498735104 link_bytes=498735104 token_bytes=557297440 weight_movement_bytes=220200960 total_nvlink_bytes=777498400 weight_to_token_ratio=0.395123 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.673823737e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44526 payload_rows_remote=39026 dup_rows=21010 dup_rows_remote=18472 dedup_ratio=0.320587 router_tokens_per_expert=[476, 543, 514, 496, 566, 532, 549, 494, 537, 529, 500, 498, 516, 518, 558, 482, 539, 535, 568, 495, 519, 511, 523, 537, 524, 484, 525, 518, 516, 499, 443, 468, 484, 498, 473, 501, 529, 539, 492, 514, 507, 525, 472, 495, 494, 517, 512, 497, 498, 530, 517, 539, 482, 495, 489, 486, 518, 517, 532, 545, 532, 491, 499, 537] payload_rows_per_dst=[5608, 5648, 5678, 5462, 5512, 5500, 5510, 5608] dup_rows_per_dst=[2732, 2628, 2776, 2492, 2548, 2538, 2562, 2734] combine_rows_per_src=[5608, 5648, 5678, 5462, 5512, 5500, 5510, 5608] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279838720 dispatch_meta_bytes=229720 combine_payload_bytes=279838720 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280068440 ingress_bytes=389939200 link_bytes=389939200 token_bytes=559907160 weight_movement_bytes=110100480 total_nvlink_bytes=670007640 weight_to_token_ratio=0.196641 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.781684028e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44248 payload_rows_remote=38722 dup_rows=21288 dup_rows_remote=18566 dedup_ratio=0.324829 router_tokens_per_expert=[544, 529, 488, 542, 510, 552, 500, 529, 502, 534, 522, 527, 517, 507, 512, 515, 539, 516, 496, 518, 523, 501, 556, 475, 487, 492, 530, 480, 508, 491, 522, 544, 521, 524, 519, 519, 492, 532, 516, 507, 524, 493, 467, 517, 517, 517, 500, 491, 490, 495, 515, 485, 472, 559, 526, 506, 499, 481, 516, 526, 489, 503, 524, 518] payload_rows_per_dst=[5606, 5572, 5526, 5480, 5546, 5490, 5492, 5536] dup_rows_per_dst=[2782, 2700, 2722, 2628, 2714, 2562, 2604, 2576] combine_rows_per_src=[5606, 5572, 5526, 5480, 5546, 5490, 5492, 5536] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44628 payload_rows_remote=38972 dup_rows=20908 dup_rows_remote=18216 dedup_ratio=0.319031 router_tokens_per_expert=[500, 506, 523, 526, 525, 507, 487, 552, 511, 542, 511, 544, 516, 509, 515, 524, 541, 527, 547, 517, 497, 498, 469, 481, 514, 484, 540, 513, 510, 500, 517, 486, 546, 528, 477, 506, 526, 526, 547, 518, 511, 520, 451, 530, 517, 498, 474, 505, 501, 499, 508, 509, 512, 510, 511, 476, 502, 522, 523, 494, 470, 549, 535, 528] payload_rows_per_dst=[5646, 5610, 5574, 5542, 5656, 5446, 5494, 5660] dup_rows_per_dst=[2606, 2734, 2580, 2586, 2692, 2566, 2558, 2586] combine_rows_per_src=[5646, 5610, 5574, 5542, 5656, 5446, 5494, 5660] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279738368 dispatch_meta_bytes=229992 combine_payload_bytes=279738368 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279968360 ingress_bytes=389838848 link_bytes=389838848 token_bytes=559706728 weight_movement_bytes=110100480 total_nvlink_bytes=669807208 weight_to_token_ratio=0.196711 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.779938741e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44220 payload_rows_remote=38664 dup_rows=21316 dup_rows_remote=18602 dedup_ratio=0.325256 router_tokens_per_expert=[553, 528, 506, 484, 509, 495, 523, 504, 493, 502, 500, 488, 531, 510, 502, 481, 516, 479, 538, 511, 493, 526, 524, 554, 512, 531, 541, 494, 501, 521, 529, 506, 515, 528, 513, 521, 514, 520, 526, 543, 522, 523, 505, 486, 532, 541, 518, 504, 497, 507, 528, 492, 488, 520, 511, 489, 545, 508, 480, 485, 542, 518, 482, 480] payload_rows_per_dst=[5202, 5266, 5188, 5708, 5666, 5814, 5820, 5556] dup_rows_per_dst=[2252, 2224, 2198, 3056, 2794, 3008, 3070, 2714] combine_rows_per_src=[5202, 5266, 5188, 5708, 5666, 5814, 5820, 5556] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44570 payload_rows_remote=38976 dup_rows=20966 dup_rows_remote=18468 dedup_ratio=0.319916 router_tokens_per_expert=[507, 494, 561, 505, 534, 522, 497, 518, 529, 483, 520, 546, 524, 515, 507, 530, 533, 501, 569, 506, 518, 506, 553, 540, 485, 511, 496, 498, 536, 478, 530, 512, 475, 534, 503, 499, 509, 528, 503, 504, 484, 484, 538, 512, 517, 535, 495, 480, 504, 546, 532, 495, 515, 531, 511, 497, 522, 486, 494, 465, 520, 488, 473, 525] payload_rows_per_dst=[5652, 5640, 5686, 5594, 5498, 5454, 5614, 5432] dup_rows_per_dst=[2624, 2668, 2766, 2498, 2612, 2636, 2648, 2514] combine_rows_per_src=[5652, 5640, 5686, 5594, 5498, 5454, 5614, 5432] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44480 payload_rows_remote=38930 dup_rows=21056 dup_rows_remote=18352 dedup_ratio=0.321289 router_tokens_per_expert=[508, 538, 515, 530, 517, 517, 470, 529, 569, 531, 491, 510, 529, 532, 523, 521, 511, 508, 495, 522, 517, 489, 525, 561, 518, 509, 527, 499, 477, 489, 482, 536, 564, 494, 503, 518, 491, 525, 490, 496, 544, 483, 530, 480, 500, 502, 518, 506, 487, 479, 517, 515, 481, 533, 489, 505, 509, 514, 552, 534, 502, 519, 502, 491] payload_rows_per_dst=[5550, 5666, 5682, 5488, 5548, 5454, 5454, 5638] dup_rows_per_dst=[2704, 2740, 2574, 2586, 2614, 2672, 2558, 2608] combine_rows_per_src=[5550, 5666, 5682, 5488, 5548, 5454, 5454, 5638] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44460 payload_rows_remote=38854 dup_rows=21076 dup_rows_remote=18400 dedup_ratio=0.321594 router_tokens_per_expert=[536, 503, 545, 532, 550, 525, 515, 490, 509, 534, 525, 479, 549, 549, 498, 498, 518, 547, 535, 537, 493, 464, 526, 468, 531, 487, 492, 532, 511, 523, 524, 537, 463, 520, 520, 481, 519, 540, 504, 494, 509, 507, 504, 497, 527, 544, 527, 488, 472, 525, 536, 473, 529, 490, 507, 491, 488, 475, 540, 513, 537, 527, 495, 464] payload_rows_per_dst=[5606, 5606, 5550, 5546, 5584, 5536, 5534, 5498] dup_rows_per_dst=[2786, 2676, 2626, 2728, 2498, 2670, 2512, 2580] combine_rows_per_src=[5606, 5606, 5550, 5546, 5584, 5536, 5534, 5498] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277559296 dispatch_meta_bytes=229152 combine_payload_bytes=277559296 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=277788448 ingress_bytes=607860736 link_bytes=607860736 token_bytes=555347744 weight_movement_bytes=330301440 total_nvlink_bytes=885649184 weight_to_token_ratio=0.594765 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057169796e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279351296 dispatch_meta_bytes=228752 combine_payload_bytes=279351296 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279580048 ingress_bytes=389451776 link_bytes=389451776 token_bytes=558931344 weight_movement_bytes=110100480 total_nvlink_bytes=669031824 weight_to_token_ratio=0.196984 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773206923e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44634 payload_rows_remote=39018 dup_rows=20902 dup_rows_remote=18266 dedup_ratio=0.318939 router_tokens_per_expert=[497, 504, 484, 505, 471, 501, 466, 525, 528, 523, 549, 479, 542, 527, 516, 509, 512, 537, 531, 527, 526, 484, 522, 505, 481, 543, 478, 493, 486, 518, 535, 520, 507, 495, 528, 509, 515, 482, 524, 485, 524, 521, 564, 528, 490, 521, 513, 519, 525, 526, 528, 484, 526, 572, 486, 479, 533, 539, 498, 518, 509, 516, 505, 475] payload_rows_per_dst=[5454, 5624, 5592, 5562, 5518, 5670, 5616, 5598] dup_rows_per_dst=[2452, 2722, 2696, 2546, 2572, 2690, 2636, 2588] combine_rows_per_src=[5454, 5624, 5592, 5562, 5518, 5670, 5616, 5598] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277143552 dispatch_meta_bytes=229064 combine_payload_bytes=277143552 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277372616 ingress_bytes=387244032 link_bytes=387244032 token_bytes=554516168 weight_movement_bytes=110100480 total_nvlink_bytes=664616648 weight_to_token_ratio=0.198552 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.734810623e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279379968 dispatch_meta_bytes=229776 combine_payload_bytes=279379968 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279609744 ingress_bytes=389480448 link_bytes=389480448 token_bytes=558989712 weight_movement_bytes=110100480 total_nvlink_bytes=669090192 weight_to_token_ratio=0.196963 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773705576e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279050240 dispatch_meta_bytes=229128 combine_payload_bytes=279050240 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279279368 ingress_bytes=609351680 link_bytes=609351680 token_bytes=558329608 weight_movement_bytes=330301440 total_nvlink_bytes=888631048 weight_to_token_ratio=0.591589 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059762793e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278505472 dispatch_meta_bytes=229016 combine_payload_bytes=278505472 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278734488 ingress_bytes=608806912 link_bytes=608806912 token_bytes=557239960 weight_movement_bytes=330301440 total_nvlink_bytes=887541400 weight_to_token_ratio=0.592745 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058815351e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44288 payload_rows_remote=38670 dup_rows=21248 dup_rows_remote=18578 dedup_ratio=0.324219 router_tokens_per_expert=[555, 530, 483, 492, 523, 496, 499, 547, 520, 510, 518, 514, 494, 526, 532, 502, 573, 495, 496, 505, 511, 489, 511, 540, 511, 524, 490, 472, 514, 531, 527, 512, 509, 549, 506, 466, 535, 528, 526, 476, 497, 527, 516, 526, 547, 518, 500, 513, 535, 523, 499, 476, 485, 482, 538, 532, 511, 514, 488, 468, 527, 488, 494, 527] payload_rows_per_dst=[5596, 5510, 5552, 5460, 5566, 5618, 5540, 5446] dup_rows_per_dst=[2654, 2722, 2688, 2702, 2624, 2670, 2600, 2588] combine_rows_per_src=[5596, 5510, 5552, 5460, 5566, 5618, 5540, 5446] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279681024 dispatch_meta_bytes=229136 combine_payload_bytes=279681024 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279910160 ingress_bytes=389781504 link_bytes=389781504 token_bytes=559591184 weight_movement_bytes=110100480 total_nvlink_bytes=669691664 weight_to_token_ratio=0.196752 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778941435e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277760000 dispatch_meta_bytes=229232 combine_payload_bytes=277760000 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=277989232 ingress_bytes=608061440 link_bytes=608061440 token_bytes=555749232 weight_movement_bytes=330301440 total_nvlink_bytes=886050672 weight_to_token_ratio=0.594335 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057518853e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279523328 dispatch_meta_bytes=229792 combine_payload_bytes=279523328 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279753120 ingress_bytes=389623808 link_bytes=389623808 token_bytes=559276448 weight_movement_bytes=110100480 total_nvlink_bytes=669376928 weight_to_token_ratio=0.196862 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.776198842e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44494 payload_rows_remote=38908 dup_rows=21042 dup_rows_remote=18296 dedup_ratio=0.321075 router_tokens_per_expert=[505, 536, 514, 523, 515, 514, 470, 526, 569, 527, 483, 512, 525, 532, 521, 519, 508, 509, 495, 523, 517, 488, 521, 560, 517, 507, 526, 502, 475, 490, 483, 538, 563, 494, 504, 522, 490, 525, 487, 495, 545, 487, 529, 480, 504, 504, 520, 507, 489, 482, 517, 515, 483, 536, 491, 507, 507, 515, 555, 537, 507, 523, 506, 492] payload_rows_per_dst=[5586, 5700, 5622, 5492, 5572, 5468, 5468, 5586] dup_rows_per_dst=[2746, 2802, 2494, 2584, 2634, 2684, 2572, 2526] combine_rows_per_src=[5586, 5700, 5622, 5492, 5572, 5468, 5468, 5586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279681024 dispatch_meta_bytes=229616 combine_payload_bytes=279681024 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279910640 ingress_bytes=389781504 link_bytes=389781504 token_bytes=559591664 weight_movement_bytes=110100480 total_nvlink_bytes=669692144 weight_to_token_ratio=0.196751 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778941435e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44280 payload_rows_remote=38628 dup_rows=21256 dup_rows_remote=18456 dedup_ratio=0.324341 router_tokens_per_expert=[511, 500, 533, 535, 517, 530, 485, 521, 489, 504, 488, 534, 514, 482, 468, 512, 511, 517, 531, 523, 521, 541, 527, 507, 461, 505, 516, 547, 555, 508, 537, 499, 529, 532, 543, 546, 539, 508, 517, 512, 485, 511, 507, 472, 559, 459, 523, 483, 522, 485, 523, 521, 503, 495, 508, 509, 537, 474, 496, 492, 517, 480, 522, 530] payload_rows_per_dst=[5588, 5428, 5562, 5560, 5652, 5506, 5524, 5460] dup_rows_per_dst=[2676, 2554, 2794, 2696, 2800, 2492, 2608, 2636] combine_rows_per_src=[5588, 5428, 5562, 5560, 5652, 5506, 5524, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44296 payload_rows_remote=38674 dup_rows=21240 dup_rows_remote=18568 dedup_ratio=0.324097 router_tokens_per_expert=[551, 526, 479, 487, 520, 495, 496, 549, 516, 509, 516, 514, 491, 525, 530, 502, 571, 494, 495, 506, 510, 489, 510, 537, 509, 523, 491, 469, 513, 533, 526, 513, 508, 552, 506, 467, 534, 529, 529, 478, 497, 527, 517, 526, 548, 518, 499, 515, 537, 523, 501, 480, 488, 484, 541, 534, 514, 516, 491, 468, 529, 492, 497, 528] payload_rows_per_dst=[5584, 5498, 5544, 5456, 5576, 5622, 5560, 5456] dup_rows_per_dst=[2622, 2708, 2680, 2698, 2630, 2672, 2616, 2614] combine_rows_per_src=[5584, 5498, 5544, 5456, 5576, 5622, 5560, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278892544 dispatch_meta_bytes=228816 combine_payload_bytes=278892544 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279121360 ingress_bytes=499093504 link_bytes=499093504 token_bytes=558013904 weight_movement_bytes=220200960 total_nvlink_bytes=778214864 weight_to_token_ratio=0.394616 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.680056902e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44390 payload_rows_remote=38740 dup_rows=21146 dup_rows_remote=18422 dedup_ratio=0.322662 router_tokens_per_expert=[521, 503, 529, 480, 500, 467, 505, 505, 500, 529, 531, 515, 488, 530, 543, 499, 499, 513, 524, 506, 527, 517, 492, 530, 554, 545, 507, 498, 498, 549, 520, 523, 544, 454, 477, 506, 487, 504, 517, 481, 518, 508, 504, 521, 524, 530, 470, 519, 501, 495, 492, 527, 513, 491, 496, 555, 541, 520, 529, 491, 527, 561, 491, 527] payload_rows_per_dst=[5490, 5572, 5538, 5570, 5386, 5616, 5568, 5650] dup_rows_per_dst=[2530, 2698, 2678, 2676, 2554, 2624, 2662, 2724] combine_rows_per_src=[5490, 5572, 5538, 5570, 5386, 5616, 5568, 5650] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44622 payload_rows_remote=38998 dup_rows=20914 dup_rows_remote=18446 dedup_ratio=0.319122 router_tokens_per_expert=[530, 506, 512, 533, 481, 510, 483, 485, 519, 501, 546, 494, 515, 510, 550, 476, 535, 543, 475, 497, 475, 506, 521, 494, 514, 521, 536, 503, 515, 465, 494, 522, 529, 538, 517, 514, 497, 507, 500, 529, 523, 536, 529, 509, 488, 524, 493, 534, 525, 472, 523, 520, 510, 514, 506, 545, 498, 511, 500, 496, 514, 526, 530, 544] payload_rows_per_dst=[5536, 5556, 5624, 5496, 5664, 5634, 5586, 5526] dup_rows_per_dst=[2544, 2666, 2468, 2644, 2598, 2638, 2644, 2712] combine_rows_per_src=[5536, 5556, 5624, 5496, 5664, 5634, 5586, 5526] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44298 payload_rows_remote=38652 dup_rows=21238 dup_rows_remote=18450 dedup_ratio=0.324066 router_tokens_per_expert=[514, 504, 534, 536, 519, 535, 488, 525, 490, 505, 490, 537, 517, 485, 471, 512, 512, 515, 531, 524, 525, 543, 529, 505, 461, 507, 515, 547, 559, 507, 539, 500, 528, 528, 544, 544, 538, 506, 518, 511, 484, 512, 507, 470, 558, 456, 521, 483, 523, 483, 518, 519, 500, 493, 503, 506, 534, 474, 495, 491, 516, 478, 518, 528] payload_rows_per_dst=[5604, 5440, 5574, 5570, 5646, 5496, 5516, 5452] dup_rows_per_dst=[2706, 2574, 2794, 2700, 2788, 2486, 2574, 2616] combine_rows_per_src=[5604, 5440, 5574, 5570, 5646, 5496, 5516, 5452] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277186560 dispatch_meta_bytes=228992 combine_payload_bytes=277186560 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277415552 ingress_bytes=387287040 link_bytes=387287040 token_bytes=554602112 weight_movement_bytes=110100480 total_nvlink_bytes=664702592 weight_to_token_ratio=0.198522 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.735558603e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44348 payload_rows_remote=38668 dup_rows=21188 dup_rows_remote=18412 dedup_ratio=0.323303 router_tokens_per_expert=[526, 504, 530, 481, 505, 467, 508, 510, 500, 530, 532, 517, 492, 532, 546, 503, 500, 509, 527, 508, 527, 517, 496, 531, 553, 546, 509, 497, 498, 547, 519, 526, 544, 456, 477, 506, 486, 505, 516, 482, 518, 505, 504, 519, 523, 530, 472, 518, 500, 489, 490, 526, 509, 490, 493, 552, 537, 518, 524, 489, 524, 559, 489, 525] payload_rows_per_dst=[5504, 5556, 5352, 5550, 5424, 5642, 5640, 5680] dup_rows_per_dst=[2558, 2672, 2394, 2676, 2592, 2704, 2816, 2776] combine_rows_per_src=[5504, 5556, 5352, 5550, 5424, 5642, 5640, 5680] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277057536 dispatch_meta_bytes=228408 combine_payload_bytes=277057536 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277285944 ingress_bytes=387158016 link_bytes=387158016 token_bytes=554343480 weight_movement_bytes=110100480 total_nvlink_bytes=664443960 weight_to_token_ratio=0.198614 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.733314664e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279537664 dispatch_meta_bytes=229776 combine_payload_bytes=279537664 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279767440 ingress_bytes=609839104 link_bytes=609839104 token_bytes=559305104 weight_movement_bytes=330301440 total_nvlink_bytes=889606544 weight_to_token_ratio=0.590557 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060610503e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44444 payload_rows_remote=38842 dup_rows=21092 dup_rows_remote=18372 dedup_ratio=0.321838 router_tokens_per_expert=[500, 484, 536, 520, 504, 515, 526, 540, 579, 500, 516, 514, 482, 518, 462, 521, 482, 510, 548, 514, 502, 522, 521, 526, 536, 486, 538, 505, 525, 518, 528, 525, 527, 531, 542, 545, 462, 527, 537, 465, 488, 514, 543, 526, 477, 513, 473, 446, 490, 513, 534, 493, 536, 526, 488, 499, 519, 498, 537, 513, 509, 508, 483, 503] payload_rows_per_dst=[5586, 5602, 5570, 5602, 5628, 5406, 5580, 5470] dup_rows_per_dst=[2664, 2582, 2680, 2720, 2644, 2554, 2578, 2670] combine_rows_per_src=[5586, 5602, 5570, 5602, 5628, 5406, 5580, 5470] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44496 payload_rows_remote=38808 dup_rows=21040 dup_rows_remote=18382 dedup_ratio=0.321045 router_tokens_per_expert=[484, 495, 500, 543, 511, 495, 531, 529, 511, 487, 518, 518, 523, 497, 535, 487, 529, 548, 546, 522, 506, 517, 534, 501, 517, 519, 520, 500, 512, 536, 533, 513, 505, 511, 519, 516, 509, 519, 525, 520, 515, 517, 516, 534, 484, 524, 460, 490, 521, 513, 490, 503, 534, 499, 506, 482, 502, 517, 504, 438, 487, 526, 547, 518] payload_rows_per_dst=[5688, 5480, 5684, 5520, 5572, 5530, 5490, 5532] dup_rows_per_dst=[2658, 2672, 2758, 2574, 2676, 2550, 2606, 2546] combine_rows_per_src=[5688, 5480, 5684, 5520, 5572, 5530, 5490, 5532] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44542 payload_rows_remote=38988 dup_rows=20994 dup_rows_remote=18390 dedup_ratio=0.320343 router_tokens_per_expert=[532, 507, 501, 537, 536, 524, 518, 511, 548, 507, 501, 510, 518, 519, 501, 475, 535, 486, 543, 495, 481, 501, 511, 503, 515, 510, 518, 550, 519, 480, 508, 507, 525, 506, 497, 531, 486, 511, 464, 492, 554, 500, 541, 509, 490, 527, 503, 500, 511, 499, 477, 495, 518, 532, 512, 513, 485, 495, 538, 533, 539, 545, 513, 520] payload_rows_per_dst=[5540, 5554, 5616, 5610, 5522, 5580, 5480, 5640] dup_rows_per_dst=[2792, 2604, 2494, 2604, 2502, 2668, 2634, 2696] combine_rows_per_src=[5540, 5554, 5616, 5610, 5522, 5580, 5480, 5640] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44452 payload_rows_remote=38984 dup_rows=21084 dup_rows_remote=18408 dedup_ratio=0.321716 router_tokens_per_expert=[521, 493, 519, 482, 538, 512, 497, 546, 540, 508, 498, 502, 508, 526, 543, 483, 534, 492, 500, 485, 478, 505, 573, 535, 523, 525, 515, 508, 511, 550, 492, 558, 510, 502, 480, 530, 525, 498, 511, 544, 501, 485, 482, 462, 520, 522, 490, 482, 492, 502, 497, 499, 531, 486, 555, 510, 507, 560, 505, 524, 523, 525, 490, 518] payload_rows_per_dst=[5578, 5514, 5572, 5634, 5646, 5446, 5468, 5594] dup_rows_per_dst=[2638, 2702, 2632, 2730, 2554, 2442, 2676, 2710] combine_rows_per_src=[5578, 5514, 5572, 5634, 5646, 5446, 5468, 5594] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277172224 dispatch_meta_bytes=228320 combine_payload_bytes=277172224 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277400544 ingress_bytes=387272704 link_bytes=387272704 token_bytes=554572768 weight_movement_bytes=110100480 total_nvlink_bytes=664673248 weight_to_token_ratio=0.198532 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.735309277e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278419456 dispatch_meta_bytes=228856 combine_payload_bytes=278419456 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=278648312 ingress_bytes=718821376 link_bytes=718821376 token_bytes=557067768 weight_movement_bytes=440401920 total_nvlink_bytes=997469688 weight_to_token_ratio=0.790572 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.250148599e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278175744 dispatch_meta_bytes=228760 combine_payload_bytes=278175744 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278404504 ingress_bytes=388276224 link_bytes=388276224 token_bytes=556580248 weight_movement_bytes=110100480 total_nvlink_bytes=666680728 weight_to_token_ratio=0.197816 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.752762140e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279465984 dispatch_meta_bytes=229512 combine_payload_bytes=279465984 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279695496 ingress_bytes=499666944 link_bytes=499666944 token_bytes=559161480 weight_movement_bytes=220200960 total_nvlink_bytes=779362440 weight_to_token_ratio=0.393806 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.690029967e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44700 payload_rows_remote=39098 dup_rows=20836 dup_rows_remote=18160 dedup_ratio=0.317932 router_tokens_per_expert=[513, 555, 535, 502, 522, 500, 528, 511, 534, 506, 487, 552, 524, 503, 517, 525, 540, 496, 499, 545, 542, 494, 484, 494, 502, 528, 469, 497, 497, 482, 488, 529, 496, 523, 514, 538, 530, 505, 509, 544, 498, 513, 537, 530, 527, 497, 499, 538, 529, 499, 512, 495, 503, 465, 519, 527, 475, 483, 480, 538, 510, 502, 514, 519] payload_rows_per_dst=[5706, 5586, 5530, 5484, 5634, 5602, 5572, 5586] dup_rows_per_dst=[2626, 2710, 2658, 2500, 2684, 2676, 2526, 2456] combine_rows_per_src=[5706, 5586, 5530, 5484, 5634, 5602, 5572, 5586] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44520 payload_rows_remote=38872 dup_rows=21016 dup_rows_remote=18284 dedup_ratio=0.320679 router_tokens_per_expert=[481, 453, 535, 493, 518, 502, 500, 531, 530, 506, 544, 508, 512, 520, 517, 470, 488, 538, 522, 527, 507, 509, 465, 483, 510, 535, 516, 560, 534, 488, 477, 511, 550, 516, 495, 518, 503, 508, 563, 537, 537, 507, 489, 496, 504, 527, 535, 516, 522, 483, 550, 507, 489, 527, 536, 510, 529, 499, 509, 495, 495, 498, 538, 490] payload_rows_per_dst=[5484, 5618, 5532, 5504, 5648, 5626, 5626, 5482] dup_rows_per_dst=[2542, 2596, 2546, 2758, 2732, 2596, 2622, 2624] combine_rows_per_src=[5484, 5618, 5532, 5504, 5648, 5626, 5626, 5482] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279437312 dispatch_meta_bytes=229568 combine_payload_bytes=279437312 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279666880 ingress_bytes=389537792 link_bytes=389537792 token_bytes=559104192 weight_movement_bytes=110100480 total_nvlink_bytes=669204672 weight_to_token_ratio=0.196923 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.774702882e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44626 payload_rows_remote=39054 dup_rows=20910 dup_rows_remote=18352 dedup_ratio=0.319061 router_tokens_per_expert=[496, 523, 529, 531, 500, 468, 493, 517, 531, 501, 528, 518, 482, 554, 540, 514, 502, 532, 494, 493, 509, 519, 494, 522, 477, 487, 518, 488, 534, 488, 527, 487, 524, 530, 520, 503, 511, 523, 485, 522, 521, 550, 536, 478, 518, 519, 483, 486, 502, 512, 490, 531, 536, 540, 518, 547, 517, 511, 543, 482, 518, 502, 537, 477] payload_rows_per_dst=[5504, 5618, 5572, 5520, 5590, 5560, 5662, 5600] dup_rows_per_dst=[2610, 2718, 2558, 2492, 2646, 2622, 2690, 2574] combine_rows_per_src=[5504, 5618, 5572, 5520, 5590, 5560, 5662, 5600] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=280254464 dispatch_meta_bytes=229032 combine_payload_bytes=280254464 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280483496 ingress_bytes=390354944 link_bytes=390354944 token_bytes=560737960 weight_movement_bytes=110100480 total_nvlink_bytes=670838440 weight_to_token_ratio=0.196349 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.788914500e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278634496 dispatch_meta_bytes=228624 combine_payload_bytes=278634496 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278863120 ingress_bytes=608935936 link_bytes=608935936 token_bytes=557497616 weight_movement_bytes=330301440 total_nvlink_bytes=887799056 weight_to_token_ratio=0.592471 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059039745e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44270 payload_rows_remote=38652 dup_rows=21266 dup_rows_remote=18492 dedup_ratio=0.324493 router_tokens_per_expert=[531, 498, 508, 530, 503, 504, 532, 477, 508, 540, 516, 543, 502, 498, 512, 494, 509, 509, 518, 513, 496, 484, 518, 538, 505, 512, 477, 520, 547, 535, 552, 514, 521, 490, 526, 475, 550, 470, 541, 518, 521, 507, 542, 524, 488, 529, 480, 494, 524, 492, 502, 548, 489, 524, 500, 543, 482, 518, 513, 511, 498, 500, 497, 508] payload_rows_per_dst=[5358, 5366, 5634, 5632, 5418, 5600, 5644, 5618] dup_rows_per_dst=[2404, 2410, 2870, 2874, 2466, 2660, 2808, 2774] combine_rows_per_src=[5358, 5366, 5634, 5632, 5418, 5600, 5644, 5618] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44358 payload_rows_remote=38768 dup_rows=21178 dup_rows_remote=18508 dedup_ratio=0.323151 router_tokens_per_expert=[543, 506, 518, 545, 522, 526, 519, 520, 514, 499, 523, 494, 453, 538, 528, 511, 503, 450, 532, 515, 501, 502, 518, 502, 519, 524, 514, 485, 535, 530, 489, 534, 514, 511, 519, 501, 483, 533, 568, 480, 514, 513, 539, 477, 499, 514, 483, 543, 511, 489, 493, 522, 511, 521, 538, 473, 523, 493, 512, 480, 559, 527, 505, 508] payload_rows_per_dst=[5598, 5616, 5526, 5590, 5510, 5482, 5504, 5532] dup_rows_per_dst=[2800, 2504, 2520, 2670, 2708, 2682, 2612, 2682] combine_rows_per_src=[5598, 5616, 5526, 5590, 5510, 5482, 5504, 5532] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279939072 dispatch_meta_bytes=229624 combine_payload_bytes=279939072 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280168696 ingress_bytes=390039552 link_bytes=390039552 token_bytes=560107768 weight_movement_bytes=110100480 total_nvlink_bytes=670208248 weight_to_token_ratio=0.196570 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.783429314e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44390 payload_rows_remote=38694 dup_rows=21146 dup_rows_remote=18370 dedup_ratio=0.322662 router_tokens_per_expert=[493, 547, 567, 538, 523, 477, 551, 529, 534, 518, 509, 529, 525, 485, 486, 561, 494, 536, 519, 499, 525, 471, 519, 506, 496, 455, 483, 519, 517, 507, 494, 515, 518, 549, 514, 535, 488, 509, 508, 507, 519, 533, 518, 535, 505, 510, 497, 494, 504, 496, 512, 542, 497, 478, 511, 520, 535, 468, 492, 493, 503, 510, 525, 516] payload_rows_per_dst=[5696, 5572, 5568, 5398, 5546, 5556, 5540, 5514] dup_rows_per_dst=[2776, 2722, 2570, 2574, 2688, 2666, 2580, 2570] combine_rows_per_src=[5696, 5572, 5568, 5398, 5546, 5556, 5540, 5514] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44366 payload_rows_remote=38758 dup_rows=21170 dup_rows_remote=18494 dedup_ratio=0.323029 router_tokens_per_expert=[496, 529, 496, 491, 512, 526, 517, 541, 484, 540, 526, 530, 502, 535, 494, 531, 484, 490, 528, 530, 521, 522, 531, 518, 494, 516, 501, 504, 531, 487, 517, 538, 528, 509, 508, 506, 508, 500, 498, 476, 538, 551, 514, 501, 491, 537, 502, 526, 449, 540, 510, 512, 464, 520, 492, 516, 514, 505, 501, 491, 549, 528, 495, 527] payload_rows_per_dst=[5550, 5608, 5536, 5572, 5520, 5600, 5412, 5568] dup_rows_per_dst=[2666, 2676, 2712, 2604, 2546, 2720, 2594, 2652] combine_rows_per_src=[5550, 5608, 5536, 5572, 5520, 5600, 5412, 5568] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44558 payload_rows_remote=39018 dup_rows=20978 dup_rows_remote=18396 dedup_ratio=0.320099 router_tokens_per_expert=[530, 503, 498, 534, 532, 523, 518, 509, 544, 505, 499, 509, 517, 516, 498, 473, 536, 486, 540, 493, 482, 501, 512, 502, 517, 509, 517, 552, 520, 480, 509, 505, 524, 507, 495, 529, 486, 512, 466, 495, 555, 500, 541, 508, 490, 529, 503, 501, 511, 502, 477, 493, 520, 533, 513, 514, 491, 495, 543, 534, 543, 549, 518, 522] payload_rows_per_dst=[5526, 5540, 5614, 5606, 5534, 5584, 5484, 5670] dup_rows_per_dst=[2768, 2582, 2490, 2612, 2494, 2670, 2642, 2720] combine_rows_per_src=[5526, 5540, 5614, 5606, 5534, 5584, 5484, 5670] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44600 payload_rows_remote=38982 dup_rows=20936 dup_rows_remote=18474 dedup_ratio=0.319458 router_tokens_per_expert=[528, 502, 507, 529, 479, 507, 478, 484, 518, 499, 544, 492, 509, 511, 547, 472, 532, 543, 476, 498, 472, 505, 518, 496, 513, 521, 535, 506, 515, 466, 493, 521, 526, 539, 519, 519, 495, 507, 497, 532, 528, 534, 528, 511, 488, 527, 498, 535, 526, 471, 530, 522, 514, 514, 508, 547, 500, 511, 502, 497, 515, 528, 536, 548] payload_rows_per_dst=[5512, 5536, 5618, 5496, 5664, 5638, 5596, 5540] dup_rows_per_dst=[2516, 2648, 2462, 2644, 2604, 2660, 2668, 2734] combine_rows_per_src=[5512, 5536, 5618, 5496, 5664, 5638, 5596, 5540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=276885504 dispatch_meta_bytes=228336 combine_payload_bytes=276885504 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277113840 ingress_bytes=386985984 link_bytes=386985984 token_bytes=553999344 weight_movement_bytes=110100480 total_nvlink_bytes=664099824 weight_to_token_ratio=0.198738 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.730322744e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44426 payload_rows_remote=38828 dup_rows=21110 dup_rows_remote=18394 dedup_ratio=0.322113 router_tokens_per_expert=[497, 482, 533, 515, 500, 514, 525, 538, 576, 500, 511, 512, 481, 520, 462, 521, 480, 504, 549, 511, 500, 523, 519, 525, 534, 487, 539, 502, 525, 519, 526, 525, 527, 532, 542, 547, 461, 526, 539, 466, 488, 517, 545, 530, 478, 515, 476, 446, 490, 515, 536, 496, 536, 529, 489, 500, 521, 501, 537, 515, 512, 509, 485, 507] payload_rows_per_dst=[5560, 5590, 5558, 5598, 5626, 5422, 5592, 5480] dup_rows_per_dst=[2648, 2576, 2664, 2716, 2654, 2568, 2590, 2694] combine_rows_per_src=[5560, 5590, 5558, 5598, 5626, 5422, 5592, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277215232 dispatch_meta_bytes=228968 combine_payload_bytes=277215232 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277444200 ingress_bytes=387315712 link_bytes=387315712 token_bytes=554659432 weight_movement_bytes=110100480 total_nvlink_bytes=664759912 weight_to_token_ratio=0.198501 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.736057257e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44610 payload_rows_remote=38972 dup_rows=20926 dup_rows_remote=18260 dedup_ratio=0.319305 router_tokens_per_expert=[494, 498, 478, 499, 467, 501, 459, 519, 527, 523, 548, 477, 540, 526, 516, 507, 510, 539, 532, 527, 525, 484, 520, 502, 481, 542, 476, 494, 485, 516, 536, 522, 509, 495, 527, 508, 513, 481, 527, 484, 524, 522, 566, 527, 493, 522, 512, 522, 527, 529, 531, 489, 529, 578, 488, 481, 536, 543, 501, 521, 513, 517, 507, 476] payload_rows_per_dst=[5408, 5628, 5592, 5554, 5504, 5674, 5638, 5612] dup_rows_per_dst=[2422, 2700, 2686, 2550, 2584, 2702, 2666, 2616] combine_rows_per_src=[5408, 5628, 5592, 5554, 5504, 5674, 5638, 5612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279422976 dispatch_meta_bytes=229824 combine_payload_bytes=279422976 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279652800 ingress_bytes=499623936 link_bytes=499623936 token_bytes=559075776 weight_movement_bytes=220200960 total_nvlink_bytes=779276736 weight_to_token_ratio=0.393866 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.689281987e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277688320 dispatch_meta_bytes=228648 combine_payload_bytes=277688320 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=277916968 ingress_bytes=607989760 link_bytes=607989760 token_bytes=555605288 weight_movement_bytes=330301440 total_nvlink_bytes=885906728 weight_to_token_ratio=0.594489 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057394190e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279681024 dispatch_meta_bytes=229656 combine_payload_bytes=279681024 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279910680 ingress_bytes=389781504 link_bytes=389781504 token_bytes=559591704 weight_movement_bytes=110100480 total_nvlink_bytes=669692184 weight_to_token_ratio=0.196751 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778941435e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278319104 dispatch_meta_bytes=228888 combine_payload_bytes=278319104 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=278547992 ingress_bytes=718721024 link_bytes=718721024 token_bytes=556867096 weight_movement_bytes=440401920 total_nvlink_bytes=997269016 weight_to_token_ratio=0.790856 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.249974070e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44410 payload_rows_remote=38620 dup_rows=21126 dup_rows_remote=18230 dedup_ratio=0.322357 router_tokens_per_expert=[484, 494, 499, 542, 508, 495, 528, 527, 511, 485, 515, 515, 521, 495, 535, 484, 526, 545, 543, 522, 506, 517, 533, 501, 518, 518, 516, 501, 514, 535, 532, 511, 505, 512, 519, 515, 512, 520, 527, 522, 517, 518, 517, 534, 483, 525, 459, 490, 523, 515, 490, 503, 536, 501, 509, 484, 503, 519, 509, 437, 487, 529, 550, 522] payload_rows_per_dst=[5790, 5510, 5514, 5528, 5578, 5532, 5500, 5458] dup_rows_per_dst=[2896, 2768, 2542, 2576, 2696, 2554, 2622, 2472] combine_rows_per_src=[5790, 5510, 5514, 5528, 5578, 5532, 5500, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279351296 dispatch_meta_bytes=228928 combine_payload_bytes=279351296 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279580224 ingress_bytes=389451776 link_bytes=389451776 token_bytes=558931520 weight_movement_bytes=110100480 total_nvlink_bytes=669032000 weight_to_token_ratio=0.196984 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773206923e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44548 payload_rows_remote=38896 dup_rows=20988 dup_rows_remote=18256 dedup_ratio=0.320251 router_tokens_per_expert=[478, 450, 530, 489, 516, 499, 495, 529, 532, 505, 545, 505, 510, 519, 513, 465, 485, 539, 519, 528, 505, 506, 465, 479, 510, 535, 514, 556, 537, 487, 476, 515, 549, 514, 496, 519, 501, 509, 566, 538, 536, 508, 490, 498, 505, 529, 534, 517, 524, 484, 551, 511, 492, 530, 539, 510, 530, 504, 514, 499, 498, 503, 540, 494] payload_rows_per_dst=[5470, 5604, 5530, 5504, 5652, 5634, 5642, 5512] dup_rows_per_dst=[2502, 2584, 2522, 2756, 2732, 2600, 2640, 2652] combine_rows_per_src=[5470, 5604, 5530, 5504, 5652, 5634, 5642, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44690 payload_rows_remote=39084 dup_rows=20846 dup_rows_remote=18146 dedup_ratio=0.318085 router_tokens_per_expert=[508, 551, 533, 500, 519, 500, 526, 508, 532, 500, 487, 547, 521, 500, 517, 525, 541, 492, 497, 545, 540, 490, 485, 494, 503, 527, 468, 498, 498, 486, 488, 528, 497, 522, 513, 536, 530, 503, 510, 541, 500, 513, 540, 529, 529, 498, 502, 542, 530, 505, 513, 497, 503, 466, 523, 528, 477, 488, 481, 543, 511, 506, 515, 523] payload_rows_per_dst=[5684, 5574, 5526, 5488, 5628, 5606, 5580, 5604] dup_rows_per_dst=[2606, 2684, 2642, 2504, 2676, 2700, 2550, 2484] combine_rows_per_src=[5684, 5574, 5526, 5488, 5628, 5606, 5580, 5604] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=276828160 dispatch_meta_bytes=227400 combine_payload_bytes=276828160 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277055560 ingress_bytes=386928640 link_bytes=386928640 token_bytes=553883720 weight_movement_bytes=110100480 total_nvlink_bytes=663984200 weight_to_token_ratio=0.198779 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.729325438e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278806528 dispatch_meta_bytes=228608 combine_payload_bytes=278806528 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279035136 ingress_bytes=499007488 link_bytes=499007488 token_bytes=557841664 weight_movement_bytes=220200960 total_nvlink_bytes=778042624 weight_to_token_ratio=0.394737 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.678560943e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44610 payload_rows_remote=39046 dup_rows=20926 dup_rows_remote=18372 dedup_ratio=0.319305 router_tokens_per_expert=[491, 520, 527, 527, 498, 468, 492, 516, 530, 500, 522, 518, 482, 550, 540, 513, 501, 530, 494, 493, 508, 519, 494, 520, 476, 486, 515, 487, 534, 487, 526, 489, 521, 533, 519, 503, 513, 524, 485, 522, 520, 551, 537, 481, 520, 519, 486, 488, 502, 513, 499, 531, 537, 541, 516, 548, 516, 516, 547, 484, 519, 502, 540, 482] payload_rows_per_dst=[5486, 5600, 5564, 5510, 5596, 5574, 5670, 5610] dup_rows_per_dst=[2592, 2710, 2554, 2490, 2644, 2630, 2704, 2602] combine_rows_per_src=[5486, 5600, 5564, 5510, 5596, 5574, 5670, 5610] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44368 payload_rows_remote=38768 dup_rows=21168 dup_rows_remote=18494 dedup_ratio=0.322998 router_tokens_per_expert=[495, 525, 494, 489, 509, 521, 513, 537, 482, 537, 527, 530, 502, 535, 493, 531, 484, 487, 527, 532, 522, 518, 529, 519, 493, 517, 502, 498, 529, 487, 522, 538, 524, 511, 508, 507, 508, 500, 497, 476, 541, 549, 512, 501, 492, 537, 500, 527, 449, 543, 513, 514, 468, 524, 494, 524, 516, 505, 501, 491, 553, 532, 496, 531] payload_rows_per_dst=[5530, 5600, 5528, 5568, 5522, 5602, 5440, 5578] dup_rows_per_dst=[2636, 2674, 2708, 2604, 2540, 2716, 2618, 2672] combine_rows_per_src=[5530, 5600, 5528, 5568, 5522, 5602, 5440, 5578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44318 payload_rows_remote=38802 dup_rows=21218 dup_rows_remote=18624 dedup_ratio=0.323761 router_tokens_per_expert=[525, 498, 504, 528, 500, 501, 529, 476, 505, 539, 515, 540, 502, 494, 510, 492, 509, 508, 515, 516, 496, 483, 517, 538, 506, 511, 476, 521, 547, 535, 551, 517, 519, 492, 522, 475, 548, 472, 539, 520, 523, 508, 546, 523, 489, 528, 480, 497, 526, 493, 504, 551, 490, 525, 502, 546, 485, 520, 514, 513, 499, 504, 503, 508] payload_rows_per_dst=[5496, 5438, 5520, 5628, 5532, 5568, 5620, 5516] dup_rows_per_dst=[2588, 2526, 2644, 2878, 2642, 2602, 2744, 2594] combine_rows_per_src=[5496, 5438, 5520, 5628, 5532, 5568, 5620, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44366 payload_rows_remote=38776 dup_rows=21170 dup_rows_remote=18504 dedup_ratio=0.323029 router_tokens_per_expert=[540, 506, 516, 542, 522, 526, 519, 515, 510, 497, 519, 493, 453, 535, 526, 509, 501, 450, 533, 514, 499, 502, 516, 502, 518, 522, 512, 486, 537, 529, 489, 535, 515, 509, 517, 501, 482, 533, 566, 480, 514, 514, 540, 477, 501, 516, 484, 543, 513, 493, 494, 526, 514, 523, 542, 475, 524, 497, 513, 482, 560, 531, 507, 509] payload_rows_per_dst=[5586, 5602, 5526, 5590, 5516, 5482, 5520, 5544] dup_rows_per_dst=[2786, 2482, 2508, 2666, 2690, 2696, 2640, 2702] combine_rows_per_src=[5586, 5602, 5526, 5590, 5516, 5482, 5520, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=280154112 dispatch_meta_bytes=228920 combine_payload_bytes=280154112 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=280383032 ingress_bytes=500355072 link_bytes=500355072 token_bytes=560537144 weight_movement_bytes=220200960 total_nvlink_bytes=780738104 weight_to_token_ratio=0.392839 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.701997645e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277057536 dispatch_meta_bytes=228576 combine_payload_bytes=277057536 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277286112 ingress_bytes=387158016 link_bytes=387158016 token_bytes=554343648 weight_movement_bytes=110100480 total_nvlink_bytes=664444128 weight_to_token_ratio=0.198614 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.733314664e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44270 payload_rows_remote=38844 dup_rows=21266 dup_rows_remote=18674 dedup_ratio=0.324493 router_tokens_per_expert=[524, 528, 509, 525, 505, 534, 524, 503, 481, 491, 566, 502, 530, 498, 560, 521, 546, 493, 514, 532, 497, 507, 537, 504, 483, 527, 527, 507, 503, 509, 508, 486, 536, 477, 512, 493, 482, 538, 504, 530, 523, 523, 509, 510, 550, 518, 482, 483, 497, 501, 518, 487, 486, 477, 544, 499, 495, 555, 465, 527, 497, 512, 516, 541] payload_rows_per_dst=[5576, 5600, 5588, 5522, 5468, 5528, 5426, 5562] dup_rows_per_dst=[2728, 2698, 2672, 2578, 2676, 2668, 2592, 2654] combine_rows_per_src=[5576, 5600, 5588, 5522, 5468, 5528, 5426, 5562] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277889024 dispatch_meta_bytes=229104 combine_payload_bytes=277889024 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278118128 ingress_bytes=387989504 link_bytes=387989504 token_bytes=556007152 weight_movement_bytes=110100480 total_nvlink_bytes=666107632 weight_to_token_ratio=0.198020 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.747775608e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277358592 dispatch_meta_bytes=228256 combine_payload_bytes=277358592 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=277586848 ingress_bytes=607660032 link_bytes=607660032 token_bytes=554945440 weight_movement_bytes=330301440 total_nvlink_bytes=885246880 weight_to_token_ratio=0.595196 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.056820739e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44680 payload_rows_remote=39076 dup_rows=20856 dup_rows_remote=18142 dedup_ratio=0.318237 router_tokens_per_expert=[503, 538, 537, 492, 493, 524, 500, 513, 510, 494, 548, 539, 549, 509, 501, 492, 535, 493, 510, 470, 506, 509, 498, 493, 539, 487, 528, 513, 500, 489, 510, 556, 504, 482, 495, 522, 517, 540, 520, 523, 561, 516, 531, 511, 521, 504, 516, 499, 535, 476, 491, 531, 508, 505, 500, 495, 531, 480, 522, 517, 510, 512, 491, 524] payload_rows_per_dst=[5604, 5678, 5482, 5688, 5556, 5604, 5502, 5566] dup_rows_per_dst=[2596, 2606, 2546, 2556, 2650, 2714, 2580, 2608] combine_rows_per_src=[5604, 5678, 5482, 5688, 5556, 5604, 5502, 5566] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277817344 dispatch_meta_bytes=229008 combine_payload_bytes=277817344 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278046352 ingress_bytes=608118784 link_bytes=608118784 token_bytes=555863696 weight_movement_bytes=330301440 total_nvlink_bytes=886165136 weight_to_token_ratio=0.594213 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057618584e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44434 payload_rows_remote=38956 dup_rows=21102 dup_rows_remote=18404 dedup_ratio=0.321991 router_tokens_per_expert=[518, 490, 517, 479, 533, 513, 493, 541, 539, 507, 497, 502, 501, 524, 543, 483, 535, 490, 498, 484, 476, 504, 573, 531, 521, 526, 517, 508, 511, 550, 493, 558, 511, 501, 484, 533, 524, 497, 514, 544, 503, 486, 482, 461, 519, 522, 491, 482, 492, 504, 497, 501, 534, 491, 555, 514, 510, 562, 509, 527, 524, 529, 491, 519] payload_rows_per_dst=[5562, 5504, 5558, 5634, 5646, 5442, 5478, 5610] dup_rows_per_dst=[2606, 2688, 2624, 2734, 2570, 2450, 2698, 2732] combine_rows_per_src=[5562, 5504, 5558, 5634, 5646, 5442, 5478, 5610] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279881728 dispatch_meta_bytes=229672 combine_payload_bytes=279881728 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280111400 ingress_bytes=389982208 link_bytes=389982208 token_bytes=559993128 weight_movement_bytes=110100480 total_nvlink_bytes=670093608 weight_to_token_ratio=0.196610 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.782432008e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277889024 dispatch_meta_bytes=229048 combine_payload_bytes=277889024 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278118072 ingress_bytes=608190464 link_bytes=608190464 token_bytes=556007096 weight_movement_bytes=330301440 total_nvlink_bytes=886308536 weight_to_token_ratio=0.594060 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057743247e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278132736 dispatch_meta_bytes=229704 combine_payload_bytes=278132736 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278362440 ingress_bytes=388233216 link_bytes=388233216 token_bytes=556495176 weight_movement_bytes=110100480 total_nvlink_bytes=666595656 weight_to_token_ratio=0.197846 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.752014160e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277946368 dispatch_meta_bytes=229120 combine_payload_bytes=277946368 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278175488 ingress_bytes=388046848 link_bytes=388046848 token_bytes=556121856 weight_movement_bytes=110100480 total_nvlink_bytes=666222336 weight_to_token_ratio=0.197979 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.748772914e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44400 payload_rows_remote=38718 dup_rows=21136 dup_rows_remote=18412 dedup_ratio=0.322510 router_tokens_per_expert=[493, 546, 562, 533, 521, 474, 549, 520, 530, 516, 506, 528, 521, 485, 486, 561, 495, 535, 518, 499, 525, 468, 517, 508, 498, 457, 482, 518, 517, 511, 492, 516, 518, 547, 514, 535, 487, 510, 508, 508, 518, 536, 518, 536, 507, 511, 498, 498, 504, 497, 512, 546, 498, 480, 511, 522, 537, 472, 492, 494, 506, 512, 532, 517] payload_rows_per_dst=[5682, 5558, 5732, 5406, 5444, 5562, 5486, 5530] dup_rows_per_dst=[2724, 2708, 2850, 2576, 2506, 2672, 2506, 2594] combine_rows_per_src=[5682, 5558, 5732, 5406, 5444, 5562, 5486, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44324 payload_rows_remote=38776 dup_rows=21212 dup_rows_remote=18464 dedup_ratio=0.323669 router_tokens_per_expert=[498, 482, 522, 516, 510, 541, 500, 496, 540, 528, 501, 505, 509, 528, 535, 494, 502, 521, 538, 480, 503, 543, 518, 537, 470, 523, 524, 508, 516, 498, 488, 522, 496, 493, 471, 558, 550, 524, 545, 511, 485, 512, 511, 513, 508, 486, 488, 493, 482, 508, 524, 477, 527, 536, 513, 503, 456, 548, 523, 513, 540, 515, 534, 529] payload_rows_per_dst=[5536, 5604, 5612, 5486, 5548, 5404, 5544, 5590] dup_rows_per_dst=[2594, 2676, 2672, 2612, 2748, 2588, 2596, 2726] combine_rows_per_src=[5536, 5604, 5612, 5486, 5548, 5404, 5544, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279236608 dispatch_meta_bytes=229440 combine_payload_bytes=279236608 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279466048 ingress_bytes=499437568 link_bytes=499437568 token_bytes=558702656 weight_movement_bytes=220200960 total_nvlink_bytes=778903616 weight_to_token_ratio=0.394129 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.686040741e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44704 payload_rows_remote=39104 dup_rows=20832 dup_rows_remote=18110 dedup_ratio=0.317871 router_tokens_per_expert=[502, 536, 535, 487, 492, 522, 497, 510, 508, 494, 546, 535, 546, 505, 496, 491, 533, 492, 510, 469, 505, 506, 496, 495, 540, 488, 528, 513, 500, 491, 512, 555, 505, 483, 496, 522, 517, 542, 523, 523, 558, 517, 529, 511, 523, 505, 517, 501, 541, 480, 491, 531, 509, 506, 501, 496, 532, 482, 525, 520, 514, 513, 494, 526] payload_rows_per_dst=[5592, 5664, 5484, 5692, 5570, 5600, 5514, 5588] dup_rows_per_dst=[2570, 2578, 2528, 2562, 2652, 2722, 2596, 2624] combine_rows_per_src=[5592, 5664, 5484, 5692, 5570, 5600, 5514, 5588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277530624 dispatch_meta_bytes=228520 combine_payload_bytes=277530624 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277759144 ingress_bytes=497731584 link_bytes=497731584 token_bytes=555289768 weight_movement_bytes=220200960 total_nvlink_bytes=775490728 weight_to_token_ratio=0.396551 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.656370874e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277946368 dispatch_meta_bytes=228960 combine_payload_bytes=277946368 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278175328 ingress_bytes=388046848 link_bytes=388046848 token_bytes=556121696 weight_movement_bytes=110100480 total_nvlink_bytes=666222176 weight_to_token_ratio=0.197979 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.748772914e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44390 payload_rows_remote=38780 dup_rows=21146 dup_rows_remote=18404 dedup_ratio=0.322662 router_tokens_per_expert=[530, 521, 494, 513, 483, 541, 523, 515, 535, 533, 487, 496, 497, 513, 507, 512, 547, 504, 512, 524, 553, 489, 566, 481, 520, 467, 539, 504, 516, 498, 528, 511, 488, 548, 561, 503, 487, 484, 513, 491, 519, 521, 511, 520, 506, 493, 477, 515, 500, 522, 514, 513, 550, 503, 528, 485, 483, 520, 520, 486, 523, 540, 475, 510] payload_rows_per_dst=[5624, 5478, 5610, 5588, 5508, 5506, 5518, 5558] dup_rows_per_dst=[2616, 2682, 2742, 2578, 2642, 2618, 2712, 2556] combine_rows_per_src=[5624, 5478, 5610, 5588, 5508, 5506, 5518, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44376 payload_rows_remote=38826 dup_rows=21160 dup_rows_remote=18542 dedup_ratio=0.322876 router_tokens_per_expert=[510, 511, 515, 536, 515, 502, 534, 515, 523, 477, 535, 523, 487, 510, 513, 516, 519, 537, 525, 491, 514, 482, 555, 553, 502, 522, 531, 505, 515, 511, 499, 516, 529, 492, 449, 508, 462, 527, 526, 509, 527, 459, 481, 525, 511, 523, 535, 516, 499, 528, 493, 504, 522, 509, 553, 528, 533, 512, 511, 489, 533, 487, 507, 482] payload_rows_per_dst=[5540, 5550, 5640, 5552, 5490, 5556, 5546, 5502] dup_rows_per_dst=[2736, 2618, 2712, 2650, 2514, 2598, 2726, 2606] combine_rows_per_src=[5540, 5550, 5640, 5552, 5490, 5556, 5546, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44358 payload_rows_remote=38766 dup_rows=21178 dup_rows_remote=18386 dedup_ratio=0.323151 router_tokens_per_expert=[490, 532, 531, 504, 552, 498, 514, 523, 529, 523, 507, 522, 518, 522, 538, 486, 524, 547, 521, 484, 517, 516, 537, 537, 524, 480, 514, 483, 509, 493, 473, 490, 528, 505, 471, 514, 463, 525, 490, 452, 503, 563, 545, 483, 496, 508, 513, 495, 499, 521, 522, 528, 507, 495, 489, 523, 509, 529, 488, 568, 535, 509, 528, 526] payload_rows_per_dst=[5400, 5292, 5522, 5720, 5578, 5590, 5664, 5592] dup_rows_per_dst=[2388, 2368, 2576, 2842, 2818, 2622, 2772, 2792] combine_rows_per_src=[5400, 5292, 5522, 5720, 5578, 5590, 5664, 5592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44500 payload_rows_remote=39030 dup_rows=21036 dup_rows_remote=18490 dedup_ratio=0.320984 router_tokens_per_expert=[508, 504, 547, 521, 526, 508, 535, 540, 487, 524, 538, 532, 493, 525, 579, 555, 518, 449, 498, 540, 540, 499, 523, 540, 502, 474, 498, 478, 545, 518, 489, 504, 537, 505, 485, 514, 486, 494, 536, 507, 520, 506, 503, 523, 517, 546, 483, 510, 470, 507, 542, 515, 499, 514, 530, 500, 499, 475, 512, 483, 504, 514, 502, 493] payload_rows_per_dst=[5674, 5664, 5622, 5470, 5524, 5556, 5540, 5450] dup_rows_per_dst=[2704, 2802, 2592, 2546, 2604, 2660, 2614, 2514] combine_rows_per_src=[5674, 5664, 5622, 5470, 5524, 5556, 5540, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280297472 dispatch_meta_bytes=228856 combine_payload_bytes=280297472 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280526328 ingress_bytes=500498432 link_bytes=500498432 token_bytes=560823800 weight_movement_bytes=220200960 total_nvlink_bytes=781024760 weight_to_token_ratio=0.392638 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.704490911e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44288 payload_rows_remote=38844 dup_rows=21248 dup_rows_remote=18634 dedup_ratio=0.324219 router_tokens_per_expert=[520, 526, 506, 523, 503, 531, 521, 496, 477, 485, 561, 499, 526, 498, 558, 520, 545, 490, 512, 528, 496, 508, 535, 503, 481, 527, 524, 506, 504, 510, 509, 483, 539, 478, 511, 500, 481, 541, 507, 531, 525, 523, 510, 514, 551, 520, 481, 487, 499, 503, 518, 489, 490, 479, 547, 504, 499, 557, 470, 530, 498, 517, 517, 541] payload_rows_per_dst=[5560, 5584, 5582, 5510, 5488, 5536, 5444, 5584] dup_rows_per_dst=[2692, 2664, 2652, 2578, 2688, 2686, 2614, 2674] combine_rows_per_src=[5560, 5584, 5582, 5510, 5488, 5536, 5444, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277975040 dispatch_meta_bytes=228736 combine_payload_bytes=277975040 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278203776 ingress_bytes=608276480 link_bytes=608276480 token_bytes=556178816 weight_movement_bytes=330301440 total_nvlink_bytes=886480256 weight_to_token_ratio=0.593876 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057892843e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278304768 dispatch_meta_bytes=229472 combine_payload_bytes=278304768 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278534240 ingress_bytes=498505728 link_bytes=498505728 token_bytes=556839008 weight_movement_bytes=220200960 total_nvlink_bytes=777039968 weight_to_token_ratio=0.395448 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.669834511e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277874688 dispatch_meta_bytes=228608 combine_payload_bytes=277874688 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=278103296 ingress_bytes=498075648 link_bytes=498075648 token_bytes=555977984 weight_movement_bytes=220200960 total_nvlink_bytes=776178944 weight_to_token_ratio=0.396061 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.662354712e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44294 payload_rows_remote=38744 dup_rows=21242 dup_rows_remote=18492 dedup_ratio=0.324127 router_tokens_per_expert=[503, 486, 523, 518, 512, 545, 503, 501, 541, 530, 505, 509, 506, 528, 536, 493, 503, 523, 537, 483, 503, 542, 522, 537, 471, 522, 524, 506, 513, 499, 489, 520, 497, 494, 471, 557, 548, 526, 548, 509, 486, 509, 513, 510, 507, 484, 486, 490, 480, 506, 523, 475, 526, 535, 510, 501, 454, 545, 519, 512, 538, 515, 532, 529] payload_rows_per_dst=[5560, 5606, 5608, 5470, 5550, 5394, 5534, 5572] dup_rows_per_dst=[2622, 2690, 2692, 2618, 2750, 2576, 2578, 2716] combine_rows_per_src=[5560, 5606, 5608, 5470, 5550, 5394, 5534, 5572] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278433792 dispatch_meta_bytes=230072 combine_payload_bytes=278433792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278663864 ingress_bytes=388534272 link_bytes=388534272 token_bytes=557097656 weight_movement_bytes=110100480 total_nvlink_bytes=667198136 weight_to_token_ratio=0.197632 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.757250019e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44388 payload_rows_remote=38762 dup_rows=21148 dup_rows_remote=18398 dedup_ratio=0.322693 router_tokens_per_expert=[535, 526, 496, 514, 490, 545, 527, 519, 536, 535, 489, 500, 498, 514, 509, 512, 549, 505, 514, 524, 555, 492, 566, 483, 518, 466, 542, 506, 516, 499, 524, 510, 488, 550, 558, 504, 487, 485, 511, 490, 518, 520, 510, 518, 502, 493, 472, 512, 497, 521, 513, 510, 547, 499, 527, 483, 481, 520, 519, 485, 520, 537, 471, 506] payload_rows_per_dst=[5640, 5482, 5626, 5588, 5508, 5496, 5506, 5542] dup_rows_per_dst=[2664, 2704, 2750, 2574, 2638, 2594, 2688, 2536] combine_rows_per_src=[5640, 5482, 5626, 5588, 5508, 5496, 5506, 5542] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280096768 dispatch_meta_bytes=228872 combine_payload_bytes=280096768 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280325640 ingress_bytes=390197248 link_bytes=390197248 token_bytes=560422408 weight_movement_bytes=110100480 total_nvlink_bytes=670522888 weight_to_token_ratio=0.196460 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.786171907e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277716992 dispatch_meta_bytes=228944 combine_payload_bytes=277716992 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277945936 ingress_bytes=387817472 link_bytes=387817472 token_bytes=555662928 weight_movement_bytes=110100480 total_nvlink_bytes=665763408 weight_to_token_ratio=0.198143 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.744783688e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44276 payload_rows_remote=38596 dup_rows=21260 dup_rows_remote=18350 dedup_ratio=0.324402 router_tokens_per_expert=[492, 535, 533, 504, 557, 502, 517, 525, 529, 526, 508, 523, 519, 523, 538, 489, 527, 547, 519, 485, 517, 518, 540, 538, 525, 482, 514, 485, 509, 491, 476, 489, 530, 503, 472, 514, 464, 526, 491, 452, 503, 561, 544, 481, 493, 509, 508, 492, 495, 518, 520, 528, 506, 495, 485, 523, 503, 525, 487, 567, 534, 507, 527, 523] payload_rows_per_dst=[5292, 5252, 5430, 5696, 5566, 5628, 5732, 5680] dup_rows_per_dst=[2184, 2338, 2480, 2856, 2904, 2672, 2916, 2910] combine_rows_per_src=[5292, 5252, 5430, 5696, 5566, 5628, 5732, 5680] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44498 payload_rows_remote=39030 dup_rows=21038 dup_rows_remote=18496 dedup_ratio=0.321014 router_tokens_per_expert=[512, 507, 547, 524, 526, 508, 538, 540, 490, 526, 544, 532, 495, 524, 582, 556, 519, 450, 499, 542, 542, 500, 522, 542, 503, 476, 496, 478, 545, 518, 487, 502, 538, 505, 484, 514, 487, 495, 538, 503, 517, 505, 503, 521, 517, 548, 481, 507, 470, 505, 541, 514, 496, 509, 532, 500, 497, 471, 511, 479, 503, 514, 501, 490] payload_rows_per_dst=[5688, 5680, 5628, 5468, 5522, 5544, 5532, 5436] dup_rows_per_dst=[2716, 2818, 2604, 2542, 2606, 2654, 2602, 2496] combine_rows_per_src=[5688, 5680, 5628, 5468, 5522, 5544, 5532, 5436] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277846016 dispatch_meta_bytes=228640 combine_payload_bytes=277846016 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278074656 ingress_bytes=608147456 link_bytes=608147456 token_bytes=555920672 weight_movement_bytes=330301440 total_nvlink_bytes=886222112 weight_to_token_ratio=0.594152 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057668449e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44242 payload_rows_remote=38656 dup_rows=21294 dup_rows_remote=18534 dedup_ratio=0.324921 router_tokens_per_expert=[514, 539, 506, 487, 539, 505, 503, 539, 527, 512, 488, 531, 511, 504, 520, 503, 532, 503, 496, 533, 515, 521, 490, 480, 496, 484, 529, 483, 543, 531, 500, 513, 519, 479, 540, 508, 542, 494, 499, 496, 490, 542, 534, 544, 507, 506, 524, 520, 537, 508, 456, 534, 521, 504, 484, 509, 528, 527, 515, 478, 510, 526, 518, 492] payload_rows_per_dst=[5586, 5484, 5490, 5478, 5574, 5604, 5518, 5508] dup_rows_per_dst=[2760, 2708, 2568, 2680, 2580, 2730, 2588, 2680] combine_rows_per_src=[5586, 5484, 5490, 5478, 5574, 5604, 5518, 5508] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44388 payload_rows_remote=38830 dup_rows=21148 dup_rows_remote=18514 dedup_ratio=0.322693 router_tokens_per_expert=[513, 515, 517, 538, 518, 502, 540, 517, 523, 479, 539, 524, 488, 511, 514, 518, 522, 538, 526, 492, 514, 483, 556, 555, 505, 522, 532, 511, 516, 510, 499, 516, 527, 492, 451, 507, 463, 524, 522, 508, 527, 456, 480, 524, 511, 523, 531, 513, 497, 524, 491, 503, 520, 508, 550, 528, 527, 512, 509, 483, 530, 485, 507, 482] payload_rows_per_dst=[5558, 5558, 5652, 5562, 5490, 5542, 5536, 5490] dup_rows_per_dst=[2762, 2634, 2720, 2660, 2498, 2588, 2706, 2580] combine_rows_per_src=[5558, 5558, 5652, 5562, 5490, 5542, 5536, 5490] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=276656128 dispatch_meta_bytes=227784 combine_payload_bytes=276656128 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276883912 ingress_bytes=386756608 link_bytes=386756608 token_bytes=553540040 weight_movement_bytes=110100480 total_nvlink_bytes=663640520 weight_to_token_ratio=0.198902 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.726333518e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44574 payload_rows_remote=39106 dup_rows=20962 dup_rows_remote=18358 dedup_ratio=0.319855 router_tokens_per_expert=[529, 524, 483, 512, 510, 476, 525, 557, 511, 497, 518, 521, 551, 498, 564, 500, 507, 507, 497, 509, 486, 522, 510, 554, 501, 505, 477, 513, 525, 505, 522, 505, 497, 511, 501, 513, 508, 522, 453, 506, 517, 537, 526, 533, 518, 540, 557, 513, 479, 517, 539, 501, 485, 494, 482, 539, 527, 487, 501, 522, 505, 485, 513, 519] payload_rows_per_dst=[5560, 5626, 5574, 5582, 5502, 5704, 5468, 5558] dup_rows_per_dst=[2672, 2694, 2610, 2524, 2520, 2778, 2604, 2560] combine_rows_per_src=[5560, 5626, 5574, 5582, 5502, 5704, 5468, 5558] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279767040 dispatch_meta_bytes=230104 combine_payload_bytes=279767040 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279997144 ingress_bytes=389867520 link_bytes=389867520 token_bytes=559764184 weight_movement_bytes=110100480 total_nvlink_bytes=669864664 weight_to_token_ratio=0.196691 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.780437395e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44546 payload_rows_remote=38998 dup_rows=20990 dup_rows_remote=18332 dedup_ratio=0.320282 router_tokens_per_expert=[519, 525, 521, 558, 475, 495, 546, 528, 490, 508, 552, 487, 485, 512, 477, 524, 520, 495, 559, 498, 487, 499, 504, 465, 530, 505, 493, 505, 523, 500, 511, 496, 513, 534, 545, 520, 542, 531, 508, 520, 541, 484, 493, 557, 525, 502, 492, 509, 530, 528, 497, 487, 495, 535, 538, 500, 512, 514, 507, 537, 491, 489, 506, 494] payload_rows_per_dst=[5698, 5522, 5544, 5534, 5598, 5548, 5588, 5514] dup_rows_per_dst=[2636, 2548, 2510, 2592, 2828, 2658, 2632, 2586] combine_rows_per_src=[5698, 5522, 5544, 5534, 5598, 5548, 5588, 5514] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277086208 dispatch_meta_bytes=228760 combine_payload_bytes=277086208 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=277314968 ingress_bytes=607387648 link_bytes=607387648 token_bytes=554401176 weight_movement_bytes=330301440 total_nvlink_bytes=884702616 weight_to_token_ratio=0.595781 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.056347018e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279767040 dispatch_meta_bytes=230080 combine_payload_bytes=279767040 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279997120 ingress_bytes=389867520 link_bytes=389867520 token_bytes=559764160 weight_movement_bytes=110100480 total_nvlink_bytes=669864640 weight_to_token_ratio=0.196691 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.780437395e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44444 payload_rows_remote=38850 dup_rows=21092 dup_rows_remote=18494 dedup_ratio=0.321838 router_tokens_per_expert=[505, 505, 505, 504, 558, 542, 509, 562, 513, 500, 491, 486, 492, 511, 492, 481, 477, 531, 548, 515, 506, 487, 487, 512, 517, 538, 522, 504, 484, 520, 483, 524, 511, 528, 525, 505, 533, 517, 516, 461, 493, 523, 499, 529, 537, 528, 541, 472, 497, 498, 503, 468, 506, 498, 527, 529, 552, 555, 472, 545, 538, 516, 500, 535] payload_rows_per_dst=[5616, 5448, 5554, 5478, 5594, 5616, 5446, 5692] dup_rows_per_dst=[2764, 2484, 2572, 2706, 2598, 2628, 2606, 2734] combine_rows_per_src=[5616, 5448, 5554, 5478, 5594, 5616, 5446, 5692] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44240 payload_rows_remote=38662 dup_rows=21296 dup_rows_remote=18564 dedup_ratio=0.324951 router_tokens_per_expert=[512, 535, 505, 483, 536, 504, 498, 535, 528, 514, 487, 528, 511, 502, 520, 501, 534, 500, 496, 532, 511, 519, 488, 479, 493, 485, 529, 485, 541, 530, 499, 513, 520, 479, 541, 507, 546, 495, 499, 495, 491, 543, 533, 545, 507, 506, 525, 522, 537, 510, 457, 534, 524, 507, 487, 512, 533, 528, 516, 479, 515, 528, 520, 494] payload_rows_per_dst=[5578, 5534, 5472, 5470, 5574, 5576, 5532, 5504] dup_rows_per_dst=[2732, 2778, 2518, 2680, 2590, 2718, 2604, 2676] combine_rows_per_src=[5578, 5534, 5472, 5470, 5574, 5576, 5532, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278433792 dispatch_meta_bytes=229912 combine_payload_bytes=278433792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278663704 ingress_bytes=388534272 link_bytes=388534272 token_bytes=557097496 weight_movement_bytes=110100480 total_nvlink_bytes=667197976 weight_to_token_ratio=0.197632 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.757250019e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44544 payload_rows_remote=38992 dup_rows=20992 dup_rows_remote=18316 dedup_ratio=0.320312 router_tokens_per_expert=[517, 519, 519, 556, 471, 495, 543, 527, 486, 506, 550, 487, 484, 514, 475, 522, 517, 490, 557, 497, 486, 503, 504, 466, 529, 505, 492, 502, 526, 500, 513, 496, 514, 534, 544, 517, 543, 531, 505, 523, 543, 486, 495, 558, 527, 502, 493, 510, 529, 531, 499, 487, 497, 537, 544, 499, 514, 517, 511, 537, 491, 492, 507, 497] payload_rows_per_dst=[5684, 5508, 5532, 5538, 5594, 5552, 5606, 5530] dup_rows_per_dst=[2610, 2540, 2508, 2588, 2828, 2676, 2640, 2602] combine_rows_per_src=[5684, 5508, 5532, 5538, 5594, 5552, 5606, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278476800 dispatch_meta_bytes=229376 combine_payload_bytes=278476800 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278706176 ingress_bytes=388577280 link_bytes=388577280 token_bytes=557182976 weight_movement_bytes=110100480 total_nvlink_bytes=667283456 weight_to_token_ratio=0.197602 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.757997999e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277129216 dispatch_meta_bytes=228904 combine_payload_bytes=277129216 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277358120 ingress_bytes=497330176 link_bytes=497330176 token_bytes=554487336 weight_movement_bytes=220200960 total_nvlink_bytes=774688296 weight_to_token_ratio=0.397125 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.649389728e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44618 payload_rows_remote=39000 dup_rows=20918 dup_rows_remote=18236 dedup_ratio=0.319183 router_tokens_per_expert=[487, 512, 487, 532, 503, 486, 513, 511, 509, 521, 481, 511, 506, 529, 516, 528, 536, 546, 504, 521, 525, 490, 515, 513, 513, 505, 489, 511, 495, 473, 518, 537, 513, 475, 495, 495, 487, 512, 533, 516, 512, 515, 520, 519, 465, 527, 527, 507, 530, 526, 514, 482, 539, 540, 511, 529, 531, 491, 502, 525, 535, 539, 530, 503] payload_rows_per_dst=[5534, 5562, 5618, 5546, 5506, 5584, 5656, 5612] dup_rows_per_dst=[2528, 2640, 2682, 2536, 2546, 2600, 2686, 2700] combine_rows_per_src=[5534, 5562, 5618, 5546, 5506, 5584, 5656, 5612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44360 payload_rows_remote=38780 dup_rows=21176 dup_rows_remote=18414 dedup_ratio=0.323120 router_tokens_per_expert=[455, 535, 490, 541, 515, 525, 516, 503, 516, 510, 515, 525, 548, 513, 516, 528, 537, 542, 490, 490, 514, 511, 519, 452, 543, 504, 518, 505, 543, 537, 517, 523, 517, 498, 509, 489, 476, 504, 534, 471, 534, 506, 517, 501, 528, 535, 505, 543, 531, 530, 500, 511, 506, 477, 539, 503, 526, 468, 508, 491, 492, 480, 509, 534] payload_rows_per_dst=[5536, 5580, 5560, 5700, 5454, 5558, 5508, 5464] dup_rows_per_dst=[2624, 2762, 2550, 2680, 2542, 2780, 2686, 2552] combine_rows_per_src=[5536, 5580, 5560, 5700, 5454, 5558, 5508, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44720 payload_rows_remote=39142 dup_rows=20816 dup_rows_remote=18266 dedup_ratio=0.317627 router_tokens_per_expert=[511, 552, 532, 489, 514, 521, 488, 513, 506, 525, 542, 494, 489, 514, 514, 511, 547, 504, 524, 542, 544, 499, 484, 516, 516, 483, 486, 529, 511, 499, 521, 476, 522, 496, 526, 515, 500, 517, 531, 537, 498, 504, 522, 517, 507, 512, 511, 467, 537, 494, 512, 523, 502, 519, 582, 519, 527, 490, 498, 468, 513, 502, 511, 493] payload_rows_per_dst=[5438, 5588, 5688, 5594, 5602, 5518, 5714, 5578] dup_rows_per_dst=[2430, 2602, 2768, 2652, 2686, 2436, 2692, 2550] combine_rows_per_src=[5438, 5588, 5688, 5594, 5602, 5518, 5714, 5578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44450 payload_rows_remote=38896 dup_rows=21086 dup_rows_remote=18468 dedup_ratio=0.321747 router_tokens_per_expert=[515, 486, 525, 565, 511, 504, 505, 553, 543, 538, 512, 513, 510, 501, 489, 523, 548, 507, 486, 514, 516, 477, 517, 530, 525, 528, 509, 539, 512, 472, 500, 501, 529, 481, 540, 511, 511, 542, 503, 519, 483, 492, 521, 508, 513, 476, 495, 555, 498, 484, 518, 518, 480, 487, 516, 516, 540, 515, 502, 478, 546, 520, 527, 470] payload_rows_per_dst=[5636, 5518, 5576, 5554, 5590, 5512, 5480, 5584] dup_rows_per_dst=[2692, 2740, 2614, 2618, 2682, 2574, 2554, 2612] combine_rows_per_src=[5636, 5518, 5576, 5554, 5590, 5512, 5480, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279494656 dispatch_meta_bytes=229232 combine_payload_bytes=279494656 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=279723888 ingress_bytes=719896576 link_bytes=719896576 token_bytes=559218544 weight_movement_bytes=440401920 total_nvlink_bytes=999620464 weight_to_token_ratio=0.787531 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.252018548e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44606 payload_rows_remote=39116 dup_rows=20930 dup_rows_remote=18320 dedup_ratio=0.319366 router_tokens_per_expert=[526, 521, 481, 510, 508, 475, 520, 552, 509, 491, 517, 521, 544, 496, 563, 497, 506, 507, 497, 508, 486, 520, 511, 553, 504, 506, 475, 510, 528, 504, 520, 506, 500, 513, 501, 514, 508, 522, 455, 507, 518, 535, 529, 532, 517, 540, 556, 517, 480, 518, 540, 502, 484, 497, 488, 541, 529, 488, 506, 524, 510, 490, 515, 520] payload_rows_per_dst=[5536, 5612, 5572, 5584, 5516, 5710, 5490, 5586] dup_rows_per_dst=[2650, 2664, 2604, 2522, 2524, 2778, 2610, 2578] combine_rows_per_src=[5536, 5612, 5572, 5584, 5516, 5710, 5490, 5586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279552000 dispatch_meta_bytes=228944 combine_payload_bytes=279552000 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279780944 ingress_bytes=389652480 link_bytes=389652480 token_bytes=559332944 weight_movement_bytes=110100480 total_nvlink_bytes=669433424 weight_to_token_ratio=0.196842 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.776697495e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277975040 dispatch_meta_bytes=228776 combine_payload_bytes=277975040 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278203816 ingress_bytes=388075520 link_bytes=388075520 token_bytes=556178856 weight_movement_bytes=110100480 total_nvlink_bytes=666279336 weight_to_token_ratio=0.197959 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.749271567e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44472 payload_rows_remote=38882 dup_rows=21064 dup_rows_remote=18472 dedup_ratio=0.321411 router_tokens_per_expert=[508, 507, 505, 508, 559, 546, 511, 562, 517, 504, 492, 490, 494, 514, 493, 482, 479, 533, 548, 518, 508, 488, 485, 513, 519, 539, 522, 508, 483, 521, 481, 521, 508, 529, 521, 506, 532, 519, 513, 463, 495, 524, 501, 530, 538, 526, 541, 473, 497, 497, 502, 466, 502, 499, 524, 527, 546, 552, 471, 540, 530, 514, 496, 528] payload_rows_per_dst=[5626, 5458, 5564, 5484, 5590, 5628, 5442, 5680] dup_rows_per_dst=[2786, 2514, 2580, 2704, 2592, 2628, 2586, 2674] combine_rows_per_src=[5626, 5458, 5564, 5484, 5590, 5628, 5442, 5680] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278333440 dispatch_meta_bytes=229376 combine_payload_bytes=278333440 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278562816 ingress_bytes=608634880 link_bytes=608634880 token_bytes=556896256 weight_movement_bytes=330301440 total_nvlink_bytes=887197696 weight_to_token_ratio=0.593111 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058516160e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280311808 dispatch_meta_bytes=229856 combine_payload_bytes=280311808 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280541664 ingress_bytes=390412288 link_bytes=390412288 token_bytes=560853472 weight_movement_bytes=110100480 total_nvlink_bytes=670953952 weight_to_token_ratio=0.196309 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.789911806e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44592 payload_rows_remote=38974 dup_rows=20944 dup_rows_remote=18244 dedup_ratio=0.319580 router_tokens_per_expert=[491, 512, 488, 535, 503, 490, 516, 512, 516, 523, 486, 513, 506, 531, 518, 528, 538, 545, 509, 522, 523, 490, 517, 515, 517, 506, 491, 513, 496, 473, 518, 536, 512, 473, 494, 494, 488, 509, 529, 519, 511, 515, 518, 518, 463, 525, 526, 507, 529, 525, 515, 479, 540, 540, 508, 527, 527, 488, 500, 524, 530, 534, 528, 496] payload_rows_per_dst=[5548, 5568, 5618, 5548, 5502, 5574, 5644, 5590] dup_rows_per_dst=[2546, 2674, 2700, 2552, 2534, 2592, 2682, 2664] combine_rows_per_src=[5548, 5568, 5618, 5548, 5502, 5574, 5644, 5590] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279537664 dispatch_meta_bytes=229320 combine_payload_bytes=279537664 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279766984 ingress_bytes=499738624 link_bytes=499738624 token_bytes=559304648 weight_movement_bytes=220200960 total_nvlink_bytes=779505608 weight_to_token_ratio=0.393705 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.691276600e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278706176 dispatch_meta_bytes=229416 combine_payload_bytes=278706176 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278935592 ingress_bytes=388806656 link_bytes=388806656 token_bytes=557641768 weight_movement_bytes=110100480 total_nvlink_bytes=667742248 weight_to_token_ratio=0.197439 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.761987225e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44658 payload_rows_remote=38970 dup_rows=20878 dup_rows_remote=18124 dedup_ratio=0.318573 router_tokens_per_expert=[514, 553, 533, 489, 518, 525, 489, 515, 508, 526, 543, 494, 491, 516, 518, 512, 547, 508, 523, 542, 545, 500, 487, 514, 516, 484, 486, 529, 511, 498, 522, 472, 522, 496, 530, 515, 501, 514, 532, 536, 496, 501, 523, 517, 505, 514, 509, 465, 538, 490, 513, 521, 502, 516, 578, 517, 527, 488, 495, 467, 512, 501, 510, 489] payload_rows_per_dst=[5306, 5558, 5656, 5510, 5622, 5546, 5772, 5688] dup_rows_per_dst=[2260, 2552, 2698, 2526, 2746, 2522, 2820, 2754] combine_rows_per_src=[5306, 5558, 5656, 5510, 5622, 5546, 5772, 5688] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44448 payload_rows_remote=38896 dup_rows=21088 dup_rows_remote=18474 dedup_ratio=0.321777 router_tokens_per_expert=[517, 494, 529, 566, 511, 504, 506, 556, 547, 538, 515, 515, 519, 502, 490, 524, 548, 507, 488, 516, 514, 478, 514, 530, 526, 530, 507, 536, 511, 470, 503, 500, 530, 479, 540, 511, 510, 542, 505, 518, 483, 494, 520, 509, 511, 473, 494, 553, 499, 482, 518, 518, 475, 486, 515, 514, 537, 513, 500, 477, 542, 519, 526, 464] payload_rows_per_dst=[5648, 5540, 5578, 5552, 5588, 5508, 5468, 5566] dup_rows_per_dst=[2718, 2760, 2612, 2614, 2682, 2566, 2546, 2590] combine_rows_per_src=[5648, 5540, 5578, 5552, 5588, 5508, 5468, 5566] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279365632 dispatch_meta_bytes=228872 combine_payload_bytes=279365632 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279594504 ingress_bytes=389466112 link_bytes=389466112 token_bytes=558960136 weight_movement_bytes=110100480 total_nvlink_bytes=669060616 weight_to_token_ratio=0.196974 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773456249e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44660 payload_rows_remote=38990 dup_rows=20876 dup_rows_remote=18212 dedup_ratio=0.318542 router_tokens_per_expert=[547, 534, 503, 508, 541, 506, 508, 505, 502, 488, 559, 508, 472, 537, 536, 488, 536, 519, 509, 484, 512, 500, 515, 509, 493, 506, 516, 532, 519, 526, 502, 527, 553, 520, 505, 491, 518, 547, 485, 501, 460, 515, 529, 541, 525, 510, 468, 532, 505, 488, 483, 513, 502, 554, 501, 508, 517, 496, 501, 501, 515, 490, 540, 507] payload_rows_per_dst=[5670, 5596, 5630, 5616, 5548, 5486, 5582, 5532] dup_rows_per_dst=[2664, 2554, 2538, 2822, 2692, 2478, 2526, 2602] combine_rows_per_src=[5670, 5596, 5630, 5616, 5548, 5486, 5582, 5532] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44638 payload_rows_remote=39188 dup_rows=20898 dup_rows_remote=18364 dedup_ratio=0.318878 router_tokens_per_expert=[518, 530, 534, 534, 485, 542, 486, 514, 557, 511, 518, 538, 499, 519, 509, 507, 510, 483, 539, 505, 494, 517, 526, 507, 542, 532, 537, 513, 541, 508, 486, 474, 530, 520, 523, 519, 553, 496, 509, 509, 505, 456, 511, 520, 476, 501, 509, 522, 494, 492, 547, 493, 473, 499, 499, 495, 519, 546, 500, 528, 499, 510, 474, 526] payload_rows_per_dst=[5570, 5632, 5570, 5656, 5642, 5510, 5450, 5608] dup_rows_per_dst=[2716, 2684, 2592, 2610, 2676, 2490, 2534, 2596] combine_rows_per_src=[5570, 5632, 5570, 5656, 5642, 5510, 5450, 5608] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279336960 dispatch_meta_bytes=228376 combine_payload_bytes=279336960 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279565336 ingress_bytes=389437440 link_bytes=389437440 token_bytes=558902296 weight_movement_bytes=110100480 total_nvlink_bytes=669002776 weight_to_token_ratio=0.196994 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.772957596e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44368 payload_rows_remote=38776 dup_rows=21168 dup_rows_remote=18388 dedup_ratio=0.322998 router_tokens_per_expert=[457, 537, 492, 543, 519, 527, 518, 506, 520, 513, 515, 528, 550, 514, 518, 528, 538, 540, 492, 494, 517, 513, 518, 450, 543, 502, 520, 504, 539, 538, 518, 524, 515, 498, 508, 490, 476, 506, 534, 469, 534, 505, 515, 500, 528, 533, 504, 541, 528, 530, 499, 510, 505, 478, 536, 502, 523, 468, 508, 490, 488, 474, 507, 531] payload_rows_per_dst=[5542, 5592, 5566, 5710, 5452, 5554, 5496, 5456] dup_rows_per_dst=[2656, 2780, 2558, 2666, 2540, 2766, 2680, 2522] combine_rows_per_src=[5542, 5592, 5566, 5710, 5452, 5554, 5496, 5456] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278806528 dispatch_meta_bytes=229480 combine_payload_bytes=278806528 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279036008 ingress_bytes=388907008 link_bytes=388907008 token_bytes=557842536 weight_movement_bytes=110100480 total_nvlink_bytes=667943016 weight_to_token_ratio=0.197368 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.763732511e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44384 payload_rows_remote=38746 dup_rows=21152 dup_rows_remote=18536 dedup_ratio=0.322754 router_tokens_per_expert=[513, 517, 530, 516, 503, 531, 532, 512, 509, 516, 503, 496, 538, 519, 523, 510, 524, 545, 519, 491, 499, 517, 504, 509, 504, 485, 498, 493, 505, 492, 484, 470, 504, 505, 510, 521, 493, 484, 521, 515, 513, 536, 531, 512, 479, 525, 496, 535, 490, 503, 544, 520, 509, 504, 512, 540, 518, 527, 503, 554, 515, 528, 465, 549] payload_rows_per_dst=[5586, 5594, 5522, 5390, 5506, 5638, 5560, 5588] dup_rows_per_dst=[2722, 2634, 2694, 2472, 2600, 2616, 2684, 2730] combine_rows_per_src=[5586, 5594, 5522, 5390, 5506, 5638, 5560, 5588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280569856 dispatch_meta_bytes=229632 combine_payload_bytes=280569856 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280799488 ingress_bytes=390670336 link_bytes=390670336 token_bytes=561369344 weight_movement_bytes=110100480 total_nvlink_bytes=671469824 weight_to_token_ratio=0.196128 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.794399685e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278806528 dispatch_meta_bytes=229456 combine_payload_bytes=278806528 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279035984 ingress_bytes=388907008 link_bytes=388907008 token_bytes=557842512 weight_movement_bytes=110100480 total_nvlink_bytes=667942992 weight_to_token_ratio=0.197368 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.763732511e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44554 payload_rows_remote=39040 dup_rows=20982 dup_rows_remote=18454 dedup_ratio=0.320160 router_tokens_per_expert=[504, 517, 534, 508, 536, 525, 536, 534, 568, 512, 505, 516, 492, 526, 509, 509, 503, 477, 481, 533, 503, 477, 523, 515, 469, 506, 535, 507, 524, 532, 478, 546, 497, 478, 526, 522, 492, 492, 511, 503, 536, 521, 541, 528, 479, 504, 485, 528, 484, 525, 528, 523, 494, 502, 536, 514, 508, 497, 513, 502, 511, 494, 539, 515] payload_rows_per_dst=[5620, 5620, 5482, 5598, 5514, 5582, 5588, 5550] dup_rows_per_dst=[2768, 2654, 2542, 2596, 2528, 2662, 2624, 2608] combine_rows_per_src=[5620, 5620, 5482, 5598, 5514, 5582, 5588, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44632 payload_rows_remote=38996 dup_rows=20904 dup_rows_remote=18282 dedup_ratio=0.318970 router_tokens_per_expert=[545, 531, 501, 503, 538, 502, 505, 504, 501, 484, 556, 505, 471, 538, 533, 485, 535, 517, 509, 482, 511, 496, 513, 506, 490, 507, 516, 534, 520, 524, 501, 528, 555, 520, 503, 492, 517, 550, 486, 503, 460, 517, 529, 541, 525, 511, 468, 538, 508, 491, 487, 517, 503, 561, 504, 510, 517, 499, 504, 501, 517, 491, 542, 510] payload_rows_per_dst=[5636, 5628, 5614, 5554, 5560, 5506, 5592, 5542] dup_rows_per_dst=[2622, 2660, 2524, 2686, 2714, 2508, 2570, 2620] combine_rows_per_src=[5636, 5628, 5614, 5554, 5560, 5506, 5592, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280383488 dispatch_meta_bytes=229744 combine_payload_bytes=280383488 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280613232 ingress_bytes=390483968 link_bytes=390483968 token_bytes=560996720 weight_movement_bytes=110100480 total_nvlink_bytes=671097200 weight_to_token_ratio=0.196259 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.791158439e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279838720 dispatch_meta_bytes=229976 combine_payload_bytes=279838720 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=280068696 ingress_bytes=610140160 link_bytes=610140160 token_bytes=559907416 weight_movement_bytes=330301440 total_nvlink_bytes=890208856 weight_to_token_ratio=0.589922 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061134089e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44420 payload_rows_remote=38772 dup_rows=21116 dup_rows_remote=18498 dedup_ratio=0.322205 router_tokens_per_expert=[510, 512, 530, 514, 497, 526, 528, 512, 509, 515, 502, 494, 534, 515, 524, 510, 523, 543, 517, 493, 498, 517, 501, 512, 504, 488, 496, 492, 504, 490, 482, 471, 501, 507, 510, 519, 493, 489, 520, 519, 514, 536, 532, 511, 480, 526, 498, 536, 495, 507, 543, 522, 511, 506, 513, 541, 518, 528, 504, 556, 516, 532, 468, 554] payload_rows_per_dst=[5568, 5590, 5526, 5386, 5516, 5648, 5584, 5602] dup_rows_per_dst=[2690, 2616, 2682, 2468, 2600, 2618, 2692, 2750] combine_rows_per_src=[5568, 5590, 5526, 5386, 5516, 5648, 5584, 5602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279523328 dispatch_meta_bytes=229112 combine_payload_bytes=279523328 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=279752440 ingress_bytes=719925248 link_bytes=719925248 token_bytes=559275768 weight_movement_bytes=440401920 total_nvlink_bytes=999677688 weight_to_token_ratio=0.787450 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.252068414e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44466 payload_rows_remote=38966 dup_rows=21070 dup_rows_remote=18540 dedup_ratio=0.321503 router_tokens_per_expert=[509, 532, 496, 526, 485, 503, 522, 504, 512, 526, 544, 526, 537, 521, 498, 568, 516, 484, 490, 484, 481, 539, 520, 501, 529, 495, 494, 507, 555, 500, 502, 505, 514, 478, 489, 487, 486, 528, 534, 486, 508, 546, 534, 508, 522, 500, 477, 522, 535, 535, 501, 529, 467, 519, 564, 524, 468, 498, 503, 527, 509, 544, 495, 520] payload_rows_per_dst=[5594, 5642, 5500, 5572, 5480, 5598, 5576, 5504] dup_rows_per_dst=[2560, 2822, 2530, 2602, 2524, 2636, 2772, 2624] combine_rows_per_src=[5594, 5642, 5500, 5572, 5480, 5598, 5576, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44494 payload_rows_remote=39016 dup_rows=21042 dup_rows_remote=18396 dedup_ratio=0.321075 router_tokens_per_expert=[517, 503, 507, 504, 535, 514, 485, 510, 493, 505, 525, 542, 516, 503, 482, 496, 523, 489, 501, 499, 512, 497, 494, 535, 546, 519, 537, 505, 522, 520, 496, 506, 514, 519, 518, 525, 518, 508, 559, 552, 562, 550, 483, 511, 536, 518, 483, 476, 550, 490, 517, 522, 509, 495, 503, 495, 497, 493, 484, 532, 478, 517, 525, 491] payload_rows_per_dst=[5522, 5478, 5492, 5626, 5688, 5608, 5576, 5504] dup_rows_per_dst=[2628, 2646, 2608, 2676, 2738, 2630, 2586, 2530] combine_rows_per_src=[5522, 5478, 5492, 5626, 5688, 5608, 5576, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44468 payload_rows_remote=38938 dup_rows=21068 dup_rows_remote=18380 dedup_ratio=0.321472 router_tokens_per_expert=[477, 534, 503, 494, 495, 547, 511, 527, 490, 486, 511, 473, 571, 506, 504, 523, 531, 496, 506, 557, 519, 479, 521, 490, 509, 500, 517, 520, 507, 492, 511, 537, 518, 505, 474, 507, 494, 510, 545, 498, 505, 525, 552, 529, 522, 524, 462, 532, 484, 511, 543, 520, 495, 511, 523, 527, 532, 542, 495, 485, 516, 524, 515, 499] payload_rows_per_dst=[5444, 5404, 5608, 5528, 5540, 5730, 5684, 5530] dup_rows_per_dst=[2502, 2420, 2590, 2658, 2560, 2802, 2848, 2688] combine_rows_per_src=[5444, 5404, 5608, 5528, 5540, 5730, 5684, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44368 payload_rows_remote=38768 dup_rows=21168 dup_rows_remote=18370 dedup_ratio=0.322998 router_tokens_per_expert=[504, 508, 526, 511, 521, 512, 504, 497, 498, 472, 503, 526, 525, 509, 494, 548, 522, 498, 498, 466, 521, 480, 496, 503, 560, 544, 531, 503, 502, 526, 520, 513, 513, 503, 499, 475, 517, 530, 513, 519, 485, 459, 583, 539, 503, 528, 542, 490, 491, 512, 461, 502, 498, 511, 493, 541, 547, 513, 517, 523, 555, 503, 553, 509] payload_rows_per_dst=[5514, 5514, 5454, 5600, 5504, 5632, 5496, 5654] dup_rows_per_dst=[2652, 2636, 2514, 2798, 2634, 2626, 2522, 2786] combine_rows_per_src=[5514, 5514, 5454, 5600, 5504, 5632, 5496, 5654] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44632 payload_rows_remote=39174 dup_rows=20904 dup_rows_remote=18352 dedup_ratio=0.318970 router_tokens_per_expert=[516, 529, 530, 532, 484, 542, 483, 508, 555, 507, 515, 538, 500, 514, 506, 502, 510, 482, 538, 505, 491, 518, 526, 504, 541, 530, 536, 514, 541, 506, 484, 478, 532, 519, 520, 518, 551, 499, 510, 512, 500, 460, 513, 523, 474, 503, 511, 521, 496, 495, 549, 494, 473, 500, 501, 497, 524, 550, 506, 531, 502, 512, 479, 528] payload_rows_per_dst=[5550, 5614, 5566, 5656, 5644, 5514, 5458, 5630] dup_rows_per_dst=[2698, 2660, 2582, 2604, 2678, 2496, 2552, 2634] combine_rows_per_src=[5550, 5614, 5566, 5656, 5644, 5514, 5458, 5630] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=277917696 dispatch_meta_bytes=229080 combine_payload_bytes=277917696 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278146776 ingress_bytes=498118656 link_bytes=498118656 token_bytes=556064472 weight_movement_bytes=220200960 total_nvlink_bytes=776265432 weight_to_token_ratio=0.395999 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.663102692e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279308288 dispatch_meta_bytes=230024 combine_payload_bytes=279308288 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279538312 ingress_bytes=389408768 link_bytes=389408768 token_bytes=558846600 weight_movement_bytes=110100480 total_nvlink_bytes=668947080 weight_to_token_ratio=0.197014 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.772458943e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279666688 dispatch_meta_bytes=229648 combine_payload_bytes=279666688 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279896336 ingress_bytes=499867648 link_bytes=499867648 token_bytes=559563024 weight_movement_bytes=220200960 total_nvlink_bytes=779763984 weight_to_token_ratio=0.393523 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.693520540e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44538 payload_rows_remote=39018 dup_rows=20998 dup_rows_remote=18468 dedup_ratio=0.320404 router_tokens_per_expert=[507, 521, 538, 512, 535, 530, 538, 536, 569, 516, 508, 516, 494, 529, 509, 508, 504, 480, 486, 533, 503, 477, 522, 516, 474, 506, 536, 503, 523, 531, 478, 546, 497, 480, 525, 522, 495, 492, 513, 501, 539, 520, 542, 527, 478, 504, 482, 526, 480, 520, 525, 520, 491, 498, 535, 511, 508, 497, 511, 500, 508, 491, 535, 511] payload_rows_per_dst=[5628, 5630, 5490, 5596, 5520, 5580, 5562, 5532] dup_rows_per_dst=[2806, 2668, 2552, 2598, 2530, 2656, 2598, 2590] combine_rows_per_src=[5628, 5630, 5490, 5596, 5520, 5580, 5562, 5532] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279480320 dispatch_meta_bytes=228808 combine_payload_bytes=279480320 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279709128 ingress_bytes=609781760 link_bytes=609781760 token_bytes=559189448 weight_movement_bytes=330301440 total_nvlink_bytes=889490888 weight_to_token_ratio=0.590679 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060510773e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=280899584 dispatch_meta_bytes=230208 combine_payload_bytes=280899584 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281129792 ingress_bytes=391000064 link_bytes=391000064 token_bytes=562029376 weight_movement_bytes=110100480 total_nvlink_bytes=672129856 weight_to_token_ratio=0.195898 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.800134198e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277946368 dispatch_meta_bytes=228656 combine_payload_bytes=277946368 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=278175024 ingress_bytes=718348288 link_bytes=718348288 token_bytes=556121392 weight_movement_bytes=440401920 total_nvlink_bytes=996523312 weight_to_token_ratio=0.791917 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.249325821e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44462 payload_rows_remote=38956 dup_rows=21074 dup_rows_remote=18522 dedup_ratio=0.321564 router_tokens_per_expert=[512, 535, 499, 529, 487, 504, 524, 505, 513, 526, 546, 528, 539, 522, 497, 569, 522, 483, 491, 486, 483, 541, 520, 503, 531, 496, 494, 506, 555, 500, 501, 506, 514, 478, 492, 487, 485, 529, 533, 485, 510, 544, 534, 509, 519, 499, 475, 521, 529, 535, 501, 527, 466, 516, 562, 524, 465, 496, 498, 526, 507, 542, 492, 515] payload_rows_per_dst=[5614, 5654, 5506, 5572, 5478, 5592, 5562, 5484] dup_rows_per_dst=[2576, 2826, 2552, 2606, 2528, 2630, 2758, 2598] combine_rows_per_src=[5614, 5654, 5506, 5572, 5478, 5592, 5562, 5484] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=277731328 dispatch_meta_bytes=229128 combine_payload_bytes=277731328 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277960456 ingress_bytes=497932288 link_bytes=497932288 token_bytes=555691784 weight_movement_bytes=220200960 total_nvlink_bytes=775892744 weight_to_token_ratio=0.396265 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.659861446e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279681024 dispatch_meta_bytes=229944 combine_payload_bytes=279681024 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279910968 ingress_bytes=389781504 link_bytes=389781504 token_bytes=559591992 weight_movement_bytes=110100480 total_nvlink_bytes=669692472 weight_to_token_ratio=0.196751 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778941435e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44456 payload_rows_remote=38824 dup_rows=21080 dup_rows_remote=18266 dedup_ratio=0.321655 router_tokens_per_expert=[481, 536, 506, 495, 496, 549, 513, 528, 495, 491, 511, 479, 572, 507, 505, 523, 532, 495, 509, 558, 519, 479, 524, 490, 509, 503, 522, 519, 508, 492, 511, 541, 518, 506, 473, 508, 492, 508, 546, 498, 506, 524, 549, 526, 520, 523, 461, 531, 481, 509, 542, 517, 495, 512, 519, 525, 529, 537, 489, 483, 516, 521, 509, 497] payload_rows_per_dst=[5330, 5330, 5794, 5498, 5554, 5548, 5770, 5632] dup_rows_per_dst=[2382, 2300, 2872, 2642, 2578, 2488, 3004, 2814] combine_rows_per_src=[5330, 5330, 5794, 5498, 5554, 5548, 5770, 5632] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279236608 dispatch_meta_bytes=229912 combine_payload_bytes=279236608 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279466520 ingress_bytes=389337088 link_bytes=389337088 token_bytes=558703128 weight_movement_bytes=110100480 total_nvlink_bytes=668803608 weight_to_token_ratio=0.197064 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.771212310e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44372 payload_rows_remote=38772 dup_rows=21164 dup_rows_remote=18362 dedup_ratio=0.322937 router_tokens_per_expert=[505, 512, 530, 512, 525, 517, 506, 498, 499, 473, 505, 529, 527, 509, 497, 548, 522, 498, 501, 464, 522, 484, 497, 505, 560, 546, 532, 504, 500, 525, 521, 513, 511, 502, 498, 476, 519, 529, 510, 518, 485, 457, 584, 539, 503, 527, 540, 488, 487, 510, 458, 501, 499, 512, 490, 536, 546, 510, 515, 522, 551, 500, 550, 509] payload_rows_per_dst=[5536, 5524, 5468, 5600, 5498, 5620, 5484, 5642] dup_rows_per_dst=[2674, 2650, 2518, 2802, 2628, 2626, 2502, 2764] combine_rows_per_src=[5536, 5524, 5468, 5600, 5498, 5620, 5484, 5642] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44444 payload_rows_remote=38944 dup_rows=21092 dup_rows_remote=18388 dedup_ratio=0.321838 router_tokens_per_expert=[510, 489, 515, 502, 519, 544, 499, 524, 490, 517, 516, 515, 516, 524, 547, 501, 513, 524, 542, 502, 505, 494, 538, 519, 569, 527, 484, 504, 516, 493, 529, 462, 491, 517, 521, 516, 557, 537, 523, 534, 528, 500, 523, 527, 530, 546, 545, 470, 508, 536, 501, 444, 480, 470, 479, 550, 468, 507, 513, 519, 501, 473, 518, 487] payload_rows_per_dst=[5500, 5570, 5578, 5582, 5666, 5650, 5436, 5462] dup_rows_per_dst=[2704, 2682, 2696, 2586, 2726, 2688, 2500, 2510] combine_rows_per_src=[5500, 5570, 5578, 5582, 5666, 5650, 5436, 5462] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278290432 dispatch_meta_bytes=228360 combine_payload_bytes=278290432 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278518792 ingress_bytes=388390912 link_bytes=388390912 token_bytes=556809224 weight_movement_bytes=110100480 total_nvlink_bytes=666909704 weight_to_token_ratio=0.197735 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.754756753e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44518 payload_rows_remote=38954 dup_rows=21018 dup_rows_remote=18474 dedup_ratio=0.320709 router_tokens_per_expert=[543, 514, 513, 527, 531, 534, 495, 523, 530, 513, 471, 515, 512, 483, 533, 481, 479, 539, 489, 521, 478, 501, 507, 526, 486, 500, 551, 517, 539, 515, 522, 536, 489, 497, 497, 548, 492, 515, 526, 488, 504, 536, 486, 503, 562, 515, 492, 507, 480, 533, 547, 483, 508, 532, 472, 499, 532, 525, 504, 558, 519, 479, 497, 519] payload_rows_per_dst=[5596, 5534, 5512, 5600, 5504, 5594, 5564, 5614] dup_rows_per_dst=[2764, 2542, 2568, 2732, 2600, 2616, 2544, 2652] combine_rows_per_src=[5596, 5534, 5512, 5600, 5504, 5594, 5564, 5614] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44506 payload_rows_remote=39016 dup_rows=21030 dup_rows_remote=18362 dedup_ratio=0.320892 router_tokens_per_expert=[519, 507, 509, 506, 539, 518, 486, 512, 499, 510, 523, 541, 519, 503, 483, 501, 525, 491, 502, 500, 519, 499, 496, 534, 550, 522, 536, 505, 520, 524, 499, 507, 513, 516, 516, 524, 517, 508, 559, 550, 561, 548, 479, 511, 533, 513, 481, 476, 545, 488, 515, 521, 507, 496, 500, 494, 494, 490, 483, 529, 470, 514, 524, 489] payload_rows_per_dst=[5542, 5490, 5506, 5640, 5674, 5602, 5570, 5482] dup_rows_per_dst=[2650, 2668, 2626, 2686, 2732, 2602, 2562, 2504] combine_rows_per_src=[5542, 5490, 5506, 5640, 5674, 5602, 5570, 5482] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44654 payload_rows_remote=39036 dup_rows=20882 dup_rows_remote=18202 dedup_ratio=0.318634 router_tokens_per_expert=[519, 524, 493, 517, 501, 515, 518, 515, 517, 497, 508, 536, 513, 509, 498, 510, 508, 479, 515, 471, 469, 488, 500, 527, 528, 538, 533, 547, 518, 508, 519, 518, 524, 524, 498, 488, 532, 480, 522, 498, 546, 530, 552, 541, 473, 489, 506, 512, 531, 509, 488, 502, 511, 521, 486, 486, 529, 505, 513, 491, 527, 523, 537, 538] payload_rows_per_dst=[5596, 5574, 5518, 5678, 5572, 5618, 5480, 5618] dup_rows_per_dst=[2608, 2602, 2396, 2740, 2560, 2680, 2588, 2708] combine_rows_per_src=[5596, 5574, 5518, 5678, 5572, 5618, 5480, 5618] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=277917696 dispatch_meta_bytes=228536 combine_payload_bytes=277917696 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278146232 ingress_bytes=608219136 link_bytes=608219136 token_bytes=556063928 weight_movement_bytes=330301440 total_nvlink_bytes=886365368 weight_to_token_ratio=0.593999 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057793112e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44464 payload_rows_remote=38932 dup_rows=21072 dup_rows_remote=18412 dedup_ratio=0.321533 router_tokens_per_expert=[503, 509, 531, 529, 510, 522, 538, 475, 504, 483, 486, 516, 502, 540, 512, 535, 499, 557, 528, 500, 485, 537, 509, 487, 537, 534, 474, 509, 523, 527, 493, 516, 528, 511, 479, 510, 515, 519, 521, 513, 535, 530, 505, 518, 537, 501, 543, 517, 507, 528, 495, 523, 496, 475, 553, 513, 487, 492, 482, 501, 497, 546, 464, 517] payload_rows_per_dst=[5616, 5480, 5542, 5522, 5532, 5656, 5628, 5488] dup_rows_per_dst=[2618, 2676, 2662, 2704, 2660, 2716, 2552, 2484] combine_rows_per_src=[5616, 5480, 5542, 5522, 5532, 5656, 5628, 5488] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279150592 dispatch_meta_bytes=229328 combine_payload_bytes=279150592 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279379920 ingress_bytes=609452032 link_bytes=609452032 token_bytes=558530512 weight_movement_bytes=330301440 total_nvlink_bytes=888831952 weight_to_token_ratio=0.591376 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059937321e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279222272 dispatch_meta_bytes=229712 combine_payload_bytes=279222272 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279451984 ingress_bytes=389322752 link_bytes=389322752 token_bytes=558674256 weight_movement_bytes=110100480 total_nvlink_bytes=668774736 weight_to_token_ratio=0.197075 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.770962983e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44484 payload_rows_remote=39074 dup_rows=21052 dup_rows_remote=18484 dedup_ratio=0.321228 router_tokens_per_expert=[509, 501, 523, 527, 494, 516, 559, 511, 499, 496, 499, 504, 524, 555, 529, 511, 519, 517, 449, 471, 551, 478, 500, 504, 529, 493, 530, 481, 537, 482, 546, 454, 506, 497, 541, 512, 485, 521, 523, 516, 501, 521, 494, 537, 504, 502, 505, 497, 519, 498, 574, 509, 555, 532, 494, 525, 526, 504, 513, 513, 508, 522, 486, 530] payload_rows_per_dst=[5568, 5520, 5410, 5586, 5578, 5630, 5648, 5544] dup_rows_per_dst=[2712, 2714, 2568, 2518, 2624, 2492, 2764, 2660] combine_rows_per_src=[5568, 5520, 5410, 5586, 5578, 5630, 5648, 5544] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279666688 dispatch_meta_bytes=229512 combine_payload_bytes=279666688 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279896200 ingress_bytes=499867648 link_bytes=499867648 token_bytes=559562888 weight_movement_bytes=220200960 total_nvlink_bytes=779763848 weight_to_token_ratio=0.393523 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.693520540e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279810048 dispatch_meta_bytes=228952 combine_payload_bytes=279810048 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280039000 ingress_bytes=389910528 link_bytes=389910528 token_bytes=559849048 weight_movement_bytes=110100480 total_nvlink_bytes=669949528 weight_to_token_ratio=0.196661 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.781185375e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279107584 dispatch_meta_bytes=229272 combine_payload_bytes=279107584 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279336856 ingress_bytes=499308544 link_bytes=499308544 token_bytes=558444440 weight_movement_bytes=220200960 total_nvlink_bytes=778645400 weight_to_token_ratio=0.394311 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.683796802e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44484 payload_rows_remote=38948 dup_rows=21052 dup_rows_remote=18394 dedup_ratio=0.321228 router_tokens_per_expert=[499, 506, 529, 526, 507, 519, 536, 474, 502, 482, 484, 518, 501, 539, 510, 536, 499, 554, 527, 498, 484, 536, 508, 486, 538, 532, 474, 507, 522, 528, 493, 517, 528, 512, 476, 510, 516, 518, 522, 515, 535, 529, 507, 519, 538, 501, 546, 518, 509, 533, 498, 525, 499, 477, 554, 516, 488, 495, 482, 502, 499, 547, 465, 518] payload_rows_per_dst=[5600, 5480, 5534, 5524, 5536, 5660, 5650, 5500] dup_rows_per_dst=[2592, 2664, 2650, 2698, 2658, 2726, 2572, 2492] combine_rows_per_src=[5600, 5480, 5534, 5524, 5536, 5660, 5650, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=277889024 dispatch_meta_bytes=228552 combine_payload_bytes=277889024 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278117576 ingress_bytes=387989504 link_bytes=387989504 token_bytes=556006600 weight_movement_bytes=110100480 total_nvlink_bytes=666107080 weight_to_token_ratio=0.198020 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.747775608e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=280799232 dispatch_meta_bytes=230104 combine_payload_bytes=280799232 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281029336 ingress_bytes=390899712 link_bytes=390899712 token_bytes=561828568 weight_movement_bytes=110100480 total_nvlink_bytes=671929048 weight_to_token_ratio=0.195968 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.798388911e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44416 payload_rows_remote=38902 dup_rows=21120 dup_rows_remote=18408 dedup_ratio=0.322266 router_tokens_per_expert=[508, 486, 513, 502, 519, 536, 497, 522, 488, 512, 515, 513, 514, 523, 547, 496, 510, 523, 540, 502, 504, 493, 536, 519, 570, 525, 485, 504, 515, 490, 529, 461, 494, 516, 521, 518, 560, 538, 522, 532, 531, 501, 525, 527, 530, 547, 546, 473, 509, 539, 507, 447, 483, 469, 481, 554, 469, 507, 514, 522, 505, 475, 520, 489] payload_rows_per_dst=[5514, 5556, 5804, 5580, 5672, 5382, 5460, 5448] dup_rows_per_dst=[2712, 2664, 3056, 2588, 2716, 2372, 2518, 2494] combine_rows_per_src=[5514, 5556, 5804, 5580, 5672, 5382, 5460, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279179264 dispatch_meta_bytes=229368 combine_payload_bytes=279179264 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279408632 ingress_bytes=389279744 link_bytes=389279744 token_bytes=558587896 weight_movement_bytes=110100480 total_nvlink_bytes=668688376 weight_to_token_ratio=0.197105 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.770215003e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278849536 dispatch_meta_bytes=229240 combine_payload_bytes=278849536 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279078776 ingress_bytes=499050496 link_bytes=499050496 token_bytes=557928312 weight_movement_bytes=220200960 total_nvlink_bytes=778129272 weight_to_token_ratio=0.394676 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.679308922e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44654 payload_rows_remote=39028 dup_rows=20882 dup_rows_remote=18196 dedup_ratio=0.318634 router_tokens_per_expert=[516, 521, 490, 515, 500, 510, 515, 513, 517, 492, 507, 534, 511, 504, 497, 510, 508, 476, 518, 470, 469, 487, 500, 523, 528, 536, 533, 547, 517, 508, 518, 518, 525, 522, 498, 488, 533, 479, 524, 500, 549, 529, 550, 540, 475, 490, 508, 515, 533, 508, 488, 504, 514, 522, 492, 486, 533, 506, 514, 494, 530, 525, 541, 545] payload_rows_per_dst=[5578, 5562, 5514, 5672, 5574, 5626, 5484, 5644] dup_rows_per_dst=[2582, 2582, 2388, 2738, 2564, 2686, 2610, 2732] combine_rows_per_src=[5578, 5562, 5514, 5672, 5574, 5626, 5484, 5644] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44500 payload_rows_remote=39094 dup_rows=21036 dup_rows_remote=18476 dedup_ratio=0.320984 router_tokens_per_expert=[505, 499, 516, 526, 492, 514, 557, 509, 497, 495, 498, 500, 522, 553, 524, 511, 516, 519, 450, 470, 550, 476, 499, 503, 528, 491, 530, 478, 537, 480, 545, 453, 508, 495, 542, 514, 487, 519, 523, 519, 503, 524, 494, 537, 504, 500, 507, 499, 520, 497, 577, 511, 561, 535, 494, 528, 530, 507, 516, 515, 513, 523, 493, 530] payload_rows_per_dst=[5550, 5510, 5406, 5586, 5586, 5638, 5656, 5568] dup_rows_per_dst=[2686, 2690, 2560, 2498, 2628, 2498, 2790, 2686] combine_rows_per_src=[5550, 5510, 5406, 5586, 5586, 5638, 5656, 5568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44428 payload_rows_remote=38852 dup_rows=21108 dup_rows_remote=18460 dedup_ratio=0.322083 router_tokens_per_expert=[520, 513, 518, 519, 488, 528, 480, 524, 525, 526, 542, 473, 499, 482, 545, 520, 525, 502, 507, 501, 518, 540, 488, 493, 482, 511, 473, 497, 529, 483, 540, 508, 478, 501, 498, 504, 519, 510, 529, 536, 548, 502, 497, 540, 522, 524, 517, 480, 498, 503, 494, 500, 512, 523, 501, 532, 535, 564, 484, 513, 542, 532, 512, 519] payload_rows_per_dst=[5542, 5576, 5548, 5522, 5538, 5578, 5472, 5652] dup_rows_per_dst=[2638, 2648, 2600, 2524, 2612, 2682, 2654, 2750] combine_rows_per_src=[5542, 5576, 5548, 5522, 5538, 5578, 5472, 5652] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5576587304 prefetch_bytes=880803840 reduce_bytes=990904320 weight_movement_bytes=1871708160 egress_bytes=2789439528 ingress_bytes=4658855936 link_bytes=4658855936 total_nvlink_bytes=7448295464 weight_to_token_ratio=0.335637 prefetch_share_of_ingress=0.189060 dedup_ratio=0.321408 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=9 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042647 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.102516723e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44386 payload_rows_remote=38764 dup_rows=21150 dup_rows_remote=18308 dedup_ratio=0.322723 router_tokens_per_expert=[537, 503, 501, 493, 494, 532, 510, 489, 531, 505, 542, 497, 543, 549, 540, 532, 505, 507, 508, 526, 500, 520, 495, 478, 499, 508, 510, 529, 476, 510, 464, 519, 473, 520, 486, 520, 530, 502, 521, 543, 502, 522, 538, 504, 488, 514, 498, 502, 516, 476, 512, 507, 492, 553, 517, 525, 481, 515, 498, 529, 524, 539, 534, 535] payload_rows_per_dst=[5178, 5688, 5828, 5400, 5604, 5532, 5534, 5622] dup_rows_per_dst=[2016, 2790, 3174, 2476, 2586, 2604, 2662, 2842] combine_rows_per_src=[5178, 5688, 5828, 5400, 5604, 5532, 5534, 5622] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279064576 dispatch_meta_bytes=229376 combine_payload_bytes=279064576 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279293952 ingress_bytes=389165056 link_bytes=389165056 token_bytes=558358528 weight_movement_bytes=110100480 total_nvlink_bytes=668459008 weight_to_token_ratio=0.197186 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.768220390e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44368 payload_rows_remote=38742 dup_rows=21168 dup_rows_remote=18310 dedup_ratio=0.322998 router_tokens_per_expert=[543, 506, 504, 496, 497, 537, 512, 492, 532, 506, 541, 500, 548, 554, 542, 533, 506, 507, 510, 524, 500, 520, 498, 478, 495, 507, 511, 531, 476, 511, 466, 518, 478, 519, 485, 520, 528, 503, 520, 541, 505, 520, 539, 504, 487, 513, 498, 501, 511, 476, 510, 502, 492, 549, 514, 523, 480, 509, 494, 526, 522, 537, 530, 531] payload_rows_per_dst=[5282, 5478, 5778, 5418, 5616, 5580, 5590, 5626] dup_rows_per_dst=[2116, 2558, 3084, 2490, 2620, 2676, 2766, 2858] combine_rows_per_src=[5282, 5478, 5778, 5418, 5616, 5580, 5590, 5626] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5594370048 prefetch_bytes=1101004800 reduce_bytes=0 weight_movement_bytes=1101004800 egress_bytes=2798333952 ingress_bytes=3897040896 link_bytes=3897040896 total_nvlink_bytes=6695374848 weight_to_token_ratio=0.196806 prefetch_share_of_ingress=0.282523 dedup_ratio=0.320877 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=0 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=6.777595071e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=280082432 dispatch_meta_bytes=230232 combine_payload_bytes=280082432 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280312664 ingress_bytes=390182912 link_bytes=390182912 token_bytes=560395096 weight_movement_bytes=110100480 total_nvlink_bytes=670495576 weight_to_token_ratio=0.196469 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.785922580e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44650 payload_rows_remote=38976 dup_rows=20886 dup_rows_remote=18206 dedup_ratio=0.318695 router_tokens_per_expert=[515, 525, 516, 519, 516, 514, 558, 524, 498, 502, 527, 510, 552, 504, 484, 540, 522, 462, 522, 532, 517, 504, 546, 545, 495, 527, 540, 510, 544, 511, 501, 549, 501, 511, 523, 515, 530, 483, 527, 522, 513, 497, 472, 503, 522, 504, 486, 498, 524, 509, 488, 488, 518, 520, 510, 477, 484, 491, 481, 490, 513, 510, 519, 508] payload_rows_per_dst=[5618, 5558, 5618, 5674, 5578, 5520, 5600, 5484] dup_rows_per_dst=[2756, 2676, 2682, 2680, 2646, 2470, 2468, 2508] combine_rows_per_src=[5618, 5558, 5618, 5674, 5578, 5520, 5600, 5484] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5582293216 prefetch_bytes=1101004800 reduce_bytes=330301440 weight_movement_bytes=1431306240 egress_bytes=2792292576 ingress_bytes=4221306880 link_bytes=4221306880 total_nvlink_bytes=7013599456 weight_to_token_ratio=0.256401 prefetch_share_of_ingress=0.260821 dedup_ratio=0.320749 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=3 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.341546950e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5576673520 prefetch_bytes=880803840 reduce_bytes=770703360 weight_movement_bytes=1651507200 egress_bytes=2789482736 ingress_bytes=4438697984 link_bytes=4438697984 total_nvlink_bytes=7228180720 weight_to_token_ratio=0.296146 prefetch_share_of_ingress=0.198437 dedup_ratio=0.321457 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=7 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.719625835e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44328 payload_rows_remote=38694 dup_rows=21208 dup_rows_remote=18392 dedup_ratio=0.323608 router_tokens_per_expert=[521, 516, 527, 503, 533, 536, 524, 550, 542, 548, 486, 497, 504, 509, 545, 569, 505, 508, 490, 503, 481, 493, 547, 484, 514, 494, 503, 510, 472, 511, 530, 548, 548, 500, 522, 501, 522, 515, 518, 526, 489, 485, 492, 526, 486, 503, 512, 531, 490, 518, 522, 495, 541, 547, 493, 518, 492, 530, 513, 529, 474, 496, 475, 456] payload_rows_per_dst=[5634, 5580, 5498, 5584, 5586, 5540, 5564, 5342] dup_rows_per_dst=[2816, 2790, 2524, 2580, 2718, 2508, 2684, 2588] combine_rows_per_src=[5634, 5580, 5498, 5584, 5586, 5540, 5564, 5342] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=277702656 dispatch_meta_bytes=228208 combine_payload_bytes=277702656 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277930864 ingress_bytes=387803136 link_bytes=387803136 token_bytes=555633520 weight_movement_bytes=110100480 total_nvlink_bytes=665734000 weight_to_token_ratio=0.198153 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.744534362e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5581004416 prefetch_bytes=1101004800 reduce_bytes=770703360 weight_movement_bytes=1871708160 egress_bytes=2791648896 ingress_bytes=4661063680 link_bytes=4661063680 total_nvlink_bytes=7452712576 weight_to_token_ratio=0.335371 prefetch_share_of_ingress=0.236213 dedup_ratio=0.321182 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=7 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.106356353e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44414 payload_rows_remote=38826 dup_rows=21122 dup_rows_remote=18456 dedup_ratio=0.322296 router_tokens_per_expert=[524, 518, 524, 523, 490, 530, 482, 527, 527, 531, 541, 474, 501, 486, 545, 522, 527, 503, 511, 502, 518, 542, 490, 495, 486, 508, 469, 496, 525, 482, 541, 508, 479, 498, 499, 503, 522, 508, 528, 534, 550, 501, 496, 540, 519, 521, 515, 478, 494, 504, 493, 497, 511, 520, 498, 531, 532, 560, 481, 508, 540, 531, 512, 517] payload_rows_per_dst=[5564, 5588, 5556, 5506, 5532, 5576, 5460, 5632] dup_rows_per_dst=[2672, 2666, 2620, 2524, 2610, 2664, 2636, 2730] combine_rows_per_src=[5564, 5588, 5556, 5506, 5532, 5576, 5460, 5632] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44648 payload_rows_remote=38976 dup_rows=20888 dup_rows_remote=18222 dedup_ratio=0.318726 router_tokens_per_expert=[514, 521, 511, 519, 510, 512, 556, 520, 497, 499, 526, 510, 551, 504, 479, 539, 521, 460, 518, 529, 518, 503, 545, 544, 493, 526, 539, 511, 543, 510, 500, 547, 502, 510, 524, 515, 529, 483, 527, 525, 515, 498, 474, 505, 523, 505, 492, 501, 525, 514, 491, 488, 519, 521, 511, 477, 488, 491, 485, 492, 515, 515, 523, 510] payload_rows_per_dst=[5598, 5556, 5608, 5672, 5572, 5536, 5610, 5496] dup_rows_per_dst=[2728, 2654, 2668, 2666, 2658, 2490, 2482, 2542] combine_rows_per_src=[5598, 5556, 5608, 5672, 5572, 5536, 5610, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44534 payload_rows_remote=38946 dup_rows=21002 dup_rows_remote=18434 dedup_ratio=0.320465 router_tokens_per_expert=[538, 514, 510, 521, 529, 531, 495, 521, 528, 512, 471, 513, 511, 481, 531, 478, 479, 539, 486, 521, 477, 499, 506, 520, 486, 499, 550, 518, 539, 515, 519, 535, 486, 498, 499, 547, 493, 517, 523, 487, 502, 540, 486, 503, 563, 519, 491, 507, 482, 534, 549, 488, 514, 534, 471, 506, 534, 525, 507, 560, 522, 480, 504, 525] payload_rows_per_dst=[5580, 5522, 5498, 5610, 5500, 5598, 5588, 5638] dup_rows_per_dst=[2738, 2528, 2556, 2712, 2600, 2624, 2568, 2676] combine_rows_per_src=[5580, 5522, 5498, 5610, 5500, 5598, 5588, 5638] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279752704 dispatch_meta_bytes=228896 combine_payload_bytes=279752704 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=279981600 ingress_bytes=389853184 link_bytes=389853184 token_bytes=559734304 weight_movement_bytes=110100480 total_nvlink_bytes=669834784 weight_to_token_ratio=0.196701 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.780188068e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=280225792 dispatch_meta_bytes=230280 combine_payload_bytes=280225792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280456072 ingress_bytes=390326272 link_bytes=390326272 token_bytes=560681864 weight_movement_bytes=110100480 total_nvlink_bytes=670782344 weight_to_token_ratio=0.196369 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4096 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.788415846e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278491136 dispatch_meta_bytes=229248 combine_payload_bytes=278491136 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278720384 ingress_bytes=498692096 link_bytes=498692096 token_bytes=557211520 weight_movement_bytes=220200960 total_nvlink_bytes=777412480 weight_to_token_ratio=0.395184 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.673075757e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=277860352 dispatch_meta_bytes=228288 combine_payload_bytes=277860352 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278088640 ingress_bytes=498061312 link_bytes=498061312 token_bytes=555948992 weight_movement_bytes=220200960 total_nvlink_bytes=776149952 weight_to_token_ratio=0.396081 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.662105386e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279379968 dispatch_meta_bytes=228792 combine_payload_bytes=279379968 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279608760 ingress_bytes=499580928 link_bytes=499580928 token_bytes=558988728 weight_movement_bytes=220200960 total_nvlink_bytes=779189688 weight_to_token_ratio=0.393927 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.688534007e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44328 payload_rows_remote=38508 dup_rows=21208 dup_rows_remote=18098 dedup_ratio=0.323608 router_tokens_per_expert=[515, 515, 528, 502, 529, 530, 523, 548, 542, 543, 485, 494, 499, 505, 539, 566, 502, 506, 488, 499, 485, 494, 544, 484, 514, 494, 502, 511, 472, 512, 533, 549, 547, 500, 521, 501, 522, 515, 518, 527, 491, 486, 494, 526, 488, 505, 513, 535, 495, 517, 523, 498, 539, 549, 494, 519, 496, 533, 516, 532, 479, 501, 478, 458] payload_rows_per_dst=[5820, 5460, 5494, 5614, 5586, 5544, 5554, 5256] dup_rows_per_dst=[3110, 2626, 2510, 2624, 2734, 2514, 2650, 2440] combine_rows_per_src=[5820, 5460, 5494, 5614, 5586, 5544, 5554, 5256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279164928 dispatch_meta_bytes=229520 combine_payload_bytes=279164928 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=279394448 ingress_bytes=389265408 link_bytes=389265408 token_bytes=558559376 weight_movement_bytes=110100480 total_nvlink_bytes=668659856 weight_to_token_ratio=0.197115 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.769965677e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5582435984 prefetch_bytes=880803840 reduce_bytes=990904320 weight_movement_bytes=1871708160 egress_bytes=2792363664 ingress_bytes=4661780480 link_bytes=4661780480 total_nvlink_bytes=7454144144 weight_to_token_ratio=0.335285 prefetch_share_of_ingress=0.188942 dedup_ratio=0.320667 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=9 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.107602986e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5582639464 prefetch_bytes=990904320 reduce_bytes=770703360 weight_movement_bytes=1761607680 egress_bytes=2792466792 ingress_bytes=4551780352 link_bytes=4551780352 total_nvlink_bytes=7344247144 weight_to_token_ratio=0.315551 prefetch_share_of_ingress=0.217696 dedup_ratio=0.321127 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=7 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.916294671e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279379968 dispatch_meta_bytes=228728 combine_payload_bytes=279379968 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=279608696 ingress_bytes=389480448 link_bytes=389480448 token_bytes=558988664 weight_movement_bytes=110100480 total_nvlink_bytes=669089144 weight_to_token_ratio=0.196964 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773705576e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=277358592 dispatch_meta_bytes=228344 combine_payload_bytes=277358592 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277586936 ingress_bytes=497559552 link_bytes=497559552 token_bytes=554945528 weight_movement_bytes=220200960 total_nvlink_bytes=775146488 weight_to_token_ratio=0.396797 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.653378954e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5558972672 prefetch_bytes=990904320 reduce_bytes=330301440 weight_movement_bytes=1321205760 egress_bytes=2780627200 ingress_bytes=4099551232 link_bytes=4099551232 total_nvlink_bytes=6880178432 weight_to_token_ratio=0.237671 prefetch_share_of_ingress=0.241710 dedup_ratio=0.322189 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=3 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042647 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.129793853e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278304768 dispatch_meta_bytes=229128 combine_payload_bytes=278304768 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=278533896 ingress_bytes=718706688 link_bytes=718706688 token_bytes=556838664 weight_movement_bytes=440401920 total_nvlink_bytes=997240584 weight_to_token_ratio=0.790897 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.249949137e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5575497608 prefetch_bytes=990904320 reduce_bytes=990904320 weight_movement_bytes=1981808640 egress_bytes=2788894600 ingress_bytes=4768411648 link_bytes=4768411648 total_nvlink_bytes=7557306248 weight_to_token_ratio=0.355450 prefetch_share_of_ingress=0.207806 dedup_ratio=0.321484 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=9 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.293052125e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5571021536 prefetch_bytes=880803840 reduce_bytes=1871708160 weight_movement_bytes=2752512000 egress_bytes=2786654944 ingress_bytes=5536878592 link_bytes=5536878592 total_nvlink_bytes=8323533536 weight_to_token_ratio=0.494077 prefetch_share_of_ingress=0.159079 dedup_ratio=0.321707 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=17 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.629542532e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5576415088 prefetch_bytes=550502400 reduce_bytes=2532311040 weight_movement_bytes=3082813440 egress_bytes=2789353328 ingress_bytes=5869875200 link_bytes=5869875200 total_nvlink_bytes=8659228528 weight_to_token_ratio=0.552831 prefetch_share_of_ingress=0.093784 dedup_ratio=0.321661 prefetch_slots=10 unused_prefetch_slots=150 slot_utilization=0.062500 reduce_slots=23 remote_demand=5 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042647 gemm_inner_pad_upper_bound=12065 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.020867840e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 anomalies=none - [2026-08-26 15:12:38] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 364.4 | throughput per GPU (TFLOP/s/GPU): 338.9 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 9.866695E+00 | loss scale: 1.0 | grad norm: 19.098 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44422 payload_rows_remote=38684 dup_rows=21114 dup_rows_remote=18316 dedup_ratio=0.322174 router_tokens_per_expert=[501, 527, 510, 512, 530, 497, 510, 523, 559, 505, 506, 512, 535, 491, 523, 524, 497, 497, 495, 517, 511, 479, 487, 495, 549, 530, 519, 503, 526, 459, 556, 534, 501, 513, 544, 465, 533, 509, 505, 497, 497, 513, 511, 520, 514, 526, 525, 469, 504, 522, 549, 478, 534, 500, 509, 513, 494, 533, 528, 521, 504, 525, 470, 523] payload_rows_per_dst=[5444, 5502, 5454, 5422, 5592, 5572, 5698, 5738] dup_rows_per_dst=[2508, 2468, 2502, 2474, 2622, 2766, 2976, 2798] combine_rows_per_src=[5444, 5502, 5454, 5422, 5592, 5572, 5698, 5738] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44362 payload_rows_remote=38744 dup_rows=21174 dup_rows_remote=18490 dedup_ratio=0.323090 router_tokens_per_expert=[522, 509, 485, 532, 503, 457, 525, 540, 523, 511, 532, 465, 500, 562, 510, 541, 477, 514, 485, 486, 531, 512, 529, 514, 538, 552, 520, 514, 509, 516, 484, 513, 508, 545, 472, 510, 523, 510, 527, 488, 539, 573, 496, 500, 497, 523, 529, 494, 492, 486, 549, 489, 493, 536, 478, 525, 517, 531, 516, 492, 517, 468, 489, 545] payload_rows_per_dst=[5534, 5644, 5508, 5612, 5458, 5618, 5516, 5472] dup_rows_per_dst=[2612, 2644, 2588, 2680, 2708, 2684, 2580, 2678] combine_rows_per_src=[5534, 5644, 5508, 5612, 5458, 5618, 5516, 5472] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44718 payload_rows_remote=39052 dup_rows=20818 dup_rows_remote=18302 dedup_ratio=0.317657 router_tokens_per_expert=[516, 525, 549, 503, 504, 546, 528, 479, 466, 510, 495, 523, 491, 503, 522, 536, 524, 525, 508, 531, 499, 518, 486, 500, 528, 521, 515, 496, 501, 531, 544, 504, 560, 500, 519, 489, 537, 501, 463, 484, 510, 486, 503, 512, 524, 520, 482, 531, 547, 547, 495, 509, 505, 504, 501, 520, 521, 524, 482, 503, 500, 525, 503, 534] payload_rows_per_dst=[5562, 5520, 5666, 5640, 5580, 5570, 5600, 5580] dup_rows_per_dst=[2738, 2572, 2516, 2640, 2526, 2566, 2656, 2604] combine_rows_per_src=[5562, 5520, 5666, 5640, 5580, 5570, 5600, 5580] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44590 payload_rows_remote=38896 dup_rows=20946 dup_rows_remote=18242 dedup_ratio=0.319611 router_tokens_per_expert=[503, 489, 532, 518, 518, 486, 512, 509, 526, 557, 506, 516, 494, 532, 517, 493, 507, 528, 481, 515, 514, 536, 491, 531, 533, 517, 523, 506, 498, 524, 554, 544, 526, 517, 498, 487, 523, 514, 509, 519, 513, 483, 489, 508, 514, 468, 522, 521, 485, 542, 501, 523, 498, 459, 517, 488, 485, 557, 549, 501, 536, 499, 522, 485] payload_rows_per_dst=[5518, 5582, 5596, 5694, 5586, 5484, 5520, 5610] dup_rows_per_dst=[2616, 2700, 2610, 2704, 2600, 2552, 2506, 2658] combine_rows_per_src=[5518, 5582, 5596, 5694, 5586, 5484, 5520, 5610] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44366 payload_rows_remote=38900 dup_rows=21170 dup_rows_remote=18524 dedup_ratio=0.323029 router_tokens_per_expert=[494, 551, 524, 493, 522, 511, 544, 518, 498, 485, 524, 484, 512, 493, 486, 495, 517, 519, 540, 493, 487, 543, 508, 533, 548, 542, 492, 548, 478, 514, 508, 528, 530, 534, 473, 517, 508, 536, 541, 513, 510, 512, 522, 520, 495, 533, 514, 499, 506, 502, 539, 470, 481, 523, 515, 520, 497, 517, 515, 501, 501, 486, 488, 518] payload_rows_per_dst=[5590, 5464, 5626, 5592, 5618, 5554, 5466, 5456] dup_rows_per_dst=[2724, 2490, 2654, 2724, 2686, 2656, 2646, 2590] combine_rows_per_src=[5590, 5464, 5626, 5592, 5618, 5554, 5466, 5456] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44548 payload_rows_remote=38892 dup_rows=20988 dup_rows_remote=18370 dedup_ratio=0.320251 router_tokens_per_expert=[549, 495, 517, 497, 522, 519, 518, 532, 534, 537, 529, 532, 496, 499, 504, 506, 506, 496, 520, 501, 522, 473, 510, 521, 495, 541, 515, 512, 478, 510, 540, 531, 489, 507, 549, 540, 522, 516, 534, 542, 532, 477, 493, 489, 523, 520, 516, 518, 464, 481, 476, 539, 525, 498, 503, 522, 490, 502, 490, 515, 524, 502, 509, 504] payload_rows_per_dst=[5578, 5656, 5538, 5566, 5694, 5520, 5468, 5528] dup_rows_per_dst=[2720, 2618, 2560, 2678, 2704, 2616, 2548, 2544] combine_rows_per_src=[5578, 5656, 5538, 5566, 5694, 5520, 5468, 5528] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44406 payload_rows_remote=38882 dup_rows=21130 dup_rows_remote=18594 dedup_ratio=0.322418 router_tokens_per_expert=[528, 496, 496, 530, 515, 506, 545, 544, 527, 548, 527, 505, 554, 519, 511, 482, 531, 497, 507, 484, 515, 457, 514, 515, 491, 518, 483, 529, 516, 536, 525, 502, 511, 491, 481, 522, 513, 515, 497, 500, 514, 521, 513, 517, 492, 507, 509, 517, 507, 512, 507, 512, 491, 567, 510, 530, 505, 493, 494, 508, 505, 510, 510, 534] payload_rows_per_dst=[5622, 5626, 5470, 5570, 5524, 5516, 5564, 5514] dup_rows_per_dst=[2698, 2720, 2570, 2630, 2536, 2664, 2708, 2604] combine_rows_per_src=[5622, 5626, 5470, 5570, 5524, 5516, 5564, 5514] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5578452032 prefetch_bytes=990904320 reduce_bytes=1321205760 weight_movement_bytes=2312110080 egress_bytes=2790372416 ingress_bytes=5100189696 link_bytes=5100189696 total_nvlink_bytes=7890562112 weight_to_token_ratio=0.414472 prefetch_share_of_ingress=0.194288 dedup_ratio=0.321655 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=12 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.039706 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.870068718e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5574491456 prefetch_bytes=880803840 reduce_bytes=1321205760 weight_movement_bytes=2202009600 egress_bytes=2788390208 ingress_bytes=4988110848 link_bytes=4988110848 total_nvlink_bytes=7776501056 weight_to_token_ratio=0.395015 prefetch_share_of_ingress=0.176581 dedup_ratio=0.321558 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=12 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.675145168e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5575297088 prefetch_bytes=990904320 reduce_bytes=990904320 weight_movement_bytes=1981808640 egress_bytes=2788794432 ingress_bytes=4768311296 link_bytes=4768311296 total_nvlink_bytes=7557105728 weight_to_token_ratio=0.355462 prefetch_share_of_ingress=0.207810 dedup_ratio=0.321536 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=9 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.292877596e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=276025344 dispatch_meta_bytes=226424 combine_payload_bytes=276025344 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=276251768 ingress_bytes=496226304 link_bytes=496226304 token_bytes=552277112 weight_movement_bytes=220200960 total_nvlink_bytes=772478072 weight_to_token_ratio=0.398715 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.630191579e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5592390304 prefetch_bytes=990904320 reduce_bytes=220200960 weight_movement_bytes=1211105280 egress_bytes=2797343392 ingress_bytes=4006152192 link_bytes=4006152192 total_nvlink_bytes=6803495584 weight_to_token_ratio=0.216563 prefetch_share_of_ingress=0.247346 dedup_ratio=0.320859 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=2 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=6.967357561e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5562444616 prefetch_bytes=990904320 reduce_bytes=990904320 weight_movement_bytes=1981808640 egress_bytes=2782364488 ingress_bytes=4761888768 link_bytes=4761888768 total_nvlink_bytes=7544253256 weight_to_token_ratio=0.356284 prefetch_share_of_ingress=0.208091 dedup_ratio=0.322052 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=9 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.281707764e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none + [2026-08-26 16:00:26] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 367.9 | throughput per GPU (TFLOP/s/GPU): 335.6 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 9.869503E+00 | loss scale: 1.0 | grad norm: 19.160 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44574 payload_rows_remote=38924 dup_rows=20962 dup_rows_remote=18364 dedup_ratio=0.319855 router_tokens_per_expert=[547, 493, 515, 494, 519, 517, 515, 529, 534, 535, 528, 527, 494, 496, 504, 506, 505, 496, 517, 500, 521, 471, 509, 522, 497, 539, 513, 511, 480, 511, 540, 532, 489, 507, 551, 542, 520, 514, 532, 542, 532, 478, 495, 488, 524, 520, 517, 518, 463, 484, 478, 542, 528, 498, 507, 522, 493, 505, 497, 518, 526, 503, 512, 506] payload_rows_per_dst=[5558, 5650, 5530, 5568, 5696, 5532, 5488, 5552] dup_rows_per_dst=[2700, 2598, 2552, 2678, 2698, 2612, 2556, 2568] combine_rows_per_src=[5558, 5650, 5530, 5568, 5696, 5532, 5488, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44484 payload_rows_remote=38854 dup_rows=21052 dup_rows_remote=18420 dedup_ratio=0.321228 router_tokens_per_expert=[497, 525, 504, 508, 526, 496, 507, 518, 558, 505, 504, 511, 537, 488, 522, 522, 493, 495, 491, 515, 509, 477, 486, 494, 546, 531, 519, 506, 525, 454, 555, 533, 502, 512, 545, 464, 535, 510, 507, 495, 500, 514, 513, 521, 516, 528, 525, 468, 504, 524, 554, 479, 537, 501, 514, 516, 495, 537, 531, 525, 509, 531, 473, 526] payload_rows_per_dst=[5546, 5654, 5438, 5496, 5556, 5552, 5612, 5630] dup_rows_per_dst=[2616, 2640, 2482, 2544, 2584, 2742, 2812, 2632] combine_rows_per_src=[5546, 5654, 5438, 5496, 5556, 5552, 5612, 5630] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44404 payload_rows_remote=38874 dup_rows=21132 dup_rows_remote=18600 dedup_ratio=0.322449 router_tokens_per_expert=[528, 491, 491, 528, 512, 501, 540, 541, 526, 544, 524, 504, 550, 519, 510, 482, 526, 496, 507, 484, 514, 459, 515, 511, 493, 519, 481, 529, 518, 536, 524, 505, 510, 488, 482, 524, 513, 515, 500, 499, 513, 523, 512, 513, 491, 509, 509, 516, 508, 518, 509, 511, 496, 567, 518, 532, 506, 499, 495, 509, 509, 517, 514, 535] payload_rows_per_dst=[5602, 5608, 5466, 5572, 5530, 5510, 5574, 5542] dup_rows_per_dst=[2662, 2710, 2558, 2638, 2532, 2662, 2744, 2626] combine_rows_per_src=[5602, 5608, 5466, 5572, 5530, 5510, 5574, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44700 payload_rows_remote=39042 dup_rows=20836 dup_rows_remote=18328 dedup_ratio=0.317932 router_tokens_per_expert=[513, 524, 546, 494, 503, 545, 526, 476, 463, 509, 493, 520, 490, 502, 518, 533, 524, 524, 505, 531, 497, 516, 484, 502, 527, 522, 517, 494, 505, 528, 544, 503, 559, 501, 520, 489, 538, 502, 465, 486, 510, 485, 504, 510, 524, 522, 483, 532, 548, 549, 498, 507, 512, 505, 503, 521, 524, 526, 484, 505, 504, 534, 505, 535] payload_rows_per_dst=[5542, 5504, 5658, 5642, 5582, 5572, 5610, 5590] dup_rows_per_dst=[2712, 2552, 2508, 2638, 2538, 2568, 2676, 2644] combine_rows_per_src=[5542, 5504, 5658, 5642, 5582, 5572, 5610, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44384 payload_rows_remote=38758 dup_rows=21152 dup_rows_remote=18468 dedup_ratio=0.322754 router_tokens_per_expert=[519, 506, 484, 529, 501, 457, 522, 540, 521, 508, 530, 463, 497, 560, 512, 541, 475, 510, 484, 486, 531, 511, 527, 515, 538, 550, 518, 513, 506, 518, 484, 512, 507, 543, 471, 509, 527, 511, 526, 492, 538, 573, 496, 501, 500, 522, 530, 495, 491, 491, 552, 491, 495, 536, 481, 527, 519, 532, 518, 495, 519, 474, 492, 546] payload_rows_per_dst=[5520, 5636, 5506, 5612, 5460, 5626, 5536, 5488] dup_rows_per_dst=[2596, 2628, 2572, 2666, 2712, 2684, 2592, 2702] combine_rows_per_src=[5520, 5636, 5506, 5612, 5460, 5626, 5536, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44578 payload_rows_remote=38886 dup_rows=20958 dup_rows_remote=18256 dedup_ratio=0.319794 router_tokens_per_expert=[501, 487, 528, 517, 517, 483, 504, 510, 525, 555, 503, 513, 492, 529, 516, 493, 505, 525, 478, 513, 514, 533, 493, 530, 532, 517, 522, 505, 498, 524, 553, 546, 527, 519, 499, 485, 523, 515, 511, 518, 515, 484, 490, 508, 514, 470, 524, 524, 490, 544, 501, 525, 498, 461, 519, 490, 486, 559, 550, 503, 539, 500, 525, 491] payload_rows_per_dst=[5504, 5570, 5588, 5692, 5592, 5488, 5522, 5622] dup_rows_per_dst=[2590, 2682, 2594, 2702, 2602, 2570, 2534, 2684] combine_rows_per_src=[5504, 5570, 5588, 5692, 5592, 5488, 5522, 5622] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44372 payload_rows_remote=38890 dup_rows=21164 dup_rows_remote=18504 dedup_ratio=0.322937 router_tokens_per_expert=[489, 551, 520, 490, 518, 512, 541, 514, 496, 482, 522, 483, 510, 491, 484, 496, 517, 519, 537, 496, 487, 541, 508, 539, 545, 543, 491, 546, 479, 513, 507, 527, 531, 534, 472, 518, 508, 533, 543, 514, 510, 512, 524, 519, 496, 534, 517, 497, 508, 505, 540, 472, 484, 523, 518, 521, 500, 518, 517, 501, 503, 489, 491, 522] payload_rows_per_dst=[5578, 5450, 5626, 5576, 5620, 5564, 5482, 5476] dup_rows_per_dst=[2692, 2478, 2662, 2726, 2686, 2654, 2660, 2606] combine_rows_per_src=[5578, 5450, 5626, 5576, 5620, 5564, 5482, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279007232 dispatch_meta_bytes=229152 combine_payload_bytes=279007232 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279236384 ingress_bytes=499208192 link_bytes=499208192 token_bytes=558243616 weight_movement_bytes=220200960 total_nvlink_bytes=778444576 weight_to_token_ratio=0.394453 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.682051515e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=277286912 dispatch_meta_bytes=228000 combine_payload_bytes=277286912 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277514912 ingress_bytes=387387392 link_bytes=387387392 token_bytes=554801824 weight_movement_bytes=110100480 total_nvlink_bytes=664902304 weight_to_token_ratio=0.198450 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.737303890e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=277716992 dispatch_meta_bytes=228936 combine_payload_bytes=277716992 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277945928 ingress_bytes=387817472 link_bytes=387817472 token_bytes=555662920 weight_movement_bytes=110100480 total_nvlink_bytes=665763400 weight_to_token_ratio=0.198143 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.744783688e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44560 payload_rows_remote=39020 dup_rows=20976 dup_rows_remote=18444 dedup_ratio=0.320068 router_tokens_per_expert=[506, 479, 468, 515, 508, 453, 564, 540, 510, 509, 525, 521, 546, 513, 507, 522, 545, 518, 531, 520, 491, 500, 524, 509, 495, 546, 513, 529, 518, 517, 506, 547, 504, 534, 526, 497, 474, 489, 533, 494, 513, 505, 511, 508, 512, 544, 484, 522, 522, 479, 500, 489, 524, 522, 505, 501, 517, 518, 488, 518, 490, 551, 463, 536] payload_rows_per_dst=[5540, 5634, 5802, 5392, 5494, 5612, 5548, 5538] dup_rows_per_dst=[2532, 2696, 2876, 2518, 2608, 2586, 2536, 2624] combine_rows_per_src=[5540, 5634, 5802, 5392, 5494, 5612, 5548, 5538] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279924736 dispatch_meta_bytes=229416 combine_payload_bytes=279924736 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280154152 ingress_bytes=390025216 link_bytes=390025216 token_bytes=560078888 weight_movement_bytes=110100480 total_nvlink_bytes=670179368 weight_to_token_ratio=0.196580 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.783179987e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278806528 dispatch_meta_bytes=228552 combine_payload_bytes=278806528 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=279035080 ingress_bytes=719208448 link_bytes=719208448 token_bytes=557841608 weight_movement_bytes=440401920 total_nvlink_bytes=998243528 weight_to_token_ratio=0.789475 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.250821781e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278835200 dispatch_meta_bytes=229696 combine_payload_bytes=278835200 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279064896 ingress_bytes=388935680 link_bytes=388935680 token_bytes=557900096 weight_movement_bytes=110100480 total_nvlink_bytes=668000576 weight_to_token_ratio=0.197348 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.764231164e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278777856 dispatch_meta_bytes=229048 combine_payload_bytes=278777856 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279006904 ingress_bytes=498978816 link_bytes=498978816 token_bytes=557784760 weight_movement_bytes=220200960 total_nvlink_bytes=777985720 weight_to_token_ratio=0.394778 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.678062289e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278706176 dispatch_meta_bytes=229904 combine_payload_bytes=278706176 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278936080 ingress_bytes=388806656 link_bytes=388806656 token_bytes=557642256 weight_movement_bytes=110100480 total_nvlink_bytes=667742736 weight_to_token_ratio=0.197439 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.761987225e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279695360 dispatch_meta_bytes=229856 combine_payload_bytes=279695360 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279925216 ingress_bytes=609996800 link_bytes=609996800 token_bytes=559620576 weight_movement_bytes=330301440 total_nvlink_bytes=889922016 weight_to_token_ratio=0.590224 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060884762e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44456 payload_rows_remote=38552 dup_rows=21080 dup_rows_remote=18034 dedup_ratio=0.321655 router_tokens_per_expert=[553, 475, 513, 542, 533, 505, 515, 509, 501, 505, 502, 527, 534, 538, 522, 517, 547, 511, 507, 514, 498, 501, 526, 538, 509, 526, 468, 527, 496, 510, 504, 481, 508, 540, 534, 491, 486, 523, 496, 496, 522, 512, 522, 494, 510, 529, 527, 500, 494, 486, 536, 536, 503, 499, 513, 475, 509, 495, 538, 497, 489, 541, 517, 496] payload_rows_per_dst=[5246, 5626, 5584, 5186, 5486, 5582, 5842, 5904] dup_rows_per_dst=[2166, 2666, 2694, 2070, 2662, 2656, 3120, 3046] combine_rows_per_src=[5246, 5626, 5584, 5186, 5486, 5582, 5842, 5904] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44558 payload_rows_remote=38960 dup_rows=20978 dup_rows_remote=18372 dedup_ratio=0.320099 router_tokens_per_expert=[508, 566, 521, 556, 539, 495, 517, 527, 516, 536, 496, 486, 522, 464, 529, 517, 511, 505, 506, 505, 514, 525, 548, 536, 526, 499, 518, 494, 490, 538, 500, 528, 550, 533, 510, 487, 554, 519, 500, 513, 502, 534, 530, 523, 514, 494, 489, 516, 506, 526, 490, 497, 515, 479, 482, 486, 488, 486, 496, 524, 471, 476, 510, 530] payload_rows_per_dst=[5610, 5582, 5616, 5554, 5644, 5598, 5494, 5460] dup_rows_per_dst=[2848, 2550, 2684, 2632, 2688, 2606, 2468, 2502] combine_rows_per_src=[5610, 5582, 5616, 5554, 5644, 5598, 5494, 5460] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44494 payload_rows_remote=38954 dup_rows=21042 dup_rows_remote=18426 dedup_ratio=0.321075 router_tokens_per_expert=[526, 513, 522, 517, 533, 497, 566, 538, 510, 504, 498, 496, 501, 522, 502, 467, 516, 475, 531, 507, 543, 467, 527, 512, 520, 521, 531, 543, 534, 571, 506, 512, 542, 507, 506, 500, 520, 500, 478, 527, 504, 478, 493, 523, 514, 491, 487, 566, 484, 508, 480, 515, 475, 520, 529, 479, 518, 544, 492, 508, 480, 519, 498, 555] payload_rows_per_dst=[5596, 5470, 5540, 5700, 5592, 5536, 5514, 5546] dup_rows_per_dst=[2828, 2530, 2616, 2776, 2568, 2576, 2466, 2682] combine_rows_per_src=[5596, 5470, 5540, 5700, 5592, 5536, 5514, 5546] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44560 payload_rows_remote=38882 dup_rows=20976 dup_rows_remote=18286 dedup_ratio=0.320068 router_tokens_per_expert=[522, 518, 498, 511, 531, 553, 528, 511, 487, 546, 491, 521, 511, 526, 504, 504, 513, 531, 509, 511, 482, 512, 522, 501, 546, 505, 514, 536, 509, 526, 519, 529, 522, 527, 521, 508, 479, 491, 510, 509, 487, 517, 494, 515, 468, 512, 534, 517, 520, 547, 517, 492, 511, 513, 468, 548, 505, 497, 525, 520, 478, 497, 494, 498] payload_rows_per_dst=[5682, 5614, 5568, 5678, 5544, 5442, 5530, 5502] dup_rows_per_dst=[2662, 2566, 2594, 2690, 2590, 2646, 2702, 2526] combine_rows_per_src=[5682, 5614, 5568, 5678, 5544, 5442, 5530, 5502] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44440 payload_rows_remote=38926 dup_rows=21096 dup_rows_remote=18490 dedup_ratio=0.321899 router_tokens_per_expert=[528, 495, 516, 502, 535, 483, 499, 504, 533, 520, 503, 523, 512, 501, 492, 472, 515, 519, 526, 509, 514, 518, 512, 499, 500, 498, 503, 527, 531, 493, 530, 513, 491, 560, 494, 525, 543, 523, 529, 512, 510, 533, 533, 503, 513, 507, 519, 489, 517, 532, 502, 522, 513, 467, 483, 524, 511, 538, 514, 502, 502, 515, 524, 493] payload_rows_per_dst=[5492, 5496, 5588, 5606, 5602, 5578, 5514, 5564] dup_rows_per_dst=[2632, 2616, 2636, 2584, 2752, 2636, 2606, 2634] combine_rows_per_src=[5492, 5496, 5588, 5606, 5602, 5578, 5514, 5564] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44398 payload_rows_remote=38874 dup_rows=21138 dup_rows_remote=18568 dedup_ratio=0.322540 router_tokens_per_expert=[516, 512, 523, 500, 515, 500, 505, 491, 494, 479, 529, 484, 546, 516, 511, 488, 545, 492, 535, 541, 502, 545, 533, 505, 519, 537, 518, 514, 531, 523, 485, 532, 497, 475, 528, 500, 505, 505, 526, 487, 490, 533, 509, 509, 559, 523, 534, 497, 479, 518, 520, 522, 491, 528, 515, 509, 504, 528, 503, 534, 486, 527, 507, 454] payload_rows_per_dst=[5536, 5524, 5594, 5604, 5456, 5636, 5536, 5512] dup_rows_per_dst=[2588, 2570, 2802, 2714, 2590, 2672, 2628, 2574] combine_rows_per_src=[5536, 5524, 5594, 5604, 5456, 5636, 5536, 5512] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=276340736 dispatch_meta_bytes=226344 combine_payload_bytes=276340736 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276567080 ingress_bytes=386441216 link_bytes=386441216 token_bytes=552907816 weight_movement_bytes=110100480 total_nvlink_bytes=663008296 weight_to_token_ratio=0.199130 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.720848333e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44530 payload_rows_remote=39012 dup_rows=21006 dup_rows_remote=18350 dedup_ratio=0.320526 router_tokens_per_expert=[469, 502, 502, 519, 508, 518, 525, 499, 524, 490, 502, 525, 504, 507, 528, 556, 524, 534, 498, 487, 545, 516, 500, 498, 529, 547, 517, 519, 502, 531, 501, 514, 519, 538, 532, 516, 484, 527, 491, 480, 494, 525, 529, 543, 506, 549, 511, 550, 516, 508, 492, 515, 506, 498, 498, 489, 509, 500, 502, 520, 489, 491, 507, 494] payload_rows_per_dst=[5530, 5680, 5594, 5642, 5518, 5604, 5508, 5454] dup_rows_per_dst=[2554, 2592, 2610, 2678, 2656, 2810, 2536, 2570] combine_rows_per_src=[5530, 5680, 5594, 5642, 5518, 5604, 5508, 5454] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279265280 dispatch_meta_bytes=229328 combine_payload_bytes=279265280 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279494608 ingress_bytes=609566720 link_bytes=609566720 token_bytes=558759888 weight_movement_bytes=330301440 total_nvlink_bytes=889061328 weight_to_token_ratio=0.591133 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060136783e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44536 payload_rows_remote=38992 dup_rows=21000 dup_rows_remote=18426 dedup_ratio=0.320435 router_tokens_per_expert=[505, 476, 463, 511, 504, 450, 561, 537, 510, 507, 521, 519, 544, 509, 507, 521, 545, 515, 531, 518, 491, 497, 521, 509, 494, 550, 511, 530, 516, 514, 505, 546, 503, 535, 528, 499, 472, 492, 533, 494, 515, 506, 510, 509, 512, 542, 485, 521, 526, 481, 503, 491, 532, 523, 512, 501, 522, 521, 493, 521, 491, 556, 464, 537] payload_rows_per_dst=[5544, 5568, 5832, 5360, 5512, 5612, 5566, 5542] dup_rows_per_dst=[2574, 2608, 2964, 2426, 2636, 2588, 2572, 2632] combine_rows_per_src=[5544, 5568, 5832, 5360, 5512, 5612, 5566, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278505472 dispatch_meta_bytes=229096 combine_payload_bytes=278505472 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278734568 ingress_bytes=498706432 link_bytes=498706432 token_bytes=557240040 weight_movement_bytes=220200960 total_nvlink_bytes=777441000 weight_to_token_ratio=0.395164 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.673325084e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279853056 dispatch_meta_bytes=229480 combine_payload_bytes=279853056 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280082536 ingress_bytes=389953536 link_bytes=389953536 token_bytes=559935592 weight_movement_bytes=110100480 total_nvlink_bytes=670036072 weight_to_token_ratio=0.196631 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.781933354e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278648832 dispatch_meta_bytes=229896 combine_payload_bytes=278648832 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278878728 ingress_bytes=388749312 link_bytes=388749312 token_bytes=557527560 weight_movement_bytes=110100480 total_nvlink_bytes=667628040 weight_to_token_ratio=0.197480 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.760989918e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=277817344 dispatch_meta_bytes=228904 combine_payload_bytes=277817344 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278046248 ingress_bytes=387917824 link_bytes=387917824 token_bytes=555863592 weight_movement_bytes=110100480 total_nvlink_bytes=665964072 weight_to_token_ratio=0.198071 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.746528975e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278734848 dispatch_meta_bytes=228568 combine_payload_bytes=278734848 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=278963416 ingress_bytes=829237248 link_bytes=829237248 token_bytes=557698264 weight_movement_bytes=550502400 total_nvlink_bytes=1108200664 weight_to_token_ratio=0.987097 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.442179960e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278763520 dispatch_meta_bytes=229576 combine_payload_bytes=278763520 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278993096 ingress_bytes=388864000 link_bytes=388864000 token_bytes=557756616 weight_movement_bytes=110100480 total_nvlink_bytes=667857096 weight_to_token_ratio=0.197399 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.762984531e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44412 payload_rows_remote=38902 dup_rows=21124 dup_rows_remote=18574 dedup_ratio=0.322327 router_tokens_per_expert=[510, 510, 520, 501, 514, 497, 501, 489, 490, 478, 527, 483, 544, 515, 511, 482, 545, 494, 533, 539, 502, 543, 530, 503, 514, 535, 517, 511, 533, 520, 486, 531, 500, 475, 527, 501, 507, 507, 526, 487, 491, 533, 510, 515, 562, 527, 536, 497, 482, 520, 521, 524, 492, 531, 516, 511, 505, 530, 506, 536, 489, 530, 508, 458] payload_rows_per_dst=[5526, 5510, 5592, 5598, 5458, 5654, 5546, 5528] dup_rows_per_dst=[2558, 2550, 2786, 2696, 2602, 2688, 2648, 2596] combine_rows_per_src=[5526, 5510, 5592, 5598, 5458, 5654, 5546, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44368 payload_rows_remote=38596 dup_rows=21168 dup_rows_remote=18290 dedup_ratio=0.322998 router_tokens_per_expert=[546, 472, 508, 542, 528, 502, 513, 505, 499, 503, 502, 527, 534, 539, 519, 516, 547, 509, 508, 516, 495, 498, 524, 538, 510, 525, 472, 527, 497, 508, 504, 478, 510, 541, 533, 491, 485, 525, 495, 497, 520, 512, 521, 497, 513, 531, 528, 500, 495, 487, 538, 536, 507, 503, 516, 478, 511, 498, 538, 501, 490, 542, 519, 499] payload_rows_per_dst=[5466, 5618, 5594, 5330, 5496, 5338, 5754, 5772] dup_rows_per_dst=[2552, 2660, 2724, 2286, 2700, 2372, 2996, 2878] combine_rows_per_src=[5466, 5618, 5594, 5330, 5496, 5338, 5754, 5772] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44490 payload_rows_remote=38964 dup_rows=21046 dup_rows_remote=18444 dedup_ratio=0.321136 router_tokens_per_expert=[526, 507, 520, 514, 526, 493, 562, 537, 508, 501, 497, 496, 500, 519, 501, 464, 514, 470, 529, 505, 545, 466, 526, 509, 514, 523, 533, 543, 533, 571, 502, 511, 545, 508, 503, 500, 525, 501, 479, 526, 505, 479, 496, 524, 515, 494, 489, 568, 486, 508, 482, 523, 477, 523, 531, 481, 521, 551, 493, 509, 481, 521, 503, 556] payload_rows_per_dst=[5572, 5458, 5526, 5690, 5596, 5548, 5538, 5562] dup_rows_per_dst=[2798, 2514, 2602, 2770, 2578, 2592, 2484, 2708] combine_rows_per_src=[5572, 5458, 5526, 5690, 5596, 5548, 5538, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279494656 dispatch_meta_bytes=229672 combine_payload_bytes=279494656 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279724328 ingress_bytes=389595136 link_bytes=389595136 token_bytes=559218984 weight_movement_bytes=110100480 total_nvlink_bytes=669319464 weight_to_token_ratio=0.196883 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.775700189e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279021568 dispatch_meta_bytes=229664 combine_payload_bytes=279021568 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279251232 ingress_bytes=389122048 link_bytes=389122048 token_bytes=558272800 weight_movement_bytes=110100480 total_nvlink_bytes=668373280 weight_to_token_ratio=0.197216 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.767472411e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44394 payload_rows_remote=38664 dup_rows=21142 dup_rows_remote=18306 dedup_ratio=0.322601 router_tokens_per_expert=[530, 515, 549, 533, 485, 477, 498, 542, 487, 520, 553, 528, 527, 540, 497, 506, 534, 506, 495, 510, 517, 536, 523, 510, 548, 507, 497, 505, 500, 494, 529, 499, 494, 509, 513, 476, 473, 519, 531, 523, 534, 502, 541, 509, 477, 509, 556, 539, 516, 493, 509, 508, 473, 542, 502, 493, 546, 494, 497, 503, 496, 500, 507, 487] payload_rows_per_dst=[5730, 5746, 5354, 5646, 5530, 5438, 5494, 5456] dup_rows_per_dst=[2836, 2708, 2462, 2776, 2658, 2520, 2578, 2604] combine_rows_per_src=[5730, 5746, 5354, 5646, 5530, 5438, 5494, 5456] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279222272 dispatch_meta_bytes=229520 combine_payload_bytes=279222272 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279451792 ingress_bytes=609523712 link_bytes=609523712 token_bytes=558674064 weight_movement_bytes=330301440 total_nvlink_bytes=888975504 weight_to_token_ratio=0.591224 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060061985e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278648832 dispatch_meta_bytes=229768 combine_payload_bytes=278648832 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278878600 ingress_bytes=388749312 link_bytes=388749312 token_bytes=557527432 weight_movement_bytes=110100480 total_nvlink_bytes=667627912 weight_to_token_ratio=0.197480 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.760989918e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278706176 dispatch_meta_bytes=228672 combine_payload_bytes=278706176 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278934848 ingress_bytes=498907136 link_bytes=498907136 token_bytes=557641024 weight_movement_bytes=220200960 total_nvlink_bytes=777841984 weight_to_token_ratio=0.394879 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.676815656e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279638016 dispatch_meta_bytes=229448 combine_payload_bytes=279638016 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279867464 ingress_bytes=389738496 link_bytes=389738496 token_bytes=559505480 weight_movement_bytes=110100480 total_nvlink_bytes=669605960 weight_to_token_ratio=0.196782 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778193455e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277143552 dispatch_meta_bytes=227880 combine_payload_bytes=277143552 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277371432 ingress_bytes=497344512 link_bytes=497344512 token_bytes=554514984 weight_movement_bytes=220200960 total_nvlink_bytes=774715944 weight_to_token_ratio=0.397106 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.649639055e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44566 payload_rows_remote=38706 dup_rows=20970 dup_rows_remote=17880 dedup_ratio=0.319977 router_tokens_per_expert=[515, 528, 503, 507, 543, 493, 547, 487, 546, 502, 513, 519, 502, 527, 545, 524, 484, 482, 544, 526, 500, 496, 516, 538, 525, 505, 509, 491, 548, 534, 482, 511, 522, 533, 527, 510, 469, 484, 547, 459, 483, 517, 519, 510, 503, 508, 521, 532, 520, 536, 529, 511, 522, 532, 491, 500, 496, 499, 481, 526, 479, 522, 514, 474] payload_rows_per_dst=[5182, 5570, 5592, 5558, 5602, 5582, 5620, 5860] dup_rows_per_dst=[2096, 2556, 2580, 2628, 2642, 2692, 2686, 3090] combine_rows_per_src=[5182, 5570, 5592, 5558, 5602, 5582, 5620, 5860] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44410 payload_rows_remote=38888 dup_rows=21126 dup_rows_remote=18550 dedup_ratio=0.322357 router_tokens_per_expert=[476, 537, 520, 504, 500, 520, 509, 531, 542, 488, 511, 537, 512, 526, 517, 505, 512, 494, 521, 537, 551, 534, 515, 487, 543, 511, 509, 501, 535, 493, 492, 518, 538, 494, 439, 504, 483, 506, 531, 519, 523, 507, 505, 488, 498, 506, 503, 519, 530, 503, 548, 519, 515, 518, 500, 502, 477, 547, 476, 531, 518, 512, 532, 489] payload_rows_per_dst=[5560, 5564, 5606, 5524, 5540, 5522, 5562, 5532] dup_rows_per_dst=[2634, 2712, 2696, 2680, 2488, 2576, 2708, 2632] combine_rows_per_src=[5560, 5564, 5606, 5524, 5540, 5522, 5562, 5532] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44636 payload_rows_remote=39092 dup_rows=20900 dup_rows_remote=18320 dedup_ratio=0.318909 router_tokens_per_expert=[552, 526, 514, 491, 553, 506, 462, 489, 505, 528, 505, 520, 492, 482, 492, 519, 524, 520, 541, 515, 515, 504, 500, 493, 520, 524, 533, 530, 498, 523, 474, 490, 502, 525, 536, 514, 541, 526, 538, 516, 492, 539, 503, 518, 524, 492, 520, 522, 516, 534, 511, 474, 511, 514, 503, 499, 494, 508, 520, 481, 522, 517, 514, 502] payload_rows_per_dst=[5568, 5500, 5614, 5606, 5648, 5616, 5544, 5540] dup_rows_per_dst=[2618, 2586, 2610, 2578, 2748, 2604, 2580, 2576] combine_rows_per_src=[5568, 5500, 5614, 5606, 5648, 5616, 5544, 5540] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44518 payload_rows_remote=38910 dup_rows=21018 dup_rows_remote=18378 dedup_ratio=0.320709 router_tokens_per_expert=[515, 549, 542, 527, 481, 521, 544, 518, 458, 482, 518, 496, 485, 495, 523, 529, 513, 524, 532, 480, 506, 524, 518, 527, 501, 473, 554, 540, 522, 522, 492, 509, 498, 540, 499, 503, 491, 503, 508, 502, 514, 520, 534, 482, 552, 492, 520, 488, 490, 528, 511, 533, 514, 494, 485, 537, 525, 493, 506, 503, 525, 519, 527, 512] payload_rows_per_dst=[5652, 5456, 5608, 5552, 5490, 5570, 5586, 5604] dup_rows_per_dst=[2742, 2516, 2640, 2674, 2598, 2634, 2598, 2616] combine_rows_per_src=[5652, 5456, 5608, 5552, 5490, 5570, 5586, 5604] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44422 payload_rows_remote=38844 dup_rows=21114 dup_rows_remote=18442 dedup_ratio=0.322174 router_tokens_per_expert=[493, 514, 521, 540, 506, 527, 503, 544, 532, 515, 564, 518, 494, 477, 501, 524, 523, 512, 518, 510, 466, 523, 516, 524, 504, 506, 539, 531, 522, 527, 474, 520, 483, 521, 558, 518, 451, 549, 541, 515, 501, 500, 495, 513, 518, 514, 506, 470, 539, 509, 516, 541, 525, 512, 508, 500, 503, 505, 526, 507, 421, 509, 515, 491] payload_rows_per_dst=[5642, 5578, 5574, 5550, 5636, 5458, 5588, 5396] dup_rows_per_dst=[2654, 2672, 2610, 2696, 2636, 2576, 2712, 2558] combine_rows_per_src=[5642, 5578, 5574, 5550, 5636, 5458, 5588, 5396] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277444608 dispatch_meta_bytes=226344 combine_payload_bytes=277444608 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277670952 ingress_bytes=387545088 link_bytes=387545088 token_bytes=555115560 weight_movement_bytes=110100480 total_nvlink_bytes=665216040 weight_to_token_ratio=0.198338 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.740046482e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44610 payload_rows_remote=38934 dup_rows=20926 dup_rows_remote=18188 dedup_ratio=0.319305 router_tokens_per_expert=[556, 537, 493, 504, 518, 518, 492, 516, 531, 554, 487, 499, 486, 509, 533, 510, 519, 507, 517, 535, 506, 504, 498, 488, 490, 526, 557, 550, 526, 519, 500, 539, 518, 545, 505, 497, 474, 466, 524, 495, 541, 479, 541, 502, 510, 497, 518, 517, 539, 526, 520, 542, 513, 490, 521, 493, 472, 544, 495, 490, 503, 501, 460, 506] payload_rows_per_dst=[5636, 5596, 5528, 5676, 5504, 5584, 5676, 5410] dup_rows_per_dst=[2632, 2622, 2620, 2738, 2544, 2626, 2612, 2532] combine_rows_per_src=[5636, 5596, 5528, 5676, 5504, 5584, 5676, 5410] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44334 payload_rows_remote=38752 dup_rows=21202 dup_rows_remote=18564 dedup_ratio=0.323517 router_tokens_per_expert=[502, 517, 578, 507, 506, 501, 497, 528, 526, 522, 493, 533, 518, 514, 505, 525, 507, 496, 555, 495, 489, 494, 493, 492, 531, 532, 515, 512, 484, 521, 521, 521, 528, 514, 511, 507, 544, 498, 523, 485, 462, 547, 537, 518, 499, 524, 527, 530, 517, 508, 492, 531, 463, 461, 512, 484, 521, 492, 521, 526, 509, 522, 496, 529] payload_rows_per_dst=[5566, 5606, 5470, 5562, 5582, 5566, 5414, 5568] dup_rows_per_dst=[2706, 2666, 2572, 2712, 2638, 2722, 2522, 2664] combine_rows_per_src=[5566, 5606, 5470, 5562, 5582, 5566, 5414, 5568] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278749184 dispatch_meta_bytes=229752 combine_payload_bytes=278749184 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278978936 ingress_bytes=388849664 link_bytes=388849664 token_bytes=557728120 weight_movement_bytes=110100480 total_nvlink_bytes=667828600 weight_to_token_ratio=0.197409 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.762735205e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280211456 dispatch_meta_bytes=229648 combine_payload_bytes=280211456 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280441104 ingress_bytes=390311936 link_bytes=390311936 token_bytes=560652560 weight_movement_bytes=110100480 total_nvlink_bytes=670753040 weight_to_token_ratio=0.196379 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.788166520e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278906880 dispatch_meta_bytes=229152 combine_payload_bytes=278906880 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279136032 ingress_bytes=389007360 link_bytes=389007360 token_bytes=558042912 weight_movement_bytes=110100480 total_nvlink_bytes=668143392 weight_to_token_ratio=0.197298 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.765477798e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44638 payload_rows_remote=38888 dup_rows=20898 dup_rows_remote=18000 dedup_ratio=0.318878 router_tokens_per_expert=[494, 528, 546, 500, 529, 507, 530, 504, 518, 560, 543, 508, 506, 498, 508, 518, 561, 516, 486, 516, 480, 521, 508, 514, 516, 504, 503, 544, 484, 492, 516, 543, 474, 521, 529, 491, 511, 529, 526, 539, 525, 477, 521, 528, 508, 544, 500, 501, 487, 510, 499, 513, 511, 487, 537, 520, 450, 498, 509, 502, 508, 508, 480, 524] payload_rows_per_dst=[5750, 5616, 5592, 5426, 5602, 5610, 5524, 5518] dup_rows_per_dst=[2898, 2720, 2624, 2376, 2638, 2598, 2604, 2440] combine_rows_per_src=[5750, 5616, 5592, 5426, 5602, 5610, 5524, 5518] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278433792 dispatch_meta_bytes=229144 combine_payload_bytes=278433792 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278662936 ingress_bytes=608735232 link_bytes=608735232 token_bytes=557096728 weight_movement_bytes=330301440 total_nvlink_bytes=887398168 weight_to_token_ratio=0.592898 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058690688e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277774336 dispatch_meta_bytes=229264 combine_payload_bytes=277774336 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278003600 ingress_bytes=387874816 link_bytes=387874816 token_bytes=555777936 weight_movement_bytes=110100480 total_nvlink_bytes=665878416 weight_to_token_ratio=0.198102 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.745780995e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279078912 dispatch_meta_bytes=228488 combine_payload_bytes=279078912 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=279307400 ingress_bytes=719480832 link_bytes=719480832 token_bytes=558386312 weight_movement_bytes=440401920 total_nvlink_bytes=998788232 weight_to_token_ratio=0.788705 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.251295501e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278749184 dispatch_meta_bytes=227552 combine_payload_bytes=278749184 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278976736 ingress_bytes=498950144 link_bytes=498950144 token_bytes=557725920 weight_movement_bytes=220200960 total_nvlink_bytes=777926880 weight_to_token_ratio=0.394819 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.677563636e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44310 payload_rows_remote=38536 dup_rows=21226 dup_rows_remote=18182 dedup_ratio=0.323883 router_tokens_per_expert=[501, 533, 521, 529, 529, 492, 522, 549, 531, 533, 509, 548, 500, 501, 523, 501, 514, 506, 523, 528, 500, 510, 528, 498, 523, 489, 545, 481, 534, 505, 516, 480, 505, 495, 500, 502, 511, 545, 502, 551, 543, 491, 521, 504, 508, 507, 483, 503, 514, 503, 519, 486, 541, 528, 532, 466, 509, 511, 511, 487, 525, 487, 488, 488] payload_rows_per_dst=[5226, 5576, 5650, 5296, 5460, 5604, 5724, 5774] dup_rows_per_dst=[2154, 2670, 2730, 2330, 2612, 2706, 2980, 3044] combine_rows_per_src=[5226, 5576, 5650, 5296, 5460, 5604, 5724, 5774] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44588 payload_rows_remote=38932 dup_rows=20948 dup_rows_remote=18276 dedup_ratio=0.319641 router_tokens_per_expert=[489, 550, 517, 558, 520, 524, 538, 534, 492, 542, 499, 492, 557, 495, 484, 528, 526, 531, 490, 482, 547, 498, 512, 522, 504, 505, 516, 545, 521, 518, 486, 537, 486, 512, 531, 494, 493, 527, 474, 489, 502, 538, 521, 524, 540, 508, 504, 527, 493, 512, 523, 505, 505, 511, 462, 508, 480, 503, 537, 463, 496, 524, 507, 510] payload_rows_per_dst=[5648, 5560, 5588, 5632, 5498, 5656, 5488, 5518] dup_rows_per_dst=[2812, 2618, 2628, 2632, 2514, 2672, 2550, 2522] combine_rows_per_src=[5648, 5560, 5588, 5632, 5498, 5656, 5488, 5518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278849536 dispatch_meta_bytes=229904 combine_payload_bytes=278849536 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279079440 ingress_bytes=388950016 link_bytes=388950016 token_bytes=557928976 weight_movement_bytes=110100480 total_nvlink_bytes=668029456 weight_to_token_ratio=0.197338 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.764480491e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=276656128 dispatch_meta_bytes=227544 combine_payload_bytes=276656128 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276883672 ingress_bytes=386756608 link_bytes=386756608 token_bytes=553539800 weight_movement_bytes=110100480 total_nvlink_bytes=663640280 weight_to_token_ratio=0.198903 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.726333518e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279293952 dispatch_meta_bytes=229632 combine_payload_bytes=279293952 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279523584 ingress_bytes=499494912 link_bytes=499494912 token_bytes=558817536 weight_movement_bytes=220200960 total_nvlink_bytes=779018496 weight_to_token_ratio=0.394048 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.687038048e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44560 payload_rows_remote=38964 dup_rows=20976 dup_rows_remote=18366 dedup_ratio=0.320068 router_tokens_per_expert=[507, 562, 519, 555, 538, 494, 513, 526, 516, 535, 494, 485, 521, 462, 527, 516, 510, 504, 505, 504, 515, 525, 548, 536, 522, 497, 517, 493, 489, 537, 503, 532, 550, 535, 510, 490, 554, 520, 502, 511, 503, 534, 529, 523, 512, 496, 490, 516, 506, 528, 494, 495, 517, 483, 483, 487, 491, 488, 498, 522, 472, 478, 512, 532] payload_rows_per_dst=[5598, 5574, 5610, 5554, 5656, 5596, 5504, 5468] dup_rows_per_dst=[2830, 2538, 2684, 2626, 2688, 2610, 2482, 2518] combine_rows_per_src=[5598, 5574, 5610, 5554, 5656, 5596, 5504, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44538 payload_rows_remote=39018 dup_rows=20998 dup_rows_remote=18342 dedup_ratio=0.320404 router_tokens_per_expert=[464, 499, 497, 515, 505, 516, 522, 496, 522, 488, 500, 521, 502, 505, 524, 555, 522, 533, 494, 489, 545, 517, 499, 499, 529, 544, 517, 517, 500, 529, 502, 513, 519, 538, 532, 517, 483, 531, 490, 478, 497, 527, 529, 545, 505, 552, 516, 554, 518, 510, 493, 517, 506, 498, 501, 493, 509, 503, 503, 522, 493, 498, 514, 497] payload_rows_per_dst=[5514, 5652, 5594, 5640, 5520, 5612, 5524, 5482] dup_rows_per_dst=[2514, 2582, 2602, 2662, 2656, 2838, 2548, 2596] combine_rows_per_src=[5514, 5652, 5594, 5640, 5520, 5612, 5524, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44428 payload_rows_remote=38914 dup_rows=21108 dup_rows_remote=18476 dedup_ratio=0.322083 router_tokens_per_expert=[524, 493, 513, 498, 533, 483, 499, 501, 531, 518, 500, 521, 509, 499, 492, 470, 514, 518, 526, 509, 512, 518, 511, 499, 498, 495, 504, 529, 530, 494, 531, 511, 488, 559, 496, 527, 542, 524, 530, 511, 508, 535, 532, 505, 515, 507, 519, 490, 520, 533, 503, 522, 515, 468, 484, 528, 516, 539, 516, 506, 504, 517, 525, 501] payload_rows_per_dst=[5478, 5482, 5586, 5600, 5600, 5584, 5514, 5584] dup_rows_per_dst=[2610, 2598, 2628, 2584, 2754, 2638, 2632, 2664] combine_rows_per_src=[5478, 5482, 5586, 5600, 5600, 5584, 5514, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44560 payload_rows_remote=38888 dup_rows=20976 dup_rows_remote=18288 dedup_ratio=0.320068 router_tokens_per_expert=[517, 515, 496, 509, 529, 550, 527, 507, 479, 545, 489, 519, 509, 525, 500, 501, 510, 530, 506, 510, 481, 508, 521, 500, 546, 505, 517, 535, 505, 526, 518, 528, 524, 527, 524, 510, 479, 491, 512, 513, 489, 518, 494, 516, 468, 513, 536, 517, 520, 549, 520, 496, 513, 515, 471, 547, 508, 501, 527, 523, 483, 501, 499, 501] payload_rows_per_dst=[5668, 5588, 5552, 5672, 5556, 5446, 5546, 5532] dup_rows_per_dst=[2632, 2546, 2580, 2688, 2604, 2656, 2716, 2554] combine_rows_per_src=[5668, 5588, 5552, 5672, 5556, 5446, 5546, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44408 payload_rows_remote=38810 dup_rows=21128 dup_rows_remote=18506 dedup_ratio=0.322388 router_tokens_per_expert=[526, 511, 547, 530, 485, 475, 496, 540, 486, 520, 551, 526, 524, 541, 497, 502, 533, 506, 495, 510, 516, 534, 523, 507, 543, 506, 497, 504, 499, 494, 528, 499, 495, 509, 512, 473, 474, 520, 531, 522, 540, 502, 543, 509, 476, 513, 560, 539, 515, 492, 512, 512, 475, 543, 504, 499, 549, 497, 499, 503, 498, 501, 509, 491] payload_rows_per_dst=[5598, 5818, 5392, 5522, 5490, 5604, 5512, 5472] dup_rows_per_dst=[2622, 2848, 2484, 2618, 2582, 2760, 2592, 2622] combine_rows_per_src=[5598, 5818, 5392, 5522, 5490, 5604, 5512, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44620 payload_rows_remote=38880 dup_rows=20916 dup_rows_remote=18044 dedup_ratio=0.319153 router_tokens_per_expert=[510, 527, 501, 506, 539, 490, 543, 483, 545, 500, 512, 517, 497, 525, 543, 523, 481, 480, 541, 527, 500, 496, 515, 536, 526, 507, 510, 488, 549, 535, 484, 509, 520, 533, 528, 511, 471, 484, 547, 461, 486, 519, 519, 510, 503, 508, 521, 537, 521, 536, 529, 512, 522, 535, 494, 502, 499, 501, 482, 530, 483, 524, 519, 476] payload_rows_per_dst=[5480, 5690, 5582, 5410, 5546, 5544, 5628, 5740] dup_rows_per_dst=[2474, 2684, 2570, 2416, 2564, 2654, 2682, 2872] combine_rows_per_src=[5480, 5690, 5582, 5410, 5546, 5544, 5628, 5740] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279293952 dispatch_meta_bytes=229320 combine_payload_bytes=279293952 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279523272 ingress_bytes=499494912 link_bytes=499494912 token_bytes=558817224 weight_movement_bytes=220200960 total_nvlink_bytes=779018184 weight_to_token_ratio=0.394048 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.687038048e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279681024 dispatch_meta_bytes=229440 combine_payload_bytes=279681024 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279910464 ingress_bytes=389781504 link_bytes=389781504 token_bytes=559591488 weight_movement_bytes=110100480 total_nvlink_bytes=669691968 weight_to_token_ratio=0.196752 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778941435e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44490 payload_rows_remote=38896 dup_rows=21046 dup_rows_remote=18412 dedup_ratio=0.321136 router_tokens_per_expert=[513, 546, 538, 524, 477, 518, 542, 513, 457, 481, 515, 493, 485, 494, 519, 527, 511, 523, 528, 479, 507, 523, 517, 526, 500, 473, 554, 537, 520, 520, 493, 501, 500, 539, 501, 505, 492, 504, 506, 502, 514, 520, 535, 485, 555, 496, 522, 488, 491, 528, 515, 534, 515, 499, 486, 537, 529, 496, 508, 504, 535, 522, 534, 517] payload_rows_per_dst=[5624, 5438, 5594, 5540, 5488, 5582, 5594, 5630] dup_rows_per_dst=[2718, 2504, 2634, 2656, 2610, 2648, 2616, 2660] combine_rows_per_src=[5624, 5438, 5594, 5540, 5488, 5582, 5594, 5630] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44408 payload_rows_remote=38838 dup_rows=21128 dup_rows_remote=18468 dedup_ratio=0.322388 router_tokens_per_expert=[487, 513, 517, 540, 504, 527, 500, 541, 528, 513, 563, 517, 493, 476, 500, 525, 520, 512, 518, 508, 467, 518, 515, 523, 503, 505, 537, 531, 522, 526, 472, 523, 484, 523, 558, 519, 450, 550, 541, 516, 503, 503, 494, 515, 518, 516, 508, 471, 542, 510, 519, 540, 526, 515, 509, 500, 504, 508, 530, 509, 420, 512, 517, 494] payload_rows_per_dst=[5622, 5570, 5562, 5542, 5640, 5470, 5594, 5408] dup_rows_per_dst=[2636, 2660, 2600, 2696, 2642, 2586, 2728, 2580] combine_rows_per_src=[5622, 5570, 5562, 5542, 5640, 5470, 5594, 5408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278935552 dispatch_meta_bytes=229560 combine_payload_bytes=278935552 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279165112 ingress_bytes=389036032 link_bytes=389036032 token_bytes=558100664 weight_movement_bytes=110100480 total_nvlink_bytes=668201144 weight_to_token_ratio=0.197277 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.765976451e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278749184 dispatch_meta_bytes=228704 combine_payload_bytes=278749184 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278977888 ingress_bytes=498950144 link_bytes=498950144 token_bytes=557727072 weight_movement_bytes=220200960 total_nvlink_bytes=777928032 weight_to_token_ratio=0.394818 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.677563636e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278190080 dispatch_meta_bytes=229264 combine_payload_bytes=278190080 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=278419344 ingress_bytes=718592000 link_bytes=718592000 token_bytes=556609424 weight_movement_bytes=440401920 total_nvlink_bytes=997011344 weight_to_token_ratio=0.791223 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.249749676e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278390784 dispatch_meta_bytes=229224 combine_payload_bytes=278390784 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278620008 ingress_bytes=388491264 link_bytes=388491264 token_bytes=557010792 weight_movement_bytes=110100480 total_nvlink_bytes=667111272 weight_to_token_ratio=0.197663 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.756502039e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278691840 dispatch_meta_bytes=227696 combine_payload_bytes=278691840 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278919536 ingress_bytes=388792320 link_bytes=388792320 token_bytes=557611376 weight_movement_bytes=110100480 total_nvlink_bytes=667711856 weight_to_token_ratio=0.197450 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.761737898e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278806528 dispatch_meta_bytes=229232 combine_payload_bytes=278806528 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279035760 ingress_bytes=388907008 link_bytes=388907008 token_bytes=557842288 weight_movement_bytes=110100480 total_nvlink_bytes=667942768 weight_to_token_ratio=0.197368 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.763732511e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44410 payload_rows_remote=38866 dup_rows=21126 dup_rows_remote=18544 dedup_ratio=0.322357 router_tokens_per_expert=[470, 531, 516, 501, 498, 520, 509, 529, 540, 487, 508, 533, 512, 522, 516, 503, 508, 493, 518, 534, 551, 533, 513, 486, 541, 511, 510, 501, 536, 495, 494, 520, 538, 494, 437, 504, 482, 506, 531, 519, 523, 508, 508, 491, 499, 510, 505, 519, 530, 504, 549, 522, 518, 519, 502, 504, 480, 547, 482, 531, 521, 516, 537, 493] payload_rows_per_dst=[5536, 5552, 5590, 5526, 5532, 5544, 5576, 5554] dup_rows_per_dst=[2612, 2690, 2682, 2690, 2490, 2582, 2720, 2660] combine_rows_per_src=[5536, 5552, 5590, 5526, 5532, 5544, 5576, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44336 payload_rows_remote=38744 dup_rows=21200 dup_rows_remote=18568 dedup_ratio=0.323486 router_tokens_per_expert=[498, 512, 574, 505, 503, 497, 494, 528, 524, 520, 494, 531, 514, 515, 503, 522, 506, 494, 552, 493, 485, 494, 496, 492, 531, 534, 513, 512, 484, 520, 521, 524, 530, 513, 513, 507, 545, 502, 519, 483, 463, 547, 539, 519, 500, 527, 526, 535, 519, 508, 491, 533, 464, 462, 512, 486, 523, 496, 523, 529, 512, 525, 499, 533] payload_rows_per_dst=[5538, 5596, 5462, 5560, 5592, 5574, 5424, 5590] dup_rows_per_dst=[2684, 2650, 2562, 2718, 2632, 2738, 2526, 2690] combine_rows_per_src=[5538, 5596, 5462, 5560, 5592, 5574, 5424, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44646 payload_rows_remote=39086 dup_rows=20890 dup_rows_remote=18300 dedup_ratio=0.318756 router_tokens_per_expert=[547, 525, 511, 488, 552, 502, 458, 489, 505, 527, 503, 515, 487, 480, 491, 518, 522, 517, 539, 512, 515, 502, 499, 492, 521, 520, 535, 529, 496, 521, 476, 491, 501, 526, 536, 517, 541, 525, 538, 519, 493, 538, 504, 521, 527, 495, 520, 526, 517, 536, 512, 479, 511, 516, 505, 499, 494, 508, 525, 487, 524, 521, 518, 504] payload_rows_per_dst=[5544, 5484, 5604, 5602, 5654, 5626, 5560, 5572] dup_rows_per_dst=[2600, 2568, 2592, 2576, 2752, 2622, 2590, 2590] combine_rows_per_src=[5544, 5484, 5604, 5602, 5654, 5626, 5560, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44618 payload_rows_remote=38944 dup_rows=20918 dup_rows_remote=18190 dedup_ratio=0.319183 router_tokens_per_expert=[552, 534, 489, 501, 517, 515, 488, 517, 529, 551, 484, 496, 483, 506, 532, 509, 514, 506, 514, 538, 505, 503, 495, 486, 490, 524, 557, 548, 524, 519, 501, 538, 519, 545, 505, 495, 476, 467, 527, 497, 538, 478, 545, 503, 514, 500, 521, 518, 540, 529, 524, 547, 517, 490, 522, 496, 474, 545, 499, 492, 505, 506, 462, 507] payload_rows_per_dst=[5620, 5580, 5518, 5674, 5516, 5592, 5690, 5428] dup_rows_per_dst=[2606, 2600, 2604, 2728, 2546, 2642, 2640, 2552] combine_rows_per_src=[5620, 5580, 5518, 5674, 5516, 5592, 5690, 5428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278591488 dispatch_meta_bytes=229640 combine_payload_bytes=278591488 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=278821128 ingress_bytes=498792448 link_bytes=498792448 token_bytes=557412616 weight_movement_bytes=220200960 total_nvlink_bytes=777613576 weight_to_token_ratio=0.395041 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.674821043e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277716992 dispatch_meta_bytes=229248 combine_payload_bytes=277716992 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277946240 ingress_bytes=387817472 link_bytes=387817472 token_bytes=555663232 weight_movement_bytes=110100480 total_nvlink_bytes=665763712 weight_to_token_ratio=0.198142 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.744783688e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44326 payload_rows_remote=38680 dup_rows=21210 dup_rows_remote=18374 dedup_ratio=0.323639 router_tokens_per_expert=[497, 531, 520, 523, 524, 493, 520, 547, 530, 531, 508, 547, 497, 499, 522, 501, 510, 507, 523, 529, 499, 509, 526, 497, 522, 490, 543, 481, 533, 505, 516, 483, 505, 495, 501, 502, 510, 543, 503, 550, 543, 493, 519, 503, 510, 509, 485, 505, 516, 505, 523, 485, 546, 531, 532, 469, 510, 510, 515, 489, 529, 488, 489, 492] payload_rows_per_dst=[5344, 5666, 5572, 5436, 5444, 5580, 5638, 5646] dup_rows_per_dst=[2344, 2788, 2628, 2540, 2580, 2646, 2848, 2836] combine_rows_per_src=[5344, 5666, 5572, 5436, 5444, 5580, 5638, 5646] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44568 payload_rows_remote=39010 dup_rows=20968 dup_rows_remote=18398 dedup_ratio=0.319946 router_tokens_per_expert=[520, 515, 521, 535, 541, 512, 490, 506, 499, 544, 498, 484, 510, 548, 559, 486, 514, 474, 514, 525, 508, 511, 514, 504, 496, 538, 530, 560, 506, 536, 499, 525, 505, 492, 519, 503, 504, 521, 501, 501, 512, 507, 548, 449, 532, 498, 502, 537, 501, 497, 523, 485, 494, 527, 519, 529, 467, 510, 517, 502, 494, 519, 517, 514] payload_rows_per_dst=[5636, 5564, 5558, 5650, 5576, 5574, 5560, 5450] dup_rows_per_dst=[2644, 2692, 2570, 2730, 2516, 2596, 2590, 2630] combine_rows_per_src=[5636, 5564, 5558, 5650, 5576, 5574, 5560, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44588 payload_rows_remote=39034 dup_rows=20948 dup_rows_remote=18414 dedup_ratio=0.319641 router_tokens_per_expert=[514, 523, 471, 522, 487, 524, 522, 491, 475, 504, 519, 510, 504, 496, 540, 496, 499, 513, 500, 516, 478, 478, 532, 491, 516, 544, 509, 520, 492, 509, 525, 519, 503, 520, 500, 537, 566, 503, 522, 511, 509, 513, 516, 466, 511, 526, 557, 519, 535, 487, 517, 504, 487, 494, 538, 563, 518, 470, 515, 533, 544, 507, 477, 561] payload_rows_per_dst=[5534, 5554, 5466, 5638, 5626, 5580, 5598, 5592] dup_rows_per_dst=[2574, 2534, 2548, 2630, 2698, 2654, 2652, 2658] combine_rows_per_src=[5534, 5554, 5466, 5638, 5626, 5580, 5598, 5592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44714 payload_rows_remote=39112 dup_rows=20822 dup_rows_remote=18182 dedup_ratio=0.317719 router_tokens_per_expert=[491, 527, 543, 493, 529, 504, 527, 498, 515, 559, 540, 502, 503, 496, 507, 513, 561, 513, 486, 516, 480, 521, 507, 516, 517, 505, 501, 544, 484, 492, 515, 543, 474, 523, 528, 491, 513, 528, 526, 538, 527, 478, 520, 528, 506, 547, 501, 504, 487, 515, 503, 514, 512, 488, 541, 523, 453, 501, 510, 507, 510, 514, 484, 526] payload_rows_per_dst=[5602, 5588, 5672, 5618, 5646, 5584, 5460, 5544] dup_rows_per_dst=[2640, 2682, 2708, 2584, 2686, 2548, 2508, 2466] combine_rows_per_src=[5602, 5588, 5672, 5618, 5646, 5584, 5460, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280168448 dispatch_meta_bytes=229544 combine_payload_bytes=280168448 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280397992 ingress_bytes=610469888 link_bytes=610469888 token_bytes=560566440 weight_movement_bytes=330301440 total_nvlink_bytes=890867880 weight_to_token_ratio=0.589228 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061707540e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279150592 dispatch_meta_bytes=228536 combine_payload_bytes=279150592 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279379128 ingress_bytes=499351552 link_bytes=499351552 token_bytes=558529720 weight_movement_bytes=220200960 total_nvlink_bytes=778730680 weight_to_token_ratio=0.394251 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.684544781e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44760 payload_rows_remote=39198 dup_rows=20776 dup_rows_remote=18218 dedup_ratio=0.317017 router_tokens_per_expert=[502, 537, 536, 527, 556, 552, 521, 500, 505, 506, 521, 549, 524, 525, 498, 520, 538, 508, 527, 492, 494, 549, 503, 521, 518, 530, 508, 481, 506, 513, 500, 502, 505, 472, 511, 528, 514, 535, 528, 517, 505, 473, 478, 456, 483, 516, 510, 501, 533, 533, 482, 474, 513, 534, 493, 498, 505, 507, 534, 522, 484, 498, 546, 511] payload_rows_per_dst=[5708, 5648, 5630, 5578, 5582, 5432, 5562, 5620] dup_rows_per_dst=[2754, 2648, 2634, 2538, 2638, 2412, 2558, 2594] combine_rows_per_src=[5708, 5648, 5630, 5578, 5582, 5432, 5562, 5620] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=276226048 dispatch_meta_bytes=226872 combine_payload_bytes=276226048 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276452920 ingress_bytes=386326528 link_bytes=386326528 token_bytes=552678968 weight_movement_bytes=110100480 total_nvlink_bytes=662779448 weight_to_token_ratio=0.199212 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.718853720e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44564 payload_rows_remote=39002 dup_rows=20972 dup_rows_remote=18392 dedup_ratio=0.320007 router_tokens_per_expert=[522, 519, 525, 536, 546, 513, 494, 512, 500, 543, 500, 484, 508, 549, 559, 487, 514, 474, 516, 526, 508, 512, 516, 505, 496, 538, 529, 559, 509, 537, 499, 524, 505, 491, 518, 500, 504, 520, 501, 501, 513, 504, 551, 449, 532, 497, 502, 538, 501, 495, 522, 486, 490, 526, 513, 530, 464, 508, 515, 498, 494, 517, 513, 511] payload_rows_per_dst=[5658, 5568, 5562, 5654, 5566, 5568, 5554, 5434] dup_rows_per_dst=[2676, 2692, 2580, 2728, 2514, 2604, 2572, 2606] combine_rows_per_src=[5658, 5568, 5562, 5654, 5566, 5568, 5554, 5434] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279064576 dispatch_meta_bytes=228832 combine_payload_bytes=279064576 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279293408 ingress_bytes=389165056 link_bytes=389165056 token_bytes=558357984 weight_movement_bytes=110100480 total_nvlink_bytes=668458464 weight_to_token_ratio=0.197186 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.768220390e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44370 payload_rows_remote=38794 dup_rows=21166 dup_rows_remote=18476 dedup_ratio=0.322968 router_tokens_per_expert=[528, 505, 509, 559, 489, 510, 494, 545, 511, 513, 506, 497, 495, 490, 484, 502, 492, 532, 540, 515, 482, 528, 484, 495, 551, 481, 522, 560, 467, 510, 536, 534, 519, 498, 503, 519, 518, 526, 520, 530, 553, 497, 515, 533, 539, 500, 551, 494, 514, 479, 530, 491, 511, 525, 500, 496, 521, 533, 518, 514, 508, 497, 463, 487] payload_rows_per_dst=[5640, 5430, 5516, 5620, 5576, 5590, 5478, 5520] dup_rows_per_dst=[2638, 2566, 2620, 2702, 2690, 2774, 2614, 2562] combine_rows_per_src=[5640, 5430, 5516, 5620, 5576, 5590, 5478, 5520] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44386 payload_rows_remote=38798 dup_rows=21150 dup_rows_remote=18504 dedup_ratio=0.322723 router_tokens_per_expert=[509, 519, 510, 502, 532, 490, 513, 523, 491, 515, 516, 530, 503, 536, 506, 522, 513, 519, 500, 490, 507, 568, 491, 532, 519, 532, 528, 507, 520, 503, 488, 520, 505, 523, 536, 529, 494, 545, 497, 540, 519, 531, 510, 489, 525, 537, 495, 493, 519, 505, 455, 545, 519, 477, 497, 508, 525, 492, 475, 498, 508, 519, 500, 504] payload_rows_per_dst=[5556, 5564, 5538, 5588, 5582, 5566, 5484, 5508] dup_rows_per_dst=[2640, 2674, 2702, 2646, 2756, 2632, 2566, 2534] combine_rows_per_src=[5556, 5564, 5538, 5588, 5582, 5566, 5484, 5508] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44592 payload_rows_remote=39018 dup_rows=20944 dup_rows_remote=18402 dedup_ratio=0.319580 router_tokens_per_expert=[519, 525, 472, 523, 487, 526, 523, 492, 479, 504, 520, 512, 506, 497, 544, 496, 501, 515, 501, 518, 479, 481, 530, 493, 519, 542, 509, 521, 497, 507, 525, 519, 504, 518, 500, 537, 568, 503, 524, 511, 510, 512, 516, 464, 512, 522, 557, 517, 530, 484, 516, 505, 488, 493, 537, 559, 515, 468, 512, 531, 538, 505, 476, 554] payload_rows_per_dst=[5546, 5574, 5488, 5640, 5626, 5572, 5584, 5562] dup_rows_per_dst=[2588, 2542, 2548, 2638, 2704, 2648, 2640, 2636] combine_rows_per_src=[5546, 5574, 5488, 5640, 5626, 5572, 5584, 5562] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44338 payload_rows_remote=38772 dup_rows=21198 dup_rows_remote=18472 dedup_ratio=0.323456 router_tokens_per_expert=[544, 499, 457, 506, 521, 546, 507, 547, 520, 517, 490, 545, 516, 499, 515, 501, 546, 531, 490, 509, 487, 519, 517, 462, 528, 518, 523, 567, 530, 468, 534, 508, 497, 523, 488, 496, 513, 491, 568, 533, 496, 516, 507, 513, 484, 511, 492, 531, 493, 512, 515, 524, 537, 505, 492, 513, 495, 494, 525, 517, 502, 516, 478, 524] payload_rows_per_dst=[5566, 5538, 5524, 5604, 5548, 5506, 5590, 5462] dup_rows_per_dst=[2726, 2668, 2598, 2710, 2670, 2594, 2592, 2640] combine_rows_per_src=[5566, 5538, 5524, 5604, 5548, 5506, 5590, 5462] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=280971264 dispatch_meta_bytes=229664 combine_payload_bytes=280971264 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281200928 ingress_bytes=391071744 link_bytes=391071744 token_bytes=562172192 weight_movement_bytes=110100480 total_nvlink_bytes=672272672 weight_to_token_ratio=0.195848 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.801380831e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279566336 dispatch_meta_bytes=229576 combine_payload_bytes=279566336 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279795912 ingress_bytes=389666816 link_bytes=389666816 token_bytes=559362248 weight_movement_bytes=110100480 total_nvlink_bytes=669462728 weight_to_token_ratio=0.196832 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.776946822e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278075392 dispatch_meta_bytes=229080 combine_payload_bytes=278075392 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278304472 ingress_bytes=498276352 link_bytes=498276352 token_bytes=556379864 weight_movement_bytes=220200960 total_nvlink_bytes=776580824 weight_to_token_ratio=0.395774 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.665845285e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278104064 dispatch_meta_bytes=229208 combine_payload_bytes=278104064 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278333272 ingress_bytes=608405504 link_bytes=608405504 token_bytes=556437336 weight_movement_bytes=330301440 total_nvlink_bytes=886738776 weight_to_token_ratio=0.593600 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058117237e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279681024 dispatch_meta_bytes=229680 combine_payload_bytes=279681024 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279910704 ingress_bytes=499881984 link_bytes=499881984 token_bytes=559591728 weight_movement_bytes=220200960 total_nvlink_bytes=779792688 weight_to_token_ratio=0.393503 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.693769866e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44454 payload_rows_remote=38810 dup_rows=21082 dup_rows_remote=18334 dedup_ratio=0.321686 router_tokens_per_expert=[511, 539, 523, 515, 523, 542, 520, 527, 487, 497, 529, 537, 509, 488, 501, 536, 540, 523, 526, 552, 467, 517, 477, 490, 507, 510, 492, 487, 498, 474, 535, 516, 474, 534, 557, 500, 519, 520, 561, 541, 467, 479, 527, 505, 478, 505, 509, 562, 552, 451, 537, 499, 467, 557, 501, 481, 488, 532, 530, 495, 490, 535, 485, 535] payload_rows_per_dst=[5378, 5438, 5618, 5422, 5610, 5572, 5772, 5644] dup_rows_per_dst=[2316, 2486, 2592, 2390, 2872, 2654, 3024, 2748] combine_rows_per_src=[5378, 5438, 5618, 5422, 5610, 5572, 5772, 5644] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277917696 dispatch_meta_bytes=228976 combine_payload_bytes=277917696 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278146672 ingress_bytes=608219136 link_bytes=608219136 token_bytes=556064368 weight_movement_bytes=330301440 total_nvlink_bytes=886365808 weight_to_token_ratio=0.593999 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057793112e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44620 payload_rows_remote=39154 dup_rows=20916 dup_rows_remote=18436 dedup_ratio=0.319153 router_tokens_per_expert=[520, 506, 487, 501, 505, 540, 526, 518, 552, 511, 527, 520, 514, 530, 516, 542, 530, 525, 515, 523, 499, 543, 513, 499, 493, 490, 517, 493, 509, 556, 514, 505, 508, 479, 500, 493, 508, 514, 531, 531, 481, 509, 531, 494, 500, 460, 475, 523, 519, 491, 475, 545, 513, 506, 506, 474, 549, 494, 521, 518, 507, 527, 517, 530] payload_rows_per_dst=[5598, 5684, 5578, 5576, 5582, 5466, 5456, 5680] dup_rows_per_dst=[2608, 2740, 2716, 2578, 2546, 2480, 2602, 2646] combine_rows_per_src=[5598, 5684, 5578, 5576, 5582, 5466, 5456, 5680] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278190080 dispatch_meta_bytes=228576 combine_payload_bytes=278190080 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278418656 ingress_bytes=388290560 link_bytes=388290560 token_bytes=556608736 weight_movement_bytes=110100480 total_nvlink_bytes=666709216 weight_to_token_ratio=0.197806 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.753011467e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44564 payload_rows_remote=38990 dup_rows=20972 dup_rows_remote=18376 dedup_ratio=0.320007 router_tokens_per_expert=[505, 519, 534, 457, 492, 498, 526, 510, 544, 517, 516, 480, 529, 516, 493, 501, 534, 477, 502, 510, 510, 492, 512, 516, 528, 547, 494, 502, 511, 516, 519, 534, 516, 493, 510, 532, 524, 496, 502, 509, 527, 512, 522, 538, 537, 535, 506, 554, 497, 465, 531, 551, 501, 482, 506, 552, 508, 524, 498, 536, 469, 516, 490, 488] payload_rows_per_dst=[5490, 5534, 5554, 5602, 5602, 5718, 5574, 5490] dup_rows_per_dst=[2592, 2658, 2552, 2700, 2562, 2744, 2596, 2568] combine_rows_per_src=[5490, 5534, 5554, 5602, 5602, 5718, 5574, 5490] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44454 payload_rows_remote=39008 dup_rows=21082 dup_rows_remote=18414 dedup_ratio=0.321686 router_tokens_per_expert=[492, 499, 522, 511, 516, 502, 530, 501, 488, 529, 486, 551, 520, 536, 520, 522, 505, 511, 506, 483, 542, 495, 520, 495, 527, 531, 531, 500, 531, 519, 487, 510, 552, 507, 521, 526, 475, 499, 528, 502, 493, 498, 552, 518, 506, 468, 505, 540, 506, 488, 533, 504, 504, 552, 516, 507, 492, 533, 489, 531, 521, 513, 472, 499] payload_rows_per_dst=[5536, 5656, 5446, 5652, 5524, 5546, 5472, 5622] dup_rows_per_dst=[2610, 2648, 2668, 2620, 2696, 2614, 2748, 2478] combine_rows_per_src=[5536, 5656, 5446, 5652, 5524, 5546, 5472, 5622] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=280655872 dispatch_meta_bytes=230360 combine_payload_bytes=280655872 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280886232 ingress_bytes=390756352 link_bytes=390756352 token_bytes=561542104 weight_movement_bytes=110100480 total_nvlink_bytes=671642584 weight_to_token_ratio=0.196068 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.795895645e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44372 payload_rows_remote=38714 dup_rows=21164 dup_rows_remote=18526 dedup_ratio=0.322937 router_tokens_per_expert=[521, 517, 516, 516, 539, 492, 511, 526, 537, 488, 513, 471, 520, 515, 512, 480, 500, 470, 509, 524, 487, 520, 520, 568, 564, 573, 517, 523, 514, 499, 477, 509, 520, 507, 536, 510, 529, 511, 534, 501, 483, 522, 490, 511, 516, 511, 517, 504, 523, 489, 523, 524, 490, 474, 505, 554, 479, 534, 526, 519, 523, 491, 490, 474] payload_rows_per_dst=[5544, 5538, 5528, 5604, 5658, 5534, 5474, 5492] dup_rows_per_dst=[2732, 2534, 2668, 2748, 2638, 2574, 2690, 2580] combine_rows_per_src=[5544, 5538, 5528, 5604, 5658, 5534, 5474, 5492] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44432 payload_rows_remote=38776 dup_rows=21104 dup_rows_remote=18426 dedup_ratio=0.322021 router_tokens_per_expert=[528, 501, 518, 535, 477, 499, 478, 521, 519, 546, 520, 514, 476, 494, 496, 489, 508, 465, 513, 516, 522, 524, 481, 475, 554, 515, 517, 505, 516, 556, 490, 514, 506, 537, 553, 512, 529, 554, 490, 490, 517, 508, 547, 497, 524, 517, 541, 549, 532, 514, 497, 491, 465, 509, 489, 520, 493, 477, 534, 553, 491, 513, 517, 520] payload_rows_per_dst=[5522, 5478, 5524, 5656, 5610, 5604, 5498, 5540] dup_rows_per_dst=[2592, 2630, 2484, 2678, 2732, 2796, 2536, 2656] combine_rows_per_src=[5522, 5478, 5524, 5656, 5610, 5604, 5498, 5540] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44610 payload_rows_remote=38994 dup_rows=20926 dup_rows_remote=18252 dedup_ratio=0.319305 router_tokens_per_expert=[481, 511, 502, 526, 507, 530, 512, 490, 501, 508, 517, 499, 515, 547, 532, 526, 489, 518, 537, 550, 574, 510, 526, 508, 514, 516, 503, 538, 526, 499, 522, 499, 495, 504, 536, 500, 495, 467, 530, 527, 504, 494, 520, 540, 476, 517, 512, 503, 509, 529, 495, 527, 499, 451, 521, 494, 515, 497, 498, 528, 473, 503, 550, 526] payload_rows_per_dst=[5548, 5616, 5634, 5564, 5582, 5572, 5522, 5572] dup_rows_per_dst=[2570, 2674, 2790, 2670, 2526, 2560, 2528, 2608] combine_rows_per_src=[5548, 5616, 5634, 5564, 5582, 5572, 5522, 5572] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279480320 dispatch_meta_bytes=229464 combine_payload_bytes=279480320 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279709784 ingress_bytes=389580800 link_bytes=389580800 token_bytes=559190104 weight_movement_bytes=110100480 total_nvlink_bytes=669290584 weight_to_token_ratio=0.196893 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.775450862e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44544 payload_rows_remote=38688 dup_rows=20992 dup_rows_remote=17948 dedup_ratio=0.320312 router_tokens_per_expert=[516, 514, 539, 550, 538, 528, 534, 543, 522, 501, 548, 505, 516, 501, 480, 538, 541, 461, 511, 508, 524, 556, 476, 515, 486, 503, 499, 481, 508, 518, 514, 529, 528, 496, 512, 507, 503, 501, 527, 512, 485, 490, 520, 506, 540, 535, 476, 499, 457, 504, 513, 483, 536, 523, 491, 515, 564, 538, 489, 512, 494, 520, 485, 504] payload_rows_per_dst=[5856, 5568, 5590, 5554, 5436, 5514, 5502, 5524] dup_rows_per_dst=[3044, 2654, 2594, 2522, 2360, 2588, 2542, 2688] combine_rows_per_src=[5856, 5568, 5590, 5554, 5436, 5514, 5502, 5524] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279609344 dispatch_meta_bytes=229688 combine_payload_bytes=279609344 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279839032 ingress_bytes=389709824 link_bytes=389709824 token_bytes=559448376 weight_movement_bytes=110100480 total_nvlink_bytes=669548856 weight_to_token_ratio=0.196802 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.777694802e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277501952 dispatch_meta_bytes=228960 combine_payload_bytes=277501952 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277730912 ingress_bytes=497702912 link_bytes=497702912 token_bytes=555232864 weight_movement_bytes=220200960 total_nvlink_bytes=775433824 weight_to_token_ratio=0.396592 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.655872220e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44334 payload_rows_remote=38654 dup_rows=21202 dup_rows_remote=18288 dedup_ratio=0.323517 router_tokens_per_expert=[483, 551, 537, 567, 484, 521, 520, 539, 492, 524, 497, 487, 509, 526, 512, 553, 494, 531, 495, 494, 517, 501, 553, 525, 536, 523, 493, 519, 512, 502, 512, 517, 523, 555, 498, 484, 490, 498, 491, 521, 491, 531, 531, 527, 506, 494, 491, 547, 507, 466, 497, 528, 500, 528, 521, 501, 524, 496, 490, 508, 509, 529, 469, 491] payload_rows_per_dst=[5408, 5468, 5246, 5584, 5724, 5678, 5546, 5680] dup_rows_per_dst=[2434, 2374, 2266, 2752, 3104, 2808, 2550, 2914] combine_rows_per_src=[5408, 5468, 5246, 5584, 5724, 5678, 5546, 5680] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277946368 dispatch_meta_bytes=228808 combine_payload_bytes=277946368 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278175176 ingress_bytes=608247808 link_bytes=608247808 token_bytes=556121544 weight_movement_bytes=330301440 total_nvlink_bytes=886422984 weight_to_token_ratio=0.593938 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057842978e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279508992 dispatch_meta_bytes=228984 combine_payload_bytes=279508992 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279737976 ingress_bytes=609810432 link_bytes=609810432 token_bytes=559246968 weight_movement_bytes=330301440 total_nvlink_bytes=889548408 weight_to_token_ratio=0.590618 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060560638e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277315584 dispatch_meta_bytes=226544 combine_payload_bytes=277315584 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277542128 ingress_bytes=497516544 link_bytes=497516544 token_bytes=554857712 weight_movement_bytes=220200960 total_nvlink_bytes=775058672 weight_to_token_ratio=0.396860 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.652630974e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44324 payload_rows_remote=38798 dup_rows=21212 dup_rows_remote=18620 dedup_ratio=0.323669 router_tokens_per_expert=[565, 497, 533, 504, 507, 523, 528, 498, 510, 510, 507, 541, 506, 523, 492, 526, 531, 525, 530, 484, 507, 552, 536, 517, 514, 513, 491, 501, 488, 500, 538, 498, 508, 513, 515, 493, 473, 495, 493, 508, 492, 501, 524, 532, 514, 502, 492, 502, 506, 534, 515, 547, 506, 505, 520, 502, 504, 501, 504, 510, 505, 483, 541, 533] payload_rows_per_dst=[5596, 5574, 5632, 5512, 5380, 5526, 5524, 5580] dup_rows_per_dst=[2714, 2656, 2732, 2574, 2616, 2592, 2746, 2582] combine_rows_per_src=[5596, 5574, 5632, 5512, 5380, 5526, 5524, 5580] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277071872 dispatch_meta_bytes=227768 combine_payload_bytes=277071872 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277299640 ingress_bytes=387172352 link_bytes=387172352 token_bytes=554371512 weight_movement_bytes=110100480 total_nvlink_bytes=664471992 weight_to_token_ratio=0.198604 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.733563990e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277258240 dispatch_meta_bytes=228216 combine_payload_bytes=277258240 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277486456 ingress_bytes=387358720 link_bytes=387358720 token_bytes=554744696 weight_movement_bytes=110100480 total_nvlink_bytes=664845176 weight_to_token_ratio=0.198471 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.736805236e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279623680 dispatch_meta_bytes=229632 combine_payload_bytes=279623680 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279853312 ingress_bytes=389724160 link_bytes=389724160 token_bytes=559476992 weight_movement_bytes=110100480 total_nvlink_bytes=669577472 weight_to_token_ratio=0.196792 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.777944128e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279795712 dispatch_meta_bytes=229792 combine_payload_bytes=279795712 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280025504 ingress_bytes=389896192 link_bytes=389896192 token_bytes=559821216 weight_movement_bytes=110100480 total_nvlink_bytes=669921696 weight_to_token_ratio=0.196671 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.780936048e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280354816 dispatch_meta_bytes=229176 combine_payload_bytes=280354816 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280583992 ingress_bytes=610656256 link_bytes=610656256 token_bytes=560938808 weight_movement_bytes=330301440 total_nvlink_bytes=891240248 weight_to_token_ratio=0.588837 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.062031665e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44400 payload_rows_remote=38820 dup_rows=21136 dup_rows_remote=18446 dedup_ratio=0.322510 router_tokens_per_expert=[524, 498, 509, 554, 487, 506, 492, 544, 511, 514, 503, 494, 494, 488, 481, 498, 492, 528, 540, 513, 478, 526, 486, 493, 546, 480, 521, 561, 467, 513, 537, 532, 519, 497, 502, 522, 520, 525, 517, 533, 554, 499, 517, 532, 540, 501, 551, 493, 516, 479, 532, 497, 515, 526, 500, 500, 526, 536, 520, 518, 518, 498, 465, 490] payload_rows_per_dst=[5624, 5422, 5510, 5618, 5580, 5594, 5502, 5550] dup_rows_per_dst=[2604, 2544, 2602, 2696, 2690, 2780, 2628, 2592] combine_rows_per_src=[5624, 5422, 5510, 5618, 5580, 5594, 5502, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44570 payload_rows_remote=38912 dup_rows=20966 dup_rows_remote=18286 dedup_ratio=0.319916 router_tokens_per_expert=[485, 547, 516, 556, 517, 523, 536, 532, 489, 536, 499, 491, 558, 494, 480, 527, 525, 532, 488, 478, 547, 497, 513, 520, 503, 506, 514, 544, 522, 513, 484, 535, 488, 511, 530, 496, 491, 528, 476, 492, 503, 541, 520, 522, 540, 510, 505, 528, 495, 512, 526, 505, 505, 513, 464, 511, 480, 507, 541, 467, 500, 526, 511, 517] payload_rows_per_dst=[5630, 5550, 5580, 5622, 5496, 5658, 5492, 5542] dup_rows_per_dst=[2794, 2598, 2620, 2620, 2528, 2680, 2570, 2556] combine_rows_per_src=[5630, 5550, 5580, 5622, 5496, 5658, 5492, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44766 payload_rows_remote=39188 dup_rows=20770 dup_rows_remote=18188 dedup_ratio=0.316925 router_tokens_per_expert=[501, 535, 533, 523, 554, 550, 519, 499, 501, 506, 520, 545, 521, 522, 494, 518, 538, 507, 522, 489, 493, 546, 504, 523, 519, 530, 507, 483, 506, 514, 500, 502, 506, 473, 512, 531, 512, 534, 528, 517, 503, 473, 479, 456, 486, 517, 511, 499, 533, 536, 485, 478, 518, 535, 497, 498, 507, 512, 534, 525, 486, 500, 549, 514] payload_rows_per_dst=[5694, 5634, 5620, 5586, 5578, 5436, 5578, 5640] dup_rows_per_dst=[2734, 2620, 2624, 2536, 2648, 2412, 2582, 2614] combine_rows_per_src=[5694, 5634, 5620, 5586, 5578, 5436, 5578, 5640] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44402 payload_rows_remote=38812 dup_rows=21134 dup_rows_remote=18500 dedup_ratio=0.322479 router_tokens_per_expert=[506, 518, 508, 500, 530, 485, 511, 522, 488, 507, 515, 527, 499, 532, 506, 518, 513, 516, 500, 490, 507, 562, 491, 532, 520, 533, 525, 507, 520, 500, 488, 519, 507, 524, 535, 529, 493, 541, 500, 542, 519, 529, 510, 490, 529, 540, 498, 495, 522, 507, 456, 546, 522, 481, 500, 514, 526, 494, 479, 503, 511, 521, 504, 506] payload_rows_per_dst=[5544, 5546, 5534, 5590, 5578, 5582, 5504, 5524] dup_rows_per_dst=[2616, 2638, 2688, 2634, 2764, 2638, 2592, 2564] combine_rows_per_src=[5544, 5546, 5534, 5590, 5578, 5582, 5504, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44456 payload_rows_remote=39020 dup_rows=21080 dup_rows_remote=18436 dedup_ratio=0.321655 router_tokens_per_expert=[490, 497, 521, 509, 514, 497, 525, 499, 487, 526, 485, 547, 518, 536, 522, 520, 503, 511, 502, 481, 538, 492, 517, 496, 524, 530, 528, 501, 532, 518, 488, 510, 551, 508, 520, 526, 479, 498, 530, 504, 494, 501, 551, 518, 510, 471, 506, 543, 509, 490, 536, 505, 505, 554, 516, 509, 493, 535, 493, 532, 521, 519, 473, 504] payload_rows_per_dst=[5518, 5642, 5436, 5650, 5528, 5556, 5486, 5640] dup_rows_per_dst=[2586, 2640, 2644, 2612, 2704, 2632, 2762, 2500] combine_rows_per_src=[5518, 5642, 5436, 5650, 5528, 5556, 5486, 5640] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44596 payload_rows_remote=39010 dup_rows=20940 dup_rows_remote=18312 dedup_ratio=0.319519 router_tokens_per_expert=[507, 533, 519, 512, 521, 541, 519, 523, 484, 495, 528, 535, 508, 488, 500, 533, 538, 523, 525, 552, 465, 514, 478, 489, 503, 511, 492, 485, 499, 474, 538, 517, 474, 533, 557, 502, 521, 521, 560, 539, 468, 480, 531, 508, 480, 506, 511, 564, 553, 451, 538, 500, 472, 558, 503, 484, 490, 533, 533, 496, 490, 537, 488, 538] payload_rows_per_dst=[5672, 5532, 5600, 5534, 5414, 5528, 5730, 5586] dup_rows_per_dst=[2678, 2606, 2568, 2504, 2558, 2568, 2830, 2628] combine_rows_per_src=[5672, 5532, 5600, 5534, 5414, 5528, 5730, 5586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278261760 dispatch_meta_bytes=229064 combine_payload_bytes=278261760 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278490824 ingress_bytes=498462720 link_bytes=498462720 token_bytes=556752584 weight_movement_bytes=220200960 total_nvlink_bytes=776953544 weight_to_token_ratio=0.395510 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.669086531e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278921216 dispatch_meta_bytes=228792 combine_payload_bytes=278921216 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279150008 ingress_bytes=389021696 link_bytes=389021696 token_bytes=558071224 weight_movement_bytes=110100480 total_nvlink_bytes=668171704 weight_to_token_ratio=0.197288 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.765727124e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44620 payload_rows_remote=39018 dup_rows=20916 dup_rows_remote=18262 dedup_ratio=0.319153 router_tokens_per_expert=[480, 507, 498, 519, 502, 525, 510, 489, 495, 505, 515, 501, 513, 545, 531, 523, 489, 518, 535, 549, 571, 508, 523, 507, 513, 516, 502, 538, 526, 500, 521, 500, 496, 505, 534, 498, 499, 467, 531, 529, 504, 497, 521, 541, 478, 518, 513, 504, 511, 532, 497, 528, 505, 452, 526, 495, 519, 499, 503, 530, 475, 506, 550, 531] payload_rows_per_dst=[5518, 5602, 5630, 5574, 5590, 5570, 5546, 5590] dup_rows_per_dst=[2542, 2654, 2770, 2658, 2528, 2582, 2546, 2636] combine_rows_per_src=[5518, 5602, 5630, 5574, 5590, 5570, 5546, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44318 payload_rows_remote=38772 dup_rows=21218 dup_rows_remote=18544 dedup_ratio=0.323761 router_tokens_per_expert=[541, 496, 456, 502, 519, 544, 503, 548, 516, 516, 491, 545, 514, 496, 512, 503, 545, 527, 491, 505, 487, 514, 517, 461, 526, 514, 523, 567, 529, 466, 534, 508, 499, 525, 488, 493, 511, 493, 573, 535, 498, 515, 507, 516, 486, 514, 494, 533, 496, 512, 518, 524, 537, 506, 493, 515, 497, 495, 527, 520, 506, 518, 481, 527] payload_rows_per_dst=[5546, 5526, 5636, 5462, 5554, 5516, 5598, 5480] dup_rows_per_dst=[2674, 2660, 2780, 2548, 2680, 2610, 2604, 2662] combine_rows_per_src=[5546, 5526, 5636, 5462, 5554, 5516, 5598, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=280899584 dispatch_meta_bytes=229504 combine_payload_bytes=280899584 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281129088 ingress_bytes=391000064 link_bytes=391000064 token_bytes=562028672 weight_movement_bytes=110100480 total_nvlink_bytes=672129152 weight_to_token_ratio=0.195898 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.800134198e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278204416 dispatch_meta_bytes=229248 combine_payload_bytes=278204416 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=278433664 ingress_bytes=718606336 link_bytes=718606336 token_bytes=556638080 weight_movement_bytes=440401920 total_nvlink_bytes=997040000 weight_to_token_ratio=0.791182 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.249774609e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279623680 dispatch_meta_bytes=229288 combine_payload_bytes=279623680 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=279852968 ingress_bytes=720025600 link_bytes=720025600 token_bytes=559476648 weight_movement_bytes=440401920 total_nvlink_bytes=999878568 weight_to_token_ratio=0.787168 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.252242942e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279695360 dispatch_meta_bytes=229824 combine_payload_bytes=279695360 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279925184 ingress_bytes=389795840 link_bytes=389795840 token_bytes=559620544 weight_movement_bytes=110100480 total_nvlink_bytes=669721024 weight_to_token_ratio=0.196741 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.779190762e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277917696 dispatch_meta_bytes=229264 combine_payload_bytes=277917696 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278146960 ingress_bytes=608219136 link_bytes=608219136 token_bytes=556064656 weight_movement_bytes=330301440 total_nvlink_bytes=886366096 weight_to_token_ratio=0.593998 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057793112e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279681024 dispatch_meta_bytes=229120 combine_payload_bytes=279681024 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=279910144 ingress_bytes=389781504 link_bytes=389781504 token_bytes=559591168 weight_movement_bytes=110100480 total_nvlink_bytes=669691648 weight_to_token_ratio=0.196752 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778941435e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44378 payload_rows_remote=38722 dup_rows=21158 dup_rows_remote=18510 dedup_ratio=0.322845 router_tokens_per_expert=[514, 512, 516, 516, 534, 491, 511, 517, 535, 487, 514, 470, 518, 515, 513, 480, 500, 470, 508, 516, 488, 517, 518, 567, 566, 573, 518, 522, 514, 496, 476, 509, 519, 507, 537, 508, 532, 511, 536, 502, 482, 523, 490, 511, 518, 511, 515, 507, 527, 488, 526, 526, 492, 474, 507, 554, 483, 538, 528, 523, 525, 493, 499, 475] payload_rows_per_dst=[5518, 5532, 5530, 5600, 5656, 5536, 5484, 5522] dup_rows_per_dst=[2704, 2532, 2638, 2748, 2648, 2578, 2704, 2606] combine_rows_per_src=[5518, 5532, 5530, 5600, 5656, 5536, 5484, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44612 payload_rows_remote=39144 dup_rows=20924 dup_rows_remote=18430 dedup_ratio=0.319275 router_tokens_per_expert=[516, 504, 480, 496, 499, 539, 528, 515, 548, 509, 522, 518, 513, 528, 513, 541, 530, 526, 513, 525, 497, 543, 511, 495, 493, 491, 517, 493, 509, 559, 517, 506, 511, 474, 500, 495, 508, 516, 535, 530, 484, 509, 532, 492, 500, 463, 477, 524, 525, 493, 475, 547, 515, 508, 505, 475, 548, 498, 525, 520, 508, 530, 518, 534] payload_rows_per_dst=[5568, 5666, 5582, 5592, 5580, 5468, 5466, 5690] dup_rows_per_dst=[2586, 2718, 2698, 2578, 2558, 2494, 2620, 2672] combine_rows_per_src=[5568, 5666, 5582, 5592, 5580, 5468, 5466, 5690] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44444 payload_rows_remote=38794 dup_rows=21092 dup_rows_remote=18408 dedup_ratio=0.321838 router_tokens_per_expert=[525, 501, 514, 531, 474, 500, 476, 517, 515, 546, 517, 510, 474, 493, 491, 486, 506, 460, 512, 514, 522, 523, 482, 473, 553, 516, 516, 505, 516, 557, 490, 514, 507, 535, 551, 511, 534, 554, 494, 491, 516, 511, 546, 498, 525, 519, 540, 551, 533, 515, 503, 492, 467, 515, 491, 523, 494, 480, 537, 556, 495, 516, 516, 523] payload_rows_per_dst=[5502, 5460, 5520, 5650, 5628, 5606, 5518, 5560] dup_rows_per_dst=[2574, 2604, 2464, 2684, 2726, 2806, 2560, 2674] combine_rows_per_src=[5502, 5460, 5520, 5650, 5628, 5606, 5518, 5560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44548 payload_rows_remote=38960 dup_rows=20988 dup_rows_remote=18372 dedup_ratio=0.320251 router_tokens_per_expert=[504, 515, 533, 457, 489, 495, 525, 505, 541, 515, 513, 478, 521, 515, 492, 500, 532, 475, 500, 508, 509, 492, 512, 517, 524, 547, 492, 502, 511, 521, 520, 534, 513, 492, 509, 532, 525, 495, 506, 509, 527, 512, 524, 542, 537, 534, 511, 554, 497, 468, 532, 556, 500, 487, 506, 556, 511, 527, 501, 539, 470, 519, 493, 490] payload_rows_per_dst=[5468, 5512, 5548, 5594, 5596, 5726, 5588, 5516] dup_rows_per_dst=[2578, 2638, 2542, 2708, 2566, 2756, 2616, 2584] combine_rows_per_src=[5468, 5512, 5548, 5594, 5596, 5726, 5588, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44370 payload_rows_remote=38784 dup_rows=21166 dup_rows_remote=18448 dedup_ratio=0.322968 router_tokens_per_expert=[479, 549, 534, 565, 482, 518, 518, 532, 488, 523, 496, 483, 506, 521, 509, 552, 492, 529, 495, 494, 517, 500, 552, 528, 536, 522, 493, 518, 510, 503, 511, 517, 525, 555, 500, 484, 488, 499, 492, 524, 491, 531, 531, 529, 508, 496, 492, 545, 507, 469, 499, 526, 502, 532, 521, 503, 526, 499, 498, 511, 511, 533, 473, 496] payload_rows_per_dst=[5628, 5576, 5262, 5524, 5716, 5620, 5458, 5586] dup_rows_per_dst=[2726, 2464, 2290, 2676, 3080, 2742, 2470, 2718] combine_rows_per_src=[5628, 5576, 5262, 5524, 5716, 5620, 5458, 5586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44568 payload_rows_remote=39062 dup_rows=20968 dup_rows_remote=18332 dedup_ratio=0.319946 router_tokens_per_expert=[472, 523, 527, 487, 531, 456, 556, 495, 522, 533, 484, 503, 535, 521, 503, 494, 502, 525, 507, 534, 514, 516, 464, 509, 502, 537, 515, 491, 518, 502, 520, 533, 512, 500, 525, 489, 509, 535, 538, 514, 495, 524, 527, 538, 516, 500, 510, 478, 484, 540, 537, 524, 545, 504, 500, 493, 491, 548, 520, 535, 514, 493, 517, 482] payload_rows_per_dst=[5556, 5598, 5506, 5590, 5564, 5566, 5648, 5540] dup_rows_per_dst=[2538, 2592, 2636, 2646, 2680, 2610, 2606, 2660] combine_rows_per_src=[5556, 5598, 5506, 5590, 5564, 5566, 5648, 5540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277559296 dispatch_meta_bytes=228928 combine_payload_bytes=277559296 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277788224 ingress_bytes=497760256 link_bytes=497760256 token_bytes=555347520 weight_movement_bytes=220200960 total_nvlink_bytes=775548480 weight_to_token_ratio=0.396510 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.656869527e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=280584192 dispatch_meta_bytes=230296 combine_payload_bytes=280584192 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280814488 ingress_bytes=390684672 link_bytes=390684672 token_bytes=561398680 weight_movement_bytes=110100480 total_nvlink_bytes=671499160 weight_to_token_ratio=0.196118 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.794649012e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44546 payload_rows_remote=38804 dup_rows=20990 dup_rows_remote=18208 dedup_ratio=0.320282 router_tokens_per_expert=[511, 513, 536, 546, 535, 525, 528, 541, 521, 497, 547, 503, 514, 498, 479, 540, 540, 460, 507, 508, 522, 555, 475, 517, 486, 503, 501, 482, 508, 521, 513, 529, 526, 497, 510, 506, 506, 501, 527, 512, 486, 489, 522, 508, 540, 535, 477, 503, 460, 504, 515, 484, 538, 526, 493, 520, 565, 537, 489, 517, 498, 520, 489, 507] payload_rows_per_dst=[5742, 5564, 5646, 5450, 5630, 5516, 5514, 5484] dup_rows_per_dst=[2782, 2634, 2720, 2400, 2626, 2614, 2566, 2648] combine_rows_per_src=[5742, 5564, 5646, 5450, 5630, 5516, 5514, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44568 payload_rows_remote=39036 dup_rows=20968 dup_rows_remote=18432 dedup_ratio=0.319946 router_tokens_per_expert=[512, 488, 507, 501, 532, 485, 520, 517, 530, 481, 526, 495, 520, 512, 479, 491, 479, 535, 536, 516, 541, 494, 541, 542, 540, 490, 563, 527, 552, 494, 504, 496, 493, 513, 531, 487, 485, 511, 528, 513, 540, 533, 509, 526, 493, 515, 503, 528, 511, 501, 545, 491, 512, 532, 477, 504, 513, 481, 512, 500, 517, 538, 508, 472] payload_rows_per_dst=[5538, 5532, 5696, 5606, 5520, 5630, 5568, 5478] dup_rows_per_dst=[2586, 2536, 2672, 2726, 2602, 2664, 2578, 2604] combine_rows_per_src=[5538, 5532, 5696, 5606, 5520, 5630, 5568, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278075392 dispatch_meta_bytes=228808 combine_payload_bytes=278075392 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278304200 ingress_bytes=608376832 link_bytes=608376832 token_bytes=556379592 weight_movement_bytes=330301440 total_nvlink_bytes=886681032 weight_to_token_ratio=0.593662 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058067372e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279265280 dispatch_meta_bytes=229328 combine_payload_bytes=279265280 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279494608 ingress_bytes=389365760 link_bytes=389365760 token_bytes=558759888 weight_movement_bytes=110100480 total_nvlink_bytes=668860368 weight_to_token_ratio=0.197044 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.771710963e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278003712 dispatch_meta_bytes=228928 combine_payload_bytes=278003712 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278232640 ingress_bytes=388104192 link_bytes=388104192 token_bytes=556236352 weight_movement_bytes=110100480 total_nvlink_bytes=666336832 weight_to_token_ratio=0.197938 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.051471 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.749770221e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279996416 dispatch_meta_bytes=229576 combine_payload_bytes=279996416 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280225992 ingress_bytes=500197376 link_bytes=500197376 token_bytes=560222408 weight_movement_bytes=220200960 total_nvlink_bytes=780423368 weight_to_token_ratio=0.393060 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.699255052e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278147072 dispatch_meta_bytes=228048 combine_payload_bytes=278147072 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278375120 ingress_bytes=388247552 link_bytes=388247552 token_bytes=556522192 weight_movement_bytes=110100480 total_nvlink_bytes=666622672 weight_to_token_ratio=0.197837 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.752263487e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279810048 dispatch_meta_bytes=229872 combine_payload_bytes=279810048 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280039920 ingress_bytes=500011008 link_bytes=500011008 token_bytes=559849968 weight_movement_bytes=220200960 total_nvlink_bytes=780050928 weight_to_token_ratio=0.393321 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.696013806e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44332 payload_rows_remote=38796 dup_rows=21204 dup_rows_remote=18602 dedup_ratio=0.323547 router_tokens_per_expert=[564, 496, 529, 502, 503, 521, 525, 494, 503, 507, 502, 541, 505, 522, 492, 527, 532, 523, 528, 487, 506, 551, 533, 515, 511, 512, 488, 501, 488, 501, 538, 500, 507, 516, 515, 493, 475, 496, 491, 509, 491, 504, 523, 536, 515, 503, 494, 503, 510, 535, 517, 550, 505, 506, 528, 506, 506, 503, 504, 510, 506, 486, 541, 537] payload_rows_per_dst=[5582, 5556, 5630, 5508, 5382, 5536, 5544, 5594] dup_rows_per_dst=[2686, 2642, 2720, 2570, 2622, 2602, 2770, 2592] combine_rows_per_src=[5582, 5556, 5630, 5508, 5382, 5536, 5544, 5594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44672 payload_rows_remote=39112 dup_rows=20864 dup_rows_remote=18280 dedup_ratio=0.318359 router_tokens_per_expert=[525, 519, 515, 494, 507, 574, 477, 538, 471, 531, 486, 495, 463, 520, 506, 549, 473, 505, 528, 499, 530, 473, 485, 504, 528, 479, 516, 491, 533, 529, 502, 570, 493, 524, 525, 510, 489, 488, 530, 513, 501, 509, 501, 529, 497, 513, 523, 540, 534, 520, 509, 509, 499, 493, 539, 534, 507, 518, 541, 514, 521, 545, 473, 512] payload_rows_per_dst=[5654, 5498, 5538, 5606, 5560, 5646, 5586, 5584] dup_rows_per_dst=[2644, 2544, 2456, 2690, 2584, 2580, 2688, 2678] combine_rows_per_src=[5654, 5498, 5538, 5606, 5560, 5646, 5586, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44460 payload_rows_remote=38846 dup_rows=21076 dup_rows_remote=18490 dedup_ratio=0.321594 router_tokens_per_expert=[523, 528, 497, 529, 513, 535, 518, 474, 540, 528, 534, 495, 518, 478, 538, 503, 501, 504, 511, 517, 507, 525, 528, 572, 509, 514, 538, 478, 531, 523, 511, 496, 458, 501, 494, 480, 512, 501, 477, 496, 539, 514, 510, 525, 533, 500, 477, 502, 517, 539, 519, 500, 500, 532, 527, 512, 514, 531, 486, 493, 505, 522, 512, 524] payload_rows_per_dst=[5572, 5584, 5582, 5614, 5326, 5594, 5596, 5592] dup_rows_per_dst=[2662, 2684, 2748, 2586, 2512, 2606, 2696, 2582] combine_rows_per_src=[5572, 5584, 5582, 5614, 5326, 5594, 5596, 5592] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44578 payload_rows_remote=38948 dup_rows=20958 dup_rows_remote=18378 dedup_ratio=0.319794 router_tokens_per_expert=[484, 514, 495, 505, 481, 511, 464, 499, 546, 572, 535, 501, 509, 537, 535, 510, 517, 540, 528, 549, 528, 532, 535, 509, 533, 506, 529, 511, 539, 478, 480, 512, 545, 536, 509, 510, 486, 512, 501, 505, 523, 518, 498, 489, 489, 472, 491, 525, 507, 512, 556, 507, 525, 502, 490, 506, 496, 504, 500, 484, 480, 510, 546, 510] payload_rows_per_dst=[5438, 5656, 5684, 5614, 5514, 5494, 5630, 5548] dup_rows_per_dst=[2468, 2834, 2792, 2562, 2694, 2516, 2580, 2512] combine_rows_per_src=[5438, 5656, 5684, 5614, 5514, 5494, 5630, 5548] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278104064 dispatch_meta_bytes=229672 combine_payload_bytes=278104064 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278333736 ingress_bytes=388204544 link_bytes=388204544 token_bytes=556437800 weight_movement_bytes=110100480 total_nvlink_bytes=666538280 weight_to_token_ratio=0.197867 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.751515507e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44566 payload_rows_remote=39050 dup_rows=20970 dup_rows_remote=18304 dedup_ratio=0.319977 router_tokens_per_expert=[474, 527, 531, 495, 533, 459, 560, 496, 522, 536, 485, 504, 541, 522, 505, 494, 504, 530, 509, 535, 516, 516, 468, 513, 505, 540, 518, 491, 520, 501, 521, 531, 515, 499, 525, 493, 507, 532, 537, 513, 493, 521, 525, 534, 514, 495, 510, 475, 483, 537, 534, 518, 543, 501, 497, 489, 488, 545, 516, 533, 512, 491, 512, 479] payload_rows_per_dst=[5568, 5612, 5516, 5594, 5570, 5552, 5638, 5516] dup_rows_per_dst=[2582, 2606, 2666, 2660, 2672, 2582, 2566, 2636] combine_rows_per_src=[5568, 5612, 5516, 5594, 5570, 5552, 5638, 5516] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44650 payload_rows_remote=39094 dup_rows=20886 dup_rows_remote=18302 dedup_ratio=0.318695 router_tokens_per_expert=[527, 522, 517, 498, 511, 576, 479, 543, 472, 534, 487, 497, 464, 523, 509, 547, 475, 507, 528, 502, 532, 472, 486, 502, 529, 480, 516, 491, 534, 530, 501, 570, 495, 521, 524, 509, 487, 490, 531, 513, 499, 508, 501, 529, 495, 513, 522, 541, 533, 520, 505, 505, 495, 492, 538, 532, 504, 516, 541, 510, 518, 541, 471, 508] payload_rows_per_dst=[5674, 5500, 5540, 5600, 5556, 5638, 5574, 5568] dup_rows_per_dst=[2672, 2566, 2468, 2702, 2584, 2578, 2666, 2650] combine_rows_per_src=[5674, 5500, 5540, 5600, 5556, 5638, 5574, 5568] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44474 payload_rows_remote=38858 dup_rows=21062 dup_rows_remote=18482 dedup_ratio=0.321381 router_tokens_per_expert=[525, 528, 497, 532, 516, 540, 521, 474, 541, 530, 534, 498, 521, 481, 539, 505, 503, 505, 512, 517, 510, 527, 529, 574, 509, 514, 538, 478, 528, 523, 512, 496, 460, 499, 495, 481, 510, 498, 477, 496, 541, 513, 510, 524, 533, 499, 477, 499, 516, 540, 515, 500, 498, 530, 524, 513, 511, 531, 484, 489, 498, 520, 509, 521] payload_rows_per_dst=[5588, 5594, 5602, 5616, 5324, 5594, 5592, 5564] dup_rows_per_dst=[2678, 2704, 2752, 2580, 2508, 2598, 2680, 2562] combine_rows_per_src=[5588, 5594, 5602, 5616, 5324, 5594, 5592, 5564] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44576 payload_rows_remote=39034 dup_rows=20960 dup_rows_remote=18414 dedup_ratio=0.319824 router_tokens_per_expert=[518, 490, 508, 504, 537, 488, 525, 520, 532, 482, 526, 496, 523, 513, 479, 493, 478, 539, 536, 517, 541, 494, 544, 544, 537, 490, 563, 527, 551, 494, 507, 499, 489, 511, 530, 488, 483, 513, 529, 512, 541, 531, 507, 527, 492, 513, 502, 527, 510, 498, 544, 488, 511, 530, 474, 501, 510, 480, 510, 497, 515, 535, 504, 471] payload_rows_per_dst=[5562, 5542, 5700, 5608, 5520, 5624, 5562, 5458] dup_rows_per_dst=[2618, 2546, 2686, 2728, 2590, 2656, 2550, 2586] combine_rows_per_src=[5562, 5542, 5700, 5608, 5520, 5624, 5562, 5458] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279179264 dispatch_meta_bytes=229304 combine_payload_bytes=279179264 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=279408568 ingress_bytes=389279744 link_bytes=389279744 token_bytes=558587832 weight_movement_bytes=110100480 total_nvlink_bytes=668688312 weight_to_token_ratio=0.197105 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.770215003e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44694 payload_rows_remote=38948 dup_rows=20842 dup_rows_remote=18052 dedup_ratio=0.318024 router_tokens_per_expert=[551, 506, 518, 486, 514, 500, 546, 542, 516, 536, 517, 521, 536, 518, 521, 469, 540, 530, 493, 516, 498, 531, 546, 484, 544, 494, 496, 520, 505, 509, 481, 481, 496, 479, 581, 514, 494, 488, 523, 515, 473, 541, 490, 532, 536, 508, 515, 455, 483, 504, 471, 516, 531, 542, 508, 518, 511, 501, 508, 496, 537, 507, 512, 518] payload_rows_per_dst=[5746, 5642, 5558, 5432, 5564, 5554, 5618, 5580] dup_rows_per_dst=[2790, 2662, 2696, 2418, 2616, 2546, 2514, 2600] combine_rows_per_src=[5746, 5642, 5558, 5432, 5564, 5554, 5618, 5580] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279910400 dispatch_meta_bytes=229416 combine_payload_bytes=279910400 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280139816 ingress_bytes=610211840 link_bytes=610211840 token_bytes=560050216 weight_movement_bytes=330301440 total_nvlink_bytes=890351656 weight_to_token_ratio=0.589771 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061258752e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=280225792 dispatch_meta_bytes=229584 combine_payload_bytes=280225792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280455376 ingress_bytes=390326272 link_bytes=390326272 token_bytes=560681168 weight_movement_bytes=110100480 total_nvlink_bytes=670781648 weight_to_token_ratio=0.196369 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.788415846e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44336 payload_rows_remote=38670 dup_rows=21200 dup_rows_remote=18272 dedup_ratio=0.323486 router_tokens_per_expert=[493, 477, 525, 480, 520, 478, 547, 552, 547, 527, 521, 538, 511, 525, 534, 531, 526, 550, 512, 520, 533, 500, 522, 499, 522, 491, 517, 516, 520, 517, 548, 479, 512, 548, 508, 519, 511, 515, 454, 486, 516, 493, 507, 534, 536, 492, 482, 502, 498, 511, 506, 474, 520, 531, 526, 477, 506, 489, 511, 526, 495, 503, 521, 481] payload_rows_per_dst=[5426, 5486, 5568, 5444, 5466, 5544, 5736, 5666] dup_rows_per_dst=[2580, 2442, 2576, 2512, 2480, 2740, 2942, 2928] combine_rows_per_src=[5426, 5486, 5568, 5444, 5466, 5544, 5736, 5666] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278534144 dispatch_meta_bytes=229360 combine_payload_bytes=278534144 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278763504 ingress_bytes=498735104 link_bytes=498735104 token_bytes=557297648 weight_movement_bytes=220200960 total_nvlink_bytes=777498608 weight_to_token_ratio=0.395123 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.673823737e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44606 payload_rows_remote=39200 dup_rows=20930 dup_rows_remote=18382 dedup_ratio=0.319366 router_tokens_per_expert=[490, 515, 479, 515, 535, 527, 512, 512, 538, 517, 505, 573, 460, 566, 522, 533, 500, 528, 513, 563, 532, 528, 506, 504, 511, 507, 519, 503, 522, 531, 540, 500, 517, 503, 531, 510, 473, 504, 497, 534, 468, 487, 482, 518, 522, 492, 514, 494, 519, 491, 503, 515, 495, 496, 539, 516, 512, 522, 513, 496, 506, 496, 485, 512] payload_rows_per_dst=[5558, 5672, 5656, 5662, 5570, 5406, 5586, 5496] dup_rows_per_dst=[2612, 2756, 2692, 2604, 2568, 2548, 2562, 2588] combine_rows_per_src=[5558, 5672, 5656, 5662, 5570, 5406, 5586, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44356 payload_rows_remote=38810 dup_rows=21180 dup_rows_remote=18440 dedup_ratio=0.323181 router_tokens_per_expert=[492, 476, 525, 477, 519, 475, 545, 549, 547, 525, 521, 536, 507, 522, 532, 527, 527, 546, 506, 518, 531, 500, 519, 498, 523, 491, 517, 515, 520, 517, 549, 479, 511, 549, 510, 517, 510, 518, 455, 490, 518, 493, 508, 537, 536, 491, 484, 502, 499, 511, 510, 476, 522, 533, 527, 479, 507, 490, 512, 527, 499, 507, 525, 484] payload_rows_per_dst=[5478, 5548, 5628, 5486, 5500, 5500, 5670, 5546] dup_rows_per_dst=[2638, 2510, 2662, 2552, 2574, 2684, 2820, 2740] combine_rows_per_src=[5478, 5548, 5628, 5486, 5500, 5500, 5670, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44486 payload_rows_remote=38952 dup_rows=21050 dup_rows_remote=18472 dedup_ratio=0.321198 router_tokens_per_expert=[535, 504, 532, 515, 506, 498, 512, 489, 503, 490, 525, 484, 524, 518, 521, 485, 538, 550, 471, 502, 507, 519, 473, 496, 499, 532, 523, 540, 519, 509, 515, 558, 518, 524, 499, 515, 473, 498, 515, 521, 506, 522, 538, 514, 498, 486, 559, 529, 532, 523, 535, 507, 526, 517, 522, 450, 502, 502, 524, 489, 487, 523, 506, 516] payload_rows_per_dst=[5542, 5524, 5534, 5682, 5580, 5596, 5534, 5494] dup_rows_per_dst=[2640, 2576, 2578, 2708, 2546, 2708, 2690, 2604] combine_rows_per_src=[5542, 5524, 5534, 5682, 5580, 5596, 5534, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44570 payload_rows_remote=38932 dup_rows=20966 dup_rows_remote=18372 dedup_ratio=0.319916 router_tokens_per_expert=[484, 512, 494, 499, 480, 509, 462, 496, 545, 572, 535, 500, 507, 532, 535, 507, 516, 538, 523, 549, 526, 530, 534, 510, 530, 506, 529, 509, 539, 477, 481, 514, 547, 536, 512, 510, 487, 510, 501, 503, 523, 518, 502, 491, 488, 473, 492, 526, 508, 512, 557, 510, 525, 505, 492, 507, 499, 509, 504, 486, 482, 511, 547, 515] payload_rows_per_dst=[5420, 5650, 5674, 5602, 5516, 5500, 5638, 5570] dup_rows_per_dst=[2452, 2816, 2778, 2568, 2696, 2526, 2594, 2536] combine_rows_per_src=[5420, 5650, 5674, 5602, 5516, 5500, 5638, 5570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=280354816 dispatch_meta_bytes=229568 combine_payload_bytes=280354816 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280584384 ingress_bytes=390455296 link_bytes=390455296 token_bytes=560939200 weight_movement_bytes=110100480 total_nvlink_bytes=671039680 weight_to_token_ratio=0.196279 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.790659786e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278089728 dispatch_meta_bytes=229592 combine_payload_bytes=278089728 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278319320 ingress_bytes=388190208 link_bytes=388190208 token_bytes=556409048 weight_movement_bytes=110100480 total_nvlink_bytes=666509528 weight_to_token_ratio=0.197877 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.751266180e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44708 payload_rows_remote=39130 dup_rows=20828 dup_rows_remote=18222 dedup_ratio=0.317810 router_tokens_per_expert=[477, 480, 504, 535, 539, 514, 518, 503, 513, 471, 535, 529, 526, 493, 516, 509, 467, 512, 486, 516, 501, 511, 482, 500, 509, 511, 542, 529, 491, 496, 491, 495, 509, 524, 544, 557, 503, 497, 509, 543, 491, 502, 525, 537, 536, 538, 536, 536, 522, 508, 524, 525, 503, 498, 478, 524, 483, 529, 515, 522, 496, 519, 529, 505] payload_rows_per_dst=[5576, 5578, 5500, 5550, 5706, 5592, 5562, 5644] dup_rows_per_dst=[2564, 2606, 2450, 2578, 2666, 2810, 2602, 2552] combine_rows_per_src=[5576, 5578, 5500, 5550, 5706, 5592, 5562, 5644] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44714 payload_rows_remote=39066 dup_rows=20822 dup_rows_remote=18178 dedup_ratio=0.317719 router_tokens_per_expert=[550, 503, 514, 482, 511, 497, 542, 539, 516, 534, 513, 519, 536, 517, 521, 466, 539, 529, 493, 515, 495, 530, 546, 485, 546, 495, 494, 519, 506, 509, 481, 480, 496, 478, 581, 515, 493, 488, 524, 513, 476, 542, 491, 531, 536, 508, 516, 457, 486, 504, 474, 517, 532, 545, 511, 521, 514, 501, 512, 502, 539, 508, 515, 520] payload_rows_per_dst=[5648, 5632, 5608, 5500, 5560, 5562, 5602, 5602] dup_rows_per_dst=[2644, 2632, 2740, 2524, 2616, 2552, 2494, 2620] combine_rows_per_src=[5648, 5632, 5608, 5500, 5560, 5562, 5602, 5602] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279795712 dispatch_meta_bytes=229792 combine_payload_bytes=279795712 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280025504 ingress_bytes=610097152 link_bytes=610097152 token_bytes=559821216 weight_movement_bytes=330301440 total_nvlink_bytes=890122656 weight_to_token_ratio=0.590012 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061059291e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44536 payload_rows_remote=38898 dup_rows=21000 dup_rows_remote=18486 dedup_ratio=0.320435 router_tokens_per_expert=[510, 500, 517, 502, 535, 532, 527, 485, 485, 533, 513, 510, 486, 497, 543, 525, 538, 508, 544, 479, 513, 478, 509, 566, 497, 514, 515, 522, 484, 503, 508, 550, 527, 518, 493, 524, 533, 544, 490, 491, 498, 526, 519, 532, 515, 511, 502, 473, 503, 515, 484, 504, 503, 565, 515, 487, 494, 484, 533, 541, 505, 502, 499, 510] payload_rows_per_dst=[5562, 5592, 5580, 5594, 5522, 5502, 5638, 5546] dup_rows_per_dst=[2654, 2592, 2690, 2592, 2718, 2650, 2514, 2590] combine_rows_per_src=[5562, 5592, 5580, 5594, 5522, 5502, 5638, 5546] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279179264 dispatch_meta_bytes=228000 combine_payload_bytes=279179264 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279407264 ingress_bytes=499380224 link_bytes=499380224 token_bytes=558586528 weight_movement_bytes=220200960 total_nvlink_bytes=778787488 weight_to_token_ratio=0.394211 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.685043435e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278448128 dispatch_meta_bytes=229344 combine_payload_bytes=278448128 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278677472 ingress_bytes=608749568 link_bytes=608749568 token_bytes=557125600 weight_movement_bytes=330301440 total_nvlink_bytes=887427040 weight_to_token_ratio=0.592867 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058715621e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278190080 dispatch_meta_bytes=229000 combine_payload_bytes=278190080 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278419080 ingress_bytes=388290560 link_bytes=388290560 token_bytes=556609160 weight_movement_bytes=110100480 total_nvlink_bytes=666709640 weight_to_token_ratio=0.197806 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.753011467e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279207936 dispatch_meta_bytes=229696 combine_payload_bytes=279207936 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279437632 ingress_bytes=389308416 link_bytes=389308416 token_bytes=558645568 weight_movement_bytes=110100480 total_nvlink_bytes=668746048 weight_to_token_ratio=0.197085 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.770713657e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279064576 dispatch_meta_bytes=229216 combine_payload_bytes=279064576 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279293792 ingress_bytes=609366016 link_bytes=609366016 token_bytes=558358368 weight_movement_bytes=330301440 total_nvlink_bytes=888659808 weight_to_token_ratio=0.591558 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059787725e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280483840 dispatch_meta_bytes=229408 combine_payload_bytes=280483840 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280713248 ingress_bytes=500684800 link_bytes=500684800 token_bytes=561197088 weight_movement_bytes=220200960 total_nvlink_bytes=781398048 weight_to_token_ratio=0.392377 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.707732157e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=280025088 dispatch_meta_bytes=228976 combine_payload_bytes=280025088 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280254064 ingress_bytes=390125568 link_bytes=390125568 token_bytes=560279152 weight_movement_bytes=110100480 total_nvlink_bytes=670379632 weight_to_token_ratio=0.196510 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.784925274e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44570 payload_rows_remote=38956 dup_rows=20966 dup_rows_remote=18374 dedup_ratio=0.319916 router_tokens_per_expert=[536, 528, 512, 485, 496, 515, 541, 530, 495, 529, 564, 460, 496, 491, 522, 539, 511, 503, 480, 536, 507, 520, 518, 478, 518, 535, 535, 495, 507, 526, 522, 535, 511, 519, 462, 581, 478, 540, 497, 515, 542, 481, 508, 469, 553, 527, 520, 514, 508, 504, 503, 522, 480, 535, 485, 514, 532, 484, 502, 502, 478, 514, 526, 497] payload_rows_per_dst=[5596, 5568, 5502, 5660, 5614, 5646, 5554, 5430] dup_rows_per_dst=[2690, 2624, 2604, 2686, 2592, 2582, 2548, 2640] combine_rows_per_src=[5596, 5568, 5502, 5660, 5614, 5646, 5554, 5430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44638 payload_rows_remote=39228 dup_rows=20898 dup_rows_remote=18350 dedup_ratio=0.318878 router_tokens_per_expert=[488, 514, 476, 514, 532, 526, 510, 510, 538, 517, 502, 573, 456, 566, 520, 533, 498, 529, 511, 560, 530, 529, 506, 503, 510, 506, 518, 502, 523, 530, 540, 500, 517, 503, 530, 508, 473, 503, 498, 533, 467, 489, 483, 517, 522, 492, 514, 495, 521, 494, 504, 517, 496, 498, 542, 521, 512, 522, 517, 501, 508, 498, 485, 518] payload_rows_per_dst=[5550, 5664, 5656, 5662, 5572, 5410, 5604, 5520] dup_rows_per_dst=[2590, 2746, 2676, 2596, 2558, 2548, 2582, 2602] combine_rows_per_src=[5550, 5664, 5656, 5662, 5572, 5410, 5604, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277186560 dispatch_meta_bytes=227768 combine_payload_bytes=277186560 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277414328 ingress_bytes=387287040 link_bytes=387287040 token_bytes=554600888 weight_movement_bytes=110100480 total_nvlink_bytes=664701368 weight_to_token_ratio=0.198522 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.735558603e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280985600 dispatch_meta_bytes=230328 combine_payload_bytes=280985600 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281215928 ingress_bytes=391086080 link_bytes=391086080 token_bytes=562201528 weight_movement_bytes=110100480 total_nvlink_bytes=672302008 weight_to_token_ratio=0.195838 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.801630157e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278820864 dispatch_meta_bytes=229536 combine_payload_bytes=278820864 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279050400 ingress_bytes=388921344 link_bytes=388921344 token_bytes=557871264 weight_movement_bytes=110100480 total_nvlink_bytes=667971744 weight_to_token_ratio=0.197358 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.763981838e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44576 payload_rows_remote=38962 dup_rows=20960 dup_rows_remote=18366 dedup_ratio=0.319824 router_tokens_per_expert=[539, 528, 517, 487, 500, 517, 543, 531, 498, 531, 563, 464, 498, 493, 524, 539, 512, 504, 484, 538, 507, 523, 516, 481, 517, 536, 535, 496, 508, 523, 522, 534, 512, 521, 463, 578, 476, 541, 500, 513, 541, 481, 507, 471, 551, 525, 521, 513, 506, 503, 499, 519, 480, 533, 484, 510, 529, 481, 501, 499, 478, 508, 523, 493] payload_rows_per_dst=[5614, 5582, 5512, 5658, 5614, 5642, 5538, 5416] dup_rows_per_dst=[2710, 2638, 2618, 2684, 2594, 2578, 2530, 2608] combine_rows_per_src=[5614, 5582, 5512, 5658, 5614, 5642, 5538, 5416] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44458 payload_rows_remote=38916 dup_rows=21078 dup_rows_remote=18486 dedup_ratio=0.321625 router_tokens_per_expert=[537, 506, 535, 517, 508, 503, 513, 494, 506, 492, 526, 487, 527, 522, 523, 486, 540, 553, 472, 503, 507, 521, 475, 496, 501, 532, 519, 542, 519, 510, 515, 557, 516, 524, 497, 513, 473, 499, 514, 519, 508, 523, 536, 513, 497, 485, 557, 526, 532, 522, 532, 506, 526, 515, 521, 443, 498, 499, 520, 487, 486, 521, 501, 515] payload_rows_per_dst=[5556, 5534, 5542, 5678, 5568, 5590, 5522, 5468] dup_rows_per_dst=[2670, 2604, 2592, 2712, 2542, 2700, 2672, 2586] combine_rows_per_src=[5556, 5534, 5542, 5678, 5568, 5590, 5522, 5468] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44548 payload_rows_remote=39024 dup_rows=20988 dup_rows_remote=18340 dedup_ratio=0.320251 router_tokens_per_expert=[470, 517, 490, 538, 574, 513, 503, 505, 508, 545, 520, 498, 492, 517, 498, 522, 464, 495, 506, 539, 540, 482, 504, 537, 496, 503, 507, 500, 540, 525, 497, 518, 539, 509, 493, 524, 525, 479, 494, 499, 517, 514, 530, 490, 510, 511, 488, 537, 487, 519, 516, 470, 515, 529, 502, 541, 526, 533, 473, 534, 488, 525, 574, 514] payload_rows_per_dst=[5604, 5604, 5514, 5524, 5496, 5608, 5600, 5598] dup_rows_per_dst=[2616, 2596, 2620, 2648, 2628, 2586, 2558, 2736] combine_rows_per_src=[5604, 5604, 5514, 5524, 5496, 5608, 5600, 5598] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44714 payload_rows_remote=39126 dup_rows=20822 dup_rows_remote=18194 dedup_ratio=0.317719 router_tokens_per_expert=[479, 482, 507, 542, 542, 517, 525, 504, 515, 473, 536, 532, 529, 494, 517, 512, 467, 513, 487, 518, 504, 511, 484, 501, 511, 510, 543, 528, 490, 494, 492, 499, 507, 526, 544, 555, 504, 491, 509, 541, 489, 502, 523, 538, 536, 537, 532, 532, 522, 506, 523, 522, 503, 492, 476, 523, 479, 531, 515, 520, 492, 516, 526, 498] payload_rows_per_dst=[5602, 5588, 5506, 5544, 5710, 5586, 5550, 5628] dup_rows_per_dst=[2594, 2628, 2464, 2590, 2644, 2792, 2584, 2526] combine_rows_per_src=[5602, 5588, 5506, 5544, 5710, 5586, 5550, 5628] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279279616 dispatch_meta_bytes=229312 combine_payload_bytes=279279616 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279508928 ingress_bytes=499480576 link_bytes=499480576 token_bytes=558788544 weight_movement_bytes=220200960 total_nvlink_bytes=778989504 weight_to_token_ratio=0.394068 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.686788721e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44474 payload_rows_remote=38982 dup_rows=21062 dup_rows_remote=18376 dedup_ratio=0.321381 router_tokens_per_expert=[553, 506, 483, 572, 513, 474, 492, 492, 519, 493, 535, 503, 492, 520, 541, 508, 549, 521, 513, 506, 562, 516, 519, 517, 442, 500, 507, 504, 509, 560, 504, 519, 511, 542, 520, 517, 542, 554, 502, 524, 491, 486, 479, 522, 525, 519, 524, 486, 512, 510, 493, 486, 553, 507, 470, 471, 544, 505, 540, 481, 524, 502, 478, 504] payload_rows_per_dst=[5492, 5608, 5714, 5484, 5710, 5506, 5432, 5528] dup_rows_per_dst=[2686, 2614, 2692, 2606, 2706, 2558, 2572, 2628] combine_rows_per_src=[5492, 5608, 5714, 5484, 5710, 5506, 5432, 5528] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44440 payload_rows_remote=38706 dup_rows=21096 dup_rows_remote=18146 dedup_ratio=0.321899 router_tokens_per_expert=[512, 541, 461, 518, 467, 503, 482, 519, 532, 490, 514, 533, 526, 535, 506, 555, 509, 534, 502, 492, 503, 487, 503, 497, 496, 493, 512, 517, 540, 521, 516, 513, 512, 512, 527, 539, 515, 504, 545, 518, 500, 506, 534, 532, 469, 493, 492, 502, 515, 524, 541, 524, 495, 515, 512, 532, 488, 491, 538, 514, 494, 514, 515, 527] payload_rows_per_dst=[5414, 5262, 5434, 5536, 5666, 5700, 5694, 5734] dup_rows_per_dst=[2370, 2172, 2508, 2688, 2782, 2872, 2754, 2950] combine_rows_per_src=[5414, 5262, 5434, 5536, 5666, 5700, 5694, 5734] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278949888 dispatch_meta_bytes=229608 combine_payload_bytes=278949888 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279179496 ingress_bytes=499150848 link_bytes=499150848 token_bytes=558129384 weight_movement_bytes=220200960 total_nvlink_bytes=778330344 weight_to_token_ratio=0.394534 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.681054209e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279724032 dispatch_meta_bytes=229456 combine_payload_bytes=279724032 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279953488 ingress_bytes=499924992 link_bytes=499924992 token_bytes=559677520 weight_movement_bytes=220200960 total_nvlink_bytes=779878480 weight_to_token_ratio=0.393443 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.694517846e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44602 payload_rows_remote=39142 dup_rows=20934 dup_rows_remote=18290 dedup_ratio=0.319427 router_tokens_per_expert=[527, 485, 510, 500, 517, 525, 499, 522, 523, 513, 531, 475, 538, 529, 533, 529, 493, 523, 492, 516, 500, 500, 494, 519, 533, 522, 537, 529, 495, 482, 539, 543, 520, 509, 518, 526, 462, 489, 543, 508, 475, 515, 497, 528, 483, 509, 547, 498, 504, 537, 491, 501, 521, 492, 517, 511, 519, 521, 475, 503, 518, 492, 546, 520] payload_rows_per_dst=[5622, 5636, 5470, 5640, 5600, 5460, 5616, 5558] dup_rows_per_dst=[2548, 2706, 2604, 2720, 2550, 2644, 2532, 2630] combine_rows_per_src=[5622, 5636, 5470, 5640, 5600, 5460, 5616, 5558] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44492 payload_rows_remote=38954 dup_rows=21044 dup_rows_remote=18438 dedup_ratio=0.321106 router_tokens_per_expert=[501, 492, 551, 511, 538, 509, 497, 507, 496, 532, 546, 517, 523, 504, 518, 525, 506, 503, 539, 515, 526, 531, 518, 518, 525, 489, 493, 484, 519, 475, 532, 542, 515, 546, 487, 492, 523, 473, 509, 516, 520, 534, 495, 503, 500, 538, 496, 562, 523, 533, 480, 513, 504, 500, 513, 506, 516, 489, 491, 500, 498, 503, 485, 523] payload_rows_per_dst=[5646, 5642, 5602, 5460, 5486, 5610, 5538, 5508] dup_rows_per_dst=[2566, 2680, 2710, 2658, 2636, 2686, 2606, 2502] combine_rows_per_src=[5646, 5642, 5602, 5460, 5486, 5610, 5538, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44566 payload_rows_remote=39044 dup_rows=20970 dup_rows_remote=18326 dedup_ratio=0.319977 router_tokens_per_expert=[467, 514, 489, 536, 574, 512, 504, 505, 507, 542, 517, 495, 494, 517, 495, 521, 461, 493, 507, 538, 538, 481, 500, 536, 495, 502, 505, 499, 538, 525, 500, 519, 538, 511, 491, 524, 526, 479, 497, 500, 516, 515, 531, 490, 509, 512, 489, 537, 487, 518, 516, 469, 518, 532, 502, 544, 528, 536, 475, 539, 492, 527, 577, 517] payload_rows_per_dst=[5602, 5594, 5492, 5522, 5504, 5616, 5614, 5622] dup_rows_per_dst=[2600, 2582, 2616, 2644, 2628, 2582, 2558, 2760] combine_rows_per_src=[5602, 5594, 5492, 5522, 5504, 5616, 5614, 5622] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279236608 dispatch_meta_bytes=229320 combine_payload_bytes=279236608 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=279465928 ingress_bytes=719638528 link_bytes=719638528 token_bytes=558702536 weight_movement_bytes=440401920 total_nvlink_bytes=999104456 weight_to_token_ratio=0.788258 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.251569760e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44412 payload_rows_remote=38786 dup_rows=21124 dup_rows_remote=18364 dedup_ratio=0.322327 router_tokens_per_expert=[505, 537, 459, 516, 463, 502, 480, 517, 531, 489, 510, 532, 523, 535, 504, 554, 505, 531, 497, 490, 502, 486, 499, 496, 496, 491, 511, 515, 537, 522, 520, 511, 516, 513, 529, 538, 512, 507, 548, 517, 499, 507, 537, 534, 469, 496, 493, 505, 517, 530, 543, 525, 500, 517, 515, 531, 493, 495, 541, 516, 496, 516, 517, 530] payload_rows_per_dst=[5462, 5376, 5494, 5532, 5640, 5644, 5638, 5626] dup_rows_per_dst=[2496, 2396, 2590, 2674, 2720, 2770, 2718, 2760] combine_rows_per_src=[5462, 5376, 5494, 5532, 5640, 5644, 5638, 5626] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44538 payload_rows_remote=38890 dup_rows=20998 dup_rows_remote=18470 dedup_ratio=0.320404 router_tokens_per_expert=[505, 496, 514, 500, 531, 528, 523, 483, 482, 532, 513, 508, 483, 490, 541, 523, 535, 506, 543, 480, 513, 476, 508, 567, 497, 514, 515, 520, 483, 504, 511, 552, 526, 516, 493, 524, 535, 545, 490, 498, 499, 527, 518, 534, 516, 513, 507, 475, 506, 516, 484, 506, 506, 567, 517, 486, 500, 487, 535, 544, 506, 504, 500, 512] payload_rows_per_dst=[5536, 5580, 5576, 5602, 5526, 5510, 5648, 5560] dup_rows_per_dst=[2624, 2564, 2680, 2590, 2728, 2668, 2528, 2616] combine_rows_per_src=[5536, 5580, 5576, 5602, 5526, 5510, 5648, 5560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44628 payload_rows_remote=38966 dup_rows=20908 dup_rows_remote=18224 dedup_ratio=0.319031 router_tokens_per_expert=[514, 540, 545, 506, 553, 498, 466, 519, 510, 523, 495, 492, 530, 512, 547, 472, 543, 496, 504, 522, 501, 513, 518, 576, 513, 511, 519, 491, 513, 498, 478, 515, 495, 539, 537, 516, 489, 488, 467, 496, 497, 517, 533, 506, 505, 517, 529, 487, 505, 540, 509, 498, 498, 523, 537, 527, 443, 505, 519, 508, 513, 526, 535, 531] payload_rows_per_dst=[5628, 5520, 5662, 5570, 5578, 5472, 5624, 5574] dup_rows_per_dst=[2654, 2642, 2684, 2506, 2476, 2710, 2650, 2586] combine_rows_per_src=[5628, 5520, 5662, 5570, 5578, 5472, 5624, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=281186304 dispatch_meta_bytes=230312 combine_payload_bytes=281186304 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281416616 ingress_bytes=391286784 link_bytes=391286784 token_bytes=562602920 weight_movement_bytes=110100480 total_nvlink_bytes=672703400 weight_to_token_ratio=0.195698 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.805120730e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44394 payload_rows_remote=38832 dup_rows=21142 dup_rows_remote=18462 dedup_ratio=0.322601 router_tokens_per_expert=[524, 529, 525, 511, 557, 537, 566, 520, 542, 525, 513, 462, 530, 519, 547, 483, 506, 523, 508, 525, 477, 505, 530, 514, 521, 507, 518, 523, 473, 516, 540, 503, 468, 539, 501, 509, 520, 517, 509, 514, 499, 491, 517, 514, 518, 530, 475, 470, 534, 481, 569, 486, 487, 507, 508, 491, 481, 504, 483, 511, 512, 527, 513, 504] payload_rows_per_dst=[5772, 5562, 5516, 5508, 5482, 5540, 5544, 5470] dup_rows_per_dst=[2766, 2680, 2660, 2694, 2672, 2488, 2582, 2600] combine_rows_per_src=[5772, 5562, 5516, 5508, 5482, 5540, 5544, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44472 payload_rows_remote=38940 dup_rows=21064 dup_rows_remote=18286 dedup_ratio=0.321411 router_tokens_per_expert=[552, 504, 476, 570, 508, 472, 492, 491, 516, 491, 533, 501, 486, 517, 541, 506, 548, 518, 513, 505, 556, 513, 519, 518, 440, 501, 507, 506, 508, 559, 504, 519, 510, 544, 520, 519, 543, 555, 504, 527, 493, 488, 480, 524, 525, 520, 524, 488, 511, 510, 493, 488, 556, 509, 472, 472, 546, 506, 543, 489, 526, 504, 480, 509] payload_rows_per_dst=[5532, 5636, 5696, 5460, 5638, 5522, 5436, 5552] dup_rows_per_dst=[2778, 2636, 2686, 2558, 2604, 2562, 2586, 2654] combine_rows_per_src=[5532, 5636, 5696, 5460, 5638, 5522, 5436, 5552] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280455168 dispatch_meta_bytes=229280 combine_payload_bytes=280455168 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280684448 ingress_bytes=500656128 link_bytes=500656128 token_bytes=561139616 weight_movement_bytes=220200960 total_nvlink_bytes=781340576 weight_to_token_ratio=0.392417 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.707233504e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279867392 dispatch_meta_bytes=229480 combine_payload_bytes=279867392 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280096872 ingress_bytes=610168832 link_bytes=610168832 token_bytes=559964264 weight_movement_bytes=330301440 total_nvlink_bytes=890265704 weight_to_token_ratio=0.589862 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061183954e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278018048 dispatch_meta_bytes=228600 combine_payload_bytes=278018048 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278246648 ingress_bytes=388118528 link_bytes=388118528 token_bytes=556264696 weight_movement_bytes=110100480 total_nvlink_bytes=666365176 weight_to_token_ratio=0.197928 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.750019547e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278763520 dispatch_meta_bytes=229440 combine_payload_bytes=278763520 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278992960 ingress_bytes=388864000 link_bytes=388864000 token_bytes=557756480 weight_movement_bytes=110100480 total_nvlink_bytes=667856960 weight_to_token_ratio=0.197399 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.762984531e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279308288 dispatch_meta_bytes=228760 combine_payload_bytes=279308288 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279537048 ingress_bytes=389408768 link_bytes=389408768 token_bytes=558845336 weight_movement_bytes=110100480 total_nvlink_bytes=668945816 weight_to_token_ratio=0.197014 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.772458943e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278347776 dispatch_meta_bytes=229176 combine_payload_bytes=278347776 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=278576952 ingress_bytes=828850176 link_bytes=828850176 token_bytes=556924728 weight_movement_bytes=550502400 total_nvlink_bytes=1107427128 weight_to_token_ratio=0.988468 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.441506778e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279121920 dispatch_meta_bytes=228904 combine_payload_bytes=279121920 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279350824 ingress_bytes=389222400 link_bytes=389222400 token_bytes=558472744 weight_movement_bytes=110100480 total_nvlink_bytes=668573224 weight_to_token_ratio=0.197146 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.769217697e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44504 payload_rows_remote=38954 dup_rows=21032 dup_rows_remote=18312 dedup_ratio=0.320923 router_tokens_per_expert=[496, 509, 496, 501, 517, 496, 498, 503, 489, 506, 529, 525, 528, 528, 533, 503, 545, 564, 510, 504, 474, 516, 477, 487, 488, 501, 568, 490, 510, 504, 489, 500, 520, 542, 466, 494, 525, 531, 534, 523, 519, 528, 489, 518, 516, 495, 498, 529, 515, 544, 502, 501, 532, 516, 509, 521, 538, 527, 484, 531, 527, 494, 499, 517] payload_rows_per_dst=[5466, 5694, 5566, 5502, 5550, 5572, 5592, 5562] dup_rows_per_dst=[2566, 2588, 2588, 2598, 2720, 2612, 2688, 2672] combine_rows_per_src=[5466, 5694, 5566, 5502, 5550, 5572, 5592, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44616 payload_rows_remote=39140 dup_rows=20920 dup_rows_remote=18262 dedup_ratio=0.319214 router_tokens_per_expert=[524, 483, 509, 498, 514, 522, 497, 518, 520, 507, 532, 473, 538, 527, 531, 528, 494, 521, 492, 513, 500, 502, 493, 518, 531, 519, 537, 530, 496, 481, 537, 544, 519, 508, 519, 525, 466, 490, 542, 512, 476, 519, 502, 529, 483, 510, 548, 500, 507, 537, 492, 501, 524, 491, 517, 511, 521, 523, 479, 506, 518, 495, 547, 522] payload_rows_per_dst=[5604, 5632, 5464, 5642, 5606, 5476, 5622, 5570] dup_rows_per_dst=[2526, 2680, 2602, 2708, 2556, 2658, 2538, 2652] combine_rows_per_src=[5604, 5632, 5464, 5642, 5606, 5476, 5622, 5570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279422976 dispatch_meta_bytes=229432 combine_payload_bytes=279422976 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279652408 ingress_bytes=609724416 link_bytes=609724416 token_bytes=559075384 weight_movement_bytes=330301440 total_nvlink_bytes=889376824 weight_to_token_ratio=0.590799 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060411042e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277444608 dispatch_meta_bytes=227408 combine_payload_bytes=277444608 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277672016 ingress_bytes=387545088 link_bytes=387545088 token_bytes=555116624 weight_movement_bytes=110100480 total_nvlink_bytes=665217104 weight_to_token_ratio=0.198338 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.740046482e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280569856 dispatch_meta_bytes=229728 combine_payload_bytes=280569856 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280799584 ingress_bytes=390670336 link_bytes=390670336 token_bytes=561369440 weight_movement_bytes=110100480 total_nvlink_bytes=671469920 weight_to_token_ratio=0.196128 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.794399685e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279222272 dispatch_meta_bytes=229568 combine_payload_bytes=279222272 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279451840 ingress_bytes=389322752 link_bytes=389322752 token_bytes=558674112 weight_movement_bytes=110100480 total_nvlink_bytes=668774592 weight_to_token_ratio=0.197075 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.770962983e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44508 payload_rows_remote=38966 dup_rows=21028 dup_rows_remote=18304 dedup_ratio=0.320862 router_tokens_per_expert=[498, 510, 497, 504, 518, 503, 501, 504, 490, 510, 530, 527, 530, 529, 537, 503, 549, 567, 509, 508, 475, 516, 476, 487, 489, 501, 569, 489, 509, 505, 493, 500, 522, 542, 464, 495, 523, 532, 533, 522, 519, 527, 488, 517, 515, 495, 496, 528, 514, 543, 503, 497, 529, 515, 506, 517, 533, 520, 480, 531, 526, 493, 494, 516] payload_rows_per_dst=[5484, 5712, 5576, 5500, 5542, 5570, 5582, 5542] dup_rows_per_dst=[2586, 2600, 2598, 2610, 2724, 2600, 2666, 2644] combine_rows_per_src=[5484, 5712, 5576, 5500, 5542, 5570, 5582, 5542] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44640 payload_rows_remote=38968 dup_rows=20896 dup_rows_remote=18204 dedup_ratio=0.318848 router_tokens_per_expert=[515, 545, 549, 508, 555, 501, 470, 521, 510, 526, 502, 495, 532, 513, 546, 476, 546, 497, 505, 523, 503, 515, 518, 575, 517, 512, 519, 490, 511, 498, 481, 515, 493, 541, 536, 516, 489, 485, 466, 492, 497, 515, 532, 502, 504, 517, 528, 486, 506, 539, 507, 495, 494, 522, 533, 524, 441, 503, 517, 506, 508, 522, 533, 530] payload_rows_per_dst=[5652, 5538, 5672, 5578, 5572, 5466, 5608, 5554] dup_rows_per_dst=[2676, 2662, 2692, 2508, 2464, 2696, 2632, 2566] combine_rows_per_src=[5652, 5538, 5672, 5578, 5572, 5466, 5608, 5554] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44656 payload_rows_remote=39092 dup_rows=20880 dup_rows_remote=18328 dedup_ratio=0.318604 router_tokens_per_expert=[519, 494, 504, 515, 507, 516, 518, 519, 497, 491, 562, 507, 520, 529, 530, 507, 501, 536, 514, 523, 528, 530, 516, 536, 498, 499, 491, 528, 523, 492, 519, 508, 520, 556, 536, 500, 513, 529, 501, 528, 499, 467, 509, 497, 514, 524, 512, 517, 512, 522, 502, 481, 517, 528, 515, 470, 485, 481, 529, 501, 486, 531, 517, 492] payload_rows_per_dst=[5578, 5628, 5646, 5564, 5624, 5504, 5564, 5548] dup_rows_per_dst=[2606, 2658, 2722, 2552, 2742, 2574, 2530, 2496] combine_rows_per_src=[5578, 5628, 5646, 5564, 5624, 5504, 5564, 5548] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44388 payload_rows_remote=38808 dup_rows=21148 dup_rows_remote=18460 dedup_ratio=0.322693 router_tokens_per_expert=[525, 532, 528, 516, 557, 543, 568, 522, 544, 526, 513, 463, 534, 524, 547, 483, 508, 523, 508, 527, 479, 505, 528, 513, 525, 509, 520, 526, 478, 518, 538, 502, 467, 536, 500, 509, 518, 515, 509, 514, 496, 493, 513, 510, 518, 530, 472, 469, 533, 479, 566, 484, 486, 506, 507, 489, 478, 504, 481, 509, 510, 524, 511, 500] payload_rows_per_dst=[5778, 5580, 5518, 5520, 5476, 5522, 5538, 5456] dup_rows_per_dst=[2804, 2688, 2664, 2712, 2660, 2480, 2562, 2578] combine_rows_per_src=[5778, 5580, 5518, 5520, 5476, 5522, 5538, 5456] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279308288 dispatch_meta_bytes=229080 combine_payload_bytes=279308288 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279537368 ingress_bytes=389408768 link_bytes=389408768 token_bytes=558845656 weight_movement_bytes=110100480 total_nvlink_bytes=668946136 weight_to_token_ratio=0.197014 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.772458943e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279322624 dispatch_meta_bytes=228688 combine_payload_bytes=279322624 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279551312 ingress_bytes=499523584 link_bytes=499523584 token_bytes=558873936 weight_movement_bytes=220200960 total_nvlink_bytes=779074896 weight_to_token_ratio=0.394008 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.687536701e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44560 payload_rows_remote=38968 dup_rows=20976 dup_rows_remote=18362 dedup_ratio=0.320068 router_tokens_per_expert=[562, 521, 503, 507, 491, 489, 517, 494, 532, 528, 527, 506, 493, 531, 508, 561, 516, 463, 532, 521, 520, 489, 513, 506, 535, 508, 522, 485, 523, 518, 534, 539, 452, 498, 524, 512, 517, 494, 493, 516, 479, 501, 474, 542, 521, 503, 524, 532, 534, 494, 492, 527, 502, 485, 510, 513, 528, 517, 528, 519, 516, 512, 471, 544] payload_rows_per_dst=[5592, 5642, 5566, 5556, 5490, 5586, 5566, 5562] dup_rows_per_dst=[2614, 2730, 2554, 2734, 2522, 2566, 2548, 2708] combine_rows_per_src=[5592, 5642, 5566, 5556, 5490, 5586, 5566, 5562] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44506 payload_rows_remote=38902 dup_rows=21030 dup_rows_remote=18256 dedup_ratio=0.320892 router_tokens_per_expert=[534, 537, 567, 507, 525, 521, 521, 490, 532, 526, 518, 478, 506, 517, 502, 502, 511, 502, 529, 513, 501, 530, 496, 539, 525, 560, 513, 466, 510, 532, 507, 479, 517, 494, 466, 515, 536, 503, 511, 505, 515, 505, 521, 517, 551, 501, 534, 503, 516, 537, 507, 521, 469, 502, 478, 532, 516, 471, 514, 495, 481, 516, 496, 527] payload_rows_per_dst=[5470, 5320, 5442, 5416, 5714, 5792, 5748, 5604] dup_rows_per_dst=[2472, 2370, 2454, 2298, 2842, 2972, 2848, 2774] combine_rows_per_src=[5470, 5320, 5442, 5416, 5714, 5792, 5748, 5604] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44566 payload_rows_remote=38996 dup_rows=20970 dup_rows_remote=18412 dedup_ratio=0.319977 router_tokens_per_expert=[478, 529, 525, 541, 525, 532, 514, 540, 498, 503, 507, 515, 507, 565, 498, 527, 508, 500, 503, 539, 510, 535, 521, 528, 506, 514, 538, 537, 511, 484, 542, 556, 500, 516, 517, 515, 492, 496, 520, 508, 520, 501, 497, 513, 514, 505, 493, 521, 502, 505, 527, 498, 491, 478, 469, 497, 521, 483, 500, 546, 476, 489, 488, 534] payload_rows_per_dst=[5636, 5556, 5598, 5648, 5508, 5570, 5486, 5564] dup_rows_per_dst=[2732, 2684, 2690, 2728, 2620, 2558, 2448, 2510] combine_rows_per_src=[5636, 5556, 5598, 5648, 5508, 5570, 5486, 5564] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280211456 dispatch_meta_bytes=229680 combine_payload_bytes=280211456 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280441136 ingress_bytes=500412416 link_bytes=500412416 token_bytes=560652592 weight_movement_bytes=220200960 total_nvlink_bytes=780853552 weight_to_token_ratio=0.392758 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.702994951e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44436 payload_rows_remote=38944 dup_rows=21100 dup_rows_remote=18390 dedup_ratio=0.321960 router_tokens_per_expert=[521, 516, 482, 567, 514, 500, 509, 561, 513, 518, 539, 503, 490, 549, 531, 527, 554, 526, 525, 514, 505, 516, 516, 526, 516, 480, 533, 525, 491, 542, 522, 494, 499, 494, 486, 523, 508, 483, 484, 500, 530, 507, 541, 476, 519, 494, 473, 495, 504, 490, 499, 540, 507, 518, 534, 509, 542, 515, 484, 485, 496, 489, 508, 511] payload_rows_per_dst=[5652, 5654, 5612, 5542, 5436, 5558, 5492, 5490] dup_rows_per_dst=[2688, 2686, 2752, 2664, 2518, 2512, 2710, 2570] combine_rows_per_src=[5652, 5654, 5612, 5542, 5436, 5558, 5492, 5490] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278175744 dispatch_meta_bytes=229072 combine_payload_bytes=278175744 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278404816 ingress_bytes=608477184 link_bytes=608477184 token_bytes=556580560 weight_movement_bytes=330301440 total_nvlink_bytes=886882000 weight_to_token_ratio=0.593448 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058241900e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44660 payload_rows_remote=39104 dup_rows=20876 dup_rows_remote=18330 dedup_ratio=0.318542 router_tokens_per_expert=[518, 490, 504, 511, 505, 513, 515, 515, 492, 486, 559, 508, 516, 528, 529, 507, 500, 532, 511, 521, 524, 532, 515, 534, 500, 497, 490, 526, 522, 491, 518, 507, 523, 557, 538, 499, 514, 530, 501, 531, 498, 468, 511, 498, 514, 525, 513, 520, 513, 525, 504, 482, 523, 533, 516, 471, 489, 484, 532, 505, 490, 533, 517, 495] payload_rows_per_dst=[5558, 5610, 5634, 5556, 5642, 5510, 5574, 5576] dup_rows_per_dst=[2584, 2640, 2704, 2546, 2744, 2584, 2560, 2514] combine_rows_per_src=[5558, 5610, 5634, 5556, 5642, 5510, 5574, 5576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279222272 dispatch_meta_bytes=229064 combine_payload_bytes=279222272 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279451336 ingress_bytes=389322752 link_bytes=389322752 token_bytes=558673608 weight_movement_bytes=110100480 total_nvlink_bytes=668774088 weight_to_token_ratio=0.197075 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.770962983e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44558 payload_rows_remote=39076 dup_rows=20978 dup_rows_remote=18378 dedup_ratio=0.320099 router_tokens_per_expert=[531, 533, 561, 506, 520, 520, 519, 489, 528, 523, 519, 477, 502, 517, 501, 500, 508, 500, 525, 514, 502, 530, 497, 536, 518, 559, 511, 466, 510, 533, 508, 479, 518, 494, 469, 518, 536, 503, 510, 506, 516, 505, 520, 516, 551, 502, 537, 503, 515, 543, 508, 525, 473, 504, 480, 533, 519, 474, 517, 497, 485, 522, 498, 529] payload_rows_per_dst=[5568, 5384, 5544, 5416, 5740, 5748, 5676, 5482] dup_rows_per_dst=[2660, 2426, 2568, 2290, 2830, 2876, 2728, 2600] combine_rows_per_src=[5568, 5384, 5544, 5416, 5740, 5748, 5676, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44490 payload_rows_remote=38952 dup_rows=21046 dup_rows_remote=18430 dedup_ratio=0.321136 router_tokens_per_expert=[501, 489, 550, 511, 537, 507, 495, 507, 492, 529, 543, 512, 523, 501, 515, 525, 504, 502, 539, 512, 522, 530, 517, 515, 525, 489, 493, 487, 518, 473, 533, 540, 515, 544, 488, 494, 521, 475, 511, 514, 521, 535, 497, 503, 503, 539, 500, 563, 525, 530, 481, 514, 505, 502, 515, 505, 520, 493, 494, 504, 500, 508, 488, 525] payload_rows_per_dst=[5636, 5622, 5594, 5458, 5490, 5620, 5538, 5532] dup_rows_per_dst=[2558, 2658, 2688, 2658, 2634, 2702, 2616, 2532] combine_rows_per_src=[5636, 5622, 5594, 5458, 5490, 5620, 5538, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44566 payload_rows_remote=38962 dup_rows=20970 dup_rows_remote=18396 dedup_ratio=0.319977 router_tokens_per_expert=[510, 450, 510, 485, 510, 505, 505, 523, 538, 544, 543, 510, 523, 508, 490, 519, 490, 512, 517, 517, 491, 514, 502, 546, 499, 518, 505, 541, 522, 498, 522, 522, 484, 516, 551, 496, 491, 491, 533, 490, 501, 507, 524, 520, 505, 488, 516, 500, 494, 535, 512, 494, 564, 484, 541, 501, 507, 538, 524, 530, 537, 524, 506, 475] payload_rows_per_dst=[5492, 5632, 5604, 5568, 5470, 5574, 5636, 5590] dup_rows_per_dst=[2504, 2718, 2574, 2686, 2634, 2548, 2614, 2692] combine_rows_per_src=[5492, 5632, 5604, 5568, 5470, 5574, 5636, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44522 payload_rows_remote=38998 dup_rows=21014 dup_rows_remote=18316 dedup_ratio=0.320648 router_tokens_per_expert=[508, 517, 495, 504, 479, 524, 506, 531, 504, 509, 507, 509, 539, 512, 494, 537, 525, 495, 511, 465, 451, 514, 540, 555, 543, 538, 517, 510, 511, 484, 499, 527, 515, 524, 500, 506, 490, 527, 502, 506, 492, 506, 493, 540, 520, 513, 506, 545, 494, 508, 520, 504, 532, 510, 475, 505, 508, 506, 528, 505, 505, 536, 531, 556] payload_rows_per_dst=[5502, 5524, 5592, 5570, 5596, 5582, 5516, 5640] dup_rows_per_dst=[2626, 2698, 2520, 2688, 2544, 2648, 2580, 2710] combine_rows_per_src=[5502, 5524, 5592, 5570, 5596, 5582, 5516, 5640] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280555520 dispatch_meta_bytes=229608 combine_payload_bytes=280555520 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280785128 ingress_bytes=390656000 link_bytes=390656000 token_bytes=561340648 weight_movement_bytes=110100480 total_nvlink_bytes=671441128 weight_to_token_ratio=0.196138 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.794150359e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44530 payload_rows_remote=38914 dup_rows=21006 dup_rows_remote=18362 dedup_ratio=0.320526 router_tokens_per_expert=[561, 515, 501, 504, 490, 487, 513, 492, 531, 527, 525, 505, 492, 529, 507, 559, 515, 460, 528, 516, 515, 490, 513, 508, 532, 507, 521, 486, 522, 517, 532, 537, 452, 496, 525, 511, 517, 495, 497, 516, 482, 502, 476, 543, 523, 503, 526, 534, 538, 496, 493, 529, 505, 490, 511, 516, 530, 519, 533, 520, 519, 515, 473, 546] payload_rows_per_dst=[5616, 5578, 5552, 5598, 5492, 5606, 5516, 5572] dup_rows_per_dst=[2644, 2658, 2538, 2786, 2534, 2572, 2536, 2738] combine_rows_per_src=[5616, 5578, 5552, 5598, 5492, 5606, 5516, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280297472 dispatch_meta_bytes=229736 combine_payload_bytes=280297472 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280527208 ingress_bytes=390397952 link_bytes=390397952 token_bytes=560824680 weight_movement_bytes=110100480 total_nvlink_bytes=670925160 weight_to_token_ratio=0.196319 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.789662480e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=280096768 dispatch_meta_bytes=229816 combine_payload_bytes=280096768 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=280326584 ingress_bytes=500297728 link_bytes=500297728 token_bytes=560423352 weight_movement_bytes=220200960 total_nvlink_bytes=780624312 weight_to_token_ratio=0.392919 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.701000338e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279207936 dispatch_meta_bytes=229528 combine_payload_bytes=279207936 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279437464 ingress_bytes=609509376 link_bytes=609509376 token_bytes=558645400 weight_movement_bytes=330301440 total_nvlink_bytes=888946840 weight_to_token_ratio=0.591254 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060037052e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279279616 dispatch_meta_bytes=229432 combine_payload_bytes=279279616 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279509048 ingress_bytes=499480576 link_bytes=499480576 token_bytes=558788664 weight_movement_bytes=220200960 total_nvlink_bytes=778989624 weight_to_token_ratio=0.394068 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.686788721e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279537664 dispatch_meta_bytes=229256 combine_payload_bytes=279537664 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279766920 ingress_bytes=499738624 link_bytes=499738624 token_bytes=559304584 weight_movement_bytes=220200960 total_nvlink_bytes=779505544 weight_to_token_ratio=0.393705 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.691276600e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278935552 dispatch_meta_bytes=229104 combine_payload_bytes=278935552 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279164656 ingress_bytes=389036032 link_bytes=389036032 token_bytes=558100208 weight_movement_bytes=110100480 total_nvlink_bytes=668200688 weight_to_token_ratio=0.197277 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.765976451e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44350 payload_rows_remote=38802 dup_rows=21186 dup_rows_remote=18478 dedup_ratio=0.323273 router_tokens_per_expert=[493, 531, 509, 527, 525, 519, 516, 524, 479, 526, 512, 515, 519, 535, 513, 504, 527, 493, 485, 521, 502, 530, 523, 518, 528, 492, 484, 507, 499, 511, 531, 481, 512, 518, 494, 503, 525, 525, 505, 546, 522, 511, 505, 526, 535, 502, 520, 495, 512, 519, 515, 483, 490, 535, 511, 508, 502, 521, 543, 493, 513, 508, 476, 516] payload_rows_per_dst=[5622, 5532, 5546, 5466, 5548, 5598, 5488, 5550] dup_rows_per_dst=[2666, 2674, 2652, 2600, 2708, 2634, 2658, 2594] combine_rows_per_src=[5622, 5532, 5546, 5466, 5548, 5598, 5488, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44592 payload_rows_remote=39024 dup_rows=20944 dup_rows_remote=18382 dedup_ratio=0.319580 router_tokens_per_expert=[476, 527, 523, 539, 520, 532, 510, 535, 493, 502, 508, 513, 505, 563, 498, 523, 507, 500, 501, 538, 507, 534, 520, 528, 507, 515, 535, 538, 509, 485, 542, 555, 502, 517, 517, 515, 495, 494, 519, 506, 520, 504, 497, 514, 514, 505, 492, 519, 503, 506, 529, 499, 498, 480, 471, 498, 524, 489, 504, 549, 479, 492, 489, 540] payload_rows_per_dst=[5624, 5550, 5594, 5644, 5514, 5568, 5508, 5590] dup_rows_per_dst=[2700, 2660, 2676, 2728, 2616, 2562, 2460, 2542] combine_rows_per_src=[5624, 5550, 5594, 5644, 5514, 5568, 5508, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278132736 dispatch_meta_bytes=229120 combine_payload_bytes=278132736 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278361856 ingress_bytes=388233216 link_bytes=388233216 token_bytes=556494592 weight_movement_bytes=110100480 total_nvlink_bytes=666595072 weight_to_token_ratio=0.197846 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.752014160e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279322624 dispatch_meta_bytes=229320 combine_payload_bytes=279322624 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279551944 ingress_bytes=609624064 link_bytes=609624064 token_bytes=558874568 weight_movement_bytes=330301440 total_nvlink_bytes=889176008 weight_to_token_ratio=0.591012 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060236513e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278849536 dispatch_meta_bytes=228632 combine_payload_bytes=278849536 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279078168 ingress_bytes=388950016 link_bytes=388950016 token_bytes=557927704 weight_movement_bytes=110100480 total_nvlink_bytes=668028184 weight_to_token_ratio=0.197338 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.764480491e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279523328 dispatch_meta_bytes=229632 combine_payload_bytes=279523328 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279752960 ingress_bytes=389623808 link_bytes=389623808 token_bytes=559276288 weight_movement_bytes=110100480 total_nvlink_bytes=669376768 weight_to_token_ratio=0.196862 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.776198842e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279150592 dispatch_meta_bytes=229336 combine_payload_bytes=279150592 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279379928 ingress_bytes=389251072 link_bytes=389251072 token_bytes=558530520 weight_movement_bytes=110100480 total_nvlink_bytes=668631000 weight_to_token_ratio=0.197125 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.769716350e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44332 payload_rows_remote=38780 dup_rows=21204 dup_rows_remote=18498 dedup_ratio=0.323547 router_tokens_per_expert=[498, 534, 510, 535, 528, 522, 517, 525, 480, 528, 518, 518, 522, 535, 515, 506, 529, 494, 487, 521, 504, 531, 526, 520, 527, 493, 485, 509, 499, 511, 530, 485, 513, 517, 500, 500, 526, 525, 506, 542, 520, 507, 502, 526, 533, 500, 518, 493, 511, 519, 514, 478, 489, 533, 508, 505, 500, 515, 540, 490, 507, 504, 472, 513] payload_rows_per_dst=[5634, 5552, 5548, 5470, 5552, 5584, 5472, 5520] dup_rows_per_dst=[2704, 2692, 2676, 2608, 2706, 2614, 2642, 2562] combine_rows_per_src=[5634, 5552, 5548, 5470, 5552, 5584, 5472, 5520] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44552 payload_rows_remote=38944 dup_rows=20984 dup_rows_remote=18392 dedup_ratio=0.320190 router_tokens_per_expert=[512, 453, 514, 488, 511, 509, 509, 527, 538, 545, 542, 516, 523, 510, 491, 523, 493, 514, 520, 515, 494, 517, 501, 546, 498, 518, 503, 543, 519, 495, 522, 523, 483, 516, 550, 496, 490, 493, 534, 494, 502, 509, 521, 520, 505, 488, 517, 498, 493, 532, 512, 493, 565, 477, 539, 498, 502, 535, 521, 524, 533, 522, 501, 473] payload_rows_per_dst=[5510, 5646, 5608, 5560, 5468, 5566, 5632, 5562] dup_rows_per_dst=[2536, 2730, 2592, 2682, 2644, 2554, 2586, 2660] combine_rows_per_src=[5510, 5646, 5608, 5560, 5468, 5566, 5632, 5562] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44522 payload_rows_remote=38876 dup_rows=21014 dup_rows_remote=18246 dedup_ratio=0.320648 router_tokens_per_expert=[521, 517, 546, 501, 538, 473, 501, 529, 477, 525, 485, 544, 493, 550, 506, 523, 495, 486, 508, 536, 547, 494, 521, 528, 531, 509, 559, 491, 497, 556, 545, 519, 516, 491, 465, 503, 511, 526, 531, 512, 511, 535, 477, 487, 509, 483, 522, 503, 483, 511, 503, 508, 515, 539, 494, 500, 512, 527, 523, 494, 519, 502, 496, 509] payload_rows_per_dst=[5588, 5602, 5600, 5646, 5504, 5520, 5548, 5514] dup_rows_per_dst=[2664, 2604, 2630, 2768, 2606, 2534, 2558, 2650] combine_rows_per_src=[5588, 5602, 5600, 5646, 5504, 5520, 5548, 5514] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44538 payload_rows_remote=38992 dup_rows=20998 dup_rows_remote=18284 dedup_ratio=0.320404 router_tokens_per_expert=[515, 523, 500, 506, 482, 524, 507, 536, 509, 514, 509, 510, 542, 516, 493, 537, 526, 497, 511, 469, 452, 516, 540, 554, 542, 538, 519, 507, 511, 482, 503, 525, 516, 524, 501, 507, 492, 528, 501, 504, 488, 504, 494, 536, 519, 513, 504, 545, 492, 506, 520, 500, 533, 509, 471, 502, 507, 503, 522, 501, 501, 532, 524, 554] payload_rows_per_dst=[5532, 5546, 5594, 5572, 5598, 5568, 5504, 5624] dup_rows_per_dst=[2654, 2714, 2536, 2682, 2548, 2638, 2562, 2664] combine_rows_per_src=[5532, 5546, 5594, 5572, 5598, 5568, 5504, 5624] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=277975040 dispatch_meta_bytes=229112 combine_payload_bytes=277975040 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278204152 ingress_bytes=388075520 link_bytes=388075520 token_bytes=556179192 weight_movement_bytes=110100480 total_nvlink_bytes=666279672 weight_to_token_ratio=0.197959 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.749271567e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44428 payload_rows_remote=38868 dup_rows=21108 dup_rows_remote=18604 dedup_ratio=0.322083 router_tokens_per_expert=[470, 520, 509, 483, 502, 498, 520, 512, 550, 513, 500, 506, 503, 519, 502, 515, 529, 504, 485, 484, 516, 499, 514, 516, 514, 529, 541, 512, 520, 511, 551, 503, 448, 534, 553, 504, 540, 513, 516, 528, 505, 502, 462, 559, 485, 462, 508, 515, 493, 536, 531, 494, 536, 511, 551, 498, 504, 527, 540, 499, 538, 515, 493, 518] payload_rows_per_dst=[5560, 5536, 5550, 5602, 5634, 5420, 5572, 5554] dup_rows_per_dst=[2504, 2644, 2654, 2650, 2638, 2576, 2728, 2714] combine_rows_per_src=[5560, 5536, 5550, 5602, 5634, 5420, 5572, 5554] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279150592 dispatch_meta_bytes=229344 combine_payload_bytes=279150592 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279379936 ingress_bytes=499351552 link_bytes=499351552 token_bytes=558530528 weight_movement_bytes=220200960 total_nvlink_bytes=778731488 weight_to_token_ratio=0.394251 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.684544781e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44344 payload_rows_remote=38738 dup_rows=21192 dup_rows_remote=18540 dedup_ratio=0.323364 router_tokens_per_expert=[495, 513, 517, 542, 500, 532, 542, 540, 502, 540, 499, 474, 508, 517, 525, 504, 511, 487, 509, 527, 524, 516, 527, 492, 505, 535, 500, 513, 520, 486, 492, 502, 512, 505, 521, 565, 483, 513, 521, 500, 498, 478, 520, 524, 523, 506, 531, 527, 512, 532, 552, 497, 516, 483, 546, 468, 519, 460, 521, 512, 528, 488, 514, 497] payload_rows_per_dst=[5456, 5392, 5474, 5602, 5500, 5660, 5654, 5606] dup_rows_per_dst=[2510, 2568, 2506, 2728, 2574, 2868, 2786, 2652] combine_rows_per_src=[5456, 5392, 5474, 5602, 5500, 5660, 5654, 5606] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44496 payload_rows_remote=38888 dup_rows=21040 dup_rows_remote=18406 dedup_ratio=0.321045 router_tokens_per_expert=[492, 474, 537, 514, 524, 525, 492, 506, 495, 508, 549, 506, 519, 492, 493, 496, 551, 533, 475, 496, 519, 515, 513, 489, 537, 540, 527, 501, 488, 471, 520, 520, 496, 530, 536, 487, 498, 507, 497, 524, 511, 573, 507, 473, 531, 527, 493, 506, 508, 532, 528, 539, 486, 496, 492, 521, 478, 528, 526, 518, 530, 515, 540, 518] payload_rows_per_dst=[5548, 5496, 5554, 5548, 5500, 5608, 5560, 5682] dup_rows_per_dst=[2580, 2620, 2628, 2660, 2650, 2634, 2644, 2624] combine_rows_per_src=[5548, 5496, 5554, 5548, 5500, 5608, 5560, 5682] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278663168 dispatch_meta_bytes=228488 combine_payload_bytes=278663168 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278891656 ingress_bytes=608964608 link_bytes=608964608 token_bytes=557554824 weight_movement_bytes=330301440 total_nvlink_bytes=887856264 weight_to_token_ratio=0.592411 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059089611e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44472 payload_rows_remote=38890 dup_rows=21064 dup_rows_remote=18590 dedup_ratio=0.321411 router_tokens_per_expert=[524, 544, 494, 545, 480, 495, 516, 503, 511, 526, 524, 523, 507, 529, 548, 555, 488, 506, 521, 498, 527, 526, 476, 498, 499, 516, 548, 499, 482, 480, 530, 514, 518, 505, 523, 531, 488, 554, 538, 468, 492, 505, 507, 474, 483, 520, 548, 528, 508, 501, 533, 518, 503, 507, 496, 462, 553, 477, 503, 484, 553, 507, 549, 500] payload_rows_per_dst=[5574, 5632, 5490, 5528, 5564, 5542, 5582, 5560] dup_rows_per_dst=[2628, 2814, 2590, 2608, 2686, 2572, 2474, 2692] combine_rows_per_src=[5574, 5632, 5490, 5528, 5564, 5542, 5582, 5560] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279494656 dispatch_meta_bytes=229104 combine_payload_bytes=279494656 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279723760 ingress_bytes=609796096 link_bytes=609796096 token_bytes=559218416 weight_movement_bytes=330301440 total_nvlink_bytes=889519856 weight_to_token_ratio=0.590648 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060535705e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278605824 dispatch_meta_bytes=229888 combine_payload_bytes=278605824 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278835712 ingress_bytes=498806784 link_bytes=498806784 token_bytes=557441536 weight_movement_bytes=220200960 total_nvlink_bytes=777642496 weight_to_token_ratio=0.395021 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.675070370e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=277673984 dispatch_meta_bytes=229112 combine_payload_bytes=277673984 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277903096 ingress_bytes=387774464 link_bytes=387774464 token_bytes=555577080 weight_movement_bytes=110100480 total_nvlink_bytes=665677560 weight_to_token_ratio=0.198173 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.744035708e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278749184 dispatch_meta_bytes=229176 combine_payload_bytes=278749184 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278978360 ingress_bytes=388849664 link_bytes=388849664 token_bytes=557727544 weight_movement_bytes=110100480 total_nvlink_bytes=667828024 weight_to_token_ratio=0.197409 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.762735205e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278763520 dispatch_meta_bytes=229920 combine_payload_bytes=278763520 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278993440 ingress_bytes=388864000 link_bytes=388864000 token_bytes=557756960 weight_movement_bytes=110100480 total_nvlink_bytes=667857440 weight_to_token_ratio=0.197399 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.762984531e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44454 payload_rows_remote=38940 dup_rows=21082 dup_rows_remote=18572 dedup_ratio=0.321686 router_tokens_per_expert=[500, 518, 549, 567, 488, 492, 506, 522, 483, 516, 498, 518, 527, 502, 528, 554, 516, 537, 508, 535, 528, 468, 530, 535, 536, 530, 508, 501, 525, 539, 502, 486, 504, 501, 511, 536, 492, 455, 526, 487, 535, 510, 485, 498, 527, 539, 511, 508, 464, 502, 520, 497, 510, 491, 530, 533, 516, 510, 497, 510, 502, 480, 512, 517] payload_rows_per_dst=[5628, 5562, 5596, 5576, 5514, 5578, 5474, 5526] dup_rows_per_dst=[2656, 2690, 2718, 2678, 2510, 2648, 2620, 2562] combine_rows_per_src=[5628, 5562, 5596, 5576, 5514, 5578, 5474, 5526] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44352 payload_rows_remote=38724 dup_rows=21184 dup_rows_remote=18484 dedup_ratio=0.323242 router_tokens_per_expert=[526, 463, 485, 502, 551, 506, 539, 532, 533, 493, 498, 494, 501, 517, 544, 534, 519, 511, 510, 510, 535, 521, 534, 524, 498, 553, 503, 485, 498, 501, 496, 462, 513, 505, 528, 544, 520, 519, 541, 504, 513, 488, 520, 471, 509, 503, 531, 478, 526, 517, 515, 467, 501, 538, 520, 528, 528, 482, 475, 533, 514, 514, 527, 518] payload_rows_per_dst=[5628, 5540, 5628, 5428, 5596, 5384, 5534, 5614] dup_rows_per_dst=[2580, 2688, 2700, 2564, 2752, 2642, 2690, 2568] combine_rows_per_src=[5628, 5540, 5628, 5428, 5596, 5384, 5534, 5614] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5549706712 prefetch_bytes=1101004800 reduce_bytes=0 weight_movement_bytes=1101004800 egress_bytes=2775991768 ingress_bytes=3874719744 link_bytes=3874719744 total_nvlink_bytes=6650711512 weight_to_token_ratio=0.198390 prefetch_share_of_ingress=0.284151 dedup_ratio=0.322253 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=0 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043382 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=6.738774917e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5589063616 prefetch_bytes=1101004800 reduce_bytes=220200960 weight_movement_bytes=1321205760 egress_bytes=2795679680 ingress_bytes=4114589696 link_bytes=4114589696 total_nvlink_bytes=6910269376 weight_to_token_ratio=0.236391 prefetch_share_of_ingress=0.267586 dedup_ratio=0.320782 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=2 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043750 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.155948215e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5589608440 prefetch_bytes=990904320 reduce_bytes=110100480 weight_movement_bytes=1101004800 egress_bytes=2795952120 ingress_bytes=3894661120 link_bytes=3894661120 total_nvlink_bytes=6690613240 weight_to_token_ratio=0.196974 prefetch_share_of_ingress=0.254426 dedup_ratio=0.320557 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=1 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=6.773456249e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44582 payload_rows_remote=38900 dup_rows=20954 dup_rows_remote=18304 dedup_ratio=0.319733 router_tokens_per_expert=[485, 526, 463, 566, 535, 525, 536, 515, 489, 536, 518, 505, 506, 499, 493, 501, 518, 529, 513, 544, 543, 520, 520, 495, 523, 539, 509, 527, 506, 521, 533, 508, 497, 505, 511, 515, 502, 517, 484, 536, 498, 490, 509, 501, 521, 497, 504, 522, 522, 472, 494, 507, 496, 535, 503, 496, 504, 491, 517, 515, 483, 515, 543, 520] payload_rows_per_dst=[5570, 5548, 5624, 5682, 5538, 5502, 5524, 5594] dup_rows_per_dst=[2732, 2546, 2740, 2650, 2596, 2582, 2526, 2582] combine_rows_per_src=[5570, 5548, 5624, 5682, 5538, 5502, 5524, 5594] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44602 payload_rows_remote=39094 dup_rows=20934 dup_rows_remote=18360 dedup_ratio=0.319427 router_tokens_per_expert=[539, 524, 540, 528, 502, 524, 502, 505, 491, 517, 496, 495, 529, 479, 532, 502, 519, 499, 525, 528, 530, 499, 495, 449, 499, 478, 484, 512, 485, 519, 506, 485, 530, 534, 558, 499, 536, 496, 587, 535, 498, 518, 516, 516, 503, 513, 521, 507, 507, 505, 526, 489, 520, 530, 525, 510, 519, 489, 519, 518, 513, 500, 526, 488] payload_rows_per_dst=[5594, 5508, 5492, 5486, 5778, 5592, 5582, 5570] dup_rows_per_dst=[2734, 2574, 2596, 2450, 2772, 2592, 2642, 2574] combine_rows_per_src=[5594, 5508, 5492, 5486, 5778, 5592, 5582, 5570] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44374 payload_rows_remote=38864 dup_rows=21162 dup_rows_remote=18554 dedup_ratio=0.322906 router_tokens_per_expert=[529, 504, 510, 515, 508, 515, 505, 473, 512, 491, 493, 541, 518, 522, 575, 527, 517, 533, 499, 515, 472, 530, 510, 524, 523, 535, 514, 514, 525, 513, 539, 511, 504, 486, 489, 527, 503, 460, 489, 545, 499, 503, 520, 526, 495, 513, 498, 512, 515, 550, 530, 523, 509, 491, 518, 486, 508, 513, 540, 492, 488, 507, 515, 502] payload_rows_per_dst=[5510, 5666, 5520, 5566, 5450, 5582, 5624, 5456] dup_rows_per_dst=[2608, 2692, 2680, 2782, 2556, 2550, 2620, 2674] combine_rows_per_src=[5510, 5666, 5520, 5566, 5450, 5582, 5624, 5456] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279121920 dispatch_meta_bytes=230048 combine_payload_bytes=279121920 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279351968 ingress_bytes=499322880 link_bytes=499322880 token_bytes=558473888 weight_movement_bytes=220200960 total_nvlink_bytes=778674848 weight_to_token_ratio=0.394291 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.684046128e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=277573632 dispatch_meta_bytes=228832 combine_payload_bytes=277573632 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277802464 ingress_bytes=497774592 link_bytes=497774592 token_bytes=555376096 weight_movement_bytes=220200960 total_nvlink_bytes=775577056 weight_to_token_ratio=0.396490 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.657118853e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278835200 dispatch_meta_bytes=228816 combine_payload_bytes=278835200 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279064016 ingress_bytes=388935680 link_bytes=388935680 token_bytes=557899216 weight_movement_bytes=110100480 total_nvlink_bytes=667999696 weight_to_token_ratio=0.197348 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.764231164e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=280225792 dispatch_meta_bytes=229816 combine_payload_bytes=280225792 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280455608 ingress_bytes=500426752 link_bytes=500426752 token_bytes=560681400 weight_movement_bytes=220200960 total_nvlink_bytes=780882360 weight_to_token_ratio=0.392738 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.703244278e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5577506848 prefetch_bytes=990904320 reduce_bytes=550502400 weight_movement_bytes=1541406720 egress_bytes=2789900320 ingress_bytes=4329013248 link_bytes=4329013248 total_nvlink_bytes=7118913568 weight_to_token_ratio=0.276361 prefetch_share_of_ingress=0.228898 dedup_ratio=0.321698 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=5 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.528866039e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278577152 dispatch_meta_bytes=229672 combine_payload_bytes=278577152 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=278806824 ingress_bytes=718979072 link_bytes=718979072 token_bytes=557383976 weight_movement_bytes=440401920 total_nvlink_bytes=997785896 weight_to_token_ratio=0.790123 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.250422858e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5586566648 prefetch_bytes=880803840 reduce_bytes=1101004800 weight_movement_bytes=1981808640 egress_bytes=2794429944 ingress_bytes=4773945344 link_bytes=4773945344 total_nvlink_bytes=7568375288 weight_to_token_ratio=0.354745 prefetch_share_of_ingress=0.184502 dedup_ratio=0.320502 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=10 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.302676132e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5579509624 prefetch_bytes=880803840 reduce_bytes=1981808640 weight_movement_bytes=2862612480 egress_bytes=2790899576 ingress_bytes=5651222528 link_bytes=5651222528 total_nvlink_bytes=8442122104 weight_to_token_ratio=0.513058 prefetch_share_of_ingress=0.155861 dedup_ratio=0.320435 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=18 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.828405443e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5588688824 prefetch_bytes=770703360 reduce_bytes=1871708160 weight_movement_bytes=2642411520 egress_bytes=2795491256 ingress_bytes=5435609088 link_bytes=5435609088 total_nvlink_bytes=8231100344 weight_to_token_ratio=0.472814 prefetch_share_of_ingress=0.141788 dedup_ratio=0.320392 prefetch_slots=14 unused_prefetch_slots=146 slot_utilization=0.087500 reduce_slots=17 remote_demand=7 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=12319 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.453418208e-05 prefetch_bytes_per_gemm_flop=1.340379902e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5574145552 prefetch_bytes=990904320 reduce_bytes=1871708160 weight_movement_bytes=2862612480 egress_bytes=2788216336 ingress_bytes=5648541696 link_bytes=5648541696 total_nvlink_bytes=8436758032 weight_to_token_ratio=0.513552 prefetch_share_of_ingress=0.175427 dedup_ratio=0.320978 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=17 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.823743035e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none - [2026-08-26 15:12:38] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 366.2 | throughput per GPU (TFLOP/s/GPU): 337.2 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.031645E+01 | loss scale: 1.0 | grad norm: 8.132 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44490 payload_rows_remote=38904 dup_rows=21046 dup_rows_remote=18424 dedup_ratio=0.321136 router_tokens_per_expert=[503, 498, 505, 506, 502, 532, 501, 529, 507, 485, 502, 532, 496, 501, 498, 541, 484, 511, 562, 529, 492, 481, 491, 554, 532, 507, 510, 522, 510, 563, 506, 478, 518, 508, 478, 545, 484, 471, 466, 549, 533, 490, 518, 544, 478, 502, 520, 529, 566, 525, 479, 489, 543, 528, 524, 476, 513, 555, 519, 484, 515, 504, 527, 518] payload_rows_per_dst=[5484, 5592, 5586, 5566, 5500, 5612, 5602, 5548] dup_rows_per_dst=[2668, 2532, 2622, 2690, 2538, 2616, 2658, 2722] combine_rows_per_src=[5484, 5592, 5586, 5566, 5500, 5612, 5602, 5548] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44408 payload_rows_remote=38800 dup_rows=21128 dup_rows_remote=18330 dedup_ratio=0.322388 router_tokens_per_expert=[525, 531, 489, 536, 503, 491, 517, 507, 490, 516, 528, 492, 535, 524, 531, 505, 558, 547, 500, 547, 551, 498, 511, 525, 503, 488, 495, 480, 511, 522, 510, 511, 527, 539, 524, 490, 518, 528, 540, 537, 507, 494, 508, 496, 528, 530, 510, 520, 509, 482, 490, 494, 521, 537, 485, 495, 485, 483, 520, 478, 502, 496, 507, 511] payload_rows_per_dst=[5574, 5508, 5710, 5488, 5608, 5594, 5420, 5506] dup_rows_per_dst=[2624, 2734, 2764, 2552, 2798, 2592, 2606, 2458] combine_rows_per_src=[5574, 5508, 5710, 5488, 5608, 5594, 5420, 5506] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44786 payload_rows_remote=39218 dup_rows=20750 dup_rows_remote=18220 dedup_ratio=0.316620 router_tokens_per_expert=[541, 543, 471, 502, 509, 500, 529, 503, 481, 509, 490, 495, 525, 564, 522, 504, 509, 487, 549, 521, 557, 476, 464, 509, 526, 523, 524, 516, 509, 477, 490, 484, 464, 513, 513, 483, 515, 508, 506, 545, 497, 528, 476, 532, 525, 547, 522, 520, 489, 483, 529, 532, 542, 488, 555, 519, 513, 533, 490, 498, 531, 523, 525, 515] payload_rows_per_dst=[5576, 5632, 5588, 5568, 5576, 5612, 5600, 5634] dup_rows_per_dst=[2620, 2548, 2556, 2530, 2518, 2682, 2674, 2622] combine_rows_per_src=[5576, 5632, 5588, 5568, 5576, 5612, 5600, 5634] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44602 payload_rows_remote=39110 dup_rows=20934 dup_rows_remote=18426 dedup_ratio=0.319427 router_tokens_per_expert=[493, 527, 517, 520, 545, 498, 536, 536, 504, 546, 540, 515, 516, 513, 510, 499, 515, 538, 522, 483, 501, 495, 525, 532, 485, 507, 537, 521, 476, 565, 517, 491, 470, 542, 514, 495, 528, 503, 524, 503, 526, 506, 491, 519, 494, 484, 494, 486, 538, 516, 525, 496, 482, 549, 527, 521, 513, 491, 497, 501, 515, 490, 481, 522] payload_rows_per_dst=[5618, 5604, 5564, 5646, 5588, 5492, 5652, 5438] dup_rows_per_dst=[2726, 2682, 2658, 2552, 2570, 2508, 2656, 2582] combine_rows_per_src=[5618, 5604, 5564, 5646, 5588, 5492, 5652, 5438] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44694 payload_rows_remote=38994 dup_rows=20842 dup_rows_remote=18010 dedup_ratio=0.318024 router_tokens_per_expert=[506, 493, 552, 511, 492, 522, 521, 508, 520, 506, 499, 531, 505, 508, 508, 495, 495, 540, 507, 542, 494, 502, 502, 514, 541, 477, 506, 529, 479, 531, 482, 512, 502, 528, 530, 512, 527, 497, 507, 515, 510, 480, 542, 497, 514, 513, 505, 523, 529, 517, 497, 481, 575, 505, 525, 511, 475, 529, 559, 478, 536, 481, 528, 510] payload_rows_per_dst=[5458, 5438, 5322, 5632, 5672, 5830, 5642, 5700] dup_rows_per_dst=[2484, 2366, 2266, 2750, 2698, 2804, 2642, 2832] combine_rows_per_src=[5458, 5438, 5322, 5632, 5672, 5830, 5642, 5700] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44570 payload_rows_remote=38932 dup_rows=20966 dup_rows_remote=18378 dedup_ratio=0.319916 router_tokens_per_expert=[516, 518, 512, 539, 522, 507, 507, 497, 520, 541, 517, 507, 509, 486, 519, 528, 514, 541, 491, 505, 496, 549, 506, 536, 493, 512, 507, 495, 490, 489, 543, 482, 544, 560, 525, 508, 513, 514, 515, 513, 501, 512, 518, 499, 511, 502, 484, 520, 494, 519, 501, 547, 495, 531, 512, 514, 531, 468, 508, 490, 497, 530, 481, 517] payload_rows_per_dst=[5600, 5524, 5596, 5512, 5626, 5562, 5638, 5512] dup_rows_per_dst=[2636, 2730, 2680, 2510, 2758, 2532, 2588, 2532] combine_rows_per_src=[5600, 5524, 5596, 5512, 5626, 5562, 5638, 5512] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44490 payload_rows_remote=38866 dup_rows=21046 dup_rows_remote=18266 dedup_ratio=0.321136 router_tokens_per_expert=[513, 529, 471, 516, 490, 498, 515, 451, 523, 505, 509, 508, 541, 533, 550, 533, 536, 519, 526, 540, 519, 535, 484, 487, 506, 534, 517, 509, 494, 540, 524, 538, 529, 488, 525, 497, 515, 481, 487, 519, 473, 505, 485, 491, 520, 517, 535, 509, 527, 510, 530, 505, 490, 491, 518, 484, 533, 507, 503, 563, 505, 497, 504, 532] payload_rows_per_dst=[5506, 5624, 5608, 5612, 5524, 5520, 5512, 5584] dup_rows_per_dst=[2460, 2780, 2684, 2712, 2558, 2550, 2598, 2704] combine_rows_per_src=[5506, 5624, 5608, 5612, 5524, 5520, 5512, 5584] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44534 payload_rows_remote=38894 dup_rows=21002 dup_rows_remote=18244 dedup_ratio=0.320465 router_tokens_per_expert=[519, 517, 541, 501, 537, 470, 497, 524, 475, 523, 483, 545, 490, 548, 504, 521, 495, 487, 509, 535, 545, 494, 520, 528, 528, 508, 559, 490, 496, 557, 543, 518, 517, 492, 467, 500, 513, 524, 531, 512, 510, 539, 478, 488, 507, 486, 523, 503, 485, 514, 504, 507, 515, 540, 496, 504, 514, 531, 530, 497, 521, 504, 499, 510] payload_rows_per_dst=[5572, 5598, 5602, 5640, 5506, 5526, 5556, 5534] dup_rows_per_dst=[2640, 2580, 2624, 2758, 2606, 2542, 2574, 2678] combine_rows_per_src=[5572, 5598, 5602, 5640, 5506, 5526, 5556, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44414 payload_rows_remote=38880 dup_rows=21122 dup_rows_remote=18540 dedup_ratio=0.322296 router_tokens_per_expert=[494, 507, 514, 539, 499, 523, 538, 536, 502, 537, 498, 472, 507, 518, 525, 504, 510, 487, 507, 523, 524, 514, 525, 494, 507, 533, 499, 510, 520, 485, 495, 504, 512, 506, 520, 565, 484, 511, 520, 500, 499, 480, 522, 525, 525, 508, 533, 529, 512, 534, 551, 502, 518, 485, 551, 468, 521, 462, 522, 513, 531, 492, 515, 502] payload_rows_per_dst=[5568, 5458, 5576, 5524, 5602, 5558, 5594, 5534] dup_rows_per_dst=[2654, 2668, 2574, 2582, 2712, 2684, 2666, 2582] combine_rows_per_src=[5568, 5458, 5576, 5524, 5602, 5558, 5594, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44436 payload_rows_remote=38934 dup_rows=21100 dup_rows_remote=18374 dedup_ratio=0.321960 router_tokens_per_expert=[521, 513, 478, 564, 511, 500, 509, 560, 512, 513, 538, 498, 488, 547, 530, 524, 554, 525, 525, 514, 506, 515, 515, 528, 512, 479, 531, 527, 492, 540, 521, 496, 498, 493, 485, 522, 508, 482, 483, 502, 532, 507, 543, 477, 519, 496, 473, 498, 508, 488, 498, 543, 513, 517, 536, 511, 545, 520, 486, 487, 501, 491, 508, 512] payload_rows_per_dst=[5644, 5636, 5608, 5540, 5432, 5570, 5502, 5504] dup_rows_per_dst=[2668, 2664, 2756, 2656, 2514, 2520, 2726, 2596] combine_rows_per_src=[5644, 5636, 5608, 5540, 5432, 5570, 5502, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279724032 dispatch_meta_bytes=229624 combine_payload_bytes=279724032 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279953656 ingress_bytes=389824512 link_bytes=389824512 token_bytes=559677688 weight_movement_bytes=110100480 total_nvlink_bytes=669778168 weight_to_token_ratio=0.196721 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.779689415e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44366 payload_rows_remote=38748 dup_rows=21170 dup_rows_remote=18474 dedup_ratio=0.323029 router_tokens_per_expert=[520, 462, 482, 502, 546, 505, 537, 531, 533, 489, 495, 490, 498, 515, 541, 534, 517, 511, 506, 510, 535, 522, 533, 523, 496, 550, 500, 481, 497, 499, 496, 462, 512, 506, 531, 546, 522, 520, 542, 503, 514, 487, 520, 471, 512, 507, 532, 479, 526, 517, 515, 466, 506, 539, 524, 532, 533, 483, 479, 538, 518, 518, 530, 522] payload_rows_per_dst=[5610, 5532, 5618, 5424, 5608, 5376, 5554, 5644] dup_rows_per_dst=[2560, 2658, 2696, 2538, 2756, 2668, 2696, 2598] combine_rows_per_src=[5610, 5532, 5618, 5424, 5608, 5376, 5554, 5644] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44572 payload_rows_remote=39082 dup_rows=20964 dup_rows_remote=18412 dedup_ratio=0.319885 router_tokens_per_expert=[537, 518, 539, 526, 499, 522, 499, 502, 492, 512, 493, 491, 524, 479, 529, 501, 521, 497, 525, 528, 531, 501, 494, 450, 498, 479, 483, 509, 485, 519, 505, 484, 529, 533, 556, 500, 533, 494, 585, 537, 499, 520, 515, 519, 504, 514, 524, 507, 511, 508, 531, 490, 521, 532, 528, 513, 520, 491, 524, 519, 518, 501, 528, 492] payload_rows_per_dst=[5576, 5490, 5488, 5478, 5768, 5592, 5600, 5580] dup_rows_per_dst=[2708, 2552, 2606, 2446, 2766, 2612, 2668, 2606] combine_rows_per_src=[5576, 5490, 5488, 5478, 5768, 5592, 5600, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44392 payload_rows_remote=38848 dup_rows=21144 dup_rows_remote=18640 dedup_ratio=0.322632 router_tokens_per_expert=[466, 517, 508, 479, 501, 497, 520, 510, 545, 509, 499, 503, 501, 517, 501, 515, 526, 502, 483, 486, 516, 497, 513, 515, 513, 528, 541, 512, 520, 512, 550, 501, 450, 535, 555, 503, 538, 514, 516, 528, 510, 509, 460, 561, 487, 465, 510, 518, 494, 536, 533, 496, 538, 512, 552, 496, 506, 528, 545, 501, 538, 516, 495, 520] payload_rows_per_dst=[5544, 5556, 5500, 5618, 5628, 5440, 5580, 5526] dup_rows_per_dst=[2504, 2704, 2576, 2668, 2650, 2600, 2734, 2708] combine_rows_per_src=[5544, 5556, 5500, 5618, 5628, 5440, 5580, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278792192 dispatch_meta_bytes=228552 combine_payload_bytes=278792192 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279020744 ingress_bytes=609093632 link_bytes=609093632 token_bytes=557812936 weight_movement_bytes=330301440 total_nvlink_bytes=888114376 weight_to_token_ratio=0.592137 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059314005e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278691840 dispatch_meta_bytes=229680 combine_payload_bytes=278691840 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=278921520 ingress_bytes=498892800 link_bytes=498892800 token_bytes=557613360 weight_movement_bytes=220200960 total_nvlink_bytes=777814320 weight_to_token_ratio=0.394899 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.676566330e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279078912 dispatch_meta_bytes=229232 combine_payload_bytes=279078912 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279308144 ingress_bytes=389179392 link_bytes=389179392 token_bytes=558387056 weight_movement_bytes=110100480 total_nvlink_bytes=668487536 weight_to_token_ratio=0.197176 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.768469717e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44448 payload_rows_remote=38926 dup_rows=21088 dup_rows_remote=18566 dedup_ratio=0.321777 router_tokens_per_expert=[497, 515, 548, 566, 486, 491, 502, 521, 478, 515, 491, 519, 522, 498, 524, 552, 514, 536, 506, 534, 525, 466, 528, 536, 537, 529, 504, 505, 522, 539, 503, 485, 506, 501, 511, 536, 492, 459, 526, 491, 534, 511, 485, 500, 532, 539, 514, 510, 464, 505, 525, 497, 511, 491, 532, 536, 516, 512, 500, 512, 505, 485, 515, 521] payload_rows_per_dst=[5614, 5528, 5588, 5580, 5522, 5592, 5486, 5538] dup_rows_per_dst=[2638, 2670, 2702, 2668, 2522, 2658, 2636, 2594] combine_rows_per_src=[5614, 5528, 5588, 5580, 5522, 5592, 5486, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=277745664 dispatch_meta_bytes=228888 combine_payload_bytes=277745664 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=277974552 ingress_bytes=608047104 link_bytes=608047104 token_bytes=555720216 weight_movement_bytes=330301440 total_nvlink_bytes=886021656 weight_to_token_ratio=0.594366 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057493920e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=280139776 dispatch_meta_bytes=229976 combine_payload_bytes=280139776 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280369752 ingress_bytes=390240256 link_bytes=390240256 token_bytes=560509528 weight_movement_bytes=110100480 total_nvlink_bytes=670610008 weight_to_token_ratio=0.196429 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.786919887e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278462464 dispatch_meta_bytes=229952 combine_payload_bytes=278462464 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278692416 ingress_bytes=498663424 link_bytes=498663424 token_bytes=557154880 weight_movement_bytes=220200960 total_nvlink_bytes=777355840 weight_to_token_ratio=0.395224 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.672577104e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5569759480 prefetch_bytes=880803840 reduce_bytes=880803840 weight_movement_bytes=1761607680 egress_bytes=2786023672 ingress_bytes=4545343488 link_bytes=4545343488 total_nvlink_bytes=7331367160 weight_to_token_ratio=0.316281 prefetch_share_of_ingress=0.193782 dedup_ratio=0.321741 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=8 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042647 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.905099906e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44500 payload_rows_remote=38880 dup_rows=21036 dup_rows_remote=18388 dedup_ratio=0.320984 router_tokens_per_expert=[486, 468, 533, 514, 522, 523, 490, 504, 495, 506, 544, 505, 512, 490, 489, 492, 549, 533, 476, 494, 522, 514, 511, 489, 538, 538, 529, 499, 485, 472, 520, 521, 494, 530, 537, 488, 500, 507, 494, 525, 512, 578, 507, 473, 531, 530, 496, 507, 512, 533, 533, 539, 487, 498, 493, 524, 481, 530, 529, 525, 533, 519, 541, 519] payload_rows_per_dst=[5526, 5474, 5552, 5542, 5504, 5620, 5580, 5702] dup_rows_per_dst=[2554, 2592, 2624, 2662, 2646, 2648, 2658, 2652] combine_rows_per_src=[5526, 5474, 5552, 5542, 5504, 5620, 5580, 5702] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5587915144 prefetch_bytes=1101004800 reduce_bytes=550502400 weight_movement_bytes=1651507200 egress_bytes=2795104648 ingress_bytes=4444317696 link_bytes=4444317696 total_nvlink_bytes=7239422344 weight_to_token_ratio=0.295550 prefetch_share_of_ingress=0.247733 dedup_ratio=0.320499 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=5 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.729399438e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5590381664 prefetch_bytes=990904320 reduce_bytes=990904320 weight_movement_bytes=1981808640 egress_bytes=2796338272 ingress_bytes=4775852032 link_bytes=4775852032 total_nvlink_bytes=7572190304 weight_to_token_ratio=0.354503 prefetch_share_of_ingress=0.207482 dedup_ratio=0.320425 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=9 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.305992176e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279021568 dispatch_meta_bytes=229968 combine_payload_bytes=279021568 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279251536 ingress_bytes=499222528 link_bytes=499222528 token_bytes=558273104 weight_movement_bytes=220200960 total_nvlink_bytes=778474064 weight_to_token_ratio=0.394432 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.682300842e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44588 payload_rows_remote=38912 dup_rows=20948 dup_rows_remote=18288 dedup_ratio=0.319641 router_tokens_per_expert=[485, 523, 456, 565, 531, 522, 532, 511, 488, 536, 516, 504, 505, 498, 489, 499, 517, 529, 510, 543, 543, 517, 520, 494, 523, 539, 511, 527, 507, 520, 534, 507, 499, 504, 511, 517, 503, 517, 487, 536, 501, 488, 510, 503, 522, 498, 504, 522, 522, 474, 495, 508, 498, 536, 504, 498, 504, 493, 519, 518, 488, 517, 549, 522] payload_rows_per_dst=[5554, 5532, 5624, 5676, 5546, 5514, 5530, 5612] dup_rows_per_dst=[2696, 2538, 2722, 2660, 2602, 2582, 2540, 2608] combine_rows_per_src=[5554, 5532, 5624, 5676, 5546, 5514, 5530, 5612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278691840 dispatch_meta_bytes=229072 combine_payload_bytes=278691840 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278920912 ingress_bytes=388792320 link_bytes=388792320 token_bytes=557612752 weight_movement_bytes=110100480 total_nvlink_bytes=667713232 weight_to_token_ratio=0.197450 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.761737898e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44484 payload_rows_remote=38890 dup_rows=21052 dup_rows_remote=18562 dedup_ratio=0.321228 router_tokens_per_expert=[521, 543, 489, 544, 479, 493, 514, 501, 509, 525, 523, 521, 507, 529, 547, 553, 487, 504, 521, 497, 525, 526, 472, 499, 497, 517, 546, 502, 480, 481, 529, 514, 516, 505, 526, 531, 487, 554, 538, 467, 491, 505, 509, 470, 486, 521, 549, 531, 510, 503, 534, 518, 504, 511, 497, 465, 557, 483, 506, 487, 553, 508, 549, 502] payload_rows_per_dst=[5560, 5626, 5484, 5532, 5560, 5550, 5594, 5578] dup_rows_per_dst=[2608, 2802, 2578, 2600, 2688, 2574, 2490, 2712] combine_rows_per_src=[5560, 5626, 5484, 5532, 5560, 5550, 5594, 5578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5577965424 prefetch_bytes=990904320 reduce_bytes=770703360 weight_movement_bytes=1761607680 egress_bytes=2790129520 ingress_bytes=4549443584 link_bytes=4549443584 total_nvlink_bytes=7339573104 weight_to_token_ratio=0.315815 prefetch_share_of_ingress=0.217808 dedup_ratio=0.321594 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=7 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.912230647e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44366 payload_rows_remote=38864 dup_rows=21170 dup_rows_remote=18562 dedup_ratio=0.323029 router_tokens_per_expert=[527, 500, 507, 512, 507, 512, 503, 472, 507, 489, 494, 540, 518, 518, 572, 525, 514, 533, 500, 514, 471, 531, 507, 523, 521, 535, 510, 510, 526, 512, 541, 511, 505, 485, 491, 530, 505, 461, 488, 548, 499, 505, 518, 526, 495, 513, 500, 514, 517, 551, 533, 525, 510, 493, 520, 488, 510, 515, 546, 495, 492, 507, 518, 503] payload_rows_per_dst=[5502, 5686, 5480, 5676, 5330, 5660, 5624, 5408] dup_rows_per_dst=[2608, 2730, 2624, 2948, 2404, 2648, 2620, 2588] combine_rows_per_src=[5502, 5686, 5480, 5676, 5330, 5660, 5624, 5408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278921216 dispatch_meta_bytes=228800 combine_payload_bytes=278921216 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279150016 ingress_bytes=389021696 link_bytes=389021696 token_bytes=558071232 weight_movement_bytes=110100480 total_nvlink_bytes=668171712 weight_to_token_ratio=0.197288 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.765727124e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5589206392 prefetch_bytes=990904320 reduce_bytes=330301440 weight_movement_bytes=1321205760 egress_bytes=2795750776 ingress_bytes=4114661376 link_bytes=4114661376 total_nvlink_bytes=6910412152 weight_to_token_ratio=0.236385 prefetch_share_of_ingress=0.240823 dedup_ratio=0.320657 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=3 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042647 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.156072878e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278763520 dispatch_meta_bytes=229808 combine_payload_bytes=278763520 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278993328 ingress_bytes=498964480 link_bytes=498964480 token_bytes=557756848 weight_movement_bytes=220200960 total_nvlink_bytes=777957808 weight_to_token_ratio=0.394797 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.677812963e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278577152 dispatch_meta_bytes=229704 combine_payload_bytes=278577152 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278806856 ingress_bytes=498778112 link_bytes=498778112 token_bytes=557384008 weight_movement_bytes=220200960 total_nvlink_bytes=777584968 weight_to_token_ratio=0.395061 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.674571717e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5580771440 prefetch_bytes=770703360 reduce_bytes=2202009600 weight_movement_bytes=2972712960 egress_bytes=2791530608 ingress_bytes=5761953792 link_bytes=5761953792 total_nvlink_bytes=8553484400 weight_to_token_ratio=0.532671 prefetch_share_of_ingress=0.133757 dedup_ratio=0.320358 prefetch_slots=14 unused_prefetch_slots=146 slot_utilization=0.087500 reduce_slots=20 remote_demand=7 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=12319 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.002098532e-04 prefetch_bytes_per_gemm_flop=1.340379902e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5588116432 prefetch_bytes=990904320 reduce_bytes=880803840 weight_movement_bytes=1871708160 egress_bytes=2795205584 ingress_bytes=4664619008 link_bytes=4664619008 total_nvlink_bytes=7459824592 weight_to_token_ratio=0.334944 prefetch_share_of_ingress=0.212430 dedup_ratio=0.320560 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=8 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042647 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.112539653e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5580745056 prefetch_bytes=990904320 reduce_bytes=1101004800 weight_movement_bytes=2091909120 egress_bytes=2791518560 ingress_bytes=4881135616 link_bytes=4881135616 total_nvlink_bytes=7672654176 weight_to_token_ratio=0.374844 prefetch_share_of_ingress=0.203007 dedup_ratio=0.320990 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=10 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.489097645e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none + [2026-08-26 16:00:27] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 366.6 | throughput per GPU (TFLOP/s/GPU): 336.8 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.033299E+01 | loss scale: 1.0 | grad norm: 8.129 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44498 payload_rows_remote=38918 dup_rows=21038 dup_rows_remote=18432 dedup_ratio=0.321014 router_tokens_per_expert=[499, 497, 505, 502, 501, 529, 499, 528, 504, 484, 498, 529, 491, 501, 495, 541, 483, 511, 561, 529, 491, 478, 488, 552, 531, 507, 509, 522, 510, 563, 508, 480, 519, 508, 480, 547, 484, 472, 464, 550, 533, 489, 520, 547, 479, 503, 521, 529, 566, 527, 478, 490, 547, 529, 525, 479, 516, 554, 522, 486, 517, 508, 530, 523] payload_rows_per_dst=[5472, 5574, 5580, 5570, 5500, 5622, 5612, 5568] dup_rows_per_dst=[2648, 2512, 2606, 2690, 2548, 2620, 2670, 2744] combine_rows_per_src=[5472, 5574, 5580, 5570, 5500, 5622, 5612, 5568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44478 payload_rows_remote=38856 dup_rows=21058 dup_rows_remote=18294 dedup_ratio=0.321320 router_tokens_per_expert=[511, 524, 468, 512, 488, 495, 514, 447, 522, 504, 508, 507, 542, 530, 549, 531, 531, 519, 524, 540, 520, 535, 483, 486, 506, 532, 514, 510, 495, 540, 522, 537, 527, 487, 526, 496, 515, 483, 487, 521, 475, 508, 488, 491, 520, 521, 536, 507, 527, 509, 531, 506, 495, 496, 517, 486, 535, 511, 507, 565, 508, 498, 509, 534] payload_rows_per_dst=[5486, 5622, 5594, 5606, 5520, 5526, 5516, 5608] dup_rows_per_dst=[2432, 2764, 2682, 2706, 2564, 2566, 2618, 2726] combine_rows_per_src=[5486, 5622, 5594, 5606, 5520, 5526, 5516, 5608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44778 payload_rows_remote=39230 dup_rows=20758 dup_rows_remote=18224 dedup_ratio=0.316742 router_tokens_per_expert=[539, 539, 469, 500, 506, 499, 527, 497, 477, 509, 491, 492, 525, 559, 521, 503, 507, 486, 546, 516, 555, 476, 462, 509, 525, 521, 521, 516, 509, 474, 491, 484, 466, 513, 511, 486, 517, 510, 507, 543, 497, 525, 479, 533, 533, 547, 524, 526, 493, 483, 531, 534, 541, 489, 559, 522, 519, 531, 493, 498, 538, 523, 526, 520] payload_rows_per_dst=[5558, 5618, 5582, 5548, 5586, 5632, 5606, 5648] dup_rows_per_dst=[2594, 2536, 2532, 2534, 2520, 2696, 2698, 2648] combine_rows_per_src=[5558, 5618, 5582, 5548, 5586, 5632, 5606, 5648] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44690 payload_rows_remote=39102 dup_rows=20846 dup_rows_remote=18198 dedup_ratio=0.318085 router_tokens_per_expert=[505, 491, 548, 510, 489, 518, 519, 505, 515, 504, 499, 528, 505, 506, 506, 496, 492, 536, 506, 541, 493, 502, 499, 513, 544, 480, 506, 527, 483, 530, 483, 512, 500, 528, 527, 512, 524, 497, 510, 516, 510, 481, 547, 497, 514, 517, 504, 523, 529, 517, 497, 486, 576, 508, 526, 513, 480, 531, 563, 480, 538, 484, 531, 511] payload_rows_per_dst=[5552, 5534, 5406, 5518, 5660, 5784, 5648, 5588] dup_rows_per_dst=[2618, 2472, 2420, 2612, 2680, 2720, 2676, 2648] combine_rows_per_src=[5552, 5534, 5406, 5518, 5660, 5784, 5648, 5588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44598 payload_rows_remote=39094 dup_rows=20938 dup_rows_remote=18430 dedup_ratio=0.319489 router_tokens_per_expert=[492, 523, 515, 518, 541, 495, 533, 532, 502, 545, 539, 513, 515, 514, 507, 493, 513, 535, 520, 484, 501, 492, 525, 530, 485, 509, 536, 520, 477, 564, 516, 492, 468, 542, 513, 495, 529, 504, 524, 504, 527, 507, 492, 519, 494, 485, 496, 486, 542, 518, 528, 497, 483, 553, 530, 526, 518, 495, 497, 503, 519, 493, 482, 523] payload_rows_per_dst=[5606, 5580, 5556, 5638, 5588, 5504, 5670, 5456] dup_rows_per_dst=[2692, 2676, 2644, 2560, 2570, 2508, 2684, 2604] combine_rows_per_src=[5606, 5580, 5556, 5638, 5588, 5504, 5670, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44590 payload_rows_remote=38940 dup_rows=20946 dup_rows_remote=18350 dedup_ratio=0.319611 router_tokens_per_expert=[514, 518, 508, 538, 519, 505, 505, 492, 518, 538, 515, 505, 506, 484, 518, 526, 515, 540, 489, 504, 497, 549, 506, 538, 490, 513, 507, 498, 488, 489, 542, 480, 543, 561, 528, 507, 511, 510, 515, 514, 503, 513, 518, 502, 512, 503, 488, 520, 496, 521, 502, 549, 497, 532, 512, 514, 532, 470, 509, 492, 501, 531, 487, 521] payload_rows_per_dst=[5574, 5512, 5608, 5508, 5630, 5574, 5650, 5534] dup_rows_per_dst=[2624, 2708, 2668, 2506, 2748, 2544, 2596, 2552] combine_rows_per_src=[5574, 5512, 5608, 5508, 5630, 5574, 5650, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44400 payload_rows_remote=38780 dup_rows=21136 dup_rows_remote=18334 dedup_ratio=0.322510 router_tokens_per_expert=[519, 527, 488, 532, 500, 490, 512, 507, 487, 511, 526, 491, 533, 522, 530, 504, 556, 545, 501, 547, 549, 497, 510, 524, 505, 487, 491, 480, 511, 522, 510, 512, 526, 541, 526, 491, 520, 528, 541, 538, 509, 493, 509, 496, 528, 530, 512, 520, 511, 484, 490, 497, 522, 539, 489, 500, 489, 486, 520, 482, 507, 498, 509, 511] payload_rows_per_dst=[5554, 5492, 5698, 5486, 5620, 5594, 5434, 5522] dup_rows_per_dst=[2596, 2716, 2760, 2550, 2802, 2600, 2630, 2482] combine_rows_per_src=[5554, 5492, 5698, 5486, 5620, 5594, 5434, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278118400 dispatch_meta_bytes=228520 combine_payload_bytes=278118400 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278346920 ingress_bytes=388218880 link_bytes=388218880 token_bytes=556465320 weight_movement_bytes=110100480 total_nvlink_bytes=666565800 weight_to_token_ratio=0.197857 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.751764834e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44502 payload_rows_remote=38848 dup_rows=21034 dup_rows_remote=18186 dedup_ratio=0.320953 router_tokens_per_expert=[573, 545, 499, 502, 551, 494, 534, 530, 555, 491, 510, 512, 510, 489, 507, 531, 544, 476, 517, 550, 521, 574, 501, 499, 545, 493, 495, 476, 509, 504, 539, 547, 479, 532, 508, 499, 498, 487, 482, 517, 502, 513, 506, 498, 476, 539, 490, 491, 541, 520, 483, 525, 491, 516, 478, 493, 524, 542, 521, 559, 501, 456, 497, 481] payload_rows_per_dst=[5654, 5576, 5684, 5570, 5486, 5520, 5474, 5538] dup_rows_per_dst=[2848, 2666, 2648, 2646, 2518, 2510, 2574, 2624] combine_rows_per_src=[5654, 5576, 5684, 5570, 5486, 5520, 5474, 5538] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278863872 dispatch_meta_bytes=229312 combine_payload_bytes=278863872 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=279093184 ingress_bytes=719265792 link_bytes=719265792 token_bytes=557957056 weight_movement_bytes=440401920 total_nvlink_bytes=998358976 weight_to_token_ratio=0.789311 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.250921511e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=281114624 dispatch_meta_bytes=229752 combine_payload_bytes=281114624 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281344376 ingress_bytes=391215104 link_bytes=391215104 token_bytes=562459000 weight_movement_bytes=110100480 total_nvlink_bytes=672559480 weight_to_token_ratio=0.195748 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.803874097e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280340480 dispatch_meta_bytes=230144 combine_payload_bytes=280340480 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280570624 ingress_bytes=390440960 link_bytes=390440960 token_bytes=560911104 weight_movement_bytes=110100480 total_nvlink_bytes=671011584 weight_to_token_ratio=0.196289 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.790410459e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279508992 dispatch_meta_bytes=228016 combine_payload_bytes=279508992 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279737008 ingress_bytes=389609472 link_bytes=389609472 token_bytes=559246000 weight_movement_bytes=110100480 total_nvlink_bytes=669346480 weight_to_token_ratio=0.196873 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.775949516e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279064576 dispatch_meta_bytes=229240 combine_payload_bytes=279064576 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279293816 ingress_bytes=499265536 link_bytes=499265536 token_bytes=558358392 weight_movement_bytes=220200960 total_nvlink_bytes=778559352 weight_to_token_ratio=0.394372 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.683048822e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278591488 dispatch_meta_bytes=228528 combine_payload_bytes=278591488 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278820016 ingress_bytes=498792448 link_bytes=498792448 token_bytes=557411504 weight_movement_bytes=220200960 total_nvlink_bytes=777612464 weight_to_token_ratio=0.395042 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.674821043e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278462464 dispatch_meta_bytes=228136 combine_payload_bytes=278462464 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278690600 ingress_bytes=498663424 link_bytes=498663424 token_bytes=557153064 weight_movement_bytes=220200960 total_nvlink_bytes=777354024 weight_to_token_ratio=0.395225 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.672577104e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44392 payload_rows_remote=38826 dup_rows=21144 dup_rows_remote=18520 dedup_ratio=0.322632 router_tokens_per_expert=[495, 531, 511, 513, 524, 501, 526, 497, 516, 472, 546, 542, 508, 499, 525, 522, 525, 496, 514, 512, 504, 529, 486, 547, 541, 494, 489, 552, 506, 496, 519, 534, 551, 517, 519, 482, 520, 518, 518, 470, 495, 494, 482, 490, 526, 513, 516, 509, 511, 465, 545, 499, 491, 572, 523, 502, 477, 499, 541, 523, 526, 455, 549, 498] payload_rows_per_dst=[5594, 5628, 5544, 5568, 5566, 5510, 5496, 5486] dup_rows_per_dst=[2602, 2632, 2682, 2694, 2624, 2540, 2720, 2650] combine_rows_per_src=[5594, 5628, 5544, 5568, 5566, 5510, 5496, 5486] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44280 payload_rows_remote=38594 dup_rows=21256 dup_rows_remote=18492 dedup_ratio=0.324341 router_tokens_per_expert=[493, 522, 537, 503, 518, 522, 480, 523, 556, 514, 503, 512, 505, 522, 538, 554, 521, 502, 555, 529, 504, 543, 505, 566, 509, 522, 537, 500, 543, 520, 495, 523, 463, 507, 511, 481, 500, 514, 481, 516, 509, 504, 495, 546, 501, 480, 495, 511, 506, 529, 484, 536, 499, 548, 485, 503, 535, 516, 475, 508, 488, 481, 497, 488] payload_rows_per_dst=[5478, 5654, 5686, 5536, 5462, 5440, 5588, 5436] dup_rows_per_dst=[2718, 2754, 2764, 2762, 2484, 2642, 2592, 2540] combine_rows_per_src=[5478, 5654, 5686, 5536, 5462, 5440, 5588, 5436] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44388 payload_rows_remote=38872 dup_rows=21148 dup_rows_remote=18488 dedup_ratio=0.322693 router_tokens_per_expert=[524, 526, 549, 491, 481, 513, 524, 480, 545, 494, 541, 525, 496, 527, 535, 541, 506, 470, 484, 529, 585, 510, 476, 520, 510, 478, 493, 503, 502, 543, 540, 519, 523, 522, 533, 462, 516, 492, 504, 528, 516, 497, 538, 530, 528, 523, 540, 519, 473, 500, 507, 482, 518, 513, 517, 503, 521, 488, 477, 501, 502, 521, 507, 507] payload_rows_per_dst=[5542, 5632, 5558, 5516, 5496, 5632, 5480, 5532] dup_rows_per_dst=[2634, 2776, 2602, 2660, 2664, 2750, 2546, 2516] combine_rows_per_src=[5542, 5632, 5558, 5516, 5496, 5632, 5480, 5532] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278304768 dispatch_meta_bytes=229384 combine_payload_bytes=278304768 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278534152 ingress_bytes=388405248 link_bytes=388405248 token_bytes=556838920 weight_movement_bytes=110100480 total_nvlink_bytes=666939400 weight_to_token_ratio=0.197724 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.755006080e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44502 payload_rows_remote=38940 dup_rows=21034 dup_rows_remote=18472 dedup_ratio=0.320953 router_tokens_per_expert=[480, 488, 489, 536, 503, 539, 545, 504, 535, 558, 517, 518, 545, 518, 526, 495, 456, 491, 487, 556, 487, 491, 522, 514, 509, 572, 545, 504, 501, 507, 493, 494, 538, 480, 483, 509, 501, 522, 510, 516, 506, 479, 537, 520, 492, 497, 513, 518, 501, 521, 501, 531, 546, 531, 529, 546, 502, 479, 505, 522, 494, 508, 503, 503] payload_rows_per_dst=[5538, 5698, 5492, 5588, 5544, 5562, 5614, 5466] dup_rows_per_dst=[2630, 2726, 2516, 2662, 2574, 2562, 2798, 2566] combine_rows_per_src=[5538, 5698, 5492, 5588, 5544, 5562, 5614, 5466] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44664 payload_rows_remote=38786 dup_rows=20872 dup_rows_remote=17698 dedup_ratio=0.318481 router_tokens_per_expert=[488, 551, 493, 529, 481, 509, 502, 531, 494, 510, 494, 507, 526, 505, 506, 525, 498, 536, 518, 495, 524, 513, 512, 553, 495, 538, 508, 495, 548, 547, 508, 489, 518, 487, 514, 480, 491, 492, 508, 497, 471, 519, 531, 469, 499, 487, 512, 501, 526, 527, 510, 561, 532, 548, 493, 520, 534, 527, 548, 477, 487, 516, 536, 522] payload_rows_per_dst=[5528, 5370, 5342, 5562, 5782, 5514, 5688, 5878] dup_rows_per_dst=[2440, 2206, 2230, 2670, 2918, 2464, 2770, 3174] combine_rows_per_src=[5528, 5370, 5342, 5562, 5782, 5514, 5688, 5878] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44530 payload_rows_remote=38954 dup_rows=21006 dup_rows_remote=18322 dedup_ratio=0.320526 router_tokens_per_expert=[499, 536, 512, 485, 518, 482, 518, 529, 511, 466, 544, 535, 542, 511, 531, 490, 482, 507, 538, 501, 508, 517, 500, 511, 508, 540, 527, 512, 494, 495, 540, 519, 531, 472, 469, 493, 526, 540, 525, 526, 512, 541, 479, 503, 538, 534, 505, 494, 519, 504, 514, 496, 543, 521, 551, 507, 480, 464, 485, 518, 537, 511, 506, 516] payload_rows_per_dst=[5620, 5576, 5528, 5604, 5576, 5554, 5630, 5442] dup_rows_per_dst=[2538, 2684, 2600, 2666, 2588, 2658, 2680, 2592] combine_rows_per_src=[5620, 5576, 5528, 5604, 5576, 5554, 5630, 5442] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44588 payload_rows_remote=38990 dup_rows=20948 dup_rows_remote=18492 dedup_ratio=0.319641 router_tokens_per_expert=[527, 496, 535, 536, 520, 496, 507, 520, 485, 550, 511, 505, 552, 485, 467, 492, 540, 534, 496, 490, 554, 525, 509, 536, 495, 516, 489, 543, 526, 549, 478, 480, 481, 531, 504, 533, 493, 540, 518, 511, 522, 517, 515, 502, 541, 524, 507, 513, 500, 499, 494, 486, 504, 482, 525, 537, 522, 523, 463, 516, 521, 492, 518, 490] payload_rows_per_dst=[5586, 5528, 5564, 5580, 5640, 5610, 5598, 5482] dup_rows_per_dst=[2688, 2566, 2804, 2572, 2582, 2672, 2456, 2608] combine_rows_per_src=[5586, 5528, 5564, 5580, 5640, 5610, 5598, 5482] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44356 payload_rows_remote=38712 dup_rows=21180 dup_rows_remote=18424 dedup_ratio=0.323181 router_tokens_per_expert=[517, 527, 530, 532, 516, 508, 516, 550, 489, 485, 515, 490, 562, 499, 532, 481, 507, 499, 493, 536, 526, 529, 524, 537, 525, 509, 531, 531, 524, 513, 483, 497, 532, 513, 519, 484, 510, 523, 479, 458, 495, 512, 533, 478, 526, 520, 517, 504, 497, 516, 500, 515, 555, 479, 490, 516, 513, 517, 528, 505, 490, 502, 517, 512] payload_rows_per_dst=[5644, 5542, 5748, 5390, 5448, 5650, 5396, 5538] dup_rows_per_dst=[2756, 2564, 2876, 2506, 2588, 2776, 2484, 2630] combine_rows_per_src=[5644, 5542, 5748, 5390, 5448, 5650, 5396, 5538] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278964224 dispatch_meta_bytes=229400 combine_payload_bytes=278964224 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=279193624 ingress_bytes=829466624 link_bytes=829466624 token_bytes=558157848 weight_movement_bytes=550502400 total_nvlink_bytes=1108660248 weight_to_token_ratio=0.986284 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.442578883e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278519808 dispatch_meta_bytes=228600 combine_payload_bytes=278519808 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278748408 ingress_bytes=498720768 link_bytes=498720768 token_bytes=557268216 weight_movement_bytes=220200960 total_nvlink_bytes=777469176 weight_to_token_ratio=0.395144 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.673574410e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44516 payload_rows_remote=38844 dup_rows=21020 dup_rows_remote=18162 dedup_ratio=0.320740 router_tokens_per_expert=[570, 542, 497, 498, 547, 493, 532, 530, 552, 488, 507, 511, 508, 486, 506, 532, 541, 477, 514, 550, 520, 572, 500, 499, 543, 492, 496, 477, 509, 503, 537, 549, 480, 532, 505, 498, 499, 487, 481, 518, 506, 513, 505, 498, 477, 540, 491, 493, 543, 521, 482, 528, 492, 520, 480, 494, 530, 543, 523, 561, 508, 457, 499, 486] payload_rows_per_dst=[5672, 5566, 5632, 5678, 5482, 5530, 5394, 5562] dup_rows_per_dst=[2858, 2654, 2568, 2826, 2518, 2516, 2434, 2646] combine_rows_per_src=[5672, 5566, 5632, 5678, 5482, 5530, 5394, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=281200640 dispatch_meta_bytes=229816 combine_payload_bytes=281200640 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281430456 ingress_bytes=391301120 link_bytes=391301120 token_bytes=562631096 weight_movement_bytes=110100480 total_nvlink_bytes=672731576 weight_to_token_ratio=0.195689 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.805370057e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280283136 dispatch_meta_bytes=229200 combine_payload_bytes=280283136 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=280512336 ingress_bytes=390383616 link_bytes=390383616 token_bytes=560795472 weight_movement_bytes=110100480 total_nvlink_bytes=670895952 weight_to_token_ratio=0.196329 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.789413153e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280225792 dispatch_meta_bytes=230096 combine_payload_bytes=280225792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280455888 ingress_bytes=390326272 link_bytes=390326272 token_bytes=560681680 weight_movement_bytes=110100480 total_nvlink_bytes=670782160 weight_to_token_ratio=0.196369 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.788415846e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279121920 dispatch_meta_bytes=229160 combine_payload_bytes=279121920 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279351080 ingress_bytes=499322880 link_bytes=499322880 token_bytes=558473000 weight_movement_bytes=220200960 total_nvlink_bytes=778673960 weight_to_token_ratio=0.394291 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.684046128e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=277975040 dispatch_meta_bytes=228456 combine_payload_bytes=277975040 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278203496 ingress_bytes=388075520 link_bytes=388075520 token_bytes=556178536 weight_movement_bytes=110100480 total_nvlink_bytes=666279016 weight_to_token_ratio=0.197959 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.749271567e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278433792 dispatch_meta_bytes=228024 combine_payload_bytes=278433792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278661816 ingress_bytes=388534272 link_bytes=388534272 token_bytes=557095608 weight_movement_bytes=110100480 total_nvlink_bytes=667196088 weight_to_token_ratio=0.197633 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.757250019e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44282 payload_rows_remote=38604 dup_rows=21254 dup_rows_remote=18494 dedup_ratio=0.324310 router_tokens_per_expert=[490, 521, 532, 499, 515, 521, 481, 522, 554, 513, 502, 511, 504, 522, 537, 552, 520, 499, 555, 528, 503, 540, 505, 569, 508, 520, 538, 501, 543, 521, 496, 523, 464, 505, 513, 478, 501, 515, 481, 514, 509, 508, 496, 547, 503, 481, 497, 512, 508, 527, 485, 539, 499, 549, 487, 502, 535, 518, 476, 510, 490, 482, 498, 494] payload_rows_per_dst=[5464, 5648, 5678, 5542, 5454, 5446, 5598, 5452] dup_rows_per_dst=[2698, 2742, 2760, 2758, 2488, 2660, 2594, 2554] combine_rows_per_src=[5464, 5648, 5678, 5542, 5454, 5446, 5598, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44548 payload_rows_remote=38984 dup_rows=20988 dup_rows_remote=18326 dedup_ratio=0.320251 router_tokens_per_expert=[494, 532, 507, 484, 514, 482, 514, 526, 509, 465, 543, 534, 538, 505, 529, 490, 479, 504, 534, 498, 506, 517, 502, 510, 508, 539, 526, 512, 494, 493, 540, 521, 535, 475, 473, 494, 527, 544, 528, 528, 512, 541, 482, 503, 539, 535, 507, 494, 520, 507, 517, 498, 544, 521, 553, 507, 483, 466, 487, 520, 539, 515, 508, 517] payload_rows_per_dst=[5596, 5564, 5518, 5608, 5596, 5566, 5642, 5458] dup_rows_per_dst=[2510, 2662, 2582, 2658, 2612, 2660, 2692, 2612] combine_rows_per_src=[5596, 5564, 5518, 5608, 5596, 5566, 5642, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44482 payload_rows_remote=38920 dup_rows=21054 dup_rows_remote=18484 dedup_ratio=0.321259 router_tokens_per_expert=[478, 488, 486, 533, 500, 533, 544, 500, 534, 558, 515, 518, 545, 514, 525, 494, 454, 491, 483, 556, 485, 492, 518, 511, 507, 571, 544, 507, 499, 507, 493, 494, 538, 479, 484, 510, 498, 522, 513, 516, 506, 476, 537, 523, 493, 498, 514, 519, 502, 522, 506, 533, 548, 536, 530, 547, 508, 481, 509, 525, 496, 510, 506, 506] payload_rows_per_dst=[5530, 5686, 5472, 5574, 5544, 5562, 5628, 5486] dup_rows_per_dst=[2594, 2720, 2508, 2670, 2576, 2570, 2820, 2596] combine_rows_per_src=[5530, 5686, 5472, 5574, 5544, 5562, 5628, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44730 payload_rows_remote=38962 dup_rows=20806 dup_rows_remote=17816 dedup_ratio=0.317474 router_tokens_per_expert=[485, 550, 491, 527, 479, 504, 500, 530, 494, 507, 490, 505, 524, 504, 503, 525, 493, 534, 519, 491, 525, 513, 511, 552, 496, 538, 510, 494, 543, 550, 510, 489, 518, 485, 515, 479, 493, 493, 508, 499, 473, 521, 531, 469, 499, 488, 514, 504, 528, 529, 510, 562, 535, 550, 493, 520, 536, 530, 551, 481, 490, 518, 537, 523] payload_rows_per_dst=[5598, 5434, 5636, 5442, 5522, 5556, 5774, 5768] dup_rows_per_dst=[2534, 2244, 2636, 2484, 2458, 2542, 2918, 2990] combine_rows_per_src=[5598, 5434, 5636, 5442, 5522, 5556, 5774, 5768] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44610 payload_rows_remote=38994 dup_rows=20926 dup_rows_remote=18454 dedup_ratio=0.319305 router_tokens_per_expert=[523, 493, 534, 532, 513, 495, 506, 520, 484, 545, 511, 502, 546, 483, 469, 491, 535, 533, 494, 490, 553, 525, 506, 535, 494, 516, 488, 544, 527, 547, 479, 479, 482, 530, 505, 534, 493, 541, 515, 511, 525, 518, 516, 503, 542, 527, 507, 514, 502, 503, 495, 486, 507, 484, 529, 538, 524, 527, 466, 518, 523, 496, 522, 493] payload_rows_per_dst=[5568, 5522, 5556, 5576, 5638, 5626, 5616, 5508] dup_rows_per_dst=[2664, 2540, 2786, 2572, 2584, 2678, 2472, 2630] combine_rows_per_src=[5568, 5522, 5556, 5576, 5638, 5626, 5616, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44392 payload_rows_remote=38826 dup_rows=21144 dup_rows_remote=18520 dedup_ratio=0.322632 router_tokens_per_expert=[493, 527, 510, 511, 523, 499, 523, 495, 517, 471, 545, 542, 507, 497, 522, 519, 523, 494, 512, 511, 502, 529, 486, 546, 541, 497, 488, 550, 509, 495, 518, 535, 551, 517, 519, 481, 518, 518, 520, 471, 495, 495, 481, 490, 527, 514, 517, 513, 512, 467, 545, 501, 492, 573, 523, 502, 481, 502, 544, 524, 528, 457, 552, 501] payload_rows_per_dst=[5578, 5620, 5538, 5564, 5566, 5514, 5502, 5510] dup_rows_per_dst=[2584, 2620, 2668, 2702, 2624, 2550, 2728, 2668] combine_rows_per_src=[5578, 5620, 5538, 5564, 5566, 5514, 5502, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44370 payload_rows_remote=38858 dup_rows=21166 dup_rows_remote=18512 dedup_ratio=0.322968 router_tokens_per_expert=[521, 523, 546, 488, 482, 506, 521, 478, 545, 488, 538, 522, 493, 530, 533, 541, 504, 470, 483, 526, 583, 509, 475, 519, 510, 478, 491, 503, 500, 546, 537, 518, 524, 525, 534, 462, 516, 493, 504, 528, 517, 497, 542, 529, 528, 526, 545, 523, 474, 503, 508, 480, 519, 512, 519, 507, 525, 488, 480, 504, 504, 524, 508, 513] payload_rows_per_dst=[5518, 5614, 5542, 5512, 5494, 5640, 5496, 5554] dup_rows_per_dst=[2612, 2766, 2596, 2654, 2678, 2774, 2548, 2538] combine_rows_per_src=[5518, 5614, 5542, 5512, 5494, 5640, 5496, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=276713472 dispatch_meta_bytes=228392 combine_payload_bytes=276713472 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=276941864 ingress_bytes=607014912 link_bytes=607014912 token_bytes=553655336 weight_movement_bytes=330301440 total_nvlink_bytes=883956776 weight_to_token_ratio=0.596583 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.055698769e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279437312 dispatch_meta_bytes=229240 combine_payload_bytes=279437312 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279666552 ingress_bytes=499638272 link_bytes=499638272 token_bytes=559103864 weight_movement_bytes=220200960 total_nvlink_bytes=779304824 weight_to_token_ratio=0.393846 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.689531314e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278634496 dispatch_meta_bytes=229440 combine_payload_bytes=278634496 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278863936 ingress_bytes=608935936 link_bytes=608935936 token_bytes=557498432 weight_movement_bytes=330301440 total_nvlink_bytes=887799872 weight_to_token_ratio=0.592471 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059039745e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=276641792 dispatch_meta_bytes=228344 combine_payload_bytes=276641792 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=276870136 ingress_bytes=496842752 link_bytes=496842752 token_bytes=553511928 weight_movement_bytes=220200960 total_nvlink_bytes=773712888 weight_to_token_ratio=0.397825 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.640912623e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279121920 dispatch_meta_bytes=229648 combine_payload_bytes=279121920 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279351568 ingress_bytes=389222400 link_bytes=389222400 token_bytes=558473488 weight_movement_bytes=110100480 total_nvlink_bytes=668573968 weight_to_token_ratio=0.197145 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.769217697e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278018048 dispatch_meta_bytes=225936 combine_payload_bytes=278018048 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278243984 ingress_bytes=388118528 link_bytes=388118528 token_bytes=556262032 weight_movement_bytes=110100480 total_nvlink_bytes=666362512 weight_to_token_ratio=0.197929 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.750019547e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277487616 dispatch_meta_bytes=228544 combine_payload_bytes=277487616 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277716160 ingress_bytes=497688576 link_bytes=497688576 token_bytes=555203776 weight_movement_bytes=220200960 total_nvlink_bytes=775404736 weight_to_token_ratio=0.396613 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.655622894e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279222272 dispatch_meta_bytes=229104 combine_payload_bytes=279222272 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279451376 ingress_bytes=499423232 link_bytes=499423232 token_bytes=558673648 weight_movement_bytes=220200960 total_nvlink_bytes=778874608 weight_to_token_ratio=0.394150 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.685791415e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279480320 dispatch_meta_bytes=229928 combine_payload_bytes=279480320 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279710248 ingress_bytes=499681280 link_bytes=499681280 token_bytes=559190568 weight_movement_bytes=220200960 total_nvlink_bytes=779391528 weight_to_token_ratio=0.393785 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.690279294e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44510 payload_rows_remote=39000 dup_rows=21026 dup_rows_remote=18304 dedup_ratio=0.320831 router_tokens_per_expert=[471, 528, 552, 519, 496, 515, 537, 533, 534, 511, 525, 523, 518, 542, 524, 534, 495, 518, 539, 498, 476, 496, 522, 473, 544, 499, 522, 495, 507, 478, 518, 544, 512, 492, 516, 506, 506, 572, 518, 494, 485, 516, 516, 518, 531, 500, 500, 554, 511, 503, 519, 490, 553, 476, 494, 521, 486, 519, 459, 527, 476, 501, 510, 501] payload_rows_per_dst=[5612, 5654, 5526, 5566, 5510, 5600, 5548, 5494] dup_rows_per_dst=[2690, 2768, 2508, 2648, 2722, 2640, 2586, 2464] combine_rows_per_src=[5612, 5654, 5526, 5566, 5510, 5600, 5548, 5494] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279552000 dispatch_meta_bytes=229216 combine_payload_bytes=279552000 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279781216 ingress_bytes=389652480 link_bytes=389652480 token_bytes=559333216 weight_movement_bytes=110100480 total_nvlink_bytes=669433696 weight_to_token_ratio=0.196842 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.776697495e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44378 payload_rows_remote=38616 dup_rows=21158 dup_rows_remote=18312 dedup_ratio=0.322845 router_tokens_per_expert=[536, 539, 489, 510, 539, 503, 508, 543, 489, 474, 511, 475, 512, 512, 550, 528, 495, 481, 492, 497, 495, 458, 562, 482, 557, 531, 585, 541, 524, 528, 518, 520, 495, 502, 521, 491, 546, 505, 478, 511, 506, 540, 521, 482, 539, 494, 497, 483, 547, 547, 492, 502, 520, 527, 494, 484, 520, 550, 503, 507, 516, 476, 480, 508] payload_rows_per_dst=[5666, 5522, 5364, 5762, 5568, 5482, 5536, 5478] dup_rows_per_dst=[2668, 2580, 2560, 2846, 2530, 2642, 2690, 2642] combine_rows_per_src=[5666, 5522, 5364, 5762, 5568, 5482, 5536, 5478] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44570 payload_rows_remote=38968 dup_rows=20966 dup_rows_remote=18432 dedup_ratio=0.319916 router_tokens_per_expert=[534, 511, 518, 492, 514, 531, 509, 537, 527, 532, 524, 509, 513, 515, 520, 493, 552, 499, 485, 492, 486, 525, 514, 541, 492, 504, 491, 502, 501, 505, 482, 517, 503, 538, 515, 524, 486, 491, 492, 513, 482, 514, 517, 506, 515, 515, 516, 503, 527, 526, 493, 523, 528, 484, 503, 526, 495, 518, 507, 516, 538, 509, 533, 545] payload_rows_per_dst=[5586, 5566, 5550, 5530, 5550, 5602, 5530, 5656] dup_rows_per_dst=[2706, 2700, 2638, 2458, 2574, 2534, 2690, 2666] combine_rows_per_src=[5586, 5566, 5550, 5530, 5550, 5602, 5530, 5656] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44422 payload_rows_remote=38820 dup_rows=21114 dup_rows_remote=18496 dedup_ratio=0.322174 router_tokens_per_expert=[525, 522, 509, 559, 506, 494, 510, 477, 504, 521, 471, 495, 488, 516, 479, 520, 511, 535, 508, 526, 508, 500, 510, 512, 477, 475, 529, 502, 503, 514, 513, 468, 494, 504, 498, 510, 523, 498, 512, 524, 522, 535, 510, 536, 517, 535, 498, 539, 522, 522, 500, 555, 528, 500, 474, 595, 521, 492, 515, 539, 510, 493, 522, 538] payload_rows_per_dst=[5572, 5474, 5602, 5474, 5490, 5612, 5614, 5584] dup_rows_per_dst=[2632, 2514, 2618, 2488, 2636, 2772, 2778, 2676] combine_rows_per_src=[5572, 5474, 5602, 5474, 5490, 5612, 5614, 5584] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44518 payload_rows_remote=38968 dup_rows=21018 dup_rows_remote=18490 dedup_ratio=0.320709 router_tokens_per_expert=[462, 527, 523, 488, 506, 485, 512, 525, 524, 479, 510, 503, 490, 501, 539, 492, 491, 515, 500, 490, 502, 510, 482, 550, 556, 512, 499, 509, 537, 532, 546, 514, 540, 536, 491, 530, 486, 524, 508, 483, 511, 536, 499, 518, 545, 499, 533, 494, 509, 457, 501, 509, 495, 513, 538, 526, 512, 534, 536, 541, 485, 541, 492, 535] payload_rows_per_dst=[5550, 5530, 5578, 5682, 5534, 5492, 5518, 5634] dup_rows_per_dst=[2528, 2546, 2750, 2706, 2662, 2530, 2578, 2718] combine_rows_per_src=[5550, 5530, 5578, 5682, 5534, 5492, 5518, 5634] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44628 payload_rows_remote=39034 dup_rows=20908 dup_rows_remote=18282 dedup_ratio=0.319031 router_tokens_per_expert=[529, 528, 501, 505, 532, 503, 527, 501, 520, 489, 550, 566, 566, 518, 481, 517, 509, 509, 520, 514, 515, 538, 517, 522, 504, 498, 485, 539, 469, 508, 495, 516, 541, 522, 484, 552, 489, 523, 479, 514, 498, 525, 529, 525, 538, 505, 510, 475, 476, 479, 549, 502, 506, 515, 507, 498, 507, 514, 505, 492, 507, 496, 507, 508] payload_rows_per_dst=[5470, 5644, 5562, 5482, 5746, 5638, 5492, 5594] dup_rows_per_dst=[2362, 2622, 2726, 2486, 2882, 2622, 2582, 2626] combine_rows_per_src=[5470, 5644, 5562, 5482, 5746, 5638, 5492, 5594] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44296 payload_rows_remote=38728 dup_rows=21240 dup_rows_remote=18630 dedup_ratio=0.324097 router_tokens_per_expert=[525, 533, 527, 521, 555, 532, 538, 479, 532, 502, 502, 520, 506, 512, 524, 491, 549, 536, 511, 510, 513, 474, 505, 527, 485, 532, 511, 520, 525, 508, 519, 516, 530, 512, 500, 521, 496, 498, 496, 527, 511, 496, 557, 501, 494, 535, 479, 512, 558, 496, 517, 512, 494, 476, 504, 524, 499, 474, 494, 465, 513, 510, 544, 483] payload_rows_per_dst=[5604, 5568, 5546, 5572, 5490, 5528, 5592, 5396] dup_rows_per_dst=[2816, 2610, 2704, 2660, 2670, 2642, 2570, 2568] combine_rows_per_src=[5604, 5568, 5546, 5572, 5490, 5528, 5592, 5396] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44388 payload_rows_remote=38826 dup_rows=21148 dup_rows_remote=18478 dedup_ratio=0.322693 router_tokens_per_expert=[521, 494, 514, 532, 495, 514, 460, 507, 516, 514, 499, 525, 539, 469, 543, 514, 509, 515, 503, 490, 515, 539, 557, 524, 543, 513, 542, 499, 457, 512, 505, 517, 489, 505, 461, 491, 493, 527, 509, 511, 521, 471, 521, 506, 516, 525, 512, 528, 487, 541, 505, 507, 508, 543, 551, 474, 502, 515, 521, 525, 494, 536, 504, 573] payload_rows_per_dst=[5492, 5550, 5558, 5538, 5438, 5584, 5562, 5666] dup_rows_per_dst=[2582, 2688, 2746, 2638, 2534, 2616, 2670, 2674] combine_rows_per_src=[5492, 5550, 5558, 5538, 5438, 5584, 5562, 5666] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278978560 dispatch_meta_bytes=229616 combine_payload_bytes=278978560 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279208176 ingress_bytes=389079040 link_bytes=389079040 token_bytes=558186736 weight_movement_bytes=110100480 total_nvlink_bytes=668287216 weight_to_token_ratio=0.197247 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.766724431e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279279616 dispatch_meta_bytes=227112 combine_payload_bytes=279279616 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279506728 ingress_bytes=389380096 link_bytes=389380096 token_bytes=558786344 weight_movement_bytes=110100480 total_nvlink_bytes=668886824 weight_to_token_ratio=0.197035 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.771960290e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279508992 dispatch_meta_bytes=229792 combine_payload_bytes=279508992 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279738784 ingress_bytes=499709952 link_bytes=499709952 token_bytes=559247776 weight_movement_bytes=220200960 total_nvlink_bytes=779448736 weight_to_token_ratio=0.393745 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.690777947e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278304768 dispatch_meta_bytes=229384 combine_payload_bytes=278304768 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278534152 ingress_bytes=388405248 link_bytes=388405248 token_bytes=556838920 weight_movement_bytes=110100480 total_nvlink_bytes=666939400 weight_to_token_ratio=0.197724 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.755006080e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278534144 dispatch_meta_bytes=229480 combine_payload_bytes=278534144 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278763624 ingress_bytes=608835584 link_bytes=608835584 token_bytes=557297768 weight_movement_bytes=330301440 total_nvlink_bytes=887599208 weight_to_token_ratio=0.592684 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058865217e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44356 payload_rows_remote=38676 dup_rows=21180 dup_rows_remote=18372 dedup_ratio=0.323181 router_tokens_per_expert=[514, 522, 531, 530, 514, 505, 512, 546, 487, 484, 513, 489, 560, 494, 529, 480, 505, 499, 493, 534, 523, 530, 524, 533, 523, 510, 530, 530, 522, 514, 483, 498, 535, 514, 517, 484, 512, 523, 480, 458, 495, 512, 536, 478, 530, 521, 518, 506, 498, 518, 499, 519, 555, 484, 493, 516, 514, 519, 529, 508, 496, 506, 519, 515] payload_rows_per_dst=[5680, 5562, 5516, 5532, 5696, 5562, 5250, 5558] dup_rows_per_dst=[2808, 2592, 2544, 2688, 3006, 2630, 2258, 2654] combine_rows_per_src=[5680, 5562, 5516, 5532, 5696, 5562, 5250, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277229568 dispatch_meta_bytes=228192 combine_payload_bytes=277229568 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277457760 ingress_bytes=387330048 link_bytes=387330048 token_bytes=554687328 weight_movement_bytes=110100480 total_nvlink_bytes=664787808 weight_to_token_ratio=0.198491 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.736306583e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44282 payload_rows_remote=38724 dup_rows=21254 dup_rows_remote=18664 dedup_ratio=0.324310 router_tokens_per_expert=[521, 532, 527, 518, 554, 529, 536, 477, 530, 501, 499, 515, 504, 512, 521, 492, 546, 537, 509, 508, 512, 474, 502, 528, 483, 530, 509, 520, 524, 509, 518, 513, 529, 512, 503, 521, 498, 498, 498, 529, 511, 496, 556, 501, 496, 536, 478, 513, 559, 499, 517, 512, 495, 479, 507, 527, 502, 475, 496, 469, 515, 514, 551, 486] payload_rows_per_dst=[5586, 5558, 5534, 5562, 5502, 5528, 5598, 5414] dup_rows_per_dst=[2802, 2590, 2698, 2650, 2674, 2646, 2592, 2602] combine_rows_per_src=[5586, 5558, 5534, 5562, 5502, 5528, 5598, 5414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44420 payload_rows_remote=38824 dup_rows=21116 dup_rows_remote=18516 dedup_ratio=0.322205 router_tokens_per_expert=[522, 521, 506, 555, 504, 493, 506, 476, 503, 518, 469, 493, 486, 516, 475, 520, 511, 530, 505, 523, 508, 502, 510, 509, 478, 477, 527, 501, 502, 513, 513, 467, 495, 503, 498, 509, 525, 499, 512, 529, 522, 536, 511, 537, 518, 535, 499, 540, 528, 526, 499, 559, 530, 501, 477, 595, 524, 491, 521, 541, 510, 495, 524, 540] payload_rows_per_dst=[5556, 5464, 5596, 5472, 5494, 5614, 5628, 5596] dup_rows_per_dst=[2610, 2496, 2600, 2484, 2646, 2782, 2802, 2696] combine_rows_per_src=[5556, 5464, 5596, 5472, 5494, 5614, 5628, 5596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=276799488 dispatch_meta_bytes=227712 combine_payload_bytes=276799488 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=277027200 ingress_bytes=607100928 link_bytes=607100928 token_bytes=553826688 weight_movement_bytes=330301440 total_nvlink_bytes=884128128 weight_to_token_ratio=0.596399 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.055848365e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279322624 dispatch_meta_bytes=229600 combine_payload_bytes=279322624 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279552224 ingress_bytes=499523584 link_bytes=499523584 token_bytes=558874848 weight_movement_bytes=220200960 total_nvlink_bytes=779075808 weight_to_token_ratio=0.394008 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.687536701e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278261760 dispatch_meta_bytes=229264 combine_payload_bytes=278261760 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278491024 ingress_bytes=388362240 link_bytes=388362240 token_bytes=556752784 weight_movement_bytes=110100480 total_nvlink_bytes=666853264 weight_to_token_ratio=0.197755 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.754258100e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279322624 dispatch_meta_bytes=229832 combine_payload_bytes=279322624 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279552456 ingress_bytes=499523584 link_bytes=499523584 token_bytes=558875080 weight_movement_bytes=220200960 total_nvlink_bytes=779076040 weight_to_token_ratio=0.394007 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.687536701e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279795712 dispatch_meta_bytes=229264 combine_payload_bytes=279795712 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280024976 ingress_bytes=389896192 link_bytes=389896192 token_bytes=559820688 weight_movement_bytes=110100480 total_nvlink_bytes=669921168 weight_to_token_ratio=0.196671 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.780936048e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44530 payload_rows_remote=38900 dup_rows=21006 dup_rows_remote=18240 dedup_ratio=0.320526 router_tokens_per_expert=[528, 533, 496, 531, 504, 497, 506, 483, 537, 504, 532, 504, 491, 523, 483, 497, 501, 505, 536, 543, 502, 473, 526, 498, 511, 546, 480, 521, 521, 517, 476, 480, 523, 532, 549, 509, 516, 524, 494, 551, 494, 542, 518, 514, 486, 513, 531, 504, 553, 503, 514, 484, 501, 485, 510, 510, 522, 490, 543, 492, 522, 508, 516, 530] payload_rows_per_dst=[5560, 5506, 5562, 5528, 5630, 5554, 5566, 5624] dup_rows_per_dst=[2596, 2636, 2606, 2576, 2766, 2650, 2554, 2622] combine_rows_per_src=[5560, 5506, 5562, 5528, 5630, 5554, 5566, 5624] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277602304 dispatch_meta_bytes=229432 combine_payload_bytes=277602304 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=277831736 ingress_bytes=387702784 link_bytes=387702784 token_bytes=555434040 weight_movement_bytes=110100480 total_nvlink_bytes=665534520 weight_to_token_ratio=0.198224 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.742789075e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278304768 dispatch_meta_bytes=229216 combine_payload_bytes=278304768 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278533984 ingress_bytes=388405248 link_bytes=388405248 token_bytes=556838752 weight_movement_bytes=110100480 total_nvlink_bytes=666939232 weight_to_token_ratio=0.197724 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.755006080e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278835200 dispatch_meta_bytes=228560 combine_payload_bytes=278835200 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279063760 ingress_bytes=499036160 link_bytes=499036160 token_bytes=557898960 weight_movement_bytes=220200960 total_nvlink_bytes=778099920 weight_to_token_ratio=0.394697 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.679059596e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44586 payload_rows_remote=38952 dup_rows=20950 dup_rows_remote=18242 dedup_ratio=0.319672 router_tokens_per_expert=[506, 557, 497, 554, 520, 528, 499, 529, 520, 466, 500, 505, 520, 512, 525, 502, 503, 527, 493, 533, 559, 519, 516, 464, 503, 540, 472, 495, 547, 541, 553, 520, 505, 492, 497, 559, 508, 506, 521, 528, 495, 500, 504, 529, 521, 491, 514, 524, 510, 513, 481, 499, 513, 515, 531, 519, 509, 488, 495, 464, 487, 491, 520, 514] payload_rows_per_dst=[5644, 5530, 5564, 5634, 5616, 5564, 5570, 5464] dup_rows_per_dst=[2736, 2570, 2664, 2708, 2616, 2592, 2592, 2472] combine_rows_per_src=[5644, 5530, 5564, 5634, 5616, 5564, 5570, 5464] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44404 payload_rows_remote=38786 dup_rows=21132 dup_rows_remote=18400 dedup_ratio=0.322449 router_tokens_per_expert=[557, 513, 469, 550, 494, 511, 556, 490, 534, 492, 512, 494, 506, 477, 538, 495, 500, 521, 503, 528, 543, 491, 489, 538, 480, 508, 512, 524, 517, 521, 535, 502, 528, 565, 481, 513, 503, 465, 507, 524, 543, 495, 547, 535, 489, 530, 515, 521, 516, 521, 500, 476, 523, 514, 515, 524, 534, 460, 529, 516, 497, 496, 487, 499] payload_rows_per_dst=[5578, 5524, 5540, 5590, 5546, 5618, 5552, 5456] dup_rows_per_dst=[2702, 2572, 2686, 2608, 2626, 2732, 2626, 2580] combine_rows_per_src=[5578, 5524, 5540, 5590, 5546, 5618, 5552, 5456] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44348 payload_rows_remote=38762 dup_rows=21188 dup_rows_remote=18348 dedup_ratio=0.323303 router_tokens_per_expert=[500, 531, 499, 523, 515, 532, 530, 545, 492, 515, 501, 481, 498, 505, 494, 484, 556, 505, 542, 490, 504, 474, 480, 563, 455, 544, 503, 497, 518, 523, 533, 514, 518, 516, 503, 527, 517, 488, 515, 493, 526, 483, 514, 490, 526, 528, 530, 526, 518, 515, 513, 511, 541, 517, 484, 490, 523, 504, 509, 558, 509, 504, 490, 536] payload_rows_per_dst=[5586, 5454, 5594, 5592, 5492, 5526, 5564, 5540] dup_rows_per_dst=[2840, 2486, 2710, 2654, 2672, 2486, 2614, 2726] combine_rows_per_src=[5586, 5454, 5594, 5592, 5492, 5526, 5564, 5540] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44498 payload_rows_remote=38912 dup_rows=21038 dup_rows_remote=18290 dedup_ratio=0.321014 router_tokens_per_expert=[531, 514, 564, 539, 490, 510, 510, 514, 507, 493, 502, 502, 508, 512, 492, 506, 506, 514, 539, 528, 499, 520, 525, 536, 503, 497, 505, 493, 476, 516, 496, 477, 520, 516, 505, 534, 522, 523, 517, 555, 536, 529, 481, 520, 521, 486, 488, 509, 521, 495, 539, 516, 538, 529, 488, 520, 508, 511, 537, 480, 520, 511, 477, 492] payload_rows_per_dst=[5640, 5526, 5586, 5422, 5656, 5530, 5656, 5482] dup_rows_per_dst=[2704, 2518, 2748, 2504, 2728, 2610, 2636, 2590] combine_rows_per_src=[5640, 5526, 5586, 5422, 5656, 5530, 5656, 5482] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44464 payload_rows_remote=38804 dup_rows=21072 dup_rows_remote=18374 dedup_ratio=0.321533 router_tokens_per_expert=[494, 494, 503, 513, 539, 492, 482, 480, 512, 515, 478, 526, 514, 496, 516, 478, 530, 492, 563, 500, 513, 498, 510, 520, 537, 531, 518, 499, 527, 534, 559, 543, 486, 509, 484, 513, 518, 535, 536, 525, 499, 539, 492, 513, 481, 543, 537, 547, 551, 505, 536, 478, 521, 484, 527, 493, 511, 524, 500, 505, 493, 510, 481, 486] payload_rows_per_dst=[5206, 5744, 5462, 5690, 5408, 5690, 5604, 5660] dup_rows_per_dst=[2180, 2934, 2452, 2806, 2542, 2612, 2848, 2698] combine_rows_per_src=[5206, 5744, 5462, 5690, 5408, 5690, 5604, 5660] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44470 payload_rows_remote=38876 dup_rows=21066 dup_rows_remote=18364 dedup_ratio=0.321442 router_tokens_per_expert=[527, 504, 533, 529, 534, 523, 507, 527, 533, 521, 521, 507, 533, 482, 546, 505, 503, 529, 525, 506, 485, 484, 512, 496, 494, 498, 502, 499, 502, 503, 526, 529, 505, 504, 543, 453, 544, 479, 505, 523, 527, 514, 530, 503, 510, 503, 474, 508, 524, 481, 470, 518, 505, 550, 533, 525, 525, 514, 528, 500, 498, 485, 521, 541] payload_rows_per_dst=[5692, 5594, 5498, 5496, 5560, 5496, 5490, 5644] dup_rows_per_dst=[2676, 2702, 2582, 2610, 2552, 2642, 2722, 2580] combine_rows_per_src=[5692, 5594, 5498, 5496, 5560, 5496, 5490, 5644] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44608 payload_rows_remote=39134 dup_rows=20928 dup_rows_remote=18460 dedup_ratio=0.319336 router_tokens_per_expert=[456, 515, 551, 536, 484, 529, 523, 504, 528, 530, 520, 507, 549, 515, 478, 493, 528, 546, 501, 511, 511, 523, 505, 516, 485, 512, 553, 471, 539, 518, 479, 502, 528, 503, 539, 497, 530, 489, 469, 508, 521, 511, 476, 529, 524, 535, 481, 540, 485, 492, 502, 483, 470, 532, 499, 508, 521, 493, 505, 540, 552, 535, 534, 519] payload_rows_per_dst=[5598, 5582, 5622, 5590, 5542, 5540, 5474, 5660] dup_rows_per_dst=[2598, 2658, 2660, 2528, 2584, 2694, 2468, 2738] combine_rows_per_src=[5598, 5582, 5622, 5590, 5542, 5540, 5474, 5660] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279207936 dispatch_meta_bytes=228776 combine_payload_bytes=279207936 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279436712 ingress_bytes=389308416 link_bytes=389308416 token_bytes=558644648 weight_movement_bytes=110100480 total_nvlink_bytes=668745128 weight_to_token_ratio=0.197085 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.770713657e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278018048 dispatch_meta_bytes=228744 combine_payload_bytes=278018048 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=278246792 ingress_bytes=718419968 link_bytes=718419968 token_bytes=556264840 weight_movement_bytes=440401920 total_nvlink_bytes=996666760 weight_to_token_ratio=0.791713 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.249450484e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44454 payload_rows_remote=39022 dup_rows=21082 dup_rows_remote=18432 dedup_ratio=0.321686 router_tokens_per_expert=[509, 504, 483, 542, 523, 505, 526, 523, 490, 536, 513, 486, 526, 541, 542, 481, 545, 523, 545, 508, 493, 528, 528, 545, 532, 526, 503, 487, 537, 500, 529, 478, 514, 513, 510, 506, 517, 495, 486, 500, 509, 530, 530, 489, 491, 540, 512, 473, 493, 510, 498, 538, 507, 535, 513, 527, 534, 475, 514, 486, 484, 489, 504, 509] payload_rows_per_dst=[5606, 5606, 5718, 5546, 5432, 5556, 5542, 5448] dup_rows_per_dst=[2624, 2624, 2712, 2638, 2650, 2592, 2700, 2542] combine_rows_per_src=[5606, 5606, 5718, 5546, 5432, 5556, 5542, 5448] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278921216 dispatch_meta_bytes=228808 combine_payload_bytes=278921216 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279150024 ingress_bytes=499122176 link_bytes=499122176 token_bytes=558071240 weight_movement_bytes=220200960 total_nvlink_bytes=778272200 weight_to_token_ratio=0.394575 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.680555556e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278147072 dispatch_meta_bytes=228712 combine_payload_bytes=278147072 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278375784 ingress_bytes=388247552 link_bytes=388247552 token_bytes=556522856 weight_movement_bytes=110100480 total_nvlink_bytes=666623336 weight_to_token_ratio=0.197836 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.752263487e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277846016 dispatch_meta_bytes=228440 combine_payload_bytes=277846016 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278074456 ingress_bytes=498046976 link_bytes=498046976 token_bytes=555920472 weight_movement_bytes=220200960 total_nvlink_bytes=776121432 weight_to_token_ratio=0.396102 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.661856059e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278663168 dispatch_meta_bytes=228960 combine_payload_bytes=278663168 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278892128 ingress_bytes=388763648 link_bytes=388763648 token_bytes=557555296 weight_movement_bytes=110100480 total_nvlink_bytes=667655776 weight_to_token_ratio=0.197470 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.761239245e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=280512512 dispatch_meta_bytes=230376 combine_payload_bytes=280512512 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280742888 ingress_bytes=390612992 link_bytes=390612992 token_bytes=561255400 weight_movement_bytes=110100480 total_nvlink_bytes=671355880 weight_to_token_ratio=0.196168 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.793402379e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279709696 dispatch_meta_bytes=229816 combine_payload_bytes=279709696 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=279939512 ingress_bytes=389810176 link_bytes=389810176 token_bytes=559649208 weight_movement_bytes=110100480 total_nvlink_bytes=669749688 weight_to_token_ratio=0.196731 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.779440088e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44462 payload_rows_remote=38898 dup_rows=21074 dup_rows_remote=18386 dedup_ratio=0.321564 router_tokens_per_expert=[527, 520, 505, 533, 530, 518, 483, 505, 470, 523, 490, 491, 516, 481, 563, 528, 545, 477, 502, 486, 512, 504, 493, 536, 491, 535, 547, 509, 518, 517, 525, 484, 494, 530, 498, 505, 521, 539, 474, 517, 507, 493, 513, 464, 545, 529, 518, 537, 487, 517, 516, 517, 510, 525, 509, 475, 487, 515, 534, 582, 497, 490, 537, 522] payload_rows_per_dst=[5620, 5500, 5508, 5564, 5606, 5532, 5532, 5600] dup_rows_per_dst=[2622, 2624, 2602, 2688, 2550, 2680, 2580, 2728] combine_rows_per_src=[5620, 5500, 5508, 5564, 5606, 5532, 5532, 5600] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44368 payload_rows_remote=38880 dup_rows=21168 dup_rows_remote=18582 dedup_ratio=0.322998 router_tokens_per_expert=[498, 530, 476, 532, 545, 495, 518, 537, 480, 517, 521, 560, 478, 523, 524, 515, 503, 586, 556, 539, 506, 498, 487, 510, 511, 500, 490, 506, 511, 487, 509, 523, 505, 508, 493, 521, 526, 546, 500, 535, 501, 525, 499, 501, 506, 484, 521, 500, 508, 490, 478, 476, 512, 522, 525, 543, 498, 515, 491, 514, 508, 525, 530, 491] payload_rows_per_dst=[5534, 5532, 5630, 5498, 5646, 5488, 5534, 5506] dup_rows_per_dst=[2728, 2704, 2740, 2576, 2622, 2586, 2574, 2638] combine_rows_per_src=[5534, 5532, 5630, 5498, 5646, 5488, 5534, 5506] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44382 payload_rows_remote=38690 dup_rows=21154 dup_rows_remote=18502 dedup_ratio=0.322784 router_tokens_per_expert=[486, 500, 517, 541, 494, 452, 538, 557, 524, 532, 532, 482, 499, 511, 518, 508, 561, 555, 473, 534, 495, 504, 529, 521, 488, 493, 520, 470, 513, 523, 512, 516, 533, 504, 509, 508, 515, 480, 534, 533, 499, 504, 533, 541, 495, 511, 512, 477, 462, 534, 493, 542, 500, 515, 523, 523, 483, 498, 485, 567, 490, 541, 523, 503] payload_rows_per_dst=[5580, 5442, 5692, 5484, 5622, 5514, 5566, 5482] dup_rows_per_dst=[2590, 2770, 2652, 2586, 2610, 2630, 2618, 2698] combine_rows_per_src=[5580, 5442, 5692, 5484, 5622, 5514, 5566, 5482] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44610 payload_rows_remote=39024 dup_rows=20926 dup_rows_remote=18266 dedup_ratio=0.319305 router_tokens_per_expert=[524, 522, 516, 464, 522, 498, 522, 528, 543, 514, 480, 541, 500, 510, 489, 553, 508, 525, 502, 500, 507, 494, 530, 505, 506, 497, 512, 518, 520, 547, 487, 517, 537, 522, 527, 527, 481, 503, 507, 491, 481, 521, 527, 481, 513, 515, 522, 506, 509, 461, 557, 509, 536, 496, 529, 511, 518, 513, 502, 515, 560, 517, 484, 489] payload_rows_per_dst=[5480, 5634, 5244, 5704, 5618, 5608, 5736, 5586] dup_rows_per_dst=[2474, 2626, 2180, 2820, 2572, 2762, 2832, 2660] combine_rows_per_src=[5480, 5634, 5244, 5704, 5618, 5608, 5736, 5586] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44422 payload_rows_remote=38790 dup_rows=21114 dup_rows_remote=18392 dedup_ratio=0.322174 router_tokens_per_expert=[517, 504, 510, 548, 518, 499, 519, 532, 494, 496, 514, 558, 514, 507, 503, 499, 510, 571, 498, 531, 522, 525, 503, 506, 482, 490, 509, 533, 482, 520, 504, 485, 510, 490, 532, 501, 540, 497, 506, 495, 520, 551, 461, 474, 535, 536, 499, 521, 493, 515, 490, 494, 518, 549, 519, 492, 486, 516, 510, 520, 519, 507, 531, 538] payload_rows_per_dst=[5632, 5534, 5594, 5460, 5582, 5532, 5478, 5610] dup_rows_per_dst=[2722, 2652, 2664, 2550, 2558, 2662, 2662, 2644] combine_rows_per_src=[5632, 5534, 5594, 5460, 5582, 5532, 5478, 5610] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44608 payload_rows_remote=39074 dup_rows=20928 dup_rows_remote=18342 dedup_ratio=0.319336 router_tokens_per_expert=[472, 472, 538, 505, 502, 517, 543, 517, 541, 494, 523, 537, 519, 524, 505, 530, 513, 511, 517, 503, 531, 499, 507, 513, 538, 520, 528, 500, 487, 497, 546, 492, 507, 510, 516, 532, 502, 503, 496, 510, 525, 537, 493, 523, 530, 477, 542, 519, 487, 491, 497, 508, 502, 530, 482, 563, 518, 486, 530, 489, 508, 499, 559, 456] payload_rows_per_dst=[5540, 5710, 5624, 5576, 5540, 5568, 5534, 5516] dup_rows_per_dst=[2592, 2636, 2564, 2640, 2612, 2724, 2586, 2574] combine_rows_per_src=[5540, 5710, 5624, 5576, 5540, 5568, 5534, 5516] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44626 payload_rows_remote=39130 dup_rows=20910 dup_rows_remote=18464 dedup_ratio=0.319061 router_tokens_per_expert=[494, 528, 513, 495, 505, 528, 508, 525, 489, 473, 525, 541, 488, 507, 486, 462, 516, 508, 544, 508, 526, 516, 528, 560, 544, 498, 475, 498, 518, 537, 499, 534, 543, 553, 506, 522, 481, 538, 499, 516, 482, 508, 479, 493, 530, 521, 511, 527, 474, 490, 492, 490, 522, 538, 498, 527, 570, 486, 498, 521, 532, 518, 501, 526] payload_rows_per_dst=[5586, 5496, 5734, 5578, 5628, 5584, 5494, 5526] dup_rows_per_dst=[2606, 2446, 2678, 2628, 2688, 2518, 2568, 2778] combine_rows_per_src=[5586, 5496, 5734, 5578, 5628, 5584, 5494, 5526] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278820864 dispatch_meta_bytes=229136 combine_payload_bytes=278820864 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279050000 ingress_bytes=388921344 link_bytes=388921344 token_bytes=557870864 weight_movement_bytes=110100480 total_nvlink_bytes=667971344 weight_to_token_ratio=0.197358 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.763981838e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278691840 dispatch_meta_bytes=229848 combine_payload_bytes=278691840 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278921688 ingress_bytes=388792320 link_bytes=388792320 token_bytes=557613528 weight_movement_bytes=110100480 total_nvlink_bytes=667714008 weight_to_token_ratio=0.197449 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.761737898e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44418 payload_rows_remote=38918 dup_rows=21118 dup_rows_remote=18624 dedup_ratio=0.322235 router_tokens_per_expert=[532, 533, 539, 493, 529, 506, 513, 523, 511, 528, 544, 492, 517, 495, 554, 505, 534, 466, 533, 491, 529, 499, 485, 535, 511, 510, 492, 516, 512, 516, 527, 546, 512, 485, 487, 474, 526, 528, 486, 499, 499, 457, 550, 476, 522, 501, 485, 510, 500, 510, 531, 515, 549, 516, 493, 503, 500, 529, 577, 496, 478, 483, 547, 528] payload_rows_per_dst=[5616, 5632, 5534, 5556, 5500, 5518, 5524, 5538] dup_rows_per_dst=[2720, 2660, 2610, 2704, 2494, 2482, 2710, 2738] combine_rows_per_src=[5616, 5632, 5534, 5556, 5500, 5518, 5524, 5538] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44554 payload_rows_remote=38960 dup_rows=20982 dup_rows_remote=18438 dedup_ratio=0.320160 router_tokens_per_expert=[533, 511, 515, 490, 510, 528, 508, 535, 523, 526, 524, 506, 512, 514, 519, 490, 551, 498, 487, 491, 482, 525, 513, 540, 491, 503, 491, 503, 501, 505, 482, 518, 500, 538, 516, 526, 485, 489, 491, 512, 484, 514, 516, 506, 515, 516, 516, 502, 530, 528, 495, 526, 530, 484, 504, 532, 497, 519, 510, 519, 547, 510, 538, 548] payload_rows_per_dst=[5570, 5556, 5544, 5530, 5542, 5594, 5542, 5676] dup_rows_per_dst=[2690, 2672, 2630, 2458, 2572, 2544, 2716, 2700] combine_rows_per_src=[5570, 5556, 5544, 5530, 5542, 5594, 5542, 5676] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44420 payload_rows_remote=38842 dup_rows=21116 dup_rows_remote=18438 dedup_ratio=0.322205 router_tokens_per_expert=[514, 493, 513, 528, 491, 514, 457, 506, 516, 514, 498, 525, 536, 464, 541, 511, 507, 515, 502, 491, 514, 537, 556, 525, 542, 513, 543, 496, 456, 513, 505, 517, 490, 503, 465, 492, 494, 528, 509, 510, 520, 471, 522, 506, 519, 527, 512, 529, 486, 541, 507, 506, 512, 545, 554, 477, 504, 518, 521, 528, 497, 540, 506, 576] payload_rows_per_dst=[5478, 5548, 5560, 5540, 5446, 5592, 5578, 5678] dup_rows_per_dst=[2554, 2662, 2734, 2630, 2536, 2620, 2678, 2702] combine_rows_per_src=[5478, 5548, 5560, 5540, 5446, 5592, 5578, 5678] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44510 payload_rows_remote=38992 dup_rows=21026 dup_rows_remote=18310 dedup_ratio=0.320831 router_tokens_per_expert=[468, 524, 551, 516, 492, 513, 538, 532, 531, 509, 521, 524, 517, 541, 525, 534, 495, 516, 534, 495, 471, 498, 521, 472, 545, 497, 520, 495, 507, 481, 517, 545, 513, 495, 513, 507, 506, 571, 519, 493, 484, 519, 515, 516, 534, 500, 506, 555, 513, 505, 521, 490, 553, 481, 496, 523, 488, 523, 460, 530, 478, 503, 511, 502] payload_rows_per_dst=[5600, 5644, 5516, 5564, 5518, 5608, 5552, 5508] dup_rows_per_dst=[2668, 2760, 2488, 2650, 2716, 2650, 2612, 2482] combine_rows_per_src=[5600, 5644, 5516, 5564, 5518, 5608, 5552, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44644 payload_rows_remote=39094 dup_rows=20892 dup_rows_remote=18306 dedup_ratio=0.318787 router_tokens_per_expert=[529, 523, 498, 498, 526, 503, 524, 500, 516, 490, 549, 564, 561, 515, 481, 515, 506, 506, 521, 512, 513, 537, 516, 525, 501, 499, 483, 540, 470, 510, 496, 516, 543, 524, 485, 549, 490, 523, 483, 516, 499, 526, 530, 525, 539, 505, 511, 479, 477, 480, 548, 504, 508, 516, 510, 501, 507, 515, 511, 493, 509, 498, 509, 512] payload_rows_per_dst=[5582, 5736, 5562, 5502, 5582, 5632, 5498, 5550] dup_rows_per_dst=[2534, 2730, 2710, 2502, 2644, 2596, 2590, 2586] combine_rows_per_src=[5582, 5736, 5562, 5502, 5582, 5632, 5498, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44394 payload_rows_remote=38644 dup_rows=21142 dup_rows_remote=18304 dedup_ratio=0.322601 router_tokens_per_expert=[533, 536, 488, 507, 537, 500, 506, 539, 485, 470, 511, 474, 510, 509, 551, 529, 493, 479, 489, 497, 496, 460, 563, 482, 553, 531, 581, 540, 523, 527, 517, 522, 497, 504, 522, 491, 545, 506, 479, 511, 508, 541, 519, 485, 540, 495, 500, 484, 547, 547, 494, 503, 523, 528, 497, 487, 521, 552, 504, 511, 517, 481, 480, 511] payload_rows_per_dst=[5648, 5514, 5364, 5750, 5570, 5494, 5554, 5500] dup_rows_per_dst=[2644, 2564, 2554, 2838, 2540, 2650, 2698, 2654] combine_rows_per_src=[5648, 5514, 5364, 5750, 5570, 5494, 5554, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277573632 dispatch_meta_bytes=229552 combine_payload_bytes=277573632 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277803184 ingress_bytes=387674112 link_bytes=387674112 token_bytes=555376816 weight_movement_bytes=110100480 total_nvlink_bytes=665477296 weight_to_token_ratio=0.198245 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.742290422e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278290432 dispatch_meta_bytes=229360 combine_payload_bytes=278290432 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278519792 ingress_bytes=388390912 link_bytes=388390912 token_bytes=556810224 weight_movement_bytes=110100480 total_nvlink_bytes=666910704 weight_to_token_ratio=0.197734 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.754756753e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279265280 dispatch_meta_bytes=229592 combine_payload_bytes=279265280 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279494872 ingress_bytes=499466240 link_bytes=499466240 token_bytes=558760152 weight_movement_bytes=220200960 total_nvlink_bytes=778961112 weight_to_token_ratio=0.394089 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.686539394e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278419456 dispatch_meta_bytes=229120 combine_payload_bytes=278419456 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278648576 ingress_bytes=498620416 link_bytes=498620416 token_bytes=557068032 weight_movement_bytes=220200960 total_nvlink_bytes=777268992 weight_to_token_ratio=0.395286 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.671829124e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279494656 dispatch_meta_bytes=229208 combine_payload_bytes=279494656 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279723864 ingress_bytes=389595136 link_bytes=389595136 token_bytes=559218520 weight_movement_bytes=110100480 total_nvlink_bytes=669319000 weight_to_token_ratio=0.196883 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.775700189e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280225792 dispatch_meta_bytes=229600 combine_payload_bytes=280225792 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280455392 ingress_bytes=610527232 link_bytes=610527232 token_bytes=560681184 weight_movement_bytes=330301440 total_nvlink_bytes=890982624 weight_to_token_ratio=0.589107 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061807271e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277000192 dispatch_meta_bytes=227792 combine_payload_bytes=277000192 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277227984 ingress_bytes=497201152 link_bytes=497201152 token_bytes=554228176 weight_movement_bytes=220200960 total_nvlink_bytes=774429136 weight_to_token_ratio=0.397311 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.647145789e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44482 payload_rows_remote=38960 dup_rows=21054 dup_rows_remote=18562 dedup_ratio=0.321259 router_tokens_per_expert=[457, 526, 521, 486, 503, 483, 511, 520, 522, 475, 510, 501, 485, 500, 537, 491, 492, 515, 494, 487, 500, 508, 481, 551, 557, 510, 500, 511, 535, 530, 546, 513, 541, 536, 489, 526, 486, 524, 507, 484, 512, 536, 500, 518, 547, 501, 534, 500, 511, 459, 502, 514, 499, 517, 539, 529, 515, 536, 538, 545, 488, 546, 495, 536] payload_rows_per_dst=[5522, 5514, 5636, 5674, 5658, 5476, 5438, 5564] dup_rows_per_dst=[2492, 2528, 2810, 2730, 2896, 2510, 2460, 2628] combine_rows_per_src=[5522, 5514, 5636, 5674, 5658, 5476, 5438, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44464 payload_rows_remote=38880 dup_rows=21072 dup_rows_remote=18384 dedup_ratio=0.321533 router_tokens_per_expert=[525, 500, 530, 528, 530, 518, 505, 525, 532, 519, 518, 502, 531, 485, 546, 503, 502, 528, 521, 505, 485, 484, 511, 495, 491, 496, 500, 499, 502, 505, 526, 526, 504, 507, 543, 451, 546, 481, 504, 524, 526, 515, 534, 504, 514, 503, 474, 509, 525, 482, 470, 520, 508, 552, 534, 531, 526, 515, 532, 502, 503, 490, 524, 542] payload_rows_per_dst=[5676, 5584, 5492, 5494, 5564, 5500, 5498, 5656] dup_rows_per_dst=[2646, 2688, 2570, 2596, 2556, 2658, 2746, 2612] combine_rows_per_src=[5676, 5584, 5492, 5494, 5564, 5500, 5498, 5656] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279265280 dispatch_meta_bytes=230088 combine_payload_bytes=279265280 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279495368 ingress_bytes=499466240 link_bytes=499466240 token_bytes=558760648 weight_movement_bytes=220200960 total_nvlink_bytes=778961608 weight_to_token_ratio=0.394088 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.686539394e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44512 payload_rows_remote=38934 dup_rows=21024 dup_rows_remote=18286 dedup_ratio=0.320801 router_tokens_per_expert=[531, 510, 559, 538, 487, 509, 508, 514, 504, 490, 499, 500, 506, 512, 490, 505, 502, 512, 537, 528, 499, 520, 524, 536, 503, 497, 505, 491, 475, 517, 496, 477, 521, 515, 505, 532, 525, 526, 517, 553, 539, 529, 481, 521, 523, 492, 488, 511, 521, 495, 540, 514, 539, 530, 486, 524, 511, 514, 540, 482, 522, 515, 483, 493] payload_rows_per_dst=[5632, 5518, 5578, 5426, 5660, 5538, 5660, 5500] dup_rows_per_dst=[2680, 2494, 2738, 2496, 2728, 2630, 2638, 2620] combine_rows_per_src=[5632, 5518, 5578, 5426, 5660, 5538, 5660, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44406 payload_rows_remote=38780 dup_rows=21130 dup_rows_remote=18394 dedup_ratio=0.322418 router_tokens_per_expert=[553, 511, 466, 545, 491, 510, 553, 488, 529, 492, 511, 493, 505, 475, 536, 494, 493, 521, 502, 526, 543, 491, 489, 537, 481, 506, 512, 526, 520, 521, 536, 504, 527, 563, 481, 512, 505, 469, 508, 526, 542, 495, 548, 535, 490, 529, 519, 523, 520, 522, 502, 477, 524, 515, 519, 526, 536, 462, 530, 517, 498, 498, 489, 501] payload_rows_per_dst=[5552, 5512, 5530, 5592, 5552, 5626, 5572, 5470] dup_rows_per_dst=[2682, 2558, 2674, 2620, 2630, 2736, 2638, 2592] combine_rows_per_src=[5552, 5512, 5530, 5592, 5552, 5626, 5572, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44534 payload_rows_remote=38900 dup_rows=21002 dup_rows_remote=18220 dedup_ratio=0.320465 router_tokens_per_expert=[526, 529, 494, 531, 502, 496, 501, 480, 535, 500, 530, 501, 489, 523, 477, 498, 498, 499, 536, 541, 501, 473, 525, 494, 511, 547, 479, 524, 521, 515, 479, 481, 524, 532, 551, 508, 517, 525, 498, 553, 491, 542, 520, 517, 490, 514, 530, 504, 556, 506, 516, 487, 503, 486, 512, 510, 524, 489, 544, 494, 525, 510, 521, 533] payload_rows_per_dst=[5552, 5494, 5548, 5536, 5634, 5552, 5576, 5642] dup_rows_per_dst=[2566, 2612, 2586, 2578, 2782, 2664, 2576, 2638] combine_rows_per_src=[5552, 5494, 5548, 5536, 5634, 5552, 5576, 5642] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44600 payload_rows_remote=39112 dup_rows=20936 dup_rows_remote=18450 dedup_ratio=0.319458 router_tokens_per_expert=[452, 515, 548, 534, 484, 528, 521, 500, 526, 526, 519, 507, 546, 515, 478, 490, 526, 544, 496, 509, 511, 520, 503, 516, 479, 510, 553, 472, 537, 517, 480, 504, 528, 506, 539, 500, 532, 488, 471, 506, 522, 509, 477, 532, 525, 535, 481, 541, 487, 493, 503, 484, 475, 532, 502, 511, 523, 498, 508, 544, 553, 538, 539, 520] payload_rows_per_dst=[5578, 5570, 5610, 5584, 5550, 5542, 5488, 5678] dup_rows_per_dst=[2586, 2644, 2640, 2520, 2590, 2702, 2486, 2768] combine_rows_per_src=[5578, 5570, 5610, 5584, 5550, 5542, 5488, 5678] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44446 payload_rows_remote=38904 dup_rows=21090 dup_rows_remote=18552 dedup_ratio=0.321808 router_tokens_per_expert=[493, 492, 500, 511, 536, 487, 479, 480, 510, 511, 476, 522, 514, 494, 512, 475, 529, 493, 563, 499, 511, 497, 508, 519, 537, 529, 522, 498, 528, 534, 558, 545, 487, 509, 485, 514, 520, 534, 536, 524, 497, 541, 493, 512, 483, 546, 537, 546, 552, 507, 540, 481, 522, 487, 525, 494, 510, 530, 504, 508, 496, 513, 486, 487] payload_rows_per_dst=[5328, 5524, 5554, 5724, 5378, 5862, 5534, 5542] dup_rows_per_dst=[2418, 2504, 2598, 2876, 2506, 2930, 2720, 2538] combine_rows_per_src=[5328, 5524, 5554, 5724, 5378, 5862, 5534, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278691840 dispatch_meta_bytes=229056 combine_payload_bytes=278691840 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278920896 ingress_bytes=388792320 link_bytes=388792320 token_bytes=557612736 weight_movement_bytes=110100480 total_nvlink_bytes=667713216 weight_to_token_ratio=0.197450 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.761737898e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44610 payload_rows_remote=38990 dup_rows=20926 dup_rows_remote=18214 dedup_ratio=0.319305 router_tokens_per_expert=[504, 553, 495, 551, 518, 522, 499, 525, 518, 463, 498, 502, 518, 508, 524, 501, 501, 525, 491, 530, 558, 517, 516, 465, 502, 538, 473, 494, 547, 541, 552, 519, 506, 493, 498, 561, 508, 509, 520, 529, 495, 501, 505, 534, 523, 492, 517, 528, 511, 514, 481, 500, 515, 515, 534, 521, 512, 491, 499, 468, 491, 493, 520, 516] payload_rows_per_dst=[5628, 5522, 5564, 5620, 5628, 5590, 5576, 5482] dup_rows_per_dst=[2706, 2542, 2642, 2712, 2620, 2600, 2606, 2498] combine_rows_per_src=[5628, 5522, 5564, 5620, 5628, 5590, 5576, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279078912 dispatch_meta_bytes=228880 combine_payload_bytes=279078912 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279307792 ingress_bytes=499279872 link_bytes=499279872 token_bytes=558386704 weight_movement_bytes=220200960 total_nvlink_bytes=778587664 weight_to_token_ratio=0.394352 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.683298148e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277975040 dispatch_meta_bytes=228696 combine_payload_bytes=277975040 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278203736 ingress_bytes=498176000 link_bytes=498176000 token_bytes=556178776 weight_movement_bytes=220200960 total_nvlink_bytes=776379736 weight_to_token_ratio=0.395918 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.664099999e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278835200 dispatch_meta_bytes=228480 combine_payload_bytes=278835200 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279063680 ingress_bytes=499036160 link_bytes=499036160 token_bytes=557898880 weight_movement_bytes=220200960 total_nvlink_bytes=778099840 weight_to_token_ratio=0.394697 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.679059596e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=280354816 dispatch_meta_bytes=230248 combine_payload_bytes=280354816 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280585064 ingress_bytes=390455296 link_bytes=390455296 token_bytes=560939880 weight_movement_bytes=110100480 total_nvlink_bytes=671040360 weight_to_token_ratio=0.196279 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.790659786e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278863872 dispatch_meta_bytes=229824 combine_payload_bytes=278863872 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279093696 ingress_bytes=388964352 link_bytes=388964352 token_bytes=557957568 weight_movement_bytes=110100480 total_nvlink_bytes=668058048 weight_to_token_ratio=0.197328 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.764729818e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44310 payload_rows_remote=38772 dup_rows=21226 dup_rows_remote=18458 dedup_ratio=0.323883 router_tokens_per_expert=[497, 530, 495, 521, 512, 532, 525, 541, 493, 510, 498, 479, 495, 504, 493, 483, 552, 506, 541, 489, 505, 473, 480, 560, 454, 540, 502, 498, 516, 522, 534, 515, 520, 515, 502, 525, 517, 489, 517, 494, 527, 483, 512, 496, 525, 530, 531, 529, 521, 516, 516, 515, 546, 521, 485, 491, 523, 507, 511, 563, 511, 507, 490, 538] payload_rows_per_dst=[5538, 5744, 5568, 5572, 5488, 5266, 5584, 5550] dup_rows_per_dst=[2768, 3026, 2644, 2590, 2670, 2140, 2638, 2750] combine_rows_per_src=[5538, 5744, 5568, 5572, 5488, 5266, 5584, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279480320 dispatch_meta_bytes=228816 combine_payload_bytes=279480320 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279709136 ingress_bytes=389580800 link_bytes=389580800 token_bytes=559189456 weight_movement_bytes=110100480 total_nvlink_bytes=669289936 weight_to_token_ratio=0.196893 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.775450862e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44594 payload_rows_remote=39114 dup_rows=20942 dup_rows_remote=18518 dedup_ratio=0.319550 router_tokens_per_expert=[492, 524, 510, 490, 502, 526, 505, 524, 488, 473, 521, 537, 487, 503, 480, 463, 517, 508, 542, 509, 524, 514, 526, 561, 546, 500, 474, 498, 517, 538, 496, 535, 544, 550, 503, 522, 481, 537, 499, 516, 483, 509, 479, 497, 531, 522, 512, 530, 474, 491, 497, 491, 527, 541, 501, 529, 571, 488, 500, 523, 536, 521, 505, 528] payload_rows_per_dst=[5562, 5480, 5726, 5572, 5622, 5590, 5508, 5534] dup_rows_per_dst=[2584, 2424, 2676, 2636, 2682, 2536, 2594, 2810] combine_rows_per_src=[5562, 5480, 5726, 5572, 5622, 5590, 5508, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44382 payload_rows_remote=38684 dup_rows=21154 dup_rows_remote=18504 dedup_ratio=0.322784 router_tokens_per_expert=[485, 497, 515, 541, 491, 448, 532, 556, 521, 530, 531, 478, 495, 508, 513, 508, 562, 555, 472, 530, 497, 505, 530, 523, 488, 493, 519, 471, 512, 522, 511, 517, 532, 503, 509, 510, 515, 481, 537, 532, 501, 504, 534, 541, 499, 514, 515, 477, 462, 533, 493, 543, 500, 518, 525, 525, 486, 500, 491, 567, 492, 545, 524, 504] payload_rows_per_dst=[5562, 5428, 5698, 5474, 5622, 5532, 5570, 5496] dup_rows_per_dst=[2568, 2740, 2650, 2592, 2616, 2638, 2628, 2722] combine_rows_per_src=[5562, 5428, 5698, 5474, 5622, 5532, 5570, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277917696 dispatch_meta_bytes=228920 combine_payload_bytes=277917696 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=278146616 ingress_bytes=718319616 link_bytes=718319616 token_bytes=556064312 weight_movement_bytes=440401920 total_nvlink_bytes=996466232 weight_to_token_ratio=0.791998 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.249275955e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277329920 dispatch_meta_bytes=228768 combine_payload_bytes=277329920 prefetch_bytes=110100480 reduce_bytes=550502400 egress_bytes=277558688 ingress_bytes=937932800 link_bytes=937932800 token_bytes=554888608 weight_movement_bytes=660602880 total_nvlink_bytes=1215491488 weight_to_token_ratio=1.190514 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.631219403e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279724032 dispatch_meta_bytes=229160 combine_payload_bytes=279724032 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279953192 ingress_bytes=389824512 link_bytes=389824512 token_bytes=559677224 weight_movement_bytes=110100480 total_nvlink_bytes=669777704 weight_to_token_ratio=0.196721 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.779689415e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278046720 dispatch_meta_bytes=228728 combine_payload_bytes=278046720 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278275448 ingress_bytes=498247680 link_bytes=498247680 token_bytes=556322168 weight_movement_bytes=220200960 total_nvlink_bytes=776523128 weight_to_token_ratio=0.395816 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.665346632e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=280082432 dispatch_meta_bytes=229664 combine_payload_bytes=280082432 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280312096 ingress_bytes=390182912 link_bytes=390182912 token_bytes=560394528 weight_movement_bytes=110100480 total_nvlink_bytes=670495008 weight_to_token_ratio=0.196470 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.785922580e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=280483840 dispatch_meta_bytes=230376 combine_payload_bytes=280483840 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280714216 ingress_bytes=390584320 link_bytes=390584320 token_bytes=561198056 weight_movement_bytes=110100480 total_nvlink_bytes=671298536 weight_to_token_ratio=0.196188 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.792903726e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278964224 dispatch_meta_bytes=230168 combine_payload_bytes=278964224 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279194392 ingress_bytes=499165184 link_bytes=499165184 token_bytes=558158616 weight_movement_bytes=220200960 total_nvlink_bytes=778359576 weight_to_token_ratio=0.394513 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.681303535e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44448 payload_rows_remote=38832 dup_rows=21088 dup_rows_remote=18510 dedup_ratio=0.321777 router_tokens_per_expert=[541, 511, 494, 550, 518, 518, 503, 539, 510, 528, 522, 525, 491, 489, 527, 529, 497, 492, 528, 460, 538, 472, 534, 512, 515, 535, 525, 513, 486, 511, 514, 498, 505, 506, 517, 492, 509, 491, 526, 534, 484, 526, 542, 510, 529, 515, 521, 499, 525, 492, 492, 525, 501, 516, 515, 528, 512, 510, 489, 485, 518, 521, 506, 502] payload_rows_per_dst=[5562, 5568, 5510, 5616, 5522, 5612, 5602, 5456] dup_rows_per_dst=[2786, 2674, 2556, 2578, 2638, 2640, 2586, 2630] combine_rows_per_src=[5562, 5568, 5510, 5616, 5522, 5612, 5602, 5456] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44688 payload_rows_remote=39132 dup_rows=20848 dup_rows_remote=18370 dedup_ratio=0.318115 router_tokens_per_expert=[511, 529, 498, 547, 501, 487, 494, 479, 519, 528, 497, 541, 517, 523, 490, 537, 534, 508, 548, 512, 482, 507, 506, 484, 533, 531, 540, 504, 535, 490, 488, 517, 537, 497, 482, 551, 514, 520, 550, 525, 506, 479, 514, 495, 502, 533, 479, 509, 500, 526, 483, 516, 498, 504, 531, 505, 473, 545, 493, 543, 525, 498, 506, 512] payload_rows_per_dst=[5510, 5606, 5590, 5622, 5648, 5556, 5566, 5590] dup_rows_per_dst=[2582, 2698, 2572, 2654, 2704, 2478, 2560, 2600] combine_rows_per_src=[5510, 5606, 5590, 5622, 5648, 5556, 5566, 5590] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44482 payload_rows_remote=38962 dup_rows=21054 dup_rows_remote=18512 dedup_ratio=0.321259 router_tokens_per_expert=[519, 531, 520, 534, 516, 512, 500, 510, 498, 541, 515, 512, 531, 530, 513, 489, 492, 518, 516, 516, 486, 483, 515, 505, 486, 516, 513, 579, 441, 508, 528, 525, 530, 544, 436, 522, 508, 519, 536, 514, 528, 458, 511, 510, 522, 482, 492, 523, 520, 485, 518, 495, 506, 507, 546, 509, 467, 539, 548, 520, 517, 528, 528, 502] payload_rows_per_dst=[5646, 5544, 5520, 5594, 5566, 5448, 5548, 5616] dup_rows_per_dst=[2638, 2714, 2542, 2598, 2652, 2604, 2624, 2682] combine_rows_per_src=[5646, 5544, 5520, 5594, 5566, 5448, 5548, 5616] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44482 payload_rows_remote=38968 dup_rows=21054 dup_rows_remote=18454 dedup_ratio=0.321259 router_tokens_per_expert=[487, 525, 535, 533, 516, 517, 501, 511, 526, 533, 523, 496, 474, 489, 535, 553, 511, 508, 499, 520, 516, 521, 518, 507, 489, 498, 502, 540, 529, 500, 527, 461, 554, 512, 475, 546, 500, 519, 529, 505, 546, 515, 480, 485, 523, 529, 526, 496, 510, 528, 505, 502, 508, 525, 505, 538, 455, 523, 488, 559, 504, 509, 463, 506] payload_rows_per_dst=[5294, 5492, 5774, 5468, 5600, 5740, 5600, 5514] dup_rows_per_dst=[2408, 2354, 2974, 2524, 2680, 2872, 2642, 2600] combine_rows_per_src=[5294, 5492, 5774, 5468, 5600, 5740, 5600, 5514] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278347776 dispatch_meta_bytes=229368 combine_payload_bytes=278347776 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278577144 ingress_bytes=608649216 link_bytes=608649216 token_bytes=556924920 weight_movement_bytes=330301440 total_nvlink_bytes=887226360 weight_to_token_ratio=0.593081 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058541092e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44638 payload_rows_remote=39116 dup_rows=20898 dup_rows_remote=18344 dedup_ratio=0.318878 router_tokens_per_expert=[518, 505, 473, 543, 502, 492, 529, 490, 548, 512, 509, 501, 519, 516, 505, 509, 500, 521, 498, 500, 500, 511, 505, 507, 478, 524, 525, 514, 537, 524, 539, 511, 532, 508, 516, 523, 517, 512, 469, 511, 513, 500, 503, 540, 523, 531, 517, 486, 499, 528, 524, 485, 515, 540, 505, 442, 532, 524, 524, 500, 535, 556, 495, 498] payload_rows_per_dst=[5568, 5658, 5476, 5652, 5516, 5602, 5522, 5644] dup_rows_per_dst=[2536, 2580, 2608, 2652, 2660, 2624, 2554, 2684] combine_rows_per_src=[5568, 5658, 5476, 5652, 5516, 5602, 5522, 5644] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44450 payload_rows_remote=39030 dup_rows=21086 dup_rows_remote=18484 dedup_ratio=0.321747 router_tokens_per_expert=[578, 491, 551, 483, 526, 527, 543, 533, 496, 511, 507, 483, 495, 506, 531, 482, 477, 509, 513, 480, 533, 509, 517, 543, 563, 529, 511, 503, 536, 480, 460, 519, 529, 515, 519, 524, 518, 493, 550, 503, 493, 486, 490, 511, 505, 494, 517, 482, 484, 496, 511, 538, 506, 494, 517, 536, 514, 474, 509, 534, 521, 523, 511, 546] payload_rows_per_dst=[5700, 5420, 5522, 5526, 5684, 5458, 5538, 5602] dup_rows_per_dst=[2764, 2602, 2640, 2676, 2618, 2498, 2626, 2662] combine_rows_per_src=[5700, 5420, 5522, 5526, 5684, 5458, 5538, 5602] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44504 payload_rows_remote=38900 dup_rows=21032 dup_rows_remote=18430 dedup_ratio=0.320923 router_tokens_per_expert=[485, 519, 518, 525, 516, 512, 480, 514, 525, 530, 509, 538, 508, 520, 524, 499, 545, 493, 518, 509, 517, 497, 472, 503, 470, 526, 482, 486, 564, 497, 520, 538, 513, 506, 504, 508, 514, 509, 504, 492, 531, 529, 531, 514, 565, 499, 487, 546, 545, 496, 496, 527, 494, 475, 506, 501, 524, 474, 520, 520, 493, 534, 513, 539] payload_rows_per_dst=[5604, 5628, 5530, 5564, 5456, 5620, 5598, 5504] dup_rows_per_dst=[2602, 2706, 2578, 2552, 2598, 2784, 2736, 2476] combine_rows_per_src=[5604, 5628, 5530, 5564, 5456, 5620, 5598, 5504] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44624 payload_rows_remote=39072 dup_rows=20912 dup_rows_remote=18336 dedup_ratio=0.319092 router_tokens_per_expert=[473, 488, 531, 529, 495, 530, 515, 519, 528, 515, 514, 531, 512, 510, 478, 485, 528, 537, 520, 504, 510, 523, 480, 494, 522, 510, 489, 508, 519, 498, 530, 541, 479, 530, 506, 515, 513, 516, 491, 514, 515, 504, 511, 502, 543, 488, 498, 557, 512, 491, 521, 530, 524, 520, 508, 530, 527, 503, 504, 508, 523, 503, 504, 512] payload_rows_per_dst=[5664, 5582, 5534, 5588, 5552, 5542, 5578, 5584] dup_rows_per_dst=[2496, 2564, 2658, 2646, 2576, 2694, 2694, 2584] combine_rows_per_src=[5664, 5582, 5534, 5588, 5552, 5542, 5578, 5584] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=280498176 dispatch_meta_bytes=230008 combine_payload_bytes=280498176 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280728184 ingress_bytes=390598656 link_bytes=390598656 token_bytes=561226360 weight_movement_bytes=110100480 total_nvlink_bytes=671326840 weight_to_token_ratio=0.196178 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.793153052e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279279616 dispatch_meta_bytes=229896 combine_payload_bytes=279279616 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279509512 ingress_bytes=389380096 link_bytes=389380096 token_bytes=558789128 weight_movement_bytes=110100480 total_nvlink_bytes=668889608 weight_to_token_ratio=0.197034 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.771960290e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279322624 dispatch_meta_bytes=229688 combine_payload_bytes=279322624 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279552312 ingress_bytes=499523584 link_bytes=499523584 token_bytes=558874936 weight_movement_bytes=220200960 total_nvlink_bytes=779075896 weight_to_token_ratio=0.394008 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.687536701e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=280383488 dispatch_meta_bytes=229840 combine_payload_bytes=280383488 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280613328 ingress_bytes=390483968 link_bytes=390483968 token_bytes=560996816 weight_movement_bytes=110100480 total_nvlink_bytes=671097296 weight_to_token_ratio=0.196259 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.791158439e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279767040 dispatch_meta_bytes=230056 combine_payload_bytes=279767040 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279997096 ingress_bytes=499968000 link_bytes=499968000 token_bytes=559764136 weight_movement_bytes=220200960 total_nvlink_bytes=779965096 weight_to_token_ratio=0.393382 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.695265826e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278835200 dispatch_meta_bytes=229320 combine_payload_bytes=278835200 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279064520 ingress_bytes=499036160 link_bytes=499036160 token_bytes=557899720 weight_movement_bytes=220200960 total_nvlink_bytes=778100680 weight_to_token_ratio=0.394696 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.679059596e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280068096 dispatch_meta_bytes=229632 combine_payload_bytes=280068096 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280297728 ingress_bytes=390168576 link_bytes=390168576 token_bytes=560365824 weight_movement_bytes=110100480 total_nvlink_bytes=670466304 weight_to_token_ratio=0.196480 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.051471 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.785673254e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44548 payload_rows_remote=39002 dup_rows=20988 dup_rows_remote=18488 dedup_ratio=0.320251 router_tokens_per_expert=[525, 509, 561, 477, 530, 495, 525, 525, 542, 527, 539, 497, 499, 545, 534, 500, 512, 496, 505, 494, 524, 529, 482, 487, 508, 499, 514, 501, 518, 493, 481, 509, 547, 480, 484, 517, 535, 526, 553, 511, 509, 508, 470, 499, 518, 506, 544, 544, 512, 523, 513, 494, 518, 494, 543, 536, 478, 512, 493, 512, 503, 492, 492, 520] payload_rows_per_dst=[5636, 5616, 5506, 5546, 5568, 5574, 5638, 5464] dup_rows_per_dst=[2658, 2750, 2552, 2500, 2738, 2622, 2628, 2540] combine_rows_per_src=[5636, 5616, 5506, 5546, 5568, 5574, 5638, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44360 payload_rows_remote=38866 dup_rows=21176 dup_rows_remote=18574 dedup_ratio=0.323120 router_tokens_per_expert=[495, 524, 474, 529, 545, 490, 517, 533, 479, 512, 518, 560, 478, 524, 524, 515, 502, 582, 553, 540, 507, 498, 486, 509, 508, 499, 488, 507, 509, 487, 510, 523, 505, 512, 494, 522, 528, 545, 500, 537, 505, 527, 499, 503, 505, 485, 521, 503, 507, 490, 479, 476, 515, 521, 529, 546, 500, 518, 491, 515, 510, 528, 532, 495] payload_rows_per_dst=[5512, 5524, 5620, 5490, 5660, 5494, 5544, 5516] dup_rows_per_dst=[2702, 2696, 2734, 2572, 2626, 2602, 2582, 2662] combine_rows_per_src=[5512, 5524, 5620, 5490, 5660, 5494, 5544, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44464 payload_rows_remote=39032 dup_rows=21072 dup_rows_remote=18430 dedup_ratio=0.321533 router_tokens_per_expert=[508, 502, 481, 538, 522, 501, 523, 521, 487, 534, 514, 484, 525, 539, 540, 478, 545, 524, 544, 508, 494, 526, 527, 546, 532, 526, 502, 486, 537, 499, 532, 476, 512, 511, 509, 506, 518, 493, 488, 500, 509, 529, 531, 492, 492, 539, 513, 473, 498, 511, 502, 539, 507, 537, 514, 525, 537, 478, 516, 491, 486, 495, 506, 510] payload_rows_per_dst=[5596, 5594, 5718, 5538, 5432, 5558, 5558, 5470] dup_rows_per_dst=[2596, 2608, 2710, 2642, 2642, 2598, 2708, 2568] combine_rows_per_src=[5596, 5594, 5718, 5538, 5432, 5558, 5558, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44618 payload_rows_remote=39070 dup_rows=20918 dup_rows_remote=18318 dedup_ratio=0.319183 router_tokens_per_expert=[469, 471, 534, 503, 499, 516, 539, 512, 540, 491, 517, 537, 519, 525, 504, 528, 512, 509, 515, 505, 530, 499, 505, 509, 538, 522, 527, 498, 488, 498, 546, 492, 507, 507, 517, 534, 503, 503, 495, 510, 527, 539, 494, 523, 533, 473, 545, 520, 489, 493, 499, 510, 502, 532, 485, 564, 520, 488, 530, 492, 509, 502, 564, 462] payload_rows_per_dst=[5532, 5704, 5612, 5570, 5542, 5568, 5548, 5542] dup_rows_per_dst=[2554, 2618, 2556, 2648, 2610, 2740, 2600, 2592] combine_rows_per_src=[5532, 5704, 5612, 5570, 5542, 5568, 5548, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=280369152 dispatch_meta_bytes=230528 combine_payload_bytes=280369152 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280599680 ingress_bytes=390469632 link_bytes=390469632 token_bytes=560968832 weight_movement_bytes=110100480 total_nvlink_bytes=671069312 weight_to_token_ratio=0.196268 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.790909113e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44730 payload_rows_remote=39138 dup_rows=20806 dup_rows_remote=18144 dedup_ratio=0.317474 router_tokens_per_expert=[518, 519, 514, 463, 521, 494, 519, 522, 540, 512, 478, 539, 499, 508, 487, 547, 506, 521, 502, 500, 505, 495, 530, 507, 507, 498, 512, 520, 520, 547, 487, 515, 537, 521, 529, 526, 480, 502, 505, 492, 481, 523, 529, 483, 513, 520, 526, 509, 512, 463, 558, 511, 537, 498, 532, 513, 522, 513, 503, 518, 561, 520, 487, 492] payload_rows_per_dst=[5554, 5624, 5390, 5742, 5612, 5564, 5652, 5592] dup_rows_per_dst=[2586, 2596, 2294, 2822, 2576, 2674, 2596, 2662] combine_rows_per_src=[5554, 5624, 5390, 5742, 5612, 5564, 5652, 5592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277286912 dispatch_meta_bytes=228752 combine_payload_bytes=277286912 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=277515664 ingress_bytes=827789312 link_bytes=827789312 token_bytes=554802576 weight_movement_bytes=550502400 total_nvlink_bytes=1105304976 weight_to_token_ratio=0.992249 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.439661762e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44452 payload_rows_remote=38888 dup_rows=21084 dup_rows_remote=18412 dedup_ratio=0.321716 router_tokens_per_expert=[525, 517, 503, 530, 524, 517, 480, 503, 469, 522, 487, 491, 511, 479, 562, 528, 543, 476, 502, 485, 508, 502, 492, 536, 489, 533, 546, 507, 520, 515, 522, 486, 493, 531, 496, 507, 522, 540, 475, 516, 508, 497, 516, 464, 546, 528, 516, 536, 487, 518, 518, 521, 511, 526, 513, 481, 488, 518, 540, 584, 500, 493, 543, 526] payload_rows_per_dst=[5606, 5490, 5502, 5564, 5600, 5526, 5552, 5612] dup_rows_per_dst=[2592, 2608, 2586, 2672, 2560, 2696, 2598, 2772] combine_rows_per_src=[5606, 5490, 5502, 5564, 5600, 5526, 5552, 5612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278591488 dispatch_meta_bytes=229760 combine_payload_bytes=278591488 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278821248 ingress_bytes=388691968 link_bytes=388691968 token_bytes=557412736 weight_movement_bytes=110100480 total_nvlink_bytes=667513216 weight_to_token_ratio=0.197521 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.759992612e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279781376 dispatch_meta_bytes=229848 combine_payload_bytes=279781376 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280011224 ingress_bytes=389881856 link_bytes=389881856 token_bytes=559792600 weight_movement_bytes=110100480 total_nvlink_bytes=669893080 weight_to_token_ratio=0.196681 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.780686721e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=280053760 dispatch_meta_bytes=229552 combine_payload_bytes=280053760 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280283312 ingress_bytes=390154240 link_bytes=390154240 token_bytes=560337072 weight_movement_bytes=110100480 total_nvlink_bytes=670437552 weight_to_token_ratio=0.196490 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.785423927e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=280541184 dispatch_meta_bytes=229128 combine_payload_bytes=280541184 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280770312 ingress_bytes=610842624 link_bytes=610842624 token_bytes=561311496 weight_movement_bytes=330301440 total_nvlink_bytes=891612936 weight_to_token_ratio=0.588446 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.062355789e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44418 payload_rows_remote=38776 dup_rows=21118 dup_rows_remote=18330 dedup_ratio=0.322235 router_tokens_per_expert=[515, 502, 506, 546, 516, 497, 518, 529, 492, 494, 514, 557, 513, 506, 500, 500, 510, 571, 498, 528, 520, 524, 501, 505, 481, 490, 509, 534, 481, 518, 503, 485, 508, 491, 531, 500, 540, 495, 505, 497, 521, 555, 463, 475, 536, 538, 498, 524, 494, 515, 490, 493, 519, 552, 520, 495, 487, 519, 514, 526, 520, 509, 535, 540] payload_rows_per_dst=[5642, 5614, 5514, 5456, 5574, 5542, 5540, 5536] dup_rows_per_dst=[2788, 2788, 2550, 2546, 2560, 2678, 2758, 2450] combine_rows_per_src=[5642, 5614, 5514, 5456, 5574, 5542, 5540, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278749184 dispatch_meta_bytes=229200 combine_payload_bytes=278749184 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278978384 ingress_bytes=388849664 link_bytes=388849664 token_bytes=557727568 weight_movement_bytes=110100480 total_nvlink_bytes=667828048 weight_to_token_ratio=0.197409 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.762735205e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44432 payload_rows_remote=39020 dup_rows=21104 dup_rows_remote=18518 dedup_ratio=0.322021 router_tokens_per_expert=[573, 489, 549, 481, 522, 526, 541, 532, 494, 506, 506, 482, 493, 505, 531, 482, 475, 508, 514, 481, 532, 511, 515, 541, 564, 529, 510, 504, 536, 480, 456, 522, 530, 514, 517, 525, 515, 492, 549, 504, 493, 488, 492, 511, 505, 496, 516, 482, 486, 498, 511, 538, 507, 498, 518, 540, 521, 475, 513, 536, 524, 523, 513, 548] payload_rows_per_dst=[5680, 5412, 5514, 5524, 5676, 5468, 5544, 5614] dup_rows_per_dst=[2746, 2586, 2640, 2678, 2616, 2498, 2648, 2692] combine_rows_per_src=[5680, 5412, 5514, 5524, 5676, 5468, 5544, 5614] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277946368 dispatch_meta_bytes=228424 combine_payload_bytes=277946368 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278174792 ingress_bytes=388046848 link_bytes=388046848 token_bytes=556121160 weight_movement_bytes=110100480 total_nvlink_bytes=666221640 weight_to_token_ratio=0.197979 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.748772914e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44486 payload_rows_remote=38962 dup_rows=21050 dup_rows_remote=18512 dedup_ratio=0.321198 router_tokens_per_expert=[511, 527, 515, 529, 514, 509, 499, 507, 496, 538, 511, 511, 532, 529, 510, 487, 493, 520, 515, 515, 483, 483, 516, 506, 485, 514, 513, 582, 440, 507, 528, 525, 531, 545, 435, 522, 508, 518, 539, 515, 531, 459, 512, 510, 525, 482, 491, 528, 521, 485, 521, 498, 507, 509, 548, 513, 469, 541, 549, 522, 518, 532, 530, 504] payload_rows_per_dst=[5622, 5528, 5524, 5594, 5574, 5448, 5560, 5636] dup_rows_per_dst=[2600, 2700, 2538, 2594, 2652, 2628, 2644, 2694] combine_rows_per_src=[5622, 5528, 5524, 5594, 5574, 5448, 5560, 5636] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44422 payload_rows_remote=38922 dup_rows=21114 dup_rows_remote=18604 dedup_ratio=0.322174 router_tokens_per_expert=[526, 532, 538, 489, 526, 505, 510, 519, 509, 527, 540, 492, 515, 491, 552, 497, 534, 464, 534, 494, 528, 497, 486, 535, 510, 510, 490, 515, 510, 514, 529, 545, 512, 487, 489, 476, 528, 530, 485, 498, 499, 458, 551, 479, 522, 503, 486, 510, 499, 512, 534, 517, 550, 519, 496, 506, 502, 532, 580, 500, 482, 484, 549, 530] payload_rows_per_dst=[5594, 5612, 5542, 5554, 5500, 5524, 5538, 5558] dup_rows_per_dst=[2696, 2634, 2602, 2692, 2510, 2492, 2728, 2760] combine_rows_per_src=[5594, 5612, 5542, 5554, 5500, 5524, 5538, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44674 payload_rows_remote=39108 dup_rows=20862 dup_rows_remote=18366 dedup_ratio=0.318329 router_tokens_per_expert=[508, 526, 497, 545, 499, 482, 492, 478, 518, 524, 496, 539, 515, 523, 490, 536, 531, 509, 548, 512, 476, 507, 507, 484, 532, 530, 540, 501, 535, 490, 487, 515, 536, 496, 480, 551, 515, 520, 555, 527, 511, 479, 514, 495, 504, 535, 481, 512, 503, 527, 483, 518, 499, 504, 532, 505, 474, 546, 498, 544, 528, 502, 507, 515] payload_rows_per_dst=[5494, 5596, 5580, 5622, 5650, 5566, 5570, 5596] dup_rows_per_dst=[2560, 2686, 2568, 2638, 2710, 2496, 2572, 2632] combine_rows_per_src=[5494, 5596, 5580, 5622, 5650, 5566, 5570, 5596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44646 payload_rows_remote=39118 dup_rows=20890 dup_rows_remote=18326 dedup_ratio=0.318756 router_tokens_per_expert=[514, 503, 469, 542, 501, 487, 529, 489, 546, 511, 507, 498, 514, 512, 503, 508, 496, 522, 497, 499, 501, 511, 504, 506, 475, 522, 525, 514, 538, 525, 542, 510, 532, 508, 515, 522, 516, 512, 470, 510, 513, 500, 502, 543, 525, 533, 519, 489, 498, 529, 525, 486, 516, 542, 507, 443, 532, 531, 529, 501, 538, 561, 498, 503] payload_rows_per_dst=[5550, 5642, 5474, 5652, 5514, 5618, 5528, 5668] dup_rows_per_dst=[2518, 2556, 2598, 2650, 2656, 2630, 2564, 2718] combine_rows_per_src=[5550, 5642, 5474, 5652, 5514, 5618, 5528, 5668] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279695360 dispatch_meta_bytes=230152 combine_payload_bytes=279695360 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279925512 ingress_bytes=499896320 link_bytes=499896320 token_bytes=559620872 weight_movement_bytes=220200960 total_nvlink_bytes=779821832 weight_to_token_ratio=0.393482 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.694019193e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44484 payload_rows_remote=38936 dup_rows=21052 dup_rows_remote=18408 dedup_ratio=0.321228 router_tokens_per_expert=[485, 524, 533, 531, 514, 513, 501, 507, 525, 535, 522, 493, 472, 489, 533, 549, 511, 504, 501, 517, 515, 520, 517, 505, 487, 497, 503, 541, 532, 502, 527, 459, 553, 511, 474, 545, 502, 519, 530, 504, 547, 513, 481, 485, 523, 527, 526, 495, 511, 531, 507, 506, 513, 528, 506, 538, 459, 529, 488, 562, 504, 511, 466, 510] payload_rows_per_dst=[5386, 5656, 5550, 5446, 5580, 5692, 5626, 5548] dup_rows_per_dst=[2540, 2578, 2630, 2516, 2652, 2792, 2700, 2644] combine_rows_per_src=[5386, 5656, 5550, 5446, 5580, 5692, 5626, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279279616 dispatch_meta_bytes=229896 combine_payload_bytes=279279616 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279509512 ingress_bytes=389380096 link_bytes=389380096 token_bytes=558789128 weight_movement_bytes=110100480 total_nvlink_bytes=668889608 weight_to_token_ratio=0.197034 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.771960290e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44442 payload_rows_remote=38834 dup_rows=21094 dup_rows_remote=18512 dedup_ratio=0.321869 router_tokens_per_expert=[538, 508, 487, 547, 517, 516, 501, 538, 507, 526, 521, 521, 489, 488, 524, 527, 500, 492, 528, 457, 537, 473, 535, 508, 513, 535, 524, 512, 486, 512, 515, 498, 507, 505, 518, 495, 510, 494, 528, 534, 485, 530, 542, 511, 530, 516, 518, 500, 526, 493, 492, 526, 502, 518, 516, 534, 512, 510, 489, 486, 523, 524, 509, 505] payload_rows_per_dst=[5546, 5556, 5504, 5608, 5532, 5618, 5610, 5468] dup_rows_per_dst=[2758, 2650, 2556, 2582, 2650, 2646, 2604, 2648] combine_rows_per_src=[5546, 5556, 5504, 5608, 5532, 5618, 5610, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278992896 dispatch_meta_bytes=230104 combine_payload_bytes=278992896 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279223000 ingress_bytes=609294336 link_bytes=609294336 token_bytes=558215896 weight_movement_bytes=330301440 total_nvlink_bytes=888517336 weight_to_token_ratio=0.591709 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059663062e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=280326144 dispatch_meta_bytes=229896 combine_payload_bytes=280326144 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280556040 ingress_bytes=390426624 link_bytes=390426624 token_bytes=560882184 weight_movement_bytes=110100480 total_nvlink_bytes=670982664 weight_to_token_ratio=0.196299 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.790161133e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=280397824 dispatch_meta_bytes=229776 combine_payload_bytes=280397824 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280627600 ingress_bytes=390498304 link_bytes=390498304 token_bytes=561025424 weight_movement_bytes=110100480 total_nvlink_bytes=671125904 weight_to_token_ratio=0.196249 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.791407766e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44532 payload_rows_remote=38904 dup_rows=21004 dup_rows_remote=18386 dedup_ratio=0.320496 router_tokens_per_expert=[485, 518, 515, 522, 514, 509, 477, 512, 521, 526, 506, 539, 508, 518, 524, 501, 545, 492, 519, 509, 515, 496, 470, 503, 467, 526, 479, 482, 562, 498, 519, 540, 516, 504, 502, 505, 516, 512, 506, 491, 531, 532, 531, 517, 566, 500, 485, 548, 545, 495, 496, 528, 494, 476, 512, 504, 525, 479, 522, 520, 498, 534, 518, 543] payload_rows_per_dst=[5628, 5570, 5764, 5596, 5460, 5628, 5522, 5364] dup_rows_per_dst=[2618, 2574, 2952, 2590, 2604, 2792, 2578, 2296] combine_rows_per_src=[5628, 5570, 5764, 5596, 5460, 5628, 5522, 5364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279093248 dispatch_meta_bytes=229376 combine_payload_bytes=279093248 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279322624 ingress_bytes=499294208 link_bytes=499294208 token_bytes=558415872 weight_movement_bytes=220200960 total_nvlink_bytes=778616832 weight_to_token_ratio=0.394331 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.683547475e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278362112 dispatch_meta_bytes=229384 combine_payload_bytes=278362112 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278591496 ingress_bytes=498563072 link_bytes=498563072 token_bytes=556953608 weight_movement_bytes=220200960 total_nvlink_bytes=777154568 weight_to_token_ratio=0.395367 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.670831817e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278863872 dispatch_meta_bytes=229160 combine_payload_bytes=278863872 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279093032 ingress_bytes=499064832 link_bytes=499064832 token_bytes=557956904 weight_movement_bytes=220200960 total_nvlink_bytes=778157864 weight_to_token_ratio=0.394656 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.679558249e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44464 payload_rows_remote=38872 dup_rows=21072 dup_rows_remote=18460 dedup_ratio=0.321533 router_tokens_per_expert=[507, 534, 512, 509, 483, 496, 505, 482, 543, 522, 491, 504, 488, 539, 487, 528, 485, 507, 491, 537, 492, 528, 507, 546, 541, 484, 516, 543, 486, 499, 523, 490, 521, 525, 525, 519, 517, 509, 518, 532, 495, 549, 524, 470, 489, 482, 538, 494, 528, 554, 517, 516, 500, 528, 527, 503, 520, 484, 508, 499, 482, 517, 531, 542] payload_rows_per_dst=[5474, 5592, 5566, 5624, 5632, 5416, 5622, 5538] dup_rows_per_dst=[2582, 2612, 2620, 2540, 2700, 2666, 2724, 2628] combine_rows_per_src=[5474, 5592, 5566, 5624, 5632, 5416, 5622, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44486 payload_rows_remote=38862 dup_rows=21050 dup_rows_remote=18446 dedup_ratio=0.321198 router_tokens_per_expert=[523, 512, 525, 536, 508, 503, 519, 502, 480, 506, 475, 478, 489, 542, 524, 516, 513, 502, 484, 500, 537, 524, 531, 523, 493, 535, 530, 536, 508, 521, 482, 496, 542, 498, 492, 486, 491, 507, 505, 490, 468, 497, 538, 516, 491, 553, 516, 511, 523, 514, 473, 496, 528, 512, 524, 526, 522, 541, 514, 528, 529, 540, 533, 511] payload_rows_per_dst=[5550, 5502, 5624, 5538, 5508, 5542, 5568, 5654] dup_rows_per_dst=[2706, 2518, 2604, 2664, 2514, 2638, 2624, 2782] combine_rows_per_src=[5550, 5502, 5624, 5538, 5508, 5542, 5568, 5654] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44634 payload_rows_remote=39086 dup_rows=20902 dup_rows_remote=18326 dedup_ratio=0.318939 router_tokens_per_expert=[471, 487, 529, 526, 495, 530, 513, 519, 527, 512, 515, 530, 510, 509, 477, 480, 525, 535, 519, 502, 509, 524, 478, 494, 523, 510, 487, 511, 518, 498, 530, 542, 480, 531, 505, 515, 512, 519, 487, 513, 515, 505, 511, 502, 544, 489, 498, 557, 515, 489, 521, 531, 524, 522, 512, 535, 527, 502, 507, 511, 528, 505, 505, 516] payload_rows_per_dst=[5656, 5578, 5526, 5590, 5548, 5544, 5590, 5602] dup_rows_per_dst=[2484, 2542, 2646, 2648, 2576, 2698, 2708, 2600] combine_rows_per_src=[5656, 5578, 5526, 5590, 5548, 5544, 5590, 5602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44418 payload_rows_remote=38880 dup_rows=21118 dup_rows_remote=18400 dedup_ratio=0.322235 router_tokens_per_expert=[509, 520, 499, 543, 503, 525, 500, 499, 536, 496, 549, 517, 508, 503, 488, 487, 486, 519, 536, 516, 476, 515, 530, 481, 489, 536, 487, 497, 528, 506, 537, 478, 506, 536, 500, 535, 523, 493, 499, 532, 513, 544, 511, 529, 505, 523, 478, 525, 522, 486, 539, 493, 489, 521, 541, 491, 522, 513, 500, 518, 515, 507, 533, 527] payload_rows_per_dst=[5546, 5510, 5520, 5494, 5676, 5538, 5554, 5580] dup_rows_per_dst=[2650, 2658, 2598, 2622, 2572, 2718, 2610, 2690] combine_rows_per_src=[5546, 5510, 5520, 5494, 5676, 5538, 5554, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44524 payload_rows_remote=38868 dup_rows=21012 dup_rows_remote=18308 dedup_ratio=0.320618 router_tokens_per_expert=[487, 483, 488, 513, 508, 531, 563, 480, 528, 518, 507, 534, 500, 485, 501, 512, 519, 496, 500, 522, 505, 531, 546, 502, 502, 489, 517, 508, 561, 495, 533, 512, 478, 532, 498, 488, 509, 494, 526, 547, 554, 484, 541, 512, 523, 502, 495, 555, 540, 539, 500, 535, 505, 520, 528, 513, 502, 520, 492, 516, 498, 516, 445, 485] payload_rows_per_dst=[5562, 5580, 5630, 5572, 5506, 5570, 5656, 5448] dup_rows_per_dst=[2544, 2590, 2612, 2662, 2638, 2762, 2704, 2500] combine_rows_per_src=[5562, 5580, 5630, 5572, 5506, 5570, 5656, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278634496 dispatch_meta_bytes=229328 combine_payload_bytes=278634496 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278863824 ingress_bytes=388734976 link_bytes=388734976 token_bytes=557498320 weight_movement_bytes=110100480 total_nvlink_bytes=667598800 weight_to_token_ratio=0.197490 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.760740592e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44590 payload_rows_remote=38940 dup_rows=20946 dup_rows_remote=18318 dedup_ratio=0.319611 router_tokens_per_expert=[493, 482, 517, 461, 466, 501, 513, 503, 522, 490, 509, 505, 509, 536, 528, 513, 533, 504, 552, 492, 522, 496, 517, 521, 492, 504, 516, 502, 529, 511, 488, 515, 501, 513, 513, 521, 482, 530, 493, 504, 515, 509, 551, 508, 515, 540, 499, 544, 550, 501, 550, 513, 513, 495, 511, 516, 513, 526, 504, 558, 522, 529, 495, 492] payload_rows_per_dst=[5404, 5638, 5592, 5564, 5558, 5572, 5612, 5650] dup_rows_per_dst=[2468, 2586, 2682, 2550, 2556, 2790, 2686, 2628] combine_rows_per_src=[5404, 5638, 5592, 5564, 5558, 5572, 5612, 5650] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44544 payload_rows_remote=39004 dup_rows=20992 dup_rows_remote=18498 dedup_ratio=0.320312 router_tokens_per_expert=[523, 507, 560, 477, 526, 493, 522, 521, 540, 523, 538, 494, 497, 544, 532, 500, 511, 493, 503, 492, 522, 526, 482, 486, 510, 497, 512, 500, 517, 492, 480, 509, 544, 482, 488, 517, 537, 527, 552, 512, 509, 508, 473, 499, 519, 511, 541, 546, 512, 523, 515, 493, 524, 494, 545, 540, 480, 515, 494, 516, 506, 496, 498, 523] payload_rows_per_dst=[5620, 5612, 5494, 5540, 5570, 5584, 5650, 5474] dup_rows_per_dst=[2638, 2724, 2536, 2494, 2748, 2628, 2642, 2582] combine_rows_per_src=[5620, 5612, 5494, 5540, 5570, 5584, 5650, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278562816 dispatch_meta_bytes=229232 combine_payload_bytes=278562816 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278792048 ingress_bytes=388663296 link_bytes=388663296 token_bytes=557354864 weight_movement_bytes=110100480 total_nvlink_bytes=667455344 weight_to_token_ratio=0.197541 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.759493959e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280168448 dispatch_meta_bytes=229648 combine_payload_bytes=280168448 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280398096 ingress_bytes=390268928 link_bytes=390268928 token_bytes=560566544 weight_movement_bytes=110100480 total_nvlink_bytes=670667024 weight_to_token_ratio=0.196409 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.787418540e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278691840 dispatch_meta_bytes=229120 combine_payload_bytes=278691840 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278920960 ingress_bytes=608993280 link_bytes=608993280 token_bytes=557612800 weight_movement_bytes=330301440 total_nvlink_bytes=887914240 weight_to_token_ratio=0.592349 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059139476e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44428 payload_rows_remote=38902 dup_rows=21108 dup_rows_remote=18402 dedup_ratio=0.322083 router_tokens_per_expert=[511, 528, 502, 545, 507, 527, 500, 502, 537, 499, 550, 521, 511, 508, 489, 491, 487, 519, 533, 517, 482, 516, 529, 482, 491, 536, 488, 495, 525, 507, 537, 478, 505, 533, 500, 537, 521, 494, 497, 531, 512, 540, 508, 526, 504, 524, 479, 523, 520, 486, 539, 491, 490, 517, 541, 491, 521, 509, 495, 513, 514, 506, 529, 522] payload_rows_per_dst=[5568, 5542, 5528, 5494, 5664, 5526, 5544, 5562] dup_rows_per_dst=[2676, 2670, 2602, 2620, 2572, 2706, 2606, 2656] combine_rows_per_src=[5568, 5542, 5528, 5494, 5664, 5526, 5544, 5562] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44486 payload_rows_remote=38854 dup_rows=21050 dup_rows_remote=18436 dedup_ratio=0.321198 router_tokens_per_expert=[527, 512, 527, 538, 509, 512, 522, 503, 484, 507, 480, 480, 488, 545, 524, 517, 514, 503, 487, 503, 536, 523, 532, 525, 494, 537, 528, 540, 506, 522, 483, 495, 542, 498, 496, 488, 491, 507, 503, 488, 467, 497, 534, 516, 490, 550, 515, 508, 522, 513, 471, 495, 525, 508, 520, 524, 519, 539, 512, 526, 526, 539, 533, 503] payload_rows_per_dst=[5570, 5520, 5632, 5542, 5504, 5530, 5542, 5646] dup_rows_per_dst=[2730, 2530, 2614, 2668, 2522, 2624, 2614, 2748] combine_rows_per_src=[5570, 5520, 5632, 5542, 5504, 5530, 5542, 5646] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44570 payload_rows_remote=38816 dup_rows=20966 dup_rows_remote=18154 dedup_ratio=0.319916 router_tokens_per_expert=[496, 487, 519, 463, 466, 502, 514, 503, 526, 493, 513, 508, 510, 537, 527, 514, 535, 509, 554, 495, 521, 494, 516, 519, 492, 506, 517, 502, 529, 511, 487, 516, 503, 514, 513, 522, 480, 528, 492, 503, 514, 510, 550, 506, 514, 537, 494, 544, 550, 501, 548, 509, 512, 495, 510, 513, 512, 524, 503, 555, 520, 528, 495, 488] payload_rows_per_dst=[5414, 5532, 5590, 5566, 5640, 5596, 5478, 5754] dup_rows_per_dst=[2486, 2444, 2688, 2554, 2692, 2800, 2490, 2812] combine_rows_per_src=[5414, 5532, 5590, 5566, 5640, 5596, 5478, 5754] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279566336 dispatch_meta_bytes=229960 combine_payload_bytes=279566336 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279796296 ingress_bytes=389666816 link_bytes=389666816 token_bytes=559362632 weight_movement_bytes=110100480 total_nvlink_bytes=669463112 weight_to_token_ratio=0.196832 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.051471 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.776946822e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44518 payload_rows_remote=38862 dup_rows=21018 dup_rows_remote=18336 dedup_ratio=0.320709 router_tokens_per_expert=[491, 487, 492, 518, 511, 533, 565, 484, 530, 519, 509, 538, 505, 487, 505, 516, 521, 496, 498, 521, 509, 532, 544, 502, 501, 488, 517, 507, 561, 495, 534, 510, 477, 529, 497, 489, 506, 492, 526, 546, 554, 483, 537, 510, 522, 501, 495, 553, 540, 537, 498, 533, 504, 518, 528, 511, 500, 519, 491, 513, 497, 515, 443, 478] payload_rows_per_dst=[5580, 5594, 5634, 5564, 5500, 5562, 5656, 5428] dup_rows_per_dst=[2582, 2624, 2612, 2662, 2624, 2748, 2682, 2484] combine_rows_per_src=[5580, 5594, 5634, 5564, 5500, 5562, 5656, 5428] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44446 payload_rows_remote=38846 dup_rows=21090 dup_rows_remote=18454 dedup_ratio=0.321808 router_tokens_per_expert=[512, 537, 512, 513, 487, 500, 508, 485, 545, 525, 493, 506, 490, 541, 488, 530, 486, 509, 491, 539, 495, 528, 511, 547, 543, 488, 516, 541, 481, 499, 524, 489, 525, 527, 524, 520, 516, 513, 516, 529, 495, 550, 523, 466, 488, 480, 538, 492, 526, 552, 515, 511, 498, 526, 524, 500, 517, 478, 503, 491, 481, 516, 529, 540] payload_rows_per_dst=[5502, 5600, 5568, 5616, 5638, 5408, 5596, 5518] dup_rows_per_dst=[2606, 2636, 2644, 2546, 2702, 2656, 2708, 2592] combine_rows_per_src=[5502, 5600, 5568, 5616, 5638, 5408, 5596, 5518] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44520 payload_rows_remote=38862 dup_rows=21016 dup_rows_remote=18306 dedup_ratio=0.320679 router_tokens_per_expert=[517, 531, 533, 536, 492, 485, 522, 510, 531, 508, 504, 519, 524, 506, 557, 506, 498, 513, 514, 535, 521, 523, 506, 488, 545, 506, 525, 497, 513, 514, 504, 518, 514, 482, 516, 479, 501, 514, 508, 508, 512, 485, 499, 518, 490, 493, 521, 546, 502, 520, 541, 485, 492, 497, 527, 535, 523, 508, 505, 534, 489, 498, 518, 507] payload_rows_per_dst=[5658, 5586, 5616, 5588, 5508, 5476, 5548, 5540] dup_rows_per_dst=[2710, 2746, 2558, 2836, 2536, 2472, 2534, 2624] combine_rows_per_src=[5658, 5586, 5616, 5588, 5508, 5476, 5548, 5540] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44416 payload_rows_remote=38846 dup_rows=21120 dup_rows_remote=18550 dedup_ratio=0.322266 router_tokens_per_expert=[530, 536, 515, 503, 519, 541, 490, 507, 496, 543, 502, 481, 517, 517, 494, 519, 522, 559, 486, 500, 498, 512, 506, 547, 524, 482, 496, 522, 515, 505, 512, 507, 490, 530, 523, 538, 494, 469, 508, 518, 554, 536, 480, 496, 526, 525, 532, 537, 478, 535, 497, 529, 506, 500, 509, 522, 480, 520, 523, 525, 501, 487, 505, 492] payload_rows_per_dst=[5652, 5508, 5546, 5544, 5570, 5602, 5500, 5494] dup_rows_per_dst=[2630, 2630, 2714, 2582, 2570, 2770, 2652, 2572] combine_rows_per_src=[5652, 5508, 5546, 5544, 5570, 5602, 5500, 5494] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278849536 dispatch_meta_bytes=229216 combine_payload_bytes=278849536 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279078752 ingress_bytes=499050496 link_bytes=499050496 token_bytes=557928288 weight_movement_bytes=220200960 total_nvlink_bytes=778129248 weight_to_token_ratio=0.394676 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.679308922e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278505472 dispatch_meta_bytes=229160 combine_payload_bytes=278505472 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=278734632 ingress_bytes=498706432 link_bytes=498706432 token_bytes=557240104 weight_movement_bytes=220200960 total_nvlink_bytes=777441064 weight_to_token_ratio=0.395164 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.673325084e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44492 payload_rows_remote=38730 dup_rows=21044 dup_rows_remote=18154 dedup_ratio=0.321106 router_tokens_per_expert=[508, 530, 529, 533, 489, 482, 517, 508, 530, 507, 498, 520, 522, 507, 556, 500, 497, 508, 514, 535, 520, 523, 506, 489, 543, 505, 521, 495, 512, 513, 504, 518, 513, 483, 518, 478, 501, 514, 508, 510, 513, 486, 500, 518, 490, 495, 522, 550, 506, 523, 541, 487, 493, 499, 530, 540, 526, 506, 506, 541, 493, 505, 524, 510] payload_rows_per_dst=[5762, 5574, 5658, 5592, 5590, 5446, 5312, 5558] dup_rows_per_dst=[2890, 2720, 2628, 2842, 2644, 2424, 2274, 2622] combine_rows_per_src=[5762, 5574, 5658, 5592, 5590, 5446, 5312, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278605824 dispatch_meta_bytes=228704 combine_payload_bytes=278605824 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=278834528 ingress_bytes=719007744 link_bytes=719007744 token_bytes=557440352 weight_movement_bytes=440401920 total_nvlink_bytes=997842272 weight_to_token_ratio=0.790043 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.250472723e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279121920 dispatch_meta_bytes=229032 combine_payload_bytes=279121920 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279350952 ingress_bytes=499322880 link_bytes=499322880 token_bytes=558472872 weight_movement_bytes=220200960 total_nvlink_bytes=778673832 weight_to_token_ratio=0.394291 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.684046128e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279580672 dispatch_meta_bytes=230008 combine_payload_bytes=279580672 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279810680 ingress_bytes=389681152 link_bytes=389681152 token_bytes=559391352 weight_movement_bytes=110100480 total_nvlink_bytes=669491832 weight_to_token_ratio=0.196822 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.051471 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.777196149e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278233088 dispatch_meta_bytes=227880 combine_payload_bytes=278233088 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278460968 ingress_bytes=388333568 link_bytes=388333568 token_bytes=556694056 weight_movement_bytes=110100480 total_nvlink_bytes=666794536 weight_to_token_ratio=0.197776 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.753759446e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278562816 dispatch_meta_bytes=228792 combine_payload_bytes=278562816 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278791608 ingress_bytes=608864256 link_bytes=608864256 token_bytes=557354424 weight_movement_bytes=330301440 total_nvlink_bytes=887655864 weight_to_token_ratio=0.592624 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058915082e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278448128 dispatch_meta_bytes=229200 combine_payload_bytes=278448128 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278677328 ingress_bytes=608749568 link_bytes=608749568 token_bytes=557125456 weight_movement_bytes=330301440 total_nvlink_bytes=887426896 weight_to_token_ratio=0.592867 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058715621e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278562816 dispatch_meta_bytes=228672 combine_payload_bytes=278562816 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278791488 ingress_bytes=388663296 link_bytes=388663296 token_bytes=557354304 weight_movement_bytes=110100480 total_nvlink_bytes=667454784 weight_to_token_ratio=0.197541 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.759493959e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278448128 dispatch_meta_bytes=229584 combine_payload_bytes=278448128 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278677712 ingress_bytes=388548608 link_bytes=388548608 token_bytes=557125840 weight_movement_bytes=110100480 total_nvlink_bytes=667226320 weight_to_token_ratio=0.197622 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.757499346e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44410 payload_rows_remote=38784 dup_rows=21126 dup_rows_remote=18354 dedup_ratio=0.322357 router_tokens_per_expert=[518, 503, 488, 538, 521, 512, 491, 469, 507, 520, 458, 470, 532, 530, 547, 507, 475, 469, 543, 514, 534, 534, 499, 548, 527, 511, 506, 544, 533, 512, 513, 553, 507, 535, 509, 510, 536, 506, 516, 512, 531, 499, 510, 449, 512, 530, 519, 499, 509, 484, 503, 469, 480, 471, 542, 506, 528, 525, 515, 512, 566, 518, 526, 508] payload_rows_per_dst=[5542, 5512, 5578, 5626, 5614, 5520, 5392, 5626] dup_rows_per_dst=[2538, 2630, 2654, 2772, 2648, 2578, 2536, 2770] combine_rows_per_src=[5542, 5512, 5578, 5626, 5614, 5520, 5392, 5626] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44512 payload_rows_remote=38868 dup_rows=21024 dup_rows_remote=18454 dedup_ratio=0.320801 router_tokens_per_expert=[549, 501, 514, 506, 497, 503, 532, 512, 507, 536, 521, 514, 541, 526, 500, 502, 548, 491, 546, 535, 532, 501, 505, 506, 504, 538, 502, 518, 500, 500, 521, 546, 541, 507, 510, 498, 517, 531, 517, 502, 535, 495, 517, 529, 513, 466, 550, 502, 467, 488, 497, 497, 500, 509, 456, 487, 518, 537, 512, 501, 494, 513, 507, 501] payload_rows_per_dst=[5560, 5610, 5594, 5628, 5570, 5644, 5404, 5502] dup_rows_per_dst=[2668, 2684, 2734, 2630, 2676, 2570, 2398, 2664] combine_rows_per_src=[5560, 5610, 5594, 5628, 5570, 5644, 5404, 5502] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44502 payload_rows_remote=38972 dup_rows=21034 dup_rows_remote=18522 dedup_ratio=0.320953 router_tokens_per_expert=[473, 514, 530, 514, 518, 511, 496, 533, 541, 511, 504, 506, 551, 563, 508, 510, 504, 482, 507, 520, 495, 501, 504, 508, 492, 520, 501, 499, 545, 493, 510, 511, 515, 487, 511, 535, 481, 516, 496, 478, 470, 527, 504, 528, 568, 529, 552, 497, 509, 525, 503, 527, 529, 490, 546, 522, 523, 508, 517, 519, 517, 477, 470, 517] payload_rows_per_dst=[5518, 5648, 5530, 5500, 5466, 5634, 5640, 5566] dup_rows_per_dst=[2660, 2740, 2512, 2642, 2572, 2716, 2662, 2530] combine_rows_per_src=[5518, 5648, 5530, 5500, 5466, 5634, 5640, 5566] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278003712 dispatch_meta_bytes=228552 combine_payload_bytes=278003712 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278232264 ingress_bytes=498204672 link_bytes=498204672 token_bytes=556235976 weight_movement_bytes=220200960 total_nvlink_bytes=776436936 weight_to_token_ratio=0.395877 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.664598652e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44350 payload_rows_remote=38704 dup_rows=21186 dup_rows_remote=18456 dedup_ratio=0.323273 router_tokens_per_expert=[493, 511, 537, 541, 519, 528, 525, 522, 489, 504, 517, 541, 476, 522, 515, 528, 522, 501, 508, 489, 523, 508, 553, 491, 490, 523, 546, 525, 502, 537, 532, 508, 523, 510, 504, 522, 527, 521, 503, 505, 533, 495, 483, 470, 538, 506, 518, 531, 520, 515, 502, 520, 521, 495, 486, 495, 512, 514, 513, 511, 469, 466, 491, 523] payload_rows_per_dst=[5478, 5344, 5522, 5430, 5690, 5396, 5844, 5646] dup_rows_per_dst=[2556, 2480, 2636, 2438, 2858, 2406, 3082, 2730] combine_rows_per_src=[5478, 5344, 5522, 5430, 5690, 5396, 5844, 5646] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44520 payload_rows_remote=39064 dup_rows=21016 dup_rows_remote=18530 dedup_ratio=0.320679 router_tokens_per_expert=[477, 515, 507, 484, 528, 528, 533, 523, 547, 519, 497, 509, 519, 492, 522, 553, 520, 514, 498, 502, 515, 547, 530, 476, 493, 520, 538, 530, 502, 505, 532, 504, 467, 542, 513, 556, 542, 489, 505, 497, 496, 534, 554, 516, 511, 528, 519, 512, 490, 522, 459, 529, 517, 479, 492, 483, 537, 496, 499, 516, 495, 488, 518, 488] payload_rows_per_dst=[5566, 5608, 5502, 5598, 5598, 5610, 5456, 5582] dup_rows_per_dst=[2624, 2708, 2702, 2650, 2624, 2730, 2486, 2492] combine_rows_per_src=[5566, 5608, 5502, 5598, 5598, 5610, 5456, 5582] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44752 payload_rows_remote=39102 dup_rows=20784 dup_rows_remote=18120 dedup_ratio=0.317139 router_tokens_per_expert=[516, 480, 519, 535, 459, 528, 531, 477, 516, 507, 517, 503, 510, 535, 534, 535, 464, 533, 526, 519, 486, 497, 513, 534, 483, 535, 497, 528, 511, 506, 518, 502, 493, 482, 508, 507, 509, 536, 503, 481, 506, 524, 524, 503, 463, 513, 489, 529, 523, 507, 508, 522, 547, 544, 522, 498, 533, 543, 527, 486, 507, 518, 532, 527] payload_rows_per_dst=[5524, 5650, 5558, 5584, 5556, 5566, 5622, 5692] dup_rows_per_dst=[2566, 2664, 2586, 2576, 2482, 2536, 2720, 2654] combine_rows_per_src=[5524, 5650, 5558, 5584, 5556, 5566, 5622, 5692] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44488 payload_rows_remote=38800 dup_rows=21048 dup_rows_remote=18282 dedup_ratio=0.321167 router_tokens_per_expert=[522, 497, 539, 501, 521, 533, 490, 544, 493, 502, 501, 524, 474, 517, 538, 543, 534, 483, 479, 523, 501, 515, 504, 545, 527, 477, 558, 528, 531, 539, 512, 496, 515, 523, 502, 482, 502, 503, 508, 486, 529, 476, 538, 520, 519, 552, 525, 518, 494, 524, 501, 500, 518, 485, 540, 509, 456, 526, 504, 501, 478, 486, 508, 549] payload_rows_per_dst=[5688, 5516, 5554, 5632, 5536, 5516, 5564, 5482] dup_rows_per_dst=[2766, 2604, 2614, 2704, 2506, 2742, 2578, 2534] combine_rows_per_src=[5688, 5516, 5554, 5632, 5536, 5516, 5564, 5482] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44522 payload_rows_remote=38910 dup_rows=21014 dup_rows_remote=18338 dedup_ratio=0.320648 router_tokens_per_expert=[499, 541, 492, 521, 501, 499, 492, 532, 508, 493, 502, 526, 521, 519, 546, 509, 542, 524, 503, 543, 503, 506, 503, 530, 497, 453, 499, 510, 515, 530, 524, 503, 509, 519, 531, 548, 498, 522, 502, 515, 528, 527, 526, 515, 539, 526, 504, 513, 491, 513, 515, 464, 531, 549, 491, 511, 506, 507, 500, 479, 519, 477, 497, 510] payload_rows_per_dst=[5574, 5542, 5590, 5518, 5612, 5622, 5556, 5508] dup_rows_per_dst=[2580, 2706, 2718, 2544, 2676, 2734, 2574, 2482] combine_rows_per_src=[5574, 5542, 5590, 5518, 5612, 5622, 5556, 5508] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279351296 dispatch_meta_bytes=229976 combine_payload_bytes=279351296 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=279581272 ingress_bytes=389451776 link_bytes=389451776 token_bytes=558932568 weight_movement_bytes=110100480 total_nvlink_bytes=669033048 weight_to_token_ratio=0.196983 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773206923e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277616640 dispatch_meta_bytes=227536 combine_payload_bytes=277616640 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277844176 ingress_bytes=387717120 link_bytes=387717120 token_bytes=555460816 weight_movement_bytes=110100480 total_nvlink_bytes=665561296 weight_to_token_ratio=0.198215 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.743038402e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44758 payload_rows_remote=39118 dup_rows=20778 dup_rows_remote=18140 dedup_ratio=0.317047 router_tokens_per_expert=[513, 477, 516, 533, 456, 526, 531, 473, 513, 505, 517, 501, 507, 534, 530, 532, 462, 533, 525, 515, 486, 499, 511, 532, 482, 535, 497, 526, 507, 506, 518, 501, 492, 481, 510, 510, 511, 534, 503, 480, 507, 524, 526, 506, 466, 517, 490, 531, 525, 509, 510, 522, 549, 548, 524, 500, 533, 544, 532, 488, 510, 518, 538, 531] payload_rows_per_dst=[5512, 5640, 5550, 5586, 5558, 5578, 5630, 5704] dup_rows_per_dst=[2538, 2638, 2576, 2558, 2484, 2556, 2744, 2684] combine_rows_per_src=[5512, 5640, 5550, 5586, 5558, 5578, 5630, 5704] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44442 payload_rows_remote=38866 dup_rows=21094 dup_rows_remote=18528 dedup_ratio=0.321869 router_tokens_per_expert=[527, 531, 509, 502, 518, 535, 486, 504, 496, 540, 498, 481, 518, 514, 494, 520, 521, 558, 486, 500, 497, 511, 506, 546, 523, 481, 495, 520, 517, 505, 515, 506, 490, 531, 522, 537, 495, 471, 510, 515, 552, 537, 479, 500, 526, 526, 535, 539, 479, 536, 498, 530, 511, 500, 509, 527, 483, 522, 527, 527, 502, 492, 507, 493] payload_rows_per_dst=[5640, 5498, 5542, 5542, 5576, 5614, 5514, 5516] dup_rows_per_dst=[2584, 2624, 2708, 2582, 2566, 2774, 2666, 2590] combine_rows_per_src=[5640, 5498, 5542, 5542, 5576, 5614, 5514, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44510 payload_rows_remote=38990 dup_rows=21026 dup_rows_remote=18534 dedup_ratio=0.320831 router_tokens_per_expert=[471, 510, 526, 512, 517, 510, 492, 534, 537, 507, 501, 505, 546, 559, 509, 505, 504, 483, 505, 517, 493, 496, 500, 508, 495, 520, 497, 500, 543, 495, 511, 511, 517, 488, 513, 536, 483, 516, 496, 478, 469, 528, 505, 531, 569, 527, 552, 499, 512, 528, 504, 527, 529, 491, 548, 526, 526, 512, 522, 523, 518, 480, 475, 521] payload_rows_per_dst=[5506, 5634, 5520, 5500, 5474, 5640, 5654, 5582] dup_rows_per_dst=[2638, 2704, 2492, 2644, 2580, 2720, 2676, 2572] combine_rows_per_src=[5506, 5634, 5520, 5500, 5474, 5640, 5654, 5582] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278605824 dispatch_meta_bytes=229288 combine_payload_bytes=278605824 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278835112 ingress_bytes=608907264 link_bytes=608907264 token_bytes=557440936 weight_movement_bytes=330301440 total_nvlink_bytes=887742376 weight_to_token_ratio=0.592532 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058989880e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277430272 dispatch_meta_bytes=228640 combine_payload_bytes=277430272 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277658912 ingress_bytes=387530752 link_bytes=387530752 token_bytes=555089184 weight_movement_bytes=110100480 total_nvlink_bytes=665189664 weight_to_token_ratio=0.198347 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.739797156e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280010752 dispatch_meta_bytes=230376 combine_payload_bytes=280010752 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280241128 ingress_bytes=390111232 link_bytes=390111232 token_bytes=560251880 weight_movement_bytes=110100480 total_nvlink_bytes=670352360 weight_to_token_ratio=0.196520 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.784675947e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280283136 dispatch_meta_bytes=228888 combine_payload_bytes=280283136 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280512024 ingress_bytes=610584576 link_bytes=610584576 token_bytes=560795160 weight_movement_bytes=330301440 total_nvlink_bytes=891096600 weight_to_token_ratio=0.588988 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061907002e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278118400 dispatch_meta_bytes=228328 combine_payload_bytes=278118400 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278346728 ingress_bytes=498319360 link_bytes=498319360 token_bytes=556465128 weight_movement_bytes=220200960 total_nvlink_bytes=776666088 weight_to_token_ratio=0.395714 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.666593265e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278906880 dispatch_meta_bytes=228992 combine_payload_bytes=278906880 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279135872 ingress_bytes=389007360 link_bytes=389007360 token_bytes=558042752 weight_movement_bytes=110100480 total_nvlink_bytes=668143232 weight_to_token_ratio=0.197298 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.765477798e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44428 payload_rows_remote=38804 dup_rows=21108 dup_rows_remote=18436 dedup_ratio=0.322083 router_tokens_per_expert=[546, 519, 503, 513, 511, 492, 546, 524, 504, 558, 533, 498, 511, 513, 505, 489, 526, 505, 498, 517, 486, 493, 511, 540, 500, 500, 523, 504, 529, 520, 527, 545, 481, 491, 483, 510, 497, 520, 575, 498, 498, 505, 517, 470, 511, 501, 547, 493, 503, 530, 560, 508, 560, 486, 504, 509, 510, 508, 502, 488, 493, 530, 511, 480] payload_rows_per_dst=[5606, 5512, 5562, 5624, 5544, 5438, 5674, 5468] dup_rows_per_dst=[2702, 2710, 2590, 2672, 2566, 2646, 2646, 2576] combine_rows_per_src=[5606, 5512, 5562, 5624, 5544, 5438, 5674, 5468] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278147072 dispatch_meta_bytes=228960 combine_payload_bytes=278147072 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278376032 ingress_bytes=498348032 link_bytes=498348032 token_bytes=556523104 weight_movement_bytes=220200960 total_nvlink_bytes=776724064 weight_to_token_ratio=0.395673 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.667091918e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44524 payload_rows_remote=38952 dup_rows=21012 dup_rows_remote=18448 dedup_ratio=0.320618 router_tokens_per_expert=[485, 522, 514, 502, 540, 509, 547, 460, 534, 497, 543, 501, 525, 516, 521, 507, 506, 489, 549, 513, 521, 536, 500, 454, 498, 523, 518, 502, 491, 512, 524, 517, 498, 531, 516, 540, 495, 489, 476, 521, 541, 542, 527, 489, 503, 517, 496, 518, 469, 501, 505, 530, 536, 541, 489, 540, 533, 509, 492, 498, 521, 514, 534, 481] payload_rows_per_dst=[5508, 5652, 5572, 5562, 5558, 5590, 5520, 5562] dup_rows_per_dst=[2650, 2636, 2564, 2608, 2574, 2676, 2702, 2602] combine_rows_per_src=[5508, 5652, 5572, 5562, 5558, 5590, 5520, 5562] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44588 payload_rows_remote=39044 dup_rows=20948 dup_rows_remote=18350 dedup_ratio=0.319641 router_tokens_per_expert=[541, 475, 487, 521, 495, 512, 538, 523, 550, 507, 510, 503, 537, 521, 514, 490, 532, 515, 495, 521, 522, 527, 495, 506, 522, 520, 517, 540, 510, 532, 468, 563, 502, 535, 501, 497, 467, 539, 524, 532, 492, 507, 518, 500, 504, 508, 530, 512, 509, 510, 498, 479, 523, 481, 504, 518, 496, 484, 527, 531, 482, 513, 529, 507] payload_rows_per_dst=[5572, 5624, 5610, 5616, 5582, 5544, 5446, 5594] dup_rows_per_dst=[2612, 2640, 2616, 2728, 2612, 2598, 2598, 2544] combine_rows_per_src=[5572, 5624, 5610, 5616, 5582, 5544, 5446, 5594] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44336 payload_rows_remote=38594 dup_rows=21200 dup_rows_remote=18018 dedup_ratio=0.323486 router_tokens_per_expert=[486, 487, 503, 527, 503, 507, 493, 518, 534, 463, 514, 490, 470, 501, 535, 502, 516, 509, 541, 512, 544, 482, 503, 533, 532, 548, 508, 535, 485, 541, 522, 522, 474, 547, 511, 538, 543, 490, 511, 517, 493, 522, 516, 483, 515, 524, 496, 478, 554, 494, 502, 496, 508, 517, 511, 531, 529, 489, 509, 512, 518, 543, 521, 510] payload_rows_per_dst=[5450, 5222, 5596, 5642, 5600, 5534, 5550, 5742] dup_rows_per_dst=[2436, 2134, 2770, 2744, 2662, 2596, 2676, 3182] combine_rows_per_src=[5450, 5222, 5596, 5642, 5600, 5534, 5550, 5742] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44614 payload_rows_remote=38954 dup_rows=20922 dup_rows_remote=18172 dedup_ratio=0.319244 router_tokens_per_expert=[485, 528, 500, 490, 520, 529, 504, 554, 506, 518, 500, 525, 484, 507, 516, 523, 497, 503, 542, 528, 535, 529, 503, 496, 501, 495, 474, 501, 525, 518, 519, 525, 487, 478, 510, 510, 537, 471, 512, 520, 559, 513, 510, 523, 455, 531, 507, 503, 529, 539, 541, 531, 527, 502, 534, 502, 518, 535, 458, 493, 494, 494, 519, 546] payload_rows_per_dst=[5604, 5536, 5674, 5526, 5550, 5588, 5660, 5476] dup_rows_per_dst=[2616, 2622, 2592, 2590, 2500, 2614, 2750, 2638] combine_rows_per_src=[5604, 5536, 5674, 5526, 5550, 5588, 5660, 5476] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44214 payload_rows_remote=38614 dup_rows=21322 dup_rows_remote=18596 dedup_ratio=0.325348 router_tokens_per_expert=[514, 520, 508, 513, 509, 537, 512, 511, 550, 512, 538, 525, 524, 488, 515, 511, 544, 506, 521, 511, 520, 544, 510, 508, 507, 508, 499, 514, 488, 493, 527, 503, 477, 483, 529, 519, 515, 498, 555, 481, 498, 504, 545, 531, 512, 497, 516, 508, 526, 487, 513, 441, 527, 504, 534, 510, 506, 507, 526, 494, 499, 533, 492, 511] payload_rows_per_dst=[5538, 5600, 5566, 5482, 5486, 5572, 5464, 5506] dup_rows_per_dst=[2710, 2726, 2762, 2596, 2628, 2650, 2620, 2630] combine_rows_per_src=[5538, 5600, 5566, 5482, 5486, 5572, 5464, 5506] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44526 payload_rows_remote=38880 dup_rows=21010 dup_rows_remote=18278 dedup_ratio=0.320587 router_tokens_per_expert=[514, 516, 532, 498, 526, 540, 533, 513, 526, 545, 504, 551, 484, 536, 519, 493, 500, 513, 496, 531, 525, 480, 504, 507, 539, 512, 507, 484, 516, 488, 501, 509, 497, 510, 527, 517, 528, 543, 468, 522, 507, 535, 498, 507, 498, 510, 522, 490, 539, 512, 517, 539, 500, 484, 503, 540, 483, 521, 516, 507, 514, 495, 510, 467] payload_rows_per_dst=[5646, 5754, 5406, 5550, 5610, 5526, 5488, 5546] dup_rows_per_dst=[2732, 2920, 2348, 2534, 2776, 2608, 2612, 2480] combine_rows_per_src=[5646, 5754, 5406, 5550, 5610, 5526, 5488, 5546] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44516 payload_rows_remote=38972 dup_rows=21020 dup_rows_remote=18366 dedup_ratio=0.320740 router_tokens_per_expert=[532, 550, 517, 471, 518, 502, 514, 500, 511, 490, 534, 525, 504, 498, 499, 524, 496, 513, 492, 496, 518, 561, 501, 492, 527, 515, 512, 509, 489, 522, 527, 514, 466, 522, 538, 557, 517, 558, 474, 467, 539, 510, 542, 515, 514, 482, 525, 510, 496, 510, 529, 566, 495, 527, 513, 488, 493, 509, 478, 503, 527, 500, 538, 487] payload_rows_per_dst=[5512, 5534, 5544, 5600, 5544, 5644, 5582, 5556] dup_rows_per_dst=[2696, 2636, 2594, 2630, 2654, 2630, 2666, 2514] combine_rows_per_src=[5512, 5534, 5544, 5600, 5544, 5644, 5582, 5556] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279207936 dispatch_meta_bytes=229600 combine_payload_bytes=279207936 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279437536 ingress_bytes=499408896 link_bytes=499408896 token_bytes=558645472 weight_movement_bytes=220200960 total_nvlink_bytes=778846432 weight_to_token_ratio=0.394169 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.685542088e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279867392 dispatch_meta_bytes=229576 combine_payload_bytes=279867392 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280096968 ingress_bytes=389967872 link_bytes=389967872 token_bytes=559964360 weight_movement_bytes=110100480 total_nvlink_bytes=670064840 weight_to_token_ratio=0.196621 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.782182681e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=276641792 dispatch_meta_bytes=226448 combine_payload_bytes=276641792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276868240 ingress_bytes=386742272 link_bytes=386742272 token_bytes=553510032 weight_movement_bytes=110100480 total_nvlink_bytes=663610512 weight_to_token_ratio=0.198913 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.726084192e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279222272 dispatch_meta_bytes=228504 combine_payload_bytes=279222272 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279450776 ingress_bytes=499423232 link_bytes=499423232 token_bytes=558673048 weight_movement_bytes=220200960 total_nvlink_bytes=778874008 weight_to_token_ratio=0.394150 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.685791415e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=276785152 dispatch_meta_bytes=228840 combine_payload_bytes=276785152 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277013992 ingress_bytes=496986112 link_bytes=496986112 token_bytes=553799144 weight_movement_bytes=220200960 total_nvlink_bytes=774000104 weight_to_token_ratio=0.397619 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.643405889e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44530 payload_rows_remote=38900 dup_rows=21006 dup_rows_remote=18376 dedup_ratio=0.320526 router_tokens_per_expert=[531, 523, 526, 509, 557, 499, 494, 561, 486, 525, 523, 555, 529, 503, 515, 511, 525, 498, 474, 483, 483, 488, 505, 505, 493, 544, 516, 516, 531, 491, 529, 510, 501, 506, 493, 504, 491, 501, 496, 532, 503, 492, 545, 516, 532, 533, 526, 538, 514, 506, 517, 496, 485, 481, 518, 533, 533, 502, 533, 509, 507, 495, 517, 475] payload_rows_per_dst=[5606, 5562, 5426, 5630, 5568, 5618, 5588, 5532] dup_rows_per_dst=[2794, 2732, 2496, 2630, 2480, 2752, 2512, 2610] combine_rows_per_src=[5606, 5562, 5426, 5630, 5568, 5618, 5588, 5532] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278691840 dispatch_meta_bytes=228632 combine_payload_bytes=278691840 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278920472 ingress_bytes=608993280 link_bytes=608993280 token_bytes=557612312 weight_movement_bytes=330301440 total_nvlink_bytes=887913752 weight_to_token_ratio=0.592350 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059139476e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279351296 dispatch_meta_bytes=229352 combine_payload_bytes=279351296 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279580648 ingress_bytes=389451776 link_bytes=389451776 token_bytes=558931944 weight_movement_bytes=110100480 total_nvlink_bytes=669032424 weight_to_token_ratio=0.196984 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773206923e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278835200 dispatch_meta_bytes=229104 combine_payload_bytes=278835200 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279064304 ingress_bytes=609136640 link_bytes=609136640 token_bytes=557899504 weight_movement_bytes=330301440 total_nvlink_bytes=888200944 weight_to_token_ratio=0.592045 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059388803e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5582810600 prefetch_bytes=880803840 reduce_bytes=440401920 weight_movement_bytes=1321205760 egress_bytes=2792551912 ingress_bytes=4111464448 link_bytes=4111464448 total_nvlink_bytes=6904016360 weight_to_token_ratio=0.236656 prefetch_share_of_ingress=0.214231 dedup_ratio=0.321304 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=4 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.150512895e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44374 payload_rows_remote=38810 dup_rows=21162 dup_rows_remote=18550 dedup_ratio=0.322906 router_tokens_per_expert=[529, 510, 493, 523, 512, 520, 529, 489, 503, 520, 499, 492, 508, 530, 509, 532, 518, 512, 522, 488, 502, 514, 514, 518, 486, 512, 523, 522, 527, 526, 543, 493, 511, 499, 508, 516, 553, 469, 500, 530, 469, 526, 502, 482, 512, 521, 530, 529, 475, 564, 539, 491, 506, 513, 518, 519, 525, 492, 494, 486, 525, 509, 502, 535] payload_rows_per_dst=[5508, 5504, 5564, 5656, 5568, 5548, 5538, 5488] dup_rows_per_dst=[2702, 2682, 2612, 2608, 2604, 2594, 2712, 2648] combine_rows_per_src=[5508, 5504, 5564, 5656, 5568, 5548, 5538, 5488] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44516 payload_rows_remote=39046 dup_rows=21020 dup_rows_remote=18384 dedup_ratio=0.320740 router_tokens_per_expert=[526, 531, 486, 590, 517, 512, 469, 487, 445, 489, 518, 532, 503, 498, 501, 517, 541, 518, 550, 500, 506, 515, 511, 496, 527, 478, 534, 508, 522, 514, 513, 508, 481, 557, 517, 510, 528, 508, 511, 520, 512, 520, 516, 511, 494, 512, 497, 491, 521, 529, 518, 553, 511, 513, 464, 543, 493, 477, 543, 488, 497, 509, 545, 517] payload_rows_per_dst=[5612, 5472, 5588, 5606, 5616, 5470, 5656, 5496] dup_rows_per_dst=[2624, 2534, 2686, 2602, 2648, 2636, 2648, 2642] combine_rows_per_src=[5612, 5472, 5588, 5606, 5616, 5470, 5656, 5496] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44474 payload_rows_remote=38954 dup_rows=21062 dup_rows_remote=18486 dedup_ratio=0.321381 router_tokens_per_expert=[487, 533, 530, 519, 524, 470, 531, 534, 498, 472, 559, 529, 507, 482, 527, 494, 524, 518, 534, 527, 496, 516, 529, 505, 507, 498, 524, 527, 488, 494, 489, 495, 507, 524, 525, 505, 520, 505, 516, 504, 564, 508, 474, 499, 528, 532, 537, 532, 503, 491, 512, 524, 510, 500, 476, 532, 467, 536, 478, 524, 503, 513, 504, 548] payload_rows_per_dst=[5596, 5530, 5606, 5500, 5576, 5604, 5520, 5542] dup_rows_per_dst=[2660, 2606, 2692, 2544, 2636, 2744, 2576, 2604] combine_rows_per_src=[5596, 5530, 5606, 5500, 5576, 5604, 5520, 5542] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44432 payload_rows_remote=38736 dup_rows=21104 dup_rows_remote=18258 dedup_ratio=0.322021 router_tokens_per_expert=[501, 525, 533, 507, 532, 497, 496, 518, 506, 510, 518, 536, 499, 526, 498, 513, 475, 515, 515, 523, 512, 529, 514, 512, 509, 497, 555, 525, 567, 544, 507, 494, 474, 478, 528, 518, 513, 505, 555, 469, 536, 543, 513, 558, 484, 473, 472, 490, 491, 543, 519, 547, 501, 510, 482, 501, 476, 507, 524, 547, 478, 479, 525, 521] payload_rows_per_dst=[5346, 5556, 5540, 5676, 5464, 5544, 5610, 5696] dup_rows_per_dst=[2290, 2768, 2650, 2622, 2654, 2608, 2666, 2846] combine_rows_per_src=[5346, 5556, 5540, 5676, 5464, 5544, 5610, 5696] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5577245768 prefetch_bytes=880803840 reduce_bytes=1321205760 weight_movement_bytes=2202009600 egress_bytes=2789768264 ingress_bytes=4989487104 link_bytes=4989487104 total_nvlink_bytes=7779255368 weight_to_token_ratio=0.394820 prefetch_share_of_ingress=0.176532 dedup_ratio=0.321039 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=12 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.677538703e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44458 payload_rows_remote=38970 dup_rows=21078 dup_rows_remote=18412 dedup_ratio=0.321625 router_tokens_per_expert=[511, 529, 530, 501, 526, 518, 541, 525, 504, 531, 510, 504, 497, 506, 540, 485, 542, 525, 480, 512, 503, 507, 517, 508, 510, 519, 500, 525, 512, 499, 470, 528, 524, 500, 522, 530, 474, 533, 475, 533, 481, 511, 521, 453, 499, 525, 530, 519, 534, 493, 531, 483, 515, 491, 532, 528, 516, 509, 533, 498, 512, 528, 503, 517] payload_rows_per_dst=[5662, 5488, 5624, 5504, 5594, 5496, 5586, 5504] dup_rows_per_dst=[2700, 2666, 2564, 2622, 2588, 2582, 2628, 2728] combine_rows_per_src=[5662, 5488, 5624, 5504, 5594, 5496, 5586, 5504] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44472 payload_rows_remote=38846 dup_rows=21064 dup_rows_remote=18462 dedup_ratio=0.321411 router_tokens_per_expert=[486, 527, 510, 533, 510, 515, 526, 507, 495, 514, 553, 537, 476, 501, 530, 527, 535, 491, 526, 494, 528, 476, 528, 498, 493, 539, 507, 486, 473, 510, 507, 538, 485, 531, 518, 547, 545, 496, 498, 551, 518, 532, 526, 536, 511, 510, 503, 496, 515, 490, 490, 485, 513, 525, 492, 514, 520, 486, 507, 505, 522, 532, 508, 485] payload_rows_per_dst=[5626, 5534, 5610, 5496, 5670, 5474, 5516, 5546] dup_rows_per_dst=[2602, 2732, 2740, 2610, 2672, 2592, 2532, 2584] combine_rows_per_src=[5626, 5534, 5610, 5496, 5670, 5474, 5516, 5546] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278190080 dispatch_meta_bytes=229440 combine_payload_bytes=278190080 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278419520 ingress_bytes=388290560 link_bytes=388290560 token_bytes=556609600 weight_movement_bytes=110100480 total_nvlink_bytes=666710080 weight_to_token_ratio=0.197806 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.753011467e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279881728 dispatch_meta_bytes=229720 combine_payload_bytes=279881728 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280111448 ingress_bytes=500082688 link_bytes=500082688 token_bytes=559993176 weight_movement_bytes=220200960 total_nvlink_bytes=780194136 weight_to_token_ratio=0.393221 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.697260439e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279222272 dispatch_meta_bytes=229760 combine_payload_bytes=279222272 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279452032 ingress_bytes=389322752 link_bytes=389322752 token_bytes=558674304 weight_movement_bytes=110100480 total_nvlink_bytes=668774784 weight_to_token_ratio=0.197075 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.770962983e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=277659648 dispatch_meta_bytes=227976 combine_payload_bytes=277659648 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277887624 ingress_bytes=387760128 link_bytes=387760128 token_bytes=555547272 weight_movement_bytes=110100480 total_nvlink_bytes=665647752 weight_to_token_ratio=0.198184 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.743786382e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279336960 dispatch_meta_bytes=229528 combine_payload_bytes=279336960 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279566488 ingress_bytes=389437440 link_bytes=389437440 token_bytes=558903448 weight_movement_bytes=110100480 total_nvlink_bytes=669003928 weight_to_token_ratio=0.196994 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.772957596e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278448128 dispatch_meta_bytes=229232 combine_payload_bytes=278448128 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=278677360 ingress_bytes=718850048 link_bytes=718850048 token_bytes=557125488 weight_movement_bytes=440401920 total_nvlink_bytes=997527408 weight_to_token_ratio=0.790490 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.250198464e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5571398488 prefetch_bytes=770703360 reduce_bytes=1651507200 weight_movement_bytes=2422210560 egress_bytes=2786845528 ingress_bytes=5206763520 link_bytes=5206763520 total_nvlink_bytes=7993609048 weight_to_token_ratio=0.434758 prefetch_share_of_ingress=0.148020 dedup_ratio=0.321838 prefetch_slots=14 unused_prefetch_slots=146 slot_utilization=0.087500 reduce_slots=15 remote_demand=7 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=12319 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.055418127e-05 prefetch_bytes_per_gemm_flop=1.340379902e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5588690928 prefetch_bytes=990904320 reduce_bytes=990904320 weight_movement_bytes=1981808640 egress_bytes=2795493360 ingress_bytes=4775006208 link_bytes=4775006208 total_nvlink_bytes=7570499568 weight_to_token_ratio=0.354611 prefetch_share_of_ingress=0.207519 dedup_ratio=0.320712 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=9 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.304521149e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5591988112 prefetch_bytes=990904320 reduce_bytes=660602880 weight_movement_bytes=1651507200 egress_bytes=2797141904 ingress_bytes=4446353408 link_bytes=4446353408 total_nvlink_bytes=7243495312 weight_to_token_ratio=0.295335 prefetch_share_of_ingress=0.222858 dedup_ratio=0.320181 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=6 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.732939876e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5571244280 prefetch_bytes=1101004800 reduce_bytes=110100480 weight_movement_bytes=1211105280 egress_bytes=2786763000 ingress_bytes=3995586560 link_bytes=3995586560 total_nvlink_bytes=6782349560 weight_to_token_ratio=0.217385 prefetch_share_of_ingress=0.275555 dedup_ratio=0.320633 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=1 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=6.948982189e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5580659888 prefetch_bytes=990904320 reduce_bytes=990904320 weight_movement_bytes=1981808640 egress_bytes=2791476400 ingress_bytes=4770992128 link_bytes=4770992128 total_nvlink_bytes=7562468528 weight_to_token_ratio=0.355121 prefetch_share_of_ingress=0.207694 dedup_ratio=0.321393 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=9 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.297540004e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5569931512 prefetch_bytes=990904320 reduce_bytes=1431306240 weight_movement_bytes=2422210560 egress_bytes=2786109688 ingress_bytes=5206032384 link_bytes=5206032384 total_nvlink_bytes=7992142072 weight_to_token_ratio=0.434873 prefetch_share_of_ingress=0.190338 dedup_ratio=0.321509 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=13 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.039706 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.054146561e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none - [2026-08-26 15:12:39] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 364.8 | throughput per GPU (TFLOP/s/GPU): 338.5 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 9.238233E+00 | loss scale: 1.0 | grad norm: 5.059 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44334 payload_rows_remote=38844 dup_rows=21202 dup_rows_remote=18578 dedup_ratio=0.323517 router_tokens_per_expert=[520, 486, 526, 545, 552, 521, 496, 503, 536, 499, 519, 519, 480, 483, 503, 510, 487, 509, 514, 549, 506, 505, 491, 496, 483, 534, 507, 507, 528, 532, 525, 507, 495, 499, 503, 539, 515, 530, 521, 552, 498, 511, 535, 549, 500, 509, 511, 518, 511, 494, 522, 468, 501, 523, 482, 512, 527, 466, 495, 511, 514, 505, 517, 557] payload_rows_per_dst=[5574, 5508, 5490, 5608, 5546, 5536, 5528, 5544] dup_rows_per_dst=[2724, 2590, 2624, 2638, 2762, 2726, 2498, 2640] combine_rows_per_src=[5574, 5508, 5490, 5608, 5546, 5536, 5528, 5544] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44464 payload_rows_remote=38884 dup_rows=21072 dup_rows_remote=18300 dedup_ratio=0.321533 router_tokens_per_expert=[538, 516, 457, 470, 532, 511, 461, 497, 489, 524, 552, 516, 480, 520, 519, 502, 518, 502, 500, 510, 509, 502, 487, 506, 482, 523, 522, 528, 525, 546, 489, 456, 518, 556, 510, 515, 528, 520, 532, 513, 544, 459, 538, 532, 542, 491, 492, 520, 507, 511, 514, 523, 510, 532, 558, 503, 514, 528, 536, 476, 506, 524, 513, 514] payload_rows_per_dst=[5426, 5600, 5454, 5448, 5666, 5684, 5606, 5580] dup_rows_per_dst=[2534, 2604, 2614, 2420, 2722, 2788, 2618, 2772] combine_rows_per_src=[5426, 5600, 5454, 5448, 5666, 5684, 5606, 5580] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44528 payload_rows_remote=38970 dup_rows=21008 dup_rows_remote=18454 dedup_ratio=0.320557 router_tokens_per_expert=[512, 525, 503, 494, 491, 543, 511, 529, 478, 499, 508, 496, 495, 535, 500, 545, 496, 508, 535, 524, 552, 497, 493, 450, 528, 528, 530, 534, 500, 494, 543, 515, 504, 509, 493, 539, 515, 519, 522, 540, 505, 526, 526, 515, 526, 484, 502, 486, 519, 500, 525, 504, 498, 500, 486, 552, 524, 502, 536, 499, 515, 480, 510, 516] payload_rows_per_dst=[5606, 5558, 5486, 5632, 5576, 5540, 5588, 5542] dup_rows_per_dst=[2610, 2554, 2624, 2712, 2706, 2600, 2580, 2622] combine_rows_per_src=[5606, 5558, 5486, 5632, 5576, 5540, 5588, 5542] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44604 payload_rows_remote=39092 dup_rows=20932 dup_rows_remote=18342 dedup_ratio=0.319397 router_tokens_per_expert=[499, 518, 522, 490, 537, 526, 498, 509, 538, 496, 513, 533, 536, 513, 512, 498, 520, 559, 508, 526, 532, 539, 487, 508, 513, 493, 532, 493, 503, 510, 511, 499, 516, 486, 489, 528, 492, 546, 505, 489, 523, 522, 515, 513, 517, 534, 500, 523, 516, 481, 516, 490, 487, 531, 501, 507, 536, 477, 526, 526, 521, 496, 479, 509] payload_rows_per_dst=[5592, 5642, 5668, 5526, 5512, 5634, 5512, 5518] dup_rows_per_dst=[2606, 2636, 2690, 2582, 2590, 2660, 2546, 2622] combine_rows_per_src=[5592, 5642, 5668, 5526, 5512, 5634, 5512, 5518] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44476 payload_rows_remote=38958 dup_rows=21060 dup_rows_remote=18376 dedup_ratio=0.321350 router_tokens_per_expert=[520, 527, 508, 518, 516, 525, 502, 538, 507, 520, 510, 487, 569, 487, 475, 495, 517, 485, 509, 520, 491, 527, 506, 518, 548, 475, 542, 475, 494, 485, 554, 528, 526, 505, 495, 519, 505, 519, 515, 511, 548, 477, 527, 467, 487, 525, 505, 525, 562, 506, 503, 493, 508, 499, 526, 532, 517, 502, 511, 529, 523, 511, 517, 495] payload_rows_per_dst=[5648, 5486, 5518, 5518, 5568, 5560, 5612, 5566] dup_rows_per_dst=[2660, 2614, 2628, 2684, 2622, 2562, 2646, 2644] combine_rows_per_src=[5648, 5486, 5518, 5518, 5568, 5560, 5612, 5566] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44534 payload_rows_remote=39064 dup_rows=21002 dup_rows_remote=18466 dedup_ratio=0.320465 router_tokens_per_expert=[540, 499, 470, 472, 493, 484, 508, 552, 466, 516, 494, 540, 489, 488, 530, 485, 518, 502, 523, 535, 482, 497, 515, 501, 526, 520, 529, 547, 532, 521, 538, 529, 489, 519, 496, 493, 506, 538, 521, 551, 510, 486, 487, 523, 524, 502, 488, 483, 512, 532, 485, 562, 522, 515, 539, 516, 493, 562, 525, 529, 496, 467, 539, 517] payload_rows_per_dst=[5526, 5466, 5548, 5696, 5578, 5470, 5614, 5636] dup_rows_per_dst=[2510, 2550, 2598, 2788, 2648, 2536, 2752, 2620] combine_rows_per_src=[5526, 5466, 5548, 5696, 5578, 5470, 5614, 5636] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44486 payload_rows_remote=38850 dup_rows=21050 dup_rows_remote=18380 dedup_ratio=0.321198 router_tokens_per_expert=[497, 523, 524, 532, 503, 528, 512, 511, 510, 545, 532, 538, 491, 522, 525, 523, 531, 538, 501, 494, 488, 549, 489, 507, 531, 516, 485, 538, 543, 550, 519, 499, 495, 527, 495, 494, 495, 499, 447, 544, 520, 503, 498, 517, 487, 469, 530, 495, 515, 520, 528, 517, 534, 514, 483, 542, 516, 489, 526, 495, 497, 512, 482, 489] payload_rows_per_dst=[5576, 5660, 5560, 5628, 5476, 5458, 5636, 5492] dup_rows_per_dst=[2684, 2712, 2634, 2734, 2516, 2580, 2670, 2520] combine_rows_per_src=[5576, 5660, 5560, 5628, 5476, 5458, 5636, 5492] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44528 payload_rows_remote=38918 dup_rows=21008 dup_rows_remote=18358 dedup_ratio=0.320557 router_tokens_per_expert=[545, 509, 513, 526, 518, 523, 516, 480, 495, 481, 523, 504, 535, 518, 529, 501, 517, 562, 525, 513, 542, 547, 473, 506, 539, 469, 481, 492, 483, 492, 507, 513, 497, 534, 495, 530, 477, 524, 531, 528, 526, 518, 518, 532, 523, 528, 477, 479, 536, 514, 536, 490, 481, 506, 494, 508, 530, 515, 478, 541, 548, 525, 472, 500] payload_rows_per_dst=[5610, 5606, 5642, 5482, 5514, 5548, 5538, 5588] dup_rows_per_dst=[2650, 2750, 2758, 2440, 2534, 2654, 2592, 2630] combine_rows_per_src=[5610, 5606, 5642, 5482, 5514, 5548, 5538, 5588] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278720512 dispatch_meta_bytes=228736 combine_payload_bytes=278720512 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278949248 ingress_bytes=388820992 link_bytes=388820992 token_bytes=557669760 weight_movement_bytes=110100480 total_nvlink_bytes=667770240 weight_to_token_ratio=0.197430 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.762236552e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278433792 dispatch_meta_bytes=229688 combine_payload_bytes=278433792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278663480 ingress_bytes=388534272 link_bytes=388534272 token_bytes=557097272 weight_movement_bytes=110100480 total_nvlink_bytes=667197752 weight_to_token_ratio=0.197632 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.757250019e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279336960 dispatch_meta_bytes=229696 combine_payload_bytes=279336960 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279566656 ingress_bytes=389437440 link_bytes=389437440 token_bytes=558903616 weight_movement_bytes=110100480 total_nvlink_bytes=669004096 weight_to_token_ratio=0.196994 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.772957596e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280010752 dispatch_meta_bytes=230120 combine_payload_bytes=280010752 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280240872 ingress_bytes=390111232 link_bytes=390111232 token_bytes=560251624 weight_movement_bytes=110100480 total_nvlink_bytes=670352104 weight_to_token_ratio=0.196520 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.784675947e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280211456 dispatch_meta_bytes=229736 combine_payload_bytes=280211456 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280441192 ingress_bytes=500412416 link_bytes=500412416 token_bytes=560652648 weight_movement_bytes=220200960 total_nvlink_bytes=780853608 weight_to_token_ratio=0.392758 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.702994951e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279250944 dispatch_meta_bytes=229336 combine_payload_bytes=279250944 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279480280 ingress_bytes=609552384 link_bytes=609552384 token_bytes=558731224 weight_movement_bytes=330301440 total_nvlink_bytes=889032664 weight_to_token_ratio=0.591163 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060111850e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278476800 dispatch_meta_bytes=228920 combine_payload_bytes=278476800 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278705720 ingress_bytes=498677760 link_bytes=498677760 token_bytes=557182520 weight_movement_bytes=220200960 total_nvlink_bytes=777383480 weight_to_token_ratio=0.395204 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.672826430e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278964224 dispatch_meta_bytes=229104 combine_payload_bytes=278964224 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=279193328 ingress_bytes=389064704 link_bytes=389064704 token_bytes=558157552 weight_movement_bytes=110100480 total_nvlink_bytes=668258032 weight_to_token_ratio=0.197257 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.766475104e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44412 payload_rows_remote=38734 dup_rows=21124 dup_rows_remote=18370 dedup_ratio=0.322327 router_tokens_per_expert=[535, 486, 500, 513, 544, 493, 537, 504, 483, 498, 554, 494, 519, 541, 497, 499, 515, 528, 483, 499, 515, 540, 512, 509, 506, 514, 549, 517, 503, 484, 502, 515, 506, 504, 515, 551, 538, 503, 512, 514, 519, 503, 513, 516, 508, 483, 538, 508, 511, 483, 522, 514, 560, 511, 531, 537, 472, 466, 503, 490, 506, 519, 508, 516] payload_rows_per_dst=[5420, 5548, 5376, 5554, 5570, 5634, 5632, 5678] dup_rows_per_dst=[2454, 2622, 2354, 2626, 2706, 2892, 2716, 2754] combine_rows_per_src=[5420, 5548, 5376, 5554, 5570, 5634, 5632, 5678] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44506 payload_rows_remote=38922 dup_rows=21030 dup_rows_remote=18412 dedup_ratio=0.320892 router_tokens_per_expert=[520, 517, 471, 539, 526, 530, 505, 490, 492, 519, 482, 496, 521, 492, 518, 518, 502, 536, 523, 515, 508, 515, 512, 490, 508, 535, 496, 504, 530, 502, 471, 521, 480, 545, 536, 511, 506, 503, 576, 577, 517, 530, 503, 541, 534, 471, 528, 495, 536, 471, 509, 472, 496, 507, 523, 520, 538, 531, 534, 486, 522, 477, 521, 468] payload_rows_per_dst=[5588, 5504, 5584, 5568, 5648, 5600, 5476, 5538] dup_rows_per_dst=[2608, 2572, 2618, 2566, 2820, 2638, 2592, 2616] combine_rows_per_src=[5588, 5504, 5584, 5568, 5648, 5600, 5476, 5538] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44496 payload_rows_remote=38968 dup_rows=21040 dup_rows_remote=18478 dedup_ratio=0.321045 router_tokens_per_expert=[499, 507, 544, 502, 528, 479, 539, 505, 484, 470, 546, 514, 521, 504, 491, 515, 522, 554, 525, 497, 540, 507, 476, 538, 497, 464, 504, 513, 501, 538, 515, 523, 519, 547, 500, 530, 485, 502, 487, 500, 489, 507, 518, 506, 506, 516, 513, 541, 531, 510, 519, 509, 528, 490, 502, 548, 530, 538, 516, 485, 504, 538, 505, 487] payload_rows_per_dst=[5554, 5528, 5636, 5486, 5548, 5570, 5604, 5570] dup_rows_per_dst=[2652, 2562, 2682, 2624, 2592, 2622, 2670, 2636] combine_rows_per_src=[5554, 5528, 5636, 5486, 5548, 5570, 5604, 5570] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44826 payload_rows_remote=39230 dup_rows=20710 dup_rows_remote=18188 dedup_ratio=0.316010 router_tokens_per_expert=[476, 522, 526, 527, 534, 513, 519, 524, 497, 532, 484, 501, 522, 507, 497, 515, 518, 524, 520, 547, 491, 488, 485, 537, 515, 511, 503, 479, 519, 516, 492, 529, 517, 510, 511, 502, 496, 534, 558, 484, 483, 512, 512, 487, 496, 548, 512, 509, 504, 507, 499, 522, 525, 496, 495, 509, 517, 507, 535, 497, 525, 508, 527, 554] payload_rows_per_dst=[5620, 5536, 5576, 5556, 5692, 5596, 5536, 5714] dup_rows_per_dst=[2662, 2574, 2644, 2572, 2532, 2522, 2578, 2626] combine_rows_per_src=[5620, 5536, 5576, 5556, 5692, 5596, 5536, 5714] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44408 payload_rows_remote=38832 dup_rows=21128 dup_rows_remote=18474 dedup_ratio=0.322388 router_tokens_per_expert=[517, 521, 496, 558, 514, 488, 502, 528, 515, 510, 514, 485, 547, 528, 503, 502, 484, 529, 540, 500, 558, 478, 539, 479, 512, 507, 483, 523, 526, 526, 522, 507, 508, 543, 471, 489, 528, 540, 550, 486, 528, 468, 500, 514, 511, 545, 498, 495, 530, 468, 505, 511, 525, 549, 555, 513, 475, 515, 510, 512, 500, 495, 508, 482] payload_rows_per_dst=[5564, 5556, 5606, 5580, 5576, 5532, 5562, 5432] dup_rows_per_dst=[2684, 2652, 2608, 2632, 2654, 2586, 2750, 2562] combine_rows_per_src=[5564, 5556, 5606, 5580, 5576, 5532, 5562, 5432] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44612 payload_rows_remote=39062 dup_rows=20924 dup_rows_remote=18378 dedup_ratio=0.319275 router_tokens_per_expert=[514, 508, 509, 507, 568, 522, 560, 529, 538, 496, 483, 536, 535, 508, 510, 551, 487, 536, 462, 515, 505, 532, 491, 538, 532, 525, 458, 508, 523, 465, 512, 525, 506, 488, 507, 515, 527, 506, 513, 491, 514, 509, 525, 506, 518, 502, 543, 532, 467, 498, 527, 516, 504, 489, 510, 503, 489, 515, 509, 502, 539, 521, 513, 476] payload_rows_per_dst=[5674, 5632, 5574, 5550, 5508, 5600, 5462, 5612] dup_rows_per_dst=[2760, 2682, 2558, 2546, 2598, 2698, 2566, 2516] combine_rows_per_src=[5674, 5632, 5574, 5550, 5508, 5600, 5462, 5612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44510 payload_rows_remote=38856 dup_rows=21026 dup_rows_remote=18456 dedup_ratio=0.320831 router_tokens_per_expert=[546, 500, 511, 504, 496, 497, 528, 509, 506, 537, 518, 513, 539, 524, 498, 501, 543, 492, 544, 537, 531, 500, 502, 508, 503, 535, 503, 517, 499, 500, 519, 547, 539, 506, 507, 499, 519, 535, 520, 502, 536, 497, 516, 528, 511, 466, 554, 504, 469, 487, 502, 499, 503, 515, 456, 490, 522, 540, 514, 501, 495, 514, 511, 504] payload_rows_per_dst=[5540, 5608, 5584, 5620, 5574, 5654, 5416, 5514] dup_rows_per_dst=[2642, 2664, 2730, 2626, 2680, 2570, 2426, 2688] combine_rows_per_src=[5540, 5608, 5584, 5620, 5574, 5654, 5416, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44366 payload_rows_remote=38808 dup_rows=21170 dup_rows_remote=18650 dedup_ratio=0.323029 router_tokens_per_expert=[491, 508, 534, 537, 520, 526, 526, 520, 485, 500, 514, 537, 475, 521, 515, 528, 522, 499, 504, 488, 519, 505, 550, 491, 489, 523, 544, 523, 499, 537, 531, 511, 523, 510, 505, 522, 528, 524, 505, 504, 539, 496, 489, 470, 539, 505, 518, 531, 523, 516, 506, 523, 521, 498, 486, 495, 514, 515, 517, 513, 472, 469, 496, 524] payload_rows_per_dst=[5578, 5462, 5512, 5538, 5682, 5294, 5742, 5558] dup_rows_per_dst=[2746, 2676, 2644, 2512, 2836, 2280, 2956, 2520] combine_rows_per_src=[5578, 5462, 5512, 5538, 5682, 5294, 5742, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44522 payload_rows_remote=39046 dup_rows=21014 dup_rows_remote=18524 dedup_ratio=0.320648 router_tokens_per_expert=[475, 512, 503, 482, 526, 526, 526, 518, 545, 517, 497, 508, 515, 488, 519, 552, 520, 513, 498, 502, 513, 547, 528, 476, 496, 519, 538, 530, 502, 504, 533, 506, 464, 542, 516, 558, 544, 488, 509, 499, 497, 533, 555, 517, 514, 526, 521, 513, 492, 525, 459, 526, 520, 481, 496, 484, 539, 501, 501, 518, 495, 489, 522, 490] payload_rows_per_dst=[5544, 5594, 5494, 5602, 5604, 5616, 5476, 5592] dup_rows_per_dst=[2592, 2688, 2700, 2654, 2636, 2736, 2490, 2518] combine_rows_per_src=[5544, 5594, 5494, 5602, 5604, 5616, 5476, 5592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280397824 dispatch_meta_bytes=229032 combine_payload_bytes=280397824 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280626856 ingress_bytes=610699264 link_bytes=610699264 token_bytes=561024680 weight_movement_bytes=330301440 total_nvlink_bytes=891326120 weight_to_token_ratio=0.588747 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.062106463e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44392 payload_rows_remote=38774 dup_rows=21144 dup_rows_remote=18378 dedup_ratio=0.322632 router_tokens_per_expert=[514, 501, 486, 533, 522, 509, 490, 468, 508, 516, 456, 469, 531, 528, 546, 504, 474, 468, 540, 510, 532, 533, 500, 546, 526, 512, 505, 544, 532, 510, 512, 551, 505, 535, 509, 509, 535, 508, 515, 508, 536, 500, 510, 450, 516, 534, 521, 499, 509, 486, 509, 471, 480, 473, 546, 509, 531, 528, 519, 513, 570, 518, 528, 512] payload_rows_per_dst=[5528, 5502, 5564, 5618, 5600, 5536, 5406, 5638] dup_rows_per_dst=[2518, 2614, 2642, 2766, 2648, 2596, 2560, 2800] combine_rows_per_src=[5528, 5502, 5564, 5618, 5600, 5536, 5406, 5638] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278591488 dispatch_meta_bytes=229576 combine_payload_bytes=278591488 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278821064 ingress_bytes=388691968 link_bytes=388691968 token_bytes=557412552 weight_movement_bytes=110100480 total_nvlink_bytes=667513032 weight_to_token_ratio=0.197521 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.759992612e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279480320 dispatch_meta_bytes=230096 combine_payload_bytes=279480320 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279710416 ingress_bytes=389580800 link_bytes=389580800 token_bytes=559190736 weight_movement_bytes=110100480 total_nvlink_bytes=669291216 weight_to_token_ratio=0.196893 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.775450862e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44466 payload_rows_remote=38788 dup_rows=21070 dup_rows_remote=18318 dedup_ratio=0.321503 router_tokens_per_expert=[520, 496, 537, 500, 517, 529, 488, 543, 492, 498, 496, 522, 474, 515, 538, 543, 534, 478, 476, 522, 501, 517, 502, 546, 525, 475, 556, 526, 531, 534, 514, 496, 517, 522, 499, 481, 502, 504, 508, 486, 531, 477, 543, 520, 519, 552, 526, 520, 497, 527, 503, 501, 520, 486, 547, 509, 460, 531, 509, 502, 478, 487, 510, 553] payload_rows_per_dst=[5678, 5442, 5594, 5576, 5540, 5562, 5576, 5498] dup_rows_per_dst=[2752, 2548, 2684, 2612, 2498, 2810, 2604, 2562] combine_rows_per_src=[5678, 5442, 5594, 5576, 5540, 5562, 5576, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278519808 dispatch_meta_bytes=229248 combine_payload_bytes=278519808 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278749056 ingress_bytes=608821248 link_bytes=608821248 token_bytes=557268864 weight_movement_bytes=330301440 total_nvlink_bytes=887570304 weight_to_token_ratio=0.592715 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058840284e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278175744 dispatch_meta_bytes=229832 combine_payload_bytes=278175744 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278405576 ingress_bytes=388276224 link_bytes=388276224 token_bytes=556581320 weight_movement_bytes=110100480 total_nvlink_bytes=666681800 weight_to_token_ratio=0.197816 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.752762140e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279881728 dispatch_meta_bytes=230280 combine_payload_bytes=279881728 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280112008 ingress_bytes=389982208 link_bytes=389982208 token_bytes=559993736 weight_movement_bytes=110100480 total_nvlink_bytes=670094216 weight_to_token_ratio=0.196610 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.782432008e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277932032 dispatch_meta_bytes=228608 combine_payload_bytes=277932032 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278160640 ingress_bytes=608233472 link_bytes=608233472 token_bytes=556092672 weight_movement_bytes=330301440 total_nvlink_bytes=886394112 weight_to_token_ratio=0.593968 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057818045e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278032384 dispatch_meta_bytes=228424 combine_payload_bytes=278032384 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278260808 ingress_bytes=388132864 link_bytes=388132864 token_bytes=556293192 weight_movement_bytes=110100480 total_nvlink_bytes=666393672 weight_to_token_ratio=0.197918 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.750268874e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44202 payload_rows_remote=38614 dup_rows=21334 dup_rows_remote=18626 dedup_ratio=0.325531 router_tokens_per_expert=[507, 521, 503, 512, 508, 533, 511, 509, 546, 507, 534, 524, 522, 488, 515, 512, 544, 504, 518, 508, 518, 541, 510, 502, 508, 508, 499, 513, 487, 493, 527, 505, 480, 483, 531, 517, 517, 501, 555, 482, 496, 507, 545, 531, 511, 499, 518, 507, 528, 489, 519, 442, 530, 506, 536, 515, 509, 509, 527, 497, 503, 536, 493, 512] payload_rows_per_dst=[5522, 5588, 5554, 5478, 5492, 5578, 5468, 5522] dup_rows_per_dst=[2686, 2708, 2736, 2602, 2640, 2650, 2662, 2650] combine_rows_per_src=[5522, 5588, 5554, 5478, 5492, 5578, 5468, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44526 payload_rows_remote=38916 dup_rows=21010 dup_rows_remote=18338 dedup_ratio=0.320587 router_tokens_per_expert=[498, 536, 491, 514, 495, 493, 490, 531, 506, 489, 502, 525, 519, 518, 547, 507, 542, 522, 502, 541, 501, 508, 503, 528, 495, 453, 500, 509, 516, 528, 523, 502, 504, 521, 533, 548, 499, 518, 502, 516, 528, 527, 529, 515, 541, 528, 506, 517, 492, 514, 514, 468, 533, 558, 491, 511, 507, 508, 508, 481, 523, 483, 498, 513] payload_rows_per_dst=[5550, 5534, 5584, 5522, 5610, 5624, 5570, 5532] dup_rows_per_dst=[2546, 2692, 2710, 2530, 2672, 2758, 2592, 2510] combine_rows_per_src=[5550, 5534, 5584, 5522, 5610, 5624, 5570, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44530 payload_rows_remote=38972 dup_rows=21006 dup_rows_remote=18450 dedup_ratio=0.320526 router_tokens_per_expert=[483, 522, 513, 498, 538, 502, 544, 459, 531, 496, 540, 498, 522, 516, 520, 507, 504, 488, 549, 510, 520, 531, 500, 455, 496, 524, 518, 499, 492, 514, 522, 518, 496, 529, 517, 541, 497, 488, 476, 521, 539, 541, 530, 489, 505, 520, 497, 520, 472, 504, 509, 532, 538, 543, 491, 542, 536, 511, 496, 500, 525, 513, 536, 485] payload_rows_per_dst=[5490, 5638, 5558, 5562, 5562, 5596, 5538, 5586] dup_rows_per_dst=[2628, 2622, 2556, 2604, 2568, 2686, 2724, 2618] combine_rows_per_src=[5490, 5638, 5558, 5562, 5562, 5596, 5538, 5586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44568 payload_rows_remote=39024 dup_rows=20968 dup_rows_remote=18350 dedup_ratio=0.319946 router_tokens_per_expert=[540, 474, 482, 518, 492, 510, 537, 521, 545, 504, 509, 505, 532, 518, 511, 488, 531, 510, 493, 520, 519, 523, 493, 503, 524, 519, 511, 541, 511, 530, 470, 561, 501, 533, 502, 502, 469, 540, 526, 533, 492, 511, 519, 501, 504, 508, 532, 514, 509, 513, 500, 481, 528, 487, 507, 518, 499, 484, 531, 535, 486, 514, 532, 512] payload_rows_per_dst=[5550, 5606, 5582, 5616, 5584, 5544, 5468, 5618] dup_rows_per_dst=[2598, 2618, 2602, 2718, 2628, 2618, 2618, 2568] combine_rows_per_src=[5550, 5606, 5582, 5616, 5584, 5544, 5468, 5618] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=276785152 dispatch_meta_bytes=228960 combine_payload_bytes=276785152 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277014112 ingress_bytes=496986112 link_bytes=496986112 token_bytes=553799264 weight_movement_bytes=220200960 total_nvlink_bytes=774000224 weight_to_token_ratio=0.397619 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.643405889e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44360 payload_rows_remote=38720 dup_rows=21176 dup_rows_remote=18206 dedup_ratio=0.323120 router_tokens_per_expert=[484, 485, 500, 522, 499, 505, 491, 512, 530, 459, 513, 486, 469, 500, 535, 501, 516, 510, 539, 511, 542, 483, 504, 533, 535, 547, 510, 534, 486, 541, 522, 522, 473, 546, 511, 542, 543, 493, 514, 515, 492, 522, 519, 485, 514, 524, 495, 480, 556, 495, 508, 499, 508, 521, 513, 531, 531, 490, 509, 512, 519, 546, 523, 513] payload_rows_per_dst=[5502, 5450, 5510, 5650, 5630, 5510, 5468, 5640] dup_rows_per_dst=[2494, 2458, 2634, 2774, 2746, 2552, 2548, 2970] combine_rows_per_src=[5502, 5450, 5510, 5650, 5630, 5510, 5468, 5640] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44606 payload_rows_remote=38938 dup_rows=20930 dup_rows_remote=18158 dedup_ratio=0.319366 router_tokens_per_expert=[484, 522, 496, 488, 516, 525, 503, 552, 500, 515, 499, 521, 483, 505, 511, 522, 496, 501, 542, 526, 539, 528, 503, 498, 502, 494, 476, 501, 523, 516, 516, 527, 485, 483, 510, 509, 538, 472, 511, 521, 561, 513, 513, 523, 459, 532, 505, 503, 529, 541, 542, 534, 529, 506, 534, 505, 519, 539, 461, 494, 496, 504, 521, 546] payload_rows_per_dst=[5588, 5518, 5682, 5516, 5552, 5592, 5668, 5490] dup_rows_per_dst=[2584, 2594, 2584, 2594, 2506, 2626, 2772, 2670] combine_rows_per_src=[5588, 5518, 5682, 5516, 5552, 5592, 5668, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278949888 dispatch_meta_bytes=229016 combine_payload_bytes=278949888 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279178904 ingress_bytes=389050368 link_bytes=389050368 token_bytes=558128792 weight_movement_bytes=110100480 total_nvlink_bytes=668229272 weight_to_token_ratio=0.197267 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.766225777e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44432 payload_rows_remote=38818 dup_rows=21104 dup_rows_remote=18442 dedup_ratio=0.322021 router_tokens_per_expert=[544, 517, 501, 510, 508, 490, 544, 519, 500, 556, 529, 498, 509, 513, 504, 487, 526, 504, 499, 514, 486, 492, 510, 539, 498, 497, 522, 505, 529, 520, 525, 542, 479, 488, 481, 509, 499, 523, 576, 500, 499, 503, 519, 470, 515, 501, 547, 496, 506, 532, 562, 509, 560, 487, 506, 512, 513, 513, 505, 491, 500, 533, 514, 483] payload_rows_per_dst=[5586, 5510, 5550, 5614, 5548, 5450, 5676, 5498] dup_rows_per_dst=[2680, 2682, 2590, 2662, 2562, 2650, 2672, 2606] combine_rows_per_src=[5586, 5510, 5550, 5614, 5548, 5450, 5676, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279351296 dispatch_meta_bytes=229688 combine_payload_bytes=279351296 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279580984 ingress_bytes=609652736 link_bytes=609652736 token_bytes=558932280 weight_movement_bytes=330301440 total_nvlink_bytes=889233720 weight_to_token_ratio=0.590951 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060286379e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44542 payload_rows_remote=38830 dup_rows=20994 dup_rows_remote=18174 dedup_ratio=0.320343 router_tokens_per_expert=[513, 513, 528, 498, 525, 536, 531, 512, 525, 544, 502, 545, 483, 535, 515, 493, 500, 512, 495, 531, 522, 479, 504, 504, 539, 510, 507, 483, 515, 483, 502, 508, 498, 511, 526, 518, 527, 545, 469, 524, 507, 534, 501, 511, 496, 512, 523, 489, 542, 512, 519, 542, 502, 488, 508, 543, 486, 524, 517, 508, 516, 496, 511, 471] payload_rows_per_dst=[5712, 5636, 5540, 5556, 5548, 5510, 5482, 5558] dup_rows_per_dst=[2820, 2682, 2564, 2538, 2688, 2592, 2610, 2500] combine_rows_per_src=[5712, 5636, 5540, 5556, 5548, 5510, 5482, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279724032 dispatch_meta_bytes=229496 combine_payload_bytes=279724032 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279953528 ingress_bytes=499924992 link_bytes=499924992 token_bytes=559677560 weight_movement_bytes=220200960 total_nvlink_bytes=779878520 weight_to_token_ratio=0.393443 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.694517846e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=277544960 dispatch_meta_bytes=227704 combine_payload_bytes=277544960 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277772664 ingress_bytes=387645440 link_bytes=387645440 token_bytes=555317624 weight_movement_bytes=110100480 total_nvlink_bytes=665418104 weight_to_token_ratio=0.198266 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.741791769e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279107584 dispatch_meta_bytes=228384 combine_payload_bytes=279107584 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279335968 ingress_bytes=609409024 link_bytes=609409024 token_bytes=558443552 weight_movement_bytes=330301440 total_nvlink_bytes=888744992 weight_to_token_ratio=0.591468 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059862523e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278247424 dispatch_meta_bytes=229040 combine_payload_bytes=278247424 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278476464 ingress_bytes=388347904 link_bytes=388347904 token_bytes=556723888 weight_movement_bytes=110100480 total_nvlink_bytes=666824368 weight_to_token_ratio=0.197765 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.754008773e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278333440 dispatch_meta_bytes=228016 combine_payload_bytes=278333440 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278561456 ingress_bytes=388433920 link_bytes=388433920 token_bytes=556894896 weight_movement_bytes=110100480 total_nvlink_bytes=666995376 weight_to_token_ratio=0.197704 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.755504733e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44454 payload_rows_remote=38982 dup_rows=21082 dup_rows_remote=18438 dedup_ratio=0.321686 router_tokens_per_expert=[510, 526, 526, 499, 524, 517, 539, 525, 501, 527, 507, 503, 493, 503, 540, 484, 542, 524, 479, 510, 498, 506, 515, 507, 507, 522, 500, 524, 513, 498, 468, 528, 523, 499, 521, 530, 477, 534, 476, 533, 484, 510, 523, 456, 500, 526, 533, 524, 534, 495, 534, 484, 517, 493, 533, 531, 520, 512, 536, 500, 512, 528, 505, 520] payload_rows_per_dst=[5650, 5472, 5610, 5502, 5600, 5506, 5600, 5514] dup_rows_per_dst=[2682, 2644, 2552, 2618, 2586, 2606, 2642, 2752] combine_rows_per_src=[5650, 5472, 5610, 5502, 5600, 5506, 5600, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44498 payload_rows_remote=38946 dup_rows=21038 dup_rows_remote=18384 dedup_ratio=0.321014 router_tokens_per_expert=[532, 548, 514, 468, 514, 497, 512, 500, 509, 488, 532, 520, 501, 497, 497, 519, 499, 513, 490, 494, 520, 561, 502, 492, 525, 513, 509, 507, 488, 522, 530, 512, 468, 523, 540, 556, 518, 557, 473, 468, 542, 509, 545, 515, 514, 485, 525, 517, 494, 515, 529, 566, 495, 530, 514, 487, 494, 510, 480, 507, 530, 505, 538, 494] payload_rows_per_dst=[5494, 5510, 5546, 5584, 5552, 5656, 5590, 5566] dup_rows_per_dst=[2676, 2616, 2596, 2628, 2654, 2648, 2670, 2550] combine_rows_per_src=[5494, 5510, 5546, 5584, 5552, 5656, 5590, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44376 payload_rows_remote=38812 dup_rows=21160 dup_rows_remote=18564 dedup_ratio=0.322876 router_tokens_per_expert=[526, 507, 493, 523, 509, 519, 527, 490, 501, 518, 496, 490, 506, 527, 505, 530, 518, 509, 521, 488, 500, 513, 514, 517, 485, 512, 525, 520, 527, 527, 540, 493, 512, 497, 507, 517, 555, 473, 500, 531, 468, 527, 502, 482, 509, 519, 531, 529, 475, 565, 542, 493, 508, 515, 522, 520, 526, 493, 498, 490, 526, 514, 507, 539] payload_rows_per_dst=[5502, 5488, 5564, 5646, 5570, 5542, 5552, 5512] dup_rows_per_dst=[2686, 2658, 2596, 2612, 2614, 2592, 2728, 2674] combine_rows_per_src=[5502, 5488, 5564, 5646, 5570, 5542, 5552, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279422976 dispatch_meta_bytes=229680 combine_payload_bytes=279422976 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279652656 ingress_bytes=389523456 link_bytes=389523456 token_bytes=559075632 weight_movement_bytes=110100480 total_nvlink_bytes=669176112 weight_to_token_ratio=0.196933 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.774453556e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44512 payload_rows_remote=39022 dup_rows=21024 dup_rows_remote=18378 dedup_ratio=0.320801 router_tokens_per_expert=[525, 526, 481, 587, 512, 510, 468, 486, 443, 490, 516, 532, 501, 495, 500, 516, 539, 515, 549, 501, 504, 513, 509, 494, 527, 476, 534, 508, 521, 514, 511, 507, 481, 560, 518, 512, 527, 508, 511, 521, 512, 521, 517, 516, 493, 514, 498, 497, 520, 529, 520, 553, 512, 515, 465, 543, 496, 478, 545, 490, 500, 513, 550, 523] payload_rows_per_dst=[5598, 5456, 5572, 5606, 5622, 5490, 5652, 5516] dup_rows_per_dst=[2592, 2530, 2676, 2590, 2654, 2646, 2662, 2674] combine_rows_per_src=[5598, 5456, 5572, 5606, 5622, 5490, 5652, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44414 payload_rows_remote=38840 dup_rows=21122 dup_rows_remote=18464 dedup_ratio=0.322296 router_tokens_per_expert=[498, 520, 532, 506, 531, 493, 495, 515, 505, 509, 517, 532, 500, 523, 496, 511, 471, 512, 515, 521, 512, 526, 516, 511, 506, 497, 554, 526, 568, 544, 510, 496, 476, 478, 527, 518, 511, 505, 556, 469, 536, 545, 518, 560, 485, 475, 474, 494, 492, 544, 520, 548, 500, 512, 484, 502, 479, 506, 526, 548, 480, 482, 529, 521] payload_rows_per_dst=[5442, 5504, 5624, 5712, 5444, 5560, 5554, 5574] dup_rows_per_dst=[2424, 2682, 2858, 2690, 2616, 2630, 2564, 2658] combine_rows_per_src=[5442, 5504, 5624, 5712, 5444, 5560, 5554, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44474 payload_rows_remote=38950 dup_rows=21062 dup_rows_remote=18470 dedup_ratio=0.321381 router_tokens_per_expert=[486, 531, 523, 518, 520, 467, 528, 533, 497, 472, 559, 528, 506, 481, 526, 491, 521, 518, 532, 528, 494, 517, 527, 505, 505, 495, 525, 525, 489, 494, 488, 494, 504, 525, 525, 505, 521, 505, 516, 508, 563, 508, 473, 501, 531, 534, 538, 533, 502, 491, 513, 525, 511, 503, 478, 535, 473, 540, 480, 525, 506, 515, 505, 553] payload_rows_per_dst=[5574, 5524, 5602, 5494, 5574, 5614, 5524, 5568] dup_rows_per_dst=[2638, 2596, 2682, 2536, 2644, 2748, 2592, 2626] combine_rows_per_src=[5574, 5524, 5602, 5494, 5574, 5614, 5524, 5568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279164928 dispatch_meta_bytes=229320 combine_payload_bytes=279164928 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279394248 ingress_bytes=499365888 link_bytes=499365888 token_bytes=558559176 weight_movement_bytes=220200960 total_nvlink_bytes=778760136 weight_to_token_ratio=0.394230 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.684794108e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44522 payload_rows_remote=38890 dup_rows=21014 dup_rows_remote=18388 dedup_ratio=0.320648 router_tokens_per_expert=[527, 521, 524, 508, 553, 493, 492, 559, 483, 522, 520, 555, 528, 501, 512, 508, 522, 498, 472, 484, 484, 487, 506, 508, 493, 544, 516, 515, 531, 493, 528, 509, 501, 505, 496, 504, 491, 503, 496, 530, 504, 494, 549, 515, 533, 531, 525, 535, 515, 510, 518, 498, 486, 485, 522, 535, 534, 504, 534, 509, 509, 502, 521, 478] payload_rows_per_dst=[5592, 5542, 5420, 5632, 5564, 5624, 5600, 5548] dup_rows_per_dst=[2762, 2716, 2502, 2626, 2488, 2748, 2538, 2634] combine_rows_per_src=[5592, 5542, 5420, 5632, 5564, 5624, 5600, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278204416 dispatch_meta_bytes=229504 combine_payload_bytes=278204416 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278433920 ingress_bytes=388304896 link_bytes=388304896 token_bytes=556638336 weight_movement_bytes=110100480 total_nvlink_bytes=666738816 weight_to_token_ratio=0.197795 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.753260793e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44484 payload_rows_remote=38856 dup_rows=21052 dup_rows_remote=18456 dedup_ratio=0.321228 router_tokens_per_expert=[484, 524, 510, 531, 508, 511, 522, 504, 494, 510, 551, 535, 475, 498, 527, 528, 533, 490, 526, 495, 525, 472, 526, 501, 492, 542, 505, 485, 468, 508, 505, 537, 484, 531, 518, 547, 547, 497, 501, 552, 521, 532, 526, 539, 513, 509, 506, 498, 517, 492, 495, 487, 513, 528, 493, 515, 522, 489, 512, 508, 523, 534, 509, 488] payload_rows_per_dst=[5628, 5646, 5520, 5484, 5674, 5444, 5522, 5566] dup_rows_per_dst=[2596, 2952, 2616, 2576, 2680, 2482, 2546, 2604] combine_rows_per_src=[5628, 5646, 5520, 5484, 5674, 5444, 5522, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5581349232 prefetch_bytes=990904320 reduce_bytes=770703360 weight_movement_bytes=1761607680 egress_bytes=2791821680 ingress_bytes=4551135232 link_bytes=4551135232 total_nvlink_bytes=7342956912 weight_to_token_ratio=0.315624 prefetch_share_of_ingress=0.217727 dedup_ratio=0.321478 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=7 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042647 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.915172702e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279709696 dispatch_meta_bytes=229600 combine_payload_bytes=279709696 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279939296 ingress_bytes=499910656 link_bytes=499910656 token_bytes=559648992 weight_movement_bytes=220200960 total_nvlink_bytes=779849952 weight_to_token_ratio=0.393463 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.694268520e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278405120 dispatch_meta_bytes=229216 combine_payload_bytes=278405120 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278634336 ingress_bytes=388505600 link_bytes=388505600 token_bytes=557039456 weight_movement_bytes=110100480 total_nvlink_bytes=667139936 weight_to_token_ratio=0.197653 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.756751366e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5582810416 prefetch_bytes=990904320 reduce_bytes=550502400 weight_movement_bytes=1541406720 egress_bytes=2792551728 ingress_bytes=4331665408 link_bytes=4331665408 total_nvlink_bytes=7124217136 weight_to_token_ratio=0.276099 prefetch_share_of_ingress=0.228758 dedup_ratio=0.321255 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=5 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.533478581e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279193600 dispatch_meta_bytes=229680 combine_payload_bytes=279193600 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279423280 ingress_bytes=609495040 link_bytes=609495040 token_bytes=558616880 weight_movement_bytes=330301440 total_nvlink_bytes=888918320 weight_to_token_ratio=0.591284 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060012119e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278763520 dispatch_meta_bytes=229112 combine_payload_bytes=278763520 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278992632 ingress_bytes=498964480 link_bytes=498964480 token_bytes=557756152 weight_movement_bytes=220200960 total_nvlink_bytes=777957112 weight_to_token_ratio=0.394798 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.677812963e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5572718032 prefetch_bytes=880803840 reduce_bytes=1651507200 weight_movement_bytes=2532311040 egress_bytes=2787505616 ingress_bytes=5317523456 link_bytes=5317523456 total_nvlink_bytes=8105029072 weight_to_token_ratio=0.454412 prefetch_share_of_ingress=0.165642 dedup_ratio=0.321774 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=15 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.248047872e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5586310480 prefetch_bytes=880803840 reduce_bytes=1101004800 weight_movement_bytes=1981808640 egress_bytes=2794302800 ingress_bytes=4773816320 link_bytes=4773816320 total_nvlink_bytes=7568119120 weight_to_token_ratio=0.354762 prefetch_share_of_ingress=0.184507 dedup_ratio=0.320859 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=10 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043015 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.302451738e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278519808 dispatch_meta_bytes=229248 combine_payload_bytes=278519808 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278749056 ingress_bytes=498720768 link_bytes=498720768 token_bytes=557268864 weight_movement_bytes=220200960 total_nvlink_bytes=777469824 weight_to_token_ratio=0.395143 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.673574410e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5585359208 prefetch_bytes=880803840 reduce_bytes=880803840 weight_movement_bytes=1761607680 egress_bytes=2793824616 ingress_bytes=4553142272 link_bytes=4553142272 total_nvlink_bytes=7346966888 weight_to_token_ratio=0.315397 prefetch_share_of_ingress=0.193450 dedup_ratio=0.320291 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=8 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.918663274e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5591585704 prefetch_bytes=1101004800 reduce_bytes=1101004800 weight_movement_bytes=2202009600 egress_bytes=2796940200 ingress_bytes=4996655104 link_bytes=4996655104 total_nvlink_bytes=7793595304 weight_to_token_ratio=0.393808 prefetch_share_of_ingress=0.220348 dedup_ratio=0.320053 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=10 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.690005034e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5577991736 prefetch_bytes=1101004800 reduce_bytes=770703360 weight_movement_bytes=1871708160 egress_bytes=2790141496 ingress_bytes=4659558400 link_bytes=4659558400 total_nvlink_bytes=7449699896 weight_to_token_ratio=0.335552 prefetch_share_of_ingress=0.236290 dedup_ratio=0.321082 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=7 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.103738423e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5566603728 prefetch_bytes=880803840 reduce_bytes=880803840 weight_movement_bytes=1761607680 egress_bytes=2784444880 ingress_bytes=4543766528 link_bytes=4543766528 total_nvlink_bytes=7328211408 weight_to_token_ratio=0.316460 prefetch_share_of_ingress=0.193849 dedup_ratio=0.321597 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=8 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.902357313e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none + [2026-08-26 16:00:27] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 369.1 | throughput per GPU (TFLOP/s/GPU): 334.5 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 9.252842E+00 | loss scale: 1.0 | grad norm: 5.076 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44520 payload_rows_remote=38982 dup_rows=21016 dup_rows_remote=18472 dedup_ratio=0.320679 router_tokens_per_expert=[509, 523, 502, 492, 489, 540, 507, 526, 479, 501, 507, 493, 492, 530, 497, 542, 493, 507, 535, 521, 550, 499, 489, 449, 530, 527, 533, 534, 499, 493, 542, 514, 505, 508, 492, 540, 514, 524, 522, 543, 503, 527, 525, 517, 526, 487, 504, 487, 518, 502, 527, 505, 499, 501, 485, 557, 527, 507, 537, 501, 517, 485, 516, 516] payload_rows_per_dst=[5584, 5538, 5478, 5636, 5580, 5538, 5596, 5570] dup_rows_per_dst=[2592, 2544, 2608, 2708, 2716, 2614, 2592, 2642] combine_rows_per_src=[5584, 5538, 5478, 5636, 5580, 5538, 5596, 5570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44474 payload_rows_remote=38820 dup_rows=21062 dup_rows_remote=18380 dedup_ratio=0.321381 router_tokens_per_expert=[493, 520, 522, 527, 499, 528, 508, 509, 509, 541, 528, 537, 488, 522, 524, 519, 529, 538, 498, 494, 487, 548, 485, 503, 532, 515, 486, 537, 543, 549, 518, 498, 495, 528, 495, 495, 496, 503, 446, 546, 521, 504, 503, 518, 490, 471, 528, 496, 518, 522, 531, 518, 535, 515, 483, 546, 520, 493, 527, 497, 501, 514, 486, 493] payload_rows_per_dst=[5548, 5642, 5540, 5630, 5486, 5466, 5654, 5508] dup_rows_per_dst=[2664, 2694, 2624, 2726, 2522, 2596, 2682, 2554] combine_rows_per_src=[5548, 5642, 5540, 5630, 5486, 5466, 5654, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44478 payload_rows_remote=38934 dup_rows=21058 dup_rows_remote=18336 dedup_ratio=0.321320 router_tokens_per_expert=[537, 513, 456, 468, 529, 510, 458, 493, 487, 520, 551, 515, 478, 519, 518, 500, 519, 502, 500, 511, 507, 499, 484, 507, 482, 522, 522, 527, 525, 543, 489, 456, 518, 553, 508, 515, 525, 520, 532, 515, 544, 460, 544, 534, 544, 491, 491, 520, 509, 513, 517, 526, 513, 532, 561, 505, 514, 530, 540, 477, 511, 527, 515, 517] payload_rows_per_dst=[5414, 5586, 5446, 5562, 5664, 5592, 5670, 5544] dup_rows_per_dst=[2514, 2590, 2612, 2570, 2704, 2664, 2682, 2722] combine_rows_per_src=[5414, 5586, 5446, 5562, 5664, 5592, 5670, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44364 payload_rows_remote=38876 dup_rows=21172 dup_rows_remote=18576 dedup_ratio=0.323059 router_tokens_per_expert=[518, 485, 524, 544, 549, 521, 492, 500, 530, 496, 516, 519, 476, 480, 500, 510, 483, 508, 515, 543, 505, 503, 490, 495, 482, 533, 508, 507, 532, 531, 526, 507, 498, 500, 502, 538, 515, 533, 522, 553, 494, 513, 539, 549, 499, 509, 512, 522, 512, 497, 526, 467, 503, 529, 484, 513, 532, 468, 498, 511, 517, 508, 519, 558] payload_rows_per_dst=[5562, 5492, 5488, 5616, 5558, 5540, 5550, 5558] dup_rows_per_dst=[2704, 2562, 2596, 2636, 2764, 2734, 2512, 2664] combine_rows_per_src=[5562, 5492, 5488, 5616, 5558, 5540, 5550, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44476 payload_rows_remote=38960 dup_rows=21060 dup_rows_remote=18372 dedup_ratio=0.321350 router_tokens_per_expert=[519, 523, 506, 517, 513, 521, 501, 536, 504, 521, 508, 487, 569, 487, 472, 491, 515, 482, 509, 518, 489, 526, 506, 518, 546, 476, 538, 477, 494, 485, 556, 530, 526, 505, 497, 522, 504, 521, 513, 511, 550, 474, 526, 467, 485, 528, 508, 527, 560, 511, 505, 492, 510, 501, 528, 532, 517, 503, 515, 532, 528, 513, 518, 499] payload_rows_per_dst=[5626, 5474, 5516, 5516, 5568, 5572, 5620, 5584] dup_rows_per_dst=[2646, 2604, 2610, 2688, 2630, 2558, 2658, 2666] combine_rows_per_src=[5626, 5474, 5516, 5516, 5568, 5572, 5620, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44530 payload_rows_remote=39048 dup_rows=21006 dup_rows_remote=18466 dedup_ratio=0.320526 router_tokens_per_expert=[537, 496, 469, 469, 492, 481, 508, 546, 465, 514, 493, 539, 489, 486, 529, 482, 516, 502, 524, 532, 482, 498, 513, 501, 526, 519, 530, 546, 531, 519, 535, 528, 488, 521, 495, 488, 509, 538, 523, 550, 509, 491, 489, 522, 524, 504, 489, 483, 515, 531, 487, 565, 523, 515, 540, 520, 495, 562, 531, 531, 497, 471, 540, 525] payload_rows_per_dst=[5504, 5454, 5542, 5694, 5580, 5482, 5622, 5652] dup_rows_per_dst=[2492, 2540, 2594, 2774, 2644, 2540, 2770, 2652] combine_rows_per_src=[5504, 5454, 5542, 5694, 5580, 5482, 5622, 5652] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44602 payload_rows_remote=39084 dup_rows=20934 dup_rows_remote=18324 dedup_ratio=0.319427 router_tokens_per_expert=[496, 512, 519, 489, 535, 524, 495, 509, 534, 492, 511, 531, 533, 513, 509, 497, 520, 556, 505, 524, 526, 537, 487, 507, 514, 494, 532, 493, 502, 508, 510, 500, 518, 487, 492, 528, 494, 546, 506, 493, 522, 520, 515, 512, 519, 537, 499, 526, 518, 482, 520, 495, 484, 533, 506, 507, 539, 480, 528, 530, 524, 498, 487, 509] payload_rows_per_dst=[5576, 5624, 5666, 5534, 5518, 5638, 5518, 5528] dup_rows_per_dst=[2582, 2616, 2658, 2572, 2610, 2662, 2572, 2662] combine_rows_per_src=[5576, 5624, 5666, 5534, 5518, 5638, 5518, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279422976 dispatch_meta_bytes=229816 combine_payload_bytes=279422976 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279652792 ingress_bytes=389523456 link_bytes=389523456 token_bytes=559075768 weight_movement_bytes=110100480 total_nvlink_bytes=669176248 weight_to_token_ratio=0.196933 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.774453556e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278261760 dispatch_meta_bytes=228800 combine_payload_bytes=278261760 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278490560 ingress_bytes=498462720 link_bytes=498462720 token_bytes=556752320 weight_movement_bytes=220200960 total_nvlink_bytes=776953280 weight_to_token_ratio=0.395510 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.669086531e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44510 payload_rows_remote=38794 dup_rows=21026 dup_rows_remote=18214 dedup_ratio=0.320831 router_tokens_per_expert=[544, 505, 507, 525, 518, 523, 516, 478, 495, 478, 520, 502, 532, 518, 528, 498, 514, 559, 523, 513, 541, 544, 471, 508, 540, 466, 481, 493, 484, 491, 505, 513, 495, 532, 493, 529, 478, 525, 534, 528, 524, 522, 517, 533, 524, 529, 478, 480, 536, 515, 540, 494, 481, 505, 499, 509, 533, 518, 480, 544, 552, 528, 476, 504] payload_rows_per_dst=[5716, 5714, 5706, 5380, 5494, 5600, 5410, 5490] dup_rows_per_dst=[2812, 2862, 2852, 2294, 2498, 2740, 2428, 2540] combine_rows_per_src=[5716, 5714, 5706, 5380, 5494, 5600, 5410, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279078912 dispatch_meta_bytes=229080 combine_payload_bytes=279078912 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279307992 ingress_bytes=609380352 link_bytes=609380352 token_bytes=558386904 weight_movement_bytes=330301440 total_nvlink_bytes=888688344 weight_to_token_ratio=0.591528 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059812658e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278663168 dispatch_meta_bytes=229808 combine_payload_bytes=278663168 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278892976 ingress_bytes=388763648 link_bytes=388763648 token_bytes=557556144 weight_movement_bytes=110100480 total_nvlink_bytes=667656624 weight_to_token_ratio=0.197470 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.761239245e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279265280 dispatch_meta_bytes=229328 combine_payload_bytes=279265280 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279494608 ingress_bytes=499466240 link_bytes=499466240 token_bytes=558759888 weight_movement_bytes=220200960 total_nvlink_bytes=778960848 weight_to_token_ratio=0.394089 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.686539394e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279896064 dispatch_meta_bytes=230056 combine_payload_bytes=279896064 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280126120 ingress_bytes=389996544 link_bytes=389996544 token_bytes=560022184 weight_movement_bytes=110100480 total_nvlink_bytes=670122664 weight_to_token_ratio=0.196600 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.782681334e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280154112 dispatch_meta_bytes=229632 combine_payload_bytes=280154112 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=280383744 ingress_bytes=610455552 link_bytes=610455552 token_bytes=560537856 weight_movement_bytes=330301440 total_nvlink_bytes=890839296 weight_to_token_ratio=0.589258 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061682608e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278075392 dispatch_meta_bytes=228032 combine_payload_bytes=278075392 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278303424 ingress_bytes=388175872 link_bytes=388175872 token_bytes=556378816 weight_movement_bytes=110100480 total_nvlink_bytes=666479296 weight_to_token_ratio=0.197888 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.751016854e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44498 payload_rows_remote=38982 dup_rows=21038 dup_rows_remote=18492 dedup_ratio=0.321014 router_tokens_per_expert=[496, 503, 542, 498, 525, 480, 537, 502, 482, 466, 545, 513, 520, 505, 489, 511, 520, 553, 524, 495, 541, 506, 474, 537, 494, 462, 505, 511, 501, 539, 517, 522, 519, 549, 500, 530, 486, 502, 489, 498, 490, 510, 520, 509, 507, 517, 514, 542, 531, 510, 522, 510, 530, 492, 501, 551, 532, 538, 519, 488, 507, 542, 510, 488] payload_rows_per_dst=[5540, 5516, 5626, 5486, 5552, 5584, 5604, 5590] dup_rows_per_dst=[2626, 2546, 2674, 2616, 2594, 2634, 2690, 2658] combine_rows_per_src=[5540, 5516, 5626, 5486, 5552, 5584, 5604, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44464 payload_rows_remote=38926 dup_rows=21072 dup_rows_remote=18372 dedup_ratio=0.321533 router_tokens_per_expert=[538, 480, 513, 508, 486, 509, 525, 494, 511, 525, 529, 516, 493, 507, 511, 532, 490, 523, 491, 508, 521, 530, 531, 529, 540, 534, 519, 501, 492, 551, 475, 495, 510, 529, 506, 515, 517, 517, 442, 552, 523, 486, 486, 503, 491, 480, 503, 557, 518, 515, 518, 513, 526, 510, 516, 503, 494, 549, 501, 505, 530, 506, 488, 552] payload_rows_per_dst=[5544, 5488, 5590, 5602, 5592, 5506, 5538, 5604] dup_rows_per_dst=[2562, 2760, 2656, 2612, 2584, 2552, 2700, 2646] combine_rows_per_src=[5544, 5488, 5590, 5602, 5592, 5506, 5538, 5604] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44456 payload_rows_remote=38920 dup_rows=21080 dup_rows_remote=18398 dedup_ratio=0.321655 router_tokens_per_expert=[541, 482, 515, 511, 488, 511, 530, 498, 510, 527, 531, 519, 492, 511, 511, 532, 489, 524, 492, 509, 522, 532, 534, 528, 541, 535, 520, 501, 492, 551, 474, 494, 509, 530, 506, 516, 517, 516, 442, 551, 523, 485, 484, 503, 488, 479, 502, 555, 518, 514, 518, 508, 526, 510, 517, 498, 494, 545, 498, 502, 530, 503, 483, 551] payload_rows_per_dst=[5556, 5492, 5594, 5604, 5590, 5496, 5536, 5588] dup_rows_per_dst=[2596, 2774, 2666, 2612, 2584, 2542, 2682, 2624] combine_rows_per_src=[5556, 5492, 5594, 5604, 5590, 5496, 5536, 5588] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277645312 dispatch_meta_bytes=228416 combine_payload_bytes=277645312 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277873728 ingress_bytes=387745792 link_bytes=387745792 token_bytes=555519040 weight_movement_bytes=110100480 total_nvlink_bytes=665619520 weight_to_token_ratio=0.198194 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.743537055e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44570 payload_rows_remote=38972 dup_rows=20966 dup_rows_remote=18420 dedup_ratio=0.319916 router_tokens_per_expert=[526, 516, 529, 497, 479, 515, 478, 487, 527, 520, 520, 489, 502, 535, 516, 529, 500, 516, 526, 536, 565, 491, 536, 515, 517, 545, 510, 552, 499, 522, 505, 518, 506, 491, 502, 499, 530, 527, 490, 521, 516, 473, 534, 515, 502, 472, 490, 502, 520, 551, 501, 543, 510, 511, 500, 509, 504, 523, 509, 488, 508, 512, 490, 501] payload_rows_per_dst=[5598, 5656, 5708, 5634, 5448, 5422, 5596, 5508] dup_rows_per_dst=[2546, 2646, 2752, 2826, 2470, 2586, 2578, 2562] combine_rows_per_src=[5598, 5656, 5708, 5634, 5448, 5422, 5596, 5508] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279322624 dispatch_meta_bytes=229784 combine_payload_bytes=279322624 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279552408 ingress_bytes=389423104 link_bytes=389423104 token_bytes=558875032 weight_movement_bytes=110100480 total_nvlink_bytes=668975512 weight_to_token_ratio=0.197004 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.772708270e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278347776 dispatch_meta_bytes=229224 combine_payload_bytes=278347776 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278577000 ingress_bytes=608649216 link_bytes=608649216 token_bytes=556924776 weight_movement_bytes=330301440 total_nvlink_bytes=887226216 weight_to_token_ratio=0.593081 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058541092e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278992896 dispatch_meta_bytes=229336 combine_payload_bytes=278992896 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279222232 ingress_bytes=499193856 link_bytes=499193856 token_bytes=558215128 weight_movement_bytes=220200960 total_nvlink_bytes=778416088 weight_to_token_ratio=0.394473 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.681802189e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=281200640 dispatch_meta_bytes=229672 combine_payload_bytes=281200640 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281430312 ingress_bytes=391301120 link_bytes=391301120 token_bytes=562630952 weight_movement_bytes=110100480 total_nvlink_bytes=672731432 weight_to_token_ratio=0.195689 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.805370057e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279996416 dispatch_meta_bytes=229760 combine_payload_bytes=279996416 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280226176 ingress_bytes=390096896 link_bytes=390096896 token_bytes=560222592 weight_movement_bytes=110100480 total_nvlink_bytes=670323072 weight_to_token_ratio=0.196530 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.784426621e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278978560 dispatch_meta_bytes=229272 combine_payload_bytes=278978560 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279207832 ingress_bytes=609280000 link_bytes=609280000 token_bytes=558186392 weight_movement_bytes=330301440 total_nvlink_bytes=888487832 weight_to_token_ratio=0.591740 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059638129e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279351296 dispatch_meta_bytes=229568 combine_payload_bytes=279351296 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279580864 ingress_bytes=499552256 link_bytes=499552256 token_bytes=558932160 weight_movement_bytes=220200960 total_nvlink_bytes=779133120 weight_to_token_ratio=0.393967 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.688035354e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44350 payload_rows_remote=38670 dup_rows=21186 dup_rows_remote=18546 dedup_ratio=0.323273 router_tokens_per_expert=[521, 537, 503, 504, 522, 508, 542, 505, 508, 513, 502, 493, 513, 526, 494, 520, 491, 526, 503, 506, 540, 499, 527, 537, 516, 497, 501, 547, 500, 471, 545, 516, 504, 547, 498, 510, 518, 505, 524, 524, 483, 509, 478, 490, 495, 475, 510, 488, 554, 511, 516, 552, 458, 506, 534, 512, 519, 527, 536, 523, 508, 510, 506, 505] payload_rows_per_dst=[5152, 5478, 5550, 5360, 5714, 5756, 5660, 5680] dup_rows_per_dst=[2168, 2608, 2520, 2398, 2974, 3064, 2814, 2640] combine_rows_per_src=[5152, 5478, 5550, 5360, 5714, 5756, 5660, 5680] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44546 payload_rows_remote=38910 dup_rows=20990 dup_rows_remote=18286 dedup_ratio=0.320282 router_tokens_per_expert=[557, 525, 533, 513, 524, 500, 545, 468, 510, 527, 540, 500, 533, 539, 499, 522, 477, 531, 528, 509, 512, 563, 524, 492, 459, 537, 529, 530, 502, 493, 504, 531, 526, 529, 501, 525, 562, 507, 469, 476, 528, 505, 489, 473, 493, 493, 480, 543, 494, 520, 525, 507, 496, 488, 494, 529, 529, 533, 470, 527, 470, 491, 551, 489] payload_rows_per_dst=[5602, 5636, 5586, 5508, 5556, 5480, 5552, 5626] dup_rows_per_dst=[2728, 2704, 2686, 2662, 2634, 2528, 2554, 2494] combine_rows_per_src=[5602, 5636, 5586, 5508, 5556, 5480, 5552, 5626] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44438 payload_rows_remote=38858 dup_rows=21098 dup_rows_remote=18470 dedup_ratio=0.321930 router_tokens_per_expert=[545, 477, 518, 488, 531, 507, 525, 536, 502, 514, 527, 523, 515, 538, 462, 498, 500, 498, 534, 526, 489, 525, 488, 527, 536, 462, 501, 577, 538, 535, 526, 506, 555, 495, 542, 502, 491, 496, 523, 500, 493, 475, 531, 497, 528, 482, 511, 499, 518, 507, 494, 501, 496, 513, 504, 490, 538, 510, 529, 476, 524, 548, 543, 483] payload_rows_per_dst=[5608, 5576, 5500, 5598, 5580, 5518, 5482, 5576] dup_rows_per_dst=[2646, 2582, 2674, 2764, 2628, 2514, 2564, 2726] combine_rows_per_src=[5608, 5576, 5500, 5598, 5580, 5518, 5482, 5576] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44354 payload_rows_remote=38874 dup_rows=21182 dup_rows_remote=18614 dedup_ratio=0.323212 router_tokens_per_expert=[521, 527, 522, 526, 520, 499, 524, 515, 512, 530, 514, 532, 500, 504, 521, 505, 486, 513, 505, 499, 539, 515, 510, 516, 504, 515, 541, 541, 516, 514, 538, 514, 503, 523, 472, 514, 473, 515, 504, 536, 529, 506, 497, 493, 489, 536, 450, 524, 507, 501, 476, 507, 543, 509, 529, 492, 523, 499, 504, 497, 517, 551, 517, 494] payload_rows_per_dst=[5574, 5642, 5506, 5562, 5516, 5480, 5574, 5500] dup_rows_per_dst=[2734, 2594, 2660, 2804, 2564, 2568, 2554, 2704] combine_rows_per_src=[5574, 5642, 5506, 5562, 5516, 5480, 5574, 5500] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44402 payload_rows_remote=38902 dup_rows=21134 dup_rows_remote=18536 dedup_ratio=0.322479 router_tokens_per_expert=[493, 528, 517, 546, 535, 489, 513, 498, 520, 517, 519, 535, 540, 531, 494, 533, 488, 558, 504, 524, 519, 437, 526, 493, 530, 531, 535, 540, 476, 501, 481, 525, 544, 504, 589, 515, 482, 494, 504, 486, 495, 498, 492, 499, 526, 496, 527, 517, 482, 482, 495, 516, 543, 520, 502, 514, 494, 526, 512, 494, 496, 517, 524, 507] payload_rows_per_dst=[5514, 5662, 5500, 5578, 5540, 5496, 5570, 5542] dup_rows_per_dst=[2724, 2716, 2598, 2660, 2696, 2604, 2538, 2598] combine_rows_per_src=[5514, 5662, 5500, 5578, 5540, 5496, 5570, 5542] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44360 payload_rows_remote=38802 dup_rows=21176 dup_rows_remote=18482 dedup_ratio=0.323120 router_tokens_per_expert=[489, 524, 521, 534, 518, 531, 488, 516, 506, 493, 524, 472, 545, 502, 503, 528, 529, 525, 498, 518, 519, 511, 534, 472, 494, 479, 482, 512, 548, 550, 512, 541, 481, 491, 513, 498, 523, 522, 531, 501, 520, 531, 474, 504, 488, 525, 491, 528, 489, 506, 508, 522, 507, 517, 562, 515, 456, 537, 527, 511, 504, 536, 525, 507] payload_rows_per_dst=[5560, 5544, 5570, 5540, 5534, 5522, 5558, 5532] dup_rows_per_dst=[2682, 2602, 2642, 2696, 2586, 2600, 2694, 2674] combine_rows_per_src=[5560, 5544, 5570, 5540, 5534, 5522, 5558, 5532] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44448 payload_rows_remote=38980 dup_rows=21088 dup_rows_remote=18458 dedup_ratio=0.321777 router_tokens_per_expert=[564, 518, 499, 500, 538, 508, 509, 522, 513, 492, 543, 511, 524, 526, 525, 494, 503, 514, 510, 523, 522, 474, 558, 490, 501, 515, 507, 505, 530, 474, 510, 507, 493, 503, 514, 460, 539, 523, 501, 505, 537, 511, 530, 526, 490, 512, 497, 512, 539, 486, 491, 534, 515, 477, 523, 506, 479, 504, 518, 524, 470, 548, 524, 548] payload_rows_per_dst=[5590, 5606, 5556, 5468, 5548, 5598, 5566, 5516] dup_rows_per_dst=[2726, 2650, 2632, 2630, 2528, 2632, 2576, 2714] combine_rows_per_src=[5590, 5606, 5556, 5468, 5548, 5598, 5566, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44482 payload_rows_remote=38890 dup_rows=21054 dup_rows_remote=18478 dedup_ratio=0.321259 router_tokens_per_expert=[529, 484, 498, 512, 540, 493, 534, 500, 483, 496, 552, 494, 518, 538, 495, 501, 514, 523, 484, 496, 514, 540, 511, 506, 507, 513, 549, 517, 502, 486, 503, 513, 507, 507, 517, 552, 536, 502, 515, 514, 518, 502, 512, 516, 505, 489, 542, 508, 510, 485, 521, 515, 562, 513, 532, 537, 475, 467, 506, 499, 509, 520, 512, 518] payload_rows_per_dst=[5550, 5540, 5472, 5638, 5518, 5586, 5586, 5592] dup_rows_per_dst=[2630, 2614, 2454, 2728, 2630, 2778, 2644, 2576] combine_rows_per_src=[5550, 5540, 5472, 5638, 5518, 5586, 5586, 5592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44510 payload_rows_remote=38924 dup_rows=21026 dup_rows_remote=18420 dedup_ratio=0.320831 router_tokens_per_expert=[518, 512, 470, 537, 526, 527, 504, 487, 490, 515, 480, 496, 519, 491, 517, 516, 503, 533, 523, 515, 507, 515, 512, 488, 505, 532, 494, 506, 529, 502, 469, 522, 479, 540, 535, 509, 505, 502, 577, 577, 520, 529, 504, 545, 535, 474, 531, 498, 539, 474, 515, 470, 497, 513, 525, 524, 539, 532, 535, 488, 524, 479, 523, 471] payload_rows_per_dst=[5578, 5494, 5586, 5564, 5632, 5608, 5494, 5554] dup_rows_per_dst=[2584, 2554, 2606, 2554, 2816, 2664, 2620, 2628] combine_rows_per_src=[5578, 5494, 5586, 5564, 5632, 5608, 5494, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44620 payload_rows_remote=39066 dup_rows=20916 dup_rows_remote=18368 dedup_ratio=0.319153 router_tokens_per_expert=[514, 507, 506, 501, 564, 520, 554, 526, 536, 495, 483, 536, 535, 507, 508, 548, 485, 537, 462, 511, 504, 532, 490, 534, 534, 527, 456, 509, 522, 464, 512, 527, 506, 488, 506, 517, 527, 506, 510, 494, 511, 514, 528, 505, 520, 504, 546, 533, 468, 501, 527, 516, 506, 492, 512, 505, 492, 515, 510, 503, 542, 524, 515, 479] payload_rows_per_dst=[5664, 5624, 5568, 5554, 5502, 5612, 5474, 5622] dup_rows_per_dst=[2720, 2672, 2542, 2548, 2606, 2710, 2580, 2538] combine_rows_per_src=[5664, 5624, 5568, 5554, 5502, 5612, 5474, 5622] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279422976 dispatch_meta_bytes=229896 combine_payload_bytes=279422976 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279652872 ingress_bytes=389523456 link_bytes=389523456 token_bytes=559075848 weight_movement_bytes=110100480 total_nvlink_bytes=669176328 weight_to_token_ratio=0.196933 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.774453556e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44826 payload_rows_remote=39232 dup_rows=20710 dup_rows_remote=18178 dedup_ratio=0.316010 router_tokens_per_expert=[474, 521, 523, 521, 534, 509, 516, 521, 493, 530, 484, 500, 520, 507, 496, 515, 514, 523, 519, 546, 492, 488, 487, 536, 515, 507, 501, 478, 519, 516, 491, 529, 518, 514, 512, 500, 496, 533, 560, 486, 483, 514, 514, 486, 496, 550, 512, 508, 503, 509, 502, 525, 528, 497, 499, 508, 523, 509, 536, 498, 529, 511, 528, 556] payload_rows_per_dst=[5598, 5528, 5578, 5546, 5702, 5594, 5554, 5726] dup_rows_per_dst=[2640, 2562, 2632, 2566, 2536, 2532, 2588, 2654] combine_rows_per_src=[5598, 5528, 5578, 5546, 5702, 5594, 5554, 5726] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279021568 dispatch_meta_bytes=229192 combine_payload_bytes=279021568 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279250760 ingress_bytes=609323008 link_bytes=609323008 token_bytes=558272328 weight_movement_bytes=330301440 total_nvlink_bytes=888573768 weight_to_token_ratio=0.591649 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059712927e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44398 payload_rows_remote=38818 dup_rows=21138 dup_rows_remote=18484 dedup_ratio=0.322540 router_tokens_per_expert=[514, 516, 489, 551, 514, 485, 498, 528, 513, 509, 513, 483, 545, 527, 500, 501, 485, 528, 535, 498, 558, 476, 539, 481, 514, 504, 485, 519, 524, 523, 524, 506, 509, 542, 468, 491, 526, 542, 551, 488, 529, 473, 502, 514, 514, 545, 501, 497, 531, 469, 507, 513, 526, 554, 554, 515, 477, 517, 512, 517, 507, 498, 510, 484] payload_rows_per_dst=[5532, 5546, 5600, 5572, 5580, 5548, 5572, 5448] dup_rows_per_dst=[2658, 2636, 2600, 2626, 2654, 2602, 2766, 2596] combine_rows_per_src=[5532, 5546, 5600, 5572, 5580, 5548, 5572, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44584 payload_rows_remote=39008 dup_rows=20952 dup_rows_remote=18418 dedup_ratio=0.319702 router_tokens_per_expert=[522, 513, 526, 496, 476, 515, 476, 484, 527, 517, 516, 490, 502, 533, 513, 525, 499, 514, 524, 537, 564, 488, 536, 514, 516, 544, 509, 550, 502, 520, 504, 517, 506, 492, 501, 497, 534, 528, 489, 519, 518, 473, 536, 516, 503, 475, 489, 503, 523, 553, 506, 545, 512, 511, 503, 515, 505, 526, 510, 490, 511, 514, 492, 504] payload_rows_per_dst=[5576, 5698, 5670, 5614, 5498, 5424, 5584, 5520] dup_rows_per_dst=[2534, 2758, 2682, 2710, 2540, 2602, 2542, 2584] combine_rows_per_src=[5576, 5698, 5670, 5614, 5498, 5424, 5584, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278763520 dispatch_meta_bytes=229472 combine_payload_bytes=278763520 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278992992 ingress_bytes=388864000 link_bytes=388864000 token_bytes=557756512 weight_movement_bytes=110100480 total_nvlink_bytes=667856992 weight_to_token_ratio=0.197399 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.762984531e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279007232 dispatch_meta_bytes=229376 combine_payload_bytes=279007232 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279236608 ingress_bytes=609308672 link_bytes=609308672 token_bytes=558243840 weight_movement_bytes=330301440 total_nvlink_bytes=888545280 weight_to_token_ratio=0.591680 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059687995e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=280025088 dispatch_meta_bytes=229736 combine_payload_bytes=280025088 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280254824 ingress_bytes=390125568 link_bytes=390125568 token_bytes=560279912 weight_movement_bytes=110100480 total_nvlink_bytes=670380392 weight_to_token_ratio=0.196510 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.784925274e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=281214976 dispatch_meta_bytes=229640 combine_payload_bytes=281214976 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281444616 ingress_bytes=391315456 link_bytes=391315456 token_bytes=562659592 weight_movement_bytes=110100480 total_nvlink_bytes=672760072 weight_to_token_ratio=0.195679 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.805619383e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278247424 dispatch_meta_bytes=229208 combine_payload_bytes=278247424 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278476632 ingress_bytes=608548864 link_bytes=608548864 token_bytes=556724056 weight_movement_bytes=330301440 total_nvlink_bytes=887025496 weight_to_token_ratio=0.593295 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058366564e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44534 payload_rows_remote=38908 dup_rows=21002 dup_rows_remote=18324 dedup_ratio=0.320465 router_tokens_per_expert=[557, 521, 530, 511, 525, 495, 540, 467, 506, 525, 537, 501, 529, 535, 498, 521, 475, 531, 525, 508, 509, 560, 520, 494, 460, 536, 531, 529, 505, 494, 501, 532, 526, 527, 500, 526, 564, 509, 470, 479, 529, 505, 489, 473, 497, 493, 481, 543, 495, 521, 527, 513, 501, 489, 498, 533, 532, 534, 471, 528, 473, 492, 552, 490] payload_rows_per_dst=[5582, 5626, 5574, 5506, 5566, 5484, 5568, 5628] dup_rows_per_dst=[2710, 2678, 2670, 2670, 2636, 2536, 2586, 2516] combine_rows_per_src=[5582, 5626, 5574, 5506, 5566, 5484, 5568, 5628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279609344 dispatch_meta_bytes=229704 combine_payload_bytes=279609344 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279839048 ingress_bytes=389709824 link_bytes=389709824 token_bytes=559448392 weight_movement_bytes=110100480 total_nvlink_bytes=669548872 weight_to_token_ratio=0.196802 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.777694802e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44342 payload_rows_remote=38772 dup_rows=21194 dup_rows_remote=18484 dedup_ratio=0.323395 router_tokens_per_expert=[487, 521, 520, 528, 515, 531, 486, 514, 502, 489, 521, 475, 543, 501, 500, 528, 526, 524, 495, 515, 521, 507, 530, 469, 489, 477, 485, 515, 550, 551, 514, 542, 481, 492, 514, 498, 522, 520, 534, 499, 519, 531, 475, 506, 490, 527, 490, 532, 490, 509, 512, 523, 507, 516, 566, 517, 459, 538, 531, 511, 510, 541, 528, 509] payload_rows_per_dst=[5546, 5526, 5556, 5538, 5532, 5524, 5570, 5550] dup_rows_per_dst=[2658, 2592, 2618, 2708, 2588, 2616, 2710, 2704] combine_rows_per_src=[5546, 5526, 5556, 5538, 5532, 5524, 5570, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278892544 dispatch_meta_bytes=228928 combine_payload_bytes=278892544 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279121472 ingress_bytes=388993024 link_bytes=388993024 token_bytes=558014016 weight_movement_bytes=110100480 total_nvlink_bytes=668114496 weight_to_token_ratio=0.197308 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.765228471e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44366 payload_rows_remote=38546 dup_rows=21170 dup_rows_remote=18310 dedup_ratio=0.323029 router_tokens_per_expert=[517, 534, 498, 499, 522, 504, 541, 505, 505, 513, 504, 491, 512, 524, 492, 519, 490, 526, 503, 502, 539, 496, 527, 538, 515, 497, 500, 542, 500, 472, 547, 516, 502, 551, 499, 509, 519, 508, 525, 522, 481, 511, 479, 492, 494, 473, 512, 490, 555, 512, 518, 556, 462, 510, 534, 513, 522, 530, 538, 525, 509, 514, 506, 507] payload_rows_per_dst=[5324, 5488, 5634, 5418, 5680, 5402, 5600, 5820] dup_rows_per_dst=[2452, 2632, 2608, 2504, 2932, 2462, 2720, 2860] combine_rows_per_src=[5324, 5488, 5634, 5418, 5680, 5402, 5600, 5820] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44398 payload_rows_remote=38902 dup_rows=21138 dup_rows_remote=18552 dedup_ratio=0.322540 router_tokens_per_expert=[490, 524, 510, 543, 532, 486, 512, 496, 515, 514, 515, 535, 539, 531, 492, 534, 487, 557, 502, 524, 519, 438, 524, 490, 531, 528, 536, 541, 479, 501, 480, 522, 543, 502, 590, 517, 482, 496, 504, 490, 493, 497, 493, 500, 529, 496, 528, 517, 484, 482, 498, 517, 544, 523, 502, 519, 499, 526, 514, 494, 501, 520, 531, 510] payload_rows_per_dst=[5492, 5646, 5496, 5574, 5546, 5494, 5584, 5566] dup_rows_per_dst=[2694, 2704, 2586, 2662, 2702, 2612, 2554, 2624] combine_rows_per_src=[5492, 5646, 5496, 5574, 5546, 5494, 5584, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44470 payload_rows_remote=38998 dup_rows=21066 dup_rows_remote=18450 dedup_ratio=0.321442 router_tokens_per_expert=[561, 516, 497, 495, 536, 505, 504, 517, 506, 488, 540, 509, 522, 526, 523, 493, 503, 513, 509, 521, 518, 473, 555, 487, 500, 512, 509, 504, 530, 473, 509, 507, 493, 505, 517, 460, 544, 525, 503, 506, 537, 513, 531, 527, 495, 513, 497, 515, 542, 486, 491, 536, 517, 478, 525, 509, 486, 506, 525, 526, 473, 551, 525, 550] payload_rows_per_dst=[5580, 5592, 5540, 5472, 5560, 5618, 5568, 5540] dup_rows_per_dst=[2682, 2622, 2618, 2616, 2546, 2638, 2600, 2744] combine_rows_per_src=[5580, 5592, 5540, 5472, 5560, 5618, 5568, 5540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44352 payload_rows_remote=38860 dup_rows=21184 dup_rows_remote=18610 dedup_ratio=0.323242 router_tokens_per_expert=[518, 526, 515, 523, 517, 497, 523, 513, 511, 529, 507, 530, 496, 504, 519, 504, 487, 509, 501, 501, 541, 515, 508, 515, 501, 514, 541, 538, 517, 516, 539, 512, 501, 521, 473, 514, 472, 516, 506, 538, 529, 508, 496, 494, 489, 539, 452, 526, 509, 502, 478, 511, 543, 513, 532, 495, 525, 501, 505, 498, 521, 553, 520, 501] payload_rows_per_dst=[5562, 5632, 5498, 5554, 5510, 5492, 5592, 5512] dup_rows_per_dst=[2702, 2568, 2656, 2802, 2572, 2574, 2574, 2736] combine_rows_per_src=[5562, 5632, 5498, 5554, 5510, 5492, 5592, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277917696 dispatch_meta_bytes=229024 combine_payload_bytes=277917696 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=278146720 ingress_bytes=498118656 link_bytes=498118656 token_bytes=556064416 weight_movement_bytes=220200960 total_nvlink_bytes=776265376 weight_to_token_ratio=0.395999 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.663102692e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44438 payload_rows_remote=38854 dup_rows=21098 dup_rows_remote=18460 dedup_ratio=0.321930 router_tokens_per_expert=[538, 477, 517, 484, 528, 504, 522, 534, 501, 507, 523, 522, 512, 534, 459, 496, 500, 497, 533, 524, 485, 525, 486, 529, 537, 461, 502, 575, 538, 535, 527, 504, 556, 497, 542, 499, 494, 495, 526, 502, 494, 476, 533, 500, 528, 486, 511, 502, 521, 506, 495, 502, 496, 514, 506, 493, 538, 516, 535, 483, 526, 551, 545, 484] payload_rows_per_dst=[5586, 5552, 5496, 5596, 5584, 5528, 5494, 5602] dup_rows_per_dst=[2622, 2556, 2662, 2762, 2638, 2532, 2572, 2754] combine_rows_per_src=[5586, 5552, 5496, 5596, 5584, 5528, 5494, 5602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44466 payload_rows_remote=38854 dup_rows=21070 dup_rows_remote=18338 dedup_ratio=0.321503 router_tokens_per_expert=[494, 516, 508, 505, 497, 548, 582, 502, 512, 493, 489, 530, 497, 545, 493, 487, 525, 553, 515, 475, 535, 503, 556, 513, 507, 513, 500, 480, 533, 535, 505, 504, 483, 523, 469, 507, 461, 516, 498, 515, 517, 492, 526, 512, 505, 525, 494, 527, 517, 512, 516, 489, 544, 527, 479, 560, 507, 503, 505, 532, 543, 478, 510, 526] payload_rows_per_dst=[5612, 5550, 5608, 5542, 5428, 5850, 5604, 5272] dup_rows_per_dst=[2732, 2572, 2716, 2612, 2516, 3174, 2640, 2108] combine_rows_per_src=[5612, 5550, 5608, 5542, 5428, 5850, 5604, 5272] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44510 payload_rows_remote=38828 dup_rows=21026 dup_rows_remote=18180 dedup_ratio=0.320831 router_tokens_per_expert=[498, 521, 508, 504, 498, 549, 584, 506, 515, 496, 489, 535, 498, 546, 492, 486, 529, 553, 519, 477, 536, 503, 556, 512, 509, 513, 500, 480, 535, 535, 505, 506, 482, 525, 470, 507, 458, 517, 498, 512, 517, 490, 522, 511, 505, 524, 492, 527, 515, 512, 516, 487, 544, 526, 479, 561, 503, 502, 502, 530, 535, 476, 506, 524] payload_rows_per_dst=[5682, 5482, 5612, 5580, 5426, 5564, 5616, 5548] dup_rows_per_dst=[2846, 2454, 2744, 2728, 2512, 2612, 2664, 2466] combine_rows_per_src=[5682, 5482, 5612, 5580, 5426, 5564, 5616, 5548] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277186560 dispatch_meta_bytes=228864 combine_payload_bytes=277186560 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277415424 ingress_bytes=497387520 link_bytes=497387520 token_bytes=554601984 weight_movement_bytes=220200960 total_nvlink_bytes=774802944 weight_to_token_ratio=0.397043 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.650387035e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278906880 dispatch_meta_bytes=228784 combine_payload_bytes=278906880 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279135664 ingress_bytes=609208320 link_bytes=609208320 token_bytes=558042544 weight_movement_bytes=330301440 total_nvlink_bytes=888343984 weight_to_token_ratio=0.591893 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059513466e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278534144 dispatch_meta_bytes=229312 combine_payload_bytes=278534144 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278763456 ingress_bytes=388634624 link_bytes=388634624 token_bytes=557297600 weight_movement_bytes=110100480 total_nvlink_bytes=667398080 weight_to_token_ratio=0.197561 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.758995305e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278648832 dispatch_meta_bytes=229952 combine_payload_bytes=278648832 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278878784 ingress_bytes=388749312 link_bytes=388749312 token_bytes=557527616 weight_movement_bytes=110100480 total_nvlink_bytes=667628096 weight_to_token_ratio=0.197480 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.760989918e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278849536 dispatch_meta_bytes=229752 combine_payload_bytes=278849536 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279079288 ingress_bytes=499050496 link_bytes=499050496 token_bytes=557928824 weight_movement_bytes=220200960 total_nvlink_bytes=778129784 weight_to_token_ratio=0.394676 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.679308922e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278132736 dispatch_meta_bytes=229136 combine_payload_bytes=278132736 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278361872 ingress_bytes=388233216 link_bytes=388233216 token_bytes=556494608 weight_movement_bytes=110100480 total_nvlink_bytes=666595088 weight_to_token_ratio=0.197846 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.752014160e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279408640 dispatch_meta_bytes=229752 combine_payload_bytes=279408640 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279638392 ingress_bytes=499609600 link_bytes=499609600 token_bytes=559047032 weight_movement_bytes=220200960 total_nvlink_bytes=779247992 weight_to_token_ratio=0.393886 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.689032661e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278319104 dispatch_meta_bytes=228032 combine_payload_bytes=278319104 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278547136 ingress_bytes=498520064 link_bytes=498520064 token_bytes=556866240 weight_movement_bytes=220200960 total_nvlink_bytes=777067200 weight_to_token_ratio=0.395429 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.670083838e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44754 payload_rows_remote=38986 dup_rows=20782 dup_rows_remote=17878 dedup_ratio=0.317108 router_tokens_per_expert=[527, 516, 502, 501, 480, 527, 502, 514, 489, 481, 498, 514, 541, 517, 510, 517, 520, 521, 554, 505, 519, 551, 554, 526, 513, 514, 514, 512, 545, 507, 517, 503, 508, 520, 524, 549, 497, 543, 506, 498, 516, 534, 491, 517, 484, 482, 505, 528, 468, 565, 511, 507, 483, 511, 508, 510, 491, 503, 526, 466, 463, 481, 543, 519] payload_rows_per_dst=[5324, 5560, 5450, 5652, 5670, 5522, 5808, 5768] dup_rows_per_dst=[2228, 2574, 2270, 2646, 2620, 2636, 2904, 2904] combine_rows_per_src=[5324, 5560, 5450, 5652, 5670, 5522, 5808, 5768] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44548 payload_rows_remote=39094 dup_rows=20988 dup_rows_remote=18396 dedup_ratio=0.320251 router_tokens_per_expert=[547, 535, 541, 533, 502, 498, 502, 524, 527, 508, 488, 464, 500, 504, 520, 512, 544, 530, 511, 492, 496, 485, 525, 521, 521, 503, 519, 492, 510, 511, 477, 503, 513, 516, 515, 532, 546, 518, 509, 518, 481, 516, 498, 499, 520, 550, 554, 566, 509, 500, 507, 513, 504, 503, 500, 473, 531, 484, 497, 489, 504, 512, 545, 501] payload_rows_per_dst=[5690, 5454, 5540, 5534, 5624, 5614, 5516, 5576] dup_rows_per_dst=[2674, 2592, 2668, 2538, 2710, 2754, 2502, 2550] combine_rows_per_src=[5690, 5454, 5540, 5534, 5624, 5614, 5516, 5576] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44336 payload_rows_remote=38750 dup_rows=21200 dup_rows_remote=18654 dedup_ratio=0.323486 router_tokens_per_expert=[498, 523, 510, 464, 487, 512, 533, 536, 572, 485, 499, 536, 546, 506, 536, 534, 493, 467, 522, 515, 509, 546, 538, 506, 524, 502, 501, 532, 502, 498, 499, 485, 490, 490, 531, 490, 538, 484, 520, 523, 486, 531, 510, 528, 537, 509, 502, 525, 509, 535, 508, 516, 496, 505, 512, 499, 528, 521, 512, 498, 501, 508, 492, 518] payload_rows_per_dst=[5530, 5602, 5550, 5528, 5586, 5516, 5558, 5466] dup_rows_per_dst=[2596, 2826, 2642, 2558, 2546, 2740, 2602, 2690] combine_rows_per_src=[5530, 5602, 5550, 5528, 5586, 5516, 5558, 5466] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44436 payload_rows_remote=38904 dup_rows=21100 dup_rows_remote=18534 dedup_ratio=0.321960 router_tokens_per_expert=[527, 473, 513, 540, 559, 509, 541, 512, 465, 506, 503, 537, 527, 523, 498, 489, 525, 488, 500, 537, 489, 505, 475, 530, 535, 517, 499, 552, 523, 510, 512, 485, 504, 502, 507, 494, 491, 518, 461, 527, 506, 533, 535, 515, 531, 511, 532, 513, 514, 493, 550, 499, 505, 525, 524, 520, 492, 499, 505, 469, 525, 512, 531, 521] payload_rows_per_dst=[5596, 5560, 5532, 5632, 5456, 5554, 5538, 5568] dup_rows_per_dst=[2752, 2536, 2566, 2634, 2552, 2798, 2722, 2540] combine_rows_per_src=[5596, 5560, 5532, 5632, 5456, 5554, 5538, 5568] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=276297728 dispatch_meta_bytes=227424 combine_payload_bytes=276297728 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=276525152 ingress_bytes=496498688 link_bytes=496498688 token_bytes=552822880 weight_movement_bytes=220200960 total_nvlink_bytes=773023840 weight_to_token_ratio=0.398321 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.634928784e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278849536 dispatch_meta_bytes=229816 combine_payload_bytes=278849536 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279079352 ingress_bytes=499050496 link_bytes=499050496 token_bytes=557928888 weight_movement_bytes=220200960 total_nvlink_bytes=778129848 weight_to_token_ratio=0.394676 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.679308922e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279537664 dispatch_meta_bytes=229792 combine_payload_bytes=279537664 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279767456 ingress_bytes=499738624 link_bytes=499738624 token_bytes=559305120 weight_movement_bytes=220200960 total_nvlink_bytes=779506080 weight_to_token_ratio=0.393705 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.691276600e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278548480 dispatch_meta_bytes=229880 combine_payload_bytes=278548480 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278778360 ingress_bytes=388648960 link_bytes=388648960 token_bytes=557326840 weight_movement_bytes=110100480 total_nvlink_bytes=667427320 weight_to_token_ratio=0.197551 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.759244632e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278505472 dispatch_meta_bytes=229256 combine_payload_bytes=278505472 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278734728 ingress_bytes=388605952 link_bytes=388605952 token_bytes=557240200 weight_movement_bytes=110100480 total_nvlink_bytes=667340680 weight_to_token_ratio=0.197582 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.758496652e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44536 payload_rows_remote=39086 dup_rows=21000 dup_rows_remote=18424 dedup_ratio=0.320435 router_tokens_per_expert=[545, 531, 538, 531, 497, 494, 498, 520, 527, 506, 489, 463, 497, 502, 519, 510, 543, 528, 511, 491, 496, 484, 525, 522, 522, 504, 519, 492, 513, 515, 475, 502, 513, 515, 512, 533, 545, 521, 510, 519, 481, 513, 503, 497, 521, 554, 553, 567, 511, 501, 506, 515, 503, 508, 502, 474, 533, 487, 499, 495, 505, 511, 546, 506] payload_rows_per_dst=[5664, 5450, 5532, 5544, 5624, 5616, 5522, 5584] dup_rows_per_dst=[2644, 2576, 2668, 2540, 2712, 2762, 2518, 2580] combine_rows_per_src=[5664, 5450, 5532, 5544, 5624, 5616, 5522, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44578 payload_rows_remote=39046 dup_rows=20958 dup_rows_remote=18382 dedup_ratio=0.319794 router_tokens_per_expert=[532, 519, 509, 498, 489, 520, 510, 473, 482, 509, 512, 550, 501, 521, 491, 530, 502, 546, 509, 518, 517, 539, 527, 539, 531, 528, 541, 495, 556, 521, 487, 490, 514, 529, 514, 527, 505, 492, 494, 497, 528, 524, 501, 496, 495, 530, 522, 515, 491, 510, 515, 486, 523, 512, 509, 508, 512, 499, 482, 520, 518, 491, 505, 512] payload_rows_per_dst=[5516, 5574, 5676, 5582, 5594, 5562, 5532, 5542] dup_rows_per_dst=[2584, 2618, 2718, 2716, 2550, 2660, 2576, 2536] combine_rows_per_src=[5516, 5574, 5676, 5582, 5594, 5562, 5532, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44418 payload_rows_remote=38906 dup_rows=21118 dup_rows_remote=18554 dedup_ratio=0.322235 router_tokens_per_expert=[523, 470, 511, 538, 556, 507, 541, 510, 463, 504, 499, 534, 523, 523, 494, 487, 526, 486, 501, 533, 486, 504, 474, 528, 536, 516, 498, 551, 521, 508, 515, 486, 505, 502, 505, 495, 494, 520, 462, 529, 505, 535, 535, 516, 533, 512, 533, 515, 517, 494, 553, 503, 506, 527, 524, 520, 493, 501, 506, 472, 528, 514, 535, 527] payload_rows_per_dst=[5582, 5546, 5512, 5624, 5462, 5558, 5552, 5582] dup_rows_per_dst=[2730, 2508, 2564, 2638, 2562, 2810, 2736, 2570] combine_rows_per_src=[5582, 5546, 5512, 5624, 5462, 5558, 5552, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44808 payload_rows_remote=39124 dup_rows=20728 dup_rows_remote=18042 dedup_ratio=0.316284 router_tokens_per_expert=[524, 512, 499, 501, 479, 526, 499, 511, 486, 479, 492, 513, 539, 517, 508, 519, 519, 519, 555, 504, 519, 552, 557, 527, 515, 514, 512, 512, 543, 510, 515, 503, 504, 520, 524, 549, 497, 542, 506, 498, 518, 535, 494, 516, 485, 482, 507, 527, 470, 562, 512, 511, 483, 514, 510, 514, 492, 503, 526, 470, 466, 485, 544, 522] payload_rows_per_dst=[5488, 5546, 5544, 5642, 5668, 5512, 5724, 5684] dup_rows_per_dst=[2442, 2560, 2416, 2606, 2612, 2616, 2790, 2686] combine_rows_per_src=[5488, 5546, 5544, 5642, 5668, 5512, 5724, 5684] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44448 payload_rows_remote=38884 dup_rows=21088 dup_rows_remote=18384 dedup_ratio=0.321777 router_tokens_per_expert=[524, 497, 519, 525, 505, 516, 543, 514, 539, 495, 503, 540, 517, 484, 542, 533, 508, 554, 518, 511, 537, 529, 527, 516, 542, 518, 530, 499, 547, 516, 473, 509, 517, 497, 511, 515, 491, 482, 523, 486, 534, 517, 512, 527, 515, 479, 463, 500, 482, 492, 548, 477, 506, 494, 483, 506, 530, 493, 522, 484, 534, 502, 523, 493] payload_rows_per_dst=[5626, 5590, 5632, 5564, 5532, 5450, 5508, 5546] dup_rows_per_dst=[2660, 2716, 2768, 2704, 2512, 2644, 2468, 2616] combine_rows_per_src=[5626, 5590, 5632, 5564, 5532, 5450, 5508, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278505472 dispatch_meta_bytes=228768 combine_payload_bytes=278505472 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278734240 ingress_bytes=608806912 link_bytes=608806912 token_bytes=557239712 weight_movement_bytes=330301440 total_nvlink_bytes=887541152 weight_to_token_ratio=0.592746 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058815351e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44576 payload_rows_remote=39048 dup_rows=20960 dup_rows_remote=18398 dedup_ratio=0.319824 router_tokens_per_expert=[538, 523, 512, 500, 495, 526, 512, 474, 486, 511, 514, 551, 501, 523, 492, 531, 501, 547, 512, 521, 516, 542, 528, 540, 531, 526, 541, 496, 557, 520, 488, 490, 512, 532, 512, 524, 505, 492, 490, 494, 528, 524, 500, 495, 491, 529, 521, 514, 490, 508, 513, 485, 522, 514, 507, 506, 509, 495, 481, 517, 516, 486, 503, 508] payload_rows_per_dst=[5544, 5586, 5680, 5580, 5588, 5552, 5528, 5518] dup_rows_per_dst=[2616, 2632, 2734, 2718, 2534, 2652, 2562, 2512] combine_rows_per_src=[5544, 5586, 5680, 5580, 5588, 5552, 5528, 5518] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44544 payload_rows_remote=38976 dup_rows=20992 dup_rows_remote=18272 dedup_ratio=0.320312 router_tokens_per_expert=[502, 513, 540, 488, 508, 524, 493, 545, 513, 478, 539, 512, 488, 501, 492, 526, 531, 511, 514, 538, 509, 524, 518, 533, 489, 521, 516, 502, 509, 552, 506, 489, 509, 505, 508, 494, 463, 534, 483, 512, 513, 521, 537, 501, 489, 505, 558, 520, 532, 527, 513, 515, 538, 469, 535, 481, 502, 541, 503, 497, 528, 500, 510, 501] payload_rows_per_dst=[5602, 5536, 5674, 5538, 5454, 5568, 5570, 5602] dup_rows_per_dst=[2624, 2562, 2682, 2630, 2562, 2720, 2650, 2562] combine_rows_per_src=[5602, 5536, 5674, 5538, 5454, 5568, 5570, 5602] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44464 payload_rows_remote=38786 dup_rows=21072 dup_rows_remote=18308 dedup_ratio=0.321533 router_tokens_per_expert=[515, 512, 483, 544, 524, 528, 519, 535, 504, 503, 529, 516, 498, 499, 508, 508, 519, 534, 494, 509, 529, 523, 526, 519, 460, 518, 512, 487, 532, 503, 504, 521, 564, 526, 502, 532, 505, 502, 462, 525, 515, 502, 532, 503, 502, 559, 529, 450, 539, 497, 516, 485, 522, 509, 487, 509, 505, 484, 495, 538, 512, 514, 506, 525] payload_rows_per_dst=[5678, 5490, 5570, 5522, 5670, 5424, 5542, 5568] dup_rows_per_dst=[2764, 2698, 2556, 2552, 2928, 2398, 2586, 2590] combine_rows_per_src=[5678, 5490, 5570, 5522, 5670, 5424, 5542, 5568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=280168448 dispatch_meta_bytes=230040 combine_payload_bytes=280168448 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280398488 ingress_bytes=390268928 link_bytes=390268928 token_bytes=560566936 weight_movement_bytes=110100480 total_nvlink_bytes=670667416 weight_to_token_ratio=0.196409 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.787418540e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279881728 dispatch_meta_bytes=229712 combine_payload_bytes=279881728 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280111440 ingress_bytes=389982208 link_bytes=389982208 token_bytes=559993168 weight_movement_bytes=110100480 total_nvlink_bytes=670093648 weight_to_token_ratio=0.196610 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.782432008e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278878208 dispatch_meta_bytes=229840 combine_payload_bytes=278878208 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=279108048 ingress_bytes=719280128 link_bytes=719280128 token_bytes=557986256 weight_movement_bytes=440401920 total_nvlink_bytes=998388176 weight_to_token_ratio=0.789270 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.250946444e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=280440832 dispatch_meta_bytes=228664 combine_payload_bytes=280440832 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280669496 ingress_bytes=390541312 link_bytes=390541312 token_bytes=561110328 weight_movement_bytes=110100480 total_nvlink_bytes=671210808 weight_to_token_ratio=0.196219 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.792155746e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278720512 dispatch_meta_bytes=229072 combine_payload_bytes=278720512 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278949584 ingress_bytes=388820992 link_bytes=388820992 token_bytes=557670096 weight_movement_bytes=110100480 total_nvlink_bytes=667770576 weight_to_token_ratio=0.197429 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.762236552e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44348 payload_rows_remote=38760 dup_rows=21188 dup_rows_remote=18648 dedup_ratio=0.323303 router_tokens_per_expert=[495, 520, 507, 460, 481, 511, 531, 534, 570, 482, 496, 536, 546, 503, 534, 530, 493, 466, 519, 514, 508, 546, 536, 507, 527, 499, 502, 532, 502, 499, 500, 486, 489, 487, 530, 489, 539, 483, 522, 525, 487, 532, 508, 530, 539, 508, 504, 528, 511, 535, 514, 519, 496, 508, 515, 501, 531, 521, 515, 500, 502, 510, 497, 521] payload_rows_per_dst=[5514, 5580, 5546, 5534, 5588, 5528, 5580, 5478] dup_rows_per_dst=[2564, 2814, 2632, 2560, 2540, 2744, 2618, 2716] combine_rows_per_src=[5514, 5580, 5546, 5534, 5588, 5528, 5580, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44546 payload_rows_remote=38964 dup_rows=20990 dup_rows_remote=18262 dedup_ratio=0.320282 router_tokens_per_expert=[502, 509, 539, 484, 504, 523, 490, 543, 512, 477, 537, 510, 485, 501, 487, 523, 529, 510, 511, 538, 509, 527, 515, 532, 489, 520, 516, 500, 507, 551, 504, 490, 509, 505, 509, 493, 461, 530, 483, 513, 512, 522, 538, 501, 488, 509, 561, 524, 536, 528, 514, 517, 539, 472, 540, 485, 506, 543, 507, 499, 529, 503, 511, 507] payload_rows_per_dst=[5584, 5520, 5670, 5534, 5452, 5582, 5592, 5612] dup_rows_per_dst=[2604, 2544, 2672, 2620, 2554, 2728, 2670, 2598] combine_rows_per_src=[5584, 5520, 5670, 5534, 5452, 5582, 5592, 5612] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44468 payload_rows_remote=38902 dup_rows=21068 dup_rows_remote=18360 dedup_ratio=0.321472 router_tokens_per_expert=[525, 497, 519, 526, 509, 516, 547, 520, 540, 496, 505, 539, 522, 486, 540, 536, 510, 554, 521, 512, 539, 530, 529, 516, 544, 518, 528, 499, 551, 516, 473, 508, 515, 496, 509, 516, 493, 479, 522, 488, 536, 517, 508, 528, 515, 478, 463, 499, 484, 492, 544, 477, 505, 489, 480, 503, 524, 493, 522, 483, 532, 495, 520, 492] payload_rows_per_dst=[5642, 5602, 5644, 5566, 5532, 5448, 5498, 5536] dup_rows_per_dst=[2676, 2726, 2778, 2708, 2504, 2640, 2450, 2586] combine_rows_per_src=[5642, 5602, 5644, 5566, 5532, 5448, 5498, 5536] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279451648 dispatch_meta_bytes=227456 combine_payload_bytes=279451648 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279679104 ingress_bytes=389552128 link_bytes=389552128 token_bytes=559130752 weight_movement_bytes=110100480 total_nvlink_bytes=669231232 weight_to_token_ratio=0.196914 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.774952209e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=280225792 dispatch_meta_bytes=229960 combine_payload_bytes=280225792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280455752 ingress_bytes=390326272 link_bytes=390326272 token_bytes=560681544 weight_movement_bytes=110100480 total_nvlink_bytes=670782024 weight_to_token_ratio=0.196369 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.788415846e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277760000 dispatch_meta_bytes=229616 combine_payload_bytes=277760000 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277989616 ingress_bytes=387860480 link_bytes=387860480 token_bytes=555749616 weight_movement_bytes=110100480 total_nvlink_bytes=665850096 weight_to_token_ratio=0.198112 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.745531668e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278863872 dispatch_meta_bytes=229752 combine_payload_bytes=278863872 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=279093624 ingress_bytes=829366272 link_bytes=829366272 token_bytes=557957496 weight_movement_bytes=550502400 total_nvlink_bytes=1108459896 weight_to_token_ratio=0.986639 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.442404354e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279379968 dispatch_meta_bytes=228992 combine_payload_bytes=279379968 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279608960 ingress_bytes=499580928 link_bytes=499580928 token_bytes=558988928 weight_movement_bytes=220200960 total_nvlink_bytes=779189888 weight_to_token_ratio=0.393927 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.688534007e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279896064 dispatch_meta_bytes=229784 combine_payload_bytes=279896064 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280125848 ingress_bytes=389996544 link_bytes=389996544 token_bytes=560021912 weight_movement_bytes=110100480 total_nvlink_bytes=670122392 weight_to_token_ratio=0.196600 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.782681334e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278018048 dispatch_meta_bytes=228376 combine_payload_bytes=278018048 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278246424 ingress_bytes=498219008 link_bytes=498219008 token_bytes=556264472 weight_movement_bytes=220200960 total_nvlink_bytes=776465432 weight_to_token_ratio=0.395857 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.664847979e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278849536 dispatch_meta_bytes=229048 combine_payload_bytes=278849536 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279078584 ingress_bytes=388950016 link_bytes=388950016 token_bytes=557928120 weight_movement_bytes=110100480 total_nvlink_bytes=668028600 weight_to_token_ratio=0.197338 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.764480491e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44392 payload_rows_remote=38850 dup_rows=21144 dup_rows_remote=18466 dedup_ratio=0.322632 router_tokens_per_expert=[519, 518, 529, 526, 477, 538, 524, 482, 517, 493, 557, 509, 481, 526, 528, 532, 499, 520, 523, 505, 508, 539, 509, 550, 517, 512, 527, 515, 506, 504, 493, 521, 471, 484, 544, 518, 501, 474, 529, 491, 500, 533, 507, 528, 488, 500, 543, 513, 544, 525, 511, 498, 482, 492, 507, 508, 528, 500, 473, 509, 486, 516, 530, 531] payload_rows_per_dst=[5532, 5310, 5382, 5660, 5652, 5716, 5598, 5542] dup_rows_per_dst=[2688, 2468, 2286, 2842, 2888, 2706, 2588, 2678] combine_rows_per_src=[5532, 5310, 5382, 5660, 5652, 5716, 5598, 5542] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44556 payload_rows_remote=39044 dup_rows=20980 dup_rows_remote=18222 dedup_ratio=0.320129 router_tokens_per_expert=[500, 501, 556, 507, 563, 479, 496, 527, 525, 550, 508, 524, 522, 534, 488, 484, 480, 500, 511, 504, 526, 524, 545, 514, 504, 518, 503, 508, 507, 535, 535, 503, 476, 496, 486, 547, 473, 464, 504, 512, 473, 470, 527, 547, 471, 550, 482, 508, 505, 485, 515, 495, 535, 553, 520, 514, 501, 489, 538, 522, 523, 521, 534, 551] payload_rows_per_dst=[5644, 5512, 5526, 5542, 5482, 5572, 5618, 5660] dup_rows_per_dst=[2614, 2758, 2682, 2684, 2434, 2484, 2626, 2698] combine_rows_per_src=[5644, 5512, 5526, 5542, 5482, 5572, 5618, 5660] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44472 payload_rows_remote=38902 dup_rows=21064 dup_rows_remote=18386 dedup_ratio=0.321411 router_tokens_per_expert=[543, 517, 491, 496, 511, 520, 534, 528, 516, 504, 507, 485, 509, 516, 507, 517, 480, 503, 516, 550, 545, 547, 508, 529, 475, 485, 557, 533, 522, 497, 509, 516, 538, 522, 501, 461, 477, 544, 543, 538, 455, 548, 509, 504, 541, 485, 531, 490, 508, 516, 509, 499, 551, 511, 483, 501, 522, 507, 495, 517, 504, 468, 512, 505] payload_rows_per_dst=[5586, 5514, 5646, 5610, 5570, 5500, 5570, 5476] dup_rows_per_dst=[2694, 2608, 2710, 2578, 2678, 2626, 2586, 2584] combine_rows_per_src=[5586, 5514, 5646, 5610, 5570, 5500, 5570, 5476] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44606 payload_rows_remote=39040 dup_rows=20930 dup_rows_remote=18252 dedup_ratio=0.319366 router_tokens_per_expert=[525, 508, 539, 518, 490, 536, 523, 485, 539, 522, 503, 496, 488, 515, 531, 495, 545, 457, 526, 532, 541, 499, 528, 494, 528, 516, 472, 494, 527, 550, 498, 520, 480, 512, 482, 528, 519, 555, 521, 541, 479, 527, 493, 510, 531, 518, 502, 511, 534, 500, 494, 486, 499, 499, 551, 513, 529, 498, 534, 511, 479, 482, 500, 510] payload_rows_per_dst=[5648, 5594, 5566, 5604, 5622, 5492, 5546, 5534] dup_rows_per_dst=[2600, 2584, 2678, 2606, 2654, 2650, 2606, 2552] combine_rows_per_src=[5648, 5594, 5566, 5604, 5622, 5492, 5546, 5534] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44870 payload_rows_remote=39208 dup_rows=20666 dup_rows_remote=18016 dedup_ratio=0.315338 router_tokens_per_expert=[519, 508, 503, 531, 486, 503, 524, 494, 523, 500, 516, 527, 539, 523, 525, 507, 493, 501, 530, 530, 514, 518, 519, 469, 502, 511, 538, 491, 477, 545, 512, 523, 521, 537, 512, 458, 515, 489, 498, 512, 516, 426, 497, 504, 506, 548, 492, 527, 517, 499, 537, 502, 510, 523, 520, 548, 554, 517, 504, 501, 539, 502, 516, 520] payload_rows_per_dst=[5574, 5674, 5586, 5616, 5498, 5566, 5662, 5694] dup_rows_per_dst=[2562, 2646, 2562, 2582, 2586, 2466, 2650, 2612] combine_rows_per_src=[5574, 5674, 5586, 5616, 5498, 5566, 5662, 5694] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44412 payload_rows_remote=38984 dup_rows=21124 dup_rows_remote=18462 dedup_ratio=0.322327 router_tokens_per_expert=[495, 524, 461, 492, 518, 490, 534, 536, 529, 546, 549, 460, 490, 519, 534, 564, 555, 534, 512, 532, 491, 504, 509, 517, 520, 532, 531, 535, 490, 494, 505, 516, 508, 518, 499, 515, 493, 489, 529, 523, 499, 488, 494, 515, 499, 521, 522, 507, 498, 489, 458, 513, 490, 509, 512, 486, 566, 516, 506, 502, 521, 551, 500, 514] payload_rows_per_dst=[5526, 5684, 5598, 5572, 5584, 5428, 5416, 5604] dup_rows_per_dst=[2574, 2698, 2710, 2674, 2564, 2662, 2494, 2748] combine_rows_per_src=[5526, 5684, 5598, 5572, 5584, 5428, 5416, 5604] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278476800 dispatch_meta_bytes=229264 combine_payload_bytes=278476800 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278706064 ingress_bytes=388577280 link_bytes=388577280 token_bytes=557182864 weight_movement_bytes=110100480 total_nvlink_bytes=667283344 weight_to_token_ratio=0.197602 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.757997999e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44416 payload_rows_remote=38980 dup_rows=21120 dup_rows_remote=18560 dedup_ratio=0.322266 router_tokens_per_expert=[470, 467, 537, 477, 538, 492, 514, 503, 514, 518, 540, 515, 508, 529, 528, 532, 529, 542, 542, 506, 504, 502, 479, 521, 478, 500, 498, 480, 503, 517, 507, 492, 488, 543, 495, 507, 513, 498, 480, 515, 492, 522, 512, 532, 526, 525, 513, 501, 522, 555, 504, 517, 524, 531, 490, 569, 527, 505, 507, 522, 516, 519, 492, 524] payload_rows_per_dst=[5436, 5598, 5586, 5460, 5522, 5524, 5672, 5618] dup_rows_per_dst=[2560, 2770, 2664, 2490, 2556, 2722, 2752, 2606] combine_rows_per_src=[5436, 5598, 5586, 5460, 5522, 5524, 5672, 5618] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279867392 dispatch_meta_bytes=229064 combine_payload_bytes=279867392 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280096456 ingress_bytes=610168832 link_bytes=610168832 token_bytes=559963848 weight_movement_bytes=330301440 total_nvlink_bytes=890265288 weight_to_token_ratio=0.589862 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061183954e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44454 payload_rows_remote=38810 dup_rows=21082 dup_rows_remote=18392 dedup_ratio=0.321686 router_tokens_per_expert=[511, 508, 482, 543, 523, 524, 515, 534, 502, 503, 526, 515, 496, 496, 505, 506, 516, 533, 491, 507, 526, 523, 525, 520, 460, 519, 511, 488, 532, 502, 505, 522, 564, 524, 502, 535, 504, 504, 464, 525, 517, 504, 533, 502, 499, 562, 529, 453, 541, 496, 519, 485, 526, 509, 487, 512, 507, 488, 499, 543, 515, 517, 506, 528] payload_rows_per_dst=[5644, 5566, 5498, 5544, 5664, 5396, 5554, 5588] dup_rows_per_dst=[2690, 2828, 2488, 2584, 2910, 2368, 2596, 2618] combine_rows_per_src=[5644, 5566, 5498, 5544, 5664, 5396, 5554, 5588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277831680 dispatch_meta_bytes=229632 combine_payload_bytes=277831680 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278061312 ingress_bytes=387932160 link_bytes=387932160 token_bytes=555892992 weight_movement_bytes=110100480 total_nvlink_bytes=665993472 weight_to_token_ratio=0.198061 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.746778301e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44548 payload_rows_remote=39046 dup_rows=20988 dup_rows_remote=18246 dedup_ratio=0.320251 router_tokens_per_expert=[499, 499, 553, 505, 560, 479, 495, 521, 523, 549, 505, 523, 520, 534, 486, 482, 479, 498, 509, 505, 525, 519, 545, 510, 503, 517, 501, 511, 506, 533, 537, 502, 476, 496, 489, 546, 471, 465, 504, 514, 473, 472, 525, 548, 470, 552, 483, 511, 506, 487, 521, 498, 539, 556, 524, 515, 504, 493, 538, 522, 525, 522, 537, 553] payload_rows_per_dst=[5630, 5502, 5514, 5540, 5480, 5574, 5632, 5676] dup_rows_per_dst=[2592, 2742, 2666, 2680, 2442, 2494, 2660, 2712] combine_rows_per_src=[5630, 5502, 5514, 5540, 5480, 5574, 5632, 5676] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44844 payload_rows_remote=39176 dup_rows=20692 dup_rows_remote=18014 dedup_ratio=0.315735 router_tokens_per_expert=[516, 503, 502, 529, 486, 499, 520, 491, 520, 499, 514, 526, 535, 518, 523, 505, 491, 498, 528, 528, 516, 519, 520, 468, 501, 509, 540, 487, 480, 546, 511, 524, 523, 536, 515, 459, 517, 490, 496, 513, 517, 428, 496, 506, 509, 545, 496, 529, 517, 500, 539, 502, 513, 529, 523, 550, 556, 516, 505, 504, 541, 504, 520, 522] payload_rows_per_dst=[5550, 5656, 5578, 5610, 5498, 5578, 5668, 5706] dup_rows_per_dst=[2542, 2624, 2558, 2586, 2600, 2474, 2678, 2630] combine_rows_per_src=[5550, 5656, 5578, 5610, 5498, 5578, 5668, 5706] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279293952 dispatch_meta_bytes=228904 combine_payload_bytes=279293952 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279522856 ingress_bytes=609595392 link_bytes=609595392 token_bytes=558816808 weight_movement_bytes=330301440 total_nvlink_bytes=889118248 weight_to_token_ratio=0.591073 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060186648e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278190080 dispatch_meta_bytes=228808 combine_payload_bytes=278190080 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278418888 ingress_bytes=498391040 link_bytes=498391040 token_bytes=556608968 weight_movement_bytes=220200960 total_nvlink_bytes=776809928 weight_to_token_ratio=0.395612 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.667839898e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44620 payload_rows_remote=39066 dup_rows=20916 dup_rows_remote=18250 dedup_ratio=0.319153 router_tokens_per_expert=[520, 505, 536, 516, 486, 535, 522, 483, 536, 519, 502, 496, 486, 511, 526, 497, 546, 453, 526, 528, 542, 497, 526, 492, 526, 516, 471, 492, 526, 551, 497, 519, 482, 510, 482, 527, 518, 553, 521, 543, 480, 529, 492, 511, 532, 522, 502, 513, 532, 500, 495, 487, 502, 502, 552, 516, 531, 500, 539, 518, 484, 486, 512, 511] payload_rows_per_dst=[5622, 5586, 5554, 5596, 5624, 5500, 5564, 5574] dup_rows_per_dst=[2584, 2560, 2666, 2600, 2648, 2662, 2608, 2588] combine_rows_per_src=[5622, 5586, 5554, 5596, 5624, 5500, 5564, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44446 payload_rows_remote=38894 dup_rows=21090 dup_rows_remote=18386 dedup_ratio=0.321808 router_tokens_per_expert=[518, 516, 525, 525, 475, 536, 519, 481, 516, 491, 552, 507, 478, 525, 527, 532, 500, 519, 520, 504, 505, 540, 508, 547, 516, 511, 526, 511, 507, 504, 490, 520, 471, 483, 544, 515, 502, 472, 531, 491, 503, 534, 509, 526, 489, 499, 545, 511, 546, 526, 515, 498, 485, 492, 509, 512, 530, 502, 480, 515, 488, 522, 536, 536] payload_rows_per_dst=[5516, 5530, 5462, 5536, 5640, 5638, 5572, 5552] dup_rows_per_dst=[2674, 2726, 2338, 2634, 2864, 2594, 2556, 2704] combine_rows_per_src=[5516, 5530, 5462, 5536, 5640, 5638, 5572, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44354 payload_rows_remote=38920 dup_rows=21182 dup_rows_remote=18596 dedup_ratio=0.323212 router_tokens_per_expert=[512, 504, 528, 556, 502, 496, 512, 519, 530, 534, 490, 509, 502, 477, 484, 527, 523, 511, 515, 521, 528, 532, 516, 521, 514, 453, 548, 492, 453, 499, 503, 548, 558, 516, 515, 498, 526, 490, 515, 503, 500, 543, 538, 532, 542, 517, 524, 520, 545, 496, 531, 474, 513, 508, 492, 530, 511, 516, 486, 485, 511, 511, 489, 474] payload_rows_per_dst=[5594, 5506, 5652, 5434, 5550, 5598, 5510, 5510] dup_rows_per_dst=[2664, 2600, 2682, 2586, 2692, 2834, 2668, 2456] combine_rows_per_src=[5594, 5506, 5652, 5434, 5550, 5598, 5510, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279881728 dispatch_meta_bytes=229168 combine_payload_bytes=279881728 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280110896 ingress_bytes=500082688 link_bytes=500082688 token_bytes=559992624 weight_movement_bytes=220200960 total_nvlink_bytes=780193584 weight_to_token_ratio=0.393221 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.697260439e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=280813568 dispatch_meta_bytes=228760 combine_payload_bytes=280813568 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=281042328 ingress_bytes=721215488 link_bytes=721215488 token_bytes=561855896 weight_movement_bytes=440401920 total_nvlink_bytes=1002257816 weight_to_token_ratio=0.783834 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.254312353e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=280025088 dispatch_meta_bytes=229264 combine_payload_bytes=280025088 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280254352 ingress_bytes=500226048 link_bytes=500226048 token_bytes=560279440 weight_movement_bytes=220200960 total_nvlink_bytes=780480400 weight_to_token_ratio=0.393020 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.699753705e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278792192 dispatch_meta_bytes=229120 combine_payload_bytes=278792192 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279021312 ingress_bytes=498993152 link_bytes=498993152 token_bytes=557813504 weight_movement_bytes=220200960 total_nvlink_bytes=778014464 weight_to_token_ratio=0.394757 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.678311616e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278978560 dispatch_meta_bytes=230064 combine_payload_bytes=278978560 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279208624 ingress_bytes=389079040 link_bytes=389079040 token_bytes=558187184 weight_movement_bytes=110100480 total_nvlink_bytes=668287664 weight_to_token_ratio=0.197247 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.766724431e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44464 payload_rows_remote=38892 dup_rows=21072 dup_rows_remote=18388 dedup_ratio=0.321533 router_tokens_per_expert=[541, 513, 487, 493, 507, 518, 534, 527, 516, 501, 504, 483, 506, 515, 508, 516, 482, 500, 512, 549, 543, 542, 506, 529, 475, 487, 556, 531, 521, 496, 509, 516, 540, 524, 499, 461, 476, 546, 543, 539, 456, 549, 512, 505, 542, 485, 532, 489, 508, 516, 510, 502, 556, 514, 483, 503, 524, 511, 498, 519, 508, 471, 515, 509] payload_rows_per_dst=[5572, 5502, 5632, 5602, 5572, 5502, 5588, 5494] dup_rows_per_dst=[2668, 2596, 2694, 2580, 2684, 2638, 2596, 2616] combine_rows_per_src=[5572, 5502, 5632, 5602, 5572, 5502, 5588, 5494] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44354 payload_rows_remote=38924 dup_rows=21182 dup_rows_remote=18596 dedup_ratio=0.323212 router_tokens_per_expert=[514, 505, 531, 558, 505, 496, 516, 524, 531, 537, 492, 510, 502, 476, 484, 527, 523, 511, 516, 521, 530, 534, 517, 525, 513, 454, 547, 493, 452, 498, 503, 548, 559, 518, 517, 498, 528, 490, 512, 501, 502, 540, 536, 532, 539, 517, 526, 519, 541, 496, 530, 473, 512, 505, 492, 527, 510, 515, 483, 482, 509, 508, 487, 471] payload_rows_per_dst=[5610, 5510, 5658, 5430, 5550, 5600, 5500, 5496] dup_rows_per_dst=[2688, 2608, 2696, 2586, 2696, 2822, 2652, 2434] combine_rows_per_src=[5610, 5510, 5658, 5430, 5550, 5600, 5500, 5496] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278849536 dispatch_meta_bytes=229152 combine_payload_bytes=278849536 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279078688 ingress_bytes=388950016 link_bytes=388950016 token_bytes=557928224 weight_movement_bytes=110100480 total_nvlink_bytes=668028704 weight_to_token_ratio=0.197338 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.764480491e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279838720 dispatch_meta_bytes=229168 combine_payload_bytes=279838720 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=280067888 ingress_bytes=720240640 link_bytes=720240640 token_bytes=559906608 weight_movement_bytes=440401920 total_nvlink_bytes=1000308528 weight_to_token_ratio=0.786563 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.252616932e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=281042944 dispatch_meta_bytes=228896 combine_payload_bytes=281042944 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281271840 ingress_bytes=391143424 link_bytes=391143424 token_bytes=562314784 weight_movement_bytes=110100480 total_nvlink_bytes=672415264 weight_to_token_ratio=0.195799 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.802627464e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279437312 dispatch_meta_bytes=229784 combine_payload_bytes=279437312 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279667096 ingress_bytes=389537792 link_bytes=389537792 token_bytes=559104408 weight_movement_bytes=110100480 total_nvlink_bytes=669204888 weight_to_token_ratio=0.196923 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.774702882e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279408640 dispatch_meta_bytes=230160 combine_payload_bytes=279408640 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279638800 ingress_bytes=499609600 link_bytes=499609600 token_bytes=559047440 weight_movement_bytes=220200960 total_nvlink_bytes=779248400 weight_to_token_ratio=0.393886 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.689032661e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279007232 dispatch_meta_bytes=230080 combine_payload_bytes=279007232 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279237312 ingress_bytes=389107712 link_bytes=389107712 token_bytes=558244544 weight_movement_bytes=110100480 total_nvlink_bytes=668345024 weight_to_token_ratio=0.197226 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.767223084e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44486 payload_rows_remote=38816 dup_rows=21050 dup_rows_remote=18188 dedup_ratio=0.321198 router_tokens_per_expert=[515, 513, 504, 486, 509, 471, 497, 547, 529, 544, 516, 513, 531, 505, 510, 532, 501, 514, 535, 474, 519, 518, 516, 524, 499, 512, 522, 466, 509, 517, 511, 550, 480, 541, 491, 493, 555, 490, 504, 526, 525, 550, 563, 533, 474, 501, 519, 495, 483, 499, 529, 501, 520, 473, 469, 504, 500, 524, 490, 529, 509, 531, 532, 526] payload_rows_per_dst=[5386, 5302, 5558, 5808, 5498, 5748, 5516, 5670] dup_rows_per_dst=[2340, 2392, 2564, 2770, 2502, 2930, 2690, 2862] combine_rows_per_src=[5386, 5302, 5558, 5808, 5498, 5748, 5516, 5670] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44548 payload_rows_remote=38940 dup_rows=20988 dup_rows_remote=18328 dedup_ratio=0.320251 router_tokens_per_expert=[519, 518, 551, 507, 496, 508, 502, 544, 524, 521, 540, 496, 487, 521, 528, 517, 518, 491, 532, 483, 492, 504, 489, 503, 498, 521, 525, 527, 490, 520, 489, 504, 499, 518, 528, 510, 494, 528, 533, 534, 541, 478, 510, 498, 498, 522, 456, 496, 521, 510, 523, 495, 503, 507, 512, 509, 527, 494, 485, 534, 536, 556, 494, 554] payload_rows_per_dst=[5566, 5608, 5550, 5538, 5582, 5518, 5534, 5652] dup_rows_per_dst=[2724, 2660, 2474, 2610, 2706, 2480, 2626, 2708] combine_rows_per_src=[5566, 5608, 5550, 5538, 5582, 5518, 5534, 5652] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44710 payload_rows_remote=39118 dup_rows=20826 dup_rows_remote=18144 dedup_ratio=0.317780 router_tokens_per_expert=[490, 512, 500, 488, 504, 547, 539, 524, 496, 512, 494, 524, 547, 474, 539, 505, 551, 531, 496, 560, 521, 444, 483, 558, 500, 500, 518, 524, 479, 486, 473, 471, 484, 534, 500, 523, 518, 533, 499, 546, 539, 500, 535, 498, 540, 503, 510, 516, 511, 536, 512, 548, 511, 524, 519, 500, 516, 505, 489, 466, 541, 479, 538, 505] payload_rows_per_dst=[5582, 5576, 5602, 5434, 5592, 5612, 5670, 5642] dup_rows_per_dst=[2626, 2606, 2686, 2468, 2682, 2670, 2652, 2436] combine_rows_per_src=[5582, 5576, 5602, 5434, 5592, 5612, 5670, 5642] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44474 payload_rows_remote=38878 dup_rows=21062 dup_rows_remote=18412 dedup_ratio=0.321381 router_tokens_per_expert=[506, 536, 534, 541, 494, 545, 489, 551, 541, 507, 550, 480, 476, 524, 550, 498, 539, 464, 514, 535, 513, 531, 510, 517, 488, 480, 498, 528, 488, 547, 512, 512, 529, 513, 472, 520, 515, 513, 517, 491, 497, 507, 553, 459, 518, 521, 516, 494, 521, 532, 549, 485, 484, 461, 502, 539, 532, 531, 460, 505, 517, 515, 502, 500] payload_rows_per_dst=[5578, 5608, 5596, 5582, 5470, 5492, 5546, 5602] dup_rows_per_dst=[2814, 2644, 2650, 2524, 2670, 2638, 2600, 2522] combine_rows_per_src=[5578, 5608, 5596, 5582, 5470, 5492, 5546, 5602] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278233088 dispatch_meta_bytes=228016 combine_payload_bytes=278233088 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278461104 ingress_bytes=388333568 link_bytes=388333568 token_bytes=556694192 weight_movement_bytes=110100480 total_nvlink_bytes=666794672 weight_to_token_ratio=0.197776 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.753759446e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44368 payload_rows_remote=38956 dup_rows=21168 dup_rows_remote=18660 dedup_ratio=0.322998 router_tokens_per_expert=[515, 483, 521, 512, 540, 482, 507, 514, 537, 516, 510, 549, 538, 520, 516, 506, 505, 555, 505, 569, 511, 515, 517, 474, 538, 491, 495, 545, 508, 538, 472, 543, 518, 543, 497, 508, 539, 464, 513, 551, 508, 488, 493, 528, 497, 494, 513, 510, 504, 496, 495, 499, 465, 490, 487, 524, 531, 534, 540, 505, 469, 481, 503, 534] payload_rows_per_dst=[5530, 5644, 5600, 5544, 5530, 5526, 5412, 5582] dup_rows_per_dst=[2618, 2740, 2702, 2716, 2736, 2536, 2508, 2612] combine_rows_per_src=[5530, 5644, 5600, 5544, 5530, 5526, 5412, 5582] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44420 payload_rows_remote=38796 dup_rows=21116 dup_rows_remote=18500 dedup_ratio=0.322205 router_tokens_per_expert=[528, 499, 498, 485, 540, 495, 545, 491, 507, 512, 548, 522, 478, 459, 512, 542, 465, 497, 540, 490, 521, 569, 519, 491, 510, 473, 538, 561, 505, 474, 506, 541, 550, 514, 528, 529, 476, 524, 549, 497, 514, 521, 523, 512, 514, 511, 500, 525, 524, 484, 522, 504, 509, 503, 536, 497, 536, 482, 481, 487, 476, 540, 512, 527] payload_rows_per_dst=[5572, 5546, 5510, 5546, 5598, 5624, 5484, 5540] dup_rows_per_dst=[2590, 2614, 2674, 2670, 2736, 2616, 2674, 2542] combine_rows_per_src=[5572, 5546, 5510, 5546, 5598, 5624, 5484, 5540] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44414 payload_rows_remote=38858 dup_rows=21122 dup_rows_remote=18502 dedup_ratio=0.322296 router_tokens_per_expert=[522, 529, 506, 498, 487, 495, 538, 512, 481, 533, 545, 490, 514, 552, 508, 547, 487, 510, 493, 518, 561, 505, 495, 524, 487, 521, 505, 512, 496, 529, 456, 495, 473, 481, 515, 498, 552, 526, 511, 531, 494, 508, 553, 495, 537, 468, 463, 498, 536, 525, 539, 504, 509, 552, 504, 522, 521, 524, 486, 532, 509, 525, 509, 517] payload_rows_per_dst=[5556, 5636, 5552, 5488, 5544, 5440, 5616, 5582] dup_rows_per_dst=[2620, 2702, 2634, 2514, 2630, 2592, 2766, 2664] combine_rows_per_src=[5556, 5636, 5552, 5488, 5544, 5440, 5616, 5582] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279121920 dispatch_meta_bytes=229072 combine_payload_bytes=279121920 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=279350992 ingress_bytes=719523840 link_bytes=719523840 token_bytes=558472912 weight_movement_bytes=440401920 total_nvlink_bytes=998874832 weight_to_token_ratio=0.788582 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.251370299e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44678 payload_rows_remote=39070 dup_rows=20858 dup_rows_remote=18142 dedup_ratio=0.318268 router_tokens_per_expert=[551, 544, 482, 511, 541, 539, 506, 543, 508, 560, 529, 512, 523, 500, 505, 492, 540, 499, 529, 500, 495, 498, 495, 536, 537, 545, 515, 511, 553, 489, 496, 516, 468, 488, 493, 532, 512, 525, 526, 486, 535, 511, 517, 466, 528, 516, 505, 479, 530, 539, 509, 512, 514, 512, 497, 508, 492, 527, 464, 489, 564, 492, 445, 487] payload_rows_per_dst=[5720, 5612, 5646, 5608, 5508, 5534, 5580, 5470] dup_rows_per_dst=[2714, 2646, 2538, 2716, 2552, 2580, 2662, 2450] combine_rows_per_src=[5720, 5612, 5646, 5608, 5508, 5534, 5580, 5470] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44394 payload_rows_remote=38966 dup_rows=21142 dup_rows_remote=18474 dedup_ratio=0.322601 router_tokens_per_expert=[493, 520, 459, 491, 514, 487, 531, 532, 523, 546, 547, 460, 489, 514, 534, 561, 551, 533, 513, 530, 486, 504, 510, 516, 521, 533, 536, 536, 489, 491, 504, 516, 508, 518, 498, 518, 497, 489, 528, 524, 498, 487, 494, 517, 499, 523, 522, 508, 498, 489, 460, 516, 492, 514, 514, 488, 566, 519, 508, 506, 524, 555, 506, 515] payload_rows_per_dst=[5506, 5662, 5592, 5574, 5588, 5428, 5424, 5620] dup_rows_per_dst=[2548, 2686, 2694, 2678, 2572, 2668, 2518, 2778] combine_rows_per_src=[5506, 5662, 5592, 5574, 5588, 5428, 5424, 5620] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44342 payload_rows_remote=38802 dup_rows=21194 dup_rows_remote=18460 dedup_ratio=0.323395 router_tokens_per_expert=[467, 467, 535, 476, 536, 489, 511, 498, 513, 518, 537, 513, 506, 527, 524, 529, 527, 539, 539, 505, 503, 498, 480, 521, 480, 500, 498, 477, 505, 517, 509, 492, 489, 540, 495, 507, 513, 500, 482, 515, 492, 525, 512, 534, 527, 527, 515, 503, 520, 557, 508, 518, 525, 534, 491, 571, 530, 508, 508, 525, 517, 525, 493, 526] payload_rows_per_dst=[5540, 5452, 5590, 5590, 5526, 5560, 5590, 5494] dup_rows_per_dst=[2734, 2606, 2664, 2710, 2556, 2794, 2704, 2426] combine_rows_per_src=[5540, 5452, 5590, 5590, 5526, 5560, 5590, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278777856 dispatch_meta_bytes=229120 combine_payload_bytes=278777856 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279006976 ingress_bytes=388878336 link_bytes=388878336 token_bytes=557784832 weight_movement_bytes=110100480 total_nvlink_bytes=667885312 weight_to_token_ratio=0.197389 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.763233858e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44540 payload_rows_remote=38938 dup_rows=20996 dup_rows_remote=18360 dedup_ratio=0.320374 router_tokens_per_expert=[515, 517, 548, 504, 495, 502, 499, 540, 521, 522, 535, 495, 483, 520, 529, 514, 516, 490, 529, 483, 491, 502, 489, 505, 497, 519, 522, 528, 491, 518, 488, 504, 499, 517, 526, 512, 495, 531, 534, 534, 539, 479, 510, 498, 500, 522, 456, 499, 524, 512, 524, 498, 503, 507, 513, 513, 529, 496, 492, 540, 540, 561, 497, 557] payload_rows_per_dst=[5538, 5602, 5538, 5534, 5588, 5518, 5544, 5678] dup_rows_per_dst=[2702, 2636, 2472, 2600, 2708, 2488, 2644, 2746] combine_rows_per_src=[5538, 5602, 5538, 5534, 5588, 5518, 5544, 5678] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279308288 dispatch_meta_bytes=229760 combine_payload_bytes=279308288 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279538048 ingress_bytes=389408768 link_bytes=389408768 token_bytes=558846336 weight_movement_bytes=110100480 total_nvlink_bytes=668946816 weight_to_token_ratio=0.197014 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.772458943e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44338 payload_rows_remote=38922 dup_rows=21198 dup_rows_remote=18656 dedup_ratio=0.323456 router_tokens_per_expert=[511, 479, 518, 506, 538, 480, 506, 513, 536, 514, 509, 546, 537, 516, 515, 506, 502, 550, 501, 567, 511, 511, 517, 476, 536, 491, 496, 545, 509, 539, 472, 541, 516, 542, 498, 509, 541, 464, 513, 553, 509, 490, 495, 528, 500, 496, 514, 511, 507, 497, 496, 502, 467, 495, 487, 528, 531, 537, 542, 511, 472, 480, 508, 535] payload_rows_per_dst=[5506, 5630, 5590, 5540, 5530, 5530, 5416, 5596] dup_rows_per_dst=[2596, 2728, 2680, 2718, 2742, 2556, 2542, 2636] combine_rows_per_src=[5506, 5630, 5590, 5540, 5530, 5530, 5416, 5596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44502 payload_rows_remote=38902 dup_rows=21034 dup_rows_remote=18316 dedup_ratio=0.320953 router_tokens_per_expert=[515, 511, 498, 482, 506, 470, 493, 543, 528, 543, 511, 511, 529, 505, 511, 533, 498, 515, 532, 473, 517, 519, 515, 523, 498, 511, 518, 466, 512, 516, 510, 549, 479, 542, 492, 495, 555, 491, 504, 528, 524, 552, 563, 532, 478, 501, 523, 498, 483, 501, 530, 502, 522, 474, 470, 509, 502, 525, 494, 533, 510, 532, 533, 530] payload_rows_per_dst=[5486, 5410, 5598, 5814, 5484, 5670, 5440, 5600] dup_rows_per_dst=[2514, 2468, 2622, 2810, 2466, 2894, 2542, 2718] combine_rows_per_src=[5486, 5410, 5598, 5814, 5484, 5670, 5440, 5600] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44474 payload_rows_remote=38882 dup_rows=21062 dup_rows_remote=18406 dedup_ratio=0.321381 router_tokens_per_expert=[503, 535, 532, 540, 489, 542, 488, 551, 539, 504, 546, 475, 474, 520, 549, 493, 538, 463, 515, 536, 513, 532, 509, 518, 487, 477, 497, 530, 488, 547, 510, 514, 526, 514, 472, 522, 514, 513, 517, 495, 496, 508, 553, 457, 522, 520, 519, 498, 526, 536, 551, 487, 484, 462, 505, 541, 533, 533, 461, 506, 517, 519, 504, 503] payload_rows_per_dst=[5564, 5584, 5592, 5576, 5476, 5504, 5570, 5608] dup_rows_per_dst=[2796, 2616, 2656, 2524, 2670, 2642, 2614, 2544] combine_rows_per_src=[5564, 5584, 5592, 5576, 5476, 5504, 5570, 5608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278132736 dispatch_meta_bytes=229048 combine_payload_bytes=278132736 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278361784 ingress_bytes=388233216 link_bytes=388233216 token_bytes=556494520 weight_movement_bytes=110100480 total_nvlink_bytes=666595000 weight_to_token_ratio=0.197846 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.752014160e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44676 payload_rows_remote=39070 dup_rows=20860 dup_rows_remote=18154 dedup_ratio=0.318298 router_tokens_per_expert=[547, 542, 478, 506, 537, 537, 505, 540, 506, 556, 527, 512, 522, 500, 504, 491, 541, 500, 528, 498, 494, 496, 494, 535, 538, 546, 509, 507, 555, 489, 495, 517, 466, 489, 493, 534, 515, 526, 525, 486, 535, 513, 521, 467, 528, 517, 505, 480, 531, 543, 509, 514, 514, 516, 500, 509, 493, 530, 467, 493, 567, 493, 447, 490] payload_rows_per_dst=[5700, 5602, 5642, 5606, 5512, 5532, 5592, 5490] dup_rows_per_dst=[2684, 2634, 2530, 2706, 2556, 2600, 2680, 2470] combine_rows_per_src=[5700, 5602, 5642, 5606, 5512, 5532, 5592, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279107584 dispatch_meta_bytes=229192 combine_payload_bytes=279107584 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279336776 ingress_bytes=499308544 link_bytes=499308544 token_bytes=558444360 weight_movement_bytes=220200960 total_nvlink_bytes=778645320 weight_to_token_ratio=0.394311 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.683796802e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=280397824 dispatch_meta_bytes=229048 combine_payload_bytes=280397824 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280626872 ingress_bytes=500598784 link_bytes=500598784 token_bytes=561024696 weight_movement_bytes=220200960 total_nvlink_bytes=781225656 weight_to_token_ratio=0.392498 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.706236197e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278677504 dispatch_meta_bytes=229160 combine_payload_bytes=278677504 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278906664 ingress_bytes=498878464 link_bytes=498878464 token_bytes=557584168 weight_movement_bytes=220200960 total_nvlink_bytes=777785128 weight_to_token_ratio=0.394920 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.676317003e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279236608 dispatch_meta_bytes=230464 combine_payload_bytes=279236608 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279467072 ingress_bytes=389337088 link_bytes=389337088 token_bytes=558703680 weight_movement_bytes=110100480 total_nvlink_bytes=668804160 weight_to_token_ratio=0.197064 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.771212310e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278089728 dispatch_meta_bytes=229184 combine_payload_bytes=278089728 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278318912 ingress_bytes=388190208 link_bytes=388190208 token_bytes=556408640 weight_movement_bytes=110100480 total_nvlink_bytes=666509120 weight_to_token_ratio=0.197877 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.751266180e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278534144 dispatch_meta_bytes=229440 combine_payload_bytes=278534144 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278763584 ingress_bytes=498735104 link_bytes=498735104 token_bytes=557297728 weight_movement_bytes=220200960 total_nvlink_bytes=777498688 weight_to_token_ratio=0.395123 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.673823737e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=280053760 dispatch_meta_bytes=228848 combine_payload_bytes=280053760 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280282608 ingress_bytes=390154240 link_bytes=390154240 token_bytes=560336368 weight_movement_bytes=110100480 total_nvlink_bytes=670436848 weight_to_token_ratio=0.196490 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.785423927e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44348 payload_rows_remote=38628 dup_rows=21188 dup_rows_remote=18292 dedup_ratio=0.323303 router_tokens_per_expert=[511, 471, 497, 540, 491, 511, 511, 507, 502, 561, 503, 492, 493, 489, 505, 503, 530, 488, 488, 485, 522, 517, 508, 504, 526, 506, 513, 539, 535, 498, 528, 503, 516, 544, 524, 528, 524, 539, 503, 517, 516, 529, 523, 486, 505, 500, 480, 539, 527, 526, 516, 519, 501, 515, 475, 501, 551, 546, 492, 542, 524, 499, 494, 490] payload_rows_per_dst=[5436, 5522, 5516, 5584, 5452, 5518, 5600, 5720] dup_rows_per_dst=[2512, 2574, 2568, 2548, 2728, 2660, 2702, 2896] combine_rows_per_src=[5436, 5522, 5516, 5584, 5452, 5518, 5600, 5720] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44674 payload_rows_remote=39092 dup_rows=20862 dup_rows_remote=18190 dedup_ratio=0.318329 router_tokens_per_expert=[512, 538, 539, 490, 517, 514, 472, 533, 520, 546, 509, 501, 471, 540, 535, 505, 507, 541, 483, 475, 514, 551, 515, 519, 507, 539, 512, 480, 498, 497, 547, 532, 522, 517, 529, 504, 510, 503, 518, 484, 499, 497, 526, 500, 501, 522, 512, 531, 554, 517, 496, 481, 504, 495, 502, 514, 502, 509, 519, 493, 515, 531, 506, 496] payload_rows_per_dst=[5666, 5582, 5622, 5618, 5556, 5542, 5566, 5522] dup_rows_per_dst=[2564, 2672, 2588, 2606, 2618, 2634, 2560, 2620] combine_rows_per_src=[5666, 5582, 5622, 5618, 5556, 5542, 5566, 5522] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=276885504 dispatch_meta_bytes=227680 combine_payload_bytes=276885504 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277113184 ingress_bytes=386985984 link_bytes=386985984 token_bytes=553998688 weight_movement_bytes=110100480 total_nvlink_bytes=664099168 weight_to_token_ratio=0.198738 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.730322744e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44626 payload_rows_remote=39050 dup_rows=20910 dup_rows_remote=18244 dedup_ratio=0.319061 router_tokens_per_expert=[488, 523, 545, 560, 542, 475, 517, 513, 497, 545, 550, 494, 513, 534, 512, 497, 513, 477, 531, 496, 526, 506, 476, 463, 491, 483, 526, 475, 531, 495, 505, 522, 519, 514, 517, 537, 526, 482, 507, 519, 528, 544, 493, 536, 536, 518, 491, 516, 481, 502, 537, 525, 534, 524, 534, 523, 489, 518, 477, 523, 500, 496, 500, 501] payload_rows_per_dst=[5602, 5598, 5490, 5554, 5576, 5656, 5618, 5532] dup_rows_per_dst=[2724, 2686, 2486, 2502, 2666, 2668, 2702, 2476] combine_rows_per_src=[5602, 5598, 5490, 5554, 5576, 5656, 5618, 5532] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44504 payload_rows_remote=39002 dup_rows=21032 dup_rows_remote=18394 dedup_ratio=0.320923 router_tokens_per_expert=[508, 511, 516, 535, 500, 530, 498, 495, 511, 493, 504, 484, 538, 514, 516, 484, 524, 553, 497, 491, 542, 465, 478, 520, 534, 494, 474, 524, 566, 520, 515, 535, 505, 506, 514, 479, 486, 535, 543, 513, 594, 504, 480, 530, 510, 530, 512, 525, 485, 487, 532, 524, 514, 533, 502, 486, 520, 483, 519, 508, 498, 497, 520, 525] payload_rows_per_dst=[5564, 5472, 5502, 5612, 5590, 5710, 5536, 5518] dup_rows_per_dst=[2622, 2616, 2638, 2712, 2572, 2660, 2590, 2622] combine_rows_per_src=[5564, 5472, 5502, 5612, 5590, 5710, 5536, 5518] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44516 payload_rows_remote=39038 dup_rows=21020 dup_rows_remote=18486 dedup_ratio=0.320740 router_tokens_per_expert=[543, 530, 541, 488, 537, 536, 503, 501, 506, 512, 508, 523, 521, 540, 510, 544, 505, 548, 521, 518, 524, 499, 547, 513, 523, 485, 495, 499, 492, 523, 526, 514, 509, 516, 534, 506, 474, 508, 481, 495, 511, 526, 499, 512, 513, 505, 505, 531, 499, 483, 486, 475, 489, 529, 555, 490, 521, 507, 513, 516, 512, 505, 492, 496] payload_rows_per_dst=[5686, 5614, 5632, 5518, 5492, 5588, 5478, 5508] dup_rows_per_dst=[2672, 2714, 2718, 2596, 2554, 2616, 2534, 2616] combine_rows_per_src=[5686, 5614, 5632, 5518, 5492, 5588, 5478, 5508] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280211456 dispatch_meta_bytes=229128 combine_payload_bytes=280211456 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280440584 ingress_bytes=390311936 link_bytes=390311936 token_bytes=560652040 weight_movement_bytes=110100480 total_nvlink_bytes=670752520 weight_to_token_ratio=0.196379 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.788166520e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44698 payload_rows_remote=39104 dup_rows=20838 dup_rows_remote=18152 dedup_ratio=0.317963 router_tokens_per_expert=[488, 507, 498, 488, 501, 546, 536, 525, 490, 508, 490, 522, 545, 472, 538, 502, 551, 531, 495, 558, 522, 446, 484, 559, 500, 494, 519, 521, 478, 488, 472, 472, 486, 536, 496, 523, 520, 533, 502, 544, 538, 502, 534, 496, 545, 504, 513, 518, 512, 539, 517, 551, 512, 524, 519, 503, 518, 503, 490, 469, 544, 481, 540, 510] payload_rows_per_dst=[5574, 5550, 5604, 5426, 5594, 5622, 5680, 5648] dup_rows_per_dst=[2604, 2584, 2688, 2462, 2686, 2678, 2674, 2462] combine_rows_per_src=[5574, 5550, 5604, 5426, 5594, 5622, 5680, 5648] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278706176 dispatch_meta_bytes=229152 combine_payload_bytes=278706176 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278935328 ingress_bytes=388806656 link_bytes=388806656 token_bytes=557641504 weight_movement_bytes=110100480 total_nvlink_bytes=667741984 weight_to_token_ratio=0.197440 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.761987225e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278849536 dispatch_meta_bytes=228872 combine_payload_bytes=278849536 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279078408 ingress_bytes=499050496 link_bytes=499050496 token_bytes=557927944 weight_movement_bytes=220200960 total_nvlink_bytes=778128904 weight_to_token_ratio=0.394676 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.679308922e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=280053760 dispatch_meta_bytes=228896 combine_payload_bytes=280053760 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280282656 ingress_bytes=390154240 link_bytes=390154240 token_bytes=560336416 weight_movement_bytes=110100480 total_nvlink_bytes=670436896 weight_to_token_ratio=0.196490 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.785423927e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44424 payload_rows_remote=38788 dup_rows=21112 dup_rows_remote=18494 dedup_ratio=0.322144 router_tokens_per_expert=[526, 499, 496, 485, 538, 494, 544, 490, 506, 510, 544, 519, 473, 461, 509, 541, 462, 493, 541, 488, 520, 569, 521, 492, 509, 473, 537, 561, 504, 475, 506, 543, 550, 514, 525, 526, 476, 524, 549, 496, 515, 522, 522, 511, 517, 511, 503, 526, 525, 486, 524, 504, 511, 503, 540, 497, 536, 488, 481, 492, 479, 545, 513, 528] payload_rows_per_dst=[5564, 5528, 5506, 5550, 5592, 5636, 5490, 5558] dup_rows_per_dst=[2580, 2598, 2666, 2666, 2728, 2618, 2690, 2566] combine_rows_per_src=[5564, 5528, 5506, 5550, 5592, 5636, 5490, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278992896 dispatch_meta_bytes=230312 combine_payload_bytes=278992896 prefetch_bytes=0 reduce_bytes=0 egress_bytes=279223208 ingress_bytes=278992896 link_bytes=279223208 token_bytes=558216104 weight_movement_bytes=0 total_nvlink_bytes=558216104 weight_to_token_ratio=0.000000 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=4.856150830e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=280297472 dispatch_meta_bytes=229024 combine_payload_bytes=280297472 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280526496 ingress_bytes=500498432 link_bytes=500498432 token_bytes=560823968 weight_movement_bytes=220200960 total_nvlink_bytes=781024928 weight_to_token_ratio=0.392638 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.704490911e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44428 payload_rows_remote=38892 dup_rows=21108 dup_rows_remote=18516 dedup_ratio=0.322083 router_tokens_per_expert=[521, 526, 500, 494, 485, 493, 536, 509, 476, 533, 542, 490, 506, 552, 505, 550, 485, 508, 491, 514, 556, 505, 493, 524, 487, 520, 504, 510, 496, 532, 457, 496, 475, 483, 515, 496, 555, 525, 511, 532, 494, 510, 554, 495, 537, 470, 463, 501, 537, 527, 542, 506, 513, 555, 504, 525, 525, 525, 487, 534, 514, 527, 510, 525] payload_rows_per_dst=[5536, 5654, 5536, 5494, 5560, 5444, 5632, 5572] dup_rows_per_dst=[2592, 2712, 2616, 2510, 2644, 2604, 2786, 2644] combine_rows_per_src=[5536, 5654, 5536, 5494, 5560, 5444, 5632, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278032384 dispatch_meta_bytes=229128 combine_payload_bytes=278032384 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278261512 ingress_bytes=388132864 link_bytes=388132864 token_bytes=556293896 weight_movement_bytes=110100480 total_nvlink_bytes=666394376 weight_to_token_ratio=0.197918 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.750268874e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44696 payload_rows_remote=39128 dup_rows=20840 dup_rows_remote=18192 dedup_ratio=0.317993 router_tokens_per_expert=[509, 535, 536, 487, 517, 510, 468, 532, 517, 543, 507, 499, 469, 537, 534, 502, 505, 539, 482, 474, 514, 553, 518, 519, 505, 538, 513, 481, 496, 497, 547, 533, 524, 517, 529, 504, 511, 501, 518, 486, 501, 499, 529, 501, 497, 521, 514, 531, 555, 517, 497, 482, 508, 498, 503, 515, 505, 513, 523, 496, 518, 532, 509, 498] payload_rows_per_dst=[5652, 5568, 5622, 5616, 5564, 5552, 5580, 5542] dup_rows_per_dst=[2536, 2648, 2586, 2604, 2616, 2634, 2570, 2646] combine_rows_per_src=[5652, 5568, 5622, 5616, 5564, 5552, 5580, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44532 payload_rows_remote=39038 dup_rows=21004 dup_rows_remote=18380 dedup_ratio=0.320496 router_tokens_per_expert=[504, 511, 513, 532, 499, 529, 496, 495, 510, 492, 501, 483, 535, 514, 511, 481, 520, 554, 496, 488, 541, 465, 477, 518, 535, 494, 474, 525, 566, 520, 514, 536, 506, 504, 513, 479, 489, 537, 543, 515, 592, 503, 481, 532, 507, 533, 513, 525, 487, 492, 534, 528, 517, 535, 503, 486, 522, 485, 521, 512, 498, 499, 522, 526] payload_rows_per_dst=[5558, 5462, 5494, 5622, 5596, 5718, 5550, 5532] dup_rows_per_dst=[2600, 2592, 2624, 2706, 2576, 2654, 2614, 2638] combine_rows_per_src=[5558, 5462, 5494, 5622, 5596, 5718, 5550, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278777856 dispatch_meta_bytes=229632 combine_payload_bytes=278777856 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=279007488 ingress_bytes=388878336 link_bytes=388878336 token_bytes=557785344 weight_movement_bytes=110100480 total_nvlink_bytes=667885824 weight_to_token_ratio=0.197389 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.763233858e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44338 payload_rows_remote=38754 dup_rows=21198 dup_rows_remote=18466 dedup_ratio=0.323456 router_tokens_per_expert=[507, 469, 494, 538, 490, 509, 509, 505, 500, 559, 501, 488, 491, 481, 501, 502, 531, 485, 487, 485, 521, 512, 509, 503, 524, 506, 512, 536, 534, 494, 525, 505, 519, 542, 526, 530, 525, 539, 507, 518, 518, 530, 525, 489, 503, 503, 481, 540, 527, 529, 519, 521, 505, 517, 480, 505, 553, 547, 495, 543, 525, 504, 497, 493] payload_rows_per_dst=[5480, 5514, 5512, 5600, 5532, 5526, 5590, 5584] dup_rows_per_dst=[2562, 2532, 2554, 2582, 2880, 2650, 2706, 2732] combine_rows_per_src=[5480, 5514, 5512, 5600, 5532, 5526, 5590, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280469504 dispatch_meta_bytes=229280 combine_payload_bytes=280469504 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280698784 ingress_bytes=390569984 link_bytes=390569984 token_bytes=561168288 weight_movement_bytes=110100480 total_nvlink_bytes=671268768 weight_to_token_ratio=0.196199 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.792654399e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44560 payload_rows_remote=38940 dup_rows=20976 dup_rows_remote=18218 dedup_ratio=0.320068 router_tokens_per_expert=[512, 506, 509, 495, 483, 499, 498, 508, 500, 476, 507, 542, 526, 492, 510, 534, 508, 524, 497, 512, 500, 499, 505, 523, 503, 493, 564, 535, 529, 537, 501, 527, 530, 486, 528, 514, 522, 509, 510, 539, 521, 490, 511, 473, 539, 523, 525, 502, 486, 562, 487, 541, 514, 505, 523, 512, 509, 499, 497, 515, 511, 557, 455, 519] payload_rows_per_dst=[5538, 5658, 5498, 5620, 5606, 5542, 5558, 5540] dup_rows_per_dst=[2482, 2516, 2638, 2758, 2670, 2626, 2702, 2584] combine_rows_per_src=[5538, 5658, 5498, 5620, 5606, 5542, 5558, 5540] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44300 payload_rows_remote=38726 dup_rows=21236 dup_rows_remote=18508 dedup_ratio=0.324036 router_tokens_per_expert=[515, 531, 519, 515, 500, 516, 527, 498, 516, 474, 500, 528, 504, 524, 520, 499, 570, 494, 474, 475, 499, 544, 531, 518, 518, 490, 546, 490, 505, 514, 505, 516, 521, 532, 499, 500, 478, 508, 500, 523, 524, 541, 521, 495, 519, 480, 522, 480, 538, 516, 534, 498, 468, 520, 497, 517, 498, 529, 527, 536, 522, 512, 513, 525] payload_rows_per_dst=[5574, 5490, 5704, 5550, 5566, 5334, 5494, 5588] dup_rows_per_dst=[2728, 2642, 2918, 2616, 2556, 2358, 2682, 2736] combine_rows_per_src=[5574, 5490, 5704, 5550, 5566, 5334, 5494, 5588] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44296 payload_rows_remote=38684 dup_rows=21240 dup_rows_remote=18412 dedup_ratio=0.324097 router_tokens_per_expert=[481, 534, 505, 539, 487, 523, 522, 479, 467, 494, 508, 516, 535, 529, 532, 496, 521, 508, 486, 493, 509, 528, 465, 489, 463, 494, 507, 536, 495, 534, 519, 508, 548, 530, 502, 492, 483, 548, 539, 530, 527, 544, 518, 578, 487, 528, 526, 512, 499, 496, 508, 513, 529, 497, 569, 509, 485, 481, 526, 492, 523, 483, 525, 539] payload_rows_per_dst=[5558, 5568, 5464, 5456, 5550, 5612, 5582, 5506] dup_rows_per_dst=[2582, 2586, 2534, 2656, 2794, 2828, 2658, 2602] combine_rows_per_src=[5558, 5568, 5464, 5456, 5550, 5612, 5582, 5506] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44574 payload_rows_remote=38938 dup_rows=20962 dup_rows_remote=18190 dedup_ratio=0.319855 router_tokens_per_expert=[514, 506, 513, 496, 485, 501, 499, 509, 501, 477, 508, 544, 527, 496, 512, 535, 511, 525, 497, 512, 502, 500, 508, 523, 505, 494, 567, 537, 530, 540, 504, 527, 530, 485, 528, 513, 522, 509, 510, 536, 521, 489, 509, 472, 535, 524, 525, 501, 485, 560, 486, 540, 512, 503, 520, 511, 507, 497, 494, 512, 507, 552, 452, 516] payload_rows_per_dst=[5548, 5678, 5506, 5636, 5598, 5536, 5550, 5522] dup_rows_per_dst=[2498, 2522, 2650, 2772, 2668, 2616, 2684, 2552] combine_rows_per_src=[5548, 5678, 5506, 5636, 5598, 5536, 5550, 5522] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279910400 dispatch_meta_bytes=229176 combine_payload_bytes=279910400 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280139576 ingress_bytes=500111360 link_bytes=500111360 token_bytes=560049976 weight_movement_bytes=220200960 total_nvlink_bytes=780250936 weight_to_token_ratio=0.393181 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.697759092e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279566336 dispatch_meta_bytes=229584 combine_payload_bytes=279566336 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279795920 ingress_bytes=389666816 link_bytes=389666816 token_bytes=559362256 weight_movement_bytes=110100480 total_nvlink_bytes=669462736 weight_to_token_ratio=0.196832 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.776946822e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279824384 dispatch_meta_bytes=230096 combine_payload_bytes=279824384 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280054480 ingress_bytes=389924864 link_bytes=389924864 token_bytes=559878864 weight_movement_bytes=110100480 total_nvlink_bytes=669979344 weight_to_token_ratio=0.196651 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.781434701e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277587968 dispatch_meta_bytes=228936 combine_payload_bytes=277587968 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277816904 ingress_bytes=497788928 link_bytes=497788928 token_bytes=555404872 weight_movement_bytes=220200960 total_nvlink_bytes=775605832 weight_to_token_ratio=0.396469 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.657368180e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44422 payload_rows_remote=38748 dup_rows=21114 dup_rows_remote=18240 dedup_ratio=0.322174 router_tokens_per_expert=[507, 535, 508, 479, 505, 488, 551, 537, 534, 486, 525, 488, 487, 526, 490, 514, 499, 538, 504, 516, 490, 516, 469, 548, 545, 516, 532, 541, 485, 526, 501, 526, 499, 516, 455, 500, 523, 540, 509, 487, 528, 534, 477, 527, 534, 557, 532, 533, 506, 534, 508, 486, 525, 498, 515, 477, 492, 495, 486, 502, 548, 483, 537, 513] payload_rows_per_dst=[5272, 5492, 5366, 5500, 5630, 5690, 5798, 5674] dup_rows_per_dst=[2126, 2608, 2358, 2626, 2616, 2784, 3122, 2874] combine_rows_per_src=[5272, 5492, 5366, 5500, 5630, 5690, 5798, 5674] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277286912 dispatch_meta_bytes=228384 combine_payload_bytes=277286912 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=277515296 ingress_bytes=607588352 link_bytes=607588352 token_bytes=554802208 weight_movement_bytes=330301440 total_nvlink_bytes=885103648 weight_to_token_ratio=0.595350 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.056696075e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279107584 dispatch_meta_bytes=228512 combine_payload_bytes=279107584 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279336096 ingress_bytes=609409024 link_bytes=609409024 token_bytes=558443680 weight_movement_bytes=330301440 total_nvlink_bytes=888745120 weight_to_token_ratio=0.591468 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059862523e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44452 payload_rows_remote=38882 dup_rows=21084 dup_rows_remote=18574 dedup_ratio=0.321716 router_tokens_per_expert=[469, 514, 525, 522, 537, 529, 548, 537, 475, 523, 515, 524, 508, 526, 492, 477, 513, 499, 492, 535, 524, 514, 492, 556, 511, 506, 532, 512, 505, 471, 535, 485, 525, 516, 493, 474, 504, 507, 513, 534, 511, 509, 536, 547, 539, 510, 532, 513, 488, 460, 471, 513, 498, 522, 541, 501, 510, 511, 509, 524, 510, 558, 479, 507] payload_rows_per_dst=[5638, 5570, 5580, 5526, 5476, 5668, 5460, 5534] dup_rows_per_dst=[2724, 2510, 2670, 2588, 2656, 2726, 2528, 2682] combine_rows_per_src=[5638, 5570, 5580, 5526, 5476, 5668, 5460, 5534] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277745664 dispatch_meta_bytes=227952 combine_payload_bytes=277745664 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277973616 ingress_bytes=387846144 link_bytes=387846144 token_bytes=555719280 weight_movement_bytes=110100480 total_nvlink_bytes=665819760 weight_to_token_ratio=0.198122 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.745282341e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44378 payload_rows_remote=38880 dup_rows=21158 dup_rows_remote=18620 dedup_ratio=0.322845 router_tokens_per_expert=[499, 536, 484, 513, 498, 503, 525, 543, 512, 543, 488, 492, 504, 524, 527, 516, 506, 514, 494, 529, 523, 510, 526, 521, 529, 503, 513, 504, 549, 501, 536, 491, 484, 516, 481, 522, 492, 517, 505, 501, 512, 519, 510, 539, 491, 532, 508, 516, 501, 531, 526, 492, 518, 510, 504, 521, 509, 497, 536, 495, 523, 494, 510, 500] payload_rows_per_dst=[5544, 5638, 5508, 5598, 5498, 5552, 5520, 5520] dup_rows_per_dst=[2658, 2574, 2738, 2654, 2538, 2702, 2686, 2608] combine_rows_per_src=[5544, 5638, 5508, 5598, 5498, 5552, 5520, 5520] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44392 payload_rows_remote=38842 dup_rows=21144 dup_rows_remote=18420 dedup_ratio=0.322632 router_tokens_per_expert=[526, 544, 531, 519, 510, 526, 502, 525, 518, 507, 512, 519, 513, 529, 493, 518, 527, 492, 562, 501, 524, 545, 478, 508, 481, 491, 535, 542, 526, 544, 537, 490, 529, 457, 507, 501, 473, 548, 530, 533, 523, 529, 467, 528, 512, 506, 475, 494, 504, 525, 517, 534, 490, 488, 491, 504, 491, 507, 504, 480, 547, 474, 488, 537] payload_rows_per_dst=[5642, 5566, 5550, 5662, 5504, 5462, 5524, 5482] dup_rows_per_dst=[2724, 2652, 2724, 2630, 2652, 2606, 2582, 2574] combine_rows_per_src=[5642, 5566, 5550, 5662, 5504, 5462, 5524, 5482] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278706176 dispatch_meta_bytes=229824 combine_payload_bytes=278706176 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278936000 ingress_bytes=388806656 link_bytes=388806656 token_bytes=557642176 weight_movement_bytes=110100480 total_nvlink_bytes=667742656 weight_to_token_ratio=0.197439 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.761987225e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44658 payload_rows_remote=39006 dup_rows=20878 dup_rows_remote=18294 dedup_ratio=0.318573 router_tokens_per_expert=[508, 506, 504, 526, 496, 470, 501, 525, 473, 486, 484, 494, 488, 511, 507, 503, 541, 523, 501, 527, 501, 562, 528, 484, 496, 513, 504, 519, 558, 530, 531, 535, 537, 519, 505, 535, 550, 523, 513, 525, 485, 524, 509, 488, 466, 509, 516, 520, 528, 534, 515, 486, 503, 491, 549, 512, 494, 502, 537, 502, 537, 510, 475, 534] payload_rows_per_dst=[5540, 5502, 5668, 5612, 5594, 5484, 5652, 5606] dup_rows_per_dst=[2532, 2390, 2666, 2760, 2820, 2550, 2584, 2576] combine_rows_per_src=[5540, 5502, 5668, 5612, 5594, 5484, 5652, 5606] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44460 payload_rows_remote=38742 dup_rows=21076 dup_rows_remote=18250 dedup_ratio=0.321594 router_tokens_per_expert=[556, 540, 514, 501, 519, 533, 525, 509, 504, 544, 533, 498, 464, 520, 540, 534, 511, 517, 511, 523, 526, 500, 489, 529, 529, 509, 515, 498, 541, 513, 486, 488, 478, 491, 523, 527, 491, 514, 497, 494, 506, 493, 480, 473, 514, 505, 532, 502, 523, 498, 514, 557, 494, 510, 488, 483, 521, 512, 528, 533, 500, 533, 491, 544] payload_rows_per_dst=[5718, 5734, 5462, 5422, 5426, 5480, 5514, 5704] dup_rows_per_dst=[2826, 2840, 2630, 2406, 2604, 2530, 2620, 2620] combine_rows_per_src=[5718, 5734, 5462, 5422, 5426, 5480, 5514, 5704] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44550 payload_rows_remote=39044 dup_rows=20986 dup_rows_remote=18428 dedup_ratio=0.320221 router_tokens_per_expert=[540, 511, 511, 530, 523, 532, 504, 506, 506, 517, 503, 525, 533, 518, 516, 542, 515, 531, 523, 523, 489, 473, 535, 534, 523, 527, 498, 536, 514, 516, 539, 497, 501, 553, 497, 529, 492, 543, 510, 551, 546, 473, 473, 479, 538, 493, 499, 531, 528, 466, 482, 522, 513, 518, 477, 510, 526, 482, 482, 510, 480, 502, 474, 498] payload_rows_per_dst=[5600, 5660, 5606, 5564, 5656, 5506, 5494, 5464] dup_rows_per_dst=[2714, 2660, 2640, 2736, 2696, 2558, 2538, 2444] combine_rows_per_src=[5600, 5660, 5606, 5564, 5656, 5506, 5494, 5464] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44326 payload_rows_remote=38694 dup_rows=21210 dup_rows_remote=18590 dedup_ratio=0.323639 router_tokens_per_expert=[503, 528, 507, 529, 507, 510, 523, 532, 489, 547, 484, 509, 515, 507, 496, 523, 486, 478, 539, 529, 572, 511, 533, 537, 530, 534, 500, 521, 512, 484, 550, 495, 526, 515, 500, 492, 490, 486, 532, 544, 526, 521, 482, 532, 495, 529, 504, 530, 464, 513, 501, 491, 453, 537, 494, 504, 513, 545, 517, 476, 537, 497, 487, 515] payload_rows_per_dst=[5544, 5484, 5580, 5632, 5496, 5596, 5424, 5570] dup_rows_per_dst=[2734, 2656, 2790, 2620, 2674, 2642, 2490, 2604] combine_rows_per_src=[5544, 5484, 5580, 5632, 5496, 5596, 5424, 5570] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278691840 dispatch_meta_bytes=230000 combine_payload_bytes=278691840 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278921840 ingress_bytes=388792320 link_bytes=388792320 token_bytes=557613680 weight_movement_bytes=110100480 total_nvlink_bytes=667714160 weight_to_token_ratio=0.197449 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.761737898e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278419456 dispatch_meta_bytes=229048 combine_payload_bytes=278419456 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278648504 ingress_bytes=608720896 link_bytes=608720896 token_bytes=557067960 weight_movement_bytes=330301440 total_nvlink_bytes=887369400 weight_to_token_ratio=0.592928 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058665756e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44508 payload_rows_remote=38706 dup_rows=21028 dup_rows_remote=18000 dedup_ratio=0.320862 router_tokens_per_expert=[520, 512, 492, 533, 535, 476, 520, 539, 533, 501, 533, 524, 489, 522, 467, 512, 534, 542, 494, 516, 494, 490, 490, 480, 523, 485, 518, 558, 514, 514, 532, 532, 509, 495, 523, 502, 552, 518, 495, 491, 526, 523, 523, 527, 495, 498, 510, 516, 533, 542, 487, 504, 510, 505, 518, 533, 480, 480, 484, 483, 513, 521, 524, 524] payload_rows_per_dst=[5256, 5524, 5506, 5558, 5622, 5652, 5588, 5802] dup_rows_per_dst=[2186, 2638, 2574, 2702, 2548, 2656, 2696, 3028] combine_rows_per_src=[5256, 5524, 5506, 5558, 5622, 5652, 5588, 5802] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279595008 dispatch_meta_bytes=229200 combine_payload_bytes=279595008 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279824208 ingress_bytes=389695488 link_bytes=389695488 token_bytes=559419216 weight_movement_bytes=110100480 total_nvlink_bytes=669519696 weight_to_token_ratio=0.196812 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4096 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.777445475e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277702656 dispatch_meta_bytes=227968 combine_payload_bytes=277702656 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277930624 ingress_bytes=497903616 link_bytes=497903616 token_bytes=555633280 weight_movement_bytes=220200960 total_nvlink_bytes=775834240 weight_to_token_ratio=0.396306 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.659362793e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279867392 dispatch_meta_bytes=229888 combine_payload_bytes=279867392 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280097280 ingress_bytes=389967872 link_bytes=389967872 token_bytes=559964672 weight_movement_bytes=110100480 total_nvlink_bytes=670065152 weight_to_token_ratio=0.196620 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.782182681e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277358592 dispatch_meta_bytes=229136 combine_payload_bytes=277358592 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=277587728 ingress_bytes=717760512 link_bytes=717760512 token_bytes=554946320 weight_movement_bytes=440401920 total_nvlink_bytes=995348240 weight_to_token_ratio=0.793594 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.248303582e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=277444608 dispatch_meta_bytes=226824 combine_payload_bytes=277444608 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277671432 ingress_bytes=387545088 link_bytes=387545088 token_bytes=555116040 weight_movement_bytes=110100480 total_nvlink_bytes=665216520 weight_to_token_ratio=0.198338 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.740046482e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44324 payload_rows_remote=38772 dup_rows=21212 dup_rows_remote=18676 dedup_ratio=0.323669 router_tokens_per_expert=[488, 514, 556, 546, 500, 519, 535, 499, 536, 483, 485, 484, 528, 491, 521, 516, 536, 507, 513, 523, 480, 540, 495, 474, 546, 520, 540, 535, 516, 501, 535, 525, 501, 551, 526, 483, 529, 524, 529, 527, 489, 524, 516, 501, 524, 510, 475, 498, 535, 467, 528, 514, 496, 540, 493, 496, 507, 508, 519, 516, 465, 524, 471, 495] payload_rows_per_dst=[5554, 5552, 5542, 5632, 5550, 5488, 5532, 5474] dup_rows_per_dst=[2760, 2536, 2594, 2804, 2790, 2586, 2606, 2536] combine_rows_per_src=[5554, 5552, 5542, 5632, 5550, 5488, 5532, 5474] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44410 payload_rows_remote=38902 dup_rows=21126 dup_rows_remote=18598 dedup_ratio=0.322357 router_tokens_per_expert=[513, 512, 523, 527, 510, 531, 520, 494, 498, 491, 507, 518, 544, 506, 496, 540, 531, 519, 514, 542, 510, 489, 498, 483, 499, 537, 498, 499, 500, 508, 510, 493, 490, 505, 525, 509, 495, 510, 511, 473, 546, 544, 544, 482, 518, 559, 491, 483, 536, 528, 513, 517, 523, 475, 539, 536, 487, 491, 493, 505, 514, 523, 505, 538] payload_rows_per_dst=[5556, 5540, 5594, 5560, 5508, 5538, 5612, 5502] dup_rows_per_dst=[2704, 2660, 2578, 2528, 2528, 2796, 2722, 2610] combine_rows_per_src=[5556, 5540, 5594, 5560, 5508, 5538, 5612, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44518 payload_rows_remote=39022 dup_rows=21018 dup_rows_remote=18462 dedup_ratio=0.320709 router_tokens_per_expert=[540, 527, 540, 488, 537, 534, 499, 499, 503, 510, 505, 519, 519, 537, 508, 544, 501, 547, 520, 516, 525, 498, 546, 511, 524, 484, 494, 497, 490, 522, 524, 516, 508, 517, 536, 507, 476, 508, 481, 494, 510, 525, 498, 512, 518, 506, 507, 533, 499, 486, 488, 476, 493, 533, 558, 493, 522, 510, 518, 519, 512, 506, 496, 499] payload_rows_per_dst=[5680, 5600, 5620, 5514, 5492, 5592, 5496, 5524] dup_rows_per_dst=[2648, 2690, 2708, 2588, 2562, 2626, 2556, 2640] combine_rows_per_src=[5680, 5600, 5620, 5514, 5492, 5592, 5496, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44630 payload_rows_remote=39050 dup_rows=20906 dup_rows_remote=18234 dedup_ratio=0.319000 router_tokens_per_expert=[487, 522, 545, 559, 539, 474, 517, 510, 495, 540, 548, 492, 508, 531, 511, 496, 511, 476, 527, 495, 526, 506, 477, 461, 491, 483, 525, 474, 531, 497, 506, 520, 516, 517, 515, 536, 532, 482, 509, 519, 529, 543, 492, 536, 537, 518, 494, 515, 481, 505, 540, 525, 535, 527, 535, 527, 492, 519, 479, 525, 502, 496, 502, 508] payload_rows_per_dst=[5596, 5584, 5484, 5546, 5580, 5654, 5632, 5554] dup_rows_per_dst=[2710, 2658, 2474, 2508, 2672, 2674, 2718, 2492] combine_rows_per_src=[5596, 5584, 5484, 5546, 5580, 5654, 5632, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279824384 dispatch_meta_bytes=229672 combine_payload_bytes=279824384 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280054056 ingress_bytes=389924864 link_bytes=389924864 token_bytes=559878440 weight_movement_bytes=110100480 total_nvlink_bytes=669978920 weight_to_token_ratio=0.196651 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.781434701e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44306 payload_rows_remote=38684 dup_rows=21230 dup_rows_remote=18404 dedup_ratio=0.323944 router_tokens_per_expert=[479, 533, 500, 536, 483, 522, 518, 477, 465, 494, 508, 515, 530, 528, 532, 494, 520, 507, 485, 492, 508, 528, 464, 488, 461, 494, 508, 537, 495, 533, 516, 507, 550, 530, 501, 493, 486, 547, 544, 529, 525, 545, 515, 580, 490, 530, 526, 513, 498, 497, 511, 513, 531, 497, 573, 511, 488, 486, 531, 493, 528, 484, 526, 540] payload_rows_per_dst=[5546, 5560, 5460, 5458, 5554, 5622, 5580, 5526] dup_rows_per_dst=[2550, 2572, 2524, 2644, 2806, 2826, 2682, 2626] combine_rows_per_src=[5546, 5560, 5460, 5458, 5554, 5622, 5580, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277788672 dispatch_meta_bytes=228880 combine_payload_bytes=277788672 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278017552 ingress_bytes=497989632 link_bytes=497989632 token_bytes=555806224 weight_movement_bytes=220200960 total_nvlink_bytes=776007184 weight_to_token_ratio=0.396183 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.660858753e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279121920 dispatch_meta_bytes=228632 combine_payload_bytes=279121920 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279350552 ingress_bytes=499322880 link_bytes=499322880 token_bytes=558472472 weight_movement_bytes=220200960 total_nvlink_bytes=778673432 weight_to_token_ratio=0.394292 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.684046128e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279910400 dispatch_meta_bytes=229136 combine_payload_bytes=279910400 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280139536 ingress_bytes=610211840 link_bytes=610211840 token_bytes=560049936 weight_movement_bytes=330301440 total_nvlink_bytes=890351376 weight_to_token_ratio=0.589771 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061258752e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279709696 dispatch_meta_bytes=229936 combine_payload_bytes=279709696 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279939632 ingress_bytes=499910656 link_bytes=499910656 token_bytes=559649328 weight_movement_bytes=220200960 total_nvlink_bytes=779850288 weight_to_token_ratio=0.393462 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.694268520e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44314 payload_rows_remote=38696 dup_rows=21222 dup_rows_remote=18442 dedup_ratio=0.323822 router_tokens_per_expert=[511, 530, 515, 512, 496, 516, 519, 497, 513, 472, 499, 526, 501, 521, 518, 498, 572, 495, 472, 474, 498, 544, 531, 516, 518, 488, 547, 490, 503, 514, 501, 515, 522, 531, 497, 499, 480, 512, 500, 523, 529, 543, 521, 494, 518, 480, 523, 479, 537, 516, 537, 501, 473, 518, 502, 520, 501, 531, 531, 541, 525, 516, 517, 529] payload_rows_per_dst=[5618, 5600, 5744, 5556, 5460, 5310, 5416, 5610] dup_rows_per_dst=[2780, 2782, 2994, 2614, 2370, 2330, 2586, 2766] combine_rows_per_src=[5618, 5600, 5744, 5556, 5460, 5310, 5416, 5610] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44450 payload_rows_remote=38888 dup_rows=21086 dup_rows_remote=18594 dedup_ratio=0.321747 router_tokens_per_expert=[466, 508, 521, 520, 535, 529, 543, 534, 471, 524, 514, 524, 503, 526, 492, 473, 513, 499, 490, 535, 521, 515, 491, 555, 511, 509, 532, 511, 504, 471, 532, 486, 529, 518, 494, 473, 504, 507, 513, 534, 511, 504, 536, 549, 539, 514, 531, 514, 490, 460, 475, 515, 501, 525, 541, 505, 511, 513, 512, 527, 510, 559, 482, 514] payload_rows_per_dst=[5616, 5562, 5578, 5530, 5478, 5670, 5468, 5548] dup_rows_per_dst=[2696, 2492, 2660, 2582, 2666, 2726, 2556, 2708] combine_rows_per_src=[5616, 5562, 5578, 5530, 5478, 5670, 5468, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277286912 dispatch_meta_bytes=228352 combine_payload_bytes=277286912 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=277515264 ingress_bytes=497487872 link_bytes=497487872 token_bytes=554802176 weight_movement_bytes=220200960 total_nvlink_bytes=775003136 weight_to_token_ratio=0.396900 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.652132321e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278749184 dispatch_meta_bytes=229928 combine_payload_bytes=278749184 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278979112 ingress_bytes=388849664 link_bytes=388849664 token_bytes=557728296 weight_movement_bytes=110100480 total_nvlink_bytes=667828776 weight_to_token_ratio=0.197409 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.762735205e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277372928 dispatch_meta_bytes=228552 combine_payload_bytes=277372928 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277601480 ingress_bytes=387473408 link_bytes=387473408 token_bytes=554974408 weight_movement_bytes=110100480 total_nvlink_bytes=665074888 weight_to_token_ratio=0.198388 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.738799849e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44386 payload_rows_remote=38848 dup_rows=21150 dup_rows_remote=18436 dedup_ratio=0.322723 router_tokens_per_expert=[524, 544, 525, 514, 509, 523, 500, 523, 517, 507, 509, 518, 513, 527, 490, 517, 523, 490, 564, 498, 523, 544, 478, 506, 479, 492, 534, 541, 525, 543, 538, 494, 530, 455, 508, 501, 474, 549, 531, 536, 524, 527, 471, 527, 510, 505, 476, 497, 507, 526, 520, 536, 492, 488, 492, 505, 493, 510, 508, 481, 551, 478, 490, 538] payload_rows_per_dst=[5624, 5564, 5538, 5658, 5516, 5456, 5532, 5498] dup_rows_per_dst=[2700, 2632, 2714, 2634, 2652, 2618, 2600, 2600] combine_rows_per_src=[5624, 5564, 5538, 5658, 5516, 5456, 5532, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44406 payload_rows_remote=38858 dup_rows=21130 dup_rows_remote=18434 dedup_ratio=0.322418 router_tokens_per_expert=[504, 532, 507, 478, 505, 485, 547, 534, 530, 482, 522, 485, 484, 525, 490, 514, 498, 536, 504, 514, 489, 515, 469, 549, 542, 517, 532, 541, 484, 527, 502, 522, 500, 517, 455, 500, 522, 538, 508, 486, 528, 535, 479, 530, 536, 560, 535, 533, 510, 537, 509, 487, 527, 500, 517, 479, 493, 497, 488, 504, 552, 486, 538, 517] payload_rows_per_dst=[5474, 5474, 5572, 5358, 5606, 5660, 5714, 5548] dup_rows_per_dst=[2406, 2590, 2688, 2428, 2614, 2718, 2990, 2696] combine_rows_per_src=[5474, 5474, 5572, 5358, 5606, 5660, 5714, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44322 payload_rows_remote=38688 dup_rows=21214 dup_rows_remote=18588 dedup_ratio=0.323700 router_tokens_per_expert=[497, 525, 505, 525, 504, 504, 520, 531, 486, 543, 482, 509, 516, 507, 492, 520, 484, 477, 537, 526, 570, 508, 532, 535, 533, 537, 497, 520, 514, 486, 551, 492, 528, 515, 500, 495, 491, 483, 534, 545, 525, 523, 483, 532, 497, 531, 508, 533, 464, 515, 503, 491, 454, 538, 500, 506, 517, 545, 521, 477, 539, 499, 490, 521] payload_rows_per_dst=[5522, 5472, 5566, 5634, 5500, 5610, 5436, 5582] dup_rows_per_dst=[2700, 2638, 2772, 2626, 2682, 2654, 2506, 2636] combine_rows_per_src=[5522, 5472, 5566, 5634, 5500, 5610, 5436, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44398 payload_rows_remote=38898 dup_rows=21138 dup_rows_remote=18606 dedup_ratio=0.322540 router_tokens_per_expert=[493, 532, 482, 509, 497, 496, 520, 544, 511, 541, 490, 491, 500, 522, 526, 514, 505, 514, 494, 528, 523, 509, 525, 520, 523, 503, 514, 506, 550, 498, 535, 492, 485, 514, 480, 520, 492, 518, 505, 502, 510, 520, 509, 539, 492, 533, 507, 515, 502, 532, 526, 497, 523, 513, 504, 525, 514, 497, 540, 500, 527, 500, 514, 506] payload_rows_per_dst=[5518, 5630, 5510, 5590, 5500, 5560, 5538, 5552] dup_rows_per_dst=[2628, 2560, 2726, 2652, 2532, 2690, 2706, 2644] combine_rows_per_src=[5518, 5630, 5510, 5590, 5500, 5560, 5538, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44644 payload_rows_remote=38972 dup_rows=20892 dup_rows_remote=18290 dedup_ratio=0.318787 router_tokens_per_expert=[508, 499, 499, 519, 496, 469, 497, 525, 471, 485, 482, 494, 488, 510, 504, 501, 538, 524, 501, 528, 503, 562, 527, 483, 497, 512, 506, 517, 558, 530, 533, 539, 536, 518, 503, 529, 549, 523, 512, 525, 487, 525, 512, 489, 467, 508, 517, 521, 527, 535, 516, 490, 507, 492, 554, 516, 499, 505, 537, 502, 538, 511, 476, 537] payload_rows_per_dst=[5508, 5498, 5668, 5610, 5582, 5490, 5672, 5616] dup_rows_per_dst=[2516, 2372, 2664, 2774, 2808, 2562, 2602, 2594] combine_rows_per_src=[5508, 5498, 5668, 5610, 5582, 5490, 5672, 5616] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278462464 dispatch_meta_bytes=229136 combine_payload_bytes=278462464 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278691600 ingress_bytes=498663424 link_bytes=498663424 token_bytes=557154064 weight_movement_bytes=220200960 total_nvlink_bytes=777355024 weight_to_token_ratio=0.395225 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.672577104e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278534144 dispatch_meta_bytes=229168 combine_payload_bytes=278534144 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278763312 ingress_bytes=388634624 link_bytes=388634624 token_bytes=557297456 weight_movement_bytes=110100480 total_nvlink_bytes=667397936 weight_to_token_ratio=0.197561 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.758995305e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44548 payload_rows_remote=39030 dup_rows=20988 dup_rows_remote=18416 dedup_ratio=0.320251 router_tokens_per_expert=[538, 510, 509, 529, 522, 529, 499, 501, 502, 515, 501, 522, 533, 516, 512, 538, 512, 527, 521, 524, 490, 470, 534, 531, 522, 525, 500, 538, 510, 517, 538, 497, 502, 556, 497, 528, 492, 544, 509, 556, 548, 479, 476, 481, 537, 494, 499, 531, 531, 466, 483, 527, 513, 520, 482, 513, 530, 482, 485, 513, 481, 502, 474, 505] payload_rows_per_dst=[5586, 5634, 5588, 5558, 5670, 5518, 5514, 5480] dup_rows_per_dst=[2688, 2644, 2630, 2736, 2698, 2572, 2556, 2464] combine_rows_per_src=[5586, 5634, 5588, 5558, 5670, 5518, 5514, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277315584 dispatch_meta_bytes=229104 combine_payload_bytes=277315584 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277544688 ingress_bytes=497516544 link_bytes=497516544 token_bytes=554860272 weight_movement_bytes=220200960 total_nvlink_bytes=775061232 weight_to_token_ratio=0.396858 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.652630974e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278820864 dispatch_meta_bytes=230016 combine_payload_bytes=278820864 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279050880 ingress_bytes=388921344 link_bytes=388921344 token_bytes=557871744 weight_movement_bytes=110100480 total_nvlink_bytes=667972224 weight_to_token_ratio=0.197358 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.763981838e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279351296 dispatch_meta_bytes=229048 combine_payload_bytes=279351296 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279580344 ingress_bytes=389451776 link_bytes=389451776 token_bytes=558931640 weight_movement_bytes=110100480 total_nvlink_bytes=669032120 weight_to_token_ratio=0.196984 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773206923e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44306 payload_rows_remote=38768 dup_rows=21230 dup_rows_remote=18700 dedup_ratio=0.323944 router_tokens_per_expert=[484, 512, 550, 544, 497, 515, 532, 499, 534, 481, 484, 481, 526, 490, 521, 517, 535, 507, 513, 523, 477, 540, 493, 474, 541, 521, 538, 533, 518, 503, 534, 523, 500, 553, 527, 483, 530, 524, 528, 529, 491, 523, 519, 501, 522, 512, 476, 498, 536, 468, 529, 516, 502, 542, 496, 498, 508, 510, 519, 519, 468, 526, 473, 502] payload_rows_per_dst=[5536, 5538, 5536, 5622, 5550, 5494, 5546, 5484] dup_rows_per_dst=[2730, 2530, 2588, 2800, 2798, 2590, 2628, 2566] combine_rows_per_src=[5536, 5538, 5536, 5622, 5550, 5494, 5546, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44428 payload_rows_remote=38784 dup_rows=21108 dup_rows_remote=18398 dedup_ratio=0.322083 router_tokens_per_expert=[549, 537, 511, 499, 518, 534, 524, 505, 503, 543, 529, 496, 462, 519, 535, 533, 509, 518, 513, 522, 524, 499, 486, 527, 530, 507, 513, 502, 543, 515, 484, 489, 477, 492, 523, 529, 492, 515, 495, 493, 506, 492, 480, 476, 516, 508, 534, 504, 522, 499, 513, 561, 494, 511, 489, 484, 523, 514, 532, 536, 502, 536, 494, 548] payload_rows_per_dst=[5644, 5788, 5268, 5570, 5432, 5488, 5516, 5722] dup_rows_per_dst=[2710, 2986, 2394, 2596, 2600, 2544, 2630, 2648] combine_rows_per_src=[5644, 5788, 5268, 5570, 5432, 5488, 5516, 5722] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279767040 dispatch_meta_bytes=229784 combine_payload_bytes=279767040 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=279996824 ingress_bytes=389867520 link_bytes=389867520 token_bytes=559763864 weight_movement_bytes=110100480 total_nvlink_bytes=669864344 weight_to_token_ratio=0.196691 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.780437395e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44456 payload_rows_remote=38930 dup_rows=21080 dup_rows_remote=18508 dedup_ratio=0.321655 router_tokens_per_expert=[539, 512, 543, 506, 504, 484, 500, 516, 498, 490, 506, 547, 496, 553, 533, 533, 467, 512, 513, 534, 503, 503, 503, 514, 486, 485, 494, 520, 552, 507, 501, 500, 483, 507, 466, 486, 549, 537, 539, 525, 528, 548, 518, 501, 522, 537, 479, 535, 469, 553, 521, 507, 518, 469, 534, 505, 497, 468, 534, 546, 502, 510, 509, 512] payload_rows_per_dst=[5530, 5600, 5526, 5504, 5552, 5610, 5544, 5590] dup_rows_per_dst=[2678, 2712, 2572, 2586, 2632, 2726, 2608, 2566] combine_rows_per_src=[5530, 5600, 5526, 5504, 5552, 5610, 5544, 5590] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=277917696 dispatch_meta_bytes=229792 combine_payload_bytes=277917696 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278147488 ingress_bytes=388018176 link_bytes=388018176 token_bytes=556065184 weight_movement_bytes=110100480 total_nvlink_bytes=666165664 weight_to_token_ratio=0.197999 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.748274261e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44446 payload_rows_remote=38902 dup_rows=21090 dup_rows_remote=18528 dedup_ratio=0.321808 router_tokens_per_expert=[527, 510, 543, 500, 489, 520, 500, 486, 503, 522, 529, 505, 501, 466, 517, 509, 557, 495, 474, 490, 541, 504, 489, 501, 539, 522, 497, 512, 510, 520, 541, 482, 502, 507, 540, 529, 531, 553, 495, 535, 522, 487, 495, 513, 518, 503, 496, 487, 515, 500, 511, 509, 509, 507, 479, 523, 518, 519, 468, 587, 550, 482, 550, 527] payload_rows_per_dst=[5494, 5504, 5516, 5620, 5690, 5462, 5544, 5616] dup_rows_per_dst=[2656, 2600, 2586, 2626, 2694, 2580, 2562, 2786] combine_rows_per_src=[5494, 5504, 5516, 5620, 5690, 5462, 5544, 5616] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=277889024 dispatch_meta_bytes=229872 combine_payload_bytes=277889024 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278118896 ingress_bytes=387989504 link_bytes=387989504 token_bytes=556007920 weight_movement_bytes=110100480 total_nvlink_bytes=666108400 weight_to_token_ratio=0.198020 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.747775608e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44468 payload_rows_remote=38946 dup_rows=21068 dup_rows_remote=18508 dedup_ratio=0.321472 router_tokens_per_expert=[538, 510, 539, 503, 500, 480, 497, 513, 494, 487, 504, 545, 493, 555, 531, 533, 469, 510, 512, 533, 504, 500, 500, 513, 485, 483, 492, 521, 553, 509, 501, 497, 483, 507, 465, 487, 550, 538, 537, 526, 529, 547, 517, 503, 524, 541, 483, 537, 470, 556, 525, 509, 523, 470, 536, 507, 497, 472, 537, 547, 503, 513, 511, 514] payload_rows_per_dst=[5506, 5590, 5522, 5506, 5554, 5624, 5564, 5602] dup_rows_per_dst=[2654, 2694, 2560, 2576, 2632, 2738, 2628, 2586] combine_rows_per_src=[5506, 5590, 5522, 5506, 5554, 5624, 5564, 5602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278003712 dispatch_meta_bytes=228728 combine_payload_bytes=278003712 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278232440 ingress_bytes=608305152 link_bytes=608305152 token_bytes=556236152 weight_movement_bytes=330301440 total_nvlink_bytes=886537592 weight_to_token_ratio=0.593815 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057942708e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44502 payload_rows_remote=38814 dup_rows=21034 dup_rows_remote=18194 dedup_ratio=0.320953 router_tokens_per_expert=[517, 512, 488, 531, 533, 476, 519, 536, 529, 499, 531, 522, 485, 520, 465, 510, 530, 539, 495, 516, 491, 490, 488, 480, 524, 484, 517, 555, 517, 516, 533, 532, 504, 498, 524, 501, 553, 519, 499, 492, 527, 523, 526, 528, 496, 500, 512, 517, 534, 543, 485, 508, 512, 506, 522, 534, 480, 482, 490, 484, 512, 521, 530, 526] payload_rows_per_dst=[5560, 5516, 5488, 5566, 5660, 5418, 5606, 5688] dup_rows_per_dst=[2628, 2606, 2570, 2736, 2610, 2318, 2726, 2840] combine_rows_per_src=[5560, 5516, 5488, 5566, 5660, 5418, 5606, 5688] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44444 payload_rows_remote=38938 dup_rows=21092 dup_rows_remote=18560 dedup_ratio=0.321838 router_tokens_per_expert=[511, 507, 522, 522, 505, 530, 518, 492, 496, 489, 505, 518, 543, 504, 493, 537, 528, 519, 514, 545, 505, 489, 499, 483, 499, 537, 499, 496, 497, 510, 511, 491, 489, 506, 524, 510, 494, 512, 512, 472, 544, 546, 547, 483, 515, 560, 492, 489, 535, 531, 512, 517, 529, 477, 542, 539, 493, 491, 493, 507, 517, 527, 507, 542] payload_rows_per_dst=[5548, 5528, 5598, 5560, 5506, 5550, 5630, 5524] dup_rows_per_dst=[2666, 2642, 2566, 2520, 2532, 2802, 2734, 2630] combine_rows_per_src=[5548, 5528, 5598, 5560, 5506, 5550, 5630, 5524] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44504 payload_rows_remote=38762 dup_rows=21032 dup_rows_remote=18016 dedup_ratio=0.320923 router_tokens_per_expert=[502, 487, 528, 531, 524, 512, 540, 539, 532, 512, 518, 546, 497, 519, 474, 509, 517, 539, 505, 515, 497, 475, 535, 533, 494, 577, 492, 523, 520, 483, 478, 504, 550, 484, 501, 524, 503, 453, 539, 533, 524, 553, 520, 497, 495, 537, 527, 528, 538, 513, 499, 512, 511, 463, 539, 542, 460, 486, 474, 468, 493, 511, 530, 504] payload_rows_per_dst=[5742, 5738, 5652, 5546, 5496, 5452, 5460, 5418] dup_rows_per_dst=[3016, 2688, 2794, 2600, 2460, 2478, 2562, 2434] combine_rows_per_src=[5742, 5738, 5652, 5546, 5496, 5452, 5460, 5418] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44404 payload_rows_remote=38864 dup_rows=21132 dup_rows_remote=18404 dedup_ratio=0.322449 router_tokens_per_expert=[530, 546, 517, 511, 470, 505, 545, 519, 483, 492, 492, 492, 538, 530, 505, 533, 511, 496, 498, 496, 574, 518, 510, 518, 507, 533, 486, 516, 532, 520, 504, 479, 541, 495, 512, 510, 516, 493, 502, 499, 533, 518, 522, 493, 527, 522, 520, 499, 536, 547, 554, 494, 483, 502, 479, 525, 530, 497, 498, 481, 502, 536, 475, 521] payload_rows_per_dst=[5540, 5518, 5616, 5504, 5572, 5540, 5578, 5536] dup_rows_per_dst=[2746, 2612, 2626, 2650, 2564, 2728, 2662, 2544] combine_rows_per_src=[5540, 5518, 5616, 5504, 5572, 5540, 5578, 5536] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278849536 dispatch_meta_bytes=230000 combine_payload_bytes=278849536 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279079536 ingress_bytes=499050496 link_bytes=499050496 token_bytes=557929072 weight_movement_bytes=220200960 total_nvlink_bytes=778130032 weight_to_token_ratio=0.394676 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.679308922e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44568 payload_rows_remote=39006 dup_rows=20968 dup_rows_remote=18414 dedup_ratio=0.319946 router_tokens_per_expert=[508, 485, 482, 510, 504, 511, 537, 522, 461, 523, 525, 524, 485, 525, 518, 496, 524, 518, 507, 511, 515, 516, 517, 522, 557, 492, 522, 501, 491, 495, 476, 524, 530, 479, 513, 513, 545, 502, 572, 511, 525, 488, 545, 528, 527, 478, 492, 539, 482, 515, 523, 523, 512, 505, 541, 529, 541, 489, 498, 537, 478, 513, 498, 493] payload_rows_per_dst=[5614, 5538, 5570, 5562, 5600, 5576, 5670, 5438] dup_rows_per_dst=[2504, 2576, 2690, 2554, 2730, 2668, 2590, 2656] combine_rows_per_src=[5614, 5538, 5570, 5562, 5600, 5576, 5670, 5438] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279050240 dispatch_meta_bytes=229752 combine_payload_bytes=279050240 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279279992 ingress_bytes=389150720 link_bytes=389150720 token_bytes=558330232 weight_movement_bytes=110100480 total_nvlink_bytes=668430712 weight_to_token_ratio=0.197196 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.767971064e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44214 payload_rows_remote=38586 dup_rows=21322 dup_rows_remote=18532 dedup_ratio=0.325348 router_tokens_per_expert=[514, 480, 489, 528, 491, 485, 530, 511, 499, 516, 546, 479, 494, 528, 511, 541, 491, 522, 482, 557, 524, 555, 507, 484, 481, 495, 507, 474, 525, 529, 539, 524, 471, 521, 518, 486, 518, 535, 556, 565, 515, 500, 502, 501, 517, 507, 517, 547, 506, 501, 536, 489, 511, 517, 484, 504, 508, 544, 529, 532, 496, 517, 469, 511] payload_rows_per_dst=[5490, 5312, 5300, 5612, 5758, 5696, 5418, 5628] dup_rows_per_dst=[2508, 2286, 2364, 2768, 3032, 2896, 2678, 2790] combine_rows_per_src=[5490, 5312, 5300, 5612, 5758, 5696, 5418, 5628] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278849536 dispatch_meta_bytes=229720 combine_payload_bytes=278849536 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279079256 ingress_bytes=499050496 link_bytes=499050496 token_bytes=557928792 weight_movement_bytes=220200960 total_nvlink_bytes=778129752 weight_to_token_ratio=0.394676 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.679308922e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=277846016 dispatch_meta_bytes=227112 combine_payload_bytes=277846016 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278073128 ingress_bytes=498046976 link_bytes=498046976 token_bytes=555919144 weight_movement_bytes=220200960 total_nvlink_bytes=776120104 weight_to_token_ratio=0.396102 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.661856059e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278577152 dispatch_meta_bytes=229072 combine_payload_bytes=278577152 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278806224 ingress_bytes=498778112 link_bytes=498778112 token_bytes=557383376 weight_movement_bytes=220200960 total_nvlink_bytes=777584336 weight_to_token_ratio=0.395062 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.674571717e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279595008 dispatch_meta_bytes=229680 combine_payload_bytes=279595008 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279824688 ingress_bytes=609896448 link_bytes=609896448 token_bytes=559419696 weight_movement_bytes=330301440 total_nvlink_bytes=889721136 weight_to_token_ratio=0.590436 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060710234e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44436 payload_rows_remote=38850 dup_rows=21100 dup_rows_remote=18422 dedup_ratio=0.321960 router_tokens_per_expert=[473, 492, 517, 553, 520, 536, 537, 549, 540, 488, 541, 489, 510, 529, 530, 505, 506, 494, 498, 567, 479, 512, 515, 495, 521, 495, 525, 529, 500, 529, 532, 478, 518, 482, 502, 520, 460, 503, 498, 518, 511, 481, 499, 486, 515, 516, 527, 529, 580, 529, 530, 528, 515, 474, 481, 535, 513, 516, 495, 506, 540, 465, 502, 510] payload_rows_per_dst=[5592, 5586, 5532, 5570, 5480, 5604, 5582, 5490] dup_rows_per_dst=[2762, 2678, 2600, 2648, 2522, 2524, 2762, 2604] combine_rows_per_src=[5592, 5586, 5532, 5570, 5480, 5604, 5582, 5490] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=276584448 dispatch_meta_bytes=228472 combine_payload_bytes=276584448 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276812920 ingress_bytes=386684928 link_bytes=386684928 token_bytes=553397368 weight_movement_bytes=110100480 total_nvlink_bytes=663497848 weight_to_token_ratio=0.198954 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.725086885e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44440 payload_rows_remote=38954 dup_rows=21096 dup_rows_remote=18490 dedup_ratio=0.321899 router_tokens_per_expert=[484, 514, 501, 489, 511, 540, 509, 496, 519, 523, 508, 542, 519, 506, 504, 546, 516, 511, 512, 534, 511, 507, 515, 561, 522, 481, 497, 525, 479, 477, 495, 545, 470, 474, 527, 509, 506, 505, 524, 531, 516, 511, 487, 522, 531, 540, 548, 517, 476, 529, 465, 510, 534, 497, 540, 512, 523, 500, 493, 498, 502, 531, 525, 516] payload_rows_per_dst=[5532, 5614, 5594, 5526, 5486, 5626, 5602, 5460] dup_rows_per_dst=[2556, 2720, 2740, 2516, 2606, 2718, 2524, 2716] combine_rows_per_src=[5532, 5614, 5594, 5526, 5486, 5626, 5602, 5460] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44482 payload_rows_remote=38868 dup_rows=21054 dup_rows_remote=18480 dedup_ratio=0.321259 router_tokens_per_expert=[510, 515, 510, 478, 556, 532, 536, 490, 529, 554, 520, 525, 554, 516, 516, 503, 495, 509, 547, 494, 502, 496, 525, 526, 515, 487, 509, 524, 515, 538, 536, 537, 499, 488, 521, 514, 510, 520, 515, 493, 494, 496, 529, 483, 506, 486, 481, 515, 501, 468, 527, 498, 534, 543, 533, 504, 513, 505, 504, 525, 481, 506, 480, 497] payload_rows_per_dst=[5540, 5642, 5614, 5614, 5476, 5482, 5588, 5526] dup_rows_per_dst=[2714, 2792, 2574, 2708, 2644, 2498, 2628, 2496] combine_rows_per_src=[5540, 5642, 5614, 5614, 5476, 5482, 5588, 5526] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278476800 dispatch_meta_bytes=229088 combine_payload_bytes=278476800 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278705888 ingress_bytes=608778240 link_bytes=608778240 token_bytes=557182688 weight_movement_bytes=330301440 total_nvlink_bytes=887484128 weight_to_token_ratio=0.592806 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058765486e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5559029968 prefetch_bytes=1101004800 reduce_bytes=110100480 weight_movement_bytes=1211105280 egress_bytes=2780655824 ingress_bytes=3989479424 link_bytes=3989479424 total_nvlink_bytes=6770135248 weight_to_token_ratio=0.217863 prefetch_share_of_ingress=0.275977 dedup_ratio=0.321976 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=1 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043015 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=6.938360875e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44630 payload_rows_remote=39094 dup_rows=20906 dup_rows_remote=18320 dedup_ratio=0.319000 router_tokens_per_expert=[516, 498, 517, 557, 503, 509, 472, 516, 510, 540, 502, 513, 486, 517, 510, 529, 518, 500, 508, 561, 531, 491, 523, 562, 514, 506, 511, 537, 493, 513, 503, 530, 495, 523, 565, 510, 498, 524, 499, 497, 529, 481, 495, 478, 491, 528, 527, 516, 516, 510, 510, 482, 487, 522, 518, 516, 504, 530, 508, 473, 493, 494, 545, 508] payload_rows_per_dst=[5562, 5580, 5726, 5600, 5552, 5482, 5536, 5592] dup_rows_per_dst=[2614, 2634, 2662, 2614, 2670, 2608, 2586, 2518] combine_rows_per_src=[5562, 5580, 5726, 5600, 5552, 5482, 5536, 5592] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44538 payload_rows_remote=38904 dup_rows=20998 dup_rows_remote=18376 dedup_ratio=0.320404 router_tokens_per_expert=[476, 521, 495, 520, 533, 527, 529, 522, 505, 527, 524, 537, 494, 517, 480, 517, 520, 528, 541, 502, 524, 492, 520, 486, 480, 488, 524, 571, 534, 488, 506, 556, 534, 542, 525, 482, 523, 539, 489, 486, 529, 530, 476, 507, 533, 467, 489, 510, 519, 502, 489, 514, 510, 525, 425, 535, 531, 527, 493, 512, 490, 486, 537, 528] payload_rows_per_dst=[5634, 5572, 5590, 5572, 5630, 5478, 5506, 5556] dup_rows_per_dst=[2622, 2630, 2636, 2722, 2610, 2604, 2522, 2652] combine_rows_per_src=[5634, 5572, 5590, 5572, 5630, 5478, 5506, 5556] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279222272 dispatch_meta_bytes=229776 combine_payload_bytes=279222272 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279452048 ingress_bytes=389322752 link_bytes=389322752 token_bytes=558674320 weight_movement_bytes=110100480 total_nvlink_bytes=668774800 weight_to_token_ratio=0.197075 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.770962983e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44538 payload_rows_remote=38956 dup_rows=20998 dup_rows_remote=18462 dedup_ratio=0.320404 router_tokens_per_expert=[526, 505, 520, 488, 481, 486, 532, 518, 510, 482, 493, 497, 548, 516, 525, 512, 498, 537, 546, 511, 507, 516, 533, 511, 499, 495, 496, 508, 497, 489, 488, 537, 534, 464, 500, 502, 479, 487, 500, 545, 543, 513, 503, 547, 467, 517, 510, 459, 540, 540, 515, 484, 525, 496, 556, 515, 510, 543, 537, 506, 548, 527, 521, 528] payload_rows_per_dst=[5490, 5542, 5646, 5506, 5498, 5582, 5596, 5678] dup_rows_per_dst=[2622, 2624, 2672, 2512, 2524, 2536, 2746, 2762] combine_rows_per_src=[5490, 5542, 5646, 5506, 5498, 5582, 5596, 5678] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278605824 dispatch_meta_bytes=229392 combine_payload_bytes=278605824 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278835216 ingress_bytes=608907264 link_bytes=608907264 token_bytes=557441040 weight_movement_bytes=330301440 total_nvlink_bytes=887742480 weight_to_token_ratio=0.592532 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058989880e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44672 payload_rows_remote=39154 dup_rows=20864 dup_rows_remote=18334 dedup_ratio=0.318359 router_tokens_per_expert=[510, 528, 535, 515, 531, 504, 528, 498, 516, 546, 538, 510, 536, 500, 505, 511, 514, 473, 557, 517, 508, 549, 513, 512, 531, 494, 533, 503, 481, 512, 492, 478, 547, 495, 484, 499, 488, 519, 484, 508, 535, 517, 510, 497, 532, 504, 523, 483, 519, 504, 505, 525, 528, 525, 485, 540, 481, 472, 512, 522, 519, 517, 507, 504] payload_rows_per_dst=[5626, 5590, 5546, 5518, 5498, 5664, 5650, 5580] dup_rows_per_dst=[2672, 2734, 2740, 2530, 2550, 2538, 2612, 2488] combine_rows_per_src=[5626, 5590, 5546, 5518, 5498, 5664, 5650, 5580] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5586481584 prefetch_bytes=990904320 reduce_bytes=1101004800 weight_movement_bytes=2091909120 egress_bytes=2794387888 ingress_bytes=4884002816 link_bytes=4884002816 total_nvlink_bytes=7678390704 weight_to_token_ratio=0.374459 prefetch_share_of_ingress=0.202888 dedup_ratio=0.320819 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=10 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043382 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.494084177e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=280225792 dispatch_meta_bytes=229656 combine_payload_bytes=280225792 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=280455448 ingress_bytes=390326272 link_bytes=390326272 token_bytes=560681240 weight_movement_bytes=110100480 total_nvlink_bytes=670781720 weight_to_token_ratio=0.196369 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.788415846e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5583844608 prefetch_bytes=880803840 reduce_bytes=880803840 weight_movement_bytes=1761607680 egress_bytes=2793069824 ingress_bytes=4552382464 link_bytes=4552382464 total_nvlink_bytes=7345452288 weight_to_token_ratio=0.315483 prefetch_share_of_ingress=0.193482 dedup_ratio=0.321255 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=8 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.917341843e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278863872 dispatch_meta_bytes=229120 combine_payload_bytes=278863872 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279092992 ingress_bytes=609165312 link_bytes=609165312 token_bytes=557956864 weight_movement_bytes=330301440 total_nvlink_bytes=888258304 weight_to_token_ratio=0.591984 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059438668e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5580890984 prefetch_bytes=880803840 reduce_bytes=1761607680 weight_movement_bytes=2642411520 egress_bytes=2791592808 ingress_bytes=5431709696 link_bytes=5431709696 total_nvlink_bytes=8223302504 weight_to_token_ratio=0.473475 prefetch_share_of_ingress=0.162160 dedup_ratio=0.321606 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=16 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.039706 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.446636524e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279236608 dispatch_meta_bytes=229672 combine_payload_bytes=279236608 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279466280 ingress_bytes=389337088 link_bytes=389337088 token_bytes=558702888 weight_movement_bytes=110100480 total_nvlink_bytes=668803368 weight_to_token_ratio=0.197064 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.771212310e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=280655872 dispatch_meta_bytes=229952 combine_payload_bytes=280655872 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280885824 ingress_bytes=390756352 link_bytes=390756352 token_bytes=561541696 weight_movement_bytes=110100480 total_nvlink_bytes=671642176 weight_to_token_ratio=0.196068 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.795895645e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5590812008 prefetch_bytes=990904320 reduce_bytes=550502400 weight_movement_bytes=1541406720 egress_bytes=2796553576 ingress_bytes=4335665152 link_bytes=4335665152 total_nvlink_bytes=7132218728 weight_to_token_ratio=0.275704 prefetch_share_of_ingress=0.228547 dedup_ratio=0.320425 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=5 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.540434794e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5571479752 prefetch_bytes=880803840 reduce_bytes=1321205760 weight_movement_bytes=2202009600 egress_bytes=2786883784 ingress_bytes=4986605568 link_bytes=4986605568 total_nvlink_bytes=7773489352 weight_to_token_ratio=0.395229 prefetch_share_of_ingress=0.176634 dedup_ratio=0.321436 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=12 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043015 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.672527238e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5585765312 prefetch_bytes=1101004800 reduce_bytes=440401920 weight_movement_bytes=1541406720 egress_bytes=2794030016 ingress_bytes=4333142016 link_bytes=4333142016 total_nvlink_bytes=7127172032 weight_to_token_ratio=0.275953 prefetch_share_of_ingress=0.254089 dedup_ratio=0.321170 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=4 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040441 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.536046645e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5588861272 prefetch_bytes=770703360 reduce_bytes=1431306240 weight_movement_bytes=2202009600 egress_bytes=2795577688 ingress_bytes=4995293184 link_bytes=4995293184 total_nvlink_bytes=7790870872 weight_to_token_ratio=0.394000 prefetch_share_of_ingress=0.154286 dedup_ratio=0.320715 prefetch_slots=14 unused_prefetch_slots=146 slot_utilization=0.087500 reduce_slots=13 remote_demand=7 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=12319 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.687636432e-05 prefetch_bytes_per_gemm_flop=1.340379902e-05 anomalies=none - [2026-08-26 15:12:39] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 362.9 | throughput per GPU (TFLOP/s/GPU): 340.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.761658E+00 | loss scale: 1.0 | grad norm: 3.644 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44552 payload_rows_remote=38884 dup_rows=20984 dup_rows_remote=18296 dedup_ratio=0.320190 router_tokens_per_expert=[574, 522, 541, 523, 520, 509, 483, 479, 535, 475, 507, 511, 519, 523, 547, 521, 518, 549, 509, 495, 524, 541, 544, 498, 496, 503, 535, 542, 521, 528, 487, 488, 495, 512, 466, 528, 530, 488, 522, 547, 479, 491, 514, 508, 496, 546, 490, 512, 502, 515, 493, 473, 520, 512, 509, 502, 553, 511, 491, 488, 504, 498, 487, 519] payload_rows_per_dst=[5628, 5610, 5668, 5582, 5600, 5482, 5474, 5508] dup_rows_per_dst=[2674, 2666, 2688, 2618, 2576, 2590, 2578, 2594] combine_rows_per_src=[5628, 5610, 5668, 5582, 5600, 5482, 5474, 5508] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44454 payload_rows_remote=38926 dup_rows=21082 dup_rows_remote=18608 dedup_ratio=0.321686 router_tokens_per_expert=[512, 527, 530, 524, 521, 497, 527, 514, 491, 536, 545, 514, 546, 485, 478, 539, 531, 496, 556, 480, 516, 514, 476, 502, 521, 496, 556, 514, 523, 501, 561, 521, 520, 461, 507, 484, 538, 519, 531, 483, 498, 512, 519, 506, 470, 526, 476, 494, 512, 496, 512, 487, 517, 531, 489, 502, 503, 512, 493, 500, 579, 530, 523, 488] payload_rows_per_dst=[5610, 5494, 5550, 5706, 5454, 5528, 5510, 5602] dup_rows_per_dst=[2694, 2774, 2592, 2680, 2632, 2474, 2582, 2654] combine_rows_per_src=[5610, 5494, 5550, 5706, 5454, 5528, 5510, 5602] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44412 payload_rows_remote=38798 dup_rows=21124 dup_rows_remote=18414 dedup_ratio=0.322327 router_tokens_per_expert=[509, 519, 513, 545, 556, 496, 520, 471, 550, 526, 506, 501, 546, 508, 522, 524, 517, 531, 526, 519, 553, 530, 511, 481, 556, 505, 479, 494, 536, 480, 519, 490, 477, 457, 507, 501, 521, 535, 498, 515, 477, 528, 508, 482, 491, 492, 523, 534, 554, 486, 521, 522, 504, 485, 526, 548, 494, 497, 456, 499, 521, 543, 503, 524] payload_rows_per_dst=[5488, 5154, 5462, 5504, 5706, 5938, 5546, 5614] dup_rows_per_dst=[2520, 2168, 2380, 2582, 2810, 3176, 2778, 2710] combine_rows_per_src=[5488, 5154, 5462, 5504, 5706, 5938, 5546, 5614] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44624 payload_rows_remote=39080 dup_rows=20912 dup_rows_remote=18266 dedup_ratio=0.319092 router_tokens_per_expert=[486, 515, 511, 519, 516, 503, 539, 501, 543, 519, 514, 529, 493, 548, 507, 502, 501, 526, 505, 533, 543, 536, 517, 539, 501, 554, 531, 502, 490, 533, 487, 497, 524, 506, 523, 532, 494, 474, 495, 508, 528, 518, 470, 500, 490, 448, 525, 519, 531, 493, 547, 494, 490, 520, 517, 509, 543, 525, 504, 529, 518, 482, 490, 482] payload_rows_per_dst=[5598, 5620, 5664, 5544, 5536, 5530, 5622, 5510] dup_rows_per_dst=[2582, 2690, 2736, 2646, 2576, 2466, 2580, 2636] combine_rows_per_src=[5598, 5620, 5664, 5544, 5536, 5530, 5622, 5510] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44450 payload_rows_remote=38874 dup_rows=21086 dup_rows_remote=18384 dedup_ratio=0.321747 router_tokens_per_expert=[475, 507, 477, 543, 486, 510, 532, 516, 531, 491, 488, 528, 476, 534, 566, 504, 485, 518, 509, 485, 546, 507, 527, 538, 509, 544, 515, 536, 466, 517, 540, 529, 503, 500, 555, 531, 505, 517, 527, 501, 507, 498, 519, 527, 504, 481, 520, 493, 479, 510, 503, 523, 526, 506, 493, 526, 524, 541, 493, 516, 510, 490, 510, 495] payload_rows_per_dst=[5534, 5546, 5550, 5672, 5576, 5522, 5528, 5522] dup_rows_per_dst=[2558, 2690, 2680, 2640, 2702, 2576, 2604, 2636] combine_rows_per_src=[5534, 5546, 5550, 5672, 5576, 5522, 5528, 5522] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44528 payload_rows_remote=38902 dup_rows=21008 dup_rows_remote=18350 dedup_ratio=0.320557 router_tokens_per_expert=[533, 534, 511, 526, 514, 513, 494, 488, 537, 528, 508, 499, 512, 544, 542, 550, 476, 524, 482, 540, 522, 499, 504, 524, 473, 524, 516, 498, 503, 534, 520, 522, 496, 535, 514, 539, 488, 495, 486, 487, 478, 543, 523, 517, 501, 494, 521, 507, 550, 523, 540, 521, 496, 478, 529, 505, 501, 504, 507, 479, 502, 500, 494, 521] payload_rows_per_dst=[5572, 5600, 5592, 5564, 5442, 5620, 5626, 5512] dup_rows_per_dst=[2654, 2702, 2550, 2616, 2638, 2686, 2658, 2504] combine_rows_per_src=[5572, 5600, 5592, 5564, 5442, 5620, 5626, 5512] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44566 payload_rows_remote=39006 dup_rows=20970 dup_rows_remote=18412 dedup_ratio=0.319977 router_tokens_per_expert=[505, 483, 480, 507, 502, 509, 533, 519, 459, 517, 525, 520, 481, 522, 514, 497, 523, 515, 510, 509, 512, 517, 516, 521, 555, 493, 520, 501, 490, 497, 478, 525, 533, 480, 511, 513, 546, 504, 572, 511, 525, 487, 544, 531, 526, 482, 492, 543, 486, 515, 523, 525, 513, 506, 544, 532, 544, 490, 501, 542, 483, 514, 501, 494] payload_rows_per_dst=[5592, 5514, 5568, 5560, 5608, 5586, 5680, 5458] dup_rows_per_dst=[2484, 2556, 2678, 2558, 2732, 2674, 2608, 2680] combine_rows_per_src=[5592, 5514, 5568, 5560, 5608, 5586, 5680, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44472 payload_rows_remote=38908 dup_rows=21064 dup_rows_remote=18480 dedup_ratio=0.321411 router_tokens_per_expert=[520, 508, 540, 496, 486, 516, 498, 486, 498, 521, 527, 503, 497, 462, 518, 507, 556, 494, 468, 487, 539, 504, 485, 502, 539, 521, 495, 514, 508, 519, 542, 483, 503, 505, 540, 529, 533, 554, 497, 536, 524, 488, 498, 513, 518, 509, 496, 490, 519, 501, 515, 511, 513, 511, 479, 525, 519, 525, 471, 589, 551, 485, 553, 529] payload_rows_per_dst=[5476, 5494, 5502, 5630, 5700, 5478, 5564, 5628] dup_rows_per_dst=[2624, 2572, 2568, 2612, 2694, 2594, 2584, 2816] combine_rows_per_src=[5476, 5494, 5502, 5630, 5700, 5478, 5564, 5628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279164928 dispatch_meta_bytes=229816 combine_payload_bytes=279164928 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279394744 ingress_bytes=389265408 link_bytes=389265408 token_bytes=558559672 weight_movement_bytes=110100480 total_nvlink_bytes=668660152 weight_to_token_ratio=0.197115 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.769965677e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44402 payload_rows_remote=38852 dup_rows=21134 dup_rows_remote=18396 dedup_ratio=0.322479 router_tokens_per_expert=[528, 544, 512, 508, 466, 505, 542, 518, 480, 490, 492, 489, 535, 529, 503, 529, 506, 499, 498, 494, 572, 516, 511, 520, 503, 532, 486, 516, 532, 520, 506, 480, 543, 494, 511, 508, 520, 495, 501, 502, 535, 519, 522, 493, 528, 524, 522, 501, 536, 548, 556, 495, 483, 504, 477, 529, 533, 501, 501, 484, 503, 539, 477, 523] payload_rows_per_dst=[5526, 5494, 5612, 5504, 5586, 5550, 5582, 5548] dup_rows_per_dst=[2720, 2600, 2620, 2646, 2562, 2738, 2674, 2574] combine_rows_per_src=[5526, 5494, 5612, 5504, 5586, 5550, 5582, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278218752 dispatch_meta_bytes=228032 combine_payload_bytes=278218752 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278446784 ingress_bytes=388319232 link_bytes=388319232 token_bytes=556665536 weight_movement_bytes=110100480 total_nvlink_bytes=666766016 weight_to_token_ratio=0.197786 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.753510120e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279107584 dispatch_meta_bytes=229992 combine_payload_bytes=279107584 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279337576 ingress_bytes=499308544 link_bytes=499308544 token_bytes=558445160 weight_movement_bytes=220200960 total_nvlink_bytes=778646120 weight_to_token_ratio=0.394311 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.683796802e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279595008 dispatch_meta_bytes=229672 combine_payload_bytes=279595008 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279824680 ingress_bytes=499795968 link_bytes=499795968 token_bytes=559419688 weight_movement_bytes=220200960 total_nvlink_bytes=779620648 weight_to_token_ratio=0.393624 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.692273907e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44434 payload_rows_remote=38860 dup_rows=21102 dup_rows_remote=18440 dedup_ratio=0.321991 router_tokens_per_expert=[471, 487, 517, 552, 519, 534, 536, 547, 536, 487, 542, 486, 508, 528, 529, 502, 499, 494, 494, 568, 477, 509, 515, 496, 521, 494, 525, 530, 498, 527, 535, 481, 517, 487, 501, 518, 462, 503, 497, 519, 511, 482, 500, 488, 517, 517, 530, 529, 579, 532, 529, 528, 518, 476, 484, 536, 515, 519, 496, 507, 543, 467, 503, 514] payload_rows_per_dst=[5584, 5574, 5514, 5578, 5486, 5606, 5586, 5506] dup_rows_per_dst=[2742, 2662, 2590, 2644, 2522, 2542, 2778, 2622] combine_rows_per_src=[5584, 5574, 5514, 5578, 5486, 5606, 5586, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278892544 dispatch_meta_bytes=229552 combine_payload_bytes=278892544 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279122096 ingress_bytes=499093504 link_bytes=499093504 token_bytes=558014640 weight_movement_bytes=220200960 total_nvlink_bytes=778215600 weight_to_token_ratio=0.394615 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.680056902e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44558 payload_rows_remote=38968 dup_rows=20978 dup_rows_remote=18282 dedup_ratio=0.320099 router_tokens_per_expert=[496, 485, 528, 531, 521, 509, 538, 534, 531, 510, 516, 541, 497, 515, 474, 509, 514, 537, 502, 514, 496, 474, 536, 535, 495, 572, 491, 524, 524, 482, 478, 504, 547, 484, 501, 524, 501, 451, 539, 531, 522, 554, 519, 501, 498, 538, 528, 532, 537, 514, 501, 513, 512, 467, 541, 544, 462, 487, 477, 471, 498, 518, 536, 507] payload_rows_per_dst=[5590, 5826, 5710, 5542, 5428, 5664, 5350, 5448] dup_rows_per_dst=[2696, 2836, 2850, 2598, 2384, 2718, 2432, 2464] combine_rows_per_src=[5590, 5826, 5710, 5542, 5428, 5664, 5350, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278491136 dispatch_meta_bytes=228992 combine_payload_bytes=278491136 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278720128 ingress_bytes=608792576 link_bytes=608792576 token_bytes=557211264 weight_movement_bytes=330301440 total_nvlink_bytes=887512704 weight_to_token_ratio=0.592776 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058790419e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278548480 dispatch_meta_bytes=229200 combine_payload_bytes=278548480 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278777680 ingress_bytes=608849920 link_bytes=608849920 token_bytes=557326160 weight_movement_bytes=330301440 total_nvlink_bytes=887627600 weight_to_token_ratio=0.592654 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058890149e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44472 payload_rows_remote=38868 dup_rows=21064 dup_rows_remote=18492 dedup_ratio=0.321411 router_tokens_per_expert=[507, 514, 509, 476, 555, 532, 530, 489, 528, 550, 517, 522, 553, 513, 515, 502, 493, 509, 548, 491, 501, 495, 525, 526, 512, 486, 509, 524, 514, 538, 534, 534, 498, 486, 520, 519, 510, 522, 513, 492, 496, 497, 529, 483, 507, 489, 483, 516, 501, 469, 530, 501, 535, 546, 536, 505, 515, 510, 509, 527, 483, 508, 482, 500] payload_rows_per_dst=[5528, 5626, 5604, 5606, 5478, 5490, 5598, 5542] dup_rows_per_dst=[2696, 2774, 2572, 2696, 2642, 2510, 2648, 2526] combine_rows_per_src=[5528, 5626, 5604, 5606, 5478, 5490, 5598, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279322624 dispatch_meta_bytes=229000 combine_payload_bytes=279322624 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279551624 ingress_bytes=499523584 link_bytes=499523584 token_bytes=558874248 weight_movement_bytes=220200960 total_nvlink_bytes=779075208 weight_to_token_ratio=0.394008 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.687536701e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44240 payload_rows_remote=38628 dup_rows=21296 dup_rows_remote=18516 dedup_ratio=0.324951 router_tokens_per_expert=[509, 477, 487, 526, 482, 484, 528, 507, 496, 514, 545, 479, 492, 528, 509, 540, 489, 524, 481, 556, 523, 552, 507, 483, 479, 494, 506, 473, 524, 529, 537, 523, 471, 521, 520, 486, 519, 535, 558, 564, 517, 505, 501, 503, 519, 507, 515, 550, 512, 501, 537, 492, 514, 522, 485, 505, 511, 546, 532, 535, 496, 523, 470, 513] payload_rows_per_dst=[5488, 5512, 5372, 5524, 5720, 5574, 5438, 5612] dup_rows_per_dst=[2512, 2554, 2450, 2606, 3020, 2660, 2714, 2780] combine_rows_per_src=[5488, 5512, 5372, 5524, 5720, 5574, 5438, 5612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44450 payload_rows_remote=38956 dup_rows=21086 dup_rows_remote=18476 dedup_ratio=0.321747 router_tokens_per_expert=[482, 511, 497, 485, 508, 540, 508, 493, 519, 521, 507, 537, 518, 504, 500, 543, 517, 510, 510, 531, 509, 505, 514, 558, 522, 480, 497, 526, 481, 477, 494, 545, 473, 475, 525, 508, 504, 508, 523, 536, 516, 512, 489, 522, 531, 540, 547, 518, 478, 532, 468, 512, 535, 499, 544, 516, 526, 503, 495, 499, 505, 533, 529, 518] payload_rows_per_dst=[5512, 5596, 5586, 5528, 5494, 5630, 5622, 5482] dup_rows_per_dst=[2536, 2702, 2722, 2516, 2610, 2720, 2546, 2734] combine_rows_per_src=[5512, 5596, 5586, 5528, 5494, 5630, 5622, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44702 payload_rows_remote=39174 dup_rows=20834 dup_rows_remote=18316 dedup_ratio=0.317902 router_tokens_per_expert=[510, 523, 532, 513, 528, 502, 523, 497, 515, 543, 537, 510, 534, 497, 502, 507, 513, 468, 554, 515, 507, 550, 512, 512, 529, 496, 532, 502, 483, 511, 495, 475, 547, 492, 484, 502, 488, 521, 485, 512, 537, 515, 512, 498, 533, 507, 523, 485, 520, 506, 508, 527, 534, 529, 486, 541, 485, 473, 513, 522, 522, 520, 509, 505] payload_rows_per_dst=[5610, 5582, 5532, 5528, 5504, 5676, 5674, 5596] dup_rows_per_dst=[2646, 2708, 2730, 2518, 2558, 2544, 2628, 2502] combine_rows_per_src=[5610, 5582, 5532, 5528, 5504, 5676, 5674, 5596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5587400216 prefetch_bytes=1101004800 reduce_bytes=440401920 weight_movement_bytes=1541406720 egress_bytes=2794847768 ingress_bytes=4333959168 link_bytes=4333959168 total_nvlink_bytes=7128806936 weight_to_token_ratio=0.275872 prefetch_share_of_ingress=0.254041 dedup_ratio=0.320889 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=4 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.537467807e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44634 payload_rows_remote=39092 dup_rows=20902 dup_rows_remote=18294 dedup_ratio=0.318939 router_tokens_per_expert=[511, 492, 514, 551, 502, 507, 471, 515, 505, 538, 499, 513, 481, 516, 508, 526, 519, 497, 503, 564, 531, 493, 522, 561, 513, 505, 511, 535, 494, 517, 504, 529, 495, 526, 565, 509, 500, 523, 498, 498, 528, 485, 494, 478, 491, 529, 531, 518, 519, 515, 512, 483, 488, 523, 515, 520, 505, 532, 509, 475, 498, 500, 547, 512] payload_rows_per_dst=[5544, 5562, 5722, 5604, 5562, 5482, 5542, 5616] dup_rows_per_dst=[2582, 2610, 2658, 2612, 2666, 2626, 2608, 2540] combine_rows_per_src=[5544, 5562, 5722, 5604, 5562, 5482, 5542, 5616] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278605824 dispatch_meta_bytes=229440 combine_payload_bytes=278605824 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278835264 ingress_bytes=608907264 link_bytes=608907264 token_bytes=557441088 weight_movement_bytes=330301440 total_nvlink_bytes=887742528 weight_to_token_ratio=0.592532 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058989880e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279236608 dispatch_meta_bytes=229728 combine_payload_bytes=279236608 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279466336 ingress_bytes=389337088 link_bytes=389337088 token_bytes=558702944 weight_movement_bytes=110100480 total_nvlink_bytes=668803424 weight_to_token_ratio=0.197064 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.771212310e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=276885504 dispatch_meta_bytes=228576 combine_payload_bytes=276885504 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277114080 ingress_bytes=497086464 link_bytes=497086464 token_bytes=553999584 weight_movement_bytes=220200960 total_nvlink_bytes=774200544 weight_to_token_ratio=0.397475 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.645151176e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44574 payload_rows_remote=38982 dup_rows=20962 dup_rows_remote=18424 dedup_ratio=0.319855 router_tokens_per_expert=[522, 502, 519, 482, 480, 484, 528, 515, 505, 480, 491, 497, 548, 515, 525, 511, 496, 536, 545, 509, 508, 517, 531, 507, 497, 491, 495, 509, 497, 490, 485, 538, 535, 464, 499, 502, 481, 485, 500, 549, 543, 510, 504, 551, 469, 519, 510, 459, 543, 541, 516, 488, 525, 498, 559, 517, 510, 550, 542, 509, 551, 532, 522, 530] payload_rows_per_dst=[5474, 5540, 5642, 5504, 5506, 5592, 5612, 5704] dup_rows_per_dst=[2590, 2604, 2656, 2500, 2524, 2538, 2762, 2788] combine_rows_per_src=[5474, 5540, 5642, 5504, 5506, 5592, 5612, 5704] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=280799232 dispatch_meta_bytes=229960 combine_payload_bytes=280799232 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281029192 ingress_bytes=390899712 link_bytes=390899712 token_bytes=561828424 weight_movement_bytes=110100480 total_nvlink_bytes=671928904 weight_to_token_ratio=0.195968 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.798388911e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=280211456 dispatch_meta_bytes=229544 combine_payload_bytes=280211456 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280441000 ingress_bytes=500412416 link_bytes=500412416 token_bytes=560652456 weight_movement_bytes=220200960 total_nvlink_bytes=780853416 weight_to_token_ratio=0.392758 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.702994951e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5582669336 prefetch_bytes=880803840 reduce_bytes=1321205760 weight_movement_bytes=2202009600 egress_bytes=2792482328 ingress_bytes=4992196608 link_bytes=4992196608 total_nvlink_bytes=7784678936 weight_to_token_ratio=0.394437 prefetch_share_of_ingress=0.176436 dedup_ratio=0.321530 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=12 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.682250977e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44524 payload_rows_remote=38776 dup_rows=21012 dup_rows_remote=18268 dedup_ratio=0.320618 router_tokens_per_expert=[471, 518, 493, 518, 530, 524, 527, 520, 505, 524, 523, 539, 490, 517, 477, 517, 521, 529, 542, 504, 524, 491, 516, 484, 480, 486, 525, 572, 534, 487, 506, 556, 531, 543, 525, 483, 523, 540, 489, 488, 528, 533, 476, 505, 531, 472, 491, 511, 521, 504, 488, 514, 514, 525, 428, 536, 533, 529, 493, 512, 493, 488, 539, 532] payload_rows_per_dst=[5748, 5474, 5580, 5660, 5632, 5558, 5408, 5464] dup_rows_per_dst=[2744, 2456, 2606, 2912, 2612, 2788, 2372, 2522] combine_rows_per_src=[5748, 5474, 5580, 5660, 5632, 5558, 5408, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5584073688 prefetch_bytes=990904320 reduce_bytes=990904320 weight_movement_bytes=1981808640 egress_bytes=2793184216 ingress_bytes=4772698112 link_bytes=4772698112 total_nvlink_bytes=7565882328 weight_to_token_ratio=0.354904 prefetch_share_of_ingress=0.207619 dedup_ratio=0.321182 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=9 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.300506991e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5569586872 prefetch_bytes=990904320 reduce_bytes=880803840 weight_movement_bytes=1871708160 egress_bytes=2785937080 ingress_bytes=4655357952 link_bytes=4655357952 total_nvlink_bytes=7441295032 weight_to_token_ratio=0.336059 prefetch_share_of_ingress=0.212852 dedup_ratio=0.321643 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=8 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043015 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.096433153e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5589119472 prefetch_bytes=1101004800 reduce_bytes=550502400 weight_movement_bytes=1651507200 egress_bytes=2795706864 ingress_bytes=4444919808 link_bytes=4444919808 total_nvlink_bytes=7240626672 weight_to_token_ratio=0.295486 prefetch_share_of_ingress=0.247700 dedup_ratio=0.320688 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=5 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.039706 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.730446610e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279422976 dispatch_meta_bytes=229624 combine_payload_bytes=279422976 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279652600 ingress_bytes=389523456 link_bytes=389523456 token_bytes=559075576 weight_movement_bytes=110100480 total_nvlink_bytes=669176056 weight_to_token_ratio=0.196933 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.774453556e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5588402296 prefetch_bytes=770703360 reduce_bytes=1321205760 weight_movement_bytes=2091909120 egress_bytes=2795348088 ingress_bytes=4884963328 link_bytes=4885193640 total_nvlink_bytes=7680311416 weight_to_token_ratio=0.374330 prefetch_share_of_ingress=0.157771 dedup_ratio=0.320514 prefetch_slots=14 unused_prefetch_slots=146 slot_utilization=0.087500 reduce_slots=12 remote_demand=7 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042647 gemm_inner_pad_upper_bound=12319 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.496155216e-05 prefetch_bytes_per_gemm_flop=1.340379902e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=277946368 dispatch_meta_bytes=228176 combine_payload_bytes=277946368 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278174544 ingress_bytes=388046848 link_bytes=388046848 token_bytes=556120912 weight_movement_bytes=110100480 total_nvlink_bytes=666221392 weight_to_token_ratio=0.197979 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.748772914e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5584818536 prefetch_bytes=660602880 reduce_bytes=990904320 weight_movement_bytes=1651507200 egress_bytes=2793556328 ingress_bytes=4442769408 link_bytes=4442769408 total_nvlink_bytes=7236325736 weight_to_token_ratio=0.295714 prefetch_share_of_ingress=0.148692 dedup_ratio=0.321133 prefetch_slots=12 unused_prefetch_slots=148 slot_utilization=0.075000 reduce_slots=9 remote_demand=6 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042647 gemm_inner_pad_upper_bound=12192 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.726706711e-05 prefetch_bytes_per_gemm_flop=1.148897059e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5570161472 prefetch_bytes=880803840 reduce_bytes=880803840 weight_movement_bytes=1761607680 egress_bytes=2786224960 ingress_bytes=4545544192 link_bytes=4545544192 total_nvlink_bytes=7331769152 weight_to_token_ratio=0.316258 prefetch_share_of_ingress=0.193773 dedup_ratio=0.321582 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=8 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.905448963e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none + [2026-08-26 16:00:27] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 368.7 | throughput per GPU (TFLOP/s/GPU): 334.9 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.756481E+00 | loss scale: 1.0 | grad norm: 3.499 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44674 payload_rows_remote=39094 dup_rows=20862 dup_rows_remote=18284 dedup_ratio=0.318329 router_tokens_per_expert=[516, 501, 517, 475, 508, 499, 562, 499, 500, 508, 567, 507, 515, 479, 500, 503, 485, 526, 536, 522, 508, 507, 512, 522, 537, 534, 516, 505, 534, 512, 502, 525, 522, 542, 533, 495, 542, 455, 498, 512, 515, 488, 495, 507, 537, 485, 490, 520, 493, 523, 520, 478, 534, 513, 515, 509, 529, 504, 513, 500, 498, 544, 497, 523] payload_rows_per_dst=[5580, 5580, 5566, 5638, 5592, 5590, 5494, 5634] dup_rows_per_dst=[2574, 2578, 2670, 2692, 2606, 2484, 2676, 2582] combine_rows_per_src=[5580, 5580, 5566, 5638, 5592, 5590, 5494, 5634] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44418 payload_rows_remote=38864 dup_rows=21118 dup_rows_remote=18520 dedup_ratio=0.322235 router_tokens_per_expert=[507, 515, 511, 543, 554, 494, 517, 468, 547, 526, 504, 500, 543, 508, 521, 525, 511, 529, 526, 517, 553, 530, 504, 479, 553, 506, 478, 496, 534, 479, 523, 492, 478, 458, 508, 504, 521, 535, 497, 516, 475, 530, 509, 483, 491, 493, 522, 535, 554, 488, 522, 524, 505, 486, 530, 550, 495, 500, 460, 502, 526, 544, 507, 527] payload_rows_per_dst=[5574, 5150, 5648, 5346, 5674, 5938, 5534, 5554] dup_rows_per_dst=[2644, 2156, 2650, 2386, 2750, 3180, 2754, 2598] combine_rows_per_src=[5574, 5150, 5648, 5346, 5674, 5938, 5534, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44454 payload_rows_remote=38922 dup_rows=21082 dup_rows_remote=18596 dedup_ratio=0.321686 router_tokens_per_expert=[507, 525, 529, 521, 518, 495, 526, 513, 485, 536, 542, 509, 545, 484, 478, 536, 530, 495, 553, 478, 515, 511, 477, 500, 526, 491, 555, 512, 523, 502, 560, 521, 520, 463, 505, 480, 539, 519, 534, 486, 498, 514, 522, 508, 471, 524, 476, 496, 514, 495, 513, 489, 516, 538, 490, 504, 507, 515, 493, 501, 585, 537, 528, 490] payload_rows_per_dst=[5596, 5482, 5540, 5710, 5452, 5532, 5522, 5620] dup_rows_per_dst=[2672, 2748, 2578, 2670, 2640, 2486, 2596, 2692] combine_rows_per_src=[5596, 5482, 5540, 5710, 5452, 5532, 5522, 5620] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44444 payload_rows_remote=38860 dup_rows=21092 dup_rows_remote=18372 dedup_ratio=0.321838 router_tokens_per_expert=[474, 507, 475, 541, 483, 508, 529, 515, 527, 489, 487, 522, 475, 532, 567, 500, 483, 517, 507, 482, 543, 506, 527, 538, 508, 542, 513, 533, 466, 517, 538, 528, 502, 503, 555, 532, 506, 524, 527, 503, 506, 500, 521, 528, 504, 482, 520, 496, 480, 513, 502, 524, 528, 508, 492, 528, 526, 545, 495, 519, 516, 494, 512, 498] payload_rows_per_dst=[5522, 5528, 5540, 5658, 5584, 5534, 5534, 5544] dup_rows_per_dst=[2542, 2670, 2666, 2632, 2720, 2580, 2616, 2666] combine_rows_per_src=[5522, 5528, 5540, 5658, 5584, 5534, 5534, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44560 payload_rows_remote=38894 dup_rows=20976 dup_rows_remote=18304 dedup_ratio=0.320068 router_tokens_per_expert=[568, 518, 538, 521, 518, 503, 483, 476, 532, 474, 501, 510, 518, 524, 545, 521, 518, 545, 510, 493, 521, 541, 544, 497, 497, 505, 531, 541, 521, 526, 487, 490, 496, 513, 467, 528, 530, 490, 523, 548, 480, 491, 514, 508, 499, 549, 493, 513, 503, 517, 493, 475, 521, 512, 511, 509, 556, 514, 494, 489, 506, 500, 488, 521] payload_rows_per_dst=[5606, 5592, 5666, 5586, 5606, 5494, 5492, 5518] dup_rows_per_dst=[2644, 2658, 2672, 2610, 2584, 2600, 2590, 2618] combine_rows_per_src=[5606, 5592, 5666, 5586, 5606, 5494, 5492, 5518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44552 payload_rows_remote=38914 dup_rows=20984 dup_rows_remote=18306 dedup_ratio=0.320190 router_tokens_per_expert=[532, 533, 511, 525, 515, 509, 493, 485, 535, 524, 506, 496, 509, 542, 540, 550, 476, 523, 480, 540, 523, 497, 497, 522, 472, 522, 515, 498, 504, 532, 519, 521, 495, 535, 514, 539, 488, 494, 487, 489, 480, 544, 525, 516, 502, 495, 525, 509, 552, 527, 540, 524, 498, 478, 534, 505, 505, 508, 510, 479, 506, 502, 496, 521] payload_rows_per_dst=[5566, 5626, 5582, 5564, 5442, 5600, 5638, 5534] dup_rows_per_dst=[2640, 2736, 2534, 2602, 2640, 2634, 2678, 2520] combine_rows_per_src=[5566, 5626, 5582, 5564, 5442, 5600, 5638, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44606 payload_rows_remote=39080 dup_rows=20930 dup_rows_remote=18280 dedup_ratio=0.319366 router_tokens_per_expert=[485, 513, 509, 516, 513, 501, 536, 500, 541, 517, 514, 530, 492, 547, 505, 498, 498, 525, 504, 529, 542, 532, 518, 538, 499, 552, 529, 504, 492, 532, 486, 494, 528, 505, 519, 532, 498, 475, 496, 509, 525, 518, 474, 500, 490, 449, 525, 519, 536, 495, 546, 493, 492, 519, 523, 512, 546, 527, 510, 529, 519, 484, 496, 488] payload_rows_per_dst=[5586, 5612, 5646, 5526, 5546, 5534, 5634, 5522] dup_rows_per_dst=[2560, 2676, 2726, 2650, 2578, 2466, 2598, 2676] combine_rows_per_src=[5586, 5612, 5646, 5526, 5546, 5534, 5634, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44492 payload_rows_remote=38964 dup_rows=21044 dup_rows_remote=18340 dedup_ratio=0.321106 router_tokens_per_expert=[552, 484, 512, 512, 531, 489, 503, 524, 494, 542, 500, 534, 527, 516, 522, 523, 548, 528, 488, 504, 533, 503, 504, 482, 480, 514, 483, 529, 527, 490, 524, 541, 542, 501, 482, 511, 539, 512, 464, 499, 529, 474, 537, 460, 475, 485, 490, 492, 525, 509, 542, 517, 511, 525, 481, 539, 522, 522, 537, 528, 546, 513, 496, 520] payload_rows_per_dst=[5528, 5722, 5566, 5406, 5500, 5496, 5594, 5680] dup_rows_per_dst=[2704, 2916, 2604, 2440, 2600, 2388, 2704, 2688] combine_rows_per_src=[5528, 5722, 5566, 5406, 5500, 5496, 5594, 5680] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278720512 dispatch_meta_bytes=228720 combine_payload_bytes=278720512 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278949232 ingress_bytes=609021952 link_bytes=609021952 token_bytes=557669744 weight_movement_bytes=330301440 total_nvlink_bytes=887971184 weight_to_token_ratio=0.592289 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059189341e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279021568 dispatch_meta_bytes=230136 combine_payload_bytes=279021568 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279251704 ingress_bytes=389122048 link_bytes=389122048 token_bytes=558273272 weight_movement_bytes=110100480 total_nvlink_bytes=668373752 weight_to_token_ratio=0.197216 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4096 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.767472411e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278104064 dispatch_meta_bytes=228848 combine_payload_bytes=278104064 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278332912 ingress_bytes=388204544 link_bytes=388204544 token_bytes=556436976 weight_movement_bytes=110100480 total_nvlink_bytes=666537456 weight_to_token_ratio=0.197867 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.751515507e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280125440 dispatch_meta_bytes=229384 combine_payload_bytes=280125440 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=280354824 ingress_bytes=610426880 link_bytes=610426880 token_bytes=560480264 weight_movement_bytes=330301440 total_nvlink_bytes=890781704 weight_to_token_ratio=0.589319 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061632742e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278648832 dispatch_meta_bytes=229032 combine_payload_bytes=278648832 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278877864 ingress_bytes=388749312 link_bytes=388749312 token_bytes=557526696 weight_movement_bytes=110100480 total_nvlink_bytes=667627176 weight_to_token_ratio=0.197480 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.760989918e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44650 payload_rows_remote=39062 dup_rows=20886 dup_rows_remote=18290 dedup_ratio=0.318695 router_tokens_per_expert=[517, 505, 518, 478, 512, 500, 566, 500, 503, 507, 572, 510, 514, 480, 501, 505, 484, 527, 536, 523, 510, 508, 515, 524, 539, 534, 515, 507, 535, 511, 501, 526, 521, 543, 530, 495, 541, 455, 498, 511, 515, 487, 497, 506, 535, 484, 489, 519, 492, 524, 519, 474, 532, 512, 513, 510, 527, 501, 510, 497, 495, 541, 494, 518] payload_rows_per_dst=[5592, 5588, 5566, 5634, 5590, 5588, 5484, 5608] dup_rows_per_dst=[2600, 2596, 2688, 2702, 2598, 2476, 2668, 2558] combine_rows_per_src=[5592, 5588, 5566, 5634, 5590, 5588, 5484, 5608] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278849536 dispatch_meta_bytes=229008 combine_payload_bytes=278849536 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279078544 ingress_bytes=388950016 link_bytes=388950016 token_bytes=557928080 weight_movement_bytes=110100480 total_nvlink_bytes=668028560 weight_to_token_ratio=0.197338 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.764480491e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279293952 dispatch_meta_bytes=229216 combine_payload_bytes=279293952 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279523168 ingress_bytes=499494912 link_bytes=499494912 token_bytes=558817120 weight_movement_bytes=220200960 total_nvlink_bytes=779018080 weight_to_token_ratio=0.394048 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.687038048e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279996416 dispatch_meta_bytes=229408 combine_payload_bytes=279996416 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280225824 ingress_bytes=500197376 link_bytes=500197376 token_bytes=560222240 weight_movement_bytes=220200960 total_nvlink_bytes=780423200 weight_to_token_ratio=0.393060 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.699255052e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44562 payload_rows_remote=39066 dup_rows=20974 dup_rows_remote=18328 dedup_ratio=0.320038 router_tokens_per_expert=[570, 505, 483, 523, 506, 519, 499, 529, 518, 518, 525, 535, 496, 566, 550, 499, 483, 484, 501, 542, 515, 515, 520, 511, 516, 496, 496, 503, 530, 498, 526, 490, 491, 511, 483, 520, 501, 525, 533, 515, 479, 515, 508, 492, 556, 530, 530, 536, 506, 517, 520, 505, 521, 502, 529, 508, 505, 503, 459, 524, 464, 510, 504, 499] payload_rows_per_dst=[5636, 5640, 5496, 5570, 5586, 5570, 5596, 5468] dup_rows_per_dst=[2632, 2774, 2646, 2540, 2572, 2722, 2620, 2468] combine_rows_per_src=[5636, 5640, 5496, 5570, 5586, 5570, 5596, 5468] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44422 payload_rows_remote=38838 dup_rows=21114 dup_rows_remote=18400 dedup_ratio=0.322174 router_tokens_per_expert=[530, 516, 531, 509, 504, 486, 536, 498, 508, 501, 504, 508, 493, 538, 513, 500, 541, 551, 509, 502, 492, 517, 538, 504, 507, 528, 499, 533, 511, 468, 505, 503, 525, 517, 527, 507, 491, 523, 512, 520, 512, 494, 494, 496, 526, 470, 509, 479, 498, 555, 540, 542, 507, 489, 465, 540, 498, 499, 490, 529, 551, 540, 499, 541] payload_rows_per_dst=[5350, 5434, 5548, 5462, 5764, 5590, 5690, 5584] dup_rows_per_dst=[2372, 2422, 2456, 2646, 2848, 2800, 2856, 2714] combine_rows_per_src=[5350, 5434, 5548, 5462, 5764, 5590, 5690, 5584] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44434 payload_rows_remote=38846 dup_rows=21102 dup_rows_remote=18392 dedup_ratio=0.321991 router_tokens_per_expert=[504, 534, 498, 542, 499, 516, 554, 523, 482, 487, 506, 534, 507, 492, 499, 486, 515, 542, 499, 519, 545, 480, 541, 516, 506, 520, 550, 527, 489, 494, 500, 500, 517, 497, 502, 504, 497, 526, 493, 530, 550, 542, 498, 534, 511, 537, 497, 480, 504, 495, 460, 451, 522, 508, 496, 494, 523, 510, 496, 541, 549, 525, 529, 544] payload_rows_per_dst=[5614, 5494, 5636, 5530, 5548, 5588, 5376, 5648] dup_rows_per_dst=[2726, 2492, 2678, 2642, 2584, 2710, 2484, 2786] combine_rows_per_src=[5614, 5494, 5636, 5530, 5548, 5588, 5376, 5648] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44608 payload_rows_remote=39020 dup_rows=20928 dup_rows_remote=18326 dedup_ratio=0.319336 router_tokens_per_expert=[542, 513, 534, 500, 504, 499, 541, 512, 506, 490, 509, 548, 511, 506, 515, 511, 512, 504, 539, 550, 509, 544, 534, 498, 488, 518, 497, 505, 522, 525, 528, 515, 519, 529, 538, 482, 468, 546, 495, 518, 488, 480, 490, 501, 498, 515, 537, 508, 524, 513, 516, 522, 465, 533, 481, 494, 566, 485, 524, 518, 479, 489, 491, 527] payload_rows_per_dst=[5606, 5532, 5666, 5596, 5588, 5606, 5510, 5504] dup_rows_per_dst=[2684, 2660, 2714, 2600, 2602, 2428, 2586, 2654] combine_rows_per_src=[5606, 5532, 5666, 5596, 5588, 5606, 5510, 5504] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44288 payload_rows_remote=38840 dup_rows=21248 dup_rows_remote=18704 dedup_ratio=0.324219 router_tokens_per_expert=[518, 537, 543, 502, 505, 507, 499, 539, 535, 504, 515, 512, 478, 520, 526, 519, 500, 529, 497, 497, 556, 492, 496, 525, 500, 485, 507, 485, 480, 517, 543, 479, 524, 519, 488, 516, 471, 542, 512, 526, 478, 497, 534, 488, 489, 533, 489, 473, 518, 510, 528, 532, 533, 502, 523, 493, 513, 562, 517, 534, 487, 539, 504, 547] payload_rows_per_dst=[5584, 5500, 5538, 5448, 5492, 5500, 5612, 5614] dup_rows_per_dst=[2716, 2718, 2646, 2544, 2704, 2462, 2666, 2792] combine_rows_per_src=[5584, 5500, 5538, 5448, 5492, 5500, 5612, 5614] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44378 payload_rows_remote=38752 dup_rows=21158 dup_rows_remote=18470 dedup_ratio=0.322845 router_tokens_per_expert=[534, 468, 507, 526, 517, 492, 564, 507, 527, 553, 537, 538, 519, 539, 457, 539, 527, 475, 486, 526, 471, 533, 487, 516, 499, 509, 484, 502, 475, 548, 541, 504, 547, 469, 495, 514, 507, 519, 479, 496, 539, 543, 475, 482, 512, 527, 497, 554, 494, 519, 519, 516, 517, 522, 513, 557, 511, 512, 507, 520, 511, 507, 464, 517] payload_rows_per_dst=[5620, 5696, 5458, 5496, 5434, 5522, 5626, 5526] dup_rows_per_dst=[2610, 2722, 2584, 2628, 2618, 2736, 2688, 2572] combine_rows_per_src=[5620, 5696, 5458, 5496, 5434, 5522, 5626, 5526] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280225792 dispatch_meta_bytes=229512 combine_payload_bytes=280225792 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280455304 ingress_bytes=500426752 link_bytes=500426752 token_bytes=560681096 weight_movement_bytes=220200960 total_nvlink_bytes=780882056 weight_to_token_ratio=0.392738 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.703244278e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44546 payload_rows_remote=39020 dup_rows=20990 dup_rows_remote=18286 dedup_ratio=0.320282 router_tokens_per_expert=[550, 482, 508, 508, 528, 484, 500, 523, 490, 540, 501, 531, 526, 511, 521, 520, 546, 527, 485, 505, 531, 503, 504, 479, 479, 513, 484, 528, 526, 485, 525, 543, 543, 503, 483, 512, 540, 513, 465, 502, 528, 475, 536, 460, 473, 485, 493, 492, 526, 510, 544, 520, 512, 531, 482, 543, 526, 526, 541, 533, 548, 516, 497, 524] payload_rows_per_dst=[5526, 5776, 5438, 5596, 5518, 5494, 5570, 5628] dup_rows_per_dst=[2704, 2968, 2414, 2670, 2604, 2390, 2610, 2630] combine_rows_per_src=[5526, 5776, 5438, 5596, 5518, 5494, 5570, 5628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278992896 dispatch_meta_bytes=230072 combine_payload_bytes=278992896 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279222968 ingress_bytes=389093376 link_bytes=389093376 token_bytes=558215864 weight_movement_bytes=110100480 total_nvlink_bytes=668316344 weight_to_token_ratio=0.197236 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4096 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.766973757e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278548480 dispatch_meta_bytes=228928 combine_payload_bytes=278548480 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278777408 ingress_bytes=388648960 link_bytes=388648960 token_bytes=557325888 weight_movement_bytes=110100480 total_nvlink_bytes=667426368 weight_to_token_ratio=0.197551 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.759244632e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278577152 dispatch_meta_bytes=229536 combine_payload_bytes=278577152 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278806688 ingress_bytes=608878592 link_bytes=608878592 token_bytes=557383840 weight_movement_bytes=330301440 total_nvlink_bytes=887685280 weight_to_token_ratio=0.592592 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058940015e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278792192 dispatch_meta_bytes=228792 combine_payload_bytes=278792192 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279020984 ingress_bytes=498993152 link_bytes=498993152 token_bytes=557813176 weight_movement_bytes=220200960 total_nvlink_bytes=778014136 weight_to_token_ratio=0.394758 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.678311616e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278935552 dispatch_meta_bytes=228880 combine_payload_bytes=278935552 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279164432 ingress_bytes=499136512 link_bytes=499136512 token_bytes=558099984 weight_movement_bytes=220200960 total_nvlink_bytes=778300944 weight_to_token_ratio=0.394555 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.680804882e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280125440 dispatch_meta_bytes=229440 combine_payload_bytes=280125440 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280354880 ingress_bytes=500326400 link_bytes=500326400 token_bytes=560480320 weight_movement_bytes=220200960 total_nvlink_bytes=780681280 weight_to_token_ratio=0.392879 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.701498992e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279695360 dispatch_meta_bytes=229224 combine_payload_bytes=279695360 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279924584 ingress_bytes=389795840 link_bytes=389795840 token_bytes=559619944 weight_movement_bytes=110100480 total_nvlink_bytes=669720424 weight_to_token_ratio=0.196742 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.779190762e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44586 payload_rows_remote=39008 dup_rows=20950 dup_rows_remote=18364 dedup_ratio=0.319672 router_tokens_per_expert=[489, 503, 500, 519, 535, 525, 508, 530, 523, 497, 510, 513, 507, 497, 522, 513, 509, 529, 508, 564, 535, 510, 504, 498, 497, 507, 519, 525, 497, 532, 508, 511, 468, 504, 539, 501, 499, 470, 497, 519, 492, 500, 536, 508, 481, 497, 527, 500, 516, 493, 488, 506, 508, 528, 494, 522, 533, 493, 551, 530, 515, 541, 538, 530] payload_rows_per_dst=[5578, 5578, 5602, 5608, 5452, 5534, 5568, 5666] dup_rows_per_dst=[2640, 2586, 2712, 2584, 2542, 2548, 2542, 2796] combine_rows_per_src=[5578, 5578, 5602, 5608, 5452, 5534, 5568, 5666] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44422 payload_rows_remote=38828 dup_rows=21114 dup_rows_remote=18400 dedup_ratio=0.322174 router_tokens_per_expert=[503, 529, 496, 539, 496, 516, 552, 522, 479, 484, 504, 530, 506, 491, 499, 484, 513, 542, 497, 516, 543, 480, 541, 513, 501, 521, 547, 528, 489, 495, 498, 502, 517, 496, 506, 504, 497, 530, 491, 531, 551, 542, 499, 533, 512, 537, 499, 481, 506, 495, 459, 454, 523, 508, 498, 498, 529, 511, 499, 545, 555, 529, 532, 545] payload_rows_per_dst=[5596, 5482, 5622, 5522, 5550, 5594, 5388, 5668] dup_rows_per_dst=[2710, 2472, 2668, 2640, 2594, 2714, 2494, 2822] combine_rows_per_src=[5596, 5482, 5622, 5522, 5550, 5594, 5388, 5668] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44618 payload_rows_remote=39028 dup_rows=20918 dup_rows_remote=18306 dedup_ratio=0.319183 router_tokens_per_expert=[539, 511, 532, 498, 500, 498, 536, 510, 503, 488, 504, 543, 510, 506, 514, 511, 510, 504, 538, 546, 509, 543, 533, 495, 489, 516, 499, 504, 520, 525, 527, 517, 520, 528, 539, 483, 470, 546, 494, 521, 489, 482, 490, 506, 500, 516, 541, 509, 524, 513, 518, 524, 472, 535, 482, 495, 568, 487, 526, 522, 480, 490, 491, 529] payload_rows_per_dst=[5592, 5518, 5664, 5596, 5590, 5618, 5520, 5520] dup_rows_per_dst=[2656, 2640, 2692, 2598, 2612, 2448, 2606, 2666] combine_rows_per_src=[5592, 5518, 5664, 5596, 5590, 5618, 5520, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44568 payload_rows_remote=39082 dup_rows=20968 dup_rows_remote=18340 dedup_ratio=0.319946 router_tokens_per_expert=[567, 501, 476, 523, 503, 517, 497, 526, 516, 515, 524, 532, 494, 563, 548, 498, 482, 480, 503, 540, 512, 514, 518, 508, 515, 499, 493, 505, 531, 498, 522, 490, 492, 513, 481, 520, 503, 526, 533, 518, 476, 517, 508, 493, 558, 531, 533, 539, 513, 518, 522, 506, 526, 503, 531, 509, 508, 506, 461, 526, 466, 514, 506, 502] payload_rows_per_dst=[5616, 5626, 5486, 5570, 5598, 5582, 5608, 5482] dup_rows_per_dst=[2604, 2754, 2628, 2536, 2574, 2728, 2648, 2496] combine_rows_per_src=[5616, 5626, 5486, 5570, 5598, 5582, 5608, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44358 payload_rows_remote=38724 dup_rows=21178 dup_rows_remote=18466 dedup_ratio=0.323151 router_tokens_per_expert=[531, 466, 506, 522, 515, 486, 561, 505, 525, 548, 534, 537, 516, 539, 456, 538, 525, 473, 484, 524, 470, 534, 486, 517, 500, 511, 479, 500, 476, 547, 541, 504, 549, 470, 495, 515, 504, 518, 482, 497, 542, 544, 474, 480, 514, 528, 502, 555, 495, 524, 522, 517, 520, 523, 514, 558, 513, 515, 506, 523, 514, 511, 466, 522] payload_rows_per_dst=[5596, 5678, 5446, 5496, 5434, 5530, 5634, 5544] dup_rows_per_dst=[2588, 2708, 2580, 2620, 2626, 2748, 2712, 2596] combine_rows_per_src=[5596, 5678, 5446, 5496, 5434, 5530, 5634, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44450 payload_rows_remote=38852 dup_rows=21086 dup_rows_remote=18350 dedup_ratio=0.321747 router_tokens_per_expert=[526, 514, 528, 506, 503, 485, 532, 494, 507, 495, 501, 507, 490, 539, 512, 499, 539, 546, 512, 501, 491, 516, 536, 502, 506, 525, 498, 535, 512, 470, 505, 502, 526, 521, 527, 505, 491, 522, 512, 520, 514, 495, 498, 498, 526, 471, 508, 478, 500, 560, 541, 542, 509, 493, 470, 543, 504, 500, 491, 530, 551, 542, 502, 544] payload_rows_per_dst=[5406, 5534, 5652, 5456, 5638, 5564, 5602, 5598] dup_rows_per_dst=[2472, 2566, 2634, 2650, 2610, 2710, 2708, 2736] combine_rows_per_src=[5406, 5534, 5652, 5456, 5638, 5564, 5602, 5598] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279609344 dispatch_meta_bytes=229488 combine_payload_bytes=279609344 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279838832 ingress_bytes=389709824 link_bytes=389709824 token_bytes=559448176 weight_movement_bytes=110100480 total_nvlink_bytes=669548656 weight_to_token_ratio=0.196802 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.777694802e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44280 payload_rows_remote=38844 dup_rows=21256 dup_rows_remote=18720 dedup_ratio=0.324341 router_tokens_per_expert=[513, 535, 540, 498, 503, 504, 499, 538, 535, 503, 512, 509, 476, 520, 520, 517, 500, 528, 494, 499, 555, 491, 497, 522, 496, 486, 507, 482, 480, 514, 542, 479, 526, 518, 488, 517, 472, 539, 513, 529, 479, 499, 535, 490, 486, 530, 494, 475, 518, 513, 530, 533, 535, 512, 525, 495, 517, 565, 518, 537, 491, 539, 508, 548] payload_rows_per_dst=[5574, 5488, 5530, 5436, 5496, 5506, 5622, 5628] dup_rows_per_dst=[2686, 2696, 2642, 2536, 2708, 2470, 2700, 2818] combine_rows_per_src=[5574, 5488, 5530, 5436, 5496, 5506, 5622, 5628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279752704 dispatch_meta_bytes=229336 combine_payload_bytes=279752704 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279982040 ingress_bytes=389853184 link_bytes=389853184 token_bytes=559734744 weight_movement_bytes=110100480 total_nvlink_bytes=669835224 weight_to_token_ratio=0.196701 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.780188068e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278319104 dispatch_meta_bytes=228912 combine_payload_bytes=278319104 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278548016 ingress_bytes=388419584 link_bytes=388419584 token_bytes=556867120 weight_movement_bytes=110100480 total_nvlink_bytes=666967600 weight_to_token_ratio=0.197714 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.755255406e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=280139776 dispatch_meta_bytes=229688 combine_payload_bytes=280139776 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280369464 ingress_bytes=500340736 link_bytes=500340736 token_bytes=560509240 weight_movement_bytes=220200960 total_nvlink_bytes=780710200 weight_to_token_ratio=0.392859 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.701748318e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277573632 dispatch_meta_bytes=228760 combine_payload_bytes=277573632 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=277802392 ingress_bytes=497774592 link_bytes=497774592 token_bytes=555376024 weight_movement_bytes=220200960 total_nvlink_bytes=775576984 weight_to_token_ratio=0.396490 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.657118853e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278491136 dispatch_meta_bytes=228808 combine_payload_bytes=278491136 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=278719944 ingress_bytes=718893056 link_bytes=718893056 token_bytes=557211080 weight_movement_bytes=440401920 total_nvlink_bytes=997613000 weight_to_token_ratio=0.790368 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.250273262e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=280025088 dispatch_meta_bytes=229576 combine_payload_bytes=280025088 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280254664 ingress_bytes=500226048 link_bytes=500226048 token_bytes=560279752 weight_movement_bytes=220200960 total_nvlink_bytes=780480712 weight_to_token_ratio=0.393020 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.699753705e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278390784 dispatch_meta_bytes=228952 combine_payload_bytes=278390784 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278619736 ingress_bytes=608692224 link_bytes=608692224 token_bytes=557010520 weight_movement_bytes=330301440 total_nvlink_bytes=887311960 weight_to_token_ratio=0.592990 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058615890e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44618 payload_rows_remote=39090 dup_rows=20918 dup_rows_remote=18326 dedup_ratio=0.319183 router_tokens_per_expert=[514, 498, 524, 505, 471, 533, 517, 498, 487, 496, 508, 516, 495, 533, 504, 540, 531, 480, 530, 518, 511, 485, 483, 526, 522, 508, 532, 497, 499, 519, 528, 528, 519, 494, 509, 511, 513, 487, 542, 524, 536, 472, 511, 518, 515, 512, 520, 532, 542, 474, 526, 515, 517, 509, 480, 515, 500, 522, 539, 494, 529, 525, 506, 524] payload_rows_per_dst=[5528, 5586, 5552, 5728, 5584, 5600, 5610, 5430] dup_rows_per_dst=[2592, 2672, 2576, 2922, 2614, 2632, 2546, 2364] combine_rows_per_src=[5528, 5586, 5552, 5728, 5584, 5600, 5610, 5430] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279695360 dispatch_meta_bytes=229384 combine_payload_bytes=279695360 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279924744 ingress_bytes=389795840 link_bytes=389795840 token_bytes=559620104 weight_movement_bytes=110100480 total_nvlink_bytes=669720584 weight_to_token_ratio=0.196741 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.779190762e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278448128 dispatch_meta_bytes=228952 combine_payload_bytes=278448128 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278677080 ingress_bytes=388548608 link_bytes=388548608 token_bytes=557125208 weight_movement_bytes=110100480 total_nvlink_bytes=667225688 weight_to_token_ratio=0.197623 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.757499346e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278405120 dispatch_meta_bytes=230176 combine_payload_bytes=278405120 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278635296 ingress_bytes=388505600 link_bytes=388505600 token_bytes=557040416 weight_movement_bytes=110100480 total_nvlink_bytes=667140896 weight_to_token_ratio=0.197653 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.756751366e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277774336 dispatch_meta_bytes=228888 combine_payload_bytes=277774336 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278003224 ingress_bytes=387874816 link_bytes=387874816 token_bytes=555777560 weight_movement_bytes=110100480 total_nvlink_bytes=665878040 weight_to_token_ratio=0.198102 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.745780995e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44582 payload_rows_remote=38992 dup_rows=20954 dup_rows_remote=18350 dedup_ratio=0.319733 router_tokens_per_expert=[495, 505, 503, 523, 537, 528, 510, 532, 524, 499, 511, 520, 507, 500, 522, 514, 513, 529, 509, 565, 540, 510, 505, 500, 496, 507, 519, 525, 497, 534, 506, 508, 467, 506, 538, 502, 498, 472, 496, 520, 493, 497, 538, 503, 482, 497, 525, 500, 513, 488, 483, 506, 506, 525, 492, 522, 529, 493, 547, 527, 513, 536, 536, 525] payload_rows_per_dst=[5594, 5590, 5618, 5602, 5454, 5526, 5552, 5646] dup_rows_per_dst=[2672, 2604, 2724, 2582, 2544, 2544, 2518, 2766] combine_rows_per_src=[5594, 5590, 5618, 5602, 5454, 5526, 5552, 5646] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=280197120 dispatch_meta_bytes=229664 combine_payload_bytes=280197120 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=280426784 ingress_bytes=610498560 link_bytes=610498560 token_bytes=560623904 weight_movement_bytes=330301440 total_nvlink_bytes=890925344 weight_to_token_ratio=0.589168 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061757406e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279494656 dispatch_meta_bytes=229368 combine_payload_bytes=279494656 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279724024 ingress_bytes=499695616 link_bytes=499695616 token_bytes=559218680 weight_movement_bytes=220200960 total_nvlink_bytes=779419640 weight_to_token_ratio=0.393765 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.690528620e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44336 payload_rows_remote=38730 dup_rows=21200 dup_rows_remote=18566 dedup_ratio=0.323486 router_tokens_per_expert=[512, 535, 487, 505, 515, 476, 519, 543, 509, 527, 544, 490, 505, 502, 493, 501, 536, 528, 493, 494, 509, 542, 516, 502, 498, 502, 503, 533, 494, 519, 489, 537, 521, 499, 522, 541, 518, 516, 470, 491, 546, 531, 547, 551, 488, 515, 486, 514, 513, 496, 545, 520, 487, 545, 493, 518, 499, 499, 511, 451, 524, 528, 517, 508] payload_rows_per_dst=[5600, 5544, 5606, 5472, 5526, 5610, 5512, 5466] dup_rows_per_dst=[2584, 2598, 2634, 2678, 2630, 2746, 2722, 2608] combine_rows_per_src=[5600, 5544, 5606, 5472, 5526, 5610, 5512, 5466] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44482 payload_rows_remote=38734 dup_rows=21054 dup_rows_remote=18196 dedup_ratio=0.321259 router_tokens_per_expert=[531, 512, 508, 504, 525, 541, 504, 510, 493, 550, 484, 452, 496, 531, 518, 520, 518, 507, 545, 499, 511, 457, 499, 503, 517, 505, 508, 528, 551, 529, 516, 556, 543, 501, 507, 495, 514, 479, 486, 506, 512, 534, 507, 517, 536, 520, 514, 502, 509, 519, 452, 504, 530, 502, 518, 552, 520, 501, 494, 512, 497, 530, 506, 521] payload_rows_per_dst=[5534, 5302, 5504, 5688, 5692, 5466, 5548, 5748] dup_rows_per_dst=[2574, 2300, 2574, 2774, 2814, 2536, 2624, 2858] combine_rows_per_src=[5534, 5302, 5504, 5688, 5692, 5466, 5548, 5748] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44370 payload_rows_remote=38894 dup_rows=21166 dup_rows_remote=18550 dedup_ratio=0.322968 router_tokens_per_expert=[519, 526, 528, 531, 508, 501, 506, 501, 502, 529, 531, 494, 496, 500, 519, 526, 499, 519, 509, 474, 508, 514, 535, 532, 542, 541, 544, 522, 476, 506, 532, 529, 476, 522, 515, 506, 493, 526, 492, 516, 521, 498, 520, 502, 525, 571, 516, 490, 509, 511, 480, 508, 464, 503, 480, 527, 513, 531, 485, 498, 538, 525, 508, 500] payload_rows_per_dst=[5580, 5528, 5544, 5684, 5476, 5586, 5438, 5534] dup_rows_per_dst=[2660, 2666, 2636, 2700, 2616, 2700, 2526, 2662] combine_rows_per_src=[5580, 5528, 5544, 5684, 5476, 5586, 5438, 5534] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44750 payload_rows_remote=39104 dup_rows=20786 dup_rows_remote=18142 dedup_ratio=0.317169 router_tokens_per_expert=[529, 477, 482, 529, 522, 503, 517, 535, 518, 556, 493, 508, 511, 528, 508, 502, 543, 506, 523, 489, 516, 501, 515, 513, 503, 521, 471, 509, 491, 504, 507, 526, 526, 505, 474, 543, 495, 501, 500, 569, 506, 541, 507, 543, 518, 481, 522, 527, 491, 534, 508, 510, 497, 505, 542, 503, 533, 510, 520, 460, 514, 532, 505, 490] payload_rows_per_dst=[5528, 5610, 5656, 5568, 5610, 5646, 5550, 5582] dup_rows_per_dst=[2660, 2638, 2556, 2496, 2616, 2644, 2630, 2546] combine_rows_per_src=[5528, 5610, 5656, 5568, 5610, 5646, 5550, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44582 payload_rows_remote=39018 dup_rows=20954 dup_rows_remote=18296 dedup_ratio=0.319733 router_tokens_per_expert=[510, 494, 521, 504, 471, 527, 516, 496, 484, 496, 506, 512, 493, 531, 503, 537, 526, 479, 529, 516, 511, 486, 481, 526, 522, 506, 529, 499, 500, 524, 527, 530, 515, 494, 510, 510, 513, 486, 543, 525, 536, 472, 512, 518, 515, 512, 521, 535, 541, 477, 531, 518, 521, 513, 479, 519, 503, 529, 541, 496, 531, 526, 509, 525] payload_rows_per_dst=[5564, 5556, 5486, 5740, 5710, 5602, 5622, 5302] dup_rows_per_dst=[2658, 2654, 2492, 2948, 2802, 2640, 2562, 2198] combine_rows_per_src=[5564, 5556, 5486, 5740, 5710, 5602, 5622, 5302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278433792 dispatch_meta_bytes=230256 combine_payload_bytes=278433792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278664048 ingress_bytes=388534272 link_bytes=388534272 token_bytes=557097840 weight_movement_bytes=110100480 total_nvlink_bytes=667198320 weight_to_token_ratio=0.197632 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.757250019e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44184 payload_rows_remote=38594 dup_rows=21352 dup_rows_remote=18702 dedup_ratio=0.325806 router_tokens_per_expert=[474, 550, 546, 547, 501, 516, 521, 485, 499, 537, 502, 505, 508, 518, 522, 529, 499, 556, 488, 512, 539, 515, 488, 511, 495, 517, 479, 547, 507, 522, 521, 512, 504, 503, 499, 529, 495, 514, 485, 476, 539, 549, 492, 551, 524, 471, 523, 499, 493, 525, 506, 532, 487, 507, 537, 526, 503, 483, 502, 534, 484, 492, 502, 534] payload_rows_per_dst=[5576, 5590, 5566, 5496, 5468, 5518, 5524, 5446] dup_rows_per_dst=[2704, 2650, 2650, 2704, 2542, 2778, 2702, 2622] combine_rows_per_src=[5576, 5590, 5566, 5496, 5468, 5518, 5524, 5446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279681024 dispatch_meta_bytes=229256 combine_payload_bytes=279681024 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279910280 ingress_bytes=499881984 link_bytes=499881984 token_bytes=559591304 weight_movement_bytes=220200960 total_nvlink_bytes=779792264 weight_to_token_ratio=0.393503 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.693769866e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44376 payload_rows_remote=38900 dup_rows=21160 dup_rows_remote=18540 dedup_ratio=0.322876 router_tokens_per_expert=[514, 524, 526, 526, 508, 496, 505, 499, 500, 524, 527, 492, 494, 500, 517, 523, 498, 515, 508, 473, 507, 511, 534, 530, 542, 541, 545, 522, 476, 507, 531, 531, 479, 521, 514, 508, 493, 525, 494, 514, 520, 500, 522, 503, 526, 570, 518, 490, 509, 514, 487, 510, 467, 506, 482, 529, 515, 536, 488, 502, 540, 528, 510, 502] payload_rows_per_dst=[5556, 5510, 5540, 5686, 5476, 5592, 5462, 5554] dup_rows_per_dst=[2640, 2644, 2612, 2704, 2620, 2706, 2546, 2688] combine_rows_per_src=[5556, 5510, 5540, 5686, 5476, 5592, 5462, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44330 payload_rows_remote=38738 dup_rows=21206 dup_rows_remote=18578 dedup_ratio=0.323578 router_tokens_per_expert=[509, 534, 482, 504, 515, 474, 514, 542, 506, 525, 543, 490, 503, 502, 491, 501, 536, 527, 491, 492, 507, 539, 517, 501, 496, 503, 500, 532, 494, 518, 492, 535, 521, 497, 524, 539, 517, 515, 470, 492, 550, 532, 549, 552, 490, 515, 486, 512, 516, 499, 550, 521, 489, 548, 492, 523, 501, 500, 511, 453, 527, 532, 519, 511] payload_rows_per_dst=[5584, 5538, 5592, 5470, 5524, 5614, 5528, 5480] dup_rows_per_dst=[2564, 2584, 2628, 2670, 2626, 2758, 2748, 2628] combine_rows_per_src=[5584, 5538, 5592, 5470, 5524, 5614, 5528, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44610 payload_rows_remote=38890 dup_rows=20926 dup_rows_remote=18196 dedup_ratio=0.319305 router_tokens_per_expert=[462, 511, 539, 523, 461, 524, 516, 522, 518, 543, 536, 488, 500, 522, 508, 514, 486, 470, 545, 492, 489, 526, 525, 512, 488, 512, 464, 489, 490, 504, 502, 511, 512, 493, 498, 539, 497, 528, 534, 519, 493, 504, 545, 527, 509, 480, 549, 482, 544, 549, 527, 517, 525, 531, 532, 500, 516, 544, 528, 526, 521, 524, 478, 505] payload_rows_per_dst=[5508, 5650, 5544, 5486, 5606, 5516, 5720, 5580] dup_rows_per_dst=[2608, 2608, 2546, 2434, 2634, 2662, 2730, 2704] combine_rows_per_src=[5508, 5650, 5544, 5486, 5606, 5516, 5720, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44744 payload_rows_remote=39094 dup_rows=20792 dup_rows_remote=18132 dedup_ratio=0.317261 router_tokens_per_expert=[527, 475, 481, 525, 519, 501, 515, 533, 516, 555, 490, 508, 511, 527, 506, 499, 543, 503, 522, 488, 513, 500, 514, 513, 502, 517, 470, 509, 490, 504, 506, 524, 527, 505, 474, 543, 495, 503, 504, 571, 509, 542, 508, 545, 518, 482, 522, 529, 492, 533, 510, 510, 501, 511, 545, 504, 531, 512, 522, 464, 517, 531, 510, 492] payload_rows_per_dst=[5514, 5606, 5648, 5558, 5618, 5650, 5556, 5594] dup_rows_per_dst=[2638, 2618, 2544, 2486, 2626, 2660, 2656, 2564] combine_rows_per_src=[5514, 5606, 5648, 5558, 5618, 5650, 5556, 5594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44490 payload_rows_remote=38860 dup_rows=21046 dup_rows_remote=18330 dedup_ratio=0.321136 router_tokens_per_expert=[522, 507, 500, 504, 525, 538, 504, 508, 492, 549, 483, 450, 494, 530, 514, 519, 518, 505, 544, 495, 509, 458, 500, 500, 517, 505, 507, 528, 551, 528, 514, 557, 545, 500, 507, 496, 512, 479, 490, 507, 514, 535, 505, 520, 536, 520, 514, 504, 511, 520, 452, 508, 535, 503, 518, 556, 522, 502, 500, 515, 500, 535, 507, 525] payload_rows_per_dst=[5570, 5472, 5492, 5678, 5540, 5570, 5538, 5630] dup_rows_per_dst=[2646, 2526, 2566, 2736, 2532, 2740, 2584, 2716] combine_rows_per_src=[5570, 5472, 5492, 5678, 5540, 5570, 5538, 5630] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=276641792 dispatch_meta_bytes=229184 combine_payload_bytes=276641792 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=276870976 ingress_bytes=606943232 link_bytes=606943232 token_bytes=553512768 weight_movement_bytes=330301440 total_nvlink_bytes=883814208 weight_to_token_ratio=0.596737 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.055574105e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44446 payload_rows_remote=38836 dup_rows=21090 dup_rows_remote=18380 dedup_ratio=0.321808 router_tokens_per_expert=[520, 530, 494, 512, 471, 552, 523, 493, 537, 524, 499, 512, 504, 551, 491, 534, 489, 537, 463, 504, 500, 477, 507, 518, 548, 461, 525, 534, 495, 492, 577, 528, 487, 489, 520, 496, 508, 536, 512, 510, 484, 474, 537, 458, 497, 526, 514, 516, 486, 531, 519, 524, 512, 539, 517, 540, 534, 493, 511, 529, 534, 531, 506, 496] payload_rows_per_dst=[5558, 5654, 5474, 5610, 5498, 5448, 5638, 5566] dup_rows_per_dst=[2632, 2650, 2516, 2710, 2618, 2564, 2698, 2702] combine_rows_per_src=[5558, 5654, 5474, 5610, 5498, 5448, 5638, 5566] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44428 payload_rows_remote=38824 dup_rows=21108 dup_rows_remote=18410 dedup_ratio=0.322083 router_tokens_per_expert=[522, 532, 494, 515, 473, 555, 527, 497, 539, 526, 501, 514, 506, 553, 495, 537, 493, 539, 461, 505, 499, 480, 510, 518, 547, 462, 527, 532, 494, 492, 573, 524, 487, 489, 519, 497, 510, 538, 511, 509, 481, 474, 535, 457, 496, 522, 515, 514, 486, 528, 519, 522, 510, 539, 515, 534, 532, 492, 509, 529, 533, 527, 502, 495] payload_rows_per_dst=[5580, 5664, 5480, 5604, 5496, 5434, 5618, 5552] dup_rows_per_dst=[2650, 2678, 2530, 2698, 2624, 2554, 2688, 2686] combine_rows_per_src=[5580, 5664, 5480, 5604, 5496, 5434, 5618, 5552] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44612 payload_rows_remote=38896 dup_rows=20924 dup_rows_remote=18202 dedup_ratio=0.319275 router_tokens_per_expert=[469, 511, 542, 524, 461, 525, 518, 523, 521, 545, 539, 488, 503, 522, 510, 518, 488, 471, 545, 495, 491, 528, 525, 514, 490, 508, 466, 489, 487, 501, 503, 509, 509, 496, 497, 538, 498, 528, 532, 521, 493, 503, 542, 525, 507, 480, 547, 481, 541, 550, 528, 517, 523, 530, 533, 497, 515, 540, 527, 523, 518, 520, 476, 504] payload_rows_per_dst=[5522, 5660, 5564, 5476, 5606, 5506, 5716, 5562] dup_rows_per_dst=[2624, 2632, 2550, 2430, 2632, 2650, 2722, 2684] combine_rows_per_src=[5522, 5660, 5564, 5476, 5606, 5506, 5716, 5562] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277616640 dispatch_meta_bytes=229184 combine_payload_bytes=277616640 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277845824 ingress_bytes=387717120 link_bytes=387717120 token_bytes=555462464 weight_movement_bytes=110100480 total_nvlink_bytes=665562944 weight_to_token_ratio=0.198214 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.743038402e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277645312 dispatch_meta_bytes=227720 combine_payload_bytes=277645312 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277873032 ingress_bytes=387745792 link_bytes=387745792 token_bytes=555518344 weight_movement_bytes=110100480 total_nvlink_bytes=665618824 weight_to_token_ratio=0.198194 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.743537055e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278792192 dispatch_meta_bytes=229776 combine_payload_bytes=278792192 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279021968 ingress_bytes=388892672 link_bytes=388892672 token_bytes=557814160 weight_movement_bytes=110100480 total_nvlink_bytes=667914640 weight_to_token_ratio=0.197378 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.763483185e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280297472 dispatch_meta_bytes=228984 combine_payload_bytes=280297472 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280526456 ingress_bytes=500498432 link_bytes=500498432 token_bytes=560823928 weight_movement_bytes=220200960 total_nvlink_bytes=781024888 weight_to_token_ratio=0.392638 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.704490911e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44646 payload_rows_remote=39088 dup_rows=20890 dup_rows_remote=18334 dedup_ratio=0.318756 router_tokens_per_expert=[521, 516, 503, 519, 513, 487, 487, 510, 555, 521, 540, 521, 543, 508, 504, 529, 552, 534, 523, 556, 489, 472, 529, 502, 510, 497, 507, 531, 512, 494, 484, 502, 482, 543, 525, 481, 542, 542, 497, 483, 480, 510, 535, 514, 495, 512, 549, 497, 511, 512, 529, 511, 529, 495, 538, 471, 502, 514, 477, 518, 536, 494, 489, 484] payload_rows_per_dst=[5558, 5680, 5696, 5570, 5564, 5592, 5476, 5510] dup_rows_per_dst=[2556, 2770, 2762, 2504, 2626, 2592, 2562, 2518] combine_rows_per_src=[5558, 5680, 5696, 5570, 5564, 5592, 5476, 5510] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278290432 dispatch_meta_bytes=228936 combine_payload_bytes=278290432 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278519368 ingress_bytes=388390912 link_bytes=388390912 token_bytes=556809800 weight_movement_bytes=110100480 total_nvlink_bytes=666910280 weight_to_token_ratio=0.197734 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.754756753e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278806528 dispatch_meta_bytes=228392 combine_payload_bytes=278806528 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279034920 ingress_bytes=609107968 link_bytes=609107968 token_bytes=557841448 weight_movement_bytes=330301440 total_nvlink_bytes=888142888 weight_to_token_ratio=0.592106 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059338937e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44174 payload_rows_remote=38578 dup_rows=21362 dup_rows_remote=18692 dedup_ratio=0.325958 router_tokens_per_expert=[476, 552, 552, 549, 504, 521, 524, 485, 502, 538, 502, 504, 510, 522, 526, 529, 503, 559, 486, 513, 542, 516, 491, 512, 495, 515, 480, 546, 506, 520, 520, 515, 504, 507, 500, 527, 493, 515, 484, 474, 539, 550, 490, 547, 522, 469, 521, 499, 492, 525, 505, 530, 484, 505, 535, 526, 499, 479, 499, 533, 483, 489, 499, 529] payload_rows_per_dst=[5600, 5596, 5574, 5492, 5460, 5512, 5514, 5426] dup_rows_per_dst=[2726, 2670, 2670, 2702, 2548, 2762, 2690, 2594] combine_rows_per_src=[5600, 5596, 5574, 5492, 5460, 5512, 5514, 5426] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280182784 dispatch_meta_bytes=229688 combine_payload_bytes=280182784 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280412472 ingress_bytes=500383744 link_bytes=500383744 token_bytes=560595256 weight_movement_bytes=220200960 total_nvlink_bytes=780796216 weight_to_token_ratio=0.392798 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.702496298e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44528 payload_rows_remote=38998 dup_rows=21008 dup_rows_remote=18450 dedup_ratio=0.320557 router_tokens_per_expert=[543, 491, 548, 559, 509, 525, 558, 521, 530, 493, 487, 487, 531, 491, 501, 495, 468, 518, 501, 489, 545, 519, 497, 507, 493, 513, 538, 496, 519, 490, 518, 514, 524, 539, 496, 509, 516, 505, 518, 473, 485, 539, 558, 494, 507, 546, 521, 494, 523, 505, 517, 532, 502, 531, 515, 509, 500, 494, 507, 503, 490, 511, 508, 503] payload_rows_per_dst=[5700, 5514, 5530, 5600, 5522, 5658, 5560, 5444] dup_rows_per_dst=[2808, 2516, 2558, 2562, 2638, 2630, 2708, 2588] combine_rows_per_src=[5700, 5514, 5530, 5600, 5522, 5658, 5560, 5444] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44522 payload_rows_remote=38766 dup_rows=21014 dup_rows_remote=18290 dedup_ratio=0.320648 router_tokens_per_expert=[522, 503, 549, 546, 501, 468, 515, 509, 542, 523, 535, 473, 508, 512, 508, 542, 520, 525, 498, 522, 506, 504, 532, 502, 491, 530, 534, 490, 463, 521, 484, 516, 487, 505, 547, 528, 495, 522, 507, 539, 506, 500, 519, 537, 519, 480, 496, 484, 555, 514, 508, 467, 505, 484, 535, 513, 522, 484, 494, 475, 557, 521, 521, 548] payload_rows_per_dst=[5512, 5588, 5480, 5386, 5642, 5636, 5522, 5756] dup_rows_per_dst=[2466, 2698, 2416, 2418, 2846, 2700, 2746, 2724] combine_rows_per_src=[5512, 5588, 5480, 5386, 5642, 5636, 5522, 5756] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44442 payload_rows_remote=39002 dup_rows=21094 dup_rows_remote=18510 dedup_ratio=0.321869 router_tokens_per_expert=[537, 501, 500, 520, 556, 481, 496, 511, 529, 506, 483, 558, 520, 517, 502, 527, 499, 562, 524, 507, 520, 515, 473, 515, 524, 513, 492, 509, 558, 510, 532, 514, 501, 517, 502, 479, 513, 503, 479, 518, 486, 544, 501, 544, 477, 500, 501, 497, 499, 509, 543, 529, 512, 513, 513, 505, 489, 518, 531, 517, 467, 488, 525, 537] payload_rows_per_dst=[5594, 5582, 5590, 5588, 5440, 5504, 5582, 5562] dup_rows_per_dst=[2610, 2702, 2640, 2716, 2584, 2596, 2664, 2582] combine_rows_per_src=[5594, 5582, 5590, 5588, 5440, 5504, 5582, 5562] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44534 payload_rows_remote=38916 dup_rows=21002 dup_rows_remote=18296 dedup_ratio=0.320465 router_tokens_per_expert=[521, 482, 539, 498, 526, 493, 483, 535, 515, 507, 529, 491, 495, 492, 506, 497, 527, 510, 552, 512, 528, 509, 569, 497, 505, 509, 526, 516, 530, 495, 512, 497, 500, 510, 482, 499, 485, 527, 515, 479, 514, 512, 533, 504, 486, 522, 544, 547, 530, 501, 544, 493, 509, 487, 478, 544, 525, 523, 517, 521, 530, 513, 498, 493] payload_rows_per_dst=[5568, 5532, 5638, 5548, 5470, 5618, 5568, 5592] dup_rows_per_dst=[2586, 2532, 2770, 2632, 2524, 2706, 2604, 2648] combine_rows_per_src=[5568, 5532, 5638, 5548, 5470, 5618, 5568, 5592] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=276527104 dispatch_meta_bytes=229080 combine_payload_bytes=276527104 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=276756184 ingress_bytes=606828544 link_bytes=606828544 token_bytes=553283288 weight_movement_bytes=330301440 total_nvlink_bytes=883584728 weight_to_token_ratio=0.596984 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.055374644e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279537664 dispatch_meta_bytes=229792 combine_payload_bytes=279537664 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279767456 ingress_bytes=609839104 link_bytes=609839104 token_bytes=559305120 weight_movement_bytes=330301440 total_nvlink_bytes=889606560 weight_to_token_ratio=0.590557 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060610503e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277874688 dispatch_meta_bytes=228224 combine_payload_bytes=277874688 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278102912 ingress_bytes=387975168 link_bytes=387975168 token_bytes=555977600 weight_movement_bytes=110100480 total_nvlink_bytes=666078080 weight_to_token_ratio=0.198030 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.747526281e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44530 payload_rows_remote=38932 dup_rows=21006 dup_rows_remote=18490 dedup_ratio=0.320526 router_tokens_per_expert=[503, 530, 505, 536, 498, 486, 526, 508, 513, 515, 507, 493, 524, 483, 506, 555, 501, 509, 527, 549, 495, 532, 553, 504, 495, 543, 452, 512, 539, 510, 512, 494, 520, 523, 536, 533, 533, 510, 568, 499, 484, 517, 477, 508, 514, 481, 505, 508, 506, 547, 519, 520, 501, 477, 503, 495, 528, 499, 497, 506, 507, 520, 514, 498] payload_rows_per_dst=[5534, 5568, 5648, 5598, 5688, 5432, 5546, 5516] dup_rows_per_dst=[2650, 2624, 2692, 2516, 2756, 2556, 2590, 2622] combine_rows_per_src=[5534, 5568, 5648, 5598, 5688, 5432, 5546, 5516] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279566336 dispatch_meta_bytes=230048 combine_payload_bytes=279566336 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279796384 ingress_bytes=389666816 link_bytes=389666816 token_bytes=559362720 weight_movement_bytes=110100480 total_nvlink_bytes=669463200 weight_to_token_ratio=0.196832 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.776946822e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278949888 dispatch_meta_bytes=228848 combine_payload_bytes=278949888 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279178736 ingress_bytes=389050368 link_bytes=389050368 token_bytes=558128624 weight_movement_bytes=110100480 total_nvlink_bytes=668229104 weight_to_token_ratio=0.197267 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.766225777e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44504 payload_rows_remote=39006 dup_rows=21032 dup_rows_remote=18400 dedup_ratio=0.320923 router_tokens_per_expert=[503, 462, 539, 515, 515, 496, 534, 499, 495, 480, 554, 526, 511, 518, 510, 517, 522, 527, 504, 489, 533, 514, 512, 482, 506, 525, 491, 543, 542, 502, 518, 514, 491, 525, 503, 488, 502, 503, 492, 547, 504, 510, 534, 508, 510, 493, 511, 513, 518, 510, 506, 497, 537, 551, 493, 513, 528, 529, 535, 488, 522, 507, 482, 520] payload_rows_per_dst=[5498, 5686, 5536, 5484, 5534, 5560, 5546, 5660] dup_rows_per_dst=[2632, 2746, 2630, 2584, 2568, 2606, 2704, 2562] combine_rows_per_src=[5498, 5686, 5536, 5484, 5534, 5560, 5546, 5660] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44518 payload_rows_remote=39002 dup_rows=21018 dup_rows_remote=18352 dedup_ratio=0.320709 router_tokens_per_expert=[517, 556, 526, 475, 517, 511, 514, 478, 500, 512, 506, 498, 528, 528, 488, 524, 503, 474, 514, 511, 541, 496, 489, 553, 506, 536, 473, 565, 581, 518, 514, 543, 491, 487, 518, 532, 501, 517, 496, 526, 491, 539, 500, 487, 470, 556, 556, 481, 503, 516, 493, 501, 531, 531, 519, 497, 496, 524, 509, 494, 496, 522, 488, 505] payload_rows_per_dst=[5592, 5564, 5528, 5732, 5572, 5528, 5516, 5486] dup_rows_per_dst=[2596, 2604, 2634, 2740, 2564, 2632, 2666, 2582] combine_rows_per_src=[5592, 5564, 5528, 5732, 5572, 5528, 5516, 5486] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279064576 dispatch_meta_bytes=229688 combine_payload_bytes=279064576 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279294264 ingress_bytes=609366016 link_bytes=609366016 token_bytes=558358840 weight_movement_bytes=330301440 total_nvlink_bytes=888660280 weight_to_token_ratio=0.591558 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059787725e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279595008 dispatch_meta_bytes=229624 combine_payload_bytes=279595008 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279824632 ingress_bytes=609896448 link_bytes=609896448 token_bytes=559419640 weight_movement_bytes=330301440 total_nvlink_bytes=889721080 weight_to_token_ratio=0.590436 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060710234e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279566336 dispatch_meta_bytes=229416 combine_payload_bytes=279566336 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279795752 ingress_bytes=389666816 link_bytes=389666816 token_bytes=559362088 weight_movement_bytes=110100480 total_nvlink_bytes=669462568 weight_to_token_ratio=0.196832 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.776946822e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44596 payload_rows_remote=39048 dup_rows=20940 dup_rows_remote=18408 dedup_ratio=0.319519 router_tokens_per_expert=[514, 488, 529, 518, 521, 513, 489, 523, 517, 495, 524, 472, 507, 489, 534, 502, 504, 502, 547, 529, 480, 517, 524, 520, 583, 514, 526, 512, 503, 498, 532, 548, 515, 491, 550, 523, 520, 489, 512, 494, 508, 478, 517, 507, 525, 532, 522, 498, 471, 507, 481, 517, 516, 488, 511, 516, 507, 519, 511, 518, 512, 512, 484, 543] payload_rows_per_dst=[5546, 5548, 5624, 5658, 5582, 5560, 5474, 5604] dup_rows_per_dst=[2644, 2532, 2622, 2774, 2606, 2614, 2540, 2608] combine_rows_per_src=[5546, 5548, 5624, 5658, 5582, 5560, 5474, 5604] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44696 payload_rows_remote=39056 dup_rows=20840 dup_rows_remote=18222 dedup_ratio=0.317993 router_tokens_per_expert=[510, 486, 526, 552, 499, 505, 517, 499, 499, 491, 482, 492, 510, 514, 501, 507, 524, 510, 534, 505, 537, 493, 528, 498, 554, 518, 495, 554, 499, 504, 548, 518, 516, 491, 543, 516, 493, 513, 494, 523, 474, 517, 493, 512, 512, 490, 514, 508, 517, 494, 520, 531, 516, 484, 501, 515, 546, 487, 534, 519, 519, 553, 458, 556] payload_rows_per_dst=[5602, 5458, 5640, 5716, 5572, 5490, 5542, 5676] dup_rows_per_dst=[2586, 2534, 2618, 2664, 2606, 2550, 2614, 2668] combine_rows_per_src=[5602, 5458, 5640, 5716, 5572, 5490, 5542, 5676] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278835200 dispatch_meta_bytes=229760 combine_payload_bytes=278835200 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279064960 ingress_bytes=388935680 link_bytes=388935680 token_bytes=557900160 weight_movement_bytes=110100480 total_nvlink_bytes=668000640 weight_to_token_ratio=0.197348 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.764231164e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277673984 dispatch_meta_bytes=229264 combine_payload_bytes=277673984 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277903248 ingress_bytes=387774464 link_bytes=387774464 token_bytes=555577232 weight_movement_bytes=110100480 total_nvlink_bytes=665677712 weight_to_token_ratio=0.198173 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.744035708e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278763520 dispatch_meta_bytes=228344 combine_payload_bytes=278763520 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278991864 ingress_bytes=609064960 link_bytes=609064960 token_bytes=557755384 weight_movement_bytes=330301440 total_nvlink_bytes=888056824 weight_to_token_ratio=0.592198 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059264139e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280225792 dispatch_meta_bytes=228904 combine_payload_bytes=280225792 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280454696 ingress_bytes=610527232 link_bytes=610527232 token_bytes=560680488 weight_movement_bytes=330301440 total_nvlink_bytes=890981928 weight_to_token_ratio=0.589108 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061807271e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44560 payload_rows_remote=38978 dup_rows=20976 dup_rows_remote=18320 dedup_ratio=0.320068 router_tokens_per_expert=[517, 515, 502, 518, 511, 482, 483, 508, 553, 513, 536, 518, 543, 510, 502, 526, 549, 532, 524, 555, 493, 471, 529, 503, 512, 499, 508, 531, 511, 494, 482, 501, 480, 543, 527, 480, 544, 543, 496, 485, 483, 510, 538, 514, 495, 513, 551, 498, 510, 513, 531, 513, 529, 496, 541, 473, 506, 515, 477, 524, 539, 495, 490, 485] payload_rows_per_dst=[5582, 5582, 5740, 5576, 5724, 5400, 5426, 5530] dup_rows_per_dst=[2656, 2654, 2878, 2532, 2884, 2360, 2480, 2532] combine_rows_per_src=[5582, 5582, 5740, 5576, 5724, 5400, 5426, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278548480 dispatch_meta_bytes=228760 combine_payload_bytes=278548480 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278777240 ingress_bytes=388648960 link_bytes=388648960 token_bytes=557325720 weight_movement_bytes=110100480 total_nvlink_bytes=667426200 weight_to_token_ratio=0.197551 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.759244632e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278376448 dispatch_meta_bytes=228864 combine_payload_bytes=278376448 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278605312 ingress_bytes=388476928 link_bytes=388476928 token_bytes=556981760 weight_movement_bytes=110100480 total_nvlink_bytes=667082240 weight_to_token_ratio=0.197673 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.756252713e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279394304 dispatch_meta_bytes=229192 combine_payload_bytes=279394304 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279623496 ingress_bytes=389494784 link_bytes=389494784 token_bytes=559017800 weight_movement_bytes=110100480 total_nvlink_bytes=669118280 weight_to_token_ratio=0.196953 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773954903e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44580 payload_rows_remote=39052 dup_rows=20956 dup_rows_remote=18442 dedup_ratio=0.319763 router_tokens_per_expert=[513, 485, 525, 515, 519, 508, 484, 519, 514, 492, 520, 470, 507, 488, 533, 497, 504, 502, 545, 524, 480, 518, 523, 522, 582, 514, 526, 511, 501, 495, 536, 548, 514, 491, 551, 522, 517, 490, 516, 495, 510, 480, 518, 506, 528, 535, 520, 501, 472, 509, 484, 520, 516, 491, 511, 516, 516, 523, 513, 522, 514, 512, 488, 547] payload_rows_per_dst=[5524, 5528, 5618, 5660, 5578, 5562, 5482, 5628] dup_rows_per_dst=[2612, 2514, 2618, 2766, 2614, 2634, 2556, 2642] combine_rows_per_src=[5524, 5528, 5618, 5660, 5578, 5562, 5482, 5628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44440 payload_rows_remote=39000 dup_rows=21096 dup_rows_remote=18516 dedup_ratio=0.321899 router_tokens_per_expert=[537, 498, 498, 517, 554, 476, 495, 504, 528, 505, 482, 557, 518, 516, 500, 525, 497, 559, 526, 509, 516, 515, 472, 510, 522, 514, 491, 509, 558, 511, 534, 513, 500, 516, 504, 479, 509, 504, 480, 518, 486, 545, 501, 547, 477, 502, 501, 498, 503, 511, 544, 532, 514, 515, 513, 509, 491, 520, 535, 521, 467, 492, 527, 541] payload_rows_per_dst=[5576, 5574, 5582, 5588, 5440, 5502, 5604, 5574] dup_rows_per_dst=[2582, 2688, 2626, 2716, 2580, 2612, 2678, 2614] combine_rows_per_src=[5576, 5574, 5582, 5588, 5440, 5502, 5604, 5574] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44454 payload_rows_remote=38760 dup_rows=21082 dup_rows_remote=18162 dedup_ratio=0.321686 router_tokens_per_expert=[496, 506, 503, 533, 514, 527, 529, 506, 521, 486, 482, 494, 527, 539, 477, 523, 499, 507, 503, 535, 544, 485, 512, 514, 499, 547, 523, 506, 474, 488, 502, 532, 498, 540, 515, 505, 513, 550, 537, 501, 483, 479, 505, 491, 543, 535, 511, 520, 535, 551, 487, 539, 465, 488, 479, 496, 551, 510, 503, 519, 548, 500, 537, 501] payload_rows_per_dst=[5508, 5510, 5380, 5530, 5696, 5488, 5648, 5694] dup_rows_per_dst=[2720, 2588, 2470, 2546, 2692, 2372, 2774, 2920] combine_rows_per_src=[5508, 5510, 5380, 5530, 5696, 5488, 5648, 5694] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44556 payload_rows_remote=39010 dup_rows=20980 dup_rows_remote=18284 dedup_ratio=0.320129 router_tokens_per_expert=[510, 492, 555, 526, 529, 472, 508, 513, 505, 524, 493, 534, 527, 474, 473, 516, 489, 511, 530, 513, 565, 513, 535, 487, 550, 498, 514, 534, 494, 504, 476, 517, 490, 487, 562, 511, 526, 512, 534, 499, 570, 533, 503, 500, 531, 472, 530, 482, 547, 484, 530, 488, 487, 492, 533, 506, 483, 534, 534, 536, 491, 496, 496, 508] payload_rows_per_dst=[5554, 5598, 5636, 5554, 5546, 5558, 5548, 5562] dup_rows_per_dst=[2656, 2494, 2650, 2620, 2696, 2684, 2586, 2594] combine_rows_per_src=[5554, 5598, 5636, 5554, 5546, 5558, 5548, 5562] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44540 payload_rows_remote=38922 dup_rows=20996 dup_rows_remote=18266 dedup_ratio=0.320374 router_tokens_per_expert=[511, 523, 493, 523, 513, 470, 497, 536, 525, 512, 514, 518, 483, 548, 529, 540, 483, 535, 519, 521, 487, 481, 554, 512, 534, 482, 487, 535, 499, 514, 520, 512, 497, 493, 538, 514, 500, 541, 527, 496, 536, 543, 516, 543, 533, 514, 508, 481, 512, 550, 507, 501, 502, 498, 520, 509, 470, 529, 511, 519, 497, 485, 484, 484] payload_rows_per_dst=[5582, 5640, 5512, 5554, 5532, 5618, 5614, 5488] dup_rows_per_dst=[2550, 2698, 2672, 2612, 2680, 2730, 2584, 2470] combine_rows_per_src=[5582, 5640, 5512, 5554, 5532, 5618, 5614, 5488] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279896064 dispatch_meta_bytes=229824 combine_payload_bytes=279896064 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280125888 ingress_bytes=389996544 link_bytes=389996544 token_bytes=560021952 weight_movement_bytes=110100480 total_nvlink_bytes=670122432 weight_to_token_ratio=0.196600 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.782681334e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279953408 dispatch_meta_bytes=229112 combine_payload_bytes=279953408 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=280182520 ingress_bytes=610254848 link_bytes=610254848 token_bytes=560135928 weight_movement_bytes=330301440 total_nvlink_bytes=890437368 weight_to_token_ratio=0.589681 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061333550e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277831680 dispatch_meta_bytes=227688 combine_payload_bytes=277831680 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278059368 ingress_bytes=387932160 link_bytes=387932160 token_bytes=555891048 weight_movement_bytes=110100480 total_nvlink_bytes=665991528 weight_to_token_ratio=0.198061 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.746778301e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44470 payload_rows_remote=38938 dup_rows=21066 dup_rows_remote=18338 dedup_ratio=0.321442 router_tokens_per_expert=[498, 507, 520, 504, 493, 513, 529, 512, 552, 488, 519, 529, 512, 536, 514, 500, 532, 538, 456, 516, 511, 538, 516, 533, 488, 500, 556, 499, 494, 509, 526, 509, 509, 487, 492, 487, 502, 521, 480, 521, 506, 562, 526, 508, 498, 550, 513, 529, 510, 524, 490, 531, 512, 518, 485, 481, 485, 512, 541, 480, 517, 543, 508, 493] payload_rows_per_dst=[5532, 5636, 5606, 5508, 5454, 5620, 5546, 5568] dup_rows_per_dst=[2728, 2676, 2674, 2534, 2544, 2764, 2556, 2590] combine_rows_per_src=[5532, 5636, 5606, 5508, 5454, 5620, 5546, 5568] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279623680 dispatch_meta_bytes=229176 combine_payload_bytes=279623680 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279852856 ingress_bytes=389724160 link_bytes=389724160 token_bytes=559476536 weight_movement_bytes=110100480 total_nvlink_bytes=669577016 weight_to_token_ratio=0.196792 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.777944128e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44412 payload_rows_remote=38886 dup_rows=21124 dup_rows_remote=18448 dedup_ratio=0.322327 router_tokens_per_expert=[505, 532, 495, 505, 552, 514, 491, 542, 492, 479, 508, 519, 533, 527, 533, 496, 519, 505, 535, 519, 488, 545, 499, 526, 527, 546, 490, 507, 533, 545, 448, 505, 511, 490, 504, 503, 501, 507, 511, 532, 511, 510, 548, 529, 500, 518, 477, 523, 516, 451, 491, 507, 494, 516, 535, 503, 485, 521, 524, 503, 542, 520, 557, 468] payload_rows_per_dst=[5522, 5548, 5608, 5526, 5500, 5606, 5476, 5626] dup_rows_per_dst=[2750, 2626, 2664, 2676, 2618, 2626, 2550, 2614] combine_rows_per_src=[5522, 5548, 5608, 5526, 5500, 5606, 5476, 5626] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278992896 dispatch_meta_bytes=228752 combine_payload_bytes=278992896 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279221648 ingress_bytes=499193856 link_bytes=499193856 token_bytes=558214544 weight_movement_bytes=220200960 total_nvlink_bytes=778415504 weight_to_token_ratio=0.394474 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.681802189e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44394 payload_rows_remote=38904 dup_rows=21142 dup_rows_remote=18490 dedup_ratio=0.322601 router_tokens_per_expert=[499, 529, 519, 464, 520, 505, 516, 509, 512, 497, 554, 511, 525, 516, 499, 518, 469, 518, 538, 495, 534, 491, 520, 504, 484, 491, 546, 532, 530, 498, 553, 511, 519, 505, 526, 512, 502, 499, 503, 547, 498, 538, 535, 557, 538, 469, 489, 530, 479, 538, 525, 494, 540, 498, 488, 509, 524, 513, 488, 515, 491, 488, 510, 494] payload_rows_per_dst=[5560, 5598, 5546, 5524, 5570, 5600, 5490, 5506] dup_rows_per_dst=[2562, 2666, 2592, 2766, 2656, 2708, 2652, 2540] combine_rows_per_src=[5560, 5598, 5546, 5524, 5570, 5600, 5490, 5506] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279107584 dispatch_meta_bytes=229104 combine_payload_bytes=279107584 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279336688 ingress_bytes=389208064 link_bytes=389208064 token_bytes=558444272 weight_movement_bytes=110100480 total_nvlink_bytes=668544752 weight_to_token_ratio=0.197156 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.768968370e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278734848 dispatch_meta_bytes=229336 combine_payload_bytes=278734848 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=278964184 ingress_bytes=719136768 link_bytes=719136768 token_bytes=557699032 weight_movement_bytes=440401920 total_nvlink_bytes=998100952 weight_to_token_ratio=0.789677 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.250697117e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278863872 dispatch_meta_bytes=229576 combine_payload_bytes=278863872 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279093448 ingress_bytes=388964352 link_bytes=388964352 token_bytes=557957320 weight_movement_bytes=110100480 total_nvlink_bytes=668057800 weight_to_token_ratio=0.197328 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.764729818e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44424 payload_rows_remote=38820 dup_rows=21112 dup_rows_remote=18450 dedup_ratio=0.322144 router_tokens_per_expert=[503, 512, 515, 490, 512, 502, 518, 510, 499, 562, 537, 465, 510, 518, 536, 506, 530, 503, 485, 495, 547, 499, 515, 560, 475, 500, 542, 482, 536, 537, 500, 531, 527, 476, 487, 537, 506, 534, 492, 528, 486, 515, 496, 505, 496, 543, 533, 487, 517, 506, 527, 546, 535, 489, 528, 530, 500, 516, 537, 502, 460, 509, 495, 491] payload_rows_per_dst=[5524, 5604, 5556, 5544, 5550, 5512, 5692, 5442] dup_rows_per_dst=[2600, 2662, 2712, 2662, 2624, 2610, 2664, 2578] combine_rows_per_src=[5524, 5604, 5556, 5544, 5550, 5512, 5692, 5442] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44586 payload_rows_remote=39032 dup_rows=20950 dup_rows_remote=18286 dedup_ratio=0.319672 router_tokens_per_expert=[514, 515, 508, 497, 516, 535, 499, 520, 491, 495, 562, 537, 504, 529, 529, 491, 511, 513, 510, 474, 574, 520, 481, 526, 517, 509, 486, 486, 518, 482, 507, 548, 527, 485, 521, 521, 529, 510, 516, 504, 536, 501, 498, 484, 534, 536, 513, 476, 542, 505, 528, 490, 508, 493, 483, 513, 528, 516, 499, 537, 515, 538, 503, 475] payload_rows_per_dst=[5498, 5628, 5554, 5544, 5580, 5592, 5568, 5622] dup_rows_per_dst=[2710, 2648, 2664, 2562, 2646, 2564, 2556, 2600] combine_rows_per_src=[5498, 5628, 5554, 5544, 5580, 5592, 5568, 5622] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44392 payload_rows_remote=38806 dup_rows=21144 dup_rows_remote=18530 dedup_ratio=0.322632 router_tokens_per_expert=[513, 511, 535, 519, 536, 549, 533, 524, 504, 489, 516, 488, 516, 501, 504, 504, 537, 535, 516, 508, 528, 510, 511, 517, 496, 488, 481, 503, 522, 531, 534, 499, 505, 511, 504, 511, 464, 549, 489, 523, 526, 535, 512, 520, 483, 520, 541, 523, 518, 493, 488, 510, 518, 575, 541, 503, 511, 469, 497, 486, 466, 515, 471, 533] payload_rows_per_dst=[5208, 5404, 5332, 5668, 5716, 5782, 5696, 5586] dup_rows_per_dst=[2272, 2626, 2364, 2682, 2868, 2938, 2780, 2614] combine_rows_per_src=[5208, 5404, 5332, 5668, 5716, 5782, 5696, 5586] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44546 payload_rows_remote=39032 dup_rows=20990 dup_rows_remote=18450 dedup_ratio=0.320282 router_tokens_per_expert=[506, 495, 514, 571, 537, 516, 519, 542, 511, 498, 545, 520, 522, 506, 493, 519, 492, 507, 490, 498, 541, 562, 523, 519, 523, 524, 533, 541, 548, 493, 465, 489, 481, 499, 537, 486, 506, 498, 533, 487, 496, 531, 511, 517, 456, 487, 504, 527, 490, 524, 542, 528, 500, 521, 465, 527, 549, 496, 497, 481, 465, 545, 551, 469] payload_rows_per_dst=[5644, 5506, 5562, 5640, 5514, 5562, 5572, 5546] dup_rows_per_dst=[2756, 2722, 2702, 2592, 2540, 2496, 2622, 2560] combine_rows_per_src=[5644, 5506, 5562, 5640, 5514, 5562, 5572, 5546] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44426 payload_rows_remote=38912 dup_rows=21110 dup_rows_remote=18524 dedup_ratio=0.322113 router_tokens_per_expert=[499, 501, 509, 509, 495, 507, 523, 498, 483, 467, 501, 513, 508, 522, 503, 573, 566, 520, 520, 518, 531, 501, 497, 535, 525, 515, 448, 571, 525, 488, 495, 491, 533, 500, 523, 490, 506, 511, 524, 503, 492, 541, 519, 500, 505, 516, 467, 510, 508, 497, 514, 527, 532, 525, 474, 475, 539, 520, 541, 518, 563, 496, 517, 525] payload_rows_per_dst=[5480, 5550, 5654, 5494, 5570, 5514, 5486, 5678] dup_rows_per_dst=[2602, 2590, 2722, 2622, 2610, 2586, 2618, 2760] combine_rows_per_src=[5480, 5550, 5654, 5494, 5570, 5514, 5486, 5678] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278261760 dispatch_meta_bytes=229080 combine_payload_bytes=278261760 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278490840 ingress_bytes=388362240 link_bytes=388362240 token_bytes=556752600 weight_movement_bytes=110100480 total_nvlink_bytes=666853080 weight_to_token_ratio=0.197755 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.754258100e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279781376 dispatch_meta_bytes=229272 combine_payload_bytes=279781376 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=280010648 ingress_bytes=720183296 link_bytes=720183296 token_bytes=559792024 weight_movement_bytes=440401920 total_nvlink_bytes=1000193944 weight_to_token_ratio=0.786724 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.252517202e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44498 payload_rows_remote=38984 dup_rows=21038 dup_rows_remote=18486 dedup_ratio=0.321014 router_tokens_per_expert=[541, 485, 544, 559, 507, 524, 552, 520, 529, 490, 487, 488, 527, 489, 497, 491, 467, 518, 500, 486, 545, 515, 496, 506, 492, 514, 539, 496, 519, 490, 520, 510, 524, 539, 497, 509, 514, 507, 519, 471, 488, 538, 559, 497, 511, 547, 522, 494, 523, 505, 520, 538, 504, 532, 518, 514, 503, 495, 511, 504, 491, 514, 512, 505] payload_rows_per_dst=[5676, 5498, 5514, 5602, 5516, 5662, 5576, 5454] dup_rows_per_dst=[2788, 2498, 2552, 2558, 2644, 2650, 2732, 2616] combine_rows_per_src=[5676, 5498, 5514, 5602, 5516, 5662, 5576, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44512 payload_rows_remote=38982 dup_rows=21024 dup_rows_remote=18322 dedup_ratio=0.320801 router_tokens_per_expert=[513, 552, 521, 469, 515, 508, 515, 472, 496, 507, 503, 496, 529, 526, 485, 523, 503, 472, 514, 509, 542, 495, 489, 551, 503, 539, 472, 564, 580, 519, 513, 543, 492, 486, 518, 531, 501, 518, 496, 526, 494, 538, 499, 487, 469, 558, 558, 481, 504, 520, 498, 505, 534, 534, 522, 499, 500, 530, 510, 498, 501, 523, 493, 507] payload_rows_per_dst=[5576, 5546, 5520, 5734, 5572, 5532, 5530, 5502] dup_rows_per_dst=[2554, 2584, 2630, 2732, 2564, 2636, 2702, 2622] combine_rows_per_src=[5576, 5546, 5520, 5734, 5572, 5532, 5530, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44542 payload_rows_remote=38914 dup_rows=20994 dup_rows_remote=18278 dedup_ratio=0.320343 router_tokens_per_expert=[520, 476, 536, 497, 525, 492, 482, 529, 513, 504, 526, 492, 498, 490, 503, 496, 525, 509, 549, 513, 528, 508, 564, 495, 505, 511, 526, 518, 530, 496, 512, 496, 500, 510, 483, 501, 485, 525, 513, 481, 517, 511, 533, 506, 489, 523, 546, 547, 531, 504, 546, 494, 510, 489, 478, 546, 528, 524, 520, 522, 532, 515, 500, 495] payload_rows_per_dst=[5544, 5524, 5632, 5556, 5470, 5628, 5580, 5608] dup_rows_per_dst=[2570, 2520, 2750, 2632, 2526, 2716, 2616, 2664] combine_rows_per_src=[5544, 5524, 5632, 5556, 5470, 5628, 5580, 5608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279924736 dispatch_meta_bytes=229976 combine_payload_bytes=279924736 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280154712 ingress_bytes=390025216 link_bytes=390025216 token_bytes=560079448 weight_movement_bytes=110100480 total_nvlink_bytes=670179928 weight_to_token_ratio=0.196580 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.783179987e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44568 payload_rows_remote=38882 dup_rows=20968 dup_rows_remote=18358 dedup_ratio=0.319946 router_tokens_per_expert=[518, 500, 548, 542, 497, 466, 514, 507, 539, 521, 535, 471, 507, 509, 505, 541, 517, 523, 493, 523, 507, 503, 531, 501, 492, 529, 531, 488, 465, 521, 483, 517, 487, 503, 547, 527, 494, 523, 511, 536, 508, 498, 521, 538, 519, 479, 501, 484, 560, 514, 509, 472, 507, 483, 537, 520, 522, 486, 498, 477, 560, 526, 525, 552] payload_rows_per_dst=[5588, 5578, 5612, 5494, 5540, 5560, 5510, 5686] dup_rows_per_dst=[2596, 2678, 2548, 2590, 2716, 2536, 2694, 2610] combine_rows_per_src=[5588, 5578, 5612, 5494, 5540, 5560, 5510, 5686] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44558 payload_rows_remote=38956 dup_rows=20978 dup_rows_remote=18470 dedup_ratio=0.320099 router_tokens_per_expert=[501, 527, 505, 533, 495, 482, 524, 504, 507, 509, 504, 489, 522, 482, 505, 553, 500, 508, 521, 550, 496, 529, 549, 506, 494, 543, 451, 509, 539, 512, 513, 494, 519, 520, 537, 532, 533, 509, 568, 500, 489, 518, 478, 508, 515, 486, 505, 510, 508, 548, 519, 529, 505, 480, 505, 497, 530, 500, 503, 510, 513, 519, 518, 501] payload_rows_per_dst=[5520, 5544, 5640, 5602, 5696, 5444, 5570, 5542] dup_rows_per_dst=[2622, 2598, 2678, 2508, 2740, 2574, 2612, 2646] combine_rows_per_src=[5520, 5544, 5640, 5602, 5696, 5444, 5570, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279552000 dispatch_meta_bytes=230064 combine_payload_bytes=279552000 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279782064 ingress_bytes=389652480 link_bytes=389652480 token_bytes=559334064 weight_movement_bytes=110100480 total_nvlink_bytes=669434544 weight_to_token_ratio=0.196842 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.051471 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.776697495e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279437312 dispatch_meta_bytes=229880 combine_payload_bytes=279437312 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279667192 ingress_bytes=609738752 link_bytes=609738752 token_bytes=559104504 weight_movement_bytes=330301440 total_nvlink_bytes=889405944 weight_to_token_ratio=0.590769 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060435975e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44378 payload_rows_remote=38856 dup_rows=21158 dup_rows_remote=18472 dedup_ratio=0.322845 router_tokens_per_expert=[501, 457, 536, 513, 512, 493, 533, 495, 492, 477, 549, 524, 511, 519, 510, 517, 518, 524, 505, 487, 533, 512, 512, 481, 505, 521, 488, 540, 543, 504, 516, 515, 491, 526, 504, 488, 504, 504, 493, 547, 504, 513, 531, 507, 510, 495, 510, 516, 520, 512, 507, 499, 539, 557, 495, 516, 533, 530, 537, 491, 524, 514, 486, 522] payload_rows_per_dst=[5522, 5746, 5468, 5336, 5616, 5612, 5476, 5602] dup_rows_per_dst=[2686, 2918, 2548, 2462, 2724, 2676, 2618, 2526] combine_rows_per_src=[5522, 5746, 5468, 5336, 5616, 5612, 5476, 5602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279422976 dispatch_meta_bytes=229216 combine_payload_bytes=279422976 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279652192 ingress_bytes=499623936 link_bytes=499623936 token_bytes=559075168 weight_movement_bytes=220200960 total_nvlink_bytes=779276128 weight_to_token_ratio=0.393866 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.689281987e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278935552 dispatch_meta_bytes=228768 combine_payload_bytes=278935552 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279164320 ingress_bytes=389036032 link_bytes=389036032 token_bytes=558099872 weight_movement_bytes=110100480 total_nvlink_bytes=668200352 weight_to_token_ratio=0.197277 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.765976451e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278706176 dispatch_meta_bytes=228960 combine_payload_bytes=278706176 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278935136 ingress_bytes=609007616 link_bytes=609007616 token_bytes=557641312 weight_movement_bytes=330301440 total_nvlink_bytes=887942752 weight_to_token_ratio=0.592319 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059164409e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279236608 dispatch_meta_bytes=229704 combine_payload_bytes=279236608 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279466312 ingress_bytes=499437568 link_bytes=499437568 token_bytes=558702920 weight_movement_bytes=220200960 total_nvlink_bytes=778903880 weight_to_token_ratio=0.394129 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.686040741e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278519808 dispatch_meta_bytes=229312 combine_payload_bytes=278519808 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278749120 ingress_bytes=388620288 link_bytes=388620288 token_bytes=557268928 weight_movement_bytes=110100480 total_nvlink_bytes=667369408 weight_to_token_ratio=0.197572 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.758745979e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44426 payload_rows_remote=38828 dup_rows=21110 dup_rows_remote=18468 dedup_ratio=0.322113 router_tokens_per_expert=[500, 510, 514, 485, 513, 498, 516, 510, 498, 561, 536, 464, 509, 514, 535, 503, 526, 501, 488, 493, 547, 499, 517, 562, 475, 501, 543, 481, 534, 539, 499, 531, 527, 476, 486, 535, 504, 534, 491, 526, 488, 513, 496, 506, 500, 544, 534, 492, 517, 505, 528, 547, 537, 490, 532, 533, 501, 518, 540, 502, 463, 513, 496, 492] payload_rows_per_dst=[5506, 5598, 5558, 5540, 5552, 5520, 5698, 5454] dup_rows_per_dst=[2586, 2642, 2708, 2666, 2606, 2626, 2680, 2596] combine_rows_per_src=[5506, 5598, 5558, 5540, 5552, 5520, 5698, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44534 payload_rows_remote=38978 dup_rows=21002 dup_rows_remote=18292 dedup_ratio=0.320465 router_tokens_per_expert=[508, 491, 554, 521, 528, 471, 505, 511, 504, 520, 492, 535, 526, 473, 470, 515, 488, 511, 527, 511, 565, 513, 535, 482, 550, 492, 511, 533, 492, 501, 475, 515, 492, 489, 564, 513, 525, 516, 534, 500, 570, 534, 503, 500, 528, 475, 532, 484, 547, 487, 533, 490, 488, 495, 535, 511, 483, 536, 538, 539, 493, 499, 498, 512] payload_rows_per_dst=[5536, 5586, 5620, 5534, 5556, 5560, 5556, 5586] dup_rows_per_dst=[2642, 2484, 2644, 2604, 2710, 2692, 2616, 2610] combine_rows_per_src=[5536, 5586, 5620, 5534, 5556, 5560, 5556, 5586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44708 payload_rows_remote=39078 dup_rows=20828 dup_rows_remote=18222 dedup_ratio=0.317810 router_tokens_per_expert=[505, 485, 525, 549, 496, 505, 513, 497, 499, 486, 479, 491, 508, 515, 500, 509, 523, 509, 533, 502, 537, 491, 527, 496, 556, 515, 494, 555, 500, 504, 546, 517, 518, 489, 544, 517, 493, 513, 492, 524, 475, 516, 493, 512, 511, 491, 515, 510, 519, 498, 521, 532, 519, 485, 507, 516, 546, 487, 534, 521, 523, 557, 464, 559] payload_rows_per_dst=[5590, 5452, 5630, 5712, 5580, 5494, 5560, 5690] dup_rows_per_dst=[2560, 2522, 2606, 2662, 2600, 2552, 2634, 2692] combine_rows_per_src=[5590, 5452, 5630, 5712, 5580, 5494, 5560, 5690] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278319104 dispatch_meta_bytes=229184 combine_payload_bytes=278319104 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278548288 ingress_bytes=388419584 link_bytes=388419584 token_bytes=556867392 weight_movement_bytes=110100480 total_nvlink_bytes=666967872 weight_to_token_ratio=0.197714 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.755255406e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44390 payload_rows_remote=38886 dup_rows=21146 dup_rows_remote=18484 dedup_ratio=0.322662 router_tokens_per_expert=[496, 527, 517, 461, 518, 506, 517, 507, 511, 497, 550, 505, 525, 515, 495, 516, 469, 518, 538, 494, 532, 490, 518, 504, 481, 490, 545, 531, 529, 502, 551, 510, 519, 504, 524, 514, 500, 501, 502, 549, 499, 539, 534, 558, 539, 469, 491, 532, 479, 540, 526, 494, 541, 500, 491, 512, 524, 519, 491, 516, 495, 491, 513, 497] payload_rows_per_dst=[5550, 5580, 5538, 5518, 5572, 5600, 5504, 5528] dup_rows_per_dst=[2548, 2648, 2588, 2760, 2654, 2722, 2662, 2564] combine_rows_per_src=[5550, 5580, 5538, 5518, 5572, 5600, 5504, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44534 payload_rows_remote=38918 dup_rows=21002 dup_rows_remote=18258 dedup_ratio=0.320465 router_tokens_per_expert=[510, 520, 492, 521, 508, 469, 494, 536, 521, 511, 512, 516, 482, 545, 529, 540, 486, 534, 517, 521, 484, 480, 553, 511, 536, 482, 487, 534, 500, 514, 517, 510, 495, 493, 539, 517, 500, 540, 525, 494, 534, 542, 523, 544, 534, 513, 508, 482, 514, 551, 508, 501, 505, 497, 520, 511, 473, 533, 510, 523, 500, 492, 487, 488] payload_rows_per_dst=[5566, 5634, 5504, 5550, 5522, 5616, 5624, 5518] dup_rows_per_dst=[2534, 2678, 2668, 2610, 2684, 2744, 2590, 2494] combine_rows_per_src=[5566, 5634, 5504, 5550, 5522, 5616, 5624, 5518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44446 payload_rows_remote=38830 dup_rows=21090 dup_rows_remote=18336 dedup_ratio=0.321808 router_tokens_per_expert=[492, 502, 501, 531, 514, 526, 528, 505, 520, 482, 480, 493, 526, 536, 475, 520, 498, 507, 504, 535, 543, 482, 510, 513, 499, 548, 523, 506, 475, 487, 503, 532, 498, 541, 516, 505, 512, 552, 538, 500, 486, 482, 505, 491, 545, 535, 511, 519, 538, 552, 491, 538, 468, 489, 480, 497, 553, 512, 504, 520, 552, 499, 539, 504] payload_rows_per_dst=[5494, 5504, 5498, 5486, 5674, 5602, 5572, 5616] dup_rows_per_dst=[2704, 2560, 2686, 2514, 2650, 2546, 2676, 2754] combine_rows_per_src=[5494, 5504, 5498, 5486, 5674, 5602, 5572, 5616] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279394304 dispatch_meta_bytes=229080 combine_payload_bytes=279394304 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279623384 ingress_bytes=389494784 link_bytes=389494784 token_bytes=559017688 weight_movement_bytes=110100480 total_nvlink_bytes=669118168 weight_to_token_ratio=0.196953 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773954903e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44422 payload_rows_remote=38898 dup_rows=21114 dup_rows_remote=18436 dedup_ratio=0.322174 router_tokens_per_expert=[504, 531, 493, 500, 548, 512, 489, 540, 491, 476, 505, 517, 534, 525, 532, 492, 519, 504, 535, 517, 485, 544, 498, 526, 527, 547, 491, 503, 535, 544, 449, 505, 510, 488, 503, 503, 502, 510, 511, 534, 511, 512, 551, 530, 502, 518, 477, 524, 517, 452, 491, 508, 496, 518, 535, 504, 490, 524, 528, 504, 544, 525, 558, 470] payload_rows_per_dst=[5508, 5536, 5600, 5524, 5506, 5614, 5490, 5644] dup_rows_per_dst=[2726, 2608, 2656, 2678, 2616, 2636, 2552, 2642] combine_rows_per_src=[5508, 5536, 5600, 5524, 5506, 5614, 5490, 5644] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=280111104 dispatch_meta_bytes=229200 combine_payload_bytes=280111104 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280340304 ingress_bytes=500312064 link_bytes=500312064 token_bytes=560451408 weight_movement_bytes=220200960 total_nvlink_bytes=780652368 weight_to_token_ratio=0.392899 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.701249665e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44444 payload_rows_remote=38820 dup_rows=21092 dup_rows_remote=18230 dedup_ratio=0.321838 router_tokens_per_expert=[493, 506, 518, 498, 488, 510, 528, 508, 548, 487, 519, 525, 510, 535, 510, 499, 529, 540, 455, 513, 507, 539, 517, 534, 488, 503, 555, 500, 495, 508, 528, 512, 507, 487, 494, 489, 501, 521, 481, 521, 507, 561, 530, 509, 499, 551, 514, 530, 511, 526, 489, 534, 513, 517, 488, 483, 486, 515, 544, 483, 520, 544, 512, 496] payload_rows_per_dst=[5624, 5690, 5496, 5560, 5482, 5462, 5554, 5576] dup_rows_per_dst=[2862, 2814, 2536, 2618, 2572, 2550, 2568, 2572] combine_rows_per_src=[5624, 5690, 5496, 5560, 5482, 5462, 5554, 5576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278964224 dispatch_meta_bytes=228704 combine_payload_bytes=278964224 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279192928 ingress_bytes=499165184 link_bytes=499165184 token_bytes=558157152 weight_movement_bytes=220200960 total_nvlink_bytes=778358112 weight_to_token_ratio=0.394514 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.681303535e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278734848 dispatch_meta_bytes=229480 combine_payload_bytes=278734848 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278964328 ingress_bytes=388835328 link_bytes=388835328 token_bytes=557699176 weight_movement_bytes=110100480 total_nvlink_bytes=667799656 weight_to_token_ratio=0.197419 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.762485878e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278333440 dispatch_meta_bytes=228664 combine_payload_bytes=278333440 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278562104 ingress_bytes=498534400 link_bytes=498534400 token_bytes=556895544 weight_movement_bytes=220200960 total_nvlink_bytes=777096504 weight_to_token_ratio=0.395408 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.670333164e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278820864 dispatch_meta_bytes=229336 combine_payload_bytes=278820864 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279050200 ingress_bytes=499021824 link_bytes=499021824 token_bytes=557871064 weight_movement_bytes=220200960 total_nvlink_bytes=778072024 weight_to_token_ratio=0.394717 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.678810269e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44402 payload_rows_remote=38752 dup_rows=21134 dup_rows_remote=18494 dedup_ratio=0.322479 router_tokens_per_expert=[508, 485, 499, 522, 543, 521, 522, 537, 523, 516, 537, 495, 559, 530, 502, 483, 531, 528, 506, 501, 516, 512, 508, 492, 511, 491, 510, 518, 486, 496, 545, 494, 494, 510, 517, 536, 521, 473, 544, 511, 487, 497, 530, 493, 505, 518, 518, 514, 526, 513, 481, 532, 532, 492, 507, 518, 515, 516, 497, 495, 511, 526, 536, 476] payload_rows_per_dst=[5524, 5650, 5584, 5486, 5536, 5512, 5588, 5522] dup_rows_per_dst=[2750, 2640, 2604, 2616, 2676, 2612, 2614, 2622] combine_rows_per_src=[5524, 5650, 5584, 5486, 5536, 5512, 5588, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278261760 dispatch_meta_bytes=228200 combine_payload_bytes=278261760 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278489960 ingress_bytes=388362240 link_bytes=388362240 token_bytes=556751720 weight_movement_bytes=110100480 total_nvlink_bytes=666852200 weight_to_token_ratio=0.197755 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.754258100e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44546 payload_rows_remote=39030 dup_rows=20990 dup_rows_remote=18442 dedup_ratio=0.320282 router_tokens_per_expert=[502, 487, 513, 567, 535, 516, 515, 541, 509, 496, 543, 519, 522, 505, 491, 515, 492, 502, 488, 497, 538, 562, 522, 519, 523, 522, 531, 543, 548, 491, 467, 491, 482, 502, 536, 486, 506, 496, 538, 486, 499, 530, 514, 517, 456, 488, 507, 530, 491, 526, 542, 527, 501, 521, 469, 527, 550, 498, 503, 485, 470, 548, 554, 471] payload_rows_per_dst=[5628, 5500, 5552, 5638, 5516, 5568, 5580, 5564] dup_rows_per_dst=[2724, 2700, 2688, 2594, 2548, 2514, 2628, 2594] combine_rows_per_src=[5628, 5500, 5552, 5638, 5516, 5568, 5580, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44588 payload_rows_remote=39040 dup_rows=20948 dup_rows_remote=18286 dedup_ratio=0.319641 router_tokens_per_expert=[510, 513, 506, 492, 511, 528, 497, 518, 490, 495, 561, 534, 502, 526, 528, 489, 509, 513, 510, 475, 577, 517, 478, 526, 516, 509, 489, 486, 518, 476, 509, 548, 528, 487, 522, 521, 529, 511, 516, 502, 536, 501, 499, 486, 536, 536, 515, 478, 544, 506, 528, 495, 509, 493, 484, 515, 533, 515, 503, 539, 516, 541, 509, 479] payload_rows_per_dst=[5476, 5614, 5548, 5542, 5590, 5602, 5580, 5636] dup_rows_per_dst=[2674, 2636, 2662, 2560, 2642, 2572, 2568, 2634] combine_rows_per_src=[5476, 5614, 5548, 5542, 5590, 5602, 5580, 5636] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277774336 dispatch_meta_bytes=228984 combine_payload_bytes=277774336 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278003320 ingress_bytes=387874816 link_bytes=387874816 token_bytes=555777656 weight_movement_bytes=110100480 total_nvlink_bytes=665878136 weight_to_token_ratio=0.198102 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.745780995e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279767040 dispatch_meta_bytes=229888 combine_payload_bytes=279767040 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279996928 ingress_bytes=389867520 link_bytes=389867520 token_bytes=559763968 weight_movement_bytes=110100480 total_nvlink_bytes=669864448 weight_to_token_ratio=0.196691 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.780437395e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44434 payload_rows_remote=38904 dup_rows=21102 dup_rows_remote=18508 dedup_ratio=0.321991 router_tokens_per_expert=[497, 498, 506, 507, 492, 508, 522, 496, 481, 462, 499, 511, 505, 519, 499, 572, 560, 517, 521, 515, 531, 502, 495, 534, 524, 514, 447, 570, 526, 486, 495, 492, 534, 504, 523, 488, 508, 512, 527, 505, 494, 541, 521, 502, 504, 519, 469, 512, 509, 499, 516, 529, 531, 527, 476, 478, 540, 521, 542, 519, 570, 497, 521, 527] payload_rows_per_dst=[5466, 5526, 5644, 5496, 5582, 5530, 5498, 5692] dup_rows_per_dst=[2586, 2570, 2706, 2612, 2620, 2594, 2632, 2782] combine_rows_per_src=[5466, 5526, 5644, 5496, 5582, 5530, 5498, 5692] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44532 payload_rows_remote=39044 dup_rows=21004 dup_rows_remote=18438 dedup_ratio=0.320496 router_tokens_per_expert=[550, 521, 516, 546, 495, 527, 497, 489, 511, 492, 515, 480, 487, 514, 538, 481, 540, 526, 552, 505, 534, 542, 504, 516, 539, 535, 518, 514, 530, 524, 507, 541, 480, 461, 483, 481, 502, 480, 516, 533, 504, 509, 547, 502, 524, 510, 499, 493, 467, 470, 490, 512, 545, 505, 531, 507, 521, 475, 477, 515, 539, 541, 539, 524] payload_rows_per_dst=[5594, 5528, 5656, 5678, 5404, 5586, 5488, 5598] dup_rows_per_dst=[2688, 2508, 2782, 2738, 2468, 2590, 2566, 2664] combine_rows_per_src=[5594, 5528, 5656, 5678, 5404, 5586, 5488, 5598] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44464 payload_rows_remote=38978 dup_rows=21072 dup_rows_remote=18618 dedup_ratio=0.321533 router_tokens_per_expert=[508, 510, 531, 514, 534, 547, 531, 523, 503, 486, 514, 486, 512, 498, 499, 499, 535, 533, 517, 507, 528, 510, 509, 518, 496, 488, 481, 504, 526, 530, 533, 502, 504, 512, 505, 511, 465, 548, 492, 524, 525, 536, 514, 520, 482, 522, 540, 524, 520, 495, 490, 513, 520, 575, 543, 506, 517, 475, 497, 486, 468, 516, 474, 537] payload_rows_per_dst=[5364, 5518, 5370, 5672, 5700, 5724, 5630, 5486] dup_rows_per_dst=[2448, 2724, 2390, 2700, 2820, 2842, 2694, 2454] combine_rows_per_src=[5364, 5518, 5370, 5672, 5700, 5724, 5630, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279838720 dispatch_meta_bytes=229304 combine_payload_bytes=279838720 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280068024 ingress_bytes=610140160 link_bytes=610140160 token_bytes=559906744 weight_movement_bytes=330301440 total_nvlink_bytes=890208184 weight_to_token_ratio=0.589922 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061134089e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44734 payload_rows_remote=39226 dup_rows=20802 dup_rows_remote=18254 dedup_ratio=0.317413 router_tokens_per_expert=[495, 510, 538, 482, 543, 506, 547, 519, 501, 500, 495, 489, 473, 512, 546, 506, 510, 494, 509, 503, 531, 498, 487, 513, 548, 474, 499, 518, 464, 523, 459, 543, 515, 503, 503, 539, 534, 527, 518, 499, 516, 520, 525, 504, 518, 542, 555, 464, 516, 510, 523, 516, 521, 527, 513, 515, 503, 553, 527, 518, 499, 509, 493, 508] payload_rows_per_dst=[5638, 5560, 5504, 5508, 5648, 5656, 5606, 5614] dup_rows_per_dst=[2642, 2484, 2586, 2548, 2628, 2632, 2676, 2606] combine_rows_per_src=[5638, 5560, 5504, 5508, 5648, 5656, 5606, 5614] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44440 payload_rows_remote=38746 dup_rows=21096 dup_rows_remote=18404 dedup_ratio=0.321899 router_tokens_per_expert=[511, 538, 485, 545, 540, 511, 503, 528, 485, 539, 542, 546, 522, 533, 519, 522, 494, 529, 480, 576, 490, 545, 520, 504, 499, 509, 493, 487, 519, 552, 506, 514, 526, 533, 481, 489, 517, 497, 493, 539, 519, 490, 526, 467, 495, 488, 488, 521, 525, 501, 521, 501, 512, 499, 524, 508, 496, 470, 529, 536, 467, 508, 491, 525] payload_rows_per_dst=[5694, 5594, 5590, 5518, 5536, 5462, 5530, 5516] dup_rows_per_dst=[2692, 2822, 2622, 2640, 2614, 2526, 2652, 2528] combine_rows_per_src=[5694, 5594, 5590, 5518, 5536, 5462, 5530, 5516] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278161408 dispatch_meta_bytes=229344 combine_payload_bytes=278161408 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278390752 ingress_bytes=388261888 link_bytes=388261888 token_bytes=556552160 weight_movement_bytes=110100480 total_nvlink_bytes=666652640 weight_to_token_ratio=0.197826 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.752512813e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279781376 dispatch_meta_bytes=229928 combine_payload_bytes=279781376 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280011304 ingress_bytes=389881856 link_bytes=389881856 token_bytes=559792680 weight_movement_bytes=110100480 total_nvlink_bytes=669893160 weight_to_token_ratio=0.196681 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.780686721e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44706 payload_rows_remote=39196 dup_rows=20830 dup_rows_remote=18284 dedup_ratio=0.317841 router_tokens_per_expert=[500, 514, 542, 486, 545, 509, 551, 521, 502, 501, 498, 491, 474, 512, 545, 507, 511, 497, 513, 502, 531, 501, 484, 515, 546, 474, 498, 518, 466, 525, 459, 542, 513, 498, 505, 540, 536, 529, 518, 499, 516, 520, 523, 503, 517, 543, 551, 463, 516, 509, 521, 515, 521, 527, 509, 512, 499, 548, 526, 512, 498, 504, 491, 506] payload_rows_per_dst=[5662, 5572, 5506, 5510, 5640, 5644, 5584, 5588] dup_rows_per_dst=[2674, 2488, 2602, 2546, 2636, 2628, 2676, 2580] combine_rows_per_src=[5662, 5572, 5506, 5510, 5640, 5644, 5584, 5588] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44528 payload_rows_remote=39058 dup_rows=21008 dup_rows_remote=18464 dedup_ratio=0.320557 router_tokens_per_expert=[553, 525, 520, 546, 498, 532, 500, 489, 517, 492, 515, 481, 489, 516, 540, 482, 540, 526, 552, 505, 538, 543, 504, 517, 537, 537, 517, 516, 529, 527, 507, 538, 481, 461, 485, 481, 503, 479, 517, 531, 502, 505, 547, 501, 524, 507, 496, 491, 462, 471, 489, 510, 545, 502, 524, 504, 521, 473, 476, 514, 538, 539, 539, 522] payload_rows_per_dst=[5616, 5536, 5662, 5672, 5406, 5574, 5470, 5592] dup_rows_per_dst=[2710, 2528, 2788, 2744, 2470, 2572, 2544, 2652] combine_rows_per_src=[5616, 5536, 5662, 5672, 5406, 5574, 5470, 5592] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278921216 dispatch_meta_bytes=229744 combine_payload_bytes=278921216 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279150960 ingress_bytes=389021696 link_bytes=389021696 token_bytes=558072176 weight_movement_bytes=110100480 total_nvlink_bytes=668172656 weight_to_token_ratio=0.197287 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.765727124e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277731328 dispatch_meta_bytes=228600 combine_payload_bytes=277731328 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=277959928 ingress_bytes=718133248 link_bytes=718133248 token_bytes=555691256 weight_movement_bytes=440401920 total_nvlink_bytes=996093176 weight_to_token_ratio=0.792530 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.248951831e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=280956928 dispatch_meta_bytes=229920 combine_payload_bytes=280956928 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281186848 ingress_bytes=391057408 link_bytes=391057408 token_bytes=562143776 weight_movement_bytes=110100480 total_nvlink_bytes=672244256 weight_to_token_ratio=0.195858 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.801131504e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279967744 dispatch_meta_bytes=230088 combine_payload_bytes=279967744 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280197832 ingress_bytes=390068224 link_bytes=390068224 token_bytes=560165576 weight_movement_bytes=110100480 total_nvlink_bytes=670266056 weight_to_token_ratio=0.196550 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.783927967e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44402 payload_rows_remote=38738 dup_rows=21134 dup_rows_remote=18476 dedup_ratio=0.322479 router_tokens_per_expert=[510, 490, 501, 526, 546, 524, 524, 538, 525, 520, 539, 495, 559, 533, 506, 484, 535, 526, 510, 500, 520, 513, 510, 492, 509, 491, 513, 516, 486, 498, 543, 495, 492, 507, 515, 534, 521, 472, 544, 510, 486, 496, 528, 489, 503, 515, 516, 515, 526, 512, 481, 531, 530, 490, 506, 515, 514, 515, 497, 494, 506, 524, 534, 473] payload_rows_per_dst=[5542, 5664, 5588, 5486, 5534, 5502, 5580, 5506] dup_rows_per_dst=[2776, 2658, 2624, 2616, 2656, 2594, 2602, 2608] combine_rows_per_src=[5542, 5664, 5588, 5486, 5534, 5502, 5580, 5506] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44682 payload_rows_remote=39046 dup_rows=20854 dup_rows_remote=18092 dedup_ratio=0.318207 router_tokens_per_expert=[542, 569, 511, 460, 486, 534, 489, 534, 545, 551, 538, 451, 512, 526, 510, 473, 522, 513, 524, 505, 549, 520, 532, 534, 498, 513, 541, 451, 527, 526, 532, 504, 512, 544, 500, 521, 515, 523, 504, 503, 492, 499, 547, 471, 504, 470, 499, 530, 468, 477, 509, 533, 513, 483, 530, 485, 506, 499, 516, 518, 528, 519, 516, 512] payload_rows_per_dst=[5644, 5552, 5636, 5580, 5622, 5486, 5580, 5582] dup_rows_per_dst=[2606, 2660, 2762, 2604, 2622, 2538, 2416, 2646] combine_rows_per_src=[5644, 5552, 5636, 5580, 5622, 5486, 5580, 5582] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44270 payload_rows_remote=38584 dup_rows=21266 dup_rows_remote=18468 dedup_ratio=0.324493 router_tokens_per_expert=[493, 535, 516, 477, 492, 503, 505, 492, 520, 502, 534, 555, 520, 526, 525, 537, 508, 501, 459, 486, 545, 497, 503, 505, 529, 517, 522, 524, 508, 470, 524, 528, 525, 509, 519, 483, 497, 484, 523, 483, 538, 550, 525, 509, 500, 490, 576, 507, 545, 497, 527, 533, 525, 535, 498, 531, 510, 503, 470, 509, 506, 491, 491, 521] payload_rows_per_dst=[5456, 5348, 5402, 5450, 5656, 5622, 5650, 5686] dup_rows_per_dst=[2456, 2400, 2456, 2496, 2798, 2970, 2892, 2798] combine_rows_per_src=[5456, 5348, 5402, 5450, 5656, 5622, 5650, 5686] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44566 payload_rows_remote=39030 dup_rows=20970 dup_rows_remote=18396 dedup_ratio=0.319977 router_tokens_per_expert=[522, 518, 517, 487, 478, 482, 522, 513, 527, 519, 511, 512, 537, 523, 506, 528, 548, 531, 528, 503, 519, 494, 518, 515, 518, 510, 506, 546, 529, 552, 498, 519, 479, 517, 519, 499, 538, 494, 504, 505, 505, 508, 458, 483, 493, 504, 505, 504, 500, 530, 531, 542, 510, 470, 477, 567, 480, 491, 522, 501, 554, 531, 506, 505] payload_rows_per_dst=[5504, 5636, 5662, 5598, 5536, 5470, 5586, 5574] dup_rows_per_dst=[2574, 2690, 2650, 2758, 2574, 2450, 2668, 2606] combine_rows_per_src=[5504, 5636, 5662, 5598, 5536, 5470, 5586, 5574] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44420 payload_rows_remote=38748 dup_rows=21116 dup_rows_remote=18454 dedup_ratio=0.322205 router_tokens_per_expert=[510, 536, 485, 542, 536, 508, 499, 525, 483, 535, 540, 545, 519, 531, 516, 520, 494, 528, 478, 574, 489, 543, 520, 504, 499, 506, 492, 486, 521, 552, 506, 512, 524, 533, 484, 491, 516, 499, 491, 541, 517, 487, 529, 470, 496, 492, 489, 521, 527, 504, 525, 505, 514, 500, 526, 508, 501, 471, 531, 536, 467, 514, 496, 529] payload_rows_per_dst=[5672, 5578, 5596, 5514, 5536, 5468, 5532, 5524] dup_rows_per_dst=[2662, 2800, 2664, 2634, 2622, 2534, 2656, 2544] combine_rows_per_src=[5672, 5578, 5596, 5514, 5536, 5468, 5532, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278863872 dispatch_meta_bytes=229648 combine_payload_bytes=278863872 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279093520 ingress_bytes=388964352 link_bytes=388964352 token_bytes=557957392 weight_movement_bytes=110100480 total_nvlink_bytes=668057872 weight_to_token_ratio=0.197328 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.764729818e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279867392 dispatch_meta_bytes=229928 combine_payload_bytes=279867392 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280097320 ingress_bytes=500068352 link_bytes=500068352 token_bytes=559964712 weight_movement_bytes=220200960 total_nvlink_bytes=780165672 weight_to_token_ratio=0.393241 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.697011112e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279394304 dispatch_meta_bytes=230384 combine_payload_bytes=279394304 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=279624688 ingress_bytes=389494784 link_bytes=389494784 token_bytes=559018992 weight_movement_bytes=110100480 total_nvlink_bytes=669119472 weight_to_token_ratio=0.196953 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773954903e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44460 payload_rows_remote=38912 dup_rows=21076 dup_rows_remote=18408 dedup_ratio=0.321594 router_tokens_per_expert=[503, 469, 519, 513, 517, 479, 561, 512, 523, 520, 500, 502, 489, 485, 535, 554, 471, 542, 485, 530, 478, 498, 527, 526, 492, 519, 522, 509, 509, 521, 502, 548, 517, 496, 517, 519, 501, 527, 498, 531, 544, 505, 512, 518, 522, 490, 464, 525, 455, 526, 509, 535, 526, 528, 480, 543, 546, 501, 503, 517, 546, 518, 498, 491] payload_rows_per_dst=[5552, 5548, 5524, 5548, 5532, 5498, 5612, 5646] dup_rows_per_dst=[2594, 2668, 2590, 2696, 2680, 2662, 2592, 2594] combine_rows_per_src=[5552, 5548, 5524, 5548, 5532, 5498, 5612, 5646] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=281171968 dispatch_meta_bytes=229920 combine_payload_bytes=281171968 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281401888 ingress_bytes=391272448 link_bytes=391272448 token_bytes=562573856 weight_movement_bytes=110100480 total_nvlink_bytes=672674336 weight_to_token_ratio=0.195708 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.804871403e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44576 payload_rows_remote=39040 dup_rows=20960 dup_rows_remote=18376 dedup_ratio=0.319824 router_tokens_per_expert=[520, 516, 512, 485, 474, 480, 521, 510, 528, 519, 507, 510, 535, 521, 505, 524, 547, 528, 526, 499, 519, 493, 518, 515, 517, 509, 507, 544, 531, 553, 499, 518, 475, 517, 518, 502, 541, 496, 504, 507, 507, 508, 458, 483, 496, 504, 505, 507, 502, 529, 532, 545, 515, 474, 480, 568, 482, 494, 523, 502, 554, 532, 510, 508] payload_rows_per_dst=[5486, 5630, 5652, 5598, 5536, 5478, 5608, 5588] dup_rows_per_dst=[2550, 2668, 2638, 2758, 2584, 2458, 2682, 2622] combine_rows_per_src=[5486, 5630, 5652, 5598, 5536, 5478, 5608, 5588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277745664 dispatch_meta_bytes=228808 combine_payload_bytes=277745664 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=277974472 ingress_bytes=718147584 link_bytes=718147584 token_bytes=555720136 weight_movement_bytes=440401920 total_nvlink_bytes=996122056 weight_to_token_ratio=0.792489 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.248976764e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44684 payload_rows_remote=39046 dup_rows=20852 dup_rows_remote=18108 dedup_ratio=0.318176 router_tokens_per_expert=[538, 564, 511, 458, 486, 532, 487, 532, 544, 549, 539, 448, 511, 523, 510, 472, 520, 512, 521, 504, 548, 520, 532, 534, 498, 512, 540, 451, 526, 527, 530, 503, 513, 544, 499, 524, 516, 521, 503, 504, 493, 500, 545, 470, 505, 471, 502, 530, 470, 480, 513, 533, 513, 486, 533, 486, 508, 500, 517, 518, 533, 523, 519, 514] payload_rows_per_dst=[5632, 5538, 5638, 5582, 5614, 5492, 5590, 5598] dup_rows_per_dst=[2584, 2654, 2744, 2592, 2634, 2540, 2438, 2666] combine_rows_per_src=[5632, 5538, 5638, 5582, 5614, 5492, 5590, 5598] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278921216 dispatch_meta_bytes=229280 combine_payload_bytes=278921216 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279150496 ingress_bytes=609222656 link_bytes=609222656 token_bytes=558071712 weight_movement_bytes=330301440 total_nvlink_bytes=888373152 weight_to_token_ratio=0.591862 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059538399e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279838720 dispatch_meta_bytes=229664 combine_payload_bytes=279838720 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280068384 ingress_bytes=389939200 link_bytes=389939200 token_bytes=559907104 weight_movement_bytes=110100480 total_nvlink_bytes=670007584 weight_to_token_ratio=0.196641 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.781684028e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44514 payload_rows_remote=39124 dup_rows=21022 dup_rows_remote=18626 dedup_ratio=0.320770 router_tokens_per_expert=[485, 513, 532, 500, 531, 531, 572, 525, 486, 543, 523, 531, 534, 535, 489, 532, 491, 551, 515, 539, 538, 540, 470, 496, 519, 507, 510, 570, 532, 509, 509, 557, 529, 507, 513, 492, 537, 522, 456, 486, 452, 485, 481, 491, 487, 465, 525, 507, 471, 506, 487, 521, 531, 526, 534, 532, 514, 520, 497, 496, 502, 509, 489, 483] payload_rows_per_dst=[5646, 5648, 5600, 5652, 5510, 5390, 5558, 5510] dup_rows_per_dst=[2732, 2698, 2680, 2774, 2574, 2396, 2658, 2510] combine_rows_per_src=[5646, 5648, 5600, 5652, 5510, 5390, 5558, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44656 payload_rows_remote=39124 dup_rows=20880 dup_rows_remote=18342 dedup_ratio=0.318604 router_tokens_per_expert=[540, 495, 526, 547, 489, 499, 491, 511, 498, 489, 507, 514, 461, 485, 489, 540, 495, 497, 509, 516, 534, 524, 501, 536, 518, 544, 478, 524, 487, 501, 535, 495, 529, 502, 508, 497, 533, 509, 511, 525, 502, 519, 535, 546, 493, 547, 540, 512, 514, 517, 480, 500, 503, 489, 526, 506, 526, 527, 545, 493, 561, 484, 517, 497] payload_rows_per_dst=[5630, 5480, 5576, 5556, 5660, 5640, 5532, 5582] dup_rows_per_dst=[2566, 2486, 2648, 2608, 2568, 2748, 2538, 2718] combine_rows_per_src=[5630, 5480, 5576, 5556, 5660, 5640, 5532, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44370 payload_rows_remote=38780 dup_rows=21166 dup_rows_remote=18534 dedup_ratio=0.322968 router_tokens_per_expert=[492, 531, 514, 474, 491, 501, 503, 488, 519, 498, 531, 553, 519, 525, 524, 536, 507, 500, 459, 485, 542, 497, 503, 505, 528, 516, 521, 523, 506, 472, 524, 529, 526, 509, 523, 483, 497, 486, 522, 481, 539, 548, 525, 509, 499, 491, 580, 508, 546, 500, 528, 534, 526, 540, 500, 533, 511, 507, 473, 511, 508, 491, 494, 524] payload_rows_per_dst=[5510, 5414, 5452, 5596, 5646, 5568, 5594, 5590] dup_rows_per_dst=[2478, 2494, 2504, 2654, 2726, 2830, 2848, 2632] combine_rows_per_src=[5510, 5414, 5452, 5596, 5646, 5568, 5594, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279881728 dispatch_meta_bytes=228616 combine_payload_bytes=279881728 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=280110344 ingress_bytes=500082688 link_bytes=500082688 token_bytes=559992072 weight_movement_bytes=220200960 total_nvlink_bytes=780193032 weight_to_token_ratio=0.393222 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.697260439e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44552 payload_rows_remote=39028 dup_rows=20984 dup_rows_remote=18392 dedup_ratio=0.320190 router_tokens_per_expert=[479, 506, 507, 535, 486, 500, 505, 509, 556, 524, 504, 504, 466, 554, 544, 523, 489, 530, 509, 493, 539, 492, 514, 476, 492, 526, 524, 502, 515, 495, 534, 470, 523, 500, 500, 503, 511, 504, 505, 526, 497, 556, 532, 505, 518, 492, 553, 502, 486, 488, 518, 551, 497, 517, 518, 511, 530, 508, 543, 481, 536, 511, 541, 503] payload_rows_per_dst=[5446, 5696, 5540, 5524, 5582, 5620, 5520, 5624] dup_rows_per_dst=[2608, 2654, 2544, 2592, 2562, 2690, 2652, 2682] combine_rows_per_src=[5446, 5696, 5540, 5524, 5582, 5620, 5520, 5624] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44678 payload_rows_remote=39014 dup_rows=20858 dup_rows_remote=18000 dedup_ratio=0.318268 router_tokens_per_expert=[514, 522, 561, 498, 515, 487, 528, 527, 478, 536, 499, 517, 478, 527, 530, 534, 470, 524, 492, 536, 536, 530, 508, 507, 515, 511, 501, 510, 552, 496, 494, 464, 524, 514, 508, 526, 520, 491, 491, 490, 502, 551, 541, 539, 520, 515, 526, 510, 509, 525, 521, 502, 529, 534, 493, 503, 518, 461, 473, 532, 485, 487, 525, 506] payload_rows_per_dst=[5664, 5586, 5606, 5392, 5546, 5756, 5634, 5494] dup_rows_per_dst=[2858, 2674, 2600, 2414, 2582, 2770, 2480, 2480] combine_rows_per_src=[5664, 5586, 5606, 5392, 5546, 5756, 5634, 5494] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44512 payload_rows_remote=39126 dup_rows=21024 dup_rows_remote=18632 dedup_ratio=0.320801 router_tokens_per_expert=[488, 515, 536, 504, 534, 533, 575, 528, 488, 546, 523, 538, 536, 538, 495, 532, 490, 550, 518, 537, 537, 540, 471, 495, 516, 507, 509, 570, 532, 510, 511, 559, 530, 507, 512, 488, 536, 520, 454, 488, 452, 485, 481, 490, 486, 464, 524, 507, 467, 506, 486, 518, 528, 523, 532, 532, 511, 518, 495, 494, 502, 506, 487, 478] payload_rows_per_dst=[5664, 5664, 5596, 5648, 5504, 5386, 5554, 5496] dup_rows_per_dst=[2762, 2728, 2680, 2780, 2566, 2392, 2630, 2486] combine_rows_per_src=[5664, 5664, 5596, 5648, 5504, 5386, 5554, 5496] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277673984 dispatch_meta_bytes=228856 combine_payload_bytes=277673984 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=277902840 ingress_bytes=387774464 link_bytes=387774464 token_bytes=555576824 weight_movement_bytes=110100480 total_nvlink_bytes=665677304 weight_to_token_ratio=0.198173 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.744035708e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279881728 dispatch_meta_bytes=228552 combine_payload_bytes=279881728 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280110280 ingress_bytes=500082688 link_bytes=500082688 token_bytes=559992008 weight_movement_bytes=220200960 total_nvlink_bytes=780192968 weight_to_token_ratio=0.393222 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.697260439e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=276570112 dispatch_meta_bytes=228208 combine_payload_bytes=276570112 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276798320 ingress_bytes=386670592 link_bytes=386670592 token_bytes=553368432 weight_movement_bytes=110100480 total_nvlink_bytes=663468912 weight_to_token_ratio=0.198964 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.724837559e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44662 payload_rows_remote=39052 dup_rows=20874 dup_rows_remote=18116 dedup_ratio=0.318512 router_tokens_per_expert=[516, 529, 566, 503, 518, 493, 532, 527, 484, 537, 501, 516, 478, 530, 529, 536, 469, 524, 496, 537, 536, 529, 509, 507, 515, 511, 500, 512, 552, 496, 495, 462, 522, 514, 505, 523, 520, 489, 490, 491, 502, 548, 543, 541, 519, 514, 526, 511, 508, 522, 514, 498, 527, 533, 491, 503, 517, 461, 473, 527, 483, 485, 519, 504] payload_rows_per_dst=[5610, 5576, 5600, 5518, 5536, 5706, 5640, 5476] dup_rows_per_dst=[2758, 2646, 2614, 2568, 2572, 2702, 2552, 2462] combine_rows_per_src=[5610, 5576, 5600, 5518, 5536, 5706, 5640, 5476] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44518 payload_rows_remote=38992 dup_rows=21018 dup_rows_remote=18416 dedup_ratio=0.320709 router_tokens_per_expert=[482, 507, 509, 538, 488, 505, 505, 513, 557, 524, 506, 506, 466, 556, 545, 526, 492, 531, 508, 496, 542, 490, 513, 477, 493, 527, 524, 503, 517, 494, 535, 471, 522, 501, 499, 504, 510, 505, 506, 526, 497, 553, 531, 503, 517, 492, 553, 503, 482, 487, 516, 552, 494, 517, 516, 509, 529, 505, 540, 478, 531, 508, 538, 498] payload_rows_per_dst=[5458, 5698, 5542, 5526, 5580, 5608, 5502, 5604] dup_rows_per_dst=[2636, 2674, 2556, 2602, 2566, 2690, 2644, 2650] combine_rows_per_src=[5458, 5698, 5542, 5526, 5580, 5608, 5502, 5604] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44672 payload_rows_remote=39146 dup_rows=20864 dup_rows_remote=18342 dedup_ratio=0.318359 router_tokens_per_expert=[541, 498, 528, 547, 490, 504, 494, 512, 504, 492, 507, 515, 462, 485, 491, 543, 494, 501, 512, 515, 536, 524, 501, 540, 519, 543, 478, 523, 486, 503, 532, 500, 529, 502, 506, 498, 531, 509, 507, 524, 501, 518, 531, 544, 492, 549, 538, 511, 512, 517, 478, 497, 503, 487, 525, 505, 524, 526, 545, 491, 555, 484, 515, 494] payload_rows_per_dst=[5642, 5492, 5590, 5560, 5650, 5638, 5526, 5574] dup_rows_per_dst=[2586, 2506, 2656, 2608, 2562, 2730, 2522, 2694] combine_rows_per_src=[5642, 5492, 5590, 5560, 5650, 5638, 5526, 5574] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279767040 dispatch_meta_bytes=229704 combine_payload_bytes=279767040 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279996744 ingress_bytes=389867520 link_bytes=389867520 token_bytes=559763784 weight_movement_bytes=110100480 total_nvlink_bytes=669864264 weight_to_token_ratio=0.196691 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.780437395e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280455168 dispatch_meta_bytes=231032 combine_payload_bytes=280455168 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280686200 ingress_bytes=390555648 link_bytes=390555648 token_bytes=561141368 weight_movement_bytes=110100480 total_nvlink_bytes=671241848 weight_to_token_ratio=0.196208 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.792405072e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279924736 dispatch_meta_bytes=228672 combine_payload_bytes=279924736 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=280153408 ingress_bytes=500125696 link_bytes=500125696 token_bytes=560078144 weight_movement_bytes=220200960 total_nvlink_bytes=780279104 weight_to_token_ratio=0.393161 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.698008419e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279494656 dispatch_meta_bytes=229632 combine_payload_bytes=279494656 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279724288 ingress_bytes=499695616 link_bytes=499695616 token_bytes=559218944 weight_movement_bytes=220200960 total_nvlink_bytes=779419904 weight_to_token_ratio=0.393765 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.690528620e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280598528 dispatch_meta_bytes=229952 combine_payload_bytes=280598528 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280828480 ingress_bytes=390699008 link_bytes=390699008 token_bytes=561427008 weight_movement_bytes=110100480 total_nvlink_bytes=671527488 weight_to_token_ratio=0.196108 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.794898339e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44458 payload_rows_remote=38896 dup_rows=21078 dup_rows_remote=18374 dedup_ratio=0.321625 router_tokens_per_expert=[506, 471, 522, 514, 519, 479, 564, 515, 525, 522, 504, 504, 494, 489, 535, 560, 474, 546, 486, 530, 478, 498, 528, 527, 489, 521, 522, 506, 509, 522, 502, 549, 517, 498, 514, 517, 500, 522, 498, 530, 543, 508, 511, 519, 521, 490, 460, 522, 456, 524, 507, 535, 526, 521, 479, 539, 545, 501, 498, 516, 541, 516, 494, 490] payload_rows_per_dst=[5570, 5562, 5544, 5552, 5516, 5490, 5602, 5622] dup_rows_per_dst=[2610, 2704, 2590, 2688, 2676, 2658, 2572, 2580] combine_rows_per_src=[5570, 5562, 5544, 5552, 5516, 5490, 5602, 5622] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44278 payload_rows_remote=38772 dup_rows=21258 dup_rows_remote=18594 dedup_ratio=0.324371 router_tokens_per_expert=[498, 481, 507, 536, 489, 516, 484, 491, 539, 507, 525, 514, 500, 518, 513, 496, 480, 497, 511, 519, 521, 520, 503, 534, 497, 567, 501, 541, 513, 483, 516, 502, 517, 517, 516, 537, 560, 538, 484, 518, 514, 485, 513, 504, 485, 490, 548, 503, 527, 525, 527, 472, 522, 528, 508, 524, 564, 514, 479, 520, 525, 487, 498, 500] payload_rows_per_dst=[5410, 5518, 5506, 5594, 5622, 5510, 5574, 5544] dup_rows_per_dst=[2594, 2706, 2664, 2646, 2752, 2574, 2692, 2630] combine_rows_per_src=[5410, 5518, 5506, 5594, 5622, 5510, 5574, 5544] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44740 payload_rows_remote=39150 dup_rows=20796 dup_rows_remote=18152 dedup_ratio=0.317322 router_tokens_per_expert=[496, 506, 487, 521, 499, 506, 521, 516, 474, 514, 501, 519, 501, 542, 518, 483, 532, 504, 479, 551, 515, 463, 505, 522, 523, 515, 541, 561, 494, 541, 529, 512, 510, 476, 494, 528, 506, 537, 494, 480, 502, 525, 493, 514, 500, 516, 546, 476, 540, 511, 508, 525, 496, 532, 525, 538, 521, 533, 516, 502, 529, 475, 540, 489] payload_rows_per_dst=[5616, 5498, 5542, 5720, 5500, 5612, 5662, 5590] dup_rows_per_dst=[2488, 2388, 2576, 2712, 2550, 2712, 2726, 2644] combine_rows_per_src=[5616, 5498, 5542, 5720, 5500, 5612, 5662, 5590] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44568 payload_rows_remote=39090 dup_rows=20968 dup_rows_remote=18336 dedup_ratio=0.319946 router_tokens_per_expert=[496, 547, 492, 524, 455, 518, 528, 492, 485, 537, 516, 540, 534, 498, 506, 515, 512, 501, 546, 530, 495, 536, 502, 540, 459, 506, 493, 475, 528, 530, 497, 509, 515, 516, 525, 476, 521, 527, 489, 486, 487, 493, 523, 556, 553, 523, 483, 513, 492, 501, 550, 496, 498, 536, 491, 523, 547, 527, 501, 530, 534, 516, 508, 490] payload_rows_per_dst=[5508, 5582, 5642, 5434, 5478, 5666, 5596, 5662] dup_rows_per_dst=[2596, 2680, 2682, 2560, 2632, 2596, 2578, 2644] combine_rows_per_src=[5508, 5582, 5642, 5434, 5478, 5666, 5596, 5662] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44526 payload_rows_remote=38970 dup_rows=21010 dup_rows_remote=18382 dedup_ratio=0.320587 router_tokens_per_expert=[495, 549, 541, 516, 481, 533, 524, 534, 486, 485, 520, 528, 479, 493, 534, 504, 496, 502, 554, 533, 523, 518, 498, 513, 496, 537, 475, 508, 524, 488, 547, 493, 498, 503, 513, 509, 526, 485, 528, 536, 506, 535, 483, 494, 505, 499, 530, 540, 519, 510, 488, 517, 494, 513, 510, 476, 525, 525, 520, 519, 506, 504, 541, 504] payload_rows_per_dst=[5656, 5518, 5570, 5534, 5594, 5556, 5544, 5554] dup_rows_per_dst=[2690, 2540, 2704, 2602, 2602, 2628, 2510, 2734] combine_rows_per_src=[5656, 5518, 5570, 5534, 5594, 5556, 5544, 5554] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278806528 dispatch_meta_bytes=229080 combine_payload_bytes=278806528 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=279035608 ingress_bytes=719208448 link_bytes=719208448 token_bytes=557842136 weight_movement_bytes=440401920 total_nvlink_bytes=998244056 weight_to_token_ratio=0.789474 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.250821781e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280627200 dispatch_meta_bytes=229208 combine_payload_bytes=280627200 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280856408 ingress_bytes=500828160 link_bytes=500828160 token_bytes=561483608 weight_movement_bytes=220200960 total_nvlink_bytes=781684568 weight_to_token_ratio=0.392177 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.710225423e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277917696 dispatch_meta_bytes=229464 combine_payload_bytes=277917696 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=278147160 ingress_bytes=498118656 link_bytes=498118656 token_bytes=556064856 weight_movement_bytes=220200960 total_nvlink_bytes=776265816 weight_to_token_ratio=0.395999 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.663102692e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44488 payload_rows_remote=38788 dup_rows=21048 dup_rows_remote=18212 dedup_ratio=0.321167 router_tokens_per_expert=[472, 508, 511, 545, 525, 499, 485, 542, 488, 512, 558, 519, 498, 544, 505, 537, 512, 510, 492, 506, 465, 507, 485, 523, 478, 479, 497, 542, 485, 481, 530, 492, 552, 519, 532, 532, 477, 521, 539, 484, 512, 525, 494, 504, 492, 531, 530, 495, 533, 504, 542, 523, 486, 497, 545, 515, 530, 509, 543, 545, 514, 512, 489, 510] payload_rows_per_dst=[5700, 5604, 5454, 5592, 5506, 5522, 5526, 5584] dup_rows_per_dst=[2836, 2724, 2540, 2554, 2586, 2644, 2622, 2542] combine_rows_per_src=[5700, 5604, 5454, 5592, 5506, 5522, 5526, 5584] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280197120 dispatch_meta_bytes=229704 combine_payload_bytes=280197120 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280426824 ingress_bytes=500398080 link_bytes=500398080 token_bytes=560623944 weight_movement_bytes=220200960 total_nvlink_bytes=780824904 weight_to_token_ratio=0.392778 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.702745625e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44530 payload_rows_remote=38976 dup_rows=21006 dup_rows_remote=18346 dedup_ratio=0.320526 router_tokens_per_expert=[528, 504, 496, 481, 499, 510, 470, 512, 530, 545, 502, 515, 535, 529, 510, 481, 469, 499, 542, 537, 535, 498, 488, 533, 513, 515, 491, 561, 491, 518, 508, 528, 545, 491, 477, 529, 514, 556, 508, 517, 494, 523, 495, 526, 523, 505, 484, 500, 529, 495, 520, 523, 496, 502, 517, 525, 499, 550, 510, 530, 484, 508, 524, 496] payload_rows_per_dst=[5456, 5602, 5592, 5552, 5538, 5524, 5554, 5712] dup_rows_per_dst=[2544, 2692, 2610, 2698, 2736, 2576, 2660, 2490] combine_rows_per_src=[5456, 5602, 5592, 5552, 5538, 5524, 5554, 5712] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44660 payload_rows_remote=39188 dup_rows=20876 dup_rows_remote=18256 dedup_ratio=0.318542 router_tokens_per_expert=[519, 515, 486, 518, 528, 536, 469, 532, 500, 479, 536, 542, 504, 539, 532, 494, 513, 490, 523, 519, 523, 515, 543, 543, 498, 511, 499, 504, 497, 514, 494, 529, 504, 525, 524, 499, 500, 533, 511, 529, 509, 546, 501, 530, 501, 520, 492, 496, 536, 503, 532, 485, 509, 496, 515, 498, 503, 504, 512, 489, 506, 524, 467, 525] payload_rows_per_dst=[5552, 5606, 5618, 5472, 5666, 5598, 5610, 5538] dup_rows_per_dst=[2654, 2646, 2720, 2620, 2584, 2592, 2538, 2522] combine_rows_per_src=[5552, 5606, 5618, 5472, 5666, 5598, 5610, 5538] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279336960 dispatch_meta_bytes=229408 combine_payload_bytes=279336960 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279566368 ingress_bytes=389437440 link_bytes=389437440 token_bytes=558903328 weight_movement_bytes=110100480 total_nvlink_bytes=669003808 weight_to_token_ratio=0.196994 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.772957596e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278032384 dispatch_meta_bytes=228000 combine_payload_bytes=278032384 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278260384 ingress_bytes=388132864 link_bytes=388132864 token_bytes=556292768 weight_movement_bytes=110100480 total_nvlink_bytes=666393248 weight_to_token_ratio=0.197918 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.750268874e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279379968 dispatch_meta_bytes=229288 combine_payload_bytes=279379968 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279609256 ingress_bytes=499580928 link_bytes=499580928 token_bytes=558989224 weight_movement_bytes=220200960 total_nvlink_bytes=779190184 weight_to_token_ratio=0.393927 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.688534007e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280899584 dispatch_meta_bytes=229776 combine_payload_bytes=280899584 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281129360 ingress_bytes=391000064 link_bytes=391000064 token_bytes=562028944 weight_movement_bytes=110100480 total_nvlink_bytes=672129424 weight_to_token_ratio=0.195898 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.800134198e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44790 payload_rows_remote=39176 dup_rows=20746 dup_rows_remote=18128 dedup_ratio=0.316559 router_tokens_per_expert=[503, 516, 504, 529, 498, 501, 554, 513, 501, 533, 525, 496, 566, 489, 505, 501, 520, 525, 527, 532, 457, 501, 515, 481, 498, 524, 527, 554, 501, 531, 484, 504, 523, 490, 500, 494, 514, 524, 519, 531, 490, 552, 508, 500, 528, 501, 510, 524, 541, 499, 526, 493, 479, 542, 493, 499, 522, 507, 558, 481, 514, 478, 496, 517] payload_rows_per_dst=[5618, 5614, 5558, 5620, 5546, 5648, 5606, 5580] dup_rows_per_dst=[2618, 2618, 2558, 2626, 2644, 2578, 2538, 2566] combine_rows_per_src=[5618, 5614, 5558, 5620, 5546, 5648, 5606, 5580] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44398 payload_rows_remote=38650 dup_rows=21138 dup_rows_remote=18490 dedup_ratio=0.322540 router_tokens_per_expert=[556, 517, 535, 534, 512, 531, 509, 494, 505, 555, 517, 508, 534, 530, 538, 478, 482, 510, 559, 555, 517, 522, 535, 530, 530, 503, 488, 493, 534, 499, 488, 548, 503, 532, 498, 467, 504, 513, 517, 496, 524, 492, 508, 494, 524, 521, 501, 477, 491, 503, 519, 497, 514, 477, 493, 490, 546, 483, 473, 525, 531, 502, 521, 486] payload_rows_per_dst=[5282, 5438, 5462, 5558, 5520, 5728, 5662, 5748] dup_rows_per_dst=[2188, 2560, 2572, 2620, 2624, 3032, 2894, 2648] combine_rows_per_src=[5282, 5438, 5462, 5558, 5520, 5728, 5662, 5748] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44380 payload_rows_remote=38844 dup_rows=21156 dup_rows_remote=18556 dedup_ratio=0.322815 router_tokens_per_expert=[520, 472, 545, 540, 534, 502, 490, 482, 487, 513, 500, 512, 518, 486, 497, 512, 497, 485, 516, 506, 529, 501, 520, 514, 520, 559, 503, 522, 483, 501, 537, 516, 496, 535, 489, 499, 530, 548, 525, 513, 520, 545, 493, 514, 443, 524, 517, 534, 516, 502, 523, 505, 502, 542, 504, 527, 528, 495, 543, 477, 503, 481, 541, 535] payload_rows_per_dst=[5570, 5516, 5536, 5542, 5560, 5542, 5558, 5556] dup_rows_per_dst=[2600, 2534, 2600, 2740, 2710, 2638, 2684, 2650] combine_rows_per_src=[5570, 5516, 5536, 5542, 5560, 5542, 5558, 5556] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44190 payload_rows_remote=38612 dup_rows=21346 dup_rows_remote=18664 dedup_ratio=0.325714 router_tokens_per_expert=[548, 504, 502, 492, 496, 516, 521, 483, 499, 516, 525, 511, 555, 506, 520, 506, 545, 546, 488, 492, 512, 471, 553, 500, 504, 529, 522, 523, 509, 539, 489, 542, 529, 557, 527, 480, 516, 472, 519, 530, 513, 486, 484, 540, 483, 492, 509, 537, 499, 480, 507, 494, 533, 516, 493, 523, 481, 493, 508, 538, 512, 523, 529, 501] payload_rows_per_dst=[5462, 5532, 5532, 5594, 5578, 5478, 5504, 5510] dup_rows_per_dst=[2662, 2744, 2682, 2720, 2682, 2610, 2586, 2660] combine_rows_per_src=[5462, 5532, 5532, 5594, 5578, 5478, 5504, 5510] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44518 payload_rows_remote=39032 dup_rows=21018 dup_rows_remote=18484 dedup_ratio=0.320709 router_tokens_per_expert=[519, 531, 516, 551, 513, 523, 507, 499, 546, 496, 491, 505, 477, 533, 504, 506, 485, 526, 566, 512, 519, 489, 530, 475, 492, 498, 516, 548, 502, 535, 544, 526, 489, 511, 543, 529, 515, 496, 515, 501, 506, 503, 508, 484, 489, 507, 493, 520, 478, 502, 529, 509, 545, 531, 483, 524, 486, 493, 490, 540, 537, 527, 487, 518] payload_rows_per_dst=[5630, 5504, 5572, 5604, 5584, 5486, 5608, 5530] dup_rows_per_dst=[2688, 2612, 2632, 2718, 2614, 2534, 2594, 2626] combine_rows_per_src=[5630, 5504, 5572, 5604, 5584, 5486, 5608, 5530] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280813568 dispatch_meta_bytes=229216 combine_payload_bytes=280813568 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=281042784 ingress_bytes=611115008 link_bytes=611115008 token_bytes=561856352 weight_movement_bytes=330301440 total_nvlink_bytes=892157792 weight_to_token_ratio=0.587875 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.062829510e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280440832 dispatch_meta_bytes=231000 combine_payload_bytes=280440832 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280671832 ingress_bytes=390541312 link_bytes=390541312 token_bytes=561112664 weight_movement_bytes=110100480 total_nvlink_bytes=671213144 weight_to_token_ratio=0.196218 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.792155746e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280440832 dispatch_meta_bytes=229864 combine_payload_bytes=280440832 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280670696 ingress_bytes=500641792 link_bytes=500641792 token_bytes=561111528 weight_movement_bytes=220200960 total_nvlink_bytes=781312488 weight_to_token_ratio=0.392437 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.706984177e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277975040 dispatch_meta_bytes=229256 combine_payload_bytes=277975040 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278204296 ingress_bytes=388075520 link_bytes=388075520 token_bytes=556179336 weight_movement_bytes=110100480 total_nvlink_bytes=666279816 weight_to_token_ratio=0.197959 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.749271567e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44780 payload_rows_remote=39168 dup_rows=20756 dup_rows_remote=18142 dedup_ratio=0.316711 router_tokens_per_expert=[501, 510, 501, 526, 498, 498, 553, 511, 504, 533, 523, 494, 565, 490, 504, 500, 519, 523, 526, 532, 458, 499, 516, 481, 496, 523, 523, 554, 500, 532, 482, 503, 522, 491, 500, 494, 514, 525, 521, 527, 492, 552, 509, 499, 527, 503, 509, 524, 542, 502, 527, 495, 480, 545, 496, 500, 524, 510, 560, 482, 516, 481, 500, 521] payload_rows_per_dst=[5610, 5612, 5554, 5606, 5544, 5646, 5614, 5594] dup_rows_per_dst=[2586, 2614, 2554, 2620, 2644, 2584, 2560, 2594] combine_rows_per_src=[5610, 5612, 5554, 5606, 5544, 5646, 5614, 5594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44560 payload_rows_remote=39074 dup_rows=20976 dup_rows_remote=18336 dedup_ratio=0.320068 router_tokens_per_expert=[493, 542, 491, 522, 451, 514, 526, 493, 482, 535, 516, 535, 531, 498, 506, 514, 511, 497, 547, 528, 493, 532, 502, 542, 458, 507, 492, 476, 527, 529, 495, 512, 518, 516, 527, 478, 520, 528, 488, 488, 488, 494, 522, 555, 555, 523, 483, 514, 492, 503, 552, 496, 499, 537, 494, 527, 549, 532, 501, 531, 539, 518, 510, 494] payload_rows_per_dst=[5500, 5566, 5622, 5434, 5486, 5658, 5610, 5684] dup_rows_per_dst=[2564, 2668, 2682, 2558, 2640, 2610, 2590, 2664] combine_rows_per_src=[5500, 5566, 5622, 5434, 5486, 5658, 5610, 5684] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279752704 dispatch_meta_bytes=229680 combine_payload_bytes=279752704 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279982384 ingress_bytes=610054144 link_bytes=610054144 token_bytes=559735088 weight_movement_bytes=330301440 total_nvlink_bytes=890036528 weight_to_token_ratio=0.590103 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060984493e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279652352 dispatch_meta_bytes=228056 combine_payload_bytes=279652352 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279880408 ingress_bytes=389752832 link_bytes=389752832 token_bytes=559532760 weight_movement_bytes=110100480 total_nvlink_bytes=669633240 weight_to_token_ratio=0.196772 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778442782e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44292 payload_rows_remote=38796 dup_rows=21244 dup_rows_remote=18590 dedup_ratio=0.324158 router_tokens_per_expert=[495, 479, 502, 535, 487, 513, 482, 488, 535, 503, 523, 513, 501, 515, 513, 494, 475, 498, 512, 518, 517, 519, 504, 532, 496, 568, 501, 541, 513, 484, 517, 504, 516, 516, 516, 537, 562, 536, 484, 519, 517, 486, 515, 503, 486, 490, 552, 502, 526, 527, 530, 474, 524, 529, 509, 525, 567, 516, 481, 526, 526, 490, 502, 502] payload_rows_per_dst=[5394, 5506, 5496, 5594, 5626, 5524, 5582, 5570] dup_rows_per_dst=[2568, 2688, 2654, 2654, 2746, 2578, 2706, 2650] combine_rows_per_src=[5394, 5506, 5496, 5594, 5626, 5524, 5582, 5570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280756224 dispatch_meta_bytes=229240 combine_payload_bytes=280756224 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=280985464 ingress_bytes=390856704 link_bytes=390856704 token_bytes=561741688 weight_movement_bytes=110100480 total_nvlink_bytes=671842168 weight_to_token_ratio=0.195998 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.797640931e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280082432 dispatch_meta_bytes=229640 combine_payload_bytes=280082432 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=280312072 ingress_bytes=720484352 link_bytes=720484352 token_bytes=560394504 weight_movement_bytes=440401920 total_nvlink_bytes=1000796424 weight_to_token_ratio=0.785878 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.253040787e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44514 payload_rows_remote=38958 dup_rows=21022 dup_rows_remote=18344 dedup_ratio=0.320770 router_tokens_per_expert=[524, 500, 491, 480, 498, 508, 469, 510, 525, 541, 505, 515, 532, 525, 509, 480, 467, 498, 542, 537, 534, 496, 488, 531, 511, 514, 492, 560, 492, 521, 507, 528, 546, 489, 480, 530, 515, 556, 512, 516, 494, 522, 497, 528, 526, 505, 483, 502, 529, 495, 522, 525, 497, 502, 519, 528, 500, 554, 512, 532, 487, 509, 526, 500] payload_rows_per_dst=[5434, 5588, 5584, 5560, 5534, 5532, 5556, 5726] dup_rows_per_dst=[2526, 2676, 2602, 2690, 2754, 2582, 2678, 2514] combine_rows_per_src=[5434, 5588, 5584, 5560, 5534, 5532, 5556, 5726] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44526 payload_rows_remote=38964 dup_rows=21010 dup_rows_remote=18382 dedup_ratio=0.320587 router_tokens_per_expert=[491, 546, 533, 512, 480, 531, 524, 534, 485, 484, 517, 526, 478, 490, 534, 504, 496, 501, 551, 532, 523, 517, 498, 514, 491, 536, 477, 509, 521, 489, 546, 494, 496, 503, 513, 509, 527, 484, 529, 535, 506, 536, 482, 493, 507, 500, 529, 542, 519, 513, 489, 519, 495, 516, 513, 478, 527, 530, 522, 521, 509, 511, 543, 508] payload_rows_per_dst=[5636, 5510, 5552, 5534, 5594, 5562, 5556, 5582] dup_rows_per_dst=[2666, 2526, 2712, 2592, 2598, 2628, 2528, 2760] combine_rows_per_src=[5636, 5510, 5552, 5534, 5594, 5562, 5556, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44708 payload_rows_remote=39050 dup_rows=20828 dup_rows_remote=18080 dedup_ratio=0.317810 router_tokens_per_expert=[494, 503, 486, 521, 497, 502, 519, 515, 473, 512, 503, 519, 501, 539, 517, 480, 529, 501, 480, 549, 514, 461, 507, 522, 520, 513, 538, 561, 493, 542, 528, 512, 512, 476, 494, 528, 508, 539, 495, 480, 503, 524, 494, 514, 499, 518, 548, 478, 541, 510, 509, 525, 497, 535, 528, 536, 521, 534, 519, 505, 532, 480, 542, 493] payload_rows_per_dst=[5600, 5574, 5548, 5712, 5390, 5556, 5670, 5658] dup_rows_per_dst=[2474, 2506, 2578, 2702, 2414, 2692, 2714, 2748] combine_rows_per_src=[5600, 5574, 5548, 5712, 5390, 5556, 5670, 5658] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278089728 dispatch_meta_bytes=229544 combine_payload_bytes=278089728 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278319272 ingress_bytes=388190208 link_bytes=388190208 token_bytes=556409000 weight_movement_bytes=110100480 total_nvlink_bytes=666509480 weight_to_token_ratio=0.197877 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.751266180e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44428 payload_rows_remote=38640 dup_rows=21108 dup_rows_remote=18084 dedup_ratio=0.322083 router_tokens_per_expert=[468, 506, 510, 541, 525, 497, 484, 538, 485, 508, 555, 519, 499, 542, 502, 535, 512, 508, 489, 506, 464, 507, 485, 523, 480, 475, 497, 542, 484, 482, 530, 490, 552, 516, 533, 529, 477, 522, 539, 483, 513, 525, 497, 504, 493, 533, 531, 494, 538, 505, 542, 525, 487, 499, 545, 515, 533, 511, 544, 552, 517, 519, 495, 512] payload_rows_per_dst=[5788, 5590, 5488, 5612, 5568, 5532, 5464, 5386] dup_rows_per_dst=[3024, 2700, 2584, 2582, 2734, 2648, 2530, 2306] combine_rows_per_src=[5788, 5590, 5488, 5612, 5568, 5532, 5464, 5386] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44656 payload_rows_remote=39190 dup_rows=20880 dup_rows_remote=18260 dedup_ratio=0.318604 router_tokens_per_expert=[516, 513, 483, 518, 525, 536, 465, 527, 497, 477, 531, 537, 503, 539, 530, 492, 511, 488, 521, 520, 520, 515, 545, 543, 498, 513, 498, 500, 497, 514, 495, 528, 505, 525, 526, 501, 496, 535, 513, 531, 509, 545, 503, 531, 502, 521, 494, 497, 535, 503, 536, 490, 511, 500, 519, 495, 508, 506, 512, 492, 507, 528, 471, 526] payload_rows_per_dst=[5540, 5578, 5612, 5466, 5668, 5600, 5636, 5556] dup_rows_per_dst=[2626, 2634, 2714, 2620, 2596, 2604, 2542, 2544] combine_rows_per_src=[5540, 5578, 5612, 5466, 5668, 5600, 5636, 5556] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279250944 dispatch_meta_bytes=229208 combine_payload_bytes=279250944 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279480152 ingress_bytes=609552384 link_bytes=609552384 token_bytes=558731096 weight_movement_bytes=330301440 total_nvlink_bytes=889032536 weight_to_token_ratio=0.591164 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060111850e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279293952 dispatch_meta_bytes=229384 combine_payload_bytes=279293952 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279523336 ingress_bytes=389394432 link_bytes=389394432 token_bytes=558817288 weight_movement_bytes=110100480 total_nvlink_bytes=668917768 weight_to_token_ratio=0.197024 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.772209616e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279910400 dispatch_meta_bytes=228520 combine_payload_bytes=279910400 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280138920 ingress_bytes=500111360 link_bytes=500111360 token_bytes=560049320 weight_movement_bytes=220200960 total_nvlink_bytes=780250280 weight_to_token_ratio=0.393181 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.697759092e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44468 payload_rows_remote=38982 dup_rows=21068 dup_rows_remote=18476 dedup_ratio=0.321472 router_tokens_per_expert=[518, 539, 551, 488, 544, 506, 492, 513, 523, 541, 490, 489, 508, 512, 502, 474, 538, 547, 498, 516, 519, 517, 553, 489, 525, 518, 495, 514, 538, 508, 490, 509, 546, 526, 498, 482, 535, 554, 513, 534, 500, 492, 505, 500, 501, 485, 482, 521, 494, 525, 486, 503, 532, 492, 529, 519, 481, 522, 531, 507, 494, 524, 471, 520] payload_rows_per_dst=[5592, 5486, 5642, 5528, 5698, 5472, 5542, 5508] dup_rows_per_dst=[2710, 2592, 2712, 2666, 2678, 2500, 2618, 2592] combine_rows_per_src=[5592, 5486, 5642, 5528, 5698, 5472, 5542, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44196 payload_rows_remote=38610 dup_rows=21340 dup_rows_remote=18656 dedup_ratio=0.325623 router_tokens_per_expert=[544, 500, 500, 490, 494, 515, 516, 481, 497, 515, 524, 512, 551, 506, 516, 503, 540, 545, 483, 492, 508, 472, 552, 499, 503, 532, 522, 521, 508, 537, 488, 544, 529, 556, 528, 482, 517, 474, 519, 530, 513, 487, 488, 539, 484, 495, 512, 537, 499, 483, 508, 497, 533, 523, 495, 525, 480, 494, 510, 539, 515, 527, 533, 507] payload_rows_per_dst=[5446, 5522, 5524, 5596, 5586, 5486, 5512, 5524] dup_rows_per_dst=[2634, 2726, 2658, 2714, 2684, 2624, 2614, 2686] combine_rows_per_src=[5446, 5522, 5524, 5596, 5586, 5486, 5512, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=276971520 dispatch_meta_bytes=226896 combine_payload_bytes=276971520 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277198416 ingress_bytes=387072000 link_bytes=387072000 token_bytes=554169936 weight_movement_bytes=110100480 total_nvlink_bytes=664270416 weight_to_token_ratio=0.198676 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.731818704e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280913920 dispatch_meta_bytes=229800 combine_payload_bytes=280913920 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281143720 ingress_bytes=391014400 link_bytes=391014400 token_bytes=562057640 weight_movement_bytes=110100480 total_nvlink_bytes=672158120 weight_to_token_ratio=0.195888 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.800383524e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=277043200 dispatch_meta_bytes=228560 combine_payload_bytes=277043200 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277271760 ingress_bytes=387143680 link_bytes=387143680 token_bytes=554314960 weight_movement_bytes=110100480 total_nvlink_bytes=664415440 weight_to_token_ratio=0.198624 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.733065337e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278433792 dispatch_meta_bytes=229600 combine_payload_bytes=278433792 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278663392 ingress_bytes=498634752 link_bytes=498634752 token_bytes=557097184 weight_movement_bytes=220200960 total_nvlink_bytes=777298144 weight_to_token_ratio=0.395265 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.672078451e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44650 payload_rows_remote=38972 dup_rows=20886 dup_rows_remote=18068 dedup_ratio=0.318695 router_tokens_per_expert=[573, 522, 543, 534, 537, 495, 498, 546, 538, 540, 537, 530, 506, 510, 520, 504, 539, 518, 517, 503, 492, 516, 493, 515, 495, 519, 490, 482, 516, 500, 499, 504, 486, 504, 514, 533, 500, 474, 501, 526, 513, 500, 512, 510, 495, 532, 512, 488, 512, 507, 497, 469, 538, 518, 509, 485, 517, 501, 513, 498, 517, 534, 484, 538] payload_rows_per_dst=[5678, 5686, 5618, 5538, 5464, 5556, 5562, 5548] dup_rows_per_dst=[2818, 2684, 2568, 2472, 2612, 2568, 2508, 2656] combine_rows_per_src=[5678, 5686, 5618, 5538, 5464, 5556, 5562, 5548] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44490 payload_rows_remote=38910 dup_rows=21046 dup_rows_remote=18440 dedup_ratio=0.321136 router_tokens_per_expert=[511, 495, 538, 533, 526, 528, 516, 520, 512, 501, 535, 503, 530, 569, 515, 502, 534, 522, 522, 503, 483, 496, 514, 509, 497, 493, 512, 524, 532, 505, 475, 531, 499, 483, 534, 491, 522, 476, 542, 522, 508, 506, 487, 507, 525, 512, 514, 510, 496, 524, 499, 491, 505, 536, 524, 518, 545, 511, 504, 478, 509, 503, 497, 504] payload_rows_per_dst=[5640, 5568, 5546, 5568, 5532, 5498, 5580, 5558] dup_rows_per_dst=[2694, 2766, 2620, 2570, 2606, 2640, 2606, 2544] combine_rows_per_src=[5640, 5568, 5546, 5568, 5532, 5498, 5580, 5558] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=276770816 dispatch_meta_bytes=229104 combine_payload_bytes=276770816 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276999920 ingress_bytes=386871296 link_bytes=386871296 token_bytes=553770736 weight_movement_bytes=110100480 total_nvlink_bytes=663871216 weight_to_token_ratio=0.198820 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.728328131e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44404 payload_rows_remote=38960 dup_rows=21132 dup_rows_remote=18482 dedup_ratio=0.322449 router_tokens_per_expert=[520, 517, 554, 508, 510, 512, 533, 486, 521, 517, 517, 514, 507, 510, 523, 536, 538, 541, 525, 508, 500, 502, 518, 485, 502, 512, 510, 513, 497, 485, 508, 520, 485, 539, 475, 522, 506, 496, 520, 499, 509, 517, 542, 486, 510, 537, 517, 517, 507, 514, 483, 528, 466, 503, 486, 546, 521, 502, 544, 482, 526, 540, 495, 499] payload_rows_per_dst=[5568, 5592, 5560, 5444, 5454, 5686, 5490, 5610] dup_rows_per_dst=[2712, 2698, 2674, 2650, 2630, 2584, 2576, 2608] combine_rows_per_src=[5568, 5592, 5560, 5444, 5454, 5686, 5490, 5610] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279781376 dispatch_meta_bytes=230064 combine_payload_bytes=279781376 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280011440 ingress_bytes=389881856 link_bytes=389881856 token_bytes=559792816 weight_movement_bytes=110100480 total_nvlink_bytes=669893296 weight_to_token_ratio=0.196681 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.780686721e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44364 payload_rows_remote=38840 dup_rows=21172 dup_rows_remote=18580 dedup_ratio=0.323059 router_tokens_per_expert=[519, 470, 542, 537, 531, 500, 485, 480, 487, 510, 500, 505, 511, 484, 495, 509, 497, 483, 517, 505, 523, 502, 518, 513, 517, 561, 504, 519, 480, 502, 540, 512, 494, 534, 490, 501, 530, 550, 527, 514, 522, 544, 497, 512, 446, 526, 521, 539, 517, 505, 523, 504, 505, 544, 505, 533, 532, 499, 544, 478, 503, 485, 545, 541] payload_rows_per_dst=[5552, 5488, 5524, 5532, 5560, 5552, 5568, 5588] dup_rows_per_dst=[2576, 2514, 2592, 2738, 2720, 2662, 2704, 2666] combine_rows_per_src=[5552, 5488, 5524, 5532, 5560, 5552, 5568, 5588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=276756480 dispatch_meta_bytes=229064 combine_payload_bytes=276756480 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276985544 ingress_bytes=386856960 link_bytes=386856960 token_bytes=553742024 weight_movement_bytes=110100480 total_nvlink_bytes=663842504 weight_to_token_ratio=0.198830 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.728078805e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279422976 dispatch_meta_bytes=229832 combine_payload_bytes=279422976 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279652808 ingress_bytes=499623936 link_bytes=499623936 token_bytes=559075784 weight_movement_bytes=220200960 total_nvlink_bytes=779276744 weight_to_token_ratio=0.393866 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.689281987e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44490 payload_rows_remote=38904 dup_rows=21046 dup_rows_remote=18420 dedup_ratio=0.321136 router_tokens_per_expert=[509, 489, 537, 532, 524, 526, 514, 520, 509, 497, 531, 501, 530, 568, 516, 497, 533, 519, 520, 504, 481, 495, 512, 507, 497, 493, 510, 522, 530, 503, 475, 534, 499, 483, 535, 493, 522, 478, 543, 523, 509, 506, 487, 507, 528, 511, 516, 516, 499, 525, 500, 492, 507, 538, 526, 519, 549, 513, 506, 482, 513, 504, 498, 506] payload_rows_per_dst=[5620, 5550, 5538, 5568, 5536, 5514, 5586, 5578] dup_rows_per_dst=[2682, 2748, 2604, 2560, 2616, 2646, 2626, 2564] combine_rows_per_src=[5620, 5550, 5538, 5568, 5536, 5514, 5586, 5578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44550 payload_rows_remote=39058 dup_rows=20986 dup_rows_remote=18452 dedup_ratio=0.320221 router_tokens_per_expert=[517, 527, 512, 550, 510, 521, 505, 494, 543, 492, 490, 503, 475, 529, 500, 505, 481, 525, 564, 508, 517, 489, 530, 475, 490, 496, 517, 546, 505, 539, 545, 527, 488, 510, 541, 529, 516, 498, 516, 503, 504, 502, 511, 481, 489, 508, 494, 524, 480, 502, 532, 513, 546, 535, 485, 529, 489, 496, 491, 542, 541, 533, 490, 523] payload_rows_per_dst=[5610, 5498, 5562, 5610, 5594, 5492, 5634, 5550] dup_rows_per_dst=[2662, 2576, 2616, 2720, 2608, 2534, 2610, 2660] combine_rows_per_src=[5610, 5498, 5562, 5610, 5594, 5492, 5634, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44438 payload_rows_remote=38780 dup_rows=21098 dup_rows_remote=18532 dedup_ratio=0.321930 router_tokens_per_expert=[553, 517, 534, 528, 512, 530, 507, 490, 502, 555, 515, 506, 533, 527, 535, 476, 481, 510, 560, 552, 516, 521, 534, 530, 530, 502, 485, 492, 536, 498, 488, 547, 503, 533, 496, 466, 503, 514, 518, 495, 524, 491, 504, 492, 524, 525, 507, 479, 493, 505, 521, 499, 517, 477, 492, 493, 551, 487, 473, 528, 537, 506, 523, 490] payload_rows_per_dst=[5404, 5578, 5512, 5558, 5506, 5686, 5536, 5658] dup_rows_per_dst=[2330, 2674, 2584, 2644, 2596, 2934, 2770, 2566] combine_rows_per_src=[5404, 5578, 5512, 5558, 5506, 5686, 5536, 5658] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278405120 dispatch_meta_bytes=229680 combine_payload_bytes=278405120 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278634800 ingress_bytes=498606080 link_bytes=498606080 token_bytes=557039920 weight_movement_bytes=220200960 total_nvlink_bytes=777240880 weight_to_token_ratio=0.395306 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.671579797e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44642 payload_rows_remote=38978 dup_rows=20894 dup_rows_remote=18088 dedup_ratio=0.318817 router_tokens_per_expert=[571, 522, 540, 531, 533, 492, 495, 546, 537, 538, 536, 529, 503, 509, 518, 501, 537, 516, 515, 497, 491, 514, 491, 513, 496, 519, 488, 482, 518, 500, 501, 500, 486, 504, 512, 531, 499, 475, 502, 528, 514, 502, 514, 514, 497, 534, 513, 487, 513, 511, 501, 472, 540, 522, 511, 487, 518, 508, 514, 502, 517, 534, 487, 540] payload_rows_per_dst=[5664, 5668, 5632, 5546, 5462, 5562, 5584, 5524] dup_rows_per_dst=[2806, 2664, 2622, 2462, 2612, 2588, 2530, 2610] combine_rows_per_src=[5664, 5668, 5632, 5546, 5462, 5562, 5584, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44412 payload_rows_remote=38964 dup_rows=21124 dup_rows_remote=18478 dedup_ratio=0.322327 router_tokens_per_expert=[514, 516, 552, 506, 511, 508, 532, 485, 517, 514, 514, 512, 507, 507, 519, 536, 537, 537, 524, 506, 500, 502, 516, 485, 503, 513, 513, 512, 496, 482, 507, 521, 484, 539, 474, 522, 506, 496, 519, 500, 508, 517, 545, 486, 514, 536, 519, 517, 507, 517, 484, 530, 470, 506, 488, 549, 521, 504, 547, 485, 528, 547, 496, 503] payload_rows_per_dst=[5560, 5566, 5552, 5448, 5454, 5690, 5512, 5630] dup_rows_per_dst=[2688, 2686, 2662, 2646, 2626, 2594, 2590, 2632] combine_rows_per_src=[5560, 5566, 5552, 5448, 5454, 5690, 5512, 5630] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278863872 dispatch_meta_bytes=229296 combine_payload_bytes=278863872 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279093168 ingress_bytes=388964352 link_bytes=388964352 token_bytes=557957040 weight_movement_bytes=110100480 total_nvlink_bytes=668057520 weight_to_token_ratio=0.197328 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.764729818e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279967744 dispatch_meta_bytes=230040 combine_payload_bytes=279967744 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280197784 ingress_bytes=390068224 link_bytes=390068224 token_bytes=560165528 weight_movement_bytes=110100480 total_nvlink_bytes=670266008 weight_to_token_ratio=0.196550 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.783927967e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279351296 dispatch_meta_bytes=228160 combine_payload_bytes=279351296 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=279579456 ingress_bytes=719753216 link_bytes=719753216 token_bytes=558930752 weight_movement_bytes=440401920 total_nvlink_bytes=999332672 weight_to_token_ratio=0.787936 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.251769222e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278906880 dispatch_meta_bytes=229400 combine_payload_bytes=278906880 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279136280 ingress_bytes=389007360 link_bytes=389007360 token_bytes=558043160 weight_movement_bytes=110100480 total_nvlink_bytes=668143640 weight_to_token_ratio=0.197297 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.765477798e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279265280 dispatch_meta_bytes=229768 combine_payload_bytes=279265280 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279495048 ingress_bytes=389365760 link_bytes=389365760 token_bytes=558760328 weight_movement_bytes=110100480 total_nvlink_bytes=668860808 weight_to_token_ratio=0.197044 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.771710963e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44296 payload_rows_remote=38418 dup_rows=21240 dup_rows_remote=17960 dedup_ratio=0.324097 router_tokens_per_expert=[546, 558, 510, 509, 516, 504, 499, 505, 542, 535, 503, 531, 513, 501, 506, 489, 573, 502, 528, 501, 489, 520, 493, 513, 510, 499, 511, 518, 535, 501, 521, 483, 507, 506, 525, 515, 492, 501, 481, 502, 530, 537, 526, 502, 522, 471, 507, 501, 521, 530, 506, 529, 477, 500, 469, 548, 489, 504, 468, 539, 538, 516, 512, 533] payload_rows_per_dst=[5170, 5516, 5642, 5470, 5496, 5590, 5534, 5878] dup_rows_per_dst=[2164, 2584, 2716, 2686, 2582, 2602, 2626, 3280] combine_rows_per_src=[5170, 5516, 5642, 5470, 5496, 5590, 5534, 5878] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44472 payload_rows_remote=38978 dup_rows=21064 dup_rows_remote=18456 dedup_ratio=0.321411 router_tokens_per_expert=[519, 543, 554, 492, 545, 509, 497, 513, 525, 546, 495, 487, 509, 513, 503, 473, 538, 548, 498, 518, 517, 518, 554, 490, 526, 518, 495, 512, 539, 512, 488, 509, 547, 523, 496, 484, 533, 557, 510, 533, 500, 491, 502, 500, 501, 483, 482, 519, 491, 521, 486, 501, 530, 491, 526, 516, 481, 522, 529, 506, 493, 523, 468, 520] payload_rows_per_dst=[5610, 5494, 5646, 5534, 5690, 5468, 5530, 5500] dup_rows_per_dst=[2734, 2608, 2716, 2664, 2676, 2488, 2594, 2584] combine_rows_per_src=[5610, 5494, 5646, 5534, 5690, 5468, 5530, 5500] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44598 payload_rows_remote=38984 dup_rows=20938 dup_rows_remote=18298 dedup_ratio=0.319489 router_tokens_per_expert=[515, 514, 546, 513, 535, 509, 542, 522, 531, 504, 528, 488, 513, 473, 516, 542, 497, 539, 519, 507, 523, 523, 520, 499, 503, 497, 488, 503, 525, 491, 506, 489, 522, 537, 528, 472, 524, 527, 497, 501, 499, 497, 518, 504, 558, 524, 498, 504, 515, 529, 533, 504, 543, 493, 516, 503, 500, 502, 507, 476, 482, 522, 504, 509] payload_rows_per_dst=[5586, 5620, 5614, 5460, 5644, 5614, 5592, 5468] dup_rows_per_dst=[2806, 2570, 2640, 2544, 2572, 2590, 2680, 2536] combine_rows_per_src=[5586, 5620, 5614, 5460, 5644, 5614, 5592, 5468] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44416 payload_rows_remote=38854 dup_rows=21120 dup_rows_remote=18562 dedup_ratio=0.322266 router_tokens_per_expert=[525, 524, 544, 525, 548, 522, 505, 498, 494, 525, 487, 521, 507, 508, 507, 549, 542, 507, 496, 535, 541, 512, 505, 557, 518, 527, 510, 510, 461, 513, 511, 513, 513, 480, 494, 464, 555, 515, 492, 547, 516, 525, 519, 511, 521, 476, 491, 515, 539, 485, 490, 522, 487, 531, 578, 453, 494, 483, 513, 530, 508, 492, 475, 507] payload_rows_per_dst=[5636, 5544, 5632, 5486, 5562, 5646, 5480, 5430] dup_rows_per_dst=[2746, 2652, 2758, 2640, 2558, 2502, 2690, 2574] combine_rows_per_src=[5636, 5544, 5632, 5486, 5562, 5646, 5480, 5430] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44662 payload_rows_remote=39232 dup_rows=20874 dup_rows_remote=18430 dedup_ratio=0.318512 router_tokens_per_expert=[549, 553, 474, 539, 486, 511, 492, 505, 473, 481, 545, 510, 563, 497, 516, 530, 502, 522, 539, 493, 516, 486, 509, 538, 554, 504, 536, 524, 482, 526, 519, 503, 504, 525, 546, 521, 533, 542, 498, 565, 468, 512, 499, 498, 487, 482, 519, 472, 507, 543, 482, 482, 538, 528, 548, 539, 504, 491, 504, 491, 483, 496, 508, 476] payload_rows_per_dst=[5616, 5626, 5596, 5666, 5668, 5430, 5620, 5440] dup_rows_per_dst=[2602, 2604, 2614, 2630, 2800, 2444, 2714, 2466] combine_rows_per_src=[5616, 5626, 5596, 5666, 5668, 5430, 5620, 5440] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=275380224 dispatch_meta_bytes=225512 combine_payload_bytes=275380224 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=275605736 ingress_bytes=385480704 link_bytes=385480704 token_bytes=550985960 weight_movement_bytes=110100480 total_nvlink_bytes=661086440 weight_to_token_ratio=0.199824 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4096 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.704143449e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44262 payload_rows_remote=38784 dup_rows=21274 dup_rows_remote=18680 dedup_ratio=0.324615 router_tokens_per_expert=[503, 500, 492, 525, 490, 509, 494, 495, 540, 520, 535, 521, 504, 517, 514, 488, 479, 506, 547, 498, 513, 519, 511, 505, 531, 498, 500, 532, 516, 518, 499, 489, 534, 527, 525, 554, 485, 493, 505, 492, 509, 506, 555, 473, 512, 534, 477, 497, 527, 527, 522, 519, 537, 531, 509, 488, 536, 529, 529, 501, 515, 486, 529, 497] payload_rows_per_dst=[5478, 5610, 5496, 5568, 5478, 5720, 5392, 5520] dup_rows_per_dst=[2594, 2674, 2660, 2798, 2552, 2920, 2352, 2724] combine_rows_per_src=[5478, 5610, 5496, 5568, 5478, 5720, 5392, 5520] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279394304 dispatch_meta_bytes=229736 combine_payload_bytes=279394304 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279624040 ingress_bytes=609695744 link_bytes=609695744 token_bytes=559018344 weight_movement_bytes=330301440 total_nvlink_bytes=889319784 weight_to_token_ratio=0.590860 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060361177e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279437312 dispatch_meta_bytes=229128 combine_payload_bytes=279437312 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279666440 ingress_bytes=389537792 link_bytes=389537792 token_bytes=559103752 weight_movement_bytes=110100480 total_nvlink_bytes=669204232 weight_to_token_ratio=0.196923 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.774702882e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44522 payload_rows_remote=38992 dup_rows=21014 dup_rows_remote=18338 dedup_ratio=0.320648 router_tokens_per_expert=[563, 483, 506, 489, 544, 516, 521, 515, 546, 548, 528, 510, 492, 486, 541, 516, 496, 501, 538, 470, 479, 503, 509, 480, 473, 493, 507, 556, 528, 529, 505, 503, 507, 524, 526, 553, 484, 521, 567, 487, 524, 523, 506, 520, 501, 506, 492, 518, 509, 530, 481, 544, 514, 499, 513, 513, 539, 504, 497, 487, 497, 513, 497, 498] payload_rows_per_dst=[5576, 5614, 5466, 5554, 5664, 5568, 5530, 5550] dup_rows_per_dst=[2698, 2720, 2486, 2634, 2674, 2612, 2676, 2514] combine_rows_per_src=[5576, 5614, 5466, 5554, 5664, 5568, 5530, 5550] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278505472 dispatch_meta_bytes=229664 combine_payload_bytes=278505472 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278735136 ingress_bytes=498706432 link_bytes=498706432 token_bytes=557240608 weight_movement_bytes=220200960 total_nvlink_bytes=777441568 weight_to_token_ratio=0.395163 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.673325084e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=281214976 dispatch_meta_bytes=230648 combine_payload_bytes=281214976 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281445624 ingress_bytes=391315456 link_bytes=391315456 token_bytes=562660600 weight_movement_bytes=110100480 total_nvlink_bytes=672761080 weight_to_token_ratio=0.195678 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.805619383e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44494 payload_rows_remote=38944 dup_rows=21042 dup_rows_remote=18476 dedup_ratio=0.321075 router_tokens_per_expert=[503, 544, 507, 540, 525, 506, 538, 522, 536, 508, 560, 489, 506, 504, 500, 512, 493, 486, 500, 537, 538, 522, 482, 551, 508, 468, 529, 529, 539, 482, 488, 515, 493, 497, 486, 537, 501, 503, 503, 510, 495, 537, 533, 456, 544, 494, 525, 549, 521, 488, 509, 546, 508, 521, 554, 506, 494, 477, 504, 512, 484, 496, 487, 531] payload_rows_per_dst=[5636, 5528, 5528, 5550, 5558, 5640, 5600, 5454] dup_rows_per_dst=[2734, 2702, 2690, 2566, 2502, 2626, 2706, 2516] combine_rows_per_src=[5636, 5528, 5528, 5550, 5558, 5640, 5600, 5454] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5557539608 prefetch_bytes=1101004800 reduce_bytes=330301440 weight_movement_bytes=1431306240 egress_bytes=2779910936 ingress_bytes=4208934912 link_bytes=4208934912 total_nvlink_bytes=6988845848 weight_to_token_ratio=0.257543 prefetch_share_of_ingress=0.261588 dedup_ratio=0.321918 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=3 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.320030063e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278003712 dispatch_meta_bytes=229856 combine_payload_bytes=278003712 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278233568 ingress_bytes=498204672 link_bytes=498204672 token_bytes=556237280 weight_movement_bytes=220200960 total_nvlink_bytes=776438240 weight_to_token_ratio=0.395876 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.664598652e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=277975040 dispatch_meta_bytes=229248 combine_payload_bytes=277975040 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278204288 ingress_bytes=498176000 link_bytes=498176000 token_bytes=556179328 weight_movement_bytes=220200960 total_nvlink_bytes=776380288 weight_to_token_ratio=0.395917 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.664099999e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44406 payload_rows_remote=38838 dup_rows=21130 dup_rows_remote=18568 dedup_ratio=0.322418 router_tokens_per_expert=[520, 521, 539, 522, 545, 519, 502, 498, 492, 523, 484, 519, 506, 505, 505, 547, 541, 508, 496, 535, 543, 511, 505, 554, 518, 529, 514, 511, 460, 512, 513, 511, 513, 480, 495, 467, 554, 515, 494, 547, 515, 526, 521, 511, 520, 476, 492, 515, 539, 487, 490, 525, 487, 538, 580, 455, 496, 483, 515, 531, 511, 495, 477, 510] payload_rows_per_dst=[5608, 5526, 5628, 5492, 5568, 5652, 5494, 5438] dup_rows_per_dst=[2724, 2636, 2758, 2644, 2562, 2500, 2708, 2598] combine_rows_per_src=[5608, 5526, 5628, 5492, 5568, 5652, 5494, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44740 payload_rows_remote=39280 dup_rows=20796 dup_rows_remote=18306 dedup_ratio=0.317322 router_tokens_per_expert=[554, 527, 484, 524, 575, 522, 528, 513, 501, 511, 496, 477, 515, 508, 465, 502, 488, 488, 474, 474, 529, 523, 481, 531, 547, 512, 505, 517, 518, 491, 525, 526, 542, 553, 466, 519, 520, 502, 548, 469, 504, 524, 479, 479, 497, 514, 528, 498, 526, 508, 509, 541, 521, 535, 509, 530, 518, 524, 501, 501, 529, 502, 521, 520] payload_rows_per_dst=[5762, 5460, 5518, 5602, 5646, 5512, 5654, 5586] dup_rows_per_dst=[2692, 2490, 2458, 2680, 2592, 2534, 2704, 2646] combine_rows_per_src=[5762, 5460, 5518, 5602, 5646, 5512, 5654, 5586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279394304 dispatch_meta_bytes=228264 combine_payload_bytes=279394304 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=279622568 ingress_bytes=719796224 link_bytes=719796224 token_bytes=559016872 weight_movement_bytes=440401920 total_nvlink_bytes=999418792 weight_to_token_ratio=0.787815 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.251844020e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279293952 dispatch_meta_bytes=229768 combine_payload_bytes=279293952 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279523720 ingress_bytes=389394432 link_bytes=389394432 token_bytes=558817672 weight_movement_bytes=110100480 total_nvlink_bytes=668918152 weight_to_token_ratio=0.197024 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.772209616e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44612 payload_rows_remote=39002 dup_rows=20924 dup_rows_remote=18292 dedup_ratio=0.319275 router_tokens_per_expert=[514, 509, 545, 509, 530, 506, 541, 522, 527, 504, 525, 486, 509, 472, 516, 539, 494, 541, 517, 502, 523, 523, 520, 501, 500, 498, 486, 501, 524, 491, 506, 491, 526, 539, 527, 472, 525, 525, 496, 501, 498, 498, 518, 505, 559, 525, 501, 504, 515, 530, 534, 509, 546, 496, 516, 510, 504, 503, 513, 476, 485, 522, 507, 511] payload_rows_per_dst=[5572, 5604, 5610, 5460, 5648, 5624, 5610, 5484] dup_rows_per_dst=[2780, 2552, 2632, 2534, 2574, 2592, 2702, 2558] combine_rows_per_src=[5572, 5604, 5610, 5460, 5648, 5624, 5610, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278390784 dispatch_meta_bytes=229624 combine_payload_bytes=278390784 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278620408 ingress_bytes=498591744 link_bytes=498591744 token_bytes=557011192 weight_movement_bytes=220200960 total_nvlink_bytes=777212152 weight_to_token_ratio=0.395326 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.671330471e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=281559040 dispatch_meta_bytes=230344 combine_payload_bytes=281559040 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281789384 ingress_bytes=391659520 link_bytes=391659520 token_bytes=563348424 weight_movement_bytes=110100480 total_nvlink_bytes=673448904 weight_to_token_ratio=0.195439 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.811603222e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44510 payload_rows_remote=38970 dup_rows=21026 dup_rows_remote=18324 dedup_ratio=0.320831 router_tokens_per_expert=[560, 481, 502, 487, 542, 512, 519, 515, 545, 543, 526, 509, 492, 486, 539, 514, 497, 498, 537, 473, 481, 504, 507, 479, 472, 493, 506, 555, 527, 526, 503, 504, 506, 523, 527, 553, 483, 518, 567, 488, 523, 523, 507, 522, 502, 505, 494, 518, 512, 530, 483, 546, 518, 501, 517, 514, 543, 509, 499, 489, 497, 515, 499, 503] payload_rows_per_dst=[5562, 5610, 5462, 5548, 5658, 5566, 5540, 5564] dup_rows_per_dst=[2674, 2698, 2490, 2624, 2672, 2622, 2702, 2544] combine_rows_per_src=[5562, 5610, 5462, 5548, 5658, 5566, 5540, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44682 payload_rows_remote=39240 dup_rows=20854 dup_rows_remote=18402 dedup_ratio=0.318207 router_tokens_per_expert=[549, 553, 471, 536, 485, 506, 490, 504, 472, 482, 543, 505, 562, 492, 514, 531, 501, 520, 539, 494, 516, 484, 506, 537, 554, 504, 529, 521, 481, 524, 518, 501, 502, 524, 547, 521, 535, 546, 500, 567, 467, 515, 500, 498, 489, 483, 520, 475, 507, 544, 485, 482, 540, 531, 552, 542, 505, 493, 508, 492, 485, 496, 515, 478] payload_rows_per_dst=[5610, 5616, 5596, 5658, 5674, 5442, 5632, 5454] dup_rows_per_dst=[2578, 2586, 2598, 2606, 2810, 2452, 2734, 2490] combine_rows_per_src=[5610, 5616, 5596, 5658, 5674, 5442, 5632, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279566336 dispatch_meta_bytes=229176 combine_payload_bytes=279566336 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279795512 ingress_bytes=389666816 link_bytes=389666816 token_bytes=559361848 weight_movement_bytes=110100480 total_nvlink_bytes=669462328 weight_to_token_ratio=0.196832 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.776946822e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5584131336 prefetch_bytes=1101004800 reduce_bytes=440401920 weight_movement_bytes=1541406720 egress_bytes=2793213192 ingress_bytes=4332324864 link_bytes=4332324864 total_nvlink_bytes=7125538056 weight_to_token_ratio=0.276033 prefetch_share_of_ingress=0.254137 dedup_ratio=0.321448 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=4 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.044118 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.534625483e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44292 payload_rows_remote=38508 dup_rows=21244 dup_rows_remote=18154 dedup_ratio=0.324158 router_tokens_per_expert=[544, 552, 503, 507, 514, 505, 491, 505, 539, 534, 503, 527, 513, 501, 504, 488, 569, 502, 529, 501, 488, 521, 493, 511, 513, 497, 509, 516, 534, 501, 521, 484, 505, 505, 524, 513, 491, 501, 481, 500, 531, 536, 523, 503, 524, 473, 508, 502, 524, 531, 509, 529, 482, 503, 471, 551, 492, 506, 470, 547, 541, 520, 515, 538] payload_rows_per_dst=[5306, 5614, 5570, 5474, 5512, 5736, 5296, 5784] dup_rows_per_dst=[2370, 2694, 2658, 2676, 2584, 2876, 2296, 3090] combine_rows_per_src=[5306, 5614, 5570, 5474, 5512, 5736, 5296, 5784] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5588604144 prefetch_bytes=770703360 reduce_bytes=990904320 weight_movement_bytes=1761607680 egress_bytes=2795449584 ingress_bytes=4554762240 link_bytes=4554762240 total_nvlink_bytes=7350211824 weight_to_token_ratio=0.315214 prefetch_share_of_ingress=0.169208 dedup_ratio=0.320526 prefetch_slots=14 unused_prefetch_slots=146 slot_utilization=0.087500 reduce_slots=9 remote_demand=7 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=12319 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.921480665e-05 prefetch_bytes_per_gemm_flop=1.340379902e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44316 payload_rows_remote=38878 dup_rows=21220 dup_rows_remote=18684 dedup_ratio=0.323792 router_tokens_per_expert=[499, 497, 492, 521, 488, 507, 491, 492, 539, 515, 533, 521, 503, 516, 514, 482, 479, 508, 546, 497, 512, 517, 510, 506, 532, 498, 500, 534, 517, 519, 499, 489, 533, 526, 524, 554, 486, 494, 503, 492, 510, 507, 556, 472, 514, 535, 476, 498, 527, 529, 523, 520, 539, 533, 512, 489, 536, 532, 531, 503, 520, 487, 531, 503] payload_rows_per_dst=[5438, 5596, 5574, 5602, 5454, 5562, 5554, 5536] dup_rows_per_dst=[2536, 2650, 2778, 2826, 2518, 2574, 2588, 2750] combine_rows_per_src=[5438, 5596, 5574, 5602, 5454, 5562, 5554, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279336960 dispatch_meta_bytes=229176 combine_payload_bytes=279336960 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279566136 ingress_bytes=609638400 link_bytes=609638400 token_bytes=558903096 weight_movement_bytes=330301440 total_nvlink_bytes=889204536 weight_to_token_ratio=0.590982 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060261446e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=281272320 dispatch_meta_bytes=230568 combine_payload_bytes=281272320 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281502888 ingress_bytes=391372800 link_bytes=391372800 token_bytes=562775208 weight_movement_bytes=110100480 total_nvlink_bytes=672875688 weight_to_token_ratio=0.195638 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.806616690e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44484 payload_rows_remote=38938 dup_rows=21052 dup_rows_remote=18488 dedup_ratio=0.321228 router_tokens_per_expert=[499, 541, 505, 536, 522, 505, 537, 517, 530, 508, 559, 487, 503, 505, 499, 511, 491, 485, 499, 535, 537, 520, 483, 553, 508, 470, 529, 528, 537, 482, 487, 514, 494, 496, 486, 534, 504, 504, 502, 512, 497, 540, 533, 456, 547, 496, 527, 550, 520, 487, 508, 550, 512, 523, 553, 508, 496, 479, 509, 516, 485, 499, 488, 535] payload_rows_per_dst=[5620, 5512, 5522, 5546, 5562, 5646, 5604, 5472] dup_rows_per_dst=[2704, 2692, 2684, 2564, 2502, 2646, 2718, 2542] combine_rows_per_src=[5620, 5512, 5522, 5546, 5562, 5646, 5604, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5586165144 prefetch_bytes=880803840 reduce_bytes=1211105280 weight_movement_bytes=2091909120 egress_bytes=2794229144 ingress_bytes=4883845120 link_bytes=4883845120 total_nvlink_bytes=7678074264 weight_to_token_ratio=0.374480 prefetch_share_of_ingress=0.180350 dedup_ratio=0.320673 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=11 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043382 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.493809918e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5584902832 prefetch_bytes=880803840 reduce_bytes=1651507200 weight_movement_bytes=2532311040 egress_bytes=2793597616 ingress_bytes=5323616256 link_bytes=5323616256 total_nvlink_bytes=8117213872 weight_to_token_ratio=0.453421 prefetch_share_of_ingress=0.165452 dedup_ratio=0.320682 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=15 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042647 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.258644254e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279494656 dispatch_meta_bytes=229320 combine_payload_bytes=279494656 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279723976 ingress_bytes=609796096 link_bytes=609796096 token_bytes=559218632 weight_movement_bytes=330301440 total_nvlink_bytes=889520072 weight_to_token_ratio=0.590648 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060535705e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5593135864 prefetch_bytes=1101004800 reduce_bytes=220200960 weight_movement_bytes=1321205760 egress_bytes=2797716216 ingress_bytes=4116625408 link_bytes=4116625408 total_nvlink_bytes=6914341624 weight_to_token_ratio=0.236219 prefetch_share_of_ingress=0.267453 dedup_ratio=0.320441 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=2 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043382 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.159488653e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5584991968 prefetch_bytes=1101004800 reduce_bytes=220200960 weight_movement_bytes=1321205760 egress_bytes=2793643744 ingress_bytes=4112553984 link_bytes=4112553984 total_nvlink_bytes=6906197728 weight_to_token_ratio=0.236564 prefetch_share_of_ingress=0.267718 dedup_ratio=0.321423 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=2 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.044118 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.152407777e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279150592 dispatch_meta_bytes=229680 combine_payload_bytes=279150592 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279380272 ingress_bytes=389251072 link_bytes=389251072 token_bytes=558530864 weight_movement_bytes=110100480 total_nvlink_bytes=668631344 weight_to_token_ratio=0.197125 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.769716350e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5585130392 prefetch_bytes=770703360 reduce_bytes=1871708160 weight_movement_bytes=2642411520 egress_bytes=2793710488 ingress_bytes=5433831424 link_bytes=5433831424 total_nvlink_bytes=8227541912 weight_to_token_ratio=0.473115 prefetch_share_of_ingress=0.141834 dedup_ratio=0.320630 prefetch_slots=14 unused_prefetch_slots=146 slot_utilization=0.087500 reduce_slots=17 remote_demand=7 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040441 gemm_inner_pad_upper_bound=12319 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.450326558e-05 prefetch_bytes_per_gemm_flop=1.340379902e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5586710096 prefetch_bytes=880803840 reduce_bytes=770703360 weight_movement_bytes=1651507200 egress_bytes=2794501712 ingress_bytes=4443715584 link_bytes=4443715584 total_nvlink_bytes=7238217296 weight_to_token_ratio=0.295614 prefetch_share_of_ingress=0.198213 dedup_ratio=0.320721 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=7 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.728352266e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44730 payload_rows_remote=39254 dup_rows=20806 dup_rows_remote=18310 dedup_ratio=0.317474 router_tokens_per_expert=[556, 530, 485, 529, 577, 526, 529, 515, 503, 514, 499, 477, 516, 509, 467, 501, 489, 487, 474, 476, 531, 527, 483, 532, 546, 515, 507, 516, 519, 492, 522, 525, 540, 552, 466, 518, 523, 498, 546, 470, 502, 524, 479, 480, 497, 511, 526, 497, 525, 509, 508, 540, 522, 534, 508, 526, 517, 521, 499, 501, 527, 497, 517, 514] payload_rows_per_dst=[5776, 5476, 5522, 5604, 5644, 5506, 5646, 5556] dup_rows_per_dst=[2718, 2496, 2476, 2680, 2582, 2526, 2698, 2630] combine_rows_per_src=[5776, 5476, 5522, 5604, 5644, 5506, 5646, 5556] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5591071208 prefetch_bytes=880803840 reduce_bytes=1101004800 weight_movement_bytes=1981808640 egress_bytes=2796683752 ingress_bytes=4776196096 link_bytes=4776196096 total_nvlink_bytes=7572879848 weight_to_token_ratio=0.354460 prefetch_share_of_ingress=0.184415 dedup_ratio=0.320886 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=10 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.306590560e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=281372672 dispatch_meta_bytes=230256 combine_payload_bytes=281372672 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=281602928 ingress_bytes=611674112 link_bytes=611674112 token_bytes=562975600 weight_movement_bytes=330301440 total_nvlink_bytes=893277040 weight_to_token_ratio=0.586706 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.063801884e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5586768016 prefetch_bytes=990904320 reduce_bytes=1541406720 weight_movement_bytes=2532311040 egress_bytes=2794530960 ingress_bytes=5324548096 link_bytes=5324548096 total_nvlink_bytes=8119079056 weight_to_token_ratio=0.453269 prefetch_share_of_ingress=0.186101 dedup_ratio=0.320560 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=14 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.039706 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.260264877e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none - [2026-08-26 15:12:39] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 362.3 | throughput per GPU (TFLOP/s/GPU): 340.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.603439E+00 | loss scale: 1.0 | grad norm: 3.962 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44586 payload_rows_remote=39012 dup_rows=20950 dup_rows_remote=18324 dedup_ratio=0.319672 router_tokens_per_expert=[477, 546, 501, 509, 510, 522, 537, 492, 499, 521, 504, 487, 503, 545, 522, 518, 525, 509, 552, 514, 528, 485, 498, 489, 509, 491, 526, 528, 514, 536, 493, 526, 493, 517, 520, 512, 522, 515, 503, 495, 483, 545, 511, 493, 497, 504, 514, 504, 501, 524, 534, 518, 508, 579, 480, 488, 488, 535, 489, 531, 535, 513, 483, 518] payload_rows_per_dst=[5478, 5574, 5574, 5642, 5564, 5572, 5608, 5574] dup_rows_per_dst=[2710, 2624, 2626, 2604, 2590, 2530, 2656, 2610] combine_rows_per_src=[5478, 5574, 5574, 5642, 5564, 5572, 5608, 5574] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44460 payload_rows_remote=38834 dup_rows=21076 dup_rows_remote=18452 dedup_ratio=0.321594 router_tokens_per_expert=[522, 504, 521, 528, 516, 489, 537, 492, 494, 511, 526, 552, 528, 500, 497, 523, 519, 526, 559, 484, 493, 554, 480, 531, 529, 456, 509, 529, 543, 518, 537, 557, 493, 494, 539, 501, 508, 494, 513, 496, 527, 522, 469, 523, 515, 454, 503, 480, 503, 523, 494, 512, 520, 456, 536, 512, 527, 484, 512, 498, 544, 526, 503, 523] payload_rows_per_dst=[5508, 5604, 5418, 5484, 5554, 5728, 5538, 5626] dup_rows_per_dst=[2458, 2622, 2324, 2606, 2808, 3060, 2574, 2624] combine_rows_per_src=[5508, 5604, 5418, 5484, 5554, 5728, 5538, 5626] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44476 payload_rows_remote=38958 dup_rows=21060 dup_rows_remote=18478 dedup_ratio=0.321350 router_tokens_per_expert=[492, 479, 554, 545, 510, 507, 503, 532, 529, 554, 515, 472, 507, 540, 502, 509, 533, 514, 521, 512, 514, 518, 535, 512, 532, 518, 504, 503, 512, 522, 488, 520, 495, 480, 528, 501, 517, 524, 515, 524, 541, 466, 529, 457, 505, 496, 525, 531, 504, 506, 525, 493, 508, 476, 488, 535, 508, 471, 505, 531, 550, 498, 515, 513] payload_rows_per_dst=[5536, 5616, 5616, 5598, 5574, 5518, 5488, 5530] dup_rows_per_dst=[2708, 2640, 2702, 2600, 2594, 2582, 2582, 2652] combine_rows_per_src=[5536, 5616, 5616, 5598, 5574, 5518, 5488, 5530] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44602 payload_rows_remote=38950 dup_rows=20934 dup_rows_remote=18302 dedup_ratio=0.319427 router_tokens_per_expert=[544, 522, 509, 537, 495, 503, 534, 503, 502, 535, 535, 514, 484, 504, 484, 508, 505, 531, 472, 493, 498, 494, 538, 528, 510, 515, 534, 498, 525, 534, 500, 526, 517, 536, 512, 518, 491, 512, 513, 510, 521, 502, 505, 533, 480, 504, 541, 491, 515, 523, 511, 498, 513, 471, 509, 529, 486, 527, 512, 518, 525, 516, 499, 516] payload_rows_per_dst=[5638, 5542, 5530, 5652, 5638, 5580, 5486, 5536] dup_rows_per_dst=[2656, 2590, 2588, 2632, 2580, 2574, 2652, 2662] combine_rows_per_src=[5638, 5542, 5530, 5652, 5638, 5580, 5486, 5536] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44256 payload_rows_remote=38760 dup_rows=21280 dup_rows_remote=18522 dedup_ratio=0.324707 router_tokens_per_expert=[529, 516, 506, 510, 505, 477, 509, 556, 507, 490, 536, 530, 505, 526, 505, 500, 525, 519, 481, 557, 523, 517, 517, 497, 557, 514, 515, 499, 497, 487, 528, 504, 537, 519, 520, 515, 481, 525, 503, 527, 530, 485, 481, 501, 514, 510, 507, 509, 535, 501, 526, 495, 485, 506, 512, 491, 541, 530, 526, 477, 514, 514, 512, 495] payload_rows_per_dst=[5520, 5592, 5534, 5568, 5496, 5486, 5454, 5606] dup_rows_per_dst=[2696, 2606, 2738, 2634, 2758, 2588, 2648, 2612] combine_rows_per_src=[5520, 5592, 5534, 5568, 5496, 5486, 5454, 5606] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44390 payload_rows_remote=38824 dup_rows=21146 dup_rows_remote=18494 dedup_ratio=0.322662 router_tokens_per_expert=[481, 506, 515, 516, 517, 513, 540, 508, 514, 502, 531, 522, 487, 484, 496, 553, 501, 513, 558, 505, 517, 522, 579, 510, 501, 553, 513, 479, 533, 503, 514, 517, 525, 463, 536, 493, 517, 488, 518, 479, 490, 459, 490, 524, 516, 476, 495, 519, 501, 518, 521, 530, 526, 478, 513, 522, 528, 520, 519, 512, 532, 524, 508, 525] payload_rows_per_dst=[5578, 5562, 5572, 5512, 5490, 5406, 5566, 5704] dup_rows_per_dst=[2614, 2616, 2838, 2714, 2548, 2532, 2652, 2632] combine_rows_per_src=[5578, 5562, 5572, 5512, 5490, 5406, 5566, 5704] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44500 payload_rows_remote=39002 dup_rows=21036 dup_rows_remote=18442 dedup_ratio=0.320984 router_tokens_per_expert=[508, 497, 516, 517, 557, 496, 466, 480, 520, 510, 516, 504, 534, 522, 487, 553, 481, 529, 482, 510, 507, 519, 527, 523, 513, 529, 517, 550, 436, 511, 516, 514, 527, 504, 500, 495, 536, 481, 539, 487, 501, 530, 510, 481, 546, 477, 584, 505, 512, 488, 520, 505, 487, 542, 543, 521, 524, 503, 493, 499, 500, 532, 508, 541] payload_rows_per_dst=[5498, 5626, 5588, 5502, 5490, 5656, 5640, 5500] dup_rows_per_dst=[2594, 2648, 2568, 2670, 2648, 2612, 2746, 2550] combine_rows_per_src=[5498, 5626, 5588, 5502, 5490, 5656, 5640, 5500] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278362112 dispatch_meta_bytes=229144 combine_payload_bytes=278362112 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278591256 ingress_bytes=498563072 link_bytes=498563072 token_bytes=556953368 weight_movement_bytes=220200960 total_nvlink_bytes=777154328 weight_to_token_ratio=0.395367 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.670831817e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279638016 dispatch_meta_bytes=229344 combine_payload_bytes=279638016 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279867360 ingress_bytes=499838976 link_bytes=499838976 token_bytes=559505376 weight_movement_bytes=220200960 total_nvlink_bytes=779706336 weight_to_token_ratio=0.393564 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.693021886e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279250944 dispatch_meta_bytes=229744 combine_payload_bytes=279250944 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279480688 ingress_bytes=389351424 link_bytes=389351424 token_bytes=558731632 weight_movement_bytes=110100480 total_nvlink_bytes=668832112 weight_to_token_ratio=0.197054 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.771461636e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279193600 dispatch_meta_bytes=229008 combine_payload_bytes=279193600 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279422608 ingress_bytes=499394560 link_bytes=499394560 token_bytes=558616208 weight_movement_bytes=220200960 total_nvlink_bytes=778817168 weight_to_token_ratio=0.394190 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.685292761e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44616 payload_rows_remote=39074 dup_rows=20920 dup_rows_remote=18388 dedup_ratio=0.319214 router_tokens_per_expert=[545, 514, 514, 510, 518, 542, 502, 527, 523, 545, 525, 501, 486, 455, 519, 483, 536, 539, 483, 522, 481, 538, 555, 506, 501, 503, 500, 511, 499, 534, 509, 502, 514, 474, 474, 521, 550, 499, 542, 528, 517, 488, 473, 530, 522, 517, 525, 484, 504, 482, 551, 503, 521, 518, 500, 544, 489, 512, 502, 503, 529, 512, 506, 506] payload_rows_per_dst=[5640, 5542, 5602, 5554, 5588, 5570, 5588, 5532] dup_rows_per_dst=[2704, 2532, 2718, 2564, 2616, 2542, 2658, 2586] combine_rows_per_src=[5640, 5542, 5602, 5554, 5588, 5570, 5588, 5532] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=277831680 dispatch_meta_bytes=229128 combine_payload_bytes=277831680 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278060808 ingress_bytes=387932160 link_bytes=387932160 token_bytes=555892488 weight_movement_bytes=110100480 total_nvlink_bytes=665992968 weight_to_token_ratio=0.198061 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.746778301e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278290432 dispatch_meta_bytes=229272 combine_payload_bytes=278290432 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278519704 ingress_bytes=388390912 link_bytes=388390912 token_bytes=556810136 weight_movement_bytes=110100480 total_nvlink_bytes=666910616 weight_to_token_ratio=0.197734 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.754756753e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279566336 dispatch_meta_bytes=229776 combine_payload_bytes=279566336 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279796112 ingress_bytes=499767296 link_bytes=499767296 token_bytes=559362448 weight_movement_bytes=220200960 total_nvlink_bytes=779563408 weight_to_token_ratio=0.393664 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.691775253e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280082432 dispatch_meta_bytes=229848 combine_payload_bytes=280082432 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=280312280 ingress_bytes=610383872 link_bytes=610383872 token_bytes=560394712 weight_movement_bytes=330301440 total_nvlink_bytes=890696152 weight_to_token_ratio=0.589409 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061557944e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44564 payload_rows_remote=38968 dup_rows=20972 dup_rows_remote=18226 dedup_ratio=0.320007 router_tokens_per_expert=[536, 512, 513, 518, 502, 478, 497, 516, 570, 515, 558, 542, 547, 491, 512, 505, 542, 489, 506, 488, 519, 544, 490, 493, 521, 515, 544, 523, 516, 521, 516, 500, 509, 490, 489, 495, 515, 491, 511, 526, 525, 496, 514, 529, 473, 510, 492, 519, 539, 531, 534, 496, 482, 557, 488, 503, 496, 501, 510, 510, 516, 519, 458, 505] payload_rows_per_dst=[5580, 5514, 5450, 5496, 5570, 5692, 5666, 5596] dup_rows_per_dst=[2564, 2500, 2400, 2596, 2620, 2726, 2820, 2746] combine_rows_per_src=[5580, 5514, 5450, 5496, 5570, 5692, 5666, 5596] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44484 payload_rows_remote=38938 dup_rows=21052 dup_rows_remote=18420 dedup_ratio=0.321228 router_tokens_per_expert=[494, 498, 524, 530, 489, 523, 529, 502, 481, 492, 507, 534, 524, 488, 544, 522, 506, 508, 511, 521, 549, 517, 509, 536, 571, 510, 501, 528, 487, 520, 481, 518, 503, 523, 481, 507, 508, 514, 465, 528, 521, 536, 488, 540, 499, 494, 506, 505, 503, 512, 503, 503, 523, 488, 482, 528, 554, 499, 523, 518, 549, 497, 489, 525] payload_rows_per_dst=[5564, 5558, 5596, 5568, 5500, 5546, 5484, 5668] dup_rows_per_dst=[2614, 2626, 2718, 2664, 2558, 2632, 2600, 2640] combine_rows_per_src=[5564, 5558, 5596, 5568, 5500, 5546, 5484, 5668] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44460 payload_rows_remote=38748 dup_rows=21076 dup_rows_remote=18388 dedup_ratio=0.321594 router_tokens_per_expert=[478, 500, 514, 527, 531, 508, 522, 550, 491, 505, 481, 559, 509, 564, 505, 548, 550, 520, 484, 520, 545, 545, 509, 527, 501, 505, 519, 503, 508, 500, 557, 501, 538, 489, 516, 511, 507, 493, 503, 542, 520, 527, 499, 506, 500, 520, 514, 504, 517, 494, 521, 475, 465, 482, 528, 519, 513, 545, 467, 496, 488, 496, 484, 503] payload_rows_per_dst=[5572, 5542, 5712, 5530, 5568, 5644, 5488, 5404] dup_rows_per_dst=[2688, 2782, 2688, 2658, 2630, 2536, 2514, 2580] combine_rows_per_src=[5572, 5542, 5712, 5530, 5568, 5644, 5488, 5404] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44420 payload_rows_remote=38966 dup_rows=21116 dup_rows_remote=18560 dedup_ratio=0.322205 router_tokens_per_expert=[468, 484, 519, 493, 534, 515, 528, 541, 512, 487, 495, 541, 512, 517, 471, 504, 491, 515, 550, 513, 504, 512, 501, 539, 488, 508, 515, 472, 495, 508, 510, 509, 509, 503, 557, 515, 550, 496, 518, 520, 516, 488, 499, 516, 500, 522, 524, 517, 485, 490, 530, 498, 531, 514, 527, 548, 493, 491, 522, 489, 533, 576, 492, 548] payload_rows_per_dst=[5550, 5478, 5638, 5454, 5630, 5526, 5572, 5572] dup_rows_per_dst=[2614, 2600, 2612, 2556, 2706, 2638, 2674, 2716] combine_rows_per_src=[5550, 5478, 5638, 5454, 5630, 5526, 5572, 5572] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=276025344 dispatch_meta_bytes=226648 combine_payload_bytes=276025344 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276251992 ingress_bytes=386125824 link_bytes=386125824 token_bytes=552277336 weight_movement_bytes=110100480 total_nvlink_bytes=662377816 weight_to_token_ratio=0.199357 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.715363147e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278677504 dispatch_meta_bytes=230248 combine_payload_bytes=278677504 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=278907752 ingress_bytes=719079424 link_bytes=719079424 token_bytes=557585256 weight_movement_bytes=440401920 total_nvlink_bytes=997987176 weight_to_token_ratio=0.789838 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.250597387e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5584673208 prefetch_bytes=880803840 reduce_bytes=1431306240 weight_movement_bytes=2312110080 egress_bytes=2793482680 ingress_bytes=5103300608 link_bytes=5103300608 total_nvlink_bytes=7896783288 weight_to_token_ratio=0.414010 prefetch_share_of_ingress=0.172595 dedup_ratio=0.320795 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=13 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.875479106e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5592848576 prefetch_bytes=990904320 reduce_bytes=440401920 weight_movement_bytes=1431306240 egress_bytes=2797572288 ingress_bytes=4226582528 link_bytes=4226582528 total_nvlink_bytes=7024154816 weight_to_token_ratio=0.255917 prefetch_share_of_ingress=0.234446 dedup_ratio=0.320370 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=4 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043750 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.350722170e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279107584 dispatch_meta_bytes=229704 combine_payload_bytes=279107584 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279337288 ingress_bytes=389208064 link_bytes=389208064 token_bytes=558444872 weight_movement_bytes=110100480 total_nvlink_bytes=668545352 weight_to_token_ratio=0.197156 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.768968370e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5578274656 prefetch_bytes=990904320 reduce_bytes=1211105280 weight_movement_bytes=2202009600 egress_bytes=2790281056 ingress_bytes=4990003200 link_bytes=4990003200 total_nvlink_bytes=7780284256 weight_to_token_ratio=0.394747 prefetch_share_of_ingress=0.198578 dedup_ratio=0.320993 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=11 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.678436279e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5570161808 prefetch_bytes=990904320 reduce_bytes=1211105280 weight_movement_bytes=2202009600 egress_bytes=2786225296 ingress_bytes=4985946112 link_bytes=4985946112 total_nvlink_bytes=7772171408 weight_to_token_ratio=0.395322 prefetch_share_of_ingress=0.198739 dedup_ratio=0.321527 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=11 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.671380336e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5592763032 prefetch_bytes=990904320 reduce_bytes=660602880 weight_movement_bytes=1651507200 egress_bytes=2797529752 ingress_bytes=4446740480 link_bytes=4446740480 total_nvlink_bytes=7244270232 weight_to_token_ratio=0.295294 prefetch_share_of_ingress=0.222838 dedup_ratio=0.320755 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=6 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.733613058e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none + [2026-08-26 16:00:28] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 369.3 | throughput per GPU (TFLOP/s/GPU): 334.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.605207E+00 | loss scale: 1.0 | grad norm: 4.264 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44608 payload_rows_remote=39072 dup_rows=20928 dup_rows_remote=18402 dedup_ratio=0.319336 router_tokens_per_expert=[540, 511, 510, 507, 517, 541, 501, 525, 521, 540, 525, 503, 486, 454, 520, 482, 536, 538, 483, 522, 483, 537, 551, 500, 499, 501, 502, 509, 500, 532, 509, 502, 514, 474, 474, 524, 548, 502, 542, 529, 516, 487, 474, 533, 521, 520, 526, 485, 506, 484, 551, 504, 523, 518, 502, 545, 491, 512, 503, 505, 535, 514, 510, 509] payload_rows_per_dst=[5626, 5536, 5590, 5550, 5592, 5574, 5596, 5544] dup_rows_per_dst=[2678, 2526, 2710, 2558, 2622, 2550, 2670, 2614] combine_rows_per_src=[5626, 5536, 5590, 5550, 5592, 5574, 5596, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44508 payload_rows_remote=38792 dup_rows=21028 dup_rows_remote=18262 dedup_ratio=0.320862 router_tokens_per_expert=[520, 498, 518, 527, 512, 488, 532, 489, 492, 509, 526, 549, 525, 498, 495, 523, 515, 522, 558, 481, 492, 554, 479, 529, 528, 457, 509, 529, 544, 517, 537, 556, 492, 494, 540, 499, 508, 496, 518, 498, 528, 523, 472, 524, 518, 457, 503, 482, 504, 527, 495, 512, 522, 460, 536, 516, 528, 488, 515, 502, 544, 529, 505, 525] payload_rows_per_dst=[5588, 5606, 5522, 5510, 5554, 5454, 5558, 5716] dup_rows_per_dst=[2580, 2628, 2506, 2628, 2768, 2560, 2592, 2766] combine_rows_per_src=[5588, 5606, 5522, 5510, 5554, 5454, 5558, 5716] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44560 payload_rows_remote=38988 dup_rows=20976 dup_rows_remote=18362 dedup_ratio=0.320068 router_tokens_per_expert=[471, 543, 498, 508, 508, 522, 535, 492, 498, 520, 502, 480, 502, 542, 517, 517, 523, 507, 553, 511, 531, 485, 496, 487, 508, 490, 525, 532, 512, 536, 490, 525, 493, 520, 521, 512, 523, 514, 503, 494, 484, 545, 513, 494, 496, 505, 516, 506, 503, 525, 536, 520, 515, 582, 481, 488, 490, 536, 491, 534, 540, 514, 486, 522] payload_rows_per_dst=[5462, 5548, 5572, 5628, 5568, 5570, 5620, 5592] dup_rows_per_dst=[2692, 2608, 2614, 2608, 2592, 2548, 2680, 2634] combine_rows_per_src=[5462, 5548, 5572, 5628, 5568, 5570, 5620, 5592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44252 payload_rows_remote=38760 dup_rows=21284 dup_rows_remote=18512 dedup_ratio=0.324768 router_tokens_per_expert=[527, 516, 502, 505, 501, 476, 506, 553, 507, 488, 531, 531, 505, 525, 501, 498, 524, 514, 481, 556, 521, 514, 517, 493, 555, 513, 516, 499, 496, 489, 528, 503, 537, 518, 524, 519, 482, 523, 503, 526, 531, 486, 482, 507, 514, 514, 507, 512, 537, 503, 526, 496, 487, 511, 513, 493, 542, 531, 528, 479, 517, 515, 515, 499] payload_rows_per_dst=[5508, 5582, 5514, 5566, 5492, 5494, 5468, 5628] dup_rows_per_dst=[2664, 2590, 2726, 2632, 2772, 2612, 2664, 2624] combine_rows_per_src=[5508, 5582, 5514, 5566, 5492, 5494, 5468, 5628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44374 payload_rows_remote=38802 dup_rows=21162 dup_rows_remote=18484 dedup_ratio=0.322906 router_tokens_per_expert=[475, 504, 512, 515, 514, 511, 538, 506, 512, 499, 529, 524, 484, 484, 496, 551, 499, 512, 559, 504, 515, 523, 579, 507, 501, 553, 512, 479, 534, 502, 514, 519, 525, 463, 535, 493, 520, 488, 518, 478, 493, 459, 492, 524, 516, 478, 494, 519, 501, 518, 525, 532, 526, 484, 514, 525, 530, 522, 521, 513, 534, 525, 509, 528] payload_rows_per_dst=[5560, 5558, 5568, 5508, 5484, 5406, 5572, 5718] dup_rows_per_dst=[2590, 2600, 2828, 2720, 2556, 2544, 2678, 2646] combine_rows_per_src=[5560, 5558, 5568, 5508, 5484, 5406, 5572, 5718] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44500 payload_rows_remote=38968 dup_rows=21036 dup_rows_remote=18460 dedup_ratio=0.320984 router_tokens_per_expert=[491, 475, 553, 543, 509, 502, 502, 530, 528, 552, 514, 470, 503, 536, 499, 505, 530, 513, 518, 511, 514, 517, 533, 511, 530, 518, 503, 501, 510, 519, 491, 521, 493, 478, 527, 503, 516, 528, 516, 526, 539, 467, 530, 457, 505, 496, 527, 533, 509, 507, 526, 496, 510, 479, 490, 537, 510, 474, 506, 533, 553, 501, 524, 520] payload_rows_per_dst=[5518, 5596, 5606, 5592, 5576, 5532, 5512, 5568] dup_rows_per_dst=[2692, 2618, 2688, 2594, 2598, 2576, 2596, 2674] combine_rows_per_src=[5518, 5596, 5606, 5592, 5576, 5532, 5512, 5568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44612 payload_rows_remote=38964 dup_rows=20924 dup_rows_remote=18306 dedup_ratio=0.319275 router_tokens_per_expert=[540, 520, 506, 538, 494, 500, 533, 500, 499, 534, 535, 511, 481, 501, 483, 506, 500, 529, 474, 490, 496, 493, 538, 526, 510, 513, 534, 497, 522, 533, 499, 525, 515, 536, 511, 521, 494, 512, 515, 509, 520, 501, 507, 536, 479, 503, 543, 493, 517, 525, 513, 501, 516, 474, 509, 532, 489, 529, 515, 521, 527, 519, 503, 523] payload_rows_per_dst=[5626, 5526, 5526, 5648, 5642, 5580, 5502, 5562] dup_rows_per_dst=[2636, 2574, 2566, 2618, 2584, 2584, 2672, 2690] combine_rows_per_src=[5626, 5526, 5526, 5648, 5642, 5580, 5502, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280068096 dispatch_meta_bytes=229896 combine_payload_bytes=280068096 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280297992 ingress_bytes=390168576 link_bytes=390168576 token_bytes=560366088 weight_movement_bytes=110100480 total_nvlink_bytes=670466568 weight_to_token_ratio=0.196480 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.785673254e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44444 payload_rows_remote=38944 dup_rows=21092 dup_rows_remote=18476 dedup_ratio=0.321838 router_tokens_per_expert=[505, 497, 515, 515, 557, 488, 463, 480, 519, 509, 515, 500, 530, 519, 486, 554, 481, 528, 480, 509, 502, 518, 528, 524, 513, 530, 517, 549, 436, 512, 518, 513, 525, 503, 498, 497, 538, 481, 537, 489, 503, 533, 510, 479, 550, 476, 585, 504, 512, 488, 521, 509, 489, 547, 543, 525, 527, 507, 494, 502, 502, 532, 509, 543] payload_rows_per_dst=[5500, 5676, 5534, 5506, 5488, 5914, 5540, 5286] dup_rows_per_dst=[2616, 2764, 2466, 2670, 2648, 3038, 2616, 2274] combine_rows_per_src=[5500, 5676, 5534, 5506, 5488, 5914, 5540, 5286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278061056 dispatch_meta_bytes=228216 combine_payload_bytes=278061056 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278289272 ingress_bytes=498262016 link_bytes=498262016 token_bytes=556350328 weight_movement_bytes=220200960 total_nvlink_bytes=776551288 weight_to_token_ratio=0.395796 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.665595958e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279465984 dispatch_meta_bytes=229400 combine_payload_bytes=279465984 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279695384 ingress_bytes=609767424 link_bytes=609767424 token_bytes=559161368 weight_movement_bytes=330301440 total_nvlink_bytes=889462808 weight_to_token_ratio=0.590709 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060485840e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=277831680 dispatch_meta_bytes=229088 combine_payload_bytes=277831680 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278060768 ingress_bytes=387932160 link_bytes=387932160 token_bytes=555892448 weight_movement_bytes=110100480 total_nvlink_bytes=665992928 weight_to_token_ratio=0.198061 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.746778301e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278132736 dispatch_meta_bytes=229144 combine_payload_bytes=278132736 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278361880 ingress_bytes=608434176 link_bytes=608434176 token_bytes=556494616 weight_movement_bytes=330301440 total_nvlink_bytes=886796056 weight_to_token_ratio=0.593539 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058167102e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279322624 dispatch_meta_bytes=229712 combine_payload_bytes=279322624 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279552336 ingress_bytes=389423104 link_bytes=389423104 token_bytes=558874960 weight_movement_bytes=110100480 total_nvlink_bytes=668975440 weight_to_token_ratio=0.197004 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.772708270e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279293952 dispatch_meta_bytes=229080 combine_payload_bytes=279293952 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279523032 ingress_bytes=499494912 link_bytes=499494912 token_bytes=558816984 weight_movement_bytes=220200960 total_nvlink_bytes=779017944 weight_to_token_ratio=0.394048 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.687038048e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279150592 dispatch_meta_bytes=229680 combine_payload_bytes=279150592 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279380272 ingress_bytes=389251072 link_bytes=389251072 token_bytes=558530864 weight_movement_bytes=110100480 total_nvlink_bytes=668631344 weight_to_token_ratio=0.197125 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.769716350e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44544 payload_rows_remote=38894 dup_rows=20992 dup_rows_remote=18264 dedup_ratio=0.320312 router_tokens_per_expert=[489, 529, 466, 519, 515, 509, 524, 510, 524, 469, 537, 569, 528, 559, 484, 519, 482, 533, 509, 510, 505, 489, 464, 549, 501, 536, 521, 546, 507, 492, 524, 527, 490, 503, 481, 534, 508, 506, 533, 532, 518, 503, 517, 517, 510, 507, 490, 511, 470, 491, 465, 495, 516, 500, 538, 533, 543, 487, 537, 522, 498, 522, 551, 495] payload_rows_per_dst=[5498, 5650, 5490, 5624, 5578, 5570, 5544, 5590] dup_rows_per_dst=[2624, 2728, 2592, 2684, 2596, 2576, 2472, 2720] combine_rows_per_src=[5498, 5650, 5490, 5624, 5578, 5570, 5544, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44560 payload_rows_remote=39066 dup_rows=20976 dup_rows_remote=18404 dedup_ratio=0.320068 router_tokens_per_expert=[534, 509, 511, 516, 500, 476, 495, 514, 568, 510, 556, 538, 546, 491, 509, 504, 543, 488, 506, 487, 518, 543, 488, 493, 519, 514, 546, 520, 517, 521, 515, 501, 511, 491, 489, 496, 517, 489, 512, 526, 525, 496, 516, 527, 474, 510, 494, 519, 540, 531, 538, 500, 486, 558, 491, 504, 497, 503, 512, 511, 520, 522, 460, 507] payload_rows_per_dst=[5568, 5682, 5472, 5548, 5566, 5630, 5600, 5494] dup_rows_per_dst=[2542, 2762, 2464, 2710, 2588, 2642, 2696, 2572] combine_rows_per_src=[5568, 5682, 5472, 5548, 5566, 5630, 5600, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44420 payload_rows_remote=38826 dup_rows=21116 dup_rows_remote=18494 dedup_ratio=0.322205 router_tokens_per_expert=[497, 474, 529, 483, 541, 543, 494, 522, 505, 504, 521, 500, 463, 503, 471, 566, 515, 522, 491, 527, 500, 511, 485, 502, 513, 531, 511, 541, 506, 512, 560, 504, 502, 558, 543, 509, 488, 517, 477, 514, 513, 521, 501, 536, 493, 506, 474, 506, 542, 510, 511, 521, 530, 504, 542, 522, 513, 508, 505, 523, 504, 543, 478, 507] payload_rows_per_dst=[5546, 5530, 5500, 5640, 5594, 5464, 5578, 5568] dup_rows_per_dst=[2620, 2536, 2606, 2716, 2622, 2636, 2786, 2594] combine_rows_per_src=[5546, 5530, 5500, 5640, 5594, 5464, 5578, 5568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278792192 dispatch_meta_bytes=228632 combine_payload_bytes=278792192 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279020824 ingress_bytes=498993152 link_bytes=498993152 token_bytes=557813016 weight_movement_bytes=220200960 total_nvlink_bytes=778013976 weight_to_token_ratio=0.394758 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.678311616e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44478 payload_rows_remote=38780 dup_rows=21058 dup_rows_remote=18366 dedup_ratio=0.321320 router_tokens_per_expert=[474, 497, 513, 524, 528, 504, 523, 546, 487, 503, 481, 560, 508, 562, 505, 542, 547, 520, 484, 523, 545, 545, 507, 524, 500, 506, 520, 501, 507, 500, 559, 503, 538, 489, 517, 510, 507, 492, 504, 544, 522, 528, 500, 506, 502, 519, 514, 505, 520, 493, 524, 474, 468, 484, 529, 522, 516, 546, 468, 498, 491, 496, 486, 508] payload_rows_per_dst=[5558, 5532, 5698, 5542, 5570, 5656, 5500, 5422] dup_rows_per_dst=[2660, 2764, 2692, 2650, 2632, 2536, 2528, 2596] combine_rows_per_src=[5558, 5532, 5698, 5542, 5570, 5656, 5500, 5422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44820 payload_rows_remote=39164 dup_rows=20716 dup_rows_remote=18074 dedup_ratio=0.316101 router_tokens_per_expert=[514, 477, 510, 531, 503, 500, 487, 496, 460, 514, 520, 558, 512, 549, 508, 475, 487, 514, 495, 508, 586, 489, 494, 573, 532, 507, 517, 522, 490, 515, 527, 515, 509, 517, 510, 519, 506, 507, 475, 518, 510, 507, 517, 497, 521, 513, 514, 527, 522, 515, 523, 485, 528, 532, 518, 526, 528, 526, 536, 525, 461, 473, 486, 532] payload_rows_per_dst=[5488, 5604, 5628, 5678, 5598, 5596, 5656, 5572] dup_rows_per_dst=[2548, 2588, 2664, 2572, 2524, 2616, 2642, 2562] combine_rows_per_src=[5488, 5604, 5628, 5678, 5598, 5596, 5656, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44484 payload_rows_remote=38924 dup_rows=21052 dup_rows_remote=18412 dedup_ratio=0.321228 router_tokens_per_expert=[492, 496, 519, 528, 487, 520, 525, 499, 479, 487, 503, 533, 523, 487, 540, 521, 506, 508, 508, 520, 553, 517, 509, 535, 572, 508, 502, 530, 482, 520, 478, 518, 503, 524, 479, 504, 511, 516, 470, 526, 521, 535, 489, 542, 501, 496, 510, 506, 506, 516, 504, 503, 525, 489, 483, 533, 555, 499, 522, 521, 551, 503, 491, 529] payload_rows_per_dst=[5540, 5544, 5594, 5562, 5506, 5560, 5502, 5676] dup_rows_per_dst=[2592, 2602, 2718, 2658, 2560, 2640, 2616, 2666] combine_rows_per_src=[5540, 5544, 5594, 5562, 5506, 5560, 5502, 5676] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44444 payload_rows_remote=38852 dup_rows=21092 dup_rows_remote=18476 dedup_ratio=0.321838 router_tokens_per_expert=[500, 475, 531, 484, 544, 546, 498, 526, 508, 507, 520, 501, 464, 503, 473, 571, 515, 523, 491, 529, 499, 510, 485, 504, 514, 531, 511, 541, 507, 510, 562, 505, 502, 559, 542, 510, 487, 516, 475, 513, 513, 521, 497, 535, 494, 503, 474, 502, 543, 506, 511, 520, 533, 502, 538, 519, 514, 505, 505, 521, 501, 539, 476, 504] payload_rows_per_dst=[5568, 5542, 5500, 5644, 5592, 5458, 5578, 5562] dup_rows_per_dst=[2640, 2552, 2612, 2718, 2616, 2620, 2766, 2568] combine_rows_per_src=[5568, 5542, 5500, 5644, 5592, 5458, 5578, 5562] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44822 payload_rows_remote=39170 dup_rows=20714 dup_rows_remote=18092 dedup_ratio=0.316071 router_tokens_per_expert=[516, 479, 511, 532, 507, 502, 491, 499, 459, 517, 522, 561, 514, 551, 510, 478, 490, 514, 498, 509, 587, 490, 495, 574, 533, 508, 517, 525, 492, 514, 527, 517, 509, 516, 512, 518, 504, 506, 477, 515, 506, 506, 513, 494, 520, 512, 514, 526, 522, 515, 522, 482, 523, 532, 516, 525, 527, 522, 534, 520, 460, 470, 482, 529] payload_rows_per_dst=[5508, 5618, 5640, 5678, 5596, 5578, 5652, 5552] dup_rows_per_dst=[2566, 2606, 2674, 2588, 2518, 2604, 2622, 2536] combine_rows_per_src=[5508, 5618, 5640, 5678, 5596, 5578, 5652, 5552] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279322624 dispatch_meta_bytes=228776 combine_payload_bytes=279322624 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279551400 ingress_bytes=389423104 link_bytes=389423104 token_bytes=558874024 weight_movement_bytes=110100480 total_nvlink_bytes=668974504 weight_to_token_ratio=0.197004 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.772708270e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279107584 dispatch_meta_bytes=229432 combine_payload_bytes=279107584 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279337016 ingress_bytes=389208064 link_bytes=389208064 token_bytes=558444600 weight_movement_bytes=110100480 total_nvlink_bytes=668545080 weight_to_token_ratio=0.197156 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.768968370e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279308288 dispatch_meta_bytes=230104 combine_payload_bytes=279308288 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279538392 ingress_bytes=609609728 link_bytes=609609728 token_bytes=558846680 weight_movement_bytes=330301440 total_nvlink_bytes=889148120 weight_to_token_ratio=0.591041 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060211581e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277745664 dispatch_meta_bytes=228544 combine_payload_bytes=277745664 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277974208 ingress_bytes=497946624 link_bytes=497946624 token_bytes=555719872 weight_movement_bytes=220200960 total_nvlink_bytes=775920832 weight_to_token_ratio=0.396245 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.660110773e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44646 payload_rows_remote=39000 dup_rows=20890 dup_rows_remote=18194 dedup_ratio=0.318756 router_tokens_per_expert=[529, 541, 521, 507, 531, 528, 509, 487, 531, 555, 542, 503, 491, 500, 473, 511, 535, 544, 467, 518, 530, 505, 510, 489, 496, 495, 514, 511, 488, 523, 483, 545, 538, 527, 536, 513, 522, 493, 531, 471, 534, 475, 507, 499, 527, 524, 500, 545, 531, 474, 554, 473, 492, 533, 505, 490, 510, 529, 510, 520, 491, 496, 515, 491] payload_rows_per_dst=[5646, 5554, 5608, 5532, 5610, 5628, 5496, 5572] dup_rows_per_dst=[2696, 2658, 2588, 2542, 2652, 2594, 2608, 2552] combine_rows_per_src=[5646, 5554, 5608, 5532, 5610, 5628, 5496, 5572] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278491136 dispatch_meta_bytes=229312 combine_payload_bytes=278491136 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278720448 ingress_bytes=388591616 link_bytes=388591616 token_bytes=557211584 weight_movement_bytes=110100480 total_nvlink_bytes=667312064 weight_to_token_ratio=0.197592 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.758247326e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=280770560 dispatch_meta_bytes=229048 combine_payload_bytes=280770560 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280999608 ingress_bytes=390871040 link_bytes=390871040 token_bytes=561770168 weight_movement_bytes=110100480 total_nvlink_bytes=671870648 weight_to_token_ratio=0.195988 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.797890258e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44518 payload_rows_remote=38858 dup_rows=21018 dup_rows_remote=18272 dedup_ratio=0.320709 router_tokens_per_expert=[492, 529, 471, 520, 518, 508, 531, 514, 524, 470, 540, 570, 529, 560, 489, 521, 484, 535, 509, 511, 507, 489, 463, 552, 502, 537, 523, 545, 509, 492, 523, 525, 491, 505, 479, 534, 506, 506, 532, 532, 514, 503, 515, 518, 508, 505, 486, 508, 468, 490, 463, 498, 515, 499, 534, 531, 544, 486, 536, 520, 496, 519, 544, 491] payload_rows_per_dst=[5512, 5660, 5492, 5622, 5576, 5554, 5530, 5572] dup_rows_per_dst=[2654, 2746, 2608, 2690, 2594, 2560, 2466, 2700] combine_rows_per_src=[5512, 5660, 5492, 5622, 5576, 5554, 5530, 5572] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279552000 dispatch_meta_bytes=228776 combine_payload_bytes=279552000 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279780776 ingress_bytes=389652480 link_bytes=389652480 token_bytes=559332776 weight_movement_bytes=110100480 total_nvlink_bytes=669433256 weight_to_token_ratio=0.196843 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.776697495e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44296 payload_rows_remote=38550 dup_rows=21240 dup_rows_remote=18232 dedup_ratio=0.324097 router_tokens_per_expert=[516, 516, 540, 523, 521, 502, 482, 524, 559, 532, 480, 490, 487, 509, 542, 521, 536, 506, 509, 511, 554, 494, 512, 521, 506, 491, 478, 512, 526, 491, 526, 501, 511, 508, 524, 461, 534, 567, 561, 494, 527, 507, 508, 487, 531, 495, 518, 475, 482, 518, 511, 519, 519, 521, 494, 516, 478, 506, 537, 536, 492, 501, 485, 527] payload_rows_per_dst=[5556, 5218, 5402, 5492, 5606, 5606, 5670, 5746] dup_rows_per_dst=[2692, 2152, 2474, 2558, 2726, 2730, 2900, 3008] combine_rows_per_src=[5556, 5218, 5402, 5492, 5606, 5606, 5670, 5746] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44578 payload_rows_remote=39088 dup_rows=20958 dup_rows_remote=18382 dedup_ratio=0.319794 router_tokens_per_expert=[506, 546, 509, 514, 534, 542, 511, 493, 537, 524, 533, 515, 525, 500, 513, 526, 483, 524, 508, 500, 529, 516, 519, 519, 479, 533, 519, 517, 526, 518, 514, 530, 531, 500, 492, 515, 480, 541, 529, 537, 502, 513, 494, 518, 487, 497, 501, 521, 509, 469, 519, 488, 506, 522, 525, 498, 506, 506, 508, 498, 505, 512, 496, 481] payload_rows_per_dst=[5604, 5686, 5544, 5610, 5644, 5490, 5544, 5456] dup_rows_per_dst=[2706, 2660, 2652, 2662, 2606, 2576, 2528, 2568] combine_rows_per_src=[5604, 5686, 5544, 5610, 5644, 5490, 5544, 5456] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278534144 dispatch_meta_bytes=228520 combine_payload_bytes=278534144 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=278762664 ingress_bytes=718936064 link_bytes=718936064 token_bytes=557296808 weight_movement_bytes=440401920 total_nvlink_bytes=997698728 weight_to_token_ratio=0.790247 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.250348060e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44438 payload_rows_remote=38970 dup_rows=21098 dup_rows_remote=18598 dedup_ratio=0.321930 router_tokens_per_expert=[512, 505, 530, 486, 528, 523, 496, 484, 515, 521, 552, 475, 519, 526, 491, 502, 485, 524, 465, 479, 485, 503, 522, 521, 499, 493, 476, 506, 544, 471, 527, 525, 538, 494, 549, 509, 523, 537, 520, 526, 534, 517, 502, 489, 545, 534, 490, 513, 530, 524, 476, 501, 496, 497, 530, 568, 535, 510, 537, 492, 511, 502, 521, 528] payload_rows_per_dst=[5482, 5582, 5468, 5520, 5618, 5620, 5508, 5640] dup_rows_per_dst=[2646, 2620, 2500, 2562, 2774, 2628, 2736, 2632] combine_rows_per_src=[5482, 5582, 5468, 5520, 5618, 5620, 5508, 5640] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44454 payload_rows_remote=38878 dup_rows=21082 dup_rows_remote=18438 dedup_ratio=0.321686 router_tokens_per_expert=[509, 498, 516, 529, 509, 507, 477, 504, 519, 511, 532, 500, 473, 492, 537, 550, 541, 481, 532, 522, 524, 489, 538, 508, 508, 513, 510, 526, 548, 492, 501, 512, 544, 491, 543, 490, 502, 502, 502, 504, 496, 523, 506, 508, 531, 495, 503, 518, 498, 527, 519, 512, 508, 513, 502, 524, 496, 488, 482, 517, 543, 528, 516, 529] payload_rows_per_dst=[5528, 5570, 5608, 5576, 5524, 5630, 5448, 5570] dup_rows_per_dst=[2570, 2658, 2662, 2644, 2632, 2530, 2758, 2628] combine_rows_per_src=[5528, 5570, 5608, 5576, 5524, 5630, 5448, 5570] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44490 payload_rows_remote=38938 dup_rows=21046 dup_rows_remote=18446 dedup_ratio=0.321136 router_tokens_per_expert=[497, 501, 500, 505, 535, 508, 518, 515, 526, 500, 519, 525, 497, 495, 528, 461, 532, 522, 550, 514, 525, 511, 513, 548, 506, 512, 492, 481, 519, 510, 553, 534, 509, 496, 520, 472, 512, 507, 513, 547, 516, 510, 510, 522, 536, 489, 535, 513, 539, 511, 497, 544, 498, 516, 493, 469, 557, 449, 504, 527, 527, 498, 482, 498] payload_rows_per_dst=[5562, 5528, 5638, 5624, 5552, 5494, 5588, 5504] dup_rows_per_dst=[2596, 2574, 2792, 2590, 2600, 2768, 2546, 2580] combine_rows_per_src=[5562, 5528, 5638, 5624, 5552, 5494, 5588, 5504] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44586 payload_rows_remote=39084 dup_rows=20950 dup_rows_remote=18434 dedup_ratio=0.319672 router_tokens_per_expert=[551, 471, 506, 533, 505, 494, 519, 526, 511, 560, 533, 496, 553, 538, 499, 530, 494, 506, 491, 524, 495, 503, 533, 534, 495, 527, 499, 540, 506, 522, 536, 510, 510, 514, 481, 488, 561, 494, 486, 514, 490, 510, 549, 517, 508, 510, 540, 521, 528, 475, 493, 496, 507, 503, 512, 495, 491, 479, 510, 542, 473, 516, 528, 487] payload_rows_per_dst=[5582, 5676, 5564, 5604, 5544, 5602, 5502, 5512] dup_rows_per_dst=[2628, 2764, 2596, 2666, 2552, 2688, 2516, 2540] combine_rows_per_src=[5582, 5676, 5564, 5604, 5544, 5602, 5502, 5512] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44572 payload_rows_remote=39018 dup_rows=20964 dup_rows_remote=18344 dedup_ratio=0.319885 router_tokens_per_expert=[498, 542, 477, 510, 488, 504, 536, 531, 520, 525, 499, 513, 532, 522, 514, 542, 490, 516, 523, 510, 574, 485, 530, 539, 517, 498, 522, 500, 486, 483, 538, 511, 525, 507, 497, 492, 503, 517, 521, 524, 520, 472, 529, 539, 483, 520, 484, 486, 488, 512, 539, 524, 521, 541, 509, 491, 490, 515, 521, 510, 471, 520, 508, 514] payload_rows_per_dst=[5554, 5680, 5576, 5556, 5526, 5498, 5632, 5550] dup_rows_per_dst=[2620, 2756, 2790, 2552, 2512, 2568, 2618, 2548] combine_rows_per_src=[5554, 5680, 5576, 5556, 5526, 5498, 5632, 5550] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=276326400 dispatch_meta_bytes=227128 combine_payload_bytes=276326400 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276553528 ingress_bytes=386426880 link_bytes=386426880 token_bytes=552879928 weight_movement_bytes=110100480 total_nvlink_bytes=662980408 weight_to_token_ratio=0.199140 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.720599006e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280182784 dispatch_meta_bytes=229880 combine_payload_bytes=280182784 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280412664 ingress_bytes=390283264 link_bytes=390283264 token_bytes=560595448 weight_movement_bytes=110100480 total_nvlink_bytes=670695928 weight_to_token_ratio=0.196399 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.787667867e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279336960 dispatch_meta_bytes=230272 combine_payload_bytes=279336960 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279567232 ingress_bytes=499537920 link_bytes=499537920 token_bytes=558904192 weight_movement_bytes=220200960 total_nvlink_bytes=779105152 weight_to_token_ratio=0.393987 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.687786027e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278677504 dispatch_meta_bytes=229264 combine_payload_bytes=278677504 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=278906768 ingress_bytes=498878464 link_bytes=498878464 token_bytes=557584272 weight_movement_bytes=220200960 total_nvlink_bytes=777785232 weight_to_token_ratio=0.394920 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.676317003e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279107584 dispatch_meta_bytes=229536 combine_payload_bytes=279107584 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279337120 ingress_bytes=609409024 link_bytes=609409024 token_bytes=558444704 weight_movement_bytes=330301440 total_nvlink_bytes=888746144 weight_to_token_ratio=0.591467 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059862523e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280154112 dispatch_meta_bytes=230072 combine_payload_bytes=280154112 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280384184 ingress_bytes=390254592 link_bytes=390254592 token_bytes=560538296 weight_movement_bytes=110100480 total_nvlink_bytes=670638776 weight_to_token_ratio=0.196419 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.787169213e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279681024 dispatch_meta_bytes=229448 combine_payload_bytes=279681024 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279910472 ingress_bytes=389781504 link_bytes=389781504 token_bytes=559591496 weight_movement_bytes=110100480 total_nvlink_bytes=669691976 weight_to_token_ratio=0.196752 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778941435e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44562 payload_rows_remote=38934 dup_rows=20974 dup_rows_remote=18190 dedup_ratio=0.320038 router_tokens_per_expert=[539, 491, 504, 528, 498, 507, 501, 537, 510, 485, 535, 552, 557, 538, 521, 508, 510, 510, 499, 477, 538, 546, 540, 515, 508, 532, 549, 547, 487, 541, 497, 499, 539, 503, 513, 476, 500, 516, 494, 519, 464, 505, 500, 529, 512, 496, 540, 508, 507, 494, 490, 527, 532, 458, 500, 513, 530, 498, 513, 466, 504, 527, 476, 513] payload_rows_per_dst=[5604, 5628, 5642, 5592, 5544, 5486, 5532, 5534] dup_rows_per_dst=[2606, 2784, 2628, 2728, 2576, 2622, 2510, 2520] combine_rows_per_src=[5604, 5628, 5642, 5592, 5544, 5486, 5532, 5534] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44404 payload_rows_remote=38732 dup_rows=21132 dup_rows_remote=18432 dedup_ratio=0.322449 router_tokens_per_expert=[544, 572, 497, 547, 513, 501, 490, 513, 516, 553, 529, 558, 514, 503, 490, 500, 498, 525, 502, 492, 542, 490, 556, 476, 502, 527, 527, 518, 485, 505, 492, 526, 509, 500, 513, 491, 550, 517, 522, 463, 513, 512, 509, 515, 527, 509, 509, 521, 523, 505, 511, 512, 484, 466, 515, 507, 467, 502, 504, 551, 485, 518, 520, 515] payload_rows_per_dst=[5266, 5430, 5570, 5540, 5530, 5620, 5776, 5672] dup_rows_per_dst=[2316, 2580, 2564, 2664, 2646, 2620, 3042, 2700] combine_rows_per_src=[5266, 5430, 5570, 5540, 5530, 5620, 5776, 5672] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44372 payload_rows_remote=38770 dup_rows=21164 dup_rows_remote=18458 dedup_ratio=0.322937 router_tokens_per_expert=[491, 524, 519, 534, 522, 509, 505, 549, 521, 507, 546, 499, 519, 548, 546, 551, 548, 520, 508, 505, 491, 509, 503, 506, 476, 501, 499, 507, 505, 564, 545, 509, 501, 548, 550, 535, 533, 491, 456, 516, 482, 538, 526, 535, 533, 533, 513, 494, 522, 478, 483, 522, 487, 480, 492, 501, 500, 480, 505, 509, 493, 483, 479, 484] payload_rows_per_dst=[5572, 5652, 5520, 5596, 5588, 5602, 5440, 5402] dup_rows_per_dst=[2734, 2822, 2660, 2616, 2672, 2706, 2490, 2464] combine_rows_per_src=[5572, 5652, 5520, 5596, 5588, 5602, 5440, 5402] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44508 payload_rows_remote=39000 dup_rows=21028 dup_rows_remote=18452 dedup_ratio=0.320862 router_tokens_per_expert=[507, 536, 541, 518, 491, 543, 524, 550, 507, 522, 498, 525, 516, 527, 544, 520, 514, 528, 480, 523, 506, 516, 454, 521, 518, 484, 483, 488, 519, 514, 492, 563, 510, 526, 492, 523, 496, 520, 519, 526, 495, 505, 494, 492, 543, 538, 495, 539, 514, 513, 505, 525, 481, 521, 493, 504, 512, 494, 488, 492, 491, 514, 510, 526] payload_rows_per_dst=[5678, 5644, 5508, 5480, 5582, 5596, 5550, 5470] dup_rows_per_dst=[2742, 2674, 2576, 2642, 2642, 2606, 2562, 2584] combine_rows_per_src=[5678, 5644, 5508, 5480, 5582, 5596, 5550, 5470] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279078912 dispatch_meta_bytes=228496 combine_payload_bytes=279078912 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279307408 ingress_bytes=609380352 link_bytes=609380352 token_bytes=558386320 weight_movement_bytes=330301440 total_nvlink_bytes=888687760 weight_to_token_ratio=0.591529 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059812658e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44690 payload_rows_remote=39102 dup_rows=20846 dup_rows_remote=18296 dedup_ratio=0.318085 router_tokens_per_expert=[495, 493, 493, 569, 500, 469, 542, 533, 517, 539, 498, 536, 516, 470, 495, 518, 520, 518, 545, 504, 486, 527, 497, 532, 524, 506, 493, 497, 498, 524, 536, 491, 464, 499, 554, 514, 504, 531, 518, 496, 496, 523, 496, 548, 488, 512, 526, 518, 512, 508, 505, 513, 502, 521, 474, 529, 531, 527, 504, 530, 519, 497, 510, 518] payload_rows_per_dst=[5596, 5592, 5614, 5588, 5566, 5546, 5578, 5610] dup_rows_per_dst=[2592, 2586, 2644, 2550, 2594, 2668, 2550, 2662] combine_rows_per_src=[5596, 5592, 5614, 5588, 5566, 5546, 5578, 5610] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44422 payload_rows_remote=38836 dup_rows=21114 dup_rows_remote=18464 dedup_ratio=0.322174 router_tokens_per_expert=[506, 485, 520, 537, 523, 567, 510, 498, 518, 542, 531, 504, 506, 525, 476, 490, 502, 479, 542, 506, 508, 504, 567, 510, 504, 525, 510, 496, 505, 533, 522, 493, 558, 542, 493, 547, 488, 505, 483, 494, 524, 521, 497, 531, 473, 515, 482, 504, 534, 539, 490, 499, 512, 538, 492, 514, 506, 529, 517, 443, 549, 501, 485, 519] payload_rows_per_dst=[5584, 5538, 5544, 5566, 5584, 5500, 5586, 5520] dup_rows_per_dst=[2708, 2646, 2692, 2610, 2636, 2594, 2650, 2578] combine_rows_per_src=[5584, 5538, 5544, 5566, 5584, 5500, 5586, 5520] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44466 payload_rows_remote=38844 dup_rows=21070 dup_rows_remote=18412 dedup_ratio=0.321503 router_tokens_per_expert=[550, 506, 488, 524, 553, 530, 494, 539, 516, 508, 547, 506, 506, 510, 528, 465, 504, 515, 478, 521, 554, 487, 507, 512, 546, 464, 538, 483, 523, 553, 521, 528, 500, 512, 503, 515, 532, 524, 553, 501, 537, 511, 500, 509, 528, 516, 503, 473, 514, 479, 502, 516, 494, 491, 511, 498, 512, 528, 506, 490, 490, 528, 479, 509] payload_rows_per_dst=[5614, 5524, 5554, 5562, 5622, 5574, 5490, 5526] dup_rows_per_dst=[2754, 2648, 2602, 2750, 2658, 2580, 2520, 2558] combine_rows_per_src=[5614, 5524, 5554, 5562, 5622, 5574, 5490, 5526] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44330 payload_rows_remote=38820 dup_rows=21206 dup_rows_remote=18604 dedup_ratio=0.323578 router_tokens_per_expert=[486, 546, 505, 473, 480, 512, 526, 504, 482, 502, 509, 506, 492, 541, 504, 495, 530, 512, 501, 505, 514, 515, 544, 497, 531, 500, 510, 505, 501, 513, 518, 496, 475, 485, 511, 526, 526, 490, 508, 524, 504, 508, 509, 475, 545, 516, 550, 512, 522, 510, 499, 491, 512, 512, 510, 526, 553, 525, 507, 551, 515, 558, 522, 536] payload_rows_per_dst=[5510, 5468, 5558, 5550, 5480, 5532, 5534, 5698] dup_rows_per_dst=[2602, 2546, 2678, 2598, 2610, 2706, 2630, 2836] combine_rows_per_src=[5510, 5468, 5558, 5550, 5480, 5532, 5534, 5698] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279552000 dispatch_meta_bytes=229808 combine_payload_bytes=279552000 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279781808 ingress_bytes=499752960 link_bytes=499752960 token_bytes=559333808 weight_movement_bytes=220200960 total_nvlink_bytes=779534768 weight_to_token_ratio=0.393684 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.691525927e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277630976 dispatch_meta_bytes=228656 combine_payload_bytes=277630976 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277859632 ingress_bytes=387731456 link_bytes=387731456 token_bytes=555490608 weight_movement_bytes=110100480 total_nvlink_bytes=665591088 weight_to_token_ratio=0.198204 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.743287729e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277903360 dispatch_meta_bytes=228912 combine_payload_bytes=277903360 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278132272 ingress_bytes=388003840 link_bytes=388003840 token_bytes=556035632 weight_movement_bytes=110100480 total_nvlink_bytes=666136112 weight_to_token_ratio=0.198010 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.748024934e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44414 payload_rows_remote=38956 dup_rows=21122 dup_rows_remote=18570 dedup_ratio=0.322296 router_tokens_per_expert=[463, 479, 518, 492, 531, 513, 525, 538, 509, 482, 494, 540, 508, 514, 472, 505, 488, 515, 551, 508, 501, 513, 502, 539, 486, 507, 516, 477, 495, 508, 509, 507, 509, 504, 552, 518, 551, 498, 517, 522, 516, 490, 497, 518, 502, 524, 526, 517, 483, 490, 532, 501, 534, 517, 526, 553, 495, 495, 525, 494, 538, 577, 493, 549] payload_rows_per_dst=[5524, 5464, 5636, 5458, 5638, 5534, 5578, 5582] dup_rows_per_dst=[2594, 2584, 2598, 2552, 2704, 2646, 2694, 2750] combine_rows_per_src=[5524, 5464, 5636, 5458, 5638, 5534, 5578, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=280025088 dispatch_meta_bytes=229880 combine_payload_bytes=280025088 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280254968 ingress_bytes=500226048 link_bytes=500226048 token_bytes=560280056 weight_movement_bytes=220200960 total_nvlink_bytes=780481016 weight_to_token_ratio=0.393019 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.699753705e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278304768 dispatch_meta_bytes=229280 combine_payload_bytes=278304768 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278534048 ingress_bytes=388405248 link_bytes=388405248 token_bytes=556838816 weight_movement_bytes=110100480 total_nvlink_bytes=666939296 weight_to_token_ratio=0.197724 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.755006080e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44650 payload_rows_remote=38900 dup_rows=20886 dup_rows_remote=18002 dedup_ratio=0.318695 router_tokens_per_expert=[528, 538, 517, 505, 529, 524, 505, 482, 531, 553, 540, 503, 491, 496, 470, 510, 534, 539, 469, 518, 529, 503, 511, 489, 498, 495, 512, 508, 486, 523, 483, 544, 539, 526, 536, 513, 522, 495, 531, 468, 534, 473, 507, 497, 534, 524, 501, 547, 532, 473, 557, 476, 494, 537, 507, 492, 512, 534, 513, 525, 495, 500, 516, 495] payload_rows_per_dst=[5750, 5474, 5600, 5480, 5608, 5634, 5512, 5592] dup_rows_per_dst=[2884, 2492, 2584, 2462, 2652, 2600, 2632, 2580] combine_rows_per_src=[5750, 5474, 5600, 5480, 5608, 5634, 5512, 5592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=280727552 dispatch_meta_bytes=228952 combine_payload_bytes=280727552 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280956504 ingress_bytes=390828032 link_bytes=390828032 token_bytes=561684056 weight_movement_bytes=110100480 total_nvlink_bytes=671784536 weight_to_token_ratio=0.196019 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.797142278e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277975040 dispatch_meta_bytes=228584 combine_payload_bytes=277975040 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278203624 ingress_bytes=608276480 link_bytes=608276480 token_bytes=556178664 weight_movement_bytes=330301440 total_nvlink_bytes=886480104 weight_to_token_ratio=0.593877 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057892843e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279007232 dispatch_meta_bytes=229344 combine_payload_bytes=279007232 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279236576 ingress_bytes=389107712 link_bytes=389107712 token_bytes=558243808 weight_movement_bytes=110100480 total_nvlink_bytes=668344288 weight_to_token_ratio=0.197227 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.767223084e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44552 payload_rows_remote=38942 dup_rows=20984 dup_rows_remote=18218 dedup_ratio=0.320190 router_tokens_per_expert=[534, 489, 501, 521, 497, 505, 498, 533, 506, 484, 533, 551, 552, 537, 519, 506, 507, 510, 496, 478, 537, 545, 541, 515, 508, 530, 551, 547, 487, 537, 498, 498, 538, 505, 512, 476, 500, 516, 496, 520, 465, 504, 504, 531, 514, 494, 541, 508, 508, 495, 492, 529, 534, 460, 500, 515, 536, 499, 517, 470, 509, 527, 480, 522] payload_rows_per_dst=[5588, 5610, 5636, 5584, 5548, 5486, 5542, 5558] dup_rows_per_dst=[2568, 2766, 2622, 2728, 2578, 2636, 2524, 2562] combine_rows_per_src=[5588, 5610, 5636, 5584, 5548, 5486, 5542, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279236608 dispatch_meta_bytes=230104 combine_payload_bytes=279236608 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279466712 ingress_bytes=609538048 link_bytes=609538048 token_bytes=558703320 weight_movement_bytes=330301440 total_nvlink_bytes=889004760 weight_to_token_ratio=0.591193 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060086917e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278835200 dispatch_meta_bytes=227608 combine_payload_bytes=278835200 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279062808 ingress_bytes=388935680 link_bytes=388935680 token_bytes=557898008 weight_movement_bytes=110100480 total_nvlink_bytes=667998488 weight_to_token_ratio=0.197349 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.764231164e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279136256 dispatch_meta_bytes=228640 combine_payload_bytes=279136256 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279364896 ingress_bytes=609437696 link_bytes=609437696 token_bytes=558501152 weight_movement_bytes=330301440 total_nvlink_bytes=888802592 weight_to_token_ratio=0.591407 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059912389e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44468 payload_rows_remote=38912 dup_rows=21068 dup_rows_remote=18472 dedup_ratio=0.321472 router_tokens_per_expert=[494, 500, 495, 501, 535, 509, 516, 512, 525, 495, 511, 525, 495, 494, 527, 461, 530, 519, 547, 511, 524, 511, 513, 546, 507, 512, 492, 481, 518, 505, 554, 534, 511, 497, 520, 469, 512, 506, 513, 548, 517, 508, 511, 523, 539, 493, 536, 513, 538, 514, 500, 544, 502, 520, 495, 471, 558, 453, 506, 534, 527, 500, 486, 505] payload_rows_per_dst=[5542, 5508, 5630, 5620, 5556, 5486, 5602, 5524] dup_rows_per_dst=[2582, 2558, 2772, 2586, 2596, 2794, 2566, 2614] combine_rows_per_src=[5542, 5508, 5630, 5620, 5556, 5486, 5602, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44308 payload_rows_remote=38682 dup_rows=21228 dup_rows_remote=18418 dedup_ratio=0.323914 router_tokens_per_expert=[514, 514, 537, 515, 520, 499, 482, 523, 554, 531, 479, 488, 487, 504, 542, 520, 536, 505, 507, 512, 550, 493, 512, 517, 506, 493, 479, 509, 525, 492, 526, 502, 516, 508, 523, 460, 529, 568, 561, 495, 526, 508, 509, 488, 533, 497, 521, 475, 483, 519, 512, 518, 522, 522, 495, 519, 479, 506, 541, 539, 499, 504, 491, 529] payload_rows_per_dst=[5538, 5492, 5462, 5496, 5546, 5554, 5594, 5626] dup_rows_per_dst=[2670, 2538, 2542, 2566, 2660, 2676, 2766, 2810] combine_rows_per_src=[5538, 5492, 5462, 5496, 5546, 5554, 5594, 5626] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44586 payload_rows_remote=39078 dup_rows=20950 dup_rows_remote=18410 dedup_ratio=0.319672 router_tokens_per_expert=[544, 466, 502, 530, 502, 491, 517, 521, 508, 560, 530, 494, 551, 536, 496, 531, 490, 505, 489, 522, 496, 503, 534, 531, 492, 526, 501, 539, 504, 526, 536, 512, 511, 513, 483, 488, 563, 497, 489, 513, 491, 512, 550, 519, 509, 510, 542, 523, 528, 480, 494, 496, 508, 505, 516, 497, 493, 482, 514, 542, 475, 518, 532, 490] payload_rows_per_dst=[5550, 5664, 5562, 5600, 5552, 5614, 5508, 5536] dup_rows_per_dst=[2596, 2748, 2578, 2672, 2562, 2698, 2540, 2556] combine_rows_per_src=[5550, 5664, 5562, 5600, 5552, 5614, 5508, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44440 payload_rows_remote=38984 dup_rows=21096 dup_rows_remote=18608 dedup_ratio=0.321899 router_tokens_per_expert=[510, 503, 529, 483, 520, 522, 492, 483, 514, 519, 546, 472, 518, 523, 491, 497, 481, 519, 467, 477, 485, 503, 521, 519, 498, 493, 476, 507, 547, 472, 525, 525, 541, 492, 551, 511, 524, 536, 520, 528, 535, 514, 503, 493, 547, 532, 491, 515, 531, 523, 481, 503, 500, 497, 531, 568, 538, 512, 543, 494, 514, 503, 529, 531] payload_rows_per_dst=[5468, 5570, 5456, 5518, 5626, 5632, 5512, 5658] dup_rows_per_dst=[2616, 2590, 2488, 2568, 2780, 2628, 2756, 2670] combine_rows_per_src=[5468, 5570, 5456, 5518, 5626, 5632, 5512, 5658] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44598 payload_rows_remote=39102 dup_rows=20938 dup_rows_remote=18366 dedup_ratio=0.319489 router_tokens_per_expert=[504, 543, 509, 512, 530, 543, 508, 492, 533, 525, 535, 514, 523, 498, 512, 524, 481, 523, 506, 499, 530, 518, 515, 518, 478, 531, 521, 515, 525, 520, 518, 530, 530, 499, 491, 513, 481, 541, 531, 539, 502, 515, 494, 514, 489, 497, 501, 522, 511, 468, 519, 487, 507, 524, 527, 502, 509, 507, 509, 502, 507, 516, 497, 484] payload_rows_per_dst=[5592, 5684, 5542, 5612, 5648, 5496, 5550, 5474] dup_rows_per_dst=[2690, 2644, 2638, 2664, 2602, 2572, 2540, 2588] combine_rows_per_src=[5592, 5684, 5542, 5612, 5648, 5496, 5550, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278921216 dispatch_meta_bytes=229536 combine_payload_bytes=278921216 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279150752 ingress_bytes=609222656 link_bytes=609222656 token_bytes=558071968 weight_movement_bytes=330301440 total_nvlink_bytes=888373408 weight_to_token_ratio=0.591862 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059538399e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277272576 dispatch_meta_bytes=228400 combine_payload_bytes=277272576 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277500976 ingress_bytes=387373056 link_bytes=387373056 token_bytes=554773552 weight_movement_bytes=110100480 total_nvlink_bytes=664874032 weight_to_token_ratio=0.198460 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.737054563e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280111104 dispatch_meta_bytes=229952 combine_payload_bytes=280111104 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280341056 ingress_bytes=390211584 link_bytes=390211584 token_bytes=560452160 weight_movement_bytes=110100480 total_nvlink_bytes=670552640 weight_to_token_ratio=0.196449 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.786421234e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44448 payload_rows_remote=38880 dup_rows=21088 dup_rows_remote=18446 dedup_ratio=0.321777 router_tokens_per_expert=[503, 497, 514, 528, 507, 505, 476, 501, 519, 509, 527, 493, 471, 490, 537, 548, 539, 480, 531, 520, 524, 489, 537, 507, 509, 514, 510, 523, 548, 491, 499, 511, 546, 490, 543, 489, 504, 501, 503, 503, 496, 526, 507, 508, 533, 496, 506, 519, 499, 528, 520, 516, 511, 517, 507, 530, 495, 491, 484, 520, 544, 528, 520, 531] payload_rows_per_dst=[5514, 5554, 5604, 5568, 5524, 5636, 5474, 5574] dup_rows_per_dst=[2548, 2634, 2650, 2642, 2634, 2546, 2782, 2652] combine_rows_per_src=[5514, 5554, 5604, 5568, 5524, 5636, 5474, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44570 payload_rows_remote=38894 dup_rows=20966 dup_rows_remote=18168 dedup_ratio=0.319916 router_tokens_per_expert=[495, 541, 476, 507, 486, 502, 535, 530, 516, 525, 498, 511, 531, 520, 513, 538, 488, 514, 520, 505, 572, 486, 526, 540, 515, 497, 522, 500, 488, 485, 538, 512, 523, 507, 496, 492, 504, 518, 523, 525, 518, 476, 528, 540, 484, 521, 484, 490, 491, 513, 540, 527, 525, 542, 510, 492, 493, 518, 523, 511, 475, 523, 508, 516] payload_rows_per_dst=[5676, 5468, 5570, 5562, 5580, 5506, 5644, 5564] dup_rows_per_dst=[2798, 2508, 2784, 2550, 2544, 2576, 2636, 2570] combine_rows_per_src=[5676, 5468, 5570, 5562, 5580, 5506, 5644, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279437312 dispatch_meta_bytes=230368 combine_payload_bytes=279437312 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279667680 ingress_bytes=499638272 link_bytes=499638272 token_bytes=559104992 weight_movement_bytes=220200960 total_nvlink_bytes=779305952 weight_to_token_ratio=0.393845 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.689531314e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280283136 dispatch_meta_bytes=229872 combine_payload_bytes=280283136 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280513008 ingress_bytes=390383616 link_bytes=390383616 token_bytes=560796144 weight_movement_bytes=110100480 total_nvlink_bytes=670896624 weight_to_token_ratio=0.196329 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.789413153e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44514 payload_rows_remote=39018 dup_rows=21022 dup_rows_remote=18346 dedup_ratio=0.320770 router_tokens_per_expert=[545, 522, 497, 540, 477, 531, 489, 512, 513, 519, 494, 504, 507, 489, 541, 519, 524, 494, 506, 517, 529, 487, 509, 545, 524, 506, 541, 538, 496, 498, 485, 515, 500, 462, 483, 547, 532, 528, 475, 541, 536, 495, 528, 533, 522, 478, 486, 475, 485, 494, 539, 514, 522, 485, 531, 515, 514, 536, 511, 521, 555, 478, 528, 506] payload_rows_per_dst=[5590, 5496, 5584, 5500, 5556, 5566, 5600, 5622] dup_rows_per_dst=[2636, 2676, 2638, 2706, 2580, 2540, 2570, 2676] combine_rows_per_src=[5590, 5496, 5584, 5500, 5556, 5566, 5600, 5622] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278691840 dispatch_meta_bytes=229304 combine_payload_bytes=278691840 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=278921144 ingress_bytes=498892800 link_bytes=498892800 token_bytes=557612984 weight_movement_bytes=220200960 total_nvlink_bytes=777813944 weight_to_token_ratio=0.394899 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.676566330e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278792192 dispatch_meta_bytes=228248 combine_payload_bytes=278792192 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279020440 ingress_bytes=388892672 link_bytes=388892672 token_bytes=557812632 weight_movement_bytes=110100480 total_nvlink_bytes=667913112 weight_to_token_ratio=0.197379 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.763483185e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279681024 dispatch_meta_bytes=229456 combine_payload_bytes=279681024 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279910480 ingress_bytes=499881984 link_bytes=499881984 token_bytes=559591504 weight_movement_bytes=220200960 total_nvlink_bytes=779792464 weight_to_token_ratio=0.393503 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.693769866e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44462 payload_rows_remote=38842 dup_rows=21074 dup_rows_remote=18406 dedup_ratio=0.321564 router_tokens_per_expert=[545, 503, 488, 523, 553, 526, 488, 536, 512, 507, 546, 503, 507, 511, 524, 467, 502, 513, 477, 520, 555, 486, 506, 512, 546, 465, 537, 481, 522, 551, 522, 528, 501, 509, 505, 515, 534, 526, 552, 502, 537, 513, 501, 509, 529, 517, 503, 471, 519, 480, 504, 516, 495, 492, 511, 502, 515, 530, 509, 491, 496, 529, 479, 514] payload_rows_per_dst=[5600, 5514, 5548, 5560, 5620, 5580, 5502, 5538] dup_rows_per_dst=[2724, 2640, 2594, 2744, 2668, 2580, 2536, 2588] combine_rows_per_src=[5600, 5514, 5548, 5560, 5620, 5580, 5502, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44422 payload_rows_remote=38812 dup_rows=21114 dup_rows_remote=18504 dedup_ratio=0.322174 router_tokens_per_expert=[541, 567, 494, 544, 510, 497, 487, 510, 515, 549, 529, 558, 513, 501, 487, 500, 496, 524, 499, 493, 542, 488, 556, 474, 501, 525, 529, 516, 485, 505, 491, 521, 512, 499, 514, 490, 550, 518, 524, 466, 514, 512, 512, 516, 527, 511, 511, 523, 524, 508, 512, 514, 484, 467, 517, 509, 467, 506, 508, 556, 486, 523, 524, 517] payload_rows_per_dst=[5334, 5560, 5570, 5522, 5510, 5628, 5688, 5610] dup_rows_per_dst=[2440, 2744, 2574, 2624, 2590, 2624, 2908, 2610] combine_rows_per_src=[5334, 5560, 5570, 5522, 5510, 5628, 5688, 5610] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44430 payload_rows_remote=38830 dup_rows=21106 dup_rows_remote=18448 dedup_ratio=0.322052 router_tokens_per_expert=[503, 482, 516, 536, 524, 566, 507, 498, 514, 542, 530, 501, 505, 522, 478, 488, 499, 480, 539, 505, 507, 504, 567, 510, 503, 521, 509, 496, 506, 533, 522, 494, 558, 540, 492, 546, 488, 508, 484, 490, 528, 521, 498, 531, 471, 515, 485, 507, 534, 538, 490, 498, 515, 542, 496, 516, 507, 530, 519, 445, 552, 504, 489, 524] payload_rows_per_dst=[5570, 5528, 5540, 5566, 5572, 5516, 5600, 5538] dup_rows_per_dst=[2694, 2632, 2682, 2602, 2640, 2596, 2658, 2602] combine_rows_per_src=[5570, 5528, 5540, 5566, 5572, 5516, 5600, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44524 payload_rows_remote=39018 dup_rows=21012 dup_rows_remote=18450 dedup_ratio=0.320618 router_tokens_per_expert=[504, 532, 534, 515, 488, 541, 519, 546, 505, 520, 497, 523, 514, 526, 544, 515, 513, 529, 479, 522, 506, 514, 451, 520, 518, 484, 484, 487, 519, 514, 492, 562, 508, 527, 491, 524, 496, 518, 523, 528, 498, 505, 498, 492, 546, 537, 495, 542, 517, 514, 506, 527, 482, 524, 500, 506, 512, 498, 487, 495, 493, 516, 511, 535] payload_rows_per_dst=[5656, 5630, 5506, 5480, 5586, 5604, 5572, 5490] dup_rows_per_dst=[2702, 2658, 2562, 2640, 2644, 2622, 2580, 2604] combine_rows_per_src=[5656, 5630, 5506, 5480, 5586, 5604, 5572, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44390 payload_rows_remote=38776 dup_rows=21146 dup_rows_remote=18428 dedup_ratio=0.322662 router_tokens_per_expert=[488, 519, 517, 532, 521, 508, 500, 543, 518, 507, 544, 498, 518, 546, 542, 550, 545, 519, 503, 502, 492, 506, 502, 507, 475, 500, 499, 506, 507, 563, 546, 509, 502, 549, 550, 535, 533, 492, 455, 516, 482, 537, 529, 537, 536, 534, 512, 499, 526, 481, 487, 526, 488, 481, 494, 503, 503, 483, 506, 511, 495, 485, 481, 488] payload_rows_per_dst=[5556, 5646, 5508, 5592, 5590, 5614, 5462, 5422] dup_rows_per_dst=[2700, 2800, 2644, 2618, 2674, 2718, 2510, 2482] combine_rows_per_src=[5556, 5646, 5508, 5592, 5590, 5614, 5462, 5422] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=280283136 dispatch_meta_bytes=229592 combine_payload_bytes=280283136 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280512728 ingress_bytes=390383616 link_bytes=390383616 token_bytes=560795864 weight_movement_bytes=110100480 total_nvlink_bytes=670896344 weight_to_token_ratio=0.196329 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.789413153e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278419456 dispatch_meta_bytes=228992 combine_payload_bytes=278419456 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278648448 ingress_bytes=498620416 link_bytes=498620416 token_bytes=557067904 weight_movement_bytes=220200960 total_nvlink_bytes=777268864 weight_to_token_ratio=0.395286 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.671829124e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278204416 dispatch_meta_bytes=229264 combine_payload_bytes=278204416 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278433680 ingress_bytes=498405376 link_bytes=498405376 token_bytes=556638096 weight_movement_bytes=220200960 total_nvlink_bytes=776839056 weight_to_token_ratio=0.395591 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.668089225e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44686 payload_rows_remote=39116 dup_rows=20850 dup_rows_remote=18318 dedup_ratio=0.318146 router_tokens_per_expert=[489, 492, 494, 563, 500, 466, 537, 531, 514, 538, 498, 533, 515, 468, 495, 516, 521, 517, 543, 503, 482, 525, 494, 530, 522, 503, 491, 493, 499, 520, 536, 487, 462, 501, 552, 517, 505, 534, 521, 496, 497, 526, 497, 547, 492, 515, 528, 521, 511, 510, 505, 517, 504, 525, 479, 529, 533, 529, 509, 533, 524, 502, 513, 519] payload_rows_per_dst=[5578, 5580, 5606, 5570, 5578, 5556, 5590, 5628] dup_rows_per_dst=[2566, 2574, 2624, 2532, 2598, 2690, 2570, 2696] combine_rows_per_src=[5578, 5580, 5606, 5570, 5578, 5556, 5590, 5628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278333440 dispatch_meta_bytes=229112 combine_payload_bytes=278333440 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278562552 ingress_bytes=388433920 link_bytes=388433920 token_bytes=556895992 weight_movement_bytes=110100480 total_nvlink_bytes=666996472 weight_to_token_ratio=0.197704 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.755504733e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44456 payload_rows_remote=38944 dup_rows=21080 dup_rows_remote=18484 dedup_ratio=0.321655 router_tokens_per_expert=[480, 525, 514, 505, 537, 515, 479, 517, 526, 479, 523, 554, 498, 504, 471, 499, 528, 538, 522, 535, 473, 489, 477, 512, 514, 529, 514, 539, 524, 505, 507, 543, 509, 488, 562, 497, 518, 502, 514, 563, 509, 550, 510, 522, 521, 494, 490, 490, 501, 511, 526, 475, 552, 514, 494, 490, 523, 493, 505, 553, 515, 505, 508, 489] payload_rows_per_dst=[5572, 5512, 5550, 5690, 5586, 5518, 5468, 5560] dup_rows_per_dst=[2572, 2596, 2598, 2660, 2720, 2654, 2658, 2622] combine_rows_per_src=[5572, 5512, 5550, 5690, 5586, 5518, 5468, 5560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44306 payload_rows_remote=38782 dup_rows=21230 dup_rows_remote=18564 dedup_ratio=0.323944 router_tokens_per_expert=[482, 541, 504, 470, 479, 508, 524, 499, 484, 499, 506, 503, 488, 541, 499, 495, 528, 513, 498, 506, 514, 515, 545, 495, 530, 499, 512, 505, 501, 513, 515, 497, 478, 486, 511, 528, 530, 491, 510, 524, 506, 508, 509, 477, 545, 517, 551, 515, 525, 511, 499, 490, 514, 517, 512, 527, 554, 527, 510, 552, 517, 560, 522, 537] payload_rows_per_dst=[5524, 5398, 5604, 5614, 5492, 5470, 5542, 5662] dup_rows_per_dst=[2666, 2460, 2752, 2666, 2638, 2632, 2648, 2768] combine_rows_per_src=[5524, 5398, 5604, 5614, 5492, 5470, 5542, 5662] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279681024 dispatch_meta_bytes=229872 combine_payload_bytes=279681024 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279910896 ingress_bytes=389781504 link_bytes=389781504 token_bytes=559591920 weight_movement_bytes=110100480 total_nvlink_bytes=669692400 weight_to_token_ratio=0.196751 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778941435e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277946368 dispatch_meta_bytes=228816 combine_payload_bytes=277946368 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278175184 ingress_bytes=608247808 link_bytes=608247808 token_bytes=556121552 weight_movement_bytes=330301440 total_nvlink_bytes=886422992 weight_to_token_ratio=0.593937 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057842978e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278376448 dispatch_meta_bytes=229200 combine_payload_bytes=278376448 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278605648 ingress_bytes=388476928 link_bytes=388476928 token_bytes=556982096 weight_movement_bytes=110100480 total_nvlink_bytes=667082576 weight_to_token_ratio=0.197673 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.756252713e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278433792 dispatch_meta_bytes=229024 combine_payload_bytes=278433792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278662816 ingress_bytes=388534272 link_bytes=388534272 token_bytes=557096608 weight_movement_bytes=110100480 total_nvlink_bytes=667197088 weight_to_token_ratio=0.197633 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.757250019e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278261760 dispatch_meta_bytes=229696 combine_payload_bytes=278261760 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278491456 ingress_bytes=498462720 link_bytes=498462720 token_bytes=556753216 weight_movement_bytes=220200960 total_nvlink_bytes=776954176 weight_to_token_ratio=0.395509 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.669086531e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44526 payload_rows_remote=39022 dup_rows=21010 dup_rows_remote=18328 dedup_ratio=0.320587 router_tokens_per_expert=[549, 523, 499, 544, 481, 535, 490, 513, 515, 521, 494, 505, 508, 491, 541, 518, 525, 496, 505, 520, 532, 489, 516, 543, 524, 507, 542, 540, 496, 500, 485, 518, 497, 463, 484, 547, 530, 528, 473, 541, 533, 494, 527, 530, 521, 477, 486, 475, 483, 488, 540, 512, 518, 484, 530, 514, 512, 535, 510, 518, 552, 473, 525, 503] payload_rows_per_dst=[5602, 5504, 5598, 5512, 5548, 5564, 5594, 5604] dup_rows_per_dst=[2666, 2682, 2654, 2712, 2578, 2522, 2544, 2652] combine_rows_per_src=[5602, 5504, 5598, 5512, 5548, 5564, 5594, 5604] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44808 payload_rows_remote=39206 dup_rows=20728 dup_rows_remote=18066 dedup_ratio=0.316284 router_tokens_per_expert=[520, 513, 561, 489, 564, 474, 469, 504, 504, 509, 490, 552, 479, 535, 522, 516, 519, 532, 482, 539, 521, 479, 503, 557, 490, 532, 511, 514, 490, 536, 470, 504, 513, 526, 514, 533, 497, 528, 536, 500, 524, 549, 493, 509, 487, 479, 503, 546, 503, 526, 545, 533, 523, 493, 485, 520, 481, 532, 535, 504, 522, 487, 466, 496] payload_rows_per_dst=[5556, 5624, 5602, 5602, 5612, 5614, 5616, 5582] dup_rows_per_dst=[2632, 2590, 2662, 2492, 2682, 2566, 2640, 2464] combine_rows_per_src=[5556, 5624, 5602, 5602, 5612, 5614, 5616, 5582] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44518 payload_rows_remote=38820 dup_rows=21018 dup_rows_remote=18300 dedup_ratio=0.320709 router_tokens_per_expert=[531, 496, 538, 510, 525, 528, 514, 497, 517, 496, 510, 528, 495, 536, 532, 496, 542, 495, 491, 511, 507, 534, 550, 499, 485, 495, 500, 511, 523, 511, 507, 508, 497, 546, 561, 545, 507, 455, 503, 502, 492, 530, 518, 504, 532, 493, 535, 523, 533, 494, 495, 515, 509, 480, 505, 522, 511, 498, 503, 522, 530, 499, 493, 498] payload_rows_per_dst=[5448, 5310, 5554, 5676, 5468, 5566, 5798, 5698] dup_rows_per_dst=[2580, 2340, 2704, 2654, 2456, 2688, 2878, 2718] combine_rows_per_src=[5448, 5310, 5554, 5676, 5468, 5566, 5798, 5698] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279709696 dispatch_meta_bytes=229400 combine_payload_bytes=279709696 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=279939096 ingress_bytes=830212096 link_bytes=830212096 token_bytes=559648792 weight_movement_bytes=550502400 total_nvlink_bytes=1110151192 weight_to_token_ratio=0.983657 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.443875381e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44394 payload_rows_remote=38878 dup_rows=21142 dup_rows_remote=18542 dedup_ratio=0.322601 router_tokens_per_expert=[509, 483, 503, 499, 506, 524, 535, 521, 529, 534, 514, 494, 495, 515, 551, 482, 511, 504, 513, 540, 502, 521, 521, 511, 536, 498, 487, 477, 509, 480, 513, 490, 516, 502, 514, 497, 511, 559, 497, 496, 498, 481, 525, 500, 498, 490, 512, 554, 499, 521, 556, 540, 543, 521, 513, 521, 534, 494, 505, 523, 507, 515, 501, 518] payload_rows_per_dst=[5520, 5598, 5490, 5452, 5590, 5516, 5644, 5584] dup_rows_per_dst=[2640, 2630, 2756, 2528, 2594, 2600, 2784, 2610] combine_rows_per_src=[5520, 5598, 5490, 5452, 5590, 5516, 5644, 5584] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44372 payload_rows_remote=38804 dup_rows=21164 dup_rows_remote=18276 dedup_ratio=0.322937 router_tokens_per_expert=[512, 550, 532, 516, 525, 521, 510, 517, 502, 518, 494, 516, 512, 511, 523, 552, 509, 513, 504, 527, 519, 509, 493, 495, 526, 514, 508, 536, 501, 531, 554, 558, 484, 517, 519, 474, 516, 484, 458, 513, 532, 517, 483, 499, 518, 510, 518, 489, 526, 468, 551, 526, 502, 506, 486, 498, 493, 491, 558, 529, 479, 499, 526, 491] payload_rows_per_dst=[5638, 5576, 5538, 5568, 5440, 5570, 5572, 5470] dup_rows_per_dst=[2728, 2680, 2600, 2888, 2490, 2562, 2554, 2662] combine_rows_per_src=[5638, 5576, 5538, 5568, 5440, 5570, 5572, 5470] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44420 payload_rows_remote=38860 dup_rows=21116 dup_rows_remote=18452 dedup_ratio=0.322205 router_tokens_per_expert=[530, 536, 479, 523, 558, 511, 509, 520, 498, 521, 538, 534, 515, 470, 512, 520, 525, 484, 511, 497, 505, 508, 538, 534, 481, 510, 463, 508, 487, 525, 499, 513, 479, 516, 492, 562, 489, 525, 523, 526, 516, 544, 493, 547, 480, 521, 535, 563, 469, 474, 519, 521, 459, 520, 465, 519, 528, 522, 513, 509, 508, 524, 501, 544] payload_rows_per_dst=[5596, 5518, 5602, 5474, 5560, 5680, 5474, 5516] dup_rows_per_dst=[2736, 2698, 2602, 2498, 2664, 2718, 2418, 2782] combine_rows_per_src=[5596, 5518, 5602, 5474, 5560, 5680, 5474, 5516] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44582 payload_rows_remote=39102 dup_rows=20954 dup_rows_remote=18472 dedup_ratio=0.319733 router_tokens_per_expert=[481, 493, 488, 506, 518, 511, 531, 512, 542, 490, 535, 534, 526, 478, 558, 508, 540, 509, 510, 508, 506, 516, 511, 527, 485, 518, 535, 544, 505, 494, 519, 538, 537, 527, 508, 514, 524, 503, 542, 505, 523, 511, 511, 497, 515, 530, 489, 491, 494, 494, 498, 473, 486, 514, 514, 508, 486, 530, 534, 504, 489, 525, 510, 506] payload_rows_per_dst=[5502, 5610, 5616, 5556, 5614, 5630, 5480, 5574] dup_rows_per_dst=[2578, 2732, 2638, 2720, 2706, 2504, 2482, 2594] combine_rows_per_src=[5502, 5610, 5616, 5556, 5614, 5630, 5480, 5574] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44610 payload_rows_remote=38972 dup_rows=20926 dup_rows_remote=18360 dedup_ratio=0.319305 router_tokens_per_expert=[571, 511, 497, 478, 525, 514, 510, 476, 534, 504, 488, 504, 548, 525, 517, 539, 489, 505, 540, 516, 494, 557, 497, 504, 482, 493, 520, 536, 508, 493, 515, 485, 500, 562, 528, 516, 510, 526, 510, 489, 513, 503, 540, 501, 526, 506, 528, 516, 487, 521, 516, 504, 508, 543, 484, 482, 524, 517, 466, 532, 482, 501, 529, 523] payload_rows_per_dst=[5638, 5662, 5496, 5534, 5566, 5660, 5502, 5552] dup_rows_per_dst=[2566, 2820, 2506, 2530, 2804, 2516, 2588, 2596] combine_rows_per_src=[5638, 5662, 5496, 5534, 5566, 5660, 5502, 5552] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=281028608 dispatch_meta_bytes=229088 combine_payload_bytes=281028608 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=281257696 ingress_bytes=611330048 link_bytes=611330048 token_bytes=562286304 weight_movement_bytes=330301440 total_nvlink_bytes=892587744 weight_to_token_ratio=0.587426 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.063203500e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278261760 dispatch_meta_bytes=228480 combine_payload_bytes=278261760 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278490240 ingress_bytes=388362240 link_bytes=388362240 token_bytes=556752000 weight_movement_bytes=110100480 total_nvlink_bytes=666852480 weight_to_token_ratio=0.197755 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.754258100e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278147072 dispatch_meta_bytes=228320 combine_payload_bytes=278147072 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278375392 ingress_bytes=388247552 link_bytes=388247552 token_bytes=556522464 weight_movement_bytes=110100480 total_nvlink_bytes=666622944 weight_to_token_ratio=0.197837 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.752263487e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278677504 dispatch_meta_bytes=229680 combine_payload_bytes=278677504 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=278907184 ingress_bytes=498878464 link_bytes=498878464 token_bytes=557584688 weight_movement_bytes=220200960 total_nvlink_bytes=777785648 weight_to_token_ratio=0.394919 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.676317003e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278548480 dispatch_meta_bytes=229248 combine_payload_bytes=278548480 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278777728 ingress_bytes=498749440 link_bytes=498749440 token_bytes=557326208 weight_movement_bytes=220200960 total_nvlink_bytes=777527168 weight_to_token_ratio=0.395102 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.674073063e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44492 payload_rows_remote=38960 dup_rows=21044 dup_rows_remote=18428 dedup_ratio=0.321106 router_tokens_per_expert=[481, 528, 516, 505, 539, 516, 482, 522, 527, 485, 530, 555, 498, 505, 473, 501, 529, 539, 522, 536, 475, 490, 478, 516, 515, 532, 516, 541, 523, 507, 505, 543, 507, 489, 559, 498, 515, 501, 514, 562, 509, 549, 510, 516, 519, 494, 487, 489, 498, 510, 526, 474, 551, 510, 494, 490, 521, 489, 502, 549, 511, 500, 507, 488] payload_rows_per_dst=[5590, 5532, 5564, 5710, 5582, 5510, 5460, 5544] dup_rows_per_dst=[2588, 2616, 2606, 2654, 2708, 2636, 2646, 2590] combine_rows_per_src=[5590, 5532, 5564, 5710, 5582, 5510, 5460, 5544] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=280283136 dispatch_meta_bytes=230296 combine_payload_bytes=280283136 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280513432 ingress_bytes=390383616 link_bytes=390383616 token_bytes=560796568 weight_movement_bytes=110100480 total_nvlink_bytes=670897048 weight_to_token_ratio=0.196329 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.789413153e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279351296 dispatch_meta_bytes=229328 combine_payload_bytes=279351296 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279580624 ingress_bytes=499552256 link_bytes=499552256 token_bytes=558931920 weight_movement_bytes=220200960 total_nvlink_bytes=779132880 weight_to_token_ratio=0.393967 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.688035354e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279265280 dispatch_meta_bytes=229552 combine_payload_bytes=279265280 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279494832 ingress_bytes=499466240 link_bytes=499466240 token_bytes=558760112 weight_movement_bytes=220200960 total_nvlink_bytes=778961072 weight_to_token_ratio=0.394089 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.686539394e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44718 payload_rows_remote=39056 dup_rows=20818 dup_rows_remote=18174 dedup_ratio=0.317657 router_tokens_per_expert=[496, 519, 488, 518, 515, 516, 527, 536, 542, 503, 536, 468, 547, 500, 504, 526, 529, 495, 528, 539, 497, 552, 493, 520, 475, 523, 517, 497, 478, 507, 505, 497, 529, 493, 509, 532, 555, 518, 513, 517, 567, 488, 515, 533, 525, 497, 491, 512, 471, 479, 491, 500, 540, 517, 482, 502, 511, 523, 516, 481, 512, 520, 538, 498] payload_rows_per_dst=[5632, 5608, 5662, 5474, 5604, 5604, 5462, 5672] dup_rows_per_dst=[2598, 2644, 2644, 2524, 2728, 2652, 2502, 2526] combine_rows_per_src=[5632, 5608, 5662, 5474, 5604, 5604, 5462, 5672] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44304 payload_rows_remote=38804 dup_rows=21232 dup_rows_remote=18526 dedup_ratio=0.323975 router_tokens_per_expert=[520, 525, 523, 532, 493, 503, 519, 505, 519, 488, 480, 546, 502, 508, 539, 523, 528, 554, 475, 514, 489, 522, 537, 505, 516, 518, 509, 510, 472, 536, 517, 527, 542, 522, 472, 509, 560, 505, 494, 471, 522, 525, 505, 508, 508, 504, 531, 544, 506, 484, 518, 507, 510, 488, 529, 538, 513, 467, 522, 520, 515, 489, 493, 493] payload_rows_per_dst=[5610, 5194, 5458, 5842, 5476, 5604, 5620, 5500] dup_rows_per_dst=[2578, 2160, 2608, 3224, 2492, 2690, 2774, 2706] combine_rows_per_src=[5610, 5194, 5458, 5842, 5476, 5604, 5620, 5500] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44546 payload_rows_remote=39032 dup_rows=20990 dup_rows_remote=18544 dedup_ratio=0.320282 router_tokens_per_expert=[509, 497, 526, 536, 523, 508, 536, 492, 526, 520, 549, 572, 538, 479, 505, 486, 513, 504, 526, 495, 541, 478, 543, 467, 519, 489, 511, 489, 478, 538, 486, 470, 487, 489, 546, 523, 485, 540, 508, 501, 523, 507, 500, 543, 492, 516, 534, 497, 506, 539, 512, 510, 513, 522, 519, 511, 524, 500, 513, 500, 513, 529, 516, 501] payload_rows_per_dst=[5598, 5662, 5504, 5514, 5492, 5540, 5600, 5636] dup_rows_per_dst=[2656, 2688, 2630, 2446, 2666, 2684, 2664, 2556] combine_rows_per_src=[5598, 5662, 5504, 5514, 5492, 5540, 5600, 5636] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44358 payload_rows_remote=38766 dup_rows=21178 dup_rows_remote=18488 dedup_ratio=0.323151 router_tokens_per_expert=[519, 541, 510, 497, 501, 519, 497, 509, 536, 517, 534, 525, 500, 536, 563, 546, 538, 503, 485, 501, 479, 522, 517, 540, 518, 518, 519, 513, 499, 508, 513, 497, 513, 501, 517, 468, 492, 505, 549, 527, 488, 512, 523, 518, 533, 500, 503, 564, 561, 459, 453, 482, 497, 534, 458, 503, 515, 508, 516, 526, 508, 520, 489, 506] payload_rows_per_dst=[5510, 5730, 5516, 5546, 5444, 5592, 5452, 5568] dup_rows_per_dst=[2676, 2784, 2654, 2624, 2700, 2690, 2442, 2608] combine_rows_per_src=[5510, 5730, 5516, 5546, 5444, 5592, 5452, 5568] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44588 payload_rows_remote=38952 dup_rows=20948 dup_rows_remote=18306 dedup_ratio=0.319641 router_tokens_per_expert=[468, 511, 521, 522, 522, 511, 519, 560, 503, 511, 518, 473, 510, 504, 546, 487, 524, 512, 506, 550, 512, 479, 519, 495, 525, 494, 512, 488, 508, 530, 501, 485, 557, 505, 470, 556, 512, 476, 516, 547, 485, 515, 511, 516, 479, 522, 520, 495, 540, 510, 529, 462, 548, 519, 522, 517, 553, 496, 509, 525, 508, 490, 516, 516] payload_rows_per_dst=[5582, 5540, 5644, 5528, 5636, 5488, 5596, 5574] dup_rows_per_dst=[2686, 2564, 2550, 2558, 2642, 2598, 2698, 2652] combine_rows_per_src=[5582, 5540, 5644, 5528, 5636, 5488, 5596, 5574] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279953408 dispatch_meta_bytes=228920 combine_payload_bytes=279953408 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280182328 ingress_bytes=500154368 link_bytes=500154368 token_bytes=560135736 weight_movement_bytes=220200960 total_nvlink_bytes=780336696 weight_to_token_ratio=0.393121 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.698507072e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44602 payload_rows_remote=38988 dup_rows=20934 dup_rows_remote=18234 dedup_ratio=0.319427 router_tokens_per_expert=[492, 519, 527, 499, 539, 523, 503, 503, 499, 570, 515, 480, 505, 540, 502, 513, 496, 497, 497, 505, 516, 511, 517, 503, 493, 501, 528, 488, 509, 477, 517, 491, 529, 493, 530, 525, 521, 561, 518, 479, 497, 544, 513, 497, 515, 504, 522, 508, 491, 582, 572, 462, 535, 514, 489, 512, 506, 512, 497, 507, 495, 532, 507, 524] payload_rows_per_dst=[5558, 5580, 5572, 5492, 5662, 5614, 5614, 5510] dup_rows_per_dst=[2652, 2668, 2512, 2516, 2650, 2586, 2700, 2650] combine_rows_per_src=[5558, 5580, 5572, 5492, 5662, 5614, 5614, 5510] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44608 payload_rows_remote=39104 dup_rows=20928 dup_rows_remote=18434 dedup_ratio=0.319336 router_tokens_per_expert=[492, 484, 494, 473, 502, 534, 518, 497, 512, 544, 507, 539, 509, 484, 545, 524, 506, 505, 522, 511, 523, 542, 508, 536, 530, 550, 459, 512, 561, 493, 487, 481, 527, 509, 500, 526, 552, 519, 505, 567, 491, 499, 482, 509, 510, 483, 532, 490, 531, 483, 505, 531, 532, 543, 521, 492, 509, 503, 489, 523, 497, 492, 520, 512] payload_rows_per_dst=[5504, 5722, 5638, 5568, 5522, 5554, 5520, 5580] dup_rows_per_dst=[2494, 2922, 2672, 2578, 2572, 2424, 2756, 2510] combine_rows_per_src=[5504, 5722, 5638, 5568, 5522, 5554, 5520, 5580] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278147072 dispatch_meta_bytes=229320 combine_payload_bytes=278147072 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278376392 ingress_bytes=388247552 link_bytes=388247552 token_bytes=556523464 weight_movement_bytes=110100480 total_nvlink_bytes=666623944 weight_to_token_ratio=0.197836 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.752263487e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279781376 dispatch_meta_bytes=230304 combine_payload_bytes=279781376 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280011680 ingress_bytes=389881856 link_bytes=389881856 token_bytes=559793056 weight_movement_bytes=110100480 total_nvlink_bytes=669893536 weight_to_token_ratio=0.196681 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.780686721e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277874688 dispatch_meta_bytes=229016 combine_payload_bytes=277874688 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=278103704 ingress_bytes=498075648 link_bytes=498075648 token_bytes=555978392 weight_movement_bytes=220200960 total_nvlink_bytes=776179352 weight_to_token_ratio=0.396060 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.662354712e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44494 payload_rows_remote=39000 dup_rows=21042 dup_rows_remote=18466 dedup_ratio=0.321075 router_tokens_per_expert=[498, 537, 501, 507, 506, 535, 499, 518, 506, 530, 483, 502, 516, 478, 510, 510, 510, 491, 527, 494, 502, 551, 515, 546, 523, 468, 520, 512, 503, 487, 490, 548, 527, 529, 490, 536, 484, 531, 534, 494, 507, 508, 510, 491, 511, 526, 550, 505, 505, 473, 558, 516, 499, 512, 498, 507, 501, 502, 551, 502, 537, 524, 493, 534] payload_rows_per_dst=[5594, 5494, 5648, 5578, 5576, 5548, 5514, 5542] dup_rows_per_dst=[2608, 2576, 2624, 2524, 2674, 2668, 2622, 2746] combine_rows_per_src=[5594, 5494, 5648, 5578, 5576, 5548, 5514, 5542] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279207936 dispatch_meta_bytes=229032 combine_payload_bytes=279207936 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279436968 ingress_bytes=609509376 link_bytes=609509376 token_bytes=558644904 weight_movement_bytes=330301440 total_nvlink_bytes=888946344 weight_to_token_ratio=0.591255 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060037052e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279465984 dispatch_meta_bytes=228888 combine_payload_bytes=279465984 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279694872 ingress_bytes=389566464 link_bytes=389566464 token_bytes=559160856 weight_movement_bytes=110100480 total_nvlink_bytes=669261336 weight_to_token_ratio=0.196903 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.775201536e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=280297472 dispatch_meta_bytes=230152 combine_payload_bytes=280297472 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280527624 ingress_bytes=500498432 link_bytes=500498432 token_bytes=560825096 weight_movement_bytes=220200960 total_nvlink_bytes=781026056 weight_to_token_ratio=0.392637 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.704490911e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279552000 dispatch_meta_bytes=229864 combine_payload_bytes=279552000 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279781864 ingress_bytes=499752960 link_bytes=499752960 token_bytes=559333864 weight_movement_bytes=220200960 total_nvlink_bytes=779534824 weight_to_token_ratio=0.393684 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.691525927e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44444 payload_rows_remote=38844 dup_rows=21092 dup_rows_remote=18492 dedup_ratio=0.321838 router_tokens_per_expert=[506, 531, 499, 517, 526, 512, 510, 553, 513, 543, 510, 514, 535, 520, 519, 551, 514, 523, 494, 513, 527, 518, 525, 486, 503, 522, 508, 505, 526, 523, 473, 479, 510, 529, 508, 502, 475, 556, 477, 518, 503, 507, 493, 480, 497, 463, 500, 512, 505, 515, 526, 551, 503, 520, 563, 471, 477, 508, 476, 485, 533, 536, 513, 558] payload_rows_per_dst=[5608, 5632, 5600, 5452, 5556, 5398, 5626, 5572] dup_rows_per_dst=[2700, 2778, 2600, 2626, 2594, 2512, 2682, 2600] combine_rows_per_src=[5608, 5632, 5600, 5452, 5556, 5398, 5626, 5572] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44548 payload_rows_remote=39068 dup_rows=20988 dup_rows_remote=18442 dedup_ratio=0.320251 router_tokens_per_expert=[512, 456, 504, 524, 540, 552, 509, 529, 507, 498, 534, 499, 513, 489, 511, 529, 538, 547, 533, 501, 513, 529, 553, 512, 507, 530, 510, 462, 516, 517, 473, 498, 486, 489, 514, 545, 484, 482, 506, 531, 518, 516, 529, 485, 506, 520, 561, 514, 507, 503, 500, 516, 525, 482, 511, 524, 516, 490, 478, 531, 547, 472, 536, 499] payload_rows_per_dst=[5552, 5530, 5676, 5480, 5542, 5658, 5570, 5540] dup_rows_per_dst=[2700, 2630, 2776, 2546, 2532, 2640, 2566, 2598] combine_rows_per_src=[5552, 5530, 5676, 5480, 5542, 5658, 5570, 5540] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44540 payload_rows_remote=38784 dup_rows=20996 dup_rows_remote=18160 dedup_ratio=0.320374 router_tokens_per_expert=[526, 488, 471, 513, 506, 537, 493, 521, 508, 515, 486, 532, 488, 528, 509, 500, 524, 525, 507, 504, 536, 485, 499, 503, 501, 514, 527, 545, 500, 510, 570, 532, 518, 486, 508, 493, 507, 511, 549, 512, 545, 560, 522, 525, 514, 460, 537, 508, 518, 505, 515, 496, 491, 478, 522, 498, 524, 498, 514, 492, 509, 509, 534, 507] payload_rows_per_dst=[5204, 5528, 5262, 5744, 5694, 5774, 5578, 5756] dup_rows_per_dst=[2290, 2510, 2194, 2948, 2766, 2786, 2666, 2836] combine_rows_per_src=[5204, 5528, 5262, 5744, 5694, 5774, 5578, 5756] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44714 payload_rows_remote=39130 dup_rows=20822 dup_rows_remote=18242 dedup_ratio=0.317719 router_tokens_per_expert=[490, 528, 523, 542, 541, 504, 524, 497, 527, 516, 512, 484, 520, 480, 507, 506, 534, 506, 542, 491, 534, 526, 531, 547, 532, 521, 518, 524, 502, 482, 517, 500, 511, 503, 489, 524, 512, 505, 537, 532, 546, 510, 506, 462, 510, 520, 529, 499, 520, 486, 500, 497, 533, 515, 511, 489, 507, 508, 487, 502, 509, 484, 535, 482] payload_rows_per_dst=[5640, 5544, 5710, 5558, 5590, 5584, 5542, 5546] dup_rows_per_dst=[2658, 2560, 2712, 2634, 2636, 2580, 2560, 2482] combine_rows_per_src=[5640, 5544, 5710, 5558, 5590, 5584, 5542, 5546] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278433792 dispatch_meta_bytes=229344 combine_payload_bytes=278433792 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278663136 ingress_bytes=608735232 link_bytes=608735232 token_bytes=557096928 weight_movement_bytes=330301440 total_nvlink_bytes=887398368 weight_to_token_ratio=0.592898 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058690688e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44548 payload_rows_remote=38910 dup_rows=20988 dup_rows_remote=18376 dedup_ratio=0.320251 router_tokens_per_expert=[504, 532, 534, 516, 540, 527, 513, 512, 541, 502, 515, 506, 540, 516, 503, 503, 491, 508, 541, 544, 506, 530, 523, 492, 496, 477, 536, 452, 525, 511, 473, 525, 522, 506, 504, 499, 535, 519, 526, 501, 499, 490, 476, 531, 528, 503, 496, 502, 531, 551, 489, 511, 500, 519, 532, 492, 562, 501, 516, 517, 486, 505, 513, 472] payload_rows_per_dst=[5618, 5550, 5590, 5474, 5560, 5546, 5638, 5572] dup_rows_per_dst=[2738, 2702, 2680, 2516, 2664, 2504, 2612, 2572] combine_rows_per_src=[5618, 5550, 5590, 5474, 5560, 5546, 5638, 5572] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44472 payload_rows_remote=38918 dup_rows=21064 dup_rows_remote=18530 dedup_ratio=0.321411 router_tokens_per_expert=[505, 536, 512, 467, 493, 514, 512, 539, 549, 532, 549, 503, 493, 510, 509, 481, 528, 529, 544, 475, 501, 510, 502, 554, 478, 505, 527, 508, 538, 496, 502, 510, 524, 479, 536, 507, 470, 499, 481, 548, 490, 516, 504, 489, 515, 515, 522, 533, 514, 495, 514, 502, 525, 528, 494, 498, 521, 525, 508, 558, 510, 517, 501, 519] payload_rows_per_dst=[5536, 5618, 5568, 5542, 5554, 5560, 5526, 5568] dup_rows_per_dst=[2620, 2634, 2718, 2586, 2534, 2608, 2614, 2750] combine_rows_per_src=[5536, 5618, 5568, 5542, 5554, 5560, 5526, 5568] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44534 payload_rows_remote=38928 dup_rows=21002 dup_rows_remote=18276 dedup_ratio=0.320465 router_tokens_per_expert=[521, 487, 509, 529, 528, 541, 537, 505, 506, 484, 521, 492, 519, 516, 514, 513, 515, 546, 534, 508, 528, 513, 516, 481, 501, 511, 483, 513, 504, 510, 514, 521, 539, 556, 500, 478, 506, 519, 506, 507, 524, 499, 557, 536, 523, 532, 503, 492, 486, 524, 493, 504, 508, 490, 501, 505, 535, 514, 509, 496, 493, 501, 499, 513] payload_rows_per_dst=[5606, 5522, 5542, 5492, 5646, 5592, 5562, 5572] dup_rows_per_dst=[2726, 2608, 2722, 2622, 2576, 2740, 2460, 2548] combine_rows_per_src=[5606, 5522, 5542, 5492, 5646, 5592, 5562, 5572] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280039424 dispatch_meta_bytes=230040 combine_payload_bytes=280039424 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280269464 ingress_bytes=390139904 link_bytes=390139904 token_bytes=560308888 weight_movement_bytes=110100480 total_nvlink_bytes=670409368 weight_to_token_ratio=0.196500 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.785174600e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44624 payload_rows_remote=39006 dup_rows=20912 dup_rows_remote=18340 dedup_ratio=0.319092 router_tokens_per_expert=[513, 548, 486, 567, 498, 541, 527, 490, 505, 501, 519, 489, 514, 544, 533, 490, 502, 502, 507, 493, 524, 517, 498, 550, 507, 503, 549, 511, 552, 525, 504, 507, 488, 485, 527, 497, 510, 523, 509, 507, 490, 468, 525, 500, 488, 537, 513, 506, 536, 514, 543, 518, 509, 481, 542, 524, 480, 511, 524, 493, 507, 503, 499, 495] payload_rows_per_dst=[5656, 5618, 5538, 5646, 5464, 5532, 5648, 5522] dup_rows_per_dst=[2684, 2572, 2648, 2670, 2628, 2522, 2686, 2502] combine_rows_per_src=[5656, 5618, 5538, 5646, 5464, 5532, 5648, 5522] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278003712 dispatch_meta_bytes=227776 combine_payload_bytes=278003712 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278231488 ingress_bytes=388104192 link_bytes=388104192 token_bytes=556235200 weight_movement_bytes=110100480 total_nvlink_bytes=666335680 weight_to_token_ratio=0.197939 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.749770221e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280483840 dispatch_meta_bytes=229488 combine_payload_bytes=280483840 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280713328 ingress_bytes=390584320 link_bytes=390584320 token_bytes=561197168 weight_movement_bytes=110100480 total_nvlink_bytes=671297648 weight_to_token_ratio=0.196189 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.792903726e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=280383488 dispatch_meta_bytes=229736 combine_payload_bytes=280383488 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280613224 ingress_bytes=390483968 link_bytes=390483968 token_bytes=560996712 weight_movement_bytes=110100480 total_nvlink_bytes=671097192 weight_to_token_ratio=0.196259 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.791158439e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279150592 dispatch_meta_bytes=229712 combine_payload_bytes=279150592 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279380304 ingress_bytes=609452032 link_bytes=609452032 token_bytes=558530896 weight_movement_bytes=330301440 total_nvlink_bytes=888832336 weight_to_token_ratio=0.591375 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059937321e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277989376 dispatch_meta_bytes=229384 combine_payload_bytes=277989376 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278218760 ingress_bytes=498190336 link_bytes=498190336 token_bytes=556208136 weight_movement_bytes=220200960 total_nvlink_bytes=776409096 weight_to_token_ratio=0.395897 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.664349325e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44416 payload_rows_remote=38850 dup_rows=21120 dup_rows_remote=18454 dedup_ratio=0.322266 router_tokens_per_expert=[530, 533, 479, 518, 557, 506, 508, 516, 498, 520, 536, 531, 515, 469, 511, 517, 523, 483, 509, 495, 500, 506, 538, 533, 480, 508, 464, 507, 487, 525, 500, 512, 480, 516, 493, 562, 490, 523, 525, 527, 519, 544, 495, 544, 483, 522, 539, 565, 472, 473, 521, 523, 461, 522, 468, 521, 530, 522, 515, 510, 510, 527, 503, 549] payload_rows_per_dst=[5576, 5508, 5590, 5476, 5566, 5684, 5486, 5530] dup_rows_per_dst=[2718, 2686, 2584, 2490, 2666, 2738, 2436, 2802] combine_rows_per_src=[5576, 5508, 5590, 5476, 5566, 5684, 5486, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44490 payload_rows_remote=38908 dup_rows=21046 dup_rows_remote=18498 dedup_ratio=0.321136 router_tokens_per_expert=[526, 495, 536, 510, 523, 524, 513, 496, 515, 492, 510, 527, 494, 535, 532, 494, 541, 494, 493, 511, 504, 535, 548, 499, 485, 493, 502, 509, 521, 513, 508, 508, 495, 546, 563, 543, 508, 456, 503, 502, 492, 530, 518, 503, 536, 493, 535, 527, 531, 496, 496, 516, 510, 484, 509, 525, 512, 498, 507, 525, 533, 499, 493, 498] payload_rows_per_dst=[5538, 5498, 5540, 5558, 5454, 5606, 5714, 5582] dup_rows_per_dst=[2708, 2644, 2684, 2520, 2456, 2718, 2768, 2548] combine_rows_per_src=[5538, 5498, 5540, 5558, 5454, 5606, 5714, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44590 payload_rows_remote=39098 dup_rows=20946 dup_rows_remote=18454 dedup_ratio=0.319611 router_tokens_per_expert=[479, 490, 487, 496, 515, 509, 530, 511, 538, 489, 532, 530, 524, 476, 558, 507, 538, 508, 508, 508, 507, 513, 509, 524, 486, 517, 534, 544, 504, 493, 520, 538, 540, 527, 511, 515, 526, 503, 544, 506, 523, 511, 515, 497, 512, 531, 493, 493, 495, 495, 502, 473, 488, 513, 517, 509, 489, 535, 537, 505, 489, 528, 512, 512] payload_rows_per_dst=[5488, 5596, 5606, 5554, 5620, 5640, 5492, 5594] dup_rows_per_dst=[2546, 2712, 2624, 2718, 2724, 2510, 2492, 2620] combine_rows_per_src=[5488, 5596, 5606, 5554, 5620, 5640, 5492, 5594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44400 payload_rows_remote=38882 dup_rows=21136 dup_rows_remote=18516 dedup_ratio=0.322510 router_tokens_per_expert=[508, 483, 501, 496, 502, 521, 530, 516, 526, 532, 510, 491, 493, 513, 548, 480, 510, 504, 513, 539, 502, 519, 520, 510, 533, 496, 489, 476, 509, 481, 514, 487, 517, 503, 517, 498, 512, 559, 498, 495, 499, 484, 527, 502, 498, 489, 513, 557, 500, 523, 558, 545, 543, 520, 513, 525, 538, 495, 509, 524, 509, 518, 505, 523] payload_rows_per_dst=[5510, 5586, 5482, 5450, 5598, 5518, 5656, 5600] dup_rows_per_dst=[2604, 2600, 2752, 2520, 2600, 2620, 2798, 2642] combine_rows_per_src=[5510, 5586, 5482, 5450, 5598, 5518, 5656, 5600] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278476800 dispatch_meta_bytes=229216 combine_payload_bytes=278476800 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278706016 ingress_bytes=498677760 link_bytes=498677760 token_bytes=557182816 weight_movement_bytes=220200960 total_nvlink_bytes=777383776 weight_to_token_ratio=0.395204 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.672826430e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44810 payload_rows_remote=39212 dup_rows=20726 dup_rows_remote=18086 dedup_ratio=0.316254 router_tokens_per_expert=[514, 510, 558, 485, 561, 473, 464, 504, 501, 505, 485, 548, 477, 533, 521, 515, 517, 529, 481, 536, 524, 477, 500, 555, 487, 531, 513, 511, 493, 535, 470, 505, 513, 526, 514, 535, 499, 531, 541, 502, 527, 548, 494, 511, 487, 480, 504, 548, 504, 528, 546, 531, 524, 497, 488, 524, 483, 531, 537, 508, 524, 491, 473, 501] payload_rows_per_dst=[5534, 5606, 5598, 5600, 5622, 5622, 5622, 5606] dup_rows_per_dst=[2604, 2564, 2640, 2490, 2700, 2576, 2662, 2490] combine_rows_per_src=[5534, 5606, 5598, 5600, 5622, 5622, 5622, 5606] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278892544 dispatch_meta_bytes=229624 combine_payload_bytes=278892544 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=279122168 ingress_bytes=388993024 link_bytes=388993024 token_bytes=558014712 weight_movement_bytes=110100480 total_nvlink_bytes=668115192 weight_to_token_ratio=0.197307 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.051471 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.765228471e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44378 payload_rows_remote=38804 dup_rows=21158 dup_rows_remote=18268 dedup_ratio=0.322845 router_tokens_per_expert=[511, 546, 526, 513, 524, 519, 508, 516, 500, 517, 490, 516, 511, 508, 521, 552, 505, 512, 504, 527, 518, 507, 492, 491, 525, 513, 507, 535, 505, 531, 556, 560, 483, 515, 520, 475, 517, 484, 458, 515, 531, 519, 486, 500, 520, 511, 518, 489, 525, 469, 552, 530, 503, 512, 487, 500, 494, 492, 561, 530, 482, 499, 529, 496] payload_rows_per_dst=[5618, 5562, 5534, 5574, 5446, 5578, 5580, 5486] dup_rows_per_dst=[2708, 2668, 2578, 2890, 2488, 2570, 2576, 2680] combine_rows_per_src=[5618, 5562, 5534, 5574, 5446, 5578, 5580, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=280254464 dispatch_meta_bytes=230208 combine_payload_bytes=280254464 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280484672 ingress_bytes=390354944 link_bytes=390354944 token_bytes=560739136 weight_movement_bytes=110100480 total_nvlink_bytes=670839616 weight_to_token_ratio=0.196349 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.788914500e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44488 payload_rows_remote=39006 dup_rows=21048 dup_rows_remote=18486 dedup_ratio=0.321167 router_tokens_per_expert=[495, 535, 499, 505, 506, 532, 497, 515, 505, 526, 481, 501, 515, 478, 510, 506, 509, 489, 526, 496, 499, 552, 516, 547, 522, 465, 521, 510, 501, 487, 488, 549, 529, 530, 489, 536, 485, 532, 536, 492, 509, 508, 513, 491, 512, 527, 552, 505, 504, 473, 557, 520, 501, 514, 499, 507, 502, 504, 558, 506, 539, 525, 494, 536] payload_rows_per_dst=[5580, 5482, 5644, 5570, 5584, 5552, 5516, 5560] dup_rows_per_dst=[2588, 2562, 2624, 2516, 2674, 2682, 2634, 2768] combine_rows_per_src=[5580, 5482, 5644, 5570, 5584, 5552, 5516, 5560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44622 payload_rows_remote=38978 dup_rows=20914 dup_rows_remote=18318 dedup_ratio=0.319122 router_tokens_per_expert=[567, 509, 495, 476, 520, 511, 508, 476, 532, 502, 485, 505, 548, 524, 514, 539, 489, 502, 537, 516, 492, 555, 497, 506, 481, 493, 516, 534, 510, 494, 514, 483, 500, 561, 527, 516, 513, 525, 512, 491, 511, 503, 540, 504, 529, 507, 530, 516, 487, 525, 517, 510, 511, 543, 486, 485, 526, 518, 469, 533, 483, 504, 532, 524] payload_rows_per_dst=[5644, 5554, 5574, 5646, 5534, 5594, 5516, 5560] dup_rows_per_dst=[2596, 2644, 2602, 2700, 2756, 2390, 2612, 2614] combine_rows_per_src=[5644, 5554, 5574, 5646, 5534, 5594, 5516, 5560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278706176 dispatch_meta_bytes=229592 combine_payload_bytes=278706176 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278935768 ingress_bytes=498907136 link_bytes=498907136 token_bytes=557641944 weight_movement_bytes=220200960 total_nvlink_bytes=777842904 weight_to_token_ratio=0.394879 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.676815656e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=281071616 dispatch_meta_bytes=229192 combine_payload_bytes=281071616 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=281300808 ingress_bytes=611373056 link_bytes=611373056 token_bytes=562372424 weight_movement_bytes=330301440 total_nvlink_bytes=892673864 weight_to_token_ratio=0.587336 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.063278298e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278147072 dispatch_meta_bytes=228288 combine_payload_bytes=278147072 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278375360 ingress_bytes=388247552 link_bytes=388247552 token_bytes=556522432 weight_movement_bytes=110100480 total_nvlink_bytes=666622912 weight_to_token_ratio=0.197837 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.752263487e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279595008 dispatch_meta_bytes=229968 combine_payload_bytes=279595008 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279824976 ingress_bytes=499795968 link_bytes=499795968 token_bytes=559419984 weight_movement_bytes=220200960 total_nvlink_bytes=779620944 weight_to_token_ratio=0.393624 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.692273907e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279394304 dispatch_meta_bytes=229184 combine_payload_bytes=279394304 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279623488 ingress_bytes=389494784 link_bytes=389494784 token_bytes=559017792 weight_movement_bytes=110100480 total_nvlink_bytes=669118272 weight_to_token_ratio=0.196953 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773954903e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44584 payload_rows_remote=38942 dup_rows=20952 dup_rows_remote=18300 dedup_ratio=0.319702 router_tokens_per_expert=[467, 506, 520, 518, 519, 505, 515, 557, 499, 510, 515, 472, 506, 499, 546, 486, 521, 513, 507, 547, 515, 478, 517, 498, 523, 496, 513, 487, 508, 529, 499, 486, 558, 506, 469, 557, 512, 477, 520, 548, 486, 517, 510, 518, 480, 520, 521, 495, 542, 514, 532, 463, 549, 524, 523, 522, 556, 499, 509, 527, 508, 492, 518, 519] payload_rows_per_dst=[5558, 5524, 5640, 5524, 5642, 5490, 5622, 5584] dup_rows_per_dst=[2656, 2542, 2552, 2558, 2652, 2604, 2716, 2672] combine_rows_per_src=[5558, 5524, 5640, 5524, 5642, 5490, 5622, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44324 payload_rows_remote=38870 dup_rows=21212 dup_rows_remote=18582 dedup_ratio=0.323669 router_tokens_per_expert=[518, 522, 521, 526, 487, 501, 518, 501, 518, 485, 478, 547, 501, 506, 540, 522, 525, 551, 473, 513, 490, 522, 540, 506, 512, 517, 509, 510, 470, 536, 516, 527, 538, 521, 474, 510, 563, 504, 494, 472, 525, 525, 505, 508, 508, 503, 530, 549, 509, 486, 520, 511, 512, 489, 531, 539, 514, 471, 524, 523, 519, 490, 498, 495] payload_rows_per_dst=[5614, 5364, 5528, 5532, 5662, 5612, 5558, 5454] dup_rows_per_dst=[2574, 2420, 2712, 2662, 2900, 2694, 2620, 2630] combine_rows_per_src=[5614, 5364, 5528, 5532, 5662, 5612, 5558, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44592 payload_rows_remote=38958 dup_rows=20944 dup_rows_remote=18228 dedup_ratio=0.319580 router_tokens_per_expert=[490, 517, 525, 493, 535, 523, 503, 503, 497, 566, 513, 477, 506, 537, 500, 511, 497, 494, 496, 505, 513, 510, 517, 502, 492, 500, 526, 486, 510, 475, 516, 489, 528, 492, 529, 528, 523, 563, 515, 484, 500, 545, 514, 498, 516, 505, 522, 511, 493, 585, 573, 464, 536, 518, 491, 515, 509, 513, 501, 510, 497, 534, 508, 524] payload_rows_per_dst=[5542, 5566, 5566, 5474, 5670, 5626, 5634, 5514] dup_rows_per_dst=[2636, 2648, 2502, 2514, 2654, 2596, 2716, 2678] combine_rows_per_src=[5542, 5566, 5566, 5474, 5670, 5626, 5634, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279136256 dispatch_meta_bytes=228968 combine_payload_bytes=279136256 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279365224 ingress_bytes=499337216 link_bytes=499337216 token_bytes=558501480 weight_movement_bytes=220200960 total_nvlink_bytes=778702440 weight_to_token_ratio=0.394271 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.684295455e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44710 payload_rows_remote=39044 dup_rows=20826 dup_rows_remote=18188 dedup_ratio=0.317780 router_tokens_per_expert=[493, 516, 488, 514, 512, 512, 527, 535, 539, 500, 533, 465, 542, 499, 503, 524, 528, 495, 523, 537, 497, 555, 494, 523, 473, 522, 518, 498, 474, 507, 506, 496, 529, 493, 507, 531, 559, 517, 513, 516, 564, 490, 517, 537, 529, 498, 493, 516, 473, 482, 492, 504, 543, 519, 483, 505, 512, 523, 517, 483, 513, 521, 542, 499] payload_rows_per_dst=[5614, 5590, 5666, 5466, 5600, 5624, 5472, 5678] dup_rows_per_dst=[2580, 2620, 2638, 2522, 2730, 2664, 2530, 2542] combine_rows_per_src=[5614, 5590, 5666, 5466, 5600, 5624, 5472, 5678] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278620160 dispatch_meta_bytes=229808 combine_payload_bytes=278620160 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278849968 ingress_bytes=498821120 link_bytes=498821120 token_bytes=557470128 weight_movement_bytes=220200960 total_nvlink_bytes=777671088 weight_to_token_ratio=0.395000 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.675319697e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44548 payload_rows_remote=39038 dup_rows=20988 dup_rows_remote=18548 dedup_ratio=0.320251 router_tokens_per_expert=[508, 495, 525, 534, 519, 507, 533, 492, 521, 518, 547, 570, 538, 477, 504, 484, 510, 503, 526, 494, 539, 478, 542, 465, 517, 489, 508, 490, 477, 538, 487, 469, 487, 488, 545, 524, 483, 538, 508, 501, 524, 509, 500, 540, 495, 518, 536, 499, 508, 542, 514, 509, 517, 525, 522, 515, 526, 500, 517, 502, 518, 533, 517, 504] payload_rows_per_dst=[5588, 5652, 5494, 5510, 5494, 5550, 5612, 5648] dup_rows_per_dst=[2638, 2666, 2620, 2440, 2654, 2692, 2692, 2586] combine_rows_per_src=[5588, 5652, 5494, 5510, 5494, 5550, 5612, 5648] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44618 payload_rows_remote=39012 dup_rows=20918 dup_rows_remote=18358 dedup_ratio=0.319183 router_tokens_per_expert=[508, 543, 484, 564, 496, 540, 527, 483, 503, 495, 519, 489, 512, 543, 535, 487, 501, 500, 510, 497, 524, 515, 499, 551, 507, 500, 547, 508, 547, 526, 502, 509, 489, 485, 524, 498, 509, 525, 509, 506, 493, 467, 525, 499, 488, 538, 514, 506, 536, 514, 545, 521, 513, 484, 547, 525, 482, 516, 524, 498, 513, 506, 501, 497] payload_rows_per_dst=[5636, 5606, 5540, 5634, 5458, 5538, 5664, 5542] dup_rows_per_dst=[2654, 2560, 2654, 2658, 2632, 2522, 2706, 2532] combine_rows_per_src=[5636, 5606, 5540, 5634, 5458, 5538, 5664, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279250944 dispatch_meta_bytes=228744 combine_payload_bytes=279250944 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279479688 ingress_bytes=499451904 link_bytes=499451904 token_bytes=558730632 weight_movement_bytes=220200960 total_nvlink_bytes=778931592 weight_to_token_ratio=0.394109 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.686290068e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44352 payload_rows_remote=38758 dup_rows=21184 dup_rows_remote=18488 dedup_ratio=0.323242 router_tokens_per_expert=[517, 540, 509, 495, 499, 517, 493, 504, 533, 514, 530, 525, 500, 538, 562, 545, 535, 502, 484, 502, 481, 521, 515, 540, 516, 515, 518, 514, 495, 507, 511, 500, 516, 500, 514, 471, 495, 505, 547, 528, 489, 512, 523, 519, 534, 504, 502, 562, 559, 459, 456, 484, 500, 537, 462, 503, 515, 510, 519, 530, 513, 522, 494, 507] payload_rows_per_dst=[5494, 5714, 5520, 5538, 5448, 5594, 5464, 5580] dup_rows_per_dst=[2654, 2780, 2640, 2614, 2704, 2696, 2456, 2640] combine_rows_per_src=[5494, 5714, 5520, 5538, 5448, 5594, 5464, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44510 payload_rows_remote=38922 dup_rows=21026 dup_rows_remote=18394 dedup_ratio=0.320831 router_tokens_per_expert=[488, 481, 492, 472, 499, 529, 516, 496, 509, 543, 507, 535, 506, 482, 545, 523, 503, 504, 518, 506, 523, 540, 508, 535, 529, 550, 457, 508, 562, 491, 489, 484, 528, 507, 501, 528, 551, 521, 507, 569, 494, 500, 482, 509, 510, 484, 536, 490, 531, 487, 507, 535, 535, 546, 524, 493, 510, 508, 491, 524, 498, 496, 521, 515] payload_rows_per_dst=[5588, 5622, 5586, 5914, 5262, 5500, 5522, 5516] dup_rows_per_dst=[2632, 2784, 2604, 3182, 2206, 2360, 2772, 2486] combine_rows_per_src=[5588, 5622, 5586, 5914, 5262, 5500, 5522, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279867392 dispatch_meta_bytes=228928 combine_payload_bytes=279867392 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280096320 ingress_bytes=500068352 link_bytes=500068352 token_bytes=559963712 weight_movement_bytes=220200960 total_nvlink_bytes=780164672 weight_to_token_ratio=0.393241 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.697011112e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279824384 dispatch_meta_bytes=230344 combine_payload_bytes=279824384 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280054728 ingress_bytes=389924864 link_bytes=389924864 token_bytes=559879112 weight_movement_bytes=110100480 total_nvlink_bytes=669979592 weight_to_token_ratio=0.196650 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4096 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.781434701e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279638016 dispatch_meta_bytes=229480 combine_payload_bytes=279638016 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279867496 ingress_bytes=389738496 link_bytes=389738496 token_bytes=559505512 weight_movement_bytes=110100480 total_nvlink_bytes=669605992 weight_to_token_ratio=0.196782 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778193455e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277817344 dispatch_meta_bytes=228984 combine_payload_bytes=277817344 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278046328 ingress_bytes=498018304 link_bytes=498018304 token_bytes=555863672 weight_movement_bytes=220200960 total_nvlink_bytes=776064632 weight_to_token_ratio=0.396142 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.661357406e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278992896 dispatch_meta_bytes=229264 combine_payload_bytes=278992896 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279222160 ingress_bytes=389093376 link_bytes=389093376 token_bytes=558215056 weight_movement_bytes=110100480 total_nvlink_bytes=668315536 weight_to_token_ratio=0.197237 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.766973757e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44460 payload_rows_remote=38900 dup_rows=21076 dup_rows_remote=18538 dedup_ratio=0.321594 router_tokens_per_expert=[503, 530, 509, 465, 493, 513, 509, 534, 549, 530, 547, 501, 493, 510, 503, 482, 527, 529, 535, 472, 503, 508, 501, 555, 480, 504, 528, 508, 537, 498, 503, 510, 525, 479, 537, 509, 469, 500, 482, 548, 490, 513, 502, 492, 516, 516, 523, 535, 515, 501, 515, 503, 529, 530, 495, 499, 522, 526, 508, 562, 512, 519, 504, 523] payload_rows_per_dst=[5512, 5602, 5562, 5544, 5560, 5556, 5544, 5580] dup_rows_per_dst=[2600, 2628, 2698, 2592, 2538, 2618, 2630, 2772] combine_rows_per_src=[5512, 5602, 5562, 5544, 5560, 5556, 5544, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44626 payload_rows_remote=38958 dup_rows=20910 dup_rows_remote=18262 dedup_ratio=0.319061 router_tokens_per_expert=[523, 486, 465, 509, 505, 535, 489, 519, 503, 512, 483, 530, 490, 525, 510, 498, 521, 523, 505, 502, 536, 484, 498, 503, 499, 513, 523, 544, 500, 509, 568, 531, 520, 489, 509, 494, 508, 510, 548, 512, 547, 562, 524, 525, 514, 463, 538, 510, 517, 507, 520, 500, 493, 481, 522, 499, 527, 501, 517, 495, 511, 510, 539, 515] payload_rows_per_dst=[5352, 5596, 5296, 5756, 5674, 5742, 5542, 5668] dup_rows_per_dst=[2420, 2626, 2244, 2978, 2730, 2708, 2556, 2648] combine_rows_per_src=[5352, 5596, 5296, 5756, 5674, 5742, 5542, 5668] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278906880 dispatch_meta_bytes=229144 combine_payload_bytes=278906880 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279136024 ingress_bytes=389007360 link_bytes=389007360 token_bytes=558042904 weight_movement_bytes=110100480 total_nvlink_bytes=668143384 weight_to_token_ratio=0.197298 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.765477798e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278964224 dispatch_meta_bytes=229792 combine_payload_bytes=278964224 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279194016 ingress_bytes=389064704 link_bytes=389064704 token_bytes=558158240 weight_movement_bytes=110100480 total_nvlink_bytes=668258720 weight_to_token_ratio=0.197257 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.766475104e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279035904 dispatch_meta_bytes=228816 combine_payload_bytes=279035904 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=279264720 ingress_bytes=719437824 link_bytes=719437824 token_bytes=558300624 weight_movement_bytes=440401920 total_nvlink_bytes=998702544 weight_to_token_ratio=0.788826 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.251220703e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279595008 dispatch_meta_bytes=229384 combine_payload_bytes=279595008 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279824392 ingress_bytes=499795968 link_bytes=499795968 token_bytes=559419400 weight_movement_bytes=220200960 total_nvlink_bytes=779620360 weight_to_token_ratio=0.393624 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.692273907e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44456 payload_rows_remote=38832 dup_rows=21080 dup_rows_remote=18422 dedup_ratio=0.321655 router_tokens_per_expert=[512, 508, 494, 522, 526, 531, 542, 536, 520, 533, 512, 508, 530, 541, 532, 456, 522, 512, 517, 520, 505, 550, 509, 506, 505, 526, 499, 541, 538, 513, 535, 504, 510, 498, 488, 496, 524, 465, 494, 506, 505, 524, 512, 524, 494, 505, 507, 490, 495, 480, 484, 508, 522, 509, 525, 503, 513, 486, 512, 525, 509, 541, 480, 529] payload_rows_per_dst=[5656, 5554, 5624, 5588, 5440, 5526, 5528, 5540] dup_rows_per_dst=[2686, 2710, 2658, 2734, 2522, 2596, 2524, 2650] combine_rows_per_src=[5656, 5554, 5624, 5588, 5440, 5526, 5528, 5540] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44756 payload_rows_remote=39114 dup_rows=20780 dup_rows_remote=18212 dedup_ratio=0.317078 router_tokens_per_expert=[503, 491, 508, 532, 504, 496, 539, 540, 512, 511, 485, 496, 502, 475, 513, 501, 545, 466, 502, 529, 510, 515, 520, 509, 494, 522, 515, 491, 506, 545, 494, 538, 514, 485, 495, 514, 525, 527, 553, 513, 481, 543, 496, 545, 526, 534, 534, 527, 496, 497, 544, 511, 494, 521, 539, 480, 509, 520, 513, 495, 506, 491, 526, 505] payload_rows_per_dst=[5638, 5532, 5520, 5642, 5622, 5618, 5580, 5604] dup_rows_per_dst=[2588, 2458, 2672, 2568, 2630, 2754, 2584, 2526] combine_rows_per_src=[5638, 5532, 5520, 5642, 5622, 5618, 5580, 5604] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44426 payload_rows_remote=38766 dup_rows=21110 dup_rows_remote=18184 dedup_ratio=0.322113 router_tokens_per_expert=[478, 478, 506, 553, 535, 515, 509, 535, 527, 486, 499, 499, 539, 524, 532, 499, 530, 472, 532, 524, 493, 543, 525, 509, 523, 517, 503, 533, 523, 532, 521, 492, 496, 536, 506, 512, 541, 458, 492, 475, 519, 495, 558, 480, 518, 511, 516, 475, 488, 507, 526, 499, 498, 547, 518, 504, 501, 491, 505, 490, 515, 557, 520, 528] payload_rows_per_dst=[5590, 5430, 5368, 5616, 5616, 5572, 5574, 5660] dup_rows_per_dst=[2578, 2438, 2502, 2602, 2758, 2622, 2684, 2926] combine_rows_per_src=[5590, 5430, 5368, 5616, 5616, 5572, 5574, 5660] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44460 payload_rows_remote=38780 dup_rows=21076 dup_rows_remote=18336 dedup_ratio=0.321594 router_tokens_per_expert=[516, 476, 522, 518, 480, 529, 494, 507, 536, 512, 480, 494, 535, 515, 530, 510, 501, 512, 525, 500, 514, 491, 503, 525, 535, 500, 520, 507, 498, 509, 498, 519, 524, 512, 510, 517, 541, 506, 487, 491, 565, 519, 496, 556, 528, 526, 520, 500, 495, 525, 487, 524, 531, 463, 534, 534, 514, 478, 507, 486, 531, 499, 515, 536] payload_rows_per_dst=[5540, 5512, 5574, 5522, 5588, 5680, 5514, 5530] dup_rows_per_dst=[2544, 2712, 2568, 2650, 2588, 2740, 2672, 2602] combine_rows_per_src=[5540, 5512, 5574, 5522, 5588, 5680, 5514, 5530] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278347776 dispatch_meta_bytes=229016 combine_payload_bytes=278347776 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278576792 ingress_bytes=608649216 link_bytes=608649216 token_bytes=556924568 weight_movement_bytes=330301440 total_nvlink_bytes=887226008 weight_to_token_ratio=0.593081 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058541092e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44430 payload_rows_remote=38886 dup_rows=21106 dup_rows_remote=18402 dedup_ratio=0.322052 router_tokens_per_expert=[481, 481, 519, 536, 518, 504, 520, 481, 501, 520, 506, 521, 473, 551, 502, 522, 516, 504, 525, 508, 511, 495, 505, 570, 532, 501, 513, 534, 497, 497, 504, 505, 520, 505, 484, 528, 478, 504, 546, 529, 509, 487, 526, 532, 542, 455, 511, 524, 495, 511, 480, 509, 525, 550, 538, 516, 536, 534, 516, 482, 521, 488, 511, 523] payload_rows_per_dst=[5588, 5568, 5540, 5512, 5560, 5558, 5544, 5560] dup_rows_per_dst=[2492, 2624, 2728, 2654, 2628, 2614, 2704, 2662] combine_rows_per_src=[5588, 5568, 5540, 5512, 5560, 5558, 5544, 5560] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44572 payload_rows_remote=38976 dup_rows=20964 dup_rows_remote=18298 dedup_ratio=0.319885 router_tokens_per_expert=[479, 481, 527, 548, 495, 519, 541, 535, 500, 533, 528, 504, 552, 468, 499, 547, 561, 520, 528, 519, 539, 526, 519, 534, 545, 529, 501, 492, 491, 482, 530, 547, 474, 499, 560, 532, 491, 479, 484, 492, 461, 502, 475, 532, 510, 517, 507, 492, 514, 477, 491, 517, 542, 549, 544, 531, 478, 505, 496, 528, 492, 504, 493, 481] payload_rows_per_dst=[5690, 5596, 5604, 5554, 5514, 5556, 5636, 5422] dup_rows_per_dst=[2560, 2666, 2888, 2680, 2508, 2436, 2694, 2532] combine_rows_per_src=[5690, 5596, 5604, 5554, 5514, 5556, 5636, 5422] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44226 payload_rows_remote=38644 dup_rows=21310 dup_rows_remote=18604 dedup_ratio=0.325165 router_tokens_per_expert=[539, 498, 532, 499, 532, 499, 489, 497, 511, 533, 537, 479, 509, 514, 543, 521, 484, 533, 519, 504, 501, 511, 498, 498, 528, 527, 494, 510, 483, 496, 488, 515, 549, 502, 565, 510, 504, 494, 517, 503, 480, 501, 520, 519, 481, 509, 538, 526, 517, 506, 524, 536, 527, 536, 531, 479, 515, 506, 483, 515, 516, 523, 510, 505] payload_rows_per_dst=[5472, 5586, 5492, 5484, 5582, 5456, 5582, 5572] dup_rows_per_dst=[2698, 2708, 2604, 2598, 2706, 2692, 2730, 2574] combine_rows_per_src=[5472, 5586, 5492, 5484, 5582, 5456, 5582, 5572] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44424 payload_rows_remote=38848 dup_rows=21112 dup_rows_remote=18472 dedup_ratio=0.322144 router_tokens_per_expert=[488, 511, 514, 529, 536, 515, 506, 508, 525, 510, 544, 526, 512, 519, 503, 531, 526, 519, 511, 515, 495, 511, 535, 486, 529, 513, 503, 512, 503, 529, 526, 502, 531, 507, 524, 501, 490, 523, 549, 512, 495, 500, 517, 496, 519, 536, 506, 489, 490, 490, 499, 506, 513, 492, 515, 488, 508, 527, 522, 484, 515, 496, 535, 501] payload_rows_per_dst=[5576, 5626, 5570, 5512, 5568, 5508, 5528, 5536] dup_rows_per_dst=[2640, 2816, 2624, 2620, 2706, 2608, 2458, 2640] combine_rows_per_src=[5576, 5626, 5570, 5512, 5568, 5508, 5528, 5536] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280369152 dispatch_meta_bytes=229304 combine_payload_bytes=280369152 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280598456 ingress_bytes=610670592 link_bytes=610670592 token_bytes=560967608 weight_movement_bytes=330301440 total_nvlink_bytes=891269048 weight_to_token_ratio=0.588807 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.062056598e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277874688 dispatch_meta_bytes=227800 combine_payload_bytes=277874688 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278102488 ingress_bytes=387975168 link_bytes=387975168 token_bytes=555977176 weight_movement_bytes=110100480 total_nvlink_bytes=666077656 weight_to_token_ratio=0.198031 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.747526281e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277975040 dispatch_meta_bytes=228464 combine_payload_bytes=277975040 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278203504 ingress_bytes=388075520 link_bytes=388075520 token_bytes=556178544 weight_movement_bytes=110100480 total_nvlink_bytes=666279024 weight_to_token_ratio=0.197959 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.749271567e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278734848 dispatch_meta_bytes=229152 combine_payload_bytes=278734848 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278964000 ingress_bytes=388835328 link_bytes=388835328 token_bytes=557698848 weight_movement_bytes=110100480 total_nvlink_bytes=667799328 weight_to_token_ratio=0.197419 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.762485878e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279379968 dispatch_meta_bytes=229096 combine_payload_bytes=279379968 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279609064 ingress_bytes=499580928 link_bytes=499580928 token_bytes=558989032 weight_movement_bytes=220200960 total_nvlink_bytes=779189992 weight_to_token_ratio=0.393927 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.688534007e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277000192 dispatch_meta_bytes=228992 combine_payload_bytes=277000192 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277229184 ingress_bytes=387100672 link_bytes=387100672 token_bytes=554229376 weight_movement_bytes=110100480 total_nvlink_bytes=664329856 weight_to_token_ratio=0.198655 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.732317357e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278462464 dispatch_meta_bytes=229280 combine_payload_bytes=278462464 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278691744 ingress_bytes=498663424 link_bytes=498663424 token_bytes=557154208 weight_movement_bytes=220200960 total_nvlink_bytes=777355168 weight_to_token_ratio=0.395224 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.672577104e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44298 payload_rows_remote=38804 dup_rows=21238 dup_rows_remote=18536 dedup_ratio=0.324066 router_tokens_per_expert=[537, 537, 534, 498, 514, 545, 466, 518, 505, 511, 486, 500, 528, 528, 512, 509, 522, 495, 519, 511, 518, 489, 520, 524, 513, 543, 533, 521, 566, 509, 491, 549, 509, 526, 513, 517, 519, 485, 509, 526, 481, 515, 511, 556, 520, 507, 488, 468, 506, 545, 529, 463, 531, 506, 497, 512, 482, 500, 497, 486, 523, 500, 488, 502] payload_rows_per_dst=[5582, 5456, 5494, 5658, 5586, 5518, 5540, 5464] dup_rows_per_dst=[2716, 2702, 2702, 2792, 2622, 2574, 2638, 2492] combine_rows_per_src=[5582, 5456, 5494, 5658, 5586, 5518, 5540, 5464] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44632 payload_rows_remote=39048 dup_rows=20904 dup_rows_remote=18284 dedup_ratio=0.318970 router_tokens_per_expert=[513, 501, 518, 528, 540, 511, 522, 508, 518, 543, 531, 461, 541, 526, 541, 499, 498, 491, 504, 477, 534, 490, 538, 499, 506, 482, 510, 528, 496, 551, 538, 491, 536, 544, 510, 505, 495, 462, 504, 523, 520, 503, 538, 505, 568, 529, 498, 506, 523, 524, 468, 497, 485, 475, 541, 516, 504, 486, 514, 497, 523, 529, 515, 491] payload_rows_per_dst=[5550, 5628, 5548, 5584, 5618, 5626, 5506, 5572] dup_rows_per_dst=[2732, 2692, 2514, 2620, 2540, 2708, 2552, 2546] combine_rows_per_src=[5550, 5628, 5548, 5584, 5618, 5626, 5506, 5572] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44326 payload_rows_remote=38572 dup_rows=21210 dup_rows_remote=18114 dedup_ratio=0.323639 router_tokens_per_expert=[512, 517, 575, 514, 531, 526, 515, 518, 490, 523, 514, 512, 539, 532, 489, 504, 519, 541, 524, 509, 527, 487, 534, 542, 488, 500, 513, 508, 517, 508, 498, 547, 522, 487, 495, 507, 482, 506, 517, 503, 538, 501, 482, 532, 511, 518, 498, 547, 497, 521, 514, 489, 468, 522, 498, 492, 539, 516, 489, 492, 483, 524, 494, 511] payload_rows_per_dst=[5470, 5438, 5764, 5222, 5500, 5610, 5568, 5754] dup_rows_per_dst=[2580, 2518, 2934, 2184, 2494, 2720, 2684, 3096] combine_rows_per_src=[5470, 5438, 5764, 5222, 5500, 5610, 5568, 5754] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278147072 dispatch_meta_bytes=229360 combine_payload_bytes=278147072 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278376432 ingress_bytes=388247552 link_bytes=388247552 token_bytes=556523504 weight_movement_bytes=110100480 total_nvlink_bytes=666623984 weight_to_token_ratio=0.197836 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.752263487e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44618 payload_rows_remote=39022 dup_rows=20918 dup_rows_remote=18290 dedup_ratio=0.319183 router_tokens_per_expert=[478, 537, 497, 482, 499, 489, 509, 504, 518, 565, 511, 539, 513, 530, 458, 481, 543, 540, 472, 509, 503, 511, 545, 532, 535, 512, 540, 496, 494, 527, 514, 495, 497, 528, 490, 504, 531, 525, 534, 534, 528, 523, 531, 526, 509, 491, 484, 520, 507, 514, 513, 528, 528, 526, 473, 498, 490, 494, 485, 516, 485, 526, 531, 521] payload_rows_per_dst=[5480, 5614, 5602, 5636, 5602, 5596, 5572, 5516] dup_rows_per_dst=[2510, 2616, 2708, 2590, 2684, 2628, 2602, 2580] combine_rows_per_src=[5480, 5614, 5602, 5636, 5602, 5596, 5572, 5516] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44496 payload_rows_remote=38944 dup_rows=21040 dup_rows_remote=18438 dedup_ratio=0.321045 router_tokens_per_expert=[531, 519, 521, 498, 513, 531, 530, 481, 498, 483, 511, 485, 471, 542, 512, 497, 490, 462, 511, 501, 537, 513, 554, 476, 551, 504, 552, 531, 538, 572, 523, 514, 512, 521, 509, 521, 504, 509, 535, 515, 503, 496, 522, 503, 535, 485, 483, 535, 566, 517, 508, 506, 480, 511, 528, 461, 475, 520, 505, 540, 518, 483, 510, 500] payload_rows_per_dst=[5552, 5522, 5480, 5740, 5564, 5540, 5552, 5546] dup_rows_per_dst=[2696, 2476, 2608, 2830, 2688, 2584, 2602, 2556] combine_rows_per_src=[5552, 5522, 5480, 5740, 5564, 5540, 5552, 5546] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44550 payload_rows_remote=38994 dup_rows=20986 dup_rows_remote=18432 dedup_ratio=0.320221 router_tokens_per_expert=[489, 487, 508, 541, 546, 507, 488, 506, 523, 464, 518, 491, 499, 503, 544, 513, 480, 507, 464, 500, 505, 502, 501, 532, 505, 550, 548, 522, 517, 492, 505, 565, 513, 533, 530, 520, 523, 516, 506, 535, 524, 480, 491, 489, 502, 501, 546, 506, 555, 485, 530, 495, 505, 491, 515, 544, 479, 521, 495, 552, 506, 518, 526, 514] payload_rows_per_dst=[5520, 5556, 5548, 5692, 5614, 5516, 5552, 5552] dup_rows_per_dst=[2624, 2554, 2434, 2716, 2738, 2562, 2688, 2670] combine_rows_per_src=[5520, 5556, 5548, 5692, 5614, 5516, 5552, 5552] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44452 payload_rows_remote=38934 dup_rows=21084 dup_rows_remote=18482 dedup_ratio=0.321716 router_tokens_per_expert=[488, 512, 513, 515, 517, 514, 515, 523, 539, 521, 523, 565, 529, 495, 503, 534, 486, 520, 516, 526, 519, 503, 467, 502, 520, 515, 496, 491, 526, 522, 499, 519, 493, 494, 495, 542, 488, 496, 520, 532, 529, 523, 529, 506, 499, 492, 464, 557, 519, 472, 545, 517, 528, 521, 531, 515, 489, 511, 544, 505, 495, 479, 491, 514] payload_rows_per_dst=[5592, 5632, 5550, 5504, 5518, 5560, 5576, 5520] dup_rows_per_dst=[2602, 2786, 2528, 2672, 2602, 2638, 2720, 2536] combine_rows_per_src=[5592, 5632, 5550, 5504, 5518, 5560, 5576, 5520] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279896064 dispatch_meta_bytes=229328 combine_payload_bytes=279896064 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280125392 ingress_bytes=610197504 link_bytes=610197504 token_bytes=560021456 weight_movement_bytes=330301440 total_nvlink_bytes=890322896 weight_to_token_ratio=0.589801 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061233820e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=276484096 dispatch_meta_bytes=226744 combine_payload_bytes=276484096 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276710840 ingress_bytes=386584576 link_bytes=386584576 token_bytes=553194936 weight_movement_bytes=110100480 total_nvlink_bytes=663295416 weight_to_token_ratio=0.199027 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.723341599e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44466 payload_rows_remote=38870 dup_rows=21070 dup_rows_remote=18336 dedup_ratio=0.321503 router_tokens_per_expert=[500, 567, 495, 519, 506, 528, 527, 523, 551, 491, 534, 537, 501, 507, 509, 500, 491, 548, 500, 476, 496, 526, 516, 508, 501, 491, 512, 572, 514, 517, 500, 484, 533, 551, 524, 504, 465, 513, 474, 519, 535, 518, 489, 498, 500, 524, 533, 481, 521, 506, 516, 506, 460, 561, 514, 508, 497, 509, 497, 513, 510, 528, 503, 511] payload_rows_per_dst=[5596, 5572, 5538, 5558, 5598, 5570, 5590, 5444] dup_rows_per_dst=[2734, 2774, 2584, 2538, 2568, 2586, 2594, 2692] combine_rows_per_src=[5596, 5572, 5538, 5558, 5598, 5570, 5590, 5444] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279709696 dispatch_meta_bytes=229248 combine_payload_bytes=279709696 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279938944 ingress_bytes=389810176 link_bytes=389810176 token_bytes=559648640 weight_movement_bytes=110100480 total_nvlink_bytes=669749120 weight_to_token_ratio=0.196731 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.779440088e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44544 payload_rows_remote=38896 dup_rows=20992 dup_rows_remote=18370 dedup_ratio=0.320312 router_tokens_per_expert=[502, 530, 531, 514, 537, 527, 513, 508, 540, 500, 518, 505, 537, 516, 499, 502, 488, 507, 542, 543, 508, 529, 522, 491, 495, 477, 532, 453, 525, 511, 470, 526, 520, 506, 504, 498, 534, 522, 524, 503, 497, 494, 478, 535, 527, 505, 498, 503, 530, 551, 492, 511, 501, 521, 536, 493, 563, 500, 517, 519, 490, 508, 515, 475] payload_rows_per_dst=[5602, 5548, 5588, 5470, 5554, 5554, 5648, 5580] dup_rows_per_dst=[2722, 2686, 2672, 2508, 2668, 2520, 2622, 2594] combine_rows_per_src=[5602, 5548, 5588, 5470, 5554, 5554, 5648, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278835200 dispatch_meta_bytes=229752 combine_payload_bytes=278835200 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279064952 ingress_bytes=388935680 link_bytes=388935680 token_bytes=557900152 weight_movement_bytes=110100480 total_nvlink_bytes=668000632 weight_to_token_ratio=0.197348 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.764231164e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279250944 dispatch_meta_bytes=228880 combine_payload_bytes=279250944 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279479824 ingress_bytes=389351424 link_bytes=389351424 token_bytes=558730768 weight_movement_bytes=110100480 total_nvlink_bytes=668831248 weight_to_token_ratio=0.197055 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.771461636e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44564 payload_rows_remote=38978 dup_rows=20972 dup_rows_remote=18326 dedup_ratio=0.320007 router_tokens_per_expert=[476, 480, 522, 547, 491, 518, 537, 531, 499, 532, 526, 499, 550, 467, 497, 546, 559, 518, 530, 514, 540, 524, 516, 534, 543, 527, 498, 486, 492, 483, 530, 547, 475, 501, 560, 530, 493, 481, 485, 494, 460, 503, 482, 530, 514, 517, 515, 494, 516, 479, 493, 520, 543, 551, 543, 532, 479, 508, 499, 530, 494, 508, 497, 483] payload_rows_per_dst=[5672, 5586, 5596, 5550, 5520, 5566, 5640, 5434] dup_rows_per_dst=[2532, 2646, 2874, 2662, 2518, 2464, 2714, 2562] combine_rows_per_src=[5672, 5586, 5596, 5550, 5520, 5566, 5640, 5434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44576 payload_rows_remote=39098 dup_rows=20960 dup_rows_remote=18418 dedup_ratio=0.319824 router_tokens_per_expert=[511, 455, 499, 520, 536, 551, 508, 528, 502, 495, 534, 496, 512, 487, 513, 528, 538, 545, 534, 500, 510, 527, 551, 512, 508, 529, 507, 461, 515, 517, 475, 498, 489, 490, 513, 542, 483, 481, 507, 533, 520, 515, 530, 488, 509, 523, 562, 513, 509, 505, 500, 522, 526, 482, 513, 525, 518, 493, 480, 532, 548, 473, 540, 502] payload_rows_per_dst=[5536, 5526, 5676, 5478, 5554, 5668, 5586, 5552] dup_rows_per_dst=[2680, 2608, 2758, 2542, 2522, 2652, 2578, 2620] combine_rows_per_src=[5536, 5526, 5676, 5478, 5554, 5668, 5586, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44414 payload_rows_remote=38830 dup_rows=21122 dup_rows_remote=18534 dedup_ratio=0.322296 router_tokens_per_expert=[504, 529, 497, 515, 522, 505, 509, 549, 511, 542, 508, 511, 535, 518, 518, 547, 515, 521, 493, 508, 524, 516, 524, 485, 503, 522, 506, 505, 527, 524, 474, 481, 512, 531, 509, 504, 475, 556, 477, 520, 505, 507, 494, 482, 499, 464, 503, 513, 503, 515, 530, 552, 505, 520, 565, 473, 479, 510, 480, 487, 535, 539, 514, 562] payload_rows_per_dst=[5580, 5618, 5584, 5440, 5568, 5412, 5630, 5582] dup_rows_per_dst=[2680, 2762, 2588, 2644, 2600, 2522, 2696, 2630] combine_rows_per_src=[5580, 5618, 5584, 5440, 5568, 5412, 5630, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278806528 dispatch_meta_bytes=229064 combine_payload_bytes=278806528 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279035592 ingress_bytes=499007488 link_bytes=499007488 token_bytes=557842120 weight_movement_bytes=220200960 total_nvlink_bytes=778043080 weight_to_token_ratio=0.394737 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.678560943e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44732 payload_rows_remote=39138 dup_rows=20804 dup_rows_remote=18214 dedup_ratio=0.317444 router_tokens_per_expert=[484, 526, 519, 536, 539, 499, 522, 496, 527, 514, 511, 482, 515, 478, 507, 504, 532, 506, 538, 492, 534, 527, 531, 546, 531, 520, 518, 524, 500, 483, 515, 499, 509, 504, 488, 525, 515, 505, 543, 529, 543, 511, 507, 463, 516, 519, 531, 502, 522, 488, 500, 500, 532, 516, 512, 492, 512, 511, 491, 506, 512, 488, 539, 482] payload_rows_per_dst=[5614, 5528, 5714, 5552, 5598, 5594, 5556, 5576] dup_rows_per_dst=[2628, 2548, 2698, 2628, 2638, 2590, 2568, 2506] combine_rows_per_src=[5614, 5528, 5714, 5552, 5598, 5594, 5556, 5576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44522 payload_rows_remote=38928 dup_rows=21014 dup_rows_remote=18288 dedup_ratio=0.320648 router_tokens_per_expert=[517, 484, 507, 528, 525, 536, 537, 502, 503, 478, 521, 492, 514, 516, 513, 510, 513, 544, 530, 507, 529, 514, 516, 479, 501, 508, 484, 513, 506, 508, 515, 521, 539, 561, 500, 478, 508, 520, 505, 510, 523, 498, 555, 537, 523, 535, 507, 492, 489, 527, 494, 507, 509, 492, 504, 507, 537, 518, 510, 500, 495, 503, 499, 515] payload_rows_per_dst=[5594, 5508, 5544, 5484, 5654, 5596, 5556, 5586] dup_rows_per_dst=[2726, 2586, 2720, 2628, 2588, 2744, 2454, 2568] combine_rows_per_src=[5594, 5508, 5544, 5484, 5654, 5596, 5556, 5586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279394304 dispatch_meta_bytes=229216 combine_payload_bytes=279394304 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279623520 ingress_bytes=499595264 link_bytes=499595264 token_bytes=559017824 weight_movement_bytes=220200960 total_nvlink_bytes=779218784 weight_to_token_ratio=0.393907 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.688783334e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280254464 dispatch_meta_bytes=230064 combine_payload_bytes=280254464 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280484528 ingress_bytes=390354944 link_bytes=390354944 token_bytes=560738992 weight_movement_bytes=110100480 total_nvlink_bytes=670839472 weight_to_token_ratio=0.196349 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.788914500e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278333440 dispatch_meta_bytes=229456 combine_payload_bytes=278333440 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278562896 ingress_bytes=608634880 link_bytes=608634880 token_bytes=556896336 weight_movement_bytes=330301440 total_nvlink_bytes=887197776 weight_to_token_ratio=0.593111 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058516160e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280541184 dispatch_meta_bytes=229408 combine_payload_bytes=280541184 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280770592 ingress_bytes=390641664 link_bytes=390641664 token_bytes=561311776 weight_movement_bytes=110100480 total_nvlink_bytes=671412256 weight_to_token_ratio=0.196149 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.793901032e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44236 payload_rows_remote=38660 dup_rows=21300 dup_rows_remote=18596 dedup_ratio=0.325012 router_tokens_per_expert=[532, 493, 529, 498, 529, 494, 489, 495, 509, 532, 534, 480, 508, 514, 541, 520, 484, 534, 518, 505, 501, 509, 493, 497, 529, 526, 495, 511, 483, 494, 488, 514, 548, 502, 565, 510, 504, 492, 515, 504, 481, 501, 520, 520, 483, 511, 541, 527, 517, 507, 526, 536, 528, 537, 534, 481, 517, 512, 485, 518, 519, 523, 516, 510] payload_rows_per_dst=[5450, 5580, 5488, 5490, 5576, 5466, 5588, 5598] dup_rows_per_dst=[2668, 2696, 2594, 2590, 2704, 2702, 2744, 2602] combine_rows_per_src=[5450, 5580, 5488, 5490, 5576, 5466, 5588, 5598] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279035904 dispatch_meta_bytes=228864 combine_payload_bytes=279035904 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=279264768 ingress_bytes=719437824 link_bytes=719437824 token_bytes=558300672 weight_movement_bytes=440401920 total_nvlink_bytes=998702592 weight_to_token_ratio=0.788826 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.251220703e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44472 payload_rows_remote=38886 dup_rows=21064 dup_rows_remote=18368 dedup_ratio=0.321411 router_tokens_per_expert=[469, 477, 503, 550, 532, 511, 506, 532, 522, 479, 499, 498, 541, 523, 529, 498, 530, 471, 531, 521, 493, 541, 523, 506, 524, 516, 500, 533, 523, 534, 519, 491, 495, 539, 504, 513, 541, 460, 494, 476, 521, 496, 559, 480, 517, 512, 522, 475, 491, 508, 530, 502, 500, 550, 522, 512, 501, 496, 508, 492, 519, 558, 522, 528] payload_rows_per_dst=[5586, 5544, 5524, 5630, 5482, 5506, 5614, 5586] dup_rows_per_dst=[2574, 2614, 2708, 2636, 2562, 2532, 2742, 2696] combine_rows_per_src=[5586, 5544, 5524, 5630, 5482, 5506, 5614, 5586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277114880 dispatch_meta_bytes=229024 combine_payload_bytes=277114880 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277343904 ingress_bytes=387215360 link_bytes=387215360 token_bytes=554458784 weight_movement_bytes=110100480 total_nvlink_bytes=664559264 weight_to_token_ratio=0.198573 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.734311970e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44422 payload_rows_remote=38864 dup_rows=21114 dup_rows_remote=18404 dedup_ratio=0.322174 router_tokens_per_expert=[479, 476, 515, 534, 516, 502, 517, 479, 498, 520, 505, 518, 472, 548, 500, 519, 516, 498, 527, 506, 512, 494, 506, 569, 534, 501, 513, 535, 496, 496, 506, 504, 522, 505, 484, 525, 480, 504, 545, 532, 510, 487, 525, 532, 541, 456, 512, 527, 497, 513, 479, 510, 527, 553, 539, 516, 539, 537, 517, 484, 526, 492, 514, 527] payload_rows_per_dst=[5566, 5562, 5532, 5504, 5558, 5560, 5558, 5582] dup_rows_per_dst=[2470, 2598, 2724, 2666, 2636, 2620, 2710, 2690] combine_rows_per_src=[5566, 5562, 5532, 5504, 5558, 5560, 5558, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278734848 dispatch_meta_bytes=229016 combine_payload_bytes=278734848 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278963864 ingress_bytes=388835328 link_bytes=388835328 token_bytes=557698712 weight_movement_bytes=110100480 total_nvlink_bytes=667799192 weight_to_token_ratio=0.197419 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.762485878e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44542 payload_rows_remote=39004 dup_rows=20994 dup_rows_remote=18470 dedup_ratio=0.320343 router_tokens_per_expert=[485, 484, 507, 538, 545, 505, 486, 503, 523, 458, 519, 488, 493, 500, 542, 508, 477, 505, 463, 498, 499, 502, 503, 534, 503, 551, 544, 521, 520, 492, 505, 565, 513, 534, 531, 520, 522, 518, 505, 538, 526, 483, 492, 491, 503, 503, 543, 508, 556, 489, 532, 495, 504, 492, 517, 548, 481, 523, 498, 555, 508, 524, 531, 519] payload_rows_per_dst=[5504, 5538, 5536, 5686, 5610, 5528, 5570, 5570] dup_rows_per_dst=[2602, 2524, 2426, 2716, 2752, 2570, 2696, 2708] combine_rows_per_src=[5504, 5538, 5536, 5686, 5610, 5528, 5570, 5570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44766 payload_rows_remote=39122 dup_rows=20770 dup_rows_remote=18208 dedup_ratio=0.316925 router_tokens_per_expert=[500, 489, 506, 527, 500, 492, 538, 533, 512, 507, 483, 494, 501, 473, 512, 498, 541, 465, 500, 527, 510, 513, 519, 508, 491, 519, 518, 490, 510, 543, 491, 541, 514, 486, 497, 515, 526, 529, 552, 515, 484, 539, 498, 545, 530, 538, 535, 527, 497, 502, 546, 512, 500, 523, 537, 483, 513, 523, 516, 498, 508, 491, 529, 509] payload_rows_per_dst=[5618, 5522, 5512, 5644, 5630, 5620, 5594, 5626] dup_rows_per_dst=[2552, 2438, 2654, 2562, 2638, 2772, 2606, 2548] combine_rows_per_src=[5618, 5522, 5512, 5644, 5630, 5620, 5594, 5626] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44466 payload_rows_remote=38852 dup_rows=21070 dup_rows_remote=18420 dedup_ratio=0.321503 router_tokens_per_expert=[508, 506, 489, 521, 525, 526, 541, 535, 516, 530, 509, 506, 529, 541, 532, 455, 520, 512, 513, 521, 502, 548, 511, 505, 506, 527, 499, 538, 536, 513, 535, 501, 508, 496, 489, 496, 524, 462, 496, 508, 509, 526, 513, 528, 496, 507, 509, 490, 497, 482, 484, 509, 527, 510, 531, 504, 516, 487, 515, 524, 510, 542, 483, 534] payload_rows_per_dst=[5646, 5540, 5614, 5592, 5436, 5536, 5552, 5550] dup_rows_per_dst=[2656, 2696, 2650, 2718, 2522, 2620, 2536, 2672] combine_rows_per_src=[5646, 5540, 5614, 5592, 5436, 5536, 5552, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44454 payload_rows_remote=38766 dup_rows=21082 dup_rows_remote=18318 dedup_ratio=0.321686 router_tokens_per_expert=[512, 476, 519, 511, 478, 527, 494, 508, 533, 509, 477, 492, 533, 513, 531, 508, 498, 508, 524, 499, 513, 492, 503, 524, 536, 499, 517, 505, 499, 510, 497, 517, 524, 512, 510, 518, 542, 504, 489, 491, 566, 521, 500, 556, 532, 525, 524, 502, 496, 527, 490, 525, 531, 466, 537, 536, 518, 478, 509, 488, 534, 501, 515, 539] payload_rows_per_dst=[5528, 5498, 5566, 5516, 5594, 5688, 5520, 5544] dup_rows_per_dst=[2522, 2694, 2556, 2644, 2586, 2764, 2696, 2620] combine_rows_per_src=[5528, 5498, 5566, 5516, 5594, 5688, 5520, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278577152 dispatch_meta_bytes=229072 combine_payload_bytes=278577152 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278806224 ingress_bytes=498778112 link_bytes=498778112 token_bytes=557383376 weight_movement_bytes=220200960 total_nvlink_bytes=777584336 weight_to_token_ratio=0.395062 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.674571717e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44402 payload_rows_remote=38722 dup_rows=21134 dup_rows_remote=18318 dedup_ratio=0.322479 router_tokens_per_expert=[487, 508, 512, 525, 533, 512, 504, 505, 522, 509, 542, 524, 512, 514, 503, 530, 524, 518, 510, 515, 493, 509, 535, 485, 529, 511, 502, 513, 501, 528, 527, 500, 531, 508, 523, 501, 490, 521, 548, 513, 494, 500, 517, 498, 521, 535, 509, 491, 492, 492, 499, 509, 516, 495, 519, 491, 514, 528, 525, 487, 518, 499, 539, 503] payload_rows_per_dst=[5680, 5588, 5470, 5520, 5680, 5508, 5398, 5558] dup_rows_per_dst=[2816, 2734, 2474, 2612, 2894, 2612, 2324, 2668] combine_rows_per_src=[5680, 5588, 5470, 5520, 5680, 5508, 5398, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279580672 dispatch_meta_bytes=229896 combine_payload_bytes=279580672 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279810568 ingress_bytes=389681152 link_bytes=389681152 token_bytes=559391240 weight_movement_bytes=110100480 total_nvlink_bytes=669491720 weight_to_token_ratio=0.196822 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.777196149e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280426496 dispatch_meta_bytes=229320 combine_payload_bytes=280426496 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280655816 ingress_bytes=610727936 link_bytes=610727936 token_bytes=561082312 weight_movement_bytes=330301440 total_nvlink_bytes=891383752 weight_to_token_ratio=0.588686 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.062156328e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278491136 dispatch_meta_bytes=229088 combine_payload_bytes=278491136 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278720224 ingress_bytes=498692096 link_bytes=498692096 token_bytes=557211360 weight_movement_bytes=220200960 total_nvlink_bytes=777412320 weight_to_token_ratio=0.395184 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.673075757e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44454 payload_rows_remote=38946 dup_rows=21082 dup_rows_remote=18484 dedup_ratio=0.321686 router_tokens_per_expert=[487, 512, 512, 510, 513, 509, 511, 520, 535, 519, 521, 565, 520, 494, 504, 534, 487, 519, 510, 526, 520, 503, 467, 497, 520, 514, 498, 489, 527, 523, 501, 525, 493, 494, 495, 538, 487, 494, 524, 528, 530, 524, 533, 507, 500, 491, 465, 557, 520, 473, 549, 518, 527, 526, 536, 522, 493, 515, 544, 506, 496, 481, 492, 518] payload_rows_per_dst=[5570, 5616, 5550, 5524, 5508, 5558, 5594, 5534] dup_rows_per_dst=[2578, 2768, 2508, 2670, 2598, 2656, 2748, 2556] combine_rows_per_src=[5570, 5616, 5550, 5524, 5508, 5558, 5594, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277874688 dispatch_meta_bytes=228336 combine_payload_bytes=277874688 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=278103024 ingress_bytes=498075648 link_bytes=498075648 token_bytes=555977712 weight_movement_bytes=220200960 total_nvlink_bytes=776178672 weight_to_token_ratio=0.396061 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.662354712e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277559296 dispatch_meta_bytes=228160 combine_payload_bytes=277559296 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277787456 ingress_bytes=387659776 link_bytes=387659776 token_bytes=555346752 weight_movement_bytes=110100480 total_nvlink_bytes=665447232 weight_to_token_ratio=0.198255 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.742041095e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44428 payload_rows_remote=38752 dup_rows=21108 dup_rows_remote=18238 dedup_ratio=0.322083 router_tokens_per_expert=[512, 513, 572, 513, 529, 524, 514, 516, 487, 522, 510, 508, 537, 529, 487, 501, 514, 540, 526, 508, 525, 484, 536, 543, 485, 500, 513, 505, 516, 511, 499, 547, 524, 485, 496, 504, 484, 508, 515, 505, 537, 505, 484, 532, 509, 519, 501, 550, 499, 522, 520, 488, 468, 521, 498, 495, 540, 522, 490, 496, 486, 528, 495, 516] payload_rows_per_dst=[5588, 5534, 5652, 5418, 5478, 5612, 5470, 5676] dup_rows_per_dst=[2750, 2628, 2700, 2360, 2494, 2754, 2552, 2870] combine_rows_per_src=[5588, 5534, 5652, 5418, 5478, 5612, 5470, 5676] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279150592 dispatch_meta_bytes=229528 combine_payload_bytes=279150592 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279380120 ingress_bytes=389251072 link_bytes=389251072 token_bytes=558530712 weight_movement_bytes=110100480 total_nvlink_bytes=668631192 weight_to_token_ratio=0.197125 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.769716350e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279508992 dispatch_meta_bytes=229704 combine_payload_bytes=279508992 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279738696 ingress_bytes=499709952 link_bytes=499709952 token_bytes=559247688 weight_movement_bytes=220200960 total_nvlink_bytes=779448648 weight_to_token_ratio=0.393745 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.690777947e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279078912 dispatch_meta_bytes=229664 combine_payload_bytes=279078912 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279308576 ingress_bytes=499279872 link_bytes=499279872 token_bytes=558387488 weight_movement_bytes=220200960 total_nvlink_bytes=778588448 weight_to_token_ratio=0.394352 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.683298148e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278620160 dispatch_meta_bytes=228824 combine_payload_bytes=278620160 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278848984 ingress_bytes=608921600 link_bytes=608921600 token_bytes=557469144 weight_movement_bytes=330301440 total_nvlink_bytes=887770584 weight_to_token_ratio=0.592502 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059014813e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44602 payload_rows_remote=39014 dup_rows=20934 dup_rows_remote=18202 dedup_ratio=0.319427 router_tokens_per_expert=[495, 496, 510, 499, 523, 505, 517, 495, 499, 459, 513, 558, 489, 522, 543, 518, 514, 508, 482, 558, 524, 517, 477, 580, 515, 518, 485, 549, 523, 514, 532, 522, 541, 475, 521, 529, 497, 501, 504, 509, 537, 493, 512, 521, 524, 490, 447, 516, 537, 528, 508, 515, 477, 521, 515, 507, 490, 512, 496, 520, 507, 496, 536, 527] payload_rows_per_dst=[5510, 5540, 5588, 5676, 5580, 5584, 5578, 5546] dup_rows_per_dst=[2570, 2662, 2732, 2640, 2574, 2496, 2638, 2622] combine_rows_per_src=[5510, 5540, 5588, 5676, 5580, 5584, 5578, 5546] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44510 payload_rows_remote=38978 dup_rows=21026 dup_rows_remote=18454 dedup_ratio=0.320831 router_tokens_per_expert=[554, 491, 490, 533, 509, 512, 522, 520, 536, 500, 504, 520, 503, 501, 511, 482, 501, 520, 529, 509, 494, 521, 509, 540, 507, 526, 479, 502, 519, 517, 480, 522, 475, 569, 536, 528, 545, 528, 484, 520, 503, 511, 532, 513, 521, 526, 517, 529, 476, 527, 505, 533, 531, 491, 495, 513, 507, 481, 498, 498, 493, 514, 523, 483] payload_rows_per_dst=[5552, 5580, 5610, 5532, 5602, 5620, 5522, 5492] dup_rows_per_dst=[2710, 2534, 2636, 2572, 2768, 2684, 2620, 2502] combine_rows_per_src=[5552, 5580, 5610, 5532, 5602, 5620, 5522, 5492] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44306 payload_rows_remote=38522 dup_rows=21230 dup_rows_remote=18156 dedup_ratio=0.323944 router_tokens_per_expert=[491, 495, 496, 492, 501, 498, 535, 544, 502, 539, 520, 505, 506, 509, 530, 526, 515, 506, 487, 499, 530, 525, 496, 494, 533, 518, 480, 524, 550, 494, 532, 524, 498, 484, 559, 524, 530, 503, 535, 528, 498, 512, 505, 535, 549, 529, 545, 494, 483, 499, 481, 488, 518, 525, 526, 500, 489, 517, 490, 508, 532, 475, 493, 520] payload_rows_per_dst=[5102, 5416, 5424, 5650, 5670, 5624, 5636, 5784] dup_rows_per_dst=[2192, 2590, 2476, 2694, 2856, 2712, 2636, 3074] combine_rows_per_src=[5102, 5416, 5424, 5650, 5670, 5624, 5636, 5784] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279652352 dispatch_meta_bytes=228864 combine_payload_bytes=279652352 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=279881216 ingress_bytes=389752832 link_bytes=389752832 token_bytes=559533568 weight_movement_bytes=110100480 total_nvlink_bytes=669634048 weight_to_token_ratio=0.196772 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778442782e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44440 payload_rows_remote=38904 dup_rows=21096 dup_rows_remote=18406 dedup_ratio=0.321899 router_tokens_per_expert=[473, 504, 515, 566, 482, 516, 501, 538, 527, 518, 492, 533, 520, 504, 513, 517, 525, 509, 519, 498, 482, 519, 508, 547, 499, 542, 522, 563, 510, 548, 473, 495, 472, 530, 500, 506, 487, 497, 479, 521, 515, 520, 457, 504, 501, 521, 508, 549, 513, 522, 525, 492, 486, 506, 490, 579, 505, 517, 515, 512, 484, 515, 561, 501] payload_rows_per_dst=[5532, 5642, 5548, 5576, 5424, 5576, 5536, 5606] dup_rows_per_dst=[2658, 2606, 2666, 2728, 2560, 2574, 2690, 2614] combine_rows_per_src=[5532, 5642, 5548, 5576, 5424, 5576, 5536, 5606] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44380 payload_rows_remote=38930 dup_rows=21156 dup_rows_remote=18526 dedup_ratio=0.322815 router_tokens_per_expert=[542, 499, 572, 499, 533, 513, 547, 534, 529, 531, 489, 499, 515, 518, 484, 474, 495, 543, 467, 509, 524, 536, 512, 509, 512, 493, 533, 464, 538, 483, 532, 528, 535, 521, 523, 527, 510, 550, 529, 509, 513, 491, 500, 488, 515, 520, 532, 481, 487, 531, 508, 497, 509, 528, 509, 503, 503, 474, 481, 512, 503, 482, 529, 512] payload_rows_per_dst=[5620, 5532, 5580, 5496, 5694, 5450, 5538, 5470] dup_rows_per_dst=[2858, 2546, 2610, 2670, 2714, 2630, 2606, 2522] combine_rows_per_src=[5620, 5532, 5580, 5496, 5694, 5450, 5538, 5470] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44418 payload_rows_remote=38788 dup_rows=21118 dup_rows_remote=18436 dedup_ratio=0.322235 router_tokens_per_expert=[511, 475, 520, 532, 530, 494, 525, 507, 519, 565, 558, 510, 491, 519, 493, 501, 518, 513, 515, 523, 532, 479, 502, 538, 469, 537, 526, 489, 514, 530, 493, 494, 511, 516, 530, 539, 456, 507, 521, 510, 534, 497, 500, 494, 514, 505, 518, 528, 509, 516, 502, 499, 511, 482, 529, 503, 526, 518, 517, 536, 511, 510, 492, 505] payload_rows_per_dst=[5530, 5630, 5504, 5546, 5520, 5550, 5542, 5596] dup_rows_per_dst=[2658, 2682, 2736, 2558, 2660, 2630, 2560, 2634] combine_rows_per_src=[5530, 5630, 5504, 5546, 5520, 5550, 5542, 5596] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279394304 dispatch_meta_bytes=229728 combine_payload_bytes=279394304 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279624032 ingress_bytes=389494784 link_bytes=389494784 token_bytes=559018336 weight_movement_bytes=110100480 total_nvlink_bytes=669118816 weight_to_token_ratio=0.196953 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773954903e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44614 payload_rows_remote=39018 dup_rows=20922 dup_rows_remote=18334 dedup_ratio=0.319244 router_tokens_per_expert=[513, 500, 497, 522, 497, 546, 547, 553, 499, 518, 554, 528, 514, 515, 544, 512, 484, 493, 509, 503, 546, 503, 546, 515, 476, 488, 506, 527, 472, 507, 480, 514, 498, 511, 525, 524, 513, 512, 497, 512, 511, 557, 452, 565, 508, 526, 493, 503, 496, 541, 512, 515, 487, 520, 516, 521, 521, 539, 494, 498, 501, 491, 490, 491] payload_rows_per_dst=[5722, 5674, 5548, 5480, 5596, 5482, 5574, 5538] dup_rows_per_dst=[2628, 2694, 2650, 2460, 2588, 2748, 2642, 2512] combine_rows_per_src=[5722, 5674, 5548, 5480, 5596, 5482, 5574, 5538] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5585963856 prefetch_bytes=770703360 reduce_bytes=1541406720 weight_movement_bytes=2312110080 egress_bytes=2794128208 ingress_bytes=5103945728 link_bytes=5103945728 total_nvlink_bytes=7898073936 weight_to_token_ratio=0.413914 prefetch_share_of_ingress=0.151001 dedup_ratio=0.320499 prefetch_slots=14 unused_prefetch_slots=146 slot_utilization=0.087500 reduce_slots=14 remote_demand=7 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=12319 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.876601076e-05 prefetch_bytes_per_gemm_flop=1.340379902e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44616 payload_rows_remote=38836 dup_rows=20920 dup_rows_remote=18006 dedup_ratio=0.319214 router_tokens_per_expert=[505, 520, 523, 506, 500, 524, 519, 510, 496, 519, 519, 508, 511, 508, 547, 496, 499, 527, 506, 498, 549, 490, 534, 511, 516, 488, 524, 509, 512, 535, 539, 506, 502, 516, 527, 489, 508, 531, 527, 505, 499, 537, 517, 492, 509, 481, 513, 540, 487, 522, 510, 509, 528, 534, 498, 521, 515, 485, 533, 505, 492, 516, 480, 486] payload_rows_per_dst=[5780, 5550, 5544, 5674, 5394, 5574, 5572, 5528] dup_rows_per_dst=[2914, 2658, 2684, 2584, 2336, 2602, 2646, 2496] combine_rows_per_src=[5780, 5550, 5544, 5674, 5394, 5574, 5572, 5528] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=276125696 dispatch_meta_bytes=226712 combine_payload_bytes=276125696 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276352408 ingress_bytes=386226176 link_bytes=386226176 token_bytes=552478104 weight_movement_bytes=110100480 total_nvlink_bytes=662578584 weight_to_token_ratio=0.199285 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.717108434e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278863872 dispatch_meta_bytes=229240 combine_payload_bytes=278863872 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279093112 ingress_bytes=388964352 link_bytes=388964352 token_bytes=557956984 weight_movement_bytes=110100480 total_nvlink_bytes=668057464 weight_to_token_ratio=0.197328 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.764729818e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279050240 dispatch_meta_bytes=229824 combine_payload_bytes=279050240 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279280064 ingress_bytes=389150720 link_bytes=389150720 token_bytes=558330304 weight_movement_bytes=110100480 total_nvlink_bytes=668430784 weight_to_token_ratio=0.197196 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.767971064e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278032384 dispatch_meta_bytes=228896 combine_payload_bytes=278032384 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278261280 ingress_bytes=608333824 link_bytes=608333824 token_bytes=556293664 weight_movement_bytes=330301440 total_nvlink_bytes=886595104 weight_to_token_ratio=0.593754 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057992574e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5592474832 prefetch_bytes=990904320 reduce_bytes=990904320 weight_movement_bytes=1981808640 egress_bytes=2797384912 ingress_bytes=4776898560 link_bytes=4776898560 total_nvlink_bytes=7574283472 weight_to_token_ratio=0.354371 prefetch_share_of_ingress=0.207437 dedup_ratio=0.320175 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=9 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040074 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.307812261e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279681024 dispatch_meta_bytes=229408 combine_payload_bytes=279681024 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279910432 ingress_bytes=499881984 link_bytes=499881984 token_bytes=559591456 weight_movement_bytes=220200960 total_nvlink_bytes=779792416 weight_to_token_ratio=0.393503 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.693769866e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5555358808 prefetch_bytes=1101004800 reduce_bytes=110100480 weight_movement_bytes=1211105280 egress_bytes=2778819672 ingress_bytes=3987644416 link_bytes=3987644416 total_nvlink_bytes=6766464088 weight_to_token_ratio=0.218007 prefetch_share_of_ingress=0.276104 dedup_ratio=0.322290 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=1 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.044485 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=6.935169494e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278376448 dispatch_meta_bytes=227368 combine_payload_bytes=278376448 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278603816 ingress_bytes=498577408 link_bytes=498577408 token_bytes=556980264 weight_movement_bytes=220200960 total_nvlink_bytes=777181224 weight_to_token_ratio=0.395348 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.671081144e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5588287568 prefetch_bytes=1101004800 reduce_bytes=0 weight_movement_bytes=1101004800 egress_bytes=2795290704 ingress_bytes=3894001664 link_bytes=3894001664 total_nvlink_bytes=6689292368 weight_to_token_ratio=0.197020 prefetch_share_of_ingress=0.282744 dedup_ratio=0.320499 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=0 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043382 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=6.772309347e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5582725048 prefetch_bytes=880803840 reduce_bytes=440401920 weight_movement_bytes=1321205760 egress_bytes=2792509368 ingress_bytes=4111421440 link_bytes=4111421440 total_nvlink_bytes=6903930808 weight_to_token_ratio=0.236660 prefetch_share_of_ingress=0.214233 dedup_ratio=0.321237 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=4 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.150438097e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5587770392 prefetch_bytes=660602880 reduce_bytes=2422210560 weight_movement_bytes=3082813440 egress_bytes=2795031576 ingress_bytes=5875552256 link_bytes=5875552256 total_nvlink_bytes=8670583832 weight_to_token_ratio=0.551707 prefetch_share_of_ingress=0.112432 dedup_ratio=0.320416 prefetch_slots=12 unused_prefetch_slots=148 slot_utilization=0.075000 reduce_slots=22 remote_demand=6 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=12192 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.021855174e-04 prefetch_bytes_per_gemm_flop=1.148897059e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5574983056 prefetch_bytes=1101004800 reduce_bytes=770703360 weight_movement_bytes=1871708160 egress_bytes=2788638096 ingress_bytes=4658053120 link_bytes=4658053120 total_nvlink_bytes=7446691216 weight_to_token_ratio=0.335733 prefetch_share_of_ingress=0.236366 dedup_ratio=0.321857 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=7 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.039338 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.101120494e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5584701192 prefetch_bytes=990904320 reduce_bytes=1321205760 weight_movement_bytes=2312110080 egress_bytes=2793496328 ingress_bytes=5103314944 link_bytes=5103314944 total_nvlink_bytes=7896811272 weight_to_token_ratio=0.414008 prefetch_share_of_ingress=0.194169 dedup_ratio=0.320517 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=12 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040074 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.875504039e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none - [2026-08-26 15:12:40] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 358.7 | throughput per GPU (TFLOP/s/GPU): 344.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.531658E+00 | loss scale: 1.0 | grad norm: 3.699 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44640 payload_rows_remote=39074 dup_rows=20896 dup_rows_remote=18368 dedup_ratio=0.318848 router_tokens_per_expert=[453, 488, 560, 511, 520, 518, 525, 533, 512, 522, 510, 547, 533, 475, 521, 506, 489, 519, 500, 502, 516, 506, 495, 520, 521, 534, 492, 528, 519, 506, 523, 472, 493, 525, 535, 501, 512, 535, 512, 499, 519, 482, 525, 514, 573, 527, 468, 553, 516, 492, 497, 478, 541, 496, 503, 492, 523, 492, 493, 511, 521, 523, 529, 512] payload_rows_per_dst=[5584, 5628, 5566, 5588, 5602, 5654, 5470, 5548] dup_rows_per_dst=[2632, 2624, 2528, 2602, 2622, 2668, 2560, 2660] combine_rows_per_src=[5584, 5628, 5566, 5588, 5602, 5654, 5470, 5548] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44588 payload_rows_remote=38864 dup_rows=20948 dup_rows_remote=18168 dedup_ratio=0.319641 router_tokens_per_expert=[526, 514, 538, 513, 486, 504, 494, 518, 516, 493, 533, 530, 504, 509, 474, 527, 521, 515, 526, 512, 513, 538, 493, 502, 554, 521, 521, 528, 501, 482, 543, 505, 472, 519, 496, 549, 496, 522, 485, 500, 481, 529, 517, 478, 534, 499, 510, 499, 508, 492, 555, 515, 544, 515, 512, 470, 497, 511, 540, 544, 492, 493, 529, 511] payload_rows_per_dst=[5374, 5414, 5530, 5642, 5616, 5502, 5786, 5724] dup_rows_per_dst=[2222, 2460, 2600, 2884, 2476, 2606, 2920, 2780] combine_rows_per_src=[5374, 5414, 5530, 5642, 5616, 5502, 5786, 5724] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44354 payload_rows_remote=38900 dup_rows=21182 dup_rows_remote=18544 dedup_ratio=0.323212 router_tokens_per_expert=[502, 520, 542, 528, 518, 518, 538, 513, 521, 520, 525, 517, 525, 519, 524, 536, 546, 506, 517, 499, 502, 513, 523, 523, 460, 532, 513, 505, 488, 505, 523, 520, 525, 515, 541, 473, 491, 520, 509, 536, 494, 568, 515, 450, 521, 501, 536, 520, 480, 502, 543, 482, 506, 504, 499, 507, 496, 504, 474, 510, 503, 518, 508, 476] payload_rows_per_dst=[5582, 5572, 5600, 5454, 5572, 5548, 5502, 5524] dup_rows_per_dst=[2776, 2802, 2658, 2638, 2648, 2662, 2544, 2454] combine_rows_per_src=[5582, 5572, 5600, 5454, 5572, 5548, 5502, 5524] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44584 payload_rows_remote=39036 dup_rows=20952 dup_rows_remote=18272 dedup_ratio=0.319702 router_tokens_per_expert=[503, 556, 508, 534, 478, 477, 523, 546, 522, 518, 518, 526, 515, 499, 524, 507, 501, 464, 499, 509, 531, 524, 485, 498, 509, 500, 535, 503, 521, 532, 494, 493, 510, 483, 517, 489, 538, 542, 529, 530, 504, 506, 529, 505, 546, 490, 489, 545, 496, 504, 514, 489, 549, 488, 507, 509, 503, 546, 516, 520, 531, 517, 481, 494] payload_rows_per_dst=[5596, 5640, 5478, 5644, 5576, 5548, 5554, 5548] dup_rows_per_dst=[2654, 2618, 2544, 2530, 2700, 2680, 2558, 2668] combine_rows_per_src=[5596, 5640, 5478, 5644, 5576, 5548, 5554, 5548] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44434 payload_rows_remote=38966 dup_rows=21102 dup_rows_remote=18514 dedup_ratio=0.321991 router_tokens_per_expert=[538, 524, 516, 520, 532, 503, 509, 542, 523, 475, 481, 521, 481, 506, 519, 523, 476, 479, 510, 531, 517, 508, 517, 506, 485, 533, 500, 551, 508, 499, 519, 539, 508, 506, 527, 524, 481, 506, 488, 488, 549, 525, 483, 529, 532, 513, 558, 519, 482, 463, 566, 518, 489, 511, 502, 529, 517, 477, 465, 505, 519, 527, 532, 539] payload_rows_per_dst=[5612, 5500, 5498, 5626, 5468, 5656, 5520, 5554] dup_rows_per_dst=[2756, 2558, 2590, 2642, 2588, 2760, 2600, 2608] combine_rows_per_src=[5612, 5500, 5498, 5626, 5468, 5656, 5520, 5554] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44494 payload_rows_remote=38850 dup_rows=21042 dup_rows_remote=18414 dedup_ratio=0.321075 router_tokens_per_expert=[502, 495, 505, 534, 563, 516, 509, 530, 541, 515, 513, 499, 482, 532, 484, 543, 536, 519, 538, 495, 529, 520, 531, 541, 486, 526, 501, 499, 523, 481, 513, 497, 514, 542, 504, 502, 523, 490, 472, 504, 484, 532, 530, 518, 503, 504, 486, 501, 546, 516, 520, 528, 535, 536, 486, 469, 501, 503, 497, 492, 523, 504, 503, 502] payload_rows_per_dst=[5642, 5510, 5652, 5540, 5568, 5492, 5644, 5446] dup_rows_per_dst=[2666, 2708, 2766, 2512, 2534, 2624, 2628, 2604] combine_rows_per_src=[5642, 5510, 5652, 5540, 5568, 5492, 5644, 5446] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280082432 dispatch_meta_bytes=229768 combine_payload_bytes=280082432 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280312200 ingress_bytes=500283392 link_bytes=500283392 token_bytes=560394632 weight_movement_bytes=220200960 total_nvlink_bytes=780595592 weight_to_token_ratio=0.392939 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.700751012e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278577152 dispatch_meta_bytes=228128 combine_payload_bytes=278577152 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278805280 ingress_bytes=388677632 link_bytes=388677632 token_bytes=557382432 weight_movement_bytes=110100480 total_nvlink_bytes=667482912 weight_to_token_ratio=0.197531 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.759743285e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278835200 dispatch_meta_bytes=229776 combine_payload_bytes=278835200 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279064976 ingress_bytes=388935680 link_bytes=388935680 token_bytes=557900176 weight_movement_bytes=110100480 total_nvlink_bytes=668000656 weight_to_token_ratio=0.197348 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.764231164e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279810048 dispatch_meta_bytes=229232 combine_payload_bytes=279810048 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280039280 ingress_bytes=389910528 link_bytes=389910528 token_bytes=559849328 weight_movement_bytes=110100480 total_nvlink_bytes=669949808 weight_to_token_ratio=0.196661 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.781185375e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44408 payload_rows_remote=38812 dup_rows=21128 dup_rows_remote=18382 dedup_ratio=0.322388 router_tokens_per_expert=[494, 515, 531, 503, 513, 515, 529, 542, 518, 501, 502, 505, 540, 531, 501, 515, 503, 495, 540, 457, 441, 507, 514, 534, 533, 486, 508, 481, 538, 489, 511, 509, 512, 518, 485, 519, 536, 500, 554, 545, 536, 516, 541, 478, 503, 478, 505, 538, 526, 560, 475, 487, 539, 513, 479, 505, 511, 526, 484, 534, 529, 512, 495, 528] payload_rows_per_dst=[5596, 5604, 5480, 5514, 5534, 5526, 5584, 5570] dup_rows_per_dst=[2746, 2622, 2502, 2596, 2746, 2664, 2584, 2668] combine_rows_per_src=[5596, 5604, 5480, 5514, 5534, 5526, 5584, 5570] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279308288 dispatch_meta_bytes=229920 combine_payload_bytes=279308288 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279538208 ingress_bytes=499509248 link_bytes=499509248 token_bytes=558846496 weight_movement_bytes=220200960 total_nvlink_bytes=779047456 weight_to_token_ratio=0.394028 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.687287374e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44446 payload_rows_remote=38874 dup_rows=21090 dup_rows_remote=18386 dedup_ratio=0.321808 router_tokens_per_expert=[491, 513, 503, 515, 518, 482, 500, 527, 494, 497, 522, 537, 515, 513, 529, 531, 589, 517, 517, 513, 546, 524, 528, 538, 503, 502, 520, 505, 526, 504, 509, 493, 541, 538, 519, 514, 520, 527, 505, 479, 512, 483, 491, 519, 502, 510, 472, 484, 527, 510, 517, 496, 470, 524, 508, 489, 503, 529, 527, 515, 500, 481, 546, 489] payload_rows_per_dst=[5522, 5572, 5710, 5518, 5602, 5492, 5484, 5546] dup_rows_per_dst=[2576, 2704, 2834, 2606, 2684, 2454, 2598, 2634] combine_rows_per_src=[5522, 5572, 5710, 5518, 5602, 5492, 5484, 5546] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278476800 dispatch_meta_bytes=229056 combine_payload_bytes=278476800 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278705856 ingress_bytes=388577280 link_bytes=388577280 token_bytes=557182656 weight_movement_bytes=110100480 total_nvlink_bytes=667283136 weight_to_token_ratio=0.197602 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.757997999e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278204416 dispatch_meta_bytes=228776 combine_payload_bytes=278204416 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278433192 ingress_bytes=608505856 link_bytes=608505856 token_bytes=556637608 weight_movement_bytes=330301440 total_nvlink_bytes=886939048 weight_to_token_ratio=0.593387 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058291766e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278648832 dispatch_meta_bytes=229040 combine_payload_bytes=278648832 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278877872 ingress_bytes=608950272 link_bytes=608950272 token_bytes=557526704 weight_movement_bytes=330301440 total_nvlink_bytes=887828144 weight_to_token_ratio=0.592441 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059064678e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44328 payload_rows_remote=38676 dup_rows=21208 dup_rows_remote=18514 dedup_ratio=0.323608 router_tokens_per_expert=[515, 507, 506, 532, 516, 524, 515, 491, 562, 485, 484, 524, 485, 529, 531, 489, 492, 480, 480, 523, 526, 492, 512, 498, 509, 527, 481, 523, 503, 523, 504, 532, 503, 497, 504, 538, 491, 541, 516, 519, 519, 507, 530, 485, 500, 524, 547, 529, 510, 533, 474, 511, 498, 536, 527, 504, 492, 551, 500, 494, 584, 515, 465, 524] payload_rows_per_dst=[5618, 5278, 5406, 5606, 5488, 5692, 5588, 5652] dup_rows_per_dst=[2594, 2362, 2430, 2768, 2638, 3046, 2676, 2694] combine_rows_per_src=[5618, 5278, 5406, 5606, 5488, 5692, 5588, 5652] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44428 payload_rows_remote=39032 dup_rows=21108 dup_rows_remote=18642 dedup_ratio=0.322083 router_tokens_per_expert=[555, 495, 491, 541, 500, 510, 552, 517, 524, 507, 499, 500, 489, 494, 500, 516, 524, 516, 496, 546, 520, 501, 559, 519, 510, 541, 516, 498, 511, 487, 486, 474, 527, 536, 533, 514, 536, 522, 518, 521, 480, 511, 497, 491, 530, 486, 450, 486, 496, 487, 563, 493, 515, 519, 503, 505, 537, 489, 542, 508, 544, 528, 484, 523] payload_rows_per_dst=[5620, 5478, 5546, 5530, 5642, 5396, 5516, 5700] dup_rows_per_dst=[2702, 2580, 2816, 2516, 2772, 2466, 2646, 2610] combine_rows_per_src=[5620, 5478, 5546, 5530, 5642, 5396, 5516, 5700] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44524 payload_rows_remote=38906 dup_rows=21012 dup_rows_remote=18292 dedup_ratio=0.320618 router_tokens_per_expert=[494, 538, 480, 547, 526, 526, 546, 513, 516, 520, 469, 517, 498, 488, 490, 517, 493, 523, 516, 506, 515, 557, 547, 521, 517, 520, 535, 523, 502, 516, 530, 526, 534, 527, 495, 473, 542, 542, 518, 504, 480, 496, 470, 507, 502, 495, 516, 513, 488, 515, 529, 541, 524, 456, 532, 525, 491, 447, 541, 496, 537, 530, 495, 475] payload_rows_per_dst=[5586, 5580, 5630, 5618, 5576, 5442, 5620, 5472] dup_rows_per_dst=[2754, 2450, 2726, 2720, 2694, 2516, 2600, 2552] combine_rows_per_src=[5586, 5580, 5630, 5618, 5576, 5442, 5620, 5472] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44326 payload_rows_remote=38704 dup_rows=21210 dup_rows_remote=18496 dedup_ratio=0.323639 router_tokens_per_expert=[483, 503, 486, 548, 527, 515, 509, 467, 560, 530, 503, 575, 547, 548, 517, 516, 542, 537, 517, 473, 501, 555, 529, 514, 535, 509, 487, 542, 528, 478, 475, 476, 505, 498, 490, 487, 451, 490, 505, 485, 528, 494, 499, 556, 541, 507, 489, 538, 536, 559, 494, 500, 484, 536, 512, 507, 507, 496, 520, 496, 515, 503, 520, 488] payload_rows_per_dst=[5542, 5606, 5622, 5488, 5376, 5610, 5540, 5542] dup_rows_per_dst=[2534, 2986, 2714, 2572, 2446, 2694, 2716, 2548] combine_rows_per_src=[5542, 5606, 5622, 5488, 5376, 5610, 5540, 5542] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44406 payload_rows_remote=38798 dup_rows=21130 dup_rows_remote=18524 dedup_ratio=0.322418 router_tokens_per_expert=[485, 516, 489, 518, 494, 548, 497, 544, 541, 494, 502, 516, 543, 502, 538, 491, 520, 524, 498, 561, 487, 527, 464, 491, 534, 515, 491, 481, 522, 537, 483, 508, 503, 517, 522, 518, 513, 496, 517, 521, 498, 532, 521, 551, 525, 506, 547, 480, 481, 519, 526, 546, 516, 484, 476, 509, 539, 520, 493, 458, 515, 535, 544, 479] payload_rows_per_dst=[5562, 5620, 5532, 5496, 5608, 5576, 5500, 5512] dup_rows_per_dst=[2620, 2634, 2612, 2646, 2606, 2744, 2614, 2654] combine_rows_per_src=[5562, 5620, 5532, 5496, 5608, 5576, 5500, 5512] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44404 payload_rows_remote=38914 dup_rows=21132 dup_rows_remote=18570 dedup_ratio=0.322449 router_tokens_per_expert=[514, 507, 495, 538, 536, 515, 492, 527, 500, 510, 546, 510, 524, 541, 536, 515, 508, 503, 510, 509, 505, 498, 498, 486, 510, 494, 533, 533, 520, 524, 541, 514, 513, 519, 536, 519, 491, 522, 547, 490, 504, 521, 494, 510, 509, 509, 464, 519, 484, 496, 527, 510, 521, 481, 511, 496, 508, 516, 507, 497, 498, 494, 548, 515] payload_rows_per_dst=[5516, 5536, 5518, 5586, 5630, 5588, 5490, 5540] dup_rows_per_dst=[2732, 2828, 2516, 2752, 2644, 2472, 2562, 2626] combine_rows_per_src=[5516, 5536, 5518, 5586, 5630, 5588, 5490, 5540] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277229568 dispatch_meta_bytes=228760 combine_payload_bytes=277229568 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277458328 ingress_bytes=387330048 link_bytes=387330048 token_bytes=554687896 weight_movement_bytes=110100480 total_nvlink_bytes=664788376 weight_to_token_ratio=0.198491 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.736306583e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279781376 dispatch_meta_bytes=230696 combine_payload_bytes=279781376 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280012072 ingress_bytes=389881856 link_bytes=389881856 token_bytes=559793448 weight_movement_bytes=110100480 total_nvlink_bytes=669893928 weight_to_token_ratio=0.196681 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.780686721e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278878208 dispatch_meta_bytes=228792 combine_payload_bytes=278878208 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279107000 ingress_bytes=388978688 link_bytes=388978688 token_bytes=557985208 weight_movement_bytes=110100480 total_nvlink_bytes=668085688 weight_to_token_ratio=0.197318 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.764979144e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277430272 dispatch_meta_bytes=228800 combine_payload_bytes=277430272 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277659072 ingress_bytes=497631232 link_bytes=497631232 token_bytes=555089344 weight_movement_bytes=220200960 total_nvlink_bytes=775290304 weight_to_token_ratio=0.396695 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.654625587e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278104064 dispatch_meta_bytes=229288 combine_payload_bytes=278104064 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=278333352 ingress_bytes=718505984 link_bytes=718505984 token_bytes=556437416 weight_movement_bytes=440401920 total_nvlink_bytes=996839336 weight_to_token_ratio=0.791467 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.249600080e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278935552 dispatch_meta_bytes=229936 combine_payload_bytes=278935552 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279165488 ingress_bytes=389036032 link_bytes=389036032 token_bytes=558101040 weight_movement_bytes=110100480 total_nvlink_bytes=668201520 weight_to_token_ratio=0.197277 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.765976451e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44442 payload_rows_remote=38954 dup_rows=21094 dup_rows_remote=18534 dedup_ratio=0.321869 router_tokens_per_expert=[506, 491, 492, 507, 502, 517, 496, 494, 525, 528, 537, 539, 511, 461, 509, 517, 477, 475, 502, 528, 530, 512, 503, 506, 524, 528, 502, 515, 549, 484, 498, 509, 510, 454, 527, 530, 547, 511, 514, 499, 529, 518, 503, 537, 488, 507, 502, 499, 526, 505, 503, 523, 556, 500, 538, 542, 516, 495, 518, 499, 518, 526, 523, 531] payload_rows_per_dst=[5488, 5568, 5496, 5560, 5548, 5566, 5616, 5600] dup_rows_per_dst=[2560, 2686, 2588, 2658, 2580, 2600, 2770, 2652] combine_rows_per_src=[5488, 5568, 5496, 5560, 5548, 5566, 5616, 5600] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44550 payload_rows_remote=38950 dup_rows=20986 dup_rows_remote=18242 dedup_ratio=0.320221 router_tokens_per_expert=[486, 508, 482, 531, 476, 519, 521, 531, 520, 535, 508, 540, 529, 494, 506, 540, 500, 499, 536, 498, 512, 518, 581, 548, 488, 499, 514, 514, 503, 495, 491, 506, 500, 495, 523, 519, 537, 517, 538, 515, 540, 489, 533, 487, 486, 491, 522, 516, 500, 524, 493, 503, 501, 504, 508, 508, 491, 519, 523, 524, 517, 491, 530, 496] payload_rows_per_dst=[5552, 5600, 5688, 5528, 5652, 5476, 5462, 5592] dup_rows_per_dst=[2556, 2744, 2696, 2492, 2636, 2652, 2620, 2590] combine_rows_per_src=[5552, 5600, 5688, 5528, 5652, 5476, 5462, 5592] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279222272 dispatch_meta_bytes=229952 combine_payload_bytes=279222272 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279452224 ingress_bytes=389322752 link_bytes=389322752 token_bytes=558674496 weight_movement_bytes=110100480 total_nvlink_bytes=668774976 weight_to_token_ratio=0.197074 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.770962983e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44548 payload_rows_remote=39056 dup_rows=20988 dup_rows_remote=18264 dedup_ratio=0.320251 router_tokens_per_expert=[496, 555, 537, 520, 533, 510, 459, 502, 504, 510, 520, 521, 489, 534, 540, 514, 510, 491, 540, 514, 536, 489, 504, 517, 526, 500, 496, 496, 481, 504, 514, 490, 493, 498, 528, 529, 498, 484, 558, 498, 552, 519, 511, 462, 511, 494, 515, 544, 527, 539, 556, 542, 515, 523, 490, 485, 481, 495, 514, 521, 530, 507, 511, 486] payload_rows_per_dst=[5608, 5688, 5542, 5446, 5558, 5492, 5644, 5570] dup_rows_per_dst=[2616, 2576, 2660, 2568, 2614, 2724, 2710, 2520] combine_rows_per_src=[5608, 5688, 5542, 5446, 5558, 5492, 5644, 5570] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44534 payload_rows_remote=38976 dup_rows=21002 dup_rows_remote=18270 dedup_ratio=0.320465 router_tokens_per_expert=[478, 533, 491, 481, 520, 536, 529, 499, 483, 548, 510, 500, 548, 514, 489, 473, 515, 539, 517, 508, 508, 500, 538, 513, 476, 511, 519, 526, 500, 501, 492, 506, 517, 509, 509, 530, 515, 505, 484, 484, 475, 497, 474, 516, 538, 544, 522, 543, 547, 533, 493, 522, 573, 495, 500, 497, 504, 507, 518, 542, 512, 498, 567, 497] payload_rows_per_dst=[5554, 5516, 5610, 5590, 5512, 5600, 5594, 5558] dup_rows_per_dst=[2580, 2614, 2666, 2472, 2594, 2618, 2726, 2732] combine_rows_per_src=[5554, 5516, 5610, 5590, 5512, 5600, 5594, 5558] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44552 payload_rows_remote=38894 dup_rows=20984 dup_rows_remote=18256 dedup_ratio=0.320190 router_tokens_per_expert=[508, 537, 491, 502, 493, 476, 502, 517, 528, 541, 490, 488, 506, 534, 524, 516, 492, 490, 508, 521, 486, 498, 515, 489, 528, 546, 479, 529, 559, 525, 514, 513, 526, 505, 511, 525, 511, 520, 542, 508, 484, 499, 502, 519, 510, 491, 494, 521, 529, 497, 544, 524, 492, 508, 508, 523, 529, 489, 509, 507, 527, 516, 518, 535] payload_rows_per_dst=[5538, 5580, 5502, 5658, 5600, 5506, 5604, 5564] dup_rows_per_dst=[2514, 2674, 2496, 2728, 2696, 2534, 2646, 2696] combine_rows_per_src=[5538, 5580, 5502, 5658, 5600, 5506, 5604, 5564] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44528 payload_rows_remote=39032 dup_rows=21008 dup_rows_remote=18380 dedup_ratio=0.320557 router_tokens_per_expert=[496, 505, 492, 495, 533, 509, 524, 542, 523, 475, 533, 492, 481, 517, 480, 498, 533, 526, 498, 502, 505, 490, 502, 506, 546, 505, 535, 496, 532, 480, 535, 517, 516, 550, 504, 492, 522, 495, 515, 500, 544, 539, 506, 543, 505, 506, 535, 492, 497, 483, 508, 498, 503, 505, 545, 520, 512, 492, 528, 521, 514, 544, 527, 504] payload_rows_per_dst=[5624, 5464, 5496, 5576, 5558, 5648, 5574, 5588] dup_rows_per_dst=[2568, 2534, 2628, 2716, 2630, 2692, 2544, 2696] combine_rows_per_src=[5624, 5464, 5496, 5576, 5558, 5648, 5574, 5588] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279193600 dispatch_meta_bytes=228768 combine_payload_bytes=279193600 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279422368 ingress_bytes=609495040 link_bytes=609495040 token_bytes=558615968 weight_movement_bytes=330301440 total_nvlink_bytes=888917408 weight_to_token_ratio=0.591285 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060012119e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44394 payload_rows_remote=38920 dup_rows=21142 dup_rows_remote=18516 dedup_ratio=0.322601 router_tokens_per_expert=[480, 510, 550, 480, 526, 502, 473, 533, 559, 506, 522, 481, 523, 477, 565, 513, 527, 527, 502, 531, 509, 506, 514, 498, 490, 518, 510, 505, 511, 530, 501, 523, 523, 528, 491, 537, 545, 513, 532, 472, 527, 489, 514, 496, 507, 512, 505, 518, 500, 526, 516, 496, 487, 474, 522, 529, 531, 520, 488, 490, 546, 534, 504, 494] payload_rows_per_dst=[5430, 5662, 5574, 5600, 5584, 5492, 5474, 5578] dup_rows_per_dst=[2678, 2630, 2654, 2576, 2698, 2644, 2626, 2636] combine_rows_per_src=[5430, 5662, 5574, 5600, 5584, 5492, 5474, 5578] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44238 payload_rows_remote=38686 dup_rows=21298 dup_rows_remote=18722 dedup_ratio=0.324982 router_tokens_per_expert=[537, 495, 502, 471, 498, 505, 511, 514, 562, 501, 527, 515, 552, 538, 525, 479, 496, 535, 501, 525, 500, 513, 520, 531, 492, 501, 533, 513, 518, 551, 494, 532, 511, 502, 521, 509, 492, 528, 494, 507, 503, 521, 546, 506, 531, 503, 506, 534, 527, 501, 505, 486, 525, 504, 500, 478, 510, 494, 460, 510, 514, 537, 510, 506] payload_rows_per_dst=[5450, 5650, 5576, 5568, 5552, 5488, 5432, 5522] dup_rows_per_dst=[2616, 2748, 2666, 2700, 2576, 2812, 2620, 2560] combine_rows_per_src=[5450, 5650, 5576, 5568, 5552, 5488, 5432, 5522] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279164928 dispatch_meta_bytes=229720 combine_payload_bytes=279164928 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279394648 ingress_bytes=499365888 link_bytes=499365888 token_bytes=558559576 weight_movement_bytes=220200960 total_nvlink_bytes=778760536 weight_to_token_ratio=0.394230 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.684794108e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44508 payload_rows_remote=38948 dup_rows=21028 dup_rows_remote=18410 dedup_ratio=0.320862 router_tokens_per_expert=[529, 516, 520, 495, 512, 527, 525, 478, 496, 482, 508, 484, 470, 536, 510, 495, 491, 465, 510, 500, 535, 512, 554, 474, 549, 504, 551, 532, 536, 572, 521, 514, 514, 520, 510, 521, 504, 509, 538, 512, 506, 496, 522, 507, 536, 486, 484, 536, 570, 516, 510, 508, 482, 511, 530, 462, 479, 520, 504, 544, 524, 489, 513, 502] payload_rows_per_dst=[5532, 5506, 5480, 5734, 5570, 5552, 5560, 5574] dup_rows_per_dst=[2672, 2456, 2602, 2824, 2686, 2594, 2618, 2576] combine_rows_per_src=[5532, 5506, 5480, 5734, 5570, 5552, 5560, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=277774336 dispatch_meta_bytes=227960 combine_payload_bytes=277774336 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278002296 ingress_bytes=387874816 link_bytes=387874816 token_bytes=555776632 weight_movement_bytes=110100480 total_nvlink_bytes=665877112 weight_to_token_ratio=0.198102 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.745780995e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44442 payload_rows_remote=38820 dup_rows=21094 dup_rows_remote=18436 dedup_ratio=0.321869 router_tokens_per_expert=[507, 472, 518, 530, 526, 493, 521, 505, 517, 561, 556, 509, 488, 517, 492, 500, 515, 513, 514, 521, 528, 480, 503, 535, 469, 536, 526, 487, 512, 532, 493, 495, 511, 518, 530, 538, 457, 507, 522, 512, 534, 499, 500, 497, 513, 504, 518, 526, 509, 518, 501, 499, 514, 486, 533, 504, 529, 523, 519, 539, 515, 513, 497, 512] payload_rows_per_dst=[5520, 5622, 5494, 5548, 5524, 5552, 5556, 5626] dup_rows_per_dst=[2624, 2658, 2724, 2552, 2666, 2630, 2572, 2668] combine_rows_per_src=[5520, 5622, 5494, 5548, 5524, 5552, 5556, 5626] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44652 payload_rows_remote=39074 dup_rows=20884 dup_rows_remote=18258 dedup_ratio=0.318665 router_tokens_per_expert=[511, 499, 515, 528, 538, 507, 518, 504, 515, 539, 526, 456, 539, 522, 536, 498, 500, 490, 502, 477, 535, 490, 538, 500, 505, 482, 511, 530, 494, 551, 539, 490, 537, 544, 509, 505, 494, 467, 508, 522, 519, 506, 539, 507, 570, 533, 501, 508, 522, 525, 472, 497, 491, 476, 542, 515, 504, 488, 518, 497, 525, 533, 516, 493] payload_rows_per_dst=[5532, 5616, 5554, 5578, 5630, 5646, 5512, 5584] dup_rows_per_dst=[2708, 2646, 2510, 2626, 2542, 2720, 2568, 2564] combine_rows_per_src=[5532, 5616, 5554, 5578, 5630, 5646, 5512, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44276 payload_rows_remote=38804 dup_rows=21260 dup_rows_remote=18564 dedup_ratio=0.324402 router_tokens_per_expert=[532, 535, 532, 494, 512, 541, 464, 516, 502, 509, 481, 496, 525, 526, 510, 507, 520, 493, 516, 505, 518, 489, 521, 522, 515, 545, 530, 525, 563, 511, 492, 549, 508, 525, 512, 519, 516, 485, 511, 526, 482, 518, 510, 556, 520, 509, 489, 469, 509, 547, 531, 465, 533, 510, 499, 513, 486, 502, 502, 489, 529, 505, 493, 504] payload_rows_per_dst=[5560, 5434, 5472, 5662, 5580, 5532, 5552, 5484] dup_rows_per_dst=[2692, 2678, 2696, 2798, 2624, 2574, 2662, 2536] combine_rows_per_src=[5560, 5434, 5472, 5662, 5580, 5532, 5552, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44628 payload_rows_remote=39012 dup_rows=20908 dup_rows_remote=18280 dedup_ratio=0.319031 router_tokens_per_expert=[474, 532, 493, 479, 498, 487, 507, 504, 515, 563, 510, 536, 510, 525, 455, 477, 542, 541, 470, 508, 502, 511, 544, 532, 534, 510, 540, 496, 493, 524, 514, 496, 499, 530, 490, 504, 532, 525, 538, 534, 530, 523, 532, 528, 511, 493, 485, 520, 510, 516, 516, 527, 531, 529, 476, 498, 492, 494, 489, 516, 490, 530, 536, 522] payload_rows_per_dst=[5460, 5594, 5602, 5632, 5610, 5616, 5586, 5528] dup_rows_per_dst=[2488, 2588, 2698, 2582, 2694, 2628, 2620, 2610] combine_rows_per_src=[5460, 5594, 5602, 5632, 5610, 5616, 5586, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279179264 dispatch_meta_bytes=229432 combine_payload_bytes=279179264 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=279408696 ingress_bytes=389279744 link_bytes=389279744 token_bytes=558587960 weight_movement_bytes=110100480 total_nvlink_bytes=668688440 weight_to_token_ratio=0.197105 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.770215003e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44494 payload_rows_remote=38916 dup_rows=21042 dup_rows_remote=18320 dedup_ratio=0.321075 router_tokens_per_expert=[498, 566, 493, 515, 503, 527, 527, 520, 550, 490, 531, 532, 501, 506, 510, 499, 490, 547, 498, 474, 494, 524, 514, 506, 497, 491, 516, 573, 514, 519, 499, 482, 534, 550, 524, 503, 464, 515, 475, 515, 533, 518, 490, 501, 500, 524, 536, 482, 524, 511, 519, 508, 464, 564, 515, 510, 500, 511, 501, 513, 510, 531, 504, 513] payload_rows_per_dst=[5578, 5614, 5706, 5426, 5594, 5502, 5620, 5454] dup_rows_per_dst=[2722, 2796, 2844, 2300, 2566, 2494, 2608, 2712] combine_rows_per_src=[5578, 5614, 5706, 5426, 5594, 5502, 5620, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278261760 dispatch_meta_bytes=229024 combine_payload_bytes=278261760 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278490784 ingress_bytes=498462720 link_bytes=498462720 token_bytes=556752544 weight_movement_bytes=220200960 total_nvlink_bytes=776953504 weight_to_token_ratio=0.395510 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.669086531e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44614 payload_rows_remote=39016 dup_rows=20922 dup_rows_remote=18332 dedup_ratio=0.319244 router_tokens_per_expert=[511, 498, 492, 519, 495, 546, 545, 552, 498, 514, 551, 526, 511, 514, 542, 508, 480, 492, 507, 503, 547, 503, 544, 514, 477, 491, 502, 526, 470, 506, 483, 512, 498, 509, 526, 524, 515, 510, 500, 512, 510, 559, 452, 564, 509, 526, 497, 507, 498, 545, 513, 518, 490, 521, 519, 527, 522, 538, 494, 499, 505, 497, 492, 493] payload_rows_per_dst=[5714, 5662, 5538, 5472, 5598, 5492, 5592, 5546] dup_rows_per_dst=[2602, 2666, 2642, 2462, 2590, 2756, 2670, 2534] combine_rows_per_src=[5714, 5662, 5538, 5472, 5598, 5492, 5592, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=280082432 dispatch_meta_bytes=229328 combine_payload_bytes=280082432 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280311760 ingress_bytes=610383872 link_bytes=610383872 token_bytes=560394192 weight_movement_bytes=330301440 total_nvlink_bytes=890695632 weight_to_token_ratio=0.589409 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061557944e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279638016 dispatch_meta_bytes=229168 combine_payload_bytes=279638016 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279867184 ingress_bytes=499838976 link_bytes=499838976 token_bytes=559505200 weight_movement_bytes=220200960 total_nvlink_bytes=779706160 weight_to_token_ratio=0.393564 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.693021886e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278147072 dispatch_meta_bytes=229472 combine_payload_bytes=278147072 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278376544 ingress_bytes=388247552 link_bytes=388247552 token_bytes=556523616 weight_movement_bytes=110100480 total_nvlink_bytes=666624096 weight_to_token_ratio=0.197836 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.752263487e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278949888 dispatch_meta_bytes=228944 combine_payload_bytes=278949888 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279178832 ingress_bytes=609251328 link_bytes=609251328 token_bytes=558128720 weight_movement_bytes=330301440 total_nvlink_bytes=888430160 weight_to_token_ratio=0.591802 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059588264e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44320 payload_rows_remote=38654 dup_rows=21216 dup_rows_remote=18340 dedup_ratio=0.323730 router_tokens_per_expert=[489, 491, 493, 490, 495, 495, 533, 542, 500, 538, 516, 503, 505, 507, 529, 523, 514, 504, 486, 497, 528, 528, 501, 492, 530, 514, 480, 524, 548, 494, 535, 525, 501, 483, 559, 526, 531, 502, 535, 528, 500, 513, 507, 536, 553, 531, 545, 499, 483, 500, 480, 490, 521, 525, 529, 501, 492, 518, 495, 509, 535, 476, 494, 522] payload_rows_per_dst=[5434, 5512, 5470, 5660, 5396, 5638, 5544, 5666] dup_rows_per_dst=[2622, 2730, 2550, 2720, 2474, 2730, 2514, 2876] combine_rows_per_src=[5434, 5512, 5470, 5660, 5396, 5638, 5544, 5666] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5588889760 prefetch_bytes=1101004800 reduce_bytes=990904320 weight_movement_bytes=2091909120 egress_bytes=2795591840 ingress_bytes=4885207040 link_bytes=4885207040 total_nvlink_bytes=7680798880 weight_to_token_ratio=0.374298 prefetch_share_of_ingress=0.225375 dedup_ratio=0.320483 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=9 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.496178521e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279666688 dispatch_meta_bytes=229392 combine_payload_bytes=279666688 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279896080 ingress_bytes=499867648 link_bytes=499867648 token_bytes=559562768 weight_movement_bytes=220200960 total_nvlink_bytes=779763728 weight_to_token_ratio=0.393523 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.693520540e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44436 payload_rows_remote=38884 dup_rows=21100 dup_rows_remote=18396 dedup_ratio=0.321960 router_tokens_per_expert=[469, 500, 514, 560, 483, 511, 497, 534, 523, 517, 491, 532, 518, 503, 508, 513, 525, 506, 518, 497, 480, 518, 508, 550, 499, 541, 523, 561, 510, 546, 475, 495, 470, 531, 500, 507, 488, 501, 480, 521, 519, 518, 459, 502, 501, 522, 510, 550, 513, 523, 528, 492, 489, 507, 495, 581, 509, 519, 515, 513, 487, 519, 566, 508] payload_rows_per_dst=[5514, 5624, 5534, 5578, 5428, 5582, 5552, 5624] dup_rows_per_dst=[2622, 2586, 2670, 2722, 2568, 2580, 2704, 2648] combine_rows_per_src=[5514, 5624, 5534, 5578, 5428, 5582, 5552, 5624] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=277071872 dispatch_meta_bytes=227976 combine_payload_bytes=277071872 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277299848 ingress_bytes=387172352 link_bytes=387172352 token_bytes=554371720 weight_movement_bytes=110100480 total_nvlink_bytes=664472200 weight_to_token_ratio=0.198604 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.733563990e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5574036712 prefetch_bytes=1101004800 reduce_bytes=770703360 weight_movement_bytes=1871708160 egress_bytes=2788164840 ingress_bytes=4657580032 link_bytes=4657580032 total_nvlink_bytes=7445744872 weight_to_token_ratio=0.335790 prefetch_share_of_ingress=0.236390 dedup_ratio=0.321951 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=7 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.039706 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.100297716e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44522 payload_rows_remote=38996 dup_rows=21014 dup_rows_remote=18458 dedup_ratio=0.320648 router_tokens_per_expert=[550, 486, 489, 528, 506, 510, 521, 519, 536, 499, 501, 520, 502, 499, 509, 482, 499, 516, 529, 510, 495, 520, 505, 540, 504, 524, 480, 500, 518, 516, 479, 520, 476, 568, 537, 527, 545, 529, 483, 522, 501, 512, 535, 514, 522, 526, 516, 528, 480, 528, 506, 539, 534, 493, 497, 514, 510, 483, 504, 499, 497, 519, 527, 485] payload_rows_per_dst=[5536, 5568, 5602, 5526, 5606, 5624, 5536, 5524] dup_rows_per_dst=[2682, 2528, 2626, 2556, 2768, 2684, 2646, 2524] combine_rows_per_src=[5536, 5568, 5602, 5526, 5606, 5624, 5536, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44368 payload_rows_remote=38920 dup_rows=21168 dup_rows_remote=18536 dedup_ratio=0.322998 router_tokens_per_expert=[537, 494, 570, 495, 534, 512, 546, 532, 526, 530, 485, 495, 514, 516, 481, 476, 494, 541, 467, 506, 524, 534, 513, 507, 513, 491, 533, 465, 538, 483, 534, 530, 536, 519, 524, 525, 509, 549, 532, 510, 513, 492, 500, 488, 515, 517, 535, 480, 488, 532, 507, 499, 515, 529, 512, 506, 505, 480, 482, 514, 507, 485, 533, 514] payload_rows_per_dst=[5610, 5516, 5564, 5494, 5694, 5448, 5550, 5492] dup_rows_per_dst=[2830, 2530, 2608, 2680, 2714, 2632, 2626, 2548] combine_rows_per_src=[5610, 5516, 5564, 5494, 5694, 5448, 5550, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44604 payload_rows_remote=39028 dup_rows=20932 dup_rows_remote=18214 dedup_ratio=0.319397 router_tokens_per_expert=[491, 494, 505, 498, 519, 502, 512, 488, 498, 458, 511, 558, 486, 522, 541, 516, 513, 505, 480, 554, 521, 516, 477, 581, 517, 517, 485, 548, 523, 516, 532, 521, 540, 479, 524, 533, 498, 502, 504, 510, 538, 494, 512, 524, 526, 491, 448, 517, 537, 530, 511, 517, 478, 526, 518, 507, 493, 515, 499, 521, 510, 497, 536, 528] payload_rows_per_dst=[5480, 5530, 5576, 5674, 5602, 5594, 5588, 5560] dup_rows_per_dst=[2538, 2650, 2718, 2644, 2578, 2506, 2660, 2638] combine_rows_per_src=[5480, 5530, 5576, 5674, 5602, 5594, 5588, 5560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278720512 dispatch_meta_bytes=229120 combine_payload_bytes=278720512 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278949632 ingress_bytes=609021952 link_bytes=609021952 token_bytes=557670144 weight_movement_bytes=330301440 total_nvlink_bytes=887971584 weight_to_token_ratio=0.592288 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059189341e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5570104704 prefetch_bytes=990904320 reduce_bytes=550502400 weight_movement_bytes=1541406720 egress_bytes=2786196864 ingress_bytes=4325314560 link_bytes=4325314560 total_nvlink_bytes=7111511424 weight_to_token_ratio=0.276729 prefetch_share_of_ingress=0.229094 dedup_ratio=0.321811 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=5 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.522433412e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44618 payload_rows_remote=38996 dup_rows=20918 dup_rows_remote=18332 dedup_ratio=0.319183 router_tokens_per_expert=[501, 517, 522, 504, 497, 522, 520, 506, 494, 518, 516, 503, 508, 507, 548, 497, 499, 527, 501, 496, 548, 490, 535, 512, 515, 488, 521, 512, 514, 532, 538, 504, 502, 518, 527, 489, 509, 531, 527, 503, 505, 538, 516, 495, 508, 480, 515, 543, 489, 523, 511, 510, 529, 538, 497, 522, 515, 486, 536, 508, 495, 517, 486, 488] payload_rows_per_dst=[5622, 5520, 5536, 5672, 5628, 5538, 5566, 5536] dup_rows_per_dst=[2586, 2634, 2684, 2576, 2702, 2550, 2660, 2526] combine_rows_per_src=[5622, 5520, 5536, 5672, 5628, 5538, 5566, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5586480192 prefetch_bytes=770703360 reduce_bytes=1101004800 weight_movement_bytes=1871708160 egress_bytes=2794386496 ingress_bytes=4663801856 link_bytes=4663801856 total_nvlink_bytes=7458188352 weight_to_token_ratio=0.335042 prefetch_share_of_ingress=0.165252 dedup_ratio=0.320523 prefetch_slots=14 unused_prefetch_slots=146 slot_utilization=0.087500 reduce_slots=10 remote_demand=7 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=12319 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.111118491e-05 prefetch_bytes_per_gemm_flop=1.340379902e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279523328 dispatch_meta_bytes=229816 combine_payload_bytes=279523328 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279753144 ingress_bytes=389623808 link_bytes=389623808 token_bytes=559276472 weight_movement_bytes=110100480 total_nvlink_bytes=669376952 weight_to_token_ratio=0.196862 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.776198842e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278978560 dispatch_meta_bytes=229824 combine_payload_bytes=278978560 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279208384 ingress_bytes=499179520 link_bytes=499179520 token_bytes=558186944 weight_movement_bytes=220200960 total_nvlink_bytes=778387904 weight_to_token_ratio=0.394493 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.681552862e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279752704 dispatch_meta_bytes=228968 combine_payload_bytes=279752704 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279981672 ingress_bytes=499953664 link_bytes=499953664 token_bytes=559734376 weight_movement_bytes=220200960 total_nvlink_bytes=779935336 weight_to_token_ratio=0.393403 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.695016499e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279523328 dispatch_meta_bytes=229312 combine_payload_bytes=279523328 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279752640 ingress_bytes=389623808 link_bytes=389623808 token_bytes=559275968 weight_movement_bytes=110100480 total_nvlink_bytes=669376448 weight_to_token_ratio=0.196863 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.776198842e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5594023512 prefetch_bytes=880803840 reduce_bytes=1101004800 weight_movement_bytes=1981808640 egress_bytes=2798159448 ingress_bytes=4777672704 link_bytes=4777672704 total_nvlink_bytes=7575832152 weight_to_token_ratio=0.354272 prefetch_share_of_ingress=0.184358 dedup_ratio=0.320065 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=10 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.309158624e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5582523712 prefetch_bytes=880803840 reduce_bytes=990904320 weight_movement_bytes=1871708160 egress_bytes=2792408384 ingress_bytes=4661823488 link_bytes=4661823488 total_nvlink_bytes=7454231872 weight_to_token_ratio=0.335280 prefetch_share_of_ingress=0.188940 dedup_ratio=0.321127 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=9 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.107677784e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5586738768 prefetch_bytes=880803840 reduce_bytes=1541406720 weight_movement_bytes=2422210560 egress_bytes=2794516048 ingress_bytes=5214433280 link_bytes=5214433280 total_nvlink_bytes=8008949328 weight_to_token_ratio=0.433564 prefetch_share_of_ingress=0.168917 dedup_ratio=0.320554 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=14 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.068757101e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5578734600 prefetch_bytes=1101004800 reduce_bytes=660602880 weight_movement_bytes=1761607680 egress_bytes=2790511624 ingress_bytes=4549830656 link_bytes=4549830656 total_nvlink_bytes=7340342280 weight_to_token_ratio=0.315772 prefetch_share_of_ingress=0.241988 dedup_ratio=0.320773 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=6 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.912903829e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none + [2026-08-26 16:00:28] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 364.5 | throughput per GPU (TFLOP/s/GPU): 338.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.536139E+00 | loss scale: 1.0 | grad norm: 4.064 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44438 payload_rows_remote=38886 dup_rows=21098 dup_rows_remote=18418 dedup_ratio=0.321930 router_tokens_per_expert=[487, 510, 501, 516, 517, 476, 497, 526, 488, 491, 521, 537, 513, 511, 527, 528, 585, 516, 515, 513, 543, 523, 525, 538, 507, 501, 518, 505, 524, 504, 507, 493, 540, 537, 522, 512, 520, 532, 507, 480, 514, 483, 492, 521, 503, 511, 473, 486, 528, 513, 520, 497, 470, 528, 509, 493, 505, 532, 531, 516, 501, 486, 548, 495] payload_rows_per_dst=[5504, 5552, 5694, 5506, 5622, 5498, 5498, 5564] dup_rows_per_dst=[2556, 2680, 2822, 2612, 2678, 2468, 2618, 2664] combine_rows_per_src=[5504, 5552, 5694, 5506, 5622, 5498, 5498, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44416 payload_rows_remote=38950 dup_rows=21120 dup_rows_remote=18524 dedup_ratio=0.322266 router_tokens_per_expert=[536, 521, 515, 517, 530, 501, 507, 539, 523, 474, 476, 520, 479, 504, 516, 526, 476, 476, 509, 530, 517, 506, 514, 506, 483, 534, 499, 553, 509, 499, 519, 536, 507, 509, 528, 525, 479, 508, 490, 485, 548, 523, 485, 531, 532, 518, 560, 521, 484, 463, 567, 520, 492, 512, 503, 533, 519, 479, 468, 506, 521, 530, 533, 539] payload_rows_per_dst=[5598, 5488, 5484, 5628, 5466, 5662, 5530, 5560] dup_rows_per_dst=[2734, 2548, 2584, 2636, 2596, 2774, 2618, 2630] combine_rows_per_src=[5598, 5488, 5484, 5628, 5466, 5662, 5530, 5560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44580 payload_rows_remote=39022 dup_rows=20956 dup_rows_remote=18272 dedup_ratio=0.319763 router_tokens_per_expert=[502, 554, 506, 531, 476, 478, 520, 546, 515, 517, 516, 522, 513, 498, 522, 507, 498, 459, 500, 507, 529, 525, 486, 494, 508, 499, 534, 503, 520, 532, 491, 494, 511, 487, 516, 490, 538, 543, 530, 529, 505, 510, 533, 504, 550, 490, 484, 545, 497, 510, 516, 491, 554, 489, 505, 511, 506, 549, 517, 523, 535, 518, 484, 496] payload_rows_per_dst=[5584, 5624, 5468, 5638, 5574, 5558, 5568, 5566] dup_rows_per_dst=[2642, 2596, 2528, 2524, 2714, 2684, 2578, 2690] combine_rows_per_src=[5584, 5624, 5468, 5638, 5574, 5558, 5568, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44640 payload_rows_remote=39084 dup_rows=20896 dup_rows_remote=18382 dedup_ratio=0.318848 router_tokens_per_expert=[451, 484, 557, 506, 518, 513, 522, 531, 508, 521, 510, 546, 532, 473, 521, 508, 489, 517, 497, 500, 514, 505, 494, 519, 518, 534, 494, 528, 516, 507, 523, 473, 496, 525, 535, 500, 513, 534, 514, 501, 516, 481, 524, 515, 573, 528, 468, 557, 520, 492, 500, 478, 542, 501, 508, 494, 522, 493, 498, 514, 523, 528, 532, 514] payload_rows_per_dst=[5554, 5622, 5556, 5582, 5618, 5654, 5490, 5564] dup_rows_per_dst=[2610, 2616, 2514, 2604, 2618, 2670, 2580, 2684] combine_rows_per_src=[5554, 5622, 5556, 5582, 5618, 5654, 5490, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44624 payload_rows_remote=38970 dup_rows=20912 dup_rows_remote=18276 dedup_ratio=0.319092 router_tokens_per_expert=[523, 510, 535, 511, 482, 503, 493, 518, 515, 486, 530, 528, 504, 508, 475, 526, 518, 515, 520, 510, 512, 537, 495, 503, 556, 518, 522, 523, 498, 482, 545, 506, 471, 520, 497, 550, 497, 522, 486, 502, 479, 530, 517, 478, 534, 500, 512, 500, 509, 494, 554, 516, 547, 518, 514, 472, 499, 514, 543, 546, 497, 495, 534, 514] payload_rows_per_dst=[5498, 5498, 5606, 5668, 5508, 5498, 5694, 5654] dup_rows_per_dst=[2376, 2578, 2682, 2908, 2364, 2596, 2772, 2636] combine_rows_per_src=[5498, 5498, 5606, 5668, 5508, 5498, 5694, 5654] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44386 payload_rows_remote=38924 dup_rows=21150 dup_rows_remote=18528 dedup_ratio=0.322723 router_tokens_per_expert=[499, 512, 539, 525, 515, 514, 536, 511, 518, 517, 522, 515, 523, 517, 525, 533, 544, 500, 515, 500, 501, 515, 524, 521, 462, 533, 512, 507, 485, 504, 521, 518, 525, 515, 541, 476, 491, 521, 510, 537, 495, 567, 516, 452, 523, 507, 539, 521, 481, 504, 544, 486, 507, 507, 501, 510, 499, 506, 476, 511, 504, 521, 509, 483] payload_rows_per_dst=[5562, 5562, 5594, 5462, 5574, 5570, 5520, 5542] dup_rows_per_dst=[2740, 2778, 2646, 2622, 2658, 2670, 2560, 2476] combine_rows_per_src=[5562, 5562, 5594, 5462, 5574, 5570, 5520, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44472 payload_rows_remote=38822 dup_rows=21064 dup_rows_remote=18418 dedup_ratio=0.321411 router_tokens_per_expert=[493, 493, 501, 531, 559, 509, 507, 528, 538, 513, 514, 498, 477, 528, 485, 542, 534, 517, 535, 495, 528, 519, 532, 541, 487, 523, 499, 501, 524, 482, 512, 496, 514, 541, 505, 502, 526, 490, 473, 502, 486, 533, 534, 519, 504, 508, 486, 501, 547, 517, 521, 528, 537, 539, 487, 472, 504, 507, 501, 494, 528, 509, 506, 506] payload_rows_per_dst=[5610, 5496, 5638, 5534, 5568, 5512, 5650, 5464] dup_rows_per_dst=[2632, 2694, 2764, 2514, 2538, 2630, 2646, 2646] combine_rows_per_src=[5610, 5496, 5638, 5534, 5568, 5512, 5650, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44382 payload_rows_remote=38786 dup_rows=21154 dup_rows_remote=18414 dedup_ratio=0.322784 router_tokens_per_expert=[491, 513, 529, 501, 513, 512, 525, 541, 516, 499, 501, 503, 539, 530, 500, 513, 500, 491, 537, 460, 442, 506, 512, 530, 527, 488, 509, 481, 537, 490, 509, 508, 510, 517, 483, 523, 536, 501, 554, 545, 537, 518, 542, 479, 502, 478, 507, 535, 528, 566, 477, 488, 542, 516, 483, 507, 511, 528, 487, 537, 533, 516, 499, 530] payload_rows_per_dst=[5596, 5594, 5470, 5506, 5582, 5490, 5602, 5542] dup_rows_per_dst=[2740, 2610, 2486, 2592, 2858, 2602, 2612, 2654] combine_rows_per_src=[5596, 5594, 5470, 5506, 5582, 5490, 5602, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278734848 dispatch_meta_bytes=229216 combine_payload_bytes=278734848 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278964064 ingress_bytes=498935808 link_bytes=498935808 token_bytes=557698912 weight_movement_bytes=220200960 total_nvlink_bytes=777899872 weight_to_token_ratio=0.394838 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.677314310e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279193600 dispatch_meta_bytes=229896 combine_payload_bytes=279193600 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279423496 ingress_bytes=499394560 link_bytes=499394560 token_bytes=558617096 weight_movement_bytes=220200960 total_nvlink_bytes=778818056 weight_to_token_ratio=0.394189 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.685292761e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279709696 dispatch_meta_bytes=229176 combine_payload_bytes=279709696 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279938872 ingress_bytes=610011136 link_bytes=610011136 token_bytes=559648568 weight_movement_bytes=330301440 total_nvlink_bytes=889950008 weight_to_token_ratio=0.590194 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060909695e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280154112 dispatch_meta_bytes=229864 combine_payload_bytes=280154112 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280383976 ingress_bytes=390254592 link_bytes=390254592 token_bytes=560538088 weight_movement_bytes=110100480 total_nvlink_bytes=670638568 weight_to_token_ratio=0.196419 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.787169213e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279336960 dispatch_meta_bytes=228984 combine_payload_bytes=279336960 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279565944 ingress_bytes=499537920 link_bytes=499537920 token_bytes=558902904 weight_movement_bytes=220200960 total_nvlink_bytes=779103864 weight_to_token_ratio=0.393988 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.687786027e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279007232 dispatch_meta_bytes=229808 combine_payload_bytes=279007232 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279237040 ingress_bytes=389107712 link_bytes=389107712 token_bytes=558244272 weight_movement_bytes=110100480 total_nvlink_bytes=668344752 weight_to_token_ratio=0.197226 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.767223084e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278276096 dispatch_meta_bytes=228960 combine_payload_bytes=278276096 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278505056 ingress_bytes=388376576 link_bytes=388376576 token_bytes=556781152 weight_movement_bytes=110100480 total_nvlink_bytes=666881632 weight_to_token_ratio=0.197745 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.754507426e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278018048 dispatch_meta_bytes=228800 combine_payload_bytes=278018048 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278246848 ingress_bytes=498219008 link_bytes=498219008 token_bytes=556264896 weight_movement_bytes=220200960 total_nvlink_bytes=776465856 weight_to_token_ratio=0.395856 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.664847979e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44552 payload_rows_remote=38958 dup_rows=20984 dup_rows_remote=18256 dedup_ratio=0.320190 router_tokens_per_expert=[484, 505, 480, 529, 473, 518, 519, 530, 519, 529, 507, 540, 531, 494, 503, 538, 496, 497, 534, 496, 511, 518, 580, 548, 486, 500, 511, 513, 504, 495, 490, 506, 498, 497, 522, 519, 534, 517, 537, 515, 543, 489, 533, 489, 485, 494, 520, 518, 502, 526, 495, 506, 504, 504, 512, 511, 495, 519, 529, 529, 522, 491, 532, 497] payload_rows_per_dst=[5540, 5594, 5682, 5522, 5650, 5478, 5470, 5616] dup_rows_per_dst=[2536, 2728, 2678, 2488, 2628, 2664, 2650, 2612] combine_rows_per_src=[5540, 5594, 5682, 5522, 5650, 5478, 5470, 5616] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44380 payload_rows_remote=38770 dup_rows=21156 dup_rows_remote=18540 dedup_ratio=0.322815 router_tokens_per_expert=[484, 511, 489, 515, 492, 545, 494, 539, 539, 494, 501, 515, 539, 500, 535, 488, 519, 522, 499, 561, 485, 524, 464, 491, 530, 514, 490, 481, 520, 537, 484, 506, 505, 514, 524, 519, 513, 497, 520, 521, 499, 533, 521, 552, 524, 505, 550, 483, 484, 522, 528, 550, 518, 484, 479, 511, 542, 519, 496, 460, 519, 540, 547, 482] payload_rows_per_dst=[5538, 5608, 5524, 5480, 5610, 5580, 5510, 5530] dup_rows_per_dst=[2600, 2614, 2606, 2644, 2616, 2754, 2642, 2680] combine_rows_per_src=[5538, 5608, 5524, 5480, 5610, 5580, 5510, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44318 payload_rows_remote=38700 dup_rows=21218 dup_rows_remote=18510 dedup_ratio=0.323761 router_tokens_per_expert=[480, 497, 479, 545, 526, 514, 508, 466, 557, 526, 503, 573, 548, 543, 514, 513, 539, 537, 518, 473, 500, 553, 529, 514, 535, 509, 488, 545, 526, 477, 474, 480, 504, 496, 492, 487, 453, 490, 505, 483, 528, 497, 500, 555, 543, 511, 487, 539, 537, 562, 493, 505, 485, 540, 513, 510, 508, 498, 523, 498, 517, 506, 523, 491] payload_rows_per_dst=[5522, 5590, 5618, 5492, 5372, 5620, 5550, 5554] dup_rows_per_dst=[2508, 2964, 2708, 2576, 2448, 2700, 2740, 2574] combine_rows_per_src=[5522, 5590, 5618, 5492, 5372, 5620, 5550, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44440 payload_rows_remote=39026 dup_rows=21096 dup_rows_remote=18628 dedup_ratio=0.321899 router_tokens_per_expert=[550, 492, 491, 536, 497, 508, 549, 517, 522, 504, 498, 501, 490, 492, 498, 516, 522, 514, 493, 546, 518, 500, 560, 517, 512, 543, 517, 498, 511, 486, 483, 474, 527, 540, 533, 514, 537, 523, 519, 520, 479, 512, 499, 494, 529, 492, 450, 486, 495, 487, 564, 496, 515, 520, 504, 507, 542, 492, 544, 509, 545, 529, 486, 524] payload_rows_per_dst=[5600, 5472, 5546, 5528, 5646, 5414, 5518, 5716] dup_rows_per_dst=[2680, 2570, 2794, 2520, 2780, 2468, 2658, 2626] combine_rows_per_src=[5600, 5472, 5546, 5528, 5646, 5414, 5518, 5716] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44346 payload_rows_remote=38686 dup_rows=21190 dup_rows_remote=18496 dedup_ratio=0.323334 router_tokens_per_expert=[510, 504, 503, 530, 514, 522, 513, 489, 561, 484, 482, 520, 484, 527, 532, 487, 492, 478, 477, 519, 525, 492, 512, 498, 508, 526, 480, 524, 502, 525, 505, 531, 505, 496, 503, 538, 492, 544, 517, 519, 520, 508, 530, 487, 502, 526, 549, 531, 511, 533, 475, 512, 499, 539, 529, 508, 493, 551, 502, 499, 585, 516, 468, 525] payload_rows_per_dst=[5596, 5506, 5458, 5534, 5544, 5516, 5532, 5660] dup_rows_per_dst=[2574, 2648, 2514, 2668, 2688, 2790, 2614, 2694] combine_rows_per_src=[5596, 5506, 5458, 5534, 5544, 5516, 5532, 5660] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44526 payload_rows_remote=38928 dup_rows=21010 dup_rows_remote=18292 dedup_ratio=0.320587 router_tokens_per_expert=[490, 536, 478, 544, 522, 525, 537, 511, 515, 520, 469, 513, 497, 486, 489, 512, 491, 522, 512, 510, 513, 558, 546, 519, 517, 520, 534, 521, 501, 511, 531, 523, 535, 526, 496, 473, 543, 543, 521, 502, 479, 496, 474, 510, 503, 496, 519, 516, 488, 514, 529, 545, 527, 461, 531, 527, 496, 449, 545, 499, 541, 532, 499, 480] payload_rows_per_dst=[5576, 5572, 5628, 5598, 5574, 5450, 5628, 5500] dup_rows_per_dst=[2710, 2430, 2714, 2718, 2704, 2536, 2616, 2582] combine_rows_per_src=[5576, 5572, 5628, 5598, 5574, 5450, 5628, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44396 payload_rows_remote=38888 dup_rows=21140 dup_rows_remote=18564 dedup_ratio=0.322571 router_tokens_per_expert=[511, 503, 491, 538, 530, 514, 491, 524, 497, 505, 548, 508, 520, 537, 532, 511, 507, 502, 505, 507, 505, 499, 497, 484, 508, 497, 535, 533, 518, 526, 539, 512, 514, 518, 539, 520, 494, 518, 550, 489, 504, 520, 496, 512, 510, 513, 465, 526, 486, 499, 528, 511, 523, 486, 511, 498, 511, 518, 509, 500, 499, 497, 551, 519] payload_rows_per_dst=[5494, 5516, 5502, 5578, 5634, 5602, 5508, 5562] dup_rows_per_dst=[2710, 2800, 2510, 2758, 2650, 2490, 2576, 2646] combine_rows_per_src=[5494, 5516, 5502, 5578, 5634, 5602, 5508, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279250944 dispatch_meta_bytes=228856 combine_payload_bytes=279250944 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279479800 ingress_bytes=499451904 link_bytes=499451904 token_bytes=558730744 weight_movement_bytes=220200960 total_nvlink_bytes=778931704 weight_to_token_ratio=0.394109 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.686290068e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44382 payload_rows_remote=38792 dup_rows=21154 dup_rows_remote=18392 dedup_ratio=0.322784 router_tokens_per_expert=[502, 489, 491, 503, 501, 515, 494, 488, 524, 525, 535, 540, 509, 457, 507, 514, 476, 470, 505, 527, 527, 514, 503, 505, 526, 530, 502, 516, 549, 486, 497, 509, 511, 453, 526, 533, 547, 512, 514, 501, 528, 520, 502, 536, 488, 508, 504, 499, 527, 505, 505, 526, 558, 501, 538, 546, 517, 496, 518, 502, 522, 527, 528, 534] payload_rows_per_dst=[5590, 5416, 5486, 5632, 5498, 5674, 5600, 5486] dup_rows_per_dst=[2762, 2508, 2568, 2800, 2494, 2800, 2724, 2498] combine_rows_per_src=[5590, 5416, 5486, 5632, 5498, 5674, 5600, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277401600 dispatch_meta_bytes=228840 combine_payload_bytes=277401600 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=277630440 ingress_bytes=497602560 link_bytes=497602560 token_bytes=555032040 weight_movement_bytes=220200960 total_nvlink_bytes=775233000 weight_to_token_ratio=0.396736 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.654126934e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277903360 dispatch_meta_bytes=229240 combine_payload_bytes=277903360 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278132600 ingress_bytes=498104320 link_bytes=498104320 token_bytes=556035960 weight_movement_bytes=220200960 total_nvlink_bytes=776236920 weight_to_token_ratio=0.396019 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.662853366e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277301248 dispatch_meta_bytes=228728 combine_payload_bytes=277301248 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277529976 ingress_bytes=497502208 link_bytes=497502208 token_bytes=554831224 weight_movement_bytes=220200960 total_nvlink_bytes=775032184 weight_to_token_ratio=0.396879 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.652381648e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279738368 dispatch_meta_bytes=230616 combine_payload_bytes=279738368 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279968984 ingress_bytes=389838848 link_bytes=389838848 token_bytes=559707352 weight_movement_bytes=110100480 total_nvlink_bytes=669807832 weight_to_token_ratio=0.196711 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.779938741e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279035904 dispatch_meta_bytes=228880 combine_payload_bytes=279035904 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279264784 ingress_bytes=389136384 link_bytes=389136384 token_bytes=558300688 weight_movement_bytes=110100480 total_nvlink_bytes=668401168 weight_to_token_ratio=0.197206 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.767721737e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278749184 dispatch_meta_bytes=229808 combine_payload_bytes=278749184 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278978992 ingress_bytes=609050624 link_bytes=609050624 token_bytes=557728176 weight_movement_bytes=330301440 total_nvlink_bytes=888029616 weight_to_token_ratio=0.592227 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059239207e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278061056 dispatch_meta_bytes=228736 combine_payload_bytes=278061056 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278289792 ingress_bytes=388161536 link_bytes=388161536 token_bytes=556350848 weight_movement_bytes=110100480 total_nvlink_bytes=666451328 weight_to_token_ratio=0.197898 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.750767527e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44248 payload_rows_remote=38688 dup_rows=21288 dup_rows_remote=18704 dedup_ratio=0.324829 router_tokens_per_expert=[533, 491, 498, 469, 496, 505, 508, 515, 562, 500, 526, 512, 548, 532, 520, 478, 495, 534, 500, 522, 498, 513, 518, 532, 493, 500, 533, 512, 518, 550, 494, 531, 510, 501, 524, 510, 495, 530, 495, 507, 503, 523, 548, 508, 532, 504, 505, 537, 528, 501, 507, 486, 526, 503, 501, 480, 511, 500, 461, 513, 519, 540, 514, 510] payload_rows_per_dst=[5438, 5634, 5570, 5558, 5560, 5494, 5442, 5552] dup_rows_per_dst=[2592, 2722, 2654, 2704, 2584, 2826, 2622, 2584] combine_rows_per_src=[5438, 5634, 5570, 5558, 5560, 5494, 5442, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44732 payload_rows_remote=39104 dup_rows=20804 dup_rows_remote=18224 dedup_ratio=0.317444 router_tokens_per_expert=[507, 515, 492, 515, 520, 485, 482, 509, 525, 514, 546, 528, 486, 483, 519, 503, 517, 493, 501, 484, 522, 535, 525, 520, 517, 510, 484, 519, 505, 538, 536, 499, 534, 529, 505, 466, 499, 541, 547, 544, 513, 493, 531, 521, 494, 505, 530, 544, 533, 480, 488, 513, 550, 475, 493, 506, 510, 524, 484, 513, 499, 510, 520, 540] payload_rows_per_dst=[5576, 5628, 5538, 5546, 5642, 5672, 5544, 5586] dup_rows_per_dst=[2474, 2580, 2656, 2670, 2688, 2590, 2532, 2614] combine_rows_per_src=[5576, 5628, 5538, 5546, 5642, 5672, 5544, 5586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44536 payload_rows_remote=39048 dup_rows=21000 dup_rows_remote=18390 dedup_ratio=0.320435 router_tokens_per_expert=[494, 501, 488, 493, 531, 506, 521, 539, 521, 475, 533, 487, 479, 516, 476, 496, 530, 527, 494, 502, 503, 488, 501, 504, 542, 504, 531, 496, 533, 480, 537, 516, 517, 549, 508, 491, 526, 496, 515, 501, 546, 541, 507, 546, 506, 509, 534, 494, 498, 483, 510, 499, 505, 511, 551, 522, 513, 494, 530, 528, 515, 545, 528, 506] payload_rows_per_dst=[5612, 5442, 5488, 5574, 5568, 5656, 5592, 5604] dup_rows_per_dst=[2534, 2524, 2610, 2704, 2638, 2710, 2566, 2714] combine_rows_per_src=[5612, 5442, 5488, 5574, 5568, 5656, 5592, 5604] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44506 payload_rows_remote=38804 dup_rows=21030 dup_rows_remote=18026 dedup_ratio=0.320892 router_tokens_per_expert=[474, 528, 488, 480, 517, 534, 527, 496, 481, 546, 509, 500, 547, 513, 486, 467, 514, 536, 515, 509, 507, 499, 538, 510, 476, 509, 519, 529, 500, 502, 493, 505, 519, 509, 505, 529, 517, 503, 486, 486, 476, 495, 475, 518, 540, 547, 526, 545, 549, 532, 497, 527, 575, 496, 501, 498, 505, 510, 520, 542, 516, 500, 569, 501] payload_rows_per_dst=[5532, 5502, 5604, 5536, 5514, 5506, 5610, 5702] dup_rows_per_dst=[2556, 2596, 2652, 2402, 2594, 2486, 2740, 3004] combine_rows_per_src=[5532, 5502, 5604, 5536, 5514, 5506, 5610, 5702] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44570 payload_rows_remote=38908 dup_rows=20966 dup_rows_remote=18242 dedup_ratio=0.319916 router_tokens_per_expert=[507, 531, 488, 498, 492, 474, 497, 517, 528, 534, 488, 488, 502, 532, 522, 517, 492, 489, 505, 518, 478, 497, 514, 488, 524, 546, 481, 529, 560, 528, 514, 511, 529, 503, 514, 525, 515, 517, 543, 507, 485, 502, 500, 519, 512, 494, 493, 522, 532, 499, 546, 524, 494, 510, 512, 529, 533, 493, 511, 510, 532, 517, 520, 537] payload_rows_per_dst=[5524, 5574, 5492, 5662, 5600, 5508, 5628, 5582] dup_rows_per_dst=[2484, 2648, 2470, 2724, 2706, 2546, 2664, 2724] combine_rows_per_src=[5524, 5574, 5492, 5662, 5600, 5508, 5628, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44536 payload_rows_remote=39040 dup_rows=21000 dup_rows_remote=18262 dedup_ratio=0.320435 router_tokens_per_expert=[492, 554, 532, 518, 531, 508, 457, 501, 500, 510, 517, 518, 489, 536, 538, 513, 506, 491, 537, 513, 534, 487, 503, 517, 526, 499, 499, 493, 482, 503, 514, 490, 492, 497, 527, 529, 498, 488, 558, 497, 554, 521, 512, 462, 515, 491, 517, 545, 528, 540, 555, 546, 517, 525, 490, 487, 486, 497, 517, 524, 534, 511, 513, 487] payload_rows_per_dst=[5586, 5684, 5530, 5444, 5554, 5496, 5654, 5588] dup_rows_per_dst=[2600, 2558, 2646, 2568, 2618, 2738, 2722, 2550] combine_rows_per_src=[5586, 5684, 5530, 5444, 5554, 5496, 5654, 5588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279953408 dispatch_meta_bytes=229280 combine_payload_bytes=279953408 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280182688 ingress_bytes=610254848 link_bytes=610254848 token_bytes=560136096 weight_movement_bytes=330301440 total_nvlink_bytes=890437536 weight_to_token_ratio=0.589681 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061333550e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279379968 dispatch_meta_bytes=228984 combine_payload_bytes=279379968 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279608952 ingress_bytes=499580928 link_bytes=499580928 token_bytes=558988920 weight_movement_bytes=220200960 total_nvlink_bytes=779189880 weight_to_token_ratio=0.393927 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.688534007e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278792192 dispatch_meta_bytes=228600 combine_payload_bytes=278792192 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279020792 ingress_bytes=498993152 link_bytes=498993152 token_bytes=557812984 weight_movement_bytes=220200960 total_nvlink_bytes=778013944 weight_to_token_ratio=0.394758 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.678311616e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279781376 dispatch_meta_bytes=229648 combine_payload_bytes=279781376 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280011024 ingress_bytes=389881856 link_bytes=389881856 token_bytes=559792400 weight_movement_bytes=110100480 total_nvlink_bytes=669892880 weight_to_token_ratio=0.196681 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.780686721e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278978560 dispatch_meta_bytes=229744 combine_payload_bytes=278978560 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279208304 ingress_bytes=389079040 link_bytes=389079040 token_bytes=558186864 weight_movement_bytes=110100480 total_nvlink_bytes=668287344 weight_to_token_ratio=0.197247 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.766724431e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277301248 dispatch_meta_bytes=229632 combine_payload_bytes=277301248 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277530880 ingress_bytes=497502208 link_bytes=497502208 token_bytes=554832128 weight_movement_bytes=220200960 total_nvlink_bytes=775033088 weight_to_token_ratio=0.396879 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.652381648e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44482 payload_rows_remote=38692 dup_rows=21054 dup_rows_remote=18164 dedup_ratio=0.321259 router_tokens_per_expert=[515, 520, 543, 509, 517, 495, 498, 521, 509, 503, 515, 496, 502, 520, 518, 522, 491, 509, 519, 469, 502, 517, 469, 512, 531, 539, 485, 511, 517, 513, 500, 544, 544, 512, 505, 519, 496, 542, 516, 490, 477, 541, 519, 485, 493, 523, 498, 535, 501, 499, 522, 499, 511, 528, 553, 532, 518, 527, 531, 502, 507, 519, 485, 508] payload_rows_per_dst=[5790, 5402, 5600, 5538, 5476, 5490, 5638, 5548] dup_rows_per_dst=[2890, 2562, 2644, 2572, 2574, 2546, 2656, 2610] combine_rows_per_src=[5790, 5402, 5600, 5538, 5476, 5490, 5638, 5548] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44746 payload_rows_remote=39106 dup_rows=20790 dup_rows_remote=18194 dedup_ratio=0.317230 router_tokens_per_expert=[509, 515, 495, 518, 522, 488, 483, 510, 531, 513, 548, 529, 488, 484, 521, 504, 519, 495, 504, 485, 521, 535, 530, 522, 518, 511, 486, 520, 506, 538, 537, 498, 535, 527, 505, 461, 499, 539, 547, 544, 511, 495, 529, 519, 494, 504, 530, 543, 531, 479, 486, 511, 550, 472, 491, 504, 508, 519, 482, 511, 493, 508, 520, 538] payload_rows_per_dst=[5592, 5640, 5552, 5552, 5638, 5672, 5530, 5570] dup_rows_per_dst=[2488, 2596, 2670, 2676, 2676, 2578, 2518, 2588] combine_rows_per_src=[5592, 5640, 5552, 5552, 5638, 5672, 5530, 5570] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277344256 dispatch_meta_bytes=227424 combine_payload_bytes=277344256 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277571680 ingress_bytes=387444736 link_bytes=387444736 token_bytes=554915936 weight_movement_bytes=110100480 total_nvlink_bytes=665016416 weight_to_token_ratio=0.198409 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.738301196e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44546 payload_rows_remote=38962 dup_rows=20990 dup_rows_remote=18300 dedup_ratio=0.320282 router_tokens_per_expert=[517, 535, 500, 501, 543, 492, 510, 547, 502, 533, 534, 521, 509, 525, 510, 517, 453, 526, 517, 539, 525, 510, 485, 540, 541, 515, 487, 485, 522, 508, 506, 513, 505, 517, 506, 521, 537, 529, 539, 495, 485, 543, 503, 518, 496, 551, 490, 551, 508, 474, 510, 495, 510, 530, 486, 504, 504, 493, 516, 497, 482, 490, 533, 482] payload_rows_per_dst=[5670, 5646, 5550, 5554, 5572, 5584, 5528, 5442] dup_rows_per_dst=[2620, 2656, 2640, 2600, 2726, 2690, 2506, 2552] combine_rows_per_src=[5670, 5646, 5550, 5554, 5572, 5584, 5528, 5442] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44410 payload_rows_remote=38646 dup_rows=21126 dup_rows_remote=18026 dedup_ratio=0.322357 router_tokens_per_expert=[504, 547, 516, 508, 525, 474, 519, 503, 500, 531, 534, 503, 527, 521, 546, 508, 532, 547, 527, 486, 532, 525, 524, 513, 492, 484, 543, 517, 537, 487, 489, 521, 523, 496, 502, 476, 505, 489, 534, 456, 495, 535, 494, 502, 515, 519, 513, 526, 496, 522, 517, 539, 507, 470, 524, 496, 518, 509, 503, 542, 487, 550, 472, 514] payload_rows_per_dst=[5304, 5504, 5418, 5594, 5604, 5642, 5580, 5764] dup_rows_per_dst=[2396, 2528, 2280, 2638, 2650, 2772, 2762, 3100] combine_rows_per_src=[5304, 5504, 5418, 5594, 5604, 5642, 5580, 5764] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280311808 dispatch_meta_bytes=229200 combine_payload_bytes=280311808 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280541008 ingress_bytes=500512768 link_bytes=500512768 token_bytes=560852816 weight_movement_bytes=220200960 total_nvlink_bytes=781053776 weight_to_token_ratio=0.392618 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.704740238e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44384 payload_rows_remote=38768 dup_rows=21152 dup_rows_remote=18544 dedup_ratio=0.322754 router_tokens_per_expert=[525, 519, 526, 507, 503, 552, 550, 508, 513, 511, 510, 529, 519, 515, 517, 535, 487, 501, 527, 493, 535, 500, 547, 512, 525, 553, 527, 516, 530, 492, 452, 517, 500, 495, 498, 530, 532, 498, 545, 506, 543, 528, 483, 446, 519, 493, 510, 502, 521, 471, 521, 481, 496, 529, 533, 537, 481, 498, 544, 490, 487, 515, 488, 495] payload_rows_per_dst=[5624, 5540, 5574, 5616, 5526, 5478, 5524, 5502] dup_rows_per_dst=[2756, 2758, 2630, 2608, 2682, 2570, 2654, 2494] combine_rows_per_src=[5624, 5540, 5574, 5616, 5526, 5478, 5524, 5502] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44414 payload_rows_remote=38772 dup_rows=21122 dup_rows_remote=18384 dedup_ratio=0.322296 router_tokens_per_expert=[509, 521, 538, 535, 517, 534, 527, 472, 507, 499, 506, 539, 522, 514, 512, 518, 527, 546, 492, 521, 537, 510, 543, 514, 472, 528, 491, 493, 498, 538, 512, 506, 538, 526, 519, 456, 503, 519, 493, 515, 513, 495, 506, 511, 508, 475, 495, 518, 529, 526, 527, 487, 512, 511, 511, 503, 501, 508, 492, 489, 527, 501, 514, 542] payload_rows_per_dst=[5556, 5604, 5642, 5496, 5544, 5446, 5598, 5528] dup_rows_per_dst=[2750, 2630, 2738, 2580, 2594, 2596, 2614, 2620] combine_rows_per_src=[5556, 5604, 5642, 5496, 5544, 5446, 5598, 5528] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44512 payload_rows_remote=38966 dup_rows=21024 dup_rows_remote=18434 dedup_ratio=0.320801 router_tokens_per_expert=[511, 504, 529, 497, 546, 502, 487, 496, 525, 530, 505, 484, 491, 494, 524, 500, 517, 520, 524, 510, 522, 501, 536, 505, 536, 510, 499, 523, 524, 498, 550, 515, 511, 486, 503, 493, 515, 491, 532, 506, 512, 554, 502, 531, 516, 511, 519, 485, 536, 497, 506, 486, 553, 496, 501, 493, 558, 476, 515, 520, 494, 522, 514, 519] payload_rows_per_dst=[5600, 5516, 5558, 5616, 5500, 5616, 5546, 5560] dup_rows_per_dst=[2544, 2590, 2712, 2694, 2574, 2644, 2590, 2676] combine_rows_per_src=[5600, 5516, 5558, 5616, 5500, 5616, 5546, 5560] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44484 payload_rows_remote=38874 dup_rows=21052 dup_rows_remote=18476 dedup_ratio=0.321228 router_tokens_per_expert=[499, 535, 484, 509, 506, 524, 527, 489, 510, 528, 512, 492, 501, 536, 524, 495, 528, 536, 541, 483, 547, 476, 547, 537, 504, 503, 542, 517, 519, 507, 510, 498, 507, 535, 504, 493, 512, 504, 509, 529, 534, 545, 483, 482, 526, 524, 466, 495, 537, 477, 540, 542, 546, 507, 485, 510, 503, 517, 527, 550, 475, 476, 465, 497] payload_rows_per_dst=[5578, 5574, 5628, 5558, 5610, 5514, 5606, 5416] dup_rows_per_dst=[2568, 2622, 2762, 2642, 2576, 2596, 2682, 2604] combine_rows_per_src=[5578, 5574, 5628, 5558, 5610, 5514, 5606, 5416] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279279616 dispatch_meta_bytes=229048 combine_payload_bytes=279279616 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279508664 ingress_bytes=389380096 link_bytes=389380096 token_bytes=558788280 weight_movement_bytes=110100480 total_nvlink_bytes=668888760 weight_to_token_ratio=0.197034 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.771960290e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44398 payload_rows_remote=38910 dup_rows=21138 dup_rows_remote=18490 dedup_ratio=0.322540 router_tokens_per_expert=[477, 505, 547, 479, 526, 502, 470, 530, 557, 503, 520, 480, 525, 474, 564, 513, 527, 529, 501, 530, 509, 504, 512, 499, 490, 516, 508, 503, 510, 529, 498, 522, 523, 526, 490, 537, 546, 510, 533, 474, 528, 490, 516, 498, 510, 513, 505, 517, 501, 526, 517, 499, 489, 479, 525, 532, 533, 523, 488, 492, 549, 536, 508, 496] payload_rows_per_dst=[5424, 5652, 5570, 5590, 5582, 5500, 5488, 5592] dup_rows_per_dst=[2648, 2620, 2652, 2562, 2696, 2654, 2648, 2658] combine_rows_per_src=[5424, 5652, 5570, 5590, 5582, 5500, 5488, 5592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44452 payload_rows_remote=38562 dup_rows=21084 dup_rows_remote=18018 dedup_ratio=0.321716 router_tokens_per_expert=[511, 519, 542, 507, 514, 494, 495, 520, 505, 500, 514, 495, 501, 519, 518, 518, 487, 506, 519, 468, 500, 517, 466, 511, 531, 536, 485, 512, 516, 513, 498, 541, 546, 515, 508, 522, 499, 544, 514, 491, 480, 539, 518, 488, 495, 523, 499, 535, 505, 499, 526, 500, 512, 528, 555, 532, 518, 530, 534, 504, 508, 520, 490, 513] payload_rows_per_dst=[5890, 5480, 5674, 5606, 5434, 5544, 5540, 5284] dup_rows_per_dst=[3066, 2692, 2730, 2686, 2558, 2610, 2544, 2198] combine_rows_per_src=[5890, 5480, 5674, 5606, 5434, 5544, 5540, 5284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277315584 dispatch_meta_bytes=229568 combine_payload_bytes=277315584 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277545152 ingress_bytes=497516544 link_bytes=497516544 token_bytes=554860736 weight_movement_bytes=220200960 total_nvlink_bytes=775061696 weight_to_token_ratio=0.396858 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.652630974e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280297472 dispatch_meta_bytes=229312 combine_payload_bytes=280297472 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280526784 ingress_bytes=390397952 link_bytes=390397952 token_bytes=560824256 weight_movement_bytes=110100480 total_nvlink_bytes=670924736 weight_to_token_ratio=0.196319 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.789662480e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279896064 dispatch_meta_bytes=229752 combine_payload_bytes=279896064 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280125816 ingress_bytes=389996544 link_bytes=389996544 token_bytes=560021880 weight_movement_bytes=110100480 total_nvlink_bytes=670122360 weight_to_token_ratio=0.196600 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.782681334e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278147072 dispatch_meta_bytes=227320 combine_payload_bytes=278147072 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278374392 ingress_bytes=498348032 link_bytes=498348032 token_bytes=556521464 weight_movement_bytes=220200960 total_nvlink_bytes=776722424 weight_to_token_ratio=0.395674 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.667091918e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278892544 dispatch_meta_bytes=228600 combine_payload_bytes=278892544 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279121144 ingress_bytes=499093504 link_bytes=499093504 token_bytes=558013688 weight_movement_bytes=220200960 total_nvlink_bytes=778214648 weight_to_token_ratio=0.394616 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.680056902e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279838720 dispatch_meta_bytes=229208 combine_payload_bytes=279838720 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280067928 ingress_bytes=500039680 link_bytes=500039680 token_bytes=559906648 weight_movement_bytes=220200960 total_nvlink_bytes=780107608 weight_to_token_ratio=0.393282 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.696512459e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278906880 dispatch_meta_bytes=229600 combine_payload_bytes=278906880 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279136480 ingress_bytes=609208320 link_bytes=609208320 token_bytes=558043360 weight_movement_bytes=330301440 total_nvlink_bytes=888344800 weight_to_token_ratio=0.591892 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059513466e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=276412416 dispatch_meta_bytes=226320 combine_payload_bytes=276412416 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276638736 ingress_bytes=386512896 link_bytes=386512896 token_bytes=553051152 weight_movement_bytes=110100480 total_nvlink_bytes=663151632 weight_to_token_ratio=0.199078 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.722094966e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44480 payload_rows_remote=38868 dup_rows=21056 dup_rows_remote=18450 dedup_ratio=0.321289 router_tokens_per_expert=[498, 531, 483, 507, 502, 519, 525, 486, 507, 523, 508, 490, 499, 533, 523, 493, 528, 534, 536, 483, 544, 476, 546, 536, 502, 500, 540, 517, 519, 511, 508, 495, 510, 538, 505, 494, 513, 506, 512, 531, 534, 546, 485, 478, 528, 524, 469, 499, 538, 477, 542, 546, 550, 510, 486, 513, 504, 520, 531, 556, 476, 481, 467, 497] payload_rows_per_dst=[5560, 5552, 5620, 5558, 5612, 5512, 5626, 5440] dup_rows_per_dst=[2542, 2600, 2746, 2626, 2606, 2614, 2698, 2624] combine_rows_per_src=[5560, 5552, 5620, 5558, 5612, 5512, 5626, 5440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44602 payload_rows_remote=39102 dup_rows=20934 dup_rows_remote=18296 dedup_ratio=0.319427 router_tokens_per_expert=[510, 534, 529, 513, 552, 531, 497, 497, 501, 537, 481, 540, 479, 514, 521, 496, 531, 540, 499, 537, 481, 532, 500, 514, 501, 515, 502, 510, 461, 549, 526, 491, 542, 532, 521, 493, 536, 507, 481, 470, 486, 512, 505, 496, 481, 560, 534, 501, 473, 511, 519, 497, 491, 530, 523, 502, 518, 510, 515, 532, 549, 511, 503, 506] payload_rows_per_dst=[5664, 5500, 5654, 5528, 5536, 5586, 5526, 5608] dup_rows_per_dst=[2662, 2638, 2614, 2582, 2628, 2564, 2566, 2680] combine_rows_per_src=[5664, 5500, 5654, 5528, 5536, 5586, 5526, 5608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44410 payload_rows_remote=38776 dup_rows=21126 dup_rows_remote=18404 dedup_ratio=0.322357 router_tokens_per_expert=[505, 515, 535, 534, 517, 532, 523, 473, 505, 494, 505, 536, 520, 513, 510, 518, 526, 543, 491, 520, 536, 510, 539, 513, 471, 527, 492, 491, 499, 540, 510, 509, 536, 525, 521, 460, 502, 519, 492, 514, 514, 496, 509, 514, 510, 478, 496, 519, 531, 530, 530, 487, 514, 511, 512, 506, 504, 510, 493, 491, 529, 503, 516, 544] payload_rows_per_dst=[5534, 5588, 5634, 5500, 5548, 5460, 5606, 5540] dup_rows_per_dst=[2734, 2614, 2722, 2578, 2590, 2612, 2636, 2640] combine_rows_per_src=[5534, 5588, 5634, 5500, 5548, 5460, 5606, 5540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44420 payload_rows_remote=38782 dup_rows=21116 dup_rows_remote=18176 dedup_ratio=0.322205 router_tokens_per_expert=[503, 543, 510, 505, 521, 469, 515, 502, 498, 529, 533, 502, 524, 520, 546, 507, 531, 546, 525, 484, 530, 525, 522, 513, 494, 483, 542, 518, 538, 488, 487, 522, 522, 496, 502, 475, 509, 493, 535, 459, 494, 535, 493, 503, 515, 519, 516, 525, 496, 523, 520, 541, 511, 470, 525, 498, 520, 511, 508, 544, 492, 555, 472, 516] payload_rows_per_dst=[5414, 5636, 5454, 5602, 5576, 5572, 5528, 5638] dup_rows_per_dst=[2490, 2666, 2372, 2660, 2638, 2694, 2656, 2940] combine_rows_per_src=[5414, 5636, 5454, 5602, 5576, 5572, 5528, 5638] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44374 payload_rows_remote=38762 dup_rows=21162 dup_rows_remote=18548 dedup_ratio=0.322906 router_tokens_per_expert=[520, 516, 525, 507, 501, 550, 547, 504, 514, 507, 509, 529, 516, 513, 515, 532, 483, 498, 521, 487, 535, 499, 545, 508, 525, 553, 527, 515, 531, 494, 453, 515, 500, 498, 496, 527, 530, 498, 544, 509, 542, 530, 486, 446, 521, 491, 512, 508, 522, 470, 526, 484, 501, 534, 536, 542, 482, 501, 551, 493, 487, 516, 494, 497] payload_rows_per_dst=[5608, 5528, 5552, 5612, 5524, 5492, 5542, 5516] dup_rows_per_dst=[2732, 2742, 2600, 2614, 2680, 2580, 2688, 2526] combine_rows_per_src=[5608, 5528, 5552, 5612, 5524, 5492, 5542, 5516] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277014528 dispatch_meta_bytes=226688 combine_payload_bytes=277014528 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277241216 ingress_bytes=387115008 link_bytes=387115008 token_bytes=554255744 weight_movement_bytes=110100480 total_nvlink_bytes=664356224 weight_to_token_ratio=0.198646 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.732566684e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44468 payload_rows_remote=38888 dup_rows=21068 dup_rows_remote=18432 dedup_ratio=0.321472 router_tokens_per_expert=[578, 541, 484, 465, 511, 508, 491, 528, 518, 552, 501, 476, 489, 516, 537, 501, 533, 502, 481, 540, 488, 480, 521, 517, 527, 527, 510, 512, 490, 544, 509, 491, 530, 493, 527, 508, 559, 488, 505, 502, 507, 527, 502, 504, 501, 536, 520, 527, 531, 515, 513, 537, 507, 523, 507, 518, 494, 477, 514, 485, 514, 512, 528, 489] payload_rows_per_dst=[5580, 5610, 5532, 5490, 5584, 5632, 5600, 5440] dup_rows_per_dst=[2636, 2590, 2568, 2730, 2640, 2616, 2702, 2586] combine_rows_per_src=[5580, 5610, 5532, 5490, 5584, 5632, 5600, 5440] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277889024 dispatch_meta_bytes=229248 combine_payload_bytes=277889024 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278118272 ingress_bytes=387989504 link_bytes=387989504 token_bytes=556007296 weight_movement_bytes=110100480 total_nvlink_bytes=666107776 weight_to_token_ratio=0.198020 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.747775608e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277917696 dispatch_meta_bytes=228624 combine_payload_bytes=277917696 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278146320 ingress_bytes=608219136 link_bytes=608219136 token_bytes=556064016 weight_movement_bytes=330301440 total_nvlink_bytes=886365456 weight_to_token_ratio=0.593999 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057793112e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279308288 dispatch_meta_bytes=229600 combine_payload_bytes=279308288 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279537888 ingress_bytes=389408768 link_bytes=389408768 token_bytes=558846176 weight_movement_bytes=110100480 total_nvlink_bytes=668946656 weight_to_token_ratio=0.197014 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.772458943e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278648832 dispatch_meta_bytes=229400 combine_payload_bytes=278648832 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278878232 ingress_bytes=388749312 link_bytes=388749312 token_bytes=557527064 weight_movement_bytes=110100480 total_nvlink_bytes=667627544 weight_to_token_ratio=0.197480 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.760989918e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44598 payload_rows_remote=39076 dup_rows=20938 dup_rows_remote=18292 dedup_ratio=0.319489 router_tokens_per_expert=[512, 536, 532, 514, 553, 531, 500, 499, 500, 540, 481, 543, 483, 515, 522, 500, 535, 543, 504, 538, 487, 531, 500, 514, 503, 517, 501, 510, 463, 548, 526, 492, 542, 529, 520, 492, 538, 507, 480, 469, 484, 510, 504, 496, 481, 562, 532, 501, 470, 507, 514, 494, 490, 529, 520, 498, 518, 509, 514, 529, 545, 508, 499, 504] payload_rows_per_dst=[5672, 5522, 5666, 5536, 5528, 5580, 5500, 5594] dup_rows_per_dst=[2682, 2646, 2638, 2584, 2626, 2560, 2544, 2658] combine_rows_per_src=[5672, 5522, 5666, 5536, 5528, 5580, 5500, 5594] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278749184 dispatch_meta_bytes=229280 combine_payload_bytes=278749184 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278978464 ingress_bytes=609050624 link_bytes=609050624 token_bytes=557727648 weight_movement_bytes=330301440 total_nvlink_bytes=888029088 weight_to_token_ratio=0.592227 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059239207e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=280096768 dispatch_meta_bytes=229472 combine_payload_bytes=280096768 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280326240 ingress_bytes=610398208 link_bytes=610398208 token_bytes=560423008 weight_movement_bytes=330301440 total_nvlink_bytes=890724448 weight_to_token_ratio=0.589379 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061582877e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44480 payload_rows_remote=38982 dup_rows=21056 dup_rows_remote=18496 dedup_ratio=0.321289 router_tokens_per_expert=[516, 530, 489, 507, 485, 504, 517, 501, 531, 512, 489, 515, 544, 511, 513, 512, 501, 536, 496, 500, 528, 505, 543, 511, 531, 517, 522, 530, 533, 500, 517, 519, 493, 493, 517, 517, 470, 507, 553, 500, 500, 487, 478, 506, 510, 491, 523, 534, 528, 502, 525, 531, 511, 515, 488, 528, 513, 515, 492, 499, 543, 506, 504, 524] payload_rows_per_dst=[5532, 5578, 5538, 5592, 5590, 5498, 5638, 5514] dup_rows_per_dst=[2566, 2676, 2702, 2746, 2510, 2560, 2618, 2678] combine_rows_per_src=[5532, 5578, 5538, 5592, 5590, 5498, 5638, 5514] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44344 payload_rows_remote=38690 dup_rows=21192 dup_rows_remote=18482 dedup_ratio=0.323364 router_tokens_per_expert=[491, 492, 509, 503, 498, 521, 483, 516, 567, 484, 529, 498, 511, 468, 553, 566, 473, 537, 509, 509, 539, 515, 496, 533, 522, 483, 526, 505, 494, 514, 528, 529, 508, 513, 524, 497, 496, 529, 513, 531, 505, 498, 538, 548, 502, 523, 495, 517, 494, 499, 495, 480, 476, 494, 507, 536, 503, 515, 526, 508, 500, 545, 512, 540] payload_rows_per_dst=[5510, 5466, 5476, 5556, 5600, 5688, 5394, 5654] dup_rows_per_dst=[2516, 2622, 2746, 2644, 2622, 2828, 2504, 2710] combine_rows_per_src=[5510, 5466, 5476, 5556, 5600, 5688, 5394, 5654] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44736 payload_rows_remote=39064 dup_rows=20800 dup_rows_remote=18156 dedup_ratio=0.317383 router_tokens_per_expert=[500, 515, 516, 539, 490, 520, 520, 501, 481, 504, 476, 525, 512, 538, 506, 524, 512, 491, 483, 501, 485, 536, 522, 505, 512, 533, 490, 512, 525, 519, 536, 531, 520, 514, 499, 511, 495, 523, 468, 516, 522, 554, 482, 524, 520, 506, 490, 506, 505, 512, 490, 488, 571, 532, 549, 498, 518, 551, 514, 549, 482, 505, 491, 503] payload_rows_per_dst=[5562, 5646, 5528, 5672, 5560, 5614, 5550, 5604] dup_rows_per_dst=[2640, 2486, 2542, 2644, 2532, 2594, 2740, 2622] combine_rows_per_src=[5562, 5646, 5528, 5672, 5560, 5614, 5550, 5604] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44768 payload_rows_remote=39192 dup_rows=20768 dup_rows_remote=18332 dedup_ratio=0.316895 router_tokens_per_expert=[490, 493, 533, 531, 550, 496, 474, 498, 506, 529, 491, 523, 489, 529, 505, 488, 475, 496, 490, 508, 502, 520, 522, 493, 532, 513, 542, 511, 511, 500, 491, 513, 528, 500, 506, 583, 506, 542, 530, 517, 543, 501, 491, 491, 536, 507, 537, 515, 509, 532, 547, 512, 530, 528, 493, 482, 518, 513, 505, 480, 471, 509, 510, 552] payload_rows_per_dst=[5560, 5566, 5576, 5594, 5666, 5574, 5636, 5596] dup_rows_per_dst=[2570, 2554, 2436, 2632, 2758, 2668, 2630, 2520] combine_rows_per_src=[5560, 5566, 5576, 5594, 5666, 5574, 5636, 5596] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44542 payload_rows_remote=39048 dup_rows=20994 dup_rows_remote=18310 dedup_ratio=0.320343 router_tokens_per_expert=[529, 502, 529, 492, 484, 521, 514, 575, 516, 516, 524, 482, 534, 524, 509, 488, 507, 531, 527, 531, 524, 495, 512, 514, 514, 489, 511, 517, 509, 507, 518, 467, 536, 527, 514, 502, 518, 496, 516, 490, 527, 501, 497, 519, 518, 528, 512, 498, 505, 501, 488, 499, 560, 506, 504, 526, 519, 505, 534, 499, 469, 490, 557, 495] payload_rows_per_dst=[5610, 5608, 5624, 5512, 5566, 5574, 5494, 5554] dup_rows_per_dst=[2682, 2578, 2658, 2552, 2632, 2626, 2684, 2582] combine_rows_per_src=[5610, 5608, 5624, 5512, 5566, 5574, 5494, 5554] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44356 payload_rows_remote=38800 dup_rows=21180 dup_rows_remote=18484 dedup_ratio=0.323181 router_tokens_per_expert=[541, 548, 499, 524, 465, 544, 553, 484, 516, 502, 495, 561, 532, 513, 550, 494, 521, 530, 504, 473, 524, 526, 496, 511, 543, 477, 512, 507, 526, 483, 543, 502, 528, 516, 505, 488, 571, 505, 504, 509, 486, 502, 488, 526, 509, 513, 510, 501, 524, 549, 464, 481, 522, 501, 551, 494, 500, 499, 527, 489, 505, 505, 510, 487] payload_rows_per_dst=[5598, 5652, 5514, 5548, 5556, 5500, 5536, 5452] dup_rows_per_dst=[2718, 2674, 2656, 2638, 2696, 2570, 2636, 2592] combine_rows_per_src=[5598, 5652, 5514, 5548, 5556, 5500, 5536, 5452] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279422976 dispatch_meta_bytes=229912 combine_payload_bytes=279422976 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279652888 ingress_bytes=389523456 link_bytes=389523456 token_bytes=559075864 weight_movement_bytes=110100480 total_nvlink_bytes=669176344 weight_to_token_ratio=0.196933 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.774453556e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277329920 dispatch_meta_bytes=228688 combine_payload_bytes=277329920 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277558608 ingress_bytes=387430400 link_bytes=387430400 token_bytes=554888528 weight_movement_bytes=110100480 total_nvlink_bytes=664989008 weight_to_token_ratio=0.198419 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.738051870e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44390 payload_rows_remote=38836 dup_rows=21146 dup_rows_remote=18478 dedup_ratio=0.322662 router_tokens_per_expert=[523, 527, 537, 486, 500, 494, 516, 523, 486, 522, 536, 499, 510, 518, 483, 531, 535, 489, 508, 496, 495, 548, 506, 505, 506, 517, 532, 484, 515, 517, 529, 533, 525, 478, 545, 524, 529, 468, 484, 493, 472, 503, 492, 498, 511, 500, 508, 512, 531, 515, 518, 537, 492, 536, 541, 472, 512, 538, 517, 526, 510, 515, 542, 518] payload_rows_per_dst=[5554, 5608, 5594, 5440, 5486, 5466, 5580, 5662] dup_rows_per_dst=[2668, 2608, 2846, 2504, 2628, 2526, 2672, 2694] combine_rows_per_src=[5554, 5608, 5594, 5440, 5486, 5466, 5580, 5662] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=280010752 dispatch_meta_bytes=228880 combine_payload_bytes=280010752 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=280239632 ingress_bytes=610312192 link_bytes=610312192 token_bytes=560250384 weight_movement_bytes=330301440 total_nvlink_bytes=890551824 weight_to_token_ratio=0.589560 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061433281e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=280928256 dispatch_meta_bytes=230096 combine_payload_bytes=280928256 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281158352 ingress_bytes=391028736 link_bytes=391028736 token_bytes=562086608 weight_movement_bytes=110100480 total_nvlink_bytes=672187088 weight_to_token_ratio=0.195878 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.800632851e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279896064 dispatch_meta_bytes=229432 combine_payload_bytes=279896064 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=280125496 ingress_bytes=720297984 link_bytes=720297984 token_bytes=560021560 weight_movement_bytes=440401920 total_nvlink_bytes=1000423480 weight_to_token_ratio=0.786402 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.252716663e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44570 payload_rows_remote=39014 dup_rows=20966 dup_rows_remote=18308 dedup_ratio=0.319916 router_tokens_per_expert=[491, 529, 511, 507, 524, 510, 558, 495, 537, 552, 512, 496, 524, 515, 508, 463, 542, 486, 532, 520, 495, 491, 484, 500, 561, 520, 511, 498, 509, 516, 510, 506, 508, 520, 522, 547, 473, 479, 524, 494, 505, 506, 479, 522, 532, 503, 556, 522, 512, 525, 537, 470, 524, 528, 522, 494, 508, 491, 513, 532, 497, 518, 517, 475] payload_rows_per_dst=[5654, 5556, 5554, 5544, 5550, 5570, 5622, 5520] dup_rows_per_dst=[2596, 2658, 2546, 2718, 2584, 2680, 2602, 2582] combine_rows_per_src=[5654, 5556, 5554, 5544, 5550, 5570, 5622, 5520] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278118400 dispatch_meta_bytes=229136 combine_payload_bytes=278118400 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278347536 ingress_bytes=388218880 link_bytes=388218880 token_bytes=556465936 weight_movement_bytes=110100480 total_nvlink_bytes=666566416 weight_to_token_ratio=0.197857 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.751764834e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278376448 dispatch_meta_bytes=229256 combine_payload_bytes=278376448 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278605704 ingress_bytes=388476928 link_bytes=388476928 token_bytes=556982152 weight_movement_bytes=110100480 total_nvlink_bytes=667082632 weight_to_token_ratio=0.197673 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.756252713e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279652352 dispatch_meta_bytes=229288 combine_payload_bytes=279652352 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279881640 ingress_bytes=499853312 link_bytes=499853312 token_bytes=559533992 weight_movement_bytes=220200960 total_nvlink_bytes=779734952 weight_to_token_ratio=0.393543 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.693271213e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44372 payload_rows_remote=38784 dup_rows=21164 dup_rows_remote=18574 dedup_ratio=0.322937 router_tokens_per_expert=[574, 479, 512, 527, 533, 484, 546, 514, 498, 499, 543, 522, 520, 493, 509, 505, 537, 498, 507, 525, 489, 537, 509, 497, 545, 545, 521, 480, 510, 489, 465, 515, 557, 511, 491, 527, 498, 540, 514, 517, 509, 540, 466, 503, 521, 515, 510, 525, 507, 491, 523, 489, 532, 537, 496, 500, 524, 512, 506, 477, 523, 498, 490, 492] payload_rows_per_dst=[5566, 5542, 5532, 5534, 5586, 5588, 5514, 5510] dup_rows_per_dst=[2772, 2636, 2666, 2606, 2724, 2590, 2636, 2534] combine_rows_per_src=[5566, 5542, 5532, 5534, 5586, 5588, 5514, 5510] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44468 payload_rows_remote=38922 dup_rows=21068 dup_rows_remote=18454 dedup_ratio=0.321472 router_tokens_per_expert=[525, 529, 525, 548, 508, 540, 459, 517, 518, 535, 505, 493, 506, 506, 523, 530, 528, 492, 501, 512, 539, 493, 530, 494, 525, 520, 507, 507, 503, 541, 505, 535, 483, 476, 545, 545, 514, 528, 532, 504, 501, 513, 528, 477, 507, 546, 496, 518, 497, 518, 482, 527, 507, 500, 496, 510, 531, 489, 536, 489, 466, 520, 496, 492] payload_rows_per_dst=[5498, 5448, 5638, 5490, 5504, 5610, 5734, 5546] dup_rows_per_dst=[2468, 2586, 2778, 2480, 2628, 2660, 2854, 2614] combine_rows_per_src=[5498, 5448, 5638, 5490, 5504, 5610, 5734, 5546] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44314 payload_rows_remote=38816 dup_rows=21222 dup_rows_remote=18634 dedup_ratio=0.323822 router_tokens_per_expert=[482, 564, 495, 476, 513, 521, 504, 510, 510, 474, 538, 516, 504, 548, 528, 507, 462, 538, 527, 519, 516, 500, 514, 543, 516, 446, 505, 503, 545, 473, 534, 521, 536, 512, 512, 495, 511, 519, 475, 479, 552, 494, 527, 518, 547, 525, 510, 503, 515, 493, 536, 479, 497, 489, 522, 512, 557, 541, 519, 493, 523, 506, 511, 508] payload_rows_per_dst=[5464, 5562, 5526, 5498, 5526, 5572, 5528, 5638] dup_rows_per_dst=[2666, 2688, 2712, 2588, 2552, 2780, 2558, 2678] combine_rows_per_src=[5464, 5562, 5526, 5498, 5526, 5572, 5528, 5638] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44490 payload_rows_remote=38918 dup_rows=21046 dup_rows_remote=18494 dedup_ratio=0.321136 router_tokens_per_expert=[525, 539, 488, 487, 509, 505, 560, 503, 504, 511, 530, 506, 518, 550, 529, 502, 488, 468, 521, 534, 519, 505, 523, 504, 520, 528, 551, 508, 537, 499, 498, 546, 494, 519, 485, 476, 495, 541, 523, 500, 482, 508, 508, 515, 470, 540, 521, 514, 494, 513, 551, 517, 531, 526, 478, 498, 506, 503, 505, 529, 521, 497, 484, 509] payload_rows_per_dst=[5542, 5726, 5572, 5612, 5508, 5496, 5528, 5506] dup_rows_per_dst=[2690, 2574, 2552, 2762, 2558, 2620, 2688, 2602] combine_rows_per_src=[5542, 5726, 5572, 5612, 5508, 5496, 5528, 5506] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44568 payload_rows_remote=39000 dup_rows=20968 dup_rows_remote=18368 dedup_ratio=0.319946 router_tokens_per_expert=[500, 506, 521, 489, 516, 514, 516, 522, 523, 558, 502, 527, 507, 503, 512, 501, 475, 517, 501, 529, 493, 522, 537, 546, 533, 497, 506, 524, 506, 489, 537, 502, 542, 491, 518, 533, 494, 486, 525, 545, 544, 519, 495, 477, 488, 512, 509, 494, 534, 551, 522, 533, 474, 498, 480, 492, 519, 545, 477, 492, 525, 537, 504, 482] payload_rows_per_dst=[5494, 5588, 5550, 5570, 5618, 5584, 5568, 5596] dup_rows_per_dst=[2674, 2678, 2690, 2618, 2650, 2492, 2600, 2566] combine_rows_per_src=[5494, 5588, 5550, 5570, 5618, 5584, 5568, 5596] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278003712 dispatch_meta_bytes=229432 combine_payload_bytes=278003712 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278233144 ingress_bytes=388104192 link_bytes=388104192 token_bytes=556236856 weight_movement_bytes=110100480 total_nvlink_bytes=666337336 weight_to_token_ratio=0.197938 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.749770221e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44480 payload_rows_remote=38950 dup_rows=21056 dup_rows_remote=18432 dedup_ratio=0.321289 router_tokens_per_expert=[495, 505, 504, 502, 487, 529, 492, 547, 521, 461, 536, 538, 522, 477, 528, 540, 532, 504, 495, 521, 511, 511, 532, 505, 485, 525, 493, 475, 527, 512, 511, 518, 514, 522, 504, 497, 526, 522, 488, 504, 508, 526, 534, 507, 529, 495, 516, 491, 478, 533, 490, 532, 496, 490, 523, 516, 526, 518, 520, 532, 508, 532, 514, 536] payload_rows_per_dst=[5550, 5554, 5586, 5526, 5530, 5548, 5568, 5618] dup_rows_per_dst=[2572, 2692, 2636, 2566, 2624, 2664, 2548, 2754] combine_rows_per_src=[5550, 5554, 5586, 5526, 5530, 5548, 5568, 5618] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44640 payload_rows_remote=39070 dup_rows=20896 dup_rows_remote=18166 dedup_ratio=0.318848 router_tokens_per_expert=[532, 521, 517, 484, 494, 510, 519, 554, 522, 495, 529, 516, 521, 499, 495, 518, 487, 505, 517, 521, 493, 506, 507, 473, 527, 499, 525, 547, 525, 519, 525, 514, 515, 539, 516, 508, 512, 537, 475, 522, 517, 524, 543, 492, 477, 517, 522, 510, 491, 510, 541, 513, 468, 526, 522, 514, 501, 470, 512, 488, 499, 517, 530, 524] payload_rows_per_dst=[5570, 5580, 5532, 5630, 5552, 5584, 5618, 5574] dup_rows_per_dst=[2730, 2578, 2486, 2732, 2690, 2620, 2552, 2508] combine_rows_per_src=[5570, 5580, 5532, 5630, 5552, 5584, 5618, 5574] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278992896 dispatch_meta_bytes=229504 combine_payload_bytes=278992896 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279222400 ingress_bytes=389093376 link_bytes=389093376 token_bytes=558215296 weight_movement_bytes=110100480 total_nvlink_bytes=668315776 weight_to_token_ratio=0.197237 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.766973757e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278233088 dispatch_meta_bytes=229800 combine_payload_bytes=278233088 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278462888 ingress_bytes=498434048 link_bytes=498434048 token_bytes=556695976 weight_movement_bytes=220200960 total_nvlink_bytes=776896936 weight_to_token_ratio=0.395550 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.668587878e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278964224 dispatch_meta_bytes=229648 combine_payload_bytes=278964224 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279193872 ingress_bytes=389064704 link_bytes=389064704 token_bytes=558158096 weight_movement_bytes=110100480 total_nvlink_bytes=668258576 weight_to_token_ratio=0.197257 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.766475104e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44592 payload_rows_remote=39002 dup_rows=20944 dup_rows_remote=18430 dedup_ratio=0.319580 router_tokens_per_expert=[512, 538, 517, 538, 514, 496, 518, 507, 481, 468, 548, 489, 521, 511, 526, 508, 496, 516, 511, 528, 496, 494, 512, 487, 523, 510, 498, 499, 531, 551, 487, 563, 500, 500, 519, 529, 516, 529, 510, 540, 498, 482, 523, 548, 519, 500, 495, 499, 509, 528, 518, 524, 489, 479, 468, 506, 525, 498, 532, 505, 520, 529, 515, 522] payload_rows_per_dst=[5628, 5590, 5540, 5636, 5562, 5478, 5510, 5648] dup_rows_per_dst=[2652, 2514, 2540, 2688, 2724, 2650, 2532, 2644] combine_rows_per_src=[5628, 5590, 5540, 5636, 5562, 5478, 5510, 5648] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279552000 dispatch_meta_bytes=229472 combine_payload_bytes=279552000 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279781472 ingress_bytes=389652480 link_bytes=389652480 token_bytes=559333472 weight_movement_bytes=110100480 total_nvlink_bytes=669433952 weight_to_token_ratio=0.196842 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.776697495e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279193600 dispatch_meta_bytes=229528 combine_payload_bytes=279193600 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279423128 ingress_bytes=499394560 link_bytes=499394560 token_bytes=558616728 weight_movement_bytes=220200960 total_nvlink_bytes=778817688 weight_to_token_ratio=0.394190 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.685292761e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=280053760 dispatch_meta_bytes=228944 combine_payload_bytes=280053760 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280282704 ingress_bytes=610355200 link_bytes=610355200 token_bytes=560336464 weight_movement_bytes=330301440 total_nvlink_bytes=890637904 weight_to_token_ratio=0.589470 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061508079e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279566336 dispatch_meta_bytes=229728 combine_payload_bytes=279566336 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279796064 ingress_bytes=609867776 link_bytes=609867776 token_bytes=559362400 weight_movement_bytes=330301440 total_nvlink_bytes=889663840 weight_to_token_ratio=0.590496 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060660368e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44472 payload_rows_remote=38984 dup_rows=21064 dup_rows_remote=18570 dedup_ratio=0.321411 router_tokens_per_expert=[511, 494, 494, 521, 542, 541, 491, 542, 518, 525, 525, 490, 507, 515, 514, 525, 546, 513, 531, 478, 495, 524, 506, 501, 519, 508, 537, 526, 500, 514, 499, 512, 511, 519, 545, 515, 544, 517, 513, 510, 467, 544, 504, 484, 502, 489, 487, 514, 514, 495, 509, 501, 520, 480, 512, 500, 515, 490, 481, 516, 538, 517, 536, 515] payload_rows_per_dst=[5636, 5576, 5520, 5576, 5602, 5488, 5520, 5554] dup_rows_per_dst=[2636, 2662, 2668, 2654, 2746, 2494, 2542, 2662] combine_rows_per_src=[5636, 5576, 5520, 5576, 5602, 5488, 5520, 5554] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44404 payload_rows_remote=38880 dup_rows=21132 dup_rows_remote=18522 dedup_ratio=0.322449 router_tokens_per_expert=[532, 512, 527, 505, 528, 476, 531, 521, 531, 545, 521, 525, 515, 496, 499, 476, 541, 499, 478, 562, 494, 507, 499, 523, 480, 539, 484, 520, 521, 500, 517, 492, 526, 529, 490, 494, 491, 508, 509, 524, 507, 504, 525, 507, 518, 500, 532, 509, 533, 541, 520, 520, 489, 481, 524, 526, 480, 496, 531, 537, 516, 485, 499, 521] payload_rows_per_dst=[5480, 5472, 5594, 5506, 5496, 5714, 5618, 5524] dup_rows_per_dst=[2338, 2626, 2612, 2602, 2682, 2936, 2726, 2610] combine_rows_per_src=[5480, 5472, 5594, 5506, 5496, 5714, 5618, 5524] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44410 payload_rows_remote=38850 dup_rows=21126 dup_rows_remote=18594 dedup_ratio=0.322357 router_tokens_per_expert=[516, 519, 533, 484, 541, 533, 532, 527, 531, 509, 501, 501, 549, 489, 481, 529, 511, 502, 486, 515, 521, 487, 500, 490, 494, 477, 535, 518, 507, 533, 461, 521, 498, 533, 505, 538, 471, 481, 507, 494, 512, 543, 531, 507, 483, 529, 479, 500, 546, 486, 522, 526, 500, 508, 534, 504, 521, 527, 526, 500, 533, 529, 514, 548] payload_rows_per_dst=[5716, 5508, 5506, 5560, 5474, 5514, 5556, 5576] dup_rows_per_dst=[2654, 2672, 2518, 2532, 2580, 2654, 2696, 2820] combine_rows_per_src=[5716, 5508, 5506, 5560, 5474, 5514, 5556, 5576] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44320 payload_rows_remote=38758 dup_rows=21216 dup_rows_remote=18596 dedup_ratio=0.323730 router_tokens_per_expert=[484, 500, 515, 518, 526, 492, 511, 508, 476, 543, 520, 504, 528, 515, 504, 549, 518, 500, 536, 485, 520, 532, 496, 504, 492, 528, 534, 505, 525, 490, 509, 516, 519, 544, 514, 517, 545, 530, 519, 514, 479, 497, 521, 527, 503, 505, 498, 487, 505, 455, 527, 502, 510, 561, 508, 493, 505, 515, 540, 514, 471, 503, 528, 529] payload_rows_per_dst=[5446, 5608, 5562, 5576, 5628, 5442, 5496, 5562] dup_rows_per_dst=[2662, 2670, 2620, 2622, 2776, 2592, 2626, 2648] combine_rows_per_src=[5446, 5608, 5562, 5576, 5628, 5442, 5496, 5562] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279437312 dispatch_meta_bytes=230216 combine_payload_bytes=279437312 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279667528 ingress_bytes=389537792 link_bytes=389537792 token_bytes=559104840 weight_movement_bytes=110100480 total_nvlink_bytes=669205320 weight_to_token_ratio=0.196923 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.774702882e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44560 payload_rows_remote=39032 dup_rows=20976 dup_rows_remote=18396 dedup_ratio=0.320068 router_tokens_per_expert=[555, 521, 545, 553, 510, 501, 513, 497, 539, 472, 519, 489, 498, 527, 525, 522, 529, 526, 504, 526, 516, 501, 518, 515, 524, 470, 485, 529, 515, 523, 545, 543, 501, 512, 498, 520, 516, 496, 519, 492, 502, 491, 530, 527, 488, 502, 518, 499, 515, 503, 529, 524, 513, 536, 444, 480, 522, 521, 494, 468, 478, 571, 500, 504] payload_rows_per_dst=[5678, 5560, 5638, 5682, 5528, 5496, 5454, 5524] dup_rows_per_dst=[2712, 2622, 2632, 2586, 2580, 2618, 2634, 2592] combine_rows_per_src=[5678, 5560, 5638, 5682, 5528, 5496, 5454, 5524] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44444 payload_rows_remote=38782 dup_rows=21092 dup_rows_remote=18402 dedup_ratio=0.321838 router_tokens_per_expert=[533, 509, 504, 503, 484, 514, 496, 516, 479, 490, 492, 509, 503, 555, 523, 536, 522, 483, 493, 466, 517, 532, 522, 504, 506, 511, 527, 531, 526, 510, 528, 503, 538, 530, 522, 471, 475, 512, 510, 494, 534, 479, 519, 521, 518, 472, 544, 519, 502, 516, 565, 516, 526, 535, 513, 503, 509, 474, 535, 496, 548, 528, 503, 514] payload_rows_per_dst=[5550, 5528, 5520, 5582, 5528, 5538, 5662, 5536] dup_rows_per_dst=[2568, 2646, 2558, 2702, 2576, 2674, 2690, 2678] combine_rows_per_src=[5550, 5528, 5520, 5582, 5528, 5538, 5662, 5536] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44366 payload_rows_remote=38826 dup_rows=21170 dup_rows_remote=18388 dedup_ratio=0.323029 router_tokens_per_expert=[519, 525, 528, 500, 527, 509, 520, 506, 501, 511, 511, 514, 500, 517, 532, 527, 537, 499, 513, 494, 533, 480, 487, 509, 491, 537, 528, 477, 487, 554, 520, 478, 536, 514, 464, 467, 523, 502, 516, 510, 527, 518, 514, 540, 468, 540, 523, 532, 487, 552, 541, 521, 498, 529, 517, 501, 528, 499, 517, 479, 486, 547, 506, 495] payload_rows_per_dst=[5540, 5640, 5550, 5578, 5526, 5572, 5466, 5494] dup_rows_per_dst=[2782, 2586, 2756, 2512, 2538, 2752, 2624, 2620] combine_rows_per_src=[5540, 5640, 5550, 5578, 5526, 5572, 5466, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44558 payload_rows_remote=38960 dup_rows=20978 dup_rows_remote=18278 dedup_ratio=0.320099 router_tokens_per_expert=[516, 535, 495, 497, 541, 491, 503, 545, 500, 529, 532, 519, 508, 524, 506, 513, 454, 523, 516, 539, 527, 509, 484, 539, 539, 513, 485, 487, 523, 508, 505, 514, 502, 513, 505, 521, 540, 528, 540, 496, 485, 544, 507, 518, 497, 550, 493, 555, 508, 477, 513, 496, 511, 534, 487, 509, 508, 495, 522, 500, 484, 490, 538, 483] payload_rows_per_dst=[5654, 5626, 5542, 5548, 5576, 5598, 5546, 5468] dup_rows_per_dst=[2592, 2636, 2640, 2600, 2714, 2700, 2524, 2572] combine_rows_per_src=[5654, 5626, 5542, 5548, 5576, 5598, 5546, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44506 payload_rows_remote=38946 dup_rows=21030 dup_rows_remote=18416 dedup_ratio=0.320892 router_tokens_per_expert=[509, 502, 526, 494, 542, 503, 485, 492, 521, 522, 505, 483, 487, 493, 523, 499, 516, 519, 524, 510, 518, 498, 535, 508, 535, 510, 500, 523, 527, 498, 548, 516, 508, 486, 502, 494, 517, 492, 534, 506, 510, 552, 501, 530, 517, 510, 525, 488, 539, 501, 508, 488, 557, 496, 503, 495, 561, 479, 516, 521, 497, 526, 517, 521] payload_rows_per_dst=[5580, 5502, 5550, 5618, 5506, 5616, 5560, 5574] dup_rows_per_dst=[2526, 2564, 2706, 2696, 2572, 2650, 2614, 2702] combine_rows_per_src=[5580, 5502, 5550, 5618, 5506, 5616, 5560, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278605824 dispatch_meta_bytes=229272 combine_payload_bytes=278605824 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278835096 ingress_bytes=388706304 link_bytes=388706304 token_bytes=557440920 weight_movement_bytes=110100480 total_nvlink_bytes=667541400 weight_to_token_ratio=0.197511 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.760241939e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277989376 dispatch_meta_bytes=227832 combine_payload_bytes=277989376 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278217208 ingress_bytes=388089856 link_bytes=388089856 token_bytes=556206584 weight_movement_bytes=110100480 total_nvlink_bytes=666307064 weight_to_token_ratio=0.197949 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.749520894e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=280283136 dispatch_meta_bytes=229592 combine_payload_bytes=280283136 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=280512728 ingress_bytes=390383616 link_bytes=390383616 token_bytes=560795864 weight_movement_bytes=110100480 total_nvlink_bytes=670896344 weight_to_token_ratio=0.196329 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.789413153e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44460 payload_rows_remote=38868 dup_rows=21076 dup_rows_remote=18424 dedup_ratio=0.321594 router_tokens_per_expert=[576, 536, 481, 460, 511, 506, 489, 526, 513, 552, 499, 474, 485, 514, 535, 500, 533, 500, 479, 537, 486, 480, 517, 514, 528, 526, 510, 513, 490, 542, 507, 491, 532, 493, 527, 508, 558, 485, 507, 505, 508, 531, 506, 509, 501, 537, 521, 530, 532, 519, 515, 537, 507, 526, 511, 519, 498, 478, 517, 488, 516, 516, 529, 492] payload_rows_per_dst=[5592, 5582, 5556, 5488, 5588, 5646, 5548, 5460] dup_rows_per_dst=[2652, 2562, 2574, 2726, 2642, 2640, 2672, 2608] combine_rows_per_src=[5592, 5582, 5556, 5488, 5588, 5646, 5548, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277946368 dispatch_meta_bytes=228720 combine_payload_bytes=277946368 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=278175088 ingress_bytes=718348288 link_bytes=718348288 token_bytes=556121456 weight_movement_bytes=440401920 total_nvlink_bytes=996523376 weight_to_token_ratio=0.791917 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.249325821e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277846016 dispatch_meta_bytes=229240 combine_payload_bytes=277846016 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278075256 ingress_bytes=387946496 link_bytes=387946496 token_bytes=555921272 weight_movement_bytes=110100480 total_nvlink_bytes=666021752 weight_to_token_ratio=0.198050 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.747027628e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279265280 dispatch_meta_bytes=228952 combine_payload_bytes=279265280 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279494232 ingress_bytes=389365760 link_bytes=389365760 token_bytes=558759512 weight_movement_bytes=110100480 total_nvlink_bytes=668859992 weight_to_token_ratio=0.197044 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.771710963e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279164928 dispatch_meta_bytes=229448 combine_payload_bytes=279164928 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279394376 ingress_bytes=609466368 link_bytes=609466368 token_bytes=558559304 weight_movement_bytes=330301440 total_nvlink_bytes=888860744 weight_to_token_ratio=0.591345 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059962254e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278605824 dispatch_meta_bytes=229168 combine_payload_bytes=278605824 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278834992 ingress_bytes=498806784 link_bytes=498806784 token_bytes=557440816 weight_movement_bytes=220200960 total_nvlink_bytes=777641776 weight_to_token_ratio=0.395021 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.675070370e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44368 payload_rows_remote=38806 dup_rows=21168 dup_rows_remote=18474 dedup_ratio=0.322998 router_tokens_per_expert=[540, 546, 496, 522, 463, 542, 549, 481, 512, 500, 495, 560, 532, 513, 548, 491, 515, 531, 499, 474, 525, 527, 496, 511, 546, 477, 513, 507, 524, 482, 542, 500, 528, 517, 504, 488, 573, 505, 504, 509, 489, 503, 487, 527, 509, 516, 511, 501, 523, 553, 466, 480, 523, 501, 555, 494, 505, 501, 529, 491, 506, 509, 513, 489] payload_rows_per_dst=[5588, 5640, 5504, 5550, 5562, 5504, 5542, 5478] dup_rows_per_dst=[2690, 2662, 2652, 2632, 2694, 2582, 2648, 2608] combine_rows_per_src=[5588, 5640, 5504, 5550, 5562, 5504, 5542, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44372 payload_rows_remote=38728 dup_rows=21164 dup_rows_remote=18464 dedup_ratio=0.322937 router_tokens_per_expert=[489, 490, 507, 501, 494, 519, 479, 514, 565, 482, 527, 498, 508, 466, 551, 565, 471, 535, 507, 506, 538, 516, 494, 530, 524, 484, 525, 503, 493, 514, 532, 529, 506, 514, 525, 499, 496, 531, 516, 532, 505, 500, 538, 549, 501, 523, 495, 515, 494, 503, 499, 479, 477, 496, 508, 539, 505, 515, 530, 510, 507, 549, 513, 543] payload_rows_per_dst=[5496, 5578, 5472, 5556, 5602, 5592, 5432, 5644] dup_rows_per_dst=[2490, 2746, 2722, 2652, 2636, 2660, 2558, 2700] combine_rows_per_src=[5496, 5578, 5472, 5556, 5602, 5592, 5432, 5644] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44558 payload_rows_remote=39016 dup_rows=20978 dup_rows_remote=18338 dedup_ratio=0.320099 router_tokens_per_expert=[489, 528, 507, 505, 522, 506, 554, 495, 529, 551, 509, 493, 524, 515, 508, 462, 542, 486, 528, 518, 495, 491, 481, 499, 560, 521, 511, 498, 511, 514, 509, 507, 506, 519, 522, 546, 473, 480, 522, 495, 508, 509, 481, 521, 533, 503, 556, 527, 513, 526, 544, 473, 524, 530, 524, 497, 512, 490, 516, 535, 500, 520, 518, 477] payload_rows_per_dst=[5636, 5542, 5542, 5550, 5544, 5574, 5642, 5528] dup_rows_per_dst=[2576, 2640, 2538, 2712, 2582, 2702, 2620, 2608] combine_rows_per_src=[5636, 5542, 5542, 5550, 5544, 5574, 5642, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44772 payload_rows_remote=39206 dup_rows=20764 dup_rows_remote=18334 dedup_ratio=0.316833 router_tokens_per_expert=[488, 492, 529, 527, 550, 495, 470, 495, 502, 529, 490, 522, 488, 527, 505, 488, 475, 497, 490, 504, 501, 518, 520, 493, 531, 514, 543, 511, 510, 497, 492, 512, 528, 497, 506, 584, 505, 542, 530, 517, 540, 500, 492, 494, 537, 507, 538, 518, 511, 534, 545, 515, 532, 529, 498, 485, 521, 516, 509, 482, 474, 510, 515, 552] payload_rows_per_dst=[5544, 5554, 5566, 5592, 5670, 5584, 5648, 5614] dup_rows_per_dst=[2548, 2548, 2430, 2628, 2748, 2668, 2650, 2544] combine_rows_per_src=[5544, 5554, 5566, 5592, 5670, 5584, 5648, 5614] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44748 payload_rows_remote=39078 dup_rows=20788 dup_rows_remote=18150 dedup_ratio=0.317200 router_tokens_per_expert=[496, 514, 512, 534, 490, 518, 516, 499, 478, 503, 475, 524, 510, 536, 503, 522, 510, 491, 481, 500, 479, 536, 527, 503, 511, 532, 489, 511, 525, 520, 536, 530, 522, 513, 497, 513, 492, 525, 467, 516, 524, 552, 481, 525, 523, 508, 491, 507, 504, 512, 490, 492, 575, 536, 555, 499, 522, 556, 516, 552, 484, 507, 496, 505] payload_rows_per_dst=[5548, 5636, 5524, 5670, 5556, 5618, 5568, 5628] dup_rows_per_dst=[2610, 2466, 2530, 2638, 2534, 2604, 2758, 2648] combine_rows_per_src=[5548, 5636, 5524, 5670, 5556, 5618, 5568, 5628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44484 payload_rows_remote=38974 dup_rows=21052 dup_rows_remote=18494 dedup_ratio=0.321228 router_tokens_per_expert=[513, 527, 487, 506, 482, 503, 513, 501, 531, 509, 488, 508, 543, 510, 512, 511, 500, 533, 497, 499, 527, 501, 541, 511, 529, 516, 522, 532, 531, 503, 518, 521, 489, 495, 515, 517, 474, 505, 553, 500, 499, 487, 480, 504, 512, 493, 526, 533, 531, 503, 526, 533, 513, 516, 493, 529, 515, 517, 495, 501, 544, 510, 507, 528] payload_rows_per_dst=[5518, 5568, 5530, 5586, 5588, 5510, 5652, 5532] dup_rows_per_dst=[2546, 2656, 2688, 2758, 2508, 2558, 2636, 2702] combine_rows_per_src=[5518, 5568, 5530, 5586, 5588, 5510, 5652, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44524 payload_rows_remote=39022 dup_rows=21012 dup_rows_remote=18308 dedup_ratio=0.320618 router_tokens_per_expert=[527, 501, 526, 489, 483, 518, 509, 571, 515, 513, 520, 480, 533, 522, 506, 487, 506, 530, 527, 529, 523, 494, 514, 510, 513, 485, 507, 517, 510, 509, 520, 467, 536, 526, 514, 502, 517, 499, 515, 490, 530, 500, 497, 521, 522, 530, 516, 500, 508, 502, 489, 499, 564, 508, 506, 527, 521, 509, 535, 502, 471, 492, 559, 500] payload_rows_per_dst=[5590, 5602, 5604, 5508, 5556, 5590, 5502, 5572] dup_rows_per_dst=[2658, 2550, 2662, 2548, 2642, 2642, 2704, 2606] combine_rows_per_src=[5590, 5602, 5604, 5508, 5556, 5590, 5502, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278161408 dispatch_meta_bytes=229120 combine_payload_bytes=278161408 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278390528 ingress_bytes=388261888 link_bytes=388261888 token_bytes=556551936 weight_movement_bytes=110100480 total_nvlink_bytes=666652416 weight_to_token_ratio=0.197826 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.752512813e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277602304 dispatch_meta_bytes=228768 combine_payload_bytes=277602304 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=277831072 ingress_bytes=828104704 link_bytes=828104704 token_bytes=555433376 weight_movement_bytes=550502400 total_nvlink_bytes=1105935776 weight_to_token_ratio=0.991122 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.440210280e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279666688 dispatch_meta_bytes=229416 combine_payload_bytes=279666688 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279896104 ingress_bytes=389767168 link_bytes=389767168 token_bytes=559562792 weight_movement_bytes=110100480 total_nvlink_bytes=669663272 weight_to_token_ratio=0.196762 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778692108e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44370 payload_rows_remote=38724 dup_rows=21166 dup_rows_remote=18314 dedup_ratio=0.322968 router_tokens_per_expert=[520, 526, 533, 487, 495, 491, 514, 524, 483, 519, 536, 496, 511, 517, 481, 529, 533, 488, 506, 497, 497, 544, 505, 506, 505, 516, 529, 487, 514, 518, 529, 536, 524, 477, 546, 525, 530, 468, 484, 492, 473, 504, 493, 499, 511, 502, 506, 516, 530, 515, 517, 538, 495, 538, 542, 474, 514, 542, 517, 528, 514, 518, 543, 521] payload_rows_per_dst=[5646, 5584, 5624, 5446, 5484, 5520, 5494, 5572] dup_rows_per_dst=[2852, 2564, 2948, 2510, 2630, 2628, 2482, 2552] combine_rows_per_src=[5646, 5584, 5624, 5446, 5484, 5520, 5494, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=281028608 dispatch_meta_bytes=230160 combine_payload_bytes=281028608 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281258768 ingress_bytes=391129088 link_bytes=391129088 token_bytes=562287376 weight_movement_bytes=110100480 total_nvlink_bytes=672387856 weight_to_token_ratio=0.195808 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.802378137e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=280111104 dispatch_meta_bytes=228912 combine_payload_bytes=280111104 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280340016 ingress_bytes=500312064 link_bytes=500312064 token_bytes=560451120 weight_movement_bytes=220200960 total_nvlink_bytes=780652080 weight_to_token_ratio=0.392899 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.701249665e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279365632 dispatch_meta_bytes=229872 combine_payload_bytes=279365632 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279595504 ingress_bytes=389466112 link_bytes=389466112 token_bytes=558961136 weight_movement_bytes=110100480 total_nvlink_bytes=669061616 weight_to_token_ratio=0.196973 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773456249e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279709696 dispatch_meta_bytes=229320 combine_payload_bytes=279709696 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279939016 ingress_bytes=499910656 link_bytes=499910656 token_bytes=559648712 weight_movement_bytes=220200960 total_nvlink_bytes=779849672 weight_to_token_ratio=0.393463 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.694268520e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277573632 dispatch_meta_bytes=228152 combine_payload_bytes=277573632 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277801784 ingress_bytes=387674112 link_bytes=387674112 token_bytes=555375416 weight_movement_bytes=110100480 total_nvlink_bytes=665475896 weight_to_token_ratio=0.198245 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.742290422e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44500 payload_rows_remote=38968 dup_rows=21036 dup_rows_remote=18404 dedup_ratio=0.320984 router_tokens_per_expert=[491, 500, 502, 498, 481, 526, 486, 543, 517, 461, 533, 538, 520, 476, 525, 537, 535, 502, 493, 523, 509, 510, 534, 508, 478, 526, 492, 471, 527, 512, 512, 520, 513, 522, 502, 498, 529, 524, 487, 507, 509, 529, 535, 512, 531, 496, 516, 492, 479, 536, 493, 533, 497, 494, 524, 521, 526, 519, 525, 533, 510, 535, 517, 538] payload_rows_per_dst=[5520, 5536, 5594, 5526, 5532, 5562, 5588, 5642] dup_rows_per_dst=[2534, 2678, 2634, 2550, 2632, 2678, 2566, 2764] combine_rows_per_src=[5520, 5536, 5594, 5526, 5532, 5562, 5588, 5642] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44436 payload_rows_remote=38912 dup_rows=21100 dup_rows_remote=18522 dedup_ratio=0.321960 router_tokens_per_expert=[519, 527, 522, 547, 505, 537, 456, 518, 516, 533, 505, 490, 503, 504, 519, 528, 529, 489, 503, 512, 537, 492, 525, 493, 524, 522, 506, 503, 500, 540, 504, 541, 484, 476, 544, 546, 520, 529, 532, 506, 501, 516, 528, 475, 508, 547, 497, 521, 497, 519, 485, 533, 507, 503, 499, 511, 534, 489, 538, 494, 467, 522, 497, 494] payload_rows_per_dst=[5634, 5514, 5532, 5602, 5554, 5556, 5520, 5524] dup_rows_per_dst=[2628, 2682, 2628, 2678, 2720, 2598, 2588, 2578] combine_rows_per_src=[5634, 5514, 5532, 5602, 5554, 5556, 5520, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44590 payload_rows_remote=39008 dup_rows=20946 dup_rows_remote=18454 dedup_ratio=0.319611 router_tokens_per_expert=[507, 533, 512, 533, 510, 495, 517, 505, 477, 467, 545, 488, 521, 508, 524, 507, 495, 514, 511, 529, 496, 493, 517, 484, 521, 508, 498, 497, 533, 550, 484, 564, 503, 502, 520, 528, 516, 530, 510, 540, 498, 484, 524, 549, 520, 501, 496, 499, 514, 529, 518, 523, 492, 482, 470, 509, 528, 498, 535, 508, 522, 536, 516, 525] payload_rows_per_dst=[5598, 5582, 5548, 5628, 5570, 5478, 5522, 5664] dup_rows_per_dst=[2626, 2492, 2530, 2682, 2728, 2664, 2552, 2672] combine_rows_per_src=[5598, 5582, 5548, 5628, 5570, 5478, 5522, 5664] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44470 payload_rows_remote=38918 dup_rows=21066 dup_rows_remote=18528 dedup_ratio=0.321442 router_tokens_per_expert=[523, 537, 483, 480, 506, 505, 557, 497, 505, 509, 528, 504, 516, 550, 527, 501, 483, 465, 522, 530, 519, 501, 522, 503, 522, 527, 553, 509, 539, 497, 500, 543, 495, 517, 487, 476, 497, 542, 522, 497, 484, 511, 510, 515, 472, 540, 523, 516, 496, 514, 555, 521, 534, 527, 480, 500, 510, 503, 506, 537, 522, 500, 486, 510] payload_rows_per_dst=[5520, 5710, 5552, 5616, 5504, 5504, 5546, 5518] dup_rows_per_dst=[2656, 2570, 2538, 2764, 2562, 2638, 2708, 2630] combine_rows_per_src=[5520, 5710, 5552, 5616, 5504, 5504, 5546, 5518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44304 payload_rows_remote=38798 dup_rows=21232 dup_rows_remote=18650 dedup_ratio=0.323975 router_tokens_per_expert=[477, 562, 495, 474, 509, 519, 497, 501, 509, 472, 535, 513, 501, 546, 523, 506, 460, 536, 526, 519, 513, 501, 514, 541, 517, 445, 504, 505, 545, 470, 535, 523, 537, 511, 514, 496, 508, 519, 478, 482, 553, 492, 527, 518, 547, 527, 514, 503, 517, 496, 539, 481, 500, 492, 524, 514, 560, 544, 525, 498, 524, 509, 515, 511] payload_rows_per_dst=[5438, 5532, 5524, 5506, 5524, 5574, 5542, 5664] dup_rows_per_dst=[2630, 2678, 2696, 2582, 2566, 2788, 2584, 2708] combine_rows_per_src=[5438, 5532, 5524, 5506, 5524, 5574, 5542, 5664] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44362 payload_rows_remote=38764 dup_rows=21174 dup_rows_remote=18572 dedup_ratio=0.323090 router_tokens_per_expert=[573, 479, 509, 525, 532, 478, 543, 513, 496, 498, 543, 522, 520, 491, 510, 501, 534, 498, 506, 524, 491, 535, 507, 497, 540, 545, 522, 481, 510, 488, 465, 516, 556, 512, 492, 528, 497, 538, 514, 517, 510, 539, 470, 504, 524, 517, 512, 524, 509, 491, 525, 489, 534, 537, 497, 501, 527, 517, 508, 478, 525, 499, 490, 495] payload_rows_per_dst=[5554, 5532, 5524, 5534, 5582, 5598, 5518, 5520] dup_rows_per_dst=[2750, 2630, 2660, 2600, 2726, 2602, 2648, 2558] combine_rows_per_src=[5554, 5532, 5524, 5534, 5582, 5598, 5518, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278921216 dispatch_meta_bytes=229736 combine_payload_bytes=278921216 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279150952 ingress_bytes=499122176 link_bytes=499122176 token_bytes=558072168 weight_movement_bytes=220200960 total_nvlink_bytes=778273128 weight_to_token_ratio=0.394574 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.680555556e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44612 payload_rows_remote=39036 dup_rows=20924 dup_rows_remote=18168 dedup_ratio=0.319275 router_tokens_per_expert=[527, 519, 517, 482, 489, 506, 517, 552, 518, 493, 525, 513, 519, 497, 493, 518, 483, 503, 515, 521, 489, 504, 505, 474, 527, 497, 525, 547, 525, 517, 523, 515, 515, 536, 521, 510, 511, 537, 476, 522, 517, 524, 543, 491, 479, 518, 523, 514, 493, 517, 543, 516, 472, 528, 525, 519, 504, 475, 514, 489, 501, 522, 533, 525] payload_rows_per_dst=[5576, 5578, 5662, 5526, 5486, 5574, 5724, 5486] dup_rows_per_dst=[2756, 2584, 2680, 2560, 2582, 2620, 2730, 2412] combine_rows_per_src=[5576, 5578, 5662, 5526, 5486, 5574, 5724, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279322624 dispatch_meta_bytes=229488 combine_payload_bytes=279322624 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279552112 ingress_bytes=609624064 link_bytes=609624064 token_bytes=558874736 weight_movement_bytes=330301440 total_nvlink_bytes=889176176 weight_to_token_ratio=0.591012 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060236513e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44574 payload_rows_remote=38990 dup_rows=20962 dup_rows_remote=18346 dedup_ratio=0.319855 router_tokens_per_expert=[492, 504, 520, 487, 511, 514, 515, 515, 521, 559, 499, 527, 503, 499, 512, 499, 474, 516, 501, 531, 491, 522, 535, 547, 531, 497, 507, 523, 504, 489, 535, 505, 541, 491, 518, 533, 495, 484, 526, 546, 548, 521, 495, 476, 489, 515, 511, 496, 539, 552, 523, 534, 478, 502, 481, 491, 522, 545, 478, 495, 525, 540, 507, 486] payload_rows_per_dst=[5482, 5576, 5542, 5570, 5618, 5600, 5584, 5602] dup_rows_per_dst=[2634, 2662, 2692, 2612, 2650, 2502, 2616, 2594] combine_rows_per_src=[5482, 5576, 5542, 5570, 5618, 5600, 5584, 5602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279609344 dispatch_meta_bytes=229848 combine_payload_bytes=279609344 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279839192 ingress_bytes=389709824 link_bytes=389709824 token_bytes=559448536 weight_movement_bytes=110100480 total_nvlink_bytes=669549016 weight_to_token_ratio=0.196802 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.777694802e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278964224 dispatch_meta_bytes=229784 combine_payload_bytes=278964224 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279194008 ingress_bytes=389064704 link_bytes=389064704 token_bytes=558158232 weight_movement_bytes=110100480 total_nvlink_bytes=668258712 weight_to_token_ratio=0.197257 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.766475104e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278104064 dispatch_meta_bytes=229792 combine_payload_bytes=278104064 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278333856 ingress_bytes=498305024 link_bytes=498305024 token_bytes=556437920 weight_movement_bytes=220200960 total_nvlink_bytes=776638880 weight_to_token_ratio=0.395733 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.666343938e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277860352 dispatch_meta_bytes=229344 combine_payload_bytes=277860352 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278089696 ingress_bytes=608161792 link_bytes=608161792 token_bytes=555950048 weight_movement_bytes=330301440 total_nvlink_bytes=886251488 weight_to_token_ratio=0.594121 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057693382e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279810048 dispatch_meta_bytes=228816 combine_payload_bytes=279810048 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280038864 ingress_bytes=389910528 link_bytes=389910528 token_bytes=559848912 weight_movement_bytes=110100480 total_nvlink_bytes=669949392 weight_to_token_ratio=0.196661 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.781185375e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279480320 dispatch_meta_bytes=229344 combine_payload_bytes=279480320 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279709664 ingress_bytes=389580800 link_bytes=389580800 token_bytes=559189984 weight_movement_bytes=110100480 total_nvlink_bytes=669290464 weight_to_token_ratio=0.196893 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.775450862e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44502 payload_rows_remote=38958 dup_rows=21034 dup_rows_remote=18396 dedup_ratio=0.320953 router_tokens_per_expert=[531, 510, 527, 502, 527, 472, 528, 519, 528, 541, 518, 523, 512, 491, 496, 473, 539, 498, 478, 562, 494, 508, 499, 522, 477, 540, 485, 520, 522, 501, 515, 494, 525, 525, 488, 493, 495, 507, 512, 523, 507, 504, 528, 507, 519, 501, 531, 513, 538, 542, 520, 524, 493, 485, 525, 527, 481, 496, 535, 538, 521, 488, 504, 521] payload_rows_per_dst=[5602, 5588, 5586, 5496, 5494, 5580, 5612, 5544] dup_rows_per_dst=[2492, 2714, 2614, 2586, 2654, 2640, 2696, 2638] combine_rows_per_src=[5602, 5588, 5586, 5496, 5494, 5580, 5612, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44578 payload_rows_remote=39052 dup_rows=20958 dup_rows_remote=18378 dedup_ratio=0.319794 router_tokens_per_expert=[552, 521, 540, 552, 506, 498, 512, 497, 537, 472, 518, 487, 497, 526, 522, 519, 527, 524, 501, 524, 516, 501, 515, 513, 523, 470, 484, 532, 516, 524, 546, 545, 502, 511, 498, 519, 519, 494, 518, 492, 505, 493, 531, 531, 490, 503, 520, 499, 516, 500, 528, 527, 512, 537, 447, 484, 524, 520, 498, 470, 483, 572, 501, 507] payload_rows_per_dst=[5668, 5548, 5630, 5692, 5526, 5512, 5464, 5538] dup_rows_per_dst=[2688, 2608, 2612, 2588, 2580, 2632, 2638, 2612] combine_rows_per_src=[5668, 5548, 5630, 5692, 5526, 5512, 5464, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44382 payload_rows_remote=38854 dup_rows=21154 dup_rows_remote=18552 dedup_ratio=0.322784 router_tokens_per_expert=[493, 520, 542, 510, 495, 524, 469, 497, 495, 537, 506, 493, 503, 526, 512, 493, 508, 470, 539, 529, 526, 531, 536, 494, 526, 519, 494, 497, 540, 502, 543, 503, 508, 512, 527, 521, 541, 499, 526, 509, 540, 524, 521, 525, 454, 472, 506, 485, 520, 524, 510, 518, 501, 511, 531, 521, 509, 478, 545, 503, 501, 522, 499, 533] payload_rows_per_dst=[5462, 5528, 5626, 5542, 5590, 5532, 5556, 5546] dup_rows_per_dst=[2638, 2602, 2640, 2706, 2696, 2522, 2716, 2634] combine_rows_per_src=[5462, 5528, 5626, 5542, 5590, 5532, 5556, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44394 payload_rows_remote=38838 dup_rows=21142 dup_rows_remote=18604 dedup_ratio=0.322601 router_tokens_per_expert=[515, 514, 530, 481, 541, 530, 528, 525, 527, 509, 499, 497, 546, 489, 481, 527, 508, 500, 481, 512, 521, 489, 498, 490, 494, 477, 536, 516, 507, 532, 462, 523, 498, 534, 506, 539, 470, 482, 507, 494, 511, 545, 531, 509, 483, 531, 480, 498, 547, 487, 525, 529, 503, 508, 542, 505, 523, 529, 526, 500, 535, 533, 520, 553] payload_rows_per_dst=[5696, 5490, 5498, 5556, 5476, 5522, 5566, 5590] dup_rows_per_dst=[2632, 2660, 2500, 2538, 2584, 2654, 2726, 2848] combine_rows_per_src=[5696, 5490, 5498, 5556, 5476, 5522, 5566, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44308 payload_rows_remote=38760 dup_rows=21228 dup_rows_remote=18612 dedup_ratio=0.323914 router_tokens_per_expert=[482, 498, 513, 516, 524, 491, 510, 507, 474, 542, 519, 502, 525, 515, 501, 547, 514, 499, 540, 482, 522, 528, 496, 501, 491, 528, 535, 507, 524, 486, 508, 518, 521, 545, 513, 517, 547, 533, 519, 514, 481, 497, 523, 527, 504, 505, 499, 487, 505, 456, 530, 503, 514, 562, 509, 497, 507, 518, 541, 514, 472, 505, 529, 529] payload_rows_per_dst=[5434, 5592, 5548, 5574, 5632, 5452, 5512, 5564] dup_rows_per_dst=[2648, 2658, 2616, 2620, 2786, 2594, 2640, 2666] combine_rows_per_src=[5434, 5592, 5548, 5574, 5632, 5452, 5512, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44480 payload_rows_remote=38978 dup_rows=21056 dup_rows_remote=18552 dedup_ratio=0.321289 router_tokens_per_expert=[506, 494, 492, 518, 540, 537, 488, 541, 515, 524, 524, 486, 504, 514, 512, 523, 544, 513, 532, 478, 495, 522, 503, 501, 514, 507, 537, 526, 496, 514, 499, 513, 509, 520, 545, 513, 544, 519, 511, 510, 468, 543, 505, 484, 505, 496, 488, 514, 517, 495, 513, 503, 522, 483, 513, 503, 522, 491, 485, 519, 538, 519, 541, 518] payload_rows_per_dst=[5628, 5560, 5514, 5568, 5598, 5502, 5536, 5574] dup_rows_per_dst=[2604, 2644, 2662, 2644, 2744, 2504, 2562, 2692] combine_rows_per_src=[5628, 5560, 5514, 5568, 5598, 5502, 5536, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279250944 dispatch_meta_bytes=229416 combine_payload_bytes=279250944 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279480360 ingress_bytes=499451904 link_bytes=499451904 token_bytes=558731304 weight_movement_bytes=220200960 total_nvlink_bytes=778932264 weight_to_token_ratio=0.394109 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.686290068e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278476800 dispatch_meta_bytes=229776 combine_payload_bytes=278476800 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278706576 ingress_bytes=498677760 link_bytes=498677760 token_bytes=557183376 weight_movement_bytes=220200960 total_nvlink_bytes=777384336 weight_to_token_ratio=0.395204 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.672826430e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278691840 dispatch_meta_bytes=229608 combine_payload_bytes=278691840 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278921448 ingress_bytes=388792320 link_bytes=388792320 token_bytes=557613288 weight_movement_bytes=110100480 total_nvlink_bytes=667713768 weight_to_token_ratio=0.197450 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.761737898e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44402 payload_rows_remote=38854 dup_rows=21134 dup_rows_remote=18522 dedup_ratio=0.322479 router_tokens_per_expert=[493, 521, 544, 510, 498, 527, 471, 505, 495, 540, 510, 493, 503, 526, 516, 497, 508, 472, 537, 532, 526, 533, 537, 496, 528, 519, 499, 501, 541, 502, 540, 503, 506, 509, 526, 521, 540, 500, 525, 511, 540, 524, 520, 521, 453, 472, 508, 488, 517, 521, 507, 517, 496, 508, 530, 517, 509, 474, 544, 501, 497, 519, 495, 529] payload_rows_per_dst=[5474, 5548, 5632, 5550, 5590, 5528, 5548, 5532] dup_rows_per_dst=[2664, 2612, 2650, 2716, 2686, 2524, 2678, 2604] combine_rows_per_src=[5474, 5548, 5632, 5550, 5590, 5528, 5548, 5532] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277817344 dispatch_meta_bytes=229416 combine_payload_bytes=277817344 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278046760 ingress_bytes=387917824 link_bytes=387917824 token_bytes=555864104 weight_movement_bytes=110100480 total_nvlink_bytes=665964584 weight_to_token_ratio=0.198071 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.746528975e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279781376 dispatch_meta_bytes=229712 combine_payload_bytes=279781376 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280011088 ingress_bytes=389881856 link_bytes=389881856 token_bytes=559792464 weight_movement_bytes=110100480 total_nvlink_bytes=669892944 weight_to_token_ratio=0.196681 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.780686721e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277989376 dispatch_meta_bytes=228736 combine_payload_bytes=277989376 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278218112 ingress_bytes=498190336 link_bytes=498190336 token_bytes=556207488 weight_movement_bytes=220200960 total_nvlink_bytes=776408448 weight_to_token_ratio=0.395897 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.664349325e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278304768 dispatch_meta_bytes=228856 combine_payload_bytes=278304768 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278533624 ingress_bytes=498505728 link_bytes=498505728 token_bytes=556838392 weight_movement_bytes=220200960 total_nvlink_bytes=777039352 weight_to_token_ratio=0.395449 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.669834511e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278505472 dispatch_meta_bytes=229504 combine_payload_bytes=278505472 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=278734976 ingress_bytes=718907392 link_bytes=718907392 token_bytes=557240448 weight_movement_bytes=440401920 total_nvlink_bytes=997642368 weight_to_token_ratio=0.790327 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.250298195e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44620 payload_rows_remote=39106 dup_rows=20916 dup_rows_remote=18348 dedup_ratio=0.319153 router_tokens_per_expert=[530, 487, 518, 540, 554, 478, 546, 523, 510, 516, 501, 520, 518, 524, 508, 515, 506, 515, 521, 490, 515, 452, 519, 512, 511, 491, 486, 548, 526, 504, 527, 484, 485, 550, 516, 502, 523, 520, 559, 519, 473, 508, 513, 489, 497, 518, 527, 516, 524, 509, 523, 468, 509, 513, 489, 511, 481, 520, 523, 530, 519, 525, 493, 521] payload_rows_per_dst=[5642, 5610, 5552, 5548, 5624, 5514, 5530, 5600] dup_rows_per_dst=[2710, 2614, 2508, 2606, 2724, 2568, 2562, 2624] combine_rows_per_src=[5642, 5610, 5552, 5548, 5624, 5514, 5530, 5600] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44318 payload_rows_remote=38804 dup_rows=21218 dup_rows_remote=18466 dedup_ratio=0.323761 router_tokens_per_expert=[512, 523, 527, 498, 527, 505, 515, 502, 495, 509, 511, 514, 499, 516, 532, 526, 537, 498, 512, 493, 527, 478, 485, 508, 487, 539, 526, 477, 487, 554, 520, 478, 536, 511, 467, 468, 525, 503, 517, 510, 528, 522, 513, 544, 469, 545, 523, 532, 490, 557, 544, 522, 503, 528, 518, 503, 530, 500, 521, 479, 487, 551, 506, 499] payload_rows_per_dst=[5514, 5630, 5590, 5604, 5530, 5696, 5398, 5356] dup_rows_per_dst=[2752, 2574, 2834, 2534, 2544, 2978, 2534, 2468] combine_rows_per_src=[5514, 5630, 5590, 5604, 5530, 5696, 5398, 5356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279924736 dispatch_meta_bytes=229720 combine_payload_bytes=279924736 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280154456 ingress_bytes=390025216 link_bytes=390025216 token_bytes=560079192 weight_movement_bytes=110100480 total_nvlink_bytes=670179672 weight_to_token_ratio=0.196580 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.783179987e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278505472 dispatch_meta_bytes=229624 combine_payload_bytes=278505472 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278735096 ingress_bytes=388605952 link_bytes=388605952 token_bytes=557240568 weight_movement_bytes=110100480 total_nvlink_bytes=667341048 weight_to_token_ratio=0.197582 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.758496652e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44446 payload_rows_remote=38768 dup_rows=21090 dup_rows_remote=18388 dedup_ratio=0.321808 router_tokens_per_expert=[531, 505, 502, 501, 481, 512, 493, 514, 479, 489, 489, 507, 501, 551, 521, 533, 520, 483, 491, 465, 516, 533, 522, 503, 506, 510, 527, 532, 527, 512, 527, 503, 534, 529, 520, 472, 479, 512, 511, 494, 533, 477, 521, 520, 517, 473, 545, 522, 503, 516, 567, 518, 529, 536, 516, 505, 512, 481, 536, 499, 550, 533, 504, 518] payload_rows_per_dst=[5526, 5520, 5514, 5578, 5526, 5540, 5678, 5564] dup_rows_per_dst=[2552, 2620, 2552, 2710, 2576, 2676, 2702, 2702] combine_rows_per_src=[5526, 5520, 5514, 5578, 5526, 5540, 5678, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278390784 dispatch_meta_bytes=229768 combine_payload_bytes=278390784 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278620552 ingress_bytes=608692224 link_bytes=608692224 token_bytes=557011336 weight_movement_bytes=330301440 total_nvlink_bytes=887312776 weight_to_token_ratio=0.592989 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058615890e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277831680 dispatch_meta_bytes=229488 combine_payload_bytes=277831680 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278061168 ingress_bytes=387932160 link_bytes=387932160 token_bytes=555892848 weight_movement_bytes=110100480 total_nvlink_bytes=665993328 weight_to_token_ratio=0.198061 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.746778301e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279394304 dispatch_meta_bytes=230120 combine_payload_bytes=279394304 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279624424 ingress_bytes=389494784 link_bytes=389494784 token_bytes=559018728 weight_movement_bytes=110100480 total_nvlink_bytes=669119208 weight_to_token_ratio=0.196953 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773954903e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278147072 dispatch_meta_bytes=229080 combine_payload_bytes=278147072 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278376152 ingress_bytes=498348032 link_bytes=498348032 token_bytes=556523224 weight_movement_bytes=220200960 total_nvlink_bytes=776724184 weight_to_token_ratio=0.395673 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.667091918e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277889024 dispatch_meta_bytes=228624 combine_payload_bytes=277889024 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278117648 ingress_bytes=608190464 link_bytes=608190464 token_bytes=556006672 weight_movement_bytes=330301440 total_nvlink_bytes=886308112 weight_to_token_ratio=0.594060 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057743247e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44280 payload_rows_remote=38742 dup_rows=21256 dup_rows_remote=18428 dedup_ratio=0.324341 router_tokens_per_expert=[508, 557, 533, 591, 502, 525, 511, 528, 520, 480, 547, 472, 518, 515, 492, 550, 495, 528, 510, 560, 502, 537, 500, 519, 531, 506, 480, 490, 513, 511, 515, 528, 521, 474, 537, 523, 520, 524, 504, 491, 472, 502, 500, 516, 507, 517, 513, 543, 508, 476, 476, 483, 486, 490, 507, 476, 535, 477, 495, 528, 532, 520, 543, 498] payload_rows_per_dst=[5482, 5604, 5598, 5438, 5432, 5610, 5578, 5538] dup_rows_per_dst=[2446, 2584, 2768, 2600, 2496, 2790, 2744, 2828] combine_rows_per_src=[5482, 5604, 5598, 5438, 5432, 5610, 5578, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44468 payload_rows_remote=38974 dup_rows=21068 dup_rows_remote=18480 dedup_ratio=0.321472 router_tokens_per_expert=[507, 508, 519, 506, 512, 489, 507, 500, 508, 471, 518, 509, 512, 483, 499, 509, 477, 479, 554, 525, 520, 521, 551, 507, 527, 549, 497, 467, 548, 518, 499, 523, 506, 531, 473, 532, 498, 502, 510, 489, 503, 521, 521, 478, 522, 535, 491, 506, 472, 510, 505, 525, 527, 531, 519, 524, 468, 510, 542, 559, 548, 551, 514, 526] payload_rows_per_dst=[5500, 5488, 5594, 5650, 5494, 5554, 5608, 5580] dup_rows_per_dst=[2596, 2530, 2674, 2606, 2588, 2600, 2618, 2856] combine_rows_per_src=[5500, 5488, 5594, 5650, 5494, 5554, 5608, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44522 payload_rows_remote=39034 dup_rows=21014 dup_rows_remote=18370 dedup_ratio=0.320648 router_tokens_per_expert=[493, 506, 511, 505, 528, 522, 534, 510, 504, 486, 532, 484, 539, 500, 511, 510, 515, 522, 513, 497, 538, 494, 518, 492, 529, 524, 516, 566, 499, 487, 505, 484, 525, 517, 524, 540, 571, 514, 533, 512, 535, 522, 503, 454, 536, 510, 513, 486, 536, 481, 500, 485, 470, 557, 553, 483, 532, 514, 521, 496, 497, 511, 485, 478] payload_rows_per_dst=[5562, 5488, 5592, 5582, 5674, 5548, 5542, 5534] dup_rows_per_dst=[2656, 2644, 2586, 2638, 2798, 2570, 2588, 2534] combine_rows_per_src=[5562, 5488, 5592, 5582, 5674, 5548, 5542, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44532 payload_rows_remote=39016 dup_rows=21004 dup_rows_remote=18394 dedup_ratio=0.320496 router_tokens_per_expert=[525, 508, 546, 499, 529, 539, 497, 494, 544, 545, 536, 500, 490, 516, 521, 497, 492, 500, 544, 511, 506, 476, 520, 510, 531, 506, 513, 473, 512, 514, 509, 505, 546, 534, 508, 530, 506, 497, 517, 510, 521, 505, 474, 552, 522, 534, 526, 501, 468, 497, 499, 530, 474, 521, 493, 506, 514, 522, 532, 553, 529, 503, 468, 468] payload_rows_per_dst=[5588, 5614, 5478, 5516, 5604, 5628, 5512, 5592] dup_rows_per_dst=[2686, 2684, 2640, 2610, 2692, 2642, 2464, 2586] combine_rows_per_src=[5588, 5614, 5478, 5516, 5604, 5628, 5512, 5592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44418 payload_rows_remote=38906 dup_rows=21118 dup_rows_remote=18586 dedup_ratio=0.322235 router_tokens_per_expert=[539, 522, 549, 490, 515, 464, 520, 495, 469, 513, 506, 518, 488, 543, 507, 549, 517, 506, 486, 496, 504, 508, 491, 514, 534, 505, 539, 536, 522, 549, 497, 516, 513, 519, 483, 546, 497, 458, 552, 525, 534, 518, 528, 505, 517, 526, 500, 509, 504, 512, 508, 511, 521, 563, 507, 503, 514, 514, 469, 519, 498, 474, 528, 486] payload_rows_per_dst=[5554, 5534, 5512, 5612, 5548, 5548, 5618, 5492] dup_rows_per_dst=[2634, 2652, 2532, 2784, 2638, 2726, 2640, 2512] combine_rows_per_src=[5554, 5534, 5512, 5612, 5548, 5548, 5618, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277702656 dispatch_meta_bytes=228680 combine_payload_bytes=277702656 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277931336 ingress_bytes=497903616 link_bytes=497903616 token_bytes=555633992 weight_movement_bytes=220200960 total_nvlink_bytes=775834952 weight_to_token_ratio=0.396306 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.659362793e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44632 payload_rows_remote=39110 dup_rows=20904 dup_rows_remote=18322 dedup_ratio=0.318970 router_tokens_per_expert=[525, 485, 514, 537, 552, 476, 544, 523, 508, 515, 504, 518, 518, 523, 503, 512, 505, 511, 519, 488, 517, 449, 520, 509, 510, 491, 487, 547, 527, 503, 527, 484, 484, 548, 518, 503, 523, 519, 560, 520, 476, 509, 516, 491, 498, 517, 529, 516, 525, 510, 524, 469, 511, 516, 489, 512, 483, 522, 525, 532, 526, 528, 496, 522] payload_rows_per_dst=[5632, 5602, 5546, 5548, 5622, 5522, 5542, 5618] dup_rows_per_dst=[2680, 2600, 2490, 2604, 2728, 2582, 2570, 2650] combine_rows_per_src=[5632, 5602, 5546, 5548, 5622, 5522, 5542, 5618] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44448 payload_rows_remote=38832 dup_rows=21088 dup_rows_remote=18300 dedup_ratio=0.321777 router_tokens_per_expert=[511, 525, 531, 540, 487, 527, 519, 556, 553, 525, 476, 484, 501, 500, 472, 468, 511, 527, 537, 529, 481, 508, 488, 503, 499, 547, 539, 517, 520, 524, 503, 481, 521, 494, 502, 522, 522, 477, 482, 498, 479, 535, 487, 550, 522, 521, 490, 480, 501, 546, 524, 530, 504, 481, 524, 521, 516, 542, 555, 493, 502, 498, 525, 535] payload_rows_per_dst=[5616, 5696, 5528, 5614, 5418, 5520, 5560, 5496] dup_rows_per_dst=[2788, 2642, 2640, 2636, 2616, 2608, 2702, 2456] combine_rows_per_src=[5616, 5696, 5528, 5614, 5418, 5520, 5560, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279365632 dispatch_meta_bytes=229816 combine_payload_bytes=279365632 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279595448 ingress_bytes=499566592 link_bytes=499566592 token_bytes=558961080 weight_movement_bytes=220200960 total_nvlink_bytes=779162040 weight_to_token_ratio=0.393947 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.688284681e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279795712 dispatch_meta_bytes=229616 combine_payload_bytes=279795712 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280025328 ingress_bytes=389896192 link_bytes=389896192 token_bytes=559821040 weight_movement_bytes=110100480 total_nvlink_bytes=669921520 weight_to_token_ratio=0.196671 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.780936048e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44544 payload_rows_remote=39030 dup_rows=20992 dup_rows_remote=18380 dedup_ratio=0.320312 router_tokens_per_expert=[529, 512, 548, 500, 530, 542, 503, 499, 548, 548, 536, 500, 491, 517, 523, 500, 493, 499, 545, 516, 506, 478, 518, 511, 528, 506, 514, 473, 514, 512, 510, 506, 547, 535, 508, 529, 505, 496, 515, 509, 523, 505, 476, 551, 522, 530, 531, 502, 466, 494, 498, 526, 472, 515, 489, 507, 510, 521, 527, 550, 524, 499, 464, 467] payload_rows_per_dst=[5614, 5632, 5478, 5514, 5604, 5636, 5494, 5572] dup_rows_per_dst=[2712, 2694, 2654, 2612, 2684, 2644, 2440, 2552] combine_rows_per_src=[5614, 5632, 5478, 5514, 5604, 5636, 5494, 5572] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44250 payload_rows_remote=38748 dup_rows=21286 dup_rows_remote=18492 dedup_ratio=0.324799 router_tokens_per_expert=[508, 560, 537, 594, 505, 525, 511, 529, 521, 481, 548, 475, 519, 517, 497, 554, 498, 529, 512, 562, 503, 537, 500, 520, 532, 506, 482, 489, 515, 511, 516, 526, 524, 476, 535, 522, 520, 523, 504, 489, 471, 500, 502, 519, 506, 514, 512, 542, 506, 473, 473, 484, 484, 489, 504, 476, 533, 472, 492, 527, 526, 518, 537, 496] payload_rows_per_dst=[5340, 5616, 5540, 5494, 5458, 5642, 5658, 5502] dup_rows_per_dst=[2282, 2608, 2688, 2666, 2518, 2854, 2876, 2794] combine_rows_per_src=[5340, 5616, 5540, 5494, 5458, 5642, 5658, 5502] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44406 payload_rows_remote=38892 dup_rows=21130 dup_rows_remote=18596 dedup_ratio=0.322418 router_tokens_per_expert=[542, 522, 552, 493, 519, 466, 523, 499, 471, 515, 507, 519, 493, 545, 512, 555, 518, 506, 485, 494, 507, 508, 492, 514, 536, 504, 540, 536, 523, 551, 499, 515, 513, 518, 482, 546, 495, 455, 553, 525, 535, 516, 529, 502, 513, 526, 500, 510, 503, 510, 506, 508, 518, 561, 505, 500, 514, 513, 464, 517, 493, 473, 524, 480] payload_rows_per_dst=[5568, 5550, 5514, 5618, 5540, 5548, 5602, 5466] dup_rows_per_dst=[2664, 2684, 2534, 2790, 2634, 2714, 2620, 2490] combine_rows_per_src=[5568, 5550, 5514, 5618, 5540, 5548, 5602, 5466] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280311808 dispatch_meta_bytes=229816 combine_payload_bytes=280311808 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280541624 ingress_bytes=390412288 link_bytes=390412288 token_bytes=560853432 weight_movement_bytes=110100480 total_nvlink_bytes=670953912 weight_to_token_ratio=0.196309 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.789911806e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44458 payload_rows_remote=38976 dup_rows=21078 dup_rows_remote=18486 dedup_ratio=0.321625 router_tokens_per_expert=[508, 512, 520, 508, 515, 492, 511, 504, 508, 472, 520, 511, 513, 483, 500, 509, 478, 478, 557, 526, 523, 520, 552, 506, 527, 553, 499, 465, 547, 520, 504, 521, 507, 531, 473, 533, 497, 500, 508, 488, 502, 520, 520, 478, 520, 533, 491, 504, 470, 508, 506, 523, 524, 529, 519, 522, 468, 509, 540, 556, 544, 548, 510, 525] payload_rows_per_dst=[5518, 5490, 5602, 5656, 5482, 5548, 5596, 5566] dup_rows_per_dst=[2622, 2542, 2678, 2616, 2592, 2588, 2606, 2834] combine_rows_per_src=[5518, 5490, 5602, 5656, 5482, 5548, 5596, 5566] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44416 payload_rows_remote=38790 dup_rows=21120 dup_rows_remote=18308 dedup_ratio=0.322266 router_tokens_per_expert=[515, 527, 534, 543, 490, 528, 522, 560, 558, 527, 477, 484, 506, 502, 474, 471, 514, 529, 541, 529, 479, 511, 488, 502, 499, 549, 539, 516, 525, 527, 505, 478, 523, 493, 501, 521, 523, 478, 482, 499, 478, 533, 484, 549, 517, 517, 490, 479, 500, 543, 520, 527, 504, 478, 522, 516, 513, 541, 554, 490, 498, 492, 523, 531] payload_rows_per_dst=[5626, 5624, 5540, 5540, 5418, 5498, 5552, 5618] dup_rows_per_dst=[2812, 2532, 2662, 2562, 2622, 2596, 2668, 2666] combine_rows_per_src=[5626, 5624, 5540, 5540, 5418, 5498, 5552, 5618] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44292 payload_rows_remote=38766 dup_rows=21244 dup_rows_remote=18548 dedup_ratio=0.324158 router_tokens_per_expert=[497, 505, 539, 529, 522, 507, 489, 492, 494, 541, 518, 494, 516, 506, 516, 511, 495, 512, 520, 532, 563, 547, 490, 534, 475, 525, 522, 526, 480, 507, 525, 486, 495, 501, 525, 516, 532, 533, 500, 488, 497, 506, 505, 520, 479, 507, 524, 502, 536, 545, 502, 468, 469, 498, 563, 530, 500, 528, 509, 512, 534, 506, 506, 517] payload_rows_per_dst=[5502, 5552, 5594, 5456, 5594, 5546, 5526, 5522] dup_rows_per_dst=[2658, 2640, 2792, 2636, 2586, 2534, 2696, 2702] combine_rows_per_src=[5502, 5552, 5594, 5456, 5594, 5546, 5526, 5522] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279767040 dispatch_meta_bytes=229640 combine_payload_bytes=279767040 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279996680 ingress_bytes=610068480 link_bytes=610068480 token_bytes=559763720 weight_movement_bytes=330301440 total_nvlink_bytes=890065160 weight_to_token_ratio=0.590073 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061009426e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44548 payload_rows_remote=39046 dup_rows=20988 dup_rows_remote=18342 dedup_ratio=0.320251 router_tokens_per_expert=[498, 511, 514, 510, 529, 523, 537, 511, 507, 487, 533, 484, 541, 502, 512, 508, 517, 524, 514, 498, 541, 497, 517, 492, 530, 524, 516, 564, 499, 486, 508, 484, 525, 514, 523, 538, 569, 511, 535, 513, 535, 523, 503, 453, 532, 508, 512, 487, 534, 479, 499, 484, 469, 556, 552, 482, 528, 514, 519, 493, 493, 508, 484, 475] payload_rows_per_dst=[5582, 5502, 5606, 5590, 5670, 5548, 5534, 5516] dup_rows_per_dst=[2684, 2646, 2594, 2632, 2786, 2558, 2576, 2512] combine_rows_per_src=[5582, 5502, 5606, 5590, 5670, 5548, 5534, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279666688 dispatch_meta_bytes=229640 combine_payload_bytes=279666688 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279896328 ingress_bytes=609968128 link_bytes=609968128 token_bytes=559563016 weight_movement_bytes=330301440 total_nvlink_bytes=889864456 weight_to_token_ratio=0.590285 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060834897e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278878208 dispatch_meta_bytes=229968 combine_payload_bytes=278878208 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279108176 ingress_bytes=388978688 link_bytes=388978688 token_bytes=557986384 weight_movement_bytes=110100480 total_nvlink_bytes=668086864 weight_to_token_ratio=0.197318 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.764979144e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44290 payload_rows_remote=38760 dup_rows=21246 dup_rows_remote=18526 dedup_ratio=0.324188 router_tokens_per_expert=[493, 501, 536, 527, 516, 505, 490, 490, 492, 537, 518, 494, 517, 505, 513, 511, 490, 511, 521, 531, 559, 544, 490, 533, 475, 524, 521, 530, 479, 507, 526, 487, 493, 499, 525, 517, 533, 535, 501, 491, 497, 506, 506, 519, 479, 509, 525, 503, 537, 545, 504, 469, 470, 504, 563, 533, 500, 531, 509, 512, 539, 510, 511, 520] payload_rows_per_dst=[5488, 5546, 5586, 5456, 5596, 5544, 5530, 5544] dup_rows_per_dst=[2628, 2628, 2772, 2642, 2592, 2544, 2720, 2720] combine_rows_per_src=[5488, 5546, 5586, 5456, 5596, 5544, 5530, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280340480 dispatch_meta_bytes=229728 combine_payload_bytes=280340480 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280570208 ingress_bytes=390440960 link_bytes=390440960 token_bytes=560910688 weight_movement_bytes=110100480 total_nvlink_bytes=671011168 weight_to_token_ratio=0.196289 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.790410459e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278347776 dispatch_meta_bytes=228528 combine_payload_bytes=278347776 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278576304 ingress_bytes=608649216 link_bytes=608649216 token_bytes=556924080 weight_movement_bytes=330301440 total_nvlink_bytes=887225520 weight_to_token_ratio=0.593082 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058541092e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277745664 dispatch_meta_bytes=228960 combine_payload_bytes=277745664 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277974624 ingress_bytes=387846144 link_bytes=387846144 token_bytes=555720288 weight_movement_bytes=110100480 total_nvlink_bytes=665820768 weight_to_token_ratio=0.198122 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.745282341e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278777856 dispatch_meta_bytes=229952 combine_payload_bytes=278777856 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279007808 ingress_bytes=498978816 link_bytes=498978816 token_bytes=557785664 weight_movement_bytes=220200960 total_nvlink_bytes=777986624 weight_to_token_ratio=0.394777 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.678062289e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279379968 dispatch_meta_bytes=229848 combine_payload_bytes=279379968 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279609816 ingress_bytes=499580928 link_bytes=499580928 token_bytes=558989784 weight_movement_bytes=220200960 total_nvlink_bytes=779190744 weight_to_token_ratio=0.393927 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.688534007e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278046720 dispatch_meta_bytes=228392 combine_payload_bytes=278046720 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278275112 ingress_bytes=608348160 link_bytes=608348160 token_bytes=556321832 weight_movement_bytes=330301440 total_nvlink_bytes=886623272 weight_to_token_ratio=0.593724 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058017506e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277874688 dispatch_meta_bytes=229256 combine_payload_bytes=277874688 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278103944 ingress_bytes=387975168 link_bytes=387975168 token_bytes=555978632 weight_movement_bytes=110100480 total_nvlink_bytes=666079112 weight_to_token_ratio=0.198030 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.747526281e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279881728 dispatch_meta_bytes=229552 combine_payload_bytes=279881728 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280111280 ingress_bytes=389982208 link_bytes=389982208 token_bytes=559993008 weight_movement_bytes=110100480 total_nvlink_bytes=670093488 weight_to_token_ratio=0.196610 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.782432008e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44414 payload_rows_remote=38898 dup_rows=21122 dup_rows_remote=18470 dedup_ratio=0.322296 router_tokens_per_expert=[508, 525, 515, 503, 528, 520, 500, 539, 534, 498, 472, 495, 519, 522, 538, 534, 566, 484, 501, 504, 517, 510, 515, 508, 522, 512, 498, 484, 521, 535, 516, 532, 531, 508, 532, 508, 519, 536, 479, 501, 513, 543, 535, 512, 481, 517, 473, 510, 503, 508, 494, 533, 494, 506, 509, 523, 519, 488, 484, 506, 513, 485, 519, 511] payload_rows_per_dst=[5616, 5560, 5524, 5532, 5604, 5516, 5584, 5478] dup_rows_per_dst=[2660, 2664, 2686, 2708, 2624, 2652, 2556, 2572] combine_rows_per_src=[5616, 5560, 5524, 5532, 5604, 5516, 5584, 5478] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44632 payload_rows_remote=39122 dup_rows=20904 dup_rows_remote=18232 dedup_ratio=0.318970 router_tokens_per_expert=[529, 518, 496, 479, 504, 532, 492, 522, 521, 513, 507, 500, 514, 525, 478, 511, 537, 532, 478, 583, 530, 479, 516, 508, 516, 506, 544, 522, 500, 486, 519, 498, 486, 509, 530, 558, 511, 517, 492, 499, 567, 501, 538, 510, 516, 498, 491, 469, 540, 513, 514, 506, 519, 524, 494, 494, 508, 487, 502, 534, 489, 512, 507, 538] payload_rows_per_dst=[5548, 5564, 5646, 5510, 5586, 5592, 5610, 5576] dup_rows_per_dst=[2596, 2574, 2680, 2672, 2618, 2588, 2598, 2578] combine_rows_per_src=[5548, 5564, 5646, 5510, 5586, 5592, 5610, 5576] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278820864 dispatch_meta_bytes=229472 combine_payload_bytes=278820864 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=279050336 ingress_bytes=388921344 link_bytes=388921344 token_bytes=557871200 weight_movement_bytes=110100480 total_nvlink_bytes=667971680 weight_to_token_ratio=0.197358 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.763981838e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44442 payload_rows_remote=38686 dup_rows=21094 dup_rows_remote=18122 dedup_ratio=0.321869 router_tokens_per_expert=[494, 507, 537, 511, 480, 509, 486, 490, 523, 517, 521, 520, 542, 513, 529, 496, 508, 500, 533, 484, 485, 506, 554, 518, 498, 525, 529, 515, 508, 503, 516, 521, 513, 487, 542, 528, 500, 503, 496, 504, 520, 542, 519, 535, 519, 520, 519, 463, 500, 477, 492, 489, 551, 505, 479, 518, 547, 515, 500, 505, 548, 554, 495, 505] payload_rows_per_dst=[5384, 5188, 5468, 5520, 5574, 5650, 5902, 5756] dup_rows_per_dst=[2358, 2154, 2444, 2710, 2732, 2624, 3100, 2972] combine_rows_per_src=[5384, 5188, 5468, 5520, 5574, 5650, 5902, 5756] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44664 payload_rows_remote=39064 dup_rows=20872 dup_rows_remote=18264 dedup_ratio=0.318481 router_tokens_per_expert=[520, 500, 495, 522, 539, 483, 514, 523, 531, 535, 516, 520, 496, 531, 513, 513, 543, 518, 499, 486, 491, 542, 521, 504, 512, 536, 519, 531, 509, 509, 514, 501, 526, 447, 549, 505, 539, 510, 504, 530, 531, 500, 509, 549, 502, 501, 511, 487, 512, 495, 497, 482, 499, 483, 463, 512, 525, 531, 554, 501, 553, 504, 466, 505] payload_rows_per_dst=[5526, 5650, 5600, 5612, 5616, 5602, 5454, 5604] dup_rows_per_dst=[2666, 2660, 2608, 2650, 2604, 2578, 2432, 2674] combine_rows_per_src=[5526, 5650, 5600, 5612, 5616, 5602, 5454, 5604] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44518 payload_rows_remote=38884 dup_rows=21018 dup_rows_remote=18454 dedup_ratio=0.320709 router_tokens_per_expert=[497, 542, 528, 542, 549, 491, 473, 549, 528, 512, 526, 476, 489, 499, 519, 512, 545, 517, 487, 493, 515, 528, 475, 541, 500, 484, 481, 534, 529, 538, 503, 449, 484, 498, 527, 500, 515, 515, 544, 516, 534, 534, 522, 514, 493, 509, 506, 539, 503, 529, 545, 482, 507, 549, 465, 520, 505, 526, 565, 485, 514, 495, 512, 465] payload_rows_per_dst=[5640, 5518, 5506, 5484, 5634, 5600, 5572, 5564] dup_rows_per_dst=[2702, 2604, 2696, 2552, 2564, 2702, 2628, 2570] combine_rows_per_src=[5640, 5518, 5506, 5484, 5634, 5600, 5572, 5564] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44570 payload_rows_remote=38998 dup_rows=20966 dup_rows_remote=18264 dedup_ratio=0.319916 router_tokens_per_expert=[509, 516, 488, 524, 515, 511, 532, 506, 529, 517, 542, 532, 483, 520, 553, 524, 518, 500, 513, 465, 509, 466, 528, 525, 513, 521, 525, 518, 498, 535, 507, 525, 515, 513, 517, 510, 520, 537, 538, 533, 521, 507, 531, 514, 522, 507, 516, 496, 498, 543, 497, 498, 487, 489, 512, 512, 495, 499, 497, 499, 478, 529, 469, 502] payload_rows_per_dst=[5572, 5820, 5512, 5610, 5446, 5600, 5548, 5462] dup_rows_per_dst=[2702, 3114, 2536, 2634, 2386, 2596, 2524, 2474] combine_rows_per_src=[5572, 5820, 5512, 5610, 5446, 5600, 5548, 5462] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44476 payload_rows_remote=38978 dup_rows=21060 dup_rows_remote=18452 dedup_ratio=0.321350 router_tokens_per_expert=[483, 553, 493, 489, 514, 478, 499, 530, 481, 518, 534, 576, 497, 514, 515, 481, 584, 489, 541, 548, 557, 529, 534, 491, 519, 499, 491, 513, 535, 530, 515, 527, 535, 523, 489, 507, 543, 509, 498, 497, 490, 475, 481, 519, 527, 507, 508, 500, 517, 482, 517, 528, 497, 492, 506, 514, 521, 485, 491, 525, 533, 483, 523, 489] payload_rows_per_dst=[5544, 5572, 5678, 5606, 5554, 5466, 5498, 5558] dup_rows_per_dst=[2534, 2660, 2868, 2652, 2648, 2548, 2608, 2542] combine_rows_per_src=[5544, 5572, 5678, 5606, 5554, 5466, 5498, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44456 payload_rows_remote=38790 dup_rows=21080 dup_rows_remote=18302 dedup_ratio=0.321655 router_tokens_per_expert=[491, 503, 535, 505, 479, 507, 483, 489, 518, 516, 519, 519, 540, 511, 528, 493, 507, 499, 534, 482, 485, 506, 553, 517, 497, 527, 529, 515, 508, 505, 517, 518, 512, 487, 542, 526, 502, 502, 493, 503, 523, 541, 523, 533, 523, 520, 520, 463, 502, 480, 498, 489, 554, 509, 480, 520, 550, 517, 502, 509, 552, 556, 497, 505] payload_rows_per_dst=[5472, 5440, 5550, 5520, 5358, 5656, 5794, 5666] dup_rows_per_dst=[2512, 2552, 2548, 2712, 2386, 2636, 2956, 2778] combine_rows_per_src=[5472, 5440, 5550, 5520, 5358, 5656, 5794, 5666] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277831680 dispatch_meta_bytes=229144 combine_payload_bytes=277831680 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278060824 ingress_bytes=387932160 link_bytes=387932160 token_bytes=555892504 weight_movement_bytes=110100480 total_nvlink_bytes=665992984 weight_to_token_ratio=0.198061 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.746778301e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=280426496 dispatch_meta_bytes=229416 combine_payload_bytes=280426496 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280655912 ingress_bytes=500627456 link_bytes=500627456 token_bytes=561082408 weight_movement_bytes=220200960 total_nvlink_bytes=781283368 weight_to_token_ratio=0.392457 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.706734851e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44572 payload_rows_remote=38980 dup_rows=20964 dup_rows_remote=18322 dedup_ratio=0.319885 router_tokens_per_expert=[578, 518, 513, 528, 525, 525, 525, 532, 521, 495, 484, 482, 540, 518, 537, 540, 505, 512, 516, 501, 504, 506, 479, 519, 522, 476, 494, 511, 528, 491, 527, 493, 535, 482, 508, 545, 549, 526, 510, 518, 532, 495, 523, 528, 535, 510, 499, 489, 460, 507, 537, 503, 485, 480, 505, 484, 509, 535, 509, 497, 479, 506, 548, 495] payload_rows_per_dst=[5762, 5592, 5498, 5480, 5552, 5626, 5480, 5582] dup_rows_per_dst=[2726, 2642, 2586, 2604, 2794, 2596, 2442, 2574] combine_rows_per_src=[5762, 5592, 5498, 5480, 5552, 5626, 5480, 5582] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=277301248 dispatch_meta_bytes=227232 combine_payload_bytes=277301248 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277528480 ingress_bytes=387401728 link_bytes=387401728 token_bytes=554829728 weight_movement_bytes=110100480 total_nvlink_bytes=664930208 weight_to_token_ratio=0.198440 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.737553216e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=280010752 dispatch_meta_bytes=229312 combine_payload_bytes=280010752 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280240064 ingress_bytes=500211712 link_bytes=500211712 token_bytes=560250816 weight_movement_bytes=220200960 total_nvlink_bytes=780451776 weight_to_token_ratio=0.393040 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.699504379e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278720512 dispatch_meta_bytes=229352 combine_payload_bytes=278720512 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278949864 ingress_bytes=498921472 link_bytes=498921472 token_bytes=557670376 weight_movement_bytes=220200960 total_nvlink_bytes=777871336 weight_to_token_ratio=0.394859 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.677064983e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279537664 dispatch_meta_bytes=229048 combine_payload_bytes=279537664 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279766712 ingress_bytes=609839104 link_bytes=609839104 token_bytes=559304376 weight_movement_bytes=330301440 total_nvlink_bytes=889605816 weight_to_token_ratio=0.590558 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060610503e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279394304 dispatch_meta_bytes=229720 combine_payload_bytes=279394304 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279624024 ingress_bytes=389494784 link_bytes=389494784 token_bytes=559018328 weight_movement_bytes=110100480 total_nvlink_bytes=669118808 weight_to_token_ratio=0.196953 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4096 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773954903e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44546 payload_rows_remote=39016 dup_rows=20990 dup_rows_remote=18496 dedup_ratio=0.320282 router_tokens_per_expert=[546, 490, 537, 517, 489, 490, 485, 475, 545, 508, 490, 500, 506, 480, 535, 516, 535, 511, 518, 493, 518, 535, 509, 529, 532, 489, 532, 499, 501, 549, 503, 533, 568, 476, 479, 541, 523, 497, 517, 497, 507, 500, 508, 503, 496, 497, 512, 489, 532, 529, 530, 459, 526, 522, 508, 500, 526, 540, 526, 511, 500, 524, 494, 536] payload_rows_per_dst=[5570, 5548, 5560, 5602, 5554, 5530, 5560, 5622] dup_rows_per_dst=[2488, 2612, 2736, 2674, 2642, 2494, 2652, 2692] combine_rows_per_src=[5570, 5548, 5560, 5602, 5554, 5530, 5560, 5622] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279408640 dispatch_meta_bytes=229208 combine_payload_bytes=279408640 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279637848 ingress_bytes=499609600 link_bytes=499609600 token_bytes=559046488 weight_movement_bytes=220200960 total_nvlink_bytes=779247448 weight_to_token_ratio=0.393887 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.689032661e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44586 payload_rows_remote=39006 dup_rows=20950 dup_rows_remote=18342 dedup_ratio=0.319672 router_tokens_per_expert=[542, 489, 490, 540, 494, 547, 533, 508, 519, 527, 520, 541, 532, 531, 534, 497, 532, 497, 524, 504, 482, 475, 524, 487, 521, 526, 486, 542, 492, 472, 537, 518, 525, 494, 508, 509, 518, 489, 496, 496, 541, 486, 505, 463, 491, 494, 533, 494, 562, 528, 544, 526, 498, 546, 501, 512, 493, 475, 488, 543, 509, 540, 501, 497] payload_rows_per_dst=[5676, 5684, 5458, 5580, 5512, 5500, 5690, 5486] dup_rows_per_dst=[2610, 2718, 2592, 2608, 2558, 2514, 2744, 2606] combine_rows_per_src=[5676, 5684, 5458, 5580, 5512, 5500, 5690, 5486] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279666688 dispatch_meta_bytes=230048 combine_payload_bytes=279666688 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279896736 ingress_bytes=389767168 link_bytes=389767168 token_bytes=559563424 weight_movement_bytes=110100480 total_nvlink_bytes=669663904 weight_to_token_ratio=0.196761 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778692108e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44296 payload_rows_remote=38780 dup_rows=21240 dup_rows_remote=18702 dedup_ratio=0.324097 router_tokens_per_expert=[510, 532, 473, 517, 485, 511, 532, 518, 512, 529, 539, 570, 537, 524, 528, 486, 525, 518, 523, 501, 474, 489, 479, 518, 513, 510, 505, 518, 474, 551, 518, 536, 487, 483, 509, 515, 514, 511, 513, 483, 535, 512, 493, 504, 529, 476, 536, 549, 469, 535, 531, 495, 529, 515, 528, 520, 496, 532, 539, 512, 449, 472, 518, 524] payload_rows_per_dst=[5462, 5716, 5516, 5504, 5464, 5498, 5564, 5572] dup_rows_per_dst=[2694, 2734, 2538, 2746, 2566, 2770, 2680, 2512] combine_rows_per_src=[5462, 5716, 5516, 5504, 5464, 5498, 5564, 5572] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44460 payload_rows_remote=38702 dup_rows=21076 dup_rows_remote=18000 dedup_ratio=0.321594 router_tokens_per_expert=[538, 544, 510, 491, 521, 505, 496, 515, 515, 508, 536, 536, 526, 480, 525, 518, 501, 514, 521, 505, 475, 518, 508, 506, 543, 507, 494, 534, 486, 522, 512, 529, 518, 512, 518, 532, 501, 510, 523, 490, 492, 538, 505, 532, 522, 514, 502, 516, 512, 510, 515, 526, 499, 499, 481, 510, 522, 483, 492, 468, 574, 508, 476, 529] payload_rows_per_dst=[5590, 5298, 5574, 5588, 5532, 5574, 5546, 5758] dup_rows_per_dst=[2650, 2260, 2522, 2666, 2676, 2668, 2558, 3076] combine_rows_per_src=[5590, 5298, 5574, 5588, 5532, 5574, 5546, 5758] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44396 payload_rows_remote=38880 dup_rows=21140 dup_rows_remote=18550 dedup_ratio=0.322571 router_tokens_per_expert=[529, 505, 529, 512, 484, 524, 489, 526, 505, 569, 502, 462, 518, 496, 533, 489, 514, 511, 506, 527, 557, 548, 486, 526, 494, 512, 473, 505, 534, 507, 512, 514, 482, 501, 517, 508, 528, 496, 517, 504, 497, 524, 490, 507, 552, 545, 527, 506, 501, 514, 517, 500, 486, 531, 521, 529, 505, 525, 516, 529, 507, 514, 477, 497] payload_rows_per_dst=[5570, 5552, 5610, 5496, 5516, 5626, 5526, 5500] dup_rows_per_dst=[2626, 2596, 2740, 2606, 2590, 2670, 2672, 2640] combine_rows_per_src=[5570, 5552, 5610, 5496, 5516, 5626, 5526, 5500] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44444 payload_rows_remote=38910 dup_rows=21092 dup_rows_remote=18494 dedup_ratio=0.321838 router_tokens_per_expert=[477, 518, 499, 522, 517, 521, 483, 514, 511, 501, 529, 530, 546, 542, 518, 509, 528, 524, 470, 531, 554, 489, 523, 462, 519, 519, 506, 508, 512, 515, 537, 475, 508, 516, 496, 504, 488, 488, 564, 542, 563, 497, 536, 516, 524, 525, 512, 531, 496, 518, 473, 524, 506, 537, 499, 513, 542, 454, 504, 503, 491, 491, 502, 496] payload_rows_per_dst=[5522, 5652, 5536, 5540, 5560, 5652, 5534, 5448] dup_rows_per_dst=[2580, 2720, 2626, 2642, 2652, 2756, 2598, 2518] combine_rows_per_src=[5522, 5652, 5536, 5540, 5560, 5652, 5534, 5448] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44552 payload_rows_remote=38864 dup_rows=20984 dup_rows_remote=18176 dedup_ratio=0.320190 router_tokens_per_expert=[515, 537, 541, 551, 521, 521, 505, 488, 503, 476, 552, 510, 507, 540, 520, 533, 481, 499, 492, 506, 522, 509, 482, 516, 527, 475, 527, 529, 519, 542, 546, 516, 503, 490, 531, 546, 509, 526, 512, 523, 492, 465, 495, 531, 482, 532, 522, 533, 487, 516, 495, 519, 532, 494, 510, 508, 495, 498, 498, 484, 530, 497, 509, 496] payload_rows_per_dst=[5688, 5582, 5618, 5456, 5586, 5580, 5484, 5558] dup_rows_per_dst=[2808, 2710, 2768, 2398, 2764, 2534, 2546, 2456] combine_rows_per_src=[5688, 5582, 5618, 5456, 5586, 5580, 5484, 5558] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279595008 dispatch_meta_bytes=229392 combine_payload_bytes=279595008 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279824400 ingress_bytes=609896448 link_bytes=609896448 token_bytes=559419408 weight_movement_bytes=330301440 total_nvlink_bytes=889720848 weight_to_token_ratio=0.590436 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060710234e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44506 payload_rows_remote=38950 dup_rows=21030 dup_rows_remote=18432 dedup_ratio=0.320892 router_tokens_per_expert=[521, 530, 513, 461, 509, 509, 489, 469, 517, 554, 494, 526, 501, 496, 496, 493, 504, 549, 503, 501, 500, 499, 530, 485, 557, 513, 503, 513, 546, 539, 523, 521, 551, 492, 523, 526, 499, 549, 529, 513, 489, 505, 559, 543, 514, 505, 480, 490, 509, 493, 523, 505, 509, 517, 529, 506, 515, 502, 508, 482, 496, 511, 544, 488] payload_rows_per_dst=[5538, 5556, 5528, 5620, 5688, 5554, 5538, 5484] dup_rows_per_dst=[2464, 2598, 2614, 2810, 2676, 2616, 2644, 2608] combine_rows_per_src=[5538, 5556, 5528, 5620, 5688, 5554, 5538, 5484] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5591272768 prefetch_bytes=990904320 reduce_bytes=330301440 weight_movement_bytes=1321205760 egress_bytes=2796784960 ingress_bytes=4115693568 link_bytes=4115693568 total_nvlink_bytes=6912478528 weight_to_token_ratio=0.236298 prefetch_share_of_ingress=0.240762 dedup_ratio=0.320969 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=3 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.157868030e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=277975040 dispatch_meta_bytes=229928 combine_payload_bytes=277975040 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278204968 ingress_bytes=388075520 link_bytes=388075520 token_bytes=556180008 weight_movement_bytes=110100480 total_nvlink_bytes=666280488 weight_to_token_ratio=0.197958 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.749271567e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=277415936 dispatch_meta_bytes=226808 combine_payload_bytes=277415936 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277642744 ingress_bytes=387516416 link_bytes=387516416 token_bytes=555058680 weight_movement_bytes=110100480 total_nvlink_bytes=665159160 weight_to_token_ratio=0.198358 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4096 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.739547829e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278691840 dispatch_meta_bytes=229720 combine_payload_bytes=278691840 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278921560 ingress_bytes=388792320 link_bytes=388792320 token_bytes=557613400 weight_movement_bytes=110100480 total_nvlink_bytes=667713880 weight_to_token_ratio=0.197449 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.761737898e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278906880 dispatch_meta_bytes=229616 combine_payload_bytes=278906880 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=279136496 ingress_bytes=719308800 link_bytes=719308800 token_bytes=558043376 weight_movement_bytes=440401920 total_nvlink_bytes=998445296 weight_to_token_ratio=0.789189 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.250996309e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5584100936 prefetch_bytes=990904320 reduce_bytes=1211105280 weight_movement_bytes=2202009600 egress_bytes=2793197128 ingress_bytes=4992913408 link_bytes=4992913408 total_nvlink_bytes=7786110536 weight_to_token_ratio=0.394336 prefetch_share_of_ingress=0.198462 dedup_ratio=0.320929 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=11 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.683497610e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278577152 dispatch_meta_bytes=228160 combine_payload_bytes=278577152 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278805312 ingress_bytes=388677632 link_bytes=388677632 token_bytes=557382464 weight_movement_bytes=110100480 total_nvlink_bytes=667482944 weight_to_token_ratio=0.197531 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.759743285e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279193600 dispatch_meta_bytes=229528 combine_payload_bytes=279193600 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279423128 ingress_bytes=499394560 link_bytes=499394560 token_bytes=558616728 weight_movement_bytes=220200960 total_nvlink_bytes=778817688 weight_to_token_ratio=0.394190 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.685292761e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5581665688 prefetch_bytes=990904320 reduce_bytes=770703360 weight_movement_bytes=1761607680 egress_bytes=2791980440 ingress_bytes=4551292928 link_bytes=4551292928 total_nvlink_bytes=7343273368 weight_to_token_ratio=0.315606 prefetch_share_of_ingress=0.217719 dedup_ratio=0.321210 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=7 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.044853 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.915446961e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5561640800 prefetch_bytes=990904320 reduce_bytes=220200960 weight_movement_bytes=1211105280 egress_bytes=2781962080 ingress_bytes=3990784000 link_bytes=3990784000 total_nvlink_bytes=6772746080 weight_to_token_ratio=0.217760 prefetch_share_of_ingress=0.248298 dedup_ratio=0.322162 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=2 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=6.940629747e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5576791792 prefetch_bytes=990904320 reduce_bytes=990904320 weight_movement_bytes=1981808640 egress_bytes=2789543664 ingress_bytes=4769056768 link_bytes=4769056768 total_nvlink_bytes=7558600432 weight_to_token_ratio=0.355367 prefetch_share_of_ingress=0.207778 dedup_ratio=0.322006 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=9 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.294174095e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5580919592 prefetch_bytes=990904320 reduce_bytes=880803840 weight_movement_bytes=1871708160 egress_bytes=2791607080 ingress_bytes=4661020672 link_bytes=4661020672 total_nvlink_bytes=7452627752 weight_to_token_ratio=0.335376 prefetch_share_of_ingress=0.212594 dedup_ratio=0.321640 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=8 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.106281555e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5575121368 prefetch_bytes=990904320 reduce_bytes=1321205760 weight_movement_bytes=2312110080 egress_bytes=2788704728 ingress_bytes=5098526720 link_bytes=5098526720 total_nvlink_bytes=7887231448 weight_to_token_ratio=0.414719 prefetch_share_of_ingress=0.194351 dedup_ratio=0.321390 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=12 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.039706 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.867176530e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5591211560 prefetch_bytes=770703360 reduce_bytes=1981808640 weight_movement_bytes=2752512000 egress_bytes=2796752424 ingress_bytes=5546971136 link_bytes=5546971136 total_nvlink_bytes=8343723560 weight_to_token_ratio=0.492293 prefetch_share_of_ingress=0.138941 dedup_ratio=0.320175 prefetch_slots=14 unused_prefetch_slots=146 slot_utilization=0.087500 reduce_slots=18 remote_demand=7 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=12319 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.647095125e-05 prefetch_bytes_per_gemm_flop=1.340379902e-05 anomalies=none - [2026-08-26 15:12:40] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 671.7 | throughput per GPU (TFLOP/s/GPU): 183.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.488188E+00 | loss scale: 1.0 | grad norm: 3.427 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44572 payload_rows_remote=39076 dup_rows=20964 dup_rows_remote=18396 dedup_ratio=0.319885 router_tokens_per_expert=[484, 518, 488, 530, 510, 550, 517, 537, 515, 445, 544, 509, 512, 493, 519, 495, 466, 497, 537, 513, 487, 533, 481, 520, 552, 535, 482, 516, 559, 518, 520, 541, 527, 524, 482, 531, 500, 512, 531, 515, 512, 553, 510, 507, 506, 512, 501, 507, 528, 494, 555, 487, 508, 511, 498, 502, 524, 515, 518, 502, 501, 503, 482, 487] payload_rows_per_dst=[5614, 5496, 5538, 5704, 5556, 5552, 5556, 5556] dup_rows_per_dst=[2654, 2568, 2530, 2742, 2688, 2664, 2610, 2508] combine_rows_per_src=[5614, 5496, 5538, 5704, 5556, 5552, 5556, 5556] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44574 payload_rows_remote=38974 dup_rows=20962 dup_rows_remote=18494 dedup_ratio=0.319855 router_tokens_per_expert=[548, 534, 498, 506, 525, 543, 533, 510, 492, 522, 554, 540, 490, 550, 551, 480, 519, 482, 506, 472, 512, 525, 486, 532, 517, 539, 500, 480, 559, 533, 524, 538, 499, 505, 505, 513, 508, 491, 513, 509, 505, 494, 475, 453, 533, 497, 520, 517, 531, 513, 462, 547, 540, 527, 505, 510, 516, 552, 517, 484, 505, 503, 472, 447] payload_rows_per_dst=[5602, 5644, 5600, 5684, 5522, 5472, 5546, 5504] dup_rows_per_dst=[2792, 2714, 2468, 2696, 2564, 2516, 2724, 2488] combine_rows_per_src=[5602, 5644, 5600, 5684, 5522, 5472, 5546, 5504] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44518 payload_rows_remote=39018 dup_rows=21018 dup_rows_remote=18490 dedup_ratio=0.320709 router_tokens_per_expert=[520, 504, 507, 563, 568, 534, 506, 531, 504, 555, 493, 469, 498, 521, 486, 532, 527, 537, 531, 541, 523, 491, 499, 509, 502, 500, 487, 501, 514, 504, 505, 501, 536, 502, 489, 491, 497, 501, 520, 498, 524, 507, 554, 487, 542, 515, 488, 497, 478, 488, 467, 514, 500, 524, 499, 535, 503, 488, 533, 527, 495, 524, 557, 525] payload_rows_per_dst=[5696, 5626, 5612, 5500, 5504, 5562, 5478, 5540] dup_rows_per_dst=[2770, 2490, 2704, 2528, 2564, 2666, 2532, 2764] combine_rows_per_src=[5696, 5626, 5612, 5500, 5504, 5562, 5478, 5540] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44590 payload_rows_remote=38986 dup_rows=20946 dup_rows_remote=18248 dedup_ratio=0.319611 router_tokens_per_expert=[509, 512, 520, 541, 523, 469, 523, 517, 513, 522, 514, 482, 521, 518, 536, 528, 538, 543, 523, 507, 510, 527, 516, 518, 547, 513, 490, 526, 554, 474, 510, 486, 500, 516, 515, 510, 573, 500, 544, 493, 512, 477, 509, 511, 549, 511, 502, 536, 490, 536, 522, 485, 459, 505, 496, 490, 480, 542, 484, 492, 488, 511, 506, 494] payload_rows_per_dst=[5634, 5530, 5626, 5550, 5604, 5568, 5542, 5536] dup_rows_per_dst=[2594, 2738, 2738, 2650, 2698, 2646, 2424, 2458] combine_rows_per_src=[5634, 5530, 5626, 5550, 5604, 5568, 5542, 5536] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44680 payload_rows_remote=39138 dup_rows=20856 dup_rows_remote=18198 dedup_ratio=0.318237 router_tokens_per_expert=[504, 542, 529, 521, 526, 511, 489, 556, 505, 501, 522, 499, 500, 502, 514, 511, 525, 480, 592, 489, 477, 508, 517, 471, 527, 501, 516, 523, 530, 519, 489, 525, 510, 516, 531, 523, 452, 470, 530, 470, 500, 508, 518, 524, 504, 568, 488, 490, 530, 524, 522, 513, 532, 509, 536, 523, 541, 501, 508, 531, 498, 492, 475, 510] payload_rows_per_dst=[5666, 5552, 5524, 5660, 5484, 5542, 5690, 5562] dup_rows_per_dst=[2690, 2556, 2594, 2600, 2520, 2658, 2688, 2550] combine_rows_per_src=[5666, 5552, 5524, 5660, 5484, 5542, 5690, 5562] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44368 payload_rows_remote=38662 dup_rows=21168 dup_rows_remote=18136 dedup_ratio=0.322998 router_tokens_per_expert=[498, 527, 503, 531, 530, 498, 513, 492, 527, 515, 515, 504, 486, 535, 518, 483, 489, 514, 557, 497, 519, 514, 532, 512, 500, 534, 511, 500, 524, 520, 511, 521, 492, 503, 518, 507, 523, 527, 518, 518, 520, 528, 532, 504, 479, 501, 523, 493, 539, 529, 498, 541, 485, 472, 513, 510, 489, 496, 536, 520, 512, 501, 513, 498] payload_rows_per_dst=[5368, 5432, 5552, 5426, 5586, 5552, 5746, 5706] dup_rows_per_dst=[2208, 2598, 2716, 2422, 2634, 2610, 2948, 3032] combine_rows_per_src=[5368, 5432, 5552, 5426, 5586, 5552, 5746, 5706] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44502 payload_rows_remote=39108 dup_rows=21034 dup_rows_remote=18496 dedup_ratio=0.320953 router_tokens_per_expert=[510, 480, 532, 516, 496, 505, 501, 516, 501, 497, 534, 555, 499, 502, 516, 542, 529, 496, 481, 540, 496, 505, 513, 478, 540, 486, 512, 551, 499, 492, 513, 501, 530, 525, 526, 504, 539, 559, 561, 515, 550, 511, 477, 490, 536, 526, 556, 517, 462, 492, 483, 545, 469, 496, 521, 498, 518, 491, 526, 485, 481, 528, 519, 498] payload_rows_per_dst=[5576, 5562, 5560, 5548, 5748, 5568, 5394, 5546] dup_rows_per_dst=[2536, 2730, 2516, 2640, 2770, 2758, 2538, 2546] combine_rows_per_src=[5576, 5562, 5560, 5548, 5748, 5568, 5394, 5546] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44520 payload_rows_remote=38894 dup_rows=21016 dup_rows_remote=18452 dedup_ratio=0.320679 router_tokens_per_expert=[490, 539, 525, 541, 547, 488, 466, 545, 527, 507, 526, 478, 485, 503, 518, 511, 542, 515, 488, 495, 513, 525, 473, 541, 500, 484, 482, 533, 530, 539, 502, 447, 482, 500, 526, 502, 513, 512, 547, 513, 532, 535, 522, 514, 493, 512, 510, 540, 505, 529, 548, 485, 507, 551, 468, 520, 507, 529, 568, 490, 518, 499, 515, 471] payload_rows_per_dst=[5610, 5512, 5502, 5496, 5626, 5602, 5578, 5594] dup_rows_per_dst=[2672, 2598, 2682, 2538, 2564, 2714, 2648, 2600] combine_rows_per_src=[5610, 5512, 5502, 5496, 5626, 5602, 5578, 5594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44596 payload_rows_remote=39020 dup_rows=20940 dup_rows_remote=18318 dedup_ratio=0.319519 router_tokens_per_expert=[573, 515, 512, 525, 521, 524, 518, 530, 519, 492, 481, 481, 540, 515, 534, 537, 504, 513, 515, 501, 503, 507, 478, 518, 523, 476, 492, 509, 528, 493, 529, 496, 537, 482, 507, 544, 545, 525, 510, 520, 533, 497, 523, 528, 534, 513, 500, 491, 462, 510, 538, 507, 488, 481, 507, 488, 513, 536, 513, 498, 480, 507, 552, 497] payload_rows_per_dst=[5740, 5576, 5492, 5490, 5552, 5634, 5504, 5608] dup_rows_per_dst=[2696, 2622, 2586, 2602, 2788, 2604, 2458, 2584] combine_rows_per_src=[5740, 5576, 5492, 5490, 5552, 5634, 5504, 5608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44654 payload_rows_remote=39058 dup_rows=20882 dup_rows_remote=18300 dedup_ratio=0.318634 router_tokens_per_expert=[518, 495, 493, 514, 535, 482, 514, 520, 528, 528, 510, 519, 496, 534, 513, 511, 538, 518, 495, 483, 490, 541, 521, 503, 510, 532, 521, 531, 506, 509, 516, 500, 529, 446, 549, 504, 538, 511, 503, 531, 533, 501, 512, 552, 506, 503, 513, 491, 514, 497, 497, 484, 504, 485, 465, 514, 526, 535, 557, 503, 555, 510, 469, 507] payload_rows_per_dst=[5502, 5640, 5596, 5608, 5602, 5618, 5464, 5624] dup_rows_per_dst=[2640, 2638, 2582, 2642, 2620, 2604, 2456, 2700] combine_rows_per_src=[5502, 5640, 5596, 5608, 5602, 5618, 5464, 5624] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44614 payload_rows_remote=39114 dup_rows=20922 dup_rows_remote=18256 dedup_ratio=0.319244 router_tokens_per_expert=[525, 514, 494, 476, 502, 527, 488, 521, 516, 512, 507, 499, 511, 522, 478, 508, 534, 531, 479, 580, 529, 475, 516, 507, 516, 502, 541, 521, 502, 486, 518, 497, 490, 510, 529, 559, 510, 517, 494, 503, 569, 498, 539, 510, 520, 501, 494, 473, 542, 519, 518, 508, 520, 524, 496, 499, 509, 493, 503, 535, 491, 512, 509, 540] payload_rows_per_dst=[5524, 5546, 5632, 5500, 5590, 5600, 5630, 5592] dup_rows_per_dst=[2570, 2560, 2670, 2666, 2634, 2608, 2622, 2592] combine_rows_per_src=[5524, 5546, 5632, 5500, 5590, 5600, 5630, 5592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278046720 dispatch_meta_bytes=228368 combine_payload_bytes=278046720 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278275088 ingress_bytes=388147200 link_bytes=388147200 token_bytes=556321808 weight_movement_bytes=110100480 total_nvlink_bytes=666422288 weight_to_token_ratio=0.197908 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.750518200e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44406 payload_rows_remote=38890 dup_rows=21130 dup_rows_remote=18468 dedup_ratio=0.322418 router_tokens_per_expert=[505, 523, 511, 503, 525, 518, 496, 536, 532, 496, 471, 494, 518, 519, 539, 531, 565, 480, 501, 504, 518, 509, 514, 507, 520, 509, 498, 484, 519, 535, 515, 532, 531, 506, 532, 505, 520, 537, 480, 500, 512, 546, 531, 514, 482, 518, 474, 512, 508, 514, 496, 537, 496, 509, 509, 526, 522, 489, 486, 508, 515, 488, 523, 515] payload_rows_per_dst=[5600, 5548, 5522, 5516, 5598, 5516, 5606, 5500] dup_rows_per_dst=[2634, 2652, 2674, 2708, 2624, 2662, 2584, 2592] combine_rows_per_src=[5600, 5548, 5522, 5516, 5598, 5516, 5606, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44588 payload_rows_remote=39012 dup_rows=20948 dup_rows_remote=18190 dedup_ratio=0.319641 router_tokens_per_expert=[506, 514, 485, 523, 513, 510, 526, 500, 529, 514, 537, 532, 481, 518, 554, 520, 517, 498, 515, 464, 505, 465, 529, 521, 514, 520, 523, 517, 498, 536, 504, 525, 514, 512, 521, 510, 519, 537, 540, 531, 522, 511, 532, 515, 523, 511, 520, 496, 500, 543, 498, 499, 490, 494, 513, 516, 497, 501, 499, 501, 480, 530, 473, 507] payload_rows_per_dst=[5576, 5662, 5480, 5622, 5670, 5544, 5560, 5474] dup_rows_per_dst=[2758, 2780, 2492, 2660, 2698, 2512, 2546, 2502] combine_rows_per_src=[5576, 5662, 5480, 5622, 5670, 5544, 5560, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278792192 dispatch_meta_bytes=229384 combine_payload_bytes=278792192 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279021576 ingress_bytes=498993152 link_bytes=498993152 token_bytes=557813768 weight_movement_bytes=220200960 total_nvlink_bytes=778014728 weight_to_token_ratio=0.394757 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.678311616e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279695360 dispatch_meta_bytes=229352 combine_payload_bytes=279695360 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279924712 ingress_bytes=499896320 link_bytes=499896320 token_bytes=559620072 weight_movement_bytes=220200960 total_nvlink_bytes=779821032 weight_to_token_ratio=0.393483 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.694019193e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279967744 dispatch_meta_bytes=229432 combine_payload_bytes=279967744 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280197176 ingress_bytes=610269184 link_bytes=610269184 token_bytes=560164920 weight_movement_bytes=330301440 total_nvlink_bytes=890466360 weight_to_token_ratio=0.589650 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061358483e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=280369152 dispatch_meta_bytes=229480 combine_payload_bytes=280369152 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280598632 ingress_bytes=390469632 link_bytes=390469632 token_bytes=560967784 weight_movement_bytes=110100480 total_nvlink_bytes=671068264 weight_to_token_ratio=0.196269 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.790909113e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278763520 dispatch_meta_bytes=229432 combine_payload_bytes=278763520 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278992952 ingress_bytes=609064960 link_bytes=609064960 token_bytes=557756472 weight_movement_bytes=330301440 total_nvlink_bytes=888057912 weight_to_token_ratio=0.592197 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059264139e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44456 payload_rows_remote=38944 dup_rows=21080 dup_rows_remote=18462 dedup_ratio=0.321655 router_tokens_per_expert=[483, 551, 489, 488, 508, 473, 497, 526, 479, 518, 532, 576, 493, 514, 513, 480, 584, 485, 538, 548, 555, 529, 535, 490, 518, 500, 493, 508, 537, 530, 515, 527, 537, 525, 490, 511, 545, 506, 499, 500, 489, 477, 482, 518, 529, 505, 510, 503, 519, 485, 516, 529, 500, 495, 506, 515, 524, 486, 493, 527, 534, 485, 524, 492] payload_rows_per_dst=[5518, 5558, 5666, 5606, 5556, 5470, 5512, 5570] dup_rows_per_dst=[2512, 2652, 2862, 2650, 2670, 2556, 2618, 2560] combine_rows_per_src=[5518, 5558, 5666, 5606, 5556, 5470, 5512, 5570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279638016 dispatch_meta_bytes=228808 combine_payload_bytes=279638016 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279866824 ingress_bytes=389738496 link_bytes=389738496 token_bytes=559504840 weight_movement_bytes=110100480 total_nvlink_bytes=669605320 weight_to_token_ratio=0.196782 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778193455e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44422 payload_rows_remote=38764 dup_rows=21114 dup_rows_remote=18196 dedup_ratio=0.322174 router_tokens_per_expert=[537, 540, 508, 488, 521, 501, 492, 512, 512, 507, 535, 533, 525, 476, 525, 517, 496, 514, 521, 505, 472, 515, 510, 508, 546, 505, 495, 530, 486, 520, 510, 528, 518, 513, 520, 532, 501, 509, 523, 491, 494, 539, 505, 532, 521, 514, 502, 518, 515, 511, 517, 526, 506, 500, 486, 513, 527, 486, 492, 471, 574, 512, 478, 532] payload_rows_per_dst=[5572, 5358, 5772, 5410, 5530, 5534, 5588, 5658] dup_rows_per_dst=[2626, 2354, 2858, 2398, 2684, 2618, 2658, 2918] combine_rows_per_src=[5572, 5358, 5772, 5410, 5530, 5534, 5588, 5658] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279150592 dispatch_meta_bytes=229624 combine_payload_bytes=279150592 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279380216 ingress_bytes=389251072 link_bytes=389251072 token_bytes=558530808 weight_movement_bytes=110100480 total_nvlink_bytes=668631288 weight_to_token_ratio=0.197125 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.769716350e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44412 payload_rows_remote=38902 dup_rows=21124 dup_rows_remote=18524 dedup_ratio=0.322327 router_tokens_per_expert=[528, 504, 528, 509, 482, 520, 489, 524, 503, 567, 499, 460, 517, 498, 529, 486, 511, 510, 506, 524, 555, 545, 487, 527, 496, 511, 473, 502, 532, 506, 512, 512, 482, 501, 519, 506, 529, 497, 517, 504, 498, 527, 492, 507, 553, 544, 529, 509, 500, 514, 518, 505, 486, 531, 522, 531, 513, 528, 519, 530, 509, 517, 480, 499] payload_rows_per_dst=[5568, 5544, 5606, 5492, 5510, 5638, 5536, 5518] dup_rows_per_dst=[2600, 2574, 2724, 2596, 2600, 2680, 2678, 2672] combine_rows_per_src=[5568, 5544, 5606, 5492, 5510, 5638, 5536, 5518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44500 payload_rows_remote=38962 dup_rows=21036 dup_rows_remote=18460 dedup_ratio=0.320984 router_tokens_per_expert=[518, 529, 509, 461, 507, 504, 486, 469, 515, 553, 491, 524, 498, 496, 490, 490, 505, 548, 499, 500, 496, 496, 528, 482, 558, 513, 503, 514, 546, 539, 523, 521, 551, 490, 526, 526, 501, 548, 530, 515, 489, 506, 559, 543, 519, 508, 483, 490, 509, 493, 523, 506, 514, 519, 529, 510, 517, 505, 510, 485, 501, 514, 548, 490] payload_rows_per_dst=[5518, 5538, 5508, 5624, 5692, 5562, 5554, 5504] dup_rows_per_dst=[2448, 2576, 2600, 2810, 2682, 2632, 2652, 2636] combine_rows_per_src=[5518, 5538, 5508, 5624, 5692, 5562, 5554, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44296 payload_rows_remote=38792 dup_rows=21240 dup_rows_remote=18714 dedup_ratio=0.324097 router_tokens_per_expert=[508, 532, 464, 515, 485, 509, 528, 518, 512, 528, 537, 567, 534, 523, 521, 487, 524, 515, 521, 501, 475, 488, 478, 513, 513, 511, 505, 516, 478, 550, 517, 534, 487, 484, 510, 515, 514, 517, 513, 484, 537, 514, 493, 503, 528, 478, 537, 553, 469, 539, 532, 496, 529, 517, 532, 522, 499, 534, 540, 514, 454, 473, 519, 525] payload_rows_per_dst=[5444, 5706, 5504, 5500, 5470, 5514, 5578, 5580] dup_rows_per_dst=[2674, 2712, 2526, 2748, 2578, 2772, 2694, 2536] combine_rows_per_src=[5444, 5706, 5504, 5500, 5470, 5514, 5578, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44568 payload_rows_remote=39000 dup_rows=20968 dup_rows_remote=18342 dedup_ratio=0.319946 router_tokens_per_expert=[538, 486, 489, 539, 493, 547, 532, 508, 518, 524, 515, 540, 531, 530, 528, 497, 529, 496, 522, 503, 480, 475, 524, 485, 526, 527, 487, 539, 491, 470, 539, 518, 523, 497, 508, 506, 520, 486, 495, 496, 542, 485, 504, 467, 494, 495, 536, 495, 564, 529, 548, 527, 500, 548, 503, 513, 494, 474, 489, 547, 510, 541, 505, 501] payload_rows_per_dst=[5668, 5668, 5450, 5568, 5510, 5510, 5700, 5494] dup_rows_per_dst=[2596, 2698, 2578, 2626, 2552, 2526, 2764, 2628] combine_rows_per_src=[5668, 5668, 5450, 5568, 5510, 5510, 5700, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=277860352 dispatch_meta_bytes=227840 combine_payload_bytes=277860352 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278088192 ingress_bytes=387960832 link_bytes=387960832 token_bytes=555948544 weight_movement_bytes=110100480 total_nvlink_bytes=666049024 weight_to_token_ratio=0.198041 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.747276954e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44556 payload_rows_remote=39010 dup_rows=20980 dup_rows_remote=18476 dedup_ratio=0.320129 router_tokens_per_expert=[542, 489, 535, 515, 485, 490, 484, 473, 544, 505, 489, 497, 503, 480, 535, 517, 534, 508, 516, 491, 515, 536, 508, 527, 531, 487, 530, 499, 500, 547, 503, 533, 566, 475, 478, 539, 526, 497, 515, 499, 506, 502, 507, 503, 499, 500, 515, 493, 534, 530, 532, 465, 530, 524, 509, 504, 527, 542, 528, 512, 505, 526, 495, 537] payload_rows_per_dst=[5556, 5542, 5546, 5604, 5552, 5546, 5572, 5638] dup_rows_per_dst=[2470, 2598, 2724, 2656, 2638, 2504, 2684, 2706] combine_rows_per_src=[5556, 5542, 5546, 5604, 5552, 5546, 5572, 5638] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44536 payload_rows_remote=38782 dup_rows=21000 dup_rows_remote=18022 dedup_ratio=0.320435 router_tokens_per_expert=[512, 535, 538, 548, 519, 519, 503, 486, 501, 475, 552, 506, 505, 539, 519, 532, 481, 498, 491, 507, 521, 506, 481, 513, 525, 474, 527, 530, 520, 543, 547, 517, 499, 489, 533, 547, 508, 528, 515, 524, 494, 465, 497, 531, 483, 532, 521, 534, 488, 518, 496, 523, 532, 495, 514, 510, 498, 498, 500, 485, 531, 500, 513, 497] payload_rows_per_dst=[5754, 5584, 5466, 5656, 5566, 5554, 5388, 5568] dup_rows_per_dst=[2978, 2736, 2530, 2658, 2758, 2512, 2352, 2476] combine_rows_per_src=[5754, 5584, 5466, 5656, 5566, 5554, 5388, 5568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278849536 dispatch_meta_bytes=229704 combine_payload_bytes=278849536 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279079240 ingress_bytes=388950016 link_bytes=388950016 token_bytes=557928776 weight_movement_bytes=110100480 total_nvlink_bytes=668029256 weight_to_token_ratio=0.197338 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.764480491e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279279616 dispatch_meta_bytes=229688 combine_payload_bytes=279279616 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279509304 ingress_bytes=499480576 link_bytes=499480576 token_bytes=558788920 weight_movement_bytes=220200960 total_nvlink_bytes=778989880 weight_to_token_ratio=0.394068 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.686788721e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278061056 dispatch_meta_bytes=230024 combine_payload_bytes=278061056 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278291080 ingress_bytes=388161536 link_bytes=388161536 token_bytes=556352136 weight_movement_bytes=110100480 total_nvlink_bytes=666452616 weight_to_token_ratio=0.197897 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.750767527e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5566603368 prefetch_bytes=990904320 reduce_bytes=1211105280 weight_movement_bytes=2202009600 egress_bytes=2784444520 ingress_bytes=4984168448 link_bytes=4984168448 total_nvlink_bytes=7768612968 weight_to_token_ratio=0.395575 prefetch_share_of_ingress=0.198810 dedup_ratio=0.321954 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=11 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.668288686e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44460 payload_rows_remote=38924 dup_rows=21076 dup_rows_remote=18474 dedup_ratio=0.321594 router_tokens_per_expert=[477, 513, 496, 518, 514, 520, 483, 511, 508, 500, 529, 531, 543, 542, 517, 504, 524, 521, 470, 530, 555, 490, 521, 461, 519, 521, 506, 509, 513, 513, 539, 475, 508, 517, 492, 507, 488, 490, 563, 542, 567, 498, 537, 516, 524, 524, 520, 531, 494, 518, 474, 524, 507, 539, 501, 512, 543, 456, 509, 504, 492, 494, 506, 498] payload_rows_per_dst=[5514, 5640, 5530, 5546, 5556, 5668, 5536, 5470] dup_rows_per_dst=[2550, 2708, 2614, 2644, 2658, 2766, 2602, 2534] combine_rows_per_src=[5514, 5640, 5530, 5546, 5556, 5668, 5536, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279552000 dispatch_meta_bytes=229368 combine_payload_bytes=279552000 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279781368 ingress_bytes=609853440 link_bytes=609853440 token_bytes=559333368 weight_movement_bytes=330301440 total_nvlink_bytes=889634808 weight_to_token_ratio=0.590527 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060635436e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279623680 dispatch_meta_bytes=229944 combine_payload_bytes=279623680 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279853624 ingress_bytes=609925120 link_bytes=609925120 token_bytes=559477304 weight_movement_bytes=330301440 total_nvlink_bytes=889778744 weight_to_token_ratio=0.590375 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060760099e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5577164200 prefetch_bytes=990904320 reduce_bytes=880803840 weight_movement_bytes=1871708160 egress_bytes=2789729704 ingress_bytes=4659142656 link_bytes=4659142656 total_nvlink_bytes=7448872360 weight_to_token_ratio=0.335602 prefetch_share_of_ingress=0.212680 dedup_ratio=0.321945 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=8 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.103015376e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=277989376 dispatch_meta_bytes=227216 combine_payload_bytes=277989376 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278216592 ingress_bytes=388089856 link_bytes=388089856 token_bytes=556205968 weight_movement_bytes=110100480 total_nvlink_bytes=666306448 weight_to_token_ratio=0.197949 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.749520894e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5582555360 prefetch_bytes=990904320 reduce_bytes=770703360 weight_movement_bytes=1761607680 egress_bytes=2792425696 ingress_bytes=4551737344 link_bytes=4551737344 total_nvlink_bytes=7344163040 weight_to_token_ratio=0.315556 prefetch_share_of_ingress=0.217698 dedup_ratio=0.321255 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=7 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043382 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.916219873e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5592531704 prefetch_bytes=990904320 reduce_bytes=550502400 weight_movement_bytes=1541406720 egress_bytes=2797413112 ingress_bytes=4336525312 link_bytes=4336525312 total_nvlink_bytes=7133938424 weight_to_token_ratio=0.275619 prefetch_share_of_ingress=0.228502 dedup_ratio=0.320264 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=5 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.541930753e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5584244464 prefetch_bytes=1101004800 reduce_bytes=990904320 weight_movement_bytes=2091909120 egress_bytes=2793268976 ingress_bytes=4882884608 link_bytes=4882884608 total_nvlink_bytes=7676153584 weight_to_token_ratio=0.374609 prefetch_share_of_ingress=0.225482 dedup_ratio=0.320959 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=9 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.039706 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.492139430e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5590096456 prefetch_bytes=660602880 reduce_bytes=1431306240 weight_movement_bytes=2091909120 egress_bytes=2796196424 ingress_bytes=4885809152 link_bytes=4885809152 total_nvlink_bytes=7682005576 weight_to_token_ratio=0.374217 prefetch_share_of_ingress=0.135208 dedup_ratio=0.320932 prefetch_slots=12 unused_prefetch_slots=148 slot_utilization=0.075000 reduce_slots=13 remote_demand=6 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042647 gemm_inner_pad_upper_bound=12192 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.497225693e-05 prefetch_bytes_per_gemm_flop=1.148897059e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279007232 dispatch_meta_bytes=229592 combine_payload_bytes=279007232 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279236824 ingress_bytes=499208192 link_bytes=499208192 token_bytes=558244056 weight_movement_bytes=220200960 total_nvlink_bytes=778445016 weight_to_token_ratio=0.394453 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.682051515e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5567490152 prefetch_bytes=1101004800 reduce_bytes=550502400 weight_movement_bytes=1651507200 egress_bytes=2784886888 ingress_bytes=4434110464 link_bytes=4434110464 total_nvlink_bytes=7218997352 weight_to_token_ratio=0.296634 prefetch_share_of_ingress=0.248303 dedup_ratio=0.321674 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=5 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.711647383e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5578624728 prefetch_bytes=880803840 reduce_bytes=1321205760 weight_movement_bytes=2202009600 egress_bytes=2790459096 ingress_bytes=4990175232 link_bytes=4990175232 total_nvlink_bytes=7780634328 weight_to_token_ratio=0.394723 prefetch_share_of_ingress=0.176508 dedup_ratio=0.321713 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=12 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042647 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.678735471e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none + [2026-08-26 16:00:29] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 638.0 | throughput per GPU (TFLOP/s/GPU): 193.5 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.491462E+00 | loss scale: 1.0 | grad norm: 3.542 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44556 payload_rows_remote=38956 dup_rows=20980 dup_rows_remote=18520 dedup_ratio=0.320129 router_tokens_per_expert=[546, 534, 495, 506, 525, 540, 532, 508, 490, 522, 551, 533, 489, 548, 547, 481, 518, 483, 503, 474, 510, 526, 486, 530, 519, 535, 497, 480, 560, 532, 524, 538, 499, 507, 504, 513, 508, 489, 514, 511, 505, 493, 478, 452, 533, 500, 520, 518, 532, 514, 463, 549, 544, 529, 505, 511, 520, 553, 520, 487, 507, 505, 475, 448] payload_rows_per_dst=[5592, 5624, 5600, 5678, 5518, 5476, 5552, 5516] dup_rows_per_dst=[2780, 2698, 2460, 2692, 2572, 2522, 2742, 2514] combine_rows_per_src=[5592, 5624, 5600, 5678, 5518, 5476, 5552, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44692 payload_rows_remote=39138 dup_rows=20844 dup_rows_remote=18172 dedup_ratio=0.318054 router_tokens_per_expert=[499, 540, 527, 521, 523, 509, 487, 549, 503, 500, 519, 494, 500, 503, 513, 507, 524, 479, 589, 489, 478, 510, 518, 470, 526, 502, 515, 522, 530, 519, 487, 525, 509, 514, 531, 520, 454, 470, 529, 470, 499, 511, 519, 526, 505, 569, 491, 493, 531, 525, 523, 514, 534, 514, 537, 528, 543, 501, 512, 534, 499, 493, 478, 515] payload_rows_per_dst=[5646, 5546, 5530, 5662, 5482, 5554, 5698, 5574] dup_rows_per_dst=[2664, 2532, 2584, 2590, 2512, 2672, 2714, 2576] combine_rows_per_src=[5646, 5546, 5530, 5662, 5482, 5554, 5698, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44376 payload_rows_remote=38758 dup_rows=21160 dup_rows_remote=18318 dedup_ratio=0.322876 router_tokens_per_expert=[495, 524, 499, 528, 530, 496, 509, 489, 524, 510, 514, 499, 484, 534, 517, 486, 489, 514, 556, 497, 521, 513, 531, 511, 499, 533, 510, 499, 523, 521, 512, 521, 493, 501, 521, 505, 521, 529, 518, 518, 520, 530, 533, 504, 483, 504, 525, 493, 540, 529, 498, 542, 488, 473, 516, 513, 492, 497, 540, 522, 514, 502, 515, 501] payload_rows_per_dst=[5486, 5464, 5556, 5434, 5588, 5558, 5672, 5618] dup_rows_per_dst=[2398, 2672, 2708, 2490, 2642, 2626, 2782, 2842] combine_rows_per_src=[5486, 5464, 5556, 5434, 5588, 5558, 5672, 5618] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44598 payload_rows_remote=38994 dup_rows=20938 dup_rows_remote=18244 dedup_ratio=0.319489 router_tokens_per_expert=[507, 507, 517, 538, 521, 465, 523, 515, 513, 520, 509, 484, 516, 517, 534, 527, 536, 541, 520, 507, 504, 528, 514, 514, 546, 511, 489, 527, 552, 475, 510, 488, 495, 517, 512, 511, 573, 501, 544, 496, 514, 480, 510, 512, 550, 512, 503, 537, 493, 542, 525, 489, 462, 508, 498, 493, 483, 542, 487, 494, 491, 512, 508, 499] payload_rows_per_dst=[5618, 5514, 5618, 5552, 5604, 5578, 5568, 5546] dup_rows_per_dst=[2568, 2726, 2710, 2644, 2694, 2658, 2452, 2486] combine_rows_per_src=[5618, 5514, 5618, 5552, 5604, 5578, 5568, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44582 payload_rows_remote=39102 dup_rows=20954 dup_rows_remote=18404 dedup_ratio=0.319733 router_tokens_per_expert=[480, 516, 482, 529, 509, 545, 513, 534, 512, 444, 542, 505, 510, 489, 519, 494, 467, 498, 534, 512, 485, 531, 484, 518, 554, 533, 482, 516, 561, 519, 518, 541, 526, 521, 480, 529, 504, 509, 532, 517, 512, 554, 513, 510, 510, 516, 505, 509, 528, 495, 558, 488, 508, 516, 499, 503, 526, 516, 520, 505, 504, 504, 484, 491] payload_rows_per_dst=[5604, 5480, 5536, 5704, 5550, 5570, 5564, 5574] dup_rows_per_dst=[2612, 2550, 2522, 2744, 2686, 2688, 2626, 2526] combine_rows_per_src=[5604, 5480, 5536, 5704, 5550, 5570, 5564, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44504 payload_rows_remote=39092 dup_rows=21032 dup_rows_remote=18476 dedup_ratio=0.320923 router_tokens_per_expert=[507, 479, 530, 513, 494, 503, 496, 515, 498, 497, 532, 550, 500, 501, 514, 540, 527, 493, 477, 541, 496, 501, 509, 475, 537, 486, 511, 552, 500, 493, 514, 502, 530, 526, 525, 505, 538, 559, 560, 516, 553, 512, 481, 489, 537, 528, 556, 517, 464, 495, 491, 547, 471, 497, 522, 497, 522, 494, 528, 486, 484, 533, 522, 500] payload_rows_per_dst=[5558, 5550, 5548, 5552, 5740, 5582, 5412, 5562] dup_rows_per_dst=[2516, 2714, 2490, 2638, 2778, 2764, 2556, 2576] combine_rows_per_src=[5558, 5550, 5548, 5552, 5740, 5582, 5412, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44522 payload_rows_remote=39034 dup_rows=21014 dup_rows_remote=18492 dedup_ratio=0.320648 router_tokens_per_expert=[516, 502, 506, 559, 564, 532, 504, 529, 505, 553, 492, 467, 498, 520, 483, 531, 524, 534, 530, 540, 521, 488, 496, 507, 505, 496, 484, 500, 513, 503, 504, 500, 535, 502, 486, 492, 495, 506, 521, 498, 524, 507, 555, 491, 542, 516, 493, 500, 481, 489, 471, 516, 502, 525, 503, 535, 504, 490, 539, 530, 498, 527, 562, 527] payload_rows_per_dst=[5682, 5618, 5602, 5488, 5506, 5566, 5500, 5560] dup_rows_per_dst=[2742, 2480, 2678, 2522, 2564, 2690, 2544, 2794] combine_rows_per_src=[5682, 5618, 5602, 5488, 5506, 5566, 5500, 5560] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280096768 dispatch_meta_bytes=229888 combine_payload_bytes=280096768 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=280326656 ingress_bytes=610398208 link_bytes=610398208 token_bytes=560423424 weight_movement_bytes=330301440 total_nvlink_bytes=890724864 weight_to_token_ratio=0.589378 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061582877e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44436 payload_rows_remote=38888 dup_rows=21100 dup_rows_remote=18512 dedup_ratio=0.321960 router_tokens_per_expert=[489, 539, 507, 511, 517, 462, 510, 503, 532, 564, 504, 522, 509, 499, 508, 512, 524, 506, 445, 492, 549, 498, 504, 534, 502, 520, 495, 515, 477, 540, 507, 537, 532, 504, 502, 486, 514, 503, 532, 474, 517, 556, 558, 511, 522, 495, 533, 516, 517, 458, 493, 511, 515, 532, 505, 529, 498, 517, 537, 494, 543, 529, 518, 484] payload_rows_per_dst=[5548, 5660, 5528, 5546, 5504, 5530, 5550, 5570] dup_rows_per_dst=[2588, 2640, 2730, 2640, 2590, 2672, 2570, 2670] combine_rows_per_src=[5548, 5660, 5528, 5546, 5504, 5530, 5550, 5570] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279365632 dispatch_meta_bytes=229872 combine_payload_bytes=279365632 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279595504 ingress_bytes=389466112 link_bytes=389466112 token_bytes=558961136 weight_movement_bytes=110100480 total_nvlink_bytes=669061616 weight_to_token_ratio=0.196973 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773456249e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=277129216 dispatch_meta_bytes=227192 combine_payload_bytes=277129216 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277356408 ingress_bytes=387229696 link_bytes=387229696 token_bytes=554485624 weight_movement_bytes=110100480 total_nvlink_bytes=664586104 weight_to_token_ratio=0.198563 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.734561297e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279681024 dispatch_meta_bytes=230032 combine_payload_bytes=279681024 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279911056 ingress_bytes=499881984 link_bytes=499881984 token_bytes=559592080 weight_movement_bytes=220200960 total_nvlink_bytes=779793040 weight_to_token_ratio=0.393503 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.693769866e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279451648 dispatch_meta_bytes=228936 combine_payload_bytes=279451648 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279680584 ingress_bytes=389552128 link_bytes=389552128 token_bytes=559132232 weight_movement_bytes=110100480 total_nvlink_bytes=669232712 weight_to_token_ratio=0.196913 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.774952209e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280541184 dispatch_meta_bytes=229344 combine_payload_bytes=280541184 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280770528 ingress_bytes=610842624 link_bytes=610842624 token_bytes=561311712 weight_movement_bytes=330301440 total_nvlink_bytes=891613152 weight_to_token_ratio=0.588446 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.062355789e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280326144 dispatch_meta_bytes=230416 combine_payload_bytes=280326144 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280556560 ingress_bytes=390426624 link_bytes=390426624 token_bytes=560882704 weight_movement_bytes=110100480 total_nvlink_bytes=670983184 weight_to_token_ratio=0.196299 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.790161133e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278749184 dispatch_meta_bytes=229600 combine_payload_bytes=278749184 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278978784 ingress_bytes=498950144 link_bytes=498950144 token_bytes=557727968 weight_movement_bytes=220200960 total_nvlink_bytes=777928928 weight_to_token_ratio=0.394818 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.677563636e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44622 payload_rows_remote=39122 dup_rows=20914 dup_rows_remote=18258 dedup_ratio=0.319122 router_tokens_per_expert=[518, 540, 551, 508, 521, 542, 494, 501, 493, 536, 517, 528, 514, 488, 494, 508, 511, 559, 488, 532, 512, 506, 514, 528, 518, 514, 530, 503, 554, 503, 491, 499, 496, 501, 548, 521, 529, 536, 511, 531, 476, 474, 524, 487, 506, 494, 541, 494, 486, 525, 526, 480, 506, 525, 497, 502, 542, 497, 502, 491, 488, 517, 477, 523] payload_rows_per_dst=[5678, 5500, 5636, 5572, 5596, 5536, 5530, 5574] dup_rows_per_dst=[2672, 2656, 2664, 2652, 2750, 2456, 2564, 2500] combine_rows_per_src=[5678, 5500, 5636, 5572, 5596, 5536, 5530, 5574] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44448 payload_rows_remote=38892 dup_rows=21088 dup_rows_remote=18410 dedup_ratio=0.321777 router_tokens_per_expert=[510, 544, 533, 499, 557, 522, 534, 510, 504, 479, 517, 512, 491, 511, 492, 523, 541, 527, 515, 511, 509, 493, 508, 513, 515, 514, 529, 523, 510, 537, 520, 488, 528, 543, 520, 502, 519, 482, 483, 500, 497, 490, 527, 528, 454, 510, 510, 508, 519, 475, 510, 494, 516, 525, 502, 495, 501, 565, 531, 519, 504, 467, 533, 520] payload_rows_per_dst=[5628, 5500, 5556, 5568, 5492, 5560, 5540, 5604] dup_rows_per_dst=[2790, 2558, 2678, 2704, 2662, 2488, 2532, 2676] combine_rows_per_src=[5628, 5500, 5556, 5568, 5492, 5560, 5540, 5604] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44314 payload_rows_remote=38708 dup_rows=21222 dup_rows_remote=18308 dedup_ratio=0.323822 router_tokens_per_expert=[508, 509, 502, 486, 500, 533, 518, 505, 507, 533, 568, 481, 489, 499, 533, 521, 548, 524, 503, 511, 538, 525, 524, 482, 522, 519, 475, 521, 512, 535, 522, 519, 466, 531, 500, 545, 549, 514, 487, 497, 494, 496, 538, 491, 550, 503, 501, 521, 488, 487, 525, 497, 526, 500, 543, 502, 541, 487, 485, 482, 518, 457, 537, 538] payload_rows_per_dst=[5198, 5556, 5546, 5426, 5570, 5626, 5786, 5606] dup_rows_per_dst=[2056, 2614, 2578, 2480, 2624, 2738, 3218, 2914] combine_rows_per_src=[5198, 5556, 5546, 5426, 5570, 5626, 5786, 5606] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44420 payload_rows_remote=38842 dup_rows=21116 dup_rows_remote=18404 dedup_ratio=0.322205 router_tokens_per_expert=[509, 526, 536, 504, 534, 532, 480, 547, 498, 554, 517, 479, 519, 505, 571, 538, 517, 527, 507, 499, 521, 514, 511, 541, 514, 529, 525, 511, 556, 515, 482, 513, 531, 475, 508, 504, 511, 516, 503, 524, 490, 534, 515, 520, 509, 545, 528, 491, 483, 510, 501, 462, 500, 524, 494, 497, 482, 470, 485, 484, 543, 504, 499, 495] payload_rows_per_dst=[5620, 5612, 5620, 5578, 5576, 5540, 5446, 5428] dup_rows_per_dst=[2716, 2750, 2654, 2712, 2568, 2724, 2496, 2496] combine_rows_per_src=[5620, 5612, 5620, 5578, 5576, 5540, 5446, 5428] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44780 payload_rows_remote=39212 dup_rows=20756 dup_rows_remote=18082 dedup_ratio=0.316711 router_tokens_per_expert=[492, 509, 477, 526, 511, 524, 529, 515, 503, 538, 539, 525, 524, 465, 521, 459, 497, 519, 497, 535, 512, 498, 522, 487, 527, 522, 505, 490, 519, 502, 515, 499, 513, 492, 491, 513, 555, 526, 534, 510, 537, 528, 518, 520, 520, 503, 496, 499, 503, 511, 517, 504, 519, 512, 519, 518, 521, 540, 475, 521, 479, 515, 529, 527] payload_rows_per_dst=[5568, 5538, 5586, 5624, 5636, 5568, 5614, 5646] dup_rows_per_dst=[2598, 2610, 2548, 2534, 2632, 2674, 2592, 2568] combine_rows_per_src=[5568, 5538, 5586, 5624, 5636, 5568, 5614, 5646] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44438 payload_rows_remote=38842 dup_rows=21098 dup_rows_remote=18354 dedup_ratio=0.321930 router_tokens_per_expert=[518, 518, 497, 523, 527, 558, 522, 502, 525, 485, 507, 524, 556, 505, 526, 471, 507, 511, 489, 501, 526, 558, 513, 494, 532, 513, 486, 501, 493, 521, 499, 531, 536, 525, 515, 500, 538, 516, 527, 513, 497, 474, 474, 520, 503, 537, 516, 503, 508, 504, 508, 514, 491, 514, 507, 527, 516, 513, 501, 500, 502, 502, 511, 517] payload_rows_per_dst=[5608, 5500, 5570, 5566, 5596, 5526, 5536, 5536] dup_rows_per_dst=[2722, 2698, 2628, 2586, 2744, 2522, 2610, 2588] combine_rows_per_src=[5608, 5500, 5570, 5566, 5596, 5526, 5536, 5536] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=280426496 dispatch_meta_bytes=229520 combine_payload_bytes=280426496 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=280656016 ingress_bytes=610727936 link_bytes=610727936 token_bytes=561082512 weight_movement_bytes=330301440 total_nvlink_bytes=891383952 weight_to_token_ratio=0.588686 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.062156328e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44700 payload_rows_remote=39176 dup_rows=20836 dup_rows_remote=18378 dedup_ratio=0.317932 router_tokens_per_expert=[526, 538, 509, 480, 545, 532, 518, 526, 539, 457, 523, 545, 494, 541, 499, 549, 487, 500, 491, 511, 475, 502, 504, 508, 514, 515, 497, 563, 537, 507, 523, 534, 484, 501, 477, 516, 508, 501, 495, 491, 536, 522, 526, 504, 552, 490, 500, 520, 530, 496, 487, 518, 502, 477, 476, 505, 508, 502, 515, 545, 500, 535, 545, 515] payload_rows_per_dst=[5570, 5642, 5456, 5718, 5486, 5620, 5524, 5684] dup_rows_per_dst=[2778, 2652, 2500, 2662, 2460, 2680, 2458, 2646] combine_rows_per_src=[5570, 5642, 5456, 5718, 5486, 5620, 5524, 5684] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278777856 dispatch_meta_bytes=229208 combine_payload_bytes=278777856 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279007064 ingress_bytes=609079296 link_bytes=609079296 token_bytes=557784920 weight_movement_bytes=330301440 total_nvlink_bytes=888086360 weight_to_token_ratio=0.592166 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059289072e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278419456 dispatch_meta_bytes=228984 combine_payload_bytes=278419456 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278648440 ingress_bytes=498620416 link_bytes=498620416 token_bytes=557067896 weight_movement_bytes=220200960 total_nvlink_bytes=777268856 weight_to_token_ratio=0.395286 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.671829124e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277458944 dispatch_meta_bytes=228064 combine_payload_bytes=277458944 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277687008 ingress_bytes=387559424 link_bytes=387559424 token_bytes=555145952 weight_movement_bytes=110100480 total_nvlink_bytes=665246432 weight_to_token_ratio=0.198327 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.740295809e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=281071616 dispatch_meta_bytes=229176 combine_payload_bytes=281071616 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281300792 ingress_bytes=391172096 link_bytes=391172096 token_bytes=562372408 weight_movement_bytes=110100480 total_nvlink_bytes=672472888 weight_to_token_ratio=0.195779 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.803126117e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278419456 dispatch_meta_bytes=228784 combine_payload_bytes=278419456 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278648240 ingress_bytes=388519936 link_bytes=388519936 token_bytes=557067696 weight_movement_bytes=110100480 total_nvlink_bytes=667168176 weight_to_token_ratio=0.197643 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.757000693e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44422 payload_rows_remote=38904 dup_rows=21114 dup_rows_remote=18562 dedup_ratio=0.322174 router_tokens_per_expert=[485, 538, 502, 511, 514, 461, 506, 502, 533, 560, 504, 520, 506, 496, 506, 510, 521, 508, 446, 489, 546, 499, 501, 532, 501, 517, 495, 515, 475, 541, 509, 536, 532, 504, 498, 486, 518, 505, 533, 474, 518, 557, 560, 515, 524, 496, 533, 517, 522, 458, 493, 513, 519, 531, 507, 534, 500, 517, 539, 499, 545, 531, 519, 486] payload_rows_per_dst=[5518, 5688, 5568, 5540, 5510, 5468, 5556, 5574] dup_rows_per_dst=[2552, 2680, 2826, 2606, 2590, 2564, 2598, 2698] combine_rows_per_src=[5518, 5688, 5568, 5540, 5510, 5468, 5556, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279236608 dispatch_meta_bytes=229904 combine_payload_bytes=279236608 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279466512 ingress_bytes=389337088 link_bytes=389337088 token_bytes=558703120 weight_movement_bytes=110100480 total_nvlink_bytes=668803600 weight_to_token_ratio=0.197064 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.771212310e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280541184 dispatch_meta_bytes=229240 combine_payload_bytes=280541184 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=280770424 ingress_bytes=500742144 link_bytes=500742144 token_bytes=561311608 weight_movement_bytes=220200960 total_nvlink_bytes=781512568 weight_to_token_ratio=0.392297 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.708729463e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=277817344 dispatch_meta_bytes=228304 combine_payload_bytes=277817344 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278045648 ingress_bytes=387917824 link_bytes=387917824 token_bytes=555862992 weight_movement_bytes=110100480 total_nvlink_bytes=665963472 weight_to_token_ratio=0.198071 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.746528975e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279508992 dispatch_meta_bytes=228952 combine_payload_bytes=279508992 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279737944 ingress_bytes=389609472 link_bytes=389609472 token_bytes=559246936 weight_movement_bytes=110100480 total_nvlink_bytes=669347416 weight_to_token_ratio=0.196873 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.775949516e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280283136 dispatch_meta_bytes=230024 combine_payload_bytes=280283136 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280513160 ingress_bytes=390383616 link_bytes=390383616 token_bytes=560796296 weight_movement_bytes=110100480 total_nvlink_bytes=670896776 weight_to_token_ratio=0.196329 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.789413153e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280211456 dispatch_meta_bytes=230272 combine_payload_bytes=280211456 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280441728 ingress_bytes=390311936 link_bytes=390311936 token_bytes=560653184 weight_movement_bytes=110100480 total_nvlink_bytes=670753664 weight_to_token_ratio=0.196379 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.788166520e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279795712 dispatch_meta_bytes=230104 combine_payload_bytes=279795712 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=280025816 ingress_bytes=610097152 link_bytes=610097152 token_bytes=559821528 weight_movement_bytes=330301440 total_nvlink_bytes=890122968 weight_to_token_ratio=0.590012 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061059291e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278863872 dispatch_meta_bytes=229864 combine_payload_bytes=278863872 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279093736 ingress_bytes=499064832 link_bytes=499064832 token_bytes=557957608 weight_movement_bytes=220200960 total_nvlink_bytes=778158568 weight_to_token_ratio=0.394655 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.679558249e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44792 payload_rows_remote=39210 dup_rows=20744 dup_rows_remote=18060 dedup_ratio=0.316528 router_tokens_per_expert=[484, 509, 473, 525, 507, 521, 525, 513, 502, 533, 538, 522, 522, 464, 521, 459, 498, 517, 498, 534, 514, 498, 521, 486, 527, 520, 506, 489, 517, 505, 515, 499, 510, 493, 493, 513, 554, 525, 533, 510, 535, 531, 522, 519, 522, 506, 495, 503, 504, 515, 516, 508, 519, 515, 523, 519, 521, 543, 477, 523, 480, 520, 531, 528] payload_rows_per_dst=[5548, 5524, 5584, 5630, 5638, 5582, 5628, 5658] dup_rows_per_dst=[2566, 2598, 2548, 2526, 2624, 2684, 2610, 2588] combine_rows_per_src=[5548, 5524, 5584, 5630, 5638, 5582, 5628, 5658] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44426 payload_rows_remote=38878 dup_rows=21110 dup_rows_remote=18434 dedup_ratio=0.322113 router_tokens_per_expert=[508, 543, 532, 498, 555, 521, 532, 508, 503, 479, 517, 511, 490, 508, 491, 520, 541, 524, 514, 509, 509, 493, 510, 512, 510, 513, 526, 524, 507, 537, 517, 485, 526, 542, 520, 504, 522, 483, 482, 500, 497, 491, 528, 528, 454, 512, 514, 509, 521, 473, 510, 495, 517, 528, 504, 496, 501, 569, 532, 524, 507, 472, 539, 521] payload_rows_per_dst=[5616, 5490, 5548, 5554, 5490, 5566, 5542, 5620] dup_rows_per_dst=[2778, 2548, 2676, 2684, 2668, 2500, 2546, 2710] combine_rows_per_src=[5616, 5490, 5548, 5554, 5490, 5566, 5542, 5620] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44316 payload_rows_remote=38800 dup_rows=21220 dup_rows_remote=18508 dedup_ratio=0.323792 router_tokens_per_expert=[507, 508, 498, 486, 499, 531, 514, 505, 500, 533, 566, 477, 489, 499, 528, 519, 546, 524, 501, 511, 535, 522, 525, 480, 522, 516, 475, 520, 513, 534, 523, 523, 465, 533, 502, 549, 547, 513, 487, 498, 496, 493, 539, 492, 552, 504, 501, 524, 488, 487, 527, 501, 527, 500, 543, 503, 543, 488, 486, 487, 520, 458, 545, 541] payload_rows_per_dst=[5248, 5554, 5566, 5554, 5582, 5558, 5738, 5516] dup_rows_per_dst=[2182, 2640, 2630, 2698, 2634, 2644, 3080, 2712] combine_rows_per_src=[5248, 5554, 5566, 5554, 5582, 5558, 5738, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44422 payload_rows_remote=38828 dup_rows=21114 dup_rows_remote=18362 dedup_ratio=0.322174 router_tokens_per_expert=[514, 513, 497, 521, 526, 552, 523, 498, 522, 483, 505, 521, 552, 500, 522, 471, 507, 512, 486, 500, 523, 558, 518, 494, 534, 509, 481, 502, 495, 521, 499, 531, 537, 525, 511, 501, 538, 520, 527, 514, 497, 476, 473, 523, 505, 542, 518, 507, 510, 506, 515, 513, 494, 516, 508, 528, 518, 515, 503, 502, 502, 503, 513, 518] payload_rows_per_dst=[5580, 5478, 5572, 5560, 5594, 5548, 5544, 5546] dup_rows_per_dst=[2708, 2674, 2624, 2584, 2752, 2534, 2636, 2602] combine_rows_per_src=[5580, 5478, 5572, 5560, 5594, 5548, 5544, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44628 payload_rows_remote=39148 dup_rows=20908 dup_rows_remote=18282 dedup_ratio=0.319031 router_tokens_per_expert=[512, 537, 548, 507, 519, 539, 494, 499, 490, 529, 512, 527, 512, 487, 492, 504, 509, 559, 486, 531, 510, 508, 509, 524, 514, 511, 531, 503, 554, 504, 489, 502, 497, 502, 547, 524, 530, 536, 511, 528, 479, 476, 522, 488, 510, 496, 548, 497, 484, 524, 528, 487, 509, 528, 500, 502, 544, 500, 506, 493, 490, 521, 483, 526] payload_rows_per_dst=[5662, 5480, 5620, 5572, 5600, 5558, 5544, 5592] dup_rows_per_dst=[2648, 2626, 2652, 2644, 2750, 2474, 2580, 2534] combine_rows_per_src=[5662, 5480, 5620, 5572, 5600, 5558, 5544, 5592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44700 payload_rows_remote=39160 dup_rows=20836 dup_rows_remote=18368 dedup_ratio=0.317932 router_tokens_per_expert=[525, 532, 509, 478, 542, 530, 515, 525, 538, 458, 520, 540, 494, 539, 498, 546, 480, 496, 491, 509, 473, 502, 502, 510, 511, 515, 499, 563, 536, 508, 520, 536, 484, 498, 477, 520, 507, 505, 498, 492, 537, 521, 522, 507, 555, 490, 502, 522, 531, 496, 491, 519, 504, 478, 478, 507, 513, 502, 516, 546, 502, 539, 552, 517] payload_rows_per_dst=[5558, 5630, 5438, 5714, 5492, 5626, 5540, 5702] dup_rows_per_dst=[2754, 2636, 2488, 2662, 2470, 2686, 2468, 2672] combine_rows_per_src=[5558, 5630, 5438, 5714, 5492, 5626, 5540, 5702] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44404 payload_rows_remote=38840 dup_rows=21132 dup_rows_remote=18438 dedup_ratio=0.322449 router_tokens_per_expert=[506, 523, 533, 499, 533, 530, 475, 543, 496, 553, 515, 476, 517, 505, 568, 537, 516, 524, 510, 498, 521, 513, 510, 540, 514, 527, 525, 508, 552, 514, 481, 508, 533, 475, 508, 506, 509, 513, 506, 525, 492, 533, 521, 524, 509, 546, 532, 493, 483, 515, 502, 463, 503, 527, 496, 497, 485, 474, 489, 488, 549, 505, 500, 497] payload_rows_per_dst=[5592, 5596, 5606, 5564, 5582, 5558, 5458, 5448] dup_rows_per_dst=[2692, 2738, 2658, 2694, 2568, 2742, 2514, 2526] combine_rows_per_src=[5592, 5596, 5606, 5564, 5582, 5558, 5458, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=281057280 dispatch_meta_bytes=229080 combine_payload_bytes=281057280 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=281286360 ingress_bytes=501258240 link_bytes=501258240 token_bytes=562343640 weight_movement_bytes=220200960 total_nvlink_bytes=782544600 weight_to_token_ratio=0.391577 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.717705222e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278677504 dispatch_meta_bytes=229248 combine_payload_bytes=278677504 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278906752 ingress_bytes=498878464 link_bytes=498878464 token_bytes=557584256 weight_movement_bytes=220200960 total_nvlink_bytes=777785216 weight_to_token_ratio=0.394920 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.676317003e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278118400 dispatch_meta_bytes=229232 combine_payload_bytes=278118400 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278347632 ingress_bytes=388218880 link_bytes=388218880 token_bytes=556466032 weight_movement_bytes=110100480 total_nvlink_bytes=666566512 weight_to_token_ratio=0.197857 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.751764834e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44522 payload_rows_remote=38906 dup_rows=21014 dup_rows_remote=18296 dedup_ratio=0.320648 router_tokens_per_expert=[526, 485, 528, 487, 490, 541, 500, 514, 520, 489, 523, 541, 545, 515, 541, 497, 531, 513, 497, 540, 493, 551, 499, 534, 502, 489, 525, 530, 527, 519, 473, 512, 498, 495, 523, 542, 495, 506, 509, 504, 490, 499, 517, 520, 518, 502, 536, 557, 497, 498, 525, 465, 534, 527, 505, 494, 520, 526, 499, 503, 526, 518, 460, 483] payload_rows_per_dst=[5616, 5586, 5624, 5624, 5578, 5504, 5534, 5456] dup_rows_per_dst=[2718, 2564, 2692, 2530, 2566, 2774, 2556, 2614] combine_rows_per_src=[5616, 5586, 5624, 5624, 5578, 5504, 5534, 5456] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=280813568 dispatch_meta_bytes=230216 combine_payload_bytes=280813568 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281043784 ingress_bytes=390914048 link_bytes=390914048 token_bytes=561857352 weight_movement_bytes=110100480 total_nvlink_bytes=671957832 weight_to_token_ratio=0.195958 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.798638238e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44588 payload_rows_remote=38906 dup_rows=20948 dup_rows_remote=18198 dedup_ratio=0.319641 router_tokens_per_expert=[496, 472, 515, 544, 529, 525, 481, 539, 504, 554, 560, 511, 494, 582, 473, 538, 491, 532, 529, 517, 504, 523, 503, 525, 491, 496, 531, 535, 505, 537, 484, 505, 495, 456, 526, 463, 500, 536, 504, 503, 519, 485, 504, 531, 499, 492, 506, 475, 518, 524, 501, 524, 491, 518, 506, 513, 500, 534, 529, 533, 524, 497, 523, 514] payload_rows_per_dst=[5562, 5682, 5564, 5580, 5546, 5514, 5578, 5562] dup_rows_per_dst=[2640, 2750, 2684, 2588, 2420, 2508, 2612, 2746] combine_rows_per_src=[5562, 5682, 5564, 5580, 5546, 5514, 5578, 5562] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278878208 dispatch_meta_bytes=228808 combine_payload_bytes=278878208 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279107016 ingress_bytes=499079168 link_bytes=499079168 token_bytes=557985224 weight_movement_bytes=220200960 total_nvlink_bytes=778186184 weight_to_token_ratio=0.394636 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.679807576e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44582 payload_rows_remote=38998 dup_rows=20954 dup_rows_remote=18212 dedup_ratio=0.319733 router_tokens_per_expert=[509, 507, 524, 548, 509, 506, 515, 498, 528, 516, 504, 494, 527, 505, 467, 499, 538, 517, 519, 494, 494, 509, 483, 513, 517, 512, 522, 516, 526, 523, 513, 534, 528, 487, 525, 517, 505, 543, 538, 515, 563, 510, 521, 456, 513, 495, 498, 523, 505, 512, 524, 520, 511, 509, 485, 509, 501, 515, 574, 474, 485, 518, 489, 514] payload_rows_per_dst=[5586, 5612, 5482, 5584, 5650, 5566, 5562, 5540] dup_rows_per_dst=[2646, 2468, 2652, 2742, 2666, 2592, 2588, 2600] combine_rows_per_src=[5586, 5612, 5482, 5584, 5650, 5566, 5562, 5540] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44386 payload_rows_remote=38900 dup_rows=21150 dup_rows_remote=18518 dedup_ratio=0.322723 router_tokens_per_expert=[509, 523, 495, 508, 508, 546, 516, 486, 518, 495, 507, 516, 501, 558, 528, 525, 513, 500, 508, 505, 511, 474, 522, 526, 500, 533, 507, 511, 537, 533, 469, 511, 539, 487, 508, 507, 540, 535, 536, 478, 536, 521, 502, 475, 496, 542, 481, 524, 524, 505, 463, 531, 479, 523, 503, 496, 515, 497, 519, 529, 518, 523, 515, 522] payload_rows_per_dst=[5576, 5534, 5486, 5608, 5530, 5586, 5500, 5566] dup_rows_per_dst=[2606, 2762, 2632, 2594, 2730, 2568, 2548, 2710] combine_rows_per_src=[5576, 5534, 5486, 5608, 5530, 5586, 5500, 5566] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44340 payload_rows_remote=38720 dup_rows=21196 dup_rows_remote=18492 dedup_ratio=0.323425 router_tokens_per_expert=[564, 480, 508, 517, 470, 498, 503, 481, 477, 548, 492, 536, 504, 512, 490, 485, 537, 520, 546, 541, 516, 520, 479, 513, 551, 531, 525, 520, 522, 513, 509, 536, 485, 553, 540, 536, 486, 504, 520, 522, 517, 530, 520, 492, 513, 552, 520, 530, 489, 500, 515, 459, 532, 470, 507, 511, 466, 504, 530, 487, 503, 499, 511, 521] payload_rows_per_dst=[5314, 5410, 5580, 5358, 5622, 5828, 5608, 5620] dup_rows_per_dst=[2282, 2502, 2702, 2414, 2894, 2942, 2756, 2704] combine_rows_per_src=[5314, 5410, 5580, 5358, 5622, 5828, 5608, 5620] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44512 payload_rows_remote=39034 dup_rows=21024 dup_rows_remote=18386 dedup_ratio=0.320801 router_tokens_per_expert=[527, 541, 520, 532, 537, 523, 540, 518, 525, 469, 513, 516, 533, 478, 515, 538, 507, 488, 521, 511, 497, 522, 571, 520, 509, 478, 556, 486, 537, 552, 521, 506, 496, 478, 488, 479, 492, 486, 495, 482, 488, 509, 493, 501, 513, 566, 481, 507, 519, 478, 553, 533, 534, 534, 488, 503, 510, 510, 501, 528, 494, 499, 514, 509] payload_rows_per_dst=[5628, 5554, 5586, 5656, 5408, 5478, 5646, 5556] dup_rows_per_dst=[2848, 2620, 2688, 2634, 2384, 2638, 2638, 2574] combine_rows_per_src=[5628, 5554, 5586, 5656, 5408, 5478, 5646, 5556] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278878208 dispatch_meta_bytes=228416 combine_payload_bytes=278878208 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279106624 ingress_bytes=499079168 link_bytes=499079168 token_bytes=557984832 weight_movement_bytes=220200960 total_nvlink_bytes=778185792 weight_to_token_ratio=0.394636 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.679807576e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44480 payload_rows_remote=38968 dup_rows=21056 dup_rows_remote=18476 dedup_ratio=0.321289 router_tokens_per_expert=[482, 519, 551, 502, 522, 521, 542, 502, 520, 500, 494, 510, 485, 508, 501, 524, 501, 508, 515, 547, 527, 472, 491, 451, 523, 505, 490, 566, 564, 530, 494, 534, 489, 485, 571, 497, 506, 481, 517, 500, 498, 499, 561, 538, 487, 512, 511, 486, 519, 529, 507, 514, 511, 500, 521, 535, 527, 499, 496, 531, 543, 520, 474, 503] payload_rows_per_dst=[5634, 5514, 5464, 5648, 5512, 5582, 5600, 5526] dup_rows_per_dst=[2648, 2570, 2560, 2764, 2580, 2602, 2672, 2660] combine_rows_per_src=[5634, 5514, 5464, 5648, 5512, 5582, 5600, 5526] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44452 payload_rows_remote=39040 dup_rows=21084 dup_rows_remote=18458 dedup_ratio=0.321716 router_tokens_per_expert=[516, 494, 523, 528, 546, 503, 516, 538, 493, 524, 543, 527, 501, 488, 534, 523, 546, 533, 546, 497, 535, 509, 516, 478, 515, 483, 514, 502, 573, 515, 501, 515, 539, 501, 530, 518, 491, 554, 519, 499, 487, 542, 491, 531, 502, 479, 515, 473, 496, 515, 515, 513, 521, 498, 463, 498, 467, 527, 547, 483, 470, 466, 511, 532] payload_rows_per_dst=[5600, 5674, 5568, 5598, 5614, 5482, 5412, 5504] dup_rows_per_dst=[2728, 2592, 2752, 2638, 2688, 2558, 2626, 2502] combine_rows_per_src=[5600, 5674, 5568, 5598, 5614, 5482, 5412, 5504] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278319104 dispatch_meta_bytes=228760 combine_payload_bytes=278319104 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278547864 ingress_bytes=498520064 link_bytes=498520064 token_bytes=556866968 weight_movement_bytes=220200960 total_nvlink_bytes=777067928 weight_to_token_ratio=0.395428 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.670083838e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44548 payload_rows_remote=38892 dup_rows=20988 dup_rows_remote=18240 dedup_ratio=0.320251 router_tokens_per_expert=[523, 483, 526, 485, 487, 538, 498, 514, 517, 487, 522, 541, 544, 514, 540, 493, 533, 514, 496, 540, 494, 550, 499, 533, 500, 487, 526, 527, 525, 521, 472, 513, 497, 494, 524, 540, 495, 506, 511, 504, 489, 497, 519, 521, 519, 504, 535, 562, 499, 499, 527, 466, 534, 527, 506, 496, 523, 529, 498, 508, 527, 522, 461, 487] payload_rows_per_dst=[5656, 5652, 5628, 5528, 5564, 5500, 5540, 5480] dup_rows_per_dst=[2748, 2672, 2698, 2368, 2528, 2776, 2568, 2630] combine_rows_per_src=[5656, 5652, 5628, 5528, 5564, 5500, 5540, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=280612864 dispatch_meta_bytes=229720 combine_payload_bytes=280612864 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280842584 ingress_bytes=500813824 link_bytes=500813824 token_bytes=561455448 weight_movement_bytes=220200960 total_nvlink_bytes=781656408 weight_to_token_ratio=0.392197 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.709976097e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=280698880 dispatch_meta_bytes=230112 combine_payload_bytes=280698880 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280928992 ingress_bytes=390799360 link_bytes=390799360 token_bytes=561627872 weight_movement_bytes=110100480 total_nvlink_bytes=671728352 weight_to_token_ratio=0.196038 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.796643625e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278405120 dispatch_meta_bytes=229112 combine_payload_bytes=278405120 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278634232 ingress_bytes=498606080 link_bytes=498606080 token_bytes=557039352 weight_movement_bytes=220200960 total_nvlink_bytes=777240312 weight_to_token_ratio=0.395306 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.671579797e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278777856 dispatch_meta_bytes=228528 combine_payload_bytes=278777856 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279006384 ingress_bytes=498978816 link_bytes=498978816 token_bytes=557784240 weight_movement_bytes=220200960 total_nvlink_bytes=777985200 weight_to_token_ratio=0.394778 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.678062289e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44500 payload_rows_remote=39012 dup_rows=21036 dup_rows_remote=18380 dedup_ratio=0.320984 router_tokens_per_expert=[523, 535, 511, 527, 535, 521, 540, 513, 526, 465, 512, 515, 532, 478, 515, 535, 507, 486, 517, 510, 497, 519, 573, 519, 509, 477, 555, 486, 535, 552, 521, 504, 496, 480, 485, 481, 493, 488, 497, 482, 490, 510, 497, 502, 515, 571, 482, 505, 523, 479, 558, 536, 537, 536, 491, 506, 509, 512, 502, 532, 495, 503, 516, 509] payload_rows_per_dst=[5610, 5538, 5578, 5646, 5416, 5488, 5658, 5566] dup_rows_per_dst=[2800, 2618, 2678, 2632, 2388, 2656, 2674, 2590] combine_rows_per_src=[5610, 5538, 5578, 5646, 5416, 5488, 5658, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44396 payload_rows_remote=38906 dup_rows=21140 dup_rows_remote=18514 dedup_ratio=0.322571 router_tokens_per_expert=[506, 519, 493, 505, 504, 543, 513, 484, 516, 494, 504, 509, 499, 553, 527, 524, 510, 496, 510, 507, 511, 475, 520, 529, 500, 532, 508, 509, 537, 534, 469, 512, 537, 487, 507, 506, 539, 536, 535, 477, 538, 522, 504, 478, 498, 541, 481, 524, 523, 507, 462, 535, 483, 527, 503, 500, 518, 504, 521, 532, 520, 527, 519, 525] payload_rows_per_dst=[5554, 5524, 5490, 5614, 5520, 5596, 5508, 5590] dup_rows_per_dst=[2580, 2728, 2626, 2588, 2728, 2576, 2572, 2742] combine_rows_per_src=[5554, 5524, 5490, 5614, 5520, 5596, 5508, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44478 payload_rows_remote=38954 dup_rows=21058 dup_rows_remote=18470 dedup_ratio=0.321320 router_tokens_per_expert=[480, 518, 551, 500, 518, 520, 538, 496, 518, 499, 492, 506, 484, 505, 500, 521, 499, 507, 513, 544, 528, 469, 491, 453, 524, 505, 487, 565, 563, 529, 495, 535, 492, 487, 573, 497, 506, 480, 517, 504, 498, 502, 562, 538, 488, 515, 511, 488, 521, 532, 509, 516, 512, 505, 520, 536, 528, 502, 497, 531, 544, 523, 475, 506] payload_rows_per_dst=[5622, 5494, 5454, 5650, 5524, 5592, 5606, 5536] dup_rows_per_dst=[2620, 2556, 2554, 2756, 2588, 2612, 2696, 2676] combine_rows_per_src=[5622, 5494, 5454, 5650, 5524, 5592, 5606, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44450 payload_rows_remote=38906 dup_rows=21086 dup_rows_remote=18542 dedup_ratio=0.321747 router_tokens_per_expert=[562, 478, 505, 512, 470, 497, 501, 479, 475, 546, 492, 533, 503, 509, 486, 483, 537, 517, 546, 541, 514, 521, 477, 510, 549, 531, 524, 519, 521, 514, 509, 537, 489, 554, 539, 534, 490, 508, 520, 523, 519, 530, 522, 493, 516, 553, 519, 532, 489, 504, 515, 459, 531, 470, 508, 513, 469, 505, 534, 489, 505, 502, 512, 524] payload_rows_per_dst=[5478, 5508, 5600, 5384, 5614, 5782, 5540, 5544] dup_rows_per_dst=[2472, 2604, 2726, 2452, 2848, 2838, 2602, 2544] combine_rows_per_src=[5478, 5508, 5600, 5384, 5614, 5782, 5540, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44622 payload_rows_remote=38948 dup_rows=20914 dup_rows_remote=18190 dedup_ratio=0.319122 router_tokens_per_expert=[493, 468, 513, 540, 527, 523, 478, 537, 503, 551, 556, 509, 490, 580, 472, 538, 490, 530, 527, 518, 505, 523, 503, 526, 492, 499, 529, 533, 500, 537, 483, 508, 494, 457, 526, 461, 499, 535, 504, 506, 524, 489, 502, 530, 499, 494, 507, 474, 520, 525, 503, 526, 491, 519, 511, 515, 505, 535, 531, 534, 526, 505, 525, 515] payload_rows_per_dst=[5546, 5674, 5570, 5580, 5552, 5528, 5590, 5582] dup_rows_per_dst=[2612, 2724, 2674, 2582, 2412, 2510, 2630, 2770] combine_rows_per_src=[5546, 5674, 5570, 5580, 5552, 5528, 5590, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44448 payload_rows_remote=39022 dup_rows=21088 dup_rows_remote=18442 dedup_ratio=0.321777 router_tokens_per_expert=[513, 493, 516, 523, 543, 501, 515, 536, 493, 520, 544, 527, 498, 489, 533, 523, 541, 532, 546, 498, 534, 508, 513, 475, 515, 482, 511, 500, 573, 510, 500, 512, 539, 501, 530, 519, 495, 554, 523, 500, 488, 545, 493, 532, 505, 480, 517, 473, 500, 518, 515, 515, 522, 502, 466, 498, 468, 533, 547, 486, 472, 467, 512, 536] payload_rows_per_dst=[5584, 5672, 5550, 5590, 5618, 5494, 5426, 5514] dup_rows_per_dst=[2696, 2582, 2744, 2616, 2704, 2572, 2646, 2528] combine_rows_per_src=[5584, 5672, 5550, 5590, 5618, 5494, 5426, 5514] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279537664 dispatch_meta_bytes=228840 combine_payload_bytes=279537664 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279766504 ingress_bytes=609839104 link_bytes=609839104 token_bytes=559304168 weight_movement_bytes=330301440 total_nvlink_bytes=889605608 weight_to_token_ratio=0.590558 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060610503e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278835200 dispatch_meta_bytes=229672 combine_payload_bytes=278835200 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279064872 ingress_bytes=499036160 link_bytes=499036160 token_bytes=557900072 weight_movement_bytes=220200960 total_nvlink_bytes=778101032 weight_to_token_ratio=0.394696 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.679059596e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277544960 dispatch_meta_bytes=228848 combine_payload_bytes=277544960 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277773808 ingress_bytes=387645440 link_bytes=387645440 token_bytes=555318768 weight_movement_bytes=110100480 total_nvlink_bytes=665419248 weight_to_token_ratio=0.198265 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.741791769e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279795712 dispatch_meta_bytes=229680 combine_payload_bytes=279795712 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280025392 ingress_bytes=389896192 link_bytes=389896192 token_bytes=559821104 weight_movement_bytes=110100480 total_nvlink_bytes=669921584 weight_to_token_ratio=0.196671 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.780936048e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44264 payload_rows_remote=38708 dup_rows=21272 dup_rows_remote=18590 dedup_ratio=0.324585 router_tokens_per_expert=[499, 488, 522, 486, 495, 524, 573, 526, 546, 544, 500, 523, 537, 488, 513, 495, 491, 545, 507, 507, 504, 487, 525, 508, 515, 500, 562, 490, 506, 517, 502, 473, 521, 543, 540, 545, 517, 502, 484, 494, 520, 522, 494, 498, 498, 486, 504, 524, 509, 535, 486, 495, 499, 568, 501, 502, 496, 485, 528, 488, 528, 539, 518, 501] payload_rows_per_dst=[5556, 5592, 5486, 5492, 5528, 5446, 5572, 5592] dup_rows_per_dst=[2682, 2700, 2650, 2638, 2764, 2646, 2618, 2574] combine_rows_per_src=[5556, 5592, 5486, 5492, 5528, 5446, 5572, 5592] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279322624 dispatch_meta_bytes=229776 combine_payload_bytes=279322624 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279552400 ingress_bytes=389423104 link_bytes=389423104 token_bytes=558875024 weight_movement_bytes=110100480 total_nvlink_bytes=668975504 weight_to_token_ratio=0.197004 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.772708270e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279838720 dispatch_meta_bytes=229992 combine_payload_bytes=279838720 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280068712 ingress_bytes=389939200 link_bytes=389939200 token_bytes=559907432 weight_movement_bytes=110100480 total_nvlink_bytes=670007912 weight_to_token_ratio=0.196641 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.781684028e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44480 payload_rows_remote=38990 dup_rows=21056 dup_rows_remote=18368 dedup_ratio=0.321289 router_tokens_per_expert=[551, 512, 515, 496, 518, 520, 596, 513, 482, 522, 488, 532, 515, 499, 498, 553, 524, 530, 526, 510, 506, 492, 537, 495, 544, 465, 539, 523, 479, 542, 478, 511, 484, 553, 501, 557, 506, 513, 479, 475, 515, 499, 493, 510, 503, 492, 513, 510, 537, 529, 486, 510, 485, 520, 495, 504, 497, 531, 521, 521, 501, 501, 522, 494] payload_rows_per_dst=[5730, 5490, 5602, 5556, 5568, 5496, 5480, 5558] dup_rows_per_dst=[2712, 2688, 2638, 2606, 2568, 2574, 2652, 2618] combine_rows_per_src=[5730, 5490, 5602, 5556, 5568, 5496, 5480, 5558] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277458944 dispatch_meta_bytes=229192 combine_payload_bytes=277458944 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=277688136 ingress_bytes=607760384 link_bytes=607760384 token_bytes=555147080 weight_movement_bytes=330301440 total_nvlink_bytes=885448520 weight_to_token_ratio=0.594980 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.056995267e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44518 payload_rows_remote=38966 dup_rows=21018 dup_rows_remote=18362 dedup_ratio=0.320709 router_tokens_per_expert=[498, 502, 486, 517, 508, 521, 548, 530, 527, 500, 502, 547, 484, 543, 515, 495, 510, 530, 492, 513, 501, 490, 522, 516, 481, 515, 523, 536, 502, 539, 484, 524, 506, 521, 529, 517, 519, 536, 500, 523, 498, 484, 514, 519, 508, 506, 502, 519, 508, 507, 505, 520, 521, 532, 525, 506, 519, 527, 507, 483, 479, 526, 513, 488] payload_rows_per_dst=[5662, 5622, 5528, 5552, 5584, 5498, 5506, 5566] dup_rows_per_dst=[2558, 2604, 2620, 2656, 2718, 2602, 2742, 2518] combine_rows_per_src=[5662, 5622, 5528, 5552, 5584, 5498, 5506, 5566] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44372 payload_rows_remote=38860 dup_rows=21164 dup_rows_remote=18514 dedup_ratio=0.322937 router_tokens_per_expert=[540, 530, 530, 486, 535, 522, 512, 518, 520, 485, 489, 495, 512, 475, 518, 490, 519, 449, 517, 523, 504, 535, 539, 495, 501, 498, 489, 532, 535, 510, 528, 505, 546, 508, 527, 508, 508, 513, 505, 513, 482, 514, 517, 504, 487, 496, 511, 515, 523, 468, 502, 520, 507, 541, 511, 505, 510, 530, 539, 535, 515, 523, 519, 530] payload_rows_per_dst=[5674, 5524, 5512, 5494, 5578, 5480, 5490, 5620] dup_rows_per_dst=[2672, 2444, 2650, 2702, 2678, 2572, 2664, 2782] combine_rows_per_src=[5674, 5524, 5512, 5494, 5578, 5480, 5490, 5620] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44444 payload_rows_remote=38952 dup_rows=21092 dup_rows_remote=18606 dedup_ratio=0.321838 router_tokens_per_expert=[544, 536, 522, 522, 475, 504, 507, 486, 537, 501, 497, 528, 521, 485, 473, 530, 492, 493, 504, 509, 535, 532, 517, 539, 489, 547, 505, 495, 541, 501, 511, 524, 519, 537, 531, 523, 539, 554, 498, 506, 517, 501, 519, 526, 480, 500, 507, 530, 552, 523, 501, 503, 477, 526, 503, 507, 476, 486, 509, 536, 512, 515, 482, 471] payload_rows_per_dst=[5266, 5480, 5572, 5500, 5710, 5848, 5576, 5492] dup_rows_per_dst=[2202, 2664, 2670, 2726, 2660, 3036, 2648, 2486] combine_rows_per_src=[5266, 5480, 5572, 5500, 5710, 5848, 5576, 5492] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279480320 dispatch_meta_bytes=229432 combine_payload_bytes=279480320 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279709752 ingress_bytes=389580800 link_bytes=389580800 token_bytes=559190072 weight_movement_bytes=110100480 total_nvlink_bytes=669290552 weight_to_token_ratio=0.196893 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.775450862e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44342 payload_rows_remote=38830 dup_rows=21194 dup_rows_remote=18710 dedup_ratio=0.323395 router_tokens_per_expert=[521, 505, 522, 490, 512, 540, 526, 489, 513, 489, 522, 505, 512, 525, 498, 525, 513, 501, 501, 494, 474, 481, 547, 514, 547, 509, 493, 488, 498, 535, 518, 497, 502, 518, 496, 506, 513, 514, 511, 518, 476, 543, 486, 493, 468, 512, 515, 505, 530, 534, 500, 533, 508, 516, 535, 517, 507, 547, 515, 532, 526, 539, 512, 537] payload_rows_per_dst=[5582, 5496, 5446, 5556, 5568, 5512, 5540, 5642] dup_rows_per_dst=[2628, 2682, 2604, 2614, 2588, 2484, 2806, 2788] combine_rows_per_src=[5582, 5496, 5446, 5556, 5568, 5512, 5540, 5642] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44454 payload_rows_remote=38950 dup_rows=21082 dup_rows_remote=18410 dedup_ratio=0.321686 router_tokens_per_expert=[519, 533, 461, 505, 519, 526, 508, 527, 527, 516, 522, 553, 506, 504, 507, 520, 517, 513, 486, 507, 512, 536, 502, 507, 484, 509, 512, 507, 512, 500, 517, 482, 516, 515, 511, 537, 519, 516, 496, 542, 489, 458, 518, 535, 509, 459, 525, 546, 501, 483, 540, 512, 518, 527, 511, 496, 540, 491, 534, 535, 514, 456, 502, 561] payload_rows_per_dst=[5572, 5650, 5590, 5512, 5530, 5538, 5504, 5558] dup_rows_per_dst=[2624, 2660, 2570, 2534, 2774, 2540, 2672, 2708] combine_rows_per_src=[5572, 5650, 5590, 5512, 5530, 5538, 5504, 5558] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44452 payload_rows_remote=38926 dup_rows=21084 dup_rows_remote=18570 dedup_ratio=0.321716 router_tokens_per_expert=[533, 520, 516, 515, 529, 521, 495, 506, 497, 486, 515, 504, 515, 524, 529, 504, 502, 507, 467, 540, 530, 531, 477, 529, 529, 521, 489, 523, 500, 506, 503, 506, 501, 518, 484, 524, 527, 492, 497, 477, 523, 502, 514, 499, 508, 509, 511, 540, 519, 511, 530, 525, 521, 526, 538, 528, 498, 519, 502, 515, 467, 524, 506, 544] payload_rows_per_dst=[5552, 5540, 5484, 5502, 5526, 5618, 5668, 5562] dup_rows_per_dst=[2718, 2608, 2682, 2652, 2514, 2594, 2728, 2588] combine_rows_per_src=[5552, 5540, 5484, 5502, 5526, 5618, 5668, 5562] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279308288 dispatch_meta_bytes=229312 combine_payload_bytes=279308288 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279537600 ingress_bytes=389408768 link_bytes=389408768 token_bytes=558845888 weight_movement_bytes=110100480 total_nvlink_bytes=668946368 weight_to_token_ratio=0.197014 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.772458943e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279207936 dispatch_meta_bytes=230232 combine_payload_bytes=279207936 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279438168 ingress_bytes=609509376 link_bytes=609509376 token_bytes=558646104 weight_movement_bytes=330301440 total_nvlink_bytes=888947544 weight_to_token_ratio=0.591253 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060037052e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44464 payload_rows_remote=38808 dup_rows=21072 dup_rows_remote=18344 dedup_ratio=0.321533 router_tokens_per_expert=[505, 520, 512, 529, 489, 559, 544, 507, 516, 487, 520, 495, 564, 538, 483, 498, 501, 520, 503, 514, 527, 546, 482, 534, 494, 480, 494, 520, 479, 522, 536, 488, 483, 524, 475, 512, 508, 513, 519, 543, 515, 498, 520, 490, 517, 496, 509, 527, 513, 495, 513, 577, 496, 492, 517, 501, 535, 513, 537, 483, 538, 500, 490, 513] payload_rows_per_dst=[5656, 5668, 5574, 5612, 5510, 5556, 5324, 5564] dup_rows_per_dst=[2728, 2716, 2794, 2762, 2462, 2588, 2536, 2486] combine_rows_per_src=[5656, 5668, 5574, 5612, 5510, 5556, 5324, 5564] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278548480 dispatch_meta_bytes=229496 combine_payload_bytes=278548480 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278777976 ingress_bytes=388648960 link_bytes=388648960 token_bytes=557326456 weight_movement_bytes=110100480 total_nvlink_bytes=667426936 weight_to_token_ratio=0.197551 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.759244632e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278333440 dispatch_meta_bytes=230160 combine_payload_bytes=278333440 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278563600 ingress_bytes=388433920 link_bytes=388433920 token_bytes=556897040 weight_movement_bytes=110100480 total_nvlink_bytes=666997520 weight_to_token_ratio=0.197703 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.755504733e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279193600 dispatch_meta_bytes=229440 combine_payload_bytes=279193600 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279423040 ingress_bytes=499394560 link_bytes=499394560 token_bytes=558616640 weight_movement_bytes=220200960 total_nvlink_bytes=778817600 weight_to_token_ratio=0.394190 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.685292761e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279021568 dispatch_meta_bytes=229984 combine_payload_bytes=279021568 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279251552 ingress_bytes=609323008 link_bytes=609323008 token_bytes=558273120 weight_movement_bytes=330301440 total_nvlink_bytes=888574560 weight_to_token_ratio=0.591648 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059712927e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44396 payload_rows_remote=38830 dup_rows=21140 dup_rows_remote=18432 dedup_ratio=0.322571 router_tokens_per_expert=[482, 519, 507, 490, 541, 549, 494, 524, 532, 506, 512, 537, 503, 497, 498, 552, 534, 523, 500, 534, 520, 519, 514, 496, 487, 569, 525, 522, 500, 535, 522, 482, 532, 517, 520, 502, 472, 491, 489, 516, 484, 543, 508, 486, 514, 527, 499, 488, 505, 518, 489, 506, 522, 478, 490, 545, 497, 511, 505, 513, 531, 514, 524, 507] payload_rows_per_dst=[5592, 5566, 5578, 5616, 5484, 5500, 5514, 5546] dup_rows_per_dst=[2620, 2708, 2702, 2668, 2594, 2598, 2592, 2658] combine_rows_per_src=[5592, 5566, 5578, 5616, 5484, 5500, 5514, 5546] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278175744 dispatch_meta_bytes=228608 combine_payload_bytes=278175744 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278404352 ingress_bytes=498376704 link_bytes=498376704 token_bytes=556580096 weight_movement_bytes=220200960 total_nvlink_bytes=776781056 weight_to_token_ratio=0.395632 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.667590571e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44590 payload_rows_remote=38952 dup_rows=20946 dup_rows_remote=18342 dedup_ratio=0.319611 router_tokens_per_expert=[557, 545, 474, 512, 567, 522, 495, 506, 531, 508, 512, 482, 511, 544, 496, 518, 484, 532, 528, 481, 516, 535, 500, 489, 556, 503, 527, 516, 525, 501, 509, 484, 510, 549, 533, 541, 513, 485, 526, 513, 513, 510, 570, 515, 495, 469, 489, 520, 495, 551, 482, 497, 519, 454, 481, 524, 500, 523, 468, 529, 488, 499, 546, 495] payload_rows_per_dst=[5580, 5574, 5562, 5638, 5684, 5538, 5464, 5550] dup_rows_per_dst=[2776, 2630, 2568, 2604, 2656, 2624, 2542, 2546] combine_rows_per_src=[5580, 5574, 5562, 5638, 5684, 5538, 5464, 5550] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278333440 dispatch_meta_bytes=229048 combine_payload_bytes=278333440 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278562488 ingress_bytes=608634880 link_bytes=608634880 token_bytes=556895928 weight_movement_bytes=330301440 total_nvlink_bytes=887197368 weight_to_token_ratio=0.593112 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058516160e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44322 payload_rows_remote=38712 dup_rows=21214 dup_rows_remote=18558 dedup_ratio=0.323700 router_tokens_per_expert=[504, 483, 505, 497, 522, 497, 511, 518, 530, 497, 556, 551, 504, 485, 548, 511, 533, 506, 510, 519, 481, 499, 548, 542, 559, 495, 484, 527, 542, 500, 495, 507, 564, 507, 507, 496, 508, 483, 508, 511, 513, 522, 465, 529, 498, 469, 520, 503, 518, 533, 550, 536, 514, 499, 496, 490, 500, 495, 533, 492, 512, 511, 531, 489] payload_rows_per_dst=[5222, 5554, 5570, 5500, 5488, 5534, 5844, 5610] dup_rows_per_dst=[2178, 2766, 2698, 2588, 2548, 2678, 3102, 2656] combine_rows_per_src=[5222, 5554, 5570, 5500, 5488, 5534, 5844, 5610] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44510 payload_rows_remote=38920 dup_rows=21026 dup_rows_remote=18358 dedup_ratio=0.320831 router_tokens_per_expert=[497, 486, 506, 528, 528, 518, 515, 508, 527, 509, 503, 525, 466, 527, 529, 490, 499, 521, 519, 515, 531, 544, 491, 509, 514, 490, 516, 502, 538, 535, 544, 494, 497, 480, 518, 511, 494, 515, 522, 522, 495, 508, 511, 511, 531, 466, 577, 503, 518, 466, 475, 535, 500, 511, 524, 522, 532, 497, 531, 568, 482, 498, 506, 518] payload_rows_per_dst=[5504, 5556, 5590, 5602, 5566, 5560, 5538, 5594] dup_rows_per_dst=[2668, 2596, 2668, 2664, 2552, 2644, 2564, 2670] combine_rows_per_src=[5504, 5556, 5590, 5602, 5566, 5560, 5538, 5594] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44442 payload_rows_remote=38848 dup_rows=21094 dup_rows_remote=18544 dedup_ratio=0.321869 router_tokens_per_expert=[547, 535, 517, 492, 547, 489, 524, 551, 519, 499, 535, 488, 545, 510, 506, 502, 480, 500, 486, 502, 504, 509, 467, 502, 496, 501, 532, 561, 495, 496, 524, 517, 517, 536, 480, 523, 538, 543, 504, 491, 519, 504, 487, 519, 493, 484, 547, 513, 499, 533, 482, 502, 494, 505, 557, 500, 539, 496, 515, 492, 521, 500, 545, 512] payload_rows_per_dst=[5612, 5560, 5408, 5582, 5596, 5506, 5594, 5584] dup_rows_per_dst=[2792, 2648, 2492, 2662, 2668, 2626, 2550, 2656] combine_rows_per_src=[5612, 5560, 5408, 5582, 5596, 5506, 5594, 5584] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44462 payload_rows_remote=38884 dup_rows=21074 dup_rows_remote=18456 dedup_ratio=0.321564 router_tokens_per_expert=[475, 507, 525, 540, 495, 520, 543, 548, 521, 487, 527, 505, 554, 520, 520, 523, 525, 515, 487, 502, 528, 522, 524, 546, 504, 532, 477, 513, 522, 514, 508, 507, 495, 527, 488, 487, 529, 506, 537, 496, 493, 497, 538, 493, 530, 486, 543, 518, 475, 522, 486, 487, 461, 528, 530, 495, 469, 522, 518, 509, 521, 498, 554, 494] payload_rows_per_dst=[5618, 5630, 5498, 5530, 5514, 5578, 5512, 5582] dup_rows_per_dst=[2688, 2684, 2800, 2624, 2616, 2618, 2456, 2588] combine_rows_per_src=[5618, 5630, 5498, 5530, 5514, 5578, 5512, 5582] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44364 payload_rows_remote=38716 dup_rows=21172 dup_rows_remote=18384 dedup_ratio=0.323059 router_tokens_per_expert=[500, 496, 542, 528, 463, 510, 537, 515, 508, 500, 495, 499, 511, 508, 531, 515, 543, 530, 500, 519, 479, 472, 525, 492, 517, 524, 504, 505, 479, 535, 500, 494, 516, 568, 501, 527, 537, 554, 475, 540, 496, 527, 477, 546, 532, 540, 566, 523, 516, 507, 535, 448, 474, 526, 503, 533, 459, 518, 505, 504, 467, 537, 531, 504] payload_rows_per_dst=[5574, 5566, 5500, 5544, 5648, 5592, 5478, 5462] dup_rows_per_dst=[2608, 2568, 2620, 2572, 2788, 2822, 2606, 2588] combine_rows_per_src=[5574, 5566, 5500, 5544, 5648, 5592, 5478, 5462] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279207936 dispatch_meta_bytes=229176 combine_payload_bytes=279207936 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279437112 ingress_bytes=499408896 link_bytes=499408896 token_bytes=558645048 weight_movement_bytes=220200960 total_nvlink_bytes=778846008 weight_to_token_ratio=0.394170 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.685542088e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277487616 dispatch_meta_bytes=229080 combine_payload_bytes=277487616 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277716696 ingress_bytes=387588096 link_bytes=387588096 token_bytes=555204312 weight_movement_bytes=110100480 total_nvlink_bytes=665304792 weight_to_token_ratio=0.198306 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.740794462e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44498 payload_rows_remote=38824 dup_rows=21038 dup_rows_remote=18208 dedup_ratio=0.321014 router_tokens_per_expert=[499, 570, 482, 536, 527, 481, 550, 511, 469, 500, 519, 513, 539, 517, 479, 525, 510, 522, 511, 531, 512, 506, 534, 515, 513, 509, 515, 475, 498, 496, 544, 521, 522, 488, 530, 488, 480, 518, 500, 541, 503, 532, 492, 509, 495, 517, 514, 497, 533, 532, 493, 516, 506, 501, 508, 501, 515, 485, 545, 528, 521, 546, 484, 499] payload_rows_per_dst=[5674, 5404, 5642, 5542, 5562, 5522, 5552, 5600] dup_rows_per_dst=[2830, 2526, 2640, 2600, 2572, 2596, 2628, 2646] combine_rows_per_src=[5674, 5404, 5642, 5542, 5562, 5522, 5552, 5600] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278978560 dispatch_meta_bytes=229112 combine_payload_bytes=278978560 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279207672 ingress_bytes=499179520 link_bytes=499179520 token_bytes=558186232 weight_movement_bytes=220200960 total_nvlink_bytes=778387192 weight_to_token_ratio=0.394494 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.681552862e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278462464 dispatch_meta_bytes=229568 combine_payload_bytes=278462464 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278692032 ingress_bytes=388562944 link_bytes=388562944 token_bytes=557154496 weight_movement_bytes=110100480 total_nvlink_bytes=667254976 weight_to_token_ratio=0.197612 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.757748672e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278720512 dispatch_meta_bytes=229360 combine_payload_bytes=278720512 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278949872 ingress_bytes=388820992 link_bytes=388820992 token_bytes=557670384 weight_movement_bytes=110100480 total_nvlink_bytes=667770864 weight_to_token_ratio=0.197429 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.762236552e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44618 payload_rows_remote=39052 dup_rows=20918 dup_rows_remote=18276 dedup_ratio=0.319183 router_tokens_per_expert=[513, 549, 499, 495, 525, 549, 478, 532, 532, 524, 527, 482, 498, 536, 502, 503, 545, 468, 513, 519, 514, 568, 515, 535, 477, 492, 502, 499, 505, 500, 521, 522, 510, 533, 511, 507, 510, 485, 522, 529, 455, 528, 510, 530, 521, 522, 495, 482, 512, 506, 497, 507, 512, 575, 503, 525, 514, 515, 503, 479, 485, 504, 508, 534] payload_rows_per_dst=[5594, 5566, 5658, 5508, 5550, 5560, 5630, 5552] dup_rows_per_dst=[2686, 2642, 2696, 2528, 2664, 2526, 2644, 2532] combine_rows_per_src=[5594, 5566, 5658, 5508, 5550, 5560, 5630, 5552] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277516288 dispatch_meta_bytes=228400 combine_payload_bytes=277516288 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277744688 ingress_bytes=497717248 link_bytes=497717248 token_bytes=555260976 weight_movement_bytes=220200960 total_nvlink_bytes=775461936 weight_to_token_ratio=0.396572 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.656121547e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278290432 dispatch_meta_bytes=228128 combine_payload_bytes=278290432 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278518560 ingress_bytes=498491392 link_bytes=498491392 token_bytes=556808992 weight_movement_bytes=220200960 total_nvlink_bytes=777009952 weight_to_token_ratio=0.395469 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.669585184e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279924736 dispatch_meta_bytes=229312 combine_payload_bytes=279924736 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=280154048 ingress_bytes=610226176 link_bytes=610226176 token_bytes=560078784 weight_movement_bytes=330301440 total_nvlink_bytes=890380224 weight_to_token_ratio=0.589741 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061283685e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44454 payload_rows_remote=38916 dup_rows=21082 dup_rows_remote=18472 dedup_ratio=0.321686 router_tokens_per_expert=[552, 479, 552, 526, 492, 506, 502, 482, 524, 492, 516, 478, 525, 507, 505, 526, 513, 500, 507, 528, 525, 513, 515, 544, 500, 519, 508, 503, 515, 519, 526, 484, 517, 520, 497, 483, 538, 548, 518, 481, 526, 513, 488, 488, 504, 541, 517, 513, 562, 553, 548, 532, 525, 496, 507, 490, 503, 482, 505, 508, 459, 525, 505, 493] payload_rows_per_dst=[5514, 5510, 5648, 5538, 5552, 5504, 5650, 5538] dup_rows_per_dst=[2668, 2636, 2642, 2610, 2652, 2676, 2776, 2422] combine_rows_per_src=[5514, 5510, 5648, 5538, 5552, 5504, 5650, 5538] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44464 payload_rows_remote=38584 dup_rows=21072 dup_rows_remote=17928 dedup_ratio=0.321533 router_tokens_per_expert=[501, 489, 537, 482, 489, 503, 548, 494, 512, 478, 494, 526, 554, 517, 544, 511, 483, 515, 500, 548, 506, 483, 499, 487, 537, 530, 517, 527, 497, 501, 516, 490, 521, 489, 526, 504, 524, 514, 518, 499, 528, 489, 551, 513, 482, 531, 511, 487, 500, 524, 508, 508, 506, 531, 525, 520, 540, 518, 533, 541, 496, 524, 520, 472] payload_rows_per_dst=[5530, 5376, 5110, 5776, 5544, 5546, 5702, 5880] dup_rows_per_dst=[2556, 2298, 2068, 3006, 2654, 2638, 2708, 3144] combine_rows_per_src=[5530, 5376, 5110, 5776, 5544, 5546, 5702, 5880] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44588 payload_rows_remote=39006 dup_rows=20948 dup_rows_remote=18238 dedup_ratio=0.319641 router_tokens_per_expert=[525, 488, 488, 525, 555, 457, 533, 505, 533, 529, 507, 501, 501, 490, 514, 494, 525, 497, 501, 544, 537, 571, 510, 504, 519, 537, 505, 533, 469, 500, 536, 512, 502, 513, 504, 490, 518, 527, 529, 534, 520, 518, 514, 497, 485, 499, 524, 507, 472, 516, 536, 539, 539, 505, 552, 487, 501, 512, 498, 536, 468, 501, 483, 497] payload_rows_per_dst=[5630, 5500, 5668, 5528, 5624, 5544, 5582, 5512] dup_rows_per_dst=[2522, 2638, 2710, 2694, 2610, 2584, 2710, 2480] combine_rows_per_src=[5630, 5500, 5668, 5528, 5624, 5544, 5582, 5512] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44558 payload_rows_remote=38940 dup_rows=20978 dup_rows_remote=18190 dedup_ratio=0.320099 router_tokens_per_expert=[517, 551, 482, 530, 490, 502, 522, 515, 525, 505, 543, 507, 516, 510, 527, 489, 555, 534, 512, 504, 505, 547, 508, 538, 520, 488, 515, 516, 513, 495, 487, 508, 508, 504, 491, 524, 494, 503, 526, 506, 508, 518, 496, 472, 521, 516, 521, 504, 504, 499, 505, 560, 537, 527, 488, 519, 545, 496, 488, 531, 478, 496, 504, 503] payload_rows_per_dst=[5626, 5484, 5618, 5558, 5538, 5562, 5652, 5520] dup_rows_per_dst=[2592, 2760, 2788, 2526, 2574, 2550, 2626, 2562] combine_rows_per_src=[5626, 5484, 5618, 5558, 5538, 5562, 5652, 5520] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44428 payload_rows_remote=39006 dup_rows=21108 dup_rows_remote=18632 dedup_ratio=0.322083 router_tokens_per_expert=[525, 489, 529, 524, 491, 494, 513, 502, 503, 497, 528, 560, 525, 507, 548, 552, 490, 527, 527, 536, 507, 480, 484, 496, 482, 465, 542, 558, 519, 474, 521, 506, 530, 549, 515, 520, 546, 535, 459, 543, 504, 506, 525, 494, 490, 479, 471, 480, 532, 536, 515, 515, 539, 520, 516, 521, 512, 512, 512, 476, 503, 532, 462, 518] payload_rows_per_dst=[5460, 5666, 5554, 5470, 5658, 5422, 5698, 5500] dup_rows_per_dst=[2674, 2774, 2540, 2664, 2736, 2476, 2690, 2554] combine_rows_per_src=[5460, 5666, 5554, 5470, 5658, 5422, 5698, 5500] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44390 payload_rows_remote=38878 dup_rows=21146 dup_rows_remote=18550 dedup_ratio=0.322662 router_tokens_per_expert=[511, 527, 492, 488, 553, 516, 527, 483, 498, 526, 509, 512, 514, 503, 530, 530, 511, 551, 500, 597, 528, 528, 498, 530, 531, 482, 497, 494, 536, 463, 507, 527, 555, 484, 525, 473, 511, 515, 486, 505, 491, 495, 494, 476, 525, 541, 491, 531, 516, 537, 506, 478, 544, 510, 487, 505, 514, 503, 535, 511, 508, 516, 511, 490] payload_rows_per_dst=[5574, 5574, 5650, 5464, 5512, 5482, 5576, 5558] dup_rows_per_dst=[2620, 2670, 2836, 2610, 2596, 2606, 2590, 2618] combine_rows_per_src=[5574, 5574, 5650, 5464, 5512, 5482, 5576, 5558] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278949888 dispatch_meta_bytes=229552 combine_payload_bytes=278949888 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279179440 ingress_bytes=389050368 link_bytes=389050368 token_bytes=558129328 weight_movement_bytes=110100480 total_nvlink_bytes=668229808 weight_to_token_ratio=0.197267 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.766225777e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44460 payload_rows_remote=38824 dup_rows=21076 dup_rows_remote=18422 dedup_ratio=0.321594 router_tokens_per_expert=[517, 513, 475, 494, 493, 500, 493, 492, 521, 559, 488, 513, 521, 531, 560, 528, 574, 526, 527, 511, 496, 478, 508, 555, 525, 539, 468, 522, 520, 478, 469, 507, 490, 523, 524, 521, 480, 518, 464, 516, 521, 542, 532, 502, 523, 507, 499, 514, 480, 534, 535, 521, 521, 513, 523, 482, 513, 501, 508, 540, 499, 493, 517, 511] payload_rows_per_dst=[5636, 5610, 5578, 5508, 5472, 5876, 5206, 5574] dup_rows_per_dst=[2654, 2832, 2772, 2548, 2600, 2984, 2096, 2590] combine_rows_per_src=[5636, 5610, 5578, 5508, 5472, 5876, 5206, 5574] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44394 payload_rows_remote=38882 dup_rows=21142 dup_rows_remote=18456 dedup_ratio=0.322601 router_tokens_per_expert=[523, 492, 526, 510, 506, 521, 517, 493, 514, 500, 524, 542, 492, 519, 505, 503, 517, 537, 518, 530, 513, 529, 529, 523, 465, 490, 549, 497, 521, 498, 536, 496, 552, 503, 519, 516, 505, 529, 492, 516, 508, 496, 491, 534, 548, 536, 474, 506, 528, 510, 547, 524, 545, 502, 515, 486, 474, 524, 497, 495, 517, 470, 497, 477] payload_rows_per_dst=[5532, 5512, 5698, 5484, 5596, 5470, 5656, 5446] dup_rows_per_dst=[2644, 2686, 2694, 2620, 2668, 2716, 2658, 2456] combine_rows_per_src=[5532, 5512, 5698, 5484, 5596, 5470, 5656, 5446] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=276570112 dispatch_meta_bytes=226048 combine_payload_bytes=276570112 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276796160 ingress_bytes=386670592 link_bytes=386670592 token_bytes=553366272 weight_movement_bytes=110100480 total_nvlink_bytes=663466752 weight_to_token_ratio=0.198965 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.724837559e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279595008 dispatch_meta_bytes=228976 combine_payload_bytes=279595008 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279823984 ingress_bytes=609896448 link_bytes=609896448 token_bytes=559418992 weight_movement_bytes=330301440 total_nvlink_bytes=889720432 weight_to_token_ratio=0.590437 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060710234e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44580 payload_rows_remote=38996 dup_rows=20956 dup_rows_remote=18218 dedup_ratio=0.319763 router_tokens_per_expert=[506, 503, 518, 545, 507, 504, 514, 495, 526, 514, 501, 492, 526, 504, 467, 495, 537, 510, 518, 491, 493, 510, 482, 509, 517, 509, 522, 513, 529, 521, 514, 536, 529, 488, 526, 517, 505, 542, 536, 516, 566, 514, 523, 457, 514, 495, 501, 524, 506, 515, 526, 520, 512, 510, 485, 514, 503, 519, 578, 477, 488, 524, 492, 518] payload_rows_per_dst=[5566, 5602, 5466, 5584, 5660, 5572, 5566, 5564] dup_rows_per_dst=[2618, 2448, 2634, 2738, 2658, 2616, 2610, 2634] combine_rows_per_src=[5566, 5602, 5466, 5584, 5660, 5572, 5566, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278878208 dispatch_meta_bytes=229680 combine_payload_bytes=278878208 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279107888 ingress_bytes=388978688 link_bytes=388978688 token_bytes=557986096 weight_movement_bytes=110100480 total_nvlink_bytes=668086576 weight_to_token_ratio=0.197318 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.764979144e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44286 payload_rows_remote=38762 dup_rows=21250 dup_rows_remote=18596 dedup_ratio=0.324249 router_tokens_per_expert=[495, 486, 517, 484, 490, 523, 570, 522, 545, 544, 497, 519, 533, 487, 514, 493, 491, 542, 505, 508, 504, 486, 526, 507, 514, 501, 563, 488, 506, 516, 502, 474, 522, 542, 541, 544, 516, 505, 488, 495, 521, 520, 494, 499, 501, 487, 504, 526, 508, 537, 489, 497, 500, 570, 500, 505, 502, 485, 530, 492, 530, 541, 520, 505] payload_rows_per_dst=[5524, 5584, 5510, 5494, 5538, 5456, 5584, 5596] dup_rows_per_dst=[2654, 2680, 2682, 2630, 2768, 2648, 2628, 2560] combine_rows_per_src=[5524, 5584, 5510, 5494, 5538, 5456, 5584, 5596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279638016 dispatch_meta_bytes=229568 combine_payload_bytes=279638016 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279867584 ingress_bytes=389738496 link_bytes=389738496 token_bytes=559505600 weight_movement_bytes=110100480 total_nvlink_bytes=669606080 weight_to_token_ratio=0.196782 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778193455e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279222272 dispatch_meta_bytes=229696 combine_payload_bytes=279222272 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279451968 ingress_bytes=389322752 link_bytes=389322752 token_bytes=558674240 weight_movement_bytes=110100480 total_nvlink_bytes=668774720 weight_to_token_ratio=0.197075 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.770962983e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279709696 dispatch_meta_bytes=229856 combine_payload_bytes=279709696 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279939552 ingress_bytes=389810176 link_bytes=389810176 token_bytes=559649248 weight_movement_bytes=110100480 total_nvlink_bytes=669749728 weight_to_token_ratio=0.196731 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.779440088e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278878208 dispatch_meta_bytes=229792 combine_payload_bytes=278878208 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279108000 ingress_bytes=499079168 link_bytes=499079168 token_bytes=557986208 weight_movement_bytes=220200960 total_nvlink_bytes=778187168 weight_to_token_ratio=0.394635 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.679807576e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279179264 dispatch_meta_bytes=228552 combine_payload_bytes=279179264 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279407816 ingress_bytes=389279744 link_bytes=389279744 token_bytes=558587080 weight_movement_bytes=110100480 total_nvlink_bytes=668687560 weight_to_token_ratio=0.197105 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.770215003e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279523328 dispatch_meta_bytes=228856 combine_payload_bytes=279523328 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=279752184 ingress_bytes=830025728 link_bytes=830025728 token_bytes=559275512 weight_movement_bytes=550502400 total_nvlink_bytes=1109777912 weight_to_token_ratio=0.984313 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.443551257e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277846016 dispatch_meta_bytes=229432 combine_payload_bytes=277846016 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278075448 ingress_bytes=498046976 link_bytes=498046976 token_bytes=555921464 weight_movement_bytes=220200960 total_nvlink_bytes=776122424 weight_to_token_ratio=0.396101 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.661856059e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44368 payload_rows_remote=38860 dup_rows=21168 dup_rows_remote=18532 dedup_ratio=0.322998 router_tokens_per_expert=[535, 528, 525, 483, 529, 521, 509, 517, 519, 484, 490, 492, 511, 475, 516, 489, 517, 448, 515, 523, 503, 531, 538, 497, 499, 499, 487, 530, 536, 507, 525, 506, 546, 508, 527, 509, 508, 512, 507, 513, 484, 513, 518, 508, 488, 499, 510, 515, 522, 471, 507, 522, 506, 544, 511, 510, 512, 532, 543, 542, 517, 527, 521, 532] payload_rows_per_dst=[5650, 5514, 5508, 5484, 5582, 5480, 5506, 5644] dup_rows_per_dst=[2644, 2438, 2636, 2694, 2678, 2590, 2680, 2808] combine_rows_per_src=[5650, 5514, 5508, 5484, 5582, 5480, 5506, 5644] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44310 payload_rows_remote=38788 dup_rows=21226 dup_rows_remote=18734 dedup_ratio=0.323883 router_tokens_per_expert=[520, 500, 518, 488, 508, 537, 527, 483, 508, 486, 522, 502, 508, 522, 494, 524, 512, 502, 499, 494, 477, 481, 546, 513, 546, 509, 493, 490, 498, 536, 520, 499, 502, 517, 496, 506, 512, 514, 511, 517, 477, 544, 486, 495, 466, 512, 519, 508, 535, 536, 500, 533, 510, 518, 539, 520, 506, 551, 517, 535, 529, 543, 514, 538] payload_rows_per_dst=[5546, 5474, 5440, 5560, 5566, 5522, 5552, 5650] dup_rows_per_dst=[2616, 2658, 2608, 2622, 2584, 2492, 2830, 2816] combine_rows_per_src=[5546, 5474, 5440, 5560, 5566, 5522, 5552, 5650] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44442 payload_rows_remote=38922 dup_rows=21094 dup_rows_remote=18570 dedup_ratio=0.321869 router_tokens_per_expert=[528, 518, 511, 513, 527, 519, 494, 504, 494, 485, 514, 500, 515, 521, 525, 506, 502, 506, 460, 538, 530, 532, 478, 528, 530, 517, 486, 525, 499, 503, 503, 506, 500, 517, 485, 525, 529, 493, 497, 476, 523, 503, 516, 498, 509, 510, 512, 544, 522, 514, 531, 528, 521, 528, 542, 532, 498, 523, 505, 518, 468, 528, 509, 547] payload_rows_per_dst=[5530, 5528, 5478, 5498, 5520, 5626, 5682, 5580] dup_rows_per_dst=[2698, 2592, 2670, 2640, 2524, 2604, 2754, 2612] combine_rows_per_src=[5530, 5528, 5478, 5498, 5520, 5626, 5682, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44470 payload_rows_remote=38948 dup_rows=21066 dup_rows_remote=18382 dedup_ratio=0.321442 router_tokens_per_expert=[518, 530, 459, 503, 514, 523, 506, 522, 523, 516, 519, 552, 506, 503, 503, 517, 513, 508, 486, 507, 513, 538, 499, 509, 483, 508, 510, 508, 513, 499, 520, 483, 518, 514, 512, 539, 520, 516, 495, 544, 489, 459, 520, 535, 510, 462, 525, 547, 504, 483, 539, 516, 522, 529, 512, 498, 541, 492, 536, 535, 520, 459, 503, 563] payload_rows_per_dst=[5554, 5642, 5586, 5508, 5540, 5546, 5522, 5572] dup_rows_per_dst=[2596, 2636, 2560, 2540, 2776, 2548, 2684, 2726] combine_rows_per_src=[5554, 5642, 5586, 5508, 5540, 5546, 5522, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44528 payload_rows_remote=39014 dup_rows=21008 dup_rows_remote=18500 dedup_ratio=0.320557 router_tokens_per_expert=[541, 535, 517, 519, 472, 503, 506, 486, 534, 498, 496, 528, 520, 482, 474, 527, 493, 493, 498, 509, 535, 531, 514, 535, 488, 546, 504, 496, 539, 499, 513, 524, 520, 534, 533, 523, 536, 553, 496, 506, 517, 503, 521, 524, 479, 501, 511, 532, 556, 525, 504, 503, 482, 527, 508, 508, 479, 487, 513, 541, 513, 515, 488, 475] payload_rows_per_dst=[5384, 5476, 5568, 5662, 5710, 5612, 5602, 5514] dup_rows_per_dst=[2368, 2642, 2648, 2962, 2662, 2564, 2654, 2508] combine_rows_per_src=[5384, 5476, 5568, 5662, 5710, 5612, 5602, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44492 payload_rows_remote=39014 dup_rows=21044 dup_rows_remote=18370 dedup_ratio=0.321106 router_tokens_per_expert=[546, 510, 510, 493, 514, 519, 593, 510, 480, 519, 487, 531, 514, 496, 497, 552, 523, 529, 525, 507, 506, 490, 537, 496, 543, 465, 539, 522, 477, 541, 477, 507, 484, 550, 501, 564, 506, 516, 479, 476, 515, 497, 494, 509, 504, 493, 515, 511, 535, 528, 489, 512, 490, 518, 502, 510, 498, 534, 524, 523, 507, 503, 527, 499] payload_rows_per_dst=[5708, 5478, 5596, 5544, 5574, 5504, 5500, 5588] dup_rows_per_dst=[2682, 2674, 2630, 2598, 2578, 2572, 2668, 2642] combine_rows_per_src=[5708, 5478, 5596, 5544, 5574, 5504, 5500, 5588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44502 payload_rows_remote=38954 dup_rows=21034 dup_rows_remote=18388 dedup_ratio=0.320953 router_tokens_per_expert=[495, 502, 484, 518, 508, 521, 546, 528, 524, 499, 498, 545, 478, 542, 515, 495, 508, 528, 492, 512, 502, 486, 519, 516, 479, 513, 524, 534, 502, 540, 482, 523, 504, 523, 529, 517, 517, 536, 505, 522, 497, 485, 515, 519, 509, 506, 505, 520, 513, 510, 506, 520, 523, 535, 527, 507, 520, 530, 512, 484, 481, 530, 515, 488] payload_rows_per_dst=[5656, 5608, 5512, 5548, 5584, 5502, 5514, 5578] dup_rows_per_dst=[2548, 2584, 2614, 2646, 2722, 2610, 2768, 2542] combine_rows_per_src=[5656, 5608, 5512, 5548, 5584, 5502, 5514, 5578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278548480 dispatch_meta_bytes=229568 combine_payload_bytes=278548480 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278778048 ingress_bytes=388648960 link_bytes=388648960 token_bytes=557326528 weight_movement_bytes=110100480 total_nvlink_bytes=667427008 weight_to_token_ratio=0.197551 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.759244632e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44408 payload_rows_remote=38762 dup_rows=21128 dup_rows_remote=18418 dedup_ratio=0.322388 router_tokens_per_expert=[502, 515, 509, 527, 486, 557, 538, 507, 515, 486, 520, 494, 562, 534, 481, 496, 498, 519, 500, 514, 527, 545, 484, 531, 495, 477, 496, 525, 479, 520, 535, 488, 483, 524, 477, 512, 513, 514, 522, 544, 514, 499, 521, 491, 519, 496, 510, 527, 514, 496, 513, 578, 499, 493, 518, 504, 536, 514, 536, 488, 542, 501, 493, 515] payload_rows_per_dst=[5646, 5746, 5610, 5474, 5494, 5782, 5216, 5440] dup_rows_per_dst=[2710, 2846, 2872, 2556, 2442, 2982, 2404, 2316] combine_rows_per_src=[5646, 5746, 5610, 5474, 5494, 5782, 5216, 5440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278032384 dispatch_meta_bytes=230088 combine_payload_bytes=278032384 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278262472 ingress_bytes=388132864 link_bytes=388132864 token_bytes=556294856 weight_movement_bytes=110100480 total_nvlink_bytes=666395336 weight_to_token_ratio=0.197917 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.750268874e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278992896 dispatch_meta_bytes=229968 combine_payload_bytes=278992896 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279222864 ingress_bytes=609294336 link_bytes=609294336 token_bytes=558215760 weight_movement_bytes=330301440 total_nvlink_bytes=888517200 weight_to_token_ratio=0.591709 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059663062e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279179264 dispatch_meta_bytes=229320 combine_payload_bytes=279179264 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279408584 ingress_bytes=499380224 link_bytes=499380224 token_bytes=558587848 weight_movement_bytes=220200960 total_nvlink_bytes=778788808 weight_to_token_ratio=0.394210 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.685043435e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279652352 dispatch_meta_bytes=230056 combine_payload_bytes=279652352 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279882408 ingress_bytes=609953792 link_bytes=609953792 token_bytes=559534760 weight_movement_bytes=330301440 total_nvlink_bytes=889836200 weight_to_token_ratio=0.590314 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060809964e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279652352 dispatch_meta_bytes=229536 combine_payload_bytes=279652352 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279881888 ingress_bytes=389752832 link_bytes=389752832 token_bytes=559534240 weight_movement_bytes=110100480 total_nvlink_bytes=669634720 weight_to_token_ratio=0.196772 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778442782e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279222272 dispatch_meta_bytes=229368 combine_payload_bytes=279222272 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279451640 ingress_bytes=389322752 link_bytes=389322752 token_bytes=558673912 weight_movement_bytes=110100480 total_nvlink_bytes=668774392 weight_to_token_ratio=0.197075 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.770962983e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277846016 dispatch_meta_bytes=228720 combine_payload_bytes=277846016 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278074736 ingress_bytes=498046976 link_bytes=498046976 token_bytes=555920752 weight_movement_bytes=220200960 total_nvlink_bytes=776121712 weight_to_token_ratio=0.396101 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.661856059e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44496 payload_rows_remote=38910 dup_rows=21040 dup_rows_remote=18378 dedup_ratio=0.321045 router_tokens_per_expert=[494, 486, 502, 526, 526, 516, 514, 505, 522, 506, 503, 525, 464, 526, 529, 489, 498, 517, 520, 514, 531, 544, 489, 511, 514, 491, 515, 501, 536, 535, 542, 487, 499, 482, 519, 512, 495, 514, 522, 522, 495, 507, 514, 512, 531, 464, 577, 503, 518, 470, 476, 538, 504, 515, 526, 522, 535, 499, 531, 570, 483, 502, 511, 522] payload_rows_per_dst=[5488, 5540, 5586, 5590, 5570, 5566, 5552, 5604] dup_rows_per_dst=[2650, 2588, 2662, 2652, 2560, 2640, 2586, 2702] combine_rows_per_src=[5488, 5540, 5586, 5590, 5570, 5566, 5552, 5604] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44444 payload_rows_remote=38864 dup_rows=21092 dup_rows_remote=18466 dedup_ratio=0.321838 router_tokens_per_expert=[469, 504, 518, 537, 495, 517, 542, 545, 519, 487, 518, 504, 553, 517, 519, 522, 527, 512, 488, 501, 528, 522, 526, 546, 500, 534, 476, 512, 523, 512, 509, 508, 496, 528, 488, 489, 531, 506, 538, 497, 491, 495, 538, 494, 533, 489, 543, 520, 476, 524, 486, 487, 463, 530, 533, 499, 469, 523, 522, 512, 521, 502, 556, 499] payload_rows_per_dst=[5596, 5612, 5502, 5520, 5520, 5580, 5516, 5598] dup_rows_per_dst=[2658, 2666, 2798, 2628, 2626, 2626, 2480, 2610] combine_rows_per_src=[5596, 5612, 5502, 5520, 5520, 5580, 5516, 5598] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44334 payload_rows_remote=38686 dup_rows=21202 dup_rows_remote=18408 dedup_ratio=0.323517 router_tokens_per_expert=[499, 493, 539, 523, 463, 506, 537, 514, 507, 499, 495, 498, 509, 506, 531, 514, 541, 529, 498, 518, 478, 470, 523, 490, 518, 524, 501, 502, 478, 535, 499, 492, 514, 568, 504, 528, 537, 554, 475, 541, 499, 525, 476, 551, 535, 542, 568, 527, 518, 509, 535, 450, 475, 530, 507, 534, 461, 520, 506, 505, 469, 538, 532, 506] payload_rows_per_dst=[5558, 5550, 5490, 5532, 5648, 5602, 5482, 5472] dup_rows_per_dst=[2590, 2568, 2604, 2566, 2794, 2844, 2634, 2602] combine_rows_per_src=[5558, 5550, 5490, 5532, 5648, 5602, 5482, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44448 payload_rows_remote=38834 dup_rows=21088 dup_rows_remote=18518 dedup_ratio=0.321777 router_tokens_per_expert=[544, 533, 512, 490, 545, 486, 523, 549, 516, 497, 532, 489, 545, 507, 504, 499, 476, 497, 487, 499, 502, 509, 465, 502, 497, 501, 530, 559, 498, 496, 523, 511, 516, 538, 480, 522, 538, 545, 505, 495, 522, 504, 485, 522, 495, 484, 548, 517, 506, 537, 483, 507, 495, 505, 558, 501, 542, 497, 517, 493, 522, 501, 549, 516] payload_rows_per_dst=[5596, 5550, 5406, 5572, 5600, 5508, 5614, 5602] dup_rows_per_dst=[2768, 2628, 2468, 2658, 2678, 2646, 2570, 2672] combine_rows_per_src=[5596, 5550, 5406, 5572, 5600, 5508, 5614, 5602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44308 payload_rows_remote=38726 dup_rows=21228 dup_rows_remote=18614 dedup_ratio=0.323914 router_tokens_per_expert=[503, 481, 503, 495, 522, 493, 509, 513, 527, 496, 551, 547, 503, 483, 547, 508, 533, 507, 508, 517, 479, 498, 546, 542, 560, 494, 481, 528, 542, 499, 491, 506, 561, 507, 508, 495, 511, 483, 507, 512, 517, 522, 468, 529, 499, 468, 520, 504, 518, 535, 556, 536, 518, 501, 502, 491, 505, 501, 533, 496, 515, 513, 534, 491] payload_rows_per_dst=[5418, 5558, 5596, 5542, 5356, 5520, 5736, 5582] dup_rows_per_dst=[2440, 2766, 2782, 2640, 2420, 2596, 2970, 2614] combine_rows_per_src=[5418, 5558, 5596, 5542, 5356, 5520, 5736, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44398 payload_rows_remote=38842 dup_rows=21138 dup_rows_remote=18436 dedup_ratio=0.322540 router_tokens_per_expert=[481, 517, 504, 488, 534, 543, 490, 523, 531, 505, 511, 533, 503, 496, 498, 552, 529, 519, 498, 534, 521, 514, 512, 494, 486, 568, 525, 520, 501, 537, 522, 482, 535, 516, 520, 502, 474, 495, 489, 515, 487, 543, 507, 486, 515, 527, 501, 488, 511, 517, 496, 508, 523, 479, 491, 546, 498, 514, 508, 514, 534, 517, 527, 514] payload_rows_per_dst=[5576, 5556, 5562, 5616, 5496, 5506, 5530, 5556] dup_rows_per_dst=[2584, 2702, 2680, 2666, 2596, 2602, 2612, 2696] combine_rows_per_src=[5576, 5556, 5562, 5616, 5496, 5506, 5530, 5556] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278906880 dispatch_meta_bytes=229152 combine_payload_bytes=278906880 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279136032 ingress_bytes=389007360 link_bytes=389007360 token_bytes=558042912 weight_movement_bytes=110100480 total_nvlink_bytes=668143392 weight_to_token_ratio=0.197298 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.765477798e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44570 payload_rows_remote=38952 dup_rows=20966 dup_rows_remote=18368 dedup_ratio=0.319916 router_tokens_per_expert=[555, 545, 472, 510, 565, 521, 495, 502, 529, 507, 509, 479, 508, 545, 494, 519, 484, 531, 527, 479, 517, 537, 501, 487, 553, 503, 526, 514, 524, 497, 507, 484, 510, 549, 532, 539, 514, 484, 524, 511, 518, 512, 570, 514, 494, 472, 487, 522, 495, 554, 484, 499, 519, 459, 483, 525, 503, 528, 469, 530, 491, 500, 550, 501] payload_rows_per_dst=[5570, 5560, 5556, 5618, 5678, 5548, 5476, 5564] dup_rows_per_dst=[2760, 2620, 2570, 2598, 2648, 2630, 2560, 2580] combine_rows_per_src=[5570, 5560, 5556, 5618, 5678, 5548, 5476, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278577152 dispatch_meta_bytes=229320 combine_payload_bytes=278577152 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278806472 ingress_bytes=388677632 link_bytes=388677632 token_bytes=557383624 weight_movement_bytes=110100480 total_nvlink_bytes=667484104 weight_to_token_ratio=0.197531 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.759743285e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44486 payload_rows_remote=38896 dup_rows=21050 dup_rows_remote=18356 dedup_ratio=0.321198 router_tokens_per_expert=[495, 565, 476, 534, 524, 478, 549, 507, 467, 497, 521, 511, 536, 515, 478, 523, 508, 521, 509, 529, 513, 506, 536, 515, 513, 507, 515, 473, 498, 499, 545, 521, 523, 491, 531, 490, 481, 516, 500, 542, 503, 532, 493, 510, 496, 516, 516, 495, 538, 533, 496, 518, 507, 504, 510, 503, 516, 485, 547, 531, 524, 549, 486, 502] payload_rows_per_dst=[5590, 5460, 5638, 5536, 5628, 5530, 5568, 5536] dup_rows_per_dst=[2694, 2612, 2636, 2602, 2718, 2592, 2650, 2546] combine_rows_per_src=[5590, 5460, 5638, 5536, 5628, 5530, 5568, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277587968 dispatch_meta_bytes=229360 combine_payload_bytes=277587968 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277817328 ingress_bytes=497788928 link_bytes=497788928 token_bytes=555405296 weight_movement_bytes=220200960 total_nvlink_bytes=775606256 weight_to_token_ratio=0.396469 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.657368180e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277301248 dispatch_meta_bytes=228376 combine_payload_bytes=277301248 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277529624 ingress_bytes=497502208 link_bytes=497502208 token_bytes=554830872 weight_movement_bytes=220200960 total_nvlink_bytes=775031832 weight_to_token_ratio=0.396879 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.652381648e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278362112 dispatch_meta_bytes=229408 combine_payload_bytes=278362112 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278591520 ingress_bytes=388462592 link_bytes=388462592 token_bytes=556953632 weight_movement_bytes=110100480 total_nvlink_bytes=667054112 weight_to_token_ratio=0.197683 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.756003386e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278419456 dispatch_meta_bytes=229112 combine_payload_bytes=278419456 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278648568 ingress_bytes=498620416 link_bytes=498620416 token_bytes=557068024 weight_movement_bytes=220200960 total_nvlink_bytes=777268984 weight_to_token_ratio=0.395286 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.671829124e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279207936 dispatch_meta_bytes=229280 combine_payload_bytes=279207936 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279437216 ingress_bytes=499408896 link_bytes=499408896 token_bytes=558645152 weight_movement_bytes=220200960 total_nvlink_bytes=778846112 weight_to_token_ratio=0.394170 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.685542088e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278806528 dispatch_meta_bytes=229008 combine_payload_bytes=278806528 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279035536 ingress_bytes=609107968 link_bytes=609107968 token_bytes=557842064 weight_movement_bytes=330301440 total_nvlink_bytes=888143504 weight_to_token_ratio=0.592106 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059338937e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44536 payload_rows_remote=38924 dup_rows=21000 dup_rows_remote=18220 dedup_ratio=0.320435 router_tokens_per_expert=[515, 549, 478, 527, 485, 500, 519, 513, 522, 503, 540, 506, 513, 509, 525, 490, 554, 535, 512, 504, 503, 545, 506, 537, 518, 485, 516, 515, 510, 497, 488, 504, 508, 500, 492, 523, 499, 503, 526, 506, 510, 520, 497, 472, 523, 514, 522, 506, 506, 502, 507, 564, 539, 535, 490, 520, 546, 498, 492, 538, 479, 498, 505, 505] payload_rows_per_dst=[5606, 5468, 5612, 5552, 5528, 5568, 5666, 5536] dup_rows_per_dst=[2566, 2748, 2780, 2514, 2586, 2560, 2660, 2586] combine_rows_per_src=[5606, 5468, 5612, 5552, 5528, 5568, 5666, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44406 payload_rows_remote=38984 dup_rows=21130 dup_rows_remote=18644 dedup_ratio=0.322418 router_tokens_per_expert=[524, 482, 527, 520, 489, 493, 508, 500, 500, 497, 524, 561, 526, 508, 546, 549, 489, 523, 528, 539, 505, 477, 481, 493, 483, 465, 542, 553, 522, 474, 517, 508, 531, 550, 513, 517, 548, 534, 463, 544, 505, 508, 524, 496, 490, 478, 472, 481, 534, 535, 518, 517, 542, 523, 516, 527, 516, 514, 513, 479, 509, 534, 465, 519] payload_rows_per_dst=[5446, 5654, 5540, 5458, 5664, 5422, 5716, 5506] dup_rows_per_dst=[2640, 2768, 2530, 2670, 2736, 2486, 2708, 2592] combine_rows_per_src=[5446, 5654, 5540, 5458, 5664, 5422, 5716, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44386 payload_rows_remote=38868 dup_rows=21150 dup_rows_remote=18546 dedup_ratio=0.322723 router_tokens_per_expert=[507, 523, 489, 487, 552, 515, 526, 483, 496, 525, 506, 507, 513, 503, 526, 529, 510, 550, 499, 592, 525, 525, 496, 529, 531, 480, 496, 492, 533, 461, 507, 526, 555, 487, 525, 474, 510, 518, 487, 505, 493, 496, 494, 477, 525, 542, 494, 535, 518, 538, 507, 479, 546, 513, 487, 508, 515, 510, 538, 513, 513, 521, 514, 492] payload_rows_per_dst=[5562, 5556, 5642, 5450, 5518, 5492, 5586, 5580] dup_rows_per_dst=[2602, 2654, 2810, 2602, 2604, 2620, 2606, 2652] combine_rows_per_src=[5562, 5556, 5642, 5450, 5518, 5492, 5586, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44590 payload_rows_remote=38990 dup_rows=20946 dup_rows_remote=18222 dedup_ratio=0.319611 router_tokens_per_expert=[523, 483, 482, 523, 552, 451, 527, 505, 531, 525, 509, 503, 497, 490, 512, 494, 523, 497, 501, 541, 536, 568, 508, 504, 521, 534, 501, 530, 468, 500, 535, 516, 498, 511, 504, 490, 519, 530, 533, 536, 520, 520, 514, 497, 489, 501, 525, 513, 475, 518, 540, 541, 539, 508, 555, 486, 503, 512, 499, 538, 470, 502, 490, 502] payload_rows_per_dst=[5604, 5494, 5666, 5528, 5622, 5550, 5600, 5526] dup_rows_per_dst=[2488, 2628, 2690, 2682, 2620, 2608, 2724, 2506] combine_rows_per_src=[5604, 5494, 5666, 5528, 5622, 5550, 5600, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44458 payload_rows_remote=38692 dup_rows=21078 dup_rows_remote=18108 dedup_ratio=0.321625 router_tokens_per_expert=[497, 488, 532, 478, 488, 500, 545, 492, 506, 477, 493, 524, 551, 514, 541, 510, 480, 514, 501, 547, 505, 485, 496, 487, 539, 528, 517, 525, 498, 501, 515, 489, 519, 488, 526, 505, 524, 515, 517, 499, 528, 490, 553, 514, 483, 530, 515, 489, 502, 527, 513, 509, 509, 533, 531, 522, 543, 520, 535, 541, 499, 529, 521, 476] payload_rows_per_dst=[5514, 5430, 5194, 5816, 5542, 5696, 5500, 5766] dup_rows_per_dst=[2526, 2366, 2158, 3044, 2644, 2944, 2426, 2970] combine_rows_per_src=[5514, 5430, 5194, 5816, 5542, 5696, 5500, 5766] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279007232 dispatch_meta_bytes=228576 combine_payload_bytes=279007232 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=279235808 ingress_bytes=719409152 link_bytes=719409152 token_bytes=558243040 weight_movement_bytes=440401920 total_nvlink_bytes=998644960 weight_to_token_ratio=0.788907 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.251170838e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44464 payload_rows_remote=38924 dup_rows=21072 dup_rows_remote=18466 dedup_ratio=0.321533 router_tokens_per_expert=[551, 477, 551, 523, 490, 503, 503, 475, 523, 489, 514, 478, 522, 503, 506, 525, 512, 500, 506, 527, 523, 511, 514, 545, 498, 519, 508, 503, 517, 520, 527, 481, 517, 517, 498, 483, 539, 548, 517, 478, 528, 512, 488, 489, 508, 543, 518, 515, 563, 555, 551, 533, 525, 497, 510, 491, 506, 486, 509, 511, 460, 526, 507, 496] payload_rows_per_dst=[5498, 5502, 5646, 5540, 5550, 5510, 5656, 5562] dup_rows_per_dst=[2648, 2618, 2630, 2606, 2644, 2692, 2794, 2440] combine_rows_per_src=[5498, 5502, 5646, 5540, 5550, 5510, 5656, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44616 payload_rows_remote=39056 dup_rows=20920 dup_rows_remote=18292 dedup_ratio=0.319214 router_tokens_per_expert=[512, 548, 496, 491, 521, 547, 475, 531, 531, 521, 527, 479, 497, 535, 501, 503, 542, 469, 511, 519, 514, 564, 514, 532, 478, 493, 501, 498, 504, 497, 521, 524, 509, 532, 509, 507, 511, 485, 522, 532, 456, 532, 513, 529, 521, 523, 495, 481, 516, 507, 498, 509, 515, 576, 505, 526, 519, 515, 507, 482, 487, 506, 509, 538] payload_rows_per_dst=[5578, 5560, 5650, 5504, 5552, 5562, 5646, 5564] dup_rows_per_dst=[2664, 2628, 2680, 2528, 2662, 2538, 2658, 2562] combine_rows_per_src=[5578, 5560, 5650, 5504, 5552, 5562, 5646, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279437312 dispatch_meta_bytes=230512 combine_payload_bytes=279437312 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279667824 ingress_bytes=389537792 link_bytes=389537792 token_bytes=559105136 weight_movement_bytes=110100480 total_nvlink_bytes=669205616 weight_to_token_ratio=0.196923 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.774702882e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278605824 dispatch_meta_bytes=229656 combine_payload_bytes=278605824 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=278835480 ingress_bytes=388706304 link_bytes=388706304 token_bytes=557441304 weight_movement_bytes=110100480 total_nvlink_bytes=667541784 weight_to_token_ratio=0.197510 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.760241939e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44484 payload_rows_remote=38720 dup_rows=21052 dup_rows_remote=18222 dedup_ratio=0.321228 router_tokens_per_expert=[515, 510, 472, 491, 492, 498, 492, 491, 519, 558, 487, 511, 521, 532, 557, 528, 574, 526, 528, 511, 492, 476, 506, 552, 525, 541, 466, 520, 516, 475, 469, 509, 488, 522, 524, 524, 481, 520, 464, 516, 525, 543, 531, 501, 522, 506, 502, 514, 483, 535, 535, 525, 522, 517, 525, 489, 514, 503, 509, 543, 500, 494, 518, 513] payload_rows_per_dst=[5764, 5608, 5558, 5502, 5466, 5644, 5356, 5586] dup_rows_per_dst=[2830, 2854, 2750, 2540, 2598, 2644, 2234, 2602] combine_rows_per_src=[5764, 5608, 5558, 5502, 5466, 5644, 5356, 5586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279480320 dispatch_meta_bytes=228848 combine_payload_bytes=279480320 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279709168 ingress_bytes=609781760 link_bytes=609781760 token_bytes=559189488 weight_movement_bytes=330301440 total_nvlink_bytes=889490928 weight_to_token_ratio=0.590679 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060510773e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277344256 dispatch_meta_bytes=227200 combine_payload_bytes=277344256 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277571456 ingress_bytes=387444736 link_bytes=387444736 token_bytes=554915712 weight_movement_bytes=110100480 total_nvlink_bytes=665016192 weight_to_token_ratio=0.198409 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.738301196e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279007232 dispatch_meta_bytes=229560 combine_payload_bytes=279007232 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279236792 ingress_bytes=389107712 link_bytes=389107712 token_bytes=558244024 weight_movement_bytes=110100480 total_nvlink_bytes=668344504 weight_to_token_ratio=0.197226 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.767223084e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279595008 dispatch_meta_bytes=230552 combine_payload_bytes=279595008 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279825560 ingress_bytes=389695488 link_bytes=389695488 token_bytes=559420568 weight_movement_bytes=110100480 total_nvlink_bytes=669521048 weight_to_token_ratio=0.196812 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.777445475e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279121920 dispatch_meta_bytes=228520 combine_payload_bytes=279121920 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279350440 ingress_bytes=609423360 link_bytes=609423360 token_bytes=558472360 weight_movement_bytes=330301440 total_nvlink_bytes=888773800 weight_to_token_ratio=0.591437 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059887456e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278677504 dispatch_meta_bytes=229712 combine_payload_bytes=278677504 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278907216 ingress_bytes=388777984 link_bytes=388777984 token_bytes=557584720 weight_movement_bytes=110100480 total_nvlink_bytes=667685200 weight_to_token_ratio=0.197460 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.761488572e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278290432 dispatch_meta_bytes=228984 combine_payload_bytes=278290432 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278519416 ingress_bytes=388390912 link_bytes=388390912 token_bytes=556809848 weight_movement_bytes=110100480 total_nvlink_bytes=666910328 weight_to_token_ratio=0.197734 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.754756753e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278706176 dispatch_meta_bytes=229352 combine_payload_bytes=278706176 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278935528 ingress_bytes=498907136 link_bytes=498907136 token_bytes=557641704 weight_movement_bytes=220200960 total_nvlink_bytes=777842664 weight_to_token_ratio=0.394879 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.676815656e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44348 payload_rows_remote=38764 dup_rows=21188 dup_rows_remote=18584 dedup_ratio=0.323303 router_tokens_per_expert=[509, 493, 510, 547, 491, 529, 481, 508, 565, 499, 527, 515, 505, 537, 513, 543, 507, 478, 525, 513, 546, 534, 485, 535, 523, 519, 487, 487, 542, 506, 522, 508, 544, 483, 496, 497, 474, 524, 526, 506, 526, 495, 542, 508, 519, 462, 477, 555, 529, 515, 461, 519, 498, 515, 567, 542, 469, 455, 503, 541, 478, 544, 507, 502] payload_rows_per_dst=[5546, 5574, 5566, 5584, 5600, 5504, 5550, 5424] dup_rows_per_dst=[2590, 2834, 2680, 2604, 2500, 2664, 2742, 2574] combine_rows_per_src=[5546, 5574, 5566, 5584, 5600, 5504, 5550, 5424] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44582 payload_rows_remote=39068 dup_rows=20954 dup_rows_remote=18298 dedup_ratio=0.319733 router_tokens_per_expert=[498, 536, 515, 528, 560, 530, 512, 517, 571, 492, 517, 479, 517, 494, 498, 502, 520, 525, 540, 503, 521, 508, 524, 480, 540, 485, 499, 482, 521, 489, 520, 498, 515, 496, 502, 545, 510, 482, 512, 552, 518, 533, 464, 512, 482, 524, 507, 482, 498, 532, 505, 497, 515, 505, 499, 534, 525, 512, 497, 561, 507, 491, 510, 523] payload_rows_per_dst=[5684, 5498, 5552, 5512, 5646, 5496, 5514, 5680] dup_rows_per_dst=[2708, 2642, 2690, 2556, 2582, 2548, 2656, 2572] combine_rows_per_src=[5684, 5498, 5552, 5512, 5646, 5496, 5514, 5680] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44554 payload_rows_remote=38876 dup_rows=20982 dup_rows_remote=18006 dedup_ratio=0.320160 router_tokens_per_expert=[496, 513, 505, 488, 472, 522, 524, 519, 542, 514, 493, 514, 529, 534, 499, 499, 527, 485, 510, 515, 517, 515, 501, 468, 524, 520, 464, 488, 515, 504, 532, 539, 491, 533, 513, 524, 508, 519, 506, 518, 495, 541, 503, 530, 526, 510, 489, 503, 510, 559, 489, 550, 500, 529, 480, 541, 502, 518, 540, 516, 502, 511, 521, 504] payload_rows_per_dst=[5546, 5544, 5364, 5624, 5614, 5566, 5618, 5678] dup_rows_per_dst=[2476, 2578, 2286, 2596, 2736, 2628, 2706, 2976] combine_rows_per_src=[5546, 5544, 5364, 5624, 5614, 5566, 5618, 5678] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44448 payload_rows_remote=38922 dup_rows=21088 dup_rows_remote=18600 dedup_ratio=0.321777 router_tokens_per_expert=[516, 487, 527, 522, 539, 517, 509, 496, 479, 517, 510, 517, 518, 466, 502, 517, 493, 536, 535, 504, 492, 504, 519, 541, 488, 513, 512, 530, 560, 536, 544, 514, 475, 557, 524, 486, 522, 488, 507, 521, 492, 483, 501, 512, 518, 483, 506, 512, 513, 531, 516, 490, 502, 508, 527, 530, 521, 504, 488, 488, 538, 538, 556, 471] payload_rows_per_dst=[5560, 5504, 5602, 5620, 5614, 5526, 5510, 5512] dup_rows_per_dst=[2666, 2548, 2646, 2774, 2546, 2488, 2724, 2696] combine_rows_per_src=[5560, 5504, 5602, 5620, 5614, 5526, 5510, 5512] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277860352 dispatch_meta_bytes=229392 combine_payload_bytes=277860352 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278089744 ingress_bytes=498061312 link_bytes=498061312 token_bytes=555950096 weight_movement_bytes=220200960 total_nvlink_bytes=776151056 weight_to_token_ratio=0.396080 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.662105386e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44402 payload_rows_remote=38826 dup_rows=21134 dup_rows_remote=18514 dedup_ratio=0.322479 router_tokens_per_expert=[511, 545, 538, 502, 536, 522, 523, 538, 547, 520, 530, 491, 467, 513, 519, 507, 500, 488, 524, 530, 490, 526, 517, 523, 524, 558, 476, 513, 535, 472, 486, 517, 514, 534, 503, 512, 480, 544, 481, 529, 513, 524, 469, 523, 541, 496, 529, 530, 476, 535, 455, 481, 482, 559, 528, 483, 495, 495, 533, 485, 535, 498, 501, 517] payload_rows_per_dst=[5594, 5494, 5576, 5546, 5504, 5616, 5542, 5530] dup_rows_per_dst=[2836, 2694, 2620, 2616, 2690, 2634, 2456, 2588] combine_rows_per_src=[5594, 5494, 5576, 5546, 5504, 5616, 5542, 5530] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44324 payload_rows_remote=38744 dup_rows=21212 dup_rows_remote=18518 dedup_ratio=0.323669 router_tokens_per_expert=[489, 486, 535, 491, 513, 471, 435, 513, 500, 542, 528, 531, 510, 477, 566, 486, 527, 497, 491, 524, 508, 544, 541, 478, 505, 524, 520, 555, 536, 520, 472, 513, 537, 522, 488, 508, 491, 560, 519, 512, 525, 491, 491, 489, 498, 483, 514, 507, 514, 515, 537, 538, 491, 537, 516, 535, 561, 513, 493, 498, 498, 522, 533, 504] payload_rows_per_dst=[5470, 5540, 5582, 5616, 5580, 5436, 5564, 5536] dup_rows_per_dst=[2396, 2740, 2638, 2674, 2694, 2560, 2802, 2708] combine_rows_per_src=[5470, 5540, 5582, 5616, 5580, 5436, 5564, 5536] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279953408 dispatch_meta_bytes=229392 combine_payload_bytes=279953408 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280182800 ingress_bytes=500154368 link_bytes=500154368 token_bytes=560136208 weight_movement_bytes=220200960 total_nvlink_bytes=780337168 weight_to_token_ratio=0.393120 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.698507072e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277544960 dispatch_meta_bytes=227768 combine_payload_bytes=277544960 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277772728 ingress_bytes=387645440 link_bytes=387645440 token_bytes=555317688 weight_movement_bytes=110100480 total_nvlink_bytes=665418168 weight_to_token_ratio=0.198266 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.741791769e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44414 payload_rows_remote=38842 dup_rows=21122 dup_rows_remote=18508 dedup_ratio=0.322296 router_tokens_per_expert=[510, 544, 534, 499, 534, 518, 521, 537, 538, 520, 525, 488, 464, 509, 519, 505, 501, 486, 524, 529, 492, 525, 516, 520, 523, 559, 478, 514, 536, 468, 486, 518, 517, 533, 500, 510, 478, 546, 484, 527, 516, 528, 469, 524, 540, 497, 534, 532, 481, 536, 453, 482, 486, 561, 530, 486, 499, 495, 536, 488, 537, 502, 502, 519] payload_rows_per_dst=[5582, 5478, 5572, 5546, 5500, 5630, 5560, 5546] dup_rows_per_dst=[2812, 2658, 2614, 2618, 2690, 2650, 2470, 2610] combine_rows_per_src=[5582, 5478, 5572, 5546, 5500, 5630, 5560, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44448 payload_rows_remote=38916 dup_rows=21088 dup_rows_remote=18582 dedup_ratio=0.321777 router_tokens_per_expert=[508, 485, 523, 519, 536, 516, 505, 492, 475, 516, 505, 512, 517, 463, 504, 518, 493, 534, 533, 508, 489, 504, 517, 540, 488, 512, 511, 532, 558, 537, 543, 513, 476, 557, 525, 489, 525, 489, 506, 521, 493, 486, 502, 513, 521, 484, 505, 515, 516, 532, 516, 492, 501, 508, 527, 531, 522, 511, 491, 494, 540, 542, 557, 475] payload_rows_per_dst=[5532, 5498, 5598, 5620, 5616, 5532, 5518, 5534] dup_rows_per_dst=[2636, 2522, 2638, 2768, 2560, 2506, 2728, 2730] combine_rows_per_src=[5532, 5498, 5598, 5620, 5616, 5532, 5518, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44304 payload_rows_remote=38708 dup_rows=21232 dup_rows_remote=18530 dedup_ratio=0.323975 router_tokens_per_expert=[485, 482, 533, 485, 509, 471, 433, 507, 496, 539, 526, 526, 511, 477, 563, 486, 528, 499, 490, 520, 510, 546, 538, 476, 503, 524, 521, 552, 536, 520, 474, 513, 539, 522, 490, 511, 492, 559, 523, 513, 526, 494, 488, 489, 495, 485, 516, 508, 516, 518, 537, 539, 491, 540, 519, 534, 565, 514, 495, 500, 501, 523, 539, 508] payload_rows_per_dst=[5450, 5520, 5574, 5612, 5596, 5440, 5568, 5544] dup_rows_per_dst=[2360, 2728, 2640, 2674, 2702, 2562, 2820, 2746] combine_rows_per_src=[5450, 5520, 5574, 5612, 5596, 5440, 5568, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44582 payload_rows_remote=39060 dup_rows=20954 dup_rows_remote=18284 dedup_ratio=0.319733 router_tokens_per_expert=[497, 532, 511, 526, 557, 525, 510, 513, 569, 492, 513, 478, 515, 488, 495, 502, 520, 523, 539, 504, 519, 509, 526, 479, 538, 487, 499, 481, 521, 485, 521, 502, 513, 494, 506, 542, 511, 484, 514, 552, 519, 532, 465, 515, 484, 525, 510, 484, 499, 531, 508, 498, 518, 505, 500, 537, 526, 514, 501, 562, 511, 492, 514, 526] payload_rows_per_dst=[5672, 5482, 5552, 5520, 5638, 5504, 5522, 5692] dup_rows_per_dst=[2670, 2622, 2686, 2548, 2594, 2564, 2670, 2600] combine_rows_per_src=[5672, 5482, 5552, 5520, 5638, 5504, 5522, 5692] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44548 payload_rows_remote=38872 dup_rows=20988 dup_rows_remote=18376 dedup_ratio=0.320251 router_tokens_per_expert=[573, 537, 472, 515, 500, 540, 486, 521, 506, 517, 527, 516, 524, 476, 514, 461, 521, 527, 520, 505, 524, 510, 485, 492, 510, 526, 535, 517, 514, 498, 534, 508, 518, 517, 500, 494, 538, 470, 497, 449, 523, 478, 485, 497, 533, 518, 534, 513, 522, 515, 549, 512, 526, 518, 567, 504, 508, 516, 512, 519, 544, 485, 492, 504] payload_rows_per_dst=[5676, 5538, 5606, 5544, 5402, 5710, 5458, 5614] dup_rows_per_dst=[2612, 2610, 2736, 2566, 2564, 2974, 2380, 2546] combine_rows_per_src=[5676, 5538, 5606, 5544, 5402, 5710, 5458, 5614] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44472 payload_rows_remote=38866 dup_rows=21064 dup_rows_remote=18472 dedup_ratio=0.321411 router_tokens_per_expert=[520, 529, 515, 539, 522, 488, 529, 512, 557, 515, 507, 517, 538, 467, 501, 497, 511, 490, 524, 510, 536, 524, 556, 508, 497, 528, 532, 513, 539, 538, 519, 543, 453, 517, 509, 502, 493, 494, 470, 521, 501, 501, 518, 484, 491, 494, 542, 470, 519, 519, 524, 549, 526, 525, 514, 477, 502, 511, 512, 471, 508, 501, 469, 560] payload_rows_per_dst=[5634, 5606, 5596, 5602, 5490, 5472, 5586, 5486] dup_rows_per_dst=[2674, 2592, 2722, 2816, 2428, 2530, 2720, 2582] combine_rows_per_src=[5634, 5606, 5596, 5602, 5490, 5472, 5586, 5486] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280039424 dispatch_meta_bytes=229464 combine_payload_bytes=280039424 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280268888 ingress_bytes=610340864 link_bytes=610340864 token_bytes=560308312 weight_movement_bytes=330301440 total_nvlink_bytes=890609752 weight_to_token_ratio=0.589499 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061483146e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278663168 dispatch_meta_bytes=227528 combine_payload_bytes=278663168 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278890696 ingress_bytes=388763648 link_bytes=388763648 token_bytes=557553864 weight_movement_bytes=110100480 total_nvlink_bytes=667654344 weight_to_token_ratio=0.197471 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.761239245e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278992896 dispatch_meta_bytes=230088 combine_payload_bytes=278992896 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279222984 ingress_bytes=389093376 link_bytes=389093376 token_bytes=558215880 weight_movement_bytes=110100480 total_nvlink_bytes=668316360 weight_to_token_ratio=0.197236 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.766973757e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278304768 dispatch_meta_bytes=229360 combine_payload_bytes=278304768 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278534128 ingress_bytes=498505728 link_bytes=498505728 token_bytes=556838896 weight_movement_bytes=220200960 total_nvlink_bytes=777039856 weight_to_token_ratio=0.395448 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.669834511e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277716992 dispatch_meta_bytes=229048 combine_payload_bytes=277716992 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277946040 ingress_bytes=387817472 link_bytes=387817472 token_bytes=555663032 weight_movement_bytes=110100480 total_nvlink_bytes=665763512 weight_to_token_ratio=0.198143 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.744783688e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278634496 dispatch_meta_bytes=228992 combine_payload_bytes=278634496 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=278863488 ingress_bytes=498835456 link_bytes=498835456 token_bytes=557497984 weight_movement_bytes=220200960 total_nvlink_bytes=777698944 weight_to_token_ratio=0.394981 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.675569023e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278591488 dispatch_meta_bytes=229352 combine_payload_bytes=278591488 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278820840 ingress_bytes=608892928 link_bytes=608892928 token_bytes=557412328 weight_movement_bytes=330301440 total_nvlink_bytes=887713768 weight_to_token_ratio=0.592562 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058964947e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44420 payload_rows_remote=38874 dup_rows=21116 dup_rows_remote=18478 dedup_ratio=0.322205 router_tokens_per_expert=[491, 504, 522, 508, 502, 517, 503, 532, 545, 540, 505, 544, 542, 532, 539, 520, 515, 495, 524, 517, 511, 580, 495, 520, 534, 500, 486, 498, 502, 537, 511, 524, 479, 478, 500, 520, 527, 509, 495, 483, 497, 510, 549, 464, 472, 533, 503, 485, 489, 534, 533, 476, 508, 523, 480, 516, 480, 503, 520, 520, 522, 484, 558, 523] payload_rows_per_dst=[5560, 5752, 5556, 5546, 5476, 5444, 5558, 5528] dup_rows_per_dst=[2598, 2782, 2758, 2638, 2506, 2582, 2560, 2692] combine_rows_per_src=[5560, 5752, 5556, 5546, 5476, 5444, 5558, 5528] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278648832 dispatch_meta_bytes=229408 combine_payload_bytes=278648832 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278878240 ingress_bytes=388749312 link_bytes=388749312 token_bytes=557527072 weight_movement_bytes=110100480 total_nvlink_bytes=667627552 weight_to_token_ratio=0.197480 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.760989918e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44402 payload_rows_remote=38884 dup_rows=21134 dup_rows_remote=18552 dedup_ratio=0.322479 router_tokens_per_expert=[519, 512, 533, 503, 513, 531, 526, 508, 505, 509, 520, 532, 536, 531, 518, 489, 529, 528, 517, 509, 494, 503, 514, 502, 497, 484, 521, 536, 496, 508, 470, 504, 477, 496, 508, 491, 516, 529, 511, 527, 560, 514, 529, 520, 496, 501, 492, 505, 506, 473, 507, 532, 496, 515, 498, 523, 536, 566, 510, 510, 505, 500, 519, 503] payload_rows_per_dst=[5592, 5532, 5576, 5534, 5462, 5628, 5518, 5560] dup_rows_per_dst=[2698, 2748, 2616, 2498, 2648, 2606, 2582, 2738] combine_rows_per_src=[5592, 5532, 5576, 5534, 5462, 5628, 5518, 5560] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44198 payload_rows_remote=38552 dup_rows=21338 dup_rows_remote=18480 dedup_ratio=0.325592 router_tokens_per_expert=[541, 475, 539, 507, 516, 519, 484, 502, 528, 523, 541, 548, 506, 515, 497, 541, 520, 504, 509, 506, 523, 524, 484, 557, 494, 509, 519, 482, 462, 487, 501, 504, 496, 530, 517, 524, 516, 528, 531, 487, 512, 504, 514, 494, 526, 490, 528, 535, 507, 519, 517, 487, 507, 533, 517, 536, 467, 539, 479, 547, 489, 523, 497, 505] payload_rows_per_dst=[5218, 5318, 5170, 5626, 5858, 5688, 5674, 5646] dup_rows_per_dst=[2278, 2320, 2352, 2758, 3136, 2776, 2860, 2858] combine_rows_per_src=[5218, 5318, 5170, 5626, 5858, 5688, 5674, 5646] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44616 payload_rows_remote=39034 dup_rows=20920 dup_rows_remote=18452 dedup_ratio=0.319214 router_tokens_per_expert=[483, 526, 542, 541, 491, 527, 541, 527, 541, 490, 509, 559, 519, 527, 499, 510, 477, 498, 515, 537, 538, 540, 527, 502, 492, 467, 514, 539, 493, 498, 527, 487, 475, 508, 493, 509, 487, 524, 512, 519, 509, 520, 533, 464, 466, 525, 500, 508, 482, 518, 542, 476, 479, 516, 505, 535, 484, 533, 525, 559, 488, 533, 550, 508] payload_rows_per_dst=[5668, 5652, 5566, 5490, 5534, 5582, 5516, 5608] dup_rows_per_dst=[2688, 2656, 2702, 2544, 2520, 2468, 2590, 2752] combine_rows_per_src=[5668, 5652, 5566, 5490, 5534, 5582, 5516, 5608] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44492 payload_rows_remote=38802 dup_rows=21044 dup_rows_remote=18304 dedup_ratio=0.321106 router_tokens_per_expert=[532, 501, 545, 518, 463, 512, 538, 510, 530, 539, 488, 516, 505, 515, 514, 514, 547, 533, 553, 536, 498, 518, 514, 516, 530, 514, 489, 509, 492, 531, 501, 502, 477, 508, 527, 512, 512, 557, 507, 506, 537, 503, 517, 535, 500, 500, 492, 507, 490, 518, 543, 477, 486, 517, 507, 518, 484, 512, 498, 491, 489, 528, 464, 526] payload_rows_per_dst=[5558, 5570, 5690, 5576, 5600, 5480, 5552, 5466] dup_rows_per_dst=[2680, 2672, 2740, 2560, 2612, 2702, 2560, 2518] combine_rows_per_src=[5558, 5570, 5690, 5576, 5600, 5480, 5552, 5466] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44454 payload_rows_remote=38954 dup_rows=21082 dup_rows_remote=18420 dedup_ratio=0.321686 router_tokens_per_expert=[523, 510, 521, 515, 522, 555, 507, 511, 511, 520, 514, 536, 501, 508, 537, 507, 533, 497, 527, 515, 503, 508, 523, 514, 497, 500, 515, 497, 506, 521, 522, 525, 501, 519, 511, 491, 519, 502, 511, 527, 479, 494, 519, 510, 541, 470, 545, 530, 474, 479, 474, 538, 509, 542, 524, 528, 525, 506, 501, 477, 525, 508, 514, 474] payload_rows_per_dst=[5640, 5604, 5540, 5608, 5500, 5498, 5566, 5498] dup_rows_per_dst=[2688, 2664, 2700, 2558, 2662, 2678, 2570, 2562] combine_rows_per_src=[5640, 5604, 5540, 5608, 5500, 5498, 5566, 5498] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44626 payload_rows_remote=38996 dup_rows=20910 dup_rows_remote=18178 dedup_ratio=0.319061 router_tokens_per_expert=[522, 456, 532, 511, 507, 528, 560, 526, 490, 519, 563, 489, 495, 510, 517, 490, 501, 549, 526, 523, 518, 522, 472, 515, 521, 528, 472, 513, 543, 530, 487, 497, 509, 485, 490, 544, 501, 524, 495, 504, 501, 512, 529, 559, 493, 573, 524, 510, 508, 494, 514, 504, 527, 486, 509, 520, 503, 504, 532, 527, 463, 501, 515, 476] payload_rows_per_dst=[5630, 5508, 5576, 5602, 5506, 5738, 5552, 5514] dup_rows_per_dst=[2732, 2560, 2676, 2580, 2598, 2664, 2572, 2528] combine_rows_per_src=[5630, 5508, 5576, 5602, 5506, 5738, 5552, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278419456 dispatch_meta_bytes=229400 combine_payload_bytes=278419456 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278648856 ingress_bytes=498620416 link_bytes=498620416 token_bytes=557068312 weight_movement_bytes=220200960 total_nvlink_bytes=777269272 weight_to_token_ratio=0.395285 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.671829124e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44604 payload_rows_remote=39014 dup_rows=20932 dup_rows_remote=18182 dedup_ratio=0.319397 router_tokens_per_expert=[493, 511, 502, 487, 470, 518, 522, 517, 539, 513, 492, 511, 526, 534, 498, 498, 524, 482, 510, 514, 517, 516, 499, 470, 522, 519, 463, 488, 518, 506, 532, 539, 489, 532, 512, 523, 509, 519, 508, 521, 494, 541, 502, 530, 527, 511, 491, 503, 513, 563, 488, 553, 503, 530, 481, 542, 505, 519, 544, 517, 505, 514, 523, 506] payload_rows_per_dst=[5566, 5572, 5566, 5562, 5588, 5562, 5598, 5590] dup_rows_per_dst=[2474, 2650, 2498, 2536, 2714, 2636, 2674, 2750] combine_rows_per_src=[5566, 5572, 5566, 5562, 5588, 5562, 5598, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44376 payload_rows_remote=38784 dup_rows=21160 dup_rows_remote=18556 dedup_ratio=0.322876 router_tokens_per_expert=[509, 487, 509, 544, 488, 527, 480, 505, 564, 496, 526, 513, 499, 534, 511, 539, 506, 479, 524, 513, 545, 534, 485, 533, 521, 519, 487, 487, 544, 506, 522, 512, 546, 484, 496, 495, 474, 523, 525, 508, 527, 496, 545, 510, 519, 462, 478, 556, 529, 517, 461, 517, 500, 516, 570, 546, 472, 459, 506, 545, 480, 545, 507, 506] payload_rows_per_dst=[5534, 5562, 5566, 5592, 5598, 5512, 5564, 5448] dup_rows_per_dst=[2564, 2802, 2672, 2604, 2504, 2674, 2748, 2592] combine_rows_per_src=[5534, 5562, 5566, 5592, 5598, 5512, 5564, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44390 payload_rows_remote=38888 dup_rows=21146 dup_rows_remote=18478 dedup_ratio=0.322662 router_tokens_per_expert=[519, 490, 526, 504, 503, 519, 513, 490, 513, 500, 522, 539, 488, 518, 503, 502, 515, 536, 517, 527, 513, 527, 529, 522, 468, 491, 549, 495, 520, 497, 536, 494, 553, 503, 518, 518, 505, 533, 491, 515, 510, 497, 493, 535, 549, 538, 475, 507, 530, 512, 546, 524, 545, 504, 519, 487, 478, 523, 499, 498, 524, 474, 499, 481] payload_rows_per_dst=[5518, 5502, 5690, 5490, 5594, 5478, 5660, 5458] dup_rows_per_dst=[2610, 2668, 2682, 2610, 2678, 2730, 2674, 2494] combine_rows_per_src=[5518, 5502, 5690, 5490, 5594, 5478, 5660, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278949888 dispatch_meta_bytes=229992 combine_payload_bytes=278949888 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279179880 ingress_bytes=389050368 link_bytes=389050368 token_bytes=558129768 weight_movement_bytes=110100480 total_nvlink_bytes=668230248 weight_to_token_ratio=0.197267 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.766225777e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277458944 dispatch_meta_bytes=228952 combine_payload_bytes=277458944 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277687896 ingress_bytes=387559424 link_bytes=387559424 token_bytes=555146840 weight_movement_bytes=110100480 total_nvlink_bytes=665247320 weight_to_token_ratio=0.198327 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.740295809e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44526 payload_rows_remote=38770 dup_rows=21010 dup_rows_remote=18210 dedup_ratio=0.320587 router_tokens_per_expert=[569, 535, 471, 514, 495, 538, 487, 519, 506, 512, 523, 514, 520, 477, 512, 459, 522, 527, 518, 504, 523, 507, 484, 491, 511, 529, 535, 518, 514, 498, 531, 505, 517, 515, 503, 495, 537, 469, 496, 450, 524, 478, 488, 499, 532, 520, 534, 515, 524, 518, 554, 511, 527, 519, 569, 505, 513, 516, 513, 520, 547, 487, 498, 507] payload_rows_per_dst=[5756, 5584, 5466, 5586, 5402, 5680, 5412, 5640] dup_rows_per_dst=[2800, 2688, 2522, 2650, 2562, 2876, 2350, 2562] combine_rows_per_src=[5756, 5584, 5466, 5586, 5402, 5680, 5412, 5640] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279982080 dispatch_meta_bytes=229376 combine_payload_bytes=279982080 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=280211456 ingress_bytes=720384000 link_bytes=720384000 token_bytes=560193536 weight_movement_bytes=440401920 total_nvlink_bytes=1000595456 weight_to_token_ratio=0.786160 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.252866259e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279652352 dispatch_meta_bytes=228784 combine_payload_bytes=279652352 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279881136 ingress_bytes=389752832 link_bytes=389752832 token_bytes=559533488 weight_movement_bytes=110100480 total_nvlink_bytes=669633968 weight_to_token_ratio=0.196772 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778442782e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278003712 dispatch_meta_bytes=229360 combine_payload_bytes=278003712 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278233072 ingress_bytes=608305152 link_bytes=608305152 token_bytes=556236784 weight_movement_bytes=330301440 total_nvlink_bytes=886538224 weight_to_token_ratio=0.593814 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057942708e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278749184 dispatch_meta_bytes=229464 combine_payload_bytes=278749184 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278978648 ingress_bytes=388849664 link_bytes=388849664 token_bytes=557727832 weight_movement_bytes=110100480 total_nvlink_bytes=667828312 weight_to_token_ratio=0.197409 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.762735205e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277903360 dispatch_meta_bytes=227920 combine_payload_bytes=277903360 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278131280 ingress_bytes=388003840 link_bytes=388003840 token_bytes=556034640 weight_movement_bytes=110100480 total_nvlink_bytes=666135120 weight_to_token_ratio=0.198010 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.748024934e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44490 payload_rows_remote=38808 dup_rows=21046 dup_rows_remote=18322 dedup_ratio=0.321136 router_tokens_per_expert=[532, 497, 543, 512, 460, 511, 536, 510, 523, 536, 486, 514, 501, 515, 514, 515, 546, 533, 550, 532, 495, 517, 515, 515, 530, 515, 485, 509, 490, 531, 500, 503, 476, 507, 527, 513, 513, 558, 510, 508, 538, 504, 518, 536, 501, 501, 494, 512, 491, 522, 545, 479, 488, 518, 509, 518, 485, 514, 501, 494, 491, 529, 467, 530] payload_rows_per_dst=[5542, 5556, 5682, 5576, 5608, 5482, 5558, 5486] dup_rows_per_dst=[2660, 2652, 2724, 2550, 2616, 2726, 2582, 2536] combine_rows_per_src=[5542, 5556, 5682, 5576, 5608, 5482, 5558, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44436 payload_rows_remote=38928 dup_rows=21100 dup_rows_remote=18430 dedup_ratio=0.321960 router_tokens_per_expert=[520, 507, 518, 512, 520, 552, 503, 509, 509, 517, 512, 533, 498, 506, 534, 502, 533, 493, 526, 515, 500, 509, 525, 514, 496, 499, 514, 498, 506, 520, 519, 526, 500, 521, 515, 492, 518, 504, 512, 527, 482, 493, 520, 511, 545, 472, 547, 532, 475, 480, 480, 540, 509, 544, 526, 530, 527, 509, 502, 480, 526, 509, 518, 477] payload_rows_per_dst=[5618, 5580, 5532, 5598, 5508, 5508, 5580, 5512] dup_rows_per_dst=[2664, 2642, 2698, 2558, 2670, 2696, 2588, 2584] combine_rows_per_src=[5618, 5580, 5532, 5598, 5508, 5508, 5580, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44614 payload_rows_remote=39034 dup_rows=20922 dup_rows_remote=18446 dedup_ratio=0.319244 router_tokens_per_expert=[479, 524, 541, 536, 489, 525, 541, 521, 538, 489, 507, 556, 518, 523, 500, 510, 478, 497, 513, 535, 538, 540, 527, 499, 491, 468, 518, 537, 493, 496, 529, 490, 478, 508, 493, 510, 490, 523, 512, 521, 508, 518, 534, 466, 468, 526, 501, 507, 483, 519, 545, 477, 479, 516, 507, 538, 484, 537, 528, 561, 490, 536, 550, 509] payload_rows_per_dst=[5654, 5644, 5556, 5492, 5540, 5580, 5530, 5618] dup_rows_per_dst=[2658, 2638, 2698, 2552, 2530, 2476, 2598, 2772] combine_rows_per_src=[5654, 5644, 5556, 5492, 5540, 5580, 5530, 5618] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44426 payload_rows_remote=38888 dup_rows=21110 dup_rows_remote=18506 dedup_ratio=0.322113 router_tokens_per_expert=[518, 510, 530, 498, 512, 530, 521, 504, 504, 507, 520, 531, 537, 529, 515, 489, 529, 528, 515, 508, 491, 502, 514, 498, 497, 483, 518, 536, 494, 509, 470, 500, 478, 498, 509, 489, 518, 530, 510, 528, 561, 515, 528, 522, 497, 502, 496, 507, 506, 472, 511, 536, 499, 516, 501, 530, 539, 566, 510, 514, 507, 501, 519, 506] payload_rows_per_dst=[5570, 5528, 5570, 5530, 5472, 5640, 5538, 5578] dup_rows_per_dst=[2676, 2736, 2600, 2484, 2648, 2616, 2604, 2746] combine_rows_per_src=[5570, 5528, 5570, 5530, 5472, 5640, 5538, 5578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44344 payload_rows_remote=38798 dup_rows=21192 dup_rows_remote=18560 dedup_ratio=0.323364 router_tokens_per_expert=[516, 495, 512, 501, 508, 543, 482, 519, 515, 500, 512, 510, 491, 528, 530, 503, 508, 504, 527, 498, 514, 526, 516, 525, 515, 494, 523, 494, 540, 535, 493, 526, 508, 476, 520, 539, 530, 503, 481, 545, 563, 491, 520, 553, 512, 504, 513, 508, 479, 512, 508, 523, 487, 515, 514, 530, 555, 479, 509, 468, 515, 471, 518, 516] payload_rows_per_dst=[5526, 5546, 5590, 5550, 5502, 5578, 5578, 5474] dup_rows_per_dst=[2626, 2632, 2646, 2690, 2702, 2750, 2558, 2588] combine_rows_per_src=[5526, 5546, 5590, 5550, 5502, 5578, 5578, 5474] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278720512 dispatch_meta_bytes=229744 combine_payload_bytes=278720512 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278950256 ingress_bytes=388820992 link_bytes=388820992 token_bytes=557670768 weight_movement_bytes=110100480 total_nvlink_bytes=667771248 weight_to_token_ratio=0.197429 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.762236552e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=276340736 dispatch_meta_bytes=228128 combine_payload_bytes=276340736 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276568864 ingress_bytes=386441216 link_bytes=386441216 token_bytes=552909600 weight_movement_bytes=110100480 total_nvlink_bytes=663010080 weight_to_token_ratio=0.199129 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.720848333e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279795712 dispatch_meta_bytes=229944 combine_payload_bytes=279795712 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280025656 ingress_bytes=389896192 link_bytes=389896192 token_bytes=559821368 weight_movement_bytes=110100480 total_nvlink_bytes=669921848 weight_to_token_ratio=0.196671 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.780936048e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278132736 dispatch_meta_bytes=228424 combine_payload_bytes=278132736 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278361160 ingress_bytes=608434176 link_bytes=608434176 token_bytes=556493896 weight_movement_bytes=330301440 total_nvlink_bytes=886795336 weight_to_token_ratio=0.593540 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058167102e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279222272 dispatch_meta_bytes=229496 combine_payload_bytes=279222272 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279451768 ingress_bytes=389322752 link_bytes=389322752 token_bytes=558674040 weight_movement_bytes=110100480 total_nvlink_bytes=668774520 weight_to_token_ratio=0.197075 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.770962983e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279523328 dispatch_meta_bytes=228696 combine_payload_bytes=279523328 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279752024 ingress_bytes=609824768 link_bytes=609824768 token_bytes=559275352 weight_movement_bytes=330301440 total_nvlink_bytes=889576792 weight_to_token_ratio=0.590588 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060585570e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44500 payload_rows_remote=39006 dup_rows=21036 dup_rows_remote=18494 dedup_ratio=0.320984 router_tokens_per_expert=[512, 493, 501, 528, 524, 487, 543, 504, 518, 522, 538, 499, 516, 526, 514, 511, 495, 494, 510, 492, 509, 498, 497, 490, 501, 448, 482, 514, 527, 544, 491, 511, 567, 507, 549, 536, 460, 536, 518, 528, 513, 496, 504, 528, 499, 524, 564, 519, 514, 510, 526, 507, 513, 484, 527, 491, 543, 522, 521, 499, 510, 485, 514, 515] payload_rows_per_dst=[5520, 5564, 5522, 5494, 5666, 5622, 5522, 5590] dup_rows_per_dst=[2664, 2724, 2448, 2542, 2736, 2672, 2622, 2628] combine_rows_per_src=[5520, 5564, 5522, 5494, 5666, 5622, 5522, 5590] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278104064 dispatch_meta_bytes=229432 combine_payload_bytes=278104064 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278333496 ingress_bytes=498305024 link_bytes=498305024 token_bytes=556437560 weight_movement_bytes=220200960 total_nvlink_bytes=776638520 weight_to_token_ratio=0.395733 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.666343938e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279595008 dispatch_meta_bytes=230000 combine_payload_bytes=279595008 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279825008 ingress_bytes=389695488 link_bytes=389695488 token_bytes=559420016 weight_movement_bytes=110100480 total_nvlink_bytes=669520496 weight_to_token_ratio=0.196812 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.777445475e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44566 payload_rows_remote=38736 dup_rows=20970 dup_rows_remote=18052 dedup_ratio=0.319977 router_tokens_per_expert=[501, 542, 502, 522, 505, 539, 508, 530, 528, 519, 534, 536, 498, 534, 531, 512, 519, 468, 496, 512, 498, 470, 518, 510, 519, 480, 526, 533, 506, 546, 525, 512, 524, 523, 529, 533, 510, 497, 514, 517, 506, 495, 509, 509, 531, 529, 498, 494, 513, 505, 546, 487, 501, 486, 509, 483, 508, 514, 501, 515, 525, 505, 489, 484] payload_rows_per_dst=[5402, 5578, 5758, 5616, 5566, 5286, 5530, 5830] dup_rows_per_dst=[2348, 2700, 2772, 2784, 2652, 2260, 2536, 2918] combine_rows_per_src=[5402, 5578, 5758, 5616, 5566, 5286, 5530, 5830] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44582 payload_rows_remote=39088 dup_rows=20954 dup_rows_remote=18472 dedup_ratio=0.319733 router_tokens_per_expert=[488, 509, 512, 525, 513, 516, 509, 503, 500, 577, 495, 496, 505, 530, 553, 533, 500, 523, 525, 507, 523, 487, 537, 512, 528, 529, 502, 507, 495, 523, 493, 506, 520, 505, 525, 461, 561, 531, 494, 471, 505, 521, 521, 531, 523, 526, 523, 530, 493, 490, 489, 523, 509, 511, 467, 506, 534, 505, 499, 502, 512, 497, 500, 522] payload_rows_per_dst=[5558, 5634, 5596, 5562, 5538, 5640, 5494, 5560] dup_rows_per_dst=[2592, 2744, 2632, 2604, 2598, 2720, 2482, 2582] combine_rows_per_src=[5558, 5634, 5596, 5562, 5538, 5640, 5494, 5560] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44602 payload_rows_remote=38956 dup_rows=20934 dup_rows_remote=18278 dedup_ratio=0.319427 router_tokens_per_expert=[522, 522, 549, 538, 523, 491, 526, 525, 509, 540, 479, 501, 467, 509, 543, 487, 517, 563, 549, 540, 499, 524, 505, 482, 513, 478, 479, 496, 515, 464, 554, 500, 532, 501, 516, 511, 487, 499, 482, 486, 558, 512, 506, 493, 496, 486, 554, 546, 493, 486, 525, 530, 530, 523, 561, 505, 492, 501, 477, 497, 510, 511, 507, 546] payload_rows_per_dst=[5662, 5462, 5704, 5476, 5502, 5646, 5616, 5534] dup_rows_per_dst=[2730, 2608, 2654, 2522, 2526, 2656, 2690, 2548] combine_rows_per_src=[5662, 5462, 5704, 5476, 5502, 5646, 5616, 5534] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44652 payload_rows_remote=39102 dup_rows=20884 dup_rows_remote=18444 dedup_ratio=0.318665 router_tokens_per_expert=[516, 523, 510, 516, 516, 516, 511, 525, 502, 501, 531, 512, 531, 520, 502, 480, 501, 532, 486, 488, 483, 489, 476, 507, 538, 522, 551, 528, 473, 539, 498, 509, 492, 475, 519, 503, 523, 490, 491, 502, 540, 497, 485, 522, 530, 511, 523, 518, 514, 561, 514, 530, 538, 532, 518, 506, 539, 509, 486, 471, 518, 518, 531, 530] payload_rows_per_dst=[5596, 5576, 5532, 5574, 5550, 5628, 5668, 5528] dup_rows_per_dst=[2670, 2582, 2392, 2742, 2440, 2624, 2758, 2676] combine_rows_per_src=[5596, 5576, 5532, 5574, 5550, 5628, 5668, 5528] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44424 payload_rows_remote=38858 dup_rows=21112 dup_rows_remote=18552 dedup_ratio=0.322144 router_tokens_per_expert=[556, 541, 505, 555, 514, 503, 528, 521, 507, 529, 481, 504, 479, 508, 520, 517, 526, 441, 537, 493, 491, 519, 532, 524, 522, 512, 508, 526, 529, 494, 525, 521, 491, 514, 513, 554, 494, 499, 498, 516, 495, 519, 465, 509, 519, 558, 506, 531, 481, 541, 531, 555, 472, 522, 517, 517, 508, 517, 483, 490, 502, 494, 483, 506] payload_rows_per_dst=[5788, 5472, 5566, 5530, 5498, 5570, 5514, 5486] dup_rows_per_dst=[2658, 2618, 2560, 2744, 2660, 2634, 2758, 2480] combine_rows_per_src=[5788, 5472, 5566, 5530, 5498, 5570, 5514, 5486] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44568 payload_rows_remote=38974 dup_rows=20968 dup_rows_remote=18366 dedup_ratio=0.319946 router_tokens_per_expert=[528, 509, 517, 509, 517, 504, 512, 485, 528, 513, 502, 500, 515, 529, 497, 456, 496, 495, 530, 528, 513, 482, 484, 554, 472, 508, 495, 524, 499, 530, 501, 524, 514, 533, 484, 549, 572, 502, 548, 498, 541, 528, 506, 535, 530, 483, 459, 543, 510, 506, 492, 546, 496, 549, 524, 507, 527, 494, 527, 492, 477, 523, 515, 502] payload_rows_per_dst=[5594, 5536, 5522, 5572, 5620, 5646, 5548, 5530] dup_rows_per_dst=[2602, 2560, 2642, 2534, 2780, 2604, 2662, 2584] combine_rows_per_src=[5594, 5536, 5522, 5572, 5620, 5646, 5548, 5530] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44642 payload_rows_remote=39064 dup_rows=20894 dup_rows_remote=18302 dedup_ratio=0.318817 router_tokens_per_expert=[543, 538, 533, 532, 513, 523, 522, 527, 501, 535, 497, 483, 515, 496, 489, 569, 525, 514, 521, 521, 499, 489, 523, 478, 555, 510, 512, 497, 502, 527, 535, 499, 492, 510, 512, 502, 502, 534, 517, 500, 454, 527, 528, 523, 504, 491, 491, 493, 531, 491, 539, 534, 548, 515, 504, 488, 446, 487, 534, 523, 503, 535, 515, 472] payload_rows_per_dst=[5732, 5578, 5568, 5594, 5554, 5480, 5594, 5542] dup_rows_per_dst=[2730, 2592, 2572, 2680, 2584, 2542, 2706, 2488] combine_rows_per_src=[5732, 5578, 5568, 5594, 5554, 5480, 5594, 5542] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=277659648 dispatch_meta_bytes=227152 combine_payload_bytes=277659648 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277886800 ingress_bytes=387760128 link_bytes=387760128 token_bytes=555546448 weight_movement_bytes=110100480 total_nvlink_bytes=665646928 weight_to_token_ratio=0.198184 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.743786382e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279236608 dispatch_meta_bytes=228936 combine_payload_bytes=279236608 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279465544 ingress_bytes=499437568 link_bytes=499437568 token_bytes=558702152 weight_movement_bytes=220200960 total_nvlink_bytes=778903112 weight_to_token_ratio=0.394129 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.686040741e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=280182784 dispatch_meta_bytes=230240 combine_payload_bytes=280182784 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280413024 ingress_bytes=610484224 link_bytes=610484224 token_bytes=560595808 weight_movement_bytes=330301440 total_nvlink_bytes=890897248 weight_to_token_ratio=0.589197 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061732473e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=280283136 dispatch_meta_bytes=230184 combine_payload_bytes=280283136 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=280513320 ingress_bytes=500484096 link_bytes=500484096 token_bytes=560796456 weight_movement_bytes=220200960 total_nvlink_bytes=780997416 weight_to_token_ratio=0.392658 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.704241584e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278534144 dispatch_meta_bytes=229640 combine_payload_bytes=278534144 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278763784 ingress_bytes=388634624 link_bytes=388634624 token_bytes=557297928 weight_movement_bytes=110100480 total_nvlink_bytes=667398408 weight_to_token_ratio=0.197561 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.758995305e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279365632 dispatch_meta_bytes=229360 combine_payload_bytes=279365632 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279594992 ingress_bytes=499566592 link_bytes=499566592 token_bytes=558960624 weight_movement_bytes=220200960 total_nvlink_bytes=779161584 weight_to_token_ratio=0.393947 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.688284681e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44666 payload_rows_remote=39084 dup_rows=20870 dup_rows_remote=18366 dedup_ratio=0.318451 router_tokens_per_expert=[509, 514, 531, 535, 506, 529, 523, 534, 479, 518, 515, 510, 521, 497, 528, 524, 498, 517, 508, 491, 508, 492, 537, 516, 496, 499, 511, 510, 515, 492, 516, 504, 536, 495, 516, 547, 557, 536, 481, 518, 533, 491, 499, 526, 513, 479, 505, 497, 537, 484, 485, 506, 532, 483, 524, 509, 503, 496, 549, 514, 504, 550, 485, 495] payload_rows_per_dst=[5626, 5606, 5544, 5582, 5680, 5512, 5572, 5544] dup_rows_per_dst=[2736, 2578, 2590, 2504, 2692, 2574, 2548, 2648] combine_rows_per_src=[5626, 5606, 5544, 5582, 5680, 5512, 5572, 5544] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=280010752 dispatch_meta_bytes=229464 combine_payload_bytes=280010752 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280240216 ingress_bytes=500211712 link_bytes=500211712 token_bytes=560250968 weight_movement_bytes=220200960 total_nvlink_bytes=780451928 weight_to_token_ratio=0.393040 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.699504379e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=280154112 dispatch_meta_bytes=229800 combine_payload_bytes=280154112 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280383912 ingress_bytes=390254592 link_bytes=390254592 token_bytes=560538024 weight_movement_bytes=110100480 total_nvlink_bytes=670638504 weight_to_token_ratio=0.196419 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.787169213e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5587398112 prefetch_bytes=550502400 reduce_bytes=2091909120 weight_movement_bytes=2642411520 egress_bytes=2794845664 ingress_bytes=5434963968 link_bytes=5434963968 total_nvlink_bytes=8229809632 weight_to_token_ratio=0.472923 prefetch_share_of_ingress=0.101289 dedup_ratio=0.320789 prefetch_slots=10 unused_prefetch_slots=150 slot_utilization=0.062500 reduce_slots=19 remote_demand=5 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040074 gemm_inner_pad_upper_bound=12065 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.452296238e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44458 payload_rows_remote=38692 dup_rows=21078 dup_rows_remote=17960 dedup_ratio=0.321625 router_tokens_per_expert=[510, 486, 486, 503, 485, 526, 514, 524, 488, 518, 486, 520, 530, 559, 502, 519, 524, 509, 526, 503, 527, 545, 481, 516, 523, 509, 490, 500, 500, 558, 514, 510, 520, 526, 537, 475, 493, 462, 507, 504, 519, 516, 509, 495, 522, 497, 482, 517, 552, 527, 543, 499, 490, 530, 501, 538, 519, 506, 513, 494, 545, 482, 523, 534] payload_rows_per_dst=[5134, 5326, 5576, 5612, 5530, 5868, 5646, 5766] dup_rows_per_dst=[1882, 2266, 2576, 2706, 2518, 3298, 2714, 3118] combine_rows_per_src=[5134, 5326, 5576, 5612, 5530, 5868, 5646, 5766] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44450 payload_rows_remote=38996 dup_rows=21086 dup_rows_remote=18602 dedup_ratio=0.321747 router_tokens_per_expert=[520, 534, 476, 508, 483, 557, 548, 534, 549, 543, 524, 530, 530, 513, 512, 501, 498, 532, 479, 534, 501, 490, 519, 525, 514, 474, 499, 479, 497, 486, 531, 498, 528, 518, 515, 525, 562, 492, 519, 487, 514, 467, 512, 507, 479, 515, 496, 479, 522, 514, 541, 544, 526, 496, 548, 502, 528, 490, 485, 505, 528, 485, 503, 518] payload_rows_per_dst=[5630, 5632, 5500, 5418, 5700, 5454, 5616, 5500] dup_rows_per_dst=[2690, 2772, 2656, 2538, 2592, 2484, 2770, 2584] combine_rows_per_src=[5630, 5632, 5500, 5418, 5700, 5454, 5616, 5500] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44576 payload_rows_remote=39024 dup_rows=20960 dup_rows_remote=18210 dedup_ratio=0.319824 router_tokens_per_expert=[513, 509, 535, 526, 524, 501, 562, 518, 502, 504, 460, 488, 518, 531, 483, 519, 477, 513, 560, 575, 566, 471, 527, 519, 521, 485, 529, 466, 502, 544, 517, 502, 517, 504, 548, 500, 537, 481, 502, 499, 511, 492, 484, 500, 465, 541, 519, 512, 495, 530, 523, 508, 519, 536, 507, 533, 501, 500, 484, 521, 541, 510, 487, 494] payload_rows_per_dst=[5592, 5502, 5648, 5568, 5578, 5578, 5552, 5558] dup_rows_per_dst=[2580, 2508, 2768, 2564, 2598, 2674, 2750, 2518] combine_rows_per_src=[5592, 5502, 5648, 5568, 5578, 5578, 5552, 5558] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44426 payload_rows_remote=38844 dup_rows=21110 dup_rows_remote=18364 dedup_ratio=0.322113 router_tokens_per_expert=[514, 530, 476, 505, 552, 551, 525, 490, 551, 544, 533, 533, 452, 527, 511, 575, 497, 494, 508, 502, 484, 534, 498, 485, 546, 532, 508, 504, 527, 487, 507, 498, 512, 517, 535, 535, 521, 505, 521, 518, 496, 509, 518, 483, 509, 494, 514, 502, 498, 494, 534, 523, 521, 525, 479, 507, 511, 492, 510, 472, 525, 512, 524, 472] payload_rows_per_dst=[5596, 5702, 5496, 5586, 5582, 5432, 5570, 5462] dup_rows_per_dst=[2690, 2750, 2508, 2632, 2746, 2618, 2592, 2574] combine_rows_per_src=[5596, 5702, 5496, 5586, 5582, 5432, 5570, 5462] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44624 payload_rows_remote=39028 dup_rows=20912 dup_rows_remote=18364 dedup_ratio=0.319092 router_tokens_per_expert=[567, 507, 503, 514, 510, 535, 487, 498, 521, 518, 513, 529, 521, 485, 573, 542, 496, 530, 509, 471, 514, 495, 506, 551, 522, 536, 518, 531, 536, 544, 494, 537, 490, 474, 499, 547, 492, 496, 515, 485, 503, 522, 509, 482, 497, 555, 527, 514, 524, 509, 529, 491, 494, 508, 536, 472, 508, 526, 501, 484, 521, 472, 479, 494] payload_rows_per_dst=[5572, 5634, 5596, 5684, 5492, 5638, 5534, 5474] dup_rows_per_dst=[2670, 2770, 2548, 2752, 2504, 2580, 2592, 2496] combine_rows_per_src=[5572, 5634, 5596, 5684, 5492, 5638, 5534, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278175744 dispatch_meta_bytes=228520 combine_payload_bytes=278175744 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278404264 ingress_bytes=498376704 link_bytes=498376704 token_bytes=556580008 weight_movement_bytes=220200960 total_nvlink_bytes=776780968 weight_to_token_ratio=0.395632 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.667590571e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44324 payload_rows_remote=38764 dup_rows=21212 dup_rows_remote=18526 dedup_ratio=0.323669 router_tokens_per_expert=[537, 473, 536, 505, 513, 512, 483, 500, 526, 522, 535, 545, 505, 513, 495, 538, 518, 505, 509, 504, 521, 523, 481, 553, 494, 508, 518, 482, 462, 487, 500, 506, 496, 531, 518, 523, 516, 530, 528, 487, 513, 506, 513, 494, 527, 491, 533, 536, 508, 523, 521, 491, 507, 539, 520, 541, 472, 542, 480, 551, 490, 524, 501, 507] payload_rows_per_dst=[5404, 5362, 5246, 5642, 5856, 5632, 5622, 5560] dup_rows_per_dst=[2470, 2402, 2382, 2776, 3056, 2714, 2726, 2686] combine_rows_per_src=[5404, 5362, 5246, 5642, 5856, 5632, 5622, 5560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44438 payload_rows_remote=38886 dup_rows=21098 dup_rows_remote=18468 dedup_ratio=0.321930 router_tokens_per_expert=[490, 500, 518, 508, 503, 515, 499, 529, 540, 537, 500, 541, 541, 529, 536, 518, 513, 495, 521, 514, 513, 581, 497, 518, 532, 501, 486, 495, 502, 537, 513, 525, 479, 480, 499, 517, 532, 509, 498, 485, 495, 514, 549, 463, 472, 535, 503, 487, 490, 536, 535, 479, 510, 524, 481, 516, 485, 504, 520, 526, 526, 487, 559, 526] payload_rows_per_dst=[5548, 5736, 5554, 5552, 5480, 5450, 5570, 5548] dup_rows_per_dst=[2576, 2748, 2750, 2630, 2518, 2586, 2572, 2718] combine_rows_per_src=[5548, 5736, 5554, 5552, 5480, 5450, 5570, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44464 payload_rows_remote=38864 dup_rows=21072 dup_rows_remote=18500 dedup_ratio=0.321533 router_tokens_per_expert=[515, 525, 511, 535, 518, 487, 526, 511, 558, 513, 507, 512, 536, 466, 501, 493, 510, 487, 521, 510, 535, 527, 556, 508, 496, 531, 532, 514, 538, 538, 519, 541, 453, 517, 510, 503, 494, 494, 469, 521, 499, 500, 522, 483, 493, 496, 542, 475, 518, 519, 525, 549, 529, 525, 515, 478, 506, 514, 516, 472, 513, 504, 471, 566] payload_rows_per_dst=[5608, 5600, 5594, 5596, 5494, 5478, 5588, 5506] dup_rows_per_dst=[2648, 2572, 2714, 2822, 2428, 2542, 2728, 2618] combine_rows_per_src=[5608, 5600, 5594, 5596, 5494, 5478, 5588, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279035904 dispatch_meta_bytes=229432 combine_payload_bytes=279035904 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279265336 ingress_bytes=389136384 link_bytes=389136384 token_bytes=558301240 weight_movement_bytes=110100480 total_nvlink_bytes=668401720 weight_to_token_ratio=0.197206 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.767721737e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279795712 dispatch_meta_bytes=229920 combine_payload_bytes=279795712 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280025632 ingress_bytes=389896192 link_bytes=389896192 token_bytes=559821344 weight_movement_bytes=110100480 total_nvlink_bytes=669921824 weight_to_token_ratio=0.196671 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.780936048e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44626 payload_rows_remote=39042 dup_rows=20910 dup_rows_remote=18254 dedup_ratio=0.319061 router_tokens_per_expert=[521, 454, 529, 509, 503, 525, 556, 523, 492, 521, 561, 486, 496, 508, 517, 489, 499, 547, 527, 521, 514, 522, 473, 515, 521, 527, 471, 513, 543, 531, 489, 494, 510, 485, 489, 545, 503, 525, 497, 502, 501, 513, 527, 562, 492, 574, 525, 511, 510, 495, 518, 503, 529, 489, 511, 521, 505, 506, 536, 528, 465, 501, 516, 477] payload_rows_per_dst=[5584, 5530, 5572, 5600, 5508, 5740, 5568, 5524] dup_rows_per_dst=[2656, 2610, 2664, 2578, 2604, 2670, 2584, 2544] combine_rows_per_src=[5584, 5530, 5572, 5600, 5508, 5740, 5568, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278749184 dispatch_meta_bytes=229576 combine_payload_bytes=278749184 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278978760 ingress_bytes=388849664 link_bytes=388849664 token_bytes=557727944 weight_movement_bytes=110100480 total_nvlink_bytes=667828424 weight_to_token_ratio=0.197409 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.762735205e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277860352 dispatch_meta_bytes=229160 combine_payload_bytes=277860352 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278089512 ingress_bytes=387960832 link_bytes=387960832 token_bytes=555949864 weight_movement_bytes=110100480 total_nvlink_bytes=666050344 weight_to_token_ratio=0.198040 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.747276954e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278734848 dispatch_meta_bytes=229416 combine_payload_bytes=278734848 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278964264 ingress_bytes=388835328 link_bytes=388835328 token_bytes=557699112 weight_movement_bytes=110100480 total_nvlink_bytes=667799592 weight_to_token_ratio=0.197419 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.762485878e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278577152 dispatch_meta_bytes=229456 combine_payload_bytes=278577152 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=278806608 ingress_bytes=718979072 link_bytes=718979072 token_bytes=557383760 weight_movement_bytes=440401920 total_nvlink_bytes=997785680 weight_to_token_ratio=0.790123 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.250422858e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279853056 dispatch_meta_bytes=229184 combine_payload_bytes=279853056 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=280082240 ingress_bytes=610154496 link_bytes=610154496 token_bytes=559935296 weight_movement_bytes=330301440 total_nvlink_bytes=890236736 weight_to_token_ratio=0.589892 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061159022e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44430 payload_rows_remote=38874 dup_rows=21106 dup_rows_remote=18560 dedup_ratio=0.322052 router_tokens_per_expert=[554, 537, 503, 552, 510, 501, 523, 518, 504, 527, 480, 500, 479, 506, 518, 513, 526, 440, 535, 492, 489, 518, 529, 522, 521, 514, 510, 525, 530, 497, 526, 524, 490, 514, 513, 554, 494, 500, 500, 519, 496, 519, 466, 510, 518, 558, 507, 531, 480, 541, 533, 558, 475, 524, 520, 521, 513, 521, 485, 493, 503, 496, 486, 507] payload_rows_per_dst=[5766, 5456, 5556, 5544, 5504, 5578, 5520, 5506] dup_rows_per_dst=[2630, 2598, 2546, 2750, 2664, 2632, 2784, 2502] combine_rows_per_src=[5766, 5456, 5556, 5544, 5504, 5578, 5520, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44626 payload_rows_remote=39068 dup_rows=20910 dup_rows_remote=18468 dedup_ratio=0.319061 router_tokens_per_expert=[515, 521, 509, 516, 514, 514, 506, 522, 502, 500, 527, 512, 530, 517, 501, 479, 500, 529, 486, 487, 482, 486, 475, 509, 536, 520, 546, 527, 472, 537, 497, 507, 491, 480, 519, 504, 524, 489, 490, 503, 539, 497, 489, 523, 531, 511, 526, 521, 513, 562, 515, 532, 538, 539, 523, 507, 543, 512, 488, 473, 522, 519, 531, 533] payload_rows_per_dst=[5580, 5568, 5518, 5550, 5558, 5632, 5674, 5546] dup_rows_per_dst=[2654, 2568, 2390, 2734, 2442, 2642, 2784, 2696] combine_rows_per_src=[5580, 5568, 5518, 5550, 5558, 5632, 5674, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44590 payload_rows_remote=38940 dup_rows=20946 dup_rows_remote=18254 dedup_ratio=0.319611 router_tokens_per_expert=[517, 522, 545, 535, 518, 487, 522, 524, 507, 538, 477, 498, 463, 508, 539, 484, 514, 561, 545, 537, 499, 523, 506, 480, 511, 477, 481, 494, 514, 466, 555, 501, 533, 499, 517, 508, 489, 500, 484, 485, 563, 513, 508, 493, 498, 488, 558, 550, 494, 490, 523, 531, 530, 525, 566, 508, 492, 502, 479, 501, 513, 519, 509, 552] payload_rows_per_dst=[5648, 5446, 5694, 5478, 5490, 5650, 5618, 5566] dup_rows_per_dst=[2692, 2582, 2636, 2520, 2540, 2692, 2716, 2568] combine_rows_per_src=[5648, 5446, 5694, 5478, 5490, 5650, 5618, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278648832 dispatch_meta_bytes=229736 combine_payload_bytes=278648832 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278878568 ingress_bytes=388749312 link_bytes=388749312 token_bytes=557527400 weight_movement_bytes=110100480 total_nvlink_bytes=667627880 weight_to_token_ratio=0.197480 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.760989918e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44600 payload_rows_remote=39092 dup_rows=20936 dup_rows_remote=18430 dedup_ratio=0.319458 router_tokens_per_expert=[485, 507, 506, 524, 507, 515, 506, 503, 498, 577, 495, 491, 502, 527, 551, 532, 498, 520, 525, 505, 519, 486, 537, 511, 525, 529, 502, 506, 493, 525, 491, 505, 519, 509, 524, 461, 564, 530, 492, 474, 506, 524, 518, 530, 525, 526, 525, 535, 494, 492, 494, 523, 513, 512, 473, 506, 540, 505, 504, 504, 514, 502, 503, 524] payload_rows_per_dst=[5548, 5620, 5588, 5564, 5540, 5654, 5508, 5578] dup_rows_per_dst=[2558, 2726, 2614, 2588, 2606, 2724, 2506, 2614] combine_rows_per_src=[5548, 5620, 5588, 5564, 5540, 5654, 5508, 5578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44598 payload_rows_remote=38852 dup_rows=20938 dup_rows_remote=18254 dedup_ratio=0.319489 router_tokens_per_expert=[500, 542, 502, 518, 503, 539, 507, 529, 527, 517, 527, 534, 497, 532, 532, 510, 518, 465, 494, 509, 497, 471, 515, 509, 517, 481, 527, 533, 506, 545, 525, 510, 523, 524, 529, 535, 511, 497, 515, 520, 507, 495, 514, 511, 531, 528, 500, 496, 516, 508, 547, 487, 501, 486, 510, 484, 509, 516, 502, 516, 527, 508, 490, 487] payload_rows_per_dst=[5526, 5660, 5540, 5576, 5608, 5534, 5408, 5746] dup_rows_per_dst=[2554, 2872, 2416, 2712, 2714, 2598, 2388, 2684] combine_rows_per_src=[5526, 5660, 5540, 5576, 5608, 5534, 5408, 5746] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44500 payload_rows_remote=39002 dup_rows=21036 dup_rows_remote=18506 dedup_ratio=0.320984 router_tokens_per_expert=[510, 490, 502, 528, 520, 481, 541, 499, 517, 522, 537, 497, 513, 521, 514, 508, 495, 491, 510, 491, 508, 497, 496, 491, 500, 446, 480, 514, 527, 545, 490, 512, 568, 506, 547, 537, 460, 535, 518, 529, 511, 498, 505, 529, 501, 523, 563, 521, 516, 511, 529, 509, 520, 487, 529, 493, 545, 527, 523, 501, 513, 486, 519, 516] payload_rows_per_dst=[5508, 5548, 5518, 5498, 5660, 5624, 5536, 5608] dup_rows_per_dst=[2634, 2710, 2440, 2530, 2740, 2678, 2652, 2652] combine_rows_per_src=[5508, 5548, 5518, 5498, 5660, 5624, 5536, 5608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=280039424 dispatch_meta_bytes=230144 combine_payload_bytes=280039424 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280269568 ingress_bytes=500240384 link_bytes=500240384 token_bytes=560308992 weight_movement_bytes=220200960 total_nvlink_bytes=780509952 weight_to_token_ratio=0.392999 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.700003032e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44320 payload_rows_remote=38786 dup_rows=21216 dup_rows_remote=18610 dedup_ratio=0.323730 router_tokens_per_expert=[515, 493, 510, 498, 507, 540, 477, 519, 514, 497, 509, 508, 490, 526, 524, 502, 505, 503, 525, 499, 513, 525, 515, 523, 513, 495, 523, 494, 541, 536, 495, 526, 509, 476, 521, 538, 529, 503, 482, 547, 565, 490, 519, 554, 514, 506, 514, 511, 480, 512, 507, 526, 486, 516, 514, 532, 556, 483, 516, 469, 520, 475, 518, 520] payload_rows_per_dst=[5512, 5534, 5580, 5548, 5498, 5582, 5572, 5494] dup_rows_per_dst=[2606, 2606, 2636, 2698, 2712, 2764, 2574, 2620] combine_rows_per_src=[5512, 5534, 5580, 5548, 5498, 5582, 5572, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279121920 dispatch_meta_bytes=228776 combine_payload_bytes=279121920 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279350696 ingress_bytes=609423360 link_bytes=609423360 token_bytes=558472616 weight_movement_bytes=330301440 total_nvlink_bytes=888774056 weight_to_token_ratio=0.591437 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059887456e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44572 payload_rows_remote=39010 dup_rows=20964 dup_rows_remote=18406 dedup_ratio=0.319885 router_tokens_per_expert=[528, 508, 514, 507, 513, 499, 509, 482, 526, 509, 500, 493, 512, 530, 495, 456, 494, 493, 532, 525, 510, 482, 485, 550, 471, 509, 496, 525, 499, 530, 500, 521, 511, 533, 484, 548, 573, 502, 551, 500, 540, 529, 509, 537, 531, 483, 462, 546, 510, 509, 495, 546, 499, 553, 525, 511, 530, 496, 527, 496, 480, 527, 517, 505] payload_rows_per_dst=[5562, 5514, 5792, 5630, 5570, 5372, 5578, 5554] dup_rows_per_dst=[2558, 2528, 3072, 2616, 2690, 2180, 2718, 2602] combine_rows_per_src=[5562, 5514, 5792, 5630, 5570, 5372, 5578, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=280211456 dispatch_meta_bytes=230088 combine_payload_bytes=280211456 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280441544 ingress_bytes=500412416 link_bytes=500412416 token_bytes=560653000 weight_movement_bytes=220200960 total_nvlink_bytes=780853960 weight_to_token_ratio=0.392758 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.702994951e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278491136 dispatch_meta_bytes=228424 combine_payload_bytes=278491136 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278719560 ingress_bytes=388591616 link_bytes=388591616 token_bytes=557210696 weight_movement_bytes=110100480 total_nvlink_bytes=667311176 weight_to_token_ratio=0.197592 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.758247326e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279566336 dispatch_meta_bytes=230032 combine_payload_bytes=279566336 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279796368 ingress_bytes=389666816 link_bytes=389666816 token_bytes=559362704 weight_movement_bytes=110100480 total_nvlink_bytes=669463184 weight_to_token_ratio=0.196832 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.776946822e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278018048 dispatch_meta_bytes=229584 combine_payload_bytes=278018048 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278247632 ingress_bytes=388118528 link_bytes=388118528 token_bytes=556265680 weight_movement_bytes=110100480 total_nvlink_bytes=666366160 weight_to_token_ratio=0.197928 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.750019547e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44638 payload_rows_remote=39038 dup_rows=20898 dup_rows_remote=18368 dedup_ratio=0.318878 router_tokens_per_expert=[565, 504, 500, 507, 508, 531, 485, 494, 520, 516, 513, 527, 522, 482, 571, 541, 494, 531, 510, 471, 511, 496, 501, 551, 520, 536, 517, 529, 538, 546, 494, 535, 490, 475, 500, 548, 494, 495, 516, 486, 500, 524, 509, 484, 496, 556, 529, 514, 527, 512, 534, 492, 494, 510, 538, 472, 510, 530, 502, 487, 524, 475, 483, 496] payload_rows_per_dst=[5548, 5626, 5600, 5690, 5496, 5642, 5550, 5486] dup_rows_per_dst=[2640, 2758, 2530, 2740, 2512, 2582, 2608, 2528] combine_rows_per_src=[5548, 5626, 5600, 5690, 5496, 5642, 5550, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279623680 dispatch_meta_bytes=229664 combine_payload_bytes=279623680 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279853344 ingress_bytes=609925120 link_bytes=609925120 token_bytes=559477024 weight_movement_bytes=330301440 total_nvlink_bytes=889778464 weight_to_token_ratio=0.590375 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.051471 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060760099e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44428 payload_rows_remote=38846 dup_rows=21108 dup_rows_remote=18364 dedup_ratio=0.322083 router_tokens_per_expert=[513, 530, 472, 504, 549, 545, 521, 488, 551, 540, 529, 531, 448, 528, 510, 575, 497, 493, 507, 501, 483, 532, 497, 481, 542, 532, 508, 503, 526, 487, 506, 499, 513, 514, 532, 530, 521, 508, 524, 521, 498, 508, 520, 483, 513, 495, 516, 503, 500, 497, 535, 526, 523, 527, 485, 509, 514, 494, 511, 476, 529, 513, 527, 475] payload_rows_per_dst=[5582, 5690, 5482, 5580, 5582, 5444, 5592, 5476] dup_rows_per_dst=[2662, 2734, 2500, 2626, 2744, 2628, 2612, 2602] combine_rows_per_src=[5582, 5690, 5482, 5580, 5582, 5444, 5592, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279824384 dispatch_meta_bytes=229624 combine_payload_bytes=279824384 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280054008 ingress_bytes=500025344 link_bytes=500025344 token_bytes=559878392 weight_movement_bytes=220200960 total_nvlink_bytes=780079352 weight_to_token_ratio=0.393301 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.696263133e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44458 payload_rows_remote=38998 dup_rows=21078 dup_rows_remote=18600 dedup_ratio=0.321625 router_tokens_per_expert=[518, 533, 476, 505, 478, 553, 550, 533, 548, 542, 517, 528, 526, 511, 510, 503, 496, 531, 480, 533, 504, 490, 518, 524, 513, 476, 497, 479, 498, 485, 528, 498, 526, 521, 515, 523, 563, 494, 518, 486, 513, 467, 512, 507, 479, 515, 496, 480, 524, 514, 541, 544, 525, 498, 550, 506, 531, 495, 489, 508, 534, 487, 505, 521] payload_rows_per_dst=[5624, 5614, 5498, 5416, 5694, 5460, 5630, 5522] dup_rows_per_dst=[2668, 2756, 2654, 2532, 2598, 2478, 2774, 2618] combine_rows_per_src=[5624, 5614, 5498, 5416, 5694, 5460, 5630, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44546 payload_rows_remote=38988 dup_rows=20990 dup_rows_remote=18216 dedup_ratio=0.320282 router_tokens_per_expert=[512, 506, 531, 524, 521, 496, 560, 513, 498, 501, 456, 488, 516, 529, 483, 517, 476, 513, 561, 573, 568, 468, 527, 519, 521, 484, 529, 466, 501, 540, 517, 504, 517, 507, 548, 500, 540, 482, 501, 499, 513, 496, 483, 502, 466, 542, 519, 513, 497, 531, 525, 509, 521, 535, 511, 537, 499, 502, 490, 521, 542, 510, 494, 498] payload_rows_per_dst=[5644, 5494, 5638, 5566, 5580, 5490, 5558, 5576] dup_rows_per_dst=[2682, 2482, 2772, 2558, 2608, 2578, 2774, 2536] combine_rows_per_src=[5644, 5494, 5638, 5566, 5580, 5490, 5558, 5576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44448 payload_rows_remote=38790 dup_rows=21088 dup_rows_remote=18144 dedup_ratio=0.321777 router_tokens_per_expert=[506, 483, 484, 496, 481, 524, 509, 523, 486, 516, 484, 516, 528, 557, 502, 517, 525, 508, 528, 503, 525, 545, 481, 518, 523, 505, 489, 499, 499, 557, 514, 508, 521, 527, 541, 475, 492, 462, 508, 502, 519, 518, 511, 496, 524, 498, 483, 517, 552, 529, 543, 500, 492, 532, 504, 541, 525, 511, 513, 496, 546, 485, 528, 538] payload_rows_per_dst=[5244, 5444, 5618, 5646, 5540, 5474, 5824, 5658] dup_rows_per_dst=[2124, 2450, 2648, 2728, 2516, 2658, 3020, 2944] combine_rows_per_src=[5244, 5444, 5618, 5646, 5540, 5474, 5824, 5658] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44668 payload_rows_remote=39090 dup_rows=20868 dup_rows_remote=18374 dedup_ratio=0.318420 router_tokens_per_expert=[508, 509, 528, 534, 502, 525, 522, 530, 476, 516, 515, 508, 520, 495, 527, 524, 498, 515, 507, 490, 507, 491, 538, 515, 495, 499, 508, 509, 513, 492, 516, 504, 534, 498, 518, 545, 559, 535, 480, 518, 534, 493, 499, 529, 512, 482, 505, 499, 538, 487, 489, 507, 535, 487, 526, 509, 505, 497, 551, 517, 504, 554, 490, 496] payload_rows_per_dst=[5604, 5596, 5544, 5578, 5686, 5518, 5582, 5560] dup_rows_per_dst=[2712, 2566, 2578, 2494, 2688, 2588, 2574, 2668] combine_rows_per_src=[5604, 5596, 5544, 5578, 5686, 5518, 5582, 5560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278448128 dispatch_meta_bytes=228840 combine_payload_bytes=278448128 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=278676968 ingress_bytes=498649088 link_bytes=498649088 token_bytes=557125096 weight_movement_bytes=220200960 total_nvlink_bytes=777326056 weight_to_token_ratio=0.395245 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.672327777e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44646 payload_rows_remote=39078 dup_rows=20890 dup_rows_remote=18308 dedup_ratio=0.318756 router_tokens_per_expert=[543, 538, 529, 532, 508, 522, 520, 524, 501, 535, 496, 482, 513, 492, 489, 567, 524, 514, 519, 520, 498, 487, 522, 477, 557, 509, 511, 497, 501, 526, 532, 501, 492, 510, 510, 501, 501, 534, 518, 501, 455, 530, 527, 523, 502, 492, 493, 496, 532, 490, 540, 534, 551, 520, 506, 490, 449, 487, 535, 524, 504, 535, 520, 480] payload_rows_per_dst=[5720, 5568, 5558, 5592, 5554, 5538, 5554, 5562] dup_rows_per_dst=[2712, 2582, 2564, 2676, 2580, 2618, 2652, 2506] combine_rows_per_src=[5720, 5568, 5558, 5592, 5554, 5538, 5554, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5578940064 prefetch_bytes=1101004800 reduce_bytes=770703360 weight_movement_bytes=1871708160 egress_bytes=2790616736 ingress_bytes=4660031488 link_bytes=4660031488 total_nvlink_bytes=7450648224 weight_to_token_ratio=0.335495 prefetch_share_of_ingress=0.236266 dedup_ratio=0.321365 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=7 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.104561201e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5585019616 prefetch_bytes=1101004800 reduce_bytes=660602880 weight_movement_bytes=1761607680 egress_bytes=2793657056 ingress_bytes=4552970240 link_bytes=4552970240 total_nvlink_bytes=7346627296 weight_to_token_ratio=0.315417 prefetch_share_of_ingress=0.241821 dedup_ratio=0.320959 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=6 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.918364082e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44674 payload_rows_remote=38924 dup_rows=20862 dup_rows_remote=17936 dedup_ratio=0.318329 router_tokens_per_expert=[547, 543, 521, 513, 509, 517, 556, 485, 554, 536, 483, 476, 523, 501, 512, 508, 526, 541, 505, 559, 531, 501, 506, 518, 492, 502, 548, 515, 513, 481, 516, 491, 502, 467, 525, 513, 520, 513, 539, 521, 463, 456, 509, 511, 493, 481, 500, 489, 474, 492, 507, 491, 528, 523, 549, 541, 527, 500, 490, 521, 523, 503, 522, 546] payload_rows_per_dst=[5750, 5618, 5650, 5580, 5634, 5408, 5444, 5590] dup_rows_per_dst=[2926, 2702, 2740, 2536, 2552, 2396, 2336, 2674] combine_rows_per_src=[5750, 5618, 5650, 5580, 5634, 5408, 5444, 5590] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=277344256 dispatch_meta_bytes=226608 combine_payload_bytes=277344256 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277570864 ingress_bytes=387444736 link_bytes=387444736 token_bytes=554915120 weight_movement_bytes=110100480 total_nvlink_bytes=665015600 weight_to_token_ratio=0.198410 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.738301196e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279523328 dispatch_meta_bytes=230392 combine_payload_bytes=279523328 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279753720 ingress_bytes=389623808 link_bytes=389623808 token_bytes=559277048 weight_movement_bytes=110100480 total_nvlink_bytes=669377528 weight_to_token_ratio=0.196862 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4096 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.776198842e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279724032 dispatch_meta_bytes=228936 combine_payload_bytes=279724032 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279952968 ingress_bytes=610025472 link_bytes=610025472 token_bytes=559677000 weight_movement_bytes=330301440 total_nvlink_bytes=889978440 weight_to_token_ratio=0.590164 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060934628e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278433792 dispatch_meta_bytes=228832 combine_payload_bytes=278433792 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278662624 ingress_bytes=498634752 link_bytes=498634752 token_bytes=557096416 weight_movement_bytes=220200960 total_nvlink_bytes=777297376 weight_to_token_ratio=0.395265 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.672078451e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279752704 dispatch_meta_bytes=229568 combine_payload_bytes=279752704 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279982272 ingress_bytes=499953664 link_bytes=499953664 token_bytes=559734976 weight_movement_bytes=220200960 total_nvlink_bytes=779935936 weight_to_token_ratio=0.393402 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.695016499e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279007232 dispatch_meta_bytes=227440 combine_payload_bytes=279007232 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279234672 ingress_bytes=499208192 link_bytes=499208192 token_bytes=558241904 weight_movement_bytes=220200960 total_nvlink_bytes=778442864 weight_to_token_ratio=0.394454 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.682051515e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5553092064 prefetch_bytes=1101004800 reduce_bytes=220200960 weight_movement_bytes=1321205760 egress_bytes=2777685472 ingress_bytes=4096612352 link_bytes=4096612352 total_nvlink_bytes=6874297824 weight_to_token_ratio=0.237923 prefetch_share_of_ingress=0.268760 dedup_ratio=0.322467 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=2 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042647 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.124682657e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5593509664 prefetch_bytes=1101004800 reduce_bytes=330301440 weight_movement_bytes=1431306240 egress_bytes=2797903648 ingress_bytes=4226912256 link_bytes=4226912256 total_nvlink_bytes=7024815904 weight_to_token_ratio=0.255887 prefetch_share_of_ingress=0.260475 dedup_ratio=0.320496 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=3 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.351295621e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5596089504 prefetch_bytes=990904320 reduce_bytes=1101004800 weight_movement_bytes=2091909120 egress_bytes=2799193248 ingress_bytes=4888805376 link_bytes=4888805376 total_nvlink_bytes=7687998624 weight_to_token_ratio=0.373816 prefetch_share_of_ingress=0.202688 dedup_ratio=0.320557 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=10 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.502436619e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5578423712 prefetch_bytes=880803840 reduce_bytes=770703360 weight_movement_bytes=1651507200 egress_bytes=2790358432 ingress_bytes=4439572480 link_bytes=4439572480 total_nvlink_bytes=7229930912 weight_to_token_ratio=0.296053 prefetch_share_of_ingress=0.198398 dedup_ratio=0.321640 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=7 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.721146727e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5578996872 prefetch_bytes=990904320 reduce_bytes=1211105280 weight_movement_bytes=2202009600 egress_bytes=2790644872 ingress_bytes=4990361600 link_bytes=4990361600 total_nvlink_bytes=7781006472 weight_to_token_ratio=0.394696 prefetch_share_of_ingress=0.198564 dedup_ratio=0.321304 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=11 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040441 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.679059596e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5575035072 prefetch_bytes=990904320 reduce_bytes=1321205760 weight_movement_bytes=2312110080 egress_bytes=2788661440 ingress_bytes=5098483712 link_bytes=5098483712 total_nvlink_bytes=7887145152 weight_to_token_ratio=0.414726 prefetch_share_of_ingress=0.194353 dedup_ratio=0.320892 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=12 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.867101732e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none - [2026-08-26 15:12:41] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 367.7 | throughput per GPU (TFLOP/s/GPU): 335.9 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.427058E+00 | loss scale: 1.0 | grad norm: 3.400 | number of skipped iterations: 0 | number of nan iterations: 0 | -[after training is done] datetime: 2026-08-26 15:12:41 -[rank0]:[W826 15:12:42.728502008 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank3]:[W826 15:12:43.710759834 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank5]:[W826 15:12:43.716647979 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank4]:[W826 15:12:43.722567356 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank1]:[W826 15:12:43.730298525 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank7]:[W826 15:12:43.748984049 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank6]:[W826 15:12:44.282671207 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank2]:[W826 15:12:44.533200910 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -W0826 15:12:45.269000 76686 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers -W0826 15:12:45.269000 76686 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 76779 closing signal SIGTERM -W0826 15:12:45.270000 76686 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 76780 closing signal SIGTERM -W0826 15:12:45.271000 76686 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 76781 closing signal SIGTERM -W0826 15:12:45.271000 76686 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 76782 closing signal SIGTERM -W0826 15:12:45.273000 76686 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 76783 closing signal SIGTERM -W0826 15:12:45.273000 76686 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 76784 closing signal SIGTERM -W0826 15:12:45.274000 76686 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 76785 closing signal SIGTERM -W0826 15:12:45.274000 76686 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 76786 closing signal SIGTERM +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44696 payload_rows_remote=39052 dup_rows=20840 dup_rows_remote=18114 dedup_ratio=0.317993 router_tokens_per_expert=[544, 540, 517, 509, 507, 514, 553, 480, 551, 534, 480, 473, 522, 502, 511, 508, 525, 541, 502, 558, 529, 501, 506, 518, 492, 501, 547, 517, 512, 479, 515, 494, 506, 467, 525, 511, 520, 512, 539, 523, 462, 458, 508, 514, 493, 484, 501, 491, 477, 494, 509, 495, 533, 524, 550, 543, 528, 501, 490, 523, 525, 505, 525, 550] payload_rows_per_dst=[5644, 5582, 5640, 5572, 5624, 5806, 5220, 5608] dup_rows_per_dst=[2726, 2594, 2712, 2542, 2534, 3034, 2012, 2686] combine_rows_per_src=[5644, 5582, 5640, 5572, 5624, 5806, 5220, 5608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279537664 dispatch_meta_bytes=230392 combine_payload_bytes=279537664 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279768056 ingress_bytes=389638144 link_bytes=389638144 token_bytes=559305720 weight_movement_bytes=110100480 total_nvlink_bytes=669406200 weight_to_token_ratio=0.196852 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4096 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.776448169e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279465984 dispatch_meta_bytes=228816 combine_payload_bytes=279465984 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279694800 ingress_bytes=499666944 link_bytes=499666944 token_bytes=559160784 weight_movement_bytes=220200960 total_nvlink_bytes=779361744 weight_to_token_ratio=0.393806 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.690029967e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278046720 dispatch_meta_bytes=227736 combine_payload_bytes=278046720 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278274456 ingress_bytes=388147200 link_bytes=388147200 token_bytes=556321176 weight_movement_bytes=110100480 total_nvlink_bytes=666421656 weight_to_token_ratio=0.197908 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.750518200e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5576158248 prefetch_bytes=880803840 reduce_bytes=880803840 weight_movement_bytes=1761607680 egress_bytes=2789225512 ingress_bytes=4548540416 link_bytes=4548540416 total_nvlink_bytes=7337765928 weight_to_token_ratio=0.315918 prefetch_share_of_ingress=0.193645 dedup_ratio=0.321817 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=8 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.910659890e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=280197120 dispatch_meta_bytes=229856 combine_payload_bytes=280197120 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280426976 ingress_bytes=390297600 link_bytes=390297600 token_bytes=560624096 weight_movement_bytes=110100480 total_nvlink_bytes=670724576 weight_to_token_ratio=0.196389 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.787917193e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=280111104 dispatch_meta_bytes=229544 combine_payload_bytes=280111104 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280340648 ingress_bytes=500312064 link_bytes=500312064 token_bytes=560451752 weight_movement_bytes=220200960 total_nvlink_bytes=780652712 weight_to_token_ratio=0.392899 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.701249665e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5591673912 prefetch_bytes=880803840 reduce_bytes=660602880 weight_movement_bytes=1541406720 egress_bytes=2796985400 ingress_bytes=4336095232 link_bytes=4336095232 total_nvlink_bytes=7133080632 weight_to_token_ratio=0.275661 prefetch_share_of_ingress=0.203133 dedup_ratio=0.320596 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=6 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040074 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.541182774e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279924736 dispatch_meta_bytes=228664 combine_payload_bytes=279924736 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280153400 ingress_bytes=610226176 link_bytes=610226176 token_bytes=560078136 weight_movement_bytes=330301440 total_nvlink_bytes=890379576 weight_to_token_ratio=0.589742 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061283685e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5594396536 prefetch_bytes=990904320 reduce_bytes=990904320 weight_movement_bytes=1981808640 egress_bytes=2798346104 ingress_bytes=4777859072 link_bytes=4777859072 total_nvlink_bytes=7576205176 weight_to_token_ratio=0.354249 prefetch_share_of_ingress=0.207395 dedup_ratio=0.320505 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=9 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043015 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.309482749e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5569186224 prefetch_bytes=990904320 reduce_bytes=550502400 weight_movement_bytes=1541406720 egress_bytes=2785737136 ingress_bytes=4324855808 link_bytes=4324855808 total_nvlink_bytes=7110592944 weight_to_token_ratio=0.276774 prefetch_share_of_ingress=0.229118 dedup_ratio=0.321884 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=5 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043382 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.521635567e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5585622176 prefetch_bytes=770703360 reduce_bytes=1431306240 weight_movement_bytes=2202009600 egress_bytes=2793958560 ingress_bytes=4993673216 link_bytes=4993673216 total_nvlink_bytes=7787631776 weight_to_token_ratio=0.394228 prefetch_share_of_ingress=0.154336 dedup_ratio=0.320947 prefetch_slots=14 unused_prefetch_slots=146 slot_utilization=0.087500 reduce_slots=13 remote_demand=7 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=12319 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.684819041e-05 prefetch_bytes_per_gemm_flop=1.340379902e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5589406320 prefetch_bytes=1101004800 reduce_bytes=770703360 weight_movement_bytes=1871708160 egress_bytes=2795850352 ingress_bytes=4665264128 link_bytes=4665264128 total_nvlink_bytes=7461114480 weight_to_token_ratio=0.334867 prefetch_share_of_ingress=0.236001 dedup_ratio=0.320743 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=7 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040074 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.113661623e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5576268912 prefetch_bytes=880803840 reduce_bytes=1541406720 weight_movement_bytes=2422210560 egress_bytes=2789278832 ingress_bytes=5209200640 link_bytes=5209200640 total_nvlink_bytes=7998479472 weight_to_token_ratio=0.434378 prefetch_share_of_ingress=0.169086 dedup_ratio=0.320901 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=14 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.059656679e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none + [2026-08-26 16:00:29] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 366.1 | throughput per GPU (TFLOP/s/GPU): 337.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.429613E+00 | loss scale: 1.0 | grad norm: 3.374 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-26 16:00:29 +[rank0]:[W826 16:00:30.919741065 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W826 16:00:31.820466013 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W826 16:00:31.977932726 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W826 16:00:32.451985469 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W826 16:00:32.460693363 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W826 16:00:32.466710713 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W826 16:00:32.466972493 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W826 16:00:32.985981190 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 16:00:33.192000 165720 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 16:00:33.193000 165720 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 165813 closing signal SIGTERM +W0826 16:00:33.193000 165720 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 165814 closing signal SIGTERM +W0826 16:00:33.195000 165720 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 165815 closing signal SIGTERM +W0826 16:00:33.196000 165720 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 165816 closing signal SIGTERM +W0826 16:00:33.197000 165720 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 165817 closing signal SIGTERM +W0826 16:00:33.199000 165720 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 165818 closing signal SIGTERM +W0826 16:00:33.199000 165720 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 165819 closing signal SIGTERM +W0826 16:00:33.200000 165720 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 165820 closing signal SIGTERM Traceback (most recent call last): File "/usr/local/bin/torchrun", line 8, in sys.exit(main()) @@ -3553,4 +3537,4 @@ Traceback (most recent call last): time.sleep(monitor_interval) File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 76686 got signal: 15 +torch.distributed.elastic.multiprocessing.api.SignalException: Process 165720 got signal: 15 diff --git a/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1/mem_samples.csv b/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1/mem_samples.csv index 5e4b42cd700a09479c0e585b3d1421e1a933256d..579a190d70228e2049cf71b03f6d17059eafaa95 100644 --- a/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1/mem_samples.csv +++ b/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1/mem_samples.csv @@ -38,38 +38,38 @@ 5, 4 6, 4 7, 4 -0, 4 -1, 4 -2, 4 -3, 629 -4, 4 -5, 4 -6, 4 -7, 4 -0, 907 -1, 893 -2, 923 -3, 907 -4, 897 -5, 893 -6, 903 -7, 911 -0, 1975 -1, 1975 -2, 1975 -3, 1975 -4, 1975 -5, 1975 -6, 1975 -7, 1975 -0, 8465 -1, 8463 -2, 8433 -3, 8431 -4, 8439 -5, 8433 -6, 8439 -7, 8437 +0, 819 +1, 801 +2, 807 +3, 819 +4, 801 +5, 801 +6, 797 +7, 811 +0, 1079 +1, 1079 +2, 1079 +3, 1079 +4, 1079 +5, 1079 +6, 1079 +7, 1079 +0, 2551 +1, 2551 +2, 2551 +3, 2551 +4, 2551 +5, 2551 +6, 2551 +7, 2551 +0, 8401 +1, 8399 +2, 8425 +3, 8409 +4, 8389 +5, 8397 +6, 8385 +7, 8407 0, 70075 1, 70075 2, 70075 @@ -80,11 +80,11 @@ 7, 70075 0, 71553 1, 71553 -2, 72097 -3, 72097 +2, 71553 +3, 71553 4, 71553 -5, 72097 -6, 72097 +5, 71553 +6, 71553 7, 71553 0, 73289 1, 73289 @@ -102,83 +102,91 @@ 5, 73289 6, 73289 7, 73289 -0, 76053 -1, 76029 -2, 76029 -3, 76029 -4, 76041 -5, 76029 -6, 76029 -7, 76029 -0, 90463 -1, 92109 -2, 89697 -3, 89375 -4, 89739 -5, 95527 -6, 89573 -7, 90665 -0, 128175 -1, 128197 -2, 128197 -3, 128197 -4, 128221 -5, 134555 -6, 128197 -7, 128197 -0, 160611 -1, 160655 -2, 160655 -3, 160655 -4, 160659 -5, 160655 -6, 160655 -7, 160655 -0, 161647 -1, 161691 -2, 161691 -3, 161691 -4, 161695 -5, 161691 -6, 161691 -7, 161691 -0, 161647 -1, 161691 -2, 161691 -3, 161691 -4, 161695 -5, 161691 -6, 161691 -7, 161691 -0, 161811 -1, 161861 -2, 161873 -3, 161853 -4, 161863 -5, 161849 -6, 161851 -7, 161879 -0, 171199 -1, 171223 -2, 171273 -3, 171309 -4, 171245 -5, 171291 -6, 171229 -7, 171271 -0, 182493 -1, 182533 -2, 182533 -3, 182533 -4, 182587 -5, 182533 -6, 182533 -7, 182533 -0, 182493 -1, 182533 -2, 182533 -3, 182533 -4, 182587 -5, 182533 -6, 182533 -7, 182533 +0, 79131 +1, 79165 +2, 79165 +3, 79165 +4, 79177 +5, 79165 +6, 79165 +7, 79165 +0, 106325 +1, 108455 +2, 119691 +3, 109981 +4, 108829 +5, 105929 +6, 107891 +7, 109535 +0, 137629 +1, 138297 +2, 147001 +3, 140081 +4, 139301 +5, 137949 +6, 138123 +7, 140143 +0, 160841 +1, 160905 +2, 160905 +3, 160883 +4, 160903 +5, 160891 +6, 160875 +7, 160879 +0, 161649 +1, 161685 +2, 161685 +3, 161685 +4, 161697 +5, 161685 +6, 161685 +7, 161685 +0, 161649 +1, 161685 +2, 161685 +3, 161685 +4, 161697 +5, 161685 +6, 161685 +7, 161685 +0, 162255 +1, 162297 +2, 162295 +3, 162297 +4, 162295 +5, 162307 +6, 162295 +7, 162295 +0, 171249 +1, 171281 +2, 171323 +3, 171375 +4, 171397 +5, 171355 +6, 171345 +7, 171339 +0, 182495 +1, 182575 +2, 182575 +3, 182575 +4, 182589 +5, 182575 +6, 182575 +7, 182575 +0, 182495 +1, 182575 +2, 182575 +3, 182575 +4, 182589 +5, 182575 +6, 182575 +7, 182575 +0, 182491 +1, 182571 +2, 182571 +3, 182571 +4, 182585 +5, 182571 +6, 182571 +7, 182571 diff --git a/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1/train.log b/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1/train.log index 7086b7e06707b06e95b0ed43d54c0f2de4b59bc3..3d8fcf124e6d248bc250ef6791fb936abb824b85 100644 --- a/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1/train.log +++ b/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1/train.log @@ -1,6 +1,6 @@ -[15:05:39] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) -[15:05:39] WARN: moonep_backend_token_routing_analysis lacks --moe-router-fusion; QWEN3_235B runs unfused (timing only, routing identical) -[15:05:39] WARN: moonep_backend_token_routing_analysis lacks --moe-debug-drop-logging; probe will have no pre/post rank loads +[15:53:40] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +[15:53:40] WARN: moonep_backend_token_routing_analysis lacks --moe-router-fusion; QWEN3_235B runs unfused (timing only, routing identical) +[15:53:40] WARN: moonep_backend_token_routing_analysis lacks --moe-debug-drop-logging; probe will have no pre/post rank loads ============================================================== QWEN3_235B / moonep (slots/rank=16 policy=matched) E=128 K=8 H'=1536 m=16 @@ -9,10 +9,23 @@ memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1/memory_snapshots/snapshot.pickle repo=/workspace/Megatron-MoE-Echo @ 35ae6e7f ============================================================== -W0826 15:05:41.313000 62367 torch/distributed/run.py:766] -W0826 15:05:41.313000 62367 torch/distributed/run.py:766] ***************************************** -W0826 15:05:41.313000 62367 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -W0826 15:05:41.313000 62367 torch/distributed/run.py:766] ***************************************** +W0826 15:53:41.745000 153251 torch/distributed/run.py:766] +W0826 15:53:41.745000 153251 torch/distributed/run.py:766] ***************************************** +W0826 15:53:41.745000 153251 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 15:53:41.745000 153251 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 @@ -568,64 +581,51 @@ INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS > initializing torch distributed ... -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7[Gloo] Rank +6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 > initialized tensor model parallel with size 1 > initialized pipeline model parallel with size 1 > setting random seeds to 1234 ... [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[rank1]:[W826 15:05:59.009647315 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank5]:[W826 15:05:59.009721570 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank3]:[W826 15:05:59.009771961 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank6]:[W826 15:05:59.009851468 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank2]:[W826 15:05:59.009887336 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank4]:[W826 15:05:59.011379545 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank7]:[W826 15:05:59.011686659 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[rank3]:[W826 15:54:00.083644589 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 15:54:00.083711591 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 15:54:00.083733902 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 15:54:00.083761185 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 15:54:00.083764111 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 15:54:00.083787541 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 15:54:00.085026140 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. > compiling dataset index builder ... make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' make: Nothing to be done for 'default'. make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' ->>> done with dataset index builder. Compilation time: 0.034 seconds +>>> done with dataset index builder. Compilation time: 0.036 seconds WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. > compiling and loading fused kernels ... -[rank0]:[W826 15:06:00.176511709 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. ->>> done with compiling and loading fused kernels. Compilation time: 3.360 seconds -time to initialize megatron (seconds): 6.914 -[after megatron is initialized] datetime: 2026-08-26 15:06:05 +[rank0]:[W826 15:54:00.233142401 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.765 seconds +time to initialize megatron (seconds): 7.191 +[after megatron is initialized] datetime: 2026-08-26 15:54:06 building GPT model ... /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. warnings.warn( @@ -649,109 +649,109 @@ INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradi Params for bucket 1 (206045184 elements, 206045184 padded size): module.output_layer.weight Params for bucket 2 (71840000 elements, 71840000 padded size): - module.decoder.layers.11.self_attention.k_layernorm.weight + module.decoder.final_layernorm.weight module.decoder.layers.11.self_attention.q_layernorm.weight - module.decoder.layers.11.mlp.router.weight module.decoder.layers.11.pre_mlp_layernorm.weight - module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.11.mlp.router.weight module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.self_attention.k_layernorm.weight + module.decoder.layers.11.self_attention.linear_qkv.weight module.decoder.layers.11.self_attention.linear_proj.weight - module.decoder.final_layernorm.weight Params for bucket 3 (71835904 elements, 71835904 padded size): module.decoder.layers.10.mlp.router.weight - module.decoder.layers.10.self_attention.linear_qkv.weight - module.decoder.layers.10.self_attention.q_layernorm.weight module.decoder.layers.10.pre_mlp_layernorm.weight module.decoder.layers.10.self_attention.k_layernorm.weight + module.decoder.layers.10.self_attention.q_layernorm.weight + module.decoder.layers.10.self_attention.linear_qkv.weight module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.10.self_attention.linear_proj.weight Params for bucket 4 (71835904 elements, 71835904 padded size): + module.decoder.layers.9.self_attention.linear_proj.weight module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.k_layernorm.weight module.decoder.layers.9.pre_mlp_layernorm.weight module.decoder.layers.9.self_attention.linear_qkv.weight - module.decoder.layers.9.self_attention.k_layernorm.weight module.decoder.layers.9.self_attention.q_layernorm.weight module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.9.self_attention.linear_proj.weight Params for bucket 5 (71835904 elements, 71835904 padded size): module.decoder.layers.8.mlp.router.weight - module.decoder.layers.8.self_attention.linear_qkv.weight module.decoder.layers.8.self_attention.q_layernorm.weight module.decoder.layers.8.pre_mlp_layernorm.weight module.decoder.layers.8.self_attention.k_layernorm.weight module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.self_attention.linear_qkv.weight Params for bucket 6 (71835904 elements, 71835904 padded size): - module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.7.mlp.router.weight - module.decoder.layers.7.self_attention.k_layernorm.weight - module.decoder.layers.7.self_attention.q_layernorm.weight module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.q_layernorm.weight module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.k_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.7.self_attention.linear_proj.weight Params for bucket 7 (71835904 elements, 71835904 padded size): module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.weight module.decoder.layers.6.self_attention.k_layernorm.weight module.decoder.layers.6.self_attention.q_layernorm.weight - module.decoder.layers.6.self_attention.linear_qkv.weight - module.decoder.layers.6.pre_mlp_layernorm.weight module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.6.self_attention.linear_proj.weight Params for bucket 8 (71835904 elements, 71835904 padded size): module.decoder.layers.5.mlp.router.weight - module.decoder.layers.5.pre_mlp_layernorm.weight module.decoder.layers.5.self_attention.q_layernorm.weight module.decoder.layers.5.self_attention.k_layernorm.weight + module.decoder.layers.5.pre_mlp_layernorm.weight module.decoder.layers.5.self_attention.linear_qkv.weight module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.5.self_attention.linear_proj.weight Params for bucket 9 (71835904 elements, 71835904 padded size): module.decoder.layers.4.mlp.router.weight - module.decoder.layers.4.pre_mlp_layernorm.weight module.decoder.layers.4.self_attention.k_layernorm.weight module.decoder.layers.4.self_attention.q_layernorm.weight module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.pre_mlp_layernorm.weight module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.4.self_attention.linear_proj.weight Params for bucket 10 (71835904 elements, 71835904 padded size): + module.decoder.layers.3.self_attention.q_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight module.decoder.layers.3.mlp.router.weight module.decoder.layers.3.self_attention.linear_qkv.weight module.decoder.layers.3.pre_mlp_layernorm.weight module.decoder.layers.3.self_attention.k_layernorm.weight - module.decoder.layers.3.self_attention.q_layernorm.weight - module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.3.self_attention.linear_proj.weight Params for bucket 11 (71835904 elements, 71835904 padded size): + module.decoder.layers.2.mlp.router.weight module.decoder.layers.2.self_attention.linear_qkv.weight module.decoder.layers.2.self_attention.k_layernorm.weight module.decoder.layers.2.self_attention.q_layernorm.weight - module.decoder.layers.2.self_attention.linear_proj.weight - module.decoder.layers.2.mlp.router.weight module.decoder.layers.2.pre_mlp_layernorm.weight module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.self_attention.linear_proj.weight Params for bucket 12 (71835904 elements, 71835904 padded size): module.decoder.layers.1.mlp.router.weight module.decoder.layers.1.pre_mlp_layernorm.weight - module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.1.self_attention.linear_proj.weight module.decoder.layers.1.self_attention.k_layernorm.weight - module.decoder.layers.1.self_attention.q_layernorm.weight module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.q_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.linear_proj.weight Params for bucket 13 (71835904 elements, 71835904 padded size): - module.decoder.layers.0.mlp.router.weight - module.decoder.layers.0.self_attention.k_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.weight module.decoder.layers.0.self_attention.q_layernorm.weight module.decoder.layers.0.pre_mlp_layernorm.weight - module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.self_attention.k_layernorm.weight module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight Params for bucket 14 (206045184 elements, 206045184 padded size): module.embedding.word_embeddings.weight INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 76 Params for bucket 1 (44040192 elements, 44040192 padded size): module.decoder.layers.11.mlp.experts.linear_fc2.weight15 - module.decoder.layers.11.mlp.experts.linear_fc2.weight13 module.decoder.layers.11.mlp.experts.linear_fc2.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight13 module.decoder.layers.11.mlp.experts.linear_fc2.weight9 module.decoder.layers.11.mlp.experts.linear_fc2.weight14 module.decoder.layers.11.mlp.experts.linear_fc2.weight12 @@ -777,8 +777,8 @@ Params for bucket 4 (50331648 elements, 50331648 padded size): module.decoder.layers.11.mlp.experts.linear_fc1.weight10 Params for bucket 5 (50331648 elements, 50331648 padded size): module.decoder.layers.11.mlp.experts.linear_fc1.weight8 - module.decoder.layers.11.mlp.experts.linear_fc1.weight7 module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 module.decoder.layers.11.mlp.experts.linear_fc1.weight6 Params for bucket 6 (50331648 elements, 50331648 padded size): module.decoder.layers.11.mlp.experts.linear_fc1.weight4 @@ -803,51 +803,51 @@ Params for bucket 8 (44040192 elements, 44040192 padded size): Params for bucket 9 (50331648 elements, 50331648 padded size): module.decoder.layers.10.mlp.experts.linear_fc2.weight3 module.decoder.layers.10.mlp.experts.linear_fc2.weight0 - module.decoder.layers.10.mlp.experts.linear_fc1.weight15 - module.decoder.layers.10.mlp.experts.linear_fc2.weight1 module.decoder.layers.10.mlp.experts.linear_fc2.weight2 module.decoder.layers.10.mlp.experts.linear_fc1.weight14 + module.decoder.layers.10.mlp.experts.linear_fc1.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 Params for bucket 10 (50331648 elements, 50331648 padded size): module.decoder.layers.10.mlp.experts.linear_fc1.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 module.decoder.layers.10.mlp.experts.linear_fc1.weight10 module.decoder.layers.10.mlp.experts.linear_fc1.weight11 - module.decoder.layers.10.mlp.experts.linear_fc1.weight12 Params for bucket 11 (50331648 elements, 50331648 padded size): module.decoder.layers.10.mlp.experts.linear_fc1.weight9 - module.decoder.layers.10.mlp.experts.linear_fc1.weight7 module.decoder.layers.10.mlp.experts.linear_fc1.weight8 module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 Params for bucket 12 (50331648 elements, 50331648 padded size): module.decoder.layers.10.mlp.experts.linear_fc1.weight5 - module.decoder.layers.10.mlp.experts.linear_fc1.weight4 module.decoder.layers.10.mlp.experts.linear_fc1.weight2 module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 Params for bucket 13 (44040192 elements, 44040192 padded size): - module.decoder.layers.10.mlp.experts.linear_fc1.weight1 module.decoder.layers.9.mlp.experts.linear_fc2.weight14 - module.decoder.layers.9.mlp.experts.linear_fc2.weight13 - module.decoder.layers.9.mlp.experts.linear_fc2.weight15 module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight13 Params for bucket 14 (44040192 elements, 44040192 padded size): module.decoder.layers.9.mlp.experts.linear_fc2.weight12 - module.decoder.layers.9.mlp.experts.linear_fc2.weight10 module.decoder.layers.9.mlp.experts.linear_fc2.weight8 + module.decoder.layers.9.mlp.experts.linear_fc2.weight10 module.decoder.layers.9.mlp.experts.linear_fc2.weight6 - module.decoder.layers.9.mlp.experts.linear_fc2.weight9 module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.9.mlp.experts.linear_fc2.weight9 module.decoder.layers.9.mlp.experts.linear_fc2.weight7 Params for bucket 15 (50331648 elements, 50331648 padded size): - module.decoder.layers.9.mlp.experts.linear_fc2.weight1 module.decoder.layers.9.mlp.experts.linear_fc2.weight4 module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 module.decoder.layers.9.mlp.experts.linear_fc1.weight15 module.decoder.layers.9.mlp.experts.linear_fc2.weight5 module.decoder.layers.9.mlp.experts.linear_fc2.weight3 module.decoder.layers.9.mlp.experts.linear_fc2.weight0 Params for bucket 16 (50331648 elements, 50331648 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight14 - module.decoder.layers.9.mlp.experts.linear_fc1.weight13 module.decoder.layers.9.mlp.experts.linear_fc1.weight11 + module.decoder.layers.9.mlp.experts.linear_fc1.weight13 module.decoder.layers.9.mlp.experts.linear_fc1.weight12 Params for bucket 17 (50331648 elements, 50331648 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight10 @@ -856,34 +856,34 @@ Params for bucket 17 (50331648 elements, 50331648 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight8 Params for bucket 18 (50331648 elements, 50331648 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight6 - module.decoder.layers.9.mlp.experts.linear_fc1.weight3 - module.decoder.layers.9.mlp.experts.linear_fc1.weight4 module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 Params for bucket 19 (44040192 elements, 44040192 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight2 - module.decoder.layers.9.mlp.experts.linear_fc1.weight0 - module.decoder.layers.8.mlp.experts.linear_fc2.weight15 module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight15 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 Params for bucket 20 (44040192 elements, 44040192 padded size): - module.decoder.layers.8.mlp.experts.linear_fc2.weight11 module.decoder.layers.8.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight11 module.decoder.layers.8.mlp.experts.linear_fc2.weight9 + module.decoder.layers.8.mlp.experts.linear_fc2.weight8 module.decoder.layers.8.mlp.experts.linear_fc2.weight14 module.decoder.layers.8.mlp.experts.linear_fc2.weight10 module.decoder.layers.8.mlp.experts.linear_fc2.weight12 - module.decoder.layers.8.mlp.experts.linear_fc2.weight8 Params for bucket 21 (44040192 elements, 44040192 padded size): - module.decoder.layers.8.mlp.experts.linear_fc2.weight7 - module.decoder.layers.8.mlp.experts.linear_fc2.weight5 module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 module.decoder.layers.8.mlp.experts.linear_fc2.weight2 - module.decoder.layers.8.mlp.experts.linear_fc2.weight1 - module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 Params for bucket 22 (44040192 elements, 44040192 padded size): module.decoder.layers.8.mlp.experts.linear_fc2.weight0 - module.decoder.layers.8.mlp.experts.linear_fc1.weight13 module.decoder.layers.8.mlp.experts.linear_fc1.weight15 + module.decoder.layers.8.mlp.experts.linear_fc1.weight13 module.decoder.layers.8.mlp.experts.linear_fc1.weight14 Params for bucket 23 (50331648 elements, 50331648 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight12 @@ -892,9 +892,9 @@ Params for bucket 23 (50331648 elements, 50331648 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight11 Params for bucket 24 (50331648 elements, 50331648 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 module.decoder.layers.8.mlp.experts.linear_fc1.weight6 module.decoder.layers.8.mlp.experts.linear_fc1.weight7 - module.decoder.layers.8.mlp.experts.linear_fc1.weight5 Params for bucket 25 (50331648 elements, 50331648 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight4 module.decoder.layers.8.mlp.experts.linear_fc1.weight3 @@ -902,36 +902,36 @@ Params for bucket 25 (50331648 elements, 50331648 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight2 Params for bucket 26 (44040192 elements, 44040192 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight0 - module.decoder.layers.7.mlp.experts.linear_fc2.weight13 module.decoder.layers.7.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 module.decoder.layers.7.mlp.experts.linear_fc2.weight11 - module.decoder.layers.7.mlp.experts.linear_fc2.weight12 module.decoder.layers.7.mlp.experts.linear_fc2.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight12 Params for bucket 27 (44040192 elements, 44040192 padded size): module.decoder.layers.7.mlp.experts.linear_fc2.weight9 module.decoder.layers.7.mlp.experts.linear_fc2.weight7 module.decoder.layers.7.mlp.experts.linear_fc2.weight5 module.decoder.layers.7.mlp.experts.linear_fc2.weight10 - module.decoder.layers.7.mlp.experts.linear_fc2.weight8 module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 module.decoder.layers.7.mlp.experts.linear_fc2.weight4 Params for bucket 28 (50331648 elements, 50331648 padded size): module.decoder.layers.7.mlp.experts.linear_fc2.weight3 module.decoder.layers.7.mlp.experts.linear_fc1.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 module.decoder.layers.7.mlp.experts.linear_fc2.weight1 module.decoder.layers.7.mlp.experts.linear_fc2.weight0 - module.decoder.layers.7.mlp.experts.linear_fc2.weight2 module.decoder.layers.7.mlp.experts.linear_fc1.weight14 Params for bucket 29 (50331648 elements, 50331648 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight13 module.decoder.layers.7.mlp.experts.linear_fc1.weight12 - module.decoder.layers.7.mlp.experts.linear_fc1.weight10 module.decoder.layers.7.mlp.experts.linear_fc1.weight11 + module.decoder.layers.7.mlp.experts.linear_fc1.weight10 Params for bucket 30 (50331648 elements, 50331648 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight9 module.decoder.layers.7.mlp.experts.linear_fc1.weight6 - module.decoder.layers.7.mlp.experts.linear_fc1.weight7 module.decoder.layers.7.mlp.experts.linear_fc1.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 Params for bucket 31 (50331648 elements, 50331648 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight5 module.decoder.layers.7.mlp.experts.linear_fc1.weight3 @@ -939,10 +939,10 @@ Params for bucket 31 (50331648 elements, 50331648 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight2 Params for bucket 32 (44040192 elements, 44040192 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight1 - module.decoder.layers.6.mlp.experts.linear_fc2.weight15 module.decoder.layers.6.mlp.experts.linear_fc2.weight13 - module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 module.decoder.layers.6.mlp.experts.linear_fc2.weight14 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 Params for bucket 33 (44040192 elements, 44040192 padded size): module.decoder.layers.6.mlp.experts.linear_fc2.weight9 module.decoder.layers.6.mlp.experts.linear_fc2.weight11 @@ -955,124 +955,124 @@ Params for bucket 34 (50331648 elements, 50331648 padded size): module.decoder.layers.6.mlp.experts.linear_fc2.weight5 module.decoder.layers.6.mlp.experts.linear_fc2.weight3 module.decoder.layers.6.mlp.experts.linear_fc2.weight0 - module.decoder.layers.6.mlp.experts.linear_fc2.weight2 - module.decoder.layers.6.mlp.experts.linear_fc2.weight4 module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 module.decoder.layers.6.mlp.experts.linear_fc1.weight15 Params for bucket 35 (50331648 elements, 50331648 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight14 module.decoder.layers.6.mlp.experts.linear_fc1.weight11 - module.decoder.layers.6.mlp.experts.linear_fc1.weight13 module.decoder.layers.6.mlp.experts.linear_fc1.weight12 + module.decoder.layers.6.mlp.experts.linear_fc1.weight13 Params for bucket 36 (50331648 elements, 50331648 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight10 - module.decoder.layers.6.mlp.experts.linear_fc1.weight7 module.decoder.layers.6.mlp.experts.linear_fc1.weight8 module.decoder.layers.6.mlp.experts.linear_fc1.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 Params for bucket 37 (50331648 elements, 50331648 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight6 - module.decoder.layers.6.mlp.experts.linear_fc1.weight4 module.decoder.layers.6.mlp.experts.linear_fc1.weight5 module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 Params for bucket 38 (44040192 elements, 44040192 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight2 module.decoder.layers.5.mlp.experts.linear_fc2.weight15 - module.decoder.layers.6.mlp.experts.linear_fc1.weight0 module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 Params for bucket 39 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight13 + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 module.decoder.layers.5.mlp.experts.linear_fc2.weight14 module.decoder.layers.5.mlp.experts.linear_fc2.weight10 module.decoder.layers.5.mlp.experts.linear_fc2.weight12 module.decoder.layers.5.mlp.experts.linear_fc2.weight8 - module.decoder.layers.5.mlp.experts.linear_fc2.weight13 - module.decoder.layers.5.mlp.experts.linear_fc2.weight11 - module.decoder.layers.5.mlp.experts.linear_fc2.weight9 Params for bucket 40 (44040192 elements, 44040192 padded size): - module.decoder.layers.5.mlp.experts.linear_fc2.weight6 - module.decoder.layers.5.mlp.experts.linear_fc2.weight4 - module.decoder.layers.5.mlp.experts.linear_fc2.weight1 module.decoder.layers.5.mlp.experts.linear_fc2.weight7 module.decoder.layers.5.mlp.experts.linear_fc2.weight3 module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 Params for bucket 41 (44040192 elements, 44040192 padded size): module.decoder.layers.5.mlp.experts.linear_fc2.weight0 - module.decoder.layers.5.mlp.experts.linear_fc1.weight13 module.decoder.layers.5.mlp.experts.linear_fc1.weight14 module.decoder.layers.5.mlp.experts.linear_fc1.weight15 + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 Params for bucket 42 (50331648 elements, 50331648 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight12 - module.decoder.layers.5.mlp.experts.linear_fc1.weight10 module.decoder.layers.5.mlp.experts.linear_fc1.weight11 module.decoder.layers.5.mlp.experts.linear_fc1.weight9 + module.decoder.layers.5.mlp.experts.linear_fc1.weight10 Params for bucket 43 (50331648 elements, 50331648 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight8 - module.decoder.layers.5.mlp.experts.linear_fc1.weight7 module.decoder.layers.5.mlp.experts.linear_fc1.weight5 module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 Params for bucket 44 (50331648 elements, 50331648 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight4 module.decoder.layers.5.mlp.experts.linear_fc1.weight2 module.decoder.layers.5.mlp.experts.linear_fc1.weight3 module.decoder.layers.5.mlp.experts.linear_fc1.weight1 Params for bucket 45 (44040192 elements, 44040192 padded size): - module.decoder.layers.4.mlp.experts.linear_fc2.weight13 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 module.decoder.layers.4.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight13 module.decoder.layers.4.mlp.experts.linear_fc2.weight11 - module.decoder.layers.5.mlp.experts.linear_fc1.weight0 module.decoder.layers.4.mlp.experts.linear_fc2.weight14 module.decoder.layers.4.mlp.experts.linear_fc2.weight12 Params for bucket 46 (44040192 elements, 44040192 padded size): - module.decoder.layers.4.mlp.experts.linear_fc2.weight7 module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 module.decoder.layers.4.mlp.experts.linear_fc2.weight5 module.decoder.layers.4.mlp.experts.linear_fc2.weight10 - module.decoder.layers.4.mlp.experts.linear_fc2.weight6 module.decoder.layers.4.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 module.decoder.layers.4.mlp.experts.linear_fc2.weight4 Params for bucket 47 (50331648 elements, 50331648 padded size): module.decoder.layers.4.mlp.experts.linear_fc2.weight3 - module.decoder.layers.4.mlp.experts.linear_fc1.weight15 module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight14 module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 module.decoder.layers.4.mlp.experts.linear_fc2.weight1 - module.decoder.layers.4.mlp.experts.linear_fc1.weight14 Params for bucket 48 (50331648 elements, 50331648 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight13 + module.decoder.layers.4.mlp.experts.linear_fc1.weight12 module.decoder.layers.4.mlp.experts.linear_fc1.weight10 module.decoder.layers.4.mlp.experts.linear_fc1.weight11 - module.decoder.layers.4.mlp.experts.linear_fc1.weight12 Params for bucket 49 (50331648 elements, 50331648 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight9 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 module.decoder.layers.4.mlp.experts.linear_fc1.weight7 module.decoder.layers.4.mlp.experts.linear_fc1.weight8 - module.decoder.layers.4.mlp.experts.linear_fc1.weight6 Params for bucket 50 (50331648 elements, 50331648 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight5 - module.decoder.layers.4.mlp.experts.linear_fc1.weight4 module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 module.decoder.layers.4.mlp.experts.linear_fc1.weight3 Params for bucket 51 (44040192 elements, 44040192 padded size): - module.decoder.layers.4.mlp.experts.linear_fc1.weight1 - module.decoder.layers.3.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 module.decoder.layers.3.mlp.experts.linear_fc2.weight13 + module.decoder.layers.3.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 module.decoder.layers.3.mlp.experts.linear_fc2.weight14 - module.decoder.layers.4.mlp.experts.linear_fc1.weight0 Params for bucket 52 (44040192 elements, 44040192 padded size): module.decoder.layers.3.mlp.experts.linear_fc2.weight11 module.decoder.layers.3.mlp.experts.linear_fc2.weight9 module.decoder.layers.3.mlp.experts.linear_fc2.weight7 module.decoder.layers.3.mlp.experts.linear_fc2.weight12 - module.decoder.layers.3.mlp.experts.linear_fc2.weight10 module.decoder.layers.3.mlp.experts.linear_fc2.weight8 + module.decoder.layers.3.mlp.experts.linear_fc2.weight10 module.decoder.layers.3.mlp.experts.linear_fc2.weight6 Params for bucket 53 (50331648 elements, 50331648 padded size): module.decoder.layers.3.mlp.experts.linear_fc2.weight5 module.decoder.layers.3.mlp.experts.linear_fc2.weight2 module.decoder.layers.3.mlp.experts.linear_fc2.weight3 module.decoder.layers.3.mlp.experts.linear_fc2.weight0 - module.decoder.layers.3.mlp.experts.linear_fc2.weight4 module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 module.decoder.layers.3.mlp.experts.linear_fc1.weight15 Params for bucket 54 (50331648 elements, 50331648 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight14 @@ -1082,25 +1082,25 @@ Params for bucket 54 (50331648 elements, 50331648 padded size): Params for bucket 55 (50331648 elements, 50331648 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight10 module.decoder.layers.3.mlp.experts.linear_fc1.weight9 - module.decoder.layers.3.mlp.experts.linear_fc1.weight7 module.decoder.layers.3.mlp.experts.linear_fc1.weight8 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 Params for bucket 56 (50331648 elements, 50331648 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight6 module.decoder.layers.3.mlp.experts.linear_fc1.weight3 - module.decoder.layers.3.mlp.experts.linear_fc1.weight4 module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 Params for bucket 57 (44040192 elements, 44040192 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight2 - module.decoder.layers.2.mlp.experts.linear_fc2.weight15 module.decoder.layers.3.mlp.experts.linear_fc1.weight1 module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight15 Params for bucket 58 (44040192 elements, 44040192 padded size): module.decoder.layers.2.mlp.experts.linear_fc2.weight11 module.decoder.layers.2.mlp.experts.linear_fc2.weight13 module.decoder.layers.2.mlp.experts.linear_fc2.weight9 module.decoder.layers.2.mlp.experts.linear_fc2.weight14 - module.decoder.layers.2.mlp.experts.linear_fc2.weight12 module.decoder.layers.2.mlp.experts.linear_fc2.weight10 + module.decoder.layers.2.mlp.experts.linear_fc2.weight12 module.decoder.layers.2.mlp.experts.linear_fc2.weight8 Params for bucket 59 (44040192 elements, 44040192 padded size): module.decoder.layers.2.mlp.experts.linear_fc2.weight7 @@ -1113,13 +1113,13 @@ Params for bucket 59 (44040192 elements, 44040192 padded size): Params for bucket 60 (44040192 elements, 44040192 padded size): module.decoder.layers.2.mlp.experts.linear_fc2.weight0 module.decoder.layers.2.mlp.experts.linear_fc1.weight15 - module.decoder.layers.2.mlp.experts.linear_fc1.weight13 module.decoder.layers.2.mlp.experts.linear_fc1.weight14 + module.decoder.layers.2.mlp.experts.linear_fc1.weight13 Params for bucket 61 (50331648 elements, 50331648 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight12 + module.decoder.layers.2.mlp.experts.linear_fc1.weight11 module.decoder.layers.2.mlp.experts.linear_fc1.weight9 module.decoder.layers.2.mlp.experts.linear_fc1.weight10 - module.decoder.layers.2.mlp.experts.linear_fc1.weight11 Params for bucket 62 (50331648 elements, 50331648 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight8 module.decoder.layers.2.mlp.experts.linear_fc1.weight5 @@ -1132,8 +1132,8 @@ Params for bucket 63 (50331648 elements, 50331648 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight2 Params for bucket 64 (44040192 elements, 44040192 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight0 - module.decoder.layers.1.mlp.experts.linear_fc2.weight14 module.decoder.layers.1.mlp.experts.linear_fc2.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 module.decoder.layers.1.mlp.experts.linear_fc2.weight13 module.decoder.layers.1.mlp.experts.linear_fc2.weight15 module.decoder.layers.1.mlp.experts.linear_fc2.weight11 @@ -1142,13 +1142,13 @@ Params for bucket 65 (44040192 elements, 44040192 padded size): module.decoder.layers.1.mlp.experts.linear_fc2.weight6 module.decoder.layers.1.mlp.experts.linear_fc2.weight8 module.decoder.layers.1.mlp.experts.linear_fc2.weight4 - module.decoder.layers.1.mlp.experts.linear_fc2.weight7 module.decoder.layers.1.mlp.experts.linear_fc2.weight9 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 module.decoder.layers.1.mlp.experts.linear_fc2.weight5 Params for bucket 66 (50331648 elements, 50331648 padded size): module.decoder.layers.1.mlp.experts.linear_fc2.weight1 - module.decoder.layers.1.mlp.experts.linear_fc2.weight2 module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 module.decoder.layers.1.mlp.experts.linear_fc1.weight14 module.decoder.layers.1.mlp.experts.linear_fc2.weight3 module.decoder.layers.1.mlp.experts.linear_fc1.weight15 @@ -1160,57 +1160,57 @@ Params for bucket 67 (50331648 elements, 50331648 padded size): Params for bucket 68 (50331648 elements, 50331648 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight9 module.decoder.layers.1.mlp.experts.linear_fc1.weight7 - module.decoder.layers.1.mlp.experts.linear_fc1.weight8 module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 Params for bucket 69 (50331648 elements, 50331648 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 module.decoder.layers.1.mlp.experts.linear_fc1.weight4 module.decoder.layers.1.mlp.experts.linear_fc1.weight3 - module.decoder.layers.1.mlp.experts.linear_fc1.weight2 Params for bucket 70 (44040192 elements, 44040192 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight0 - module.decoder.layers.0.mlp.experts.linear_fc2.weight14 - module.decoder.layers.1.mlp.experts.linear_fc1.weight1 module.decoder.layers.0.mlp.experts.linear_fc2.weight15 module.decoder.layers.0.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight14 Params for bucket 71 (44040192 elements, 44040192 padded size): - module.decoder.layers.0.mlp.experts.linear_fc2.weight12 - module.decoder.layers.0.mlp.experts.linear_fc2.weight8 - module.decoder.layers.0.mlp.experts.linear_fc2.weight10 - module.decoder.layers.0.mlp.experts.linear_fc2.weight6 module.decoder.layers.0.mlp.experts.linear_fc2.weight9 module.decoder.layers.0.mlp.experts.linear_fc2.weight11 module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight12 + module.decoder.layers.0.mlp.experts.linear_fc2.weight10 + module.decoder.layers.0.mlp.experts.linear_fc2.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 Params for bucket 72 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 module.decoder.layers.0.mlp.experts.linear_fc2.weight2 module.decoder.layers.0.mlp.experts.linear_fc2.weight1 module.decoder.layers.0.mlp.experts.linear_fc2.weight4 module.decoder.layers.0.mlp.experts.linear_fc1.weight15 - module.decoder.layers.0.mlp.experts.linear_fc2.weight5 - module.decoder.layers.0.mlp.experts.linear_fc2.weight3 - module.decoder.layers.0.mlp.experts.linear_fc2.weight0 Params for bucket 73 (50331648 elements, 50331648 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc1.weight13 module.decoder.layers.0.mlp.experts.linear_fc1.weight12 module.decoder.layers.0.mlp.experts.linear_fc1.weight11 - module.decoder.layers.0.mlp.experts.linear_fc1.weight13 Params for bucket 74 (50331648 elements, 50331648 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight10 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 module.decoder.layers.0.mlp.experts.linear_fc1.weight9 module.decoder.layers.0.mlp.experts.linear_fc1.weight8 - module.decoder.layers.0.mlp.experts.linear_fc1.weight7 Params for bucket 75 (50331648 elements, 50331648 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight6 - module.decoder.layers.0.mlp.experts.linear_fc1.weight5 module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 module.decoder.layers.0.mlp.experts.linear_fc1.weight3 Params for bucket 76 (37748736 elements, 37748736 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight2 - module.decoder.layers.0.mlp.experts.linear_fc1.weight0 module.decoder.layers.0.mlp.experts.linear_fc1.weight1 -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine -[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 15:06:08 +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 15:54:09 > building train, validation, and test datasets ... > datasets target sizes (minimum size): train: 96 @@ -1218,7 +1218,7 @@ INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine test: 0 INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] > building train, validation, and test datasets for GPT ... -INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers @@ -1231,14 +1231,14 @@ INFO:megatron.core.datasets.gpt_dataset: Load the sample index from eadc3756447e INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-shuffle_index.npy INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 > finished creating GPT datasets ... -[after dataloaders are built] datetime: 2026-08-26 15:06:08 +[after dataloaders are built] datetime: 2026-08-26 15:54:09 done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3030.24, 3042.89) + train/valid/test-data-iterators-setup ..........: (59.14, 68.17) training ... Overwriting rerun_state_machine.current_iteration from -1 to 0... -(min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (2942.54, 2967.03) - train/valid/test-data-iterators-setup ..........: (59.93, 65.85) -[before the start of training step] datetime: 2026-08-26 15:06:08 +[before the start of training step] datetime: 2026-08-26 15:54:09 INFO:moonep.api:MoonEP comm buffer sizes: config : R=8, S=8192, H=4096, E=128, K=8, NvS=69600, NvS_padded=69632 sm_count : num_sms=32, num_sms_dedup=148 @@ -1247,15 +1247,156 @@ INFO:moonep.api:MoonEP comm buffer sizes: local_temp_buffer_size : 2.52 MiB total_size : 548.52 MiB WARNING:megatron.core.rerun_state_machine:Result validation enabled - [2026-08-26 15:06:35] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 26843.1 | throughput per GPU (TFLOP/s/GPU): 6.7 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.116256E+01 | loss scale: 1.0 | grad norm: 44.737 | number of skipped iterations: 0 | number of nan iterations: 0 | -Number of parameters in transformer block in billions: 29.85 + [2026-08-26 15:54:34] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 25523.0 | throughput per GPU (TFLOP/s/GPU): 7.1 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.116256E+01 | loss scale: 1.0 | grad norm: 44.736 | number of skipped iterations: 0 | number of nan iterations: 0 |Number of parameters in transformer block in billions: 29.85 + Number of parameters in embedding layers in billions: 0.41 Total number of parameters in billions: 30.26 Number of parameters in most loaded shard in billions: 30.2590 Theoretical memory footprints: weight and optimizer=216428.96 MB -[Rank 0] (after 1 iterations) memory (MB) | allocated: 138673.02490234375 | max allocated: 138673.025390625 | reserved: 142242.0 | max reserved: 142242.0 -WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 408.00 MiB. GPU 0 has a total capacity of 178.35 GiB of which 138.81 MiB is free. Including non-PyTorch memory, this process has 178.20 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.72 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -['Traceback (most recent call last):\n', ' File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute\n output, shared_expert_output, mlp_bias = self.experts_compute(\n ^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute\n expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 875, in forward\n intermediate_parallel, bias_parallel = self.linear_fc1(\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward\n out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn\n return fn(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward\n out = linear_fn(*args)\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward\n out = torch.empty(\n ^^^^^^^^^^^^\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 0 has a total capacity of 178.35 GiB of which 138.81 MiB is free. Including non-PyTorch memory, this process has 178.20 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.72 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] +[Rank 0] (after 1 iterations) memory (MB) | allocated: 138672.52490234375 | max allocated: 138672.525390625 | reserved: 142244.0 | max reserved: 142244.0 +WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 408.00 MiB. GPU 4 has a total capacity of 178.35 GiB of which 42.81 MiB is free. Including non-PyTorch memory, this process has 178.29 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.81 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute\n output, shared_expert_output, mlp_bias = self.experts_compute(\n ^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute\n expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 875, in forward\n intermediate_parallel, bias_parallel = self.linear_fc1(\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward\n out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn\n return fn(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward\n out = linear_fn(*args)\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward\n out = torch.empty(\n ^^^^^^^^^^^^\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 4 has a total capacity of 178.35 GiB of which 42.81 MiB is free. Including non-PyTorch memory, this process has 178.29 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.81 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] +WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 408.00 MiB. GPU 0 has a total capacity of 178.35 GiB of which 136.81 MiB is free. Including non-PyTorch memory, this process has 178.20 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.72 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute\n output, shared_expert_output, mlp_bias = self.experts_compute(\n ^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute\n expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 875, in forward\n intermediate_parallel, bias_parallel = self.linear_fc1(\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward\n out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn\n return fn(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward\n out = linear_fn(*args)\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward\n out = torch.empty(\n ^^^^^^^^^^^^\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 0 has a total capacity of 178.35 GiB of which 136.81 MiB is free. Including non-PyTorch memory, this process has 178.20 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.72 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] +[rank4]: Traceback (most recent call last): +[rank4]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 394, in +[rank4]: pretrain( +[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 917, in pretrain +[rank4]: iteration, num_floating_point_operations_so_far = train( +[rank4]: ^^^^^^ +[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 2326, in train +[rank4]: ) = train_step( +[rank4]: ^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 1433, in train_step +[rank4]: losses_reduced = forward_backward_func( +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining +[rank4]: output_tensor, num_tokens = forward_step( +[rank4]: ^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step +[rank4]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step +[rank4]: output_tensor = model( +[rank4]: ^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank4]: return self._call_impl(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank4]: return forward_call(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward +[rank4]: return self.module(*inputs, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank4]: return self._call_impl(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank4]: return inner() +[rank4]: ^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank4]: result = forward_call(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward +[rank4]: outputs = self.module(*inputs, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank4]: return self._call_impl(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank4]: return inner() +[rank4]: ^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank4]: result = forward_call(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward +[rank4]: hidden_states = self.decoder( +[rank4]: ^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank4]: return self._call_impl(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank4]: return inner() +[rank4]: ^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank4]: result = forward_call(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward +[rank4]: hidden_states, context = layer( +[rank4]: ^^^^^^ +[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__ +[rank4]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank4]: return self._call_impl(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank4]: return inner() +[rank4]: ^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank4]: result = forward_call(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward +[rank4]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp +[rank4]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank4]: return self._call_impl(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank4]: return inner() +[rank4]: ^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank4]: result = forward_call(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward +[rank4]: output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute +[rank4]: output, shared_expert_output, mlp_bias = self.experts_compute( +[rank4]: ^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute +[rank4]: expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank4]: return self._call_impl(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank4]: return inner() +[rank4]: ^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank4]: result = forward_call(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 875, in forward +[rank4]: intermediate_parallel, bias_parallel = self.linear_fc1( +[rank4]: ^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank4]: return self._call_impl(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank4]: return inner() +[rank4]: ^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank4]: result = forward_call(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward +[rank4]: out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn +[rank4]: return fn(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward +[rank4]: out = linear_fn(*args) +[rank4]: ^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply +[rank4]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward +[rank4]: out = torch.empty( +[rank4]: ^^^^^^^^^^^^ +[rank4]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 4 has a total capacity of 178.35 GiB of which 42.81 MiB is free. Including non-PyTorch memory, this process has 178.29 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.81 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) [rank0]: Traceback (most recent call last): [rank0]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 394, in [rank0]: pretrain( @@ -1394,432 +1535,154 @@ WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 408.00 MiB. GP [rank0]: File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward [rank0]: out = torch.empty( [rank0]: ^^^^^^^^^^^^ -[rank0]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 0 has a total capacity of 178.35 GiB of which 138.81 MiB is free. Including non-PyTorch memory, this process has 178.20 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.72 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 408.00 MiB. GPU 1 has a total capacity of 178.35 GiB of which 98.81 MiB is free. Including non-PyTorch memory, this process has 178.24 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.75 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -['Traceback (most recent call last):\n', ' File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute\n output, shared_expert_output, mlp_bias = self.experts_compute(\n ^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute\n expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 875, in forward\n intermediate_parallel, bias_parallel = self.linear_fc1(\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward\n out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn\n return fn(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward\n out = linear_fn(*args)\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward\n out = torch.empty(\n ^^^^^^^^^^^^\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 1 has a total capacity of 178.35 GiB of which 98.81 MiB is free. Including non-PyTorch memory, this process has 178.24 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.75 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] -[rank1]: Traceback (most recent call last): -[rank1]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 394, in -[rank1]: pretrain( -[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 917, in pretrain -[rank1]: iteration, num_floating_point_operations_so_far = train( -[rank1]: ^^^^^^ -[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 2326, in train -[rank1]: ) = train_step( -[rank1]: ^^^^^^^^^^^ -[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 1433, in train_step -[rank1]: losses_reduced = forward_backward_func( -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining -[rank1]: output_tensor, num_tokens = forward_step( -[rank1]: ^^^^^^^^^^^^^ -[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step -[rank1]: output_tensor, loss_func = forward_step_func(data_iterator, model) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step -[rank1]: output_tensor = model( -[rank1]: ^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward -[rank1]: return self.module(*inputs, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank1]: return inner() -[rank1]: ^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank1]: result = forward_call(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward -[rank1]: outputs = self.module(*inputs, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank1]: return inner() -[rank1]: ^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank1]: result = forward_call(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward -[rank1]: hidden_states = self.decoder( -[rank1]: ^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank1]: return inner() -[rank1]: ^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank1]: result = forward_call(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward -[rank1]: hidden_states, context = layer( -[rank1]: ^^^^^^ -[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__ -[rank1]: return super(MegatronModule, self).__call__(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank1]: return inner() -[rank1]: ^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank1]: result = forward_call(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward -[rank1]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp -[rank1]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank1]: return inner() -[rank1]: ^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank1]: result = forward_call(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward -[rank1]: output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute -[rank1]: output, shared_expert_output, mlp_bias = self.experts_compute( -[rank1]: ^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute -[rank1]: expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank1]: return inner() -[rank1]: ^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank1]: result = forward_call(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 875, in forward -[rank1]: intermediate_parallel, bias_parallel = self.linear_fc1( -[rank1]: ^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank1]: return inner() -[rank1]: ^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank1]: result = forward_call(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward -[rank1]: out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn -[rank1]: return fn(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward -[rank1]: out = linear_fn(*args) -[rank1]: ^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply -[rank1]: return super().apply(*args, **kwargs) # type: ignore[misc] -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward -[rank1]: out = torch.empty( -[rank1]: ^^^^^^^^^^^^ -[rank1]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 1 has a total capacity of 178.35 GiB of which 98.81 MiB is free. Including non-PyTorch memory, this process has 178.24 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.75 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 408.00 MiB. GPU 4 has a total capacity of 178.35 GiB of which 44.81 MiB is free. Including non-PyTorch memory, this process has 178.29 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.81 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -['Traceback (most recent call last):\n', ' File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute\n output, shared_expert_output, mlp_bias = self.experts_compute(\n ^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute\n expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 875, in forward\n intermediate_parallel, bias_parallel = self.linear_fc1(\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward\n out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn\n return fn(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward\n out = linear_fn(*args)\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward\n out = torch.empty(\n ^^^^^^^^^^^^\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 4 has a total capacity of 178.35 GiB of which 44.81 MiB is free. Including non-PyTorch memory, this process has 178.29 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.81 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] -[rank4]: Traceback (most recent call last): -[rank4]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 394, in -[rank4]: pretrain( -[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 917, in pretrain -[rank4]: iteration, num_floating_point_operations_so_far = train( -[rank4]: ^^^^^^ -[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 2326, in train -[rank4]: ) = train_step( -[rank4]: ^^^^^^^^^^^ -[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 1433, in train_step -[rank4]: losses_reduced = forward_backward_func( -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining -[rank4]: output_tensor, num_tokens = forward_step( -[rank4]: ^^^^^^^^^^^^^ -[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step -[rank4]: output_tensor, loss_func = forward_step_func(data_iterator, model) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step -[rank4]: output_tensor = model( -[rank4]: ^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank4]: return self._call_impl(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank4]: return forward_call(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward -[rank4]: return self.module(*inputs, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank4]: return self._call_impl(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank4]: return inner() -[rank4]: ^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank4]: result = forward_call(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward -[rank4]: outputs = self.module(*inputs, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank4]: return self._call_impl(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank4]: return inner() -[rank4]: ^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank4]: result = forward_call(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward -[rank4]: hidden_states = self.decoder( -[rank4]: ^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank4]: return self._call_impl(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank4]: return inner() -[rank4]: ^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank4]: result = forward_call(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward -[rank4]: hidden_states, context = layer( -[rank4]: ^^^^^^ -[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__ -[rank4]: return super(MegatronModule, self).__call__(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank4]: return self._call_impl(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank4]: return inner() -[rank4]: ^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank4]: result = forward_call(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward -[rank4]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp -[rank4]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank4]: return self._call_impl(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank4]: return inner() -[rank4]: ^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank4]: result = forward_call(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward -[rank4]: output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute -[rank4]: output, shared_expert_output, mlp_bias = self.experts_compute( -[rank4]: ^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute -[rank4]: expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank4]: return self._call_impl(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank4]: return inner() -[rank4]: ^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank4]: result = forward_call(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 875, in forward -[rank4]: intermediate_parallel, bias_parallel = self.linear_fc1( -[rank4]: ^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank4]: return self._call_impl(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank4]: return inner() -[rank4]: ^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank4]: result = forward_call(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward -[rank4]: out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn -[rank4]: return fn(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward -[rank4]: out = linear_fn(*args) -[rank4]: ^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply -[rank4]: return super().apply(*args, **kwargs) # type: ignore[misc] -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward -[rank4]: out = torch.empty( -[rank4]: ^^^^^^^^^^^^ -[rank4]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 4 has a total capacity of 178.35 GiB of which 44.81 MiB is free. Including non-PyTorch memory, this process has 178.29 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.81 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 408.00 MiB. GPU 3 has a total capacity of 178.35 GiB of which 98.81 MiB is free. Including non-PyTorch memory, this process has 178.24 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.75 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -['Traceback (most recent call last):\n', ' File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute\n output, shared_expert_output, mlp_bias = self.experts_compute(\n ^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute\n expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 875, in forward\n intermediate_parallel, bias_parallel = self.linear_fc1(\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward\n out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn\n return fn(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward\n out = linear_fn(*args)\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward\n out = torch.empty(\n ^^^^^^^^^^^^\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 3 has a total capacity of 178.35 GiB of which 98.81 MiB is free. Including non-PyTorch memory, this process has 178.24 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.75 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] -WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 408.00 MiB. GPU 7 has a total capacity of 178.35 GiB of which 98.81 MiB is free. Including non-PyTorch memory, this process has 178.24 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.75 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -['Traceback (most recent call last):\n', ' File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute\n output, shared_expert_output, mlp_bias = self.experts_compute(\n ^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute\n expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 875, in forward\n intermediate_parallel, bias_parallel = self.linear_fc1(\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward\n out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn\n return fn(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward\n out = linear_fn(*args)\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward\n out = torch.empty(\n ^^^^^^^^^^^^\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 7 has a total capacity of 178.35 GiB of which 98.81 MiB is free. Including non-PyTorch memory, this process has 178.24 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.75 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] -[rank3]: Traceback (most recent call last): -[rank3]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 394, in -[rank3]: pretrain( -[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 917, in pretrain -[rank3]: iteration, num_floating_point_operations_so_far = train( -[rank3]: ^^^^^^ -[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 2326, in train -[rank3]: ) = train_step( -[rank3]: ^^^^^^^^^^^ -[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 1433, in train_step -[rank3]: losses_reduced = forward_backward_func( -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining -[rank3]: output_tensor, num_tokens = forward_step( -[rank3]: ^^^^^^^^^^^^^ -[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step -[rank3]: output_tensor, loss_func = forward_step_func(data_iterator, model) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step -[rank3]: output_tensor = model( -[rank3]: ^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank3]: return self._call_impl(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank3]: return forward_call(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward -[rank3]: return self.module(*inputs, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank3]: return self._call_impl(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank3]: return inner() -[rank3]: ^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank3]: result = forward_call(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward -[rank3]: outputs = self.module(*inputs, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank3]: return self._call_impl(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank3]: return inner() -[rank3]: ^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank3]: result = forward_call(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward -[rank3]: hidden_states = self.decoder( -[rank3]: ^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank3]: return self._call_impl(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank3]: return inner() -[rank3]: ^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank3]: result = forward_call(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward -[rank3]: hidden_states, context = layer( -[rank3]: ^^^^^^ -[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__ -[rank3]: return super(MegatronModule, self).__call__(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank3]: return self._call_impl(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank3]: return inner() -[rank3]: ^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank3]: result = forward_call(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward -[rank3]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp -[rank3]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank3]: return self._call_impl(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank3]: return inner() -[rank3]: ^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank3]: result = forward_call(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward -[rank3]: output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute -[rank3]: output, shared_expert_output, mlp_bias = self.experts_compute( -[rank3]: ^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute -[rank3]: expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank3]: return self._call_impl(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank3]: return inner() -[rank3]: ^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank3]: result = forward_call(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 875, in forward -[rank3]: intermediate_parallel, bias_parallel = self.linear_fc1( -[rank3]: ^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank3]: return self._call_impl(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank3]: return inner() -[rank3]: ^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank3]: result = forward_call(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward -[rank3]: out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn -[rank3]: return fn(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward -[rank3]: out = linear_fn(*args) -[rank3]: ^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply -[rank3]: return super().apply(*args, **kwargs) # type: ignore[misc] -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward -[rank3]: out = torch.empty( -[rank3]: ^^^^^^^^^^^^ -[rank3]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 3 has a total capacity of 178.35 GiB of which 98.81 MiB is free. Including non-PyTorch memory, this process has 178.24 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.75 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +[rank0]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 0 has a total capacity of 178.35 GiB of which 136.81 MiB is free. Including non-PyTorch memory, this process has 178.20 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.72 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 408.00 MiB. GPU 6 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute\n output, shared_expert_output, mlp_bias = self.experts_compute(\n ^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute\n expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 875, in forward\n intermediate_parallel, bias_parallel = self.linear_fc1(\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward\n out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn\n return fn(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward\n out = linear_fn(*args)\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward\n out = torch.empty(\n ^^^^^^^^^^^^\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 6 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] +[rank6]: Traceback (most recent call last): +[rank6]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 394, in +[rank6]: pretrain( +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 917, in pretrain +[rank6]: iteration, num_floating_point_operations_so_far = train( +[rank6]: ^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 2326, in train +[rank6]: ) = train_step( +[rank6]: ^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 1433, in train_step +[rank6]: losses_reduced = forward_backward_func( +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining +[rank6]: output_tensor, num_tokens = forward_step( +[rank6]: ^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step +[rank6]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step +[rank6]: output_tensor = model( +[rank6]: ^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank6]: return forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward +[rank6]: return self.module(*inputs, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank6]: return inner() +[rank6]: ^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank6]: result = forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward +[rank6]: outputs = self.module(*inputs, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank6]: return inner() +[rank6]: ^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank6]: result = forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward +[rank6]: hidden_states = self.decoder( +[rank6]: ^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank6]: return inner() +[rank6]: ^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank6]: result = forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward +[rank6]: hidden_states, context = layer( +[rank6]: ^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__ +[rank6]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank6]: return inner() +[rank6]: ^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank6]: result = forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward +[rank6]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp +[rank6]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank6]: return inner() +[rank6]: ^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank6]: result = forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward +[rank6]: output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute +[rank6]: output, shared_expert_output, mlp_bias = self.experts_compute( +[rank6]: ^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute +[rank6]: expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank6]: return inner() +[rank6]: ^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank6]: result = forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 875, in forward +[rank6]: intermediate_parallel, bias_parallel = self.linear_fc1( +[rank6]: ^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank6]: return inner() +[rank6]: ^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank6]: result = forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward +[rank6]: out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn +[rank6]: return fn(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward +[rank6]: out = linear_fn(*args) +[rank6]: ^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply +[rank6]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward +[rank6]: out = torch.empty( +[rank6]: ^^^^^^^^^^^^ +[rank6]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 6 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 408.00 MiB. GPU 7 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute\n output, shared_expert_output, mlp_bias = self.experts_compute(\n ^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute\n expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 875, in forward\n intermediate_parallel, bias_parallel = self.linear_fc1(\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward\n out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn\n return fn(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward\n out = linear_fn(*args)\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward\n out = torch.empty(\n ^^^^^^^^^^^^\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 7 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] +WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 408.00 MiB. GPU 1 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute\n output, shared_expert_output, mlp_bias = self.experts_compute(\n ^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute\n expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 875, in forward\n intermediate_parallel, bias_parallel = self.linear_fc1(\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward\n out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn\n return fn(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward\n out = linear_fn(*args)\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward\n out = torch.empty(\n ^^^^^^^^^^^^\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 1 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] +WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 408.00 MiB. GPU 5 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute\n output, shared_expert_output, mlp_bias = self.experts_compute(\n ^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute\n expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 875, in forward\n intermediate_parallel, bias_parallel = self.linear_fc1(\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward\n out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn\n return fn(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward\n out = linear_fn(*args)\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward\n out = torch.empty(\n ^^^^^^^^^^^^\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 5 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] [rank7]: Traceback (most recent call last): [rank7]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 394, in [rank7]: pretrain( @@ -1958,150 +1821,146 @@ WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 408.00 MiB. GP [rank7]: File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward [rank7]: out = torch.empty( [rank7]: ^^^^^^^^^^^^ -[rank7]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 7 has a total capacity of 178.35 GiB of which 98.81 MiB is free. Including non-PyTorch memory, this process has 178.24 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.75 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 408.00 MiB. GPU 2 has a total capacity of 178.35 GiB of which 98.81 MiB is free. Including non-PyTorch memory, this process has 178.24 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.75 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -['Traceback (most recent call last):\n', ' File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute\n output, shared_expert_output, mlp_bias = self.experts_compute(\n ^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute\n expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 875, in forward\n intermediate_parallel, bias_parallel = self.linear_fc1(\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward\n out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn\n return fn(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward\n out = linear_fn(*args)\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward\n out = torch.empty(\n ^^^^^^^^^^^^\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 2 has a total capacity of 178.35 GiB of which 98.81 MiB is free. Including non-PyTorch memory, this process has 178.24 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.75 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] -[rank2]: Traceback (most recent call last): -[rank2]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 394, in -[rank2]: pretrain( -[rank2]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 917, in pretrain -[rank2]: iteration, num_floating_point_operations_so_far = train( -[rank2]: ^^^^^^ -[rank2]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 2326, in train -[rank2]: ) = train_step( -[rank2]: ^^^^^^^^^^^ -[rank2]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 1433, in train_step -[rank2]: losses_reduced = forward_backward_func( -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/workspace/Megatron-MoE-Echo/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining -[rank2]: output_tensor, num_tokens = forward_step( -[rank2]: ^^^^^^^^^^^^^ -[rank2]: File "/workspace/Megatron-MoE-Echo/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step -[rank2]: output_tensor, loss_func = forward_step_func(data_iterator, model) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step -[rank2]: output_tensor = model( -[rank2]: ^^^^^^ -[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank2]: return self._call_impl(*args, **kwargs) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank2]: return forward_call(*args, **kwargs) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward -[rank2]: return self.module(*inputs, **kwargs) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank2]: return self._call_impl(*args, **kwargs) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank2]: return inner() -[rank2]: ^^^^^^^ -[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank2]: result = forward_call(*args, **kwargs) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward -[rank2]: outputs = self.module(*inputs, **kwargs) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank2]: return self._call_impl(*args, **kwargs) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank2]: return inner() -[rank2]: ^^^^^^^ -[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank2]: result = forward_call(*args, **kwargs) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward -[rank2]: hidden_states = self.decoder( -[rank2]: ^^^^^^^^^^^^^ -[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank2]: return self._call_impl(*args, **kwargs) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank2]: return inner() -[rank2]: ^^^^^^^ -[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank2]: result = forward_call(*args, **kwargs) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward -[rank2]: hidden_states, context = layer( -[rank2]: ^^^^^^ -[rank2]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__ -[rank2]: return super(MegatronModule, self).__call__(*args, **kwargs) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank2]: return self._call_impl(*args, **kwargs) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank2]: return inner() -[rank2]: ^^^^^^^ -[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank2]: result = forward_call(*args, **kwargs) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward -[rank2]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp -[rank2]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank2]: return self._call_impl(*args, **kwargs) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank2]: return inner() -[rank2]: ^^^^^^^ -[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank2]: result = forward_call(*args, **kwargs) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward -[rank2]: output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute -[rank2]: output, shared_expert_output, mlp_bias = self.experts_compute( -[rank2]: ^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute -[rank2]: expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank2]: return self._call_impl(*args, **kwargs) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank2]: return inner() -[rank2]: ^^^^^^^ -[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank2]: result = forward_call(*args, **kwargs) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 875, in forward -[rank2]: intermediate_parallel, bias_parallel = self.linear_fc1( -[rank2]: ^^^^^^^^^^^^^^^^ -[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank2]: return self._call_impl(*args, **kwargs) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank2]: return inner() -[rank2]: ^^^^^^^ -[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank2]: result = forward_call(*args, **kwargs) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward -[rank2]: out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn -[rank2]: return fn(*args, **kwargs) -[rank2]: ^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward -[rank2]: out = linear_fn(*args) -[rank2]: ^^^^^^^^^^^^^^^^ -[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply -[rank2]: return super().apply(*args, **kwargs) # type: ignore[misc] -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward -[rank2]: out = torch.empty( -[rank2]: ^^^^^^^^^^^^ -[rank2]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 2 has a total capacity of 178.35 GiB of which 98.81 MiB is free. Including non-PyTorch memory, this process has 178.24 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.75 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 408.00 MiB. GPU 5 has a total capacity of 178.35 GiB of which 98.81 MiB is free. Including non-PyTorch memory, this process has 178.24 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.75 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -['Traceback (most recent call last):\n', ' File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute\n output, shared_expert_output, mlp_bias = self.experts_compute(\n ^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute\n expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 875, in forward\n intermediate_parallel, bias_parallel = self.linear_fc1(\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward\n out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn\n return fn(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward\n out = linear_fn(*args)\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward\n out = torch.empty(\n ^^^^^^^^^^^^\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 5 has a total capacity of 178.35 GiB of which 98.81 MiB is free. Including non-PyTorch memory, this process has 178.24 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.75 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] +[rank7]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 7 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +[rank1]: Traceback (most recent call last): +[rank1]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 394, in +[rank1]: pretrain( +[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 917, in pretrain +[rank1]: iteration, num_floating_point_operations_so_far = train( +[rank1]: ^^^^^^ +[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 2326, in train +[rank1]: ) = train_step( +[rank1]: ^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 1433, in train_step +[rank1]: losses_reduced = forward_backward_func( +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining +[rank1]: output_tensor, num_tokens = forward_step( +[rank1]: ^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step +[rank1]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step +[rank1]: output_tensor = model( +[rank1]: ^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward +[rank1]: return self.module(*inputs, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank1]: return inner() +[rank1]: ^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank1]: result = forward_call(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward +[rank1]: outputs = self.module(*inputs, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank1]: return inner() +[rank1]: ^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank1]: result = forward_call(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward +[rank1]: hidden_states = self.decoder( +[rank1]: ^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank1]: return inner() +[rank1]: ^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank1]: result = forward_call(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward +[rank1]: hidden_states, context = layer( +[rank1]: ^^^^^^ +[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__ +[rank1]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank1]: return inner() +[rank1]: ^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank1]: result = forward_call(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward +[rank1]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp +[rank1]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank1]: return inner() +[rank1]: ^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank1]: result = forward_call(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward +[rank1]: output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute +[rank1]: output, shared_expert_output, mlp_bias = self.experts_compute( +[rank1]: ^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute +[rank1]: expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank1]: return inner() +[rank1]: ^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank1]: result = forward_call(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 875, in forward +[rank1]: intermediate_parallel, bias_parallel = self.linear_fc1( +[rank1]: ^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank1]: return inner() +[rank1]: ^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank1]: result = forward_call(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward +[rank1]: out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn +[rank1]: return fn(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward +[rank1]: out = linear_fn(*args) +[rank1]: ^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply +[rank1]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward +[rank1]: out = torch.empty( +[rank1]: ^^^^^^^^^^^^ +[rank1]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 1 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) [rank5]: Traceback (most recent call last): [rank5]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 394, in [rank5]: pretrain( @@ -2240,157 +2099,298 @@ WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 408.00 MiB. GP [rank5]: File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward [rank5]: out = torch.empty( [rank5]: ^^^^^^^^^^^^ -[rank5]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 5 has a total capacity of 178.35 GiB of which 98.81 MiB is free. Including non-PyTorch memory, this process has 178.24 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.75 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 408.00 MiB. GPU 6 has a total capacity of 178.35 GiB of which 98.81 MiB is free. Including non-PyTorch memory, this process has 178.24 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.75 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -['Traceback (most recent call last):\n', ' File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute\n output, shared_expert_output, mlp_bias = self.experts_compute(\n ^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute\n expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 875, in forward\n intermediate_parallel, bias_parallel = self.linear_fc1(\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward\n out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn\n return fn(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward\n out = linear_fn(*args)\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward\n out = torch.empty(\n ^^^^^^^^^^^^\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 6 has a total capacity of 178.35 GiB of which 98.81 MiB is free. Including non-PyTorch memory, this process has 178.24 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.75 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] -[rank6]: Traceback (most recent call last): -[rank6]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 394, in -[rank6]: pretrain( -[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 917, in pretrain -[rank6]: iteration, num_floating_point_operations_so_far = train( -[rank6]: ^^^^^^ -[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 2326, in train -[rank6]: ) = train_step( -[rank6]: ^^^^^^^^^^^ -[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 1433, in train_step -[rank6]: losses_reduced = forward_backward_func( -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining -[rank6]: output_tensor, num_tokens = forward_step( -[rank6]: ^^^^^^^^^^^^^ -[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step -[rank6]: output_tensor, loss_func = forward_step_func(data_iterator, model) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step -[rank6]: output_tensor = model( -[rank6]: ^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank6]: return self._call_impl(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank6]: return forward_call(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward -[rank6]: return self.module(*inputs, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank6]: return self._call_impl(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank6]: return inner() -[rank6]: ^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank6]: result = forward_call(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward -[rank6]: outputs = self.module(*inputs, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank6]: return self._call_impl(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank6]: return inner() -[rank6]: ^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank6]: result = forward_call(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward -[rank6]: hidden_states = self.decoder( -[rank6]: ^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank6]: return self._call_impl(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank6]: return inner() -[rank6]: ^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank6]: result = forward_call(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward -[rank6]: hidden_states, context = layer( -[rank6]: ^^^^^^ -[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__ -[rank6]: return super(MegatronModule, self).__call__(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank6]: return self._call_impl(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank6]: return inner() -[rank6]: ^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank6]: result = forward_call(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward -[rank6]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp -[rank6]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank6]: return self._call_impl(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank6]: return inner() -[rank6]: ^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank6]: result = forward_call(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward -[rank6]: output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute -[rank6]: output, shared_expert_output, mlp_bias = self.experts_compute( -[rank6]: ^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute -[rank6]: expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank6]: return self._call_impl(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank6]: return inner() -[rank6]: ^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank6]: result = forward_call(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 875, in forward -[rank6]: intermediate_parallel, bias_parallel = self.linear_fc1( -[rank6]: ^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank6]: return self._call_impl(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank6]: return inner() -[rank6]: ^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank6]: result = forward_call(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward -[rank6]: out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn -[rank6]: return fn(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward -[rank6]: out = linear_fn(*args) -[rank6]: ^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply -[rank6]: return super().apply(*args, **kwargs) # type: ignore[misc] -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward -[rank6]: out = torch.empty( -[rank6]: ^^^^^^^^^^^^ -[rank6]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 6 has a total capacity of 178.35 GiB of which 98.81 MiB is free. Including non-PyTorch memory, this process has 178.24 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.75 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -W0826 15:06:39.968000 62367 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers -W0826 15:06:39.969000 62367 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 62466 closing signal SIGTERM -W0826 15:06:39.969000 62367 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 62467 closing signal SIGTERM -W0826 15:06:39.973000 62367 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 62468 closing signal SIGTERM -W0826 15:06:39.975000 62367 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 62469 closing signal SIGTERM -W0826 15:06:39.979000 62367 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 62470 closing signal SIGTERM -W0826 15:06:39.980000 62367 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 62471 closing signal SIGTERM -W0826 15:06:39.986000 62367 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 62472 closing signal SIGTERM -W0826 15:06:39.987000 62367 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 62473 closing signal SIGTERM +[rank5]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 5 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 408.00 MiB. GPU 3 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute\n output, shared_expert_output, mlp_bias = self.experts_compute(\n ^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute\n expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 875, in forward\n intermediate_parallel, bias_parallel = self.linear_fc1(\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward\n out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn\n return fn(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward\n out = linear_fn(*args)\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward\n out = torch.empty(\n ^^^^^^^^^^^^\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 3 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] +WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 408.00 MiB. GPU 2 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute\n output, shared_expert_output, mlp_bias = self.experts_compute(\n ^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute\n expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 875, in forward\n intermediate_parallel, bias_parallel = self.linear_fc1(\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward\n out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn\n return fn(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward\n out = linear_fn(*args)\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward\n out = torch.empty(\n ^^^^^^^^^^^^\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 2 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] +[rank3]: Traceback (most recent call last): +[rank3]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 394, in +[rank3]: pretrain( +[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 917, in pretrain +[rank3]: iteration, num_floating_point_operations_so_far = train( +[rank3]: ^^^^^^ +[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 2326, in train +[rank3]: ) = train_step( +[rank3]: ^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 1433, in train_step +[rank3]: losses_reduced = forward_backward_func( +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining +[rank3]: output_tensor, num_tokens = forward_step( +[rank3]: ^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step +[rank3]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step +[rank3]: output_tensor = model( +[rank3]: ^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank3]: return self._call_impl(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank3]: return forward_call(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward +[rank3]: return self.module(*inputs, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank3]: return self._call_impl(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank3]: return inner() +[rank3]: ^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank3]: result = forward_call(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward +[rank3]: outputs = self.module(*inputs, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank3]: return self._call_impl(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank3]: return inner() +[rank3]: ^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank3]: result = forward_call(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward +[rank3]: hidden_states = self.decoder( +[rank3]: ^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank3]: return self._call_impl(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank3]: return inner() +[rank3]: ^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank3]: result = forward_call(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward +[rank3]: hidden_states, context = layer( +[rank3]: ^^^^^^ +[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__ +[rank3]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank3]: return self._call_impl(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank3]: return inner() +[rank3]: ^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank3]: result = forward_call(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward +[rank3]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp +[rank3]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank3]: return self._call_impl(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank3]: return inner() +[rank3]: ^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank3]: result = forward_call(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward +[rank3]: output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute +[rank3]: output, shared_expert_output, mlp_bias = self.experts_compute( +[rank3]: ^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute +[rank3]: expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank3]: return self._call_impl(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank3]: return inner() +[rank3]: ^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank3]: result = forward_call(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 875, in forward +[rank3]: intermediate_parallel, bias_parallel = self.linear_fc1( +[rank3]: ^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank3]: return self._call_impl(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank3]: return inner() +[rank3]: ^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank3]: result = forward_call(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward +[rank3]: out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn +[rank3]: return fn(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward +[rank3]: out = linear_fn(*args) +[rank3]: ^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply +[rank3]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward +[rank3]: out = torch.empty( +[rank3]: ^^^^^^^^^^^^ +[rank3]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 3 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +[rank2]: Traceback (most recent call last): +[rank2]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 394, in +[rank2]: pretrain( +[rank2]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 917, in pretrain +[rank2]: iteration, num_floating_point_operations_so_far = train( +[rank2]: ^^^^^^ +[rank2]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 2326, in train +[rank2]: ) = train_step( +[rank2]: ^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 1433, in train_step +[rank2]: losses_reduced = forward_backward_func( +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-MoE-Echo/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining +[rank2]: output_tensor, num_tokens = forward_step( +[rank2]: ^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-MoE-Echo/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step +[rank2]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step +[rank2]: output_tensor = model( +[rank2]: ^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank2]: return self._call_impl(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank2]: return forward_call(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward +[rank2]: return self.module(*inputs, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank2]: return self._call_impl(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank2]: return inner() +[rank2]: ^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank2]: result = forward_call(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward +[rank2]: outputs = self.module(*inputs, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank2]: return self._call_impl(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank2]: return inner() +[rank2]: ^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank2]: result = forward_call(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward +[rank2]: hidden_states = self.decoder( +[rank2]: ^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank2]: return self._call_impl(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank2]: return inner() +[rank2]: ^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank2]: result = forward_call(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward +[rank2]: hidden_states, context = layer( +[rank2]: ^^^^^^ +[rank2]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__ +[rank2]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank2]: return self._call_impl(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank2]: return inner() +[rank2]: ^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank2]: result = forward_call(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward +[rank2]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp +[rank2]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank2]: return self._call_impl(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank2]: return inner() +[rank2]: ^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank2]: result = forward_call(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward +[rank2]: output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute +[rank2]: output, shared_expert_output, mlp_bias = self.experts_compute( +[rank2]: ^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute +[rank2]: expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank2]: return self._call_impl(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank2]: return inner() +[rank2]: ^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank2]: result = forward_call(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 875, in forward +[rank2]: intermediate_parallel, bias_parallel = self.linear_fc1( +[rank2]: ^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank2]: return self._call_impl(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank2]: return inner() +[rank2]: ^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank2]: result = forward_call(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward +[rank2]: out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn +[rank2]: return fn(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward +[rank2]: out = linear_fn(*args) +[rank2]: ^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply +[rank2]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward +[rank2]: out = torch.empty( +[rank2]: ^^^^^^^^^^^^ +[rank2]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 2 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +W0826 15:54:40.424000 153251 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 15:54:40.424000 153251 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 153347 closing signal SIGTERM +W0826 15:54:40.426000 153251 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 153348 closing signal SIGTERM +W0826 15:54:40.428000 153251 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 153349 closing signal SIGTERM +W0826 15:54:40.430000 153251 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 153350 closing signal SIGTERM +W0826 15:54:40.436000 153251 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 153351 closing signal SIGTERM +W0826 15:54:40.446000 153251 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 153352 closing signal SIGTERM +W0826 15:54:40.450000 153251 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 153353 closing signal SIGTERM +W0826 15:54:40.455000 153251 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 153354 closing signal SIGTERM Traceback (most recent call last): File "/usr/local/bin/torchrun", line 8, in sys.exit(main()) @@ -2418,4 +2418,4 @@ Traceback (most recent call last): time.sleep(monitor_interval) File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 62367 got signal: 15 +torch.distributed.elastic.multiprocessing.api.SignalException: Process 153251 got signal: 15 diff --git a/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1/mem_samples.csv b/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1/mem_samples.csv index 37e4f638f5de7b4a7969d9c701d21d6b79c77af8..6cc3a68c9c35e632bb33272bfaed542d656121ec 100644 --- a/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1/mem_samples.csv +++ b/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1/mem_samples.csv @@ -14,7 +14,7 @@ 5, 0 6, 0 7, 0 -0, 4 +0, 1 1, 1 2, 1 3, 1 @@ -39,21 +39,21 @@ 6, 4 7, 4 0, 4 -1, 4 -2, 4 -3, 4 -4, 4 -5, 4 -6, 4 -7, 4 -0, 821 -1, 821 -2, 821 -3, 821 -4, 821 -5, 821 -6, 821 -7, 821 +1, 629 +2, 629 +3, 629 +4, 629 +5, 629 +6, 629 +7, 629 +0, 913 +1, 933 +2, 917 +3, 921 +4, 911 +5, 909 +6, 905 +7, 931 0, 1831 1, 1831 2, 1831 @@ -62,38 +62,38 @@ 5, 1831 6, 1831 7, 1831 -0, 3735 -1, 3745 -2, 3723 -3, 3727 -4, 3737 -5, 3743 -6, 3733 -7, 3725 -0, 15371 -1, 15373 -2, 15373 -3, 15373 -4, 15373 -5, 15373 -6, 15373 -7, 15373 -0, 55341 -1, 55343 -2, 55343 -3, 55343 -4, 55343 -5, 55343 -6, 55343 -7, 55343 -0, 67357 -1, 67409 -2, 67367 -3, 67315 -4, 67353 -5, 67413 -6, 67383 -7, 67383 +0, 3761 +1, 3773 +2, 3775 +3, 3781 +4, 3787 +5, 3781 +6, 3787 +7, 3761 +0, 15875 +1, 15877 +2, 15877 +3, 15877 +4, 15877 +5, 15877 +6, 15877 +7, 15877 +0, 28147 +1, 28147 +2, 28151 +3, 28151 +4, 28153 +5, 28151 +6, 28153 +7, 28151 +0, 67361 +1, 67399 +2, 67435 +3, 67303 +4, 67393 +5, 67387 +6, 67373 +7, 67407 0, 68527 1, 68565 2, 68543 @@ -118,107 +118,75 @@ 5, 68691 6, 68713 7, 68691 -0, 71871 -1, 71625 -2, 71653 -3, 71883 -4, 71605 -5, 72187 -6, 71755 -7, 71857 -0, 117205 -1, 114877 -2, 116895 -3, 117519 -4, 114367 -5, 117101 -6, 116825 -7, 116873 -0, 152069 -1, 149561 -2, 150911 -3, 152077 -4, 148449 -5, 150911 -6, 150223 -7, 150475 -0, 156087 -1, 156099 -2, 156085 -3, 156093 -4, 156089 -5, 156085 -6, 156101 -7, 156085 -0, 156777 -1, 156735 -2, 156791 -3, 156803 -4, 156741 -5, 156731 -6, 156711 -7, 156763 +0, 86169 +1, 88235 +2, 88343 +3, 87983 +4, 88333 +5, 85463 +6, 87383 +7, 87205 +0, 125825 +1, 125839 +2, 127169 +3, 125837 +4, 125833 +5, 125825 +6, 125839 +7, 125825 +0, 155969 +1, 155993 +2, 155985 +3, 155977 +4, 155987 +5, 155973 +6, 155979 +7, 155947 +0, 156825 +1, 156873 +2, 156865 +3, 156877 +4, 156799 +5, 156833 +6, 156809 +7, 156831 0, 156877 -1, 156889 -2, 156875 -3, 156883 -4, 156879 -5, 156875 +1, 156891 +2, 156877 +3, 156889 +4, 156885 +5, 156877 6, 156891 -7, 156875 +7, 156877 0, 157525 -1, 157537 -2, 157523 -3, 157531 -4, 157527 -5, 157523 +1, 157539 +2, 157525 +3, 157537 +4, 157533 +5, 157525 6, 157539 -7, 157523 -0, 167421 -1, 167335 -2, 167449 -3, 167299 -4, 167619 -5, 167435 -6, 167361 -7, 167425 -0, 182487 -1, 182571 -2, 182505 -3, 182525 -4, 182511 -5, 182505 -6, 182625 -7, 182505 -0, 182487 -1, 182571 -2, 182505 -3, 182525 -4, 182511 -5, 182505 -6, 182625 -7, 182505 -0, 182487 -1, 182571 -2, 182505 -3, 182525 -4, 182511 -5, 182505 -6, 182625 -7, 182505 +7, 157525 +0, 167413 +1, 167353 +2, 167427 +3, 167247 +4, 167341 +5, 167447 +6, 167601 +7, 167441 0, 182487 -1, 182571 -2, 182505 -3, 182525 -4, 182511 -5, 182505 -6, 182625 -7, 182505 +1, 182585 +2, 182519 +3, 182531 +4, 182599 +5, 182519 +6, 182545 +7, 182519 0, 182487 -1, 182571 -2, 182505 -3, 182525 -4, 182511 -5, 182505 -6, 182625 -7, 182505 +1, 182585 +2, 182517 +3, 182531 +4, 182599 +5, 182519 +6, 182545 +7, 182519 diff --git a/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1/train.log b/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1/train.log index 7a66793244c6bdc44199bdd6236e9079be30a886..abf8d740d06165b99f62c5d42b07ec6d446ffdc2 100644 --- a/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1/train.log +++ b/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1/train.log @@ -1,6 +1,6 @@ -[15:06:51] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) -[15:06:51] WARN: moonep_backend_token_routing_analysis lacks --moe-router-fusion; QWEN3_30B runs unfused (timing only, routing identical) -[15:06:51] WARN: moonep_backend_token_routing_analysis lacks --moe-debug-drop-logging; probe will have no pre/post rank loads +[15:54:51] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +[15:54:51] WARN: moonep_backend_token_routing_analysis lacks --moe-router-fusion; QWEN3_30B runs unfused (timing only, routing identical) +[15:54:51] WARN: moonep_backend_token_routing_analysis lacks --moe-debug-drop-logging; probe will have no pre/post rank loads ============================================================== QWEN3_30B / moonep (slots/rank=16 policy=matched) E=128 K=8 H'=768 m=16 @@ -9,16 +9,10 @@ memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1/memory_snapshots/snapshot.pickle repo=/workspace/Megatron-MoE-Echo @ 35ae6e7f ============================================================== -W0826 15:06:52.826000 65221 torch/distributed/run.py:766] -W0826 15:06:52.826000 65221 torch/distributed/run.py:766] ***************************************** -W0826 15:06:52.826000 65221 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -W0826 15:06:52.826000 65221 torch/distributed/run.py:766] ***************************************** -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +W0826 15:54:53.347000 155599 torch/distributed/run.py:766] +W0826 15:54:53.347000 155599 torch/distributed/run.py:766] ***************************************** +W0826 15:54:53.347000 155599 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 15:54:53.347000 155599 torch/distributed/run.py:766] ***************************************** using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer Number of virtual stages per pipeline stage: None @@ -569,63 +563,69 @@ using torch.bfloat16 for parameters ... -------------------- end of arguments --------------------- INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 > building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") - > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) -WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS -> initializing torch distributed ... WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -[Gloo] Rank [Gloo] Rank 2 is connected to 6[Gloo] Rank 7 is connected to peer ranks. 07Expected number of connected peer ranks is : is connected to peer ranks. 77Expected number of connected peer ranks is : - peer ranks. 7Expected number of connected peer ranks is : -7 -[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank 7[Gloo] Rank 61 is connected to [Gloo] Rank is connected to is connected to 7077 peer ranks. is connected to peer ranks. peer ranks. Expected number of connected peer ranks is : 7Expected number of connected peer ranks is : Expected number of connected peer ranks is : 7 peer ranks. 7[Gloo] Rank 7 -Expected number of connected peer ranks is : -4 -7 is connected to 7 - peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 > initialized tensor model parallel with size 1 > initialized pipeline model parallel with size 1 > setting random seeds to 1234 ... [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 00 is connected to [Gloo] Rank is connected to 000 peer ranks. is connected to peer ranks. Expected number of connected peer ranks is : 0Expected number of connected peer ranks is : 0 peer ranks. 0 +Expected number of connected peer ranks is : +0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[rank5]:[W826 15:07:11.681615793 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank7]:[W826 15:07:11.681703741 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank6]:[W826 15:07:11.681903516 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank1]:[W826 15:07:11.684860635 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank3]:[W826 15:07:11.684997644 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank2]:[W826 15:07:11.686194224 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank4]:[W826 15:07:11.686280151 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 15:55:11.649123242 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 15:55:11.649145568 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 15:55:11.649744199 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 15:55:11.651375573 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 15:55:11.651466022 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 15:55:11.651612793 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 15:55:11.658944426 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. > compiling dataset index builder ... make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' make: Nothing to be done for 'default'. make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' ->>> done with dataset index builder. Compilation time: 0.034 seconds +>>> done with dataset index builder. Compilation time: 0.033 seconds WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. > compiling and loading fused kernels ... -[rank0]:[W826 15:07:11.836804628 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. ->>> done with compiling and loading fused kernels. Compilation time: 4.096 seconds -time to initialize megatron (seconds): 7.258 -[after megatron is initialized] datetime: 2026-08-26 15:07:18 +[rank0]:[W826 15:55:11.801466905 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.671 seconds +time to initialize megatron (seconds): 7.060 +[after megatron is initialized] datetime: 2026-08-26 15:55:17 building GPT model ... /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. warnings.warn( @@ -649,28 +649,30 @@ INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradi Params for bucket 1 (103022592 elements, 103022592 padded size): module.output_layer.weight Params for bucket 2 (49033984 elements, 49033984 padded size): - module.decoder.layers.47.pre_mlp_layernorm.weight - module.decoder.layers.47.self_attention.linear_proj.weight - module.decoder.layers.46.self_attention.linear_proj.weight - module.decoder.layers.46.self_attention.linear_qkv.weight - module.decoder.layers.45.self_attention.linear_qkv.weight - module.decoder.final_layernorm.weight module.decoder.layers.47.self_attention.linear_qkv.weight - module.decoder.layers.46.self_attention.k_layernorm.weight + module.decoder.layers.47.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.46.self_attention.linear_qkv.weight module.decoder.layers.46.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.45.mlp.router.weight + module.decoder.layers.46.self_attention.k_layernorm.weight + module.decoder.layers.45.self_attention.linear_qkv.weight module.decoder.layers.45.self_attention.k_layernorm.weight + module.decoder.final_layernorm.weight module.decoder.layers.47.self_attention.k_layernorm.weight - module.decoder.layers.47.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.47.self_attention.linear_proj.weight module.decoder.layers.46.mlp.router.weight + module.decoder.layers.45.mlp.router.weight module.decoder.layers.46.self_attention.q_layernorm.weight - module.decoder.layers.45.pre_mlp_layernorm.weight - module.decoder.layers.47.mlp.router.weight module.decoder.layers.47.self_attention.q_layernorm.weight + module.decoder.layers.47.mlp.router.weight module.decoder.layers.46.pre_mlp_layernorm.weight + module.decoder.layers.45.pre_mlp_layernorm.weight + module.decoder.layers.46.self_attention.linear_proj.weight + module.decoder.layers.47.pre_mlp_layernorm.weight module.decoder.layers.45.self_attention.q_layernorm.weight Params for bucket 3 (46672384 elements, 46672384 padded size): + module.decoder.layers.45.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.44.mlp.router.weight + module.decoder.layers.45.self_attention.linear_proj.weight module.decoder.layers.44.pre_mlp_layernorm.weight module.decoder.layers.44.self_attention.k_layernorm.weight module.decoder.layers.44.self_attention.q_layernorm.weight @@ -684,70 +686,68 @@ Params for bucket 3 (46672384 elements, 46672384 padded size): module.decoder.layers.43.self_attention.linear_qkv.weight module.decoder.layers.43.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.43.self_attention.linear_proj.weight - module.decoder.layers.45.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.45.self_attention.linear_proj.weight Params for bucket 4 (49031936 elements, 49031936 padded size): module.decoder.layers.42.self_attention.linear_qkv.weight + module.decoder.layers.41.mlp.router.weight module.decoder.layers.40.mlp.router.weight - module.decoder.layers.40.self_attention.k_layernorm.weight - module.decoder.layers.41.pre_mlp_layernorm.weight - module.decoder.layers.41.self_attention.linear_qkv.weight - module.decoder.layers.42.mlp.router.weight module.decoder.layers.42.self_attention.k_layernorm.weight module.decoder.layers.42.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.41.pre_mlp_layernorm.weight module.decoder.layers.40.pre_mlp_layernorm.weight module.decoder.layers.40.self_attention.q_layernorm.weight - module.decoder.layers.41.self_attention.q_layernorm.weight - module.decoder.layers.41.self_attention.linear_proj.weight + module.decoder.layers.41.self_attention.linear_qkv.weight + module.decoder.layers.42.mlp.router.weight module.decoder.layers.42.self_attention.q_layernorm.weight - module.decoder.layers.42.pre_mlp_layernorm.weight module.decoder.layers.42.self_attention.linear_proj.weight module.decoder.layers.41.self_attention.k_layernorm.weight - module.decoder.layers.40.self_attention.linear_qkv.weight - module.decoder.layers.41.mlp.router.weight + module.decoder.layers.41.self_attention.q_layernorm.weight + module.decoder.layers.42.pre_mlp_layernorm.weight module.decoder.layers.41.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.40.self_attention.linear_qkv.weight + module.decoder.layers.41.self_attention.linear_proj.weight + module.decoder.layers.40.self_attention.k_layernorm.weight Params for bucket 5 (46672384 elements, 46672384 padded size): module.decoder.layers.40.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.39.mlp.router.weight - module.decoder.layers.39.pre_mlp_layernorm.weight - module.decoder.layers.39.self_attention.k_layernorm.weight module.decoder.layers.40.self_attention.linear_proj.weight - module.decoder.layers.39.self_attention.q_layernorm.weight module.decoder.layers.39.self_attention.linear_qkv.weight - module.decoder.layers.39.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.39.self_attention.linear_proj.weight - module.decoder.layers.38.self_attention.linear_qkv.weight - module.decoder.layers.38.self_attention.linear_proj.weight module.decoder.layers.38.mlp.router.weight module.decoder.layers.38.pre_mlp_layernorm.weight module.decoder.layers.38.self_attention.k_layernorm.weight module.decoder.layers.38.self_attention.q_layernorm.weight + module.decoder.layers.38.self_attention.linear_qkv.weight module.decoder.layers.38.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.39.mlp.router.weight + module.decoder.layers.39.pre_mlp_layernorm.weight + module.decoder.layers.39.self_attention.k_layernorm.weight + module.decoder.layers.39.self_attention.q_layernorm.weight + module.decoder.layers.39.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.38.self_attention.linear_proj.weight Params for bucket 6 (49031936 elements, 49031936 padded size): - module.decoder.layers.37.pre_mlp_layernorm.weight - module.decoder.layers.35.self_attention.k_layernorm.weight - module.decoder.layers.36.self_attention.linear_proj.weight - module.decoder.layers.35.self_attention.linear_qkv.weight - module.decoder.layers.36.self_attention.linear_qkv.weight - module.decoder.layers.35.self_attention.q_layernorm.weight - module.decoder.layers.37.self_attention.linear_qkv.weight module.decoder.layers.37.self_attention.k_layernorm.weight module.decoder.layers.37.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.36.mlp.router.weight - module.decoder.layers.36.self_attention.k_layernorm.weight - module.decoder.layers.36.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.35.pre_mlp_layernorm.weight + module.decoder.layers.35.self_attention.q_layernorm.weight module.decoder.layers.37.mlp.router.weight module.decoder.layers.37.self_attention.q_layernorm.weight module.decoder.layers.37.self_attention.linear_proj.weight + module.decoder.layers.36.self_attention.k_layernorm.weight + module.decoder.layers.36.self_attention.linear_qkv.weight + module.decoder.layers.37.pre_mlp_layernorm.weight + module.decoder.layers.36.mlp.router.weight + module.decoder.layers.36.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.35.self_attention.k_layernorm.weight + module.decoder.layers.35.self_attention.linear_qkv.weight module.decoder.layers.36.pre_mlp_layernorm.weight - module.decoder.layers.35.pre_mlp_layernorm.weight - module.decoder.layers.35.mlp.router.weight module.decoder.layers.36.self_attention.q_layernorm.weight + module.decoder.layers.36.self_attention.linear_proj.weight + module.decoder.layers.35.mlp.router.weight + module.decoder.layers.37.self_attention.linear_qkv.weight Params for bucket 7 (46672384 elements, 46672384 padded size): module.decoder.layers.35.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.35.self_attention.linear_proj.weight + module.decoder.layers.33.mlp.router.weight module.decoder.layers.33.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.33.self_attention.q_layernorm.weight module.decoder.layers.34.mlp.router.weight module.decoder.layers.34.pre_mlp_layernorm.weight module.decoder.layers.34.self_attention.k_layernorm.weight @@ -755,325 +755,317 @@ Params for bucket 7 (46672384 elements, 46672384 padded size): module.decoder.layers.34.self_attention.linear_qkv.weight module.decoder.layers.34.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.34.self_attention.linear_proj.weight - module.decoder.layers.33.mlp.router.weight module.decoder.layers.33.pre_mlp_layernorm.weight module.decoder.layers.33.self_attention.k_layernorm.weight + module.decoder.layers.33.self_attention.q_layernorm.weight module.decoder.layers.33.self_attention.linear_qkv.weight module.decoder.layers.33.self_attention.linear_proj.weight Params for bucket 8 (49031936 elements, 49031936 padded size): - module.decoder.layers.31.self_attention.k_layernorm.weight module.decoder.layers.31.self_attention.linear_qkv.weight module.decoder.layers.30.mlp.router.weight - module.decoder.layers.30.pre_mlp_layernorm.weight - module.decoder.layers.31.mlp.router.weight - module.decoder.layers.31.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.32.self_attention.linear_qkv.weight module.decoder.layers.32.self_attention.k_layernorm.weight - module.decoder.layers.31.pre_mlp_layernorm.weight - module.decoder.layers.31.self_attention.q_layernorm.weight - module.decoder.layers.31.self_attention.linear_proj.weight - module.decoder.layers.30.self_attention.k_layernorm.weight - module.decoder.layers.30.self_attention.linear_qkv.weight module.decoder.layers.32.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.31.mlp.router.weight + module.decoder.layers.31.self_attention.k_layernorm.weight + module.decoder.layers.31.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.30.pre_mlp_layernorm.weight module.decoder.layers.30.self_attention.q_layernorm.weight module.decoder.layers.32.mlp.router.weight - module.decoder.layers.32.pre_mlp_layernorm.weight module.decoder.layers.32.self_attention.q_layernorm.weight module.decoder.layers.32.self_attention.linear_proj.weight + module.decoder.layers.31.pre_mlp_layernorm.weight + module.decoder.layers.31.self_attention.q_layernorm.weight + module.decoder.layers.32.pre_mlp_layernorm.weight + module.decoder.layers.31.self_attention.linear_proj.weight + module.decoder.layers.30.self_attention.k_layernorm.weight + module.decoder.layers.30.self_attention.linear_qkv.weight Params for bucket 9 (46672384 elements, 46672384 padded size): - module.decoder.layers.30.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.29.mlp.router.weight - module.decoder.layers.30.self_attention.linear_proj.weight + module.decoder.layers.29.self_attention.k_layernorm.weight module.decoder.layers.29.pre_mlp_layernorm.weight module.decoder.layers.29.self_attention.q_layernorm.weight module.decoder.layers.29.self_attention.linear_qkv.weight module.decoder.layers.29.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.29.self_attention.linear_proj.weight module.decoder.layers.28.mlp.router.weight - module.decoder.layers.28.pre_mlp_layernorm.weight module.decoder.layers.28.self_attention.k_layernorm.weight - module.decoder.layers.28.self_attention.linear_qkv.weight - module.decoder.layers.28.self_attention.linear_proj.weight - module.decoder.layers.29.self_attention.k_layernorm.weight - module.decoder.layers.29.self_attention.linear_proj.weight module.decoder.layers.28.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.30.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.30.self_attention.linear_proj.weight + module.decoder.layers.28.pre_mlp_layernorm.weight module.decoder.layers.28.self_attention.q_layernorm.weight + module.decoder.layers.28.self_attention.linear_qkv.weight + module.decoder.layers.28.self_attention.linear_proj.weight Params for bucket 10 (49031936 elements, 49031936 padded size): - module.decoder.layers.26.self_attention.linear_qkv.weight - module.decoder.layers.27.self_attention.linear_qkv.weight - module.decoder.layers.27.pre_mlp_layernorm.weight - module.decoder.layers.27.self_attention.linear_proj.weight - module.decoder.layers.26.pre_mlp_layernorm.weight - module.decoder.layers.26.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.25.self_attention.k_layernorm.weight module.decoder.layers.25.self_attention.linear_qkv.weight - module.decoder.layers.27.self_attention.q_layernorm.weight - module.decoder.layers.27.self_attention.k_layernorm.weight - module.decoder.layers.26.self_attention.k_layernorm.weight - module.decoder.layers.26.self_attention.q_layernorm.weight - module.decoder.layers.25.mlp.router.weight - module.decoder.layers.25.self_attention.q_layernorm.weight - module.decoder.layers.27.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.26.mlp.router.weight module.decoder.layers.26.self_attention.linear_proj.weight module.decoder.layers.25.pre_mlp_layernorm.weight + module.decoder.layers.27.self_attention.linear_qkv.weight + module.decoder.layers.27.self_attention.k_layernorm.weight + module.decoder.layers.27.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.26.self_attention.k_layernorm.weight + module.decoder.layers.26.self_attention.linear_qkv.weight + module.decoder.layers.25.self_attention.k_layernorm.weight module.decoder.layers.27.mlp.router.weight + module.decoder.layers.27.self_attention.q_layernorm.weight + module.decoder.layers.26.pre_mlp_layernorm.weight + module.decoder.layers.26.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.25.self_attention.q_layernorm.weight + module.decoder.layers.27.pre_mlp_layernorm.weight + module.decoder.layers.27.self_attention.linear_proj.weight + module.decoder.layers.26.self_attention.q_layernorm.weight + module.decoder.layers.25.mlp.router.weight Params for bucket 11 (46672384 elements, 46672384 padded size): - module.decoder.layers.25.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.24.mlp.router.weight - module.decoder.layers.24.pre_mlp_layernorm.weight module.decoder.layers.25.self_attention.linear_proj.weight module.decoder.layers.24.self_attention.k_layernorm.weight - module.decoder.layers.24.self_attention.q_layernorm.weight + module.decoder.layers.24.pre_mlp_layernorm.weight module.decoder.layers.24.self_attention.linear_qkv.weight module.decoder.layers.24.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.24.self_attention.linear_proj.weight module.decoder.layers.23.mlp.router.weight module.decoder.layers.23.pre_mlp_layernorm.weight module.decoder.layers.23.self_attention.k_layernorm.weight - module.decoder.layers.23.self_attention.q_layernorm.weight + module.decoder.layers.25.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.24.self_attention.q_layernorm.weight module.decoder.layers.23.self_attention.linear_qkv.weight module.decoder.layers.23.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.23.self_attention.q_layernorm.weight module.decoder.layers.23.self_attention.linear_proj.weight Params for bucket 12 (49031936 elements, 49031936 padded size): - module.decoder.layers.21.mlp.router.weight - module.decoder.layers.21.self_attention.k_layernorm.weight - module.decoder.layers.21.self_attention.linear_qkv.weight - module.decoder.layers.21.self_attention.q_layernorm.weight - module.decoder.layers.22.self_attention.linear_qkv.weight module.decoder.layers.22.self_attention.k_layernorm.weight - module.decoder.layers.22.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.21.pre_mlp_layernorm.weight + module.decoder.layers.22.self_attention.linear_qkv.weight module.decoder.layers.21.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.20.self_attention.linear_qkv.weight + module.decoder.layers.22.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.20.self_attention.k_layernorm.weight + module.decoder.layers.20.self_attention.linear_qkv.weight module.decoder.layers.22.mlp.router.weight + module.decoder.layers.21.pre_mlp_layernorm.weight + module.decoder.layers.21.self_attention.q_layernorm.weight module.decoder.layers.22.self_attention.q_layernorm.weight - module.decoder.layers.20.self_attention.q_layernorm.weight - module.decoder.layers.21.self_attention.linear_proj.weight - module.decoder.layers.20.mlp.router.weight module.decoder.layers.22.pre_mlp_layernorm.weight module.decoder.layers.22.self_attention.linear_proj.weight + module.decoder.layers.20.mlp.router.weight + module.decoder.layers.21.self_attention.linear_proj.weight module.decoder.layers.20.pre_mlp_layernorm.weight + module.decoder.layers.20.self_attention.q_layernorm.weight + module.decoder.layers.21.mlp.router.weight + module.decoder.layers.21.self_attention.k_layernorm.weight + module.decoder.layers.21.self_attention.linear_qkv.weight Params for bucket 13 (46672384 elements, 46672384 padded size): module.decoder.layers.20.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.20.self_attention.linear_proj.weight + module.decoder.layers.18.self_attention.linear_proj.weight + module.decoder.layers.18.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.19.mlp.router.weight module.decoder.layers.19.pre_mlp_layernorm.weight - module.decoder.layers.20.self_attention.linear_proj.weight module.decoder.layers.19.self_attention.k_layernorm.weight module.decoder.layers.19.self_attention.q_layernorm.weight module.decoder.layers.19.self_attention.linear_qkv.weight module.decoder.layers.19.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.19.self_attention.linear_proj.weight module.decoder.layers.18.mlp.router.weight - module.decoder.layers.18.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.18.self_attention.q_layernorm.weight module.decoder.layers.18.pre_mlp_layernorm.weight - module.decoder.layers.18.self_attention.linear_qkv.weight module.decoder.layers.18.self_attention.k_layernorm.weight - module.decoder.layers.18.self_attention.linear_proj.weight + module.decoder.layers.18.self_attention.linear_qkv.weight + module.decoder.layers.18.self_attention.q_layernorm.weight Params for bucket 14 (49031936 elements, 49031936 padded size): module.decoder.layers.16.pre_mlp_layernorm.weight - module.decoder.layers.16.self_attention.q_layernorm.weight - module.decoder.layers.15.self_attention.linear_qkv.weight - module.decoder.layers.17.self_attention.k_layernorm.weight - module.decoder.layers.17.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.15.pre_mlp_layernorm.weight module.decoder.layers.16.self_attention.linear_proj.weight - module.decoder.layers.17.mlp.router.weight - module.decoder.layers.17.self_attention.q_layernorm.weight module.decoder.layers.15.self_attention.k_layernorm.weight + module.decoder.layers.17.self_attention.linear_qkv.weight + module.decoder.layers.17.self_attention.k_layernorm.weight + module.decoder.layers.17.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.16.self_attention.linear_qkv.weight - module.decoder.layers.17.pre_mlp_layernorm.weight - module.decoder.layers.17.self_attention.linear_proj.weight + module.decoder.layers.15.mlp.router.weight module.decoder.layers.15.self_attention.q_layernorm.weight + module.decoder.layers.17.mlp.router.weight module.decoder.layers.16.mlp.router.weight module.decoder.layers.16.self_attention.k_layernorm.weight module.decoder.layers.16.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.17.self_attention.linear_qkv.weight - module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.17.self_attention.q_layernorm.weight + module.decoder.layers.17.pre_mlp_layernorm.weight + module.decoder.layers.17.self_attention.linear_proj.weight + module.decoder.layers.16.self_attention.q_layernorm.weight + module.decoder.layers.15.self_attention.linear_qkv.weight Params for bucket 15 (46672384 elements, 46672384 padded size): - module.decoder.layers.13.self_attention.k_layernorm.weight - module.decoder.layers.13.self_attention.linear_qkv.weight module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.14.mlp.router.weight module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.k_layernorm.weight + module.decoder.layers.13.self_attention.q_layernorm.weight + module.decoder.layers.13.self_attention.linear_qkv.weight + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.13.self_attention.linear_proj.weight module.decoder.layers.14.pre_mlp_layernorm.weight module.decoder.layers.14.self_attention.k_layernorm.weight module.decoder.layers.14.self_attention.q_layernorm.weight module.decoder.layers.14.self_attention.linear_qkv.weight module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.14.self_attention.linear_proj.weight - module.decoder.layers.13.mlp.router.weight - module.decoder.layers.13.self_attention.linear_proj.weight - module.decoder.layers.13.pre_mlp_layernorm.weight - module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.13.self_attention.q_layernorm.weight Params for bucket 16 (49031936 elements, 49031936 padded size): - module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.self_attention.q_layernorm.weight + module.decoder.layers.11.self_attention.k_layernorm.weight + module.decoder.layers.11.self_attention.linear_qkv.weight module.decoder.layers.10.mlp.router.weight - module.decoder.layers.11.pre_mlp_layernorm.weight - module.decoder.layers.12.self_attention.linear_qkv.weight - module.decoder.layers.10.pre_mlp_layernorm.weight module.decoder.layers.10.self_attention.linear_qkv.weight - module.decoder.layers.11.self_attention.linear_qkv.weight - module.decoder.layers.12.mlp.router.weight - module.decoder.layers.12.self_attention.k_layernorm.weight - module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.10.self_attention.q_layernorm.weight - module.decoder.layers.11.self_attention.k_layernorm.weight - module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.12.self_attention.q_layernorm.weight - module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.self_attention.linear_proj.weight module.decoder.layers.10.self_attention.k_layernorm.weight module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.12.self_attention.linear_qkv.weight + module.decoder.layers.12.self_attention.k_layernorm.weight + module.decoder.layers.11.pre_mlp_layernorm.weight module.decoder.layers.11.self_attention.q_layernorm.weight module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight Params for bucket 17 (46672384 elements, 46672384 padded size): module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.9.self_attention.k_layernorm.weight - module.decoder.layers.10.self_attention.linear_proj.weight module.decoder.layers.9.pre_mlp_layernorm.weight module.decoder.layers.9.mlp.router.weight + module.decoder.layers.10.self_attention.linear_proj.weight module.decoder.layers.9.self_attention.q_layernorm.weight module.decoder.layers.9.self_attention.linear_qkv.weight module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.9.self_attention.linear_proj.weight module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.linear_qkv.weight module.decoder.layers.8.pre_mlp_layernorm.weight module.decoder.layers.8.self_attention.k_layernorm.weight - module.decoder.layers.8.self_attention.q_layernorm.weight - module.decoder.layers.8.self_attention.linear_qkv.weight module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.self_attention.q_layernorm.weight Params for bucket 18 (49031936 elements, 49031936 padded size): + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.linear_qkv.weight module.decoder.layers.7.pre_mlp_layernorm.weight - module.decoder.layers.7.self_attention.q_layernorm.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.q_layernorm.weight module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.self_attention.k_layernorm.weight + module.decoder.layers.6.self_attention.k_layernorm.weight module.decoder.layers.6.self_attention.linear_proj.weight module.decoder.layers.5.self_attention.linear_qkv.weight - module.decoder.layers.7.self_attention.linear_qkv.weight - module.decoder.layers.6.self_attention.k_layernorm.weight - module.decoder.layers.6.self_attention.linear_qkv.weight - module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.5.self_attention.q_layernorm.weight - module.decoder.layers.7.self_attention.k_layernorm.weight + module.decoder.layers.7.self_attention.q_layernorm.weight module.decoder.layers.7.self_attention.linear_proj.weight - module.decoder.layers.6.mlp.router.weight - module.decoder.layers.6.self_attention.q_layernorm.weight module.decoder.layers.5.mlp.router.weight - module.decoder.layers.7.mlp.router.weight - module.decoder.layers.6.pre_mlp_layernorm.weight - module.decoder.layers.5.pre_mlp_layernorm.weight module.decoder.layers.5.self_attention.k_layernorm.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.5.self_attention.q_layernorm.weight Params for bucket 19 (46672384 elements, 46672384 padded size): - module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.linear_proj.weight module.decoder.layers.3.mlp.router.weight module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.k_layernorm.weight module.decoder.layers.3.self_attention.q_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.weight module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.self_attention.linear_proj.weight module.decoder.layers.4.self_attention.k_layernorm.weight module.decoder.layers.4.mlp.router.weight module.decoder.layers.4.pre_mlp_layernorm.weight module.decoder.layers.4.self_attention.q_layernorm.weight - module.decoder.layers.4.self_attention.linear_qkv.weight module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.4.self_attention.linear_proj.weight - module.decoder.layers.3.self_attention.k_layernorm.weight - module.decoder.layers.3.self_attention.linear_qkv.weight - module.decoder.layers.3.self_attention.linear_proj.weight Params for bucket 20 (49031936 elements, 49031936 padded size): + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.k_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.0.self_attention.k_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.pre_mlp_layernorm.weight module.decoder.layers.2.self_attention.q_layernorm.weight - module.decoder.layers.2.self_attention.linear_proj.weight - module.decoder.layers.1.mlp.router.weight module.decoder.layers.1.self_attention.q_layernorm.weight module.decoder.layers.1.self_attention.linear_proj.weight - module.decoder.layers.2.pre_mlp_layernorm.weight - module.decoder.layers.1.pre_mlp_layernorm.weight - module.decoder.layers.0.pre_mlp_layernorm.weight - module.decoder.layers.0.self_attention.q_layernorm.weight + module.decoder.layers.1.mlp.router.weight module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.0.self_attention.q_layernorm.weight + module.decoder.layers.1.pre_mlp_layernorm.weight module.decoder.layers.2.mlp.router.weight - module.decoder.layers.2.self_attention.linear_qkv.weight module.decoder.layers.2.self_attention.k_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.0.pre_mlp_layernorm.weight module.decoder.layers.0.mlp.router.weight - module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.1.self_attention.k_layernorm.weight - module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.1.self_attention.linear_qkv.weight - module.decoder.layers.0.self_attention.k_layernorm.weight Params for bucket 21 (111413248 elements, 111413248 padded size): module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight module.embedding.word_embeddings.weight module.decoder.layers.0.self_attention.linear_proj.weight INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 89 Params for bucket 1 (40894464 elements, 40894464 padded size): - module.decoder.layers.47.mlp.experts.linear_fc2.weight11 - module.decoder.layers.47.mlp.experts.linear_fc2.weight3 + module.decoder.layers.47.mlp.experts.linear_fc2.weight12 + module.decoder.layers.47.mlp.experts.linear_fc2.weight4 module.decoder.layers.47.mlp.experts.linear_fc2.weight0 module.decoder.layers.47.mlp.experts.linear_fc1.weight14 + module.decoder.layers.47.mlp.experts.linear_fc2.weight9 + module.decoder.layers.47.mlp.experts.linear_fc1.weight11 + module.decoder.layers.47.mlp.experts.linear_fc2.weight14 + module.decoder.layers.47.mlp.experts.linear_fc2.weight6 + module.decoder.layers.47.mlp.experts.linear_fc2.weight11 + module.decoder.layers.47.mlp.experts.linear_fc2.weight3 + module.decoder.layers.47.mlp.experts.linear_fc1.weight13 module.decoder.layers.47.mlp.experts.linear_fc2.weight8 module.decoder.layers.47.mlp.experts.linear_fc2.weight1 - module.decoder.layers.47.mlp.experts.linear_fc1.weight11 module.decoder.layers.47.mlp.experts.linear_fc2.weight13 module.decoder.layers.47.mlp.experts.linear_fc2.weight5 + module.decoder.layers.47.mlp.experts.linear_fc1.weight15 module.decoder.layers.47.mlp.experts.linear_fc2.weight10 module.decoder.layers.47.mlp.experts.linear_fc2.weight2 - module.decoder.layers.47.mlp.experts.linear_fc1.weight13 + module.decoder.layers.47.mlp.experts.linear_fc1.weight12 module.decoder.layers.47.mlp.experts.linear_fc2.weight15 module.decoder.layers.47.mlp.experts.linear_fc2.weight7 - module.decoder.layers.47.mlp.experts.linear_fc2.weight12 - module.decoder.layers.47.mlp.experts.linear_fc2.weight4 - module.decoder.layers.47.mlp.experts.linear_fc1.weight15 - module.decoder.layers.47.mlp.experts.linear_fc2.weight9 - module.decoder.layers.47.mlp.experts.linear_fc1.weight12 - module.decoder.layers.47.mlp.experts.linear_fc2.weight14 - module.decoder.layers.47.mlp.experts.linear_fc2.weight6 Params for bucket 2 (40894464 elements, 40894464 padded size): module.decoder.layers.47.mlp.experts.linear_fc1.weight10 - module.decoder.layers.47.mlp.experts.linear_fc1.weight6 module.decoder.layers.47.mlp.experts.linear_fc1.weight8 + module.decoder.layers.47.mlp.experts.linear_fc1.weight6 module.decoder.layers.47.mlp.experts.linear_fc1.weight4 module.decoder.layers.47.mlp.experts.linear_fc1.weight3 module.decoder.layers.47.mlp.experts.linear_fc1.weight0 - module.decoder.layers.46.mlp.experts.linear_fc2.weight15 - module.decoder.layers.46.mlp.experts.linear_fc2.weight13 + module.decoder.layers.46.mlp.experts.linear_fc2.weight14 module.decoder.layers.46.mlp.experts.linear_fc2.weight12 - module.decoder.layers.47.mlp.experts.linear_fc1.weight7 module.decoder.layers.47.mlp.experts.linear_fc1.weight9 + module.decoder.layers.47.mlp.experts.linear_fc1.weight7 module.decoder.layers.47.mlp.experts.linear_fc1.weight5 module.decoder.layers.47.mlp.experts.linear_fc1.weight2 module.decoder.layers.47.mlp.experts.linear_fc1.weight1 - module.decoder.layers.46.mlp.experts.linear_fc2.weight14 + module.decoder.layers.46.mlp.experts.linear_fc2.weight15 + module.decoder.layers.46.mlp.experts.linear_fc2.weight13 Params for bucket 3 (40894464 elements, 40894464 padded size): - module.decoder.layers.46.mlp.experts.linear_fc2.weight10 - module.decoder.layers.46.mlp.experts.linear_fc2.weight1 - module.decoder.layers.46.mlp.experts.linear_fc1.weight9 - module.decoder.layers.46.mlp.experts.linear_fc2.weight7 - module.decoder.layers.46.mlp.experts.linear_fc2.weight2 - module.decoder.layers.46.mlp.experts.linear_fc1.weight14 module.decoder.layers.46.mlp.experts.linear_fc2.weight4 - module.decoder.layers.46.mlp.experts.linear_fc1.weight11 + module.decoder.layers.46.mlp.experts.linear_fc1.weight10 module.decoder.layers.46.mlp.experts.linear_fc2.weight9 module.decoder.layers.46.mlp.experts.linear_fc2.weight0 + module.decoder.layers.46.mlp.experts.linear_fc1.weight15 module.decoder.layers.46.mlp.experts.linear_fc2.weight6 - module.decoder.layers.46.mlp.experts.linear_fc1.weight13 + module.decoder.layers.46.mlp.experts.linear_fc1.weight12 module.decoder.layers.46.mlp.experts.linear_fc2.weight11 module.decoder.layers.46.mlp.experts.linear_fc2.weight3 - module.decoder.layers.46.mlp.experts.linear_fc1.weight10 + module.decoder.layers.46.mlp.experts.linear_fc1.weight9 module.decoder.layers.46.mlp.experts.linear_fc2.weight8 - module.decoder.layers.46.mlp.experts.linear_fc1.weight15 + module.decoder.layers.46.mlp.experts.linear_fc2.weight2 + module.decoder.layers.46.mlp.experts.linear_fc1.weight14 module.decoder.layers.46.mlp.experts.linear_fc2.weight5 - module.decoder.layers.46.mlp.experts.linear_fc1.weight12 + module.decoder.layers.46.mlp.experts.linear_fc1.weight11 + module.decoder.layers.46.mlp.experts.linear_fc2.weight10 + module.decoder.layers.46.mlp.experts.linear_fc2.weight1 + module.decoder.layers.46.mlp.experts.linear_fc2.weight7 + module.decoder.layers.46.mlp.experts.linear_fc1.weight13 Params for bucket 4 (40894464 elements, 40894464 padded size): - module.decoder.layers.46.mlp.experts.linear_fc1.weight7 - module.decoder.layers.46.mlp.experts.linear_fc1.weight5 - module.decoder.layers.46.mlp.experts.linear_fc1.weight2 - module.decoder.layers.46.mlp.experts.linear_fc1.weight1 - module.decoder.layers.45.mlp.experts.linear_fc2.weight15 - module.decoder.layers.45.mlp.experts.linear_fc2.weight13 - module.decoder.layers.45.mlp.experts.linear_fc2.weight11 - module.decoder.layers.45.mlp.experts.linear_fc2.weight9 module.decoder.layers.46.mlp.experts.linear_fc1.weight8 module.decoder.layers.46.mlp.experts.linear_fc1.weight6 module.decoder.layers.46.mlp.experts.linear_fc1.weight4 @@ -1083,15 +1075,15 @@ Params for bucket 4 (40894464 elements, 40894464 padded size): module.decoder.layers.45.mlp.experts.linear_fc2.weight12 module.decoder.layers.45.mlp.experts.linear_fc2.weight10 module.decoder.layers.45.mlp.experts.linear_fc2.weight8 + module.decoder.layers.46.mlp.experts.linear_fc1.weight5 + module.decoder.layers.46.mlp.experts.linear_fc1.weight7 + module.decoder.layers.46.mlp.experts.linear_fc1.weight2 + module.decoder.layers.46.mlp.experts.linear_fc1.weight1 + module.decoder.layers.45.mlp.experts.linear_fc2.weight15 + module.decoder.layers.45.mlp.experts.linear_fc2.weight13 + module.decoder.layers.45.mlp.experts.linear_fc2.weight11 + module.decoder.layers.45.mlp.experts.linear_fc2.weight9 Params for bucket 5 (40894464 elements, 40894464 padded size): - module.decoder.layers.45.mlp.experts.linear_fc2.weight7 - module.decoder.layers.45.mlp.experts.linear_fc2.weight3 - module.decoder.layers.45.mlp.experts.linear_fc2.weight5 - module.decoder.layers.45.mlp.experts.linear_fc2.weight0 - module.decoder.layers.45.mlp.experts.linear_fc1.weight14 - module.decoder.layers.45.mlp.experts.linear_fc1.weight12 - module.decoder.layers.45.mlp.experts.linear_fc1.weight10 - module.decoder.layers.45.mlp.experts.linear_fc1.weight8 module.decoder.layers.45.mlp.experts.linear_fc2.weight6 module.decoder.layers.45.mlp.experts.linear_fc2.weight4 module.decoder.layers.45.mlp.experts.linear_fc2.weight2 @@ -1101,26 +1093,34 @@ Params for bucket 5 (40894464 elements, 40894464 padded size): module.decoder.layers.45.mlp.experts.linear_fc1.weight11 module.decoder.layers.45.mlp.experts.linear_fc1.weight9 module.decoder.layers.45.mlp.experts.linear_fc1.weight7 + module.decoder.layers.45.mlp.experts.linear_fc2.weight7 + module.decoder.layers.45.mlp.experts.linear_fc2.weight5 + module.decoder.layers.45.mlp.experts.linear_fc2.weight3 + module.decoder.layers.45.mlp.experts.linear_fc2.weight0 + module.decoder.layers.45.mlp.experts.linear_fc1.weight14 + module.decoder.layers.45.mlp.experts.linear_fc1.weight12 + module.decoder.layers.45.mlp.experts.linear_fc1.weight10 + module.decoder.layers.45.mlp.experts.linear_fc1.weight8 Params for bucket 6 (40894464 elements, 40894464 padded size): + module.decoder.layers.45.mlp.experts.linear_fc1.weight2 + module.decoder.layers.44.mlp.experts.linear_fc2.weight9 + module.decoder.layers.45.mlp.experts.linear_fc1.weight6 + module.decoder.layers.44.mlp.experts.linear_fc2.weight14 + module.decoder.layers.44.mlp.experts.linear_fc2.weight6 + module.decoder.layers.45.mlp.experts.linear_fc1.weight1 + module.decoder.layers.44.mlp.experts.linear_fc2.weight11 module.decoder.layers.45.mlp.experts.linear_fc1.weight0 + module.decoder.layers.44.mlp.experts.linear_fc2.weight8 + module.decoder.layers.45.mlp.experts.linear_fc1.weight5 module.decoder.layers.44.mlp.experts.linear_fc2.weight13 module.decoder.layers.44.mlp.experts.linear_fc2.weight5 - module.decoder.layers.45.mlp.experts.linear_fc1.weight5 - module.decoder.layers.44.mlp.experts.linear_fc2.weight10 module.decoder.layers.45.mlp.experts.linear_fc1.weight3 + module.decoder.layers.44.mlp.experts.linear_fc2.weight10 module.decoder.layers.44.mlp.experts.linear_fc2.weight15 module.decoder.layers.44.mlp.experts.linear_fc2.weight7 + module.decoder.layers.45.mlp.experts.linear_fc1.weight4 module.decoder.layers.44.mlp.experts.linear_fc2.weight12 module.decoder.layers.44.mlp.experts.linear_fc2.weight4 - module.decoder.layers.45.mlp.experts.linear_fc1.weight4 - module.decoder.layers.44.mlp.experts.linear_fc2.weight9 - module.decoder.layers.45.mlp.experts.linear_fc1.weight2 - module.decoder.layers.44.mlp.experts.linear_fc2.weight14 - module.decoder.layers.44.mlp.experts.linear_fc2.weight6 - module.decoder.layers.45.mlp.experts.linear_fc1.weight6 - module.decoder.layers.44.mlp.experts.linear_fc2.weight11 - module.decoder.layers.45.mlp.experts.linear_fc1.weight1 - module.decoder.layers.44.mlp.experts.linear_fc2.weight8 Params for bucket 7 (40894464 elements, 40894464 padded size): module.decoder.layers.44.mlp.experts.linear_fc2.weight3 module.decoder.layers.44.mlp.experts.linear_fc2.weight0 @@ -1130,14 +1130,24 @@ Params for bucket 7 (40894464 elements, 40894464 padded size): module.decoder.layers.44.mlp.experts.linear_fc1.weight8 module.decoder.layers.44.mlp.experts.linear_fc1.weight6 module.decoder.layers.44.mlp.experts.linear_fc2.weight1 - module.decoder.layers.44.mlp.experts.linear_fc2.weight2 module.decoder.layers.44.mlp.experts.linear_fc1.weight15 + module.decoder.layers.44.mlp.experts.linear_fc2.weight2 module.decoder.layers.44.mlp.experts.linear_fc1.weight13 module.decoder.layers.44.mlp.experts.linear_fc1.weight11 module.decoder.layers.44.mlp.experts.linear_fc1.weight9 module.decoder.layers.44.mlp.experts.linear_fc1.weight7 module.decoder.layers.44.mlp.experts.linear_fc1.weight5 Params for bucket 8 (40894464 elements, 40894464 padded size): + module.decoder.layers.44.mlp.experts.linear_fc1.weight4 + module.decoder.layers.43.mlp.experts.linear_fc2.weight10 + module.decoder.layers.43.mlp.experts.linear_fc2.weight2 + module.decoder.layers.44.mlp.experts.linear_fc1.weight2 + module.decoder.layers.43.mlp.experts.linear_fc2.weight15 + module.decoder.layers.43.mlp.experts.linear_fc2.weight7 + module.decoder.layers.43.mlp.experts.linear_fc2.weight12 + module.decoder.layers.43.mlp.experts.linear_fc2.weight4 + module.decoder.layers.44.mlp.experts.linear_fc1.weight1 + module.decoder.layers.43.mlp.experts.linear_fc2.weight9 module.decoder.layers.44.mlp.experts.linear_fc1.weight0 module.decoder.layers.43.mlp.experts.linear_fc2.weight14 module.decoder.layers.43.mlp.experts.linear_fc2.weight6 @@ -1149,16 +1159,6 @@ Params for bucket 8 (40894464 elements, 40894464 padded size): module.decoder.layers.43.mlp.experts.linear_fc2.weight1 module.decoder.layers.43.mlp.experts.linear_fc2.weight13 module.decoder.layers.43.mlp.experts.linear_fc2.weight5 - module.decoder.layers.44.mlp.experts.linear_fc1.weight4 - module.decoder.layers.43.mlp.experts.linear_fc2.weight10 - module.decoder.layers.43.mlp.experts.linear_fc2.weight2 - module.decoder.layers.44.mlp.experts.linear_fc1.weight2 - module.decoder.layers.43.mlp.experts.linear_fc2.weight15 - module.decoder.layers.43.mlp.experts.linear_fc2.weight7 - module.decoder.layers.43.mlp.experts.linear_fc2.weight12 - module.decoder.layers.43.mlp.experts.linear_fc2.weight4 - module.decoder.layers.44.mlp.experts.linear_fc1.weight1 - module.decoder.layers.43.mlp.experts.linear_fc2.weight9 Params for bucket 9 (40894464 elements, 40894464 padded size): module.decoder.layers.43.mlp.experts.linear_fc1.weight15 module.decoder.layers.43.mlp.experts.linear_fc1.weight13 @@ -1166,14 +1166,23 @@ Params for bucket 9 (40894464 elements, 40894464 padded size): module.decoder.layers.43.mlp.experts.linear_fc1.weight9 module.decoder.layers.43.mlp.experts.linear_fc1.weight7 module.decoder.layers.43.mlp.experts.linear_fc1.weight5 - module.decoder.layers.43.mlp.experts.linear_fc1.weight14 module.decoder.layers.43.mlp.experts.linear_fc1.weight12 + module.decoder.layers.43.mlp.experts.linear_fc1.weight14 module.decoder.layers.43.mlp.experts.linear_fc1.weight10 module.decoder.layers.43.mlp.experts.linear_fc1.weight8 module.decoder.layers.43.mlp.experts.linear_fc1.weight6 module.decoder.layers.43.mlp.experts.linear_fc1.weight4 module.decoder.layers.43.mlp.experts.linear_fc1.weight3 Params for bucket 10 (40894464 elements, 40894464 padded size): + module.decoder.layers.43.mlp.experts.linear_fc1.weight2 + module.decoder.layers.42.mlp.experts.linear_fc2.weight15 + module.decoder.layers.42.mlp.experts.linear_fc2.weight7 + module.decoder.layers.42.mlp.experts.linear_fc2.weight12 + module.decoder.layers.42.mlp.experts.linear_fc2.weight4 + module.decoder.layers.42.mlp.experts.linear_fc1.weight15 + module.decoder.layers.43.mlp.experts.linear_fc1.weight1 + module.decoder.layers.42.mlp.experts.linear_fc2.weight9 + module.decoder.layers.43.mlp.experts.linear_fc1.weight0 module.decoder.layers.42.mlp.experts.linear_fc2.weight14 module.decoder.layers.42.mlp.experts.linear_fc2.weight6 module.decoder.layers.42.mlp.experts.linear_fc2.weight11 @@ -1184,18 +1193,15 @@ Params for bucket 10 (40894464 elements, 40894464 padded size): module.decoder.layers.42.mlp.experts.linear_fc2.weight1 module.decoder.layers.42.mlp.experts.linear_fc2.weight13 module.decoder.layers.42.mlp.experts.linear_fc2.weight5 - module.decoder.layers.43.mlp.experts.linear_fc1.weight2 module.decoder.layers.42.mlp.experts.linear_fc2.weight10 module.decoder.layers.42.mlp.experts.linear_fc2.weight2 - module.decoder.layers.42.mlp.experts.linear_fc2.weight15 - module.decoder.layers.42.mlp.experts.linear_fc2.weight7 - module.decoder.layers.43.mlp.experts.linear_fc1.weight1 - module.decoder.layers.42.mlp.experts.linear_fc2.weight12 - module.decoder.layers.42.mlp.experts.linear_fc2.weight4 - module.decoder.layers.42.mlp.experts.linear_fc1.weight15 - module.decoder.layers.43.mlp.experts.linear_fc1.weight0 - module.decoder.layers.42.mlp.experts.linear_fc2.weight9 Params for bucket 11 (40894464 elements, 40894464 padded size): + module.decoder.layers.42.mlp.experts.linear_fc1.weight12 + module.decoder.layers.42.mlp.experts.linear_fc1.weight10 + module.decoder.layers.42.mlp.experts.linear_fc1.weight8 + module.decoder.layers.42.mlp.experts.linear_fc1.weight6 + module.decoder.layers.42.mlp.experts.linear_fc1.weight4 + module.decoder.layers.42.mlp.experts.linear_fc1.weight3 module.decoder.layers.42.mlp.experts.linear_fc1.weight13 module.decoder.layers.42.mlp.experts.linear_fc1.weight9 module.decoder.layers.42.mlp.experts.linear_fc1.weight11 @@ -1203,99 +1209,93 @@ Params for bucket 11 (40894464 elements, 40894464 padded size): module.decoder.layers.42.mlp.experts.linear_fc1.weight5 module.decoder.layers.42.mlp.experts.linear_fc1.weight2 module.decoder.layers.42.mlp.experts.linear_fc1.weight1 - module.decoder.layers.42.mlp.experts.linear_fc1.weight10 - module.decoder.layers.42.mlp.experts.linear_fc1.weight12 - module.decoder.layers.42.mlp.experts.linear_fc1.weight8 - module.decoder.layers.42.mlp.experts.linear_fc1.weight6 - module.decoder.layers.42.mlp.experts.linear_fc1.weight4 - module.decoder.layers.42.mlp.experts.linear_fc1.weight3 Params for bucket 12 (40894464 elements, 40894464 padded size): module.decoder.layers.41.mlp.experts.linear_fc2.weight9 module.decoder.layers.41.mlp.experts.linear_fc2.weight0 - module.decoder.layers.41.mlp.experts.linear_fc1.weight15 module.decoder.layers.41.mlp.experts.linear_fc2.weight14 module.decoder.layers.41.mlp.experts.linear_fc2.weight6 - module.decoder.layers.41.mlp.experts.linear_fc1.weight12 + module.decoder.layers.41.mlp.experts.linear_fc1.weight13 module.decoder.layers.41.mlp.experts.linear_fc2.weight11 module.decoder.layers.41.mlp.experts.linear_fc2.weight3 - module.decoder.layers.42.mlp.experts.linear_fc1.weight0 module.decoder.layers.41.mlp.experts.linear_fc2.weight8 - module.decoder.layers.41.mlp.experts.linear_fc2.weight2 - module.decoder.layers.41.mlp.experts.linear_fc1.weight14 + module.decoder.layers.41.mlp.experts.linear_fc1.weight15 module.decoder.layers.41.mlp.experts.linear_fc2.weight13 module.decoder.layers.41.mlp.experts.linear_fc2.weight5 + module.decoder.layers.41.mlp.experts.linear_fc1.weight12 module.decoder.layers.41.mlp.experts.linear_fc2.weight10 module.decoder.layers.41.mlp.experts.linear_fc2.weight1 + module.decoder.layers.42.mlp.experts.linear_fc1.weight0 module.decoder.layers.41.mlp.experts.linear_fc2.weight15 module.decoder.layers.41.mlp.experts.linear_fc2.weight7 - module.decoder.layers.41.mlp.experts.linear_fc1.weight13 + module.decoder.layers.41.mlp.experts.linear_fc2.weight2 + module.decoder.layers.41.mlp.experts.linear_fc1.weight14 module.decoder.layers.41.mlp.experts.linear_fc2.weight12 module.decoder.layers.41.mlp.experts.linear_fc2.weight4 Params for bucket 13 (40894464 elements, 40894464 padded size): - module.decoder.layers.41.mlp.experts.linear_fc1.weight11 - module.decoder.layers.41.mlp.experts.linear_fc1.weight9 - module.decoder.layers.41.mlp.experts.linear_fc1.weight7 - module.decoder.layers.41.mlp.experts.linear_fc1.weight5 - module.decoder.layers.41.mlp.experts.linear_fc1.weight2 - module.decoder.layers.41.mlp.experts.linear_fc1.weight1 - module.decoder.layers.40.mlp.experts.linear_fc2.weight15 - module.decoder.layers.41.mlp.experts.linear_fc1.weight10 module.decoder.layers.41.mlp.experts.linear_fc1.weight8 + module.decoder.layers.41.mlp.experts.linear_fc1.weight10 module.decoder.layers.41.mlp.experts.linear_fc1.weight6 module.decoder.layers.41.mlp.experts.linear_fc1.weight4 module.decoder.layers.41.mlp.experts.linear_fc1.weight3 module.decoder.layers.41.mlp.experts.linear_fc1.weight0 + module.decoder.layers.40.mlp.experts.linear_fc2.weight15 + module.decoder.layers.41.mlp.experts.linear_fc1.weight11 + module.decoder.layers.41.mlp.experts.linear_fc1.weight9 + module.decoder.layers.41.mlp.experts.linear_fc1.weight7 + module.decoder.layers.41.mlp.experts.linear_fc1.weight5 + module.decoder.layers.41.mlp.experts.linear_fc1.weight2 + module.decoder.layers.41.mlp.experts.linear_fc1.weight1 module.decoder.layers.40.mlp.experts.linear_fc2.weight14 Params for bucket 14 (40894464 elements, 40894464 padded size): - module.decoder.layers.40.mlp.experts.linear_fc2.weight13 - module.decoder.layers.40.mlp.experts.linear_fc2.weight5 - module.decoder.layers.40.mlp.experts.linear_fc1.weight12 - module.decoder.layers.40.mlp.experts.linear_fc2.weight10 - module.decoder.layers.40.mlp.experts.linear_fc2.weight2 module.decoder.layers.40.mlp.experts.linear_fc2.weight7 - module.decoder.layers.40.mlp.experts.linear_fc1.weight14 + module.decoder.layers.40.mlp.experts.linear_fc1.weight10 module.decoder.layers.40.mlp.experts.linear_fc2.weight12 module.decoder.layers.40.mlp.experts.linear_fc2.weight4 - module.decoder.layers.40.mlp.experts.linear_fc1.weight11 + module.decoder.layers.40.mlp.experts.linear_fc1.weight13 module.decoder.layers.40.mlp.experts.linear_fc2.weight9 + module.decoder.layers.40.mlp.experts.linear_fc1.weight12 module.decoder.layers.40.mlp.experts.linear_fc2.weight6 - module.decoder.layers.40.mlp.experts.linear_fc1.weight13 module.decoder.layers.40.mlp.experts.linear_fc2.weight11 module.decoder.layers.40.mlp.experts.linear_fc2.weight3 module.decoder.layers.40.mlp.experts.linear_fc2.weight0 - module.decoder.layers.40.mlp.experts.linear_fc1.weight10 + module.decoder.layers.40.mlp.experts.linear_fc1.weight15 module.decoder.layers.40.mlp.experts.linear_fc2.weight8 module.decoder.layers.40.mlp.experts.linear_fc2.weight1 - module.decoder.layers.40.mlp.experts.linear_fc1.weight15 + module.decoder.layers.40.mlp.experts.linear_fc1.weight11 + module.decoder.layers.40.mlp.experts.linear_fc2.weight13 + module.decoder.layers.40.mlp.experts.linear_fc2.weight5 + module.decoder.layers.40.mlp.experts.linear_fc2.weight10 + module.decoder.layers.40.mlp.experts.linear_fc2.weight2 + module.decoder.layers.40.mlp.experts.linear_fc1.weight14 Params for bucket 15 (40894464 elements, 40894464 padded size): - module.decoder.layers.40.mlp.experts.linear_fc1.weight8 module.decoder.layers.40.mlp.experts.linear_fc1.weight6 - module.decoder.layers.40.mlp.experts.linear_fc1.weight5 + module.decoder.layers.40.mlp.experts.linear_fc1.weight8 module.decoder.layers.40.mlp.experts.linear_fc1.weight4 + module.decoder.layers.40.mlp.experts.linear_fc1.weight3 module.decoder.layers.40.mlp.experts.linear_fc1.weight0 module.decoder.layers.39.mlp.experts.linear_fc2.weight15 module.decoder.layers.39.mlp.experts.linear_fc2.weight13 - module.decoder.layers.39.mlp.experts.linear_fc2.weight11 + module.decoder.layers.39.mlp.experts.linear_fc2.weight12 module.decoder.layers.40.mlp.experts.linear_fc1.weight9 module.decoder.layers.40.mlp.experts.linear_fc1.weight7 - module.decoder.layers.40.mlp.experts.linear_fc1.weight3 + module.decoder.layers.40.mlp.experts.linear_fc1.weight5 module.decoder.layers.40.mlp.experts.linear_fc1.weight2 module.decoder.layers.40.mlp.experts.linear_fc1.weight1 module.decoder.layers.39.mlp.experts.linear_fc2.weight14 - module.decoder.layers.39.mlp.experts.linear_fc2.weight12 + module.decoder.layers.39.mlp.experts.linear_fc2.weight11 module.decoder.layers.39.mlp.experts.linear_fc2.weight10 Params for bucket 16 (40894464 elements, 40894464 padded size): module.decoder.layers.39.mlp.experts.linear_fc2.weight9 module.decoder.layers.39.mlp.experts.linear_fc2.weight7 - module.decoder.layers.39.mlp.experts.linear_fc2.weight6 + module.decoder.layers.39.mlp.experts.linear_fc2.weight5 module.decoder.layers.39.mlp.experts.linear_fc2.weight3 module.decoder.layers.39.mlp.experts.linear_fc2.weight0 module.decoder.layers.39.mlp.experts.linear_fc1.weight15 module.decoder.layers.39.mlp.experts.linear_fc1.weight13 module.decoder.layers.39.mlp.experts.linear_fc1.weight11 module.decoder.layers.39.mlp.experts.linear_fc1.weight9 + module.decoder.layers.39.mlp.experts.linear_fc2.weight6 module.decoder.layers.39.mlp.experts.linear_fc2.weight8 - module.decoder.layers.39.mlp.experts.linear_fc2.weight5 module.decoder.layers.39.mlp.experts.linear_fc2.weight4 module.decoder.layers.39.mlp.experts.linear_fc2.weight2 module.decoder.layers.39.mlp.experts.linear_fc2.weight1 @@ -1323,45 +1323,51 @@ Params for bucket 17 (40894464 elements, 40894464 padded size): module.decoder.layers.38.mlp.experts.linear_fc2.weight9 module.decoder.layers.38.mlp.experts.linear_fc2.weight7 Params for bucket 18 (40894464 elements, 40894464 padded size): - module.decoder.layers.38.mlp.experts.linear_fc2.weight4 module.decoder.layers.38.mlp.experts.linear_fc2.weight2 module.decoder.layers.38.mlp.experts.linear_fc2.weight1 - module.decoder.layers.38.mlp.experts.linear_fc1.weight15 - module.decoder.layers.38.mlp.experts.linear_fc1.weight13 - module.decoder.layers.38.mlp.experts.linear_fc1.weight11 - module.decoder.layers.38.mlp.experts.linear_fc1.weight9 - module.decoder.layers.38.mlp.experts.linear_fc1.weight7 - module.decoder.layers.38.mlp.experts.linear_fc2.weight5 - module.decoder.layers.38.mlp.experts.linear_fc2.weight3 + module.decoder.layers.38.mlp.experts.linear_fc2.weight4 module.decoder.layers.38.mlp.experts.linear_fc2.weight0 module.decoder.layers.38.mlp.experts.linear_fc1.weight14 module.decoder.layers.38.mlp.experts.linear_fc1.weight12 module.decoder.layers.38.mlp.experts.linear_fc1.weight10 module.decoder.layers.38.mlp.experts.linear_fc1.weight8 module.decoder.layers.38.mlp.experts.linear_fc1.weight6 + module.decoder.layers.38.mlp.experts.linear_fc2.weight5 + module.decoder.layers.38.mlp.experts.linear_fc2.weight3 + module.decoder.layers.38.mlp.experts.linear_fc1.weight15 + module.decoder.layers.38.mlp.experts.linear_fc1.weight13 + module.decoder.layers.38.mlp.experts.linear_fc1.weight11 + module.decoder.layers.38.mlp.experts.linear_fc1.weight9 + module.decoder.layers.38.mlp.experts.linear_fc1.weight7 Params for bucket 19 (40894464 elements, 40894464 padded size): + module.decoder.layers.38.mlp.experts.linear_fc1.weight0 + module.decoder.layers.37.mlp.experts.linear_fc2.weight8 module.decoder.layers.38.mlp.experts.linear_fc1.weight5 + module.decoder.layers.37.mlp.experts.linear_fc2.weight13 + module.decoder.layers.37.mlp.experts.linear_fc2.weight5 + module.decoder.layers.38.mlp.experts.linear_fc1.weight3 + module.decoder.layers.37.mlp.experts.linear_fc2.weight10 + module.decoder.layers.37.mlp.experts.linear_fc2.weight2 + module.decoder.layers.37.mlp.experts.linear_fc2.weight15 + module.decoder.layers.37.mlp.experts.linear_fc2.weight7 + module.decoder.layers.38.mlp.experts.linear_fc1.weight4 module.decoder.layers.37.mlp.experts.linear_fc2.weight12 module.decoder.layers.37.mlp.experts.linear_fc2.weight4 - module.decoder.layers.38.mlp.experts.linear_fc1.weight3 + module.decoder.layers.38.mlp.experts.linear_fc1.weight2 module.decoder.layers.37.mlp.experts.linear_fc2.weight9 module.decoder.layers.37.mlp.experts.linear_fc2.weight14 module.decoder.layers.37.mlp.experts.linear_fc2.weight6 - module.decoder.layers.38.mlp.experts.linear_fc1.weight4 + module.decoder.layers.38.mlp.experts.linear_fc1.weight1 module.decoder.layers.37.mlp.experts.linear_fc2.weight11 module.decoder.layers.37.mlp.experts.linear_fc2.weight3 - module.decoder.layers.38.mlp.experts.linear_fc1.weight2 - module.decoder.layers.37.mlp.experts.linear_fc2.weight8 - module.decoder.layers.37.mlp.experts.linear_fc2.weight13 - module.decoder.layers.37.mlp.experts.linear_fc2.weight5 - module.decoder.layers.38.mlp.experts.linear_fc1.weight1 - module.decoder.layers.37.mlp.experts.linear_fc2.weight10 - module.decoder.layers.37.mlp.experts.linear_fc2.weight2 - module.decoder.layers.38.mlp.experts.linear_fc1.weight0 - module.decoder.layers.37.mlp.experts.linear_fc2.weight15 - module.decoder.layers.37.mlp.experts.linear_fc2.weight7 Params for bucket 20 (40894464 elements, 40894464 padded size): module.decoder.layers.37.mlp.experts.linear_fc2.weight1 + module.decoder.layers.37.mlp.experts.linear_fc1.weight13 + module.decoder.layers.37.mlp.experts.linear_fc1.weight15 + module.decoder.layers.37.mlp.experts.linear_fc1.weight11 + module.decoder.layers.37.mlp.experts.linear_fc1.weight9 + module.decoder.layers.37.mlp.experts.linear_fc1.weight7 + module.decoder.layers.37.mlp.experts.linear_fc1.weight5 module.decoder.layers.37.mlp.experts.linear_fc1.weight14 module.decoder.layers.37.mlp.experts.linear_fc2.weight0 module.decoder.layers.37.mlp.experts.linear_fc1.weight12 @@ -1369,60 +1375,44 @@ Params for bucket 20 (40894464 elements, 40894464 padded size): module.decoder.layers.37.mlp.experts.linear_fc1.weight8 module.decoder.layers.37.mlp.experts.linear_fc1.weight6 module.decoder.layers.37.mlp.experts.linear_fc1.weight4 - module.decoder.layers.37.mlp.experts.linear_fc1.weight15 - module.decoder.layers.37.mlp.experts.linear_fc1.weight13 - module.decoder.layers.37.mlp.experts.linear_fc1.weight11 - module.decoder.layers.37.mlp.experts.linear_fc1.weight9 - module.decoder.layers.37.mlp.experts.linear_fc1.weight7 - module.decoder.layers.37.mlp.experts.linear_fc1.weight5 Params for bucket 21 (40894464 elements, 40894464 padded size): - module.decoder.layers.36.mlp.experts.linear_fc2.weight13 - module.decoder.layers.36.mlp.experts.linear_fc2.weight5 module.decoder.layers.37.mlp.experts.linear_fc1.weight1 - module.decoder.layers.36.mlp.experts.linear_fc2.weight10 - module.decoder.layers.36.mlp.experts.linear_fc2.weight1 - module.decoder.layers.36.mlp.experts.linear_fc2.weight15 - module.decoder.layers.37.mlp.experts.linear_fc1.weight0 - module.decoder.layers.36.mlp.experts.linear_fc2.weight7 - module.decoder.layers.36.mlp.experts.linear_fc2.weight12 - module.decoder.layers.36.mlp.experts.linear_fc2.weight4 - module.decoder.layers.37.mlp.experts.linear_fc1.weight3 module.decoder.layers.36.mlp.experts.linear_fc2.weight9 - module.decoder.layers.36.mlp.experts.linear_fc2.weight0 - module.decoder.layers.36.mlp.experts.linear_fc1.weight15 + module.decoder.layers.37.mlp.experts.linear_fc1.weight0 module.decoder.layers.36.mlp.experts.linear_fc2.weight14 module.decoder.layers.36.mlp.experts.linear_fc2.weight6 module.decoder.layers.36.mlp.experts.linear_fc2.weight11 module.decoder.layers.36.mlp.experts.linear_fc2.weight3 - module.decoder.layers.37.mlp.experts.linear_fc1.weight2 + module.decoder.layers.36.mlp.experts.linear_fc2.weight0 + module.decoder.layers.37.mlp.experts.linear_fc1.weight3 module.decoder.layers.36.mlp.experts.linear_fc2.weight8 + module.decoder.layers.36.mlp.experts.linear_fc2.weight1 + module.decoder.layers.36.mlp.experts.linear_fc1.weight15 + module.decoder.layers.36.mlp.experts.linear_fc2.weight13 + module.decoder.layers.36.mlp.experts.linear_fc2.weight5 + module.decoder.layers.36.mlp.experts.linear_fc2.weight10 module.decoder.layers.36.mlp.experts.linear_fc2.weight2 + module.decoder.layers.36.mlp.experts.linear_fc2.weight15 + module.decoder.layers.37.mlp.experts.linear_fc1.weight2 + module.decoder.layers.36.mlp.experts.linear_fc2.weight7 + module.decoder.layers.36.mlp.experts.linear_fc2.weight12 + module.decoder.layers.36.mlp.experts.linear_fc2.weight4 Params for bucket 22 (40894464 elements, 40894464 padded size): - module.decoder.layers.36.mlp.experts.linear_fc1.weight13 - module.decoder.layers.36.mlp.experts.linear_fc1.weight11 - module.decoder.layers.36.mlp.experts.linear_fc1.weight9 - module.decoder.layers.36.mlp.experts.linear_fc1.weight7 - module.decoder.layers.36.mlp.experts.linear_fc1.weight5 - module.decoder.layers.36.mlp.experts.linear_fc1.weight2 module.decoder.layers.36.mlp.experts.linear_fc1.weight14 - module.decoder.layers.36.mlp.experts.linear_fc1.weight10 module.decoder.layers.36.mlp.experts.linear_fc1.weight12 + module.decoder.layers.36.mlp.experts.linear_fc1.weight10 module.decoder.layers.36.mlp.experts.linear_fc1.weight8 module.decoder.layers.36.mlp.experts.linear_fc1.weight6 module.decoder.layers.36.mlp.experts.linear_fc1.weight4 module.decoder.layers.36.mlp.experts.linear_fc1.weight3 + module.decoder.layers.36.mlp.experts.linear_fc1.weight13 + module.decoder.layers.36.mlp.experts.linear_fc1.weight11 + module.decoder.layers.36.mlp.experts.linear_fc1.weight9 + module.decoder.layers.36.mlp.experts.linear_fc1.weight7 + module.decoder.layers.36.mlp.experts.linear_fc1.weight5 + module.decoder.layers.36.mlp.experts.linear_fc1.weight2 Params for bucket 23 (40894464 elements, 40894464 padded size): - module.decoder.layers.36.mlp.experts.linear_fc1.weight1 - module.decoder.layers.35.mlp.experts.linear_fc2.weight10 - module.decoder.layers.35.mlp.experts.linear_fc2.weight2 - module.decoder.layers.35.mlp.experts.linear_fc1.weight13 module.decoder.layers.36.mlp.experts.linear_fc1.weight0 - module.decoder.layers.35.mlp.experts.linear_fc2.weight15 - module.decoder.layers.35.mlp.experts.linear_fc2.weight7 - module.decoder.layers.35.mlp.experts.linear_fc2.weight12 - module.decoder.layers.35.mlp.experts.linear_fc2.weight4 - module.decoder.layers.35.mlp.experts.linear_fc1.weight15 - module.decoder.layers.35.mlp.experts.linear_fc2.weight9 module.decoder.layers.35.mlp.experts.linear_fc2.weight14 module.decoder.layers.35.mlp.experts.linear_fc2.weight6 module.decoder.layers.35.mlp.experts.linear_fc2.weight11 @@ -1433,6 +1423,16 @@ Params for bucket 23 (40894464 elements, 40894464 padded size): module.decoder.layers.35.mlp.experts.linear_fc2.weight1 module.decoder.layers.35.mlp.experts.linear_fc2.weight13 module.decoder.layers.35.mlp.experts.linear_fc2.weight5 + module.decoder.layers.35.mlp.experts.linear_fc2.weight10 + module.decoder.layers.35.mlp.experts.linear_fc2.weight2 + module.decoder.layers.35.mlp.experts.linear_fc1.weight13 + module.decoder.layers.35.mlp.experts.linear_fc2.weight15 + module.decoder.layers.35.mlp.experts.linear_fc2.weight7 + module.decoder.layers.35.mlp.experts.linear_fc2.weight12 + module.decoder.layers.35.mlp.experts.linear_fc2.weight4 + module.decoder.layers.35.mlp.experts.linear_fc1.weight15 + module.decoder.layers.36.mlp.experts.linear_fc1.weight1 + module.decoder.layers.35.mlp.experts.linear_fc2.weight9 Params for bucket 24 (40894464 elements, 40894464 padded size): module.decoder.layers.35.mlp.experts.linear_fc1.weight11 module.decoder.layers.35.mlp.experts.linear_fc1.weight9 @@ -1441,51 +1441,55 @@ Params for bucket 24 (40894464 elements, 40894464 padded size): module.decoder.layers.35.mlp.experts.linear_fc1.weight2 module.decoder.layers.35.mlp.experts.linear_fc1.weight1 module.decoder.layers.35.mlp.experts.linear_fc1.weight12 - module.decoder.layers.35.mlp.experts.linear_fc1.weight10 module.decoder.layers.35.mlp.experts.linear_fc1.weight8 + module.decoder.layers.35.mlp.experts.linear_fc1.weight10 module.decoder.layers.35.mlp.experts.linear_fc1.weight6 module.decoder.layers.35.mlp.experts.linear_fc1.weight4 module.decoder.layers.35.mlp.experts.linear_fc1.weight3 module.decoder.layers.35.mlp.experts.linear_fc1.weight0 Params for bucket 25 (40894464 elements, 40894464 padded size): - module.decoder.layers.34.mlp.experts.linear_fc2.weight12 - module.decoder.layers.34.mlp.experts.linear_fc2.weight4 - module.decoder.layers.34.mlp.experts.linear_fc2.weight9 - module.decoder.layers.34.mlp.experts.linear_fc2.weight0 - module.decoder.layers.34.mlp.experts.linear_fc1.weight15 - module.decoder.layers.34.mlp.experts.linear_fc2.weight14 - module.decoder.layers.34.mlp.experts.linear_fc2.weight6 - module.decoder.layers.34.mlp.experts.linear_fc1.weight12 - module.decoder.layers.34.mlp.experts.linear_fc2.weight11 - module.decoder.layers.34.mlp.experts.linear_fc2.weight3 module.decoder.layers.34.mlp.experts.linear_fc2.weight8 - module.decoder.layers.34.mlp.experts.linear_fc2.weight2 - module.decoder.layers.34.mlp.experts.linear_fc1.weight14 + module.decoder.layers.34.mlp.experts.linear_fc1.weight15 module.decoder.layers.34.mlp.experts.linear_fc2.weight13 module.decoder.layers.34.mlp.experts.linear_fc2.weight5 - module.decoder.layers.34.mlp.experts.linear_fc1.weight11 + module.decoder.layers.34.mlp.experts.linear_fc1.weight12 module.decoder.layers.34.mlp.experts.linear_fc2.weight10 module.decoder.layers.34.mlp.experts.linear_fc2.weight1 module.decoder.layers.34.mlp.experts.linear_fc2.weight15 module.decoder.layers.34.mlp.experts.linear_fc2.weight7 + module.decoder.layers.34.mlp.experts.linear_fc2.weight2 + module.decoder.layers.34.mlp.experts.linear_fc1.weight14 + module.decoder.layers.34.mlp.experts.linear_fc2.weight12 + module.decoder.layers.34.mlp.experts.linear_fc2.weight4 + module.decoder.layers.34.mlp.experts.linear_fc1.weight11 + module.decoder.layers.34.mlp.experts.linear_fc2.weight9 + module.decoder.layers.34.mlp.experts.linear_fc2.weight0 + module.decoder.layers.34.mlp.experts.linear_fc2.weight14 + module.decoder.layers.34.mlp.experts.linear_fc2.weight6 module.decoder.layers.34.mlp.experts.linear_fc1.weight13 + module.decoder.layers.34.mlp.experts.linear_fc2.weight11 + module.decoder.layers.34.mlp.experts.linear_fc2.weight3 Params for bucket 26 (40894464 elements, 40894464 padded size): + module.decoder.layers.34.mlp.experts.linear_fc1.weight7 + module.decoder.layers.34.mlp.experts.linear_fc1.weight9 + module.decoder.layers.34.mlp.experts.linear_fc1.weight5 + module.decoder.layers.34.mlp.experts.linear_fc1.weight2 + module.decoder.layers.34.mlp.experts.linear_fc1.weight1 + module.decoder.layers.33.mlp.experts.linear_fc2.weight15 + module.decoder.layers.33.mlp.experts.linear_fc2.weight13 module.decoder.layers.34.mlp.experts.linear_fc1.weight10 - module.decoder.layers.34.mlp.experts.linear_fc1.weight6 module.decoder.layers.34.mlp.experts.linear_fc1.weight8 + module.decoder.layers.34.mlp.experts.linear_fc1.weight6 module.decoder.layers.34.mlp.experts.linear_fc1.weight4 module.decoder.layers.34.mlp.experts.linear_fc1.weight3 module.decoder.layers.34.mlp.experts.linear_fc1.weight0 - module.decoder.layers.33.mlp.experts.linear_fc2.weight15 - module.decoder.layers.33.mlp.experts.linear_fc2.weight13 - module.decoder.layers.34.mlp.experts.linear_fc1.weight9 - module.decoder.layers.34.mlp.experts.linear_fc1.weight7 - module.decoder.layers.34.mlp.experts.linear_fc1.weight5 - module.decoder.layers.34.mlp.experts.linear_fc1.weight2 - module.decoder.layers.34.mlp.experts.linear_fc1.weight1 module.decoder.layers.33.mlp.experts.linear_fc2.weight14 module.decoder.layers.33.mlp.experts.linear_fc2.weight12 Params for bucket 27 (40894464 elements, 40894464 padded size): + module.decoder.layers.33.mlp.experts.linear_fc2.weight7 + module.decoder.layers.33.mlp.experts.linear_fc1.weight14 + module.decoder.layers.33.mlp.experts.linear_fc2.weight4 + module.decoder.layers.33.mlp.experts.linear_fc1.weight11 module.decoder.layers.33.mlp.experts.linear_fc2.weight9 module.decoder.layers.33.mlp.experts.linear_fc2.weight6 module.decoder.layers.33.mlp.experts.linear_fc1.weight13 @@ -1501,14 +1505,10 @@ Params for bucket 27 (40894464 elements, 40894464 padded size): module.decoder.layers.33.mlp.experts.linear_fc2.weight10 module.decoder.layers.33.mlp.experts.linear_fc2.weight2 module.decoder.layers.33.mlp.experts.linear_fc1.weight9 - module.decoder.layers.33.mlp.experts.linear_fc2.weight7 - module.decoder.layers.33.mlp.experts.linear_fc1.weight14 - module.decoder.layers.33.mlp.experts.linear_fc2.weight4 - module.decoder.layers.33.mlp.experts.linear_fc1.weight11 Params for bucket 28 (40894464 elements, 40894464 padded size): module.decoder.layers.33.mlp.experts.linear_fc1.weight8 - module.decoder.layers.33.mlp.experts.linear_fc1.weight6 module.decoder.layers.33.mlp.experts.linear_fc1.weight4 + module.decoder.layers.33.mlp.experts.linear_fc1.weight6 module.decoder.layers.33.mlp.experts.linear_fc1.weight3 module.decoder.layers.33.mlp.experts.linear_fc1.weight0 module.decoder.layers.32.mlp.experts.linear_fc2.weight15 @@ -1525,9 +1525,8 @@ Params for bucket 28 (40894464 elements, 40894464 padded size): module.decoder.layers.32.mlp.experts.linear_fc2.weight8 Params for bucket 29 (40894464 elements, 40894464 padded size): module.decoder.layers.32.mlp.experts.linear_fc2.weight7 - module.decoder.layers.32.mlp.experts.linear_fc2.weight5 module.decoder.layers.32.mlp.experts.linear_fc2.weight3 - module.decoder.layers.32.mlp.experts.linear_fc2.weight0 + module.decoder.layers.32.mlp.experts.linear_fc2.weight5 module.decoder.layers.32.mlp.experts.linear_fc1.weight14 module.decoder.layers.32.mlp.experts.linear_fc1.weight12 module.decoder.layers.32.mlp.experts.linear_fc1.weight10 @@ -1536,73 +1535,74 @@ Params for bucket 29 (40894464 elements, 40894464 padded size): module.decoder.layers.32.mlp.experts.linear_fc2.weight6 module.decoder.layers.32.mlp.experts.linear_fc2.weight2 module.decoder.layers.32.mlp.experts.linear_fc2.weight1 + module.decoder.layers.32.mlp.experts.linear_fc2.weight0 module.decoder.layers.32.mlp.experts.linear_fc1.weight15 module.decoder.layers.32.mlp.experts.linear_fc1.weight13 module.decoder.layers.32.mlp.experts.linear_fc1.weight11 module.decoder.layers.32.mlp.experts.linear_fc1.weight9 module.decoder.layers.32.mlp.experts.linear_fc1.weight7 Params for bucket 30 (40894464 elements, 40894464 padded size): - module.decoder.layers.32.mlp.experts.linear_fc1.weight3 + module.decoder.layers.32.mlp.experts.linear_fc1.weight6 + module.decoder.layers.31.mlp.experts.linear_fc2.weight15 + module.decoder.layers.31.mlp.experts.linear_fc2.weight7 + module.decoder.layers.32.mlp.experts.linear_fc1.weight1 + module.decoder.layers.31.mlp.experts.linear_fc2.weight12 + module.decoder.layers.31.mlp.experts.linear_fc2.weight4 + module.decoder.layers.32.mlp.experts.linear_fc1.weight0 module.decoder.layers.31.mlp.experts.linear_fc2.weight9 + module.decoder.layers.32.mlp.experts.linear_fc1.weight5 module.decoder.layers.31.mlp.experts.linear_fc2.weight14 module.decoder.layers.31.mlp.experts.linear_fc2.weight6 - module.decoder.layers.32.mlp.experts.linear_fc1.weight4 + module.decoder.layers.32.mlp.experts.linear_fc1.weight3 module.decoder.layers.31.mlp.experts.linear_fc2.weight11 - module.decoder.layers.32.mlp.experts.linear_fc1.weight2 module.decoder.layers.31.mlp.experts.linear_fc2.weight8 - module.decoder.layers.32.mlp.experts.linear_fc1.weight6 + module.decoder.layers.32.mlp.experts.linear_fc1.weight4 module.decoder.layers.31.mlp.experts.linear_fc2.weight13 module.decoder.layers.31.mlp.experts.linear_fc2.weight5 - module.decoder.layers.32.mlp.experts.linear_fc1.weight1 + module.decoder.layers.32.mlp.experts.linear_fc1.weight2 module.decoder.layers.31.mlp.experts.linear_fc2.weight10 - module.decoder.layers.32.mlp.experts.linear_fc1.weight0 - module.decoder.layers.31.mlp.experts.linear_fc2.weight15 - module.decoder.layers.31.mlp.experts.linear_fc2.weight7 - module.decoder.layers.32.mlp.experts.linear_fc1.weight5 - module.decoder.layers.31.mlp.experts.linear_fc2.weight12 - module.decoder.layers.31.mlp.experts.linear_fc2.weight4 Params for bucket 31 (40894464 elements, 40894464 padded size): module.decoder.layers.31.mlp.experts.linear_fc2.weight3 - module.decoder.layers.31.mlp.experts.linear_fc1.weight14 - module.decoder.layers.31.mlp.experts.linear_fc1.weight12 - module.decoder.layers.31.mlp.experts.linear_fc1.weight10 - module.decoder.layers.31.mlp.experts.linear_fc1.weight8 - module.decoder.layers.31.mlp.experts.linear_fc1.weight6 - module.decoder.layers.31.mlp.experts.linear_fc2.weight2 - module.decoder.layers.31.mlp.experts.linear_fc2.weight0 - module.decoder.layers.31.mlp.experts.linear_fc2.weight1 module.decoder.layers.31.mlp.experts.linear_fc1.weight15 module.decoder.layers.31.mlp.experts.linear_fc1.weight13 module.decoder.layers.31.mlp.experts.linear_fc1.weight11 module.decoder.layers.31.mlp.experts.linear_fc1.weight9 module.decoder.layers.31.mlp.experts.linear_fc1.weight7 module.decoder.layers.31.mlp.experts.linear_fc1.weight5 + module.decoder.layers.31.mlp.experts.linear_fc2.weight0 + module.decoder.layers.31.mlp.experts.linear_fc2.weight2 + module.decoder.layers.31.mlp.experts.linear_fc2.weight1 + module.decoder.layers.31.mlp.experts.linear_fc1.weight14 + module.decoder.layers.31.mlp.experts.linear_fc1.weight12 + module.decoder.layers.31.mlp.experts.linear_fc1.weight10 + module.decoder.layers.31.mlp.experts.linear_fc1.weight8 + module.decoder.layers.31.mlp.experts.linear_fc1.weight6 Params for bucket 32 (40894464 elements, 40894464 padded size): + module.decoder.layers.31.mlp.experts.linear_fc1.weight1 + module.decoder.layers.30.mlp.experts.linear_fc2.weight10 + module.decoder.layers.30.mlp.experts.linear_fc2.weight2 + module.decoder.layers.31.mlp.experts.linear_fc1.weight0 + module.decoder.layers.30.mlp.experts.linear_fc2.weight15 + module.decoder.layers.30.mlp.experts.linear_fc2.weight7 + module.decoder.layers.30.mlp.experts.linear_fc2.weight12 + module.decoder.layers.30.mlp.experts.linear_fc2.weight4 + module.decoder.layers.31.mlp.experts.linear_fc1.weight3 + module.decoder.layers.30.mlp.experts.linear_fc2.weight9 module.decoder.layers.30.mlp.experts.linear_fc2.weight14 module.decoder.layers.30.mlp.experts.linear_fc2.weight6 - module.decoder.layers.31.mlp.experts.linear_fc1.weight1 + module.decoder.layers.31.mlp.experts.linear_fc1.weight4 module.decoder.layers.30.mlp.experts.linear_fc2.weight11 module.decoder.layers.30.mlp.experts.linear_fc2.weight3 - module.decoder.layers.30.mlp.experts.linear_fc2.weight0 - module.decoder.layers.31.mlp.experts.linear_fc1.weight0 + module.decoder.layers.31.mlp.experts.linear_fc1.weight2 module.decoder.layers.30.mlp.experts.linear_fc2.weight8 module.decoder.layers.30.mlp.experts.linear_fc2.weight1 module.decoder.layers.30.mlp.experts.linear_fc2.weight13 module.decoder.layers.30.mlp.experts.linear_fc2.weight5 - module.decoder.layers.31.mlp.experts.linear_fc1.weight3 - module.decoder.layers.30.mlp.experts.linear_fc2.weight10 - module.decoder.layers.30.mlp.experts.linear_fc2.weight2 - module.decoder.layers.30.mlp.experts.linear_fc2.weight15 - module.decoder.layers.30.mlp.experts.linear_fc2.weight7 - module.decoder.layers.31.mlp.experts.linear_fc1.weight4 - module.decoder.layers.30.mlp.experts.linear_fc2.weight12 - module.decoder.layers.30.mlp.experts.linear_fc2.weight4 - module.decoder.layers.31.mlp.experts.linear_fc1.weight2 - module.decoder.layers.30.mlp.experts.linear_fc2.weight9 + module.decoder.layers.30.mlp.experts.linear_fc2.weight0 Params for bucket 33 (40894464 elements, 40894464 padded size): module.decoder.layers.30.mlp.experts.linear_fc1.weight15 - module.decoder.layers.30.mlp.experts.linear_fc1.weight11 module.decoder.layers.30.mlp.experts.linear_fc1.weight13 + module.decoder.layers.30.mlp.experts.linear_fc1.weight11 module.decoder.layers.30.mlp.experts.linear_fc1.weight9 module.decoder.layers.30.mlp.experts.linear_fc1.weight7 module.decoder.layers.30.mlp.experts.linear_fc1.weight5 @@ -1614,108 +1614,100 @@ Params for bucket 33 (40894464 elements, 40894464 padded size): module.decoder.layers.30.mlp.experts.linear_fc1.weight4 module.decoder.layers.30.mlp.experts.linear_fc1.weight3 Params for bucket 34 (40894464 elements, 40894464 padded size): - module.decoder.layers.30.mlp.experts.linear_fc1.weight2 + module.decoder.layers.30.mlp.experts.linear_fc1.weight1 + module.decoder.layers.29.mlp.experts.linear_fc2.weight9 + module.decoder.layers.29.mlp.experts.linear_fc2.weight0 + module.decoder.layers.30.mlp.experts.linear_fc1.weight0 + module.decoder.layers.29.mlp.experts.linear_fc2.weight14 + module.decoder.layers.29.mlp.experts.linear_fc2.weight6 + module.decoder.layers.29.mlp.experts.linear_fc2.weight11 + module.decoder.layers.29.mlp.experts.linear_fc2.weight3 module.decoder.layers.29.mlp.experts.linear_fc2.weight8 - module.decoder.layers.29.mlp.experts.linear_fc1.weight14 + module.decoder.layers.29.mlp.experts.linear_fc1.weight15 module.decoder.layers.29.mlp.experts.linear_fc2.weight13 module.decoder.layers.29.mlp.experts.linear_fc2.weight5 - module.decoder.layers.30.mlp.experts.linear_fc1.weight1 module.decoder.layers.29.mlp.experts.linear_fc2.weight10 module.decoder.layers.29.mlp.experts.linear_fc2.weight1 - module.decoder.layers.30.mlp.experts.linear_fc1.weight0 + module.decoder.layers.30.mlp.experts.linear_fc1.weight2 module.decoder.layers.29.mlp.experts.linear_fc2.weight15 module.decoder.layers.29.mlp.experts.linear_fc2.weight7 module.decoder.layers.29.mlp.experts.linear_fc2.weight2 + module.decoder.layers.29.mlp.experts.linear_fc1.weight14 module.decoder.layers.29.mlp.experts.linear_fc2.weight12 module.decoder.layers.29.mlp.experts.linear_fc2.weight4 - module.decoder.layers.29.mlp.experts.linear_fc2.weight9 - module.decoder.layers.29.mlp.experts.linear_fc2.weight0 - module.decoder.layers.29.mlp.experts.linear_fc1.weight15 - module.decoder.layers.29.mlp.experts.linear_fc2.weight14 - module.decoder.layers.29.mlp.experts.linear_fc2.weight6 - module.decoder.layers.29.mlp.experts.linear_fc2.weight11 - module.decoder.layers.29.mlp.experts.linear_fc2.weight3 Params for bucket 35 (40894464 elements, 40894464 padded size): - module.decoder.layers.29.mlp.experts.linear_fc1.weight10 module.decoder.layers.29.mlp.experts.linear_fc1.weight12 + module.decoder.layers.29.mlp.experts.linear_fc1.weight10 module.decoder.layers.29.mlp.experts.linear_fc1.weight8 module.decoder.layers.29.mlp.experts.linear_fc1.weight6 module.decoder.layers.29.mlp.experts.linear_fc1.weight4 module.decoder.layers.29.mlp.experts.linear_fc1.weight3 module.decoder.layers.29.mlp.experts.linear_fc1.weight13 - module.decoder.layers.29.mlp.experts.linear_fc1.weight11 module.decoder.layers.29.mlp.experts.linear_fc1.weight9 + module.decoder.layers.29.mlp.experts.linear_fc1.weight11 module.decoder.layers.29.mlp.experts.linear_fc1.weight7 module.decoder.layers.29.mlp.experts.linear_fc1.weight5 module.decoder.layers.29.mlp.experts.linear_fc1.weight2 module.decoder.layers.29.mlp.experts.linear_fc1.weight1 Params for bucket 36 (40894464 elements, 40894464 padded size): - module.decoder.layers.28.mlp.experts.linear_fc2.weight14 - module.decoder.layers.28.mlp.experts.linear_fc2.weight6 - module.decoder.layers.28.mlp.experts.linear_fc2.weight11 - module.decoder.layers.28.mlp.experts.linear_fc2.weight3 - module.decoder.layers.28.mlp.experts.linear_fc2.weight0 module.decoder.layers.29.mlp.experts.linear_fc1.weight0 module.decoder.layers.28.mlp.experts.linear_fc2.weight8 module.decoder.layers.28.mlp.experts.linear_fc2.weight1 - module.decoder.layers.28.mlp.experts.linear_fc1.weight15 module.decoder.layers.28.mlp.experts.linear_fc2.weight13 module.decoder.layers.28.mlp.experts.linear_fc2.weight5 module.decoder.layers.28.mlp.experts.linear_fc2.weight10 module.decoder.layers.28.mlp.experts.linear_fc2.weight2 - module.decoder.layers.28.mlp.experts.linear_fc1.weight14 + module.decoder.layers.28.mlp.experts.linear_fc1.weight13 module.decoder.layers.28.mlp.experts.linear_fc2.weight15 module.decoder.layers.28.mlp.experts.linear_fc2.weight7 module.decoder.layers.28.mlp.experts.linear_fc2.weight12 module.decoder.layers.28.mlp.experts.linear_fc2.weight4 - module.decoder.layers.28.mlp.experts.linear_fc1.weight12 + module.decoder.layers.28.mlp.experts.linear_fc1.weight14 module.decoder.layers.28.mlp.experts.linear_fc2.weight9 - module.decoder.layers.28.mlp.experts.linear_fc1.weight13 + module.decoder.layers.28.mlp.experts.linear_fc1.weight12 + module.decoder.layers.28.mlp.experts.linear_fc2.weight14 + module.decoder.layers.28.mlp.experts.linear_fc2.weight6 + module.decoder.layers.28.mlp.experts.linear_fc2.weight11 + module.decoder.layers.28.mlp.experts.linear_fc2.weight3 + module.decoder.layers.28.mlp.experts.linear_fc2.weight0 + module.decoder.layers.28.mlp.experts.linear_fc1.weight15 Params for bucket 37 (40894464 elements, 40894464 padded size): module.decoder.layers.28.mlp.experts.linear_fc1.weight11 - module.decoder.layers.28.mlp.experts.linear_fc1.weight7 module.decoder.layers.28.mlp.experts.linear_fc1.weight9 + module.decoder.layers.28.mlp.experts.linear_fc1.weight7 module.decoder.layers.28.mlp.experts.linear_fc1.weight5 module.decoder.layers.28.mlp.experts.linear_fc1.weight2 module.decoder.layers.28.mlp.experts.linear_fc1.weight1 - module.decoder.layers.27.mlp.experts.linear_fc2.weight14 - module.decoder.layers.28.mlp.experts.linear_fc1.weight8 + module.decoder.layers.27.mlp.experts.linear_fc2.weight15 module.decoder.layers.28.mlp.experts.linear_fc1.weight10 + module.decoder.layers.28.mlp.experts.linear_fc1.weight8 module.decoder.layers.28.mlp.experts.linear_fc1.weight6 module.decoder.layers.28.mlp.experts.linear_fc1.weight4 module.decoder.layers.28.mlp.experts.linear_fc1.weight3 module.decoder.layers.28.mlp.experts.linear_fc1.weight0 - module.decoder.layers.27.mlp.experts.linear_fc2.weight15 + module.decoder.layers.27.mlp.experts.linear_fc2.weight14 Params for bucket 38 (40894464 elements, 40894464 padded size): - module.decoder.layers.27.mlp.experts.linear_fc2.weight6 + module.decoder.layers.27.mlp.experts.linear_fc2.weight10 + module.decoder.layers.27.mlp.experts.linear_fc2.weight1 + module.decoder.layers.27.mlp.experts.linear_fc2.weight7 module.decoder.layers.27.mlp.experts.linear_fc1.weight13 - module.decoder.layers.27.mlp.experts.linear_fc2.weight3 + module.decoder.layers.27.mlp.experts.linear_fc2.weight12 + module.decoder.layers.27.mlp.experts.linear_fc2.weight4 module.decoder.layers.27.mlp.experts.linear_fc1.weight10 module.decoder.layers.27.mlp.experts.linear_fc2.weight9 - module.decoder.layers.27.mlp.experts.linear_fc2.weight12 - module.decoder.layers.27.mlp.experts.linear_fc2.weight8 + module.decoder.layers.27.mlp.experts.linear_fc2.weight0 module.decoder.layers.27.mlp.experts.linear_fc1.weight15 - module.decoder.layers.27.mlp.experts.linear_fc2.weight5 + module.decoder.layers.27.mlp.experts.linear_fc2.weight6 module.decoder.layers.27.mlp.experts.linear_fc1.weight12 module.decoder.layers.27.mlp.experts.linear_fc2.weight11 - module.decoder.layers.27.mlp.experts.linear_fc2.weight1 - module.decoder.layers.27.mlp.experts.linear_fc2.weight7 + module.decoder.layers.27.mlp.experts.linear_fc2.weight3 + module.decoder.layers.27.mlp.experts.linear_fc2.weight13 + module.decoder.layers.27.mlp.experts.linear_fc2.weight8 module.decoder.layers.27.mlp.experts.linear_fc2.weight2 module.decoder.layers.27.mlp.experts.linear_fc1.weight14 - module.decoder.layers.27.mlp.experts.linear_fc2.weight4 + module.decoder.layers.27.mlp.experts.linear_fc2.weight5 module.decoder.layers.27.mlp.experts.linear_fc1.weight11 - module.decoder.layers.27.mlp.experts.linear_fc2.weight13 - module.decoder.layers.27.mlp.experts.linear_fc2.weight10 - module.decoder.layers.27.mlp.experts.linear_fc2.weight0 Params for bucket 39 (40894464 elements, 40894464 padded size): - module.decoder.layers.27.mlp.experts.linear_fc1.weight9 - module.decoder.layers.27.mlp.experts.linear_fc1.weight7 - module.decoder.layers.27.mlp.experts.linear_fc1.weight5 - module.decoder.layers.27.mlp.experts.linear_fc1.weight2 - module.decoder.layers.27.mlp.experts.linear_fc1.weight1 - module.decoder.layers.26.mlp.experts.linear_fc2.weight15 - module.decoder.layers.26.mlp.experts.linear_fc2.weight13 - module.decoder.layers.26.mlp.experts.linear_fc2.weight11 module.decoder.layers.27.mlp.experts.linear_fc1.weight8 module.decoder.layers.27.mlp.experts.linear_fc1.weight6 module.decoder.layers.27.mlp.experts.linear_fc1.weight4 @@ -1724,16 +1716,15 @@ Params for bucket 39 (40894464 elements, 40894464 padded size): module.decoder.layers.26.mlp.experts.linear_fc2.weight14 module.decoder.layers.26.mlp.experts.linear_fc2.weight12 module.decoder.layers.26.mlp.experts.linear_fc2.weight10 + module.decoder.layers.27.mlp.experts.linear_fc1.weight9 + module.decoder.layers.27.mlp.experts.linear_fc1.weight7 + module.decoder.layers.27.mlp.experts.linear_fc1.weight5 + module.decoder.layers.27.mlp.experts.linear_fc1.weight2 + module.decoder.layers.27.mlp.experts.linear_fc1.weight1 + module.decoder.layers.26.mlp.experts.linear_fc2.weight15 + module.decoder.layers.26.mlp.experts.linear_fc2.weight13 + module.decoder.layers.26.mlp.experts.linear_fc2.weight11 Params for bucket 40 (40894464 elements, 40894464 padded size): - module.decoder.layers.26.mlp.experts.linear_fc2.weight9 - module.decoder.layers.26.mlp.experts.linear_fc2.weight7 - module.decoder.layers.26.mlp.experts.linear_fc2.weight5 - module.decoder.layers.26.mlp.experts.linear_fc2.weight3 - module.decoder.layers.26.mlp.experts.linear_fc2.weight0 - module.decoder.layers.26.mlp.experts.linear_fc1.weight14 - module.decoder.layers.26.mlp.experts.linear_fc1.weight12 - module.decoder.layers.26.mlp.experts.linear_fc1.weight10 - module.decoder.layers.26.mlp.experts.linear_fc1.weight8 module.decoder.layers.26.mlp.experts.linear_fc2.weight6 module.decoder.layers.26.mlp.experts.linear_fc2.weight8 module.decoder.layers.26.mlp.experts.linear_fc2.weight4 @@ -1743,16 +1734,16 @@ Params for bucket 40 (40894464 elements, 40894464 padded size): module.decoder.layers.26.mlp.experts.linear_fc1.weight13 module.decoder.layers.26.mlp.experts.linear_fc1.weight11 module.decoder.layers.26.mlp.experts.linear_fc1.weight9 + module.decoder.layers.26.mlp.experts.linear_fc2.weight9 + module.decoder.layers.26.mlp.experts.linear_fc2.weight7 + module.decoder.layers.26.mlp.experts.linear_fc2.weight5 + module.decoder.layers.26.mlp.experts.linear_fc2.weight3 + module.decoder.layers.26.mlp.experts.linear_fc2.weight0 + module.decoder.layers.26.mlp.experts.linear_fc1.weight14 + module.decoder.layers.26.mlp.experts.linear_fc1.weight12 + module.decoder.layers.26.mlp.experts.linear_fc1.weight10 + module.decoder.layers.26.mlp.experts.linear_fc1.weight8 Params for bucket 41 (40894464 elements, 40894464 padded size): - module.decoder.layers.26.mlp.experts.linear_fc1.weight4 - module.decoder.layers.26.mlp.experts.linear_fc1.weight3 - module.decoder.layers.26.mlp.experts.linear_fc1.weight6 - module.decoder.layers.26.mlp.experts.linear_fc1.weight0 - module.decoder.layers.25.mlp.experts.linear_fc2.weight14 - module.decoder.layers.25.mlp.experts.linear_fc2.weight12 - module.decoder.layers.25.mlp.experts.linear_fc2.weight10 - module.decoder.layers.25.mlp.experts.linear_fc2.weight8 - module.decoder.layers.25.mlp.experts.linear_fc2.weight6 module.decoder.layers.26.mlp.experts.linear_fc1.weight7 module.decoder.layers.26.mlp.experts.linear_fc1.weight5 module.decoder.layers.26.mlp.experts.linear_fc1.weight2 @@ -1762,15 +1753,16 @@ Params for bucket 41 (40894464 elements, 40894464 padded size): module.decoder.layers.25.mlp.experts.linear_fc2.weight11 module.decoder.layers.25.mlp.experts.linear_fc2.weight9 module.decoder.layers.25.mlp.experts.linear_fc2.weight7 + module.decoder.layers.26.mlp.experts.linear_fc1.weight4 + module.decoder.layers.26.mlp.experts.linear_fc1.weight3 + module.decoder.layers.26.mlp.experts.linear_fc1.weight6 + module.decoder.layers.26.mlp.experts.linear_fc1.weight0 + module.decoder.layers.25.mlp.experts.linear_fc2.weight14 + module.decoder.layers.25.mlp.experts.linear_fc2.weight12 + module.decoder.layers.25.mlp.experts.linear_fc2.weight10 + module.decoder.layers.25.mlp.experts.linear_fc2.weight8 + module.decoder.layers.25.mlp.experts.linear_fc2.weight6 Params for bucket 42 (40894464 elements, 40894464 padded size): - module.decoder.layers.25.mlp.experts.linear_fc2.weight2 - module.decoder.layers.25.mlp.experts.linear_fc2.weight1 - module.decoder.layers.25.mlp.experts.linear_fc2.weight4 - module.decoder.layers.25.mlp.experts.linear_fc1.weight15 - module.decoder.layers.25.mlp.experts.linear_fc1.weight13 - module.decoder.layers.25.mlp.experts.linear_fc1.weight11 - module.decoder.layers.25.mlp.experts.linear_fc1.weight9 - module.decoder.layers.25.mlp.experts.linear_fc1.weight7 module.decoder.layers.25.mlp.experts.linear_fc2.weight5 module.decoder.layers.25.mlp.experts.linear_fc2.weight3 module.decoder.layers.25.mlp.experts.linear_fc2.weight0 @@ -1779,35 +1771,36 @@ Params for bucket 42 (40894464 elements, 40894464 padded size): module.decoder.layers.25.mlp.experts.linear_fc1.weight10 module.decoder.layers.25.mlp.experts.linear_fc1.weight8 module.decoder.layers.25.mlp.experts.linear_fc1.weight6 + module.decoder.layers.25.mlp.experts.linear_fc2.weight2 + module.decoder.layers.25.mlp.experts.linear_fc2.weight1 + module.decoder.layers.25.mlp.experts.linear_fc2.weight4 + module.decoder.layers.25.mlp.experts.linear_fc1.weight15 + module.decoder.layers.25.mlp.experts.linear_fc1.weight13 + module.decoder.layers.25.mlp.experts.linear_fc1.weight11 + module.decoder.layers.25.mlp.experts.linear_fc1.weight9 + module.decoder.layers.25.mlp.experts.linear_fc1.weight7 Params for bucket 43 (40894464 elements, 40894464 padded size): - module.decoder.layers.25.mlp.experts.linear_fc1.weight1 - module.decoder.layers.24.mlp.experts.linear_fc2.weight11 - module.decoder.layers.24.mlp.experts.linear_fc2.weight3 - module.decoder.layers.25.mlp.experts.linear_fc1.weight0 - module.decoder.layers.24.mlp.experts.linear_fc2.weight8 - module.decoder.layers.25.mlp.experts.linear_fc1.weight5 module.decoder.layers.24.mlp.experts.linear_fc2.weight13 module.decoder.layers.24.mlp.experts.linear_fc2.weight5 - module.decoder.layers.25.mlp.experts.linear_fc1.weight3 + module.decoder.layers.25.mlp.experts.linear_fc1.weight1 module.decoder.layers.24.mlp.experts.linear_fc2.weight10 module.decoder.layers.24.mlp.experts.linear_fc2.weight2 + module.decoder.layers.25.mlp.experts.linear_fc1.weight0 module.decoder.layers.24.mlp.experts.linear_fc2.weight15 module.decoder.layers.24.mlp.experts.linear_fc2.weight7 - module.decoder.layers.25.mlp.experts.linear_fc1.weight4 + module.decoder.layers.25.mlp.experts.linear_fc1.weight5 module.decoder.layers.24.mlp.experts.linear_fc2.weight12 module.decoder.layers.24.mlp.experts.linear_fc2.weight4 - module.decoder.layers.25.mlp.experts.linear_fc1.weight2 + module.decoder.layers.25.mlp.experts.linear_fc1.weight3 module.decoder.layers.24.mlp.experts.linear_fc2.weight9 module.decoder.layers.24.mlp.experts.linear_fc2.weight14 module.decoder.layers.24.mlp.experts.linear_fc2.weight6 + module.decoder.layers.25.mlp.experts.linear_fc1.weight4 + module.decoder.layers.24.mlp.experts.linear_fc2.weight11 + module.decoder.layers.24.mlp.experts.linear_fc2.weight3 + module.decoder.layers.25.mlp.experts.linear_fc1.weight2 + module.decoder.layers.24.mlp.experts.linear_fc2.weight8 Params for bucket 44 (40894464 elements, 40894464 padded size): - module.decoder.layers.24.mlp.experts.linear_fc1.weight13 - module.decoder.layers.24.mlp.experts.linear_fc1.weight15 - module.decoder.layers.24.mlp.experts.linear_fc1.weight11 - module.decoder.layers.24.mlp.experts.linear_fc1.weight9 - module.decoder.layers.24.mlp.experts.linear_fc1.weight7 - module.decoder.layers.24.mlp.experts.linear_fc1.weight5 - module.decoder.layers.24.mlp.experts.linear_fc2.weight1 module.decoder.layers.24.mlp.experts.linear_fc1.weight14 module.decoder.layers.24.mlp.experts.linear_fc2.weight0 module.decoder.layers.24.mlp.experts.linear_fc1.weight12 @@ -1815,145 +1808,144 @@ Params for bucket 44 (40894464 elements, 40894464 padded size): module.decoder.layers.24.mlp.experts.linear_fc1.weight8 module.decoder.layers.24.mlp.experts.linear_fc1.weight6 module.decoder.layers.24.mlp.experts.linear_fc1.weight4 + module.decoder.layers.24.mlp.experts.linear_fc2.weight1 + module.decoder.layers.24.mlp.experts.linear_fc1.weight13 + module.decoder.layers.24.mlp.experts.linear_fc1.weight15 + module.decoder.layers.24.mlp.experts.linear_fc1.weight11 + module.decoder.layers.24.mlp.experts.linear_fc1.weight9 + module.decoder.layers.24.mlp.experts.linear_fc1.weight7 + module.decoder.layers.24.mlp.experts.linear_fc1.weight5 Params for bucket 45 (40894464 elements, 40894464 padded size): - module.decoder.layers.23.mlp.experts.linear_fc2.weight15 - module.decoder.layers.24.mlp.experts.linear_fc1.weight2 - module.decoder.layers.23.mlp.experts.linear_fc2.weight7 module.decoder.layers.23.mlp.experts.linear_fc2.weight12 module.decoder.layers.23.mlp.experts.linear_fc2.weight4 module.decoder.layers.23.mlp.experts.linear_fc2.weight0 - module.decoder.layers.24.mlp.experts.linear_fc1.weight1 + module.decoder.layers.24.mlp.experts.linear_fc1.weight2 module.decoder.layers.23.mlp.experts.linear_fc2.weight9 - module.decoder.layers.24.mlp.experts.linear_fc1.weight0 module.decoder.layers.23.mlp.experts.linear_fc2.weight14 module.decoder.layers.23.mlp.experts.linear_fc2.weight6 + module.decoder.layers.24.mlp.experts.linear_fc1.weight1 module.decoder.layers.23.mlp.experts.linear_fc2.weight11 module.decoder.layers.23.mlp.experts.linear_fc2.weight3 - module.decoder.layers.24.mlp.experts.linear_fc1.weight3 + module.decoder.layers.24.mlp.experts.linear_fc1.weight0 module.decoder.layers.23.mlp.experts.linear_fc2.weight8 module.decoder.layers.23.mlp.experts.linear_fc2.weight1 module.decoder.layers.23.mlp.experts.linear_fc2.weight13 module.decoder.layers.23.mlp.experts.linear_fc2.weight5 module.decoder.layers.23.mlp.experts.linear_fc1.weight15 + module.decoder.layers.24.mlp.experts.linear_fc1.weight3 module.decoder.layers.23.mlp.experts.linear_fc2.weight10 module.decoder.layers.23.mlp.experts.linear_fc2.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight15 + module.decoder.layers.23.mlp.experts.linear_fc2.weight7 Params for bucket 46 (40894464 elements, 40894464 padded size): - module.decoder.layers.23.mlp.experts.linear_fc1.weight11 - module.decoder.layers.23.mlp.experts.linear_fc1.weight13 - module.decoder.layers.23.mlp.experts.linear_fc1.weight9 - module.decoder.layers.23.mlp.experts.linear_fc1.weight7 - module.decoder.layers.23.mlp.experts.linear_fc1.weight5 - module.decoder.layers.23.mlp.experts.linear_fc1.weight2 module.decoder.layers.23.mlp.experts.linear_fc1.weight14 - module.decoder.layers.23.mlp.experts.linear_fc1.weight12 module.decoder.layers.23.mlp.experts.linear_fc1.weight10 + module.decoder.layers.23.mlp.experts.linear_fc1.weight12 module.decoder.layers.23.mlp.experts.linear_fc1.weight8 module.decoder.layers.23.mlp.experts.linear_fc1.weight6 module.decoder.layers.23.mlp.experts.linear_fc1.weight4 module.decoder.layers.23.mlp.experts.linear_fc1.weight3 + module.decoder.layers.23.mlp.experts.linear_fc1.weight13 + module.decoder.layers.23.mlp.experts.linear_fc1.weight11 + module.decoder.layers.23.mlp.experts.linear_fc1.weight9 + module.decoder.layers.23.mlp.experts.linear_fc1.weight7 + module.decoder.layers.23.mlp.experts.linear_fc1.weight5 + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 Params for bucket 47 (40894464 elements, 40894464 padded size): - module.decoder.layers.22.mlp.experts.linear_fc2.weight10 - module.decoder.layers.22.mlp.experts.linear_fc2.weight1 module.decoder.layers.22.mlp.experts.linear_fc2.weight15 module.decoder.layers.22.mlp.experts.linear_fc2.weight7 - module.decoder.layers.22.mlp.experts.linear_fc2.weight2 - module.decoder.layers.22.mlp.experts.linear_fc1.weight14 + module.decoder.layers.22.mlp.experts.linear_fc1.weight13 module.decoder.layers.23.mlp.experts.linear_fc1.weight1 module.decoder.layers.22.mlp.experts.linear_fc2.weight12 module.decoder.layers.22.mlp.experts.linear_fc2.weight4 module.decoder.layers.23.mlp.experts.linear_fc1.weight0 module.decoder.layers.22.mlp.experts.linear_fc2.weight9 module.decoder.layers.22.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc1.weight15 module.decoder.layers.22.mlp.experts.linear_fc2.weight14 module.decoder.layers.22.mlp.experts.linear_fc2.weight6 - module.decoder.layers.22.mlp.experts.linear_fc1.weight13 module.decoder.layers.22.mlp.experts.linear_fc2.weight11 module.decoder.layers.22.mlp.experts.linear_fc2.weight3 module.decoder.layers.22.mlp.experts.linear_fc2.weight8 - module.decoder.layers.22.mlp.experts.linear_fc1.weight15 + module.decoder.layers.22.mlp.experts.linear_fc2.weight2 + module.decoder.layers.22.mlp.experts.linear_fc1.weight14 module.decoder.layers.22.mlp.experts.linear_fc2.weight13 module.decoder.layers.22.mlp.experts.linear_fc2.weight5 + module.decoder.layers.22.mlp.experts.linear_fc2.weight10 + module.decoder.layers.22.mlp.experts.linear_fc2.weight1 Params for bucket 48 (40894464 elements, 40894464 padded size): - module.decoder.layers.22.mlp.experts.linear_fc1.weight9 module.decoder.layers.22.mlp.experts.linear_fc1.weight11 + module.decoder.layers.22.mlp.experts.linear_fc1.weight9 module.decoder.layers.22.mlp.experts.linear_fc1.weight7 module.decoder.layers.22.mlp.experts.linear_fc1.weight5 module.decoder.layers.22.mlp.experts.linear_fc1.weight2 module.decoder.layers.22.mlp.experts.linear_fc1.weight1 module.decoder.layers.22.mlp.experts.linear_fc1.weight12 - module.decoder.layers.22.mlp.experts.linear_fc1.weight8 module.decoder.layers.22.mlp.experts.linear_fc1.weight10 + module.decoder.layers.22.mlp.experts.linear_fc1.weight8 module.decoder.layers.22.mlp.experts.linear_fc1.weight6 module.decoder.layers.22.mlp.experts.linear_fc1.weight4 module.decoder.layers.22.mlp.experts.linear_fc1.weight3 module.decoder.layers.22.mlp.experts.linear_fc1.weight0 Params for bucket 49 (40894464 elements, 40894464 padded size): + module.decoder.layers.21.mlp.experts.linear_fc2.weight14 + module.decoder.layers.21.mlp.experts.linear_fc2.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight13 module.decoder.layers.21.mlp.experts.linear_fc2.weight11 module.decoder.layers.21.mlp.experts.linear_fc2.weight3 + module.decoder.layers.21.mlp.experts.linear_fc2.weight0 module.decoder.layers.21.mlp.experts.linear_fc2.weight8 module.decoder.layers.21.mlp.experts.linear_fc2.weight1 - module.decoder.layers.21.mlp.experts.linear_fc1.weight14 + module.decoder.layers.21.mlp.experts.linear_fc1.weight15 module.decoder.layers.21.mlp.experts.linear_fc2.weight13 module.decoder.layers.21.mlp.experts.linear_fc2.weight5 - module.decoder.layers.21.mlp.experts.linear_fc1.weight11 + module.decoder.layers.21.mlp.experts.linear_fc1.weight12 module.decoder.layers.21.mlp.experts.linear_fc2.weight10 module.decoder.layers.21.mlp.experts.linear_fc2.weight2 module.decoder.layers.21.mlp.experts.linear_fc2.weight15 module.decoder.layers.21.mlp.experts.linear_fc2.weight7 - module.decoder.layers.21.mlp.experts.linear_fc1.weight13 + module.decoder.layers.21.mlp.experts.linear_fc1.weight14 module.decoder.layers.21.mlp.experts.linear_fc2.weight12 module.decoder.layers.21.mlp.experts.linear_fc2.weight4 - module.decoder.layers.21.mlp.experts.linear_fc2.weight0 + module.decoder.layers.21.mlp.experts.linear_fc1.weight11 module.decoder.layers.21.mlp.experts.linear_fc2.weight9 - module.decoder.layers.21.mlp.experts.linear_fc1.weight15 - module.decoder.layers.21.mlp.experts.linear_fc2.weight14 - module.decoder.layers.21.mlp.experts.linear_fc2.weight6 - module.decoder.layers.21.mlp.experts.linear_fc1.weight12 Params for bucket 50 (40894464 elements, 40894464 padded size): - module.decoder.layers.21.mlp.experts.linear_fc1.weight7 module.decoder.layers.21.mlp.experts.linear_fc1.weight9 - module.decoder.layers.21.mlp.experts.linear_fc1.weight5 + module.decoder.layers.21.mlp.experts.linear_fc1.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight4 module.decoder.layers.21.mlp.experts.linear_fc1.weight2 module.decoder.layers.21.mlp.experts.linear_fc1.weight1 module.decoder.layers.20.mlp.experts.linear_fc2.weight15 module.decoder.layers.20.mlp.experts.linear_fc2.weight13 module.decoder.layers.21.mlp.experts.linear_fc1.weight10 - module.decoder.layers.21.mlp.experts.linear_fc1.weight8 - module.decoder.layers.21.mlp.experts.linear_fc1.weight6 - module.decoder.layers.21.mlp.experts.linear_fc1.weight4 + module.decoder.layers.21.mlp.experts.linear_fc1.weight8 + module.decoder.layers.21.mlp.experts.linear_fc1.weight5 module.decoder.layers.21.mlp.experts.linear_fc1.weight3 module.decoder.layers.21.mlp.experts.linear_fc1.weight0 module.decoder.layers.20.mlp.experts.linear_fc2.weight14 module.decoder.layers.20.mlp.experts.linear_fc2.weight12 Params for bucket 51 (40894464 elements, 40894464 padded size): module.decoder.layers.20.mlp.experts.linear_fc2.weight9 - module.decoder.layers.20.mlp.experts.linear_fc1.weight11 + module.decoder.layers.20.mlp.experts.linear_fc1.weight12 module.decoder.layers.20.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc1.weight9 module.decoder.layers.20.mlp.experts.linear_fc2.weight11 module.decoder.layers.20.mlp.experts.linear_fc2.weight3 - module.decoder.layers.20.mlp.experts.linear_fc1.weight13 + module.decoder.layers.20.mlp.experts.linear_fc2.weight1 + module.decoder.layers.20.mlp.experts.linear_fc1.weight14 module.decoder.layers.20.mlp.experts.linear_fc2.weight8 - module.decoder.layers.20.mlp.experts.linear_fc1.weight10 + module.decoder.layers.20.mlp.experts.linear_fc1.weight11 module.decoder.layers.20.mlp.experts.linear_fc2.weight5 module.decoder.layers.20.mlp.experts.linear_fc2.weight2 - module.decoder.layers.20.mlp.experts.linear_fc1.weight15 module.decoder.layers.20.mlp.experts.linear_fc2.weight10 - module.decoder.layers.20.mlp.experts.linear_fc1.weight12 + module.decoder.layers.20.mlp.experts.linear_fc1.weight13 module.decoder.layers.20.mlp.experts.linear_fc2.weight7 - module.decoder.layers.20.mlp.experts.linear_fc1.weight9 + module.decoder.layers.20.mlp.experts.linear_fc1.weight10 module.decoder.layers.20.mlp.experts.linear_fc2.weight4 - module.decoder.layers.20.mlp.experts.linear_fc2.weight1 module.decoder.layers.20.mlp.experts.linear_fc2.weight0 - module.decoder.layers.20.mlp.experts.linear_fc1.weight14 + module.decoder.layers.20.mlp.experts.linear_fc1.weight15 Params for bucket 52 (40894464 elements, 40894464 padded size): - module.decoder.layers.20.mlp.experts.linear_fc1.weight5 - module.decoder.layers.20.mlp.experts.linear_fc1.weight7 - module.decoder.layers.20.mlp.experts.linear_fc1.weight2 - module.decoder.layers.20.mlp.experts.linear_fc1.weight1 - module.decoder.layers.19.mlp.experts.linear_fc2.weight15 - module.decoder.layers.19.mlp.experts.linear_fc2.weight13 - module.decoder.layers.19.mlp.experts.linear_fc2.weight11 - module.decoder.layers.19.mlp.experts.linear_fc2.weight9 module.decoder.layers.20.mlp.experts.linear_fc1.weight8 module.decoder.layers.20.mlp.experts.linear_fc1.weight6 module.decoder.layers.20.mlp.experts.linear_fc1.weight4 @@ -1963,53 +1955,53 @@ Params for bucket 52 (40894464 elements, 40894464 padded size): module.decoder.layers.19.mlp.experts.linear_fc2.weight12 module.decoder.layers.19.mlp.experts.linear_fc2.weight10 module.decoder.layers.19.mlp.experts.linear_fc2.weight8 + module.decoder.layers.20.mlp.experts.linear_fc1.weight7 + module.decoder.layers.20.mlp.experts.linear_fc1.weight5 + module.decoder.layers.20.mlp.experts.linear_fc1.weight2 + module.decoder.layers.20.mlp.experts.linear_fc1.weight1 + module.decoder.layers.19.mlp.experts.linear_fc2.weight15 + module.decoder.layers.19.mlp.experts.linear_fc2.weight13 + module.decoder.layers.19.mlp.experts.linear_fc2.weight11 + module.decoder.layers.19.mlp.experts.linear_fc2.weight9 Params for bucket 53 (40894464 elements, 40894464 padded size): - module.decoder.layers.19.mlp.experts.linear_fc2.weight7 - module.decoder.layers.19.mlp.experts.linear_fc2.weight5 - module.decoder.layers.19.mlp.experts.linear_fc2.weight3 + module.decoder.layers.19.mlp.experts.linear_fc2.weight6 + module.decoder.layers.19.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight1 module.decoder.layers.19.mlp.experts.linear_fc1.weight15 module.decoder.layers.19.mlp.experts.linear_fc1.weight13 module.decoder.layers.19.mlp.experts.linear_fc1.weight11 module.decoder.layers.19.mlp.experts.linear_fc1.weight9 module.decoder.layers.19.mlp.experts.linear_fc1.weight7 - module.decoder.layers.19.mlp.experts.linear_fc2.weight6 - module.decoder.layers.19.mlp.experts.linear_fc2.weight4 - module.decoder.layers.19.mlp.experts.linear_fc2.weight2 - module.decoder.layers.19.mlp.experts.linear_fc2.weight1 + module.decoder.layers.19.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc2.weight3 + module.decoder.layers.19.mlp.experts.linear_fc2.weight5 module.decoder.layers.19.mlp.experts.linear_fc2.weight0 module.decoder.layers.19.mlp.experts.linear_fc1.weight14 module.decoder.layers.19.mlp.experts.linear_fc1.weight12 module.decoder.layers.19.mlp.experts.linear_fc1.weight10 module.decoder.layers.19.mlp.experts.linear_fc1.weight8 Params for bucket 54 (40894464 elements, 40894464 padded size): - module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 + module.decoder.layers.18.mlp.experts.linear_fc2.weight11 + module.decoder.layers.19.mlp.experts.linear_fc1.weight0 + module.decoder.layers.18.mlp.experts.linear_fc2.weight8 + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight13 + module.decoder.layers.18.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 module.decoder.layers.18.mlp.experts.linear_fc2.weight10 - module.decoder.layers.19.mlp.experts.linear_fc1.weight6 module.decoder.layers.18.mlp.experts.linear_fc2.weight15 module.decoder.layers.18.mlp.experts.linear_fc2.weight7 - module.decoder.layers.19.mlp.experts.linear_fc1.weight1 + module.decoder.layers.19.mlp.experts.linear_fc1.weight4 module.decoder.layers.18.mlp.experts.linear_fc2.weight12 module.decoder.layers.18.mlp.experts.linear_fc2.weight4 - module.decoder.layers.19.mlp.experts.linear_fc1.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight2 module.decoder.layers.18.mlp.experts.linear_fc2.weight9 - module.decoder.layers.19.mlp.experts.linear_fc1.weight5 + module.decoder.layers.19.mlp.experts.linear_fc1.weight6 module.decoder.layers.18.mlp.experts.linear_fc2.weight14 module.decoder.layers.18.mlp.experts.linear_fc2.weight6 - module.decoder.layers.19.mlp.experts.linear_fc1.weight3 - module.decoder.layers.18.mlp.experts.linear_fc2.weight11 - module.decoder.layers.18.mlp.experts.linear_fc2.weight8 - module.decoder.layers.19.mlp.experts.linear_fc1.weight4 - module.decoder.layers.18.mlp.experts.linear_fc2.weight13 - module.decoder.layers.18.mlp.experts.linear_fc2.weight5 Params for bucket 55 (40894464 elements, 40894464 padded size): - module.decoder.layers.18.mlp.experts.linear_fc2.weight2 - module.decoder.layers.18.mlp.experts.linear_fc2.weight1 - module.decoder.layers.18.mlp.experts.linear_fc1.weight15 - module.decoder.layers.18.mlp.experts.linear_fc1.weight13 - module.decoder.layers.18.mlp.experts.linear_fc1.weight11 - module.decoder.layers.18.mlp.experts.linear_fc1.weight9 - module.decoder.layers.18.mlp.experts.linear_fc1.weight7 - module.decoder.layers.18.mlp.experts.linear_fc1.weight5 module.decoder.layers.18.mlp.experts.linear_fc2.weight3 module.decoder.layers.18.mlp.experts.linear_fc2.weight0 module.decoder.layers.18.mlp.experts.linear_fc1.weight14 @@ -2017,35 +2009,37 @@ Params for bucket 55 (40894464 elements, 40894464 padded size): module.decoder.layers.18.mlp.experts.linear_fc1.weight10 module.decoder.layers.18.mlp.experts.linear_fc1.weight8 module.decoder.layers.18.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight15 + module.decoder.layers.18.mlp.experts.linear_fc2.weight2 + module.decoder.layers.18.mlp.experts.linear_fc1.weight13 + module.decoder.layers.18.mlp.experts.linear_fc1.weight11 + module.decoder.layers.18.mlp.experts.linear_fc1.weight9 + module.decoder.layers.18.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight5 Params for bucket 56 (40894464 elements, 40894464 padded size): + module.decoder.layers.17.mlp.experts.linear_fc2.weight15 + module.decoder.layers.17.mlp.experts.linear_fc2.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight1 module.decoder.layers.17.mlp.experts.linear_fc2.weight12 module.decoder.layers.17.mlp.experts.linear_fc2.weight4 - module.decoder.layers.18.mlp.experts.linear_fc1.weight3 + module.decoder.layers.18.mlp.experts.linear_fc1.weight0 module.decoder.layers.17.mlp.experts.linear_fc2.weight9 - module.decoder.layers.17.mlp.experts.linear_fc2.weight0 module.decoder.layers.17.mlp.experts.linear_fc2.weight14 module.decoder.layers.17.mlp.experts.linear_fc2.weight6 - module.decoder.layers.18.mlp.experts.linear_fc1.weight4 + module.decoder.layers.18.mlp.experts.linear_fc1.weight3 module.decoder.layers.17.mlp.experts.linear_fc2.weight11 module.decoder.layers.17.mlp.experts.linear_fc2.weight3 - module.decoder.layers.18.mlp.experts.linear_fc1.weight2 + module.decoder.layers.17.mlp.experts.linear_fc2.weight0 module.decoder.layers.17.mlp.experts.linear_fc2.weight8 + module.decoder.layers.17.mlp.experts.linear_fc2.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight4 module.decoder.layers.17.mlp.experts.linear_fc2.weight13 module.decoder.layers.17.mlp.experts.linear_fc2.weight5 - module.decoder.layers.18.mlp.experts.linear_fc1.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight2 module.decoder.layers.17.mlp.experts.linear_fc2.weight10 - module.decoder.layers.17.mlp.experts.linear_fc2.weight1 module.decoder.layers.17.mlp.experts.linear_fc2.weight2 - module.decoder.layers.18.mlp.experts.linear_fc1.weight0 - module.decoder.layers.17.mlp.experts.linear_fc2.weight15 - module.decoder.layers.17.mlp.experts.linear_fc2.weight7 Params for bucket 57 (40894464 elements, 40894464 padded size): - module.decoder.layers.17.mlp.experts.linear_fc1.weight15 - module.decoder.layers.17.mlp.experts.linear_fc1.weight13 - module.decoder.layers.17.mlp.experts.linear_fc1.weight11 - module.decoder.layers.17.mlp.experts.linear_fc1.weight9 - module.decoder.layers.17.mlp.experts.linear_fc1.weight7 - module.decoder.layers.17.mlp.experts.linear_fc1.weight5 module.decoder.layers.17.mlp.experts.linear_fc1.weight14 module.decoder.layers.17.mlp.experts.linear_fc1.weight12 module.decoder.layers.17.mlp.experts.linear_fc1.weight10 @@ -2053,7 +2047,15 @@ Params for bucket 57 (40894464 elements, 40894464 padded size): module.decoder.layers.17.mlp.experts.linear_fc1.weight6 module.decoder.layers.17.mlp.experts.linear_fc1.weight4 module.decoder.layers.17.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc1.weight15 + module.decoder.layers.17.mlp.experts.linear_fc1.weight13 + module.decoder.layers.17.mlp.experts.linear_fc1.weight11 + module.decoder.layers.17.mlp.experts.linear_fc1.weight9 + module.decoder.layers.17.mlp.experts.linear_fc1.weight7 + module.decoder.layers.17.mlp.experts.linear_fc1.weight5 Params for bucket 58 (40894464 elements, 40894464 padded size): + module.decoder.layers.16.mlp.experts.linear_fc2.weight15 + module.decoder.layers.16.mlp.experts.linear_fc2.weight7 module.decoder.layers.17.mlp.experts.linear_fc1.weight1 module.decoder.layers.16.mlp.experts.linear_fc2.weight12 module.decoder.layers.16.mlp.experts.linear_fc2.weight4 @@ -2073,15 +2075,7 @@ Params for bucket 58 (40894464 elements, 40894464 padded size): module.decoder.layers.17.mlp.experts.linear_fc1.weight2 module.decoder.layers.16.mlp.experts.linear_fc2.weight10 module.decoder.layers.16.mlp.experts.linear_fc2.weight2 - module.decoder.layers.16.mlp.experts.linear_fc2.weight15 - module.decoder.layers.16.mlp.experts.linear_fc2.weight7 Params for bucket 59 (40894464 elements, 40894464 padded size): - module.decoder.layers.16.mlp.experts.linear_fc1.weight12 - module.decoder.layers.16.mlp.experts.linear_fc1.weight10 - module.decoder.layers.16.mlp.experts.linear_fc1.weight8 - module.decoder.layers.16.mlp.experts.linear_fc1.weight6 - module.decoder.layers.16.mlp.experts.linear_fc1.weight4 - module.decoder.layers.16.mlp.experts.linear_fc1.weight3 module.decoder.layers.16.mlp.experts.linear_fc1.weight13 module.decoder.layers.16.mlp.experts.linear_fc1.weight11 module.decoder.layers.16.mlp.experts.linear_fc1.weight9 @@ -2089,44 +2083,58 @@ Params for bucket 59 (40894464 elements, 40894464 padded size): module.decoder.layers.16.mlp.experts.linear_fc1.weight5 module.decoder.layers.16.mlp.experts.linear_fc1.weight2 module.decoder.layers.16.mlp.experts.linear_fc1.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight12 + module.decoder.layers.16.mlp.experts.linear_fc1.weight10 + module.decoder.layers.16.mlp.experts.linear_fc1.weight8 + module.decoder.layers.16.mlp.experts.linear_fc1.weight6 + module.decoder.layers.16.mlp.experts.linear_fc1.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight3 Params for bucket 60 (40894464 elements, 40894464 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight8 + module.decoder.layers.15.mlp.experts.linear_fc1.weight15 + module.decoder.layers.15.mlp.experts.linear_fc2.weight13 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight12 + module.decoder.layers.15.mlp.experts.linear_fc2.weight10 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight0 module.decoder.layers.15.mlp.experts.linear_fc2.weight15 module.decoder.layers.15.mlp.experts.linear_fc2.weight7 - module.decoder.layers.15.mlp.experts.linear_fc1.weight13 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight14 module.decoder.layers.15.mlp.experts.linear_fc2.weight12 module.decoder.layers.15.mlp.experts.linear_fc2.weight4 - module.decoder.layers.16.mlp.experts.linear_fc1.weight0 module.decoder.layers.15.mlp.experts.linear_fc2.weight9 module.decoder.layers.15.mlp.experts.linear_fc2.weight0 - module.decoder.layers.15.mlp.experts.linear_fc1.weight15 module.decoder.layers.15.mlp.experts.linear_fc2.weight14 module.decoder.layers.15.mlp.experts.linear_fc2.weight6 - module.decoder.layers.15.mlp.experts.linear_fc1.weight12 + module.decoder.layers.15.mlp.experts.linear_fc1.weight13 module.decoder.layers.15.mlp.experts.linear_fc2.weight11 module.decoder.layers.15.mlp.experts.linear_fc2.weight3 - module.decoder.layers.15.mlp.experts.linear_fc2.weight8 - module.decoder.layers.15.mlp.experts.linear_fc2.weight2 - module.decoder.layers.15.mlp.experts.linear_fc1.weight14 - module.decoder.layers.15.mlp.experts.linear_fc2.weight13 - module.decoder.layers.15.mlp.experts.linear_fc2.weight5 - module.decoder.layers.15.mlp.experts.linear_fc2.weight10 - module.decoder.layers.15.mlp.experts.linear_fc2.weight1 Params for bucket 61 (40894464 elements, 40894464 padded size): module.decoder.layers.15.mlp.experts.linear_fc1.weight11 - module.decoder.layers.15.mlp.experts.linear_fc1.weight9 module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight9 module.decoder.layers.15.mlp.experts.linear_fc1.weight5 module.decoder.layers.15.mlp.experts.linear_fc1.weight2 module.decoder.layers.15.mlp.experts.linear_fc1.weight1 module.decoder.layers.14.mlp.experts.linear_fc2.weight15 - module.decoder.layers.15.mlp.experts.linear_fc1.weight10 module.decoder.layers.15.mlp.experts.linear_fc1.weight8 + module.decoder.layers.15.mlp.experts.linear_fc1.weight10 module.decoder.layers.15.mlp.experts.linear_fc1.weight6 module.decoder.layers.15.mlp.experts.linear_fc1.weight4 module.decoder.layers.15.mlp.experts.linear_fc1.weight3 module.decoder.layers.15.mlp.experts.linear_fc1.weight0 module.decoder.layers.14.mlp.experts.linear_fc2.weight14 Params for bucket 62 (40894464 elements, 40894464 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight14 + module.decoder.layers.14.mlp.experts.linear_fc2.weight12 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight11 + module.decoder.layers.14.mlp.experts.linear_fc2.weight9 + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight13 module.decoder.layers.14.mlp.experts.linear_fc2.weight11 module.decoder.layers.14.mlp.experts.linear_fc2.weight3 module.decoder.layers.14.mlp.experts.linear_fc2.weight0 @@ -2139,14 +2147,6 @@ Params for bucket 62 (40894464 elements, 40894464 padded size): module.decoder.layers.14.mlp.experts.linear_fc1.weight12 module.decoder.layers.14.mlp.experts.linear_fc2.weight10 module.decoder.layers.14.mlp.experts.linear_fc2.weight2 - module.decoder.layers.14.mlp.experts.linear_fc2.weight7 - module.decoder.layers.14.mlp.experts.linear_fc1.weight14 - module.decoder.layers.14.mlp.experts.linear_fc2.weight12 - module.decoder.layers.14.mlp.experts.linear_fc2.weight4 - module.decoder.layers.14.mlp.experts.linear_fc1.weight11 - module.decoder.layers.14.mlp.experts.linear_fc2.weight9 - module.decoder.layers.14.mlp.experts.linear_fc2.weight6 - module.decoder.layers.14.mlp.experts.linear_fc1.weight13 Params for bucket 63 (40894464 elements, 40894464 padded size): module.decoder.layers.14.mlp.experts.linear_fc1.weight8 module.decoder.layers.14.mlp.experts.linear_fc1.weight6 @@ -2157,8 +2157,8 @@ Params for bucket 63 (40894464 elements, 40894464 padded size): module.decoder.layers.13.mlp.experts.linear_fc2.weight13 module.decoder.layers.13.mlp.experts.linear_fc2.weight11 module.decoder.layers.14.mlp.experts.linear_fc1.weight9 - module.decoder.layers.14.mlp.experts.linear_fc1.weight7 module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 module.decoder.layers.14.mlp.experts.linear_fc1.weight2 module.decoder.layers.14.mlp.experts.linear_fc1.weight1 module.decoder.layers.13.mlp.experts.linear_fc2.weight14 @@ -2166,16 +2166,16 @@ Params for bucket 63 (40894464 elements, 40894464 padded size): module.decoder.layers.13.mlp.experts.linear_fc2.weight10 Params for bucket 64 (40894464 elements, 40894464 padded size): module.decoder.layers.13.mlp.experts.linear_fc2.weight9 - module.decoder.layers.13.mlp.experts.linear_fc2.weight7 module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 module.decoder.layers.13.mlp.experts.linear_fc2.weight3 module.decoder.layers.13.mlp.experts.linear_fc2.weight0 module.decoder.layers.13.mlp.experts.linear_fc1.weight14 module.decoder.layers.13.mlp.experts.linear_fc1.weight12 module.decoder.layers.13.mlp.experts.linear_fc1.weight10 module.decoder.layers.13.mlp.experts.linear_fc1.weight8 - module.decoder.layers.13.mlp.experts.linear_fc2.weight6 module.decoder.layers.13.mlp.experts.linear_fc2.weight8 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 module.decoder.layers.13.mlp.experts.linear_fc2.weight4 module.decoder.layers.13.mlp.experts.linear_fc2.weight2 module.decoder.layers.13.mlp.experts.linear_fc2.weight1 @@ -2184,9 +2184,9 @@ Params for bucket 64 (40894464 elements, 40894464 padded size): module.decoder.layers.13.mlp.experts.linear_fc1.weight11 module.decoder.layers.13.mlp.experts.linear_fc1.weight9 Params for bucket 65 (40894464 elements, 40894464 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 module.decoder.layers.13.mlp.experts.linear_fc1.weight4 module.decoder.layers.13.mlp.experts.linear_fc1.weight3 - module.decoder.layers.13.mlp.experts.linear_fc1.weight6 module.decoder.layers.13.mlp.experts.linear_fc1.weight0 module.decoder.layers.12.mlp.experts.linear_fc2.weight15 module.decoder.layers.12.mlp.experts.linear_fc2.weight13 @@ -2211,43 +2211,36 @@ Params for bucket 66 (40894464 elements, 40894464 padded size): module.decoder.layers.12.mlp.experts.linear_fc1.weight10 module.decoder.layers.12.mlp.experts.linear_fc1.weight8 module.decoder.layers.12.mlp.experts.linear_fc1.weight6 - module.decoder.layers.12.mlp.experts.linear_fc2.weight4 module.decoder.layers.12.mlp.experts.linear_fc2.weight2 module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 module.decoder.layers.12.mlp.experts.linear_fc1.weight15 module.decoder.layers.12.mlp.experts.linear_fc1.weight13 module.decoder.layers.12.mlp.experts.linear_fc1.weight11 module.decoder.layers.12.mlp.experts.linear_fc1.weight9 module.decoder.layers.12.mlp.experts.linear_fc1.weight7 Params for bucket 67 (40894464 elements, 40894464 padded size): - module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight8 + module.decoder.layers.11.mlp.experts.linear_fc2.weight13 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 module.decoder.layers.11.mlp.experts.linear_fc2.weight10 module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 module.decoder.layers.11.mlp.experts.linear_fc2.weight15 module.decoder.layers.11.mlp.experts.linear_fc2.weight7 - module.decoder.layers.12.mlp.experts.linear_fc1.weight4 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 module.decoder.layers.11.mlp.experts.linear_fc2.weight12 module.decoder.layers.11.mlp.experts.linear_fc2.weight4 - module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 module.decoder.layers.11.mlp.experts.linear_fc2.weight9 module.decoder.layers.11.mlp.experts.linear_fc2.weight14 module.decoder.layers.11.mlp.experts.linear_fc2.weight6 - module.decoder.layers.12.mlp.experts.linear_fc1.weight1 - module.decoder.layers.11.mlp.experts.linear_fc2.weight11 - module.decoder.layers.11.mlp.experts.linear_fc2.weight3 - module.decoder.layers.12.mlp.experts.linear_fc1.weight0 - module.decoder.layers.11.mlp.experts.linear_fc2.weight8 - module.decoder.layers.12.mlp.experts.linear_fc1.weight5 - module.decoder.layers.11.mlp.experts.linear_fc2.weight13 - module.decoder.layers.11.mlp.experts.linear_fc2.weight5 Params for bucket 68 (40894464 elements, 40894464 padded size): - module.decoder.layers.11.mlp.experts.linear_fc2.weight1 - module.decoder.layers.11.mlp.experts.linear_fc1.weight15 - module.decoder.layers.11.mlp.experts.linear_fc1.weight13 - module.decoder.layers.11.mlp.experts.linear_fc1.weight11 - module.decoder.layers.11.mlp.experts.linear_fc1.weight9 - module.decoder.layers.11.mlp.experts.linear_fc1.weight7 - module.decoder.layers.11.mlp.experts.linear_fc1.weight5 module.decoder.layers.11.mlp.experts.linear_fc2.weight0 module.decoder.layers.11.mlp.experts.linear_fc1.weight14 module.decoder.layers.11.mlp.experts.linear_fc1.weight12 @@ -2255,7 +2248,22 @@ Params for bucket 68 (40894464 elements, 40894464 padded size): module.decoder.layers.11.mlp.experts.linear_fc1.weight8 module.decoder.layers.11.mlp.experts.linear_fc1.weight6 module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight13 + module.decoder.layers.11.mlp.experts.linear_fc1.weight15 + module.decoder.layers.11.mlp.experts.linear_fc1.weight11 + module.decoder.layers.11.mlp.experts.linear_fc1.weight9 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 Params for bucket 69 (40894464 elements, 40894464 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight12 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight9 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight14 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 module.decoder.layers.10.mlp.experts.linear_fc2.weight11 module.decoder.layers.10.mlp.experts.linear_fc2.weight3 module.decoder.layers.11.mlp.experts.linear_fc1.weight3 @@ -2269,49 +2277,41 @@ Params for bucket 69 (40894464 elements, 40894464 padded size): module.decoder.layers.11.mlp.experts.linear_fc1.weight2 module.decoder.layers.10.mlp.experts.linear_fc2.weight7 module.decoder.layers.10.mlp.experts.linear_fc2.weight2 - module.decoder.layers.10.mlp.experts.linear_fc2.weight12 - module.decoder.layers.10.mlp.experts.linear_fc2.weight4 - module.decoder.layers.11.mlp.experts.linear_fc1.weight1 - module.decoder.layers.10.mlp.experts.linear_fc2.weight9 - module.decoder.layers.10.mlp.experts.linear_fc2.weight0 - module.decoder.layers.11.mlp.experts.linear_fc1.weight0 - module.decoder.layers.10.mlp.experts.linear_fc2.weight14 - module.decoder.layers.10.mlp.experts.linear_fc2.weight6 Params for bucket 70 (40894464 elements, 40894464 padded size): - module.decoder.layers.10.mlp.experts.linear_fc1.weight14 - module.decoder.layers.10.mlp.experts.linear_fc1.weight10 - module.decoder.layers.10.mlp.experts.linear_fc1.weight12 - module.decoder.layers.10.mlp.experts.linear_fc1.weight8 - module.decoder.layers.10.mlp.experts.linear_fc1.weight6 - module.decoder.layers.10.mlp.experts.linear_fc1.weight4 - module.decoder.layers.10.mlp.experts.linear_fc1.weight3 module.decoder.layers.10.mlp.experts.linear_fc1.weight11 module.decoder.layers.10.mlp.experts.linear_fc1.weight13 module.decoder.layers.10.mlp.experts.linear_fc1.weight9 module.decoder.layers.10.mlp.experts.linear_fc1.weight7 module.decoder.layers.10.mlp.experts.linear_fc1.weight5 module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight14 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 + module.decoder.layers.10.mlp.experts.linear_fc1.weight10 + module.decoder.layers.10.mlp.experts.linear_fc1.weight8 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 Params for bucket 71 (40894464 elements, 40894464 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 module.decoder.layers.9.mlp.experts.linear_fc2.weight8 module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight15 module.decoder.layers.9.mlp.experts.linear_fc2.weight13 module.decoder.layers.9.mlp.experts.linear_fc2.weight5 - module.decoder.layers.9.mlp.experts.linear_fc2.weight0 module.decoder.layers.9.mlp.experts.linear_fc2.weight10 module.decoder.layers.9.mlp.experts.linear_fc2.weight2 - module.decoder.layers.9.mlp.experts.linear_fc1.weight15 + module.decoder.layers.9.mlp.experts.linear_fc1.weight14 module.decoder.layers.9.mlp.experts.linear_fc2.weight15 module.decoder.layers.9.mlp.experts.linear_fc2.weight7 module.decoder.layers.9.mlp.experts.linear_fc2.weight12 module.decoder.layers.9.mlp.experts.linear_fc2.weight4 - module.decoder.layers.9.mlp.experts.linear_fc1.weight14 - module.decoder.layers.10.mlp.experts.linear_fc1.weight1 module.decoder.layers.9.mlp.experts.linear_fc2.weight9 - module.decoder.layers.10.mlp.experts.linear_fc1.weight0 - module.decoder.layers.9.mlp.experts.linear_fc2.weight14 - module.decoder.layers.9.mlp.experts.linear_fc2.weight6 - module.decoder.layers.9.mlp.experts.linear_fc2.weight11 - module.decoder.layers.9.mlp.experts.linear_fc2.weight3 module.decoder.layers.9.mlp.experts.linear_fc1.weight13 Params for bucket 72 (40894464 elements, 40894464 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight12 @@ -2328,84 +2328,92 @@ Params for bucket 72 (40894464 elements, 40894464 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight3 module.decoder.layers.9.mlp.experts.linear_fc1.weight0 Params for bucket 73 (40894464 elements, 40894464 padded size): - module.decoder.layers.8.mlp.experts.linear_fc2.weight11 - module.decoder.layers.8.mlp.experts.linear_fc2.weight8 - module.decoder.layers.8.mlp.experts.linear_fc1.weight15 module.decoder.layers.8.mlp.experts.linear_fc2.weight5 - module.decoder.layers.8.mlp.experts.linear_fc1.weight12 + module.decoder.layers.8.mlp.experts.linear_fc1.weight11 module.decoder.layers.8.mlp.experts.linear_fc2.weight14 - module.decoder.layers.8.mlp.experts.linear_fc2.weight10 module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight11 module.decoder.layers.8.mlp.experts.linear_fc2.weight7 module.decoder.layers.8.mlp.experts.linear_fc2.weight2 - module.decoder.layers.8.mlp.experts.linear_fc1.weight14 - module.decoder.layers.8.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc1.weight13 module.decoder.layers.8.mlp.experts.linear_fc2.weight4 - module.decoder.layers.8.mlp.experts.linear_fc1.weight11 - module.decoder.layers.8.mlp.experts.linear_fc2.weight9 + module.decoder.layers.8.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight10 module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight15 module.decoder.layers.8.mlp.experts.linear_fc2.weight6 - module.decoder.layers.8.mlp.experts.linear_fc1.weight13 + module.decoder.layers.8.mlp.experts.linear_fc1.weight12 module.decoder.layers.8.mlp.experts.linear_fc2.weight15 - module.decoder.layers.8.mlp.experts.linear_fc2.weight12 module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight9 + module.decoder.layers.8.mlp.experts.linear_fc2.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight14 Params for bucket 74 (40894464 elements, 40894464 padded size): - module.decoder.layers.8.mlp.experts.linear_fc1.weight7 module.decoder.layers.8.mlp.experts.linear_fc1.weight9 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 module.decoder.layers.8.mlp.experts.linear_fc1.weight5 module.decoder.layers.8.mlp.experts.linear_fc1.weight2 module.decoder.layers.8.mlp.experts.linear_fc1.weight1 - module.decoder.layers.7.mlp.experts.linear_fc2.weight15 - module.decoder.layers.7.mlp.experts.linear_fc2.weight13 + module.decoder.layers.7.mlp.experts.linear_fc2.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight12 module.decoder.layers.8.mlp.experts.linear_fc1.weight10 - module.decoder.layers.8.mlp.experts.linear_fc1.weight8 module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight8 module.decoder.layers.8.mlp.experts.linear_fc1.weight4 module.decoder.layers.8.mlp.experts.linear_fc1.weight3 module.decoder.layers.8.mlp.experts.linear_fc1.weight0 - module.decoder.layers.7.mlp.experts.linear_fc2.weight14 - module.decoder.layers.7.mlp.experts.linear_fc2.weight12 + module.decoder.layers.7.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 Params for bucket 75 (40894464 elements, 40894464 padded size): - module.decoder.layers.7.mlp.experts.linear_fc2.weight11 - module.decoder.layers.7.mlp.experts.linear_fc2.weight3 - module.decoder.layers.7.mlp.experts.linear_fc1.weight9 - module.decoder.layers.7.mlp.experts.linear_fc2.weight8 - module.decoder.layers.7.mlp.experts.linear_fc2.weight1 - module.decoder.layers.7.mlp.experts.linear_fc1.weight14 - module.decoder.layers.7.mlp.experts.linear_fc2.weight5 - module.decoder.layers.7.mlp.experts.linear_fc1.weight11 module.decoder.layers.7.mlp.experts.linear_fc2.weight10 module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight9 module.decoder.layers.7.mlp.experts.linear_fc2.weight7 - module.decoder.layers.7.mlp.experts.linear_fc1.weight13 + module.decoder.layers.7.mlp.experts.linear_fc1.weight14 module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight11 + module.decoder.layers.7.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight13 + module.decoder.layers.7.mlp.experts.linear_fc2.weight11 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 module.decoder.layers.7.mlp.experts.linear_fc2.weight0 module.decoder.layers.7.mlp.experts.linear_fc1.weight10 - module.decoder.layers.7.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 module.decoder.layers.7.mlp.experts.linear_fc1.weight15 - module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 module.decoder.layers.7.mlp.experts.linear_fc1.weight12 Params for bucket 76 (40894464 elements, 40894464 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight7 module.decoder.layers.7.mlp.experts.linear_fc1.weight5 module.decoder.layers.7.mlp.experts.linear_fc1.weight2 module.decoder.layers.7.mlp.experts.linear_fc1.weight1 - module.decoder.layers.6.mlp.experts.linear_fc2.weight14 - module.decoder.layers.6.mlp.experts.linear_fc2.weight12 - module.decoder.layers.6.mlp.experts.linear_fc2.weight10 - module.decoder.layers.6.mlp.experts.linear_fc2.weight8 + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 + module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc2.weight9 module.decoder.layers.7.mlp.experts.linear_fc1.weight8 module.decoder.layers.7.mlp.experts.linear_fc1.weight6 module.decoder.layers.7.mlp.experts.linear_fc1.weight4 module.decoder.layers.7.mlp.experts.linear_fc1.weight3 module.decoder.layers.7.mlp.experts.linear_fc1.weight0 - module.decoder.layers.6.mlp.experts.linear_fc2.weight15 - module.decoder.layers.6.mlp.experts.linear_fc2.weight13 - module.decoder.layers.6.mlp.experts.linear_fc2.weight11 - module.decoder.layers.6.mlp.experts.linear_fc2.weight9 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight8 Params for bucket 77 (40894464 elements, 40894464 padded size): - module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight14 + module.decoder.layers.6.mlp.experts.linear_fc1.weight12 + module.decoder.layers.6.mlp.experts.linear_fc1.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight8 module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 module.decoder.layers.6.mlp.experts.linear_fc2.weight2 module.decoder.layers.6.mlp.experts.linear_fc2.weight1 module.decoder.layers.6.mlp.experts.linear_fc1.weight15 @@ -2413,35 +2421,35 @@ Params for bucket 77 (40894464 elements, 40894464 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight11 module.decoder.layers.6.mlp.experts.linear_fc1.weight9 module.decoder.layers.6.mlp.experts.linear_fc1.weight7 - module.decoder.layers.6.mlp.experts.linear_fc2.weight7 - module.decoder.layers.6.mlp.experts.linear_fc2.weight3 - module.decoder.layers.6.mlp.experts.linear_fc2.weight5 - module.decoder.layers.6.mlp.experts.linear_fc2.weight0 - module.decoder.layers.6.mlp.experts.linear_fc1.weight14 - module.decoder.layers.6.mlp.experts.linear_fc1.weight12 - module.decoder.layers.6.mlp.experts.linear_fc1.weight10 - module.decoder.layers.6.mlp.experts.linear_fc1.weight8 Params for bucket 78 (40894464 elements, 40894464 padded size): - module.decoder.layers.6.mlp.experts.linear_fc1.weight2 - module.decoder.layers.5.mlp.experts.linear_fc2.weight9 - module.decoder.layers.6.mlp.experts.linear_fc1.weight6 - module.decoder.layers.5.mlp.experts.linear_fc2.weight14 - module.decoder.layers.5.mlp.experts.linear_fc2.weight6 - module.decoder.layers.6.mlp.experts.linear_fc1.weight1 - module.decoder.layers.5.mlp.experts.linear_fc2.weight11 - module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 module.decoder.layers.5.mlp.experts.linear_fc2.weight8 - module.decoder.layers.6.mlp.experts.linear_fc1.weight5 module.decoder.layers.5.mlp.experts.linear_fc2.weight13 module.decoder.layers.5.mlp.experts.linear_fc2.weight5 - module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 module.decoder.layers.5.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 module.decoder.layers.5.mlp.experts.linear_fc2.weight15 module.decoder.layers.5.mlp.experts.linear_fc2.weight7 - module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 module.decoder.layers.5.mlp.experts.linear_fc2.weight12 module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight14 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 Params for bucket 79 (40894464 elements, 40894464 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 + module.decoder.layers.5.mlp.experts.linear_fc1.weight11 + module.decoder.layers.5.mlp.experts.linear_fc1.weight9 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 module.decoder.layers.5.mlp.experts.linear_fc2.weight3 module.decoder.layers.5.mlp.experts.linear_fc2.weight0 module.decoder.layers.5.mlp.experts.linear_fc1.weight14 @@ -2449,24 +2457,7 @@ Params for bucket 79 (40894464 elements, 40894464 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight10 module.decoder.layers.5.mlp.experts.linear_fc1.weight8 module.decoder.layers.5.mlp.experts.linear_fc1.weight6 - module.decoder.layers.5.mlp.experts.linear_fc2.weight2 - module.decoder.layers.5.mlp.experts.linear_fc1.weight15 - module.decoder.layers.5.mlp.experts.linear_fc2.weight1 - module.decoder.layers.5.mlp.experts.linear_fc1.weight13 - module.decoder.layers.5.mlp.experts.linear_fc1.weight11 - module.decoder.layers.5.mlp.experts.linear_fc1.weight9 - module.decoder.layers.5.mlp.experts.linear_fc1.weight7 - module.decoder.layers.5.mlp.experts.linear_fc1.weight5 Params for bucket 80 (40894464 elements, 40894464 padded size): - module.decoder.layers.5.mlp.experts.linear_fc1.weight0 - module.decoder.layers.4.mlp.experts.linear_fc2.weight14 - module.decoder.layers.4.mlp.experts.linear_fc2.weight6 - module.decoder.layers.4.mlp.experts.linear_fc2.weight11 - module.decoder.layers.4.mlp.experts.linear_fc2.weight3 - module.decoder.layers.4.mlp.experts.linear_fc2.weight0 - module.decoder.layers.5.mlp.experts.linear_fc1.weight3 - module.decoder.layers.4.mlp.experts.linear_fc2.weight8 - module.decoder.layers.4.mlp.experts.linear_fc2.weight1 module.decoder.layers.4.mlp.experts.linear_fc2.weight13 module.decoder.layers.4.mlp.experts.linear_fc2.weight5 module.decoder.layers.5.mlp.experts.linear_fc1.weight4 @@ -2479,13 +2470,16 @@ Params for bucket 80 (40894464 elements, 40894464 padded size): module.decoder.layers.4.mlp.experts.linear_fc2.weight4 module.decoder.layers.5.mlp.experts.linear_fc1.weight1 module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight14 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight11 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 Params for bucket 81 (40894464 elements, 40894464 padded size): - module.decoder.layers.4.mlp.experts.linear_fc1.weight15 - module.decoder.layers.4.mlp.experts.linear_fc1.weight11 - module.decoder.layers.4.mlp.experts.linear_fc1.weight13 - module.decoder.layers.4.mlp.experts.linear_fc1.weight9 - module.decoder.layers.4.mlp.experts.linear_fc1.weight7 - module.decoder.layers.4.mlp.experts.linear_fc1.weight5 module.decoder.layers.4.mlp.experts.linear_fc1.weight14 module.decoder.layers.4.mlp.experts.linear_fc1.weight12 module.decoder.layers.4.mlp.experts.linear_fc1.weight10 @@ -2493,28 +2487,34 @@ Params for bucket 81 (40894464 elements, 40894464 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight6 module.decoder.layers.4.mlp.experts.linear_fc1.weight4 module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 + module.decoder.layers.4.mlp.experts.linear_fc1.weight13 + module.decoder.layers.4.mlp.experts.linear_fc1.weight11 + module.decoder.layers.4.mlp.experts.linear_fc1.weight9 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 Params for bucket 82 (40894464 elements, 40894464 padded size): - module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight13 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight10 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight15 module.decoder.layers.3.mlp.experts.linear_fc2.weight15 module.decoder.layers.3.mlp.experts.linear_fc2.weight7 - module.decoder.layers.3.mlp.experts.linear_fc2.weight2 - module.decoder.layers.3.mlp.experts.linear_fc1.weight14 module.decoder.layers.3.mlp.experts.linear_fc2.weight12 module.decoder.layers.3.mlp.experts.linear_fc2.weight4 - module.decoder.layers.4.mlp.experts.linear_fc1.weight1 module.decoder.layers.3.mlp.experts.linear_fc2.weight9 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 module.decoder.layers.3.mlp.experts.linear_fc2.weight0 - module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight14 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 module.decoder.layers.3.mlp.experts.linear_fc2.weight14 module.decoder.layers.3.mlp.experts.linear_fc2.weight6 module.decoder.layers.3.mlp.experts.linear_fc2.weight11 module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 module.decoder.layers.3.mlp.experts.linear_fc2.weight8 - module.decoder.layers.3.mlp.experts.linear_fc1.weight15 - module.decoder.layers.3.mlp.experts.linear_fc2.weight13 - module.decoder.layers.3.mlp.experts.linear_fc2.weight5 - module.decoder.layers.3.mlp.experts.linear_fc2.weight10 - module.decoder.layers.3.mlp.experts.linear_fc2.weight1 Params for bucket 83 (40894464 elements, 40894464 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight12 module.decoder.layers.3.mlp.experts.linear_fc1.weight10 @@ -2530,49 +2530,43 @@ Params for bucket 83 (40894464 elements, 40894464 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight2 module.decoder.layers.3.mlp.experts.linear_fc1.weight1 Params for bucket 84 (40894464 elements, 40894464 padded size): - module.decoder.layers.2.mlp.experts.linear_fc2.weight9 - module.decoder.layers.2.mlp.experts.linear_fc1.weight15 module.decoder.layers.2.mlp.experts.linear_fc2.weight14 module.decoder.layers.2.mlp.experts.linear_fc2.weight6 - module.decoder.layers.2.mlp.experts.linear_fc1.weight12 - module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight13 module.decoder.layers.2.mlp.experts.linear_fc2.weight11 module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 module.decoder.layers.2.mlp.experts.linear_fc2.weight8 module.decoder.layers.2.mlp.experts.linear_fc2.weight1 - module.decoder.layers.2.mlp.experts.linear_fc1.weight14 + module.decoder.layers.2.mlp.experts.linear_fc1.weight15 module.decoder.layers.2.mlp.experts.linear_fc2.weight13 module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight12 module.decoder.layers.2.mlp.experts.linear_fc2.weight10 module.decoder.layers.2.mlp.experts.linear_fc2.weight2 module.decoder.layers.2.mlp.experts.linear_fc2.weight15 module.decoder.layers.2.mlp.experts.linear_fc2.weight7 - module.decoder.layers.2.mlp.experts.linear_fc1.weight13 + module.decoder.layers.2.mlp.experts.linear_fc1.weight14 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 module.decoder.layers.2.mlp.experts.linear_fc2.weight12 module.decoder.layers.2.mlp.experts.linear_fc2.weight4 - module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 Params for bucket 85 (40894464 elements, 40894464 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight11 - module.decoder.layers.2.mlp.experts.linear_fc1.weight7 module.decoder.layers.2.mlp.experts.linear_fc1.weight9 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 module.decoder.layers.2.mlp.experts.linear_fc1.weight5 module.decoder.layers.2.mlp.experts.linear_fc1.weight2 module.decoder.layers.2.mlp.experts.linear_fc1.weight1 - module.decoder.layers.1.mlp.experts.linear_fc2.weight14 - module.decoder.layers.2.mlp.experts.linear_fc1.weight10 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 module.decoder.layers.2.mlp.experts.linear_fc1.weight8 + module.decoder.layers.2.mlp.experts.linear_fc1.weight10 module.decoder.layers.2.mlp.experts.linear_fc1.weight6 module.decoder.layers.2.mlp.experts.linear_fc1.weight4 module.decoder.layers.2.mlp.experts.linear_fc1.weight3 module.decoder.layers.2.mlp.experts.linear_fc1.weight0 - module.decoder.layers.1.mlp.experts.linear_fc2.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 Params for bucket 86 (40894464 elements, 40894464 padded size): - module.decoder.layers.1.mlp.experts.linear_fc2.weight12 - module.decoder.layers.1.mlp.experts.linear_fc2.weight4 - module.decoder.layers.1.mlp.experts.linear_fc1.weight15 - module.decoder.layers.1.mlp.experts.linear_fc2.weight9 - module.decoder.layers.1.mlp.experts.linear_fc1.weight12 - module.decoder.layers.1.mlp.experts.linear_fc2.weight6 module.decoder.layers.1.mlp.experts.linear_fc2.weight11 module.decoder.layers.1.mlp.experts.linear_fc2.weight3 module.decoder.layers.1.mlp.experts.linear_fc2.weight0 @@ -2587,7 +2581,21 @@ Params for bucket 86 (40894464 elements, 40894464 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight13 module.decoder.layers.1.mlp.experts.linear_fc2.weight7 module.decoder.layers.1.mlp.experts.linear_fc1.weight10 + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 Params for bucket 87 (40894464 elements, 40894464 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 + module.decoder.layers.0.mlp.experts.linear_fc2.weight13 + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 module.decoder.layers.1.mlp.experts.linear_fc1.weight9 module.decoder.layers.1.mlp.experts.linear_fc1.weight5 module.decoder.layers.1.mlp.experts.linear_fc1.weight7 @@ -2596,45 +2604,37 @@ Params for bucket 87 (40894464 elements, 40894464 padded size): module.decoder.layers.0.mlp.experts.linear_fc2.weight14 module.decoder.layers.0.mlp.experts.linear_fc2.weight12 module.decoder.layers.0.mlp.experts.linear_fc2.weight10 - module.decoder.layers.1.mlp.experts.linear_fc1.weight6 - module.decoder.layers.1.mlp.experts.linear_fc1.weight8 - module.decoder.layers.1.mlp.experts.linear_fc1.weight4 - module.decoder.layers.1.mlp.experts.linear_fc1.weight3 - module.decoder.layers.1.mlp.experts.linear_fc1.weight0 - module.decoder.layers.0.mlp.experts.linear_fc2.weight15 - module.decoder.layers.0.mlp.experts.linear_fc2.weight13 - module.decoder.layers.0.mlp.experts.linear_fc2.weight11 Params for bucket 88 (40894464 elements, 40894464 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 module.decoder.layers.0.mlp.experts.linear_fc2.weight5 - module.decoder.layers.0.mlp.experts.linear_fc2.weight8 - module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 module.decoder.layers.0.mlp.experts.linear_fc1.weight15 module.decoder.layers.0.mlp.experts.linear_fc1.weight13 module.decoder.layers.0.mlp.experts.linear_fc1.weight11 module.decoder.layers.0.mlp.experts.linear_fc1.weight9 - module.decoder.layers.0.mlp.experts.linear_fc1.weight8 - module.decoder.layers.0.mlp.experts.linear_fc2.weight9 - module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight8 module.decoder.layers.0.mlp.experts.linear_fc2.weight6 - module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 module.decoder.layers.0.mlp.experts.linear_fc2.weight2 module.decoder.layers.0.mlp.experts.linear_fc2.weight1 module.decoder.layers.0.mlp.experts.linear_fc2.weight0 module.decoder.layers.0.mlp.experts.linear_fc1.weight14 module.decoder.layers.0.mlp.experts.linear_fc1.weight12 module.decoder.layers.0.mlp.experts.linear_fc1.weight10 + module.decoder.layers.0.mlp.experts.linear_fc1.weight8 Params for bucket 89 (25165824 elements, 25165824 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight7 module.decoder.layers.0.mlp.experts.linear_fc1.weight5 module.decoder.layers.0.mlp.experts.linear_fc1.weight2 module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 module.decoder.layers.0.mlp.experts.linear_fc1.weight6 module.decoder.layers.0.mlp.experts.linear_fc1.weight4 - module.decoder.layers.0.mlp.experts.linear_fc1.weight3 module.decoder.layers.0.mlp.experts.linear_fc1.weight0 -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine -[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 15:07:24 +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 15:55:24 > building train, validation, and test datasets ... > datasets target sizes (minimum size): train: 96 @@ -2642,7 +2642,7 @@ INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine test: 0 INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] > building train, validation, and test datasets for GPT ... -INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers @@ -2655,14 +2655,14 @@ INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 466b63c71b42 INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-shuffle_index.npy INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 > finished creating GPT datasets ... -[after dataloaders are built] datetime: 2026-08-26 15:07:24 +[after dataloaders are built] datetime: 2026-08-26 15:55:24 done with setup ... -training ... (min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (6316.47, 6363.05) - train/valid/test-data-iterators-setup ..........: (56.55, 67.58) + model-and-optimizer-setup ......................: (6409.42, 6447.87) + train/valid/test-data-iterators-setup ..........: (57.62, 70.27) +training ... Overwriting rerun_state_machine.current_iteration from -1 to 0... -[before the start of training step] datetime: 2026-08-26 15:07:24 +[before the start of training step] datetime: 2026-08-26 15:55:24 INFO:moonep.api:MoonEP comm buffer sizes: config : R=8, S=4096, H=2048, E=128, K=8, NvS=36832, NvS_padded=36864 sm_count : num_sms=32, num_sms_dedup=148 @@ -2671,15 +2671,15 @@ INFO:moonep.api:MoonEP comm buffer sizes: local_temp_buffer_size : 1.26 MiB total_size : 147.26 MiB WARNING:megatron.core.rerun_state_machine:Result validation enabled - [2026-08-26 15:07:51] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 27256.9 | throughput per GPU (TFLOP/s/GPU): 3.3 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.095192E+01 | loss scale: 1.0 | grad norm: 54.489 | number of skipped iterations: 0 | number of nan iterations: 0 | -Number of parameters in transformer block in billions: 29.90 + [2026-08-26 15:55:49] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 25332.8 | throughput per GPU (TFLOP/s/GPU): 3.5 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.095192E+01 | loss scale: 1.0 | grad norm: 54.492 | number of skipped iterations: 0 | number of nan iterations: 0 |Number of parameters in transformer block in billions: 29.90 + Number of parameters in embedding layers in billions: 0.21 Total number of parameters in billions: 30.10 Number of parameters in most loaded shard in billions: 30.1034 Theoretical memory footprints: weight and optimizer=215316.59 MB -[Rank 0] (after 1 iterations) memory (MB) | allocated: 139095.5478515625 | max allocated: 139095.54833984375 | reserved: 140448.0 | max reserved: 140448.0 -WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 144.00 MiB. GPU 2 has a total capacity of 178.35 GiB of which 126.81 MiB is free. Including non-PyTorch memory, this process has 178.21 GiB memory in use. Of the allocated memory 150.05 GiB is allocated by PyTorch, and 1.41 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -['Traceback (most recent call last):\n', ' File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 541, in dispatch_and_compute\n output = self.combine(output, metadata, shared_expert_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 333, in combine\n output = self.token_dispatcher.token_combine(output, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/token_dispatcher.py", line 1773, in token_combine\n return self._comm_manager.combine(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/token_dispatcher.py", line 1516, in combine\n hidden_states = zero_moonep_tail(hidden_states, metadata.cu_seqlens)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/fused_moonep.py", line 367, in zero_moonep_tail\n return torch.where(keep, x, x.new_zeros(()))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 144.00 MiB. GPU 2 has a total capacity of 178.35 GiB of which 126.81 MiB is free. Including non-PyTorch memory, this process has 178.21 GiB memory in use. Of the allocated memory 150.05 GiB is allocated by PyTorch, and 1.41 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] +[Rank 0] (after 1 iterations) memory (MB) | allocated: 139094.9228515625 | max allocated: 139094.92333984375 | reserved: 140448.0 | max reserved: 140448.0 +WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 144.00 MiB. GPU 2 has a total capacity of 178.35 GiB of which 112.81 MiB is free. Including non-PyTorch memory, this process has 178.23 GiB memory in use. Of the allocated memory 150.05 GiB is allocated by PyTorch, and 1.43 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 541, in dispatch_and_compute\n output = self.combine(output, metadata, shared_expert_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 333, in combine\n output = self.token_dispatcher.token_combine(output, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/token_dispatcher.py", line 1773, in token_combine\n return self._comm_manager.combine(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/token_dispatcher.py", line 1516, in combine\n hidden_states = zero_moonep_tail(hidden_states, metadata.cu_seqlens)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/fused_moonep.py", line 367, in zero_moonep_tail\n return torch.where(keep, x, x.new_zeros(()))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 144.00 MiB. GPU 2 has a total capacity of 178.35 GiB of which 112.81 MiB is free. Including non-PyTorch memory, this process has 178.23 GiB memory in use. Of the allocated memory 150.05 GiB is allocated by PyTorch, and 1.43 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] [rank2]: Traceback (most recent call last): [rank2]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 394, in [rank2]: pretrain( @@ -2791,150 +2791,150 @@ WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 144.00 MiB. GP [rank2]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/fused_moonep.py", line 367, in zero_moonep_tail [rank2]: return torch.where(keep, x, x.new_zeros(())) [rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 144.00 MiB. GPU 2 has a total capacity of 178.35 GiB of which 126.81 MiB is free. Including non-PyTorch memory, this process has 178.21 GiB memory in use. Of the allocated memory 150.05 GiB is allocated by PyTorch, and 1.41 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 144.00 MiB. GPU 4 has a total capacity of 178.35 GiB of which 120.81 MiB is free. Including non-PyTorch memory, this process has 178.22 GiB memory in use. Of the allocated memory 149.95 GiB is allocated by PyTorch, and 1.52 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -['Traceback (most recent call last):\n', ' File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute\n output, shared_expert_output, mlp_bias = self.experts_compute(\n ^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute\n expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 932, in forward\n output, output_bias = self.linear_fc2(intermediate_parallel, tokens_per_expert)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward\n out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn\n return fn(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward\n out = linear_fn(*args)\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward\n out = torch.empty(\n ^^^^^^^^^^^^\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 144.00 MiB. GPU 4 has a total capacity of 178.35 GiB of which 120.81 MiB is free. Including non-PyTorch memory, this process has 178.22 GiB memory in use. Of the allocated memory 149.95 GiB is allocated by PyTorch, and 1.52 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] -[rank4]: Traceback (most recent call last): -[rank4]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 394, in -[rank4]: pretrain( -[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 917, in pretrain -[rank4]: iteration, num_floating_point_operations_so_far = train( -[rank4]: ^^^^^^ -[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 2326, in train -[rank4]: ) = train_step( -[rank4]: ^^^^^^^^^^^ -[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 1433, in train_step -[rank4]: losses_reduced = forward_backward_func( -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining -[rank4]: output_tensor, num_tokens = forward_step( -[rank4]: ^^^^^^^^^^^^^ -[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step -[rank4]: output_tensor, loss_func = forward_step_func(data_iterator, model) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step -[rank4]: output_tensor = model( -[rank4]: ^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank4]: return self._call_impl(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank4]: return forward_call(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward -[rank4]: return self.module(*inputs, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank4]: return self._call_impl(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank4]: return inner() -[rank4]: ^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank4]: result = forward_call(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward -[rank4]: outputs = self.module(*inputs, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank4]: return self._call_impl(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank4]: return inner() -[rank4]: ^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank4]: result = forward_call(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward -[rank4]: hidden_states = self.decoder( -[rank4]: ^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank4]: return self._call_impl(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank4]: return inner() -[rank4]: ^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank4]: result = forward_call(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward -[rank4]: hidden_states, context = layer( -[rank4]: ^^^^^^ -[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__ -[rank4]: return super(MegatronModule, self).__call__(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank4]: return self._call_impl(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank4]: return inner() -[rank4]: ^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank4]: result = forward_call(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward -[rank4]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp -[rank4]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank4]: return self._call_impl(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank4]: return inner() -[rank4]: ^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank4]: result = forward_call(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward -[rank4]: output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute -[rank4]: output, shared_expert_output, mlp_bias = self.experts_compute( -[rank4]: ^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute -[rank4]: expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank4]: return self._call_impl(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank4]: return inner() -[rank4]: ^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank4]: result = forward_call(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 932, in forward -[rank4]: output, output_bias = self.linear_fc2(intermediate_parallel, tokens_per_expert) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank4]: return self._call_impl(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank4]: return inner() -[rank4]: ^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank4]: result = forward_call(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward -[rank4]: out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn -[rank4]: return fn(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward -[rank4]: out = linear_fn(*args) -[rank4]: ^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply -[rank4]: return super().apply(*args, **kwargs) # type: ignore[misc] -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward -[rank4]: out = torch.empty( -[rank4]: ^^^^^^^^^^^^ -[rank4]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 144.00 MiB. GPU 4 has a total capacity of 178.35 GiB of which 120.81 MiB is free. Including non-PyTorch memory, this process has 178.22 GiB memory in use. Of the allocated memory 149.95 GiB is allocated by PyTorch, and 1.52 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 144.00 MiB. GPU 7 has a total capacity of 178.35 GiB of which 126.81 MiB is free. Including non-PyTorch memory, this process has 178.21 GiB memory in use. Of the allocated memory 150.05 GiB is allocated by PyTorch, and 1.41 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -['Traceback (most recent call last):\n', ' File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 541, in dispatch_and_compute\n output = self.combine(output, metadata, shared_expert_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 333, in combine\n output = self.token_dispatcher.token_combine(output, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/token_dispatcher.py", line 1773, in token_combine\n return self._comm_manager.combine(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/token_dispatcher.py", line 1516, in combine\n hidden_states = zero_moonep_tail(hidden_states, metadata.cu_seqlens)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/fused_moonep.py", line 367, in zero_moonep_tail\n return torch.where(keep, x, x.new_zeros(()))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 144.00 MiB. GPU 7 has a total capacity of 178.35 GiB of which 126.81 MiB is free. Including non-PyTorch memory, this process has 178.21 GiB memory in use. Of the allocated memory 150.05 GiB is allocated by PyTorch, and 1.41 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] +[rank2]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 144.00 MiB. GPU 2 has a total capacity of 178.35 GiB of which 112.81 MiB is free. Including non-PyTorch memory, this process has 178.23 GiB memory in use. Of the allocated memory 150.05 GiB is allocated by PyTorch, and 1.43 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 144.00 MiB. GPU 6 has a total capacity of 178.35 GiB of which 86.81 MiB is free. Including non-PyTorch memory, this process has 178.25 GiB memory in use. Of the allocated memory 149.95 GiB is allocated by PyTorch, and 1.56 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute\n output, shared_expert_output, mlp_bias = self.experts_compute(\n ^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute\n expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 932, in forward\n output, output_bias = self.linear_fc2(intermediate_parallel, tokens_per_expert)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward\n out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn\n return fn(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward\n out = linear_fn(*args)\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward\n out = torch.empty(\n ^^^^^^^^^^^^\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 144.00 MiB. GPU 6 has a total capacity of 178.35 GiB of which 86.81 MiB is free. Including non-PyTorch memory, this process has 178.25 GiB memory in use. Of the allocated memory 149.95 GiB is allocated by PyTorch, and 1.56 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] +[rank6]: Traceback (most recent call last): +[rank6]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 394, in +[rank6]: pretrain( +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 917, in pretrain +[rank6]: iteration, num_floating_point_operations_so_far = train( +[rank6]: ^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 2326, in train +[rank6]: ) = train_step( +[rank6]: ^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 1433, in train_step +[rank6]: losses_reduced = forward_backward_func( +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining +[rank6]: output_tensor, num_tokens = forward_step( +[rank6]: ^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step +[rank6]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step +[rank6]: output_tensor = model( +[rank6]: ^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank6]: return forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward +[rank6]: return self.module(*inputs, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank6]: return inner() +[rank6]: ^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank6]: result = forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward +[rank6]: outputs = self.module(*inputs, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank6]: return inner() +[rank6]: ^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank6]: result = forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward +[rank6]: hidden_states = self.decoder( +[rank6]: ^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank6]: return inner() +[rank6]: ^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank6]: result = forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward +[rank6]: hidden_states, context = layer( +[rank6]: ^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__ +[rank6]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank6]: return inner() +[rank6]: ^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank6]: result = forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward +[rank6]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp +[rank6]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank6]: return inner() +[rank6]: ^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank6]: result = forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward +[rank6]: output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute +[rank6]: output, shared_expert_output, mlp_bias = self.experts_compute( +[rank6]: ^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute +[rank6]: expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank6]: return inner() +[rank6]: ^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank6]: result = forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 932, in forward +[rank6]: output, output_bias = self.linear_fc2(intermediate_parallel, tokens_per_expert) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank6]: return inner() +[rank6]: ^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank6]: result = forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward +[rank6]: out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn +[rank6]: return fn(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward +[rank6]: out = linear_fn(*args) +[rank6]: ^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply +[rank6]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward +[rank6]: out = torch.empty( +[rank6]: ^^^^^^^^^^^^ +[rank6]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 144.00 MiB. GPU 6 has a total capacity of 178.35 GiB of which 86.81 MiB is free. Including non-PyTorch memory, this process has 178.25 GiB memory in use. Of the allocated memory 149.95 GiB is allocated by PyTorch, and 1.56 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 144.00 MiB. GPU 7 has a total capacity of 178.35 GiB of which 112.81 MiB is free. Including non-PyTorch memory, this process has 178.23 GiB memory in use. Of the allocated memory 150.05 GiB is allocated by PyTorch, and 1.43 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 541, in dispatch_and_compute\n output = self.combine(output, metadata, shared_expert_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 333, in combine\n output = self.token_dispatcher.token_combine(output, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/token_dispatcher.py", line 1773, in token_combine\n return self._comm_manager.combine(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/token_dispatcher.py", line 1516, in combine\n hidden_states = zero_moonep_tail(hidden_states, metadata.cu_seqlens)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/fused_moonep.py", line 367, in zero_moonep_tail\n return torch.where(keep, x, x.new_zeros(()))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 144.00 MiB. GPU 7 has a total capacity of 178.35 GiB of which 112.81 MiB is free. Including non-PyTorch memory, this process has 178.23 GiB memory in use. Of the allocated memory 150.05 GiB is allocated by PyTorch, and 1.43 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] [rank7]: Traceback (most recent call last): [rank7]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 394, in [rank7]: pretrain( @@ -3046,9 +3046,9 @@ WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 144.00 MiB. GP [rank7]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/fused_moonep.py", line 367, in zero_moonep_tail [rank7]: return torch.where(keep, x, x.new_zeros(())) [rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank7]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 144.00 MiB. GPU 7 has a total capacity of 178.35 GiB of which 126.81 MiB is free. Including non-PyTorch memory, this process has 178.21 GiB memory in use. Of the allocated memory 150.05 GiB is allocated by PyTorch, and 1.41 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 144.00 MiB. GPU 5 has a total capacity of 178.35 GiB of which 126.81 MiB is free. Including non-PyTorch memory, this process has 178.21 GiB memory in use. Of the allocated memory 150.05 GiB is allocated by PyTorch, and 1.41 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -['Traceback (most recent call last):\n', ' File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 541, in dispatch_and_compute\n output = self.combine(output, metadata, shared_expert_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 333, in combine\n output = self.token_dispatcher.token_combine(output, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/token_dispatcher.py", line 1773, in token_combine\n return self._comm_manager.combine(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/token_dispatcher.py", line 1516, in combine\n hidden_states = zero_moonep_tail(hidden_states, metadata.cu_seqlens)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/fused_moonep.py", line 367, in zero_moonep_tail\n return torch.where(keep, x, x.new_zeros(()))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 144.00 MiB. GPU 5 has a total capacity of 178.35 GiB of which 126.81 MiB is free. Including non-PyTorch memory, this process has 178.21 GiB memory in use. Of the allocated memory 150.05 GiB is allocated by PyTorch, and 1.41 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] +[rank7]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 144.00 MiB. GPU 7 has a total capacity of 178.35 GiB of which 112.81 MiB is free. Including non-PyTorch memory, this process has 178.23 GiB memory in use. Of the allocated memory 150.05 GiB is allocated by PyTorch, and 1.43 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 144.00 MiB. GPU 5 has a total capacity of 178.35 GiB of which 112.81 MiB is free. Including non-PyTorch memory, this process has 178.23 GiB memory in use. Of the allocated memory 150.05 GiB is allocated by PyTorch, and 1.43 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 541, in dispatch_and_compute\n output = self.combine(output, metadata, shared_expert_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 333, in combine\n output = self.token_dispatcher.token_combine(output, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/token_dispatcher.py", line 1773, in token_combine\n return self._comm_manager.combine(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/token_dispatcher.py", line 1516, in combine\n hidden_states = zero_moonep_tail(hidden_states, metadata.cu_seqlens)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/fused_moonep.py", line 367, in zero_moonep_tail\n return torch.where(keep, x, x.new_zeros(()))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 144.00 MiB. GPU 5 has a total capacity of 178.35 GiB of which 112.81 MiB is free. Including non-PyTorch memory, this process has 178.23 GiB memory in use. Of the allocated memory 150.05 GiB is allocated by PyTorch, and 1.43 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] [rank5]: Traceback (most recent call last): [rank5]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 394, in [rank5]: pretrain( @@ -3160,9 +3160,9 @@ WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 144.00 MiB. GP [rank5]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/fused_moonep.py", line 367, in zero_moonep_tail [rank5]: return torch.where(keep, x, x.new_zeros(())) [rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 144.00 MiB. GPU 5 has a total capacity of 178.35 GiB of which 126.81 MiB is free. Including non-PyTorch memory, this process has 178.21 GiB memory in use. Of the allocated memory 150.05 GiB is allocated by PyTorch, and 1.41 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 144.00 MiB. GPU 0 has a total capacity of 178.35 GiB of which 144.81 MiB is free. Including non-PyTorch memory, this process has 178.20 GiB memory in use. Of the allocated memory 150.05 GiB is allocated by PyTorch, and 1.40 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -['Traceback (most recent call last):\n', ' File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 541, in dispatch_and_compute\n output = self.combine(output, metadata, shared_expert_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 333, in combine\n output = self.token_dispatcher.token_combine(output, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/token_dispatcher.py", line 1773, in token_combine\n return self._comm_manager.combine(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/token_dispatcher.py", line 1516, in combine\n hidden_states = zero_moonep_tail(hidden_states, metadata.cu_seqlens)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/fused_moonep.py", line 367, in zero_moonep_tail\n return torch.where(keep, x, x.new_zeros(()))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 144.00 MiB. GPU 0 has a total capacity of 178.35 GiB of which 144.81 MiB is free. Including non-PyTorch memory, this process has 178.20 GiB memory in use. Of the allocated memory 150.05 GiB is allocated by PyTorch, and 1.40 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] +[rank5]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 144.00 MiB. GPU 5 has a total capacity of 178.35 GiB of which 112.81 MiB is free. Including non-PyTorch memory, this process has 178.23 GiB memory in use. Of the allocated memory 150.05 GiB is allocated by PyTorch, and 1.43 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 144.00 MiB. GPU 0 has a total capacity of 178.35 GiB of which 144.81 MiB is free. Including non-PyTorch memory, this process has 178.20 GiB memory in use. Of the allocated memory 150.04 GiB is allocated by PyTorch, and 1.41 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 541, in dispatch_and_compute\n output = self.combine(output, metadata, shared_expert_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 333, in combine\n output = self.token_dispatcher.token_combine(output, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/token_dispatcher.py", line 1773, in token_combine\n return self._comm_manager.combine(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/token_dispatcher.py", line 1516, in combine\n hidden_states = zero_moonep_tail(hidden_states, metadata.cu_seqlens)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/fused_moonep.py", line 367, in zero_moonep_tail\n return torch.where(keep, x, x.new_zeros(()))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 144.00 MiB. GPU 0 has a total capacity of 178.35 GiB of which 144.81 MiB is free. Including non-PyTorch memory, this process has 178.20 GiB memory in use. Of the allocated memory 150.04 GiB is allocated by PyTorch, and 1.41 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] [rank0]: Traceback (most recent call last): [rank0]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 394, in [rank0]: pretrain( @@ -3274,16 +3274,16 @@ WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 144.00 MiB. GP [rank0]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/fused_moonep.py", line 367, in zero_moonep_tail [rank0]: return torch.where(keep, x, x.new_zeros(())) [rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank0]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 144.00 MiB. GPU 0 has a total capacity of 178.35 GiB of which 144.81 MiB is free. Including non-PyTorch memory, this process has 178.20 GiB memory in use. Of the allocated memory 150.05 GiB is allocated by PyTorch, and 1.40 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -W0826 15:08:01.492000 65221 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers -W0826 15:08:01.493000 65221 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 65317 closing signal SIGTERM -W0826 15:08:01.495000 65221 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 65318 closing signal SIGTERM -W0826 15:08:01.497000 65221 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 65319 closing signal SIGTERM -W0826 15:08:01.498000 65221 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 65320 closing signal SIGTERM -W0826 15:08:01.500000 65221 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 65321 closing signal SIGTERM -W0826 15:08:01.503000 65221 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 65322 closing signal SIGTERM -W0826 15:08:01.505000 65221 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 65323 closing signal SIGTERM -W0826 15:08:01.510000 65221 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 65324 closing signal SIGTERM +[rank0]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 144.00 MiB. GPU 0 has a total capacity of 178.35 GiB of which 144.81 MiB is free. Including non-PyTorch memory, this process has 178.20 GiB memory in use. Of the allocated memory 150.04 GiB is allocated by PyTorch, and 1.41 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +W0826 15:55:52.010000 155599 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 15:55:52.010000 155599 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 155695 closing signal SIGTERM +W0826 15:55:52.013000 155599 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 155696 closing signal SIGTERM +W0826 15:55:52.015000 155599 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 155697 closing signal SIGTERM +W0826 15:55:52.025000 155599 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 155698 closing signal SIGTERM +W0826 15:55:52.031000 155599 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 155699 closing signal SIGTERM +W0826 15:55:52.036000 155599 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 155700 closing signal SIGTERM +W0826 15:55:52.037000 155599 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 155701 closing signal SIGTERM +W0826 15:55:52.044000 155599 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 155702 closing signal SIGTERM Traceback (most recent call last): File "/usr/local/bin/torchrun", line 8, in sys.exit(main()) @@ -3311,4 +3311,4 @@ Traceback (most recent call last): time.sleep(monitor_interval) File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 65221 got signal: 15 +torch.distributed.elastic.multiprocessing.api.SignalException: Process 155599 got signal: 15 diff --git a/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv b/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv index 3fe28e1095cc74686820bb6a4f50d12979d6e4f3..5397dc70ac4ef04d476b07d01adb228ef9fdd515 100644 --- a/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv +++ b/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv @@ -1,7 +1,7 @@ model backend purpose capacity status peak_mib zero_drop drop_rate_pct required_capacity run_dir -QWEN3_235B moonep natural none OOM 182587 1 0 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1 -QWEN3_30B moonep natural none OOM 182625 1 0 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1 +QWEN3_235B moonep natural none OOM 182589 1 0 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1 +QWEN3_30B moonep natural none OOM 182599 1 0 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1 DSV2_LITE_L24 moonep natural none OK 157151 1 0 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1 DSV2_LITE_L24 moonep forced none OK 157151 1 0 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1 -QWEN2_57B moonep natural none OK 141553 1 0 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1 -QWEN2_57B moonep forced none OK 141551 1 0 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1 +QWEN2_57B moonep natural none OK 141555 1 0 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1 +QWEN2_57B moonep forced none OK 141555 1 0 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1 diff --git a/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv b/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv index 07167ae737f7c10e194aed8487daeda1d6ca351f..2e23d93e81f9f458909ee5a192ba1ed8d099618a 100644 --- a/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv +++ b/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv @@ -1,5 +1,5 @@ model backend status selected_capacity required_capacity drop_rate_pct peak_mib run_dir -QWEN3_235B moonep ERROR_OOM 182587 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1 -QWEN3_30B moonep ERROR_OOM 182625 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1 +QWEN3_235B moonep DOES_NOT_FIT 182589 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1 +QWEN3_30B moonep DOES_NOT_FIT 182599 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1 DSV2_LITE_L24 moonep SELECTED none 0 157151 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1 -QWEN2_57B moonep SELECTED none 0 141553 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1 +QWEN2_57B moonep SELECTED none 0 141555 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1 diff --git a/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/probe_summary.json b/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/probe_summary.json index c633a4c7d4c31815432d6e8baba4f39ff659c913..f782734cb3f03a10a84183a97b141b0911084eca 100644 --- a/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/probe_summary.json +++ b/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/probe_summary.json @@ -232,8 +232,8 @@ "branch": "moonep_backend_token_routing_analysis", "capacity": "none", "capacity_mode": "none", - "cold_expert_fraction_max": 0.796875, - "cold_expert_fraction_mean": 0.39234375, + "cold_expert_fraction_max": 0.8125, + "cold_expert_fraction_mean": 0.39734375, "cold_expert_fraction_min": 0.0, "cold_expert_fraction_n": 100, "commit": "35ae6e7f", @@ -245,8 +245,8 @@ "global_batch_size": "8", "global_snapshots": 100, "hottest_expert_token_fraction_max": 1.0, - "hottest_expert_token_fraction_mean": 0.9268069458007813, - "hottest_expert_token_fraction_min": 0.2415771484375, + "hottest_expert_token_fraction_mean": 0.9260787963867188, + "hottest_expert_token_fraction_min": 0.238616943359375, "hottest_expert_token_fraction_n": 100, "incomplete_snapshots": 0, "malformed_records": 0, @@ -254,77 +254,77 @@ "moonep_bytes_global_snapshots": 100, "moonep_bytes_incomplete_snapshots": 0, "moonep_bytes_malformed_records": 0, - "moonep_global_combine_payload_bytes_max": 3287445504.0, - "moonep_global_combine_payload_bytes_mean": 2789546475.52, - "moonep_global_combine_payload_bytes_min": 2180663296.0, + "moonep_global_combine_payload_bytes_max": 3287646208.0, + "moonep_global_combine_payload_bytes_mean": 2803923763.2, + "moonep_global_combine_payload_bytes_min": 2199099392.0, "moonep_global_combine_payload_bytes_n": 100, - "moonep_global_dispatch_payload_bytes_max": 3287445504.0, - "moonep_global_dispatch_payload_bytes_mean": 2789546475.52, - "moonep_global_dispatch_payload_bytes_min": 2180663296.0, + "moonep_global_dispatch_payload_bytes_max": 3287646208.0, + "moonep_global_dispatch_payload_bytes_mean": 2803923763.2, + "moonep_global_dispatch_payload_bytes_min": 2199099392.0, "moonep_global_dispatch_payload_bytes_n": 100, "moonep_global_prefetch_bytes_max": 990904320.0, - "moonep_global_prefetch_bytes_mean": 823551590.4, - "moonep_global_prefetch_bytes_min": 660602880.0, + "moonep_global_prefetch_bytes_mean": 822450585.6, + "moonep_global_prefetch_bytes_min": 550502400.0, "moonep_global_prefetch_bytes_n": 100, "moonep_global_reduce_bytes_max": 990904320.0, - "moonep_global_reduce_bytes_mean": 823551590.4, - "moonep_global_reduce_bytes_min": 660602880.0, + "moonep_global_reduce_bytes_mean": 822450585.6, + "moonep_global_reduce_bytes_min": 550502400.0, "moonep_global_reduce_bytes_n": 100, - "moonep_global_token_bytes_max": 6576725968.0, - "moonep_global_token_bytes_mean": 5580893210.72, - "moonep_global_token_bytes_min": 4363119160.0, + "moonep_global_token_bytes_max": 6577127376.0, + "moonep_global_token_bytes_mean": 5609650465.52, + "moonep_global_token_bytes_min": 4399994752.0, "moonep_global_token_bytes_n": 100, - "moonep_global_total_nvlink_bytes_max": 8328785856.0, - "moonep_global_total_nvlink_bytes_mean": 7227996391.52, - "moonep_global_total_nvlink_bytes_min": 5904525880.0, + "moonep_global_total_nvlink_bytes_max": 8322845104.0, + "moonep_global_total_nvlink_bytes_mean": 7254551636.72, + "moonep_global_total_nvlink_bytes_min": 5941401472.0, "moonep_global_total_nvlink_bytes_n": 100, "moonep_global_weight_movement_bytes_max": 1981808640.0, - "moonep_global_weight_movement_bytes_mean": 1647103180.8, - "moonep_global_weight_movement_bytes_min": 1321205760.0, + "moonep_global_weight_movement_bytes_mean": 1644901171.2, + "moonep_global_weight_movement_bytes_min": 1101004800.0, "moonep_global_weight_movement_bytes_n": 100, - "moonep_global_weight_to_token_ratio_max": 0.4380461607026548, - "moonep_global_weight_to_token_ratio_mean": 0.2995223715370738, - "moonep_global_weight_to_token_ratio_min": 0.20089110697762314, + "moonep_global_weight_to_token_ratio_max": 0.4405786409882186, + "moonep_global_weight_to_token_ratio_mean": 0.29776532604308203, + "moonep_global_weight_to_token_ratio_min": 0.17134294648250742, "moonep_global_weight_to_token_ratio_n": 100, "moonep_remote_demand_max": 2.0, - "moonep_remote_demand_mean": 0.935, + "moonep_remote_demand_mean": 0.93375, "moonep_remote_demand_min": 0.0, "moonep_remote_demand_n": 800, "moonep_slot_utilization_max": 0.25, - "moonep_slot_utilization_mean": 0.116875, + "moonep_slot_utilization_mean": 0.11671875, "moonep_slot_utilization_min": 0.0, "moonep_slot_utilization_n": 800, "moonep_unused_prefetch_slots_max": 16.0, - "moonep_unused_prefetch_slots_mean": 14.13, + "moonep_unused_prefetch_slots_mean": 14.1325, "moonep_unused_prefetch_slots_min": 12.0, "moonep_unused_prefetch_slots_n": 800, "num_experts": "64", "num_layers": "10", - "post_physical_rank_cv_max": 0.20167894084637375, - "post_physical_rank_cv_mean": 0.08583132480611003, - "post_physical_rank_cv_min": 0.00019943885935289461, + "post_physical_rank_cv_max": 0.18401159803181436, + "post_physical_rank_cv_mean": 0.07955254388706673, + "post_physical_rank_cv_min": 0.00013516465086261357, "post_physical_rank_cv_n": 100, - "post_physical_rank_imax_max": 1.2419115350424064, - "post_physical_rank_imax_mean": 1.1036399766472182, - "post_physical_rank_imax_min": 1.0002518334236372, + "post_physical_rank_imax_max": 1.2594853026197383, + "post_physical_rank_imax_mean": 1.0986239515089382, + "post_physical_rank_imax_min": 1.000156431828154, "post_physical_rank_imax_n": 100, - "pre_home_rank_cv_max": 1.4500421285803669, - "pre_home_rank_cv_mean": 0.7161558574760766, - "pre_home_rank_cv_min": 0.06359546419183554, + "pre_home_rank_cv_max": 1.2988435350844798, + "pre_home_rank_cv_mean": 0.6853757589365178, + "pre_home_rank_cv_min": 0.09817255863046992, "pre_home_rank_cv_n": 100, - "pre_home_rank_imax_max": 4.147735595703125, - "pre_home_rank_imax_mean": 2.322430419921875, - "pre_home_rank_imax_min": 1.11138916015625, + "pre_home_rank_imax_max": 3.904144287109375, + "pre_home_rank_imax_mean": 2.241612854003906, + "pre_home_rank_imax_min": 1.22235107421875, "pre_home_rank_imax_n": 100, - "raw_expert_cv_max": 2.6452554407770648, - "raw_expert_cv_mean": 2.184735306615502, - "raw_expert_cv_min": 0.4993102636020994, + "raw_expert_cv_max": 2.645313117775842, + "raw_expert_cv_mean": 2.1887209762080544, + "raw_expert_cv_min": 0.497937540133204, "raw_expert_cv_n": 100, "raw_expert_imax_ceiling": 8.0, "raw_expert_imax_hit_ceiling": 1, "raw_expert_imax_max": 8.0, - "raw_expert_imax_mean": 7.41445556640625, - "raw_expert_imax_min": 1.9326171875, + "raw_expert_imax_mean": 7.40863037109375, + "raw_expert_imax_min": 1.908935546875, "raw_expert_imax_n": 100, "records": { "moonep_bytes": 800, @@ -357,9 +357,9 @@ "force_balance": "1", "global_batch_size": "8", "global_snapshots": 100, - "hottest_expert_token_fraction_max": 0.1317138671875, - "hottest_expert_token_fraction_mean": 0.12946014404296874, - "hottest_expert_token_fraction_min": 0.128021240234375, + "hottest_expert_token_fraction_max": 0.13189697265625, + "hottest_expert_token_fraction_mean": 0.1293536376953125, + "hottest_expert_token_fraction_min": 0.1279296875, "hottest_expert_token_fraction_n": 100, "incomplete_snapshots": 0, "malformed_records": 0, @@ -367,77 +367,77 @@ "moonep_bytes_global_snapshots": 100, "moonep_bytes_incomplete_snapshots": 0, "moonep_bytes_malformed_records": 0, - "moonep_global_combine_payload_bytes_max": 2237204480.0, - "moonep_global_combine_payload_bytes_mean": 2231120138.24, - "moonep_global_combine_payload_bytes_min": 2224459776.0, + "moonep_global_combine_payload_bytes_max": 2236903424.0, + "moonep_global_combine_payload_bytes_mean": 2231590215.68, + "moonep_global_combine_payload_bytes_min": 2225391616.0, "moonep_global_combine_payload_bytes_n": 100, - "moonep_global_dispatch_payload_bytes_max": 2237204480.0, - "moonep_global_dispatch_payload_bytes_mean": 2231120138.24, - "moonep_global_dispatch_payload_bytes_min": 2224459776.0, + "moonep_global_dispatch_payload_bytes_max": 2236903424.0, + "moonep_global_dispatch_payload_bytes_mean": 2231590215.68, + "moonep_global_dispatch_payload_bytes_min": 2225391616.0, "moonep_global_dispatch_payload_bytes_n": 100, "moonep_global_prefetch_bytes_max": 770703360.0, - "moonep_global_prefetch_bytes_mean": 762996326.4, - "moonep_global_prefetch_bytes_min": 660602880.0, + "moonep_global_prefetch_bytes_mean": 761895321.6, + "moonep_global_prefetch_bytes_min": 550502400.0, "moonep_global_prefetch_bytes_n": 100, "moonep_global_reduce_bytes_max": 770703360.0, - "moonep_global_reduce_bytes_mean": 762996326.4, - "moonep_global_reduce_bytes_min": 660602880.0, + "moonep_global_reduce_bytes_mean": 761895321.6, + "moonep_global_reduce_bytes_min": 550502400.0, "moonep_global_reduce_bytes_n": 100, - "moonep_global_token_bytes_max": 4476243024.0, - "moonep_global_token_bytes_mean": 4464073623.92, - "moonep_global_token_bytes_min": 4450749096.0, + "moonep_global_token_bytes_max": 4475642264.0, + "moonep_global_token_bytes_mean": 4465014321.2, + "moonep_global_token_bytes_min": 4452613880.0, "moonep_global_token_bytes_n": 100, - "moonep_global_total_nvlink_bytes_max": 6017649744.0, - "moonep_global_total_nvlink_bytes_mean": 5990066276.72, - "moonep_global_total_nvlink_bytes_min": 5771954856.0, + "moonep_global_total_nvlink_bytes_max": 6017048984.0, + "moonep_global_total_nvlink_bytes_mean": 5988804964.4, + "moonep_global_total_nvlink_bytes_min": 5568474336.0, "moonep_global_total_nvlink_bytes_n": 100, "moonep_global_weight_movement_bytes_max": 1541406720.0, - "moonep_global_weight_movement_bytes_mean": 1525992652.8, - "moonep_global_weight_movement_bytes_min": 1321205760.0, + "moonep_global_weight_movement_bytes_mean": 1523790643.2, + "moonep_global_weight_movement_bytes_min": 1101004800.0, "moonep_global_weight_movement_bytes_n": 100, - "moonep_global_weight_to_token_ratio_max": 0.3460953541488461, - "moonep_global_weight_to_token_ratio_mean": 0.34183623349354486, - "moonep_global_weight_to_token_ratio_min": 0.2956613023580652, + "moonep_global_weight_to_token_ratio_max": 0.34618019023019353, + "moonep_global_weight_to_token_ratio_mean": 0.3412749588086127, + "moonep_global_weight_to_token_ratio_min": 0.2464493134486592, "moonep_global_weight_to_token_ratio_n": 100, "moonep_remote_demand_max": 1.0, - "moonep_remote_demand_mean": 0.86625, + "moonep_remote_demand_mean": 0.865, "moonep_remote_demand_min": 0.0, "moonep_remote_demand_n": 800, "moonep_slot_utilization_max": 0.125, - "moonep_slot_utilization_mean": 0.10828125, + "moonep_slot_utilization_mean": 0.108125, "moonep_slot_utilization_min": 0.0, "moonep_slot_utilization_n": 800, "moonep_unused_prefetch_slots_max": 16.0, - "moonep_unused_prefetch_slots_mean": 14.2675, + "moonep_unused_prefetch_slots_mean": 14.27, "moonep_unused_prefetch_slots_min": 14.0, "moonep_unused_prefetch_slots_n": 800, "num_experts": "64", "num_layers": "10", - "post_physical_rank_cv_max": 0.004173107469722948, - "post_physical_rank_cv_mean": 0.0021866389418670955, - "post_physical_rank_cv_min": 0.0008634926254374552, + "post_physical_rank_cv_max": 0.004151405986167901, + "post_physical_rank_cv_mean": 0.002179447588207395, + "post_physical_rank_cv_min": 0.0007203705850357968, "post_physical_rank_cv_n": 100, - "post_physical_rank_imax_max": 1.0063246661981728, - "post_physical_rank_imax_mean": 1.0033137485288246, - "post_physical_rank_imax_min": 1.001134997246789, + "post_physical_rank_imax_max": 1.0062412574645103, + "post_physical_rank_imax_mean": 1.0033121152297226, + "post_physical_rank_imax_min": 1.0013429305103698, "post_physical_rank_imax_n": 100, - "pre_home_rank_cv_max": 0.00995108219738984, - "pre_home_rank_cv_mean": 0.0057066499140920315, - "pre_home_rank_cv_min": 0.0010965114882337452, + "pre_home_rank_cv_max": 0.009250589744190636, + "pre_home_rank_cv_mean": 0.004711613912774553, + "pre_home_rank_cv_min": 0.002294812301292466, "pre_home_rank_cv_n": 100, - "pre_home_rank_imax_max": 1.014678955078125, - "pre_home_rank_imax_mean": 1.009002685546875, - "pre_home_rank_imax_min": 1.001373291015625, + "pre_home_rank_imax_max": 1.015838623046875, + "pre_home_rank_imax_mean": 1.00729736328125, + "pre_home_rank_imax_min": 1.002899169921875, "pre_home_rank_imax_n": 100, - "raw_expert_cv_max": 0.0179363414576349, - "raw_expert_cv_mean": 0.014900499877977456, - "raw_expert_cv_min": 0.011926166451983852, + "raw_expert_cv_max": 0.018080370892918116, + "raw_expert_cv_mean": 0.014541988425652842, + "raw_expert_cv_min": 0.011942398269672016, "raw_expert_cv_n": 100, "raw_expert_imax_ceiling": 8.0, "raw_expert_imax_hit_ceiling": 0, - "raw_expert_imax_max": 1.0537109375, - "raw_expert_imax_mean": 1.03568115234375, - "raw_expert_imax_min": 1.024169921875, + "raw_expert_imax_max": 1.05517578125, + "raw_expert_imax_mean": 1.0348291015625, + "raw_expert_imax_min": 1.0234375, "raw_expert_imax_n": 100, "records": { "moonep_bytes": 800, diff --git a/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/probe_summary.tsv b/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/probe_summary.tsv index 6f5c88f848853fd81a79720798669959462687c8..2517557bb6c3de3bd056a58e40ce239a461c733b 100644 --- a/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/probe_summary.tsv +++ b/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/probe_summary.tsv @@ -1,5 +1,5 @@ run_dir arm config backend num_experts topk experts_per_rank num_layers moe_layers dense_first_layers seq_length global_batch_size capacity_mode capacity force_balance cuda_graph branch commit router_balancing aux_loss_coeff shared_expert_overlap cold_expert_fraction_max cold_expert_fraction_mean cold_expert_fraction_min cold_expert_fraction_n global_snapshots hottest_expert_token_fraction_max hottest_expert_token_fraction_mean hottest_expert_token_fraction_min hottest_expert_token_fraction_n incomplete_snapshots malformed_records moonep_bytes_global_snapshots moonep_bytes_incomplete_snapshots moonep_bytes_malformed_records moonep_global_combine_payload_bytes_max moonep_global_combine_payload_bytes_mean moonep_global_combine_payload_bytes_min moonep_global_combine_payload_bytes_n moonep_global_dispatch_payload_bytes_max moonep_global_dispatch_payload_bytes_mean moonep_global_dispatch_payload_bytes_min moonep_global_dispatch_payload_bytes_n moonep_global_prefetch_bytes_max moonep_global_prefetch_bytes_mean moonep_global_prefetch_bytes_min moonep_global_prefetch_bytes_n moonep_global_reduce_bytes_max moonep_global_reduce_bytes_mean moonep_global_reduce_bytes_min moonep_global_reduce_bytes_n moonep_global_token_bytes_max moonep_global_token_bytes_mean moonep_global_token_bytes_min moonep_global_token_bytes_n moonep_global_total_nvlink_bytes_max moonep_global_total_nvlink_bytes_mean moonep_global_total_nvlink_bytes_min moonep_global_total_nvlink_bytes_n moonep_global_weight_movement_bytes_max moonep_global_weight_movement_bytes_mean moonep_global_weight_movement_bytes_min moonep_global_weight_movement_bytes_n moonep_global_weight_to_token_ratio_max moonep_global_weight_to_token_ratio_mean moonep_global_weight_to_token_ratio_min moonep_global_weight_to_token_ratio_n moonep_remote_demand_max moonep_remote_demand_mean moonep_remote_demand_min moonep_remote_demand_n moonep_slot_utilization_max moonep_slot_utilization_mean moonep_slot_utilization_min moonep_slot_utilization_n moonep_unused_prefetch_slots_max moonep_unused_prefetch_slots_mean moonep_unused_prefetch_slots_min moonep_unused_prefetch_slots_n post_physical_rank_cv_max post_physical_rank_cv_mean post_physical_rank_cv_min post_physical_rank_cv_n post_physical_rank_imax_max post_physical_rank_imax_mean post_physical_rank_imax_min post_physical_rank_imax_n pre_home_rank_cv_max pre_home_rank_cv_mean pre_home_rank_cv_min pre_home_rank_cv_n pre_home_rank_imax_max pre_home_rank_imax_mean pre_home_rank_imax_min pre_home_rank_imax_n raw_expert_cv_max raw_expert_cv_mean raw_expert_cv_min raw_expert_cv_n raw_expert_imax_ceiling raw_expert_imax_hit_ceiling raw_expert_imax_max raw_expert_imax_mean raw_expert_imax_min raw_expert_imax_n DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1 natural DSV2_LITE_L24 moonep 64 6 8 24 23 1 8192 8 none none 0 0 moonep_backend_token_routing_analysis 35ae6e7f seq_aux_loss 1e-3 0 0.578125 0.15625 0.0 230 230 0.999969482421875 0.7797577567722487 0.162750244140625 230 0 0 230 0 0 2592899072.0 2282338571.130435 2026799104.0 230 2592899072.0 2282338571.130435 2026799104.0 230 346030080.0 252601958.4 242221056.0 230 346030080.0 252601958.4 242221056.0 230 5188482536.0 4567339392.869565 4056064752.0 230 5672924648.0 5072543309.669565 4540506864.0 230 692060160.0 505203916.8 484442112.0 230 0.14851639788299403 0.11096021713514241 0.09336874676530664 230 3.0 0.9125 0.0 1840 0.375 0.1140625 0.0 1840 16.0 14.175 10.0 1840 0.24644338469771185 0.0866314272142518 0.016159622971429973 230 1.2001146488570558 1.1121655106296537 1.0188653892666146 230 1.3626457501059894 0.6592086328754161 0.06474057713157837 230 4.328043619791667 2.2370494897814766 1.1056925455729167 230 3.0868156557140782 1.9632045302354437 0.37852206422070017 230 10.666666666666666 1 10.666341145833334 8.317416072237318 1.7360026041666667 230 DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1 forced DSV2_LITE_L24 moonep 64 6 8 24 23 1 8192 8 none none 1 0 moonep_backend_token_routing_analysis 35ae6e7f seq_aux_loss 1e-3 0 0.0 0.0 0.0 230 230 0.0982818603515625 0.09643501613451087 0.09539794921875 230 0 0 230 0 0 2133762048.0 2129465023.4434783 2124537856.0 230 2133762048.0 2129465023.4434783 2124537856.0 230 242221056.0 238309411.61739132 207618048.0 230 242221056.0 238309411.61739132 207618048.0 230 4270279408.0 4261681018.2608695 4251824656.0 230 4754721520.0 4738299841.495652 4667434440.0 230 484442112.0 476618823.23478264 415236096.0 230 0.11393746242954192 0.11183841293995254 0.09728537714366937 230 1.0 0.8608695652173913 0.0 1840 0.125 0.10760869565217392 0.0 1840 16.0 14.278260869565218 14.0 1840 0.0031667036922078573 0.0016501228109707747 0.0006791931370339787 230 1.006201201251668 1.0026117364992109 1.0009196904719393 230 0.006673533732490181 0.003923942735502633 0.0014708009315538319 230 1.0122884114583333 1.0061054616734602 1.00213623046875 230 0.015123587213954862 0.012068912140234289 0.008171875787738593 230 10.666666666666666 0 1.04833984375 1.0286401721014493 1.017578125 230 -QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1 natural QWEN2_57B moonep 64 8 8 10 10 0 4096 8 none none 0 0 moonep_backend_token_routing_analysis 35ae6e7f aux_loss 1e-2 0 0.796875 0.39234375 0.0 100 100 1.0 0.9268069458007813 0.2415771484375 100 0 0 100 0 0 3287445504.0 2789546475.52 2180663296.0 100 3287445504.0 2789546475.52 2180663296.0 100 990904320.0 823551590.4 660602880.0 100 990904320.0 823551590.4 660602880.0 100 6576725968.0 5580893210.72 4363119160.0 100 8328785856.0 7227996391.52 5904525880.0 100 1981808640.0 1647103180.8 1321205760.0 100 0.4380461607026548 0.2995223715370738 0.20089110697762314 100 2.0 0.935 0.0 800 0.25 0.116875 0.0 800 16.0 14.13 12.0 800 0.20167894084637375 0.08583132480611003 0.00019943885935289461 100 1.2419115350424064 1.1036399766472182 1.0002518334236372 100 1.4500421285803669 0.7161558574760766 0.06359546419183554 100 4.147735595703125 2.322430419921875 1.11138916015625 100 2.6452554407770648 2.184735306615502 0.4993102636020994 100 8.0 1 8.0 7.41445556640625 1.9326171875 100 -QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1 forced QWEN2_57B moonep 64 8 8 10 10 0 4096 8 none none 1 0 moonep_backend_token_routing_analysis 35ae6e7f aux_loss 1e-2 0 0.0 0.0 0.0 100 100 0.1317138671875 0.12946014404296874 0.128021240234375 100 0 0 100 0 0 2237204480.0 2231120138.24 2224459776.0 100 2237204480.0 2231120138.24 2224459776.0 100 770703360.0 762996326.4 660602880.0 100 770703360.0 762996326.4 660602880.0 100 4476243024.0 4464073623.92 4450749096.0 100 6017649744.0 5990066276.72 5771954856.0 100 1541406720.0 1525992652.8 1321205760.0 100 0.3460953541488461 0.34183623349354486 0.2956613023580652 100 1.0 0.86625 0.0 800 0.125 0.10828125 0.0 800 16.0 14.2675 14.0 800 0.004173107469722948 0.0021866389418670955 0.0008634926254374552 100 1.0063246661981728 1.0033137485288246 1.001134997246789 100 0.00995108219738984 0.0057066499140920315 0.0010965114882337452 100 1.014678955078125 1.009002685546875 1.001373291015625 100 0.0179363414576349 0.014900499877977456 0.011926166451983852 100 8.0 0 1.0537109375 1.03568115234375 1.024169921875 100 +QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1 natural QWEN2_57B moonep 64 8 8 10 10 0 4096 8 none none 0 0 moonep_backend_token_routing_analysis 35ae6e7f aux_loss 1e-2 0 0.8125 0.39734375 0.0 100 100 1.0 0.9260787963867188 0.238616943359375 100 0 0 100 0 0 3287646208.0 2803923763.2 2199099392.0 100 3287646208.0 2803923763.2 2199099392.0 100 990904320.0 822450585.6 550502400.0 100 990904320.0 822450585.6 550502400.0 100 6577127376.0 5609650465.52 4399994752.0 100 8322845104.0 7254551636.72 5941401472.0 100 1981808640.0 1644901171.2 1101004800.0 100 0.4405786409882186 0.29776532604308203 0.17134294648250742 100 2.0 0.93375 0.0 800 0.25 0.11671875 0.0 800 16.0 14.1325 12.0 800 0.18401159803181436 0.07955254388706673 0.00013516465086261357 100 1.2594853026197383 1.0986239515089382 1.000156431828154 100 1.2988435350844798 0.6853757589365178 0.09817255863046992 100 3.904144287109375 2.241612854003906 1.22235107421875 100 2.645313117775842 2.1887209762080544 0.497937540133204 100 8.0 1 8.0 7.40863037109375 1.908935546875 100 +QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1 forced QWEN2_57B moonep 64 8 8 10 10 0 4096 8 none none 1 0 moonep_backend_token_routing_analysis 35ae6e7f aux_loss 1e-2 0 0.0 0.0 0.0 100 100 0.13189697265625 0.1293536376953125 0.1279296875 100 0 0 100 0 0 2236903424.0 2231590215.68 2225391616.0 100 2236903424.0 2231590215.68 2225391616.0 100 770703360.0 761895321.6 550502400.0 100 770703360.0 761895321.6 550502400.0 100 4475642264.0 4465014321.2 4452613880.0 100 6017048984.0 5988804964.4 5568474336.0 100 1541406720.0 1523790643.2 1101004800.0 100 0.34618019023019353 0.3412749588086127 0.2464493134486592 100 1.0 0.865 0.0 800 0.125 0.108125 0.0 800 16.0 14.27 14.0 800 0.004151405986167901 0.002179447588207395 0.0007203705850357968 100 1.0062412574645103 1.0033121152297226 1.0013429305103698 100 0.009250589744190636 0.004711613912774553 0.002294812301292466 100 1.015838623046875 1.00729736328125 1.002899169921875 100 0.018080370892918116 0.014541988425652842 0.011942398269672016 100 8.0 0 1.05517578125 1.0348291015625 1.0234375 100 diff --git a/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/results.tsv b/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/results.tsv index 60c8bdd182b99a5198dddbdd4abee51929731c0f..0aa69eb27538ec1cc7d1eacfff1a36e982d382ef 100644 --- a/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/results.tsv +++ b/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/results.tsv @@ -1,7 +1,7 @@ config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir -QWEN3_235B moonep matched 16 8192 8 none none 1 OOM NA NA 182587 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1 -QWEN3_30B moonep matched 16 4096 8 none none 1 OOM NA NA 182625 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1 -DSV2_LITE_L24 moonep matched 8 8192 8 none none 1 OK 848.31 791.80 157151 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1 -DSV2_LITE_L24 moonep matched 8 8192 8 none none+forcebal 1 OK 791.39 788.20 157151 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1 -QWEN2_57B moonep matched 8 4096 8 none none 1 OK 366.93 362.10 141553 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1 -QWEN2_57B moonep matched 8 4096 8 none none+forcebal 1 OK 407.76 358.70 141551 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1 +QWEN3_235B moonep matched 16 8192 8 none none 1 OOM NA NA 182589 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1 +QWEN3_30B moonep matched 16 4096 8 none none 1 OOM NA NA 182599 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1 +DSV2_LITE_L24 moonep matched 8 8192 8 none none 1 OK 793.84 791.50 157151 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1 +DSV2_LITE_L24 moonep matched 8 8192 8 none none+forcebal 1 OK 794.89 790.90 157151 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1 +QWEN2_57B moonep matched 8 4096 8 none none 1 OK 401.09 360.60 141555 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1 +QWEN2_57B moonep matched 8 4096 8 none none+forcebal 1 OK 406.04 364.50 141555 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1